From 4eb08dc15113cc4f73b65b239dfeb4f8ec0e23cf Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 08:50:34 +0000 Subject: [PATCH 001/147] fix(p184): Phase A DL closure - REST refresh quiet proof, binding V3 policy, quota log, SPOF runbook - IDR-040 fix 1: a Binance REST refresh proves a quiet book only when the stream already covers it (lastUpdateId <= last_sequence); REST_AHEAD_OF_STREAM otherwise - no publication, no re-stamp. - IDR-002: consumer route binding V3 seals the manifest request policy per product (additive; manifest digest and JWT revision unchanged). - IDR-029: quota exhaustion logged once per consumer and minute. - docs/runbooks/kn-single-points-of-failure.md: dependency map and limits. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 25 +++++++ docs/runbooks/kn-single-points-of-failure.md | 55 ++++++++++++++ qdl/consumer/universal_release.py | 74 ++++++++++++++++++- qdl/runtime/sandbox_extension.py | 9 +++ qdl/security/data_plane.py | 19 ++++- rust/qdl-core/src/l2_adapter.rs | 16 +++- rust/qdl-core/src/l2_book.rs | 4 + rust/qdl-realtime-core/src/lib.rs | 58 +++++++++++++++ .../phase115_render_consumer_route_binding.py | 14 +++- tests/test_fund_phase71_beta_runtime.py | 25 +++++++ tests/test_phase115_universal_release.py | 53 +++++++++++++ 11 files changed, 345 insertions(+), 7 deletions(-) create mode 100644 docs/runbooks/kn-single-points-of-failure.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index d92375d2..68664dde 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -68973,3 +68973,28 @@ cleanup delta. Evidence final-scoped-cleanup.json. Prior testclients/buildcontex already cleaned; active/rollback/preparedimages unchanged. This is artifact housekeeping still open, not a market-data correctness or release gate claim. Known unstaged owner whitespace atline13073 is preserved; stagedclosure diffclean. + +#### P18.4 Phase A Data Layer Closure Fixes (2026-10-07, Claude) + +Branch `fix/p184-dl-closure` from dev f012cac; source and local tests only - no image, role, binding mount, +Kafka, Redis or ledger change. Scope agreed with the owner: fix inside DL, keep every cross-system interface +versioned and additive (owner rule 2026-10-07). + +- IDR-040 fix 1: a Binance REST refresh is a quiet proof only when `lastUpdateId <= last_sequence` of the + stream-applied book; a snapshot ahead of the stream returns the new outcome `REST_AHEAD_OF_STREAM` (no + publication, no re-stamp, logged with the other frame refusals). Rust qdl-core + qdl-realtime-core 98/0, + including `binance_rest_refresh_ahead_of_stream_does_not_materialize`. Fix 2 (heartbeat quiet proof) needs a new + quality field and is held for the owner decision on tolerant SDK response readers (qdl_sdk models are + extra=forbid, so any new response field breaks deployed consumers). +- IDR-002: consumer route binding V3 = V2 + `request_policies` (requirement_id, event_recency_policy, + max_session_liveness_ms) rendered from the consumer manifest; products and the universal manifest digest are + unchanged (no manifest or JWT revision bump); V3 requires the manifest revision and covers every product once; + the sandbox extension carries policies. The TS reader (SDK 2.1.0) accepts V1/V2/V3 and is deployed first. +- IDR-029: quota exhaustion is logged once per consumer and minute (`qdl_consumer_quota_exhausted`, count/limit). +- SPOF: `docs/runbooks/kn-single-points-of-failure.md` - dependency map, effect, recovery and stated limits for + both ingestors, bar edge and stable_redis; no new service. +Tests: full unittest discover 2512 run - 7 errors were mount-only (`/app/logs` not writable, `noexec` /tmp) and +pass with correct mounts (17/17); universal release + sandbox 21/0; quota 14/0. The full Rust workspace needs the +builder's system libraries (openssl) and is compiled by the release image build. +Next: one Rust + one Python image from this commit (digests recorded) for the Phase C packet; closure table and +owner decisions D1-D3 handed over in Phase B. diff --git a/docs/runbooks/kn-single-points-of-failure.md b/docs/runbooks/kn-single-points-of-failure.md new file mode 100644 index 00000000..e50bd520 --- /dev/null +++ b/docs/runbooks/kn-single-points-of-failure.md @@ -0,0 +1,55 @@ +# KN single points of failure - dependency map, recovery, and stated limits + +Status: documentation of the current production topology (KN, release v2.2.2 certificate +`upgrade/evidence/releases/v2.2.2/certificate.json`). Written 2026-10-07 for the consolidated Data Layer +release scope (P18.4 Phase A). It does not change runtime and does not claim high availability. + +## What is single, and why a replica is not a fix + +| Role | Instances | Why a second instance is a design change, not a replica count | +|---|---|---| +| `ingestor_binance_usdm` | 1 | `lease_epoch` is a static fencing token fixed at 1, not leader election; the core refuses two bindings sharing a `source_id` (plan "What is actually single", B3) | +| `ingestor_okx_swap` | 1 | same as above | +| `binance_bar_edge` | 1 | owns BAR checkpoints; active/passive (B2) touches checkpoint ownership, the part that can corrupt rather than stop | +| `stable_redis` | 1 | deliberately ephemeral (`--appendonly no`); holds lease and quota keys in the stable runtime wiring (`qdl/runtime/stable.py` quota wiring). Its exact role set in KN mode is **not verified** here | + +All of them share one host and, for the producers, one Kafka cluster. Running a second copy on the same host +does not remove the host or Kafka as a common failure, so even B2/B3 would not make this independently highly +available. + +## Dependency map (data path) + +``` +venue WS/REST -> ingestor_{binance_usdm,okx_swap} -> Kafka raw -> rust_core x3 -> Kafka canonical + -> market_projector x2 -> market_cache -> query_kn x2 / stream_kn x2 -> consumers (TS, alpha) +venue REST -> binance_bar_edge -> BAR history (closed bars, > 1m intervals) +stable_redis -> leases and per-consumer request quota (query/stream admission) +``` + +## Failure effect and recovery + +| Role down | Consumer-visible effect | Recovery today | Measured | +|---|---|---|---| +| one ingestor | its venue's TRADE/QUOTE/BOOK/MARK stop advancing; consumers see quality STALE / not execution-eligible (fail closed), never fabricated data | container restart policy `unless-stopped`; the ingestor re-subscribes and the core resyncs books (gap -> quarantine -> snapshot) | not measured end to end (outage -> first execution-eligible view) | +| `binance_bar_edge` | intervals above 1m stop gaining closed bars; 1m bars are unaffected (since R1.28); longer intervals tolerate 540 s+ | restart policy; checkpoints resume | restart 18 s (plan, B2 note) | +| `stable_redis` | admission depends on it: quota consume fails closed with 503 `DEPENDENCY_UNAVAILABLE` (`qdl/security/data_plane.py` RedisMinuteQuota) | restart; quota counters restart empty (one-minute windows) | not measured | +| `market_cache` (KN) | reads fall back to recovery until rebuilt | rebuilt from Kafka state topics without offset reset or flush (plan, KN incident record) | not measured as a drill | + +## Commands (no new tooling) + +- Status: `docker ps --filter name=qdl_v2_stable_candidate` and the per-role healthchecks (B1). +- Restart one role: `docker start ` - the KN containers were created from private packets, not + this repository's compose files, so never `docker compose up` (CLAUDE.md section 1). +- Resource and liveness watch: `scripts/kn_production_watch.py`. + +## Missing (recorded, not built here) + +1. Recovery-time measurement per role (outage -> first execution-eligible consumer view), across both query + replicas - needed before any availability statement. +2. B3 Rust ingestor HA: a lease client and a fenced takeover in the core (the `lease_epoch` change). +3. B2 bar-edge active/passive, reusing `ActivePassiveGatewayLease` (`qdl/runtime/lease.py`). +4. B4 Redis redundancy: a decision about where lease/quota identity lives, not a replica count. +5. A KN boot runbook script; `scripts/v2_stable_boot_recovery.py` and + `scripts/rebuild_v2_stable_projection_cache.py` target the retired SQLite-spool topology. + +No new service is added for any of these (owner rule: extend existing services, reclaim wrong-direction ones). diff --git a/qdl/consumer/universal_release.py b/qdl/consumer/universal_release.py index 3ca23355..c5d62caf 100644 --- a/qdl/consumer/universal_release.py +++ b/qdl/consumer/universal_release.py @@ -36,6 +36,10 @@ _COVERAGE_SCHEMA = "qdl.v2.universal-release-coverage.v1" _CONSUMER_ROUTE_BINDING_SCHEMA_V1 = "qdl.v2.consumer-route-binding.v1" _CONSUMER_ROUTE_BINDING_SCHEMA_V2 = "qdl.v2.consumer-route-binding.v2" +# v3 = v2 + request_policies: the event-recency entitlement of every product, sealed so a consumer never +# configures it independently of the consumer manifest (IDR-002). Additive: v1/v2 stay readable. +_CONSUMER_ROUTE_BINDING_SCHEMA_V3 = "qdl.v2.consumer-route-binding.v3" +_EVENT_RECENCY_POLICIES = frozenset({"UNSPECIFIED", "BLOCK", "PAUSE", "OBSERVE"}) _V2_VENUES = frozenset({"BINANCE", "OKX"}) _REALTIME_FEEDS = frozenset({DemandFeed.TRADE, DemandFeed.QUOTE, DemandFeed.BAR}) _BOOK_FEEDS = frozenset({DemandFeed.BOOK_SNAPSHOT, DemandFeed.BOOK_DELTA}) @@ -829,6 +833,49 @@ def from_canonical_mapping( return manifest +@dataclass(frozen=True, slots=True) +class ConsumerRequestPolicy: + """The request entitlement a consumer must send for one sealed product (consumer manifest requirement). + + `event_recency_policy` None means the manifest leaves it unset (strict event recency); the Data Layer admits a + request only when both fields equal the manifest (qdl/consumer/manifest.py requirement_allowed). + """ + + requirement_id: str + event_recency_policy: str | None + max_session_liveness_ms: int | None + + def __post_init__(self) -> None: + object.__setattr__(self, "requirement_id", _sha256(self.requirement_id, "request policy requirement_id")) + if self.event_recency_policy is not None and self.event_recency_policy not in _EVENT_RECENCY_POLICIES: + raise ValueError("request policy event recency is invalid") + if self.max_session_liveness_ms is not None: + _positive_int(self.max_session_liveness_ms, "request policy max_session_liveness_ms") + + def canonical_mapping(self) -> dict[str, object]: + return { + "requirement_id": self.requirement_id, + "event_recency_policy": self.event_recency_policy, + "max_session_liveness_ms": self.max_session_liveness_ms, + } + + @classmethod + def from_canonical_mapping(cls, value: object) -> "ConsumerRequestPolicy": + if not isinstance(value, Mapping) or set(value) != { + "requirement_id", "event_recency_policy", "max_session_liveness_ms" + }: + raise ValueError("consumer route binding request policy fields are invalid") + recency = value["event_recency_policy"] + if recency is not None and not isinstance(recency, str): + raise ValueError("request policy event recency is invalid") + return cls( + requirement_id=str(value["requirement_id"]), + event_recency_policy=recency, + max_session_liveness_ms=_optional_positive_int( + value["max_session_liveness_ms"], "request policy max_session_liveness_ms"), + ) + + @dataclass(frozen=True, slots=True) class ConsumerRouteBinding: """One sealed universal-release slice for exactly one workload consumer.""" @@ -849,6 +896,8 @@ class ConsumerRouteBinding: # binding seals the server-side consumer-manifest revision as well, so a # JWT minted for another revision fails before any data-plane connection. consumer_manifest_revision: int | None = None + # v3: exactly one request policy per product (IDR-002); empty keeps the v1/v2 shape. + request_policies: tuple[ConsumerRequestPolicy, ...] = () def __post_init__(self) -> None: object.__setattr__(self, "consumer_id", _text(self.consumer_id, "binding consumer_id")) @@ -886,6 +935,14 @@ def __post_init__(self) -> None: sorted_products = tuple(sorted(self.products, key=lambda item: item.requirement_id)) if self.products != sorted_products: raise ValueError("consumer route binding products are not canonical") + if self.request_policies: + if self.consumer_manifest_revision is None: + raise ValueError("consumer route binding v3 needs the consumer manifest revision") + policies = tuple(sorted(self.request_policies, key=lambda item: item.requirement_id)) + if self.request_policies != policies: + raise ValueError("consumer route binding request policies are not canonical") + if [item.requirement_id for item in policies] != [item.requirement_id for item in self.products]: + raise ValueError("consumer route binding request policies must cover every product exactly once") expected = _digest(self._canonical_without_digest()) if self.binding_sha256 is not None and _sha256(self.binding_sha256, "binding_sha256") != expected: raise ValueError("consumer route binding checksum differs") @@ -894,7 +951,9 @@ def __post_init__(self) -> None: def _canonical_without_digest(self) -> dict[str, object]: canonical = { "schema": ( - _CONSUMER_ROUTE_BINDING_SCHEMA_V2 + _CONSUMER_ROUTE_BINDING_SCHEMA_V3 + if self.request_policies + else _CONSUMER_ROUTE_BINDING_SCHEMA_V2 if self.consumer_manifest_revision is not None else _CONSUMER_ROUTE_BINDING_SCHEMA_V1 ), @@ -915,6 +974,8 @@ def _canonical_without_digest(self) -> dict[str, object]: } if self.consumer_manifest_revision is not None: canonical["consumer_manifest_revision"] = self.consumer_manifest_revision + if self.request_policies: + canonical["request_policies"] = [item.canonical_mapping() for item in self.request_policies] return canonical def canonical_mapping(self) -> dict[str, object]: @@ -964,8 +1025,10 @@ def from_canonical_mapping(cls, value: Mapping[str, object]) -> "ConsumerRouteBi if schema == _CONSUMER_ROUTE_BINDING_SCHEMA_V1: expected = base_expected consumer_manifest_revision = None - elif schema == _CONSUMER_ROUTE_BINDING_SCHEMA_V2: + elif schema in {_CONSUMER_ROUTE_BINDING_SCHEMA_V2, _CONSUMER_ROUTE_BINDING_SCHEMA_V3}: expected = base_expected | {"consumer_manifest_revision"} + if schema == _CONSUMER_ROUTE_BINDING_SCHEMA_V3: + expected |= {"request_policies"} consumer_manifest_revision = _positive_int( value.get("consumer_manifest_revision"), "consumer route binding consumer manifest revision", @@ -1002,6 +1065,12 @@ def from_canonical_mapping(cls, value: Mapping[str, object]) -> "ConsumerRouteBi "execution_grade", "route", "fallback", "fallback_rule_id", "blocked_reason", "gap_policy", } + policies_raw = value.get("request_policies", []) + if not isinstance(policies_raw, list): + raise ValueError("consumer route binding request policies must be a list") + policies = tuple(ConsumerRequestPolicy.from_canonical_mapping(item) for item in policies_raw) + if schema == _CONSUMER_ROUTE_BINDING_SCHEMA_V3 and not policies: + raise ValueError("consumer route binding v3 needs request policies") products = [] for raw in products_raw: if not isinstance(raw, Mapping) or set(raw) != product_expected: @@ -1060,6 +1129,7 @@ def from_canonical_mapping(cls, value: Mapping[str, object]) -> "ConsumerRouteBi products=tuple(products), binding_sha256=str(value["binding_sha256"]), consumer_manifest_revision=consumer_manifest_revision, + request_policies=policies, ) diff --git a/qdl/runtime/sandbox_extension.py b/qdl/runtime/sandbox_extension.py index b2d96286..6c29ef54 100644 --- a/qdl/runtime/sandbox_extension.py +++ b/qdl/runtime/sandbox_extension.py @@ -123,6 +123,13 @@ def prepare_sandbox_extension(*, catalog, acquisition, manifest, binding, demand route["requirement_id"] = hashlib.sha256( f"{consumer.consumer_id}:{requirement_key(parsed)}".encode()).hexdigest() target_binding["products"].append(route) + if "request_policies" in target_binding: # v3: every sealed product carries its manifest entitlement + recency = parsed.event_recency_policy + target_binding["request_policies"].append({ + "requirement_id": route["requirement_id"], + "event_recency_policy": None if recency is None else str(getattr(recency, "value", recency)), + "max_session_liveness_ms": parsed.max_session_liveness_ms, + }) provenance = { "schema": "qdl.v2.sandbox-extension-preparation.v1", "status": "PREPARED_NOT_ACTIVATED_NOT_CERTIFIED", @@ -143,6 +150,8 @@ def prepare_sandbox_extension(*, catalog, acquisition, manifest, binding, demand universal_manifest_sha256=_digest(provenance), inventory_sha256=provenance["demand_sha256"]) target_binding["products"].sort(key=lambda r: r["requirement_id"]) + if "request_policies" in target_binding: + target_binding["request_policies"].sort(key=lambda r: r["requirement_id"]) target_binding.pop("binding_sha256") target_binding["binding_sha256"] = _digest(target_binding) ConsumerRouteBinding.from_canonical_mapping(target_binding) diff --git a/qdl/security/data_plane.py b/qdl/security/data_plane.py index b8636591..70ffa686 100644 --- a/qdl/security/data_plane.py +++ b/qdl/security/data_plane.py @@ -2,6 +2,7 @@ import json import hashlib +import logging import os import threading import time @@ -151,8 +152,15 @@ def consume(self, manifest: ConsumerManifest) -> None: """ +logger = logging.getLogger(__name__) + + class RedisMinuteQuota: - """Shared, atomic beta quota; Redis failure denies access rather than bypassing.""" + """Shared, atomic beta quota; Redis failure denies access rather than bypassing. + + Exhaustion is reported once per consumer and minute (IDR-029: the per-identity spend was only visible by + reading the Redis minute key), so the log is bounded by consumers x minutes, never by request rate. + """ def __init__(self, redis: Redis, *, prefix: str) -> None: normalized = prefix.strip(": ") @@ -162,6 +170,7 @@ def __init__(self, redis: Redis, *, prefix: str) -> None: ) self.redis = redis self.prefix = normalized + self._reported: dict[str, int] = {} @classmethod def from_url(cls, url: str, *, prefix: str) -> "RedisMinuteQuota": @@ -183,7 +192,7 @@ def _key(self, manifest: ConsumerManifest, minute: int) -> str: def consume(self, manifest: ConsumerManifest) -> None: minute = int(time.time() // 60) try: - allowed, _ = self.redis.eval( + allowed, count = self.redis.eval( _REDIS_MINUTE_QUOTA, 1, self._key(manifest, minute), @@ -197,6 +206,12 @@ def consume(self, manifest: ConsumerManifest) -> None: status_code=503, ) from error if int(allowed) != 1: + if self._reported.get(manifest.consumer_id) != minute: + self._reported[manifest.consumer_id] = minute + logger.warning(json.dumps({ + "event": "qdl_consumer_quota_exhausted", "consumer_id": manifest.consumer_id, + "minute": minute, "count": int(count), "limit": manifest.quotas.requests_per_minute, + }, sort_keys=True)) raise DataPlaneAccessError( "RATE_LIMITED", "consumer request quota is exhausted", diff --git a/rust/qdl-core/src/l2_adapter.rs b/rust/qdl-core/src/l2_adapter.rs index 9178fe2a..972618c6 100644 --- a/rust/qdl-core/src/l2_adapter.rs +++ b/rust/qdl-core/src/l2_adapter.rs @@ -240,10 +240,13 @@ impl L2BookAdapter { // window and can manufacture a false sequence gap. We still parse the // snapshot strictly so malformed provider data is never hidden. if generation == self.core.generation() && self.core.status() == BookStatus::Ready { + // Only a snapshot the stream-applied book already covers proves the book current at that time; + // a snapshot ahead of it would re-stamp a book that is behind the venue (IDR-040). + let covered = self.core.last_sequence().is_some_and(|last| sequence_end <= last); return Ok(self.transition( BookTransitionKind::Snapshot, BookPublication::None, - BookOutcome::Keepalive, + if covered { BookOutcome::Keepalive } else { BookOutcome::RestAheadOfStream }, TransitionMetadata::empty(observed_at_ms), )); } @@ -861,6 +864,17 @@ mod tests { assert_eq!(refresh.snapshot_sequence, Some(100)); assert_eq!(refresh.last_sequence, Some(101)); assert_eq!(refresh.view.unwrap().bids[0].quantity.canonical_text(), "2"); + // IDR-040: a REST snapshot ahead of the stream-applied book (venue at 105, book at 101) is no quiet proof. + let ahead = adapter + .apply_binance_rest_snapshot( + &json!({"lastUpdateId": 105, "E": 4, "bids": [["10", "7"]], "asks": [["11", "7"]]}), + 1, + 4, + ) + .unwrap(); + assert_eq!(ahead.outcome, BookOutcome::RestAheadOfStream); + assert_eq!((ahead.status, ahead.last_sequence), (BookStatus::Ready, Some(101))); + assert_eq!(ahead.view.unwrap().bids[0].quantity.canonical_text(), "2"); // the book is not replaced assert!(adapter .apply_binance_rest_snapshot(&json!({"bids": [], "asks": []}), 1, 4) .is_err()); diff --git a/rust/qdl-core/src/l2_book.rs b/rust/qdl-core/src/l2_book.rs index 1aff8cdb..551c9a49 100644 --- a/rust/qdl-core/src/l2_book.rs +++ b/rust/qdl-core/src/l2_book.rs @@ -422,6 +422,9 @@ pub enum BookOutcome { DeltaUnsupported, ResyncRequested, Disconnected, + /// A periodic REST snapshot is ahead of the stream-applied book: it is not a quiet proof of the current + /// book (IDR-040), so nothing is published or re-stamped; the stream deltas catch up on their own. + RestAheadOfStream, } impl BookOutcome { @@ -444,6 +447,7 @@ impl BookOutcome { Self::DeltaUnsupported => "DELTA_UNSUPPORTED", Self::ResyncRequested => "RESYNC_REQUESTED", Self::Disconnected => "DISCONNECTED", + Self::RestAheadOfStream => "REST_AHEAD_OF_STREAM", } } } diff --git a/rust/qdl-realtime-core/src/lib.rs b/rust/qdl-realtime-core/src/lib.rs index c0818924..f15f9369 100644 --- a/rust/qdl-realtime-core/src/lib.rs +++ b/rust/qdl-realtime-core/src/lib.rs @@ -1211,6 +1211,7 @@ impl RealtimeCore { BookOutcome::IgnoredStaleGeneration | BookOutcome::SnapshotSourceRejected | BookOutcome::IdentityMismatch + | BookOutcome::RestAheadOfStream ) { eprintln!( "{{\"event\":\"qdl_realtime_core_l2_frame_refused\",\"binding\":\"{}\",\ @@ -2608,6 +2609,63 @@ mod tests { assert_ne!(initial_snapshot.event_id, snapshot.event_id); } + #[test] + fn binance_rest_refresh_ahead_of_stream_does_not_materialize() { + // IDR-040: the venue is at 110 but the stream-applied book is at 102; a due REST refresh must not + // re-stamp the older book as current. + let binding = binance_book_binding(); + let mut core = core(binding.clone(), true); + let _ = core + .process( + raw( + &binding, + br#"{"s":"BTCUSDT","U":99,"u":101,"pu":98,"E":1001,"b":[["60000","2"]],"a":[["60001","1"]]}"#, + 1, + ), + 10, + ) + .unwrap(); + let _ = core + .process( + with_transport( + raw( + &binding, + br#"{"lastUpdateId":100,"E":1000,"bids":[["60000","1"]],"asks":[["60001","1"]]}"#, + 1, + ), + TransportProtocol::Http, + ), + 11, + ) + .unwrap(); + let _ = core + .process( + raw( + &binding, + br#"{"s":"BTCUSDT","U":102,"u":102,"pu":101,"E":2000,"b":[["60000","3"]],"a":[]}"#, + 1, + ), + 12, + ) + .unwrap(); + let ahead = core + .process( + with_transport( + raw( + &binding, + br#"{"lastUpdateId":110,"E":31001,"bids":[["1","1"]],"asks":[["2","1"]]}"#, + 1, + ), + TransportProtocol::Http, + ), + 13, + ) + .unwrap(); + assert!(ahead.canonical.is_empty(), "a REST snapshot ahead of the stream is no quiet proof"); + assert!(ahead.quarantines.is_empty()); + assert_eq!(ahead.filtered_outcome, Some("REST_AHEAD_OF_STREAM")); + } + #[test] fn binance_l2_gap_is_quarantined_and_requires_resync() { let binding = binance_book_binding(); diff --git a/scripts/phase115_render_consumer_route_binding.py b/scripts/phase115_render_consumer_route_binding.py index f1fe77d4..4189c219 100644 --- a/scripts/phase115_render_consumer_route_binding.py +++ b/scripts/phase115_render_consumer_route_binding.py @@ -18,6 +18,7 @@ import yaml from qdl.consumer.universal_release import ( + ConsumerRequestPolicy, ConsumerRouteBinding, UniversalReleaseManifest, UniversalReleaseProduct, @@ -38,7 +39,7 @@ def binding_from_stable_release(path: Path, root: Path, consumer_id: str) -> Con policy = UniversalReleasePolicy.load(root / "config/v2/universal-release-policy.yaml", manifest_root=root) catalog = StableSourceCatalog.load(plan.source_catalog.path) routes = {r.requirement_key: r for r in consumer.products} - products = [] + products, policies = [], [] consumer_class = UniversalConsumerClass.TRADING_SYSTEM if not consumer_id.startswith("trading-system."): raise ValueError("stable renderer currently supports the Trading System consumer only") @@ -61,9 +62,16 @@ def binding_from_stable_release(path: Path, root: Path, consumer_id: str) -> Con feed, requirement.interval)), None) if rule is None: raise ValueError("stable V1 route has no matching compatibility rule") + requirement_id = hashlib.sha256(f"{consumer_id}:{key}".encode()).hexdigest() + recency = requirement.event_recency_policy + policies.append(ConsumerRequestPolicy( + requirement_id=requirement_id, + event_recency_policy=None if recency is None else str(getattr(recency, "value", recency)), + max_session_liveness_ms=requirement.max_session_liveness_ms, + )) products.append(UniversalReleaseProduct( consumer_id=consumer_id, consumer_class=consumer_class, - requirement_id=hashlib.sha256(f"{consumer_id}:{key}".encode()).hexdigest(), + requirement_id=requirement_id, instrument_uid=identity.instrument_uid, instrument_id=identity.instrument_id, venue=identity.venue, market=identity.market, product_type=identity.product_type.value, native_symbol=instrument.native_symbol, @@ -87,6 +95,8 @@ def binding_from_stable_release(path: Path, root: Path, consumer_id: str) -> Con independent_v1_venues=("DNSE",), products=tuple(sorted(products, key=lambda p: p.requirement_id)), consumer_manifest_revision=consumer.manifest.manifest_revision, + # v3 (IDR-002): the manifest's request entitlement is sealed with the routes. + request_policies=tuple(sorted(policies, key=lambda p: p.requirement_id)), ) diff --git a/tests/test_fund_phase71_beta_runtime.py b/tests/test_fund_phase71_beta_runtime.py index 09cb1433..169c1f59 100644 --- a/tests/test_fund_phase71_beta_runtime.py +++ b/tests/test_fund_phase71_beta_runtime.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json import tempfile import threading import time @@ -463,6 +464,30 @@ def close(self): second.consume(manifest) self.assertEqual(captured.exception.status_code, 503) + def test_quota_exhaustion_is_reported_once_per_consumer_and_minute(self): + """IDR-029: the per-identity spend becomes visible without a log line per request.""" + payload = manifest_mapping( + consumer_id="phase71-quota-log", + subject="spiffe://qdl/paper/phase71-quota-log", + instrument_uid="00000000-0000-0000-0000-000000000001", + ) + payload["spec"]["quotas"]["requests_per_minute"] = 1 + manifest = ConsumerManifestLoader.from_mapping(payload) + + class Exhausted: + def eval(self, script, key_count, key, limit, ttl): + return [0, 5] + + quota = RedisMinuteQuota(Exhausted(), prefix="qdl:beta:v2:paper:phase71") + with self.assertLogs("qdl.security.data_plane", level="WARNING") as logs: + for _ in range(4): + with self.assertRaises(DataPlaneAccessError): + quota.consume(manifest) + self.assertEqual(len(logs.records), 1) + event = json.loads(logs.records[0].getMessage()) + self.assertEqual((event["event"], event["consumer_id"], event["count"], event["limit"]), + ("qdl_consumer_quota_exhausted", "phase71-quota-log", 5, 1)) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_phase115_universal_release.py b/tests/test_phase115_universal_release.py index 20268bc0..b90becc7 100644 --- a/tests/test_phase115_universal_release.py +++ b/tests/test_phase115_universal_release.py @@ -359,6 +359,59 @@ def test_generation_bound_consumer_route_binding_round_trips_and_keeps_v1_compat self.assertEqual(legacy.canonical_mapping()["schema"], "qdl.v2.consumer-route-binding.v1") self.assertIsNone(legacy.consumer_manifest_revision) + def test_v3_binding_seals_request_policy_and_keeps_v1_v2_bytes(self): + """IDR-002: v3 carries the manifest request entitlement per product; v1/v2 are unchanged.""" + from dataclasses import replace + + from qdl.consumer.universal_release import ConsumerRequestPolicy + + manifest, *_ = _manifest() + consumer_id = manifest.products[0].consumer_id + v2 = ConsumerRouteBinding.from_manifest(manifest, consumer_id=consumer_id, consumer_manifest_revision=7) + policies = tuple( + ConsumerRequestPolicy(item.requirement_id, "OBSERVE" if index % 2 else None, 45000 if index % 2 else None) + for index, item in enumerate(v2.products) + ) + v3 = replace(v2, binding_sha256=None, request_policies=policies) + mapping = v3.canonical_mapping() + self.assertEqual(mapping["schema"], "qdl.v2.consumer-route-binding.v3") + self.assertEqual(len(mapping["request_policies"]), len(v2.products)) + self.assertNotEqual(v3.binding_sha256, v2.binding_sha256) + self.assertEqual(v3.universal_manifest_sha256, v2.universal_manifest_sha256) # manifest untouched + parsed = ConsumerRouteBinding.from_canonical_mapping(mapping) + self.assertEqual(parsed.request_policies, policies) + self.assertEqual(parsed.canonical_mapping(), mapping) + # v1/v2 bytes do not change with v3 support. + self.assertNotIn("request_policies", v2.canonical_mapping()) + self.assertEqual(ConsumerRouteBinding.from_canonical_mapping(v2.canonical_mapping()).request_policies, ()) + + def test_v3_binding_rejects_partial_unrevisioned_or_invalid_policies(self): + from dataclasses import replace + + from qdl.consumer.universal_release import ConsumerRequestPolicy + + manifest, *_ = _manifest() + consumer_id = manifest.products[0].consumer_id + v2 = ConsumerRouteBinding.from_manifest(manifest, consumer_id=consumer_id, consumer_manifest_revision=7) + full = tuple(ConsumerRequestPolicy(item.requirement_id, "BLOCK", None) for item in v2.products) + if len(full) > 1: + with self.assertRaisesRegex(ValueError, "cover every product"): + replace(v2, binding_sha256=None, request_policies=full[1:]) + with self.assertRaisesRegex(ValueError, "manifest revision"): + replace(v2, binding_sha256=None, consumer_manifest_revision=None, request_policies=full) + with self.assertRaisesRegex(ValueError, "event recency"): + ConsumerRequestPolicy(v2.products[0].requirement_id, "LOOSE", None) + with self.assertRaises(ValueError): + ConsumerRequestPolicy(v2.products[0].requirement_id, "OBSERVE", 0) + mapping = replace(v2, binding_sha256=None, request_policies=full).canonical_mapping() + mapping["request_policies"] = [] + with self.assertRaises(ValueError): + ConsumerRouteBinding.from_canonical_mapping(mapping) + tampered = replace(v2, binding_sha256=None, request_policies=full).canonical_mapping() + tampered["request_policies"][0]["event_recency_policy"] = "OBSERVE" + with self.assertRaisesRegex(ValueError, "checksum"): + ConsumerRouteBinding.from_canonical_mapping(tampered) + def test_consumer_route_binding_rejects_unknown_tampered_and_duplicate_routes(self): manifest, *_ = _manifest() with self.assertRaisesRegex(ValueError, "no admitted product"): From d64bbca80fb44152ba7f4479a7a4a18f8e9e7367 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 10:29:29 +0000 Subject: [PATCH 002/147] docs(p184): Phase B handover - DL closure table and owner decisions D1-D3 TS Phase A closed on candidate r59b (93/93, strict verifier PASS); D2 tolerant SDK response readers in this release, D1 heartbeat deferred, D3 HA separate. Lists remaining Codex work and close conditions per item. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 39 +++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 68664dde..3c007fc1 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -68998,3 +68998,42 @@ pass with correct mounts (17/17); universal release + sandbox 21/0; quota 14/0. builder's system libraries (openssl) and is compiled by the release image build. Next: one Rust + one Python image from this commit (digests recorded) for the Phase C packet; closure table and owner decisions D1-D3 handed over in Phase B. + +#### P18.4 Phase B Handover - Data Layer closure for the consolidated release (2026-10-07, Claude -> Codex) + +Status: HANDOVER. Trading System Phase A is closed on candidates: TS r59b (image +sha256:9150385e2bb3596793b6ee209dbc99696c3ed07439d6f4199356f417a97b6105, source 054cfe3 on +feat/v2-rust-first-okx-demo) passed surface 93/93 with the strict verifier, and the TS Data Layer consumer +(writer market_data_service_p184w) runs that image since 2026-10-07 10:20 UTC (owner packet; TS plan journal +"P18.4 Phase A - candidate r59"). Nothing in this section has run on a Data Layer role. + +Owner decisions 2026-10-07: **D2 in this release** (tolerant SDK response readers), **D1 deferred** (a quiet Binance +book stays stale, never falsely fresh), **D3 separate project** (ingestor/bar-edge/Redis HA; limits recorded in +the SPOF runbook). Interface rule (owner): versioned, additive-only interfaces, readers accept every supported +version; fix internals, not endpoints. + +Candidate images from 4eb08dc (built, not run on any role): qdl-v2-rust:2.2.3-rc1-4eb08dc +sha256:3566a6e3e6c7f7d6b82e7beee9075a02e42f1c0ebb292d3802a1bd8423e19520, qdl-v2-python:2.2.3-rc1-4eb08dc +sha256:acf5fd06b6a3f8c7d1540180cf8f429bfef89af0d479fc432b205993cf353e96. They are superseded once D2 lands +(rebuild both from the final commit; delete these by digest). + +| Item | State on this branch | Remaining work (Codex) | Close condition | +|---|---|---|---| +| IDR-040 fix 1 - REST refresh ahead of stream | fixed 4eb08dc, Rust 98/0 | none in source | candidate rust_core: no REST_AHEAD materialization; refusal log visible | +| IDR-040 fix 2 - heartbeat quiet proof | not built (D1 deferred) | none this release | recorded limit: quiet Binance book = stale | +| D2 - tolerant SDK response readers | not started. `qdl_sdk/models.py:119-120` ClosedModel is `extra="forbid"` for every model; SDK 2.0.7 | response models ignore unknown fields; requests, signed payloads and binding/manifest documents stay closed; tests both ways; SDK minor bump (2.1.0) with changelog | old-server/new-SDK and new-server/old-field round trips green; TS can upgrade the SDK before any new response field ships | +| IDR-002 - request policy sealed | binding V3 fixed 4eb08dc | render V3 for trading-system.paper.stable at release | TS reader 2.1.0 deployed first (done: r59 writer), then mount V3; rollback = V2 file | +| IDR-029 - quota visibility | log added 4eb08dc | none in source | TS measured (minute quota counter, read-only): 1204 -> 975 requests/min after TS batching; no quota or manifest change | +| BAR 15m/30m gaps | KN cache keeps by open time (`rust/qdl-projector/src/expiry.rs`) | read-only gap diagnostic on the KN cache for the affected bindings | 0 gaps reported, or a defect row | +| IDR-039 - mixed revisions | 5 revisions across 13 roles (v2.2.2 certificate) | one commit -> one Rust + one Python image | Phase C packet recreates the 13 non-infrastructure roles from two digests, certified once | +| SPOF | runbook `docs/runbooks/kn-single-points-of-failure.md` | recovery-time per role (optional this release) | measured or stated as not measured in the certificate | +| Consumer gate | tooling hardcodes `query_v2_1/2` | parameterise replica names for `query_kn_1/2`; measure event -> TS apply, usable/refusal by reason | numbers recorded per the four latency quantities rule | + +Not Data Layer (stays TS): IDR-062 (TS package rule requires book top == quote exactly; mitigated by the TS writer +change, fixed in TS), POV volume source, reservations, writer CPU. + +Order: Codex implements D2 and the diagnostics on this branch (same worktree +`/home/bobby/.worktrees/data-layer-p184-closure`), local CI, one commit per tested slice with this journal; then +rebuild both images from that commit and write the Phase C packet (roles, digests, rollback, what is not touched). +Phase C runtime changes, push and merge need the owner's approval per packet. CLAUDE.md section 4 couplings apply +(manifest revision, image-baked catalog, stable_redis ephemeral, no compose up on KN containers). From ca053804dddd163d962bd7e37f5f0eb6d7f96f02 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 10:42:09 +0000 Subject: [PATCH 003/147] docs(p184): add IDR-062 (BOOK top lags QUOTE, timer-only L2 materialization) to the DL closure TS measured 0 overlapping / 18-21% crossed book-vs-quote reads with book age p50 ~730 ms; the core materializes BOOK_SNAPSHOT on a 1000 ms timer only. Fix belongs in the DL (top-change trigger with a floor), measured for resources before release. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 3c007fc1..4bee8fe6 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69027,12 +69027,12 @@ sha256:acf5fd06b6a3f8c7d1540180cf8f429bfef89af0d479fc432b205993cf353e96. They ar | BAR 15m/30m gaps | KN cache keeps by open time (`rust/qdl-projector/src/expiry.rs`) | read-only gap diagnostic on the KN cache for the affected bindings | 0 gaps reported, or a defect row | | IDR-039 - mixed revisions | 5 revisions across 13 roles (v2.2.2 certificate) | one commit -> one Rust + one Python image | Phase C packet recreates the 13 non-infrastructure roles from two digests, certified once | | SPOF | runbook `docs/runbooks/kn-single-points-of-failure.md` | recovery-time per role (optional this release) | measured or stated as not measured in the certificate | +| IDR-062 - BOOK top lags QUOTE | confirmed by code: the core materializes BOOK_SNAPSHOT only on a timer (`rust/qdl-realtime-core/src/lib.rs:1392-1399`, interval `qdl/runtime/execution_l2.py:23` = 1000 ms). TS measured on the TS cache, 120 samples/venue: book age p50 ~730 ms p95 ~1.2 s vs QUOTE p50 ~200 ms; 18-21% of reads the quote crosses the book, so TS package admission (exact book top == quote, a kept safety rule) refuses | also materialize when best bid/ask changes, bounded by a minimum interval (keep the 1000 ms timer as the quiet floor); same feed and contract; tests for top-change, floor and quiet cases | candidate measurement, one variable at a time: BOOK_SNAPSHOT msg/s, Kafka, rust_core/query/stream CPU (DL budget <= 5 vcore) and the TS writer CPU; TS coherence probe >= 39/40 sustained and book age p95 well under the 2000 ms package cap | | Consumer gate | tooling hardcodes `query_v2_1/2` | parameterise replica names for `query_kn_1/2`; measure event -> TS apply, usable/refusal by reason | numbers recorded per the four latency quantities rule | -Not Data Layer (stays TS): IDR-062 (TS package rule requires book top == quote exactly; mitigated by the TS writer -change, fixed in TS), POV volume source, reservations, writer CPU. +Not Data Layer (stays TS): POV volume source, reservations, writer CPU. -Order: Codex implements D2 and the diagnostics on this branch (same worktree +Order: Codex implements D2, the IDR-062 top-change materialization and the diagnostics on this branch (same worktree `/home/bobby/.worktrees/data-layer-p184-closure`), local CI, one commit per tested slice with this journal; then rebuild both images from that commit and write the Phase C packet (roles, digests, rollback, what is not touched). Phase C runtime changes, push and merge need the owner's approval per packet. CLAUDE.md section 4 couplings apply From fa0cfa176353a4b7fe259a06030bc19151820365 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 10:53:44 +0000 Subject: [PATCH 004/147] docs(p184): Phase C waits for the TS DR matrix and broker delta (owner 2026-10-07) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 4bee8fe6..e254f399 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69035,5 +69035,5 @@ Not Data Layer (stays TS): POV volume source, reservations, writer CPU. Order: Codex implements D2, the IDR-062 top-change materialization and the diagnostics on this branch (same worktree `/home/bobby/.worktrees/data-layer-p184-closure`), local CI, one commit per tested slice with this journal; then rebuild both images from that commit and write the Phase C packet (roles, digests, rollback, what is not touched). -Phase C runtime changes, push and merge need the owner's approval per packet. CLAUDE.md section 4 couplings apply +Phase C runtime changes, push and merge need the owner's approval per packet. Owner 2026-10-07: build and test now, but the Phase C release WAITS for the TS DR01-DR08 matrix and the broker delta (TS plan journal "P18.4 gap audit"); Data Layer defects they find join this closure first, so the release is cut once. CLAUDE.md section 4 couplings apply (manifest revision, image-baked catalog, stable_redis ephemeral, no compose up on KN containers). From eb7c649898749bc1613b6a37e580a91dba58121f Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 11:39:38 +0000 Subject: [PATCH 005/147] feat(sdk): tolerate additive response fields in version 2.1.0 --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 35 ++++++++++ qdl_sdk/CHANGELOG.md | 10 +++ qdl_sdk/__init__.py | 2 +- qdl_sdk/models.py | 81 +++++++++++++---------- qdl_sdk/reference.py | 15 +++-- scripts/build_qdl_sdk_release.py | 2 +- tests/test_phaseb_stable_release.py | 2 +- tests/test_qdl_sdk_additive_responses.py | 70 ++++++++++++++++++++ tests/test_qdl_sdk_release.py | 2 +- 9 files changed, 172 insertions(+), 47 deletions(-) create mode 100644 qdl_sdk/CHANGELOG.md create mode 100644 tests/test_qdl_sdk_additive_responses.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index e254f399..36e8d3e6 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69037,3 +69037,38 @@ Order: Codex implements D2, the IDR-062 top-change materialization and the diagn rebuild both images from that commit and write the Phase C packet (roles, digests, rollback, what is not touched). Phase C runtime changes, push and merge need the owner's approval per packet. Owner 2026-10-07: build and test now, but the Phase C release WAITS for the TS DR01-DR08 matrix and the broker delta (TS plan journal "P18.4 gap audit"); Data Layer defects they find join this closure first, so the release is cut once. CLAUDE.md section 4 couplings apply (manifest revision, image-baked catalog, stable_redis ephemeral, no compose up on KN containers). + + +#### P18.4 Phase B implementation start (Codex, 2026-10-07) + +Status: IN_PROGRESS, source/local candidate only. Approved scope is the Phase B +handover above at fa0cfa1: D2 tolerant responses with strict requests/documents, +IDR-062 bounded top-change snapshots, read-only BAR 15m/30m coverage, configurable +probe replicas. D1/D3 remain deferred. No TS file or canonical checkout edit. +Tests target changed contracts and continuity first; inherited certificates are +not rerun. Two final images must share one source SHA. Phase C packet is a draft, +blocked on TS DR01-DR08/broker delta and explicit runtime approval. No shared data +write, role restart, binding mount, push or merge. Rollback retains all active +digests/config and V2 binding; only the two named rc1 candidates may be removed +after replacement and reference checks. Record exact tests, resource limits and +any unmeasured runtime gates per tested slice below. apply_patch failed with the +known mountinfo helper error; exact scripted edits under workspace rule 32. + + +##### Phase B slice D2 - tolerant SDK response readers + +Status: IMPLEMENTED / TESTED_LOCALLY, SDK 2.1.0; no server response field added. +ResponseModel ignores additive unknown fields recursively through typed market, +reference and problem responses. Known enums/decimals/domain validators remain +strict. Warmup/reference requests and signed/binding/manifest documents unchanged. +The shared response classes keep the published OpenAPI schema unchanged via +json_schema_extra; initial regression found OpenAPI drift and the corrected run +passed. Changelog: qdl_sdk/CHANGELOG.md. Version builder and two current-version +tests updated, historical certificates untouched. +Tests in existing acf5fd06 image, source mounted read-only, network none, 2 CPUs, +1 GiB, tmpfs /tmp (exec) and /app/logs: SDK additive/release/projection/stable release +28 PASS; stream SDK/reference V2/universal release/sandbox/quality/pre5/status/offset +91 PASS, zero fail/skip in both final runs. First container failed before tests +because /app/logs mount nested inside a read-only checkout; fixed mount layout +(/src checkout, /app/logs tmpfs), not permissions on source. Containers --rm. +No new image, shared mutation, push/merge or cleanup of retained artifacts. diff --git a/qdl_sdk/CHANGELOG.md b/qdl_sdk/CHANGELOG.md new file mode 100644 index 00000000..e4ae7562 --- /dev/null +++ b/qdl_sdk/CHANGELOG.md @@ -0,0 +1,10 @@ +# SDK Changelog + +## 2.1.0 + +Response models, including nested market/reference payloads and error diagnostics, +ignore unknown additive fields. Known fields, enum values and domain validators +remain enforced. Request models, signed payloads, manifests and sealed binding +documents remain strict. Existing server responses remain compatible. No new +server response field is introduced by this release. Deploy tolerant readers +before introducing additive response fields; older 2.0.x readers remain strict. diff --git a/qdl_sdk/__init__.py b/qdl_sdk/__init__.py index aa66c2ca..4419207f 100644 --- a/qdl_sdk/__init__.py +++ b/qdl_sdk/__init__.py @@ -1,6 +1,6 @@ """Stable Python SDK V2 with typed public models and generated transports.""" -__version__ = "2.0.7" +__version__ = "2.1.0" from qdl_sdk.client import AsyncDataLayerClient, DataLayerClientV2, WarmupStreamSession from qdl_sdk.credentials import ( diff --git a/qdl_sdk/models.py b/qdl_sdk/models.py index 17b8c5aa..8e2a73d0 100644 --- a/qdl_sdk/models.py +++ b/qdl_sdk/models.py @@ -120,6 +120,15 @@ class ClosedModel(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) +class ResponseModel(BaseModel): + """Accept additive response fields without relaxing known-field validation.""" + + # These types also describe the current server output in OpenAPI. Keep that + # published shape closed; tolerance is a reader behavior, not a new field. + model_config = ConfigDict(extra="ignore", populate_by_name=True, + json_schema_extra={"additionalProperties": False}) + + class WarmupTimeRange(ClosedModel): start_time_ns: int = Field(gt=0) end_time_ns: int = Field(gt=0) @@ -147,7 +156,7 @@ def exactly_one_horizon(self): return self -class ProblemDiagnostics(ClosedModel): +class ProblemDiagnostics(ResponseModel): """Quality as Query evaluated it for the refused request (KN-4 review): the same non-secret metadata a served item carries, taken at the refusal, so a failure is never diagnosed from a status read seconds later.""" @@ -168,7 +177,7 @@ class ProblemDiagnostics(ClosedModel): received_at_ns: int | None = Field(default=None, ge=0) -class ProblemDetails(ClosedModel): +class ProblemDetails(ResponseModel): type: str title: str status: int @@ -190,7 +199,7 @@ def serialize_problem(self, handler): return result -class DecimalValue(ClosedModel): +class DecimalValue(ResponseModel): coefficient: str = Field(pattern=r"^-?(0|[1-9][0-9]*)$") scale: int = Field(ge=-38, le=38) source_text: str = Field(min_length=1, max_length=128) @@ -247,7 +256,7 @@ class BasisKind(StrEnum): DERIVED = "DERIVED" -class SourceView(ClosedModel): +class SourceView(ResponseModel): venue: str = Field(min_length=1, max_length=40) provider: str = Field(min_length=1, max_length=80) source_id: str = Field(min_length=1, max_length=120) @@ -255,7 +264,7 @@ class SourceView(ClosedModel): authoritative: bool -class QualityView(ClosedModel): +class QualityView(ResponseModel): state: Literal[ "DISABLED", "STARTING", "CONNECTING", "SUBSCRIBING", "SYNCING", "LIVE", "DEGRADED", "GAPPED", "RESYNCING", "STALE", "OFFLINE", @@ -277,7 +286,7 @@ class QualityView(ClosedModel): flags: list[str] -class ContractView(ClosedModel): +class ContractView(ResponseModel): schema_digest: str = Field(pattern=r"^[0-9a-f]{64}$") contract_version: str = Field(min_length=1, max_length=40) normalizer_version: str = Field(min_length=1, max_length=80) @@ -289,7 +298,7 @@ class ContractView(ClosedModel): correlation_id: str = Field(min_length=1, max_length=200) -class TradePayload(ClosedModel): +class TradePayload(ResponseModel): feed: Literal[Feed.TRADE] = Feed.TRADE native_trade_id: str = Field(min_length=1, max_length=200) price: DecimalValue @@ -301,7 +310,7 @@ class TradePayload(ClosedModel): is_buyer_maker: bool = False -class QuotePayload(ClosedModel): +class QuotePayload(ResponseModel): feed: Literal[Feed.QUOTE] = Feed.QUOTE bid_price: DecimalValue bid_quantity: DecimalValue @@ -311,7 +320,7 @@ class QuotePayload(ClosedModel): level: int = Field(default=1, ge=1) -class ResampleLineageView(ClosedModel): +class ResampleLineageView(ResponseModel): base_interval: str = Field(min_length=1, max_length=20) constituent_count: int = Field(ge=1) first_watermark: int = Field(ge=0) @@ -325,7 +334,7 @@ def increasing_watermark(self): return self -class BarPayload(ClosedModel): +class BarPayload(ResponseModel): feed: Literal[Feed.BAR] = Feed.BAR interval: str = Field(min_length=1, max_length=20) open_time_ns: int = Field(gt=0) @@ -357,7 +366,7 @@ def validate_lifecycle(self): return self -class BookLevel(ClosedModel): +class BookLevel(ResponseModel): side: Literal["BID", "ASK"] price: DecimalValue quantity: DecimalValue @@ -365,7 +374,7 @@ class BookLevel(ClosedModel): order_count: int = Field(default=0, ge=0) -class BookSnapshotPayload(ClosedModel): +class BookSnapshotPayload(ResponseModel): feed: Literal[Feed.BOOK_SNAPSHOT] = Feed.BOOK_SNAPSHOT native_sequence: str = Field(min_length=1, max_length=200) checksum: str | None = Field(default=None, max_length=200) @@ -382,7 +391,7 @@ def verified_snapshot_has_generation(self): return self -class BookDeltaPayload(ClosedModel): +class BookDeltaPayload(ResponseModel): feed: Literal[Feed.BOOK_DELTA] = Feed.BOOK_DELTA native_sequence_start: str = Field(min_length=1, max_length=200) native_sequence_end: str = Field(min_length=1, max_length=200) @@ -400,14 +409,14 @@ def verified_delta_has_generation(self): return self -class FundingRatePayload(ClosedModel): +class FundingRatePayload(ResponseModel): feed: Literal[Feed.FUNDING_RATE] = Feed.FUNDING_RATE rate: DecimalValue funding_time_ns: int = Field(gt=0) next_funding_time_ns: int | None = Field(default=None, gt=0) -class OpenInterestPayload(ClosedModel): +class OpenInterestPayload(ResponseModel): feed: Literal[Feed.OPEN_INTEREST] = Feed.OPEN_INTEREST quantity: DecimalValue quantity_unit: QuantityUnit @@ -421,13 +430,13 @@ def sampling_interval_is_not_blank(self): return self -class MarkIndexPricePayload(ClosedModel): +class MarkIndexPricePayload(ResponseModel): feed: Literal[Feed.MARK_INDEX_PRICE] = Feed.MARK_INDEX_PRICE mark_price: DecimalValue index_price: DecimalValue -class LongShortRatioPayload(ClosedModel): +class LongShortRatioPayload(ResponseModel): feed: Literal[Feed.LONG_SHORT_RATIO] = Feed.LONG_SHORT_RATIO population: LongShortRatioPopulation sampling_interval: str = Field(min_length=1, max_length=20) @@ -445,7 +454,7 @@ def ratio_has_ratio_unit(self): return self -class TakerFlowPayload(ClosedModel): +class TakerFlowPayload(ResponseModel): feed: Literal[Feed.TAKER_FLOW] = Feed.TAKER_FLOW sampling_interval: str = Field(min_length=1, max_length=20) buy_volume: DecimalValue @@ -460,7 +469,7 @@ def sampling_interval_is_not_blank(self): return self -class BasisPayload(ClosedModel): +class BasisPayload(ResponseModel): feed: Literal[Feed.BASIS] = Feed.BASIS kind: BasisKind sampling_interval: str = Field(min_length=1, max_length=20) @@ -489,7 +498,7 @@ def derived_basis_has_explicit_lineage(self): return self -class ContractMetadataPayload(ClosedModel): +class ContractMetadataPayload(ResponseModel): feed: Literal[Feed.CONTRACT_METADATA] = Feed.CONTRACT_METADATA contract_kind: str = Field(min_length=1, max_length=40) settlement_asset: str = Field(min_length=1, max_length=40) @@ -517,7 +526,7 @@ def tradable_rules_are_positive(self): return self -class TickerPayload(ClosedModel): +class TickerPayload(ResponseModel): feed: Literal[Feed.TICKER] = Feed.TICKER last_price: DecimalValue last_quantity: DecimalValue | None = None @@ -546,7 +555,7 @@ def quantity_units_match_optional_values(self): ] -class MarketDataView(ClosedModel): +class MarketDataView(ResponseModel): instrument_uid: str instrument_id: str instrument_revision: int = Field(ge=1) @@ -583,13 +592,13 @@ def feed_matches_payload(self): return self -class SnapshotResponse(ClosedModel): +class SnapshotResponse(ResponseModel): contract_schema: str = Field("qdl.marketdata.snapshot.v2", alias="schema") request_id: str data: MarketDataView -class InstrumentView(ClosedModel): +class InstrumentView(ResponseModel): instrument_uid: str instrument_id: str venue: str @@ -602,7 +611,7 @@ class InstrumentView(ClosedModel): status: str -class InstrumentPageResponse(ClosedModel): +class InstrumentPageResponse(ResponseModel): contract_schema: str = Field("qdl.instruments.page.v2", alias="schema") items: list[InstrumentView] next_cursor: str | None = None @@ -612,7 +621,7 @@ class InstrumentResponse(InstrumentView): contract_schema: str = Field("qdl.instrument.v2", alias="schema") -class WarmupResponse(ClosedModel): +class WarmupResponse(ResponseModel): contract_schema: str = Field("qdl.marketdata.warmup.v2", alias="schema") request_id: str snapshot_id: str = Field(min_length=1) @@ -624,14 +633,14 @@ class WarmupResponse(ClosedModel): data: list[MarketDataView] -class BatchItemResponse(ClosedModel): +class BatchItemResponse(ResponseModel): instrument_uid: str status: str data: WarmupResponse | None = None problem: ProblemDetails | None = None -class BatchResponse(ClosedModel): +class BatchResponse(ResponseModel): contract_schema: str = Field("qdl.marketdata.batch.v2", alias="schema") request_id: str partial: bool @@ -648,14 +657,14 @@ def counts_match(self): return self -class ReadinessItemResponse(ClosedModel): +class ReadinessItemResponse(ResponseModel): instrument_uid: str status: str quality: QualityView | None = None problem: ProblemDetails | None = None -class ReadinessResponse(ClosedModel): +class ReadinessResponse(ResponseModel): contract_schema: str = Field("qdl.system-readiness.v2", alias="schema") request_id: str ready: bool @@ -663,14 +672,14 @@ class ReadinessResponse(ClosedModel): results: list[ReadinessItemResponse] -class FeedStatusResponse(ClosedModel): +class FeedStatusResponse(ResponseModel): contract_schema: str = Field("qdl.feed-status.v2", alias="schema") instrument_uid: str feed: Feed quality: QualityView -class GapView(ClosedModel): +class GapView(ResponseModel): gap_id: str instrument_uid: str feed: Feed @@ -680,7 +689,7 @@ class GapView(ClosedModel): detected_at_ns: int -class GapProductCoverage(ClosedModel): +class GapProductCoverage(ResponseModel): binding_id: str instrument_uid: str feed: Feed @@ -692,7 +701,7 @@ class GapProductCoverage(ClosedModel): reason: str | None = None -class GapScanCoverage(ClosedModel): +class GapScanCoverage(ResponseModel): scope: Literal["RETAINED_WINDOW"] scan_complete: bool materialization_complete: bool @@ -702,13 +711,13 @@ class GapScanCoverage(ClosedModel): products: list[GapProductCoverage] -class GapListResponse(ClosedModel): +class GapListResponse(ResponseModel): contract_schema: str = Field("qdl.data-quality.gaps.v2", alias="schema") items: list[GapView] coverage: GapScanCoverage | None = None -class SystemReadinessSummary(ClosedModel): +class SystemReadinessSummary(ResponseModel): contract_schema: str = Field("qdl.system-readiness.v2", alias="schema") status: str authority: str diff --git a/qdl_sdk/reference.py b/qdl_sdk/reference.py index 4f61e060..e3d04d37 100644 --- a/qdl_sdk/reference.py +++ b/qdl_sdk/reference.py @@ -15,6 +15,7 @@ from qdl_sdk.models import ( ClosedModel, + ResponseModel, DecimalValue, Grade, ProblemDetails, @@ -195,13 +196,13 @@ def no_duplicate_items(self): return self -class ReferenceFieldView(ClosedModel): +class ReferenceFieldView(ResponseModel): name: str = Field(min_length=1, max_length=160) value: DecimalValue unit: str = Field(min_length=1, max_length=80) -class ReferenceLineageView(ClosedModel): +class ReferenceLineageView(ResponseModel): provider: str = Field(min_length=1, max_length=80) provider_endpoint: str = Field(min_length=1, max_length=240) source_role: str = Field(min_length=1, max_length=40) @@ -210,7 +211,7 @@ class ReferenceLineageView(ClosedModel): capability_constraint: str | None = Field(default=None, max_length=240) -class ReferenceCoverageView(ClosedModel): +class ReferenceCoverageView(ResponseModel): requested_start_ms: int | None = Field(default=None, ge=0) requested_end_ms: int | None = Field(default=None, ge=0) observed_min_ms: int | None = Field(default=None, ge=0) @@ -237,7 +238,7 @@ def ordered_window(self): return self -class ReferenceObservationView(ClosedModel): +class ReferenceObservationView(ResponseModel): instrument_uid: str = Field(min_length=1, max_length=200) instrument_revision: int = Field(ge=1) product: ReferenceProduct @@ -246,7 +247,7 @@ class ReferenceObservationView(ClosedModel): labels: dict[str, str] = Field(default_factory=dict) -class ReferenceDataResponse(ClosedModel): +class ReferenceDataResponse(ResponseModel): instrument_uid: str = Field(min_length=1, max_length=200) product: ReferenceProduct status: Literal["OK", "MISSING", "UNAVAILABLE", "ERROR"] @@ -270,7 +271,7 @@ def status_matches_observations(self): return self -class ReferenceBatchItemResponse(ClosedModel): +class ReferenceBatchItemResponse(ResponseModel): instrument_uid: str = Field(min_length=1, max_length=200) product: ReferenceProduct status: str = Field(min_length=1, max_length=80) @@ -278,7 +279,7 @@ class ReferenceBatchItemResponse(ClosedModel): problem: ProblemDetails | None = None -class ReferenceBatchResponse(ClosedModel): +class ReferenceBatchResponse(ResponseModel): contract_schema: str = Field("qdl.reference.batch.v2", alias="schema") request_id: str = Field(min_length=1) partial: bool diff --git a/scripts/build_qdl_sdk_release.py b/scripts/build_qdl_sdk_release.py index d6b78b02..a339e889 100755 --- a/scripts/build_qdl_sdk_release.py +++ b/scripts/build_qdl_sdk_release.py @@ -15,7 +15,7 @@ ROOT = Path(__file__).resolve().parents[1] NAME = "qdl-sdk" NORMALIZED_NAME = "qdl_sdk" -VERSION = "2.0.7" +VERSION = "2.1.0" DIST_INFO = f"{NORMALIZED_NAME}-{VERSION}.dist-info" DEPENDENCIES = ( "grpcio>=1.70.0,<2.0.0", diff --git a/tests/test_phaseb_stable_release.py b/tests/test_phaseb_stable_release.py index e6789af8..98a6ebaa 100644 --- a/tests/test_phaseb_stable_release.py +++ b/tests/test_phaseb_stable_release.py @@ -317,7 +317,7 @@ def test_service_openapi_and_sdk_versions_are_explicit(self): ) generated = build_openapi() self.assertEqual(package["project"]["version"], "2.0.0") - self.assertEqual(qdl_sdk.__version__, "2.0.7") + self.assertEqual(qdl_sdk.__version__, "2.1.0") self.assertEqual(generated["info"]["version"], "2.0.0") self.assertEqual(snapshot, generated) # ``reference:batch`` is a governed V2 public path in the checked-in diff --git a/tests/test_qdl_sdk_additive_responses.py b/tests/test_qdl_sdk_additive_responses.py new file mode 100644 index 00000000..d3962364 --- /dev/null +++ b/tests/test_qdl_sdk_additive_responses.py @@ -0,0 +1,70 @@ +"""D2: additive response compatibility does not weaken requests or domain checks.""" +import copy +import unittest +from pydantic import ValidationError +from qdl_sdk.models import (Feed, SnapshotResponse, WarmupSpecification, + WarmupTimeRange, DecimalValue, ProblemDetails) +from qdl_sdk.reference import ReferenceRequirement, ReferenceBatchRequest, ReferenceBatchResponse +from tests.test_qdl_sdk_stream_projection import template, payload_fixture + + +def added_fields(value): + if isinstance(value, dict): + return {**{k: added_fields(v) for k, v in value.items()}, + "future_server_field": {"revision": 3}} + if isinstance(value, list): + return [added_fields(v) for v in value] + return value + + +class AdditiveResponseTests(unittest.TestCase): + def test_old_server_and_additive_server_all_feed_payloads(self): + for feed in Feed: + if feed is Feed.UNSPECIFIED: + continue + with self.subTest(feed=feed): + try: + payload_fixture(feed) + except KeyError: + continue + original = {"request_id": "d2", "data": template(feed).model_dump(mode="json")} + old = SnapshotResponse.model_validate(original) + new = SnapshotResponse.model_validate(added_fields(original)) + self.assertEqual(new.model_dump(), old.model_dump()) + self.assertEqual(SnapshotResponse.model_validate_json(new.model_dump_json()), old) + + def test_problem_and_nested_reference_response_ignore_additions(self): + problem = dict(type="test", title="stale", status=409, code="DATA_STALE", + detail="stale", request_id="d2", retryable=False) + self.assertEqual(ProblemDetails.model_validate(added_fields(problem)), + ProblemDetails.model_validate(problem)) + response = dict(schema="qdl.reference.batch.v2", request_id="d2", partial=True, + success_count=0, error_count=1, + results=[dict(instrument_uid="test", product="MARK_INDEX_PRICE", + status="ERROR", problem=problem)]) + self.assertEqual(ReferenceBatchResponse.model_validate(added_fields(response)), + ReferenceBatchResponse.model_validate(response)) + + def test_known_response_fields_still_reject_invalid_domain_values(self): + original = added_fields({"request_id": "d2", "data": template(Feed.QUOTE).model_dump(mode="json")}) + for key, value in (("feed", "UNKNOWN"), ("quality", {"state": "BOGUS"}), + ("payload", {"feed": "QUOTE"})): + with self.subTest(key=key): + malformed = copy.deepcopy(original) + malformed["data"][key] = value + with self.assertRaises(ValidationError): + SnapshotResponse.model_validate(malformed) + with self.assertRaises(ValidationError): + DecimalValue(coefficient="100", scale=2, source_text="2", new_field=True) + + def test_requests_remain_closed(self): + for model, payload in ( + (WarmupTimeRange, dict(start_time_ns=1, end_time_ns=2)), + (WarmupSpecification, dict(rows=700)), + (ReferenceRequirement, dict(instrument_uid="test", product="MARK_INDEX_PRICE")), + (ReferenceBatchRequest, dict(requirements=[])), + ): + with self.subTest(model=model.__name__): + with self.assertRaises(ValidationError) as caught: + model.model_validate({**payload, "future_request_field": True}) + self.assertIn("extra_forbidden", [e["type"] for e in caught.exception.errors()]) diff --git a/tests/test_qdl_sdk_release.py b/tests/test_qdl_sdk_release.py index 49d1d5cf..0131765b 100644 --- a/tests/test_qdl_sdk_release.py +++ b/tests/test_qdl_sdk_release.py @@ -33,7 +33,7 @@ def test_wheel_is_reproducible_self_contained_and_recorded(self): self.assertEqual({row[0] for row in rows}, names) release = json.loads(Path(first_result["manifest"]).read_text()) self.assertFalse(release["contains_service_internals"]) - self.assertEqual(release["version"], "2.0.7") + self.assertEqual(release["version"], "2.1.0") self.assertEqual(len(release["generated_contract_digest"]), 64) sbom = json.loads(Path(first_result["sbom"]).read_text()) self.assertEqual(sbom["bomFormat"], "CycloneDX") From f8f377008904d10f03dcfa7ec1a9207a323ab1a0 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 11:49:11 +0000 Subject: [PATCH 006/147] test(p184): parameterize KN replicas and diagnose retained BAR gaps --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 23 ++++++++ scripts/benchmark_kn_ts_consumer.py | 18 +++++-- scripts/diagnose_kn_bar_window.py | 64 +++++++++++++++++++++++ scripts/measure_kn_ts_cache_delivery.py | 2 + tests/test_kn_bar_window_diagnostic.py | 31 +++++++++++ tests/test_kn_consumer_probe_replicas.py | 25 +++++++++ 6 files changed, 160 insertions(+), 3 deletions(-) create mode 100644 scripts/diagnose_kn_bar_window.py create mode 100644 tests/test_kn_bar_window_diagnostic.py create mode 100644 tests/test_kn_consumer_probe_replicas.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 36e8d3e6..730207e2 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69072,3 +69072,26 @@ Tests in existing acf5fd06 image, source mounted read-only, network none, 2 CPUs because /app/logs mount nested inside a read-only checkout; fixed mount layout (/src checkout, /app/logs tmpfs), not permissions on source. Containers --rm. No new image, shared mutation, push/merge or cleanup of retained artifacts. + + +##### Phase B slice - consumer probe names and BAR diagnostic + +Status: IMPLEMENTED / TESTED_LOCALLY; BAR retained-window observation READ_ONLY. +KN_QUERY_REPLICAS selects exactly two distinct DNS names in benchmark_kn_ts_consumer +(defaults preserved); all coverage aggregation uses the same selected pair. +measure_kn_ts_cache_delivery has no Query RPC or hardcoded Query name: it observes +TS Redis only. Added explicitly labelled declared-replica metadata, not invented +per-replica attribution. No consumer/load measurement run on shared runtime. +Added diagnose_kn_bar_window.py using existing KN generation/source-fenced reader. +It checks only 15m/30m retained windows; empty generation, topic mismatch, diagnostic +flags or missing opens cannot PASS. Eight focused diagnostic/replica unit tests +PASS (zero fail/skip), acf5fd06 image, network none, read-only source. +Executed approved read-only diagnostic inside query_kn_1: 20 bindings, 214752 +retained rows, 0 missing opens/flags/errors. No coverage claim before/after retained +bounds, no repair/publication. Evidence external p184-phaseb-20261007/evidence/ +bar-window-query-kn-1.json SHA256 b80d38099ec27aeb3520cc8c15d506a7d6e5193799195d691e12b6d1d1408241. +Inventory correction: 12 active non-infrastructure application roles, not the +handoff's 13: 3 core + 2 ingestor + 2 projector + 2 stream + 2 query + 1 BAR edge. +Phase C must explicitly resolve this count; do not start an invented/thirteenth +role, touch V1, or recreate state/TLS initializers to satisfy an incorrect count. +No shared write/restart, test containers --rm, no TS edits or image cleanup yet. diff --git a/scripts/benchmark_kn_ts_consumer.py b/scripts/benchmark_kn_ts_consumer.py index 82866428..8c69dd02 100644 --- a/scripts/benchmark_kn_ts_consumer.py +++ b/scripts/benchmark_kn_ts_consumer.py @@ -52,6 +52,16 @@ def _market(venue): return ("binance", "futures") if venue == "BINANCE" else ("okx", "swap") +def query_replicas(): + """Explicit DNS names; keep legacy defaults for existing isolated packets.""" + import re + names = tuple(os.environ.get("KN_QUERY_REPLICAS", "query_v2_1,query_v2_2").split(",")) + if (len(names) != 2 or len(set(names)) != 2 + or any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,252}", name) for name in names)): + raise ValueError("KN_QUERY_REPLICAS requires two distinct DNS names") + return names + + async def ts60(results): sys.path.insert(0, "/app") from adapters.market_data.data_layer_v2 import Feed, build_versioned_data_layer_client @@ -80,7 +90,8 @@ def validate_recording(view, *args, **kwargs): rounds = int(os.environ.get("KN4_TS60_ROUNDS", "5")) good = settings.DATA_LAYER_V2_QUERY_URL phase_a = [] - for replica in ("query_v2_1", "query_v2_2"): + replicas = query_replicas() + for replica in replicas: settings.DATA_LAYER_V2_QUERY_URL = f"https://{replica}:8200" client = build_versioned_data_layer_client(settings, consumer_id="trading-system.paper.stable", mode_override="V2_PRIMARY") @@ -176,11 +187,12 @@ async def bounded(product): for metric in ("sdk_ms", "total_ms", "ages_ms"): cell[metric] = _dist(cell[metric]) results.append({"consumer": "ts60", "phase": "A_snapshots_per_replica", "rounds": rounds, + "query_replicas": list(replicas), "concurrency_per_replica": int(os.environ.get("KN4_TS60_CONCURRENCY", "8")), "reads": len(phase_a), "failed": sum(1 for row in phase_a if not row["ok"]), "products": len(by_product), "products_ok_on_both_replicas": sum(1 for e in by_product.values() - if e["replicas_ok"] == {"query_v2_1", "query_v2_2"}), + if e["replicas_ok"] == set(replicas)), "timing_basis": "call_to_usable = consumer queue (semaphore) + SDK call incl. server queue " "and client validation; all from the call, per read", "by_feed": {feed: {"reads": g["reads"], "failed": g["failed"], @@ -190,7 +202,7 @@ async def bounded(product): "per_binding_replica": per_binding, "products_without_a_positive_read_on_both_replicas": sorted( product for product, cells in per_binding.items() - if not all(cells.get(r, {}).get("ok") for r in ("query_v2_1", "query_v2_2"))), + if not all(cells.get(r, {}).get("ok") for r in replicas)), "failures": {k: {"ok": v["ok"], "failed": v["failed"], "errors": v["errors"]} for k, v in sorted(by_product.items()) if v["failed"]}}) diff --git a/scripts/diagnose_kn_bar_window.py b/scripts/diagnose_kn_bar_window.py new file mode 100644 index 00000000..31cd8dcc --- /dev/null +++ b/scripts/diagnose_kn_bar_window.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Read-only P18.4 retained-window diagnostic for existing KN 15m/30m bindings. + +Uses the production generation/fence-checked reader; never repairs or publishes. +An empty/missing generation is a finding, never zero gaps. Does not certify missing +history before/after the retained window. No credential or payload is reported. +""" +from __future__ import annotations +import json +import os +import time +from qdl.adapters.intervals import canonical_interval_ms +from qdl.runtime.kn_bar_readback import binding_product_key +from qdl.runtime.kn_market_cache import reader_from_environment +from qdl.runtime.stable_catalog import StableSourceCatalog + + +def diagnose(reader, bindings, topic_id, *, max_rows=12064, seconds=30): + deadline = time.monotonic() + seconds + rows = [] + def budget(): + if time.monotonic() >= deadline: + raise TimeoutError("BAR diagnostic deadline exceeded") + for binding in bindings: + if binding.feed.value != "BAR" or binding.interval not in {"15m", "30m"}: + continue + row = {"binding_id": binding.binding_id, "interval": binding.interval} + try: + budget() + step = canonical_interval_ms(binding.interval) + boundary, spans, flags = reader.bar_diagnostic_ranges( + binding_product_key(binding, reader.environment), step, + last=max_rows, check_budget=budget) + if boundary.topic_id != topic_id: + raise ValueError("source topic identity mismatch") + if not spans: + raise ValueError("no retained BAR generation/window") + missing = sum(max(0, (start - end) // step - 1) + for (_, end), (start, _) in zip(spans, spans[1:])) + row.update(state="PASS" if not missing and not flags else "GAP", + missing_opens=missing, flagged_rows=len(flags), + retained_rows=sum((end-start)//step+1 for start,end in spans), + first_open_ms=spans[0][0], last_open_ms=spans[-1][1], + boundary={"topic_id": boundary.topic_id, "partition": boundary.partition, + "offset": boundary.offset}, + missing_ranges=[[end+step, start-step] for (_,end),(start,_) in zip(spans,spans[1:]) if start-end>step][:20]) + except Exception as error: + row.update(state="ERROR", error_type=type(error).__name__) + rows.append(row) + time.sleep(.025) + return {"schema": "qdl.p184.bar-retained-window.v1", "writes": 0, + "scope": "retained window only; no assertion before/after retained bounds", + "status": "PASS" if rows and all(r["state"] == "PASS" for r in rows) else "FINDINGS", + "bindings": rows} + + +if __name__ == "__main__": + reader = reader_from_environment(os.environ) + try: + catalog = StableSourceCatalog.load(os.environ["QDL_STABLE_SOURCE_BINDINGS"]) + report = diagnose(reader, catalog.bindings, os.environ["QDL_KN_TOPIC_ID"]) + print(json.dumps(report, sort_keys=True)) + finally: + reader.client.close() diff --git a/scripts/measure_kn_ts_cache_delivery.py b/scripts/measure_kn_ts_cache_delivery.py index f305b302..f210b56e 100644 --- a/scripts/measure_kn_ts_cache_delivery.py +++ b/scripts/measure_kn_ts_cache_delivery.py @@ -92,6 +92,8 @@ async def main(): await pub.aclose() await client.aclose() print(json.dumps({"schema": "qdl.kn5.actual-ts-cache-availability.v1", "seconds": time.monotonic() - started, + "query_replicas_declared": os.environ.get("KN_QUERY_REPLICAS", "").split(",") if os.environ.get("KN_QUERY_REPLICAS") else [], + "replica_attribution": "observer reads TS Redis only; declared names are packet metadata, not per-replica measurements", "boundary": "source event (BAR close) -> external GET after actual TS Redis SET/PUBLISH; not publisher ACK", "sampling": "at most2Hz per channel; max2000 samples per product; no source event dedup", "event_to_cache_readable": {k: summary(v) for k,v in samples.items()}, diff --git a/tests/test_kn_bar_window_diagnostic.py b/tests/test_kn_bar_window_diagnostic.py new file mode 100644 index 00000000..fffd108b --- /dev/null +++ b/tests/test_kn_bar_window_diagnostic.py @@ -0,0 +1,31 @@ +import unittest +from types import SimpleNamespace as NS +from unittest.mock import patch +from scripts.diagnose_kn_bar_window import diagnose + + +class BarWindowDiagnosticTests(unittest.TestCase): + def run_case(self, spans, flags=(), topic="canonical"): + reader = NS(environment="paper", bar_diagnostic_ranges=lambda *a, **k: + (NS(topic_id=topic, partition=0, offset=12), spans, flags)) + binding = NS(feed=NS(value="BAR"), interval="15m", binding_id="test") + with patch("scripts.diagnose_kn_bar_window.binding_product_key", return_value="test"): + return diagnose(reader, [binding], "canonical") + + def test_contiguous_retained_window(self): + result = self.run_case([(0, 1800000)]) + self.assertEqual(result["status"], "PASS") + self.assertEqual(result["bindings"][0]["retained_rows"], 3) + + def test_gap_is_not_pass(self): + result = self.run_case([(0, 0), (1800000, 1800000)]) + self.assertEqual(result["status"], "FINDINGS") + self.assertEqual(result["bindings"][0]["missing_opens"], 1) + + def test_no_generation_and_wrong_topic_fail_closed(self): + self.assertEqual(self.run_case([])["status"], "FINDINGS") + self.assertEqual(self.run_case([(0, 0)], topic="old")["status"], "FINDINGS") + + def test_flags_and_empty_inventory_not_pass(self): + self.assertEqual(self.run_case([(0, 0)], [(0, 1)])["status"], "FINDINGS") + self.assertEqual(diagnose(None, [], "canonical")["status"], "FINDINGS") diff --git a/tests/test_kn_consumer_probe_replicas.py b/tests/test_kn_consumer_probe_replicas.py new file mode 100644 index 00000000..743fc5b1 --- /dev/null +++ b/tests/test_kn_consumer_probe_replicas.py @@ -0,0 +1,25 @@ +"""Replica configuration stays explicit without touching any live consumer.""" +import unittest +from unittest.mock import patch +from scripts.benchmark_kn_ts_consumer import query_replicas, _dist + + +class ProbeReplicaTests(unittest.TestCase): + def test_existing_packets_keep_default(self): + with patch.dict("os.environ", {}, clear=True): + self.assertEqual(query_replicas(), ("query_v2_1", "query_v2_2")) + + def test_kn_names_and_fqdn(self): + for value in ("query_kn_1,query_kn_2", "query-1.test,query-2.test"): + with self.subTest(value=value), patch.dict("os.environ", KN_QUERY_REPLICAS=value): + self.assertEqual(query_replicas(), tuple(value.split(","))) + + def test_duplicate_missing_url_and_injection_rejected(self): + for value in ("", "query_kn_1", "a,a", "a,b,c", "https://a,b", "a,b/path", "a, b"): + with self.subTest(value=value), patch.dict("os.environ", KN_QUERY_REPLICAS=value): + with self.assertRaises(ValueError): + query_replicas() + + def test_sparse_samples_do_not_claim_p99(self): + self.assertIsNone(_dist([1, 2, 3])["p99"]) + self.assertEqual(_dist(list(range(100)))["p99"], 98) From 43bbcf0209f49e9cc1af6802af8f8a207a53ee3c Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 11:52:33 +0000 Subject: [PATCH 007/147] fix(l2): materialize verified top changes with bounded cadence --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 29 ++++ rust/qdl-realtime-core/src/lib.rs | 203 +++++++++++++++++++++- 2 files changed, 224 insertions(+), 8 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 730207e2..d4b25f2e 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69095,3 +69095,32 @@ handoff's 13: 3 core + 2 ingestor + 2 projector + 2 stream + 2 query + 1 BAR edg Phase C must explicitly resolve this count; do not start an invented/thirteenth role, touch V1, or recreate state/TLS initializers to satisfy an incorrect count. No shared write/restart, test containers --rm, no TS edits or image cleanup yet. + + +##### Phase B slice IDR-062 - bounded top-change materialization + +Status: IMPLEMENTED / TESTED_LOCALLY, not runtime-certified. Rust retains exact +best bid/ask of the last emitted snapshot (not every delta); a changed top emits +an additional snapshot after 100 ms of provider observation time. Suppressed +changes remain pending. Unchanged top retains configured periodic cadence (1000 ms +on execution bindings), evaluated only on verified provider transitions. No wall +timer/heartbeat proof, timestamp rewrite, endpoint or binding/schema change. +Comparison borrows exact decimal prices; allocations occur only when recording a +published snapshot. Delta publication, event identity, sequence and generation +fences stay intact. The floor bounds provider-time amplification, NOT wall-clock +replay throughput or independent quote/book equality (channels are asynchronous). +Tests: pinned Rust 1.82/OpenSSL builder, --locked, 2 CPU / 3 GiB; qdl-core 50 PASS, +qdl-realtime-core 52 PASS, 0 fail, 1 existing ignored isolated-Redis admission test +(outside changed BOOK path). New dual-venue tests: top/ask-only change, 99/100 ms +floor, pending change on quantity-only delta, unchanged top at 999/1000 ms, exact +delta/snapshot source sequence and timestamp, duplicate/gap rejection, 1000 +transitions yielding exactly 10 extra snapshots per provider second. Prior tests +cover delayed replay, resync, lane and generation changes. Initial failures were +test macro qualification and duplicate price rows in a new fixture; corrected +without changing production validation. Offline builder cache was incomplete; +locked crates downloaded in disposable builder, no runtime network credentials. +Evidence rust-tests-final.log SHA256 fd9b80ac3b837485b41628b444de63b419edba8aa244c144ff262a3ff390f553. +Real/capture candidate BOOK rate, Kafka/whole-stack CPU and TS coherence >=39/40 +remain unmeasured; do not infer those gates from synthetic source tests. Phase C +remains held pending these measurements and TS DR01-DR08/broker delta. No role +restart, shared state write, resource quota change, push or merge. diff --git a/rust/qdl-realtime-core/src/lib.rs b/rust/qdl-realtime-core/src/lib.rs index f15f9369..0f26d7e9 100644 --- a/rust/qdl-realtime-core/src/lib.rs +++ b/rust/qdl-realtime-core/src/lib.rs @@ -13,7 +13,7 @@ use qdl_core::canonical::{ TradeContext, TradeFixture, }; use qdl_core::l2_adapter::{BookPublication, BookTransition, L2BookAdapter}; -use qdl_core::l2_book::{BookIdentity, BookOutcome, BookStatus}; +use qdl_core::l2_book::{BookIdentity, BookOutcome, BookStatus, ExactDecimal}; use qdl_core::okx::expand_data_frame; use qdl_core::transport::DurableRecord; use qdl_provider_envelope::validate as validate_raw; @@ -396,11 +396,35 @@ struct MarkIndexPairState { pub const L2_RESNAPSHOT_REQUIRED_V1: &str = "L2_RESNAPSHOT_REQUIRED_V1"; +// Space hot snapshots by at least 100 ms of provider observation time. +// Replay may run faster than wall time; it must retain deterministic output. +// This is not a freshness proof: only verified provider transitions can emit. +const L2_TOP_CHANGE_MIN_INTERVAL_MS: i64 = 100; + +struct L2SnapshotObservation { + observed_at_ms: i64, + generation: u64, + bid: Option, + ask: Option, +} + +impl L2SnapshotObservation { + fn from_transition(transition: &BookTransition, observed_at_ms: i64) -> Option { + let view = transition.view.as_ref()?; + Some(Self { + observed_at_ms, + generation: transition.generation, + bid: view.bids.first().map(|level| level.price.clone()), + ask: view.asks.first().map(|level| level.price.clone()), + }) + } +} + pub struct RealtimeCore { config: RealtimeCoreConfig, bindings: BTreeMap, l2_adapters: BTreeMap, - l2_snapshot_observed_at_ms: BTreeMap, + l2_snapshot_observations: BTreeMap, mark_index_pairs: BTreeMap, partition_sequences: BTreeMap, /// The provider session that currently owns each L2 book, so a lane change @@ -448,7 +472,7 @@ impl RealtimeCore { config, bindings, l2_adapters, - l2_snapshot_observed_at_ms: BTreeMap::new(), + l2_snapshot_observations: BTreeMap::new(), mark_index_pairs: BTreeMap::new(), partition_sequences: BTreeMap::new(), l2_sessions: BTreeMap::new(), @@ -1354,8 +1378,12 @@ impl RealtimeCore { .insert(partition_key.clone(), canonical.partition_sequence); self.remember(canonical.event_id.clone()); if let Some(observed_at_ms) = snapshot_observed_at_ms { - self.l2_snapshot_observed_at_ms - .insert(binding_key.to_owned(), observed_at_ms); + if let Some(observation) = + L2SnapshotObservation::from_transition(&transition, observed_at_ms) + { + self.l2_snapshot_observations + .insert(binding_key.to_owned(), observation); + } } durable.push(canonical_record( &self.config.canonical_stream, @@ -1390,10 +1418,21 @@ impl RealtimeCore { } let observed_at_ms = l2_observed_at_ms(transition, raw)?; let refresh_ms = i64::try_from(l2.materialized_snapshot_interval_ms()).ok()?; + let view = transition.view.as_ref()?; + let bid = view.bids.first().map(|level| &level.price); + let ask = view.asks.first().map(|level| &level.price); let due = self - .l2_snapshot_observed_at_ms + .l2_snapshot_observations .get(binding_key) - .map(|previous| observed_at_ms.saturating_sub(*previous) >= refresh_ms) + .map(|previous| { + let elapsed = observed_at_ms.saturating_sub(previous.observed_at_ms); + let top_changed = previous.bid.as_ref() != bid || previous.ask.as_ref() != ask; + // Compare against the last emitted snapshot, not the prior delta: + // a change suppressed by the floor must remain pending. + previous.generation != transition.generation + || elapsed >= refresh_ms + || (top_changed && elapsed >= L2_TOP_CHANGE_MIN_INTERVAL_MS) + }) .unwrap_or(true); if !due { return None; @@ -2661,7 +2700,10 @@ mod tests { 13, ) .unwrap(); - assert!(ahead.canonical.is_empty(), "a REST snapshot ahead of the stream is no quiet proof"); + assert!( + ahead.canonical.is_empty(), + "a REST snapshot ahead of the stream is no quiet proof" + ); assert!(ahead.quarantines.is_empty()); assert_eq!(ahead.filtered_outcome, Some("REST_AHEAD_OF_STREAM")); } @@ -2993,6 +3035,151 @@ mod tests { } } + fn hot_book_core(okx: bool) -> (RealtimeCore, CoreBinding) { + let mut binding = if okx { + okx_book_binding() + } else { + binance_book_binding() + }; + binding + .l2 + .as_mut() + .unwrap() + .materialized_snapshot_interval_ms = Some(1_000); + let mut engine = core(binding.clone(), true); + let initial = if okx { + raw(&binding, br#"{"arg":{"channel":"books","instId":"BTC-USDT-SWAP"},"action":"snapshot","data":[{"seqId":"101","prevSeqId":"-1","ts":"1000","bids":[["60000","1","0","1"]],"asks":[["60001","1","0","1"]]}]}"#, 1) + } else { + engine.process(raw(&binding, br#"{"s":"BTCUSDT","U":99,"u":101,"pu":98,"E":1000,"b":[["60000","1"]],"a":[]}"#, 1), 1).unwrap(); + with_transport(raw(&binding, br#"{"lastUpdateId":100,"E":1000,"bids":[["60000","1"]],"asks":[["60001","1"]]}"#, 1), TransportProtocol::Http) + }; + assert_eq!(engine.process(initial, 2).unwrap().canonical.len(), 1); + (engine, binding) + } + + fn hot_delta( + binding: &CoreBinding, + seq: u64, + stamp: i64, + bid: &str, + ask: &str, + ) -> RawProviderEnvelope { + let asks = if ask == "60001" { + serde_json::json!([[ask, "2", "0", "1"]]) + } else { + serde_json::json!([["60001", "0", "0", "0"], [ask, "2", "0", "1"]]) + }; + let payload = if binding.venue == "OKX" { + serde_json::json!({"arg":{"channel":"books","instId":"BTC-USDT-SWAP"},"action":"update", + "data":[{"seqId":seq.to_string(),"prevSeqId":(seq-1).to_string(),"ts":stamp.to_string(), + "bids":[[bid,"2","0","1"]],"asks":asks}]}) + } else { + serde_json::json!({"s":"BTCUSDT","U":seq,"u":seq,"pu":seq-1,"E":stamp, + "b":[[bid,"2"]],"a":asks}) + }; + raw(binding, &serde_json::to_vec(&payload).unwrap(), 1) + } + + #[test] + fn hot_top_change_coalesces_until_floor_on_both_venues() { + for okx in [false, true] { + let (mut engine, binding) = hot_book_core(okx); + // Changed top before the floor remains pending even if the next + // frame only changes quantity. Every original delta is still emitted. + for (seq, stamp, count) in [ + (102, 1050, 1), + (103, 1099, 1), + (104, 1100, 2), + (105, 1200, 1), + ] { + let batch = engine + .process(hot_delta(&binding, seq, stamp, "60000.5", "60002"), 3) + .unwrap(); + assert!(batch.quarantines.is_empty()); + assert_eq!( + batch.canonical.len(), + count, + "venue={} stamp={stamp}", + binding.venue + ); + if count == 2 { + let delta = + EventEnvelope::decode(batch.canonical[0].payload.as_slice()).unwrap(); + let snapshot = + EventEnvelope::decode(batch.canonical[1].payload.as_slice()).unwrap(); + assert_eq!(snapshot.source_event_time_ns, stamp * 1_000_000); + assert_eq!(snapshot.source_sequence, delta.source_sequence); + assert_eq!(snapshot.partition_sequence, delta.partition_sequence + 1); + assert_ne!(snapshot.event_id, delta.event_id); + assert!(matches!( + snapshot.payload, + Some(event_envelope::Payload::BookSnapshot(_)) + )); + } + } + } + } + + #[test] + fn hot_snapshot_amplification_is_bounded_in_provider_time() { + for okx in [false, true] { + let (mut engine, binding) = hot_book_core(okx); + let mut snapshots = 0; + for n in 1..=1_000 { + let bid = format!("60000.{n:04}"); + let batch = engine + .process( + hot_delta(&binding, 101 + n, 1000 + n as i64, &bid, "60002"), + 3, + ) + .unwrap(); + assert!(batch.quarantines.is_empty()); + assert!(matches!( + EventEnvelope::decode(batch.canonical[0].payload.as_slice()) + .unwrap() + .payload, + Some(event_envelope::Payload::BookDelta(_)) + )); + snapshots += batch.canonical.len() - 1; + } + assert_eq!(snapshots, 10); + } + } + + #[test] + fn unchanged_top_waits_for_periodic_provider_transition_on_both_venues() { + for okx in [false, true] { + let (mut engine, binding) = hot_book_core(okx); + for (seq, stamp, count) in [(102, 1100, 1), (103, 1999, 1), (104, 2000, 2)] { + let batch = engine + .process(hot_delta(&binding, seq, stamp, "60000", "60001"), 3) + .unwrap(); + assert!(batch.quarantines.is_empty()); + assert_eq!(batch.canonical.len(), count); + } + } + } + + #[test] + fn ask_only_top_change_emits_at_floor_without_waiting_one_second() { + for okx in [false, true] { + let (mut engine, binding) = hot_book_core(okx); + let batch = engine + .process(hot_delta(&binding, 102, 1100, "60000", "60002"), 3) + .unwrap(); + assert_eq!(batch.canonical.len(), 2); + let duplicate = engine + .process(hot_delta(&binding, 102, 1100, "60000", "60002"), 4) + .unwrap(); + assert!(duplicate.canonical.is_empty()); + let gap = engine + .process(hot_delta(&binding, 104, 1200, "60000.5", "60002"), 5) + .unwrap(); + assert!(gap.canonical.is_empty()); + assert_eq!(gap.quarantines.len(), 1); + } + } + #[test] fn okx_l2_snapshot_then_update_emit_distinct_public_events() { let binding = okx_book_binding(); From 5eb453dcad51419f6d09f6580c4592402ea4dc7d Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 12:16:53 +0000 Subject: [PATCH 008/147] docs(p184): record candidate evidence and held Phase C packet --- CERTIFICATION_LEDGER.md | 23 ++++ DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 55 ++++++++++ upgrade/evidence/p184-dl-phase-c-packet.md | 122 +++++++++++++++++++++ 3 files changed, 200 insertions(+) create mode 100644 upgrade/evidence/p184-dl-phase-c-packet.md diff --git a/CERTIFICATION_LEDGER.md b/CERTIFICATION_LEDGER.md index 5c1cbf1f..5dc8b6a3 100644 --- a/CERTIFICATION_LEDGER.md +++ b/CERTIFICATION_LEDGER.md @@ -2102,3 +2102,26 @@ Published with `scripts/publish_github_release.py` against a fine-grained token held in a 0600 file outside the repository; the token was never printed and the tool refused to replace an existing release. `gh` is not installed on this host, which is why the script exists. + + +## P18.4 DL closure candidate - 2026-10-07 (not certified) + +Status: SOURCE_CANDIDATE_READY_MEASUREMENT_PENDING. This entry does not replace +previous immutable certificates or certify a runtime rollout. Published v2.2.2 +provenance is in upgrade/evidence/releases/v2.2.2/ and the Unified Plan publication +closure; historical entries above are not a current runtime inventory. + +Candidate source 43bbcf0209f49e9cc1af6802af8f8a207a53ee3c, branch +fix/p184-dl-closure. Rust 69e3b2b214331...; Python 77682eea5054...; exact +digests/roles/rollback in [Phase C packet](upgrade/evidence/p184-dl-phase-c-packet.md). +D2 SDK 2.1.0 response tolerance and IDR-062 top-change materialization are locally +tested. Rust 102 PASS/1 existing ignored isolated-Redis test; SDK suites 28 and +91 PASS; tools 8 PASS; packaged Python 29 PASS (overlapping source checks). +Read-only BAR diagnostic: 20 retained 15m/30m windows, 214752 rows, zero gaps; not +a proof beyond retained bounds. No shared mutation or latency improvement claim. + +External evidence: ~/.local/state/qdl-v2/p184-phaseb-20261007/evidence/phase-b-receipt.json. +Inherited Phase A 4eb08dc, no catalogue/C2 rerun. Release waits for real/capture +candidate resource/coherence evidence, TS DR01-DR08 and broker delta, and owner +approval of the actual 12-role packet (handoff listed 13). D1/D3 remain deferred. +Only source/docs, two candidate images and scoped test cleanup completed. diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index d4b25f2e..074f2182 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69124,3 +69124,58 @@ Real/capture candidate BOOK rate, Kafka/whole-stack CPU and TS coherence >=39/40 remain unmeasured; do not infer those gates from synthetic source tests. Phase C remains held pending these measurements and TS DR01-DR08/broker delta. No role restart, shared state write, resource quota change, push or merge. + + +##### Phase B candidate artifact and Phase C handoff (Codex, 2026-10-07) + +Status: SOURCE_CANDIDATE_READY_MEASUREMENT_PENDING. Not production-certified, +not released, no Phase C execution. Source slices: eb7c649 (D2), f8f3770 +(probes/BAR), 43bbcf0 (L2); same application SHA for both completed images: + +- Rust qdl-v2-rust:2.2.3-rc2-43bbcf0 = sha256:69e3b2b214331a7ebb6a110dd8159c899bbbd6439aaf87293c8c68d145e3713d. +- Python qdl-v2-python:2.2.3-rc2-43bbcf0 = sha256:77682eea5054ae9ddbd858323b083286669a55eeb4bc7bc065b04bae8854e32f. + +Canonical Dockerfiles, full native build includes 12 binaries, four runtime native +role binaries have resolved shared libraries. Python dependency receipt PASS: 48 +locked packages, 5954 hashed files; ten packaged source files match candidate. +Initial Python build correctly refused tag-only dependency input; rebuilt with +immutable acf5fd06 digest, without bypassing dependency verification. Packaged +Python affected tests 29 PASS / 0 fail / 0 skip. Source counts above remain separate +(overlap with packaged tests; do not sum as unique coverage). SDK wheel 2.1.0 SHA256 +3ed58182f3b6a077e21b218589c33b0a1289eb5cd1609f4910200aae6a138c47, not published. + +Binding compatibility check found source renderer r27 differs from the deployed +writer's r22 in inventory/release/universal hash despite identical 60 products. +DO NOT mount generic r27 output. Prepared r22 V3 by canonical dataclass constructor +with identical products and manifest-10 policies; only schema, binding checksum +and request_policies change. Old V2 remains untouched. An initial constructor +probe retained the old checksum, then an empty checksum; both correctly rejected. +Using documented None builds a NEW checksum, never overwrites old evidence. +V3 binding hash a7a4380581a923223bf40a067fd4f718c2ccf0b23705abf216abc38f86de84bd; +file hash 4bdaa49410190948a8d15ad07ffae5bd73ccb48cc327fd2936b36b815daba5dc. +Universal b4e43688...9afe, release r22, JWT/manifest 10 and 60 products unchanged. +No binding was mounted; no TS source/image/config was edited. + +Draft exact-role packet: [P18.4 Phase C](upgrade/evidence/p184-dl-phase-c-packet.md). +It preserves the observed 12-role roster (not 13), all rollback digests, config +fingerprints, TLS/state and source/group ownership. Owner must resolve roster and +approve packet after TS DR01-DR08/broker delta and the unmeasured IDR-062 real/capture +resource/coherence gates. No current improvement in consumer latency, <=5 vCPU +whole-stack usage or uninterrupted execution eligibility is claimed. D1/D3 remain +explicit owner deferrals, not secretly implemented. + +External receipt: p184-phaseb-20261007/evidence/phase-b-receipt.json SHA256 7ab598b514087c22d46645691d8e3632eec59ff8f13b05d3f43a4a03d4900ecf. +Runtime post-check: all 19 inventoried DL containers (17 running, 2 prior stopped +initializers) retain image/start/restart values; zero scoped test containers remain. +Cleanup: removed exactly rc1 digests 3566a6e3... and acf5fd06... after zero-reference +checks; removed the task's 695 MiB Cargo target (root-owned build files required +a disposable cleanup container bound only to that directory). Active digests and +named rollbacks untouched; no volume/network/shared data deletion. Free disk +100953247744 -> 101003780096 bytes; shared-host delta is NOT +a precise attribution of bytes reclaimed. BuildKit cache was inventoried, not +broad-pruned: ownership-safe exact cache cleanup remains for Phase C closure. +Retain new rc2 images specifically for this active handoff, not as indefinite +archive. Canonical /home/bobby/data_layer main5bf5cd5 and owner plan hunks untouched; +only active feature worktree modified. Stable published line remains v2.2.2. +No push/merge. Next: Claude/owner review this packet and pending measurements, +complete TS gates, then approve Phase C; do not silently promote this receipt. diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md new file mode 100644 index 00000000..447c33fb --- /dev/null +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -0,0 +1,122 @@ +# P18.4 Data Layer Phase C Packet + +Status: DRAFT_NOT_AUTHORIZED. Source/local candidate only, not a release certificate. +Phase B handover and journal: DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md, final +P18.4 Phase B entries. Architecture: DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md +section 18.13. Candidate application source: `43bbcf0209f49e9cc1af6802af8f8a207a53ee3c`. +Evidence root: `/home/bobby/.local/state/qdl-v2/p184-phaseb-20261007/evidence/`. +The JSON packet there contains exact container names, config fingerprints and +rollback digests, without environment secrets. No Phase C runtime or binding-mount action has run. + +## Blocking prerequisites + +1. TS owner completes DR01-DR08 and broker delta, with new DL findings consolidated + here before a single release. Do not substitute prior E09 or synthetic tests. +2. IDR-062 candidate performance remains unmeasured: paired baseline/candidate + on identical approved provider capture/load; change only top materialization. + Record BOOK_SNAPSHOT messages/s and bytes/s, Kafka input/lag, and CPU/RSS/throttle + for core/query/stream/projector plus whole DL stack. <=5 steady-state vCPU is a + measured budget, not authority to lower existing container quotas. TS measures + its writer CPU. Keep all quotas, buffers and freshness gates unchanged. +3. TS coherence >=39/40 sustained, with sample counts and rejected exact-view + reasons; report BOOK p95 versus existing 2000 ms package cap and paired baseline. + Keep exact book-top/QUOTE equality. Independent channels may still diverge; + faster materialization alone is not proof this gate passes. +4. Owner approves exact runtime packet, role names/digests, maintenance/refusal + window and rollback. Handoff says 13 roles; read-only inventory finds **12**. + No inferred thirteenth role, V1 replacement or state initializer recreation. + +## Images and scope + +One Rust and one Python image from the same application SHA. Both builds completed. Python packaged checks and native binary/linkage checks +passed; neither image has been applied to a runtime role. + +- Python: `sha256:77682eea5054ae9ddbd858323b083286669a55eeb4bc7bc065b04bae8854e32f` +- Rust: `sha256:69e3b2b214331a7ebb6a110dd8159c899bbbd6439aaf87293c8c68d145e3713d` + +| Existing role | Candidate | Retained rollback image | +| --- | --- | --- | +| `ingestor_binance_usdm` | Rust | `sha256:0f6876e16e51600419e1f172aae596bed9c2d369d1afcef93d1da30c18f262ec` | +| `query_kn_1` | Python | `sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9` | +| `query_kn_2` | Python | `sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9` | +| `market_projector_1` | Rust | `sha256:a696cfba6ab383173607f96413c814f74c7abcc98a4c41f57c9c6e78a41bb70b` | +| `market_projector_2` | Rust | `sha256:a696cfba6ab383173607f96413c814f74c7abcc98a4c41f57c9c6e78a41bb70b` | +| `stream_kn_1` | Rust | `sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9` | +| `stream_kn_2` | Rust | `sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9` | +| `ingestor_okx_swap` | Rust | `sha256:0f6876e16e51600419e1f172aae596bed9c2d369d1afcef93d1da30c18f262ec` | +| `rust_core` | Rust | `sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023` | +| `rust_core_2` | Rust | `sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023` | +| `rust_core_3` | Rust | `sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023` | +| `binance_bar_edge` | Python | `sha256:f7351c3bda080a6d1266d49480dc65d7e590375f178252f858f87a04b1a4d685` | + +Keep original command/entrypoint, user, mounts, network aliases, TLS, healthcheck, +restart policy, CPU/RAM and source/group/config identities. Compare normalized +Docker config against recorded fingerprints immediately before action; stop on +unexpected drift. Never use a default source-tree Compose to recreate KN roles. +`docker start` only restarts an existing stopped container; it does not change its +image. The approved future replacement must be rendered from actual private +runtime configuration, preserving these fields, not reconstructed by guessing. + +## Binding V3 order + +The current shared TS writer r59 image is `9150385e...`; its TS binding reader 2.1.0 +is distinct from this release's qdl-sdk 2.1.0. Reverify supported V1/V2/V3 parsing +on every actual caller before a mount change. No change to TS source in this packet. + +Use `trading-system.paper.stable-r22-v3.json` in the evidence root, not the generic +r27 renderer output. Both contain 60 equal products, but r27 changes release and +inventory hashes; it is **not approved for the existing writer**. The r22 V3 file +only changes schema/checksum and adds 60 manifest-derived request policies: + +- release revision 22; consumer manifest/JWT revision **10 unchanged**; +- universal hash `b4e43688ef714e6cef28c0bf2e565b0fbd7b5c70703c9271e1189a9c5bf99afe`; +- V3 binding hash `a7a4380581a923223bf40a067fd4f718c2ccf0b23705abf216abc38f86de84bd`; +- V3 file hash `4bdaa49410190948a8d15ad07ffae5bd73ccb48cc327fd2936b36b815daba5dc`. + +Sequence: confirm readers first -> stage immutable V3 file externally -> owner/TS +mount switch in its approved window -> same-identity no-order refresh proof -> +resume qualification. Preserve original V2 file unchanged for rollback. Do not +rewrite a bind-mounted live file in place or assume all consumers share this file. +No consumer manifest edit, JWT regeneration, global hash update or new entitlement. + +## Proposed bounded rollout, after approval only + +Preflight ISR/quorum, cache generations/checkpoints, actual consumer quality, +TLS expiry/SAN, image digests, supported bindings/cursors and per-role config diff. +Freeze pre-action inventory and typed refusals; keep active images as named rollback. + +Update existing readers one replica at a time (Query then Stream); then the two +projectors separately, three cores separately, ingestors one venue at a time, and +BAR edge last. Between every role verify assignment, lag progression, first cache +apply and actual eligible consumer view, not process health alone. Stop on loss of +ownership, gap/generation/identity mismatch, internal refusal or unbounded lag. +Do not roll the next role merely because Kafka ISR recovered. This is a bounded +maintenance handoff, not a zero-downtime or independent-HA certificate. + +No shared-runtime load or restart in Phase B. Phase C inherits unchanged evidence; +run only packaging/affected protocol and consumer gates. Do not repeat whole C2 +or the catalogue to discover errors. Real-provider quiet/stale refusals remain +separate from internal defects; no timestamp rewrite, hidden fallback or SLA lift. + +## Rollback and non-changes + +Stop the failing replacement before restoring its exact recorded prior digest and +configuration. Restore V2 binding before rolling a reader back if it lacks V3; +readers already supporting V3 may keep the old compatible file. Preserve Kafka +committed offsets, topics/ACL, state generations, cache, TLS and credentials. +No flush, reset, repair publication, DB changes or deleted operational volumes. +Check bounded catch-up and consumer usable view on return; old image health alone +is insufficient. D1 remains deferred: quiet Binance BOOK may legitimately go stale. +D3 remains separate: no independent ingestor/bar-edge/Redis/host HA claim. + +## Publication and cleanup + +After runtime acceptance and explicit approval: feature -> dev CI -> main release, +immutable tag/assets with source/image/config/evidence and supported SDK/binding +versions. Never mutate v2.2.2. These rc2 images are not published stable artifacts. +Remove only the two superseded rc1 image digests after replacement checks and no +container reference; retain active, candidate and named rollback. Test containers +use --rm; remove this task's build target after preserving logs. No broad prune, +shared cache cleanup, volume/network deletion or canonical/user-hunk modification. +Record disk pre/post and running-role image/start/restart comparison. Feature +worktree remains active until reviewed/merged; no worktree cleanup before that. From 5a261e0d60f303502955e52b7fd37527ff0e4b5f Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 13:03:26 +0000 Subject: [PATCH 009/147] test(p184): measure paired L2 materialization on existing core images --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 62 +++ scripts/measure_l2_top_materialization.py | 422 ++++++++++++++++++ scripts/phase114_l2_real_provider_capture.py | 104 ++++- ...test_l2_top_materialization_measurement.py | 91 ++++ upgrade/evidence/p184-idr062-offline-pair.md | 127 ++++++ 5 files changed, 803 insertions(+), 3 deletions(-) create mode 100644 scripts/measure_l2_top_materialization.py create mode 100644 tests/test_l2_top_materialization_measurement.py create mode 100644 upgrade/evidence/p184-idr062-offline-pair.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 074f2182..e8482866 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69179,3 +69179,65 @@ archive. Canonical /home/bobby/data_layer main5bf5cd5 and owner plan hunks untou only active feature worktree modified. Stable published line remains v2.2.2. No push/merge. Next: Claude/owner review this packet and pending measurements, complete TS gates, then approve Phase C; do not silently promote this receipt. + + +##### IDR-062 paired offline measurement - approved follow-up start + +Status: IN_PROGRESS (2026-10-07). Same worktree/branch, no runtime mutation, +no push/merge. Owner scope: capture active public ETH BOOK on Binance/OKX using +phase114_l2_real_provider_capture.py; replay identical bytes/config/resource limits +through the deployed baseline image and rc2-43bbcf0. No new image builds. Existing +core executable requires Kafka: use disposable --rm broker/core/client containers +on one internal test network, fresh test TLS and tmpfs state, no production mounts +or credentials. This is offline replay, not production/consumer latency evidence. +Guide: architecture review section 18.13; packet upgrade/evidence/p184-dl-phase-c-packet.md. +Record snapshot msg/s and serialized bytes/s over captured provider duration; +core CPU/RSS over paced replay, top-change-to-matching-snapshot delay in ms +(p50/p95/max/n), superseded/unmatched changes and sequence/quarantine outcomes. +Only downstream workload ratios are estimates, not measured whole-stack CPU. +Activity must be demonstrated by actual top changes; no claim about market stress +from an arbitrary short capture. Preserve compact hashes/metrics, delete this +run's raw capture and disposable resources after measurement. Retain existing +active/candidate/rollback images. Tests cover metric censoring and capture bounds. +Second slice: move coherence to POST-core-rollout acceptance (>=117/120 per venue), +explicit CPU/lag rollback and core-first order. Runtime execution remains held. +Rollback for this offline slice is stop/remove exact disposable resources only. +apply_patch helper failed (mountinfo path); workspace rule 32 exact edits used. + + +##### IDR-062 offline pair completed (Codex, 2026-10-07) + +Status: PAIRED_OFFLINE_MEASURED_NOT_RUNTIME_CERTIFIED. Source/image/runtime unchanged; +measurement tools and this journal only. Detailed immutable-image results, metric +semantics, invalid harness attempts and cleanup: +[IDR-062 paired report](upgrade/evidence/p184-idr062-offline-pair.md). +Existing baseline 658a9570 vs rc2 69e3b2b2 on identical capture/config, one core per +run, 1 CPU / 256 MiB, fresh internal Kafka --rm and tmpfs. No shared network/state. +Capture started 12:52:17.878 UTC, 592 Binance ETH frames / 601 OKX ETH frames, +44 / 41 actual top changes. Delta payload chains match (587 / 600 deltas); no +post-bootstrap quarantine. One expected Binance bootstrap resnapshot advisory +per core retained separately. Snapshot counts 59->87 and 61->87; msg/s .9832->1.4498 +and 1.0167->1.4500; protobuf bytes/s 7807.9->11509.5 and 8329.5->11877.4. +Top-change -> next verified snapshot on provider clock: Binance baseline +p50/p95/max 510/922/924 ms n43 plus 1 censored; OKX 500/900/900 ms n41. +Candidate 0/0/0 ms n44/n41 (same-event materialization, NOT zero consumer latency). +Exact-top and superseded-change denominators remain in report, no p99 claim. +Core process CPU observed 3.60->3.76 s / 60 s (.0600->.0627 vCPU), observed RSS HWM +19324->19612 KiB, 101 samples each. One short pair, not peak load or all-bindings +certification. Combined snapshot workload estimate x1.45; combined BOOK messages +x1.0413. No measured downstream CPU/consumer latency claim. +Tests: existing Python candidate, network none, unittest modules +`tests.test_l2_top_materialization_measurement tests.test_phase114_l2_provider_capture`: +17 PASS, zero fail/skip. Initial pytest command unavailable in runtime image; +used the suites' actual standard-library unittest runner. Four invalid/incomplete +harness attempts kept as such; bounded bootstrap advisory classification and +container auto-remove race fixes were tested, no production source changes. +Evidence run5/paired-receipt.json SHA256 b9d14de49256e7fb15b80faa430c52a610f18d5aa83c1be6dfc88f85e4de4ecc; +capture SHA256 3afa271cb5f1e99c84ebb68a90bdd1719b740c39c1bca7826f53c4727868d4a9, +root ~/.local/state/qdl-v2/idr062-paired-20261007/. Captures/canonical raw dumps, +fresh test keys, containers and internal network deleted; runtime image/start/ +restart inventory unchanged. Disk free 100669689856->100420591616 bytes (shared-host +writes, not a reclaimed-byte claim). No image built or deleted; active/rc2/rollback +retained. Shared cache inventoried 16.83 GB, not broad-pruned. Canonical main5bf5cd5 +and owner hunk untouched; published v2.2.2 unchanged. No push/merge, no TS edit. +Next separate docs slice: core-first Phase C packet, post-core coherence gate. diff --git a/scripts/measure_l2_top_materialization.py b/scripts/measure_l2_top_materialization.py new file mode 100644 index 00000000..4c57f1a2 --- /dev/null +++ b/scripts/measure_l2_top_materialization.py @@ -0,0 +1,422 @@ +#!/usr/bin/env python3 +"""Paired real-capture replay through existing core images, isolated Kafka only. + +No builds or shared writes. Raw captures/output are temporary and deleted after +hashing. Message rates use provider-window seconds, not accelerated replay time. +""" +from __future__ import annotations +import argparse +import asyncio +import base64 +from decimal import Decimal +import hashlib +import json +import math +import os +from pathlib import Path +import shutil +import subprocess +import threading +import time + +BASELINE = "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023" +CANDIDATE = "sha256:69e3b2b214331a7ebb6a110dd8159c899bbbd6439aaf87293c8c68d145e3713d" +PYTHON = "sha256:77682eea5054ae9ddbd858323b083286669a55eeb4bc7bc065b04bae8854e32f" +KAFKA = "apache/kafka@sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9" +ROOT = Path(__file__).resolve().parents[1] +PREFIX = "qdl-idr062-pair" + + +def digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def write(path, value): + Path(path).write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def distribution(values): + values = sorted(values) + if not values: + return {"n": 0, "unit": "ms", "p50": None, "p95": None, "max": None} + return {"n": len(values), "unit": "ms", "p50": values[math.ceil(.5 * len(values))-1], + "p95": values[math.ceil(.95 * len(values))-1], "max": values[-1]} + + +def top_observations(capture): + """Provider sequence oracle, no approximation of the core publication policy.""" + bids, asks = {}, {} + sequence = None + buffered = [] + observations = [] + venue = capture["binding"]["venue"] + def apply(levels, side): + for level in levels: + price, qty = Decimal(level[0]), Decimal(level[1]) + if qty == 0: + side.pop(price, None) + else: + side[price] = qty + def emit(seq, stamp): + if not bids or not asks or max(bids) >= min(asks): + raise ValueError("oracle empty/crossed book") + observations.append({"seq": str(seq), "ms": stamp, + "top": (str(max(bids).normalize()), str(min(asks).normalize()))}) + def delta(frame): + nonlocal sequence + end = int(frame["u"]) + if end <= sequence: + return + if not observations: + if not (int(frame["U"]) <= sequence + 1 <= end or int(frame["pu"]) == sequence): + raise ValueError("oracle Binance bridge gap") + elif int(frame["pu"]) != sequence: + raise ValueError("oracle Binance sequence gap") + apply(frame["b"], bids); apply(frame["a"], asks) + sequence = end + emit(sequence, int(frame["E"])) + for row in capture["rows"]: + frame = json.loads(row["raw"]) + if venue == "BINANCE": + if row["protocol"] == "HTTP": + bids.clear(); asks.clear() + apply(frame["bids"], bids); apply(frame["asks"], asks) + sequence = int(frame["lastUpdateId"]) + for item in buffered: + delta(item) + buffered.clear() + elif sequence is None: + buffered.append(frame) + else: + delta(frame) + else: + for item in frame["data"]: + if frame["action"] == "snapshot": + bids.clear(); asks.clear() + elif int(item["prevSeqId"]) != sequence: + raise ValueError("oracle OKX sequence gap") + apply(item["bids"], bids); apply(item["asks"], asks) + sequence = int(item["seqId"]) + emit(sequence, int(item["ts"])) + return observations + + +def top_delay(observations, snapshots): + """Matching top at its own source sequence; count all superseded intervals.""" + by_seq = {r["seq"]: r for r in observations} + emitted = {} + for snap in snapshots: + row = by_seq.get(snap["seq"]) + if row is None or tuple(snap["top"]) != tuple(row["top"]): + raise ValueError("snapshot differs from exact provider top/sequence") + emitted[(snap["seq"], snap.get("ms", row["ms"]))] = snap + delays, pending, last = [], None, None + coverage_delays, uncovered = [], [] + changed = superseded = 0 + for row in observations: + if last is not None and row["top"] != last: + changed += 1 + uncovered.append(row["ms"]) + if pending is not None: + superseded += 1 + pending = row["ms"] + if (row["seq"], row["ms"]) in emitted: + coverage_delays.extend(row["ms"] - stamp for stamp in uncovered) + uncovered.clear() + if pending is not None and (row["seq"], row["ms"]) in emitted: + delay = row["ms"] - pending + if delay < 0: + raise ValueError("provider clock regressed") + delays.append(delay) + pending = None + last = row["top"] + return {"matched_delay": distribution(delays), "top_changes": changed, + "next_verified_snapshot_delay": distribution(coverage_delays), + "next_snapshot_uncovered_tail": len(uncovered), + "coverage_semantics": "first snapshot at same or later sequence; superseded tops coalesce, not exact historical price delivery", + "superseded_before_snapshot": superseded, "unmatched_tail": int(pending is not None), + "denominator_check": changed == len(delays) + superseded + int(pending is not None), + "scope": "provider event time to snapshot materialization source time, NOT wall-clock consumer latency"} + + +def client_capture(work, seconds): + from scripts.phase114_l2_real_provider_capture import capture_replay_window + cfg = json.loads((work / "core.json").read_text()) + result = asyncio.run(capture_replay_window(cfg["core"]["bindings"], seconds)) + for cap in result["captures"]: + if len(top_observations(cap)) < 2: + raise ValueError("no continuous oracle window") + write(work / "capture.json", result) + + +def client_replay(work): + from confluent_kafka import Producer, Consumer, TopicPartition + from confluent_kafka.admin import AdminClient, NewTopic + from qdl.provider.v1.raw_provider_pb2 import RawProviderEnvelope + from qdl.marketdata.v2.market_data_pb2 import EventEnvelope + security = {"bootstrap.servers": "kafka:9092", "security.protocol": "SSL", + "ssl.ca.location": "/work/ca.crt", "ssl.certificate.location": "/work/client.crt", + "ssl.key.location": "/work/client.key"} + admin = AdminClient(security) + for future in admin.create_topics([NewTopic(n, 1, 1) for n in ("idr.raw", "idr.canonical", "idr.quarantine")]).values(): + future.result(30) + capture = json.loads((work / "capture.json").read_text()) + rows = sorted([(r["received_at_ns"], cap["binding"], r) for cap in capture["captures"] for r in cap["rows"]], key=lambda x:x[0]) + cfg = json.loads((work / "core.json").read_text()); cfg["max_events"] = len(rows); write(work / "core.json", cfg) + (work / "client-ready").touch() + deadline = time.monotonic() + 90 + while not (work / "core-ready").exists(): + if time.monotonic() > deadline: raise TimeoutError("core orchestration") + time.sleep(.1) + producer = Producer(dict(security, **{"enable.idempotence": True, "acks": "all", "linger.ms": 0})) + errors = [] + def delivery(error, _): + if error: errors.append(str(error)) + def publish(): + try: + start = time.monotonic() + for idx, (ns, b, row) in enumerate(rows): + time.sleep(max(0, (ns - rows[0][0])/1e9 - (time.monotonic()-start))) + raw = row["raw"].encode(); sha = hashlib.sha256(raw).digest() + env = RawProviderEnvelope(raw_schema_name="qdl.provider.raw", raw_schema_major=1, + capture_id=hashlib.sha256(str(idx).encode()+sha).digest()[:16], provider=b["provider"], venue=b["venue"], + market=b["market"], product_type=b["product_type"], native_symbol=b["native_symbol"], + native_channel=b["native_channel"], subscription_id=b["source_id"], + source_session_id="idr062-offline-"+b["venue"], connection_generation=1, lease_epoch=1, + authority_revision=1, partition_plan_epoch=1, received_at_ns=ns, + transport_protocol=2 if row["protocol"]=="HTTP" else 1, transport_compression=1, + capture_boundary=2, raw_frame_bytes=raw, raw_frame_sha256=sha, + adapter_version="idr062-real-capture/1", config_revision=1, + instrument_catalog_revision=b["instrument_catalog_revision"], correlation_id=str(idx), test_provenance=False) + producer.produce("idr.raw", partition=0, key=b["source_id"].encode(), value=env.SerializeToString(), headers=[("qdl-event-id", env.capture_id)], callback=delivery) + producer.poll(0) + if producer.flush(15): errors.append("producer flush incomplete") + except Exception as exc: + errors.append(str(exc)) + worker = threading.Thread(target=publish); worker.start() + consumer = Consumer(dict(security, **{"group.id":"idr062-observer", "enable.auto.commit":False, + "isolation.level":"read_committed", "auto.offset.reset":"earliest"})) + consumer.assign([TopicPartition("idr.canonical",0,0), TopicPartition("idr.quarantine",0,0)]) + output = []; last = time.monotonic(); deadline = last + capture["seconds"] + 75 + try: + while time.monotonic() < deadline: + msg = consumer.poll(.1) + if msg is None: + if not worker.is_alive() and time.monotonic()-last > 8: break + continue + if msg.error(): raise RuntimeError(msg.error()) + last = time.monotonic() + output.append({"topic":msg.topic(), "bytes":base64.b64encode(msg.value()).decode()}) + finally: + worker.join(20); consumer.close() + if worker.is_alive() or errors: raise RuntimeError(errors or "publisher alive") + write(work / "output.json", output) + + +def analyse(work): + from qdl.marketdata.v2.market_data_pb2 import EventEnvelope + capture = json.loads((work/"capture.json").read_text()) + output = json.loads((work/"output.json").read_text()) + rows = {} + quarantines = [] + for item in output: + if item["topic"] != "idr.canonical": + from qdl.provider.v1.raw_provider_pb2 import QuarantineRecord, QUARANTINE_REASON_SEQUENCE_GAP + quarantines.append(QuarantineRecord.FromString(base64.b64decode(item["bytes"]))) + continue + payload = base64.b64decode(item["bytes"]); event = EventEnvelope.FromString(payload) + key = event.venue+"/"+event.native_symbol + entry = rows.setdefault(key, {"snapshots":[], "snapshot_bytes":0, "deltas":0, "delta_hashes":[]}) + kind = event.WhichOneof("payload") + if kind == "book_snapshot": + snap = event.book_snapshot + if not snap.sequence_verified: raise ValueError("unverified snapshot") + levels = {1:[], 2:[]} + for level in snap.levels: + p = level.price + value = Decimal(p.mantissa_text if p.WhichOneof("coefficient")=="mantissa_text" else p.mantissa).scaleb(-p.scale) + levels[level.side].append(value) + entry.setdefault("bootstrap_received_at_ns", event.received_at_ns) + entry["snapshots"].append({"seq":snap.native_sequence,"ms":event.source_event_time_ns//1000000, + "top":[str(max(levels[1]).normalize()), str(min(levels[2]).normalize())]}) + entry["snapshot_bytes"] += len(payload) + elif kind == "book_delta": + entry["deltas"] += 1 + entry["delta_hashes"].append(hashlib.sha256(event.book_delta.SerializeToString()).hexdigest()) + for q in quarantines: + key = q.raw.venue+"/"+q.raw.native_symbol + entry = rows[key] + if q.reason != QUARANTINE_REASON_SEQUENCE_GAP or q.safe_summary != "L2_RESNAPSHOT_REQUIRED_V1" or q.raw.received_at_ns > entry["bootstrap_received_at_ns"]: + raise ValueError(f"non-bootstrap quarantine: {q.reason} {q.safe_summary}, raw={q.raw.received_at_ns}, bootstrap={entry['bootstrap_received_at_ns']}") + entry["bootstrap_resnapshot_requests"] = entry.get("bootstrap_resnapshot_requests", 0) + 1 + for cap in capture["captures"]: + key = cap["binding"]["venue"]+"/"+cap["binding"]["native_symbol"] + entry = rows[key]; obs = top_observations(cap) + elapsed = (obs[-1]["ms"]-obs[0]["ms"])/1000 + if elapsed <= 0: raise ValueError("empty provider duration") + entry["provider_seconds"] = elapsed + entry["captured_frames"] = len(cap["rows"]) + entry["provider_observations"] = len(obs) + entry["received_start_ns"] = cap["rows"][0]["received_at_ns"] + entry["received_end_ns"] = cap["rows"][-1]["received_at_ns"] + entry["snapshot_count"] = len(entry["snapshots"]) + entry["snapshot_messages_per_second"] = entry["snapshot_count"]/elapsed + entry["snapshot_protobuf_bytes_per_second"] = entry["snapshot_bytes"]/elapsed + entry["top_delay"] = top_delay(obs, entry.pop("snapshots")) + entry["delta_payload_chain_sha256"] = hashlib.sha256("".join(entry.pop("delta_hashes")).encode()).hexdigest() + write(work/"metrics.json", rows) + + +def command(args, **kwargs): + return subprocess.check_output(args, text=True, **kwargs).strip() + + +def stop_disposable(name): + if not name.startswith(PREFIX + "-"): + raise ValueError("refusing non-test container") + subprocess.run(["docker", "stop", "-t", "3", name], capture_output=True) + deadline = time.monotonic() + 20 + while subprocess.run(["docker", "inspect", name], capture_output=True).returncode == 0: + if time.monotonic() > deadline: + raise TimeoutError("disposable auto-remove did not finish: " + name) + time.sleep(.2) + + +def orchestrate(work, seconds): + work.mkdir(parents=True, exist_ok=False); work.chmod(0o755) + runtime = json.loads(command(["docker","inspect","qdl_v2_stable_candidate-rust_core-1"]))[0] + if runtime["Image"] != BASELINE: raise ValueError("deployed baseline drift") + cfg = json.loads(command(["docker","exec",runtime["Id"],"cat","/runtime/core.json"])) + cfg["core"]["bindings"] = [b for b in cfg["core"]["bindings"] if b.get("l2") and b["native_symbol"] in {"ETHUSDT","ETH-USDT-SWAP"}] + if len(cfg["core"]["bindings"]) != 2: raise ValueError("expected two active ETH BOOK bindings") + cfg["core"].update(canonical_stream="idr.canonical", quarantine_stream="idr.quarantine") + cfg.update(raw_topics=["idr.raw"], transactional_id="idr062-core", shard_id="idr062-core", metrics_every_batches=10) + cfg["authority"].update(mode="RUST_SHADOW",slice_id="idr062-offline",approved_by="BobbyAxerol offline replay") + write(work/"core.json",cfg) + before = {x["Name"]:{"image":x["Image"],"started":x["State"]["StartedAt"],"restarts":x["RestartCount"]} + for x in json.loads(command(["docker","inspect",*command(["docker","ps","-aq","--filter","name=qdl_v2_stable_candidate-"]).splitlines()]))} + write(work/"runtime-before.json",before) + disk_before = shutil.disk_usage(work).free + def client(mode, network, detach=False): + args=["docker","run","--rm","--name",PREFIX+("-client" if mode=="replay" else "-"+mode),"--user",str(os.getuid()),"--cpus","1","--memory","512m", + "--network",network,"--tmpfs","/tmp:rw,exec,size=32m","--tmpfs","/app/logs:rw,uid=1001", + "-e","PYTHONDONTWRITEBYTECODE=1","-e","PYTHONPATH=/src:/src/generated/python", + "-v",str(ROOT)+":/src:ro","-v",str(work)+":/work","-w","/src","--entrypoint","python"] + if detach: args.insert(3,"-d") + return command(args+[PYTHON,"-B","scripts/measure_l2_top_materialization.py",mode,"--work","/work","--seconds",str(seconds)]) + client("capture","bridge") + capture_hash=digest(work/"capture.json") + # Fresh self-signed test identity only; never read production TLS. + subprocess.run(["openssl","req","-x509","-newkey","rsa:2048","-nodes","-days","1","-subj","/CN=kafka", + "-addext","subjectAltName=DNS:kafka","-keyout",str(work/"client.key"),"-out",str(work/"ca.crt")],check=True,capture_output=True) + shutil.copyfile(work/"ca.crt",work/"client.crt") + (work/"client.key").chmod(0o644) + subprocess.run(["openssl","pkcs12","-export","-in",str(work/"ca.crt"),"-inkey",str(work/"client.key"), + "-out",str(work/"server.p12"),"-passout","pass:idr062-test-only"],check=True,capture_output=True) + (work/"server.p12").chmod(0o644) + properties = """process.roles=broker,controller +node.id=1 +controller.quorum.voters=1@kafka:9093 +listeners=SSL://:9092,CONTROLLER://:9093 +advertised.listeners=SSL://kafka:9092 +controller.listener.names=CONTROLLER +listener.security.protocol.map=SSL:SSL,CONTROLLER:PLAINTEXT +inter.broker.listener.name=SSL +ssl.keystore.type=PKCS12 +ssl.keystore.location=/work/server.p12 +ssl.keystore.password=idr062-test-only +ssl.key.password=idr062-test-only +ssl.truststore.type=PEM +ssl.truststore.location=/work/ca.crt +ssl.client.auth=required +log.dirs=/data +num.partitions=1 +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 +group.initial.rebalance.delay.ms=0 +""" + (work/"broker.properties").write_text(properties) + result={"capture_sha256":capture_hash,"source":command(["git","-C",str(ROOT),"rev-parse","HEAD"]), + "baseline":BASELINE,"candidate":CANDIDATE,"runs":{},"resource_scope":"core container only, 1 CPU / 256 MiB, paced same capture"} + command(["docker","network","create","--internal",PREFIX]) + try: + for label,image in [("baseline",BASELINE),("candidate",CANDIDATE)]: + for f in ["client-ready","core-ready"]: (work/f).unlink(missing_ok=True) + broker=PREFIX+"-kafka";core=PREFIX+"-core" + command(["docker","run","--rm","-d","--name",broker,"--network",PREFIX,"--network-alias","kafka", + "--cpus","1","--memory","1g","--tmpfs","/data:rw,size=512m,mode=1777", + "-e","KAFKA_HEAP_OPTS=-Xms256m -Xmx512m","-v",str(work)+":/work:ro","--entrypoint","/bin/sh",KAFKA,"-c", + "/opt/kafka/bin/kafka-storage.sh format -t MkU3OEVBNTcwNTJENDM2Qk -c /work/broker.properties && exec /opt/kafka/bin/kafka-server-start.sh /work/broker.properties"]) + try: + time.sleep(12) + client("replay",PREFIX,True) + end=time.monotonic()+45 + while not (work/"client-ready").exists(): + if time.monotonic()>end: raise TimeoutError("replay client ready") + time.sleep(.2) + args=["docker","run","--rm","-d","--name",core,"--network",PREFIX,"--cpus","1","--memory","256m", + "-v",str(work)+":/work:ro","--entrypoint","/bin/sh"] + for k,v in {"QDL_KAFKA_CERT_ROOT":"/work","QDL_KAFKA_BOOTSTRAP_SERVERS":"kafka:9092", + "QDL_KAFKA_CLIENT_ID":"idr062-core","QDL_KAFKA_GROUP_ID":"idr062-core"}.items(): args += ["-e",k+"="+v] + command(args+[image,"-c","/usr/local/bin/qdl-realtime-core /work/core.json; code=$?; echo IDR062_EXIT=$code; sleep 45; exit $code"]) + time.sleep(2);(work/"core-ready").touch() + samples=[];end=time.monotonic()+seconds+65 + while time.monotonic() OkxRepla return replay +async def capture_replay_window(bindings: Sequence[Mapping[str, Any]], seconds: float) -> dict[str, Any]: + """Opt-in bounded raw capture for offline replay; caller must hash then delete it. + + Reuses the public protocol/continuity validators. No production client state, + credentials or Kafka connection is used. Original provider text is retained. + """ + if not 10 <= seconds <= 120 or not 1 <= len(bindings) <= 12: + raise ValueError("replay window must be 10..120 seconds and 1..12 bindings") + + async def one(binding): + venue, symbol = binding["venue"], binding["native_symbol"] + rows = [] + captured_bytes = 0 + deadline = time.monotonic() + seconds + 15 + def add(raw, protocol): + nonlocal captured_bytes + captured_bytes += len(raw.encode("utf-8")) + if len(rows) >= 10000 or captured_bytes > 64 * 1024 * 1024: + raise ProviderCaptureError("replay capture bound exceeded") + rows.append({"raw": raw, "protocol": protocol, "received_at_ns": time.time_ns()}) + if venue == "BINANCE": + async with connect(BINANCE_WS, open_timeout=8, close_timeout=4, ping_interval=10) as socket: + await socket.send(json.dumps({"method": "SUBSCRIBE", "params": [symbol.lower() + "@depth@100ms"], "id": 114})) + while not rows: + raw, frame = await _recv_json(socket, deadline) + if frame.get("e") == "depthUpdate" and frame.get("s") == symbol: + add(raw, "WS") + async def snapshot(): + async with httpx.AsyncClient(timeout=8) as client: + response = await client.get(BINANCE_DEPTH, params={"symbol": symbol, "limit": 1000}) + response.raise_for_status() + add(response.text, "HTTP") + return response.json() + task = asyncio.create_task(snapshot()) + try: + while not task.done(): + try: + raw, frame = await _recv_json(socket, min(deadline, time.monotonic() + .25)) + except TimeoutError: + continue + if frame.get("e") == "depthUpdate" and frame.get("s") == symbol: + add(raw, "WS") + anchor = await task + stop = time.monotonic() + seconds + while time.monotonic() < stop: + try: + raw, frame = await _recv_json(socket, stop) + except (TimeoutError, ProviderCaptureError): + break + if frame.get("e") == "depthUpdate" and frame.get("s") == symbol: + add(raw, "WS") + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + ws = [r["raw"] for r in rows if r["protocol"] == "WS"] + result = validate_binance_replay(symbol=symbol, snapshot_sequence=int(anchor["lastUpdateId"]), + frames=[json.loads(r) for r in ws], raw_frames=ws, min_chain_frames=2) + elif venue == "OKX": + async with connect(OKX_WS, open_timeout=8, close_timeout=4, ping_interval=10) as socket: + await socket.send(json.dumps({"op": "subscribe", "args": [{"channel": "books", "instId": symbol}]})) + stop = None + while stop is None or time.monotonic() < stop: + try: + raw, frame = await _recv_json(socket, stop or deadline) + except TimeoutError: + if stop is None: + raise ProviderCaptureError("OKX initial snapshot timed out") + break + if frame.get("arg", {}).get("instId") != symbol or frame.get("action") not in {"snapshot", "update"}: + continue + if stop is None: + if frame["action"] != "snapshot": + raise ProviderCaptureError("OKX update before snapshot") + stop = time.monotonic() + seconds + add(raw, "WS") + raw_frames = [r["raw"] for r in rows] + # Existing validator stops at the first proven update; validate every + # subsequent sequence too, not just the bootstrap prefix. + result = validate_okx_replay(symbol=symbol, frames=[json.loads(r) for r in raw_frames], raw_frames=raw_frames) + previous = None + for raw in raw_frames: + frame = json.loads(raw) + for row in frame["data"]: + seq, prev = int(row["seqId"]), int(row["prevSeqId"]) + if frame["action"] != "snapshot" and prev != previous: + raise ProviderCaptureError("OKX window sequence gap/reset") + previous = seq + else: + raise ProviderCaptureError("unsupported replay venue") + if result is None: + raise ProviderCaptureError("replay window has no verified bridge") + return {"binding": dict(binding), "rows": rows, "bootstrap": asdict(result)} + return {"schema": "qdl.l2-offline-capture.v1", "seconds": seconds, + "provenance": "REAL_PUBLIC_PROVIDER_BYTES_OFFLINE_ONLY", + "captures": list(await asyncio.gather(*(one(b) for b in bindings)))} + + async def _capture_okx_symbols( symbols: Sequence[str], deadline: float, diff --git a/tests/test_l2_top_materialization_measurement.py b/tests/test_l2_top_materialization_measurement.py new file mode 100644 index 00000000..49dc8a42 --- /dev/null +++ b/tests/test_l2_top_materialization_measurement.py @@ -0,0 +1,91 @@ +"""Offline measurement must not hide censored changes or invent matching tops.""" +import asyncio +import unittest +from scripts.measure_l2_top_materialization import distribution, top_delay, top_observations +from scripts.phase114_l2_real_provider_capture import capture_replay_window + +class MeasurementTests(unittest.TestCase): + def test_censored_changes_stay_in_denominator(self): + rows=[{"seq":str(i),"ms":i*100,"top":top} for i,top in enumerate([("1","3"),("2","3"),("2","4"),("2","4"),("1","4")])] + result=top_delay(rows,[{"seq":"3","top":("2","4")}]) + self.assertEqual(result["top_changes"],3) + self.assertEqual(result["superseded_before_snapshot"],1) + self.assertEqual(result["unmatched_tail"],1) + self.assertEqual(result["matched_delay"]["p50"],100) + self.assertTrue(result["denominator_check"]) + self.assertEqual(result["next_verified_snapshot_delay"]["n"],2) + self.assertEqual(result["next_verified_snapshot_delay"]["max"],200) + self.assertEqual(result["next_snapshot_uncovered_tail"],1) + + def test_snapshot_must_match_its_sequence(self): + with self.assertRaises(ValueError): + top_delay([{"seq":"1","ms":1,"top":("1","2")}],[{"seq":"2","top":("1","2")}]) + with self.assertRaises(ValueError): + top_delay([{"seq":"1","ms":1,"top":("1","2")}],[{"seq":"1","top":("0","2")}]) + + def test_keepalive_sequence_does_not_backdate_later_snapshot(self): + rows=[{"seq":"1","ms":0,"top":("1","3")}, + {"seq":"2","ms":100,"top":("2","3")}, + {"seq":"2","ms":1000,"top":("2","3")}] + r=top_delay(rows,[{"seq":"2","ms":1000,"top":("2","3")}]) + self.assertEqual(r["next_verified_snapshot_delay"]["p50"],900) + + def test_no_samples_not_zero_latency(self): + self.assertIsNone(distribution([])["p95"]) + self.assertEqual(distribution([1,2,3])["max"],3) + + def test_same_event_snapshot_has_zero_policy_delay(self): + rows=[{"seq":"1","ms":100,"top":("1","3")},{"seq":"2","ms":200,"top":("2","3")}] + self.assertEqual(top_delay(rows,[{"seq":"2","top":("2","3")}])["matched_delay"]["max"],0) + + def test_window_bounds_fail_before_network(self): + for seconds,bindings in [(0,[{}]),(121,[{}]),(60,[]),(60,[{}]*13)]: + with self.assertRaises(ValueError): + asyncio.run(capture_replay_window(bindings,seconds)) + + def test_oracle_rejects_sequence_gap(self): + import json + frames=[{"action":"snapshot","data":[{"seqId":1,"prevSeqId":-1,"ts":"100","bids":[["1","2"]],"asks":[["3","2"]]}]}, + {"action":"update","data":[{"seqId":3,"prevSeqId":2,"ts":"200","bids":[],"asks":[]}]}] + with self.assertRaises(ValueError): + top_observations({"binding":{"venue":"OKX"},"rows":[{"raw":json.dumps(f),"protocol":"WS"} for f in frames]}) + + def test_analysis_classifies_bootstrap_using_protobuf_enum(self): + import base64 + import json + import tempfile + from pathlib import Path + from scripts.measure_l2_top_materialization import analyse + from qdl.provider.v1.raw_provider_pb2 import QuarantineRecord, QUARANTINE_REASON_SEQUENCE_GAP + from qdl.marketdata.v2.market_data_pb2 import EventEnvelope + with tempfile.TemporaryDirectory() as folder: + work=Path(folder) + frames=[]; output=[] + for seq, stamp, bid in [(1,100,1),(2,200,2)]: + frame={"action":"snapshot" if seq==1 else "update","data":[{"seqId":seq,"prevSeqId":seq-1,"ts":str(stamp),"bids":[[str(bid),"1"]],"asks":[["3","1"]]}]} + frames.append({"protocol":"WS","received_at_ns":stamp*1000000,"raw":json.dumps(frame)}) + e=EventEnvelope(venue="OKX",native_symbol="TEST",source_event_time_ns=stamp*1000000,received_at_ns=stamp*1000000) + e.book_snapshot.native_sequence=str(seq);e.book_snapshot.sequence_verified=True + for side, price in [(1,bid),(2,3)]: + level=e.book_snapshot.levels.add(side=side);level.price.mantissa=price + output.append({"topic":"idr.canonical","bytes":base64.b64encode(e.SerializeToString()).decode()}) + q=QuarantineRecord(reason=QUARANTINE_REASON_SEQUENCE_GAP,safe_summary="L2_RESNAPSHOT_REQUIRED_V1") + q.raw.venue="OKX";q.raw.native_symbol="TEST";q.raw.received_at_ns=1 + output.append({"topic":"idr.quarantine","bytes":base64.b64encode(q.SerializeToString()).decode()}) + (work/"capture.json").write_text(json.dumps({"captures":[{"binding":{"venue":"OKX","native_symbol":"TEST"},"rows":frames}]})) + (work/"output.json").write_text(json.dumps(output));analyse(work) + metrics=json.loads((work/"metrics.json").read_text())["OKX/TEST"] + self.assertEqual(metrics["bootstrap_resnapshot_requests"],1) + self.assertEqual(metrics["top_delay"]["next_verified_snapshot_delay"]["n"],1) + q.raw.received_at_ns=300000000 + output[-1]["bytes"]=base64.b64encode(q.SerializeToString()).decode() + (work/"output.json").write_text(json.dumps(output)) + with self.assertRaises(ValueError): analyse(work) + + def test_cleanup_waits_for_auto_remove_and_refuses_shared_names(self): + from unittest.mock import patch, Mock + from scripts.measure_l2_top_materialization import stop_disposable + with self.assertRaises(ValueError): stop_disposable("rust_core") + with patch("scripts.measure_l2_top_materialization.subprocess.run", side_effect=[Mock(returncode=0),Mock(returncode=0),Mock(returncode=1)]) as run, patch("scripts.measure_l2_top_materialization.time.sleep"): + stop_disposable("qdl-idr062-pair-core") + self.assertEqual(run.call_count,3) diff --git a/upgrade/evidence/p184-idr062-offline-pair.md b/upgrade/evidence/p184-idr062-offline-pair.md new file mode 100644 index 00000000..ca872f99 --- /dev/null +++ b/upgrade/evidence/p184-idr062-offline-pair.md @@ -0,0 +1,127 @@ +# IDR-062 Offline Paired Measurement + +Status: PAIRED_OFFLINE_MEASURED_NOT_RUNTIME_CERTIFIED, 2026-10-07. +No shared-runtime mutation, image build, TS edit, push or merge. + +## Provenance and method + +- Existing baseline: `sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023`. +- Existing rc2-43bbcf0: `sha256:69e3b2b214331a7ebb6a110dd8159c899bbbd6439aaf87293c8c68d145e3713d`. +- Public capture begins 2026-10-07 12:52:17.878 UTC, approximately 60 s per venue: + ETHUSDT Binance USD-M (592 frames) and ETH-USDT-SWAP OKX (601 frames). + Bindings copied read-only from the running core, not a reference-symbol guess. +- Capture SHA256: `3afa271cb5f1e99c84ebb68a90bdd1719b740c39c1bca7826f53c4727868d4a9`. +- Both core configs SHA256: `54c0fece8e9535113d8960ede65bedaf7c1d9f02d6c470387d07b2f667d34cbc`. +- Same capture, pacing, config, TLS, partition count and 1 CPU / 256 MiB limit; + only core image changes. Each run uses a fresh disposable Kafka broker on an + internal test network (no host ports, production secrets or shared state). + Core is the unmodified executable packaged in each image, not a Python model. +- Capture API: `scripts/phase114_l2_real_provider_capture.py:capture_replay_window`; + orchestration/oracle: `scripts/measure_l2_top_materialization.py`. +- Raw capture/output files deleted after hashing. Compact evidence retained at + `/home/bobby/.local/state/qdl-v2/idr062-paired-20261007/run5/`. + `paired-receipt.json` SHA256: + `b9d14de49256e7fb15b80faa430c52a610f18d5aa83c1be6dfc88f85e4de4ecc`. + +## Snapshot output + +Includes initial bootstrap snapshots. Rates divide by the actual provider window, +60.008 s Binance / 60.000 s OKX. Bytes are serialized EventEnvelope protobuf bytes, +NOT Kafka framing, TLS, compressed on-disk size or raw provider payload bytes. + +| Venue | Baseline snapshots n | Candidate n | msg/s baseline -> candidate | bytes/s baseline -> candidate | +| --- | ---: | ---: | ---: | ---: | +| Binance ETH | 59 | 87 | 0.9832 -> 1.4498 | 7807.9 -> 11509.5 | +| OKX ETH | 61 | 87 | 1.0167 -> 1.4500 | 8329.5 -> 11877.4 | + +587 Binance and 600 OKX delta payloads have equal sequence-ordered SHA256 chains +between cores. Snapshot top at each source sequence matches an exact Decimal +provider-book oracle. No post-bootstrap gap/refusal/quarantine; each run records +one expected Binance `L2_RESNAPSHOT_REQUIRED_V1` before initial snapshot readiness. +That advisory is counted, not hidden. Four pre-bootstrap frames are buffered, not +lost. No duplicate canonical payload claim is inferred from container health. + +## Top-change delay + +Time from provider top-change event to first verified snapshot at the same or a +later source sequence, in **provider event-time ms**. A later sequence may coalesce +intermediate tops. This isolates publication-policy delay. It is NOT request RTT, +normalization CPU time, Kafka commit latency or consumer usable latency. Zero means +snapshot emitted on the same event, not zero wall-clock processing time. + +| Venue / image | p50 ms | p95 ms | max ms | matched n | right-censored tail | +| --- | ---: | ---: | ---: | ---: | ---: | +| Binance baseline | 510 | 922 | 924 | 43 | 1 | +| Binance candidate | 0 | 0 | 0 | 44 | 0 | +| OKX baseline | 500 | 900 | 900 | 41 | 0 | +| OKX candidate | 0 | 0 | 0 | 41 | 0 | + +Activity: 44 Binance and 41 OKX top changes, so this is not a motionless-book probe. +It is one short public-market window, NOT a high-volatility/peak-load certificate. +Only initial bootstrap is captured; periodic REST renewal, disconnect and resync +faults are not replayed in this measurement (unchanged source tests are inherited). +For the stricter metric requiring that exact top before it is superseded: +Binance baseline p50/p95/max 408/922/924 ms, n=24, 19 superseded, 1 censored; +OKX baseline 300/900/900 ms, n=26, 15 superseded, 0 censored. +Candidate: 0/0/0 ms, n=44 and 41 respectively, none superseded/censored. +No p99 is claimed from these small denominators; quantiles use nearest rank. +A test prevents a later keepalive snapshot with the same sequence from being +backdated onto an earlier update. + +## Core resources and downstream estimates + +Two-binding replay through one core, not three production cores or the entire DL. +Process CPU is sampled from /proc user+system ticks (100 ticks/s); RSS is maximum +observed VmHWM. 101 samples per image, target cadence 0.5 s plus probe overhead; +observed spans 60.642 / 60.835 s. The last process sample can precede exit by one +sample interval. Startup is included; no confidence interval from one pair. + +| Core image | observed CPU s | CPU s / 60 s (vCPU) | observed RSS HWM KiB | +| --- | ---: | ---: | ---: | +| Baseline | 3.60 | 0.0600 | 19324 | +| Candidate | 3.76 | 0.0627 | 19612 | + +Observed process CPU +4.44%, RSS +288 KiB. Container CPU including docker-exec +instrumentation is separately recorded (7.588 / 7.770 s), NOT passed off as core +CPU. Neither measured container was throttled in sampled cgroup counters. + +**Estimates only**, at unchanged workload/fan-out and payload mix: +- Snapshot-only publication work: Binance x1.475, OKX x1.426, combined x1.450. +- Total BOOK message work (snapshot + unchanged delta): Binance x1.0433, + OKX x1.0393, combined 1307 -> 1361 messages, x1.0413. This is the relevant + message-volume estimate for a projector processing both types. +- Stream snapshot-only subscribers see the snapshot multiplier; delta-only + subscribers have unchanged counts. Mixed subscription work depends on fan-out. +- Query RPC demand does not increase merely because core publishes more snapshots. + Cache write/materialization work increases; Query CPU/latency cannot be derived + by multiplying snapshot count. No measured projector/Query/Stream CPU is claimed. +- No extrapolation to all bindings, 5 vCPU whole-stack PASS, 20-50 alpha capacity, + TS coherence or consumer latency. Those require the held post-core runtime gate. + +## Verification, failed harness attempts and cleanup + +17 unittest cases PASS (9 measurement + 8 existing capture), zero fail/skip on the +final run using the existing Python candidate image. Tests cover censored changes, +sequence/top mismatch, keepalive timing, capture bounds, bootstrap-only advisory, +provider continuity and exact test-container auto-remove handling. +Earlier attempts are NOT accepted measurements: run1 invalid 32-byte capture ID +(correct contract is 16) plus client name race; run2 rejected the expected bootstrap +advisory; run3 used the wrong numeric quarantine enum (now imports the protobuf +constant); run4 had valid baseline but failed candidate start on broker auto-remove +race. The final harness waits for auto-remove with a deadline; no production code +or provider rules were weakened. Those runs' compact logs/digests remain. + +All test containers and the internal network removed; all five runs' raw captures, +canonical payload dumps and fresh private test keys removed. No images built or +removed: active/candidate/named rollback digests retained, shared BuildKit cache +not pruned. Docker inventory: 92 images / 28.19 GB; cache 227 entries / 16.83 GB. +Disk free for final run: 100669689856 -> 100420591616 bytes. Concurrent host writes +mean this decrease is NOT test residue attribution or a claimed reclaimed amount. +All inventoried DL image/start/restart fields unchanged. Canonical remains +`/home/bobby/data_layer main@5bf5cd5`, owner plan hunk untouched; feature worktree +`/home/bobby/.worktrees/data-layer-p184-closure`, `fix/p184-dl-closure` retained. +Published release remains v2.2.2. Runtime roles/digests/config fingerprints and +rollback set remain those in the Phase C packet; no binding V3 was mounted. + +Next: review this result and the amended core-first Phase C packet. STOP; no +runtime deployment authority is inferred from this offline result. From 9dbb93c6fc5eff4d02fcdaffafba28bd54caa2fc Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 13:06:30 +0000 Subject: [PATCH 010/147] docs(p184): gate remaining rollout on post-core TS coherence --- CERTIFICATION_LEDGER.md | 16 ++++ DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 33 ++++++++ upgrade/evidence/p184-dl-phase-c-packet.md | 98 ++++++++++++++++------ 3 files changed, 123 insertions(+), 24 deletions(-) diff --git a/CERTIFICATION_LEDGER.md b/CERTIFICATION_LEDGER.md index 5dc8b6a3..7038ca16 100644 --- a/CERTIFICATION_LEDGER.md +++ b/CERTIFICATION_LEDGER.md @@ -2125,3 +2125,19 @@ Inherited Phase A 4eb08dc, no catalogue/C2 rerun. Release waits for real/capture candidate resource/coherence evidence, TS DR01-DR08 and broker delta, and owner approval of the actual 12-role packet (handoff listed 13). D1/D3 remain deferred. Only source/docs, two candidate images and scoped test cleanup completed. + + +### P18.4 IDR-062 offline pair and held core-first packet - 2026-10-07 + +Status: PAIRED_OFFLINE_MEASURED_NOT_RUNTIME_CERTIFIED. Does not supersede any release +certificate. [Paired result](upgrade/evidence/p184-idr062-offline-pair.md): identical +ETH Binance/OKX public capture and configs on deployed 658a9570 / rc2 69e3b2b2; +no shared changes. Top-change -> next snapshot p95 922/900 ms -> same-event 0 ms +(provider timeline, NOT zero consumer latency). 44/41 actual top changes, exact +delta parity; CPU observed .0600->.0627 vCPU and RSS HWM 19324->19612 KiB for one +core/two bindings. 17 measurement/capture tests PASS. Capture hashed/deleted; +test network/containers/keys cleaned; no new images or broad cache cleanup. +[Phase C](upgrade/evidence/p184-dl-phase-c-packet.md): cores first, then Claude's +post-rollout >=117/120 per-venue coherence receipt plus CPU/lag gates, then other +roles. Prior pre-rollout coherence wording is superseded. Runtime remains HELD, +TS DR01-DR08/broker delta and runtime approval still required. No release/push/merge. diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index e8482866..30521f80 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69241,3 +69241,36 @@ writes, not a reclaimed-byte claim). No image built or deleted; active/rc2/rollb retained. Shared cache inventoried 16.83 GB, not broad-pruned. Canonical main5bf5cd5 and owner hunk untouched; published v2.2.2 unchanged. No push/merge, no TS edit. Next separate docs slice: core-first Phase C packet, post-core coherence gate. + + +##### Phase C core-first packet amendment (Codex, 2026-10-07) + +Status: PACKET_UPDATED_RUNTIME_HELD. Measurement slice commit 5a261e0; no runtime +approval or production certificate inferred. Guide section 18.13 unchanged; +[packet](upgrade/evidence/p184-dl-phase-c-packet.md) now supersedes the prior +pre-rollout coherence prerequisite and reader-first ordering. Approved user scope: +three cores first -> Claude TS coherence -> remaining existing roles only. + +Post-core gate: 120 consecutive 1 Hz samples EACH venue, >=117 usable coherent +samples EACH (39/40). Failures/timeouts remain in denominator; 4 failures permit +early rollback. No exact top equality, freshness or risk contract relaxation. +Incomplete measurement cannot open the remaining roles. CPU keeps the existing +whole-stack 5.0 steady-state vCPU budget and accounting denominator, not the offline +.0627 vCPU two-binding result. Proposed executable stop definition frozen in packet: +two consecutive non-overlapping 30 s CPU means over budget; lag five consecutive +increases over six 10 s readings. Missing offset/ownership is HOLD, never zero lag. +These are packet operational observation rules, not claims they have run or new +freshness/quality thresholds. Preflight already over budget means HOLD before +changing any core. Stop on OOM/integrity/fencing failure as well. + +Rollback: all cores actually changed, reverse order, exact prior 658a9570...54023 +and per-role original config/mounts; no offsets reset, shared data delete, or +reader rollback when readers never changed. Confirm recovery to consumer-usable, +not just broker health. All other roles wait for Claude's complete accepted receipt. +Reader/ingestor/projector/BAR digests and r22 V3 binding fields remain unchanged. +Roster correction resolved: exactly 12 application roles, no invented 13th. +Validation: packet diff, exact role/digest roster unchanged versus 5eb453d, +117/120 arithmetic, linked paired report present, no stale reader-first sequence. +No push/merge, no new test resources/images, no further runtime action. Cleanup +and canonical/runtime/rollback inventory remain as recorded in paired slice above. +Next permitted action is owner/Claude review; STOP and wait. diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md index 447c33fb..66e1b40d 100644 --- a/upgrade/evidence/p184-dl-phase-c-packet.md +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -5,25 +5,37 @@ Phase B handover and journal: DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md, final P18.4 Phase B entries. Architecture: DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md section 18.13. Candidate application source: `43bbcf0209f49e9cc1af6802af8f8a207a53ee3c`. Evidence root: `/home/bobby/.local/state/qdl-v2/p184-phaseb-20261007/evidence/`. -The JSON packet there contains exact container names, config fingerprints and -rollback digests, without environment secrets. No Phase C runtime or binding-mount action has run. +The JSON inventory there contains exact container names, config fingerprints and +rollback digests, without environment secrets; it is not a rollout-order runner. +This amended Markdown governs ordering/gates and supersedes the old pre-core +coherence prerequisite. No Phase C runtime or binding-mount action has run. -## Blocking prerequisites +Offline follow-up: [paired report](p184-idr062-offline-pair.md), measured on existing +images without runtime changes. Receipt SHA256 +`b9d14de49256e7fb15b80faa430c52a610f18d5aa83c1be6dfc88f85e4de4ecc`. +Snapshot workload x1.45; total BOOK messages x1.0413 on two ETH bindings; +observed core CPU .0600 -> .0627 vCPU. These are NOT whole-stack or TS acceptance. + +## Prerequisites and post-core gate 1. TS owner completes DR01-DR08 and broker delta, with new DL findings consolidated here before a single release. Do not substitute prior E09 or synthetic tests. -2. IDR-062 candidate performance remains unmeasured: paired baseline/candidate - on identical approved provider capture/load; change only top materialization. - Record BOOK_SNAPSHOT messages/s and bytes/s, Kafka input/lag, and CPU/RSS/throttle - for core/query/stream/projector plus whole DL stack. <=5 steady-state vCPU is a - measured budget, not authority to lower existing container quotas. TS measures - its writer CPU. Keep all quotas, buffers and freshness gates unchanged. -3. TS coherence >=39/40 sustained, with sample counts and rejected exact-view - reasons; report BOOK p95 versus existing 2000 ms package cap and paired baseline. - Keep exact book-top/QUOTE equality. Independent channels may still diverge; - faster materialization alone is not proof this gate passes. +2. Complete the IDR-062 offline paired report using the existing deployed and rc2 + Rust images, identical captured bytes/configuration and resource limits. Core + CPU/RSS and BOOK_SNAPSHOT message/byte rates are measured; downstream ratios + are estimates only. Offline core replay does NOT measure live Kafka lag or + whole-stack CPU, and is not a TS coherence certificate. Freeze the paired report + hash before runtime approval; no image rebuild for measurement-only changes. +3. TS coherence is explicitly a **POST-CORE-ROLLOUT** gate, not a prerequisite that + the unchanged old core must pass. Claude measures 120 consecutive samples per + venue at 1 Hz after the three cores have caught up. Require >=117/120 per venue + (39/40 = 97.5%); do not pool venues, drop refused/time-out samples, substitute + a later green window or relax exact book-top/QUOTE equality. Preserve exact-view + rejection reasons; report BOOK p95 against the existing 2000 ms package cap. + Independent channels can still diverge: offline improvement does not imply PASS. 4. Owner approves exact runtime packet, role names/digests, maintenance/refusal - window and rollback. Handoff says 13 roles; read-only inventory finds **12**. + window and rollback. Roster is the reviewed **12** existing application roles; + the old handoff count of 13 was incorrect and is resolved by this inventory. No inferred thirteenth role, V1 replacement or state initializer recreation. ## Images and scope @@ -85,13 +97,51 @@ Preflight ISR/quorum, cache generations/checkpoints, actual consumer quality, TLS expiry/SAN, image digests, supported bindings/cursors and per-role config diff. Freeze pre-action inventory and typed refusals; keep active images as named rollback. -Update existing readers one replica at a time (Query then Stream); then the two -projectors separately, three cores separately, ingestors one venue at a time, and -BAR edge last. Between every role verify assignment, lag progression, first cache -apply and actual eligible consumer view, not process health alone. Stop on loss of -ownership, gap/generation/identity mismatch, internal refusal or unbounded lag. -Do not roll the next role merely because Kafka ISR recovered. This is a bounded -maintenance handoff, not a zero-downtime or independent-HA certificate. +1. Roll **rust_core -> rust_core_2 -> rust_core_3**, one at a time, to the Rust + candidate above. Query/Stream/projectors/ingestors/BAR edge remain unchanged. + Between cores verify assignment, committed-offset progress, first cache apply + and actual consumer eligibility, not just ISR or process health. Stop on an + ownership, integrity, generation or identity violation. +2. **Hold all remaining roles. Claude measures TS coherence** on these new core + outputs using the post-core gate above, preserving 120 samples for each venue. + Collect Kafka lag and CPU on the same interval. This is a bounded maintenance + handoff, not a zero-downtime or independent-HA certificate. +3. Only after that receipt PASS: Query replicas individually -> Stream replicas + individually -> projectors individually -> ingestors individually -> BAR edge. + Recheck application recovery and affected eligible reads after each role. Keep + SDK/binding compatibility checks in the binding section; no blanket catalogue or C2 rerun. + +### Core-stage rollback conditions (freeze before action) + +- Coherence **<117/120 on either venue**; four failures already make the window + impossible to pass and allow an early stop. A read error/timeout counts as failed, + not as a missing sample. An incomplete window is NOT PASS; hold or rollback, + never resume the other roles without a complete accepted receipt. +- Exceed the CPU budget: the existing **5.0 steady-state vCPU whole-DL serving-stack + budget** remains the ceiling (`config/v2/kn-v220-candidate-budget.json`, cpu + denominator and production_acceptance: prior measured 4.6687 vCPU / 374.4 s, + not a measurement of this new candidate). Scope: Kafka/Redis, ingestors, cores, projectors, Query, + Stream and BAR edge; TS writer reported separately. Record the exact accounting + roster before action. Sample cgroup CPU counters every 10 s; two successive + non-overlapping 30 s means >5.0 trigger rollback. If a tighter measured operational + envelope is approved before action, use that lower frozen value. Never infer + whole-stack consumption from the offline two-binding core result. If the old + stack already exceeds this ceiling during preflight, HOLD before rollout rather + than attribute it to the candidate. No quota change is authorized here. +- Kafka backlog rises on **six consecutive 10 s readings** (five strictly increasing + intervals) for any affected raw/core group or canonical/projector group. Record + per-partition high watermark minus committed offset, not message-age substitutes. + Unknown/missing offsets, counter reset or a rebalance invalidates the measurement: + HOLD and establish ownership first, not reinterpret an unknown lag as zero. +- Any new loss/duplicate/integrity/fencing failure or OOM also stops this packet. + +On a triggered gate, stop further roles and restore every core already changed, +reverse rollout order, to +`sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023` +with its recorded original config/mounts. Do not reset Kafka offsets/cache or roll +unmodified readers. Confirm catch-up and consumer usable data on return. Retain the +failed interval and rollback receipt; a retry needs an explained cause, not a +fresh window chosen to replace failed evidence. No shared-runtime load or restart in Phase B. Phase C inherits unchanged evidence; run only packaging/affected protocol and consumer gates. Do not repeat whole C2 @@ -114,9 +164,9 @@ D3 remains separate: no independent ingestor/bar-edge/Redis/host HA claim. After runtime acceptance and explicit approval: feature -> dev CI -> main release, immutable tag/assets with source/image/config/evidence and supported SDK/binding versions. Never mutate v2.2.2. These rc2 images are not published stable artifacts. -Remove only the two superseded rc1 image digests after replacement checks and no -container reference; retain active, candidate and named rollback. Test containers -use --rm; remove this task's build target after preserving logs. No broad prune, +The two superseded rc1 images and Phase B build target have already been removed; +do not repeat that cleanup. Retain active, candidate and named rollback images. +New disposable tests use --rm and are removed after their evidence is recorded. No broad prune, shared cache cleanup, volume/network deletion or canonical/user-hunk modification. Record disk pre/post and running-role image/start/restart comparison. Feature worktree remains active until reviewed/merged; no worktree cleanup before that. From a7587b272101b2b243b7b820f3a3269d56924d0f Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 13:45:30 +0000 Subject: [PATCH 011/147] docs: measure active L2 pair and bound release capacity --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 41 ++++++++++ .../evidence/p184-idr062-capacity-envelope.md | 80 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 upgrade/evidence/p184-idr062-capacity-envelope.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 30521f80..2a5dae2a 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69274,3 +69274,44 @@ Validation: packet diff, exact role/digest roster unchanged versus 5eb453d, No push/merge, no new test resources/images, no further runtime action. Cleanup and canonical/runtime/rollback inventory remain as recorded in paired slice above. Next permitted action is owner/Claude review; STOP and wait. + + +##### Release-readiness read-only audit - owner decision 2026-10-07 + +Status: IN_PROGRESS. Same fix/p184-dl-closure worktree, inherited reviewed commits +5a261e0/9dbb93c. DL release waits on TS DR01-DR08 INCLUDING DR07 and broker delta, +NOT TS Paper feature construction. Runtime/certificate gates remain fail-closed. +Scope: actual BOOK binding count and 10/s per-binding provider-time ceiling; +bounded higher-activity public capture replay using unchanged 5a261e0 harness and +existing baseline/rc2 images; read-only current KN health/resource/quota/cache/log +observation; capability/endpoint/version inventory; readiness report and packet. +Guide: architecture review 18.13, Phase C packet and certification ledger. +No production config/data/identity changes, order, restart, rollout, push/merge or +image build. Passive observations first; additional request probes only at low +rate after measured quota headroom, zero new stream subscriptions. Never run +kn_production_watch.main (it can stop services); reuse sample() only. Never run +full benchmark_kn_ts_consumer on shared runtime (isolated Redis writer/streams). +Preregister higher-activity criterion: top changes/minute >=1.5x old 44/41 on BOTH +venues. One 120 s capture in this bounded session; if criterion misses, report +NOT_MEASURED for peak rather than relabel quiet data or wait indefinitely. +Latency boundaries/p50/p95/max/n in ms; p99 only n>=100; no substitution of event +age for request/commit/delivery timing. Unknowns explicitly NOT_MEASURED. rc2 is +not deployed and current runtime measurements do not certify its rollout. +Use scoped --rm offline resources only; hash/delete captures/test keys, preserve +compact evidence. No new image/cache cleanup required; retain active/rc2/rollback. +Each coherent slice committed with journal. Stop after report/packet for owner +review. apply_patch sandbox failed; exact file edits per workspace rule 32. + +### Release-readiness slice 1 - active IDR-062 pair and capacity (2026-10-07) + +Status: MEASURED; current whole-stack CPU gate FAIL, candidate stack NOT_MEASURED. +Readiness report: [capacity envelope](upgrade/evidence/p184-idr062-capacity-envelope.md). +17 BOOK acquisition bindings confirmed in actual ingestors/core (7 Binance, 10 OKX), not multiplied by replicas. +One unchanged-harness 120 s public capture: 270/312 top changes, about 135/156 per minute; preregistered active criterion PASS. +Same-byte delta chains; rc2 top-change materialization delay 0/0/0 ms on n=270/312, source-time metric, not network latency. +Snapshot counts 239 -> 639; core CPU 9.81 -> 10.51 CPU-s; RSS 21844 -> 23612 KiB, samples 199/198. +Passive current-stack measurement: 61 samples/60 intervals, 307.156 s; mean 5.9618 vCPU exceeds unchanged 5-vCPU gate. +No resource increase, production write, rebuild or rollout. Raw capture and isolated replay resources removed; disk accounting and retained images in report. +Commands: unchanged scripts/measure_l2_top_materialization.py run --seconds 120; read-only kn_production_watch.sample wrapper; bounded docker inspect/config reads. +Next: finish contract/latency/health report and amend Phase C HOLD conditions; no TS Paper feature dependency added. +Verification: receipt invariant checks PASS (same config/delta chains, denominator, zero candidate tails, 17 bindings, capture deletion). Harness bytes equal 5a261e0. Attempted pytest on host and existing Python candidate: both lack pytest, so no new unit-pass claim; prior 9 measurement tests remain inherited unchanged. Temporary qdl-idr062-readiness-tests --rm container removed, no dependency install/build. diff --git a/upgrade/evidence/p184-idr062-capacity-envelope.md b/upgrade/evidence/p184-idr062-capacity-envelope.md new file mode 100644 index 00000000..aa7aeaf3 --- /dev/null +++ b/upgrade/evidence/p184-idr062-capacity-envelope.md @@ -0,0 +1,80 @@ +# IDR-062 Active Capture And Capacity Envelope + +Status: MEASUREMENT_COMPLETE; whole-stack 5-vCPU gate FAIL on current runtime; rc2 full-stack NOT_MEASURED. + +Owner scope: read-only production inventory and isolated replay. No runtime change, build, push or merge. Candidate application source `43bbcf0209f49e9cc1af6802af8f8a207a53ee3c`; harness is unchanged from `5a261e0`. + +## Actual BOOK Roster + +Runtime core `/runtime/core.json` (host bind `~/.local/state/qdl-v2/releases/v2.2.1-0e00c81-recovery/rust_core.json`) declares **17 unique BOOK bindings**, not 51 across three core replicas. Ingestor configs independently confirm 7 Binance and 10 OKX subscriptions. These are configured/running acquisition subscriptions, not a claim that every tick of every product is currently eligible. + +| Venue | Native symbol | Core config line | Materialization timer | +| --- | --- | ---: | ---: | +| BINANCE | BNBUSDT | 1193 | 1000 ms | +| BINANCE | BTCUSDT_261225 | 1348 | 30000 ms | +| BINANCE | BTCUSDT | 1624 | 1000 ms | +| BINANCE | DOGEUSDT | 2158 | 1000 ms | +| BINANCE | ETHUSDT_261225 | 2511 | 30000 ms | +| BINANCE | ETHUSDT | 2787 | 1000 ms | +| BINANCE | SOLUSDT | 5175 | 1000 ms | +| OKX | BTC-USD-261225 | 6338 | 30000 ms | +| OKX | BTC-USD-270326 | 6361 | 30000 ms | +| OKX | ETH-USD-261225 | 6384 | 30000 ms | +| OKX | ETH-USD-270326 | 6407 | 30000 ms | +| OKX | BNB-USDT-SWAP | 7493 | 1000 ms | +| OKX | BTC-USDT-SWAP | 7925 | 1000 ms | +| OKX | DOGE-USDT-SWAP | 8501 | 1000 ms | +| OKX | ETH-USDT-SWAP | 9113 | 1000 ms | +| OKX | SOL-USDT-SWAP | 11633 | 1000 ms | +| OKX | BTC-USD-SWAP | 12731 | 1000 ms | + +The two ingestor `/runtime/config.json` evidence lists are `ingestor_binance_usdm-book-lines.json` (first BOOK line 46) and `ingestor_okx_swap-book-lines.json` (first line 33). Catalog declarations alone are not used as the count. Source default at `rust/qdl-realtime-core/src/lib.rs:68` is snapshot_refresh_seconds x 1000: **six research futures use 30 s, not 1 s**; eleven perpetual bindings explicitly use 1000 ms. Candidate top-change floor is 100 ms at `rust/qdl-realtime-core/src/lib.rs:402`, applied at line 1434. Bootstrap/generation changes bypass steady-state timing; 170/s is not a hard bound during reset storms. + +## Paired Higher-Activity Capture + +Command: `python3 -B scripts/measure_l2_top_materialization.py run --work /home/bobby/.local/state/qdl-v2/release-readiness-20261007/active-pair --seconds 120`. +One public capture, identical provider bytes, config, resource limits, replay timing, broker and consumer. Only core image changes. Baseline `658a9570...54023`; candidate `69e3b2b2...713d`. Capture implementation: `scripts/phase114_l2_real_provider_capture.py:578`. No production Kafka replay/write. + +| Venue | Window s / frames | Top changes / minute | Snapshot count old/new | msg/s old/new | protobuf bytes/s old/new | +| --- | ---: | ---: | ---: | ---: | ---: | +| BINANCE/ETHUSDT | 119.896 / 1179 | 270 / 135.12 | 118/304 | 0.984/2.536 | 7767.0/19995.4 | +| OKX/ETH-USDT-SWAP | 120.100 / 1202 | 312 / 155.87 | 121/335 | 1.007/2.789 | 8234.2/22775.4 | + +Preregistered high-activity criterion: >=1.5 x old 44/41 changes/min on BOTH venues. PASS: about 3.07x/3.80x. This is a measured active window, not proof of the daily maximum. + +| Venue | Delay basis | Baseline p50/p95/max ms (n) | rc2 p50/p95/max ms (n) | +| --- | --- | --- | --- | +| BINANCE/ETHUSDT | next_verified_snapshot_delay | 408/918/924 (269) | 0/0/0 (270) | +| BINANCE/ETHUSDT | matched_delay | 204/818/924 (99) | 0/0/0 (270) | +| OKX/ETH-USDT-SWAP | next_verified_snapshot_delay | 500/900/900 (312) | 0/0/0 (312) | +| OKX/ETH-USDT-SWAP | matched_delay | 200/800/900 (110) | 0/0/0 (312) | + +Delay is **provider event time at top change -> source time of materialized snapshot**, not wall-clock delivery. Zero means emitted for the same source event, not zero network/commit latency. Baseline Binance: 170 superseded tops + 1 tail; OKX: 202 superseded. Candidate: zero superseded/tails; exact delta payload chains match on both venues. No p99 invented. + +| Core resource | Baseline | rc2 | +| --- | ---: | ---: | +| CPU seconds | 9.81 | 10.51 | +| vCPU / 120 s | 0.08175 | 0.08758333333333333 | +| RSS high-water KiB | 21844 | 23612 | +| Resource samples | 199 | 198 | + +CPU is process CPU; cgroup CPU includes sampling exec overhead and is retained separately. RSS is sampled process high-water, not whole-stack RSS. + +## Steady-State Upper Envelope + +17 bindings x 10 snapshots/s = **170 snapshot/s**, 70 Binance + 100 OKX. Observed mean ETH encoded sizes: 7886.10 B Binance and 8165.17 B OKX. Extrapolated payload rate: **1,368,544 B/s (1.369 MB/s)**. This byte rate uses ETH means, NOT a mathematical worst-case size across all symbols, variable decimal strings, Kafka headers/TLS or replication. Snapshot amplification measured on active pair: 639/239 = 2.6736; all BOOK messages: 3014/2614 = 1.1530. + +Illustrative message-cost model (NOT measured capacity): assume every binding also emits 10 deltas/s. Baseline timer snapshots = 11 + 6/30 = 11.2/s; total 181.2/s. Candidate ceiling = 170 deltas + 170 snapshots = 340/s; multiplier R=1.8764. Query request rate stays fixed; snapshots are not 10 additional Query calls/s. Stream cost also depends on subscriptions/fan-out. No buffer/quotas changed. + +| Role group | Current measured mean vCPU | Illustrative vCPU at R | Assumption | +| --- | ---: | ---: | --- | +| core | 1.1895 | 2.2319 | all group CPU scales with total BOOK message ratio; non-BOOK/base costs not separated | +| projector | 0.2978 | 0.5587 | all group CPU scales with total BOOK message ratio; non-BOOK/base costs not separated | +| query | 0.2950 | 0.2950 | fixed HTTP request rate, not measured candidate CPU | +| stream | 0.2211 | 0.4149 | all group CPU scales with total BOOK message ratio; non-BOOK/base costs not separated | + +Current full-stack mean **5.9618 vCPU**, p95 6.2197, max 6.3726; n=60 intervals over 307.156 s, 17 running roles including three Kafka brokers and both Redis roles. Measurement completed BEFORE isolated replay started. The 5-vCPU gate is **FAIL for this current workload**; headroom is -0.9618 vCPU. Candidate whole-stack maximum remains **NOT_MEASURED**. Even the illustrative model holding Kafka/ingestors/edge/Redis fixed is 7.459 vCPU; not a safe capacity ceiling, since broker/Redis costs can also rise. Do not use the two-binding core result to certify <=5 vCPU. + +## Evidence And Cleanup + +External root: `/home/bobby/.local/state/qdl-v2/release-readiness-20261007`. Paired receipt SHA256 `381760f55659bb5b98879526d7100e118f3c1cbb82e9b05674d5b28579105aa4`. Capture SHA256 `0a116c39866bbeb325b3833a21e0d865f1c599262176f0b8aa6e3135b8394a8d`; deleted after replay/hashing. All isolated containers/network and generated TLS keys removed by unchanged harness. No image built or removed. Cleanup free bytes 102468628480 -> 100905811968; concurrent host use means this is NOT reclaimed-space accounting. Active+rc2+rollback images retained. Full resource result: `resources-summary.json`; original 61 samples: `resource-samples.jsonl`. From 397cf825e75e09a972e3cf91d2b7659724409fa6 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 13:57:20 +0000 Subject: [PATCH 012/147] docs: record DL readiness and hold phase C on measured gaps --- CERTIFICATION_LEDGER.md | 13 + DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 44 +++ upgrade/evidence/p184-dl-phase-c-packet.md | 43 ++- upgrade/evidence/p184-dl-release-readiness.md | 303 ++++++++++++++++++ 4 files changed, 401 insertions(+), 2 deletions(-) create mode 100644 upgrade/evidence/p184-dl-release-readiness.md diff --git a/CERTIFICATION_LEDGER.md b/CERTIFICATION_LEDGER.md index 7038ca16..b47330b0 100644 --- a/CERTIFICATION_LEDGER.md +++ b/CERTIFICATION_LEDGER.md @@ -2141,3 +2141,16 @@ test network/containers/keys cleaned; no new images or broad cache cleanup. post-rollout >=117/120 per-venue coherence receipt plus CPU/lag gates, then other roles. Prior pre-rollout coherence wording is superseded. Runtime remains HELD, TS DR01-DR08/broker delta and runtime approval still required. No release/push/merge. + +### 2026-10-07 - Read-only release-readiness audit (not promotion) + +Application rc2 remains43bbcf0, undeployed. See +[readiness report](upgrade/evidence/p184-dl-release-readiness.md) and +[active pair/capacity](upgrade/evidence/p184-idr062-capacity-envelope.md). +Active pair PASS at135/156top-changes/minute; current whole-stack5vCPU gate FAIL +(mean5.9618, n60intervals/307.156s). TS snapshot119/120usable; one strict LAST refusal. +Stream overflow+4 and TS cache lag require DR07 attribution. Exact delivery lag/ +Redis-ACK/alpha cache latency NOT_MEASURED. Phase C HOLD, not authorized/deployed. +Only TS DR01-DR08 includingDR07 +broker delta are release dependencies; no Paper +feature construction added. Prior certificates kept immutable and scoped, no +freshness/quota/resource threshold change. No required endpoint replacement identified. diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 2a5dae2a..2ffdd854 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69315,3 +69315,47 @@ No resource increase, production write, rebuild or rollout. Raw capture and isol Commands: unchanged scripts/measure_l2_top_materialization.py run --seconds 120; read-only kn_production_watch.sample wrapper; bounded docker inspect/config reads. Next: finish contract/latency/health report and amend Phase C HOLD conditions; no TS Paper feature dependency added. Verification: receipt invariant checks PASS (same config/delta chains, denominator, zero candidate tails, 17 bindings, capture deletion). Harness bytes equal 5a261e0. Attempted pytest on host and existing Python candidate: both lack pytest, so no new unit-pass claim; prior 9 measurement tests remain inherited unchanged. Temporary qdl-idr062-readiness-tests --rm container removed, no dependency install/build. + +### Release-readiness slice 2 - read-only report / Phase C HOLD (2026-10-07) + +Status: AUDIT_COMPLETE_WITH_GAPS; PHASE_C_HOLD_NOT_AUTHORIZED. Not a release certificate. +Owner decision: DL waits only on TS DR01-DR08 including DR07 plus broker delta, +not TS Paper construction. Existing safety/resource gates still apply. +Report: [DL readiness](upgrade/evidence/p184-dl-release-readiness.md). +Packet: [Phase C core-first](upgrade/evidence/p184-dl-phase-c-packet.md). + +Measured on deployed KN, not rc2: 12 app +5 infrastructure roles unchanged; +61 resource samples/60 intervals,307.156s;5.9618vCPU mean >5 gate. All container +IDs/images/restart counters stable; BAR-edge historical3 restarts retained. +Correct-TLS serial TS60 reads through both replicas:119/120usable; per-feed n19/20, +request p50/p95/max in report, no p99. Exact BNB/OKX TRADE refusal age4262ms, +session live59ms, stale LAST; no provider-side attribution from a later green read. +Shared quota measured at correct :identity namespace: previous961/1500 before +probe; later885/1500. No new identity, stream, quota, manifest or order. +Cache GET observer:1800reads/30rounds,zero missing; BOOK_DELTA age up to22825ms. +Stream active log metrics:overflow+4 over about10min, reader/decode errors delta0. +These are open DR07 attribution/availability evidence, not silently fixed. + +Measurement limitations/errors retained: first120 requests used wrong TLS alias, +ConnectError; aliases corrected after2-request smoke, no TLS bypass. First PUBSUB +observer connection closed; bounded60s observer may include notification backlog, so not +used as exact Redis write latency. GET cache age/first-observation upper bounds +are distinct from SET ACK timing; canonical delivery lag and alpha cache latency +NOT_MEASURED. Initial quota lookup lacked :identity suffix; corrected before +requests. Missing pytest on host/candidate, unchanged source tests inherited; +receipt/schema/table consistency checked independently. No app source changed. + +Kafka per-partition lag and exact role digests/config references, 60 binding rows, +all HTTP/gRPC surfaces, SDK2.0.7 deployed vs2.1.0 candidate, V1/V2/V3 r22/JWT +revisions, quota/concurrency/headroom and known D1/D3 limits are in report. +BAR3d remains excluded; existing history errors/inflight admission deferrals are +reported, not fixed under this audit. Historical18s BAR-edge restart not remeasured. + +Cleanup: isolated pair --rm containers/network/capture/TLS keys removed; diagnostic +raw cache payload hashed then removed. No new image/build/cache; Docker92images +28.19GB, BuildKit227entries16.83GB. Keep active+rc2+named rollback; broad prune not +permitted by read-only scope. Disk pre/post102468628480/100905811968bytes includes +other host activity. Canonical main5bf5cd5 untouched; same active feature worktree. +No runtime rollout, push or merge. Stop and return readiness/packet for owner review; +do not continue into TS Paper features or roll a candidate against failing baseline. +Final audit checks PASS:60unique binding products x2replicas=120reads (119usable);1800cache GETs/zero missing; report60-row coverage and local links valid; qdl-idr062 containers/networks zero. TLS negative explicitly confirms hostname mismatch; no TLS bypass. Readiness measurements are not a replacement certificate. diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md index 66e1b40d..f15860db 100644 --- a/upgrade/evidence/p184-dl-phase-c-packet.md +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -1,6 +1,6 @@ # P18.4 Data Layer Phase C Packet -Status: DRAFT_NOT_AUTHORIZED. Source/local candidate only, not a release certificate. +Status: DRAFT_HOLD_NOT_AUTHORIZED. Source/local candidate only, not a release certificate. Phase B handover and journal: DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md, final P18.4 Phase B entries. Architecture: DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md section 18.13. Candidate application source: `43bbcf0209f49e9cc1af6802af8f8a207a53ee3c`. @@ -16,9 +16,48 @@ images without runtime changes. Receipt SHA256 Snapshot workload x1.45; total BOOK messages x1.0413 on two ETH bindings; observed core CPU .0600 -> .0627 vCPU. These are NOT whole-stack or TS acceptance. +## Readiness audit update - 2026-10-07 + +[Readiness report](p184-dl-release-readiness.md) and [active capacity envelope](p184-idr062-capacity-envelope.md) +are required inputs to this packet. They supersede any inference that the quiet +pair or old whole-stack certificate proves present resource headroom. + +- Owner release dependency: **TS DR01-DR08, including DR07, plus broker delta**. + Do NOT wait for construction of TS Paper features or reopen unrelated phases. +- Active pair: 270/312 top changes in about120s (135/156 per minute); rc2 source-time + materialization delay p50/p95/max0/0/0ms, n270/312. This is NOT client latency. + Snapshot count239 ->639; core9.81 ->10.51CPU-s; no delta-chain difference. +- Actual acquisition roster17BOOK bindings. Steady-state scenario170snapshots/s; + observed-size estimate1.369MB/s protobuf, not a packet/replication byte ceiling. + Query RPC rate is not multiplied automatically by snapshot publication rate. +- **Pre-action resource HOLD:** current full stack mean5.9618vCPU, p956.2197, + max6.3726 over60 intervals/307.156s, before isolated replay. The existing5vCPU + gate already fails; do not roll core then call rollback PASS against this + failing baseline. No cap/budget increase is implied. Owner must resolve the + measured resource prerequisite before authorizing this packet. +- Current correct-TLS snapshot reads119/120usable; one BNB/OKX TRADE refusal + retained (4262ms event age, live session). No quiet-market root cause claimed. +- Stream1 overflow+4 in approximately10min; cache polling observed BOOK_DELTA + up to22825ms while nearby Query reads remained fresh. Exact matched-watermark + attribution/recovery and full delivery measurements belong in DR07. No Paper + feature work is required to supply that handoff evidence. Do not label them + fixed or merely observer artifacts; distinguish the possible PUBSUB + observer lag from the separate GET cache observation. +- qdl-sdk2.1.0 is candidate only; actual TS writer package is2.0.7. Binding r22 + V2 remains mounted, V3 remains staged. No required endpoint replacement has + been identified, and no interface change is authorized by this report. + +After readiness/resource prerequisites and owner approval, retain the exact +**core-first -> Claude TS coherence -> other roles** order below. No action was +performed by this audit. Existing thresholds are unchanged: >=117/120 per venue, +whole-stack <=5vCPU, no five consecutive rising lag intervals. A future change +to an approved resource budget must be versioned before measuring, not chosen +after seeing candidate results. Preserve all negative evidence. + ## Prerequisites and post-core gate -1. TS owner completes DR01-DR08 and broker delta, with new DL findings consolidated +1. TS owner completes DR01-DR08 (including DR07) and broker delta, independently + of TS Paper feature construction, with new DL findings consolidated here before a single release. Do not substitute prior E09 or synthetic tests. 2. Complete the IDR-062 offline paired report using the existing deployed and rc2 Rust images, identical captured bytes/configuration and resource limits. Core diff --git a/upgrade/evidence/p184-dl-release-readiness.md b/upgrade/evidence/p184-dl-release-readiness.md new file mode 100644 index 00000000..9c984ae1 --- /dev/null +++ b/upgrade/evidence/p184-dl-release-readiness.md @@ -0,0 +1,303 @@ +# DL Release Readiness - rc2 / Deployed KN, 2026-10-07 + +**Decision: HOLD / NOT READY to claim the complete rc2 release production-qualified.** This is the requested read-only report, not a release certificate or permission to deploy. The active-capture source improvement is verified, but current full-stack CPU exceeds the existing gate; consumer cache/stream observations still need attribution and DR07 acceptance. No interface replacement is currently required by these findings. + +Owner boundary: DL release depends on TS **DR01-DR08 (including DR07) + broker delta**, not completion of TS Paper features. No E03/E04 rerun, new TS phase, orders, runtime changes or new consumer/quota. Existing Phase C safety/resource/coherence gates remain mandatory. + +## Evidence And Scope + +- Worktree `/home/bobby/.worktrees/data-layer-p184-closure`, branch `fix/p184-dl-closure`. Capacity slice `a7587b2`; application rc2 source `43bbcf0209f49e9cc1af6802af8f8a207a53ee3c`. +- Canonical `/home/bobby/data_layer`, `main@5bf5cd540498b82959342c7cfb2ebeaac3007426`, tag ancestry `v2.2.2-3-g5bf5cd5`. Canonical owner changes untouched. Only this active feature worktree retained. +- rc2 Rust `sha256:69e3b2b214331a7ebb6a110dd8159c899bbbd6439aaf87293c8c68d145e3713d`; Python `sha256:77682eea5054ae9ddbd858323b083286669a55eeb4bc7bc065b04bae8854e32f`. Neither is deployed. These images already existed; no build. +- Current TS writer `market_data_service_p184w`, image `sha256:9150385e2bb3596793b6ee209dbc99696c3ed07439d6f4199356f417a97b6105`; its qdl-sdk package is **2.0.7**, not candidate 2.1.0. Its binding reader version and qdl-sdk version are separate contracts. +- External evidence root `/home/bobby/.local/state/qdl-v2/release-readiness-20261007`. All tables below identify measured versus estimated/inherited quantities. Each PASS is scoped to its row, not blanket execution certification. + +## Verdict Matrix + +| Item | Verdict | Basis / remaining condition | +| --- | --- | --- | +| IDR-062 active paired correctness | PASS | Identical capture/config and delta chains; same-event snapshots n=270 Binance,312 OKX. | +| Current whole stack <=5 vCPU | FAIL | 5.9618 mean,6.2197 p95,6.3726 max;60 intervals/307.156 s before isolated replay. | +| rc2 whole-stack CPU and coherence | NOT_MEASURED | Offline core only; post-core 120 samples/venue still required. | +| All12 app roles /5 infrastructure roles remain running | PASS | Same container IDs/images/restarts before/after. Existing BAR-edge restart count3 retained. | +| Current TS60 Query usability | FAIL for 100%-usable claim | 119/120 usable; one exact-view TRADE freshness refusal, not an unexplained SDK success. | +| Public contract preservation | PASS for reviewed candidate | Existing endpoints unchanged; SDK2.1.0 response tolerance; V3 r22 additive policy binding, caller compatibility gate retained. | +| Continuous Stream delivery | FAIL for uninterrupted claim | Active Stream overflow counter +4 in60 metric samples over approx10 min; no new decode/reader errors. | +| Exact source-event -> Redis SET/ACK latency / alpha cache | NOT_MEASURED | Possible observer lag; bounded GET availability upper bounds below, no alpha cache process measured. | +| All four latency quantities including delivery lag | NOT_MEASURED in full | Source event age available; publication timestamp absent from served cache views, no fabricated commit time. | +| DR01-DR08 /broker delta handoff | NOT_MEASURED by this audit | Owner/Claude must attach current receipts; TS Paper feature construction is not a dependency. | + +## A. Latency, Milliseconds + +Read path: actual TS adapter/identity/policy -> HTTPS Query -> SDK validation. 120 serial reads, one per second, one inflight,60 products on each replica. TLS aliases `query_v2_1`/`query_v2_2`, both actual KN Query replicas. No additional subscriptions; no bypass of freshness or TLS. Source reference: `scripts/benchmark_kn_ts_consumer.py:102` through line135; only its snapshot read logic was reused, NOT the isolated bridge/writer phase. External commands: `python3 -B request-launch-tls.py` (inside writer via `docker exec -i ... python -B -`). + +| Feed | Request->validated usable p50/p95/max ms (n successes) | Durable-view event age at SDK return p50/p95/max ms (n) | Refused/total | +| --- | --- | --- | ---: | +| BAR | 20.68/45.57/47.49 (n=20) | 18669.75/52430.53/52690.49 (n=20) | 0/20 | +| BOOK_DELTA | 18.17/56.21/123.14 (n=20) | 401.62/533.01/579.98 (n=20) | 0/20 | +| BOOK_SNAPSHOT | 29.49/50.72/51.39 (n=20) | 1061.58/1356.35/1368.47 (n=20) | 0/20 | +| MARK_INDEX_PRICE | 34.01/62.29/64.68 (n=20) | 644.33/1433.34/1464.64 (n=20) | 0/20 | +| QUOTE | 12.27/91.69/93.56 (n=20) | 328.71/473.29/582.06 (n=20) | 0/20 | +| TRADE | 13.66/46.89/46.89 (n=19) | 464.79/2828.97/2828.97 (n=19) | 1/20 | + +These per-feed samples are pooled across venues/replicas; per-binding rows are below. n=19/20 is small: p95 is a nearest-rank sample statistic (n=19 p95 equals maximum), **no p99**. Initial clock includes route lookup, SDK request/server queue, decode/validation; pacing wait before starting the request is not counted. No concurrency queue was imposed. + +Durable-view event age means client receipt wall-clock minus the **source event timestamp of the returned canonical/materialized view**, BAR uses close time. It is not age since Kafka commit, and not request duration. A 52-second-old final1m BAR is not a 52-second HTTP response. MARK combined timestamp is not a substitute for each component eligibility/quiet proof. + +Refusal retained: Query replica1, OKX BNB-USDT-SWAP TRADE; request79.78 ms, exact-view freshness4262 ms, event STALE, session LIVE/liveness59 ms, no gap, complete=true, execution_eligible=false, LAST_EVENT_STALE. This is correct blocking at3,000 ms; no simultaneous provider TRADE capture, so quiet-market versus delayed pipeline attribution is NOT_MEASURED. Second replica positive read does not erase this refusal. + +### Delivery Lag And Actual Cache Boundary + +Delivery lag (source event -> canonical publication, matching `scripts/kn_native_slice_probe.py:387`) is **NOT_MEASURED for QUOTE/TRADE/MARK/BOOK/BAR** in this read-only run: served TS cache does not preserve `published_at_ns`. No new Stream identity/subscription was opened to recover it. `received_at_ns` is provider ingest, NOT Kafka commit/publication. Commit-to-consumer is likewise NOT_MEASURED. Aggregate Stream dispatch histogram is not a per-feed replacement. + +Existing `scripts/measure_kn_ts_cache_delivery.py:66` was run passively first180s: Redis closed the PUBSUB connection and it produced no summary. Preserved failure. A bounded60s copy retained partial/error output without changing runtime;37,723 notifications,50 groups, no missing-key/watermark error, but notification-derived ages approximately20-44 s were much larger than later direct GET ages; these were not simultaneous. The difference is not conclusively attributable to the observer alone. It computes age of the queued notification while permitting a newer cached watermark, so those numbers are **observer-inclusive upper bounds, not a TS write-delay certificate**. BAR absent in that notification window. No claim that changing an observer fixes a runtime issue. + +A separate1Hz pipelined GET observer then read all60 actual TS cache keys for30 rounds:1,800 reads, zero missing keys, no writes. Boundary: event/BAR-close -> external observation of present TS cache; excludes alpha caches and does not timestamp actual SET ACK. Values below include repeated/current views and stored quality is NOT revalidated into execution eligibility. This is useful consumer-visible age, not exact event-application timing. + +| Feed | Current cache age p50/p95/max ms (n) | New watermark after initial round: first-observed availability upper bound ms (n) | Verdict | +| --- | --- | --- | --- | +| BAR | 15663.58/60184.83/61343.88 (n=300) | 2208.46/2208.46/2208.46 (n=10) | Measured age; exact apply latency NOT_MEASURED | +| BOOK_DELTA | 7042.15/19358.58/22825.17 (n=300) | 7850.59/20401.21/22825.17 (n=189) | Measured age; exact apply latency NOT_MEASURED | +| BOOK_SNAPSHOT | 1339.19/2043.93/2436.88 (n=300) | 1324.19/2003.98/2436.88 (n=267) | Measured age; exact apply latency NOT_MEASURED | +| MARK_INDEX_PRICE | 1983.34/3147.93/3343.88 (n=300) | 1830.17/2331.13/3208.46 (n=205) | Measured age; exact apply latency NOT_MEASURED | +| QUOTE | 863.15/1301.52/1648.88 (n=300) | 886.98/1299.88/1648.88 (n=288) | Measured age; exact apply latency NOT_MEASURED | +| TRADE | 1023.59/8603.54/15411.21 (n=300) | 920.45/2679.88/4521.93 (n=229) | Measured age; exact apply latency NOT_MEASURED | + +**Open attribution:** BOOK_DELTA cache reached22,825 ms despite Query BOOK_DELTA max580 ms in a different, nearby window. Snapshot cache max2437 ms exceeds a2,000 ms package-use age threshold at that observation. MARK age alone does not decide quiet/component eligibility. These observations cannot certify uninterrupted execution or blame a specific pipeline segment without paired watermark tracing. Hand them to DR07; do not silently waive them or trigger a new DL rewrite. One stored value per feed being present is not readiness. + +Measurement harness errors preserved: first120 reads used TLS-incompatible `query_kn_*` aliases and all failed ConnectError before application response; two-request alias check succeeded with canonical TLS names; final120-read run uses correct aliases. This was an audit setup error, not120 production service failures. Initial quota lookup omitted the `:identity` prefix and returned null; corrected using `qdl/security/data_plane.py:190` and actual key namespace before issuing requests. No quota headroom was inferred from the incorrect lookup. + +## B. Runtime Health And Resources + +Command/method: bounded docker inspect, cgroup cpu.stat/memory.current at5s intervals using **only** `kn_production_watch.sample()`; never its main/auto-stop path.61 samples/60 deltas over307.156s. CPU columns are measured mean/p95/max vCPU; throttle is delta nr_throttled/nr_periods, not CPU time lost. RSS uses one later `docker top -eo pid,rss,comm` sum of role processes (n=1; shared pages can double-count), not tini PID RSS. Cgroup peak includes page cache and is not RSS. + +| Role | Image digest prefix | Restarts / OOM | CPU mean/p95/max (n=60) | throttle periods % | RSS MiB (n=1) | cgroup peak/cap MiB | +| --- | --- | --- | --- | ---: | ---: | ---: | +| ingestor_binance_usdm | `0f6876e16e51` | 0 / False | 0.161/0.194/0.232 | 0.521 | 41.4 | 39.4/256.0 | +| query_kn_1 | `514506122111` | 0 / False | 0.161/0.176/0.385 | 0.035 | 693.2 | 714.1/1536.0 | +| query_kn_2 | `514506122111` | 0 / False | 0.134/0.151/0.156 | 0.000 | 606.6 | 612.8/1536.0 | +| market_projector_1 | `a696cfba6ab3` | 0 / False | 0.141/0.150/0.186 | 0.000 | 145.3 | 143.9/512.0 | +| market_projector_2 | `a696cfba6ab3` | 0 / False | 0.156/0.172/0.214 | 0.000 | 158.2 | 157.5/512.0 | +| stream_kn_1 | `1f13408fe946` | 0 / False | 0.146/0.173/0.266 | 0.130 | 393.4 | 391.6/512.0 | +| stream_kn_2 | `1f13408fe946` | 0 / False | 0.076/0.079/0.086 | 0.000 | 333.9 | 331.0/512.0 | +| ingestor_okx_swap | `0f6876e16e51` | 0 / False | 0.145/0.174/0.196 | 0.195 | 39.0 | 43.4/256.0 | +| kafka3 | `9516fb7634ba` | 0 / False | 1.053/1.093/1.115 | 0.554 | 899.4 | 1535.9/1536.0 | +| kafka2 | `9516fb7634ba` | 0 / False | 1.026/1.065/1.076 | 0.684 | 860.2 | 2048.0/2048.0 | +| kafka1 | `9516fb7634ba` | 0 / False | 0.903/0.962/0.982 | 2.345 | 1061.1 | 1536.0/1536.0 | +| rust_core | `658a9570c5fc` | 0 / False | 0.599/0.639/0.670 | 0.065 | 129.6 | 128.3/256.0 | +| rust_core_2 | `658a9570c5fc` | 0 / False | 0.419/0.447/0.510 | 0.000 | 162.7 | 160.4/256.0 | +| rust_core_3 | `658a9570c5fc` | 0 / False | 0.172/0.185/0.204 | 0.033 | 130.1 | 127.7/256.0 | +| binance_bar_edge | `f7351c3bda08` | 3 / False | 0.568/0.630/0.639 | 2.851 | 184.1 | 183.9/512.0 | +| market_cache | `dfa18828cbc0` | 0 / False | 0.081/0.088/0.090 | 0.000 | 1153.3 | 1154.5/9216.0 | +| stable_redis | `dfa18828cbc0` | 0 / False | 0.021/0.026/0.029 | 0.000 | 8.7 | 8.9/160.0 | + +Exact full image digests/config retention set are in the [Phase C packet](p184-dl-phase-c-packet.md) and hashed inventory. No restart increment or OOM increment during observation. Projector status has no new conflict/rewind/cache reconnect and owned partitions remain split even/odd. Historical projector2 abort count1 is not zero-history proof. Cache has about1.15GiB charged against9GiB cap; near-cap Kafka charged memory includes file cache, so cannot infer imminent heap OOM from memory.current alone. + +Logs bounded to10 min/1,000 lines per role: no WARN/ERROR severity for core/Stream/projectors; projector string `stale:0` caused keyword matches and is NOT an error. BAR edge has INFLIGHT_CAPACITY deferrals and known Binance3d history discontinuity errors. Binance3d stays explicitly excluded by owner, not secretly repaired or counted as complete. No output from several docker logs paths is not proof of no errors; stable_redis docker logs timed out. Current TS log had a MARK/INDEX reconnect, not enough evidence for cause. Process health alone is insufficient. + +### Kafka Partition Lag (records, one snapshot per group) + +Command: existing broker `kafka-consumer-groups.sh --bootstrap-server kafka1:9092 --command-config /etc/kafka/secrets/admin.properties --describe --group `. No offset commit/reset/group membership change. Groups sampled sequentially; not an atomic cluster snapshot. + +| Group | Topic | Partition:lag | +| --- | --- | --- | +| kn-projector-v3-a | md.canonical.v2 | 0:34, 1:27, 2:15, 3:38, 4:54, 5:28 | +| kn-projector-v3-b | md.bars.v2 | 0:1, 1:1, 2:1, 3:1, 4:1, 5:1 | +| kn-projector-v3-b | md.latest.v2 | 0:41, 1:40, 2:45, 3:61, 4:25, 5:48 | +| qdl-v2-realtime-core-v2 | md.raw.realtime.v2 | 0:40, 1:43, 2:4, 3:20, 4:27, 5:14 | +| qdl-v2-book-resync-ingestor_okx_swap-okx-public-001 | md.quarantine.stable.v1 | 0:76, 1:1, 2:1, 3:1, 4:1, 5:1 | +| qdl-v2-book-resync-ingestor_binance_usdm-binance-public-book-001 | md.quarantine.stable.v1 | 0:76, 1:1, 2:1, 3:1, 4:1, 5:1 | + +No under-replicated partitions returned by read-only kafka-topics --describe --under-replicated-partitions (exit0). Lag trend/maximum recovery during a fault is NOT_MEASURED in this read-only audit; one bounded lag sample is not a recovery test. Keep six10s samples during proposed rollout as packet requires. + +## C. TS60 r22 Capability And Observed Reads + +Verdict: PASS for declared roster/route preservation; FAIL for claiming every observation execution-usable. + +Actual mounted schema `qdl.v2.consumer-route-binding.v2`, revision22, manifest10;60 products. Ten linear perpetual instruments, six feeds each; BAR interval1m. Explicit V2_PRIMARY and fail-closed fallback; no silent V1 price substitution. No inverse/spot/futures/options inferred from this60-product claim, though17 acquisition BOOK bindings include other products. TS paper manifest has61 requirements (entitlement roster is not identical to sealed60-product usage); sandbox has64, requiring its own identity checks. + +| Venue / native product | Feed | Interval | Binding max freshness ms | Current replica1/2 usable | Request max ms (2 observations) | +| --- | --- | --- | ---: | --- | ---: | +| BINANCE / BNBUSDT | BAR | 1m | 180000 | yes/yes | 22.86 | +| BINANCE / BNBUSDT | BOOK_DELTA | - | 2000 | yes/yes | 27.52 | +| BINANCE / BNBUSDT | BOOK_SNAPSHOT | - | 60000 | yes/yes | 29.15 | +| BINANCE / BNBUSDT | MARK_INDEX_PRICE | - | 2000 | yes/yes | 62.29 | +| BINANCE / BNBUSDT | QUOTE | - | 2000 | yes/yes | 12.27 | +| BINANCE / BNBUSDT | TRADE | - | 3000 | yes/yes | 14.07 | +| BINANCE / BTCUSDT | BAR | 1m | 180000 | yes/yes | 18.50 | +| BINANCE / BTCUSDT | BOOK_DELTA | - | 2000 | yes/yes | 123.14 | +| BINANCE / BTCUSDT | BOOK_SNAPSHOT | - | 60000 | yes/yes | 38.93 | +| BINANCE / BTCUSDT | MARK_INDEX_PRICE | - | 2000 | yes/yes | 34.61 | +| BINANCE / BTCUSDT | QUOTE | - | 2000 | yes/yes | 12.03 | +| BINANCE / BTCUSDT | TRADE | - | 3000 | yes/yes | 46.89 | +| BINANCE / DOGEUSDT | BAR | 1m | 180000 | yes/yes | 45.57 | +| BINANCE / DOGEUSDT | BOOK_DELTA | - | 2000 | yes/yes | 19.41 | +| BINANCE / DOGEUSDT | BOOK_SNAPSHOT | - | 60000 | yes/yes | 38.13 | +| BINANCE / DOGEUSDT | MARK_INDEX_PRICE | - | 2000 | yes/yes | 34.47 | +| BINANCE / DOGEUSDT | QUOTE | - | 2000 | yes/yes | 14.18 | +| BINANCE / DOGEUSDT | TRADE | - | 3000 | yes/yes | 13.16 | +| BINANCE / ETHUSDT | BAR | 1m | 180000 | yes/yes | 20.68 | +| BINANCE / ETHUSDT | BOOK_DELTA | - | 2000 | yes/yes | 56.21 | +| BINANCE / ETHUSDT | BOOK_SNAPSHOT | - | 60000 | yes/yes | 28.56 | +| BINANCE / ETHUSDT | MARK_INDEX_PRICE | - | 2000 | yes/yes | 64.68 | +| BINANCE / ETHUSDT | QUOTE | - | 2000 | yes/yes | 12.78 | +| BINANCE / ETHUSDT | TRADE | - | 3000 | yes/yes | 22.94 | +| BINANCE / SOLUSDT | BAR | 1m | 180000 | yes/yes | 17.72 | +| BINANCE / SOLUSDT | BOOK_DELTA | - | 2000 | yes/yes | 31.74 | +| BINANCE / SOLUSDT | BOOK_SNAPSHOT | - | 60000 | yes/yes | 27.61 | +| BINANCE / SOLUSDT | MARK_INDEX_PRICE | - | 2000 | yes/yes | 32.78 | +| BINANCE / SOLUSDT | QUOTE | - | 2000 | yes/yes | 39.71 | +| BINANCE / SOLUSDT | TRADE | - | 3000 | yes/yes | 12.20 | +| OKX / BNB-USDT-SWAP | BAR | 1m | 180000 | yes/yes | 27.97 | +| OKX / BNB-USDT-SWAP | BOOK_DELTA | - | 2000 | yes/yes | 13.52 | +| OKX / BNB-USDT-SWAP | BOOK_SNAPSHOT | - | 60000 | yes/yes | 40.69 | +| OKX / BNB-USDT-SWAP | MARK_INDEX_PRICE | - | 2000 | yes/yes | 34.17 | +| OKX / BNB-USDT-SWAP | QUOTE | - | 2000 | yes/yes | 12.42 | +| OKX / BNB-USDT-SWAP | TRADE | - | 3000 | refused/yes | 79.78 | +| OKX / BTC-USDT-SWAP | BAR | 1m | 180000 | yes/yes | 42.97 | +| OKX / BTC-USDT-SWAP | BOOK_DELTA | - | 2000 | yes/yes | 18.17 | +| OKX / BTC-USDT-SWAP | BOOK_SNAPSHOT | - | 60000 | yes/yes | 31.19 | +| OKX / BTC-USDT-SWAP | MARK_INDEX_PRICE | - | 2000 | yes/yes | 34.05 | +| OKX / BTC-USDT-SWAP | QUOTE | - | 2000 | yes/yes | 12.56 | +| OKX / BTC-USDT-SWAP | TRADE | - | 3000 | yes/yes | 11.91 | +| OKX / DOGE-USDT-SWAP | BAR | 1m | 180000 | yes/yes | 21.69 | +| OKX / DOGE-USDT-SWAP | BOOK_DELTA | - | 2000 | yes/yes | 18.49 | +| OKX / DOGE-USDT-SWAP | BOOK_SNAPSHOT | - | 60000 | yes/yes | 51.39 | +| OKX / DOGE-USDT-SWAP | MARK_INDEX_PRICE | - | 2000 | yes/yes | 33.09 | +| OKX / DOGE-USDT-SWAP | QUOTE | - | 2000 | yes/yes | 11.69 | +| OKX / DOGE-USDT-SWAP | TRADE | - | 3000 | yes/yes | 22.79 | +| OKX / ETH-USDT-SWAP | BAR | 1m | 180000 | yes/yes | 47.49 | +| OKX / ETH-USDT-SWAP | BOOK_DELTA | - | 2000 | yes/yes | 25.25 | +| OKX / ETH-USDT-SWAP | BOOK_SNAPSHOT | - | 60000 | yes/yes | 35.37 | +| OKX / ETH-USDT-SWAP | MARK_INDEX_PRICE | - | 2000 | yes/yes | 34.01 | +| OKX / ETH-USDT-SWAP | QUOTE | - | 2000 | yes/yes | 11.29 | +| OKX / ETH-USDT-SWAP | TRADE | - | 3000 | yes/yes | 20.79 | +| OKX / SOL-USDT-SWAP | BAR | 1m | 180000 | yes/yes | 18.55 | +| OKX / SOL-USDT-SWAP | BOOK_DELTA | - | 2000 | yes/yes | 16.07 | +| OKX / SOL-USDT-SWAP | BOOK_SNAPSHOT | - | 60000 | yes/yes | 31.63 | +| OKX / SOL-USDT-SWAP | MARK_INDEX_PRICE | - | 2000 | yes/yes | 44.48 | +| OKX / SOL-USDT-SWAP | QUOTE | - | 2000 | yes/yes | 93.56 | +| OKX / SOL-USDT-SWAP | TRADE | - | 3000 | yes/yes | 14.52 | + +Quality contract: QUOTE ON_CHANGE/OBSERVE does not invent ticks; session/generation/gap checks remain. TRADE session-live and executable LAST price are distinct; stale LAST cannot price an order. MARK/INDEX requires same-instrument component lineage, component/session proof and native timestamps. BOOK requires sequence integrity, generation and matching execution intent; a60s generic binding cap does not override2s package/top-coherence gate. BAR is final-bar validity/close recency, not tick SLA. Binding policy V3 seals these distinctions; it does not widen entitlement. + +D1 deferred: Binance quiet BOOK heartbeat/materialization can still legitimately expire. D3 separate: ingestor, BAR edge, Redis, Kafka/host share failure domains; no independent HA or zero-downtime claim. Backup through Kafka-native Stream still shares Kafka/host. These limitations are in the runbook/packet, not solved by green Query reads. + +## D. Endpoints, Schemas And Change Policy + +Endpoint source: `qdl/api_v2/router.py:738` through1199 (mounted under `/v2`); gRPC `contracts/proto/qdl/query/v2/query.proto:190`. Paths are resource selectors with feed/interval/requirements, not separate routes per symbol. + +| Surface | Consumer purpose | Contract | +| --- | --- | --- | +| GET /v2/instruments; /v2/instruments/{identity} | TS/alpha metadata/native instrument resolution | V2 typed instrument schema | +| GET /v2/market-data/{instrument_uid}/snapshot | latest TRADE/QUOTE/MARK/BOOK/BAR | V2 SnapshotResponse; typed quality/authority | +| GET /v2/market-data/{instrument_uid}/warmup; /history | alpha bounded final BAR history | V2 WarmupResponse; interval + limit | +| POST /v2/market-data/warmup:batch | universe1d / multi-symbol equal requested limit | V2 BatchResponse, require_all semantics; no invented pre-listing history | +| POST /v2/market-data/reference:batch | funding, OI, long/short, taker and other admitted references | V2 ReferenceBatchResponse; venue-specific capability/units | +| GET /v2/feeds/{instrument_uid}/status | feed quality/liveness | V2 FeedStatusResponse | +| POST /v2/system/readiness:check; GET /v2/system/readiness | scoped readiness / service summary | V2, not an order-admission guarantee | +| GET /v2/data-quality/gaps | bounded retained-window diagnostics | V2 GapListResponse; incomplete/partial is not full-history PASS | +| MarketDataStreamService.Subscribe / Replay | event stream, cursor recovery | protobuf qdl.query.v2; cursor v3 consumer/realm/generation binding | +| MarketDataStreamService.GetSnapshot / GetFeedStatus | stream read/backup status | same scoped typed contract, not independent HA | + +Public endpoint/schema presence is source-reviewed here; this audit directly exercised snapshot reads, not all warmup/reference/diagnostic APIs. Inherit prior scoped packet evidence; do not label uncalled APIs live-PASS on this window. Alpha universe is batch1d, not automatic subscription entitlement for all universe members. + +Candidate qdl-sdk2.1.0 is locally packaged/tested (CERTIFICATION_LEDGER.md:2114); actual TS writer qdl-sdk2.0.7. Binding V1/V2 readers retained, V3r22 additive policies staged, not mounted. Release r22 universal hash `b4e43688ef714e6cef28c0bf2e565b0fbd7b5c70703c9271e1189a9c5bf99afe`; actual V2 binding hash `60118db19bfca5d6d39aec83c1f0b6a0b4068cfe085d4a786e554792e99ebd5d`. Candidate V3 file hash and rollback file are in packet. SDK version, release revision and JWT manifest revision are not interchangeable. + +**Required endpoint replacement identified now: none.** Intended evolution is additive optional response fields/new products under versioned manifests; existing request semantics remain strict. New venue/product/FX/collateral capability can require explicit new schema/version/entitlement, not a guarantee that every possible future feature fits today's schema. Removal/change of a required field, unit, cursor meaning or freshness policy requires explicit versioned migration; no unreviewed breaking interface change is authorized. Adding an alpha uses an existing allowed manifest or a versioned config/identity activation with compatibility preflight, not rebuilding the DL core merely to add a consumer name. + +## E. Capacity, Quota And Headroom + +Verdict: PASS for observed request quota headroom; FAIL for <=5vCPU; NOT_MEASURED for added-consumer concurrency capacity. + +Actual manifests from Query mounts and shared Redis minute counters (`...:identity:quota:minute::`). Table uses completed minute before probe and a later completed minute, not partially elapsed-minute extrapolation. Query+Stream request accounting shares identity budget; stream counts are separate. Limits are not per-container freebies. + +| Consumer | JWT/manifest rev | req/min | prior used / remaining | later used / remaining | batch / warmup / streams | +| --- | ---: | ---: | --- | --- | --- | +| monitoring.multivenue.stable | 2 | 180 | 0 / 180 | 0 / 180 | 50 / 2000 / 20 | +| alpha.binance.paper.stable | 14 | 2400 | 0 / 2400 | 0 / 2400 | 50 / 10000 / 60 | +| alpha.okx.paper.stable | 14 | 2400 | 0 / 2400 | 0 / 2400 | 50 / 10000 / 60 | +| alpha.vn.paper.stable | 1 | 180 | 0 / 180 | 0 / 180 | 50 / 2000 / 20 | +| trading-system.paper.stable | 10 | 1500 | 961 / 539 | 885 / 615 | 50 / 2000 / 50 | +| qdl.crypto.reference-l2.stable | 6 | 120 | 0 / 120 | 0 / 120 | 100 / 10000 / 24 | +| alpha.binance.live.stable | 1 | 2400 | 0 / 2400 | 0 / 2400 | 50 / 10000 / 60 | +| alpha.binance.sandbox.stable | 1 | 2400 | 0 / 2400 | 0 / 2400 | 50 / 10000 / 60 | +| alpha.okx.live.stable | 1 | 2400 | 0 / 2400 | 0 / 2400 | 50 / 10000 / 60 | +| alpha.okx.sandbox.stable | 1 | 2400 | 0 / 2400 | 0 / 2400 | 50 / 10000 / 60 | +| trading-system.live.stable | 1 | 1500 | 0 / 1500 | 0 / 1500 | 50 / 2000 / 50 | +| trading-system.sandbox.stable | 2 | 1500 | 0 / 1500 | 0 / 1500 | 50 / 2000 / 50 | + +50 active subscriptions observed on Stream1, zero on Stream2; role limit384 subscriptions +32 replay RPCs each. Slots remaining334/384 at those instants are technical slot counts, **not a promise of334 more consumers**. Per-identity active occupancy is not exported in these sampled metrics; do not infer all50 belong to one identity. If a60-stream alpha identity uses2-5 subscriptions per process, theoretical per-replica identity quota supports12-30 such processes before other requirements; not a load certificate. The prior50-logical-alpha+TS60 acceptance in candidate budget is historical evidence, not a rerun or current5-vCPU PASS. + +Current container CPU ceilings sum 16.50 vCPU. They are burst ceilings, not observed demand and not a shared hard cap. Sum of per-role sampled cgroup peaks 9.10 GiB is conservative/non-simultaneous; sum caps 20.91 GiB. CPU budget headroom against5 is **negative0.962 vCPU**. No extra consumer admission capacity can be certified under that budget now. [Capacity envelope](p184-idr062-capacity-envelope.md) distinguishes measured active pair,170snapshot/s scenario and downstream estimates; no resource cap changed. + +## F. Stability And Recovery + +Verdict: PASS for unchanged process/restart inventory during observation; FAIL for an uninterrupted-stream claim; rc2 fault recovery NOT_MEASURED. + +| Role | Current process start UTC | Current recorded restart count | +| --- | --- | ---: | +| ingestor_binance_usdm | 2026-10-02T12:24:58.369037654Z | 0 | +| query_kn_1 | 2026-10-02T11:13:49.101943789Z | 0 | +| query_kn_2 | 2026-10-02T11:12:13.488673234Z | 0 | +| market_projector_1 | 2026-10-01T12:16:20.78036445Z | 0 | +| market_projector_2 | 2026-10-01T12:15:13.766866129Z | 0 | +| stream_kn_1 | 2026-10-01T12:10:50.691735492Z | 0 | +| stream_kn_2 | 2026-10-01T12:07:24.322915897Z | 0 | +| ingestor_okx_swap | 2026-10-01T05:51:24.840836395Z | 0 | +| kafka3 | 2026-10-01T03:58:43.963843285Z | 0 | +| kafka2 | 2026-10-01T03:57:34.401411155Z | 0 | +| kafka1 | 2026-10-01T03:56:13.899839854Z | 0 | +| rust_core | 2026-09-30T17:46:56.312111099Z | 0 | +| rust_core_2 | 2026-09-30T17:46:23.058376424Z | 0 | +| rust_core_3 | 2026-09-30T17:45:38.534537186Z | 0 | +| binance_bar_edge | 2026-10-01T04:03:57.875233715Z | 3 | +| market_cache | 2026-09-27T05:29:58.000340326Z | 0 | +| stable_redis | 2026-09-16T06:38:55.588316335Z | 0 | + +These start timestamps are process uptime, not duration of error-free eligible data. Stream1 metrics60 samples/approx10min:449,679 delivered, overflow+4, expired/refused/reader/decode/duplicate deltas0. Stream2 delivered0,overflow delta0. Historical counters remain in evidence; an idle replica is not exercised failover. + +BAR edge18s restart is **inherited historical measurement** at main plan line43467, not current rc2 fault testing or a bound for all roles. Recovery under broker failure, restart/rollback rc2, rolling continuity and alpha consumer cache latency: NOT_MEASURED here, no runtime fault injected. Paired replay used an isolated broker and source capture, not production HA. + +## G. Handoff To TS + +**TS can rely on now:** existing versioned Query/gRPC interfaces and r22 product identities; strict fail-closed typed quality; preserved source timestamps/units/cursor isolation; observed119 usable snapshot reads with measured latency; native top-change fix correctness on the active ETH pair. Continue coding against these contracts without waiting for unrelated Paper construction. + +**TS must not rely on yet:** rc2 already deployed; all60 feeds always eligible; <=5vCPU capacity; continuous Stream/TS cache freshness during this window; inverse entitlement inferred from linearTS60; daily-universe history being complete before listing; quiet heartbeat supplying a new price; independent HA/zero downtime; exact E2E-to-alpha/Redis-ACK or canonical delivery lag that was not measured. + +**Release boundary:** attach DR01-DR08 including attribution/closure in DR07 and broker delta. Existing CPU gate fails before Phase C, so hold and present the measured mismatch for owner decision, not silently raise budget. Post-core TS coherence remains after the three Rust cores, not a precondition that old core must pass. No TS Paper feature building added to the release dependency. + +## Cleanup, Reproducibility And Limitations + +All commands were reads except private evidence files and temporary isolated replay resources. No production container ID/image/config/offset/manifest/credential change; no order. Capture hashed then deleted; exact isolated containers/network/TLS keys cleaned. Bounded diagnostic cache payload hashed then deleted; no raw cursor-bearing payload committed. Two failed pytest invocations (host and existing candidate lack pytest) were recorded; no package install/build. The one disposable test container used --rm. Source tests inherited unchanged; receipt consistency assertions passed. + +Docker inventory after:92 images28.19GB; BuildKit227 entries16.83GB (1.089GB reclaimable), no new builds/images. Active+rc2+explicit rollback images retained; broad prune intentionally not done under read-only task. Pair disk free102468628480 ->100905811968 bytes includes concurrent host use, not claimed reclaimed bytes. No test process is left running. + +External compact evidence hashes (not raw credentials/log dumps): + +| File | SHA256 | +| --- | --- | +| `active-pair/paired-receipt.json` | `381760f55659bb5b98879526d7100e118f3c1cbb82e9b05674d5b28579105aa4` | +| `active-pair/cleanup.json` | `a2fa60030476010b8a6453d9c6f2ac6e3a8cd849fa16d2e284ce33ebb901e4e0` | +| `resources-summary.json` | `06ad61ff7b202733830640c934be0473f168c86d40bbd71fbd4645ba441b0dca` | +| `resource-samples.jsonl` | `5313cab1e7fc5abc1f533f33b1dc74c75ca8a9ee20642d712b3d297dabf2603e` | +| `inventory-before.json` | `68dee779e1eb4f1471c02af33da78e926be80979749f4eedb2ac9bda9a2eaae9` | +| `inventory-after.json` | `cf1ffe56f1eebd38b1e5acdce9da3f050a5e022107447909089378e94f9b69b2` | +| `requests-tls.json` | `86bbd396369b424559340e960d97071bd19753ac4c3203ad98684e46f3b6a608` | +| `requests.json` | `860d529b7bf5535a2c8cfbead981ee22dcab1435f5c39a2a4a89a6b0304162a0` | +| `request-alias-check.json` | `dd6dc81b86efe566d24552fed6e976f7c26f8495ec1830ed11d24e8a1ee86354` | +| `cache-observation-bounded.json` | `96c63c629a23bb82afe257b900b1a04b03b9165fe62489c51ac93d8670bc9689` | +| `cache-observation.stderr` | `8f7426a70db8c2bfaa2203fabc6664cdb42092600b72808c4c412fe6d65a0861` | +| `cache-poll.json` | `1ad12f4d5ec633b3b83ca8be5246fee523f599d304f13f3a2a884d1c210ca04d` | +| `quota-before-corrected.json` | `de195058533dc2deb810f2f140f314e149814325c4d599b6b30ad2458378964c` | +| `quota-after.json` | `1e1c75b1b5b80c6ba2bf76701ef3fa4347028aea352fd31cc5ceb3649ea63c69` | +| `kafka-lag.json` | `a0a8443c3e77daec409753398b5cf628af8618dc3ef0cfa8fdf0ba074c0e6849` | +| `logs-summary.json` | `341904e43f147d274ef92e4bd471a7b82ebee856b10c871b2cb666e726255d0f` | +| `logs-examples.json` | `e7974240928a9d35ebad6bb93fb7533ee6740662a3c4a971db31e45992abfe3e` | +| `stream-metrics.json` | `a31429b948e3c1aab322c32d14c052779c09084be8ff8418ca538868b5051071` | +| `stream-limits.json` | `a4083513ee1366853ba0612ec5c135401b6c77886d2815235dc033d4ccf1a07d` | +| `process-rss.json` | `4c60eadebe7b2eb23cc7696381b2968580384c6a4e5999d239c3a2a41cc75ff2` | +| `ts-binding.json` | `aa85d6cf41011da8f05fac335e4b642a7d9f0a3157a660b464fb19f827172159` | +| `binding-projector-status.json` | `92971c3061422b41b29ef1861fa078ed155a3d2b0d38dcf66dcd579b1725b6b5` | + +TLS alias negative diagnostic SHA256 `1fb11d834c348a68604ab2289d5656af62ac2ad81f5b961c81526935b79c2d96`: explicit CERTIFICATE_VERIFY_FAILED / hostname mismatch for query_kn_1, not provider unavailability. From 24ff39b688173e41b09f6d71594d400d724cb6a7 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 14:43:07 +0000 Subject: [PATCH 013/147] fix(observability): bound and correlate TS cache delivery measurements --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 61 +++++ scripts/measure_kn_ts_cache_delivery.py | 244 ++++++++++++++---- tests/test_kn_ts_cache_delivery_observer.py | 188 ++++++++++++++ .../evidence/p184-dr07-hardening-progress.md | 119 +++++++++ 4 files changed, 559 insertions(+), 53 deletions(-) create mode 100644 tests/test_kn_ts_cache_delivery_observer.py create mode 100644 upgrade/evidence/p184-dr07-hardening-progress.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 2ffdd854..9138abe8 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69359,3 +69359,64 @@ other host activity. Canonical main5bf5cd5 untouched; same active feature worktr No runtime rollout, push or merge. Stop and return readiness/packet for owner review; do not continue into TS Paper features or roll a candidate against failing baseline. Final audit checks PASS:60unique binding products x2replicas=120reads (119usable);1800cache GETs/zero missing; report60-row coverage and local links valid; qdl-idr062 containers/networks zero. TLS negative explicitly confirms hostname mismatch; no TLS bypass. Readiness measurements are not a replacement certificate. + +### DR07 hardening - owner approval 2026-10-07 + +Status: IN_PROGRESS. Owner approved the five-part corrective direction after +readiness slice 2; existing DR07/Phase C scope, not a new phase. Goal: attribute +and remove internal delivery/backpressure defects and obtain four-boundary latency +and CPU evidence. Guide: architecture review 18.13-18.14 and existing Phase C packet. +Order: bounded offset-correlated trace + CPU profile -> reproduce responsible stage +-> narrow tested fix -> isolated replay/fault acceptance -> converged candidate and +approved core-first rollout/coherence/remaining roles. Missing timestamps remain +NOT_MEASURED; no PUBSUB/callback substitution for Redis ACK. TRADE quiet vs lag +requires contemporaneous provider identity; preserve strict LAST/intent dependencies. +Invariants: no endpoint/schema/realm/quota/freshness change, bigger buffer, order, +offset reset/state deletion or durability reduction. Observation/isolated tests +first; Phase C HOLD until resource gate, DR01-DR08/broker delta and packet approval. +No automatic budget increase or dependency on TS Paper feature construction. +Tests: identity/time/ordering negatives, lossless replay/monotonic ACK, slow-consumer +isolation, bounded recovery and same-capture CPU/latency. No full catalogue/C2 to debug. +Rollback: active digests/config in packet; no runtime action yet. Cleanup exact test +resources, retain active/rc2/rollback. Journal each coherent tested slice. +Concurrent TS Paper changes owned by Claude are preserved; delegated read-only +BOOK_DELTA/ACK review, no edits to those files. apply_patch helper failed (mountinfo); +exact file writes with immediate diff review follow workspace rule32. + +DR07 investigation checkpoint: delegated source reproducer confirmed64BOOK deltas +perform64MGET+64CAS EVAL sequentially, and reset control+new-generation delta in +one batch yields delta before Query anchor (batch1 hides it). These are source +findings, not proven cause of historical22s observation. TS plan records narrow +ordering fix/test slice; concurrent Paper source remains untouched. +Read-only31.801s thread profile: TS Python1.507vCPU, broker network threads about +2.069vCPU combined; not a function-level profile. No CPU configuration changes. +Observer fix separates PUBSUB drain from batched MGET and requires matching view +for correlated samples.19 stdlib regression tests passed; runtime observation pending. + +### DR07 slice - bounded cache observer and paired TRADE attribution + +Status: IMPLEMENTED_TESTED_DIAGNOSTIC / DR07_IN_PROGRESS; Phase C remains HOLD. +Report: [DR07 hardening progress](upgrade/evidence/p184-dr07-hardening-progress.md). +Observer fixes its own serial GET/backlog and false event-to-current-cache matching; +MARK pairs use existing component lineage, never a fabricated scalar cursor. +25 stdlib tests PASS, diff-check PASS. Two read-only60s windows retained; second +36659notifications/80groups,106MGET,p50/p95/max26.298/189.489/294.093ms. +Second result INCOMPLETE because2oversize notifications; not a full-latency gate. +BOOK_DELTA current-cache ages max1469.907ms in this window, not proof old22.825s fixed. +Independent public BNB comparison75.189s:278cache samples per venue;719Binance/33OKX +public events. Stale>3s: OKX141 (132same last ID/time,8provider-newer,1unattributed), +Binance10 (10same ID/time). No quiet inference for unobserved original refusal. +First attribution attempt had wrong Binance product/channel; preserved as failed +measurement, corrected from production config; no cache/config mutation. +CPU thread attribution, original observer limitations and all hashes in report. +No runtime rollout, order, new image, push/merge, shared Redis write or cleanup +of active/rollback artifacts. Read-only processes finished; no new DL resources. +Remain: function-level CPU evidence/optimization, overflow/backlog attribution, +actual commit/Redis ACK boundaries, isolated lossless/fault acceptance and approved +core-first Phase C. Narrow TS reset fix is separately tested/journaled. + +CPU profiling limit: a15s49Hz user-space perf sample on Kafka2 was denied by +perf_event_paranoid=4/missing CAP_PERFMON. No sysctl/capability change or retry +through elevated credentials. Thread attribution remains valid but insufficient +for a function-level CPU root cause. Empty perf output removed; command/error +receipt retained externally. This is not evidence that optimization is impossible. diff --git a/scripts/measure_kn_ts_cache_delivery.py b/scripts/measure_kn_ts_cache_delivery.py index f210b56e..04549a6f 100644 --- a/scripts/measure_kn_ts_cache_delivery.py +++ b/scripts/measure_kn_ts_cache_delivery.py @@ -36,70 +36,208 @@ def q(p): "p99_ms": q(.99) if len(x) >= 100 else None, "max_ms": q(1)} -async def main(): - import redis.asyncio as redis - seconds = int(os.environ.get("KN_OBSERVE_SECONDS", "300")) - if not 1 <= seconds <= 600: - raise ValueError("bounded observation requires1..600 seconds") - client = redis.from_url(os.environ["KN_TS_REDIS_URL"], socket_timeout=5, - socket_connect_timeout=5, max_connections=3) - pub = client.pubsub() - await pub.psubscribe("events.market.v2.*") - samples, reads, counts, errors, eligible = defaultdict(list), defaultdict(list), Counter(), Counter(), Counter() - last = {} +def compare_observation(channel, payload, current, now_ms): + """Only identical source observations may supply matched delivery timing.""" + _, feed, event_ms, watermark = describe(channel, payload) + _, current_feed, current_ms, current_watermark = describe(channel, current) + if (current.get("canonical_instrument_id") != payload.get("canonical_instrument_id") + or current_feed != feed or current.get("interval") != payload.get("interval")): + raise ValueError("IDENTITY_MISMATCH") + for stamp in (event_ms, current_ms): + if type(stamp) is not int or stamp <= 0 or stamp > now_ms: + raise ValueError("INVALID_OR_FUTURE_TIMESTAMP") + def reference_pair(value): + observations = value.get("contract", {}).get("reference_observations") + if not isinstance(observations, list) or len(observations) != 1: + raise ValueError("REFERENCE_PAIR_MISSING") + labels = observations[0].get("labels", {}) + pair = [] + for name in ("mark", "index"): + capture = labels.get(f"component_{name}_capture_id") + if (not isinstance(capture, str) or len(capture) != 32 + or any(c not in "0123456789abcdef" for c in capture)): + raise ValueError("REFERENCE_PAIR_MISSING") + times = [] + for field in ("source_event_time_ns", "received_at_ns"): + raw = labels.get(f"component_{name}_{field}") + if type(raw) not in (str, int) or not str(raw).isascii() or not str(raw).isdecimal(): + raise ValueError("REFERENCE_PAIR_MISSING") + stamp = int(raw) + if stamp <= 0 or stamp / 1e6 > now_ms: + raise ValueError("INVALID_OR_FUTURE_TIMESTAMP") + times.append(stamp) + price = value.get("payload", {}).get(f"{name}_price") + if price is None: + raise ValueError("REFERENCE_PAIR_MISSING") + pair.append((capture, *times, price)) + generation = labels.get("connection_generation") + if type(generation) not in (str, int) or not str(generation).isascii() or not str(generation).isdecimal() or int(generation) <= 0: + raise ValueError("REFERENCE_PAIR_MISSING") + return (int(generation), pair) + if feed == "MARK_INDEX_PRICE": + previous_pair, current_pair = reference_pair(payload), reference_pair(current) + elif any(type(offset) is not int or offset < 0 for offset in (watermark, current_watermark)): + raise ValueError("WATERMARK_MISSING") + def context(value): + raw = value.get("raw", {}).get("v2", {}) + contract = value.get("contract", raw.get("contract", {})) + body = value.get("payload", raw.get("payload", {})) + return (contract.get("config_revision"), contract.get("instrument_catalog_revision"), + contract.get("authority_revision"), contract.get("source_policy_revision"), + value.get("source", raw.get("source")), value.get("metadata_version"), + body.get("book_generation")) + if context(current) != context(payload): + match = "CONTEXT_CHANGED" + elif feed == "MARK_INDEX_PRICE": + # A paired reference has no scalar cursor. Never infer ordering across + # independently advancing components or manufacture a watermark. + if previous_pair != current_pair: + match = "PAIR_CHANGED_AT_READ" + elif current_ms != event_ms: + raise ValueError("EVENT_IDENTITY_MISMATCH") + else: + match = "EXACT" + elif current_watermark < watermark: + raise ValueError("WATERMARK_REGRESSION") + elif current_watermark != watermark: + match = "SUPERSEDED_AT_READ" + elif current_ms != event_ms: + raise ValueError("EVENT_IDENTITY_MISMATCH") + else: + match = "EXACT" + return {"match": match, "notification_age_ms": now_ms - event_ms, + "current_cache_age_ms": now_ms - current_ms, + "watermark": watermark, "current_watermark": current_watermark} + + +async def observe(client, pub, seconds, *, period=.5, max_channels=128, max_bytes=262144): + # Draining never awaits Redis GET. One bounded MGET samples the latest + # notification per channel; this is observer sampling, not feed coalescing. + pending, counts, errors = {}, Counter(), Counter() + samples, current_ages, notification_ages, residence = (defaultdict(list) for _ in range(4)) + matches, eligible = Counter(), Counter() + admitted_groups = set() + batches = [] started = time.monotonic() - try: - while time.monotonic() - started < seconds: - item = await pub.get_message(ignore_subscribe_messages=True, timeout=.5) + deadline = started + seconds + async def drain(): + received = 0 + while time.monotonic() < deadline: + item = await pub.get_message(ignore_subscribe_messages=True, timeout=.25) if not item: + await asyncio.sleep(0) continue channel = item["channel"].decode() - now = time.monotonic() + if channel not in counts and len(counts) >= max_channels: + errors["CHANNEL_LIMIT"] += 1 + continue counts[channel] += 1 - if now - last.get(channel, 0) < .5: + if len(item["data"]) > max_bytes: + errors["MESSAGE_SIZE_LIMIT"] += 1 continue - last[channel] = now - try: - payload = json.loads(item["data"]) - key, feed, event_ms, watermark = describe(channel, payload) - group = payload["venue"] + "/" + payload["venue_symbol"] + "/" + feed - t0 = time.perf_counter_ns() - current_raw = await client.get(key) - t1 = time.perf_counter_ns() - if current_raw is None: + pending[channel] = (item["data"], time.monotonic()) + received += 1 + if received % 128 == 0: + await asyncio.sleep(0) + task = asyncio.create_task(drain()) + try: + while time.monotonic() < deadline: + await asyncio.sleep(min(period, max(0, deadline - time.monotonic()))) + if task.done(): + await task + selected = list(pending.items()) + pending.clear() + decoded = [] + for channel, (raw, received) in selected: + try: + payload = json.loads(raw) + key, feed, _, _ = describe(channel, payload) + group = payload["venue"] + "/" + payload["venue_symbol"] + "/" + feed + "/" + channel.split(".")[3] + if group not in admitted_groups and len(admitted_groups) >= max_channels: + errors["GROUP_LIMIT"] += 1 + continue + admitted_groups.add(group) + decoded.append((channel, payload, key, group, received)) + except (KeyError, TypeError, ValueError, IndexError): + errors["INVALID_NOTIFICATION"] += 1 + if not decoded: + continue + t0 = time.perf_counter_ns() + values = await client.mget([item[2] for item in decoded]) + elapsed = (time.perf_counter_ns() - t0) / 1e6 + if len(values) != len(decoded): + raise ValueError("MGET_LENGTH_MISMATCH") + if len(batches) < 2000: + batches.append(elapsed) + now_ms, observed = time.time_ns() / 1e6, time.monotonic() + for (channel, payload, _, group, received), value in zip(decoded, values): + if value is None: errors["CACHE_MISSING:" + group] += 1 continue - current = json.loads(current_raw) - if current["canonical_instrument_id"] != payload["canonical_instrument_id"]: - raise ValueError("identity changed at cache readback") - current_watermark = current.get("watermark_offset", current.get("raw", {}).get("v2", {}).get("watermark_offset")) - if watermark is not None and (current_watermark is None or current_watermark < watermark): - errors["WATERMARK_REGRESSION:" + group] += 1 - continue - age = None if event_ms is None else time.time_ns() / 1e6 - event_ms - if age is not None and age < 0: - errors["FUTURE_TIMESTAMP:" + group] += 1 - continue - if len(reads[group]) < 2000: - reads[group].append((t1 - t0) / 1e6) - if age is not None: - samples[group].append(age) - quality = payload.get("quality", payload.get("raw", {}).get("quality", {})) - eligible[group + ":" + str(quality.get("execution_eligible"))] += 1 - except (KeyError, ValueError, TypeError) as error: - errors[type(error).__name__] += 1 + try: + current = json.loads(value) + result = compare_observation(channel, payload, current, now_ms) + matches[group + ":" + result["match"]] += 1 + if len(current_ages[group]) < 2000: + current_ages[group].append(result["current_cache_age_ms"]) + notification_ages[group].append(result["notification_age_ms"]) + residence[group].append((observed - received) * 1000) + if result["match"] == "EXACT": + samples[group].append(result["notification_age_ms"]) + quality = current.get("quality", current.get("raw", {}).get("quality", {})) + eligible[group + ":" + str(quality.get("execution_eligible"))] += 1 + except (KeyError, TypeError, ValueError) as error: + code = str(error) if str(error) in { + "IDENTITY_MISMATCH", "INVALID_OR_FUTURE_TIMESTAMP", "WATERMARK_MISSING", + "WATERMARK_REGRESSION", "EVENT_IDENTITY_MISMATCH", "REFERENCE_PAIR_MISSING"} else "INVALID_CACHE_VIEW" + errors[code] += 1 + except Exception as error: + errors["OBSERVER_FAILED:" + type(error).__name__] += 1 + finally: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception as error: + errors.setdefault("OBSERVER_FAILED:" + type(error).__name__, 1) + if not current_ages: + errors["NO_VALID_CACHE_OBSERVATIONS"] += 1 + return {"schema": "qdl.kn5.actual-ts-cache-availability.v2", + "seconds": time.monotonic() - started, + "status": "INCOMPLETE" if errors else "OBSERVED", + "boundary": "source event/BAR close -> MGET of identical source observation (watermark or MARK pair); availability upper bound, NOT Redis SET ACK", + "replica_attribution": "TS Redis only; no Query replica attribution", + "sampling": {"mode": "latest notification per channel; time samples, not unique events; execution/last paths separate", + "period_seconds": period, "max_channels": max_channels, + "max_notification_bytes": max_bytes, "max_samples_per_group": 2000}, + "event_to_cache_readable": {k: summary(v) for k,v in samples.items()}, + "current_cache_age_ms": {k: summary(v) for k,v in current_ages.items()}, + "notification_age_ms": {k: summary(v) for k,v in notification_ages.items()}, + "observer_residence_ms": {k: summary(v) for k,v in residence.items()}, + "cache_mget_batch_ms": summary(batches), "observed_messages": dict(counts), + "matches": dict(matches), "eligibility_at_write_not_revalidated": dict(eligible), + "errors": dict(errors), "writes": 0, "order_actions": 0, + "redis_set_ack_latency": "NOT_MEASURED", "canonical_commit_latency": "NOT_MEASURED"} + + +async def main(): + import redis.asyncio as redis + seconds = int(os.environ.get("KN_OBSERVE_SECONDS", "300")) + if not 1 <= seconds <= 600: + raise ValueError("bounded observation requires1..600 seconds") + client = redis.from_url(os.environ["KN_TS_REDIS_URL"], socket_timeout=5, + socket_connect_timeout=5, max_connections=3) + pub = client.pubsub() + try: + await pub.psubscribe("events.market.v2.*") + result = await observe(client, pub, seconds) finally: await pub.aclose() await client.aclose() - print(json.dumps({"schema": "qdl.kn5.actual-ts-cache-availability.v1", "seconds": time.monotonic() - started, - "query_replicas_declared": os.environ.get("KN_QUERY_REPLICAS", "").split(",") if os.environ.get("KN_QUERY_REPLICAS") else [], - "replica_attribution": "observer reads TS Redis only; declared names are packet metadata, not per-replica measurements", - "boundary": "source event (BAR close) -> external GET after actual TS Redis SET/PUBLISH; not publisher ACK", - "sampling": "at most2Hz per channel; max2000 samples per product; no source event dedup", - "event_to_cache_readable": {k: summary(v) for k,v in samples.items()}, - "cache_get": {k: summary(v) for k,v in reads.items()}, "observed_messages": dict(counts), - "eligibility": dict(eligible), "errors": dict(errors), "writes": 0, "order_actions": 0})) + print(json.dumps(result)) + return 0 if result["status"] == "OBSERVED" else 1 if __name__ == "__main__": - asyncio.run(main()) + raise SystemExit(asyncio.run(main())) diff --git a/tests/test_kn_ts_cache_delivery_observer.py b/tests/test_kn_ts_cache_delivery_observer.py new file mode 100644 index 00000000..63a388c2 --- /dev/null +++ b/tests/test_kn_ts_cache_delivery_observer.py @@ -0,0 +1,188 @@ +"""Observer-only fixtures, never provider/certification data.""" +import asyncio +from copy import deepcopy +import importlib.util +import json +from pathlib import Path +import time +import unittest + +spec = importlib.util.spec_from_file_location("observer", Path(__file__).parents[1] / "scripts/measure_kn_ts_cache_delivery.py") +observer = importlib.util.module_from_spec(spec) +spec.loader.exec_module(observer) +CHANNEL = "events.market.v2.execution.book_delta.OKX.SWAP.ETH-USDT-SWAP" + + +def view(offset=3, stamp=None): + return {"canonical_instrument_id": "OKX:SWAP:ETH-USDT-SWAP", "venue": "OKX", + "venue_symbol": "ETH-USDT-SWAP", "feed": "BOOK_DELTA", + "observed_at_ms": stamp or int(time.time() * 1000) - 100, + "watermark_offset": offset, "payload": {"book_generation": 7}, + "contract": {"config_revision": 20}, "quality": {"execution_eligible": True}} + + +class ComparisonTests(unittest.TestCase): + def test_exact_view(self): + v=view(stamp=1000) + self.assertEqual(observer.compare_observation(CHANNEL,v,v,1200)["match"],"EXACT") + + def test_newer_cache_is_not_matched_delivery(self): + result=observer.compare_observation(CHANNEL,view(3,1000),view(4,1900),2000) + self.assertEqual(result["match"],"SUPERSEDED_AT_READ") + self.assertEqual(result["current_cache_age_ms"],100) + self.assertEqual(result["notification_age_ms"],1000) + + def test_offset_regression(self): + with self.assertRaisesRegex(ValueError,"WATERMARK_REGRESSION"): + observer.compare_observation(CHANNEL,view(4),view(3),time.time()*1000) + + def test_generation_reset_is_not_offset_regression(self): + old=view(100);new=view(2);new["payload"]["book_generation"]=8 + self.assertEqual(observer.compare_observation(CHANNEL,old,new,time.time()*1000)["match"],"CONTEXT_CHANGED") + + def test_config_change(self): + old=view();new=deepcopy(old);new["contract"]["config_revision"]=21 + self.assertEqual(observer.compare_observation(CHANNEL,old,new,time.time()*1000)["match"],"CONTEXT_CHANGED") + + def test_authority_change_cannot_be_matched(self): + old=view();new=deepcopy(old);new["contract"]["authority_revision"]=99 + self.assertEqual(observer.compare_observation(CHANNEL,old,new,time.time()*1000)["match"],"CONTEXT_CHANGED") + + def test_same_offset_changed_event(self): + with self.assertRaisesRegex(ValueError,"EVENT_IDENTITY_MISMATCH"): + observer.compare_observation(CHANNEL,view(3,1000),view(3,1001),2000) + + def test_identity_mismatch(self): + old=view();new=deepcopy(old);new["canonical_instrument_id"]="OKX:SWAP:BTC-USDT-SWAP" + with self.assertRaisesRegex(ValueError,"IDENTITY_MISMATCH"): + observer.compare_observation(CHANNEL,old,new,time.time()*1000) + + def test_offsets_must_be_integers(self): + for bad in [None,True,-1,"3"]: + v=view();v["watermark_offset"]=bad + with self.subTest(bad=bad),self.assertRaisesRegex(ValueError,"WATERMARK_MISSING"): + observer.compare_observation(CHANNEL,v,v,time.time()*1000) + + def test_future_event(self): + v=view(stamp=2000) + with self.assertRaisesRegex(ValueError,"INVALID_OR_FUTURE_TIMESTAMP"): + observer.compare_observation(CHANNEL,v,v,1999) + + def test_bar_close(self): + v={"canonical_instrument_id":"OKX:SWAP:ETH-USDT-SWAP","timestamp_ms":1000, + "interval":"1m","watermark_offset":1} + self.assertEqual(observer.describe("events.market.v2.bar.OKX.SWAP.ETH-USDT-SWAP.1m",v)[2],61000) + v["interval"]="1h" + with self.assertRaises(ValueError):observer.describe("events.market.v2.bar",v) + + def mark(self): + v=view(stamp=1000);v["feed"]="MARK_INDEX_PRICE";v["watermark_offset"]=None + labels={"connection_generation":"1"} + for name in ("mark","index"): + labels.update({f"component_{name}_capture_id":"a"*32, + f"component_{name}_source_event_time_ns":"1000000000", + f"component_{name}_received_at_ns":"1010000000"}) + v["contract"]["reference_observations"]=[{"labels":labels}] + v["payload"]={"mark_price":"12.1","index_price":"12.2"} + return v + + def test_mark_pair_without_scalar_watermark(self): + v=self.mark() + self.assertEqual(observer.compare_observation(CHANNEL,v,v,1200)["match"],"EXACT") + + def test_mark_one_component_changed_is_not_matched(self): + old=self.mark();new=deepcopy(old) + new["contract"]["reference_observations"][0]["labels"]["component_index_capture_id"]="b"*32 + self.assertEqual(observer.compare_observation(CHANNEL,old,new,1200)["match"],"PAIR_CHANGED_AT_READ") + + def test_mark_price_changed_is_not_matched(self): + old=self.mark();new=deepcopy(old);new["payload"]["mark_price"]="13" + self.assertEqual(observer.compare_observation(CHANNEL,old,new,1200)["match"],"PAIR_CHANGED_AT_READ") + + def test_mark_missing_component_proof(self): + v=self.mark();v["contract"]["reference_observations"][0]["labels"].pop("component_mark_capture_id") + with self.assertRaisesRegex(ValueError,"REFERENCE_PAIR_MISSING"): + observer.compare_observation(CHANNEL,v,v,1200) + + def test_mark_future_component(self): + v=self.mark();v["contract"]["reference_observations"][0]["labels"]["component_index_received_at_ns"]="2000000000" + with self.assertRaisesRegex(ValueError,"INVALID_OR_FUTURE_TIMESTAMP"): + observer.compare_observation(CHANNEL,v,v,1200) + + def test_p99_requires_100(self): + self.assertIsNone(observer.summary(list(range(99)))["p99_ms"]) + self.assertIsNotNone(observer.summary(list(range(100)))["p99_ms"]) + + +class Pub: + def __init__(self, items):self.items=list(items);self.reads=0 + async def get_message(self, **kwargs): + await asyncio.sleep(.001) + if self.items: + self.reads+=1 + return self.items.pop(0) + return None + + +def message(v, channel=CHANNEL):return {"channel":channel.encode(),"data":json.dumps(v).encode()} + + +class Client: + def __init__(self,v):self.value=v;self.calls=0 + async def mget(self,keys): + self.calls+=1 + return [json.dumps(self.value) if self.value else None for _ in keys] + + +class AsyncTests(unittest.IsolatedAsyncioTestCase): + async def test_reads_are_batched(self): + v=view();client=Client(v);pub=Pub([message(v) for _ in range(20)]) + result=await observer.observe(client,pub,.06,period=.01) + self.assertEqual(result["status"],"OBSERVED") + self.assertLess(client.calls,pub.reads) + self.assertTrue(result["event_to_cache_readable"]) + self.assertEqual(result["writes"],0) + self.assertEqual(result["redis_set_ack_latency"],"NOT_MEASURED") + + async def test_drain_continues_during_slow_mget(self): + v=view();pub=Pub([message(v) for _ in range(30)]) + class Slow(Client): + async def mget(self,keys): + await asyncio.sleep(.045) + self.reads_while_waiting=pub.reads + return await super().mget(keys) + client=Slow(v) + await observer.observe(client,pub,.08,period=.005) + self.assertEqual(client.reads_while_waiting,30) + + async def test_superseded_is_not_a_latency_sample(self): + old=view(3);new=view(4) + result=await observer.observe(Client(new),Pub([message(old)]),.03,period=.005) + self.assertEqual(result["event_to_cache_readable"],{}) + self.assertTrue(result["current_cache_age_ms"]) + + async def test_missing_cache_is_incomplete(self): + result=await observer.observe(Client(None),Pub([message(view())]),.03,period=.005) + self.assertEqual(result["status"],"INCOMPLETE") + + async def test_transport_failure_is_retained(self): + class Broken(Pub): + async def get_message(self,**kwargs):raise ConnectionError("test") + result=await observer.observe(Client(view()),Broken([]),.03,period=.005) + self.assertEqual(result["status"],"INCOMPLETE") + self.assertIn("OBSERVER_FAILED:ConnectionError",result["errors"]) + + async def test_empty_window_is_not_pass(self): + result=await observer.observe(Client(view()),Pub([]),.02,period=.005) + self.assertEqual(result["status"],"INCOMPLETE") + + async def test_size_limit(self): + result=await observer.observe(Client(view()),Pub([message(view())]),.02,period=.005,max_bytes=5) + self.assertIn("MESSAGE_SIZE_LIMIT",result["errors"]) + + async def test_channel_limit(self): + v=view();result=await observer.observe(Client(v),Pub([message(v),message(v,CHANNEL+".extra")]),.03,period=.005,max_channels=1) + self.assertIn("CHANNEL_LIMIT",result["errors"]) + + +if __name__ == "__main__": unittest.main() diff --git a/upgrade/evidence/p184-dr07-hardening-progress.md b/upgrade/evidence/p184-dr07-hardening-progress.md new file mode 100644 index 00000000..2551850a --- /dev/null +++ b/upgrade/evidence/p184-dr07-hardening-progress.md @@ -0,0 +1,119 @@ +# P18.4 DR07 Hardening Progress - 2026-10-07 + +Status: IN_PROGRESS / PHASE_C_HOLD. This is not a release certificate. + +## Scope And Findings + +- Existing DR07 only: CPU attribution, truthful consumer measurement, BOOK reset/recovery, TRADE attribution. No Paper features or Data Layer interface changes. +- Correctness defect reproduced in TS source: a BOOK reset followed by deltas in the same batch could yield the new-generation deltas before the verified Query anchor. Narrow TS adapter fix is under test/review; not deployed. +- Source inspection: 64 BOOK deltas perform 64 MGET and 64 CAS EVAL calls sequentially. This is a performance candidate, not yet proof of the historical Stream overflows or 22.825s cache age. +- Thread CPU attribution over31.801s: TS Python1.507vCPU; Kafka network-named threads2.069vCPU combined. These are thread names, not function stacks; TS is outside the DL5vCPU denominator. No CPU configuration change or claim that the5vCPU gate now passes. A bounded Kafka2 perf sample was denied by perf_event_paranoid=4/missing CAP_PERFMON; no security setting changed. Thread attribution cannot substitute for function-level profiling. + +## Observer Correction + +PUBSUB drain no longer awaits a GET per notification. Bounded batched MGET samples the latest notification per channel. Matched availability requires the same source observation; superseded cache views cannot supply delivery latency. MARK compares component capture/time/price/generation rather than inventing a scalar offset. Authority/context changes cannot match. Execution and last-value paths are reported separately. + +25 stdlib tests PASS: `python3 -B tests/test_kn_ts_cache_delivery_observer.py`. No pytest claim (not installed in host/candidate). Source snapshot used for the second live window precedes the final authority-context negative and sampling-label adjustment; those final diagnostic-only edits are unit-tested, not a new runtime benchmark. + +Second window:60.060s,36659notifications,80cache-path groups,106MGET batches. MGET p50/p95/p99/max26.298/189.489/265.391/294.093ms. Status INCOMPLETE:2notifications exceeded the256KiB observer bound. No missing-MARK-watermark error remains. First diagnostic window is retained (502MARK watermark errors,7oversize messages), not replaced or called a production defect. + +All numbers below are time-sampled **current cache event age**, not request latency, Redis SET ACK, execution eligibility or distinct source-event latency. MARK event age alone does not evaluate component/session eligibility. Samples are notification-driven, not a complete availability census. The read-only observer adds its own process/subscriber overhead; this is not a capacity workload. + +| Venue/instrument/feed/path | n | p50 ms | p95 ms | max ms | Matched availability samples | +|---|---:|---:|---:|---:|---:| +| BINANCE/BNBUSDT/BAR/bar | 1 | 1137.854 | None | 1137.854 | 1 | +| BINANCE/BNBUSDT/BOOK_DELTA/execution | 106 | 270.303 | 784.167 | 938.862 | 68 | +| BINANCE/BNBUSDT/BOOK_SNAPSHOT/execution | 56 | 626.325 | 1174.698 | 1300.583 | 56 | +| BINANCE/BNBUSDT/MARK_INDEX_PRICE/execution | 55 | 1256.764 | 1827.012 | 2169.425 | 55 | +| BINANCE/BNBUSDT/QUOTE/execution | 105 | 311.073 | 614.464 | 792.698 | 76 | +| BINANCE/BNBUSDT/TRADE/execution | 63 | 462.534 | 1030.002 | 1101.675 | 56 | +| BINANCE/BTCUSDT/BAR/bar | 1 | 618.591 | None | 618.591 | 1 | +| BINANCE/BTCUSDT/BOOK_DELTA/execution | 106 | 283.235 | 1122.862 | 1469.907 | 76 | +| BINANCE/BTCUSDT/BOOK_SNAPSHOT/execution | 58 | 612.024 | 1101.152 | 1268.167 | 58 | +| BINANCE/BTCUSDT/MARK_INDEX_PRICE/execution | 53 | 1329.087 | 1716.708 | 2021.862 | 53 | +| BINANCE/BTCUSDT/QUOTE/execution | 106 | 213.282 | 418.123 | 500.583 | 68 | +| BINANCE/BTCUSDT/TRADE/execution | 104 | 336.024 | 707.773 | 917.166 | 79 | +| BINANCE/DOGEUSDT/BAR/bar | 1 | 1137.854 | None | 1137.854 | 1 | +| BINANCE/DOGEUSDT/BOOK_DELTA/execution | 106 | 278.203 | 716.182 | 766.583 | 74 | +| BINANCE/DOGEUSDT/BOOK_SNAPSHOT/execution | 57 | 619.708 | 1119.511 | 1382.583 | 57 | +| BINANCE/DOGEUSDT/MARK_INDEX_PRICE/execution | 55 | 1256.764 | 1956.167 | 2206.511 | 55 | +| BINANCE/DOGEUSDT/QUOTE/execution | 106 | 221.087 | 437.464 | 548.055 | 75 | +| BINANCE/DOGEUSDT/TRADE/execution | 71 | 488.26 | 1091.862 | 1292.708 | 66 | +| BINANCE/ETHUSDT/BAR/bar | 1 | 618.591 | None | 618.591 | 1 | +| BINANCE/ETHUSDT/BOOK_DELTA/execution | 106 | 240.712 | 918.637 | 1285.907 | 72 | +| BINANCE/ETHUSDT/BOOK_SNAPSHOT/execution | 58 | 550.336 | 982.167 | 1230.862 | 58 | +| BINANCE/ETHUSDT/MARK_INDEX_PRICE/execution | 54 | 1237.692 | 1896.29 | 2021.862 | 54 | +| BINANCE/ETHUSDT/QUOTE/execution | 106 | 265.235 | 523.166 | 792.698 | 70 | +| BINANCE/ETHUSDT/TRADE/execution | 104 | 288.708 | 681.595 | 772.235 | 75 | +| BINANCE/SOLUSDT/BAR/bar | 1 | 618.591 | None | 618.591 | 1 | +| BINANCE/SOLUSDT/BOOK_DELTA/execution | 106 | 220.087 | 594.583 | 822.637 | 85 | +| BINANCE/SOLUSDT/BOOK_SNAPSHOT/execution | 57 | 484.055 | 1078.425 | 1145.992 | 57 | +| BINANCE/SOLUSDT/MARK_INDEX_PRICE/execution | 54 | 1298.732 | 1934.339 | 2386.675 | 54 | +| BINANCE/SOLUSDT/QUOTE/execution | 106 | 205.317 | 451.691 | 653.698 | 68 | +| BINANCE/SOLUSDT/TRADE/execution | 93 | 358.073 | 699.698 | 1089.583 | 80 | +| OKX/BNB-USDT-SWAP/BAR/bar | 1 | 1717.953 | None | 1717.953 | 1 | +| OKX/BNB-USDT-SWAP/BOOK_DELTA/execution | 106 | 290.294 | 735.583 | 919.862 | 80 | +| OKX/BNB-USDT-SWAP/BOOK_SNAPSHOT/execution | 59 | 566.907 | 1119.862 | 1235.583 | 59 | +| OKX/BNB-USDT-SWAP/MARK_INDEX_PRICE/execution | 54 | 1175.258 | 1875.583 | 2323.698 | 54 | +| OKX/BNB-USDT-SWAP/QUOTE/execution | 103 | 324.675 | 634.639 | 859.637 | 70 | +| OKX/BNB-USDT-SWAP/TRADE/execution | 17 | 685.008 | None | 8760.912 | 15 | +| OKX/BTC-USDT-SWAP/BAR/bar | 1 | 1717.953 | None | 1717.953 | 1 | +| OKX/BTC-USDT-SWAP/BOOK_DELTA/execution | 106 | 293.406 | 994.992 | 1281.182 | 67 | +| OKX/BTC-USDT-SWAP/BOOK_SNAPSHOT/execution | 59 | 568.971 | 1005.764 | 1215.862 | 59 | +| OKX/BTC-USDT-SWAP/MARK_INDEX_PRICE/execution | 53 | 995.406 | 1391.907 | 1472.708 | 53 | +| OKX/BTC-USDT-SWAP/QUOTE/execution | 106 | 244.235 | 440.654 | 597.123 | 62 | +| OKX/BTC-USDT-SWAP/TRADE/execution | 101 | 354.277 | 669.83 | 847.698 | 81 | +| OKX/DOGE-USDT-SWAP/BAR/bar | 1 | 1137.854 | None | 1137.854 | 1 | +| OKX/DOGE-USDT-SWAP/BOOK_DELTA/execution | 106 | 293.286 | 736.583 | 820.862 | 76 | +| OKX/DOGE-USDT-SWAP/BOOK_SNAPSHOT/execution | 60 | 604.431 | 999.992 | 1286.182 | 60 | +| OKX/DOGE-USDT-SWAP/MARK_INDEX_PRICE/execution | 55 | 1201.949 | 2056.123 | 2142.235 | 55 | +| OKX/DOGE-USDT-SWAP/QUOTE/execution | 106 | 312.503 | 518.691 | 680.698 | 79 | +| OKX/DOGE-USDT-SWAP/TRADE/execution | 40 | 471.645 | 838.425 | 1208.583 | 40 | +| OKX/ETH-USDT-SWAP/BAR/bar | 1 | 1137.854 | None | 1137.854 | 1 | +| OKX/ETH-USDT-SWAP/BOOK_DELTA/execution | 106 | 290.286 | 983.182 | 1264.907 | 75 | +| OKX/ETH-USDT-SWAP/BOOK_SNAPSHOT/execution | 60 | 596.992 | 1053.167 | 1433.583 | 59 | +| OKX/ETH-USDT-SWAP/MARK_INDEX_PRICE/execution | 54 | 962.455 | 1461.166 | 1711.764 | 54 | +| OKX/ETH-USDT-SWAP/QUOTE/execution | 106 | 236.258 | 448.166 | 647.698 | 67 | +| OKX/ETH-USDT-SWAP/TRADE/execution | 96 | 308.306 | 671.862 | 882.583 | 77 | +| OKX/SOL-USDT-SWAP/BAR/bar | 1 | 1137.854 | None | 1137.854 | 1 | +| OKX/SOL-USDT-SWAP/BOOK_DELTA/execution | 106 | 293.732 | 732.583 | 916.862 | 67 | +| OKX/SOL-USDT-SWAP/BOOK_SNAPSHOT/execution | 60 | 639.235 | 1063.907 | 1206.764 | 59 | +| OKX/SOL-USDT-SWAP/MARK_INDEX_PRICE/execution | 54 | 1123.425 | 2042.008 | 2642.595 | 54 | +| OKX/SOL-USDT-SWAP/QUOTE/execution | 106 | 224.406 | 451.654 | 552.583 | 71 | +| OKX/SOL-USDT-SWAP/TRADE/execution | 83 | 418.152 | 916.235 | 1158.583 | 75 | + +## Contemporaneous BNB Trade Attribution + +75.189s,278cache samples per venue;719Binance raw trade events and33OKX trade events, no capture errors. Public frames hashed in memory, not retained. Read-only existing TS cache, no Query identity quota or stream allocation consumed. No canonical commit timestamp measured. + +| Venue | Cache samples >3000ms | Same public trade ID + source time | Public trade newer than stale cache | Unattributed | +|---|---:|---:|---:|---:| +| OKX | 141 | 132 | 8 | 1 | +| BINANCE | 10 | 10 | 0 | 0 | + +These are contemporaneous observations, not a guarantee that a separate provider socket sees every production event. They do not explain the old4262ms refusal retroactively. Across all behind-cache observations, time since the independent public receiver saw the newer event: OKX n15,p50/p95/max156.090/340.280/340.280ms; Binance n64,83.633/940.982/1784.090ms. This is neither production-ingress lag nor a paired commit/apply latency. A newly arrived provider event with stale preceding trade does not itself prove an internal outage. + +First capture attempt retained: Binance used the wrong cache product and aggregate channel while production uses USD_M/raw trade. It supplies no Binance attribution. Corrected from acquisition config (`config/v2/stable-acquisition-bindings.yaml`, native channel `bnbusdt@trade`) before the second window. OKX observations from that first attempt are not mixed into the second denominator. + +## Gates Still Open + +| Gate | Current verdict | +|---|---| +| Whole DL stack <=5vCPU | Previous measured FAIL; not remeasured after a runtime optimization | +| Four Stream overflows | Attribution/fix still open; no buffer increase | +| Historical BOOK_DELTA22.825s | Root cause not proved; reset ordering source defect fixed separately | +| Redis ACK / canonical commit / alpha apply | NOT_MEASURED | +| Consumer usable after candidate rollout | NOT_MEASURED; Phase C still HOLD | + +## Evidence And Runtime Safety + +External root: `/home/bobby/.local/state/qdl-v2/dr07-hardening-20261007/`. Bounded metrics and public trade identifiers only, no credentials/provider raw payloads. + +- `thread-profile.json` sha256 `b73b2d50b48e56d07e20e6efa500ad6108cb5c6b5737a5e9ab583e04b67710e7` +- `observer-candidate.json` sha256 `a67d3605f1a42eed03dffa2f15d11e96bd357b9f97a6407cd46d43797dfcf177` +- `observer-pair-candidate.json` sha256 `afb872d039b3df345acc3de21231a2fd4ff82c5e318ec2ec320c48e9584a1552` +- `trade-attribution.json` sha256 `48df88ec69a01d712f4cb54732b52efd066d16aed62bbb16102350f7e4ae9a5e` +- `trade-attribution-v2.json` sha256 `2826d7791f050f0a48858351adc3db828dd8f0aebc0e6dc8c2a6306e416c9f8c` +- `trade-attribution-v2-summary.json` sha256 `85e39a900326133b8aa2608b8e9ca2769405a758c49756106d7e36a0b213647f` + +No runtime recreation, cap/config/manifest/quota/offset or order mutations. Existing rc2 and active/rollback images retained, no image build or broad prune. Disposable TS reset tests are documented in the TS journal; DL observations leave no container/network/image. Canonical `/home/bobby/data_layer` main5bf5cd5 unchanged; working branch fix/p184-dl-closure; no push/merge. Active role/config/digest and rollback map remains the [Phase C packet](p184-dl-phase-c-packet.md). From 0fb731de7ddd62756a6907211e2be21d292646d6 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 15:09:39 +0000 Subject: [PATCH 014/147] docs(dr07): record isolated TS source fix and reviewer handoff --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 35 +++++++++++++++++++ .../evidence/p184-dr07-hardening-progress.md | 20 +++++++++++ 2 files changed, 55 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 9138abe8..cd102412 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69420,3 +69420,38 @@ perf_event_paranoid=4/missing CAP_PERFMON. No sysctl/capability change or retry through elevated credentials. Thread attribution remains valid but insufficient for a function-level CPU root cause. Empty perf output removed; command/error receipt retained externally. This is not evidence that optimization is impossible. + +DR07 follow-up within approved recovery scope: existing late BOOK_DELTA integration +cases fail even before the reset patch. Trace the SDK OBSERVE result path (returns +noneligible view rather than DATA_STALE exception) through strict projection. +If confirmed, preserve complete authoritative ordered deltas as replay-only facts, +never renew execution eligibility/session/TTL from stale replay. Keep gap, source, +generation and sequence guards and ACK-after-apply. Required tests include all10 +venue/symbol late cases and negatives, plus Query-ahead/reset/replay. Do not treat +pre-existing failures as waived or change the Data Layer freshness contract. + +DR07 owner isolation handoff2026-10-07: stopped delegated edits on Claude's TS +worktree; saved exact adapter patch/test/Codex journal under +/tmp/codex-dr07-move-20261007 (hashed migration-receipt.json). Created branch +codex/dr07-reset-anchor in /home/bobby/.worktrees/ts-codex-dr07 from be0c15f. +Commit58287fcfbd2c06fd714d12ce32ca8497bd63637c contains only adapter, new reset +regressions and TS journal.532affected tests PASS in13.69s in this worktree, +including real isolated Redis and all10formerly failing late cases. Exact trace +showed BOOK projection used market2000ms rather than the BOOK45000ms already +declared by its request; corrected policy selection without new thresholds or +replay exceptions. Reset Query-anchor/ACK ordering also fixed, not deployed. + +After commit, removed only the identical copied adapter/test/Codex journal from +Claude worktree. Owner blank-line deletion preserved; unrelated new risk_pool +and price_match changes preserved. New worktree clean, no push/merge. Claude +reviews58287fc before bringing it into feat/v2-rust-first-okx-demo. +Test evidence: ~/.local/state/qdl-v2/dr07-hardening-20261007/isolated-worktree/ +attempt-2/{commands.json,affected-tests.log,source-receipt.json}. First setup +failed before pytest due missing tmpfs mountpoint; retained, fixed harness only. +Cleanup: RedisDB15zero; ts-codex-dr07-tests network and exact test containers +removed; empty logs mountpoint removed; no build/prune,92images28.19GB and +227BuildKit entries16.83GB retained. Available disk90497032192->90442244096bytes, +concurrent host activity. Production writer f63785a27af8/image9150385e2bb3, +restart0 unchanged. DL canonical main5bf5cd5 and active12-role/config set +unchanged; active/rc2/named rollback retained. DR07 remains IN_PROGRESS; CPU, +causal overflow attribution and commit->RedisACK measurement remain open. diff --git a/upgrade/evidence/p184-dr07-hardening-progress.md b/upgrade/evidence/p184-dr07-hardening-progress.md index 2551850a..73db7135 100644 --- a/upgrade/evidence/p184-dr07-hardening-progress.md +++ b/upgrade/evidence/p184-dr07-hardening-progress.md @@ -117,3 +117,23 @@ External root: `/home/bobby/.local/state/qdl-v2/dr07-hardening-20261007/`. Bound - `trade-attribution-v2-summary.json` sha256 `85e39a900326133b8aa2608b8e9ca2769405a758c49756106d7e36a0b213647f` No runtime recreation, cap/config/manifest/quota/offset or order mutations. Existing rc2 and active/rollback images retained, no image build or broad prune. Disposable TS reset tests are documented in the TS journal; DL observations leave no container/network/image. Canonical `/home/bobby/data_layer` main5bf5cd5 unchanged; working branch fix/p184-dl-closure; no push/merge. Active role/config/digest and rollback map remains the [Phase C packet](p184-dl-phase-c-packet.md). + +## TS Source Handoff Update + +Owner moved Codex work to `/home/bobby/.worktrees/ts-codex-dr07`, branch +`codex/dr07-reset-anchor`, base `be0c15f`, commit +`58287fcfbd2c06fd714d12ce32ca8497bd63637c`. Claude's Paper worktree is no longer +used for Codex edits/tests. Copied changes removed only after this tested commit; +owner blank-line edit and concurrent risk/price-match work preserved. + +532affected tests PASS in13.69s, including the10late cases that initially failed. +Their exact source cause was mismatched policy wiring: BOOK request45000ms, +projection incorrectly using market2000ms. Final code selects the declared BOOK +limit; TRADE and execution freshness unchanged. No new replay bypass was added. +Query-ahead/reset replay preserves cache and TTL, never ACKs Query watermark; +equal-sequence conflict fails closed. Real disposable Redis, synthetic transport. +No runtime throughput/recovery or release certification is inferred. + +Evidence/cleanup receipt: external `isolated-worktree/attempt-2/` under the root +above; migration `/tmp/codex-dr07-move-20261007/migration-receipt.json`. New worktree +clean, no push/merge/deploy. The historical overflow cause remains unproved. From 63238bf7098cdb7131187e45489d0044dff8246b Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 16:19:10 +0000 Subject: [PATCH 015/147] perf(core): avoid whole-book copies after validated L2 deltas --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 83 +++++++++++ rust/qdl-core/src/l2_book.rs | 133 +++++++++++++++++- scripts/measure_l2_apply_cpu.py | 96 +++++++++++++ scripts/support/l2_apply_probe.rs | 105 ++++++++++++++ upgrade/evidence/p184-dl-phase-c-packet.md | 14 ++ .../evidence/p184-dr07-hardening-progress.md | 47 +++++++ 6 files changed, 471 insertions(+), 7 deletions(-) create mode 100644 scripts/measure_l2_apply_cpu.py create mode 100644 scripts/support/l2_apply_probe.rs diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index cd102412..eeecaf6c 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69455,3 +69455,86 @@ concurrent host activity. Production writer f63785a27af8/image9150385e2bb3, restart0 unchanged. DL canonical main5bf5cd5 and active12-role/config set unchanged; active/rc2/named rollback retained. DR07 remains IN_PROGRESS; CPU, causal overflow attribution and commit->RedisACK measurement remain open. + +### DR07 continuation - bounded delivery trace and CPU attribution (2026-10-07) + +Status: IN_PROGRESS; owner reconfirmed the approved five-part attachment. +Scope/guide: existing DR07 hardening, architecture sections18.13-18.14 and Phase C +packet. No new phase or Paper dependency. Keep reset slice58287fc for Claude review. +TS changes only in ts-codex-dr07/codex/dr07-reset-anchor; never edit Claude's +trading-system-next-upgrade worktree. Next slice: optional bounded internal +measurement of existing apply/Redis ACK boundary with source identity, offsets, +monotonic durations and honest missing commit/receive timestamps. No public schema +change, new service, default hot-path logging or false PUBSUB-as-ACK claim. +Tests: same apply semantics tracing off/on, negative/timeout/replay/no-ACK cases, +bounded observer overhead and real isolated Redis. CPU: inspect existing bounded +metrics/config; no sysctl/privilege, resource cap or durability change. Profile +and optimize only with evidence. No rollout without converged packet; active +digests/config unchanged, source revert is rollback. Journal/commit each tested +slice; remove only exact test resources. No push/merge and no whole-catalogue/C2. + +CPU slice pre-edit evidence: isolated TEST_ONLY synthetic L2 microbenchmark, +existing rust:1.83-slim image, no build/network/runtime mutation. Five alternating +pairs per depth, 2,000 deltas/pair, 10 updates/delta, identical final view hashes. +At depth400 each side, old clone-every-delta median324.303ms versus experimental +in-place25.614ms; allocations/delta2790 versus258. At depth20:30.115/16.400ms; +depth1000:852.838/24.977ms. Evidence: dr07-hardening-20261007/l2-cpu/. +CPU optimization removes only two whole-book clones after all fallible +validation/parsing completes. Exclusive &mut ownership, synchronous infallible +update loop, no intermediate publication. Keep decimal comparator, protocol +checks, snapshot frequency, Kafka durability and output unchanged. Validate +existing golden fixture and baseline/candidate action parity including invalid +updates/checksum/gap/generation/replay. Microbenchmark is not real-provider or +full-stack capacity evidence; <=5vCPU remains UNPROVEN. + +## DR07 CPU Source Slice - 2026-10-07 + +Status: TESTED_LOCALLY, not deployed, Phase C remains HOLD. Profiling already +showed core normalization cost but did not provide a function-level CPU profile. +A controlled TEST_ONLY allocation microbenchmark now isolates whole-book cloning +on every L2 delta. All validations complete before the synchronous update loop; +exclusive Rust ownership allows updating both sides without copying untouched +levels. No changes to comparator, decimal precision, integrity/sequence/generation, +publish cadence, Kafka durability or provider semantics. + +Reproducer: `python3 scripts/measure_l2_apply_cpu.py --baseline-ref 0fb731d +--output `. Existing compiler image only, +network none, 1 CPU / 512 MiB ceiling, alternating baseline/candidate order. +Each sample applies 2,000 deltas of 10 updates. Five samples per side/depth; +these are elapsed times with allocation instrumentation, NOT vCPU or live latency. + +| Levels per side | Baseline median / max (ms) | Candidate median / max (ms) | Allocations/delta old -> new | +| --- | --- | --- | --- | +| 20 | 35.635 / 38.471 | 15.646 / 18.101 | 280 -> 152 | +| 400 | 330.114 / 359.657 | 24.486 / 27.836 | 2790 -> 258 | +| 1000 | 816.535 / 833.999 | 23.063 / 27.816 | 6610 -> 276 | + +All paired final views identical. n=5 per row/variant; no p99 claim. Source +regression: 3 Rust unit tests PASS, 0 fail/ignore, 2.23 s. Includes shared golden +fixture (6 cases / 30 actions), exact decimal checks, and new full-depth clone +oracle across 2,000 mixed deltas + duplicates, followed by invalid ask after valid +bid. No partial state becomes visible after invalidation. Standalone test crate +imports the unchanged module from the checkout; this is not a full workspace or +provider acceptance. Candidate packaging / affected capture replay is still needed. + +External evidence: `dr07-hardening-20261007/l2-cpu/final-paired/receipt.json`, sha256 +`2ff2eec750555d7cd798a34ffc3634a4b01fc001ac8cd9abd6aee120258cec9f`, +plus `l2-cpu/unit.log`, `source-hashes.json`, `cleanup.json`. +Do not extrapolate the microbenchmark speedup to Kafka/projector/query/stream or +claim the 5 vCPU full-stack gate passes. Kafka network CPU attribution remains +open; perf is unavailable under current host policy and was not bypassed. No +resource/durability/timeouts were changed to manufacture a pass. + +Cleanup: --rm compiler/benchmark containers gone; removed precisely 93 MiB test +target and two exploratory binaries after hashes. Final harness removes its own +scratch. Disk available 91,463,634,944 -> 91,565,522,944 bytes (concurrent host); +no image builds or pruning. Inventory92 images28.19GB, BuildKit227entries16.83GB +unchanged; active/rc2/named rollback retained. TS writer IDf63785a27af8, +image9150385e2bb3, restart0/OOMfalse unchanged. Canonical DL main5bf5cd5 and +12-role/config set unchanged. Source changes require a new immutable candidate +before Phase C; rc2 cannot attest this new core optimization. + +Post-test formatting: rustfmt applied only to the new oracle test and standalone +probe; the optimized production block is unchanged. Evidence hashes retain the +measured pre-format source/harness, not silently rewritten. No new timing claim +from formatting. New benchmark script parsed successfully; diff --check clean. diff --git a/rust/qdl-core/src/l2_book.rs b/rust/qdl-core/src/l2_book.rs index 551c9a49..4da43376 100644 --- a/rust/qdl-core/src/l2_book.rs +++ b/rust/qdl-core/src/l2_book.rs @@ -626,13 +626,10 @@ impl L2BookCore { } }; - // Parse all updates before cloning/committing either side. A consumer - // can never observe a bid mutation without its matching ask mutation. - let mut next_bids = self.bids.clone(); - let mut next_asks = self.asks.clone(); - apply_updates(&mut next_bids, &mut next_asks, updates); - self.bids = next_bids; - self.asks = next_asks; + // All fallible validation is complete. Exclusive ownership and this + // synchronous, infallible update loop prevent partial publication; + // cloning the whole book would add work proportional to untouched depth. + apply_updates(&mut self.bids, &mut self.asks, updates); self.last_sequence = Some(frame.sequence_end); self.range_bridge_complete = self.range_bridge_complete || self.config.sequence_policy == SequencePolicy::RangeBridgeThenPrevious; @@ -1196,6 +1193,128 @@ mod tests { } } + #[test] + fn delta_mutation_matches_clone_commit_oracle() { + let id = BookIdentity::new("TEST_ONLY", "TEST_BOOK", "TEST_CHANNEL").unwrap(); + let mut core = L2BookCore::new( + BookConfig::new( + id.clone(), + SequencePolicy::PreviousSequence, + ChecksumPolicy::RequireVerified, + 512, + ) + .unwrap(), + ); + let initial: Vec<_> = (0..400) + .flat_map(|i| { + [ + BookLevelInput { + side: BookSide::Bid, + price: (10000 - i).to_string(), + quantity: "1.25".into(), + order_count: Some(2), + }, + BookLevelInput { + side: BookSide::Ask, + price: (10001 + i).to_string(), + quantity: "2.5".into(), + order_count: None, + }, + ] + }) + .collect(); + assert_eq!( + core.apply_snapshot(&BookSnapshot { + identity: id.clone(), + generation: 1, + sequence_end: 1, + checksum: ChecksumEvidence::Verified, + origin: SnapshotOrigin::WebSocket, + levels: initial + }), + BookOutcome::SnapshotApplied + ); + let mut bids = core.bids.clone(); + let mut asks = core.asks.clone(); + for seq in 2..2002 { + let updates: Vec<_> = (0..10) + .map(|j| { + let n = (seq * 17 + j * 31) % 430; + let side = if j % 2 == 0 { + BookSide::Bid + } else { + BookSide::Ask + }; + BookLevelInput { + side, + price: if side == BookSide::Bid { + (10000 - n).to_string() + } else { + (10001 + n).to_string() + }, + quantity: if (seq + j) % 5 == 0 { + "0".into() + } else { + format!("{}.0125", (seq + j) % 7) + }, + order_count: Some(seq + j), + } + }) + .collect(); + let mut next_bids = bids.clone(); + let mut next_asks = asks.clone(); + super::apply_updates( + &mut next_bids, + &mut next_asks, + super::parse_delta_levels(&updates).unwrap(), + ); + let frame = BookDelta { + identity: id.clone(), + generation: 1, + sequence_start: None, + previous_sequence: Some(seq - 1), + sequence_end: seq, + checksum: ChecksumEvidence::Verified, + updates, + }; + assert_eq!(core.apply_delta(&frame), BookOutcome::DeltaApplied); + assert_eq!(core.bids, next_bids); + assert_eq!(core.asks, next_asks); + assert_eq!(core.last_sequence, Some(seq)); + assert_eq!(core.apply_delta(&frame), BookOutcome::Duplicate); + assert_eq!(core.bids, next_bids); + assert_eq!(core.asks, next_asks); + bids = next_bids; + asks = next_asks; + } + let invalid = BookDelta { + identity: id, + generation: 1, + sequence_start: None, + previous_sequence: Some(2001), + sequence_end: 2002, + checksum: ChecksumEvidence::Verified, + updates: vec![ + BookLevelInput { + side: BookSide::Bid, + price: "9999".into(), + quantity: "8".into(), + order_count: None, + }, + BookLevelInput { + side: BookSide::Ask, + price: "10001".into(), + quantity: "-1".into(), + order_count: None, + }, + ], + }; + assert_eq!(core.apply_delta(&invalid), BookOutcome::InvalidFrame); + assert!(core.view().is_none()); + assert!(core.bids.is_empty() && core.asks.is_empty()); + assert_eq!(core.last_sequence, None); + } + #[test] fn exact_decimal_is_float_free_normalized_and_ordered() { let one = ExactDecimal::parse("1.000").unwrap(); diff --git a/scripts/measure_l2_apply_cpu.py b/scripts/measure_l2_apply_cpu.py new file mode 100644 index 00000000..516a1b2e --- /dev/null +++ b/scripts/measure_l2_apply_cpu.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Offline TEST_ONLY allocation microbenchmark, not a serving-stack CPU gate. + +Compare the same L2 workload against a Git baseline and current source using an +existing compiler image. No image builds, credentials, networks or provider I/O. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import uuid + +SOURCE = Path("rust/qdl-core/src/l2_book.rs") +IMAGE = "sha256:540c902e99c384163b688bbd8b5b8520e94e7731b27f7bd0eaa56ae1960627ab" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline-ref", required=True) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + output = args.output.resolve() + if output.is_relative_to(root): + parser.error("evidence output must be outside the checkout") + output.mkdir(parents=True, exist_ok=False) + baseline_sha = subprocess.check_output( + ["git", "rev-parse", "--verify", "--end-of-options", f"{args.baseline_ref}^{{commit}}"], cwd=root, text=True + ).strip() + sources = { + "baseline": subprocess.check_output(["git", "show", f"{baseline_sha}:{SOURCE}"], cwd=root), + "candidate": (root / SOURCE).read_bytes(), + } + harness = (root / "scripts/support/l2_apply_probe.rs").read_bytes() + receipt = { + "provenance": "TEST_ONLY_SYNTHETIC_MICROBENCH", + "baseline_sha": baseline_sha, + "compiler_image": IMAGE, + "source_sha256": {k: hashlib.sha256(v).hexdigest() for k, v in sources.items()}, + "harness_sha256": hashlib.sha256(harness).hexdigest(), + "cpu_vcpu": "NOT_MEASURED", + "full_stack_budget": "NOT_MEASURED", + "measurements": [], + "status": "INCOMPLETE", + } + scratch = Path(tempfile.mkdtemp(prefix="l2-apply-", dir=output)) + try: + def run(variant, command): + name = f"dr07-l2-{uuid.uuid4().hex}" + try: + return subprocess.run( + ["docker", "run", "--rm", "--name", name, "--network", "none", "--cpus", "1", + "--memory", "512m", "--user", f"{os.getuid()}:{os.getgid()}", + "-v", f"{scratch}:/work", "-w", f"/work/{variant}", IMAGE, *command], + capture_output=True, text=True, timeout=120, check=True, + ) + except BaseException: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=15) + raise + + for variant, source in sources.items(): + directory = scratch / variant + directory.mkdir() + (directory / "book.rs").write_bytes(source) + (directory / "bench.rs").write_bytes(harness) + result = run(variant, ["rustc", "--edition=2021", "-O", "-Awarnings", "bench.rs", "-o", "bench"]) + (output / f"{variant}-compile.log").write_text(result.stdout + result.stderr) + for depth in (20, 400, 1000): + hashes = set() + for pair in range(5): + order = ("baseline", "candidate") if pair % 2 == 0 else ("candidate", "baseline") + for variant in order: + result = run(variant, ["./bench", str(depth), "2000"]) + row = json.loads(result.stdout) + digest = hashlib.sha256(result.stderr.encode()).hexdigest() + hashes.add(digest) + row.update(variant=variant, pair=pair, state_sha256=digest) + receipt["measurements"].append(row) + if len(hashes) != 1: + raise RuntimeError(f"baseline/candidate final state differs at depth {depth}") + receipt["status"] = "PASS_MICROBENCH_ONLY" + finally: + shutil.rmtree(scratch) + receipt["scratch_removed"] = True + (output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + print(json.dumps({"status": receipt["status"], "receipt": str(output / "receipt.json")})) + + +if __name__ == "__main__": + main() diff --git a/scripts/support/l2_apply_probe.rs b/scripts/support/l2_apply_probe.rs new file mode 100644 index 00000000..88adf73a --- /dev/null +++ b/scripts/support/l2_apply_probe.rs @@ -0,0 +1,105 @@ +#[path = "book.rs"] +mod book; +use book::*; +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicU64, Ordering}; +struct CountAlloc; +static ALLOCS: AtomicU64 = AtomicU64::new(0); +unsafe impl GlobalAlloc for CountAlloc { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + ALLOCS.fetch_add(1, Ordering::Relaxed); + System.alloc(l) + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + System.dealloc(p, l) + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, n: usize) -> *mut u8 { + ALLOCS.fetch_add(1, Ordering::Relaxed); + System.realloc(p, l, n) + } +} +#[global_allocator] +static ALLOC: CountAlloc = CountAlloc; +fn main() { + let depth: usize = std::env::args().nth(1).unwrap().parse().unwrap(); + let count: u64 = std::env::args().nth(2).unwrap().parse().unwrap(); + let id = BookIdentity::new("TEST_ONLY", "TEST_BOOK", "TEST_CHANNEL").unwrap(); + let mut core = L2BookCore::new( + BookConfig::new( + id.clone(), + SequencePolicy::PreviousSequence, + ChecksumPolicy::RequireVerified, + 20, + ) + .unwrap(), + ); + let levels: Vec<_> = (0..depth) + .flat_map(|i| { + [ + BookLevelInput { + side: BookSide::Bid, + price: format!("{}", 10000 - i), + quantity: "1.25".into(), + order_count: None, + }, + BookLevelInput { + side: BookSide::Ask, + price: format!("{}", 10001 + i), + quantity: "2.5".into(), + order_count: None, + }, + ] + }) + .collect(); + assert_eq!( + core.apply_snapshot(&BookSnapshot { + identity: id.clone(), + generation: 1, + sequence_end: 1, + checksum: ChecksumEvidence::Verified, + origin: SnapshotOrigin::WebSocket, + levels + }), + BookOutcome::SnapshotApplied + ); + let mut delta = BookDelta { + identity: id, + generation: 1, + sequence_start: None, + previous_sequence: Some(1), + sequence_end: 2, + checksum: ChecksumEvidence::Verified, + updates: (0..5) + .flat_map(|i| { + [ + BookLevelInput { + side: BookSide::Bid, + price: format!("{}", 10000 - i), + quantity: "3.25".into(), + order_count: None, + }, + BookLevelInput { + side: BookSide::Ask, + price: format!("{}", 10001 + i), + quantity: "4.5".into(), + order_count: None, + }, + ] + }) + .collect(), + }; + let start_alloc = ALLOCS.load(Ordering::Relaxed); + let start = std::time::Instant::now(); + for seq in 2..count + 2 { + delta.previous_sequence = Some(seq - 1); + delta.sequence_end = seq; + assert_eq!( + std::hint::black_box(core.apply_delta(&delta)), + BookOutcome::DeltaApplied + ); + } + let elapsed = start.elapsed().as_secs_f64() * 1000.; + let allocations = ALLOCS.load(Ordering::Relaxed) - start_alloc; + println!("{{\"provenance\":\"TEST_ONLY_SYNTHETIC_MICROBENCH\",\"depth_per_side\":{depth},\"deltas\":{count},\"elapsed_ms\":{elapsed},\"allocations\":{allocations}}}"); + eprintln!("FINAL {:?}", core.view()); +} diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md index f15860db..27304a0f 100644 --- a/upgrade/evidence/p184-dl-phase-c-packet.md +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -209,3 +209,17 @@ New disposable tests use --rm and are removed after their evidence is recorded. shared cache cleanup, volume/network deletion or canonical/user-hunk modification. Record disk pre/post and running-role image/start/restart comparison. Feature worktree remains active until reviewed/merged; no worktree cleanup before that. + +## DR07 source convergence amendment (2026-10-07) + +HOLD remains. The new L2 no-whole-book-clone source slice is unit/oracle tested +and paired offline in `p184-dr07-hardening-progress.md`, but NOT inside rc2. +Do not deploy rc2 and claim this optimization runs. After source review, converge +one immutable core candidate and check affected provider capture/packaging before +using the existing core-first packet. Keep baseline rollback digest and all +coherence/lag/integrity gates above unchanged. Microbenchmark allocation savings +are not whole-stack vCPU evidence; the old 5.962 vCPU baseline still fails5. + +TS reset source58287fc is isolated in codex/dr07-reset-anchor for Claude review; +no shared writer recreation is authorized by this DL packet. Optional Redis-ACK +trace is a separate TS source slice, not evidence of current deployed ACK latency. diff --git a/upgrade/evidence/p184-dr07-hardening-progress.md b/upgrade/evidence/p184-dr07-hardening-progress.md index 73db7135..9fa5ac31 100644 --- a/upgrade/evidence/p184-dr07-hardening-progress.md +++ b/upgrade/evidence/p184-dr07-hardening-progress.md @@ -137,3 +137,50 @@ No runtime throughput/recovery or release certification is inferred. Evidence/cleanup receipt: external `isolated-worktree/attempt-2/` under the root above; migration `/tmp/codex-dr07-move-20261007/migration-receipt.json`. New worktree clean, no push/merge/deploy. The historical overflow cause remains unproved. + +## DR07 CPU Source Slice - 2026-10-07 + +Status: TESTED_LOCALLY, not deployed, Phase C remains HOLD. Profiling already +showed core normalization cost but did not provide a function-level CPU profile. +A controlled TEST_ONLY allocation microbenchmark now isolates whole-book cloning +on every L2 delta. All validations complete before the synchronous update loop; +exclusive Rust ownership allows updating both sides without copying untouched +levels. No changes to comparator, decimal precision, integrity/sequence/generation, +publish cadence, Kafka durability or provider semantics. + +Reproducer: `python3 scripts/measure_l2_apply_cpu.py --baseline-ref 0fb731d +--output `. Existing compiler image only, +network none, 1 CPU / 512 MiB ceiling, alternating baseline/candidate order. +Each sample applies 2,000 deltas of 10 updates. Five samples per side/depth; +these are elapsed times with allocation instrumentation, NOT vCPU or live latency. + +| Levels per side | Baseline median / max (ms) | Candidate median / max (ms) | Allocations/delta old -> new | +| --- | --- | --- | --- | +| 20 | 35.635 / 38.471 | 15.646 / 18.101 | 280 -> 152 | +| 400 | 330.114 / 359.657 | 24.486 / 27.836 | 2790 -> 258 | +| 1000 | 816.535 / 833.999 | 23.063 / 27.816 | 6610 -> 276 | + +All paired final views identical. n=5 per row/variant; no p99 claim. Source +regression: 3 Rust unit tests PASS, 0 fail/ignore, 2.23 s. Includes shared golden +fixture (6 cases / 30 actions), exact decimal checks, and new full-depth clone +oracle across 2,000 mixed deltas + duplicates, followed by invalid ask after valid +bid. No partial state becomes visible after invalidation. Standalone test crate +imports the unchanged module from the checkout; this is not a full workspace or +provider acceptance. Candidate packaging / affected capture replay is still needed. + +External evidence: `dr07-hardening-20261007/l2-cpu/final-paired/receipt.json`, sha256 +`2ff2eec750555d7cd798a34ffc3634a4b01fc001ac8cd9abd6aee120258cec9f`, +plus `l2-cpu/unit.log`, `source-hashes.json`, `cleanup.json`. +Do not extrapolate the microbenchmark speedup to Kafka/projector/query/stream or +claim the 5 vCPU full-stack gate passes. Kafka network CPU attribution remains +open; perf is unavailable under current host policy and was not bypassed. No +resource/durability/timeouts were changed to manufacture a pass. + +Cleanup: --rm compiler/benchmark containers gone; removed precisely 93 MiB test +target and two exploratory binaries after hashes. Final harness removes its own +scratch. Disk available 91,463,634,944 -> 91,565,522,944 bytes (concurrent host); +no image builds or pruning. Inventory92 images28.19GB, BuildKit227entries16.83GB +unchanged; active/rc2/named rollback retained. TS writer IDf63785a27af8, +image9150385e2bb3, restart0/OOMfalse unchanged. Canonical DL main5bf5cd5 and +12-role/config set unchanged. Source changes require a new immutable candidate +before Phase C; rc2 cannot attest this new core optimization. From c33c90f993049f06742a7d29a9dfcb017c588c9b Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 16:24:29 +0000 Subject: [PATCH 016/147] docs: record isolated DR07 Redis trace handoff --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index eeecaf6c..87bf029f 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69538,3 +69538,20 @@ Post-test formatting: rustfmt applied only to the new oracle test and standalone probe; the optimized production block is unchanged. Evidence hashes retain the measured pre-format source/harness, not silently rewritten. No new timing claim from formatting. New benchmark script parsed successfully; diff --check clean. + +DR07 TS trace handoff recorded2026-10-07: commit19d77e2 on +codex/dr07-reset-anchor, /home/bobby/.worktrees/ts-codex-dr07, follows reset58287fc. +Claude's Paper worktree untouched. BOOK/MARK writer opt-in fixed ring records real +Redis reply monotonic clock, generation/native sequence and UNKNOWN lost-reply +outcome; no PUBSUB-as-ACK, default OFF, no hot-path I/O/logging. 66 focused /725 +affected total PASS,0fail/skip,real isolated Redis; do not sum overlapping counts. +Receipt dr07-hardening-20261007/ack-trace/source-test-receipt.json sha256 +f5af8b5beded6fcc484eefa106db8a9724d10eb8eb46113fffaabb5b6ffd286c. +Five rounds of300calls: enabled diagnostic delta0.040279ms/apply (same-host), +not live delivery latency. Canonical commit/SDK receive/other feeds still unmeasured. +DB15zero, exact test containers/network removed; disk91,342,974,976->91,437,936,640 +bytes, no attributable whole-host reclaim. No builds/prune/runtime changes. +DL core63238bf and TS19d77e2 are source handoffs only; Phase C HOLD. Do not use +rc2 to claim new core bytes or deployed TS trace. No push/merge or active quota, +freshness/manifest changes; next gate is affected source/candidate review and +converged packet, not another catalogue/C2 debugging run. From 2071d99e4e84752264446eb1eb7ee21e767c17dc Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 17:20:07 +0000 Subject: [PATCH 017/147] Harden read-only DR07 monitoring and record rc3 paired evidence --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 54 +++++++ scripts/kn_production_watch.py | 26 +++- scripts/measure_l2_top_materialization.py | 18 ++- tests/test_kn_production_watch.py | 31 ++++ ...test_l2_top_materialization_measurement.py | 12 ++ upgrade/evidence/p184-dl-phase-c-packet.md | 17 +++ upgrade/evidence/p184-dr07-runtime-closure.md | 135 ++++++++++++++++++ 7 files changed, 284 insertions(+), 9 deletions(-) create mode 100644 upgrade/evidence/p184-dr07-runtime-closure.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 87bf029f..0a9f1c96 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69555,3 +69555,57 @@ DL core63238bf and TS19d77e2 are source handoffs only; Phase C HOLD. Do not use rc2 to claim new core bytes or deployed TS trace. No push/merge or active quota, freshness/manifest changes; next gate is affected source/candidate review and converged packet, not another catalogue/C2 debugging run. + +### DR07 runtime closure continuation - 2026-10-07 + +Status: IN_PROGRESS. Owner asks to finish full-stack candidate CPU, actual + overflow/replay recovery and canonical-to-Redis/consumer latency, plus related +in-scope gaps. No Paper feature dependency, full catalogue/C2 rerun, hidden policy +relaxation, new service or interface. Governing guide sections18.13-18.14 and +Phase C packet still apply. Source baseline: DL c33c90f (core63238bf), isolated TS +19d77e2; Claude worktree remains read-only for coordination, never edited/tested. + +Execution sequence: capture uncontaminated read-only baseline; verify source/image +and DR matrix handoff; build one immutable candidate because core source changed; +affected native tests and captured-provider replay; then apply only the exact +eligible bounded runtime packet with named rollback. Keep existing CPU5vCPU, +coherence117/120 per venue and per-partition lag gates; never bypass a failed +prerequisite. No duplicate writer or competing same-identity probe. No orders. +Measurement must name canonical durable-visibility/commit boundary explicitly; +Kafka CreateTime and Redis PUBSUB are not commit/ACK clocks. Preserve missing and +negative results; distinguish current baseline from candidate observations. +Cleanup exact test/build resources after hashes; preserve active/named rollback. +No push/merge/release until evidence convergence. New blockers are recorded here, +not renamed technical debt or closed by an isolated microbenchmark. + +New in-scope monitoring safety finding before runtime action: existing +kn_production_watch.main() auto-stops seven canonical roles by historical KN +names on a guard event, even without an explicit stop roster. Current audit uses +sample() only and has performed NO stops. Fix diagnostic default to read-only; +optional emergency stop must specify exact immutable container IDs and validate +against the current eligible roster, never infer authority from phase names. +Add negative tests proving missing/mismatched IDs cannot stop production. This +is observer tooling only, not an application/runtime/API/config change. +Measurement harness also now accepts an explicit immutable candidate digest; +10 targeted unit tests passed before capture, no new provider calls yet. + + +DR07 closure checkpoint: read-only baseline31samples/30intervals,299.996s; +12application+5infra mean5.779832/p955.978048/max6.012644vCPU, FAIL5gate. +Stream overflow+0/+0 during approximately290s is an observation, NOT proof +historical replay loop fixed. Canonical->actual TS Redis ACK NOT_MEASURED. +Built exactly one changed-source Rust image rc3-c33c90f, digest0d8da150...38411; +12binary/linkage checks PASS. Paired60s real ETH Binance/OKX capture: same delta +chains,586/602deltas, core3.59->3.01CPU-s, RSS18420->18424KiB,n101 each; +49/53top changes candidate source-time delay0ms (not wall-clock delivery). +Snapshot message rate x1.492, totalBOOK x1.045; downstream CPU not measured. +Tooling16tests PASS in existingPythonimage; host lackedhttpx for capturemodule, +recorded rather than installing/changing host. Watcher auto-stop finding fixed: +defaultread-only, explicit exactcontainerIDs required tostop. No runtime action. +Report upgrade/evidence/p184-dr07-runtime-closure.md and Phase C packet updated; +statusIN_PROGRESS/HOLD. OldCPUprerequisite still fails; boundedtrialproposal has +NOT waived final5vCPU or authorized new image. No catalogue/C2 or Paperret tests. +Cleanup pairedcontainers/network/capture/TLS and buildcontext complete; full disk, +imagecache inventory and retainedactive/rollback/candidate set recorded in report. +No push/merge; no changes to Claude TSworktree. Source/app image provenance remains +c33c90f; this slice only changes diagnostic/test/report tooling. diff --git a/scripts/kn_production_watch.py b/scripts/kn_production_watch.py index 219f7e99..4b112217 100644 --- a/scripts/kn_production_watch.py +++ b/scripts/kn_production_watch.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 -"""Deadline-bounded KN-5 resource journal; emergency stop affects new roles only.""" +"""Bounded read-only resource journal; stopping requires explicit container IDs.""" import argparse import json +import re from pathlib import Path import subprocess import time @@ -54,6 +55,19 @@ def stop_reasons(row, low_idle_seconds=0): return reasons +def explicit_stop_ids(row, allowed): + """No lifecycle authority comes from a historic phase name or guard alert.""" + allowed = list(allowed) + if len(set(allowed)) != len(allowed) or any( + re.fullmatch(r"[0-9a-f]{64}", value) is None for value in allowed + ): + raise ValueError("stop authorization requires distinct full container IDs") + eligible = {row["containers"].get(f"{PROJECT}-{role}-1", {}).get("id") for role in NATIVE} + if any(value not in eligible for value in allowed): + raise ValueError("authorized stop ID no longer belongs to the observed role") + return allowed + + def cpu_ticks(): values = [int(x) for x in Path("/proc/stat").read_text().splitlines()[0].split()[1:9]] return sum(values), values[3] + values[4] @@ -64,6 +78,8 @@ def main(): parser.add_argument("--out", type=Path, required=True) parser.add_argument("--seconds", type=int, default=7200) parser.add_argument("--stop-file", type=Path, required=True) + parser.add_argument("--stop-container-id", action="append", default=[], + help="Explicitly authorized full container ID; default is read-only") args = parser.parse_args() end = time.monotonic() + args.seconds previous_cpu = cpu_ticks() @@ -73,6 +89,7 @@ def main(): with args.out.open("x") as handle: while time.monotonic() < end and not args.stop_file.exists(): row = sample() + stop_ids = explicit_stop_ids(row, args.stop_container_id) now_cpu = cpu_ticks() total_delta = now_cpu[0] - previous_cpu[0] idle = 100 * (now_cpu[1] - previous_cpu[1]) / total_delta if total_delta else 100 @@ -92,10 +109,9 @@ def main(): handle.write(json.dumps(row, separators=(",", ":")) + "\n") handle.flush() if danger: - names = [f"{PROJECT}-{r}-1" for r in NATIVE if f"{PROJECT}-{r}-1" in row["containers"]] - if names: - subprocess.run(["docker", "stop", "-t", "15", *names], check=True) - print("GUARD_STOP_NEW_KN_ROLES", flush=True) + if stop_ids: + subprocess.run(["docker", "stop", "-t", "15", *stop_ids], check=True) + print("GUARD_STOP_EXPLICIT_IDS" if stop_ids else "GUARD_ALERT_NO_RUNTIME_MUTATION", flush=True) return 3 time.sleep(5) print("WATCH_COMPLETE", flush=True) diff --git a/scripts/measure_l2_top_materialization.py b/scripts/measure_l2_top_materialization.py index 4c57f1a2..dd108b82 100644 --- a/scripts/measure_l2_top_materialization.py +++ b/scripts/measure_l2_top_materialization.py @@ -13,6 +13,7 @@ import json import math import os +import re from pathlib import Path import shutil import subprocess @@ -282,7 +283,15 @@ def stop_disposable(name): time.sleep(.2) -def orchestrate(work, seconds): +def image_digest(value): + if not isinstance(value, str) or re.fullmatch(r"sha256:[0-9a-f]{64}", value) is None: + raise ValueError("candidate requires an immutable sha256 image digest") + return value + + +def orchestrate(work, seconds, candidate=CANDIDATE): + candidate = image_digest(candidate) + command(["docker", "image", "inspect", candidate, "--format", "{{.Id}}"]) work.mkdir(parents=True, exist_ok=False); work.chmod(0o755) runtime = json.loads(command(["docker","inspect","qdl_v2_stable_candidate-rust_core-1"]))[0] if runtime["Image"] != BASELINE: raise ValueError("deployed baseline drift") @@ -338,10 +347,10 @@ def client(mode, network, detach=False): """ (work/"broker.properties").write_text(properties) result={"capture_sha256":capture_hash,"source":command(["git","-C",str(ROOT),"rev-parse","HEAD"]), - "baseline":BASELINE,"candidate":CANDIDATE,"runs":{},"resource_scope":"core container only, 1 CPU / 256 MiB, paced same capture"} + "baseline":BASELINE,"candidate":candidate,"runs":{},"resource_scope":"core container only, 1 CPU / 256 MiB, paced same capture"} command(["docker","network","create","--internal",PREFIX]) try: - for label,image in [("baseline",BASELINE),("candidate",CANDIDATE)]: + for label,image in [("baseline",BASELINE),("candidate",candidate)]: for f in ["client-ready","core-ready"]: (work/f).unlink(missing_ok=True) broker=PREFIX+"-kafka";core=PREFIX+"-core" command(["docker","run","--rm","-d","--name",broker,"--network",PREFIX,"--network-alias","kafka", @@ -413,8 +422,9 @@ def field(text, name): def main(): p=argparse.ArgumentParser();p.add_argument("mode",choices=["run","capture","replay","analyse"]) p.add_argument("--work",type=Path,required=True);p.add_argument("--seconds",type=float,default=60) + p.add_argument("--candidate-image", type=image_digest, default=CANDIDATE) a=p.parse_args() - if a.mode=="run": orchestrate(a.work,a.seconds) + if a.mode=="run": orchestrate(a.work,a.seconds,a.candidate_image) elif a.mode=="capture": client_capture(a.work,a.seconds) elif a.mode=="replay": client_replay(a.work) else: analyse(a.work) diff --git a/tests/test_kn_production_watch.py b/tests/test_kn_production_watch.py index 12b82a8f..02fa5f4e 100644 --- a/tests/test_kn_production_watch.py +++ b/tests/test_kn_production_watch.py @@ -22,3 +22,34 @@ def test_resource_and_ts_fences(self): row["available_memory"] = 4*1024**3 row["ts_mark_index_unavailable_60s"] = 9 assert stop_reasons(row, 59) == [] + + +class ExplicitAuthorityTests(unittest.TestCase): + def test_stop_authority_never_comes_from_name(self): + from scripts.kn_production_watch import explicit_stop_ids + row = {"containers": {PROJECT + "-market_projector_1-1": {"id": "a" * 64}}} + self.assertEqual(explicit_stop_ids(row, []), []) + self.assertEqual(explicit_stop_ids(row, ["a" * 64]), ["a" * 64]) + for ids in (["market_projector_1"], ["a" * 12], ["b" * 64], ["a" * 64] * 2): + with self.assertRaises(ValueError): + explicit_stop_ids(row, ids) + + def test_guard_is_read_only_unless_exact_ids_supplied(self): + import tempfile + from pathlib import Path + from unittest.mock import patch, Mock + from scripts.kn_production_watch import main + row = {"available_memory": 1024**3, "containers": { + PROJECT + "-market_projector_1-1": {"id": "a" * 64}}} + for approved in (False, True): + with self.subTest(approved=approved), tempfile.TemporaryDirectory() as folder: + argv = ["watch", "--out", str(Path(folder) / "result.jsonl"), + "--seconds", "1", "--stop-file", str(Path(folder) / "stop")] + if approved: + argv += ["--stop-container-id", "a" * 64] + with patch("sys.argv", argv), patch("scripts.kn_production_watch.sample", return_value=row), \ + patch("scripts.kn_production_watch.cpu_ticks", return_value=(100, 50)), \ + patch("scripts.kn_production_watch.subprocess.run", return_value=Mock(stdout="", stderr="")) as run: + self.assertEqual(main(), 3) + stops = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "stop"]] + self.assertEqual(stops, [["docker", "stop", "-t", "15", "a" * 64]] if approved else []) diff --git a/tests/test_l2_top_materialization_measurement.py b/tests/test_l2_top_materialization_measurement.py index 49dc8a42..9ae64638 100644 --- a/tests/test_l2_top_materialization_measurement.py +++ b/tests/test_l2_top_materialization_measurement.py @@ -5,6 +5,18 @@ from scripts.phase114_l2_real_provider_capture import capture_replay_window class MeasurementTests(unittest.TestCase): + def test_candidate_digest_must_be_immutable_before_side_effects(self): + from unittest.mock import patch + from pathlib import Path + from scripts.measure_l2_top_materialization import image_digest, orchestrate + good = "sha256:" + "a" * 64 + self.assertEqual(image_digest(good), good) + for bad in ("latest", "qdl:tag", "sha256:a", None, "sha256:" + "A" * 64): + with patch("scripts.measure_l2_top_materialization.command") as command: + with self.assertRaises(ValueError): + orchestrate(Path("/must-not-create"), 60, bad) + command.assert_not_called() + def test_censored_changes_stay_in_denominator(self): rows=[{"seq":str(i),"ms":i*100,"top":top} for i,top in enumerate([("1","3"),("2","3"),("2","4"),("2","4"),("1","4")])] result=top_delay(rows,[{"seq":"3","top":("2","4")}]) diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md index 27304a0f..39779b5d 100644 --- a/upgrade/evidence/p184-dl-phase-c-packet.md +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -223,3 +223,20 @@ are not whole-stack vCPU evidence; the old 5.962 vCPU baseline still fails5. TS reset source58287fc is isolated in codex/dr07-reset-anchor for Claude review; no shared writer recreation is authorized by this DL packet. Optional Redis-ACK trace is a separate TS source slice, not evidence of current deployed ACK latency. + + +## DR07 RC3 checkpoint - 2026-10-07 (not execution authorization) + +[Runtime closure report](p184-dr07-runtime-closure.md) supersedes candidate-resource +inferences, not prior evidence. New proposed Rust candidate is +`sha256:0d8da150820ed684bda084e92f714537cb40bd81e2e3a6bc93fa9a4b33538411` +from c33c90f, optimization63238bf; rc2 does not include that optimization. +Paired real capture:3.59->3.01CPU-seconds, equal delta chains,49/53 top changes +materialized at source-time0ms delay (not consumer latency). Current full-stack +baseline remains5.779832vCPU, p955.978048,n30/299.996s: prerequisite FAIL. +Do not execute the rc2 roster above or silently substitute rc3 into an old approval. +Proposed bounded-trial amendment must resolve old-runtime CPU prerequisite while +retaining final<=5vCPU, coherence>=117/120 each venue, identity/integrity fencing +and lag gates. No amendment applied or caps raised in this slice. Named rollback +images/mounts/TLS/offsets unchanged. Claude's TS worktree remains protected; +reset/ACK trace integration and real DR/broker receipts are still required. diff --git a/upgrade/evidence/p184-dr07-runtime-closure.md b/upgrade/evidence/p184-dr07-runtime-closure.md new file mode 100644 index 00000000..8ac42ea9 --- /dev/null +++ b/upgrade/evidence/p184-dr07-runtime-closure.md @@ -0,0 +1,135 @@ +# DR07 Runtime Closure Checkpoint - 2026-10-07 + +Status: IN_PROGRESS / PHASE_C_HOLD. This is not a release certificate. +Main plan: DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md, DR07 runtime closure continuation. +Guide: DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md sections18.13-18.14. +Evidence root: /home/bobby/.local/state/qdl-v2/dr07-runtime-20261007/. +No production restart/config/credential/manifest/cap or order mutation in this slice. +No TS Paper feature dependency; DR01-DR08 and broker delta remain release inputs. + +## Actual Current Runtime Baseline + +`audit/baseline-20261007T163940Z/summary.json`, sha256 +`6902bd0c96e2ec1e58a08b944bc13fe412e1bea0c60d769ae79d434a4c8ca192`. +Read-only cgroup observations, 16:39:40.979-16:44:41.089 UTC, 31 samples, +30 intervals /299.996 seconds. Build and replay started AFTER this window. + +| Measurement | Result | Verdict / boundary | +| --- | --- | --- | +| Full DL, 12 application +5 infrastructure roles | mean5.779832, p955.978048, max6.012644 vCPU; n30 | FAIL existing5vCPU gate; old runtime, not candidate | +| Five non-overlapping60s averages |5.801806 /5.722398 /5.794909 /5.817657 /5.762390 vCPU | all above5 | +| TS writer CPU, separately |mean1.461520 vCPU; n30 | excluded from DL total, not hidden | +| Three Kafka brokers combined |2.885854 vCPU | measured, not attributed solely to L2 | +| Three cores combined |1.181349 vCPU | measured old cores | +| Two projectors combined |0.280874 vCPU | measured | +| Stream overflow increments |0/0, approximately290s, n30 metric samples each | observed clean window only; historical replay loop NOT closed | +| Active Stream delivered |119727 events /50 subscriptions | not a stress capacity certificate | +| OOM/restart delta |0, observed roles | startup/process health alone is not execution acceptance | +| TS replay/reset deltas |NOT_MEASURED | Docker logs empty; actual mounted log has no timestamped entries in window | +| Kafka lag trend |NOT_MEASURED | one partition snapshot after window, not continuous convergence proof | +| Candidate CPU / canonical-to-production-Redis ACK |NOT_MEASURED | candidate not deployed; no substitute from isolated tests | + +All canonical roles stayed on their previous images. Initial raw Mounts array hashes +were order-sensitive; preserve originals and see mount-order-correction.json. +Eight repeated inspections had identical normalized mounts, IDs/config/start times. +No mount/config drift established. Do not rewrite the original audit as a PASS. + +Kafka memory.events.max increased2294/1651/1770; no OOM. Follow-up memory PSI +avg10/60/300 was0.00. A memory-limit event is not proof of a GC or pressure root +cause; no resource/durability/timeout changes have been inferred from this. +Latest completed quota minutes943/1003 of1500 before the probe; headroom is not +reserved capacity and must be re-read immediately before any same-identity calls. + +## New Candidate And Paired Real Capture + +Frozen application source `c33c90f993049f06742a7d29a9dfcb017c588c9b`, containing +core optimization63238bf. Built once with tracked Dockerfile, pinned bases and +CARGO_BUILD_JOBS=1: `qdl-v2-rust:2.2.3-rc3-c33c90f`, immutable digest +`sha256:0d8da150820ed684bda084e92f714537cb40bd81e2e3a6bc93fa9a4b33538411`. +All12 packaged executables found; no missing dynamic libraries. This is NOT rc2. +Python application paths have no diff43bbcf0..c33c90f; existing Python rc2 reused +only for capture/test tooling. It does not pretend to bear the new source label. +Future release attestation must explicitly bind unchanged Python bytes to source. + +Command: `python3 scripts/measure_l2_top_materialization.py run --work +/paired-rc3 --seconds60 --candidate-image sha256:0d8da150...38411` +(use full digest above and `--seconds 60` when executing). +One pair, same real provider bytes, same config, baseline core658a9570 versus +candidate0d8da150; each bounded1CPU/256MiB. No shared Kafka writes. +Receipt sha256 `360c9bc5c8ae8164c8ed26d697648ba6759d09b4a458bf25c69aadf8f13e47ae`. + +| Metric | Baseline | Candidate | +| --- | --- | --- | +| Core observed CPU seconds over60s capture |3.59 |3.01 (-16.16%) | +| Core CPU per capture second |0.059833 vCPU |0.050167 vCPU | +| Core sampled RSS high water |18420 KiB |18424 KiB | +| Resource samples |101 |101 | +| Binance ETH snapshot/s; protobuf bytes/s |0.984580;7802.45 |1.468527;11630.93 | +| OKX ETH snapshot/s; protobuf bytes/s |1.013289;8320.25 |1.511628;12403.24 | +| Binance next verified snapshot source-time delay p50/p95/max, n49 |410/920/924ms |0/0/0ms | +| OKX equivalent, n53 |400/900/900ms |0/0/0ms | + +Source-time delay is NOT wall-clock consumer latency. Candidate emits on the +same source event that changes top; real Kafka/Query/consumer transit remains. +586 Binance +602 OKX deltas have identical payload chain hashes in both runs. +Capture frame counts592/603; windows59.924/60.2s.49/53 top changes, no uncovered +tail on either candidate binding. This is not the previously measured busy window +(135/156 changes/minute); do not relabel it peak-load acceptance. +Snapshot count multiplier approximately1.492; all BOOK message multiplier1.045. +Those are downstream workload estimates, not projector/Query/Stream CPU measures. + +Core savings cannot certify whole-stack5vCPU. Old non-core remainder is4.598483vCPU; +ignoring every downstream/workload change, cores would need approximately66% +less CPU to reach5. The paired replay is16.16% lower. This arithmetic is a +scenario, not a runtime forecast and not grounds to waive the gate or raise caps. + +## Observer Safety And Verification + +Fixed kn_production_watch default: alert only, no automatic production stops based +on historical KN role names. Optional stop requires distinct full container IDs +matching current eligible roles; no implied rollout authority. +Measurement harness accepts an explicit immutable candidate digest and checks +it before side effects. These are tools, not application contract changes. +Tests in existing Python rc2 image:16 PASS,0fail/skip. Host attempt first ran6 +watch tests but could not import capture test (host lacks httpx); retained as an +environment error, not disguised as16 host passes. No dependency installation. + +## Remaining Closure Gates + +1. Runtime admission: Phase C still requires old stack<=5 before rollout. It + fails. Proposed decision is a bounded candidate trial with named rollback, + NOT waiver of final5vCPU or silent change of the rollout packet. Needs explicit + resolution before action. Existing core-first/coherence117/120 per venue and + continuous lag safety checks remain unchanged. +2. Source/caller convergence: TS58287fc +19d77e2 remain on codex/dr07-reset-anchor. + No recorded Claude integration acceptance found; do not overwrite its Paper + worktree or deploy an unreviewed union. Current shared writer is9150385e2bb3, + actual SDK2.0.7, bindingV2r22/JWT10; SDK2.1.0/V3 remain candidate. +3. Runtime recovery: observe exact matched reset/anchor/ACK/cursor sequence on + converged actual writer; zero new overflow in an idle window alone cannot + close the historical overflow/replay defect. +4. Latency: retain NOT_MEASURED canonical commit and actual Redis ACK until those + clocks/identity are joined on deployed writer. A bounded real Query-to-isolated + Redis probe is useful affected-path evidence but cannot close this runtime gate. +5. Release inputs: no newer accepted case-indexed DR01-DR08/broker-delta packet + was found. Static surface93/93 is not a substitute. Owner removed dependence + on building Paper features, not on these data/broker checks. +6. Final immutable source/image/config/receipt convergence and scoped cleanup. + No push/merge/release, no broad catalogue/C2 run to hunt these defects. + +## Cleanup And Runtime Identity + +Paired capture/test Kafka/core/client/network and private test TLS removed; +capture hashcbbddcf85b016d771435c0d2000ea221cacd023d888e9219af127719bd36f499 +retained, raw capture deleted. Runtime inventory unchanged. Pair disk available +89,861,836,800 ->89,632,673,792 bytes (whole-host concurrent activity, not claimed +reclaim). Build context24,557,465 logical bytes removed after provenance: +89,559,764,992 ->89,587,806,208 available bytes. No volume/shared-state cleanup. +Inventory93 images28.31GB; BuildKit248 records18.11GB. No broad prune: shared +builds and active/named rollback/candidate artifacts retained pending closure. +DL canonical /home/bobby/data_layer main5bf5cd5 (v2.2.2 ancestry), feature +/home/bobby/.worktrees/data-layer-p184-closure fix/p184-dl-closure. +TS canonical /home/bobby/trading_system fix/dev-consumer-recovery-integration668e103; +Codex TS worktree remains separate. Old DL core658a9570, projector a696cfba, +Stream1f13408f, Query51450612, ingestors0f6876e1, BARf7351c3b unchanged. +Full immutable rollback roster remains in the Phase C packet and inventory JSON. From 4da52a8e6cb1bb97f1ce17d005bc65442c7af8ac Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 17:35:23 +0000 Subject: [PATCH 018/147] Freeze owner-approved core-only trial and rollback gates --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 29 ++++++++++++++++++++++ upgrade/evidence/p184-dl-phase-c-packet.md | 29 ++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 0a9f1c96..ac6a6578 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69609,3 +69609,32 @@ Cleanup pairedcontainers/network/capture/TLS and buildcontext complete; full dis imagecache inventory and retainedactive/rollback/candidate set recorded in report. No push/merge; no changes to Claude TSworktree. Source/app image provenance remains c33c90f; this slice only changes diagnostic/test/report tooling. + + +### DR07 owner-approved core-only trial - 2026-10-07 + +Owner explicitly approved changing gate order to trial the candidate. This +supersedes ONLY the old-runtime<=5vCPU pre-action prerequisite for this bounded +trial, not the final5vCPU release gate, not DR/broker release evidence, not TS +source integration, and not permission to roll the other9application roles. +Exact sequence rust_core -> rust_core_2 -> rust_core_3. Candidate +sha256:0d8da150820ed684bda084e92f714537cb40bd81e2e3a6bc93fa9a4b33538411; +rollback sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023. +Render private compose from current inspect using existing kn_production_packet +helpers; preserve config/TLS/state/offsets/aliases/caps, only image changes. +Readiness is actual partition progress and usable consumer data, not process-up. + +Freeze before first action: trial safety CPU boundary=5.978048vCPU (already +measured old-runtime p95), two consecutive non-overlapping30s whole-stack means +above it stop further steps and roll changed roles back in reverse order. +This boundary governs transitional safety only; after convergence300s mean<=5 +remains required for promotion. OOM/restart loop, integrity/ownership failure, +missing ISR/quorum, five consecutive10s per-partition lag increases, or failure +to recover application progress within120s also stop/rollback. A failed final +CPU gate or post-core TS coherence<117/120 on either venue means trialFAIL and +rollback, not changing limits to keep the candidate. Preserve failed samples. +Consumer QUOTE/BOOK freshness/policies remain unchanged; observations use actual +Redis read path without shared writes/orders/subscriptions. Coherence result is +review input for Claude; no replacement of complete TS DR01-DR08 acceptance. +Observer begins before first lifecycle call; only this frozen scope is approved. +A successful trial does not authorize release or rolling remaining roles. diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md index 39779b5d..5181976d 100644 --- a/upgrade/evidence/p184-dl-phase-c-packet.md +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -240,3 +240,32 @@ retaining final<=5vCPU, coherence>=117/120 each venue, identity/integrity fencin and lag gates. No amendment applied or caps raised in this slice. Named rollback images/mounts/TLS/offsets unchanged. Claude's TS worktree remains protected; reset/ACK trace integration and real DR/broker receipts are still required. + + +## DR07 owner-approved core-only trial - 2026-10-07 + +Owner explicitly approved changing gate order to trial the candidate. This +supersedes ONLY the old-runtime<=5vCPU pre-action prerequisite for this bounded +trial, not the final5vCPU release gate, not DR/broker release evidence, not TS +source integration, and not permission to roll the other9application roles. +Exact sequence rust_core -> rust_core_2 -> rust_core_3. Candidate +sha256:0d8da150820ed684bda084e92f714537cb40bd81e2e3a6bc93fa9a4b33538411; +rollback sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023. +Render private compose from current inspect using existing kn_production_packet +helpers; preserve config/TLS/state/offsets/aliases/caps, only image changes. +Readiness is actual partition progress and usable consumer data, not process-up. + +Freeze before first action: trial safety CPU boundary=5.978048vCPU (already +measured old-runtime p95), two consecutive non-overlapping30s whole-stack means +above it stop further steps and roll changed roles back in reverse order. +This boundary governs transitional safety only; after convergence300s mean<=5 +remains required for promotion. OOM/restart loop, integrity/ownership failure, +missing ISR/quorum, five consecutive10s per-partition lag increases, or failure +to recover application progress within120s also stop/rollback. A failed final +CPU gate or post-core TS coherence<117/120 on either venue means trialFAIL and +rollback, not changing limits to keep the candidate. Preserve failed samples. +Consumer QUOTE/BOOK freshness/policies remain unchanged; observations use actual +Redis read path without shared writes/orders/subscriptions. Coherence result is +review input for Claude; no replacement of complete TS DR01-DR08 acceptance. +Observer begins before first lifecycle call; only this frozen scope is approved. +A successful trial does not authorize release or rolling remaining roles. From 60a3cf79fb14455dacd424ab1c3860ec6aa6f703 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 17:46:05 +0000 Subject: [PATCH 019/147] Pin pre-action load-relative trial safety without waiving final gate --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 12 ++++++++++++ upgrade/evidence/p184-dl-phase-c-packet.md | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index ac6a6578..840ce0bc 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69638,3 +69638,15 @@ Redis read path without shared writes/orders/subscriptions. Coherence result is review input for Claude; no replacement of complete TS DR01-DR08 acceptance. Observer begins before first lifecycle call; only this frozen scope is approved. A successful trial does not authorize release or rolling remaining roles. + + +Pre-action safety refinement (no core has changed yet): the observer saw natural +old-runtime CPU7.07vCPU/30s, later around6.0, before rollout. The prior calm-window +5.978048 transient boundary would reinstate precisely the old-runtime resource +prerequisite owner removed. Preserve those pre-action alerts, do not call them +candidate failures. Before any trial, freeze revision2 safety at1.10 times the +greater of old p95 and latest unmodified60s mean: baseline5.843292, +trial stop threshold6.575853vCPU (two30s means). Full evidence and exact +intervals core-trial/trial-safety-v2.json. This is a transitional non-regression +guard, not resource-cap increase or release-gate change. Final5vCPU, coherence, +lag, quorum, integrity and reverse rollback rules remain mandatory. diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md index 5181976d..6be20056 100644 --- a/upgrade/evidence/p184-dl-phase-c-packet.md +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -269,3 +269,15 @@ Redis read path without shared writes/orders/subscriptions. Coherence result is review input for Claude; no replacement of complete TS DR01-DR08 acceptance. Observer begins before first lifecycle call; only this frozen scope is approved. A successful trial does not authorize release or rolling remaining roles. + + +Pre-action safety refinement (no core has changed yet): the observer saw natural +old-runtime CPU7.07vCPU/30s, later around6.0, before rollout. The prior calm-window +5.978048 transient boundary would reinstate precisely the old-runtime resource +prerequisite owner removed. Preserve those pre-action alerts, do not call them +candidate failures. Before any trial, freeze revision2 safety at1.10 times the +greater of old p95 and latest unmodified60s mean: baseline5.843292, +trial stop threshold6.575853vCPU (two30s means). Full evidence and exact +intervals core-trial/trial-safety-v2.json. This is a transitional non-regression +guard, not resource-cap increase or release-gate change. Final5vCPU, coherence, +lag, quorum, integrity and reverse rollback rules remain mandatory. From 893a30e39ae4981383ce0f2d7c13bee8d94815b3 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 18:10:36 +0000 Subject: [PATCH 020/147] Record failed core trial and verified reverse rollback --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 24 ++++++ upgrade/evidence/p184-dl-phase-c-packet.md | 13 ++++ upgrade/evidence/p184-dr07-runtime-closure.md | 76 ++++++++++++++++++- 3 files changed, 111 insertions(+), 2 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 840ce0bc..454161af 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69650,3 +69650,27 @@ trial stop threshold6.575853vCPU (two30s means). Full evidence and exact intervals core-trial/trial-safety-v2.json. This is a transitional non-regression guard, not resource-cap increase or release-gate change. Final5vCPU, coherence, lag, quorum, integrity and reverse rollback rules remain mandatory. + + +DR07 coretrial RESULT: FAILED_ROLLED_BACK (owner-approved gate-order trial). +Sourcec33c90f/image0d8da150 deployed exactly3cores, configchecksPASS; nootherroles. +TSdeployedcache coherence103/120Binance105/120OKX<117, FAIL. Reverse rollback +3->2->1 complete658a9570;13app+TSrolesrunning/noOOM. No replacementgreenwindow. +19purecandidateintervals/189.995s CPUmean5.158006,p95/max5.405644vCPU, NOT300sPASS. +Originalaggregate hitmissingcgroupkeyduringrollback; rawsamples preserved and +segmentedbyID/time beforedecision. Per-rolecores0.325581/Kafka3.117959vCPU. +Streamoverflow+4 BEFOREtrial,0additionalduringtrial/rollback; cannotcloseoldloop. +Guardgenerationresetfalsealarms corrected with5checks; overlap32.938s, no rawgap. +Coherencepredicate corrected(strictfresh vsquietproof) with10testchecks; allold +observeroutputsretained. Continuouslagtrendmissing, do notclaimgatePASS. + +Fullreport upgrade/evidence/p184-dr07-runtime-closure.md now records exactlatencies, +scopes, CPU,evidence androllback; packetstillHOLD for remainingroles/release. +Current canonical->actual RedisACK stillNOT_MEASURED; isolatedQueryprobe0requests, +stoppedbyharness/config/quota reserve androlloutcoordination, notproviderrefusal. +TSprobe38testsPASS inexistingimage; sourcehandoff only, noTSruntimeimagechange. +Alltestcontainers/network/captures/privateprobecopies removed;guardprobesfinished. +Keepactiveold/explicitcandidate0d8/namedrollback; nobroadprune orpush/merge. +StatusoverallIN_PROGRESS: close residualcoherence attribution + TSreset/ACK +integration + fullruntimeCPU/recovery/delivery gates + DR/broker evidence. +Do notwaitforTS Paperfeatures, do notrebuildunchangedimages orrepeatC2 toguess. diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md index 6be20056..90c4c36f 100644 --- a/upgrade/evidence/p184-dl-phase-c-packet.md +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -281,3 +281,16 @@ trial stop threshold6.575853vCPU (two30s means). Full evidence and exact intervals core-trial/trial-safety-v2.json. This is a transitional non-regression guard, not resource-cap increase or release-gate change. Final5vCPU, coherence, lag, quorum, integrity and reverse rollback rules remain mandatory. + + +## Executed trial outcome: FAIL, fully rolled back + +Owner-approved core-only trial executed; threecandidate0d8da150cores restored +inreverseorder to658a9570 because post-coreTScoherence was103/120and105/120, +not117/120. No otherrolewasrolled. PartialcandidateCPU5.158006vCPU,n19/189.995s, +not300sPASS. Full receipt andmeasurementlimitations in +[p184-dr07-runtime-closure.md](p184-dr07-runtime-closure.md). +Release/remainingroleauthorization remainsHOLD. This is not a newapprovalrequest +for the same failedimage: attribute residualBOOK/QUOTEconsumeralignment first, +coordinate pendingTSreset/ACK integration, then affected-proofbeforeanyretrial. +No source/manifest/freshness changes were made toforcecoherencePASS. diff --git a/upgrade/evidence/p184-dr07-runtime-closure.md b/upgrade/evidence/p184-dr07-runtime-closure.md index 8ac42ea9..bbe542c4 100644 --- a/upgrade/evidence/p184-dr07-runtime-closure.md +++ b/upgrade/evidence/p184-dr07-runtime-closure.md @@ -1,10 +1,12 @@ # DR07 Runtime Closure Checkpoint - 2026-10-07 -Status: IN_PROGRESS / PHASE_C_HOLD. This is not a release certificate. +Status: TRIAL_FAILED_ROLLED_BACK / RELEASE_HOLD. This is not a release certificate. Main plan: DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md, DR07 runtime closure continuation. Guide: DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md sections18.13-18.14. Evidence root: /home/bobby/.local/state/qdl-v2/dr07-runtime-20261007/. -No production restart/config/credential/manifest/cap or order mutation in this slice. +The pre-trial checkpoint below had no production mutation. The later owner-approved +core-only image trial and reverse rollback are recorded at the end; no other +role/config/credential/manifest/cap or order changes. No TS Paper feature dependency; DR01-DR08 and broker delta remain release inputs. ## Actual Current Runtime Baseline @@ -133,3 +135,73 @@ TS canonical /home/bobby/trading_system fix/dev-consumer-recovery-integration668 Codex TS worktree remains separate. Old DL core658a9570, projector a696cfba, Stream1f13408f, Query51450612, ingestors0f6876e1, BARf7351c3b unchanged. Full immutable rollback roster remains in the Phase C packet and inventory JSON. + + +## Actual Owner-Approved Core Trial And Rollback + +Owner explicitly approved gate-order change after this report's first checkpoint. +Plans committed4da52a8/60a3cf7 BEFORE lifecycle calls. Current-load transition +boundary6.5758528vCPU (two30s windows) and final5vCPU are distinct; no resource cap +was raised. Three cores rolled serially17:46-17:50UTC to0d8da150; private inspected +compose preserved effectiveconfig/mounts/TLS/aliases/caps, receipt checks passed. +Quorum leader2,epoch149,followerlag0; under-replicated partition list empty before +action. One subsequent admin read showed raw lag14-31records across6partitions, +canonical projector19-106. Continuous five-increasing-interval lag was NOT +collected: operational sampling was incomplete, do not claim that gatePASS. + +Frozen post-core gate via deployed TS MarketCacheReader, ETH each venue,1Hz: + +| Result (n=120 each venue) | Binance | OKX | +| --- | --- | --- | +| Exact BOOK top equals QUOTE, both eligible at read |103/120 |105/120 | +| Gate>=117/120 |FAIL |FAIL | +| Stored BOOK observed_at age p50/p95/max |654.197/1104.025/1218.918ms |652.626/1119.492/1276.319ms | +| Stored QUOTE observed_at age p50/p95/max |181.010/426.622/767.293ms |192.039/362.477/880.194ms | +| Cache reader call p50/p95/max |16.412/150.317/249.192ms |15.700/155.352/246.358ms | + +These are TS Redis read and stored-view-age measurements, NOT SDKrequest latency, +canonical commit latency, Redis ACK or alpha latency. Gate failed, so no second +window was used to replace it. Do not diagnose the residual stage from age alone. +BOOK and QUOTE remain independently updated streams. Core-only improvement did +not prove their end-to-end alignment; matched event/offset trace is still needed +at canonical, Stream, SDK and TSwriter before deciding the next source change. + +Core-only candidate CPU:17DLroles,19complete same-ID pre-decision intervals, +189.994570s, mean5.158006vCPU,p95/max5.405644. NOT a completed300s acceptance. +Measured cores combined0.325581vCPU; Kafka3.117959; projectors0.290687; +Query0.296130; Stream0.209910; BAR0.551557. Workload differed from earlierbaseline, +so before/after CPU is not a controlled causal performance ratio. +Full300s recorder continued across rollback and its original aggregate failed +on a missing cgroup usage key during recreate. Raw31samples retained. Segmented +report excludes11mixed/post-decision intervals and interpolates nothing: +core-trial/candidate-cpu-segmented.json. This does not change failed gate status. + +At17:56-17:57UTC rolled back3->2->1 to658a9570. Every rollback config checkPASS. +Final12DLapp+TSwriter13/13running, noOOM; unchanged9otherDLroles andshared TS. +Rollback consumer read5samples/venue: allQUOTE/BOOKfresh,4/5coherent each. This is +return-to-baseline evidence only, NOT coherence acceptance or fullTSqualification. + +Observer v1 falsely flagged6stalls when generation reset processed counters; +corrected identity=(containerID,generation),5controlledchecksPASS. Segments +actually overlapped32.938s, maxsamplegap10.141s; no rawcollectiongap. Preserve +original alerts, do not call them application crashes. Initial coherence +observer incorrectly required quiet-session receipt fields on strict fresh +BOOK/QUOTE; corrected feed-specific predicate,10TEST_ONLYchecksPASS; old5sample +outputs retained and not used tocertify. No new markettimestamp or freshnesspolicy. + +Stream1 overflow+4 beforefirstrollout, then0additional observed during trial/ +rollback; replayed+23202 acrosswholewindow. Stream2overflow+0. Counters have not +proved successful reset/cursor/ACK closure; unchangedTSwriter lacks the pending +DR07 integration. Wholewindow approximately17:41-17:59, not a steady-state proof. +Canonical-to-actual RedisACK andalpha cache remainNOT_MEASURED. The isolated +QueryACKattempt made0providerrequests (config/harness/quota-preflight stops); +no substituted latency is reported. Three remaining original gates are open, +plus residualBOOK/QUOTEcoherence andTS DR/broker case-indexed releaseinputs. + +Next permitted source work is attribution and coordinatedTSreset/ACK integration +in its own branch; no repeat imagebuild or release trial without an affected +fix. No Paperfeaturedependency, nounrelatedDLendpointchanges, nofreshnesswaiver. +Release and remaining9role packetHOLD. Allobserver processes/testresources stopped; +activeoldimages+explicitrc3candidate andnamedrollback kept for investigatedscope. +No push/merge/tag, noextraAPI/manifest change. Private rollbackpacket retained600 +outsideGit. Reportedbuild/paircleanup above remainsvalid; no broadDockerprune. From 794c90ea47e2d0d1af71d34e846fa033367c0c4e Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Wed, 7 Oct 2026 19:30:41 +0000 Subject: [PATCH 021/147] docs: record actual caller ACK trace and retain release gates --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 43 ++++++ upgrade/evidence/p184-dl-phase-c-packet.md | 14 ++ upgrade/evidence/p184-dr07-runtime-closure.md | 127 ++++++++++++++++++ 3 files changed, 184 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 454161af..3c802340 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69674,3 +69674,46 @@ Keepactiveold/explicitcandidate0d8/namedrollback; nobroadprune orpush/merge. StatusoverallIN_PROGRESS: close residualcoherence attribution + TSreset/ACK integration + fullruntimeCPU/recovery/delivery gates + DR/broker evidence. Do notwaitforTS Paperfeatures, do notrebuildunchangedimages orrepeatC2 toguess. + + +### DR07 exact-event caller continuation - 2026-10-07 + +Status IN_PROGRESS/RELEASE_HOLD, owner explicitly approves finishing exact-event +canonical -> Stream -> SDK -> TS writer trace, reset/ACK caller integration, +recovery, full-stack CPU/continuous lag and DR01-DR08 plus broker delta receipts. +Guide18.13-18.14/Phase C invariants unchanged. Prior three-core trial remains +FAILED_ROLLED_BACK; do not repeat it without new attribution. No Paper dependency. +Work only DL fix/p184-dl-closure and TS codex/dr07-reset-anchor, never edit/test +Claude checkout. TS patch scope actualmarket_data_service_p184w only; inspect +source/image/config first, retain9150385e rollback and existing state/identity. +No new credentials, quotas, freshness, publicinterface, datareset, or newservice. +Freeze concrete packet/digest before mutation; receipt reports exact changes. +Canonical read_committed visibility is not a broker commit clock. Pair same +identity/partition/offset/generation; bounded trace must identify missingpoints. +Stop on integrity/ownership failure or readiness regression; preserve errors, +roll back changed writer if gates fail. No push/merge/release in this slice. + +TS source finding reproduced: bridge BOOK_DELTA used market session budget, +not adapter BOOK-specific budget. Commit83131d4 changes selector only;7newcases +3FAIL before, full bridge/reset144PASS0skip with isolatedRedis after. Exact +RedisDB15zero and testnetwork removed. Deployedwriter unchanged, SDK2.0.7; +fixedworkers/main/cache_projector/cache_reader/execution_ordering bytes match +Codex source before trace edits. Overlay will not import unrelated Paper code. + + +### DR07 caller runtime receipt - 2026-10-07 + +Status CALLER_DEPLOYED_OBSERVED / RELEASE_HOLD. Full continuation receipt: +[Actual caller rollout](upgrade/evidence/p184-dr07-runtime-closure.md#actual-caller-rollout-and-exact-event-receipt---2026-10-07). +TS source83131d4/452955f, only writer5849d716 deployed, rollback9150385e retained. +144/529/491 PASS separate overlapping suites, zero skip. 2556 exact BOOK event +joins to actual Redis ACK, zero identity mismatches. 31/31 READY60/60sessions; +execution eligibility minimum47/60, not FULL. Trace timeout retained UNKNOWN +with successful subsequent same-generation MARK apply1121ms later. +Continuous lag2160 valid samples, no five-increase alarm; no overflow/replay growth +in300s. DL17-role mean5.303766vCPU FAIL, old/current DL runtime unchanged. +Prior core trial remains failed/rolled back. DR01-08/broker indexing still pending; +DL DR07 hardening must not be confused with TS DR07 reference-metrics family. +Do not wait for Paper features; do not issue release on these partial results. +Cleanup context removed, disk93753360384->93753577472bytes; tests/networks gone. +No broad prune concurrent caches, no data reset, no push/merge/release. diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md index 90c4c36f..3b09af84 100644 --- a/upgrade/evidence/p184-dl-phase-c-packet.md +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -294,3 +294,17 @@ Release/remainingroleauthorization remainsHOLD. This is not a newapprovalrequest for the same failedimage: attribute residualBOOK/QUOTEconsumeralignment first, coordinate pendingTSreset/ACK integration, then affected-proofbeforeanyretrial. No source/manifest/freshness changes were made toforcecoherencePASS. + + +## Caller integration continuation: release remains HOLD + +2026-10-07 actual TS writer now image5849d716 (source452955f); named rollback9150385e. +Only caller changed. 2556 exact BOOK event-to-Redis-ACK joins,31/31 READY60/60sessions, +no Stream overflow/replay growth in300s. All17 DL roles unchanged; CPUmean5.303766 +p955.502266vCPU,n30/299.992s FAIL5gate. Continuous lag2160valid/0unknown, longest +increase streak4. This does not reopen authorization to retry failed0d8da150 +cores without new attribution, nor certify candidate CPU or all65DR+11brokercases. +See [full receipt](p184-dr07-runtime-closure.md#actual-caller-rollout-and-exact-event-receipt---2026-10-07). +Prior coherence and rollback conditions remain mandatory; no extra interface, +freshness, resource-cap or policy change. Broker packet ownership must be reconciled +without making unfinished Paper features a Data Layer release prerequisite. diff --git a/upgrade/evidence/p184-dr07-runtime-closure.md b/upgrade/evidence/p184-dr07-runtime-closure.md index bbe542c4..62462c7c 100644 --- a/upgrade/evidence/p184-dr07-runtime-closure.md +++ b/upgrade/evidence/p184-dr07-runtime-closure.md @@ -205,3 +205,130 @@ Release and remaining9role packetHOLD. Allobserver processes/testresources stopp activeoldimages+explicitrc3candidate andnamedrollback kept for investigatedscope. No push/merge/tag, noextraAPI/manifest change. Private rollbackpacket retained600 outsideGit. Reportedbuild/paircleanup above remainsvalid; no broadDockerprune. + + +## Actual caller rollout and exact-event receipt - 2026-10-07 + +Status: CALLER_DEPLOYED_OBSERVED / RELEASE_HOLD. This supersedes only the earlier +NOT_MEASURED statement for event-to-actual-TS-Redis-ACK, not the failed core trial. +No DL runtime role changed in this continuation. No push, merge or release. + +### Source, packaging and runtime + +TS branch codex/dr07-reset-anchor: 83131d4 corrects BOOK session-budget selection; +452955f adds bounded opt-in tracing to actual fixed/single workers. Bridge/reset +144 PASS, source trace suite 529 PASS, packaged image 491 PASS, all zero skip; +these overlapping suites must not be summed. Tests used disposable Redis; no +shared financial rows, credentials or order submission. Bridge counterfactual: +3/7 regressions failed before correction. Actual deployed market and BOOK budgets +are both 45000ms, so this selector is a latent mismatch fix, NOT attribution of +the observed runtime delay. + +Only market_data_service_p184w was replaced at 19:15:49 UTC with +sha256:5849d7167561273af08bda2103c38cec5f4480f7a8809ea592c57abff7716ea1, +source 452955f7b9f32a2b757caf7d9bcfac73e3cf188f, six app files over current base. +SDK2.0.7, binding V2 r22, JWT10 and state/TLS/namespace/policies/caps retained. +No whole Paper upgrade. Explicit rollback container +market_data_service_p184w-dr07-rollback retains +sha256:9150385e2bb3596793b6ee209dbc99696c3ed07439d6f4199356f417a97b6105. +Old writer stopped after 30s ceiling with exit137, no OOM: do NOT certify graceful +shutdown. New writer restart0/OOMfalse. No concurrent active writer or cache reset. +Trace is bounded 4096 records/process with absolute expiry; four exports, +16384 records, one process omitted three in-flight records at cap. Expired trace +settings are inert; no extra recreate solely to remove them. + +### Actual measurements (all latency in ms) + +Exact canonical record joins require raw protobuf event-id SHA256, offset, feed, +source hash, instrument, generation, native sequence and all envelope clocks. +2556 exact matched ETH BOOK events, zero validation mismatches, all Redis EVAL1. +Canonical observer is read_committed, explicit assignment, no group join/commit. +Its observation clock is NOT the broker commit clock. Negative observer-to-SDK +intervals reflect independently scheduled readers; never label them negative +processing latency. Core normalized/received clocks were identical in the 701 +baseline records, not independent stages. Stream dispatch clock NOT_MEASURED. + +| Feed / venue | n | SDK return -> Redis ACK p50 / p95 / max | Event -> Redis ACK p50 / p95 / max | +| --- | ---: | --- | --- | +| BOOK_SNAPSHOT Binance | 169 | 57.893 / 136.682 / 360.574 | 168.772 / 275.901 / 1050.128 | +| BOOK_SNAPSHOT OKX | 173 | 58.736 / 191.736 / 557.328 | 171.002 / 319.197 / 1083.462 | +| BOOK_DELTA Binance | 1127 | 39.487 / 230.395 / 1335.608 | 155.282 / 474.117 / 1708.935 | +| BOOK_DELTA OKX | 1087 | 40.281 / 243.220 / 1470.926 | 159.590 / 440.066 / 1639.998 | + +BOOK_DELTA SDK->writer p99: Binance650.2/OKX543.0ms. Writer duration p95 +66.2/61.3ms, Redis apply wait p95 32.0/29.4ms. Tail is predominantly before +writer entry; batching/projection/preceding work are included. Without batch IDs +and SDK ACK clocks this does NOT establish a specific queue/ACK root cause. +Largest backlog events precede the resource sample window. 1192 unmatched trace +keys are bounded-capture/sampling coverage, not proof of loss; 348 rows lack a +stream-event join key (query/warmup). No extrapolation to alpha-cache ACK. + +Preserved negatives: 85 ordered older-state no-write decisions, seven duplicates, +one MARK UNKNOWN/TIMEOUT. The OKX SOL generation104 timeout was followed by EVAL1 +with the same identity/source/generation and advancing component clocks 1121ms +later. Subsequent recovery is observed; original ACK remains UNKNOWN. +Four MARK startup timeouts reconnected. In the subsequent 300s no-order window, +31/31 worker samples READY, session60/60; minimum execution-eligible47/60. +Session-ready is NOT all-prices-eligible. Cursor ACK23021->90457, +commits17772->69078, persistence errors0. This proves actual restart/progression, +not full deliberate reset-fault, zero-downtime or 5000event/s certification. + +### Resource and continuous lag gates + +DL unchanged stack: 17 containers, 30 intervals/299.992s, +mean5.303766/p95 5.502266/max5.545844 vCPU: FAIL <=5 gate. TS writer measured +separately mean1.323835vCPU, not included in DL sum. All sampled identities stable, +no new OOM. Kafka alone mean2.970528vCPU; no resource-cap changes. +Active Stream overflow3389->3389 and replay1891705->1891705; passive109->109 and +1138433->1138433 over the observed window. No loop observed, not a burst guarantee. + +Lag union19:14:50.963-19:24:36.602UTC: five groups, 36 group/topic/partition +series, 60 observations each, 2160/2160 valid, unknown0. 473/2124 comparisons +increased, longest consecutive streak4; zero five-increase alarms, including +across file boundary. Maximum sample gap10.046s, process overlap4.318s. These +are read-uncommitted log-end minus committed offsets, not LSO/processing latency. +All bounded observer processes finished. No duplicate consumer/quota consumption. + +### Evidence and remaining release boundary + +Evidence root: /home/bobby/.local/state/qdl-v2/dr07-runtime-20261007/continuation/. +Commands and raw bounded outputs: writer_packet.py + writer-packet/roll-*.json; +writer-image-test/receipt.json; caller_resource_window.py + caller-window-300s/; +readiness_observer.py + writer-packet/readiness-300s.jsonl; +readonly_kafka_observer.py + writer-window-canonical-420s/; +writer-window-lag-union-final-1791401299658010600.json; +writer-same-event-preliminary/{summary,tail-attribution,outcome-denominators}.json. +Core candidate0d8da150 remains undeployed after failed trial; active three cores +remain658a9570. Do not substitute this old-DL-runtime CPU window for candidate +qualification or erase prior coherence103/120 and105/120 failures. + +DR label warning: this runtime-hardening DR07 is NOT R2's DR07 reference-metrics +case family. TS case-matrix has65 DR cases (9/8/6/10/12/4/12/4), broker matrix11. +Read-only source inspection found no complete indexed DR01-DR08/broker receipt; +this is missing acceptance indexing/evidence, not a claim every case failed. +Broker-delta packet is still DRAFT; its Paper-completion prerequisite contradicts +owner's latest release boundary. Reconcile it with Claude's active ownership; +do not deploy its twelve-role packet blindly, wait for Paper features, or mark +these receipts PASS from the BOOK trace. + +Next permitted work stays narrow: attribute BOOK/QUOTE alignment on exact frames +before retrying the failed core candidate; profile pre-writer batching if tails +violate the applicable feed contract, prove affected reset/fault path; qualify +candidate CPU <=5 and the existing coherence gate; collect/index actual DR and +broker delta evidence independently of Paper feature construction. No interface, +freshness, durability, quota or benchmark threshold waiver is implied. + +### Cleanup and provenance + +Disposable tests/Redis/networks removed; observer processes completed. Build +context (seven files) removed after source/image hashes saved. Free disk +93753360384 ->93753577472 bytes; concurrent host activity prevents exclusive +attribution. Docker inventory94 images/28.31GB, BuildKit444/18.29GB. No broad prune: +concurrent Claude cache ownership is not isolated; active image5849, named9150 +rollback and existing explicit DL candidate/rollback retained. No shared volume, +financial data, cursor or raw provider state deleted. Bounded trace evidence +retained outside Git; private packet/inspect remain mode0600 for rollback. +Canonical DL /home/bobby/data_layer main5bf5cd5, published v2.2.2 unchanged; +feature fix/p184-dl-closure. Canonical TS /home/bobby/trading_system +fix/dev-consumer-recovery-integration@668e103 unchanged; active patch source lives +only in codex/dr07-reset-anchor. Claude's upgrade worktree was not modified. From 1cd24e367bc9b3216318da740d0e7ab3f3ddba54 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 03:01:20 +0000 Subject: [PATCH 022/147] docs: remove bar edge retry storm and gate coherence on the TS product path Single owner-approved runtime change: binance_bar_edge to the v2.2.2 Python image (fba8863), ending the Binance 3d overlap retry storm (0.56 vCPU, ~850 req/s per broker). DL observed 4.37/4.30 mean vCPU in two 300 s windows. Product-path coherence 120/120 per venue on current cores with TS IDR-076. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 12 +++++ upgrade/evidence/p184-dr07-runtime-closure.md | 45 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 3c802340..72567999 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69717,3 +69717,15 @@ DL DR07 hardening must not be confused with TS DR07 reference-metrics family. Do not wait for Paper features; do not issue release on these partial results. Cleanup context removed, disk93753360384->93753577472bytes; tests/networks gone. No broad prune concurrent caches, no data reset, no push/merge/release. + +### DR07 Claude continuation - bar edge storm removed, product coherence gate - 2026-10-08 + +Owner-authorized continuation of Codex's DR07 closure. Found by CPU attribution: binance_bar_edge on 2.2.0-rc4 +(without fba8863) retried Binance 3d overlapping history every ~2.4 s forever (attempt ~19,500), costing 0.56 vCPU, +~850 Kafka request segments/s per broker, rust_core admissions and Binance REST quota. Owner-approved single-role +recreate 02:17:33 UTC to qdl-v2-python:2.2.2-d6d2637 (sha256:51450612...); config/mounts/limits and baked +acquisition/catalog/manifests unchanged; rollback compose retained. Whole DL 5.1996/5.4454 (mean/p95) -> +4.3657/4.5680 and 4.2965/4.6351 in two later 300 s windows (lower load; one earlier window unattributed spike). +Coherence gate redefined by the owner to the TS product path; with TS IDR-076 (bounded re-read) the current cores +reach 120/120 per venue. Core candidate not needed for coherence. Evidence: +upgrade/evidence/p184-dr07-runtime-closure.md (Claude continuation). Status RELEASE_HOLD for DR/broker receipts. diff --git a/upgrade/evidence/p184-dr07-runtime-closure.md b/upgrade/evidence/p184-dr07-runtime-closure.md index 62462c7c..6d91936b 100644 --- a/upgrade/evidence/p184-dr07-runtime-closure.md +++ b/upgrade/evidence/p184-dr07-runtime-closure.md @@ -332,3 +332,48 @@ Canonical DL /home/bobby/data_layer main5bf5cd5, published v2.2.2 unchanged; feature fix/p184-dl-closure. Canonical TS /home/bobby/trading_system fix/dev-consumer-recovery-integration@668e103 unchanged; active patch source lives only in codex/dr07-reset-anchor. Claude's upgrade worktree was not modified. + + +## Claude continuation - CPU source, bar edge storm and product-path coherence - 2026-10-08 + +Status: CPU_GATE_OBSERVED_PASS_TWO_WINDOWS / COHERENCE_PRODUCT_GATE_PASS_CURRENT_CORE / RELEASE_HOLD (DR/broker +case receipts still open). Owner authorized this continuation; the single runtime change was approved explicitly. +Evidence root (outside Git): /home/bobby/.local/state/qdl-v2/p184-claude-dl-closure-20261008/ (cpu_window.py, roll_bar_edge.py, +coherence_attempts.py, cpu-*.json, rates-*.json, coherence-attempts-current-core.json, private/ mode 0600). + +CPU attribution (read-only): Kafka per-thread CPU (/proc task stat) put most broker CPU in data-plane network/request +threads; per-peer socket segment rates (nsenter ss -ti, 20 s) showed inter-broker replication largest and +binance_bar_edge at ~850 segments/s on EVERY broker with ~50 KiB/s. canonical carries 511 records/s plus 193 +transaction markers/s (core batch_wait_ms 25); raw has no markers. Bar edge log: Binance 3d history refused +OVERLAPPING_PROVIDER_WINDOWS attempt ~19,500, retry ~2.4 s, image 2.2.0-rc4-4fe7e92 without fba8863 (the v2.2.2 +release record kept f7351c3b, recreated:false). + +Runtime change (owner-approved, only role binance_bar_edge, 02:17:33 UTC): f7351c3b -> qdl-v2-python:2.2.2-d6d2637 +sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9 (image query already runs). Compose from +inspect, only image differs; config_unchanged true, running, OOM false, restarts 0. stable-acquisition-bindings.yaml, +consumer manifests and catalog sha256 identical in both images; checkpoint mount unchanged. Rollback: +private/rollback.compose.json (f7351c3b). After restart a one-time history coverage recheck ran ~25 min (settled +02:44 UTC, published_rows 0 on rechecked bindings). Overlap now attempt=1 retry 86400 s, 383-row recent suffix. + +| 300 s window (17 DL roles, cgroup usage_usec) | mean | p95 | max | bar edge | Kafka canonical/raw msg/s | +| --- | ---: | ---: | ---: | ---: | --- | +| before (01:47, local CI ran on host meanwhile) | 5.1996 | 5.4454 | 5.4784 | 0.5643 | 645/528 (01:30, not simultaneous) | +| after 1 (02:44) | 5.0258 | 6.4545 | 6.5009 | 0.0171 | not measured; series not recorded | +| after 2 | 4.3657 | 4.5680 | 4.6772 | - | 608.0/463.7 (same window) | +| after 3 | 4.2965 | 4.6351 | 4.6714 | - | 560.3/419.0 (same window) | + +Load was 10-15% lower in windows 2-3; the direct, load-independent saving is the bar edge 0.55 vCPU. Window 1's +spike is unattributed. This is observed runtime evidence, not a peak-load certificate; the formal gate still needs a +measured busy window. + +Product-path coherence (owner 2026-10-08: gate on the TS product path, not single reads). TS fix IDR-076 (TS +2cebb7d): package admission re-reads BOOK+QUOTE after rollback up to 15 times, 100 ms apart, inside its 3 s readset +deadline; exact-equality rule unchanged. Measured on the CURRENT cores 658a9570 through the deployed TS +MarketCacheReader (Redis GET; re-read policy emulated in the probe), 120 attempts/venue at 1 Hz: +Binance 120/120 coherent within deadline (first read 108/120; time p50/p95/max 10.8/352.3/1091.2 ms; reads p95/max +4/11); OKX 120/120 (108/120; 10.6/332.8/416.3 ms; 3/4). The top-change core candidate 0d8da150 is therefore not +needed for this gate; it raises snapshots x1.47 and should not ride the release for coherence. + +Open: DR01-DR08 + broker-delta case receipts (TS runners; DR07 runner TS 74852a8), deliberate reset/fault proof on +the writer (Codex reset-anchor now merged into the TS feature branch, dfc70bd), a busy-window CPU gate, and a DL +release record that names the new bar edge image. No push/merge/release. From 704552d8736905cdb73feec86f3f01d339d02cbb Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 03:41:47 +0000 Subject: [PATCH 023/147] feat: prepare paired Paper inverse entitlement and record busy release evidence --- CERTIFICATION_LEDGER.md | 18 ++ DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 77 ++++++++ scripts/prepare_paper_read_extension.py | 155 +++++++++++++++ tests/test_paper_read_extension.py | 137 +++++++++++++ ...YER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md | 17 ++ upgrade/evidence/p184-dl-phase-c-packet.md | 50 +++++ .../evidence/releases/v2.2.3/RELEASE_NOTES.md | 65 ++++++ .../releases/v2.2.3/release-preparation.json | 186 ++++++++++++++++++ 8 files changed, 705 insertions(+) create mode 100644 scripts/prepare_paper_read_extension.py create mode 100644 tests/test_paper_read_extension.py create mode 100644 upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md create mode 100644 upgrade/evidence/releases/v2.2.3/release-preparation.json diff --git a/CERTIFICATION_LEDGER.md b/CERTIFICATION_LEDGER.md index b47330b0..5655de60 100644 --- a/CERTIFICATION_LEDGER.md +++ b/CERTIFICATION_LEDGER.md @@ -2154,3 +2154,21 @@ Redis-ACK/alpha cache latency NOT_MEASURED. Phase C HOLD, not authorized/deploye Only TS DR01-DR08 includingDR07 +broker delta are release dependencies; no Paper feature construction added. Prior certificates kept immutable and scoped, no freshness/quota/resource threshold change. No required endpoint replacement identified. + + +### 2026-10-08 - Bar edge inherited; v2.2.3 preparation only + +Owner-approved bar edge51450612 (sourced6d2637) replacedf7351c3b at02:17:33UTC. +No other role changed in this Codex continuation. Existing658a9570 cores retained; +coherence uses owner-approved TS IDR076 productpath. Failed0d8da150 excluded. +[Prepared v2.2.3](upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md) names17runtime +images and rollback; NOT certificate/tag. Busywindowmean4.882532vCPU,n30/299.992s +meets frozen300smean<=5; p955.564052/max5.870669 exceed5. Canonical/raw952.48/ +826.28offsets/s exceed threebaselines, markersincluded, not3000payload/sproof. +Streamoverflow+2/replay+15901: TSPaperBTC TRADEqueueclosures bothvenues, followed +byBinance replayOUT_OF_RANGE remain affectedrecoveryfindings. No evidence erased. +IDR075prepared/tested: DL25/TS28PASS, realpublicpacket validatedbothsides. Target +manifest/JWT11,bindingV3r23/64products; runtime10/r22. Fourinversefeeds addedonly +topreparedartifacts, linearINDEX alreadyexists. Pairedruntimepreflight, DR01-08/ +brokerreceipts, releaseattestation/CI and approvedrollout remain gates. No new +build/push/merge/tag or further runtime mutation. See mainplan for cleanup. diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 72567999..f862c736 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69729,3 +69729,80 @@ acquisition/catalog/manifests unchanged; rollback compose retained. Whole DL 5.1 Coherence gate redefined by the owner to the TS product path; with TS IDR-076 (bounded re-read) the current cores reach 120/120 per venue. Core candidate not needed for coherence. Evidence: upgrade/evidence/p184-dr07-runtime-closure.md (Claude continuation). Status RELEASE_HOLD for DR/broker receipts. + + +### IDR-075 paired Paper entitlement and release-record continuation - 2026-10-08 (Codex) + +Status IN_PROGRESS / PREPARED_ONLY until approved runtime packet. Owner handoff +supersedes the old core-first coherence gate: current658a9570 cores retained; +TS product-path IDR-076 governs coherent reads. Existing bar edge51450612 +rollout is inherited with immutable evidence, not repeated. Scope: simultaneous +readonly300s CPU/input-rate sample; prepare next DL release record; additive +Paper inverse entitlement and TS manifest-revision/hash pairing. No other role +may change without its exact approved packet. No push/merge/tag/publication. +Read guide18.13-18.14 and PhaseC packet; do not wait for Paper feature completion. +Source inspection shows BTC-USDT-SWAP MARK_INDEX_PRICE already entitled: preserve +and prove its INDEX component path, do not add duplicate or confuse standalone +INDEX reference products. Four inverse feeds only; copy ETH policy without +weakening freshness/realm/quota, no TRADE/BAR additions. Use deployed catalog +identity and bindingr22, not the source catalog that lacks runtime inverse rows. +Prepare public artifacts outside runtime; sealedV3 rev11/r23 and paired TSenv +change are not live authorization. Negatives: wrong realm/source/units, checksum, +revision drift, missing pair, old policy mutation, duplicate/unapproved feed. +Rollback preserves original paper manifest10/bindingr22/JWT10 together; no cache, +cursor or financial mutation. Busy window must be supported by simultaneous +input rates, not inferred from a CPU average. Cleanup only owned temp resources. + + +### IDR-075 preparation and busy-window receipt - 2026-10-08 (Codex) + +Status PREPARED_NOT_ACTIVATED / RELEASE_HOLD. Owner's latest scope observed: +no core candidate rollout, no reader/writer/runtime changes, no push/merge/tag. +Release preparation v2.2.3 now records actual17role composition and inherited +bar edge51450612 instead of stalef7351c3b. Published v2.2.2 not rewritten. +See upgrade/evidence/releases/v2.2.3/{RELEASE_NOTES.md,release-preparation.json} +and governing2026-10-08 section in upgrade/evidence/p184-dl-phase-c-packet.md. + +Busy readonlywindow03:21:03-03:26:03UTC,31samples/30intervals/299.992s: +DLmean4.882532,p955.564052,max5.870669vCPU. Frozen300smean<=5 observedPASS; +not allintervals<=5, not3000payload/s certificate. Input canonical952.48/raw826.28 +log-end offsets/s over302.110s, higher than all3priorbaselines; includesmarkers. +BARmean0.016960vCPU. Noimage/config/restart/OOMchanges. ActiveStream overflow+2, +replay+15901/subscriptions+3. Exactboundedlogs identify TS Paper BTC TRADE on +Binance/OKX queued1000, then Binance replay10000 -> OUT_OF_RANGE. Do not call +missingdata or infinite loop proved; affectedrecovery remainsopen. One TS +BinanceBTC MARK timeout. No new consumerquota/probe used. Passiveunchanged. +Evidence: ~/.local/state/qdl-v2/p184-codex-release-20261008/cpu/receipt.json +SHA256e1434fe1 (fullhash in release record); overflow-attribution.json retains +three exactclosures, commands/hash/samples preserved. No repeatedmeasurement. + +IDR075: scripts/prepare_paper_read_extension.py uses deployed public catalog and +paperrev10/bindingr22, adds four already-acquired inverse feeds with ETHpolicy, +keeps all60 products/61 priorrequirements/quotas. Target65manifestrequirements, +64V2products, manifest/JWT11,bindingV3r23. BTC-USDT-SWAP MARK_INDEX_PRICE was +alreadygranted; performance referenceINDEX uses same feed, no duplicategrant. +No catalog/acquisition/sourcepolicy update or USD=USDT assumption. V3policies +sealedfromactualmanifest, not guesseddefaults; consumerenvironmentpair validated. +DLtests25PASS0fail0skip (unittest,89.939s) including9newnegatives/positives and +existing sandbox/consumer/inversecontracts. Host lackedredis; image lackedpytest; +used existing image's unittest, no dependencyinstall/build. TS28PASS0skip and +actualpublicpacket validationPASS by strict TSparser, sourcefeature notoldbranch. +Prepared packet SHA25627470ade38a9e638884de3f37fe41c8a66dc1c15aeca7de3d00bba808889a088; +binding5a9a19385ae34d34213284f06aac8e5c7eb687068702f493590f2664dc0accdf; +pairedreleasehash6d35f2cf26d9a348d3000ffbf80dc5266dbbe6e8c7cbc0fdc0675345f501ad27. +Public artifacts/commands/receipts under sameevidenceroot/idr075/. Runtime still +JWT10/bindingV2r22, no intended-read acceptance claimed for inversePaper. + +Next: reviewed exactreader/callerpacket, simultaneousgenerationcutover, actual +Paperidentity no-orderpreflight, DR01-08 andbrokerreceipt; no waitforPaperfeatures. +Do not infer permission to recreate any role from purepreparedfiles. Record new +TRADE recovery finding with Claude before closing release, no buffer/gatewaiver. + +Cleanup: all --rm/no-networktestcontainers gone, no newimages/buildcache. Agents +andboundedobserversfinished. Mergedclean codex/dr07-reset-anchor worktree/branch +removed afterancestorcheck(dfc70bd merge), noClaude/Portal +checkout removed. Only owncompiledscriptcache removed. Diskfree105165643776 -> +105203245056bytes, concurrenthost noexclusiveattribution. Inventory94images28.31GB, +BuildKit444/18.29GB; retaincurrent/candidate/namedrollback, no broadprune. Canonical +DL /home/bobby/data_layer main5bf5cd5/v2.2.2 unchanged, featurefix/p184-dl-closure; +TS canonical668e103 unchanged, workon feat/v2-rust-first-okx-demo perowner. diff --git a/scripts/prepare_paper_read_extension.py b/scripts/prepare_paper_read_extension.py new file mode 100644 index 00000000..16125823 --- /dev/null +++ b/scripts/prepare_paper_read_extension.py @@ -0,0 +1,155 @@ +"""Prepare the owner-approved IDR-075 Paper entitlement; never activate it.""" +from __future__ import annotations + +import argparse +from copy import deepcopy +import hashlib +import json +from pathlib import Path + +import yaml + +from qdl.consumer.manifest import ConsumerManifestLoader +from qdl.consumer.realtime_route import requirement_key +from qdl.consumer.universal_release import ConsumerRouteBinding +from qdl.runtime.stable_catalog import StableSourceCatalog + +CONSUMER = "trading-system.paper.stable" +SUBJECT = "spiffe://qdl/paper/trading-system-stable" +FEEDS = frozenset({"QUOTE", "MARK_INDEX_PRICE", "BOOK_SNAPSHOT", "BOOK_DELTA"}) + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), + allow_nan=False).encode()).hexdigest() + + +def key(row): + return (row["instrument_uid"], row["feed"], row.get("interval"), row["source_policy_id"]) + + +def prepare_paper_extension(*, catalog, manifest, binding): + """Reuse deployed inverse feeds; no acquisition, credential or runtime mutation.""" + current = StableSourceCatalog.from_mapping(catalog) + consumer = ConsumerManifestLoader.from_mapping(manifest) + base = ConsumerRouteBinding.from_canonical_mapping(binding) + if (consumer.consumer_id != CONSUMER or consumer.subject != SUBJECT + or consumer.environment != "paper" or consumer.execution_dependency != "PAPER_ONLY" + or base.consumer_id != CONSUMER + or base.consumer_manifest_revision != consumer.manifest_revision): + raise ValueError("Paper identity/revision mismatch") + requirements = {key(r): r for r in manifest["spec"]["requirements"]} + products = {key(r): r for r in binding["products"]} + if (len(requirements) != len(manifest["spec"]["requirements"]) + or len(products) != len(binding["products"]) + or not products.keys() <= requirements.keys()): + raise ValueError("duplicate or unentitled base product") + old_policies = {p["requirement_id"]: p for p in binding.get("request_policies", [])} + for k, route in products.items(): + req = requirements[k] + record = current.instrument_for(k[0]) + if (route["instrument_id"], route["native_symbol"], route["venue"], + route["market"], route["product_type"]) != ( + record.identity.instrument_id, record.native_symbol, record.identity.venue, + record.identity.market, record.identity.product_type.value): + raise ValueError("base catalog identity mismatch") + if (route["max_freshness_ms"] != req["max_freshness_ms"] + or route["require_final_bars"] != req.get("require_final_bars", True)): + raise ValueError("base product quality mismatch") + expected = dict(requirement_id=route["requirement_id"], + event_recency_policy=req.get("event_recency_policy"), + max_session_liveness_ms=req.get("max_session_liveness_ms")) + if old_policies and old_policies.get(route["requirement_id"]) != expected: + raise ValueError("base sealed request policy mismatch") + + def instrument(symbol): + rows = [r for r in catalog["instruments"] if r.get("native_symbol") == symbol + and r.get("venue") == "OKX" and r.get("market") == "SWAP" + and r.get("product_type") == "PERPETUAL"] + if len(rows) != 1: + raise ValueError("missing or ambiguous already-acquired instrument") + return rows[0] + + inverse, linear, template = map(instrument, ("BTC-USD-SWAP", "BTC-USDT-SWAP", "ETH-USDT-SWAP")) + if (inverse["base_asset"], inverse["quote_asset"], inverse["settlement_asset"], + inverse.get("attributes", {}).get("ctType")) != ("BTC", "USD", "BTC", "inverse"): + raise ValueError("inverse collateral identity mismatch") + fx_key = (linear["instrument_uid"], "MARK_INDEX_PRICE", None, "crypto_liquid_v2") + if fx_key not in products: + raise ValueError("existing BTC-USDT-SWAP MARK/INDEX FX entitlement missing") + if any(k[0] == inverse["instrument_uid"] for k in requirements): + raise ValueError("inverse extension already present; explicit revision review required") + target, new_binding = deepcopy(manifest), deepcopy(binding) + target["metadata"]["revision"] += 1 + for feed in sorted(FEEDS): + policy = "crypto_primary_v2" if feed == "QUOTE" else "crypto_liquid_v2" + k = (template["instrument_uid"], feed, None, policy) + if k not in requirements or k not in products: + raise ValueError("missing exact ETH execution template") + req, route = deepcopy(requirements[k]), deepcopy(products[k]) + if (req["consumer_grade"] != "EXECUTION" or req.get("stale_policy") != "BLOCK" + or req.get("gap_policy") != "BLOCK" or not req.get("require_full_coverage") + or route["route"] != "V2_PRIMARY" or route["fallback"] != "BLOCKED" + or not route["execution_grade"]): + raise ValueError("template is not fail-closed execution policy") + req["instrument_uid"] = inverse["instrument_uid"] + route.update(instrument_uid=inverse["instrument_uid"], + instrument_id=inverse["instrument_id"], native_symbol=inverse["native_symbol"]) + target["spec"]["requirements"].append(req) + new_binding["products"].append(route) + parsed = ConsumerManifestLoader.from_mapping(target) + parsed_requirements = {requirement_key(r): r for r in parsed.requirements} + new_binding["request_policies"] = [] + for row in new_binding["products"]: + k = f"{row['instrument_uid']}:{row['feed']}:{row.get('interval') or ''}:{row['source_policy_id']}" + req = parsed_requirements[k] + current.binding_for(req) + expected_id = hashlib.sha256(f"{CONSUMER}:{k}".encode()).hexdigest() + if key(row) in products and row["requirement_id"] != expected_id: + raise ValueError("base requirement identity mismatch") + row["requirement_id"] = expected_id + recency = req.event_recency_policy + new_binding["request_policies"].append(dict( + requirement_id=expected_id, + event_recency_policy=None if recency is None else str(recency.value), + max_session_liveness_ms=req.max_session_liveness_ms)) + provenance = dict(schema="qdl.v2.paper-extension-preparation.v1", + status="PREPARED_NOT_ACTIVATED_NOT_CERTIFIED", runtime_mutations=0, + base_binding_sha256=base.binding_sha256, + base_manifest_sha256=consumer.manifest_sha256, + consumer_manifest_sha256=parsed.manifest_sha256, + deployed_catalog_sha256=digest(catalog), + added_products=4, existing_fx_product_preserved=True) + new_binding.update(schema="qdl.v2.consumer-route-binding.v3", + consumer_manifest_revision=parsed.manifest_revision, + release_revision=base.release_revision + 1, + universal_manifest_sha256=digest(provenance)) + for field in ("products", "request_policies"): + new_binding[field].sort(key=lambda row: row["requirement_id"]) + new_binding.pop("binding_sha256") + new_binding["binding_sha256"] = digest(new_binding) + ConsumerRouteBinding.from_canonical_mapping(new_binding) + return dict(manifest=target, binding=new_binding, provenance=provenance, + consumer_environment={ + "DATA_LAYER_V2_JWT_MANIFEST_REVISION": str(parsed.manifest_revision), + "DATA_LAYER_V2_RELEASE_MANIFEST_SHA256": new_binding["universal_manifest_sha256"]}) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("catalog", "manifest", "binding", "output"): + parser.add_argument("--" + name, type=Path, required=True) + args = parser.parse_args() + result = prepare_paper_extension(**{name: yaml.safe_load(getattr(args, name).read_text()) + for name in ("catalog", "manifest", "binding")}) + # Never overwrite an earlier preparation or a live manifest. + with args.output.open("x") as out: + json.dump(result, out, indent=2, sort_keys=True) + out.write("\n") + print(json.dumps(dict(status=result["provenance"]["status"], + products=len(result["binding"]["products"]), + manifest_revision=result["manifest"]["metadata"]["revision"]))) + + +if __name__ == "__main__": + main() diff --git a/tests/test_paper_read_extension.py b/tests/test_paper_read_extension.py new file mode 100644 index 00000000..9ee8d6c6 --- /dev/null +++ b/tests/test_paper_read_extension.py @@ -0,0 +1,137 @@ +"""IDR-075 preparation tests; metadata fixtures are TEST_ONLY, not live proof.""" +from copy import deepcopy +from pathlib import Path +import unittest + +import yaml + +from qdl.consumer.manifest import ConsumerManifestLoader +from qdl.consumer.universal_release import ConsumerRouteBinding +from qdl.runtime.production_catalog import ProductionCatalogBuilder, ProductionDemandManifest +from scripts.phase115_render_consumer_route_binding import binding_from_stable_release +from scripts.prepare_paper_read_extension import digest, prepare_paper_extension + +ROOT = Path(__file__).resolve().parents[1] + + +class PaperReadExtensionTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + catalog = yaml.safe_load((ROOT / "config/v2/stable-source-bindings.yaml").read_text()) + inverse = ProductionCatalogBuilder(catalog_revision=catalog["catalog_revision"], + source_policy_revision=catalog["source_policy_revision"], + authority_revision=catalog["authority_revision"]).build( + demand=ProductionDemandManifest.load_many([ROOT / "config/v2/okx-inverse-sandbox-demand.yaml"]), + binance_usdm=None, okx_rows=[dict(instId="BTC-USD-SWAP", instType="SWAP", + instFamily="BTC-USD", ctType="inverse", ctValCcy="USD", settleCcy="BTC", + ctVal="100", ctMult="1", lotSz="0.1", tickSz="0.1", state="live")]) + for field in ("instruments", "bindings"): + catalog[field].extend(inverse.source_catalog[field]) + cls.inputs = dict(catalog=catalog, + manifest=yaml.safe_load((ROOT / "consumers/stable/trading-system-paper.yaml").read_text()), + binding=binding_from_stable_release(ROOT / "config/v2/stable-v2-release-routing.yaml", + ROOT, "trading-system.paper.stable").canonical_mapping()) + + def setUp(self): + self.source = deepcopy(self.inputs) + + def prepare(self): + return prepare_paper_extension(**self.source) + + def reseal(self): + self.source["binding"].pop("binding_sha256") + self.source["binding"]["binding_sha256"] = digest(self.source["binding"]) + + def test_preserves_every_old_requirement_and_product_and_budget(self): + result = self.prepare() + self.assertEqual(self.source, self.inputs) + self.assertEqual(result["manifest"]["metadata"]["revision"], 11) + self.assertEqual(len(result["binding"]["products"]), 64) + self.assertEqual(result["manifest"]["spec"]["quotas"], self.source["manifest"]["spec"]["quotas"]) + for row in self.source["manifest"]["spec"]["requirements"]: + self.assertIn(row, result["manifest"]["spec"]["requirements"]) + for row in self.source["binding"]["products"]: + self.assertIn(row, result["binding"]["products"]) + ConsumerRouteBinding.from_canonical_mapping(result["binding"]) + + def test_exact_inverse_feeds_native_units_and_no_duplicate_fx(self): + result = self.prepare() + inverse = [p for p in result["binding"]["products"] if p["native_symbol"] == "BTC-USD-SWAP"] + self.assertEqual({p["feed"] for p in inverse}, {"QUOTE", "MARK_INDEX_PRICE", "BOOK_SNAPSHOT", "BOOK_DELTA"}) + self.assertTrue(all(p["venue"] == "OKX" and p["interval"] is None for p in inverse)) + self.assertEqual(sum(p["native_symbol"] == "BTC-USDT-SWAP" and p["feed"] == "MARK_INDEX_PRICE" + for p in result["binding"]["products"]), 1) + self.assertEqual(result["consumer_environment"]["DATA_LAYER_V2_JWT_MANIFEST_REVISION"], "11") + self.assertEqual(result["consumer_environment"]["DATA_LAYER_V2_RELEASE_MANIFEST_SHA256"], + result["binding"]["universal_manifest_sha256"]) + + def test_v2_input_seals_manifest_policy_without_schema_break(self): + self.source["binding"].pop("request_policies") + self.source["binding"]["schema"] = "qdl.v2.consumer-route-binding.v2" + self.reseal() + result = self.prepare() + self.assertEqual(result["binding"]["schema"], "qdl.v2.consumer-route-binding.v3") + self.assertEqual(len(result["binding"]["request_policies"]), 64) + manifest = ConsumerManifestLoader.from_mapping(result["manifest"]) + self.assertEqual(manifest.manifest_sha256, result["provenance"]["consumer_manifest_sha256"]) + + def test_wrong_realm_subject_or_revision_fails(self): + for field, value in (("environment", "live"), ("subject", "wrong"), ("revision", 11)): + with self.subTest(field=field): + self.source = deepcopy(self.inputs) + self.source["manifest"]["metadata"][field] = value + with self.assertRaises(ValueError): + self.prepare() + + def test_wrong_catalog_identity_or_collateral_fails(self): + for field, value in (("settlement_asset", "USDT"), ("native_symbol", "BTC-USDT-SWAP")): + with self.subTest(field=field): + self.source = deepcopy(self.inputs) + row = next(i for i in self.source["catalog"]["instruments"] if i["native_symbol"] == "BTC-USD-SWAP") + row[field] = value + with self.assertRaises(ValueError): + self.prepare() + + def test_unacquired_inverse_feed_fails(self): + uid = next(i["instrument_uid"] for i in self.source["catalog"]["instruments"] if i["native_symbol"] == "BTC-USD-SWAP") + self.source["catalog"]["bindings"] = [b for b in self.source["catalog"]["bindings"] + if not (b["instrument_uid"] == uid and b["feed"] == "BOOK_DELTA")] + with self.assertRaises((ValueError, KeyError)): + self.prepare() + + def test_missing_linear_fx_fails(self): + self.source["binding"]["products"] = [p for p in self.source["binding"]["products"] + if not (p["native_symbol"] == "BTC-USDT-SWAP" and p["feed"] == "MARK_INDEX_PRICE")] + ids = {p["requirement_id"] for p in self.source["binding"]["products"]} + self.source["binding"]["request_policies"] = [p for p in self.source["binding"]["request_policies"] + if p["requirement_id"] in ids] + self.reseal() + with self.assertRaisesRegex(ValueError, "FX entitlement"): + self.prepare() + + def test_policy_or_quality_drift_fails_even_when_checksum_resealed(self): + for field in ("policy", "quality"): + with self.subTest(field=field): + self.source = deepcopy(self.inputs) + if field == "policy": + self.source["binding"]["request_policies"][0]["max_session_liveness_ms"] = 99999 + else: + self.source["binding"]["products"][0]["max_freshness_ms"] += 1 + self.reseal() + with self.assertRaises(ValueError): + self.prepare() + + def test_bad_checksum_and_duplicate_and_repeat_fail(self): + self.source["binding"]["binding_sha256"] = "0" * 64 + with self.assertRaises(ValueError): + self.prepare() + self.source = deepcopy(self.inputs) + self.source["manifest"]["spec"]["requirements"].append( + deepcopy(self.source["manifest"]["spec"]["requirements"][0])) + with self.assertRaises(ValueError): + self.prepare() + self.source = deepcopy(self.inputs) + result = self.prepare() + self.source.update(manifest=result["manifest"], binding=result["binding"]) + with self.assertRaisesRegex(ValueError, "already present"): + self.prepare() diff --git a/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md b/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md index f09821c5..c37f03fd 100644 --- a/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md +++ b/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md @@ -3156,3 +3156,20 @@ No alpha/mainnet activation, TS accounting redesign or P18.3E money certificate is implied. The original planning entry changed no runtime. The subsequent scoped TS handoff and additive proof migration are recorded in the receipt above; Data Layer runtime, limits and bindings remain unchanged. + + +### Owner continuation 2026-10-08: existing-core coherence and Paper entitlement + +This narrow operational amendment preserves KN architecture. Coherence is now +measured through the TS IDR-076 bounded product read path; a new core is not +required merely for coherence. The failed0d8da150 trial is historical evidence, +not the release target. Owner's already-applied bar-edge51450612 fixes runaway +history retry. Busy-window/resource and release status are recorded in +[evidence/releases/v2.2.3/RELEASE_NOTES.md](evidence/releases/v2.2.3/RELEASE_NOTES.md). +IDR075 prepares four additive inversePaper reads, preserving existing linear +MARK/INDEX and allpolicies; revision11/bindingr23 must activate with TSJWT11/hash +in one approved reader/caller window. No newinterface, service, acquisition or +realmpermission. Prepared files are not activation/certification. ActiveStream +BTC TRADE overflow/replay observed under highertraffic remains a qualification +finding. DR01-08/brokerdelta remain prerequisites; no dependency on building +Paperfeatures. [PhaseCpacket](evidence/p184-dl-phase-c-packet.md) governs scope. diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md index 3b09af84..5af48db1 100644 --- a/upgrade/evidence/p184-dl-phase-c-packet.md +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -1,5 +1,55 @@ # P18.4 Data Layer Phase C Packet +## Governing owner amendment - 2026-10-08 + +This supersedes the old core-first requirement below. Historical trial evidence +remains immutable. Owner's gate is TS product-path coherent reads (IDR-076 bounded +re-read) on existing658a9570 cores. Do not roll0d8da150 solely for coherence. +Only approved bar edge51450612 has changed. No other role authorized by this file. +Busy-window300s mean4.882532/p955.564052/max5.870669vCPU,n30/299.992s: observed +mean<=5 PASS, not peak/every-interval PASS. Canonical/raw952.48/826.28 offsets/s +exceed all three measured baselines. Retain overflow+2/replay+15901 and actual +BTC TRADE queue closures; DR/recovery/broker gates remain open. See +[v2.2.3 preparation](releases/v2.2.3/RELEASE_NOTES.md). + +### IDR-075 paired window: prepared, not applied + +Public artifacts: ~/.local/state/qdl-v2/p184-codex-release-20261008/idr075/. +prepared.json SHA25627470ade38a9e638884de3f37fe41c8a66dc1c15aeca7de3d00bba808889a088. +Only four inverse grants; linear MARK/INDEX already entitled. No acquisition, +catalog, quota, freshness, realm or account change. Existing physical inverse +source is in deployed catalog; no ingestor/core rollout. + +1. Reinspect exact active Paper callers before action. At preparation only + market_data_service_p184w uses this active Paper subject. Stopped rollback + containers are NOT rollout targets. Any newly created Paper/Risk/Gateway/ + performance caller belongs in Claude's reviewed packet. +2. Stage manifest11 and sealed V3 bindingr23 in new immutable paths; verify both + DL/TS preparers, hashes and policy fingerprints. Existing writer5849 supports + V1/V2/V3 parsing; no speculative rebuild for schema alone. +3. Future approved packet must name Query query_kn_1/2, Stream stream_kn_1/2 and + affected Paper callers with exact images/config hashes. Reader manifests load + at startup: changing a source file alone does NOT activate. Preserve other + realms/keys. No global reader image upgrade or other role is implied. +4. Quiesce affected Paper callers during the bounded mixed-generation window. + Roll readers sequentially with existing images and only exact Paper manifest + mount/list changes; confirm both replicas. Then activate caller revision11 + AND binding/hash together. Do not accept both revisions silently or claim + zero downtime. Reuse identity key, never log its contents. +5. No-order preflight each actual caller: inverse QUOTE/MARK_INDEX_PRICE/ + BOOK_SNAPSHOT/BOOK_DELTA and existing linear INDEX component, both Query + replicas and affected Stream behavior. Wrong revision/realm/unapproved feed + must fail. Check original clocks, collateral/FX proof, cursor/recovery. This + read grant does not add inverse to persistent TS60 or authorize an order. +6. On mixed generation/denied intended route/integrity failure, stop new callers; + restore both Query/Stream Paper manifest10, then caller bindingr22/hash/JWT10. + Preserve Redis/cursors/financial state. No old operational container deletion. + +This is preparation, not the executable private Compose packet. Owner approval +is required before additional role recreates. DR01-DR08 and broker delta remain +release prerequisites, independently of Paper feature completion. + + Status: DRAFT_HOLD_NOT_AUTHORIZED. Source/local candidate only, not a release certificate. Phase B handover and journal: DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md, final P18.4 Phase B entries. Architecture: DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md diff --git a/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md b/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md new file mode 100644 index 00000000..8a9e578e --- /dev/null +++ b/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md @@ -0,0 +1,65 @@ +# v2.2.3 Preparation: Not Published + +Status: `PREPARED_RELEASE_HOLD_NOT_PUBLISHED`. This record is not a certificate, +Git tag, deploy authorization or proof that the feature HEAD runs on production. +The published stable release remains v2.2.2. + +## Inherited Runtime Change + +Owner-approved bar-edge-only recreate at 2026-10-08 02:17:33 UTC: +`f7351c3b` -> `514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9` +(`qdl-v2-python:2.2.2-d6d2637`). This removes the old endless Binance3d history +retry storm using the already-built implementation. No additional role changed +in this continuation. No claim that Binance3d history gaps have been repaired. +Exact rollback compose remains in the prior Claude private evidence directory. + +The three cores remain `658a9570`. Owner moved coherence acceptance to TS's +bounded product read path (IDR-076). Prior paired core trial remains historical +FAIL/rollback; core candidate `0d8da150` is explicitly excluded from this release +proposal. No new core is needed merely to pass the revised coherence gate. + +## Busy-Window Evidence + +2026-10-08 03:21:03-03:26:03 UTC, 17 DL roles, 31 samples/30 intervals over +299.992s: mean **4.882532**, p95 **5.564052**, max **5.870669 vCPU**. The existing +300s mean<=5 gate passes this observed window; p95/max do not stay below5. +No peak-capacity or every-interval<=5 claim. TS CPU is measured separately. + +Simultaneous log-end rates: canonical952.48/raw826.28 offsets/s over302.110s; +both exceed all three prior baselines (canonical +47.7% to70.1%, raw +56.5% to97.2%). +Canonical offsets include transaction markers, not payload-only events. Bar edge +mean0.016960vCPU, no restart/OOM or image/config changes during observation. + +**Retained finding:** active Stream overflow+2, replay+15901, subscriptions+3. +Two queue-full closures are TS Paper BTC TRADE, one Binance and one OKX, each +queued1000. A subsequent Binance replay closes OUT_OF_RANGE after10000 replayed +records. This does not establish data loss or an endless loop, but it precludes +claiming fully qualified burst recovery. One TS Binance BTC MARK timeout also +appears in that window. Passive Stream counters remain unchanged. + +## IDR-075 Preparation + +Pure preparation reuses deployed catalog identities and preserves all60 existing +V2 products, all prior manifest requirements, quotas and fail-closed policies. +Adds only OKX BTC-USD-SWAP QUOTE, MARK_INDEX_PRICE, BOOK_SNAPSHOT and BOOK_DELTA, +with USD quote and BTC settlement identity. No TRADE/BAR acquisition or grants. +BTC-USDT-SWAP MARK_INDEX_PRICE already exists; its INDEX component supplies the +typed FX path, not a new standalone INDEX feed or a USD=USDT assumption. + +Target: Paper manifest11/JWT11, bindingV3 release23,64 products. The matching +binding hash and environment projection are in `release-preparation.json`. +Runtime remains manifest10/bindingV2r22/JWT10. Prepared files are outside runtime; +no token, mount, reader, writer or account was changed. No live/sandbox grants. + +## Publication Gates + +DR01-DR08 and broker delta receipts remain mandatory, independently of Paper +feature completion. Retain the new overflow/replay finding for affected recovery +qualification. IDR-075 needs one approved paired reader/caller packet and actual +Paper-identity no-order reads before activation is claimed. Final release must +reconcile source, immutable images, schemas/configs, CI and owner-authorized +publication; never relabel this prepared inventory as a completed release. + +Evidence: `~/.local/state/qdl-v2/p184-codex-release-20261008/{cpu,idr075}/`. +The main-plan entry and [Phase C packet](../../p184-dl-phase-c-packet.md) govern +the next action. No other role may change without its owner-approved packet. diff --git a/upgrade/evidence/releases/v2.2.3/release-preparation.json b/upgrade/evidence/releases/v2.2.3/release-preparation.json new file mode 100644 index 00000000..dd44a88f --- /dev/null +++ b/upgrade/evidence/releases/v2.2.3/release-preparation.json @@ -0,0 +1,186 @@ +{ + "active_runtime_roles": [ + { + "image": "sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9", + "name": "qdl_v2_stable_candidate-binance_bar_edge-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-08T02:17:35.393509241Z" + }, + { + "image": "sha256:0f6876e16e51600419e1f172aae596bed9c2d369d1afcef93d1da30c18f262ec", + "name": "qdl_v2_stable_candidate-ingestor_binance_usdm-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-02T12:24:58.369037654Z" + }, + { + "image": "sha256:0f6876e16e51600419e1f172aae596bed9c2d369d1afcef93d1da30c18f262ec", + "name": "qdl_v2_stable_candidate-ingestor_okx_swap-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T05:51:24.840836395Z" + }, + { + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "name": "qdl_v2_stable_candidate-kafka1-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T03:56:13.899839854Z" + }, + { + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "name": "qdl_v2_stable_candidate-kafka2-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T03:57:34.401411155Z" + }, + { + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "name": "qdl_v2_stable_candidate-kafka3-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T03:58:43.963843285Z" + }, + { + "image": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", + "name": "qdl_v2_stable_candidate-market_cache-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-09-27T05:29:58.000340326Z" + }, + { + "image": "sha256:a696cfba6ab383173607f96413c814f74c7abcc98a4c41f57c9c6e78a41bb70b", + "name": "qdl_v2_stable_candidate-market_projector_1-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T12:16:20.78036445Z" + }, + { + "image": "sha256:a696cfba6ab383173607f96413c814f74c7abcc98a4c41f57c9c6e78a41bb70b", + "name": "qdl_v2_stable_candidate-market_projector_2-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T12:15:13.766866129Z" + }, + { + "image": "sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9", + "name": "qdl_v2_stable_candidate-query_kn_1-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-02T11:13:49.101943789Z" + }, + { + "image": "sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9", + "name": "qdl_v2_stable_candidate-query_kn_2-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-02T11:12:13.488673234Z" + }, + { + "image": "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023", + "name": "qdl_v2_stable_candidate-rust_core-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-07T17:56:52.401430746Z" + }, + { + "image": "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023", + "name": "qdl_v2_stable_candidate-rust_core_2-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-07T17:56:25.704876381Z" + }, + { + "image": "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023", + "name": "qdl_v2_stable_candidate-rust_core_3-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-07T17:56:10.228521642Z" + }, + { + "image": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", + "name": "qdl_v2_stable_candidate-stable_redis-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-09-16T06:38:55.588316335Z" + }, + { + "image": "sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9", + "name": "qdl_v2_stable_candidate-stream_kn_1-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T12:10:50.691735492Z" + }, + { + "image": "sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9", + "name": "qdl_v2_stable_candidate-stream_kn_2-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T12:07:24.322915897Z" + } + ], + "bar_edge": { + "image": "sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9", + "owner_rollout_at": "2026-10-08T02:17:33Z", + "rollback": "sha256:f7351c3bda080a6d1266d49480dc65d7e590375f178252f858f87a04b1a4d685", + "source": "d6d2637557c3759f829421d2174e21a44762f7fc" + }, + "busy_window": { + "cpu": { + "max": 5.870669239231109, + "mean": 4.882531768271189, + "n": 30, + "p95_nearest_rank": 5.564052047081731 + }, + "duration_ms": 299992.35502200003, + "mean_lte5": true, + "not_peak_capacity": true, + "rates": { + "basis": "LOG_END_OFFSETS_INCLUDES_TRANSACTION_MARKERS_NOT_PAYLOAD_COUNT", + "covers_cpu_window_with_small_snapshot_setup_margin": true, + "offset_increments": { + "md.canonical.v2": 287754, + "md.raw.realtime.v2": 249628 + }, + "partitions": 12, + "rate": { + "md.canonical.v2": 952.4805598220488, + "md.raw.realtime.v2": 826.2815362679872 + }, + "seconds": 302.110102965, + "topics": 2 + }, + "receipt_sha256": "e1434fe1c534a1c20946779a89b5a53fd9013cc5378f0b88b923058bc6a24e80", + "stream_overflow_delta": 2, + "stream_replayed_delta": 15901 + }, + "excluded_core_candidate": "sha256:0d8da150820ed684bda084e92f714537cb40bd81e2e3a6bc93fa9a4b33538411", + "idr075": { + "binding_sha256": "5a9a19385ae34d34213284f06aac8e5c7eb687068702f493590f2664dc0accdf", + "existing_linear_index_preserved": true, + "new_binding_revision": 23, + "new_manifest_revision": 11, + "old_binding_revision": 22, + "old_manifest_revision": 10, + "paired_environment": { + "DATA_LAYER_V2_JWT_MANIFEST_REVISION": "11", + "DATA_LAYER_V2_RELEASE_MANIFEST_SHA256": "6d35f2cf26d9a348d3000ffbf80dc5266dbbe6e8c7cbc0fdc0675345f501ad27" + }, + "products": 64, + "status": "PREPARED_NOT_ACTIVATED" + }, + "previous_published_release": "v2.2.2", + "release_gates_open": [ + "DR01-DR08 indexed receipts", + "broker delta receipt", + "TRADE overflow/replay affected recovery proof", + "IDR075 approved paired runtime packet and actual Paper identity preflight", + "release source/image/config attestation and CI/approved publication" + ], + "runtime_changed_this_slice": false, + "schema": "qdl.release-preparation.v1", + "source_review_base": "1cd24e367bc9b3216318da740d0e7ab3f3ddba54", + "status": "PREPARED_RELEASE_HOLD_NOT_PUBLISHED", + "version": "2.2.3" +} From cf4d82cb2c9cbc8e85e0e9a65579bbe308b1e464 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 05:01:20 +0000 Subject: [PATCH 024/147] docs(p184): attribute TRADE overflow and bound DR runtime evidence scope --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 43 ++++ ...YER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md | 11 + .../p184-dr01-dr08-trade-attribution.md | 236 ++++++++++++++++++ 3 files changed, 290 insertions(+) create mode 100644 upgrade/evidence/p184-dr01-dr08-trade-attribution.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index f862c736..cb3e6863 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69806,3 +69806,46 @@ checkout removed. Only owncompiledscriptcache removed. Diskfree105165643776 -> BuildKit444/18.29GB; retaincurrent/candidate/namedrollback, no broadprune. Canonical DL /home/bobby/data_layer main5bf5cd5/v2.2.2 unchanged, featurefix/p184-dl-closure; TS canonical668e103 unchanged, workon feat/v2-rust-first-okx-demo perowner. + + +### DR01/DR08 isolated runtime evidence and TRADE attribution - 2026-10-08 (Codex) + +Status IN_PROGRESS / v2.2.3 HOLD. Approved scope: nine DR01 runtime cells using +disposable test credentials; four DR08 ISOLATED_FAULT cells on temporary +Kafka/core/Stream; bounded readonly attribution of the 03:21 TRADE incident. +Roster and evidence contract: TS scripts/p184_case_matrix.py and +scripts/p184_dr07_reference.py; architecture guide owner continuation 2026-10-08 +and upgrade/evidence/p184-dl-phase-c-packet.md remain governing references. +No shared runtime, credential, manifest, quota, topic or consumer mutation. +No push/merge/release. Keep bar-edge rollback f7351c3b until release approval. +Runner development uses explicitly dummy TS provenance; final evidence requires +Claude's final source SHA and image digest, not a relabelled development receipt. +Correctness gates: exact nine/four cell identities, fail-closed negative auth, +bounded queues, independent fast-consumer progress, observable overflow/range +signals and measured restart recovery. Do not infer account authorization from +an unchecked JWT claim. Test data provenance must remain explicitly test-only. +LATEST_STATE opt-in TRADE and batch_wait_ms 25->50 remain proposals only. +Rollback: no operational changes to undo; remove only named disposable test +containers/networks/credentials and retain bounded evidence/hashes. Record exact +commands, tests, measurement limitations and cleanup with each completed slice. + + +### DR TRADE attribution slice - 2026-10-08 (Codex) + +Status READONLY_ATTRIBUTION_COMPLETE / RELEASE_HOLD. Detailed report: +upgrade/evidence/p184-dr01-dr08-trade-attribution.md. Recomputed retained hashes +and paired counter deltas; independently checked rates with Python datetime/JSON. +The 9.996720s interval enclosing both overflows has aggregate hub offers +2370.778/s and local outbound deliveries1057.447/s. NOT per-subscription rates, +NOT client Redis ACK; lifetime totals135298/240841 cannot be divided by300s. +Closure lacks per-subscription offered sampling and transport window telemetry. +Two queues1000 and Binance replay10000/OUT_OF_RANGE are observed; exact TS vs +network cause remains NOT_ATTRIBUTED. TS IDR078 later fresh/advancing samples do +not reconstruct incident-time recovery latency. No additional consumer requests. +Inspected actual realtime-core entrypoint and its one shared batch deadline; +25->50ms remains an isolated paired-experiment proposal, not an implemented fix. +Per-subscription TRADE LATEST_STATE also remains proposal-only, default LOSSLESS; +no changes to existing QUOTE mode, canonical durability or consumer contracts. +No runtime/image/config changes, no disposable resources created in this slice, +no cleanup required. Bar-edge f7351c3b retained. DR01/DR08 runner work remains +in progress; final TS SHA/image still required before final case evidence. diff --git a/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md b/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md index c37f03fd..62b693cc 100644 --- a/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md +++ b/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md @@ -3173,3 +3173,14 @@ realmpermission. Prepared files are not activation/certification. ActiveStream BTC TRADE overflow/replay observed under highertraffic remains a qualification finding. DR01-08/brokerdelta remain prerequisites; no dependency on building Paperfeatures. [PhaseCpacket](evidence/p184-dl-phase-c-packet.md) governs scope. + + +### DR01/DR08 evidence and TRADE diagnosis follow-up + +Owner scope 2026-10-08: isolated runtime auth/fault runners, not a shared-runtime +rollout. [Incident attribution and proposal boundaries](evidence/p184-dr01-dr08-trade-attribution.md) +record why replica offers/local delivery counters cannot prove client apply rate. +Use exact TS case roster/provenance; development placeholders are not acceptance. +Retain v2.2.3 HOLD until DR/broker receipts and own DR01/DR08 meet their contracts. +TRADE latest-state opt-in and realtime-core batch25->50 remain proposals only; +no delivery-mode default, interface, broker config or core binary is changed. diff --git a/upgrade/evidence/p184-dr01-dr08-trade-attribution.md b/upgrade/evidence/p184-dr01-dr08-trade-attribution.md new file mode 100644 index 00000000..5f04d727 --- /dev/null +++ b/upgrade/evidence/p184-dr01-dr08-trade-attribution.md @@ -0,0 +1,236 @@ +# P18.4 DR01/DR08 TRADE Incident Attribution + +Date: 2026-10-08. Status: READONLY_ATTRIBUTION_COMPLETE / +INCIDENT_WINDOW_RECOVERY_NOT_MEASURED / PROPOSALS_NOT_EXECUTED. +This report is not a DR01-DR08 acceptance certificate. No implementation source +change, runtime mutation, new measurement, test execution or cleanup was +performed here. + +## Scope And Provenance + +Rules read: `/home/bobby/AGENTS.md` and repository `AGENTS.md`. +Main journal: [Data Layer plan](../../DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md), +sections "IDR-075 preparation and busy-window receipt" and "DR01/DR08 isolated +runtime evidence and TRADE attribution". Governing guide: [KN architecture +review](../DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md), sections 18.4, +18.13-18.14; [Phase C packet](p184-dl-phase-c-packet.md), 2026-10-08 amendment. +No permission to change delivery contracts or roll runtime is implied. + +Evidence root: `/home/bobby/.local/state/qdl-v2/p184-codex-release-20261008/cpu`. +The existing observation covered 03:21:03-03:26:03 UTC, 31 CPU samples / 30 +intervals / 299.992355 seconds. Rates below use their own counter timestamps. +Retained evidence SHA256, recomputed read-only: + +| File relative to evidence root | SHA256 | +| --- | --- | +| `receipt.json` | `e1434fe1c534a1c20946779a89b5a53fd9013cc5378f0b88b923058bc6a24e80` | +| `overflow-attribution.json` | `24dd25eb08133a6e6d1bb586301b95cb99e479b5ff95c1cb1f4f4fe55e925147` | +| `rates-summary.json` | `1311726a644235960e193d657dae2e0ce94d6bb9d8e9ebab6b8220ddc2e3bc33` | +| `window/stream_kn_1-window.json` | `7021fb73b3694a520ae47320d2617470fbed01dff9af8b80a1091e19afb7e809` | +| `window/stream_kn_2-window.json` | `0dc1edd99145623d0130c6e09a77d07a7a40ab105cde2699798e83f02b786f82` | +| `window/inventory-before.json` | `4016c9a322bffcb6257c73472c61d6ebddca88e46a168ed36a76a8b28465fcd7` | + +## Exact Subscription Evidence + +All three closures belong to `trading-system.paper.stable`, feed `TRADE`, +interval null, with `coalesced=0`, `rejected_at_push=0`, `aged_out_at_read=0`. +Counters are subscription-lifetime totals, NOT the five-minute window totals. + +| Subscription / venue | Closure UTC | Initial after | Live delivered | Replayed | Queued / bytes | Outcome | +| --- | --- | ---: | ---: | ---: | --- | --- | +| 6635 / Binance | 03:21:34.481092968 | 442388521 | 135298 | 2715 | 1000 / 5261765 | RESOURCE_EXHAUSTED | +| 6625 / OKX | 03:21:34.487181266 | 389823200 | 240841 | 1 | 1000 / 4939480 | RESOURCE_EXHAUSTED | +| 6651 / Binance retry | 03:22:01.676587057 | 442924424 | 0 | 10000 | 1000 / 5253332 | OUT_OF_RANGE | + +Physical keys: + +- Binance: `a953e16e-7138-5562-b5e8-c337a44d0b65/trade/binance-usdm-btcusdt-trade-stable-001`. +- OKX: `fb26214c-7b9b-5961-95b2-55154755af0f/trade/okx-swap-btcusdt-trade-stable-001`. + +The first two queues reached 1000 records. The Binance retry handed off 10000 +replay records without live delivery, consistent with the replay bound. Its +OUT_OF_RANGE status alone does not prove retention loss: backlog and retention +have distinct details but share that status. Recovery was not measured within +this incident observation window. + +Later evidence: TS IDR-078 at commit +`4b1606608f3f27d6f32b3c9636f18b162a0389de` documents both BTC TRADE views fresh +(approximately 200ms), eligible and advancing at 04:10 UTC, with +910 / +1218 +offsets over 10 seconds. See `INTEGRATION_DEFECT_REGISTER.md`, IDR-078, and +`TRADING_SYSTEM_UNIFIED_IMPLEMENTATION_PLAN.md`, P18.4 IDR-078, in that TS +revision. This later sample supports eventual recovery; it does not measure +immediate recovery time (RTO), reconstruct the incident-window recovery +timeline or prove the precise root cause of the stall. Offset advancement is +not a payload-throughput measurement. IDR-078's worker logging fix is recorded +as tested locally, not a deployed observability guarantee for this incident. + +## Rates And Their Denominators + +Formula: `(last counter - first counter) / elapsed timestamp seconds`. +These are replica aggregates across subscriptions, not BTC-specific rates. + +| Replica / UTC interval | Seconds | Hub offer delta / rate per second | Live delivered delta / rate per second | Replay delta / rate per second | +| --- | ---: | --- | --- | --- | +| stream_kn_1 03:21:11.172417483-03:26:01.171030676 | 289.998613193 | 213282 / 735.46 | 161090 / 555.49 | 15901 / 54.83 | +| stream_kn_1 03:21:21.173973554-03:21:31.173673629 | 9.999700075 | 26103 / 2610.38 | 18924 / 1892.46 | 0 / 0 | +| stream_kn_1 03:21:31.173673629-03:21:41.170393711 | 9.996720082 | 23700 / 2370.78 | 10571 / 1057.45 | 0 / 0 | +| stream_kn_2 03:21:04.812892649-03:25:54.811353809 | 289.998461160 | 0 / 0 | 0 / 0 | 0 / 0 | + +The third row encloses both overflow closures (+2). Stream 1 opened three +subscriptions over its retained window. Stream 2 had zero active subscribers +in retained samples but still read canonical records. Hub record rates were +762.88/s and 763.73/s respectively; do not sum replica reads as unique ingress. + +`hub_offers` increments per physical-key subscriber before `offer` filtering +(`rust/qdl-stream-gateway/src/hub.rs:295-303`). It includes attempts later +rejected as duplicate, wrong product, stale, closed or overflowed. It is not +the accepted subscription-input counter. `SubscriptionCounters.offered` +increments after filtering, before capacity admission, including the failing +offer (`subscription.rs:196-246`), but is absent from the closure JSON +(`service.rs:948-974`) and periodic per-replica metrics (`main.rs:314-338`). + +Therefore neither affected subscription has a measured offered or delivered +rate in this evidence. Lifetime delivery totals lack opening timestamps and +paired window counters. Dividing them by 300s, dividing replica rates by 50, +or treating aggregate offers minus delivery as lost records is invalid. + +Kafka log-end snapshots cover 302.110102965s: canonical +287754 = 952.48056 +offsets/s; raw +249628 = 826.28154 offsets/s, six partitions per topic. These +include transaction markers, not just payloads; partition offsets also mix +products and do not identify one subscription's traffic. CPU mean 4.882532, +p95 5.564052, max 5.870669 vCPU is not proof of 3000 payload/s capacity or +absence of brief stalls. + +## Precise Stream Flow Control + +References below are in `rust/qdl-stream-gateway/src/` unless stated otherwise. + +1. `subscription.rs:190-266`: the shared live reader offers synchronously and + never awaits subscriber capacity. Admission requires item space, per-queue + byte space and replica byte budget. If no legal lifecycle coalescing can + make space, mark overflow. TRADE is lossless by + `rust/qdl-contracts/src/delivery.rs:38-45`. +2. `subscription.rs:273-295`: overflow takes precedence over queued records; + this is termination/recovery, not draining the queue before failure. +3. `service.rs:44,472-499`: the subscription task sends through a two-slot + Tokio mpsc channel. Tonic polls `ReceiverStream`; the memory charge drops + when the mapped item is taken, not when the remote application acknowledges + it. This budget does not count all downstream encoded/transport buffers. +4. `service.rs:985-1014`: send awaits local channel capacity with configurable + `slow_consumer_after` (default 10000ms in `main.rs:212`), raced against + authorization revocation. Success means local enqueue. Live `delivered` + increments at `service.rs:1282-1287`; replay increments separately at + `service.rs:1114-1123`. Neither counter proves socket transmission, SDK + receipt, application processing or durable checkpoint. +5. HTTP/2/gRPC transport flow control can indirectly slow Tonic polling and + fill that handoff; it does not pause this shared reader or canonical + production. Successful gRPC writes need not have reached the network: + [official gRPC flow-control guide](https://grpc.io/docs/guides/flow-control/). + This evidence does not measure HTTP/2 windows or pinpoint a transport stall. +6. `service.rs:1021-1052,1313`: overflow attempts BACKPRESSURE/RATE_LIMITED + control with resume token followed by RESOURCE_EXHAUSTED. Control enqueue + and terminal-status enqueue each have a five-second timeout with ignored + result. Logged closure is not proof the client received either signal. + Queue overflow uses the task's last progress offset; send timeout has its + separate `record.offset - 1` path (`service.rs:1084-1088`). +7. `service.rs:452-497,1138-1215`: registration admits live records while replay + runs first; the live queue can fill during replay. The 10000 replay-event + default is `main.rs:209`; `service.rs:766-789` maps backlog to OUT_OF_RANGE / + CURSOR_EXPIRED with a fresh-snapshot detail, separately from retention and + retryable replay time-limit outcomes. + +No measured stage attribution separates client processing, transport pressure, +task scheduling, serialization and arrival bursts. The collector at +`/home/bobby/.local/state/qdl-v2/dr07-runtime-20261007/continuation/caller_resource_window.py:97` +kept numeric scalars only, excluding the queue/handoff histogram objects emitted +by `main.rs:332-333`. Even those histograms are aggregate; handoff timing wraps +delivery work, not a pure HTTP/2 stall timer. Missing data, persistent infinite +retry, client-applied throughput and immediate recovery time are not proved by +the incident window. The later IDR-078 sample supports eventual recovery as +described above, without resolving precise stall attribution. + +## Proposal: Per-Subscription Delivery Mode + +NOT IMPLEMENTED OR AUTHORIZED FOR RUNTIME. Add explicit per-subscription TRADE +delivery mode: omitted/default remains `LOSSLESS`; TRADE `LATEST_STATE` requires +an approved consumer/product capability. Existing QUOTE and other latest-state +modes remain unchanged. This is not a global LOSSLESS default. Do not change +canonical ingest, durable history or another subscriber's delivery. Current +mode is feed/lifecycle-derived, not a subscriber choice; preserve existing +request semantics for all other feeds. + +TRADE latest-state opt-in would change current invariant 27 / guide 18.4.1 and +needs explicit domain approval, especially for execution, volume and audit +consumers. Define whether the consumer needs a latest-trade state projection +instead of the complete TRADE event feed. Preserve quality/authority/session +transitions, bounded bytes/items, identity/order, original clocks, and explicit +coalesced counts. Version and validate mode/cursor/replay semantics, rejecting +unauthorized modes or mismatched resumes. Never silently coalesce lossless +TRADE, book lifecycle or final BAR/revisions. + +Required future tests: concurrent lossless/latest subscriptions on one product, +fast-reader independence, alternating transition preservation, bounded memory, +auth negatives, mode/cursor mismatch, cancellation and reconnect/recovery. +Instrument per-subscription offered/accepted/dequeued/local-enqueued/coalesced/ +rejected counters with monotonic intervals and client received/applied progress. +Keep replay separate; capture queue/byte high-water and wait distributions. + +## Proposal: Paired batch_wait_ms 25 -> 50 + +NOT RUN. Owner verified running Entrypoint: +`/usr/local/bin/qdl-realtime-core /runtime/core.json`. +Use `rust/qdl-kafka/src/bin/qdl-realtime-core.rs:326-335`: after the first input, +one `batch_deadline = collect_started + batch_wait_ms` is shared by all remaining +receives. `receive_batch_input` at line 220 enforces that deadline, including +already-ready receives. It does not reset the wait per record. The different +loop in `qdl-production-core.rs:529` is NOT the running binary and must not be +used to model this proposal. Commit is awaited at realtime-core.rs:473-475. +Thus 25 -> 50 extends the maximum additional collection window after the first +record, unless batch_size wins; it is not 50ms per record or a fixed total +end-to-end latency bound. More batching may reduce commit cost but increase +waiting, burst size and downstream queue pressure. Improvement is a hypothesis. + +Use isolated paired runs over identical durably captured provider bytes and +arrival timing, fixed image, batch_size, partitions, quotas and consumers; +change only batch_wait_ms. Repeat with order reversal; do not combine with a +LATEST_STATE experiment. State replay provenance, not live-provider or HA +certification. Any later live comparison must record simultaneous real payload +rates/mix and acknowledge that two live windows are not identical workloads. + +Before execution, record exact isolated roles/configs, retention, stop gates +and rollback to 25 in the main plan. Measure batch size, commits/s, commit and raw-age +latencies, venue-to-durable/client-applied p50/p95/p99, per-subscription input and +output, queue/handoff waits, overflow/replay/recovery, CPU/RAM/I/O and broker lag. +Require identity/order/decimal/time parity, no unexplained loss/duplicates/gaps, +unchanged freshness/latency gates and bounded resources before judging CPU. +No source/config edit, rollout, restart or measurement is authorized by this text. + +## Inspection And Runtime Scope + +Read-only commands used: scoped `cat`/`sed`/`rg`/`jq`, `sha256sum`, +`git --no-optional-locks status --short --branch`, `git worktree list --porcelain`, +and scoped `git diff` against recorded deployed revisions. Stream service, +subscription, hub, reporter and delivery-policy files have no diff from +`c8015cac822c8f2759dafabe9a56f97409c39c73`; realtime-core.rs has no diff from +`0e00c817289ce32ea767cea002c859814349fc72`. No test suite was run or claimed. +Rates were calculated with jq over retained JSON, not a new observation. + +Canonical checkout: `/home/bobby/data_layer`, `main@5bf5cd540498b82959342c7cfb2ebeaac3007426`, +published v2.2.2. Registered feature checkout: +`/home/bobby/.worktrees/data-layer-p184-closure`, +`fix/p184-dl-closure@704552d8736905cdb73feec86f3f01d339d02cbb` at inspection. + +Incident inventory, not freshly revalidated runtime: streams +`sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9`; +three cores `sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023`. +Full 17-role DL plus TS caller service/image/config hashes are in the hashed +inventory above. Journal records manifest/JWT10 and binding V2r22. Stable +v2.2.2 remains published; v2.2.3 preparation is under +`upgrade/evidence/releases/v2.2.3/`, not release approval. Retain active and +named rollback artifacts, including bar edge +`sha256:f7351c3bda080a6d1266d49480dc65d7e590375f178252f858f87a04b1a4d685`. +No cleanup or runtime restart occurred; no new images, caches or disposable +runtime resources were created. DR acceptance, incident-window RTO, broker and +release gates remain open; eventual recovery evidence is distinguished above. +Future instrumentation, contract changes or an exact isolated measurement +packet require review before execution. From 4c435a7032240c39d04784f8fbe3779633746068 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 05:19:34 +0000 Subject: [PATCH 025/147] test(p184): add isolated DR01 auth runner and strict account probe contract --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 63 +++ scripts/p184_dr01_runtime.py | 462 ++++++++++++++++++++++ tests/test_p184_dr01_runtime.py | 353 +++++++++++++++++ 3 files changed, 878 insertions(+) create mode 100644 scripts/p184_dr01_runtime.py create mode 100644 tests/test_p184_dr01_runtime.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index cb3e6863..eab77f75 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69849,3 +69849,66 @@ no changes to existing QUOTE mode, canonical durability or consumer contracts. No runtime/image/config changes, no disposable resources created in this slice, no cleanup required. Bar-edge f7351c3b retained. DR01/DR08 runner work remains in progress; final TS SHA/image still required before final case evidence. + + +### DR isolated runner rehearsal packet - 2026-10-08 (Codex) + +Status IMPLEMENTING_AND_TESTING / NOT_FINAL_TS_EVIDENCE. DR01 loopback HTTP +uses existing Python rc2 image77682eea, --rm/network none, .5CPU/512MiB, +fresh RS256 keys removed on exit; public source mounted read-only. Initial +13 tests PASS0skip; HTTP rehearsal7/9 passes, two account cells explicitly open. +Exact TS verifier independently confirms those two failures, not a roster or +provenance error. A consumer mismatch is not broker/account authorization. +Final account probes must GET TS balances from an isolated final-image Gateway: +A->A and B->B positive balances, A->B same venue rejected with +ACCOUNT_ALPHA_SCOPE_MISMATCH, invalid-key negative, all four accounts within +writer scope. No order command, broker key or shared financial-data access. + +DR08 disposable packet uses unique p184-dr08- names and --rm. Kafka +network none; other roles join only that container namespace, no published +ports or production network. Limits: Kafka1CPU/1536MiB, core.5/256MiB, +Stream.5/512MiB, Query.5/512MiB, quotaRedis.1/64MiB, driver.75/512MiB. +Reuse existing immutable images, new TLS/JWT/cursor keys mode0600, read-only +source and fresh scoped evidence/state only. Query is minimal existing read-view +support for real GetSnapshot RPC, not a new production service or schema. +Copy only current core public configuration read-only and restrict to test +BTC TRADE; synthetic records explicitly test_provenance=true in test-only +core authority. No claims of real-provider latency, HA or TS durable ACK. +Queue1000/replay10000 unchanged. Overall driver deadline600s; preserve failed +attempts and cleanup all intended names by ownership label even startup/CLI +failure. Remove keys after hashes/logs, verify production image/start/restart +inventory unchanged; retain active and bar-edge f7351c3b rollback images. +No sourceimage build, runtime rollout, batch-wait change or LATEST_STATE change. +Final case acceptance awaits actual TS SHA/image, fresh run, and DR/broker receipts. + + +### DR01 runner implementation receipt - 2026-10-08 (Codex) + +Status RUNNER_TESTED / FINAL_RUNTIME_ACCEPTANCE_PENDING. New runner +scripts/p184_dr01_runtime.py emits the exact nine TS cell IDs and wrapped +p184.surface-evidence-provenance.v1 header. RS256 ephemeral keys use0600 and +TemporaryDirectory cleanup; actual DL app handles loopback HTTP in --network +none existingPythonimage77682eea. Existing compiler seals a one-product +BAR1m ALPHA binding pervenue; positive wire control reads metadata, NOT all +r22 product data or mTLS/image-byte certification. Scope is explicit in rows. +Source tests plus actual HTTP:23PASS0fail0skip39.818s. Scope-label-only followup +host run22PASS1skip (HTTP requires containerdeps), not counted as runtime proof. +First source version had header wrapping/placeholder and boundary issues caught +before commit; corrected with tests. Account option uses public TS GET balances, +strict positive exact-decimal balances, same-venue A->B403scope denial and an +independent invalid-key403. No broker JWT account claim is invented. Account +option tests are HTTP unit mocks, NOT actual final-image Gateway evidence. + +Rehearsal JSONL (before account-option addition) retained outside Git: +~/.local/state/qdl-v2/p184-dr01-20261008-agent/dr01.rehearsal.jsonl +SHA2563b6e1a90ad4129738f822b79d530d8d7d43e561ded2d445be418508fbbcbcf1f. +Actual TS scripts/p184_case_matrix.py independently parsed exact header/roster: +7/9 closed, only two WRONG_ACCOUNT_DENIED cells failed/NOT_OBSERVED. Dummy TS +SHA/image means this is DEVELOPMENT ONLY, never included in final release +bundle or relabelled. Fresh final-provenance run with isolated Gateway packet +is required for nine-cell acceptance; Claude final TS SHA/image not yet supplied. +Runner --account-packet schema and setup contract are documented in code; use +four Paper accounts within writer scope, two active alphas and isolated PG/Redis. +No runtime, real credentials, consumer quota, manifest or TS source changed. +Test containers removed (--rm); no images built or removed, private keys removed. +No push/merge; v2.2.3 HOLD and f7351c3b rollback retained. DR08 remains in progress. diff --git a/scripts/p184_dr01_runtime.py b/scripts/p184_dr01_runtime.py new file mode 100644 index 00000000..6aa61a7e --- /dev/null +++ b/scripts/p184_dr01_runtime.py @@ -0,0 +1,462 @@ +"""Bounded isolated DR01 HTTP probe; not shipped-image or mTLS qualification. + +Use --source-sha (40 hex) --image-digest (sha256:64 hex) --output FILE. +Placeholder provenance requires --development and FILE.rehearsal.jsonl. +Real provenance requires a fresh run, never copying or relabelling a rehearsal. +Starts only an ephemeral loopback HTTP server, not containers or shared services. +The actual DL app authenticates ephemeral RS256 keys against isolated manifests. +Bindings use the existing DL compiler for ONE BAR-1m test projection per venue. +The positive HTTP read is instrument METADATA, not market data. This is NOT the +full deployed r22 roster or all-feed runtime qualification; TS binding consumption, +warmup/data reads and deployed request-quality parity are not demonstrated. +Consumer isolation is not account authorization: those two cells stay open unless +--account-packet supplies the isolated TS Gateway probe inputs documented below. +""" +from __future__ import annotations + +import argparse +import asyncio +from contextlib import contextmanager +import hashlib +import json +import os +import stat +from decimal import Decimal +from pathlib import Path +import secrets +import re +import tempfile +import socket +import sys +import threading +import time +from urllib.error import HTTPError +from urllib.parse import urlencode, urlsplit +from urllib.request import ProxyHandler, build_opener + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +VENUES = ("BINANCE", "OKX") +CASES = ("BINDING_COMPILE", "WRONG_REALM_DENIED", "WRONG_REVISION_DENIED", + "WRONG_ACCOUNT_DENIED") +CELLS = tuple(f"DR01.{case}@{venue}" for case in CASES for venue in VENUES) + ( + "DR01.NO_GLOBAL_GRANT@ANY",) +DUMMY_SHA = "0" * 40 +DUMMY_IMAGE = "sha256:" + "0" * 64 +MAX_RESPONSE = 65536 + + +def result(cell, ok, evidence, *, observed=True): + if cell not in CELLS or type(ok) is not bool or type(observed) is not bool: + raise ValueError("known cell and strict boolean results required") + if ok and not observed: + raise ValueError("unobserved evidence cannot pass") + return {"cell": cell, "level": "RUNTIME", "ok": ok, + "outcome": "OBSERVED" if observed else "NOT_OBSERVED", + "evidence": evidence, "at_ms": time.time_ns() // 1_000_000} + + +def provenance(development, source_sha, image_digest): + if type(development) is not bool: + raise ValueError("development must be a strict boolean") + if not re.fullmatch(r"[0-9a-f]{40}", source_sha) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_digest): + raise ValueError("source SHA and image digest must be format-valid") + if not development and (len(set(source_sha)) == 1 or len(set(image_digest[7:])) == 1): + raise ValueError("placeholder provenance requires --development") + return {"schema": "p184.surface-evidence-provenance.v1", + "source_sha": source_sha, "image_digest": image_digest, + "development": development, "final_qualification": False, + "test_provenance": True, "runner": "p184_dr01_runtime", + "scope": "isolated DL HTTP authentication and binding compilation", + "shipped_image_verified": False, "mtls_verified": False, + "binding_scope": "ONE_BAR_1M_TEST_PROJECTION_PER_VENUE", + "full_r22_roster_verified": False, "all_feeds_runtime_verified": False, + "positive_http_scope": "INSTRUMENT_METADATA_ONLY", + "ts_provenance": "caller_supplied_not_attested", "broker_account_covered": False} + + +def denied(observation, status, code): + return (type(observation.get("status")) is int + and observation["status"] == status and observation.get("code") == code) + + +def assess(observations): + """Evaluate wire observations, not an authentication model.""" + rows = [] + for venue in VENUES: + obs = observations[venue] + baseline = obs["baseline"].get("status") == 200 and obs["baseline"].get("identity_matches") is True + rows.append(result(f"DR01.BINDING_COMPILE@{venue}", + baseline and obs["binding"].get("compiled") is True, + {"binding": obs["binding"], "dl_manifest_http": obs["baseline"], + "ts_process_consumption_observed": False, + "full_r22_roster_verified": False, "market_data_read_observed": False, + "binding_scope": "ONE_BAR_1M_TEST_PROJECTION_PER_VENUE"})) + for case, key in (("WRONG_REALM_DENIED", "realm"), ("WRONG_REVISION_DENIED", "revision")): + rows.append(result(f"DR01.{case}@{venue}", baseline and denied(obs[key], 401, "UNAUTHENTICATED"), + {"baseline": obs["baseline"], "negative": obs[key]})) + rows.append(result(f"DR01.WRONG_ACCOUNT_DENIED@{venue}", False, + {"blocker": "BROKER_ACCOUNT_AUTHORIZATION_NOT_PRESENT_IN_DL", + "consumer_boundary_observed": baseline and denied(obs["consumer"], 403, "CONSUMER_MISMATCH"), + "key_subject_boundary_observed": baseline and denied(obs["principal"], 401, "UNAUTHENTICATED"), + "consumer_negative": obs["consumer"], "principal_negative": obs["principal"], + "broker_account_covered": False}, observed=False)) + rows.append(result("DR01.NO_GLOBAL_GRANT@ANY", all( + observations[v]["baseline"].get("identity_matches") is True + and observations[v]["baseline"].get("status") == 200 + and denied(observations[v]["ungranted"], 403, "PERMISSION_DENIED") for v in VENUES), + {v: observations[v]["ungranted"] for v in VENUES})) + return sorted(rows, key=lambda row: CELLS.index(row["cell"])) + + +def build_fixture(directory): + """Reuse the existing manifest/token harness with newly generated secrets.""" + from dataclasses import replace + from qdl.api_v2 import create_v2_app + from qdl.consumer import ConsumerManifestLoader, ConsumerManifestRegistry + from qdl.domain.instrument import InstrumentRegistry + from qdl.runtime.stable_catalog import StableSourceCatalog + from qdl.query import EntitlementPolicy, InstrumentQuery, MemoryMarketDataBackend, V2QueryService + from qdl.security import DataPlaneIdentityService, DataPlaneSecurityConfig + from tests.phase7_support import TEST_AUDIENCE, TEST_ISSUER, manifest_mapping + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.hazmat.primitives import serialization + from qdl.consumer.universal_release import ConsumerRouteBinding + from qdl.consumer.realtime_route import requirement_key + from scripts.phase115_render_consumer_route_binding import binding_from_stable_release + + template = binding_from_stable_release(ROOT / "config/v2/stable-v2-release-routing.yaml", + ROOT, "trading-system.paper.stable") + catalog = StableSourceCatalog.load(ROOT / "config/v2/stable-source-bindings.yaml") + + registry, manifests, credentials, records = InstrumentRegistry(), [], {}, {} + nonce = secrets.token_hex(8) + for venue in VENUES: + symbol = "ETHUSDT" if venue == "BINANCE" else "ETH-USDT-SWAP" + product = next(p for p in template.products if p.venue == venue and p.native_symbol == symbol + and p.product_type == "PERPETUAL" and p.feed == "BAR" and p.interval == "1m") + item = catalog.instrument_for(product.instrument_uid) + registry.register(item, []) + cid = f"p184-dr01-test.{nonce}.{venue.lower()}" + subject = f"spiffe://qdl/paper/{cid}" + mapping = manifest_mapping(consumer_id=cid, subject=subject, + instrument_uid=item.instrument_uid, source_policy_id=product.source_policy_id) + requirement = mapping["spec"]["requirements"][0] + requirement["max_freshness_ms"] = product.max_freshness_ms + requirement["require_final_bars"] = product.require_final_bars + manifest = ConsumerManifestLoader.from_mapping(mapping) + manifests.append(manifest) + rid = hashlib.sha256(f"{cid}:{requirement_key(manifest.requirements[0])}".encode()).hexdigest() + policy = next(p for p in template.request_policies if p.requirement_id == product.requirement_id) + product = replace(product, consumer_id=cid, requirement_id=rid, execution_grade=False) + policy = replace(policy, requirement_id=rid, event_recency_policy=None, max_session_liveness_ms=None) + binding = replace(template, consumer_id=cid, products=(product,), request_policies=(policy,), + consumer_manifest_revision=1, binding_sha256=None) + sealed = ConsumerRouteBinding.from_canonical_mapping(binding.canonical_mapping()) + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + key_path = Path(directory) / (venue + ".key") + with key_path.open("xb") as key_file: + key_path.chmod(0o600) + key_file.write(key.private_bytes(serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, serialization.NoEncryption())) + credentials[venue] = {"cid": cid, "subject": subject, "key_path": key_path, + "public_key": key.public_key().public_bytes(serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo), "policy": product.source_policy_id, + "binding": {"compiled": True, "binding_sha256": sealed.binding_sha256, + "manifest_sha256": manifest.manifest_sha256, + "compiler": "binding_from_stable_release + ConsumerRouteBinding", + "test_scope": "isolated one-product ALPHA projection", "venue": venue}} + records[venue] = item + config = DataPlaneSecurityConfig(environment="paper", issuer=TEST_ISSUER, audience=TEST_AUDIENCE, + algorithms=("RS256",), keys_by_id={v: c["public_key"] for v, c in credentials.items()}, + subjects_by_key_id={v: c["subject"] for v, c in credentials.items()}, + environments_by_key_id={v: "paper" for v in VENUES}) + service = V2QueryService(instruments=InstrumentQuery(registry), backend=MemoryMarketDataBackend(), + entitlements=EntitlementPolicy(())) + app = create_v2_app(service, identity_service=DataPlaneIdentityService( + config, ConsumerManifestRegistry(tuple(manifests)))) + return app, credentials, records + + +@contextmanager +def isolated_server(app): + import uvicorn + + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + server = uvicorn.Server(uvicorn.Config(app, log_level="critical", access_log=False, + lifespan="off", timeout_graceful_shutdown=2)) + thread = threading.Thread(target=server.run, kwargs={"sockets": [listener]}, daemon=True) + thread.start() + try: + deadline = time.monotonic() + 10 + while not server.started: + if not thread.is_alive() or time.monotonic() >= deadline: + raise RuntimeError("isolated HTTP server startup failed") + time.sleep(.01) + yield f"http://127.0.0.1:{listener.getsockname()[1]}" + finally: + server.should_exit = True + thread.join(5) + if thread.is_alive(): + server.force_exit = True + thread.join(3) + if thread.is_alive(): + raise RuntimeError("isolated HTTP server cleanup failed") + + +def request_json(base_url, route, headers): + from urllib.request import Request + + # No environment proxies, redirects or retry loops; this endpoint is ours. + from urllib.request import HTTPRedirectHandler + + class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + opener = build_opener(ProxyHandler({}), NoRedirect()) + try: + response = opener.open(Request(base_url + route, headers=headers), timeout=3) + except HTTPError as error: + response = error + with response: + raw = response.read(MAX_RESPONSE + 1) + if len(raw) > MAX_RESPONSE: + raise ValueError("HTTP response bound exceeded") + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ValueError("HTTP response must be an object") + return response.code, payload, hashlib.sha256(raw).hexdigest() + + +def request(base_url, route, headers): + status, payload, digest = request_json(base_url, route, headers) + return {"status": status, "code": payload.get("code"), + "instrument_uid": payload.get("instrument_uid"), "response_sha256": digest} + + +def _funds(rows): + if not isinstance(rows, list) or not 1 <= len(rows) <= 32: + raise ValueError("nonempty bounded expected balances required") + values = {} + for row in rows: + currency = row["currency"] + if not isinstance(currency, str) or not re.fullmatch(r"[A-Z0-9]{1,20}", currency) or currency in values: + raise ValueError("invalid or duplicate balance currency") + amounts = [] + for field in ("total", "locked", "free"): + raw = row[field] + if not isinstance(raw, str) or len(raw) > 80: + raise ValueError("balance values must be bounded decimal strings") + amount = Decimal(raw) + if not amount.is_finite() or amount < 0: + raise ValueError("invalid balance amount") + amounts.append(amount) + # Exact decimal comparison below never passes through a float. + values[currency] = tuple(amounts) + return values + + +def _private_key(path): + path = Path(path) + if not path.is_absolute() or path.resolve().is_relative_to(ROOT): + raise ValueError("account key must be external and absolute") + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as handle: + info = os.fstat(handle.fileno()) + if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600: + raise ValueError("account key must be a mode-0600 regular file") + raw = handle.read(4097) + if not 1 <= len(raw) <= 4096: + raise ValueError("account key size invalid") + key = raw.decode("ascii").strip() + if not key or any(ord(c) < 33 or ord(c) > 126 for c in key): + raise ValueError("account key content invalid") + return key + + +def load_account_packet(path, *, source_sha, image_digest): + """Validate everything before requests; packet and keys never enter evidence. + + JSON schema p184.dr01-account-packet.v1: isolated=true, mode="paper", + source_sha/image_digest equal CLI, gateway_url="http://127.0.0.1:PORT", + bindings={"BINANCE":{"A":binding,"B":binding},"OKX":{...}}. + Each binding has alpha_id, account_id, api_key_file (external absolute 0600), + balances=[{"currency":"USDT","total":"1000.125","locked":"25.125","free":"975"}]. + Provision these four accounts and two active alphas in the isolated Gateway + before running. This runner performs no setup. API GETs can consume isolated + auth/rate counters; they do not place orders or mutate business records. + """ + with Path(path).open("rb") as handle: + raw = handle.read(MAX_RESPONSE + 1) + if len(raw) > MAX_RESPONSE: + raise ValueError("account packet too large") + packet = json.loads(raw) + if (packet.get("schema") != "p184.dr01-account-packet.v1" + or packet.get("isolated") is not True or packet.get("mode") != "paper" + or packet.get("source_sha") != source_sha or packet.get("image_digest") != image_digest): + raise ValueError("isolated Paper packet and matching provenance required") + url = urlsplit(packet["gateway_url"]) + if (url.scheme != "http" or url.hostname != "127.0.0.1" or not url.port + or url.username or url.password or url.path not in ("", "/") or url.query or url.fragment + or packet["gateway_url"].rstrip("/") != f"http://127.0.0.1:{url.port}"): + raise ValueError("literal loopback HTTP origin with explicit port required") + if set(packet["bindings"]) != set(VENUES): + raise ValueError("exact two-venue account packet required") + accounts = set() + for venue in VENUES: + pair = packet["bindings"][venue] + if set(pair) != {"A", "B"}: + raise ValueError("two isolated alpha/account bindings per venue required") + for binding in pair.values(): + if set(binding) != {"alpha_id", "account_id", "api_key_file", "balances"}: + raise ValueError("account binding fields invalid") + for field in ("alpha_id", "account_id"): + if not isinstance(binding[field], str) or not re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", binding[field]): + raise ValueError("account identity invalid") + if binding["account_id"] in accounts: + raise ValueError("account ids must be distinct") + accounts.add(binding["account_id"]) + _funds(binding["balances"]) + binding["key"] = _private_key(binding["api_key_file"]) + if pair["A"]["alpha_id"] == pair["B"]["alpha_id"] or pair["A"]["key"] == pair["B"]["key"]: + raise ValueError("independent alpha credentials required") + for label in ("A", "B"): + first, second = (packet["bindings"][v][label] for v in VENUES) + if first["alpha_id"] != second["alpha_id"] or first["key"] != second["key"]: + raise ValueError("each alpha must use the same identity/key across venues") + packet["gateway_url"] = packet["gateway_url"].rstrip("/") + packet["packet_sha256"] = hashlib.sha256(raw).hexdigest() + return packet + + +def account_probe(packet, venue): + """GET the actual Gateway boundary; no model or account JWT claim.""" + pair = packet["bindings"][venue] + a, b = pair["A"], pair["B"] + detail = {"boundary": "TS_GATEWAY_PAPER_ACCOUNT_ALPHA_SCOPE", "test_scope": "ISOLATED_TEST_ONLY", + "packet_sha256": packet["packet_sha256"], "broker_authoritative": False, "checks": {}, + "bindings": {label: {"alpha_id": b["alpha_id"], "account_id": b["account_id"], + "expected_balances_sha256": hashlib.sha256(json.dumps(b["balances"], + sort_keys=True, separators=(",", ":")).encode()).hexdigest()} for label, b in pair.items()}} + cases = (("a_to_a", a, a, a["key"]), ("b_to_b", b, b, b["key"]), + ("a_to_b", a, b, a["key"]), ("invalid_key", a, a, secrets.token_urlsafe(48))) + for name, principal, target, key in cases: + params = {"alpha_id": principal["alpha_id"], "account_id": target["account_id"], + "mode": "paper", "venue": venue} + route = f"/v1/accounts/{target['account_id']}/balances?" + urlencode(params) + try: + status, payload, digest = request_json(packet["gateway_url"], route, {"X-API-Key": key}) + if name in {"a_to_a", "b_to_b"}: + rows = payload.get("balances") + passed = (status == 200 and payload.get("status") == "OK" + and isinstance(rows, list) and type(payload.get("count")) is int + and payload["count"] == len(rows) and 1 <= len(rows) <= 32 + and all(r.get("strategy_id") == principal["alpha_id"] + and r.get("account_id") == target["account_id"] and r.get("mode") == "paper" + and r.get("venue") == venue for r in rows) + and _funds(rows) == _funds(target["balances"])) + else: + reason = "ACCOUNT_ALPHA_SCOPE_MISMATCH" if name == "a_to_b" else "INVALID_API_KEY" + passed = status == 403 and payload.get("status") == "REJECTED" and payload.get("reason") == reason + detail["checks"][name] = {"ok": passed, "http_status": status, "response_sha256": digest, + "outcome": "OBSERVED"} + if name not in {"a_to_a", "b_to_b"}: + detail["checks"][name]["expected_reason"] = reason + detail["checks"][name]["reason_matches"] = payload.get("reason") == reason + except Exception as error: + detail["checks"][name] = {"ok": False, "outcome": "NOT_OBSERVED", + "error_type": type(error).__name__} + return result(f"DR01.WRONG_ACCOUNT_DENIED@{venue}", + all(check["ok"] is True for check in detail["checks"].values()), detail, + observed=all(check["outcome"] == "OBSERVED" for check in detail["checks"].values())) + + +def observe(): + with tempfile.TemporaryDirectory(prefix="p184-dr01-credentials-") as directory: + return _observe(directory) + + +def _observe(directory): + from tests.phase7_support import TEST_AUDIENCE, TEST_ISSUER + from qdl_sdk.credentials import RotatingJwtCredentialProvider + + app, credentials, records = build_fixture(directory) + observations = {} + with isolated_server(app) as base_url: + for venue in VENUES: + cred = credentials[venue] + route = f"/v2/instruments/{records[venue].instrument_uid}" + def headers(**overrides): + settings = dict(private_key_file=cred["key_path"], key_id=venue, algorithm="RS256", + issuer=TEST_ISSUER, audience=TEST_AUDIENCE, subject=cred["subject"], environment="paper", + roles=("market_data_reader", "historical_reader", "stream_consumer"), + consumer_manifest_revision=1, lifetime_seconds=300, refresh_before_seconds=30) + settings.update(overrides) + token = asyncio.run(RotatingJwtCredentialProvider(**settings).get_token()) + return {"Authorization": "Bearer " + token, "X-QDL-Consumer-ID": cred["cid"], + "X-QDL-Purpose": "INTERNAL_ALPHA"} + baseline = request(base_url, route, headers()) + baseline["identity_matches"] = baseline["instrument_uid"] == records[venue].instrument_uid + other = credentials["OKX" if venue == "BINANCE" else "BINANCE"] + forbidden = f"/v2/market-data/{records[venue].instrument_uid}/snapshot?" + urlencode({ + "feed": "QUOTE", "consumer_grade": "ALPHA", "source_policy_id": cred["policy"]}) + observations[venue] = {"baseline": baseline, "binding": cred["binding"], + "realm": request(base_url, route, headers(environment="live")), + "revision": request(base_url, route, headers(consumer_manifest_revision=2)), + "consumer": request(base_url, route, headers() | {"X-QDL-Consumer-ID": other["cid"]}), + "principal": request(base_url, route, headers(subject=other["subject"]) | { + "X-QDL-Consumer-ID": other["cid"]}), + "ungranted": request(base_url, forbidden, headers())} + return observations + + +def run(output, *, development, source_sha, image_digest, account_packet=None): + header = provenance(development, source_sha, image_digest) + packet = None + if account_packet is not None: + try: + packet = load_account_packet(account_packet, source_sha=source_sha, image_digest=image_digest) + except Exception: + raise ValueError("invalid isolated account packet or external key file") from None + header["account_packet_sha256"] = packet["packet_sha256"] + header["account_boundary_requested"] = "TS_GATEWAY_PAPER_ACCOUNT_ALPHA_SCOPE" + if development and not Path(output).name.endswith(".rehearsal.jsonl"): + raise ValueError("development output must end with .rehearsal.jsonl; never relabel as final") + # Exclusive creation occurs before service startup. Never relabel/reuse a receipt. + with Path(output).open("x", encoding="utf-8") as handle: + handle.write(json.dumps({"provenance": header}, sort_keys=True) + "\n") + handle.flush() + try: + rows = assess(observe()) + except Exception as error: + # Exception messages can contain credentials; retain only the class. + rows = [result(cell, False, {"blocker": "HTTP_RUN_NOT_COMPLETED", + "error_type": type(error).__name__}, observed=False) for cell in CELLS] + if packet is not None: + accounts = {f"DR01.WRONG_ACCOUNT_DENIED@{v}": account_probe(packet, v) for v in VENUES} + rows = [accounts.get(row["cell"], row) for row in rows] + for row in rows: + handle.write(json.dumps(row, sort_keys=True) + "\n") + return rows + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--development", action="store_true") + parser.add_argument("--source-sha", required=True) + parser.add_argument("--image-digest", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--account-packet", type=Path, help="isolated Paper Gateway account probe packet") + args = parser.parse_args(argv) + rows = run(**vars(args)) + print(json.dumps({"development": args.development, "final_qualification": False, + "passed": sum(r["ok"] is True for r in rows), "total": len(rows)})) + return 0 if all(r["ok"] is True for r in rows) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_p184_dr01_runtime.py b/tests/test_p184_dr01_runtime.py new file mode 100644 index 00000000..3f9f2f9f --- /dev/null +++ b/tests/test_p184_dr01_runtime.py @@ -0,0 +1,353 @@ +"""Source checks are not runtime evidence; opt-in test uses real loopback HTTP.""" +import json +import os +from pathlib import Path +import tempfile +import unittest +import threading +from copy import deepcopy +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, urlsplit +from unittest.mock import patch + +from scripts import p184_dr01_runtime as dr + + +def observations(): + return {venue: {"baseline": {"status": 200, "identity_matches": True}, + "binding": {"compiled": True}, + "realm": {"status": 401, "code": "UNAUTHENTICATED"}, + "revision": {"status": 401, "code": "UNAUTHENTICATED"}, + "consumer": {"status": 403, "code": "CONSUMER_MISMATCH"}, + "principal": {"status": 401, "code": "UNAUTHENTICATED"}, + "ungranted": {"status": 403, "code": "PERMISSION_DENIED"}} for venue in dr.VENUES} + + +class SourceTests(unittest.TestCase): + def test_exact_nine_ids(self): + expected = {f"DR01.{case}@{venue}" for case in ( + "BINDING_COMPILE", "WRONG_REALM_DENIED", "WRONG_REVISION_DENIED", "WRONG_ACCOUNT_DENIED") + for venue in ("BINANCE", "OKX")} | {"DR01.NO_GLOBAL_GRANT@ANY"} + self.assertEqual(set(dr.CELLS), expected) + self.assertEqual(len(dr.CELLS), 9) + + def test_strict_booleans(self): + for bad in (1, 0, "false", [], None): + with self.subTest(bad=bad), self.assertRaises(ValueError): + dr.result(dr.CELLS[0], bad, {}) + with self.assertRaises(ValueError): + dr.result(dr.CELLS[0], True, {}, observed=False) + with self.assertRaises(ValueError): + dr.result("invented", False, {}) + + def test_provenance_format_and_placeholder_gate(self): + for values in ((False, dr.DUMMY_SHA, dr.DUMMY_IMAGE), + (True, "dummy-ts-sha", "dummy-ts-image"), + (False, "a" * 40, "sha256:" + "b" * 64), + (1, dr.DUMMY_SHA, dr.DUMMY_IMAGE)): + with self.assertRaises(ValueError): + dr.provenance(*values) + header = dr.provenance(True, dr.DUMMY_SHA, dr.DUMMY_IMAGE) + self.assertIs(header["final_qualification"], False) + supplied = dr.provenance(False, "0123456789" * 4, "sha256:" + "0123456789abcdef" * 4) + self.assertIs(supplied["development"], False) + self.assertIs(supplied["shipped_image_verified"], False) + self.assertIs(supplied["mtls_verified"], False) + + def test_account_stays_open_and_binding_uses_compile_result(self): + rows = dr.assess(observations()) + self.assertEqual(sum(row["ok"] is True for row in rows), 7) + self.assertEqual(tuple(r["cell"] for r in rows), dr.CELLS) + for row in rows: + if "BINDING_COMPILE" in row["cell"]: + self.assertIs(row["evidence"]["full_r22_roster_verified"], False) + self.assertIs(row["evidence"]["market_data_read_observed"], False) + self.assertEqual(row["evidence"]["binding_scope"], "ONE_BAR_1M_TEST_PROJECTION_PER_VENUE") + for row in rows: + if "WRONG_ACCOUNT" in row["cell"]: + self.assertIs(row["ok"], False) + self.assertEqual(row["outcome"], "NOT_OBSERVED") + + def test_compile_must_be_strict_positive(self): + for bad in (False, "true", 1, None): + obs = observations() + obs["BINANCE"]["binding"]["compiled"] = bad + row = next(r for r in dr.assess(obs) if r["cell"] == "DR01.BINDING_COMPILE@BINANCE") + self.assertIs(row["ok"], False) + + def test_key_cleanup_on_probe_failure(self): + with patch.object(dr, "_observe", side_effect=RuntimeError) as probe: + with self.assertRaises(RuntimeError): + dr.observe() + self.assertFalse(Path(probe.call_args.args[0]).exists()) + + def test_outage_or_generic_refusal_never_passes(self): + for status, code in ((500, "UNAUTHENTICATED"), (403, "PERMISSION_DENIED"), + (401, None), (200, "UNAUTHENTICATED")): + obs = observations() + obs["BINANCE"]["realm"] = {"status": status, "code": code} + row = next(r for r in dr.assess(obs) if r["cell"] == "DR01.WRONG_REALM_DENIED@BINANCE") + self.assertIs(row["ok"], False) + + def test_positive_control_required(self): + obs = observations() + obs["BINANCE"]["baseline"]["identity_matches"] = False + rows = dr.assess(obs) + self.assertFalse(any(r["ok"] for r in rows if r["cell"].endswith("@BINANCE"))) + self.assertIs(rows[-1]["ok"], False) + + def test_both_venues_need_no_global_grant(self): + obs = observations() + obs["OKX"]["ungranted"] = {"status": 200} + self.assertIs(dr.assess(obs)[-1]["ok"], False) + + def test_rehearsal_filename_required(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "final.jsonl" + with patch.object(dr, "observe") as observe, self.assertRaises(ValueError): + dr.run(path, development=True, source_sha=dr.DUMMY_SHA, image_digest=dr.DUMMY_IMAGE) + observe.assert_not_called() + self.assertFalse(path.exists()) + + def test_output_is_exclusive_before_any_runtime(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "existing.rehearsal.jsonl" + path.write_text("untouched") + with patch.object(dr, "observe") as observe, self.assertRaises(FileExistsError): + dr.run(path, development=True, source_sha=dr.DUMMY_SHA, image_digest=dr.DUMMY_IMAGE) + observe.assert_not_called() + self.assertEqual(path.read_text(), "untouched") + + def test_runtime_failure_is_nine_blockers_without_secret_messages(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "development.rehearsal.jsonl" + with patch.object(dr, "observe", side_effect=TimeoutError("secret-token")): + rows = dr.run(path, development=True, source_sha=dr.DUMMY_SHA, image_digest=dr.DUMMY_IMAGE) + text = path.read_text() + self.assertNotIn("secret-token", text) + self.assertEqual(len(text.splitlines()), 10) + header = json.loads(text.splitlines()[0]) + self.assertEqual(set(header), {"provenance"}) + self.assertIs(header["provenance"]["development"], True) + self.assertEqual(header["provenance"]["schema"], "p184.surface-evidence-provenance.v1") + self.assertEqual(header["provenance"]["source_sha"], dr.DUMMY_SHA) + self.assertEqual(header["provenance"]["image_digest"], dr.DUMMY_IMAGE) + self.assertTrue(all(r["ok"] is False and r["outcome"] == "NOT_OBSERVED" for r in rows)) + + +class AccountHttpUnitTests(unittest.TestCase): + """Scripted local HTTP mock, UNIT ONLY; never actual TS Gateway evidence.""" + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="p184-account-unit-") + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.calls, self.answers = [], [] + owner = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + owner.calls.append((self.command, self.path, self.headers.get("X-API-Key"))) + status, body = owner.answers.pop(0) if owner.answers else (500, {}) + self.send_response(status) + if status == 302: + self.send_header("Location", "/redirect-must-not-be-followed") + self.end_headers() + self.wfile.write(body if isinstance(body, bytes) else json.dumps(body).encode()) + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.thread = threading.Thread(target=self.server.serve_forever, kwargs={"poll_interval": .01}) + self.thread.start() + self.addCleanup(self.close_server) + self.packet = {"schema": "p184.dr01-account-packet.v1", "isolated": True, "mode": "paper", + "source_sha": dr.DUMMY_SHA, "image_digest": dr.DUMMY_IMAGE, + "gateway_url": f"http://127.0.0.1:{self.server.server_port}", "bindings": {}} + for venue in dr.VENUES: + pair = {} + for label in ("A", "B"): + key = self.root / (venue + label + ".key") + key.write_text("unit-only-secret-" + label) + key.chmod(0o600) + pair[label] = {"alpha_id": "unit-alpha-" + label, + "account_id": "unit-paper-" + venue + label, "api_key_file": str(key), + "balances": [{"currency": "USDT", "total": "1000.125", "locked": "25.125", "free": "975"}]} + self.packet["bindings"][venue] = pair + self.path = self.root / "account-packet.json" + + def close_server(self): + self.server.shutdown() + self.server.server_close() + self.thread.join(2) + self.assertFalse(self.thread.is_alive()) + + def load(self): + self.path.write_text(json.dumps(self.packet)) + return dr.load_account_packet(self.path, source_sha=dr.DUMMY_SHA, image_digest=dr.DUMMY_IMAGE) + + def positive(self, venue, label): + binding = self.packet["bindings"][venue][label] + return (200, {"status": "OK", "count": 1, "balances": [{**binding["balances"][0], + "strategy_id": binding["alpha_id"], "account_id": binding["account_id"], + "mode": "paper", "venue": venue}]}) + + def answers_for(self, venue): + return [self.positive(venue, "A"), self.positive(venue, "B"), + (403, {"status": "REJECTED", "reason": "ACCOUNT_ALPHA_SCOPE_MISMATCH"}), + (403, {"status": "REJECTED", "reason": "INVALID_API_KEY"})] + + def test_unit_http_exact_route_headers_and_independent_controls(self): + packet = self.load() + for venue in dr.VENUES: + self.answers.extend(self.answers_for(venue)) + row = dr.account_probe(packet, venue) + self.assertIs(row["ok"], True) + self.assertEqual(row["cell"], f"DR01.WRONG_ACCOUNT_DENIED@{venue}") + pair = packet["bindings"][venue] + for index, (method, route, key) in enumerate(self.calls[-4:]): + self.assertEqual(method, "GET") + target = pair["B"] if index in (1, 2) else pair["A"] + principal = pair["B"] if index == 1 else pair["A"] + parsed = urlsplit(route) + self.assertEqual(parsed.path, f"/v1/accounts/{target['account_id']}/balances") + self.assertEqual(parse_qs(parsed.query), {"alpha_id": [principal["alpha_id"]], + "account_id": [target["account_id"]], "mode": ["paper"], "venue": [venue]}) + self.assertTrue(key == principal["key"] if index != 3 else key not in {pair["A"]["key"], pair["B"]["key"]}) + serialized = json.dumps(row) + self.assertNotIn("unit-only-secret", serialized) + self.assertNotIn(str(self.root), serialized) + self.assertNotIn("1000.125", serialized) + + def test_unit_positive_identity_amount_and_nonempty_gates(self): + packet = self.load() + for mutation in ("empty", "account", "strategy", "venue", "mode", "amount", "count", "float", "duplicate"): + with self.subTest(mutation=mutation): + self.answers = self.answers_for("BINANCE") + body = self.answers[0][1] + row = body["balances"][0] + if mutation == "empty": + body.update(balances=[], count=0) + elif mutation == "count": + body["count"] = True + elif mutation == "duplicate": + body["balances"].append(dict(row)) + body["count"] = 2 + else: + field = {"account": "account_id", "strategy": "strategy_id", "amount": "total", "float": "total"}.get(mutation, mutation) + row[field] = 1000.125 if mutation == "float" else "wrong" if mutation not in {"amount"} else "1000.1250000000000000000001" + self.assertIs(dr.account_probe(packet, "BINANCE")["ok"], False) + + def test_unit_wrong_reason_and_independent_invalid_key_gate(self): + packet = self.load() + for index, status, body in ((2, 403, {"status": "REJECTED", "reason": "INVALID_API_KEY"}), + (2, 403, {"status": "REJECTED", "reason": "ACCOUNT_SCOPE_MISMATCH"}), + (2, 500, {"status": "REJECTED", "reason": "ACCOUNT_ALPHA_SCOPE_MISMATCH"}), + (3, 403, {"status": "REJECTED", "reason": "ACCOUNT_ALPHA_SCOPE_MISMATCH"}), + (3, 200, {"status": "OK"})): + self.answers = self.answers_for("OKX") + self.answers[index] = status, body + self.assertIs(dr.account_probe(packet, "OKX")["ok"], False) + + def test_unit_timeout_is_never_account_denial(self): + packet = self.load() + with patch.object(dr, "request_json", side_effect=TimeoutError("sensitive-message")) as request: + row = dr.account_probe(packet, "BINANCE") + self.assertEqual(request.call_count, 4) + self.assertIs(row["ok"], False) + self.assertEqual(row["outcome"], "NOT_OBSERVED") + self.assertNotIn("sensitive-message", json.dumps(row)) + + def test_unit_transport_uses_bounded_timeout(self): + with patch.object(dr, "build_opener") as factory: + factory.return_value.open.side_effect = TimeoutError + with self.assertRaises(TimeoutError): + dr.request_json(self.packet["gateway_url"], "/test", {"X-API-Key": "unit-only"}) + self.assertEqual(factory.return_value.open.call_args.kwargs["timeout"], 3) + + def test_unit_redirect_malformed_and_response_bounds(self): + packet = self.load() + for status, body in ((302, {}), (200, b"not-json"), (200, b"x" * (dr.MAX_RESPONSE + 1))): + before = len(self.calls) + self.answers = self.answers_for("BINANCE") + self.answers[0] = status, body + self.assertIs(dr.account_probe(packet, "BINANCE")["ok"], False) + self.assertEqual(len(self.calls) - before, 4) + + def test_packet_provenance_isolation_origin_and_pair_fail_closed(self): + good = deepcopy(self.packet) + for field, value in (("isolated", 1), ("mode", "live"), ("source_sha", "a" * 40), + ("image_digest", "sha256:" + "a" * 64), ("gateway_url", "http://example.com:1234"), + ("gateway_url", "http://localhost:1234"), ("gateway_url", "http://127.0.0.1:1234/a"), + ("gateway_url", "http://user:secret@127.0.0.1:1234")): + self.packet = deepcopy(good) + self.packet[field] = value + with self.assertRaises(ValueError): + self.load() + self.packet = deepcopy(good) + self.packet["bindings"]["BINANCE"]["B"]["alpha_id"] = self.packet["bindings"]["BINANCE"]["A"]["alpha_id"] + with self.assertRaises(ValueError): + self.load() + self.assertEqual(self.calls, []) + + def test_keys_require_external_regular_0600_files(self): + binding = self.packet["bindings"]["BINANCE"]["A"] + path = Path(binding["api_key_file"]) + path.chmod(0o644) + with self.assertRaises(ValueError): + self.load() + path.chmod(0o600) + link = self.root / "symlink.key" + link.symlink_to(path) + binding["api_key_file"] = str(link) + with self.assertRaises(OSError): + self.load() + binding["api_key_file"] = str(dr.ROOT / "not-an-external-key") + with self.assertRaises(ValueError): + self.load() + self.assertEqual(self.calls, []) + + def test_unit_option_replaces_only_account_rows_and_never_logs_keys(self): + self.load() + for venue in dr.VENUES: + self.answers.extend(self.answers_for(venue)) + output = self.root / "unit-contract.rehearsal.jsonl" + with patch.object(dr, "observe", return_value=observations()): + rows = dr.run(output, development=True, source_sha=dr.DUMMY_SHA, + image_digest=dr.DUMMY_IMAGE, account_packet=self.path) + self.assertEqual(len(rows), 9) + self.assertTrue(all(row["ok"] is True for row in rows)) + self.assertEqual(len(self.calls), 8) + text = output.read_text() + self.assertNotIn("unit-only-secret", text) + self.assertNotIn(str(self.root), text) + self.assertIs(json.loads(text.splitlines()[0])["provenance"]["final_qualification"], False) + + def test_invalid_packet_does_not_create_output_or_start_dl(self): + self.packet["mode"] = "live" + self.path.write_text(json.dumps(self.packet)) + output = self.root / "unit-contract.rehearsal.jsonl" + with patch.object(dr, "observe") as probe, self.assertRaisesRegex(ValueError, "invalid isolated account packet"): + dr.run(output, development=True, source_sha=dr.DUMMY_SHA, + image_digest=dr.DUMMY_IMAGE, account_packet=self.path) + probe.assert_not_called() + self.assertFalse(output.exists()) + self.assertEqual(self.calls, []) + + +@unittest.skipUnless(os.environ.get("P184_DR01_HTTP_TEST") == "1", + "requires approved isolated image with app dependencies; no runtime claim") +class ActualHttpTests(unittest.TestCase): + def test_real_application_ephemeral_credentials(self): + obs = dr.observe() + rows = dr.assess(obs) + self.assertEqual(sum(row["ok"] is True for row in rows), 7, rows) + for venue in dr.VENUES: + self.assertTrue(dr.denied(obs[venue]["consumer"], 403, "CONSUMER_MISMATCH")) + self.assertTrue(dr.denied(obs[venue]["principal"], 401, "UNAUTHENTICATED")) + self.assertIs(obs[venue]["baseline"]["identity_matches"], True) + + +if __name__ == "__main__": + unittest.main() From 5a23049e3447b668daadd99675a01d163690cb77 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 06:00:52 +0000 Subject: [PATCH 026/147] test(p184): rehearse isolated stream faults and record DR handoff --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 65 ++ scripts/p184_dr01_runtime.py | 5 + scripts/p184_dr08_isolated.py | 1028 +++++++++++++++++ tests/test_p184_dr01_runtime.py | 4 + tests/test_p184_dr08_isolated.py | 264 +++++ upgrade/evidence/p184-dr-runtime-runbook.md | 95 ++ upgrade/evidence/p184-dr07-runtime-closure.md | 9 + 7 files changed, 1470 insertions(+) create mode 100644 scripts/p184_dr08_isolated.py create mode 100644 tests/test_p184_dr08_isolated.py create mode 100644 upgrade/evidence/p184-dr-runtime-runbook.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index eab77f75..4c396d93 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69912,3 +69912,68 @@ four Paper accounts within writer scope, two active alphas and isolated PG/Redis No runtime, real credentials, consumer quota, manifest or TS source changed. Test containers removed (--rm); no images built or removed, private keys removed. No push/merge; v2.2.3 HOLD and f7351c3b rollback retained. DR08 remains in progress. + + +### DR08 isolated fault receipt and final handoff - 2026-10-08 (Codex) + +Status RUNNERS_IMPLEMENTED_TESTED / FINAL_DR_ACCEPTANCE_PENDING / RELEASE_HOLD. +Guide and final-provenance procedure: upgrade/evidence/p184-dr-runtime-runbook.md. +Attribution/proposal report: upgrade/evidence/p184-dr01-dr08-trade-attribution.md. +DR01 full source binding is now canonical-validated before deriving disposable +one-BAR identities; source roster counts/hash are recorded, not implied all-feed +wire coverage. Final check in existing Python image77682eea with --rm/network +none/.5CPU/512MiB, source read-only: python -B -m unittest + tests.test_p184_dr01_runtime -v:23PASS0fail0skip38.615s. +No final TS SHA/image supplied yet. DR01 retains7/9 rehearsal, two account cells +require isolated final-image Gateway. Unit mocks do not close those cells. + +DR08 scripts/p184_dr08_isolated.py owns disposable Kafka/core/Stream/Query/Redis +and driver. Existing images only, no published ports or external network, +explicit synthetic test provenance, queue1000/replay10000 unchanged. Slow client +process paused while independent fast client progresses. Actual gRPC overflow, +OUT_OF_RANGE and newer GetSnapshot recovery observed. This is ISOLATED_FAULT +rehearsal, not production/provider latency, HA, projector recovery or TS Redis ACK. +Tests: python3 -B -m unittest tests.test_p184_dr08_isolated -q: +25PASS0fail0skip0.037s (parent rerun). + +Attempts1-11 preserved outside Git; harness setup failures covered permissions, +Kafka file/segment sizing, quota/group namespaces, startup and test lineage. +No production code/policy fix inferred from those test-packet errors. Cold-start +seed expiry fixed by publishing a NEW test event after readiness, never changing +old timestamps or relaxing freshness. Attempt12 raw receipt preserved unchanged: +~/.local/state/qdl-v2/p184-dr08-development-20261008-attempt12/evidence.jsonl +SHA256 3e9e0b93581e6b65bc9b0e04c2895ffc1aff1f60481c0e2af0e50513bc5c4471. +One original check failed because parser expected the wrong textual status name. +Corrected parser plus regression and independently verified raw closure matched +exact consumer/subscription and actual wire RESOURCE_EXHAUSTED. Supplement: +queue-parser-supplement.json SHA256 +87b218116e287b8b4dbc1a30f6ad5aa01d81277faf473b2acc276ec37c8f5834. +stream-runtime.log SHA256 +f179343f670ae103acc7e5842a0439dfb6174aa37a4e6079fed89613b845710e. +Combined4/4 DEVELOPMENT checks; original failed receipt is not overwritten. + +Measured queue at closure1000events/4719110bytes; sampled queue bytes max4989098. +Replay10000 then OUT_OF_RANGE. Independent fast consumer10001ordered events, +71 progress points while slow stopped. Controls/counters: overflow+1, expired+1, +69 metrics samples. Recovery n=1, units ms: snapshot1744.467, +client restart->first event5241.271, OUT_OF_RANGE->first event6999.346. +Snapshot tradeID2->10004 and offset3->10107; not a p99 or consumer durable ACK. +Development JSONL remains nonqualifying. Final provenance requires a fresh run, +not rewriting its header. No conclusion about TS-vs-network03:21 root cause. + +Cleanup: all8 owned attempt12 containers removed, private keys removed, +cleanup_failures=[], shared_runtime_unchanged=true. Disk free before91924062208B, +after91898748928B; concurrent jobs mean delta is not attributable to this test. +All earlier failed attempts cleaned; final DR01 container --rm removed. Images +built0/removed0. Docker inventory:95images/41active/28.96GB; BuildKit19.8GB, +2.125GB reclaimable. No broad prune during Claude concurrent work. Retain active +images plus explicitly retained bar-edge rollback f7351c3b (inspect confirmed). + +Canonical /home/bobby/data_layer main5bf5cd5, stable releasev2.2.2; feature +/home/bobby/.worktrees/data-layer-p184-closure fix/p184-dl-closure. Runtime service +set/config remains unchanged:3cores658a9570,2Stream1f13408f,2projector a696cfba, +BAR/2Query51450612, ingestors0f6876e1, existing infrastructure. TS writer5849d716, +manifest/JWT10 and bindingV2r22 unchanged. No TS source edits, rollout, push/merge. +v2.2.3 HOLD for final DR/broker receipts. LATEST_STATE and batch25->50 proposals +were neither implemented nor measured. Next permitted work: final-image isolated +account fixture plus fresh provenance-bound DR01/DR08, then owner packet review. diff --git a/scripts/p184_dr01_runtime.py b/scripts/p184_dr01_runtime.py index 6aa61a7e..f2594833 100644 --- a/scripts/p184_dr01_runtime.py +++ b/scripts/p184_dr01_runtime.py @@ -129,6 +129,8 @@ def build_fixture(directory): template = binding_from_stable_release(ROOT / "config/v2/stable-v2-release-routing.yaml", ROOT, "trading-system.paper.stable") + # Validate the entire compiled artifact before deriving disposable identities. + template = ConsumerRouteBinding.from_canonical_mapping(template.canonical_mapping()) catalog = StableSourceCatalog.load(ROOT / "config/v2/stable-source-bindings.yaml") registry, manifests, credentials, records = InstrumentRegistry(), [], {}, {} @@ -167,6 +169,9 @@ def build_fixture(directory): "binding": {"compiled": True, "binding_sha256": sealed.binding_sha256, "manifest_sha256": manifest.manifest_sha256, "compiler": "binding_from_stable_release + ConsumerRouteBinding", + "source_roster_products": len(template.products), + "source_roster_venue_products": sum(p.venue == venue for p in template.products), + "source_roster_binding_sha256": template.binding_sha256, "test_scope": "isolated one-product ALPHA projection", "venue": venue}} records[venue] = item config = DataPlaneSecurityConfig(environment="paper", issuer=TEST_ISSUER, audience=TEST_AUDIENCE, diff --git a/scripts/p184_dr08_isolated.py b/scripts/p184_dr08_isolated.py new file mode 100644 index 00000000..dcce0aca --- /dev/null +++ b/scripts/p184_dr08_isolated.py @@ -0,0 +1,1028 @@ +#!/usr/bin/env python3 +"""DR08 driver for a parent-owned, disposable Kafka/core/Stream packet. + +Host entrypoint --orchestrate NEW_EXTERNAL_DIRECTORY --development creates the +bounded disposable stack using existing images and public core configuration. +The inner --packet/--output driver runs only in its isolated container namespace. +Fresh TLS/JWT/cursor material is generated locally; Slice is reused with explicit +isolated manifest overrides. A real Query read view behind +Stream GetSnapshot is required; hot-read or locally signed offsets are NOT a +replacement for fresh snapshot recovery. + +Packet JSON keys: namespace, development, ts_source_sha, ts_image_digest, +images (kafka/core/stream/python immutable sha256 IDs), bootstrap (localhost:9092), +raw_topic, raw_template (base64 RawProviderEnvelope, test_provenance=true), +consumer_id, fast_consumer_id, instrument_uid, metrics_path, and slice (the +existing Slice args: bundle, cursor_keys, profile, probes, topic_id, +route_generation, target=localhost:8210). Metrics must be a parent-owned bounded +JSONL capture of this isolated Stream's stdout, beginning before this driver. +Both consumers must have the same BINANCE USDM trade requirement, quota1000. +Stream must retain max_replay_events=10000, report metrics every second, and +use a queue byte budget <=32MiB. No production credentials or mounts are allowed. + +Parent bounds: Kafka 1CPU/1536MiB; core .5CPU/256MiB; Stream .5CPU/512MiB; +Python .75CPU/512MiB; quota Redis .1CPU/64MiB; Query .5CPU/512MiB. --rm, no published ports, no shared mounts, tmpfs/bounded logs, +ten-minute deadline. The orchestrator validates labels and cleans only its owned names. +""" +from __future__ import annotations + +import argparse +import asyncio +import base64 +import hashlib +import json +import multiprocessing +import os +from pathlib import Path +import re +import signal +import sys +import time +from types import SimpleNamespace +import uuid + +SCHEMA = "p184.surface-evidence-provenance.v1" +CELLS = tuple("DR08." + name + "@ANY" for name in ( + "SLOW_CONSUMER_BOUNDED_QUEUE", "CONSUMER_RESTART_RECOVERY_MEASURED", + "NO_CROSS_CONSUMER_STARVATION", "ALERT_PATH_OBSERVABLE")) +QUEUE = 1000 +REPLAY = 10000 +MAX_METRICS_BYTES = 8 * 1024 * 1024 +QUEUE_BYTES = 32 * 1024 * 1024 + + +def validate_packet(packet, *, development): + """Reject accidental shared endpoints and ambiguous provenance before I/O.""" + namespace = packet.get("namespace", "") + if not re.fullmatch(r"p184-dr08-[a-z0-9-]{8,64}", namespace): + raise ValueError("a unique p184-dr08 namespace is required") + if os.environ.get("P184_DR08_ISOLATED_NAMESPACE") != namespace: + raise ValueError("parent must attest the isolated container namespace") + if type(packet.get("development")) is not bool or packet["development"] != development: + raise ValueError("packet and execution mode differ; prepare a fresh packet") + if packet.get("bootstrap") != "localhost:9092" or packet["slice"].get("target") != "localhost:8210": + raise ValueError("only the isolated namespace's loopback endpoints are allowed") + if packet.get("consumer_id") == packet.get("fast_consumer_id"): + raise ValueError("two distinct consumer identities are required") + for role in ("kafka", "core", "stream", "python"): + if not re.fullmatch(r"sha256:[0-9a-f]{64}", packet.get("images", {}).get(role, "")): + raise ValueError(f"immutable {role} image ID required") + if not development: + sha, image = packet.get("ts_source_sha", ""), packet.get("ts_image_digest", "") + if not re.fullmatch(r"[0-9a-f]{40}", sha) or len(set(sha)) == 1: + raise ValueError("final TS source SHA required") + if not re.fullmatch(r"sha256:[0-9a-f]{64}", image) or len(set(image[7:])) == 1: + raise ValueError("final TS image digest required") + if not packet.get("raw_topic") or not packet.get("metrics_path"): + raise ValueError("isolated raw topic and metrics capture required") + + +def evidence_rows(packet, facts, *, development, run_id): + """Development rows deliberately cannot satisfy the TS matrix checker.""" + mode = "DEVELOPMENT" if development else "QUALIFICATION" + header = {"provenance": { + "schema": SCHEMA, "source_sha": packet["ts_source_sha"], + "image_digest": packet["ts_image_digest"], "mode": mode, "run_id": run_id, + "data_provenance": "SYNTHETIC_TEST_ONLY", "images": packet["images"], + "packet_sha256": hashlib.sha256(json.dumps(packet, sort_keys=True).encode()).hexdigest(), + "scope": "same-host isolated DL fault test; not TS consumer or real-provider certification", + }} + rows = [header] + for cell in CELLS: + fact = facts.get(cell, {}) + observed = fact.get("observed") is True + passed = observed and fact.get("passed") is True + rows.append({"cell": cell, "level": "ISOLATED_FAULT", "mode": mode, + "run_id": run_id, "ok": passed and not development, + "outcome": "OBSERVED" if observed and not development else "NOT_OBSERVED", + "development_pass": passed if development else None, + "measurement": fact}) + return rows + + +def read_metrics(path, *, since_ns): + path = Path(path) + if path.stat().st_size > MAX_METRICS_BYTES: + raise ValueError("metrics capture exceeds 8MiB bound") + rows = [] + for line in path.read_text().splitlines(): + try: + row = json.loads(line) + except json.JSONDecodeError: + continue # Rust startup text and an in-progress final line. + if row.get("event") == "qdl_kn_gateway_metrics": + rows.append(row) + # since_ns is carried in evidence; the parent starts an exclusive capture + # before the run. Counter deltas, not file timestamps, delimit the test. + if not rows: + raise ValueError(f"no native Stream metrics available for run {since_ns}") + return rows + + +def slow_queue_observation(path, consumer, *, subscription_id, wire_code): + path = Path(path) + if path.stat().st_size > MAX_METRICS_BYTES: + raise ValueError("metrics capture exceeds bound") + if wire_code != "RESOURCE_EXHAUSTED": + return None + for line in path.read_text().splitlines(): + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if (isinstance(row, dict) and row.get("event") == "qdl_kn_subscription_closed" + and row.get("consumer_id") == consumer and row.get("subscription_id") == subscription_id + and row.get("outcome") in ("backpressure", "Some resource has been exhausted") + and row.get("queued") == QUEUE and 0 < row.get("queued_bytes", 0) <= QUEUE_BYTES): + return {key: row[key] for key in ("subscription_id", "queued", "queued_bytes", "outcome")} + return None + + +def wait_metrics(path, *, since_ns, timeout=20): + deadline = time.monotonic() + timeout + while True: + try: + return read_metrics(path, since_ns=since_ns) + except (FileNotFoundError, ValueError): + if time.monotonic() >= deadline: + raise TimeoutError("isolated metrics baseline unavailable") + time.sleep(.1) + + +def metric_verdict(before, after): + required = ("overflow", "expired", "queue_bytes", "rss_bytes", "cpu_seconds") + if any(key not in row for row in (before, after) for key in required): + return False + return (after["overflow"] > before["overflow"] and after["expired"] > before["expired"] + and 0 <= after["queue_bytes"] <= QUEUE_BYTES + and after["cpu_seconds"] >= before["cpu_seconds"]) + + +def harness(packet): + # Keep all profile, manifest, JWT and cursor behavior in the native harness. + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from kn_native_slice_probe import Slice + from qdl.consumer.manifest import ConsumerManifestLoader + runner = Slice(SimpleNamespace(**packet["slice"])) + runner.manifests = {m.consumer_id: m for m in ( + ConsumerManifestLoader.load(Path(path)) for path in packet["isolated_manifest_paths"])} + required = {packet["consumer_id"], packet["fast_consumer_id"]} + if set(runner.manifests) != required or set(runner.identities) != required: + raise ValueError("only the two disposable consumer identities may be loaded") + if any(not cid.startswith(packet["namespace"] + ".") for cid in required): + raise ValueError("consumer identities must belong to this isolated namespace") + return runner + + +async def channel_for(runner, consumer): + import grpc + identity = runner.identities[consumer]["tls"] + credentials = grpc.ssl_channel_credentials( + Path(identity["ca_file"]).read_bytes(), Path(identity["key_file"]).read_bytes(), + Path(identity["cert_file"]).read_bytes()) + channel = grpc.aio.secure_channel(runner.args.target, credentials, options=( + ("grpc.http2.bdp_probe", 0), ("grpc.enable_retries", 0), ("grpc.enable_http_proxy", 0), + ("grpc.max_receive_message_length", 1024 * 1024))) + try: + await asyncio.wait_for(channel.channel_ready(), 20) + except TimeoutError: + state = channel.get_state().name + await channel.close() + raise TimeoutError("CHANNEL_READY_TIMEOUT:" + state) from None + return channel + + +def rpc_inputs(runner, packet, consumer): + from qdl.certification.phase103_consumer_acceptance import sdk_requirement + domain = runner.requirement(consumer, packet["instrument_uid"], "TRADE") + proto = sdk_requirement(SimpleNamespace(requirement=domain)).to_proto() + metadata = (("authorization", "Bearer " + runner.jwt(consumer)), + ("x-qdl-consumer-id", consumer), ("x-qdl-purpose", "INTERNAL_ALPHA")) + return domain, proto, metadata + + +async def snapshot(packet, consumer, expected_trade_id): + import grpc + from qdl.query.v2 import query_pb2 as pb + runner = harness(packet) + domain, requirement, metadata = rpc_inputs(runner, packet, consumer) + channel = await channel_for(runner, consumer) + try: + rpc = channel.unary_unary("/qdl.query.v2.MarketDataStreamService/GetSnapshot", + request_serializer=pb.GetSnapshotRequest.SerializeToString, + response_deserializer=pb.GetSnapshotResponse.FromString) + deadline = time.monotonic() + 20 + last_refusal = "NO_MATCHING_MARKER" + while True: + try: + result = await rpc(pb.GetSnapshotRequest(consumer_id=consumer, requirement=requirement), + metadata=metadata, timeout=min(3, max(.01, deadline - time.monotonic()))) + if result.events and int(result.events[-1].trade.native_trade_id) == expected_trade_id: + break + except grpc.aio.AioRpcError as error: + last_refusal = error.code().name + ":" + error.details()[:160] + if error.code() not in (grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.FAILED_PRECONDITION): + raise + if time.monotonic() >= deadline: + raise TimeoutError("SNAPSHOT_MARKER_TIMEOUT:" + last_refusal) + await asyncio.sleep(.1) + if not result.events or not runner.cursor_valid(result.stream_cursor, consumer, domain): + raise ValueError("fresh snapshot requires actual events and a valid server cursor") + if any(event.instrument_uid != packet["instrument_uid"] or not event.HasField("trade") + for event in result.events): + raise ValueError("snapshot product mismatch") + return result.stream_cursor, result.watermark_offset + finally: + await channel.close() + + +async def consume(packet, consumer, token, pipe, *, stop_after, stall): + import grpc + from qdl.query.v2 import query_pb2 as pb + runner = harness(packet) + _, requirement, metadata = rpc_inputs(runner, packet, consumer) + channel = await channel_for(runner, consumer) + count, last, controls, first_ns = 0, None, [], None + try: + rpc = channel.unary_stream("/qdl.query.v2.MarketDataStreamService/Subscribe", + request_serializer=pb.SubscribeRequest.SerializeToString, + response_deserializer=pb.SubscribeResponse.FromString) + call = rpc(pb.SubscribeRequest(consumer_id=consumer, requirement=requirement, + cursor_token=token, max_buffer_events=QUEUE), metadata=metadata, timeout=180) + try: + async for response in call: + record = response.record + if record.HasField("control"): + controls.append(record.control.code) + if record.control.code == "LIVE": + pipe.send({"kind": "live", "count": count}) + if stall: + os.kill(os.getpid(), signal.SIGSTOP) + continue + if not record.HasField("event") or not record.event.HasField("trade"): + raise ValueError("unexpected stream product") + if record.event.instrument_uid != packet["instrument_uid"]: + raise ValueError("stream identity mismatch") + current = int(record.event.trade.native_trade_id) + if last is not None and current != last + 1: + raise ValueError("trade identity loss, duplicate or reordering") + last = current + count += 1 + first_ns = first_ns or time.monotonic_ns() + if count % 100 == 0: + pipe.send({"kind": "progress", "count": count, "last": last}) + if stop_after and count >= stop_after: + break + pipe.send({"kind": "done", "count": count, "last": last, + "first_ns": first_ns, "controls": controls, "code": "OK"}) + except grpc.aio.AioRpcError as error: + pipe.send({"kind": "done", "count": count, "last": last, "controls": controls, + "code": error.code().name, "detail": error.details()[:256]}) + finally: + call.cancel() + finally: + await channel.close() + + +def child_main(packet, consumer, token, pipe, stop_after, stall): + try: + asyncio.run(consume(packet, consumer, token, pipe, stop_after=stop_after, stall=stall)) + except Exception as error: + # No exception strings: third-party errors can include auth material. + pipe.send({"kind": "error", "type": type(error).__name__}) + finally: + pipe.close() + + +class Consumer: + def __init__(self, packet, consumer, token, *, stop_after=0, stall=False): + ctx = multiprocessing.get_context("spawn") + self.pipe, sender = ctx.Pipe(duplex=False) + self.process = ctx.Process(target=child_main, + args=(packet, consumer, token, sender, stop_after, stall)) + self.started_ns = time.monotonic_ns() + self.process.start() + sender.close() + self.progress = 0 + + def wait(self, kind, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not self.pipe.poll(min(.2, max(0, deadline - time.monotonic()))): + continue + row = self.pipe.recv() + self.progress = max(self.progress, row.get("count", 0)) + if row["kind"] == kind: + return row + if row["kind"] in ("done", "error"): + raise RuntimeError(f"consumer ended before {kind}: {row}") + raise TimeoutError(f"consumer did not reach {kind}") + + def resume(self): + os.kill(self.process.pid, signal.SIGCONT) + + def close(self): + if self.process.is_alive(): + self.resume() + self.process.terminate() + self.process.join(5) + if self.process.is_alive(): + self.process.kill() + self.process.join(5) + self.pipe.close() + + +class RawProducer: + def __init__(self, packet): + from confluent_kafka import Producer + from qdl.provider.v1.raw_provider_pb2 import RawProviderEnvelope + from qdl.raw.envelope import validate_raw_envelope + self.packet = packet + self.template = RawProviderEnvelope.FromString(base64.b64decode(packet["raw_template"], validate=True)) + validate_raw_envelope(self.template) + if not self.template.test_provenance or self.template.venue != "BINANCE" or self.template.market != "USDM": + raise ValueError("requires explicitly test-only BINANCE USDM raw template") + if not self.template.native_channel.lower().endswith(("@aggtrade", "@trade")): + raise ValueError("requires a native trade template") + self.producer = Producer({**kafka_security(packet), + "enable.idempotence": True, "message.timeout.ms": 20000, + "queue.buffering.max.messages": 1000, "queue.buffering.max.kbytes": 4096}) + self.next_id = 1 + + def publish(self, count): + from qdl.provider.v1.raw_provider_pb2 import RawProviderEnvelope + errors = [] + def delivered(error, _message): + if error is not None: + errors.append(str(error.code())) + for _ in range(count): + raw = RawProviderEnvelope() + raw.CopyFrom(self.template) + index, now = self.next_id, time.time_ns() + self.next_id += 1 + raw.capture_id = uuid.uuid4().bytes + raw.received_at_ns = now + raw.raw_frame_bytes = json.dumps({"e": "trade" if raw.native_channel.lower().endswith("@trade") else "aggTrade", "E": now // 1_000_000, + "T": now // 1_000_000, "s": raw.native_symbol, "a": index, "t": index, + "p": "100", "q": "1", "f": index, "l": index, "m": False}).encode() + raw.raw_frame_sha256 = hashlib.sha256(raw.raw_frame_bytes).digest() + self.producer.produce(self.packet["raw_topic"], key=raw.subscription_id.encode(), + value=raw.SerializeToString(), headers=[("qdl-event-id", raw.capture_id)], on_delivery=delivered) + self.producer.poll(0) + if self.producer.flush(25) or errors: + raise RuntimeError("isolated raw publication failed") + + +def wait_seed(packet): + from confluent_kafka import Consumer as KafkaConsumer, TopicPartition + from qdl.marketdata.v2.market_data_pb2 import EventEnvelope + topic = json.loads(Path(packet["slice"]["bundle"]).read_text())["catalog"]["canonical_stream"] + consumer = KafkaConsumer({**kafka_security(packet), "group.id": "kn-" + packet["namespace"] + "-oracle", + "enable.auto.commit": False, "enable.auto.offset.store": False, "isolation.level": "read_committed"}) + started = time.monotonic() + try: + consumer.assign([TopicPartition(topic, 0, 0)]) + while time.monotonic() - started < 25: + message = consumer.poll(.2) + if message is None: + continue + if message.error(): + raise RuntimeError("isolated canonical seed read failed") + event = EventEnvelope.FromString(message.value()) + if event.instrument_uid != packet["instrument_uid"] or event.trade.native_trade_id != "1": + raise ValueError("isolated canonical seed identity mismatch") + from qdl.runtime.stable_catalog import StableSourceCatalog + catalog = StableSourceCatalog.load(Path(packet["slice"]["bundle"]).parent / "catalog.yaml") + catalog.binding_for_envelope(event) + return {"partition": message.partition(), "offset": message.offset(), + "committed_seed_wait_ms": (time.monotonic() - started) * 1000} + raise TimeoutError("SEED_CANONICAL_COMMIT_TIMEOUT") + finally: + consumer.close() + + +def run_faults(packet): + facts, children = {}, [] + stage = "baseline_metrics" + started = time.time_ns() + slow_id, fast_id = packet["consumer_id"], packet["fast_consumer_id"] + def start(consumer, token, **kwargs): + child = Consumer(packet, consumer, token, **kwargs) + children.append(child) + return child + try: + before = wait_metrics(packet["metrics_path"], since_ns=started)[-1] + stage = "raw_producer" + producer = RawProducer(packet) + producer.publish(1) + stage = "seed_canonical_commit" + seed = wait_seed(packet) + # Cold assignment is outside the fault window; keep catalog freshness. + producer.publish(1) + baseline_id = producer.next_id - 1 + stage = "initial_snapshot" + old_token, old_offset = asyncio.run(snapshot(packet, slow_id, baseline_id)) + fast_token, _ = asyncio.run(snapshot(packet, fast_id, baseline_id)) + stage = "slow_subscribe" + slow = start(slow_id, old_token, stall=True) + slow.wait("live") + stage = "fast_subscribe" + fast = start(fast_id, fast_token, stop_after=REPLAY + 1) + fast.wait("live") + stage = "stalled_consumer_live_progress" + progress_samples = [] + stalled_started_ns = time.monotonic_ns() + slow_resumed = False + for _ in range(REPLAY // 100): + producer.publish(100) + # Backpressure the test producer on the independent fast consumer, + # not the stalled one. This also bounds IPC and records progress. + progress = fast.wait("progress") + if not slow_resumed: + progress_samples.append({"count": progress["count"], + "elapsed_ms": (time.monotonic_ns() - stalled_started_ns) / 1_000_000}) + samples = read_metrics(packet["metrics_path"], since_ns=started) + if samples and samples[-1].get("overflow", 0) > before.get("overflow", 0): + slow.resume() + slow_resumed = True + stage = "slow_terminal" + if not slow_resumed: + slow.resume() + slow_end = slow.wait("done") + slow.close() + bounded = slow_end["code"] == "RESOURCE_EXHAUSTED" and "RATE_LIMITED" in slow_end["controls"] + facts[CELLS[0]] = {"observed": True, "passed": False, + "configured_queue_events": QUEUE, "slow_terminal": slow_end, "seed_commit": seed} + stage = "exact_10000_replay" + replay = start(slow_id, old_token, stop_after=REPLAY) + replay_end = replay.wait("done", 90) + replay.close() + exact_replay = replay_end["code"] == "OK" and replay_end["count"] == REPLAY and replay_end["last"] == baseline_id + REPLAY + producer.publish(1) + fast_end = fast.wait("done") + fast.close() + stage = "out_of_range" + expired = start(slow_id, old_token) + range_end = expired.wait("done") + range_observed_ns = time.monotonic_ns() + expired.close() + out_of_range = range_end["code"] == "OUT_OF_RANGE" and "backlog" in range_end.get("detail", "") + stage = "fresh_snapshot" + producer.publish(1) + fresh_trade_id = producer.next_id - 1 + snapshot_started_ns = time.monotonic_ns() + fresh_token, fresh_offset = asyncio.run(snapshot(packet, slow_id, fresh_trade_id)) + snapshot_ms = (time.monotonic_ns() - snapshot_started_ns) / 1_000_000 + stage = "consumer_restart" + restart = start(slow_id, fresh_token, stop_after=1) + restart.wait("live") + producer.publish(1) + recovered = restart.wait("done") + restart_ms = (recovered.get("first_ns", 0) - restart.started_ns) / 1_000_000 + restart.close() + facts[CELLS[1]] = {"observed": True, + "passed": exact_replay and out_of_range and fresh_offset > old_offset + and recovered["code"] == "OK" and recovered["count"] == 1 + and recovered["last"] == fresh_trade_id + 1 and 0 < restart_ms < 60000, + "replayed": replay_end["count"], "range_code": range_end["code"], + "old_snapshot_offset": old_offset, "fresh_snapshot_offset": fresh_offset, + "baseline_trade_id": baseline_id, "fresh_snapshot_trade_id": fresh_trade_id, + "restart_to_first_event_ms": restart_ms, "restart_kind": "consumer_process", + "snapshot_request_ms": snapshot_ms, "measurement_n": 1, + "range_observed_to_first_event_ms": (recovered.get("first_ns", 0) - range_observed_ns) / 1_000_000, + "ack_measured": False, + "fresh_snapshot_rpc": "MarketDataStreamService/GetSnapshot", + "snapshot_path": "isolated Query to canonical hot read; not projector recovery proof"} + facts[CELLS[2]] = {"observed": True, + "passed": fast_end["code"] == "OK" and fast_end["count"] == REPLAY + 1 and fast_end["last"] == baseline_id + REPLAY + 1 + and bool(progress_samples) and slow_resumed, + "fast_events": fast_end["count"], "distinct_identities": True, + "identity_order_checked": True, "progress_checked_every": 100, + "progress_while_slow_stopped": progress_samples, "measurement_n": len(progress_samples)} + time.sleep(2) + stage = "final_metrics" + metrics = read_metrics(packet["metrics_path"], since_ns=started) + after = metrics[-1] + max_bytes = max(row.get("queue_bytes", QUEUE_BYTES + 1) for row in metrics) + closed = slow_queue_observation(packet["metrics_path"], slow_id, + subscription_id=before["subscriptions_opened"] + 1, wire_code=slow_end["code"]) + facts[CELLS[0]]["passed"] = bounded and max_bytes <= QUEUE_BYTES and closed is not None + facts[CELLS[0]]["observed_closed_queue"] = closed + facts[CELLS[0]]["sampled_max_queue_bytes"] = max_bytes + facts[CELLS[3]] = {"observed": True, + "passed": bounded and out_of_range and metric_verdict(before, after), + "overflow_delta": after.get("overflow", 0) - before.get("overflow", 0), + "expired_delta": after.get("expired", 0) - before.get("expired", 0), + "metrics_samples": len(metrics), "alert_scope": "native controls, status and structured counters; not paging delivery"} + except Exception as error: + # Keep any completed observations without turning an incomplete run green. + for cell in CELLS: + facts.setdefault(cell, {"observed": False, "passed": False, + "failure_type": type(error).__name__, "failure_stage": stage, + "failure_detail": str(error)[:128] if isinstance(error, TimeoutError) else None, + "failure_code": error.code().name if callable(getattr(error, "code", None)) else None}) + finally: + for child in children: + child.close() + return facts + + +def dependency_smoke(): + """Exercise real imports and protobuf construction, without runtime traffic.""" + import grpc + from confluent_kafka import Producer, TopicCollection + from qdl.runtime.stable_source import build_stable_query_stack + from qdl.runtime.kn_read_view import install_kn_read_view + from qdl.runtime.kn_query_backend import KnMarketCacheQueryBackend, KnCursorSettings, KnCursorV3Issuer + from qdl.runtime.kn_market_cache import reader_from_environment + from qdl.runtime.kn_hot_view import CanonicalHotClient + import uvicorn + from qdl.raw.envelope import build_raw_envelope, validate_raw_envelope + from qdl.provider.v1 import raw_provider_pb2 as raw_pb + from qdl.query.v2 import query_pb2 as pb + from qdl.marketdata.v2.market_data_pb2 import EventEnvelope + from qdl.consumer.manifest import ConsumerManifestLoader + from qdl.certification.phase103_consumer_acceptance import sdk_requirement + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from kn_native_slice_probe import Slice + raw = build_raw_envelope(capture_id=uuid.uuid4().bytes, provider="BINANCE_DIRECT", + venue="BINANCE", market="USDM", product_type="PERPETUAL", native_symbol="BTCUSDT", + native_channel="btcusdt@aggTrade", subscription_id="test-only", source_session_id="test-only", + connection_generation=1, lease_epoch=1, authority_revision=1, partition_plan_epoch=1, + received_at_ns=time.time_ns(), transport_protocol=raw_pb.TRANSPORT_PROTOCOL_WEBSOCKET, + transport_compression=raw_pb.TRANSPORT_COMPRESSION_NONE, + capture_boundary=raw_pb.CAPTURE_BOUNDARY_POST_DECOMPRESSION, + raw_frame_bytes=b'{"e":"aggTrade","a":1}', adapter_version="p184-test-only", + config_revision=1, instrument_catalog_revision=1, correlation_id="p184-test-only", + test_provenance=True) + validate_raw_envelope(raw_pb.RawProviderEnvelope.FromString(raw.SerializeToString())) + from unittest.mock import patch, MagicMock + fake = MagicMock() + fake.flush.return_value = 0 + with patch("confluent_kafka.Producer", return_value=fake): + driver = RawProducer({"raw_template": base64.b64encode(raw.SerializeToString()).decode(), + "bootstrap": "localhost:9092", "raw_topic": "test-only", "client_tls": { + "ca": "/test/ca", "cert": "/test/cert", "key": "/test/key"}}) + driver.publish(1) + sent = fake.produce.call_args.kwargs + parsed_raw = raw_pb.RawProviderEnvelope.FromString(sent["value"]) + validate_raw_envelope(parsed_raw) + assert sent["headers"] == [("qdl-event-id", parsed_raw.capture_id)] + assert parsed_raw.test_provenance and json.loads(parsed_raw.raw_frame_bytes)["t"] == 1 + event = EventEnvelope(instrument_uid="test-only") + event.trade.native_trade_id = "1" + answer = pb.GetSnapshotResponse(stream_cursor="test-only", watermark_offset=1, events=[event]) + decoded = pb.GetSnapshotResponse.FromString(answer.SerializeToString()) + assert decoded.events[0].trade.native_trade_id == "1" and decoded.watermark_offset == 1 + request = pb.SubscribeRequest(consumer_id="test-only", max_buffer_events=QUEUE) + assert request.max_buffer_events == QUEUE + return {"imports": "PASS", "raw_proto_roundtrip": "PASS", "snapshot_proto_roundtrip": "PASS", + "runtime_tested": False} + + +IMAGES = { + "core": "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023", + "stream": "sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9", + "python": "sha256:77682eea5054ae9ddbd858323b083286669a55eeb4bc7bc065b04bae8854e32f", + "kafka": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "redis": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", +} + + +def prepare_material(work): + """Inside the network-disabled Python image; only fresh test credentials.""" + import secrets + import shutil + import yaml + from datetime import datetime, timedelta, timezone + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.hazmat.primitives.serialization import pkcs12 + from cryptography.x509.oid import NameOID, ExtendedKeyUsageOID + from qdl.runtime.stable_catalog import StableSourceCatalog + from qdl.provider.v1.raw_provider_pb2 import RawProviderEnvelope + from qdl.raw.envelope import build_raw_envelope + from tests.phase7_support import manifest_mapping + from scripts.kn_gateway_bundle import compile_bundle + from scripts.kn_native_slice_probe import ISSUER, AUDIENCE + + def save(name, value, private=False): + path = work / name + with path.open("x") as handle: + path.chmod(0o600 if private else 0o644) + handle.write(value if isinstance(value, str) else json.dumps(value)) + + cfg = json.loads((work / "core.json").read_text()) + packet = json.loads((work / "packet.json").read_text()) + b = cfg["core"]["bindings"][0] + namespace = packet["namespace"] + root = Path(__file__).resolve().parents[1] + catalog = yaml.safe_load((root / "config/v2/stable-source-bindings.yaml").read_text()) + catalog["canonical_stream"] = cfg["core"]["canonical_stream"] + # The synthetic adapter has its own declared lineage in this test-only catalog. + for row in catalog["bindings"]: + if row["source"]["source_id"] == b["source_id"]: + row["source"]["adapter_version"] = "p184-synthetic-test-only" + save("catalog.yaml", yaml.safe_dump(catalog)) + loaded = StableSourceCatalog.load(work / "catalog.yaml") + binding = next(x for x in loaded.bindings if x.instrument.instrument_uid == b["instrument_uid"] + and x.feed.value == "TRADE" and x.source_id == b["source_id"]) + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + now = datetime.now(timezone.utc) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, namespace)]) + ca = (x509.CertificateBuilder().subject_name(name).issuer_name(name) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)).not_valid_after(now + timedelta(hours=2)) + .add_extension(x509.BasicConstraints(ca=True, path_length=0), True) + .sign(key, hashes.SHA256())) + leaf_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + leaf = (x509.CertificateBuilder().subject_name(x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, "localhost")])).issuer_name(name) + .public_key(leaf_key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)).not_valid_after(now + timedelta(hours=2)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), True) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("localhost")]), False) + .add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH, + ExtendedKeyUsageOID.CLIENT_AUTH]), False) + .sign(key, hashes.SHA256())) + pem = serialization.Encoding.PEM + save("ca.crt", ca.public_bytes(pem).decode()) + save("client.crt", leaf.public_bytes(pem).decode()) + save("client.key", leaf_key.private_bytes(pem, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption()).decode(), True) + password = secrets.token_hex(16) + with (work / "server.p12").open("xb") as handle: + (work / "server.p12").chmod(0o600) + handle.write(pkcs12.serialize_key_and_certificates(b"dr08", leaf_key, leaf, [ca], + serialization.BestAvailableEncryption(password.encode()))) + props = """process.roles=broker,controller +node.id=1 +controller.quorum.voters=1@localhost:9093 +listeners=SSL://:9092,CONTROLLER://:9093 +advertised.listeners=SSL://localhost:9092 +controller.listener.names=CONTROLLER +listener.security.protocol.map=SSL:SSL,CONTROLLER:PLAINTEXT +inter.broker.listener.name=SSL +ssl.keystore.type=PKCS12 +ssl.keystore.location=/work/server.p12 +ssl.keystore.password=PASSWORD +ssl.key.password=PASSWORD +ssl.truststore.type=PEM +ssl.truststore.location=/work/ca.crt +ssl.client.auth=required +log.dirs=/data +log.segment.bytes=16777216 +metadata.log.segment.bytes=16777216 +log.index.size.max.bytes=1048576 +offsets.topic.num.partitions=1 +transaction.state.log.num.partitions=1 +num.partitions=1 +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 +group.initial.rebalance.delay.ms=0 +""" + save("broker.properties", props.replace("PASSWORD", password), True) + manifests, identities, keys, subjects = [], [], {}, {} + for label in ("slow", "fast"): + cid = namespace + "." + label + subject = "spiffe://qdl/paper/" + cid + mapping = manifest_mapping(consumer_id=cid, subject=subject, instrument_uid=b["instrument_uid"], + feed="TRADE", interval=None, source_policy_id=binding.source_policy_id) + mapping["spec"]["quotas"]["max_buffer_events"] = QUEUE + mapping["spec"]["requirements"][0].update(warmup_limit=1, require_full_coverage=False, + require_final_bars=False, max_freshness_ms=60000) + manifest_path = work / (label + ".yaml") + save(label + ".yaml", yaml.safe_dump(mapping)) + manifests.append(manifest_path) + jwt_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + save(label + ".key", jwt_key.private_bytes(pem, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption()).decode(), True) + keys[label] = jwt_key.public_key().public_bytes(pem, serialization.PublicFormat.SubjectPublicKeyInfo).decode() + subjects[label] = subject + identities.append({"id": cid, "tls": {"ca_file": "/work/ca.crt", "cert_file": "/work/client.crt", + "key_file": "/work/client.key"}, "jwt": {"private_key_file": "/work/" + label + ".key", "key_id": label}}) + save("jwt-config.json", {"issuer": ISSUER, "audience": AUDIENCE, "algorithms": "RS256", + "keys": keys, "subjects": subjects, "environments": {k: "paper" for k in keys}, "max_lifetime_seconds": 900}) + save("profile.json", {"identities": identities}) + save("bundle.json", compile_bundle(environment="paper", catalog_path=work / "catalog.yaml", manifest_paths=manifests)) + save("cursor-keys.json", {"dr08": secrets.token_hex(32)}, True) + save("read-view.secret", secrets.token_hex(32), True) + save("probes.json", []) + raw = build_raw_envelope(capture_id=uuid.uuid4().bytes, + **{k: b[k] for k in ("provider", "venue", "market", "product_type", "native_symbol", "native_channel")}, + subscription_id=b["source_id"], source_session_id=namespace, connection_generation=1, + lease_epoch=1, authority_revision=cfg["authority"]["revision"], partition_plan_epoch=1, + received_at_ns=time.time_ns(), transport_protocol=1, transport_compression=1, capture_boundary=2, + raw_frame_bytes=b'{"e":"aggTrade","a":1}', adapter_version="p184-synthetic-test-only", + config_revision=1, instrument_catalog_revision=b["instrument_catalog_revision"], + correlation_id=namespace, test_provenance=True) + packet.update(consumer_id=namespace + ".slow", fast_consumer_id=namespace + ".fast", + instrument_uid=b["instrument_uid"], raw_template=base64.b64encode(raw.SerializeToString()).decode(), + raw_topic=cfg["raw_topics"][0], metrics_path="/work/stream-runtime.log", + client_tls={"ca": "/work/ca.crt", "cert": "/work/client.crt", "key": "/work/client.key"}, + isolated_manifest_paths=["/work/slow.yaml", "/work/fast.yaml"], + slice={"bundle": "/work/bundle.json", "cursor_keys": "/work/cursor-keys.json", + "profile": "/work/profile.json", "probes": "/work/probes.json", + "topic_id": "pending", "route_generation": namespace, "target": "localhost:8210"}) + (work / "packet.json").write_text(json.dumps(packet)) + # Keep all newly generated private material owned by the container's host UID. + + +def kafka_security(packet): + tls = packet["client_tls"] + return {"bootstrap.servers": packet["bootstrap"], "security.protocol": "SSL", + "ssl.ca.location": tls["ca"], "ssl.certificate.location": tls["cert"], + "ssl.key.location": tls["key"]} + + +def prepare_topics(work): + from confluent_kafka.admin import AdminClient, NewTopic + packet = json.loads((work / "packet.json").read_text()) + cfg = json.loads((work / "core.json").read_text()) + admin = AdminClient(kafka_security(packet)) + deadline = time.monotonic() + 60 + while True: + try: + admin.list_topics(timeout=2) + break + except Exception: + if time.monotonic() >= deadline: + raise TimeoutError("isolated TLS broker readiness") + time.sleep(.5) + topics = cfg["raw_topics"] + [cfg["core"]["canonical_stream"], cfg["core"]["quarantine_stream"]] + for future in admin.create_topics([NewTopic(name, 1, 1) for name in topics]).values(): + future.result(20) + # describe_topics exposes the broker's immutable topic UUID (not a guessed ID). + from confluent_kafka import TopicCollection + result = admin.describe_topics(TopicCollection([cfg["core"]["canonical_stream"]])) + packet["slice"]["topic_id"] = str(result[cfg["core"]["canonical_stream"]].result(20).topic_id) + (work / "packet.json").write_text(json.dumps(packet)) + + +def serve_query(work): + """Real KN Query read-view stack, backed by the isolated canonical hot view.""" + import ssl + import uvicorn + from fastapi import FastAPI + from qdl.runtime.stable_catalog import StableSourceCatalog + from qdl.runtime.stable_source import build_stable_query_stack + from qdl.runtime.kn_query_backend import KnMarketCacheQueryBackend, KnCursorSettings, KnCursorV3Issuer + from qdl.runtime.kn_market_cache import reader_from_environment + from qdl.runtime.kn_hot_view import CanonicalHotClient + from qdl.runtime.kn_read_view import install_kn_read_view + + packet = json.loads((work / "packet.json").read_text()) + catalog = StableSourceCatalog.load(work / "catalog.yaml") + settings = KnCursorSettings.from_environment(os.environ, environment="paper") + expectation = dict(environment="paper", stream=catalog.canonical_stream, + source_topic_id=settings.topic_id, partition_plan_epoch=1, + source_policy_revision=catalog.source_policy_revision, catalog_revision=catalog.catalog_revision, + route_generation=settings.route_generation, schema_major=2) + hot = CanonicalHotClient.connect(targets=("localhost:8210",), + ca=(work / "ca.crt").read_bytes(), cert=(work / "client.crt").read_bytes(), + key=(work / "client.key").read_bytes(), expectation=expectation, + secret=bytes.fromhex((work / "read-view.secret").read_text())) + reader = reader_from_environment({"QDL_KN3_MARKET_CACHE_URL": "redis://localhost:6379/1", + "QDL_KN3_ENVIRONMENT": "paper"}) + backend = KnMarketCacheQueryBackend(reader, catalog, schema_digest="a" * 64, + topic_id=settings.topic_id, hot_client=hot) + issuer = KnCursorV3Issuer(settings, catalog) + service, _, _ = build_stable_query_stack(spool=None, catalog=catalog, schema_digest="a" * 64, + handoff=None, cursor_ttl_seconds=3600, backend=backend, issuer=issuer) + app = FastAPI() + install_kn_read_view(app, service=service, backend=backend, issuer=issuer, + secret=bytes.fromhex((work / "read-view.secret").read_text())) + try: + uvicorn.run(app, host="127.0.0.1", port=8200, access_log=False, + ssl_keyfile=str(work / "client.key"), ssl_certfile=str(work / "client.crt"), + ssl_ca_certs=str(work / "ca.crt"), ssl_cert_reqs=ssl.CERT_REQUIRED) + finally: + hot.close() + + +def orchestrate(work, *, development, source_sha, image_digest): + """Adapt measure_l2_top_materialization's isolated broker/core packet.""" + import shutil + import subprocess + root = Path(__file__).resolve().parents[1] + if not re.fullmatch(r"[0-9a-f]{40}", source_sha) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_digest): + raise ValueError("TS provenance must use full SHA/digest syntax even in development") + if not development: + if not re.fullmatch(r"[0-9a-f]{40}", source_sha) or len(set(source_sha)) == 1: + raise ValueError("final TS source SHA required before setup") + if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_digest) or len(set(image_digest[7:])) == 1: + raise ValueError("final TS image digest required before setup") + work = work.resolve() + work.mkdir(mode=0o700, parents=True, exist_ok=False) + namespace = "p184-dr08-" + uuid.uuid4().hex[:16] + print(json.dumps({"namespace": namespace, "roles": ["prepare", "kafka", "topics", "quota", "core", "stream", "query", "driver"], "mode": "DEVELOPMENT" if development else "QUALIFICATION"}), flush=True) + owned, log_process, log_file = [], None, None + disk_before = shutil.disk_usage(work).free + def command(args, timeout=90): + return subprocess.run(args, check=True, text=True, capture_output=True, timeout=timeout).stdout.strip() + def inventory(): + ids = command(["docker", "ps", "-q"]).splitlines() + rows = json.loads(command(["docker", "inspect", *ids])) if ids else [] + return {r["Id"]: {"image": r["Image"], "started": r["State"]["StartedAt"], + "restarts": r["RestartCount"]} for r in rows if r["Name"].startswith("/qdl_v2_stable_candidate-") + or r["Name"] == "/market_data_service_p184w"} + before = inventory() + try: + for image in IMAGES.values(): + command(["docker", "image", "inspect", image, "--format", "{{.Id}}"]) + runtime = json.loads(command(["docker", "inspect", "qdl_v2_stable_candidate-rust_core-1"]))[0] + if runtime["Image"] != IMAGES["core"]: + raise ValueError("deployed core image drift") + cfg = json.loads(command(["docker", "exec", runtime["Id"], "cat", "/runtime/core.json"])) + selected = [b for b in cfg["core"]["bindings"] if b["venue"] == "BINANCE" and b["market"] == "USDM" + and b["native_symbol"] == "BTCUSDT" and b["native_channel"].lower().endswith(("@aggtrade", "@trade"))] + if len(selected) != 1: + raise ValueError("expected one deployed BTCUSDT trade binding") + cfg["core"].update(bindings=selected, allow_test_provenance=True, + canonical_stream=namespace + ".canonical", quarantine_stream=namespace + ".quarantine") + cfg.update(raw_topics=[namespace + ".raw"], transactional_id=namespace, shard_id=namespace, + max_events=0, metrics_every_batches=10) + cfg["authority"].update(mode="RUST_SHADOW", slice_id=namespace, approved_by="isolated DR08 test") + (work / "core.json").write_text(json.dumps(cfg)) + packet = {"namespace": namespace, "development": development, "ts_source_sha": source_sha, + "ts_image_digest": image_digest, "images": IMAGES, "bootstrap": "localhost:9092"} + (work / "packet.json").write_text(json.dumps(packet)) + def launch(role, image, args, *, memory, cpus, network, env=None, detach=True, extra=()): + name = namespace + "-" + role + cmd = ["docker", "run", "--rm", "--name", name, "--label", "p184.dr08=" + namespace, + "--user", str(os.getuid()) + ":" + str(os.getgid()), "--cpus", str(cpus), "--memory", memory, + "--pids-limit", "128", "--network", network, "--read-only", + "--tmpfs", ("/tmp:rw,exec,size=64m,mode=1777" if role == "kafka" else "/tmp:rw,size=64m,mode=1777"), "--log-opt", "max-size=4m", + "--log-opt", "max-file=1", "-v", str(work) + ":/work", + "-v", str(root) + ":/app:ro", "-w", "/app"] + if detach: + cmd.insert(3, "-d") + for k, v in (env or {}).items(): + cmd += ["-e", k + "=" + v] + cmd += list(extra) + if detach: + # Redirect before exec: --rm can erase Docker logs before inspection. + args = ["/bin/sh", "-c", 'ulimit -f ' + ("65536" if role == "kafka" else "8192") + '; exec "$@" > /work/' + role + '-runtime.log 2>&1', + "dr08-log", *args] + cmd += ["--entrypoint", args[0], image, *args[1:]] + # Register before Docker: a failed or timed-out CLI can leave a live container. + owned.append((name, name)) + try: + return command(cmd, timeout=660 if role == "driver" else 100) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: + diagnostic = (getattr(error, "stderr", None) or "") + if isinstance(diagnostic, bytes): + diagnostic = diagnostic.decode(errors="replace") + (work / (role + "-command-error.log")).write_text(diagnostic[-16384:]) + raise + penv = {"PYTHONPATH": "/app:/app/generated/python", "PYTHONDONTWRITEBYTECODE": "1", + "P184_DR08_ISOLATED_NAMESPACE": namespace} + script = ["python", "-B", "scripts/p184_dr08_isolated.py"] + launch("prepare", IMAGES["python"], script + ["--prepare-material", "/work"], + memory="512m", cpus=.5, network="none", env=penv, detach=False) + broker = launch("kafka", IMAGES["kafka"], ["/bin/sh", "-c", + "/opt/kafka/bin/kafka-storage.sh format -t MkU3OEVBNTcwNTJENDM2Qk -c /work/broker.properties" + " && exec /opt/kafka/bin/kafka-server-start.sh /work/broker.properties"], + memory="1536m", cpus=1, network="none", env={"KAFKA_HEAP_OPTS": "-Xms256m -Xmx512m", "LOG_DIR": "/tmp/kafka-logs"}, + extra=("--hostname", "localhost", "--tmpfs", "/data:rw,size=512m,mode=1777")) + net = "container:" + broker + launch("topics", IMAGES["python"], script + ["--prepare-topics", "/work"], + memory="256m", cpus=.5, network=net, env=penv, detach=False) + packet = json.loads((work / "packet.json").read_text()) + launch("quota", IMAGES["redis"], ["redis-server", "--save", "", "--appendonly", "no", + "--maxmemory", "48mb", "--maxmemory-policy", "noeviction"], memory="64m", cpus=.1, network=net) + common = {"QDL_ENVIRONMENT": "paper", "QDL_KN_CURSOR_KEYS_FILE": "/work/cursor-keys.json", + "QDL_KN_CURSOR_ACTIVE_KEY_ID": "dr08", "QDL_KN_TOPIC_ID": packet["slice"]["topic_id"], + "QDL_KN_ROUTE_GENERATION": namespace, "QDL_KN_READ_VIEW_SECRET_FILE": "/work/read-view.secret"} + launch("core", IMAGES["core"], ["/usr/local/bin/qdl-realtime-core", "/work/core.json"], + memory="256m", cpus=.5, network=net, env={"QDL_KAFKA_CERT_ROOT": "/work", + "QDL_KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", "QDL_KAFKA_CLIENT_ID": namespace, + "QDL_KAFKA_GROUP_ID": namespace, "QDL_PROVIDER_ADMISSION_ENABLED": "false"}) + stream = launch("stream", IMAGES["stream"], ["/usr/local/bin/qdl-stream-gateway", "serve"], + memory="512m", cpus=.5, network=net, env={**common, + "QDL_KN_BUNDLE_FILE": "/work/bundle.json", "QDL_KN_JWT_CONFIG_FILE": "/work/jwt-config.json", + "QDL_KN_QUOTA_REDIS_URL": "redis://localhost:6379/0", "QDL_KN_QUOTA_PREFIX": "qdl:stable:v2:paper:" + namespace, + "QDL_KN_TLS_CERT_FILE": "/work/client.crt", "QDL_KN_TLS_KEY_FILE": "/work/client.key", + "QDL_KN_TLS_CLIENT_CA_FILE": "/work/ca.crt", "QDL_KN_KAFKA_BOOTSTRAP": "localhost:9092", + "QDL_KN_KAFKA_CA": "/work/ca.crt", "QDL_KN_KAFKA_CERT": "/work/client.crt", + "QDL_KN_KAFKA_KEY": "/work/client.key", "QDL_KN_KAFKA_TOPIC": cfg["core"]["canonical_stream"], + "QDL_KN_KAFKA_GROUP": "kn-" + namespace + "-stream", "QDL_KN_KAFKA_CLIENT_ID": namespace + "-stream", + "QDL_KN_READ_VIEW_URLS": "https://localhost:8200", "QDL_KN_READ_VIEW_CA_FILE": "/work/ca.crt", + "QDL_KN_HOT_READ_ENABLED": "true", "QDL_KN_METRICS_SECONDS": "1", + "QDL_KN_MAX_REPLAY_EVENTS": str(REPLAY), "QDL_KN_QUEUE_BYTES_TOTAL": str(QUEUE_BYTES)}) + launch("query", IMAGES["python"], script + ["--query", "/work"], memory="512m", cpus=.5, + network=net, env={**penv, **common}) + ready_deadline = time.monotonic() + 30 + while True: + core_state = json.loads(command(["docker", "inspect", namespace + "-core"]))[0] + if not core_state["State"]["Running"]: + raise RuntimeError("isolated core exited before driver") + core_log = work / "core-runtime.log" + core_ready = core_log.exists() and '"event":"qdl_realtime_core_started"' in core_log.read_text() + state = json.loads(command(["docker", "inspect", stream]))[0] + if not state["State"]["Running"]: + raise RuntimeError("isolated Stream exited before startup") + log = work / "stream-runtime.log" + text = log.read_text() if log.exists() else "" + if '"event":"qdl_kn_gateway_error"' in text: + raise RuntimeError("isolated Stream startup configuration refused; see stream-runtime.log") + if core_ready and '"event":"qdl_kn_gateway_start"' in text and '"event":"qdl_kn_gateway_metrics"' in text: + break + if time.monotonic() >= ready_deadline: + raise TimeoutError("isolated Stream startup deadline") + time.sleep(.1) + driver_args = script + ["--packet", "/work/packet.json", "--output", "/work/evidence.jsonl"] + if development: + driver_args += ["--development"] + launch("driver", IMAGES["python"], driver_args, memory="512m", cpus=.75, network=net, env=penv, detach=False) + finally: + if log_process is not None: + log_process.terminate() + log_process.wait(timeout=5) + if log_file is not None: + log_file.close() + cleanup_failures = [] + for name, cid in reversed(owned): + try: + row = json.loads(command(["docker", "inspect", cid]))[0] + if row["Config"]["Labels"].get("p184.dr08") != namespace: + raise ValueError("cleanup ownership mismatch") + logs = subprocess.run(["docker", "logs", "--tail", "100", row["Id"]], + text=True, capture_output=True, timeout=10) + (work / (name + "-tail.log")).write_text((logs.stdout + logs.stderr)[-32768:]) + command(["docker", "stop", "--time", "5", row["Id"]], timeout=20) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + if command(["docker", "ps", "-aq", "--filter", "name=^/" + name + "$"]): + cleanup_failures.append(name) + for filename in ("client.key", "server.p12", "slow.key", "fast.key", "cursor-keys.json", + "read-view.secret", "broker.properties"): + (work / filename).unlink(missing_ok=True) + after = inventory() + unchanged = all(after.get(cid) == state for cid, state in before.items()) + (work / "cleanup.json").write_text(json.dumps({"shared_runtime_unchanged": unchanged, + "cleanup_failures": cleanup_failures, "owned_containers": [name for name, _ in owned], + "disk_free_before": disk_before, "disk_free_after": shutil.disk_usage(work).free, + "images_built": 0, "images_removed": 0})) + if cleanup_failures or not unchanged: + raise RuntimeError("cleanup or shared runtime verification failed") + + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--dependency-smoke", action="store_true") + parser.add_argument("--orchestrate", type=Path) + parser.add_argument("--prepare-material", type=Path) + parser.add_argument("--prepare-topics", type=Path) + parser.add_argument("--query", type=Path) + parser.add_argument("--ts-source-sha", default="0" * 40) + parser.add_argument("--ts-image-digest", default="sha256:" + "0" * 64) + parser.add_argument("--packet", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--development", action="store_true") + args = parser.parse_args(argv) + if args.orchestrate: + def orchestration_deadline(_signum, _frame): + raise TimeoutError("whole DR08 orchestration deadline") + previous = signal.signal(signal.SIGALRM, orchestration_deadline) + signal.alarm(600) + try: + orchestrate(args.orchestrate, development=args.development, source_sha=args.ts_source_sha, image_digest=args.ts_image_digest) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, previous) + return 0 + for directory, function in ((args.prepare_material, prepare_material), (args.prepare_topics, prepare_topics), (args.query, serve_query)): + if directory is not None: + function(directory) + return 0 + if args.dependency_smoke: + print(json.dumps(dependency_smoke(), sort_keys=True)) + return 0 + if args.packet is None or args.output is None: + parser.error("--packet and --output are required") + packet = json.loads(args.packet.read_text()) + validate_packet(packet, development=args.development) + # Exclusive creation happens BEFORE any traffic. No resume/relabel operation. + with args.output.open("x", encoding="utf-8") as output: + run_id = str(uuid.uuid4()) + def deadline(_signum, _frame): + raise TimeoutError("DR08 ten-minute deadline") + previous = signal.signal(signal.SIGALRM, deadline) + signal.alarm(600) + try: + facts = run_faults(packet) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, previous) + for row in evidence_rows(packet, facts, development=args.development, run_id=run_id): + output.write(json.dumps(row, sort_keys=True) + "\n") + output.flush() + os.fsync(output.fileno()) + return 0 if all(facts.get(cell, {}).get("passed") is True for cell in CELLS) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_p184_dr01_runtime.py b/tests/test_p184_dr01_runtime.py index 3f9f2f9f..b6f1dfdd 100644 --- a/tests/test_p184_dr01_runtime.py +++ b/tests/test_p184_dr01_runtime.py @@ -347,6 +347,10 @@ def test_real_application_ephemeral_credentials(self): self.assertTrue(dr.denied(obs[venue]["consumer"], 403, "CONSUMER_MISMATCH")) self.assertTrue(dr.denied(obs[venue]["principal"], 401, "UNAUTHENTICATED")) self.assertIs(obs[venue]["baseline"]["identity_matches"], True) + self.assertGreater(obs[venue]["binding"]["source_roster_venue_products"], 1) + self.assertGreaterEqual(obs[venue]["binding"]["source_roster_products"], + obs[venue]["binding"]["source_roster_venue_products"]) + self.assertEqual(len(obs[venue]["binding"]["source_roster_binding_sha256"]), 64) if __name__ == "__main__": diff --git a/tests/test_p184_dr08_isolated.py b/tests/test_p184_dr08_isolated.py new file mode 100644 index 00000000..811d0b62 --- /dev/null +++ b/tests/test_p184_dr08_isolated.py @@ -0,0 +1,264 @@ +"""Offline runner guards, not runtime fault qualification.""" +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("dr08", ROOT / "scripts/p184_dr08_isolated.py") +R = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(R) + + +def packet(development=True): + return {"namespace": "p184-dr08-test-12345678", "development": development, + "bootstrap": "localhost:9092", "slice": {"target": "localhost:8210"}, + "consumer_id": "slow", "fast_consumer_id": "fast", "raw_topic": "test-raw", + "metrics_path": "/isolated/metrics", "ts_source_sha": "1234567890" * 4, + "ts_image_digest": "sha256:" + "12345678" * 8, + "images": {role: "sha256:" + "abcdef01" * 8 for role in ("core", "stream", "kafka", "python")}} + + +class Guards(unittest.TestCase): + def setUp(self): + self.p = packet() + env = patch.dict(os.environ, {"P184_DR08_ISOLATED_NAMESPACE": self.p["namespace"]}) + env.start() + self.addCleanup(env.stop) + + def test_valid_modes(self): + R.validate_packet(self.p, development=True) + R.validate_packet(packet(False), development=False) + + def test_shared_broker_refused(self): + for host in ("kafka1:9092", "localhost:9092,shared:9092", "127.0.0.1:9092"): + with self.subTest(host=host): + self.p["bootstrap"] = host + with self.assertRaises(ValueError): + R.validate_packet(self.p, development=True) + + def test_shared_stream_refused(self): + self.p["slice"]["target"] = "qdl-v2-stream-a:8210" + with self.assertRaises(ValueError): + R.validate_packet(self.p, development=True) + + def test_parent_attestation_required(self): + with patch.dict(os.environ, {"P184_DR08_ISOLATED_NAMESPACE": ""}): + with self.assertRaises(ValueError): + R.validate_packet(self.p, development=True) + + def test_distinct_consumers(self): + self.p["fast_consumer_id"] = self.p["consumer_id"] + with self.assertRaises(ValueError): + R.validate_packet(self.p, development=True) + + def test_immutable_images(self): + for role in self.p["images"]: + p = packet() + p["images"][role] = "latest" + with self.subTest(role=role), self.assertRaises(ValueError): + R.validate_packet(p, development=True) + + def test_development_cannot_be_final(self): + with self.assertRaisesRegex(ValueError, "fresh packet"): + R.validate_packet(self.p, development=False) + + def test_dummy_final_provenance_refused(self): + for field, value in (("ts_source_sha", "dummy"), ("ts_source_sha", "0" * 40), + ("ts_image_digest", "sha256:" + "a" * 64)): + p = packet(False) + p[field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + R.validate_packet(p, development=False) + + +class Evidence(unittest.TestCase): + def test_exact_cells(self): + self.assertEqual(set(R.CELLS), { + "DR08.SLOW_CONSUMER_BOUNDED_QUEUE@ANY", + "DR08.CONSUMER_RESTART_RECOVERY_MEASURED@ANY", + "DR08.NO_CROSS_CONSUMER_STARVATION@ANY", + "DR08.ALERT_PATH_OBSERVABLE@ANY"}) + + def test_missing_facts_fail_closed(self): + rows = R.evidence_rows(packet(False), {}, development=False, run_id="new") + self.assertEqual(rows[0]["provenance"]["schema"], "p184.surface-evidence-provenance.v1") + self.assertEqual(len(rows), 5) + self.assertTrue(all(not row["ok"] and row["outcome"] == "NOT_OBSERVED" for row in rows[1:])) + + def test_relabelling_header_cannot_qualify_development(self): + facts = {cell: {"observed": True, "passed": True} for cell in R.CELLS} + rows = R.evidence_rows(packet(), facts, development=True, run_id="dev") + rows[0] = R.evidence_rows(packet(False), {}, development=False, run_id="final")[0] + self.assertTrue(all(not row["ok"] and row["outcome"] == "NOT_OBSERVED" for row in rows[1:])) + self.assertTrue(all(row["development_pass"] and row["mode"] == "DEVELOPMENT" for row in rows[1:])) + + def test_observation_and_pass_required(self): + for fact, expected in (({"observed": True, "passed": True}, True), + ({"observed": False, "passed": True}, False), + ({"observed": True, "passed": False}, False), + ({"observed": 1, "passed": True}, False)): + rows = R.evidence_rows(packet(False), {R.CELLS[0]: fact}, development=False, run_id="new") + self.assertIs(rows[1]["ok"], expected) + + def test_alert_deltas_and_bounds(self): + before = dict(overflow=2, expired=3, queue_bytes=0, rss_bytes=100, cpu_seconds=1) + after = dict(overflow=3, expired=4, queue_bytes=100, rss_bytes=200, cpu_seconds=2) + self.assertTrue(R.metric_verdict(before, after)) + for key, value in (("overflow", 2), ("expired", 3), ("queue_bytes", R.QUEUE_BYTES + 1), ("cpu_seconds", 0)): + self.assertFalse(R.metric_verdict(before, {**after, key: value})) + self.assertFalse(R.metric_verdict({}, after)) + + def test_partial_metrics_and_no_samples(self): + with tempfile.TemporaryDirectory() as d: + path = Path(d) / "metrics" + path.write_text('startup\n{"event":"qdl_kn_gateway_metrics","overflow":1}\n{') + self.assertEqual(len(R.read_metrics(path, since_ns=1)), 1) + path.write_text("no samples") + with self.assertRaises(ValueError): + R.read_metrics(path, since_ns=1) + + def test_metrics_size_bound(self): + with tempfile.TemporaryDirectory() as d: + path = Path(d) / "metrics" + with path.open("wb") as f: + f.truncate(R.MAX_METRICS_BYTES + 1) + with self.assertRaisesRegex(ValueError, "8MiB"): + R.read_metrics(path, since_ns=1) + + def test_startup_failure_is_not_observed(self): + with patch.object(R, "wait_metrics", side_effect=ValueError("missing")): + facts = R.run_faults(packet()) + self.assertEqual(set(facts), set(R.CELLS)) + self.assertTrue(all(not f["observed"] and not f["passed"] for f in facts.values())) + + def test_output_exclusive_before_traffic(self): + with tempfile.TemporaryDirectory() as d: + config, output = Path(d) / "packet", Path(d) / "evidence" + p = packet() + config.write_text(json.dumps(p)) + output.write_text("existing evidence") + with patch.dict(os.environ, {"P184_DR08_ISOLATED_NAMESPACE": p["namespace"]}), patch.object(R, "run_faults") as run: + with self.assertRaises(FileExistsError): + R.main(["--packet", str(config), "--output", str(output), "--development"]) + run.assert_not_called() + self.assertEqual(output.read_text(), "existing evidence") + + def test_fresh_run_writes_header_and_four_rows(self): + with tempfile.TemporaryDirectory() as d: + config, output = Path(d) / "packet", Path(d) / "evidence" + p = packet() + config.write_text(json.dumps(p)) + facts = {cell: {"observed": True, "passed": True} for cell in R.CELLS} + with patch.dict(os.environ, {"P184_DR08_ISOLATED_NAMESPACE": p["namespace"]}), patch.object(R, "run_faults", return_value=facts): + self.assertEqual(R.main(["--packet", str(config), "--output", str(output), "--development"]), 0) + rows = [json.loads(line) for line in output.read_text().splitlines()] + self.assertEqual(len(rows), 5) + self.assertEqual(rows[0]["provenance"]["data_provenance"], "SYNTHETIC_TEST_ONLY") + self.assertTrue(all(row["ok"] is False for row in rows[1:])) + + def test_metrics_baseline_wait_is_bounded(self): + with patch.object(R, "read_metrics", side_effect=FileNotFoundError), \ + patch.object(R.time, "monotonic", side_effect=[0, 21]): + with self.assertRaises(TimeoutError): + R.wait_metrics("absent", since_ns=1, timeout=20) + + def test_metrics_baseline_retries_empty_capture(self): + row = {"event": "qdl_kn_gateway_metrics"} + with patch.object(R, "read_metrics", side_effect=[ValueError("empty"), [row]]), \ + patch.object(R.time, "sleep") as sleep: + self.assertEqual(R.wait_metrics("capture", since_ns=1), [row]) + sleep.assert_called_once_with(.1) + + def test_tls_producer_settings(self): + p = packet() + p["client_tls"] = {"ca": "/work/ca.crt", "cert": "/work/client.crt", "key": "/work/client.key"} + settings = R.kafka_security(p) + self.assertEqual(settings["security.protocol"], "SSL") + self.assertEqual(settings["ssl.key.location"], "/work/client.key") + self.assertEqual(settings["bootstrap.servers"], "localhost:9092") + + def test_invalid_provenance_rejected_before_setup(self): + with tempfile.TemporaryDirectory() as d: + output = Path(d) / "absent" + with self.assertRaises(ValueError): + R.orchestrate(output, development=True, source_sha="DEVELOPMENT", image_digest="DEVELOPMENT") + self.assertFalse(output.exists()) + + + def test_queue_observation_is_measured_not_configured(self): + with tempfile.TemporaryDirectory() as d: + path = Path(d) / "metrics" + row = {"event": "qdl_kn_subscription_closed", "consumer_id": "slow", + "outcome": "backpressure", "subscription_id": 1, "queued": 1000, "queued_bytes": 10000} + path.write_text(json.dumps(row)) + self.assertEqual(R.slow_queue_observation(path, "slow", subscription_id=1, wire_code="RESOURCE_EXHAUSTED")["queued"], 1000) + self.assertIsNone(R.slow_queue_observation(path, "other", subscription_id=1, wire_code="RESOURCE_EXHAUSTED")) + row["queued"] = 999 + path.write_text(json.dumps(row)) + self.assertIsNone(R.slow_queue_observation(path, "slow", subscription_id=1, wire_code="RESOURCE_EXHAUSTED")) + + + def test_actual_rust_close_description_requires_wire_and_subscription(self): + with tempfile.TemporaryDirectory() as d: + path = Path(d) / "exclusive-stream.log" + row = {"event": "qdl_kn_subscription_closed", "consumer_id": "isolated.slow", + "outcome": "Some resource has been exhausted", "subscription_id": 1, + "queued": 1000, "queued_bytes": 4719110} + path.write_text(json.dumps(row)) + self.assertEqual(R.slow_queue_observation(path, "isolated.slow", subscription_id=1, + wire_code="RESOURCE_EXHAUSTED")["queued_bytes"], 4719110) + for consumer, subscription, code in (("other.slow", 1, "RESOURCE_EXHAUSTED"), + ("isolated.slow", 2, "RESOURCE_EXHAUSTED"), ("isolated.slow", 1, "OK")): + self.assertIsNone(R.slow_queue_observation(path, consumer, + subscription_id=subscription, wire_code=code)) + row["outcome"] = "ended" + path.write_text(json.dumps(row)) + self.assertIsNone(R.slow_queue_observation(path, "isolated.slow", subscription_id=1, + wire_code="RESOURCE_EXHAUSTED")) + + def test_foreground_timeout_stops_pre_registered_owned_container(self): + import subprocess + from contextlib import redirect_stdout + import io + calls, owned = [], {} + cfg = {"core": {"bindings": [{"venue": "BINANCE", "market": "USDM", + "native_symbol": "BTCUSDT", "native_channel": "btcusdt@trade"}]}, "authority": {}} + + def run(args, **kwargs): + calls.append(args) + out = "" + if args[:3] == ["docker", "image", "inspect"]: + out = args[3] + elif args[:3] == ["docker", "inspect", "qdl_v2_stable_candidate-rust_core-1"]: + out = json.dumps([{"Image": R.IMAGES["core"], "Id": "shared-core"}]) + elif args[:3] == ["docker", "exec", "shared-core"]: + out = json.dumps(cfg) + elif args[:2] == ["docker", "run"]: + name = args[args.index("--name") + 1] + namespace = args[args.index("--label") + 1].split("=", 1)[1] + owned.update(name=name, namespace=namespace) + raise subprocess.TimeoutExpired(args, 100) + elif args[:2] == ["docker", "inspect"]: + self.assertEqual(args[2], owned["name"]) + out = json.dumps([{"Id": "owned-immutable-id", "Config": { + "Labels": {"p184.dr08": owned["namespace"]}}}]) + return subprocess.CompletedProcess(args, 0, stdout=out, stderr="") + + with tempfile.TemporaryDirectory() as d, patch("subprocess.run", side_effect=run), redirect_stdout(io.StringIO()): + target = Path(d) / "attempt" + with self.assertRaises(subprocess.TimeoutExpired): + R.orchestrate(target, development=True, source_sha="0" * 40, image_digest="sha256:" + "0" * 64) + cleanup = json.loads((target / "cleanup.json").read_text()) + self.assertEqual(cleanup["cleanup_failures"], []) + self.assertTrue(cleanup["shared_runtime_unchanged"]) + self.assertIn(["docker", "stop", "--time", "5", "owned-immutable-id"], calls) + self.assertFalse(any(a[:2] == ["docker", "stop"] and "shared-core" in a for a in calls)) + + + +if __name__ == "__main__": + unittest.main() diff --git a/upgrade/evidence/p184-dr-runtime-runbook.md b/upgrade/evidence/p184-dr-runtime-runbook.md new file mode 100644 index 00000000..d7a6fc17 --- /dev/null +++ b/upgrade/evidence/p184-dr-runtime-runbook.md @@ -0,0 +1,95 @@ +# P18.4 DR Runtime Runner Handoff + +Status: DEVELOPMENT_RUNNERS / RELEASE_HOLD. Follow the current journal for +actual results; this runbook is not a receipt or a shared-runtime packet. + +## Provenance + +`scripts/p184_dr01_runtime.py` and `scripts/p184_dr08_isolated.py` write JSONL +with the TS `p184.surface-evidence-provenance.v1` header and exact case IDs. +The final TS source SHA/image must come from Claude's final build. Development +uses forty zeroes and `sha256:` plus sixty-four zeroes. Never relabel a rehearsal +or include its failed/pending rows in a final acceptance bundle. Keep its hash +as historical evidence and run again in a fresh output directory. + +Before final acceptance, bind actual DL images, TS image, configuration and +test source hashes into the enclosing packet. A caller-supplied header alone +does not attest code running inside an image. No runner grants mainnet/order +authority or permits changing deployed readers/consumers. + +## DR01 + +Use the existing Python image with dependencies, source read-only, a fresh +evidence mount, disposable credentials, bounded CPU/RAM and `--rm`. Initial +DL-only rehearsal uses `--network none`; the HTTP server binds loopback only. + +```text +python -B scripts/p184_dr01_runtime.py --development \ + --source-sha <40-zeroes> --image-digest sha256:<64-zeroes> \ + --output /evidence/dr01.rehearsal.jsonl +``` + +The runner validates the complete source binding artifact and records its hash +and per-venue product counts before deriving disposable identities. The initial +fixture compiles one BAR-1m ALPHA projection per venue and exercises +real HTTP authentication plus metadata access. It is not all-r22 data access, +production mTLS, or a claim that TS consumed the sealed binding. Wrong account +cannot be inferred from wrong consumer/subject. Without an account packet, +those two cells explicitly remain NOT_OBSERVED and the runner exits nonzero. + +For account checks, run an isolated final-image TS Gateway and isolated +PostgreSQL/Redis. Reuse TS's existing `test_p18e_current_gateway_http.py` setup +contract; do not modify the shared Gateway or write into shared accounts. +Create two active test alphas with separate generated API keys, each with a +distinct Paper account per venue. Hash keys through TS's existing helper. +Include all four accounts in `WRITER_ACCOUNT_SCOPE_JSON`, enable API-key auth, +disable plaintext migration, and seed distinguishable test-only balances. + +Use `--account-packet /private/account-packet.json`; the exact schema is in +`load_account_packet()`. It binds source/image, literal loopback Gateway URL, +two venue pairs, account/alpha IDs, expected decimal balances, and external +mode-0600 key files. Run the driver in the Gateway's isolated network namespace, +never host networking. Keep fixture infrastructure within the same isolated +namespace and no broker credentials. No shared ports/networks/mounts. + +Per venue the probe requires A->A and B->B nonempty exact balances, A->B on the +same venue returning `403 ACCOUNT_ALPHA_SCOPE_MISMATCH`, and a separate invalid +key refusal. GET-only, no redirects, no order endpoints. This proves application +account authorization, not broker-side permissions. Auth/rate counters may change +only inside disposable Redis. Inspect financial/command tables before/after and +retire credentials with the fixture. Merely setting `GATEWAY_NO_ORDER_ONLY` is +not a security boundary; isolated topology and the GET-only driver are required. + +## DR08 + +The runner's `--orchestrate ` mode owns its temporary stack. +It uses immutable existing images, no image build, no published ports and a +broker network namespace without an external interface. Other test roles share +only that namespace. Public core configuration is copied read-only, restricted +to the isolated test binding; production configuration is not rewritten. + +```text +python3 scripts/p184_dr08_isolated.py --orchestrate \ + --development --ts-source-sha <40-zeroes> \ + --ts-image-digest sha256:<64-zeroes> +``` + +The synthetic input is explicitly test-only. Test Query read-view support is +needed for the actual GetSnapshot RPC, not a locally invented recovery cursor. +Queue1000 and replay10000 remain fixed. Two distinct consumers must demonstrate +slow-consumer overflow, fast-consumer progress, observable control/status/counters, +and recovery via a newer snapshot after OUT_OF_RANGE. Separate configured limits +from measured high-water values; distinguish gRPC receive from TS Redis ACK. +This experiment does not certify production freshness, independent HA or provider +latency. Development rows cannot qualify the final TS matrix. + +Preserve failed attempts. Verify `cleanup.json`, no remaining owned container, +no private keys, unchanged shared image/start/restart inventory and disk pre/post. +Keep active images and bar-edge rollback `f7351c3b`; no broad prune. Final TS +provenance requires a fresh run, not editing the development JSONL. + +## Release Boundary + +v2.2.3 remains HOLD for final DR01-DR08/broker evidence, independent of building +Paper features. [TRADE attribution and proposals](p184-dr01-dr08-trade-attribution.md) +do not authorize LATEST_STATE, a batch-wait change, rollout, push or merge. diff --git a/upgrade/evidence/p184-dr07-runtime-closure.md b/upgrade/evidence/p184-dr07-runtime-closure.md index 6d91936b..24ca68f1 100644 --- a/upgrade/evidence/p184-dr07-runtime-closure.md +++ b/upgrade/evidence/p184-dr07-runtime-closure.md @@ -377,3 +377,12 @@ needed for this gate; it raises snapshots x1.47 and should not ride the release Open: DR01-DR08 + broker-delta case receipts (TS runners; DR07 runner TS 74852a8), deliberate reset/fault proof on the writer (Codex reset-anchor now merged into the TS feature branch, dfc70bd), a busy-window CPU gate, and a DL release record that names the new bar edge image. No push/merge/release. + + +## DR01/DR08 runner handoff - 2026-10-08 + +See [runner runbook](p184-dr-runtime-runbook.md) and the main-plan DR08 receipt. +DR01 has7/9 development HTTP cells, account authorization still needs isolated +final-image Gateway; DR08 has4/4 development checks with preserved raw receipt +and parser supplement. Neither is final TS provenance-bound acceptance. +Runtime remains unchanged, v2.2.3 HOLD; retain f7351c3b until approved release. From b6a01190587e8771c62b5b4988782b06c5e2578a Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 08:30:12 +0000 Subject: [PATCH 027/147] fix(stream): report per-subscription offered and dequeued counters --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 43 ++++++++ rust/qdl-stream-gateway/src/service.rs | 2 + rust/qdl-stream-gateway/src/subscription.rs | 103 ++++++++++++++++++ .../p184-dr01-dr08-trade-attribution.md | 30 ++++- 4 files changed, 175 insertions(+), 3 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 4c396d93..90a87331 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -69977,3 +69977,46 @@ manifest/JWT10 and bindingV2r22 unchanged. No TS source edits, rollout, push/mer v2.2.3 HOLD for final DR/broker receipts. LATEST_STATE and batch25->50 proposals were neither implemented nor measured. Next permitted work: final-image isolated account fixture plus fresh provenance-bound DR01/DR08, then owner packet review. + + +### DR owner correction and runtime-runner scope - 2026-10-08 (Codex) + +Status IMPLEMENTING / RELEASE_HOLD. Owner supersedes prior DR01 handoff: +existing local application runner is CONTROLLED_SIM only, never closes DR01. +New stdin runner targets deployed query_kn using existing alpha r22/JWT14/mTLS, +GET-only, baseline and exact negative realm/revision/entitlement checks plus +compiled binding hash comparison. Do not execute it on shared runtime here. +WRONG_ACCOUNT_DENIED belongs to Claude's TS Gateway, not DL. DR08 must name +Query2.2.3-rc2-43bbcf0 explicitly and final recovery needs>=5 restart samples +with p50/max. Add per-subscription offered/dequeued closure counters only; no +queue, contract, freshness, quota or topology change. Add owner-supplied03:21 +worker correlation, distinguish correlation from exact transport attribution. +LATEST_STATE and batch25->50 proposals are withdrawn, not implemented. +TS candidate2abbe394ea0fb8f7f5ebcbb745ff70a8c83c9769 / +sha256:a6548ee36aeb0242c1734c071678c525c1506d05c10a70b533eb7110e7eaf16a +is provisional until Claude confirms CI green; do not certify using it yet. +Tests: isolated source/unit/protocol only; no shared runtime, credentials or +consumer traffic. Rollback is source-only; cleanup own test resources, retain +f7351c3b. Each coherent slice gets tested commit+journal; no push/merge. +Guide: upgrade/evidence/p184-dr-runtime-runbook.md. + + +### DR Stream closure counter slice - 2026-10-08 (Codex) + +Status SOURCE_TESTED / NOT_DEPLOYED / RELEASE_HOLD. Owner-approved additive +closure JSON now includes subscription-local offered/dequeued. Offered counts +eligible live enqueue attempts including overflow; dequeued counts actual live +queue pops before read-time filtering/transport, excludes replay/discard/coalesce. +No new queue, TTL, delivery mode, quota or runtime behavior. Neither is client ACK. +Regression covers duplicate/wrong product/stale, capacity/byte overflow, failure, +close discard, independent subscriptions and coalescing without false dequeues. +Existing offline builder/cache command: cargo test --offline --locked +-p qdl-stream-gateway --lib --test native_stream -- --test-threads=2: +74PASS0fail0ignored. Two bounded closure-log checks also passed, including two +subscriptions each emitting offered1/dequeued1. No image build or shared rollout. +Disposable builder containers removed; shared caches retained for concurrent work. +Report upgrade/evidence/p184-dr01-dr08-trade-attribution.md now records owner +worker1 correlation, exact6.088298ms closure separation, no other family overflow +in supplied window, and withdrawal of LATEST_STATE/batch25->50. Exact stall stage +remains unmeasured; owner TS-side working attribution is not invented telemetry. +No push/merge; keep bar-edge rollbackf7351c3b. Runtime unchanged. diff --git a/rust/qdl-stream-gateway/src/service.rs b/rust/qdl-stream-gateway/src/service.rs index ba699eda..16b0d934 100644 --- a/rust/qdl-stream-gateway/src/service.rs +++ b/rust/qdl-stream-gateway/src/service.rs @@ -962,6 +962,8 @@ impl Drop for SubscriptionTask { "replayed": load(&self.report.replayed), "delivered": load(&self.report.delivered), "aged_out_at_read": load(&self.report.aged_out), + "offered": load(&counters.offered), + "dequeued": load(&counters.dequeued), "rejected_at_push": load(&counters.rejected_at_push), "coalesced": load(&counters.coalesced), "queued": queued, diff --git a/rust/qdl-stream-gateway/src/subscription.rs b/rust/qdl-stream-gateway/src/subscription.rs index de2ae8ea..79901394 100644 --- a/rust/qdl-stream-gateway/src/subscription.rs +++ b/rust/qdl-stream-gateway/src/subscription.rs @@ -118,7 +118,10 @@ pub struct SubscriptionCounters { pub other_product: AtomicU64, pub rejected_at_push: AtomicU64, pub coalesced: AtomicU64, + /// Eligible live enqueue attempts, including the attempt that overflows. pub offered: AtomicU64, + /// Live queue pops, before read-time filtering or transport handoff; not ACKs. + pub dequeued: AtomicU64, } #[derive(Debug)] @@ -287,6 +290,7 @@ impl Subscription { } if let Some(item) = state.queue.pop_front() { state.bytes -= item.weight; + self.counters.dequeued.fetch_add(1, Ordering::Relaxed); return Next::Record(item.record, item.enqueued, item.charge); } if state.closed { @@ -384,6 +388,93 @@ mod tests { offsets } + fn counts(subscription: &Subscription) -> (u64, u64) { + ( + subscription.counters.offered.load(Ordering::Relaxed), + subscription.counters.dequeued.load(Ordering::Relaxed), + ) + } + + #[tokio::test] + async fn counters_count_eligible_attempts_and_local_pops_only() { + let mut subscription = queue(query::FeedType::Trade, 2); + subscription.requirement.delivery.max_freshness_ms = Some(1); + let trade = |offset| { + record( + offset, + event_envelope::Payload::Trade(Trade::default()), + vec![], + ) + }; + assert_eq!(counts(&subscription), (0, 0)); + subscription.offer(&trade(0), 0); // At cursor floor. + subscription.offer("e(1, vec![]), 0); // Other product. + subscription.offer(&trade(1), 2_000_000); // Too old at push. + assert_eq!(counts(&subscription), (0, 0)); + assert_eq!( + subscription.counters.other_product.load(Ordering::Relaxed), + 1 + ); + assert_eq!( + subscription + .counters + .rejected_at_push + .load(Ordering::Relaxed), + 1 + ); + subscription.offer(&trade(1), 0); + subscription.offer(&trade(1), 0); // Duplicate. + assert_eq!(counts(&subscription), (1, 0)); + let charge = match subscription.next().await { + Next::Record(record, _, charge) => { + // This record is already counted even if a later read rejects it. + assert_eq!( + delivery_decision(&subscription.requirement, &record.envelope, 2_000_000), + Delivery::TooOld + ); + charge + } + other => panic!("{other:?}"), + }; + assert_eq!(counts(&subscription), (1, 1)); + assert!( + subscription.budget.used() > 0, + "handoff still owns its bytes" + ); + drop(charge); + assert_eq!(subscription.budget.used(), 0); + subscription.offer(&trade(2), 0); + subscription.close(); // Discard is not a dequeue. + subscription.offer(&trade(3), 0); + assert!(matches!(subscription.next().await, Next::Closed)); + assert_eq!(counts(&subscription), (2, 1)); + } + + #[tokio::test] + async fn failure_and_budget_overflow_do_not_count_as_dequeues() { + let failed = queue(query::FeedType::Trade, 2); + let trade = record(1, event_envelope::Payload::Trade(Trade::default()), vec![]); + failed.offer(&trade, 0); + failed.fail("test-only failure"); + failed.offer( + &record(2, event_envelope::Payload::Trade(Trade::default()), vec![]), + 0, + ); + assert!(matches!(failed.next().await, Next::Failed(_))); + assert_eq!(counts(&failed), (1, 0)); + failed.close(); + assert_eq!(failed.budget.used(), 0); + + let mut overflow = queue(query::FeedType::Trade, 2); + overflow.budget = ByteBudget::new(0); + overflow.offer(&trade, 0); + assert!(matches!(overflow.next().await, Next::Overflow)); + assert_eq!(counts(&overflow), (1, 0)); + assert_eq!(overflow.queued(), (0, 0)); + assert_eq!(overflow.budget.used(), 0); + assert_eq!(counts(&failed), (1, 0), "counters are subscription-local"); + } + #[tokio::test] async fn a_latest_state_queue_keeps_the_newest_and_every_transition() { let budget = ByteBudget::new(1 << 20); @@ -414,6 +505,11 @@ mod tests { ); assert!(delivered.windows(2).all(|pair| pair[1] > pair[0])); assert_eq!(budget.used(), 0, "every byte released after draining"); + assert_eq!(counts(&subscription), (20, delivered.len() as u64)); + assert_eq!( + subscription.counters.coalesced.load(Ordering::Relaxed), + 20 - delivered.len() as u64 + ); } #[tokio::test] @@ -440,7 +536,14 @@ mod tests { } assert!(matches!(subscription.next().await, Next::Overflow)); assert_eq!(subscription.counters.coalesced.load(Ordering::Relaxed), 0); + assert_eq!(counts(&subscription), (5, 0)); + subscription.offer( + &record(6, event_envelope::Payload::Trade(Trade::default()), vec![]), + 0, + ); + assert_eq!(counts(&subscription), (5, 0)); subscription.close(); + assert_eq!(counts(&subscription), (5, 0)); assert_eq!(budget.used(), 0); } diff --git a/upgrade/evidence/p184-dr01-dr08-trade-attribution.md b/upgrade/evidence/p184-dr01-dr08-trade-attribution.md index 5f04d727..f4e54107 100644 --- a/upgrade/evidence/p184-dr01-dr08-trade-attribution.md +++ b/upgrade/evidence/p184-dr01-dr08-trade-attribution.md @@ -1,7 +1,8 @@ # P18.4 DR01/DR08 TRADE Incident Attribution Date: 2026-10-08. Status: READONLY_ATTRIBUTION_COMPLETE / -INCIDENT_WINDOW_RECOVERY_NOT_MEASURED / PROPOSALS_NOT_EXECUTED. +INCIDENT_WINDOW_RECOVERY_NOT_MEASURED / PROPOSALS_WITHDRAWN. +Owner update below supersedes the original proposal text. This report is not a DR01-DR08 acceptance certificate. No implementation source change, runtime mutation, new measurement, test execution or cleanup was performed here. @@ -149,7 +150,7 @@ retry, client-applied throughput and immediate recovery time are not proved by the incident window. The later IDR-078 sample supports eventual recovery as described above, without resolving precise stall attribution. -## Proposal: Per-Subscription Delivery Mode +## Withdrawn Proposal: Per-Subscription Delivery Mode NOT IMPLEMENTED OR AUTHORIZED FOR RUNTIME. Add explicit per-subscription TRADE delivery mode: omitted/default remains `LOSSLESS`; TRADE `LATEST_STATE` requires @@ -175,7 +176,7 @@ Instrument per-subscription offered/accepted/dequeued/local-enqueued/coalesced/ rejected counters with monotonic intervals and client received/applied progress. Keep replay separate; capture queue/byte high-water and wait distributions. -## Proposal: Paired batch_wait_ms 25 -> 50 +## Withdrawn Proposal: Paired batch_wait_ms 25 -> 50 NOT RUN. Owner verified running Entrypoint: `/usr/local/bin/qdl-realtime-core /runtime/core.json`. @@ -234,3 +235,26 @@ runtime resources were created. DR acceptance, incident-window RTO, broker and release gates remain open; eventual recovery evidence is distinguished above. Future instrumentation, contract changes or an exact isolated measurement packet require review before execution. + + +## Owner Follow-Up And Additive Counters (2026-10-08) + +Owner/Claude reports worker1 owns both BTC Binance and BTC OKX TRADE streams. +The retained closure timestamps above differ by6.088298ms; other feed families +did not overflow in the supplied incident window. This common worker and +TRADE-only failure are consistent with a TS-side bottleneck and justify the +owner decision to pursue TS recovery rather than alter TRADE semantics or core +batching. Precise stall stage (TS processing/fsync versus gRPC backpressure) +is not measured by those closure records; IDR-078 likewise retains that limit. +Worker ownership is owner-supplied correlation, not reconstructed offered rates. +LATEST_STATE and batch_wait_ms25->50 are withdrawn, not queued work. Historical +proposal text above is retained for audit and does not authorize implementation. + +Additive source patch now emits subscription-local offered/dequeued in closure +JSON. offered counts eligible live enqueue attempts including the overflow +attempt, after duplicate/product/stale filtering. dequeued counts successful +queue pops before read-time filtering/local handoff; excludes replay, coalescing +and close-time discard. Neither is remote receipt or durable ACK. Existing +delivered/replayed/queued/quality counters remain unchanged. Lifetime counts +still require time bounds before reporting rates. Patch is source-tested, NOT +deployed; it cannot retroactively fill the03:21 missing counters. From 233bd522483729c06659cc3c989058fc44c0fb5d Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 08:31:34 +0000 Subject: [PATCH 028/147] test(p184): bind DR08 Query provenance and require repeated restarts --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 16 ++ scripts/p184_dr08_isolated.py | 137 +++++++++++++--- tests/test_p184_dr08_isolated.py | 185 +++++++++++++++++++++- 3 files changed, 316 insertions(+), 22 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 90a87331..2ed66d09 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70020,3 +70020,19 @@ worker1 correlation, exact6.088298ms closure separation, no other family overflo in supplied window, and withdrawal of LATEST_STATE/batch25->50. Exact stall stage remains unmeasured; owner TS-side working attribution is not invented telemetry. No push/merge; keep bar-edge rollbackf7351c3b. Runtime unchanged. + + +### DR08 provenance and repeated-restart slice - 2026-10-08 (Codex) + +Status SOURCE_TESTED / FINAL_RUN_PENDING. Runner now records and validates +Query image2.2.3-rc2-43bbcf0 / sha256:77682eea5054ae9ddbd858323b083286669a55eeb4bc7bc065b04bae8854e32f +against actual container inspection. Never labels it productionQueryv2.2.2. +Final mode requires5-10 independently started consumer processes, strictly +advancing trade IDs, raw monotonic start/first-event samples, n/p50/max ms. +Copied, missing, out-of-order or oldn1 samples cannot qualify. Initial full +OUT_OF_RANGE recovery and snapshot timing remain separately labelledn1. +Unit: python3 -B -m unittest tests.test_p184_dr08_isolated -q: +40PASS0fail0skip. Docker orchestration mocked in new tests; no fault rerun, +no fresh recovery measurements and no new acceptance claim. Previous rehearsal +receipt unchanged. No containers/images created by this slice; no rollout. +Final TS provenance remains gated on Claude CI confirmation. v2.2.3 HOLD. diff --git a/scripts/p184_dr08_isolated.py b/scripts/p184_dr08_isolated.py index dcce0aca..3394441b 100644 --- a/scripts/p184_dr08_isolated.py +++ b/scripts/p184_dr08_isolated.py @@ -10,7 +10,9 @@ replacement for fresh snapshot recovery. Packet JSON keys: namespace, development, ts_source_sha, ts_image_digest, -images (kafka/core/stream/python immutable sha256 IDs), bootstrap (localhost:9092), +images (kafka/core/stream/python/query immutable sha256 IDs), query_provenance +(candidate image_ref and inspected actual_image_digest), restart_samples (final +5-10, development 1-10; defaults 5 and 1), bootstrap (localhost:9092), raw_topic, raw_template (base64 RawProviderEnvelope, test_provenance=true), consumer_id, fast_consumer_id, instrument_uid, metrics_path, and slice (the existing Slice args: bundle, cursor_keys, profile, probes, topic_id, @@ -36,6 +38,7 @@ from pathlib import Path import re import signal +import statistics import sys import time from types import SimpleNamespace @@ -49,6 +52,72 @@ REPLAY = 10000 MAX_METRICS_BYTES = 8 * 1024 * 1024 QUEUE_BYTES = 32 * 1024 * 1024 +QUERY_IMAGE_REF = "qdl-v2-python:2.2.3-rc2-43bbcf0" +QUERY_IMAGE_DIGEST = "sha256:77682eea5054ae9ddbd858323b083286669a55eeb4bc7bc065b04bae8854e32f" +MIN_FINAL_RESTARTS = 5 +MAX_RESTARTS = 10 + + +def restart_count(packet): + count = packet.get("restart_samples", 1 if packet.get("development") is True else MIN_FINAL_RESTARTS) + minimum = 1 if packet.get("development") is True else MIN_FINAL_RESTARTS + if type(count) is not int or not minimum <= count <= MAX_RESTARTS: + raise ValueError(f"restart_samples must be an integer in [{minimum}, {MAX_RESTARTS}]") + return count + + +def validate_query_provenance(packet): + query = packet.get("query_provenance", {}) + if (not isinstance(query, dict) or packet.get("images", {}).get("query") != QUERY_IMAGE_DIGEST + or query.get("image_ref") != QUERY_IMAGE_REF + or query.get("actual_image_digest") != QUERY_IMAGE_DIGEST): + raise ValueError("Query must be verified against candidate 2.2.3-rc2-43bbcf0, not production v2.2.2") + + +def recovery_summary(samples): + """Reject copied samples and require ordered, independently timed restarts.""" + if not isinstance(samples, list) or not 1 <= len(samples) <= MAX_RESTARTS: + raise ValueError("bounded measured restart sample list required") + previous_end, previous_trade = 0, 0 + values = [] + for sample in samples: + start, end, trade = (sample.get(key) for key in ("started_ns", "first_event_ns", "trade_id")) + if (any(type(value) is not int for value in (start, end, trade)) + or not previous_end < start < end or trade <= previous_trade + or sample.get("code") != "OK" or sample.get("count") != 1): + raise ValueError("independent successful consumer restart measurements required") + elapsed = (end - start) / 1_000_000 + if not 0 < elapsed < 60000 or sample.get("restart_to_first_event_ms") != elapsed: + raise ValueError("invalid measured restart latency") + values.append(elapsed) + previous_end, previous_trade = end, trade + return {"measurement_n": len(values), "restart_samples": samples, + "restart_to_first_event_ms_samples": values, + "restart_to_first_event_ms_p50": statistics.median(values), + "restart_to_first_event_ms_max": max(values)} + + +def measure_restarts(packet, producer, start, token, trade_id): + samples = [] + for index in range(restart_count(packet)): + if index: + producer.publish(1) + trade_id = producer.next_id - 1 + token, _ = asyncio.run(snapshot(packet, packet["consumer_id"], trade_id)) + child = start(packet["consumer_id"], token, stop_after=1) + try: + child.wait("live") + producer.publish(1) + recovered = child.wait("done") + if recovered.get("last") != trade_id + 1: + raise ValueError("restart recovered the wrong trade") + first_ns = recovered.get("first_ns", 0) + samples.append({"started_ns": child.started_ns, "first_event_ns": first_ns, + "trade_id": recovered["last"], "code": recovered["code"], "count": recovered["count"], + "restart_to_first_event_ms": (first_ns - child.started_ns) / 1_000_000}) + finally: + child.close() + return recovery_summary(samples) def validate_packet(packet, *, development): @@ -67,6 +136,8 @@ def validate_packet(packet, *, development): for role in ("kafka", "core", "stream", "python"): if not re.fullmatch(r"sha256:[0-9a-f]{64}", packet.get("images", {}).get(role, "")): raise ValueError(f"immutable {role} image ID required") + restart_count(packet) + validate_query_provenance(packet) if not development: sha, image = packet.get("ts_source_sha", ""), packet.get("ts_image_digest", "") if not re.fullmatch(r"[0-9a-f]{40}", sha) or len(set(sha)) == 1: @@ -79,11 +150,13 @@ def validate_packet(packet, *, development): def evidence_rows(packet, facts, *, development, run_id): """Development rows deliberately cannot satisfy the TS matrix checker.""" + validate_query_provenance(packet) mode = "DEVELOPMENT" if development else "QUALIFICATION" header = {"provenance": { "schema": SCHEMA, "source_sha": packet["ts_source_sha"], "image_digest": packet["ts_image_digest"], "mode": mode, "run_id": run_id, "data_provenance": "SYNTHETIC_TEST_ONLY", "images": packet["images"], + "query_provenance": packet.get("query_provenance", {}), "packet_sha256": hashlib.sha256(json.dumps(packet, sort_keys=True).encode()).hexdigest(), "scope": "same-host isolated DL fault test; not TS consumer or real-provider certification", }} @@ -92,6 +165,15 @@ def evidence_rows(packet, facts, *, development, run_id): fact = facts.get(cell, {}) observed = fact.get("observed") is True passed = observed and fact.get("passed") is True + if cell == CELLS[1] and not development: + try: + summary = recovery_summary(fact.get("restart_samples")) + passed = passed and summary["measurement_n"] == restart_count(packet) + fact = {**fact, **summary} + except (ValueError, TypeError, AttributeError): + passed = False + if not passed: + observed = False rows.append({"cell": cell, "level": "ISOLATED_FAULT", "mode": mode, "run_id": run_id, "ok": passed and not development, "outcome": "OBSERVED" if observed and not development else "NOT_OBSERVED", @@ -474,25 +556,19 @@ def start(consumer, token, **kwargs): fresh_token, fresh_offset = asyncio.run(snapshot(packet, slow_id, fresh_trade_id)) snapshot_ms = (time.monotonic_ns() - snapshot_started_ns) / 1_000_000 stage = "consumer_restart" - restart = start(slow_id, fresh_token, stop_after=1) - restart.wait("live") - producer.publish(1) - recovered = restart.wait("done") - restart_ms = (recovered.get("first_ns", 0) - restart.started_ns) / 1_000_000 - restart.close() + recovery = measure_restarts(packet, producer, start, fresh_token, fresh_trade_id) facts[CELLS[1]] = {"observed": True, - "passed": exact_replay and out_of_range and fresh_offset > old_offset - and recovered["code"] == "OK" and recovered["count"] == 1 - and recovered["last"] == fresh_trade_id + 1 and 0 < restart_ms < 60000, + "passed": exact_replay and out_of_range and fresh_offset > old_offset, "replayed": replay_end["count"], "range_code": range_end["code"], "old_snapshot_offset": old_offset, "fresh_snapshot_offset": fresh_offset, "baseline_trade_id": baseline_id, "fresh_snapshot_trade_id": fresh_trade_id, - "restart_to_first_event_ms": restart_ms, "restart_kind": "consumer_process", - "snapshot_request_ms": snapshot_ms, "measurement_n": 1, - "range_observed_to_first_event_ms": (recovered.get("first_ns", 0) - range_observed_ns) / 1_000_000, + **recovery, "restart_kind": "consumer_process", + "snapshot_request_ms": snapshot_ms, "snapshot_request_measurement_n": 1, + "range_observed_to_first_event_ms": (recovery["restart_samples"][0]["first_event_ns"] - range_observed_ns) / 1_000_000, + "range_recovery_measurement_n": 1, "ack_measured": False, "fresh_snapshot_rpc": "MarketDataStreamService/GetSnapshot", - "snapshot_path": "isolated Query to canonical hot read; not projector recovery proof"} + "snapshot_path": "isolated candidate Query 2.2.3-rc2-43bbcf0 to canonical hot read; not production v2.2.2 or projector recovery proof"} facts[CELLS[2]] = {"observed": True, "passed": fast_end["code"] == "OK" and fast_end["count"] == REPLAY + 1 and fast_end["last"] == baseline_id + REPLAY + 1 and bool(progress_samples) and slow_resumed, @@ -581,6 +657,7 @@ def dependency_smoke(): IMAGES = { + "query": QUERY_IMAGE_DIGEST, "core": "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023", "stream": "sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9", "python": "sha256:77682eea5054ae9ddbd858323b083286669a55eeb4bc7bc065b04bae8854e32f", @@ -802,7 +879,7 @@ def serve_query(work): hot.close() -def orchestrate(work, *, development, source_sha, image_digest): +def orchestrate(work, *, development, source_sha, image_digest, restart_samples=None): """Adapt measure_l2_top_materialization's isolated broker/core packet.""" import shutil import subprocess @@ -814,6 +891,8 @@ def orchestrate(work, *, development, source_sha, image_digest): raise ValueError("final TS source SHA required before setup") if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_digest) or len(set(image_digest[7:])) == 1: raise ValueError("final TS image digest required before setup") + count = restart_count({"development": development, **( + {"restart_samples": restart_samples} if restart_samples is not None else {})}) work = work.resolve() work.mkdir(mode=0o700, parents=True, exist_ok=False) namespace = "p184-dr08-" + uuid.uuid4().hex[:16] @@ -831,7 +910,9 @@ def inventory(): before = inventory() try: for image in IMAGES.values(): - command(["docker", "image", "inspect", image, "--format", "{{.Id}}"]) + actual = command(["docker", "image", "inspect", image, "--format", "{{.Id}}"]) + if actual != image: + raise ValueError("local image digest mismatch") runtime = json.loads(command(["docker", "inspect", "qdl_v2_stable_candidate-rust_core-1"]))[0] if runtime["Image"] != IMAGES["core"]: raise ValueError("deployed core image drift") @@ -847,7 +928,9 @@ def inventory(): cfg["authority"].update(mode="RUST_SHADOW", slice_id=namespace, approved_by="isolated DR08 test") (work / "core.json").write_text(json.dumps(cfg)) packet = {"namespace": namespace, "development": development, "ts_source_sha": source_sha, - "ts_image_digest": image_digest, "images": IMAGES, "bootstrap": "localhost:9092"} + "ts_image_digest": image_digest, "images": IMAGES, "bootstrap": "localhost:9092", + "restart_samples": count, "query_provenance": {"image_ref": QUERY_IMAGE_REF, + "scope": "isolated candidate Query; not production v2.2.2"}} (work / "packet.json").write_text(json.dumps(packet)) def launch(role, image, args, *, memory, cpus, network, env=None, detach=True, extra=()): name = namespace + "-" + role @@ -912,8 +995,15 @@ def launch(role, image, args, *, memory, cpus, network, env=None, detach=True, e "QDL_KN_READ_VIEW_URLS": "https://localhost:8200", "QDL_KN_READ_VIEW_CA_FILE": "/work/ca.crt", "QDL_KN_HOT_READ_ENABLED": "true", "QDL_KN_METRICS_SECONDS": "1", "QDL_KN_MAX_REPLAY_EVENTS": str(REPLAY), "QDL_KN_QUEUE_BYTES_TOTAL": str(QUEUE_BYTES)}) - launch("query", IMAGES["python"], script + ["--query", "/work"], memory="512m", cpus=.5, + query = launch("query", IMAGES["query"], script + ["--query", "/work"], memory="512m", cpus=.5, network=net, env={**penv, **common}) + query_state = json.loads(command(["docker", "inspect", query]))[0] + packet["query_provenance"]["actual_image_digest"] = query_state["Image"] + validate_query_provenance(packet) + # Query may be reading its startup packet; publish the attestation atomically. + attested_packet = work / "packet.attested.json" + attested_packet.write_text(json.dumps(packet)) + attested_packet.replace(work / "packet.json") ready_deadline = time.monotonic() + 30 while True: core_state = json.loads(command(["docker", "inspect", namespace + "-core"]))[0] @@ -982,6 +1072,8 @@ def main(argv=None): parser.add_argument("--packet", type=Path) parser.add_argument("--output", type=Path) parser.add_argument("--development", action="store_true") + parser.add_argument("--restart-samples", type=int, + help="orchestrator restart count: development 1-10, final 5-10") args = parser.parse_args(argv) if args.orchestrate: def orchestration_deadline(_signum, _frame): @@ -989,11 +1081,13 @@ def orchestration_deadline(_signum, _frame): previous = signal.signal(signal.SIGALRM, orchestration_deadline) signal.alarm(600) try: - orchestrate(args.orchestrate, development=args.development, source_sha=args.ts_source_sha, image_digest=args.ts_image_digest) + orchestrate(args.orchestrate, development=args.development, source_sha=args.ts_source_sha, image_digest=args.ts_image_digest, restart_samples=args.restart_samples) finally: signal.alarm(0) signal.signal(signal.SIGALRM, previous) return 0 + if args.restart_samples is not None: + parser.error("--restart-samples requires --orchestrate; inner driver uses its sealed packet") for directory, function in ((args.prepare_material, prepare_material), (args.prepare_topics, prepare_topics), (args.query, serve_query)): if directory is not None: function(directory) @@ -1017,11 +1111,12 @@ def deadline(_signum, _frame): finally: signal.alarm(0) signal.signal(signal.SIGALRM, previous) - for row in evidence_rows(packet, facts, development=args.development, run_id=run_id): + rows = evidence_rows(packet, facts, development=args.development, run_id=run_id) + for row in rows: output.write(json.dumps(row, sort_keys=True) + "\n") output.flush() os.fsync(output.fileno()) - return 0 if all(facts.get(cell, {}).get("passed") is True for cell in CELLS) else 1 + return 0 if all(row["development_pass"] if args.development else row["ok"] for row in rows[1:]) else 1 if __name__ == "__main__": diff --git a/tests/test_p184_dr08_isolated.py b/tests/test_p184_dr08_isolated.py index 811d0b62..66dcc059 100644 --- a/tests/test_p184_dr08_isolated.py +++ b/tests/test_p184_dr08_isolated.py @@ -19,7 +19,9 @@ def packet(development=True): "consumer_id": "slow", "fast_consumer_id": "fast", "raw_topic": "test-raw", "metrics_path": "/isolated/metrics", "ts_source_sha": "1234567890" * 4, "ts_image_digest": "sha256:" + "12345678" * 8, - "images": {role: "sha256:" + "abcdef01" * 8 for role in ("core", "stream", "kafka", "python")}} + "query_provenance": {"image_ref": R.QUERY_IMAGE_REF, "actual_image_digest": R.QUERY_IMAGE_DIGEST}, + "images": {"query": R.QUERY_IMAGE_DIGEST, **{ + role: "sha256:" + "abcdef01" * 8 for role in ("core", "stream", "kafka", "python")}}} class Guards(unittest.TestCase): @@ -75,6 +77,187 @@ def test_dummy_final_provenance_refused(self): R.validate_packet(p, development=False) +class RestartMeasurements(unittest.TestCase): + @staticmethod + def samples(n=5): + return [{"started_ns": (i + 1) * 100_000_000, + "first_event_ns": (i + 1) * 100_000_000 + (i + 1) * 1_000_000, + "trade_id": i + 1, "code": "OK", "count": 1, + "restart_to_first_event_ms": float(i + 1)} for i in range(n)] + + def test_count_defaults_and_bounds(self): + self.assertEqual(R.restart_count(packet()), 1) + self.assertEqual(R.restart_count(packet(False)), 5) + for development, counts in ((True, (0, 11, True, 1.0, "5")), + (False, (0, 1, 4, 11, False, 5.0))): + for count in counts: + with self.subTest(development=development, count=count), self.assertRaises(ValueError): + R.restart_count({**packet(development), "restart_samples": count}) + self.assertEqual(R.restart_count({**packet(False), "restart_samples": 10}), 10) + + def test_summary_uses_measured_samples(self): + summary = R.recovery_summary(self.samples()) + self.assertEqual(summary["measurement_n"], 5) + self.assertEqual(summary["restart_to_first_event_ms_samples"], [1, 2, 3, 4, 5]) + self.assertEqual(summary["restart_to_first_event_ms_p50"], 3) + self.assertEqual(summary["restart_to_first_event_ms_max"], 5) + self.assertEqual(R.recovery_summary(self.samples(6))["restart_to_first_event_ms_p50"], 3.5) + + def test_invalid_or_duplicated_measurements_rejected(self): + bad_lists = [None, [], self.samples(11), [self.samples(1)[0]] * 5] + for key, value in (("started_ns", 0), ("first_event_ns", 0), + ("trade_id", 0), ("code", "OUT_OF_RANGE"), ("count", 2), + ("restart_to_first_event_ms", float("nan")), + ("restart_to_first_event_ms", 60000)): + samples = self.samples() + samples[2][key] = value + bad_lists.append(samples) + for samples in bad_lists: + with self.subTest(samples=samples), self.assertRaises(ValueError): + R.recovery_summary(samples) + + def test_old_n1_or_forged_n5_cannot_qualify(self): + for extra in ({"measurement_n": 1, "restart_to_first_event_ms": 5241.271}, + {"measurement_n": 5}, {"restart_samples": self.samples(1)}, + {"restart_samples": [self.samples(1)[0]] * 5}): + fact = {"observed": True, "passed": True, **extra} + row = R.evidence_rows(packet(False), {R.CELLS[1]: fact}, development=False, run_id="fresh")[2] + self.assertFalse(row["ok"]) + self.assertEqual(row["outcome"], "NOT_OBSERVED") + + def test_five_qualifies_but_development_never_does(self): + fact = {"observed": True, "passed": True, "restart_samples": self.samples()} + final = R.evidence_rows(packet(False), {R.CELLS[1]: fact}, development=False, run_id="fresh")[2] + self.assertTrue(final["ok"]) + self.assertEqual(final["measurement"]["restart_to_first_event_ms_p50"], 3) + for n in (1, 5): + fact["restart_samples"] = self.samples(n) + dev = R.evidence_rows(packet(), {R.CELLS[1]: fact}, development=True, run_id="dev")[2] + self.assertFalse(dev["ok"]) + self.assertTrue(dev["development_pass"]) + + def test_independent_consumers_started_and_closed(self): + from unittest.mock import AsyncMock, Mock + for development, n in ((True, 1), (False, 5)): + with self.subTest(development=development): + producer = Mock(next_id=11) + producer.publish.side_effect = lambda count: setattr(producer, "next_id", producer.next_id + count) + children = [] + def start(consumer, token, **kwargs): + self.assertEqual(consumer, "slow") + self.assertEqual(kwargs, {"stop_after": 1}) + if children: + children[-1].close.assert_called_once() + child = Mock(started_ns=(len(children) + 1) * 100_000_000) + def wait(kind): + return {} if kind == "live" else { + "first_ns": child.started_ns + 1_000_000, + "last": producer.next_id - 1, "code": "OK", "count": 1} + child.wait.side_effect = wait + children.append(child) + return child + with patch.object(R, "snapshot", new_callable=AsyncMock, return_value=("fresh", 20)) as snapshot: + summary = R.measure_restarts(packet(development), producer, start, "initial", 10) + self.assertEqual(len(children), n) + self.assertEqual(snapshot.await_count, n - 1) + self.assertEqual(summary["measurement_n"], n) + self.assertEqual(summary["restart_to_first_event_ms_samples"], [1] * n) + for child in children: + child.close.assert_called_once() + + def test_failed_restart_is_closed_and_not_summarized(self): + from unittest.mock import Mock + child = Mock(started_ns=1) + child.wait.side_effect = TimeoutError("test only") + with self.assertRaises(TimeoutError): + R.measure_restarts(packet(), Mock(), Mock(return_value=child), "token", 10) + child.close.assert_called_once() + + def test_incomplete_requested_sample_count_cannot_qualify(self): + p = {**packet(False), "restart_samples": 6} + fact = {"observed": True, "passed": True, "restart_samples": self.samples(5)} + row = R.evidence_rows(p, {R.CELLS[1]: fact}, development=False, run_id="fresh")[2] + self.assertFalse(row["ok"]) + + def test_wrong_query_provenance_cannot_emit_evidence(self): + p = packet(False) + p["query_provenance"]["actual_image_digest"] = "sha256:" + "1" * 64 + with self.assertRaises(ValueError): + R.evidence_rows(p, {}, development=False, run_id="fresh") + + def test_final_exit_succeeds_with_five_measured_restarts(self): + with tempfile.TemporaryDirectory() as d: + config, output = Path(d) / "packet", Path(d) / "evidence" + p = packet(False) + config.write_text(json.dumps(p)) + facts = {cell: {"observed": True, "passed": True} for cell in R.CELLS} + facts[R.CELLS[1]]["restart_samples"] = self.samples() + with patch.dict(os.environ, {"P184_DR08_ISOLATED_NAMESPACE": p["namespace"]}), \ + patch.object(R, "run_faults", return_value=facts): + self.assertEqual(R.main(["--packet", str(config), "--output", str(output)]), 0) + row = json.loads(output.read_text().splitlines()[2]) + self.assertTrue(row["ok"]) + self.assertEqual(row["measurement"]["measurement_n"], 5) + + def test_final_exit_fails_for_old_single_sample(self): + with tempfile.TemporaryDirectory() as d: + config, output = Path(d) / "packet", Path(d) / "evidence" + p = packet(False) + config.write_text(json.dumps(p)) + facts = {cell: {"observed": True, "passed": True} for cell in R.CELLS} + facts[R.CELLS[1]]["measurement_n"] = 1 + with patch.dict(os.environ, {"P184_DR08_ISOLATED_NAMESPACE": p["namespace"]}), \ + patch.object(R, "run_faults", return_value=facts): + self.assertEqual(R.main(["--packet", str(config), "--output", str(output)]), 1) + self.assertFalse(json.loads(output.read_text().splitlines()[2])["ok"]) + + def test_query_candidate_digest_and_label_required(self): + p = packet() + R.validate_query_provenance(p) + for field, value in (("actual_image_digest", "sha256:" + "1" * 64), + ("image_ref", "qdl-v2-python:2.2.2")): + bad = packet() + bad["query_provenance"][field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + R.validate_query_provenance(bad) + p["images"]["query"] = "sha256:" + "1" * 64 + with self.assertRaises(ValueError): + R.validate_query_provenance(p) + p = packet() + del p["query_provenance"] + with self.assertRaises(ValueError): + R.validate_query_provenance(p) + + def test_query_candidate_named_in_evidence(self): + provenance = R.evidence_rows(packet(), {}, development=True, run_id="dev")[0]["provenance"] + self.assertEqual(provenance["query_provenance"]["image_ref"], "qdl-v2-python:2.2.3-rc2-43bbcf0") + self.assertEqual(provenance["query_provenance"]["actual_image_digest"], R.QUERY_IMAGE_DIGEST) + + def test_final_one_run_refused_before_setup(self): + with tempfile.TemporaryDirectory() as d, patch("subprocess.run") as run: + target = Path(d) / "absent" + with self.assertRaises(ValueError): + R.orchestrate(target, development=False, source_sha=packet(False)["ts_source_sha"], + image_digest=packet(False)["ts_image_digest"], restart_samples=1) + self.assertFalse(target.exists()) + run.assert_not_called() + + def test_actual_local_image_mismatch_fails_before_launch(self): + import subprocess + from contextlib import redirect_stdout + import io + calls = [] + def run(args, **kwargs): + calls.append(args) + out = "sha256:" + "1" * 64 if args[:3] == ["docker", "image", "inspect"] else "" + return subprocess.CompletedProcess(args, 0, stdout=out, stderr="") + with tempfile.TemporaryDirectory() as d, patch("subprocess.run", side_effect=run), redirect_stdout(io.StringIO()): + with self.assertRaisesRegex(ValueError, "image digest mismatch"): + R.orchestrate(Path(d) / "attempt", development=True, + source_sha="0" * 40, image_digest="sha256:" + "0" * 64) + self.assertFalse(any(args[:2] == ["docker", "run"] for args in calls)) + + class Evidence(unittest.TestCase): def test_exact_cells(self): self.assertEqual(set(R.CELLS), { From 351cc72c300d7a3174627f322eb5accc6447947c Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 08:34:39 +0000 Subject: [PATCH 029/147] test(p184): separate DR01 controlled regression from deployed alpha probe --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 53 ++++ scripts/p184_dr01_deployed.py | 274 ++++++++++++++++++++ scripts/p184_dr01_runtime.py | 4 +- tests/test_p184_dr01_deployed.py | 153 +++++++++++ tests/test_p184_dr01_runtime.py | 3 + upgrade/evidence/p184-dr-runtime-runbook.md | 114 +++++--- 6 files changed, 562 insertions(+), 39 deletions(-) create mode 100644 scripts/p184_dr01_deployed.py create mode 100644 tests/test_p184_dr01_deployed.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 2ed66d09..a1d1ea05 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70036,3 +70036,56 @@ Unit: python3 -B -m unittest tests.test_p184_dr08_isolated -q: no fresh recovery measurements and no new acceptance claim. Previous rehearsal receipt unchanged. No containers/images created by this slice; no rollout. Final TS provenance remains gated on Claude CI confirmation. v2.2.3 HOLD. + + +### DR01 deployed runner handoff - 2026-10-08 (Codex) + +Status RUNNER_SOURCE_TESTED / DEPLOYED_PROBE_NOT_RUN / RELEASE_HOLD. Existing +scripts/p184_dr01_runtime.py now emits CONTROLLED_SIM exclusively. Its historical +account helpers remain regression-only, never used to close DR01. New +scripts/p184_dr01_deployed.py separates offline prepare, stdin probe and record. +Prepare reuses compile_alpha_deployment_bindings.compile_inventory against +stable-v2-release-routing.yaml and original alpha inventory; rejects wildcards, +seals expected binding+manifest hashes. Probe compares actual mounted binding +bytes/checksum, identity, release hash and JWT14 before any request. GET-only, +mTLS/hostname verification, no redirect/proxy/retry, bounded body/time. Four +requests per venue: positive bound QUOTE, live401, revision13/401, valid catalog +product absent from actual access manifest403PERMISSION_DENIED. Positive control +failure never passes negatives. Runtime record requires both venues and explicit +CI-green acknowledgement; exact7DL cells, no account cells. Claude supplies the +remaining2Gateway account cells. Tokens, credentials and response bodies are not +logged. No order/stream action, no actual alpha identity used in this slice. + +Offline actual compiler in existing image77682eea --rm/network none/.5CPU/768MiB: +both BB deployment bindings match the files mounted by alpha-noorder-sdk2b. +Observed release_revision27 (not22), access JWTmanifest14. Binance binding hash +bbc8b0de1194778f70f94d5cd003fa251f3084250dee3590d4944b4e5f8b8fb8; +OKX29e321e0c6067b34070227598fe8d694f3b491d017ffaa25d4e1073706225ac4. +No configuration was rewritten to resolve naming confusion. Source selector +inspection: rg -n '\*|\?' consumers/stable/alpha-binance-paper.yaml +consumers/stable/alpha-okx-paper.yaml: exit1,0matches. Parsed concrete-selector +audit also clean, hashes in prepared packet. This does not replace live403. +Initial negative-selector prepare assumed same instrument had an ungranted feed; +offline real inventory disproved it (StopIteration). Fixed selection to valid +same-venue catalog instrument/feed outside access manifest, then both compile +checks passed. No runtime requests occurred. + +Final tests in existing Python image, sourceRO/network none, ephemeral loopback +HTTP only: python -B -m unittest tests.test_p184_dr01_deployed +tests.test_p184_dr01_runtime tests.test_p184_dr08_isolated -q: +73PASS0fail0skip38.268s (9new runner +24controlled regression +40DR08). Earlier +test-mock environment parameter collision fixed; failed run retained in session +output, not described as production defect. Readiness/latency/DR01 runtime cells +not certified by these unit/controlled tests. + +Guide updated: upgrade/evidence/p184-dr-runtime-runbook.md includes exact +prepare/probe/record commands, revision distinction and minimum5restart gate. +TS SHA2abbe394/digesta6548ee3 remains provisional until Claude confirms CI green. +No push/merge, shared runtime or manifest changes. Own --rm compile/test containers +removed (docker ps -a --filter name=qdl-dr01 empty); no new images, secrets or +networks retained. Rust cached build artifacts retained for concurrent work; +no broad prune. Canonical /home/bobby/data_layer main5bf5cd5 releasev2.2.2; +feature /home/bobby/.worktrees/data-layer-p184-closure fix/p184-dl-closure. +Active runtime remains as previous inventory; patch not deployed. Keep BAR +rollbackf7351c3b. Next: Claude reviews and runs DR window with final CI-green +provenance, then combines Gateway cells/DR08/broker delta; v2.2.3 staysHOLD. diff --git a/scripts/p184_dr01_deployed.py b/scripts/p184_dr01_deployed.py new file mode 100644 index 00000000..b32b72a3 --- /dev/null +++ b/scripts/p184_dr01_deployed.py @@ -0,0 +1,274 @@ +"""DR01 deployed alpha GET-only probe, delivered through python -B - probe. + +prepare is offline DL compilation; probe runs only when Claude explicitly invokes +it in the alpha compose --rm container; record joins the two venue observations. +No account cells, orchestration, retries, entitlement edits or order calls. +""" +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import os +from pathlib import Path +import re +import ssl +import sys +import time +from urllib.error import HTTPError +from urllib.parse import urlencode, urlsplit +from urllib.request import HTTPSHandler, HTTPRedirectHandler, ProxyHandler, Request, build_opener + +VENUES = ("BINANCE", "OKX") +CASES = ("BINDING_COMPILE", "WRONG_REALM_DENIED", "WRONG_REVISION_DENIED") +MAX_BYTES = 2_000_000 + + +def digest(raw): + return hashlib.sha256(raw).hexdigest() + + +def canonical(obj): + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + + +def read_json(path): + raw = Path(path).read_bytes() + if len(raw) > MAX_BYTES: + raise ValueError("input exceeds bound") + return json.loads(raw) + + +def identity(req): + return tuple(req.get(k) for k in ("instrument_uid", "feed", "interval", "source_policy_id")) + + +def wildcard_paths(obj, path="$"): + if isinstance(obj, dict): + return [p for k, v in obj.items() for p in wildcard_paths(v, f"{path}.{k}")] + if isinstance(obj, list): + return [p for i, v in enumerate(obj) for p in wildcard_paths(v, f"{path}[{i}]")] + return [path] if isinstance(obj, str) and ("*" in obj or "?" in obj) else [] + + +def prepare(args): + # Reuse the deployed alpha compiler, not the TS-only stable renderer. + root = Path(args.root).resolve() + sys.path.insert(0, str(root)) + import yaml + from scripts.compile_alpha_deployment_bindings import compile_inventory + from qdl.consumer.universal_release import ConsumerRouteBinding + paths = {"catalog": Path(args.catalog) if args.catalog else root / "config/v2/stable-source-bindings.yaml", + "reference_manifest": root / "consumers/stable/reference-l2-stable.yaml", + "release_routing": root / "config/v2/stable-v2-release-routing.yaml", + "policy": root / "config/v2/universal-release-policy.yaml"} + raw = {k: p.read_bytes() for k, p in paths.items()} + values = {k: yaml.safe_load(v) for k, v in raw.items()} + report = compile_inventory(inventory=read_json(args.inventory), **values, + **{k + "_sha256": digest(v) for k, v in raw.items()}) + entry = next(x for x in report["deployments"] if x["deployment_id"] == args.deployment) + if entry["status"] != "ADMITTED": + raise ValueError("deployment not admitted") + binding = entry["binding"] + ConsumerRouteBinding.from_canonical_mapping(binding) + manifest_path = root / f"consumers/stable/alpha-{args.venue.lower()}-paper.yaml" + manifest_raw = manifest_path.read_bytes() + manifest = yaml.safe_load(manifest_raw) + requirements = manifest["spec"]["requirements"] + # Check concrete source entitlement selectors as well as consumer selectors. + selectors = {"consumer": manifest["spec"], "bindings": [ + {k: b.get(k) for k in ("instrument_uid", "feed", "interval", "source")} + for b in values["catalog"]["bindings"]]} + wildcards = wildcard_paths(selectors) + if wildcards or manifest["metadata"]["revision"] != 14: + raise ValueError("wildcard entitlement or manifest revision drift") + products = [p for p in binding["products"] if p["venue"] == args.venue] + baseline = next(r for p in products if p["feed"] == "QUOTE" + for r in requirements if identity(r) == identity(p)) + # An absent deployment route alone is insufficient: deny against the actual + # access manifest, using an otherwise valid instrument/feed in the catalog. + own_uids = {p["instrument_uid"] for p in values["catalog"]["instruments"] if p["venue"] == args.venue} + forbidden = next(b for b in values["catalog"]["bindings"] + if b["instrument_uid"] in own_uids + and not any(r["instrument_uid"] == b["instrument_uid"] + and r["feed"] == b["feed"] and r.get("interval") == b.get("interval") + for r in requirements)) + negative = dict(baseline, instrument_uid=forbidden["instrument_uid"], feed=forbidden["feed"], + interval=forbidden.get("interval"), source_policy_id=forbidden["source"]["source_policy_id"]) + evidence = {"manifest_path": str(manifest_path.relative_to(root)), + "manifest_sha256": digest(manifest_raw), "wildcard_paths": wildcards, + "selector_count": len(requirements), + "grep_equivalent_command": "rg -n 'instrument_uid:|feed:|source_policy_id:' consumers/stable/alpha-*-paper.yaml", + "source_files": {str(paths[k]): digest(v) for k, v in raw.items()}} + packet = {"schema": "p184.dr01.deployed-packet.v1", "venue": args.venue, + "access_consumer_id": manifest["metadata"]["id"], "subject": manifest["metadata"]["subject"], + "jwt_revision": 14, "environment": "paper", "binding": binding, + "baseline": baseline, "forbidden": negative, "entitlements": evidence} + with Path(args.output).open("x") as out: + json.dump(packet, out, sort_keys=True) + + +def validate(packet, binding, env): + if packet.get("schema") != "p184.dr01.deployed-packet.v1" or packet.get("venue") not in VENUES: + raise ValueError("invalid packet") + expected = packet["binding"] + for obj in (binding, expected): + claimed = obj["binding_sha256"] + if claimed != digest(canonical({k: v for k, v in obj.items() if k != "binding_sha256"})): + raise ValueError("binding checksum mismatch") + if canonical(expected) != canonical(binding): + raise ValueError("compiled and mounted binding differ") + required = {"DATA_LAYER_V2_ACCESS_CONSUMER_ID": packet["access_consumer_id"], + "DATA_LAYER_V2_CONSUMER_ID": binding["consumer_id"], + "DATA_LAYER_V2_JWT_SUBJECT": packet["subject"], + "DATA_LAYER_V2_JWT_ENVIRONMENT": "paper", "DATA_LAYER_V2_JWT_MANIFEST_REVISION": "14", + "DATA_LAYER_V2_RELEASE_MANIFEST_SHA256": binding["universal_manifest_sha256"]} + if any(env.get(k) != v for k, v in required.items()): + raise ValueError("alpha identity or revision drift") + if packet["entitlements"]["wildcard_paths"] or packet["jwt_revision"] != 14: + raise ValueError("entitlement drift") + if identity(packet["baseline"]) not in {identity(p) for p in binding["products"]}: + raise ValueError("baseline outside binding") + url = urlsplit(env["DATA_LAYER_V2_QUERY_URL"]) + if url.scheme != "https" or not url.hostname or url.username or url.password or url.query or url.fragment: + raise ValueError("authenticated TLS query origin required") + if url.path not in ("", "/"): + raise ValueError("query URL must be an origin") + + +def route(req): + keys = ("feed", "source_policy_id", "consumer_grade", "interval", "max_freshness_ms", + "event_recency_policy", "max_session_liveness_ms", "require_full_coverage", "require_final_bars", + "stale_policy", "gap_policy", "recovery", "bar_revision_policy") + params = {k: str(req[k]).lower() if isinstance(req[k], bool) else req[k] + for k in keys if req.get(k) is not None} + uid = req["instrument_uid"] + if not re.fullmatch(r"[a-f0-9-]{36}", uid): + raise ValueError("invalid instrument UID") + return f"/v2/market-data/{uid}/snapshot?" + urlencode(params) + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + +def get(opener, origin, path, headers): + started = time.monotonic_ns() + try: + response = opener.open(Request(origin.rstrip("/") + path, headers=headers, method="GET"), timeout=10) + except HTTPError as exc: + response = exc + with response: + raw = response.read(MAX_BYTES + 1) + if len(raw) > MAX_BYTES: + raise ValueError("response exceeds bound") + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ValueError("response must be an object") + data = payload.get("data") or {} + return {"status": response.code, "code": payload.get("code"), + "instrument_uid": data.get("instrument_uid"), "feed": data.get("feed"), + "body_sha256": digest(raw), "request_ms": (time.monotonic_ns() - started) / 1e6} + + +async def token(env, **overrides): + from qdl_sdk.credentials import RotatingJwtCredentialProvider + settings = dict(private_key_file=env["DATA_LAYER_V2_JWT_PRIVATE_KEY_FILE"], + key_id=env["DATA_LAYER_V2_JWT_KEY_ID"], algorithm=env.get("DATA_LAYER_V2_JWT_ALGORITHM", "RS256"), + issuer=env["DATA_LAYER_V2_JWT_ISSUER"], audience=env["DATA_LAYER_V2_JWT_AUDIENCE"], + subject=env["DATA_LAYER_V2_JWT_SUBJECT"], environment="paper", consumer_manifest_revision=14, + roles=tuple(env.get("DATA_LAYER_V2_JWT_ROLES", "market_data_reader,historical_reader,stream_consumer").split(",")), + venues=tuple(env.get("DATA_LAYER_V2_JWT_VENUES", "BINANCE,OKX").split(",")), + lifetime_seconds=120, refresh_before_seconds=30) + settings.update(overrides) + return await RotatingJwtCredentialProvider(**settings).get_token() + + +def evaluate(packet, observations): + base = observations["baseline"] + good = base["status"] == 200 and base["instrument_uid"] == packet["baseline"]["instrument_uid"] and base["feed"] == packet["baseline"]["feed"] + checks = {"BINDING_COMPILE": good, + "WRONG_REALM_DENIED": good and observations["realm"]["status"] == 401 and observations["realm"].get("code") == "UNAUTHENTICATED", + "WRONG_REVISION_DENIED": good and observations["revision"]["status"] == 401 and observations["revision"].get("code") == "UNAUTHENTICATED", + "NO_GLOBAL_GRANT": good and observations["forbidden"]["status"] == 403 + and observations["forbidden"]["code"] == "PERMISSION_DENIED"} + return [{"cell": f"DR01.{case}@{packet['venue'] if case != 'NO_GLOBAL_GRANT' else 'ANY'}", + "level": "RUNTIME", "ok": bool(ok), "outcome": "OBSERVED", "venue": packet["venue"], + "evidence": {"observations": observations, "binding_sha256": packet["binding"]["binding_sha256"], + "release_revision": packet["binding"]["release_revision"], "jwt_revision": 14, + "entitlements": packet["entitlements"], "mtls": True}} + for case, ok in checks.items()] + + +def probe(args): + packet = read_json(args.packet) + binding = read_json(os.environ.get("DATA_LAYER_V2_ROUTE_MANIFEST_FILE", "/run/p184/binding.json")) + validate(packet, binding, os.environ) + context = ssl.create_default_context(cafile=os.environ["DATA_LAYER_V2_TLS_CA_FILE"]) + context.load_cert_chain(os.environ["DATA_LAYER_V2_TLS_CERT_FILE"], os.environ["DATA_LAYER_V2_TLS_KEY_FILE"]) + opener = build_opener(ProxyHandler({}), HTTPSHandler(context=context), NoRedirect()) + observations = {} + for name, changes, req in (("baseline", {}, packet["baseline"]), + ("realm", {"environment": "live"}, packet["baseline"]), + ("revision", {"consumer_manifest_revision": 13}, packet["baseline"]), + ("forbidden", {}, packet["forbidden"])): + headers = {"Authorization": "Bearer " + asyncio.run(token(os.environ, **changes)), + "X-QDL-Consumer-ID": packet["access_consumer_id"], "X-QDL-Purpose": "INTERNAL_ALPHA"} + observations[name] = get(opener, os.environ["DATA_LAYER_V2_QUERY_URL"], route(req), headers) + time.sleep(.25) + rows = evaluate(packet, observations) + for row in rows: + print(json.dumps(row, sort_keys=True), flush=True) + return 0 if all(r["ok"] for r in rows) else 1 + + +def record(args): + if not args.ci_confirmed or len(set(args.source_sha)) == 1 or len(set(args.image_digest[7:])) == 1 or not re.fullmatch(r"[a-f0-9]{40}", args.source_sha) or not re.fullmatch(r"sha256:[a-f0-9]{64}", args.image_digest): + raise ValueError("CI confirmation and full TS provenance required") + rows = [json.loads(line) for path in args.rows for line in Path(path).read_text().splitlines() if line] + expected = {(v, f"DR01.{c}@{v if c != 'NO_GLOBAL_GRANT' else 'ANY'}") + for v in VENUES for c in (*CASES, "NO_GLOBAL_GRANT")} + keys = [(r.get("venue"), r.get("cell")) for r in rows] + if len(keys) != len(expected) or set(keys) != expected or any( + r.get("level") != "RUNTIME" or type(r.get("ok")) is not bool or r.get("outcome") != "OBSERVED" for r in rows): + raise ValueError("exact two-venue runtime rows required") + combined = [r for r in rows if r["cell"] != "DR01.NO_GLOBAL_GRANT@ANY"] + grants = [r for r in rows if r["cell"] == "DR01.NO_GLOBAL_GRANT@ANY"] + combined.append({"cell": "DR01.NO_GLOBAL_GRANT@ANY", "level": "RUNTIME", "ok": all(r["ok"] for r in grants), + "outcome": "OBSERVED", "evidence": grants}) + with Path(args.output).open("x") as out: + out.write(json.dumps({"provenance": {"schema": "p184.surface-evidence-provenance.v1", + "source_sha": args.source_sha, "image_digest": args.image_digest}}) + "\n") + for row in combined: + out.write(json.dumps(row, sort_keys=True) + "\n") + return 0 if all(r["ok"] for r in combined) else 1 + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + subs = parser.add_subparsers(dest="mode", required=True) + p = subs.add_parser("prepare") + for name in ("root", "inventory", "deployment", "output"): + p.add_argument("--" + name, required=True) + p.add_argument("--venue", choices=VENUES, required=True) + p.add_argument("--catalog") + p = subs.add_parser("probe") + p.add_argument("--packet", required=True) + p = subs.add_parser("record") + p.add_argument("--rows", nargs=2, required=True) + for name in ("source-sha", "image-digest", "output"): + p.add_argument("--" + name, required=True) + p.add_argument("--ci-confirmed", action="store_true") + args = parser.parse_args(argv) + return {"prepare": prepare, "probe": probe, "record": record}[args.mode](args) or 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + # Never echo credentials, HTTP response bodies or signed JWTs. + print(json.dumps({"runner_failed": type(exc).__name__}), file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/p184_dr01_runtime.py b/scripts/p184_dr01_runtime.py index f2594833..30097930 100644 --- a/scripts/p184_dr01_runtime.py +++ b/scripts/p184_dr01_runtime.py @@ -1,4 +1,4 @@ -"""Bounded isolated DR01 HTTP probe; not shipped-image or mTLS qualification. +"""CONTROLLED_SIM authentication regression; NEVER closes a DR01 runtime cell. Use --source-sha (40 hex) --image-digest (sha256:64 hex) --output FILE. Placeholder provenance requires --development and FILE.rehearsal.jsonl. @@ -53,7 +53,7 @@ def result(cell, ok, evidence, *, observed=True): raise ValueError("known cell and strict boolean results required") if ok and not observed: raise ValueError("unobserved evidence cannot pass") - return {"cell": cell, "level": "RUNTIME", "ok": ok, + return {"cell": cell, "level": "CONTROLLED_SIM", "ok": ok, "outcome": "OBSERVED" if observed else "NOT_OBSERVED", "evidence": evidence, "at_ms": time.time_ns() // 1_000_000} diff --git a/tests/test_p184_dr01_deployed.py b/tests/test_p184_dr01_deployed.py new file mode 100644 index 00000000..56b1ec9f --- /dev/null +++ b/tests/test_p184_dr01_deployed.py @@ -0,0 +1,153 @@ +"""Offline runner regression. No deployed endpoint/credential is accessed.""" +import argparse +from copy import deepcopy +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + +from scripts import p184_dr01_deployed as dr + + +def fixture(): + req = dict(instrument_uid="e49b54ae-c23d-5351-9e64-47934aac28f8", feed="QUOTE", + interval=None, source_policy_id="crypto_primary_v2", consumer_grade="ALPHA") + binding = dict(consumer_id="alpha.deployment", release_revision=27, + universal_manifest_sha256="c" * 64, products=[req]) + binding["binding_sha256"] = dr.digest(dr.canonical(binding)) + packet = dict(schema="p184.dr01.deployed-packet.v1", venue="OKX", binding=binding, + access_consumer_id="alpha.okx.paper.stable", subject="spiffe://qdl/paper/alpha-okx-stable", + jwt_revision=14, baseline=req, forbidden=dict(req, feed="BOOK_DELTA"), + entitlements={"wildcard_paths": []}) + env = dict(DATA_LAYER_V2_ACCESS_CONSUMER_ID=packet["access_consumer_id"], + DATA_LAYER_V2_CONSUMER_ID=binding["consumer_id"], DATA_LAYER_V2_JWT_SUBJECT=packet["subject"], + DATA_LAYER_V2_JWT_ENVIRONMENT="paper", DATA_LAYER_V2_JWT_MANIFEST_REVISION="14", + DATA_LAYER_V2_RELEASE_MANIFEST_SHA256="c" * 64, DATA_LAYER_V2_QUERY_URL="https://query_kn:8200") + return packet, binding, env + + +def observations(packet): + return dict(baseline=dict(status=200, instrument_uid=packet["baseline"]["instrument_uid"], feed="QUOTE"), + realm=dict(status=401, code="UNAUTHENTICATED"), revision=dict(status=401, code="UNAUTHENTICATED"), forbidden=dict(status=403, code="PERMISSION_DENIED")) + + +class DeployedTests(unittest.TestCase): + def test_binding_identity_and_revisions(self): + packet, binding, env = fixture() + dr.validate(packet, binding, env) + for key in env: + bad = dict(env, **{key: "bad"}) + with self.subTest(key=key), self.assertRaises(ValueError): + dr.validate(packet, binding, bad) + + def test_compiled_binding_diff_and_rehashed_mutation(self): + packet, binding, env = fixture() + for rehash in (False, True): + changed = dict(binding, release_revision=22) + if rehash: + changed["binding_sha256"] = dr.digest(dr.canonical({k: v for k, v in changed.items() if k != "binding_sha256"})) + with self.assertRaises(ValueError): + dr.validate(packet, changed, env) + + def test_no_redirect_credentials_or_query_in_origin(self): + packet, binding, env = fixture() + for url in ("http://query_kn", "https://user:password@query_kn", "https://query_kn/?key=x", "https://query_kn/v2"): + with self.subTest(url=url), self.assertRaises(ValueError): + dr.validate(packet, binding, dict(env, DATA_LAYER_V2_QUERY_URL=url)) + self.assertIsNone(dr.NoRedirect().redirect_request(None, None, 302, "", {}, "https://other")) + + def test_wildcard_and_baseline_outside_binding(self): + packet, binding, env = fixture() + packet["entitlements"]["wildcard_paths"] = ["$.instrument"] + with self.assertRaises(ValueError): + dr.validate(packet, binding, env) + packet["entitlements"]["wildcard_paths"] = [] + packet["baseline"] = dict(packet["baseline"], feed="TRADE") + with self.assertRaises(ValueError): + dr.validate(packet, binding, env) + self.assertEqual(dr.wildcard_paths({"a": ["uid", "*"]}), ["$.a[1]"]) + + def test_baseline_failure_never_negative_pass(self): + packet, _, _ = fixture() + for mutation in (dict(status=503), dict(instrument_uid="wrong"), dict(feed="TRADE")): + obs = observations(packet) + obs["baseline"].update(mutation) + self.assertFalse(any(r["ok"] for r in dr.evaluate(packet, obs))) + + def test_exact_wire_denials_and_no_account_cell(self): + packet, _, _ = fixture() + obs = observations(packet) + rows = dr.evaluate(packet, obs) + self.assertEqual(len(rows), 4) + self.assertTrue(all(r["ok"] and r["level"] == "RUNTIME" for r in rows)) + self.assertFalse(any("ACCOUNT" in r["cell"] for r in rows)) + for key, status in (("realm", 403), ("revision", 429), ("forbidden", 404)): + bad = deepcopy(obs); bad[key]["status"] = status + self.assertFalse(all(r["ok"] for r in dr.evaluate(packet, bad))) + obs["forbidden"]["code"] = "RATE_LIMITED" + self.assertFalse(dr.evaluate(packet, obs)[-1]["ok"]) + + def test_get_only_bounded_no_body_logging(self): + packet, _, _ = fixture() + response = Mock() + response.__enter__ = Mock(return_value=response); response.__exit__ = Mock(return_value=False) + response.code = 200 + response.read.return_value = json.dumps({"data": {"instrument_uid": packet["baseline"]["instrument_uid"], + "feed": "QUOTE", "private": "not-logged"}}).encode() + opener = Mock(); opener.open.return_value = response + result = dr.get(opener, "https://query_kn", dr.route(packet["baseline"]), {"Authorization": "Bearer secret"}) + self.assertEqual(opener.open.call_args.args[0].method, "GET") + self.assertEqual(opener.open.call_args.kwargs["timeout"], 10) + self.assertNotIn("not-logged", json.dumps(result)) + response.read.return_value = b"x" * (dr.MAX_BYTES + 1) + with self.assertRaises(ValueError): + dr.get(opener, "https://query_kn", "/v2/anything", {}) + + def test_probe_stdin_shape_mtls_and_exact_claim_changes(self): + packet, binding, env = fixture() + env.update(DATA_LAYER_V2_TLS_CA_FILE="/test/ca", DATA_LAYER_V2_TLS_CERT_FILE="/test/cert", + DATA_LAYER_V2_TLS_KEY_FILE="/test/key") + calls = [] + async def signed(env, **changes): + calls.append(changes) + return "SECRET_TEST_TOKEN" + wire = observations(packet) + with patch.dict(dr.os.environ, env, clear=True), patch.object(dr, "read_json", side_effect=[packet, binding]), \ + patch.object(dr.ssl, "create_default_context") as tls, patch.object(dr, "HTTPSHandler"), \ + patch.object(dr, "build_opener"), patch.object(dr, "token", side_effect=signed), \ + patch.object(dr, "get", side_effect=[wire[k] for k in ("baseline", "realm", "revision", "forbidden")]) as get, \ + patch.object(dr.time, "sleep"), patch("sys.stdout", new_callable=io.StringIO) as output: + self.assertEqual(dr.probe(argparse.Namespace(packet="/packet")), 0) + tls.assert_called_once_with(cafile="/test/ca") + tls.return_value.load_cert_chain.assert_called_once_with("/test/cert", "/test/key") + self.assertEqual(calls, [{}, {"environment": "live"}, {"consumer_manifest_revision": 13}, {}]) + self.assertEqual(get.call_count, 4) + self.assertNotIn("SECRET_TEST_TOKEN", output.getvalue()) + self.assertEqual(len(output.getvalue().splitlines()), 4) + + def test_record_exact_two_venue_union_exclusive_and_ci(self): + with tempfile.TemporaryDirectory() as tmp: + paths = [] + for venue in dr.VENUES: + packet, _, _ = fixture(); packet["venue"] = venue + rows = dr.evaluate(packet, observations(packet)) + path = Path(tmp) / venue; path.write_text("\n".join(json.dumps(x) for x in rows)); paths.append(str(path)) + args = argparse.Namespace(rows=paths, output=str(Path(tmp) / "out"), ci_confirmed=False, + source_sha="0123456789" * 4, image_digest="sha256:" + "0123456789abcdef" * 4) + with self.assertRaises(ValueError): dr.record(args) + self.assertFalse(Path(args.output).exists()) + args.ci_confirmed = True + self.assertEqual(dr.record(args), 0) + result = [json.loads(line) for line in Path(args.output).read_text().splitlines()] + self.assertEqual(len(result), 8) + self.assertEqual(result[0]["provenance"]["source_sha"], args.source_sha) + self.assertEqual(sum(r.get("cell") == "DR01.NO_GLOBAL_GRANT@ANY" for r in result), 1) + with self.assertRaises(FileExistsError): dr.record(args) + args.rows = [paths[0], paths[0]] + with self.assertRaises(ValueError): dr.record(args) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p184_dr01_runtime.py b/tests/test_p184_dr01_runtime.py index b6f1dfdd..6d3f7f6c 100644 --- a/tests/test_p184_dr01_runtime.py +++ b/tests/test_p184_dr01_runtime.py @@ -31,6 +31,9 @@ def test_exact_nine_ids(self): self.assertEqual(set(dr.CELLS), expected) self.assertEqual(len(dr.CELLS), 9) + def test_never_runtime_level(self): + self.assertTrue(all(r["level"] == "CONTROLLED_SIM" for r in dr.assess(observations()))) + def test_strict_booleans(self): for bad in (1, 0, "false", [], None): with self.subTest(bad=bad), self.assertRaises(ValueError): diff --git a/upgrade/evidence/p184-dr-runtime-runbook.md b/upgrade/evidence/p184-dr-runtime-runbook.md index d7a6fc17..c9d1a12e 100644 --- a/upgrade/evidence/p184-dr-runtime-runbook.md +++ b/upgrade/evidence/p184-dr-runtime-runbook.md @@ -17,48 +17,81 @@ test source hashes into the enclosing packet. A caller-supplied header alone does not attest code running inside an image. No runner grants mainnet/order authority or permits changing deployed readers/consumers. -## DR01 +## DR01: Regression Versus Deployed Runtime -Use the existing Python image with dependencies, source read-only, a fresh -evidence mount, disposable credentials, bounded CPU/RAM and `--rm`. Initial -DL-only rehearsal uses `--network none`; the HTTP server binds loopback only. +`scripts/p184_dr01_runtime.py` is CONTROLLED_SIM only, including its old account +fixture helpers. It must NEVER close a DR01 runtime cell. Account acceptance +belongs to Claude's TS Gateway runner. New DL script: +`scripts/p184_dr01_deployed.py`; no account cells and no automatic orchestration. + +Offline prepare uses the existing alpha deployment compiler with +stable-v2-release-routing.yaml, catalog, policy, reference manifest and original +alpha inventory. It seals full expected binding bytes and source hashes, exact +manifest request policy and a valid catalog product outside access entitlements. +Do not substitute the TS-only stable renderer for the alpha compiler. + +```text +python3 -B scripts/p184_dr01_deployed.py prepare \ + --root /home/bobby/.worktrees/data-layer-p184-closure \ + --inventory \ + --deployment --venue BINANCE --output +``` + +Repeat for OKX. Host prepare is offline; it does not mount credentials or call +Query. If the exact deployed catalog is supplied with --catalog, its hash is +recorded too. Prepared packets are secret-free but must be retained read-only. + +Claude runs probe through stdin with the existing alpha-noorder-sdk2b.compose.json +identity/mTLS mounts, once per venue, sequentially. Use a disposable compose run, +not exec into a running strategy. Preserve the deployed image/identity; do not +start dependencies, change credentials/entitlements, or launch the alpha entrypoint: + +```text +docker compose -f run --rm --no-deps -T \ + --entrypoint python -v :/run/p184/dr01-packet.json:ro \ + -B - probe --packet /run/p184/dr01-packet.json \ + < scripts/p184_dr01_deployed.py > +``` + +Only four sequential GETs per venue: positive QUOTE from binding, same signed +identity with environment=live (401 UNAUTHENTICATED), revision13 (401 +UNAUTHENTICATED), and real out-of-entitlement instrument/feed (403 +PERMISSION_DENIED). Normal JWT14, realm paper, TLS CA/hostname/client certificate +checks remain enabled. No retries/proxy/redirects. Positive failure, drift or +transport failure cannot become negative PASS. Token/signing material is never +printed or copied. Budget:4 requests per venue, no streams or provider-direct. +Save compose image/config hashes with the packet; caller-supplied TS header alone +is not image attestation. `probe` imports SDK credentials, not DL server code. + +BINDING_COMPILE compares canonical compiled bytes/hash with the actual +/run/p184/binding.json (or declared route-manifest env path). Source inspection +and offline compilation on2026-10-08 found alpha bindings release_revision27, +JWT14, not release_revision22. Both BB deployment hashes matched mounted files: +Binance bbc8b0de1194778f70f94d5cd003fa251f3084250dee3590d4944b4e5f8b8fb8; +OKX29e321e0c6067b34070227598fe8d694f3b491d017ffaa25d4e1073706225ac4. +This is offline binding parity, NOT a runtime DR01 PASS or permission to rewrite +revision. Keep TS bindingr22 terminology separate from alpha release revision. + +Evidence of no wildcard includes parsed concrete selector audit and file hashes. +Archive a bounded human-readable grep alongside the packet: +`rg -n 'instrument_uid:|feed:|source_policy_id:' consumers/stable/alpha-*-paper.yaml`. +Source licensing grants are concrete source IDs in stable_catalog.entitlements; +wire403 is mandatory, not replaceable by this config inspection. + +After both probes, and ONLY when Claude confirms CI green: ```text -python -B scripts/p184_dr01_runtime.py --development \ - --source-sha <40-zeroes> --image-digest sha256:<64-zeroes> \ - --output /evidence/dr01.rehearsal.jsonl +python3 -B scripts/p184_dr01_deployed.py record --rows \ + --ci-confirmed --source-sha --image-digest \ + --output ``` -The runner validates the complete source binding artifact and records its hash -and per-venue product counts before deriving disposable identities. The initial -fixture compiles one BAR-1m ALPHA projection per venue and exercises -real HTTP authentication plus metadata access. It is not all-r22 data access, -production mTLS, or a claim that TS consumed the sealed binding. Wrong account -cannot be inferred from wrong consumer/subject. Without an account packet, -those two cells explicitly remain NOT_OBSERVED and the runner exits nonzero. - -For account checks, run an isolated final-image TS Gateway and isolated -PostgreSQL/Redis. Reuse TS's existing `test_p18e_current_gateway_http.py` setup -contract; do not modify the shared Gateway or write into shared accounts. -Create two active test alphas with separate generated API keys, each with a -distinct Paper account per venue. Hash keys through TS's existing helper. -Include all four accounts in `WRITER_ACCOUNT_SCOPE_JSON`, enable API-key auth, -disable plaintext migration, and seed distinguishable test-only balances. - -Use `--account-packet /private/account-packet.json`; the exact schema is in -`load_account_packet()`. It binds source/image, literal loopback Gateway URL, -two venue pairs, account/alpha IDs, expected decimal balances, and external -mode-0600 key files. Run the driver in the Gateway's isolated network namespace, -never host networking. Keep fixture infrastructure within the same isolated -namespace and no broker credentials. No shared ports/networks/mounts. - -Per venue the probe requires A->A and B->B nonempty exact balances, A->B on the -same venue returning `403 ACCOUNT_ALPHA_SCOPE_MISMATCH`, and a separate invalid -key refusal. GET-only, no redirects, no order endpoints. This proves application -account authorization, not broker-side permissions. Auth/rate counters may change -only inside disposable Redis. Inspect financial/command tables before/after and -retire credentials with the fixture. Merely setting `GATEWAY_NO_ORDER_ONLY` is -not a security boundary; isolated topology and the GET-only driver are required. +Recorder requires both venue rosters, merges NO_GLOBAL_GRANT into one all-venue +cell and emits exactly7 DL cells. Claude separately supplies2 WRONG_ACCOUNT_DENIED +cells; never fabricate them here. Output is exclusive, no receipt overwrite. +Provisional TS source2abbe394ea0fb8f7f5ebcbb745ff70a8c83c9769 and +sha256:a6548ee36aeb0242c1734c071678c525c1506d05c10a70b533eb7110e7eaf16a +are NOT certified by this handoff; CI confirmation still required. ## DR08 @@ -76,6 +109,13 @@ python3 scripts/p184_dr08_isolated.py --orchestrate \ The synthetic input is explicitly test-only. Test Query read-view support is needed for the actual GetSnapshot RPC, not a locally invented recovery cursor. +Query provenance MUST explicitly name qdl-v2-python:2.2.3-rc2-43bbcf0 and actual +inspected digest77682eea; this is NOT production Queryv2.2.2 evidence. Final mode +requires --restart-samples5 (default5; allowed5-10), each a new consumer process +with an independently observed advancing trade. Receipt retains raw monotonic +start/receive times, sample list, n, p50 and max in ms. The first OUT_OF_RANGE to +snapshot recovery remains n=1, reported separately from repeated restart latency. +Old developmentn=1 is not promoted. No final measurement has run in this slice. Queue1000 and replay10000 remain fixed. Two distinct consumers must demonstrate slow-consumer overflow, fast-consumer progress, observable control/status/counters, and recovery via a newer snapshot after OUT_OF_RANGE. Separate configured limits From b2c8c5696a7e6b9bdcf2d1f97e94f837f5ba0fe8 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 09:06:39 +0000 Subject: [PATCH 030/147] test(sdk): verify complete wheel records for 2.1.0 handoff --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 35 +++++++++++++++++++++++ qdl_sdk/README.md | 2 +- tests/test_qdl_sdk_release.py | 12 ++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index a1d1ea05..a90965dd 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70089,3 +70089,38 @@ feature /home/bobby/.worktrees/data-layer-p184-closure fix/p184-dl-closure. Active runtime remains as previous inventory; patch not deployed. Keep BAR rollbackf7351c3b. Next: Claude reviews and runs DR window with final CI-green provenance, then combines Gateway cells/DR08/broker delta; v2.2.3 staysHOLD. + + +### Pre-stage3 consolidated build, SDK and DR05 - 2026-10-08 (Codex) + +Status IMPLEMENTING / RELEASE_HOLD. Owner confirms TS2abbe394 CI green +(unit11415/integration3068 no failures), imagea6548ee3. This authorizes provenance +when scheduled, NOT immediate DR01 deployed/DR08 final execution. Work now: +(1) single-source commit build recipe/CI for IDR039, including IDR040fix1, bindingV3, +quota logs, b6a0119 counters and IDR075; release rebuild waits until TS tests end; +(2) reproducible qdl-sdk2.1.0 wheel handoff for TS/alpha without consumer rollout; +(3) isolated DL DR05 GAP/GENERATION preparation/tests. No shared runtime, grants, +credential or image changes. No DR01/DR08 final probes. New TS data findings +arrive via INTEGRATION_DEFECT_REGISTER, not speculative broad changes. +Tests: source/build-contract, wheel RECORD/reproducibility/clean-install SDK +compatibility; DR05 fault fixture isolation and exact typed gap/generation safety. +Every slice tested+journalled, no push/merge. Cleanup owned --rm resources only; +keep active and BARf7351c3b rollback. Release rebuild/rollout remains later packet. +Guides: upgrade/evidence/p184-dl-phase-c-packet.md and p184-dr-runtime-runbook.md. + + +### SDK2.1.0 reproducible package preparation - 2026-10-08 (Codex) + +Status SOURCE_TESTED / ARTIFACT_PUBLICATION_IN_PROGRESS / DL_RELEASE_HOLD. +Existing build_qdl_sdk_release.py is the canonical deterministic wheel builder; +no new packaging system. Correct packaged README2.0.4->2.1.0 and strengthen +RECORD test to verify every payload digest/size, no duplicate entries. SDK code, +dependencies and generated contracts unchanged. Existing image77682eea, network +none, sourceRO, --rm: python -B -m unittest discover -s tests -p '*sdk*.py' -q: +89PASS0fail0skip11.467s. Includes reconnect/offsetzero, replica, executionquality, +projection, public API and reproduciblewheel tests. Disposable removed. +Freeze this package source before building consumer artifact outside Git; then +clean-install test on exact TS a6548ee3 and alpha a03c62f0 images without shared +network/config/identity. No application image rebuild or consumer restart. +Public GitHub wheel publication follows existing certified DL tag workflow; +local immutable hash-pinned handoff does not publish v2.2.3 or bypass HOLD. diff --git a/qdl_sdk/README.md b/qdl_sdk/README.md index 05ea9d98..e018b49f 100644 --- a/qdl_sdk/README.md +++ b/qdl_sdk/README.md @@ -146,7 +146,7 @@ Build the standalone artifact with: python scripts/build_qdl_sdk_release.py --output-dir dist/qdl-sdk ``` -The output contains a reproducible `qdl_sdk-2.0.4-py3-none-any.whl`, a release +The output contains a reproducible `qdl_sdk-2.1.0-py3-none-any.whl`, a release manifest with the wheel/source/generated-contract SHA-256 digests, and a CycloneDX SBOM. The wheel contains only the public SDK plus generated Protobuf contracts; it does not package `qdl.api_v2`, runtime adapters, provider code or diff --git a/tests/test_qdl_sdk_release.py b/tests/test_qdl_sdk_release.py index 0131765b..f8f7dedf 100644 --- a/tests/test_qdl_sdk_release.py +++ b/tests/test_qdl_sdk_release.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 import csv import hashlib import io @@ -31,6 +32,17 @@ def test_wheel_is_reproducible_self_contained_and_recorded(self): record_name = f"qdl_sdk-{first_result['version']}.dist-info/RECORD" rows = list(csv.reader(io.StringIO(archive.read(record_name).decode()))) self.assertEqual({row[0] for row in rows}, names) + self.assertEqual(len(rows), len(names), "duplicate RECORD entries") + for name, encoded, size in rows: + if name == record_name: + self.assertEqual((encoded, size), ("", "")) + continue + payload = archive.read(name) + expected = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()).rstrip(b"=").decode() + self.assertEqual(encoded, "sha256=" + expected) + self.assertEqual(int(size), len(payload)) + self.assertIn("qdl_sdk-2.1.0-py3-none-any.whl", archive.read("qdl_sdk/README.md").decode()) + release = json.loads(Path(first_result["manifest"]).read_text()) self.assertFalse(release["contains_service_internals"]) self.assertEqual(release["version"], "2.1.0") From 52642e52e01981590bc433bf95ac12d4e941cf10 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 09:18:25 +0000 Subject: [PATCH 031/147] docs(sdk): publish local 2.1.0 artifact and consumer install receipt --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 30 ++++++++ .../qdl-sdk-2.1.0-consumer-handoff.md | 76 +++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 upgrade/evidence/qdl-sdk-2.1.0-consumer-handoff.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index a90965dd..dc3e3589 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70124,3 +70124,33 @@ clean-install test on exact TS a6548ee3 and alpha a03c62f0 images without shared network/config/identity. No application image rebuild or consumer restart. Public GitHub wheel publication follows existing certified DL tag workflow; local immutable hash-pinned handoff does not publish v2.2.3 or bypass HOLD. + + +### SDK2.1.0 consumer artifact handoff receipt - 2026-10-08 (Codex) + +Status LOCAL_ARTIFACT_PUBLISHED / TS_INSTALL_PASS / ALPHA_DEPENDENCY_UPDATE_REQUIRED. +Guide upgrade/evidence/qdl-sdk-2.1.0-consumer-handoff.md. Package source +b2c8c5696a7e6b9bdcf2d1f97e94f837f5ba0fe8; actual source files checked against +that commit before attestation. Artifact outsideGit: +~/.local/state/qdl-v2/sdk-2.1.0-b2c8c56/artifacts/qdl_sdk-2.1.0-py3-none-any.whl +SHA2566923b46f46bd07c850b18a6074439bbcb4085d84428a8e72c476437dd51524e7. +Manifest/SBOM alongside. consumer-install-receipt.json SHA256 +8a52a9b6967a5e516882f1988761c1a078cc7f2320ccdafb7489b06f17095b00. + +Isolated --rm/network none/pip --no-index --no-deps target installs: TS image +a6548ee3 PASS; alpha a03c62f0 wheel imports PASS but existing PyJWT2.13.0 is below +SDKdeclared>=2.15.1, so alpha AS-IS DEPENDENCY CHECK FAIL. Do not lower requirement. +Downloaded exact public PyJWT2.15.1 wheel only (33KB), SHA256 +42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193; alpha disposable +install with this wheel PASS, actual sdk/jwt paths under/tmp/sdk-only verified. +All5declared dependency ranges checked with packaging.Requirement, protobuf +query/marketdata imports PASS. This is packaging evidence, not DR or runtime +acceptance; Claude must update alpha build dependency before consumer rollout. +Initial smoke errors (venv pip absent, helper packaging absent, import cache +selecting preinstalled SDK) corrected in harness only; record retained. + +All own test containers removed, no images built, no shared runtime/config/key +mount. Retained deliverable128KB, not cleanup waste. No public tag/PyPI/GitHub +publish: existing certified-tag release workflow still owns public distribution; +local hash-pinned artifact is now available to both consumer builds. v2.2.3 HOLD. +Canonical main5bf5cd5/v2.2.2 unchanged; feature fix/p184-dl-closure; no push/merge. diff --git a/upgrade/evidence/qdl-sdk-2.1.0-consumer-handoff.md b/upgrade/evidence/qdl-sdk-2.1.0-consumer-handoff.md new file mode 100644 index 00000000..c55eb26f --- /dev/null +++ b/upgrade/evidence/qdl-sdk-2.1.0-consumer-handoff.md @@ -0,0 +1,76 @@ +# QDL SDK 2.1.0 Consumer Artifact + +Status: LOCAL_ARTIFACT_PUBLISHED / TS_INSTALL_PASS / +ALPHA_REQUIRES_PYJWT_UPDATE. No shared runtime or consumer was changed. +Public GitHub publication remains part of the certified DL release workflow; +this handoff does not publish the held v2.2.3 tag or certify that DL runtime. + +## Artifact + +Package source: `b2c8c5696a7e6b9bdcf2d1f97e94f837f5ba0fe8`. +Builder: `scripts/build_qdl_sdk_release.py` (existing deterministic builder). +Immutable handoff directory, outside Git: +`/home/bobby/.local/state/qdl-v2/sdk-2.1.0-b2c8c56/`. + +- Wheel: `artifacts/qdl_sdk-2.1.0-py3-none-any.whl`. +- SHA256: `6923b46f46bd07c850b18a6074439bbcb4085d84428a8e72c476437dd51524e7`. +- Manifest: `artifacts/qdl_sdk-2.1.0.release.json`; SBOM: `artifacts/qdl_sdk-2.1.0.cdx.json`. +- SDK source digest: `f17c883fa78d0b77bd29830d74971ccc2b53c79c90a8d98e7fbcda040432be9d`. +- Contract digest: `2a25a601ae80b2ee4b36e6d1cb8ca44a13b8a037bf69a96f45659d1782a3a624`. +- Install receipt: `consumer-install-receipt.json` in that directory. + +TS and alpha must consume the SAME wheel hash. Do not copy selected SDK files, +install a different artifact under version 2.1.0, or change live site-packages. +If the eventual one-SHA DL build changes package bytes, fail artifact parity and +review that change before promotion; do not overwrite this artifact/evidence. + +## Verification + +SDK source suite: `python -B -m unittest discover -s tests -p '*sdk*.py' -q`: +**89 PASS, zero failures/skips**, 11.467 seconds in existing Python image +`sha256:77682eea5054ae9ddbd858323b083286669a55eeb4bc7bc065b04bae8854e32f`. +The wheel test builds twice and checks deterministic bytes plus every RECORD +digest/size, including generated protobuf files and absence of service internals. + +Clean target installs used disposable containers, `--network none`, no credentials, +wheel mounted read-only, no source mount, `--no-index --no-deps`, and `/tmp/sdk-only`. +Python import caches were invalidated after installation; version AND module path +were asserted. Both generated query and market-data protobuf modules imported. +This is package compatibility, not a live consumer acceptance window. + +| Consumer image | Result | +| --- | --- | +| TS `sha256:a6548ee36aeb0242c1734c071678c525c1506d05c10a70b533eb7110e7eaf16a` | PASS with existing dependencies | +| Alpha `sha256:a03c62f01504e024205c535b9a7fae0afcde1d504e1386c636d9a1ea5ea71089` | Existing PyJWT 2.13.0 is BELOW declared minimum; not dependency-compatible as-is | +| Same alpha image, disposable target with PyJWT 2.15.1 | PASS, including exact target module paths | + +Passing dependency versions: grpcio 1.83.0, httpx 0.28.1, protobuf 6.33.6, +pydantic 2.13.4, PyJWT 2.15.1, cryptography 50.0.0. SDK declared version +specifiers were checked using `packaging.requirements.Requirement`. + +The alpha correction is a build dependency input for Claude, NOT a change to the +running alpha or an excuse to lower the SDK requirement. Offline dependency: +`dependencies/pyjwt-2.15.1-py3-none-any.whl`, downloaded from PyPI without running +provider code, SHA256 `42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193`. +Keep all other locked dependencies unchanged; do not broadly upgrade packages. + +Initial smoke harness failures are retained in the receipt: alpha venv has no pip, +TS has no standalone packaging library, and importing immediately after target +installation initially selected the existing SDK due to import-cache state. +These were corrected in the isolated test method, not application code. No +success was claimed until imports came from the installed wheel's target path. + +## Consumer Build Handoff + +1. Verify both wheel hashes before adding artifacts to an isolated build context. +2. Pin qdl-sdk 2.1.0 by the exact artifact hash in TS and alpha build inputs. +3. In alpha only, update the locked PyJWT dependency to the tested 2.15.1 artifact; + keep crypto extras and the other approved dependency locks consistent. +4. Run caller-specific tests and packaging probes on the resulting immutable + image. Do not treat this install smoke as DR01, DR05 or DR08 acceptance. +5. Wait for Claude's stage-3 window. No deployed DR01 or final DR08 has run here. + +All disposable install/test containers were `--rm`; no application images were +built, no shared Redis/DB/network/credentials were mounted, and no alpha was +started. The small artifact directory is the retained consumer deliverable, not +disposable build cache. BAR rollback `f7351c3b` remains retained. From 635647ac7c5304534d41a6627b1c86c97806dd34 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 09:25:03 +0000 Subject: [PATCH 032/147] build(p184): prepare one-commit DL release recipe with CI preflight --- .github/workflows/ci.yml | 72 ++++++++ DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 31 ++++ scripts/p184_consolidated_build.py | 183 +++++++++++++++++++ tests/test_p184_consolidated_build.py | 175 ++++++++++++++++++ upgrade/evidence/p184-consolidated-build.md | 187 ++++++++++++++++++++ 5 files changed, 648 insertions(+) create mode 100644 scripts/p184_consolidated_build.py create mode 100644 tests/test_p184_consolidated_build.py create mode 100644 upgrade/evidence/p184-consolidated-build.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 820ae835..43de8799 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,78 @@ on: branches: ["dev"] jobs: + consolidated-source-preflight: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout exact CI revision with prerequisite history + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + + - name: Portable consolidated recipe unit tests + run: python -B -m unittest tests.test_p184_consolidated_build -v + + - name: Verify exact committed source and expected HOLD + run: | + python -B - <<'PY' + import json + import os + import re + import subprocess + import sys + from scripts.p184_consolidated_build import CONTRACTS, PREREQUISITES + + sha = os.environ["GITHUB_SHA"] + assert re.fullmatch(r"[0-9a-f]{40}", sha), "Expected full CI commit SHA" + head = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + assert head == sha, "Checkout differs from exact CI source" + result = subprocess.run( + [sys.executable, "-B", "scripts/p184_consolidated_build.py", + "--repo", ".", "--source-commit", sha, "--release", "v2.2.3"], + capture_output=True, text=True, check=False, + ) + assert result.returncode == 2, (result.returncode, result.stderr) + assert not result.stderr, result.stderr + recipe = json.loads(result.stdout) + assert recipe["schema"] == "qdl.p184.consolidated-build-recipe.v1" + assert recipe["source_commit"] == sha + assert recipe["status"] == "HOLD" + assert recipe["build_script"] is None + assert recipe["build_script_sha256"] is None + assert recipe["prerequisites"] == PREREQUISITES + assert recipe["gates"] == { + "ts_tests": "pending", "ts_source_commit": None, + "ts_receipt_sha256": None, "dl_ci": "pending", + "dl_ci_source_commit": sha, "dl_ci_receipt_sha256": None, + "build_approval": "pending", + } + assert recipe["holds"] == [ + "TS tests must finish and pass before any rebuild", + "DL CI must pass for the exact source commit", + "explicit build approval required", + ] + blobs = recipe["source_git_blobs"] + assert set(blobs) == set(CONTRACTS) + assert all(re.fullmatch(r"[0-9a-f]{40}", value) for value in blobs.values()) + images = recipe["images"] + assert {(image["name"], image["dockerfile"]) for image in images} == { + ("qdl-v2-python", "Dockerfile"), + ("qdl-v2-rust", "Dockerfile.qdl-rust-runtime"), + } + assert len(images) == 2 + assert all(image["source_commit"] == sha for image in images) + assert all(image["tag"] == f'{image["name"]}:v2.2.3-{sha}' for image in images) + print(f"Committed-source preflight PASS: {sha}; build remains HOLD") + PY + contract-tests: runs-on: ubuntu-latest steps: diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index dc3e3589..b090da87 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70154,3 +70154,34 @@ mount. Retained deliverable128KB, not cleanup waste. No public tag/PyPI/GitHub publish: existing certified-tag release workflow still owns public distribution; local hash-pinned artifact is now available to both consumer builds. v2.2.3 HOLD. Canonical main5bf5cd5/v2.2.2 unchanged; feature fix/p184-dl-closure; no push/merge. + + +### IDR039 one-commit build recipe and CI slice - 2026-10-08 (Codex) + +Status IMPLEMENTED_TESTED_LOCALLY / CI_CONFIGURED_NOT_RUN / BUILD_HOLD. +Guide upgrade/evidence/p184-consolidated-build.md; generator +scripts/p184_consolidated_build.py reads only full committed Git objects, checks +ancestry/source contracts for IDR040fix1, bindingV3, quota log, b6a0119, IDR075. +One private git archive is the source context for BOTH Python and Rust images; +exact OCI revision/IDs checked, no overlays or per-role application source. +Pending TS acceptance completion, exact-source DL CI and build approval produce +HOLD with NO executable build script. Confirmed TS unit/integration CI is not +substituted for completion of the pending TS acceptance window. No build executed. +Final archived-source SDK rebuild must match handed-off wheel6923b46f; drift +requires explicit review, never overwrite consumer artifacts. + +New CI consolidated-source-preflight job uses pinned checkout/setupPython and +full Git history; proves exact github.sha source contracts and expectedHOLD. +Existing deployment/release jobs unchanged. Initial real-Git unittest dependency +would fail in application image without git/.git; moved real-source check to +host CIjob and kept portable tests mocked. Parent verification:18PASS0fail0skip +in network-none image77682eea; actual CI Pythonblock locally PASS against +52642e52e01981590bc433bf95ac12d4e941cf10 (host python3, since no python alias). +This is NOT remote CIgreen; workflow not dispatched or pushed. Source-marker +checks do not replace semantic/package/DR regression. Operator receipt digests +are attestations, not automatic remoteCI validation. +Cleanup: no images built, temporary test container --rm removed, no runtime +changes. Active/rollback retained, f7351c3b protected. Latest readonly disk +inventory95images/28.96GB, BuildKit20.52GB/2.852GB reclaimable; no broadprune +during Claude concurrent work. Canonicalmain5bf5cd5/v2.2.2 unchanged; same +featureworktree/branch, no push/merge. Rebuild only after owner timing gate. diff --git a/scripts/p184_consolidated_build.py b/scripts/p184_consolidated_build.py new file mode 100644 index 00000000..a25f7b55 --- /dev/null +++ b/scripts/p184_consolidated_build.py @@ -0,0 +1,183 @@ +"""Offline IDR039 recipe preparation. Never execute builds or activate runtime.""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import shlex +import subprocess + + +PREREQUISITES = { + "IDR040fix1": "4eb08dc15113cc4f73b65b239dfeb4f8ec0e23cf", + "bindingV3": "4eb08dc15113cc4f73b65b239dfeb4f8ec0e23cf", + "quota_log": "4eb08dc15113cc4f73b65b239dfeb4f8ec0e23cf", + "b6a0119": "b6a01190587e8771c62b5b4988782b06c5e2578a", + "IDR075": "704552d8736905cdb73feec86f3f01d339d02cbb", +} +CONTRACTS = { + "rust/qdl-realtime-core/src/lib.rs": ["REST_AHEAD_OF_STREAM"], + "qdl/consumer/universal_release.py": ["qdl.v2.consumer-route-binding.v3"], + "qdl/security/data_plane.py": ["qdl_consumer_quota_exhausted"], + "rust/qdl-stream-gateway/src/subscription.rs": ["pub offered:", "pub dequeued:"], + "scripts/prepare_paper_read_extension.py": ["def prepare_paper_extension("], + "Dockerfile": ["ARG QDL_GIT_SHA=", "ARG QDL_RELEASE=", "COPY --chown=qdl:qdl . /app"], + "Dockerfile.qdl-rust-runtime": [ + "ARG QDL_GIT_SHA=", "ARG QDL_RELEASE=", "COPY rust ./rust", + "COPY generated/rust ./generated/rust", "cargo build --release --locked", + "--bin qdl-projector", "--bin qdl-stream-gateway", + ], + ".dockerignore": [".git"], + "scripts/verify_runtime_dependencies.py": ["dependency image must be digest-pinned"], + "poetry.lock": [], + "Cargo.lock": [], + "rust-toolchain.toml": [], + "generated/rust/qdl.query.v2.tonic.rs": [], + "tests/test_paper_read_extension.py": [], + "tests/test_phase115_universal_release.py": [], + "tests/test_fund_phase71_beta_runtime.py": [], +} + + +def git(repo, *args): + return subprocess.check_output( + ["git", "-C", str(repo), *args], stderr=subprocess.PIPE + ).decode().strip() + + +def full_sha(value, field, length=40): + if not re.fullmatch(r"[0-9a-f]{%d}" % length, value or ""): + raise ValueError(f"{field} must be a full lowercase {length}-hex digest") + return value + + +def source_contract(repo, commit): + full_sha(commit, "source commit") + if git(repo, "rev-parse", commit + "^{commit}") != commit: + raise ValueError("source must identify a commit, not a tag object") + for name, ancestor in PREREQUISITES.items(): + try: + git(repo, "merge-base", "--is-ancestor", ancestor, commit) + except subprocess.CalledProcessError as exc: + raise ValueError(f"missing prerequisite ancestry: {name}") from exc + hashes = {} + for path, markers in CONTRACTS.items(): + content = git(repo, "show", f"{commit}:{path}") + if any(marker not in content for marker in markers): + raise ValueError(f"source contract changed: {path}") + hashes[path] = git(repo, "rev-parse", f"{commit}:{path}") + return hashes + + +def prepare(repo, commit, release, *, ts_status="pending", ts_commit=None, + ts_receipt=None, dl_ci_status="pending", dl_ci_receipt=None, + build_approval="pending", dependency_image="builder"): + full_sha(commit, "source commit") + if not re.fullmatch(r"v2\.\d+\.\d+(?:-rc\.\d+)?", release): + raise ValueError("release must be v2.N.N or v2.N.N-rc.N") + if ts_status not in {"pending", "pass"} or dl_ci_status not in {"pending", "pass"}: + raise ValueError("test statuses must be pending or pass") + if build_approval not in {"pending", "approved"}: + raise ValueError("build approval must be pending or approved") + if ts_status == "pass": + full_sha(ts_commit, "TS tested source commit") + full_sha(ts_receipt, "TS completed-test receipt SHA256", 64) + if dl_ci_status == "pass": + full_sha(dl_ci_receipt, "DL CI receipt SHA256 for source commit", 64) + if dependency_image != "builder" and not re.fullmatch( + r"[a-z0-9][a-z0-9./:_-]*@sha256:[0-9a-f]{64}", dependency_image + ): + raise ValueError("dependency image must be builder or a digest-pinned image") + blobs = source_contract(repo, commit) + holds = [] + if ts_status != "pass": + holds.append("TS tests must finish and pass before any rebuild") + if dl_ci_status != "pass": + holds.append("DL CI must pass for the exact source commit") + if build_approval != "approved": + holds.append("explicit build approval required") + images = [ + {"name": "qdl-v2-python", "dockerfile": "Dockerfile"}, + {"name": "qdl-v2-rust", "dockerfile": "Dockerfile.qdl-rust-runtime"}, + ] + for image in images: + image["tag"] = f"{image['name']}:{release}-{commit}" + image["source_commit"] = commit + script = None + if not holds: + # A private archive context excludes all staged/untracked source overlays. + lines = [ + "set -eu", + 'build_dir=$(mktemp -d)', + "trap 'rm -rf -- \"$build_dir\"' EXIT", + 'mkdir "$build_dir/source"', + f"git -C {shlex.quote(str(Path(repo).resolve()))} archive --format=tar {commit} > \"$build_dir/source.tar\"", + 'sha256sum "$build_dir/source.tar"', + 'tar -xf "$build_dir/source.tar" -C "$build_dir/source"', + ] + for image in images: + iid = f"$build_dir/{image['name']}.iid" + dependency_arg = ( + f"--build-arg QDL_DEPENDENCY_IMAGE={dependency_image} " + if image["dockerfile"] == "Dockerfile" else "" + ) + lines.extend([ + f"docker build --file \"$build_dir/source/{image['dockerfile']}\" " + f"--build-arg QDL_GIT_SHA={commit} --build-arg QDL_RELEASE={release} " + f"{dependency_arg}--iidfile \"{iid}\" --tag {image['tag']} \"$build_dir/source\"", + f'test "$(docker image inspect --format \'{{{{index .Config.Labels "org.opencontainers.image.revision"}}}}\' "$(cat "{iid}")")" = {commit}', + f'docker image inspect --format \'{{{{.Id}}}} {{{{json .Config.Labels}}}}\' "$(cat "{iid}")"', + ]) + script = "\n".join(lines) + "\n" + return { + "schema": "qdl.p184.consolidated-build-recipe.v1", + "status": "HOLD" if holds else "PREPARED_NOT_EXECUTED", + "source_commit": commit, "release": release, + "dependency_image": dependency_image, + "prerequisites": PREREQUISITES, "source_git_blobs": blobs, + "gates": {"ts_tests": ts_status, "ts_source_commit": ts_commit, + "ts_receipt_sha256": ts_receipt, "dl_ci": dl_ci_status, + "dl_ci_source_commit": commit, "dl_ci_receipt_sha256": dl_ci_receipt, + "build_approval": build_approval}, + "holds": holds, "images": images, "build_script": script, + "build_script_sha256": hashlib.sha256(script.encode()).hexdigest() if script else None, + "limitations": [ + "Gate inputs are operator attestations, not independently verified CI results.", + "Ancestry and source markers do not replace semantic regression tests.", + "One application commit does not make upstream package resolution reproducible.", + "No image built, pushed, deployed or certified by this generator.", + "IDR075 remains prepared, not activated; IDR040fix2 remains deferred.", + ], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--source-commit", required=True) + parser.add_argument("--release", required=True) + parser.add_argument("--ts-tests-status", choices=["pending", "pass"], default="pending") + parser.add_argument("--ts-source-commit") + parser.add_argument("--ts-tests-receipt-sha256") + parser.add_argument("--dl-ci-status", choices=["pending", "pass"], default="pending") + parser.add_argument("--dl-ci-receipt-sha256") + parser.add_argument("--build-approval", choices=["pending", "approved"], default="pending") + parser.add_argument("--dependency-image", default="builder") + args = parser.parse_args() + try: + recipe = prepare( + args.repo, args.source_commit, args.release, + ts_status=args.ts_tests_status, ts_commit=args.ts_source_commit, + ts_receipt=args.ts_tests_receipt_sha256, dl_ci_status=args.dl_ci_status, + dl_ci_receipt=args.dl_ci_receipt_sha256, build_approval=args.build_approval, + dependency_image=args.dependency_image, + ) + except (ValueError, subprocess.CalledProcessError) as exc: + parser.error(str(exc)) + print(json.dumps(recipe, indent=2, sort_keys=True)) + return 2 if recipe["holds"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_p184_consolidated_build.py b/tests/test_p184_consolidated_build.py new file mode 100644 index 00000000..1e79aa91 --- /dev/null +++ b/tests/test_p184_consolidated_build.py @@ -0,0 +1,175 @@ +"""Offline recipe tests. No Docker or network calls.""" + +import io +import json +from pathlib import Path +import subprocess +import sys +import unittest +from unittest.mock import patch + +from scripts import p184_consolidated_build as build + +ROOT = Path(__file__).resolve().parents[1] +SHA = "a" * 40 +READY = dict(ts_status="pass", ts_commit="b" * 40, ts_receipt="c" * 64, + dl_ci_status="pass", dl_ci_receipt="d" * 64, build_approval="approved") + + +class NoProcessTests(unittest.TestCase): + def setUp(self): + self.process_guard = patch.object( + subprocess, "Popen", side_effect=AssertionError("unit tests must not launch processes") + ) + self.process_guard.start() + self.addCleanup(self.process_guard.stop) + + +class RecipeTests(NoProcessTests): + def prepare(self, **kwargs): + with patch.object(build, "source_contract", return_value={"Dockerfile": SHA}): + return build.prepare(ROOT, SHA, "v2.2.3-rc.1", **kwargs) + + def test_default_hold_emits_no_build_script(self): + recipe = self.prepare() + self.assertEqual(recipe["status"], "HOLD") + self.assertEqual(len(recipe["holds"]), 3) + self.assertIsNone(recipe["build_script"]) + + def test_each_gate_is_independently_required(self): + for key in ("ts_status", "dl_ci_status", "build_approval"): + with self.subTest(key=key): + recipe = self.prepare(**{**READY, key: "pending"}) + self.assertEqual(recipe["status"], "HOLD") + self.assertIsNone(recipe["build_script"]) + + def test_pass_requires_full_evidence_digests_and_ts_commit(self): + for key in ("ts_commit", "ts_receipt", "dl_ci_receipt"): + for invalid in (None, "", "abc", "G" * 64): + with self.subTest(key=key, invalid=invalid), self.assertRaises(ValueError): + self.prepare(**{**READY, key: invalid}) + + def test_statuses_are_not_truthy_strings(self): + for key in ("ts_status", "dl_ci_status", "build_approval"): + with self.subTest(key=key), self.assertRaises(ValueError): + self.prepare(**{**READY, key: "true"}) + + def test_release_rejects_shell_and_mutable_tags(self): + for release in ("latest", "v2.2.3;true", "v2.2.3\n", "v2.2.3/overlay"): + with self.subTest(release=release), self.assertRaises(ValueError): + build.prepare(ROOT, SHA, release) + + def test_exactly_two_images_share_commit_and_archive(self): + recipe = self.prepare(**READY) + self.assertEqual(recipe["status"], "PREPARED_NOT_EXECUTED") + self.assertEqual(len(recipe["images"]), 2) + script = recipe["build_script"] + self.assertEqual(script.count(" archive --format=tar "), 1) + self.assertEqual(script.count("docker build "), 2) + self.assertEqual(script.count(f"QDL_GIT_SHA={SHA}"), 2) + self.assertEqual(script.count('"$build_dir/source"\n'), 4) + self.assertEqual(script.count("org.opencontainers.image.revision"), 2) + for image in recipe["images"]: + self.assertEqual(image["source_commit"], SHA) + self.assertTrue(image["tag"].endswith(SHA)) + for forbidden in ("docker push", "docker run", "docker compose", "--build-context", "git checkout"): + self.assertNotIn(forbidden, script) + + def test_recipe_generation_executes_no_commands_with_mocked_source(self): + with patch.object(build.subprocess, "check_output", side_effect=AssertionError("execution")): + self.prepare(**READY) + + def test_dependency_image_must_be_pinned(self): + for invalid in ("qdl:latest", "sha256:" + "a" * 64, "x;true@sha256:" + "a" * 64): + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + self.prepare(**READY, dependency_image=invalid) + + def test_dependency_reuse_is_python_only_not_source_overlay(self): + dependency = "registry.example/qdl-deps@sha256:" + "e" * 64 + script = self.prepare(**READY, dependency_image=dependency)["build_script"] + self.assertEqual(script.count("QDL_DEPENDENCY_IMAGE=" + dependency), 1) + self.assertNotIn("--build-context", script) + self.assertEqual(script.count(" archive --format=tar "), 1) + + def test_prerequisite_roster(self): + self.assertEqual(set(build.PREREQUISITES), + {"IDR040fix1", "bindingV3", "quota_log", "b6a0119", "IDR075"}) + + +class SourceTests(NoProcessTests): + def test_reject_mutable_refs_and_abbreviations_before_git(self): + for value in ("HEAD", "fix/p184-dl-closure", "b6a0119", "a" * 39, "A" * 40): + with self.subTest(value=value), patch.object(build, "git") as git: + with self.assertRaises(ValueError): + build.source_contract(ROOT, value) + git.assert_not_called() + + def test_missing_ancestor_fails_closed(self): + with patch.object(build, "git", side_effect=[SHA, subprocess.CalledProcessError(1, "git")]): + with self.assertRaisesRegex(ValueError, "prerequisite ancestry"): + build.source_contract(ROOT, SHA) + + def test_missing_marker_fails_closed(self): + with patch.object(build, "git", side_effect=[SHA] + [""] * 6): + with self.assertRaisesRegex(ValueError, "source contract changed"): + build.source_contract(ROOT, SHA) + + def test_source_contract_uses_only_requested_commit(self): + def fake_git(repo, *args): + if args == ("rev-parse", SHA + "^{commit}"): + return SHA + if args[0] == "merge-base": + self.assertEqual(args[1], "--is-ancestor") + self.assertIn(args[2], build.PREREQUISITES.values()) + self.assertEqual(args[3], SHA) + return "" + self.assertTrue(args[1].startswith(SHA + ":")) + path = args[1].split(":", 1)[1] + self.assertIn(path, build.CONTRACTS) + return "\n".join(build.CONTRACTS[path]) if args[0] == "show" else "f" * 40 + + with patch.object(build, "git", side_effect=fake_git): + blobs = build.source_contract(ROOT, SHA) + self.assertEqual(blobs, dict.fromkeys(build.CONTRACTS, "f" * 40)) + + def test_git_wrapper_is_mocked(self): + with patch.object(subprocess, "check_output", return_value=(SHA + "\n").encode()) as call: + self.assertEqual(build.git(ROOT, "rev-parse", "HEAD"), SHA) + call.assert_called_once_with( + ["git", "-C", str(ROOT), "rev-parse", "HEAD"], stderr=subprocess.PIPE + ) + + def test_cli_hold_is_machine_readable_and_nonzero(self): + argv = ["recipe", "--source-commit", SHA, "--release", "v2.2.3"] + with patch.object(sys, "argv", argv), patch.object( + build, "source_contract", return_value={"Dockerfile": "f" * 40} + ), patch("sys.stdout", new_callable=io.StringIO) as output: + self.assertEqual(build.main(), 2) + recipe = json.loads(output.getvalue()) + self.assertEqual(recipe["status"], "HOLD") + self.assertEqual(recipe["source_commit"], SHA) + self.assertIsNone(recipe["build_script"]) + + def test_cli_source_error_is_not_hold_json(self): + argv = ["recipe", "--source-commit", SHA, "--release", "v2.2.3"] + with patch.object(sys, "argv", argv), patch.object( + build, "source_contract", side_effect=ValueError("missing prerequisite") + ), patch("sys.stdout", new_callable=io.StringIO) as output, patch( + "sys.stderr", new_callable=io.StringIO + ) as error: + with self.assertRaises(SystemExit) as raised: + build.main() + self.assertEqual(raised.exception.code, 2) + self.assertEqual(output.getvalue(), "") + self.assertIn("missing prerequisite", error.getvalue()) + + def test_prepare_checks_sha_even_when_source_checker_is_mocked(self): + for invalid in ("HEAD", "a" * 39, "A" * 40, SHA + "\n"): + with self.subTest(invalid=invalid), patch.object(build, "source_contract") as source: + with self.assertRaisesRegex(ValueError, "source commit"): + build.prepare(ROOT, invalid, "v2.2.3", **READY) + source.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/upgrade/evidence/p184-consolidated-build.md b/upgrade/evidence/p184-consolidated-build.md new file mode 100644 index 00000000..1aa5cb96 --- /dev/null +++ b/upgrade/evidence/p184-consolidated-build.md @@ -0,0 +1,187 @@ +# P18.4 Consolidated Build Preparation (IDR039) + +Status: IMPLEMENTED / TESTED LOCALLY / RELEASE_HOLD, 2026-10-08. +This is recipe preparation, not an image build or CI/release certificate. +Main journal: [approved parent-owned scope](../../DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md), +"Pre-stage3 consolidated build, SDK and DR05". Parent handles journal and commit; +neither was edited by this slice. + +## Authority and Scope + +- [Phase C packet](p184-dl-phase-c-packet.md), governing owner amendment first. +- [Architecture](../DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md), section 18.13. +- IDR source read: /home/bobby/.worktrees/trading-system-next-upgrade/INTEGRATION_DEFECT_REGISTER.md, + IDR-002, IDR-029, IDR-039, IDR-040 and IDR-075. IDR039 remains OPEN until + the consolidated image set is built and certified; preparation does not close it. +- Own files: scripts/p184_consolidated_build.py, tests/test_p184_consolidated_build.py, + this evidence, and only the new consolidated-source-preflight job in + .github/workflows/ci.yml (parent-review authorization). No existing CI job, + Dockerfile, manifest or runtime change. + +## Recipe Contract + +The generator reads committed Git objects, never working-tree application files. +It requires a full lowercase 40-character commit SHA, prerequisite ancestry and +source/path markers. It records source Git blob IDs. No abbreviated SHA, branch +or mutable tag is accepted. + +| Required inclusion | Immutable ancestor | Meaning preserved | +| --- | --- | --- | +| IDR040fix1 | 4eb08dc15113cc4f73b65b239dfeb4f8ec0e23cf | REST ahead of stream is refused; no fresh timestamp manufacture | +| bindingV3 | same 4eb08dc ancestor | Sealed per-product request policy | +| quota log | same 4eb08dc ancestor | Bounded exhaustion logging, no quota increase | +| b6a0119 | b6a01190587e8771c62b5b4988782b06c5e2578a | Offered/dequeued counters, not client-apply proof | +| IDR075 | 704552d8736905cdb73feec86f3f01d339d02cbb | Preparation only, no entitlement activation | + +Future build commands use ONE private git archive context for BOTH images: +qdl-v2-python via Dockerfile and qdl-v2-rust via Dockerfile.qdl-rust-runtime. +Both use verified existing QDL_GIT_SHA and QDL_RELEASE arguments and tags with +the full application SHA. No overlay, source bind mount, named build context, +per-role image or mixed application source is supplied. The Rust Dockerfile +includes core, native ingestor, projector and stream gateway binaries. +Commands capture image IDs through --iidfile, assert OCI revision labels against +the source SHA, print archive SHA256/image IDs/labels, and remove only their own +temporary archive directory. Future build logs must be retained externally. +A partial build failure does not yield a certified image set. + +Python optionally uses --dependency-image REPOSITORY@sha256:DIGEST through the +existing QDL_DEPENDENCY_IMAGE argument. Only /opt/venv is reused; application files +remain from the archive. The existing verifier checks locked versions and RECORD +hashes. Default builder performs a clean dependency build. The main plan records +historically withdrawn vnstock4.0.2/vnai2.4.8: no availability was tested here. +An approved dependency image must be accessible to the future builder; local-only +images may need an independently approved distribution path. No dependency image +was selected, exported, retagged or pulled here. + +## Explicit Gates and CI Handoff + +Standard-library-only generator and unittest suite work offline. Full Git history +containing all prerequisite objects is required in CI; shallow history fails closed. +Parent review identified that the original real-Git unit tests were incompatible +with the Docker unittest image (.git excluded; git absent) and shallow checkout. +The revised unit suite mocks Git and invokes the CLI in-process with mocked argv +and source checks. A suite-wide Popen guard rejects any attempted process launch; +no Git binary, checkout, Bash or third-party test dependency is required. +prepare() validates the full source SHA before calling even a mocked source checker. + +The sole ci.yml addition is consolidated-source-preflight: pinned checkout action, +ref github.sha, fetch-depth: 0, pinned Python setup, portable stdlib tests, then +real committed-source CLI preflight against GITHUB_SHA. It checks checkout HEAD +matches that SHA and requires exit 2 PLUS empty stderr and valid JSON containing +the exact HOLD status, pending gates, three hold reasons, prerequisite roster, +source blob inventory, both same-SHA images, and null build script/hash. +Argument/source failures also exit 2, so exit status alone is never accepted. +No continue-on-error or silent error catch. No build, push, deploy or runtime +operation in the added job. Existing jobs and release.yml remain unchanged. +A successful source preflight is not an exact-source full DL CI certificate. + +Preparation with gates pending (JSON HOLD, exit 2, no executable build script): + +```sh +python3 -B scripts/p184_consolidated_build.py \ + --source-commit FULL_DL_COMMIT --release v2.2.3 +``` + +Only after TS tests FINISH and pass, exact-source DL CI passes, and the owner +approves the future build, provide all explicit inputs: + +```sh +python3 -B scripts/p184_consolidated_build.py \ + --source-commit FULL_DL_COMMIT --release v2.2.3 \ + --ts-tests-status pass --ts-source-commit FULL_TESTED_TS_COMMIT \ + --ts-tests-receipt-sha256 TS_COMPLETED_RECEIPT_SHA256 \ + --dl-ci-status pass --dl-ci-receipt-sha256 EXACT_DL_COMMIT_CI_RECEIPT_SHA256 \ + --build-approval approved +``` + +This still ONLY prints JSON, with PREPARED_NOT_EXECUTED and a future build_script. +Gate receipts are explicit operator attestations, not remote CI verification. +Do not use placeholder digests as acceptance evidence. The parent must review the +receipt contents and their commit association; this tool checks digest syntax only. +Never pipe preparation output into execution as part of this source-only task. + +## Same-Source SDK Promotion Gate + +Documentation-only gate; no SDK rebuild or image build performed in this slice. +The parent-owned [SDK 2.1.0 consumer handoff](qdl-sdk-2.1.0-consumer-handoff.md) +is authoritative for the consumer artifact and compatibility limitations: + +- Package source: b2c8c5696a7e6b9bdcf2d1f97e94f837f5ba0fe8. +- Wheel: qdl_sdk-2.1.0-py3-none-any.whl. +- Wheel SHA256: 6923b46f46bd07c850b18a6074439bbcb4085d84428a8e72c476437dd51524e7. + +Before promotion, rebuild the SDK using scripts/build_qdl_sdk_release.py from +the SAME archived final full SHA used for BOTH consolidated application images. +Run that archived script with --output-dir pointing to a NEW isolated directory +outside the archive and outside the immutable consumer handoff directory. +Do not invoke a builder from the current working tree or overlay SDK sources. +Record the final source SHA, archive hash, new wheel SHA256, release manifest and +SBOM identifiers, and compare the new wheel bytes/hash to the exact consumer +artifact above. Require exact wheel parity before promotion; matching version +2.1.0 alone is insufficient. Preserve the existing TS/alpha handoff unchanged. + +Any wheel drift is a promotion HOLD: preserve both artifacts and receipts, +investigate the source/build difference, and obtain parent review plus a new +explicit consumer handoff/compatibility decision before proceeding. Never +overwrite, relabel or silently replace the old wheel or its evidence. SDK parity +does not waive the documented alpha PyJWT dependency gate or DR/broker gates. +The recipe generator does not automate or attest this SDK promotion gate. + +## Verification and Limits + +Executed: python3 -B -m unittest tests.test_p184_consolidated_build -v. +Revised result: 18 PASS, 0 FAIL, 0 SKIP. All Git and CLI source checks mocked; +includes independent pending gates, malformed digests/statuses, mutable refs, +full-SHA validation even with mocked source checker, two-image single-context +contract, dependency pin validation, and CLI source error vs HOLD output. +All tests block subprocess creation. Earlier 14/16-test passes were local +checkout evidence only, not evidence of Docker/CI portability. + +Portability rehearsal: copied only the recipe and test into a TemporaryDirectory, +set PATH='' and PYTHONPATH='', then invoked absolute sys.executable with +-B -m unittest discover -s tests -p test_p184_consolidated_build.py -v: +18 PASS, 0 FAIL, 0 SKIP with no .git, Git or Bash. Temporary copy auto-removed. + +Executed the exact inline Python preflight from the new job locally against +52642e52e01981590bc433bf95ac12d4e941cf10: PASS with build HOLD. +Negative harness checks: source error exit2, malformed JSON object exit2 and +unexpected exit0 all rejected (3/3). This is local rehearsal, NOT hosted CI. +Real committed-source checks now belong to the dedicated full-history job, not +Docker unittest discovery. No Docker command was executed. +git diff --check passed; no-index --check on new code/test files had no whitespace +diagnostics (exit 1 means files differ from /dev/null). + +Not run: CI, Docker packaging/linkage, Rust compilation, provider/DR/broker tests, +image scans, registry operations or runtime certification. Marker/ancestry checks +do not establish semantic correctness or exclude later reverts; exact-source CI +and packaged-image verification remain required. Base package resolution can vary +despite one application SHA. Image IDs are local content IDs, not registry manifest +digests. IDR040fix2 is deferred and IDR075 remains prepared, not activated. +No release readiness or CI-green claim is made. + +## Runtime, Cleanup and Handoff + +Canonical source: /home/bobby/data_layer, main, +5bf5cd540498b82959342c7cfb2ebeaac3007426; stable release v2.2.2 per main journal. +Active feature: /home/bobby/.worktrees/data-layer-p184-closure, +fix/p184-dl-closure. Observed HEAD advanced during concurrent parent work from +351cc72c300d7a3174627f322eb5accc6447947c to +b2c8c5696a7e6b9bdcf2d1f97e94f837f5ba0fe8; this slice made no commit. +Parent advanced the feature to 52642e52e01981590bc433bf95ac12d4e941cf10 +during the review; local CI-block rehearsal checked this revision. +Uncommitted recipe/test/CI edits are not claimed to be in any built image. + +No runtime/Docker access or mutation, network request, push, merge or workflow +dispatch. Active service set, image digests and config revision were NOT +re-inspected; inherit only the packet's historical inventory, not a fresh claim. +Named BAR rollback f7351c3b remains protected per parent scope; other active and +rollback artifacts untouched. No test images/containers/caches created. The no-Git rehearsal used only a +TemporaryDirectory, auto-removed on completion; no retained test files. No Docker +cleanup or disk-capacity measurements were performed. Docker/BuildKit inventory and cleanup intentionally +deferred because this is source-only preparation, not image-phase closure. +Concurrent DR05 and parent work were left untouched. + +Next permitted step: parent review, journal update and eventual commit; obtain +final TS completion and exact-source DL CI receipts before separately authorizing +a consolidated rebuild. Building does not authorize deployment. DR/broker gates +and paired IDR075 activation still need the reviewed runtime packet and approval. From c0814f4c38ae32ac390d13a51acf3a3bf96addfb Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 09:37:52 +0000 Subject: [PATCH 033/147] test(p184): prepare isolated DR05 stack and scoped lifecycle cleanup --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 34 ++ scripts/p184_dr05_dl_fault.py | 612 +++++++++++++++++++++ tests/test_p184_dr05_dl_fault.py | 354 ++++++++++++ upgrade/evidence/p184-dl-phase-c-packet.md | 19 + upgrade/evidence/p184-dr05-dl-fault.md | 101 ++++ 5 files changed, 1120 insertions(+) create mode 100644 scripts/p184_dr05_dl_fault.py create mode 100644 tests/test_p184_dr05_dl_fault.py create mode 100644 upgrade/evidence/p184-dr05-dl-fault.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index b090da87..2df44d28 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70185,3 +70185,37 @@ changes. Active/rollback retained, f7351c3b protected. Latest readonly disk inventory95images/28.96GB, BuildKit20.52GB/2.852GB reclaimable; no broadprune during Claude concurrent work. Canonicalmain5bf5cd5/v2.2.2 unchanged; same featureworktree/branch, no push/merge. Rebuild only after owner timing gate. + + +### DR05 isolated stack and stage3 timing handoff - 2026-10-08 (Codex) + +Status STACK_STARTUP_TESTED / TS_FAULT_ACCEPTANCE_NOT_RUN / RELEASE_HOLD. +Guide upgrade/evidence/p184-dr05-dl-fault.md; scripts/p184_dr05_dl_fault.py. +Five bounded --rm roles, network-none Kafka and shared ONLY disposable namespace, +fresh TLS/JWT, four ETH BOOK products across Binance/OKX; sanitized public config +allowlist, no shared credentials/state. Query is rc2-43bbcf0 (77682eea), not +runtime v2.2.2; no new application image. Source-mounted test harness is not +release-image certification. Prepared TEST_ONLY GAP/generation schedules retain +positive control, recovery anchor, old-generation fence and post-fence control. +Actual raw publisher/TS observation adapter remains NOT_IMPLEMENTED; no DR05 +cell closed. Pending rows cannot be submitted as runtime evidence or emit PASS. + +Parent source tests: existing77682eea image, network none, .5CPU/512MiB, +python -B -m unittest tests.test_p184_dr05_dl_fault -q:26PASS0fail0skip/68.539s. +Agent isolated startup smoke:55.13s, all5roles process-ready/services-reachable, +functional_ready=false/acceptance=false. Namespace p184-dr05-dl-fa9ce97d336d41a2. +Initial cleanup check raced asynchronous Docker --rm; fixed bounded removal wait +and vanished-between-list/inspect handling, regression tested. Full rerun removed +all5roles/helper, credentials and temporary directory; parent scoped ps empty. +Smoke metrics transcribed from agent tool session, not archived DR acceptance. +Diskfree79,907,880,960 ->79,716,106,240bytes, concurrent host delta not attributed +to this smoke. No test images built; no broad prune; active/BARf7351c3b retained. + +PhaseC packet now records confirmed TS CI2abbe394/a6548ee3 but keeps TS test +completion separate: no consolidated image build until TS acceptance finishes +and exact-source DL CI passes. DR01 deployed/DR08 final await Claude stage3. +SDK local publication complete, hosted publication not performed; alpha requires +PyJWT2.15.1 dependency update as documented. Recipe18PASS, SDK89PASS inherited; +no rerun of unrelated gates. Hosted DL CI remains NOT_RUN, not claimed green. +Canonical main5bf5cd5/v2.2.2 and shared runtime unchanged by this work. Active +feature fix/p184-dl-closure; no push/merge. Main plan/guide/packet agree on HOLD. diff --git a/scripts/p184_dr05_dl_fault.py b/scripts/p184_dr05_dl_fault.py new file mode 100644 index 00000000..92c5aab2 --- /dev/null +++ b/scripts/p184_dr05_dl_fault.py @@ -0,0 +1,612 @@ +#!/usr/bin/env python3 +"""Offline DR05 preparation. No runtime acceptance or shared-runtime discovery. + +--prepare writes a NEW external directory with topology, fault schedules and +nonqualifying verifier rows. --readiness/--cleanup are owner stage3 utilities. +The publisher/TS adapter handoff is deliberately explicit, not a status mock. +""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +from scripts.p184_dr08_isolated import IMAGES # immutable image inventory only + +VENUES = ("BINANCE", "OKX") +FAULTS = ("GAP_REFUSAL", "GENERATION_CHANGE_FENCED") +LABEL = "p184.dr05.dl" +ROLES = ("kafka", "quota", "core", "stream", "query") +CELLS = tuple(f"DR05.{fault}@{venue}" for venue in VENUES for fault in FAULTS) + + +def namespace(value): + if not re.fullmatch(r"p184-dr05-dl-[0-9a-f]{16}", value): + raise ValueError("exact disposable DR05 namespace required") + return value + + +def pending_rows(): + # No provenance header: this file must not be submitted as runtime evidence. + return [{"cell": cell, "level": "ISOLATED_FAULT", "ok": False, + "outcome": "PREPARED_NOT_RUN", "provenance": "TEST_ONLY", + "detail": {"gate": "stage3 real DL path AND TS adapter observations required"}} + for cell in CELLS] + + +def frame(venue, snapshot, sequence, previous, generation, quantity, at_ns): + """Provider-shaped test bytes, following realtime-core delayed_book_frame. + + Stamp at publish time, not preparation time. Same lane across generations; + a lane rename tests a different fence and is not this case. + """ + if venue not in VENUES or type(at_ns) is not int or at_ns <= 0: + raise ValueError("venue and positive integer timestamp required") + symbol = "ETHUSDT" if venue == "BINANCE" else "ETH-USDT-SWAP" + ms = at_ns // 1_000_000 + if venue == "OKX": + payload = {"arg": {"channel": "books", "instId": symbol}, + "action": "snapshot" if snapshot else "update", "data": [{ + "seqId": str(sequence), "prevSeqId": "-1" if snapshot else str(previous), + "ts": str(ms), "bids": [["60000", quantity, "0", "1"]], + "asks": [["60001", "1", "0", "1"]]}]} + elif snapshot: + payload = {"lastUpdateId": sequence, "E": ms, + "bids": [["60000", quantity]], "asks": [["60001", "1"]]} + else: + payload = {"s": symbol, "U": previous, "u": sequence, "pu": previous, + "E": ms, "b": [["60000", quantity]], "a": [["60001", "1"]]} + return {"venue": venue, "native_symbol": symbol, "connection_generation": generation, + "sequence": sequence, "previous": previous, "snapshot": snapshot, + "transport": "HTTP" if snapshot and venue == "BINANCE" else "WEBSOCKET", + "received_at_ns": at_ns, "test_provenance": True, "provenance": "TEST_ONLY", + "adapter_version": "p184-synthetic-test-only", "payload": payload} + + +def schedule(venue, fault): + if venue not in VENUES or fault not in FAULTS: + raise ValueError("unknown DR05 cell") + # tuples: phase, snapshot, seq, previous, generation, bid quantity + steps = [("positive_anchor", True, 100, 0, 7, "1"), + ("positive_control", False, 101, 100, 7, "2")] + if fault == "GAP_REFUSAL": + steps += [("gap", False, 110, 109, 7, "3"), + ("still_unanchored", False, 111, 110, 7, "4")] + else: + steps += [("new_generation_unanchored", False, 201, 200, 8, "3")] + steps += [("return_anchor", True, 200, 0, 8, "5"), + ("return_control", False, 202, 200, 8, "6"), + ("old_generation", True, 300, 0, 7, "99"), + ("post_fence_control", False, 203, 202, 8, "7")] + return [{"phase": phase, "venue": venue, "snapshot": snap, "sequence": seq, + "previous": prev, "generation": gen, "quantity": qty} + for phase, snap, seq, prev, gen, qty in steps] + + +def drive(adapter, venue, fault): + """Owner adapter contract; NOT a verifier or an implemented runtime adapter. + + fresh_case must start/reset only a disposable case (generation starts at 7). + observe must wait for this raw Kafka offset through real core + Query/Stream + and actual TS validation; it must never manufacture a status from phase. + All calls must have bounded deadlines. Returned observations remain untrusted + until owner review. Even an adapter returning all-success cannot emit ok=True. + """ + adapter.fresh_case(venue, fault, timeout=30) + observations = [] + for step in schedule(venue, fault): + values = {k: v for k, v in step.items() if k != "phase"} + raw = frame(**values, at_ns=time.time_ns()) + offset = adapter.publish(raw, timeout=5) + observations.append({"phase": step["phase"], "raw_offset": offset, + "observation": adapter.observe(offset, timeout=10)}) + return {"cell": f"DR05.{fault}@{venue}", "level": "ISOLATED_FAULT", + "ok": False, "outcome": "OWNER_REVIEW_REQUIRED", "provenance": "TEST_ONLY", + "observations": observations} + + +def topology(ns, work, topic_id=""): + """Generate executable Docker argv, but do not execute or discover config. + + Material preparation is a stage3 gate; topic UUID comes from the new broker. + Query reuses DR08's generic real hot-read service, NOT its TRADE preparer. + """ + namespace(ns) + work = Path(work).resolve() + if work == ROOT or ROOT in work.parents or ":" in str(work) or "," in str(work): + raise ValueError("external unambiguous work directory required") + common = {"QDL_ENVIRONMENT": "paper", "QDL_KN_CURSOR_KEYS_FILE": "/work/cursor-keys.json", + "QDL_KN_CURSOR_ACTIVE_KEY_ID": "dr08", "QDL_KN_TOPIC_ID": topic_id, + "QDL_KN_ROUTE_GENERATION": ns, "QDL_KN_READ_VIEW_SECRET_FILE": "/work/read-view.secret"} + spec = { + "kafka": ("kafka", "1536m", "1", {}, ["/bin/sh", "-ec", + "/opt/kafka/bin/kafka-storage.sh format -t MkU3OEVBNTcwNTJENDM2Qk -c /work/broker.properties && exec /opt/kafka/bin/kafka-server-start.sh /work/broker.properties"]), + "quota": ("redis", "64m", "0.1", {}, ["redis-server", "--save", "", "--appendonly", "no", + "--maxmemory", "48mb", "--maxmemory-policy", "noeviction"]), + "core": ("core", "256m", "0.5", {"QDL_KAFKA_CERT_ROOT": "/work", + "QDL_KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", "QDL_KAFKA_CLIENT_ID": ns, + "QDL_KAFKA_GROUP_ID": ns, "QDL_PROVIDER_ADMISSION_ENABLED": "false"}, + ["/usr/local/bin/qdl-realtime-core", "/work/core.json"]), + "stream": ("stream", "512m", "0.5", {**common, + "QDL_KN_BUNDLE_FILE": "/work/bundle.json", "QDL_KN_JWT_CONFIG_FILE": "/work/jwt-config.json", + "QDL_KN_QUOTA_REDIS_URL": "redis://localhost:6379/0", "QDL_KN_QUOTA_PREFIX": "qdl:stable:v2:paper:" + ns, + "QDL_KN_TLS_CERT_FILE": "/work/client.crt", "QDL_KN_TLS_KEY_FILE": "/work/client.key", + "QDL_KN_TLS_CLIENT_CA_FILE": "/work/ca.crt", "QDL_KN_KAFKA_BOOTSTRAP": "localhost:9092", + "QDL_KN_KAFKA_CA": "/work/ca.crt", "QDL_KN_KAFKA_CERT": "/work/client.crt", + "QDL_KN_KAFKA_KEY": "/work/client.key", "QDL_KN_KAFKA_TOPIC": ns + ".canonical", + "QDL_KN_KAFKA_GROUP": "kn-" + ns, "QDL_KN_KAFKA_CLIENT_ID": ns + "-stream", + "QDL_KN_READ_VIEW_URLS": "https://localhost:8200", "QDL_KN_READ_VIEW_CA_FILE": "/work/ca.crt", + "QDL_KN_HOT_READ_ENABLED": "true", "QDL_KN_METRICS_SECONDS": "1", + "QDL_KN_MAX_REPLAY_EVENTS": "10000", "QDL_KN_QUEUE_BYTES_TOTAL": "33554432"}, + ["/usr/local/bin/qdl-stream-gateway", "serve"]), + "query": ("query", "512m", "0.5", {**common, "PYTHONPATH": "/app:/app/generated/python", + "PYTHONDONTWRITEBYTECODE": "1"}, + ["python", "-B", "scripts/p184_dr08_isolated.py", "--query", "/work"]), + } + result = {} + for role, (image, memory, cpus, env, command) in spec.items(): + network = "none" if role == "kafka" else "container:" + ns + "-kafka" + argv = ["docker", "run", "-d", "--rm", "--pull=never", "--name", ns + "-" + role, + "--label", LABEL + "=" + ns, "--network", network, "--read-only", + "--user", f"{os.getuid()}:{os.getgid()}", "--memory", memory, "--cpus", cpus, + "--pids-limit", "128", "--log-opt", "max-size=4m", "--log-opt", "max-file=1", + "--tmpfs", "/tmp:rw," + ("exec," if role == "kafka" else "") + "size=64m,mode=1777", + "-v", str(work) + ":/work", "-v", str(ROOT) + ":/app:ro", "-w", "/app"] + if role == "kafka": + argv += ["--hostname", "localhost", "--tmpfs", "/data:rw,size=512m,mode=1777"] + env = {"KAFKA_HEAP_OPTS": "-Xms256m -Xmx512m", "LOG_DIR": "/tmp/kafka-logs"} + for key, value in env.items(): + argv += ["-e", key + "=" + value] + result[role] = argv + ["--entrypoint", command[0], IMAGES[image], *command[1:]] + return result + + +def sh(argv): + return subprocess.run(argv, check=True, capture_output=True, text=True, timeout=15).stdout + + +def owned(ns, role, call=sh): + name = namespace(ns) + "-" + role + if role not in (*ROLES, "helper"): + raise ValueError("unknown owned role") + ids = call(["docker", "ps", "-aq", "--filter", "name=^/" + name + "$"]).splitlines() + if not ids: + return None + if len(ids) != 1: + raise ValueError("ambiguous container") + try: + row = json.loads(call(["docker", "inspect", ids[0]]))[0] + except subprocess.CalledProcessError: + # --rm can finish between the exact-name listing and inspection. + if not call(["docker", "ps", "-aq", "--filter", "name=^/" + name + "$"]).strip(): + return None + raise + if row["Name"] != "/" + name or row["Config"].get("Labels", {}).get(LABEL) != ns: + raise ValueError("cleanup/readiness ownership mismatch") + return row + + +def readiness(ns, call=sh): + rows = {role: owned(ns, role, call) for role in ROLES} + broker = rows["kafka"] + checks = {} + for role, row in rows.items(): + expected_image = IMAGES[{"quota": "redis"}.get(role, role)] + network = "none" if role == "kafka" else "container:" + (broker or {}).get("Id", "MISSING") + checks[role] = bool(row and row["State"]["Running"] and row["Image"] == expected_image + and row["HostConfig"]["NetworkMode"] in + (network, "container:" + ns + "-kafka" if role != "kafka" else network) + and not row["HostConfig"].get("PortBindings") + and row["HostConfig"].get("AutoRemove") is True) + return {"process_isolation_ready": all(checks.values()), "roles": checks, + "functional_ready": False, "acceptance": False, + "remaining": "mTLS broker metadata, real Query/Stream positive controls and TS adapter"} + + +def cleanup(ns, call=sh): + removed = [] + # Preflight ALL ownership before stopping anything. Never remove networks, + # images, shared volumes, files or a container selected only by a label. + rows = [(role, owned(ns, role, call)) for role in reversed((*ROLES, "helper"))] + if any(row and row["HostConfig"].get("AutoRemove") is not True for _, row in rows): + raise ValueError("refuse non-disposable container") + failures = [] + for role, row in rows: + if row: + try: + call(["docker", "stop", "--time", "5", row["Id"]]) + deadline = time.monotonic() + 5 + while (remaining := owned(ns, role, call)) is not None: + if remaining.get("State", {}).get("Running") or time.monotonic() >= deadline: + raise RuntimeError("--rm container still present") + time.sleep(.1) + removed.append(role) + except Exception as error: + failures.append(role + ":" + type(error).__name__) + if failures: + raise RuntimeError("cleanup incomplete: " + ",".join(failures)) + return {"removed": removed, "scope": ns} + + +def prepare(work): + ns = "p184-dr05-dl-" + uuid.uuid4().hex[:16] + topology(ns, work) # Validate path before writing. + work = Path(work).resolve() + work.mkdir(mode=0o700, parents=True, exist_ok=False) + packet = {"namespace": ns, "status": "PREPARED_HANDOFF_NOT_RUN", "provenance": "TEST_ONLY", + "level": "ISOLATED_FAULT", "images": IMAGES, + "deadline_seconds": 600, "acceptance": False, + "material_ready": False, "adapter_ready": False, + "schedules": {f"DR05.{f}@{v}": schedule(v, f) for v in VENUES for f in FAULTS}} + (work / "handoff.json").write_text(json.dumps(packet, indent=2) + "\n") + (work / "pending-cells.jsonl").write_text("".join(json.dumps(r) + "\n" for r in pending_rows())) + return packet + + +def isolated_config(exported, ns): + """Take only public binding fields; never copy transport, credentials or authority.""" + namespace(ns) + required = {"provider", "venue", "market", "product_type", "native_symbol", "native_channel", + "provider_kind", "instrument_uid", "instrument_id", "instrument_revision", + "instrument_catalog_revision", "source_id", "source_role", "normalizer_version", + "sequence_policy", "l2"} + optional = {"physical_native_symbol", "physical_native_channel", "require_final_bar"} + selected = [] + for venue, market, symbol in (("BINANCE", "USDM", "ETHUSDT"), ("OKX", "SWAP", "ETH-USDT-SWAP")): + matches = [b for b in exported["core"]["bindings"] if b.get("l2") and + (b.get("venue"), b.get("market"), b.get("native_symbol")) == (venue, market, symbol)] + if len(matches) != 1 or not required <= matches[0].keys(): + raise ValueError("exactly one complete exported ETH BOOK binding required: " + venue) + binding = {k: copy.deepcopy(v) for k, v in matches[0].items() if k in required | optional} + if binding["product_type"] != "PERPETUAL" or binding["source_role"] != "PRIMARY": + raise ValueError("ETH primary perpetual BOOK required") + allowed_l2 = {"provider_protocol", "depth_per_side", "snapshot_refresh_seconds", + "materialized_snapshot_interval_ms"} + if set(binding["l2"]) - allowed_l2: + raise ValueError("unexpected L2 configuration") + selected.append(binding) + digest = hashlib.sha256(json.dumps(selected, sort_keys=True).encode()).hexdigest() + return {"core": {"bindings": selected, "canonical_stream": ns + ".canonical", + "quarantine_stream": ns + ".quarantine", "allow_test_provenance": True, + "dedup_capacity": 10000}, + "raw_topics": [ns + ".raw"], "shard_id": ns, "transactional_id": ns, + "batch_size": 25, "batch_wait_ms": 10, "max_events": 0, "metrics_every_batches": 1, + "strict_subscription_scope": True, "book_resync_quarantine": True, + "authority": {"schema": "qdl.authority-record.v1", "slice_id": ns, "revision": 1, + "mode": "RUST_SHADOW", "candidate_image_digest": IMAGES["core"], + "capability_manifest_digest": digest, "contract_digest": digest, + "partition_plan_digest": digest, "public_write_allowed": False, + "legacy_write_allowed": False, "approved_by": "TEST_ONLY ISOLATED_FAULT", + "effective_at_ns": time.time_ns()}} + + +def material(work, exported): + """Offline, fresh credentials. No shared discovery; source catalog is read-only.""" + import secrets + import yaml + from datetime import datetime, timedelta, timezone + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.hazmat.primitives.serialization import pkcs12 + from cryptography.x509.oid import NameOID, ExtendedKeyUsageOID + from qdl.runtime.stable_catalog import StableSourceCatalog + from tests.phase7_support import manifest_mapping + from scripts.kn_gateway_bundle import compile_bundle + from scripts.kn_native_slice_probe import ISSUER, AUDIENCE + + work = Path(work) + packet = json.loads((work / "handoff.json").read_text()) + namespace = packet["namespace"] + cfg = isolated_config(exported, namespace) + + def save(name, value, private=False): + with (work / name).open("x") as handle: + os.chmod(work / name, 0o600 if private else 0o644) + handle.write(value if isinstance(value, str) else json.dumps(value)) + + catalog = yaml.safe_load((ROOT / "config/v2/stable-source-bindings.yaml").read_text()) + catalog["canonical_stream"] = cfg["core"]["canonical_stream"] + ids = {b["source_id"] for b in cfg["core"]["bindings"]} + catalog["bindings"] = [b for b in catalog["bindings"] if b["source"]["source_id"] in ids + and b["feed"] in ("BOOK_SNAPSHOT", "BOOK_DELTA")] + if len(catalog["bindings"]) != 4: + raise ValueError("both ETH BOOK products must match source catalog") + for binding in catalog["bindings"]: + binding["source"]["adapter_version"] = "p184-synthetic-test-only" + save("catalog.yaml", yaml.safe_dump(catalog)) + loaded = StableSourceCatalog.load(work / "catalog.yaml") + for b in cfg["core"]["bindings"]: + matches = [x for x in loaded.bindings if x.source_id == b["source_id"]] + if len(matches) != 2 or any(x.instrument.instrument_uid != b["instrument_uid"] for x in matches): + raise ValueError("export/catalog identity mismatch") + save("core.json", cfg) + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + now = datetime.now(timezone.utc) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, namespace)]) + ca = (x509.CertificateBuilder().subject_name(name).issuer_name(name) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)).not_valid_after(now + timedelta(hours=2)) + .add_extension(x509.BasicConstraints(ca=True, path_length=0), True) + .sign(key, hashes.SHA256())) + leaf_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + leaf = (x509.CertificateBuilder().subject_name(x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, "localhost")])).issuer_name(name) + .public_key(leaf_key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)).not_valid_after(now + timedelta(hours=2)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), True) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("localhost")]), False) + .add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH, + ExtendedKeyUsageOID.CLIENT_AUTH]), False) + .sign(key, hashes.SHA256())) + pem = serialization.Encoding.PEM + save("ca.crt", ca.public_bytes(pem).decode()) + save("client.crt", leaf.public_bytes(pem).decode()) + save("client.key", leaf_key.private_bytes(pem, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption()).decode(), True) + password = secrets.token_hex(16) + with (work / "server.p12").open("xb") as handle: + (work / "server.p12").chmod(0o600) + handle.write(pkcs12.serialize_key_and_certificates(b"dr08", leaf_key, leaf, [ca], + serialization.BestAvailableEncryption(password.encode()))) + props = """process.roles=broker,controller +node.id=1 +controller.quorum.voters=1@localhost:9093 +listeners=SSL://:9092,CONTROLLER://:9093 +advertised.listeners=SSL://localhost:9092 +controller.listener.names=CONTROLLER +listener.security.protocol.map=SSL:SSL,CONTROLLER:PLAINTEXT +inter.broker.listener.name=SSL +ssl.keystore.type=PKCS12 +ssl.keystore.location=/work/server.p12 +ssl.keystore.password=PASSWORD +ssl.key.password=PASSWORD +ssl.truststore.type=PEM +ssl.truststore.location=/work/ca.crt +ssl.client.auth=required +log.dirs=/data +log.segment.bytes=16777216 +metadata.log.segment.bytes=16777216 +log.index.size.max.bytes=1048576 +offsets.topic.num.partitions=1 +transaction.state.log.num.partitions=1 +num.partitions=1 +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 +group.initial.rebalance.delay.ms=0 +""" + save("broker.properties", props.replace("PASSWORD", password), True) + + manifests, identities, keys, subjects = [], [], {}, {} + for b in cfg["core"]["bindings"]: + label = b["venue"].lower() + cid = namespace + "." + label + subject = "spiffe://qdl/paper/" + cid + requirements = [] + for feed in ("BOOK_SNAPSHOT", "BOOK_DELTA"): + binding = next(x for x in loaded.bindings if x.source_id == b["source_id"] and x.feed.value == feed) + mapping = manifest_mapping(consumer_id=cid, subject=subject, instrument_uid=b["instrument_uid"], + feed=feed, interval=None, source_policy_id=binding.source_policy_id) + requirement = mapping["spec"]["requirements"][0] + requirement.update(warmup_limit=1, require_full_coverage=False, require_final_bars=False, + max_freshness_ms=2000) + requirements.append(requirement) + mapping["spec"]["requirements"] = requirements + mapping["spec"]["quotas"]["max_buffer_events"] = 1000 + save(label + ".yaml", yaml.safe_dump(mapping)) + manifests.append(work / (label + ".yaml")) + jwt_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + save(label + ".key", jwt_key.private_bytes(pem, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption()).decode(), True) + keys[label] = jwt_key.public_key().public_bytes(pem, serialization.PublicFormat.SubjectPublicKeyInfo).decode() + subjects[label] = subject + identities.append({"id": cid, "tls": {"ca_file": "/work/ca.crt", "cert_file": "/work/client.crt", + "key_file": "/work/client.key"}, "jwt": {"private_key_file": "/work/" + label + ".key", "key_id": label}}) + save("jwt-config.json", {"issuer": ISSUER, "audience": AUDIENCE, "algorithms": "RS256", + "keys": keys, "subjects": subjects, "environments": {k: "paper" for k in keys}, "max_lifetime_seconds": 900}) + save("profile.json", {"identities": identities}) + save("bundle.json", compile_bundle(environment="paper", catalog_path=work / "catalog.yaml", manifest_paths=manifests)) + save("cursor-keys.json", {"dr08": secrets.token_hex(32)}, True) + save("read-view.secret", secrets.token_hex(32), True) + save("probes.json", []) + runtime_packet = {"namespace": namespace, "bootstrap": "localhost:9092", "provenance": "TEST_ONLY", + "client_tls": {"ca": "/work/ca.crt", "cert": "/work/client.crt", "key": "/work/client.key"}, + "isolated_manifest_paths": ["/work/binance.yaml", "/work/okx.yaml"], + "slice": {"bundle": "/work/bundle.json", "cursor_keys": "/work/cursor-keys.json", + "profile": "/work/profile.json", "probes": "/work/probes.json", + "route_generation": namespace, "target": "localhost:8210"}} + save("packet.json", runtime_packet) + packet.update(material_ready=True, status="MATERIAL_READY_NOT_STARTED") + (work / "handoff.json").write_text(json.dumps(packet, indent=2) + "\n") + return packet + + +def helper_command(ns, work, args, *, network="none", exported=None): + namespace(ns) + if network not in ("none", "container:" + ns + "-kafka"): + raise ValueError("helper must stay in the disposable namespace") + topology(ns, work) + if exported and any(c in str(Path(exported).resolve()) for c in (":", ",")): + raise ValueError("unambiguous export path required") + argv = ["docker", "run", "--rm", "--pull=never", "--name", ns + "-helper", "--label", LABEL + "=" + ns, + "--network", network, "--read-only", "--user", f"{os.getuid()}:{os.getgid()}", + "--memory", "512m", "--cpus", "0.5", "--pids-limit", "128", + "--log-opt", "max-size=4m", "--log-opt", "max-file=1", + "--tmpfs", "/tmp:rw,size=64m,mode=1777", "-v", str(Path(work).resolve()) + ":/work", + "-v", str(ROOT) + ":/app:ro", "-w", "/app", "-e", "PYTHONPATH=/app:/app/generated/python", + "-e", "PYTHONDONTWRITEBYTECODE=1"] + if exported: + argv += ["-v", str(Path(exported).resolve()) + ":/export.json:ro"] + return argv + ["--entrypoint", "python", IMAGES["python"], "-B", "scripts/p184_dr05_dl_fault.py", *args] + + +def bounded(argv, timeout=120): + return subprocess.run(argv, check=True, capture_output=True, text=True, timeout=timeout).stdout + + +PRIVATE_FILES = ("client.key", "server.p12", "binance.key", "okx.key", "cursor-keys.json", + "read-view.secret", "broker.properties") + + +def erase_private(work): + work = Path(work).resolve() + namespace(json.loads((work / "handoff.json").read_text())["namespace"]) + for name in PRIVATE_FILES: + (work / name).unlink(missing_ok=True) + packet = json.loads((work / "handoff.json").read_text()) + packet.update(material_ready=False, status="STOPPED_PRIVATE_MATERIAL_REMOVED") + (work / "handoff.json").write_text(json.dumps(packet, indent=2) + "\n") + + +def init(work, export_path): + # Runtime-free initialization in the already-local dependency image. + packet = prepare(work) + ns = packet["namespace"] + try: + bounded(helper_command(ns, work, ["--material", "/work", "--core-config", "/export.json"], exported=export_path)) + except BaseException: + try: + cleanup(ns) + finally: + erase_private(work) + raise + return {"namespace": ns, "material_ready": True, "acceptance": False} + + +def up(work): + work = Path(work).resolve() + packet = json.loads((work / "handoff.json").read_text()) + ns = namespace(packet["namespace"]) + if not packet.get("material_ready"): + raise ValueError("run --init with sanitized export first") + if any(owned(ns, role) is not None for role in (*ROLES, "helper")): + raise ValueError("namespace already in use; no implicit restart") + for digest in set(IMAGES.values()): + if sh(["docker", "image", "inspect", digest, "--format", "{{.Id}} "]).strip() != digest: + raise ValueError("local immutable image mismatch") + try: + bounded(topology(ns, work)["kafka"]) + bounded(helper_command(ns, work, ["--topics", "/work"], network="container:" + ns + "-kafka")) + topic = json.loads((work / "packet.json").read_text())["slice"]["topic_id"] + for role, argv in topology(ns, work, topic).items(): + if role != "kafka": + bounded(argv) + deadline = time.monotonic() + 45 + while True: + result = readiness(ns) + if result["process_isolation_ready"]: + try: + bounded(helper_command(ns, work, ["--probe", "/work"], network="container:" + ns + "-kafka"), 25) + return {**result, "services_reachable": True, "acceptance": False} + except subprocess.CalledProcessError: + pass + if time.monotonic() >= deadline: + raise TimeoutError("isolated services startup deadline") + time.sleep(.5) + except BaseException: + try: + cleanup(ns) + finally: + erase_private(work) + raise + + +def probe(work): + """Authenticated service connectivity only, never a market-data acceptance.""" + import socket + import ssl + import asyncio + import grpc + from confluent_kafka.admin import AdminClient + from scripts.p184_dr08_isolated import kafka_security + packet = json.loads((work / "packet.json").read_text()) + AdminClient(kafka_security(packet)).list_topics(timeout=3) + context = ssl.create_default_context(cafile=str(work / "ca.crt")) + context.load_cert_chain(str(work / "client.crt"), str(work / "client.key")) + with socket.create_connection(("localhost", 8200), timeout=3) as sock: + with context.wrap_socket(sock, server_hostname="localhost"): + pass + async def connect(): + credentials = grpc.ssl_channel_credentials((work / "ca.crt").read_bytes(), + (work / "client.key").read_bytes(), (work / "client.crt").read_bytes()) + async with grpc.aio.secure_channel("localhost:8210", credentials, + options=(("grpc.enable_http_proxy", 0),)) as channel: + await asyncio.wait_for(channel.channel_ready(), 5) + asyncio.run(connect()) + return {"services_reachable": True, "functional_ready": False, "acceptance": False} + + +def dispatch(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + for flag in ("prepare", "init", "material", "up", "down", "topics", "probe", "ready"): + mode.add_argument("--" + flag, type=Path) + mode.add_argument("--readiness") + mode.add_argument("--cleanup") + parser.add_argument("--core-config", type=Path) + args = parser.parse_args(argv) + if args.init or args.material: + if args.core_config is None: + parser.error("--core-config sanitized exported JSON required") + if args.init: + result = init(args.init, args.core_config) + else: + result = material(args.material, json.loads(args.core_config.read_text())) + result = {k: result[k] for k in ("namespace", "status", "material_ready")} + elif args.up: + result = up(args.up) + elif args.down or args.ready: + work = args.down or args.ready + ns = namespace(json.loads((work / "handoff.json").read_text())["namespace"]) + if args.down: + try: + result = cleanup(ns) + finally: + erase_private(work) + else: + result = readiness(ns) + if result["process_isolation_ready"]: + result.update(json.loads(bounded(helper_command(ns, work, ["--probe", "/work"], + network="container:" + ns + "-kafka")))) + elif args.topics: + from scripts.p184_dr08_isolated import prepare_topics + prepare_topics(args.topics) + result = {"topics_ready": True} + elif args.probe: + result = probe(args.probe) + elif args.prepare: + packet = prepare(args.prepare) + result = {k: packet[k] for k in ("namespace", "status", "acceptance")} + else: + result = readiness(args.readiness) if args.readiness else cleanup(args.cleanup) + print(json.dumps(result)) + return 1 if result.get("process_isolation_ready") is False else 0 + + +def main(argv=None): + def deadline(_signum, _frame): + raise TimeoutError("DR05 command ten-minute deadline") + previous = signal.signal(signal.SIGALRM, deadline) + signal.alarm(600) + try: + return dispatch(argv) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, previous) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_p184_dr05_dl_fault.py b/tests/test_p184_dr05_dl_fault.py new file mode 100644 index 00000000..82cbca13 --- /dev/null +++ b/tests/test_p184_dr05_dl_fault.py @@ -0,0 +1,354 @@ +"""Offline contracts only; no Docker daemon or provider traffic.""" +import copy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +from scripts import p184_dr05_dl_fault as dr + +NS = "p184-dr05-dl-0123456789abcdef" + + +class PreparationTests(unittest.TestCase): + def test_exact_nonqualifying_cells(self): + rows = dr.pending_rows() + self.assertEqual({r["cell"] for r in rows}, { + "DR05.GAP_REFUSAL@BINANCE", "DR05.GAP_REFUSAL@OKX", + "DR05.GENERATION_CHANGE_FENCED@BINANCE", "DR05.GENERATION_CHANGE_FENCED@OKX"}) + self.assertTrue(all(r["ok"] is False and r["outcome"] == "PREPARED_NOT_RUN" + and r["level"] == "ISOLATED_FAULT" for r in rows)) + + def test_namespace_rejects_shared_and_injection(self): + for value in ("production", NS + ";sh", "", "p184-dr08-0123456789abcdef"): + with self.subTest(value=value), self.assertRaises(ValueError): + dr.namespace(value) + + def test_topology_is_bounded_and_disconnected(self): + commands = dr.topology(NS, "/tmp/dr05-test") + self.assertEqual(set(commands), set(dr.ROLES)) + for role, argv in commands.items(): + with self.subTest(role=role): + for required in ("--rm", "--pull=never", "--read-only", "--memory", "--cpus", "--pids-limit", "--log-opt"): + self.assertIn(required, argv) + self.assertIn(dr.LABEL + "=" + NS, argv) + self.assertEqual(argv[argv.index("--network") + 1], + "none" if role == "kafka" else "container:" + NS + "-kafka") + self.assertFalse(set(argv) & {"-p", "--publish", "--privileged", "--network=host"}) + self.assertNotIn("/run/secrets", " ".join(argv)) + self.assertTrue(any(a.startswith("sha256:") for a in argv)) + self.assertIn("scripts/p184_dr08_isolated.py", commands["query"]) + self.assertIn("--query", commands["query"]) + + def test_external_directory_required(self): + for work in (dr.ROOT, dr.ROOT / "runtime", "/tmp/a:rw", "/tmp/a,b"): + with self.subTest(work=work), self.assertRaises(ValueError): + dr.topology(NS, work) + + def test_prepare_exclusive_no_subprocess(self): + with tempfile.TemporaryDirectory() as tmp, patch.object(dr, "sh") as command: + work = Path(tmp) / "case" + packet = dr.prepare(work) + self.assertFalse(packet["acceptance"]) + self.assertFalse(packet["adapter_ready"]) + self.assertEqual(len(packet["schedules"]), 4) + self.assertEqual(work.stat().st_mode & 0o777, 0o700) + self.assertEqual(len((work / "pending-cells.jsonl").read_text().splitlines()), 4) + with self.assertRaises(FileExistsError): + dr.prepare(work) + command.assert_not_called() + + def test_provider_frames(self): + for venue in dr.VENUES: + snap = dr.frame(venue, True, 100, 0, 7, "1", 123000000) + delta = dr.frame(venue, False, 110, 109, 7, "3", 124000000) + self.assertTrue(snap["test_provenance"]) + self.assertEqual(snap["provenance"], "TEST_ONLY") + self.assertEqual(delta["transport"], "WEBSOCKET") + if venue == "BINANCE": + self.assertEqual(snap["transport"], "HTTP") + self.assertEqual(delta["payload"]["pu"], 109) + self.assertEqual(snap["payload"]["lastUpdateId"], 100) + else: + self.assertEqual(snap["payload"]["data"][0]["prevSeqId"], "-1") + self.assertEqual(delta["payload"]["data"][0]["seqId"], "110") + + def test_schedules_positive_gap_return_and_stale(self): + for venue in dr.VENUES: + for fault in dr.FAULTS: + steps = dr.schedule(venue, fault) + self.assertEqual(steps[1]["phase"], "positive_control") + self.assertEqual(steps[-1]["phase"], "post_fence_control") + stale = next(s for s in steps if s["phase"] == "old_generation") + self.assertEqual((stale["generation"], stale["quantity"]), (7, "99")) + self.assertEqual(steps[-1]["generation"], 8) + self.assertEqual(steps[2]["previous"], 109 if fault == "GAP_REFUSAL" else 200) + + def test_unknown_cells_rejected(self): + with self.assertRaises(ValueError): + dr.schedule("BINANCE", "STALE_TYPED") + with self.assertRaises(ValueError): + dr.frame("OTHER", True, 1, 0, 1, "1", 123) + + def test_adapter_status_cannot_fabricate_pass(self): + class Adapter: + def fresh_case(self, venue, fault, timeout): + self.offset = 0 + + def publish(self, raw, timeout): + self.offset += 1 + return self.offset + + def observe(self, offset, timeout): + return {"ok": True, "status": "FAKE_SUCCESS"} + + for venue in dr.VENUES: + for fault in dr.FAULTS: + row = dr.drive(Adapter(), venue, fault) + self.assertFalse(row["ok"]) + self.assertEqual(row["outcome"], "OWNER_REVIEW_REQUIRED") + self.assertEqual(len(row["observations"]), len(dr.schedule(venue, fault))) + + def test_adapter_failure_propagates(self): + class Adapter: + def fresh_case(self, *args, **kwargs): + raise TimeoutError("no positive control") + with self.assertRaises(TimeoutError): + dr.drive(Adapter(), "OKX", "GAP_REFUSAL") + + +class LifecycleTests(unittest.TestCase): + def setUp(self): + self.rows = {} + self.calls = [] + for role in dr.ROLES: + self.rows[role] = {"Name": "/" + NS + "-" + role, "Id": role + "-id", + "Config": {"Labels": {dr.LABEL: NS}}, "State": {"Running": True}, + "Image": dr.IMAGES[{"quota": "redis"}.get(role, role)], + "HostConfig": {"AutoRemove": True, "PortBindings": {}, + "NetworkMode": "none" if role == "kafka" else "container:kafka-id"}} + + def call(self, argv): + self.calls.append(argv) + if argv[1] == "ps": + role = argv[-1].removesuffix("$").split(NS + "-")[1] + return self.rows[role]["Id"] if role in self.rows else "" + role = argv[-1].removesuffix("-id") + if argv[1] == "inspect": + return json.dumps([self.rows[role]]) + if argv[1] == "stop": + del self.rows[role] + return "" + raise AssertionError(argv) + + def test_process_ready_is_not_acceptance(self): + result = dr.readiness(NS, self.call) + self.assertTrue(result["process_isolation_ready"]) + self.assertFalse(result["functional_ready"]) + self.assertFalse(result["acceptance"]) + + def test_readiness_rejects_bad_image_network_ports_and_state(self): + original = copy.deepcopy(self.rows) + for section, key, value in ((None, "Image", "bad"), ("HostConfig", "NetworkMode", "host"), + ("HostConfig", "PortBindings", {"8210": [{}]}), + ("HostConfig", "AutoRemove", False), ("State", "Running", False)): + self.rows = copy.deepcopy(original) + target = self.rows["stream"] if section is None else self.rows["stream"][section] + target[key] = value + self.assertFalse(dr.readiness(NS, self.call)["process_isolation_ready"]) + + def test_missing_role_not_ready(self): + del self.rows["query"] + self.assertFalse(dr.readiness(NS, self.call)["process_isolation_ready"]) + + def test_cleanup_reverse_order_and_idempotent(self): + self.assertEqual(dr.cleanup(NS, self.call)["removed"], list(reversed(dr.ROLES))) + self.assertEqual(dr.cleanup(NS, self.call)["removed"], []) + self.assertFalse(any("rm" in cmd or "prune" in cmd for cmd in self.calls)) + + def test_cleanup_ownership_preflight_stops_nothing(self): + self.rows["kafka"]["Config"]["Labels"][dr.LABEL] = "other" + with self.assertRaises(ValueError): + dr.cleanup(NS, self.call) + self.assertFalse(any(cmd[1] == "stop" for cmd in self.calls)) + + def test_cleanup_rejects_retained_container(self): + self.rows["kafka"]["HostConfig"]["AutoRemove"] = False + with self.assertRaises(ValueError): + dr.cleanup(NS, self.call) + self.assertFalse(any(cmd[1] == "stop" for cmd in self.calls)) + + def test_cleanup_failure_is_not_hidden(self): + def no_stop(argv): + return "" if argv[1] == "stop" else self.call(argv) + with self.assertRaises(RuntimeError): + dr.cleanup(NS, no_stop) + + def test_cleanup_waits_for_async_auto_remove(self): + original = self.call + pending = {} + def call(argv): + if argv[1] == "stop": + role = argv[-1].removesuffix("-id") + self.rows[role]["State"]["Running"] = False + pending[role] = 2 + return "" + if argv[1] == "ps": + role = argv[-1].removesuffix("$").split(NS + "-")[1] + if role in pending: + pending[role] -= 1 + if pending[role] == 0: + del self.rows[role] + return original(argv) + with patch.object(dr.time, "sleep") as sleep: + self.assertEqual(dr.cleanup(NS, call)["removed"], list(reversed(dr.ROLES))) + self.assertEqual(sleep.call_count, 5) + + def test_disappeared_between_list_and_inspect(self): + import subprocess + original = self.call + def call(argv): + if argv[1] == "inspect": + del self.rows["query"] + raise subprocess.CalledProcessError(1, argv) + return original(argv) + self.assertIsNone(dr.owned(NS, "query", call)) + + +class TimeoutTests(unittest.TestCase): + def test_helper_timeout_cleans_container_and_private_files(self): + import subprocess + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "case" + def fail(argv, timeout=120): + for name in dr.PRIVATE_FILES: + (work / name).write_text("test-only") + raise subprocess.TimeoutExpired(argv, timeout) + with patch.object(dr, "bounded", side_effect=fail), patch.object(dr, "cleanup") as cleanup: + with self.assertRaises(subprocess.TimeoutExpired): + dr.init(work, Path(tmp) / "export.json") + cleanup.assert_called_once() + self.assertTrue(all(not (work / n).exists() for n in dr.PRIVATE_FILES)) + self.assertFalse(json.loads((work / "handoff.json").read_text())["material_ready"]) + + def test_private_cleanup_does_not_touch_other_files(self): + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "case" + dr.prepare(work) + (work / "unrelated.key").write_text("retained") + for name in dr.PRIVATE_FILES: + (work / name).write_text("test-only") + dr.erase_private(work) + dr.erase_private(work) + self.assertEqual((work / "unrelated.key").read_text(), "retained") + self.assertTrue(all(not (work / n).exists() for n in dr.PRIVATE_FILES)) + + def test_timed_out_helper_is_owned_and_removed(self): + calls = [] + live = True + def call(argv): + nonlocal live + calls.append(argv) + if argv[1] == "ps": + return "helper-id" if live and argv[-1].endswith("-helper$") else "" + if argv[1] == "inspect": + return json.dumps([{"Name": "/" + NS + "-helper", "Id": "helper-id", + "Config": {"Labels": {dr.LABEL: NS}}, "HostConfig": {"AutoRemove": True}}]) + if argv[1] == "stop": + live = False + return "" + raise AssertionError(argv) + self.assertEqual(dr.cleanup(NS, call)["removed"], ["helper"]) + self.assertFalse(live) + + +class MaterialTests(unittest.TestCase): + @staticmethod + def exported(): + import yaml + catalog = yaml.safe_load((dr.ROOT / "config/v2/stable-source-bindings.yaml").read_text()) + bindings = [] + for venue, market, symbol in (("BINANCE", "USDM", "ETHUSDT"), ("OKX", "SWAP", "ETH-USDT-SWAP")): + instrument = next(i for i in catalog["instruments"] if + (i["venue"], i["market"], i["native_symbol"]) == (venue, market, symbol)) + source = next(b["source"] for b in catalog["bindings"] if + b["instrument_uid"] == instrument["instrument_uid"] and b["feed"] == "BOOK_DELTA") + bindings.append({"provider": source["provider"], "venue": venue, "market": market, + "product_type": "PERPETUAL", "native_symbol": symbol, + "native_channel": "ethusdt@depth@100ms" if venue == "BINANCE" else "books", + "provider_kind": "binance_usdm_book" if venue == "BINANCE" else "okx_book", + "instrument_uid": instrument["instrument_uid"], "instrument_id": instrument["instrument_id"], + "instrument_revision": instrument["metadata_revision"], + "instrument_catalog_revision": catalog["catalog_revision"], "source_id": source["source_id"], + "source_role": "PRIMARY", "normalizer_version": source["normalizer_version"], + "sequence_policy": "CONTIGUOUS", "l2": { + "provider_protocol": "BINANCE_DIFF_DEPTH" if venue == "BINANCE" else "OKX_PUBLIC_BOOKS", + "depth_per_side": 100, "snapshot_refresh_seconds": 30, + "materialized_snapshot_interval_ms": 100}}) + return {"core": {"bindings": bindings}, "password": "MUST_NOT_COPY", "bootstrap": "production"} + + def test_sanitized_configuration(self): + exported = self.exported() + cfg = dr.isolated_config(exported, NS) + self.assertEqual({b["venue"] for b in cfg["core"]["bindings"]}, set(dr.VENUES)) + self.assertTrue(cfg["core"]["allow_test_provenance"]) + self.assertEqual(cfg["raw_topics"], [NS + ".raw"]) + self.assertFalse(cfg["authority"]["public_write_allowed"]) + self.assertNotIn("MUST_NOT_COPY", json.dumps(cfg)) + self.assertNotIn("production", json.dumps(cfg)) + exported["core"]["bindings"].pop() + with self.assertRaises(ValueError): + dr.isolated_config(exported, NS) + + def test_duplicate_book_rejected(self): + exported = self.exported() + exported["core"]["bindings"].append(exported["core"]["bindings"][0]) + with self.assertRaises(ValueError): + dr.isolated_config(exported, NS) + + def test_real_offline_material_and_fresh_keys(self): + from cryptography import x509 + from scripts.kn_gateway_bundle import compile_bundle + from qdl.consumer.manifest import ConsumerManifestLoader + from qdl.runtime.stable_catalog import StableSourceCatalog + with tempfile.TemporaryDirectory() as tmp: + fingerprints = [] + for index in range(2): + work = Path(tmp) / str(index) + packet = dr.prepare(work) + result = dr.material(work, self.exported()) + self.assertTrue(result["material_ready"]) + self.assertFalse(result["adapter_ready"]) + cfg = json.loads((work / "core.json").read_text()) + self.assertEqual(len(cfg["core"]["bindings"]), 2) + catalog = StableSourceCatalog.load(work / "catalog.yaml") + self.assertEqual(len(catalog.bindings), 4) + manifests = [work / (venue.lower() + ".yaml") for venue in dr.VENUES] + for path in manifests: + manifest = ConsumerManifestLoader.load(path) + self.assertTrue(manifest.consumer_id.startswith(packet["namespace"] + ".")) + self.assertEqual({r.feed.value for r in manifest.requirements}, {"BOOK_DELTA", "BOOK_SNAPSHOT"}) + self.assertEqual(json.loads((work / "bundle.json").read_text()), compile_bundle( + environment="paper", catalog_path=work / "catalog.yaml", manifest_paths=manifests)) + for name in ("client.key", "binance.key", "okx.key", "server.p12", "cursor-keys.json", "read-view.secret"): + self.assertEqual((work / name).stat().st_mode & 0o777, 0o600) + cert = x509.load_pem_x509_certificate((work / "client.crt").read_bytes()) + fingerprints.append(cert.serial_number) + self.assertIn("localhost", cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value.get_values_for_type(x509.DNSName)) + runtime = json.loads((work / "packet.json").read_text()) + self.assertNotIn("topic_id", runtime["slice"]) + self.assertEqual(runtime["bootstrap"], "localhost:9092") + with self.assertRaises(FileExistsError): + dr.material(work, self.exported()) + self.assertNotEqual(*fingerprints) + + def test_correct_quota_prefix_and_helper_network(self): + stream = dr.topology(NS, "/tmp/dr05-test", "test-topic-id")["stream"] + self.assertIn("QDL_KN_QUOTA_PREFIX=qdl:stable:v2:paper:" + NS, stream) + self.assertIn("QDL_KN_TOPIC_ID=test-topic-id", stream) + with self.assertRaises(ValueError): + dr.helper_command(NS, "/tmp/dr05-test", [], network="production") + + +if __name__ == "__main__": + unittest.main() diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md index 5af48db1..1a4747ab 100644 --- a/upgrade/evidence/p184-dl-phase-c-packet.md +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -2,6 +2,25 @@ ## Governing owner amendment - 2026-10-08 +### Consolidated-build timing and consumer handoff + +Owner confirms TS CI at `2abbe394ea0fb8f7f5ebcbb745ff70a8c83c9769` +(unit 11415/0, integration 3068/0), image +`sha256:a6548ee36aeb0242c1734c071678c525c1506d05c10a70b533eb7110e7eaf16a`. +This confirmation does NOT complete the pending TS acceptance window. +Prepare [one-commit build/CI](p184-consolidated-build.md) now; execute the +consolidated v2.2.3 rebuild only AFTER TS tests finish, with exact-source DL CI. +No runtime changes or new image builds are authorized by this preparation. + +[SDK 2.1.0 local artifact handoff](qdl-sdk-2.1.0-consumer-handoff.md) is available +to both consumer builds. Alpha must satisfy the declared PyJWT >=2.15.1 +dependency; its current image has 2.13.0. Public release remains HOLD. +DR01 deployed and DR08 final remain unrun until Claude signals stage3. +DR05 isolated-stack preparation is not a passed GAP/generation acceptance; +actual TS validation and linked evidence remain required in that window. + +### Existing runtime and entitlement authority + This supersedes the old core-first requirement below. Historical trial evidence remains immutable. Owner's gate is TS product-path coherent reads (IDR-076 bounded re-read) on existing658a9570 cores. Do not roll0d8da150 solely for coherence. diff --git a/upgrade/evidence/p184-dr05-dl-fault.md b/upgrade/evidence/p184-dr05-dl-fault.md new file mode 100644 index 00000000..cba75f95 --- /dev/null +++ b/upgrade/evidence/p184-dr05-dl-fault.md @@ -0,0 +1,101 @@ +# DR05 Isolated DL Stack Handoff + +Status: STACK_STARTUP_TESTED / TS_FAULT_ACCEPTANCE_NOT_RUN, 2026-10-08. +Governing scope: Unified Plan, "Pre-stage3 consolidated build, SDK and DR05"; +[Phase C packet](p184-dl-phase-c-packet.md). No shared runtime changes. + +## Implemented + +`scripts/p184_dr05_dl_fault.py` prepares a new external directory, sanitizes an +explicitly supplied core binding export and creates fresh short-lived TLS/JWT, +catalog, manifests and Stream bundle. It never discovers shared credentials. +Only ETHUSDT Binance USDM and ETH-USDT-SWAP OKX BOOK snapshot/delta are admitted. +Public binding identity is retained; Kafka topics, groups, Redis and credentials +are isolated. Synthetic provider-shaped fault frames carry TEST_ONLY provenance. + +Five bounded roles use existing immutable images from `p184_dr08_isolated.IMAGES`: +Kafka, Redis quota, Rust core, Rust Stream and the real KN Query read-view service. +Query is **2.2.3-rc2-43bbcf0**, digest +`sha256:77682eea5054ae9ddbd858323b083286669a55eeb4bc7bc065b04bae8854e32f`, +not production v2.2.2. Core/Stream are the existing images recorded by that +inventory, not a consolidated v2.2.3 build. Query source is mounted read-only +from this checkout; this tests a harness stack, not release-image packaging. + +Kafka uses network none; the other roles share only its network namespace. +There are no published ports, shared networks, shared state or broker credentials. +All containers use --rm, immutable image IDs, bounded CPU/RAM/PIDs/logs and a +read-only root filesystem. Readiness verifies identity/isolation separately from +Kafka mTLS, Query TLS and gRPC connectivity. Neither check certifies market data. + +Cleanup prechecks exact names, ownership labels and AutoRemove for all roles, +including bootstrap helpers, then stops them in reverse order. It waits boundedly +for asynchronous Docker removal and handles disappearance between list/inspect. +No broad prune, image removal, shared volume or network deletion occurs. + +## Commands For The Scheduled Window + +Use a NEW directory outside the repository and an explicitly sanitized exported +core JSON with both BOOK bindings. Do not copy secrets or use shared bootstrap. + +```sh +python3 -B scripts/p184_dr05_dl_fault.py --init "$NEW_WORK" --core-config "$PUBLIC_EXPORT" +python3 -B scripts/p184_dr05_dl_fault.py --up "$NEW_WORK" +python3 -B scripts/p184_dr05_dl_fault.py --ready "$NEW_WORK" +# Run the separately integrated real fault publisher / TS observer here. +# Always execute down in the outer harness finally/trap, including on failure: +python3 -B scripts/p184_dr05_dl_fault.py --down "$NEW_WORK" +``` + +The directory must not already exist. TLS lifetime is two hours; regenerate a +fresh packet for a later window. Down removes private files and invalidates the +material marker. Retain bounded receipts, not keys, before removing test files. + +## Remaining Runtime Integration + +The four prepared cells are GAP_REFUSAL and GENERATION_CHANGE_FENCED for each +venue. Schedules cover positive anchor/control, missing sequence or unanchored +generation, new anchor/recovery, late old-generation snapshot and post-fence +control. Timestamps are stamped at publication, never reused from preparation. + +`drive()` defines the adapter interface; it is **not an implemented Kafka +publisher/TS observer**. The scheduled acceptance still needs that integration: +publish real raw-envelope bytes through this core, retain assigned Kafka offsets, +observe canonical/Stream/Query and invoke actual TS validation. Reset only the +disposable case between cells; use bounded deadlines. Do not infer refusal or +recovery from the injected phase name or fabricate status evidence. + +Pending rows have ok=false and no verifier provenance header. Even an adapter +returning all-success cannot make drive emit PASS. Claude must supply reviewed +observations and the final verifier-format evidence under the confirmed TS +SHA/image. Stack startup does not close DR05. DR01 deployed and DR08 final remain +scheduled separately for stage3; no such probes ran in this slice. + +## Tests And Recorded Smoke + +Parent verification in existing image77682eea, --network none, 0.5CPU/512MiB: +`python -B -m unittest tests.test_p184_dr05_dl_fault -q`: +**26 PASS, 0 FAIL, 0 SKIP, 68.539 seconds**. Includes real offline bundle and +manifest compilation, independent keys, private modes, namespace/network/image +checks, fault schedules, false-PASS rejection, timeout cleanup and removal races. + +Agent startup/connectivity smoke used the source catalog-derived +`MaterialTests.exported()` fixture, init -> up -> cleanup -> erase_private in +TemporaryDirectory. Successful rerun exit0, namespace +`p184-dr05-dl-fa9ce97d336d41a2`, elapsed55.13s, all5roles process-ready and +services-reachable; functional_ready=false and acceptance=false. Removed +query/stream/core/quota/kafka, zero private files remaining, temporary directory +removed. Initial cleanup race was fixed and regression-tested before this rerun. + +These smoke numbers are transcribed from the agent's tool-session output, **not +an archived DR acceptance receipt**. No provider/public market capture or TS +fault validation was run. Parent independently confirmed scoped containers gone. +Free disk before/after smoke: 79,907,880,960 / 79,716,106,240 bytes; concurrent host +activity prevents attributing that difference to this test. No new images built. + +## Release And Cleanup Boundary + +Canonical /home/bobby/data_layer main5bf5cd5, published v2.2.2 remains unchanged. +Active worktree /home/bobby/.worktrees/data-layer-p184-closure, +fix/p184-dl-closure. Shared service images/config/manifest untouched. Retain active +images and explicitly protected BAR rollback f7351c3b. No push/merge/public release. +v2.2.3 HOLD; rebuild only after TS test completion and exact-source DL CI. From a96a835d8fea42c0657e2df17d9dbbbcf8ecc3c8 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 09:54:42 +0000 Subject: [PATCH 034/147] test(p184): add bounded DR05 raw publisher and Claude handoff --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 43 ++++++ scripts/p184_dr05_raw_publisher.py | 107 +++++++++++++++ tests/test_p184_dr05_raw_publisher.py | 122 ++++++++++++++++++ .../evidence/p184-claude-stage3-alignment.md | 72 +++++++++++ upgrade/evidence/p184-dr05-dl-fault.md | 6 +- 5 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 scripts/p184_dr05_raw_publisher.py create mode 100644 tests/test_p184_dr05_raw_publisher.py create mode 100644 upgrade/evidence/p184-claude-stage3-alignment.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 2df44d28..22a13a0c 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70219,3 +70219,46 @@ PyJWT2.15.1 dependency update as documented. Recipe18PASS, SDK89PASS inherited; no rerun of unrelated gates. Hosted DL CI remains NOT_RUN, not claimed green. Canonical main5bf5cd5/v2.2.2 and shared runtime unchanged by this work. Active feature fix/p184-dl-closure; no push/merge. Main plan/guide/packet agree on HOLD. + + +### Claude alignment and DR05 raw publisher - 2026-10-08 (Codex) + +Status IMPLEMENTING / NO_SHARED_RUNTIME. Read Claude HEAD2abbe39 and current +dirty roster: Gateway/SDK history pagination is in progress; no TS files edited. +His p184_dr05_isolated.py owns stale/disconnect/fallback and explicitly delegates +GAP/generation upstream injection to the isolated DL stack. Implement the missing +bounded raw-envelope publisher using existing qdl.raw.envelope builder, preserve +broker-confirmed partition/offset and TEST_ONLY provenance. Test identity mismatch, +delivery error/timeout and invalid offset; no invented status/ACK or cell PASS. +Align handoff with Claude, without running DR01 deployed/DR08 final or building +images. Keep IDR040fix2 deferred, IDR075 paired activation and release timing gates. +Scope excludes TS edits, dependency changes in alpha and shared consumer probes. +Use existing image, isolated tests; cleanup only own containers/private files. + + +DR05 raw publisher slice RESULT: IMPLEMENTED_TESTED / TS_ACCEPTANCE_STILL_OPEN. +8unit tests PASS0fail0skip/.230s in existing77682eea network-none container. +Actual isolated Kafka/core/Stream/Query/Redis stack:16TEST_ONLY envelopes, +16exact byte-hash/capture-ID/offset readbacks, Binance+OKX gap schedule including +new-generation anchor/late old-generation. ACK is not canonical/TS acceptance; +no DR05 PASS emitted. Runtime-free tests also reject wrong identity/provenance, +wrong namespace/endpoint/topic, sentinel offset, delivery error and timeout; +ambiguous attempt fences publisher rather than resending. Existing raw builder +preserves original payload clocks/generation; no public contract changes. + +Receipt ~/.local/state/qdl-v2/dr05-publisher-20261008/receipt-1791453132486033908.json +SHA2560764e3bb6ea676ba9e4b5009112ed4bacab95ca53ac354817c8e2a5aec9cbcfe. +Source file digest0a0580078a4dd8fa4449aa528d9e027423cd1d44dbb988a6ffb00ab6aa2d0c4a. +Elapsed66.087s includes material/startup/test/cleanup, not recovery latency. +All5roles/helper removed, privatefiles0, parent label-scoped ps empty; no images +built. Diskfree83,987,636,224 ->83,501,821,952bytes, concurrent-host delta not +attributed to test. No broadprune; active and BARrollbackf7351c3b retained. + +Alignment guide upgrade/evidence/p184-claude-stage3-alignment.md records read-only +inspection of Claude's dirty history-pagination work and alpha_sdk2.4.0, distinct +from qdl_sdk2.1.0. His DR05 runner owns consumer stale/disconnect/fallback; Codex +upstream observer integration remains open, not delegated away as done. CIgreen +2abbe39 cannot certify uncommitted changes; re-pin source/image when finalized. +No actual acknowledgement from Claude is claimed. No TS/alpha edits, shared +runtime/manifest changes, DR01 deployed/DR08 final, push/merge or release. +Canonicalmain5bf5cd5/v2.2.2 unchanged; feature fix/p184-dl-closure, releaseHOLD. diff --git a/scripts/p184_dr05_raw_publisher.py b/scripts/p184_dr05_raw_publisher.py new file mode 100644 index 00000000..7dcd8779 --- /dev/null +++ b/scripts/p184_dr05_raw_publisher.py @@ -0,0 +1,107 @@ +"""Test-only DR05 publisher; broker ACK is not core/TS acceptance.""" +from __future__ import annotations + +import hashlib +import json +import os +import time +import uuid + +from scripts.p184_dr05_dl_fault import namespace + + +def raw_envelope(binding, frame, ns): + from qdl.raw.envelope import build_raw_envelope + + namespace(ns) + if frame.get("test_provenance") is not True or frame.get("provenance") != "TEST_ONLY": + raise ValueError("explicit test provenance required") + if (binding["venue"], binding["native_symbol"]) != (frame["venue"], frame["native_symbol"]): + raise ValueError("frame/binding identity mismatch") + if frame.get("adapter_version") != "p184-synthetic-test-only": + raise ValueError("unexpected test adapter") + if frame.get("transport") not in ("HTTP", "WEBSOCKET"): + raise ValueError("explicit transport required") + if frame["transport"] == "HTTP" and not (frame["venue"] == "BINANCE" and frame["snapshot"] is True): + raise ValueError("HTTP only allowed for Binance snapshot") + payload = json.dumps(frame["payload"], separators=(",", ":"), allow_nan=False).encode() + return build_raw_envelope( + capture_id=uuid.uuid4().bytes, provider=binding["provider"], venue=binding["venue"], + market=binding["market"], product_type=binding["product_type"], + native_symbol=binding["native_symbol"], native_channel=binding["native_channel"], + subscription_id=binding["source_id"], source_session_id=ns + "." + binding["venue"], + connection_generation=frame["connection_generation"], lease_epoch=1, + authority_revision=1, partition_plan_epoch=1, received_at_ns=frame["received_at_ns"], + transport_protocol=2 if frame["transport"] == "HTTP" else 1, + transport_compression=1, capture_boundary=2, raw_frame_bytes=payload, + adapter_version=frame["adapter_version"], config_revision=1, + instrument_catalog_revision=binding["instrument_catalog_revision"], + correlation_id=str(uuid.uuid4()), test_provenance=True) + + +class RawPublisher: + """Single-flight, one attempt; never retry an ambiguous delivery automatically. + + Caller must run inside the disposable broker network namespace and attest it + through P184_DR05_ISOLATED_NAMESPACE. No externally supplied Kafka endpoint + or TLS override is accepted. Positive offsets alone cannot qualify a cell. + """ + + def __init__(self, packet, config): + from confluent_kafka import Producer + + self.ns = namespace(packet["namespace"]) + if os.environ.get("P184_DR05_ISOLATED_NAMESPACE") != self.ns: + raise ValueError("disposable namespace attestation required") + if packet.get("bootstrap") != "localhost:9092": + raise ValueError("isolated loopback broker required") + if config.get("raw_topics") != [self.ns + ".raw"] or config["core"].get("allow_test_provenance") is not True: + raise ValueError("isolated test-only raw topic required") + self.topic = config["raw_topics"][0] + self.bindings = {b["venue"]: b for b in config["core"]["bindings"]} + if len(config["core"]["bindings"]) != 2 or set(self.bindings) != {"BINANCE", "OKX"}: + raise ValueError("exact two-venue binding roster required") + self.producer = Producer({"bootstrap.servers": "localhost:9092", "security.protocol": "SSL", + "ssl.ca.location": "/work/ca.crt", "ssl.certificate.location": "/work/client.crt", + "ssl.key.location": "/work/client.key", "enable.idempotence": True, "acks": "all", + "message.timeout.ms": 5000, "queue.buffering.max.messages": 8, + "queue.buffering.max.kbytes": 1024, "linger.ms": 0}) + self.failed = False + + def publish(self, frame, *, timeout=5): + if self.failed: + raise RuntimeError("publisher fenced after unsuccessful/ambiguous delivery") + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not 0 < timeout <= 5: + raise ValueError("publication deadline must be in (0, 5] seconds") + raw = raw_envelope(self.bindings[frame["venue"]], frame, self.ns) + wire = raw.SerializeToString() + delivered = [] + start = time.monotonic() + + def ack(error, message): + if error is not None: + delivered.append({"error": str(error.code())}) + elif message.topic() != self.topic or message.partition() != 0 or message.offset() < 0: + delivered.append({"error": "BROKER_OFFSET_OR_IDENTITY_UNKNOWN"}) + else: + delivered.append({"topic": message.topic(), "partition": message.partition(), + "offset": message.offset()}) + + try: + self.producer.produce(self.topic, partition=0, key=raw.subscription_id.encode(), + value=wire, headers=[("qdl-event-id", raw.capture_id)], on_delivery=ack) + deadline = start + timeout + while not delivered: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("raw delivery unknown; do not resend this attempt") + self.producer.poll(min(.1, remaining)) + if len(delivered) != 1 or "error" in delivered[0]: + raise RuntimeError("raw publication failed: " + str(delivered)) + except BaseException: + self.failed = True + raise + return {**delivered[0], "raw_sha256": hashlib.sha256(wire).hexdigest(), + "capture_id": raw.capture_id.hex(), "generation": raw.connection_generation, + "broker_ack_ms": (time.monotonic() - start) * 1000, + "provenance": "TEST_ONLY", "acceptance": False} diff --git a/tests/test_p184_dr05_raw_publisher.py b/tests/test_p184_dr05_raw_publisher.py new file mode 100644 index 00000000..618b344d --- /dev/null +++ b/tests/test_p184_dr05_raw_publisher.py @@ -0,0 +1,122 @@ +import copy +import json +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from scripts.p184_dr05_dl_fault import frame +from scripts.p184_dr05_raw_publisher import RawPublisher, raw_envelope + +NS = "p184-dr05-dl-0123456789abcdef" + + +def binding(venue): + return dict(provider=venue, venue=venue, market="USDM" if venue == "BINANCE" else "SWAP", + product_type="PERPETUAL", native_symbol="ETHUSDT" if venue == "BINANCE" else "ETH-USDT-SWAP", + native_channel="ethusdt@depth@100ms" if venue == "BINANCE" else "books", + source_id="test-" + venue, instrument_catalog_revision=10) + + +class Producer: + def __init__(self, config): + self.config, self.calls, self.failure = config, [], None + self.offset, self.partition, self.topic_override = 0, 0, None + self.deliver = True + + def produce(self, topic, **kwargs): + self.calls.append((topic, kwargs)) + self.callback = kwargs["on_delivery"] + + def poll(self, timeout): + if self.deliver: + self.deliver = False + self.callback(self.failure, SimpleNamespace(topic=lambda: self.topic_override or self.calls[-1][0], + partition=lambda: self.partition, offset=lambda: self.offset)) + + +class PublisherTests(unittest.TestCase): + def setUp(self): + self.packet = {"namespace": NS, "bootstrap": "localhost:9092"} + self.config = {"raw_topics": [NS + ".raw"], "core": {"allow_test_provenance": True, + "bindings": [binding(v) for v in ("BINANCE", "OKX")]}} + self.frame = frame("BINANCE", True, 100, 0, 7, "1", 123000000) + self.addCleanup(patch.stopall) + patch.dict("os.environ", {"P184_DR05_ISOLATED_NAMESPACE": NS}).start() + patch("confluent_kafka.Producer", Producer).start() + + def test_envelope_both_venues_and_generations(self): + from qdl.raw.envelope import validate_raw_envelope + for venue in ("BINANCE", "OKX"): + for gen in (7, 8): + value = raw_envelope(binding(venue), frame(venue, True, 100, 0, gen, "1", 123000000), NS) + validate_raw_envelope(value) + self.assertTrue(value.test_provenance) + self.assertEqual(value.connection_generation, gen) + self.assertEqual(value.source_session_id, NS + "." + venue) + self.assertEqual(value.received_at_ns, 123000000) + self.assertIsInstance(json.loads(value.raw_frame_bytes), dict) + + def test_bad_provenance_identity_transport_rejected(self): + for key, value in (("test_provenance", False), ("provenance", "LIVE"), + ("native_symbol", "BTCUSDT"), ("transport", "OTHER"), + ("adapter_version", "production")): + changed = {**self.frame, key: value} + with self.subTest(key=key), self.assertRaises(ValueError): + raw_envelope(binding("BINANCE"), changed, NS) + + def test_zero_offset_valid_and_no_acceptance(self): + pub = RawPublisher(self.packet, self.config) + receipt = pub.publish(self.frame) + self.assertEqual(receipt["offset"], 0) + self.assertFalse(receipt["acceptance"]) + self.assertEqual(len(receipt["raw_sha256"]), 64) + self.assertEqual(len(pub.producer.calls), 1) + + def test_negative_offset_wrong_partition_topic_fences(self): + for attr, value in (("offset", -1001), ("partition", 1), ("topic_override", "shared")): + pub = RawPublisher(self.packet, self.config) + setattr(pub.producer, attr, value) + with self.subTest(attr=attr), self.assertRaises(RuntimeError): + pub.publish(self.frame) + with self.assertRaises(RuntimeError): + pub.publish(self.frame) + self.assertEqual(len(pub.producer.calls), 1) + + def test_delivery_error_fences(self): + pub = RawPublisher(self.packet, self.config) + pub.producer.failure = SimpleNamespace(code=lambda: 123) + with self.assertRaises(RuntimeError): + pub.publish(self.frame) + self.assertTrue(pub.failed) + + def test_timeout_no_retry(self): + pub = RawPublisher(self.packet, self.config) + pub.producer.deliver = False + with patch("scripts.p184_dr05_raw_publisher.time.monotonic", side_effect=[10, 11]), self.assertRaises(TimeoutError): + pub.publish(self.frame, timeout=.5) + self.assertTrue(pub.failed) + self.assertEqual(len(pub.producer.calls), 1) + + def test_scope_checks(self): + for key, value in (("bootstrap", "production:9092"), ("namespace", "production")): + with self.assertRaises(ValueError): + RawPublisher({**self.packet, key: value}, self.config) + with patch.dict("os.environ", {"P184_DR05_ISOLATED_NAMESPACE": "wrong"}), self.assertRaises(ValueError): + RawPublisher(self.packet, self.config) + for modify in (lambda c: c.update(raw_topics=["shared"]), + lambda c: c["core"].update(allow_test_provenance=False), + lambda c: c["core"]["bindings"].pop()): + cfg = copy.deepcopy(self.config); modify(cfg) + with self.assertRaises(ValueError): + RawPublisher(self.packet, cfg) + + def test_bad_deadline_never_sends(self): + pub = RawPublisher(self.packet, self.config) + for value in (0, -1, 6, True, float("nan")): + with self.assertRaises(ValueError): + pub.publish(self.frame, timeout=value) + self.assertFalse(pub.producer.calls) + + +if __name__ == "__main__": + unittest.main() diff --git a/upgrade/evidence/p184-claude-stage3-alignment.md b/upgrade/evidence/p184-claude-stage3-alignment.md new file mode 100644 index 00000000..f6d5f696 --- /dev/null +++ b/upgrade/evidence/p184-claude-stage3-alignment.md @@ -0,0 +1,72 @@ +# Claude / Codex Stage3 Alignment + +Read-only TS inspection on 2026-10-08; this document is a handoff, not a message +acknowledged by Claude and not a runtime certificate. Unified Plan journal: +"Claude alignment and DR05 raw publisher". DL release remains HOLD. + +## Concurrent Ownership + +TS worktree `/home/bobby/.worktrees/trading-system-next-upgrade` is owned by +Claude. HEAD at inspection: `2abbe394ea0fb8f7f5ebcbb745ff70a8c83c9769`. +Dirty work includes Gateway order/fill history pagination, cursor module, +database index, tests, compatibility manifests and alpha_sdk2.4.0. No TS file was +changed by Codex. These are source observations, not claims that the new work +passed CI or was deployed. Owner's plan-heading blank-line edit is preserved. + +The previous CI/image attestation at2abbe39/a6548ee3 remains valid only for that +revision. New source must not silently inherit that identity. Confirm the final +TS SHA/image before writing final provenance. `alpha_sdk`2.4.0 is the TS trading +SDK, not `qdl_sdk`2.1.0, the Data Layer SDK. + +## Work Which Does Not Wait + +- Codex: DR05 raw injection plumbing and isolated regressions, using the already + prepared five-role DL stack. No shared provider, credentials, quota or runtime. +- Claude: actual TS validation/observation integration for upstream fault cells. + His `scripts/p184_dr05_isolated.py` already owns STALE_TYPED, + DISCONNECT_RECONNECT and APPROVED_FALLBACK_AND_RETURN_NO_FORBIDDEN_V1. It + explicitly excludes GAP_REFUSAL and GENERATION_CHANGE_FENCED, which require + upstream faults; do not duplicate those existing consumer-transport cases. +- Consumer packaging: use the immutable SDK wheel in + [SDK handoff](qdl-sdk-2.1.0-consumer-handoff.md). Alpha's inspected image has + PyJWT2.13.0; install against the declared >=2.15.1 requirement. Exact2.15.1 plus + the SDK was already verified in the isolated alpha image. Do not lower the + dependency requirement, change strategy logic, or rebuild unrelated services. +- [One-commit recipe/CI](p184-consolidated-build.md) is implemented; no need to + rebuild it or rerun unrelated acceptance while TS continues. + +## Gates Which Still Wait + +DR01 deployed and DR08 final need Claude's stage3 signal. DR05 upstream cells +need raw -> canonical -> reader -> actual TS refusal/recovery proof, not only +Kafka ACK or process readiness. Wrong-account DR01 remains Gateway-owned. +IDR075 grants require the paired reader/caller revision window. IDR040fix2 +heartbeat/quiet-field contract remains deferred; no silent freshness change. +Consolidated DL rebuild waits for TS acceptance completion and exact-source CI; +public release additionally requires DR01-DR08/broker delta and rollout receipt. + +## New Raw Publisher Contract + +`scripts/p184_dr05_raw_publisher.py` uses the existing validated raw-envelope +builder. `RawPublisher(packet, config).publish(frame, timeout=5)` returns the +actual broker topic/partition/offset, raw-byte hash and capture ID. It preserves +the frame's clock, generation and explicit TEST_ONLY provenance. Same venue, +symbol, namespace and transport checks run before publishing. Its endpoints and +TLS paths are fixed to the disposable loopback namespace. + +The calling disposable helper must set `P184_DR05_ISOLATED_NAMESPACE` to its +packet namespace. This attestation supplements Docker namespace isolation; it +does not replace it. No automatic retries after timeout or delivery errors. +Offset0 is valid. Unknown/sentinel offsets fail the test because this observer +needs an exact offset; this does not change production ingestor ACK semantics. + +Broker ACK means publication only. Every receipt has acceptance=false, and the +publisher cannot close a DR cell. Match capture ID and committed offset to +canonical/quarantine observations, then test the actual TS reader/validator. +Do not manufacture status from the injected phase or treat a quiet poll as proof +that an old-generation event was fenced. Final evidence must retain controls, +typed refusal, post-fault recovery and both venue identities. + +Canonical DL remains main5bf5cd5/v2.2.2, feature fix/p184-dl-closure. No push, +merge, shared runtime change or image build. Retain active/rollback artifacts, +including BARf7351c3b; cleanup is restricted to owned disposable tests. diff --git a/upgrade/evidence/p184-dr05-dl-fault.md b/upgrade/evidence/p184-dr05-dl-fault.md index cba75f95..6a9fc806 100644 --- a/upgrade/evidence/p184-dr05-dl-fault.md +++ b/upgrade/evidence/p184-dr05-dl-fault.md @@ -57,8 +57,10 @@ venue. Schedules cover positive anchor/control, missing sequence or unanchored generation, new anchor/recovery, late old-generation snapshot and post-fence control. Timestamps are stamped at publication, never reused from preparation. -`drive()` defines the adapter interface; it is **not an implemented Kafka -publisher/TS observer**. The scheduled acceptance still needs that integration: +`drive()` defines the adapter interface. The bounded Kafka publisher is now +implemented in `scripts/p184_dr05_raw_publisher.py` and has a 16/16 exact-byte/ +offset readback on an isolated broker. See [alignment handoff](p184-claude-stage3-alignment.md). +The canonical/Query/Stream observer and TS integration remain incomplete: publish real raw-envelope bytes through this core, retain assigned Kafka offsets, observe canonical/Stream/Query and invoke actual TS validation. Reset only the disposable case between cells; use bounded deadlines. Do not infer refusal or From 47d906e5028751284041432d0255e09d5b262817 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 10:18:43 +0000 Subject: [PATCH 035/147] fix(query): preserve standard-base64 Kafka topic UUID in cursor coordinates --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 30 +++++++++++++++++++++++ qdl/runtime/kn_query_backend.py | 2 +- tests/test_kn_cursor_topic_id.py | 27 ++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 tests/test_kn_cursor_topic_id.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 22a13a0c..9154fb10 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70262,3 +70262,33 @@ upstream observer integration remains open, not delegated away as done. CIgreen No actual acknowledgement from Claude is claimed. No TS/alpha edits, shared runtime/manifest changes, DR01 deployed/DR08 final, push/merge or release. Canonicalmain5bf5cd5/v2.2.2 unchanged; feature fix/p184-dl-closure, releaseHOLD. + + +### DR05 canonical reader and TS validation closure - 2026-10-08 (Codex) + +Status IMPLEMENTING. Owner approves completing the isolated observation path now. +Guide upgrade/evidence/p184-dr05-dl-fault.md, alignment guide; preserve shared +runtime, TS worktree, release HOLD, no DR01 deployed/DR08 final. Raw broker ACK +is not acceptance. Match raw capture IDs through read_committed canonical or +quarantine, capture real Query/Stream response and typed quality, then validate +with code inside fixed TS image (no app-source overlay). Independent positive, +gap/unanchored-generation, recovery and old-generation fencing controls on both +venues. No manufactured quality/timestamp/status; unknown evidence fails closed. +Tests: bounded observer correlation, wrong identity/offset, genuine refusal, +positive/recovered book, isolated real stack. Cleanup exact owned labels/names +and private material; retain bounded hashed receipts; no images built/push/merge. + +DR05 integration discovery: exact Kafka Uuid(2^63-1,2^63-1) string contains '/'. +Query intermediate cursor regex rejects it, while signed cursor V3 accepts it. +New regression on old source:3tests/2errors. Necessary narrow source correction: +accept '/' without rewriting topic identity, retain delimiter/negative checks. +No public contract or deployed image changes. Initial Binance observation cursor +failure kept; OKX alternate UUID succeeded, not used to erase initial failure. + +Cursor parser slice RESULT: fixed only intermediate coordinate character class +for standard-base64 Kafka UUID '/'. No rewriting/signature/schema changes. +Counterfactual oldsource:3tests,2errors; candidate3PASS. Combined cursor/DR05/ +hot-view suite60PASS0fail0skip/33.706s in existing77682eea image network-none. +Real Binance snapshot now returned signed cursor and reached actual pinned TS +SDK/adapter/BOOK validator (positive control accepted). Runtime/shared images +unchanged; no certification from this partial case, recovery adapter ongoing. diff --git a/qdl/runtime/kn_query_backend.py b/qdl/runtime/kn_query_backend.py index e75b8958..0b9af175 100644 --- a/qdl/runtime/kn_query_backend.py +++ b/qdl/runtime/kn_query_backend.py @@ -84,7 +84,7 @@ from qdl.transport.contracts import DurableEvent PLACEHOLDER_PREFIX = "kn3-source:" -_PLACEHOLDER = re.compile(r"kn3-source:([A-Za-z0-9._@|+=-]{1,256}):([0-9]{1,10}):([0-9]{1,19})") +_PLACEHOLDER = re.compile(r"kn3-source:([A-Za-z0-9._/@|+=-]{1,256}):([0-9]{1,10}):([0-9]{1,19})") QUERY_BACKEND_ENV = "QDL_STABLE_QUERY_BACKEND" QUERY_BACKENDS = ("spool", "kn3") CURSOR_KEYS_FILE_ENV = "QDL_KN_CURSOR_KEYS_FILE" diff --git a/tests/test_kn_cursor_topic_id.py b/tests/test_kn_cursor_topic_id.py new file mode 100644 index 00000000..437bb259 --- /dev/null +++ b/tests/test_kn_cursor_topic_id.py @@ -0,0 +1,27 @@ +"""Kafka UUIDs use standard base64, including '/', without identity rewriting.""" +import unittest +from types import SimpleNamespace +from qdl.runtime.kn_query_backend import parse_placeholder, source_placeholder + + +class KafkaTopicCoordinateTests(unittest.TestCase): + def test_librdkafka_uuid_roundtrip(self): + from confluent_kafka import Uuid + topic_id = str(Uuid(9223372036854775807, 9223372036854775807)) + self.assertIn("/", topic_id) + view = SimpleNamespace(boundary=SimpleNamespace(topic_id=topic_id, partition=0, offset=0)) + self.assertEqual(parse_placeholder(source_placeholder(view)), (topic_id, 0, 0)) + + def test_legacy_and_urlsafe_coordinates_unchanged(self): + for value in ("topic", "topic_a-b", "a+b/c=="): + self.assertEqual(parse_placeholder(f"kn3-source:{value}:12:345"), (value, 12, 345)) + + def test_delimiters_whitespace_and_negative_offset_refused(self): + for value in ("kn3-source:a:b:0:0", "kn3-source:a\n:0:0", "kn3-source:a:0:-1001", + "kn3-source::0:0", "kn3-source:a:0:0\n"): + with self.subTest(value=value), self.assertRaises(ValueError): + parse_placeholder(value) + + +if __name__ == "__main__": + unittest.main() From 6617f6d6d3535ad525756a6b2b71abaddcb6e399 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 10:47:08 +0000 Subject: [PATCH 036/147] test(dr05): observe canonical faults through readers and TS cache validation --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 76 ++++ scripts/p184_dr05_dl_fault.py | 16 +- scripts/p184_dr05_observe_stack.py | 115 +++++ scripts/p184_dr05_observer.py | 262 ++++++++++++ scripts/p184_dr05_ts_validate.py | 399 ++++++++++++++++++ scripts/p184_dr05_verdict.py | 53 +++ tests/test_p184_dr05_dl_fault.py | 2 +- tests/test_p184_dr05_observer.py | 80 ++++ tests/test_p184_dr05_ts_validate.py | 366 ++++++++++++++++ tests/test_p184_dr05_verdict.py | 92 ++++ .../evidence/p184-claude-stage3-alignment.md | 41 ++ upgrade/evidence/p184-dr05-dl-fault.md | 65 ++- .../evidence/p184-dr05-observer-results.md | 93 ++++ 13 files changed, 1648 insertions(+), 12 deletions(-) create mode 100644 scripts/p184_dr05_observe_stack.py create mode 100644 scripts/p184_dr05_observer.py create mode 100644 scripts/p184_dr05_ts_validate.py create mode 100644 scripts/p184_dr05_verdict.py create mode 100644 tests/test_p184_dr05_observer.py create mode 100644 tests/test_p184_dr05_ts_validate.py create mode 100644 tests/test_p184_dr05_verdict.py create mode 100644 upgrade/evidence/p184-dr05-observer-results.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 9154fb10..73502e90 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70292,3 +70292,79 @@ hot-view suite60PASS0fail0skip/33.706s in existing77682eea image network-none. Real Binance snapshot now returned signed cursor and reached actual pinned TS SDK/adapter/BOOK validator (positive control accepted). Runtime/shared images unchanged; no certification from this partial case, recovery adapter ongoing. + + +DR05 observer/TS slice IN_PROGRESS: actual core read_committed canonical and +quarantine are correlated by capture ID, with a post-commit EOF barrier for +filtered frames. Query handoffs are captured from the actual public serializer; +Stream unary GetFeedStatus/GetSnapshot use mTLS/JWT isolated EXECUTION identities. +The fixed TS image a6548ee3 loads its own SDK/adapter/BOOK validator, then calls +MarketCacheProjector.project_execution_snapshot and re-reads isolated Redis DB2. +No TS application overlays, fabricated quality, clock override or state reset +between generation changes. This is NOT the full TS writer Subscribe/ACK loop. + +NEW FINDING: real isolated OKX unanchored generation was quarantined by core, +but both reader BOOK views remained LIVE and returned the old generation while +within freshness. Initial exact receipt okx-generation-r1 SHA256 +0e7353eb42938c7ebfd2ae8317c6691c1b7913317fcd59375c0efc70dc5974bf. +BOOK_SNAPSHOT age589ms, BOOK_DELTA731ms, each GetSnapshot OK. Later control +recovers with genuine anchor and old-generation raw frame is filtered. Core +src/lib.rs sequence-gap branch returns quarantine/resync, not reader invalidation. +Do not close GAP/generation by waiting for DATA_STALE; no production incident +is inferred from this TEST_ONLY injection. Shared runtime remains unchanged. + +Harness correction: a Redis NO_WRITE/cache equality false is not typed refusal. +The preliminary okx-generation-final receipt (d5c2b8ce...) retains that weaker +subcheck; the final verdict requires explicit ENDPOINT_REFUSED/APPLICATION_REFUSED +and records BOOK validation separately from cache application. No old receipt +rewritten. Parent final regression51PASS0fail0skip/31.718s. Four final isolated +venue/fault cells now executing sequentially, using existing images only. + +Read-only coordination update: Claude TS HEAD01eb0d7 adds p184_dr05_upstream.py +and WriterClone. Its JSONL reset_case/publish/canonical_observe orchestration is +not yet the same as this per-case observer + in-image validator harness. Do not +claim his full writer clone was run by these receipts. Alignment handoff will +record the interfaces and the newly exposed upstream fault, without TS edits. + + +DR05 slice RESULT: IMPLEMENTED_TESTED_ISOLATED / FAULT_PROPAGATION_FAIL. +Report upgrade/evidence/p184-dr05-observer-results.md (exact hashes, commands, +counts, images, cleanup); guide and Claude alignment updated together. +Four selected cells (Binance/OKX x GAP/unanchored generation) all FAIL the real +fault propagation check: previous BOOK_SNAPSHOT age473/544/465/542ms remains +LIVE/eligible and passes actual TS BOOK validation. Core quarantines the exact +fault capture. Redis NO_WRITE is not refusal. Twelve positive/recovery/post-fence +controls match exact canonical snapshot byte hashes and reach TS Redis apply. +Four late old-generation frames are filtered after raw commit; no generation +rollback. No provider evidence or execution certification inferred. + +One fixture repair: Binance buffers delta201 and replays it at new anchor200; +return_control must reference201, not200. Earlier generation-v2 failure retained; +only that affected case rerun as generation-v3. Selected receipt SHA256 values: +8e4eae0dc523256305a70e4066ba6397d3e18d6a86ddb1e0387db1a836b4b25e +3703f46edf9bd308620724982a86d09b54e7b74b3a7b50ef0127278f744e53ac +590291f159d34c89919ff9a864a43987800f99e70025d11426c2c087df294bdb +f89356e55d6e688555c1fe943d97619a0fb4ca3567460e3a74f5b818658fd9c2 +(root ~/.local/state/qdl-v2/dr05-observer-20261008/, directories in report). + +Final parent DL suite52PASS0fail0skip32.861s (verdict, observer, raw publisher, +stack material and cursor regression). Agent TS image+isolated Redis22PASS0skip +5.992s, followed by3focused marker/cache tests PASS3.638s. Parent independent +TS network-none19PASS3Redis skips2.311s. Overlap not summed. Exact production +MarketCacheProjector handles new generation; no deletion/reset/clock fabrication. +Helper application modules load from TSimagea6548ee3, not source mounts. + +Cleanup all owned cases complete, privatefiles0; report has byte-level disk +pre/post. No images built; inventory95images28.96GB/BuildKit20.52GB is shared and +not broad-pruned during Claude work. Retain active images and BARrollbackf7351c3b. +Final read-only shared snapshot12app+5infra: restart0/OOMfalse, same role digests; +not a runtime acceptance. Canonicalmain5bf5cd5/v2.2.2 and feature location unchanged. +No runtime edits, TS edits, push/merge or release. v2.2.3 HOLD. + +Remaining implementation (not approval debt): propagate core BOOK gap/generation +invalidation to canonical reader quality, then close on valid anchor. Both hot +backup and primary need the fence. Full Claude WriterClone JSONL orchestration +is not claimed completed: his new01eb0d7 interface needs fresh-case mapping and +must not count eventual DATA_STALE as immediate GAP refusal. These interface +points are recorded in p184-claude-stage3-alignment.md. Reuse this counterexample; +no C2/full catalogue rerun, no shared rollout from this observer slice. diff --git a/scripts/p184_dr05_dl_fault.py b/scripts/p184_dr05_dl_fault.py index 92c5aab2..a7f5f495 100644 --- a/scripts/p184_dr05_dl_fault.py +++ b/scripts/p184_dr05_dl_fault.py @@ -86,8 +86,11 @@ def schedule(venue, fault): ("still_unanchored", False, 111, 110, 7, "4")] else: steps += [("new_generation_unanchored", False, 201, 200, 8, "3")] + # Binance replays the buffered 201 delta when snapshot200 arrives. + # OKX rejects the unanchored delta and starts from snapshot200 instead. + return_previous = 201 if venue == "BINANCE" and fault == "GENERATION_CHANGE_FENCED" else 200 steps += [("return_anchor", True, 200, 0, 8, "5"), - ("return_control", False, 202, 200, 8, "6"), + ("return_control", False, 202, return_previous, 8, "6"), ("old_generation", True, 300, 0, 7, "99"), ("post_fence_control", False, 203, 202, 8, "7")] return [{"phase": phase, "venue": venue, "snapshot": snap, "sequence": seq, @@ -153,7 +156,7 @@ def topology(ns, work, topic_id=""): ["/usr/local/bin/qdl-stream-gateway", "serve"]), "query": ("query", "512m", "0.5", {**common, "PYTHONPATH": "/app:/app/generated/python", "PYTHONDONTWRITEBYTECODE": "1"}, - ["python", "-B", "scripts/p184_dr08_isolated.py", "--query", "/work"]), + ["python", "-B", "scripts/p184_dr05_observer.py", "--query", "/work"]), } result = {} for role, (image, memory, cpus, env, command) in spec.items(): @@ -179,7 +182,7 @@ def sh(argv): def owned(ns, role, call=sh): name = namespace(ns) + "-" + role - if role not in (*ROLES, "helper"): + if role not in (*ROLES, "helper", "tsvalidator"): raise ValueError("unknown owned role") ids = call(["docker", "ps", "-aq", "--filter", "name=^/" + name + "$"]).splitlines() if not ids: @@ -219,7 +222,7 @@ def cleanup(ns, call=sh): removed = [] # Preflight ALL ownership before stopping anything. Never remove networks, # images, shared volumes, files or a container selected only by a label. - rows = [(role, owned(ns, role, call)) for role in reversed((*ROLES, "helper"))] + rows = [(role, owned(ns, role, call)) for role in reversed((*ROLES, "helper", "tsvalidator"))] if any(row and row["HostConfig"].get("AutoRemove") is not True for _, row in rows): raise ValueError("refuse non-disposable container") failures = [] @@ -399,9 +402,10 @@ def save(name, value, private=False): for feed in ("BOOK_SNAPSHOT", "BOOK_DELTA"): binding = next(x for x in loaded.bindings if x.source_id == b["source_id"] and x.feed.value == feed) mapping = manifest_mapping(consumer_id=cid, subject=subject, instrument_uid=b["instrument_uid"], - feed=feed, interval=None, source_policy_id=binding.source_policy_id) + feed=feed, interval=None, source_policy_id=binding.source_policy_id, + grade="EXECUTION", purposes=("INTERNAL_EXECUTION",)) requirement = mapping["spec"]["requirements"][0] - requirement.update(warmup_limit=1, require_full_coverage=False, require_final_bars=False, + requirement.update(warmup_limit=1, require_full_coverage=True, require_final_bars=False, max_freshness_ms=2000) requirements.append(requirement) mapping["spec"]["requirements"] = requirements diff --git a/scripts/p184_dr05_observe_stack.py b/scripts/p184_dr05_observe_stack.py new file mode 100644 index 00000000..dedfeb47 --- /dev/null +++ b/scripts/p184_dr05_observe_stack.py @@ -0,0 +1,115 @@ +"""Disposable DR05 observer rehearsal; no shared runtime or certification.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import time + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +from scripts import p184_dr05_dl_fault as dr + + +def exercise(output, exported, venue, fault): + output = output.resolve() + dr.topology("p184-dr05-dl-0123456789abcdef", output) + output.mkdir(mode=0o700, parents=True, exist_ok=False) + started, free_before = time.monotonic(), shutil.disk_usage(output).free + receipt = {"schema": "p184.dr05.stack-observation.v1", "acceptance": False, + "venue": venue, "fault": fault, "images": dr.IMAGES, "source_files": {}} + for name in ("p184_dr05_observer.py", "p184_dr05_raw_publisher.py", "p184_dr05_dl_fault.py"): + receipt["source_files"][name] = hashlib.sha256((ROOT / "scripts" / name).read_bytes()).hexdigest() + ok = False + with tempfile.TemporaryDirectory(prefix="p184-dr05-observe-") as tmp: + work = Path(tmp) / "work" + initial = dr.init(work, exported) + ns = initial["namespace"] + receipt["namespace"] = ns + try: + receipt["startup"] = dr.up(work) + ts_image = "sha256:a6548ee36aeb0242c1734c071678c525c1506d05c10a70b533eb7110e7eaf16a" + live_args = ["docker", "run", "-d", "--rm", "--pull=never", "--name", ns + "-tsvalidator", + "--label", dr.LABEL + "=" + ns, "--network", "container:" + ns + "-kafka", "--read-only", + "--user", "1001:1001", "--memory", "512m", "--cpus", "0.5", "--pids-limit", "128", + "--log-opt", "max-size=4m", "--log-opt", "max-file=1", + "--tmpfs", "/tmp:rw,size=64m,mode=1777", "-e", "PYTHONDONTWRITEBYTECODE=1", + "-e", "PYTHONPATH=/app", "-e", "P184_DR05_ISOLATED_NAMESPACE=" + ns, + "-v", str(ROOT / "scripts/p184_dr05_ts_validate.py") + ":/probe.py:ro", + "--entrypoint", "python", ts_image, "-B", "/probe.py", "--serve", "--namespace", ns] + dr.bounded(live_args) + argv = dr.helper_command(ns, work, [], network="container:" + ns + "-kafka") + index = argv.index(dr.IMAGES["python"]) + argv[index:index] = ["-e", "P184_DR05_ISOLATED_NAMESPACE=" + ns] + argv[-1:] = ["scripts/p184_dr05_observer.py", "--work", "/work", "--venue", venue, "--fault", fault] + receipt["observation"] = json.loads(dr.bounded(argv, 120)) + from scripts.p184_dr05_verdict import evaluate + receipt["case_verdict"] = evaluate(receipt["observation"]) + ts_image = "sha256:a6548ee36aeb0242c1734c071678c525c1506d05c10a70b533eb7110e7eaf16a" + ts_args = ["docker", "run", "--rm", "--pull=never", "--name", ns + "-helper", + "--label", dr.LABEL + "=" + ns, "--network", "none", "--read-only", + "--user", "1001:1001", "--memory", "512m", "--cpus", "0.5", "--pids-limit", "128", + "--tmpfs", "/tmp:rw,size=64m,mode=1777", "-e", "PYTHONDONTWRITEBYTECODE=1", + "-e", "PYTHONPATH=/app", "-v", str(ROOT / "scripts/p184_dr05_ts_validate.py") + ":/probe.py:ro", + "-i", "--entrypoint", "python", ts_image, "-B", "/probe.py"] + ts = subprocess.run(ts_args, input=(work / "ts-input.json").read_text(), text=True, + capture_output=True, timeout=45) + receipt["ts_validation"] = {"image": ts_image, "exit_code": ts.returncode, + "output": json.loads(ts.stdout), "stderr": ts.stderr[-2000:]} + ok = ts.returncode == 0 + except Exception as error: + receipt["error"] = {"type": type(error).__name__, "detail": str(error)[:2000], + "stderr": getattr(error, "stderr", "")[-3000:]} + partial = work / "observations.json" + if partial.exists(): + receipt["partial_observations"] = json.loads(partial.read_text()) + finally: + try: + for role in (*dr.ROLES, "tsvalidator"): + row = dr.owned(ns, role) + if row: + logs = subprocess.run(["docker", "logs", "--tail", "100", row["Id"]], + capture_output=True, text=True, timeout=10) + (output / (role + ".log")).write_text((logs.stdout + logs.stderr)[-32000:]) + except Exception as error: + receipt["log_capture_error"] = type(error).__name__ + try: + receipt["cleanup"] = dr.cleanup(ns) + finally: + dr.erase_private(work) + receipt["private_files_remaining"] = sum((work / name).exists() for name in dr.PRIVATE_FILES) + receipt.update(elapsed_seconds=time.monotonic() - started, disk_free_before=free_before, + disk_free_after=shutil.disk_usage(output).free) + def redact(value): + if isinstance(value, dict): + return {key: ("sha256:" + hashlib.sha256(item.encode()).hexdigest() + if key in {"stream_cursor", "cursor"} and isinstance(item, str) else redact(item)) + for key, item in value.items()} + if isinstance(value, list): + return [redact(item) for item in value] + return value + data = json.dumps(redact(receipt), indent=2) + "\n" + (output / "receipt.json").write_text(data) + print(json.dumps({"output": str(output), "sha256": hashlib.sha256(data.encode()).hexdigest(), + "observer_completed": ok, "acceptance": False})) + return ok + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--exported", type=Path, required=True) + parser.add_argument("--venue", choices=dr.VENUES, required=True) + parser.add_argument("--fault", choices=dr.FAULTS, required=True) + args = parser.parse_args() + return 0 if exercise(args.output, args.exported, args.venue, args.fault) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/p184_dr05_observer.py b/scripts/p184_dr05_observer.py new file mode 100644 index 00000000..a712c411 --- /dev/null +++ b/scripts/p184_dr05_observer.py @@ -0,0 +1,262 @@ +"""Bounded DR05 capture through committed canonical and real Query/Stream RPCs. + +Test-only development evidence, never a standalone DR acceptance certificate. +Run only inside p184_dr05_dl_fault's disposable broker network namespace. +""" +from __future__ import annotations + +import argparse +import asyncio +import base64 +import hashlib +import json +from pathlib import Path +import sys +import time +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.p184_dr05_dl_fault import frame, namespace, schedule +from scripts.p184_dr05_raw_publisher import RawPublisher + + +def record_summary(topic, offset, wire, canonical): + from qdl.marketdata.v2.market_data_pb2 import EventEnvelope + from qdl.provider.v1.raw_provider_pb2 import QuarantineRecord, QuarantineReason + + row = {"topic": topic, "offset": offset, "sha256": hashlib.sha256(wire).hexdigest()} + if canonical: + event = EventEnvelope.FromString(wire) + row.update(kind="canonical", capture_id=event.raw_capture_id.hex(), + instrument_uid=event.instrument_uid, venue=event.venue, generation=event.connection_generation, + sequence=event.source_sequence, payload=event.WhichOneof("payload"), + event_b64=base64.b64encode(wire).decode()) + else: + event = QuarantineRecord.FromString(wire) + row.update(kind="quarantine", capture_id=event.raw.capture_id.hex(), venue=event.raw.venue, + generation=event.raw.connection_generation, reason=QuarantineReason.Name(event.reason), + summary=event.safe_summary) + return row + + +class CanonicalObserver: + def __init__(self, packet, config): + from confluent_kafka import Consumer, TopicPartition + from scripts.p184_dr08_isolated import kafka_security + + self.tp = TopicPartition + self.raw = config["raw_topics"][0] + self.canonical = config["core"]["canonical_stream"] + self.topics = [self.canonical, config["core"]["quarantine_stream"]] + self.consumer = Consumer({**kafka_security(packet), "group.id": packet["namespace"] + ".observer", + "enable.auto.commit": False, "isolation.level": "read_committed", "enable.partition.eof": True, + "fetch.wait.max.ms": 10}) + self.positions = {topic: 0 for topic in self.topics} + self.consumer.assign([TopicPartition(topic, 0, 0) for topic in self.topics]) + # committed() reads offsets without subscribe/join or offset mutation. + self.offset_reader = Consumer({**kafka_security(packet), "group.id": packet["namespace"], + "enable.auto.commit": False}) + + def observe(self, receipt, timeout=8): + from confluent_kafka import KafkaError + + deadline, records, eof = time.monotonic() + timeout, [], set() + committed = -1 + next_check = 0 + after_commit = False + while time.monotonic() < deadline: + message = self.consumer.poll(.05) + if message is not None: + if message.error(): + if message.error().code() != KafkaError._PARTITION_EOF: + raise RuntimeError(str(message.error())) + eof.add(message.topic()) + self.positions[message.topic()] = message.offset() + else: + eof.discard(message.topic()) + self.positions[message.topic()] = message.offset() + 1 + records.append(record_summary(message.topic(), message.offset(), message.value(), + message.topic() == self.canonical)) + if len(records) > 64: + raise RuntimeError("unexpected unbounded canonical output") + if time.monotonic() >= next_check: + next_check = time.monotonic() + .2 + committed = self.offset_reader.committed([self.tp(self.raw, 0)], timeout=1)[0].offset + if committed > receipt["offset"] and not after_commit: + # Fetch both partitions again AFTER the transaction committed. + # An EOF observed before the raw offset commit proves nothing. + self.consumer.assign([self.tp(topic, 0, self.positions[topic]) for topic in self.topics]) + eof.clear() + after_commit = True + if after_commit and set(self.topics) <= eof: + return {"raw_committed_next": committed, "read_committed_eof": sorted(eof), + "records": records, "matched": [r for r in records if r["capture_id"] == receipt["capture_id"]]} + raise TimeoutError(f"canonical barrier missing: committed={committed}, raw={receipt['offset']}, eof={sorted(eof)}") + + def close(self): + self.consumer.close() + self.offset_reader.close() + + +def reader_harness(packet): + from scripts.kn_native_slice_probe import Slice + from qdl.consumer.manifest import ConsumerManifestLoader + + runner = Slice(SimpleNamespace(**packet["slice"])) + runner.manifests = {m.consumer_id: m for m in ( + ConsumerManifestLoader.load(Path(path)) for path in packet["isolated_manifest_paths"])} + expected = {packet["namespace"] + ".binance", packet["namespace"] + ".okx"} + if set(runner.manifests) != expected or set(runner.identities) != expected: + raise ValueError("isolated manifest/identity roster mismatch") + return runner + + +async def read_views(runner, packet, binding): + import grpc + from google.protobuf.json_format import MessageToDict + from qdl.query.v2 import query_pb2 as pb + from qdl.certification.phase103_consumer_acceptance import sdk_requirement + from scripts.p184_dr08_isolated import channel_for + + cid = packet["namespace"] + "." + binding["venue"].lower() + channel = await channel_for(runner, cid) + result = {} + try: + for feed in ("BOOK_SNAPSHOT", "BOOK_DELTA"): + domain = runner.requirement(cid, binding["instrument_uid"], feed) + requirement = sdk_requirement(SimpleNamespace(requirement=domain)).to_proto() + metadata = (("authorization", "Bearer " + runner.jwt(cid)), ("x-qdl-consumer-id", cid), + ("x-qdl-purpose", "INTERNAL_EXECUTION")) + row = {"requested_at_ns": time.time_ns(), "instrument_uid": binding["instrument_uid"]} + for method, request_type, response_type in ( + ("GetFeedStatus", pb.GetFeedStatusRequest, pb.GetFeedStatusResponse), + ("GetSnapshot", pb.GetSnapshotRequest, pb.GetSnapshotResponse)): + rpc = channel.unary_unary("/qdl.query.v2.MarketDataStreamService/" + method, + request_serializer=request_type.SerializeToString, + response_deserializer=response_type.FromString) + try: + value = await rpc(request_type(consumer_id=cid, requirement=requirement), metadata=metadata, timeout=2) + if method == "GetSnapshot": + row[method] = {"code": "OK", "events_b64": [base64.b64encode(e.SerializeToString()).decode() + for e in value.events], "watermark_offset": value.watermark_offset, + "cursor_valid": runner.cursor_valid(value.stream_cursor, cid, domain), + "stream_cursor": value.stream_cursor, + "data_as_of_ns": value.data_as_of_ns} + else: + row[method] = {"code": "OK", "status": MessageToDict(value, preserving_proto_field_name=True, + always_print_fields_with_no_presence=True)} + except grpc.aio.AioRpcError as error: + row[method] = {"code": error.code().name, "detail": error.details()[:1000]} + row["completed_at_ns"] = time.time_ns() + result[feed] = row + finally: + await channel.close() + return result + + +async def run(work, venue, fault): + packet = json.loads((work / "packet.json").read_text()) + config = json.loads((work / "core.json").read_text()) + namespace(packet["namespace"]) + producer = RawPublisher(packet, config) + observer = CanonicalObserver(packet, config) + runner = reader_harness(packet) + binding = producer.bindings[venue] + packet_base = ts_packet(work, runner, packet, binding, []) + rows = [] + try: + for step in schedule(venue, fault): + values = {key: value for key, value in step.items() if key != "phase"} + raw = frame(**values, at_ns=time.time_ns()) + receipt = producer.publish(raw) + observed = observer.observe(receipt) + views = await read_views(runner, packet, binding) + row = {"phase": step["phase"], "publication": receipt, "canonical": observed, "views": views} + import urllib.request + live_input = ts_packet(work, runner, packet, binding, [row], base=packet_base) + live_input["namespace"] = packet["namespace"] + live_input["requirements"] = {"BOOK_SNAPSHOT": live_input["requirements"]["BOOK_SNAPSHOT"]} + live_input["observations"][0]["views"] = {"BOOK_SNAPSHOT": row["views"]["BOOK_SNAPSHOT"]} + body = json.dumps(live_input).encode() + request = urllib.request.Request("http://127.0.0.1:8260/validate", data=body, + headers={"Content-Type": "application/json", "X-P184-Namespace": packet["namespace"]}, method="POST") + try: + with urllib.request.urlopen(request, timeout=5) as response: + row["ts_live"] = json.loads(response.read(4 * 1024 * 1024)) + except Exception as error: + row["ts_live"] = {"error": type(error).__name__, "detail": str(error)[:500], "acceptance": False} + rows.append(row) + (work / "observations.json").write_text(json.dumps(rows)) + finally: + observer.close() + (work / "ts-input.json").write_text(json.dumps(ts_packet(work, runner, packet, binding, rows))) + (work / "ts-input.json").chmod(0o600) + return {"schema": "p184.dr05.observation.v1", "venue": venue, "fault": fault, + "namespace": packet["namespace"], "provenance": "TEST_ONLY", "acceptance": False, "observations": rows} + + +def ts_packet(work, runner, packet, binding, rows, base=None): + venue = binding["venue"] + cid = packet["namespace"] + "." + venue.lower() + if base is None: + from dataclasses import asdict + from qdl.runtime.stable_catalog import StableSourceCatalog + from qdl.certification.phase103_consumer_acceptance import sdk_requirement + catalog = StableSourceCatalog.load(work / "catalog.yaml") + instrument = catalog.instrument_for(binding["instrument_uid"]) + cid = packet["namespace"] + "." + venue.lower() + requirements = {} + for feed in ("BOOK_SNAPSHOT", "BOOK_DELTA"): + sdk = sdk_requirement(SimpleNamespace(requirement=runner.requirement(cid, binding["instrument_uid"], feed))) + requirements[feed] = asdict(sdk) + base = {"schema_version": 1, "side": "BUY", "requirements": requirements, + "instrument": {**instrument.identity.__dict__, "metadata_revision": instrument.metadata_revision, + "asset_class": instrument.asset_class.value, "native_symbol": instrument.native_symbol, + "status": instrument.status.value}} + handoffs = {} + path = work / "query-handoffs.jsonl" + if path.exists(): + for line in path.read_text().splitlines(): + handoff = json.loads(line) + if handoff["consumer"] == cid: + handoffs.setdefault(handoff["feed"], handoff["view"]) + ts_input = {**base, "handoffs": handoffs, + "observations": [{"id": row["phase"], "views": row["views"]} for row in rows]} + return json.loads(json.dumps(ts_input, default=lambda value: value.value)) + + +def diagnostic_query(work): + # Capture the REAL Query handoff from the same RPC, without changing it. + from qdl.runtime.kn_query_backend import KnCursorV3Issuer + from qdl.api_v2.router import _market_item + from scripts.p184_dr08_isolated import serve_query + original = KnCursorV3Issuer.bind_history + def bind(self, requirement, history, *, consumer_id): + bound = original(self, requirement, history, consumer_id=consumer_id) + item = _market_item(bound.items[-1]).model_dump(mode="json") + with (work / "query-handoffs.jsonl").open("a") as handle: + handle.write(json.dumps({"consumer": consumer_id, "feed": requirement.feed.value, + "cursor_sha256": hashlib.sha256(bound.stream_cursor.encode()).hexdigest(), + "view": item}) + "\n") + return bound + KnCursorV3Issuer.bind_history = bind + serve_query(work) + + +def main(): + if len(sys.argv) == 3 and sys.argv[1] == "--query": + diagnostic_query(Path(sys.argv[2])) + return + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--work", type=Path, required=True) + parser.add_argument("--venue", choices=("BINANCE", "OKX"), required=True) + parser.add_argument("--fault", choices=("GAP_REFUSAL", "GENERATION_CHANGE_FENCED"), required=True) + args = parser.parse_args() + print(json.dumps(asyncio.run(run(args.work, args.venue, args.fault)))) + + +if __name__ == "__main__": + main() diff --git a/scripts/p184_dr05_ts_validate.py b/scripts/p184_dr05_ts_validate.py new file mode 100644 index 00000000..9481ebb2 --- /dev/null +++ b/scripts/p184_dr05_ts_validate.py @@ -0,0 +1,399 @@ +"""TS-image BOOK validation: offline replay or isolated live Redis server. + +stdin JSON: {schema_version: 1, instrument: InstrumentView JSON, +requirements: {BOOK_SNAPSHOT|BOOK_DELTA: DataRequirement JSON}, +handoffs: {feed: actual Query MarketDataView JSON}, side: BUY|SELL, +observations: [{id: str, views: }]}. +Each successful GetSnapshot must also retain its actual stream_cursor. +GetFeedStatus quality is required, never inferred from raw envelope flags. +All application imports must resolve from the existing TS image. Mount this +file only, outside /app, with PYTHONPATH=/app. Offline uses network none; +--serve --namespace NS requires P184_DR05_ISOLATED_NAMESPACE=NS and the +disposable Kafka network namespace. POST /validate needs namespace in JSON +and X-P184-Namespace; Redis is fixed to 127.0.0.1:6379/2. GET /health. +Live mode uses wall clock, never captured-time freshness for application writes. +book_validated records only successful BOOK validation. In live mode accepted +means the complete serialized eligible cache view matches the incoming view, +including volatile quality fields. A mismatch is not a validation refusal. +cache_apply reports cache acceptance and writes separately. A no-write or fence +is not an endpoint/SDK/BOOK-validator rejection or a fault-specific DR verdict. +Synthetic fixtures belong only in tests. Caller owns evidence provenance. +""" + +import base64 +import hashlib +import json +import sys +import time + +MAX_INPUT = 4 * 1024 * 1024 +IMAGE = "sha256:a6548ee36aeb0242c1734c071678c525c1506d05c10a70b533eb7110e7eaf16a" +ENDPOINT = "/qdl.query.v2.MarketDataStreamService/" + + +def requirement_from_json(raw): + from qdl_sdk import models + + values = dict(raw) + for name, kind in { + "feed": models.Feed, "consumer_grade": models.Grade, + "stale_policy": models.StalePolicy, "gap_policy": models.GapPolicy, + "event_recency_policy": models.StalePolicy, + "recovery": models.RecoveryPolicy, "bar_revision_policy": models.BarRevisionPolicy, + }.items(): + if name in values and values[name] is not None: + values[name] = kind(values[name]) + result = models.DataRequirement(**values) + if result.feed.value not in {"BOOK_SNAPSHOT", "BOOK_DELTA"}: + raise ValueError("BOOK feeds only") + if result.consumer_grade is not models.Grade.EXECUTION: + raise ValueError("EXECUTION requirement required") + if result.max_freshness_ms is None or "max_session_liveness_ms" not in raw: + raise ValueError("explicit freshness and session policy (possibly null) required") + return result + + +def positive_clock(value): + if type(value) is not int or value <= 0: + raise ValueError("observation clocks must be positive integer nanoseconds") + return value + + +def validate(packet, *, live=None): + from dataclasses import replace + from qdl.marketdata.v2 import market_data_pb2 + from qdl_sdk.client import _validate_query_payload + from qdl_sdk.models import InstrumentView, MarketDataView, QualityView, StreamEvent + from qdl_sdk.projection import market_data_view_from_stream + from qdl_sdk.errors import DataLayerError + from adapters.market_data.data_layer_client import DataLayerClientError + from adapters.market_data.data_layer_v2 import execution_feed_from_v2 + from services.market_data.execution_ordering import ordering, compare, ExecutionOrderingError + from services.market_data.execution_state import execution_payload + from services.market_data.cache_reader import MarketCacheReader + from services.risk_engine.execution_market_context import _validate_book, ExecutionMarketContextError + + if packet.get("schema_version") != 1 or packet.get("side") not in {"BUY", "SELL"}: + raise ValueError("schema_version=1 and explicit BUY/SELL side required") + instrument = InstrumentView.model_validate(packet["instrument"]) + requirements = {k: requirement_from_json(v) for k, v in packet["requirements"].items()} + handoffs = {k: MarketDataView.model_validate(v) for k, v in packet["handoffs"].items()} + observations = packet["observations"] + if not isinstance(observations, list) or not 1 <= len(observations) <= 64: + raise ValueError("1..64 observations required") + previous, results, ids = {}, [], set() + app_errors = (DataLayerError, DataLayerClientError, ExecutionOrderingError, ExecutionMarketContextError) + for observation in observations: + identifier = observation["id"] + if not isinstance(identifier, str) or not identifier or identifier in ids: + raise ValueError("observation ids must be unique nonempty strings") + ids.add(identifier) + if not observation["views"] or set(observation["views"]) - {"BOOK_SNAPSHOT", "BOOK_DELTA"}: + raise ValueError("BOOK views required") + for feed, row in observation["views"].items(): + result = {"id": identifier, "feed": feed, "accepted": False, + "stage": "input", "application_called": False, "current_generation": None, + "book_validated": False, "cache_apply": None} + results.append(result) + try: + started = positive_clock(row["requested_at_ns"]) + completed = positive_clock(row["completed_at_ns"]) + if completed < started or row["instrument_uid"] != instrument.instrument_uid: + raise ValueError("clock regression or instrument mismatch") + if live is not None and completed > time.time_ns(): + raise ValueError("future observation clock") + status, snapshot = row["GetFeedStatus"], row["GetSnapshot"] + result["endpoint_codes"] = {"GetFeedStatus": status["code"], "GetSnapshot": snapshot["code"]} + if status["code"] != "OK" or snapshot["code"] != "OK": + result.update(outcome="ENDPOINT_REFUSED", stage="endpoint") + continue + requirement, template = requirements[feed], handoffs[feed] + if requirement.feed.value != feed or template.feed.value != feed: + raise ValueError("feed mismatch") + if requirement.instrument_uid != instrument.instrument_uid: + raise ValueError("requirement identity mismatch") + # Require every captured status field: model defaults cannot create proof. + fields = set(QualityView.model_fields) + if set(status["status"]) != fields: + raise ValueError("complete actual GetFeedStatus quality required") + quality = QualityView.model_validate(status["status"]) + if snapshot["cursor_valid"] is not True: + raise ValueError("verified snapshot cursor required") + offset = snapshot["watermark_offset"] + if type(offset) is not int or offset < 0: + raise ValueError("nonnegative actual watermark required") + positive_clock(snapshot["data_as_of_ns"]) + events = snapshot["events_b64"] + if not isinstance(events, list) or len(events) != 1: + raise ValueError("exactly one snapshot event required; no silent tail selection") + raw = base64.b64decode(events[0], validate=True) + result["event_sha256"] = hashlib.sha256(raw).hexdigest() + envelope = market_data_pb2.EventEnvelope.FromString(raw) + if envelope.instrument_id != instrument.instrument_id: + raise ValueError("envelope instrument_id mismatch") + if not 0 < envelope.source_event_time_ns <= completed or not 0 < envelope.received_at_ns <= completed: + raise ValueError("missing/future envelope clocks") + result.update(stage="sdk_status", application_called=True) + # The actual status must pass SDK policy before raw projection runs. + typed = template.model_dump(mode="json") + typed["quality"] = quality.model_dump(mode="json") + status_template = MarketDataView.model_validate(typed) + _validate_query_payload(requirement, {"request_id": identifier, "data": status_template}, warmup=False) + result["stage"] = "sdk_projection" + view = market_data_view_from_stream( + StreamEvent(offset, snapshot["stream_cursor"], envelope), template=status_template, + requirement=requirement, now_ns=time.time_ns() if live is not None else completed, + ) + # SDK computes envelope recency independently. Keep actual status quality + # for TS admission; do not replace a refusal with computed healthy flags. + typed = view.model_dump(mode="json") + typed["quality"] = quality.model_dump(mode="json") + view = MarketDataView.model_validate(typed) + result["stage"] = "ts_adapter" + execution = execution_feed_from_v2(view, instrument, session_max_ms=requirement.max_session_liveness_ms) + execution = replace(execution, receipt_ns=started, state_refresh=True) + result["stage"] = "ts_ordering" + incoming = ordering(execution) + if live is None and feed in previous and compare(previous[feed], incoming) < 0: + result.update(outcome="FENCED_OLDER", generation=incoming.generation) + continue + if feed == "BOOK_SNAPSHOT": + result["stage"] = "ts_book_validator" + book = MarketCacheReader(None)._typed_execution_feed( + json.loads(execution_payload(execution)), expected_feed=feed) + if book is None: + raise ValueError("TS cache reader rejected BOOK shape") + result["book_proof"] = _validate_book( + book, now_ms=(time.time_ns() if live is not None else completed) // 1_000_000, + max_age_ms=requirement.max_freshness_ms, + side=packet["side"], require_levels=True, + ) + result["book_validated"] = True + if live is not None: + result["stage"] = "ts_redis_apply" + result.update(live.apply(execution, requirement.max_freshness_ms)) + continue + previous[feed] = incoming + result.update(outcome="ACCEPTED", accepted=True, generation=incoming.generation, + sequence=incoming.sequence, quality=view.quality.model_dump(mode="json")) + except app_errors as error: + result.update(outcome="APPLICATION_REFUSED", error_type=type(error).__name__, + error=str(error)[:600], code=str(getattr(error, "code", ""))) + except Exception as error: + result.update(outcome="INPUT_ERROR", error_type=type(error).__name__, error=str(error)[:600]) + return {"schema_version": 1, "live": live is not None, "controlled_replay": live is None, "expected_image": IMAGE, "results": results, + "endpoints": [ENDPOINT + x for x in ("GetSnapshot", "GetFeedStatus")], + "application_modules": {"sdk_projection": sys.modules[market_data_view_from_stream.__module__].__file__, + "ts_adapter": sys.modules[execution_feed_from_v2.__module__].__file__, + "ts_book_validator": sys.modules[_validate_book.__module__].__file__}, + "acceptance": False, "missing_coverage": ["full Risk admission (MARK/QUOTE absent)", + "Redis cache application and generation adoption", "live Subscribe transport and ACK", + "independent verification of caller capture provenance"]} + + +class LiveStore: + """One isolated DB2 writer. Production CAS owns ordering and adoption.""" + + def __init__(self): + self.owned = set() + self.previous = {} + self.failed = False + + def apply(self, execution, max_age_ms): + import asyncio + if self.failed: + raise RuntimeError("live writer fenced after ambiguous failure") + try: + return asyncio.run(self._apply(execution, max_age_ms)) + except Exception: + # A transport failure may follow a successful CAS: never blindly retry. + self.failed = True + raise + + async def _apply(self, execution, max_age_ms): + import asyncio + from redis.asyncio import Redis + from services.market_data.cache_projector import MarketCacheProjector + from services.market_data.execution_state import state_keys, execution_payload + from services.market_data.execution_ordering import ordering, compare + redis = Redis.from_url("redis://127.0.0.1:6379/2", decode_responses=True, + socket_connect_timeout=1, socket_timeout=1) + try: + async with _deadline(): + keys = state_keys(execution) + before = await redis.mget(*keys) + if keys not in self.owned: + if any(value is not None for value in before): + raise ValueError("isolated TS keys already exist; refusing ownership") + self.owned.add(keys) + projector = MarketCacheProjector(redis) + deadline = time.monotonic() + 2 + applied = await projector.project_execution_snapshot( + execution, deadline=deadline, max_age_ms=max_age_ms, + previous=self.previous.get(keys), wall_clock_ns=time.time_ns, + ) + after = await redis.mget(*keys) + current = await projector.current_execution_snapshot( + execution.canonical_identity, execution.feed, deadline=deadline, + max_age_ms=max_age_ms, wall_clock_ns=time.time_ns, + ) + if current is None: + raise ValueError("no current eligible cache after apply") + incoming, retained = ordering(execution), ordering(current) + relation = compare(retained, incoming) + matches = execution_payload(current) == execution_payload(execution) + self.previous[keys] = current + return {"outcome": "APPLIED" if applied else "UNCHANGED" if matches else "FENCED_OLDER" if relation < 0 else "NO_WRITE", + "accepted": bool(matches), "applied": bool(applied), + "cache_apply": {"accepted": bool(matches), "written": bool(applied)}, + "generation": incoming.generation, "sequence": incoming.sequence, + "current_generation": retained.generation, "retained_generation": retained.generation, "retained_sequence": retained.sequence, + "cache_unchanged": before == after, + "redis_apply_proof": {"method": "MarketCacheProjector.project_execution_snapshot", + "applied": bool(applied), "re_read": True, "db": 2}, + "cache_sha256": hashlib.sha256(after[1].encode()).hexdigest(), + "validated_at_ns": time.time_ns()} + finally: + await redis.aclose() + + def cleanup(self): + import asyncio + from redis.asyncio import Redis + async def remove(): + redis = Redis.from_url("redis://127.0.0.1:6379/2", socket_connect_timeout=1, socket_timeout=1) + try: + for keys in self.owned: + await redis.delete(*keys) + if any(v is not None for v in await redis.mget(*keys)): + raise RuntimeError("owned TS cleanup failed") + finally: + await redis.aclose() + asyncio.run(remove()) + + +class _deadline: + """Python 3.10-compatible bounded async operation.""" + async def __aenter__(self): + import asyncio + self.task = asyncio.current_task() + self.handle = asyncio.get_running_loop().call_later(3, self.task.cancel) + + async def __aexit__(self, kind, value, traceback): + import asyncio + self.handle.cancel() + if kind is asyncio.CancelledError: + raise TimeoutError("live Redis deadline") from value + + +def live_packet(packet, namespace): + if packet.get("namespace") != namespace: + raise ValueError("namespace mismatch") + rows = packet.get("observations") + if not isinstance(rows, list) or len(rows) != 1 or set(rows[0].get("views", {})) != {"BOOK_SNAPSHOT"}: + raise ValueError("one BOOK_SNAPSHOT observation per live request required") + if set(packet.get("requirements", {})) != {"BOOK_SNAPSHOT"}: + raise ValueError("live BOOK_SNAPSHOT requirement only") + if any(key in packet for key in ("wall_clock_ns", "now_ns", "redis_url", "clock")): + raise ValueError("clock and Redis overrides forbidden") + + +def serve(namespace): + import os + import re + import signal + from http.server import BaseHTTPRequestHandler, HTTPServer + if not re.fullmatch(r"p184-dr05-dl-[0-9a-f]{16}", namespace or ""): + raise ValueError("invalid isolated namespace") + if os.environ.get("P184_DR05_ISOLATED_NAMESPACE") != namespace: + raise ValueError("namespace environment confirmation required") + # Warm application imports before health becomes available. + from services.market_data.cache_projector import MarketCacheProjector + from services.risk_engine.execution_market_context import _validate_book + store = LiveStore() + + class Handler(BaseHTTPRequestHandler): + def setup(self): + super().setup() + self.connection.settimeout(3) + + def log_message(self, *args): + pass + + def reply(self, status, payload): + body = json.dumps(payload, sort_keys=True).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + if self.path != "/health": + self.reply(404, {"acceptance": False}) + return + self.reply(200, {"namespace": namespace, "ready": not store.failed, "acceptance": False}) + + def do_POST(self): + if self.path != "/validate" or self.headers.get("X-P184-Namespace") != namespace: + self.reply(403, {"acceptance": False, "error": "path or namespace"}) + return + try: + if self.headers.get("Transfer-Encoding") or len(self.headers.get_all("Content-Length", [])) != 1: + raise ValueError("one Content-Length required; no transfer encoding") + size = int(self.headers["Content-Length"]) + if not 0 < size <= MAX_INPUT: + raise ValueError("invalid body size") + raw = self.rfile.read(size) + if len(raw) != size: + raise ValueError("truncated body") + packet = json.loads(raw) + live_packet(packet, namespace) + result = validate(packet, live=store) + result["mode"] = "LIVE_REDIS_WALL_CLOCK" + result["missing_coverage"] = ["full Risk admission (MARK/QUOTE absent)", + "live Subscribe transport and ACK", "fault-specific verdict owned by parent"] + self.reply(200, result) + except Exception as error: + self.reply(400, {"acceptance": False, "outcome": "INPUT_ERROR", "error": str(error)[:300]}) + + server = HTTPServer(("127.0.0.1", 8260), Handler) + server.timeout = 0.5 + stopping = False + def stop(*args): + nonlocal stopping + stopping = True + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + expires = time.monotonic() + 300 + try: + while not stopping and time.monotonic() < expires: + server.handle_request() + finally: + server.server_close() + store.cleanup() + + +def main(): + if len(sys.argv) > 1: + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--serve", action="store_true", required=True) + parser.add_argument("--namespace", default=__import__("os").environ.get("P184_DR05_ISOLATED_NAMESPACE")) + args = parser.parse_args() + serve(args.namespace) + return 0 + try: + raw = sys.stdin.buffer.read(MAX_INPUT + 1) + if len(raw) > MAX_INPUT: + raise ValueError("input exceeds 4 MiB") + result = validate(json.loads(raw)) + except Exception as error: + print(json.dumps({"acceptance": False, "outcome": "INPUT_ERROR", + "error_type": type(error).__name__, "error": str(error)[:600]})) + return 2 + print(json.dumps(result, sort_keys=True)) + return 2 if any(r["outcome"] == "INPUT_ERROR" for r in result["results"]) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/p184_dr05_verdict.py b/scripts/p184_dr05_verdict.py new file mode 100644 index 00000000..3e0f95b2 --- /dev/null +++ b/scripts/p184_dr05_verdict.py @@ -0,0 +1,53 @@ +"""Fail-closed observer checks; no release certificate or synthesized evidence.""" + + +def evaluate(observation): + from scripts.p184_dr05_dl_fault import schedule + + venue, fault = observation["venue"], observation["fault"] + expected = [step["phase"] for step in schedule(venue, fault)] + rows = observation.get("observations", []) + checks = {"exact_phase_roster": [row.get("phase") for row in rows] == expected} + if not checks["exact_phase_roster"]: + return {"ok": False, "checks": checks} + phases = {row["phase"]: row for row in rows} + for phase in ("positive_control", "return_control", "post_fence_control"): + row = phases[phase] + snapshot = row["views"]["BOOK_SNAPSHOT"]["GetSnapshot"] + checks[phase + ".reader_usable"] = snapshot.get("code") == "OK" and snapshot.get("cursor_valid") is True + matches = row["canonical"].get("matched", []) + checks[phase + ".canonical_match"] = any( + value.get("kind") == "canonical" and value.get("venue") == venue + and value.get("capture_id") == row["publication"]["capture_id"] for value in matches) + live = row.get("ts_live", {}) + values = [value for value in live.get("results", []) if value.get("feed") == "BOOK_SNAPSHOT"] + checks[phase + ".ts_live_usable"] = live.get("live") is True and len(values) == 1 and values[0].get("accepted") is True + negative = "gap" if fault == "GAP_REFUSAL" else "new_generation_unanchored" + row = phases[negative] + checks[negative + ".canonical_refused"] = any( + value.get("kind") == "quarantine" and value.get("reason") == "QUARANTINE_REASON_SEQUENCE_GAP" + and value.get("capture_id") == row["publication"]["capture_id"] + for value in row["canonical"].get("matched", [])) + checks[negative + ".feed_roster"] = set(row["views"]) == {"BOOK_SNAPSHOT", "BOOK_DELTA"} + for feed, view in row["views"].items(): + status = view["GetFeedStatus"].get("status", {}) + snapshot = view["GetSnapshot"] + checks[negative + "." + feed + ".not_eligible"] = status.get("execution_eligible") is not True + checks[negative + "." + feed + ".snapshot_refused"] = snapshot.get("code") == "FAILED_PRECONDITION" + # Expiring the old price by waiting is not proof of gap propagation. + reason = snapshot.get("detail", "") + checks[negative + "." + feed + ".fault_typed"] = ( + status.get("gap_open") is True or any(word in reason for word in ("GAP", "GENERATION", "DATA_NOT_READY"))) + live = row.get("ts_live", {}) + values = [value for value in live.get("results", []) if value.get("feed") == "BOOK_SNAPSHOT"] + checks[negative + ".ts_live_refused"] = live.get("live") is True and len(values) == 1 and values[0].get("accepted") is False and values[0].get("outcome") in {"ENDPOINT_REFUSED", "APPLICATION_REFUSED"} + old = phases["old_generation"] + checks["old_generation.raw_committed"] = old["canonical"].get("raw_committed_next", -1) > old["publication"]["offset"] + checks["old_generation.not_canonical"] = not any( + value.get("kind") == "canonical" for value in old["canonical"].get("matched", [])) + checks["post_fence_generation"] = any( + value.get("kind") == "canonical" and value.get("generation") == 8 + for value in phases["post_fence_control"]["canonical"].get("matched", [])) + return {"cell": "DR05." + fault + "@" + venue, "level": "ISOLATED_FAULT", + "ok": all(checks.values()), "checks": checks, + "outcome": "OBSERVED_CONTROLS_PASS" if all(checks.values()) else "OBSERVED_CONTROLS_FAIL"} diff --git a/tests/test_p184_dr05_dl_fault.py b/tests/test_p184_dr05_dl_fault.py index 82cbca13..9981f542 100644 --- a/tests/test_p184_dr05_dl_fault.py +++ b/tests/test_p184_dr05_dl_fault.py @@ -37,7 +37,7 @@ def test_topology_is_bounded_and_disconnected(self): self.assertFalse(set(argv) & {"-p", "--publish", "--privileged", "--network=host"}) self.assertNotIn("/run/secrets", " ".join(argv)) self.assertTrue(any(a.startswith("sha256:") for a in argv)) - self.assertIn("scripts/p184_dr08_isolated.py", commands["query"]) + self.assertIn("scripts/p184_dr05_observer.py", commands["query"]) self.assertIn("--query", commands["query"]) def test_external_directory_required(self): diff --git a/tests/test_p184_dr05_observer.py b/tests/test_p184_dr05_observer.py new file mode 100644 index 00000000..ffa11e41 --- /dev/null +++ b/tests/test_p184_dr05_observer.py @@ -0,0 +1,80 @@ +import base64 +import json +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from scripts.p184_dr05_observer import CanonicalObserver, record_summary + + +class RecordTests(unittest.TestCase): + def test_canonical_capture_identity_and_exact_bytes(self): + from qdl.marketdata.v2.market_data_pb2 import EventEnvelope + event = EventEnvelope(venue="OKX", instrument_uid="instrument", connection_generation=8, + raw_capture_id=b"1234567890123456", source_sequence="202") + event.book_snapshot.native_sequence = "202" + raw = event.SerializeToString() + result = record_summary("isolated.canonical", 0, raw, True) + self.assertEqual(result["capture_id"], event.raw_capture_id.hex()) + self.assertEqual(result["generation"], 8) + self.assertEqual(result["payload"], "book_snapshot") + self.assertEqual(base64.b64decode(result["event_b64"]), raw) + + def test_quarantine_preserves_reason_not_phase_label(self): + from qdl.provider.v1.raw_provider_pb2 import QuarantineRecord, QUARANTINE_REASON_SEQUENCE_GAP + event = QuarantineRecord(reason=QUARANTINE_REASON_SEQUENCE_GAP, safe_summary="test") + event.raw.capture_id = b"1234567890123456" + event.raw.venue = "BINANCE" + event.raw.connection_generation = 7 + row = record_summary("isolated.quarantine", 2, event.SerializeToString(), False) + self.assertEqual(row["reason"], "QUARANTINE_REASON_SEQUENCE_GAP") + self.assertEqual(row["capture_id"], event.raw.capture_id.hex()) + self.assertNotIn("acceptance", row) + + +class BarrierTests(unittest.TestCase): + def observer(self, committed=2): + from confluent_kafka import KafkaError + observer = CanonicalObserver.__new__(CanonicalObserver) + observer.raw, observer.canonical, observer.topics = "raw", "canonical", ["canonical", "quarantine"] + observer.positions = {topic: 0 for topic in observer.topics} + observer.tp = lambda topic, partition, offset=None: (topic, partition, offset) + error = SimpleNamespace(code=lambda: KafkaError._PARTITION_EOF) + class Consumer: + def __init__(self): + self.queue, self.assignments, self.closed = [], [], False + def assign(self, values): + self.assignments.append(values) + self.queue = [SimpleNamespace(error=lambda: error, topic=lambda t=t: t, offset=lambda: 0) + for t in observer.topics] + def poll(self, timeout): + return self.queue.pop(0) if self.queue else None + def close(self): + self.closed = True + observer.consumer = Consumer() + observer.consumer.assign([]) + observer.offset_reader = SimpleNamespace(committed=lambda *a, **k: [SimpleNamespace(offset=committed)], close=lambda: None) + return observer + + def test_barrier_refetches_after_commit(self): + observer = self.observer() + result = observer.observe({"offset": 1, "capture_id": "absent"}, timeout=1) + self.assertEqual(result["raw_committed_next"], 2) + self.assertEqual(result["matched"], []) + self.assertEqual(len(observer.consumer.assignments), 2) + self.assertEqual(result["read_committed_eof"], ["canonical", "quarantine"]) + + def test_uncommitted_offset_is_not_evidence_of_filtered_frame(self): + observer = self.observer(committed=1) + with self.assertRaises(TimeoutError): + observer.observe({"offset": 1, "capture_id": "absent"}, timeout=.01) + + def test_non_eof_kafka_error_propagates(self): + observer = self.observer() + observer.consumer.queue = [SimpleNamespace(error=lambda: SimpleNamespace(code=lambda: 123))] + with self.assertRaises(RuntimeError): + observer.observe({"offset": 1, "capture_id": "absent"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p184_dr05_ts_validate.py b/tests/test_p184_dr05_ts_validate.py new file mode 100644 index 00000000..248a96b4 --- /dev/null +++ b/tests/test_p184_dr05_ts_validate.py @@ -0,0 +1,366 @@ +"""TEST_ONLY synthetic protobuf controls, executed against imports in TS image.""" + +import base64 +import copy +import importlib.util +import json +import os +import unittest + +from qdl.common.v1 import common_pb2 +from qdl.marketdata.v2 import market_data_pb2 + +if os.path.exists("/helper.py"): + spec = importlib.util.spec_from_file_location("helper", "/helper.py") + helper = importlib.util.module_from_spec(spec) + spec.loader.exec_module(helper) +else: + from scripts import p184_dr05_ts_validate as helper + +NOW = 1_800_000_000_000_000_000 +UID = "11111111-1111-5111-8111-111111111111" + + +def fixture(venue="BINANCE", generation=1): + symbol = "ETHUSDT" if venue == "BINANCE" else "ETH-USDT-SWAP" + instrument = dict(instrument_uid=UID, instrument_id=f"{venue}:PERPETUAL:{symbol}", + venue=venue, market="USDM" if venue == "BINANCE" else "SWAP", + product_type="PERPETUAL", canonical_symbol="ETH-USDT", + metadata_revision=7, asset_class="CRYPTO", native_symbol=symbol, status="ACTIVE") + event = market_data_pb2.EventEnvelope( + schema_name="TEST_ONLY.canonical", schema_major=2, instrument_uid=UID, + instrument_id=instrument["instrument_id"], instrument_revision=7, + venue=venue, provider="TEST_ONLY.provider", source_id="TEST_ONLY.source", + source_role=common_pb2.SOURCE_ROLE_PRIMARY, authority_revision=3, + normalizer_version="TEST_ONLY.normalizer", adapter_version="TEST_ONLY.adapter", + config_revision=5, source_event_time_ns=NOW, received_at_ns=NOW, + correlation_id="TEST_ONLY.capture", + ) + event.book_snapshot.native_sequence = "42" + event.book_snapshot.book_generation = generation + event.book_snapshot.sequence_verified = True + event.book_snapshot.depth = 1 + for side, price in ((common_pb2.BOOK_SIDE_BID, 100), (common_pb2.BOOK_SIDE_ASK, 101)): + level = event.book_snapshot.levels.add(side=side, quantity_unit=common_pb2.QUANTITY_UNIT_BASE_ASSET) + level.price.mantissa = price + level.price.source_text = str(price) + level.quantity.mantissa = 2 + level.quantity.source_text = "2" + from qdl_sdk.projection import _payload + payload = _payload(event)[3] + quality = dict(state="LIVE", freshness_ms=1, gap_open=False, complete=True, + execution_eligible=True, policy_id="TEST_ONLY.policy", flags=[], + event_recency_state="LIVE", provider_session_state="LIVE", provider_session_liveness_ms=1) + handoff = dict(instrument_uid=UID, instrument_id=instrument["instrument_id"], + instrument_revision=7, feed="BOOK_SNAPSHOT", observed_at_ns=NOW, received_at_ns=NOW, + revision=0, interval=None, payload=payload, quality=quality, + source=dict(venue=venue, provider=event.provider, source_id=event.source_id, + source_role="PRIMARY", authoritative=True), + contract=dict(schema_digest="a" * 64, contract_version="2.0.0", + normalizer_version=event.normalizer_version, adapter_version=event.adapter_version, + instrument_catalog_revision=7, source_policy_revision=2, + authority_revision=3, config_revision=5, correlation_id=event.correlation_id), + watermark_offset=12) + row = dict(requested_at_ns=NOW, completed_at_ns=NOW + 1_000_000, instrument_uid=UID, + GetFeedStatus=dict(code="OK", status=copy.deepcopy(quality)), + GetSnapshot=dict(code="OK", events_b64=[base64.b64encode(event.SerializeToString()).decode()], + watermark_offset=12, cursor_valid=True, stream_cursor="TEST_ONLY.cursor", data_as_of_ns=NOW)) + packet = dict(schema_version=1, side="BUY", instrument=instrument, + requirements={"BOOK_SNAPSHOT": dict(instrument_uid=UID, feed="BOOK_SNAPSHOT", + consumer_grade="EXECUTION", source_policy_id="TEST_ONLY.policy", + max_freshness_ms=2000, max_session_liveness_ms=2000)}, + handoffs={"BOOK_SNAPSHOT": handoff}, observations=[dict(id="positive", views={"BOOK_SNAPSHOT": row})]) + return packet + + +def row(packet): + return packet["observations"][0]["views"]["BOOK_SNAPSHOT"] + + +def change_event(packet, callback): + target = row(packet)["GetSnapshot"] + event = market_data_pb2.EventEnvelope.FromString(base64.b64decode(target["events_b64"][0])) + callback(event) + target["events_b64"] = [base64.b64encode(event.SerializeToString()).decode()] + + +class ValidationTests(unittest.TestCase): + def result(self, packet): + return helper.validate(packet)["results"][0] + + def test_positive_actual_book_validator_both_venues(self): + for venue in ("BINANCE", "OKX"): + with self.subTest(venue=venue): + result = self.result(fixture(venue)) + self.assertEqual(result["outcome"], "ACCEPTED", result) + self.assertEqual(result["stage"], "ts_book_validator") + self.assertEqual(result["book_proof"]["generation"], "1") + + def test_actual_status_gap_refuses_despite_clean_raw(self): + packet = fixture() + row(packet)["GetFeedStatus"]["status"].update(gap_open=True, state="GAPPED", complete=False) + result = self.result(packet) + self.assertEqual(result["outcome"], "APPLICATION_REFUSED", result) + self.assertEqual(result["stage"], "sdk_status") + self.assertFalse(result["book_validated"]) + self.assertIsNone(result["cache_apply"]) + + def test_raw_gap_refuses_despite_healthy_status(self): + packet = fixture() + change_event(packet, lambda e: e.quality_flags.append(common_pb2.QUALITY_FLAG_SEQUENCE_GAP_BEFORE)) + result = self.result(packet) + self.assertEqual(result["outcome"], "APPLICATION_REFUSED", result) + self.assertEqual(result["stage"], "sdk_projection") + + def test_unverified_refuses(self): + packet = fixture() + change_event(packet, lambda e: setattr(e.book_snapshot, "sequence_verified", False)) + self.assertEqual(self.result(packet)["outcome"], "APPLICATION_REFUSED") + + def test_recovered_then_old_generation_fenced(self): + packet = fixture(generation=2) + old = fixture(generation=1)["observations"][0] + old["id"] = "late-old" + packet["observations"].append(old) + results = helper.validate(packet)["results"] + self.assertEqual([r["outcome"] for r in results], ["ACCEPTED", "FENCED_OLDER"]) + + def test_new_generation_requires_real_adoption(self): + packet = fixture() + new = fixture(generation=2)["observations"][0] + new["id"] = "new-unadopted" + packet["observations"].append(new) + result = helper.validate(packet)["results"][1] + self.assertEqual(result["outcome"], "APPLICATION_REFUSED") + self.assertIn("GENERATION_HANDOFF_REQUIRED", result["error"]) + + def test_endpoint_refusal_not_application_success(self): + packet = fixture() + row(packet)["GetSnapshot"] = {"code": "FAILED_PRECONDITION"} + result = self.result(packet) + self.assertEqual(result["outcome"], "ENDPOINT_REFUSED") + self.assertFalse(result["application_called"]) + + def test_missing_quality_never_uses_model_defaults(self): + packet = fixture() + del row(packet)["GetFeedStatus"]["status"]["provider_session_state"] + self.assertEqual(self.result(packet)["outcome"], "INPUT_ERROR") + + def test_missing_handoff_does_not_invent_authority(self): + packet = fixture() + packet["handoffs"] = {} + self.assertEqual(self.result(packet)["outcome"], "INPUT_ERROR") + + def test_stale_validation_clock_refuses(self): + packet = fixture() + row(packet)["completed_at_ns"] += 10_000_000_000 + self.assertEqual(self.result(packet)["outcome"], "APPLICATION_REFUSED") + + def test_invalid_inputs_fail_closed(self): + for key, value in (("cursor_valid", False), ("watermark_offset", -1), ("events_b64", ["!!"]), + ("events_b64", []), ("data_as_of_ns", 0)): + with self.subTest(key=key, value=value): + packet = fixture() + row(packet)["GetSnapshot"][key] = value + self.assertEqual(self.result(packet)["outcome"], "INPUT_ERROR") + + def test_foreign_identity_fails_closed(self): + packet = fixture() + change_event(packet, lambda e: setattr(e, "instrument_uid", "foreign")) + self.assertEqual(self.result(packet)["outcome"], "APPLICATION_REFUSED") + + def test_delta_projection_is_not_snapshot_risk_admission(self): + packet = fixture() + target = row(packet) + event = market_data_pb2.EventEnvelope.FromString(base64.b64decode(target["GetSnapshot"]["events_b64"][0])) + event.ClearField("book_snapshot") + event.book_delta.native_sequence_start = "42" + event.book_delta.native_sequence_end = "42" + event.book_delta.snapshot_sequence = "1" + event.book_delta.book_generation = 1 + event.book_delta.sequence_verified = True + target["GetSnapshot"]["events_b64"] = [base64.b64encode(event.SerializeToString()).decode()] + from qdl_sdk.projection import _payload + for name in ("requirements", "handoffs"): + value = packet[name].pop("BOOK_SNAPSHOT") + value["feed"] = "BOOK_DELTA" + packet[name]["BOOK_DELTA"] = value + packet["handoffs"]["BOOK_DELTA"]["payload"] = _payload(event)[3] + packet["observations"][0]["views"] = {"BOOK_DELTA": target} + result = self.result(packet) + self.assertEqual(result["outcome"], "ACCEPTED", result) + self.assertEqual(result["stage"], "ts_ordering") + self.assertNotIn("book_proof", result) + + def test_stdio_contract(self): + import subprocess + import sys + process = subprocess.run([sys.executable, "-B", helper.__file__], + input=json.dumps(fixture()), text=True, capture_output=True, timeout=10) + self.assertEqual(process.returncode, 0, process.stderr) + result = json.loads(process.stdout) + self.assertEqual(result["results"][0]["outcome"], "ACCEPTED") + self.assertFalse(result["acceptance"]) + + def test_real_status_uint64_strings(self): + packet = fixture() + status = row(packet)["GetFeedStatus"]["status"] + status["freshness_ms"] = "1" + status["provider_session_liveness_ms"] = "1" + self.assertEqual(self.result(packet)["outcome"], "ACCEPTED") + + def test_not_applicable_session_preserves_explicit_policy(self): + packet = fixture() + row(packet)["GetFeedStatus"]["status"]["provider_session_state"] = "NOT_APPLICABLE" + self.assertEqual(self.result(packet)["outcome"], "APPLICATION_REFUSED") + packet["requirements"]["BOOK_SNAPSHOT"]["max_session_liveness_ms"] = None + self.assertEqual(self.result(packet)["outcome"], "ACCEPTED") + + def test_endpoint_refusal_without_handoff(self): + packet = fixture() + packet["handoffs"] = {} + row(packet)["GetSnapshot"] = {"code": "INVALID_ARGUMENT"} + self.assertEqual(self.result(packet)["outcome"], "ENDPOINT_REFUSED") + + def test_missing_cursor_not_fabricated(self): + packet = fixture() + del row(packet)["GetSnapshot"]["stream_cursor"] + self.assertEqual(self.result(packet)["outcome"], "INPUT_ERROR") + + def test_never_claims_full_acceptance(self): + result = helper.validate(fixture()) + self.assertFalse(result["acceptance"]) + self.assertTrue(result["missing_coverage"]) + self.assertNotIn("/helper.py", result["application_modules"].values()) + + +@unittest.skipUnless(os.environ.get("P184_DR05_LIVE_TEST") == "YES", "isolated Redis DB2 required") +class LiveTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + import subprocess + import sys + import time + import urllib.request + cls.ns = "p184-dr05-dl-0123456789abcdef" + cls.server = subprocess.Popen([sys.executable, "-B", helper.__file__, "--serve"], + env={**os.environ, "P184_DR05_ISOLATED_NAMESPACE": cls.ns}, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + for _ in range(60): + if cls.server.poll() is not None: + raise AssertionError(cls.server.communicate()) + try: + with urllib.request.urlopen("http://127.0.0.1:8260/health", timeout=.2) as response: + assert json.load(response)["ready"] + return + except OSError: + time.sleep(.05) + raise AssertionError("server startup timeout") + except BaseException: + cls.server.terminate() + cls.server.communicate(timeout=5) + raise + + @classmethod + def tearDownClass(cls): + cls.server.terminate() + stdout, stderr = cls.server.communicate(timeout=6) + if cls.server.returncode: + raise AssertionError((stdout, stderr)) + from redis import Redis + client = Redis(host="127.0.0.1", port=6379, db=2) + try: + if client.dbsize() != 0: + raise AssertionError("test-owned DB2 keys remain") + finally: + client.close() + + def fresh(self, generation=1): + import time + global NOW + old = NOW + NOW = time.time_ns() - 10_000_000 + try: + packet = fixture(generation=generation) + finally: + NOW = old + packet["namespace"] = self.ns + return packet + + def post(self, packet, namespace=None): + import urllib.request + request = urllib.request.Request("http://127.0.0.1:8260/validate", + data=json.dumps(packet).encode(), headers={"Content-Type": "application/json", + "X-P184-Namespace": namespace or self.ns}) + with urllib.request.urlopen(request, timeout=5) as response: + return json.load(response) + + def test_01_live_adoption_and_fencing(self): + packet = self.fresh() + positive = self.post(packet) + first = positive["results"][0] + self.assertEqual(first["outcome"], "APPLIED", first) + self.assertTrue(first["accepted"]) + self.assertTrue(first["book_validated"]) + refreshed = copy.deepcopy(packet) + row(refreshed)["GetFeedStatus"]["status"]["freshness_ms"] = "17" + row(refreshed)["GetFeedStatus"]["status"]["provider_session_liveness_ms"] = "17" + unchanged = self.post(refreshed)["results"][0] + self.assertEqual(unchanged["outcome"], "NO_WRITE", unchanged) + self.assertTrue(unchanged["book_validated"]) + self.assertFalse(unchanged["accepted"]) + self.assertFalse(unchanged["applied"]) + self.assertEqual(unchanged["cache_apply"], {"accepted": False, "written": False}) + self.assertEqual(unchanged["cache_sha256"], first["cache_sha256"]) + self.assertTrue(positive["live"]) + self.assertFalse(positive["acceptance"]) + recovered = self.post(self.fresh(2))["results"][0] + self.assertEqual(recovered["outcome"], "APPLIED", recovered) + self.assertEqual(recovered["current_generation"], 2) + old = self.post(self.fresh(1))["results"][0] + self.assertEqual(old["outcome"], "FENCED_OLDER", old) + self.assertFalse(old["accepted"]) + self.assertTrue(old["book_validated"]) + self.assertEqual(old["cache_apply"], {"accepted": False, "written": False}) + self.assertTrue(old["cache_unchanged"]) + self.assertEqual(old["cache_sha256"], recovered["cache_sha256"]) + self.assertEqual(old["current_generation"], 2) + + def test_02_expired_and_endpoint_refused(self): + packet = fixture() + packet["namespace"] = self.ns + # Captured clocks lie in the future here: input must refuse before writing. + self.assertFalse(self.post(packet)["results"][0]["accepted"]) + packet = self.fresh(3) + target = row(packet) + target["completed_at_ns"] -= 10_000_000_000 + target["requested_at_ns"] -= 10_000_000_000 + change_event(packet, lambda e: (setattr(e, "source_event_time_ns", e.source_event_time_ns - 10_000_000_000), + setattr(e, "received_at_ns", e.received_at_ns - 10_000_000_000))) + result = self.post(packet)["results"][0] + self.assertEqual(result["outcome"], "APPLICATION_REFUSED", result) + packet = self.fresh(3) + row(packet)["GetSnapshot"] = {"code": "FAILED_PRECONDITION"} + result = self.post(packet)["results"][0] + self.assertEqual(result["outcome"], "ENDPOINT_REFUSED") + self.assertFalse(result["accepted"]) + + def test_03_namespace_and_shape_protection(self): + from urllib.error import HTTPError + packet = self.fresh() + with self.assertRaises(HTTPError) as error: + self.post(packet, "wrong") + self.assertEqual(error.exception.code, 403) + packet["namespace"] = "wrong" + with self.assertRaises(HTTPError) as error: + self.post(packet) + self.assertEqual(error.exception.code, 400) + packet = self.fresh() + packet["now_ns"] = NOW + with self.assertRaises(HTTPError): + self.post(packet) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p184_dr05_verdict.py b/tests/test_p184_dr05_verdict.py new file mode 100644 index 00000000..518131ea --- /dev/null +++ b/tests/test_p184_dr05_verdict.py @@ -0,0 +1,92 @@ +import copy +import unittest + +from scripts.p184_dr05_dl_fault import schedule +from scripts.p184_dr05_verdict import evaluate + + +def observation(venue="OKX", fault="GAP_REFUSAL"): + rows = [] + for step in schedule(venue, fault): + phase = step["phase"] + negative = phase in {"gap", "new_generation_unanchored"} + matches = [] if phase == "old_generation" else [{ + "kind": "quarantine" if negative else "canonical", "venue": venue, + "capture_id": phase, "generation": 8, + "reason": "QUARANTINE_REASON_SEQUENCE_GAP"}] + rows.append({"phase": phase, "publication": {"offset": 1, "capture_id": phase}, + "canonical": {"raw_committed_next": 2, "matched": matches}, + "views": {feed: {"GetSnapshot": {"code": "FAILED_PRECONDITION" if negative else "OK", + "cursor_valid": True, "detail": "DATA_GAP" if negative else ""}, + "GetFeedStatus": {"status": {"execution_eligible": not negative, "gap_open": negative}}} + for feed in ("BOOK_SNAPSHOT", "BOOK_DELTA")}, + "ts_live": {"live": True, "results": [{"feed": "BOOK_SNAPSHOT", "accepted": not negative, + "outcome": "ENDPOINT_REFUSED" if negative else "APPLIED"}]}}) + return {"venue": venue, "fault": fault, "observations": rows} + + +class VerdictTests(unittest.TestCase): + def test_both_venues_and_faults(self): + for venue in ("BINANCE", "OKX"): + for fault in ("GAP_REFUSAL", "GENERATION_CHANGE_FENCED"): + self.assertTrue(evaluate(observation(venue, fault))["ok"]) + + def test_missing_or_duplicate_phase(self): + value = observation() + value["observations"].pop() + self.assertFalse(evaluate(value)["ok"]) + value = observation() + value["observations"].append(copy.deepcopy(value["observations"][-1])) + self.assertFalse(evaluate(value)["ok"]) + + def test_old_book_still_usable_is_failure(self): + value = observation() + row = next(r for r in value["observations"] if r["phase"] == "gap") + row["views"]["BOOK_SNAPSHOT"]["GetSnapshot"]["code"] = "OK" + self.assertFalse(evaluate(value)["ok"]) + + def test_waiting_for_staleness_does_not_prove_gap(self): + value = observation() + row = next(r for r in value["observations"] if r["phase"] == "gap") + for view in row["views"].values(): + view["GetFeedStatus"]["status"]["gap_open"] = False + view["GetSnapshot"]["detail"] = "DATA_STALE" + self.assertFalse(evaluate(value)["ok"]) + + def test_unrelated_quarantine_not_evidence(self): + value = observation() + row = next(r for r in value["observations"] if r["phase"] == "gap") + row["canonical"]["matched"][0]["capture_id"] = "other" + self.assertFalse(evaluate(value)["ok"]) + + def test_offline_ts_does_not_count_as_live(self): + value = observation() + value["observations"][1]["ts_live"]["live"] = False + self.assertFalse(evaluate(value)["ok"]) + + def test_old_generation_canonical_is_failure(self): + value = observation() + row = next(r for r in value["observations"] if r["phase"] == "old_generation") + row["canonical"]["matched"] = [{"kind": "canonical", "generation": 7}] + self.assertFalse(evaluate(value)["ok"]) + + def test_uncommitted_filter_does_not_prove_fencing(self): + value = observation() + row = next(r for r in value["observations"] if r["phase"] == "old_generation") + row["canonical"]["raw_committed_next"] = 1 + self.assertFalse(evaluate(value)["ok"]) + + def test_no_write_is_not_a_refusal(self): + value = observation() + row = next(r for r in value["observations"] if r["phase"] == "gap") + row["ts_live"]["results"][0]["outcome"] = "NO_WRITE" + self.assertFalse(evaluate(value)["ok"]) + + def test_binance_recovery_continues_buffered_generation_delta(self): + for venue, fault, expected in [ + ("BINANCE", "GENERATION_CHANGE_FENCED", 201), + ("OKX", "GENERATION_CHANGE_FENCED", 200), + ("BINANCE", "GAP_REFUSAL", 200), + ]: + step = next(s for s in schedule(venue, fault) if s["phase"] == "return_control") + self.assertEqual(step["previous"], expected) diff --git a/upgrade/evidence/p184-claude-stage3-alignment.md b/upgrade/evidence/p184-claude-stage3-alignment.md index f6d5f696..46f807f7 100644 --- a/upgrade/evidence/p184-claude-stage3-alignment.md +++ b/upgrade/evidence/p184-claude-stage3-alignment.md @@ -70,3 +70,44 @@ typed refusal, post-fault recovery and both venue identities. Canonical DL remains main5bf5cd5/v2.2.2, feature fix/p184-dl-closure. No push, merge, shared runtime change or image build. Retain active/rollback artifacts, including BARf7351c3b; cleanup is restricted to owned disposable tests. + + +## DR05 Observer Handoff After TS01eb0d7 + +Read-only TS inspection now finds `01eb0d7`, adding +`scripts/p184_dr05_upstream.py` and WriterClone. This source is newer than the +confirmed2abbe39/a6548ee3 pair: do not put01eb0d7 into provenance for that image. +Codex has not changed or run Claude's TS checkout. + +Available DL implementation: RawPublisher + CanonicalObserver (bounded raw +commit/EOF proof), real unary Query/Stream observations, and actual in-image TS +SDK/BOOK validation + Redis adoption. See [DR05 guide](p184-dr05-dl-fault.md). +This is NOT a substitute for the full writer clone stream/ACK path. + +Integration points to reconcile before claiming the new TS runner is wired: + +1. Claude's DlChannel expects JSONL reset_case/publish/canonical_observe. DL's + standalone runner owns one NEW five-role stack per cell and has no in-place + reset RPC. Reuse the Python publisher/observer APIs; bind each cell to a fresh + disposable stack. Do not implement reset by flushing shared state or merely + restarting sequence7 against an already advanced core. +2. CanonicalObserver returns raw_committed_next, read_committed_eof, records and + capture-matched rows. `observed=true` must derive from those proofs, never the + phase label or broker publish ACK alone. +3. Binance bootstrap anchor alone is intentionally not yet canonical LIVE; the + following continuity delta establishes the book. The TS evaluator currently + demands positive_anchor APPROVED on both venues. Respect this venue protocol; + do not force a bootstrap-only frame to become executable. +4. The TS evaluator currently accepts any EXECUTION_CONTEXT_* refusal after + polling. DATA_STALE after a wait is not evidence of GAP propagation. Preserve + early snapshots while the prior book is still fresh and require a typed fault + fence (not INPUT_ERROR or Redis NO_WRITE). All repeated old views need their + canonical generation/sequence and actual validator outcome recorded. +5. BOOK quality fault propagation is currently FAIL on the tested isolated path; + fixing the harness alone cannot turn that into PASS. Keep releaseHOLD and the + four cells open. Positive recovery and old-generation filtering are retained, + but do not imply immediate invalidation after a gap. + +No acknowledgement from Claude or full writer-clone acceptance is claimed here. +The next shared handoff should pin the actual TS image and use these observations +as the counterexample for the narrow upstream invalidation fix, not rerun C2. diff --git a/upgrade/evidence/p184-dr05-dl-fault.md b/upgrade/evidence/p184-dr05-dl-fault.md index 6a9fc806..a5f55145 100644 --- a/upgrade/evidence/p184-dr05-dl-fault.md +++ b/upgrade/evidence/p184-dr05-dl-fault.md @@ -60,11 +60,12 @@ control. Timestamps are stamped at publication, never reused from preparation. `drive()` defines the adapter interface. The bounded Kafka publisher is now implemented in `scripts/p184_dr05_raw_publisher.py` and has a 16/16 exact-byte/ offset readback on an isolated broker. See [alignment handoff](p184-claude-stage3-alignment.md). -The canonical/Query/Stream observer and TS integration remain incomplete: -publish real raw-envelope bytes through this core, retain assigned Kafka offsets, -observe canonical/Stream/Query and invoke actual TS validation. Reset only the -disposable case between cells; use bounded deadlines. Do not infer refusal or -recovery from the injected phase name or fabricate status evidence. +The canonical/Query/Stream observer and pinned-image TS BOOK validator are now +implemented in `p184_dr05_observer.py`, `p184_dr05_observe_stack.py` and +`p184_dr05_ts_validate.py`. This is a diagnostic isolated path, NOT Claude's full +WriterClone subscription/ACK acceptance. One fresh disposable stack per cell; +no phase-derived quality or rewritten clocks. Actual results below keep the +upstream gap/generation refusal gate open. Pending rows have ok=false and no verifier provenance header. Even an adapter returning all-success cannot make drive emit PASS. Claude must supply reviewed @@ -101,3 +102,57 @@ Active worktree /home/bobby/.worktrees/data-layer-p184-closure, fix/p184-dl-closure. Shared service images/config/manifest untouched. Retain active images and explicitly protected BAR rollback f7351c3b. No push/merge/public release. v2.2.3 HOLD; rebuild only after TS test completion and exact-source DL CI. + + +## Observer And In-Image TS Validation (2026-10-08) + +Run from the feature checkout with existing images (no build): + +```bash +python3 scripts/p184_dr05_observe_stack.py \ + --output "$EVIDENCE/new-case-directory" --exported "$PUBLIC_EXPORT" \ + --venue OKX --fault GENERATION_CHANGE_FENCED +``` + +`PUBLIC_EXPORT` is sanitized test material, never a credentials export. All raw +frames are explicit TEST_ONLY. Receipt output is immutable per new directory. +The observer follows raw capture ID -> core committed offset -> read_committed +canonical/quarantine; negative absence needs EOF fetched after the core commit. +Real Stream unary GetFeedStatus/GetSnapshot traverse Query and native hot-read +views with temporary mTLS/JWT EXECUTION identities. Query's public serializer +captures real handoff context; the harness never manufactures authority/quality. + +A helper-only mount runs in the pinned TS image a6548ee3; application bytes come +from that image. Live mode uses actual wall clock, production SDK/adapter/BOOK +validator and MarketCacheProjector, isolated Redis DB2, then a cache re-read. +New generation uses production adoption, not deletion of the previous key. +`book_validated` is separate from Redis write/cache equality: NO_WRITE is NOT a +refusal. The offline replay is labelled separately and cannot replace live proof. + +Limits: no full Risk admission (MARK/QUOTE absent), no persistent Subscribe/ACK +writer loop, no provider/mainnet evidence, no production modification. Query +Python uses feature source mounted over the existing image; core and Stream are +existing immutable binaries. These observations cannot certify a future image. + +### Finding Which Keeps DR05 Open + +A gap/unanchored generation can be quarantined correctly by core while the old +BOOK_SNAPSHOT remains LIVE/eligible and passes the TS BOOK validator until its +freshness deadline. Recovery with a new anchor works; old-generation input is +filtered. This is a fault propagation problem, not proof that TS should loosen +freshness. The current sequence-gap/resync branch in +`rust/qdl-realtime-core/src/lib.rs` returns quarantine without invalidating the +reader's previously healthy canonical book. Do not call DATA_STALE after waiting +proof of immediate typed GAP/generation refusal. + +The candidate also exposed a source-coordinate parser defect for '/' in genuine +Kafka standard-base64 topic UUID. Commit47d906e fixes the intermediate parser, +not the signed cursor schema. No image or shared runtime was changed. + +Next narrow implementation must propagate authoritative book invalidation and +recovery through existing canonical quality/read paths, fence by identity and +generation, and close only on a valid replacement anchor. Both primary and hot +backup must see the fence; no timestamp rewrite, Redis flush, timeout relaxation, +provider substitution or extra service. The observer regression is the fixed +counterexample for that repair. Core build/rollout remains under the approved +single-revision release window, not silently included in this observer slice. diff --git a/upgrade/evidence/p184-dr05-observer-results.md b/upgrade/evidence/p184-dr05-observer-results.md new file mode 100644 index 00000000..f066cb3d --- /dev/null +++ b/upgrade/evidence/p184-dr05-observer-results.md @@ -0,0 +1,93 @@ +# DR05 Canonical / Reader / TS Validator Results + +Status: IMPLEMENTED_TESTED_ISOLATED; DR05 FAULT_PROPAGATION_FAIL; release HOLD. +Date: 2026-10-08. This is TEST_ONLY fault injection, not provider/live evidence. +Guide: [DR05](p184-dr05-dl-fault.md). Coordination: [Claude handoff](p184-claude-stage3-alignment.md). + +## Exact Result + +| Venue | Fault | Old snapshot age at status read (ms), n=1 | Reader / TS BOOK validator | Cell | +|---|---|---:|---|---| +| Binance | GAP | 473 | LIVE, eligible / accepted BOOK | FAIL | +| OKX | GAP | 544 | LIVE, eligible / accepted BOOK | FAIL | +| Binance | Unanchored generation | 465 | LIVE, eligible / accepted BOOK | FAIL | +| OKX | Unanchored generation | 542 | LIVE, eligible / accepted BOOK | FAIL | + +These are event ages, not request latencies or outage durations. No percentile +or capacity claim. In all four cases core quarantines the exact fault capture, +but the prior healthy BOOK_SNAPSHOT remains readable while fresh. Redis reports +NO_WRITE for its repeated sequence; that is not a data-quality refusal. The +helper now reports book_validated separately from cache-write/equality outcome. + +12/12 positive/return/post-fence controls returned snapshot bytes with SHA256 +matching the exact capture-correlated canonical BOOK_SNAPSHOT. All reached the +real TS image BOOK validator, production Redis apply and re-read. Four old- +generation raw records were committed/filtered without a canonical publication; +post-fence controls stayed generation8. No synthetic quality is substituted. + +Core source `rust/qdl-realtime-core/src/lib.rs` sequence-gap branch requests +resync and returns quarantine. The tested reader hot path does not receive an +invalidation of the previously healthy canonical book. This is the next narrow +repair target; not justification to loosen freshness or classify it as quiet. + +## Evidence + +Root: `/home/bobby/.local/state/qdl-v2/dr05-observer-20261008/`. +Each row names an immutable `receipt.json` directory and SHA256. + +| Directory | SHA256 | +|---|---| +| binance-gap-v2 | 8e4eae0dc523256305a70e4066ba6397d3e18d6a86ddb1e0387db1a836b4b25e | +| okx-gap-v2 | 3703f46edf9bd308620724982a86d09b54e7b74b3a7b50ef0127278f744e53ac | +| binance-generation-v3 | 590291f159d34c89919ff9a864a43987800f99e70025d11426c2c087df294bdb | +| okx-generation-v2 | f89356e55d6e688555c1fe943d97619a0fb4ca3567460e3a74f5b818658fd9c2 | + +Earlier attempts are retained. Binance generation-v2 exposed a fixture error: +anchor200 replays buffered delta201, so the next delta must reference201. Fixed +only that venue/fault schedule, added regression, reran only that affected cell. +The older OKX preliminary subcheck incorrectly equated NO_WRITE with refusal; +final verdict requires explicit typed refusal. No earlier receipt was edited. + +## Verification And Boundaries + +- Parent final selected DL suite: 52 PASS, 0 FAIL, 0 SKIP, 32.861s. +- Agent TS-image suite with disposable Redis: 22 PASS, 0 FAIL, 0 SKIP, 5.992s. +- Agent final marker/cache semantics tests: 3 PASS, 0 FAIL, 0 SKIP, 3.638s. +- Parent independent network-none TS suite: 19 PASS, 3 explicit Redis skips. +- These suites overlap; do not sum them as unique coverage. +- Cursor parser fix47d906e: old source3tests/2errors; corrected source3PASS. + +Application images: core658a9570, Stream1f13408f, Query base77682eea +(2.2.3-rc2-43bbcf0 with feature Python source mounted), Kafka9516fb76, +Redisdfa18828. TS application imagea6548ee3/source2abbe39, helper-only mount. +Receipts retain full image digests. No app-source overlay into TS. No new image +build, shared manifest/ACL/config/credential change, provider call or order. + +This covers real unary Stream -> Query/hot view and TS BOOK validator/Redis +application, NOT full WriterClone Subscribe/ACK, projector-primary materialization, +or full Risk admission requiring MARK/QUOTE. Claude's newer TS01eb0d7 adapter +still needs protocol alignment described in the linked handoff. Do not declare +all TS DR05 integration complete or emit a PASS certificate from these receipts. + +## Cleanup And Runtime + +All four selected cases removed their six isolated roles plus the --rm helper; +private files remaining0 in each receipt. No shared data flush/delete or image +prune. Per-case free bytes before/after: + +- Binance GAP: 74,227,871,744 -> 74,229,985,280. +- OKX GAP: 74,228,985,856 -> 73,978,429,440. +- Binance generation: 74,422,525,952 -> 74,396,819,456. +- OKX generation: 74,523,852,800 -> 74,509,160,448. + +Host activity is concurrent; disk deltas are not attributed solely to tests. +Inventory: 95images/28.96GB, BuildKit20.52GB. No images/cache were built by this +slice; broad pruning deferred to the approved release cleanup, preserving active +and BAR rollbackf7351c3b plus artifacts owned by Claude's active work. + +Canonical `/home/bobby/data_layer` main5bf5cd5/v2.2.2 unchanged. Only active DL +feature worktree is data-layer-p184-closure/fix/p184-dl-closure. Read-only final +snapshot of 12 shared application roles plus five infrastructure roles shows +restart0/OOMfalse. This is not a fresh runtime acceptance. Shared core658a9570, +Query/BAR51450612, Stream1f13408f, projectora696cfba, ingestor0f6876e1 unchanged. +No push/merge/release. v2.2.3 remains HOLD. From d68fb2ff47622db8224dc4b9e004dca4e20f643e Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 11:55:46 +0000 Subject: [PATCH 037/147] fix(dr05): fence invalid books and complete isolated TS channel handoff --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 70 ++++ qdl/runtime/kn_hot_view.py | 9 +- qdl/runtime/kn_query_backend.py | 37 +- rust/qdl-core/src/canonical.rs | 54 ++- rust/qdl-kafka/src/bin/qdl-production-core.rs | 299 +++++++++++++-- rust/qdl-projector/src/products.rs | 21 + rust/qdl-realtime-core/src/lib.rs | 150 ++++++-- scripts/p184_dr05_case.py | 237 ++++++++++++ scripts/p184_dr05_channel.py | 361 ++++++++++++++++++ scripts/p184_dr05_channel_smoke.py | 116 ++++++ scripts/p184_dr05_dl_fault.py | 75 +++- scripts/p184_dr05_observe_stack.py | 11 +- scripts/p184_dr05_observer.py | 6 +- scripts/p184_dr05_reset.py | 177 +++++++++ tests/test_kn_hot_book_invalidation.py | 167 ++++++++ tests/test_p184_dr05_case.py | 276 +++++++++++++ tests/test_p184_dr05_channel.py | 335 ++++++++++++++++ tests/test_p184_dr05_observer.py | 3 +- tests/test_p184_dr05_reset.py | 79 ++++ .../evidence/p184-claude-stage3-alignment.md | 15 + upgrade/evidence/p184-dr05-dl-fault.md | 4 +- .../p184-dr05-fence-channel-handoff.md | 185 +++++++++ 22 files changed, 2614 insertions(+), 73 deletions(-) create mode 100644 scripts/p184_dr05_case.py create mode 100644 scripts/p184_dr05_channel.py create mode 100644 scripts/p184_dr05_channel_smoke.py create mode 100644 scripts/p184_dr05_reset.py create mode 100644 tests/test_kn_hot_book_invalidation.py create mode 100644 tests/test_p184_dr05_case.py create mode 100644 tests/test_p184_dr05_channel.py create mode 100644 tests/test_p184_dr05_reset.py create mode 100644 upgrade/evidence/p184-dr05-fence-channel-handoff.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 73502e90..a653c369 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70368,3 +70368,73 @@ is not claimed completed: his new01eb0d7 interface needs fresh-case mapping and must not count eventual DATA_STALE as immediate GAP refusal. These interface points are recorded in p184-claude-stage3-alignment.md. Reuse this counterexample; no C2/full catalogue rerun, no shared rollout from this observer slice. + + +### DR05 upstream invalidation and TS packet integration - 2026-10-08 (Codex) + +Status IMPLEMENTING. Owner approves fixing canonical BOOK gap/generation +propagation and providing isolated TS consumer bindings plus reset_case and +canonical_observe APIs. TS Claude owns TsDr05Adapter and final evidence; drive +continues ok=false. Guide p184-dr05-dl-fault.md and owner packet contract apply. +No shared runtime, manifests, ACLs, TS source edits, push/merge or release. +Tests: valid book -> fault -> immediate typed refusal -> valid anchor recovery; +old generation/identity fencing, duplicate replay, primary/hot reader parity; +packet V3 compiler/policies, path-only secret references, namespace ownership, +bounded reset/canonical observation. Reserved TS ports6391/5433 untouched. +Reuse schemas/quality flags and real committed observations; no stale wait, +timestamp rewriting, invented prices or buffer/timeout relaxation. Isolated +compile/tests only; any candidate binary is test-only until release packet. +Cleanup exact disposable labels/private files, retain active/rollback artifacts. + + +DR05 tested-slice update: IMPLEMENTED_TESTED_ISOLATED / TS_FINAL_ACCEPTANCE_PENDING. +Current guide/evidence upgrade/evidence/p184-dr05-fence-channel-handoff.md. +Core emits bounded canonical invalidation for both BOOK products on GAP or +unanchored generation; primary validator and hot reader preserve typed refusal. +Four isolated counterexample cells now pass diagnostic checks;12 positive/ +recovery controls retain exact canonical bytes. Previous failing receipts intact. +Python120PASS0fail0skip43.876s; separate reset4PASS0fail0skip0.006s. +Rust core52PASS0fail1ignored; projector product8PASS0fail0ignored. +JSONL handoff16 actual publish/observe steps,2 real resets21670.885/19807.860ms; +broker ID stable, topic UUID changed, raw starts0. TS ports6391/5433 unused. +No Docker socket in helper; exact owned roles/topics/groups only. No TS edits. +Scope kept: final TS WriterClone/control/cache evidence remains Claude-owned; +drive ok=false, no final provenance/certificate. No shared runtime/push/merge. +Disposable stack cleanup/privatefiles0 in all5receipts, exact hashes in report. +Final binary rebuild after redundant-borrow cleanup is recorded separately; +prior receipts retain original core hash, not relabelled as final image tests. + +DR05 self-review before commit found two in-scope compatibility/recovery cases: +production-core alternate runner rejects mixed quarantine+canonical fence batches; +same-generation/native-sequence recovery anchor can be deduplicated. Fix both +with targeted regression before closing this slice; retain quarantine evidence +and stable replay identity. No runtime changes or expanded catalogue testing. + +Pre-commit recovery regression reproduced0published records for a valid OKX +post-fence snapshot at the same generation/native sequence. Fixed using the +new anchor's immutable raw capture discriminator, retaining native sequence, +prices, generation and clocks; duplicate replay remains suppressed. Final core +suite53PASS0fail1ignored0.11s. Packet exact-schema/field-set/V3 parity recheck +1PASS9.114s (overlaps120suite; not added to unique total). +Read-only shared inventory17roles(12app+5infra), restart0/OOMfalse; no shared +mutations. Images95/28.96GB, BuildKit20.52GB shared with Claude; no broad prune. + +DR05 slice FINAL: IMPLEMENTED_TESTED_ISOLATED / TS_FINAL_ACCEPTANCE_PENDING. +Alternate runner fixed narrowly: retain paired BOOK invalidation and quarantine +in one transaction; other mixed outcomes rejected. Three pre-fix regressions +failed; final runner7PASS0fail0ignored. Parent core53PASS0fail1ignored; projector +products8PASS; Python120PASS plus separate4resetPASS (overlapping checks not summed). +Final binarybaafcb489bb17fabdd73bef383dcf3448581cc8bb63578e9df83e2a57107b6f2 +retained at ~/.local/state/qdl-v2/dr05-fence-20261008/bin/qdl-realtime-core-baafcb48. +Affected OKX generation final-binary rerun PASS; receiptfcf4b6593732cde3fc6897e750346dea2ed9d3744d80ce29add82aa1a824000a. +Earlier4cell/JSONL receipts preserve earlier binary hashes; no false provenance. +All6case stacks/privatefiles removed; no testcontainers left. Removed exact +Cargo target1225121125filebytes, free59893260288->60988051456bytes; concurrent +activity prevents attribution. Root-owned copied build artifacts required one +network-none cleanup helper mounted only to that target. No new image or shared +prune. Full report/commands/proof limits in p184-dr05-fence-channel-handoff.md. +Canonicalmain5bf5cd5/v2.2.2 unchanged, featurefix/p184-dl-closure only. +Shared digests/config untouched;17read-only roles restart0/OOMfalse. Retain all +active and BARrollbackf7351c3b; no push/merge/deploy. Claude next runs his actual +WriterClone/control/cache/ACK validation with honest TS source/image provenance. +DR05 final acceptance and v2.2.3 remain HOLD pending joint DR/broker evidence. diff --git a/qdl/runtime/kn_hot_view.py b/qdl/runtime/kn_hot_view.py index 847a61bd..4a490319 100644 --- a/qdl/runtime/kn_hot_view.py +++ b/qdl/runtime/kn_hot_view.py @@ -30,6 +30,10 @@ class HotViewUnavailable(RuntimeError): """Bounded backup refused; never permission to relax primary quality.""" +class HotViewGap(HotViewUnavailable): + """Authoritative book refusal; neither a replica nor cache may hide it.""" + + @dataclass(frozen=True) class CanonicalHotView: """Source coordinates, not a fabricated materialized-cache generation/fence.""" @@ -97,11 +101,14 @@ def latest(self, binding, lpk) -> CanonicalHotView: reply = call(BytesValue(value=body), timeout=remaining / (len(self._calls) - index), metadata=(("x-qdl-hot-signature-bin", signature),), wait_for_ready=False) except grpc.RpcError as failure: + detail = getattr(failure, "details", lambda: None)() + if failure.code() == grpc.StatusCode.FAILED_PRECONDITION and detail in { + "HOT_RECORD_GAPPED", "HOT_RECORD_GENERATION_MISMATCH", "HOT_BOOK_UNVERIFIED"}: + raise HotViewGap(detail) from failure # Auth/contract failures are not healed by trying another replica. if failure.code() not in {grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.DEADLINE_EXCEEDED, grpc.StatusCode.RESOURCE_EXHAUSTED}: raise HotViewUnavailable("HOT_AUTHORITY_OR_PROTOCOL_REFUSED") from failure - detail = getattr(failure, "details", lambda: None)() error = detail if detail in { "HOT_BROKER_UNCONFIRMED", "HOT_READER_CATCHING_UP", "HOT_READER_STALLED", "HOT_READER_UNAVAILABLE", "HOT_READER_BUSY", diff --git a/qdl/runtime/kn_query_backend.py b/qdl/runtime/kn_query_backend.py index 0b9af175..1579490f 100644 --- a/qdl/runtime/kn_query_backend.py +++ b/qdl/runtime/kn_query_backend.py @@ -55,7 +55,7 @@ from qdl.query.results import NON_REPLAYABLE_STREAM_CURSOR, QueryBackendError, GapScanResult from qdl.query.row_cache import BoundedRowCache from qdl.replay.cursor_v3 import CursorV3Claims, SignedCursorV3Codec, requirement_digest -from qdl.runtime.kn_hot_view import HOT_FEEDS, HotViewUnavailable +from qdl.runtime.kn_hot_view import HOT_FEEDS, HotViewGap, HotViewUnavailable from qdl.runtime.kn_bar_readback import binding_product_key from qdl.runtime.kn_market_cache import ( KnCacheError, @@ -264,7 +264,12 @@ def _hot_candidate(self, binding, primary=None): """Authenticated committed bytes, validated by the same lineage oracle.""" if self.hot_client is None or binding.feed.value not in HOT_FEEDS: return None - candidate = self.hot_client.latest(binding, self.product_key(binding)) + try: + candidate = self.hot_client.latest(binding, self.product_key(binding)) + except HotViewGap as error: + raise QueryBackendError(QueryProblem( + CanonicalErrorCode.OPEN_SEQUENCE_GAP, str(error), True, + )) from error if primary is not None: if candidate.boundary.topic_id != primary.boundary.topic_id or candidate.boundary.partition != primary.boundary.partition: raise _not_ready("HOT_SOURCE_GENERATION_MISMATCH") @@ -425,7 +430,33 @@ def _validate_records(self, binding, records) -> None: for parsed in records: entry = self._row(binding, parsed.stored.payload_sha256, parsed.stored.event.payload) if not entry.validated: - super()._validate_records(binding, (parsed,)) + try: + super()._validate_records(binding, (parsed,)) + except ValueError as error: + envelope = parsed.envelope + payload = envelope.WhichOneof("payload") + # Catalog checks identity/provenance before these payload + # errors. Empty fences are refusals, never readable books. + expected_error = { + "book_snapshot": "stable book snapshot sequence/depth is incomplete", + "book_delta": "stable book delta lacks continuity/snapshot sequence", + }.get(payload) + flags = set(envelope.quality_flags) + if (expected_error is not None and str(error) == expected_error + and envelope.instrument_uid == binding.instrument.instrument_uid + and envelope.source_id == binding.source_id + and payload == binding.feed.value.lower() + and {common_pb2.QUALITY_FLAG_SEQUENCE_GAP_BEFORE, + common_pb2.QUALITY_FLAG_RESYNC_REQUIRED} <= flags): + book = getattr(envelope, payload) + empty = (not book.levels if payload == "book_snapshot" + else not book.updates and book.reset) + if empty and not book.sequence_verified: + raise QueryBackendError(QueryProblem( + CanonicalErrorCode.OPEN_SEQUENCE_GAP, + "BOOK_INVALIDATED_RESYNC_REQUIRED", True, + )) from error + raise entry.validated = True def _item(self, requirement, binding, stored, envelope, gap_open): diff --git a/rust/qdl-core/src/canonical.rs b/rust/qdl-core/src/canonical.rs index eb31dfe9..3de4c189 100644 --- a/rust/qdl-core/src/canonical.rs +++ b/rust/qdl-core/src/canonical.rs @@ -837,6 +837,28 @@ fn canonicalize_deribit_fixture(fixture: &TradeFixture) -> Result Result { + l2_ready_snapshot(fixture, transition, None) +} + +/// A real post-fence anchor may repeat a native sequence. Its capture identity +/// distinguishes recovery from the pre-fault snapshot without changing prices. +pub fn canonicalize_l2_recovery_snapshot( + fixture: &TradeFixture, + transition: &BookTransition, +) -> Result { + if fixture.context.raw_capture_id.is_empty() { + return Err("L2 recovery requires raw capture provenance".into()); + } + let mut discriminator = b"recovery:".to_vec(); + discriminator.extend_from_slice(&fixture.context.raw_capture_id); + l2_ready_snapshot(fixture, transition, Some(&discriminator)) +} + +fn l2_ready_snapshot( + fixture: &TradeFixture, + transition: &BookTransition, + recovery_discriminator: Option<&[u8]>, ) -> Result { let view = transition .view @@ -853,7 +875,7 @@ pub fn canonicalize_l2_ready_snapshot( "book_snapshot", source_sequence, source_time_ms, - materialization_discriminator.as_deref().map(str::as_bytes), + recovery_discriminator.or(materialization_discriminator.as_deref().map(str::as_bytes)), )?; let unit = quantity_unit(&fixture.context)?; let mut levels = l2_levels(&view.bids, unit)?; @@ -879,6 +901,36 @@ pub fn canonicalize_l2_ready_snapshot( Ok(envelope) } +/// Publish a non-readable fence for both book products. No price/level or native +/// sequence is invented; the receipt clock dates the invalidation, not a price. +/// Existing quality/reset fields make old consumers fail closed as well. +pub fn canonicalize_l2_invalidation(fixture: &TradeFixture) -> Result, String> { + let generation = fixture.context.connection_generation; + if generation == 0 || fixture.context.raw_capture_id.is_empty() { + return Err("L2 invalidation requires generation and raw provenance".into()); + } + let source_sequence = format!("g{generation}:invalid:{}", fixture.context.partition_sequence); + let at_ms = fixture.context.received_at_ns / 1_000_000; + let mut snapshot = base_envelope(fixture, "book_snapshot", source_sequence.clone(), at_ms)?; + snapshot.quality_flags.extend([ + QualityFlag::SequenceGapBefore as i32, QualityFlag::ResyncRequired as i32, + ]); + snapshot.payload = Some(event_envelope::Payload::BookSnapshot(OrderBookSnapshot { + book_generation: generation, sequence_verified: false, ..Default::default() + })); + set_payload_hash(&mut snapshot)?; + let mut delta_fixture = fixture.clone(); + delta_fixture.context.partition_sequence = fixture.context.partition_sequence.checked_add(1) + .ok_or("L2 invalidation sequence overflow")?; + let mut delta = base_envelope(&delta_fixture, "book_delta", source_sequence, at_ms)?; + delta.quality_flags = snapshot.quality_flags.clone(); + delta.payload = Some(event_envelope::Payload::BookDelta(OrderBookDelta { + book_generation: generation, reset: true, sequence_verified: false, ..Default::default() + })); + set_payload_hash(&mut delta)?; + Ok(vec![snapshot, delta]) +} + /// Convert only a publication-approved L2 transition into its canonical V2 /// representation. A caller can persist raw bootstrap/gap lifecycle records /// for audit, but those transitions intentionally yield no readable book event. diff --git a/rust/qdl-kafka/src/bin/qdl-production-core.rs b/rust/qdl-kafka/src/bin/qdl-production-core.rs index 3eecc69b..062e0498 100644 --- a/rust/qdl-kafka/src/bin/qdl-production-core.rs +++ b/rust/qdl-kafka/src/bin/qdl-production-core.rs @@ -421,6 +421,54 @@ fn validate_raw_authority( Ok(()) } +fn is_book_invalidation(result: &ProcessBatch) -> bool { + use qdl_contracts::qdl::{ + common::v1::QualityFlag, + marketdata::v2::{event_envelope::Payload, EventEnvelope}, + }; + if result.canonical.len() != 2 + || result.duplicates != 0 + || result.canonical[0].partition_key != result.canonical[1].partition_key + { + return false; + } + let mut generation = None; + result.canonical.iter().enumerate().all(|(index, record)| { + let Ok(event) = EventEnvelope::decode(record.payload.as_slice()) else { + return false; + }; + if !event + .quality_flags + .contains(&(QualityFlag::SequenceGapBefore as i32)) + || !event + .quality_flags + .contains(&(QualityFlag::ResyncRequired as i32)) + { + return false; + } + match (index, event.payload) { + (0, Some(Payload::BookSnapshot(book))) => { + generation = Some(book.book_generation); + book.book_generation > 0 + && !book.sequence_verified + && book.levels.is_empty() + && book.depth == 0 + && book.native_sequence.is_empty() + } + (1, Some(Payload::BookDelta(book))) => { + Some(book.book_generation) == generation + && !book.sequence_verified + && book.reset + && book.updates.is_empty() + && book.native_sequence_start.is_empty() + && book.native_sequence_end.is_empty() + && book.snapshot_sequence.is_empty() + } + _ => false, + } + }) +} + fn decision(result: &ProcessBatch) -> Result { let active = [ !result.canonical.is_empty(), @@ -428,7 +476,9 @@ fn decision(result: &ProcessBatch) -> Result { result.duplicates > 0, result.filtered > 0, ]; - if active.into_iter().filter(|value| *value).count() != 1 { + // A book fence is canonical output plus its raw rejection evidence, not + // competing normalization results. Keep other mixed outcomes fail-closed. + if active.into_iter().filter(|value| *value).count() != 1 && !is_book_invalidation(result) { return Err("normalizer produced an ambiguous decision for one raw event".into()); } if !result.canonical.is_empty() { @@ -442,6 +492,41 @@ fn decision(result: &ProcessBatch) -> Result { } } +fn publication_outputs( + result: &ProcessBatch, + item_decision: Phase92Decision, + topics: &ProductionTopicConfig, + publication: &Phase92PublicationContext, + raw: &[u8], +) -> Result, RuntimeError> { + let mut outputs = Vec::new(); + if item_decision == Phase92Decision::Canonical { + for record in &result.canonical { + let mut projected = record.clone(); + projected.stream = output_stream(topics, publication.target)?.into(); + outputs.push(Phase92TransactionalOutput { + record: projected, + publication: publication.clone(), + raw_provider_envelope: Some(raw.to_vec()), + }); + } + } + // Keep quarantine evidence in the same transaction/progress watermark as + // canonical fences, on the existing primary-only quarantine route. + if !result.quarantines.is_empty() && publication.target == SinkTarget::PrimaryCanonical { + for record in &result.quarantines { + let mut quarantined = record.clone(); + quarantined.stream = topics.quarantine.clone(); + outputs.push(Phase92TransactionalOutput { + record: quarantined, + publication: publication.clone(), + raw_provider_envelope: Some(raw.to_vec()), + }); + } + } + Ok(outputs) +} + fn output_stream(topics: &ProductionTopicConfig, target: SinkTarget) -> Result<&str, RuntimeError> { match target { SinkTarget::CanaryCanonical => Ok(&topics.canary_canonical), @@ -591,29 +676,13 @@ async fn run_generation( source_cursor: input.cursor.clone(), source_event_id: input.record.event_id.clone(), }); - if item_decision == Phase92Decision::Canonical { - for record in &result.canonical { - let mut projected = record.clone(); - projected.stream = output_stream(&config.topics, target)?.into(); - outputs.push(Phase92TransactionalOutput { - record: projected, - publication: publication.clone(), - raw_provider_envelope: Some(input.record.payload.clone()), - }); - } - } else if item_decision == Phase92Decision::Quarantine - && target == SinkTarget::PrimaryCanonical - { - for record in &result.quarantines { - let mut quarantined = record.clone(); - quarantined.stream = config.topics.quarantine.clone(); - outputs.push(Phase92TransactionalOutput { - record: quarantined, - publication: publication.clone(), - raw_provider_envelope: Some(input.record.payload.clone()), - }); - } - } + outputs.extend(publication_outputs( + &result, + item_decision, + &config.topics, + &publication, + &input.record.payload, + )?); } } bridge @@ -729,6 +798,188 @@ mod tests { assert!(decision(&ambiguous).is_err()); } + fn fence_batch(quarantine: bool) -> ProcessBatch { + use qdl_contracts::qdl::{ + common::v1::QualityFlag, + marketdata::v2::{ + event_envelope::Payload, EventEnvelope, OrderBookDelta, OrderBookSnapshot, + }, + }; + let canonical = [ + Payload::BookSnapshot(OrderBookSnapshot { + book_generation: 7, + ..Default::default() + }), + Payload::BookDelta(OrderBookDelta { + book_generation: 7, + reset: true, + ..Default::default() + }), + ] + .into_iter() + .enumerate() + .map(|(index, payload)| { + let event = EventEnvelope { + quality_flags: vec![ + QualityFlag::SequenceGapBefore as i32, + QualityFlag::ResyncRequired as i32, + ], + payload: Some(payload), + ..Default::default() + }; + qdl_core::transport::DurableRecord { + stream: "canonical".into(), + partition_key: "TEST_ONLY.book".into(), + event_id: vec![index as u8 + 1], + payload: event.encode_to_vec(), + accepted_at_ns: 1, + } + }) + .collect(); + ProcessBatch { + canonical, + quarantines: if quarantine { + vec![qdl_core::transport::DurableRecord { + stream: "quarantine".into(), + partition_key: "TEST_ONLY.raw".into(), + event_id: vec![3], + payload: b"TEST_ONLY.quarantine".to_vec(), + accepted_at_ns: 1, + }] + } else { + vec![] + }, + duplicates: 0, + filtered: usize::from(!quarantine), + filtered_outcome: (!quarantine).then_some("BUFFERED_AWAITING_BOOTSTRAP"), + } + } + + #[test] + fn book_fence_with_quarantine_has_canonical_progress() { + assert_eq!( + decision(&fence_batch(true)).unwrap(), + Phase92Decision::Canonical + ); + } + + #[test] + fn book_fence_with_filtered_bootstrap_has_canonical_progress() { + assert_eq!( + decision(&fence_batch(false)).unwrap(), + Phase92Decision::Canonical + ); + } + + #[test] + fn mixed_non_fences_remain_rejected() { + use qdl_contracts::qdl::marketdata::v2::{event_envelope::Payload, EventEnvelope}; + for case in 0..7 { + let mut batch = fence_batch(true); + let mut event = EventEnvelope::decode(batch.canonical[0].payload.as_slice()).unwrap(); + match case { + 0 => event.quality_flags.clear(), + 1 => { + if let Some(Payload::BookSnapshot(book)) = &mut event.payload { + book.sequence_verified = true; + } + } + 2 => { + if let Some(Payload::BookSnapshot(book)) = &mut event.payload { + book.native_sequence = "100".into(); + } + } + 3 => { + batch.canonical.pop(); + } + 4 => batch.canonical[1].partition_key = "other-book".into(), + 5 => batch.duplicates = 1, + 6 => { + if let Some(Payload::BookSnapshot(book)) = &mut event.payload { + book.book_generation = 8; + } + } + _ => unreachable!(), + } + batch.canonical[0].payload = event.encode_to_vec(); + assert!( + decision(&batch).is_err(), + "case {case} must not be admitted as a fence" + ); + } + let mut empty = fence_batch(false); + empty.canonical.clear(); + empty.filtered = 0; + empty.filtered_outcome = None; + assert!(decision(&empty).is_err()); + } + + #[test] + fn book_fence_outputs_preserve_quarantine_and_exact_bytes() { + let batch = fence_batch(true); + let topics = ProductionTopicConfig { + raw_inputs: vec!["raw".into()], + authority_control: "authority".into(), + target_checkpoints: "checkpoints".into(), + canary_canonical: "canary".into(), + primary_canonical: "primary".into(), + public_v2: "public".into(), + legacy_v1: "legacy".into(), + quarantine: "quarantine".into(), + }; + for target in [ + SinkTarget::PrimaryCanonical, + SinkTarget::CanaryCanonical, + SinkTarget::PublicV2, + SinkTarget::LegacyV1, + ] { + let publication = Phase92PublicationContext { + slice_id: "TEST_ONLY.slice".into(), + owner_id: "TEST_ONLY.owner".into(), + authority_revision: 1, + shard_id: "TEST_ONLY.shard".into(), + lease_epoch: 1, + partition_plan_epoch: 1, + source_watermark: 42, + target, + }; + let outputs = publication_outputs( + &batch, + Phase92Decision::Canonical, + &topics, + &publication, + b"TEST_ONLY.raw", + ) + .unwrap(); + assert_eq!( + outputs.len(), + if target == SinkTarget::PrimaryCanonical { + 3 + } else { + 2 + } + ); + for (index, output) in outputs.iter().enumerate() { + let mut expected = if index < 2 { + batch.canonical[index].clone() + } else { + batch.quarantines[0].clone() + }; + expected.stream = if index < 2 { + output_stream(&topics, target).unwrap().into() + } else { + topics.quarantine.clone() + }; + assert_eq!(output.record, expected); + assert_eq!(output.publication, publication); + assert_eq!( + output.raw_provider_envelope.as_deref(), + Some(b"TEST_ONLY.raw".as_slice()) + ); + } + } + } + #[test] fn shared_raw_scope_accepts_exact_binding_and_ignores_valid_unbound_envelope() { let binding = RuntimeSliceBinding { diff --git a/rust/qdl-projector/src/products.rs b/rust/qdl-projector/src/products.rs index ace79512..06d4def1 100644 --- a/rust/qdl-projector/src/products.rs +++ b/rust/qdl-projector/src/products.rs @@ -388,6 +388,27 @@ mod tests { ); } + #[test] + fn book_invalidation_retains_flags_and_separate_latest_products() { + use qdl_contracts::qdl::common::v1::QualityFlag; + for (feed, payload) in [ + ("BOOK_SNAPSHOT", Payload::BookSnapshot(OrderBookSnapshot { + book_generation: 7, sequence_verified: false, ..Default::default() + })), + ("BOOK_DELTA", Payload::BookDelta(OrderBookDelta { + book_generation: 7, reset: true, sequence_verified: false, ..Default::default() + })), + ] { + let mut envelope = EventEnvelope::decode(encoded(UID, payload).as_slice()).unwrap(); + envelope.quality_flags = vec![QualityFlag::SequenceGapBefore as i32, QualityFlag::ResyncRequired as i32]; + let bytes = envelope.encode_to_vec(); + let (_, frame) = only(transform().transform(&record(BOOK, bytes.clone(), 12)).unwrap()); + assert_eq!(frame.lpk.feed, feed); + assert_eq!(frame.kind, FrameKind::Latest); + assert_eq!(frame.envelope, bytes); + } + } + #[test] fn bar_goes_to_the_bars_topic_with_its_fact_key_and_envelope_unchanged() { let payload = encoded(UID, final_bar()); diff --git a/rust/qdl-realtime-core/src/lib.rs b/rust/qdl-realtime-core/src/lib.rs index 0f26d7e9..95246278 100644 --- a/rust/qdl-realtime-core/src/lib.rs +++ b/rust/qdl-realtime-core/src/lib.rs @@ -9,7 +9,8 @@ use qdl_contracts::qdl::provider::v1::{ QuarantineReason, QuarantineRecord, RawProviderEnvelope, TransportProtocol, }; use qdl_core::canonical::{ - canonicalize_l2_publication, canonicalize_mark_index, canonicalize_trade, MarkIndexInput, + canonicalize_l2_invalidation, canonicalize_l2_publication, canonicalize_l2_recovery_snapshot, + canonicalize_mark_index, canonicalize_trade, MarkIndexInput, TradeContext, TradeFixture, }; use qdl_core::l2_adapter::{BookPublication, BookTransition, L2BookAdapter}; @@ -425,6 +426,7 @@ pub struct RealtimeCore { bindings: BTreeMap, l2_adapters: BTreeMap, l2_snapshot_observations: BTreeMap, + l2_invalidated: BTreeMap, mark_index_pairs: BTreeMap, partition_sequences: BTreeMap, /// The provider session that currently owns each L2 book, so a lane change @@ -473,6 +475,7 @@ impl RealtimeCore { bindings, l2_adapters, l2_snapshot_observations: BTreeMap::new(), + l2_invalidated: BTreeMap::new(), mark_index_pairs: BTreeMap::new(), partition_sequences: BTreeMap::new(), l2_sessions: BTreeMap::new(), @@ -1109,6 +1112,48 @@ impl RealtimeCore { binding: &CoreBinding, raw: RawProviderEnvelope, payload: Value, + processing_at_ns: i64, + transport_offset: Option, + ) -> ProcessBatch { + let mut batch = self.process_l2_inner(binding_key, binding, &raw, payload, + processing_at_ns, transport_offset); + let Some(adapter) = self.l2_adapters.get(binding_key) else { return batch; }; + if adapter.core().status() == BookStatus::Ready { + if !batch.canonical.is_empty() { + self.l2_invalidated.remove(binding_key); + } + return batch; + } + // A superseded/foreign frame cannot invalidate the current owner. + if adapter.core().generation() != raw.connection_generation + || self.l2_sessions.get(binding_key) != Some(&raw.source_session_id) + || matches!(batch.filtered_outcome, Some("IGNORED_STALE_GENERATION" | "IDENTITY_MISMATCH")) + { return batch; } + let scope = (raw.source_session_id.clone(), raw.connection_generation); + if self.l2_invalidated.get(binding_key) == Some(&scope) { return batch; } + let Some(sequence) = self.next_l2_partition_sequence(binding, transport_offset) else { + return batch; + }; + let Ok(payload) = serde_json::from_slice(&raw.raw_frame_bytes) else { return batch; }; + let fixture = l2_fixture(binding, &raw, payload, sequence); + let Ok(events) = canonicalize_l2_invalidation(&fixture) else { return batch; }; + for event in events { + if self.seen_ids.contains(&event.event_id) { continue; } + self.partition_sequences.insert(l2_partition_key(binding), event.partition_sequence); + self.remember(event.event_id.clone()); + batch.canonical.push(canonical_record(&self.config.canonical_stream, event, raw.received_at_ns)); + } + self.l2_invalidated.insert(binding_key.to_owned(), scope); + self.l2_snapshot_observations.remove(binding_key); + batch + } + + fn process_l2_inner( + &mut self, + binding_key: &str, + binding: &CoreBinding, + raw: &RawProviderEnvelope, + payload: Value, _processing_at_ns: i64, transport_offset: Option, ) -> ProcessBatch { @@ -1124,7 +1169,7 @@ impl RealtimeCore { Ok(value) => value, Err(_) => { return self.quarantine( - &raw, + raw, QuarantineReason::SemanticInvalid, "L2 raw transport protocol is invalid", materialized_at_ns, @@ -1135,7 +1180,7 @@ impl RealtimeCore { Some(value) => value, None => { return self.quarantine( - &raw, + raw, QuarantineReason::SemanticInvalid, "L2 transport-derived partition sequence overflow", materialized_at_ns, @@ -1190,7 +1235,7 @@ impl RealtimeCore { let transition = { let Some(adapter) = self.l2_adapters.get_mut(binding_key) else { return self.quarantine( - &raw, + raw, QuarantineReason::SemanticInvalid, "L2 binding has no initialized state machine", materialized_at_ns, @@ -1216,7 +1261,7 @@ impl RealtimeCore { Ok(value) => value, Err(_) => { return self.quarantine( - &raw, + raw, QuarantineReason::SemanticInvalid, "L2 provider frame failed strict parser validation", materialized_at_ns, @@ -1291,7 +1336,7 @@ impl RealtimeCore { QuarantineReason::SemanticInvalid }; return self.quarantine( - &raw, + raw, reason, "L2 state machine rejected continuity; resync required", materialized_at_ns, @@ -1300,17 +1345,22 @@ impl RealtimeCore { transition }; - let canonical = match canonicalize_l2_publication( - &l2_fixture(binding, &raw, fixture_payload.clone(), partition_sequence), - &transition, - ) { + let fixture = l2_fixture(binding, raw, fixture_payload.clone(), partition_sequence); + let publication = if self.l2_invalidated.contains_key(binding_key) + && transition.publication == BookPublication::Snapshot + { + canonicalize_l2_recovery_snapshot(&fixture, &transition).map(Some) + } else { + canonicalize_l2_publication(&fixture, &transition) + }; + let canonical = match publication { Ok(value) => value, Err(_) => { if let Some(adapter) = self.l2_adapters.get_mut(binding_key) { adapter.request_resync(raw.connection_generation); } return self.quarantine( - &raw, + raw, QuarantineReason::SemanticInvalid, "L2 canonical projection failed strict validation", materialized_at_ns, @@ -1318,11 +1368,11 @@ impl RealtimeCore { } }; let materialized_snapshot = - self.due_l2_materialized_snapshot(binding_key, l2, &raw, &transition); + self.due_l2_materialized_snapshot(binding_key, l2, raw, &transition); let mut publications = Vec::new(); if let Some(canonical) = canonical { let snapshot_observed_at_ms = if transition.publication == BookPublication::Snapshot { - l2_observed_at_ms(&transition, &raw) + l2_observed_at_ms(&transition, raw) } else { None }; @@ -1331,14 +1381,14 @@ impl RealtimeCore { if let Some(snapshot_transition) = materialized_snapshot { let Some(snapshot_partition_sequence) = partition_sequence.checked_add(1) else { return self.quarantine( - &raw, + raw, QuarantineReason::SemanticInvalid, "L2 materialized snapshot partition sequence overflow", materialized_at_ns, ); }; let canonical = match canonicalize_l2_publication( - &l2_fixture(binding, &raw, fixture_payload, snapshot_partition_sequence), + &l2_fixture(binding, raw, fixture_payload, snapshot_partition_sequence), &snapshot_transition, ) { Ok(Some(value)) => value, @@ -1347,7 +1397,7 @@ impl RealtimeCore { adapter.request_resync(raw.connection_generation); } return self.quarantine( - &raw, + raw, QuarantineReason::SemanticInvalid, "L2 materialized snapshot failed strict canonical projection", materialized_at_ns, @@ -1854,6 +1904,34 @@ mod tests { } } + fn assert_book_fence(batch: &ProcessBatch) { + use qdl_contracts::qdl::common::v1::QualityFlag; + assert_eq!(batch.canonical.len(), 2); + let events: Vec<_> = batch.canonical.iter().map(|record| + EventEnvelope::decode(record.payload.as_slice()).unwrap()).collect(); + assert_eq!(events[1].partition_sequence, events[0].partition_sequence + 1); + for event in &events { + assert!(event.quality_flags.contains(&(QualityFlag::SequenceGapBefore as i32))); + assert!(event.quality_flags.contains(&(QualityFlag::ResyncRequired as i32))); + assert!(!event.raw_capture_id.is_empty()); + assert_eq!(event.source_event_time_ns, event.received_at_ns / 1_000_000 * 1_000_000); + match event.payload.as_ref().unwrap() { + event_envelope::Payload::BookSnapshot(book) => { + assert!(book.levels.is_empty()); + assert!(!book.sequence_verified); + assert_eq!(book.book_generation, event.connection_generation); + } + event_envelope::Payload::BookDelta(book) => { + assert!(book.updates.is_empty()); + assert!(book.reset); + assert!(!book.sequence_verified); + assert_eq!(book.book_generation, event.connection_generation); + } + _ => panic!("book fence must not publish any price product"), + } + } + } + fn core(binding: CoreBinding, allow_test: bool) -> RealtimeCore { core_many(vec![binding], allow_test) } @@ -2280,7 +2358,7 @@ mod tests { 20, ) .unwrap(); - assert!(buffered.canonical.is_empty()); + assert_book_fence(&buffered); assert_ne!( buffered.filtered_outcome, Some("IGNORED_STALE_GENERATION"), @@ -2397,7 +2475,7 @@ mod tests { 10, ) .unwrap(); - assert!(buffered.canonical.is_empty()); + assert_book_fence(&buffered); assert_eq!(buffered.filtered, 1); let ready = core @@ -2536,7 +2614,7 @@ mod tests { 11, ) .unwrap(); - assert!(buffered.canonical.is_empty()); + assert_book_fence(&buffered); let initial = core .process( @@ -2936,7 +3014,7 @@ mod tests { now, ) .unwrap(); - assert!(gap.canonical.is_empty()); + assert_book_fence(&gap); assert_eq!(gap.quarantines.len(), 1); assert!(core.l2_adapters[&binding.key()].core().view().is_none()); let after = core @@ -2963,6 +3041,32 @@ mod tests { } } + #[test] + fn same_generation_recovery_republishes_deduplicated_anchor() { + let binding = okx_book_binding(); + let mut engine = core(binding.clone(), true); + let at = 1_786_352_400_000_000_000; + let original = engine.process_at_transport_offset( + delayed_book_frame(&binding, true, 100, 0, 7, at, "1"), at, 0).unwrap(); + let gap = engine.process_at_transport_offset( + delayed_book_frame(&binding, false, 110, 109, 7, at + 1_000_000, "2"), at, 1).unwrap(); + assert_book_fence(&gap); + let recovery = delayed_book_frame(&binding, true, 100, 0, 7, at + 2_000_000, "1"); + let restored = engine.process_at_transport_offset(recovery.clone(), at, 2).unwrap(); + assert_eq!(restored.canonical.len(), 1, "recovery anchor must replace fence"); + assert_ne!(restored.canonical[0].event_id, original.canonical[0].event_id); + let event = EventEnvelope::decode(restored.canonical[0].payload.as_slice()).unwrap(); + match event.payload.unwrap() { + event_envelope::Payload::BookSnapshot(book) => { + assert_eq!(book.native_sequence, "100"); + assert_eq!(book.book_generation, 7); + assert!(book.sequence_verified); + } + _ => panic!("recovery requires snapshot"), + } + assert!(engine.process_at_transport_offset(recovery, at, 2).unwrap().canonical.is_empty()); + } + #[test] fn delayed_l2_invalid_level_still_invalidates_verified_state() { for binding in [okx_book_binding(), binance_book_binding()] { @@ -2981,7 +3085,7 @@ mod tests { now, ) .unwrap(); - assert!(invalid.canonical.is_empty()); + assert_book_fence(&invalid); assert_eq!(invalid.quarantines.len(), 1); assert!(core.l2_adapters[&binding.key()].core().view().is_none()); let fresh = core @@ -3012,7 +3116,7 @@ mod tests { now, ) .unwrap(); - assert!(unanchored.canonical.is_empty()); + assert_book_fence(&unanchored); assert!(core.l2_adapters[&binding.key()].core().view().is_none()); let anchor = core .process( @@ -3175,7 +3279,7 @@ mod tests { let gap = engine .process(hot_delta(&binding, 104, 1200, "60000.5", "60002"), 5) .unwrap(); - assert!(gap.canonical.is_empty()); + assert_book_fence(&gap); assert_eq!(gap.quarantines.len(), 1); } } diff --git a/scripts/p184_dr05_case.py b/scripts/p184_dr05_case.py new file mode 100644 index 00000000..52e5719a --- /dev/null +++ b/scripts/p184_dr05_case.py @@ -0,0 +1,237 @@ +"""DR05 isolated consumer packet and bounded case APIs; never acceptance evidence. + +reset_case runs on the Docker host before TS clones start, preserving the owned +broker and long-lived JSONL channel. The channel requests reset through bounded +file IPC; no Docker socket enters a container. canonical_observe runs inside the +attested broker namespace. Callers must serialize all case operations. +""" +from __future__ import annotations + +import hashlib +import json +import math +import os +from pathlib import Path +import re +import time + +from scripts import p184_dr05_dl_fault as dr + + +def consumer_packet(manifest_path, catalog, venue, issuer, audience): + from qdl.consumer.manifest import ConsumerManifestLoader + from qdl.consumer.realtime_route import requirement_key + from qdl.consumer.universal_release import ( + ConsumerRequestPolicy, ConsumerRouteBinding, UniversalConsumerClass, + UniversalReleaseProduct, UniversalReleasePolicy, + ) + + manifest = ConsumerManifestLoader.load(manifest_path) + label = venue.lower() + cid = manifest.consumer_id + dr.namespace(cid.removesuffix("." + label)) + if venue not in dr.VENUES or not cid.endswith("." + label): + raise ValueError("isolated venue consumer required") + policy = UniversalReleasePolicy.load(dr.ROOT / "config/v2/universal-release-policy.yaml", manifest_root=dr.ROOT) + # A test-only routing generation, not a production release certificate. + digest = hashlib.sha256(Path(manifest_path).read_bytes()).hexdigest() + products, policies = [], [] + for req in manifest.requirements: + instrument = catalog.instrument_for(req.instrument_uid) + identity = instrument.identity + if identity.venue != venue or req.feed.value not in ("BOOK_SNAPSHOT", "BOOK_DELTA"): + raise ValueError("isolated venue BOOK requirements only") + rid = hashlib.sha256(f"{cid}:{requirement_key(req)}".encode()).hexdigest() + products.append(UniversalReleaseProduct( + consumer_id=cid, consumer_class=UniversalConsumerClass.TRADING_SYSTEM, + requirement_id=rid, instrument_uid=identity.instrument_uid, + instrument_id=identity.instrument_id, venue=venue, market=identity.market, + product_type=identity.product_type.value, native_symbol=instrument.native_symbol, + feed=req.feed.value, interval=req.interval, source_policy_id=req.source_policy_id, + provider_plane="L2", max_freshness_ms=req.max_freshness_ms, + require_final_bars=req.require_final_bars, require_live=True, + execution_grade=req.consumer_grade.value == "EXECUTION", route="V2_PRIMARY", + fallback="BLOCKED", fallback_rule_id=None, blocked_reason="TEST_ONLY_NO_V1")) + recency = req.event_recency_policy + policies.append(ConsumerRequestPolicy(rid, None if recency is None else recency.value, + req.max_session_liveness_ms)) + binding = ConsumerRouteBinding( + consumer_id=cid, consumer_class=UniversalConsumerClass.TRADING_SYSTEM, + release_revision=manifest.manifest_revision, universal_manifest_sha256=digest, + policy_sha256=policy.policy_sha256, capability_matrix_sha256=policy.capability_matrix_sha256, + capability_matrix_revision=policy.capability_matrix_revision, inventory_sha256=digest, + v1_rollback=policy.v1_rollback, independent_v1_venues=("DNSE",), + products=tuple(sorted(products, key=lambda p: p.requirement_id)), + consumer_manifest_revision=manifest.manifest_revision, + request_policies=tuple(sorted(policies, key=lambda p: p.requirement_id))) + sources = {p.source_policy_id for p in products} + if len(products) != 2 or len(sources) != 1: + raise ValueError("exact BOOK snapshot/delta source policy required") + return {"schema": "p184.dr05.ts-consumer.v1", "venue": venue, + "consumer_id": cid, "subject": "spiffe://qdl/paper/" + cid, + "jwt_private_key_file": "/work/" + label + ".key", "jwt_key_id": label, + "issuer": issuer, "audience": audience, "environment": "paper", + "manifest_revision": manifest.manifest_revision, + "query_url": "https://localhost:8200", "grpc_target": "localhost:8210", + "tls_ca_file": "/work/ca.crt", "tls_cert_file": "/work/client.crt", + "tls_key_file": "/work/client.key", "binding_file": "/work/" + label + ".binding.json", + "release_manifest_sha256": digest, "execution_book_source_policy_id": sources.pop()}, binding.canonical_mapping() + + +def _deadline(timeout): + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not math.isfinite(timeout) or not 0 < timeout <= 120: + raise ValueError("deadline must be in (0, 120] seconds") + return time.monotonic() + timeout + + +def _remaining(deadline): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("DR05 case deadline") + return remaining + + +def _packet(work): + work = Path(work).resolve() + handoff = json.loads((work / "handoff.json").read_text()) + packet = json.loads((work / "packet.json").read_text()) + config = json.loads((work / "core.json").read_text()) + ns = dr.namespace(handoff["namespace"]) + if (handoff.get("provenance") != "TEST_ONLY" or handoff.get("material_ready") is not True + or packet.get("namespace") != ns or packet.get("provenance") != "TEST_ONLY" + or packet.get("bootstrap") != "localhost:9092" + or packet.get("client_tls") != {"ca": "/work/ca.crt", "cert": "/work/client.crt", "key": "/work/client.key"} + or config.get("raw_topics") != [ns + ".raw"] + or config.get("core", {}).get("canonical_stream") != ns + ".canonical" + or config["core"].get("quarantine_stream") != ns + ".quarantine" + or config["core"].get("allow_test_provenance") is not True): + raise ValueError("isolated packet/config identity mismatch") + return work, ns, packet, config + + +def _reset_preflight(work, ns, call, core_binary=None, *, allow_channel=False): + core_binary = dr.core_override(core_binary) + rows = {role: dr.owned(ns, role, call) for role in (*dr.ROLES, "helper", "tsvalidator")} + if rows["tsvalidator"] or (rows["helper"] and not allow_channel): + raise ValueError("stop helpers and TS clones before reset") + if allow_channel: + channel = rows["helper"] + if (not channel or not channel.get("State", {}).get("Running") + or "P184_DR05_ISOLATED_NAMESPACE=" + ns not in channel["Config"].get("Env", []) + or "scripts/p184_dr05_channel.py" not in channel["Config"].get("Cmd", []) + or "--channel" not in channel["Config"].get("Cmd", [])): + raise ValueError("exact owned attested JSONL channel required") + broker = rows["kafka"] + if any(rows.values()) and broker is None: + raise ValueError("partial stack without owned broker; cleanup explicitly") + allowed = {row["Id"] for row in rows.values() if row} + for role, row in rows.items(): + if not row: + continue + host = row["HostConfig"] + expected = "none" if role == "kafka" else "container:" + broker["Id"] + mounts = row.get("Mounts", []) + binds = {m["Destination"]: m["Source"] for m in mounts if m["Type"] == "bind"} + expected_binds = {"/work": str(work), "/app": str(dr.ROOT)} + if role == "core" and core_binary: + expected_binds["/usr/local/bin/qdl-realtime-core"] = core_binary["path"] + if any(m.get("RW") is not False for m in mounts if m["Destination"] == "/usr/local/bin/qdl-realtime-core"): + raise ValueError("candidate core binary mount must be read-only") + if (row["Image"] != dr.IMAGES[{"quota": "redis", "helper": "python"}.get(role, role)] + or host.get("AutoRemove") is not True or host.get("PortBindings") + or host.get("NetworkMode") not in (expected, "container:" + ns + "-kafka" if role != "kafka" else expected) + or host.get("ReadonlyRootfs") is not True + or binds != expected_binds + or any(m["Type"] not in ("bind", "tmpfs") for m in mounts) + or (role == "kafka" and "/data" not in host.get("Tmpfs", {}))): + raise ValueError("reset requires exact disposable images, mounts and tmpfs broker") + # Read only IDs/network modes, never shared container environments or secrets. + ids = call(["docker", "ps", "-aq"]).split() + if ids: + modes = call(["docker", "inspect", "--format", "{{.Id}} {{.HostConfig.NetworkMode}}", *ids]) + targets = {"container:" + ns + "-kafka"} + if broker: + targets.add("container:" + broker["Id"]) + for line in modes.splitlines(): + ident, mode = line.split(maxsplit=1) + if mode in targets and ident not in allowed: + raise ValueError("unowned container or TS clone still attached; reset refused") + return broker + + +def reset_case(work, venue, timeout=30): + """Host-only bounded reset preserving the broker and JSONL network namespace.""" + from scripts.p184_dr05_reset import reset_preserving_broker + return reset_preserving_broker(work, venue, timeout) + + +def canonical_observe(work, raw_receipt, timeout=10): + """Verify actual raw bytes then delegate committed canonical/quarantine barrier.""" + deadline = _deadline(timeout) + _, ns, packet, config = _packet(work) + if os.environ.get("P184_DR05_ISOLATED_NAMESPACE") != ns: + raise ValueError("disposable namespace attestation required") + r = raw_receipt + if (not isinstance(r, dict) or r.get("topic") != ns + ".raw" + or type(r.get("partition")) is not int or r["partition"] != 0 + or type(r.get("offset")) is not int or r["offset"] < 0 + or r.get("provenance") != "TEST_ONLY" + or not re.fullmatch("[0-9a-f]{32}", str(r.get("capture_id", ""))) + or not re.fullmatch("[0-9a-f]{64}", str(r.get("raw_sha256", "")))): + raise ValueError("isolated raw receipt coordinates/hash required") + from confluent_kafka import Consumer, TopicPartition + from qdl.provider.v1.raw_provider_pb2 import RawProviderEnvelope + from scripts.p184_dr08_isolated import kafka_security + from scripts.p184_dr05_observer import CanonicalObserver + + reader = Consumer({**kafka_security(packet), "group.id": ns + ".receipt", + "enable.auto.commit": False, "isolation.level": "read_committed", + "fetch.wait.max.ms": 10}) + try: + reader.assign([TopicPartition(r["topic"], 0, r["offset"])]) + message = reader.poll(_remaining(deadline)) + if message is None: + raise TimeoutError("raw receipt readback deadline") + if message.error(): + raise RuntimeError("raw receipt readback failed") + wire = message.value() + raw = RawProviderEnvelope.FromString(wire) + if ((message.topic(), message.partition(), message.offset()) != (r["topic"], 0, r["offset"]) + or hashlib.sha256(wire).hexdigest() != r["raw_sha256"] + or raw.capture_id.hex() != r["capture_id"] or not raw.test_provenance): + raise ValueError("raw receipt does not match committed test bytes") + finally: + reader.close() + observer = CanonicalObserver(packet, config) + + class BoundedConsumer: + def __init__(self, consumer): + self.consumer = consumer + + def __getattr__(self, key): + return getattr(self.consumer, key) + + def poll(self, timeout): + return self.consumer.poll(min(timeout, _remaining(deadline))) + + def committed(self, partitions, timeout): + return self.consumer.committed(partitions, timeout=min(timeout, _remaining(deadline))) + + observer.consumer = BoundedConsumer(observer.consumer) + observer.offset_reader = BoundedConsumer(observer.offset_reader) + try: + result = observer.observe(r, timeout=_remaining(deadline)) + finally: + observer.close() + if (result["raw_committed_next"] <= r["offset"] + or set(result["read_committed_eof"]) != set(observer.topics)): + raise RuntimeError("committed canonical barrier missing") + matched = result["matched"] + records = [dict(row) for row in matched] + if any(row.get("partition") != 0 for row in records): + raise RuntimeError("canonical observer partition missing or unexpected") + return {"observed": True, "raw": {k: r[k] for k in ("topic", "partition", "offset", "capture_id")}, + "raw_committed_next": result["raw_committed_next"], + "read_committed_eof": result["read_committed_eof"], "records": records, + "outcome": "COMMITTED_OUTPUT" if records else "FILTERED_AFTER_COMMIT", + "provenance": "TEST_ONLY", "acceptance": False} diff --git a/scripts/p184_dr05_channel.py b/scripts/p184_dr05_channel.py new file mode 100644 index 00000000..a46421bd --- /dev/null +++ b/scripts/p184_dr05_channel.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +"""DR05 JSONL channel inside the owned broker namespace; no Docker access. + +Host watcher owns reset. IPC is control/reset-request.json -> +control/reset-.json. Run --channel /work with the isolated namespace +attestation. stdout contains protocol replies only; failures never echo inputs. +""" +from __future__ import annotations + +import argparse +import contextlib +import fcntl +import json +import math +import os +from pathlib import Path +import re +import stat +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.p184_dr05_dl_fault import FAULTS, VENUES + +MAX_BYTES = 4 * 1024 * 1024 +MAX_RECEIPTS = 64 +FRAME_FIELDS = {"venue", "native_symbol", "connection_generation", "sequence", "previous", + "snapshot", "transport", "received_at_ns", "test_provenance", "provenance", + "adapter_version", "payload"} +RECEIPT_FIELDS = {"topic", "partition", "offset", "raw_sha256", "capture_id", "generation", + "broker_ack_ms", "provenance", "acceptance"} + + +def _pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + +def decode(value): + def invalid(_value): + raise ValueError("non-finite JSON number") + return json.loads(value, object_pairs_hook=_pairs, parse_constant=invalid) + + +def _remaining(deadline): + value = deadline - time.monotonic() + if value <= 0: + raise TimeoutError("channel deadline") + return value + + +def _timeout(value): + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or not 0 < value <= 60: + raise ValueError("timeout must be in (0, 60]") + return value + + +def _read(path): + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as handle: + if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): + raise ValueError("regular control response required") + data = handle.read(MAX_BYTES + 1) + if len(data) > MAX_BYTES: + raise ValueError("oversize control response") + return decode(data) + + +def _atomic(path, document): + temporary = path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp") + data = json.dumps(document, allow_nan=False, separators=(",", ":")).encode() + if len(data) > MAX_BYTES: + raise ValueError("oversize control request") + try: + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def load_packet(work): + from scripts.p184_dr05_case import _packet + work, ns, packet, config = _packet(work) + if os.environ.get("P184_DR05_ISOLATED_NAMESPACE") != ns: + raise ValueError("isolated namespace attestation required") + topic = packet.get("slice", {}).get("topic_id") + if not isinstance(topic, str) or not topic or len(topic) > 128: + raise ValueError("broker-assigned canonical topic UUID required") + return work, ns, packet, config + + +class Channel: + def __init__(self, work, *, loader=load_packet, publisher_factory=None, observer_factory=None): + from scripts.p184_dr05_raw_publisher import RawPublisher + from scripts.p184_dr05_observer import CanonicalObserver + self.loader = loader + self.publisher_factory = publisher_factory or RawPublisher + self.observer_factory = observer_factory or CanonicalObserver + self.work, self.ns, self.packet, self.config = loader(work) + self.publisher = self.observer = None + self.receipts, self.observations = {}, {} + self.pending = None + self.poisoned = False + self.control = self.work / "control" + if self.control.is_symlink(): + raise ValueError("control directory may not be a symlink") + self.control.mkdir(mode=0o700, exist_ok=True) + fd = os.open(self.control / "channel.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + self.lock = os.fdopen(fd, "a") + try: + fcntl.flock(self.lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if (self.control / "reset-request.json").exists(): + raise ValueError("unresolved reset request; host reconciliation required") + self._open_clients() + except BaseException: + self.lock.close() + raise + + def _open_clients(self): + self.observer = self.observer_factory(self.packet, self.config) + try: + self.publisher = self.publisher_factory(self.packet, self.config) + except BaseException: + self.observer.close() + self.observer = None + raise + + def _close_clients(self): + observer, publisher = self.observer, self.publisher + self.observer = self.publisher = None + try: + if observer is not None: + observer.close() + finally: + if publisher is not None: + # RawPublisher has no close method. Acknowledged single-flight + # publication leaves no queued records; never flush/retry here. + del publisher + + def close(self): + try: + self._close_clients() + finally: + self.lock.close() + + def reset(self, venue, fault, deadline): + if venue not in VENUES or fault not in FAULTS: + raise ValueError("unknown cell") + request_id = uuid.uuid4().hex + request = self.control / "reset-request.json" + response = self.control / ("reset-" + request_id + ".json") + old_topic = self.packet["slice"]["topic_id"] + self._close_clients() + try: + if request.exists(): + raise ValueError("one pending reset only") + _atomic(request, {"request_id": request_id, "namespace": self.ns, + "venue": venue, "fault": fault, "timeout": _remaining(deadline)}) + while True: + _remaining(deadline) + try: + reply = _read(response) + break + except FileNotFoundError: + time.sleep(min(.05, _remaining(deadline))) + if not isinstance(reply, dict) or type(reply.get("ok")) is not bool: + raise ValueError("invalid host reset response") + if reply["ok"] is not True: + raise RuntimeError("host reset refused") + if set(reply) != {"ok", "result"} or not isinstance(reply["result"], dict): + raise ValueError("invalid host reset result") + work, ns, packet, config = self.loader(self.work) + if work != self.work or ns != self.ns or packet["slice"]["topic_id"] == old_topic: + raise ValueError("reset did not establish a fresh topic UUID") + result = reply["result"] + if result.get("namespace", self.ns) != self.ns or result.get("topic_id", packet["slice"]["topic_id"]) != packet["slice"]["topic_id"]: + raise ValueError("reset result identity mismatch") + _remaining(deadline) + self.packet, self.config = packet, config + self._open_clients() + self.receipts.clear() + self.observations.clear() + self.pending = None + # Host may already have consumed the request. Never erase another ID. + if request.exists(): + if _read(request).get("request_id") != request_id: + raise ValueError("reset request replaced unexpectedly") + request.unlink() + response.unlink() + return result + except BaseException: + self.poisoned = True + raise + + def publish(self, frame, deadline): + if self.pending is not None or len(self.receipts) >= MAX_RECEIPTS: + raise ValueError("observe pending publication or reset case first") + if not isinstance(frame, dict) or set(frame) != FRAME_FIELDS: + raise ValueError("exact frame fields required") + if (frame["venue"] not in VENUES or frame["provenance"] != "TEST_ONLY" + or frame["test_provenance"] is not True or type(frame["snapshot"]) is not bool + or not isinstance(frame["payload"], dict) + or any(type(frame[k]) is not int or frame[k] < 0 for k in + ("connection_generation", "sequence", "previous", "received_at_ns")) + or frame["connection_generation"] == 0 or frame["received_at_ns"] == 0): + raise ValueError("invalid test frame") + try: + receipt = self.publisher.publish(frame, timeout=min(5, _remaining(deadline))) + self._validate_receipt(receipt) + capture = receipt["capture_id"] + if capture in self.receipts: + raise ValueError("duplicate capture identity") + self.receipts[capture] = dict(receipt) + self.pending = capture + return receipt + except BaseException: + self.poisoned = True + raise + + def _validate_receipt(self, receipt): + if (not isinstance(receipt, dict) or set(receipt) != RECEIPT_FIELDS + or receipt["topic"] != self.ns + ".raw" + or type(receipt["partition"]) is not int or receipt["partition"] != 0 + or type(receipt["offset"]) is not int or receipt["offset"] < 0 + or type(receipt["generation"]) is not int or receipt["generation"] < 1 + or isinstance(receipt["broker_ack_ms"], bool) + or not isinstance(receipt["broker_ack_ms"], (int, float)) + or not math.isfinite(receipt["broker_ack_ms"]) or receipt["broker_ack_ms"] < 0 + or receipt["provenance"] != "TEST_ONLY" or receipt["acceptance"] is not False + or not re.fullmatch("[0-9a-f]{32}", str(receipt["capture_id"])) + or not re.fullmatch("[0-9a-f]{64}", str(receipt["raw_sha256"]))): + raise ValueError("invalid broker receipt") + + def observe(self, receipt, deadline): + self._validate_receipt(receipt) + capture = receipt["capture_id"] + if self.receipts.get(capture) != receipt: + raise ValueError("receipt was not published by this case/channel") + if capture in self.observations: + return self.observations[capture] + observer = self.observer + consumer, offsets = observer.consumer, observer.offset_reader + + class BoundedConsumer: + def __init__(self, target): + self.target = target + + def __getattr__(self, key): + return getattr(self.target, key) + + def poll(self, timeout): + return self.target.poll(min(timeout, _remaining(deadline))) + + def committed(self, partitions, timeout): + return self.target.committed(partitions, timeout=min(timeout, _remaining(deadline))) + + try: + observer.consumer, observer.offset_reader = BoundedConsumer(consumer), BoundedConsumer(offsets) + result = observer.observe(receipt, timeout=_remaining(deadline)) + expected = {self.ns + ".canonical", self.ns + ".quarantine"} + if (type(result.get("raw_committed_next")) is not int or result["raw_committed_next"] <= receipt["offset"] + or set(result.get("read_committed_eof", [])) != expected): + raise ValueError("committed observation barrier missing") + records = result["matched"] + if not isinstance(records, list) or len(records) > 64: + raise ValueError("unbounded observation") + for row in records: + if (row.get("capture_id") != capture or row.get("topic") not in expected + or type(row.get("partition")) is not int or row["partition"] != 0 + or type(row.get("offset")) is not int or row["offset"] < 0 + or row.get("kind") not in ("canonical", "quarantine") + or row["topic"] != self.ns + "." + row["kind"] + or (row["kind"] == "quarantine" and not row.get("reason"))): + raise ValueError("canonical coordinate/reason mismatch") + output = {"observed": True, "records": records, + "raw": {k: receipt[k] for k in ("topic", "partition", "offset", "capture_id")}, + "raw_committed_next": result["raw_committed_next"], + "read_committed_eof": result["read_committed_eof"], "provenance": "TEST_ONLY", "acceptance": False} + if len(json.dumps(output).encode()) > MAX_BYTES - 128: + raise ValueError("oversize observation") + self.observations[capture] = output + self.pending = None + return output + except BaseException: + self.poisoned = True + raise + finally: + observer.consumer, observer.offset_reader = consumer, offsets + + def dispatch(self, request): + if self.poisoned: + raise RuntimeError("channel fenced") + if not isinstance(request, dict) or set(request) != {"op", "args", "timeout"}: + raise ValueError("exact request fields required") + deadline = time.monotonic() + _timeout(request["timeout"]) + op, args = request["op"], request["args"] + fields = {"publish": {"frame"}, "reset_case": {"venue", "fault"}, "canonical_observe": {"receipt"}} + if not isinstance(op, str) or op not in fields or not isinstance(args, dict) or set(args) != fields[op]: + raise ValueError("invalid operation arguments") + if op == "reset_case": + return self.reset(args["venue"], args["fault"], deadline) + if op == "publish": + return self.publish(args["frame"], deadline) + return self.observe(args["receipt"], deadline) + + +def serve(channel, incoming, outgoing): + while True: + wire = incoming.readline(MAX_BYTES + 1) + if not wire: + return + oversize = len(wire) > MAX_BYTES + try: + if oversize: + channel.poisoned = True + raise ValueError("oversize request") + with contextlib.redirect_stdout(sys.stderr): + result = channel.dispatch(decode(wire)) + reply = {"ok": True, "result": result} + except Exception as error: + # Never echo input, native client diagnostics, keys or host errors. + reply = {"ok": False, "error": "DR05_CHANNEL_" + type(error).__name__} + outgoing.write(json.dumps(reply, allow_nan=False, separators=(",", ":")) + "\n") + outgoing.flush() + if channel.poisoned or oversize: + return + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--channel", required=True, type=Path) + args = parser.parse_args(argv) + channel = None + try: + with contextlib.redirect_stdout(sys.stderr): + channel = Channel(args.channel) + serve(channel, sys.stdin.buffer, sys.stdout) + return 1 if channel.poisoned else 0 + except Exception as error: + print(json.dumps({"ok": False, "error": "DR05_CHANNEL_" + type(error).__name__})) + return 1 + finally: + if channel is not None: + with contextlib.redirect_stdout(sys.stderr): + channel.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/p184_dr05_channel_smoke.py b/scripts/p184_dr05_channel_smoke.py new file mode 100644 index 00000000..9d8f0d1a --- /dev/null +++ b/scripts/p184_dr05_channel_smoke.py @@ -0,0 +1,116 @@ +"""Opt-in disposable JSONL handoff smoke; not TS acceptance evidence.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import selectors +import shutil +import subprocess +import sys +import tempfile +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts import p184_dr05_dl_fault as dr + + +def exercise(output, exported, core_binary): + output.mkdir(parents=True, exist_ok=False, mode=0o700) + receipt = {"schema": "p184.dr05.channel-smoke.v1", "acceptance": False, + "provenance": "TEST_ONLY", "resets": [], "steps": [], + "disk_free_before": shutil.disk_usage(output).free} + channel = watcher = None + passed = False + with tempfile.TemporaryDirectory(prefix="p184-dr05-channel-") as tmp: + work = Path(tmp) / "work" + ns = dr.init(work, exported)["namespace"] + receipt["namespace"] = ns + with (output / "channel.stderr").open("w") as errors, (output / "reset.stderr").open("w") as reset_errors: + try: + receipt["startup"] = dr.up(work, core_binary=core_binary) + argv = dr.helper_command(ns, work, ["--channel", "/work"], network="container:" + ns + "-kafka") + argv.insert(2, "-i") + argv[argv.index("scripts/p184_dr05_dl_fault.py")] = "scripts/p184_dr05_channel.py" + pos = argv.index(dr.IMAGES["python"]) + argv[pos:pos] = ["-e", "P184_DR05_ISOLATED_NAMESPACE=" + ns] + channel = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=errors, text=True, bufsize=1) + watcher = subprocess.Popen([sys.executable, "-B", str(ROOT / "scripts/p184_dr05_reset.py"), + "--serve-reset-host", str(work), "--duration", "300"], stdout=reset_errors, stderr=reset_errors) + + def rpc(op, args, timeout): + started = time.monotonic() + channel.stdin.write(json.dumps({"op": op, "args": args, "timeout": timeout}) + "\n") + channel.stdin.flush() + with selectors.DefaultSelector() as selector: + selector.register(channel.stdout, selectors.EVENT_READ) + if not selector.select(timeout + 2): + raise TimeoutError("JSONL reply deadline") + line = channel.stdout.readline() + if not line: + raise RuntimeError("JSONL channel exited") + result = json.loads(line) + if result.get("ok") is not True: + raise RuntimeError("channel reply " + str(result)) + return result["result"], (time.monotonic() - started) * 1000 + + for venue in dr.VENUES: + fault = "GAP_REFUSAL" + reset, elapsed = rpc("reset_case", {"venue": venue, "fault": fault}, 30) + receipt["resets"].append({**reset, "elapsed_ms": elapsed}) + for index, step in enumerate(dr.schedule(venue, fault)): + values = {k: v for k, v in step.items() if k != "phase"} + frame = dr.frame(**values, at_ns=time.time_ns()) + raw, publish_ms = rpc("publish", {"frame": frame}, 5) + observed, observe_ms = rpc("canonical_observe", {"receipt": raw}, 10) + if index == 0 and raw["offset"] != 0: + raise AssertionError("fresh raw topic did not start at zero") + if observed.get("observed") is not True: + raise AssertionError("actual committed observation missing") + receipt["steps"].append({"venue": venue, "phase": step["phase"], + "raw": raw, "observed": observed, "publish_ms": publish_ms, "observe_ms": observe_ms}) + if len({r["broker_id"] for r in receipt["resets"]}) != 1: + raise AssertionError("broker namespace changed") + if len({r["topic_id"] for r in receipt["resets"]}) != 2: + raise AssertionError("topic UUID reused") + passed = True + except Exception as error: + receipt["error"] = type(error).__name__ + ":" + str(error)[:1000] + finally: + if channel: + channel.stdin.close() + try: + channel.wait(timeout=10) + except subprocess.TimeoutExpired: + channel.terminate() + channel.wait(timeout=10) + if watcher: + watcher.terminate() + try: + watcher.wait(timeout=35) + except subprocess.TimeoutExpired: + watcher.kill() + watcher.wait(timeout=5) + try: + receipt["cleanup"] = dr.cleanup(ns) + finally: + dr.erase_private(work) + receipt["private_files_remaining"] = sum((work / f).exists() for f in dr.PRIVATE_FILES) + receipt.update(passed=passed, disk_free_after=shutil.disk_usage(output).free) + wire = json.dumps(receipt, indent=2) + "\n" + (output / "receipt.json").write_text(wire) + print(json.dumps({"passed": passed, "acceptance": False, "output": str(output), + "sha256": hashlib.sha256(wire.encode()).hexdigest(), "error": receipt.get("error")})) + return passed + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--exported", type=Path, required=True) + parser.add_argument("--core-binary", type=Path, required=True) + args = parser.parse_args() + raise SystemExit(0 if exercise(args.output, args.exported, args.core_binary) else 1) diff --git a/scripts/p184_dr05_dl_fault.py b/scripts/p184_dr05_dl_fault.py index a7f5f495..61043bcc 100644 --- a/scripts/p184_dr05_dl_fault.py +++ b/scripts/p184_dr05_dl_fault.py @@ -120,13 +120,29 @@ def drive(adapter, venue, fault): "observations": observations} -def topology(ns, work, topic_id=""): +def core_override(value): + """Pin an explicitly supplied test binary; never discover or build one.""" + if value is None: + return None + supplied = value if isinstance(value, dict) else {"path": str(value)} + path = Path(supplied["path"]).resolve(strict=True) + if not path.is_file() or not os.access(path, os.X_OK) or any(c in str(path) for c in (":", ",")): + raise ValueError("explicit executable core binary path required") + with path.open("rb") as handle: + digest = hashlib.file_digest(handle, "sha256").hexdigest() + if "sha256" in supplied and supplied["sha256"] != digest: + raise ValueError("candidate core binary digest changed") + return {"path": str(path), "sha256": digest} + + +def topology(ns, work, topic_id="", *, core_binary=None): """Generate executable Docker argv, but do not execute or discover config. Material preparation is a stage3 gate; topic UUID comes from the new broker. Query reuses DR08's generic real hot-read service, NOT its TRADE preparer. """ namespace(ns) + core_binary = core_override(core_binary) work = Path(work).resolve() if work == ROOT or ROOT in work.parents or ":" in str(work) or "," in str(work): raise ValueError("external unambiguous work directory required") @@ -169,7 +185,14 @@ def topology(ns, work, topic_id=""): "-v", str(work) + ":/work", "-v", str(ROOT) + ":/app:ro", "-w", "/app"] if role == "kafka": argv += ["--hostname", "localhost", "--tmpfs", "/data:rw,size=512m,mode=1777"] + # Override image-declared VOLUMEs: reset must never retain broker state. + for target in ("/etc/kafka/secrets", "/mnt/shared/config", "/var/lib/kafka/data"): + argv += ["--tmpfs", target + ":rw,size=1m,mode=1777"] env = {"KAFKA_HEAP_OPTS": "-Xms256m -Xmx512m", "LOG_DIR": "/tmp/kafka-logs"} + elif role == "quota": + argv += ["--tmpfs", "/data:rw,size=1m,mode=1777"] + if role == "core" and core_binary: + argv += ["-v", core_binary["path"] + ":/usr/local/bin/qdl-realtime-core:ro"] for key, value in env.items(): argv += ["-e", key + "=" + value] result[role] = argv + ["--entrypoint", command[0], IMAGES[image], *command[1:]] @@ -411,6 +434,10 @@ def save(name, value, private=False): mapping["spec"]["requirements"] = requirements mapping["spec"]["quotas"]["max_buffer_events"] = 1000 save(label + ".yaml", yaml.safe_dump(mapping)) + from scripts.p184_dr05_case import consumer_packet + consumer, route = consumer_packet(work / (label + ".yaml"), loaded, b["venue"], ISSUER, AUDIENCE) + save(label + ".binding.json", route) + save("ts-consumer." + label + ".json", consumer) manifests.append(work / (label + ".yaml")) jwt_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) save(label + ".key", jwt_key.private_bytes(pem, serialization.PrivateFormat.PKCS8, @@ -490,31 +517,43 @@ def init(work, export_path): return {"namespace": ns, "material_ready": True, "acceptance": False} -def up(work): +def up(work, timeout=120, *, core_binary=None): + deadline = time.monotonic() + timeout + + def run(argv, limit=120): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("isolated services startup deadline") + return bounded(argv, min(limit, remaining)) + work = Path(work).resolve() packet = json.loads((work / "handoff.json").read_text()) ns = namespace(packet["namespace"]) if not packet.get("material_ready"): raise ValueError("run --init with sanitized export first") - if any(owned(ns, role) is not None for role in (*ROLES, "helper")): + if any(owned(ns, role, run) is not None for role in (*ROLES, "helper")): raise ValueError("namespace already in use; no implicit restart") + selected_core = core_override(core_binary if core_binary is not None else packet.get("core_binary")) + if selected_core: + packet["core_binary"] = selected_core + (work / "handoff.json").write_text(json.dumps(packet, indent=2) + "\n") for digest in set(IMAGES.values()): - if sh(["docker", "image", "inspect", digest, "--format", "{{.Id}} "]).strip() != digest: + if run(["docker", "image", "inspect", digest, "--format", "{{.Id}} "]).strip() != digest: raise ValueError("local immutable image mismatch") try: - bounded(topology(ns, work)["kafka"]) - bounded(helper_command(ns, work, ["--topics", "/work"], network="container:" + ns + "-kafka")) + run(topology(ns, work, core_binary=selected_core)["kafka"]) + run(helper_command(ns, work, ["--topics", "/work"], network="container:" + ns + "-kafka")) topic = json.loads((work / "packet.json").read_text())["slice"]["topic_id"] - for role, argv in topology(ns, work, topic).items(): + for role, argv in topology(ns, work, topic, core_binary=selected_core).items(): if role != "kafka": - bounded(argv) - deadline = time.monotonic() + 45 + run(argv) + deadline = min(deadline, time.monotonic() + 45) while True: - result = readiness(ns) + result = readiness(ns, run) if result["process_isolation_ready"]: try: - bounded(helper_command(ns, work, ["--probe", "/work"], network="container:" + ns + "-kafka"), 25) - return {**result, "services_reachable": True, "acceptance": False} + run(helper_command(ns, work, ["--probe", "/work"], network="container:" + ns + "-kafka"), 25) + return {**result, "services_reachable": True, "acceptance": False, "core_binary": selected_core} except subprocess.CalledProcessError: pass if time.monotonic() >= deadline: @@ -553,6 +592,18 @@ async def connect(): return {"services_reachable": True, "functional_ready": False, "acceptance": False} +def reset_case(work, venue, timeout=30): + """Host-only reset; the owned JSONL helper may remain, TS clones must stop.""" + from scripts.p184_dr05_case import reset_case as reset + return reset(work, venue, timeout) + + +def canonical_observe(work, raw_receipt, timeout=10): + """Run inside the attested isolated broker namespace.""" + from scripts.p184_dr05_case import canonical_observe as observe + return observe(work, raw_receipt, timeout) + + def dispatch(argv=None): parser = argparse.ArgumentParser(description=__doc__) mode = parser.add_mutually_exclusive_group(required=True) diff --git a/scripts/p184_dr05_observe_stack.py b/scripts/p184_dr05_observe_stack.py index dedfeb47..d3d8a567 100644 --- a/scripts/p184_dr05_observe_stack.py +++ b/scripts/p184_dr05_observe_stack.py @@ -17,13 +17,15 @@ from scripts import p184_dr05_dl_fault as dr -def exercise(output, exported, venue, fault): +def exercise(output, exported, venue, fault, core_binary=None): + override = dr.core_override(core_binary) output = output.resolve() dr.topology("p184-dr05-dl-0123456789abcdef", output) output.mkdir(mode=0o700, parents=True, exist_ok=False) started, free_before = time.monotonic(), shutil.disk_usage(output).free receipt = {"schema": "p184.dr05.stack-observation.v1", "acceptance": False, - "venue": venue, "fault": fault, "images": dr.IMAGES, "source_files": {}} + "venue": venue, "fault": fault, "images": dr.IMAGES, "source_files": {}, + "test_core_binary": override} for name in ("p184_dr05_observer.py", "p184_dr05_raw_publisher.py", "p184_dr05_dl_fault.py"): receipt["source_files"][name] = hashlib.sha256((ROOT / "scripts" / name).read_bytes()).hexdigest() ok = False @@ -33,7 +35,7 @@ def exercise(output, exported, venue, fault): ns = initial["namespace"] receipt["namespace"] = ns try: - receipt["startup"] = dr.up(work) + receipt["startup"] = dr.up(work, core_binary=override) ts_image = "sha256:a6548ee36aeb0242c1734c071678c525c1506d05c10a70b533eb7110e7eaf16a" live_args = ["docker", "run", "-d", "--rm", "--pull=never", "--name", ns + "-tsvalidator", "--label", dr.LABEL + "=" + ns, "--network", "container:" + ns + "-kafka", "--read-only", @@ -107,8 +109,9 @@ def main(): parser.add_argument("--exported", type=Path, required=True) parser.add_argument("--venue", choices=dr.VENUES, required=True) parser.add_argument("--fault", choices=dr.FAULTS, required=True) + parser.add_argument("--core-binary", type=Path, help="isolated test-only compiled binary; no image build") args = parser.parse_args() - return 0 if exercise(args.output, args.exported, args.venue, args.fault) else 1 + return 0 if exercise(args.output, args.exported, args.venue, args.fault, args.core_binary) else 1 if __name__ == "__main__": diff --git a/scripts/p184_dr05_observer.py b/scripts/p184_dr05_observer.py index a712c411..d2301f44 100644 --- a/scripts/p184_dr05_observer.py +++ b/scripts/p184_dr05_observer.py @@ -23,11 +23,11 @@ from scripts.p184_dr05_raw_publisher import RawPublisher -def record_summary(topic, offset, wire, canonical): +def record_summary(topic, offset, wire, canonical, *, partition=0): from qdl.marketdata.v2.market_data_pb2 import EventEnvelope from qdl.provider.v1.raw_provider_pb2 import QuarantineRecord, QuarantineReason - row = {"topic": topic, "offset": offset, "sha256": hashlib.sha256(wire).hexdigest()} + row = {"topic": topic, "partition": partition, "offset": offset, "sha256": hashlib.sha256(wire).hexdigest()} if canonical: event = EventEnvelope.FromString(wire) row.update(kind="canonical", capture_id=event.raw_capture_id.hex(), @@ -79,7 +79,7 @@ def observe(self, receipt, timeout=8): eof.discard(message.topic()) self.positions[message.topic()] = message.offset() + 1 records.append(record_summary(message.topic(), message.offset(), message.value(), - message.topic() == self.canonical)) + message.topic() == self.canonical, partition=message.partition())) if len(records) > 64: raise RuntimeError("unexpected unbounded canonical output") if time.monotonic() >= next_check: diff --git a/scripts/p184_dr05_reset.py b/scripts/p184_dr05_reset.py new file mode 100644 index 00000000..ecf9f8d4 --- /dev/null +++ b/scripts/p184_dr05_reset.py @@ -0,0 +1,177 @@ +"""Bounded host reset bridge for the disposable DR05 JSONL channel. + +The broker/network namespace survives. Only owned readers/core/quota and the +three exact TEST_ONLY topics are reset. No Docker socket enters a container. +""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import re +import signal +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +from scripts import p184_dr05_dl_fault as dr +from scripts.p184_dr05_case import _deadline, _remaining, _packet, _reset_preflight + + +def reset_topics(work, timeout): + from confluent_kafka import KafkaError, KafkaException, TopicCollection + from confluent_kafka.admin import AdminClient, NewTopic + from scripts.p184_dr08_isolated import kafka_security + + deadline = _deadline(timeout) + work, ns, packet, config = _packet(work) + if os.environ.get("P184_DR05_ISOLATED_NAMESPACE") != ns: + raise ValueError("isolated namespace attestation required") + topics = [ns + suffix for suffix in (".raw", ".canonical", ".quarantine")] + admin = AdminClient(kafka_security(packet)) + for future in admin.delete_topics(topics, operation_timeout=min(5, _remaining(deadline))).values(): + future.result(_remaining(deadline)) + while any(name in admin.list_topics(timeout=min(2, _remaining(deadline))).topics for name in topics): + time.sleep(min(.1, _remaining(deadline))) + for future in admin.delete_consumer_groups([ns, "kn-" + ns], request_timeout=_remaining(deadline)).values(): + try: + future.result(_remaining(deadline)) + except KafkaException as error: + if error.args[0].code() != KafkaError.GROUP_ID_NOT_FOUND: + raise + for future in admin.create_topics([NewTopic(name, 1, 1) for name in topics], + operation_timeout=min(5, _remaining(deadline))).values(): + future.result(_remaining(deadline)) + desc = admin.describe_topics(TopicCollection([ns + ".canonical"]), request_timeout=_remaining(deadline)) + new_id = str(desc[ns + ".canonical"].result(_remaining(deadline)).topic_id) + old_id = packet["slice"]["topic_id"] + if not new_id or new_id == old_id: + raise RuntimeError("topic incarnation did not change") + packet["slice"]["topic_id"] = new_id + temp = work / "packet.reset.tmp" + temp.write_text(json.dumps(packet)) + temp.replace(work / "packet.json") + return {"topic_id": new_id, "previous_topic_id": old_id, "topics": topics} + + +def reset_preserving_broker(work, venue, timeout=30): + deadline = _deadline(timeout) + if venue not in dr.VENUES: + raise ValueError("unknown venue") + work, ns, packet, _ = _packet(work) + selected = dr.core_override(json.loads((work / "handoff.json").read_text()).get("core_binary")) + + def call(argv): + return dr.bounded(argv, min(15, _remaining(deadline))) + + channel = dr.owned(ns, "helper", call) + broker = _reset_preflight(work, ns, call, selected, allow_channel=channel is not None) + if not broker: + raise ValueError("running owned broker required") + rows = [dr.owned(ns, role, call) for role in ("query", "stream", "core", "quota")] + if not all(rows): + raise ValueError("complete owned stack required before reset") + ids = [row["Id"] for row in rows] + call(["docker", "stop", "--time", "1", *ids]) + while any(dr.owned(ns, role, call) for role in ("query", "stream", "core", "quota")): + time.sleep(min(.1, _remaining(deadline))) + + def inside(script, args): + if channel: + argv = ["docker", "exec", ns + "-helper", "python", "-B", script, *args] + else: + argv = dr.helper_command(ns, work, args, network="container:" + ns + "-kafka") + argv[argv.index("scripts/p184_dr05_dl_fault.py")] = script + pos = argv.index(dr.IMAGES["python"]) + argv[pos:pos] = ["-e", "P184_DR05_ISOLATED_NAMESPACE=" + ns] + return call(argv) + + topic = json.loads(inside("scripts/p184_dr05_reset.py", + ["--topics", "/work", "--timeout", str(_remaining(deadline))])) + (work / "query-handoffs.jsonl").unlink(missing_ok=True) + commands = dr.topology(ns, work, topic["topic_id"], core_binary=selected) + for role in ("quota", "core", "stream", "query"): + call(commands[role]) + while True: + _remaining(deadline) + try: + inside("scripts/p184_dr05_dl_fault.py", ["--probe", "/work"]) + break + except __import__("subprocess").CalledProcessError: + time.sleep(min(.2, _remaining(deadline))) + current = dr.owned(ns, "kafka", call) + if not current or current["Id"] != broker["Id"]: + raise RuntimeError("broker namespace changed during reset") + return {"namespace": ns, "venue": venue, "generation": 7, "reset": True, + "broker_id": current["Id"], "topic_id": topic["topic_id"], + "previous_topic_id": topic["previous_topic_id"], "acceptance": False, + "core_binary": selected, "services_reachable": True} + + +def serve_host(work, duration=1800): + work, ns, _, _ = _packet(work) + if not 0 < duration <= 3600: + raise ValueError("host watcher duration must be bounded") + control = work / "control" + if control.is_symlink(): + raise ValueError("control path cannot be a symlink") + control.mkdir(mode=0o700, exist_ok=True) + stopped = False + + def stop(*_): + nonlocal stopped + stopped = True + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + deadline = time.monotonic() + duration + seen = set() + while not stopped and time.monotonic() < deadline: + request = control / "reset-request.json" + if not request.exists(): + time.sleep(.05) + continue + if request.is_symlink() or request.stat().st_size > 8192: + raise ValueError("invalid reset request file") + row = json.loads(request.read_text()) + ident = row.get("request_id", "") + if (not re.fullmatch(r"[0-9a-f]{32}", ident) or ident in seen + or row.get("namespace") != ns or row.get("fault") not in dr.FAULTS + or set(row) != {"request_id", "namespace", "venue", "fault", "timeout"}): + raise ValueError("invalid reset identity/shape") + seen.add(ident) + if len(seen) > 16: + raise ValueError("reset attempt cap exceeded") + request.unlink() + try: + result = {"ok": True, "result": reset_preserving_broker(work, row["venue"], + min(float(row["timeout"]), deadline - time.monotonic()))} + except Exception as error: + result = {"ok": False, "error": type(error).__name__ + ":" + str(error)[:400]} + temp = control / (ident + ".tmp") + temp.write_text(json.dumps(result)) + temp.replace(control / ("reset-" + ident + ".json")) + if not result["ok"]: + return 1 + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--topics", type=Path) + group.add_argument("--serve-reset-host", type=Path) + parser.add_argument("--timeout", type=float, default=30) + parser.add_argument("--duration", type=float, default=1800) + args = parser.parse_args() + if args.topics: + print(json.dumps(reset_topics(args.topics, args.timeout))) + return 0 + return serve_host(args.serve_reset_host, args.duration) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_kn_hot_book_invalidation.py b/tests/test_kn_hot_book_invalidation.py new file mode 100644 index 00000000..cecaa548 --- /dev/null +++ b/tests/test_kn_hot_book_invalidation.py @@ -0,0 +1,167 @@ +"""TEST_ONLY DR05 book fences through the actual Python read path.""" +import base64 +import json +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import Mock + +import grpc +from google.protobuf.wrappers_pb2 import BytesValue +from qdl.common.v1 import common_pb2 as common +from qdl.marketdata.v2 import market_data_pb2 as md +from qdl.query import ConsumerGrade, DataRequirement +from qdl.query.contracts import CanonicalErrorCode +from qdl.query.results import QueryBackendError +from qdl.runtime.kn_bar_readback import binding_product_key +from qdl.runtime.kn_hot_view import CanonicalHotClient, HotViewGap, HotViewUnavailable, SCHEMA +from qdl.runtime.kn_market_cache import CacheRow, ProductView, SourceBoundary, KnCacheError, KnCacheNotReady +from qdl.runtime.kn_query_backend import KnMarketCacheQueryBackend +from qdl.runtime.stable_catalog import StableSourceCatalog + + +class Refused(grpc.RpcError): + def __init__(self, reason, code=grpc.StatusCode.FAILED_PRECONDITION): + self.reason, self.status = reason, code + + def code(self): + return self.status + + def details(self): + return self.reason + + +class BookInvalidationTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.catalog = StableSourceCatalog.load(Path(__file__).resolve().parents[1] / + "config/v2/stable-source-bindings.yaml") + cls.bindings = [b for b in cls.catalog.bindings if b.feed.value in + {"BOOK_SNAPSHOT", "BOOK_DELTA"} and + b.instrument.native_symbol in {"ETHUSDT", "ETH-USDT-SWAP"}] + assert len(cls.bindings) == 4 + + def setup_binding(self, binding): + self.binding = binding + self.lpk = binding_product_key(binding, "paper") + self.now = 1800000000000000000 + self.requirement = DataRequirement(instrument_uid=binding.instrument.instrument_uid, + feed=binding.feed, consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id=binding.source_policy_id, max_freshness_ms=1000) + self.hot = Mock() + self.reader = SimpleNamespace(environment="paper", latest=lambda key: self.view) + self.backend = KnMarketCacheQueryBackend(self.reader, self.catalog, schema_digest="a"*64, + topic_id="topic", clock_ns=lambda: self.now, hot_client=self.hot) + self.set_view(self.event(), 10) + + def event(self, fence=False): + b = self.binding + e = md.EventEnvelope(schema_major=2, event_id=b"e"*16, + instrument_uid=b.instrument.instrument_uid, instrument_id=b.instrument.instrument_id, + instrument_revision=b.instrument.metadata_revision, venue=b.instrument.identity.venue, + market=b.instrument.identity.market, product_type=b.instrument.identity.product_type.value, + native_symbol=b.instrument.native_symbol, provider=b.provider, source_id=b.source_id, + source_role=getattr(common, "SOURCE_ROLE_"+b.source_role), + authority_revision=self.catalog.authority_revision, adapter_version=b.adapter_version, + normalizer_version=b.normalizer_version, raw_capture_id=b"r"*16, raw_payload_hash=b"h"*32, + source_session_id="TEST_ONLY.session", connection_generation=2, lease_epoch=1, + partition_sequence=20, source_sequence="g2:invalid:20" if fence else "g2:42", + received_at_ns=self.now, normalized_at_ns=self.now, published_at_ns=self.now, + source_event_time_ns=self.now, correlation_id="TEST_ONLY.DR05") + book = getattr(e, b.feed.value.lower()) + book.book_generation = 2 + book.sequence_verified = not fence + if fence: + e.quality_flags.extend([common.QUALITY_FLAG_SEQUENCE_GAP_BEFORE, + common.QUALITY_FLAG_RESYNC_REQUIRED]) + if b.feed.value == "BOOK_DELTA": + book.reset = True + elif b.feed.value == "BOOK_SNAPSHOT": + book.native_sequence = "42" + book.depth = 1 + for side, price in ((common.BOOK_SIDE_BID, 100), (common.BOOK_SIDE_ASK, 101)): + level = book.levels.add(side=side, quantity_unit=common.QUANTITY_UNIT_BASE_ASSET) + level.price.CopyFrom(common.DecimalValue(mantissa=price, source_text=str(price))) + level.quantity.CopyFrom(common.DecimalValue(mantissa=1, source_text="1")) + else: + book.native_sequence_start = book.native_sequence_end = "42" + book.snapshot_sequence = "41" + return e + + def set_view(self, event, offset): + self.view = ProductView(self.lpk, 1, 1, (CacheRow(event.SerializeToString(), offset),), + SourceBoundary("topic", 0, offset)) + + def assert_gap(self, reason): + with self.assertRaises(QueryBackendError) as caught: + self.backend.latest(self.requirement) + self.assertEqual(caught.exception.problem.code, CanonicalErrorCode.OPEN_SEQUENCE_GAP) + self.assertIn(reason, caught.exception.problem.detail) + + def test_primary_empty_fence_and_valid_anchor_recovery(self): + for binding in self.bindings: + with self.subTest(binding=binding.binding_id): + self.setup_binding(binding) + self.assertTrue(self.backend.latest(self.requirement).quality.execution_eligible) + self.set_view(self.event(fence=True), 20) + self.assert_gap("BOOK_INVALIDATED_RESYNC_REQUIRED") + self.hot.latest.assert_not_called() + self.set_view(self.event(), 30) + self.assertTrue(self.backend.latest(self.requirement).quality.execution_eligible) + + def test_invalid_identity_and_unflagged_empty_book_are_not_typed_gap(self): + for binding in self.bindings: + for mutation in ("source_id", "raw_capture_id", "flags"): + with self.subTest(binding=binding.binding_id, mutation=mutation): + self.setup_binding(binding) + event = self.event(fence=True) + if mutation == "flags": + del event.quality_flags[:] + else: + setattr(event, mutation, "wrong" if mutation == "source_id" else b"") + self.set_view(event, 20) + with self.assertRaises(ValueError): + self.backend.latest(self.requirement) + self.hot.latest.assert_not_called() + + def test_rpc_gap_never_retries_replica_or_masks_with_primary_error(self): + for reason in ("HOT_RECORD_GAPPED", "HOT_RECORD_GENERATION_MISMATCH", "HOT_BOOK_UNVERIFIED"): + for primary in ("stale", "unavailable", "missing"): + with self.subTest(reason=reason, primary=primary): + self.setup_binding(self.bindings[0]) + self.now += 10000000000 + if primary == "missing": + self.reader.latest = Mock(side_effect=KnCacheNotReady(self.lpk, "MISSING")) + elif primary == "unavailable": + self.reader.latest = Mock(side_effect=KnCacheError("offline")) + second = Mock(side_effect=AssertionError("old healthy replica must not be read")) + client = CanonicalHotClient(calls=(Mock(side_effect=Refused(reason)), second), + expectation={}, secret=b"x"*32) + self.backend.hot_client = client + self.assert_gap(reason) + second.assert_not_called() + self.assertTrue(client._slots.acquire(False)) + client._slots.release() + + def test_hot_canonical_empty_fence_does_not_become_lineage_error(self): + for binding in self.bindings: + with self.subTest(binding=binding.binding_id): + self.setup_binding(binding) + self.now += 10000000000 + reply = dict(schema=SCHEMA, binding_id=binding.binding_id, + product_key=self.lpk.encode(), physical_key=binding.partition_key, + source_topic_id="topic", source_partition=0, source_offset=20, record_offset=20, + canonical=base64.b64encode(self.event(fence=True).SerializeToString()).decode()) + transport = Mock(return_value=BytesValue(value=json.dumps(reply).encode())) + self.backend.hot_client = CanonicalHotClient(calls=(transport,), + expectation={"source_topic_id":"topic"}, secret=b"x"*32) + self.assert_gap("BOOK_INVALIDATED_RESYNC_REQUIRED") + + def test_gap_text_under_auth_failure_is_not_trusted(self): + client = CanonicalHotClient(calls=(Mock(side_effect=Refused("HOT_RECORD_GAPPED", + grpc.StatusCode.PERMISSION_DENIED)),), expectation={}, secret=b"x"*32) + self.setup_binding(self.bindings[0]) + with self.assertRaises(HotViewUnavailable) as caught: + client.latest(self.binding, self.lpk) + self.assertNotIsInstance(caught.exception, HotViewGap) + self.assertEqual(str(caught.exception), "HOT_AUTHORITY_OR_PROTOCOL_REFUSED") diff --git a/tests/test_p184_dr05_case.py b/tests/test_p184_dr05_case.py new file mode 100644 index 00000000..c9a20fec --- /dev/null +++ b/tests/test_p184_dr05_case.py @@ -0,0 +1,276 @@ +import hashlib +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + +from scripts import p184_dr05_case as case +from scripts import p184_dr05_dl_fault as dr +from tests import test_p184_dr05_dl_fault as fixtures + +NS = "p184-dr05-dl-0123456789abcdef" +WORK = Path("/tmp/dr05-case-test") + + +def runtime_packet(): + return {"namespace": NS, "provenance": "TEST_ONLY", "bootstrap": "localhost:9092", + "client_tls": {"ca": "/work/ca.crt", "cert": "/work/client.crt", "key": "/work/client.key"}, + "slice": {"topic_id": "old-topic"}} + + +def config(): + return {"raw_topics": [NS + ".raw"], "core": {"canonical_stream": NS + ".canonical", + "quarantine_stream": NS + ".quarantine", "allow_test_provenance": True}} + + +def owned_rows(): + rows = {} + for role in dr.ROLES: + rows[role] = {"Id": role + "-id", "Image": dr.IMAGES[{"quota": "redis"}.get(role, role)], + "HostConfig": {"AutoRemove": True, "ReadonlyRootfs": True, "PortBindings": {}, + "NetworkMode": "none" if role == "kafka" else "container:kafka-id", + "Tmpfs": {"/data": "rw"} if role == "kafka" else {}}, + "Mounts": [{"Type": "bind", "Destination": "/work", "Source": str(WORK)}, + {"Type": "bind", "Destination": "/app", "Source": str(dr.ROOT)}]} + return rows + + +class PacketTests(unittest.TestCase): + def test_material_packet_binding_v3_parity(self): + from qdl.consumer.manifest import ConsumerManifestLoader + from qdl.consumer.universal_release import ConsumerRouteBinding + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "case" + dr.prepare(work) + dr.material(work, fixtures.MaterialTests.exported()) + for venue in dr.VENUES: + label = venue.lower() + spec = json.loads((work / f"ts-consumer.{label}.json").read_text()) + route = json.loads((work / f"{label}.binding.json").read_text()) + binding = ConsumerRouteBinding.from_canonical_mapping(route) + manifest = ConsumerManifestLoader.load(work / f"{label}.yaml") + self.assertEqual(spec["schema"], "p184.dr05.ts-consumer.v1") + self.assertEqual(spec["venue"], venue) + self.assertEqual(set(spec), {"schema", "venue", "consumer_id", "subject", + "query_url", "grpc_target", "tls_ca_file", "tls_cert_file", "tls_key_file", + "binding_file", "jwt_private_key_file", "jwt_key_id", "issuer", "audience", + "environment", "manifest_revision", "release_manifest_sha256", + "execution_book_source_policy_id"}) + self.assertEqual(len(binding.products), 2) + self.assertEqual(len(binding.request_policies), 2) + self.assertEqual(route["schema"], "qdl.v2.consumer-route-binding.v3") + self.assertEqual(spec["consumer_id"], manifest.consumer_id) + self.assertEqual(spec["manifest_revision"], manifest.manifest_revision) + self.assertEqual(spec["release_manifest_sha256"], binding.universal_manifest_sha256) + self.assertEqual(spec["release_manifest_sha256"], hashlib.sha256((work / f"{label}.yaml").read_bytes()).hexdigest()) + self.assertEqual(spec["query_url"], "https://localhost:8200") + self.assertEqual(spec["grpc_target"], "localhost:8210") + self.assertEqual(spec["subject"], "spiffe://qdl/paper/" + manifest.consumer_id) + self.assertNotIn("PRIVATE KEY", json.dumps(spec)) + for key, value in spec.items(): + if key.endswith("_file"): + self.assertTrue(value.startswith("/work/")) + self.assertTrue((work / Path(value).name).is_file()) + for product, policy in zip(binding.products, binding.request_policies): + req = next(r for r in manifest.requirements if r.feed.value == product.feed) + self.assertEqual(product.venue, venue) + self.assertEqual(product.fallback, "BLOCKED") + self.assertEqual(policy.requirement_id, product.requirement_id) + self.assertEqual(policy.event_recency_policy, None if req.event_recency_policy is None else req.event_recency_policy.value) + self.assertEqual(policy.max_session_liveness_ms, req.max_session_liveness_ms) + self.assertEqual(product.max_freshness_ms, req.max_freshness_ms) + + @patch.object(dr.os, "access", return_value=True) + def test_core_override_pinned_readonly_and_tampering_refused(self, _access): + with tempfile.TemporaryDirectory() as tmp: + binary = Path(tmp) / "qdl-realtime-core" + binary.write_bytes(b"test-only-not-a-real-binary") + binary.chmod(0o700) + selected = dr.core_override(binary) + self.assertEqual(selected["sha256"], hashlib.sha256(binary.read_bytes()).hexdigest()) + commands = dr.topology(NS, WORK, core_binary=selected) + self.assertIn(str(binary) + ":/usr/local/bin/qdl-realtime-core:ro", commands["core"]) + self.assertNotIn(str(binary), json.dumps({k: v for k, v in commands.items() if k != "core"})) + binary.write_bytes(b"changed") + with self.assertRaisesRegex(ValueError, "digest changed"): + dr.topology(NS, WORK, core_binary=selected) + + def test_reserved_ports_absent(self): + commands = json.dumps(dr.topology(NS, WORK)) + self.assertNotIn("6391", commands) + self.assertNotIn("5433", commands) + + def test_image_declared_volumes_are_tmpfs(self): + commands = dr.topology(NS, WORK) + for role, paths in (("kafka", ("/etc/kafka/secrets", "/mnt/shared/config", "/var/lib/kafka/data", "/data")), ("quota", ("/data",))): + tmpfs = [commands[role][i + 1].split(":")[0] for i, arg in enumerate(commands[role]) if arg == "--tmpfs"] + self.assertTrue(set(paths) <= set(tmpfs)) + + def test_deadline_rejects_invalid(self): + for value in (True, 0, -1, 121, float("inf"), float("nan"), "10"): + with self.subTest(value=value), self.assertRaises(ValueError): + case._deadline(value) + + def test_packet_rejects_shared_transport_and_topics(self): + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + handoff = {"namespace": NS, "provenance": "TEST_ONLY", "material_ready": True} + (work / "handoff.json").write_text(json.dumps(handoff)) + for target in ("bootstrap", "tls", "raw", "canonical", "quarantine", "test"): + packet, cfg = runtime_packet(), config() + if target == "bootstrap": packet["bootstrap"] = "production:9092" + if target == "tls": packet["client_tls"]["key"] = "/shared/key" + if target == "raw": cfg["raw_topics"] = ["shared.raw"] + if target in ("canonical", "quarantine"): cfg["core"][target + "_stream"] = "shared" + if target == "test": cfg["core"]["allow_test_provenance"] = False + (work / "packet.json").write_text(json.dumps(packet)) + (work / "core.json").write_text(json.dumps(cfg)) + with self.subTest(target=target), self.assertRaises(ValueError): + case._packet(work) + + +class ResetTests(unittest.TestCase): + def preflight(self, rows, outsider=""): + call = Mock(side_effect=["one", outsider or "kafka-id none"]) + with patch.object(dr, "owned", side_effect=lambda ns, role, call: rows.get(role)): + return case._reset_preflight(WORK, NS, call) + + def test_exact_disposable_accepted(self): + self.assertEqual(self.preflight(owned_rows())["Id"], "kafka-id") + + def test_refuses_attached_clone_or_channel(self): + for mode in ("container:kafka-id", "container:" + NS + "-kafka"): + with self.subTest(mode=mode), self.assertRaisesRegex(ValueError, "still attached"): + self.preflight(owned_rows(), "clone-id " + mode) + + def test_refuses_wrong_image_mount_network_or_persistence(self): + for field in ("Image", "AutoRemove", "NetworkMode", "ReadonlyRootfs", "PortBindings", "Tmpfs", "mount"): + rows = owned_rows() + if field == "Image": rows["kafka"][field] = "wrong-image" + elif field == "mount": rows["kafka"]["Mounts"].append({"Type": "volume", "Destination": "/data", "Source": "shared"}) + else: rows["kafka"]["HostConfig"][field] = {"AutoRemove": False, "NetworkMode": "host", "ReadonlyRootfs": False, "PortBindings": {"9092": [{}]}, "Tmpfs": {}}[field] + with self.subTest(field=field), self.assertRaises(ValueError): + self.preflight(rows) + + def test_helpers_and_orphan_roles_refused(self): + for role in ("helper", "tsvalidator", "missing-broker"): + rows = owned_rows() + if role == "missing-broker": rows.pop("kafka") + else: rows[role] = {"Id": "helper-id"} + with self.subTest(role=role), self.assertRaises(ValueError): + self.preflight(rows) + + @patch.object(dr.os, "access", return_value=True) + def test_candidate_override_mount_must_match_and_be_readonly(self, _access): + with tempfile.TemporaryDirectory() as tmp: + binary = Path(tmp) / "core" + binary.write_bytes(b"test-only") + binary.chmod(0o700) + selected = dr.core_override(binary) + rows = owned_rows() + mount = {"Type": "bind", "Destination": "/usr/local/bin/qdl-realtime-core", "Source": str(binary), "RW": False} + rows["core"]["Mounts"].append(mount) + with patch.object(dr, "owned", side_effect=lambda ns, role, call: rows.get(role)): + self.assertEqual(case._reset_preflight(WORK, NS, Mock(return_value=""), selected)["Id"], "kafka-id") + mount["RW"] = True + with self.assertRaisesRegex(ValueError, "read-only"): + case._reset_preflight(WORK, NS, Mock(return_value=""), selected) + + def test_reset_delegates_to_broker_preserving_controller(self): + result = {"broker_id": "same-broker", "topic_id": "new-topic", "acceptance": False} + with patch("scripts.p184_dr05_reset.reset_preserving_broker", return_value=result) as reset: + self.assertIs(dr.reset_case(WORK, "BINANCE", 7), result) + reset.assert_called_once_with(WORK, "BINANCE", 7) + + def test_reset_delegation_preserves_controller_error(self): + with patch("scripts.p184_dr05_reset.reset_preserving_broker", side_effect=ValueError("foreign")), patch.object(dr, "cleanup") as cleanup, patch.object(dr, "up") as up: + with self.assertRaisesRegex(ValueError, "foreign"): + case.reset_case(WORK, "OKX", 30) + cleanup.assert_not_called() + up.assert_not_called() + + def test_reset_delegation_preserves_override_receipt(self): + selected = {"path": "/tmp/candidate", "sha256": "a" * 64} + result = {"broker_id": "same-broker", "core_binary": selected} + with patch("scripts.p184_dr05_reset.reset_preserving_broker", return_value=result) as reset: + self.assertEqual(case.reset_case(WORK, "OKX")["core_binary"], selected) + reset.assert_called_once_with(WORK, "OKX", 30) + + +class ObserveTests(unittest.TestCase): + def setUp(self): + from qdl.provider.v1.raw_provider_pb2 import RawProviderEnvelope + self.wire = RawProviderEnvelope(capture_id=b"x" * 16, test_provenance=True).SerializeToString() + self.receipt = {"topic": NS + ".raw", "partition": 0, "offset": 3, "capture_id": (b"x" * 16).hex(), + "raw_sha256": hashlib.sha256(self.wire).hexdigest(), "provenance": "TEST_ONLY"} + self.packet = patch.object(case, "_packet", return_value=(WORK, NS, runtime_packet(), config())) + self.env = patch.dict(os.environ, {"P184_DR05_ISOLATED_NAMESPACE": NS}) + self.packet.start(); self.env.start() + self.addCleanup(self.packet.stop); self.addCleanup(self.env.stop) + + def test_rejects_receipt_before_io(self): + for key, value in (("topic", "shared"), ("partition", 1), ("partition", False), ("offset", -1), ("offset", True), ("capture_id", "bad"), ("raw_sha256", "bad"), ("provenance", "LIVE")): + with self.subTest(key=key), patch("confluent_kafka.Consumer") as consumer: + with self.assertRaises(ValueError): + dr.canonical_observe(WORK, {**self.receipt, key: value}) + consumer.assert_not_called() + + def test_attestation_required(self): + with patch.dict(os.environ, {"P184_DR05_ISOLATED_NAMESPACE": "shared"}), self.assertRaises(ValueError): + dr.canonical_observe(WORK, self.receipt) + + def observe(self, records=(), mismatch=False, timeout=False, barrier=True, invalid=False): + message = Mock() + message.error.return_value = None + message.topic.return_value = self.receipt["topic"] + message.partition.return_value = 0 + message.offset.return_value = 4 if mismatch else 3 + message.value.return_value = self.wire + reader = Mock() + reader.poll.return_value = None if timeout else message + observer = Mock() + observer.topics = [NS + ".canonical", NS + ".quarantine"] + observer.observe.return_value = {"raw_committed_next": 4 if barrier else 3, + "read_committed_eof": observer.topics, "matched": list(records)} + with patch("confluent_kafka.Consumer", return_value=reader) as factory, patch("scripts.p184_dr05_observer.CanonicalObserver", return_value=observer): + try: + return dr.canonical_observe(WORK, self.receipt, 2) + finally: + self.assertEqual(factory.call_args.args[0]["fetch.wait.max.ms"], 10) + reader.close.assert_called_once() + if not mismatch and not timeout and not invalid: + observer.close.assert_called_once() + + def test_canonical_and_quarantine_preserve_coordinates_reason(self): + row = {"kind": "quarantine", "topic": NS + ".quarantine", "partition": 0, "offset": 9, "reason": "SEQUENCE_GAP", "capture_id": self.receipt["capture_id"]} + result = self.observe([row]) + self.assertTrue(result["observed"]) + self.assertEqual(result["records"], [{**row, "partition": 0}]) + self.assertFalse(result["acceptance"]) + + def test_filtered_requires_commit_barrier(self): + self.assertEqual(self.observe()["outcome"], "FILTERED_AFTER_COMMIT") + with self.assertRaises(RuntimeError): + self.observe(barrier=False) + + def test_stale_receipt_refused(self): + with self.assertRaises(ValueError): + self.observe(mismatch=True) + + def test_wrong_hash_or_capture_refused(self): + for key, value in (("raw_sha256", "0" * 64), ("capture_id", "0" * 32)): + original = self.receipt[key] + self.receipt[key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + self.observe(invalid=True) + self.receipt[key] = original + + def test_raw_timeout_refused_and_closed(self): + with self.assertRaises(TimeoutError): + self.observe(timeout=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p184_dr05_channel.py b/tests/test_p184_dr05_channel.py new file mode 100644 index 00000000..d3f9ca08 --- /dev/null +++ b/tests/test_p184_dr05_channel.py @@ -0,0 +1,335 @@ +import copy +import io +import json +import os +from pathlib import Path +import tempfile +import threading +import time +import unittest +from unittest.mock import Mock, patch + +from scripts import p184_dr05_channel as ch +from scripts.p184_dr05_dl_fault import frame + +NS = "p184-dr05-dl-0123456789abcdef" + + +def receipt(): + return {"topic": NS + ".raw", "partition": 0, "offset": 0, "capture_id": "a" * 32, + "raw_sha256": "b" * 64, "generation": 7, "broker_ack_ms": 1.5, + "provenance": "TEST_ONLY", "acceptance": False} + + +def row(kind="canonical"): + result = {"kind": kind, "topic": NS + "." + kind, "partition": 0, "offset": 2, + "capture_id": "a" * 32} + if kind == "quarantine": + result["reason"] = "QUARANTINE_REASON_SEQUENCE_GAP" + return result + + +def barrier(rows=None): + return {"raw_committed_next": 1, "read_committed_eof": [NS + ".canonical", NS + ".quarantine"], + "matched": [row()] if rows is None else rows} + + +class ChannelTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.work = Path(self.temp.name) + self.packet = {"namespace": NS, "slice": {"topic_id": "old-topic"}} + self.config = {"raw_topics": [NS + ".raw"]} + self.loader = Mock(side_effect=lambda work: (Path(work), NS, copy.deepcopy(self.packet), copy.deepcopy(self.config))) + self.publishers, self.observers = [], [] + def publisher(packet, config): + value = Mock() + value.publish.return_value = receipt() + self.publishers.append(value) + return value + def observer(packet, config): + value = Mock() + value.observe.return_value = barrier() + self.observers.append(value) + return value + self.publisher_factory, self.observer_factory = publisher, observer + self.channel = ch.Channel(self.work, loader=self.loader, publisher_factory=publisher, observer_factory=observer) + self.addCleanup(self.channel.close) + + def call(self, op, args, timeout=1): + return self.channel.dispatch({"op": op, "args": args, "timeout": timeout}) + + def publish(self): + return self.call("publish", {"frame": frame("OKX", True, 100, 0, 7, "1", 1000000)}) + + def host_reply(self, *, ok=True, changed=True, bad_namespace=False): + original = ch._atomic + def write(path, request): + self.assertEqual(set(request), {"request_id", "namespace", "venue", "fault", "timeout"}) + self.assertEqual(request["namespace"], NS) + self.assertEqual(len(request["request_id"]), 32) + self.assertGreater(request["timeout"], 0) + self.assertLessEqual(request["timeout"], 1) + self.assertIsNone(self.channel.publisher) + self.assertIsNone(self.channel.observer) + original(path, request) + self.observers[0].close.assert_called_once() + if changed: + self.packet["slice"]["topic_id"] = "new-topic" + if ok: + result = {"namespace": "shared" if bad_namespace else NS, + "topic_id": self.packet["slice"]["topic_id"], "reset": True} + reply = {"ok": True, "result": result} + else: + reply = {"ok": False, "error": "PRIVATE KEY must never be echoed"} + original(path.parent / ("reset-" + request["request_id"] + ".json"), reply) + return patch.object(ch, "_atomic", side_effect=write) + + def test_real_clients_are_default_factories(self): + from scripts.p184_dr05_raw_publisher import RawPublisher + from scripts.p184_dr05_observer import CanonicalObserver + self.assertTrue(callable(RawPublisher)) + self.assertTrue(callable(CanonicalObserver)) + + def test_publish_and_observe_use_cached_instances(self): + r = self.publish() + result = self.call("canonical_observe", {"receipt": r}) + self.assertTrue(result["observed"]) + self.assertFalse(result["acceptance"]) + self.assertEqual(result["records"], [row()]) + again = self.call("canonical_observe", {"receipt": r}) + self.assertEqual(again, result) + self.observers[0].observe.assert_called_once() + self.assertEqual(len(self.publishers), 1) + self.assertEqual(len(self.observers), 1) + + def test_quarantine_preserves_actual_coordinates_and_reason(self): + r = self.publish() + self.observers[0].observe.return_value = barrier([row("quarantine")]) + self.assertEqual(self.call("canonical_observe", {"receipt": r})["records"], [row("quarantine")]) + + def test_filtered_requires_actual_postcommit_barrier(self): + r = self.publish() + self.observers[0].observe.return_value = barrier([]) + self.assertEqual(self.call("canonical_observe", {"receipt": r})["records"], []) + + def test_unobserved_publish_cannot_be_overwritten(self): + self.publish() + with self.assertRaises(ValueError): + self.publish() + self.publishers[0].publish.assert_called_once() + + def test_publish_ambiguity_fences_all_followup_operations(self): + self.publishers[0].publish.side_effect = TimeoutError("private diagnostic") + with self.assertRaises(TimeoutError): + self.publish() + self.assertTrue(self.channel.poisoned) + with self.assertRaisesRegex(RuntimeError, "fenced"): + self.call("reset_case", {"venue": "OKX", "fault": "GAP_REFUSAL"}) + self.assertFalse((self.work / "control/reset-request.json").exists()) + + def test_foreign_or_modified_receipt_never_reaches_observer(self): + r = self.publish() + changes = {"topic": "shared.raw", "partition": 1, "offset": 1, "capture_id": "c" * 32, + "raw_sha256": "d" * 64, "generation": 8, "provenance": "LIVE", "extra": "bad"} + for key, value in changes.items(): + with self.subTest(key=key), self.assertRaises(ValueError): + self.call("canonical_observe", {"receipt": {**r, key: value}}) + self.observers[0].observe.assert_not_called() + + def test_sentinel_or_boolean_offsets_rejected(self): + for key, value in (("offset", -1), ("offset", True), ("partition", False)): + with self.subTest(key=key), self.assertRaises(ValueError): + self.channel._validate_receipt({**receipt(), key: value}) + + def test_malformed_requests_do_not_reach_publisher(self): + requests = [[], {}, {"op": "publish", "args": {}, "timeout": 1}, + {"op": "other", "args": {}, "timeout": 1}, + {"op": "reset_case", "args": {"venue": "OKX", "fault": "GAP_REFUSAL", "path": "/shared"}, "timeout": 1}] + for request in requests: + with self.subTest(request=request), self.assertRaises(ValueError): + self.channel.dispatch(request) + self.publishers[0].publish.assert_not_called() + + def test_unknown_frame_fields_refused_before_publish(self): + f = frame("OKX", True, 100, 0, 7, "1", 1000000) + with self.assertRaises(ValueError): + self.call("publish", {"frame": {**f, "phase": "positive"}}) + self.publishers[0].publish.assert_not_called() + + def test_deadline_bounds_native_poll_and_commit(self): + r = self.publish() + observer = self.observers[0] + consumer, offsets = observer.consumer, observer.offset_reader + def observe(receipt, timeout): + observer.consumer.poll(3) + observer.offset_reader.committed(["partition"], timeout=3) + return barrier() + observer.observe.side_effect = observe + self.call("canonical_observe", {"receipt": r}, timeout=.1) + self.assertLessEqual(consumer.poll.call_args.args[0], .1) + self.assertLessEqual(offsets.committed.call_args.kwargs["timeout"], .1) + self.assertIs(observer.consumer, consumer) + self.assertIs(observer.offset_reader, offsets) + + def test_invalid_barrier_or_coordinates_fences(self): + r = self.publish() + self.observers[0].observe.return_value = {**barrier(), "raw_committed_next": 0} + with self.assertRaises(ValueError): + self.call("canonical_observe", {"receipt": r}) + self.assertTrue(self.channel.poisoned) + + def test_missing_partition_not_fabricated(self): + r = self.publish() + record = row(); record.pop("partition") + self.observers[0].observe.return_value = barrier([record]) + with self.assertRaises(ValueError): + self.call("canonical_observe", {"receipt": r}) + + def test_reset_atomic_ipc_reopens_clients_and_clears_receipts(self): + old = self.publish() + with self.host_reply(): + result = self.call("reset_case", {"venue": "OKX", "fault": "GAP_REFUSAL"}) + self.assertTrue(result["reset"]) + self.assertEqual(len(self.publishers), 2) + self.assertEqual(len(self.observers), 2) + self.assertFalse(self.channel.receipts) + self.assertIsNone(self.channel.pending) + self.assertEqual(self.channel.packet["slice"]["topic_id"], "new-topic") + self.assertEqual(list((self.work / "control").glob("*.json")), []) + with self.assertRaises(ValueError): + self.call("canonical_observe", {"receipt": old}) + + def test_parent_watcher_consumes_request_and_replies_on_same_ipc(self): + from scripts import p184_dr05_reset as host + result, errors = [], [] + + def reset(work, venue, timeout): + self.assertEqual(Path(work), self.work) + self.assertEqual(venue, "OKX") + self.assertGreater(timeout, 0) + self.assertFalse((self.work / "control/reset-request.json").exists()) + self.assertIsNone(self.channel.publisher) + self.assertIsNone(self.channel.observer) + self.observers[0].close.assert_called_once() + self.packet["slice"]["topic_id"] = "host-created-topic" + return {"namespace": NS, "venue": venue, "topic_id": "host-created-topic", + "previous_topic_id": "old-topic", "broker_id": "unchanged-broker", + "reset": True, "acceptance": False} + + def client(): + try: + result.append(self.call("reset_case", {"venue": "OKX", "fault": "GAP_REFUSAL"}, timeout=2)) + except Exception as error: + errors.append(error) + + with patch.object(host, "_packet", side_effect=self.loader), patch.object(host, "reset_preserving_broker", side_effect=reset), patch.object(host.signal, "signal"), patch.object(host.dr, "bounded", side_effect=AssertionError("no Docker in IPC test")): + thread = threading.Thread(target=client, daemon=True) + thread.start() + try: + self.assertEqual(host.serve_host(self.work, duration=.3), 0) + finally: + thread.join(3) + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(result[0]["broker_id"], "unchanged-broker") + self.assertEqual(self.channel.packet["slice"]["topic_id"], "host-created-topic") + self.assertEqual(len(self.publishers), 2) + self.assertEqual(list((self.work / "control").glob("*.json")), []) + + def test_reset_same_uuid_refused_and_fenced(self): + with self.host_reply(changed=False), self.assertRaisesRegex(ValueError, "fresh topic"): + self.call("reset_case", {"venue": "OKX", "fault": "GAP_REFUSAL"}) + self.assertTrue(self.channel.poisoned) + self.assertEqual(len(self.publishers), 1) + + def test_reset_foreign_response_refused(self): + with self.host_reply(bad_namespace=True), self.assertRaises(ValueError): + self.call("reset_case", {"venue": "OKX", "fault": "GAP_REFUSAL"}) + self.assertTrue(self.channel.poisoned) + + def test_reset_timeout_leaves_one_request_and_no_clients(self): + start = time.monotonic() + with self.assertRaises(TimeoutError): + self.call("reset_case", {"venue": "BINANCE", "fault": "GAP_REFUSAL"}, timeout=.06) + self.assertLess(time.monotonic() - start, .5) + request = json.loads((self.work / "control/reset-request.json").read_text()) + self.assertEqual(request["namespace"], NS) + self.assertEqual(len(list((self.work / "control").glob("reset-request*"))), 1) + self.assertIsNone(self.channel.publisher) + self.assertTrue(self.channel.poisoned) + + def test_single_channel_lock_refuses_second_owner(self): + with self.assertRaises(BlockingIOError): + ch.Channel(self.work, loader=self.loader, publisher_factory=self.publisher_factory, observer_factory=self.observer_factory) + + def test_stdout_is_jsonl_and_errors_do_not_echo_secrets(self): + incoming = io.BytesIO((json.dumps({"op": "reset_case", "args": {"venue": "OKX", "fault": "GAP_REFUSAL"}, "timeout": 1}) + "\n").encode()) + outgoing = io.StringIO() + with self.host_reply(ok=False): + ch.serve(self.channel, incoming, outgoing) + reply = json.loads(outgoing.getvalue()) + self.assertFalse(reply["ok"]) + self.assertNotIn("PRIVATE KEY", outgoing.getvalue()) + self.assertEqual(len(outgoing.getvalue().splitlines()), 1) + + def test_oversize_request_fences_without_parsing_or_reading_rest(self): + incoming = io.BytesIO(b"x" * (ch.MAX_BYTES + 10) + b"\n") + outgoing = io.StringIO() + ch.serve(self.channel, incoming, outgoing) + self.assertFalse(json.loads(outgoing.getvalue())["ok"]) + self.assertTrue(self.channel.poisoned) + self.assertEqual(incoming.tell(), ch.MAX_BYTES + 1) + + +class ProtocolTests(unittest.TestCase): + def test_duplicate_keys_and_nonfinite_json_refused(self): + for value in ('{"op":1,"op":2}', '{"timeout":NaN}', '{"timeout":Infinity}'): + with self.subTest(value=value), self.assertRaises(ValueError): + ch.decode(value) + + def test_invalid_deadlines(self): + for value in (True, 0, -1, 61, float("inf"), float("nan"), "2"): + with self.subTest(value=value), self.assertRaises(ValueError): + ch._timeout(value) + + def test_atomic_file_private_and_no_temporary_leftovers(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "request.json" + ch._atomic(path, {"request_id": "a" * 32}) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.assertEqual(ch._read(path), {"request_id": "a" * 32}) + self.assertEqual(list(Path(tmp).iterdir()), [path]) + + def test_response_symlink_refused(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "response.json" + target = Path(tmp) / "target.json" + target.write_text('{"ok":true}') + path.symlink_to(target) + with self.assertRaises(OSError): + ch._read(path) + + def test_fifo_response_refused_without_blocking(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "response.json" + os.mkfifo(path) + with self.assertRaises(ValueError): + ch._read(path) + + def test_load_requires_attestation_and_topic_uuid(self): + work = Path("/work") + packet = {"namespace": NS, "slice": {"topic_id": "actual-topic"}} + with patch("scripts.p184_dr05_case._packet", return_value=(work, NS, packet, {})): + with patch.dict(os.environ, {"P184_DR05_ISOLATED_NAMESPACE": "foreign"}), self.assertRaises(ValueError): + ch.load_packet(work) + with patch.dict(os.environ, {"P184_DR05_ISOLATED_NAMESPACE": NS}): + self.assertEqual(ch.load_packet(work)[1], NS) + packet["slice"].pop("topic_id") + with self.assertRaises(ValueError): + ch.load_packet(work) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p184_dr05_observer.py b/tests/test_p184_dr05_observer.py index ffa11e41..003dddba 100644 --- a/tests/test_p184_dr05_observer.py +++ b/tests/test_p184_dr05_observer.py @@ -14,7 +14,8 @@ def test_canonical_capture_identity_and_exact_bytes(self): raw_capture_id=b"1234567890123456", source_sequence="202") event.book_snapshot.native_sequence = "202" raw = event.SerializeToString() - result = record_summary("isolated.canonical", 0, raw, True) + result = record_summary("isolated.canonical", 0, raw, True, partition=3) + self.assertEqual(result["partition"], 3) self.assertEqual(result["capture_id"], event.raw_capture_id.hex()) self.assertEqual(result["generation"], 8) self.assertEqual(result["payload"], "book_snapshot") diff --git a/tests/test_p184_dr05_reset.py b/tests/test_p184_dr05_reset.py new file mode 100644 index 00000000..f8252702 --- /dev/null +++ b/tests/test_p184_dr05_reset.py @@ -0,0 +1,79 @@ +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + +from scripts import p184_dr05_reset as reset +from tests.test_p184_dr05_case import NS, runtime_packet, config + + +class ResetControllerTests(unittest.TestCase): + def run_reset(self, *, preflight_error=None, missing=False, changed=False): + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "handoff.json").write_text("{}") + stopped = False + started = set() + calls = [] + + def owned(ns, role, call): + self.assertEqual(ns, NS) + if role == "kafka": + return {"Id": "changed" if changed else "broker"} + if role == "helper": + return {"Id": "channel"} + if missing and role == "query": + return None + return {"Id": role} if not stopped or role in started else None + + def bounded(argv, timeout): + nonlocal stopped + self.assertGreater(timeout, 0) + calls.append(argv) + if argv[:2] == ["docker", "stop"]: + stopped = True + return "" + if argv[0] == "start": + started.add(argv[1]) + return "id" + if "scripts/p184_dr05_reset.py" in argv: + return json.dumps({"topic_id": "new", "previous_topic_id": "old"}) + return "{}" + + preflight = Mock(side_effect=preflight_error, return_value={"Id": "broker"}) + with patch.object(reset, "_packet", return_value=(work, NS, runtime_packet(), config())), \ + patch.object(reset, "_reset_preflight", preflight), \ + patch.object(reset.dr, "owned", side_effect=owned), \ + patch.object(reset.dr, "bounded", side_effect=bounded), \ + patch.object(reset.dr, "topology", return_value={r: ["start", r] for r in ("quota", "core", "stream", "query")}): + try: + return reset.reset_preserving_broker(work, "OKX", 10), calls + finally: + self.calls = calls + + def test_only_owned_readers_restarted_broker_untouched(self): + result, calls = self.run_reset() + self.assertEqual(result["generation"], 7) + self.assertEqual(result["broker_id"], "broker") + self.assertFalse(result["acceptance"]) + stop = next(c for c in calls if c[:2] == ["docker", "stop"]) + self.assertEqual(stop[4:], ["query", "stream", "core", "quota"]) + self.assertNotIn("broker", stop) + self.assertNotIn("channel", stop) + + def test_foreign_or_missing_stack_refused_before_mutation(self): + for options in ({"preflight_error": ValueError("foreign")}, {"missing": True}): + with self.subTest(options=options), self.assertRaises(ValueError): + self.run_reset(**options) + self.assertFalse(self.calls) + + def test_broker_replacement_fails_closed(self): + with self.assertRaisesRegex(RuntimeError, "namespace changed"): + self.run_reset(changed=True) + + def test_invalid_deadline_and_venue_before_docker(self): + for venue, timeout in (("OKX", float("nan")), ("OKX", 0), ("OTHER", 1)): + with patch.object(reset.dr, "bounded") as call, self.assertRaises(ValueError): + reset.reset_preserving_broker(Path("/unused"), venue, timeout) + call.assert_not_called() diff --git a/upgrade/evidence/p184-claude-stage3-alignment.md b/upgrade/evidence/p184-claude-stage3-alignment.md index 46f807f7..62d84f1b 100644 --- a/upgrade/evidence/p184-claude-stage3-alignment.md +++ b/upgrade/evidence/p184-claude-stage3-alignment.md @@ -111,3 +111,18 @@ Integration points to reconcile before claiming the new TS runner is wired: No acknowledgement from Claude or full writer-clone acceptance is claimed here. The next shared handoff should pin the actual TS image and use these observations as the counterexample for the narrow upstream invalidation fix, not rerun C2. + + +## Current DR05 Handoff: Canonical Fence And JSONL (2026-10-08) + +The prior five integration points above are retained as history. DL now supplies +per-venue ts-consumer.v1 packets and V3 bindings, a namespace-attested JSONL +channel, and a broker-preserving host reset bridge. Four fault cells pass the +isolated diagnostic;16 channel publish/observe steps and2 reset cycles pass. +Use [current guide and exact receipts](p184-dr05-fence-channel-handoff.md). + +Remaining TS boundaries are unchanged: actual WriterClone control/Redis/ACK +validation, Binance bootstrap semantics, typed immediate refusal rather than +waiting for stale, and honest final SHA/image provenance. Do not mark final +DR05 PASS from the diagnostic helper. The DL fixes have not been deployed or +packaged as the final consolidated release. No TS source was edited. diff --git a/upgrade/evidence/p184-dr05-dl-fault.md b/upgrade/evidence/p184-dr05-dl-fault.md index a5f55145..a86002ab 100644 --- a/upgrade/evidence/p184-dr05-dl-fault.md +++ b/upgrade/evidence/p184-dr05-dl-fault.md @@ -1,6 +1,8 @@ # DR05 Isolated DL Stack Handoff -Status: STACK_STARTUP_TESTED / TS_FAULT_ACCEPTANCE_NOT_RUN, 2026-10-08. +Status: IMPLEMENTED_TESTED_ISOLATED / TS_FINAL_ACCEPTANCE_PENDING, 2026-10-08. +Current packet/API and fence results: [DR05 fence/channel handoff](p184-dr05-fence-channel-handoff.md). +Earlier receipts and implementation checkpoints below remain historical. Governing scope: Unified Plan, "Pre-stage3 consolidated build, SDK and DR05"; [Phase C packet](p184-dl-phase-c-packet.md). No shared runtime changes. diff --git a/upgrade/evidence/p184-dr05-fence-channel-handoff.md b/upgrade/evidence/p184-dr05-fence-channel-handoff.md new file mode 100644 index 00000000..159a6e83 --- /dev/null +++ b/upgrade/evidence/p184-dr05-fence-channel-handoff.md @@ -0,0 +1,185 @@ +# DR05 Fence And JSONL Handoff + +Status: IMPLEMENTED_TESTED_ISOLATED; TS_FINAL_ACCEPTANCE_PENDING. +2026-10-08. Governing journal: Unified Plan, "DR05 upstream invalidation and +TS packet integration". This supersedes the open plumbing/fence implementation +items in the earlier observer handoff, not its retained failing receipts. + +## Domain Fix + +Core previously quarantined a broken book but left its last readable canonical +view in place. Readers could therefore approve that old view while still fresh. +The core now emits one non-readable snapshot/delta fence per invalid owner / +generation scope. It uses existing sequence-gap/resync/reset flags, no prices, +no fabricated native sequence, and the actual raw capture identity. Its timestamp +dates invalidation, not a refresh of an old price. A valid anchor restores the +existing venue-specific state machine. Superseded sessions/generations cannot +invalidate the current owner; repeated unanchored frames do not amplify fences. + +Projector StageA retains the fence bytes and separate product keys. Query's +primary validator recognizes the non-readable fence after catalog identity and +provenance checks. The hot-read path preserves the existing Rust Stream typed +refusal; it must not retry another replica or return an older healthy primary +to hide that refusal. No protobuf, public endpoint, policy or freshness changes. + +## TS Interface + +`--init` now emits `ts-consumer.binance.json` and `ts-consumer.okx.json`: +schema `p184.dr05.ts-consumer.v1`, venue, consumer ID/subject, issuer/audience, +environment/manifest revision, JWT key ID, and path-only credential references. +The five required `*_file` paths reside under `/work/`. No secret is inline. +Each binding is V3 rendered from that venue's isolated manifest, with request +policies and a TEST_ONLY manifest digest, not a production release certificate. +Query `https://localhost:8200`; Stream `localhost:8210`. TS ports6391/5433 free. + +The JSONL channel runs in existing DL Python image77682eea (Query rc2-43bbcf0), +inside `container:-kafka`, with the attestation environment variable. +Requests are exactly `{op,args,timeout}`; replies `{ok:true,result}` or +`{ok:false,error}`. Invalid/oversized input, ambiguous publish or timeout fails +closed. It never echoes credentials or raw input on errors. + +- `publish {frame}`: actual RawPublisher ACK and coordinates; one outstanding + publication until observation, no retry on unknown delivery. +- `canonical_observe {receipt}`: exact receipt issued by this channel, matched + capture ID, core committed offset plus read_committed EOF barrier. Returns + actual canonical/quarantine topic, partition, offset, payload hashes and reason. + Filtered frames have an empty record list with a proven commit/EOF barrier. +- `reset_case {venue,fault}`: bounded host bridge resets only the owned core, + Stream, Query, quota and three TEST_ONLY topics / two consumer groups. Broker + and channel network survive. New canonical topic UUID, generation starts at7. + An attached TS clone or foreign container makes reset refuse before mutation. + +No Docker socket is mounted in any container. Start the host watcher before +Claude's adapter starts the channel; it watches only its private work directory: + +```bash +python3 -B scripts/p184_dr05_reset.py --serve-reset-host "$WORK" --duration 1800 +``` + +Generate the channel argv through the existing helper to preserve limits/mounts: + +```python +argv = dr.helper_command(ns, work, ["--channel", "/work"], + network="container:" + ns + "-kafka") +argv.insert(2, "-i") +argv[argv.index("scripts/p184_dr05_dl_fault.py")] = "scripts/p184_dr05_channel.py" +pos = argv.index(dr.IMAGES["python"]) +argv[pos:pos] = ["-e", "P184_DR05_ISOLATED_NAMESPACE=" + ns] +# shlex.join(argv) is Claude's --dl-channel argument; stdin/stdout are JSONL. +``` + +Use `dr.up(work, core_binary=)` for the isolated patched +core. The optional binary is mounted read-only and pinned by SHA256; reset checks +the same pin. This is source/binary-overlay evidence, not image packaging. +Stop TS clones and channel, terminate the watcher, then `--down` in finally. +The host bridge allows at most16 resets, each bounded by the request deadline. + +## Verification + +All fixtures are TEST_ONLY, not exchange or production evidence. + +- Python:120 PASS,0fail,0skip,43.876s across dr05_dl_fault, case, channel, + observer, raw_publisher, verdict, kn_hot_view and kn_hot_book_invalidation. +- Reset controller:4 PASS,0fail,0skip,0.006s. Foreign/missing stack prechecks, + exact owned stop set, unchanged broker and invalid deadline rejection. +- Rust realtime-core:53 PASS,0fail,1ignored (final source,0.11s). The ignored pre-existing test + requires separate Redis provider admission; not represented as passed. +- Rust projector product tests:8 PASS,0fail,0ignored (23 unrelated filtered). +- Four isolated before/after cells: baseline FAIL is retained in the previous + observer report; patched core gives4/4 diagnostic PASS on Binance/OKX x + GAP/unanchored generation.12 positive/recovery/post-fence controls match exact + canonical bytes and actual TS in-image SDK/BOOK validator/Redis application. + Refusal is immediate typed OPEN_SEQUENCE_GAP, not eventual stale or NO_WRITE. +- JSONL smoke:16 actual publications/committed observations;2 resets preserve + broker ID, replace topic UUID and restart raw offset0. Reset durations: + Binance21670.885ms, OKX19807.860ms (n=2; no p99 claim). Both under30s. + This budget is DL-only; Claude's fresh_case also starts its TS clone and must + enforce its overall deadline rather than silently extending it. + +Evidence root outside Git: `~/.local/state/qdl-v2/dr05-fence-20261008/`. + +| Receipt | SHA256 | +| --- | --- | +| binance-gap-candidate-r1/receipt.json | 374495d08fa6d69a079a8f3ec6248929776dce0a3abceb8391d40d5fc900a3c8 | +| binance-generation-candidate-r1/receipt.json | be8f7417528ab7318f8624623778cd44067ff343bdada2ce9ab96637ee1ed41a | +| okx-gap-candidate-r1/receipt.json | 7fc9265e8ff76ff7c240dfe9301a5015ac7b007ab4a4137c11b83c17fadd4629 | +| okx-generation-candidate-r1/receipt.json | 6fc44d71487c937fb347daf868783ef9a69a6f25927c8945ec5d459bba185a4e | +| channel-r1/receipt.json | 2e6539b18805b393e345b8d3712389e1fd29ab93181a54397d2e234ea07f6152 | + +Those runs pin core binary0ca2dad272dbf1948a907c45df23e16fcd9a3e4651ff1466d0d9d4740bf60f90. +Final source removes redundant borrows from the extracted Rust inner function; +recompile succeeds (28.60s) as e76d38173fe9d5dc456df0cda8c889275b4dd8faaff4c5438b3e0f8ad7b11c8e. +Do not rewrite prior binary hashes or label prior receipts as final-image tests. + +## Still Required From Joint TS Acceptance + +Claude owns WriterClone subscribe/control/Redis/production validator and final +TS evidence header. `drive()` continues ok=false. The diagnostic helper is NOT +full WriterClone ACK acceptance. In particular verify old cached books are +invalidated on the live TS control path, not only that a new unary read refuses. +Binance bootstrap snapshot alone is not a ready book until its continuity delta. +Require GAP/generation refusal while previous data remains fresh; generic +DATA_STALE after a wait is insufficient. Keep the TS SHA/image pair honest. + +Primary projector coverage here is the StageA product test plus Query primary +validation tests, not a new full projector/cache runtime acceptance. No shared +rollout is approved by these isolated receipts. Final consolidated build/CI, +DR01-DR08/broker delta and approved packet remain required; v2.2.3 HOLD. + +## Runtime And Cleanup + +Five isolated case stacks (four observers and JSONL smoke) removed; private +files remaining0 per receipt. No images built; reused existing immutable images. +Smoke free disk66834546688 ->66607972352 bytes; concurrent host changes mean +this is an observation, not attributed test disk consumption. No broad prune. +Canonical `/home/bobby/data_layer` main5bf5cd5 / publishedv2.2.2 unchanged. +Feature `/home/bobby/.worktrees/data-layer-p184-closure` fix/p184-dl-closure. +No TS edits, push, merge, shared runtime/config/manifest changes. Retain active +images and explicit BAR rollbackf7351c3b. Final local build-cache cleanup and +retained handoff binary inventory are appended in the journal. + + +### Pre-Commit Self-Review + +A same-generation recovery snapshot can legitimately repeat its prior native +sequence. Regression failed on the first fence patch (no recovery publication). +The fix distinguishes the new real recovery anchor by immutable raw capture ID; +native sequence, generation, values and timestamps remain unchanged. A repeated +copy is still deduplicated. The fence latch only clears after publication. +The alternate production-core runner must preserve mixed fence/quarantine +outputs atomically rather than rejecting or dropping one; its result is recorded +in the final journal before commit. These are narrow source changes, not a +shared rollout or a new certification campaign. + + +## Final Handoff Freeze + +Alternate runner regression:3/3 tests fail on old behavior; final suite7PASS, +0fail/ignored using cargo test --release --offline --locked -p qdl-kafka +--bin qdl-production-core -- --test-threads=1. It accepts only the paired empty +unverified BOOK fence as a mixed outcome and preserves primary quarantine in +the same transaction/watermark. Other ambiguous outputs still fail closed. + +Final core binary after recovery fix: +`~/.local/state/qdl-v2/dr05-fence-20261008/bin/qdl-realtime-core-baafcb48` +SHA256 baafcb489bb17fabdd73bef383dcf3448581cc8bb63578e9df83e2a57107b6f2. +Affected OKX generation rerun on this binary: diagnostic PASS, receipt +`okx-generation-final-r2/receipt.json`, SHA256 +fcf4b6593732cde3fc6897e750346dea2ed9d3744d80ce29add82aa1a824000a. +Earlier four receipts and channel smoke retain their earlier binary identity; +do not claim every run used this final binary or that TS WriterClone passed. + +All six isolated case stacks and helpers removed; privatefiles0, no test +containers remain. Scoped Cargo target removed1225121125 file bytes, retaining +only the named handoff executable. Free disk59893260288 ->60988051456 bytes; +concurrent host activity prevents exact attribution. Cleanup initially met +root-owned copied build files; a network-none helper removed only that target +mount. No image build, image deletion, volume prune or shared cache prune. +Receipt `cleanup-final.json` records scope and retained hash. + +Read-only shared snapshot `shared-runtime-readonly.json`:12app+5infra roles, +restart0/OOMfalse. Core658a9570, ingestors0f6876e1, Query/BAR51450612, +projectora696cfba, Stream1f13408f, Kafka9516fb76, Redisdfa18828 (full digests in JSON). These were observed, not modified or recertified. +Source/runtime remain deliberately different until the approved consolidated +release packet. No newly created Docker image needs deletion. Shared inventory +95images28.96GB/BuildKit20.52GB is retained during Claude's concurrent work. From 0b374bc918af02476cdfb1502d5df132426f901b Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 13:49:23 +0000 Subject: [PATCH 038/147] fix(stream): preserve terminal errors across stalled handoffs --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 68 +++++++++ rust/qdl-stream-gateway/src/service.rs | 27 +++- .../qdl-stream-gateway/tests/native_stream.rs | 70 +++++++++ scripts/p184_dr08_isolated.py | 69 +++++++-- tests/test_p184_dr08_isolated.py | 69 +++++++++ .../evidence/p184-claude-stage3-alignment.md | 11 ++ upgrade/evidence/p184-dr-runtime-runbook.md | 17 +++ .../p184-dr01-dr08-trade-attribution.md | 11 ++ .../evidence/p184-dr08-continuity-closure.md | 134 ++++++++++++++++++ 9 files changed, 461 insertions(+), 15 deletions(-) create mode 100644 upgrade/evidence/p184-dr08-continuity-closure.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index a653c369..dec4fbc8 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70438,3 +70438,71 @@ Shared digests/config untouched;17read-only roles restart0/OOMfalse. Retain all active and BARrollbackf7351c3b; no push/merge/deploy. Claude next runs his actual WriterClone/control/cache/ACK validation with honest TS source/image provenance. DR05 final acceptance and v2.2.3 remain HOLD pending joint DR/broker evidence. + +### DR08 continuity attribution and bounded recovery - 2026-10-08 (Codex) + +Status IN_PROGRESS. Approved scope: runtime/source attribution, bounded passive +same-event tracing, isolated DR08 reproductions/fixes. Guides: upgrade/evidence/ +p184-dr-runtime-runbook.md and p184-dr01-dr08-trade-attribution.md. No shared +runtime changes, extra TS subscription, orders, TS edits, push/merge or release. +Prior observation: roughly15min overflow+59/replay+84263/expired+4; TRADE queue1000, +replay10000/OUT_OF_RANGE; BOOK_DELTA queue~7.9MB/RESOURCE_EXHAUSTED. TS cache +600reads/60keys had0missing but BOOK_DELTA source-age max8288.56ms. These are not +same-event latency joins or proof of transport cause. Preserve LOSSLESS semantics. +Fresh runtime: writer5849d716, Stream1f13408f, core658a9570, Query/BAR51450612, +projectora696cfba. TS source8b41608; Claude owns OR and IDR078/081. IDR081 OPEN +colocates both BTC venues in one worker; do not duplicate TS ownership changes. +Linked DL report is the incident register for BOOK continuity; do not edit +Claude's TS INTEGRATION_DEFECT_REGISTER.md. Missing trace spans NOT_MEASURED. +Tests: bounded item/byte queue, slow/fast isolation, replay/refusal, valid snapshot +anchor/generation, no rewind, cursor after apply. Reuse DR08 existing60s restart +and600s run limits; no increased quota/buffer/freshness or retrospective gates. +Source rollback only; no runtime deployment. Cleanup owned disposable resources, +retain active/named rollback images/evidence, no shared prune. apply_patch helper +failed twice (mountinfo sandbox); use exact-match replacement and inspect diff. + +DR08 source review: terminal error/backpressure sends currently ignore timeout +when the two-slot transport handoff stays full; closing sender can become clean +EOF. Reproduce before fixing terminal status outside the bounded data channel. +Also strengthen exact10000 replay proof to require LIVE, and add BOOK_DELTA +coverage (existing DR08 is TRADE-only). These are in-scope defects, not measured +cause of every current overflow. Runtime trace expiry is1791401054410448004: +new SDK/apply/ACK spans cannot be acquired read-only on that expired hook; retain +NOT_MEASURED rather than rearming/restarting writer without a runtime packet. + +DR08 progress: report upgrade/evidence/p184-dr08-continuity-closure.md links raw +bounded evidence and Claude handoff. Python43PASS; real isolated TRADE DR08 +4development checksPASS, exact10000->LIVE->marker,5restarts p505516.627/max5545.290ms. +This is test-only input/actualKafka+gRPC, not TS final evidence. Passive canonical +1465/cache720 joins380unique noidentity mismatch; ETH delta external visibility +p505033/4556ms remains an open consumer path finding. All6trace inner boundaries +not present are explicitly NOT_MEASURED, not filled using cache GET timestamps. +No shared changes or TS edits. DR08 stack removed, cleanup errors0; disk values +and named rollback retention in report. Rust terminal-status regression pending. + +Rust counterfactual: both prolonged-stall regressions fail on old service in +10.31s (terminal error missing). Candidate39library+37nativePASS0fail0ignored, +including BOOK_DELTA suffix replay and peer independence. These tests invoke +service directly, not Kafka/wire. Binaryb5e1c2c retained, target1480911808bytes +removed; free45976342528->47115079680 (concurrent activity). Source changed, so +reuse existing image with development-only hashed candidate executable for +isolated DR08 wire validation; no new image or runtime rollout. Harness44PASS, +including development-only overlay/size/hash checks. Final image qualification +cannot use this option. Report/runbook updated before wire experiment. + +DR08 tested slice status IMPLEMENTED_TESTED_ISOLATED / TRACE_PARTIAL, not full +three-step runtime closure. Final harness46PASS; Rust39library+37nativePASS, +0fail0ignored, oldsource2regressionsFAIL. Candidate binaryb5e1c2c actual isolated +Kafka/gRPC4development checksPASS;5restarts p505400.475/max5542.696ms, expired +backlog->new snapshot->firstevent7063.953ms(n1). BOOKDELTA prolongedstall/suffix +replay/fastpeer are direct-service tests, not full TS+Redis crash acceptance. +Receipt41da3328, cleanupd235cadc, noownedcontainers/privatefiles/temporarycopy. +Sharedruntimeunchanged=true. Source/interface/buffer/freshness unchanged except +terminal error preservation; replay proof strengthened rather than weakening gate. +Inventory95images28.96GB/cache467entries20.52GB; no images built/pruned. Exact +Cargo target1.48GB removed, one121.77MB review binary retained. Disk pre/post in +report; concurrent activity precludes attribution. Agent/observers finished. +Runtime pending: SDK2.0.7 writer has expired tracing; canonical/cache join is not +SDK/apply/ACK/cursor trace. ETH BOOK delay remains open alongside TS IDR081. +Canonicalmain5bf5cd5/v2.2.2, featurefix/p184-dl-closure only; active digests and +BARrollbackf7351c3b untouched. No TS edits/push/merge/deploy/release; v2.2.3 HOLD. diff --git a/rust/qdl-stream-gateway/src/service.rs b/rust/qdl-stream-gateway/src/service.rs index 16b0d934..7fe13da7 100644 --- a/rust/qdl-stream-gateway/src/service.rs +++ b/rust/qdl-stream-gateway/src/service.rs @@ -470,6 +470,7 @@ impl MarketDataStreamService for Gateway { subscription: subscription.clone(), }; let (sender, receiver) = mpsc::channel(OUTBOUND_HANDOFF); + let terminal = Arc::new(Mutex::new(None)); let auth = Authorized::new( state.authority.clone(), &access, @@ -487,6 +488,7 @@ impl MarketDataStreamService for Gateway { _registered: registered, subscription, sender, + terminal: terminal.clone(), claims, request_token: body.cursor_token, auth, @@ -496,7 +498,19 @@ impl MarketDataStreamService for Gateway { }; tokio::spawn(task.run(registration.barrier, ring)); Ok(Response::new(Box::pin( - ReceiverStream::new(receiver).map(|(item, _charge)| item), + ReceiverStream::new(receiver) + .map(|(item, _charge)| item) + // Drain the bounded handoff in order, then emit the terminal + // status even if a stalled receiver outlived the producer task. + .chain(tokio_stream::once(()).map(move |_| { + Err(terminal + .lock() + .ok() + .and_then(|mut status| status.take()) + .unwrap_or_else(|| { + Status::internal("subscription task ended without a terminal status") + })) + })), ))) } @@ -930,6 +944,7 @@ struct SubscriptionTask { _registered: Registered, subscription: Arc, sender: mpsc::Sender>, + terminal: Arc>>, claims: CursorV3Claims, request_token: String, auth: Arc, @@ -1022,11 +1037,11 @@ impl SubscriptionTask { async fn fail(&self, status: Status) { self.outcome(status.code().description()); - let _ = tokio::time::timeout( - Duration::from_secs(5), - self.sender.send((Err(status), None)), - ) - .await; + // One terminal slot cannot be blocked by the data channel. Dropping + // the task closes that channel and wakes its receiver to drain it. + if let Ok(mut terminal) = self.terminal.lock() { + terminal.get_or_insert(status); + } } /// End the stream with the public BACKPRESSURE contract. diff --git a/rust/qdl-stream-gateway/tests/native_stream.rs b/rust/qdl-stream-gateway/tests/native_stream.rs index fe2a4cc0..095b05e1 100644 --- a/rust/qdl-stream-gateway/tests/native_stream.rs +++ b/rust/qdl-stream-gateway/tests/native_stream.rs @@ -1389,6 +1389,76 @@ async fn t05_a_slow_lossless_reader_ends_typed_and_resources_return_to_baseline( assert!(harness.state().metrics.overflow.load(Ordering::Relaxed) >= 1); } +async fn prolonged_lossless_stall(feed: &str, key: &str, event: fn(u64) -> EventEnvelope) { + let mut options = Options::default(); + options.limits.slow_consumer_after = Duration::from_millis(200); + let harness = Harness::new(options); + let cursor = harness.log.append(0, key, &event(1)); + harness.settle(); + let mut slow = harness.subscribe(feed, None, cursor, 10).await.unwrap(); + let mut fast = harness.subscribe(feed, None, cursor, 100).await.unwrap(); + slow.until_live().await; + fast.until_live().await; + let mut expected = Vec::new(); + for sequence in 2..32 { + let offset = harness.log.append(0, key, &event(sequence)) as u64; + expected.push(offset); + assert_eq!(fast.collect(1).await.1, vec![offset]); + } + // Keep the slow receiver unpolled until its task releases its slot. The + // wait budget covers both legacy notification deadlines; the peer stays open. + let deadline = Instant::now() + Duration::from_secs(12); + while harness.state().open_streams() != 1 && Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!( + harness.state().open_streams(), + 1, + "stalled task releases its slot" + ); + assert_eq!(harness.state().metrics.overflow.load(Ordering::Relaxed), 1); + let mut received = Vec::new(); + loop { + match slow.next().await { + Item::Event(offset, _) => received.push(offset), + Item::Control(_) => {} + Item::Error(code, message) => { + assert_eq!(code, Code::ResourceExhausted); + assert!(message.starts_with("RATE_LIMITED:"), "{message}"); + break; + } + Item::End => panic!("{feed}: stalled receiver ended without a terminal status"), + } + } + assert!(!received.is_empty(), "queued handoff records are preserved"); + assert!( + expected.starts_with(&received), + "handoff order is preserved" + ); + assert!( + slow.0.next().await.is_none(), + "terminal status is emitted once" + ); + let after = *received.last().unwrap() as i64; + let mut resumed = harness.subscribe(feed, None, after, 100).await.unwrap(); + assert_eq!(resumed.until_live().await, expected[received.len()..]); + let next = harness.log.append(0, key, &event(32)) as u64; + assert_eq!(fast.collect(1).await.1, vec![next]); + assert_eq!(resumed.collect(1).await.1, vec![next]); + drop((slow, fast, resumed)); + harness.baseline().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn t05_prolonged_trade_stall_preserves_terminal_status_and_peer_progress() { + prolonged_lossless_stall("TRADE", TRADE_KEY, trade).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn t05_prolonged_book_delta_stall_preserves_terminal_status_and_peer_progress() { + prolonged_lossless_stall("BOOK_DELTA", BOOK_KEY, |sequence| delta(sequence, false)).await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn t05_the_replica_byte_budget_bounds_all_queues_together() { // A budget for about three records: a stalled reader cannot hold more. diff --git a/scripts/p184_dr08_isolated.py b/scripts/p184_dr08_isolated.py index 3394441b..293fea2b 100644 --- a/scripts/p184_dr08_isolated.py +++ b/scripts/p184_dr08_isolated.py @@ -138,6 +138,10 @@ def validate_packet(packet, *, development): raise ValueError(f"immutable {role} image ID required") restart_count(packet) validate_query_provenance(packet) + if packet.get("stream_binary_override"): + declared = packet["stream_binary_override"] + if declared != candidate_stream_provenance("/work/candidate-stream", development=development): + raise ValueError("candidate binary digest mismatch") if not development: sha, image = packet.get("ts_source_sha", ""), packet.get("ts_image_digest", "") if not re.fullmatch(r"[0-9a-f]{40}", sha) or len(set(sha)) == 1: @@ -151,12 +155,15 @@ def validate_packet(packet, *, development): def evidence_rows(packet, facts, *, development, run_id): """Development rows deliberately cannot satisfy the TS matrix checker.""" validate_query_provenance(packet) + if packet.get("stream_binary_override") and not development: + raise ValueError("binary overlay cannot emit qualification evidence") mode = "DEVELOPMENT" if development else "QUALIFICATION" header = {"provenance": { "schema": SCHEMA, "source_sha": packet["ts_source_sha"], "image_digest": packet["ts_image_digest"], "mode": mode, "run_id": run_id, "data_provenance": "SYNTHETIC_TEST_ONLY", "images": packet["images"], "query_provenance": packet.get("query_provenance", {}), + "stream_binary_override": packet.get("stream_binary_override"), "packet_sha256": hashlib.sha256(json.dumps(packet, sort_keys=True).encode()).hexdigest(), "scope": "same-host isolated DL fault test; not TS consumer or real-provider certification", }} @@ -483,6 +490,25 @@ def wait_seed(packet): consumer.close() +def replay_boundary(start, consumer, token, producer, baseline_id): + """A full replay must reach LIVE before a new marker proves handoff.""" + child = start(consumer, token, stop_after=REPLAY + 1) + try: + live = child.wait("live", 90) + if live.get("count") != REPLAY: + raise ValueError("exact replay did not reach LIVE at the expected boundary") + producer.publish(1) + done = child.wait("done") + passed = (done.get("code") == "OK" and done.get("count") == REPLAY + 1 + and done.get("last") == baseline_id + REPLAY + 1 + and "LIVE" in done.get("controls", [])) + if not passed: + raise ValueError("post-replay live marker missing or out of order") + return {**done, "replay_count_at_live": live["count"], "live_marker_count": 1} + finally: + child.close() + + def run_faults(packet): facts, children = {}, [] stage = "baseline_metrics" @@ -536,11 +562,8 @@ def start(consumer, token, **kwargs): facts[CELLS[0]] = {"observed": True, "passed": False, "configured_queue_events": QUEUE, "slow_terminal": slow_end, "seed_commit": seed} stage = "exact_10000_replay" - replay = start(slow_id, old_token, stop_after=REPLAY) - replay_end = replay.wait("done", 90) - replay.close() - exact_replay = replay_end["code"] == "OK" and replay_end["count"] == REPLAY and replay_end["last"] == baseline_id + REPLAY - producer.publish(1) + replay_end = replay_boundary(start, slow_id, old_token, producer, baseline_id) + exact_replay = replay_end["replay_count_at_live"] == REPLAY fast_end = fast.wait("done") fast.close() stage = "out_of_range" @@ -559,7 +582,7 @@ def start(consumer, token, **kwargs): recovery = measure_restarts(packet, producer, start, fresh_token, fresh_trade_id) facts[CELLS[1]] = {"observed": True, "passed": exact_replay and out_of_range and fresh_offset > old_offset, - "replayed": replay_end["count"], "range_code": range_end["code"], + "replayed": replay_end["replay_count_at_live"], "replay_handoff": replay_end, "range_code": range_end["code"], "old_snapshot_offset": old_offset, "fresh_snapshot_offset": fresh_offset, "baseline_trade_id": baseline_id, "fresh_snapshot_trade_id": fresh_trade_id, **recovery, "restart_kind": "consumer_process", @@ -879,7 +902,23 @@ def serve_query(work): hot.close() -def orchestrate(work, *, development, source_sha, image_digest, restart_samples=None): +def candidate_stream_provenance(path, *, development): + if path is None: + return None + if not development: + raise ValueError("binary overlay is development-only, not an image qualification") + path = Path(path).resolve(strict=True) + if not path.is_file() or not 0 < path.stat().st_size <= 512 * 1024 * 1024: + raise ValueError("bounded candidate binary required") + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return {"sha256": digest.hexdigest(), "bytes": path.stat().st_size, + "container_path": "/work/candidate-stream", "scope": "development binary overlay; base image unchanged"} + + +def orchestrate(work, *, development, source_sha, image_digest, restart_samples=None, stream_binary=None): """Adapt measure_l2_top_materialization's isolated broker/core packet.""" import shutil import subprocess @@ -893,6 +932,7 @@ def orchestrate(work, *, development, source_sha, image_digest, restart_samples= raise ValueError("final TS image digest required before setup") count = restart_count({"development": development, **( {"restart_samples": restart_samples} if restart_samples is not None else {})}) + candidate = candidate_stream_provenance(stream_binary, development=development) work = work.resolve() work.mkdir(mode=0o700, parents=True, exist_ok=False) namespace = "p184-dr08-" + uuid.uuid4().hex[:16] @@ -931,6 +971,13 @@ def inventory(): "ts_image_digest": image_digest, "images": IMAGES, "bootstrap": "localhost:9092", "restart_samples": count, "query_provenance": {"image_ref": QUERY_IMAGE_REF, "scope": "isolated candidate Query; not production v2.2.2"}} + if candidate: + shutil.copyfile(stream_binary, work / "candidate-stream") + (work / "candidate-stream").chmod(0o555) + copied = candidate_stream_provenance(work / "candidate-stream", development=True) + if copied != candidate: + raise ValueError("candidate changed during staging") + packet["stream_binary_override"] = candidate (work / "packet.json").write_text(json.dumps(packet)) def launch(role, image, args, *, memory, cpus, network, env=None, detach=True, extra=()): name = namespace + "-" + role @@ -983,7 +1030,7 @@ def launch(role, image, args, *, memory, cpus, network, env=None, detach=True, e memory="256m", cpus=.5, network=net, env={"QDL_KAFKA_CERT_ROOT": "/work", "QDL_KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", "QDL_KAFKA_CLIENT_ID": namespace, "QDL_KAFKA_GROUP_ID": namespace, "QDL_PROVIDER_ADMISSION_ENABLED": "false"}) - stream = launch("stream", IMAGES["stream"], ["/usr/local/bin/qdl-stream-gateway", "serve"], + stream = launch("stream", IMAGES["stream"], ["/work/candidate-stream" if candidate else "/usr/local/bin/qdl-stream-gateway", "serve"], memory="512m", cpus=.5, network=net, env={**common, "QDL_KN_BUNDLE_FILE": "/work/bundle.json", "QDL_KN_JWT_CONFIG_FILE": "/work/jwt-config.json", "QDL_KN_QUOTA_REDIS_URL": "redis://localhost:6379/0", "QDL_KN_QUOTA_PREFIX": "qdl:stable:v2:paper:" + namespace, @@ -1046,6 +1093,7 @@ def launch(role, image, args, *, memory, cpus, network, env=None, detach=True, e except (subprocess.CalledProcessError, subprocess.TimeoutExpired): if command(["docker", "ps", "-aq", "--filter", "name=^/" + name + "$"]): cleanup_failures.append(name) + (work / "candidate-stream").unlink(missing_ok=True) for filename in ("client.key", "server.p12", "slow.key", "fast.key", "cursor-keys.json", "read-view.secret", "broker.properties"): (work / filename).unlink(missing_ok=True) @@ -1072,6 +1120,7 @@ def main(argv=None): parser.add_argument("--packet", type=Path) parser.add_argument("--output", type=Path) parser.add_argument("--development", action="store_true") + parser.add_argument("--stream-binary", type=Path, help="development-only isolated candidate executable") parser.add_argument("--restart-samples", type=int, help="orchestrator restart count: development 1-10, final 5-10") args = parser.parse_args(argv) @@ -1081,11 +1130,13 @@ def orchestration_deadline(_signum, _frame): previous = signal.signal(signal.SIGALRM, orchestration_deadline) signal.alarm(600) try: - orchestrate(args.orchestrate, development=args.development, source_sha=args.ts_source_sha, image_digest=args.ts_image_digest, restart_samples=args.restart_samples) + orchestrate(args.orchestrate, development=args.development, source_sha=args.ts_source_sha, image_digest=args.ts_image_digest, restart_samples=args.restart_samples, stream_binary=args.stream_binary) finally: signal.alarm(0) signal.signal(signal.SIGALRM, previous) return 0 + if args.stream_binary is not None: + parser.error("--stream-binary requires --orchestrate") if args.restart_samples is not None: parser.error("--restart-samples requires --orchestrate; inner driver uses its sealed packet") for directory, function in ((args.prepare_material, prepare_material), (args.prepare_topics, prepare_topics), (args.query, serve_query)): diff --git a/tests/test_p184_dr08_isolated.py b/tests/test_p184_dr08_isolated.py index 66dcc059..4376c492 100644 --- a/tests/test_p184_dr08_isolated.py +++ b/tests/test_p184_dr08_isolated.py @@ -24,6 +24,75 @@ def packet(development=True): role: "sha256:" + "abcdef01" * 8 for role in ("core", "stream", "kafka", "python")}}} +class ReplayBoundary(unittest.TestCase): + def test_reaches_live_then_receives_new_marker(self): + from unittest.mock import Mock + child, producer, start = Mock(), Mock(), Mock() + start.return_value = child + def wait(kind, *args): + if kind == "live": + producer.publish.assert_not_called() + return {"count": 10000} + producer.publish.assert_called_once_with(1) + return {"count": 10001, "last": 10003, "code": "OK", "controls": ["REPLAYING", "LIVE"]} + child.wait.side_effect = wait + result = R.replay_boundary(start, "slow", "old", producer, 2) + self.assertEqual(result["replay_count_at_live"], 10000) + start.assert_called_once_with("slow", "old", stop_after=10001) + child.close.assert_called_once() + + def test_receiving_limit_without_live_is_not_success(self): + from unittest.mock import Mock + child, producer = Mock(), Mock() + child.wait.side_effect = RuntimeError("done before live: OUT_OF_RANGE") + with self.assertRaises(RuntimeError): + R.replay_boundary(Mock(return_value=child), "slow", "old", producer, 2) + producer.publish.assert_not_called() + child.close.assert_called_once() + + def test_wrong_replay_count_and_bad_live_marker_fail_closed(self): + from unittest.mock import Mock + for live, end in ((9999, {}), (10000, {"count": 10001, "last": 10004, "code": "OK", "controls": ["LIVE"]}), + (10000, {"count": 10001, "last": 10003, "code": "OK", "controls": []})): + with self.subTest(live=live, end=end): + child, producer = Mock(), Mock() + child.wait.side_effect = [{"count": live}, end] + with self.assertRaises(ValueError): + R.replay_boundary(Mock(return_value=child), "slow", "old", producer, 2) + child.close.assert_called_once() + + +class CandidateBinary(unittest.TestCase): + def test_overlay_cannot_be_relabelled_as_final(self): + p = packet(False) + p["stream_binary_override"] = {"sha256": "a" * 64} + with self.assertRaisesRegex(ValueError, "qualification"): + R.evidence_rows(p, {}, development=False, run_id="forged") + + def test_inner_driver_checks_actual_binary(self): + p = packet() + p["stream_binary_override"] = {"sha256": "a" * 64} + with patch.dict(os.environ, {"P184_DR08_ISOLATED_NAMESPACE": p["namespace"]}), \ + patch.object(R, "candidate_stream_provenance", return_value={"sha256": "b" * 64}): + with self.assertRaisesRegex(ValueError, "digest mismatch"): + R.validate_packet(p, development=True) + + def test_overlay_is_development_only_and_hashed(self): + import hashlib + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "candidate" + p.write_bytes(b"test-only-binary") + result = R.candidate_stream_provenance(p, development=True) + self.assertEqual(result["sha256"], hashlib.sha256(p.read_bytes()).hexdigest()) + self.assertEqual(result["bytes"], 16) + with self.assertRaises(ValueError): + R.candidate_stream_provenance(p, development=False) + p.write_bytes(b"") + with self.assertRaises(ValueError): + R.candidate_stream_provenance(p, development=True) + self.assertIsNone(R.candidate_stream_provenance(None, development=False)) + + class Guards(unittest.TestCase): def setUp(self): self.p = packet() diff --git a/upgrade/evidence/p184-claude-stage3-alignment.md b/upgrade/evidence/p184-claude-stage3-alignment.md index 62d84f1b..3d43cac4 100644 --- a/upgrade/evidence/p184-claude-stage3-alignment.md +++ b/upgrade/evidence/p184-claude-stage3-alignment.md @@ -126,3 +126,14 @@ validation, Binance bootstrap semantics, typed immediate refusal rather than waiting for stale, and honest final SHA/image provenance. Do not mark final DR05 PASS from the diagnostic helper. The DL fixes have not been deployed or packaged as the final consolidated release. No TS source was edited. + + +## 2026-10-08 Continuity Follow-Up (Codex) + +Owner approved passive attribution + isolated DR08 corrections. See +[p184-dr08-continuity-closure.md](p184-dr08-continuity-closure.md). Current writer +5849d716 still SDK2.0.7; its trace expired. No TS edits/restarts. IDR081 remains +Claude-owned; new paired evidence also finds ETH BOOK_DELTA cache delay while +BOOK_SNAPSHOT is timely, so BTC colocation does not close the whole finding. +Need same-event SDK/apply/ACK/cursor capture on intended writer before runtime +closure. New Rust terminal-status fix is isolated, not proof of stall root cause. diff --git a/upgrade/evidence/p184-dr-runtime-runbook.md b/upgrade/evidence/p184-dr-runtime-runbook.md index c9d1a12e..5ec8a4f5 100644 --- a/upgrade/evidence/p184-dr-runtime-runbook.md +++ b/upgrade/evidence/p184-dr-runtime-runbook.md @@ -128,6 +128,23 @@ no private keys, unchanged shared image/start/restart inventory and disk pre/pos Keep active images and bar-edge rollback `f7351c3b`; no broad prune. Final TS provenance requires a fresh run, not editing the development JSONL. +### Continuity Regression Addendum (2026-10-08) + +Exact replay10000 now requires the actual LIVE control with count10000 and a +subsequent newly published live marker. Receiving10000 then cancelling is not +successful replay-to-live evidence. TRADE wire runner is not BOOK qualification. +Prolonged TRADE/BOOK_DELTA handoff stalls are also covered by native direct-service +regressions: terminal error survives a full data channel, fast peer advances, +ordered suffix replay succeeds and task resources return to baseline. + +For a changed Rust executable only, development orchestration accepts +`--stream-binary `; it copies a bounded executable into +the owned work directory, records SHA256/bytes separately from the base image, +verifies it inside the driver, and removes the copy during cleanup. This option +is forbidden in final qualification. It does not claim an image rebuild or +allow shared rollout. Keep the original binary and hashes for reviewer replay. +See p184-dr08-continuity-closure.md for counterfactual, wire and trace limits. + ## Release Boundary v2.2.3 remains HOLD for final DR01-DR08/broker evidence, independent of building diff --git a/upgrade/evidence/p184-dr01-dr08-trade-attribution.md b/upgrade/evidence/p184-dr01-dr08-trade-attribution.md index f4e54107..3c484f54 100644 --- a/upgrade/evidence/p184-dr01-dr08-trade-attribution.md +++ b/upgrade/evidence/p184-dr01-dr08-trade-attribution.md @@ -1,5 +1,16 @@ # P18.4 DR01/DR08 TRADE Incident Attribution +## Open Incident Register: 2026-10-08 Continuity Follow-Up + +DL-DR08-CONTINUITY: IN_PROGRESS, linked TS IDR-078/081 (Claude-owned). +Approved passive attribution and isolated correction, not rollout. Prior window: +overflow+59/replay+84263/expired+4 over approximately15min, including BOOK_DELTA +(~7.9MB queue) and TRADE (1000 queue,10000 replay,OUT_OF_RANGE). Cache600/600 +present is not usable: BOOK_DELTA source-age max8288.56ms. Historical observations, +not same-event joins or proven cause. Runtime writer5849d716/Stream1f13408f lack +pending worker/counter patches. Preserve unavailable spans and distinguish +lossless recovery from snapshot resynchronization. DR08/release remain OPEN. + Date: 2026-10-08. Status: READONLY_ATTRIBUTION_COMPLETE / INCIDENT_WINDOW_RECOVERY_NOT_MEASURED / PROPOSALS_WITHDRAWN. Owner update below supersedes the original proposal text. diff --git a/upgrade/evidence/p184-dr08-continuity-closure.md b/upgrade/evidence/p184-dr08-continuity-closure.md new file mode 100644 index 00000000..44eb241a --- /dev/null +++ b/upgrade/evidence/p184-dr08-continuity-closure.md @@ -0,0 +1,134 @@ +# DR08 Continuity Attribution And Recovery Hardening + +Status IMPLEMENTED_TESTED_ISOLATED / TRACE_PARTIAL / NOT_RUNTIME_CLOSED, 2026-10-08. Owner scope is three steps: +freeze source/runtime, bounded passive attribution, isolated DR08 correction. +Main journal: DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md, DR08 continuity entry. +No TS edits, shared rollout, new TS subscription, orders, push/merge or release. + +## Runtime And Ownership + +Canonical DL main5bf5cd5/v2.2.2; feature fix/p184-dl-closure base d68fb2f. +Claude TS source8b41608 owns OR plus IDR078/081; do not amend his worktree. +Writer sha256:5849d7167561273af08bda2103c38cec5f4480f7a8809ea592c57abff7716ea1, +SDK2.0.7, started2026-10-07T19:15:49Z. Stream replicas image +sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9. +Both Stream and writer restart0/OOMfalse. Core658a9570, Query/BAR51450612, +projectora696cfba unchanged. Pending worker logging/offered/dequeued source is +not this deployed pair. TS trace absolute expiry1791401054410448004 is past; +read-only inspection cannot reactivate SDK/apply/ACK capture. No silent restart. + +## Incident And Passive Trace + +Previous read-only finding retained: about15min overflow+59/replay+84263/expired+4; +TRADE queue1000/replay10000/OUT_OF_RANGE; BOOK_DELTA ~7.9MB exhausted. TS cache +600/600present, delta source-age max8288.56ms. Not equivalent to usable data. +New bounded passive trace uses existing readonly_kafka_observer.py --mode canonical +--seconds90 --max-records2000 --sample-ms100 --launch, assigned read_committed, +no group join/commit. Cache MGET four ETH BOOK keys every500ms for180rounds, +no subscriptions, payloads, writes or key changes. Source identities/offsets only. +Evidence root ~/.local/state/qdl-v2/dr08-continuity-20261008/. +1465canonical observations,720cache reads,380unique matched product/source/ +authority/timestamp/offset events,0identity mismatch. Generation independently +matches all596joined poll rows (repeated offsets included). Native sequence uses +different encoded representations; no assertion of scalar sequence equality. + +| Feed/venue | Unique n | Event -> external cache observation p50/p95/max ms | +| --- | ---: | --- | +| BOOK_SNAPSHOT Binance ETH |83|704.880 /1028.558 /1228.310| +| BOOK_SNAPSHOT OKX ETH |85|609.930 /1116.752 /1419.858| +| BOOK_DELTA Binance ETH |101|5033.440 /9615.223 /9971.622| +| BOOK_DELTA OKX ETH |111|4555.829 /7690.110 /8554.624| + +This is first external visibility sampled within the window, NOT Redis ACK, +canonical commit or execution-eligibility proof. Initial sampling and polling +add delay. Canonical observer -> cache poll delta p50 is4830.661/4329.095ms +for Binance/OKX delta, respectively. It narrows the path after durable visibility +but does not separate transport, SDK and TS processing. It also shows why BTC +worker ownership alone cannot explain all ETH BOOK observations. + +Stream aggregate interval13:31:31->13:37:31UTC:258943delivered,0new overflow, +0replay/expired. Queue-wait buckets all<=50ms, handoff buckets all<=20ms; +producer CreateTime->dispatch all<=200ms. These are aggregate bucket bounds, +NOT exact subscription/ACK trace or a retroactive explanation for prior errors. +Unmeasured on current writer: Stream per-event enqueue/dequeue, SDK return, +RedisACK, cursor persist. No production root-cause closure from these aggregates. + +Evidence hashes: +- runtime-observation-01.json:2feb47115d7f0f17ba00ea5f10d5cf78d2e58c36098590ae688e546b779cb15b +- runtime-observation-02.json:db7b5c272c21baaad50603d2dea0b9a05467f820039477cc8684cb8a0e5ae537 +- cache-eth-paired-02.json:6beaa8be443900bfbc6ddcc5685612e09f8a4b7dc75c8b5fa5d322fbe6945295 +- paired-join-02.json:8ffdce5206d9ca5c4f2fda56b2c50c649990757c3a1d7365efd31d28b034cc0d + +## Isolated DR08 Reproduction + +Fixed a harness false-positive: stopping at10000 replay events did not prove +LIVE. Now require LIVE with exactly10000 events before publishing marker10001, +then prove its ordered delivery; backlog10001 separately returns OUT_OF_RANGE. +Unit43PASS0fail0skip (python3 -B -m unittest discover -s tests -p +test_p184_dr08_isolated.py -v). Runtime --orchestrate replay-boundary-01, +--development --restart-samples5 with dummy TS provenance (never final evidence): +all4development checks PASS on existing Stream image, candidate Queryrc2 explicitly +named. Queue observed1000/4719110bytes, sampled aggregate peak4955925bytes; +independent fast consumer10001events ordered,64progress observations while slow +stopped. Exact10000->LIVE->marker PASS. OUT_OF_RANGE->snapshot->firstevent +7347.908ms(n1); five independent consumer restarts p50 5516.627/max5545.290ms. +These are gRPC receive boundaries, NOT TS apply/RedisACK. Input SYNTHETIC_TEST_ONLY; +not provider, production, BOOK or lossless recovery after expired backlog proof. + +New Rust finding: a full outbound handoff can time out sending both control and +terminal status; old task can exit with clean EOF. Narrow candidate preserves +one terminal status outside the bounded data channel and drains prior data in +order. Both new prolonged TRADE/BOOK_DELTA stall regressions failed before the change +(10.31s, missing terminal), then passed. Final Rust suites39library+37native +PASS0fail0ignored,0.17s/12.60s. Direct-service tests, not prolonged-stall wire +proof. Test log ../dr08-terminal-20261008/final-tests.log SHA256 +0b32adeb22227569702487378fb6b7e1482a1f49812d252b3e775ad997ba99f6. +Final harness46PASS0fail0skip; extra negatives forbid relabelling binary overlay +as final qualification and detect changed executable bytes. + +Candidate executableb5e1c2c48d4009e1a3861bcb7e80554afb89c3d24e38ebe9fced3860dfb01325 +was copied into a second isolated stack (candidate-wire-02), existing immutable +base image unchanged. Actual Kafka/core/gRPC DR08 all4development checks PASS: +queue1000/4719110bytes, sampled aggregate peak4941708bytes, fast10001events ordered +and69progress observations while slow stopped, replay10000->LIVE->new marker, +10001 backlog->OUT_OF_RANGE. Five restarts p50 5400.475/max5542.696ms; +range->snapshot->firstevent7063.953ms(n1). No TS RedisACK measurement. +Wire receipt41da3328df76b7d8ffa15cd533f0b310842fd80fa01df85f545a63a777188085. +This tests TRADE gRPC; BOOK_DELTA prolonged stall remains direct-service proof. +The two last harness-only negative checks do not change the wire workload. +Do not claim this source defect caused all observed runtime stalls. + +## Cleanup And Remaining Work + +DR08 cleanup shared_runtime_unchanged=true, failures0, all8owned container names +retired, private material removed by orchestrator. No images built/removed. +Disk free53510434816->52509847552bytes; concurrent host activity, not attributable +to this run. Both passive observers finished. Active/rollbackBARf7351c3b retained. +Candidate stack cleanup unchanged=true/errors0; all8owned names retired, +private files and staging executable removed. Cleanup receipt + d235cadc1161b4e88ac462e27db706de5b73b6f18e07c9be6e536d90a2a03ff6. +Candidate stack free45993418752->45420568576bytes (concurrent host activity). +Agent removed exact private Cargo target1480911808bytes; free45976342528 -> +47115079680bytes. Retained one121770368byte candidate executable for review, +not a production image. Docker inventory95images28.96GB/cache467entries20.52GB; +no test images built, no shared prune during Claude's builds. No dr08 containers +remain. Agent closed; observer/driver processes completed. + +Claude handoff: retain IDR081, but trace ETH BOOK_DELTA as well as BTC TRADE on +the intended writer. New worker logs and bounded SDK->ACK trace must run together +with canonical/Stream counters; current expired trace cannot supply those spans. +No ownership repartition, timestamp relaxation, larger buffers or DATA_STALE +relabeling is implemented here. DR08 final and v2.2.3 remain HOLD. + + +## Three-Step Exit Accounting + +1. Finding/runtime freeze DONE; incident linked to TS078/081 without touching TS. +2. Passive trace PARTIAL:380exact canonical/cache joins, but current writer lacks +active SDK/apply/ACK/cursor capture. No complete seven-stage causal attribution. +3. Narrow recovery defect IMPLEMENTED_TESTED_ISOLATED; production backlog root +cause and full TS crash-between-apply/ACK evidence remain OPEN. Existing direct +service replay tests do not replace writer/Redis crash recovery acceptance. +Next permitted coordination: Claude's intended writer logging/trace (bounded +activation in approved writer packet), then exact same-event join and affected +runtime acceptance; no C2/full catalogue/strategy retest. No new freshness gate. From 5b67da86bb771bf0bc429ab2db837c0e48d0aeae Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 15:49:54 +0000 Subject: [PATCH 039/147] Require current recovery fixes in consolidated DL build preflight --- .github/workflows/ci.yml | 5 ++- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 41 +++++++++++++++++++++ scripts/p184_consolidated_build.py | 17 ++++++++- tests/test_p184_consolidated_build.py | 27 +++++++++++++- upgrade/evidence/p184-consolidated-build.md | 22 ++++++++++- upgrade/evidence/p184-dl-phase-c-packet.md | 21 +++++++++++ 6 files changed, 125 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43de8799..31ef9f69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,13 +55,14 @@ jobs: assert recipe["build_script_sha256"] is None assert recipe["prerequisites"] == PREREQUISITES assert recipe["gates"] == { - "ts_tests": "pending", "ts_source_commit": None, + "ts_tests": "pending", "ts_tests_scope": "MARKET_DATA_INTEGRATION_ONLY", + "ts_source_commit": None, "ts_receipt_sha256": None, "dl_ci": "pending", "dl_ci_source_commit": sha, "dl_ci_receipt_sha256": None, "build_approval": "pending", } assert recipe["holds"] == [ - "TS tests must finish and pass before any rebuild", + "TS market-data integration regressions must pass; Paper/OR feature completion is not required", "DL CI must pass for the exact source commit", "explicit build approval required", ] diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index dec4fbc8..4825bbbc 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70506,3 +70506,44 @@ Runtime pending: SDK2.0.7 writer has expired tracing; canonical/cache join is no SDK/apply/ACK/cursor trace. ETH BOOK delay remains open alongside TS IDR081. Canonicalmain5bf5cd5/v2.2.2, featurefix/p184-dl-closure only; active digests and BARrollbackf7351c3b untouched. No TS edits/push/merge/deploy/release; v2.2.3 HOLD. + +### Owner amendment: 100 alpha plus TS capacity and independent DL closure - 2026-10-08 + +Status IN_PROGRESS, approved by owner in current conversation. Target is100alpha +logical consumers with2-5declared data products each PLUS TS60, not100idle sockets. +Optimize before sizing, but the previous fixed <=5vCPU target is no longer an +absolute acceptance ceiling: owner permits adequate CPU/RAM and quota for the +measured workload. Preserve feed freshness/integrity/LOSSLESS and bounded memory; +no unlimited buffers, wildcard entitlement, fabricated evidence or hidden skips. +Reconcile independent identities versus logical sessions and per-subscription +fanout; quota admission is not capacity proof. Freeze workload/measurement and +headroom assumptions before a load run. Report throughput, RAM, CPU/throttle, +queue, recovery and actual consumer apply latency under sustained/burst traffic. +Guides: p184-consolidated-build.md, p184-dr-runtime-runbook.md, +p184-dr05-fence-channel-handoff.md and approved Kafka-native architecture review. +Do not wait for TS Paper/OR feature completion. Claude retains exclusive TS edits; +Codex may run existing TS helper against disposable DL/TS clone namespaces only. +Prepare resource/identity candidate and scoped tests; no shared role change until +exact digest/config/rollback packet is recorded and its safety checks pass. +Immediate parallel work: capacity100 workload/quota design using existing planner; +DR05 actual WriterClone integration on existing approved interfaces. Existing +0b374bc recovery proof inherited, only affected tests rerun. Runtime currently +v2.2.2 unchanged, v2.2.3 HOLD; no push/merge/order or broad cleanup. Disposable +stack labels, ports, state/credentials isolated; cleanup exact resources after run. +Apply-patch helper remains unavailable (mountinfo error previously reproduced), +manual edits use exact-match fallback with diff verification per workspace rule32. + +Build-preflight slice: require DR05 fence d68fb2f and terminal-status0b374bc ancestry +and source paths in consolidated recipe; keep old certified artifacts immutable. +Clarify TS gate is market-data integration regressions, not completion of Paper/OR. +Candidate build/CI preconditions remain explicit; final DR/broker runtime acceptance +still follows packaging. No fake CI attestations or build/release execution here. + +Build-preflight verification: portable unittest20PASS0fail0skip; exact inline CI +block rehearsed against committed0b374bc PASS with legitimate HOLD, not hosted CI. +Real old d68fb2f rejected for missing terminal-status ancestry. CI expectations, +guide and governing PhaseC packet updated atomically. No image/build/runtime +change; no test resources created by this slice, cleanup intentionally unnecessary. +Canonicalmain5bf5cd5/v2.2.2, active featurefix/p184-dl-closure. Active/runtime and +named rollback digests inherited unchanged from prior inventory, not remeasured. +Capacity and WriterClone agents remain independent work in progress; no TS edits. diff --git a/scripts/p184_consolidated_build.py b/scripts/p184_consolidated_build.py index a25f7b55..0459a967 100644 --- a/scripts/p184_consolidated_build.py +++ b/scripts/p184_consolidated_build.py @@ -15,9 +15,21 @@ "quota_log": "4eb08dc15113cc4f73b65b239dfeb4f8ec0e23cf", "b6a0119": "b6a01190587e8771c62b5b4988782b06c5e2578a", "IDR075": "704552d8736905cdb73feec86f3f01d339d02cbb", + "DR05_book_fence": "d68fb2ff47622db8224dc4b9e004dca4e20f643e", + "DR08_terminal_status": "0b374bc918af02476cdfb1502d5df132426f901b", } CONTRACTS = { "rust/qdl-realtime-core/src/lib.rs": ["REST_AHEAD_OF_STREAM"], + "rust/qdl-core/src/canonical.rs": [], + "rust/qdl-kafka/src/bin/qdl-production-core.rs": [], + "rust/qdl-projector/src/products.rs": [], + "qdl/runtime/kn_hot_view.py": [], + "qdl/runtime/kn_query_backend.py": [], + "rust/qdl-stream-gateway/src/service.rs": ["terminal.get_or_insert(status)"], + "rust/qdl-stream-gateway/tests/native_stream.rs": [ + "t05_prolonged_trade_stall_preserves_terminal_status_and_peer_progress", + "t05_prolonged_book_delta_stall_preserves_terminal_status_and_peer_progress"], + "tests/test_kn_hot_book_invalidation.py": [], "qdl/consumer/universal_release.py": ["qdl.v2.consumer-route-binding.v3"], "qdl/security/data_plane.py": ["qdl_consumer_quota_exhausted"], "rust/qdl-stream-gateway/src/subscription.rs": ["pub offered:", "pub dequeued:"], @@ -92,7 +104,7 @@ def prepare(repo, commit, release, *, ts_status="pending", ts_commit=None, blobs = source_contract(repo, commit) holds = [] if ts_status != "pass": - holds.append("TS tests must finish and pass before any rebuild") + holds.append("TS market-data integration regressions must pass; Paper/OR feature completion is not required") if dl_ci_status != "pass": holds.append("DL CI must pass for the exact source commit") if build_approval != "approved": @@ -136,7 +148,7 @@ def prepare(repo, commit, release, *, ts_status="pending", ts_commit=None, "source_commit": commit, "release": release, "dependency_image": dependency_image, "prerequisites": PREREQUISITES, "source_git_blobs": blobs, - "gates": {"ts_tests": ts_status, "ts_source_commit": ts_commit, + "gates": {"ts_tests": ts_status, "ts_tests_scope": "MARKET_DATA_INTEGRATION_ONLY", "ts_source_commit": ts_commit, "ts_receipt_sha256": ts_receipt, "dl_ci": dl_ci_status, "dl_ci_source_commit": commit, "dl_ci_receipt_sha256": dl_ci_receipt, "build_approval": build_approval}, @@ -147,6 +159,7 @@ def prepare(repo, commit, release, *, ts_status="pending", ts_commit=None, "Ancestry and source markers do not replace semantic regression tests.", "One application commit does not make upstream package resolution reproducible.", "No image built, pushed, deployed or certified by this generator.", + "Final DR/broker delta and consumer capacity acceptance remain required after packaging; no dependency on Paper/OR feature completion.", "IDR075 remains prepared, not activated; IDR040fix2 remains deferred.", ], } diff --git a/tests/test_p184_consolidated_build.py b/tests/test_p184_consolidated_build.py index 1e79aa91..060e5ac7 100644 --- a/tests/test_p184_consolidated_build.py +++ b/tests/test_p184_consolidated_build.py @@ -93,7 +93,30 @@ def test_dependency_reuse_is_python_only_not_source_overlay(self): def test_prerequisite_roster(self): self.assertEqual(set(build.PREREQUISITES), - {"IDR040fix1", "bindingV3", "quota_log", "b6a0119", "IDR075"}) + {"IDR040fix1", "bindingV3", "quota_log", "b6a0119", "IDR075", "DR05_book_fence", "DR08_terminal_status"}) + + +class IntegrationScopeTests(NoProcessTests): + def prepare(self, **kwargs): + with patch.object(build, "source_contract", return_value={"Dockerfile": SHA}): + return build.prepare(ROOT, SHA, "v2.2.3-rc.1", **kwargs) + + def test_scope_is_market_data_not_paper_feature_completion(self): + result = self.prepare(**READY) + self.assertEqual(result["gates"]["ts_tests_scope"], "MARKET_DATA_INTEGRATION_ONLY") + self.assertEqual(result["status"], "PREPARED_NOT_EXECUTED") + + def test_missing_new_recovery_prerequisite_refuses_old_candidate(self): + for missing in ("DR05_book_fence", "DR08_terminal_status"): + def fake(repo, *args): + if args[0] == "rev-parse": + return SHA + if args[0] == "merge-base" and args[2] == build.PREREQUISITES[missing]: + raise subprocess.CalledProcessError(1, "git") + return "" + with self.subTest(missing=missing), patch.object(build, "git", side_effect=fake): + with self.assertRaisesRegex(ValueError, missing): + build.source_contract(ROOT, SHA) class SourceTests(NoProcessTests): @@ -110,7 +133,7 @@ def test_missing_ancestor_fails_closed(self): build.source_contract(ROOT, SHA) def test_missing_marker_fails_closed(self): - with patch.object(build, "git", side_effect=[SHA] + [""] * 6): + with patch.object(build, "git", side_effect=[SHA] + [""] * (len(build.PREREQUISITES) + 1)): with self.assertRaisesRegex(ValueError, "source contract changed"): build.source_contract(ROOT, SHA) diff --git a/upgrade/evidence/p184-consolidated-build.md b/upgrade/evidence/p184-consolidated-build.md index 1aa5cb96..dbca5dd6 100644 --- a/upgrade/evidence/p184-consolidated-build.md +++ b/upgrade/evidence/p184-consolidated-build.md @@ -6,6 +6,24 @@ Main journal: [approved parent-owned scope](../../DATA_LAYER_UNIFIED_IMPLEMENTAT "Pre-stage3 consolidated build, SDK and DR05". Parent handles journal and commit; neither was edited by this slice. +## Governing Update: Independent Closure, 2026-10-08 + +The TS build gate means completed market-data integration regressions, NOT +completion of Paper/OR features. Final DR/broker acceptance and exact-source DL CI +remain mandatory; existing receipts do not certify later source automatically. +The recipe now also requires DR05 fence d68fb2ff47622db8224dc4b9e004dca4e20f643e +and DR08 terminal-status 0b374bc918af02476cdfb1502d5df132426f901b, including the +changed core/projector/read-view paths and prolonged-stall regressions. Seven +prerequisite entries and 24 source paths are inventoried. Older candidate d68fb2f +is refused because terminal-status ancestry is absent. + +Affected portable suite: 20 PASS, 0 FAIL, 0 SKIP. The real Git preflight against +0b374bc validates ancestry/paths and remains HOLD (no CI or build attestation). +CI's expected gate/hold fields are updated together with the generator. +100-alpha capacity is a candidate workload, not a measured certification; see +owner amendment in the main plan. Fixed 5-vCPU acceptance is superseded by +measured adequate resources plus headroom, without relaxing correctness/latency. + ## Authority and Scope - [Phase C packet](p184-dl-phase-c-packet.md), governing owner amendment first. @@ -82,7 +100,7 @@ python3 -B scripts/p184_consolidated_build.py \ --source-commit FULL_DL_COMMIT --release v2.2.3 ``` -Only after TS tests FINISH and pass, exact-source DL CI passes, and the owner +Only after TS market-data integration tests pass, exact-source DL CI passes, and the owner approves the future build, provide all explicit inputs: ```sh @@ -182,6 +200,6 @@ deferred because this is source-only preparation, not image-phase closure. Concurrent DR05 and parent work were left untouched. Next permitted step: parent review, journal update and eventual commit; obtain -final TS completion and exact-source DL CI receipts before separately authorizing +final TS market-data integration and exact-source DL CI receipts before separately authorizing a consolidated rebuild. Building does not authorize deployment. DR/broker gates and paired IDR075 activation still need the reviewed runtime packet and approval. diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md index 1a4747ab..321ec0d6 100644 --- a/upgrade/evidence/p184-dl-phase-c-packet.md +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -1,5 +1,26 @@ # P18.4 Data Layer Phase C Packet +## Governing Owner Amendment: 100 Alpha Plus TS, 2026-10-08 + +This amendment supersedes older fixed <=5-vCPU and Paper-feature wait wording, +not freshness, authority, integrity or LOSSLESS gates. Target: 100 active logical +alpha consumers with 2-5 declared products each, plus the TS60 workload. Quota and +resource changes must be sized from frozen offered rates, fanout, recovery and +single-replica capacity; configured slots are not measured capacity. Preserve TS +reservation and bounded per-client queues; do not increase buffers to conceal lag. + +Proceed independently of Claude's Paper/OR implementation: complete DL source, +isolated DR05/DR08 and capacity preparation now. Building still requires exact +source CI and TS market-data integration evidence; release requires final DR and +broker delta. No shared runtime change is authorized by an offline candidate. +Before activation, record exact images/config revisions, resource ceilings, +per-identity quotas, rollback, measured queue/latency/CPU/RAM limits and cleanup. +The consolidated build preflight must include DR05 fence d68fb2f and DR08 terminal +fix 0b374bc, not merely the older rc2 candidate. See +[p184-consolidated-build.md](p184-consolidated-build.md) and the main-plan entry +"Owner amendment: 100 alpha plus TS capacity and independent DL closure". + + ## Governing owner amendment - 2026-10-08 ### Consolidated-build timing and consumer handoff From fa10d1b5627cbec0a44eb9f70760e96d672a76c2 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 16:01:22 +0000 Subject: [PATCH 040/147] Model bounded 100-alpha workload and separate TS reservation --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 33 +++ scripts/p184_capacity100.py | 218 ++++++++++++++++ tests/test_p184_capacity100.py | 235 ++++++++++++++++++ .../evidence/p184-capacity100-candidate.md | 129 ++++++++++ 4 files changed, 615 insertions(+) create mode 100644 scripts/p184_capacity100.py create mode 100644 tests/test_p184_capacity100.py create mode 100644 upgrade/evidence/p184-capacity100-candidate.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 4825bbbc..71abb8c1 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70547,3 +70547,36 @@ change; no test resources created by this slice, cleanup intentionally unnecessa Canonicalmain5bf5cd5/v2.2.2, active featurefix/p184-dl-closure. Active/runtime and named rollback digests inherited unchanged from prior inventory, not remeasured. Capacity and WriterClone agents remain independent work in progress; no TS edits. + +DR05 WriterClone attempt found startup/deadline/evaluator issues in the TS-owned +harness, not a demonstrated new provider failure. No final fault cell passed; +canonical positive anchor was observed but TS writer exited with local PG5433 +connection refusal. Report: upgrade/evidence/p184-dr05-writerclone-followup.md. +Do not edit Claude's worktree or rerun blind. DL-owned cleanup also has an +AutoRemove race after channel exit: assign a narrow exact-identity absence check +plus negative tests. Failure inspection/replacement must still fail closed; no +broad cleanup and no swallowing arbitrary stop errors. Record results below. + +Capacity100 guide: upgrade/evidence/p184-capacity100-candidate.md records exact +mixed/stress workload, logical versus authenticated identities, TS reservation, +quota arithmetic, one-replica headroom and source MemoryPlan limits. No measured +100-alpha capacity or resource activation claimed. Driver integration assigned +narrowly; prior stage50 certified budget/evaluator must remain unchanged. Final +CPU/RAM sizing needs offered/dequeued/apply/recovery evidence, not quota arithmetic. + +Parent independently reproduced TS evaluator counterexample read-only: gap step, +old quantity2 instead of faulty3, actual validator EXECUTION_CONTEXT_STALE yielded +(True,[]) rather than requiredFalse. This is a TS harness correctness finding, +not proof of DL GAP closure; no TS source edited. WriterClone final acceptance +must wait for this narrow adapter correction, not for Paper/OR feature work. + +Capacity100 tested slice: typed planner reuses Phase3 and generates180streams +mixed or500stress streams (+assumedTS50),660reservedslots stress. Existing384 +single-replica slots insufficient; existing1024model307MiB is an estimate only. +Fixed startup arithmetic to include snapshot/warmup AND Subscribe on reconnect: +stress4950RPM peralpha identity BEFORE driverprobe/coldoverhead, not4500. +Unit6modules103PASS0fail0skip in existing image77682,4.535s; parent32PASS subset, +not additive. Full commands and limitations in capacity100 guide. No runtime +quota/resource changes. Stage100 hostdriver remains unfinished; do not claim +capacity from slot arithmetic or inherit stage50 PASS. This work does not wait +Paper/OR. Testcontainers--rm removed; noimagesbuilt, rollback/active preserved. diff --git a/scripts/p184_capacity100.py b/scripts/p184_capacity100.py new file mode 100644 index 00000000..104efe9c --- /dev/null +++ b/scripts/p184_capacity100.py @@ -0,0 +1,218 @@ +"""Offline candidate only: reuse the frozen Phase-3 workload, never run traffic. + +Inputs are existing manifest/acceptance objects, not credentials. Quota admission +is not capacity certification. The parent owns the journal and runtime packet. +""" +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, replace +import math + +from qdl.certification.phase3_consumer_load import ( + TargetIdentityDemand, TargetPoll, TargetWorkloadPlan, + build_target_workload_plan, +) + + +@dataclass(frozen=True) +class TSReservation: + consumer_id: str + demanded_routes: int + streams: int + requests_per_minute: int + + def __post_init__(self): + if (not self.consumer_id or self.demanded_routes != 60 + or any(type(v) is not int or v < 1 for v in + (self.streams, self.requests_per_minute))): + raise ValueError("TS60 needs explicit positive stream and request reservations") + + +def _startup_requests(session): + # SDK handoff is snapshot/warmup + Subscribe, also on resume. + return (2 * len(session.streams) + len(session.startup_snapshots) + + sum(p.feed.value == "BAR" for p in session.streams)) + + +def build_capacity100_candidate(*, manifests, products_by_consumer, + venue_identity, instruments, ts: TSReservation, + gateway_subscription_limit: int, + profile: str = "realistic"): + """Return (TargetWorkloadPlan, report), with no writes, sockets or admission. + + realistic: two unchanged 50-session plans, 40/30/20/10 class mix. + stress5: 100 sessions each consuming BAR/TRADE/QUOTE/BOOK_DELTA/BOOK_SNAPSHOT, + polling QUOTE at 1 Hz and anchoring BOOK_DELTA at startup. No extra sixth + product. Both plans require explicit DURABLE delivery for every stream. + + TS reservations are caller-supplied workload assumptions, not observed usage. + Gateway limit is an explicit per-replica config input. The check conservatively + places all traffic on one replica (including headroom); no assumed balancing + or HA multiplier. Replay RPCs have a separate limit and remain an open gate. + """ + if profile not in {"realistic", "stress5"}: + raise ValueError("unknown capacity profile") + if type(gateway_subscription_limit) is not int or gateway_subscription_limit < 1: + raise ValueError("gateway subscription limit must be a positive integer") + alpha_ids = set(venue_identity.values()) + if not alpha_ids or ts.consumer_id in alpha_ids: + raise ValueError("TS reservation must use a separate identity") + if set(venue_identity) != set(instruments): + raise ValueError("venue and instrument scopes differ") + if any(len(values) < 2 or len(set(values)) != len(values) + for values in instruments.values()): + raise ValueError("each venue needs at least two distinct explicit instruments") + if any(not value or "*" in value for values in instruments.values() for value in values): + raise ValueError("wildcard or empty instruments are not a frozen workload") + required_ids = alpha_ids | {ts.consumer_id} + if not required_ids <= set(manifests) or not alpha_ids <= set(products_by_consumer): + raise ValueError("missing alpha or TS manifest/products") + for consumer_id in required_ids: + quotas = manifests[consumer_id].quotas + if any(type(v) is not int or v < 1 for v in + (quotas.requests_per_minute, quotas.max_streams, quotas.max_buffer_events, + quotas.max_batch_items, quotas.max_warmup_rows)): + raise ValueError("invalid sealed quota") + if quotas.max_batch_items > 100 or quotas.max_warmup_rows > 10_000: + raise ValueError("query quota exceeds V2 batch/warmup service boundary") + if quotas.max_buffer_events > 10_000: + raise ValueError("buffer exceeds V2 10000-event service boundary") + for consumer_id in alpha_ids: + if any(p.consumer_id != consumer_id for p in products_by_consumer[consumer_id]): + raise ValueError("products mix authenticated identities") + base = build_target_workload_plan( + stage=50, manifests=manifests, products_by_consumer=products_by_consumer, + venue_identity=venue_identity, instruments=instruments, enforce_quota=False) + sessions = tuple(replace(s, ordinal=i + 1) + for i, s in enumerate(base.sessions + base.sessions)) + if profile == "stress5": + result = [] + for session in sessions: + first = session.streams[0] + selected = [] + for feed in ("BAR", "TRADE", "QUOTE", "BOOK_DELTA", "BOOK_SNAPSHOT"): + matches = [p for p in products_by_consumer[session.consumer_id] + if p.venue == first.venue and p.native_symbol == first.native_symbol + and p.feed.value == feed and (feed != "BAR" or p.interval == "1m")] + if len(matches) != 1: + raise ValueError(f"stress5 needs exactly one {feed} for {first.native_symbol}") + selected.append(matches[0]) + result.append(replace(session, alpha_class="GRID", streams=tuple(selected), + polls=(TargetPoll((selected[2],), 1.0, "SNAPSHOT"),), + startup_snapshots=(selected[4],))) + sessions = tuple(result) + for session in sessions: + products = (*session.streams, *session.startup_snapshots, + *(p for poll in session.polls for p in poll.products)) + if not 2 <= len({p.identity for p in products}) <= 5: + raise ValueError("capacity session must declare 2..5 distinct products including startup") + if any(getattr(getattr(p, "delivery", None), "value", None) != "DURABLE" + for p in session.streams): + raise ValueError("every planned stream must explicitly declare DURABLE delivery") + coverage = {(p.venue, p.native_symbol) for s in sessions for p in s.streams} + expected = {(v, symbol) for v, symbols in instruments.items() for symbol in symbols} + if not expected <= coverage: + raise ValueError("100-session profile does not cover the declared instruments") + demands, rows = [], [] + for consumer_id in sorted(required_ids): + mine = [s for s in sessions if s.consumer_id == consumer_id] + is_ts = consumer_id == ts.consumer_id + rpm = ts.requests_per_minute if is_ts else math.ceil(sum(s.requests_per_minute for s in mine)) + streams = ts.streams if is_ts else sum(len(s.streams) for s in mine) + quota = manifests[consumer_id].quotas + demand = TargetIdentityDemand(consumer_id, len(mine), rpm, streams, + quota.requests_per_minute, quota.max_streams) + # Bound one synchronized startup + one reconnect in a 25% burst minute. + # For TS, caller's RPM must include its non-subscribe recovery requests. + startup = ts.streams if is_ts else sum(_startup_requests(s) for s in mine) + batch_items = max((len(p.products) for s in mine for p in s.polls), default=0) + warmup_rows = 5000 if any(p.feed.value == "BAR" for s in mine for p in s.streams) else 0 + needed_rpm = max(demand.quota_needed, math.ceil(rpm * 1.25) + 2 * startup) + rows.append(dict(consumer_id=consumer_id, role="TS" if is_ts else "ALPHA", + logical_alphas=len(mine), steady_rpm=rpm, streams=streams, + startup_requests=startup, candidate_rpm=needed_rpm, + candidate_streams=demand.streams_needed, + sealed_rpm=quota.requests_per_minute, sealed_streams=quota.max_streams, + rpm_shortfall=max(0, needed_rpm - quota.requests_per_minute), + stream_shortfall=max(0, demand.streams_needed - quota.max_streams), + rpm_headroom=needed_rpm - rpm, + stream_headroom=demand.streams_needed - streams, + max_buffer_events=quota.max_buffer_events, + required_batch_items=batch_items, required_warmup_rows=warmup_rows, + sealed_batch_items=quota.max_batch_items, + sealed_warmup_rows=quota.max_warmup_rows, + batch_shortfall=max(0, batch_items - quota.max_batch_items), + warmup_shortfall=max(0, warmup_rows - quota.max_warmup_rows))) + if not is_ts: + demands.append(demand) + plan = TargetWorkloadPlan(100, sessions, tuple(demands)) + total_streams = sum(row["streams"] for row in rows) + reserved_streams = sum(row["candidate_streams"] for row in rows) + report = dict( + schema="p184.capacity100.candidate.v1", status="CANDIDATE_NOT_CERTIFIED", + profile=profile, logical_alphas=100, alpha_identity_count=len(alpha_ids), + total_identity_count=len(required_ids), ts_demanded_routes=ts.demanded_routes, + ts_reservation_source="CALLER_ASSUMPTION_NOT_MEASURED", + workload_groups=(dict(zip(("CANDLE", "REALTIME", "GRID", "MULTI"), plan.class_counts)) + if profile == "realistic" else {"STRESS5": 100}), + products_per_alpha=dict(sorted(Counter(len({p.identity for p in + (*s.streams, *s.startup_snapshots, *(p for poll in s.polls for p in poll.products))}) + for s in sessions).items())), + alpha_streams=plan.stream_count, total_streams=total_streams, + alpha_hot_rps=plan.hot_requests_per_second, + total_steady_rpm=sum(row["steady_rpm"] for row in rows), + candidate_total_rpm=sum(row["candidate_rpm"] for row in rows), + identities=rows, reserved_streams=reserved_streams, + gateway_subscription_limit=gateway_subscription_limit, + gateway_stream_shortfall=max(0, reserved_streams - gateway_subscription_limit), + sealed_identity_quotas_fit=all(not any(r[key] for key in + ("rpm_shortfall", "stream_shortfall", "batch_shortfall", "warmup_shortfall")) for r in rows), + gateway_slots_fit=reserved_streams <= gateway_subscription_limit, + manifest_event_slots_at_reserved_streams=sum(r["candidate_streams"] * r["max_buffer_events"] for r in rows), + assumptions=["100 logical sessions are not 100 independently authenticated identities", + "Fixed offered rates; no quota pacing or idle socket capacity claim", + "1.5x steady RPM floor; 1.25x burst plus startup and reconnect minute", + "1.2x stream headroom; all reserved slots must fit one replica", + "TS RPM input includes non-subscribe recovery operations; TS batch/warmup shape needs separate validation", + "BAR startup requests 5000 rows; MULTI uses a two-item reference batch", + "Five-stream stress consumes all five products, not five idle sockets"], + pending_gates=["Owner approval of TS reservation and exact identity/config revisions", + "Driver stage100/duration and stress5 integration; existing driver stops at50", + "Rust MemoryPlan for exact ring/queue/replay/transport/cgroup bounds; event slots are not RSS", + "Replay RPC admission, per-stream byte/event bounds and recovery window", + "Real-provider sustained/burst throughput, CPU/throttle, RAM, I/O and queue/lag", + "Per-session delivery/apply latency, LOSSLESS ordering/gaps and reconnect; TS60 ready throughout"], + cpu_budget="MEASURE_AND_SIZE_NO_FIXED_5_VCPU_CEILING", + ) + # These are the existing Rust MemoryPlan envelopes, not new pool sizing. + # Include Replay transport slots even though alpha_streams counts Subscribe. + report["resource_envelopes"] = [ + dict(subscriptions=slots, replay_rpcs=32, + transport_bytes_per_stream=96 * 1024, + ring_mib=48, live_queue_mib=32, replay_mib=32, + kafka_mib=32, process_reserve_mib=64, + estimated_required_bytes=(48 + 32 + 32 + 32 + 64) * 2**20 + + (slots + 32) * 96 * 1024, + observed_container_limit_bytes=512 * 2**20, + active_stream_shortfall=max(0, total_streams - slots), + reserved_stream_shortfall=max(0, reserved_streams - slots), + status="SOURCE_BOUND_ESTIMATE_NOT_CAPACITY") + for slots in (384, 1024) + ] + report["resource_observation"] = dict( + source="HISTORICAL_RUNTIME_READBACK_NOT_REMEASURED", host_vcpu=16, + stream_replica_vcpu=0.75, stream_replica_memory_mib=512, + per_client_buffers="UNCHANGED", aggregate_live_queue_mib=32, + warning="32 MiB global live pool does not guarantee fanout to 100 clients; " + "measure pool contention and apply lag before sizing aggregate pools. " + "Increasing per-client buffers to hide backlog is not optimization.") + report["sources"] = [ + "qdl/certification/phase3_consumer_load.py: frozen stage50 and quota headroom", + "qdl/consumer/manifest.py: ConsumerQuotas service boundaries", + "rust/qdl-stream-gateway/src/memory.rs: existing 384/1024 MemoryPlan", + "rust/qdl-stream-gateway/src/service.rs: separate Subscribe/Replay limits", + "scripts/phase3_consumer_load_acceptance.py: existing driver stage restrictions", + "DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md: owner amendment 2026-10-08", + ] + return plan, report diff --git a/tests/test_p184_capacity100.py b/tests/test_p184_capacity100.py new file mode 100644 index 00000000..6612af31 --- /dev/null +++ b/tests/test_p184_capacity100.py @@ -0,0 +1,235 @@ +"""Offline TEST_ONLY fixtures; no sockets, manifests or runtime writes.""" +from dataclasses import dataclass, replace +import json +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from scripts.p184_capacity100 import TSReservation, build_capacity100_candidate +from tests.test_phase3_target_workload import ( + VENUES, SYMBOLS, _Product, _Feed, _products, +) +from qdl.certification.phase3_consumer_load import TARGET_STAGE_MIX + + +@dataclass(frozen=True) +class Product(_Product): + delivery: _Feed = _Feed("DURABLE") + + +def inputs(): + products = {key: tuple(Product(**vars(p)) for p in values) + for key, values in _products().items()} + manifests = {key: SimpleNamespace(quotas=SimpleNamespace( + requests_per_minute=2400, max_streams=60, max_buffer_events=2000, + max_batch_items=50, max_warmup_rows=10000)) + for key in VENUES.values()} + manifests["ts"] = SimpleNamespace(quotas=SimpleNamespace( + requests_per_minute=1500, max_streams=50, max_buffer_events=2000, + max_batch_items=50, max_warmup_rows=10000)) + return dict(manifests=manifests, products_by_consumer=products, + venue_identity=VENUES, instruments=SYMBOLS, + ts=TSReservation("ts", 60, 50, 1000), gateway_subscription_limit=384) + + +class Capacity100Tests(unittest.TestCase): + def setUp(self): + self.args = inputs() + self.network = patch("socket.socket", side_effect=AssertionError("offline only")) + self.network.start() + self.addCleanup(self.network.stop) + + def plan(self, **changes): + return build_capacity100_candidate(**(self.args | changes)) + + def test_realistic_frozen_mix_and_products(self): + plan, report = self.plan() + self.assertEqual(plan.class_counts, (40, 30, 20, 10)) + self.assertEqual(plan.stream_count, 180) + self.assertEqual(plan.hot_requests_per_second, 100) + self.assertEqual(report["products_per_alpha"], {2: 40, 3: 30, 5: 30}) + self.assertEqual([s.ordinal for s in plan.sessions], list(range(1, 101))) + self.assertEqual(len(plan.covered_instruments), 10) + self.assertEqual(report["alpha_identity_count"], 2) + self.assertEqual(report["total_identity_count"], 3) + self.assertEqual(report["total_streams"], 230) + self.assertEqual(report["reserved_streams"], 276) + self.assertEqual(report["total_steady_rpm"], 7010) + + def test_current_quotas_report_shortfalls_without_shrinking_load(self): + plan, report = self.plan() + self.assertFalse(report["sealed_identity_quotas_fit"]) + alpha = [r for r in report["identities"] if r["role"] == "ALPHA"] + self.assertEqual([r["logical_alphas"] for r in alpha], [50, 50]) + self.assertEqual([r["streams"] for r in alpha], [90, 90]) + self.assertEqual([r["candidate_rpm"] for r in alpha], [4506, 4509]) + self.assertEqual([r["candidate_streams"] for r in alpha], [108, 108]) + self.assertEqual([r["rpm_shortfall"] for r in alpha], [2106, 2109]) + self.assertEqual([r["stream_shortfall"] for r in alpha], [48, 48]) + self.assertEqual(plan.hot_requests_per_second, 100) + + def test_stress500_is_active_and_keeps_exactly_five_products(self): + plan, report = self.plan(profile="stress5") + self.assertEqual(plan.stream_count, 500) + self.assertEqual(report["products_per_alpha"], {5: 100}) + self.assertEqual(report["workload_groups"], {"STRESS5": 100}) + for session in plan.sessions: + self.assertEqual(len({p.identity for p in session.streams}), 5) + self.assertEqual({p.feed.value for p in session.streams}, + {"BAR", "TRADE", "QUOTE", "BOOK_DELTA", "BOOK_SNAPSHOT"}) + self.assertEqual(session.polls[0].period_seconds, 1) + self.assertEqual(session.startup_snapshots[0].feed.value, "BOOK_SNAPSHOT") + self.assertEqual(report["total_streams"], 550) + self.assertEqual(report["reserved_streams"], 660) + self.assertEqual(report["gateway_stream_shortfall"], 276) + self.assertFalse(report["gateway_slots_fit"]) + self.assertEqual([d.streams_needed for d in plan.demands], [300, 300]) + self.assertEqual([d.quota_needed for d in plan.demands], [4500, 4500]) + + def test_sdk_snapshot_plus_subscribe_handoff_is_reserved(self): + _, report = self.plan(profile="stress5") + alpha = [r for r in report["identities"] if r["role"] == "ALPHA"] + # 50 sessions x (five two-request handoffs + BAR warmup + book anchor). + self.assertEqual([r["startup_requests"] for r in alpha], [600, 600]) + # Same minute: 1.25 x 3000 steady + startup and full reconnect envelopes. + self.assertEqual([r["candidate_rpm"] for r in alpha], [4950, 4950]) + self.assertEqual([r["rpm_headroom"] for r in alpha], [1950, 1950]) + + def test_existing1024_envelope_not_a_capacity_certificate(self): + _, report = self.plan(profile="stress5", gateway_subscription_limit=1024) + self.assertTrue(report["gateway_slots_fit"]) + self.assertFalse(report["sealed_identity_quotas_fit"]) + self.assertEqual(report["status"], "CANDIDATE_NOT_CERTIFIED") + old, candidate = report["resource_envelopes"] + self.assertEqual(old["estimated_required_bytes"], 247 * 2**20) + self.assertEqual(candidate["estimated_required_bytes"], 307 * 2**20) + self.assertEqual(old["active_stream_shortfall"], 166) + self.assertEqual(candidate["reserved_stream_shortfall"], 0) + self.assertEqual(candidate["live_queue_mib"], 32) + self.assertEqual(report["resource_observation"]["per_client_buffers"], "UNCHANGED") + self.assertTrue(report["pending_gates"]) + + def test_exact_quota_candidate_can_fit_without_certification(self): + _, first = self.plan(profile="stress5", gateway_subscription_limit=1024) + for row in first["identities"]: + quotas = self.args["manifests"][row["consumer_id"]].quotas + quotas.requests_per_minute = row["candidate_rpm"] + quotas.max_streams = row["candidate_streams"] + _, report = self.plan(profile="stress5", gateway_subscription_limit=1024) + self.assertTrue(report["sealed_identity_quotas_fit"]) + self.assertEqual(report["status"], "CANDIDATE_NOT_CERTIFIED") + self.args["manifests"]["ts"].quotas.requests_per_minute -= 1 + self.assertFalse(self.plan(profile="stress5")[1]["sealed_identity_quotas_fit"]) + + def test_ts_routes_are_not_assumed_to_be_streams(self): + _, report = self.plan(ts=TSReservation("ts", 60, 70, 2000)) + ts = next(r for r in report["identities"] if r["role"] == "TS") + self.assertEqual(report["ts_demanded_routes"], 60) + self.assertEqual(ts["streams"], 70) + self.assertEqual(ts["candidate_streams"], 84) + self.assertEqual(ts["candidate_rpm"], 3000) + + def test_burst_startup_reconnect_can_exceed_floor_for_ts(self): + _, report = self.plan(ts=TSReservation("ts", 60, 50, 1)) + ts = next(r for r in report["identities"] if r["role"] == "TS") + self.assertEqual(ts["candidate_rpm"], 102) + + def test_no_global_stage_or_input_mutation_and_deterministic(self): + before = dict(TARGET_STAGE_MIX) + first = self.plan() + self.assertEqual(first, self.plan()) + self.assertEqual(before, TARGET_STAGE_MIX) + self.assertNotIn(100, TARGET_STAGE_MIX) + self.assertEqual(self.args["manifests"]["ts"].quotas.max_streams, 50) + json.dumps(first[1], allow_nan=False) + + def test_missing_stress_feed_refused(self): + key = VENUES["BINANCE"] + self.args["products_by_consumer"][key] = tuple( + p for p in self.args["products_by_consumer"][key] if p.feed.value != "TRADE") + with self.assertRaisesRegex(ValueError, "TRADE"): + self.plan(profile="stress5") + + def test_ambiguous_stress_feed_refused(self): + key = VENUES["BINANCE"] + values = self.args["products_by_consumer"][key] + self.args["products_by_consumer"][key] += tuple(p for p in values if p.feed.value == "TRADE") + with self.assertRaisesRegex(ValueError, "exactly one TRADE"): + self.plan(profile="stress5") + + def test_on_demand_cannot_be_counted_as_live(self): + key = VENUES["BINANCE"] + self.args["products_by_consumer"][key] = tuple( + replace(p, delivery=_Feed("ON_DEMAND")) if p.feed.value == "BOOK_DELTA" else p + for p in self.args["products_by_consumer"][key]) + for profile in ("realistic", "stress5"): + with self.subTest(profile=profile), self.assertRaisesRegex(ValueError, "DURABLE"): + self.plan(profile=profile) + + def test_missing_ts_manifest_refused(self): + del self.args["manifests"]["ts"] + with self.assertRaisesRegex(ValueError, "missing"): + self.plan() + + def test_shared_ts_alpha_identity_refused(self): + with self.assertRaisesRegex(ValueError, "separate identity"): + self.plan(ts=TSReservation(VENUES["BINANCE"], 60, 50, 1000)) + + def test_cross_identity_product_refused(self): + key = VENUES["BINANCE"] + products = self.args["products_by_consumer"][key] + self.args["products_by_consumer"][key] = (replace(products[0], consumer_id="other"), *products[1:]) + with self.assertRaisesRegex(ValueError, "mix authenticated"): + self.plan() + + def test_invalid_reservations_and_limits(self): + for value in (0, -1, 1.5, float("nan"), True): + with self.subTest(value=value): + with self.assertRaises(ValueError): + TSReservation("ts", 60, value, 1) + with self.assertRaises(ValueError): + TSReservation("ts", 60, 1, value) + with self.assertRaises(ValueError): + self.plan(gateway_subscription_limit=value) + with self.assertRaises(ValueError): + TSReservation("ts", 59, 50, 1000) + with self.assertRaises(ValueError): + self.plan(profile="idle100") + + def test_service_buffer_boundary(self): + self.args["manifests"]["ts"].quotas.max_buffer_events = 10001 + with self.assertRaisesRegex(ValueError, "10000-event"): + self.plan() + + def test_nonpositive_sealed_quota_refused(self): + self.args["manifests"]["ts"].quotas.requests_per_minute = 0 + with self.assertRaisesRegex(ValueError, "invalid sealed"): + self.plan() + + def test_batch_and_warmup_shortfalls_do_not_shrink_workload(self): + for key in VENUES.values(): + self.args["manifests"][key].quotas.max_batch_items = 1 + self.args["manifests"][key].quotas.max_warmup_rows = 4999 + plan, report = self.plan() + self.assertEqual(plan.stream_count, 180) + alpha = [r for r in report["identities"] if r["role"] == "ALPHA"] + self.assertEqual([r["batch_shortfall"] for r in alpha], [1, 1]) + self.assertEqual([r["warmup_shortfall"] for r in alpha], [1, 1]) + self.assertFalse(report["sealed_identity_quotas_fit"]) + + def test_query_service_boundaries(self): + for field, value in (("max_batch_items", 101), ("max_warmup_rows", 10001)): + with self.subTest(field=field): + args = inputs() + setattr(args["manifests"]["ts"].quotas, field, value) + with self.assertRaisesRegex(ValueError, "batch/warmup"): + build_capacity100_candidate(**args) + + def test_nonexplicit_or_duplicate_scope_refused(self): + for values in ((), ("BTCUSDT",), ("BTCUSDT", "BTCUSDT"), ("*", "BTCUSDT")): + with self.subTest(values=values), self.assertRaises(ValueError): + self.plan(instruments=SYMBOLS | {"BINANCE": values}) + + +if __name__ == "__main__": + unittest.main() diff --git a/upgrade/evidence/p184-capacity100-candidate.md b/upgrade/evidence/p184-capacity100-candidate.md new file mode 100644 index 00000000..e9321032 --- /dev/null +++ b/upgrade/evidence/p184-capacity100-candidate.md @@ -0,0 +1,129 @@ +# 100 Alpha Plus TS: Candidate Capacity Contract + +Status: IMPLEMENTED_PLANNER / LOAD_NOT_MEASURED / NOT_CERTIFIED, 2026-10-08. +Authority: Unified Plan, "Owner amendment: 100 alpha plus TS capacity and +independent DL closure". Existing Kafka-native architecture and endpoint contracts +remain unchanged. No shared quota, resource, identity or runtime was changed. + +## Workload, Not Merely Quota + +`scripts/p184_capacity100.py` reuses the existing Phase-3 typed workload. Inputs +must be actual versioned manifests and explicit instrument/product selections. +100 means logical alpha sessions, NOT 100 authenticated identities. This first +candidate multiplexes 50 sessions through each venue's alpha identity and keeps +TS on a separate identity. It does not certify arbitrary strategy demand, +unlimited universe scans, additional symbols, or 100 different credentials. + +Two frozen profiles, both with real data processing rather than idle sockets: + +| Profile | Alpha classes | Alpha streams | Hot read calls/s | +| --- | --- | --- | --- | +| realistic | 40 candle / 30 realtime / 20 grid / 10 multi | 180 | 100 | +| stress5 | BAR1m, TRADE, QUOTE, BOOK_DELTA, BOOK_SNAPSHOT per alpha | 500 | 100 | + +Each session uses 2-5 distinct products INCLUDING startup anchors. All stream +products explicitly require DURABLE delivery. A duplicated stream, missing feed, +wildcard or mixed identity is rejected. The ordinary universe warmup requirement +is not expanded by this profile. Shared instrument selection must be recorded; +500 streams do not mean 500 independently acquired provider subscriptions. + +The unit-fixture TS reservation is 60 routes, 50 streams, 1000 requests/minute. +These are INPUT ASSUMPTIONS, not observed TS usage. Before load, replace/confirm +with writer's actual reconnect/snapshot demand; include probes and recovery calls +under the same identity in the request budget. TS heartbeat from production is +only a protection guard, never proof that TS consumed the candidate workload. + +## Candidate Quotas And Memory + +The following is arithmetic on the typed TEST_ONLY fixtures, not an entitlement +change or a capacity measurement. The planner also reports exact deficits versus +its input manifests. Startup plus one reconnect minute is counted separately +from steady traffic. Snapshot/warmup plus Subscribe are both counted. Driver +probe/cold-read overhead still needs addition before any admission; these values +are not a complete runnable quota packet. Steady RPM headroom is 1.5x, stream headroom is 1.2x. + +| Identity | Realistic RPM / slots | Stress5 RPM / slots | +| --- | --- | --- | +| Binance alpha | 4506 / 108 | 4950 / 300 | +| OKX alpha | 4509 / 108 | 4950 / 300 | +| TS, assumed workload above | 1500 / 60 | 1500 / 60 | + +Stress5 needs 550 active streams and 660 reserved slots. The CURRENT observed +384-slot Stream replica cannot carry all of these after its peer is lost. +Do not multiply capacity by two and claim failover. The existing 1024-slot +MemoryPlan is the candidate to measure, with replay RPCs accounted separately. + +Existing Rust MemoryPlan arithmetic for 1024 Subscribe + 32 Replay slots: +48 MiB ring + 32 MiB live pool + 32 MiB replay + 32 MiB Kafka + 64 MiB process +reserve + 99 MiB transport = 307 MiB. Historical container cap is 512 MiB. +This is a modeled allocation budget, NOT measured RSS or an OOM guarantee. In particular, +32 MiB shared live queue may contend with fanout. Size aggregate pools from +measured event sizes and concurrent draining after correctness; keep per-client +backpressure bounded. Do not enlarge a slow client's backlog to hide throughput. + +Host observation: 16 vCPU, Stream 0.75 vCPU per replica. These values were read +previously, not a fresh resource-availability claim. The old <=5-vCPU ceiling is +superseded by owner's adequate-resource direction. There is deliberately no +invented CPU requirement for 100 alpha: measure cost under the actual profile, +then assign CPU/RAM with headroom while protecting TS and co-hosted services. +100 alpha container compute itself is outside DL serving-capacity measurement. + +## Correctness And Measurement Order + +1. Finish DR05 typed GAP/generation and DR08 terminal/replay on exact candidate. + Paper/OR feature completion is not a dependency. Keep failed receipts. +2. Freeze image/config/manifest, workload, offered-rate schedule, test identities, + TS reservation, client resource budget and allowed stop conditions BEFORE load. + A quota candidate is not permission to load the shared identity without this. +3. Reuse affected fast matrix and current driver: ramp 20/50 then 100 only when + previous stage has no internal errors, queue growth or unsafe eligibility. + Passing earlier unrelated catalogue/economic tests remains inherited. +4. Include sustained and burst workload, one slow alpha versus healthy peers, + disconnect/replay, worker crash after apply before cursor commit, and one + Stream replica unavailable. Do not reset a cursor or skip records to catch up. +5. Report server offered/dequeued, SDK receive, apply, Redis ACK and cursor commit + for the SAME product/generation/offset. External Redis polls are not ACK time. + Report backlog high-water/events/bytes and bounded recovery independently. +6. Measure request-to-usable, event age, canonical delivery and consumer apply + in ms, p50/p95/max and n; p99 only with n>=100. Refusals remain in denominator, + separating valid market quietness from internal pipeline failures. Active + sessions must process data; health, quota slots and idle sockets do not pass. +7. Measure CPU/throttling, RSS/OOM, I/O and Kafka lag over matching intervals for + all DL roles AND the test clients separately. Choose operational headroom from + peaks/bursts, not a post-hoc threshold chosen to make a failing result pass. + +Existing freshness, schema, source identity and execution gates are unchanged. +The acceptance cannot pass while a GAP leaves a fresh old book execution-eligible, +while queue/replay loops never catch up, or while a consumer loses ordering. +Only then prepare the exact role-by-role resource/config/rollback packet. + +## Runtime And Cleanup Boundary + +No production load, quota activation, image build, rollout, push or merge in this +planner slice. Canonical /home/bobby/data_layer main5bf5cd5, published v2.2.2; +feature fix/p184-dl-closure. v2.2.3 remains HOLD. Preserve active digests and named +BAR rollback f7351c3b. Unit Docker runs use existing images, network none, read-only +source and --rm; no persistent volumes. Load resources must use owned namespaces +and be removed after each attempt, preserving bounded receipts/hashes and faults. + +## Executed Verification And Remaining Implementation + +Existing image77682, network none, read-only source, bounded64MiB/tmp and --rm: +`python -B -m unittest tests.test_p184_capacity100 + tests.test_phase3_target_workload tests.test_phase3_consumer_load + tests.test_phase3_target_driver tests.test_phase3_consumer_load_driver + tests.test_kn_capacity_budget`:103PASS0fail0skip,4.535s (agent transcript). +Parent independently ran capacity+target_workload:32PASS0fail0skip,0.092s; +this is a SUBSET, not32additional tests. No persisted performance receipt was +produced by these unit tests. Containers removed; no images/volumes created. +Host-only driver import initially lacked protobuf/pydantic, and first read-only +container lacked writable/tmp; neither failed attempt is called a product test +failure or hidden. Final image test used the declared dependencies and tmpfs. + +The existing host driver still intentionally rejects stage100. It was not +silently widened to reuse the stage50 acceptance budget. Required implementation: +explicit candidate-only stage100 configuration, complete probe/cold/startup quota +preflight, module/worker wiring, validated per-stream activity/ACK evidence and +real TS-clone consumption. Load execution and CPU/RAM sizing remain unfinished, +not a Paper dependency or an external approval excuse. This report closes only +the tested planner/design slice, NOT the100-alpha acceptance. From f5e0fa07f34387a4c83e101afc07686e40af53b5 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 16:10:02 +0000 Subject: [PATCH 041/147] Harden isolated cleanup and record actual DR05 writer handoff gaps --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 29 +++ scripts/p184_dr05_dl_fault.py | 29 ++- tests/test_p184_dr05_dl_fault.py | 107 +++++++- .../evidence/p184-capacity100-candidate.md | 13 + .../p184-dr05-writerclone-followup.md | 230 ++++++++++++++++++ 5 files changed, 403 insertions(+), 5 deletions(-) create mode 100644 upgrade/evidence/p184-dr05-writerclone-followup.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 71abb8c1..4b90a787 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70580,3 +70580,32 @@ not additive. Full commands and limitations in capacity100 guide. No runtime quota/resource changes. Stage100 hostdriver remains unfinished; do not claim capacity from slot arithmetic or inherit stage50 PASS. This work does not wait Paper/OR. Testcontainers--rm removed; noimagesbuilt, rollback/active preserved. + +DR05 cleanup slice verified: exact-ID AutoRemove disappearance accepted only +for matching NoSuchContainer; replacement, inspection/auth/otherstop errors +failclosed. Oldsource12lifecycle tests3fail+1error; final82PASS in image77682, +30portablePASS overlapping. Parent in-image dr05_dl_fault30PASS36.196s; host +attempt lacked redis dependency (1error), not silently counted PASS. Reports +retain both failed upstream attempts; no finalDR05/ACK qualification. Both stack +namespaces/privatefiles removed, noimagesbuilt/pruned. No TS edits or rollout. + +CRITICAL shared-host disk observation at end of slice: df -B1 reports root100%, +available229122048bytes (~218.5MiB). Earlier agent observation1.46GB followed +11.18GB after stack cleanup. Concurrent host work prevents attribution; no +claim that DL test consumed the decrease. Do not launch load/build before disk +headroom is restored. All owned p184-dr05 containers absent. No shared cleanup, +no stopping Claude or production, no deletion of financial state/active images. +Evidence/Claude handoff: upgrade/evidence/p184-dr05-writerclone-followup.md. + +Offline actual-manifest capacity compile additionally succeeded for both mixed +andstress profiles at sourcefa10d1b (image77682,networknone,readonly): same230/550 +active and276/660reserved streams. Actual alpha source manifests2400RPM/60streams +each are insufficient; assumedTS50active needs60reserved vssealed50. This is +source contract arithmetic, not an observed runtime quota or performance result. +Runtime/release/config unchanged; canonicalmain5bf5cd5/v2.2.2, branchfix/p184-dl-closure. +v2.2.3 HOLD, phaseIN_PROGRESS. No push/merge; kept active and namedBARrollback. + +Final readonly disk recheck worsened to28430336bytes (~27.1MiB) available, root100%. +Do not build/load until the concurrent writer/build owner restores disk headroom. +All p184-capacity100 and p184-dr05 containers absent, worktree clean before this +journal-only update. No causal attribution without process/disk investigation. diff --git a/scripts/p184_dr05_dl_fault.py b/scripts/p184_dr05_dl_fault.py index 61043bcc..c5a88ef3 100644 --- a/scripts/p184_dr05_dl_fault.py +++ b/scripts/p184_dr05_dl_fault.py @@ -203,7 +203,7 @@ def sh(argv): return subprocess.run(argv, check=True, capture_output=True, text=True, timeout=15).stdout -def owned(ns, role, call=sh): +def owned(ns, role, call=sh, *, allow_disappearance=True): name = namespace(ns) + "-" + role if role not in (*ROLES, "helper", "tsvalidator"): raise ValueError("unknown owned role") @@ -215,6 +215,8 @@ def owned(ns, role, call=sh): try: row = json.loads(call(["docker", "inspect", ids[0]]))[0] except subprocess.CalledProcessError: + if not allow_disappearance: + raise # --rm can finish between the exact-name listing and inspection. if not call(["docker", "ps", "-aq", "--filter", "name=^/" + name + "$"]).strip(): return None @@ -245,19 +247,38 @@ def cleanup(ns, call=sh): removed = [] # Preflight ALL ownership before stopping anything. Never remove networks, # images, shared volumes, files or a container selected only by a label. - rows = [(role, owned(ns, role, call)) for role in reversed((*ROLES, "helper", "tsvalidator"))] + rows = [(role, owned(ns, role, call, allow_disappearance=False)) + for role in reversed((*ROLES, "helper", "tsvalidator"))] if any(row and row["HostConfig"].get("AutoRemove") is not True for _, row in rows): raise ValueError("refuse non-disposable container") failures = [] for role, row in rows: if row: try: - call(["docker", "stop", "--time", "5", row["Id"]]) + stop = ["docker", "stop", "--time", "5", row["Id"]] + try: + call(stop) + except subprocess.CalledProcessError as error: + # Only this pinned disposable's exact absence can excuse a failed stop. + missing = "Error response from daemon: No such container: " + row["Id"] + if error.returncode != 1 or error.cmd != stop or (error.stderr or "").strip() != missing: + raise + if call(["docker", "ps", "-aq", "--no-trunc", "--filter", "id=" + row["Id"]]).strip(): + raise + if call(["docker", "ps", "-aq", "--no-trunc", "--filter", + "name=^/" + ns + "-" + role + "$"]).strip(): + raise RuntimeError("container identity replaced during cleanup") from error + removed.append(role) + continue deadline = time.monotonic() + 5 - while (remaining := owned(ns, role, call)) is not None: + while (remaining := owned(ns, role, call, allow_disappearance=False)) is not None: + if remaining["Id"] != row["Id"]: + raise RuntimeError("container identity replaced during cleanup") if remaining.get("State", {}).get("Running") or time.monotonic() >= deadline: raise RuntimeError("--rm container still present") time.sleep(.1) + if call(["docker", "ps", "-aq", "--no-trunc", "--filter", "id=" + row["Id"]]).strip(): + raise RuntimeError("inspected container still present during cleanup") removed.append(role) except Exception as error: failures.append(role + ":" + type(error).__name__) diff --git a/tests/test_p184_dr05_dl_fault.py b/tests/test_p184_dr05_dl_fault.py index 9981f542..1aa75177 100644 --- a/tests/test_p184_dr05_dl_fault.py +++ b/tests/test_p184_dr05_dl_fault.py @@ -2,6 +2,7 @@ import copy import json from pathlib import Path +import subprocess import tempfile import unittest from unittest.mock import patch @@ -131,6 +132,9 @@ def setUp(self): def call(self, argv): self.calls.append(argv) if argv[1] == "ps": + if argv[-1].startswith("id="): + identity = argv[-1].removeprefix("id=") + return "\n".join(row["Id"] for row in self.rows.values() if row["Id"] == identity) role = argv[-1].removesuffix("$").split(NS + "-")[1] return self.rows[role]["Id"] if role in self.rows else "" role = argv[-1].removesuffix("-id") @@ -184,6 +188,107 @@ def no_stop(argv): with self.assertRaises(RuntimeError): dr.cleanup(NS, no_stop) + def test_cleanup_stop_race_requires_exact_inspected_absence(self): + original = copy.deepcopy(self.rows["query"]) + cases = ("absent", "present", "replacement", "foreign_replacement", + "unrelated_error", "wrong_id_error", "extra_error", "list_error", "name_list_error") + for case in cases: + with self.subTest(case=case): + self.rows, self.calls = {"query": copy.deepcopy(original)}, [] + stopped = False + def call(argv): + nonlocal stopped + if argv[1] == "stop": + self.calls.append(argv) + stopped = True + if case != "present": + del self.rows["query"] + if case in ("replacement", "foreign_replacement"): + self.rows["query"] = copy.deepcopy(original) + self.rows["query"]["Id"] = "replacement-id" + if case == "foreign_replacement": + self.rows["query"]["Config"]["Labels"][dr.LABEL] = "foreign" + message = "Error response from daemon: No such container: query-id" + if case == "unrelated_error": + message = "permission denied" + elif case == "wrong_id_error": + message = "Error response from daemon: No such container: other-id" + elif case == "extra_error": + message += "\nadditional failure" + raise subprocess.CalledProcessError(1, argv, stderr=message + "\n") + if stopped and argv[1] == "ps": + if case == "list_error" and argv[-1] == "id=query-id": + raise subprocess.CalledProcessError(1, argv, stderr="daemon unavailable") + if case == "name_list_error" and argv[-1].startswith("name="): + raise subprocess.CalledProcessError(1, argv, stderr="daemon unavailable") + return self.call(argv) + if case == "absent": + self.assertEqual(dr.cleanup(NS, call)["removed"], ["query"]) + self.assertIn(["docker", "ps", "-aq", "--no-trunc", "--filter", "id=query-id"], self.calls) + else: + with self.assertRaisesRegex(RuntimeError, "cleanup incomplete"): + dr.cleanup(NS, call) + self.assertEqual([cmd[-1] for cmd in self.calls if cmd[1] == "stop"], ["query-id"]) + if case in ("replacement", "foreign_replacement"): + self.assertEqual(self.rows["query"]["Id"], "replacement-id") + + def test_cleanup_successful_stop_rejects_replacement_without_waiting(self): + self.rows = {"query": self.rows["query"]} + def call(argv): + if argv[1] == "stop": + self.calls.append(argv) + self.rows["query"]["Id"] = "replacement-id" + self.rows["query"]["State"]["Running"] = False + return "" + if argv[1] == "inspect" and argv[-1] == "replacement-id": + self.calls.append(argv) + return json.dumps([self.rows["query"]]) + return self.call(argv) + with patch.object(dr.time, "sleep", side_effect=AssertionError("replacement must not be polled")) as sleep: + with self.assertRaisesRegex(RuntimeError, "cleanup incomplete"): + dr.cleanup(NS, call) + sleep.assert_not_called() + self.assertEqual([cmd[-1] for cmd in self.calls if cmd[1] == "stop"], ["query-id"]) + + def test_cleanup_successful_stop_requires_pinned_id_absence_not_just_name(self): + self.rows = {"query": self.rows["query"]} + stopped = False + def call(argv): + nonlocal stopped + if argv[1] == "stop": + self.calls.append(argv) + stopped = True + return "" + if stopped and argv[1] == "ps" and argv[-1].startswith("name="): + return "" + return self.call(argv) + with self.assertRaisesRegex(RuntimeError, "cleanup incomplete"): + dr.cleanup(NS, call) + self.assertEqual(self.rows["query"]["Id"], "query-id") + self.assertEqual([cmd[-1] for cmd in self.calls if cmd[1] == "stop"], ["query-id"]) + + def test_cleanup_inspection_errors_never_become_absence(self): + original = copy.deepcopy(self.rows["query"]) + for after_stop in (False, True): + with self.subTest(after_stop=after_stop): + self.rows, self.calls = {"query": copy.deepcopy(original)}, [] + stopped = False + def call(argv): + nonlocal stopped + if argv[1] == "stop": + self.calls.append(argv) + stopped = True + return "" + if argv[1] == "inspect" and (stopped or not after_stop): + del self.rows["query"] + raise subprocess.CalledProcessError(1, argv, stderr="inspection failed") + return self.call(argv) + expected = RuntimeError if after_stop else subprocess.CalledProcessError + with self.assertRaises(expected): + dr.cleanup(NS, call) + if not after_stop: + self.assertFalse(any(cmd[1] == "stop" for cmd in self.calls)) + def test_cleanup_waits_for_async_auto_remove(self): original = self.call pending = {} @@ -193,7 +298,7 @@ def call(argv): self.rows[role]["State"]["Running"] = False pending[role] = 2 return "" - if argv[1] == "ps": + if argv[1] == "ps" and argv[-1].startswith("name="): role = argv[-1].removesuffix("$").split(NS + "-")[1] if role in pending: pending[role] -= 1 diff --git a/upgrade/evidence/p184-capacity100-candidate.md b/upgrade/evidence/p184-capacity100-candidate.md index e9321032..10d65173 100644 --- a/upgrade/evidence/p184-capacity100-candidate.md +++ b/upgrade/evidence/p184-capacity100-candidate.md @@ -127,3 +127,16 @@ preflight, module/worker wiring, validated per-stream activity/ACK evidence and real TS-clone consumption. Load execution and CPU/RAM sizing remain unfinished, not a Paper dependency or an external approval excuse. This report closes only the tested planner/design slice, NOT the100-alpha acceptance. + +### Real Manifest Compile And Disk Stop + +Parent repeated offline compile at sourcefa10d1b with actual source catalogue, +acquisition and both alpha manifests plus TS. Both profiles compile; active/ +reserved counts remain230/276 and550/660. Alpha manifests currently declare +2400RPM/60streams each; both fail the proposed workload quota check. Assumed TS +requires60reserved streams versus source manifest50. This is NOT runtime quota +readback and NOT traffic. Docker--rm/networknone/readonly removed after success. + +Before further load/build: root filesystem observed100%,229122048bytes free. +Restore measured disk headroom with an ownership-safe cleanup packet; no broad +prune while Claude may be building. No test or runtime data deleted to mask this. diff --git a/upgrade/evidence/p184-dr05-writerclone-followup.md b/upgrade/evidence/p184-dr05-writerclone-followup.md new file mode 100644 index 00000000..ae7b5653 --- /dev/null +++ b/upgrade/evidence/p184-dr05-writerclone-followup.md @@ -0,0 +1,230 @@ +# DR05 WriterClone Follow-up + +Status: BLOCKED_BEFORE_TS_FAULT_VALIDATION / NOT_QUALIFIED, 2026-10-08. + +Owner authorized independent isolated DR05 investigation without waiting for TS +Paper. Parent journal: `DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md`, section +"Owner amendment: 100 alpha plus TS capacity and independent DL closure". +Governing handoffs: `p184-dr05-fence-channel-handoff.md`, +`p184-dr05-dl-fault.md`. Parent owns journal integration; this report is the only +repository file originally owned by this investigation. No TS or DL harness +source changed during the original attempts; the assigned DL-only cleanup fix +and its verification are appended below. + +## Result And Exact Blockers + +No GAP_REFUSAL or GENERATION_CHANGE_FENCED cell passed. No fault was published: +the corrected bounded OKX run stopped at its first positive anchor failure. +One real TEST_ONLY raw publication was acknowledged at partition0/offset0 and +observed as canonical BOOK_SNAPSHOT generation7/sequence g7:100, with committed +raw next-offset1 and read_committed EOF for canonical/quarantine. TS observation +was null because its writer process had exited. This is not typed refusal proof. + +1. Actual runtime blocker: retained-image `services/market_data/main.py:360` + exited in `asyncpg.create_pool`; connections to both localhost IPv6 and IPv4 + port5433 were refused. TS `scripts/p184_dr05_isolated.py` starts Postgres, + sleeps five seconds, then starts the writer without database readiness. + The receipt has the writer traceback, not Postgres logs; whether Postgres was + still initializing or had failed is not established. Small TS-owned proposal: + bounded pg_isready plus Redis ping before writer startup, bounded failure-log + capture, and continued writer liveness checking. Keep the same overall budget. +2. Deadline violation observed: unchanged `TsDr05Adapter.fresh_case(timeout=30)` + returned after32.194860803s. Its deadline is not enforced after successful + probes or during WriterClone startup (`p184_dr05_upstream.py:131-144`). + Proposal: propagate remaining deadline to clone operations and check it before + returning; do not increase the budget to conceal startup cost. +3. Deterministic evaluator counterexample: unchanged upstream `evaluate()` at + lines208-213 accepts a gap observation with previous quantity2 (fault quantity3) + and `ExecutionMarketContextError:EXECUTION_CONTEXT_STALE`: actual `(True, [])`, + required false. This was a runtime-free test, not market evidence. Proposal: + require the actual gap/generation refusal and retained-book age at fault time; + reject generic STALE, missing-book or unrelated EXECUTION_CONTEXT errors. +4. Static Binance mismatch: `APPROVE_PHASES` includes positive_anchor and + return_anchor, requiring their exact quantities to be approved before the next + delta. The DL handoff explicitly requires Binance bootstrap continuity first; + generation recovery can replay a buffered delta instead of exposing anchor + quantity5. Coordinate venue-aware expectations with existing DL schedule; + do not change domain logic or fabricate visibility. Binance was not run. +5. Both cleanup attempts encountered `cleanup incomplete: + helper:CalledProcessError` after killing the JSONL Docker client. Exact-name + rechecks found no containers; private erasure then completed. Small DL-owned + proposal, only after parent assigns ownership of + `scripts/p184_dr05_dl_fault.py` and its test: when stop fails, accept absence + only after rechecking the exact previously verified disposable identity. + +No source fix was attempted across another owner's files. Next permitted step: +coordinate TS startup/deadline/evaluator fixes and narrow DL cleanup ownership, +then one fresh bounded positive-control run before attempting fault cells. +Final TS qualification, release-image packaging and v2.2.3 remain HOLD. + +## Method And Provenance + +Unchanged TS candidate modules were imported read-only from +`/home/bobby/.worktrees/trading-system-next-upgrade`, inspected HEAD +`cc76c05ab3f3a18babe14b3ab20dd51a790e5599`, adapter commit +`01eb0d79161687eba578ce185c93d314fdc30255`. Existing DL init/up, host reset watcher, +JSONL channel, TS WriterClone/TsDr05Adapter and production BOOK_PROBE were used. +A host inline orchestrator used the real DL schedule, stopped at the first failed +step, and retained acceptance=false rather than emitting final CaseEvidence. +No monkeypatch to domain functions, timeout/freshness relaxation or image build. + +| Artifact | SHA256 / revision | +| --- | --- | +| TS candidate p184_dr05_upstream.py | 57186fc65e134cc87ba66399fabeeabf53486de183a7c1a002ef8c6b44b2803b | +| TS candidate p184_dr05_isolated.py | e29860f6cfee54bf498d258a8fc0b413e20ff13c626f4449bacc998fbd920c88 | +| Retained TS image | a6548ee36aeb0242c1734c071678c525c1506d05c10a70b533eb7110e7eaf16a | +| TS image revision label | 2abbe394ea0fb8f7f5ebcbb745ff70a8c83c9769 | +| TS in-image market_data/main.py | 4f69633bbc83076f3bf68aff587008467a2aad4882efe9c9ffcf3b8640e82bf7 | +| TS in-image execution_market_context.py | 54169bd15e254223891d257d0538ce7d2d825febfe87821b20feefef4750aeb3 | +| Retained core executable | baafcb489bb17fabdd73bef383dcf3448581cc8bb63578e9df83e2a57107b6f2 | + +The TS image does NOT contain p184_dr05_upstream.py. Its older isolated.py hashes +71648d9a489c66ab29dc73738d1b686470cecf6b89dc2473de0e621a8c340dfd, +not the imported candidate bytes. Do not claim that this image contains adapter +01eb0d7 or that candidate TS application code ran in it. Network-none file/hash +preflight inspected the retained image; it did not validate application behavior. +Core reused the explicit handoff binary at +`~/.local/state/qdl-v2/dr05-fence-20261008/bin/qdl-realtime-core-baafcb48`. +DL source at start: `0b374bc918af02476cdfb1502d5df132426f901b`. +Query used existing image77682eea with DL source mounted read-only, not a final +release image. Full reused image digests are in the corrected-run receipt. + +Synthetic catalog-derived MaterialTests.exported core bindings only; fresh +TEST_ONLY TLS/JWT generated offline. Broker network none, no published ports; +all other disposable roles joined its namespace. Clone Redis6391/PG5433 exist +only there. A supplied WriterClone inspect argument contained an explicit clean +environment, no shared writer inspection/secrets/mounts. UID/GID matched the +owned work directory for private-file access. V2_PRIMARY and EPHEMERAL_ONLY were +explicit; fallback URL was loopback port1. This is a configured disposable clone, +not a byte-identical production environment or shared-runtime acceptance. + +## Attempts, Receipts And Cleanup + +All receipt paths below are under `~/.local/state/qdl-v2/` and outside Git. + +| Attempt | Result | Receipt SHA256 | +| --- | --- | --- | +| dr05-writerclone-followup-20261008T154437Z-4e5659/cleanup-recovery.json | First anchor aborted; recovery-only receipt | 1e1cf02a1850a51ca6ef7a60b74aae42818ea0bc2fdecac8008e082667a4f9a4 | +| dr05-writerclone-followup-20261008T154751Z-fbf95f/receipt.json | Corrected attempt94.413106s; canonical anchor observed, TS unavailable | 587c1244bbf474deea8e17b8f2d1a6e0d0ef7c8f37e24ca639c4606dd514eeb1 | + +First attempt supplied invalid temporary mode PROJECTED_PRIMARY; corrected to +the declared EPHEMERAL_ONLY before the second attempt. First-attempt in-memory +observations were lost when cleanup raised; its receipt is expressly cleanup +recovery only, not reconstructed validation. Its writer-exit root cause was not +retained. No third/blind repeat occurred. Both outer runs had a240s bound, fresh +namespaces and stop-on-first-failure. These are attempts, not passing test cases. + +Namespaces `p184-dr05-dl-01fcf029155e4389` and +`p184-dr05-dl-cc0a5f9ba9854619`: owned DL five-role stacks, channels, reset +watchers and TS writer/Redis/Postgres removed; privatefiles0 and work directories +removed. Final `docker ps -a --filter name=p184-dr05` returned no containers. +No image was built/deleted and no shared volume/network/cache was pruned. +First recovery cleanup free bytes13006569472 ->12999852032; corrected full run +12793827328 ->11175157760. Concurrent host builds prevent attribution to this +small test; no claim of disk consumption or reclamation from these deltas. +Read-only `docker system df`:95images28.96GB, BuildKit467entries20.52GB. +Shared cache cleanup intentionally deferred during concurrent parent/TS work. +Retain active images, explicitly protected BAR rollbackf7351c3b and handoff binary. + +## Runtime And Ownership Handoff + +Canonical DL `/home/bobby/data_layer`, main +`5bf5cd540498b82959342c7cfb2ebeaac3007426`, publishedv2.2.2 unchanged. +Active DL feature worktree `/home/bobby/.worktrees/data-layer-p184-closure`, +fix/p184-dl-closure at0b374bc when inspected; only this active feature worktree +was listed. TS checkout/branch feat/v2-rust-first-okx-demo is Claude-owned and +had unrelated dirty files; none touched. Parent concurrently staged DL changes; +this investigation did not stage, commit, merge or push anything. + +Read-only shared runtime snapshot:17roles, restart0/OOMfalse. Services: +rust_core/2/3 (658a9570), ingestor_binance_usdm and ingestor_okx_swap (0f6876e1), +query_kn_1/2 and binance_bar_edge (51450612), market_projector_1/2 (a696cfba), +stream_kn_1/2 (1f13408f), kafka1/2/3 (9516fb76), market_cache and stable_redis +(dfa18828). Full image identities match the prior fence handoff snapshot. +Compose config hashes were inspected read-only; no configuration/release +revision, service, provider, authority, shared credentials or shared data changed. +This snapshot is not new certification or a claim of final-candidate deployment. + +Parent must integrate this BLOCKED result and receipt hashes into the main +journal. The journal and final acceptance remain open; this report does not +close the phase. In particular actual TS typed gap/generation, recovery, ACK and +control/cache invalidation evidence is still missing. + +## Assigned DL Cleanup Fix And Claude Handoff + +Cleanup status: IMPLEMENTED_TESTED_LOCALLY, not upstream fault qualification. +Owner assigned only scripts/p184_dr05_dl_fault.py and its tests after reviewing +this report. Parent recorded the pre-edit approval in the main journal. No plan +edit/staging/commit by this agent; parent integrates these results. Rollback is +source-only withdrawal of this narrow uncommitted diff, not a runtime operation. + +Cleanup now pins each preflight-inspected, correctly named/labelled AutoRemove +container. It accepts a failed stop only for exit1 on the exact stop command, +with Docker's exact "No such container: " stderr, followed by successful +empty full-ID and exact-name listings. Other stop errors, failed listings, failed +inspection, retained containers and either authorized or unauthorized same-name +replacements fail closed. No replacement is stopped. Successful stop also needs +the pinned ID to be absent, not just the name. Cleanup disables owned()'s old +inspection-disappearance tolerance; unrelated owned() callers keep their existing +behavior. No protobuf, market semantics, freshness, TS source or release changes. + +Verification, with overlapping suites NOT summed: +- Old-source bounded lifecycle counterfactual:12tests,3failures+1error,0skip, + 0.011s. Detects missing stop-race handling, both inspection-error paths and + waiting on a replacement. The first counterfactual used a mocked sleep that + produced excessive failure output; it was bounded before the recorded rerun. +- Initial portable focused suite:29PASS,0fail/error/skip,0.025s. +- Initial dependency-image four-module suite:81PASS,0fail/error/skip,89.028s. +- Final portable focused suite:30PASS,0fail/error/skip,0.047s. +- Final dependency-image four-module suite:82PASS,0fail/error/skip,87.401s; + outer elapsed90.286593s, exit0. This is the final-source total. +- Agent expanded host attempt:78tests,70pass,0fail,8import errors,0skip, + 0.614s, due to absent host google.protobuf/redis. These are dependency + limitations, not source failures; the corresponding image suite passed. +- Parent-reported separate host attempt:29tests with1Redis import error. + Preserve this failed attempt as host-dependency evidence, not source failure. + Parent's image rerun p184-dr05-cleanup-parent-test is parent-owned; its result + is not invented or included in the agent totals. +- git diff --check passed. New cases cover exact absence, still-present ID, + same-name authorized/foreign replacement, wrong-ID/unrelated/multi-error stop + responses, failed ID/name listings, inspection errors before/after stop, + stopped replacements and name disappearance while the pinned ID remains. + +Final image command (inside existing image77682eea): +python -B -m unittest tests.test_p184_dr05_dl_fault tests.test_p184_dr05_case +tests.test_p184_dr05_channel tests.test_p184_dr05_reset -q +Final portable command: +python3 -B -m unittest tests.test_p184_dr05_dl_fault.LifecycleTests +tests.test_p184_dr05_dl_fault.PreparationTests +tests.test_p184_dr05_dl_fault.TimeoutTests tests.test_p184_dr05_reset -q + +Both image checks used fresh --rm network-none/read-only helpers,0.5CPU/512MiB, +128PIDs,64MiB /tmp tmpfs, read-only DL source, no Docker socket/secrets, no image +build and a180s host deadline. Fake Docker lifecycle tests plus offline material +generation only; no WriterClone, upstream fault retry or shared runtime mutation. +Helpers p184-dr05-cleanup-unit-57e89c7bcba3 and +p184-dr05-cleanup-unit-55928093b694 were removed, each verified remaining0. +Temporary material lived in deleted test directories/tmpfs; no new private files +or test state retained. Parent helper was not stopped or otherwise touched. +Shared images/cache and named BAR rollbackf7351c3b remain protected; no prune. +Pre-final-helper-cleanup disk observation:1461370880available bytes; concurrent +host builds mean this is not attributed test consumption. No persistent host +artifact cleanup was performed by this fix. + +Current inspected parent HEAD fa10d1b5627cbec0a44eb9f70760e96d672a76c2, +fix/p184-dl-closure, canonical main5bf5cd5/v2.2.2 and previously recorded runtime +image/config/service identities unchanged by this work. Parent capacity changes +are preserved. Only the two assigned source/test files and this report were edited. +No commit, push, merge or deployment. Parent main-journal closure remains owned +by parent; the broader DR05 phase is still BLOCKED_BEFORE_TS_FAULT_VALIDATION. + +Claude handoff for parent delivery: the DL cleanup race is locally fixed; the +original TS findings1-4 and both runtime receipts above remain unchanged. +Address bounded Postgres/Redis readiness, overall startup deadline enforcement, +typed gap/generation (not generic STALE) verification, and venue-aware Binance +anchor expectations in the TS-owned helper. Parent independently confirmed the +generic-STALE counterexample. No new WriterClone attempt is authorized by this +cleanup-only assignment. Candidate helper byte provenance is not image packaging, +and no actual TS gap/generation/ACK/control-cache qualification was obtained. +No direct Claude messaging transport is available in this session; this report +is the handoff artifact for the parent to relay. From 2dbfe313b35b747dcacae72ec1c8e77caf2018a6 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 16:25:10 +0000 Subject: [PATCH 042/147] Record scoped unused-image and build-cache cleanup --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 4b90a787..a49221d8 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70609,3 +70609,50 @@ Final readonly disk recheck worsened to28430336bytes (~27.1MiB) available, root1 Do not build/load until the concurrent writer/build owner restores disk headroom. All p184-capacity100 and p184-dr05 containers absent, worktree clean before this journal-only update. No causal attribution without process/disk investigation. + +### Owner-approved unused image and build-cache cleanup - 2026-10-08 + +Status IN_PROGRESS. Owner explicitly authorizes deleting unnecessary images and +unused build cache. Before cleanup root available17297408bytes,100%; Docker95 +images28.96GB/reclaimable8.491GB,467cacheentries20.52GB/reclaimable2.852GB. +Kafka brokers already restart-loop before action. Scope: unused BuildKit cache +older than1h (BuildKit retains in-use refs), then exact superseded DL/test image +IDs after excluding ALL running/stopped container references, active candidates, +named rollbacks and build bases. No force image deletion, container/volume/network +removal, service restart or data/log deletion. Preserve source and evidence. +Capture bounded pre/post image/container/restart inventory outside Git; compare +runtime changes and record actual reclaimed bytes. No push/merge or release. + +First action: docker buildx prune --force --filter until=1h exited0, reported +2.852GB reclaimed. Root free rose to101073723392bytes; gain exceeds this action, +so concurrent cleanup/storage activity remains unattributed. No active cargo, +rustc or docker-build/buildx process found in sanitized process scan afterward. +Exact image cleanup allowlist: DLsuperseded0ed57643,b40b49c3,65918a65,f2040ac9, +3af57ddf and retiredrc3candidate0d8da150; TSsupersededtest7a02c808,f2cc1f8d, +3b4ee780 (no current script/Dockerfile/compose references). Preserve all69 +container image references including stopped rollback/operational containers; +retainrc2Python77682/Rust69e3,buildbases,TSfinala654,alphaimages,allrollbacktags, +BARf7351c3b,corehistorical7fe348 pendingretentionreview. Recheck beforeeachrmi; +no --force. Detailed inventory/receipt outsideGit cleanup-20261008-unused-artifacts. + +Cleanup result SCOPED_CLEANUP_COMPLETE_KAFKA_UNRECOVERED. All9 allowlistedimages +removed withoutforce, all69containers retained (including stopped rollback), no +runtimeimage/start/restartcounter delta during imagecleanup, no volumes/networks +or Kafka/DB/logdata touched. Second identical cacheprune reclaimed128.5MB after +image deletion. Buildcache467->196entries,20.52->17.54GB,reportedreclaimable0B; +35activecache refs retained. Images95->86. Remaining9.003GB Dockerlabelsreclaimable +is NOT permission to delete activecandidate/buildbase/namedrollback artifacts. +Exact retention inventory: retained-images.json f27db1c65b58b16c1ba00cf4e6cc10ec76646477b7c24d3062cc71677611cbe9. +No cargo/rustc/dockerbuild process was seen during the sanitized safety scan. +Disk17297408->101289484288freebytes (100%->68%used). Only2.852GB+128.5MB cache +reclamation attributed to these cachecommands; muchlarger free-spacegain has +concurrent/unidentified cause. Do not claim cleanup recovered101GB. +Kafka1/2/3 stillrestart-loop, logsconfirm ENOSPC at2026-10-08T16:09:12Z before +cleanup. No manual restart attempted; quorum/freshness NOT recertified. Broker +recovery/integrity requires its own scoped operation, not another broadcleanup. +Receipt 0af529653048ce4b8662f9c16a6c72ec080eef86f75abfb03e0bc0a2d3ec1e07, outsideGitroot cleanup-20261008-unused-artifacts; +fullimagecommands/digests/containerinventory and exactretentions in that directory. +Canonical /home/bobby/data_layer main5bf5cd5/v2.2.2 unchanged; featureworktree +/home/bobby/.worktrees/data-layer-p184-closure fix/p184-dl-closure retainedactive. +All runtime digests/configs preserved, BARrollbackf7351c3b andTSrollback artifacts +retained. No source/runtimecodechange, no testsrerun, no push/merge/release. From f44c0a9f15650a628b075e5964b037ae2bf549ed Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 16:41:46 +0000 Subject: [PATCH 043/147] Record read-only post-disk recovery and remaining consumer lag --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 27 +++++++ upgrade/evidence/p184-post-disk-recovery.md | 88 +++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 upgrade/evidence/p184-post-disk-recovery.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index a49221d8..7b2eba01 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70656,3 +70656,30 @@ Canonical /home/bobby/data_layer main5bf5cd5/v2.2.2 unchanged; featureworktree /home/bobby/.worktrees/data-layer-p184-closure fix/p184-dl-closure retainedactive. All runtime digests/configs preserved, BARrollbackf7351c3b andTSrollback artifacts retained. No source/runtimecodechange, no testsrerun, no push/merge/release. + +### Post-disk Kafka and consumer recovery verification - 2026-10-08 + +Status IN_PROGRESS; owner approved read-only recovery verification before any load. +Scope bounded Kafka quorum/ISR and committed group-lag sampling, projector/cache +progress, actual TS cache observation and bounded error logs. Reuse existing +admin identity and observers; no second TS Stream subscription/quota use, orders, +config/ACL/schema edits, reset-offsets, lifecycle action or stress. Process health +alone cannot pass recovery. Record missing evidence/refusals separately. No broad +catalogue rerun or resource increase. Evidence private directory outside Git; +only bounded receipts/hashes into journal. Temporary read-only helpers --rm. + +Read-only recovery verdict INFRA_PROGRESS_RECOVERED_CONSUMER_ACCEPTANCE_INCOMPLETE. +Quorum3voters/leader3/followerlag0;ISRempty twice.6samples99.814s/36group-partitions, +committedoffsetsnondecreasing,noevery-samplegrowinglag. Corepeak172/final71records; +projectorA142/90,B116/16. Projectorinputs+30933/+62098,latestapply+16094/+12570, +no newstatuserrors/conflicts/cache-reconnects. Runtimeidentities/restartsunchanged. +Initial adminport9093 mistake rejected, corrected9092; not a provider/runtimebug. +TS120.095s observer saw60execution/BARgroups but INCOMPLETE:7oversizedmessages +hit observer256KiB limit. BOOKDELTAmax16676.651ms(BinanceETH,n147),TRADE9810.625ms +(n170). These are eventages at cache-read, not requestlatency/RedisACK. TSheartbeat +READY60/60 n1 does not certify all-vieweligibility. Heavyload remainsHOLD. +Guide/report upgrade/evidence/p184-post-disk-recovery.md includes feedtable, +limits,commands and receiptd0fc2764c74aa5bd5a8c7f28b8d72c44da35f1df63b62518b30295b9efe3e542. No newcontainers/images/TSsourceedits or +runtimeconfig/ACL changes; allreadprocessesended, noadditionalcleanupneeded. +Canonicalmain5bf5cd5/v2.2.2 unchanged;featurefix/p184-dl-closure retained, +v2.2.3HOLD. No push/merge. Existingdownstreamgap not proven caused bydisk. diff --git a/upgrade/evidence/p184-post-disk-recovery.md b/upgrade/evidence/p184-post-disk-recovery.md new file mode 100644 index 00000000..2c2e6588 --- /dev/null +++ b/upgrade/evidence/p184-post-disk-recovery.md @@ -0,0 +1,88 @@ +# Post-Disk Recovery Verification + +2026-10-08. Status: INFRA_PROGRESS_RECOVERED_CONSUMER_ACCEPTANCE_INCOMPLETE. +Owner scope: bounded read-only verification before load; no rollout, reset, +offset alteration, orders, source fix or freshness/authority policy change. + +## Kafka And Materialization + +Existing admin.properties, broker endpoints kafka1/2/3:9092. Quorum describe: +leader3, epoch150,3voters, max follower lag0 (52ms reported time). Under-replicated +partition describe exited0 with empty output both before and after observation. +First3 admin commands mistakenly targeted controller9093 and were refused with +UnsupportedVersionException. Corrected to the existing broker9092 contract; +this was an observer mistake, not a new runtime fault or configuration change. + +Six all-group committed-offset samples span99.814s,36 group/partition rows: +all sampled offsets nondecreasing, no partition lag strictly grew at every sample. +These are record-count samples, NOT a proof of zero event delay or no historical +loss during disk exhaustion. Kafka transaction/control markers may contribute to +reported offset lag; do not force lag0 by offset reset. + +| Group | Peak lag, records | Final max lag, records | +| --- | ---: | ---: | +| Core | 172 | 71 | +| Projector A canonical | 142 | 90 | +| Projector B bars/latest | 116 | 16 | +| Binance resync feedback | 3 | 3 | +| OKX resync feedback | 5 | 5 | + +Projector1/2 input advancement30933/62098; latest cache applied16094/12570. +No new status errors, conflict increments or cache reconnects across these samples. +The bounded3minute log check found no selected ERROR/Traceback/ENOSPC lines in +three brokers or TS stdout/stderr. This is not an exhaustive audit of mounted +application logs. Runtime IDs/images/start times/restart counts stayed unchanged +during the measured interval; no observed OOM. No manual restart was performed. + +## Real TS Cache + +Existing scripts/measure_kn_ts_cache_delivery.py ran for120.095s via stdin inside +market_data_service_p184w, using its own configured Redis URL without printing it. +No additional DL Stream subscription or TS quota consumed. External PubSub/MGET +observation, NOT Redis SET ACK measurement. All60 execution/BAR groups observed; +legacy last-trade/quote paths counted separately and not added to60. + +The table lists the single worst-max instrument in each feed, NOT pooled +percentiles. Samples are observer time samples, not necessarily unique events. +All values ms. BAR time origin is1m close; MARK age is not component eligibility. + +| Feed / worst instrument | n | p50 age | p95 age | max age | +| --- | ---: | ---: | ---: | ---: | +| QUOTE / OKX ETH | 178 | 391.192 | 883.025 | 1461.599 | +| TRADE / Binance ETH | 170 | 469.686 | 5633.734 | 9810.625 | +| BOOK_DELTA / Binance ETH | 147 | 1708.987 | 14003.404 | 16676.651 | +| BOOK_SNAPSHOT / OKX SOL | 109 | 851.333 | 1498.599 | 2237.455 | +| MARK_INDEX / Binance DOGE | 93 | 1663.906 | 2625.370 | 2951.333 | +| BAR1m / OKX DOGE | 2 | 1610.568 | insufficient n | 2160.823 | + +MGET batch n183,p50106.778,p95348.949,max612.189ms. This includes observer/client +cost, not Query request latency. Observer exit1/INCOMPLETE:7MESSAGE_SIZE_LIMIT +notifications exceeded its256KiB bound; discarded by observer, not evidence of +DL dropping these records. Do not relabel this run PASS or silently change limits. +No simultaneous provider witness establishes quiet market versus pipeline delay +for TRADE. BOOK_DELTA delay remains open; no inference that disk caused the +pre-existing downstream issue. Canonical-to-ACK/root-cause trace still required. + +One real writer heartbeat read:READY60/60,age7.3s,fallback0,v2error0. Readiness +is NOT proof every stored view remains execution-eligible at the time of use. +No decision to start100-alpha load follows from this heartbeat. + +## Evidence, Cleanup And Next Boundary + +OutsideGit: /home/bobby/.local/state/qdl-v2/recovery-after-disk-20261008. +Receipt SHA256 d0fc2764c74aa5bd5a8c7f28b8d72c44da35f1df63b62518b30295b9efe3e542. Per-command outputs and bounded metric hashes in receipt. +Source helper bytes from feature2dbfe31; no application changes. No containers, +images or persistent test services created; exec observer/admin processes ended. +No cleanup needed beyond process completion, no shared data deletion/prune. +Prior cleanup retention set and namedBARrollbackf7351c3b preserved. + +Canonical /home/bobby/data_layer main5bf5cd5, publishedv2.2.2. Activefeature +/home/bobby/.worktrees/data-layer-p184-closure fix/p184-dl-closure. +Runtime unchanged: core658a9570,ingestor0f6876e1,Query/BAR51450612, +projectora696cfba,Stream1f13408f,writer5849d716. Detailed full digests/config and +state identities in runtime-during.json; releasev2.2.3HOLD remains. + +Recovery observation is completed, not whole-system certification. Kafka and +materialization are moving again; downstream BOOK continuity/throughput and +observer payload coverage must close before heavy load. Continue existing +DR05/DR08/caller work, do not create a new broad Data Layer campaign. From 78d3aff0e6d41aa9dabfae97ef99f4d8c4434436 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 17:07:41 +0000 Subject: [PATCH 044/147] Fix bounded cache observer coverage for large BOOK notifications --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 28 ++++++++++ scripts/measure_kn_ts_cache_delivery.py | 32 ++++++++++-- tests/test_kn_ts_cache_delivery_observer.py | 33 ++++++++++++ upgrade/evidence/p184-post-disk-recovery.md | 57 +++++++++++++++++++++ 4 files changed, 146 insertions(+), 4 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 7b2eba01..104737bc 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70683,3 +70683,31 @@ limits,commands and receiptd0fc2764c74aa5bd5a8c7f28b8d72c44da35f1df63b62518b3029 runtimeconfig/ACL changes; allreadprocessesended, noadditionalcleanupneeded. Canonicalmain5bf5cd5/v2.2.2 unchanged;featurefix/p184-dl-closure retained, v2.2.3HOLD. No push/merge. Existingdownstreamgap not proven caused bydisk. + +### Cache observer bounded payload repair - 2026-10-08 + +IN_PROGRESS. Owner approved closing the seven observer size refusals and continuing +BOOK consumer attribution, not broad catalogue/load or runtime changes. Guide: +upgrade/evidence/p184-post-disk-recovery.md and KN architecture review. Observer +only: finite per-message and aggregate retained bytes, bounded size attribution, +cache payload validation, regression for large valid BOOK and over-budget inputs. +No production buffer, freshness or eligibility relaxation. Real cache probe uses +Redis PubSub/MGET only; ACK stays NOT_MEASURED unless instrumented at the writer. +TS source is Claude-owned and read-only. Rollback is observer source only. Tests +and a bounded real run precede receipt; no new image. apply_patch helper failed +with mountinfo-path error; exact scripted replacement used and diff verified. + +Observer repair tested:32unittest PASS; oldsource large-valid-message regression +fails as expected. Real120.322s/118215notifications includes8above256KiB, max496519 +bytes; zeroobservererrors,60execution/BARgroups. Pendingpeak1029467bytes below8MiB. +New bounds affect only observer; freshness/productionqueues unchanged. +BOOK_DELTA still max13375.109ms(OKXBTC,n135); freshRedisACK NOT_MEASURED because +writertrace expired. Existing trace hook is deployed, newer slice logging not. +Status OBSERVER_REPAIRED_BOOK_CONTINUITY_OPEN. Report and next caller dependency: +upgrade/evidence/p184-post-disk-recovery.md#observer-repair-and-fresh-observation. +NoTSedit/rollout/newimage; temporary --rm container removed.86images/69containers/ +196buildcacheentries retained,0Breclaimablecache; free99461668864bytes. No pruning. +Two pytest attempts unavailable; all32tests run with stdlibunittest instead. +Canonicalmain5bf5cd5/v2.2.2 and runtimeidentities unchanged; feature retained, +v2.2.3HOLD, no push/merge. Continue caller trace under single-writer packet, +not broad catalogue or load. diff --git a/scripts/measure_kn_ts_cache_delivery.py b/scripts/measure_kn_ts_cache_delivery.py index 04549a6f..f46984d4 100644 --- a/scripts/measure_kn_ts_cache_delivery.py +++ b/scripts/measure_kn_ts_cache_delivery.py @@ -110,10 +110,14 @@ def context(value): "watermark": watermark, "current_watermark": current_watermark} -async def observe(client, pub, seconds, *, period=.5, max_channels=128, max_bytes=262144): +async def observe(client, pub, seconds, *, period=.5, max_channels=128, max_bytes=1048576, max_pending_bytes=8388608): # Draining never awaits Redis GET. One bounded MGET samples the latest # notification per channel; this is observer sampling, not feed coalescing. pending, counts, errors = {}, Counter(), Counter() + pending_bytes = peak_pending_bytes = 0 + message_sizes = {} + if min(max_bytes, max_pending_bytes, max_channels) <= 0: + raise ValueError("observer bounds must be positive") samples, current_ages, notification_ages, residence = (defaultdict(list) for _ in range(4)) matches, eligible = Counter(), Counter() admitted_groups = set() @@ -121,6 +125,7 @@ async def observe(client, pub, seconds, *, period=.5, max_channels=128, max_byte started = time.monotonic() deadline = started + seconds async def drain(): + nonlocal pending_bytes, peak_pending_bytes received = 0 while time.monotonic() < deadline: item = await pub.get_message(ignore_subscribe_messages=True, timeout=.25) @@ -132,9 +137,23 @@ async def drain(): errors["CHANNEL_LIMIT"] += 1 continue counts[channel] += 1 - if len(item["data"]) > max_bytes: + size = len(item["data"]) + stats = message_sizes.setdefault(channel, {"n": 0, "max_bytes": 0, "over_256k": 0, "rejected": 0}) + stats["n"] += 1 + stats["max_bytes"] = max(stats["max_bytes"], size) + stats["over_256k"] += int(size > 262144) + if size > max_bytes: errors["MESSAGE_SIZE_LIMIT"] += 1 + stats["rejected"] += 1 continue + previous_size = len(pending[channel][0]) if channel in pending else 0 + next_bytes = pending_bytes - previous_size + size + if next_bytes > max_pending_bytes: + errors["PENDING_BYTES_LIMIT"] += 1 + stats["rejected"] += 1 + continue + pending_bytes = next_bytes + peak_pending_bytes = max(peak_pending_bytes, pending_bytes) pending[channel] = (item["data"], time.monotonic()) received += 1 if received % 128 == 0: @@ -147,6 +166,7 @@ async def drain(): await task selected = list(pending.items()) pending.clear() + pending_bytes = 0 decoded = [] for channel, (raw, received) in selected: try: @@ -174,6 +194,9 @@ async def drain(): if value is None: errors["CACHE_MISSING:" + group] += 1 continue + if len(value) > max_bytes: + errors["CACHE_SIZE_LIMIT"] += 1 + continue try: current = json.loads(value) result = compare_observation(channel, payload, current, now_ms) @@ -210,13 +233,14 @@ async def drain(): "replica_attribution": "TS Redis only; no Query replica attribution", "sampling": {"mode": "latest notification per channel; time samples, not unique events; execution/last paths separate", "period_seconds": period, "max_channels": max_channels, - "max_notification_bytes": max_bytes, "max_samples_per_group": 2000}, + "max_notification_bytes": max_bytes, "max_pending_bytes": max_pending_bytes, + "peak_pending_bytes": peak_pending_bytes, "max_samples_per_group": 2000}, "event_to_cache_readable": {k: summary(v) for k,v in samples.items()}, "current_cache_age_ms": {k: summary(v) for k,v in current_ages.items()}, "notification_age_ms": {k: summary(v) for k,v in notification_ages.items()}, "observer_residence_ms": {k: summary(v) for k,v in residence.items()}, "cache_mget_batch_ms": summary(batches), "observed_messages": dict(counts), - "matches": dict(matches), "eligibility_at_write_not_revalidated": dict(eligible), + "message_sizes": message_sizes, "matches": dict(matches), "eligibility_at_write_not_revalidated": dict(eligible), "errors": dict(errors), "writes": 0, "order_actions": 0, "redis_set_ack_latency": "NOT_MEASURED", "canonical_commit_latency": "NOT_MEASURED"} diff --git a/tests/test_kn_ts_cache_delivery_observer.py b/tests/test_kn_ts_cache_delivery_observer.py index 63a388c2..22ac8b5f 100644 --- a/tests/test_kn_ts_cache_delivery_observer.py +++ b/tests/test_kn_ts_cache_delivery_observer.py @@ -180,6 +180,39 @@ async def test_size_limit(self): result=await observer.observe(Client(view()),Pub([message(view())]),.02,period=.005,max_bytes=5) self.assertIn("MESSAGE_SIZE_LIMIT",result["errors"]) + async def test_large_valid_book_is_observed(self): + v = view() + v["padding"] = "x" * 300000 + result = await observer.observe(Client(v), Pub([message(v)]), .03, period=.005) + self.assertEqual(result["status"], "OBSERVED") + self.assertEqual(result["message_sizes"][CHANNEL]["over_256k"], 1) + self.assertTrue(result["event_to_cache_readable"]) + self.assertEqual(result["redis_set_ack_latency"], "NOT_MEASURED") + + async def test_aggregate_budget_rejects_without_hiding_loss(self): + v = view() + size = len(message(v)["data"]) + result = await observer.observe(Client(v), Pub([message(v), message(v, CHANNEL+".extra")]), + .03, period=.005, max_pending_bytes=size) + self.assertEqual(result["status"], "INCOMPLETE") + self.assertIn("PENDING_BYTES_LIMIT", result["errors"]) + self.assertLessEqual(result["sampling"]["peak_pending_bytes"], size) + + async def test_replacement_does_not_accumulate_bytes(self): + v = view() + size = len(message(v)["data"]) + result = await observer.observe(Client(v), Pub([message(v)] * 20), + .04, period=.005, max_pending_bytes=size) + self.assertEqual(result["status"], "OBSERVED") + self.assertEqual(result["sampling"]["peak_pending_bytes"], size) + + async def test_oversized_cache_is_incomplete(self): + v = view() + current = dict(v, padding="x" * 1100000) + result = await observer.observe(Client(current), Pub([message(v)]), .03, period=.005) + self.assertEqual(result["status"], "INCOMPLETE") + self.assertIn("CACHE_SIZE_LIMIT", result["errors"]) + async def test_channel_limit(self): v=view();result=await observer.observe(Client(v),Pub([message(v),message(v,CHANNEL+".extra")]),.03,period=.005,max_channels=1) self.assertIn("CHANNEL_LIMIT",result["errors"]) diff --git a/upgrade/evidence/p184-post-disk-recovery.md b/upgrade/evidence/p184-post-disk-recovery.md index 2c2e6588..526226fa 100644 --- a/upgrade/evidence/p184-post-disk-recovery.md +++ b/upgrade/evidence/p184-post-disk-recovery.md @@ -86,3 +86,60 @@ Recovery observation is completed, not whole-system certification. Kafka and materialization are moving again; downstream BOOK continuity/throughput and observer payload coverage must close before heavy load. Continue existing DR05/DR08/caller work, do not create a new broad Data Layer campaign. + +## Observer Repair And Fresh Observation + +2026-10-08: OBSERVER_REPAIRED_BOOK_CONTINUITY_OPEN. Observer per-message +limit is now 1 MiB; latest pending notifications have an aggregate 8 MiB cap. +These are observer bounds, not changes to production queues or eligibility. +Counters retain channel, size, over-256KiB count and rejections without payload. +Cache objects exceeding the decode bound remain INCOMPLETE. Pending serialized +bytes are bounded; this is not an RSS bound on Redis parser/decoded objects. + +Real run 120.322s: 118215 messages, 8 above256KiB, max496519 bytes on Binance +BTC BOOK_DELTA; zero size/budget/other observer errors. Peak pending1029467 bytes. +All60 execution/BAR groups observed. Status OBSERVED is sampling completeness, +NOT execution eligibility or full event delivery. Old7refusals remain unchanged. +MGET176batches p50137.536/p95350.644/max477.204ms: observer roundtrip only. + +Worst-max instrument per feed, source-event age at cache read, ms; time samples +not unique events or a paired improvement benchmark: + +| Feed / instrument | n | p50 | p95 | max | +| --- | ---: | ---: | ---: | ---: | +| QUOTE OKX BTC | 175 | 359.840 | 678.661 | 1171.773 | +| TRADE OKX BNB | 86 | 740.769 | 2251.782 | 5306.195 | +| BOOK_DELTA OKX BTC | 135 | 6828.955 | 11777.762 | 13375.109 | +| BOOK_SNAPSHOT OKX ETH | 112 | 931.998 | 1436.762 | 1972.581 | +| MARK_INDEX Binance BTC | 90 | 1592.823 | 2501.769 | 3370.146 | +| BAR1m OKX BNB | 2 | 626.366 | insufficient n | 3202.268 | + +Evidence outsideGit: observer-payload-20261008/observer.json sha256 +a6ebfac67aa730fdf71890df052c9f04d71e2eebdf60cb8b54355e8a69c66146. Receipt and18selectedruntimeidentities alongside. +32unittest PASS; large-payload regression fails on previous source as expected. +Host and retainedPythonimage have no pytest; those attempts failed before tests. +Used stdlib unittest for both test modules; no test silently skipped. + +### BOOK Caller Handoff + +Existing writer5849d716, SDK2.0.7 retains trace hooks but expiry is +2026-10-07T19:24:14Z. Fresh canonical-to-Redis-ACK remains NOT_MEASURED. +Historical same-event summary at dr07-runtime-20261007/continuation/ +writer-same-event-preliminary/summary.json has2556ETH BOOKjoins, no validation +failures; SDK-to-RedisACK p95 Binance230.4/OKX243.2ms. This is historical, not +current recovery proof and does not isolate Stream versus SDK pre-writer delay. +Current BOOK writes and cursor ACK are serialized per batch. That is a candidate +bottleneck, not a proved cause warranting concurrent writes that could break CAS. +Do not overwrite Claude-owned TS source or run a second subscription. + +Next exact dependency: coordinate one current-writer packet, preserved credentials/ +namespace/cursors and named rollback, bounded trace expiry <=600s, max4096records +perprocess, plus SDK ACK/cursor correlation in Claude-owned caller. Existing +SDK-return/Redis-EVAL hooks alone cannot prove exact cursor persistence timing. +No runtime activation performed here. No claim of complete BOOK remediation. + +No images built. Disposable failedpytest container auto-removed. Dockerinventory +86images/69containers/196cacheentries unchanged; buildcache reclaimable0B. +Disk available99461668864bytes at check. No deletion/prune needed this slice; +active images and explicit rollbacks retained. Canonicalmain5bf5cd5/v2.2.2, +featurefix/p184-dl-closure unchanged; v2.2.3 remains HOLD. From bf26e1c62c27a1ba0515c277a523a70e905d885a Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 17:31:15 +0000 Subject: [PATCH 045/147] Measure BOOK consumer wait through exact-event Redis ACK trace --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 33 +++ scripts/p184_book_trace_join.py | 253 ++++++++++++++++++ tests/test_p184_book_trace_join.py | 216 +++++++++++++++ .../evidence/p184-book-ack-trace-20261008.md | 131 +++++++++ 4 files changed, 633 insertions(+) create mode 100644 scripts/p184_book_trace_join.py create mode 100644 tests/test_p184_book_trace_join.py create mode 100644 upgrade/evidence/p184-book-ack-trace-20261008.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 104737bc..6c3d2963 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70711,3 +70711,36 @@ Two pytest attempts unavailable; all32tests run with stdlibunittest instead. Canonicalmain5bf5cd5/v2.2.2 and runtimeidentities unchanged; feature retained, v2.2.3HOLD, no push/merge. Continue caller trace under single-writer packet, not broad catalogue or load. + +### BOOK exact-event trace reactivation - 2026-10-08 + +IN_PROGRESS. Owner explicitly approved recreate ONLY market_data_service_p184w +with same image5849d716, unchanged identity/binding/Redis/cursors/mounts, trace +expiry <=300s and4096records/process. Same-image old-config rollback prepared +before action; never two writers. Guide p184-post-disk-recovery.md BOOK caller +handoff. NoTSsourceedit, noDLrolechanges or freshness relaxation. Read-only +assigned Kafka observer filters BTC/ETH BOOK, no group join/offset commits; join +real event hashes/offsets against writer trace. Redis ACK is not SDK ACK/cursor. +Stop/rollback on config drift, crash/OOM or writer unable to recover; preserve +all failed observations. Evidence private outsideGit; no full env/keys in logs. +Record runtime diff, window coverage, cleanup and unresolved stages after run. + +Trace RESULT: MEASURED_CONSUMER_BACKLOG_OPEN. Sameimage writer recreate completed, +onlytraceexpiry changed; expired naturally after<=300s. NoDLrolechange, noTSedit. +Canonical4096records/150.136s;4writerexports16384attempts.3425exactmatches with +zeroidentity/generation/clockfailures:2885applied,523rejected,17other. Export +omissions1+6inflight retained. OKXBTC matched622successes SDK->writerp953882.532ms, +writer->ACK301.008ms,SDK->ACK3956.305ms. Significant postSDK/prewriterwait proven; +which preparation/scheduling/prior-write cost dominates still needs batchtrace. +Cursor acknowledged14961->35617/errors0;aggregate notsameeventACK/cursor proof. +Heartbeat13/13READY60 but executionready50; freshcache BOOKstill~11.6-14.6sage. +No completeconsumer/releasecertification. New boundedoffline exactjoin analyzer +15unittestPASS;invalididentity/clock/generation/ambiguity failclosed. Guide and +Claudehandoff: upgrade/evidence/p184-book-ack-trace-20261008.md. Source architecture +unchanged; no concurrent TSedits. 8boundedwriter timeouts retained, noclaim all +feeds continuous. Unrelated trusting_ptolemy container appeared concurrently; +wholehostequalityassertionfailed, scopedwriterconfigverifiedunchangedexceptexpiry. +Observer initial330s/10000 rejected by tool, reduced to300s/4096beforeexecution. +Allmeasurementprocessesended; noimagesbuilt orcleanupneeded, BuildKit0Breclaimable. +Activeplusnamedrollback retained; free98887442432bytes. Canonicalmain5bf5cd5, +releasev2.2.2/runtimeDLunchanged,featurefix/p184-dl-closure;v2.2.3HOLD,no push/merge. diff --git a/scripts/p184_book_trace_join.py b/scripts/p184_book_trace_join.py new file mode 100644 index 00000000..9b7c8949 --- /dev/null +++ b/scripts/p184_book_trace_join.py @@ -0,0 +1,253 @@ +"""Bounded, offline BOOK exact-event diagnostics; never delivery certification.""" +import argparse +from collections import Counter, defaultdict +import hashlib +import json +import math +import os +import re + +MAX_FILES = 16 +MAX_BYTES = 32 * 1024 * 1024 +MAX_LINE_BYTES = 64 * 1024 +MAX_ROWS = 20000 +MAX_TRACE_ROWS = 4096 +BOOK = {"BOOK_SNAPSHOT", "BOOK_DELTA"} +BOUNDARY = "DURABLE_READ_COMMITTED_OBSERVATION_NOT_COMMIT_CLOCK" +CLOCKS = {"source_event_time_ns": "source_event_ns", + "envelope_received_at_ns": "received_at_ns", + "envelope_normalized_at_ns": "normalized_at_ns", + "envelope_published_at_ns": "published_at_ns"} + + +def uint(value, positive=False): + if type(value) is not int or not (int(positive) <= value < 2**128): + raise ValueError("invalid unsigned integer") + return value + + +def digest(value): + if type(value) is not str or not re.fullmatch(r"[0-9a-f]{64}", value): + raise ValueError("invalid SHA256") + return value + + +def bounded(items, limit): + for index, item in enumerate(items): + if index >= limit: + raise ValueError("row/file limit exceeded") + if type(item) is not dict: + raise ValueError("expected object") + yield item + + +def canonical_key(row): + source = row["identity"]["source_id"] + if type(source) is not str or not 1 <= len(source) <= 256: + raise ValueError("invalid source ID") + return (digest(row["event_id_sha256"]), uint(row["offset"]), + row["feed"].upper(), hashlib.sha256(source.encode()).hexdigest()) + + +def trace_key(row): + if row.get("sdk_receive_basis") != "SDK_STREAM_RETURN": + raise ValueError("not a stream receive") + if row.get("event_id_hash_basis") != "SHA256_PROTOBUF_EVENT_ID_BYTES": + raise ValueError("wrong event hash basis") + return (digest(row["event_id_sha256"]), uint(row["watermark_offset"]), + row["feed"], digest(row["source_id_sha256"])) + + +def validate(canonical, trace): + identity = canonical["identity"] + market = "USD_M" if (identity["venue"], identity["market"]) == ("BINANCE", "USDM") else identity["market"] + expected = ":".join((identity["venue"], market, identity["native_symbol"])) + if trace["canonical_instrument_id"] != expected: + raise ValueError("instrument_identity") + generation = uint(identity["connection_generation"]) + if uint(trace["source_generation"]) != generation: + raise ValueError("generation") + sequence = re.fullmatch(r"g([0-9]{1,39}):([0-9]{1,39})", identity["source_sequence"]) + if (not sequence or int(sequence[1]) != generation + or uint(trace["native_sequence"]) != int(sequence[2])): + raise ValueError("native_sequence") + for target, source in CLOCKS.items(): + if uint(trace[target], True) != uint(canonical[source], True): + raise ValueError(target) + if canonical.get("boundary") != BOUNDARY: + raise ValueError("observer_boundary") + uint(canonical["partition"]) + if not isinstance(canonical["topic"], str) or not canonical["topic"]: + raise ValueError("topic") + uint(canonical["observer_receipt_wall_ns"], True) + sdk = uint(trace["sdk_receive_monotonic_ns"], True) + writer = uint(trace["writer_started_monotonic_ns"], True) + uint(trace["sdk_receive_wall_ns"], True) + if writer < sdk: + raise ValueError("monotonic_order") + + +def metrics(canonical, trace): + sdk = trace["sdk_receive_monotonic_ns"] + writer = trace["writer_started_monotonic_ns"] + reply = uint(trace["redis_reply_monotonic_ns"], True) + wall = uint(trace["redis_reply_wall_ns"], True) + if reply < writer: + raise ValueError("monotonic_order") + if wall < trace["source_event_time_ns"]: + raise ValueError("negative_event_to_reply") + return {"sdk_to_writer_ms": (writer - sdk) / 1e6, + "writer_to_redis_reply_ms": (reply - writer) / 1e6, + "sdk_to_redis_reply_ms": (reply - sdk) / 1e6, + "event_to_redis_reply_ms": (wall - trace["source_event_time_ns"]) / 1e6, + "observer_to_sdk_signed_ms": (trace["sdk_receive_wall_ns"] - + canonical["observer_receipt_wall_ns"]) / 1e6} + + +def stats(values): + values = sorted(values) + return {"n": len(values), "negative": sum(v < 0 for v in values), + "min": values[0], "max": values[-1], + **{name: (values[max(0, math.ceil(len(values) * q) - 1)] + if len(values) >= minimum else None) + for name, q, minimum in (("p50", .5, 1), ("p95", .95, 20), ("p99", .99, 100))}} + + +def summarize(canonical_rows, trace_reports): + """Return JSON-safe sampled diagnostics without I/O or input mutation. + + Invalid report structure and resource bounds raise ValueError. Invalid event + rows are counted and excluded. Duplicate canonical keys are refused, including + identical rows: trace records cannot disambiguate topic/partition provenance. + """ + counts = Counter({key: 0 for key in ( + "canonical_observations", "invalid_canonical_rows", "non_book_canonical_rows", + "trace_records", "non_book_trace_records", "no_exact_join_key", + "unmatched_trace_records", "ambiguous_trace_records", "validation_failures", + "exact_validated_matches", "matched_EVAL1_APPLIED", "rejected_records", + "other_non_success_records", "dropped_at_export", + "in_flight_at_export_not_recorded")}) + index = defaultdict(list) + for row in bounded(canonical_rows, MAX_ROWS): + if row.get("kind") != "canonical_observation": + continue + counts["canonical_observations"] += 1 + try: + key = canonical_key(row) + if key[2] not in BOOK: + counts["non_book_canonical_rows"] += 1 + continue + index[key].append(row) + except (KeyError, TypeError, AttributeError, ValueError): + counts["invalid_canonical_rows"] += 1 + joined, failures, categories = [], Counter(), Counter() + grouped = defaultdict(lambda: defaultdict(list)) + matched = set() + for report in bounded(trace_reports, MAX_FILES): + rows = report.get("records") + if type(rows) is not list or len(rows) > MAX_TRACE_ROWS: + raise ValueError("invalid trace records") + if (uint(report["recorded"]) != len(rows) + or not len(rows) <= uint(report["max_records"]) <= MAX_TRACE_ROWS + or report["clock_basis"] != {"wall": "UNIX_TIME_NS", "monotonic": "PROCESS_MONOTONIC_NS"}): + raise ValueError("invalid trace report metadata") + for field in ("dropped_at_export", "in_flight_at_export_not_recorded"): + counts[field] += uint(report[field]) + for trace in bounded(rows, MAX_TRACE_ROWS): + counts["trace_records"] += 1 + if trace.get("feed") not in BOOK: + counts["non_book_trace_records"] += 1 + continue + success = (type(trace.get("redis_eval_reply")) is int + and trace["redis_eval_reply"] == 1 + and trace.get("cas_outcome") == "APPLIED" + and trace.get("outcome") == "APPLIED" and trace.get("failure") == "NONE") + rejected = trace.get("cas_outcome") == "REJECTED" or trace.get("outcome") == "REJECTED" + category = "EVAL1_APPLIED" if success else "REJECTED" if rejected else "OTHER_NON_SUCCESS" + categories[category] += 1 + if not success: + counts["rejected_records" if rejected else "other_non_success_records"] += 1 + try: + key = trace_key(trace) + except (KeyError, TypeError, ValueError): + counts["no_exact_join_key"] += 1 + continue + candidates = index.get(key, []) + if not candidates: + counts["unmatched_trace_records"] += 1 + continue + if len(candidates) != 1: + counts["ambiguous_trace_records"] += 1 + continue + canonical = candidates[0] + try: + validate(canonical, trace) + measurements = metrics(canonical, trace) if success else {} + except (KeyError, TypeError, ValueError) as error: + counts["validation_failures"] += 1 + failures[str(error)[:96]] += 1 + continue + counts["exact_validated_matches"] += 1 + matched.add(key) + if success: + counts["matched_EVAL1_APPLIED"] += 1 + group = trace["canonical_instrument_id"] + " " + trace["feed"] + for name, value in measurements.items(): + grouped[group][name].append(value) + joined.append({"event_id_sha256": key[0], "watermark_offset": key[1], + "feed": key[2], "source_id_sha256": key[3], + "canonical_instrument_id": trace["canonical_instrument_id"], + "topic": canonical["topic"], "partition": canonical["partition"], + "category": category, "metrics_ms": measurements}) + counts["canonical_keys_without_validated_match"] = len(set(index) - matched) + counts["ambiguous_canonical_keys"] = sum(len(rows) > 1 for rows in index.values()) + invalid = any(counts[key] for key in ("validation_failures", "invalid_canonical_rows", + "ambiguous_canonical_keys", "ambiguous_trace_records")) + return {"status": "INVALID_EVIDENCE" if invalid else "PARTIAL_SAME_EVENT_MEASUREMENT", + "counts": dict(counts), + "categories": dict(categories), "validation_failure_reasons": dict(failures), + "groups": {g: {"metrics_ms": {k: stats(v) for k, v in m.items()}} + for g, m in grouped.items()}, "joined_events": joined, + "canonical_commit_ns": "NOT_MEASURED", "sdk_ack": "NOT_MEASURED", + "cursor_persistence": "NOT_MEASURED", "full_delivery": "NOT_ESTABLISHED", + "notes": ["Monotonic SDK-to-writer/Redis reply; wall event-to-Redis reply.", + "Observer-to-SDK is SIGNED durable observation timing, not commit latency.", + "Sampled/capped exports: unmatched rows are not proof of loss.", + "Success metrics cover only exact validated EVAL1 APPLIED failure NONE attempts."]} + + +def read_bounded(path): + with open(path, "rb") as source: + data = source.read(MAX_BYTES + 1) + if len(data) > MAX_BYTES: + raise ValueError("file byte limit exceeded") + return data + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--canonical", required=True) + parser.add_argument("--trace", action="append", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args(argv) + if len(args.trace) > MAX_FILES or len(set(map(os.path.realpath, args.trace))) != len(args.trace): + parser.error("too many or duplicate trace files") + try: + raw = read_bounded(args.canonical) + lines = raw.splitlines() + if len(lines) > MAX_ROWS or any(len(line) > MAX_LINE_BYTES for line in lines): + raise ValueError("canonical line/row limit exceeded") + # Refuse partial trailing JSON while the collector is still writing. + canonical = [json.loads(line) for line in lines] + report = summarize(canonical, (json.loads(read_bounded(p)) for p in args.trace)) + descriptor = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "w") as output: + json.dump(report, output, allow_nan=False, separators=(",", ":")) + output.write("\n") + except (OSError, ValueError, KeyError, TypeError) as error: + parser.exit(2, f"BOOK trace analysis refused: {error}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_p184_book_trace_join.py b/tests/test_p184_book_trace_join.py new file mode 100644 index 00000000..37642aba --- /dev/null +++ b/tests/test_p184_book_trace_join.py @@ -0,0 +1,216 @@ +"""Synthetic test-only BOOK trace fixtures; no runtime or provider calls.""" +import copy +import hashlib +import importlib.util +import json +import math +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location( + "book_trace_join", Path(__file__).resolve().parents[1] / "scripts/p184_book_trace_join.py") +join = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(join) + + +def fixture(): + canonical = {"kind": "canonical_observation", "topic": "md.canonical.v2", + "partition": 3, "offset": 42, "event_id_sha256": "a" * 64, + "feed": "book_delta", "source_event_ns": 1000000, + "received_at_ns": 2000000, "normalized_at_ns": 3000000, + "published_at_ns": 4000000, "observer_receipt_wall_ns": 10000000, + "boundary": join.BOUNDARY, + "identity": {"venue": "BINANCE", "market": "USDM", + "native_symbol": "BTCUSDT", "source_id": "test-only-source", + "connection_generation": 7, "source_sequence": "g7:99"}} + trace = {"canonical_instrument_id": "BINANCE:USD_M:BTCUSDT", + "source_id_sha256": hashlib.sha256(b"test-only-source").hexdigest(), + "feed": "BOOK_DELTA", "watermark_offset": 42, "source_generation": 7, + "native_sequence": 99, "event_id_sha256": "a" * 64, + "event_id_hash_basis": "SHA256_PROTOBUF_EVENT_ID_BYTES", + "sdk_receive_basis": "SDK_STREAM_RETURN", "sdk_receive_wall_ns": 9000000, + "sdk_receive_monotonic_ns": 100, "writer_started_monotonic_ns": 1000100, + "redis_reply_monotonic_ns": 3000100, "redis_reply_wall_ns": 12000000, + "redis_eval_reply": 1, "cas_outcome": "APPLIED", "outcome": "APPLIED", + "failure": "NONE"} + trace.update({target: canonical[source] for target, source in join.CLOCKS.items()}) + report = {"records": [trace], "recorded": 1, "max_records": 4096, + "dropped_at_export": 0, "in_flight_at_export_not_recorded": 0, + "clock_basis": {"wall": "UNIX_TIME_NS", "monotonic": "PROCESS_MONOTONIC_NS"}} + return canonical, trace, report + + +class BookTraceTests(unittest.TestCase): + def test_exact_signed_metrics_and_purity(self): + c, t, r = fixture() + before = copy.deepcopy((c, r)) + result = join.summarize([c], [r]) + self.assertEqual((c, r), before) + self.assertEqual(result["counts"]["matched_EVAL1_APPLIED"], 1) + self.assertEqual(result["joined_events"][0]["metrics_ms"], { + "sdk_to_writer_ms": 1, "sdk_to_redis_reply_ms": 3, + "writer_to_redis_reply_ms": 2, + "event_to_redis_reply_ms": 11, "observer_to_sdk_signed_ms": -1}) + self.assertEqual(result["sdk_ack"], "NOT_MEASURED") + self.assertEqual(result["cursor_persistence"], "NOT_MEASURED") + self.assertEqual(result["full_delivery"], "NOT_ESTABLISHED") + self.assertNotIn("PASS", result["status"]) + + def test_wrong_identity_generation_sequence_and_clocks(self): + changes = {"canonical_instrument_id": "BINANCE:USD_M:ETHUSDT", + "source_generation": 8, "native_sequence": 100, + **{key: 123 for key in join.CLOCKS}, + "sdk_receive_monotonic_ns": 2000100, + "redis_reply_monotonic_ns": 1, "redis_reply_wall_ns": 1} + for field, value in changes.items(): + with self.subTest(field=field): + c, t, r = fixture() + t[field] = value + result = join.summarize([c], [r]) + self.assertEqual(result["counts"]["validation_failures"], 1) + self.assertEqual(result["status"], "INVALID_EVIDENCE") + self.assertFalse(result["groups"]) + + def test_wrong_source_offset_feed_event_never_fuzzy_joins(self): + for field, value in (("source_id_sha256", "b" * 64), ("watermark_offset", 43), + ("event_id_sha256", "b" * 64), ("feed", "BOOK_SNAPSHOT")): + with self.subTest(field=field): + c, t, r = fixture() + t[field] = value + result = join.summarize([c], [r]) + self.assertEqual(result["counts"]["unmatched_trace_records"], 1) + self.assertEqual(result["status"], "PARTIAL_SAME_EVENT_MEASUREMENT") + self.assertFalse(result["joined_events"]) + + def test_ambiguous_partitions_topics_and_duplicates_refused(self): + for field, value in (("partition", 4), ("topic", "other"), ("partition", 3)): + with self.subTest(field=field, value=value): + c, t, r = fixture() + other = copy.deepcopy(c) + other[field] = value + result = join.summarize([c, other], [r]) + self.assertEqual(result["counts"]["ambiguous_trace_records"], 1) + self.assertEqual(result["status"], "INVALID_EVIDENCE") + self.assertFalse(result["joined_events"]) + + def test_only_eval1_applied_none_succeeds(self): + for field, value in (("redis_eval_reply", 0), ("redis_eval_reply", True), + ("failure", "TIMEOUT"), ("outcome", "DUPLICATE"), + ("cas_outcome", "REJECTED")): + with self.subTest(field=field): + c, t, r = fixture() + t[field] = value + result = join.summarize([c], [r]) + self.assertEqual(result["counts"]["matched_EVAL1_APPLIED"], 0) + self.assertEqual(result["counts"]["exact_validated_matches"], 1) + self.assertFalse(result["groups"]) + + def test_reject_unmatched_and_export_omissions_separate(self): + c, t, r = fixture() + t.update(cas_outcome="REJECTED", outcome="REJECTED", redis_eval_reply=0, + watermark_offset=43) + r.update(dropped_at_export=2, in_flight_at_export_not_recorded=3) + counts = join.summarize([c], [r])["counts"] + for key, value in (("rejected_records", 1), ("unmatched_trace_records", 1), + ("dropped_at_export", 2), ("in_flight_at_export_not_recorded", 3)): + self.assertEqual(counts[key], value) + + def test_missing_hash_or_wrong_basis(self): + for field, value in (("event_id_sha256", "NOT_MEASURED"), + ("event_id_hash_basis", "WRONG"), + ("sdk_receive_basis", "SDK_QUERY_RETURN"), + ("watermark_offset", True)): + c, t, r = fixture() + t[field] = value + self.assertEqual(join.summarize([c], [r])["counts"]["no_exact_join_key"], 1) + + def test_observer_and_native_generation_validation(self): + for field, value in (("boundary", "COMMIT"), ("observer_receipt_wall_ns", 0)): + c, t, r = fixture() + c[field] = value + self.assertEqual(join.summarize([c], [r])["counts"]["validation_failures"], 1) + c, t, r = fixture() + c["identity"]["source_sequence"] = "g8:99" + self.assertEqual(join.summarize([c], [r])["counts"]["validation_failures"], 1) + + def test_empty_metadata_and_non_book(self): + result = join.summarize([{"kind": "scope"}], []) + self.assertFalse(result["groups"]) + c, t, r = fixture() + t["feed"] = "MARK_INDEX_PRICE" + self.assertEqual(join.summarize([c], [r])["counts"]["non_book_trace_records"], 1) + + def test_percentile_sample_thresholds(self): + for n in (1, 19, 20, 99, 100): + with self.subTest(n=n): + result = join.stats(list(range(1, n + 1))) + self.assertEqual(result["p95"], None if n < 20 else math.ceil(n * .95)) + self.assertEqual(result["p99"], None if n < 100 else 99) + + def test_invalid_canonical_evidence_status(self): + c, t, r = fixture() + c["event_id_sha256"] = "invalid" + result = join.summarize([c], [r]) + self.assertEqual(result["counts"]["invalid_canonical_rows"], 1) + self.assertEqual(result["status"], "INVALID_EVIDENCE") + + def test_writer_reply_percentile_uses_paired_durations(self): + c, t, r = fixture() + r["records"] = [] + for i in range(20): + row = dict(t) + row["writer_started_monotonic_ns"] = 100 + i * 1000000 + row["redis_reply_monotonic_ns"] = 100 + 20000000 + r["records"].append(row) + r["recorded"] = 20 + result = join.summarize([c], [r]) + metrics = next(iter(result["groups"].values()))["metrics_ms"] + self.assertEqual(metrics["writer_to_redis_reply_ms"]["p95"], 19) + self.assertNotEqual(metrics["writer_to_redis_reply_ms"]["p95"], + metrics["sdk_to_redis_reply_ms"]["p95"] - + metrics["sdk_to_writer_ms"]["p95"]) + + def test_pure_bounds_and_report_metadata(self): + c, t, r = fixture() + for name, args in (("MAX_ROWS", ([c, c], [])), ("MAX_FILES", ([], [r, r])), + ("MAX_TRACE_ROWS", ([c], [{**r, "records": [t, t]}]))): + with self.subTest(name=name), patch.object(join, name, 1): + with self.assertRaises(ValueError): + join.summarize(*args) + r["recorded"] = 2 + with self.assertRaises(ValueError): + join.summarize([c], [r]) + + def test_cli_exclusive_output_and_repeat_trace(self): + c, t, r = fixture() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + canonical, trace, trace2, output = (root / name for name in ("c", "t", "t2", "o")) + canonical.write_text(json.dumps(c) + "\n") + trace.write_text(json.dumps(r)) + trace2.write_text(json.dumps({**r, "records": [], "recorded": 0})) + args = ["--canonical", str(canonical), "--trace", str(trace), + "--trace", str(trace2), "--output", str(output)] + self.assertEqual(join.main(args), 0) + original = output.read_bytes() + with self.assertRaises(SystemExit) as error: + join.main(args) + self.assertEqual(error.exception.code, 2) + self.assertEqual(output.read_bytes(), original) + + def test_cli_malformed_partial_and_byte_bounds(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "partial" + path.write_text('{"kind":') + with patch.object(join, "MAX_BYTES", 2), self.assertRaises(ValueError): + join.read_bounded(path) + with self.assertRaises(SystemExit): + join.main(["--canonical", str(path), "--trace", str(path), + "--output", str(Path(directory) / "out")]) + self.assertFalse((Path(directory) / "out").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/upgrade/evidence/p184-book-ack-trace-20261008.md b/upgrade/evidence/p184-book-ack-trace-20261008.md new file mode 100644 index 00000000..9ddc2d18 --- /dev/null +++ b/upgrade/evidence/p184-book-ack-trace-20261008.md @@ -0,0 +1,131 @@ +# BOOK Consumer Exact-Event Trace + +Status: MEASURED_CONSUMER_BACKLOG_OPEN. Not release acceptance. + +## Approved Runtime Action + +Owner approved one recreate of market_data_service_p184w, same immutable image +sha256:5849d7167561273af08bda2103c38cec5f4480f7a8809ea592c57abff7716ea1. +Only TS_DELIVERY_TRACE_UNTIL_UNIX_NS changed: 300s,4096records/process. +Identity, binding r22, SDK2.0.7, mounts, Redis/cursors, resource settings and +network configuration preserved. Same-image prior-config rollback compose is +private in book-trace-20261008/rollback.compose.json. No two-writer interval. +Trace expired naturally; no additional recreate needed to disable it. +Writer running, restart0/OOMfalse. No preexisting non-writer role changed. +A concurrent unrelated trusting_ptolemy container appeared: whole-host equality +assertion therefore failed; it was not created or cleaned by this operation. + +Canonical observer initial330s/10000 request rejected by tool; reduced to +300s/4096, no rejected action executed. Existing read_committed assigned reader, +BTC/ETH BOOK only; no group join, commits, Stream quota or orders. It hit4096 +records at150.136s. Observer wall time is durable visibility, NOT commit time. + +## Matched Evidence + +Four writer files,16384 attempts; exports capped,1 omitted at export and6 +in-flight not recorded. Canonical4096 rows. Exact join on event hash, offset, +feed, source hash, with identity/generation/native sequence/envelope clocks +validated.3425 matched attempts:2885 EVAL1 APPLIED/NONE,523 rejected,17 other. +Zero validation failures/partition ambiguities. Exact counts: +```json +{ + "canonical_observations": 4096, + "invalid_canonical_rows": 0, + "non_book_canonical_rows": 0, + "trace_records": 16384, + "non_book_trace_records": 1181, + "no_exact_join_key": 102, + "unmatched_trace_records": 11676, + "ambiguous_trace_records": 0, + "validation_failures": 0, + "exact_validated_matches": 3425, + "matched_EVAL1_APPLIED": 2885, + "rejected_records": 1784, + "other_non_success_records": 64, + "dropped_at_export": 1, + "in_flight_at_export_not_recorded": 6, + "canonical_keys_without_validated_match": 671, + "ambiguous_canonical_keys": 0 +} +``` + +Unmatched/capped observations are NOT loss. Rejected older updates are NOT +orders rejected: trace reports no EVAL for those older BOOK observations. +Success latencies must not hide their separate denominator. The runtime source +returns False on relation<0 before CAS; no inference that every replay is faulty. + +| Same-event BOOK_DELTA | n | SDK->writer p95 ms | Writer->Redis ACK p95 ms | SDK->Redis ACK p95 ms | Event->Redis ACK p95 ms | +| --- | ---: | ---: | ---: | ---: | ---: | +| BINANCE:USD_M:BTCUSDT BOOK_DELTA | 542 | 7345.375 | 307.010 | 7595.909 | 13490.411 | +| BINANCE:USD_M:ETHUSDT BOOK_DELTA | 587 | 3766.698 | 284.820 | 3890.387 | 5506.640 | +| OKX:SWAP:BTC-USDT-SWAP BOOK_DELTA | 622 | 3882.532 | 301.008 | 3956.305 | 5815.562 | +| OKX:SWAP:ETH-USDT-SWAP BOOK_DELTA | 635 | 3355.554 | 249.270 | 3432.634 | 6966.953 | + +SDK/writer/ACK durations use process monotonic clocks; event age uses wall +clocks. Quantiles computed per stage, not subtracted. No p99 with n<100. +The exact OKX BTC sample still has pre-SDK durable-observer->SDK p95 +3527.101ms; this signed measurement includes observer scheduling and replay. +It cannot isolate Stream delivery from caller read pacing, and is not commit lag. + +## What This Establishes + +A material wait occurs AFTER SDK return and BEFORE writer entry; provider quiet +alone cannot explain that measured interval. Prior writer intervals overlap the +long waits in the same worker. For waits>1s, median overlap fraction1.0 on BTC +OKX(n598) and Binance(n681). This is wall-interval overlap, NOT CPU accounting +or proof which serialization/encoding/I/O stage dominates. Both BTCs share the +same worker, consistent with existing IDR-081; ETH is affected too. + +Fresh120.231s cache probe OBSERVED, all60groups. BOOK_DELTA max age still +OKX BTC11589.312ms(n154), Binance BTC14553.248ms(n138). These time samples +are not the exact trace subset and are NOT a paired improvement benchmark. +Heartbeat13/13 READY60/session60, reported execution-ready50, not60. No claim +that status READY means every feed can be used for an order. +Cursor counters acknowledged14961->35617, commits10603->25041,errors0,pending +8->0,peak29. Cumulative io_seconds153.811->384.214 over the120s window is +aggregated acrossworkers and includes lock/executor waits; NOT CPU or per-event +cursor latency. SDK ACK/cursor correlation remains NOT_MEASURED. + +Bounded Docker log check:8writer TimeoutError, no denied/trace-export failures; +Stream stdout contained no matching overflow, but mounted logs were NOT exhaustively +read. Do not declare overflow fixed from this check. Restart-window/trace sampling +does not replace steady-state acceptance or prove trace has zero overhead. + +## Narrow Handoff To Claude + +1. Keep ownership of TS source; do not rerun E03/E04 or modify DL freshness. +2. Integrate planned venue/family ownership (IDR-081) with an explicit single-writer + packet. It addresses shared BTC worker, not all ETH delay by itself. +3. Profile current project_book_deltas preparation, prior per-item store_advancing + MGET/CAS, scheduler and SDK ACK as one bounded batch. Include batch/event IDs, + ACK admission/completion and cursor persistence correlation. Reuse this trace. +4. Optimize measured batch path while maintaining per-product sequence, generation + fence, CAS adoption and deadline. Prefer reducing repeated serialization/round + trips over unrestricted concurrent writes. Do not advance cursor before all + intended batch effects are applied or safely recognized as replay. +5. Regression: delayed/failed CAS, partial batch failure, crash before/after ACK, + generation/reset, old replay, same/different venue ownership, slow consumer + isolation. Then one affected BOOK/caller comparison; no full catalogue. +6. If pre-SDK delay remains after caller catches up, correlate offered/dequeued + Stream counters with transport and caller pacing before changing DL service. + +This run closes measurement activation, not the throughput defect. No TS source +was edited; no unreviewed caller optimization deployed. + +## Reproduction, Cleanup And Provenance + +Source analyzer scripts/p184_book_trace_join.py;15 unittest PASS,0skip. +Command: python3 -B -m unittest discover -s tests -p test_p184_book_trace_join.py -v. +Use CLI --canonical canonical/observations.jsonl --trace delivery-*.json (repeat +flag perfile) --output new exclusive path. Negative identity/source/clock/ +generation/offset/partition tests included; malformed evidence fails closed. +Evidence root: /home/bobby/.local/state/qdl-v2/book-trace-20261008. +fresh-summary-v2.json SHA256 e9d776f78b8eb587ce4ee1df48acbfe88a71b8bdacbb135900e7fb70b4073cd9 +Original summary retained. Packet/env snapshots0600; no credential output/Git. +All observer processes completed; no new image/build/container/network/volume. +Docker inventory86images,69containers,196cacheentries,0Breclaimable BuildKit. +No cleanup of concurrent artifacts or rollback. Disk free98887442432bytes at end. +Canonical /home/bobby/data_layer main5bf5cd5, publishedv2.2.2; active worktree +/home/bobby/.worktrees/data-layer-p184-closure fix/p184-dl-closure. ExistingDL +core658a9570,ingestor0f6876e1,Query/BAR51450612,projectora696cfba,Stream1f13408f +unchanged. Runtime fullidentity receipts retained; v2.2.3HOLD. No push/merge. From 832788030edb32d908bc6c1a68b5d485d3a2493b Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 17:54:57 +0000 Subject: [PATCH 046/147] Add bounded disk pressure guards and fence legacy recovery on KN --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 55 +++++++++ scripts/kn_disk_pressure.py | 96 +++++++++++++++ scripts/kn_production_watch.py | 44 ++++++- scripts/p184_storage_pressure_probe.py | 74 +++++++++++ scripts/v2_stable_boot_recovery.py | 22 ++++ tests/test_kn_disk_pressure.py | 130 ++++++++++++++++++++ tests/test_kn_production_watch.py | 95 ++++++++++++++ tests/test_v2_stable_boot_recovery.py | 28 +++++ upgrade/evidence/p184-resilience-closure.md | 117 ++++++++++++++++++ 9 files changed, 657 insertions(+), 4 deletions(-) create mode 100644 scripts/kn_disk_pressure.py create mode 100644 scripts/p184_storage_pressure_probe.py create mode 100644 tests/test_kn_disk_pressure.py create mode 100644 upgrade/evidence/p184-resilience-closure.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 6c3d2963..4bc28bfb 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70744,3 +70744,58 @@ Observer initial330s/10000 rejected by tool, reduced to300s/4096beforeexecution. Allmeasurementprocessesended; noimagesbuilt orcleanupneeded, BuildKit0Breclaimable. Activeplusnamedrollback retained; free98887442432bytes. Canonicalmain5bf5cd5, releasev2.2.2/runtimeDLunchanged,featurefix/p184-dl-closure;v2.2.3HOLD,no push/merge. + +### Resilience closure - three approved packages - 2026-10-08 + +Status IN_PROGRESS. Owner approved one scope, no architecture rewrite or broad +catalogue rerun. Governing KN architecture guide and evidence +upgrade/evidence/p184-book-ack-trace-20261008.md remain authoritative. + +R1 Consumer throughput / BOOK recovery: TS source remains Claude-owned. Reuse +exact-event evidence and coordinate bounded ordered batch/ownership change, +snapshot plus sequence/generation boundary resync, ACK only after confirmed +apply. LOSSLESS TRADE must not silently become latest-state. Require slow-reader, +partial apply, crash/ACK, old replay and generation/reset tests. No TS edits here. + +R2 Disk pressure / restart: extend existing read-only watcher with bytes/inodes, +measured growth and explicit predeclared headroom policy; do not infer safety from +process-up. No destructive auto-prune, FLUSH, offset reset or legacy spool recovery +on KN. Check recovery tooling topology before mutation. Fail-closed on missing +pressure/progress evidence, bounded retries and stable recovery before READY. +Production policy derives from measured growth/retention, not guessed constants. + +R3 Fault acceptance / rollout: isolated storage/consumer faults first, fixed +budgets before runs, exact role/image/config packet and rollback before deployment. +Inherited unrelated tests remain inherited. Gate includes no loss/regression, +quorum/ISR, projector progress, cache eligibility and consumer ACK/cursor; cleanup +only owned test artifacts. Do not promote mocked faults to live acceptance. + +Initial slices: watcher disk safety and legacy-recovery topology fence, unit and +isolated filesystem fault tests; no production mutations or release authority +implied until concrete changed-role packet passes. Journal results per slice. + +R2 progress: source legacybootrecovery fence14unittestPASS; actualread-only +assessment refuses KN with MANUAL before spool/cache I/O. Installedsystemdunit +is inactive legacy canonical path, not KN controller. Broker/projector restart +policy unless-stopped confirmed. No installedunit orruntime changed. Detailed +threepackage tracking and currentOPENstates: upgrade/evidence/p184-resilience-closure.md. + +R2 initialsliceRESULT:35unitPASS(14boot/10watcher/11disk),0skip. Real isolated +16MiB/64inode tmpfs exhausted bytes theninodes, pressure detected, ownedfiles +removed, guard cleared andwrite/read succeeded in0.110404s. NoKafkafault +certification inferred. Testcontainer --rm verifiedabsent; noimagebuild. +Watchersource supports explicitpolicy/matchedTScontainer, failclosedunknown and +hysteresis; no productionthresholds invented, noinstalledcontroller activated. +Sourcefence rejects legacybootrecovery onKN (counterfactualRECOVER->MANUAL). +R1 consumerfix, R2native recoverycoordinator and R3 endtoendfault/rollout remain +implementationOPEN, nottechnicaldebt orcomplete. Guide includes evidencehashes: +upgrade/evidence/p184-resilience-closure.md. NoTSworktreeedit;Claudeplan change +preserved. No sharedruntimewrites/restart/flush/offsetreset, no push/merge. +Images86 retainedactive/rollback;concurrentBuildKit219entries/50active observed, +notowned/pruned. Testtmpfs removedautomatically, no sharedvolumes touched. + +Diskpost94548148224freebytes vs95856402432before; concurrenthost/buildusage, +notattributed to16MiBtest(torn down). No reclaimclaim. Canonicalmain5bf5cd5 and +v2.2.2 unchanged;DLcore658a9570/ingestor0f6876e1/QueryBAR51450612/projectora696cfba/ +Stream1f13408f,TSwriter5849d716 unchangedbyslice. Featurefix/p184-dl-closure +retainedactive;v2.2.3HOLD, explicitrollbackimagesretained. diff --git a/scripts/kn_disk_pressure.py b/scripts/kn_disk_pressure.py new file mode 100644 index 00000000..25c5731c --- /dev/null +++ b/scripts/kn_disk_pressure.py @@ -0,0 +1,96 @@ +"""Read-only filesystem observations and explicit, operator-owned disk policy.""" +import math +import os +import time + + +def sample_disk(path): + try: + device = os.stat(path).st_dev + fs = os.statvfs(path) + return {"device": device, "monotonic": time.monotonic(), + "total_bytes": fs.f_blocks * fs.f_frsize, + "available_bytes": fs.f_bavail * fs.f_frsize, + "total_inodes": fs.f_files, "available_inodes": fs.f_favail} + except OSError: + return None + + +class DiskPressure: + """Per-path hysteresis; no lifecycle authority and no production defaults. + + First/missing/non-comparable observations fail closed. Recovery requires both + capacity recovery floors and time-to-reserve above the recovery horizon. + Rates measure net available-capacity depletion, not cumulative write volume; + a null time-to-reserve with zero growth means no observed depletion. + State is process-local and does not certify sustained application recovery. + """ + + def __init__(self, reserve_bytes, recovery_bytes, reserve_inodes, + recovery_inodes, reserve_seconds, recovery_seconds): + values = (reserve_bytes, recovery_bytes, reserve_inodes, + recovery_inodes, reserve_seconds, recovery_seconds) + if any(type(v) not in (int, float) or not math.isfinite(v) or v <= 0 for v in values): + raise ValueError("disk thresholds must be finite and positive") + if any(high <= low for low, high in zip(values[::2], values[1::2])): + raise ValueError("disk recovery thresholds must exceed reserve thresholds") + self.reserve = (reserve_bytes, reserve_inodes) + self.recovery = (recovery_bytes, recovery_inodes) + self.reserve_seconds = reserve_seconds + self.recovery_seconds = recovery_seconds + self.previous = {} + self.pressured = set() + + def observe(self, path, observation): + previous = self.previous.pop(path, None) + result = {"observation": None, "reasons": []} + reasons = result["reasons"] + try: + if not isinstance(observation, dict): + raise ValueError("missing observation") + for key in ("device", "total_bytes", "available_bytes", "total_inodes", "available_inodes"): + if type(observation.get(key)) is not int or observation[key] < 0: + raise ValueError("invalid capacity") + stamp = observation.get("monotonic") + if type(stamp) not in (int, float) or not math.isfinite(stamp) or stamp < 0: + raise ValueError("invalid clock") + for unit in ("bytes", "inodes"): + if not 0 <= observation["available_" + unit] <= observation["total_" + unit] or observation["total_" + unit] == 0: + raise ValueError("invalid capacity") + # Journal only the validated schema, never arbitrary malformed input. + observation = {key: observation[key] for key in ( + "device", "monotonic", "total_bytes", "available_bytes", + "total_inodes", "available_inodes")} + result["observation"] = observation + self.previous[path] = dict(observation) + for unit, reserve in zip(("bytes", "inodes"), self.reserve): + if observation["available_" + unit] <= reserve: + reasons.append("DISK_" + unit.upper() + "_RESERVE") + if previous is None or any(observation[k] != previous[k] for k in ("device", "total_bytes", "total_inodes")): + raise ValueError("missing comparable baseline") + elapsed = stamp - previous["monotonic"] + if elapsed <= 0 or not math.isfinite(elapsed): + raise ValueError("non-increasing clock") + recovering = path in self.pressured + horizon = self.recovery_seconds if recovering else self.reserve_seconds + metrics = {} + for unit, reserve, recovery in zip(("bytes", "inodes"), self.reserve, self.recovery): + available = observation["available_" + unit] + growth = max(0.0, (previous["available_" + unit] - available) / elapsed) + seconds = max(0, available - reserve) / growth if growth else None + if not math.isfinite(growth) or (seconds is not None and not math.isfinite(seconds)): + raise ValueError("invalid rate") + metrics[unit] = {"growth_per_second": growth, "seconds_to_reserve": seconds} + if seconds is not None and seconds <= horizon: + reasons.append("DISK_" + unit.upper() + "_TIME_TO_RESERVE") + if recovering and available < recovery: + reasons.append("DISK_" + unit.upper() + "_RECOVERY_PENDING") + result["metrics"] = metrics + except (ValueError, TypeError, OverflowError): + reasons.append("DISK_OBSERVATION_UNKNOWN") + if reasons: + self.pressured.add(path) + else: + self.pressured.discard(path) + result["pressure"] = bool(reasons) + return result diff --git a/scripts/kn_production_watch.py b/scripts/kn_production_watch.py index 4b112217..92067147 100644 --- a/scripts/kn_production_watch.py +++ b/scripts/kn_production_watch.py @@ -7,6 +7,11 @@ import subprocess import time +if __package__: + from .kn_disk_pressure import DiskPressure, sample_disk +else: + from kn_disk_pressure import DiskPressure, sample_disk + PROJECT = "qdl_v2_stable_candidate" NATIVE = ("market_cache", "market_projector_1", "market_projector_2", "query_kn_1", "query_kn_2", "stream_kn_1", "stream_kn_2") @@ -18,9 +23,9 @@ def read_values(path): return {} -def sample(): +def sample(ts_container="market_data_service"): ids = subprocess.check_output(["docker", "ps", "-q", "--filter", f"label=com.docker.compose.project={PROJECT}"], text=True).split() - extra = subprocess.run(["docker", "inspect", "market_data_service"], capture_output=True, text=True) + extra = subprocess.run(["docker", "inspect", ts_container], capture_output=True, text=True) containers = json.loads(subprocess.check_output(["docker", "inspect", *ids])) if ids else [] if extra.returncode == 0: containers.extend(json.loads(extra.stdout)) @@ -80,15 +85,41 @@ def main(): parser.add_argument("--stop-file", type=Path, required=True) parser.add_argument("--stop-container-id", action="append", default=[], help="Explicitly authorized full container ID; default is read-only") + parser.add_argument("--ts-container", default="market_data_service", + help="Same TS writer target for inspect and logs") + parser.add_argument("--disk-path", action="append", default=[], + help="Writer filesystem path; enables disk policy (repeatable)") + for name in ("reserve-bytes", "recovery-bytes", "reserve-inodes", "recovery-inodes", + "reserve-seconds", "recovery-seconds"): + parser.add_argument("--disk-" + name, type=float if name.endswith("seconds") else int, + help="Explicit operator threshold; recovery must exceed reserve. " + "Seconds are measured time to the capacity reserve.") args = parser.parse_args() + policy_values = [getattr(args, "disk_" + name) for name in + ("reserve_bytes", "recovery_bytes", "reserve_inodes", "recovery_inodes", + "reserve_seconds", "recovery_seconds")] + disk_policy = None + if args.disk_path or any(v is not None for v in policy_values): + if not args.disk_path or any(v is None for v in policy_values): + parser.error("disk paths and all six disk policy thresholds are required together") + if len(set(args.disk_path)) != len(args.disk_path): + parser.error("disk paths must be distinct") + try: + disk_policy = DiskPressure(*policy_values) + except (ValueError, OverflowError) as exc: + parser.error(str(exc)) end = time.monotonic() + args.seconds previous_cpu = cpu_ticks() low_idle_seconds = 0 tick = 0 ts_count = 0 with args.out.open("x") as handle: + if disk_policy: + # Prime measurement only; no readiness is asserted by a baseline. + for path in args.disk_path: + disk_policy.observe(path, sample_disk(path)) while time.monotonic() < end and not args.stop_file.exists(): - row = sample() + row = sample(args.ts_container) stop_ids = explicit_stop_ids(row, args.stop_container_id) now_cpu = cpu_ticks() total_delta = now_cpu[0] - previous_cpu[0] @@ -96,7 +127,7 @@ def main(): previous_cpu = now_cpu low_idle_seconds = low_idle_seconds + 5 if idle < 5 else 0 if tick % 6 == 0: - logs = subprocess.run(["docker", "logs", "--since", "60s", "--tail", "1000", "market_data_service"], + logs = subprocess.run(["docker", "logs", "--since", "60s", "--tail", "1000", args.ts_container], capture_output=True, text=True, timeout=10) ts_count = sum("MARK_INDEX" in line and "SOURCE_UNAVAILABLE" in line for line in (logs.stdout + logs.stderr).splitlines()) @@ -104,6 +135,11 @@ def main(): row["ts_mark_index_unavailable_60s"] = ts_count row["host_idle_percent"] = idle row["stop_reasons"] = stop_reasons(row, low_idle_seconds) + if disk_policy: + row["disk_pressure"] = {path: disk_policy.observe(path, sample_disk(path)) + for path in args.disk_path} + row["stop_reasons"].extend(reason + ":" + path + for path, result in row["disk_pressure"].items() for reason in result["reasons"]) danger = bool(row["stop_reasons"]) row["guard_stop"] = danger handle.write(json.dumps(row, separators=(",", ":")) + "\n") diff --git a/scripts/p184_storage_pressure_probe.py b/scripts/p184_storage_pressure_probe.py new file mode 100644 index 00000000..50b7c50b --- /dev/null +++ b/scripts/p184_storage_pressure_probe.py @@ -0,0 +1,74 @@ +"""Test-only ENOSPC probe: dedicated tiny /fault tmpfs, never host storage.""" +import errno +import json +import os +from pathlib import Path +import time +from scripts.kn_disk_pressure import DiskPressure, sample_disk + + +def main(): + root = Path("/fault") + mounts = Path("/proc/self/mountinfo").read_text().splitlines() + if os.environ.get("P184_ISOLATED_STORAGE_FAULT") != "1" or not any( + line.split()[4] == str(root) and line.split(" - ")[1].split()[0] == "tmpfs" + for line in mounts + ): + raise RuntimeError("DEDICATED_TEST_TMPFS_REQUIRED") + fs = os.statvfs(root) + if not 0 < fs.f_blocks * fs.f_frsize <= 32 * 1024**2 or not 0 < fs.f_files <= 128 or list(root.iterdir()): + raise RuntimeError("EMPTY_BOUNDED_TEST_TMPFS_REQUIRED") + # Test-only capacities; not deployment policy or Kafka recovery budgets. + policy = DiskPressure(1024**2, 2 * 1024**2, 4, 8, 1, 2) + observations = [] + started = time.monotonic() + def observe(label): + row = policy.observe(str(root), sample_disk(root)) + observations.append({"stage": label, **row}) + return row + def check_deadline(): + if time.monotonic() - started > 10: + raise TimeoutError("ISOLATED_FAULT_BUDGET") + try: + observe("baseline") + time.sleep(.01) + assert not observe("initial_ready")["pressure"] + with (root / "bytes").open("wb", buffering=0) as output: + try: + while True: + check_deadline() + output.write(b"x" * 65536) + except OSError as error: + if error.errno != errno.ENOSPC: + raise + row = observe("bytes_exhausted") + assert "DISK_BYTES_RESERVE" in row["reasons"] + (root / "bytes").unlink() + time.sleep(.01) + assert not observe("bytes_recovered")["pressure"] + try: + for i in range(256): + check_deadline() + (root / ("inode-" + str(i))).touch(exist_ok=False) + raise AssertionError("inode fault not reached") + except OSError as error: + if error.errno != errno.ENOSPC: + raise + row = observe("inodes_exhausted") + assert "DISK_INODES_RESERVE" in row["reasons"] + for path in root.glob("inode-*"): + path.unlink() + time.sleep(.01) + assert not observe("inodes_recovered")["pressure"] + (root / "verification").write_bytes(b"test-only-after-recovery") + assert (root / "verification").read_bytes() == b"test-only-after-recovery" + print(json.dumps({"status": "ISOLATED_FILESYSTEM_PASS", "seconds": time.monotonic() - started, + "provider_data": False, "kafka_recovery": "NOT_TESTED", "observations": observations})) + finally: + for path in root.iterdir(): + if path.name in {"bytes", "verification"} or path.name.startswith("inode-"): + path.unlink() + + +if __name__ == "__main__": + main() diff --git a/scripts/v2_stable_boot_recovery.py b/scripts/v2_stable_boot_recovery.py index 2bf2d572..80e52fd8 100644 --- a/scripts/v2_stable_boot_recovery.py +++ b/scripts/v2_stable_boot_recovery.py @@ -138,7 +138,26 @@ class Assessment: reasons: list[str] = field(default_factory=list) +KN_CACHE_SERVICES = ("market_cache", "market_projector_1", "market_projector_2", + "query_kn_1", "query_kn_2", "stream_kn_1", "stream_kn_2") + + +def native_topology_present(run: Runner) -> bool: + # Even stopped KN roles make legacy FLUSH/reset recovery the wrong authority. + return any(inspect(run, service)["status"] != "absent" for service in KN_CACHE_SERVICES) + + +def require_legacy_topology(run: Runner) -> None: + if native_topology_present(run): + raise RuntimeError("KN_TOPOLOGY_REQUIRES_NATIVE_RECOVERY_NO_FLUSH_OR_OFFSET_RESET") + if any(inspect(run, service)["status"] == "absent" for service in STOP_SERVICES): + raise RuntimeError("LEGACY_RECOVERY_TOPOLOGY_UNVERIFIED") + + def assess(run: Runner) -> Assessment: + if native_topology_present(run): + return Assessment("MANUAL", {}, {}, None, None, + ["KN_TOPOLOGY_REQUIRES_NATIVE_RECOVERY_NO_FLUSH_OR_OFFSET_RESET"]) infra = {s: inspect(run, s) for s in INFRA_SERVICES} users = {s: inspect(run, s) for s in STOP_SERVICES} not_ready = [s for s, st in infra.items() if st["health"] != "healthy"] @@ -183,6 +202,7 @@ def wait_healthy(self, services: tuple[str, ...], deadline: float) -> None: def simulate_crash(self) -> None: """Reproduce the post-boot state on a running stack: cache users down, Redis tmpfs wiped.""" + require_legacy_topology(self.run) _out(self.run, ["docker", "stop", "-t", "30", *[container(s) for s in STOP_SERVICES]]) self.step("simulate_crash_stopped_cache_users", services=list(STOP_SERVICES)) _out(self.run, ["docker", "restart", container(REDIS_SERVICE)]) @@ -190,6 +210,7 @@ def simulate_crash(self) -> None: self.step("simulate_crash_redis_restarted", redis_keys=redis_keys(self.run)) def recover(self, *, timeout_seconds: float) -> dict[str, Any]: + require_legacy_topology(self.run) deadline = self.clock() + timeout_seconds running = [s for s in STOP_SERVICES if inspect(self.run, s)["status"] == "running"] if running: @@ -244,6 +265,7 @@ def recover(self, *, timeout_seconds: float) -> dict[str, Any]: return {"status": "PASS", "redis_keys": keys, "lag": {"total": last[0], "partitions": last[1], "max_partition": last[2]}} def start_only(self, *, timeout_seconds: float) -> dict[str, Any]: + require_legacy_topology(self.run) deadline = self.clock() + timeout_seconds for group in (STREAM_SERVICES, PROJECTOR_SERVICES, BAR_EDGE_SERVICES, QUERY_SERVICES): _out(self.run, ["docker", "start", *[container(s) for s in group]]) diff --git a/tests/test_kn_disk_pressure.py b/tests/test_kn_disk_pressure.py new file mode 100644 index 00000000..95de8808 --- /dev/null +++ b/tests/test_kn_disk_pressure.py @@ -0,0 +1,130 @@ +"""Synthetic policy regressions; no Docker or production filesystem mutation.""" +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from scripts.kn_disk_pressure import DiskPressure, sample_disk + + +def observation(stamp=0, size=800, inodes=800, **updates): + row = dict(device=1, monotonic=stamp, total_bytes=1000, + available_bytes=size, total_inodes=1000, available_inodes=inodes) + row.update(updates) + return row + + +class DiskPressureTests(unittest.TestCase): + def policy(self): + return DiskPressure(100, 200, 50, 100, 10, 20) + + def test_statvfs_uses_unprivileged_capacity(self): + fs = SimpleNamespace(f_blocks=100, f_frsize=4096, f_bavail=20, + f_bfree=30, f_files=1000, f_favail=50, f_ffree=60) + with patch("scripts.kn_disk_pressure.os.stat", return_value=SimpleNamespace(st_dev=2)), \ + patch("scripts.kn_disk_pressure.os.statvfs", return_value=fs), \ + patch("scripts.kn_disk_pressure.time.monotonic", return_value=42): + self.assertEqual(sample_disk("/test"), dict(device=2, monotonic=42, + total_bytes=409600, available_bytes=81920, total_inodes=1000, available_inodes=50)) + + def test_missing_path_and_real_temporary_filesystem(self): + with tempfile.TemporaryDirectory() as folder: + self.assertIsNone(sample_disk(str(Path(folder) / "absent"))) + row = sample_disk(folder) + self.assertGreater(row["total_bytes"], 0) + self.assertGreaterEqual(row["available_inodes"], 0) + + def test_baseline_unknown_then_stable_healthy(self): + p = self.policy() + self.assertEqual(p.observe("/a", observation())["reasons"], ["DISK_OBSERVATION_UNKNOWN"]) + result = p.observe("/a", observation(5)) + self.assertFalse(result["pressure"]) + self.assertEqual(result["metrics"]["bytes"], {"growth_per_second": 0, "seconds_to_reserve": None}) + + def test_byte_and_inode_exhaustion_inclusive(self): + for unit, value, code in (("available_bytes", 100, "DISK_BYTES_RESERVE"), + ("available_bytes", 0, "DISK_BYTES_RESERVE"), + ("available_inodes", 50, "DISK_INODES_RESERVE"), + ("available_inodes", 0, "DISK_INODES_RESERVE")): + with self.subTest(unit=unit, value=value): + p = self.policy() + p.observe("/a", observation()) + self.assertIn(code, p.observe("/a", observation(5, **{unit: value}))["reasons"]) + + def test_growth_measured_from_monotonic_not_wall_time(self): + p = self.policy() + p.observe("/a", observation()) + p.observe("/a", observation(5)) + result = p.observe("/a", observation(10, size=500, inodes=500)) + self.assertEqual(result["metrics"]["bytes"]["growth_per_second"], 60) + self.assertAlmostEqual(result["metrics"]["bytes"]["seconds_to_reserve"], 400 / 60) + self.assertIn("DISK_BYTES_TIME_TO_RESERVE", result["reasons"]) + self.assertIn("DISK_INODES_TIME_TO_RESERVE", result["reasons"]) + + def test_missing_malformed_and_nonfinite_fail_closed(self): + bad = [None, {}, [], observation(5, available_bytes=-1), + observation(5, available_bytes=1001), observation(5, available_bytes=True), + observation(5, total_bytes=0), observation(5, total_inodes=0), + observation(float("nan")), observation(float("inf")), + observation(5, available_inodes="100"), observation(5, device=None)] + for value in bad: + with self.subTest(value=value): + p = self.policy() + p.observe("/a", observation()) + self.assertIn("DISK_OBSERVATION_UNKNOWN", p.observe("/a", value)["reasons"]) + self.assertIn("DISK_OBSERVATION_UNKNOWN", p.observe("/a", observation(10))["reasons"]) + + def test_observation_journal_is_strict_json_and_schema_bounded(self): + import json + bad = [observation(float("nan")), observation(float("inf")), + observation(5, available_bytes=float("nan")), + {"unexpected": object()}, None] + for value in bad: + with self.subTest(value=value): + result = self.policy().observe("/a", value) + self.assertIsNone(result["observation"]) + self.assertIn("DISK_OBSERVATION_UNKNOWN", result["reasons"]) + json.dumps(result, allow_nan=False) + row = observation(extra=float("nan"), payload=object()) + result = self.policy().observe("/a", row) + self.assertEqual(result["observation"], observation()) + json.dumps(result, allow_nan=False) + + def test_clock_and_filesystem_changes_fail_closed(self): + for value in (observation(0), observation(-1), observation(5, device=2), + observation(5, total_bytes=2000), observation(5, total_inodes=2000)): + with self.subTest(value=value): + p = self.policy() + p.observe("/a", observation()) + self.assertIn("DISK_OBSERVATION_UNKNOWN", p.observe("/a", value)["reasons"]) + + def test_capacity_recovery_hysteresis_and_independent_paths(self): + p = self.policy() + p.observe("/a", observation()) + p.observe("/b", observation()) + p.observe("/a", observation(5, size=90, inodes=40)) + result = p.observe("/a", observation(10, size=150, inodes=75)) + self.assertIn("DISK_BYTES_RECOVERY_PENDING", result["reasons"]) + self.assertIn("DISK_INODES_RECOVERY_PENDING", result["reasons"]) + self.assertFalse(p.observe("/b", observation(5))["pressure"]) + self.assertFalse(p.observe("/a", observation(15, size=200, inodes=100))["pressure"]) + + def test_time_hysteresis_requires_recovery_horizon(self): + p = self.policy() + p.observe("/a", observation(0, size=100)) + result = p.observe("/a", observation(5, size=700)) + self.assertFalse(result["pressure"]) + self.assertTrue(p.observe("/a", observation(10, size=400))["pressure"]) + result = p.observe("/a", observation(15, size=325)) + self.assertEqual(result["metrics"]["bytes"]["seconds_to_reserve"], 15) + self.assertIn("DISK_BYTES_TIME_TO_RESERVE", result["reasons"]) + self.assertFalse(p.observe("/a", observation(20, size=325))["pressure"]) + + def test_invalid_policies_rejected(self): + for values in ((0, 200, 50, 100, 10, 20), (100, 100, 50, 100, 10, 20), + (100, 200, 50, 50, 10, 20), (100, 200, 50, 100, 20, 10), + (100, 200, 50, 100, float("nan"), 20), + (100, 200, 50, 100, 10, float("inf"))): + with self.subTest(values=values), self.assertRaises(ValueError): + DiskPressure(*values) diff --git a/tests/test_kn_production_watch.py b/tests/test_kn_production_watch.py index 02fa5f4e..a83c1eb6 100644 --- a/tests/test_kn_production_watch.py +++ b/tests/test_kn_production_watch.py @@ -53,3 +53,98 @@ def test_guard_is_read_only_unless_exact_ids_supplied(self): self.assertEqual(main(), 3) stops = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "stop"]] self.assertEqual(stops, [["docker", "stop", "-t", "15", "a" * 64]] if approved else []) + + +class DiskWatchIntegrationTests(unittest.TestCase): + def test_disk_alert_unknown_and_exhaustion_never_authorize_stop(self): + import json + import tempfile + from pathlib import Path + from unittest.mock import Mock, patch + from scripts import kn_production_watch as watch + from tests.test_kn_disk_pressure import observation + + for current in (None, observation(5, size=0), observation(5, inodes=0), + observation(5, size=500)): + with self.subTest(current=current), tempfile.TemporaryDirectory() as folder: + out = Path(folder) / "result.jsonl" + argv = ["watch", "--out", str(out), "--seconds", "1", + "--stop-file", str(Path(folder) / "stop"), + "--ts-container", "explicit-writer", "--disk-path", "/test", + "--disk-reserve-bytes", "100", "--disk-recovery-bytes", "200", + "--disk-reserve-inodes", "50", "--disk-recovery-inodes", "100", + "--disk-reserve-seconds", "10", "--disk-recovery-seconds", "20"] + row = {"available_memory": 8 * 1024**3, "containers": { + PROJECT + "-market_projector_1-1": {"id": "a" * 64}}} + with patch("sys.argv", argv), patch.object(watch, "sample", return_value=row) as sample, \ + patch.object(watch, "sample_disk", side_effect=[observation(), current]), \ + patch.object(watch, "cpu_ticks", return_value=(100, 50)), \ + patch.object(watch.subprocess, "run", return_value=Mock(stdout="", stderr="")) as run: + self.assertEqual(watch.main(), 3) + sample.assert_called_once_with("explicit-writer") + self.assertEqual([c.args[0] for c in run.call_args_list], + [["docker", "logs", "--since", "60s", "--tail", "1000", "explicit-writer"]]) + result = json.loads(out.read_text()) + self.assertTrue(result["guard_stop"]) + self.assertTrue(result["disk_pressure"]["/test"]["pressure"]) + self.assertTrue(all(r.startswith("DISK_") for r in result["stop_reasons"])) + + def test_cli_incomplete_invalid_policy_rejected_before_sampling(self): + import contextlib + import io + from unittest.mock import patch + from scripts import kn_production_watch as watch + complete = ["--disk-path", "/test", "--disk-reserve-bytes", "100", + "--disk-recovery-bytes", "200", "--disk-reserve-inodes", "50", + "--disk-recovery-inodes", "100", "--disk-reserve-seconds", "10", + "--disk-recovery-seconds", "20"] + cases = [["--disk-path", "/test"], ["--disk-reserve-bytes", "100"], + complete + ["--disk-path", "/test"], + complete + ["--disk-reserve-seconds", "nan"], + complete + ["--disk-recovery-bytes", "100"]] + for flags in cases: + with self.subTest(flags=flags), patch("sys.argv", [ + "watch", "--out", "/unused", "--stop-file", "/unused-stop"] + flags), \ + patch.object(watch, "sample") as sample, \ + patch.object(watch, "sample_disk") as disk, \ + contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as error: + watch.main() + self.assertEqual(error.exception.code, 2) + sample.assert_not_called() + disk.assert_not_called() + + def test_sampler_inspects_selected_ts_container(self): + from unittest.mock import Mock, patch + from scripts import kn_production_watch as watch + for name in ("market_data_service", "explicit-writer"): + with self.subTest(name=name), \ + patch.object(watch.subprocess, "check_output", return_value=""), \ + patch.object(watch.subprocess, "run", return_value=Mock(returncode=1)) as run, \ + patch.object(watch.Path, "read_text", return_value="MemAvailable: 4096 kB"): + result = watch.sample(name) + self.assertEqual(result["available_memory"], 4096 * 1024) + run.assert_called_once_with(["docker", "inspect", name], capture_output=True, text=True) + + def test_default_healthy_watcher_does_not_sample_disks(self): + import tempfile + from pathlib import Path + from unittest.mock import Mock, patch + from scripts import kn_production_watch as watch + with tempfile.TemporaryDirectory() as folder: + out = Path(folder) / "result.jsonl" + argv = ["watch", "--out", str(out), "--seconds", "1", + "--stop-file", str(Path(folder) / "stop")] + with patch("sys.argv", argv), \ + patch.object(watch, "sample", return_value={ + "available_memory": 8 * 1024**3, "containers": {}}) as sample, \ + patch.object(watch, "sample_disk") as disk, \ + patch.object(watch, "cpu_ticks", return_value=(100, 50)), \ + patch.object(watch.time, "monotonic", side_effect=[0, 0, 2]), \ + patch.object(watch.time, "sleep"), \ + patch.object(watch.subprocess, "run", return_value=Mock(stdout="", stderr="")) as run: + self.assertEqual(watch.main(), 0) + sample.assert_called_once_with("market_data_service") + disk.assert_not_called() + self.assertEqual(len(run.call_args_list), 1) + self.assertNotIn("disk_pressure", out.read_text()) diff --git a/tests/test_v2_stable_boot_recovery.py b/tests/test_v2_stable_boot_recovery.py index 5b93c318..f19ac46a 100644 --- a/tests/test_v2_stable_boot_recovery.py +++ b/tests/test_v2_stable_boot_recovery.py @@ -192,3 +192,31 @@ def test_simulate_crash_refuses_unless_healthy(self): if __name__ == "__main__": unittest.main() + + +class NativeTopologyFenceTests(unittest.TestCase): + def test_kn_recovery_never_reads_or_deletes_legacy_data(self): + for status in ("running", "exited", "restarting"): + st = healthy_states() + st["market_projector_1"] = {"status": status} + fake = FakeDocker(states=st, keys=0, spool=100) + result = boot.assess(fake) + self.assertEqual(result.decision, "MANUAL") + self.assertIn("KN_TOPOLOGY", result.reasons[0]) + self.assertTrue(all(c[:2] == ["docker", "inspect"] for c in fake.commands)) + + def test_direct_mutation_methods_are_fenced(self): + for method in ("simulate_crash", "start_only", "recover"): + st = healthy_states() + st["market_cache"] = {"status": "running"} + fake = FakeDocker(states=st) + recovery = boot.Recovery(fake, receipt={}) + with self.assertRaisesRegex(RuntimeError, "KN_TOPOLOGY"): + getattr(recovery, method)(**({} if method == "simulate_crash" else {"timeout_seconds": 1})) + self.assertTrue(all(c[:2] == ["docker", "inspect"] for c in fake.commands)) + + def test_missing_legacy_topology_is_not_authority_to_start(self): + fake = FakeDocker(states={}) + with self.assertRaisesRegex(RuntimeError, "TOPOLOGY_UNVERIFIED"): + boot.Recovery(fake, receipt={}).start_only(timeout_seconds=1) + self.assertTrue(all(c[:2] == ["docker", "inspect"] for c in fake.commands)) diff --git a/upgrade/evidence/p184-resilience-closure.md b/upgrade/evidence/p184-resilience-closure.md new file mode 100644 index 00000000..11498b2a --- /dev/null +++ b/upgrade/evidence/p184-resilience-closure.md @@ -0,0 +1,117 @@ +# Resilience Closure: Three Approved Packages + +Status: IN_PROGRESS, source and isolated tests first. Owner2026-10-08. +No architecture rewrite, no broad catalogue or order acceptance rerun. +Architecture: DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md. +Prior exact-event evidence: p184-book-ack-trace-20261008.md. + +## R1 Consumer Throughput And Recovery + +Claude owns TS worktree; Codex owns DL/SDK and shared measurement tooling. +No unsynchronized edits to TS. Existing writer5849d716/SDK2.0.7 is distinct from +Claude candidate. Freeze source/image/config before each affected rehearsal. + +Acceptance contract: +- Preserve lossless contract unless explicitly negotiated latest-state feed. +- BOOK resync uses authoritative snapshot + exact boundary and generation. +- No blind deletion of delta cache, offset reset or cursor skip over unapplied data. +- Ordered bounded batch: eliminate measured repeated work, not increase buffers. +- Venue/family ownership addresses shared BTC pressure, not every ETH issue. +- Per-event or batch correlation covers SDK return, prepare, lock/read/CAS, + Redis reply, SDK ACK admission/completion and durable cursor; report missing. +- Prove before/after against same capture/workload, preserve all failed outcomes. +- Negative tests: partial apply/failure, generation changes, replay, crash between + Redis and cursor, slow consumer isolation. Recovery must catch up after burst. + +R1 remains OPEN. Exact measured post-SDK/pre-writer delay is not fixed by disk +watcher changes or a process restart. Link final TS commit/receipt here, not an +unsupported claim of cross-agent coordination or a quota-only capacity claim. + +## R2 Disk Pressure And Safe Recovery + +Existing watcher receives bytes/inodes, projected consumption, explicit operator +policy and stable recovery observations. Missing telemetry fails closed for new +load admission. Thresholds must be derived from measured growth, retention, +maintenance lead time and reserve; test thresholds are not production defaults. +No broad prune, deletion of Kafka logs, DB/Redis state or named rollback images. + +Actual read-only findings: +- Broker1/2/3 and projector1/2 have Docker unless-stopped restart policy. +- Installed qdl-v2-stable-boot-recovery.service is inactive/dead, points at + canonical scripts/v2_stable_boot_recovery.py --apply. It is legacy spool logic, + not KN auto-recovery. NRestarts0 at inspection, not proof no historical fault. +- Legacy recovery can FLUSH and reset offsets. New source fence refuses running, + stopped or restarting KN roles before destructive methods. Does not deploy + itself to canonical/systemd. Mixed/unknown topology is not authority to reset. + +Recovery state sequence to integrate with KN supervisor: +PRESSURE/UNKNOWN -> HOLD_NONESSENTIAL_WORK -> SPACE_RECOVERED -> WAIT_QUORUM_ISR +-> WAIT_PROJECTOR_PROGRESS -> WAIT_CONSUMER_ELIGIBILITY -> SERVING. +Use hysteresis and one bounded deadline; resumed storage does not equal READY. +Transaction fencing/fatal corruption stays BLOCKED/operator-reviewed. Restart +only exact declared role when eligible, with backoff/retry ceiling and receipt. +Do not automatically restart healthy brokers for stale consumer data. + +## R3 Fault And Promotion Gates + +Faults are isolated: no filling host disk or production Kafka volumes. +For each case declare cap, timeout and workload BEFORE running: +1. Byte exhaustion and inode exhaustion; monitor unknown; pressure oscillation. +2. Real storage ENOSPC then space return: observe guard and successful writes; + this alone is not Kafka transaction recovery certification. +3. Broker unavailable/transaction abort/fencing and restart replay from committed + offsets. Reuse existing isolated Kafka recovery/DR08 harness and captures. +4. Consumer overflow/replay/expired cursor -> snapshot boundary -> contiguous + usable stream. Slow client must not starve another client. +5. Partial batch failure/crash before and after ACK: no backward state or skips. + +Exit: source tests + isolated faults + scoped candidate consumer evidence; no +unexplained loss/order mismatch, no false eligibility, bounded recovery under +predeclared deadlines. Only changed roles may rollout, exact immutable digest, +config drift check and named rollback. Preserve DR/broker acceptance gates. +No release until R1/R2/R3 final receipts are linked; no fabricated Full status. + +## Cleanup And Runtime Boundary + +No new images unless source requires them. Reuse retained test images, isolated +--rm test containers, network none for filesystem tests, tiny bounded tmpfs. +No host data mount for fault writes. Remove owned test artifacts immediately; +keep hashed bounded evidence and active/named rollback set. Do not touch concurrent +Claude resources. Journal actual disk and runtime checks after each tested slice. +Canonical /home/bobby/data_layer main5bf5cd5, publishedv2.2.2; +feature /home/bobby/.worktrees/data-layer-p184-closure fix/p184-dl-closure. +v2.2.3 stays HOLD until final gates, not waiting on unrelated Paper features. + +## Tested Initial R2 Slice + +Source changes: kn_production_watch.py/kn_disk_pressure.py expose explicit disk +policy, byte/inode available-to-writer capacity, monotonic consumption rate, +time-to-reserve and capacity/horizon hysteresis. Unknown measurements fail closed. +--ts-container selects the same writer for inspection and log checks, preventing +accidental observation of the stopped legacy market_data_service. No new default +production thresholds or automatic lifecycle authority. Existing exact-ID stop +mechanism unchanged; default remains alert-only. This watcher is NOT yet a native +Kafka recovery coordinator or ongoing production admission controller. + +35unit tests PASS0fail0error0skip across boot recovery14, watcher10, disk11. +Counterfactual mixed KN/legacy fixture: previousassess RECOVER, new MANUAL. +Newsource actualread-only assess also MANUAL/KN topology. No cache reads/writes. + +Real isolated fault: existingPythonimage77682eea,networknone,read-onlyroot/source, +128MiBRAM/.25CPU,pids64,/fault tmpfs16MiB64inodes,container --rm. Both actual +ENOSPC(bytefull andinodefull) detected. Removing exactlyownedtestfiles restored +capacity and policy cleared, verified write/read after recovery. Duration +0.110404s. No provider data, no hostfaultwrite; Kafka recovery NOT_TESTED. +Temporarycontainer qdl-resilience-storage-fault-20261008 confirmedabsent. +Evidence filesystem-fault.json SHA256 e5c30a8e87ab2e62cdbc3c2de7b7133fca0f6ffef923e8e972954bb53c7dc0e3. +Root ~/.local/state/qdl-v2/resilience-20261008, unit.json/unit.log retained. + +R1 remainsOPEN withClaude; R2 nativeprogress/restartcoordination remainsOPEN; +R3 Kafka/consumerfaults+rollout+release remainOPEN. No runtime/installedsystemd +change. Do not claim self-healing fullstack or releasecertificate. + +No build/images created. Existingactive/rollback retained. Docker86images69containers; +concurrentBuildKit growth196->219entries/50active refs,1.126GB labeledreclaimable. +Not owned by thisslice; no broadprune during concurrentClaude builds. No volumes +removed; tmpfs test ephemeral. Diskavailable95856402432bytes beforetestinventory; +postmeasurement recorded in mainjournal. From 1fc1a3ac7106e42a0ef4fcbcc3984a952f7720bb Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Thu, 8 Oct 2026 18:10:06 +0000 Subject: [PATCH 047/147] Add bounded KN recovery coordinator and exact-container start guard --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 45 +++ scripts/kn_recovery_runner.py | 127 +++++++++ scripts/kn_recovery_state.py | 193 +++++++++++++ tests/test_kn_recovery_runner.py | 72 +++++ tests/test_kn_recovery_state.py | 292 ++++++++++++++++++++ upgrade/evidence/p184-resilience-closure.md | 65 +++++ 6 files changed, 794 insertions(+) create mode 100644 scripts/kn_recovery_runner.py create mode 100644 scripts/kn_recovery_state.py create mode 100644 tests/test_kn_recovery_runner.py create mode 100644 tests/test_kn_recovery_state.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 4bc28bfb..dbbd4e06 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70799,3 +70799,48 @@ notattributed to16MiBtest(torn down). No reclaimclaim. Canonicalmain5bf5cd5 and v2.2.2 unchanged;DLcore658a9570/ingestor0f6876e1/QueryBAR51450612/projectora696cfba/ Stream1f13408f,TSwriter5849d716 unchangedbyslice. Featurefix/p184-dl-closure retainedactive;v2.2.3HOLD, explicitrollbackimagesretained. + +### Resilience R2 native recovery coordination - 2026-10-08 + +IN_PROGRESS, continuing approved R1/R2/R3 scope; guide +upgrade/evidence/p184-resilience-closure.md. Implement bounded non-destructive +KN recovery controller with explicit observed-role identity and deadlines. Gates: +pressure cleared -> quorum/ISR -> projector committed progress -> exact consumer +eligibility. Missing/stale observations never imply SERVING. Default observation +only; exact-ID approved packet needed for Docker start of exited roles, never +restart healthy roles, FLUSH, reset-offset, recreate or delete storage. Keep +retry/backoff bounded and prevent repeated deadline extension. Unit/fault tests +first; runtime changes remain role-specific packet-gated. TS worktree is active +Claude-owned (274fb5f observed), no concurrent edits. Preserve plan dirty hunk. + +### Resilience R2 coordinator tested slice - 2026-10-08 (Codex) + +Guide: upgrade/evidence/p184-resilience-closure.md, R2 Coordinator Slice and R1 +Exact TS Handoff. Added scripts/kn_recovery_state.py, kn_recovery_runner.py and +their two test files. Ordered gates, stable distinct observations, absolute +deadline, bounded retries; default propose-only, exact approved exited-container +start with fresh ID/image/state validation and shared inspect/start budget. +No reset/delete/recreate or automatic OOM restart. Runner is bounded invocation, +NOT installed native recovery service. Actual native observation adapter and +Kafka/consumer fault acceptance still required. R1/R2/R3 remain IN_PROGRESS. + +Tests: python3 -B -m unittest tests.test_kn_recovery_state + tests.test_kn_recovery_runner tests.test_kn_disk_pressure + tests.test_kn_production_watch tests.test_v2_stable_boot_recovery: +68PASS0FAIL0ERROR0SKIP in0.371s. Direct --help passed. Docker operations mocked; +probe subprocess timeout/output-limit exercised with real temporary processes. +Source-only, no runtime change. apply_patch failed with mountinfo-path sandbox +helper error; exact scripted writes used under rule32, then diff verification. + +Claude TS ownership preserved at274fb5f; recorded specific bounded BOOK CAS +batch handoff in guide, not a claim Claude has applied it. No TS files edited. +Canonical DL /home/bobby/data_layer main5bf5cd5 / stablev2.2.2; feature +/home/bobby/.worktrees/data-layer-p184-closure fix/p184-dl-closure; v2.2.3 HOLD. +Runtime observed: core658a9570, ingestors0f6876e1, Query/BAR51450612, +projectorsa696cfba, Stream1f13408f; writer5849d716, bindingr22/JWT10 per last +packet (config not revalidated here). Named BAR rollbackf7351c3b retained. +Cleanup: local temporary probe files/processes disposed; no image/build/container +created by this slice. Inventory87images/73containers/223cacheentries39active; +2.62GB build cache reclaimable but concurrent Claude CI/build resources active, +so no broad prune. Disk available87240540160bytes at inventory; no freed-space +claim. Production listing healthy/up is NOT consumer eligibility acceptance. diff --git a/scripts/kn_recovery_runner.py b/scripts/kn_recovery_runner.py new file mode 100644 index 00000000..38fbc633 --- /dev/null +++ b/scripts/kn_recovery_runner.py @@ -0,0 +1,127 @@ +"""Bounded KN recovery orchestration with explicit exact-container start authority. + +The probe is an operator-selected executable returning a current typed observation. +It must measure actual disk/quorum/progress and production consumer validation; +heartbeat is not consumer eligibility. Defaults to observation only. Never deletes, +flushes, resets offsets, recreates, stops or restarts running containers. +""" +import argparse +import json +import math +import os +from pathlib import Path +import subprocess +import time + +if __package__: + from scripts.kn_recovery_state import RecoveryController, RecoveryPolicy +else: + from kn_recovery_state import RecoveryController, RecoveryPolicy + + +MAX_BYTES = 262144 + + +def read_packet(path): + with open(path, "rb") as handle: + data = handle.read(MAX_BYTES + 1) + if len(data) > MAX_BYTES: + raise ValueError("packet too large") + value = json.loads(data) + if value.get("schema") != "qdl.kn.recovery-packet.v1": + raise ValueError("packet schema") + command = value.get("probe_command") + if (not isinstance(command, list) or not command or len(command) > 32 + or any(not isinstance(v, str) or not v or len(v) > 4096 for v in command) + or not Path(command[0]).is_absolute()): + raise ValueError("absolute bounded probe executable required") + return value + + +def bounded_probe(command, timeout): + # Temporary output files avoid unbounded PIPE buffering from a broken probe. + import tempfile + import resource + def limit_output(): + resource.setrlimit(resource.RLIMIT_FSIZE, (MAX_BYTES, MAX_BYTES)) + with tempfile.TemporaryFile() as output, tempfile.TemporaryFile() as errors: + result = subprocess.run(command, stdout=output, stderr=errors, timeout=timeout, check=False, + preexec_fn=limit_output) + output.seek(0) + raw = output.read(MAX_BYTES + 1) + if result.returncode or len(raw) > MAX_BYTES: + raise ValueError("PROBE_FAILED_OR_TOO_LARGE") + return json.loads(raw) + + +def start_exact(role, expected, approved_ids, timeout, run=subprocess.run, clock=time.monotonic): + if expected["id"] not in approved_ids: + return {"action": "PROPOSED_START", "role": role, "executed": False} + deadline = clock() + timeout + command = ["docker", "inspect", "--format", + "{{.Id}}|{{.Image}}|{{.Name}}|{{.State.Status}}|{{.State.OOMKilled}}", expected["id"]] + result = run(command, capture_output=True, text=True, timeout=timeout, check=False) + if result.returncode: + raise ValueError("PRE_START_INSPECT_FAILED") + parts = result.stdout.strip().split("|") + if len(parts) != 5 or parts[:3] != [expected["id"], expected["image"], "/" + role]: + raise ValueError("PRE_START_IDENTITY_DRIFT") + if parts[3] != "exited" or parts[4] != "false": + raise ValueError("PRE_START_STATE_NOT_AUTHORIZED") + remaining = deadline - clock() + if remaining <= 0: + raise ValueError("PRE_START_DEADLINE_EXPIRED") + result = run(["docker", "start", expected["id"]], capture_output=True, text=True, + timeout=remaining, check=False) + return {"action": "START", "role": role, "executed": True, + "accepted": result.returncode == 0} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--packet", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--approve-start-id", action="append", default=[]) + args = parser.parse_args(argv) + packet = read_packet(args.packet) + policy = RecoveryPolicy(**packet["policy"]) + controller = RecoveryController(policy, packet["roles"]) + approved = set(args.approve_start_id) + if len(approved) != len(args.approve_start_id) or not approved <= {r["id"] for r in packet["roles"].values()}: + raise ValueError("start approval must match exact packet IDs") + interval = packet["poll_seconds"] + if type(interval) not in (int, float) or not math.isfinite(interval) or not .1 <= interval <= 60: + raise ValueError("invalid polling interval") + if policy.deadline_seconds > 3600 or len(packet["roles"]) > 64: + raise ValueError("bounded run requires <= 3600 seconds and <= 64 roles") + end = time.monotonic() + policy.deadline_seconds + descriptor = os.open(args.out, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "w") as output: + while True: + remaining = end - time.monotonic() + try: + observation = bounded_probe(packet["probe_command"], max(.001, min(10, remaining))) if remaining > 0 else {} + except (ValueError, OSError, subprocess.TimeoutExpired): + observation = {} + decision = controller.step(observation) + actions = [] + for action in decision.get("actions", []): + remaining = end - time.monotonic() + if remaining <= 0: + break + try: + actions.append(start_exact(action["role"], packet["roles"][action["role"]], approved, + min(10, remaining))) + except (ValueError, OSError, subprocess.TimeoutExpired): + actions.append({"role": action["role"], "action": "START_REFUSED_OR_UNKNOWN", "executed": None}) + output.write(json.dumps({"at_ns": time.time_ns(), "decision": decision, "actions": actions}, allow_nan=False) + "\n") + output.flush() + if decision["state"] == "SERVING": + return 0 + if decision["state"] == "BLOCKED": + return 3 + time.sleep(min(interval, max(0, end - time.monotonic()))) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/kn_recovery_state.py b/scripts/kn_recovery_state.py new file mode 100644 index 00000000..95555b73 --- /dev/null +++ b/scripts/kn_recovery_state.py @@ -0,0 +1,193 @@ +"""Pure, process-local KN recovery decisions; never performs lifecycle operations.""" + +from dataclasses import dataclass +import math +import re +import time + + +SCHEMA = "qdl.kn.recovery-observation.v1" +GATES = ("disk_safe", "quorum_isr_ready", "projector_progress_ready", "consumer_eligible") +STAGES = ("infra", "projector", "reader") +ROLE_STATES = frozenset(("created", "running", "paused", "restarting", "removing", "exited", "dead")) + + +def _number(value): + try: + return type(value) in (int, float) and math.isfinite(value) and value >= 0 + except OverflowError: + return False + + +def _identity(role): + return (isinstance(role, dict) + and isinstance(role.get("id"), str) + and re.fullmatch(r"[0-9a-f]{64}", role["id"]) is not None + and isinstance(role.get("image"), str) + and re.fullmatch(r"sha256:[0-9a-f]{64}", role["image"]) is not None) + + +@dataclass(frozen=True) +class RecoveryPolicy: + deadline_seconds: float + max_observation_age_seconds: float + stable_samples: int + retry_seconds: float + max_start_attempts: int + + def __post_init__(self): + for name in ("deadline_seconds", "max_observation_age_seconds", "retry_seconds"): + value = getattr(self, name) + if not _number(value) or value <= 0: + raise ValueError(name + " must be finite and positive") + for name in ("stable_samples", "max_start_attempts"): + if type(getattr(self, name)) is not int or getattr(self, name) <= 0: + raise ValueError(name + " must be a positive integer") + + +class RecoveryController: + """One absolute deadline per episode, including unknown/pressure/backoff time. + + step returns a JSON-compatible dict. actions is empty or contains one + {action: 'start', role, id, image, attempt} proposal, not an execution receipt. + A proposal consumes an attempt even if an external adapter declines it. + Retry delay doubles up to deadline_seconds. BLOCKED is sticky; an operator + must create a new controller after review. SERVING keeps monitoring, and + the first unhealthy/unknown poll starts a new bounded episode. + + Identical fresh timestamps preserve stability but never add credit or retry. + Changed data at an identical timestamp and regressing timestamps are unknown. + Only validated gate/role fields participate in duplicate comparison. + """ + + def __init__(self, policy, expected_roles, clock=time.monotonic): + if not isinstance(policy, RecoveryPolicy): + raise ValueError("policy must be RecoveryPolicy") + if not isinstance(expected_roles, dict) or not expected_roles: + raise ValueError("expected_roles must be a nonempty roster") + self._roles = {} + for name, role in expected_roles.items(): + if (not isinstance(name, str) or not name or not _identity(role) + or role.get("stage") not in STAGES): + raise ValueError("invalid expected role") + self._roles[name] = {key: role[key] for key in ("id", "image", "stage")} + if len({role["id"] for role in self._roles.values()}) != len(self._roles): + raise ValueError("duplicate expected container ID") + self.policy = policy + self._clock = clock + now = clock() + if not _number(now) or not math.isfinite(now + policy.deadline_seconds): + raise ValueError("invalid clock/deadline") + self._last_now = now + self._deadline = now + policy.deadline_seconds + self._last_observation = None + self._stable = 0 + self._attempts = {} + self._retry_at = {} + self._delay = {} + self.state = "HOLD_UNKNOWN" + self._reason = "awaiting observation" + + def _result(self, state, reason, actions=None): + self.state, self._reason = state, reason + return {"state": state, "reason": reason, "actions": actions or [], + "stable_samples": self._stable, "deadline_monotonic": self._deadline} + + def _validate(self, observation, now): + if not isinstance(observation, dict) or observation.get("schema") != SCHEMA: + return None, "invalid schema" + stamp = observation.get("observed_monotonic") + if not _number(stamp) or stamp > now or now - stamp > self.policy.max_observation_age_seconds: + return None, "invalid, future or stale observation time" + if any(type(observation.get(gate)) is not bool for gate in GATES): + return None, "missing or non-boolean gate" + roles = observation.get("roles") + if not isinstance(roles, dict) or roles.keys() != self._roles.keys(): + return None, "role roster mismatch" + normalized = {"observed_monotonic": stamp, "roles": {}} + normalized.update({gate: observation[gate] for gate in GATES}) + for name, role in roles.items(): + if (not _identity(role) or not isinstance(role.get("state"), str) + or role["state"] not in ROLE_STATES): + return None, "malformed role" + normalized["roles"][name] = {key: role[key] for key in ("id", "image", "state")} + return normalized, None + + def _start(self, stage, roles, now, duplicate): + if duplicate: + return [], None + for name, expected in self._roles.items(): + if expected["stage"] != stage or roles[name]["state"] != "exited": + continue + if now < self._retry_at.get(name, 0): + continue + attempts = self._attempts.get(name, 0) + if attempts >= self.policy.max_start_attempts: + return [], "start attempts exhausted: " + name + attempts += 1 + self._attempts[name] = attempts + delay = min(self.policy.deadline_seconds, + self._delay[name] * 2 if name in self._delay else self.policy.retry_seconds) + self._delay[name] = delay + self._retry_at[name] = now + delay + return [{"action": "start", "role": name, "id": expected["id"], + "image": expected["image"], "attempt": attempts}], None + return [], None + + def step(self, observation): + if self.state == "BLOCKED": + return self._result("BLOCKED", self._reason) + now = self._clock() + if not _number(now) or now < self._last_now: + self._stable = 0 + return self._result("BLOCKED", "invalid controller clock") + self._last_now = now + was_serving = self.state == "SERVING" + if not was_serving and now >= self._deadline: + self._stable = 0 + return self._result("BLOCKED", "recovery deadline exceeded") + sample, error = self._validate(observation, now) + duplicate = False + if sample is not None: + for name, role in sample["roles"].items(): + if any(role[key] != self._roles[name][key] for key in ("id", "image")): + self._stable = 0 + return self._result("BLOCKED", "role identity drift: " + name) + previous = self._last_observation + if previous is not None: + delta = sample["observed_monotonic"] - previous["observed_monotonic"] + if delta < 0 or (delta == 0 and sample != previous): + error = "regressing or conflicting observation" + duplicate = delta == 0 + healthy = (sample is not None and error is None + and all(sample[gate] for gate in GATES) + and all(role["state"] == "running" for role in sample["roles"].values())) + if was_serving and not healthy: + self._deadline = now + self.policy.deadline_seconds + self._attempts.clear() + self._retry_at.clear() + self._delay.clear() + if error is not None: + self._stable = 0 + return self._result("HOLD_UNKNOWN", error) + self._last_observation = sample + if not healthy: + self._stable = 0 + if not sample["disk_safe"]: + return self._result("HOLD_PRESSURE", "disk pressure holds all starts") + roles = sample["roles"] + for stage, gate, state in ( + ("infra", "quorum_isr_ready", "WAIT_QUORUM"), + ("projector", "projector_progress_ready", "WAIT_PROJECTOR"), + ("reader", "consumer_eligible", "WAIT_CONSUMER")): + running = all(roles[name]["state"] == "running" for name, role in self._roles.items() + if role["stage"] == stage) + if not running or not sample[gate]: + actions, blocked = self._start(stage, roles, now, duplicate) + return self._result("BLOCKED" if blocked else state, + blocked or "waiting for " + stage + " readiness", actions) + if not duplicate: + self._stable = min(self.policy.stable_samples, self._stable + 1) + if self._stable >= self.policy.stable_samples: + return self._result("SERVING", "all gates stable") + return self._result("WAIT_CONSUMER", "waiting for stable observations") diff --git a/tests/test_kn_recovery_runner.py b/tests/test_kn_recovery_runner.py new file mode 100644 index 00000000..c69d10d8 --- /dev/null +++ b/tests/test_kn_recovery_runner.py @@ -0,0 +1,72 @@ +"""No Docker mutation: execution-boundary regression with explicit fake CLI.""" +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import Mock + +from scripts.kn_recovery_runner import bounded_probe, read_packet, start_exact + + +class RunnerTests(unittest.TestCase): + def setUp(self): + self.expected = {"id": "a" * 64, "image": "sha256:" + "b" * 64} + self.output = "|".join([self.expected["id"], self.expected["image"], "/reader", "exited", "false"]) + + def test_default_never_inspects_or_starts(self): + run = Mock() + self.assertFalse(start_exact("reader", self.expected, set(), 2, run=run)["executed"]) + run.assert_not_called() + + def test_exact_start_uses_remaining_budget(self): + run = Mock(side_effect=[subprocess.CompletedProcess([], 0, self.output), subprocess.CompletedProcess([], 0, "")]) + clock = Mock(side_effect=[1, 2.5]) + result = start_exact("reader", self.expected, {self.expected["id"]}, 2, run=run, clock=clock) + self.assertTrue(result["accepted"]) + self.assertEqual(run.call_args.kwargs["timeout"], .5) + self.assertEqual(run.call_args.args[0], ["docker", "start", self.expected["id"]]) + + def test_expired_budget_never_starts(self): + run = Mock(return_value=subprocess.CompletedProcess([], 0, self.output)) + with self.assertRaisesRegex(ValueError, "DEADLINE"): + start_exact("reader", self.expected, {self.expected["id"]}, 2, run=run, clock=Mock(side_effect=[1, 3])) + self.assertEqual(run.call_count, 1) + + def test_drift_running_and_oom_never_start(self): + for output in (self.output.replace("/reader", "/other"), self.output.replace("exited", "running"), self.output.replace("false", "true"), self.output.replace("a" * 64, "c" * 64)): + with self.subTest(output=output): + run = Mock(return_value=subprocess.CompletedProcess([], 0, output)) + with self.assertRaises(ValueError): + start_exact("reader", self.expected, {self.expected["id"]}, 2, run=run) + self.assertEqual(run.call_count, 1) + + def test_inspect_failure(self): + run = Mock(return_value=subprocess.CompletedProcess([], 1, "")) + with self.assertRaisesRegex(ValueError, "INSPECT"): + start_exact("reader", self.expected, {self.expected["id"]}, 2, run=run) + self.assertEqual(run.call_count, 1) + + def test_probe_real_subprocess(self): + self.assertEqual(bounded_probe([sys.executable, "-c", 'print("{}")'], 2), {}) + + def test_probe_failure_or_oversize(self): + for code in ('raise SystemExit(1)', 'print("x" * 300000)'): + with self.assertRaises(ValueError): + bounded_probe([sys.executable, "-c", code], 2) + + def test_probe_timeout(self): + with self.assertRaises(subprocess.TimeoutExpired): + bounded_probe([sys.executable, "-c", "import time; time.sleep(3)"], .05) + + def test_packet_requires_absolute_executable(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "packet.json" + path.write_text(json.dumps({"schema": "qdl.kn.recovery-packet.v1", "probe_command": ["sh"]})) + with self.assertRaises(ValueError): + read_packet(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_kn_recovery_state.py b/tests/test_kn_recovery_state.py new file mode 100644 index 00000000..6cb15100 --- /dev/null +++ b/tests/test_kn_recovery_state.py @@ -0,0 +1,292 @@ +"""Deterministic TEST_ONLY observations; no Docker, brokers, or provider claims.""" + +import copy +import unittest + +from scripts.kn_recovery_state import RecoveryController, RecoveryPolicy, GATES, SCHEMA + + +class RecoveryTests(unittest.TestCase): + def setUp(self): + self.now = 100.0 + self.roles = {name: {"id": digit * 64, "image": "sha256:" + digit * 64, "stage": stage} + for name, digit, stage in (("broker", "a", "infra"), + ("projector", "b", "projector"), + ("reader", "c", "reader"))} + self.controller = self.make_controller() + + def make_controller(self, **kwargs): + policy = dict(deadline_seconds=30, max_observation_age_seconds=5, + stable_samples=3, retry_seconds=2, max_start_attempts=2) + policy.update(kwargs) + return RecoveryController(RecoveryPolicy(**policy), self.roles, lambda: self.now) + + def observation(self, **updates): + sample = {"schema": SCHEMA, "observed_monotonic": self.now, + "roles": {name: {"id": role["id"], "image": role["image"], "state": "running"} + for name, role in self.roles.items()}} + sample.update(dict.fromkeys(GATES, True)) + sample.update(updates) + return sample + + def step(self, **updates): + return self.controller.step(self.observation(**updates)) + + def serve(self): + for _ in range(3): + result = self.step() + self.now += 1 + self.assertEqual(result["state"], "SERVING") + + def test_all_gates_need_consecutive_distinct_samples(self): + for expected in (1, 2, 3): + result = self.step() + self.assertEqual(result["stable_samples"], expected) + self.assertEqual(result["state"], "SERVING" if expected == 3 else "WAIT_CONSUMER") + self.assertEqual(result["actions"], []) + self.now += 1 + + def test_false_gates_fail_closed_in_order(self): + states = ("HOLD_PRESSURE", "WAIT_QUORUM", "WAIT_PROJECTOR", "WAIT_CONSUMER") + for gate, state in zip(GATES, states): + with self.subTest(gate=gate): + self.controller = self.make_controller() + result = self.step(**{gate: False}) + self.assertEqual(result["state"], state) + self.assertEqual(result["actions"], []) + self.assertEqual(result["stable_samples"], 0) + + def test_pressure_prevents_every_start(self): + sample = self.observation(disk_safe=False) + for role in sample["roles"].values(): + role["state"] = "exited" + self.assertEqual(self.controller.step(sample)["actions"], []) + + def test_sequential_start_proposals_require_running_upstream_and_gate(self): + sample = self.observation() + for role in sample["roles"].values(): + role["state"] = "exited" + for name, gate, state in (("broker", "quorum_isr_ready", "WAIT_QUORUM"), + ("projector", "projector_progress_ready", "WAIT_PROJECTOR"), + ("reader", "consumer_eligible", "WAIT_CONSUMER")): + sample["observed_monotonic"] = self.now + result = self.controller.step(sample) + self.assertEqual(result["state"], state) + self.assertEqual(result["actions"], [{"action": "start", "role": name, + "id": self.roles[name]["id"], "image": self.roles[name]["image"], "attempt": 1}]) + self.now += 1 + sample["roles"][name]["state"] = "running" + sample[gate] = False + sample["observed_monotonic"] = self.now + held = self.controller.step(sample) + self.assertEqual(held["state"], state) + self.assertEqual(held["actions"], []) + sample[gate] = True + self.now += 1 + + def test_never_start_running_restarting_or_other_non_exited_states(self): + for state in ("running", "restarting", "paused", "created", "removing", "dead"): + with self.subTest(state=state): + sample = self.observation(quorum_isr_ready=False) + sample["roles"]["broker"]["state"] = state + self.assertEqual(self.make_controller().step(sample)["actions"], []) + + def test_malformed_observations_hold_unknown(self): + variants = [None, [], {}, self.observation(schema="wrong")] + for gate in GATES: + for value in (None, "true", "false", 0, 1, [], {}): + variants.append(self.observation(**{gate: value})) + sample = self.observation() + del sample[gate] + variants.append(sample) + for value in (None, True, "100", -1, float("nan"), float("inf"), 10 ** 1000): + variants.append(self.observation(observed_monotonic=value)) + for field, value in (("id", "short"), ("image", "tag:latest"), ("state", "unknown"), + ("state", []), ("id", None)): + sample = self.observation() + sample["roles"]["broker"][field] = value + variants.append(sample) + for sample in variants: + with self.subTest(sample=repr(sample)[:160]): + result = self.make_controller().step(sample) + self.assertEqual(result["state"], "HOLD_UNKNOWN") + self.assertEqual(result["actions"], []) + + def test_exact_roster_required(self): + for change in ("missing", "extra", "list", "missing_state"): + sample = self.observation() + if change == "missing": + del sample["roles"]["reader"] + elif change == "extra": + sample["roles"]["other"] = dict(sample["roles"]["reader"]) + elif change == "list": + sample["roles"] = list(sample["roles"].values()) + else: + del sample["roles"]["broker"]["state"] + self.assertEqual(self.make_controller().step(sample)["state"], "HOLD_UNKNOWN") + + def test_stale_future_and_boundary_age(self): + for stamp in (94.999, 100.001): + self.assertEqual(self.make_controller().step(self.observation(observed_monotonic=stamp))["state"], + "HOLD_UNKNOWN") + self.assertEqual(self.step(observed_monotonic=95)["stable_samples"], 1) + + def test_identity_drift_blocks_sticky(self): + for field, value in (("id", "d" * 64), ("image", "sha256:" + "d" * 64)): + controller = self.make_controller() + sample = self.observation() + sample["roles"]["reader"][field] = value + self.assertEqual(controller.step(sample)["state"], "BLOCKED") + self.now += 1 + self.assertEqual(controller.step(self.observation())["state"], "BLOCKED") + + def test_duplicate_timestamp_has_no_credit_or_actions(self): + sample = self.observation() + self.controller.step(sample) + self.now += 1 + self.assertEqual(self.controller.step(sample)["stable_samples"], 1) + controller = self.make_controller() + sample = self.observation() + sample["roles"]["broker"]["state"] = "exited" + self.assertEqual(len(controller.step(sample)["actions"]), 1) + self.now += 3 + self.assertEqual(controller.step(sample)["actions"], []) + + def test_conflicting_and_regressing_timestamps_reset_stability(self): + self.step() + result = self.step(consumer_eligible=False) + self.assertEqual(result["state"], "HOLD_UNKNOWN") + self.assertEqual(result["stable_samples"], 0) + self.assertEqual(self.step(observed_monotonic=99)["state"], "HOLD_UNKNOWN") + + def test_unknown_and_false_reset_stability(self): + for unknown in (True, False): + self.controller = self.make_controller() + self.step() + self.now += 1 + result = self.controller.step(None) if unknown else self.step(consumer_eligible=False) + self.assertEqual(result["stable_samples"], 0) + self.now += 1 + self.assertEqual(self.step()["stable_samples"], 1) + + def test_deadline_absolute_sticky_across_stage_changes_and_unknown(self): + deadline = 130 + for offset, gate in ((0, "disk_safe"), (10, "quorum_isr_ready"), + (20, "projector_progress_ready"), (29, "consumer_eligible")): + self.now = 100 + offset + self.assertEqual(self.step(**{gate: False})["deadline_monotonic"], deadline) + self.now = deadline + self.assertEqual(self.controller.step(None)["state"], "BLOCKED") + self.assertEqual(self.step()["state"], "BLOCKED") + + def test_slow_status_cannot_reset_or_beat_deadline(self): + sample = self.observation() + self.now = 130 + result = self.controller.step(sample) + self.assertEqual(result["state"], "BLOCKED") + self.assertEqual(result["actions"], []) + + def test_retry_backoff_and_ceiling(self): + for offset, count, state in ((0, 1, "WAIT_QUORUM"), (1, 0, "WAIT_QUORUM"), + (2, 1, "WAIT_QUORUM"), (5, 0, "WAIT_QUORUM"), + (6, 0, "BLOCKED")): + self.now = 100 + offset + sample = self.observation() + sample["roles"]["broker"]["state"] = "exited" + result = self.controller.step(sample) + self.assertEqual(result["state"], state) + self.assertEqual(len(result["actions"]), count) + + def test_last_attempt_has_time_to_succeed(self): + self.controller = self.make_controller(max_start_attempts=1) + sample = self.observation() + sample["roles"]["broker"]["state"] = "exited" + self.controller.step(sample) + self.now += 3 + self.assertEqual(self.step()["state"], "WAIT_CONSUMER") + + def test_multiple_exited_roles_only_one_action_each_step(self): + self.roles["broker2"] = {"id": "d" * 64, "image": "sha256:" + "d" * 64, "stage": "infra"} + self.controller = self.make_controller() + for expected in ("broker", "broker2"): + sample = self.observation() + for name in ("broker", "broker2"): + sample["roles"][name]["state"] = "exited" + actions = self.controller.step(sample)["actions"] + self.assertEqual(len(actions), 1) + self.assertEqual(actions[0]["role"], expected) + self.now += 1 + + def test_serving_continues_beyond_original_deadline(self): + self.serve() + self.now = 200 + self.assertEqual(self.step()["state"], "SERVING") + sample = self.observation() + self.now += 1 + self.assertEqual(self.controller.step(sample)["state"], "SERVING") + + def test_after_serving_failure_starts_one_new_bounded_episode(self): + self.serve() + self.now = 200 + result = self.step(disk_safe=False) + self.assertEqual(result["state"], "HOLD_PRESSURE") + self.assertEqual(result["deadline_monotonic"], 230) + self.now = 210 + self.assertEqual(self.controller.step(None)["deadline_monotonic"], 230) + self.now = 230 + self.assertEqual(self.step()["state"], "BLOCKED") + + def test_recovery_after_serving_resets_attempts_and_stability(self): + sample = self.observation() + sample["roles"]["reader"]["state"] = "exited" + self.controller.step(sample) + self.now += 1 + self.serve() + sample = self.observation() + sample["roles"]["reader"]["state"] = "exited" + result = self.controller.step(sample) + self.assertEqual(result["stable_samples"], 0) + self.assertEqual(result["actions"][0]["attempt"], 1) + self.now += 1 + self.serve() + + def test_stale_poll_after_serving_is_not_healthy(self): + self.serve() + sample = self.observation() + self.now += 6 + result = self.controller.step(sample) + self.assertEqual(result["state"], "HOLD_UNKNOWN") + self.assertEqual(result["deadline_monotonic"], self.now + 30) + + def test_input_and_returned_proposal_cannot_mutate_pinned_roles(self): + sample = self.observation() + sample["roles"]["broker"]["state"] = "exited" + original = copy.deepcopy(sample) + result = self.controller.step(sample) + self.assertEqual(sample, original) + result["actions"][0]["id"] = "d" * 64 + self.roles["broker"]["id"] = "e" * 64 + self.now += 2 + sample["observed_monotonic"] = self.now + self.assertEqual(self.controller.step(sample)["actions"][0]["id"], "a" * 64) + + def test_policy_and_roster_validation(self): + for field in ("deadline_seconds", "max_observation_age_seconds", "retry_seconds", + "stable_samples", "max_start_attempts"): + for value in (True, 0, -1, "2", None, float("inf"), float("nan")): + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + self.make_controller(**{field: value}) + for field in ("stable_samples", "max_start_attempts"): + with self.assertRaises(ValueError): + self.make_controller(**{field: 1.5}) + for roles in ({}, {"x": {}}, {"x": self.roles["broker"], "y": self.roles["broker"]}): + with self.assertRaises(ValueError): + RecoveryController(self.controller.policy, roles) + + def test_controller_clock_regression_blocks(self): + self.now = 99 + self.assertEqual(self.step()["state"], "BLOCKED") + + +if __name__ == "__main__": + unittest.main() diff --git a/upgrade/evidence/p184-resilience-closure.md b/upgrade/evidence/p184-resilience-closure.md index 11498b2a..e0a4034e 100644 --- a/upgrade/evidence/p184-resilience-closure.md +++ b/upgrade/evidence/p184-resilience-closure.md @@ -115,3 +115,68 @@ concurrentBuildKit growth196->219entries/50active refs,1.126GB labeledreclaimabl Not owned by thisslice; no broadprune during concurrentClaude builds. No volumes removed; tmpfs test ephemeral. Diskavailable95856402432bytes beforetestinventory; postmeasurement recorded in mainjournal. + +## R2 Coordinator Slice - 2026-10-08 + +Status: IMPLEMENTED_TESTED_LOCALLY; overall R1/R2/R3 IN_PROGRESS. +`kn_recovery_state.py` supplies the pure state machine. Unknown/stale/conflicting +observations fail closed. Disk, Kafka quorum/ISR, projector progress and consumer +eligibility are independent ordered gates. Stable distinct samples are required; +heartbeat alone cannot produce SERVING. One absolute recovery deadline, bounded +backoff/attempts and sticky BLOCKED prevent infinite retry. After SERVING the +state machine can start a new bounded episode when health deteriorates. + +`kn_recovery_runner.py` is a bounded invocation (not an installed daemon). It +exits on SERVING/BLOCKED. Default is observe/propose only. Explicit approval IDs +must belong to the packet; each start reinspects exact ID/image/name/exited state +and refuses OOM. Inspect and start share one timeout. It never stops/recreates, +flushes cache or resets offsets. Probe subprocess output is capped at 256 KiB; +packet <=256 KiB, <=64 roles, run <=3600s, polling >=0.1s. Evidence output is new, +exclusive mode0600; existing receipts cannot be overwritten. + +Required production integration STILL MISSING: supply a native probe which +reads actual quorum/ISR, committed progress/cache catch-up and the production TS +validator result, with a fresh monotonic timestamp. Arbitrary JSON booleans or +READY60 are not acceptable certification evidence. No production packet or +systemd installation has been made. The existing boot service has NOT received +the legacy fence patch. Do not claim automatic full-stack recovery is deployed. + +Verification: `python3 -B -m unittest tests.test_kn_recovery_state + tests.test_kn_recovery_runner tests.test_kn_disk_pressure + tests.test_kn_production_watch tests.test_v2_stable_boot_recovery`: 68 PASS, +0 failures/errors/skips, 0.371s. Coordinator24 + runner9 + prior35. Runner tests +use mocked Docker commands; subprocess output overflow and timeout are real local +process tests, not Kafka faults. Direct script --help passed. No image build, +container creation or runtime mutation in this slice. + +## R1 Exact TS Handoff For Claude - 2026-10-08 + +Read-only review at TS274fb5f; no edits to Claude's worktree. IDR-081 remains +open. Reset anchors exist (58287fc/dfc70bd), but cache_projector still applies +BOOK entries serially through execution_state.store_advancing. Per item this +includes marker/value read, JSON comparison, CAS carrying old/new JSON, retries +and some rejection readback. This identifies avoidable work, not a measured +allocation of all observed latency. Existing writer5849d716 still SDK2.0.7. + +Proposed write ownership: Claude edits execution_state.py/cache_projector.py. +Implement bounded runs (at most64) of same-key, same-generation BOOK_DELTA. +Read/verify initial marker and value once, prepare all input before writing and +simulate EVERY transition with existing ordering semantics. A bounded Lua CAS +must check initial marker AND value; validate every argument before mutation, +then preserve accepted writes and publish order/TTL. Do not retain only the last +book delta. Initial CAS conflict performs no writes; recompute inside the SAME +deadline. Lost/unknown Redis reply does not ACK; replay from confirmed cursor. +Adoption, reset anchor and generation transitions remain scalar initially. +Do not pipeline dependent same-key CAS or reuse TRADE semantics for BOOK. + +Required real Redis comparisons against scalar: both venues; sizes1/64/65; +older/duplicate/conflicting sequence; corrupt marker/value; generation/reset; +publication order, TTL and exact bytes; lost reply, partial failure, crash before +and after ACK; no backward state or skipped delta. Coordinate tests in +integration/test_edc_book_delta_redis.py and test_edc_execution_state_redis.py. +Then reuse the SAME event-joined trace through SDK/apply/ACK/cursor. No claim +of runtime latency improvement until that test and narrow rollout are complete. + +R3 remains open: actual Kafka/consumer fault integration, compatible narrow +rollout and real consumer usability acceptance. Existing filesystem ENOSPC +receipt is inherited, not substituted for Kafka durability/recovery. From 8d243a24b1c0b67e8fb9fe70a2509feb1863bd4a Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 06:03:42 +0000 Subject: [PATCH 048/147] Align resilience handoff with Redis APPLY evidence and phase D gates --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 16 ++++++++++++++++ upgrade/evidence/p184-resilience-closure.md | 14 ++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index dbbd4e06..17438714 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70844,3 +70844,19 @@ created by this slice. Inventory87images/73containers/223cacheentries39active; 2.62GB build cache reclaimable but concurrent Claude CI/build resources active, so no broad prune. Disk available87240540160bytes at inventory; no freed-space claim. Production listing healthy/up is NOT consumer eligibility acceptance. + +### Phase D G4-G6 handoff and Redis APPLY attribution correction - 2026-10-09 + +Owner delegated TS upgrade/p184-phase-d-gates.md to Codex. Documentation review +filled G4-G6 with verified v2.2.0 certificate/endpoint hashes and TS paper source +manifest revision10 semantics; no invented all-feed canonical baseline or batch +p95 from n=1 history samples. Status DRAFT_PENDING_OWNER_FREEZE; missing numeric +coverage and G2/G9 semantic conflicts listed for Claude/owner, not waived. +TS phase-D a4507abb identifies Redis APPLY contention; updated resilience guide +to supersede speculative BOOK-CAS batching. No unsynchronised TS application edit. +Four workers unchanged; full mixed benchmark then explicit runtime packet needed. +Validation: JSON/hash, per-row numeric cross-check, diff/Markdown checks only. +No runtime/probe/build/container/secret changes, no resource cleanup needed. +Canonical DL /home/bobby/data_layer main5bf5cd5, stablev2.2.2 as previously recorded; +closure branch fix/p184-dl-closure; runtime digests/config/rollback unchanged by +this docs-only review, not re-inspected. v2.2.3 HOLD. No push/merge. diff --git a/upgrade/evidence/p184-resilience-closure.md b/upgrade/evidence/p184-resilience-closure.md index e0a4034e..e9362dbb 100644 --- a/upgrade/evidence/p184-resilience-closure.md +++ b/upgrade/evidence/p184-resilience-closure.md @@ -180,3 +180,17 @@ of runtime latency improvement until that test and narrow rollout are complete. R3 remains open: actual Kafka/consumer fault integration, compatible narrow rollout and real consumer usability acceptance. Existing filesystem ENOSPC receipt is inherited, not substituted for Kafka durability/recovery. + +## R1 Handoff Correction - 2026-10-09 + +Claude TS phase-D slices3-6 (sourcea4507abb) provide newer profiling: Redis atomic +APPLY contention blocks BOOK/MARK/readers. Previous same-key BOOK CAS batch +proposal above is NOT an instruction to implement speculatively. Real BOOK +batches are often size1; prioritise APPLY encoding/EVALSHA/source-contract dedup +already under Claude ownership. Full mixed workload must include production-size +BOOK_SNAPSHOT and MARK, absent in the earlier idle-Redis benchmark. Per-callCPU +improvement alone is not end-to-end closure; keep four workers per owner. +See TS upgrade/p184-phase-d-gates.md, Codex DL Review2026-10-09: G4-G6 receipts, +semantic objections and freeze blockers. G4 all-feed canonical baseline and G5 +warmup/batch numeric limits are not available as certified stage50 values. No +runtime changes, load, release or new phase authorised by this documentation. From 16c342f9f1bd267a18d9ad437d873934e70cc8a6 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 08:24:05 +0000 Subject: [PATCH 049/147] Record DR05 export, paired canonical latency and build lock audit --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 34 ++++++ .../p184-dr05-g4-build-handoff-20261009.md | 108 ++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 upgrade/evidence/p184-dr05-g4-build-handoff-20261009.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 17438714..c8779d5f 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70860,3 +70860,37 @@ No runtime/probe/build/container/secret changes, no resource cleanup needed. Canonical DL /home/bobby/data_layer main5bf5cd5, stablev2.2.2 as previously recorded; closure branch fix/p184-dl-closure; runtime digests/config/rollback unchanged by this docs-only review, not re-inspected. v2.2.3 HOLD. No push/merge. + +### DR05 public export, paired TRADE/MARK G4 and build lock audit - 2026-10-09 + +Status IN_PROGRESS; owner requests existing public export or isolated init, +concurrent TRADE/MARK canonical timing and uv-lock audit. Reuse public export +dr05-observer-20261008/public-export.json, validate through isolated_config; do not +start a duplicate DR05 stack while Claude starts adapter. Bounded read-only assigned +Kafka observer, no joins/offset commits, no second TS stream, credentials only +in-memory existing mount paths. At most120s/10000 observations/16MiB output. +Measure exact-event source->published separately from read_committed visibility +upper bound, never call CreateTime commit time. Keep missing/negative timestamps +explicit. Same current window for TRADE and MARK; historical5-11s attribution +requires matching TS event trace, not today's aggregate alone. No rollout/build, +no broad runtime dependency changes. Audit Dockerfiles/lock verifier and document +mutable toolchain/OS resolution separately from application lock. Cleanup only +own temporary observer; retain bounded receipt. + +### DR05/G4/build handoff result - 2026-10-09 + +Guide/report upgrade/evidence/p184-dr05-g4-build-handoff-20261009.md. +Public export validated, existing path+hash handed to Claude; no duplicate stack. +Real read-only canonical window08:19:24-08:21:24UTC,120s,2689sampled TRADE/MARK +BTC/ETH observations. TRADE p95103-190ms; BinanceMARK150-181ms; OKXMARK928-985ms; +max1885.003ms. Exact per-product counts and p50/p95/p99/max in report. Boundary +read_committed observed visibility upper bound, not publish/commitclock; no same +window TSACK trace so no retrospective5-11s causal claim. Writer observed162ea147, +not previous5849d716. Observer no joins/commits/writes and exited0. +Build audit: no uv sync; Poetrylock/main RECORD verification and cargolocked. +Mutable setuptools>=78.1.1/bootstrap/OS dependencies remain reproducibility risk, +not silently fixed or claimed fully reproducible. No new image or dependency. +Host test36with2missing-dependency errors; existing-image final38PASS0ERROR0SKIP. +Scoped --rm test removed, no persistent stack/build/volume created or broadprune. +Report records immutable summary/capture/script hashes. Runtime roles unchanged +by this turn; no push/merge/release. G4 fullscope/DR05 acceptance remain open. diff --git a/upgrade/evidence/p184-dr05-g4-build-handoff-20261009.md b/upgrade/evidence/p184-dr05-g4-build-handoff-20261009.md new file mode 100644 index 00000000..e40274d5 --- /dev/null +++ b/upgrade/evidence/p184-dr05-g4-build-handoff-20261009.md @@ -0,0 +1,108 @@ +# DR05 Export, G4 TRADE/MARK and Build Lock Handoff + +Status: HANDOFF_READY / G4_BOUNDED_OBSERVATION / NOT_PHASE_D_CERTIFIED. +Documentation and read-only measurement only; no rollout, mutation, push or merge. + +## DR05 public export + +`PUBLIC_EXPORT=/home/bobby/.local/state/qdl-v2/dr05-observer-20261008/public-export.json` + +SHA256 `9cf857a223cd9f370452c4121c0d14704df87f2843a368536898054e0ceef31a`, +1359 bytes. Checked exact top-level core/bindings and exact equality to +`isolated_config()` public allowlist. Exactly BINANCE ETHUSDT and OKX +ETH-USDT-SWAP, PRIMARY PERPETUAL BOOK with L2 parameters. No transport, credentials +or production authority. This file does not expire; newly generated TLS/JWT does. +Use a NEW work directory and current DR05 init/up/ready/reset-host guide. This +turn supplied the requested file rather than starting a duplicate disposable +stack. No watcher is running on Claude's behalf. Do not interpret this as DR05 PASS. +Existing `upgrade/evidence/p184-dr05-dl-fault.md` and `scripts/p184_dr05_reset.py` +remain authoritative for clean broker-preserving resets; TS clones must stop as +required before reset. Ports6391/5433 remain reserved by that packet. + +## G4 simultaneous real observations + +Window UTC: 2026-10-09T08:19:24.396767+00:00 to 2026-10-09T08:21:24.288447+00:00. +BTC and ETH, both venues, TRADE and MARK_INDEX_PRICE. 120.000s, +2689 observations, zero decode/Kafka errors or missing/negative timestamp pairs. +Quantiles nearest-rank, ms. At most one emitted event per product/feed per100ms: +this is a bounded time-throttled sample, NOT every-event or uniform-reservoir +statistics. Six partitions assigned at their current end; read_committed, +no group join, auto-store/commit disabled, no broker writes. Existing Query image +executes temporary Python observer; no service/image/config changes. + +| Venue/symbol/feed | n | source -> visible p50 | p95 | p99 | max | ingest -> visible p95 | +|---|---:|---:|---:|---:|---:|---:| +| BINANCE/BTCUSDT/mark_index_price | 120 | 123.628 | 150.156 | 161.474 | 211.744 | 73.226 | +| BINANCE/BTCUSDT/trade | 371 | 131.515 | 188.369 | 224.310 | 236.418 | 161.551 | +| BINANCE/ETHUSDT/mark_index_price | 120 | 128.756 | 181.198 | 209.775 | 211.934 | 96.785 | +| BINANCE/ETHUSDT/trade | 307 | 126.992 | 190.452 | 211.739 | 239.268 | 160.767 | +| OKX/BTC-USDT-SWAP/mark_index_price | 614 | 385.625 | 985.058 | 1343.653 | 1707.399 | 913.008 | +| OKX/BTC-USDT-SWAP/trade | 267 | 82.607 | 103.174 | 112.896 | 120.435 | 72.996 | +| OKX/ETH-USDT-SWAP/mark_index_price | 619 | 332.750 | 927.811 | 1309.909 | 1885.003 | 865.238 | +| OKX/ETH-USDT-SWAP/trade | 271 | 80.675 | 105.900 | 119.849 | 138.854 | 74.511 | + +Boundary: source envelope time -> reader observes the exact canonical +partition/offset under read_committed. This is an UPPER BOUND on visibility, +including observer fetch/decode/scheduling; not a broker commit clock. Source +clock synchronisation was not independently calibrated. MARK component cadence, +lineage and quiet eligibility are NOT certified by envelope timing alone. +Core `rust/qdl-realtime-core/src/lib.rs:1760` sets TRADE published_at_ns from +raw.received_at_ns; it is NOT a trustworthy commit timestamp. Raw timings remain +in evidence; do not subtract unrelated quantiles to allocate delay. + +No sampled canonical tail of5-11s in THIS window. This cannot attribute a +historical/different-window TS tail or prove every event is fast. For attribution, +join TS SDK/apply/ACK/cursor trace on exact identity/generation/partition/offset +within this UTC interval. No TS trace rearm or restart was performed. G4 all-feed +phase-D baseline and stage100 remain open; no inverse or other-symbol extrapolation. + +Observed writer already changed by another operator to162ea147668f, started +2026-10-09T07:31:40Z, restarts0/OOMfalse. Do not compare with old5849d716 trace as +if same runtime. Query51450612, core trio658a9570; no changes made by this turn. + +Evidence root `/home/bobby/.local/state/qdl-v2/g4-trade-mark-20261009`. +- `summary.json` SHA256 `d603e34bd6a800c305a3c9142f58a4a989698786ec773cf1f97ba5afc65b0e30`. +- `observer.py` SHA256 `66ce6625d664641eac2fc5aa50d778c654ad09529c7d17d17b79d79609389624`. +- `capture/observations.jsonl` SHA256 `7998bb5b35107ef75a134f4e61e5e2292e3caec6536b3d9bb50d558f8607f51e`. +- `capture/receipt.json` SHA256 `59531419e7533599f8d166f75743ed2515200d97499e7908289e6074cf437246`. + +Command: python3 -B $ROOT/observer.py --launch --mode canonical --seconds120 +--tail-records0 --max-records10000 --sample-ms100 --max-output-bytes16777216 +--output $ROOT/capture (CLI uses a space between flag and value). +Worker DONE and host process exit0; no remaining observer session. Bounded +metadata retained, no payload values or credentials. No extra TS subscription. + +## Build lock audit + +No `uv sync` found in tracked DL Dockerfiles, workflow YAML or shell build paths. +- Python Dockerfile14-20 copies pyproject.toml + poetry.lock and uses + Poetry2.3.4 install --only main. Dockerfile25-29 runs + verify_runtime_dependencies.py: locked main versions and installed RECORD + hashes; reused dependency images must be digest-pinned. This is not the same + unlocked-uv application-resolution bug. +- Rust runtime/core Dockerfiles use cargo build --locked and Cargo.lock. +- Residual reproducibility risk DOES exist: Dockerfile20 upgrades + setuptools>=78.1.1 outside the main lock; Poetry's bootstrap dependencies and + apt-get upgrade/install resolve against current indexes. The lock verifier + only checks locked main entries and RECORD integrity, not a complete immutable + OS/toolchain closure or wheel-archive hashes. Do not promise byte-identical + builds from application SHA alone. Consolidated-build guide already records + this limitation; fix requires reviewed dependency/toolchain pinning, not adding + a meaningless uv flag or silently reusing an unverified mutable image. +No dependency/image change made in this audit. Before final build, freeze builder +and runtime package inventory/artifacts and record resulting immutable digests; +any changed dependency requires affected tests and packaged verification. + +## Verification and cleanup + +Export validated by actual isolated_config without generating material. +Host test attempt:36 discovered,2 import errors (missing pandas/redis); not PASS. +Same source in existing image77682eea, --rm/networknone/read-onlysource, +512MiB/.5CPU/pids128/tmpfs64MiB: +`python -B -m unittest tests.test_runtime_dependency_image + tests.test_kn_dependency_boundary tests.test_p184_dr05_dl_fault`: +38PASS0FAIL0ERROR0SKIP in62.692s. Includes offline material/key tests only, +not a new Kafka/TS acceptance. Test container qdl-g4-handoff-unit-20261009 absent +after exit. No builds/volumes/image creation; active/rollback retained, no prune. +Canonical DL /home/bobby/data_layer main5bf5cd5, publishedv2.2.2 per ledger; +feature fix/p184-dl-closure. v2.2.3 HOLD; no release or runtime certification. From 1207a1a10d8d130815134601fbeac876f20da649 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 09:31:34 +0000 Subject: [PATCH 050/147] Diagnose isolated Query HTTP surface and measure OKX TRADE G4 --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 30 ++++++ .../p184-dr05-http404-g4-trade3-20261009.md | 93 +++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 upgrade/evidence/p184-dr05-http404-g4-trade3-20261009.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index c8779d5f..ccb14bde 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70894,3 +70894,33 @@ Host test36with2missing-dependency errors; existing-image final38PASS0ERROR0SKIP Scoped --rm test removed, no persistent stack/build/volume created or broadprune. Report records immutable summary/capture/script hashes. Runtime roles unchanged by this turn; no push/merge/release. G4 fullscope/DR05 acceptance remain open. + +### DR05 r8 HTTP404 and OKX three-symbol TRADE G4 - 2026-10-09 + +Owner requests exact failing Query route from fresh isolated run and concurrent +BNB/DOGE/SOL OKX TRADE canonical observation. Status IN_PROGRESS. Existing DR05 +Query runner installs only internal read-view routes; inspect against TS SDK +HTTP requirements before attributing image failure. New isolated init/up/readiness +may be used, no shared config changes; cleanup exact namespace and private keys. +G4 same bounded assigned read_committed observer120s/10000rows/16MiB, no join or +commit, no TS quota/stream. Announce UTC window; do not claim paired TS evidence +without matching same-window offset trace. No dependency/image build or rollout. + +### DR05 r8 HTTP404 reproduction and OKX TRADE G4 result - 2026-10-09 + +Report upgrade/evidence/p184-dr05-http404-g4-trade3-20261009.md. +Fresh isolated Query actualHTTPS mTLS8/8routes404: instruments/snapshot/warmup/ +status for both ETHbindings. OpenAPI paths empty. Query logs only startup because +access_log=False; actual URL/status captured from requests, not invented log. +Root cause DL harness serve_query uses bareFastAPI+internalreadview only, missing +public production router/auth wiring required by TS SDK. No SDK blame, no +productionQuery defect inferred. Fix remains implementation next; r8 rerun alone +cannot pass. No code/auth weakening or shared rollout performed. +G4 OKX BNB/DOGE/SOL same09:26:17-09:28:17UTCwindow,227samples; per-symbol n30/94/103, +p95380.293/146.769/174.404ms,max391.672/412.521/386.183ms. Read_committed visibility +upper bounds; matching Claude TSprobe NOT_VERIFIED. No5-11s tail in this sample, +no quiet/LAST or runtimeFULL claim. Hashes/limits in report. No complete-suite rerun. +Cleanup exact isolated5roles and credentials; label inventoryempty. Observerexit0. +No build/image/volume created; broadprune deferred unrelatedownedcache. Diskafter +113936941056bytes; no pre/post freed claim. CanonicalDL main5bf5cd5/stablev2.2.2 +per ledger, featurefix/p184-dl-closure; runtime unchanged by turn, v2.2.3HOLD. diff --git a/upgrade/evidence/p184-dr05-http404-g4-trade3-20261009.md b/upgrade/evidence/p184-dr05-http404-g4-trade3-20261009.md new file mode 100644 index 00000000..2488f892 --- /dev/null +++ b/upgrade/evidence/p184-dr05-http404-g4-trade3-20261009.md @@ -0,0 +1,93 @@ +# DR05 r8 HTTP404 and OKX TRADE G4 follow-up + +Status: ROOT_CAUSE_REPRODUCED / G4_BOUNDED_MEASUREMENT, not DR05 acceptance. + +## DR05 404 belongs to the isolated DL harness + +A fresh namespace p184-dr05-dl-77abcda572244bbc was initialised and started using +current scripts and existing immutable images. Query image is rc2-43bbcf0; +/app is the read-only SOURCE mount used by this harness, not an assertion that +its entire route assembly is baked into that image. No TS adapter or shared +production service was restarted. Core binary is base image, no fence overlay: +this is an HTTP-route reproduction, not a GAP/generation retest. + +Actual Query startup log: server started, application startup complete, serving +https://127.0.0.1:8200. Access logging is disabled in serve_query, so the log has +NO request URLs; do not pretend r8's individual paths were read from that log. +Fresh mTLS HTTPS requests reproduced8/8 HTTP404 with body {"detail":"Not Found"}: + +| Venue | instrument_uid | +|---|---| +| BINANCE | ee93fabf-68df-5b50-8924-51bf25a5a757 | +| OKX | e49b54ae-c23d-5351-9e64-47934aac28f8 | + +For EACH UID the routes were: +- GET /v2/instruments/{uid} +- GET /v2/market-data/{uid}/snapshot +- GET /v2/market-data/{uid}/warmup +- GET /v2/feeds/{uid}/status + +/openapi.json returns200 with paths={} (internal routes intentionally hidden). +These diagnostic requests supplied mTLS, NOT JWT or full market query params; +this establishes absent HTTP route, NOT an auth/data-read acceptance. Source +confirms the cause independent of request validation: p184_dr05_dl_fault.py +launches p184_dr05_observer.diagnostic_query; that calls +p184_dr08_isolated.serve_query (line864). It creates bare FastAPI then ONLY +install_kn_read_view, without public qdl.api_v2.router.create_v2_app wiring. +SDK qdl_sdk/transport.py lines62/123/145/179 requests these public paths. +This is NOT evidence SDK2.0.7 has the wrong URL or production Query is missing +routes. Earlier gRPC-only DR05 checks exercised internal read-view successfully +and therefore missed the HTTP surface required by the real TS writer clone. + +Next DL implementation: construct isolated public Query through the existing +production app/router and real auth/manifest/quota components, using fresh packet +identities. Retain internal read-view beside it. Do not redirect TS to internal +secret endpoints, disable auth, or fabricate HTTP response fixtures. Add tests +for genuine SDK instrument + snapshot/warmup/status, positive control and wrong +JWT/realm/consumer denials BEFORE asking Claude to rerun the full adapter. +Current code is unchanged: rerunning r8 alone will still404. Isolated fix is DL +ownership; no shared Query rollout required merely to fix this harness. + +Evidence root /home/bobby/.local/state/qdl-v2/dr05-http404-20261009: +- http-paths.json: SHA256 `b88e080cb1b16f2a29fa3ec10493db64c40336cee13d0071fdb9bb593fece2c0`. +- query.log: SHA256 `6328584aa2d3a3720d967d1a5fee0733b27beabd2081249f0080be331d7b517c`. + +## G4 OKX BNB/DOGE/SOL, one shared UTC window + +Window2026-10-09T09:26:17.028434Z to09:28:16.986444Z;120s;227 observations. +Window announced in chat when started. Matching Claude probe has NOT been +verified, so do not call this a paired TS result without its timestamp/offsets. +Six canonical partitions assigned at current end, read_committed, no group join, +auto-store/commit disabled. Sampling at most1event/product/100ms. No full-event +coverage or quiet-market inference from low sample count. Exact topic/partition/ +offset/generation retained for joining. Units ms, nearest-rank quantiles. + +| OKX TRADE | n | source -> visible p50 | p95 | max | p99 | ingest -> visible p95 | +|---|---:|---:|---:|---:|---:|---:| +| BNB-USDT-SWAP | 30 | 93.214 | 380.293 | 391.672 | N/A (n<100) | 283.621 | +| DOGE-USDT-SWAP | 94 | 93.904 | 146.769 | 412.521 | N/A (n<100) | 117.891 | +| SOL-USDT-SWAP | 103 | 98.290 | 174.404 | 386.183 | 370.930 | 143.001 | + +No decode/Kafka errors; no missing/negative paired timestamps. Source->visible +is event timestamp to this observer's read_committed receipt: an upper bound on +visibility including observer scheduling/fetch, NOT exact broker commit time. +No5-11s tail in these227 samples. This does not prove LAST price is always fresh, +or explain a different-window TS tail. No Redis ACK or consumer eligibility +claim. DL/TS clock sync was not independently recalibrated in this window. +- `summary.json` SHA256 `9b6fdb61508734e2256a3cbc5d89ae066a457c368233e45dfab4c89f8ba3132f`. +- `observer.py` SHA256 `873fa1f04f4279d74d0a1bfcd75cbdca6882bebc27ad3a48f4d5f3467124e111`. +- `capture/observations.jsonl` SHA256 `9a8836ddb92b40086fa91cfd23ea1f3b119f29c32b80177e09b2c29b4af676a5`. +- `capture/receipt.json` SHA256 `de91027f3135067635453143849c41b05be5250a40df742ad3a34d7627a22f63`. + +## Cleanup and status + +DR05 down removed exactly query/stream/core/quota/kafka, erased packet private +material; label-scoped docker ps-a returned no container. Ephemeral tmpfs only, +no image build or persistent volume. Canonical observer DONE/exit0; no extra TS +stream. Retained bounded public export, route receipt, startup log and timing +metadata. No broad prune. Disk after113936941056bytes free; no before sample, +therefore no reclaimed-space attribution. Docker89images/69containers, +BuildKit241entries39active/3.541GB reclaimable, not owned by this measurement. +Shared runtime/config/rollback unchanged by this turn. Stablev2.2.2 per ledger; +feature fix/p184-dl-closure, v2.2.3 HOLD. No push/merge. No application test suite +rerun for this diagnosis;8 HTTP probes above are not data acceptance tests. From 7487e3c9e872526083ce40bfb64b83b6415a8453 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 10:30:29 +0000 Subject: [PATCH 051/147] fix(dr05): mount authenticated public Query routes in isolated harness --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 24 +++++ scripts/p184_dr05_observer.py | 35 ++++++- scripts/p184_dr08_isolated.py | 5 +- tests/test_p184_dr05_public_query.py | 96 +++++++++++++++++++ .../p184-dr05-public-router-20261009.md | 65 +++++++++++++ 5 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 tests/test_p184_dr05_public_query.py create mode 100644 upgrade/evidence/p184-dr05-public-router-20261009.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index ccb14bde..d7e95a97 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70924,3 +70924,27 @@ Cleanup exact isolated5roles and credentials; label inventoryempty. Observerexit No build/image/volume created; broadprune deferred unrelatedownedcache. Diskafter 113936941056bytes; no pre/post freed claim. CanonicalDL main5bf5cd5/stablev2.2.2 per ledger, featurefix/p184-dl-closure; runtime unchanged by turn, v2.2.3HOLD. + +### DR05 public router and TS MARK quota - 2026-10-09 + +Status IN_PROGRESS. Owner approved connecting public router in isolated Query and +confirming ten additional TS MARK streams within stage100. Reuse production +create_v2_app, packet JWT key/realm/subject pins, manifests, bounded requests and +shared isolated Redis quota; keep mTLS/internal read-view and DR08 default intact. +Tests: public route presence, authorized reads, realm/revision/identity/feed +negatives and quota failure. No shared runtime or manifest activation. Rollback +is reverting isolated harness source only; dispose owned test resources. Quota +is workload arithmetic, not measured100-alpha capacity. Guide: +upgrade/evidence/p184-capacity100-candidate.md. No push/merge/image build. + +### DR05 public router result - 2026-10-09 + +Status IMPLEMENTED_ISOLATED_HTTP_TESTED. Guide/evidence: +upgrade/evidence/p184-dr05-public-router-20261009.md. Production public router +now mounted in DR05 with real isolated JWT/manifest/Redis quota and mTLS; DR08 +default unchanged. Existing-image scoped suite62PASS0fail0skip; includes next +quota slice. Actual SDK transport HTTPS8/8 reads and4/4negative401. Test-only +canonical data; not TS fault acceptance. Initial probe fixture gap retained and +corrected, not reader policy. Exact5roles/keys cleaned; no build/sharedmutation. +Diskafter111828414464bytesfree; unrelated cache retained for concurrentwork. +Canonicalmain5bf5cd5/stablev2.2.2, runtime unchanged, v2.2.3HOLD. No push/merge. diff --git a/scripts/p184_dr05_observer.py b/scripts/p184_dr05_observer.py index d2301f44..6826c6aa 100644 --- a/scripts/p184_dr05_observer.py +++ b/scripts/p184_dr05_observer.py @@ -228,6 +228,39 @@ def ts_packet(work, runner, packet, binding, rows, base=None): return json.loads(json.dumps(ts_input, default=lambda value: value.value)) +def public_query_app(work, service, issuer): + """Use the production HTTP boundary with this disposable packet's authority.""" + from qdl.api_v2.router import create_v2_app + from qdl.consumer import ConsumerManifestLoader, ConsumerManifestRegistry + from qdl.runtime.bounds import RequestBounds + from qdl.security.data_plane import ( + DataPlaneIdentityService, DataPlaneSecurityConfig, RedisMinuteQuota, + ) + from scripts.p184_dr05_dl_fault import namespace + + packet = json.loads((work / "packet.json").read_text()) + ns = namespace(packet["namespace"]) + if packet.get("provenance") != "TEST_ONLY": + raise ValueError("isolated public Query requires TEST_ONLY packet") + paths = packet["isolated_manifest_paths"] + if paths != ["/work/binance.yaml", "/work/okx.yaml"]: + raise ValueError("unexpected isolated manifest paths") + manifests = ConsumerManifestRegistry(tuple( + ConsumerManifestLoader.load(work / Path(path).name) for path in paths)) + raw = json.loads((work / "jwt-config.json").read_text()) + security = DataPlaneSecurityConfig( + environment="paper", issuer=raw["issuer"], audience=raw["audience"], + algorithms=tuple(raw["algorithms"].split(",")), keys_by_id=raw["keys"], + subjects_by_key_id=raw["subjects"], environments_by_key_id=raw["environments"], + max_token_lifetime_seconds=raw["max_lifetime_seconds"]) + quota = RedisMinuteQuota.from_url("redis://localhost:6379/0", + prefix="qdl:stable:v2:paper:" + ns) + identity = DataPlaneIdentityService(security, manifests, quota=quota) + return create_v2_app(service, identity_service=identity, cursor_issuer=issuer, + request_bounds=RequestBounds(max_concurrent_requests=16), + contract_version="2.0.0", authority="INTERNAL_STABLE") + + def diagnostic_query(work): # Capture the REAL Query handoff from the same RPC, without changing it. from qdl.runtime.kn_query_backend import KnCursorV3Issuer @@ -243,7 +276,7 @@ def bind(self, requirement, history, *, consumer_id): "view": item}) + "\n") return bound KnCursorV3Issuer.bind_history = bind - serve_query(work) + serve_query(work, public_app_factory=public_query_app) def main(): diff --git a/scripts/p184_dr08_isolated.py b/scripts/p184_dr08_isolated.py index 293fea2b..d228e222 100644 --- a/scripts/p184_dr08_isolated.py +++ b/scripts/p184_dr08_isolated.py @@ -861,7 +861,7 @@ def prepare_topics(work): (work / "packet.json").write_text(json.dumps(packet)) -def serve_query(work): +def serve_query(work, *, public_app_factory=None): """Real KN Query read-view stack, backed by the isolated canonical hot view.""" import ssl import uvicorn @@ -891,7 +891,8 @@ def serve_query(work): issuer = KnCursorV3Issuer(settings, catalog) service, _, _ = build_stable_query_stack(spool=None, catalog=catalog, schema_digest="a" * 64, handoff=None, cursor_ttl_seconds=3600, backend=backend, issuer=issuer) - app = FastAPI() + app = (public_app_factory(work, service, issuer) + if public_app_factory is not None else FastAPI()) install_kn_read_view(app, service=service, backend=backend, issuer=issuer, secret=bytes.fromhex((work / "read-view.secret").read_text())) try: diff --git a/tests/test_p184_dr05_public_query.py b/tests/test_p184_dr05_public_query.py new file mode 100644 index 00000000..91914688 --- /dev/null +++ b/tests/test_p184_dr05_public_query.py @@ -0,0 +1,96 @@ +"""Isolated public HTTP wiring: real JWT/manifest, bounded fake quota only.""" +import asyncio +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient +from qdl.query import InstrumentQuery, MemoryMarketDataBackend, V2QueryService +from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.security.data_plane import InMemoryMinuteQuota +from qdl_sdk.credentials import RotatingJwtCredentialProvider +from scripts import p184_dr05_dl_fault as dr +from scripts.p184_dr05_observer import public_query_app +from tests import test_p184_dr05_dl_fault as fixtures + + +class PublicQueryTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.work = Path(self.tmp.name) / "work" + dr.prepare(self.work) + dr.material(self.work, fixtures.MaterialTests.exported()) + catalog = StableSourceCatalog.load(self.work / "catalog.yaml") + service = V2QueryService(instruments=InstrumentQuery(catalog.instrument_registry()), + backend=MemoryMarketDataBackend(), entitlements=catalog.entitlements()) + self.quota = InMemoryMinuteQuota() + with patch("qdl.security.data_plane.RedisMinuteQuota.from_url", return_value=self.quota) as factory: + self.app = public_query_app(self.work, service, None) + ns = json.loads((self.work / "packet.json").read_text())["namespace"] + factory.assert_called_once_with("redis://localhost:6379/0", prefix="qdl:stable:v2:paper:" + ns) + self.client = TestClient(self.app) + self.addCleanup(self.client.close) + self.consumers = {v: json.loads((self.work / ("ts-consumer." + v + ".json")).read_text()) + for v in ("binance", "okx")} + self.uids = {b["venue"].lower(): b["instrument_uid"] + for b in fixtures.MaterialTests.exported()["core"]["bindings"]} + + def headers(self, venue, **overrides): + c = self.consumers[venue] + settings = dict(private_key_file=self.work / (venue + ".key"), key_id=c["jwt_key_id"], + algorithm="RS256", issuer=c["issuer"], audience=c["audience"], subject=c["subject"], + environment=c["environment"], consumer_manifest_revision=c["manifest_revision"], + roles=("market_data_reader", "historical_reader", "stream_consumer"), + lifetime_seconds=300, refresh_before_seconds=30) + settings.update(overrides) + token = asyncio.run(RotatingJwtCredentialProvider(**settings).get_token()) + return {"Authorization": "Bearer " + token, "X-QDL-Consumer-ID": c["consumer_id"], + "X-QDL-Purpose": "INTERNAL_EXECUTION"} + + def test_public_routes_present_and_metadata_authorized(self): + paths = self.client.get("/openapi.json").json()["paths"] + for route in ("/v2/instruments/{identity}", + "/v2/market-data/{instrument_uid}/snapshot", + "/v2/market-data/{instrument_uid}/warmup", + "/v2/feeds/{instrument_uid}/status"): + self.assertIn(route, paths) + for venue, uid in self.uids.items(): + response = self.client.get(f"/v2/instruments/{uid}", headers=self.headers(venue)) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(response.json()["instrument_uid"], uid) + + def test_auth_negatives(self): + for venue, uid in self.uids.items(): + url = f"/v2/instruments/{uid}" + self.assertEqual(self.client.get(url).status_code, 401) + for override in ({"environment": "live"}, {"consumer_manifest_revision": 999}): + self.assertEqual(self.client.get(url, headers=self.headers(venue, **override)).status_code, 401) + other = "okx" if venue == "binance" else "binance" + headers = self.headers(venue) | {"X-QDL-Consumer-ID": self.consumers[other]["consumer_id"]} + self.assertEqual(self.client.get(url, headers=headers).status_code, 403) + + def test_no_feed_global_grant(self): + for venue, uid in self.uids.items(): + response = self.client.get(f"/v2/market-data/{uid}/snapshot", + params={"feed": "QUOTE", "consumer_grade": "EXECUTION", "source_policy_id": "ungranted"}, + headers=self.headers(venue)) + self.assertEqual(response.status_code, 403, response.text) + + def test_quota_fail_closed(self): + from qdl.security.data_plane import DataPlaneAccessError + with patch.object(self.quota, "consume", side_effect=DataPlaneAccessError( + "DEPENDENCY_UNAVAILABLE", "isolated quota unavailable", status_code=503)): + venue = "binance" + response = self.client.get(f"/v2/instruments/{self.uids[venue]}", headers=self.headers(venue)) + self.assertEqual(response.status_code, 503, response.text) + + def test_nonisolated_packet_rejected(self): + path = self.work / "packet.json" + packet = json.loads(path.read_text()) + packet["provenance"] = "PRODUCTION" + path.write_text(json.dumps(packet)) + with self.assertRaisesRegex(ValueError, "TEST_ONLY"): + public_query_app(self.work, None, None) diff --git a/upgrade/evidence/p184-dr05-public-router-20261009.md b/upgrade/evidence/p184-dr05-public-router-20261009.md new file mode 100644 index 00000000..b8b2e8da --- /dev/null +++ b/upgrade/evidence/p184-dr05-public-router-20261009.md @@ -0,0 +1,65 @@ +# DR05 Public Query Router Handoff + +Status: IMPLEMENTED / ISOLATED_HTTP_TESTED, not DR05 certified. +Scope: owner request 2026-10-09, public router in the disposable Query harness. +No shared role, credential, quota, manifest or application source changed. + +## Change + +`scripts/p184_dr05_observer.py::public_query_app` uses production +`create_v2_app` with packet JWT keys pinned to subject/realm, both isolated +manifests and RedisMinuteQuota. HTTP and Rust Stream share the exact namespace +quota key prefix and Redis DB0. Query remains mTLS-only, loopback8200, with +bounded16 concurrent requests. Internal read-view remains installed, unchanged. +`scripts/p184_dr08_isolated.py::serve_query` accepts an optional factory; DR08 +without it retains its original app. No duplicated endpoint or permissive auth. + +## Tests And Actual HTTP + +Existing image77682eea (rc2-43bbcf0), read-only source, no build: +`python -B -m unittest tests.test_p184_dr05_public_query + tests.test_p184_capacity100 tests.test_p184_dr05_dl_fault + tests.test_p184_dr05_observer`: **62 PASS, 0 fail/error/skip**,80.643s. +Includes the quota arithmetic regression committed in the next slice. +Public tests check route presence, valid signed instrument reads, missing token, +wrong realm/revision/consumer, ungranted feed, quota dependency failure and +non-test packet rejection. Earlier fixture failures (wrong InstrumentRecord +attribute; URL placeholder identity vs instrument_uid) were fixed, not hidden. + +Fresh namespace `p184-dr05-dl-a8d47c9c951a4e11`: existing Kafka/core/Stream/Query/ +Redis images; core658a9570, Query77682eea plus source-mounted harness. +`--init`, `--up`, then SDK RestQueryTransport over realHTTPS+mTLS: +- instrument/snapshot/warmup/feed_status x Binance/OKX: **8/8 success**; +- wrong realm/revision x both venues: **4/4 HTTP401**. +Synthetic provider frames carry TEST_ONLY; core emits canonical, Stream hot view +serves it to Query. This is NOT a real-provider or TS writer/validator receipt. +First probe incorrectly repeated Binance snapshots with discontinuous deltas: +core rejected gaps as designed (3 read failures). Corrected the probe to one +anchor and contiguous deltas in a fresh generation; kept failed receipt/script. +No reader correctness or freshness gate was relaxed. + +External evidence: `~/.local/state/qdl-v2/dr05-public-router-20261009/`: +- `work/public-http-receipt.json`: SHA256 + `84cbb14608b471c559126eb0383dd65f9f5b14acf545d1636df3f88ab9f76126`. +- `probe.py`: `129444eec51327f13c0e56d1decc424e916639b5afc1b8ad307aa9401f0d82fd`. +- `receipt-attempt1.json`: `8c1ec9f197ca1f4e3cb7b5302a0012ed97ae929649c87c2021d9670f35af1990`. +- `probe-attempt1.py`: `f9226c8b2105afb9b19e4eb3a2bb13c13a64ded9bedd073a8b1512ccd93c0e15`. + +## Claude Next Step + +Use unchanged DR05 init/up/reset/channel protocol. Each newly started Query now +exposes SDK public routes. Keep the previously required GAP/generation candidate +core binary for final fault cases: this HTTP check used the base core and does +NOT replace that patch. TS writer-clone/validator final rows remain Claude-owned; +`drive()` still cannot issue acceptance. Use fresh material because private test +keys of this run have been erased. Ports6391/5433 remain reserved for TS clones. + +## Cleanup And Runtime + +`--down` removed exact query/stream/core/quota/kafka; private keys erased; exact +label inventory empty. Unit container --rm removed. No image built, no volume +created. Existing production/rollback images retained; unrelated BuildKit cache +not pruned while Claude works. Inventory89images/30.82GB, cache23.3GB; disk after +111828414464bytes free (no measured pre/post reclaimed-byte claim). +Canonical `/home/bobby/data_layer` main5bf5cd5, stablev2.2.2; active feature +`fix/p184-dl-closure`, v2.2.3HOLD. Shared runtime untouched. No push/merge. From d4e8d11be3129c4246d005fd0cba6d9035e96197 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 10:30:56 +0000 Subject: [PATCH 052/147] docs(capacity): account for ten TS MARK streams in stage100 quota --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 12 +++++ tests/test_p184_capacity100.py | 13 +++++ .../evidence/p184-capacity100-candidate.md | 52 +++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index d7e95a97..373529b3 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70948,3 +70948,15 @@ canonical data; not TS fault acceptance. Initial probe fixture gap retained and corrected, not reader policy. Exact5roles/keys cleaned; no build/sharedmutation. Diskafter111828414464bytesfree; unrelated cache retained for concurrentwork. Canonicalmain5bf5cd5/stablev2.2.2, runtime unchanged, v2.2.3HOLD. No push/merge. + +### TS ten MARK stream quota confirmation - 2026-10-09 + +Status QUOTA_DESIGN_CONFIRMED_RUNTIME_NOT_ACTIVATED. Appended amendment in +upgrade/evidence/p184-capacity100-candidate.md. Actual deployed bundle TSrev10: +50streams/1500RPM; both Stream384+32replay, restart0/OOMfalse. Ten extraMARK +requires60active,72reservedTS slots. Stage100realistic240/288 andstress560/672 +(active/reserved), tested with existingplanner. Current384doesnotfitstress +failover.1024modeledplanfits but no capacitycertification. Inverseadditional +subscriptions must be counted separately. No quota/buffer/manifest/runtimechange. +Scoped62test final suite includes thisregression; no further build or resources. +No push/merge; v2.2.3HOLD. Runtime/cleanup as precedingDR05entry. diff --git a/tests/test_p184_capacity100.py b/tests/test_p184_capacity100.py index 6612af31..7871ebd2 100644 --- a/tests/test_p184_capacity100.py +++ b/tests/test_p184_capacity100.py @@ -86,6 +86,19 @@ def test_stress500_is_active_and_keeps_exactly_five_products(self): self.assertEqual([d.streams_needed for d in plan.demands], [300, 300]) self.assertEqual([d.quota_needed for d in plan.demands], [4500, 4500]) + def test_ten_mark_streams_need_seventy_two_ts_slots(self): + for profile, total, reserved in (("realistic", 240, 288), ("stress5", 560, 672)): + _, report = self.plan(profile=profile, ts=TSReservation("ts", 60, 60, 1000), + gateway_subscription_limit=1024) + self.assertEqual(report["total_streams"], total) + self.assertEqual(report["reserved_streams"], reserved) + ts = next(row for row in report["identities"] if row["role"] == "TS") + self.assertEqual(ts["candidate_streams"], 72) + self.assertEqual(ts["stream_shortfall"], 22) + self.assertEqual(ts["candidate_rpm"], 1500) + self.assertTrue(report["gateway_slots_fit"]) + self.assertFalse(report["sealed_identity_quotas_fit"]) + def test_sdk_snapshot_plus_subscribe_handoff_is_reserved(self): _, report = self.plan(profile="stress5") alpha = [r for r in report["identities"] if r["role"] == "ALPHA"] diff --git a/upgrade/evidence/p184-capacity100-candidate.md b/upgrade/evidence/p184-capacity100-candidate.md index 10d65173..1f1d1bd2 100644 --- a/upgrade/evidence/p184-capacity100-candidate.md +++ b/upgrade/evidence/p184-capacity100-candidate.md @@ -140,3 +140,55 @@ readback and NOT traffic. Docker--rm/networknone/readonly removed after success. Before further load/build: root filesystem observed100%,229122048bytes free. Restore measured disk headroom with an ownership-safe cleanup packet; no broad prune while Claude may be building. No test or runtime data deleted to mask this. + +## TS Ten MARK Streams Amendment - 2026-10-09 + +Status QUOTA_DESIGN_CONFIRMED / RUNTIME_NOT_ACTIVATED / CAPACITY_NOT_CERTIFIED. +This supersedes the50-stream assumption above for the proposed linear TS60 +stream workload; it does not alter historical measurements. + +Read-only runtime evidence: both deployed Stream replicas use image +`1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9`, +`QDL_KN_MAX_SUBSCRIPTIONS=384`, `QDL_KN_MAX_REPLAY_RPCS=32`, restart0/OOMfalse. +Read `/consumer-realms/bundle.json` inside active Stream: TS paper revision10, +max_streams50, RPM1500, max_buffer_events2000, max_batch_items50, max_warmup_rows2000. +Source confirms `consumers/stable/trading-system-paper.yaml:23`/`:26`. +Therefore **ten additional concurrent MARK streams are NOT admitted by the +current TS quota**. Existing route entitlement is not an extra stream slot. + +| Quantity | Realistic100 | Stress5 x100 | +| --- | ---: | ---: | +| Alpha active streams | 180 | 500 | +| TS active streams, including ten MARK | 60 | 60 | +| Total active | 240 | 560 | +| Alpha reserved,1.2x | 216 | 600 | +| TS reserved,ceil(60x1.2) | 72 | 72 | +| Total reserved | 288 | 672 | + +The existing planner with TSReservation(ts,60,60,1000) confirms both profiles; +new regression `test_ten_mark_streams_need_seventy_two_ts_slots` checks total, +reserved, quota shortfall22 and unchanged1500RPM candidate. The22 deficit is +72reserved minus50configured; immediate active deficit is10. +RPM1500 remains conditional on1000steadyrequests/minute including recovery and +probes. Do NOT subtract ten polling lanes without measuring actual caller changes. +The planner accounts startup/reconnect calls; quota is not latency acceptance. + +Candidate request: TS max_streams72, no per-client buffer increase. For full +stress100, reserve672slots on **one** replica so peer loss fits; current384fails +by288reserved slots (176active). Existing modeled1024-slot plan fits arithmetically +with352reserved slots left, plus separately bounded32Replay. Its307MiB modeled +RAM is not a measurement. No CPU/RAM or quota activation was performed. +Alpha current2400RPM/60slots each still falls short of the stage100 candidates +4506/4509RPM+108slots(realistic),4950RPM+300slots(stress). + +Pending IDR075 inverse products are not silently included in the linear60. +If all four additional products become simultaneous streams, TS64 needs77 +reserved slots; stress total becomes564active/677reserved. Freeze the actual +caller subscription roster before rendering the coordinated manifest/bundle +revision and JWT/consumer config. Do not spend recovery headroom on uncounted +feeds or open a second probe on the shared TS identity. + +Decision boundary: owner/Claude can use72 as the TS60 quota requirement in the +next reviewed packet. Current deployment cannot accept the extra10 unchanged. +Stage100 remains unmeasured, not blocked on Paper implementation. Approval to +plan100alphas is not evidence that the existing384slot runtime can serve them. From 86dd0aeed5f72176557c26f478634f72ae5b7a5c Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 13:47:57 +0000 Subject: [PATCH 053/147] docs(p184): freeze DL phase D closure work order and approval boundaries --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 165 ++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 373529b3..3e5af51c 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -70960,3 +70960,168 @@ failover.1024modeledplanfits but no capacitycertification. Inverseadditional subscriptions must be counted separately. No quota/buffer/manifest/runtimechange. Scoped62test final suite includes thisregression; no further build or resources. No push/merge; v2.2.3HOLD. Runtime/cleanup as precedingDR05entry. + +### Phase D DL closure plan - 2026-10-09 + +Status: PLANNED_PENDING_OWNER_GO_AHEAD. Documentation only, not implementation +or certification of D1-D6. Accept all six items of Claude's delegated prompt. +Its section0 explicitly requires this plan commit before further runtime action. +Previous rollout approvals do not authorize this new packet. + +#### Inputs and authority + +- DL working source d4e8d11, fix/p184-dl-closure, clean before this entry. +- TS source inspected read-only aca225231b2bed91e86abcae879ea90d665c9167. +- TS upgrade/p184-phase-d-codex-dl-prompt.md SHA256 + 8df6bf886bf570833b0c51fcd2d32fe2fd635ce59a0c23b3f43ca199773c6bba. +- TS upgrade/p184-phase-d-gates.md SHA256 + a2e10504531421068475d4960f14aee860eca4c81d49e4a987b1a8f033f31d4a. +- TS upgrade/p184-phase-d-mark-quota-handoff.md SHA256 + 6b53e54902689f5d12e1885196af2af6d94cf96eec2e3efea1ab0b7a002448a7. +- Read workspace/project rules, CLAUDE.md sections2/3/3b/4, ledger, this journal + from100-alpha amendment, TS phase-D slices/runtime incident and IDR002/003/ + 028/029/039/040/062/075/077. Existing detailed guides under upgrade/evidence/: + p184-capacity100-candidate.md, p184-consolidated-build.md, + p184-dl-phase-c-packet.md, p184-dr-runtime-runbook.md, + p184-resilience-closure.md, p184-dr05-fence-channel-handoff.md, + p184-dr05-public-router-20261009.md, p184-dr05-g4-build-handoff-20261009.md, + p184-dl-release-readiness.md. Historical observations are not current runtime. + +Frozen G1-G14 remain unchanged. G4 all-feed baseline/comparison and G5 missing +shape-specific numeric limits are unresolved inputs, not permission to invent +thresholds after results. Owner records those inputs before load; an actual gate +change needs a reasoned owner amendment. Historical5-vCPU and core-first text in +PhaseC is superseded by governing amendments. Preserve all failed evidence. + +#### Order and ownership + +Order: D1 packet -> D2 attribution -> D3 runnable measurement preparation -> D4 +resilience -> D5 final build/qualification/release. D6 reset diagnostics precedes +further fault runs; D6 compatibility decisions precede final source freeze. +These are slices of existing phaseD, not extra phases. Codex owns DL source, +harnesses, packet/journal. Claude exclusively owns TS source/writer/validator, +trace/alpha and broker delta. Never edit TS worktree or wait for Paper features. + +**D1: one Paper rev11 packet for inverse plus MARK wake, no double bump.** +Compile demand from sealed products AND actual bridge paths. Sixty linear slices +already include ten polled MARK slices; current stream count50 is not60. +Adding ten wakes yields60 linear streams, NOT70. TS bridge lines881-884 select +MARK separately. With four inverse QUOTE/MARK/BOOK_SNAPSHOT/BOOK_DELTA streams, +planning demand becomes64 active/ceil(1.2x64)=77 reserved; verify with executable +caller roster before sealing. Realistic100 would be244 active/293 reserved, +stress100564/677. Current384 on one replica cannot cover stress peer loss. Do not +sum two replicas to certify failover; do not increase per-client buffers. + +Audit request costs separately: TS _book_fast_retry_slot spaces attempts at1.5/s +per process, not necessarily charged requests. qdl_sdk/client.py:957-1002 can +snapshot/warmup then Subscribe; security/data_plane.py:395 and Rust auth.rs:554 +both consume quota. Warmup-refused and stream-fault-after-handoff cost different +amounts. Verify real Redis URL/prefix parity, identity/global-vs-replica scope, +fixed-minute boundaries, SDK internal retries, initial bursts, cold resolve, +batch charging, all Paper callers and new wake reads before accepting1000+360 +against1500RPM. That arithmetic is NOT yet a proven worst-case request bound. +Do not double-count replacement traffic. Freeze a measured recovery reserve. + +Deliver immutable rev11 manifest/bundle, r23 V3 binding and policy fingerprint, +old/new hashes and paired JWT11 checklist. Explicitly supersede prior packet's +no-quota-change statement, preserve its artifacts. Future exact scope: Query/ +Stream replicas and affected Paper callers, freshly inventoried digests/mounts; +keep other realms/keys, ingestors/core/projectors, offsets/cache and orders intact. +Quiesce callers during revision transition, verify readers then paired callers. +Rollback readers to manifest10 and callers to r22/JWT10 together, never accept +both revisions silently. Live/sandbox need separate owner decisions based on +actual data identities/subscriptions, not the broker's execution mode. +Owner approves exact quota/reader/caller packet; Claude confirms inverse roster +and callers, then enables MARK wake only after admission/preflight. + +**D2: periodic TRADE/QUOTE attribution, not speculative code.** +Read supplied trade-split.json; join offsets534102073/460611644/534133976 with UTC, +identity/generation. Inspect dispatch/flush/keepalive, cursor refresh/signing, +quota windows, Kafka poll/fetch/rebalance and projector cadence with file:line. +A90.2s coincidence is a hypothesis, not causal proof. Check existing Stream send +fields; aggregate counters are not per-event send time. Coordinate Claude's +bounded rotating SDK/apply/ACK/cursor trace. No new TS stream, Redis MONITOR, +offset commits or broad log dump. A read-only observer packet states duration, +partitions/output bound/stop conditions; missing joins remain explicit. Include +OKX QUOTE tail, separate quiet-state age from new-event transit. Only reproduced +causality permits a proposed source fix/affected regression. Owner approves the +observer window and any later runtime action; Claude supplies matching trace. + +**D3: stage100 driver and measurement freeze before load.** +Finish candidate-only config, quota preflight, client/worker wiring, real TS +clone and active per-stream apply/ACK proof. Do not reinterpret certified stage50. +Freeze mixed/stress rosters including inverse/MARK, rates/fanout, startup/reconnect, +probe overhead, batch symbols/interval/limit, warm/cold state, concurrency/bytes. +Propose G5 limits per shape for owner freeze before results. G4 needs every +feed/venue/interval paired by event/generation/partition/offset; read_committed +observation is a visibility upper bound, not broker commit time. Owner freezes +comparison tolerance after baseline but BEFORE candidate load. Missing/negative/ +skewed samples remain counted. BAR uses close; MARK component age is separate. +Windows: one variable per>=10min comparison; >=30min stage100 steady, declared +burst and faults; ramp20/50/100 with integrity/backlog/pressure stops. Report +n,p50/p95/max, p99 only n>=100, refusals/errors included. G6 typed at-use quality +is separate from G14 per-role CPU/throttle/RSS/cgroup memory/I/O/disk/queue; +measure clients separately on aligned windows. Owner sets resource headroom from +measurements, not a fixed5CPU ceiling. Confirm disk bytes/inodes/growth reserve +before any approved load. Claude provides real receiver and ACK boundary. + +**D4: native recovery adapter and isolated fault acceptance.** +Finish observation adapter for kn_recovery_runner/state: actual pressure, +quorum/ISR, committed offsets/projector catch-up and TS at-use validator. READY +or arbitrary success booleans cannot certify SERVING. Default observe/propose, +typed unknown/fatal/fenced refusal, absolute deadline and bounded attempts; +exact exited-container identity before an approved start. No auto-prune, FLUSH, +offset reset, healthy-role restart or legacy spool recovery on KN. +Use isolated broker/projector/Stream faults and production-shaped4-worker TS +clone. Record outage start/fault cleared/first valid apply/caught-up per slice. +G12 caught-up<=600s then G1-G3 inside; G10 loss/duplicate apply/overflow/replay +expiry rules unchanged. Deliberately faulted slow-client controls separately +labelled, never silently removed. Inherit unchanged ENOSPC/unit evidence. +Owner approves fault/resource packet, then separate shared packet if needed; +Claude supplies clone/validator/ACK. No mocked test becomes runtime acceptance. + +**D5: one-source image set and fresh final receipts.** +Verify effective source plus ancestry for IDR040fix1,bindingV3,quota log,b6a0119, +IDR075+D1,IDR062,d68fb2f,0b374bc and deployed IDR077 fact. A commit list alone +cannot detect a revert. Exact-source CI/preflight before owner-approved build; +hosted workflow/push needs separate permission, local rehearsal is not hosted CI. +Build both images from one immutable archive, no overlays in final certificate. +Rebuild SDK into fresh output; require exact wheel hash +6923b46f46bd07c850b18a6074439bbcb4085d84428a8e72c476437dd51524e7. +Drift HOLD, no overwrite; preserve mutable toolchain/OS risk and package inventory. +Final DR01 joint account cells, DR05/DR07 with Claude, DR08>=5restarts use actual +final DL+TS digests/configs in fresh directories. Final DR/broker delta is not +Paper feature completion. G1-G14/source/images/artifacts must agree before +certificate and owner release decision. Feature->dev CI->main/tag/assets only +when authorized. No premature SDK publication, runtime freeze, push or deploy. + +**D6: narrow diagnostics and explicit compatibility decisions.** +Keep bounded sanitized stderr/return code/stage/deadline for r13 reset failure; +fail typed, immutable output, no blind retry or extending deadline. Regression: +timeout/nonzero/topic error and existing reset isolation checks. Do this before +another fault run, not after losing another diagnostic. +For IDR040fix2 and IDR003 propose tolerant-reader-first rollout of an optional +quality field, or explicit owner deferral with quiet-source refusal limits. +SDK2.1.0 package availability does not mean every deployed reader tolerates it. +Claude verifies TS/alpha/Risk reader versions before DL emits anything new. +No silent schema/semantics change, invented heartbeat timestamp or eligibility. + +#### Safety, cleanup and this slice + +Before a runtime packet: fresh role IDs/digests/config/caps, proposed env checked +against actual parser bounds. Guard RestartCount/OOM/heartbeat and real feed +progress, not just memory. Do not repeat32768 vs4096 trace-cap incident. Reuse +unchanged certified evidence; no broad catalogue/E03/E04 rerun. +After each attempt stop owned observers/watchers, remove owned containers/private +credentials/test artifacts after hashes, preserve immutable receipts and active/ +named rollback images. Record disk pre/post for actual cleanup; never prune +Claude's concurrent work or shared operational state. +This turn changes documentation only: no tests rerun, probes, Docker action, +build, config/manifest change, runtime measurement or TS edit. Claude's writer +0705cce3 and DR05r16 remain attributed reports, not independently recertified. +CanonicalDL main5bf5cd5/v2.2.2 and active closure worktree confirmed by git worktree +list. Runtime digests/config/rollback unchanged by this turn, not re-inspected. +No resources created; Docker/BuildKit inventory/cleanup deferred for this +planning-only slice. v2.2.3 remains HOLD. Commit plan and await owner go-ahead +per prompt section0 before runtime action. apply_patch failed mountinfo helper; +exact append fallback used under rule32, followed by diff verification. From 47607ad4edd65bffca673838c470a3f58455b890 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 14:29:26 +0000 Subject: [PATCH 054/147] docs: align phase D quota and close trade attribution search --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 31 ++++++++++++ .../evidence/p184-capacity100-candidate.md | 47 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 3e5af51c..592c0136 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71125,3 +71125,34 @@ No resources created; Docker/BuildKit inventory/cleanup deferred for this planning-only slice. v2.2.3 remains HOLD. Commit plan and await owner go-ahead per prompt section0 before runtime action. apply_patch failed mountinfo helper; exact append fallback used under rule32, followed by diff verification. + +### Phase D Claude alignment - 2026-10-09 (9ca122a/e292195) + +Status: D1_ROSTER_CONFIRMED_RPM_NOT_FROZEN; D2_TRADE_QUOTE_ATTRIBUTION_CLOSED. +Documentation-only continuation of86dd0ae; no runtime action authorized by this +entry. Scope: incorporate Claude's updated handoff, preserve G1-G14, no TS edits. +Guide: upgrade/evidence/p184-capacity100-candidate.md, final confirmed-demand +section. D1 rev11 now explicitly sizes RPM together with77 reserved slots for64 +active streams. Baseline900/898 units/min (n=2) and fastretry240 units/min are +reported by Claude, not newly measured here. MARK/inverse incremental requests, +SDK reconnect/startup and observer reserve still need numerical inputs;1500RPM +is not accepted by default. Guide records formula and paired rollback boundary. + +D2 supersedes the earlier proposed TRADE/QUOTE periodic-delay investigation. +Read TS phase-D runtime action6: image923de4c448db6d8b1d4c5454f2841a462a791bf1aa5896a4fb8fc69527e88eab, +trace14:07:43-14:17:33Z. Repeated quiet TRADE snapshots with newer watermarks +caused the apparent tails (IDR082), not demonstrated Stream delivery delay. +Claude's stream-event maxima: TRADE847/726ms, QUOTE806/809ms (Binance/OKX). +These are attributed diagnostic-window results, not independent stage100 +acceptance; no sample count invented. Do not reopen DL TRADE/QUOTE search without +independent reproducible evidence. IDR082 APPLIED-versus-REFRESHED semantics +remain TS-owned/open; MARK wake/G2, G4 attribution and D3-D6 remain separate. + +Validation: read latest TS handoff/journal, checked quota arithmetic and SDK +retry boundary; documentation diff checked. No source suite rerun, probe/build, +runtime/manifest/quota mutation, push or merge. CanonicalDL main5bf5cd5/v2.2.2, +active worktree fix/p184-dl-closure; v2.2.3HOLD. Runtime digests/config and named +rollback artifacts unchanged by this slice, not freshly inventoried. No test +resources created; Docker/BuildKit cleanup deferred for documentation-only work. +apply_patch failed with mountinfo helper; exact append fallback under rule32, +followed by diff verification. No historical journal evidence overwritten. diff --git a/upgrade/evidence/p184-capacity100-candidate.md b/upgrade/evidence/p184-capacity100-candidate.md index 1f1d1bd2..d8b2e485 100644 --- a/upgrade/evidence/p184-capacity100-candidate.md +++ b/upgrade/evidence/p184-capacity100-candidate.md @@ -192,3 +192,50 @@ Decision boundary: owner/Claude can use72 as the TS60 quota requirement in the next reviewed packet. Current deployment cannot accept the extra10 unchanged. Stage100 remains unmeasured, not blocked on Paper implementation. Approval to plan100alphas is not evidence that the existing384slot runtime can serve them. + +## Confirmed TS Demand And Joint RPM Decision - 2026-10-09 + +Status: D1_ROSTER_CONFIRMED_RPM_NOT_FROZEN. Supersedes the conditional72-slot +request above, not its historical evidence. Authority: Claude TS commits9ca122a +and e292195, upgrade/p184-phase-d-mark-quota-handoff.md, appended answers to +Codex86dd0ae. No deployment or fresh runtime measurement in this amendment. + +- Only market_data_service_p184w uses trading-system.paper.stable, per Claude's + runtime inventory. Linear50 + MARKwake10 + inverse4 =64 active streams. +- Proposed rev11 max_streams=77 (ceil(64*1.2)); r23 inverse is paired with + MARKET_DATA_OKX_SYMBOLS activation by Claude. Existing50-slot grant is not enough. +- Realistic100:244 active/293 reserved; stress100:564 active/677 reserved. + These are demand arithmetic, not measured capacity. One384-slot replica cannot + carry stress100 during peer loss; no buffer increase is implied. +- Measured pre-wake identity traffic:900/898 quota units per complete minute, + n=2, writer12be5f9. This is a baseline, not a busy-market upper bound. +- Fast BOOK retry now0.5/s/process:4*0.5*2*60=240 units/min. Two units cover + snapshot and successful Subscribe; failed snapshot costs less. This excludes + SDK in-session reconnects, initial subscription/resolve, and MARK/inverse delta. + qdl_sdk/client.py reconnect default5 is per session, not a global minute cap. + +Freeze requests_per_minute and max_streams together in the same rev11 packet. +Use quota units counted by the real shared identity limiter, not event counts: + + projected_peak = 900 + mark_delta + inverse_delta + 240 + + sdk_reconnect_reserve + startup_resolve_reserve + + observer_reserve + proposed_rpm = ceil(projected_peak * (1 + approved_headroom_fraction)) + +All terms are quota units/min. Subtract replaced polling only with paired +evidence; avoid counting it twice. Deduplicate overlapping reconnect/startup +paths when measured, or label their sum a conservative bound. Headroom fraction +is a packet decision, not a new acceptance gate. Do not silently retain1500RPM. + +MARK delta is NOT_MEASURED. Observe canonical MARK wake rate per product and +evaluate actual TS reference batching/coalescing over complete minute windows. +Account for Query plus Subscribe across both replicas and fixed-minute reset +bursts. MARK event count alone does not establish charged requests or compute +capacity. Include inverse reference polls/resolve/wake, not just its four slots. +Use existing read-only observations or an approved bounded observer, never a +second Paper-identity stream. Record measured vs modeled terms separately and +verify the selected quota with the actual caller before calling D1 accepted. + +Exit: numerical RPM with explicit inputs, shared-limiter namespace verified, +old/new bundle hashes, r23/JWT11 paired reader/writer packet, rollback to rev10 +and r22/JWT10. Neither final RPM nor runtime activation is certified here. From 4fc987a052bcce977dbb419d3fd2b3ee058b6121 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 14:53:47 +0000 Subject: [PATCH 055/147] docs: freeze D1 rev11 quota and paired TS checklist --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 27 +++++++ .../evidence/p184-capacity100-candidate.md | 33 ++++++++ upgrade/evidence/p184-dl-phase-c-packet.md | 75 +++++++++++++++++++ 3 files changed, 135 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 592c0136..e2d44dec 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71156,3 +71156,30 @@ rollback artifacts unchanged by this slice, not freshly inventoried. No test resources created; Docker/BuildKit cleanup deferred for documentation-only work. apply_patch failed with mountinfo helper; exact append fallback under rule32, followed by diff verification. No historical journal evidence overwritten. + +### Phase D D1 rev11 packet closure - 2026-10-09 (TS f457e7a) + +Status D1_PACKET_COMPLETE_NOT_APPLIED. Owner requests packet only. Freeze Paper +manifest11 at77 stream slots for64 active,3000RPM; paired TS r23/JWT11 with +MARK_WAKE_MIN_INTERVAL_MS=500. Detailed checklist/rollback and sizing are in +upgrade/evidence/p184-dl-phase-c-packet.md and p184-capacity100-candidate.md. +Scope documentation only; no deployed manifest, credentials, runtime or TS edit. +The old pending-MARK-rate sizing prerequisite is superseded by TS bounded wake. +G1-G14 and final acceptance remain; D2 closed, IDR082 remains TS-owned. + +Reviewed TS f457e7a bridge interval guard/coalescing and its flood/bounds tests +(source review, not a fresh test execution). Proposed load2480units/min leaves +520units/min reserve at3000;900baseline and10-20reconnect/probe allowance are +estimates, not hard workload bounds under arbitrary failures. SDK reconnects +remain outside BOOK limiter; acceptance checks actual charged minute counters. +No choice of2400 without evidence decomposing non-MARK<=300. Guide records +startup/recovery bursts and immutable packet rerender requirements. + +Validation: arithmetic assertions, three-file documentation diff and diff-check. +No source suite/build/probe or runtime action. apply_patch helper failed mountinfo; +exact append fallback under rule32 used and diff reviewed. No push/merge. +Canonical /home/bobby/data_layer main5bf5cd5/v2.2.2; active worktree +/home/bobby/.worktrees/data-layer-p184-closure on fix/p184-dl-closure. +Runtime image/config set and rollback artifacts unchanged, not re-inventoried. +No disposable resources created; Docker/BuildKit cleanup intentionally deferred +for this documentation slice. v2.2.3HOLD; D1 runtime acceptance NOT_RUN. diff --git a/upgrade/evidence/p184-capacity100-candidate.md b/upgrade/evidence/p184-capacity100-candidate.md index d8b2e485..aaf5d6ec 100644 --- a/upgrade/evidence/p184-capacity100-candidate.md +++ b/upgrade/evidence/p184-capacity100-candidate.md @@ -239,3 +239,36 @@ verify the selected quota with the actual caller before calling D1 accepted. Exit: numerical RPM with explicit inputs, shared-limiter namespace verified, old/new bundle hashes, r23/JWT11 paired reader/writer packet, rollback to rev10 and r22/JWT10. Neither final RPM nor runtime activation is certified here. + +## D1 Sizing Frozen For Rev11 Packet - 2026-10-09 + +D1_PACKET_COMPLETE_NOT_APPLIED. Supersedes RPM_NOT_FROZEN above. TS f457e7a +bounds MARK wake with DATA_LAYER_V2_MARK_WAKE_MIN_INTERVAL_MS=500, constructor +range100..1000ms. Pin500 in the paired TS packet; parser accepting100 does not +make a100ms configuration fit this budget. Wake and poll share the read cadence. +No provider event-rate measurement or900-request decomposition is needed to +select the conservative3000RPM option requested by owner. + +| Component | Quota units/min | Evidence class | +| --- | ---: | --- | +| Existing traffic counted conservatively as non-MARK | 900 | Measured total900/898,n2; not a hard future bound | +| Linear MARK | 1200 | 10symbols x2reads/s x60; modeled unbatched ceiling | +| Inverse MARK | 120 | 1symbol x2reads/s x60; modeled unbatched ceiling | +| Fast BOOK retry | 240 | 4processes x0.5/s x2units x60 | +| Incremental probes/reconnects | 20 | Planning allowance, not a storm bound | +| Total planning load | 2480 | Conservative steady/retry scenario, not measured capacity | +| Selected requests_per_minute | 3000 | Packet decision | +| Remaining reserve | 520 | 20.97% above planning load;17.33% of quota | + +max_streams=77 for64 active. The64/77 and stage100244/293 or564/677 arithmetic +is unchanged.3000 replaces1500 in the proposed Paper rev11 only; no other +consumer, buffer, CPU/RAM limit or freshness gate changes. Do not choose2400 +without measured non-MARK<=300 decomposition and a new reviewed budget. + +Review caveat: reference_batch costs one authenticated HTTP call; batching can +reduce charges. SDK resubscriptions, cold resolve, fixed-minute edge and startup +can consume the520 reserve; repeated failures are not bounded by the20 estimate. +Stagger startup and measure real shared Query+Stream minute counters,429 and +reconnect counts in the approved window. Do not disable limits or open a second +Paper probe to force admission. Quota admission is not stage100 capacity proof. +Paired execution/rollback checklist: p184-dl-phase-c-packet.md, D1 rev11 section. diff --git a/upgrade/evidence/p184-dl-phase-c-packet.md b/upgrade/evidence/p184-dl-phase-c-packet.md index 321ec0d6..99a4254e 100644 --- a/upgrade/evidence/p184-dl-phase-c-packet.md +++ b/upgrade/evidence/p184-dl-phase-c-packet.md @@ -398,3 +398,78 @@ See [full receipt](p184-dr07-runtime-closure.md#actual-caller-rollout-and-exact- Prior coherence and rollback conditions remain mandatory; no extra interface, freshness, resource-cap or policy change. Broker packet ownership must be reconciled without making unfinished Paper features a Data Layer release prerequisite. + +## D1 Rev11 Paired Quota Packet - 2026-10-09 + +Status: D1_PACKET_COMPLETE_NOT_APPLIED. Owner explicitly requests no apply. +This governs D1 over the old IDR075 no-quota-change/TS60-only statements above. +It does not authorize any role recreation, release or the failed core candidate. +D2 TRADE/QUOTE attribution is closed; IDR082 remains with TS. + +### Frozen proposed configuration + +| Item | Before | Paired target | +| --- | --- | --- | +| Data identity | trading-system.paper.stable | Unchanged | +| Consumer manifest / JWT revision | 10 / 10 | 11 / 11 | +| Route binding | r22 | r23, sealed V3 | +| max_streams | 50 | 77;64 expected active | +| requests_per_minute | 1500 | 3000 | +| MARK stream wake | false | true | +| MARK wake minimum interval | New config, not assumed deployed | 500ms | +| OKX symbols | Existing linear list | Preserve list, add BTC-USD-SWAP | + +Ten linear products have50 ordinary streams plus10MARK wakes. Inverse adds +QUOTE/BOOK_SNAPSHOT/BOOK_DELTA plusMARKwake. Preserve all existing linear +entitlements and policies. No new TRADE/BAR inverse entitlement implied. +TS must contain f457e7a and the0.5/s/process BOOK retry cap; an older writer +cannot be accepted with this budget merely by setting the environment variable. +Sizing:900+1200+120+240+20=2480units/min, reserve520 at3000. Last20 is estimated +reconnect/probe traffic, not an unlimited-fault guarantee. See capacity guide. + +### Paired operator checklist (not executed) + +- [ ] Reinventory actual callers: expected only market_data_service_p184w uses + this identity. If others now exist, stop and account for them before action. +- [ ] Render NEW immutable manifest11/bundle plus r23binding from the approved + catalogue with77/3000. The earlier IDR075 prepared.json hash refers to the + old packet and MUST NOT be reused as proof for this quota change. Validate + binding policies/compiler, write new hashes and old/new semantic diff; only + intended Paper grants/quota/revision may differ. Do not overwrite old packet. +- [ ] Claude provides final TS digest including f457e7a, env parser check and + paired JWT11/binding hash. Keep key material private; keep subject/realm/key + identity unchanged. Confirm500ms effective runtime value and four workers. +- [ ] Capture current exact image/config/mount hashes for query_kn_1/2, + stream_kn_1/2 and writer plus rollback compose. This documentation deliberately + does not invent fresh digests. Preserve TLS, endpoints, Redis/cursors and caps. +- [ ] Confirm shared Query/Stream identity limiter uses matching Redis/prefix + and charges per request/Subscribe, not one independent allowance per replica. +- [ ] Owner approves rendered role/config/digest packet. Quiesce the affected + writer; never run old and new writer together. Sequentially recreate only + four readers with current approved images and new Paper manifest/bundle. + Check other consumers after each step. No ingestor/core/projector/Kafka changes. +- [ ] Claude activates the sole writer with JWT11+r23/hash, inverse list, + MARK_STREAM_WAKE=true and MARK_WAKE_MIN_INTERVAL_MS=500 together. Stagger + subscription startup; keep existing recovery/backoff, no extra probe stream. +- [ ] No-order preflight both replicas using the actual caller: expected + linear/inverse grants work, wrong revision/realm and ungranted feeds reject. + Confirm64 active subscriptions within77, original timestamp/component proof, + source identity/generation and cache/cursor monotonicity. No trading permission. +- [ ] Observe complete quota minute windows including startup and recovery; + record peak usage,429, reconnects, wake/failure counters and resource/backlog. + Measure MARK G2 max<1000ms,p95<=500ms on new-event transit; keep quiet age + separate. Reuse unchanged evidence; do not reopen TRADE/QUOTE investigation. + Report n and units, no p99 below100samples. Stage100 remains a separate gate. + +Stop progression for intended-route auth rejection, duplicate writer, integrity +or generation error, OOM/restart loop, quota429 under declared workload, or +failure of frozen applicable G1-G14 gates. Keep failed evidence. If rollback is +needed, stop new writer FIRST, restore four readers to saved manifest10/config, +then restore writer r22/JWT10, previous symbol list and wake=false with saved +image/config. Check both replicas and actual consumer before resuming. Never +flush data, reset offsets, weaken freshness, grant both revisions or claim +zero-downtime during the paired transition. Keep stopped operational rollback +containers/images until acceptance; cleanup only disposable owned resources. + +D1 design/packet is closed; rendering/sealing and runtime checklist remain +NOT_RUN until the execution window. v2.2.3 remains HOLD on remaining gates. From 76db472a45c221bea26ebb7f4f43da3a9a77c110 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 15:03:06 +0000 Subject: [PATCH 056/147] feat(p184): optional upward Paper quota change in the IDR-075 preparation (D1 rev11) Only max_streams/requests_per_minute, only raised, recorded in provenance; without it the packet is unchanged. --- scripts/prepare_paper_read_extension.py | 25 ++++++++++++++++++++++--- tests/test_paper_read_extension.py | 23 +++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_paper_read_extension.py b/scripts/prepare_paper_read_extension.py index 16125823..6852e7e1 100644 --- a/scripts/prepare_paper_read_extension.py +++ b/scripts/prepare_paper_read_extension.py @@ -1,4 +1,7 @@ -"""Prepare the owner-approved IDR-075 Paper entitlement; never activate it.""" +"""Prepare the owner-approved IDR-075 Paper entitlement; never activate it. + +Optional quota raise (P18.4 D1, owner 2026-10-09): only max_streams / requests_per_minute, only upward, recorded in the +provenance. Without it the packet is byte-identical to the original IDR-075 preparation.""" from __future__ import annotations import argparse @@ -28,7 +31,10 @@ def key(row): return (row["instrument_uid"], row["feed"], row.get("interval"), row["source_policy_id"]) -def prepare_paper_extension(*, catalog, manifest, binding): +QUOTA_KEYS = frozenset({"max_streams", "requests_per_minute"}) + + +def prepare_paper_extension(*, catalog, manifest, binding, quotas=None): """Reuse deployed inverse feeds; no acquisition, credential or runtime mutation.""" current = StableSourceCatalog.from_mapping(catalog) consumer = ConsumerManifestLoader.from_mapping(manifest) @@ -81,6 +87,13 @@ def instrument(symbol): raise ValueError("inverse extension already present; explicit revision review required") target, new_binding = deepcopy(manifest), deepcopy(binding) target["metadata"]["revision"] += 1 + quota_change = None + if quotas: + current_quotas = manifest["spec"]["quotas"] + if (set(quotas) - QUOTA_KEYS or any(type(v) is not int or v <= current_quotas[k] for k, v in quotas.items())): + raise ValueError("quota change must raise max_streams/requests_per_minute only") + quota_change = {k: {"from": current_quotas[k], "to": v} for k, v in sorted(quotas.items())} + target["spec"]["quotas"].update(quotas) for feed in sorted(FEEDS): policy = "crypto_primary_v2" if feed == "QUOTE" else "crypto_liquid_v2" k = (template["instrument_uid"], feed, None, policy) @@ -120,6 +133,8 @@ def instrument(symbol): consumer_manifest_sha256=parsed.manifest_sha256, deployed_catalog_sha256=digest(catalog), added_products=4, existing_fx_product_preserved=True) + if quota_change is not None: + provenance["quota_change"] = quota_change new_binding.update(schema="qdl.v2.consumer-route-binding.v3", consumer_manifest_revision=parsed.manifest_revision, release_revision=base.release_revision + 1, @@ -139,9 +154,13 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) for name in ("catalog", "manifest", "binding", "output"): parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--max-streams", type=int) + parser.add_argument("--requests-per-minute", type=int) args = parser.parse_args() + quotas = {k: v for k, v in (("max_streams", args.max_streams), + ("requests_per_minute", args.requests_per_minute)) if v is not None} result = prepare_paper_extension(**{name: yaml.safe_load(getattr(args, name).read_text()) - for name in ("catalog", "manifest", "binding")}) + for name in ("catalog", "manifest", "binding")}, quotas=quotas or None) # Never overwrite an earlier preparation or a live manifest. with args.output.open("x") as out: json.dump(result, out, indent=2, sort_keys=True) diff --git a/tests/test_paper_read_extension.py b/tests/test_paper_read_extension.py index 9ee8d6c6..0f6a0f29 100644 --- a/tests/test_paper_read_extension.py +++ b/tests/test_paper_read_extension.py @@ -135,3 +135,26 @@ def test_bad_checksum_and_duplicate_and_repeat_fail(self): self.source.update(manifest=result["manifest"], binding=result["binding"]) with self.assertRaisesRegex(ValueError, "already present"): self.prepare() + + def test_quota_raise_is_applied_only_to_the_named_keys_and_recorded(self): + plain = self.prepare() + raised = prepare_paper_extension(**self.source, quotas={"max_streams": 77, "requests_per_minute": 3000}) + quotas = raised["manifest"]["spec"]["quotas"] + before = self.source["manifest"]["spec"]["quotas"] + self.assertEqual((quotas["max_streams"], quotas["requests_per_minute"]), (77, 3000)) + self.assertEqual({k: v for k, v in quotas.items() if k not in ("max_streams", "requests_per_minute")}, + {k: v for k, v in before.items() if k not in ("max_streams", "requests_per_minute")}) + self.assertEqual(raised["provenance"]["quota_change"], + {"max_streams": {"from": before["max_streams"], "to": 77}, + "requests_per_minute": {"from": before["requests_per_minute"], "to": 3000}}) + self.assertNotIn("quota_change", plain["provenance"]) + self.assertEqual(raised["binding"]["products"], plain["binding"]["products"]) + self.assertNotEqual(raised["consumer_environment"], plain["consumer_environment"]) # new manifest hash pairs + ConsumerManifestLoader.from_mapping(raised["manifest"]) + + def test_quota_change_never_lowers_or_touches_other_keys(self): + current = self.source["manifest"]["spec"]["quotas"] + for bad in ({"max_streams": current["max_streams"]}, {"requests_per_minute": 1000}, + {"max_batch_items": 100}, {"max_streams": "77"}): + with self.assertRaisesRegex(ValueError, "quota change"): + prepare_paper_extension(**self.source, quotas=bad) From 9c4f759c7282653d7822e69f9e48181ceeba8e82 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 15:05:50 +0000 Subject: [PATCH 057/147] docs(p184): D1 rev11 rendered with runtime-equal inputs and bundle parity; not applied --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index e2d44dec..08d5cd5f 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71183,3 +71183,31 @@ Canonical /home/bobby/data_layer main5bf5cd5/v2.2.2; active worktree Runtime image/config set and rollback artifacts unchanged, not re-inventoried. No disposable resources created; Docker/BuildKit cleanup intentionally deferred for this documentation slice. v2.2.3HOLD; D1 runtime acceptance NOT_RUN. + +### D1 rev11 rendered by Claude (owner reassigned DL closure to Claude) - 2026-10-09 + +Status RENDERED_NOT_APPLIED. Owner reassigned the DL phase D work to Claude. Packet design is unchanged from 4fc987a. +- `scripts/prepare_paper_read_extension.py`: optional upward quota change (`--max-streams`, `--requests-per-minute`), only + those two keys, only raised, recorded as `provenance.quota_change`. Without it the output is identical to the original + IDR-075 preparation. `tests.test_paper_read_extension` 11 PASS in the rc2 image (commit 76db472). +- Inputs equal runtime byte for byte: + - catalog `fc3a1e1e` = `releases/v2.2.0-02cd827/runtime/stable-source-bindings.yaml`; + - Paper manifest `1e489237` = the file baked in Query 514506122111; + - binding `5703582a` = the writer mount (r22, rev 10). + Parity: `kn_gateway_bundle.py` from the reader image with those inputs reproduces the deployed bundle exactly + (`c5d964c4`). +- Rendered in `~/.local/state/qdl-v2/p184-d1-rev11-20261009/`: + - `prepared.json` 553cab32; + - `trading-system-paper-r11.yaml` efd482fa: rev 11, quotas 77/3000, +4 inverse requirements, nothing removed; + - `trading-system.paper.stable-r23-v3.json` 685fdf6f: binding r23 V3, 64 products, binding_sha256 c9a28653; + - `bundle-r11.json` 65b790ec: gateway bundle sha256 f04ba6dc; only the Paper manifest entry differs, catalog, + environment and schema are equal, and manifest order follows paths. + - Paired consumer env: `DATA_LAYER_V2_JWT_MANIFEST_REVISION=11`, + `DATA_LAYER_V2_RELEASE_MANIFEST_SHA256=7b8af009...`. + - The TS validator `p184_validate_qdl_paper_extension.py` returns VALIDATED_PREPARATION_ONLY (60 preserved, 4 added). +- Reader step rendered (TS state dir `p184-rev11-window/readers_rev11.py`): query_kn_1/2 and stream_kn_1/2 keep their images; + only the `/consumer-realms` mount moves to the new realm dir, and for Query the Paper entry of + QDL_STABLE_CONSUMER_MANIFESTS changes. +- OPEN: the new realm dir `p184-d1-rev11-20261009/reader-public` (root-owned copy of the deployed one + the rev11 manifest) + still holds the OLD bundle.json. Installing the rev11 bundle needs a root write that requires owner permission. It must + not be mounted before that. No runtime change was made. From 09dc87b4d0a977c288530d3d6fd4a6242d4e890c Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 15:19:26 +0000 Subject: [PATCH 058/147] docs(p184): D1 rev11 window rolled back on TS IDR-083; readers clean --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 08d5cd5f..20a98e08 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71211,3 +71211,11 @@ Status RENDERED_NOT_APPLIED. Owner reassigned the DL phase D work to Claude. Pac - OPEN: the new realm dir `p184-d1-rev11-20261009/reader-public` (root-owned copy of the deployed one + the rev11 manifest) still holds the OLD bundle.json. Installing the rev11 bundle needs a root write that requires owner permission. It must not be mounted before that. No runtime change was made. + +### D1 rev11 window executed and rolled back - 2026-10-09 (Claude) + +Owner-approved window, 15:11:05-15:18:28Z. The four readers (query_kn_1/2, stream_kn_1/2) took the rev11 realm without +error: healthy, bundle f04ba6dc, restarts 0. The TS writer reached only 54/64 ready: TS defect IDR-083 (the MARK ordering +authority includes the release manifest hash, and the marker has no TTL). Everything was rolled back to rev10 / bundle +c5d964c4 / writer 9ca122a, READY 60/60. DL artifacts and reader step are unchanged and reusable. The retry waits for the +TS IDR-083 fix. Receipts are in the TS journal (runtime action 7). v2.2.3 HOLD. From 8c90ddc979e5e109b548c39a0904d32345585209 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 16:05:29 +0000 Subject: [PATCH 059/147] docs(p184): D1 rev11 applied on readers (manifest rev11, bundle f04ba6dc) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 20a98e08..2e6c14d6 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71219,3 +71219,18 @@ error: healthy, bundle f04ba6dc, restarts 0. The TS writer reached only 54/64 re authority includes the release manifest hash, and the marker has no TTL). Everything was rolled back to rev10 / bundle c5d964c4 / writer 9ca122a, READY 60/60. DL artifacts and reader step are unchanged and reusable. The retry waits for the TS IDR-083 fix. Receipts are in the TS journal (runtime action 7). v2.2.3 HOLD. + +### D1 rev11 APPLIED - 2026-10-09 (Claude) + +Owner-approved retry window 15:47:40-15:50:02Z. +- Runtime now: + - query_kn_1/2: same image 514506122111, /consumer-realms = `~/.local/state/qdl-v2/p184-d1-rev11-20261009/reader-public`, + Paper manifest path `/consumer-realms/trading-system.paper.stable-r11.yaml` (rev 11, 77 streams / 3000 RPM, +4 inverse); + - stream_kn_1/2: same image 1f13408fe946, bundle f04ba6dc. +- TS writer 71404a7 on JWT 11 / binding r23: READY 64/64, 64 active subscriptions on stream_kn_1, identity quota + 1575-1593/min, no 429. +- Rollback per reader: the saved composes in the TS state dir `p184-rev11-window/readers/*-rollback.compose.json` (old + realm dir + baked Paper manifest). +- No ingestor/core/projector/Kafka/Redis change. +- Open: the live/sandbox TS realms remain at 50 streams/1500 RPM (no owner decision). MARK G2 is a TS item (Binance ~1.18 s). + v2.2.3 HOLD. From 922a6ec38a8da94d30c634f025fd23c03838c24d Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:10:35 +0000 Subject: [PATCH 060/147] fix(query): execution MARK/INDEX takes the newer committed hot record over the projected cache (IDR-084) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 24 ++++++++ qdl/runtime/kn_query_backend.py | 15 +++++ tests/test_kn_query_backend.py | 74 +++++++++++++++++++++++ 3 files changed, 113 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 2e6c14d6..239db7a4 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71234,3 +71234,27 @@ Owner-approved retry window 15:47:40-15:50:02Z. - No ingestor/core/projector/Kafka/Redis change. - Open: the live/sandbox TS realms remain at 50 streams/1500 RPM (no owner decision). MARK G2 is a TS item (Binance ~1.18 s). v2.2.3 HOLD. + +### IDR-084 fixed in source: execution MARK takes the newer committed hot record - 2026-10-09 (Claude) + +Status IMPLEMENTED / TESTED_LOCALLY (not deployed; it goes into the consolidated D5 Query image). +- Finding (TS register IDR-084, TS writer windows 2026-10-09): + - Query serves execution MARK/INDEX from the projected kn3 latest view (CacheRefreshingMarkIndexView -> + KnMarketCacheQueryBackend.latest_stored_event). + - That view is visible about 300 ms after the venue event (Binance p50 292-305 / p95 367-377 ms), while canonical + read_committed visibility is 150-181 ms p95 (2026-10-09 G4 observer). + - The TS MARK floor stayed about 400 ms p50 whatever the TS read cadence; OKX G2 p95 was 508-534 ms. +- Fix in `qdl/runtime/kn_query_backend.py` latest_stored_event: for MARK_INDEX_PRICE with a valid cache view, read the + existing Rust canonical hot view (QDL_KN_HOT_READ_TARGETS) and take it only when its source offset is newer. + - It passes the same `_hot_candidate` lineage/monotonic/generation gates. + - Behind or unavailable -> the cache view stands (previous behaviour). + - Same offset with different bytes -> INTEGRITY. + - Other feeds are unchanged and never consult the hot view for a valid cache view. +- Tests (disposable Redis, rc2 image 77682eea): tests.test_kn_query_backend + test_kn_hot_view + + test_kn_hot_book_invalidation + test_execution_mark_index_live_view: 90 PASS, 0 fail, 0 skip. New cases: + - newer hot record wins on both venues; + - hot outage keeps the valid cache view; + - a hot record behind the cache is never taken; + - non-MARK feeds never consult hot. +- Load note: about 1 extra hot gRPC read per execution MARK read (TS: up to 2/s per symbol, 10 linear + 1 inverse) on the + Stream gateways. To measure in the D5 rollout; it is not a capacity claim. diff --git a/qdl/runtime/kn_query_backend.py b/qdl/runtime/kn_query_backend.py index 1579490f..76340360 100644 --- a/qdl/runtime/kn_query_backend.py +++ b/qdl/runtime/kn_query_backend.py @@ -408,6 +408,21 @@ def latest_stored_event(self, binding: StableSourceBinding, *, prefer_hot: bool return None, "BACKUP_UNAVAILABLE" except QueryBackendError as error: return None, "FENCED" if error.problem.code is CanonicalErrorCode.DATA_NOT_READY else "INTEGRITY" + elif self.hot_client is not None and view is not None and binding.feed is FeedType.MARK_INDEX_PRICE: + # IDR-084: the projected cache trails the committed canonical record by ~150 ms (Binance MARK visible p50 + # ~300 ms after the venue event). The Rust hot view serves the same committed bytes without the projector, + # so MARK/INDEX takes the newer of the two. Every lineage/monotonic gate of the backup path applies; a hot + # read that is behind or unavailable leaves the cache view in place (the previous behaviour). + try: + candidate = self._hot_candidate(binding, view) + except HotViewUnavailable: + candidate = None + except QueryBackendError as error: + if error.problem.code is CanonicalErrorCode.INTERNAL_ERROR: + return None, "INTEGRITY" # same offset, different bytes: never pick one silently + candidate = None + if candidate is not None and candidate.rows[0].source_offset > view.rows[0].source_offset: + view = candidate if view is None: return None, "NOT_READY" records = self._parsed(binding, view) diff --git a/tests/test_kn_query_backend.py b/tests/test_kn_query_backend.py index 652d6794..9a1bdd3c 100644 --- a/tests/test_kn_query_backend.py +++ b/tests/test_kn_query_backend.py @@ -913,6 +913,80 @@ def read(): self.assertIsNone(read().record) self.assertGreaterEqual(len(exercised), 2, "both Binance and OKX fixtures required") + def test_execution_mark_index_takes_the_newer_hot_record_over_a_valid_cache_view(self): + """IDR-084: a usable but older projected MARK is replaced by the newer committed hot record.""" + import asyncio + import time as _time + from types import SimpleNamespace + from qdl.reference.local_mark_index import CacheRefreshingMarkIndexView + from qdl.runtime.kn_hot_view import CanonicalHotView, HotViewUnavailable + from qdl.runtime.kn_market_cache import CacheRow, SourceBoundary + + exercised = set() + for binding, payload in self.latest_records: + if binding.feed is not FeedType.MARK_INDEX_PRICE or not binding.authoritative or binding.source_role != "PRIMARY": + continue + with self.subTest(binding=binding.binding_id): + exercised.add(binding.instrument.identity.venue) + now = _time.time_ns() + def timed(age): + env = market_data_pb2.EventEnvelope.FromString(retimed_mark_index(payload, now_ns=now, age_ms=age)) + env.normalized_at_ns = env.received_at_ns + 1 + env.published_at_ns = env.received_at_ns + 2 + return env.SerializeToString(deterministic=True) + cached, newer, older = timed(400), timed(100), timed(900) + self.put_latest(binding, cached, offset=10, mark=20) + primary = self.reader.latest(self.lpk(binding)) + def hot_view(payload_bytes, offset, mark): + return CanonicalHotView(self.lpk(binding), (CacheRow(payload_bytes, offset),), + SourceBoundary(TOPIC_ID, primary.boundary.partition, mark)) + replies = [hot_view(newer, 30, 40)] + def hot(*args): + if isinstance(replies[0], Exception): raise replies[0] + return replies[0] + backend = KnMarketCacheQueryBackend(self.reader, self.catalog, schema_digest=DIGEST, + topic_id=TOPIC_ID, hot_client=SimpleNamespace(latest=hot)) + view = CacheRefreshingMarkIndexView.from_catalog(self.catalog).attach_cache(backend=backend, relax_to_alpha=False) + def read(): + return asyncio.run(view.read(instrument_uid=binding.instrument.instrument_uid, + instrument_revision=binding.instrument.metadata_revision, source_policy_id=binding.source_policy_id, + max_freshness_ms=2000, gateway_epoch=1, now_ns=now)) + result = read() + self.assertIsNotNone(result.record, result.reason) + self.assertEqual(result.record.canonical, newer) # the newer committed record wins + # A hot outage leaves the valid cache view in place (never worse than before). + fresh_backend = KnMarketCacheQueryBackend(self.reader, self.catalog, schema_digest=DIGEST, + topic_id=TOPIC_ID, hot_client=SimpleNamespace(latest=hot)) + fresh_view = CacheRefreshingMarkIndexView.from_catalog(self.catalog).attach_cache( + backend=fresh_backend, relax_to_alpha=False) + replies[0] = HotViewUnavailable("test outage") + stored, state = fresh_backend.latest_stored_event(binding) + self.assertEqual(state, "OK") + self.assertEqual(stored.event.payload, cached) + # A hot record behind the cache is never taken. + replies[0] = hot_view(older, 5, 8) + stored, state = fresh_backend.latest_stored_event(binding) + self.assertEqual((state, stored.event.payload), ("OK", cached)) + del fresh_view + self.assertGreaterEqual(len(exercised), 2, "both Binance and OKX fixtures required") + + def test_non_mark_feeds_never_consult_the_hot_view_for_a_valid_cache_view(self): + from types import SimpleNamespace + + def hot(*args): + raise AssertionError("hot view consulted for a non-MARK feed") + backend = KnMarketCacheQueryBackend(self.reader, self.catalog, schema_digest=DIGEST, + topic_id=TOPIC_ID, hot_client=SimpleNamespace(latest=hot)) + checked = 0 + for binding, payload in self.latest_records: + if binding.feed in (FeedType.MARK_INDEX_PRICE, FeedType.BAR): + continue + self.put_latest(binding, payload, offset=10, mark=20) + stored, state = backend.latest_stored_event(binding) + self.assertEqual(state, "OK", binding.binding_id) + checked += 1 + self.assertGreater(checked, 0) + # ------------------------------------------------------------ D29 read view def test_the_stream_read_view_runs_the_python_oracle_on_one_view(self): From 1ee84fd6bb271946df76f3f467bfb4437205cfc6 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:13:40 +0000 Subject: [PATCH 061/147] docs(p184): owner decisions for DL closure (deferrals, realms, G4/G5 rule, stage-100 driver) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 239db7a4..7e26ec65 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71258,3 +71258,15 @@ Status IMPLEMENTED / TESTED_LOCALLY (not deployed; it goes into the consolidated - non-MARK feeds never consult hot. - Load note: about 1 extra hot gRPC read per execution MARK read (TS: up to 2/s per symbol, 10 linear + 1 inverse) on the Stream gateways. To measure in the D5 rollout; it is not a capacity claim. + +### Owner decisions for the DL closure - 2026-10-09 (Claude) + +AskUserQuestion answers, recorded before the respective work. +- IDR-040 fix 2 and IDR-003 (new optional quality field): DEFERRED, not in v2.2.3. Recorded limit: a quiet Binance book is + refused as stale beyond its last frame, never falsely fresh. A later release adds the field with the tolerant reader first. +- trading-system.live / .sandbox realms stay at 50 streams / 1500 RPM. A separate rev packet follows when a caller enables + MARK wake or inverse there. +- G4/G5: the baseline rule is in TS upgrade/p184-phase-d-gates.md "Owner amendment 2026-10-09": + - G4 candidate p95 <= baseline x 1.2 per feed/venue; + - G5 stage-50 targets where they exist, otherwise baseline p95 x 1.5 per shape, frozen before candidate load. +- Stage-100 load: synthetic driver with real SDK sessions on the capacity100 profiles plus the real TS writer. From 522cf3e1034af467339a76fe092432a0ddc29e00 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:15:06 +0000 Subject: [PATCH 062/147] fix(dr05): reset failures name their stage and keep a bounded stderr tail (D6) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 12 +++++ scripts/p184_dr05_reset.py | 56 +++++++++++++++++++---- tests/test_p184_dr05_reset.py | 40 ++++++++++++++++ 3 files changed, 99 insertions(+), 9 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 7e26ec65..8902b612 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71270,3 +71270,15 @@ AskUserQuestion answers, recorded before the respective work. - G4 candidate p95 <= baseline x 1.2 per feed/venue; - G5 stage-50 targets where they exist, otherwise baseline p95 x 1.5 per shape, frozen before candidate load. - Stage-100 load: synthetic driver with real SDK sessions on the capacity100 profiles plus the real TS writer. + +### D6 reset diagnostics - 2026-10-09 (Claude) + +Status IMPLEMENTED / TESTED_LOCALLY. DR05 r13 lost the cause of a reset failure: `serve_host` kept only +`CalledProcessError:...`, and `reset_preserving_broker` had already stopped query/stream/core/quota, so `--down` found +only kafka. +- `scripts/p184_dr05_reset.py`: a failure after the readers were stopped raises `ResetStageError` with + `stage=stop_readers|reset_topics|start_readers|probe|verify_broker`, `readers_stopped=true`, the return code and a printable + 240-character stderr tail (message <= 380 characters, inside the 400 the host result keeps). The broker-changed refusal keeps + its text. No retry was added and no deadline was extended. +- Tests in the rc2 image: tests.test_p184_dr05_reset + channel + case: 53 PASS. The new case is a topic reset failing with rc 1 + and stderr; it checks stage, rc, bound and non-printable stripping. diff --git a/scripts/p184_dr05_reset.py b/scripts/p184_dr05_reset.py index ecf9f8d4..8432aa33 100644 --- a/scripts/p184_dr05_reset.py +++ b/scripts/p184_dr05_reset.py @@ -57,6 +57,19 @@ def reset_topics(work, timeout): return {"topic_id": new_id, "previous_topic_id": old_id, "topics": topics} +class ResetStageError(RuntimeError): + """A reset that failed after it stopped the readers: names the stage and keeps a bounded stderr tail (D6, r13).""" + + def __init__(self, stage, error): + stderr = getattr(error, "stderr", None) or "" + if isinstance(stderr, bytes): + stderr = stderr.decode("utf-8", "replace") + tail = "".join(ch if ch.isprintable() else " " for ch in stderr)[-240:] + self.stage = stage + super().__init__(f"stage={stage} readers_stopped=true {type(error).__name__}:{str(error)[:80]}" + f" rc={getattr(error, 'returncode', None)} stderr_tail={tail!r}"[:380]) + + def reset_preserving_broker(work, venue, timeout=30): deadline = _deadline(timeout) if venue not in dr.VENUES: @@ -76,8 +89,24 @@ def call(argv): raise ValueError("complete owned stack required before reset") ids = [row["Id"] for row in rows] call(["docker", "stop", "--time", "1", *ids]) - while any(dr.owned(ns, role, call) for role in ("query", "stream", "core", "quota")): - time.sleep(min(.1, _remaining(deadline))) + stage = "stop_readers" + try: + while any(dr.owned(ns, role, call) for role in ("query", "stream", "core", "quota")): + time.sleep(min(.1, _remaining(deadline))) + return _reset_after_stop(work, ns, venue, selected, broker, call, deadline, channel) + except _StageFailure as failure: + raise ResetStageError(failure.stage, failure.error) from failure.error + except Exception as error: + raise ResetStageError(stage, error) from error + + +class _StageFailure(Exception): + def __init__(self, stage, error): + super().__init__(stage) + self.stage, self.error = stage, error + + +def _reset_after_stop(work, ns, venue, selected, broker, call, deadline, channel): def inside(script, args): if channel: @@ -89,14 +118,23 @@ def inside(script, args): argv[pos:pos] = ["-e", "P184_DR05_ISOLATED_NAMESPACE=" + ns] return call(argv) - topic = json.loads(inside("scripts/p184_dr05_reset.py", - ["--topics", "/work", "--timeout", str(_remaining(deadline))])) + try: + topic = json.loads(inside("scripts/p184_dr05_reset.py", + ["--topics", "/work", "--timeout", str(_remaining(deadline))])) + except Exception as error: + raise _StageFailure("reset_topics", error) from error (work / "query-handoffs.jsonl").unlink(missing_ok=True) - commands = dr.topology(ns, work, topic["topic_id"], core_binary=selected) - for role in ("quota", "core", "stream", "query"): - call(commands[role]) + try: + commands = dr.topology(ns, work, topic["topic_id"], core_binary=selected) + for role in ("quota", "core", "stream", "query"): + call(commands[role]) + except Exception as error: + raise _StageFailure("start_readers", error) from error while True: - _remaining(deadline) + try: + _remaining(deadline) + except Exception as error: + raise _StageFailure("probe", error) from error try: inside("scripts/p184_dr05_dl_fault.py", ["--probe", "/work"]) break @@ -104,7 +142,7 @@ def inside(script, args): time.sleep(min(.2, _remaining(deadline))) current = dr.owned(ns, "kafka", call) if not current or current["Id"] != broker["Id"]: - raise RuntimeError("broker namespace changed during reset") + raise _StageFailure("verify_broker", RuntimeError("broker namespace changed during reset")) return {"namespace": ns, "venue": venue, "generation": 7, "reset": True, "broker_id": current["Id"], "topic_id": topic["topic_id"], "previous_topic_id": topic["previous_topic_id"], "acceptance": False, diff --git a/tests/test_p184_dr05_reset.py b/tests/test_p184_dr05_reset.py index f8252702..b87cc165 100644 --- a/tests/test_p184_dr05_reset.py +++ b/tests/test_p184_dr05_reset.py @@ -77,3 +77,43 @@ def test_invalid_deadline_and_venue_before_docker(self): with patch.object(reset.dr, "bounded") as call, self.assertRaises(ValueError): reset.reset_preserving_broker(Path("/unused"), venue, timeout) call.assert_not_called() + + +class ResetStageDiagnosticTests(unittest.TestCase): + """D6 (DR05 r13): a failure after the readers were stopped names its stage and keeps a bounded stderr tail.""" + + def test_a_failed_topic_reset_reports_stage_rc_and_stderr(self): + import subprocess + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "handoff.json").write_text("{}") + stopped = [] + + def owned(ns, role, call): + if role == "kafka": + return {"Id": "broker"} + if role == "helper": + return {"Id": "channel"} + return None if stopped else {"Id": role} + + def bounded(argv, timeout): + if argv[:2] == ["docker", "stop"]: + stopped.append(True) + return "" + if "scripts/p184_dr05_reset.py" in argv: + raise subprocess.CalledProcessError(1, argv, output="", stderr="Traceback ...\nKafkaException: TOPIC_ALREADY_EXISTS\x07" + "x" * 500) + return "{}" + + with patch.object(reset, "_packet", return_value=(work, NS, runtime_packet(), config())), \ + patch.object(reset, "_reset_preflight", Mock(return_value={"Id": "broker"})), \ + patch.object(reset.dr, "owned", side_effect=owned), \ + patch.object(reset.dr, "bounded", side_effect=bounded): + with self.assertRaises(reset.ResetStageError) as caught: + reset.reset_preserving_broker(work, "OKX", 10) + message = str(caught.exception) + self.assertEqual(caught.exception.stage, "reset_topics") + self.assertIn("stage=reset_topics", message) + self.assertIn("readers_stopped=true", message) + self.assertIn("rc=1", message) + self.assertLessEqual(len(message), 380) + self.assertNotIn("\x07", message) From f831571c96b51e3b88007f4d7f064006e1c954bf Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:19:27 +0000 Subject: [PATCH 063/147] feat(p184): stage-100 ramp in the target driver with a frozen G5 budget schema (D3) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 22 ++++++ qdl/certification/phase3_consumer_load.py | 45 ++++++++++- scripts/phase3_consumer_load_acceptance.py | 23 ++++-- tests/test_p184_stage100.py | 87 ++++++++++++++++++++++ 4 files changed, 168 insertions(+), 9 deletions(-) create mode 100644 tests/test_p184_stage100.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 8902b612..c5449ca4 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71282,3 +71282,25 @@ only kafka. its text. No retry was added and no deadline was extended. - Tests in the rc2 image: tests.test_p184_dr05_reset + channel + case: 53 PASS. The new case is a topic reset failing with rc 1 and stderr; it checks stage, rc, bound and non-printable stripping. + +### D3 slice 1: stage-100 ramp in the existing target driver - 2026-10-09 (Claude) + +Status IMPLEMENTED / TESTED_LOCALLY (no traffic, no runtime change). +- `qdl/certification/phase3_consumer_load.py`, additive: + - P184_STAGE_MIX {20: (8,6,4,2), 50: (20,15,10,5), 100: (40,30,20,10)} and P184_STAGE_SECONDS {20: 600, 50: 600, + 100: 1800}. The frozen v2.1.1 tables and budget are untouched. + - build_target_workload_plan accepts stage 100. + - load_p184_stage100_budget (schema `qdl.p184.stage100-budget.v1`) refuses budgets that are not marked + frozen_before_candidate, lack G5 shape limits or lack the TS route count. + - evaluate_target_acceptance adds `g5:COLD_:` gates: max of the cold reads within the frozen limit, + small-sample rule, so a missing or failed read fails. +- `scripts/phase3_consumer_load_acceptance.py`: optional `--budget` selects the stage-100 budget, its stages and + durations; stage 100 is the final run. Without it the v2.1.1 behaviour is byte-identical. +- Tests (rc2 image 77682eea, network none): tests.test_p184_stage100 (new, 7) + target workload/consumer load/target + driver/consumer load driver/capacity100/kn capacity budget: 111 run, 1 failure. + - The failure is pre-existing: `test_phase3_target_driver.test_a_typed_rejection_is_counted_by_code` counts 5 instead + of 4 rejections in 0.2 s, 3/3 at HEAD 522cf3e without this change. + - It is a tick fencepost in the test expectation, not caused by this slice. It is recorded, not hidden, and remains to + fix in the driver's own tests. +- The stage-100 plan does not fit the deployed alpha manifests (2400 RPM / 60 streams). The capacity100 candidate quota + (4950 / 300) carries it. That runtime packet is the next D3 slice. diff --git a/qdl/certification/phase3_consumer_load.py b/qdl/certification/phase3_consumer_load.py index 71a57f42..e85aff76 100644 --- a/qdl/certification/phase3_consumer_load.py +++ b/qdl/certification/phase3_consumer_load.py @@ -437,6 +437,11 @@ def assert_required_instrument_coverage( 50: (20, 15, 10, 5), } TARGET_STAGE_SECONDS: dict[int, int] = {5: 90, 20: 120, 35: 180, 50: 300} +# P18.4 phase D stage-100 ramp (owner 2026-10-08/09: 100 logical alphas plus TS, synthetic driver with real SDK sessions, +# steady >= 30 min at stage 100). Additive: the frozen v2.1.1 table above and its budget are unchanged; stage 100 keeps +# the same four-class proportions (40/30/20/10). +P184_STAGE_MIX: dict[int, tuple[int, int, int, int]] = {20: (8, 6, 4, 2), 50: (20, 15, 10, 5), 100: (40, 30, 20, 10)} +P184_STAGE_SECONDS: dict[int, int] = {20: 600, 50: 600, 100: 1800} TARGET_CLASSES = ("CANDLE", "REALTIME", "GRID", "MULTI") TARGET_HOT_PERIOD_SECONDS = 1.0 TARGET_REFERENCE_PERIOD_SECONDS = 60.0 @@ -569,8 +574,9 @@ def build_target_workload_plan( the declared headroom. """ - if stage not in TARGET_STAGE_MIX: - raise ValueError(f"target stage must be one of {sorted(TARGET_STAGE_MIX)}") + mix = TARGET_STAGE_MIX.get(stage) or P184_STAGE_MIX.get(stage) + if mix is None: + raise ValueError(f"target stage must be one of {sorted(set(TARGET_STAGE_MIX) | set(P184_STAGE_MIX))}") venues = sorted(venue_identity) # Interleave venues symbol by symbol so every stage, including five # sessions, spreads over both venues and both identities rather than @@ -580,7 +586,7 @@ def build_target_workload_plan( for venue, column in zip(venues, columns) if row < len(column)] if not pairs: raise ValueError("target workload has no instruments") - classes = [name for name, count in zip(TARGET_CLASSES, TARGET_STAGE_MIX[stage]) for _ in range(count)] + classes = [name for name, count in zip(TARGET_CLASSES, mix) for _ in range(count)] sessions: list[TargetAlphaSession] = [] for ordinal, alpha_class in enumerate(classes, start=1): venue, symbol = pairs[(ordinal - 1) % len(pairs)] @@ -766,6 +772,28 @@ def load_target_budget(raw: Mapping[str, object]) -> Mapping[str, object]: return raw +def load_p184_stage100_budget(raw: Mapping[str, object]) -> Mapping[str, object]: + """The P18.4 stage-100 budget: same gate shape as v2.1.1 plus frozen G5 shape limits and the TS route count.""" + if raw.get("schema") != "qdl.p184.stage100-budget.v1": + raise ValueError("stage-100 budget schema is unknown") + stages = raw.get("stages") + if not isinstance(stages, Mapping) or {int(key) for key in stages} != set(P184_STAGE_MIX): + raise ValueError("stage-100 budget stages differ from the P18.4 ramp") + for key, value in stages.items(): + if tuple(value["mix"]) != P184_STAGE_MIX[int(key)] or value["seconds"] != P184_STAGE_SECONDS[int(key)]: + raise ValueError("stage-100 budget stage mix or duration differs from the P18.4 ramp") + if raw.get("frozen_before_candidate") is not True: + raise ValueError("stage-100 budget must be frozen (baseline-derived limits) before candidate load") + shapes = raw.get("g5_shape_limits_ms") + if not isinstance(shapes, Mapping) or not shapes or any( + not isinstance(v, Mapping) or not isinstance(v.get("p95"), (int, float)) or v["p95"] <= 0 + for v in shapes.values()): + raise ValueError("stage-100 budget needs frozen G5 shape limits") + if type(raw.get("trading_system", {}).get("demanded_routes")) is not int: + raise ValueError("stage-100 budget needs the TS demanded route count") + return raw + + def evaluate_latency(values_ms: Sequence[float], target: Mapping[str, object], rule: Mapping[str, object]) -> dict[str, object]: count = len(values_ms) @@ -845,6 +873,17 @@ def evaluate_target_acceptance( done = {(item["venue"], item["rows"]) for item in cold if not item.get("error")} gates.append(_gate("cold:2500_5000_overlap_per_venue", not cold_bad and wanted <= done, runs=len(cold), bad=len(cold_bad))) + for shape, limit in sorted((budget.get("g5_shape_limits_ms") or {}).items()): + # P18.4 G5 (owner 2026-10-09): shapes without a certified target use baseline p95 x 1.5, frozen before load. + if not shape.startswith("COLD_"): + continue + rows = int(shape.split("_", 1)[1]) + for venue in venues: + values = [item["ms"] for item in cold if item.get("rows") == rows and item.get("venue") == venue + and not item.get("error") and isinstance(item.get("ms"), (int, float))] + gates.append(_gate(f"g5:{shape}:{venue}", bool(values) and max(values) <= limit["p95"], + count=len(values), max_ms=max(values) if values else None, limit_ms=limit["p95"], + rule="small sample: max within the frozen limit")) setup = receipt["setup"] gates.append(_gate("startup:bounded", not setup["failures"] and setup["seconds"] <= budget["startup"]["max_setup_seconds"], diff --git a/scripts/phase3_consumer_load_acceptance.py b/scripts/phase3_consumer_load_acceptance.py index c41972d5..9fb71d27 100644 --- a/scripts/phase3_consumer_load_acceptance.py +++ b/scripts/phase3_consumer_load_acceptance.py @@ -3196,15 +3196,25 @@ def run_target_host(args: argparse.Namespace) -> int: profile = validate_profile(_read_json(args.profile)) if {item["id"] for item in profile["identities"]} != set(_TARGET_VENUE_IDENTITY.values()): raise ValueError("target mode runs exactly the two alpha platform identities") - budget_bytes = _TARGET_BUDGET_PATH.read_bytes() - budget = load_target_budget(json.loads(budget_bytes)) matrix = args.mode in {"target-matrix", "kn4-matrix"} - if args.sessions not in TARGET_STAGE_SECONDS: - raise ValueError("target sessions must be exactly one of 5,20,35,50") - duration = 0 if matrix else TARGET_STAGE_SECONDS[args.sessions] + budget_path = getattr(args, "budget", None) + if budget_path is not None: + # P18.4 stage-100 ramp: its own frozen budget; the v2.1.1 contract below stays untouched. + budget_bytes = Path(budget_path).read_bytes() + budget = planner.load_p184_stage100_budget(json.loads(budget_bytes)) + stage_seconds = planner.P184_STAGE_SECONDS + final_stage = 100 + else: + budget_bytes = _TARGET_BUDGET_PATH.read_bytes() + budget = load_target_budget(json.loads(budget_bytes)) + stage_seconds = TARGET_STAGE_SECONDS + final_stage = 50 + if args.sessions not in stage_seconds: + raise ValueError(f"target sessions must be exactly one of {','.join(map(str, sorted(stage_seconds)))}") + duration = 0 if matrix else stage_seconds[args.sessions] if args.duration_seconds != duration: raise ValueError(f"target stage {args.sessions} runs exactly {duration} s from the frozen budget") - final = not matrix and args.sessions == 50 + final = not matrix and args.sessions == final_stage if final: assert_no_disposable_probes() workers = 1 if matrix else target_worker_count(args.sessions) @@ -3353,6 +3363,7 @@ def parser() -> argparse.ArgumentParser: value.add_argument("--inside-worker", nargs=2, type=int, metavar=("INDEX", "COUNT"), help=argparse.SUPPRESS) value.add_argument("--sessions", type=int) value.add_argument("--duration-seconds", type=int) + value.add_argument("--budget", type=Path, help="P18.4 stage-100 budget (qdl.p184.stage100-budget.v1)") return value diff --git a/tests/test_p184_stage100.py b/tests/test_p184_stage100.py new file mode 100644 index 00000000..d307336f --- /dev/null +++ b/tests/test_p184_stage100.py @@ -0,0 +1,87 @@ +"""P18.4 phase D stage-100 ramp (D3): planner, frozen budget and G5 shape gates. TEST_ONLY fixtures, no traffic.""" +from __future__ import annotations + +import unittest + +from qdl.certification.phase3_consumer_load import ( + P184_STAGE_MIX, + P184_STAGE_SECONDS, + TARGET_STAGE_MIX, + evaluate_target_acceptance, + load_p184_stage100_budget, + load_target_budget, +) +from tests.test_phase3_target_workload import _plan + + +def _budget(**overrides): + raw = { + "schema": "qdl.p184.stage100-budget.v1", "frozen_before_candidate": True, + "stages": {str(k): {"mix": list(P184_STAGE_MIX[k]), "seconds": P184_STAGE_SECONDS[k]} for k in P184_STAGE_MIX}, + "g5_shape_limits_ms": {"COLD_2500": {"p95": 4000.0}, "COLD_5000": {"p95": 6000.0}}, + "trading_system": {"demanded_routes": 64}, + } + raw.update(overrides) + return raw + + +class Stage100PlanTests(unittest.TestCase): + def test_stage_100_keeps_the_four_class_proportions_and_the_v211_table_is_unchanged(self): + self.assertEqual(P184_STAGE_MIX[100], (40, 30, 20, 10)) + self.assertEqual(set(TARGET_STAGE_MIX), {5, 20, 35, 50}) # frozen v2.1.1 table untouched + self.assertEqual(P184_STAGE_SECONDS[100], 1800) # owner: >= 30 min steady at stage 100 + + def test_stage_100_plans_100_sessions_and_needs_more_than_todays_alpha_quota(self): + with self.assertRaisesRegex(ValueError, "does not fit sealed quota"): + _plan(100, rpm=2400, streams=60) # the deployed alpha manifests cannot carry stage 100 + plan = _plan(100, rpm=4950, streams=300) # the capacity100 candidate quota + self.assertEqual(len(plan.sessions), 100) + self.assertTrue(all(d.fits for d in plan.demands)) + self.assertEqual(round(plan.hot_requests_per_second), 100) + + +class Stage100BudgetTests(unittest.TestCase): + def test_a_frozen_budget_loads(self): + self.assertEqual(load_p184_stage100_budget(_budget())["trading_system"]["demanded_routes"], 64) + + def test_unfrozen_or_incomplete_budgets_are_refused(self): + for bad in (_budget(frozen_before_candidate=False), _budget(g5_shape_limits_ms={}), + _budget(g5_shape_limits_ms={"COLD_2500": {"p95": None}}), _budget(trading_system={}), + _budget(schema="qdl.v211.target-acceptance-budget.v1")): + with self.subTest(bad=bad), self.assertRaises(ValueError): + load_p184_stage100_budget(bad) + + def test_the_v211_loader_still_refuses_a_stage_100_budget(self): + with self.assertRaises(ValueError): + load_target_budget({**_budget(), "schema": "qdl.v211.target-acceptance-budget.v1"}) + + +class Stage100G5GateTests(unittest.TestCase): + def _evaluate(self, cold): + budget = {**_budget(), "latency_ms": {}, "sample_rule": {"p99_min_samples": 100, "p95_min_samples": 20}, + "requests": {"max_missed_ticks": 0, "max_failed_steady": 0, "client_validity_scheduler_lag_p99_ms": 250}, + "streams": {"max_stream_errors": 0, "first_event_required_feeds": [], "min_final_bars_per_bar_stream": 0}, + "cold": {"rows": [2500, 5000]}, "startup": {"max_setup_seconds": 60}, + "trading_system": {"demanded_routes": 64, "max_fallback": 0, "max_v2_error_increase": 0, + "forbidden_disconnect_codes": [], "max_disconnect_rate_over_baseline_per_minute": 1}} + receipt = {"latency_series": {"QUOTE_SNAPSHOT|BINANCE|STEADY": [10.0]}, "poll_ledgers": [], + "scheduler_lag_ms": {"p99": 1}, "streams": [], "cold": cold, + "setup": {"failures": 0, "seconds": 1, "retries": 0}, "leaked_tasks": 0} + result = evaluate_target_acceptance(budget=budget, stage=100, final=False, receipt=receipt, + trading_system={"samples": []}) + return {g["gate"]: g for g in result["gates"]} + + def test_cold_shapes_are_gated_against_their_frozen_limit(self): + gates = self._evaluate([{"venue": "BINANCE", "rows": 2500, "ms": 3500.0}, + {"venue": "BINANCE", "rows": 5000, "ms": 6500.0}]) + self.assertEqual(gates["g5:COLD_2500:BINANCE"]["status"], "PASS") + self.assertEqual(gates["g5:COLD_5000:BINANCE"]["status"], "FAIL") + + def test_a_missing_or_failed_cold_read_fails_its_shape_gate(self): + gates = self._evaluate([{"venue": "BINANCE", "rows": 2500, "ms": 100.0, "error": {"type": "X"}}]) + self.assertEqual(gates["g5:COLD_2500:BINANCE"]["status"], "FAIL") + self.assertEqual(gates["g5:COLD_5000:BINANCE"]["status"], "FAIL") + + +if __name__ == "__main__": + unittest.main() From a78696a128f42c88b4eee97448d4e6c4691258f3 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:22:03 +0000 Subject: [PATCH 064/147] feat(p184): tracked G4 observer with all-feed filter and offline per feed/venue summary (D3) --- scripts/p184_g4_observer.py | 224 +++++++++++++++++++++++++++++++++ tests/test_p184_g4_observer.py | 39 ++++++ 2 files changed, 263 insertions(+) create mode 100644 scripts/p184_g4_observer.py create mode 100644 tests/test_p184_g4_observer.py diff --git a/scripts/p184_g4_observer.py b/scripts/p184_g4_observer.py new file mode 100644 index 00000000..0a534194 --- /dev/null +++ b/scripts/p184_g4_observer.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Explicitly launched, bounded Kafka lag/assigned-tail observer; no writes/joins (P18.4 G4 baseline/candidate). + +Origin: ~/.local/state/qdl-v2/g4-trade-mark-20261009/observer.py (sha256 66ce6625...), which hard-coded BTC/ETH TRADE/MARK. +This tracked copy takes the symbol/feed filter as arguments (default: every symbol and feed) and adds `--summarize`, +an offline per feed/venue (BAR per interval) p50/p95/max/n of the visibility upper bound +(observer read_committed receipt - source event time) and of venue -> DL receipt. The observer joins no group, +commits nothing and keeps no payload values.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import time +from decimal import Decimal + +PROJECTOR='qdl_v2_stable_candidate-market_projector_1-1' +QUERY='qdl_v2_stable_candidate-query_kn_1-1' +GROUPS=[('core','qdl-v2-realtime-core-v2'),('projector','kn-projector-v3-a'),('projector','kn-projector-v3-b'),('producer','qdl-v2-book-resync-ingestor_binance_usdm-binance-public-book-001'),('producer','qdl-v2-book-resync-ingestor_okx_swap-okx-public-001')] + +def bounded_decimal(value): + coefficient=value.WhichOneof('coefficient') + if coefficient is None or abs(value.scale)>30:return None + raw=str(getattr(value,coefficient)) + if len(raw)>64 or not raw.lstrip('-').isdigit():return None + number=Decimal(raw) + parts=number.as_tuple() + result=format(Decimal((parts.sign,parts.digits,-value.scale)), 'f') + return result if len(result)<=96 else None + +def top_record(event): + feed=event.WhichOneof('payload') + result={'bid_price':None,'ask_price':None,'basis':'UNAVAILABLE_SAMPLED_DELTA_NOT_RECONSTRUCTED'} + if feed=='quote': + result.update(bid_price=bounded_decimal(event.quote.bid_price),ask_price=bounded_decimal(event.quote.ask_price),basis='QUOTE_EVENT') + elif feed=='book_snapshot': + sides={1:[],2:[]} + for level in event.book_snapshot.levels: + price,quantity=bounded_decimal(level.price),bounded_decimal(level.quantity) + if level.side in sides and price is not None and quantity is not None and Decimal(quantity)>0: + sides[level.side].append(price) + result.update(bid_price=max(sides[1],key=Decimal) if sides[1] else None,ask_price=min(sides[2],key=Decimal) if sides[2] else None,basis='SNAPSHOT_EVENT_POSITIVE_QUANTITY_LEVELS') + return result + +ALL_FEEDS='trade,quote,mark_index_price,book_delta,book_snapshot,bar' + + +def accepted(venue,symbol,feed,symbols,feeds): + return venue in ('BINANCE','OKX') and feed in feeds and (symbols is None or symbol in symbols) + + +def summarize(lines): + """Per feed/venue (BAR per interval): visibility upper bound and venue->DL receipt, ms; missing clocks counted.""" + def q(values,f): + values=sorted(values);return round(values[min(len(values)-1,int(f*len(values)))],1) if values else None + groups={};missing={} + for line in lines: + item=json.loads(line) + if item.get('kind')!='canonical_observation':continue + key='|'.join(str(x) for x in (item['feed'],item['identity']['venue'],item.get('interval') or '')) + source,received,seen=item.get('source_event_ns'),item.get('received_at_ns'),item.get('observer_receipt_wall_ns') + if not source or not seen or not received: + missing[key]=missing.get(key,0)+1;continue + g=groups.setdefault(key,{'visibility_ms':[],'venue_to_dl_ms':[]}) + g['visibility_ms'].append((seen-source)/1e6);g['venue_to_dl_ms'].append((received-source)/1e6) + out={} + for key,g in sorted(groups.items()): + out[key]={name:{'n':len(v),'p50':q(v,.5),'p95':q(v,.95),'p99':q(v,.99) if len(v)>=100 else None,'max':round(max(v),1), + 'negative':sum(1 for x in v if x<0)} for name,v in g.items()} + out[key]['missing_clock']=missing.get(key,0) + for key,count in missing.items(): + out.setdefault(key,{'missing_clock':count}) + return out + + +def should_emit(feed,mono,last,interval_ms): + return feed=='book_snapshot' or mono-last>=interval_ms*1_000_000 + +def identity_record(message,event,wall,mono): + kind,stamp=message.timestamp() + return {'kind':'canonical_observation','topic':message.topic(),'partition':message.partition(),'offset':message.offset(), + 'event_id_sha256':hashlib.sha256(bytes(event.event_id)).hexdigest() if event.event_id else None, + 'identity':{k:getattr(event,k) for k in ('instrument_uid','venue','market','product_type','native_symbol','source_id','source_session_id','connection_generation','source_sequence','partition_sequence','authority_revision','partition_plan_epoch')}, + 'feed':event.WhichOneof('payload'),'source_event_ns':event.source_event_time_ns or None, + 'received_at_ns':event.received_at_ns or None,'normalized_at_ns':event.normalized_at_ns or None,'published_at_ns':event.published_at_ns or None, + 'producer_timestamp_ms':stamp if kind==1 and stamp>=0 else None,'kafka_timestamp_type':kind, + 'observer_receipt_wall_ns':wall,'observer_receipt_monotonic_ns':mono, + 'boundary':'DURABLE_READ_COMMITTED_OBSERVATION_NOT_COMMIT_CLOCK', + 'interval':(str(getattr(event.bar,'interval','')) or None) if event.WhichOneof('payload')=='bar' else None} + +def worker(args): + from confluent_kafka import Consumer,ConsumerGroupTopicPartitions,TopicPartition,KafkaError,IsolationLevel + from confluent_kafka.admin import AdminClient,OffsetSpec + from qdl.marketdata.v2.market_data_pb2 import EventEnvelope + config=json.load(sys.stdin) + def emit(value):print(json.dumps(value,separators=(',',':')),flush=True) + clients={k:AdminClient(v) for k,v in config.items()} if args.mode in ('lag','both') else {} + reader=None; count=0; last_emit={}; seen={}; streak={}; begin=time.monotonic(); end=begin+args.seconds; next_lag=begin + symbols=set(args.symbols.split(',')) if args.symbols else None + feeds=set(args.feeds.split(',')) + if args.include_quote:feeds.add('quote') + emit({'kind':'scope','mode':args.mode,'seconds':args.seconds,'max_records':args.max_records,'filter':{'symbols':sorted(symbols) if symbols else 'ALL','feeds':sorted(feeds),'venues':['BINANCE','OKX']},'sample_ms_per_venue_feed':args.sample_ms,'snapshot_sampling':'ALL_UNTIL_GLOBAL_CAP','public_top_prices':args.include_top,'kafka_commits':False,'group_joins':False,'production_writes':0,'timestamp_note':'Kafka CreateTime is producer time, never transaction commit time. Observer clocks include fetch/scheduling delay.'}) + try: + if args.mode in ('canonical','both'): + c=dict(config['projector']) + c.update({'group.id':'kn-projector-v3-a','client.id':'dr07-assigned-readonly-observer','enable.auto.commit':False,'enable.auto.offset.store':False,'isolation.level':'read_committed','auto.offset.reset':'error','enable.partition.eof':True,'queued.max.messages.kbytes':1024,'fetch.max.bytes':1048576,'max.partition.fetch.bytes':262144,'fetch.wait.max.ms':100}) + reader=Consumer(c); assignments=[] + for partition in args.partitions: + low,high=reader.get_watermark_offsets(TopicPartition('md.canonical.v2',partition),timeout=3) + offset=max(low,high-args.tail_records) + assignments.append(TopicPartition('md.canonical.v2',partition,offset)) + emit({'kind':'assignment','topic':'md.canonical.v2','partition':partition,'offset':offset,'low':low,'high':high,'no_subscription':True}) + reader.assign(assignments) + emit({'kind':'READY','at_ns':time.time_ns(),'setup_seconds':time.monotonic()-begin}) + while time.monotonic()=next_lag: + next_lag+=10 + for role,group in GROUPS: + started=time.monotonic_ns() + try: + result=clients[role].list_consumer_group_offsets([ConsumerGroupTopicPartitions(group)],request_timeout=3)[group].result(4) + offsets=result.topic_partitions + if len(offsets)>36:raise ValueError('partition roster exceeded bound') + future=clients[role].list_offsets({TopicPartition(p.topic,p.partition):OffsetSpec.latest() for p in offsets},isolation_level=IsolationLevel.READ_UNCOMMITTED,request_timeout=3) + records=[] + for p in offsets: + key=(group,p.topic,p.partition) + e=future[TopicPartition(p.topic,p.partition)].result(4).offset + valid=p.error is None and p.offset>=0 and e>=p.offset + lag=e-p.offset if valid else None + prior=streak.get(key) + rises=prior[2]+1 if valid and prior and prior[0] is not None and lag>prior[0] and started-prior[1]<=15_000_000_000 else 0 + streak[key]=(lag,started,rises) + records.append({'topic':p.topic,'partition':p.partition,'committed':p.offset,'log_end':e,'lag_records':lag,'consecutive_increases':rises,'five_increases_alarm':rises>=5,'valid':valid}) + emit({'kind':'lag','at_ns':time.time_ns(),'group':group,'elapsed_ns':time.monotonic_ns()-started,'end_basis':'LOG_END_READ_UNCOMMITTED_NOT_LSO','rows':records}) + except Exception as exc: + for key in list(streak): + if key[0]==group:del streak[key] + error=exc.args[0] if exc.args else None + emit({'kind':'lag_unknown','at_ns':time.time_ns(),'group':group,'error_type':type(exc).__name__,'error_code':error.code() if hasattr(error,'code') else None}) + if reader and count=args.max_records: + reader.close();reader=None;emit({'kind':'canonical_record_cap_reached','records':count}) + if not clients:break + else:time.sleep(min(.1,max(0,end-time.monotonic()))) + emit({'kind':'DONE','at_ns':time.time_ns(),'records':count,'elapsed_seconds':time.monotonic()-begin,'live_semantics':'observed durable visibility, exact broker commit clock unavailable'}) + finally: + if reader is not None:reader.close() + +def launch(args): + if not args.launch:raise SystemExit('STAGED ONLY: parent coordination required; add --launch explicitly.') + args.output.mkdir(parents=True,exist_ok=False) + current=json.loads(subprocess.check_output(['docker','inspect',PROJECTOR],text=True))[0] + env=dict(item.split('=',1) for item in current['Config']['Env'] if '=' in item) + p={k:env[v] for k,v in {'bootstrap.servers':'QDL_KN_KAFKA_BOOTSTRAP','ssl.ca.location':'QDL_KN_KAFKA_CA','ssl.certificate.location':'QDL_KN_KAFKA_CERT','ssl.key.location':'QDL_KN_KAFKA_KEY'}.items()} + p.update({'security.protocol':'ssl','socket.timeout.ms':3000,'log_level':0}) + config={'projector':p} + for role in ('core','producer'): + config[role]={**p,'ssl.ca.location':f'/stable-certs/{role}/ca.crt','ssl.certificate.location':f'/stable-certs/{role}/client.crt','ssl.key.location':f'/stable-certs/{role}/client.key'} + command=['docker','exec','-i',QUERY,'python','-B','-c',Path(__file__).read_text(),'--worker','--mode',args.mode,'--seconds',str(args.seconds),'--partitions',','.join(map(str,args.partitions)),'--tail-records',str(args.tail_records),'--max-records',str(args.max_records),'--sample-ms',str(args.sample_ms)] + if args.include_quote:command.append('--include-quote') + if args.symbols:command += ['--symbols',args.symbols] + command += ['--feeds',args.feeds] + if args.include_top:command.append('--include-top') + process=subprocess.Popen(command,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True) + process.stdin.write(json.dumps(config));process.stdin.close() + written=0 + with (args.output/'observations.jsonl').open('x') as evidence: + for line in process.stdout: + written+=len(line.encode()) + if written>args.max_output_bytes: + process.terminate();raise RuntimeError('Evidence byte cap exceeded; inspect observer completion before further work') + evidence.write(line);evidence.flush() + item=json.loads(line) + if item['kind'] in ('READY','DONE','lag_unknown','canonical_error'):print(line.strip(),flush=True) + code=process.wait(timeout=15) + (args.output/'receipt.json').write_text(json.dumps({'exit':code,'output_bytes':written,'script_sha256':hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),'observer_only':True,'shared_writes':0,'payloads_retained':False,'public_top_prices_retained':args.include_top},indent=2)+'\n') + if code:raise SystemExit(code) + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--mode',choices=['lag','canonical','both'],default='both') + parser.add_argument('--seconds',type=int,default=60) + parser.add_argument('--partitions',default='0,1,2,3,4,5') + parser.add_argument('--tail-records',type=int,default=16) + parser.add_argument('--max-records',type=int,default=2000) + parser.add_argument('--sample-ms',type=int,default=100) + parser.add_argument('--include-quote',action='store_true') + parser.add_argument('--include-top',action='store_true') + parser.add_argument('--max-output-bytes',type=int,default=8*1024*1024) + parser.add_argument('--output',type=Path) + parser.add_argument('--launch',action='store_true') + parser.add_argument('--worker',action='store_true',help=argparse.SUPPRESS) + parser.add_argument('--symbols',default='',help='comma list of native symbols; empty = all') + parser.add_argument('--feeds',default=ALL_FEEDS,help='comma list of canonical payload kinds') + parser.add_argument('--summarize',type=Path,help='offline: summarize an observations.jsonl and exit') + args=parser.parse_args();args.partitions=[int(p) for p in args.partitions.split(',')] + if args.summarize: + print(json.dumps(summarize(args.summarize.read_text().splitlines()),indent=1));return + if set(args.feeds.split(','))-set(ALL_FEEDS.split(',')):parser.error('unknown feed') + if not 1<=args.seconds<=1200 or not 1<=args.max_records<=10000 or not 0<=args.tail_records<=100 or not 0<=args.sample_ms<=1000 or not args.partitions or any(p not in range(6) for p in args.partitions):parser.error('Bounded observer arguments violated') + if args.worker:worker(args) + else: + if args.output is None:parser.error('--output is required') + launch(args) +if __name__=='__main__':main() diff --git a/tests/test_p184_g4_observer.py b/tests/test_p184_g4_observer.py new file mode 100644 index 00000000..3441f190 --- /dev/null +++ b/tests/test_p184_g4_observer.py @@ -0,0 +1,39 @@ +"""P18.4 G4 observer: filter and offline summary (no Kafka).""" +import json +import unittest + +from scripts.p184_g4_observer import ALL_FEEDS, accepted, summarize + + +def obs(feed, venue, source, received, seen, interval=None): + return json.dumps({"kind": "canonical_observation", "feed": feed, "identity": {"venue": venue}, + "source_event_ns": source, "received_at_ns": received, "observer_receipt_wall_ns": seen, + "interval": interval}) + + +class ObserverTests(unittest.TestCase): + def test_default_filter_takes_every_symbol_and_feed_of_both_venues(self): + feeds = set(ALL_FEEDS.split(",")) + self.assertTrue(accepted("OKX", "DOGE-USDT-SWAP", "book_delta", None, feeds)) + self.assertTrue(accepted("BINANCE", "BNBUSDT", "bar", None, feeds)) + self.assertFalse(accepted("KRAKEN", "XBTUSD", "trade", None, feeds)) + self.assertFalse(accepted("OKX", "BTC-USDT-SWAP", "trade", {"ETH-USDT-SWAP"}, feeds)) + + def test_summary_groups_by_feed_venue_interval_and_counts_missing_and_negative(self): + lines = [obs("trade", "OKX", 1_000_000_000, 1_030_000_000, 1_150_000_000), + obs("trade", "OKX", 2_000_000_000, 2_020_000_000, 1_990_000_000), + obs("trade", "OKX", None, 1, 1), + obs("bar", "BINANCE", 3_000_000_000, 3_010_000_000, 3_200_000_000, interval="1m"), + json.dumps({"kind": "lag"})] + out = summarize(lines) + trade = out["trade|OKX|"] + self.assertEqual(trade["visibility_ms"]["n"], 2) + self.assertEqual(trade["visibility_ms"]["max"], 150.0) + self.assertEqual(trade["visibility_ms"]["negative"], 1) # skewed sample kept and counted, not clamped + self.assertEqual(trade["missing_clock"], 1) + self.assertEqual(out["bar|BINANCE|1m"]["venue_to_dl_ms"]["p50"], 10.0) + self.assertIsNone(trade["visibility_ms"]["p99"]) # p99 only with n >= 100 + + +if __name__ == "__main__": + unittest.main() From 52cff77939530cc3b753adbcbd9dbf8150c56890 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:22:52 +0000 Subject: [PATCH 065/147] docs(p184): D3 stage-100 measurement packet (windows A/B/C, not run) --- upgrade/evidence/p184-d3-stage100-packet.md | 65 +++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 upgrade/evidence/p184-d3-stage100-packet.md diff --git a/upgrade/evidence/p184-d3-stage100-packet.md b/upgrade/evidence/p184-d3-stage100-packet.md new file mode 100644 index 00000000..b354bad1 --- /dev/null +++ b/upgrade/evidence/p184-d3-stage100-packet.md @@ -0,0 +1,65 @@ +# P18.4 D3: Stage-100 Measurement Packet + +Status: PREPARED, NOT RUN. Each window needs its own owner approval. Owner decisions 2026-10-09 (journal "Owner decisions +for the DL closure"): +- G4/G5 baseline rule: G4 <= baseline p95 x 1.2; G5 uses baseline p95 x 1.5 for shapes without a stage-50 target. +- The load comes from a synthetic driver with real SDK sessions. +- Live/sandbox realms stay unchanged. + +## Tools (committed, tested locally) + +- `scripts/phase3_consumer_load_acceptance.py --mode target [--budget ]`. It is the certified stage-50 + driver. With `--budget` it runs stages 20/50/100 (600/600/1800 s), 40/30/20/10 classes at 100, and adds G5 COLD gates + (f831571). +- `scripts/p184_g4_observer.py`: read_committed assigned reader with no group join and no commit, all symbols/feeds + (a78696a). `--summarize` gives per feed/venue the visibility upper bound and venue->DL receipt. +- Identities: the alpha platform identities `alpha.binance.paper.stable` / `alpha.okx.paper.stable`, set + `~/.local/state/qdl-v2/r135-local-batch-fair-e905747-20260920T135150Z/identities`. + - TLS chain verified against the deployed query-client CA bundle; expiry 2026-11-20 / 2026-12-19. + - JWT key ids `stable-alpha-{binance,okx}-rs256-v1` are present in the deployed jwt-config. +- Client containers: driver default 512 MiB / 1 CPU per worker, up to 4 workers, `--rm`. Image + `qdl-v2-python:2.2.2-d6d2637` (sha256:514506122111), runtime dir `releases/v2.2.0-02cd827/runtime` read-only. + +## Window A - baseline on the current runtime (no config change) + +1. G4 observer, `--mode canonical`, all feeds, 4 x 120 s windows, `--sample-ms 1000`, `--max-records 10000`, output + `~/.local/state/qdl-v2/p184-d3-baseline-/g4-/`. Load: one assigned consumer inside query_kn_1 for 8 min. +2. Driver stage 50 with the v2.1.1 budget (`--mode target --sessions 50 --duration-seconds 300`). It is the certified + stage-50 workload on the deployed alpha quotas (2400 RPM / 60 streams fit stage 50), with real data and the TS writer + watched. +3. Freeze `config/v2/p184-stage100-budget.json` from these results, committed BEFORE any candidate load: + - G5 COLD_2500/COLD_5000 per venue = baseline max x 1.5 (n = 1 per venue/rows, small-sample rule); + - the latency classes keep the certified stage-50 targets; + - TS demanded routes 64; + - the G4 baseline table is attached. + +Stop: any Query/Stream/projector restart or OOM, TS writer not READY, 429 on the TS identity, disk < 20 GB free. +Rollback: none needed (no config change); stop the driver/observer. + +## Window B - capacity configuration (separate approval) + +1. Alpha manifests `alpha.binance.paper.stable` and `alpha.okx.paper.stable`: revision 14 -> 15, `requests_per_minute` + 2400 -> 4950, `max_streams` 60 -> 300 (capacity100 candidate). Same rendering method as D1: new realm dir copied from + the active rev11 dir, Query manifest list paths for the two alpha files, new gateway bundle with parity proof, no other + change. +2. Stream `QDL_KN_MAX_SUBSCRIPTIONS` 384 -> 1024 on stream_kn_1/2. MemoryPlan for 1024 + 32 replay is 307 MiB modeled, + under the unchanged 512 MiB cap; this is not a measured RSS claim. +3. Readers are recreated one at a time (same images), as in the D1 window. No alpha caller runs today. Before Phase E an + alpha must mint JWT revision 15 (documented hand-off). + +Rollback: the rev11 realm dir and the 384 env, per reader (saved composes). + +## Window C - ramp 20 / 50 / 100 (separate approval) + +`--mode target --budget config/v2/p184-stage100-budget.json --sessions {20,50,100}` with durations 600/600/1800 s. The +final stage 100 includes the BURST / SLOW_READER / RECONNECT windows. The G4 observer runs during stage 100. The driver +monitor samples CPU, throttling, memory and restarts of every DL role plus the clients (G14). + +Stop at the first: internal errors, unbounded queue growth, an unsafe execution eligibility, TS writer not READY 64/64, +Query/Stream/projector OOM or restart, a 429 on the TS identity, disk < 20 GB. Each stage must pass before the next. + +## Open question for the owner + +The target driver implements the realistic (mixed) profile. The capacity100 stress profile (500 alpha streams) is not +implemented in it. Options: a driver extension plus a stress window (more work), or the mixed profile plus the final +fault windows as acceptance and stress reported only through the planner arithmetic. From fc224914dad569daf9865d3555af6f494d38459d Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:29:59 +0000 Subject: [PATCH 066/147] feat(p184): freeze the stage-100 budget from a baseline (G5 x1.5, G4 x1.2, TS 64 routes) --- scripts/p184_freeze_stage100_budget.py | 86 ++++++++++++++++++++++++++ tests/test_p184_stage100.py | 26 ++++++++ 2 files changed, 112 insertions(+) create mode 100644 scripts/p184_freeze_stage100_budget.py diff --git a/scripts/p184_freeze_stage100_budget.py b/scripts/p184_freeze_stage100_budget.py new file mode 100644 index 00000000..adf0469f --- /dev/null +++ b/scripts/p184_freeze_stage100_budget.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Freeze the P18.4 stage-100 budget from a baseline (owner rule 2026-10-09), BEFORE any candidate load. + +Inputs: +- the certified v2.1.1 target budget (latency classes, request/stream/startup gates); +- the baseline stage-50 target receipt (cold reads 2500/5000 per venue); +- the G4 observer summaries of the baseline windows. + +Output: config/v2/p184-stage100-budget.json (schema qdl.p184.stage100-budget.v1): +- P18.4 stages 20/50/100; +- G5 COLD_ limit = max(baseline cold ms over both venues) x 1.5; +- G4 table: candidate p95 limit = baseline p95 x 1.2 per feed/venue(/interval); +- TS demanded routes 64. +Pure file transform: reads JSON, writes one new file (never overwrites), no runtime access.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +from qdl.certification.phase3_consumer_load import P184_STAGE_MIX, P184_STAGE_SECONDS, load_p184_stage100_budget + +G5_FACTOR = 1.5 +G4_FACTOR = 1.2 + + +def freeze(base: dict, receipt: dict, g4_summaries: list[dict], *, ts_routes: int, sources: dict) -> dict: + cold = [item for item in receipt["cold"] if not item.get("error") and isinstance(item.get("ms"), (int, float))] + shapes = {} + for rows in (2500, 5000): + values = [item["ms"] for item in cold if item.get("rows") == rows] + if not values: + raise ValueError(f"baseline has no successful COLD_{rows} read; cannot freeze G5") + shapes[f"COLD_{rows}"] = {"p95": round(max(values) * G5_FACTOR, 1), "baseline_max_ms": max(values), + "baseline_n": len(values), "rule": "baseline max x 1.5 (n per venue small)"} + g4 = {} + for summary in g4_summaries: + for key, value in summary.items(): + vis = value.get("visibility_ms") + if not vis: + continue + g4.setdefault(key, []).append(vis["p95"]) + g4_table = {key: {"baseline_p95_ms_windows": values, "baseline_p95_ms": max(values), + "candidate_p95_limit_ms": round(max(values) * G4_FACTOR, 1)} + for key, values in sorted(g4.items())} + budget = { + **{k: v for k, v in base.items() if k not in {"schema", "frozen_at", "stages", "source", "note"}}, + "schema": "qdl.p184.stage100-budget.v1", + "frozen_before_candidate": True, + "note": "Frozen from the baseline before any candidate load (owner 2026-10-09). A missed limit is an open exit " + "to optimize, never a threshold edited after results.", + "stages": {str(k): {"mix": list(P184_STAGE_MIX[k]), "seconds": P184_STAGE_SECONDS[k]} for k in P184_STAGE_MIX}, + "g5_shape_limits_ms": shapes, + "g4_delivery_visibility": {"rule": "candidate p95 <= baseline p95 x 1.2 per feed/venue(/interval); " + "read_committed observation is a visibility upper bound", + "table": g4_table}, + "trading_system": {**base["trading_system"], "demanded_routes": ts_routes}, + "sources": sources, + } + load_p184_stage100_budget(budget) + return budget + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--base", type=Path, default=Path("config/v2/v211-target-acceptance-budget.json")) + parser.add_argument("--receipt", type=Path, required=True) + parser.add_argument("--g4", type=Path, nargs="+", required=True, help="observer --summarize JSON files") + parser.add_argument("--ts-routes", type=int, default=64) + parser.add_argument("--output", type=Path, default=Path("config/v2/p184-stage100-budget.json")) + args = parser.parse_args() + digest = lambda path: hashlib.sha256(path.read_bytes()).hexdigest() + sources = {"base": digest(args.base), "receipt": digest(args.receipt), "g4": [digest(p) for p in args.g4]} + budget = freeze(json.loads(args.base.read_text()), json.loads(args.receipt.read_text()), + [json.loads(p.read_text()) for p in args.g4], ts_routes=args.ts_routes, sources=sources) + with args.output.open("x") as handle: + json.dump(budget, handle, indent=1, sort_keys=True) + handle.write("\n") + print(json.dumps({"output": str(args.output), "g5": budget["g5_shape_limits_ms"], + "g4_rows": len(budget["g4_delivery_visibility"]["table"])})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_p184_stage100.py b/tests/test_p184_stage100.py index d307336f..5cd5dde5 100644 --- a/tests/test_p184_stage100.py +++ b/tests/test_p184_stage100.py @@ -85,3 +85,29 @@ def test_a_missing_or_failed_cold_read_fails_its_shape_gate(self): if __name__ == "__main__": unittest.main() + + +class FreezeBudgetTests(unittest.TestCase): + def test_freeze_derives_g5_and_g4_from_the_baseline_and_validates(self): + import json + from pathlib import Path + from scripts.p184_freeze_stage100_budget import freeze + base = json.loads(Path("config/v2/v211-target-acceptance-budget.json").read_text()) + receipt = {"cold": [{"venue": "BINANCE", "rows": 2500, "ms": 1000.0}, {"venue": "OKX", "rows": 2500, "ms": 2000.0}, + {"venue": "OKX", "rows": 5000, "ms": 3000.0}, + {"venue": "BINANCE", "rows": 5000, "ms": 9.0, "error": {"type": "X"}}]} + g4 = [{"trade|OKX|": {"visibility_ms": {"p95": 200.0}}}, {"trade|OKX|": {"visibility_ms": {"p95": 250.0}}}] + budget = freeze(base, receipt, g4, ts_routes=64, sources={}) + self.assertEqual(budget["g5_shape_limits_ms"]["COLD_2500"]["p95"], 3000.0) + self.assertEqual(budget["g5_shape_limits_ms"]["COLD_5000"]["p95"], 4500.0) # the failed read is excluded + self.assertEqual(budget["g4_delivery_visibility"]["table"]["trade|OKX|"]["candidate_p95_limit_ms"], 300.0) + self.assertEqual(budget["trading_system"]["demanded_routes"], 64) + self.assertEqual(budget["latency_ms"], base["latency_ms"]) # certified stage-50 targets kept + + def test_freeze_refuses_without_a_successful_cold_read(self): + import json + from pathlib import Path + from scripts.p184_freeze_stage100_budget import freeze + base = json.loads(Path("config/v2/v211-target-acceptance-budget.json").read_text()) + with self.assertRaisesRegex(ValueError, "COLD_5000"): + freeze(base, {"cold": [{"venue": "OKX", "rows": 2500, "ms": 1.0}]}, [], ts_routes=64, sources={}) From 06b77e08519edf0b86e6c8cb792d8d1e6cf81c5e Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:30:37 +0000 Subject: [PATCH 067/147] feat(p184): quota-only manifest revision renderer for D3 window B --- scripts/p184_render_quota_revision.py | 68 ++++++++++++++++++++++++ tests/test_p184_render_quota_revision.py | 30 +++++++++++ 2 files changed, 98 insertions(+) create mode 100644 scripts/p184_render_quota_revision.py create mode 100644 tests/test_p184_render_quota_revision.py diff --git a/scripts/p184_render_quota_revision.py b/scripts/p184_render_quota_revision.py new file mode 100644 index 00000000..4fd49f69 --- /dev/null +++ b/scripts/p184_render_quota_revision.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Render a consumer manifest revision that only RAISES quotas (P18.4 D3 window B: alpha capacity100 candidate). + +Takes the deployed manifest YAML and writes a new file in which only `metadata.revision` (+1) and the named quotas +(`requests_per_minute`, `max_streams`, upward only) change. It refuses anything else and validates the result with the +existing ConsumerManifestLoader. The caller then compiles the gateway bundle with scripts/kn_gateway_bundle.py and +proves parity, as in D1. Pure file transform, never overwrites, no runtime access.""" +from __future__ import annotations + +import argparse +import copy +import json +from pathlib import Path + +import yaml + +from qdl.consumer.manifest import ConsumerManifestLoader + +QUOTA_KEYS = frozenset({"requests_per_minute", "max_streams"}) + + +def render(manifest: dict, quotas: dict[str, int]) -> dict: + if not quotas or set(quotas) - QUOTA_KEYS: + raise ValueError("only requests_per_minute/max_streams may change") + current = manifest["spec"]["quotas"] + if any(type(v) is not int or v <= current[k] for k, v in quotas.items()): + raise ValueError("quota change must raise the named quotas") + target = copy.deepcopy(manifest) + target["metadata"]["revision"] += 1 + target["spec"]["quotas"].update(quotas) + ConsumerManifestLoader.from_mapping(target) + diff = _diff(manifest, target) + expected = {("metadata", "revision")} | {("spec", "quotas", k) for k in quotas} + if set(diff) != expected: + raise ValueError(f"unexpected manifest difference: {sorted(diff)}") + return target + + +def _diff(a, b, path=()): + if isinstance(a, dict) and isinstance(b, dict): + out = {} + for key in set(a) | set(b): + out.update(_diff(a.get(key), b.get(key), path + (key,))) + return out + return {} if a == b else {path: (a, b)} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--requests-per-minute", type=int) + parser.add_argument("--max-streams", type=int) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + quotas = {k: v for k, v in (("requests_per_minute", args.requests_per_minute), + ("max_streams", args.max_streams)) if v is not None} + source = yaml.safe_load(args.manifest.read_text()) + target = render(source, quotas) + with args.output.open("x") as handle: + handle.write(yaml.safe_dump(target, sort_keys=False)) + parsed = ConsumerManifestLoader.from_mapping(target) + print(json.dumps({"consumer_id": parsed.consumer_id, "revision": parsed.manifest_revision, + "quotas": target["spec"]["quotas"], "manifest_sha256": parsed.manifest_sha256})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_p184_render_quota_revision.py b/tests/test_p184_render_quota_revision.py new file mode 100644 index 00000000..ed28e3c4 --- /dev/null +++ b/tests/test_p184_render_quota_revision.py @@ -0,0 +1,30 @@ +"""P18.4 D3 window B renderer: only the revision and the named quotas may change, upward only.""" +from pathlib import Path +import unittest + +import yaml + +from scripts.p184_render_quota_revision import render + +ROOT = Path(__file__).resolve().parents[1] + + +class RenderQuotaRevisionTests(unittest.TestCase): + def setUp(self): + self.manifest = yaml.safe_load((ROOT / "consumers/stable/alpha-binance-paper.yaml").read_text()) + + def test_only_revision_and_named_quotas_change(self): + out = render(self.manifest, {"requests_per_minute": 4950, "max_streams": 300}) + self.assertEqual(out["metadata"]["revision"], self.manifest["metadata"]["revision"] + 1) + self.assertEqual((out["spec"]["quotas"]["requests_per_minute"], out["spec"]["quotas"]["max_streams"]), (4950, 300)) + self.assertEqual(out["spec"]["requirements"], self.manifest["spec"]["requirements"]) + self.assertEqual(out["spec"]["quotas"]["max_warmup_rows"], self.manifest["spec"]["quotas"]["max_warmup_rows"]) + + def test_lowering_other_keys_or_nothing_is_refused(self): + for bad in ({}, {"requests_per_minute": 1}, {"max_batch_items": 99}, {"max_streams": "300"}): + with self.subTest(bad=bad), self.assertRaises(ValueError): + render(self.manifest, bad) + + +if __name__ == "__main__": + unittest.main() From ed2b28437c1f7e6bad8c60e983fefa38d0b6a460 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:31:33 +0000 Subject: [PATCH 068/147] fix(load): declared-rate ticker no longer offers a phantom tick at the window end DeclaredRateTicker.offered absorbed float noise with 1e-9 of a period. The driver's start/end are time.monotonic() values (host uptime); once uptime passes 2**21 s (~24 days) (start + 0.2) - start rounds to 0.20000000019, the quotient becomes 4.0000000037 and a fifth tick due exactly at `end` was offered (and, for a rejecting server, sent and counted). This is why test_a_typed_rejection_is_counted_by_code saw 5 RATE_LIMITED instead of 4 on this host (uptime 2.12e6 s). The ledger contract is "every due time before end", so the product was wrong, not the test. A due time within 1 us of `end` is now treated as at the boundary. Adds a deterministic regression at long-uptime start values (fails on the old formula at 2123918.89 and 2**21 + 0.02). Tests (qdl-v2 test image sha256:77682eea, --network none): target driver 32, consumer load 6, consumer load driver 32, target workload 11, stage100 7 - all OK. --- qdl/certification/phase3_consumer_load.py | 10 ++++++++-- tests/test_phase3_target_driver.py | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/qdl/certification/phase3_consumer_load.py b/qdl/certification/phase3_consumer_load.py index e85aff76..9e49eb78 100644 --- a/qdl/certification/phase3_consumer_load.py +++ b/qdl/certification/phase3_consumer_load.py @@ -667,11 +667,17 @@ def __init__(self, *, start: float, period: float, phase: float, end: float) -> self._end = end self._next = 0 + # A due time within this many seconds of ``end`` is at the boundary, not before it. Monotonic clocks are host + # uptime: past 2**21 s (~24 days) ``(start + 0.2) - start`` carries ~2e-10 s of rounding, which a tolerance on + # the period quotient (1e-9 of a period) does not absorb and which offered one phantom tick due exactly at ``end``. + _BOUNDARY_S = 1e-6 + @property def offered(self) -> int: - if self._first >= self._end: + window = self._end - self._first - self._BOUNDARY_S + if window <= 0: return 0 - return math.ceil((self._end - self._first) / self._period - 1e-9) + return math.ceil(window / self._period) def take(self, now: float) -> tuple[float | None, int]: missed = 0 diff --git a/tests/test_phase3_target_driver.py b/tests/test_phase3_target_driver.py index da656451..2c317d04 100644 --- a/tests/test_phase3_target_driver.py +++ b/tests/test_phase3_target_driver.py @@ -65,6 +65,13 @@ def test_offered_counts_every_due_tick_inside_the_window(self): self.assertEqual(DeclaredRateTicker(start=0.0, period=1.0, phase=phase, end=90.0).offered, 90) self.assertEqual(DeclaredRateTicker(start=10.0, period=60.0, phase=5.0, end=310.0).offered, 5) + def test_a_tick_due_at_the_end_is_not_offered_on_a_long_uptime_clock(self): + # Past 2**21 s of monotonic uptime (start + 0.2) - start rounds to 0.2000000002: still 4 ticks, not 5. + for start in (0.0, 2123918.89, 2.0 ** 21 + 0.02, 1e7 + 0.02): + self.assertEqual(DeclaredRateTicker(start=start, period=0.05, phase=0.0, end=start + 0.2).offered, 4) + self.assertEqual(DeclaredRateTicker(start=start, period=0.05, phase=0.0, end=start + 0.5).offered, 10) + self.assertEqual(DeclaredRateTicker(start=start, period=0.05, phase=0.0, end=start + 0.21).offered, 5) + def test_a_client_a_full_period_behind_misses_ticks_it_cannot_retime(self): ticker = DeclaredRateTicker(start=0.0, period=1.0, phase=0.0, end=10.0) self.assertEqual(ticker.take(0.5), (0.0, 0)) From c458fbf52536f2ca238d719bf38a70c6cd481b39 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:33:09 +0000 Subject: [PATCH 069/147] build(p184): require IDR-084 in the consolidated recipe; journal D3/D5 tooling slices --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 18 ++++++++++++++++++ scripts/p184_consolidated_build.py | 3 ++- tests/test_p184_consolidated_build.py | 3 ++- upgrade/evidence/p184-consolidated-build.md | 5 +++-- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index c5449ca4..bf4092d7 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71304,3 +71304,21 @@ Status IMPLEMENTED / TESTED_LOCALLY (no traffic, no runtime change). fix in the driver's own tests. - The stage-100 plan does not fit the deployed alpha manifests (2400 RPM / 60 streams). The capacity100 candidate quota (4950 / 300) carries it. That runtime packet is the next D3 slice. + +### D3/D5 slices: ticker fix, budget freeze tooling, quota renderer, recipe prerequisite - 2026-10-09 (Claude) + +Status IMPLEMENTED / TESTED_LOCALLY. +- ed2b284 (cherry-picked from a subagent worktree, which has since been removed): DeclaredRateTicker offered a phantom tick + at the window end on long-uptime hosts. time.monotonic() is about 2.1e6 s here, so `(start+0.2)-start` = 0.20000000019 + and the quotient exceeded the 1e-9 tolerance. A due time within 1 us of the end now counts as the boundary; a + regression test was added. This is the root cause of the "pre-existing" test_a_typed_rejection_is_counted_by_code + failure noted in D3 slice 1, which was a product bug, not the test. Suites: 112 OK. +- fc22491 scripts/p184_freeze_stage100_budget.py: v2.1.1 latency classes kept, G5 COLD_ = baseline max x 1.5, + G4 table = baseline p95 x 1.2, TS routes 64; it validates through load_p184_stage100_budget. 9 tests OK. +- 06b77e0 scripts/p184_render_quota_revision.py: revision +1 and named quotas upward only; any other difference is + refused. The repo alpha manifests equal the copies baked in Query 514506122111 (sha d201b2e7 / e8fb3ea7). +- Recipe: PREREQUISITES + IDR084_hot_mark 922a6ec, and the kn_query_backend contract marker + `binding.feed is FeedType.MARK_INDEX_PRICE`; tests.test_p184_consolidated_build 20 OK. The preflight at HEAD stays HOLD. + Pending gates: TS integration receipt, exact-source DL CI, owner build approval. +- Since 0b374bc, served runtime changed only in 922a6ec. 76db472/522cf3e/f831571/ed2b284 are scripts or certification + tooling, listed for the release notes. diff --git a/scripts/p184_consolidated_build.py b/scripts/p184_consolidated_build.py index 0459a967..0fb1481a 100644 --- a/scripts/p184_consolidated_build.py +++ b/scripts/p184_consolidated_build.py @@ -17,6 +17,7 @@ "IDR075": "704552d8736905cdb73feec86f3f01d339d02cbb", "DR05_book_fence": "d68fb2ff47622db8224dc4b9e004dca4e20f643e", "DR08_terminal_status": "0b374bc918af02476cdfb1502d5df132426f901b", + "IDR084_hot_mark": "922a6ec38a8da94d30c634f025fd23c03838c24d", } CONTRACTS = { "rust/qdl-realtime-core/src/lib.rs": ["REST_AHEAD_OF_STREAM"], @@ -24,7 +25,7 @@ "rust/qdl-kafka/src/bin/qdl-production-core.rs": [], "rust/qdl-projector/src/products.rs": [], "qdl/runtime/kn_hot_view.py": [], - "qdl/runtime/kn_query_backend.py": [], + "qdl/runtime/kn_query_backend.py": ["binding.feed is FeedType.MARK_INDEX_PRICE"], "rust/qdl-stream-gateway/src/service.rs": ["terminal.get_or_insert(status)"], "rust/qdl-stream-gateway/tests/native_stream.rs": [ "t05_prolonged_trade_stall_preserves_terminal_status_and_peer_progress", diff --git a/tests/test_p184_consolidated_build.py b/tests/test_p184_consolidated_build.py index 060e5ac7..4ff50414 100644 --- a/tests/test_p184_consolidated_build.py +++ b/tests/test_p184_consolidated_build.py @@ -93,7 +93,8 @@ def test_dependency_reuse_is_python_only_not_source_overlay(self): def test_prerequisite_roster(self): self.assertEqual(set(build.PREREQUISITES), - {"IDR040fix1", "bindingV3", "quota_log", "b6a0119", "IDR075", "DR05_book_fence", "DR08_terminal_status"}) + {"IDR040fix1", "bindingV3", "quota_log", "b6a0119", "IDR075", "DR05_book_fence", "DR08_terminal_status", + "IDR084_hot_mark"}) class IntegrationScopeTests(NoProcessTests): diff --git a/upgrade/evidence/p184-consolidated-build.md b/upgrade/evidence/p184-consolidated-build.md index dbca5dd6..8a2ea3ca 100644 --- a/upgrade/evidence/p184-consolidated-build.md +++ b/upgrade/evidence/p184-consolidated-build.md @@ -13,8 +13,9 @@ completion of Paper/OR features. Final DR/broker acceptance and exact-source DL remain mandatory; existing receipts do not certify later source automatically. The recipe now also requires DR05 fence d68fb2ff47622db8224dc4b9e004dca4e20f643e and DR08 terminal-status 0b374bc918af02476cdfb1502d5df132426f901b, including the -changed core/projector/read-view paths and prolonged-stall regressions. Seven -prerequisite entries and 24 source paths are inventoried. Older candidate d68fb2f +changed core/projector/read-view paths and prolonged-stall regressions. Eight +prerequisite entries (2026-10-09: + IDR084_hot_mark 922a6ec, the only served-runtime change since 0b374bc, with its +kn_query_backend contract marker) and 24 source paths are inventoried. Older candidate d68fb2f is refused because terminal-status ancestry is absent. Affected portable suite: 20 PASS, 0 FAIL, 0 SKIP. The real Git preflight against From 82a71a2b4b317d6e278e1e828141a14585dc16f7 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:38:05 +0000 Subject: [PATCH 070/147] fix(load): sample the actual TS writer (container, env, log files); freeze the stage-100 budget from window A --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 38 +++ config/v2/p184-stage100-budget.json | 336 +++++++++++++++++++++ scripts/phase3_consumer_load_acceptance.py | 42 ++- 3 files changed, 404 insertions(+), 12 deletions(-) create mode 100644 config/v2/p184-stage100-budget.json diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index bf4092d7..dfff6794 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71322,3 +71322,41 @@ Status IMPLEMENTED / TESTED_LOCALLY. Pending gates: TS integration receipt, exact-source DL CI, owner build approval. - Since 0b374bc, served runtime changed only in 922a6ec. 76db472/522cf3e/f831571/ed2b284 are scripts or certification tooling, listed for the release notes. + +### D3 window A (baseline) executed; stage-100 budget frozen - 2026-10-09 (Claude) + +Status: owner-approved window A, runtime unchanged. Budget FROZEN before any candidate load. +- G4 observer (scripts/p184_g4_observer.py, assigned read_committed, no join/commit), 4 x 120 s, 18:25:54-18:34:28Z. + Window 1 was idle; windows 2-4 overlapped the stage-50 run. 0 missing clocks, 0 negative samples. Per feed/venue p95 + visibility (max of the four windows): + - trade 224/215 ms; quote 244/236; book_delta 219/241; book_snapshot 217/225; + - mark_index_price 309/1322 (Binance/OKX). The OKX composite carries the older (index) component time. + - OKX bar 1m 61.5 s: the OKX bar source time is the bar open. The same metric is used for baseline and candidate. +- Stage-50 target run (v2.1.1 budget, alpha.binance/okx.paper.stable, image 514506122111, 4 workers), 300 s steady + + windows: + - every DL gate PASS. Latency p95 BAR 27/31, L2 37/75, MARK ref 24/28, QUOTE 34/39, TRADE 34/28 ms (Binance/OKX). + - requests, streams and cold all PASS. Cold 2500: 2337/4177 ms; cold 5000: 6572/10178 ms. + - The TS gates FAILED for a measurement reason: the driver sampled container market_data_service via core.config and + counted disconnects from docker logs only, so 0 TS samples were read. The writer is market_data_service_p184w, uses + shared.config and logs to /app/logs/*.log. + - Independent watcher (watch.log, 19 samples every 20 s, 18:29:22-18:35:30Z): TS READY 64/64 throughout, readers and + projectors restarts 0, disk 87 GB. + - With the fixed counter: TS disconnects 0 in the run window (18:28-18:36) and 0 in the 10 min before. +- Driver fix (this commit): optional profile field ts_container; the heartbeat reads TRADING_REDIS_URL from the writer + environment; disconnects also read the writer's log files within the window. Checked live: heartbeat READY 64/64; + log-file counting finds 105 disconnects in an earlier window that had rolls. Driver suites 79 OK. +- Frozen config/v2/p184-stage100-budget.json (scripts/p184_freeze_stage100_budget.py): + - G5 COLD_2500 <= 6265.1 ms, COLD_5000 <= 15267.0 ms (baseline max x 1.5); + - 17 G4 rows, candidate p95 <= baseline x 1.2; + - v2.1.1 latency targets kept; TS 64 routes. +- Evidence ~/.local/state/qdl-v2/p184-d3-baseline-20261009/ (file, sha256 prefix): + - stage50/receipt.json 43ecd594aee10c3e + - stage50/gates.json 5ebec6252e9d1c96 + - g4-1-summary.json 09bd7219c41e570a + - g4-2-summary.json 83c8ab8c3630bde9 + - g4-3-summary.json f7174be2ebeaf4d9 + - g4-4-summary.json a902171d065a9352 + - watch.log 2c03e0528088e821 + - profile-stage50.json 878c069f93dd63cc +- Cleanup: driver client containers --rm (none left); the watcher was stopped; the observer ran inside query_kn_1 with no + leftover process. diff --git a/config/v2/p184-stage100-budget.json b/config/v2/p184-stage100-budget.json new file mode 100644 index 00000000..a20ed7ab --- /dev/null +++ b/config/v2/p184-stage100-budget.json @@ -0,0 +1,336 @@ +{ + "cold": { + "max_errors": 0, + "per_venue": true, + "rows": [ + 2500, + 5000 + ], + "rows_must_equal_request": true + }, + "final": { + "burst": { + "at_fraction": 0.4, + "extra_fraction": 0.25, + "seconds": 10 + }, + "reconnect": { + "at_fraction": 0.7, + "fraction": 0.1 + }, + "slow_reader": { + "at_fraction": 0.2, + "pause_seconds": 5, + "streams": 1 + } + }, + "frozen_before_candidate": true, + "g4_delivery_visibility": { + "rule": "candidate p95 <= baseline p95 x 1.2 per feed/venue(/interval); read_committed observation is a visibility upper bound", + "table": { + "bar|BINANCE|15m": { + "baseline_p95_ms": 33940.6, + "baseline_p95_ms_windows": [ + 33940.6 + ], + "candidate_p95_limit_ms": 40728.7 + }, + "bar|BINANCE|1m": { + "baseline_p95_ms": 214.7, + "baseline_p95_ms_windows": [ + 214.7, + 190.2, + 155.5, + 146.6 + ], + "candidate_p95_limit_ms": 257.6 + }, + "bar|BINANCE|30m": { + "baseline_p95_ms": 21530.5, + "baseline_p95_ms_windows": [ + 21530.5 + ], + "candidate_p95_limit_ms": 25836.6 + }, + "bar|BINANCE|3m": { + "baseline_p95_ms": 22727.7, + "baseline_p95_ms_windows": [ + 8968.9, + 22727.7, + 9862.3 + ], + "candidate_p95_limit_ms": 27273.2 + }, + "bar|OKX|1m": { + "baseline_p95_ms": 61514.3, + "baseline_p95_ms_windows": [ + 61514.3, + 61359.2, + 61385.2, + 61109.4 + ], + "candidate_p95_limit_ms": 73817.2 + }, + "bar|OKX|30m": { + "baseline_p95_ms": 1800657.3, + "baseline_p95_ms_windows": [ + 1800657.3 + ], + "candidate_p95_limit_ms": 2160788.8 + }, + "bar|OKX|3m": { + "baseline_p95_ms": 181318.4, + "baseline_p95_ms_windows": [ + 181318.4, + 180839.5 + ], + "candidate_p95_limit_ms": 217582.1 + }, + "book_delta|BINANCE|": { + "baseline_p95_ms": 219.4, + "baseline_p95_ms_windows": [ + 219.4, + 164.8, + 161.0, + 160.1 + ], + "candidate_p95_limit_ms": 263.3 + }, + "book_delta|OKX|": { + "baseline_p95_ms": 240.9, + "baseline_p95_ms_windows": [ + 240.9, + 174.5, + 172.1, + 173.5 + ], + "candidate_p95_limit_ms": 289.1 + }, + "book_snapshot|BINANCE|": { + "baseline_p95_ms": 216.7, + "baseline_p95_ms_windows": [ + 216.7, + 160.5, + 150.5, + 150.1 + ], + "candidate_p95_limit_ms": 260.0 + }, + "book_snapshot|OKX|": { + "baseline_p95_ms": 225.1, + "baseline_p95_ms_windows": [ + 225.1, + 169.2, + 164.8, + 157.8 + ], + "candidate_p95_limit_ms": 270.1 + }, + "mark_index_price|BINANCE|": { + "baseline_p95_ms": 309.1, + "baseline_p95_ms_windows": [ + 309.1, + 219.9, + 210.4, + 215.6 + ], + "candidate_p95_limit_ms": 370.9 + }, + "mark_index_price|OKX|": { + "baseline_p95_ms": 1322.2, + "baseline_p95_ms_windows": [ + 1322.2, + 1289.4, + 1296.2, + 1195.2 + ], + "candidate_p95_limit_ms": 1586.6 + }, + "quote|BINANCE|": { + "baseline_p95_ms": 243.9, + "baseline_p95_ms_windows": [ + 243.9, + 167.3, + 163.8, + 177.2 + ], + "candidate_p95_limit_ms": 292.7 + }, + "quote|OKX|": { + "baseline_p95_ms": 236.1, + "baseline_p95_ms_windows": [ + 236.1, + 180.3, + 182.1, + 181.5 + ], + "candidate_p95_limit_ms": 283.3 + }, + "trade|BINANCE|": { + "baseline_p95_ms": 224.4, + "baseline_p95_ms_windows": [ + 224.4, + 173.5, + 168.2, + 175.5 + ], + "candidate_p95_limit_ms": 269.3 + }, + "trade|OKX|": { + "baseline_p95_ms": 214.5, + "baseline_p95_ms_windows": [ + 214.5, + 166.7, + 165.5, + 163.2 + ], + "candidate_p95_limit_ms": 257.4 + } + } + }, + "g5_shape_limits_ms": { + "COLD_2500": { + "baseline_max_ms": 4176.758, + "baseline_n": 2, + "p95": 6265.1, + "rule": "baseline max x 1.5 (n per venue small)" + }, + "COLD_5000": { + "baseline_max_ms": 10178.021, + "baseline_n": 2, + "p95": 15267.0, + "rule": "baseline max x 1.5 (n per venue small)" + } + }, + "latency_ms": { + "BAR_LATEST": { + "feed": "BAR", + "operation": "SNAPSHOT", + "p95": 1000, + "p99": 2000 + }, + "L2_SNAPSHOT": { + "feed": "BOOK_SNAPSHOT", + "operation": "SNAPSHOT", + "p95": 300, + "p99": 750 + }, + "MARK_INDEX_REFERENCE": { + "feed": "MARK_INDEX_PRICE", + "operation": "REFERENCE_BATCH", + "p95": 250, + "p99": 500 + }, + "QUOTE_SNAPSHOT": { + "feed": "QUOTE", + "operation": "SNAPSHOT", + "p95": 100, + "p99": 250 + }, + "TRADE_SNAPSHOT": { + "feed": "TRADE", + "operation": "SNAPSHOT", + "p95": 100, + "p99": 250 + } + }, + "latency_scope": "Gated per latency class and venue on the steady window, across symbols and replicas; every venue/symbol/replica group is reported with its own count.", + "note": "Frozen from the baseline before any candidate load (owner 2026-10-09). A missed limit is an open exit to optimize, never a threshold edited after results.", + "probes": { + "feeds": [ + "TRADE", + "BOOK_SNAPSHOT", + "BAR" + ], + "note": "Per-venue latency probes, rotating the five symbols; not part of the 50-alpha offered load.", + "period_seconds": 5.0 + }, + "requests": { + "client_validity_scheduler_lag_p99_ms": 250, + "max_failed_fault_windows": 0, + "max_failed_steady": 0, + "max_missed_ticks": 0 + }, + "sample_rule": { + "below_p95_min": "max must be within the p99 target", + "p95_min_samples": 20, + "p99_min_samples": 100 + }, + "schema": "qdl.p184.stage100-budget.v1", + "sources": { + "base": "473ea6b12f0d56278c3830ca6484e9129c4e9646de0021e3505395debf01baa8", + "g4": [ + "09bd7219c41e570aa8524dbb022204f4854ff75be548727287f1e5aa0ee41b87", + "83c8ab8c3630bde923b70417c6c0e5347ebbb6267e498795b2fce6671509e215", + "f7174be2ebeaf4d9f070cf8fc7ac7a974172f3a3ada25c83a6ae2be5449f5c7e", + "a902171d065a9352ca0ad706c73d607dac8bc7be2da3f5b40624753526a72bcf" + ], + "receipt": "43ecd594aee10c3ec538e969f3356256aa810196bc0d22f75c8160f8503184b4" + }, + "stages": { + "100": { + "mix": [ + 40, + 30, + 20, + 10 + ], + "seconds": 1800 + }, + "20": { + "mix": [ + 8, + 6, + 4, + 2 + ], + "seconds": 600 + }, + "50": { + "mix": [ + 20, + 15, + 10, + 5 + ], + "seconds": 600 + } + }, + "startup": { + "max_setup_seconds": 240, + "retry_base_seconds": 0.25, + "retry_codes": [ + "RATE_LIMITED" + ], + "retry_max_attempts": 10, + "retry_max_seconds": 4.0 + }, + "streams": { + "bar_continuity": "consecutive final opens differ by exactly one interval; a repeated open is a revision, an older open is a FIFO violation", + "first_event_required_feeds": [ + "TRADE", + "QUOTE", + "BOOK_DELTA" + ], + "max_stream_errors": 0, + "min_final_bars_per_bar_stream": 1 + }, + "trading_system": { + "allowed_quiet_slices": [], + "demanded_routes": 64, + "every_sample_ready": true, + "forbidden_disconnect_codes": [ + "UNAUTHENTICATED", + "PERMISSION_DENIED", + "MANIFEST_REVISION_MISMATCH", + "INVALID_ARGUMENT" + ], + "max_disconnect_rate_over_baseline_per_minute": 10.0, + "max_fallback": 0, + "max_v2_error_increase": 0 + }, + "writer_health_report_only": { + "note": "Reported, not a gate: the single-writer ceiling is recorded at dl-v2-r1-outcome-20260916.", + "projector_canonical_age_ms_max": 60000 + } +} diff --git a/scripts/phase3_consumer_load_acceptance.py b/scripts/phase3_consumer_load_acceptance.py index 9fb71d27..0de9eb48 100644 --- a/scripts/phase3_consumer_load_acceptance.py +++ b/scripts/phase3_consumer_load_acceptance.py @@ -232,7 +232,7 @@ def validate_profile(raw: dict[str, object]) -> dict[str, object]: # is recorded, never used to relax a gate. # KN-4 D41: a shadow kn4-matrix names the isolated canonical broker its # handoff oracle reads (never a production broker). - optional = {"monitored_containers", "scope", "oracle_bootstrap"} + optional = {"monitored_containers", "scope", "oracle_bootstrap", "ts_container"} if not expected <= set(raw) or set(raw) - expected - optional: raise ValueError("Phase-3 profile fields are incomplete or unknown") scope = raw.get("scope", "production") @@ -312,6 +312,8 @@ def validate_profile(raw: dict[str, object]) -> dict[str, object]: "monitored_containers": list(monitored) if monitored is not None else None, "scope": scope, "oracle_bootstrap": oracle, + "ts_container": raw.get("ts_container") if isinstance(raw.get("ts_container"), str) + and _NETWORK_RE.fullmatch(raw["ts_container"]) else None, } @@ -1696,10 +1698,11 @@ async def run_inside() -> dict[str, object]: }) _TS_HEARTBEAT = """ import asyncio, json, time +import os import redis.asyncio as redis -from core.config import settings async def main(): - client = redis.from_url(settings.TRADING_REDIS_URL, decode_responses=True) + # The writer's own environment (current images use shared.config; older ones core.config). + client = redis.from_url(os.environ["TRADING_REDIS_URL"], decode_responses=True) rows = [] async for key in client.scan_iter(match="service:heartbeat:market_data:*", count=100): value = await client.get(key) @@ -3117,9 +3120,12 @@ async def sdk_posts(): } -def _ts_heartbeat() -> dict[str, object] | None: +_TS_LOG_LINE = re.compile(r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})") + + +def _ts_heartbeat(container: str = "market_data_service") -> dict[str, object] | None: try: - result = subprocess.run(["docker", "exec", "-i", "market_data_service", "python", "-"], + result = subprocess.run(["docker", "exec", "-i", container, "python", "-"], input=_TS_HEARTBEAT, capture_output=True, text=True, timeout=20) value = json.loads(result.stdout.strip().splitlines()[-1]) value["at_ns"] = time.time_ns() @@ -3128,11 +3134,22 @@ def _ts_heartbeat() -> dict[str, object] | None: return {"at_ns": time.time_ns(), "error": _safe_error(error)} -def _ts_disconnects(since: str, until: str) -> tuple[int, Counter]: - result = subprocess.run(["docker", "logs", "--since", since, "--until", until, "market_data_service"], +def _ts_disconnects(since: str, until: str, container: str = "market_data_service") -> tuple[int, Counter]: + """Writer disconnects in [since, until): its docker log AND its worker log files (fixed workers log to + /app/logs/*.log, not stdout - counting docker logs alone reported 0 for the P18.4 writer).""" + result = subprocess.run(["docker", "logs", "--since", since, "--until", until, container], capture_output=True, text=True, timeout=60) + lines = (result.stdout + result.stderr).splitlines() + files = subprocess.run(["docker", "exec", container, "sh", "-c", + "cat /app/logs/market_data_worker-*.log /app/logs/market_data_service.log 2>/dev/null"], + capture_output=True, text=True, timeout=60) + low, high = since.replace("T", " ").rstrip("Z"), until.replace("T", " ").rstrip("Z") + for line in files.stdout.splitlines(): + stamp = _TS_LOG_LINE.match(line) + if stamp and low <= stamp.group(1) < high: + lines.append(line) codes: Counter[str] = Counter() - for line in (result.stdout + result.stderr).splitlines(): + for line in lines: if "slice disconnected" in line: match = re.search(r"code=([A-Z_]+)", line) codes[match.group(1) if match else "UNKNOWN"] += 1 @@ -3238,11 +3255,12 @@ def run_target_host(args: argparse.Namespace) -> int: *query_names, f"{prefix}-stream_v2_active-1", f"{prefix}-stream_v2_passive-1", "market_data_service", ] + ts_container = profile.get("ts_container") or "market_data_service" shadow = profile.get("scope") == "shadow" # A shadow run still samples the production Trading System: it is the # packet's stop condition (TS ready routes must not drop), not a consumer # of the shadow targets. - ts_samples = [] if matrix else [_ts_heartbeat()] + ts_samples = [] if matrix else [_ts_heartbeat(ts_container)] started_iso = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) baseline_iso = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 600)) started = time.monotonic() @@ -3262,7 +3280,7 @@ def run_target_host(args: argparse.Namespace) -> int: cleanup_error = _safe_error(error) ended_iso = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) if not matrix: - ts_samples.append(_ts_heartbeat()) + ts_samples.append(_ts_heartbeat(ts_container)) stdout = process.stdout if process is not None else "" stderr = process.stderr if process is not None else "" receipt = None @@ -3291,8 +3309,8 @@ def run_target_host(args: argparse.Namespace) -> int: evaluation = None if not matrix: run_minutes = max(1e-6, (time.monotonic() - started) / 60.0) - base_count, base_codes = _ts_disconnects(baseline_iso, started_iso) - run_count, run_codes = _ts_disconnects(started_iso, ended_iso) + base_count, base_codes = _ts_disconnects(baseline_iso, started_iso, ts_container) + run_count, run_codes = _ts_disconnects(started_iso, ended_iso, ts_container) trading_system.update( baseline_window=[baseline_iso, started_iso], run_window=[started_iso, ended_iso], baseline_disconnects_per_minute=round(base_count / 10.0, 3), From 1f2844857d9a4570663286df49b5b21dee386523 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:40:04 +0000 Subject: [PATCH 071/147] feat(load): profile manifest_overrides mount a deployed manifest revision over the client's baked copy (D3 window B) --- scripts/phase3_consumer_load_acceptance.py | 18 +++++++++++++++++- tests/test_p184_stage100.py | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/phase3_consumer_load_acceptance.py b/scripts/phase3_consumer_load_acceptance.py index 0de9eb48..33b5040e 100644 --- a/scripts/phase3_consumer_load_acceptance.py +++ b/scripts/phase3_consumer_load_acceptance.py @@ -232,7 +232,7 @@ def validate_profile(raw: dict[str, object]) -> dict[str, object]: # is recorded, never used to relax a gate. # KN-4 D41: a shadow kn4-matrix names the isolated canonical broker its # handoff oracle reads (never a production broker). - optional = {"monitored_containers", "scope", "oracle_bootstrap", "ts_container"} + optional = {"monitored_containers", "scope", "oracle_bootstrap", "ts_container", "manifest_overrides"} if not expected <= set(raw) or set(raw) - expected - optional: raise ValueError("Phase-3 profile fields are incomplete or unknown") scope = raw.get("scope", "production") @@ -314,9 +314,23 @@ def validate_profile(raw: dict[str, object]) -> dict[str, object]: "oracle_bootstrap": oracle, "ts_container": raw.get("ts_container") if isinstance(raw.get("ts_container"), str) and _NETWORK_RE.fullmatch(raw["ts_container"]) else None, + "manifest_overrides": _manifest_overrides(raw.get("manifest_overrides")), } +def _manifest_overrides(value: object) -> dict[str, str]: + """P18.4 D3 window B: a deployed consumer manifest revision (e.g. alpha rev15) mounted over the client image's baked + copy, so the client loads the same quotas and mints JWTs for the same revision as the readers.""" + if value is None: + return {} + if not isinstance(value, dict) or set(value) - set(_PHASE3_CONSUMER_MANIFESTS): + raise ValueError("Phase-3 manifest overrides must name known consumers") + for path in value.values(): + if not isinstance(path, str) or not path.endswith(".yaml") or not Path(path).is_file(): + raise ValueError("Phase-3 manifest override file is unavailable") + return dict(value) + + def _docker_image_id(image: str) -> str: value = subprocess.run( ["docker", "image", "inspect", image, "--format", "{{.Id}}"], @@ -507,6 +521,8 @@ def docker_command( "--env", "PYTHONPATH=/app:/driver", "--env", "QDL_PHASE3_LOAD_CONFIG=" + json.dumps(inner, sort_keys=True, separators=(",", ":")), "--mount", f"type=bind,src={profile['runtime_dir']},dst=/runtime,readonly", + *[arg for consumer, path in sorted((profile.get("manifest_overrides") or {}).items()) + for arg in ("--mount", f"type=bind,src={path},dst={_PHASE3_CONSUMER_MANIFESTS[consumer]},readonly")], "--mount", f"type=bind,src={ROOT / 'qdl/certification/phase3_consumer_load.py'},dst=/app/qdl/certification/phase3_consumer_load.py,readonly", "--mount", f"type=bind,src={Path(__file__).resolve()},dst=/driver/phase3_consumer_load_acceptance.py,readonly", "--mount", f"type=bind,src={ROOT / 'scripts/phase3_consumer_load_bootstrap.sh'},dst=/driver/phase3_consumer_load_bootstrap.sh,readonly", diff --git a/tests/test_p184_stage100.py b/tests/test_p184_stage100.py index 5cd5dde5..3bf4e643 100644 --- a/tests/test_p184_stage100.py +++ b/tests/test_p184_stage100.py @@ -111,3 +111,19 @@ def test_freeze_refuses_without_a_successful_cold_read(self): base = json.loads(Path("config/v2/v211-target-acceptance-budget.json").read_text()) with self.assertRaisesRegex(ValueError, "COLD_5000"): freeze(base, {"cold": [{"venue": "OKX", "rows": 2500, "ms": 1.0}]}, [], ts_routes=64, sources={}) + + +class ManifestOverrideTests(unittest.TestCase): + def test_overrides_are_validated_and_mounted_over_the_baked_manifest(self): + import tempfile + from pathlib import Path + import scripts.phase3_consumer_load_acceptance as driver + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "alpha.binance.paper.stable-r15.yaml" + path.write_text("x") + self.assertEqual(driver._manifest_overrides({"alpha.binance.paper.stable": str(path)}), + {"alpha.binance.paper.stable": str(path)}) + for bad in ({"unknown.consumer": str(path)}, {"alpha.binance.paper.stable": tmp + "/missing.yaml"}, ["x"]): + with self.subTest(bad=bad), self.assertRaises(ValueError): + driver._manifest_overrides(bad) + self.assertEqual(driver._manifest_overrides(None), {}) From 4772b76212b3c43534546d0465bc97115d4c34fc Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:40:18 +0000 Subject: [PATCH 072/147] docs(p184): D3 window B rendered (alpha rev15, bundle bce26969, Stream 1024), not applied --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index dfff6794..dbaa0b93 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71360,3 +71360,21 @@ Status: owner-approved window A, runtime unchanged. Budget FROZEN before any can - profile-stage50.json 878c069f93dd63cc - Cleanup: driver client containers --rm (none left); the watcher was stopped; the observer ran inside query_kn_1 with no leftover process. + +### D3 window B prepared (alpha rev15 quotas + Stream 1024 subscriptions) - 2026-10-09 (Claude) + +Status RENDERED_NOT_APPLIED. +- Manifests (scripts/p184_render_quota_revision.py, from the repo copies that equal the Query-baked ones): + - alpha.binance.paper.stable rev 15: 4950 RPM / 300 streams, manifest_sha256 06db86f0, file sha256 8fed780a; + - alpha.okx.paper.stable rev 15: 4950 / 300, manifest_sha256 b2ba5729, file sha256 33b9cfd4; + - only the revision and those two quotas differ. +- Gateway bundle: rebuilding the deployed inputs reproduces the live f04ba6dc (parity). The new bundle bce26969 (file + aa113b28) changes only the two alpha manifests; catalog/environment/schema are equal. +- Reader step (~/.local/state/qdl-v2/p184-d3-capacity-20261009/readers_capacity.py, file sha256 5531bf81), same images: + - Query: two QDL_STABLE_CONSUMER_MANIFESTS entries now point at /consumer-realms/alpha.*-r15.yaml; + - Stream: QDL_KN_MAX_SUBSCRIPTIONS 384 -> 1024; + - all four readers: /consumer-realms -> the new realm dir (rev11 realm + the two r15 files + bundle bce26969). + Rollback composes = the current rev11 realm and 384. +- Driver: profile field manifest_overrides (1f28448) mounts the r15 files over the client's baked manifests, so the client + quotas and JWT revision match the readers. Profile profile-stage100.json validates (ts_container market_data_service_p184w). +- No alpha caller runs today. Before Phase E alphas must mint JWT revision 15. From d8b0f67b7b5e98bb4ddc879a287dd8db35cf9cfa Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 18:57:38 +0000 Subject: [PATCH 073/147] docs(p184): D3 window B applied (alpha rev15, Stream 1024) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index dbaa0b93..4da32cb4 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71378,3 +71378,19 @@ Status RENDERED_NOT_APPLIED. - Driver: profile field manifest_overrides (1f28448) mounts the r15 files over the client's baked manifests, so the client quotas and JWT revision match the readers. Profile profile-stage100.json validates (ts_container market_data_service_p184w). - No alpha caller runs today. Before Phase E alphas must mint JWT revision 15. + +### D3 window B APPLIED (alpha rev15 quotas + Stream 1024) - 2026-10-09 (Claude) + +Owner-approved. The owner created the root-owned realm dir; checked: only bundle.json differs from the rev11 realm, plus the +two r15 manifests (8fed780a / 33b9cfd4). +- 18:55:30-18:56:52Z, readers recreated one at a time, same images: + - query_kn_1/2 healthy, restarts 0, manifest list now /consumer-realms/alpha.{binance,okx}.paper.stable-r15.yaml; + - stream_kn_1/2 restarts 0, bundle bce26969, QDL_KN_MAX_SUBSCRIPTIONS 1024; startup MemoryPlan streams 1056, + required 321,912,832 B, under the unchanged 512 MiB limit. +- TS writer READY 64/64 at 18:57:11Z, restarts 0 (resubscribed after the Stream restarts). +- Rollback per reader: ~/.local/state/qdl-v2/p184-d3-capacity-20261009/readers/*-rollback.compose.json (rev11 realm, 384). +- Receipts (file, sha256 prefix): + - query_kn_1-candidate-1791572130606223397.json 088cbae06a6f28c6 + - query_kn_2-candidate-1791572150685196750.json ba47ba27933bf346 + - stream_kn_1-candidate-1791572170561766250.json 21ff33a4f2eda79a + - stream_kn_2-candidate-1791572184336405277.json 52353e1dea321214 From 902c28018acdaea0f2c9a828e86310bfe6217ad5 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 19:04:05 +0000 Subject: [PATCH 074/147] feat(kn-recovery): read-only native observation adapter for recovery coordinator (D4/R2) Adds scripts/kn_recovery_observe.py producing RecoveryController input from disk pressure (reused DiskPressure), Kafka quorum/ISR, projector committed progress (READ_COMMITTED end) and TS writer heartbeat eligibility. One absolute deadline, no retries, UNKNOWN/FATAL fail closed. 22 new unit tests incl. end-to-end gate progression. TESTED_LOCALLY only. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 48 ++ scripts/kn_recovery_observe.py | 534 ++++++++++++++++++++++ tests/test_kn_recovery_observe.py | 439 ++++++++++++++++++ 3 files changed, 1021 insertions(+) create mode 100644 scripts/kn_recovery_observe.py create mode 100644 tests/test_kn_recovery_observe.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 4da32cb4..9e0ceb4e 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71394,3 +71394,51 @@ two r15 manifests (8fed780a / 33b9cfd4). - query_kn_2-candidate-1791572150685196750.json ba47ba27933bf346 - stream_kn_1-candidate-1791572170561766250.json 21ff33a4f2eda79a - stream_kn_2-candidate-1791572184336405277.json 52353e1dea321214 + +### D4 R2 native observation adapter - 2026-10-09 (Claude subagent) + +Status: IMPLEMENTED / TESTED_LOCALLY. Not runtime-certified; no fault acceptance. +Base d8b0f67 (fix/p184-dl-closure). Source + tests only. + +Files: +- scripts/kn_recovery_observe.py (new): read-only adapter producing the + `qdl.kn.recovery-observation.v1` input of kn_recovery_state.RecoveryController. + Disk reuses kn_disk_pressure.sample_disk/DiskPressure (operator thresholds, no + defaults). Quorum/ISR via AdminClient describe_cluster/describe_topics (broker + count + controller, leader in ISR, ISR >= min_isr, under-replicated <= explicit + max, default 0). Projector progress via list_consumer_group_offsets vs + list_offsets(READ_COMMITTED) per declared group/partition roster; regression of a + committed offset is FATAL. Consumer eligibility from the TS writer heartbeat + (status READY, session_ready == demanded > 0, ready/unhealthy consistent, + last_seen_unix fresh, optional exact demanded/instance roster) through a provided + reader. Roles via provided reader (default one `docker inspect`, identity checked). +- tests/test_kn_recovery_observe.py (new): 22 tests, fakes for every source + (healthy, missing, stale, future, unparseable, timeout, late answer past deadline, + partial ISR, offline, brokers missing, lagging/stalled/regressed projector, TS + DEGRADED/partial, role source failure), real confluent_kafka result types with a + fake AdminClient, and end-to-end adapter -> controller progression + HOLD_UNKNOWN -> WAIT_QUORUM(start proposal) -> WAIT_QUORUM -> WAIT_PROJECTOR -> + WAIT_CONSUMER -> HOLD_UNKNOWN(heartbeat missing, reader Up) -> SERVING -> + HOLD_UNKNOWN(stale); persistent UNKNOWN ends BLOCKED, never SERVING. + +Semantics: UNKNOWN/FATAL gates are emitted non-boolean, so the controller holds +HOLD_UNKNOWN. A stage role observed not running turns that stage's UNKNOWN into +NOT_READY (start may be proposed); "running" never makes a gate READY. Gates +short-circuit in coordinator order. One absolute deadline per cycle, every call +gets the remaining budget, single attempt, late answers UNKNOWN. mode OBSERVE_ONLY, +mutations 0; no group join/commit, no Redis write, no lifecycle call. CLI prints +the observation (plus adapter_report) for use as runner probe_command. + +Tests: image sha256:77682eea, --network none, read-only source: +`python -B -m unittest tests.test_kn_recovery_state tests.test_kn_recovery_runner +tests.test_kn_recovery_observe tests.test_kn_disk_pressure +tests.test_kn_production_watch` -> 76 PASS (24+9+22+11+10), 0 fail/error/skip. + +Open limits: not run against live brokers/Redis/Docker; no production packet, +thresholds or installed service. Heartbeat is the TS writer's self-reported +aggregate, not the at-use validator/ACK proof D4 ultimately requires (Claude TS +side). Lag uses LSO; an idle dead-but-running projector with lag 0 reads READY. +statvfs is not deadline-bounded (local paths only). Redis call time is bounded by +client socket timeouts plus deadline checks between calls. Isolated +broker/projector/Stream fault acceptance (G12) remains OPEN. No runtime change, +no image build, test containers --rm. No push/merge. diff --git a/scripts/kn_recovery_observe.py b/scripts/kn_recovery_observe.py new file mode 100644 index 00000000..5b8e1e65 --- /dev/null +++ b/scripts/kn_recovery_observe.py @@ -0,0 +1,534 @@ +#!/usr/bin/env python3 +"""Read-only native observation adapter for the KN recovery coordinator (D4/R2). + +Purpose: turn real observation sources into the input of +``kn_recovery_state.RecoveryController.step``: + +- disk pressure: ``kn_disk_pressure.sample_disk`` + operator ``DiskPressure`` + policy (reused, not duplicated; the same module the production watcher uses); +- Kafka quorum/ISR: AdminClient ``describe_cluster``/``describe_topics``; +- projector committed progress: ``list_consumer_group_offsets`` against the + read_committed log end (``list_offsets``, LSO) for the projector groups; +- consumer eligibility: the TS writer heartbeat + (``service:heartbeat:market_data:*``: status / demanded_v2_slices / + session_ready_v2_slices / last_seen_unix) read through a provided callable; +- role identity/state: a provided callable (default ``docker inspect``). + +Boundary: observe/propose only. No Kafka group join or offset commit, no Redis +write, no container lifecycle call; ``mutations`` is always 0. Every source call +receives the remaining time of ONE absolute deadline; there are no retries, and +an answer that arrives after the deadline is UNKNOWN. A container being "Up" +never makes a gate READY; a stage role that is observed not running only turns +that stage's UNKNOWN gate into a definitive NOT_READY so the coordinator may +propose an exact start. Missing, stale, unparseable, late, FATAL or UNKNOWN +evidence is emitted as a non-boolean gate, which the coordinator holds as +HOLD_UNKNOWN; it can never become SERVING. + +Gates are evaluated in the coordinator's order and short-circuit: once a gate is +not READY, later sources are not queried (reported NOT_EVALUATED, emitted False, +which the coordinator never reaches because it stops at the first failed gate). +""" +import argparse +import json +import math +import os +from pathlib import Path +import subprocess +import time + +if __package__: + from scripts.kn_disk_pressure import DiskPressure, sample_disk + from scripts.kn_recovery_state import GATES, SCHEMA, STAGES +else: + from kn_disk_pressure import DiskPressure, sample_disk + from kn_recovery_state import GATES, SCHEMA, STAGES + + +REPORT_SCHEMA = "qdl.kn.recovery-native-report.v1" +READY, NOT_READY, UNKNOWN, FATAL, NOT_EVALUATED = ( + "READY", "NOT_READY", "UNKNOWN", "FATAL", "NOT_EVALUATED") +GATE_STAGE = {"quorum_isr_ready": "infra", "projector_progress_ready": "projector", + "consumer_eligible": "reader"} +HEARTBEAT_PATTERN = "service:heartbeat:market_data:*" +MAX_PARTITIONS = 1024 +MAX_HEARTBEATS = 64 +MAX_HEARTBEAT_BYTES = 1 << 20 + + +class DeadlineExpired(Exception): + pass + + +class Deadline: + """One absolute monotonic deadline shared by every call of one cycle.""" + + def __init__(self, seconds, clock=time.monotonic): + if type(seconds) not in (int, float) or not math.isfinite(seconds) or seconds <= 0: + raise ValueError("budget must be finite and positive") + self._clock = clock + self.at = clock() + seconds + + def remaining(self): + left = self.at - self._clock() + if not left > 0: + raise DeadlineExpired("DEADLINE_EXCEEDED") + return left + + def expired(self): + return not self.at - self._clock() > 0 + + +def _gate(status, reason, /, **evidence): + return {"status": status, "reason": reason, "evidence": evidence} + + +def _combine(results): + for status in (FATAL, UNKNOWN, NOT_READY): + for result in results: + if result["status"] == status: + return result + return results[0] if results else _gate(UNKNOWN, "NO_SOURCES") + + +def _count(value): + return type(value) is int and value >= 0 + + +class ObservePolicy: + """Explicit operator inputs; no production threshold is defaulted here.""" + + def __init__(self, *, disk_paths, expected_brokers, topics, min_isr, projector_groups, + max_projector_lag_records, heartbeat_max_age_seconds, role_stages, + max_under_replicated=0, heartbeat_future_skew_seconds=1.0, + expected_demanded_slices=None, expected_heartbeat_instances=None): + if (not isinstance(disk_paths, (list, tuple)) or not disk_paths + or len(set(disk_paths)) != len(disk_paths) + or any(not isinstance(p, str) or not p for p in disk_paths)): + raise ValueError("disk_paths must be distinct nonempty strings") + for name, value in (("expected_brokers", expected_brokers), ("min_isr", min_isr)): + if type(value) is not int or value <= 0: + raise ValueError(name + " must be a positive integer") + for name, value in (("max_under_replicated", max_under_replicated), + ("max_projector_lag_records", max_projector_lag_records)): + if not _count(value): + raise ValueError(name + " must be a non-negative integer") + for name, value in (("heartbeat_max_age_seconds", heartbeat_max_age_seconds), + ("heartbeat_future_skew_seconds", heartbeat_future_skew_seconds)): + if type(value) not in (int, float) or not math.isfinite(value) or value <= 0: + raise ValueError(name + " must be finite and positive") + for name, value in (("expected_demanded_slices", expected_demanded_slices), + ("expected_heartbeat_instances", expected_heartbeat_instances)): + if value is not None and (type(value) is not int or value <= 0): + raise ValueError(name + " must be a positive integer or None") + self.topics = self._roster(topics, "topics") + if not isinstance(projector_groups, dict) or not projector_groups: + raise ValueError("projector_groups must be a nonempty mapping") + self.projector_groups = {} + for group, roster in projector_groups.items(): + if not isinstance(group, str) or not group: + raise ValueError("invalid projector group") + self.projector_groups[group] = self._roster(roster, "projector group roster") + if (not isinstance(role_stages, dict) or not role_stages + or any(not isinstance(k, str) or v not in STAGES for k, v in role_stages.items())): + raise ValueError("role_stages must map role names to stages") + self.disk_paths = tuple(disk_paths) + self.expected_brokers = expected_brokers + self.min_isr = min_isr + self.max_under_replicated = max_under_replicated + self.max_projector_lag_records = max_projector_lag_records + self.heartbeat_max_age_seconds = heartbeat_max_age_seconds + self.heartbeat_future_skew_seconds = heartbeat_future_skew_seconds + self.expected_demanded_slices = expected_demanded_slices + self.expected_heartbeat_instances = expected_heartbeat_instances + self.role_stages = dict(role_stages) + + @staticmethod + def _roster(value, name): + if (not isinstance(value, dict) or not value + or any(not isinstance(t, str) or not t or type(n) is not int or n <= 0 + for t, n in value.items()) + or sum(value.values()) > MAX_PARTITIONS): + raise ValueError(name + " must map topics to bounded partition counts") + return dict(value) + + +class NativeObserver: + """Stateful across cycles only for disk baselines and committed-offset regression. + + Sources (all read-only; each receives the cycle ``Deadline``): + - ``kafka``: object with describe_cluster(deadline) -> {broker_ids, controller_id}, + describe_topics(topics, deadline) -> {topic: [{partition, leader, replicas, isr}]}, + committed_offsets(group, roster, deadline) -> {(topic, partition): offset}, + log_end_offsets(partitions, deadline) -> {(topic, partition): offset} + - ``read_heartbeats(deadline)`` -> list of raw JSON str/bytes or dicts + - ``read_roles(deadline)`` -> {role: {id, image, state}} + - ``sample(path)`` -> kn_disk_pressure.sample_disk shaped dict or None + """ + + def __init__(self, policy, disk_pressure, kafka, read_heartbeats, read_roles, + sample=sample_disk, clock=time.monotonic, wall_clock=time.time): + if not isinstance(policy, ObservePolicy) or not isinstance(disk_pressure, DiskPressure): + raise ValueError("policy and DiskPressure are required") + self.policy = policy + self.disk = disk_pressure + self.kafka = kafka + self.read_heartbeats = read_heartbeats + self.read_roles = read_roles + self.sample = sample + self.clock = clock + self.wall_clock = wall_clock + self._committed = {} + + # ---- sources ------------------------------------------------------- + def _call(self, deadline, function, *args): + """Single bounded attempt; late answers are not trusted.""" + try: + deadline.remaining() + value = function(*args, deadline) + except DeadlineExpired: + return None, _gate(UNKNOWN, "DEADLINE_EXCEEDED") + except Exception as exc: # source failures are evidence gaps, never success + return None, _gate(UNKNOWN, "SOURCE_ERROR", error_type=type(exc).__name__) + if deadline.expired(): + return None, _gate(UNKNOWN, "DEADLINE_EXCEEDED", late=True) + return value, None + + def observe_disk(self): + results = [] + for path in self.policy.disk_paths: + outcome = self.disk.observe(path, self.sample(path)) + reasons = list(outcome["reasons"]) + definitive = [r for r in reasons if r != "DISK_OBSERVATION_UNKNOWN"] + if definitive: + results.append(_gate(NOT_READY, "DISK_PRESSURE", path=path, reasons=reasons)) + elif reasons: + results.append(_gate(UNKNOWN, "DISK_OBSERVATION_UNKNOWN", path=path)) + else: + results.append(_gate(READY, "DISK_SAFE", path=path, metrics=outcome.get("metrics"))) + return _combine(results) + + def observe_quorum(self, deadline): + cluster, error = self._call(deadline, self.kafka.describe_cluster) + if error: + return error + try: + brokers = cluster["broker_ids"] + controller = cluster["controller_id"] + if (not isinstance(brokers, (list, tuple)) or any(type(b) is not int for b in brokers) + or len(set(brokers)) != len(brokers)): + raise ValueError + except (KeyError, TypeError, ValueError): + return _gate(UNKNOWN, "CLUSTER_UNPARSEABLE") + if len(brokers) < self.policy.expected_brokers: + return _gate(NOT_READY, "BROKERS_MISSING", brokers=sorted(brokers), + expected=self.policy.expected_brokers) + if len(brokers) > self.policy.expected_brokers or controller not in brokers: + return _gate(UNKNOWN, "CLUSTER_ROSTER_MISMATCH", brokers=sorted(brokers), + controller=controller) + topics, error = self._call(deadline, self.kafka.describe_topics, tuple(self.policy.topics)) + if error: + return error + offline, below_min, under = [], [], [] + try: + if set(topics) != set(self.policy.topics): + return _gate(UNKNOWN, "TOPIC_ROSTER_MISMATCH") + for topic, count in self.policy.topics.items(): + partitions = topics[topic] + if sorted(p["partition"] for p in partitions) != list(range(count)): + return _gate(UNKNOWN, "PARTITION_ROSTER_MISMATCH", topic=topic) + for row in partitions: + replicas, isr, leader = list(row["replicas"]), list(row["isr"]), row["leader"] + if (any(type(v) is not int for v in replicas + isr) + or not set(isr) <= set(replicas) or not replicas): + return _gate(UNKNOWN, "PARTITION_UNPARSEABLE", topic=topic) + where = (topic, row["partition"]) + if leader is None or type(leader) is not int or leader < 0 or leader not in isr: + offline.append(where) + if len(isr) < self.policy.min_isr: + below_min.append(where) + if len(isr) < len(replicas): + under.append(where) + except (KeyError, TypeError, ValueError): + return _gate(UNKNOWN, "PARTITION_UNPARSEABLE") + evidence = {"brokers": sorted(brokers), "offline": len(offline), + "below_min_isr": len(below_min), "under_replicated": len(under), + "first": [list(x) for x in (offline + below_min + under)[:16]]} + if offline: + return _gate(NOT_READY, "OFFLINE_PARTITION", **evidence) + if below_min: + return _gate(NOT_READY, "ISR_BELOW_MIN", **evidence) + if len(under) > self.policy.max_under_replicated: + return _gate(NOT_READY, "UNDER_REPLICATED", **evidence) + return _gate(READY, "QUORUM_ISR_READY", **evidence) + + def observe_projector(self, deadline): + results = [] + for group, roster in self.policy.projector_groups.items(): + expected = {(t, p) for t, n in roster.items() for p in range(n)} + committed, error = self._call(deadline, self.kafka.committed_offsets, group, roster) + if error: + return dict(error, evidence=dict(error["evidence"], group=group)) + ends, error = self._call(deadline, self.kafka.log_end_offsets, tuple(sorted(expected))) + if error: + return dict(error, evidence=dict(error["evidence"], group=group)) + if (not isinstance(committed, dict) or not isinstance(ends, dict) + or set(committed) != expected or set(ends) != expected + or any(type(v) is not int for v in list(committed.values()) + list(ends.values()))): + return _gate(UNKNOWN, "OFFSET_ROSTER_MISMATCH", group=group) + if any(v < 0 for v in committed.values()): + return _gate(UNKNOWN, "NO_COMMITTED_OFFSET", group=group) + if any(committed[k] > ends[k] for k in expected): + return _gate(UNKNOWN, "COMMITTED_BEYOND_END", group=group) + previous = self._committed.get(group, {}) + if any(committed[k] < previous.get(k, -1) for k in expected): + # A committed offset moving backwards means reset/replay: operator review. + return _gate(FATAL, "COMMITTED_REGRESSION", group=group) + self._committed[group] = dict(committed) + lags = {k: ends[k] - committed[k] for k in expected} + worst = max(lags.values()) + advanced = bool(previous) and any(committed[k] > previous[k] for k in expected) + evidence = {"group": group, "max_lag_records": worst, + "total_lag_records": sum(lags.values()), + "advanced_since_previous": advanced if previous else None} + if worst > self.policy.max_projector_lag_records: + reason = "PROJECTOR_STALLED" if previous and not advanced else "PROJECTOR_LAGGING" + results.append(_gate(NOT_READY, reason, **evidence)) + else: + results.append(_gate(READY, "PROJECTOR_CAUGHT_UP", **evidence)) + return _combine(results) + + def observe_consumer(self, deadline): + rows, error = self._call(deadline, self.read_heartbeats) + if error: + return error + if not isinstance(rows, (list, tuple)) or not rows: + return _gate(UNKNOWN, "HEARTBEAT_MISSING") + if len(rows) > MAX_HEARTBEATS: + return _gate(UNKNOWN, "HEARTBEAT_ROSTER_TOO_LARGE") + expected = self.policy.expected_heartbeat_instances + if expected is not None and len(rows) != expected: + return _gate(UNKNOWN, "HEARTBEAT_INSTANCE_MISMATCH", instances=len(rows)) + now = self.wall_clock() + results = [] + for raw in rows: + try: + if isinstance(raw, (bytes, str)): + if len(raw) > MAX_HEARTBEAT_BYTES: + raise ValueError + raw = json.loads(raw) + beat = raw + seen, status, details = beat["last_seen_unix"], beat["status"], beat["details"] + if (type(seen) not in (int, float) or not math.isfinite(seen) + or not isinstance(status, str) or not isinstance(details, dict)): + raise ValueError + demanded = details["demanded_v2_slices"] + session = details["session_ready_v2_slices"] + if not _count(demanded) or not _count(session) or session > demanded: + raise ValueError + extra = {k: details[k] for k in ("ready_v2_slices", "unhealthy_v2_slices") if k in details} + if any(not _count(v) for v in extra.values()): + raise ValueError + except (KeyError, TypeError, ValueError): + results.append(_gate(UNKNOWN, "HEARTBEAT_UNPARSEABLE")) + continue + age = now - seen + evidence = {"status": status, "demanded": demanded, "session_ready": session, + "age_seconds": round(age, 3)} + if age < -self.policy.heartbeat_future_skew_seconds: + results.append(_gate(UNKNOWN, "HEARTBEAT_FROM_FUTURE", **evidence)) + elif age > self.policy.heartbeat_max_age_seconds: + results.append(_gate(UNKNOWN, "HEARTBEAT_STALE", **evidence)) + elif (self.policy.expected_demanded_slices is not None + and demanded != self.policy.expected_demanded_slices): + results.append(_gate(UNKNOWN, "DEMAND_ROSTER_MISMATCH", **evidence)) + elif (status != "READY" or demanded == 0 or session != demanded + or extra.get("ready_v2_slices", session) != session + or extra.get("unhealthy_v2_slices", 0) != 0): + results.append(_gate(NOT_READY, "TS_NOT_READY", **evidence)) + else: + results.append(_gate(READY, "TS_SESSION_READY", **evidence)) + return _combine(results) + + def observe_roles(self, deadline): + roles, error = self._call(deadline, self.read_roles) + if error: + return None, error + if not isinstance(roles, dict) or set(roles) != set(self.policy.role_stages): + return None, _gate(UNKNOWN, "ROLE_ROSTER_MISMATCH") + clean = {} + for name, role in roles.items(): + if not isinstance(role, dict) or any(not isinstance(role.get(k), str) + for k in ("id", "image", "state")): + return None, _gate(UNKNOWN, "ROLE_UNPARSEABLE", role=name) + clean[name] = {k: role[k] for k in ("id", "image", "state")} + return clean, None + + # ---- composition --------------------------------------------------- + def observe(self, budget_seconds): + """One bounded cycle -> {schema, mode, observation, gates, roles_status}.""" + started = self.clock() + deadline = Deadline(budget_seconds, self.clock) + roles, role_error = self.observe_roles(deadline) + gates = {"disk_safe": self.observe_disk()} + observers = {"quorum_isr_ready": self.observe_quorum, + "projector_progress_ready": self.observe_projector, + "consumer_eligible": self.observe_consumer} + blocked = gates["disk_safe"]["status"] != READY + for gate in GATES[1:]: + if blocked: + gates[gate] = _gate(NOT_EVALUATED, "UPSTREAM_GATE_NOT_READY") + continue + result = observers[gate](deadline) + if result["status"] == UNKNOWN and roles is not None: + stopped = sorted(n for n, s in self.policy.role_stages.items() + if s == GATE_STAGE[gate] and roles[n]["state"] != "running") + if stopped: + # Not-running is definitive negative evidence; running is never positive. + result = _gate(NOT_READY, "STAGE_ROLE_NOT_RUNNING", roles=stopped, + unknown=result["reason"]) + gates[gate] = result + blocked = result["status"] != READY + observation = {"schema": SCHEMA, "observed_monotonic": started, + "roles": roles if roles is not None else None} + for gate in GATES: + status = gates[gate]["status"] + observation[gate] = (True if status == READY else + False if status in (NOT_READY, NOT_EVALUATED) else None) + return {"schema": REPORT_SCHEMA, "mode": "OBSERVE_ONLY", "mutations": 0, + "deadline_monotonic": deadline.at, "observation": observation, "gates": gates, + "roles_status": role_error or _gate(READY, "ROLES_OBSERVED")} + + +# ---- native source wiring (lazy imports; read-only calls only) --------------- +class KafkaAdminSource: + """confluent_kafka AdminClient wrapper; every request uses the remaining budget.""" + + def __init__(self, admin): + self.admin = admin + + def describe_cluster(self, deadline): + result = self.admin.describe_cluster(request_timeout=deadline.remaining()).result( + timeout=deadline.remaining()) + controller = getattr(result, "controller", None) + return {"broker_ids": [node.id for node in result.nodes], + "controller_id": controller.id if controller is not None else None} + + def describe_topics(self, topics, deadline): + from confluent_kafka import TopicCollection + futures = self.admin.describe_topics(TopicCollection(list(topics)), + request_timeout=deadline.remaining()) + out = {} + for topic, future in futures.items(): + description = future.result(timeout=deadline.remaining()) + out[topic] = [{"partition": p.id, + "leader": p.leader.id if p.leader is not None else None, + "replicas": [n.id for n in p.replicas], "isr": [n.id for n in p.isr]} + for p in description.partitions] + return out + + def committed_offsets(self, group, roster, deadline): + from confluent_kafka import ConsumerGroupTopicPartitions, TopicPartition + request = ConsumerGroupTopicPartitions( + group, [TopicPartition(t, p) for t, n in sorted(roster.items()) for p in range(n)]) + result = self.admin.list_consumer_group_offsets( + [request], request_timeout=deadline.remaining())[group].result(timeout=deadline.remaining()) + out = {} + for item in result.topic_partitions: + if item.error is not None: + raise ValueError("partition offset error") + out[(item.topic, item.partition)] = item.offset + return out + + def log_end_offsets(self, partitions, deadline): + from confluent_kafka import IsolationLevel, TopicPartition + from confluent_kafka.admin import OffsetSpec + futures = self.admin.list_offsets( + {TopicPartition(t, p): OffsetSpec.latest() for t, p in partitions}, + isolation_level=IsolationLevel.READ_COMMITTED, request_timeout=deadline.remaining()) + return {(tp.topic, tp.partition): future.result(timeout=deadline.remaining()).offset + for tp, future in futures.items()} + + +def redis_heartbeat_reader(client, pattern=HEARTBEAT_PATTERN): + """Read-only SCAN/GET; client must carry socket timeouts <= the cycle budget.""" + def read(deadline): + rows = [] + for key in client.scan_iter(match=pattern, count=100): + deadline.remaining() + if len(rows) >= MAX_HEARTBEATS: + raise ValueError("heartbeat roster exceeds bound") + value = client.get(key) + if value is not None: + rows.append(value) + return rows + return read + + +def docker_role_reader(roles, run=subprocess.run): + """One read-only `docker inspect` for the pinned roster IDs; no lifecycle call.""" + by_id = {role["id"]: name for name, role in roles.items()} + + def read(deadline): + result = run(["docker", "inspect", "--format", + "{{.Id}}|{{.Image}}|{{.Name}}|{{.State.Status}}", *by_id], + capture_output=True, text=True, timeout=deadline.remaining(), check=False) + if result.returncode: + raise ValueError("inspect failed") + out = {} + for line in result.stdout.splitlines(): + ident, image, name, state = line.split("|") + role = by_id.get(ident) + if role is None or "/" + role != name or role in out: + raise ValueError("inspect identity drift") + out[role] = {"id": ident, "image": image, "state": state} + return out + return read + + +def kafka_tls_config(bootstrap, tls): + """Same mTLS shape as scripts/p184_dr08_isolated.py:kafka_security.""" + config = {"bootstrap.servers": bootstrap} + if tls: + config.update({"security.protocol": "SSL", "ssl.ca.location": tls["ca"], + "ssl.certificate.location": tls["cert"], "ssl.key.location": tls["key"]}) + return config + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--config", required=True, type=Path, + help="JSON: bootstrap, tls{ca,cert,key}, roles, policy, disk_thresholds, " + "redis_url_env, heartbeat_pattern") + parser.add_argument("--budget-seconds", type=float, default=8.0) + parser.add_argument("--disk-baseline-seconds", type=float, default=1.0, + help="Second disk sample spacing so growth is measured, not assumed") + args = parser.parse_args(argv) + config = json.loads(args.config.read_text()) + roles = config["roles"] + policy = ObservePolicy(role_stages={n: r["stage"] for n, r in roles.items()}, **config["policy"]) + disk = DiskPressure(*(config["disk_thresholds"][k] for k in ( + "reserve_bytes", "recovery_bytes", "reserve_inodes", "recovery_inodes", + "reserve_seconds", "recovery_seconds"))) + if not 0 < args.disk_baseline_seconds < args.budget_seconds: + parser.error("disk baseline must be inside the budget") + from confluent_kafka.admin import AdminClient + import redis + url = os.environ[config["redis_url_env"]] + socket_timeout = min(2.0, args.budget_seconds) + client = redis.Redis.from_url(url, socket_timeout=socket_timeout, + socket_connect_timeout=socket_timeout) + observer = NativeObserver( + policy, disk, KafkaAdminSource(AdminClient(kafka_tls_config(config["bootstrap"], config.get("tls")))), + redis_heartbeat_reader(client, config.get("heartbeat_pattern", HEARTBEAT_PATTERN)), + docker_role_reader(roles)) + for path in policy.disk_paths: # baseline only; never asserts readiness + disk.observe(path, sample_disk(path)) + time.sleep(args.disk_baseline_seconds) + report = observer.observe(args.budget_seconds - args.disk_baseline_seconds) + output = dict(report["observation"], adapter_report=report) + print(json.dumps(output, separators=(",", ":"), allow_nan=False, default=list)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_kn_recovery_observe.py b/tests/test_kn_recovery_observe.py new file mode 100644 index 00000000..56ca7207 --- /dev/null +++ b/tests/test_kn_recovery_observe.py @@ -0,0 +1,439 @@ +"""TEST_ONLY fakes for the KN native observation adapter; no broker, Redis or Docker.""" +import json +import subprocess +import unittest +from concurrent.futures import Future +from unittest.mock import Mock + +from scripts.kn_disk_pressure import DiskPressure +from scripts.kn_recovery_observe import ( + Deadline, DeadlineExpired, KafkaAdminSource, NativeObserver, ObservePolicy, + docker_role_reader, redis_heartbeat_reader) +from scripts.kn_recovery_state import GATES, RecoveryController, RecoveryPolicy + +TOPICS = {"md.canonical.v2": 2} +GROUPS = {"kn-projector-v3-a": {"md.canonical.v2": 2}} +ROLES = {name: {"id": d * 64, "image": "sha256:" + d * 64, "stage": stage} + for name, d, stage in (("kafka1", "a", "infra"), ("market_projector_1", "b", "projector"), + ("market_data_service", "c", "reader"))} + + +class Clock: + def __init__(self): + self.now = 1000.0 + + def __call__(self): + return self.now + + +class FakeKafka: + def __init__(self, clock): + self.clock = clock + self.calls = [] + self.brokers = [1, 2, 3] + self.isr = [1, 2, 3] + self.leader = 1 + self.committed = {("md.canonical.v2", 0): 100, ("md.canonical.v2", 1): 200} + self.ends = {("md.canonical.v2", 0): 100, ("md.canonical.v2", 1): 200} + self.fail = {} + self.delay = {} + + def _enter(self, name, deadline): + self.calls.append(name) + self.clock.now += self.delay.get(name, 0) + if name in self.fail: + raise self.fail[name] + + def describe_cluster(self, deadline): + self._enter("describe_cluster", deadline) + return {"broker_ids": list(self.brokers), "controller_id": self.brokers[0] if self.brokers else None} + + def describe_topics(self, topics, deadline): + self._enter("describe_topics", deadline) + return {t: [{"partition": p, "leader": self.leader, "replicas": [1, 2, 3], "isr": list(self.isr)} + for p in range(TOPICS[t])] for t in topics} + + def committed_offsets(self, group, roster, deadline): + self._enter("committed_offsets", deadline) + return dict(self.committed) + + def log_end_offsets(self, partitions, deadline): + self._enter("log_end_offsets", deadline) + return dict(self.ends) + + +def beat(clock_wall, **details): + row = {"demanded_v2_slices": 60, "ready_v2_slices": 60, "session_ready_v2_slices": 60, + "unhealthy_v2_slices": 0} + row.update(details) + status = row.pop("status", "READY") + age = row.pop("age", 1) + return json.dumps({"service_name": "market_data", "status": status, + "last_seen_unix": clock_wall - age, "details": row}) + + +class Harness: + def __init__(self, **policy): + self.clock = Clock() + self.wall = 2_000_000_000.0 + self.kafka = FakeKafka(self.clock) + self.states = {name: "running" for name in ROLES} + self.heartbeats = [beat(self.wall)] + self.heartbeat_error = None + self.disk_available = 800 + self.disk_calls = 0 + values = dict(disk_paths=["/kafka"], expected_brokers=3, topics=TOPICS, min_isr=2, + projector_groups=GROUPS, max_projector_lag_records=10, + heartbeat_max_age_seconds=30, role_stages={n: r["stage"] for n, r in ROLES.items()}, + expected_demanded_slices=60) + values.update(policy) + self.observer = NativeObserver(ObservePolicy(**values), DiskPressure(100, 200, 50, 100, 10, 20), + self.kafka, self.read_heartbeats, self.read_roles, + sample=self.sample, clock=self.clock, + wall_clock=lambda: self.wall) + + def sample(self, path): + self.disk_calls += 1 + if self.disk_available is None: + return None + return {"device": 1, "monotonic": self.clock.now, "total_bytes": 1000, + "available_bytes": self.disk_available, "total_inodes": 1000, "available_inodes": 800} + + def read_heartbeats(self, deadline): + if self.heartbeat_error: + raise self.heartbeat_error + return list(self.heartbeats) + + def read_roles(self, deadline): + return {n: {"id": r["id"], "image": r["image"], "state": self.states[n]} for n, r in ROLES.items()} + + def cycle(self, budget=5): + report = self.observer.observe(budget) + self.clock.now += 1 + self.wall += 1 + return report + + def primed(self): + self.cycle() + return self + + +class SourceTests(unittest.TestCase): + def test_first_cycle_without_disk_baseline_is_unknown(self): + report = Harness().cycle() + self.assertEqual(report["gates"]["disk_safe"]["status"], "UNKNOWN") + self.assertIsNone(report["observation"]["disk_safe"]) + self.assertEqual(report["gates"]["consumer_eligible"]["status"], "NOT_EVALUATED") + self.assertEqual(report["mode"], "OBSERVE_ONLY") + self.assertEqual(report["mutations"], 0) + + def test_healthy_all_gates_true(self): + report = Harness().primed().cycle() + for gate in GATES: + self.assertIs(report["observation"][gate], True, gate) + self.assertEqual(report["observation"]["roles"]["kafka1"]["state"], "running") + + def test_disk_missing_and_pressure(self): + h = Harness().primed() + h.disk_available = None + self.assertIsNone(h.cycle()["observation"]["disk_safe"]) + h = Harness().primed() + h.disk_available = 90 + report = h.cycle() + self.assertIs(report["observation"]["disk_safe"], False) + self.assertEqual(report["gates"]["disk_safe"]["reason"], "DISK_PRESSURE") + self.assertEqual(h.kafka.calls, []) # short-circuit: no Kafka call under pressure + + def test_quorum_partial_isr_offline_and_brokers_missing(self): + for setup, reason in ((lambda h: setattr(h.kafka, "isr", [1, 2]), "UNDER_REPLICATED"), + (lambda h: setattr(h.kafka, "isr", [1]), "ISR_BELOW_MIN"), + (lambda h: setattr(h.kafka, "leader", None), "OFFLINE_PARTITION"), + (lambda h: setattr(h.kafka, "brokers", [1, 2]), "BROKERS_MISSING")): + with self.subTest(reason=reason): + h = Harness().primed() + setup(h) + report = h.cycle() + self.assertEqual(report["gates"]["quorum_isr_ready"]["reason"], reason) + self.assertIs(report["observation"]["quorum_isr_ready"], False) + self.assertNotIn("committed_offsets", h.kafka.calls) + + def test_partial_isr_tolerated_only_by_explicit_policy(self): + h = Harness(max_under_replicated=2).primed() + h.kafka.isr = [1, 2] + self.assertIs(h.cycle()["observation"]["quorum_isr_ready"], True) + + def test_quorum_timeout_and_error_are_unknown_without_retry(self): + h = Harness().primed() + h.kafka.fail["describe_cluster"] = TimeoutError() + report = h.cycle() + self.assertIsNone(report["observation"]["quorum_isr_ready"]) + self.assertEqual(report["gates"]["quorum_isr_ready"]["evidence"]["error_type"], "TimeoutError") + self.assertEqual(h.kafka.calls.count("describe_cluster"), 1) # single attempt, no retry + + def test_late_answer_after_absolute_deadline_is_unknown(self): + h = Harness().primed() + h.kafka.delay["describe_topics"] = 6 + report = h.cycle(budget=5) + self.assertEqual(report["gates"]["quorum_isr_ready"]["reason"], "DEADLINE_EXCEEDED") + self.assertIsNone(report["observation"]["quorum_isr_ready"]) + + def test_expired_deadline_skips_remaining_calls(self): + h = Harness().primed() + h.kafka.delay["describe_cluster"] = 2 + h.kafka.delay["describe_topics"] = 2 + h.kafka.delay["committed_offsets"] = 2 + report = h.cycle(budget=5) + self.assertEqual(report["gates"]["projector_progress_ready"]["reason"], "DEADLINE_EXCEEDED") + self.assertNotIn("log_end_offsets", h.kafka.calls) + + def test_projector_lagging_stalled_regression_and_missing_commit(self): + h = Harness().primed() + h.cycle() # committed-offset baseline + h.kafka.ends = {k: v + 50 for k, v in h.kafka.ends.items()} + self.assertEqual(h.cycle()["gates"]["projector_progress_ready"]["reason"], "PROJECTOR_STALLED") + h.kafka.committed = {k: v + 5 for k, v in h.kafka.committed.items()} + report = h.cycle() + self.assertEqual(report["gates"]["projector_progress_ready"]["reason"], "PROJECTOR_LAGGING") + self.assertIs(report["observation"]["projector_progress_ready"], False) + h.kafka.committed = {k: v - 10 for k, v in h.kafka.committed.items()} + report = h.cycle() + self.assertEqual(report["gates"]["projector_progress_ready"]["status"], "FATAL") + self.assertIsNone(report["observation"]["projector_progress_ready"]) + h = Harness().primed() + h.kafka.committed[("md.canonical.v2", 1)] = -1001 + self.assertEqual(h.cycle()["gates"]["projector_progress_ready"]["reason"], "NO_COMMITTED_OFFSET") + h = Harness().primed() + del h.kafka.ends[("md.canonical.v2", 1)] + self.assertEqual(h.cycle()["gates"]["projector_progress_ready"]["reason"], "OFFSET_ROSTER_MISMATCH") + + def test_consumer_heartbeat_cases(self): + cases = ( + (lambda h: setattr(h, "heartbeats", []), "UNKNOWN", "HEARTBEAT_MISSING"), + (lambda h: setattr(h, "heartbeats", [beat(h.wall, age=31)]), "UNKNOWN", "HEARTBEAT_STALE"), + (lambda h: setattr(h, "heartbeats", [beat(h.wall, age=-5)]), "UNKNOWN", "HEARTBEAT_FROM_FUTURE"), + (lambda h: setattr(h, "heartbeats", ["{not json"]), "UNKNOWN", "HEARTBEAT_UNPARSEABLE"), + (lambda h: setattr(h, "heartbeats", [beat(h.wall, session_ready_v2_slices="60")]), + "UNKNOWN", "HEARTBEAT_UNPARSEABLE"), + (lambda h: setattr(h, "heartbeat_error", TimeoutError()), "UNKNOWN", "SOURCE_ERROR"), + (lambda h: setattr(h, "heartbeats", [beat(h.wall, demanded_v2_slices=50, + ready_v2_slices=50, session_ready_v2_slices=50)]), + "UNKNOWN", "DEMAND_ROSTER_MISMATCH"), + (lambda h: setattr(h, "heartbeats", [beat(h.wall, status="DEGRADED")]), "NOT_READY", "TS_NOT_READY"), + (lambda h: setattr(h, "heartbeats", [beat(h.wall, session_ready_v2_slices=59, ready_v2_slices=59, + unhealthy_v2_slices=1)]), "NOT_READY", "TS_NOT_READY"), + (lambda h: setattr(h, "heartbeats", [beat(h.wall), beat(h.wall, status="DEGRADED")]), + "NOT_READY", "TS_NOT_READY"), + ) + for setup, status, reason in cases: + with self.subTest(reason=reason, status=status): + h = Harness().primed() + setup(h) + gate = h.cycle()["gates"]["consumer_eligible"] + self.assertEqual((gate["status"], gate["reason"]), (status, reason)) + + def test_running_container_never_makes_unknown_ready(self): + h = Harness().primed() + h.heartbeats = [] + report = h.cycle() + self.assertEqual(report["observation"]["roles"]["market_data_service"]["state"], "running") + self.assertIsNone(report["observation"]["consumer_eligible"]) + + def test_stopped_stage_role_turns_unknown_into_not_ready(self): + h = Harness().primed() + h.states["kafka1"] = "exited" + h.kafka.fail["describe_cluster"] = TimeoutError() + gate = h.cycle()["gates"]["quorum_isr_ready"] + self.assertEqual((gate["status"], gate["reason"]), ("NOT_READY", "STAGE_ROLE_NOT_RUNNING")) + self.assertEqual(gate["evidence"]["unknown"], "SOURCE_ERROR") + + def test_fatal_never_downgraded(self): + h = Harness().primed() + h.cycle() + h.states["market_projector_1"] = "exited" + h.kafka.committed = {k: v - 1 for k, v in h.kafka.committed.items()} + self.assertEqual(h.cycle()["gates"]["projector_progress_ready"]["status"], "FATAL") + + def test_role_source_failure_yields_null_roles(self): + h = Harness().primed() + h.read_roles = Mock(side_effect=OSError()) + h.observer.read_roles = h.read_roles + report = h.cycle() + self.assertIsNone(report["observation"]["roles"]) + self.assertEqual(report["roles_status"]["status"], "UNKNOWN") + + def test_policy_validation(self): + base = dict(disk_paths=["/k"], expected_brokers=3, topics=TOPICS, min_isr=2, + projector_groups=GROUPS, max_projector_lag_records=0, + heartbeat_max_age_seconds=30, role_stages={"x": "infra"}) + ObservePolicy(**base) + for key, value in (("disk_paths", []), ("expected_brokers", 0), ("min_isr", True), + ("topics", {"t": 0}), ("projector_groups", {}), + ("max_projector_lag_records", -1), ("heartbeat_max_age_seconds", float("inf")), + ("role_stages", {"x": "other"}), ("topics", {"t": 2000})): + with self.subTest(key=key), self.assertRaises(ValueError): + ObservePolicy(**dict(base, **{key: value})) + + def test_deadline(self): + clock = Clock() + deadline = Deadline(2, clock) + self.assertEqual(deadline.remaining(), 2) + clock.now += 2 + with self.assertRaises(DeadlineExpired): + deadline.remaining() + for value in (0, -1, float("nan"), "1"): + with self.assertRaises(ValueError): + Deadline(value, clock) + + +class NativeWiringTests(unittest.TestCase): + """Real confluent_kafka request/result types with a fake AdminClient (no network).""" + + def setUp(self): + self.clock = Clock() + self.deadline = Deadline(5, self.clock) + + @staticmethod + def done(value): + future = Future() + future.set_result(value) + return future + + def test_admin_calls_are_bounded_and_parsed(self): + from confluent_kafka import TopicPartition, TopicPartitionInfo, Node, ConsumerGroupTopicPartitions + from confluent_kafka.admin import DescribeClusterResult, TopicDescription, ListOffsetsResultInfo + nodes = [Node(i, "h", 9092) for i in (1, 2, 3)] + admin = Mock() + admin.describe_cluster.return_value = self.done(DescribeClusterResult(nodes[0], nodes)) + admin.describe_topics.return_value = {"t": self.done(TopicDescription( + "t", None, False, [TopicPartitionInfo(0, nodes[0], nodes, nodes[:2])]))} + admin.list_consumer_group_offsets.return_value = {"g": self.done( + ConsumerGroupTopicPartitions("g", [TopicPartition("t", 0, 7)]))} + admin.list_offsets.return_value = {TopicPartition("t", 0): self.done(ListOffsetsResultInfo(9, -1, -1))} + source = KafkaAdminSource(admin) + self.assertEqual(source.describe_cluster(self.deadline), {"broker_ids": [1, 2, 3], "controller_id": 1}) + self.assertEqual(source.describe_topics(("t",), self.deadline), + {"t": [{"partition": 0, "leader": 1, "replicas": [1, 2, 3], "isr": [1, 2]}]}) + self.assertEqual(source.committed_offsets("g", {"t": 1}, self.deadline), {("t", 0): 7}) + self.assertEqual(source.log_end_offsets((("t", 0),), self.deadline), {("t", 0): 9}) + for call in (admin.describe_cluster, admin.describe_topics, + admin.list_consumer_group_offsets, admin.list_offsets): + self.assertLessEqual(call.call_args.kwargs["request_timeout"], 5) + from confluent_kafka import IsolationLevel + self.assertEqual(admin.list_offsets.call_args.kwargs["isolation_level"], IsolationLevel.READ_COMMITTED) + self.assertFalse(any("alter" in name or "delete" in name or "create" in name + for name, _, _ in admin.mock_calls)) + + def test_expired_deadline_makes_no_admin_call(self): + admin = Mock() + self.clock.now += 5 + with self.assertRaises(DeadlineExpired): + KafkaAdminSource(admin).describe_cluster(self.deadline) + admin.describe_cluster.assert_not_called() + + def test_redis_reader_is_scan_get_only_and_bounded(self): + client = Mock() + client.scan_iter.return_value = iter(["service:heartbeat:market_data:a"]) + client.get.return_value = b"{}" + self.assertEqual(redis_heartbeat_reader(client)(self.deadline), [b"{}"]) + self.assertEqual({c[0] for c in client.mock_calls}, {"scan_iter", "get"}) + client.scan_iter.return_value = iter(["k%d" % i for i in range(100)]) + with self.assertRaises(ValueError): + redis_heartbeat_reader(client)(self.deadline) + + def test_docker_reader_inspect_only_and_identity_checked(self): + rows = "\n".join("%s|%s|/%s|running" % (r["id"], r["image"], n) for n, r in ROLES.items()) + run = Mock(return_value=subprocess.CompletedProcess([], 0, rows)) + roles = docker_role_reader(ROLES, run=run)(self.deadline) + self.assertEqual(roles["kafka1"]["state"], "running") + self.assertEqual(run.call_args.args[0][:2], ["docker", "inspect"]) + self.assertLessEqual(run.call_args.kwargs["timeout"], 5) + run.return_value = subprocess.CompletedProcess([], 0, rows.replace("/kafka1", "/other")) + with self.assertRaises(ValueError): + docker_role_reader(ROLES, run=run)(self.deadline) + + +class EndToEndTests(unittest.TestCase): + """Adapter -> existing RecoveryController: ordered progression and fail-closed UNKNOWN.""" + + def test_ordered_gate_progression_and_unknown(self): + h = Harness() + controller = RecoveryController( + RecoveryPolicy(deadline_seconds=120, max_observation_age_seconds=10, stable_samples=2, + retry_seconds=2, max_start_attempts=3), ROLES, h.clock) + + def step(): + report = h.observer.observe(5) + h.clock.now += 1 + h.wall += 1 + return controller.step(report["observation"]) + + trace = [] + # 1: no disk baseline -> HOLD_UNKNOWN, no proposal. + result = step() + trace.append(result["state"]) + self.assertEqual((result["state"], result["actions"]), ("HOLD_UNKNOWN", [])) + # 2: broker exited and unreachable -> definitive NOT_READY -> exact start proposal. + h.states["kafka1"] = "exited" + h.kafka.fail["describe_cluster"] = TimeoutError() + result = step() + trace.append(result["state"]) + self.assertEqual(result["state"], "WAIT_QUORUM") + self.assertEqual(result["actions"][0]["role"], "kafka1") + # 3: broker running, ISR partial -> still WAIT_QUORUM, no further action. + h.states["kafka1"] = "running" + del h.kafka.fail["describe_cluster"] + h.kafka.isr = [1, 2] + result = step() + trace.append(result["state"]) + self.assertEqual((result["state"], result["actions"]), ("WAIT_QUORUM", [])) + # 4: ISR full, projector lagging -> WAIT_PROJECTOR. + h.kafka.isr = [1, 2, 3] + h.kafka.ends = {k: v + 500 for k, v in h.kafka.ends.items()} + result = step() + trace.append(result["state"]) + self.assertEqual(result["state"], "WAIT_PROJECTOR") + # 5: projector caught up, TS not ready -> WAIT_CONSUMER. + h.kafka.committed = dict(h.kafka.ends) + h.heartbeats = [beat(h.wall, status="DEGRADED", session_ready_v2_slices=40, + ready_v2_slices=40, unhealthy_v2_slices=20)] + result = step() + trace.append(result["state"]) + self.assertEqual(result["state"], "WAIT_CONSUMER") + # 6: TS heartbeat missing while the reader container is Up -> HOLD_UNKNOWN. + h.heartbeats = [] + result = step() + trace.append(result["state"]) + self.assertEqual(result["state"], "HOLD_UNKNOWN") + # 7-8: TS session-ready -> stable samples -> SERVING. + for expected in ("WAIT_CONSUMER", "SERVING"): + h.heartbeats = [beat(h.wall)] + result = step() + trace.append(result["state"]) + self.assertEqual(result["state"], expected) + # 9: stale heartbeat after SERVING -> HOLD_UNKNOWN (new bounded episode). + h.heartbeats = [beat(h.wall, age=60)] + result = step() + trace.append(result["state"]) + self.assertEqual(result["state"], "HOLD_UNKNOWN") + self.assertEqual(trace, ["HOLD_UNKNOWN", "WAIT_QUORUM", "WAIT_QUORUM", "WAIT_PROJECTOR", + "WAIT_CONSUMER", "HOLD_UNKNOWN", "WAIT_CONSUMER", "SERVING", + "HOLD_UNKNOWN"]) + + def test_unknown_gate_never_reaches_serving(self): + h = Harness().primed() + h.heartbeats = [] + controller = RecoveryController( + RecoveryPolicy(deadline_seconds=20, max_observation_age_seconds=10, stable_samples=1, + retry_seconds=1, max_start_attempts=1), ROLES, h.clock) + states = set() + for _ in range(25): + report = h.observer.observe(5) + h.clock.now += 1 + h.wall += 1 + states.add(controller.step(report["observation"])["state"]) + self.assertNotIn("SERVING", states) + self.assertIn("BLOCKED", states) + + +if __name__ == "__main__": + unittest.main() From 5c81c25b2e61f3359bfd0d0b39dcdd7fcd53cd15 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 19:58:17 +0000 Subject: [PATCH 075/147] fix(sdk): replicated Query transport starts round-robin on a random replica (P18.4 D3 stage-50 G5) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 31 ++++++++++++++++++++ qdl_sdk/CHANGELOG.md | 6 ++++ qdl_sdk/transport.py | 11 ++++++- tests/test_qdl_sdk_replica_read_transport.py | 23 ++++++++++++--- 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 9e0ceb4e..2edb7472 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71442,3 +71442,34 @@ statvfs is not deadline-bounded (local paths only). Redis call time is bounded b client socket timeouts plus deadline checks between calls. Isolated broker/projector/Stream fault acceptance (G12) remains OPEN. No runtime change, no image build, test containers --rm. No push/merge. + +### D3 window C stages 20/50; stage 50 stopped on G5; SDK replica start fix - 2026-10-09 (Claude) + +Status: window C (owner-approved) ran on window B config (alpha rev15, Stream 1024). The SDK fix is implemented and +tested locally; not deployed and not yet re-measured. +- Stage 20 (600 s, P184 mix, 2 workers): every gate PASS. 12,739 requests, 0 missed/failed/starved; 36 streams, 0 + errors; G5 cold B/O 2500 = 3254/5722 ms, 5000 = 4685/6617 ms; TS READY 64/64 throughout, restarts 0, OOM 0. +- Stage 50 (600 s, 4 workers): 28/29 gates PASS, including every latency class, requests, streams and the four TS + gates. FAIL: g5:COLD_2500:BINANCE 6552.6 ms > frozen 6265.1 ms. Ramp stopped at the first failure (stage 100 not + run). Guard: READY 64/64, restarts 0, OOM 0, disk 87 GB. +- Cause, measured: + - Server time of one 2500-row cold read (query_kn_2, read-only, cold_work marker) is 1.57-1.65 s at idle. bars() is + 6-9 % of it: 2 head calls, 1 pipeline, and no `changed` retry in 8 reads. Most of the time is per-row + construction and render, plus the duty-cycle pauses. + - The driver runs both venues' cold reads at once (`_target_cold` via `asyncio.gather`). Every new SDK client + started its round-robin on replica 0 (qdl_sdk/transport.py `_next = 0`). So both reads hit query_kn_1, whose + large-warmup lane is `max_active=1` (qdl/query/service.py `_queued_local_batch_lane_policy`). + - In every receipt, the later venue finishes one warmup after the earlier one: + - baseline 2337 -> 4177 ms; + - stage 20 3254 -> 5722 ms; + - stage 50 3418 -> 6553 ms. + The frozen limit came from such a queued read. Under the P184 stage-50 mix, each read's own service time grew + 35-45 % (duty cycle under heavier hot load). + - The same start-at-0 rule in production sends every alpha session's startup warmup to one replica. +- Fix (this commit): `ReplicatedRestQueryTransport` starts on a random replica per instance. The new keyword-only + `start_index` pins it. Two tests: random starts cover both replicas; out-of-range start is rejected. The order-based + tests pin start_index=0. SDK CHANGELOG Unreleased; the version bump comes with the D5 release cut. +- Tests: tests.test_qdl_sdk_replica_read_transport 7 OK; 11 SDK modules 91 OK (image 77682eea, --network none). +- Not enough alone: with two independent clients, the chance they pick the same replica stays 1/2. The first-read + (row-cache-miss) server cost is the deterministic lever and is being optimized in a separate slice. The frozen + limit is unchanged. diff --git a/qdl_sdk/CHANGELOG.md b/qdl_sdk/CHANGELOG.md index e4ae7562..2980ec56 100644 --- a/qdl_sdk/CHANGELOG.md +++ b/qdl_sdk/CHANGELOG.md @@ -1,5 +1,11 @@ # SDK Changelog +## Unreleased + +`ReplicatedRestQueryTransport` starts its round-robin on a random replica per +instance instead of always replica 0. A new keyword `start_index` pins the start +(tests, diagnostics). Additive; no request or response change. + ## 2.1.0 Response models, including nested market/reference payloads and error diagnostics, diff --git a/qdl_sdk/transport.py b/qdl_sdk/transport.py index eefda9e2..ec91fc8c 100644 --- a/qdl_sdk/transport.py +++ b/qdl_sdk/transport.py @@ -2,6 +2,7 @@ import asyncio import ipaddress +import random import time from dataclasses import dataclass from collections.abc import AsyncIterator, Sequence @@ -258,6 +259,7 @@ def __init__( *, max_attempts: int = 2, cooldown_seconds: float = 1.0, + start_index: int | None = None, ) -> None: values = tuple(replicas) if len(values) < 2: @@ -269,11 +271,18 @@ def __init__( raise ValueError("replicated Query max_attempts is outside replica bounds") if cooldown_seconds <= 0: raise ValueError("replicated Query cooldown must be positive") + if start_index is not None and not 0 <= start_index < len(values): + raise ValueError("replicated Query start_index is outside replica bounds") self.replicas = values self._health = [_ReplicaReadHealth() for _ in values] self._max_attempts = max_attempts self._cooldown_seconds = cooldown_seconds - self._next = 0 + # Each instance starts its round-robin on a random replica. A fixed + # start sent the first read of every new client (an alpha session's + # startup warmup) to replica 0, where large warmups serialize behind + # the one-active lane (P18.4 D3 stage 50: one cold read waited a whole + # warmup while the other replica was idle). + self._next = random.randrange(len(values)) if start_index is None else start_index async def warmup(self, requirement: DataRequirement, *, consumer_id: str) -> dict: return await self._read("warmup", requirement, consumer_id=consumer_id) diff --git a/tests/test_qdl_sdk_replica_read_transport.py b/tests/test_qdl_sdk_replica_read_transport.py index f3edd075..9f2915b9 100644 --- a/tests/test_qdl_sdk_replica_read_transport.py +++ b/tests/test_qdl_sdk_replica_read_transport.py @@ -35,16 +35,31 @@ class ReplicaReadTransportTests(unittest.IsolatedAsyncioTestCase): async def test_round_robin_uses_both_healthy_replicas(self): first = _Replica("https://query-a", ({"replica": "a"},)) second = _Replica("https://query-b", ({"replica": "b"},)) - transport = ReplicatedRestQueryTransport((first, second)) + transport = ReplicatedRestQueryTransport((first, second), start_index=0) self.assertEqual(await transport.snapshot(object(), consumer_id="alpha"), {"replica": "a"}) self.assertEqual(await transport.snapshot(object(), consumer_id="alpha"), {"replica": "b"}) self.assertEqual((first.calls, second.calls), (1, 1)) + async def test_default_start_spreads_first_reads_over_replicas(self): + firsts = set() + for _ in range(64): + first = _Replica("https://query-a", ({"replica": "a"},)) + second = _Replica("https://query-b", ({"replica": "b"},)) + transport = ReplicatedRestQueryTransport((first, second)) + firsts.add((await transport.snapshot(object(), consumer_id="alpha"))["replica"]) + self.assertEqual(firsts, {"a", "b"}) + + def test_start_index_must_name_a_replica(self): + replicas = (_Replica("https://query-a", ()), _Replica("https://query-b", ())) + for bad in (-1, 2): + with self.assertRaises(ValueError): + ReplicatedRestQueryTransport(replicas, start_index=bad) + async def test_transport_failure_uses_one_alternate_and_cools_failed_replica(self): first = _Replica("https://query-a", (_connect_error(),)) second = _Replica("https://query-b", ({"replica": "b1"}, {"replica": "b2"})) - transport = ReplicatedRestQueryTransport((first, second), cooldown_seconds=60) + transport = ReplicatedRestQueryTransport((first, second), cooldown_seconds=60, start_index=0) self.assertEqual(await transport.snapshot(object(), consumer_id="alpha"), {"replica": "b1"}) self.assertEqual(await transport.snapshot(object(), consumer_id="alpha"), {"replica": "b2"}) @@ -59,7 +74,7 @@ async def test_typed_v2_error_never_falls_back_to_another_replica(self): (DataLayerError("DATA_STALE", "authoritative stale", retryable=True),), ) second = _Replica("https://query-b", ({"replica": "b"},)) - transport = ReplicatedRestQueryTransport((first, second)) + transport = ReplicatedRestQueryTransport((first, second), start_index=0) with self.assertRaisesRegex(DataLayerError, "authoritative stale"): await transport.snapshot(object(), consumer_id="alpha") @@ -68,7 +83,7 @@ async def test_typed_v2_error_never_falls_back_to_another_replica(self): async def test_close_closes_every_owned_replica(self): first = _Replica("https://query-a", ()) second = _Replica("https://query-b", ()) - transport = ReplicatedRestQueryTransport((first, second)) + transport = ReplicatedRestQueryTransport((first, second), start_index=0) await transport.close() self.assertTrue(first.closed) self.assertTrue(second.closed) From 65e075715184f7b72a290f8d6c631ac3ed1e5ded Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 19:54:58 +0000 Subject: [PATCH 076/147] perf(query): reuse encoded static warmup views per cached row (P18.4 D3 G5) Measured in query_kn_2: bars() is 6-9% of a 2500-row warmup with no changed-retries; per-row view building dominates (render ~240 us/row, row derivation ~330 us/row on a miss). Market-cache items now carry their row's RenderSlot; after the first render only quality/cursor/ snapshot/watermark are validated and spliced, byte-identical output. Bounded decimal parse cache and cached QualityFlag names. Repeated 2500-row read 607 -> 278 ms offline; first read neutral. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 51 ++++++++++ qdl/api_v2/router.py | 116 +++++++++++++++++++--- qdl/domain/decimal.py | 17 +++- qdl/query/results.py | 3 + qdl/query/row_cache.py | 22 ++++ qdl/runtime/kn_query_backend.py | 15 +-- qdl/runtime/stable_source.py | 15 ++- tests/test_fund_phase5_api.py | 50 ++++++++++ tests/test_kn_query_backend.py | 6 +- tests/test_phase1_contracts.py | 20 ++++ 10 files changed, 288 insertions(+), 27 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 2edb7472..4e516ce5 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71473,3 +71473,54 @@ tested locally; not deployed and not yet re-measured. - Not enough alone: with two independent clients, the chance they pick the same replica stays 1/2. The first-read (row-cache-miss) server cost is the deterministic lever and is being optimized in a separate slice. The frozen limit is unchanged. + +### D3 window C G5 COLD_2500:BINANCE miss - cold read cost measured, render reuse - 2026-10-09 (Claude subagent) + +Status: IMPLEMENTED / TESTED_LOCALLY; not deployed. Base 902c280 (branch p184-cold-fix). +Gate: g5:COLD_2500:BINANCE 6552.6 ms > frozen 6265.1 ms (limit unchanged). + +Measured, read-only, inside query_kn_2 (image 514506122111, same BAR path as this branch), one +process, reader/backend built like `create_stable_query_app`, run under `cold_work()` with +`configure_query_interpreter()`, 8 reads, Redis reads only: +- 2500 rows, empty row cache (Binance/OKX): bars() 94/97 ms, `_parsed` 43/56, `_history_from_records` + 819/865, render 611/632, total 1567/1651 ms; duty-cycle pauses 491/505 ms of that. +- 2500 rows, warm row cache: bars 112/98, parsed 45/37, history 227/196, render 610/685, total 994/1015. +- 5000 rows: total 2500/2622 (empty cache), 1925/2003 (warm); render 1240-1306 ms. +- bars(): 2 head calls, 1 pipeline, 24 (2500) / 46 (5000) buckets per read in every read; 0 `changed` + retries. The cache read is 6-9 % of the server time; the retry hypothesis is not the cause. +- Dominant: per-row view building. Render (`router._market_item` + pydantic validate/dump) ~240 us/row, + and it re-derives the immutable payload decimals/source/contract on every request; row derivation on + a row-cache miss ~330 us/row (3-4 decimal parses per value across catalog, item and public layers). +- Client side: the gate times `client.warmup` + per-row `validate_product_view` in the load driver. In + all three receipts the second venue of each concurrent pair finishes about one first-venue warmup + later (stage50: OKX 3418, Binance 6553), consistent with the single-active large-warmup lane per + replica (`qdl/query/service.py` `_queued_local_batch_lane_policy`, max_active=1). Not measured. + +Fix (source): +- `qdl/api_v2/router.py` `_market_item_json`/`_render_encoded_chunks`: market-cache items carry their + row's `RenderSlot` (`qdl/query/row_cache.py`, in `_RowEntry`, bounded/evicted with the row cache). + The row's first render validates the whole `MarketDataView` as before and keeps the encoded static + members; later renders validate only quality/cursor/snapshot_id/watermark_offset with the same field + definitions (`_MarketDataViewDynamic`) and splice them. Byte-identical output. +- `qdl/domain/decimal.py`: bounded lru (8192) for `CanonicalDecimal.from_text` (frozen result, errors + not cached); `map(str, digits)` instead of a generator (also `router._decimal`). +- `qdl/runtime/stable_source.py`: cached `QualityFlag.Name` per value. +- bars(), validation layers and the cold duty cycle are unchanged. + +Offline microbenchmark (image 77682eea, --cpus 1, fake Redis, synthetic random-walk rows from the +golden OKX bar, median of 5; base 902c280 vs this commit; bodies sha256-identical): +- 2500 first read 1029.5 -> 1036.9 ms (history 522 -> 463, render 409 -> 489); + repeated read 607.5 -> 277.6 ms (render 409 -> 91). +- 5000 first read 2012.4 -> 2143.3 ms (+6.5 %; render 789 -> 995, history 1011 -> 954); + repeated read 1213.1 -> 569.8 ms (render 811 -> 181). +- Slot memory 1734 B/row (getsizeof); at the default 20,000 row-cache entries <= ~35 MB per replica. + +Tests (image 77682eea, --network none): 29 Query/KN modules, 550 run, 36 skipped (Redis-backed cases +without QDL_KN_TEST_REDIS), 1 failure `test_fund_phase7_contract_security...provider_fallthrough` +which fails identically on 902c280 (ResponseModel extra="ignore"; pre-existing). New: render slot +byte identity / request-field validation (`test_fund_phase5_api`), decimal cache equivalence +(`test_phase1_contracts`); `test_kn_query_backend` row-cache test updated (skipped here). + +Open: a row read once gets no gain (first read neutral 2500, +6.5 % 5000); whether the probe's rows +are in the row cache at stage 50 is unverified (no row-cache stats for single warmups). Effect on +G5 needs a reader rollout (owner-approved packet) and a stage-50 rerun. No runtime change. diff --git a/qdl/api_v2/router.py b/qdl/api_v2/router.py index 70cc9170..75f6153a 100644 --- a/qdl/api_v2/router.py +++ b/qdl/api_v2/router.py @@ -11,8 +11,10 @@ from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse, Response from fastapi.security import APIKeyHeader, HTTPAuthorizationCredentials, HTTPBearer +from pydantic import create_model from qdl.query.cold_work import await_in_thread, cold_yield +from qdl_sdk.models import ResponseModel from qdl.api_v2.models import ( BatchItemResponse, BatchRequirementModel, @@ -182,7 +184,7 @@ def _decimal(value: object) -> DecimalValue: if not parsed.is_finite(): raise ValueError("canonical decimal field must be finite") sign, digits, exponent = parsed.as_tuple() - coefficient = int("".join(str(digit) for digit in digits) or "0") + coefficient = int("".join(map(str, digits)) or "0") if sign: coefficient = -coefficient return DecimalValue( @@ -369,10 +371,15 @@ def _typed_payload(item) -> dict: raise ValueError(f"public typed payload is undefined for {item.feed.value}") -def _market_item(item) -> MarketDataView: +def _quality_fields(item) -> dict: quality = {**asdict(item.quality), "flags": list(item.quality.flags)} if item.feed not in EXECUTION_PRICE_VALIDATION_FEEDS: quality["execution_eligible"] = False + return quality + + +def _market_item(item) -> MarketDataView: + quality = _quality_fields(item) return MarketDataView( instrument_uid=item.instrument_uid, instrument_id=item.instrument_id, @@ -392,6 +399,87 @@ def _market_item(item) -> MarketDataView: ) +# MarketDataView fields that depend on the request (time, cursor, view); +# every other field is a function of the row's immutable content (KN-4 D30). +_VIEW_DYNAMIC_FIELDS = ("quality", "cursor", "snapshot_id", "watermark_offset") +# The same field definitions (types, bounds) as MarketDataView, validated +# alone once a row's static fields were validated and encoded. +_MarketDataViewDynamic = create_model( + "MarketDataViewDynamic", + __base__=ResponseModel, + **{name: (MarketDataView.model_fields[name].annotation, MarketDataView.model_fields[name]) + for name in _VIEW_DYNAMIC_FIELDS}, +) + + +# What ``json.dumps(value, ensure_ascii=False, allow_nan=False, indent=None, +# separators=(",", ":"))`` builds per call (the ``JSONResponse`` encoding); +# one stateless instance, without a constructor per row. +_JSON_ENCODER = json.JSONEncoder(ensure_ascii=False, allow_nan=False, indent=None, separators=(",", ":")) + + +def _encode_json(value) -> bytes: + return _JSON_ENCODER.encode(value).encode("utf-8") + + +_VIEW_DYNAMIC_PREFIX = {name: _encode_json(name) + b":" for name in _VIEW_DYNAMIC_FIELDS} + + +def _view_layout(dump: dict) -> tuple: + """A view's members in field order: each run of static members encoded + once as ``"k":v,...`` bytes, each dynamic member as its field name.""" + + parts, run = [], {} + for key, value in dump.items(): + if key in _VIEW_DYNAMIC_FIELDS: + if run: + parts.append(_encode_json(run)[1:-1]) + run = {} + parts.append(key) + else: + run[key] = value + if run: + parts.append(_encode_json(run)[1:-1]) + return tuple(parts) + + +def _splice_view(layout: tuple, dynamic: dict) -> bytes: + return b"{" + b",".join( + part if isinstance(part, bytes) else _VIEW_DYNAMIC_PREFIX[part] + _encode_json(dynamic[part]) + for part in layout + ) + b"}" + + +def _market_item_json(item) -> bytes: + """``_encode_json(_market_item(item).model_dump(mode="json", by_alias=True))``. + + P18.4 D3 (measured): building, validating and dumping a ``MarketDataView`` + was ~170 us per row, two thirds of a warm 2,500-row warmup, and most of it + re-derives the row's immutable fields (payload decimals, source, contract) + on every request. An item from the market-cache backend carries its row's + ``render_slot``: the row's first render validates the whole view as + before and keeps the encoded static members there (~1.7 KB per row, + bounded and evicted with the row cache); later renders validate only the + request-dependent fields (same field definitions) and splice them in. The view's cross-field rule reads static + fields only. Bytes are identical; items without a slot take the full path. + """ + + slot = item.render_slot + layout = slot.value if slot is not None else None + if layout is None: + dump = _market_item(item).model_dump(mode="json", by_alias=True) + if slot is None: + return _encode_json(dump) + slot.value = layout = _view_layout(dump) + return _splice_view(layout, dump) + return _splice_view(layout, _MarketDataViewDynamic( + quality=_quality_fields(item), + cursor=item.cursor, + snapshot_id=item.snapshot_id, + watermark_offset=item.watermark_offset, + ).model_dump(mode="json", by_alias=True)) + + def _warmup(result) -> WarmupResponse: return _warmup_envelope(result).model_copy( update={"data": [_market_item(item) for item in _cold_rows(result.history.items)]} @@ -466,10 +554,10 @@ def _render_warmup_result(result) -> bytes: """ items = result.history.items - return _render_chunked( + return _render_encoded_chunks( _warmup_envelope(result), len(items), - lambda start, stop: (_market_item(item) for item in _cold_rows(items[start:stop])), + lambda start, stop: b",".join(_market_item_json(item) for item in _cold_rows(items[start:stop])), ) @@ -486,24 +574,24 @@ def _render_chunked(envelope: WarmupResponse, count: int, views) -> bytes: never exists next to its bytes. """ - def encode(value) -> bytes: - return json.dumps( - value, ensure_ascii=False, allow_nan=False, indent=None, separators=(",", ":"), - ).encode("utf-8") + def rows(start: int, stop: int) -> bytes: + return _encode_json([view.model_dump(mode="json", by_alias=True) for view in views(start, stop)])[1:-1] + + return _render_encoded_chunks(envelope, count, rows) + + +def _render_encoded_chunks(envelope: WarmupResponse, count: int, rows) -> bytes: + """Splice ``rows(start, stop)`` (the comma-joined encoded rows) into the envelope.""" suffix = b',"data":[]}' - head = encode(envelope.model_copy(update={"data": []}).model_dump(mode="json", by_alias=True)) + head = _encode_json(envelope.model_copy(update={"data": []}).model_dump(mode="json", by_alias=True)) if not head.endswith(suffix): raise RuntimeError("warmup response layout no longer ends with its data list") parts = [head[: -len(suffix)], b',"data":['] for start in range(0, count, _RENDER_CHUNK_ROWS): - chunk = encode([ - view.model_dump(mode="json", by_alias=True) - for view in views(start, start + _RENDER_CHUNK_ROWS) - ]) if start: parts.append(b",") - parts.append(chunk[1:-1]) + parts.append(rows(start, start + _RENDER_CHUNK_ROWS)) parts.append(b"]}") return b"".join(parts) diff --git a/qdl/domain/decimal.py b/qdl/domain/decimal.py index be84576f..cffaf760 100644 --- a/qdl/domain/decimal.py +++ b/qdl/domain/decimal.py @@ -1,6 +1,7 @@ from __future__ import annotations from dataclasses import dataclass +from functools import lru_cache from decimal import Decimal, InvalidOperation @@ -19,6 +20,12 @@ class CanonicalDecimal: @classmethod def from_text(cls, value: str) -> "CanonicalDecimal": source_text = str(value).strip() + if cls is CanonicalDecimal: + return _canonical_from_text(source_text) + return cls._parse(source_text) + + @classmethod + def _parse(cls, source_text: str) -> "CanonicalDecimal": if not source_text: raise ValueError("decimal value is required") try: @@ -29,7 +36,7 @@ def from_text(cls, value: str) -> "CanonicalDecimal": raise ValueError("canonical decimals must be finite") sign, digits, exponent = parsed.as_tuple() - coefficient_value = int("".join(str(digit) for digit in digits) or "0") + coefficient_value = int("".join(map(str, digits)) or "0") if sign: coefficient_value = -coefficient_value scale = max(0, -exponent) @@ -47,3 +54,11 @@ def as_decimal(self) -> Decimal: def uses_text_coefficient(self) -> bool: return isinstance(self.coefficient, str) + +# A cold 2,500-row BAR warmup parses ~13 decimal texts per row through +# layered checks (catalog lineage, item fields), and adjacent bars repeat +# their prices (P18.4 D3 profile). The result is a frozen value of its text +# alone, so it is shared; only valid texts are stored (a parse error raises +# and is not cached) and the cache is bounded. +_canonical_from_text = lru_cache(maxsize=8192)(CanonicalDecimal._parse) + diff --git a/qdl/query/results.py b/qdl/query/results.py index 0f02efb1..ea550b92 100644 --- a/qdl/query/results.py +++ b/qdl/query/results.py @@ -143,6 +143,9 @@ class MarketDataItem: # + canonical hash) so a renderer may reuse the validated static views of # the same row (KN-4 D30). Quality, cursor and watermark never enter it. render_key: str | None = field(default=None, compare=False, repr=False) + # Internal only, never rendered: the row cache's ``RenderSlot`` of this + # row, where the public renderer keeps the encoded static view fields. + render_slot: Any = field(default=None, compare=False, repr=False) def __post_init__(self) -> None: if not self.instrument_uid.strip() or not self.instrument_id.strip(): diff --git a/qdl/query/row_cache.py b/qdl/query/row_cache.py index 4c584fd4..e318b932 100644 --- a/qdl/query/row_cache.py +++ b/qdl/query/row_cache.py @@ -20,6 +20,28 @@ ROW_CACHE_ENTRIES_ENV = "QDL_KN_ROW_CACHE_ENTRIES" +class RenderSlot: + """A renderer's derivation of one cached row, shared by reference. + + The backend creates one slot per row-cache entry and hands it to every + item it builds from that row; the public renderer stores there the + encoded fields that depend only on the row's content (P18.4 D3). The slot + lives and is evicted with its entry, so the bound is the row cache's. + Copies return the same slot: it caches, it is not item state. + """ + + __slots__ = ("value",) + + def __init__(self) -> None: + self.value: Any = None + + def __copy__(self) -> "RenderSlot": + return self + + def __deepcopy__(self, memo) -> "RenderSlot": + return self + + class BoundedRowCache: def __init__(self, max_entries: int) -> None: if max_entries < 0: diff --git a/qdl/runtime/kn_query_backend.py b/qdl/runtime/kn_query_backend.py index 76340360..f9903ddd 100644 --- a/qdl/runtime/kn_query_backend.py +++ b/qdl/runtime/kn_query_backend.py @@ -36,7 +36,7 @@ """ from __future__ import annotations -from dataclasses import dataclass, fields, replace +from dataclasses import dataclass, field, fields, replace import hashlib import json from pathlib import Path @@ -53,7 +53,7 @@ from qdl.query import CoverageStatus, DataRequirement, FeedType, GapRecord, HistoryResult, MarketDataItem, RecoveryPolicy from qdl.query.contracts import CanonicalErrorCode, QueryProblem from qdl.query.results import NON_REPLAYABLE_STREAM_CURSOR, QueryBackendError, GapScanResult -from qdl.query.row_cache import BoundedRowCache +from qdl.query.row_cache import BoundedRowCache, RenderSlot from qdl.replay.cursor_v3 import CursorV3Claims, SignedCursorV3Codec, requirement_digest from qdl.runtime.kn_hot_view import HOT_FEEDS, HotViewGap, HotViewUnavailable from qdl.runtime.kn_bar_readback import binding_product_key @@ -95,18 +95,20 @@ CURSOR_TTL_ENV = "QDL_KN_CURSOR_TTL_SECONDS" DEFAULT_ROW_CACHE_ENTRIES = 20_000 # Item fields rebuilt per request (time/request dependent); never cached. -_DYNAMIC_ITEM_FIELDS = frozenset({"quality", "watermark_offset", "cursor", "snapshot_id", "render_key"}) +_DYNAMIC_ITEM_FIELDS = frozenset({"quality", "watermark_offset", "cursor", "snapshot_id", "render_key", "render_slot"}) @dataclass class _RowEntry: """Derivations of one immutable row (D30): its envelope, whether its - lineage was proven against the catalog binding, and its static item - fields. Quality, cursor and watermark are not here.""" + lineage was proven against the catalog binding, its static item fields + and the renderer's slot for its encoded static view. Quality, cursor and + watermark are not here.""" envelope: market_data_pb2.EventEnvelope validated: bool = False static: dict | None = None + render: RenderSlot = field(default_factory=RenderSlot) def source_placeholder(view: ProductView) -> str: @@ -486,12 +488,13 @@ def _item(self, requirement, binding, stored, envelope, gap_open): for item_field in fields(item) if item_field.name not in _DYNAMIC_ITEM_FIELDS } - return replace(item, render_key=key) + return replace(item, render_key=key, render_slot=entry.render) quality = self._quality( requirement, binding, envelope, gap_open=gap_open, watermark_offset=stored.cursor.offset, ) return MarketDataItem( **entry.static, quality=quality, watermark_offset=stored.cursor.offset, render_key=key, + render_slot=entry.render, ) def warmup_stats(self) -> dict[str, int]: diff --git a/qdl/runtime/stable_source.py b/qdl/runtime/stable_source.py index 45e28c91..11a62af6 100644 --- a/qdl/runtime/stable_source.py +++ b/qdl/runtime/stable_source.py @@ -3,6 +3,7 @@ import hashlib import time from dataclasses import dataclass, replace +from functools import lru_cache from typing import Callable from qdl.query.cold_work import cold_yield @@ -95,11 +96,17 @@ def _interval_ns(interval: str) -> int: raise ValueError(f"stable BAR interval is unsupported: {interval}") from error +@lru_cache(maxsize=None) +def _quality_flag_name(value: int) -> str: + # ``EnumTypeWrapper.Name`` costs ~15 us and runs for every flag of every + # row of every warmup (P18.4 D3 profile). Only defined values are cached + # (an unknown value raises and is never stored), so the cache is bounded + # by the enum. + return common_pb2.QualityFlag.Name(value).removeprefix("QUALITY_FLAG_") + + def _quality_flag_names(envelope: market_data_pb2.EventEnvelope) -> tuple[str, ...]: - return tuple( - common_pb2.QualityFlag.Name(value).removeprefix("QUALITY_FLAG_") - for value in envelope.quality_flags - ) + return tuple(_quality_flag_name(value) for value in envelope.quality_flags) def _metric_unit_name(value: int) -> str: diff --git a/tests/test_fund_phase5_api.py b/tests/test_fund_phase5_api.py index 334025c4..cfea9775 100644 --- a/tests/test_fund_phase5_api.py +++ b/tests/test_fund_phase5_api.py @@ -544,6 +544,56 @@ def test_batch_chunk_renderer_matches_full_bytes_for_success_and_errors(self): JSONResponse(content=router_module._warmup_batch_response(None, None, empty, ()) .model_dump(mode="json", by_alias=True)).body) + def test_cached_static_view_renders_identical_bytes_and_still_validates_request_fields(self): + # P18.4 D3: a row's encoded static members are kept in its render slot + # after its first render; later renders validate and splice only the + # request-dependent fields (quality, cursor, snapshot, watermark). + import copy + from dataclasses import replace + from pydantic import ValidationError + from qdl.query.row_cache import RenderSlot + from qdl.query.service import WarmupResult + + history = self.backend.history(self.requirement) + self.assertTrue(history.items) + slots = [RenderSlot() for _ in history.items] + + def slotted(value): + return replace(value, items=tuple( + replace(item, render_slot=slot) for item, slot in zip(value.items, slots, strict=True))) + + def reference(value): + return JSONResponse(content=router_module._warmup(WarmupResult("r", value)) + .model_dump(mode="json", by_alias=True)).body + + self.assertEqual(router_module._render_warmup_result(WarmupResult("r", slotted(history))), + reference(history)) + self.assertTrue(all(isinstance(slot.value, tuple) for slot in slots)) + later = replace(history, items=tuple(replace( + item, cursor=f"cursor-{index}", snapshot_id="snapshot-2", watermark_offset=item.watermark_offset + 9, + quality=replace(item.quality, state="STALE", freshness_ms=item.quality.freshness_ms + 61_000, + flags=(*item.quality.flags, "STALE"), execution_eligible=False), + ) for index, item in enumerate(history.items))) + with patch.object(router_module, "_market_item", side_effect=AssertionError("static view rebuilt")): + cached = router_module._render_warmup_result(WarmupResult("r", slotted(later))) + self.assertEqual(cached, reference(later)) + self.assertNotEqual(cached, reference(history)) + # The request fields keep the public field definitions on the fast path. + for bad in (dict(cursor=123), dict(watermark_offset=-1), + dict(quality=replace(history.items[0].quality, state="BOGUS"))): + item = replace(history.items[0], render_slot=slots[0]) + try: + item = replace(item, **bad) + except ValueError: + continue # the internal item already refuses it + with self.subTest(bad=bad), self.assertRaises(ValidationError): + router_module._market_item_json(item) + with self.subTest(bad=bad, path="full"), self.assertRaises(ValidationError): + router_module._market_item_json(replace(item, render_slot=None)) + # A copied item shares its slot; equality ignores it. + self.assertIs(copy.deepcopy(slots[0]), slots[0]) + self.assertEqual(replace(history.items[0], render_slot=slots[0]), history.items[0]) + def test_stale_and_unentitled_sources_return_stable_problem_details(self): stale_requirement = DataRequirement( **{**self.requirement.__dict__, "consumer_grade": ConsumerGrade.EXECUTION} diff --git a/tests/test_kn_query_backend.py b/tests/test_kn_query_backend.py index 9a1bdd3c..4f83672a 100644 --- a/tests/test_kn_query_backend.py +++ b/tests/test_kn_query_backend.py @@ -703,7 +703,8 @@ def test_row_cache_reuses_static_views_byte_identically_and_never_a_verdict(self second = backend.history(req) self.assertEqual(backend.rows.stats()["misses"], 40, "the second read reuses every row") self.assertEqual(second, first) - self.assertTrue(all(item.render_key for item in second.items)) + self.assertTrue(all(item.render_key and item.render_slot is not None for item in second.items)) + self.assertIs(second.items[0].render_slot, first.items[0].render_slot, "one slot per cached row") # A later read at a later time rebuilds quality: no cached verdict. clock["now"] += 3_600_000_000_000 later = backend.history(req) @@ -717,7 +718,8 @@ def render(history) -> bytes: return body def stripped(history): - return replace(history, items=tuple(replace(item, render_key=None) for item in history.items)) + return replace(history, items=tuple(replace(item, render_key=None, render_slot=None) + for item in history.items)) # Cached row derivations render exactly like fresh ones. self.assertEqual(render(later), render(stripped(later))) diff --git a/tests/test_phase1_contracts.py b/tests/test_phase1_contracts.py index 11f8f366..d72b8917 100644 --- a/tests/test_phase1_contracts.py +++ b/tests/test_phase1_contracts.py @@ -40,6 +40,26 @@ def test_rejects_non_finite_and_invalid_values(self): with self.subTest(value=invalid), self.assertRaises(ValueError): CanonicalDecimal.from_text(invalid) + def test_shared_parse_cache_returns_exactly_the_uncached_parse(self): + # P18.4 D3: repeated texts are parsed once; results are frozen values + # of the text, invalid texts are never stored and keep raising. + from qdl.domain import decimal as decimal_module + + texts = ("61234.10", " 61234.10 ", "0", "-0.000", "1E+3", "-12.5e-2", "000.10", + "123456789012345678901.123", "9223372036854775807", "-9223372036854775809") + for text in texts: + with self.subTest(text=text): + cached = CanonicalDecimal.from_text(text) + self.assertEqual(cached, CanonicalDecimal._parse(text.strip())) + self.assertIs(CanonicalDecimal.from_text(text), cached) + before = decimal_module._canonical_from_text.cache_info().currsize + for invalid in ("NaN", "x1", " "): + for _ in range(2): + with self.subTest(value=invalid), self.assertRaises(ValueError): + CanonicalDecimal.from_text(invalid) + self.assertEqual(decimal_module._canonical_from_text.cache_info().currsize, before) + self.assertGreater(decimal_module._canonical_from_text.cache_info().maxsize, 0) + class DeterministicEventIdTests(unittest.TestCase): def test_is_stable_and_length_delimited(self): From 76950c8d1918b4cd650d49f909f431399a333a17 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 20:32:36 +0000 Subject: [PATCH 077/147] perf(query): cheaper first (row-cache-miss) cold BAR read (P18.4 D3 G5) Decimal texts in plain ASCII form are read off the text (same result as Decimal); decimals and metadata go to MarketDataView as model input; the first render uses pydantic's JSON writer (byte-identical); the render slot keeps a layout only from a row's second render; render fields are set at item construction and view-field copies skip a redundant __post_init__. Validation layers and the cold duty cycle unchanged. First read offline: 2500 rows 1091 -> 748 ms, 5000 rows 2065 -> 1540 ms. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 39 +++++++++ qdl/api_v2/router.py | 102 +++++++++++++++++----- qdl/domain/decimal.py | 31 ++++--- qdl/query/results.py | 24 ++++- qdl/runtime/kn_query_backend.py | 24 +++-- qdl/runtime/stable_source.py | 20 ++--- tests/test_fund_phase5_api.py | 58 +++++++++++- tests/test_phase1_contracts.py | 42 +++++---- 8 files changed, 257 insertions(+), 83 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 4e516ce5..5736de95 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71524,3 +71524,42 @@ byte identity / request-field validation (`test_fund_phase5_api`), decimal cache Open: a row read once gets no gain (first read neutral 2500, +6.5 % 5000); whether the probe's rows are in the row cache at stage 50 is unverified (no row-cache stats for single warmups). Effect on G5 needs a reader rollout (owner-approved packet) and a stage-50 rerun. No runtime change. + +### D3 window C G5 - first (row-cache-miss) cold read made cheaper - 2026-10-09 (Claude subagent) + +Status: IMPLEMENTED / TESTED_LOCALLY; not deployed. Supersedes the first-read part of the previous entry: the gate's +read waits one full warmup in the single-active lane (max_active=1), so the first read's service time is what matters. +- Per-row profile of the miss path (offline, image 77682eea, --cpus 1): history derivation ~200 us/row (catalog lineage + check, `bar_item_fields`, `_quality`, item construction), render ~160 us/row (`MarketDataView` input building and + validation, Python dump + `json.dumps`). Each decimal text was parsed by `Decimal` 3 times (catalog, item, public). +- Changes: + - `qdl/domain/decimal.py` `CanonicalDecimal.from_text`: ASCII `-?digits[.digits]` read off the text (identical to the + Decimal result, ~5x cheaper); other spellings keep the Decimal path. Same fast path in `router._decimal_parts`. + The earlier per-text LRU caches were dropped: measured no faster than the fast path, and they thrash at 5000 rows. + - `router._decimal` and source/contract/quality metadata are passed to `MarketDataView` as model input (same + schemas, same validators, one pydantic-core call) instead of Python-built sub-models; `_flat_asdict` replaces a + recursive `asdict` for flat metadata (falls back to `asdict` otherwise). + - First render serializes with `model_dump_json` (byte-identical to `json.dumps(model_dump(mode="json"))` for every + shape tested, incl. control/non-BMP/U+2028 characters; a lone surrogate fails on both paths). The render slot keeps + its layout only from a row's second render on, so a row read once pays no layout encoding. + - `StableSpoolQueryBackend._item(..., extras=)` sets `render_key`/`render_slot` at construction (no second copy). + `MarketDataItem.with_view_fields` copies cursor/snapshot_id without re-running `__post_init__` (which reads + neither); used by `history_with_envelopes` and the cursor issuer's `bind_history`. `ContractMetadata` digest check + uses a set comparison (same rule). + - Unchanged: bars(), every validation layer, the cold duty cycle. +- Microbenchmark (offline, synthetic random-walk BAR rows from the golden OKX bar, fake Redis, process parse caches + cleared before every first read, median of 5; base 902c280 vs this commit; body sha256 identical + 2500 `76a976e71ac4d6b2`, 5000 `b5a1edd37e6a70ed`): + - production entry `history_with_envelopes` + render, first read: 2500 1091.0 -> 747.9 ms (-31 %); + 5000 2065.2 -> 1539.7 ms (-25 %); + - first read by phase, 2500: history 520 -> 385, render 410 -> 243, total 1025.5 -> 728.0 ms (-29 %); + 5000: history 1008 -> 791, render 801 -> 495, total 1996.3 -> 1479.5 ms (-26 %); + - repeated read (3rd): 2500 615.8 -> 275.1 ms; 5000 1178.9 -> 551.6 ms; 2nd read 614.5 -> 583.9 / 1197.2 -> 1158.5. + - The 35 % aim is not reached: what remains is pydantic validation/serialization (~70 us/row) and the protobuf + field reads of the catalog/item/quality layers, which this change keeps. +- Tests (image 77682eea, --network none): 32 Query/KN modules, 569 run, 36 skipped (Redis-backed), 1 failure + (`test_fund_phase7_contract_security...provider_fallthrough`, identical on 902c280, pre-existing); plus 4 decimal + consumer modules (test_phase1_instrument_domain, test_phasec36_*) 39 OK. New: decimal fast path = Decimal reading over + edge spellings; first-render JSON identity over control/unicode text; out-of-contract decimals (scale 39, 29 digits, + 129 chars) still fail in the view; `with_view_fields` equals `replace`. +- Not verified: live effect at stage 50 (needs a reader rollout packet and a rerun). diff --git a/qdl/api_v2/router.py b/qdl/api_v2/router.py index 75f6153a..5ff4af64 100644 --- a/qdl/api_v2/router.py +++ b/qdl/api_v2/router.py @@ -3,7 +3,8 @@ import asyncio from concurrent.futures import ThreadPoolExecutor import json -from dataclasses import asdict +from dataclasses import asdict, fields as dataclass_fields +from functools import lru_cache from decimal import Decimal, InvalidOperation from typing import Annotated @@ -14,12 +15,12 @@ from pydantic import create_model from qdl.query.cold_work import await_in_thread, cold_yield +from qdl.domain.decimal import PLAIN_DECIMAL_TEXT from qdl_sdk.models import ResponseModel from qdl.api_v2.models import ( BatchItemResponse, BatchRequirementModel, BatchResponse, - DecimalValue, FeedStatusResponse, GapListResponse, InstrumentPageResponse, @@ -33,7 +34,6 @@ ReferenceBatchRequest, ReferenceBatchResponse, SnapshotResponse, - SourceView, SystemReadinessSummary, WarmupResponse, ) @@ -171,13 +171,15 @@ def _reference_requirement(model) -> ReferenceDataRequirement: ) -def _decimal(value: object) -> DecimalValue: - if isinstance(value, dict): - return DecimalValue.model_validate(value) - if isinstance(value, float) or isinstance(value, bool): - raise ValueError("public V2 decimal fields cannot originate from binary float/bool") +def _decimal_parts(source: str) -> tuple[str, int]: + """``(coefficient, scale)`` of a decimal text, as ``Decimal`` reads it.""" + + plain = PLAIN_DECIMAL_TEXT.fullmatch(source) + if plain is not None: + # ASCII ``-?digits[.digits]``: exactly the digits/exponent Decimal yields. + fraction = plain.group(2) or "" + return str(int(plain.group(1) + fraction)), len(fraction) try: - source = str(value) parsed = Decimal(source) except (InvalidOperation, TypeError, ValueError) as error: raise ValueError("canonical decimal field is invalid") from error @@ -187,11 +189,27 @@ def _decimal(value: object) -> DecimalValue: coefficient = int("".join(map(str, digits)) or "0") if sign: coefficient = -coefficient - return DecimalValue( - coefficient=str(coefficient), - scale=-exponent, - source_text=source, - ) + return str(coefficient), -exponent + + +def _decimal(value: object) -> dict: + """The ``DecimalValue`` input of a canonical decimal text. + + Returned as model input, not as a model: the enclosing public view + validates it with the same ``DecimalValue`` schema and validator in one + pydantic-core call, instead of one Python model construction per value + (P18.4 D3: 8 per BAR row).""" + + if isinstance(value, dict): + return value + if isinstance(value, float) or isinstance(value, bool): + raise ValueError("public V2 decimal fields cannot originate from binary float/bool") + try: + source = str(value) + except (TypeError, ValueError) as error: + raise ValueError("canonical decimal field is invalid") from error + coefficient, scale = _decimal_parts(source) + return {"coefficient": coefficient, "scale": scale, "source_text": source} def _book_levels(values: object) -> list[dict]: @@ -371,8 +389,34 @@ def _typed_payload(item) -> dict: raise ValueError(f"public typed payload is undefined for {item.feed.value}") +_SCALARS = (str, int, float, type(None)) + + +@lru_cache(maxsize=None) +def _field_names(kind: type) -> tuple[str, ...]: + return tuple(field.name for field in dataclass_fields(kind)) + + +def _flat_asdict(value) -> dict: + """``dataclasses.asdict`` for a dataclass of scalars and tuples of scalars. + + ``asdict`` recurses and copies every value (~15 us for the three metadata + objects of a view, P18.4 D3); for these values its result is this shallow + dict. Anything else takes ``asdict`` itself.""" + + result = {} + for name in _field_names(type(value)): + member = getattr(value, name) + if not isinstance(member, _SCALARS) and not ( + type(member) is tuple and all(isinstance(element, _SCALARS) for element in member) + ): + return asdict(value) + result[name] = member + return result + + def _quality_fields(item) -> dict: - quality = {**asdict(item.quality), "flags": list(item.quality.flags)} + quality = {**_flat_asdict(item.quality), "flags": list(item.quality.flags)} if item.feed not in EXECUTION_PRICE_VALIDATION_FEEDS: quality["execution_eligible"] = False return quality @@ -390,9 +434,10 @@ def _market_item(item) -> MarketDataView: received_at_ns=item.received_at_ns, revision=item.revision, payload=_typed_payload(item), - source=SourceView(**asdict(item.source)), - quality=QualityView(**quality), - contract=asdict(item.contract), + # Model inputs: validated by the view with the same schemas. + source=_flat_asdict(item.source), + quality=quality, + contract=_flat_asdict(item.contract), cursor=item.cursor, snapshot_id=item.snapshot_id, watermark_offset=item.watermark_offset, @@ -423,6 +468,7 @@ def _encode_json(value) -> bytes: _VIEW_DYNAMIC_PREFIX = {name: _encode_json(name) + b":" for name in _VIEW_DYNAMIC_FIELDS} +_RENDERED_ONCE = object() def _view_layout(dump: dict) -> tuple: @@ -457,19 +503,27 @@ def _market_item_json(item) -> bytes: was ~170 us per row, two thirds of a warm 2,500-row warmup, and most of it re-derives the row's immutable fields (payload decimals, source, contract) on every request. An item from the market-cache backend carries its row's - ``render_slot``: the row's first render validates the whole view as - before and keeps the encoded static members there (~1.7 KB per row, - bounded and evicted with the row cache); later renders validate only the - request-dependent fields (same field definitions) and splice them in. The view's cross-field rule reads static + ``render_slot``: a render validates the whole view as before, and from + the row's second render on the encoded static members are kept there + (~1.7 KB per row, bounded and evicted with the row cache); later renders + validate only the request-dependent fields (same field definitions) and + splice them in. The view's cross-field rule reads static fields only. Bytes are identical; items without a slot take the full path. """ slot = item.render_slot layout = slot.value if slot is not None else None if layout is None: + # A row's first render keeps no layout: a row read once pays no + # layout encoding. Pydantic's own JSON serializer writes exactly the + # bytes of ``_encode_json(model_dump(mode="json"))`` (compact, UTF-8, + # same escapes; checked over every BAR shape and all BMP characters + # in tests) at half the cost of a Python dump plus ``json.dumps``. + if slot is not None: + slot.value = _RENDERED_ONCE + return _market_item(item).model_dump_json(by_alias=True).encode("utf-8") + if layout is _RENDERED_ONCE: dump = _market_item(item).model_dump(mode="json", by_alias=True) - if slot is None: - return _encode_json(dump) slot.value = layout = _view_layout(dump) return _splice_view(layout, dump) return _splice_view(layout, _MarketDataViewDynamic( diff --git a/qdl/domain/decimal.py b/qdl/domain/decimal.py index cffaf760..a7689189 100644 --- a/qdl/domain/decimal.py +++ b/qdl/domain/decimal.py @@ -1,12 +1,16 @@ from __future__ import annotations from dataclasses import dataclass -from functools import lru_cache from decimal import Decimal, InvalidOperation +import re INT64_MIN = -(2**63) INT64_MAX = 2**63 - 1 +# ASCII ``-?digits[.digits]`` (no '+', exponent, '_', whitespace or non-ASCII +# digits). For such a text ``Decimal`` yields exactly these digits with +# exponent -len(fraction), so coefficient and scale are read off the text. +PLAIN_DECIMAL_TEXT = re.compile(r"(-?[0-9]+)(?:\.([0-9]+))?") @dataclass(frozen=True) @@ -20,14 +24,18 @@ class CanonicalDecimal: @classmethod def from_text(cls, value: str) -> "CanonicalDecimal": source_text = str(value).strip() - if cls is CanonicalDecimal: - return _canonical_from_text(source_text) - return cls._parse(source_text) - - @classmethod - def _parse(cls, source_text: str) -> "CanonicalDecimal": if not source_text: raise ValueError("decimal value is required") + plain = PLAIN_DECIMAL_TEXT.fullmatch(source_text) + if plain is not None: + # The venue spelling of nearly every price/quantity; ~5x cheaper + # than the Decimal path below and identical in result (P18.4 D3: + # a cold BAR warmup parses ~13 decimal texts per row). + fraction = plain.group(2) or "" + coefficient_value = int(plain.group(1) + fraction) + if INT64_MIN <= coefficient_value <= INT64_MAX: + return cls(coefficient=coefficient_value, scale=len(fraction), source_text=source_text) + return cls(coefficient=str(coefficient_value), scale=len(fraction), source_text=source_text) try: parsed = Decimal(source_text) except InvalidOperation as exc: @@ -53,12 +61,3 @@ def as_decimal(self) -> Decimal: @property def uses_text_coefficient(self) -> bool: return isinstance(self.coefficient, str) - - -# A cold 2,500-row BAR warmup parses ~13 decimal texts per row through -# layered checks (catalog lineage, item fields), and adjacent bars repeat -# their prices (P18.4 D3 profile). The result is a frozen value of its text -# alone, so it is shared; only valid texts are stored (a parse error raises -# and is not cached) and the cache is bounded. -_canonical_from_text = lru_cache(maxsize=8192)(CanonicalDecimal._parse) - diff --git a/qdl/query/results.py b/qdl/query/results.py index ea550b92..aec2eaf8 100644 --- a/qdl/query/results.py +++ b/qdl/query/results.py @@ -82,6 +82,9 @@ def __post_init__(self) -> None: raise ValueError("provider session liveness cannot be negative") +_LOWER_HEX = frozenset("0123456789abcdef") + + @dataclass(frozen=True) class ContractMetadata: schema_digest: str @@ -95,9 +98,7 @@ class ContractMetadata: correlation_id: str def __post_init__(self) -> None: - if len(self.schema_digest) != 64 or any( - character not in "0123456789abcdef" for character in self.schema_digest - ): + if len(self.schema_digest) != 64 or not set(self.schema_digest) <= _LOWER_HEX: raise ValueError("contract schema digest must be lowercase SHA-256") if not all( value.strip() @@ -187,6 +188,23 @@ def __post_init__(self) -> None: ): raise ValueError("bar lifecycle metadata is valid only for bar items") + def with_view_fields(self, **changes: str | None) -> "MarketDataItem": + """``dataclasses.replace`` restricted to ``cursor``/``snapshot_id``. + + ``__post_init__`` reads neither field, so the copy does not re-run it + on fields that did not change; a warmup binds both on every row + (P18.4 D3). Any other field goes through ``replace``.""" + + if not changes.keys() <= _VIEW_FIELDS: + raise TypeError(f"with_view_fields changes only {sorted(_VIEW_FIELDS)}") + clone = object.__new__(type(self)) + clone.__dict__.update(self.__dict__) + clone.__dict__.update(changes) + return clone + + +_VIEW_FIELDS = frozenset({"cursor", "snapshot_id"}) + @dataclass(frozen=True) class HistoryResult: diff --git a/qdl/runtime/kn_query_backend.py b/qdl/runtime/kn_query_backend.py index f9903ddd..f674eace 100644 --- a/qdl/runtime/kn_query_backend.py +++ b/qdl/runtime/kn_query_backend.py @@ -37,6 +37,7 @@ from __future__ import annotations from dataclasses import dataclass, field, fields, replace +from functools import lru_cache import hashlib import json from pathlib import Path @@ -98,6 +99,14 @@ _DYNAMIC_ITEM_FIELDS = frozenset({"quality", "watermark_offset", "cursor", "snapshot_id", "render_key", "render_slot"}) +@lru_cache(maxsize=1) +def _static_item_fields() -> tuple[str, ...]: + names = tuple(item_field.name for item_field in fields(MarketDataItem)) + if not {"quality", "watermark_offset", "cursor", "snapshot_id"} <= set(names): + raise RuntimeError("MarketDataItem no longer has its request-dependent fields") + return tuple(name for name in names if name not in _DYNAMIC_ITEM_FIELDS) + + @dataclass class _RowEntry: """Derivations of one immutable row (D30): its envelope, whether its @@ -482,13 +491,10 @@ def _item(self, requirement, binding, stored, envelope, gap_open): key = self._row_key(binding, stored.payload_sha256) entry = self._row(binding, stored.payload_sha256, stored.event.payload) if entry.static is None: - item = super()._item(requirement, binding, stored, envelope, gap_open) - entry.static = { - item_field.name: getattr(item, item_field.name) - for item_field in fields(item) - if item_field.name not in _DYNAMIC_ITEM_FIELDS - } - return replace(item, render_key=key, render_slot=entry.render) + item = super()._item(requirement, binding, stored, envelope, gap_open, + extras={"render_key": key, "render_slot": entry.render}) + entry.static = {name: getattr(item, name) for name in _static_item_fields()} + return item quality = self._quality( requirement, binding, envelope, gap_open=gap_open, watermark_offset=stored.cursor.offset, ) @@ -565,7 +571,7 @@ def history_with_envelopes( snapshot_id=snapshot_id, stream_cursor=source_placeholder(view), watermark_offset=view.boundary.offset, - items=tuple(replace(item, snapshot_id=snapshot_id) for item in result.items), + items=tuple(item.with_view_fields(snapshot_id=snapshot_id) for item in result.items), ), envelopes def _history_view(self, binding, requested, start_ns, end_ns): @@ -845,7 +851,7 @@ def bind_history( history, stream_cursor=token, items=tuple( - replace(item, snapshot_id=history.snapshot_id, cursor=token) for item in history.items + item.with_view_fields(snapshot_id=history.snapshot_id, cursor=token) for item in history.items ), ) diff --git a/qdl/runtime/stable_source.py b/qdl/runtime/stable_source.py index 11a62af6..587effe1 100644 --- a/qdl/runtime/stable_source.py +++ b/qdl/runtime/stable_source.py @@ -3,7 +3,6 @@ import hashlib import time from dataclasses import dataclass, replace -from functools import lru_cache from typing import Callable from qdl.query.cold_work import cold_yield @@ -96,17 +95,11 @@ def _interval_ns(interval: str) -> int: raise ValueError(f"stable BAR interval is unsupported: {interval}") from error -@lru_cache(maxsize=None) -def _quality_flag_name(value: int) -> str: - # ``EnumTypeWrapper.Name`` costs ~15 us and runs for every flag of every - # row of every warmup (P18.4 D3 profile). Only defined values are cached - # (an unknown value raises and is never stored), so the cache is bounded - # by the enum. - return common_pb2.QualityFlag.Name(value).removeprefix("QUALITY_FLAG_") - - def _quality_flag_names(envelope: market_data_pb2.EventEnvelope) -> tuple[str, ...]: - return tuple(_quality_flag_name(value) for value in envelope.quality_flags) + return tuple( + common_pb2.QualityFlag.Name(value).removeprefix("QUALITY_FLAG_") + for value in envelope.quality_flags + ) def _metric_unit_name(value: int) -> str: @@ -1244,7 +1237,11 @@ def _item( stored: StoredEvent, envelope: market_data_pb2.EventEnvelope, gap_open: bool, + *, + extras: dict[str, Any] | None = None, ) -> MarketDataItem: + """``extras``: internal, non-rendered item fields (``render_key``, + ``render_slot``) set at construction instead of by a second copy.""" payload_name = envelope.WhichOneof("payload") quality = self._quality( requirement, @@ -1282,6 +1279,7 @@ def _item( correlation_id=envelope.correlation_id or stored.event.event_id.hex(), ), watermark_offset=stored.cursor.offset, + **(extras or {}), ) if payload_name == "trade": side = common_pb2.AggressorSide.Name( diff --git a/tests/test_fund_phase5_api.py b/tests/test_fund_phase5_api.py index cfea9775..089f6162 100644 --- a/tests/test_fund_phase5_api.py +++ b/tests/test_fund_phase5_api.py @@ -566,9 +566,12 @@ def reference(value): return JSONResponse(content=router_module._warmup(WarmupResult("r", value)) .model_dump(mode="json", by_alias=True)).body - self.assertEqual(router_module._render_warmup_result(WarmupResult("r", slotted(history))), - reference(history)) - self.assertTrue(all(isinstance(slot.value, tuple) for slot in slots)) + # First render: full validation, no layout kept (a row read once pays + # nothing extra); second render: full validation, layout kept. + for render in ("first", "second"): + self.assertEqual(router_module._render_warmup_result(WarmupResult("r", slotted(history))), + reference(history), render) + self.assertEqual(all(isinstance(slot.value, tuple) for slot in slots), render == "second") later = replace(history, items=tuple(replace( item, cursor=f"cursor-{index}", snapshot_id="snapshot-2", watermark_offset=item.watermark_offset + 9, quality=replace(item.quality, state="STALE", freshness_ms=item.quality.freshness_ms + 61_000, @@ -594,6 +597,55 @@ def reference(value): self.assertIs(copy.deepcopy(slots[0]), slots[0]) self.assertEqual(replace(history.items[0], render_slot=slots[0]), history.items[0]) + def test_first_render_json_and_decimal_inputs_match_the_model_path(self): + # P18.4 D3: the first render serializes with pydantic's JSON writer and + # passes decimals/metadata as model input; bytes and failures must be + # those of the model + json.dumps path. + from dataclasses import replace + from decimal import Decimal + from pydantic import ValidationError + + history = self.backend.history(self.requirement) + item = history.items[-1] + + def reference(value): + return router_module._encode_json(router_module._market_item(value).model_dump(mode="json", by_alias=True)) + + texts = ["é

\x00\x01\x1f\x7f\"\\/\t\n\r\b\f😀", *(chr(code) for code in range(0, 0xD800, 97))] + for text in texts: + changed = replace(item, cursor="c" + text, snapshot_id=text + "s") + self.assertEqual(router_module._market_item_json(changed), reference(changed)) + broken = replace(item, cursor="a\ud800b") + for path in (lambda: router_module._market_item_json(broken), lambda: reference(broken)): + with self.assertRaises(ValueError): # unencodable on both paths + path() + # Decimal parts: the ASCII fast path equals Decimal's reading. + for text in ("61234.10", "-0.000", "000.10", "9" * 40 + ".5", "1E+3", "1e-5", " 1.5", "+2", "1_0", ".5"): + sign, digits, exponent = Decimal(text).as_tuple() + value = int("".join(map(str, digits)) or "0") * (-1 if sign else 1) + with self.subTest(text=text): + self.assertEqual(router_module._decimal_parts(text), (str(value), -exponent)) + for invalid in ("NaN", "x", "1.2.3", 1.5, True): + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + router_module._decimal(invalid) + # Out-of-contract decimals still fail in the public view validation. + for text in ("1e-39", "1" * 29, "1" * 129): + bad = replace(item, payload={**item.payload, "close": text}) + with self.subTest(text=text), self.assertRaises(ValidationError): + router_module._market_item_json(bad) + + def test_view_field_copy_equals_replace_and_changes_only_view_fields(self): + from dataclasses import replace + + item = self.backend.history(self.requirement).items[-1] + fast = item.with_view_fields(cursor="c-1", snapshot_id="s-1") + self.assertEqual(fast, replace(item, cursor="c-1", snapshot_id="s-1")) + self.assertEqual(vars(fast), vars(replace(item, cursor="c-1", snapshot_id="s-1"))) + self.assertIsNot(fast, item) + self.assertNotEqual(item.cursor, "c-1") + with self.assertRaises(TypeError): + item.with_view_fields(watermark_offset=-1) + def test_stale_and_unentitled_sources_return_stable_problem_details(self): stale_requirement = DataRequirement( **{**self.requirement.__dict__, "consumer_grade": ConsumerGrade.EXECUTION} diff --git a/tests/test_phase1_contracts.py b/tests/test_phase1_contracts.py index d72b8917..3f784dc5 100644 --- a/tests/test_phase1_contracts.py +++ b/tests/test_phase1_contracts.py @@ -40,25 +40,33 @@ def test_rejects_non_finite_and_invalid_values(self): with self.subTest(value=invalid), self.assertRaises(ValueError): CanonicalDecimal.from_text(invalid) - def test_shared_parse_cache_returns_exactly_the_uncached_parse(self): - # P18.4 D3: repeated texts are parsed once; results are frozen values - # of the text, invalid texts are never stored and keep raising. - from qdl.domain import decimal as decimal_module - - texts = ("61234.10", " 61234.10 ", "0", "-0.000", "1E+3", "-12.5e-2", "000.10", - "123456789012345678901.123", "9223372036854775807", "-9223372036854775809") + def test_plain_text_fast_path_equals_the_decimal_parse(self): + # P18.4 D3: ASCII ``-?digits[.digits]`` is read off the text; every + # other spelling takes the Decimal path. Both must agree exactly. + from decimal import Decimal + + def by_decimal(text: str) -> CanonicalDecimal: + sign, digits, exponent = Decimal(text).as_tuple() + value = int("".join(map(str, digits)) or "0") * (-1 if sign else 1) + if exponent > 0: + value *= 10**exponent + coefficient = value if -(2**63) <= value <= 2**63 - 1 else str(value) + return CanonicalDecimal(coefficient, max(0, -exponent), text) + + texts = ["61234.10", "0", "-0", "-0.000", "000.10", "00", "1.0", "-12.50", + "123456789012345678901.123", "9223372036854775807", "9223372036854775808", + "-9223372036854775808", "-9223372036854775809", "0.000000000000000000000000000000001", + "1E+3", "-12.5e-2", "+1.5", "1_000.5", ".5", "5.", "٣.1"] + texts += [f"{whole}.{fraction}" for whole in ("0", "7", "-31", "100000000000") + for fraction in ("0", "05", "1234567890", "9" * 30)] for text in texts: with self.subTest(text=text): - cached = CanonicalDecimal.from_text(text) - self.assertEqual(cached, CanonicalDecimal._parse(text.strip())) - self.assertIs(CanonicalDecimal.from_text(text), cached) - before = decimal_module._canonical_from_text.cache_info().currsize - for invalid in ("NaN", "x1", " "): - for _ in range(2): - with self.subTest(value=invalid), self.assertRaises(ValueError): - CanonicalDecimal.from_text(invalid) - self.assertEqual(decimal_module._canonical_from_text.cache_info().currsize, before) - self.assertGreater(decimal_module._canonical_from_text.cache_info().maxsize, 0) + parsed = CanonicalDecimal.from_text(text) + self.assertEqual(parsed, by_decimal(text.strip())) + self.assertEqual(type(parsed.coefficient), type(by_decimal(text.strip()).coefficient)) + for invalid in ("", " ", "NaN", "-Infinity", "x1", "1.2.3", "--1", "1-"): + with self.subTest(value=invalid), self.assertRaises(ValueError): + CanonicalDecimal.from_text(invalid) class DeterministicEventIdTests(unittest.TestCase): From 407932aaedd9b86ad47c27440a6eae8d6d19ecc7 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 20:48:01 +0000 Subject: [PATCH 078/147] test(query): history stub follows MarketDataItem.with_view_fields (76950c8) --- tests/test_binance_history_overlap.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_binance_history_overlap.py b/tests/test_binance_history_overlap.py index ec836c70..d5d50943 100644 --- a/tests/test_binance_history_overlap.py +++ b/tests/test_binance_history_overlap.py @@ -212,6 +212,10 @@ def test_verified_provider_correction_can_complete_without_rewinding_live(self): class Item: snapshot_id: str = "test-only" + def with_view_fields(self, **fields): + # Mirrors MarketDataItem.with_view_fields, which history_with_envelopes uses. + return replace(self, **fields) + class SuffixQueryCoverageTests(unittest.TestCase): def result(self, requested, *, venue="BINANCE", interval="3d", start_ns=None): From d2b99316cb213453604b6080af84e2043faa961a Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 20:48:42 +0000 Subject: [PATCH 079/147] feat(phase3): opt-in sdk_from_source runs driver clients on the checkout SDK (P18.4 D3 window C) --- scripts/phase3_consumer_load_acceptance.py | 16 +++++++++++++++- tests/test_p184_stage100.py | 8 ++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/scripts/phase3_consumer_load_acceptance.py b/scripts/phase3_consumer_load_acceptance.py index 33b5040e..1a1aeba4 100644 --- a/scripts/phase3_consumer_load_acceptance.py +++ b/scripts/phase3_consumer_load_acceptance.py @@ -232,7 +232,8 @@ def validate_profile(raw: dict[str, object]) -> dict[str, object]: # is recorded, never used to relax a gate. # KN-4 D41: a shadow kn4-matrix names the isolated canonical broker its # handoff oracle reads (never a production broker). - optional = {"monitored_containers", "scope", "oracle_bootstrap", "ts_container", "manifest_overrides"} + optional = {"monitored_containers", "scope", "oracle_bootstrap", "ts_container", "manifest_overrides", + "sdk_from_source"} if not expected <= set(raw) or set(raw) - expected - optional: raise ValueError("Phase-3 profile fields are incomplete or unknown") scope = raw.get("scope", "production") @@ -315,9 +316,20 @@ def validate_profile(raw: dict[str, object]) -> dict[str, object]: "ts_container": raw.get("ts_container") if isinstance(raw.get("ts_container"), str) and _NETWORK_RE.fullmatch(raw["ts_container"]) else None, "manifest_overrides": _manifest_overrides(raw.get("manifest_overrides")), + "sdk_from_source": _sdk_from_source(raw.get("sdk_from_source")), } +def _sdk_from_source(value: object) -> bool: + """P18.4 D3 window C: run the clients on this checkout's qdl_sdk (the SDK under release, e.g. the replica + round-robin start fix) instead of the copy baked into the client image. Opt-in; the server side is unchanged.""" + if value is None: + return False + if not isinstance(value, bool): + raise ValueError("Phase-3 sdk_from_source must be a boolean") + return value + + def _manifest_overrides(value: object) -> dict[str, str]: """P18.4 D3 window B: a deployed consumer manifest revision (e.g. alpha rev15) mounted over the client image's baked copy, so the client loads the same quotas and mints JWTs for the same revision as the readers.""" @@ -523,6 +535,8 @@ def docker_command( "--mount", f"type=bind,src={profile['runtime_dir']},dst=/runtime,readonly", *[arg for consumer, path in sorted((profile.get("manifest_overrides") or {}).items()) for arg in ("--mount", f"type=bind,src={path},dst={_PHASE3_CONSUMER_MANIFESTS[consumer]},readonly")], + *(("--mount", f"type=bind,src={ROOT / 'qdl_sdk'},dst=/app/qdl_sdk,readonly") + if profile.get("sdk_from_source") else ()), "--mount", f"type=bind,src={ROOT / 'qdl/certification/phase3_consumer_load.py'},dst=/app/qdl/certification/phase3_consumer_load.py,readonly", "--mount", f"type=bind,src={Path(__file__).resolve()},dst=/driver/phase3_consumer_load_acceptance.py,readonly", "--mount", f"type=bind,src={ROOT / 'scripts/phase3_consumer_load_bootstrap.sh'},dst=/driver/phase3_consumer_load_bootstrap.sh,readonly", diff --git a/tests/test_p184_stage100.py b/tests/test_p184_stage100.py index 3bf4e643..2fcdecf0 100644 --- a/tests/test_p184_stage100.py +++ b/tests/test_p184_stage100.py @@ -127,3 +127,11 @@ def test_overrides_are_validated_and_mounted_over_the_baked_manifest(self): with self.subTest(bad=bad), self.assertRaises(ValueError): driver._manifest_overrides(bad) self.assertEqual(driver._manifest_overrides(None), {}) + + def test_sdk_from_source_is_an_opt_in_boolean(self): + import scripts.phase3_consumer_load_acceptance as driver + self.assertFalse(driver._sdk_from_source(None)) + self.assertTrue(driver._sdk_from_source(True)) + for bad in ("true", 1, []): + with self.assertRaises(ValueError): + driver._sdk_from_source(bad) From d39468b61034e5e2809f96a167532cbf8db1d3c2 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 21:05:03 +0000 Subject: [PATCH 080/147] docs(plan): P18.4 D3 window C stage 50 rerun PASS with SDK replica-start fix --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 5736de95..70b0675f 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71563,3 +71563,21 @@ read waits one full warmup in the single-active lane (max_active=1), so the firs edge spellings; first-render JSON identity over control/unicode text; out-of-contract decimals (scale 39, 29 digits, 129 chars) still fail in the view; `with_view_fields` equals `replace`. - Not verified: live effect at stage 50 (needs a reader rollout packet and a rerun). + +### D3 window C stage 50 rerun PASS with the SDK replica-start fix; stage 100 started - 2026-10-09 (Claude) + +Status: owner goal "run everything per Claude's recommendation". Server runtime unchanged. +- Readers keep image 514506122111 (the rc3 Query rollout is blocked by the session permission classifier and left + to the owner). +- The only change is client-side. Driver profile `sdk_from_source: true` (d2b9931) mounts this checkout's qdl_sdk, + including 5c81c25, over the client image copy. Mount verified on the client container (/app/qdl_sdk). +- Stage 50 rerun (600 s, frozen p184-stage100-budget): 29/29 gates PASS, 0 failed. G5 cold: + - 2500 rows: B/O 2247/2578 ms (limit 6265.1); + - 5000 rows: 5621/5998 ms (limit 15267.0). + Both 2500-row reads took about one service time, without the earlier one-warmup step. This is consistent with the + two venues' cold reads using different replicas. The receipt has no per-read replica, so the routing is inferred + from timing, not observed. +- Guard (20 s samples): TS READY 64/64, restarts 0, OOM 0, disk 88 GB. 0 client containers left. +- Caveat: with two independent clients, random starts still collide with probability 1/2. The rc3 first-read speedup + (65e0757/76950c8) is the deterministic lever. It is built but not deployed. +- Stage 100 (1800 s, 4 x 120 s G4 observer windows) started 21:01Z with the same SDK profile. From 5a8b063631cad4b9ca770f5bef581340f3e163db Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 9 Oct 2026 21:33:50 +0000 Subject: [PATCH 081/147] docs(plan): P18.4 D3 window C stage 100 FAIL (G5 OKX collision, startup RATE_LIMITED); G4 PASS --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 70b0675f..21363b86 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71581,3 +71581,38 @@ Status: owner goal "run everything per Claude's recommendation". Server runtime - Caveat: with two independent clients, random starts still collide with probability 1/2. The rc3 first-read speedup (65e0757/76950c8) is the deterministic lever. It is built but not deployed. - Stage 100 (1800 s, 4 x 120 s G4 observer windows) started 21:01Z with the same SDK profile. + +### D3 window C stage 100 FAIL (2 of 32 gates); both point to the undeployed rc3 Query - 2026-10-09 (Claude) + +Run: 1800 s, 21:01Z, profile with sdk_from_source (5c81c25), readers on 514506122111, frozen p184-stage100-budget. +Guard: 93 samples, TS READY 64/64, restarts 0, OOM 0, disk 88 GB. Client containers removed, guard stopped. +- PASS (30/32): + - every latency class (p95 18-40 ms, p99 <= 207 ms); + - requests: 166,507 offered, 0 missed/failed/starved; + - streams: 161, 0 errors, all 51 bar streams delivered a final bar; + - fault windows BURST / RECONNECT measured; + - the four TS gates (0 disconnects, v2_error flat). +- FAIL g5:COLD_2500:OKX 6658.3 ms > 6265.1. Binance was 3544.4 ms, so OKX ran 3114 ms later, about one warmup. Both + reads picked the same replica (probability 1/2 with random starts) and serialized behind the one-active lane. +- FAIL startup:bounded: 9/100 sessions (s13 s29 s35 s37 s73 s77 s79 s81 s89) ended RATE_LIMITED in SETUP after 271 + retries; setup 81.2 s (< 240 s). + - Arithmetic: 51 BAR sessions warm up 2500/5000 rows at once, about 25 per alpha identity. + - Lane per replica (qdl/query/service.py `_queued_local_batch_lane_policy`): max_active 1, 8 pending per consumer. + So about 18 slots per identity over 2 replicas. + - Each replica serves warmups one at a time at ~3-6 s under load. The frozen retry budget (10 attempts, <= 4 s) + gives up after ~28 s. + - The Query logs carry no line for these refusals, so the source (lane vs identity quota) is inferred. +- G4 (4 x 120 s, observer assigned read_committed, 1 DONE each): every observed row is within baseline p95 x 1.2. + - trade 169/168, quote 178/183, book_delta 161/170, book_snapshot 157/162; + - mark 226/1407 (limits 371/1587); + - bar 1m 175/61356 (OKX open-time metric). + - bar 30m was NOT OBSERVED (no 30m close inside the windows). 5m rows have no baseline. +- Both failures cost per-warmup service time on the one-active lane. + - rc3 (65e0757/76950c8) cuts the first read by 25-31 %. Repeated reads of the same product, which is what a fleet + start mostly is, get cheaper with it: 615 -> 275 ms at 2500 rows and 1179 -> 552 ms at 5000 (offline + microbench). + - rc3 image qdl-v2-python:2.2.3-rc3-76950c8 is built. + - The Query rollout was refused by the session permission classifier and is left to the owner. + - No gate or budget was changed. +- Status: window C stage 100 NOT PASSED. Next: owner rolls query_kn_1, then query_kn_2, to rc3 (image only; + rollback 514506122111), then a stage-100 rerun. From 616342c9a784a25c7d8cc69189d7231ad9b92ea6 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 03:37:11 +0000 Subject: [PATCH 082/147] feat(query): expose canonical event_id on the public market-data view (IDR-082) Additive v2.2.3 field MarketDataView.event_id = lowercase hex of the canonical EventEnvelope.event_id. Query (stable spool and KN market-cache rows, via StableSpoolQueryBackend._item; canary bars) and the SDK stream projection derive it from the same canonical bytes, so a snapshot of an event and its stream delivery carry the same value and TS can classify a same-event session-probe snapshot as REFRESHED. Resampled aggregates and pass-through windows carry none. Appended last in the view, so existing members keep their bytes; the render-slot static layout includes it. SDK field optional (default None); 2.1.0 readers ignore it (extra=ignore). OpenAPI snapshot regenerated; contract doc and SDK changelog updated. --- contracts/v2/kn-v220-contracts.md | 11 ++ contracts/v2/openapi.snapshot.json | 12 ++ qdl/api_v2/router.py | 1 + qdl/query/results.py | 12 ++ qdl/runtime/canary_source.py | 1 + qdl/runtime/stable_source.py | 2 + qdl/warmup/handoff.py | 2 + qdl_sdk/CHANGELOG.md | 6 + qdl_sdk/models.py | 6 + qdl_sdk/projection.py | 2 + tests/test_kn_query_backend.py | 33 ++++ tests/test_p184_view_event_identity.py | 210 +++++++++++++++++++++++++ 12 files changed, 298 insertions(+) create mode 100644 tests/test_p184_view_event_identity.py diff --git a/contracts/v2/kn-v220-contracts.md b/contracts/v2/kn-v220-contracts.md index d7244237..c3268005 100644 --- a/contracts/v2/kn-v220-contracts.md +++ b/contracts/v2/kn-v220-contracts.md @@ -110,6 +110,17 @@ assert the rule and the message prefix (`rule_messages`). Refusal is identities or partitions the result is `NOT_COMPARABLE` and the sink fence decides. Offset gaps from filtering or transaction markers are valid; source gaps are proven from source sequence, never from `offset == previous + 1`. +- **Event identity** (v2.2.3, additive; IDR-082): the public market-data + view (`MarketDataView` of Query snapshot, latest and warmup, and the SDK + stream projection) carries `event_id`, the lowercase hex of the canonical + `EventEnvelope.event_id` bytes (deterministic from venue, market, + instrument, source and the native identity; 16 bytes on the stable path). + It is not a coordinate and never appears in cursors or watermarks. A Query + snapshot of an event and the Stream delivery of the same event carry the + same value, across replays and cache rebuilds; consumers compare it to tell + a re-read of the same event from a new one. It is `null` when the server + predates it or the row is not one canonical event (provider pass-through + window, resampled aggregate); readers must accept its absence. ## 3. Logical product key diff --git a/contracts/v2/openapi.snapshot.json b/contracts/v2/openapi.snapshot.json index f47c6f68..9c2d544c 100644 --- a/contracts/v2/openapi.snapshot.json +++ b/contracts/v2/openapi.snapshot.json @@ -1311,6 +1311,18 @@ ], "title": "Cursor" }, + "event_id": { + "anyOf": [ + { + "pattern": "^(?:[0-9a-f]{2}){1,64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Event Id" + }, "feed": { "$ref": "#/components/schemas/FeedType" }, diff --git a/qdl/api_v2/router.py b/qdl/api_v2/router.py index 5ff4af64..2ed80115 100644 --- a/qdl/api_v2/router.py +++ b/qdl/api_v2/router.py @@ -441,6 +441,7 @@ def _market_item(item) -> MarketDataView: cursor=item.cursor, snapshot_id=item.snapshot_id, watermark_offset=item.watermark_offset, + event_id=item.event_id, ) diff --git a/qdl/query/results.py b/qdl/query/results.py index aec2eaf8..e9dd9972 100644 --- a/qdl/query/results.py +++ b/qdl/query/results.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable @@ -18,6 +19,9 @@ if TYPE_CHECKING: from qdl.warmup.handoff import ResampleLineage +# Public view event identity: canonical event ids are 16 or 32 bytes today. +_EVENT_ID_HEX = re.compile(r"(?:[0-9a-f]{2}){1,64}") + # A fresh provider snapshot has no durable canonical-log position to resume. # Keeping one contract sentinel prevents edge adapters from fabricating a cursor. @@ -140,6 +144,12 @@ class MarketDataItem: supersedes_event_id: str | None = None received_at_ns: int | None = None resample_lineage: "ResampleLineage | None" = None + # IDR-082 (v2.2.3, additive): lowercase hex of the canonical + # ``EventEnvelope.event_id`` this row was read from. The same canonical + # bytes reach Stream, so a stream-delivered event and a snapshot of it + # carry the same value. ``None`` when the row is not one canonical event + # (pass-through window, resampled aggregate) or the envelope has no id. + event_id: str | None = None # Internal only, never rendered: identifies immutable row content (binding # + canonical hash) so a renderer may reuse the validated static views of # the same row (KN-4 D30). Quality, cursor and watermark never enter it. @@ -157,6 +167,8 @@ def __post_init__(self) -> None: raise ValueError("market-data watermark_offset cannot be negative") if self.received_at_ns is not None and self.received_at_ns <= 0: raise ValueError("market-data received_at_ns must be positive") + if self.event_id is not None and _EVENT_ID_HEX.fullmatch(self.event_id) is None: + raise ValueError("market-data event_id must be lowercase hex of 1..64 bytes") if self.feed is FeedType.BAR and not self.interval: raise ValueError("bar item requires interval") if self.feed in METRIC_INTERVAL_FEEDS and not self.interval: diff --git a/qdl/runtime/canary_source.py b/qdl/runtime/canary_source.py index 574e848e..766f30f4 100644 --- a/qdl/runtime/canary_source.py +++ b/qdl/runtime/canary_source.py @@ -615,6 +615,7 @@ def _item( correlation_id=envelope.correlation_id or stored.event.event_id.hex(), ), watermark_offset=stored.cursor.offset, + event_id=bytes(envelope.event_id).hex() or None, bar_lifecycle=BarLifecycle(lifecycle_name), supersedes_event_id=( bytes(bar.supersedes_event_id).hex() if bar.HasField("supersedes_event_id") else None diff --git a/qdl/runtime/stable_source.py b/qdl/runtime/stable_source.py index 587effe1..ffbcaf75 100644 --- a/qdl/runtime/stable_source.py +++ b/qdl/runtime/stable_source.py @@ -1279,6 +1279,8 @@ def _item( correlation_id=envelope.correlation_id or stored.event.event_id.hex(), ), watermark_offset=stored.cursor.offset, + # IDR-082: the canonical event identity Stream delivers unchanged. + event_id=bytes(envelope.event_id).hex() or None, **(extras or {}), ) if payload_name == "trade": diff --git a/qdl/warmup/handoff.py b/qdl/warmup/handoff.py index 39a76539..8ab536aa 100644 --- a/qdl/warmup/handoff.py +++ b/qdl/warmup/handoff.py @@ -125,6 +125,8 @@ def resample_final_bars( watermark_offset=max(item.watermark_offset for item in group), bar_lifecycle=BarLifecycle.FINAL, supersedes_event_id=None, + # An aggregate of several canonical events is none of them. + event_id=None, contract=replace( last.contract, correlation_id=f"resample:{constituent_sha256}", diff --git a/qdl_sdk/CHANGELOG.md b/qdl_sdk/CHANGELOG.md index 2980ec56..0a5e00e7 100644 --- a/qdl_sdk/CHANGELOG.md +++ b/qdl_sdk/CHANGELOG.md @@ -6,6 +6,12 @@ instance instead of always replica 0. A new keyword `start_index` pins the start (tests, diagnostics). Additive; no request or response change. +`MarketDataView.event_id` (optional, default `None`): lowercase hex of the canonical +`EventEnvelope.event_id`, rendered by the v2.2.3 server on Query snapshot/latest/warmup +rows and set by `market_data_view_from_stream` from the delivered envelope, so a +snapshot of an event and its stream delivery carry the same value (IDR-082). Additive +response field; 2.1.0 readers ignore it, existing fields are unchanged. + ## 2.1.0 Response models, including nested market/reference payloads and error diagnostics, diff --git a/qdl_sdk/models.py b/qdl_sdk/models.py index 8e2a73d0..335a7299 100644 --- a/qdl_sdk/models.py +++ b/qdl_sdk/models.py @@ -571,6 +571,12 @@ class MarketDataView(ResponseModel): cursor: str | None = None snapshot_id: str | None = None watermark_offset: int = Field(default=0, ge=0) + # Additive (server v2.2.3, IDR-082): lowercase hex of the canonical + # ``EventEnvelope.event_id``. A Query snapshot of an event and the Stream + # delivery of the same event carry the same value; ``None`` when the + # server predates it or the row is not one canonical event (pass-through + # window, resampled aggregate). + event_id: str | None = Field(default=None, pattern=r"^(?:[0-9a-f]{2}){1,64}$") @model_validator(mode="after") def feed_matches_payload(self): diff --git a/qdl_sdk/projection.py b/qdl_sdk/projection.py index a5f36435..432e77d4 100644 --- a/qdl_sdk/projection.py +++ b/qdl_sdk/projection.py @@ -546,5 +546,7 @@ def market_data_view_from_stream( }, "cursor": event.resume_token, "watermark_offset": event.logical_offset, + # The canonical event identity, as the Query view renders it. + "event_id": bytes(envelope.event_id).hex() or None, } ) diff --git a/tests/test_kn_query_backend.py b/tests/test_kn_query_backend.py index 4f83672a..826dcd11 100644 --- a/tests/test_kn_query_backend.py +++ b/tests/test_kn_query_backend.py @@ -735,6 +735,39 @@ def stripped(history): router._market_item(replace(item, render_key=None)).model_dump(mode="json", by_alias=True), ) + def test_cached_latest_views_carry_the_canonical_event_identity_stream_delivers(self): + """IDR-082: the KN view's ``event_id`` is the canonical envelope id, the + same value the SDK stream projection derives from the same record, on + the first (derived) and later (row-cache) reads alike.""" + + import importlib + + from qdl_sdk import DataRequirement as SdkRequirement, Feed, Grade, StreamEvent + from qdl_sdk import market_data_view_from_stream + from qdl_sdk.models import MarketDataView + + router = importlib.import_module("qdl.api_v2.router") + backend = self.backend() + trades = [(b, p) for b, p in self.latest_records if b.feed is FeedType.TRADE] + self.assertGreaterEqual(len(trades), 2, "the golden has real Binance and OKX trades") + for binding, payload in trades: + with self.subTest(binding=binding.binding_id): + envelope = market_data_pb2.EventEnvelope.FromString(payload) + self.put_latest(binding, payload, mark=5) + first = backend.latest(requirement(binding)) + cached = backend.latest(requirement(binding)) + self.assertEqual(first.event_id, bytes(envelope.event_id).hex()) + self.assertEqual(cached.event_id, first.event_id) + snapshot = MarketDataView.model_validate_json(router._market_item_json(cached)) + stream = market_data_view_from_stream( + StreamEvent(51, "signed", envelope), template=snapshot, + requirement=SdkRequirement( + instrument_uid=binding.instrument.instrument_uid, feed=Feed.TRADE, + consumer_grade=Grade.ALPHA, source_policy_id=binding.source_policy_id), + now_ns=int(envelope.received_at_ns) + 1_000_000, + ) + self.assertEqual(stream.event_id, snapshot.event_id) + # ------------------------------------------------------------ D31 MARK/INDEX def test_execution_mark_index_comes_from_the_cache_with_execution_bounds(self): diff --git a/tests/test_p184_view_event_identity.py b/tests/test_p184_view_event_identity.py new file mode 100644 index 00000000..004806b1 --- /dev/null +++ b/tests/test_p184_view_event_identity.py @@ -0,0 +1,210 @@ +"""IDR-082 (v2.2.3, additive): the public market-data view carries the event identity. + +A Query snapshot of a canonical event and the Stream delivery of the same event +must expose the same ``event_id`` (lowercase hex of ``EventEnvelope.event_id``), +so a consumer can tell a re-read of the last trade from a new trade. These +cases use the real canonical TRADE records of the KN codec golden, the stable +spool Query backend (whose ``_item`` the KN market-cache backend inherits), +the public HTTP renderer and the SDK stream projection. The KN Redis path is +covered by ``tests.test_kn_query_backend`` (needs ``QDL_KN_TEST_REDIS``). +""" +from __future__ import annotations + +import importlib +import tempfile +import unittest +from dataclasses import replace +from pathlib import Path + +from pydantic import ValidationError + +from qdl.marketdata.v2 import market_data_pb2 +from qdl.query import ConsumerGrade, DataRequirement, FeedType +from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_source import StableSpoolQueryBackend +from qdl.transport import DurableEvent, SQLiteDurableSpool, SpoolConfig +from qdl_sdk import DataRequirement as SdkRequirement, Feed, Grade as SdkGrade, StreamEvent, market_data_view_from_stream +from qdl_sdk.models import MarketDataView, ResponseModel, SnapshotResponse +from tests.test_kn_bar_legacy_import import golden_records +from tests.test_qdl_sdk_stream_projection import template + +ROOT = Path(__file__).resolve().parents[1] +CATALOG_PATH = ROOT / "config/v2/stable-source-bindings.yaml" +router = importlib.import_module("qdl.api_v2.router") + + +def _golden_trades(): + return [ + (pk, payload) for pk, payload in golden_records(bar=False) + if market_data_pb2.EventEnvelope.FromString(payload).WhichOneof("payload") == "trade" + ] + + +class ViewEventIdentityTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.catalog = StableSourceCatalog.load(CATALOG_PATH) + cls.trades = _golden_trades() + if not cls.trades: + raise AssertionError("the KN codec golden has no real TRADE record") + + def binding(self, partition_key: str): + return next( + item for item in self.catalog.bindings + if item.partition_key == partition_key and item.feed is FeedType.TRADE + ) + + def query_backend(self, binding, payloads, now_ns: int) -> StableSpoolQueryBackend: + directory = tempfile.TemporaryDirectory(prefix="idr082-spool-") + self.addCleanup(directory.cleanup) + spool = SQLiteDurableSpool(SpoolConfig( + path=Path(directory.name) / "spool.sqlite3", max_records=1_000, max_payload_bytes=10_000_000, + max_event_bytes=64_000, max_storage_bytes=40_000_000, min_free_disk_bytes=0, + max_partition_records=1_000)) + self.addCleanup(spool.close) + spool.append_many([ + DurableEvent(stream=binding.canonical_stream, partition_key=binding.partition_key, + event_id=bytes(market_data_pb2.EventEnvelope.FromString(payload).event_id), + payload=payload, accepted_at_ns=1_000 + index) + for index, payload in enumerate(payloads) + ]) + return StableSpoolQueryBackend(spool, self.catalog, schema_digest="a" * 64, clock_ns=lambda: now_ns) + + def snapshot_and_stream(self, partition_key: str, payload: bytes, *, offset: int = 41): + envelope = market_data_pb2.EventEnvelope.FromString(payload) + binding = self.binding(partition_key) + now_ns = int(envelope.received_at_ns) + 1_000_000 + item = self.query_backend(binding, [payload], now_ns).latest(DataRequirement( + instrument_uid=binding.instrument.instrument_uid, feed=FeedType.TRADE, + consumer_grade=ConsumerGrade.ALPHA, source_policy_id=binding.source_policy_id, + )) + self.assertIsNotNone(item) + # The public HTTP bytes, read back by the SDK as a consumer does. + body = b'{"request_id":"idr082","data":' + router._market_item_json(item) + b"}" + snapshot = SnapshotResponse.model_validate_json(body).data + stream = market_data_view_from_stream( + StreamEvent(offset, "signed", envelope), + template=snapshot, + requirement=SdkRequirement( + instrument_uid=binding.instrument.instrument_uid, feed=Feed.TRADE, + consumer_grade=SdkGrade.ALPHA, source_policy_id=binding.source_policy_id, + ), + now_ns=now_ns, + ) + return envelope, item, snapshot, stream + + def test_query_snapshot_and_stream_delivery_of_one_trade_share_its_event_id(self): + seen = set() + for partition_key, payload in self.trades: + with self.subTest(partition_key=partition_key): + envelope, item, snapshot, stream = self.snapshot_and_stream(partition_key, payload) + expected = bytes(envelope.event_id).hex() + self.assertTrue(expected) + self.assertEqual(item.event_id, expected) + self.assertEqual(snapshot.event_id, expected) + self.assertEqual(stream.event_id, snapshot.event_id) + # Same trade, same identity; the transport fields are what differ. + self.assertEqual(stream.payload.native_trade_id, snapshot.payload.native_trade_id) + seen.add(expected) + self.assertEqual(len(seen), len(self.trades), "distinct trades have distinct identities") + + def test_a_later_snapshot_of_the_same_event_keeps_its_identity(self): + partition_key, payload = self.trades[0] + _, _, first, _ = self.snapshot_and_stream(partition_key, payload, offset=41) + _, _, again, stream = self.snapshot_and_stream(partition_key, payload, offset=97) + self.assertEqual(again.event_id, first.event_id) + self.assertEqual(stream.watermark_offset, 97) + self.assertEqual(stream.event_id, first.event_id) + + def test_render_slot_paths_emit_identical_bytes_with_the_identity(self): + partition_key, payload = self.trades[0] + _, item, _, _ = self.snapshot_and_stream(partition_key, payload) + + class Slot: + value = None + + slotted = replace(item, render_slot=Slot()) + full = router._market_item(item).model_dump_json(by_alias=True).encode("utf-8") + # First render (full model), second (layout built), third (static splice). + renders = [router._market_item_json(slotted) for _ in range(3)] + self.assertEqual(renders, [full] * 3) + self.assertIn(b'"event_id":"' + item.event_id.encode() + b'"', full) + # Additive: the new member is appended; every existing member is unchanged. + without = router._market_item(replace(item, event_id=None)).model_dump_json(by_alias=True) + self.assertEqual(full.decode(), without[:-len(',"event_id":null}')] + f',"event_id":"{item.event_id}"}}') + + def test_sdk_view_tolerates_absence_and_validates_the_identity(self): + view = template(Feed.TRADE).model_dump(mode="json") + self.assertNotIn("event_id", {k for k, v in view.items() if v is not None}) + self.assertIsNone(MarketDataView.model_validate(view).event_id, "older server: absent") + self.assertEqual(MarketDataView.model_validate({**view, "event_id": "0a" * 16}).event_id, "0a" * 16) + for bad in ("", "ABCD", "abc", "zz" * 16, "0a" * 65): + with self.subTest(bad=bad), self.assertRaises(ValidationError): + MarketDataView.model_validate({**view, "event_id": bad}) + + def test_a_reader_without_the_field_ignores_it(self): + # SDK 2.1.0 response models are ``extra="ignore"``: the same view model + # without ``event_id`` reads a v2.2.3 body to the same known fields. + fields = {name: (info.annotation, info) for name, info in MarketDataView.model_fields.items() + if name != "event_id"} + from pydantic import create_model + old_view = create_model("OldMarketDataView", __base__=ResponseModel, **fields) + body = {**template(Feed.TRADE).model_dump(mode="json"), "event_id": "0a" * 16} + old = old_view.model_validate(body) + self.assertFalse(hasattr(old, "event_id")) + self.assertEqual(old.model_dump(), {k: v for k, v in MarketDataView.model_validate(body).model_dump().items() + if k != "event_id"}) + + def test_stable_query_always_has_an_identity_and_stream_tolerates_none(self): + # The stable catalog admits only a 16-byte canonical event id + # (``binding_for_envelope``), so every stable Query row has one. + partition_key, trade = self.trades[0] + _, item, snapshot, _ = self.snapshot_and_stream(partition_key, trade) + self.assertRegex(item.event_id, r"\A[0-9a-f]{32}\Z") + # Stream projection adds no new refusal: an id-less frame projects with none. + stripped = market_data_pb2.EventEnvelope.FromString(trade) + stripped.ClearField("event_id") + stream = market_data_view_from_stream( + StreamEvent(42, "signed", stripped), template=snapshot, + requirement=SdkRequirement( + instrument_uid=snapshot.instrument_uid, feed=Feed.TRADE, + consumer_grade=SdkGrade.ALPHA, source_policy_id=snapshot.quality.policy_id, + ), + now_ns=int(stripped.received_at_ns) + 1_000_000, + ) + self.assertIsNone(stream.event_id) + + def test_resample_clears_the_constituent_identity(self): + from qdl.query.lifecycle import BarLifecycle + from qdl.query.results import ContractMetadata, MarketDataItem, QualityMetadata, SourceMetadata + from qdl.warmup.handoff import resample_final_bars + + def bar(open_ns: int, event_id: str) -> MarketDataItem: + return MarketDataItem( + instrument_uid="uid-1", instrument_id="BINANCE.USDM.PERPETUAL.BTC-USDT", instrument_revision=1, + feed=FeedType.BAR, interval="1m", observed_at_ns=open_ns + 59_999_999_999, + payload={"open_time_ns": open_ns, "close_time_ns": open_ns + 59_999_999_999, "open": "1", + "high": "2", "low": "1", "close": "2", "volume": "3", "volume_unit": "BASE_ASSET", + "trade_count": 1, "origin": "PROVIDER", "is_final": True}, + source=SourceMetadata(venue="BINANCE", provider="BINANCE_DIRECT", source_id="s", + source_role="PRIMARY", authoritative=True), + quality=QualityMetadata(state="LIVE", freshness_ms=0, gap_open=False, complete=True, + execution_eligible=False, policy_id="p", flags=()), + contract=ContractMetadata(schema_digest="a" * 64, contract_version="2.0.0", + normalizer_version="n", adapter_version="a", + instrument_catalog_revision=1, source_policy_revision=1, + authority_revision=1, config_revision=1, correlation_id=event_id), + bar_lifecycle=BarLifecycle.FINAL, event_id=event_id, + ) + + base = 1_700_000_100 // 300 * 300 * 1_000_000_000 + items = [bar(base + index * 60_000_000_000, f"{index:02x}" * 16) for index in range(5)] + (aggregate,) = resample_final_bars(items, target_interval="5m") + self.assertIsNone(aggregate.event_id) + self.assertEqual(items[-1].event_id, "04" * 16) + with self.assertRaisesRegex(ValueError, "event_id"): + replace(items[0], event_id="NOT-HEX") + + +if __name__ == "__main__": + unittest.main() From f5d3671cc8acbb0dc78914a3461eed8751eb47f5 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 03:38:20 +0000 Subject: [PATCH 083/147] docs(plan): IDR-082 DL view event identity (616342c) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 21363b86..32435472 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71616,3 +71616,26 @@ Guard: 93 samples, TS READY 64/64, restarts 0, OOM 0, disk 88 GB. Client contain - No gate or budget was changed. - Status: window C stage 100 NOT PASSED. Next: owner rolls query_kn_1, then query_kn_2, to rc3 (image only; rollback 514506122111), then a stage-100 rerun. + +### IDR-082 DL side: view event identity (v2.2.3, additive) - 2026-10-10 (Claude, subagent) + +Owner decision 2026-10-10: DL adds the trade identity; TS uses it in phase E. Commit 616342c, cherry-picked from +3d5e14a. +- The public market-data view now carries `event_id`: the lowercase hex of the canonical `EventEnvelope.event_id`. + - The id is a deterministic blake2b of venue, market, instrument, source and native trade id + (qdl/canonical/trade.py:129-139, qdl/domain/event_id.py). + - The stable catalog admits only 16-byte ids. +- Query derives it in `StableSpoolQueryBackend._item`, which the KN backend inherits and caches as a static row field. + The SDK stream projection derives it from the same canonical envelope bytes. A snapshot of a trade and its stream + delivery therefore carry the same value. +- Resampled aggregates and pass-through windows carry none. +- The field is appended last, so existing members keep their bytes; the render-slot static layout includes it. + The SDK field is optional, and 2.1.0 readers ignore it (extra=ignore). +- Also regenerated: OpenAPI snapshot; updated: KN contract doc and SDK changelog. +- Tests: + - new tests/test_p184_view_event_identity.py (7, real Binance/OKX golden trades) and a KN Redis case; + - subagent: 26 modules, 386 tests, 1 known phase7 failure, 33 skipped; 127 more OK; KN on a disposable --rm Redis + 33 OK; + - after the cherry-pick: 5 modules OK (33 Redis skips). +- Cost: about 45 bytes more per row; warmup throughput not measured. +- Status: tested locally, not deployed. From bd74faa8f1a98af7600bee2e5243947119aaa3bc Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 03:41:53 +0000 Subject: [PATCH 084/147] fix(query): single-flight MARK hot lookups and env cold slice (P18.4 D3 stage 100 on rc3) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 28 ++++++ qdl/query/cold_work.py | 21 +++- qdl/runtime/kn_query_backend.py | 40 +++++++- tests/test_kn_mark_hot_memo.py | 115 ++++++++++++++++++++++ tests/test_query_cold_work.py | 13 +++ 5 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 tests/test_kn_mark_hot_memo.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 32435472..e59a5d5a 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71639,3 +71639,31 @@ Owner decision 2026-10-10: DL adds the trade identity; TS uses it in phase E. Co - after the cherry-pick: 5 modules OK (33 Redis skips). - Cost: about 45 bytes more per row; warmup throughput not measured. - Status: tested locally, not deployed. + +### D3 window C stage 100 on rc3: 30/32; DATA_NOT_READY root cause and rc4 fixes - 2026-10-10 (Claude) + +Run: owner-run roll of query_kn_1/2 to qdl-v2-python:2.2.3-rc3-76950c8 (sha256:672503b7fe87, label revision 76950c8, +03:06Z); stage 100 at 1800 s, SDK profile, frozen budget, 4 x 120 s G4 windows; guard READY 64/64, restarts 0, +OOM 0. +- PASS: startup (setup 61.5 s, 0 failures, 190 retries; was 9 RATE_LIMITED on the old image); every latency class; + streams; TS gates. +- FAIL requests:no_failures, 2 DATA_NOT_READY on OKX BNB-USDT-SWAP (TRADE probe 03:31:41Z, QUOTE snapshot 03:33:59Z, + query_v2_1). + - The detail hash matches `HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND: HOT_READER_BUSY` (kn_query_backend.py + `_latest_view`). Query logs carry no line for it. + - The Rust hot reader scans its ring under `try_lock` (rust/qdl-stream-gateway/src/hub.rs latest/check_hot_reader), + so concurrent lookups refuse each other. + - rc3 deployed IDR-084, which queries the hot reader on every MARK read. The stage-50/100 runs on the old image had + 0 request failures. The added MARK lookup load is the likely trigger: inferred, not measured. +- FAIL g5:COLD_2500:OKX 6992.6 ms. Binance took 3510.9 ms; the difference of 3482 ms is one warmup, so the two reads + shared a replica again. + - The Binance read barely improved (3544 -> 3511 ms) even though rc3 cuts first-read compute by about 31 % + offline. Under load the read time is dominated by GIL wait after each duty-cycle pause, not by compute. +- Fixes (this commit, tested locally): + - MARK hot lookups are single-flight per binding for 50 ms (`_hot_latest`, MARK_HOT_MEMO_NS). The outcome is + shared, refusals included and re-raised fresh. Other feeds are unchanged. The caller keeps every + lineage/monotonic gate. + - Cold slice from env QDL_QUERY_COLD_SLICE_MS (integer 1..50, default 4; out of range refused at import). This lets + one variable be tuned on the same image against the frozen G5 limit while the hot latency classes are watched. + - Tests: new tests/test_kn_mark_hot_memo.py (6), ColdSliceEnvTests; 96 related tests OK (33 Redis skips). +- G4 (g4b-1..4) is summarized with the rc4 run. diff --git a/qdl/query/cold_work.py b/qdl/query/cold_work.py index db02fa4b..42f893d1 100644 --- a/qdl/query/cold_work.py +++ b/qdl/query/cold_work.py @@ -22,10 +22,29 @@ import asyncio from contextlib import contextmanager from functools import partial +import os import threading import time -COLD_SLICE_SECONDS = 0.004 +COLD_SLICE_ENV = "QDL_QUERY_COLD_SLICE_MS" + + +def cold_slice_seconds(value: str | None) -> float: + """The cold work slice from ``QDL_QUERY_COLD_SLICE_MS`` (integer 1..50, default 4). + + P18.4 D3 stage 100: under fleet load a cold warmup spends most of its time + waiting to re-take the GIL after each pause, not computing. A longer slice + gives cold reads more of the GIL at the cost of at most one slice of extra + hot latency. Out-of-range values are refused at import, never clamped.""" + + if value is None or value == "": + return 0.004 + if not value.isdigit() or not 1 <= int(value) <= 50: + raise ValueError(f"{COLD_SLICE_ENV} must be an integer 1..50 (ms)") + return int(value) / 1000.0 + + +COLD_SLICE_SECONDS = cold_slice_seconds(os.environ.get(COLD_SLICE_ENV)) COLD_PAUSE_SECONDS = 0.002 _state = threading.local() diff --git a/qdl/runtime/kn_query_backend.py b/qdl/runtime/kn_query_backend.py index f674eace..e8e6ccb4 100644 --- a/qdl/runtime/kn_query_backend.py +++ b/qdl/runtime/kn_query_backend.py @@ -95,6 +95,8 @@ ROUTE_GENERATION_ENV = "QDL_KN_ROUTE_GENERATION" CURSOR_TTL_ENV = "QDL_KN_CURSOR_TTL_SECONDS" DEFAULT_ROW_CACHE_ENTRIES = 20_000 +# A MARK/INDEX hot-view lookup is shared by every read of that binding on one replica for this long. +MARK_HOT_MEMO_NS = 50_000_000 # Item fields rebuilt per request (time/request dependent); never cached. _DYNAMIC_ITEM_FIELDS = frozenset({"quality", "watermark_offset", "cursor", "snapshot_id", "render_key", "render_slot"}) @@ -195,6 +197,10 @@ def __init__( # Bounded by the immutable catalog, coordinates only (no second cache). self._hot_boundaries: dict[str, tuple[object, int, str]] = {} self._hot_boundary_lock = threading.Lock() + # IDR-084 MARK lookups, single-flight per binding (see _hot_latest). + self._mark_hot_memo: dict[str, tuple[int, object]] = {} + self._mark_hot_locks: dict[str, threading.Lock] = {} + self._mark_hot_locks_guard = threading.Lock() # ------------------------------------------------------------ cache view @@ -276,7 +282,7 @@ def _hot_candidate(self, binding, primary=None): if self.hot_client is None or binding.feed.value not in HOT_FEEDS: return None try: - candidate = self.hot_client.latest(binding, self.product_key(binding)) + candidate = self._hot_latest(binding) except HotViewGap as error: raise QueryBackendError(QueryProblem( CanonicalErrorCode.OPEN_SEQUENCE_GAP, str(error), True, @@ -301,6 +307,38 @@ def _hot_candidate(self, binding, primary=None): raise HotViewUnavailable("HOT_BACKUP_BEHIND_RETURNED_VIEW") return candidate + def _hot_latest(self, binding): + """One hot-view lookup; MARK/INDEX lookups are shared per binding for MARK_HOT_MEMO_NS. + + P18.4 D3 stage 100 on rc3: the IDR-084 path made every MARK read query the + Rust hot reader, whose ring scan runs under try_lock + (rust/qdl-stream-gateway/src/hub.rs check_hot_reader/latest). Concurrent + lookups refused each other HOT_READER_BUSY, and two TRADE/QUOTE reads that + needed the backup ended DATA_NOT_READY. Many sessions read the same MARK + product, so concurrent and back-to-back MARK reads on one replica now share + one lookup (outcome, including a refusal) for 50 ms. The caller still takes + the newer of cache and hot view under every lineage/monotonic gate, so a + shared result can only be older, never less verified. Other feeds are + unchanged.""" + if binding.feed is not FeedType.MARK_INDEX_PRICE: + return self.hot_client.latest(binding, self.product_key(binding)) + key = binding.binding_id + with self._mark_hot_locks_guard: + lock = self._mark_hot_locks.setdefault(key, threading.Lock()) + with lock: + now = self._monotonic_ns() + memo = self._mark_hot_memo.get(key) + if memo is None or now - memo[0] >= MARK_HOT_MEMO_NS: + try: + outcome = self.hot_client.latest(binding, self.product_key(binding)) + except HotViewUnavailable as error: + outcome = error + memo = (now, outcome) + self._mark_hot_memo[key] = memo + if isinstance(memo[1], HotViewUnavailable): + raise type(memo[1])(*memo[1].args) # a fresh exception per reader thread + return memo[1] + def _latest_view(self, requirement, binding): primary_error = None try: diff --git a/tests/test_kn_mark_hot_memo.py b/tests/test_kn_mark_hot_memo.py new file mode 100644 index 00000000..c3c07a40 --- /dev/null +++ b/tests/test_kn_mark_hot_memo.py @@ -0,0 +1,115 @@ +"""IDR-084 MARK hot lookups are shared per binding (P18.4 D3 stage 100, HOT_READER_BUSY).""" + +from __future__ import annotations + +import threading +import unittest +from types import SimpleNamespace + +from qdl.query import FeedType +from qdl.runtime.kn_hot_view import HotViewGap, HotViewUnavailable +from qdl.runtime.kn_query_backend import MARK_HOT_MEMO_NS, KnMarketCacheQueryBackend + + +class _Hot: + def __init__(self, outcomes): + self.outcomes = list(outcomes) + self.calls = 0 + + def latest(self, binding, key): + self.calls += 1 + outcome = self.outcomes[min(self.calls, len(self.outcomes)) - 1] + if isinstance(outcome, BaseException): + raise outcome + return outcome + + +def _backend(hot, clock): + backend = KnMarketCacheQueryBackend.__new__(KnMarketCacheQueryBackend) + backend.hot_client = hot + backend.product_key = lambda binding: binding.binding_id + backend._monotonic_ns = lambda: clock[0] + backend._mark_hot_memo = {} + backend._mark_hot_locks = {} + backend._mark_hot_locks_guard = threading.Lock() + return backend + + +MARK = SimpleNamespace(binding_id="mark-a", feed=FeedType.MARK_INDEX_PRICE) +MARK_B = SimpleNamespace(binding_id="mark-b", feed=FeedType.MARK_INDEX_PRICE) +QUOTE = SimpleNamespace(binding_id="quote-a", feed=FeedType.QUOTE) + + +class MarkHotMemoTests(unittest.TestCase): + def test_mark_reads_inside_the_window_share_one_lookup(self): + clock = [1_000] + hot = _Hot(["v1", "v2"]) + backend = _backend(hot, clock) + self.assertEqual(backend._hot_latest(MARK), "v1") + clock[0] += MARK_HOT_MEMO_NS - 1 + self.assertEqual(backend._hot_latest(MARK), "v1") + self.assertEqual(hot.calls, 1) + clock[0] += 1 + self.assertEqual(backend._hot_latest(MARK), "v2") + self.assertEqual(hot.calls, 2) + + def test_bindings_do_not_share(self): + clock = [0] + hot = _Hot(["a", "b"]) + backend = _backend(hot, clock) + self.assertEqual(backend._hot_latest(MARK), "a") + self.assertEqual(backend._hot_latest(MARK_B), "b") + self.assertEqual(hot.calls, 2) + + def test_other_feeds_always_ask(self): + clock = [0] + hot = _Hot(["q1", "q2"]) + backend = _backend(hot, clock) + self.assertEqual(backend._hot_latest(QUOTE), "q1") + self.assertEqual(backend._hot_latest(QUOTE), "q2") + self.assertEqual(hot.calls, 2) + + def test_a_refusal_is_shared_and_raised_fresh(self): + clock = [0] + hot = _Hot([HotViewUnavailable("HOT_READER_BUSY")]) + backend = _backend(hot, clock) + errors = [] + for _ in range(2): + with self.assertRaises(HotViewUnavailable) as caught: + backend._hot_latest(MARK) + errors.append(caught.exception) + self.assertEqual(hot.calls, 1) + self.assertEqual([str(e) for e in errors], ["HOT_READER_BUSY", "HOT_READER_BUSY"]) + self.assertIsNot(errors[0], errors[1]) + + def test_a_gap_keeps_its_type(self): + clock = [0] + backend = _backend(_Hot([HotViewGap("HOT_RECORD_GAPPED")]), clock) + for _ in range(2): + with self.assertRaises(HotViewGap): + backend._hot_latest(MARK) + + def test_concurrent_readers_make_one_lookup(self): + clock = [0] + gate = threading.Event() + + class SlowHot(_Hot): + def latest(self, binding, key): + gate.wait(1) + return super().latest(binding, key) + + hot = SlowHot(["v"]) + backend = _backend(hot, clock) + results = [] + threads = [threading.Thread(target=lambda: results.append(backend._hot_latest(MARK))) for _ in range(8)] + for thread in threads: + thread.start() + gate.set() + for thread in threads: + thread.join(2) + self.assertEqual(results, ["v"] * 8) + self.assertEqual(hot.calls, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_query_cold_work.py b/tests/test_query_cold_work.py index 7f7e04fd..392d13ed 100644 --- a/tests/test_query_cold_work.py +++ b/tests/test_query_cold_work.py @@ -227,5 +227,18 @@ def build(): self.assertLessEqual(len(set(names)), 2) + +class ColdSliceEnvTests(unittest.TestCase): + def test_default_and_bounds(self): + self.assertEqual(cold_work.cold_slice_seconds(None), 0.004) + self.assertEqual(cold_work.cold_slice_seconds(""), 0.004) + self.assertEqual(cold_work.cold_slice_seconds("1"), 0.001) + self.assertEqual(cold_work.cold_slice_seconds("20"), 0.02) + self.assertEqual(cold_work.cold_slice_seconds("50"), 0.05) + for bad in ("0", "51", "-1", "4.5", "x", " 4"): + with self.subTest(bad=bad), self.assertRaises(ValueError): + cold_work.cold_slice_seconds(bad) + + if __name__ == "__main__": unittest.main() From 75673bce235d0f7534314847810c0545d8305626 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 03:39:34 +0000 Subject: [PATCH 085/147] feat(p184): G12 isolated broker/projector/Stream fault harness and packet (D4-R3) scripts/p184_g12_isolated_fault.py runs G12 in a disposable holder-owned loopback namespace: three KRaft brokers (RF3, min ISR 2, mTLS), the deployed core/projector/Stream/Query images, a TEST_ONLY BOOK publisher, two TS writer clones (TS WriterClone, read-only from the TS worktree) and a sampler using the TS at-use book validator. Records the G11 four timestamps per fault and G1 transit in a fixed window. DR05 namespace guard also accepts the exact G12 shape. Packet: upgrade/evidence/p184-g12-isolated-fault-packet.md (written before the run). Tests: test_p184_g12_isolated_fault + test_p184_dr05_dl_fault 46 PASS. --- scripts/p184_dr05_dl_fault.py | 6 +- scripts/p184_g12_isolated_fault.py | 1190 +++++++++++++++++ tests/test_p184_dr05_dl_fault.py | 5 +- tests/test_p184_g12_isolated_fault.py | 210 +++ .../p184-g12-isolated-fault-packet.md | 130 ++ 5 files changed, 1538 insertions(+), 3 deletions(-) create mode 100644 scripts/p184_g12_isolated_fault.py create mode 100644 tests/test_p184_g12_isolated_fault.py create mode 100644 upgrade/evidence/p184-g12-isolated-fault-packet.md diff --git a/scripts/p184_dr05_dl_fault.py b/scripts/p184_dr05_dl_fault.py index c5a88ef3..e5a90c2e 100644 --- a/scripts/p184_dr05_dl_fault.py +++ b/scripts/p184_dr05_dl_fault.py @@ -33,8 +33,10 @@ def namespace(value): - if not re.fullmatch(r"p184-dr05-dl-[0-9a-f]{16}", value): - raise ValueError("exact disposable DR05 namespace required") + # G12 (scripts/p184_g12_isolated_fault.py) reuses this material/frame code + # for its own disposable namespace; both shapes stay exact and disposable. + if not re.fullmatch(r"p184-(?:dr05|g12)-dl-[0-9a-f]{16}", value): + raise ValueError("exact disposable DR05/G12 namespace required") return value diff --git a/scripts/p184_g12_isolated_fault.py b/scripts/p184_g12_isolated_fault.py new file mode 100644 index 00000000..0bd5cfd8 --- /dev/null +++ b/scripts/p184_g12_isolated_fault.py @@ -0,0 +1,1190 @@ +#!/usr/bin/env python3 +"""P18.4 G12 / D4-R3: isolated Kafka broker / projector / Stream fault acceptance. + +Purpose: run the G12 gate (TS upgrade/p184-phase-d-gates.md) once, in a disposable +namespace: three KRaft brokers (RF3, min ISR 2, mTLS), the production core, +projector, Stream and Query images, a test-provenance BOOK publisher, two TS +writer clones (the TS-owned ``WriterClone``, production entrypoint, 4 workers) +and a sampler that reads each clone's BOOK view through the TS at-use validator. +Faults: stop/start one broker, restart the projector, restart Stream. Per fault +it records the G11 four timestamps (injected, cleared, first valid apply, +caught up) and then G1 transit inside a fixed window. + +Reuse, not a new engine: material/frames/raw envelopes come from the DR05 tools +(``p184_dr05_dl_fault``, ``p184_dr05_raw_publisher``), the Query read view is the +DR05/DR08 ``serve_query``, cluster/ISR/committed-offset reads are the D4-R2 +``kn_recovery_observe.KafkaAdminSource``, the TS clone is +``scripts/p184_dr05_isolated.WriterClone`` loaded read-only from the TS worktree. + +Boundary: every container is ``--rm``, named ``-`` and labelled +``p184.g12=``; all join one holder's ``--network none`` namespace (no Docker +network, no published port). The shared stack, shared Redis/Kafka/Postgres and +any container not created here are never touched; the deployed TS writer is only +``docker inspect``-ed (WriterClone's own contract). Production containers' +restart counts are watched as a stop condition. No secret is printed. Synthetic +data carries TEST_ONLY provenance; results are ISOLATED_FAULT rehearsal evidence, +not production or HA evidence. + +Host modes (stdlib only): ``--run BASE`` (orchestrate, absolute deadline, teardown +in ``finally``), ``--down BASE`` (exact-name, label-checked teardown), +``--analyze BASE`` (recompute the verdict from the JSONL evidence). +In-container modes (python image, inside the namespace): ``--topics``, +``--publish``, ``--monitor``; each bounded by ``--seconds`` / its own deadline. +""" +from __future__ import annotations + +import argparse +import json +import math +import os +from pathlib import Path +import re +import shutil +import signal +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +LABEL = "p184.g12" +NS_PATTERN = r"p184-g12-dl-[0-9a-f]{16}" +TS_ROOT = Path("/home/bobby/.worktrees/trading-system-next-upgrade") +TS_WRITER = "market_data_service_p184w" +IMAGES = { + "kafka": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "core": "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023", + "projector": "sha256:a696cfba6ab383173607f96413c814f74c7abcc98a4c41f57c9c6e78a41bb70b", + "stream": "sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9", + "python": "sha256:672503b7fe87265d12af3d45fc4d344fdef0a76e94e06efe52e8dc59cab24d0a", + "redis": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", + "ts": "sha256:15524c0bea8c51949d7ced79291d4c72722292fea2f0a5d4f8e9c7fd61dbb5ca", + "postgres": "sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685", + "holder": "sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc", +} +BROKERS = (1, 2, 3) +CLUSTER_ID = "MkU3OEVBNTcwNTJENDM2Qk" +BOOTSTRAP = ",".join(f"localhost:{9092 + 100 * (n - 1)}" for n in BROKERS) +VENUES = ("BINANCE", "OKX") +CLONE_PORTS = {"BINANCE": (6391, 5441), "OKX": (6392, 5442)} +PROJECTOR_GROUPS = ("kn-projector-v3-a", "kn-projector-v3-b") +# Frozen before the run (packet sections 5-6). +GATE = {"caught_up_s": 600, "age_ms": 1000, "hold_s": 30, "window_s": 300, + "p95_ms": 500, "p99_ms": 1000, "max_ms": 2000, "p99_min_n": 100} +LIMITS = {"disk_free_bytes": 20 * 10**9, "mem_available_bytes": 3 * 2**30, + "deadline_s": 3600, "holder_s": 5400, "broker_outage_s": 60, + "role_return_s": 120, "clone_ready_s": 300, "baseline_s": 300} +PRIVATE_FILES = ("client.key", "server.p12", "binance.key", "okx.key", "cursor-keys.json", + "read-view.secret", "broker.properties", "broker-1.properties", + "broker-2.properties", "broker-3.properties") + + +def namespace(value): + if not isinstance(value, str) or not re.fullmatch(NS_PATTERN, value): + raise ValueError("exact disposable G12 namespace required") + return value + + +def broker_port(node): + if node not in BROKERS: + raise ValueError("unknown broker") + return 9092 + 100 * (node - 1) + + +def topics(ns): + """name -> (partitions, config). RF3 and min ISR 2 everywhere (production shape).""" + namespace(ns) + data = {"min.insync.replicas": "2"} + state = {**data, "cleanup.policy": "compact", "delete.retention.ms": "604800000"} + return {ns + ".raw": (1, data), ns + ".canonical": (1, data), ns + ".quarantine": (1, data), + ns + ".latest": (3, state), ns + ".bars": (3, {**state, "min.compaction.lag.ms": "3600000"})} + + +def broker_password(text): + """The keystore password DR05 material wrote; never logged.""" + found = re.findall(r"^ssl\.keystore\.password=(\S+)$", text, flags=re.M) + if len(found) != 1: + raise ValueError("exactly one keystore password required") + return found[0] + + +def broker_properties(password, node): + port = broker_port(node) + voters = ",".join(f"{n}@localhost:{broker_port(n) + 1}" for n in BROKERS) + return f"""process.roles=broker,controller +node.id={node} +controller.quorum.voters={voters} +listeners=SSL://:{port},CONTROLLER://:{port + 1} +advertised.listeners=SSL://localhost:{port} +controller.listener.names=CONTROLLER +listener.security.protocol.map=SSL:SSL,CONTROLLER:PLAINTEXT +inter.broker.listener.name=SSL +ssl.keystore.type=PKCS12 +ssl.keystore.location=/work/server.p12 +ssl.keystore.password={password} +ssl.key.password={password} +ssl.truststore.type=PEM +ssl.truststore.location=/work/ca.crt +ssl.client.auth=required +log.dirs=/data +log.segment.bytes=16777216 +metadata.log.segment.bytes=16777216 +log.index.size.max.bytes=1048576 +num.partitions=1 +default.replication.factor=3 +min.insync.replicas=2 +offsets.topic.num.partitions=1 +offsets.topic.replication.factor=3 +transaction.state.log.num.partitions=1 +transaction.state.log.replication.factor=3 +transaction.state.log.min.isr=2 +unclean.leader.election.enable=false +auto.create.topics.enable=false +group.initial.rebalance.delay.ms=0 +""" + + +def _base(ns, role, *, network, memory, cpus, readonly=True, uid=None): + namespace(ns) + argv = ["docker", "run", "-d", "--rm", "--pull=never", "--name", f"{ns}-{role}", + "--label", f"{LABEL}={ns}", "--network", network, "--memory", memory, "--cpus", cpus, + "--pids-limit", "256", "--log-opt", "max-size=4m", "--log-opt", "max-file=1"] + if readonly: + argv.append("--read-only") + if uid is not None: + argv += ["--user", uid] + return argv + + +def _env(argv, env): + for key, value in env.items(): + argv += ["-e", f"{key}={value}"] + return argv + + +def topology(ns, work, topic_id="", *, uid=None): + """Docker argv per role (adapted from p184_dr05_dl_fault.topology: 3 brokers, + a projector, one holder-owned loopback namespace). Executes nothing.""" + namespace(ns) + work = Path(work).resolve() + if work == ROOT or ROOT in work.parents or any(c in str(work) for c in (":", ",")): + raise ValueError("external unambiguous work directory required") + uid = uid or f"{os.getuid()}:{os.getgid()}" + net = "container:" + ns + "-net" + tls = {"QDL_KN_KAFKA_CA": "/work/ca.crt", "QDL_KN_KAFKA_CERT": "/work/client.crt", + "QDL_KN_KAFKA_KEY": "/work/client.key"} + common = {"QDL_ENVIRONMENT": "paper", "QDL_KN_CURSOR_KEYS_FILE": "/work/cursor-keys.json", + "QDL_KN_CURSOR_ACTIVE_KEY_ID": "dr08", "QDL_KN_TOPIC_ID": topic_id, + "QDL_KN_ROUTE_GENERATION": ns, "QDL_KN_READ_VIEW_SECRET_FILE": "/work/read-view.secret"} + mount = ["-v", f"{work}:/work"] + roles = {"net": _base(ns, "net", network="none", memory="16m", cpus="0.05") + + ["--entrypoint", "sleep", IMAGES["holder"], str(LIMITS["holder_s"])]} + for node in BROKERS: + argv = _base(ns, f"kafka-{node}", network=net, memory="1024m", cpus="0.75", uid=uid) + argv += ["--tmpfs", "/tmp:rw,exec,size=64m,mode=1777", "-v", f"{work}/kafka-{node}:/data", + "-v", f"{work}:/work:ro"] + for target in ("/etc/kafka/secrets", "/mnt/shared/config", "/var/lib/kafka/data"): + argv += ["--tmpfs", target + ":rw,size=1m,mode=1777"] # no anonymous volumes + _env(argv, {"KAFKA_HEAP_OPTS": "-Xms256m -Xmx512m", "LOG_DIR": "/tmp/kafka-logs"}) + props = f"/work/broker-{node}.properties" + roles[f"kafka-{node}"] = argv + ["--entrypoint", "/bin/sh", IMAGES["kafka"], "-ec", + f"/opt/kafka/bin/kafka-storage.sh format --ignore-formatted -t {CLUSTER_ID} -c {props}" + f" && exec /opt/kafka/bin/kafka-server-start.sh {props}"] + roles["quota"] = _base(ns, "quota", network=net, memory="192m", cpus="0.1", uid=uid) + [ + "--tmpfs", "/data:rw,size=1m,mode=1777", "--entrypoint", "redis-server", IMAGES["redis"], + "--save", "", "--appendonly", "no", "--maxmemory", "128mb", "--maxmemory-policy", "noeviction"] + roles["core"] = _env(_base(ns, "core", network=net, memory="256m", cpus="0.5", uid=uid) + + ["--tmpfs", "/tmp:rw,size=64m,mode=1777"] + mount, + {"QDL_KAFKA_CERT_ROOT": "/work", "QDL_KAFKA_BOOTSTRAP_SERVERS": BOOTSTRAP, + "QDL_KAFKA_CLIENT_ID": ns, "QDL_KAFKA_GROUP_ID": ns, "QDL_PROVIDER_ADMISSION_ENABLED": "false"}) + [ + "--entrypoint", "/usr/local/bin/qdl-realtime-core", IMAGES["core"], "/work/core.json"] + roles["projector"] = _env(_base(ns, "projector", network=net, memory="256m", cpus="0.5", uid=uid) + + ["--tmpfs", "/tmp:rw,size=64m,mode=1777"] + mount, + {**tls, "QDL_KN_KAFKA_BOOTSTRAP": BOOTSTRAP, "QDL_KN_BUNDLE_PATH": "/work/bundle.json", + "QDL_KN_TOPIC_ID": topic_id, "QDL_KN_REPLICA": "a", "QDL_KN_MATERIALIZER_EPOCH": "1", + "QDL_KN_MARKET_CACHE_URL": "redis://localhost:6379/1", + "QDL_KN_STATUS_PATH": "/work/g12-projector-status.json", "QDL_KN_STATUS_INTERVAL_S": "5", + "QDL_KN_CANONICAL_TOPIC": ns + ".canonical", "QDL_KN_LATEST_TOPIC": ns + ".latest", + "QDL_KN_BARS_TOPIC": ns + ".bars"}) + [ + "--entrypoint", "/usr/local/bin/qdl-projector", IMAGES["projector"], "run"] + roles["stream"] = _env(_base(ns, "stream", network=net, memory="512m", cpus="0.5", uid=uid) + + ["--tmpfs", "/tmp:rw,size=64m,mode=1777"] + mount, + {**common, **tls, "QDL_KN_BUNDLE_FILE": "/work/bundle.json", + "QDL_KN_JWT_CONFIG_FILE": "/work/jwt-config.json", + "QDL_KN_QUOTA_REDIS_URL": "redis://localhost:6379/0", "QDL_KN_QUOTA_PREFIX": "qdl:stable:v2:paper:" + ns, + "QDL_KN_TLS_CERT_FILE": "/work/client.crt", "QDL_KN_TLS_KEY_FILE": "/work/client.key", + "QDL_KN_TLS_CLIENT_CA_FILE": "/work/ca.crt", "QDL_KN_KAFKA_BOOTSTRAP": BOOTSTRAP, + "QDL_KN_KAFKA_TOPIC": ns + ".canonical", "QDL_KN_KAFKA_GROUP": "kn-" + ns, + "QDL_KN_KAFKA_CLIENT_ID": ns + "-stream", "QDL_KN_READ_VIEW_URLS": "https://localhost:8200", + "QDL_KN_READ_VIEW_CA_FILE": "/work/ca.crt", "QDL_KN_HOT_READ_ENABLED": "true", + "QDL_KN_METRICS_SECONDS": "10", "QDL_KN_MAX_REPLAY_EVENTS": "10000", + "QDL_KN_QUEUE_BYTES_TOTAL": "33554432"}) + [ + "--entrypoint", "/usr/local/bin/qdl-stream-gateway", IMAGES["stream"], "serve"] + roles["query"] = _env(_base(ns, "query", network=net, memory="512m", cpus="0.5", uid=uid) + + ["--tmpfs", "/tmp:rw,size=64m,mode=1777"] + mount + ["-v", f"{ROOT}:/app:ro", "-w", "/app"], + {**common, "PYTHONPATH": "/app:/app/generated/python", "PYTHONDONTWRITEBYTECODE": "1"}) + [ + "--entrypoint", "python", IMAGES["python"], "-B", "scripts/p184_dr05_observer.py", "--query", "/work"] + return roles + + +SERVICE_ORDER = ("quota", "core", "projector", "stream", "query") +TEARDOWN_ORDER = ("sampler", "publisher", "monitor", "query", "stream", "projector", "core", "quota", + "kafka-3", "kafka-2", "kafka-1", "helper", "net") + + +def helper_argv(ns, work, args, *, role="helper", detach=False, evidence=None, network=None, + cpus="0.5", memory="512m", script="scripts/p184_g12_isolated_fault.py", extra=()): + namespace(ns) + argv = _base(ns, role, network=network or "container:" + ns + "-net", memory=memory, cpus=cpus, + uid=f"{os.getuid()}:{os.getgid()}") + if not detach: + argv.remove("-d") + argv += ["--tmpfs", "/tmp:rw,size=64m,mode=1777", "-v", f"{Path(work).resolve()}:/work", + "-v", f"{ROOT}:/app:ro", "-w", "/app", "-e", "PYTHONPATH=/app:/app/generated/python", + "-e", "PYTHONDONTWRITEBYTECODE=1", "-e", "P184_G12_NAMESPACE=" + ns, *extra] + if evidence is not None: + argv += ["-v", f"{Path(evidence).resolve()}:/evidence"] + return argv + ["--entrypoint", "python", IMAGES["python"], "-B", script, *args] + + +SAMPLER = r''' +import asyncio, json, signal, sys, time +from redis.asyncio import Redis +from services.market_data.cache_reader import MarketCacheReader +from services.risk_engine.execution_market_context import _mapping, _missing_reason, _validate_book +from domain.instrument_identity import CanonicalInstrumentId +IDS = {"BINANCE": ("BINANCE", "USD_M", "ETHUSDT", "ETHUSDT"), "OKX": ("OKX", "SWAP", "ETHUSDT", "ETH-USDT-SWAP")} +STOP = [] +signal.signal(signal.SIGTERM, lambda *_: STOP.append(1)) +async def view(r, venue): + now_ms = time.time_ns() // 1_000_000 + c = await MarketCacheReader(r).execution_market_context(CanonicalInstrumentId(*IDS[venue]), + required_feeds=frozenset({"BOOK_SNAPSHOT"})) + book = (c or {}).get("book") or {} + seq = book.get("sequence") or book.get("native_sequence_end") + try: + _validate_book(_mapping((c or {}).get("book"), reason=_missing_reason(c or {}, "book", + "EXECUTION_CONTEXT_BOOK_MISSING")), now_ms=now_ms, max_age_ms=2000, side="BUY", + require_levels=True) + verdict = "APPROVED" + except Exception as e: + verdict = (type(e).__name__ + ":" + str(e))[:160] + try: + seq = None if seq is None else int(seq) + except (TypeError, ValueError): + seq = None + return seq, verdict +async def beat(r): + keys = [k async for k in r.scan_iter(match="service:heartbeat:market_data:*", count=100)] + if not keys: + return {"heartbeat": False} + b = json.loads(await r.get(keys[0]) or "{}"); d = b.get("details") or {} + return {"heartbeat": True, "status": b.get("status"), "demanded": d.get("demanded_v2_slices"), + "session_ready": d.get("session_ready_v2_slices")} +async def main(out, seconds, interval_ms, targets): + views = {v: Redis.from_url(u, socket_timeout=1, socket_connect_timeout=1) for v, u in targets.items()} + beats = {v: Redis.from_url(u.rsplit("/", 1)[0] + "/2", socket_timeout=1, socket_connect_timeout=1, + decode_responses=True) for v, u in targets.items()} + end = time.monotonic() + seconds + last, last_t, last_beat = {}, {}, 0.0 + with open(out, "a", buffering=1) as f: + while time.monotonic() < end and not STOP: + for venue, r in views.items(): + t = time.time_ns() + try: + seq, verdict = await view(r, venue) + row = {"k": "view", "t": t, "venue": venue, "seq": seq, "v": verdict} + except Exception as e: + row = {"k": "view", "t": t, "venue": venue, "seq": None, + "v": "READ_ERROR:" + type(e).__name__} + key = (row["seq"], row["v"]) + if last.get(venue) != key or t - last_t.get(venue, 0) >= 500_000_000: + f.write(json.dumps(row) + "\n"); last[venue] = key; last_t[venue] = t + if time.monotonic() - last_beat >= 2: + last_beat = time.monotonic() + for venue, r in beats.items(): + try: + row = await beat(r) + except Exception as e: + row = {"heartbeat": None, "error": type(e).__name__} + f.write(json.dumps({"k": "hb", "t": time.time_ns(), "venue": venue, **row}) + "\n") + await asyncio.sleep(interval_ms / 1000) + for r in (*views.values(), *beats.values()): + await r.aclose() +asyncio.run(main(sys.argv[1], float(sys.argv[2]), float(sys.argv[3]), + dict(item.split("=", 1) for item in sys.argv[4].split(",")))) +''' + + +def sampler_argv(ns, evidence, seconds, interval_ms=50): + targets = ",".join(f"{v}=redis://localhost:{CLONE_PORTS[v][0]}/1" for v in VENUES) + argv = _base(ns, "sampler", network="container:" + ns + "-net", memory="384m", cpus="0.25", + uid=f"{os.getuid()}:{os.getgid()}") + return argv + ["--tmpfs", "/tmp:rw,size=16m,mode=1777", "-v", f"{Path(evidence).resolve()}:/evidence", + "-e", "PYTHONDONTWRITEBYTECODE=1", "--entrypoint", "python", IMAGES["ts"], "-B", "-c", + SAMPLER, "/evidence/sampler.jsonl", str(int(seconds)), str(interval_ms), targets] + + +# ---------------------------------------------------------------- analysis (pure) + +def load_jsonl(path): + rows = [] + try: + with open(path) as handle: + for line in handle: + line = line.strip() + if line: + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + pass # a torn last line while the writer is live + except FileNotFoundError: + pass + return rows + + +def publish_index(rows): + """(venue, sequence) -> publish ns (the frame's own event time).""" + return {(r["venue"], r["seq"]): r["t"] for r in rows if r.get("k") == "pub"} + + +def view_series(rows): + series = {} + for r in rows: + if r.get("k") == "view": + series.setdefault(r["venue"], []).append((r["t"], r.get("seq"), r.get("v") == "APPROVED")) + for values in series.values(): + values.sort(key=lambda item: item[0]) + return series + + +def caught_up(series, index, venue, cleared_ns, *, age_ms=GATE["age_ms"], hold_s=GATE["hold_s"]): + """First valid apply and stable caught-up for one venue after ``cleared_ns``. + + A row counts when the view is APPROVED, shows a frame published at/after + cleared_ns, and its age (row time - publish time) is <= age_ms; caught-up + needs every row of the following hold_s to count as well, and rows that + reach past the hold (otherwise: not yet decidable -> None). + """ + rows = [r for r in series.get(venue, ()) if r[0] >= cleared_ns] + first_valid = None + age_ns, hold_ns = age_ms * 1_000_000, int(hold_s * 1e9) + + def fresh(row): + published = index.get((venue, row[1])) + return (row[2] and published is not None and published >= cleared_ns + and row[0] - published <= age_ns) + + for t, seq, ok in rows: + published = index.get((venue, seq)) + if ok and published is not None and published >= cleared_ns: + first_valid = t + break + i = 0 + while i < len(rows): + if not fresh(rows[i]): + i += 1 + continue + start, j, broken = rows[i][0], i, None + while j < len(rows) and rows[j][0] <= start + hold_ns: + if not fresh(rows[j]): + broken = j + break + j += 1 + if broken is not None: + i = broken + 1 + continue + if j >= len(rows): + return {"first_valid_apply_ns": first_valid, "caught_up_ns": None} + return {"first_valid_apply_ns": first_valid, "caught_up_ns": start} + return {"first_valid_apply_ns": first_valid, "caught_up_ns": None} + + +def first_seen(series, venue): + seen = {} + for t, seq, ok in series.get(venue, ()): + if seq is not None and ok and seq not in seen: + seen[seq] = t + return seen + + +def transit(series, index, venue, start_ns, end_ns): + """ms from publish to the first APPROVED sample showing each frame, first seen in the window.""" + out = [] + for seq, t in first_seen(series, venue).items(): + published = index.get((venue, seq)) + if published is not None and start_ns <= t < end_ns: + out.append((t - published) / 1e6) + return out + + +def quantile(sorted_values, q): + if not sorted_values: + return None + position = (len(sorted_values) - 1) * q + low, high = math.floor(position), math.ceil(position) + if low == high: + return sorted_values[low] + return sorted_values[low] + (sorted_values[high] - sorted_values[low]) * (position - low) + + +def g1_stats(values): + values = sorted(values) + n = len(values) + stats = {"n": n, "p50_ms": quantile(values, .5), "p95_ms": quantile(values, .95), + "p99_ms": quantile(values, .99) if n >= GATE["p99_min_n"] else None, + "max_ms": values[-1] if values else None} + stats["pass"] = bool(n and stats["p95_ms"] <= GATE["p95_ms"] and stats["max_ms"] < GATE["max_ms"] + and (stats["p99_ms"] is None or stats["p99_ms"] <= GATE["p99_ms"])) + if n and stats["p99_ms"] is None: + stats["note"] = "p99 not reported (n < 100)" + return {k: (round(v, 3) if isinstance(v, float) else v) for k, v in stats.items()} + + +def cleared_at(kind, monitor_rows, restart_ns, node=None): + """Fault-cleared time from the monitor (packet section 5).""" + for row in monitor_rows: + if row.get("t", 0) < restart_ns: + continue + if kind == "broker" and node in ((row.get("cluster") or {}).get("broker_ids") or ()): + return row["t"] + if kind == "projector" and ((row.get("groups") or {}).get("kn-projector-v3-a") or {}).get("members", 0) >= 1: + return row["t"] + if kind == "stream" and (row.get("ports") or {}).get("8210") is True: + return row["t"] + return None + + +def first_monitor(monitor_rows, start_ns, predicate): + for row in monitor_rows: + if row.get("t", 0) >= start_ns and predicate(row): + return row["t"] + return None + + +def evaluate_fault(fault, publish_rows, sampler_rows, monitor_rows, *, window_s=GATE["window_s"]): + index, series = publish_index(publish_rows), view_series(sampler_rows) + cleared = cleared_at(fault["kind"], monitor_rows, fault["restart_ns"], fault.get("node")) + result = {"fault": fault["name"], "kind": fault["kind"], "node": fault.get("node"), + "fault_injected_ns": fault["injected_ns"], "restart_issued_ns": fault["restart_ns"], + "fault_cleared_ns": cleared, "venues": {}, "pass": False} + if cleared is None: + result["reason"] = "FAULT_NOT_CLEARED" + return result + ends = [] + for venue in VENUES: + state = caught_up(series, index, venue, cleared) + state["caught_up_s"] = (None if state["caught_up_ns"] is None + else round((state["caught_up_ns"] - cleared) / 1e9, 3)) + rows = [r for r in series.get(venue, ()) if fault["injected_ns"] <= r[0] <= (state["caught_up_ns"] or r[0])] + ages = [(t - index[(venue, s)]) / 1e6 for t, s, ok in rows if (venue, s) in index] + state["max_view_age_ms_injected_to_caught_up"] = round(max(ages), 3) if ages else None + state["not_approved_samples_injected_to_caught_up"] = sum(1 for _, _, ok in rows if not ok) + result["venues"][venue] = state + ends.append(state["caught_up_ns"]) + if any(end is None for end in ends): + result["reason"] = "NOT_CAUGHT_UP" + return result + caught = max(ends) + result["consumers_caught_up_ns"] = caught + result["caught_up_s"] = round((caught - cleared) / 1e9, 3) + window = (caught, caught + int(window_s * 1e9)) + result["g1_window_ns"] = list(window) + result["g1"] = {venue: g1_stats(transit(series, index, venue, *window)) for venue in VENUES} + last = max((r["t"] for r in sampler_rows if r.get("k") == "view"), default=0) + complete = last >= window[1] + result["g1_window_complete"] = complete + result["projector_lag0_ns"] = first_monitor(monitor_rows, cleared, lambda r: r.get("projector_lag") == 0) + result["isr_restored_ns"] = first_monitor(monitor_rows, cleared, lambda r: r.get("under_replicated") == 0) + result["g12_caught_up_pass"] = result["caught_up_s"] <= GATE["caught_up_s"] + result["pass"] = (result["g12_caught_up_pass"] and complete + and all(result["g1"][v]["pass"] for v in VENUES)) + return result + + +def evaluate(evidence): + evidence = Path(evidence) + publish_rows = load_jsonl(evidence / "publish.jsonl") + sampler_rows = load_jsonl(evidence / "sampler.jsonl") + monitor_rows = sorted(load_jsonl(evidence / "monitor.jsonl"), key=lambda r: r.get("t", 0)) + faults = [r for r in load_jsonl(evidence / "faults.jsonl") if r.get("k") == "fault"] + index, series = publish_index(publish_rows), view_series(sampler_rows) + out = {"schema": "p184.g12.verdict.v1", "provenance": "TEST_ONLY", "level": "ISOLATED_FAULT", + "gate": GATE, "faults": [], "g2": "NOT_MEASURED (BOOK-only isolated source)", + "g3": "NOT_MEASURED (BOOK-only isolated source)"} + base = [r for r in load_jsonl(evidence / "faults.jsonl") if r.get("k") == "baseline"] + if base: + window = (base[0]["start_ns"], base[0]["end_ns"]) + out["baseline_g1"] = {v: g1_stats(transit(series, index, v, *window)) for v in VENUES} + for fault in faults: + out["faults"].append(evaluate_fault(fault, publish_rows, sampler_rows, monitor_rows)) + acks = [r for r in publish_rows if r.get("k") == "ack"] + out["publisher"] = {"frames": sum(1 for r in publish_rows if r.get("k") == "pub"), + "acked": sum(1 for r in acks if "error" not in r), + "delivery_errors": sum(1 for r in acks if "error" in r), + "resyncs": sum(1 for r in publish_rows if r.get("k") == "resync")} + out["pass"] = bool(out["faults"]) and len(out["faults"]) == 3 and all(f["pass"] for f in out["faults"]) + return out + + +# ---------------------------------------------------------------- in-container modes + +def _packet(work): + work = Path(work) + packet = json.loads((work / "packet.json").read_text()) + ns = namespace(packet["namespace"]) + if os.environ.get("P184_G12_NAMESPACE") != ns or packet.get("bootstrap") != BOOTSTRAP: + raise ValueError("G12 namespace attestation and isolated bootstrap required") + return packet + + +def _security(packet): + from scripts.p184_dr08_isolated import kafka_security + return kafka_security(packet) + + +def create_topics(work): + from confluent_kafka.admin import AdminClient, NewTopic + from confluent_kafka import TopicCollection + packet = _packet(work) + admin = AdminClient(_security(packet)) + deadline = time.monotonic() + 120 + while True: + try: + if len(admin.describe_cluster(request_timeout=3).result(timeout=4).nodes) == len(BROKERS): + break + except Exception: # noqa: BLE001 - not ready yet; the deadline decides + pass + if time.monotonic() >= deadline: + raise TimeoutError("three-broker cluster readiness") + time.sleep(1) + wanted = topics(packet["namespace"]) + futures = admin.create_topics([NewTopic(name, parts, len(BROKERS), config=dict(config)) + for name, (parts, config) in wanted.items()], request_timeout=30) + for future in futures.values(): + future.result(40) + canonical = packet["namespace"] + ".canonical" + described = admin.describe_topics(TopicCollection([canonical]))[canonical].result(20) + packet["slice"]["topic_id"] = str(described.topic_id) + (Path(work) / "packet.json").write_text(json.dumps(packet)) + return {"topics": sorted(wanted), "replication_factor": len(BROKERS), "min_isr": 2} + + +class _Stop: + def __init__(self): + self.requested = False + signal.signal(signal.SIGTERM, self._set) + + def _set(self, *_): + self.requested = True + + +def publish(work, out, seconds, hz): + """Steady provider-shaped BOOK frames per venue; resync (new generation + + snapshot) only after a failed delivery, as a venue adapter would.""" + from confluent_kafka import Producer + from scripts.p184_dr05_dl_fault import frame + from scripts.p184_dr05_raw_publisher import raw_envelope + packet = _packet(work) + ns = packet["namespace"] + config = json.loads((Path(work) / "core.json").read_text()) + if config.get("raw_topics") != [ns + ".raw"] or config["core"].get("allow_test_provenance") is not True: + raise ValueError("isolated test-only raw topic required") + bindings = {b["venue"]: b for b in config["core"]["bindings"]} + if set(bindings) != set(VENUES) or not 0 < hz <= 20 or not 0 < seconds <= LIMITS["deadline_s"]: + raise ValueError("two-venue roster and bounded rate/duration required") + producer = Producer({**_security(packet), "enable.idempotence": True, "acks": "all", + "linger.ms": 0, "message.timeout.ms": 30000, "queue.buffering.max.messages": 10000}) + state = {v: {"gen": 7, "seq": 1000, "snapshot": True} for v in VENUES} + stop, end, period = _Stop(), time.monotonic() + seconds, 1.0 / hz + with open(out, "a", buffering=1) as log: + def ack(venue, seq, gen, sent_ns): + def callback(error, message): + row = {"k": "ack", "venue": venue, "seq": seq, "gen": gen, + "ack_ms": round((time.time_ns() - sent_ns) / 1e6, 3)} + if error is not None: + row["error"] = str(error.code()) + if state[venue]["gen"] == gen: + state[venue]["gen"] += 1 + state[venue]["snapshot"] = True + log.write(json.dumps({"k": "resync", "t": time.time_ns(), "venue": venue, + "after_seq": seq, "new_gen": gen + 1}) + "\n") + else: + row["offset"] = message.offset() + log.write(json.dumps(row) + "\n") + return callback + + next_at = time.monotonic() + while time.monotonic() < end and not stop.requested: + for venue in VENUES: + s = state[venue] + snapshot = s["snapshot"] + seq = s["seq"] + 1 + at_ns = time.time_ns() + raw = raw_envelope(bindings[venue], frame(venue, snapshot, seq, 0 if snapshot else s["seq"], + s["gen"], str(seq % 1000 + 1), at_ns), ns) + try: + producer.produce(config["raw_topics"][0], partition=0, key=raw.subscription_id.encode(), + value=raw.SerializeToString(), headers=[("qdl-event-id", raw.capture_id)], + on_delivery=ack(venue, seq, s["gen"], at_ns)) + except BufferError: + log.write(json.dumps({"k": "ack", "venue": venue, "seq": seq, "gen": s["gen"], + "error": "LOCAL_QUEUE_FULL"}) + "\n") + s["gen"] += 1 + s["snapshot"] = True + continue + s["seq"], s["snapshot"] = seq, False + log.write(json.dumps({"k": "pub", "t": at_ns, "venue": venue, "seq": seq, "gen": s["gen"], + "snapshot": snapshot}) + "\n") + next_at += period + while True: + producer.poll(0) + left = next_at - time.monotonic() + if left <= 0: + break + time.sleep(min(left, .02)) + remaining = producer.flush(10) + log.write(json.dumps({"k": "end", "t": time.time_ns(), "unflushed": remaining}) + "\n") + return {"stopped": True} + + +def monitor(work, out, seconds, period=1.0): + """Read-only 1 s observation: cluster/ISR, projector committed vs LSO, + group membership, Query/Stream ports, disk. No group join or commit.""" + import socket + from confluent_kafka.admin import AdminClient + from scripts.kn_recovery_observe import Deadline, KafkaAdminSource + packet = _packet(work) + ns = packet["namespace"] + admin = AdminClient(_security(packet)) + source = KafkaAdminSource(admin) + roster = topics(ns) + rosters = {"kn-projector-v3-a": {ns + ".canonical": roster[ns + ".canonical"][0]}, + "kn-projector-v3-b": {ns + ".latest": roster[ns + ".latest"][0], + ns + ".bars": roster[ns + ".bars"][0]}} + groups = (*PROJECTOR_GROUPS, "kn-" + ns, ns) + stop, end = _Stop(), time.monotonic() + seconds + status_path = Path(work) / "g12-projector-status.json" + with open(out, "a", buffering=1) as log: + while time.monotonic() < end and not stop.requested: + started = time.monotonic() + row = {"t": time.time_ns()} + deadline = Deadline(3.0) + try: + row["cluster"] = source.describe_cluster(deadline) + except Exception as error: # noqa: BLE001 - recorded, never inferred + row["cluster_error"] = type(error).__name__ + try: + described = source.describe_topics(sorted(roster), Deadline(3.0)) + parts = [p for values in described.values() for p in values] + row["under_replicated"] = sum(1 for p in parts if len(p["isr"]) < len(p["replicas"])) + row["offline"] = sum(1 for p in parts if p["leader"] is None) + row["leaders"] = {f"{t}/{p['partition']}": p["leader"] for t, ps in described.items() for p in ps} + row["min_isr_size"] = min(len(p["isr"]) for p in parts) + except Exception as error: # noqa: BLE001 + row["topics_error"] = type(error).__name__ + try: + lag = {} + for group, group_roster in rosters.items(): + committed = source.committed_offsets(group, group_roster, Deadline(3.0)) + ends = source.log_end_offsets(sorted(committed), Deadline(3.0)) + lag[group] = sum(max(0, ends[tp] - max(committed[tp], 0)) for tp in committed) + row["lag"] = lag + row["projector_lag"] = lag["kn-projector-v3-a"] + except Exception as error: # noqa: BLE001 + row["lag_error"] = type(error).__name__ + try: + described = admin.describe_consumer_groups(list(groups), request_timeout=3) + row["groups"] = {} + for group, future in described.items(): + value = future.result(timeout=4) + row["groups"][group] = {"state": str(getattr(value, "state", "")).split(".")[-1], + "members": len(value.members or ())} + except Exception as error: # noqa: BLE001 + row["groups_error"] = type(error).__name__ + row["ports"] = {} + for port in (8200, 8210): + try: + with socket.create_connection(("localhost", port), timeout=.5): + row["ports"][str(port)] = True + except OSError: + row["ports"][str(port)] = False + usage = shutil.disk_usage("/work") + row["disk_free_bytes"] = usage.free + try: + stat = status_path.stat() + row["projector_status_mtime"] = stat.st_mtime + except FileNotFoundError: + row["projector_status_mtime"] = None + log.write(json.dumps(row, default=list) + "\n") + time.sleep(max(0.0, period - (time.monotonic() - started))) + return {"stopped": True} + + +# ---------------------------------------------------------------- host orchestration + +def sh(argv, timeout=60, check=True): + result = subprocess.run(argv, capture_output=True, text=True, timeout=timeout) + if check and result.returncode: + raise RuntimeError(f"{argv[:4]} -> {result.returncode}: {result.stderr.strip()[-400:]}") + return result.stdout + + +def owned_names(ns): + """Exact names of running/stopped containers carrying this namespace's label.""" + namespace(ns) + out = sh(["docker", "ps", "-a", "--filter", f"label={LABEL}={ns}", "--format", "{{.Names}}"]) + names = [n for n in out.split() if n] + if any(not n.startswith(ns + "-") for n in names): + raise RuntimeError("label/name ownership mismatch") + return names + + +def running_names(ns): + out = sh(["docker", "ps", "--filter", f"label={LABEL}={ns}", "--format", "{{.Names}}"]) + return set(out.split()) + + +def stop_role(ns, role, timeout=30): + """Stop one owned --rm container and wait until it is gone (identity checked).""" + name = f"{ns}-{role}" + rows = json.loads(sh(["docker", "inspect", name], check=False) or "[]") + if not rows: + return False + row = rows[0] + if row["Config"]["Labels"].get(LABEL) != ns or row["Name"] != "/" + name: + raise RuntimeError("refuse to stop a container this run does not own: " + name) + if row["HostConfig"].get("AutoRemove") is not True: + raise RuntimeError("refuse non-disposable container: " + name) + sh(["docker", "stop", "--time", str(timeout), row["Id"]], timeout=timeout + 30, check=False) + limit = time.monotonic() + 30 + while sh(["docker", "ps", "-aq", "--no-trunc", "--filter", "id=" + row["Id"]]).strip(): + if time.monotonic() >= limit: + raise RuntimeError("--rm container still present: " + name) + time.sleep(.5) + return True + + +def production_guard(): + """Read-only: name -> [restart count, started at, status] of the running shared stack and TS writer.""" + names = sh(["docker", "ps", "--filter", "name=qdl_v2_stable_candidate-", + "--format", "{{.Names}}"]).split() + [TS_WRITER] + out = sh(["docker", "inspect", "--format", "{{.Name}}|{{.RestartCount}}|{{.State.StartedAt}}|{{.State.Status}}", + *names]) + rows = {} + for line in out.splitlines(): + if line: + name, count, started, status = line.split("|") + rows[name.lstrip("/")] = [int(count), started, status] + return rows + + +def guard_breaches(before, now): + """Stop on a crash signal (restart count up, or a running container gone/not running). + + A changed start time with the restart count unchanged is a recreate by + another operator (the shared stack is not ours); it is recorded, not a stop. + """ + breaches, recreated = {}, {} + for name, (count, started, status) in before.items(): + current = now.get(name) + if current is None or current[2] != "running" or current[0] > count: + breaches[name] = [[count, started, status], current] + elif current[1] != started: + recreated[name] = [started, current[1]] + return breaches, recreated + + +def host_resources(): + free = shutil.disk_usage("/").free + available = None + with open("/proc/meminfo") as handle: + for line in handle: + if line.startswith("MemAvailable:"): + available = int(line.split()[1]) * 1024 + return {"disk_free_bytes": free, "mem_available_bytes": available} + + +def load_writer_clone(): + import importlib.util + path = TS_ROOT / "scripts/p184_dr05_isolated.py" + spec = importlib.util.spec_from_file_location("ts_p184_dr05_isolated", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def labelled_run(ns): + """WriterClone's runner with this namespace's label on every container it starts.""" + def run(*args, check=True, capture=True, timeout=120): + args = list(args) + if args[:3] == ["docker", "run", "-d"]: + args[3:3] = ["--label", f"{LABEL}={ns}"] + result = subprocess.run(args, capture_output=capture, text=True, timeout=timeout) + if check and result.returncode: + raise RuntimeError(f"{args[:3]} -> {result.returncode}: {result.stderr[-300:]}") + return result.stdout.strip() if capture else "" + return run + + +class Run: + def __init__(self, base, export, ns): + self.base = Path(base).resolve() + self.work, self.evidence = self.base / "work", self.base / "evidence" + self.export = Path(export).resolve() + self.ns = namespace(ns) + self.clones = {} + self.expected = set() + self.started = time.monotonic() + self.guard0 = None + self.last_guard = 0.0 + self.last_stats = 0.0 + + # -- evidence + def note(self, step, **fields): + row = {"t": time.time_ns(), "step": step, **fields} + with (self.evidence / "orchestrator.jsonl").open("a") as handle: + handle.write(json.dumps(row, default=str) + "\n") + print(step, json.dumps(fields, default=str)[:400], flush=True) + + def fault_row(self, row): + with (self.evidence / "faults.jsonl").open("a") as handle: + handle.write(json.dumps(row) + "\n") + + # -- checks + def check(self, faulted=()): + resources = host_resources() + if resources["disk_free_bytes"] < LIMITS["disk_free_bytes"]: + raise RuntimeError(f"STOP disk free {resources['disk_free_bytes']} < 20 GB") + if resources["mem_available_bytes"] < LIMITS["mem_available_bytes"]: + raise RuntimeError(f"STOP memory available {resources['mem_available_bytes']} < 3 GiB") + missing = {f"{self.ns}-{r}" for r in self.expected - set(faulted)} - running_names(self.ns) + if missing: + raise RuntimeError("STOP role exited: " + ",".join(sorted(missing))) + now = time.monotonic() + if now - self.last_guard >= 60: + self.last_guard = now + breaches, recreated = guard_breaches(self.guard0, production_guard()) + if recreated and recreated != getattr(self, "recreated", None): + self.recreated = recreated + self.note("production_external_recreate_observed", containers=recreated) + if breaches: + raise RuntimeError("STOP production guard breach: " + json.dumps(breaches)) + if now - self.last_stats >= 30: + self.last_stats = now + names = sorted(running_names(self.ns)) + if names: + out = sh(["docker", "stats", "--no-stream", "--format", "{{json .}}", *names], timeout=60) + with (self.evidence / "stats.jsonl").open("a") as handle: + for line in out.splitlines(): + item = json.loads(line) + handle.write(json.dumps({"t": time.time_ns(), "name": item.get("Name"), + "cpu": item.get("CPUPerc"), "mem": item.get("MemUsage")}) + "\n") + return resources + + def wait(self, seconds, faulted=(), until=None, step=5.0): + limit = time.monotonic() + seconds + while time.monotonic() < limit: + self.check(faulted) + if until is not None: + value = until() + if value: + return value + time.sleep(min(step, max(0.0, limit - time.monotonic()))) + return until() if until is not None else None + + def start(self, role, topic_id): + argv = topology(self.ns, self.work, topic_id)[role] + sh(argv, timeout=60) + self.expected.add(role) + + # -- phases + def prepare(self): + resources = host_resources() + self.note("preflight", **resources) + if resources["disk_free_bytes"] < LIMITS["disk_free_bytes"]: + raise RuntimeError("disk below 20 GB before start") + for image in IMAGES.values(): + if sh(["docker", "image", "inspect", image, "--format", "{{.Id}}"]).strip() != image: + raise RuntimeError("local image mismatch " + image) + if owned_names(self.ns) or sh(["docker", "ps", "-aq", "--filter", "name=" + self.ns]).strip(): + raise RuntimeError("namespace already in use") + self.guard0 = production_guard() + self.note("production_guard_before", containers=len(self.guard0), guard=self.guard0) + self.volumes0 = len(sh(["docker", "volume", "ls", "-q"]).split()) + self.networks0 = len(sh(["docker", "network", "ls", "-q"]).split()) + self.base.mkdir(mode=0o700, parents=True, exist_ok=True) + self.work.mkdir(mode=0o700) + self.evidence.mkdir(mode=0o755) + (self.work / "handoff.json").write_text(json.dumps({ + "namespace": self.ns, "status": "PREPARED_HANDOFF_NOT_RUN", "provenance": "TEST_ONLY", + "level": "ISOLATED_FAULT", "acceptance": False, "material_ready": False}, indent=2) + "\n") + sh(helper_argv(self.ns, self.work, ["--material", "/work", "--core-config", "/export.json"], + network="none", script="scripts/p184_dr05_dl_fault.py", + extra=["-v", f"{self.export}:/export.json:ro"]), timeout=180) + packet = json.loads((self.work / "packet.json").read_text()) + packet["bootstrap"] = BOOTSTRAP + (self.work / "packet.json").write_text(json.dumps(packet)) + password = broker_password((self.work / "broker.properties").read_text()) + for node in BROKERS: + path = self.work / f"broker-{node}.properties" + with path.open("x") as handle: + os.chmod(path, 0o600) + handle.write(broker_properties(password, node)) + (self.work / f"kafka-{node}").mkdir(mode=0o700) + self.note("material_ready", namespace=self.ns, volumes_before=self.volumes0, + networks_before=self.networks0) + + def up(self): + sh(topology(self.ns, self.work)["net"]) + self.expected.add("net") + for node in BROKERS: + self.start(f"kafka-{node}", "") + result = sh(helper_argv(self.ns, self.work, ["--topics", "/work"]), timeout=180) + self.note("topics", result=result.strip()[-400:]) + topic_id = json.loads((self.work / "packet.json").read_text())["slice"]["topic_id"] + self.topic_id = topic_id + for role in SERVICE_ORDER: + self.start(role, topic_id) + limit = time.monotonic() + LIMITS["role_return_s"] + while True: + self.check() + probe = subprocess.run(helper_argv(self.ns, self.work, ["--probe", "/work"], + script="scripts/p184_dr05_dl_fault.py"), + capture_output=True, text=True, timeout=60) + if probe.returncode == 0: + self.note("services_reachable", out=probe.stdout.strip()[-300:]) + break + if time.monotonic() >= limit: + raise RuntimeError("services not reachable: " + probe.stderr.strip()[-400:]) + time.sleep(2) + remaining = LIMITS["deadline_s"] - (time.monotonic() - self.started) - 60 + sh(helper_argv(self.ns, self.work, ["--monitor", "/work", "--out", "/evidence/monitor.jsonl", + "--seconds", str(int(remaining))], role="monitor", detach=True, + evidence=self.evidence, cpus="0.25", memory="256m")) + self.expected.add("monitor") + sh(helper_argv(self.ns, self.work, ["--publish", "/work", "--out", "/evidence/publish.jsonl", + "--seconds", str(int(remaining)), "--hz", "4"], role="publisher", + detach=True, evidence=self.evidence, cpus="0.25", memory="256m")) + self.expected.add("publisher") + self.note("publisher_monitor_started", seconds=int(remaining)) + iso = load_writer_clone() + import tempfile + logs = self.work / "tslogs" + logs.mkdir(mode=0o700) + tempfile.tempdir = str(logs) # WriterClone's stderr dir stays under the run, not /tmp + run = labelled_run(self.ns) + for venue in VENUES: + redis_port, pg_port = CLONE_PORTS[venue] + spec = json.loads((self.work / f"ts-consumer.{venue.lower()}.json").read_text()) + clone = iso.WriterClone(IMAGES["ts"], redis_image=IMAGES["redis"], postgres_image=IMAGES["postgres"], + venues=(venue,), network="container:" + self.ns + "-net", + redis_port=redis_port, pg_port=pg_port, consumer=spec, + dl_work=str(self.work), run=run, tag=venue.lower()) + label = venue.lower() + clone.name = f"{self.ns}-tswriter-{label}" + clone.redis_name = f"{self.ns}-tsredis-{label}" + clone.pg_name = f"{self.ns}-tspg-{label}" + clone.deadline = clone.clock() + LIMITS["clone_ready_s"] + self.clones[venue] = clone + clone.__enter__() + self.expected |= {f"tswriter-{label}", f"tsredis-{label}", f"tspg-{label}"} + self.note("ts_clones_started", names=[c.name for c in self.clones.values()]) + sh(sampler_argv(self.ns, self.evidence, remaining - 30)) + self.expected.add("sampler") + started = time.time_ns() + + def ready(): + index = publish_index(load_jsonl(self.evidence / "publish.jsonl")) + series = view_series(load_jsonl(self.evidence / "sampler.jsonl")) + return all(caught_up(series, index, v, started, hold_s=10)["caught_up_ns"] for v in VENUES) + + if not self.wait(LIMITS["clone_ready_s"], until=ready): + raise RuntimeError("STOP TS clones never showed a fresh APPROVED book view") + self.note("ts_views_fresh") + + def baseline(self): + start = time.time_ns() + self.wait(LIMITS["baseline_s"]) + self.fault_row({"k": "baseline", "start_ns": start, "end_ns": time.time_ns()}) + self.note("baseline_done") + + def canonical_leader(self): + rows = load_jsonl(self.evidence / "monitor.jsonl") + for row in reversed(rows): + leader = (row.get("leaders") or {}).get(f"{self.ns}.canonical/0") + if leader in BROKERS: + return leader + raise RuntimeError("canonical leader unknown") + + def fault(self, name, kind): + self.check() + node = self.canonical_leader() if kind == "broker" else None + role = f"kafka-{node}" if kind == "broker" else kind + injected = time.time_ns() + stop_role(self.ns, role) + self.note("fault_injected", fault=name, role=role) + if kind == "broker": + self.wait(LIMITS["broker_outage_s"], faulted=(role,)) + restart = time.time_ns() + sh(topology(self.ns, self.work, self.topic_id)[role], timeout=60) + self.note("restart_issued", fault=name, role=role) + row = {"k": "fault", "name": name, "kind": kind, "node": node, "role": role, + "injected_ns": injected, "restart_ns": restart} + self.fault_row(row) + monitor_rows = lambda: sorted(load_jsonl(self.evidence / "monitor.jsonl"), key=lambda r: r.get("t", 0)) + cleared = self.wait(LIMITS["role_return_s"], + until=lambda: cleared_at(kind, monitor_rows(), restart, node)) + if not cleared: + raise RuntimeError(f"STOP {name}: fault not cleared within {LIMITS['role_return_s']} s") + self.note("fault_cleared", fault=name, cleared_ns=cleared) + + def caught(): + result = evaluate_fault(row, load_jsonl(self.evidence / "publish.jsonl"), + load_jsonl(self.evidence / "sampler.jsonl"), monitor_rows(), window_s=0) + return result if result.get("consumers_caught_up_ns") else None + + result = self.wait(GATE["caught_up_s"] + GATE["hold_s"] + 10, until=caught) + if not result or result["caught_up_s"] > GATE["caught_up_s"]: + raise RuntimeError(f"STOP {name}: consumers not caught up within {GATE['caught_up_s']} s") + self.note("caught_up", fault=name, caught_up_s=result["caught_up_s"]) + end = result["consumers_caught_up_ns"] + int(GATE["window_s"] * 1e9) + self.wait(max(0.0, (end - time.time_ns()) / 1e9) + 2) + verdict = evaluate_fault(row, load_jsonl(self.evidence / "publish.jsonl"), + load_jsonl(self.evidence / "sampler.jsonl"), monitor_rows()) + self.note("fault_verdict", fault=name, verdict=verdict) + if not verdict["pass"]: + raise RuntimeError(f"STOP {name}: G12/G1 verdict FAIL") + + def down(self): + removed, errors = [], [] + for clone in self.clones.values(): + try: + for item in clone.close(): + removed.append(item) + except Exception as error: # noqa: BLE001 - cleanup continues + errors.append(type(error).__name__) + report = teardown(self.ns) + report["writer_clone_removed"] = removed + report["errors"] = errors + report.get("errors", []) + report["volumes_before"] = getattr(self, "volumes0", None) + report["networks_before"] = getattr(self, "networks0", None) + report["volumes_after"] = len(sh(["docker", "volume", "ls", "-q"]).split()) + report["networks_after"] = len(sh(["docker", "network", "ls", "-q"]).split()) + erase_private(self.work) + if self.guard0 is not None: + breaches, recreated = guard_breaches(self.guard0, production_guard()) + report["production_guard_breaches"] = breaches + report["production_external_recreates"] = recreated + with (self.evidence / "teardown.json").open("w") as handle: + json.dump(report, handle, indent=2) + self.note("teardown", **report) + return report + + +def teardown(ns): + """Exact-name, label-checked stop of every owned container, then verify zero.""" + removed, errors = [], [] + for name in owned_names(ns): + role = name[len(ns) + 1:] + if role not in TEARDOWN_ORDER and not re.fullmatch(r"ts(writer|redis|pg)-(binance|okx)", role): + errors.append("unknown role " + role) + order = list(TEARDOWN_ORDER[:3]) + [f"ts{k}-{v}" for v in ("binance", "okx") for k in ("writer", "redis", "pg")] \ + + list(TEARDOWN_ORDER[3:]) + for role in order: + try: + if stop_role(ns, role, timeout=10): + removed.append(role) + except Exception as error: # noqa: BLE001 - continue, then report + errors.append(f"{role}:{type(error).__name__}:{error}") + left = owned_names(ns) + by_name = sh(["docker", "ps", "-aq", "--filter", "name=" + ns]).split() + networks = sh(["docker", "network", "ls", "-q", "--filter", "name=" + ns]).split() + volumes = sh(["docker", "volume", "ls", "-q", "--filter", "label=" + LABEL]).split() + return {"removed_containers": removed, "removed_count": len(removed), "left_by_label": len(left), + "left_by_name": len(by_name), "networks_by_name": len(networks), "volumes_by_label": len(volumes), + "errors": errors} + + +def erase_private(work): + work = Path(work) + for name in PRIVATE_FILES: + (work / name).unlink(missing_ok=True) + for node in BROKERS: + shutil.rmtree(work / f"kafka-{node}", ignore_errors=True) + shutil.rmtree(work / "tslogs", ignore_errors=True) + + +def orchestrate(base, export, ns): + run = Run(base, export, ns) + + def expire(_signum, _frame): + raise TimeoutError("G12 orchestrator absolute deadline") + previous = signal.signal(signal.SIGALRM, expire) + signal.alarm(LIMITS["deadline_s"]) + status, error = "FAIL", None + try: + run.prepare() + run.up() + run.baseline() + for name, kind in (("broker_stop_start", "broker"), ("projector_restart", "projector"), + ("stream_restart", "stream")): + run.fault(name, kind) + status = "COMPLETED" + except BaseException as exc: # noqa: BLE001 - evidence first, then teardown + error = f"{type(exc).__name__}: {exc}" + if run.evidence.exists(): + run.note("stopped", error=error) + finally: + signal.alarm(0) + try: + if run.evidence.exists(): + verdict = evaluate(run.evidence) + verdict.update(status=status, error=error, namespace=ns) + (run.evidence / "verdict.json").write_text(json.dumps(verdict, indent=2) + "\n") + finally: + run.down() if run.evidence.exists() else teardown(ns) + signal.signal(signal.SIGALRM, previous) + return 0 if status == "COMPLETED" else 1 + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + mode = parser.add_mutually_exclusive_group(required=True) + for flag in ("run", "down", "analyze", "topics", "publish", "monitor"): + mode.add_argument("--" + flag, type=Path) + parser.add_argument("--export", type=Path) + parser.add_argument("--namespace") + parser.add_argument("--out") + parser.add_argument("--seconds", type=float) + parser.add_argument("--hz", type=float, default=4.0) + args = parser.parse_args(argv) + if args.run: + if args.export is None or args.namespace is None: + parser.error("--run needs --export and --namespace") + return orchestrate(args.run, args.export, args.namespace) + if args.down: + ns = namespace(json.loads((args.down / "work" / "handoff.json").read_text())["namespace"]) + report = teardown(ns) + erase_private(args.down / "work") + print(json.dumps(report)) + return 0 if not report["errors"] and report["left_by_label"] == 0 else 1 + if args.analyze: + print(json.dumps(evaluate(args.analyze / "evidence" if (args.analyze / "evidence").exists() + else args.analyze), indent=2)) + return 0 + if args.seconds is None and not args.topics: + parser.error("--seconds required") + if args.topics: + print(json.dumps(create_topics(args.topics))) + elif args.publish: + print(json.dumps(publish(args.publish, args.out, args.seconds, args.hz))) + else: + print(json.dumps(monitor(args.monitor, args.out, args.seconds))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_p184_dr05_dl_fault.py b/tests/test_p184_dr05_dl_fault.py index 1aa75177..ada97d6a 100644 --- a/tests/test_p184_dr05_dl_fault.py +++ b/tests/test_p184_dr05_dl_fault.py @@ -21,9 +21,12 @@ def test_exact_nonqualifying_cells(self): and r["level"] == "ISOLATED_FAULT" for r in rows)) def test_namespace_rejects_shared_and_injection(self): - for value in ("production", NS + ";sh", "", "p184-dr08-0123456789abcdef"): + for value in ("production", NS + ";sh", "", "p184-dr08-0123456789abcdef", + "p184-g12-dl-0123456789abcde", "p184-g12-dl-0123456789abcdef;sh", + "p184-g13-dl-0123456789abcdef"): with self.subTest(value=value), self.assertRaises(ValueError): dr.namespace(value) + self.assertEqual(dr.namespace("p184-g12-dl-0123456789abcdef"), "p184-g12-dl-0123456789abcdef") def test_topology_is_bounded_and_disconnected(self): commands = dr.topology(NS, "/tmp/dr05-test") diff --git a/tests/test_p184_g12_isolated_fault.py b/tests/test_p184_g12_isolated_fault.py new file mode 100644 index 00000000..f069560e --- /dev/null +++ b/tests/test_p184_g12_isolated_fault.py @@ -0,0 +1,210 @@ +"""Offline contracts of the G12 isolated fault harness; no Docker daemon or broker.""" +import unittest +from unittest.mock import patch + +from scripts import p184_g12_isolated_fault as g12 + +NS = "p184-g12-dl-0123456789abcdef" +S = 1_000_000_000 # one second in ns +MS = 1_000_000 + + +def stream_rows(venue, start, end, *, step_ms=250, transit_ms=100, approved=True, first_seq=1001): + """Publisher rows every step_ms and sampler rows showing each frame transit_ms later.""" + pubs, views = [], [] + t, seq = start, first_seq + while t < end: + pubs.append({"k": "pub", "t": t, "venue": venue, "seq": seq, "gen": 7, "snapshot": False}) + views.append({"k": "view", "t": t + transit_ms * MS, "venue": venue, "seq": seq, + "v": "APPROVED" if approved else "ExecutionMarketContextError:STALE"}) + t += step_ms * MS + seq += 1 + return pubs, views + + +class NamespaceAndTopologyTests(unittest.TestCase): + def test_namespace_is_exact(self): + self.assertEqual(g12.namespace(NS), NS) + for value in ("p184-dr05-dl-0123456789abcdef", NS + ";sh", "", None, "qdl_v2_stable_candidate"): + with self.subTest(value=value), self.assertRaises(ValueError): + g12.namespace(value) + + def test_topology_is_disposable_isolated_and_bounded(self): + roles = g12.topology(NS, "/home/bobby/.local/state/qdl-v2/g12-test/work", "TOPIC", uid="1001:1001") + self.assertEqual(set(roles), {"net", "kafka-1", "kafka-2", "kafka-3", *g12.SERVICE_ORDER}) + for role, argv in roles.items(): + with self.subTest(role=role): + for flag in ("--rm", "--pull=never", "--memory", "--cpus", "--pids-limit", "--log-opt", "--read-only"): + self.assertIn(flag, argv) + self.assertEqual(argv[argv.index("--name") + 1], f"{NS}-{role}") + self.assertIn(f"{g12.LABEL}={NS}", argv) + network = argv[argv.index("--network") + 1] + self.assertEqual(network, "none" if role == "net" else f"container:{NS}-net") + self.assertFalse({"-p", "--publish", "--privileged"} & set(argv)) + self.assertFalse(any("qdl_v2_stable_candidate" in a or "/var/run/docker.sock" in a for a in argv)) + self.assertTrue(any(a.startswith("sha256:") for a in argv)) + # Brokers keep their log on a per-node bind dir so a restart is a real broker restart. + self.assertIn("/home/bobby/.local/state/qdl-v2/g12-test/work/kafka-2:/data", roles["kafka-2"]) + self.assertIn("--ignore-formatted", roles["kafka-3"][-1]) + self.assertIn(f"QDL_KN_KAFKA_BOOTSTRAP={g12.BOOTSTRAP}", roles["stream"]) + self.assertIn("QDL_KN_TOPIC_ID=TOPIC", roles["projector"]) + + def test_topology_refuses_work_inside_repository(self): + with self.assertRaises(ValueError): + g12.topology(NS, g12.ROOT / "work") + + def test_topics_are_rf3_min_isr2(self): + wanted = g12.topics(NS) + self.assertEqual(set(wanted), {NS + s for s in (".raw", ".canonical", ".quarantine", ".latest", ".bars")}) + self.assertTrue(all(cfg["min.insync.replicas"] == "2" for _, cfg in wanted.values())) + self.assertEqual(wanted[NS + ".latest"][1]["cleanup.policy"], "compact") + + def test_broker_properties(self): + password = g12.broker_password("a=b\nssl.keystore.password=s3cr3t\nssl.key.password=s3cr3t\n") + self.assertEqual(password, "s3cr3t") + with self.assertRaises(ValueError): + g12.broker_password("ssl.key.password=x\n") + text = g12.broker_properties(password, 2) + self.assertIn("node.id=2\n", text) + self.assertIn("listeners=SSL://:9192,CONTROLLER://:9193\n", text) + self.assertIn("advertised.listeners=SSL://localhost:9192\n", text) + self.assertIn("controller.quorum.voters=1@localhost:9093,2@localhost:9193,3@localhost:9293\n", text) + for line in ("transaction.state.log.min.isr=2", "offsets.topic.replication.factor=3", + "min.insync.replicas=2", "unclean.leader.election.enable=false"): + self.assertIn(line + "\n", text) + with self.assertRaises(ValueError): + g12.broker_properties(password, 4) + + def test_helpers_and_sampler_stay_in_namespace(self): + argv = g12.helper_argv(NS, "/home/bobby/x/work", ["--topics", "/work"]) + self.assertNotIn("-d", argv) + self.assertIn(f"container:{NS}-net", argv) + self.assertIn("P184_G12_NAMESPACE=" + NS, argv) + sampler = g12.sampler_argv(NS, "/home/bobby/x/evidence", 100) + self.assertIn("-d", sampler) + self.assertEqual(sampler[sampler.index("--name") + 1], NS + "-sampler") + self.assertIn(g12.IMAGES["ts"], sampler) + self.assertIn("BINANCE=redis://localhost:6391/1,OKX=redis://localhost:6392/1", sampler) + + def test_labelled_run_labels_every_detached_container(self): + calls = [] + + def fake(args, **_): + calls.append(args) + return type("R", (), {"returncode": 0, "stdout": "ok\n", "stderr": ""})() + with patch.object(g12.subprocess, "run", side_effect=fake): + run = g12.labelled_run(NS) + run("docker", "run", "-d", "--rm", "--name", "x", "img") + run("docker", "inspect", "x") + self.assertEqual(calls[0][:5], ["docker", "run", "-d", "--label", f"{g12.LABEL}={NS}"]) + self.assertEqual(calls[1], ["docker", "inspect", "x"]) + + +class AnalysisTests(unittest.TestCase): + def test_caught_up_requires_post_clear_frame_age_and_hold(self): + cleared = 100 * S + pubs, views = stream_rows("BINANCE", 90 * S, 200 * S) + index, series = g12.publish_index(pubs), g12.view_series(views) + state = g12.caught_up(series, index, "BINANCE", cleared) + self.assertEqual(state["first_valid_apply_ns"], cleared + 100 * MS) + self.assertEqual(state["caught_up_ns"], cleared + 100 * MS) + + def test_stale_tail_delays_caught_up(self): + cleared = 100 * S + pubs, _ = stream_rows("OKX", 100 * S, 200 * S) + # The consumer shows frames 1.5 s late until t=130 s, then 100 ms late. + applied = sorted(({"k": "view", "t": p["t"] + (1500 if p["t"] < 130 * S else 100) * MS, "venue": "OKX", + "seq": p["seq"], "v": "APPROVED"} for p in pubs), key=lambda r: r["t"]) + views, newest = [], 0 # the view shows the newest applied frame; an older late apply stays hidden + for row in applied: + if row["seq"] > newest: + views.append(row) + newest = row["seq"] + state = g12.caught_up(g12.view_series(views), g12.publish_index(pubs), "OKX", cleared) + self.assertEqual(state["first_valid_apply_ns"], cleared + 1500 * MS) + self.assertEqual(state["caught_up_ns"], 130 * S + 100 * MS) + + def test_not_approved_or_undecidable_is_not_caught_up(self): + pubs, views = stream_rows("OKX", 100 * S, 200 * S, approved=False) + state = g12.caught_up(g12.view_series(views), g12.publish_index(pubs), "OKX", 100 * S) + self.assertIsNone(state["caught_up_ns"]) + self.assertIsNone(state["first_valid_apply_ns"]) + pubs, views = stream_rows("OKX", 100 * S, 110 * S) # shorter than the 30 s hold + self.assertIsNone(g12.caught_up(g12.view_series(views), g12.publish_index(pubs), "OKX", 100 * S) + ["caught_up_ns"]) + + def test_frames_published_before_clear_do_not_count(self): + pubs, views = stream_rows("OKX", 0, 200 * S) + # The view is frozen on a pre-clear frame until 150 s. + frozen = [v for v in views if v["t"] < 50 * S][-1] + views = [dict(frozen, t=t) for t in range(50 * S, 150 * S, S // 2)] + [v for v in views if v["t"] >= 150 * S] + state = g12.caught_up(g12.view_series(views), g12.publish_index(pubs), "OKX", 100 * S) + self.assertEqual(state["caught_up_ns"], 150 * S + 100 * MS) + + def test_transit_and_g1_stats(self): + pubs, views = stream_rows("BINANCE", 0, 100 * S, transit_ms=120) + index, series = g12.publish_index(pubs), g12.view_series(views) + values = g12.transit(series, index, "BINANCE", 0, 100 * S) + self.assertEqual(len(values), 400) + stats = g12.g1_stats(values) + self.assertEqual((stats["p95_ms"], stats["max_ms"]), (120.0, 120.0)) + self.assertTrue(stats["pass"]) + self.assertFalse(g12.g1_stats([100.0] * 99 + [2500.0])["pass"]) + small = g12.g1_stats([100.0] * 10) + self.assertIsNone(small["p99_ms"]) + self.assertFalse(g12.g1_stats([])["pass"]) + + def test_cleared_from_monitor(self): + rows = [{"t": 5, "cluster": {"broker_ids": [1, 3]}, "groups": {"kn-projector-v3-a": {"members": 1}}, + "ports": {"8210": True}}, + {"t": 10, "cluster": {"broker_ids": [1, 3]}, "groups": {"kn-projector-v3-a": {"members": 0}}, + "ports": {"8210": False}}, + {"t": 20, "cluster": {"broker_ids": [1, 2, 3]}, "groups": {"kn-projector-v3-a": {"members": 1}}, + "ports": {"8210": True}}] + self.assertEqual(g12.cleared_at("broker", rows, 8, 2), 20) + self.assertEqual(g12.cleared_at("projector", rows, 8), 20) + self.assertEqual(g12.cleared_at("stream", rows, 8), 20) + self.assertIsNone(g12.cleared_at("broker", rows, 25, 2)) + + def test_evaluate_fault_end_to_end(self): + pubs, views = [], [] + for venue in g12.VENUES: + p, v = stream_rows(venue, 0, 500 * S) + pubs += p + views += v + monitor = [{"t": t * S, "ports": {"8210": t >= 62}, "projector_lag": 0, "under_replicated": 0} + for t in range(0, 500)] + fault = {"k": "fault", "name": "stream_restart", "kind": "stream", "injected_ns": 60 * S, + "restart_ns": 61 * S} + result = g12.evaluate_fault(fault, pubs, views, monitor) + self.assertEqual(result["fault_cleared_ns"], 62 * S) + self.assertTrue(result["g12_caught_up_pass"]) + self.assertLess(result["caught_up_s"], 1) + self.assertTrue(result["g1_window_complete"]) + self.assertTrue(result["pass"]) + self.assertEqual(result["g1"]["OKX"]["n"], 1200) + short = g12.evaluate_fault(fault, pubs, [v for v in views if v["t"] < 200 * S], monitor) + self.assertFalse(short["g1_window_complete"]) + self.assertFalse(short["pass"]) + never = g12.evaluate_fault(dict(fault, restart_ns=600 * S), pubs, views, monitor) + self.assertEqual(never["reason"], "FAULT_NOT_CLEARED") + + +class GuardTests(unittest.TestCase): + def test_guard_breach_and_external_recreate(self): + before = {"a": [0, "t1", "running"], "b": [0, "t1", "running"], "c": [1, "t1", "running"]} + now = {"a": [0, "t1", "running"], "b": [0, "t2", "running"], "c": [2, "t3", "running"]} + breaches, recreated = g12.guard_breaches(before, now) + self.assertEqual(set(breaches), {"c"}) + self.assertEqual(recreated, {"b": ["t1", "t2"]}) + breaches, _ = g12.guard_breaches(before, {"a": [0, "t1", "exited"], "c": [1, "t1", "running"]}) + self.assertEqual(set(breaches), {"a", "b"}) + + def test_teardown_refuses_foreign_names(self): + with patch.object(g12, "sh", return_value="qdl_v2_stable_candidate-kafka1-1\n"): + with self.assertRaises(RuntimeError): + g12.owned_names(NS) + + +if __name__ == "__main__": + unittest.main() diff --git a/upgrade/evidence/p184-g12-isolated-fault-packet.md b/upgrade/evidence/p184-g12-isolated-fault-packet.md new file mode 100644 index 00000000..763fd41c --- /dev/null +++ b/upgrade/evidence/p184-g12-isolated-fault-packet.md @@ -0,0 +1,130 @@ +# P18.4 G12 / D4-R3 isolated fault acceptance - packet + +Status: PACKET (written before the run). Owner approval 2026-10-10: "Duyet chay co lap" (isolated run only). +Gate: TS `upgrade/p184-phase-d-gates.md` G12 - Kafka broker / projector / Stream restart: fault cleared -> +consumers caught up to live <= 600 s, then G1-G3 inside gate; the same four timestamps as G11. +Base: `fix/p184-dl-closure` tip `5a8b063`, branch `p184-g12`. +Harness: `scripts/p184_g12_isolated_fault.py` (new; reuses DR05 material/frames/publisher envelope, DR08 Query +read view, the D4-R2 `kn_recovery_observe.KafkaAdminSource`, and the TS-owned `WriterClone` read-only from the TS +worktree). Level: `ISOLATED_FAULT`, `TEST_ONLY` provenance. Same-host rehearsal - not HA, not independent failure +domains, not production evidence. + +## 1. Namespace and isolation + +- Namespace: `p184-g12-dl-cfcba18d978b452f`. Every container is named `p184-g12-dl-cfcba18d978b452f-` and + labelled `p184.g12=p184-g12-dl-cfcba18d978b452f`. +- Network: one holder container (`-net`, `alpine:3.20`, `--network none`, `sleep` with a deadline) owns a + loopback-only network namespace; every other role joins it with `--network container:-net`. No Docker + network is created, no port is published, nothing in the namespace can reach any other container or the + internet. Brokers listen on distinct loopback ports (9092/9093, 9192/9193, 9292/9293). +- Not touched: `qdl_v2_stable_candidate-*`, `market_data_service_p184w` (read-only `docker inspect` of its + config only, as DR05's WriterClone does; its DL identity env is dropped and no shared secret is mounted), + shared Redis/Kafka/Postgres, `qdl-phase3-target-*`, any container not created here. No `redis-cli MONITOR`. + No secret or key file is printed; TLS/JWT/cursor material is freshly generated for this namespace only. +- Production guard (read-only `docker inspect`, every 60 s and at teardown): `RestartCount`/`StartedAt`/status of + every running `qdl_v2_stable_candidate-*` container and of `market_data_service_p184w`. A restart-count increase + or a container no longer running stops the run; a new `StartedAt` with an unchanged restart count is a recreate + by another operator (the shared stack is not ours) and is recorded, not a stop. + +## 2. Images (all already local, by ID; no build - no source change in any image) + +| Role | Image ID | Tag (informational) | +|---|---|---| +| kafka-1/2/3 | `sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9` | apache/kafka (= production brokers) | +| core | `sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023` | qdl-v2-rust:2.2.1-0e00c81 (= rust_core) | +| projector | `sha256:a696cfba6ab383173607f96413c814f74c7abcc98a4c41f57c9c6e78a41bb70b` | qdl-v2-rust:2.2.1-c8015ca-projector (= market_projector_1/2) | +| stream | `sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9` | qdl-v2-rust:2.2.1-c8015ca-stream (= stream_kn_1/2) | +| query, helper, publisher, monitor | `sha256:672503b7fe87265d12af3d45fc4d344fdef0a76e94e06efe52e8dc59cab24d0a` | qdl-v2-python:2.2.3-rc3-76950c8 (= query_kn_1/2); worktree mounted read-only at /app (`git log 76950c8..HEAD -- qdl/` is empty, so library code equals the image) | +| quota + market cache Redis | `sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8` | redis (= market_cache) | +| TS writer clone x2, sampler | `sha256:15524c0bea8c51949d7ced79291d4c72722292fea2f0a5d4f8e9c7fd61dbb5ca` | local/trading-system:p184-writer-9b86ed3 (= market_data_service_p184w) | +| writer clone Redis x2 | `sha256:dfa18828...` (as above) | | +| writer clone Postgres x2 | `sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685` | postgres | +| holder | `sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc` | alpine:3.20 | + +Unit tests run in `sha256:77682eea...` (qdl-v2-python:2.2.3-rc2-43bbcf0), `--network none`, read-only source. + +## 3. Roles started (all `--rm`, `--pull=never`, `--read-only` where the image allows, tmpfs, bounded logs) + +| Role | CPU | Memory | Notes | +|---|---|---|---| +| net (holder) | 0.05 | 16 MiB | `sleep 5400` (absolute lifetime) | +| kafka-1, kafka-2, kafka-3 | 0.75 each | 1024 MiB each (heap 256-512 MiB) | KRaft combined, RF3, min ISR 2 (data, offsets, transactions), mTLS SSL listener, data on a bind dir under the work dir so a restarted broker keeps its log | +| quota (Redis) | 0.1 | 192 MiB (maxmemory 128 MB, noeviction) | db0 quota, db1 market cache written by the projector and read by Query | +| core | 0.5 | 256 MiB | ETH BOOK bindings for BINANCE USDM and OKX SWAP (DR05 public export) | +| projector (replica a, one replica) | 0.5 | 256 MiB | stage A canonical -> latest/bars; stage B -> market cache | +| stream (one replica) | 0.5 | 512 MiB | production gateway; hot read on; read view from Query | +| query | 0.5 | 512 MiB | real KN Query read view (DR08 `serve_query`), JWT + quota | +| publisher | 0.25 | 256 MiB | test-provenance provider-shaped BOOK frames, 4 Hz per venue, idempotent acks=all | +| monitor | 0.25 | 256 MiB | 1 s: cluster/ISR, projector committed vs LSO, group members, Query/Stream port open, disk | +| sampler | 0.25 | 384 MiB | 50 ms: each TS clone's BOOK view through `MarketCacheReader` + at-use `_validate_book` | +| tswriter-binance, tswriter-okx | 2 each | 2 GiB each | TS `WriterClone` (production entrypoint and env, 4 fixed workers, ETH only, own Redis/Postgres) | +| tsredis-*, tspg-* (x2 each) | uncapped CPU | 256 MiB each | WriterClone disposables, tmpfs | +| helper (one-shot) | 0.5 | 512 MiB | material, topics, readiness probe | + +Totals (caps, not reservations): ~9.2 vCPU and ~10.9 GiB. Expected draw at 8 frames/s is far below; the +measured per-container CPU/RSS is recorded every 30 s (`docker stats --no-stream`) and reported. One projector +and one Stream replica (production runs two of each): a restart is a full outage of that role - the worst case. + +## 4. Faults (sequential, one stack lifetime) + +0. Baseline: after both TS clones show an APPROVED book view, 300 s G1 window. +1. Kafka broker: stop the broker that leads the canonical partition (`docker stop -t 30`), keep it down 60 s, + then start it again with the same argv and the same data dir (KRaft `format --ignore-formatted`). +2. Projector: `docker stop -t 30` the projector, start it again immediately (same argv, same epoch). +3. Stream: `docker stop -t 30` the Stream gateway, start it again immediately (same argv). + +## 5. What is measured (per fault, per venue; the G11 four timestamps) + +- `fault_injected_at`: the stop command is issued (outage start). +- `fault_cleared_at`: broker - the restarted broker id is back in `describe_cluster` (ISR fully restored is + recorded separately); projector - the projector's stage-A group has a member again; Stream - TCP 8210 accepts + again. All from the monitor; the container start time is also recorded. +- `first_valid_apply_at`: first sampler row at/after `fault_cleared_at` whose view is `APPROVED` and shows a + frame published at/after `fault_cleared_at`. +- `caught_up_at`: first sampler row at/after `fault_cleared_at` showing a frame published at/after + `fault_cleared_at` with view age (sample time - frame publish time) <= 1000 ms, APPROVED, and staying APPROVED + with age <= 1000 ms for the next 30 s. Consumers caught up = the later of the two venues. +- Then a 300 s G1 window from `caught_up_at`: transit per newly observed frame (first sample showing it - + its publish time): n, p50, p95, p99 (n >= 100), max. Also: projector lag (committed vs LSO) and when it + reached 0, under-replicated partitions and when they reached 0, publisher delivery errors, TS heartbeat + status / demanded / session-ready. + +## 6. Pass rule + +- G12 per fault: `caught_up_at - fault_cleared_at <= 600 s` for both venues, then G1 inside gate in the 300 s + window: p95 <= 500 ms, p99 <= 1000 ms, max < 2000 ms (BOOK_DELTA/BOOK_SNAPSHOT at the TS cache). +- G2 (TRADE/QUOTE/MARK) and G3 (BAR) are NOT measured: the isolated synthetic source is BOOK-only (DR05 + material). Reported as NOT_MEASURED, not as PASS. +- Measurement resolution: the sampler polls every 50 ms; a frame skipped between two polls has no transit + sample (it is counted as skipped, not imputed). + +## 7. Stop conditions (stop at the first; evidence is kept) + +- caught-up not reached within 600 s after a fault cleared (G12 FAIL); +- any role other than the faulted one exits, or the faulted role does not come back within 120 s; +- a TS clone exits; the sampler/monitor/publisher exits early; +- free disk on `/` < 20 GB (checked before start and every 10 s by the orchestrator; the monitor also logs it); +- host available memory < 3 GiB; +- any production guard breach (section 1); +- the orchestrator's absolute deadline (3600 s). + +## 8. Deadlines + +Orchestrator: SIGALRM at 3600 s, teardown in `finally`. Holder: `sleep 5400`. Publisher, monitor and sampler: +`--seconds` (run length) and exit cleanly at it or on SIGTERM. Every `docker` call: subprocess timeout +(15-180 s). Readiness waits: brokers 120 s, services 120 s, TS clones 300 s. + +## 9. Teardown and verification + +Remove (exact names, label checked before every stop): sampler, monitor, publisher, the two TS writer clones and +their Redis/Postgres (WriterClone.close), query, stream, projector, core, quota, kafka-3/2/1, net. Then: +`docker ps -a --filter label=p184.g12=` = 0 and `docker ps -a --filter name=` = 0; +`docker network ls --filter name=` = 0 (none created); `docker volume ls` count before = after (no named +volume; image VOLUMEs are overridden by tmpfs or removed with `--rm`). Private material (keys, keystore, +passwords, cursor keys) and the broker data dirs are deleted from the work dir; JSONL evidence, the verdict and +public config remain under `~/.local/state/qdl-v2/p184-g12-20261010/`. No build cache is created. + +## 10. Rollback + +Nothing shared changes, so there is nothing to roll back; a failed/aborted run is cleaned by +`python3 -B scripts/p184_g12_isolated_fault.py --down ` (exact-name, label-checked). From 2a5e80936f14708fcc606be442d22a75be0d43b1 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 04:13:47 +0000 Subject: [PATCH 086/147] fix(p184): G12 harness hostname, log capture, clone volume cleanup; record G12 isolated PASS Holder gets --hostname localhost (brokers exited on unresolvable hostname), bounded log tails are kept before --rm teardown, WriterClone rm -f gets -v (two anonymous Redis volumes were left and removed by ID), projector steady lag is <= 1 (LSO counts the commit marker). Packet section 11 records the run: broker/projector/Stream caught up 0.202/0.256/2.355 s, G1 inside gate in every window; G2/G3 not measured. Tests: test_p184_g12_isolated_fault + test_p184_dr05_dl_fault 46 PASS. --- scripts/p184_g12_isolated_fault.py | 33 ++++++++++++++--- tests/test_p184_g12_isolated_fault.py | 2 ++ .../p184-g12-isolated-fault-packet.md | 35 +++++++++++++++++++ 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/scripts/p184_g12_isolated_fault.py b/scripts/p184_g12_isolated_fault.py index 0bd5cfd8..f1079a1f 100644 --- a/scripts/p184_g12_isolated_fault.py +++ b/scripts/p184_g12_isolated_fault.py @@ -179,8 +179,9 @@ def topology(ns, work, topic_id="", *, uid=None): "QDL_KN_CURSOR_ACTIVE_KEY_ID": "dr08", "QDL_KN_TOPIC_ID": topic_id, "QDL_KN_ROUTE_GENERATION": ns, "QDL_KN_READ_VIEW_SECRET_FILE": "/work/read-view.secret"} mount = ["-v", f"{work}:/work"] + # The holder's hostname is the namespace's: Kafka/log4j resolve it (DR05 kafka: --hostname localhost). roles = {"net": _base(ns, "net", network="none", memory="16m", cpus="0.05") - + ["--entrypoint", "sleep", IMAGES["holder"], str(LIMITS["holder_s"])]} + + ["--hostname", "localhost", "--entrypoint", "sleep", IMAGES["holder"], str(LIMITS["holder_s"])]} for node in BROKERS: argv = _base(ns, f"kafka-{node}", network=net, memory="1024m", cpus="0.75", uid=uid) argv += ["--tmpfs", "/tmp:rw,exec,size=64m,mode=1777", "-v", f"{work}/kafka-{node}:/data", @@ -496,7 +497,10 @@ def evaluate_fault(fault, publish_rows, sampler_rows, monitor_rows, *, window_s= last = max((r["t"] for r in sampler_rows if r.get("k") == "view"), default=0) complete = last >= window[1] result["g1_window_complete"] = complete - result["projector_lag0_ns"] = first_monitor(monitor_rows, cleared, lambda r: r.get("projector_lag") == 0) + # Committed offset vs LSO keeps 1 for the transaction's commit marker in steady state + # (run 2026-10-10: lag 1 in 921 of 1,257 samples, 0 only twice), so "steady" is <= 1. + result["projector_lag_steady_ns"] = first_monitor( + monitor_rows, cleared, lambda r: r.get("projector_lag") is not None and r["projector_lag"] <= 1) result["isr_restored_ns"] = first_monitor(monitor_rows, cleared, lambda r: r.get("under_replicated") == 0) result["g12_caught_up_pass"] = result["caught_up_s"] <= GATE["caught_up_s"] result["pass"] = (result["g12_caught_up_pass"] and complete @@ -822,6 +826,10 @@ def run(*args, check=True, capture=True, timeout=120): args = list(args) if args[:3] == ["docker", "run", "-d"]: args[3:3] = ["--label", f"{LABEL}={ns}"] + elif args[:3] == ["docker", "rm", "-f"]: + # WriterClone removes with rm -f; the redis image's VOLUME /data would + # otherwise stay behind as an anonymous volume (G12 run 2026-10-10: 2 left). + args[3:3] = ["-v"] result = subprocess.run(args, capture_output=capture, text=True, timeout=timeout) if check and result.returncode: raise RuntimeError(f"{args[:3]} -> {result.returncode}: {result.stderr[-300:]}") @@ -902,6 +910,9 @@ def start(self, role, topic_id): # -- phases def prepare(self): + self.base.mkdir(mode=0o700, parents=True, exist_ok=True) + self.work.mkdir(mode=0o700) + self.evidence.mkdir(mode=0o755) resources = host_resources() self.note("preflight", **resources) if resources["disk_free_bytes"] < LIMITS["disk_free_bytes"]: @@ -915,9 +926,6 @@ def prepare(self): self.note("production_guard_before", containers=len(self.guard0), guard=self.guard0) self.volumes0 = len(sh(["docker", "volume", "ls", "-q"]).split()) self.networks0 = len(sh(["docker", "network", "ls", "-q"]).split()) - self.base.mkdir(mode=0o700, parents=True, exist_ok=True) - self.work.mkdir(mode=0o700) - self.evidence.mkdir(mode=0o755) (self.work / "handoff.json").write_text(json.dumps({ "namespace": self.ns, "status": "PREPARED_HANDOFF_NOT_RUN", "provenance": "TEST_ONLY", "level": "ISOLATED_FAULT", "acceptance": False, "material_ready": False}, indent=2) + "\n") @@ -1058,8 +1066,22 @@ def caught(): if not verdict["pass"]: raise RuntimeError(f"STOP {name}: G12/G1 verdict FAIL") + def keep_logs(self): + """Bounded log tails of every owned container (--rm loses them at stop).""" + logs = self.evidence / "logs" + logs.mkdir(exist_ok=True) + for name in owned_names(self.ns): + result = subprocess.run(["docker", "logs", "--tail", "120", name], capture_output=True, + text=True, timeout=30) + (logs / (name[len(self.ns) + 1:] + ".log")).write_text( + (result.stdout + result.stderr)[-60000:]) + def down(self): removed, errors = [], [] + try: + self.keep_logs() + except Exception as error: # noqa: BLE001 - evidence is best effort, teardown is not + errors.append("logs:" + type(error).__name__) for clone in self.clones.values(): try: for item in clone.close(): @@ -1135,6 +1157,7 @@ def expire(_signum, _frame): status = "COMPLETED" except BaseException as exc: # noqa: BLE001 - evidence first, then teardown error = f"{type(exc).__name__}: {exc}" + print("STOPPED", error, flush=True) if run.evidence.exists(): run.note("stopped", error=error) finally: diff --git a/tests/test_p184_g12_isolated_fault.py b/tests/test_p184_g12_isolated_fault.py index f069560e..b7be17e1 100644 --- a/tests/test_p184_g12_isolated_fault.py +++ b/tests/test_p184_g12_isolated_fault.py @@ -96,8 +96,10 @@ def fake(args, **_): run = g12.labelled_run(NS) run("docker", "run", "-d", "--rm", "--name", "x", "img") run("docker", "inspect", "x") + run("docker", "rm", "-f", "x") self.assertEqual(calls[0][:5], ["docker", "run", "-d", "--label", f"{g12.LABEL}={NS}"]) self.assertEqual(calls[1], ["docker", "inspect", "x"]) + self.assertEqual(calls[2], ["docker", "rm", "-f", "-v", "x"]) # no anonymous volume left class AnalysisTests(unittest.TestCase): diff --git a/upgrade/evidence/p184-g12-isolated-fault-packet.md b/upgrade/evidence/p184-g12-isolated-fault-packet.md index 763fd41c..e1504dca 100644 --- a/upgrade/evidence/p184-g12-isolated-fault-packet.md +++ b/upgrade/evidence/p184-g12-isolated-fault-packet.md @@ -128,3 +128,38 @@ public config remain under `~/.local/state/qdl-v2/p184-g12-20261010/`. No build Nothing shared changes, so there is nothing to roll back; a failed/aborted run is cleaned by `python3 -B scripts/p184_g12_isolated_fault.py --down ` (exact-name, label-checked). + +## 11. Result - 2026-10-10 (run 03:48-04:12 UTC) + +Verdict: G12 PASS on all three faults at `ISOLATED_FAULT` level (BOOK, TS writer clone cache). G2/G3 NOT_MEASURED. +Evidence: `~/.local/state/qdl-v2/p184-g12-20261010/evidence/` (`verdict.json` from the run, `verdict.reanalyzed.json` +after the lag-predicate fix below, `publish/sampler/monitor/faults/orchestrator/stats.jsonl`, `logs/`, `teardown.json`). +Failed setup attempts kept under `p184-g12-20261010-attempts/` (a1: no output, evidence dir created after first +log line; a2/a3: brokers exited - holder had no resolvable hostname, fixed with `--hostname localhost`; a4 debug). + +| Fault | injected | cleared | first valid apply (B / O) | caught up (B / O) | caught-up s | G1 window p95 / p99 / max ms (B; O) | +|---|---|---|---|---|---|---| +| broker kafka-3 (canonical leader) stop 60 s / start | 03:54:20.566 | 03:55:44.768 (ISR full 03:55:45.768) | 03:55:44.969 / .970 | 03:55:44.969 / .970 | 0.202 | 103.9 / 118.6 / 843.8; 103.8 / 120.5 / 844.3 | +| projector restart | 04:00:47.079 | 04:00:48.796 | 04:00:49.052 / .052 | 04:00:49.052 / .052 | 0.256 | 102.7 / 112.6 / 131.9; 103.6 / 117.3 / 183.4 | +| Stream restart | 04:05:51.182 | 04:05:54.827 | 04:05:57.182 / 04:05:54.960 | 04:05:57.182 / 04:05:54.960 | 2.355 | 101.8 / 111.4 / 304.7; 101.6 / 112.2 / 134.9 | + +Baseline (300 s, before faults): p95 103.9 / 103.9 ms, p99 112.8 / 116.6 ms, max 139.4 / 140.1 ms; n 1,200 per venue. +Every G1 window n >= 1,198. Consumer-visible outage: broker and projector faults none (max view age 176 ms, 0 +non-APPROVED samples); Stream restart BINANCE max view age 3,062 ms with 5 non-APPROVED samples, OKX 1,772 ms, 0. +Projector stage-A lag back to steady (<= 1) 4.0 s / 1.0 s / 0 s after each clear; lag peak 31 records on its restart. +Publisher 10,564 frames, 10,564 acked, 0 delivery errors, 0 resyncs. TS clone heartbeat DEGRADED with demanded 2 = +session-ready 2 throughout (BOOK-only isolated bundle; other writer feeds unavailable by construction). + +Resources (docker stats, 41 samples of all 18 roles): total CPU max 162 % (mean 64 %), total memory max 2,614 MiB; +peaks kafka-3 79 % / 478 MiB, kafka-2 594 MiB, query 52 % / 152 MiB, TS writers 7 % / 264 MiB each. +Disk free 109.9 GB before, 104.8 GB at start of the final attempt, >= 97 GB after. + +Teardown: 12 namespace containers + 6 WriterClone containers (2 writers, 2 Redis, 2 Postgres) + 2 log dirs removed; +by label 0, by name 0, networks 0 created/0 left, volumes 314 before -> 316 after the harness -> 2 anonymous volumes +(the WriterClone Redis `VOLUME /data`, `docker rm -f` without `-v`) removed by exact ID -> 314. Fixed in the harness: +its runner adds `-v` to WriterClone's `rm -f`. Private material and broker data dirs deleted. Production guard: 0 +breaches (query_kn_1/2 were recreated by another operator at 03:43-03:44, before the final attempt's baseline). + +Limits: same-host loopback rehearsal, one projector and one Stream replica, synthetic TEST_ONLY BOOK at 4 Hz/venue +(not production load), sampler resolution 50 ms. The monitor's "projector lag 0" predicate was wrong (committed +offset vs LSO keeps 1 for the commit marker); the re-analysis uses <= 1, no G12 number changed. From 890a08093e76b9f7a3f03f53ec650a25716868e4 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 04:15:12 +0000 Subject: [PATCH 087/147] docs(plan): D4-R3 G12 isolated fault acceptance PASS (75673bc, 2a5e809) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index e59a5d5a..02de9868 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71667,3 +71667,28 @@ OOM 0. one variable be tuned on the same image against the frozen G5 limit while the hot latency classes are watched. - Tests: new tests/test_kn_mark_hot_memo.py (6), ColdSliceEnvTests; 96 related tests OK (33 Redis skips). - G4 (g4b-1..4) is summarized with the rc4 run. + +### D4-R3 G12 isolated broker/projector/Stream fault acceptance - 2026-10-10 (Claude, subagent) + +Owner-approved isolated run. Status: G12 PASS at ISOLATED_FAULT (BOOK at the TS writer-clone cache); G2/G3 +NOT_MEASURED. Commits 75673bc (harness and packet, written before the run) and 2a5e809 (run fixes and result), cherry-picked. +Packet: upgrade/evidence/p184-g12-isolated-fault-packet.md. +- scripts/p184_g12_isolated_fault.py, namespace p184-g12-dl-cfcba18d978b452f inside one --network none holder: + - 3 KRaft brokers (RF3, min ISR 2, mTLS); + - deployed core/projector/Stream/Query images by ID; + - TEST_ONLY BOOK at 4 Hz per venue; + - 2 TS WriterClones (4 workers); + - a 50 ms sampler with the TS at-use book validator. + It reuses the DR05 material, DR08 serve_query and kn_recovery_observe KafkaAdminSource (first run against real brokers). +- Caught up after the fault cleared (<= 600 s), B/O: + - broker kafka-3 (partition leader) stopped 60 s: 0.202 s; + - projector restart: 0.256 s; + - Stream restart: 2.355 s. + G1 300 s windows p95 <= 103.9 ms, p99 <= 120.5 ms, max <= 844 ms, n >= 1,198 per venue; baseline p95 ~104 ms; + 0 publisher errors. +- Seen by the TS clones: Stream restart B max view age 3,062 ms (5 non-APPROVED samples), O 1,772 ms. Broker and + projector faults: max 176 ms. +- Resources: peak 162 % CPU / 2.6 GiB total. Teardown: 18 containers, 0 networks, 2 WriterClone anonymous volumes + removed by ID (the harness now passes -v). Production guard: 0 breaches. +- Tests: 46 PASS; after the cherry-pick, g12 + dr05_dl_fault + kn_recovery_observe OK. +- Limits: same host, a single projector/Stream replica, synthetic BOOK only. Not HA or production evidence. From 1a2daf1762f01572db28b01dba62affb78a63755 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 04:28:04 +0000 Subject: [PATCH 088/147] feat(l2): IDR-040 fix 2 in-band quiet-book proof (proof_observed_at_ns) A quiet Binance book publishes nothing, so it could not be shown current beyond its last frame. The native ingestor (opt-in book_liveness_proof_ms, default off) now writes a transport liveness marker into each quiet book's own raw partition, stamped with the receipt time of the newest WebSocket message (data frame or pong to a ping it sends when idle). TCP order plus partition order put every earlier frame of the book ahead of the marker. The realtime core turns a marker into a quiet-proof BOOK_SNAPSHOT only for the exact owning session/generation of a READY, non-invalidated book, at most once per materialization interval: the current verified view, the original provider event time, and the new additive field OrderBookSnapshot.proof_observed_at_ns = 8. Markers never start, change, reset or fence a book; anything unprovable is filtered TRANSPORT_LIVENESS_UNPROVEN / _NOT_DUE, malformed ones are quarantined. Query and the SDK stream projection report quality.proof_observed_at_ns (and refusal diagnostics) for verified book snapshots only; freshness_ms, state and execution_eligible are unchanged. The key is emitted only when a proof exists, so strict 2.0.x readers see no change until proofs are enabled. OpenAPI snapshot and SDK changelog updated (additive). --- .../proto/qdl/marketdata/v2/market_data.proto | 6 + contracts/v2/openapi.snapshot.json | 24 + .../qdl/marketdata/v2/market_data_pb2.py | 64 +-- .../qdl/marketdata/v2/qdl.marketdata.v2.rs | 7 + qdl/query/results.py | 9 + qdl/query/service.py | 7 +- qdl/runtime/stable_source.py | 5 + qdl_sdk/CHANGELOG.md | 9 + qdl_sdk/models.py | 24 + qdl_sdk/projection.py | 6 + rust/qdl-core/src/canonical.rs | 19 +- rust/qdl-core/src/l2_adapter.rs | 6 + .../src/bin/qdl-native-raw-ingestor.rs | 268 ++++++++- rust/qdl-realtime-core/src/lib.rs | 529 +++++++++++++++++- tests/test_idr040_quiet_book_proof.py | 204 +++++++ 15 files changed, 1140 insertions(+), 47 deletions(-) create mode 100644 tests/test_idr040_quiet_book_proof.py diff --git a/contracts/proto/qdl/marketdata/v2/market_data.proto b/contracts/proto/qdl/marketdata/v2/market_data.proto index 8b12d087..c8f3f66b 100644 --- a/contracts/proto/qdl/marketdata/v2/market_data.proto +++ b/contracts/proto/qdl/marketdata/v2/market_data.proto @@ -113,6 +113,12 @@ message OrderBookSnapshot { // The readable projection contains the requested top-N depth rather than // every provider level. Clients must not assume omitted levels are absent. bool truncated = 7; + // Additive (IDR-040 fix 2). Zero/absent: no quiet proof; the book is current + // only as of `source_event_time_ns`. Positive: local receipt time of in-band + // transport evidence on the same provider connection, after every earlier + // frame of this book was applied without a gap. It never re-dates + // `source_event_time_ns` and is set only on a quiet-proof snapshot. + int64 proof_observed_at_ns = 8; } message OrderBookDelta { diff --git a/contracts/v2/openapi.snapshot.json b/contracts/v2/openapi.snapshot.json index 9c2d544c..71c79473 100644 --- a/contracts/v2/openapi.snapshot.json +++ b/contracts/v2/openapi.snapshot.json @@ -1664,6 +1664,18 @@ ], "title": "Observed At Ns" }, + "proof_observed_at_ns": { + "anyOf": [ + { + "exclusiveMinimum": 0.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Proof Observed At Ns" + }, "provider_session_liveness_ms": { "anyOf": [ { @@ -1786,6 +1798,18 @@ "title": "Policy Id", "type": "string" }, + "proof_observed_at_ns": { + "anyOf": [ + { + "exclusiveMinimum": 0.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Proof Observed At Ns" + }, "provider_session_liveness_ms": { "anyOf": [ { diff --git a/generated/python/qdl/marketdata/v2/market_data_pb2.py b/generated/python/qdl/marketdata/v2/market_data_pb2.py index 087a2097..bb7277f6 100644 --- a/generated/python/qdl/marketdata/v2/market_data_pb2.py +++ b/generated/python/qdl/marketdata/v2/market_data_pb2.py @@ -26,23 +26,23 @@ from qdl.quality.v1 import quality_pb2 as qdl_dot_quality_dot_v1_dot_quality__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#qdl/marketdata/v2/market_data.proto\x12\x11qdl.marketdata.v2\x1a\x1aqdl/common/v1/common.proto\x1a\x1cqdl/quality/v1/quality.proto\"\xb9\x03\n\x05Trade\x12&\n\x0fnative_trade_id\x18\x01 \x01(\tR\rnativeTradeId\x12\x31\n\x05price\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x05price\x12\x37\n\x08quantity\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x08quantity\x12\x43\n\x0e\x61ggressor_side\x18\x04 \x01(\x0e\x32\x1c.qdl.common.v1.AggressorSideR\raggressorSide\x12$\n\x0eis_block_trade\x18\x05 \x01(\x08R\x0cisBlockTrade\x12$\n\x0eis_buyer_maker\x18\x06 \x01(\x08R\x0cisBuyerMaker\x12@\n\rquantity_unit\x18\x07 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\x0cquantityUnit\x12I\n\ridentity_kind\x18\x08 \x01(\x0e\x32$.qdl.marketdata.v2.TradeIdentityKindR\x0cidentityKind\"\xd3\x02\n\x05Quote\x12\x38\n\tbid_price\x18\x01 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x08\x62idPrice\x12>\n\x0c\x62id_quantity\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x0b\x62idQuantity\x12\x38\n\task_price\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x08\x61skPrice\x12>\n\x0c\x61sk_quantity\x18\x04 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x0b\x61skQuantity\x12\x14\n\x05level\x18\x05 \x01(\rR\x05level\x12@\n\rquantity_unit\x18\x06 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\x0cquantityUnit\"\xbc\x07\n\x03\x42\x61r\x12\x1a\n\x08interval\x18\x01 \x01(\tR\x08interval\x12 \n\x0copen_time_ns\x18\x02 \x01(\x03R\nopenTimeNs\x12\"\n\rclose_time_ns\x18\x03 \x01(\x03R\x0b\x63loseTimeNs\x12/\n\x04open\x18\x04 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x04open\x12/\n\x04high\x18\x05 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x04high\x12-\n\x03low\x18\x06 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x03low\x12\x31\n\x05\x63lose\x18\x07 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x05\x63lose\x12\x33\n\x06volume\x18\x08 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x06volume\x12\x1f\n\x0btrade_count\x18\t \x01(\x04R\ntradeCount\x12\x19\n\x08is_final\x18\n \x01(\x08R\x07isFinal\x12\x1a\n\x08revision\x18\x0b \x01(\rR\x08revision\x12\x30\n\x06origin\x18\x0c \x01(\x0e\x32\x18.qdl.common.v1.BarOriginR\x06origin\x12=\n\tlifecycle\x18\r \x01(\x0e\x32\x1f.qdl.marketdata.v2.BarLifecycleR\tlifecycle\x12\x33\n\x13supersedes_event_id\x18\x0e \x01(\x0cH\x00R\x11supersedesEventId\x88\x01\x01\x12<\n\x0bvolume_unit\x18\x0f \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\nvolumeUnit\x12\x41\n\x0b\x62\x61se_volume\x18\x10 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x01R\nbaseVolume\x88\x01\x01\x12\x43\n\x0cquote_volume\x18\x11 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x02R\x0bquoteVolume\x88\x01\x01\x12I\n\x0f\x63ontract_volume\x18\x12 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x03R\x0e\x63ontractVolume\x88\x01\x01\x42\x16\n\x14_supersedes_event_idB\x0e\n\x0c_base_volumeB\x0f\n\r_quote_volumeB\x12\n\x10_contract_volume\"\x87\x02\n\tBookLevel\x12+\n\x04side\x18\x01 \x01(\x0e\x32\x17.qdl.common.v1.BookSideR\x04side\x12\x31\n\x05price\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x05price\x12\x37\n\x08quantity\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x08quantity\x12\x1f\n\x0border_count\x18\x04 \x01(\rR\norderCount\x12@\n\rquantity_unit\x18\x05 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\x0cquantityUnit\"\x98\x02\n\x11OrderBookSnapshot\x12\'\n\x0fnative_sequence\x18\x01 \x01(\tR\x0enativeSequence\x12\x1a\n\x08\x63hecksum\x18\x02 \x01(\tR\x08\x63hecksum\x12\x34\n\x06levels\x18\x03 \x03(\x0b\x32\x1c.qdl.marketdata.v2.BookLevelR\x06levels\x12\x14\n\x05\x64\x65pth\x18\x04 \x01(\rR\x05\x64\x65pth\x12\'\n\x0f\x62ook_generation\x18\x05 \x01(\x04R\x0e\x62ookGeneration\x12+\n\x11sequence_verified\x18\x06 \x01(\x08R\x10sequenceVerified\x12\x1c\n\ttruncated\x18\x07 \x01(\x08R\ttruncated\"\xe1\x02\n\x0eOrderBookDelta\x12\x32\n\x15native_sequence_start\x18\x01 \x01(\tR\x13nativeSequenceStart\x12.\n\x13native_sequence_end\x18\x02 \x01(\tR\x11nativeSequenceEnd\x12+\n\x11snapshot_sequence\x18\x03 \x01(\tR\x10snapshotSequence\x12\x1a\n\x08\x63hecksum\x18\x04 \x01(\tR\x08\x63hecksum\x12\x36\n\x07updates\x18\x05 \x03(\x0b\x32\x1c.qdl.marketdata.v2.BookLevelR\x07updates\x12\x14\n\x05reset\x18\x06 \x01(\x08R\x05reset\x12\'\n\x0f\x62ook_generation\x18\x07 \x01(\x04R\x0e\x62ookGeneration\x12+\n\x11sequence_verified\x18\x08 \x01(\x08R\x10sequenceVerified\"\xb5\x01\n\x0b\x46undingRate\x12/\n\x04rate\x18\x01 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x04rate\x12&\n\x0f\x66unding_time_ns\x18\x02 \x01(\x03R\rfundingTimeNs\x12\x34\n\x14next_funding_time_ns\x18\x03 \x01(\x03H\x00R\x11nextFundingTimeNs\x88\x01\x01\x42\x17\n\x15_next_funding_time_ns\"\x81\x02\n\x0cOpenInterest\x12\x37\n\x08quantity\x18\x01 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x08quantity\x12<\n\x08notional\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x00R\x08notional\x88\x01\x01\x12@\n\rquantity_unit\x18\x03 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\x0cquantityUnit\x12+\n\x11sampling_interval\x18\x04 \x01(\tR\x10samplingIntervalB\x0b\n\t_notional\"\x8a\x01\n\x0eMarkIndexPrice\x12:\n\nmark_price\x18\x01 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\tmarkPrice\x12<\n\x0bindex_price\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\nindexPrice\"\x89\x03\n\x0eLongShortRatio\x12K\n\npopulation\x18\x01 \x01(\x0e\x32+.qdl.marketdata.v2.LongShortRatioPopulationR\npopulation\x12+\n\x11sampling_interval\x18\x02 \x01(\tR\x10samplingInterval\x12:\n\nlong_value\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\tlongValue\x12<\n\x0bshort_value\x18\x04 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\nshortValue\x12\x45\n\x10long_short_ratio\x18\x05 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x0elongShortRatio\x12<\n\nvalue_unit\x18\x06 \x01(\x0e\x32\x1d.qdl.marketdata.v2.MetricUnitR\tvalueUnit\"\xb7\x02\n\tTakerFlow\x12+\n\x11sampling_interval\x18\x01 \x01(\tR\x10samplingInterval\x12:\n\nbuy_volume\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\tbuyVolume\x12<\n\x0bsell_volume\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\nsellVolume\x12\x41\n\x0e\x62uy_sell_ratio\x18\x04 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x0c\x62uySellRatio\x12@\n\rquantity_unit\x18\x05 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\x0cquantityUnit\"\xc6\x03\n\x05\x42\x61sis\x12\x30\n\x04kind\x18\x01 \x01(\x0e\x32\x1c.qdl.marketdata.v2.BasisKindR\x04kind\x12+\n\x11sampling_interval\x18\x02 \x01(\tR\x10samplingInterval\x12\x31\n\x05\x62\x61sis\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x05\x62\x61sis\x12<\n\nbasis_unit\x18\x04 \x01(\x0e\x32\x1d.qdl.marketdata.v2.MetricUnitR\tbasisUnit\x12K\n\x10\x61nnualized_basis\x18\x05 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x00R\x0f\x61nnualizedBasis\x88\x01\x01\x12\x38\n\x18reference_instrument_uid\x18\x06 \x01(\tR\x16referenceInstrumentUid\x12\x1d\n\nformula_id\x18\x07 \x01(\tR\tformulaId\x12\x32\n\x15input_instrument_uids\x18\x08 \x03(\tR\x13inputInstrumentUidsB\x13\n\x11_annualized_basis\"\x95\x04\n\x10\x43ontractMetadata\x12#\n\rcontract_kind\x18\x01 \x01(\tR\x0c\x63ontractKind\x12)\n\x10settlement_asset\x18\x02 \x01(\tR\x0fsettlementAsset\x12L\n\x13\x63ontract_multiplier\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x12\x63ontractMultiplier\x12:\n\nprice_tick\x18\x04 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\tpriceTick\x12@\n\rquantity_step\x18\x05 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x0cquantityStep\x12)\n\x0e\x65xpiry_time_ns\x18\x06 \x01(\x03H\x00R\x0c\x65xpiryTimeNs\x88\x01\x01\x12\x33\n\x13\x66unding_interval_ns\x18\x07 \x01(\x03H\x01R\x11\x66undingIntervalNs\x88\x01\x01\x12\x1e\n\ncontinuous\x18\x08 \x01(\x08R\ncontinuous\x12:\n\x19underlying_instrument_uid\x18\t \x01(\tR\x17underlyingInstrumentUidB\x11\n\x0f_expiry_time_nsB\x16\n\x14_funding_interval_ns\"\xd8\x04\n\x06Ticker\x12:\n\nlast_price\x18\x01 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\tlastPrice\x12\x45\n\rlast_quantity\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x00R\x0clastQuantity\x88\x01\x01\x12;\n\x08open_24h\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x01R\x07open24h\x88\x01\x01\x12;\n\x08high_24h\x18\x04 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x02R\x07high24h\x88\x01\x01\x12\x39\n\x07low_24h\x18\x05 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x03R\x06low24h\x88\x01\x01\x12?\n\nvolume_24h\x18\x06 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x04R\tvolume24h\x88\x01\x01\x12I\n\x12last_quantity_unit\x18\x07 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\x10lastQuantityUnit\x12\x43\n\x0fvolume_24h_unit\x18\x08 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\rvolume24hUnitB\x10\n\x0e_last_quantityB\x0b\n\t_open_24hB\x0b\n\t_high_24hB\n\n\x08_low_24hB\r\n\x0b_volume_24h\"\xa7\x12\n\rEventEnvelope\x12\x1f\n\x0bschema_name\x18\x01 \x01(\tR\nschemaName\x12!\n\x0cschema_major\x18\x02 \x01(\rR\x0bschemaMajor\x12!\n\x0cschema_minor\x18\x03 \x01(\rR\x0bschemaMinor\x12\x19\n\x08\x65vent_id\x18\x04 \x01(\x0cR\x07\x65ventId\x12%\n\x0einstrument_uid\x18\x05 \x01(\tR\rinstrumentUid\x12#\n\rinstrument_id\x18\x06 \x01(\tR\x0cinstrumentId\x12/\n\x13instrument_revision\x18\x07 \x01(\x04R\x12instrumentRevision\x12\x14\n\x05venue\x18\x08 \x01(\tR\x05venue\x12\x16\n\x06market\x18\t \x01(\tR\x06market\x12!\n\x0cproduct_type\x18\n \x01(\tR\x0bproductType\x12#\n\rnative_symbol\x18\x0b \x01(\tR\x0cnativeSymbol\x12\x1a\n\x08provider\x18\x0c \x01(\tR\x08provider\x12\x1b\n\tsource_id\x18\r \x01(\tR\x08sourceId\x12:\n\x0bsource_role\x18\x0e \x01(\x0e\x32\x19.qdl.common.v1.SourceRoleR\nsourceRole\x12\x1f\n\x0blease_epoch\x18\x0f \x01(\x04R\nleaseEpoch\x12/\n\x14source_event_time_ns\x18\x10 \x01(\x03R\x11sourceEventTimeNs\x12$\n\x0ereceived_at_ns\x18\x11 \x01(\x03R\x0creceivedAtNs\x12(\n\x10normalized_at_ns\x18\x12 \x01(\x03R\x0enormalizedAtNs\x12&\n\x0fpublished_at_ns\x18\x13 \x01(\x03R\rpublishedAtNs\x12\'\n\x0fsource_sequence\x18\x14 \x01(\tR\x0esourceSequence\x12-\n\x12partition_sequence\x18\x15 \x01(\x04R\x11partitionSequence\x12-\n\x12normalizer_version\x18\x16 \x01(\tR\x11normalizerVersion\x12\'\n\x0f\x61\x64\x61pter_version\x18\x17 \x01(\tR\x0e\x61\x64\x61pterVersion\x12?\n\rquality_flags\x18\x18 \x03(\x0e\x32\x1a.qdl.common.v1.QualityFlagR\x0cqualityFlags\x12(\n\x10raw_payload_hash\x18\x19 \x01(\x0cR\x0erawPayloadHash\x12%\n\x0e\x63orrelation_id\x18\x1a \x01(\tR\rcorrelationId\x12\'\n\x0f\x63onfig_revision\x18\x1b \x01(\x04R\x0e\x63onfigRevision\x12*\n\x11source_session_id\x18\x1c \x01(\tR\x0fsourceSessionId\x12\x33\n\x15\x63onnection_generation\x18\x1d \x01(\x04R\x14\x63onnectionGeneration\x12-\n\x12\x61uthority_revision\x18\x1e \x01(\x04R\x11\x61uthorityRevision\x12\x30\n\x14partition_plan_epoch\x18\x1f \x01(\x04R\x12partitionPlanEpoch\x12\x34\n\x16\x63\x61nonical_payload_hash\x18 \x01(\x0cR\x14\x63\x61nonicalPayloadHash\x12$\n\x0eraw_capture_id\x18! \x01(\x0cR\x0crawCaptureId\x12\x30\n\x05trade\x18( \x01(\x0b\x32\x18.qdl.marketdata.v2.TradeH\x00R\x05trade\x12\x30\n\x05quote\x18) \x01(\x0b\x32\x18.qdl.marketdata.v2.QuoteH\x00R\x05quote\x12*\n\x03\x62\x61r\x18* \x01(\x0b\x32\x16.qdl.marketdata.v2.BarH\x00R\x03\x62\x61r\x12K\n\rbook_snapshot\x18+ \x01(\x0b\x32$.qdl.marketdata.v2.OrderBookSnapshotH\x00R\x0c\x62ookSnapshot\x12\x42\n\nbook_delta\x18, \x01(\x0b\x32!.qdl.marketdata.v2.OrderBookDeltaH\x00R\tbookDelta\x12\x43\n\x0c\x66unding_rate\x18- \x01(\x0b\x32\x1e.qdl.marketdata.v2.FundingRateH\x00R\x0b\x66undingRate\x12\x46\n\ropen_interest\x18. \x01(\x0b\x32\x1f.qdl.marketdata.v2.OpenInterestH\x00R\x0copenInterest\x12M\n\x10mark_index_price\x18/ \x01(\x0b\x32!.qdl.marketdata.v2.MarkIndexPriceH\x00R\x0emarkIndexPrice\x12\x33\n\x06ticker\x18\x30 \x01(\x0b\x32\x19.qdl.marketdata.v2.TickerH\x00R\x06ticker\x12?\n\nfeed_state\x18\x31 \x01(\x0b\x32\x1e.qdl.quality.v1.FeedStateEventH\x00R\tfeedState\x12G\n\rquality_event\x18\x32 \x01(\x0b\x32 .qdl.quality.v1.DataQualityEventH\x00R\x0cqualityEvent\x12M\n\x10long_short_ratio\x18\x33 \x01(\x0b\x32!.qdl.marketdata.v2.LongShortRatioH\x00R\x0elongShortRatio\x12=\n\ntaker_flow\x18\x34 \x01(\x0b\x32\x1c.qdl.marketdata.v2.TakerFlowH\x00R\ttakerFlow\x12\x30\n\x05\x62\x61sis\x18\x35 \x01(\x0b\x32\x18.qdl.marketdata.v2.BasisH\x00R\x05\x62\x61sis\x12R\n\x11\x63ontract_metadata\x18\x36 \x01(\x0b\x32#.qdl.marketdata.v2.ContractMetadataH\x00R\x10\x63ontractMetadataB\t\n\x07payload*\x85\x01\n\x11TradeIdentityKind\x12#\n\x1fTRADE_IDENTITY_KIND_UNSPECIFIED\x10\x00\x12\x1e\n\x1aTRADE_IDENTITY_KIND_NATIVE\x10\x01\x12+\n\'TRADE_IDENTITY_KIND_DERIVED_RAW_CAPTURE\x10\x02*\x9d\x01\n\x0c\x42\x61rLifecycle\x12\x1d\n\x19\x42\x41R_LIFECYCLE_UNSPECIFIED\x10\x00\x12\x1d\n\x19\x42\x41R_LIFECYCLE_IN_PROGRESS\x10\x01\x12\x17\n\x13\x42\x41R_LIFECYCLE_FINAL\x10\x02\x12\x19\n\x15\x42\x41R_LIFECYCLE_REVISED\x10\x03\x12\x1b\n\x17\x42\x41R_LIFECYCLE_CANCELLED\x10\x04*\xd2\x01\n\x18LongShortRatioPopulation\x12+\n\'LONG_SHORT_RATIO_POPULATION_UNSPECIFIED\x10\x00\x12.\n*LONG_SHORT_RATIO_POPULATION_GLOBAL_ACCOUNT\x10\x01\x12+\n\'LONG_SHORT_RATIO_POPULATION_TOP_ACCOUNT\x10\x02\x12,\n(LONG_SHORT_RATIO_POPULATION_TOP_POSITION\x10\x03*_\n\tBasisKind\x12\x1a\n\x16\x42\x41SIS_KIND_UNSPECIFIED\x10\x00\x12\x1e\n\x1a\x42\x41SIS_KIND_PROVIDER_NATIVE\x10\x01\x12\x16\n\x12\x42\x41SIS_KIND_DERIVED\x10\x02*\xfb\x01\n\nMetricUnit\x12\x1b\n\x17METRIC_UNIT_UNSPECIFIED\x10\x00\x12\x15\n\x11METRIC_UNIT_RATIO\x10\x01\x12\x15\n\x11METRIC_UNIT_PRICE\x10\x02\x12\x1a\n\x16METRIC_UNIT_BASE_ASSET\x10\x03\x12\x1b\n\x17METRIC_UNIT_QUOTE_ASSET\x10\x04\x12\x18\n\x14METRIC_UNIT_CONTRACT\x10\x05\x12\x18\n\x14METRIC_UNIT_NOTIONAL\x10\x06\x12\x17\n\x13METRIC_UNIT_PERCENT\x10\x07\x12\x1c\n\x18METRIC_UNIT_BASIS_POINTS\x10\x08\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#qdl/marketdata/v2/market_data.proto\x12\x11qdl.marketdata.v2\x1a\x1aqdl/common/v1/common.proto\x1a\x1cqdl/quality/v1/quality.proto\"\xb9\x03\n\x05Trade\x12&\n\x0fnative_trade_id\x18\x01 \x01(\tR\rnativeTradeId\x12\x31\n\x05price\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x05price\x12\x37\n\x08quantity\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x08quantity\x12\x43\n\x0e\x61ggressor_side\x18\x04 \x01(\x0e\x32\x1c.qdl.common.v1.AggressorSideR\raggressorSide\x12$\n\x0eis_block_trade\x18\x05 \x01(\x08R\x0cisBlockTrade\x12$\n\x0eis_buyer_maker\x18\x06 \x01(\x08R\x0cisBuyerMaker\x12@\n\rquantity_unit\x18\x07 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\x0cquantityUnit\x12I\n\ridentity_kind\x18\x08 \x01(\x0e\x32$.qdl.marketdata.v2.TradeIdentityKindR\x0cidentityKind\"\xd3\x02\n\x05Quote\x12\x38\n\tbid_price\x18\x01 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x08\x62idPrice\x12>\n\x0c\x62id_quantity\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x0b\x62idQuantity\x12\x38\n\task_price\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x08\x61skPrice\x12>\n\x0c\x61sk_quantity\x18\x04 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x0b\x61skQuantity\x12\x14\n\x05level\x18\x05 \x01(\rR\x05level\x12@\n\rquantity_unit\x18\x06 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\x0cquantityUnit\"\xbc\x07\n\x03\x42\x61r\x12\x1a\n\x08interval\x18\x01 \x01(\tR\x08interval\x12 \n\x0copen_time_ns\x18\x02 \x01(\x03R\nopenTimeNs\x12\"\n\rclose_time_ns\x18\x03 \x01(\x03R\x0b\x63loseTimeNs\x12/\n\x04open\x18\x04 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x04open\x12/\n\x04high\x18\x05 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x04high\x12-\n\x03low\x18\x06 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x03low\x12\x31\n\x05\x63lose\x18\x07 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x05\x63lose\x12\x33\n\x06volume\x18\x08 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x06volume\x12\x1f\n\x0btrade_count\x18\t \x01(\x04R\ntradeCount\x12\x19\n\x08is_final\x18\n \x01(\x08R\x07isFinal\x12\x1a\n\x08revision\x18\x0b \x01(\rR\x08revision\x12\x30\n\x06origin\x18\x0c \x01(\x0e\x32\x18.qdl.common.v1.BarOriginR\x06origin\x12=\n\tlifecycle\x18\r \x01(\x0e\x32\x1f.qdl.marketdata.v2.BarLifecycleR\tlifecycle\x12\x33\n\x13supersedes_event_id\x18\x0e \x01(\x0cH\x00R\x11supersedesEventId\x88\x01\x01\x12<\n\x0bvolume_unit\x18\x0f \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\nvolumeUnit\x12\x41\n\x0b\x62\x61se_volume\x18\x10 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x01R\nbaseVolume\x88\x01\x01\x12\x43\n\x0cquote_volume\x18\x11 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x02R\x0bquoteVolume\x88\x01\x01\x12I\n\x0f\x63ontract_volume\x18\x12 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x03R\x0e\x63ontractVolume\x88\x01\x01\x42\x16\n\x14_supersedes_event_idB\x0e\n\x0c_base_volumeB\x0f\n\r_quote_volumeB\x12\n\x10_contract_volume\"\x87\x02\n\tBookLevel\x12+\n\x04side\x18\x01 \x01(\x0e\x32\x17.qdl.common.v1.BookSideR\x04side\x12\x31\n\x05price\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x05price\x12\x37\n\x08quantity\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x08quantity\x12\x1f\n\x0border_count\x18\x04 \x01(\rR\norderCount\x12@\n\rquantity_unit\x18\x05 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\x0cquantityUnit\"\xc9\x02\n\x11OrderBookSnapshot\x12\'\n\x0fnative_sequence\x18\x01 \x01(\tR\x0enativeSequence\x12\x1a\n\x08\x63hecksum\x18\x02 \x01(\tR\x08\x63hecksum\x12\x34\n\x06levels\x18\x03 \x03(\x0b\x32\x1c.qdl.marketdata.v2.BookLevelR\x06levels\x12\x14\n\x05\x64\x65pth\x18\x04 \x01(\rR\x05\x64\x65pth\x12\'\n\x0f\x62ook_generation\x18\x05 \x01(\x04R\x0e\x62ookGeneration\x12+\n\x11sequence_verified\x18\x06 \x01(\x08R\x10sequenceVerified\x12\x1c\n\ttruncated\x18\x07 \x01(\x08R\ttruncated\x12/\n\x14proof_observed_at_ns\x18\x08 \x01(\x03R\x11proofObservedAtNs\"\xe1\x02\n\x0eOrderBookDelta\x12\x32\n\x15native_sequence_start\x18\x01 \x01(\tR\x13nativeSequenceStart\x12.\n\x13native_sequence_end\x18\x02 \x01(\tR\x11nativeSequenceEnd\x12+\n\x11snapshot_sequence\x18\x03 \x01(\tR\x10snapshotSequence\x12\x1a\n\x08\x63hecksum\x18\x04 \x01(\tR\x08\x63hecksum\x12\x36\n\x07updates\x18\x05 \x03(\x0b\x32\x1c.qdl.marketdata.v2.BookLevelR\x07updates\x12\x14\n\x05reset\x18\x06 \x01(\x08R\x05reset\x12\'\n\x0f\x62ook_generation\x18\x07 \x01(\x04R\x0e\x62ookGeneration\x12+\n\x11sequence_verified\x18\x08 \x01(\x08R\x10sequenceVerified\"\xb5\x01\n\x0b\x46undingRate\x12/\n\x04rate\x18\x01 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x04rate\x12&\n\x0f\x66unding_time_ns\x18\x02 \x01(\x03R\rfundingTimeNs\x12\x34\n\x14next_funding_time_ns\x18\x03 \x01(\x03H\x00R\x11nextFundingTimeNs\x88\x01\x01\x42\x17\n\x15_next_funding_time_ns\"\x81\x02\n\x0cOpenInterest\x12\x37\n\x08quantity\x18\x01 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x08quantity\x12<\n\x08notional\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x00R\x08notional\x88\x01\x01\x12@\n\rquantity_unit\x18\x03 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\x0cquantityUnit\x12+\n\x11sampling_interval\x18\x04 \x01(\tR\x10samplingIntervalB\x0b\n\t_notional\"\x8a\x01\n\x0eMarkIndexPrice\x12:\n\nmark_price\x18\x01 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\tmarkPrice\x12<\n\x0bindex_price\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\nindexPrice\"\x89\x03\n\x0eLongShortRatio\x12K\n\npopulation\x18\x01 \x01(\x0e\x32+.qdl.marketdata.v2.LongShortRatioPopulationR\npopulation\x12+\n\x11sampling_interval\x18\x02 \x01(\tR\x10samplingInterval\x12:\n\nlong_value\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\tlongValue\x12<\n\x0bshort_value\x18\x04 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\nshortValue\x12\x45\n\x10long_short_ratio\x18\x05 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x0elongShortRatio\x12<\n\nvalue_unit\x18\x06 \x01(\x0e\x32\x1d.qdl.marketdata.v2.MetricUnitR\tvalueUnit\"\xb7\x02\n\tTakerFlow\x12+\n\x11sampling_interval\x18\x01 \x01(\tR\x10samplingInterval\x12:\n\nbuy_volume\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\tbuyVolume\x12<\n\x0bsell_volume\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\nsellVolume\x12\x41\n\x0e\x62uy_sell_ratio\x18\x04 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x0c\x62uySellRatio\x12@\n\rquantity_unit\x18\x05 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\x0cquantityUnit\"\xc6\x03\n\x05\x42\x61sis\x12\x30\n\x04kind\x18\x01 \x01(\x0e\x32\x1c.qdl.marketdata.v2.BasisKindR\x04kind\x12+\n\x11sampling_interval\x18\x02 \x01(\tR\x10samplingInterval\x12\x31\n\x05\x62\x61sis\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x05\x62\x61sis\x12<\n\nbasis_unit\x18\x04 \x01(\x0e\x32\x1d.qdl.marketdata.v2.MetricUnitR\tbasisUnit\x12K\n\x10\x61nnualized_basis\x18\x05 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x00R\x0f\x61nnualizedBasis\x88\x01\x01\x12\x38\n\x18reference_instrument_uid\x18\x06 \x01(\tR\x16referenceInstrumentUid\x12\x1d\n\nformula_id\x18\x07 \x01(\tR\tformulaId\x12\x32\n\x15input_instrument_uids\x18\x08 \x03(\tR\x13inputInstrumentUidsB\x13\n\x11_annualized_basis\"\x95\x04\n\x10\x43ontractMetadata\x12#\n\rcontract_kind\x18\x01 \x01(\tR\x0c\x63ontractKind\x12)\n\x10settlement_asset\x18\x02 \x01(\tR\x0fsettlementAsset\x12L\n\x13\x63ontract_multiplier\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x12\x63ontractMultiplier\x12:\n\nprice_tick\x18\x04 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\tpriceTick\x12@\n\rquantity_step\x18\x05 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\x0cquantityStep\x12)\n\x0e\x65xpiry_time_ns\x18\x06 \x01(\x03H\x00R\x0c\x65xpiryTimeNs\x88\x01\x01\x12\x33\n\x13\x66unding_interval_ns\x18\x07 \x01(\x03H\x01R\x11\x66undingIntervalNs\x88\x01\x01\x12\x1e\n\ncontinuous\x18\x08 \x01(\x08R\ncontinuous\x12:\n\x19underlying_instrument_uid\x18\t \x01(\tR\x17underlyingInstrumentUidB\x11\n\x0f_expiry_time_nsB\x16\n\x14_funding_interval_ns\"\xd8\x04\n\x06Ticker\x12:\n\nlast_price\x18\x01 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueR\tlastPrice\x12\x45\n\rlast_quantity\x18\x02 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x00R\x0clastQuantity\x88\x01\x01\x12;\n\x08open_24h\x18\x03 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x01R\x07open24h\x88\x01\x01\x12;\n\x08high_24h\x18\x04 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x02R\x07high24h\x88\x01\x01\x12\x39\n\x07low_24h\x18\x05 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x03R\x06low24h\x88\x01\x01\x12?\n\nvolume_24h\x18\x06 \x01(\x0b\x32\x1b.qdl.common.v1.DecimalValueH\x04R\tvolume24h\x88\x01\x01\x12I\n\x12last_quantity_unit\x18\x07 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\x10lastQuantityUnit\x12\x43\n\x0fvolume_24h_unit\x18\x08 \x01(\x0e\x32\x1b.qdl.common.v1.QuantityUnitR\rvolume24hUnitB\x10\n\x0e_last_quantityB\x0b\n\t_open_24hB\x0b\n\t_high_24hB\n\n\x08_low_24hB\r\n\x0b_volume_24h\"\xa7\x12\n\rEventEnvelope\x12\x1f\n\x0bschema_name\x18\x01 \x01(\tR\nschemaName\x12!\n\x0cschema_major\x18\x02 \x01(\rR\x0bschemaMajor\x12!\n\x0cschema_minor\x18\x03 \x01(\rR\x0bschemaMinor\x12\x19\n\x08\x65vent_id\x18\x04 \x01(\x0cR\x07\x65ventId\x12%\n\x0einstrument_uid\x18\x05 \x01(\tR\rinstrumentUid\x12#\n\rinstrument_id\x18\x06 \x01(\tR\x0cinstrumentId\x12/\n\x13instrument_revision\x18\x07 \x01(\x04R\x12instrumentRevision\x12\x14\n\x05venue\x18\x08 \x01(\tR\x05venue\x12\x16\n\x06market\x18\t \x01(\tR\x06market\x12!\n\x0cproduct_type\x18\n \x01(\tR\x0bproductType\x12#\n\rnative_symbol\x18\x0b \x01(\tR\x0cnativeSymbol\x12\x1a\n\x08provider\x18\x0c \x01(\tR\x08provider\x12\x1b\n\tsource_id\x18\r \x01(\tR\x08sourceId\x12:\n\x0bsource_role\x18\x0e \x01(\x0e\x32\x19.qdl.common.v1.SourceRoleR\nsourceRole\x12\x1f\n\x0blease_epoch\x18\x0f \x01(\x04R\nleaseEpoch\x12/\n\x14source_event_time_ns\x18\x10 \x01(\x03R\x11sourceEventTimeNs\x12$\n\x0ereceived_at_ns\x18\x11 \x01(\x03R\x0creceivedAtNs\x12(\n\x10normalized_at_ns\x18\x12 \x01(\x03R\x0enormalizedAtNs\x12&\n\x0fpublished_at_ns\x18\x13 \x01(\x03R\rpublishedAtNs\x12\'\n\x0fsource_sequence\x18\x14 \x01(\tR\x0esourceSequence\x12-\n\x12partition_sequence\x18\x15 \x01(\x04R\x11partitionSequence\x12-\n\x12normalizer_version\x18\x16 \x01(\tR\x11normalizerVersion\x12\'\n\x0f\x61\x64\x61pter_version\x18\x17 \x01(\tR\x0e\x61\x64\x61pterVersion\x12?\n\rquality_flags\x18\x18 \x03(\x0e\x32\x1a.qdl.common.v1.QualityFlagR\x0cqualityFlags\x12(\n\x10raw_payload_hash\x18\x19 \x01(\x0cR\x0erawPayloadHash\x12%\n\x0e\x63orrelation_id\x18\x1a \x01(\tR\rcorrelationId\x12\'\n\x0f\x63onfig_revision\x18\x1b \x01(\x04R\x0e\x63onfigRevision\x12*\n\x11source_session_id\x18\x1c \x01(\tR\x0fsourceSessionId\x12\x33\n\x15\x63onnection_generation\x18\x1d \x01(\x04R\x14\x63onnectionGeneration\x12-\n\x12\x61uthority_revision\x18\x1e \x01(\x04R\x11\x61uthorityRevision\x12\x30\n\x14partition_plan_epoch\x18\x1f \x01(\x04R\x12partitionPlanEpoch\x12\x34\n\x16\x63\x61nonical_payload_hash\x18 \x01(\x0cR\x14\x63\x61nonicalPayloadHash\x12$\n\x0eraw_capture_id\x18! \x01(\x0cR\x0crawCaptureId\x12\x30\n\x05trade\x18( \x01(\x0b\x32\x18.qdl.marketdata.v2.TradeH\x00R\x05trade\x12\x30\n\x05quote\x18) \x01(\x0b\x32\x18.qdl.marketdata.v2.QuoteH\x00R\x05quote\x12*\n\x03\x62\x61r\x18* \x01(\x0b\x32\x16.qdl.marketdata.v2.BarH\x00R\x03\x62\x61r\x12K\n\rbook_snapshot\x18+ \x01(\x0b\x32$.qdl.marketdata.v2.OrderBookSnapshotH\x00R\x0c\x62ookSnapshot\x12\x42\n\nbook_delta\x18, \x01(\x0b\x32!.qdl.marketdata.v2.OrderBookDeltaH\x00R\tbookDelta\x12\x43\n\x0c\x66unding_rate\x18- \x01(\x0b\x32\x1e.qdl.marketdata.v2.FundingRateH\x00R\x0b\x66undingRate\x12\x46\n\ropen_interest\x18. \x01(\x0b\x32\x1f.qdl.marketdata.v2.OpenInterestH\x00R\x0copenInterest\x12M\n\x10mark_index_price\x18/ \x01(\x0b\x32!.qdl.marketdata.v2.MarkIndexPriceH\x00R\x0emarkIndexPrice\x12\x33\n\x06ticker\x18\x30 \x01(\x0b\x32\x19.qdl.marketdata.v2.TickerH\x00R\x06ticker\x12?\n\nfeed_state\x18\x31 \x01(\x0b\x32\x1e.qdl.quality.v1.FeedStateEventH\x00R\tfeedState\x12G\n\rquality_event\x18\x32 \x01(\x0b\x32 .qdl.quality.v1.DataQualityEventH\x00R\x0cqualityEvent\x12M\n\x10long_short_ratio\x18\x33 \x01(\x0b\x32!.qdl.marketdata.v2.LongShortRatioH\x00R\x0elongShortRatio\x12=\n\ntaker_flow\x18\x34 \x01(\x0b\x32\x1c.qdl.marketdata.v2.TakerFlowH\x00R\ttakerFlow\x12\x30\n\x05\x62\x61sis\x18\x35 \x01(\x0b\x32\x18.qdl.marketdata.v2.BasisH\x00R\x05\x62\x61sis\x12R\n\x11\x63ontract_metadata\x18\x36 \x01(\x0b\x32#.qdl.marketdata.v2.ContractMetadataH\x00R\x10\x63ontractMetadataB\t\n\x07payload*\x85\x01\n\x11TradeIdentityKind\x12#\n\x1fTRADE_IDENTITY_KIND_UNSPECIFIED\x10\x00\x12\x1e\n\x1aTRADE_IDENTITY_KIND_NATIVE\x10\x01\x12+\n\'TRADE_IDENTITY_KIND_DERIVED_RAW_CAPTURE\x10\x02*\x9d\x01\n\x0c\x42\x61rLifecycle\x12\x1d\n\x19\x42\x41R_LIFECYCLE_UNSPECIFIED\x10\x00\x12\x1d\n\x19\x42\x41R_LIFECYCLE_IN_PROGRESS\x10\x01\x12\x17\n\x13\x42\x41R_LIFECYCLE_FINAL\x10\x02\x12\x19\n\x15\x42\x41R_LIFECYCLE_REVISED\x10\x03\x12\x1b\n\x17\x42\x41R_LIFECYCLE_CANCELLED\x10\x04*\xd2\x01\n\x18LongShortRatioPopulation\x12+\n\'LONG_SHORT_RATIO_POPULATION_UNSPECIFIED\x10\x00\x12.\n*LONG_SHORT_RATIO_POPULATION_GLOBAL_ACCOUNT\x10\x01\x12+\n\'LONG_SHORT_RATIO_POPULATION_TOP_ACCOUNT\x10\x02\x12,\n(LONG_SHORT_RATIO_POPULATION_TOP_POSITION\x10\x03*_\n\tBasisKind\x12\x1a\n\x16\x42\x41SIS_KIND_UNSPECIFIED\x10\x00\x12\x1e\n\x1a\x42\x41SIS_KIND_PROVIDER_NATIVE\x10\x01\x12\x16\n\x12\x42\x41SIS_KIND_DERIVED\x10\x02*\xfb\x01\n\nMetricUnit\x12\x1b\n\x17METRIC_UNIT_UNSPECIFIED\x10\x00\x12\x15\n\x11METRIC_UNIT_RATIO\x10\x01\x12\x15\n\x11METRIC_UNIT_PRICE\x10\x02\x12\x1a\n\x16METRIC_UNIT_BASE_ASSET\x10\x03\x12\x1b\n\x17METRIC_UNIT_QUOTE_ASSET\x10\x04\x12\x18\n\x14METRIC_UNIT_CONTRACT\x10\x05\x12\x18\n\x14METRIC_UNIT_NOTIONAL\x10\x06\x12\x17\n\x13METRIC_UNIT_PERCENT\x10\x07\x12\x1c\n\x18METRIC_UNIT_BASIS_POINTS\x10\x08\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'qdl.marketdata.v2.market_data_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None - _globals['_TRADEIDENTITYKIND']._serialized_start=8004 - _globals['_TRADEIDENTITYKIND']._serialized_end=8137 - _globals['_BARLIFECYCLE']._serialized_start=8140 - _globals['_BARLIFECYCLE']._serialized_end=8297 - _globals['_LONGSHORTRATIOPOPULATION']._serialized_start=8300 - _globals['_LONGSHORTRATIOPOPULATION']._serialized_end=8510 - _globals['_BASISKIND']._serialized_start=8512 - _globals['_BASISKIND']._serialized_end=8607 - _globals['_METRICUNIT']._serialized_start=8610 - _globals['_METRICUNIT']._serialized_end=8861 + _globals['_TRADEIDENTITYKIND']._serialized_start=8053 + _globals['_TRADEIDENTITYKIND']._serialized_end=8186 + _globals['_BARLIFECYCLE']._serialized_start=8189 + _globals['_BARLIFECYCLE']._serialized_end=8346 + _globals['_LONGSHORTRATIOPOPULATION']._serialized_start=8349 + _globals['_LONGSHORTRATIOPOPULATION']._serialized_end=8559 + _globals['_BASISKIND']._serialized_start=8561 + _globals['_BASISKIND']._serialized_end=8656 + _globals['_METRICUNIT']._serialized_start=8659 + _globals['_METRICUNIT']._serialized_end=8910 _globals['_TRADE']._serialized_start=117 _globals['_TRADE']._serialized_end=558 _globals['_QUOTE']._serialized_start=561 @@ -52,25 +52,25 @@ _globals['_BOOKLEVEL']._serialized_start=1862 _globals['_BOOKLEVEL']._serialized_end=2125 _globals['_ORDERBOOKSNAPSHOT']._serialized_start=2128 - _globals['_ORDERBOOKSNAPSHOT']._serialized_end=2408 - _globals['_ORDERBOOKDELTA']._serialized_start=2411 - _globals['_ORDERBOOKDELTA']._serialized_end=2764 - _globals['_FUNDINGRATE']._serialized_start=2767 - _globals['_FUNDINGRATE']._serialized_end=2948 - _globals['_OPENINTEREST']._serialized_start=2951 - _globals['_OPENINTEREST']._serialized_end=3208 - _globals['_MARKINDEXPRICE']._serialized_start=3211 - _globals['_MARKINDEXPRICE']._serialized_end=3349 - _globals['_LONGSHORTRATIO']._serialized_start=3352 - _globals['_LONGSHORTRATIO']._serialized_end=3745 - _globals['_TAKERFLOW']._serialized_start=3748 - _globals['_TAKERFLOW']._serialized_end=4059 - _globals['_BASIS']._serialized_start=4062 - _globals['_BASIS']._serialized_end=4516 - _globals['_CONTRACTMETADATA']._serialized_start=4519 - _globals['_CONTRACTMETADATA']._serialized_end=5052 - _globals['_TICKER']._serialized_start=5055 - _globals['_TICKER']._serialized_end=5655 - _globals['_EVENTENVELOPE']._serialized_start=5658 - _globals['_EVENTENVELOPE']._serialized_end=8001 + _globals['_ORDERBOOKSNAPSHOT']._serialized_end=2457 + _globals['_ORDERBOOKDELTA']._serialized_start=2460 + _globals['_ORDERBOOKDELTA']._serialized_end=2813 + _globals['_FUNDINGRATE']._serialized_start=2816 + _globals['_FUNDINGRATE']._serialized_end=2997 + _globals['_OPENINTEREST']._serialized_start=3000 + _globals['_OPENINTEREST']._serialized_end=3257 + _globals['_MARKINDEXPRICE']._serialized_start=3260 + _globals['_MARKINDEXPRICE']._serialized_end=3398 + _globals['_LONGSHORTRATIO']._serialized_start=3401 + _globals['_LONGSHORTRATIO']._serialized_end=3794 + _globals['_TAKERFLOW']._serialized_start=3797 + _globals['_TAKERFLOW']._serialized_end=4108 + _globals['_BASIS']._serialized_start=4111 + _globals['_BASIS']._serialized_end=4565 + _globals['_CONTRACTMETADATA']._serialized_start=4568 + _globals['_CONTRACTMETADATA']._serialized_end=5101 + _globals['_TICKER']._serialized_start=5104 + _globals['_TICKER']._serialized_end=5704 + _globals['_EVENTENVELOPE']._serialized_start=5707 + _globals['_EVENTENVELOPE']._serialized_end=8050 # @@protoc_insertion_point(module_scope) diff --git a/generated/rust/qdl/marketdata/v2/qdl.marketdata.v2.rs b/generated/rust/qdl/marketdata/v2/qdl.marketdata.v2.rs index 95a9acc3..742648fc 100644 --- a/generated/rust/qdl/marketdata/v2/qdl.marketdata.v2.rs +++ b/generated/rust/qdl/marketdata/v2/qdl.marketdata.v2.rs @@ -109,6 +109,13 @@ pub struct OrderBookSnapshot { /// every provider level. Clients must not assume omitted levels are absent. #[prost(bool, tag="7")] pub truncated: bool, + /// Additive (IDR-040 fix 2). Zero/absent: no quiet proof; the book is current + /// only as of `source_event_time_ns`. Positive: local receipt time of in-band + /// transport evidence on the same provider connection, after every earlier + /// frame of this book was applied without a gap. It never re-dates + /// `source_event_time_ns` and is set only on a quiet-proof snapshot. + #[prost(int64, tag="8")] + pub proof_observed_at_ns: i64, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct OrderBookDelta { diff --git a/qdl/query/results.py b/qdl/query/results.py index e9dd9972..4772d5cd 100644 --- a/qdl/query/results.py +++ b/qdl/query/results.py @@ -63,8 +63,17 @@ class QualityMetadata: event_recency_state: str = "LIVE" provider_session_state: str = "NOT_APPLICABLE" provider_session_liveness_ms: int | None = None + # IDR-040 fix 2 (additive): receipt time of the in-band quiet proof a + # verified BOOK_SNAPSHOT carries. It never changes ``freshness_ms``/state. + proof_observed_at_ns: int | None = None def __post_init__(self) -> None: + if self.proof_observed_at_ns is not None and ( + isinstance(self.proof_observed_at_ns, bool) + or not isinstance(self.proof_observed_at_ns, int) + or self.proof_observed_at_ns <= 0 + ): + raise ValueError("quiet proof time must be a positive integer") if self.freshness_ms < 0: raise ValueError("freshness_ms cannot be negative") if not self.state.strip() or not self.policy_id.strip(): diff --git a/qdl/query/service.py b/qdl/query/service.py index 1ce21f8f..4e348bdd 100644 --- a/qdl/query/service.py +++ b/qdl/query/service.py @@ -1985,7 +1985,7 @@ def _refusal_diagnostics( ) -> dict[str, Any]: """The quality this refusal was decided on, taken now (not re-read).""" - return { + diagnostics = { "evaluated_at_ns": self._clock_ns(), "state": quality.state, "freshness_ms": quality.freshness_ms, @@ -2001,6 +2001,11 @@ def _refusal_diagnostics( "observed_at_ns": item.observed_at_ns if item is not None else None, "received_at_ns": (item.received_at_ns or None) if item is not None else None, } + # Additive (IDR-040 fix 2): only when the refused book carries a proof, + # so a strict older reader never meets the key on any other refusal. + if quality.proof_observed_at_ns is not None: + diagnostics["proof_observed_at_ns"] = quality.proof_observed_at_ns + return diagnostics def _with_execution_eligibility( self, diff --git a/qdl/runtime/stable_source.py b/qdl/runtime/stable_source.py index ffbcaf75..88ad9918 100644 --- a/qdl/runtime/stable_source.py +++ b/qdl/runtime/stable_source.py @@ -1092,6 +1092,10 @@ def _quality( ) if book_unverified: flags = flags + ("BOOK_SEQUENCE_UNVERIFIED",) + # IDR-040 fix 2: reported, never folded into freshness or state. + proof_observed_at_ns = None + if payload_name == "book_snapshot" and not book_unverified: + proof_observed_at_ns = int(envelope.book_snapshot.proof_observed_at_ns) or None now_ns = self._clock_ns() decision = evaluate_binding_quality( BindingQualityInput( @@ -1165,6 +1169,7 @@ def _quality( event_recency_state=decision.event_recency_state, provider_session_state=decision.provider_session_state, provider_session_liveness_ms=decision.provider_session_liveness_ms, + proof_observed_at_ns=proof_observed_at_ns, ) def _gaps( diff --git a/qdl_sdk/CHANGELOG.md b/qdl_sdk/CHANGELOG.md index 0a5e00e7..46dd8fa9 100644 --- a/qdl_sdk/CHANGELOG.md +++ b/qdl_sdk/CHANGELOG.md @@ -12,6 +12,15 @@ rows and set by `market_data_view_from_stream` from the delivered envelope, so a snapshot of an event and its stream delivery carry the same value (IDR-082). Additive response field; 2.1.0 readers ignore it, existing fields are unchanged. +IDR-040 fix 2: `QualityView.proof_observed_at_ns` and +`ProblemDiagnostics.proof_observed_at_ns` (optional, positive int, BOOK_SNAPSHOT +only; also `OrderBookSnapshot.proof_observed_at_ns = 8` in the market-data +proto). It is the local receipt time of in-band transport evidence that a quiet, +sequence-verified book was still current; it never re-dates `observed_at_ns` +and never changes `freshness_ms`, `state` or `execution_eligible`. The server +emits the key only when a proof exists (absent otherwise), so 2.0.x strict +readers are unaffected until quiet proofs are enabled; 2.1.0+ readers accept it. + ## 2.1.0 Response models, including nested market/reference payloads and error diagnostics, diff --git a/qdl_sdk/models.py b/qdl_sdk/models.py index 335a7299..5c5a88cb 100644 --- a/qdl_sdk/models.py +++ b/qdl_sdk/models.py @@ -175,6 +175,16 @@ class ProblemDiagnostics(ResponseModel): watermark_offset: int | None = Field(default=None, ge=0) observed_at_ns: int | None = Field(default=None, ge=0) received_at_ns: int | None = Field(default=None, ge=0) + # Additive (IDR-040 fix 2): see ``QualityView.proof_observed_at_ns``. + proof_observed_at_ns: int | None = Field(default=None, gt=0) + + @model_serializer(mode="wrap") + def serialize_diagnostics(self, handler): + result = handler(self) + # Strict older SDKs reject a new key; it is emitted only when present. + if self.proof_observed_at_ns is None: + result.pop("proof_observed_at_ns", None) + return result class ProblemDetails(ResponseModel): @@ -284,6 +294,20 @@ class QualityView(ResponseModel): execution_eligible: bool policy_id: str = Field(min_length=1, max_length=200) flags: list[str] + # Additive (IDR-040 fix 2), BOOK_SNAPSHOT only. Local receipt time (ns) of + # in-band transport evidence proving the verified book was still current + # although no provider frame changed it. It never re-dates the event: + # ``observed_at_ns``/``freshness_ms`` stay the provider event's, and + # ``state``/``execution_eligible`` are unchanged by it. Absent = no proof. + proof_observed_at_ns: int | None = Field(default=None, gt=0) + + @model_serializer(mode="wrap") + def serialize_quality(self, handler): + result = handler(self) + # Strict older SDKs reject a new key; it is emitted only when present. + if self.proof_observed_at_ns is None: + result.pop("proof_observed_at_ns", None) + return result class ContractView(ResponseModel): diff --git a/qdl_sdk/projection.py b/qdl_sdk/projection.py index 432e77d4..6a182567 100644 --- a/qdl_sdk/projection.py +++ b/qdl_sdk/projection.py @@ -534,6 +534,12 @@ def market_data_view_from_stream( ), "policy_id": requirement.source_policy_id, "flags": flags, + # IDR-040 fix 2: reported only; never changes state/freshness. + "proof_observed_at_ns": ( + int(envelope.book_snapshot.proof_observed_at_ns) or None + if feed is Feed.BOOK_SNAPSHOT and not book_unverified + else None + ), }, "contract": { **template.contract.model_dump(mode="json"), diff --git a/rust/qdl-core/src/canonical.rs b/rust/qdl-core/src/canonical.rs index 3de4c189..c4b3406b 100644 --- a/rust/qdl-core/src/canonical.rs +++ b/rust/qdl-core/src/canonical.rs @@ -826,6 +826,7 @@ fn canonicalize_deribit_fixture(fixture: &TradeFixture) -> Result { + return Err("L2 quiet proof time must be positive".into()); + } + value => value, + }; + // A quiet proof repeats the book's sequence and provider time; only its + // receipt time makes the observation distinct, so it names the event. + let materialization_discriminator = match proof_observed_at_ns { + Some(value) => Some(format!("proof:{value}")), + None => transition + .materialized_snapshot_at_ms + .map(|value| format!("materialized:{value}")), + }; let mut envelope = base_envelope_with_discriminator( fixture, "book_snapshot", @@ -896,6 +908,7 @@ fn l2_ready_snapshot( book_generation: view.generation, sequence_verified: true, truncated: view.truncated, + proof_observed_at_ns: proof_observed_at_ns.unwrap_or(0), })); set_payload_hash(&mut envelope)?; Ok(envelope) diff --git a/rust/qdl-core/src/l2_adapter.rs b/rust/qdl-core/src/l2_adapter.rs index 972618c6..8083556e 100644 --- a/rust/qdl-core/src/l2_adapter.rs +++ b/rust/qdl-core/src/l2_adapter.rs @@ -54,6 +54,11 @@ pub struct BookTransition { /// an already verified current view. It keeps native sequence semantics /// intact while making the periodic observation idempotency-distinct. pub materialized_snapshot_at_ms: Option, + /// Set only on a quiet-proof snapshot (IDR-040 fix 2): the local receipt + /// time of in-band transport evidence on the connection that owns this + /// book, ordered after every earlier frame of the book. It leaves + /// `observed_at_ms` (the provider event time) untouched. + pub proof_observed_at_ns: Option, } impl BookTransition { @@ -461,6 +466,7 @@ impl L2BookAdapter { updates: metadata.updates, view: self.core.view(), materialized_snapshot_at_ms: None, + proof_observed_at_ns: None, } } diff --git a/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs b/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs index 31b169b0..4f8cffe1 100644 --- a/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs +++ b/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs @@ -37,7 +37,9 @@ use qdl_kafka::{ shutdown_signal, FencedKafkaSink, KafkaEventSource, KafkaTlsConfig, KafkaTransportConfig, KafkaTransportError, PendingKafkaAppend, }; -use qdl_realtime_core::L2_RESNAPSHOT_REQUIRED_V1; +use qdl_realtime_core::{ + L2_RESNAPSHOT_REQUIRED_V1, L2_TRANSPORT_LIVENESS_KEY, L2_TRANSPORT_LIVENESS_VERSION, +}; use qdl_venue_core::authority::{AuthorityMode, AuthorityRecord, PublicationContext, SinkTarget}; use qdl_venue_core::backpressure::DeliveryClass; use serde::{Deserialize, Serialize}; @@ -243,6 +245,11 @@ struct IngestorConfig { #[serde(default = "default_max_subscriptions_per_connection")] max_subscriptions_per_connection: usize, latest_state_flush_ms: u64, + /// IDR-040 fix 2: cadence of the in-band transport liveness marker for + /// quiet Binance BOOK bindings. Absent or zero (the default) emits none. + /// The realtime core that reads the marker must be deployed first. + #[serde(default)] + book_liveness_proof_ms: u64, authority: AuthorityRecord, bindings: Vec, #[serde(default)] @@ -393,6 +400,8 @@ impl IngestorConfig { || self.max_subscriptions_per_connection > 1_024 || self.latest_state_flush_ms == 0 || self.latest_state_flush_ms > 1_000 + || (self.book_liveness_proof_ms != 0 + && !(250..=5_000).contains(&self.book_liveness_proof_ms)) || self.bindings.is_empty() { return Err( @@ -1514,6 +1523,105 @@ async fn book_resync_requested( Ok(requested) } +/// In-band transport evidence for the books of one Binance connection +/// (IDR-040 fix 2). +/// +/// Binance pushes nothing for a book that does not change, so a quiet book is +/// otherwise indistinguishable from a stalled one. TCP delivers a connection's +/// messages in order: when any message (a data frame or the pong to our ping) +/// is read at `t`, every frame the venue sent before it on this connection has +/// been read and enqueued. A marker stamped `t` and enqueued into a book's own +/// raw partition therefore follows all of that book's earlier frames. It is +/// emitted only for a book with no frame for one interval, never for a book +/// that is not on this connection, and never with evidence older than one +/// interval (the connection is pinged instead). +struct BookLivenessProof { + interval_ns: i64, + last_transport_at_ns: i64, + last_frame_at_ns: HashMap, + last_marker_at_ns: HashMap, +} + +impl BookLivenessProof { + fn new( + interval: Duration, + bindings: &HashMap, + connected_at_ns: i64, + ) -> Self { + let interval_ns = i64::try_from(interval.as_nanos()).unwrap_or(i64::MAX); + let books = bindings + .values() + .filter(|binding| binding.feed == RawFeed::Book) + .map(|binding| (binding.native_channel.clone(), connected_at_ns)); + Self { + interval_ns, + last_transport_at_ns: connected_at_ns, + last_frame_at_ns: books.collect(), + last_marker_at_ns: HashMap::new(), + } + } + + fn transport(&mut self, at_ns: i64) { + self.last_transport_at_ns = self.last_transport_at_ns.max(at_ns); + } + + fn frame(&mut self, channel: &str, at_ns: i64) { + if let Some(last) = self.last_frame_at_ns.get_mut(channel) { + *last = (*last).max(at_ns); + } + } + + /// Markers due now, and whether the connection should be pinged so the + /// next tick has evidence newer than half an interval. + fn due( + &mut self, + bindings: &HashMap, + session_id: &str, + generation: u64, + now_ns: i64, + ) -> (Vec, bool) { + let evidence = self.last_transport_at_ns; + let age = now_ns.saturating_sub(evidence); + let ping = age >= self.interval_ns / 2; + if age > self.interval_ns { + return (vec![], ping); + } + let mut channels = self.last_frame_at_ns.keys().cloned().collect::>(); + channels.sort(); + let mut frames = Vec::new(); + for channel in channels { + let quiet = self + .last_frame_at_ns + .get(&channel) + .is_some_and(|last| evidence.saturating_sub(*last) >= self.interval_ns); + let newer = self + .last_marker_at_ns + .get(&channel) + .map_or(true, |last| evidence > *last); + let Some(binding) = bindings.get(&channel) else { + continue; + }; + if !quiet || !newer { + continue; + } + let marker = json!({ + L2_TRANSPORT_LIVENESS_KEY: L2_TRANSPORT_LIVENESS_VERSION, + "stream": channel, + }); + self.last_marker_at_ns.insert(channel.clone(), evidence); + frames.push(PendingRawFrame { + binding: binding.clone(), + session_id: session_id.into(), + generation, + raw_frame: marker.to_string().into_bytes(), + received_at_ns: evidence, + transport_protocol: TransportProtocol::Websocket, + }); + } + (frames, ping) + } +} + async fn run_binance_connection( config: Arc, route: BinanceRoute, @@ -1568,6 +1676,8 @@ async fn run_binance_connection( } None => None, }; + let liveness_proof_interval = (snapshot_period.is_some() && config.book_liveness_proof_ms > 0) + .then(|| Duration::from_millis(config.book_liveness_proof_ms)); let backoff = BackoffPolicy { initial_ms: 250, maximum_ms: 30_000, @@ -1628,7 +1738,16 @@ async fn run_binance_connection( now_ns()? ); let mut liveness = SessionLivenessWriter::new(&config, &lane)?; - liveness.live(&session_id, generation, now_ns()?)?; + let connected_at_ns = now_ns()?; + liveness.live(&session_id, generation, connected_at_ns)?; + let mut book_proof = liveness_proof_interval + .map(|interval| BookLivenessProof::new(interval, &bindings, connected_at_ns)); + let mut book_proof_tick = liveness_proof_interval.map(tokio::time::interval); + if let Some(tick) = book_proof_tick.as_mut() { + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + tick.tick().await; + } + let mut ping_due = false; let mut inflight = FuturesUnordered::::new(); let mut latest = LatestStateBuffer::default(); let mut latest_tick = @@ -1727,6 +1846,10 @@ async fn run_binance_connection( } } } + if ping_due { + ping_due = false; + socket.send(Message::Ping(Vec::new())).await?; + } let read = tokio::time::timeout( Duration::from_secs(config.heartbeat_seconds), socket.next(), @@ -1734,6 +1857,42 @@ async fn run_binance_connection( tokio::pin!(read); // Control backlog must not take priority over socket and publish progress. let outcome = tokio::select! { + _ = async { + match book_proof_tick.as_mut() { + Some(tick) => { + tick.tick().await; + } + None => std::future::pending::<()>().await, + } + }, if book_proof_tick.is_some() => { + let Some(proof) = book_proof.as_mut() else { + continue; + }; + let (markers, ping) = + proof.due(&bindings, &session_id, generation, now_ns()?); + ping_due = ping; + for marker in markers { + if !publish_pending_frame( + marker, + &mut PendingPublishWindow { + publisher: &publisher, + latest: &mut latest, + inflight: &mut inflight, + accepted: &accepted, + coalesced_latest: &coalesced_latest, + max_events: config.max_events, + max_inflight: config.max_inflight_publishes, + stopped: &stopped, + }, + ) + .await? + { + exhausted = true; + break; + } + } + continue; + } requested = book_resync_requested(&mut resync_listener, &config, &bindings, &session_id, generation) => { if requested? { eprintln!("{{\"event\":\"qdl_native_book_resync_requested\",\"session\":\"{}\",\"generation\":{}}}", session_id, generation); @@ -1821,7 +1980,11 @@ async fn run_binance_connection( let message = match outcome.expect("Binance read outcome is present") { Ok(Some(Ok(message))) => { if !matches!(message, Message::Close(_)) { - liveness.live(&session_id, generation, now_ns()?)?; + let read_at_ns = now_ns()?; + liveness.live(&session_id, generation, read_at_ns)?; + if let Some(proof) = book_proof.as_mut() { + proof.transport(read_at_ns); + } } message } @@ -1866,12 +2029,20 @@ async fn run_binance_connection( .await { Ok(Some(Ok(Message::Pong(_)))) => { - liveness.live(&session_id, generation, now_ns()?)?; + let read_at_ns = now_ns()?; + liveness.live(&session_id, generation, read_at_ns)?; + if let Some(proof) = book_proof.as_mut() { + proof.transport(read_at_ns); + } continue; } Ok(Some(Ok(message))) => { if !matches!(message, Message::Close(_)) { - liveness.live(&session_id, generation, now_ns()?)?; + let read_at_ns = now_ns()?; + liveness.live(&session_id, generation, read_at_ns)?; + if let Some(proof) = book_proof.as_mut() { + proof.transport(read_at_ns); + } } message } @@ -1916,6 +2087,9 @@ async fn run_binance_connection( raw_text, now_ns()?, )?; + if let Some(proof) = book_proof.as_mut() { + proof.frame(&frame.binding.native_channel, frame.received_at_ns); + } if !publish_pending_frame( frame, &mut PendingPublishWindow { @@ -2625,9 +2799,11 @@ mod tests { book_snapshot_renewal_period, feed_lane_name, next_connection_generation, partition_binance_bindings, partition_binance_route, partition_bindings, partition_okx_bindings, pending_binance_frame, pending_okx_frame, AuthorityMode, - BinanceRoute, BookSessionBootstrap, DeliveryClass, KafkaTransportError, LatestStateBuffer, - PendingRawFrame, ProviderRuntime, RawBinding, RawFeed, SessionLivenessWriter, + BinanceRoute, BookLivenessProof, BookSessionBootstrap, DeliveryClass, KafkaTransportError, + LatestStateBuffer, PendingRawFrame, ProviderRuntime, RawBinding, RawFeed, + SessionLivenessWriter, TransportProtocol, }; + use serde_json::{json, Value}; use std::collections::HashMap; use std::fs; use std::path::PathBuf; @@ -2669,6 +2845,83 @@ mod tests { } } + fn proof_bindings() -> HashMap { + let book = binding(RawFeed::Book, DeliveryClass::Lossless); + let mut other = book.clone(); + other.native_symbol = "ETHUSDT".into(); + other.native_channel = "ethusdt@depth@100ms".into(); + let trade = binding(RawFeed::Trade, DeliveryClass::Lossless); + [book, other, trade] + .into_iter() + .map(|binding| (binding.native_channel.clone(), binding)) + .collect() + } + + #[test] + fn book_liveness_marker_needs_a_quiet_book_and_fresh_in_band_evidence() { + const MS: i64 = 1_000_000; + let bindings = proof_bindings(); + let start = 1_000_000 * MS; + let mut proof = BookLivenessProof::new(Duration::from_millis(1_000), &bindings, start); + + // Nothing is quiet for a full interval yet. + let (markers, ping) = proof.due(&bindings, "lane-1-1", 1, start + 400 * MS); + assert!(markers.is_empty()); + assert!(!ping); + + // BTC keeps trading; ETH stays quiet while frames keep arriving on the connection. + proof.frame("btcusdt@depth@100ms", start + 900 * MS); + proof.transport(start + 1_200 * MS); + let (markers, ping) = proof.due(&bindings, "lane-1-1", 1, start + 1_300 * MS); + assert!(!ping); + assert_eq!(markers.len(), 1, "only the quiet book gets a marker"); + let marker = &markers[0]; + assert_eq!(marker.binding.native_channel, "ethusdt@depth@100ms"); + assert_eq!( + marker.received_at_ns, + start + 1_200 * MS, + "stamped with the read, not the tick" + ); + assert_eq!(marker.transport_protocol, TransportProtocol::Websocket); + assert_eq!( + (marker.session_id.as_str(), marker.generation), + ("lane-1-1", 1) + ); + let body: Value = serde_json::from_slice(&marker.raw_frame).unwrap(); + assert_eq!( + body, + json!({"qdl_transport_liveness": "v1", "stream": "ethusdt@depth@100ms"}) + ); + + // No new evidence: no repeated marker; the trade binding never gets one. + let (markers, _) = proof.due(&bindings, "lane-1-1", 1, start + 1_400 * MS); + assert!(markers.is_empty()); + + // An idle connection asks for a ping, and stale evidence emits nothing. + let (markers, ping) = proof.due(&bindings, "lane-1-1", 1, start + 1_800 * MS); + assert!(markers.is_empty()); + assert!(ping); + let (markers, ping) = proof.due(&bindings, "lane-1-1", 1, start + 2_300 * MS); + assert!( + markers.is_empty(), + "evidence older than one interval proves nothing" + ); + assert!(ping); + + // The pong is new evidence: both books are now quiet for an interval. + proof.transport(start + 2_350 * MS); + let (markers, _) = proof.due(&bindings, "lane-1-1", 1, start + 2_400 * MS); + let mut channels = markers + .iter() + .map(|marker| marker.binding.native_channel.as_str()) + .collect::>(); + channels.sort(); + assert_eq!(channels, ["btcusdt@depth@100ms", "ethusdt@depth@100ms"]); + assert!(markers + .iter() + .all(|marker| marker.received_at_ns == start + 2_350 * MS)); + } + #[test] fn book_delivery_bound_is_exact_and_never_applies_to_non_book_frames() { let mut book = binding(RawFeed::Book, DeliveryClass::Lossless); @@ -3337,6 +3590,7 @@ mod resync_tests { max_inflight_publishes: 16, max_subscriptions_per_connection: 100, latest_state_flush_ms: 50, + book_liveness_proof_ms: 0, authority: AuthorityRecord { schema: "qdl.authority-record.v1".into(), slice_id: "unit".into(), diff --git a/rust/qdl-realtime-core/src/lib.rs b/rust/qdl-realtime-core/src/lib.rs index 95246278..86e7ab48 100644 --- a/rust/qdl-realtime-core/src/lib.rs +++ b/rust/qdl-realtime-core/src/lib.rs @@ -9,11 +9,11 @@ use qdl_contracts::qdl::provider::v1::{ QuarantineReason, QuarantineRecord, RawProviderEnvelope, TransportProtocol, }; use qdl_core::canonical::{ - canonicalize_l2_invalidation, canonicalize_l2_publication, canonicalize_l2_recovery_snapshot, - canonicalize_mark_index, canonicalize_trade, MarkIndexInput, + canonicalize_l2_invalidation, canonicalize_l2_publication, canonicalize_l2_ready_snapshot, + canonicalize_l2_recovery_snapshot, canonicalize_mark_index, canonicalize_trade, MarkIndexInput, TradeContext, TradeFixture, }; -use qdl_core::l2_adapter::{BookPublication, BookTransition, L2BookAdapter}; +use qdl_core::l2_adapter::{BookPublication, BookTransition, BookTransitionKind, L2BookAdapter}; use qdl_core::l2_book::{BookIdentity, BookOutcome, BookStatus, ExactDecimal}; use qdl_core::okx::expand_data_frame; use qdl_core::transport::DurableRecord; @@ -402,6 +402,50 @@ pub const L2_RESNAPSHOT_REQUIRED_V1: &str = "L2_RESNAPSHOT_REQUIRED_V1"; // This is not a freshness proof: only verified provider transitions can emit. const L2_TOP_CHANGE_MIN_INTERVAL_MS: i64 = 100; +/// In-band transport liveness marker for one L2 book (IDR-040 fix 2). +/// +/// The native ingestor writes `{"qdl_transport_liveness":"v1","stream":}` into the book's own raw partition, behind every provider frame of +/// that book it had read, stamped with the receipt time of the newest message +/// (data frame or pong) read on the same WebSocket. TCP delivers a connection's +/// frames in order, so at that receipt time every frame the venue sent earlier +/// on the connection had been read, and the partition order puts all of this +/// book's frames ahead of the marker. A marker carries no market data and is +/// never a reason to start, change or invalidate a book. +pub const L2_TRANSPORT_LIVENESS_KEY: &str = "qdl_transport_liveness"; +pub const L2_TRANSPORT_LIVENESS_VERSION: &str = "v1"; +const L2_LIVENESS_UNPROVEN: &str = "TRANSPORT_LIVENESS_UNPROVEN"; +const L2_LIVENESS_NOT_DUE: &str = "TRANSPORT_LIVENESS_NOT_DUE"; + +/// Provider time and receipt facts for the book's current verified view. The +/// quiet proof re-publishes this view with its original provider time. +#[derive(Clone, Copy, Debug)] +struct L2ViewTimes { + generation: u64, + observed_at_ms: i64, + last_snapshot_received_ns: i64, +} + +fn l2_transport_liveness(payload: &Value) -> Option> { + let object = payload.as_object()?; + object.get(L2_TRANSPORT_LIVENESS_KEY)?; + if object.len() != 2 + || object + .get(L2_TRANSPORT_LIVENESS_KEY) + .and_then(Value::as_str) + != Some(L2_TRANSPORT_LIVENESS_VERSION) + { + return Some(Err("L2 transport liveness marker schema is invalid")); + } + Some( + object + .get("stream") + .and_then(Value::as_str) + .filter(|stream| !stream.is_empty()) + .ok_or("L2 transport liveness marker has no stream"), + ) +} + struct L2SnapshotObservation { observed_at_ms: i64, generation: u64, @@ -426,6 +470,7 @@ pub struct RealtimeCore { bindings: BTreeMap, l2_adapters: BTreeMap, l2_snapshot_observations: BTreeMap, + l2_view_times: BTreeMap, l2_invalidated: BTreeMap, mark_index_pairs: BTreeMap, partition_sequences: BTreeMap, @@ -475,6 +520,7 @@ impl RealtimeCore { bindings, l2_adapters, l2_snapshot_observations: BTreeMap::new(), + l2_view_times: BTreeMap::new(), l2_invalidated: BTreeMap::new(), mark_index_pairs: BTreeMap::new(), partition_sequences: BTreeMap::new(), @@ -1115,6 +1161,8 @@ impl RealtimeCore { processing_at_ns: i64, transport_offset: Option, ) -> ProcessBatch { + // A transport liveness marker never starts, changes or fences a book. + let liveness_marker = l2_transport_liveness(&payload).is_some(); let mut batch = self.process_l2_inner(binding_key, binding, &raw, payload, processing_at_ns, transport_offset); let Some(adapter) = self.l2_adapters.get(binding_key) else { return batch; }; @@ -1127,7 +1175,12 @@ impl RealtimeCore { // A superseded/foreign frame cannot invalidate the current owner. if adapter.core().generation() != raw.connection_generation || self.l2_sessions.get(binding_key) != Some(&raw.source_session_id) - || matches!(batch.filtered_outcome, Some("IGNORED_STALE_GENERATION" | "IDENTITY_MISMATCH")) + || matches!( + batch.filtered_outcome, + Some("IGNORED_STALE_GENERATION" | "IDENTITY_MISMATCH") + | Some(L2_LIVENESS_UNPROVEN | L2_LIVENESS_NOT_DUE) + ) + || liveness_marker { return batch; } let scope = (raw.source_session_id.clone(), raw.connection_generation); if self.l2_invalidated.get(binding_key) == Some(&scope) { return batch; } @@ -1161,6 +1214,18 @@ impl RealtimeCore { let Some(l2) = binding.l2.as_ref() else { unreachable!("process_l2 only accepts an L2 binding") }; + if let Some(marker) = l2_transport_liveness(&payload) { + let stream = marker.map(str::to_owned); + return self.process_l2_liveness( + binding_key, + binding, + l2, + raw, + stream, + payload, + transport_offset, + ); + } // Processing backlog is not a provider continuity failure. Replay every // validated book frame losslessly with its original source/receipt times; // readers still enforce freshness. Dropping an aged delta and clearing @@ -1344,6 +1409,7 @@ impl RealtimeCore { } transition }; + self.record_l2_view_time(binding_key, &transition, raw); let fixture = l2_fixture(binding, raw, fixture_payload.clone(), partition_sequence); let publication = if self.l2_invalidated.contains_key(binding_key) @@ -1434,6 +1500,12 @@ impl RealtimeCore { self.l2_snapshot_observations .insert(binding_key.to_owned(), observation); } + if let Some(times) = self.l2_view_times.get_mut(binding_key) { + if times.generation == transition.generation { + times.last_snapshot_received_ns = + times.last_snapshot_received_ns.max(raw.received_at_ns); + } + } } durable.push(canonical_record( &self.config.canonical_stream, @@ -1450,6 +1522,202 @@ impl RealtimeCore { } } + /// Keep the provider time of the current verified view. Only transitions + /// that leave a readable, sequence-verified view count; a transition that + /// leaves the book not READY (gap, resync, bootstrap) forgets the times, so + /// a quiet proof can never outlive the view it describes. + fn record_l2_view_time( + &mut self, + binding_key: &str, + transition: &BookTransition, + raw: &RawProviderEnvelope, + ) { + let applied = transition.sequence_verified() + && matches!( + transition.outcome, + BookOutcome::DeltaApplied | BookOutcome::SnapshotApplied | BookOutcome::Keepalive + ); + if !applied { + if transition.status != BookStatus::Ready { + self.l2_view_times.remove(binding_key); + } + return; + } + let Some(observed_at_ms) = l2_observed_at_ms(transition, raw) else { + return; + }; + let fresh = L2ViewTimes { + generation: transition.generation, + observed_at_ms, + last_snapshot_received_ns: 0, + }; + let entry = self + .l2_view_times + .entry(binding_key.to_owned()) + .or_insert(fresh); + if entry.generation != transition.generation { + *entry = fresh; + } + entry.observed_at_ms = entry.observed_at_ms.max(observed_at_ms); + } + + /// IDR-040 fix 2: a quiet book re-publishes its current verified view with + /// the original provider time and `proof_observed_at_ns` set to the marker's + /// receipt time. Every condition fails closed to "no proof"; nothing here + /// can start, change, reset or invalidate the book. + #[allow(clippy::too_many_arguments)] + fn process_l2_liveness( + &mut self, + binding_key: &str, + binding: &CoreBinding, + l2: &L2Binding, + raw: &RawProviderEnvelope, + stream: Result, + payload: Value, + transport_offset: Option, + ) -> ProcessBatch { + let materialized_at_ns = raw.received_at_ns; + let stream = match stream { + Ok(value) => value, + Err(summary) => { + return self.quarantine( + raw, + QuarantineReason::SemanticInvalid, + summary, + materialized_at_ns, + ) + } + }; + if stream != raw.native_channel + || TransportProtocol::try_from(raw.transport_protocol) + != Ok(TransportProtocol::Websocket) + { + return self.quarantine( + raw, + QuarantineReason::SemanticInvalid, + "L2 transport liveness marker is not in-band for this book", + materialized_at_ns, + ); + } + let unproven = |outcome: &'static str| ProcessBatch { + canonical: vec![], + quarantines: vec![], + duplicates: 0, + filtered: 1, + filtered_outcome: Some(outcome), + }; + let Some(adapter) = self.l2_adapters.get(binding_key) else { + return unproven(L2_LIVENESS_UNPROVEN); + }; + let core = adapter.core(); + // The marker must come from the very connection that owns the book: a + // byte-identical session and the book's own generation. A different + // lane or a newer connection proves nothing about this view. + if self.l2_sessions.get(binding_key) != Some(&raw.source_session_id) + || core.generation() != raw.connection_generation + || core.status() != BookStatus::Ready + || self.l2_invalidated.contains_key(binding_key) + { + return unproven(L2_LIVENESS_UNPROVEN); + } + let Some(view) = core.view() else { + return unproven(L2_LIVENESS_UNPROVEN); + }; + let Some(times) = self + .l2_view_times + .get(binding_key) + .copied() + .filter(|times| times.generation == core.generation()) + else { + return unproven(L2_LIVENESS_UNPROVEN); + }; + // Quiet only: a book that published a snapshot within its cadence (on + // the same receipt clock) already carries current evidence. + let refresh_ns = i64::try_from(l2.materialized_snapshot_interval_ms()) + .ok() + .and_then(|value| value.checked_mul(1_000_000)) + .unwrap_or(i64::MAX); + if raw.received_at_ns <= times.last_snapshot_received_ns + || raw + .received_at_ns + .saturating_sub(times.last_snapshot_received_ns) + < refresh_ns + { + return unproven(L2_LIVENESS_NOT_DUE); + } + let transition = BookTransition { + kind: BookTransitionKind::Snapshot, + publication: BookPublication::Snapshot, + outcome: BookOutcome::Keepalive, + status: core.status(), + generation: core.generation(), + snapshot_sequence: core.snapshot_sequence(), + last_sequence: core.last_sequence(), + observed_at_ms: Some(times.observed_at_ms), + native_sequence_start: None, + previous_sequence: None, + updates: vec![], + view: Some(view), + materialized_snapshot_at_ms: None, + proof_observed_at_ns: Some(raw.received_at_ns), + }; + let Some(partition_sequence) = self.next_l2_partition_sequence(binding, transport_offset) + else { + return self.quarantine( + raw, + QuarantineReason::SemanticInvalid, + "L2 transport-derived partition sequence overflow", + materialized_at_ns, + ); + }; + let canonical = match canonicalize_l2_ready_snapshot( + &l2_fixture(binding, raw, payload, partition_sequence), + &transition, + ) { + Ok(value) => value, + Err(_) => { + return self.quarantine( + raw, + QuarantineReason::SemanticInvalid, + "L2 quiet proof failed strict canonical projection", + materialized_at_ns, + ) + } + }; + if self.seen_ids.contains(&canonical.event_id) { + return ProcessBatch { + canonical: vec![], + quarantines: vec![], + duplicates: 1, + filtered: 0, + filtered_outcome: None, + }; + } + self.partition_sequences + .insert(l2_partition_key(binding), canonical.partition_sequence); + self.remember(canonical.event_id.clone()); + if let Some(observation) = + L2SnapshotObservation::from_transition(&transition, times.observed_at_ms) + { + self.l2_snapshot_observations + .insert(binding_key.to_owned(), observation); + } + if let Some(entry) = self.l2_view_times.get_mut(binding_key) { + entry.last_snapshot_received_ns = raw.received_at_ns; + } + ProcessBatch { + canonical: vec![canonical_record( + &self.config.canonical_stream, + canonical, + materialized_at_ns, + )], + quarantines: vec![], + duplicates: 0, + filtered: 0, + filtered_outcome: None, + } + } + fn due_l2_materialized_snapshot( &self, binding_key: &str, @@ -2786,6 +3054,259 @@ mod tests { assert_eq!(ahead.filtered_outcome, Some("REST_AHEAD_OF_STREAM")); } + const QUIET_RECEIPT_NS: i64 = 1_786_352_400_000_000_000; + + fn quiet_book_binding() -> CoreBinding { + let mut binding = binance_book_binding(); + binding + .l2 + .as_mut() + .unwrap() + .materialized_snapshot_interval_ms = Some(1_000); + binding + } + + fn liveness_marker( + binding: &CoreBinding, + generation: u64, + received_at_ns: i64, + ) -> RawProviderEnvelope { + let frame = format!( + r#"{{"qdl_transport_liveness":"v1","stream":"{}"}}"#, + binding.native_channel + ); + raw_with_receipt(binding, frame.as_bytes(), generation, received_at_ns) + } + + /// READY at receipt +1 ms with one published bootstrap snapshot (provider time 1001 ms). + fn quiet_ready_core(binding: &CoreBinding) -> (RealtimeCore, EventEnvelope) { + let mut core = core(binding.clone(), true); + let _ = core + .process( + raw_with_receipt( + binding, + br#"{"s":"BTCUSDT","U":99,"u":101,"pu":98,"E":1001,"b":[["60000","2"]],"a":[["60001","1"]]}"#, + 1, + QUIET_RECEIPT_NS, + ), + 10, + ) + .unwrap(); + let ready = core + .process( + with_transport( + raw_with_receipt( + binding, + br#"{"lastUpdateId":100,"E":1000,"bids":[["60000","1"]],"asks":[["60001","1"]]}"#, + 1, + QUIET_RECEIPT_NS + 1_000_000, + ), + TransportProtocol::Http, + ), + 11, + ) + .unwrap(); + assert_eq!(ready.canonical.len(), 1); + let snapshot = EventEnvelope::decode(ready.canonical[0].payload.as_slice()).unwrap(); + (core, snapshot) + } + + fn proof_book(batch: &ProcessBatch) -> (EventEnvelope, i64, String) { + assert_eq!(batch.canonical.len(), 1, "{batch:?}"); + let envelope = EventEnvelope::decode(batch.canonical[0].payload.as_slice()).unwrap(); + let Some(event_envelope::Payload::BookSnapshot(book)) = envelope.payload.clone() else { + panic!("a quiet proof is a verified BOOK_SNAPSHOT"); + }; + assert!(book.sequence_verified); + (envelope, book.proof_observed_at_ns, book.native_sequence) + } + + #[test] + fn quiet_binance_book_publishes_in_band_proof_without_redating_the_event() { + // IDR-040 fix 2: no depth frame arrives, but the owning connection keeps delivering. + let binding = quiet_book_binding(); + let (mut core, bootstrap) = quiet_ready_core(&binding); + let Some(event_envelope::Payload::BookSnapshot(bootstrap_book)) = bootstrap.payload.clone() + else { + panic!("bootstrap must be a snapshot"); + }; + assert_eq!( + bootstrap_book.proof_observed_at_ns, 0, + "an ordinary snapshot carries no proof" + ); + + let early = core + .process( + liveness_marker(&binding, 1, QUIET_RECEIPT_NS + 600_000_000), + 12, + ) + .unwrap(); + assert!(early.canonical.is_empty() && early.quarantines.is_empty()); + assert_eq!(early.filtered_outcome, Some("TRANSPORT_LIVENESS_NOT_DUE")); + + let proof_at = QUIET_RECEIPT_NS + 1_500_000_000; + let marker = liveness_marker(&binding, 1, proof_at); + let proven = core.process(marker.clone(), 13).unwrap(); + let (envelope, proof, sequence) = proof_book(&proven); + assert_eq!(proof, proof_at); + assert_eq!(sequence, "101"); + // The provider event time is the book's own, never the proof's. + assert_eq!( + envelope.source_event_time_ns, + bootstrap.source_event_time_ns + ); + assert_eq!(envelope.source_event_time_ns, 1_001_000_000); + assert_eq!(envelope.received_at_ns, proof_at); + assert_ne!(envelope.event_id, bootstrap.event_id); + assert_eq!(envelope.raw_capture_id, marker.capture_id); + + // Redelivery of the same marker is the same event, and the cadence holds. + let replay = core.process(marker, 14).unwrap(); + assert!(replay.canonical.is_empty()); + assert_eq!(replay.filtered_outcome, Some("TRANSPORT_LIVENESS_NOT_DUE")); + let soon = core + .process(liveness_marker(&binding, 1, proof_at + 400_000_000), 15) + .unwrap(); + assert_eq!(soon.filtered_outcome, Some("TRANSPORT_LIVENESS_NOT_DUE")); + let next = core + .process(liveness_marker(&binding, 1, proof_at + 1_000_000_000), 16) + .unwrap(); + let (next_envelope, next_proof, _) = proof_book(&next); + assert_eq!(next_proof, proof_at + 1_000_000_000); + assert_eq!(next_envelope.source_event_time_ns, 1_001_000_000); + assert_ne!(next_envelope.event_id, envelope.event_id); + assert!(next_envelope.partition_sequence > envelope.partition_sequence); + + // A real delta after the proof still applies on the same continuity. + let delta = core + .process( + raw_with_receipt( + &binding, + br#"{"s":"BTCUSDT","U":102,"u":102,"pu":101,"E":4000,"b":[["60000","3"]],"a":[]}"#, + 1, + proof_at + 1_100_000_000, + ), + 17, + ) + .unwrap(); + assert!(!delta.canonical.is_empty()); + assert!(delta.quarantines.is_empty()); + } + + #[test] + fn liveness_marker_without_an_owned_ready_book_proves_nothing_and_changes_nothing() { + let binding = quiet_book_binding(); + let key = binding.key(); + + // No book yet: no proof, no fence, no session claimed. + let mut cold = core(binding.clone(), true); + let before = cold + .process(liveness_marker(&binding, 1, QUIET_RECEIPT_NS), 10) + .unwrap(); + assert!(before.canonical.is_empty() && before.quarantines.is_empty()); + assert_eq!(before.filtered_outcome, Some("TRANSPORT_LIVENESS_UNPROVEN")); + assert!(!cold.l2_sessions.contains_key(&key)); + + let (mut core, _) = quiet_ready_core(&binding); + let session = core.l2_sessions[&key].clone(); + let later = QUIET_RECEIPT_NS + 5_000_000_000; + + // A newer connection generation proves nothing about this view. + let newer = core + .process(liveness_marker(&binding, 2, later), 12) + .unwrap(); + assert_eq!(newer.filtered_outcome, Some("TRANSPORT_LIVENESS_UNPROVEN")); + assert!(newer.canonical.is_empty()); + + // Another lane with the same generation proves nothing either. + let mut foreign = liveness_marker(&binding, 1, later); + foreign.source_session_id = "qdl-other-lane-001-1-1700000000000000000".into(); + let foreign = core.process(foreign, 13).unwrap(); + assert_eq!( + foreign.filtered_outcome, + Some("TRANSPORT_LIVENESS_UNPROVEN") + ); + assert_eq!(core.l2_sessions[&key], session); + assert_eq!(core.l2_adapters[&key].core().status(), BookStatus::Ready); + + // A marker that names another stream, or that did not travel in-band, is invalid. + let mut wrong_stream = raw_with_receipt( + &binding, + br#"{"qdl_transport_liveness":"v1","stream":"ethusdt@depth@100ms"}"#, + 1, + later, + ); + wrong_stream.correlation_id = "wrong-stream".into(); + let wrong_stream = core.process(wrong_stream, 14).unwrap(); + assert!(wrong_stream.canonical.is_empty()); + assert_eq!(wrong_stream.quarantines.len(), 1); + let rest = core + .process( + with_transport( + liveness_marker(&binding, 1, later + 1), + TransportProtocol::Http, + ), + 15, + ) + .unwrap(); + assert_eq!(rest.quarantines.len(), 1); + let unknown_version = core + .process( + raw_with_receipt( + &binding, + br#"{"qdl_transport_liveness":"v2","stream":"btcusdt@depth@100ms"}"#, + 1, + later + 2, + ), + 16, + ) + .unwrap(); + assert_eq!(unknown_version.quarantines.len(), 1); + assert_eq!(core.l2_adapters[&key].core().status(), BookStatus::Ready); + + // After a real gap the book is not current; a marker cannot vouch for it. + let gap = core + .process( + raw_with_receipt( + &binding, + br#"{"s":"BTCUSDT","U":105,"u":105,"pu":104,"E":6000,"b":[],"a":[]}"#, + 1, + later + 3, + ), + 17, + ) + .unwrap(); + assert_eq!(gap.quarantines.len(), 1); + let after_gap = core + .process(liveness_marker(&binding, 1, later + 2_000_000_000), 18) + .unwrap(); + assert!(after_gap.canonical.is_empty() && after_gap.quarantines.is_empty()); + assert_eq!( + after_gap.filtered_outcome, + Some("TRANSPORT_LIVENESS_UNPROVEN") + ); + } + + #[test] + fn quiet_proof_replay_is_byte_deterministic_on_transport_offsets() { + let binding = quiet_book_binding(); + let run = || { + let (mut core, _) = quiet_ready_core(&binding); + core.process_at_transport_offset( + liveness_marker(&binding, 1, QUIET_RECEIPT_NS + 2_000_000_000), + 99, + 42, + ) + .unwrap() + .canonical + }; + let first = run(); + let second = run(); + assert_eq!(first.len(), 1); + assert_eq!(first[0].payload, second[0].payload); + assert_eq!(first[0].event_id, second[0].event_id); + } + #[test] fn binance_l2_gap_is_quarantined_and_requires_resync() { let binding = binance_book_binding(); diff --git a/tests/test_idr040_quiet_book_proof.py b/tests/test_idr040_quiet_book_proof.py new file mode 100644 index 00000000..54f74361 --- /dev/null +++ b/tests/test_idr040_quiet_book_proof.py @@ -0,0 +1,204 @@ +"""IDR-040 fix 2: the quiet-book proof is an additive, report-only quality field. + +The Rust core publishes a quiet-proof BOOK_SNAPSHOT whose +``OrderBookSnapshot.proof_observed_at_ns`` is the receipt time of in-band +transport evidence (see ``rust/qdl-realtime-core``). These tests pin the Python +side of the contract: + +* the generated descriptor carries the new field as int64 tag 8 and decodes + bytes written without it (old producers) as "no proof"; +* Query reports it as ``quality.proof_observed_at_ns`` only for a verified book + snapshot and never lets it change freshness, state or eligibility; +* the public response carries the key only when a proof exists, so a strict + (2.0.x, ``extra="forbid"``) reader keeps parsing every other response, and + the tolerant 2.1.0 reader accepts it either way. +""" + +from __future__ import annotations + +import json +import unittest +from pathlib import Path + +from pydantic import BaseModel, ConfigDict + +from qdl.marketdata.v2 import market_data_pb2 +from qdl.query import ConsumerGrade, DataRequirement, FeedType +from qdl.query.results import QualityMetadata +from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_source import StableSpoolQueryBackend +from qdl_sdk.models import ProblemDetails, ProblemDiagnostics, QualityView +import tests.test_qdl_sdk_stream_projection as projection_tests +from qdl_sdk import Feed, StreamEvent, market_data_view_from_stream + +ROOT = Path(__file__).resolve().parents[1] +CATALOG_PATH = ROOT / "config/v2/stable-source-bindings.yaml" +PROOF_NS = 1_800_000_000_500_000_000 + + +class LegacyQualityView(BaseModel): + """The QualityView shape a strict 2.0.x reader validates.""" + + model_config = ConfigDict(extra="forbid") + state: str + freshness_ms: int + event_recency_state: str = "LIVE" + provider_session_state: str = "NOT_APPLICABLE" + provider_session_liveness_ms: int | None = None + gap_open: bool + complete: bool + execution_eligible: bool + policy_id: str + flags: list[str] + + +def _quality(**extra) -> QualityView: + return QualityView(state="STALE", freshness_ms=4_000, event_recency_state="STALE", + gap_open=False, complete=True, execution_eligible=False, + policy_id="crypto_primary_v2", flags=["LAST_EVENT_STALE"], **extra) + + +class GeneratedContractTests(unittest.TestCase): + def test_field_is_additive_int64_tag_8(self) -> None: + field = market_data_pb2.OrderBookSnapshot.DESCRIPTOR.fields_by_name["proof_observed_at_ns"] + self.assertEqual((field.number, field.type), (8, field.TYPE_INT64)) + old_fields = {f.name: f.number for f in market_data_pb2.OrderBookSnapshot.DESCRIPTOR.fields} + self.assertEqual( + {name: number for name, number in old_fields.items() if number < 8}, + {"native_sequence": 1, "checksum": 2, "levels": 3, "depth": 4, + "book_generation": 5, "sequence_verified": 6, "truncated": 7}, + ) + + def test_old_bytes_mean_no_proof_and_unset_field_keeps_old_bytes(self) -> None: + book = market_data_pb2.OrderBookSnapshot(native_sequence="101", depth=1, + book_generation=1, sequence_verified=True) + encoded = book.SerializeToString() + self.assertEqual(market_data_pb2.OrderBookSnapshot.FromString(encoded).proof_observed_at_ns, 0) + proven = market_data_pb2.OrderBookSnapshot() + proven.CopyFrom(book) + proven.proof_observed_at_ns = PROOF_NS + self.assertTrue(proven.SerializeToString().startswith(encoded)) + self.assertEqual( + market_data_pb2.OrderBookSnapshot.FromString(proven.SerializeToString()).proof_observed_at_ns, + PROOF_NS, + ) + + +class ResponseCompatibilityTests(unittest.TestCase): + def test_no_proof_keeps_the_old_wire_shape(self) -> None: + wire = json.loads(_quality().model_dump_json()) + self.assertNotIn("proof_observed_at_ns", wire) + LegacyQualityView.model_validate(wire) + + def test_proof_is_emitted_and_read_back_by_the_tolerant_reader(self) -> None: + quality = _quality(proof_observed_at_ns=PROOF_NS) + wire = json.loads(quality.model_dump_json()) + self.assertEqual(wire["proof_observed_at_ns"], PROOF_NS) + self.assertEqual(QualityView.model_validate(wire), quality) + # A strict 2.0.x reader would refuse it: the reason the key is gated. + with self.assertRaises(ValueError): + LegacyQualityView.model_validate(wire) + + def test_non_positive_proof_is_rejected(self) -> None: + for value in (0, -1): + with self.assertRaises(ValueError): + _quality(proof_observed_at_ns=value) + with self.assertRaises(ValueError): + QualityMetadata(state="LIVE", freshness_ms=0, gap_open=False, complete=True, + execution_eligible=True, policy_id="p", proof_observed_at_ns=0) + + def test_refusal_diagnostics_carry_the_key_only_with_a_proof(self) -> None: + base = dict(evaluated_at_ns=1, state="STALE", freshness_ms=4_000, + event_recency_state="STALE", provider_session_state="NOT_APPLICABLE", + execution_eligible=False, gap_open=False, complete=True) + plain = ProblemDiagnostics(**base).model_dump(mode="json") + self.assertNotIn("proof_observed_at_ns", plain) + problem = ProblemDetails(type="urn:qdl:DATA_STALE", title="DATA_STALE", status=409, + code="DATA_STALE", detail="stale", request_id="test-only", + retryable=True, + diagnostics=ProblemDiagnostics(**base, proof_observed_at_ns=PROOF_NS)) + wire = json.loads(problem.model_dump_json()) + self.assertEqual(wire["diagnostics"]["proof_observed_at_ns"], PROOF_NS) + self.assertEqual(ProblemDetails.model_validate(wire), problem) + + +class QueryQualityTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.catalog = StableSourceCatalog.load(CATALOG_PATH) + cls.binding = next( + item for item in cls.catalog.bindings + if item.feed is FeedType.BOOK_SNAPSHOT + and item.instrument.identity.venue == "BINANCE" + ) + + def backend(self, now_ns: int) -> StableSpoolQueryBackend: + return StableSpoolQueryBackend(None, self.catalog, schema_digest="a" * 64, + clock_ns=lambda: now_ns) + + def requirement(self) -> DataRequirement: + return DataRequirement( + instrument_uid=self.binding.instrument.instrument_uid, + feed=self.binding.feed, + interval=self.binding.interval, + consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id=self.binding.source_policy_id, + max_freshness_ms=2_000, + ) + + def book(self, *, proof: int = 0, verified: bool = True) -> market_data_pb2.EventEnvelope: + envelope = market_data_pb2.EventEnvelope( + source_event_time_ns=1_800_000_000_000_000_000, + received_at_ns=1_800_000_000_010_000_000, + ) + envelope.book_snapshot.native_sequence = "101" + envelope.book_snapshot.depth = 1 + envelope.book_snapshot.book_generation = 1 if verified else 0 + envelope.book_snapshot.sequence_verified = verified + envelope.book_snapshot.proof_observed_at_ns = proof + return envelope + + def quality(self, envelope, now_ns: int) -> QualityMetadata: + return self.backend(now_ns)._quality(self.requirement(), self.binding, envelope, + gap_open=False, watermark_offset=1) + + def test_proof_is_reported_but_never_changes_freshness_or_state(self) -> None: + now = 1_800_000_005_000_000_000 # 5 s after the provider event + plain = self.quality(self.book(), now) + proven = self.quality(self.book(proof=PROOF_NS), now) + self.assertIsNone(plain.proof_observed_at_ns) + self.assertEqual(proven.proof_observed_at_ns, PROOF_NS) + for field in ("state", "freshness_ms", "event_recency_state", "execution_eligible", + "complete", "gap_open", "flags", "provider_session_state"): + self.assertEqual(getattr(proven, field), getattr(plain, field), field) + self.assertEqual(proven.freshness_ms, 5_000) + self.assertEqual(proven.state, "STALE") + self.assertFalse(proven.execution_eligible) + + def test_unverified_book_never_reports_a_proof(self) -> None: + quality = self.quality(self.book(proof=PROOF_NS, verified=False), 1_800_000_000_100_000_000) + self.assertIsNone(quality.proof_observed_at_ns) + + +class StreamProjectionTests(unittest.TestCase): + def test_sdk_stream_projection_reports_the_proof_and_keeps_policy(self) -> None: + requirement = projection_tests.SdkStreamProjectionTests.requirement(None, Feed.BOOK_SNAPSHOT) + proven = projection_tests.envelope(Feed.BOOK_SNAPSHOT) + proven.book_snapshot.proof_observed_at_ns = PROOF_NS + view = market_data_view_from_stream( + StreamEvent(12, "signed", proven), template=projection_tests.template(Feed.BOOK_SNAPSHOT), + requirement=requirement, now_ns=projection_tests.NOW + 100_000_000) + plain = market_data_view_from_stream( + StreamEvent(12, "signed", projection_tests.envelope(Feed.BOOK_SNAPSHOT)), + template=projection_tests.template(Feed.BOOK_SNAPSHOT), + requirement=requirement, now_ns=projection_tests.NOW + 100_000_000) + self.assertEqual(view.quality.proof_observed_at_ns, PROOF_NS) + self.assertIsNone(plain.quality.proof_observed_at_ns) + self.assertEqual( + view.quality.model_dump(exclude={"proof_observed_at_ns"}), + plain.quality.model_dump(exclude={"proof_observed_at_ns"}), + ) + + +if __name__ == "__main__": + unittest.main() From dac490d5bbbb39498d30a33967888f9d72681018 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 04:30:03 +0000 Subject: [PATCH 089/147] docs(plan): IDR-040 fix 2 in-band quiet-book proof (1a2daf1) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 02de9868..21acef0a 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71692,3 +71692,33 @@ Packet: upgrade/evidence/p184-g12-isolated-fault-packet.md. removed by ID (the harness now passes -v). Production guard: 0 breaches. - Tests: 46 PASS; after the cherry-pick, g12 + dr05_dl_fault + kn_recovery_observe OK. - Limits: same host, a single projector/Stream replica, synthetic BOOK only. Not HA or production evidence. + +### IDR-040 fix 2: in-band quiet-book proof (v2.2.3, additive, default off) - 2026-10-10 (Claude, subagent) + +Owner decision 2026-10-10: fix 2 goes into v2.2.3. Commit 1a2daf1, cherry-picked from 5062d4e; the CHANGELOG conflict +was resolved by keeping all entries, and the regenerated OpenAPI snapshot is byte-identical to the merged one. +Status: tested locally, not deployed. +- Native ingestor (opt-in `book_liveness_proof_ms` 250..5000, default off): for a quiet Binance book it writes + `{"qdl_transport_liveness":"v1","stream":...}` into that book's own raw partition. + - The marker is stamped with the receipt time of the newest WebSocket message (frame or pong); the ingestor pings + when idle. + - TCP and partition order put every earlier frame of that book ahead of the marker. +- Realtime core: + - A marker becomes a quiet-proof BOOK_SNAPSHOT only for the exact owning session/generation of a READY, + non-invalidated book, at most once per materialization interval. + - The snapshot is the current view with the original provider time and the new additive + `OrderBookSnapshot.proof_observed_at_ns = 8`. + - Otherwise it filters TRANSPORT_LIVENESS_UNPROVEN/_NOT_DUE or quarantines the marker. Markers never start, reset or + fence a book. +- Query and the SDK projection report `quality.proof_observed_at_ns` (and the refusal diagnostics) for verified + snapshots only. freshness_ms, state and eligibility are unchanged. The key is emitted only when present. +- Tests (subagent): + - Rust core 51/0 + 56/0 (3 new); ingestor 28/0; workspace 360/0 with 66 ignored; clippy -D warnings clean; + - Python new 9/0; affected 75 OK; + - full discover: only failures that also fail at base or come from the mount. +- After merging with IDR-082 and the rc4 fixes: 13 modules, 128 tests OK (33 Redis skips). +- Rollout order: + 1. new core and Query first; + 2. readers on SDK 2.1.0+; + 3. then render the ingestor flag (stable_deployment.py does not emit it yet). +- The contract digest changes at the next bundle refresh. Ping cadence and marker volume are not measured. From 359c9eeae0f4c109ae20803ab24965b0ca7c0d97 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 04:42:27 +0000 Subject: [PATCH 090/147] release(sdk): qdl_sdk 2.2.0 for Data Layer v2.2.3 (start_index, event_id, proof_observed_at_ns; additive) --- qdl_sdk/CHANGELOG.md | 2 +- qdl_sdk/README.md | 2 +- qdl_sdk/__init__.py | 2 +- scripts/build_qdl_sdk_release.py | 2 +- tests/test_qdl_sdk_release.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/qdl_sdk/CHANGELOG.md b/qdl_sdk/CHANGELOG.md index 46dd8fa9..71f76375 100644 --- a/qdl_sdk/CHANGELOG.md +++ b/qdl_sdk/CHANGELOG.md @@ -1,6 +1,6 @@ # SDK Changelog -## Unreleased +## 2.2.0 (v2.2.3 Data Layer) `ReplicatedRestQueryTransport` starts its round-robin on a random replica per instance instead of always replica 0. A new keyword `start_index` pins the start diff --git a/qdl_sdk/README.md b/qdl_sdk/README.md index e018b49f..ec978b0d 100644 --- a/qdl_sdk/README.md +++ b/qdl_sdk/README.md @@ -146,7 +146,7 @@ Build the standalone artifact with: python scripts/build_qdl_sdk_release.py --output-dir dist/qdl-sdk ``` -The output contains a reproducible `qdl_sdk-2.1.0-py3-none-any.whl`, a release +The output contains a reproducible `qdl_sdk-2.2.0-py3-none-any.whl`, a release manifest with the wheel/source/generated-contract SHA-256 digests, and a CycloneDX SBOM. The wheel contains only the public SDK plus generated Protobuf contracts; it does not package `qdl.api_v2`, runtime adapters, provider code or diff --git a/qdl_sdk/__init__.py b/qdl_sdk/__init__.py index 4419207f..c8186dc7 100644 --- a/qdl_sdk/__init__.py +++ b/qdl_sdk/__init__.py @@ -1,6 +1,6 @@ """Stable Python SDK V2 with typed public models and generated transports.""" -__version__ = "2.1.0" +__version__ = "2.2.0" from qdl_sdk.client import AsyncDataLayerClient, DataLayerClientV2, WarmupStreamSession from qdl_sdk.credentials import ( diff --git a/scripts/build_qdl_sdk_release.py b/scripts/build_qdl_sdk_release.py index a339e889..500a3c6a 100755 --- a/scripts/build_qdl_sdk_release.py +++ b/scripts/build_qdl_sdk_release.py @@ -15,7 +15,7 @@ ROOT = Path(__file__).resolve().parents[1] NAME = "qdl-sdk" NORMALIZED_NAME = "qdl_sdk" -VERSION = "2.1.0" +VERSION = "2.2.0" DIST_INFO = f"{NORMALIZED_NAME}-{VERSION}.dist-info" DEPENDENCIES = ( "grpcio>=1.70.0,<2.0.0", diff --git a/tests/test_qdl_sdk_release.py b/tests/test_qdl_sdk_release.py index f8f7dedf..a064adc3 100644 --- a/tests/test_qdl_sdk_release.py +++ b/tests/test_qdl_sdk_release.py @@ -41,11 +41,11 @@ def test_wheel_is_reproducible_self_contained_and_recorded(self): expected = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()).rstrip(b"=").decode() self.assertEqual(encoded, "sha256=" + expected) self.assertEqual(int(size), len(payload)) - self.assertIn("qdl_sdk-2.1.0-py3-none-any.whl", archive.read("qdl_sdk/README.md").decode()) + self.assertIn("qdl_sdk-2.2.0-py3-none-any.whl", archive.read("qdl_sdk/README.md").decode()) release = json.loads(Path(first_result["manifest"]).read_text()) self.assertFalse(release["contains_service_internals"]) - self.assertEqual(release["version"], "2.1.0") + self.assertEqual(release["version"], "2.2.0") self.assertEqual(len(release["generated_contract_digest"]), 64) sbom = json.loads(Path(first_result["sbom"]).read_text()) self.assertEqual(sbom["bomFormat"], "CycloneDX") From 5c67d5f0ecc10618eb80cc6f5cdca8ac0de194da Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 04:43:25 +0000 Subject: [PATCH 091/147] docs(p184): D5 v2.2.3 release rollout packet (draft, digests after build) --- .../p184-d5-release-rollout-packet.md | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 upgrade/evidence/p184-d5-release-rollout-packet.md diff --git a/upgrade/evidence/p184-d5-release-rollout-packet.md b/upgrade/evidence/p184-d5-release-rollout-packet.md new file mode 100644 index 00000000..242bec51 --- /dev/null +++ b/upgrade/evidence/p184-d5-release-rollout-packet.md @@ -0,0 +1,71 @@ +# P18.4 D5: Data Layer v2.2.3 Release Rollout Packet + +Status: DRAFT. Digests are filled in after the one-commit build. Each runtime step needs owner approval of this +exact blast radius and rollback. + +## Goal + +Close IDR-039: every application role of `qdl_v2_stable_candidate` runs images built from ONE commit of +`fix/p184-dl-closure`. That commit carries: +- IDR-062 (BOOK_SNAPSHOT on top-of-book change); +- IDR-040 fix 1 (REST_AHEAD_OF_STREAM) and fix 2 (quiet-book proof, ingestor flag default off); +- IDR-082 (`event_id` on the view); +- IDR-084 and the rc4 Query fixes (MARK single-flight, env cold slice); +- the SDK 2.2.0 transport fix. + +## Inventory before the rollout (2026-10-10 04:4xZ, `docker inspect`) + +| Role | Image ID | Label revision | +|---|---|---| +| query_kn_1, query_kn_2 | sha256:5a9e1b4b8be1 (2.2.3-rc4) | bd74faa | +| binance_bar_edge | sha256:514506122111 (2.2.2) | d6d2637 | +| ingestor_binance_usdm, ingestor_okx_swap | sha256:0f6876e16e51 | 855c914 | +| rust_core, rust_core_2, rust_core_3 | sha256:658a9570c5fc | 0e00c81 | +| market_projector_1, market_projector_2 | sha256:a696cfba6ab3 | c8015ca | +| stream_kn_1, stream_kn_2 | sha256:1f13408fe946 | c8015ca | + +Not touched by this packet: +- kafka1-3 (sha256:9516fb7634ba); +- market_cache and stable_redis (redis@sha256:dfa188...); +- the one-shots stable_state_init and stable_tls_init; +- the TS writer `market_data_service_p184w`; +- the realm dirs and consumer manifests (rev11 TS, rev15 alpha); +- every env value except the image, including QDL_KN_MAX_SUBSCRIPTIONS=1024 and any tuned QDL_QUERY_COLD_SLICE_MS. + +## Build (one commit, both images) + +- Source: `git archive ` into a private dir. One archive feeds both Dockerfiles (`Dockerfile` and + `Dockerfile.qdl-rust-runtime`), following scripts/p184_consolidated_build.py. +- Tags: `qdl-v2-python:2.2.3-` and `qdl-v2-rust:2.2.3-`. Record the image IDs and label revisions; + the revision must equal ``. +- Before any recreate, check that every Rust role's current command/entrypoint binary exists in the new Rust image + (`docker run --rm --entrypoint ls `). +- Local exact-source CI equivalent of `.github/workflows/ci.yml` on the same archive. The DL repo is not pushed, by + owner rule. Record the receipt. + +## Rollout order (one role at a time; next role only after health + guard green) + +Guard during every step: TS writer READY 64/64, RestartCount and OOM of the changed role, Kafka ISR, disk >= 20 GB. + +1. query_kn_1, then query_kn_2: python release image. Wait for healthy. +2. binance_bar_edge: python release image. Coupling C.19/C.32: bar edge reads its acquisition plan from the image. + Compare the plan revision in the old and new image first; if it differs, STOP (it would strand the checkpoint). +3. stream_kn_1, then stream_kn_2: Rust release image. Expect TS stream reconnects (G11 bound). +4. market_projector_1, then market_projector_2. G12 isolated: 0.26 s caught up. +5. rust_core, rust_core_2, rust_core_3, one at a time. Wait for quorum and READY each time. Books reset and TS sees + typed gaps (DR05 path). +6. ingestor_okx_swap, then ingestor_binance_usdm: provider sessions reconnect. `book_liveness_proof_ms` stays unset + (IDR-040 fix 2 off). + +## Rollback + +Per role, recreate from the saved live compose (from `docker inspect` before its step) on the previous image ID in +the inventory above. Roll back in reverse order. No data migration: Kafka, Redis and the realm dirs are untouched. + +## Acceptance after the rollout + +- G13: DR01-DR08 on the final digests. +- G1-G14 scored on the final digests. G4/G5 use the frozen `config/v2/p184-stage100-budget.json`. +- Then release notes, digests, freeze. Known limits, recorded rather than release-blocking: + - G12 was isolated: same host, a single replica, BOOK only. + - IDR-040 fix 2 is off; ping cadence and marker volume are not measured. From ec89a31ce87140cf984b97b173a8a1ff6e7f8a8c Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 04:46:02 +0000 Subject: [PATCH 092/147] fix(query): MARK hot single-flight only on the IDR-084 branch, never on the backup path (rc4 regression) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 18 ++++++++++++++ qdl/runtime/kn_query_backend.py | 22 +++++++++++------ tests/test_kn_mark_hot_memo.py | 30 +++++++++++++++++++++++ 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 21acef0a..3418f2d3 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71722,3 +71722,21 @@ Status: tested locally, not deployed. 2. readers on SDK 2.1.0+; 3. then render the ingestor flag (stable_deployment.py does not emit it yet). - The contract digest changes at the next bundle refresh. Ping cadence and marker volume are not measured. + +### rc4 regression: MARK SOURCE_UNAVAILABLE under load; stage 100c stopped, Query rolled back to rc3 - 2026-10-10 (Claude) + +- rc4 (bd74faa, sha256:5a9e1b4b8be1) was rolled to query_kn_1/2 at 03:43-03:44Z, and stage 100c started at 04:30Z. +- From 04:30Z the TS writer logged `MARK_INDEX_PRICE ... IncompleteExecutionReferenceError code=SOURCE_UNAVAILABLE`: + 53 and 28 per 10 min, against 0-6 per 10 min before. Guard: DEGRADED 63/64 at 04:32:59 and 04:42:56-04:43:16. +- Cause, from code (Query logs carry no refusal line): bd74faa routed `_hot_candidate` through `_hot_latest` on EVERY + path, including the `_latest_view` backup. A shared hot record up to 50 ms old can be older than the projected + primary. The backup then fails its monotonic check, and the MARK read returns not-ready. +- Action: + - stage 100c stopped at about 04:45Z (driver, guard, observer, client killed; 0 left); + - query_kn_1 then query_kn_2 recreated from the saved rc3 composes (sha256:672503b7fe87): healthy, restarts 0. +- Fix (this commit): the single-flight applies only to the IDR-084 opportunistic branch (`shared=True`); the + backup path asks the hot reader every time. + - Regression tests BackupPathNeverSharesTests. + - kn_mark_hot_memo, kn_query_backend, kn_hot_book_invalidation and kn_hot_view OK (33 Redis skips). +- Lesson: a shared cache on a path with a monotonic gate needs a test for that path before rollout; the rc4 unit + tests covered only the memo itself. diff --git a/qdl/runtime/kn_query_backend.py b/qdl/runtime/kn_query_backend.py index e8e6ccb4..ef797b19 100644 --- a/qdl/runtime/kn_query_backend.py +++ b/qdl/runtime/kn_query_backend.py @@ -277,12 +277,16 @@ def _hot_monotonic(self, binding, view, *, record=False) -> bool: self._hot_boundaries[binding.binding_id] = (view.boundary, record_offset, digest) return True - def _hot_candidate(self, binding, primary=None): - """Authenticated committed bytes, validated by the same lineage oracle.""" + def _hot_candidate(self, binding, primary=None, *, shared=False): + """Authenticated committed bytes, validated by the same lineage oracle. + + ``shared`` (IDR-084 opportunistic MARK path only) may reuse a lookup made + up to MARK_HOT_MEMO_NS earlier; the backup path of ``_latest_view`` never + does, because a shared, older record could fail its monotonic check.""" if self.hot_client is None or binding.feed.value not in HOT_FEEDS: return None try: - candidate = self._hot_latest(binding) + candidate = self._hot_latest(binding) if shared else self.hot_client.latest(binding, self.product_key(binding)) except HotViewGap as error: raise QueryBackendError(QueryProblem( CanonicalErrorCode.OPEN_SEQUENCE_GAP, str(error), True, @@ -316,10 +320,12 @@ def _hot_latest(self, binding): lookups refused each other HOT_READER_BUSY, and two TRADE/QUOTE reads that needed the backup ended DATA_NOT_READY. Many sessions read the same MARK product, so concurrent and back-to-back MARK reads on one replica now share - one lookup (outcome, including a refusal) for 50 ms. The caller still takes - the newer of cache and hot view under every lineage/monotonic gate, so a - shared result can only be older, never less verified. Other feeds are - unchanged.""" + one lookup (outcome, including a refusal) for 50 ms. Used ONLY by the + IDR-084 opportunistic branch, which keeps the cache view whenever the hot + record is not newer. rc4 also shared it on the _latest_view backup path: + under stage-100 load an older shared record failed the backup's monotonic + check and the TS writer saw MARK SOURCE_UNAVAILABLE (2026-10-10 04:30Z). + Other feeds are unchanged.""" if binding.feed is not FeedType.MARK_INDEX_PRICE: return self.hot_client.latest(binding, self.product_key(binding)) key = binding.binding_id @@ -463,7 +469,7 @@ def latest_stored_event(self, binding: StableSourceBinding, *, prefer_hot: bool # so MARK/INDEX takes the newer of the two. Every lineage/monotonic gate of the backup path applies; a hot # read that is behind or unavailable leaves the cache view in place (the previous behaviour). try: - candidate = self._hot_candidate(binding, view) + candidate = self._hot_candidate(binding, view, shared=True) except HotViewUnavailable: candidate = None except QueryBackendError as error: diff --git a/tests/test_kn_mark_hot_memo.py b/tests/test_kn_mark_hot_memo.py index c3c07a40..88157fea 100644 --- a/tests/test_kn_mark_hot_memo.py +++ b/tests/test_kn_mark_hot_memo.py @@ -111,5 +111,35 @@ def latest(self, binding, key): self.assertEqual(hot.calls, 1) +class BackupPathNeverSharesTests(unittest.TestCase): + """rc4 regression (2026-10-10 04:30Z): the _latest_view backup path must ask the hot reader every time.""" + + def _candidate_backend(self, hot, clock): + backend = _backend(hot, clock) + record = SimpleNamespace(source_offset=5, canonical=b"x") + view = SimpleNamespace(boundary=SimpleNamespace(topic_id="t", partition=0, offset=5), rows=(record,)) + hot.outcomes = [view] * 10 + backend._parsed = lambda binding, candidate: ("parsed",) + backend._validate_records = lambda binding, records: None + backend._hot_monotonic = lambda binding, candidate, record=False: True + return backend + + def test_backup_path_asks_every_time(self): + clock = [0] + hot = _Hot([]) + backend = self._candidate_backend(hot, clock) + backend._hot_candidate(MARK) + backend._hot_candidate(MARK) + self.assertEqual(hot.calls, 2) + + def test_opportunistic_path_shares(self): + clock = [0] + hot = _Hot([]) + backend = self._candidate_backend(hot, clock) + backend._hot_candidate(MARK, shared=True) + backend._hot_candidate(MARK, shared=True) + self.assertEqual(hot.calls, 1) + + if __name__ == "__main__": unittest.main() From 3939d292e564bbdc7c208743b3e70048b78bd211 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 06:04:23 +0000 Subject: [PATCH 093/147] tooling(p184): local exact-source CI runner (D5) and G2 MARK trace transit summary scripts/p184_local_ci.py replays .github/workflows/ci.yml (parsed with yaml) against git archive per job (private clone when the checkout asks for full history): same run: text in order, per-step exit code/duration/log, summary.json with the sha and PASS/FAIL/SKIPPED. Never runs sudo, never pulls or downloads unless allowed (buf from a local bufbuild/buf image, trivy only with --trivy-image), refuses to start below 40 GB free or when a tag/container it creates already exists, isolated COMPOSE_PROJECT_NAME, cleans its containers/networks/volumes, built images (by ID) and new build-cache records after each job. --dry-run prints the predicted plan. scripts/p184_g2_mark_trace_transit.py summarises MARK_INDEX_PRICE new-event transit (venue newest component -> TS Redis APPLY reply) per venue/instrument from the TS writer delivery trace files, with component ages and the DL/SDK vs writer split reported separately and a G2 verdict (max < 1000 ms, p95 <= 500 ms). Tests: tests/test_p184_local_ci.py (15), tests/test_p184_g2_mark_trace_transit.py (5), offline, run in qdl-v2-python 2.2.3-rc2 with --network none. --- scripts/p184_g2_mark_trace_transit.py | 159 ++++++ scripts/p184_local_ci.py | 690 +++++++++++++++++++++++ tests/test_p184_g2_mark_trace_transit.py | 86 +++ tests/test_p184_local_ci.py | 186 ++++++ 4 files changed, 1121 insertions(+) create mode 100644 scripts/p184_g2_mark_trace_transit.py create mode 100644 scripts/p184_local_ci.py create mode 100644 tests/test_p184_g2_mark_trace_transit.py create mode 100644 tests/test_p184_local_ci.py diff --git a/scripts/p184_g2_mark_trace_transit.py b/scripts/p184_g2_mark_trace_transit.py new file mode 100644 index 00000000..e22e3ed9 --- /dev/null +++ b/scripts/p184_g2_mark_trace_transit.py @@ -0,0 +1,159 @@ +"""P18.4 phase D gate G2 (MARK_INDEX_PRICE): new-event transit from the TS writer delivery trace files. + +Purpose: summarise, per venue and per instrument, the time from the venue MARK/INDEX event to the TS execution Redis +APPLY reply, read from the files that TS services/market_data/delivery_trace.py exports +(delivery---[-pNN].json). Gate G2 (TS upgrade/p184-phase-d-gates.md, frozen 2026-10-09): +MARK max < 1000 ms and p95 <= 500 ms per venue, measured per NEW event; the age of a repeated quiet state is reported +separately and not gated. + +Definitions (per MARK record with outcome APPLIED and both wall clocks present): +* newest = max(mark_source_event_time_ns, index_source_event_time_ns) (the composite changes when either component does); +* new_event_transit_ms = redis_reply_wall_ns - newest, counted only for the first APPLIED record of each instrument whose + newest is strictly later than every earlier applied newest of that instrument (gated quantity; same basis as the + 2026-10-09 mark_probe "newest" and tsmark first-apply measurements); +* first_apply_per_mark_event_ms = the same for distinct mark_source_event_time_ns only (tsmark-comparable); +* all_applied_newest_age_ms, mark_age_ms, index_age_ms = every applied record (repeat/quiet ages, NOT gated); +* upstream_ms = sdk_receive_wall_ns - newest (DL + SDK read), writer_ms = redis_reply_wall_ns - sdk_receive_wall_ns. +Negative durations (clock skew) and records without the clocks are counted as invalid, never clamped. +Quantiles are nearest-rank (sorted[min(n-1, int(q*n))]) as in the earlier probes; p99 only when n >= 100. + +Boundary: read-only over the given files; no Redis, Docker or network access; it never enables tracing (that is a +runtime env change on the writer, owned by the coordinator). Output is JSON on stdout, or --out (created exclusively). +usage: p184_g2_mark_trace_transit.py FILE_OR_GLOB [...] [--skip-seconds 40] [--from-ns N] [--to-ns N] [--out PATH] +""" +from __future__ import annotations + +import argparse +import collections +import glob +import json +import sys + +FEED = "MARK_INDEX_PRICE" +GATE = {"max_lt_ms": 1000.0, "p95_le_ms": 500.0} + + +def quantiles(values): + if not values: + return {"n": 0} + ordered = sorted(values) + n = len(ordered) + pick = lambda q: round(ordered[min(n - 1, int(q * n))], 1) # noqa: E731 + return {"n": n, "p50": pick(.5), "p95": pick(.95), "p99": pick(.99) if n >= 100 else None, + "max": round(ordered[-1], 1), "min": round(ordered[0], 1)} + + +def as_int(value): + return value if type(value) is int and value > 0 else None + + +def venue_of(instrument): + return instrument.replace(".", ":").split(":", 1)[0].upper() + + +def load(paths): + files, records = [], [] + for path in paths: + with open(path) as handle: + report = json.load(handle) + rows = report.get("records") or [] + starts = [r["writer_started_wall_ns"] for r in rows if as_int(r.get("writer_started_wall_ns"))] + files.append({"file": path, "worker": report.get("worker"), "part": report.get("part"), + "stop_reason": report.get("stop_reason"), "recorded": report.get("recorded"), + "dropped_at_export": report.get("dropped_at_export"), + "first_start_ns": min(starts) if starts else None, "last_start_ns": max(starts) if starts else None}) + records.extend(rows) + return files, records + + +def summarise(records, *, skip_seconds=40.0, from_ns=None, to_ns=None): + starts = [r["writer_started_wall_ns"] for r in records if as_int(r.get("writer_started_wall_ns"))] + if not starts: + return {"error": "no records with writer_started_wall_ns"} + lower = from_ns if from_ns is not None else min(starts) + int(skip_seconds * 1e9) + upper = to_ns if to_ns is not None else max(starts) + marks = [r for r in records if r.get("feed") == FEED and as_int(r.get("writer_started_wall_ns")) + and lower <= r["writer_started_wall_ns"] <= upper] + marks.sort(key=lambda r: (r["writer_started_wall_ns"], r.get("redis_reply_wall_ns") or 0)) + outcomes, bases = collections.Counter(), collections.Counter() + invalid = collections.Counter() + series = collections.defaultdict(lambda: collections.defaultdict(list)) + last_newest, last_mark = {}, {} + for r in marks: + instrument = r.get("canonical_instrument_id", "?") + venue = venue_of(instrument) + outcomes[(venue, r.get("outcome"))] += 1 + bases[(venue, r.get("sdk_receive_basis"))] += 1 + if r.get("outcome") != "APPLIED": + continue + reply, mark, index = (as_int(r.get(k)) for k in + ("redis_reply_wall_ns", "mark_source_event_time_ns", "index_source_event_time_ns")) + if reply is None or mark is None or index is None: + invalid[(venue, "missing_clock")] += 1 + continue + newest = max(mark, index) + transit = (reply - newest) / 1e6 + if transit < 0: + invalid[(venue, "negative_transit")] += 1 + continue + for key in (venue, instrument): + series[key]["all_applied_newest_age_ms"].append(transit) + series[key]["mark_age_ms"].append((reply - mark) / 1e6) + series[key]["index_age_ms"].append((reply - index) / 1e6) + receive = as_int(r.get("sdk_receive_wall_ns")) + if instrument not in last_newest or newest > last_newest[instrument]: + last_newest[instrument] = newest + for key in (venue, instrument): + series[key]["new_event_transit_ms"].append(transit) + if receive is not None and receive >= newest and reply >= receive: + series[key]["upstream_ms"].append((receive - newest) / 1e6) + series[key]["writer_ms"].append((reply - receive) / 1e6) + if receive is None or receive < newest or reply < receive: + invalid[(venue, "split_unavailable")] += 1 + if instrument not in last_mark or mark > last_mark[instrument]: + last_mark[instrument] = mark + for key in (venue, instrument): + series[key]["first_apply_per_mark_event_ms"].append(transit) + venues = sorted(k for k in series if ":" not in k) + result = {"window": {"from_ns": lower, "to_ns": upper, "seconds": round((upper - lower) / 1e9, 1)}, + "gate": GATE, "venues": {}, "instruments": {}} + for key in sorted(series): + block = {name: quantiles(values) for name, values in sorted(series[key].items())} + (result["venues"] if key in venues else result["instruments"])[key] = block + for venue in venues: + gated = result["venues"][venue].get("new_event_transit_ms", {"n": 0}) + result["venues"][venue]["g2_verdict"] = ( + "NO_DATA" if not gated["n"] else + "INSIDE" if gated["max"] < GATE["max_lt_ms"] and gated["p95"] <= GATE["p95_le_ms"] else "OUTSIDE") + result["outcomes"] = {f"{v}:{o}": n for (v, o), n in sorted(outcomes.items(), key=str)} + result["sdk_receive_basis"] = {f"{v}:{b}": n for (v, b), n in sorted(bases.items(), key=str)} + result["invalid"] = {f"{v}:{k}": n for (v, k), n in sorted(invalid.items())} + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("inputs", nargs="+", help="trace files or quoted globs") + parser.add_argument("--skip-seconds", type=float, default=40.0, + help="drop the first N s after the first record (writer start-up); ignored with --from-ns") + parser.add_argument("--from-ns", type=int) + parser.add_argument("--to-ns", type=int) + parser.add_argument("--out") + args = parser.parse_args(argv) + paths = sorted({path for pattern in args.inputs for path in (glob.glob(pattern) or [pattern])}) + files, records = load(paths) + report = {"schema": "qdl.p184.g2-mark-trace-transit.v1", "files": len(files), + "records": len(records), "dropped_at_export_total": sum(f["dropped_at_export"] or 0 for f in files), + "stop_reasons": dict(collections.Counter(f["stop_reason"] for f in files)), + **summarise(records, skip_seconds=args.skip_seconds, from_ns=args.from_ns, to_ns=args.to_ns), + "per_file": files} + text = json.dumps(report, indent=1, sort_keys=True) + if args.out: + with open(args.out, "x") as handle: + handle.write(text + "\n") + print(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/p184_local_ci.py b/scripts/p184_local_ci.py new file mode 100644 index 00000000..0011c90f --- /dev/null +++ b/scripts/p184_local_ci.py @@ -0,0 +1,690 @@ +"""P18.4 D5: run the Data Layer GitHub Actions CI locally against one exact commit. + +Purpose: the Data Layer repository is not pushed (owner rule), so `.github/workflows/ci.yml` never runs on +GitHub. This runner parses the workflow with yaml and replays every job's steps, in order and with the same +`run:` text, against a private copy of ``: `git archive` extracted per job (a throw-away git index is +added so `git diff` steps work), or a private clone when the checkout asks for full history +(`fetch-depth: 0`). Each step records name, command, exit code, duration and log path; `summary.json` +(schema qdl.p184.local-ci-receipt.v1) carries the sha and PASS/FAIL/SKIPPED per step; its SHA-256 is the +`--dl-ci-receipt-sha256` input of scripts/p184_consolidated_build.py only when the overall status is PASS. + +Boundary (what it will not do): +* never pushes, publishes or changes the source repository; `--out` must be outside it; +* never runs sudo: steps that call sudo are reported SKIPPED (requires_sudo); +* never downloads tools on the host or pulls images unless `--allow-pull` / `--allow-download` is given: + steps needing a missing image, the buf binary (a local bufbuild/buf: image is used instead), + trivy or a host-side curl are reported SKIPPED with the reason. Package fetches that a step performs + inside its own container (apt, cargo, pip) are part of that step and are not intercepted; +* never touches running roles: it refuses to start while an image tag or container name it would create + already exists, and every compose call uses an isolated COMPOSE_PROJECT_NAME; +* cleans what it created after each job: leftover containers/networks/volumes matching its ownership + predicate, images it built (removed by ID, only when every tag on the ID is one it created), BuildKit + cache records that appeared during the job (pruned by record ID), and the job's source copy. Counts go + into summary.json. The root disk is shared with the runtime (Kafka/Postgres, 2026-10-08 incident), so it + aborts when less than 40 GB is free on the output or Docker filesystem. + +usage: p184_local_ci.py --sha SHA --out DIR [--jobs a,b] [--dry-run] [--allow-pull] [--allow-download] +""" +from __future__ import annotations + +import argparse +import dataclasses +import datetime as dt +import hashlib +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import time +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA = "qdl.p184.local-ci-receipt.v1" +WORKFLOW = ".github/workflows/ci.yml" +EVENT = "push" +MIN_FREE_BYTES = 40 * 1024 ** 3 +DEFAULT_STEP_TIMEOUT_S = 3600 +HOST_PYTHON = "/usr/bin/python3" +# Images whose tags appear in run scripts; anything else is matched only by @sha256 digest. +KNOWN_REPOSITORIES = ("rust", "redis", "postgres", "alpine", "python", "debian", "apache/kafka", + "bufbuild/buf", "aquasec/trivy") +TAG_RE = re.compile(r"(? 0 for s in steps)) + + def test_conditions_follow_push_event_semantics(self): + self.assertEqual(ci.evaluate_condition("github.event_name == 'pull_request'", False)[0], False) + self.assertEqual(ci.evaluate_condition("always()", True), (True, "")) + self.assertFalse(ci.evaluate_condition(None, True)[0]) + self.assertTrue(ci.evaluate_condition(None, False)[0]) + self.assertFalse(ci.evaluate_condition("contains(github.ref, 'x')", False)[0]) + + def test_real_workflow_classification_offline(self): + present = {"bufbuild/buf:1.50.0", "rust:1.82-slim", "redis:7.2-alpine", "alpine:3.20"} + files = {"docker-compose.yml": "services:\n data_layer:\n build:\n dockerfile: Dockerfile\n", + "Dockerfile": "FROM python:3.12-slim@sha256:" + "1" * 64 + " AS builder\nFROM builder\n", + "scripts/phase5_migration_smoke.sh": 'image="${QDL_PHASE5_POSTGRES_IMAGE:-postgres:16-alpine}"\n'} + decisions, python_version, tools = {}, None, {"buf": False} + for step in self.steps: + decision = decide(step, present=present, python_version=python_version, tools=tools, files=files) + if decision.action == "setup" and step.uses.startswith("actions/setup-python"): + python_version = {"version": step.with_["python-version"], "host": decision.executor == "host"} + if decision.action == "setup" and step.uses.startswith("bufbuild/buf-setup-action"): + tools["buf"] = True + decisions[(step.job, step.index)] = decision + by_name = {(s.job, s.name): decisions[(s.job, s.index)] for s in self.steps} + self.assertEqual(by_name[("consolidated-source-preflight", + "Checkout exact CI revision with prerequisite history")].reason, "clone") + self.assertEqual(by_name[("contract-tests", "Checkout")].reason, "archive") + self.assertIn("requires_sudo", by_name[("unit-tests", "Prepare non-root runtime bind mounts")].reason) + build = by_name[("unit-tests", "Build data_layer image")] + self.assertEqual((build.action, build.blocking), ("skip", True)) + self.assertIn("python:3.12-slim@sha256:" + "1" * 64, build.reason) + self.assertIn("postgres:16-alpine", by_name[("unit-tests", "Run additive PostgreSQL migration gate")].reason) + self.assertEqual(by_name[("contract-tests", "Test Python generated contracts")].executor, "container") + self.assertIn("--allow-download", + by_name[("contract-tests", "Check Rust dependency, license and advisory policy")].reason) + sdk = by_name[("sdk-python310", "Set up declared minimum Python")] + self.assertEqual((sdk.action, sdk.blocking), ("skip", True)) + predicted = ci.predict(self.steps, decisions) + unit = [p for s, p in zip(self.steps, predicted) if s.job == "unit-tests"] + self.assertEqual(unit[-1][0], "RUN") # `if: always()` compose cleanup still runs after a blocking skip + self.assertEqual(unit[6][0], "SKIPPED") + + +class ClassificationTests(NoProcessTests): + def test_sudo_publication_and_unknown_actions_are_never_run(self): + self.assertIn("requires_sudo", decide(run_step("sudo scripts/prepare_nonroot_runtime.sh")).reason) + self.assertEqual(decide(run_step('gh release create "$TAG"')).action, "skip") + self.assertEqual(decide(run_step(None, uses="someone/else@v1")).action, "skip") + + def test_buf_uses_local_image_or_skips_without_download(self): + setup = run_step(None, uses="bufbuild/buf-setup-action@x", with_={"version": "1.50.0"}) + self.assertEqual(decide(setup).action, "skip") + ok = decide(setup, present={"bufbuild/buf:1.50.0"}) + self.assertEqual((ok.action, ok.images), ("setup", ("bufbuild/buf:1.50.0",))) + self.assertEqual(decide(run_step("buf lint\n")).action, "skip") + self.assertEqual(decide(run_step("buf lint\n"), tools={"buf": True}).action, "run") + + def test_missing_images_skip_unless_pull_allowed(self): + step = run_step("docker run --rm redis@sha256:" + "a" * 64 + " true\n") + self.assertEqual(decide(step).action, "skip") + self.assertEqual(decide(step, allow_pull=True).action, "run") + self.assertEqual(decide(step, present={"redis@sha256:" + "a" * 64}).action, "run") + + def test_own_build_outputs_are_not_required_images(self): + step = run_step(f"docker run --rm qdl-core:{SHA} x\nQDL_TEST_IMAGE=data-layer:v0.1.0 scripts/s.sh\n", + env={}) + self.assertEqual(ci.required_images(step, fake_reader({}), SHA), ()) + + def test_skipped_docker_build_skips_its_dependents_only(self): + unavailable = set() + build = run_step(f"docker build -f Dockerfile.qdl-core\n-t qdl-core:{SHA} .\n") + skipped = ci.propagate_unavailable(build, ci.Decision("skip", reason="missing base"), unavailable) + self.assertEqual(skipped.action, "skip") + self.assertEqual(unavailable, {f"qdl-core:{SHA}"}) + user = ci.propagate_unavailable(run_step(f"docker run qdl-core:{SHA}"), ci.Decision("run"), unavailable) + self.assertEqual(user.action, "skip") + other = ci.propagate_unavailable(run_step("docker run data-layer:v0.1.0"), ci.Decision("run"), unavailable) + self.assertEqual(other.action, "run") + + def test_dockerfile_bases_ignore_stages_and_args(self): + text = "ARG X=builder\nFROM rust@sha256:" + "b" * 64 + " AS builder\nFROM ${X} AS dep\nFROM builder\n" \ + "FROM debian@sha256:" + "c" * 64 + "\n" + self.assertEqual(ci.dockerfile_bases(text), ["rust@sha256:" + "b" * 64, "debian@sha256:" + "c" * 64]) + + def test_python_steps_use_container_when_pip_or_interpreter_differs(self): + pip = run_step("python -m pip install x\npython -m unittest t\n") + host = {"version": "3.12", "host": True} + self.assertEqual(decide(pip, python_version=host).executor, "container") + self.assertEqual(decide(run_step("python -B -m unittest t\n"), python_version=host).executor, "host") + other = {"version": "3.10", "host": False} + self.assertEqual(decide(run_step("python x.py\n"), python_version=other).executor, "container") + self.assertTrue(decide(run_step("docker run x\npython y.py\n"), python_version=other).blocking) + + +class SafetyTests(NoProcessTests): + def test_protected_compose_projects_are_never_owned(self): + for project in ci.PROTECTED_PROJECTS: + data = {"Name": "/kn-kafka", "Config": {"Labels": {"com.docker.compose.project": project}}} + self.assertFalse(ci.owned("container", data, {project})) + mine = {"Name": "/x", "Config": {"Labels": {"com.docker.compose.project": "qdl-localci-abc"}}} + self.assertTrue(ci.owned("container", mine, {"qdl-localci-abc"})) + self.assertFalse(ci.owned("container", {"Name": "/redis_marketdata", "Config": {"Labels": {}}}, set())) + self.assertFalse(ci.owned("volume", {"Name": "kn-x", "Labels": {}}, set())) + + def test_overall_status_distinguishes_incomplete_from_pass(self): + ok = {"status": "PASS"} + pr = {"status": "SKIPPED", "reason": "condition false for local push event: x"} + self.assertEqual(ci.overall_status([ok, pr]), "PASS") + self.assertEqual(ci.overall_status([ok, {"status": "SKIPPED", "reason": "requires_sudo"}]), "INCOMPLETE") + self.assertEqual(ci.overall_status([ok, {"status": "FAIL"}]), "FAIL") + + def test_disk_floor_is_forty_gigabytes(self): + self.assertEqual(ci.MIN_FREE_BYTES, 40 * 1024 ** 3) + + +if __name__ == "__main__": + unittest.main() From 713aa904537aeb6911c9156a74df88b6a6699e80 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 06:28:38 +0000 Subject: [PATCH 094/147] fix(phase3): TS readiness samples the configured writer and fails on lost observation (Codex review); stage 100e journal --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 30 ++++++++++++++++++++++ qdl/certification/phase3_consumer_load.py | 9 +++++-- scripts/phase3_consumer_load_acceptance.py | 15 ++++++++--- tests/test_phase3_target_driver.py | 16 +++++++++++- 4 files changed, 64 insertions(+), 6 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 3418f2d3..1d3f7295 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71740,3 +71740,33 @@ Status: tested locally, not deployed. - kn_mark_hot_memo, kn_query_backend, kn_hot_book_invalidation and kn_hot_view OK (33 Redis skips). - Lesson: a shared cache on a path with a monotonic gate needs a test for that path before rollout; the rc4 unit tests covered only the memo itself. + +### D3 stage 100e on rc5: 29/32; Codex review of the measurement harness accepted - 2026-10-10 (Claude) + +Run 05:49-06:26Z, Query rc5 (ec89a31, sha256:30261623e5d0), SDK profile, frozen budget. Guard: 92 samples, 1 DEGRADED +63/64 at 06:18:37. The MARK error watcher read 0 errors in 34 of 40 minutes (max 7 in one minute, around the OKX +disconnect). The rc4 regression did not recur. +- PASS: every latency class (p95 34-89 ms); startup. +- FAIL requests:no_failures (1 DATA_STALE) and streams:no_errors (3) on OKX BNB-USDT-SWAP at 06:11:48-54Z. + - The OKX PUBLIC session disconnected: `qdl_native_session_disconnected` generation 167 at 06:11:52Z. + - DL refused with the correct type (STALE, SOURCE_SESSION_UNKNOWN, freshness 6045 ms > 5000). + - The OKX ingestor has logged 37 disconnects since 2026-10-09 00:00Z, about 1.2 per hour. Reconnect speed is under + investigation. +- FAIL g5:COLD_2500:OKX 7531.5 ms. Binance took 4288.6 ms, and OKX finished one warmup later (third time in four + runs). Same-replica queuing vs client-side serial validation in the single cold worker is not yet distinguished. +- Codex review (accepted after checking the code): + 1. The TS readiness gate was scored on 2 samples. The periodic observer called `_ts_heartbeat()` with its default + container `market_data_service` instead of `market_data_service_p184w`. Receipts: stage100 2 good / 157 + unreadable, stage100b 2/155, stage50b 2/55; unreadable samples were dropped before scoring. + - Fix (this commit): the observer samples the configured writer; None samples count as unreadable; + `expected_samples` is recorded. + - `ts:ready_60_every_sample` now fails on any unreadable sample or on coverage below expected. + - Tests: test_lost_observation_fails_readiness; 50 driver/stage tests OK. + - Earlier TS-gate PASSes rest on 2 samples plus the independent 20 s guard log, which was not part of the + verifier. They are re-certified only by a run with this fix. + 2. The supervisor and MARK watcher stopped processes and containers by generic name patterns. + - The rc5 supervisor even self-matched a stale shell and never saw 100e finish. + - Replaced by run-scoped (process-group) versions; the old ones were stopped. + 3. The MARK watcher counted an unreadable log as 0 and its e copy still matched `stage100d.sh`. Replaced: UNKNOWN on + a read failure. + 4. The rc5 regression test mocked the monotonic check. A real-ordering case is being added. diff --git a/qdl/certification/phase3_consumer_load.py b/qdl/certification/phase3_consumer_load.py index 9e49eb78..e80de734 100644 --- a/qdl/certification/phase3_consumer_load.py +++ b/qdl/certification/phase3_consumer_load.py @@ -907,8 +907,13 @@ def sample_ready(item: Mapping[str, object]) -> bool: and not item.get("unhealthy")) not_ready = [item for item in samples if not sample_ready(item)] - gates.append(_gate("ts:ready_60_every_sample", bool(samples) and not not_ready, - samples=len(samples), not_ready=len(not_ready))) + # Lost observation is a failure, never a skipped sample (Codex review 2026-10-10). + unreadable = len(trading_system.get("unreadable_samples", [])) + expected = trading_system.get("expected_samples") + covered = isinstance(expected, int) and expected > 0 and len(samples) == expected + gates.append(_gate("ts:ready_60_every_sample", bool(samples) and not not_ready and not unreadable and covered, + samples=len(samples), not_ready=len(not_ready), unreadable=unreadable, + expected=expected)) errors = [item.get("v2_error") for item in samples if isinstance(item.get("v2_error"), int)] gates.append(_gate("ts:v2_error_not_increased", bool(errors) and max(errors) - min(errors) <= ts_budget["max_v2_error_increase"], diff --git a/scripts/phase3_consumer_load_acceptance.py b/scripts/phase3_consumer_load_acceptance.py index 1a1aeba4..72ef8423 100644 --- a/scripts/phase3_consumer_load_acceptance.py +++ b/scripts/phase3_consumer_load_acceptance.py @@ -3301,7 +3301,10 @@ def run_target_host(args: argparse.Namespace) -> int: try: process, telemetry, runtime_fault = _run_monitored_client( command, monitored, timeout=duration + 720, - observer=None if matrix else _ts_heartbeat, observations=ts_samples, + # The periodic observer samples the SAME writer as the first/last sample + # (Codex review 2026-10-10: it used the default container and 155 of 157 + # stage-100b samples were unreadable, while the gate scored only 2). + observer=None if matrix else (lambda: _ts_heartbeat(ts_container)), observations=ts_samples, ) finally: try: @@ -3334,8 +3337,14 @@ def run_target_host(args: argparse.Namespace) -> int: if isinstance(value, dict) and "failure" in value: client_failure = str(value["failure"])[:500] break - trading_system: dict[str, object] = {"samples": [item for item in ts_samples if item and "ready" in item], - "unreadable_samples": [item for item in ts_samples if item and "error" in item]} + # Every sample is either readable or counted as unreadable (None included); + # expected_samples lets the verifier fail lost coverage instead of skipping it. + trading_system: dict[str, object] = { + "samples": [item for item in ts_samples if item and "ready" in item], + "unreadable_samples": [item if item else {"error": {"type": "NoSample"}} + for item in ts_samples if not item or "ready" not in item], + "expected_samples": len(ts_samples), + } evaluation = None if not matrix: run_minutes = max(1e-6, (time.monotonic() - started) / 60.0) diff --git a/tests/test_phase3_target_driver.py b/tests/test_phase3_target_driver.py index 2c317d04..eaea3fb0 100644 --- a/tests/test_phase3_target_driver.py +++ b/tests/test_phase3_target_driver.py @@ -170,7 +170,8 @@ def _passing_receipt(final: bool = False) -> dict[str, object]: def _passing_ts() -> dict[str, object]: sample = {"ready": 60, "demanded": 60, "fallback": 0, "v2_error": 7} - return {"samples": [sample] * 12, "disconnect_codes": {"DATA_STALE": 4}, + return {"samples": [sample] * 12, "unreadable_samples": [], "expected_samples": 12, + "disconnect_codes": {"DATA_STALE": 4}, "run_disconnects_per_minute": 2.0, "baseline_disconnects_per_minute": 1.5} @@ -229,6 +230,19 @@ def test_trading_system_regression_fails(self): ts["run_disconnects_per_minute"] = 30.0 self.assertIn("ts:disconnects_within_baseline", self._evaluate(ts=ts)["failed_gates"]) + def test_lost_observation_fails_readiness(self): + # Codex review 2026-10-10: 155 unreadable periodic samples were skipped and the gate passed on 2. + ts = _passing_ts() + ts["unreadable_samples"] = [{"error": {"type": "IndexError"}}] + ts["expected_samples"] = 13 + self.assertIn("ts:ready_60_every_sample", self._evaluate(ts=ts)["failed_gates"]) + ts = _passing_ts() + ts["expected_samples"] = 157 + self.assertIn("ts:ready_60_every_sample", self._evaluate(ts=ts)["failed_gates"]) + ts = _passing_ts() + del ts["expected_samples"] + self.assertIn("ts:ready_60_every_sample", self._evaluate(ts=ts)["failed_gates"]) + def test_quiet_or_disconnected_slices_cannot_bypass_readiness(self): for state in ("STALE", "DISCONNECTED", "UNKNOWN"): doge = {"provider": "OKX", "symbol": "DOGE-USDT-SWAP", "feed": "QUOTE", "state": state} From 7f974c354f8df4a55f7267aa87a79ed3c909c27e Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 06:30:33 +0000 Subject: [PATCH 095/147] feat(phase3): cold reads record serving replica and transport vs client time (G5 split) --- scripts/phase3_consumer_load_acceptance.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/phase3_consumer_load_acceptance.py b/scripts/phase3_consumer_load_acceptance.py index 72ef8423..96671e5e 100644 --- a/scripts/phase3_consumer_load_acceptance.py +++ b/scripts/phase3_consumer_load_acceptance.py @@ -1722,6 +1722,8 @@ async def run_inside() -> dict[str, object]: ("FUNDING_RATE", "REFERENCE_BATCH"): "FUNDING_REFERENCE", } _SERVED_BY: ContextVar[str | None] = ContextVar("phase3_served_by", default=None) +# Wall time at which the attributed replica returned the transport response (P18.4 G5 split: server+network vs client). +_SERVED_DONE_NS: ContextVar[int | None] = ContextVar("qdl_phase3_served_done_ns", default=None) _QUERY_METHODS = frozenset({ "snapshot", "warmup", "warmup_batch", "reference_batch", "feed_status", "instruments", "instrument", @@ -1768,7 +1770,9 @@ def __getattr__(self, attribute): async def call(*args, **kwargs): _SERVED_BY.set(self.name) - return await value(*args, **kwargs) + result = await value(*args, **kwargs) + _SERVED_DONE_NS.set(time.time_ns()) + return result return call @@ -2239,8 +2243,18 @@ async def _target_cold(*, client, product, recorder) -> None: started = time.monotonic() entry = {"venue": product.venue, "native_symbol": product.native_symbol, "rows": rows, "window": recorder.window(started)} + _SERVED_BY.set(None) + _SERVED_DONE_NS.set(None) + sent_ns = time.time_ns() try: response = await client.warmup(replace(sdk_requirement(product), warmup_limit=rows)) + # G5 split (2026-10-10): which replica served it, transport time (server + network + body read) and the + # client remainder (SDK model parse + per-row validation), so queueing is told apart from client CPU. + done_ns = _SERVED_DONE_NS.get() + entry["served_by"] = _SERVED_BY.get() + entry["sent_at_ns"] = sent_ns + if done_ns is not None: + entry["transport_ms"] = round((done_ns - sent_ns) / 1e6, 3) entry["returned"] = len(response.data) for item in response.data: validate_product_view(product, item, require_current_quality=False) @@ -2257,6 +2271,8 @@ async def _target_cold(*, client, product, recorder) -> None: recorder.error(operation="COLD", product=_product_evidence(product), rows=rows, error=_safe_error(error)) entry["ms"] = round((time.monotonic() - started) * 1000.0, 3) + if "transport_ms" in entry: + entry["client_ms"] = round(entry["ms"] - entry["transport_ms"], 3) recorder.cold.append(entry) From 4e19966d1b51a494555dd355375a0df00e76c9cf Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 06:33:00 +0000 Subject: [PATCH 096/147] test(query): MARK hot memo vs backup through the real ordering gates (rc5 review) Real MARK/INDEX golden records in ProductView/CanonicalHotView; _hot_candidate comparisons, _hot_monotonic/_hot_boundaries, lineage and quality run unmocked. (a) cache advancing past the shared memo keeps the cache view; backups do a fresh lookup inside the window; (b) rc4 (shared backup) refuses on a cache outage (DEPENDENCY_UNAVAILABLE) and with primary behind (DATA_NOT_READY HOT_BACKUP_BEHIND_RETURNED_VIEW) where rc5 serves the fresh record; (c) IDR-084 newer hot record still wins. No production change. --- tests/test_kn_mark_hot_memo.py | 239 +++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) diff --git a/tests/test_kn_mark_hot_memo.py b/tests/test_kn_mark_hot_memo.py index 88157fea..13f12c10 100644 --- a/tests/test_kn_mark_hot_memo.py +++ b/tests/test_kn_mark_hot_memo.py @@ -141,5 +141,244 @@ def test_opportunistic_path_shares(self): self.assertEqual(hot.calls, 1) +class RealOrderingChecksTests(unittest.TestCase): + """The shared/direct split through the REAL ordering gates (nothing mocked inside the backend). + + Real canonical MARK/INDEX golden records (retimed with their Rust lineage + kept whole, as in tests/test_kn_query_backend.py) in ProductView and + CanonicalHotView objects; the cache reader is a Redis-free stand-in that + returns one ProductView, as in tests/test_kn_hot_view.py + HotQuerySelectionTests. ``_hot_candidate``'s boundary/offset comparisons, + ``_hot_monotonic``/``_hot_boundaries``, ``_parsed``, ``_validate_records`` + and the quality oracle all run unmodified. Records, in canonical order: + C10 (record 10, boundary 20) < H1 (30, 40) < H2 (50, 60) < H3 (70, 80).""" + + OFFSETS = {"C10": (10, 20), "H1": (30, 40), "H2": (50, 60), "H3": (70, 80)} + + @classmethod + def setUpClass(cls): + from pathlib import Path + + from qdl.marketdata.v2 import market_data_pb2 + from qdl.runtime.stable_catalog import StableSourceCatalog + from tests.test_kn_bar_legacy_import import golden_records + from tests.test_kn_query_backend import binding_of + + cls.catalog = StableSourceCatalog.load( + Path(__file__).resolve().parents[1] / "config/v2/stable-source-bindings.yaml") + marks = ( + (binding_of(cls.catalog, pk, payload), payload) + for pk, payload in golden_records(bar=False) + if market_data_pb2.EventEnvelope.FromString(payload).WhichOneof("payload") == "mark_index_price" + ) + cls.binding, cls.golden = next( + (binding, payload) for binding, payload in marks + if binding.authoritative and binding.source_role == "PRIMARY" + ) + + def setUp(self): + import time as _time + + from qdl.marketdata.v2 import market_data_pb2 + from qdl.query import ConsumerGrade, DataRequirement + from qdl.runtime.kn_bar_readback import binding_product_key + from tests.test_kn_query_backend import retimed_mark_index + + self.now = _time.time_ns() + self.mono = [10_000_000_000] + self.lpk = binding_product_key(self.binding, "paper") + + def payload(age_ms): + env = market_data_pb2.EventEnvelope.FromString( + retimed_mark_index(self.golden, now_ns=self.now, age_ms=age_ms)) + env.normalized_at_ns = env.received_at_ns + 1 + env.published_at_ns = env.received_at_ns + 2 + return env.SerializeToString(deterministic=True) + + self.payloads = {"C10": payload(900), "H1": payload(400), "H2": payload(200), "H3": payload(100)} + self.primary = self.cache("C10") + self.reply = self.hot("H1") + self.calls = 0 + self.requirement = DataRequirement( + instrument_uid=self.binding.instrument.instrument_uid, feed=self.binding.feed, + consumer_grade=ConsumerGrade.EXECUTION, source_policy_id=self.binding.source_policy_id, + max_freshness_ms=2000, + ) + + def cache(self, name): + from qdl.runtime.kn_market_cache import CacheRow, ProductView, SourceBoundary + + record, mark = self.OFFSETS[name] + return ProductView(self.lpk, 1, 1, (CacheRow(self.payloads[name], record),), SourceBoundary("topic", 0, mark)) + + def hot(self, name): + from qdl.runtime.kn_hot_view import CanonicalHotView + from qdl.runtime.kn_market_cache import CacheRow, SourceBoundary + + record, mark = self.OFFSETS[name] + return CanonicalHotView(self.lpk, (CacheRow(self.payloads[name], record),), SourceBoundary("topic", 0, mark)) + + def backend(self, *, rc4=False): + from qdl.runtime.kn_query_backend import KnMarketCacheQueryBackend + + def primary(key): + if isinstance(self.primary, Exception): + raise self.primary + return self.primary + + def latest(binding, key): + self.calls += 1 + if isinstance(self.reply, Exception): + raise self.reply + return self.reply + + backend = KnMarketCacheQueryBackend( + SimpleNamespace(environment="paper", latest=primary), self.catalog, schema_digest="a" * 64, + topic_id="topic", clock_ns=lambda: self.now, monotonic_ns=lambda: self.mono[0], + hot_client=SimpleNamespace(latest=latest), + ) + if rc4: + # rc4 (before ec89a31): every _hot_candidate caller read through the shared memo. + real = backend._hot_candidate + backend._hot_candidate = lambda binding, primary=None, *, shared=False: real(binding, primary, shared=True) + return backend + + def name_of(self, payload): + return next(name for name, value in self.payloads.items() if value == payload) + + def served(self, backend): + """Which record the Query latest path (_latest_view) served, by its cursor boundary.""" + from qdl.runtime.kn_query_backend import parse_placeholder + + mark = parse_placeholder(backend.latest(self.requirement).cursor)[2] + return next(name for name, (_record, boundary) in self.OFFSETS.items() if boundary == mark) + + def recorded(self, backend): + return backend._hot_boundaries[self.binding.binding_id][1] + + def fill_memo_with_h1(self, backend): + # IDR-084: cache C10 is older than hot H1 -> H1 served, memo holds H1, H1 recorded. + stored, state = backend.latest_stored_event(self.binding) + self.assertEqual((state, self.name_of(stored.event.payload), self.calls), ("OK", "H1", 1)) + self.assertEqual(self.recorded(backend), 30) + + # (c) IDR-084 preserved ------------------------------------------------ + + def test_c_newer_hot_record_inside_the_window_is_used_by_the_idr084_branch(self): + backend = self.backend() + # The valid, usable cache view C10 is replaced by the newer committed H1 (one lookup). + self.assertEqual(self.served(backend), "C10") # the Query path alone keeps a usable cache view + self.fill_memo_with_h1(backend) + # The cache catches up to exactly H1; inside the window the shared H1 passes the real + # gates (equal offset, equal bytes) and is not newer, so the cache view stays. + self.primary = self.cache("H1") + self.reply = self.hot("H2") + self.mono[0] += MARK_HOT_MEMO_NS - 1 + stored, state = backend.latest_stored_event(self.binding) + self.assertEqual((state, self.name_of(stored.event.payload), self.calls), ("OK", "H1", 1)) + # Window over: a fresh lookup on the same IDR-084 branch, and the newer H2 wins. + self.mono[0] += 1 + stored, state = backend.latest_stored_event(self.binding) + self.assertEqual((state, self.name_of(stored.event.payload), self.calls), ("OK", "H2", 2)) + self.assertEqual(self.recorded(backend), 50) + # A cache view behind the recorded record never reaches the shared branch: inside the + # window it takes the direct backup (a fresh lookup) and gets the newest H3. + self.primary = self.cache("C10") + self.reply = self.hot("H3") + self.mono[0] += 1 + stored, state = backend.latest_stored_event(self.binding) + self.assertEqual((state, self.name_of(stored.event.payload), self.calls), ("OK", "H3", 3)) + + # (a) primary advances past the shared memo ----------------------------- + + def test_a_primary_advances_past_the_shared_memo(self): + backend = self.backend() + self.fill_memo_with_h1(backend) + # The cache catches up to H2 while the memo (10 ms old) still holds H1; hot is now at H3. + self.primary = self.cache("H2") + self.reply = self.hot("H3") + self.mono[0] += 10_000_000 + # The real offset comparison refuses the shared H1 against primary H2 ... + with self.assertRaisesRegex(HotViewUnavailable, "HOT_BACKUP_BEHIND_PRIMARY"): + backend._hot_candidate(self.binding, self.primary, shared=True) + # ... so the IDR-084 branch keeps the cache view: never H1, never a refusal. + stored, state = backend.latest_stored_event(self.binding) + self.assertEqual((state, self.name_of(stored.event.payload)), ("OK", "H2")) + self.assertEqual(self.calls, 1, "memo hit, no hot lookup") + self.assertEqual(self.recorded(backend), 50) + # MARK-view backup (prefer_hot) inside the window: a fresh lookup, H3 accepted. + stored, state = backend.latest_stored_event(self.binding, prefer_hot=True) + self.assertEqual((state, self.name_of(stored.event.payload), self.calls), ("OK", "H3", 2)) + # _latest_view backup: primary H2 is now behind the recorded H3 -> fresh lookup, H3 accepted. + self.assertEqual(self.served(backend), "H3") + self.assertEqual(self.calls, 3) + self.assertEqual(self.recorded(backend), 70) + # The memo was never refreshed by the backup reads and still holds H1 inside its window. + taken_ns, memo = backend._mark_hot_memo[self.binding.binding_id] + self.assertLess(self.mono[0] - taken_ns, MARK_HOT_MEMO_NS) + self.assertEqual(memo.rows[0].canonical, self.payloads["H1"]) + + # (b) the rc4 failure --------------------------------------------------- + + def _cache_outage_sequence(self, backend): + """Memo H1; the cache serves H2 (recorded); then a cache read fails while hot is at H3.""" + from qdl.runtime.kn_market_cache import KnCacheError + + self.fill_memo_with_h1(backend) + self.primary = self.cache("H2") + self.mono[0] += 10_000_000 + self.assertEqual(self.served(backend), "H2") # usable and monotonic: no hot lookup + self.assertEqual(self.recorded(backend), 50) + self.primary = KnCacheError("cache read timed out") + self.reply = self.hot("H3") + self.mono[0] += 10_000_000 + return self.served(backend) + + def test_b_rc4_shared_backup_refuses_on_a_cache_outage(self): + from qdl.query.contracts import CanonicalErrorCode + from qdl.query.results import QueryBackendError + + backend = self.backend(rc4=True) + with self.assertRaises(QueryBackendError) as refused: + self._cache_outage_sequence(backend) + self.assertIs(refused.exception.problem.code, CanonicalErrorCode.DEPENDENCY_UNAVAILABLE) + self.assertEqual(self.calls, 1, "the rc4 backup never asked the hot reader") + # Why: the real monotonic gate refuses the stale shared H1 behind the recorded H2. + self.assertFalse(backend._hot_monotonic(self.binding, self.hot("H1"))) + with self.assertRaisesRegex(HotViewUnavailable, "HOT_BACKUP_BEHIND_RETURNED_VIEW"): + backend._hot_candidate(self.binding, None, shared=True) + + def test_b_rc5_direct_backup_serves_the_fresh_record_on_a_cache_outage(self): + backend = self.backend() + self.assertEqual(self._cache_outage_sequence(backend), "H3") + self.assertEqual(self.calls, 2) + self.assertEqual(self.recorded(backend), 70) + + def test_b_stale_record_on_the_backup_path_is_refused_primary_behind(self): + """The backup called with a stale record (the memo forced onto it) vs a fresh lookup.""" + from qdl.query.contracts import CanonicalErrorCode + from qdl.query.results import QueryBackendError + + rc4, rc5 = self.backend(rc4=True), self.backend() + for backend in (rc4, rc5): + self.calls = 0 + self.reply = self.hot("H1") + self.fill_memo_with_h1(backend) + # Any read records H2 (here the cache served it), then the cache view is C10 again + # (another replica's older generation); hot is at H2; the memo window is still open. + backend._hot_monotonic(self.binding, self.cache("H2"), record=True) + self.reply = self.hot("H2") + self.mono[0] += 10_000_000 + with self.assertRaises(QueryBackendError) as refused: + self.served(rc4) + self.assertIs(refused.exception.problem.code, CanonicalErrorCode.DATA_NOT_READY) + self.assertIn("HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND: HOT_BACKUP_BEHIND_RETURNED_VIEW", + str(refused.exception)) + self.assertEqual(rc4.latest_stored_event(self.binding, prefer_hot=True), (None, "BACKUP_UNAVAILABLE")) + self.assertEqual(self.served(rc5), "H2") + stored, state = rc5.latest_stored_event(self.binding, prefer_hot=True) + self.assertEqual((state, self.name_of(stored.event.payload)), ("OK", "H2")) + + if __name__ == "__main__": unittest.main() From 1fb68ed7f1a1e836596761d3c7848028c5f674c3 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 06:44:40 +0000 Subject: [PATCH 097/147] docs: add evidence-ranked DL architecture closure advisory --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 32 ++ ...84-astra-architecture-advisory-20261010.md | 450 ++++++++++++++++++ 2 files changed, 482 insertions(+) create mode 100644 upgrade/evidence/p184-astra-architecture-advisory-20261010.md diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 1d3f7295..e558f7e2 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71770,3 +71770,35 @@ disconnect). The rc4 regression did not recur. 3. The MARK watcher counted an unreadable log as 0 and its e copy still matched `stage100d.sh`. Replaced: UNKNOWN on a read failure. 4. The rc5 regression test mocked the monotonic check. A real-ordering case is being added. + +### Astra advisory architecture review handoff - 2026-10-10 + +Status REVIEW_HANDOFF_COMPLETE_IMPLEMENTATION_NOT_AUTHORIZED; owner requested a +detailed advisory Markdown for Claude, not implementation or rollout. Review +baseline `4e19966`. Scope existing +Phase D correctness, hot-read contention/deadlines, quiet-book liveness, warmup, +acceptance and release convergence. Guide: Kafka-native architecture review +sections18.3-18.5 and Execution Handoff Hot Backup. Preserve source truth, +freshness/generation/ordering, prior receipts and Claude's concurrent edits. +Planned artifact upgrade/evidence/p184-astra-architecture-advisory-20261010.md. +Only docs/journal edits; optional network-none source reproduction, no shared +runtime calls, no builds/push/merge. Record findings as evidence-ranked options, +not new gates or an instruction to rebuild architecture; cleanup any owned test. + +Completed advisory: [Astra architecture and closure review](upgrade/evidence/p184-astra-architecture-advisory-20261010.md). +Seven evidence-ranked review groups R1-R7 include alternatives, trade-offs, +focused tests and recommended closure order; these are advisory, not new gates. +Acknowledges fixes 713aa90/7f974c3/4e19966 instead of reopening superseded findings. +R1 reproduction uses golden MARK bytes and actual parser/lineage/ordering with +stand-in transports: hard GAP and topic mismatch both return old primary C10/OK; +transient HOT_READER_BUSY does too. Three behavioral observations, not acceptance +PASSes or proof of incorrect live execution. R4 newly identified timer interaction +is source-confirmed, default-off and still needs an async silent-peer fault test. +Command: existing image 77682eea5054, docker run --rm --network none, read-only +source, Python -B; full recipe and resource bounds in advisory section 13. +Exit 0; disposable qdl-astra-review-20261010-fence confirmed absent afterwards. +No new image, network, volume, credential or shared mutation; no full suite rerun. +No broad cleanup or disk reclamation claimed; retain active/rollback artifacts +while Claude works. Only this plan and advisory changed; no push/merge/rollout. +Remaining decision: Claude assesses each recommendation and selects or disproves +options against existing gates; this review does not certify release readiness. diff --git a/upgrade/evidence/p184-astra-architecture-advisory-20261010.md b/upgrade/evidence/p184-astra-architecture-advisory-20261010.md new file mode 100644 index 00000000..af8bb79b --- /dev/null +++ b/upgrade/evidence/p184-astra-architecture-advisory-20261010.md @@ -0,0 +1,450 @@ +# P18.4 Data Layer: Astra Architecture And Closure Advisory + +Date: 2026-10-10 UTC. Reviewer: Astra/Codex. Audience: Bobby and Claude. +Status: REVIEW_HANDOFF; recommendations, not implementation instructions, +new gates, runtime approval, or a release certificate. +Reviewed source: `4e19966d1b51a494555dd355375a0df00e76c9cf`, branch +`fix/p184-dl-closure`, `/home/bobby/.worktrees/data-layer-p184-closure`. +Main journal: `DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md`, entry +"Astra advisory architecture review handoff - 2026-10-10". + +## 1. Decision Summary + +My recommendation is to KEEP the approved Kafka-native architecture and close +specific implementation/qualification risks. The material reviewed does not +justify a new service topology, another broad migration, or another full-catalogue +investigation. It also does not establish that the entire release is stable yet. + +The highest-value sequence appears to be: + +1. Decide hard-refusal handling in the opportunistic MARK path (R1). +2. Test silent-socket recovery before enabling quiet-book proof (R4). +3. Attribute hot-reader contention and cold-warmup time with focused tests (R2/R3/R5). +4. Complete affected proof and one converged acceptance, preserving old failures. + +Claude can choose alternatives, combine slices, or disprove a finding with a +counterexample grounded in the actual contract. A recommendation below is not a +new "must-have" feature. The existing owner-approved invariants and gates remain +the authority; choosing a different implementation does not require adopting my +preferred design. Claims should match what the selected evidence actually proves. + +## 2. Scope, Evidence, And Architecture + +### What was examined + +- Approved architecture guide, sections 18.3-18.5 and Execution Handoff Hot Backup: + [guide](../DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md). +- Rust Hub hot reader, native Binance ingest/liveness, core L2 proof generation, + projector transactional pipe and fenced Redis apply. +- Python KN cache reader, Query selection/memo, hot client, cold-work scheduling. +- Current MARK tests, stage-100 evaluator/driver, latest plan and capacity receipts. +- Prior supervisor/readiness findings and Claude's subsequent fixes. + +This is a targeted architecture/code review, NOT an exhaustive security audit, +all-provider verification, or a fresh production load test. No source application +changes, shared runtime actions, provider calls, credentials or orders were made +by this review. One disposable network-none Python container reproduced R1. + +### Evidence vocabulary + +- REPRODUCED_LOCAL: source executed with explicitly described test dependencies. +- SOURCE_CONFIRMED: exact control flow/data structure observed; not automatically + a demonstrated production incident. +- HYPOTHESIS: plausible causal explanation needing attribution. +- ACCEPTANCE_GAP: evidence cannot yet support a particular breadth of claim. +- REPORTED: journal/receipt result inherited from Claude, not rerun here. + +### What is structurally sound + +The serving model remains: + + provider -> native ingestor -> raw Kafka -> Rust core + | + committed canonical + | | + Rust Stream + replay projector A + | | + | Kafka state topics + | | + | projector B + | | + | Redis market cache + | | + | Python Query + | | + +-> SDK <- - - + + | + TS / alpha + +Query can also request the existing Stream hot view. Stage A writes state and +input offsets in +one Kafka transaction; stage B applies with Redis owner/generation fencing. +Redis market data is rebuildable, not the financial or durable source of truth. +There is no evidence in the reviewed path of returning to six SQLite spool writers. +Class names such as StableSpoolQueryBackend alone do not prove spool I/O: the KN +backend overrides the read path and reuses domain derivation/quality logic. + +Positive anchors: +- `rust/qdl-projector/src/kafka_pipe.rs:196-240`: shared transaction deadline, + bounded QueueFull handling, transactional offset commit. +- `rust/qdl-projector/src/apply.lua:82-99`: ownership/pointer checks before apply; + exact decimal coordinate handling, checkpoint in the atomic script. +- `qdl/runtime/kn_market_cache.py:71-83`: atomic latest payload/pointer/watermark read; + BAR read checks generation and detects changes across page reads. +- Query uses canonical bytes, source coordinate and lineage checks for the hot view. +- The backup is projector-path redundancy on existing Stream roles, not independent + provider/host HA. Shared Kafka, quota Redis and host remain dependencies. + +## 3. Finding Index + +| ID | Review priority | Confidence / status | Suggested focus | +| --- | --- | --- | --- | +| R1 | P1 correctness | REPRODUCED_LOCAL | Do not downgrade a hard hot-view refusal to an old MARK OK | +| R2 | P2 availability | SOURCE_CONFIRMED; incident attribution incomplete | Isolate hot lookup from unrelated partitions and long ring scans | +| R3 | P2 bounded latency | SOURCE_CONFIRMED | Include memo lock wait in the same operation deadline | +| R4 | P1 when proof flag enabled | SOURCE_CONFIRMED timer interaction; needs async fault reproduction | Quiet-book timer can defer socket timeout indefinitely | +| R5 | P2 performance/qualification | Measured failure; causal split still open | Attribute cold queue/GIL/client validation before further tuning | +| R6 | P2 diagnostics | REPORTED failures plus source error taxonomy | Preserve bounded typed refusal reason at the failing layer | +| R7 | P2 release evidence | ACCEPTANCE_GAP | Match final code, SDK, flags, schema and recovery scope | + +Priority is review triage, not a demand to implement every option before release. +R4 concerns the new opt-in path; flag-off serving is a different claim. R2/R3 +may be closed by bounded measurements without structural changes if they meet the +existing gates. No finding here asserts that live orders used corrupt prices. + +## 4. R1: Hard Hot-View Refusal Can Become Old Primary OK + +### Evidence and impact + +`qdl/runtime/kn_query_backend.py:436-495`, particularly the opportunistic MARK +branch at 469-479. `_hot_candidate` maps HotViewGap to OPEN_SEQUENCE_GAP and +rejects topic/partition mismatch as DATA_NOT_READY. The outer branch preserves +only INTERNAL_ERROR as INTEGRITY; other QueryBackendError values become +`candidate=None`. It then parses/validates the primary and returns `stored, OK`. + +Local reproduction used the existing RealOrderingChecksTests fixture at +`tests/test_kn_mark_hot_memo.py`, REAL golden canonical MARK bytes and the actual +backend parser, lineage checks and monotonic checks. Only cache/hot transports +were stand-ins. Results on 4e19966: + +| Hot result | Returned primary | State | +| --- | --- | --- | +| HotViewGap(HOT_RECORD_GAPPED) | C10 | OK | +| Hot view with different topic identity | C10 | OK | +| HotViewUnavailable(HOT_READER_BUSY) | C10 | OK | + +The first two are hard source-validity questions; the third can be a legitimate +availability fallback. This proves backend refusal suppression, NOT that a full +public request or TS intent was incorrectly approved in production. Session and +component checks later in the caller may independently reject the old record. +Whether MARK currently emits every GAP variant is separate from the confirmed +branch behavior; topic mismatch does not depend on a MARK GAP event existing. + +Related taxonomy: `qdl/runtime/kn_hot_view.py:100-150` collapses protocol/auth and +invalid reply identity/payload into HotViewUnavailable. These deserve explicit +review before treating every HotViewUnavailable as harmless transient loss. + +### Options + +A. Preferred narrow option: classify hard versus transient outcomes explicitly. +Propagate fence/generation/gap/integrity to a typed refusal; retain an independently +valid primary only for transient transport/busy/capacity failures. This can be a +small selection-path change with a focused reason table rather than a new cache. + +B. Carry a typed hot-read outcome (available, transient unavailable, hard refusal) +through both normal and prefer_hot branches. Slightly more code, less chance of +future exception subclasses accidentally changing the fallback policy. + +C. Keep the current fallback ONLY if Claude establishes a contract proof that a +particular refusal cannot invalidate this primary. Document that narrow case; +"the primary is fresh" alone does not prove generation/authority consistency. + +Suggested test matrix: valid primary + transient busy; hard gap; wrong topic; +wrong partition; same offset/different payload; expired session; older hot record; +newer primary; recovery after fence. Exercise both latest_stored_event and the +MARK component-quality/TS at-use path. Existing ordinary E03/E04 tests are unrelated. + +Suggested closure evidence: counterfactual regression for hard refusals, positive +primary fallback retained for transient errors, then affected packaged read tests. +No need to manufacture a production fault just to confirm this selection logic. + +## 5. R2: Hot Reader Couples Unrelated Products + +### Evidence and limits + +`rust/qdl-stream-gateway/src/hub.rs:333-447` loops over all partitions, acquires +each partition via try_lock, and walks ring entries backward. A busy unrelated +partition can return HOT_READER_BUSY even if the requested product has a valid +committed record elsewhere. Looking up a quiet product may scan much more than +one recent active product. Per-record deadline checks bound scanning time but +do not remove contention. The scan holds the partition lock used by dispatch. + +This is SOURCE_CONFIRMED. The stage-100 DATA_NOT_READY detail was attributed to +HOT_READER_BUSY in Claude's journal, but the proposition that MARK alone caused +those collisions remains a hypothesis. Memo per MARK binding reduces request +volume; it does not make the underlying operation product-local. + +### Options + +A. Smallest experiment: use the already-authoritative partition mapping where it +is provable, then examine only that partition; retain conflict detection through +mapping validation. Do not assume Python's partition_key is itself a numeric +partition or invent a hash inconsistent with Kafka routing. + +B. Preferred if measurements still show contention: keep a bounded latest index +inside the EXISTING Hub, maintained atomically with dispatch. Store Arc references +and both latest product and newest book-sibling/fence scope. Reads can take a +short lock to copy references, then decode/check outside it. Cost: extra bounded +metadata and eviction logic; benefit: predictable lookup instead of ring length. + +C. Retain ring scanning, shorten lock-held work and retry under one absolute +budget. This is less structural but can still fail under sustained contention. +Replacing try_lock with an unbounded blocking lock is not the same optimization. + +Index safety considerations: BOOK_DELTA and BOOK_SNAPSHOT share physical scope; +a newer reset/gap/unverified sibling defeats an old clean snapshot. Preserve +original record offset AND partition watermark; retain broker-confirmed progress, +read_committed, retention and generation semantics. Memory should remain bounded +by declared products/retention, not every historical event ID. + +Focused test: hold an unrelated partition lock; concurrent MARK and BOOK reads; +quiet product behind many busy-product events; newest unsafe sibling; broker +confirmation expiry; ring eviction; same offset conflict. Measure lookup/lock/ +decode time, refusal counts and dispatch delay in one isolated workload. + +A release choice can be "current path is adequate under the frozen workload" +with evidence; it need not automatically be an index implementation. + +## 6. R3: Memo Wait Is Outside The Hot RPC Budget + +`qdl/runtime/kn_query_backend.py:332-347` uses `with lock` without an acquisition +timeout. `CanonicalHotClient.latest` starts its timeout AFTER obtaining that +lock. Each request therefore does not have one bound for wait + lookup. If the +leader exceeds the 50ms memo window, queued callers may each start a fresh lookup +because the memo timestamp is from lookup start. Cancellation of an async caller +does not itself interrupt a Python thread waiting on this mutex. + +This is SOURCE_CONFIRMED budget composition, not evidence of an infinite loop: +RPC timeout and bounded admission may impose a larger outer bound. No latency +threshold is invented here. + +Options: +- A: pass a remaining absolute budget through lock acquisition and hot call; + return a transient typed timeout when exhausted. Keep the worker's admission + permit until it actually finishes, as the current cancellation wrapper intends. +- B: use a per-binding in-flight future; each waiter has its own remaining budget. + More implementation complexity, better explicit cancellation semantics. +- C: retain the mutex if a measured worst-case wait is within the frozen budget; + make the composite bound explicit in telemetry and a deterministic test. + +Tests: delayed leader, multiple waiters, repeated unavailable outcomes, caller +cancellation, recovery after failure, primary advances during the wait. Broker +proof age and quality are evaluated at use time even when reusing a result. +Do not start a fresh full timeout at every stage or lengthen freshness to hide it. + +## 7. R4: Quiet-Proof Tick Can Prevent Silent-Socket Recovery + +### New finding, activation-dependent + +`rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs:1745-1750,1853-1895,2023-2068`. +The Binance loop constructs a NEW timeout(heartbeat_seconds, socket.next()) each +iteration. The new book_proof_tick branch runs every 250..5000 ms and then +`continue`s. If a socket stays open but sends neither data nor pong and proof +interval < heartbeat timeout, the tick can repeatedly win and discard the read +timeout before it expires. Sending ping can still succeed on a half-open socket; +that is not proof of a received pong. The timeout-specific disconnect path may +therefore never run until another transport error/OS timeout occurs. + +This follows from source scheduling; no Rust silent-peer fault run was performed +in this review. It is not attributed to the observed OKX disconnects, which use +a different path. The flag is default-off, so distinguish new-feature activation +risk from already-active production behavior. + +The proof generator correctly stops emitting fresh evidence once the last +received transport message is too old. Thus the immediate concern is bounded +RECOVERY/availability, not proof that it fabricates an executable price. + +### Options + +A. Preferred narrow option: track an absolute receive/pong deadline independent +of periodic work. Renew it only on actual accepted inbound progress; proof tick, +publish completion and sending ping do not reset it. Check deadline in the loop +and provide a dedicated select timer which remains expired across iterations. + +B. Separate a small persistent liveness state machine from the read timeout. +This may be clearer if several periodic branches already affect the same loop; +no extra service is needed. + +C. Ship parsing/schema support with proof flag OFF until this fault case is +checked, if owner prefers release staging. Describe quiet-book availability as +not yet improved; do not call proof activation certified while leaving it off. + +Suggested deterministic test: local fake WebSocket accepts ping but never sends +pong/data; fake clock or paused Tokio time, proof tick faster than heartbeat. +Expect bounded disconnect/new generation, no fresh proof after deadline. Add +normal pong recovery, data traffic, publish backpressure, shutdown and flag-off +controls. Test the real loop/state machine, not a Python simulation of it. + +## 8. R5: Cold Warmup Needs Attribution, Not More Lucky Runs + +Stage100e on rc5 (REPORTED in plan) still failed OKX COLD_2500 at 7531.5 ms; +Binance 4288.6 ms, frozen limit 6265.1 ms, n=1 each. Earlier first-read microbenchmarks +improved 25-31%, but production cold reads did not improve correspondingly. +A second request finishing roughly one service-time later is consistent with +same-replica queuing, but also client-side synchronous validation in one worker. +It does not by itself prove GIL wait is the dominant cause. + +Claude commit `7f974c3` now records serving replica and transport/client timing. +This is the right direction; inspect those results before choosing the next fix. +`qdl/query/cold_work.py` implements cooperative CPU duty cycling, not separate CPU +isolation. Increasing COLD_SLICE_MS trades hot responsiveness for cold progress; +one pause slice is not a strict end-to-end hot latency bound over many stages. + +Options ordered by scope: +1. Use the new timings to fix the actual unfair admission, redundant per-row + work or client validation scheduling. Preserve bounded large-read lanes. +2. Choose a measured cold slice with hot feed tail/refusal controls on the SAME + workload; tuning is useful but is not proof that compute is isolated. +3. If unavoidable Python CPU contention still misses the gate, evaluate a bounded + process/native render segment within existing roles. This is a larger option, + not an immediate recommendation to rewrite Query or introduce a new service. + +A focused matrix can be short: deliberately same replica and split replicas, +2500/5000 rows, cold and warm cache, cancellation/queue saturation, concurrent +hot MARK/BOOK. Record request-to-validated-consumer, queue wait, server work, +response decode/validation and bytes. Do not rely on random placement eventually +producing a PASS, or force distinct replicas only for the certification run. +Unchanged tests/evidence can be inherited; final mixed load follows attribution. + +## 9. R6: Refusal Diagnostics Still Lose The Layer Of Failure + +The stage100c/rc3 reports repeatedly say Query logs carry no refusal line and +causality is inferred from a detail hash. Hashes protect payloads and preserve +identity, but without an indexed typed reason they make triage unnecessarily slow. +The OKX incident in stage100e correlates with an ingestor disconnect; correlation +alone does not say whether the provider, client heartbeat, network, resubscribe +or materialization accounts for the full outage. + +Options: +- A: bounded structured failure-only counters/logs at existing boundaries: + problem code, origin (admission/identity quota/hot/cache/session), binding ID, + replica, request/run ID, generation and queue/RPC duration. No tokens or raw data. +- B: retain payload-free sanitized reason fields directly in harness receipts; + look up exact requests rather than turning on broad service DEBUG. +- C: short sampled trace for only the affected binding, with expiry/size cap. + +Suggested reconnect timeline: last valid receive -> disconnect -> reconnect -> +subscription ACK -> first canonical -> Query/Stream usable -> TS apply ACK. +Split a correct refusal during actual loss from internally delayed recovery. +Do not treat the observed 37 OKX disconnects as proof of a venue defect or a reason +to disable fail-closed behavior. This advisory does not propose new fallback venues +or replacement market prices. + +## 10. R7: Release Convergence And Scope Of Proof + +Several distinct questions remain even if the next load receipt turns green: + +- SDK now targets 2.2.0 after replica-start and additive view fields. A test client + mounting checkout SDK over an older image is development evidence, not proof + of the final published wheel/image. Check the final artifact explicitly. +- IDR082 adds event identity; TS adopting it is separate from Query emitting it. +- IDR040 proof flag, generated ingest config, core protobuf, Query contract digest, + bundle and tolerant readers need a coherent packet. A field being optional does + not establish that all pinned clients tolerate it or use its semantics. +- G12 PASS is isolated synthetic BOOK recovery with a single Stream/projector and + TS clones. It is useful, but it does not prove TRADE losslessness, MARK component + eligibility, BAR finality or independent-host HA during those faults. +- G4 rows absent from observation are NOT_MEASURED. OKX BAR open-time is not + close-to-canonical latency; MARK composite older-component age is not pure + transit of a new MARK event. Keep the frozen raw metric for comparison and + report the semantically correct companion metric; do not relabel silently. +- Historical full-suite failures/skips are not automatically waived. Categorize + required coverage, reproduced base failures, environment skips and remedies. + +Options for a compact release: +A. One final immutable source/image/SDK packet with affected regressions, scoped +real-provider reads and the agreed final load/fault receipts. Inherit unchanged +proof with explicit source/dependency mapping. +B. Stage optional producer flags off, certify the supported behavior accurately, +then activate in a narrow reviewed window. This changes the advertised feature +scope, not the meaning of PASS, and is an owner/Claude release decision. + +I would avoid repeated full-catalogue tests, unrelated TS accounting changes, +more revision bumps merely for a retry, and hidden permissive fallback policy. + +## 11. Previously Reported Items: Do Not Duplicate Work + +| Earlier finding | Latest source/report at review | Remaining check | +| --- | --- | --- | +| TS periodic observer wrong container / dropped unreadables | 713aa90 fixes source and adds regression | Actual run using fixed harness; earlier 2-sample PASS stays limited | +| Generic PID/container cleanup and unreadable logs treated as zero | Claude journal says run-scoped versions replaced old scripts | Inspect actual running script/PID receipt, not just old scratch path | +| rc5 tests mocked ordering | 4e19966 adds RealOrderingChecksTests | No longer report this as missing; use those fixtures for R1 hard-refusal tests | +| rc4 memo on backup path | ec89a31 separates paths | Keep rc4 failed evidence; R1/R2/R3 are distinct residual questions | + +Do not rerun E03/E04 or other TS order qualification to settle these DL findings. +No new TRADE latency campaign is suggested for IDR082's already-attributed quiet +snapshot effect. The new OKX disconnect observation is a separate bounded case. + +## 12. Suggested Handoff And Closure Order + +This is an execution OPTION for Claude, not a replacement phase plan: + +1. A short correctness slice: R1 decision table and regression; R4 focused fault + reproduction before enabling proof. Decide explicit flag scope for release. +2. A focused performance slice: use 7f974c3 timing; R2/R3 contention/deadline tests. + Pick only the bottleneck proven relevant to the frozen workload. +3. Integrate selected changes once, test affected interfaces/artifacts, then final + acceptance on the agreed SHA/digests. Score G4/typed refusals/readiness coverage + together with the load gates; keep old failures and incomplete attempts. +4. Cleanup exact owned runs after evidence; retain active plus named rollback. + Remote dev/main release sequence follows existing owner approval, not this file. + +Suggested response from Claude per item: + +| ID | Accept / disprove / not applicable / staged | Selected option or alternative | Test/receipt | Runtime included? | +| --- | --- | --- | --- | --- | +| R1-R7 | To be filled by Claude | Explain trade-off briefly | Hash/path and scope | Exact digest or not deployed | + +Do not label an unimplemented chosen fix as external debt. Equally, do not turn +all alternative designs into mandatory scope: a reasoned rejection with evidence +is a valid review response. Existing safety invariants are the common contract. + +## 13. Review Reproduction And Hygiene + +R1 executed once in image +`sha256:77682eea5054ae9ddbd858323b083286669a55eeb4bc7bc065b04bae8854e32f`, +network none, source bind read-only, python -B, 0.5 CPU, 768 MiB, 128 PIDs, 32 MiB tmpfs. +Container `qdl-astra-review-20261010-fence` used --rm and was confirmed absent. +No image build, provider access, Redis/DB service, persistent test volume or new +credential. No full suite was rerun. The three local outcomes above are NOT three +acceptance PASSes; they expose current behavior for review. + +Reproduction recipe (inside the same existing test image, at reviewed source): + +```python +from dataclasses import replace +from tests.test_kn_mark_hot_memo import RealOrderingChecksTests +from qdl.runtime.kn_hot_view import HotViewGap, HotViewUnavailable + +RealOrderingChecksTests.setUpClass() +for scenario in ("gap", "generation_mismatch", "temporary_busy"): + case = RealOrderingChecksTests("test_a_primary_advances_past_the_shared_memo") + case.setUp() + backend = case.backend() + if scenario == "gap": + case.reply = HotViewGap("HOT_RECORD_GAPPED") + elif scenario == "generation_mismatch": + hot = case.hot("H1") + case.reply = replace(hot, boundary=replace(hot.boundary, topic_id="different-topic")) + else: + case.reply = HotViewUnavailable("HOT_READER_BUSY") + stored, state = backend.latest_stored_event(case.binding) + print(scenario, case.name_of(stored.event.payload) if stored else None, state) +``` + +No retained runtime, rollback image, build cache or shared state was removed. +Disk pre-test sample was not taken; no disk-reclamation claim is made. General +image pruning was intentionally avoided while Claude was concurrently working. +Canonical checkout remains `/home/bobby/data_layer`, released line `main@5bf5cd5` / +v2.2.2 at the last check; v2.2.3 remains a candidate. This document does not attest +current container health or supersede a newer journal/owner decision. From 57881a2a56dc0484eee772395eaa24c4fb72cf42 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 07:02:09 +0000 Subject: [PATCH 098/147] fix(query): Astra R1/R3/R6 - hard hot refusals not downgraded, one MARK hot budget, bounded refusal lines R1: the IDR-084 opportunistic MARK branch of latest_stored_event turned every typed hard refusal from _hot_candidate except INTERNAL_ERROR into "keep the older cache view, OK". A gap on the newest committed record (HotViewGap) and a source partition/generation conflict now refuse with the backup path's mapping (DATA_NOT_READY -> FENCED, else INTEGRITY). Transient HotViewUnavailable (busy/capacity/deadline/catching up/behind/reply not attributable to this Query's authority) still keeps the independently validated cache view. R3: _hot_latest waited on the per-binding lock without a bound and each queued reader could start a fresh full-deadline lookup. One budget (the hot client's deadline) now covers lock wait plus lookup; CanonicalHotClient.latest takes an additive timeout_seconds that can only shorten its deadline. An exhausted budget is the transient HOT_READ_DEADLINE and is never memoised. R6: qdl/runtime/refusal_log.py emits one rate-bounded, payload-free qdl_refusal line (origin, code, typed reason tokens, detail sha256, request id, instrument, route, replica) from the Query problem handler, the data-plane access handler, the kn read view and the private MARK/INDEX edge. The load harness receipt adds reason_tokens (same extraction) next to detail_sha256. No response, status or refusal decision changes. --- qdl/api_v2/router.py | 13 ++ qdl/runtime/execution_mark_index.py | 12 +- qdl/runtime/kn_hot_view.py | 14 +- qdl/runtime/kn_query_backend.py | 34 ++- qdl/runtime/kn_read_view.py | 5 + qdl/runtime/refusal_log.py | 150 ++++++++++++ scripts/phase3_consumer_load_acceptance.py | 11 + tests/test_fund_phase5_api.py | 34 +++ tests/test_kn_hot_view.py | 2 +- tests/test_kn_mark_hot_memo.py | 255 ++++++++++++++++++++- tests/test_kn_query_backend.py | 6 +- tests/test_phase3_consumer_load_driver.py | 12 + tests/test_refusal_log.py | 117 ++++++++++ 13 files changed, 646 insertions(+), 19 deletions(-) create mode 100644 qdl/runtime/refusal_log.py create mode 100644 tests/test_refusal_log.py diff --git a/qdl/api_v2/router.py b/qdl/api_v2/router.py index 2ed80115..748b854a 100644 --- a/qdl/api_v2/router.py +++ b/qdl/api_v2/router.py @@ -68,6 +68,7 @@ ) from qdl.runtime.bounds import BoundedRequestMiddleware, RequestBounds from qdl.runtime.readiness import FailClosedReadiness +from qdl.runtime.refusal_log import REFUSALS from qdl.warmup.contracts import ( IntervalSourcePolicy, WarmupSpecification, @@ -1414,9 +1415,17 @@ def data_plane_openapi(): app.openapi = data_plane_openapi + def _route_of(request: Request) -> str | None: + route = request.scope.get("route") + return getattr(route, "path", None) + @app.exception_handler(DataPlaneAccessError) async def data_access_error_handler(_request: Request, error: DataPlaneAccessError): request_id = service.request_id() if service is not None else "unavailable" + # Astra R6: the identity/quota layer's refusal, bounded and payload-free. + REFUSALS.record(origin="data_plane_access", code=error.code, detail=error.detail, + status=error.status_code, retryable=error.status_code in {429, 503}, + request_id=request_id, route=_route_of(_request)) problem = ProblemDetails( type=f"urn:qdl:error:{error.code.lower().replace('_', '-')}", title=error.code.replace("_", " ").title(), @@ -1435,6 +1444,10 @@ async def data_access_error_handler(_request: Request, error: DataPlaneAccessErr @app.exception_handler(QueryServiceError) async def query_error_handler(_request: Request, error: QueryServiceError): problem = _problem(error) + # Astra R6: one bounded, payload-free line per Query refusal (typed reasons + detail digest). + REFUSALS.record(origin="query", code=problem.code, detail=problem.detail, status=problem.status, + retryable=problem.retryable, request_id=problem.request_id, + instrument_uid=problem.instrument_uid, route=_route_of(_request)) return JSONResponse( status_code=problem.status, content=problem.model_dump(exclude_none=True), diff --git a/qdl/runtime/execution_mark_index.py b/qdl/runtime/execution_mark_index.py index 2027f9ff..dd9f757f 100644 --- a/qdl/runtime/execution_mark_index.py +++ b/qdl/runtime/execution_mark_index.py @@ -30,6 +30,7 @@ from qdl.runtime.internal_auth import stable_hmac_signature from qdl.runtime.lease import GatewayFenced from qdl.runtime.mark_index_lineage import paired_mark_index_lineage +from qdl.runtime.refusal_log import REFUSALS from qdl.runtime.stable_catalog import StableSourceBinding, StableSourceCatalog from qdl.runtime.stable_deployment import StableAcquisitionPlan from qdl.runtime.session_liveness import ( @@ -655,10 +656,13 @@ async def latest_mark_index( except GatewayFenced as error: raise HTTPException(status_code=409, detail="execution MARK/INDEX gateway fenced") from error if result.record is None: - raise HTTPException( - status_code=409, - detail=f"execution MARK/INDEX live view unavailable:{result.reason}", - ) + detail = f"execution MARK/INDEX live view unavailable:{result.reason}" + # Astra R6: the view's own reason (e.g. MARKET_CACHE_BACKUP_UNAVAILABLE) is + # normalised away by the reference reader; keep it at this layer. + REFUSALS.record(origin="execution_mark_index", code="LIVE_VIEW_UNAVAILABLE", detail=detail, + status=409, retryable=True, instrument_uid=instrument_uid, + route="/internal/v2/execution/mark-index/latest") + raise HTTPException(status_code=409, detail=detail) record = result.record # A header keeps the private JSON response additive-compatible with a # rolling reader deployment while carrying the stream-authoritative diff --git a/qdl/runtime/kn_hot_view.py b/qdl/runtime/kn_hot_view.py index 4a490319..22a913f5 100644 --- a/qdl/runtime/kn_hot_view.py +++ b/qdl/runtime/kn_hot_view.py @@ -81,13 +81,23 @@ def close(self): for channel in getattr(self, "_channels", ()): channel.close() - def latest(self, binding, lpk) -> CanonicalHotView: + @property + def timeout_seconds(self) -> float: + """The configured per-lookup deadline (callers may only shorten it).""" + return self._timeout + + def latest(self, binding, lpk, *, timeout_seconds: float | None = None) -> CanonicalHotView: + """``timeout_seconds`` (optional) is what is left of the caller's own budget; it can only shorten the + configured deadline, never extend it.""" + timeout = self._timeout if timeout_seconds is None else min(self._timeout, timeout_seconds) if binding.feed.value not in HOT_FEEDS or binding.interval is not None: raise HotViewUnavailable("HOT_FEED_UNSUPPORTED") + if timeout <= 0: + raise HotViewUnavailable("HOT_READ_DEADLINE") if not self._slots.acquire(blocking=False): raise HotViewUnavailable("HOT_READ_CAPACITY") try: - deadline = time.monotonic() + self._timeout + deadline = time.monotonic() + timeout error = "HOT_READER_UNAVAILABLE" for index, call in enumerate(self._calls): remaining = deadline - time.monotonic() diff --git a/qdl/runtime/kn_query_backend.py b/qdl/runtime/kn_query_backend.py index ef797b19..32a1ca68 100644 --- a/qdl/runtime/kn_query_backend.py +++ b/qdl/runtime/kn_query_backend.py @@ -97,6 +97,8 @@ DEFAULT_ROW_CACHE_ENTRIES = 20_000 # A MARK/INDEX hot-view lookup is shared by every read of that binding on one replica for this long. MARK_HOT_MEMO_NS = 50_000_000 +# Wait + lookup budget of one shared MARK read when the hot client declares no deadline (CanonicalHotClient does). +MARK_HOT_DEFAULT_BUDGET_SECONDS = 0.25 # Item fields rebuilt per request (time/request dependent); never cached. _DYNAMIC_ITEM_FIELDS = frozenset({"quality", "watermark_offset", "cursor", "snapshot_id", "render_key", "render_slot"}) @@ -331,16 +333,32 @@ def _hot_latest(self, binding): key = binding.binding_id with self._mark_hot_locks_guard: lock = self._mark_hot_locks.setdefault(key, threading.Lock()) - with lock: + # Astra R3: ONE budget (the hot client's own deadline) covers the wait for + # this binding's lookup slot AND the lookup itself. Before, the wait was + # unbounded and every queued reader whose memo had expired started a + # fresh full-deadline lookup in turn. A reader whose budget runs out is + # refused HOT_READ_DEADLINE, which the IDR-084 caller treats as transient + # (the valid cache view stands); that refusal is this reader's own wait, + # not the hot reader's answer, so it is never memoised. + budget = float(getattr(self.hot_client, "timeout_seconds", MARK_HOT_DEFAULT_BUDGET_SECONDS)) + started = time.monotonic() + if not lock.acquire(timeout=budget): + raise HotViewUnavailable("HOT_READ_DEADLINE") + try: now = self._monotonic_ns() memo = self._mark_hot_memo.get(key) if memo is None or now - memo[0] >= MARK_HOT_MEMO_NS: + remaining = budget - (time.monotonic() - started) + if remaining <= 0: + raise HotViewUnavailable("HOT_READ_DEADLINE") try: - outcome = self.hot_client.latest(binding, self.product_key(binding)) + outcome = self.hot_client.latest(binding, self.product_key(binding), timeout_seconds=remaining) except HotViewUnavailable as error: outcome = error memo = (now, outcome) self._mark_hot_memo[key] = memo + finally: + lock.release() if isinstance(memo[1], HotViewUnavailable): raise type(memo[1])(*memo[1].args) # a fresh exception per reader thread return memo[1] @@ -468,14 +486,20 @@ def latest_stored_event(self, binding: StableSourceBinding, *, prefer_hot: bool # ~300 ms after the venue event). The Rust hot view serves the same committed bytes without the projector, # so MARK/INDEX takes the newer of the two. Every lineage/monotonic gate of the backup path applies; a hot # read that is behind or unavailable leaves the cache view in place (the previous behaviour). + # + # Astra R1: only a TRANSIENT hot outcome leaves the cache view in place - HotViewUnavailable (busy, + # capacity, deadline, catching up, behind the cache view, or a reply not attributable to this Query's + # authority, which says nothing about the canonical record). A typed hard refusal from _hot_candidate is a + # fact about the newest committed record of THIS product and is never downgraded to the older cache view: + # a gap/resync on the newest record (HotViewGap -> OPEN_SEQUENCE_GAP; once projected, the MARK view drops + # it as GAP_OR_RESYNC anyway), a source partition/generation conflict (DATA_NOT_READY) or a payload/lineage + # conflict (INTERNAL_ERROR). Same state mapping as the backup path above. try: candidate = self._hot_candidate(binding, view, shared=True) except HotViewUnavailable: candidate = None except QueryBackendError as error: - if error.problem.code is CanonicalErrorCode.INTERNAL_ERROR: - return None, "INTEGRITY" # same offset, different bytes: never pick one silently - candidate = None + return None, "FENCED" if error.problem.code is CanonicalErrorCode.DATA_NOT_READY else "INTEGRITY" if candidate is not None and candidate.rows[0].source_offset > view.rows[0].source_offset: view = candidate if view is None: diff --git a/qdl/runtime/kn_read_view.py b/qdl/runtime/kn_read_view.py index 898b9e35..f5234d5f 100644 --- a/qdl/runtime/kn_read_view.py +++ b/qdl/runtime/kn_read_view.py @@ -44,6 +44,7 @@ from qdl.query.contracts import CanonicalErrorCode, QueryProblem from qdl.query.service import QueryServiceError from qdl.runtime.internal_auth import stable_hmac_signature +from qdl.runtime.refusal_log import REFUSALS from qdl.stream.grpc_service import requirement_from_proto READ_VIEW_PATH = "/internal/v2/kn/read-view" @@ -57,6 +58,10 @@ def _problem(error: QueryServiceError) -> JSONResponse: + # Astra R6: the Stream GetSnapshot read view's refusal, bounded and payload-free. + REFUSALS.record(origin="kn_read_view", code=error.problem.code.value, detail=error.problem.detail, + status=409, retryable=error.problem.retryable, request_id=error.request_id, + instrument_uid=error.instrument_uid) return JSONResponse( status_code=409, content={"code": error.problem.code.value, "detail": error.problem.detail}, diff --git a/qdl/runtime/refusal_log.py b/qdl/runtime/refusal_log.py new file mode 100644 index 00000000..2255edc7 --- /dev/null +++ b/qdl/runtime/refusal_log.py @@ -0,0 +1,150 @@ +"""Bounded, payload-free refusal lines at the layer that refused (Astra advisory R6). + +Purpose: P18.4 stage-100 triage had only a hash of a refusal's detail in the +harness receipt and no Query line at all (the HTTP problem handler and the +private execution MARK/INDEX edge returned refusals silently), so the failing +layer was inferred from code. Each refusal now leaves one structured line: +origin (which edge refused), canonical code, HTTP status, retryability, the +typed reason tokens of the detail (UPPER_SNAKE vocabulary such as +``HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND`` / ``HOT_READER_BUSY``), the sha256 of +the detail (the same digest the load harness stores, so a receipt row joins its +server line exactly), request id, instrument uid, route template and replica. + +Boundaries: +* failure-only and payload-free: never the detail text itself, a token, a + price, a cursor or a request body; +* bounded volume: at most ``burst`` lines per (origin, code, first reason) key + per ``window_seconds``; the number suppressed is reported once per key with + the first line of a later window, so an incident costs a fixed log rate and + its size is still recorded. At most ``max_keys`` keys per window (excess keys + share one overflow key); +* WARNING level on logger ``qdl.refusal``: the Query role installs no handler of + its own (``serve_stable_query``), so the line reaches stderr through the + logging module's last-resort handler without changing any other log volume. +Changing no response, status or refusal decision. +""" +from __future__ import annotations + +import hashlib +import json +import logging +import re +import socket +import threading +import time +from typing import Callable + +logger = logging.getLogger("qdl.refusal") + +_REASON = re.compile(r"\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b") +MAX_REASONS = 4 +_MAX_TOKEN = 64 +_OVERFLOW = "OVERFLOW" + + +def reason_tokens(detail: object, *, limit: int = MAX_REASONS) -> tuple[str, ...]: + """The typed UPPER_SNAKE reason codes inside a refusal detail, in order, deduplicated.""" + + found: list[str] = [] + for token in _REASON.findall(str(detail or "")[:4096]): + if len(token) <= _MAX_TOKEN and token not in found: + found.append(token) + if len(found) == limit: + break + return tuple(found) + + +def detail_sha256(detail: object) -> str: + return hashlib.sha256(str(detail or "").encode()).hexdigest() + + +def _bounded(value: object, size: int = 256) -> str | None: + if value is None: + return None + text = str(value) + return text if len(text) <= size else text[:size] + + +class RefusalLog: + """Rate-bounded structured refusal lines (thread-safe).""" + + def __init__( + self, + *, + burst: int = 5, + window_seconds: float = 10.0, + max_keys: int = 256, + clock: Callable[[], float] = time.monotonic, + emit: Callable[[str], None] | None = None, + replica: str | None = None, + ) -> None: + if burst < 1 or window_seconds <= 0 or max_keys < 1: + raise ValueError("refusal log bounds must be positive") + self._burst = burst + self._window = window_seconds + self._max_keys = max_keys + self._clock = clock + self._emit = emit or logger.warning + self._replica = replica if replica is not None else socket.gethostname() + self._lock = threading.Lock() + self._window_start = clock() + self._counts: dict[tuple[str, str, str], int] = {} + self._carried: dict[tuple[str, str, str], int] = {} + + def record( + self, + *, + origin: str, + code: str, + detail: object, + status: int | None = None, + retryable: bool | None = None, + request_id: str | None = None, + instrument_uid: str | None = None, + route: str | None = None, + ) -> bool: + """Emit one line for this refusal unless its key is over budget; True when emitted.""" + + reasons = reason_tokens(detail) + key = (str(origin), str(code), reasons[0] if reasons else "") + with self._lock: + now = self._clock() + if now - self._window_start >= self._window: + for prior, count in self._counts.items(): + if count > self._burst: + self._carried[prior] = self._carried.get(prior, 0) + count - self._burst + # Carried counts are bounded by the same key budget. + while len(self._carried) > self._max_keys: + self._carried.pop(next(iter(self._carried))) + self._counts = {} + self._window_start = now + if key not in self._counts and len(self._counts) >= self._max_keys: + key = (key[0], key[1], _OVERFLOW) + count = self._counts.get(key, 0) + 1 + self._counts[key] = count + if count > self._burst: + return False + suppressed = self._carried.pop(key, 0) + line = { + "event": "qdl_refusal", + "origin": key[0], + "code": key[1], + "status": status, + "retryable": retryable, + "reasons": list(reasons), + "detail_sha256": detail_sha256(detail), + "request_id": _bounded(request_id, 64), + "instrument_uid": _bounded(instrument_uid), + "route": _bounded(route), + "replica": _bounded(self._replica, 128), + } + if suppressed: + line["suppressed_before"] = suppressed + try: + self._emit(json.dumps(line, separators=(",", ":"), sort_keys=True)) + except Exception: # noqa: BLE001 - diagnostics never change a response + return False + return True + + +REFUSALS = RefusalLog() diff --git a/scripts/phase3_consumer_load_acceptance.py b/scripts/phase3_consumer_load_acceptance.py index 96671e5e..630c345c 100644 --- a/scripts/phase3_consumer_load_acceptance.py +++ b/scripts/phase3_consumer_load_acceptance.py @@ -77,6 +77,17 @@ def _safe_error(error: BaseException) -> dict[str, object]: reason = re.search(r"\(([A-Z_]{1,40})\)$", detail) if reason: out["reason"] = reason.group(1) + # Astra R6: every typed UPPER_SNAKE code in the detail (e.g. "HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND: + # HOT_READER_BUSY"), bounded; the same extraction as qdl/runtime/refusal_log.py reason_tokens, whose + # Query line carries this detail_sha256. Additive receipt field. + tokens: list[str] = [] + for token in re.findall(r"\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b", detail[:4096]): + if len(token) <= 64 and token not in tokens: + tokens.append(token) + if len(tokens) == 4: + break + if tokens: + out["reason_tokens"] = tokens # Query's quality at the refusal (KN-4 review): only the known, # non-secret fields, bounded. diagnostics = getattr(error, "diagnostics", None) diff --git a/tests/test_fund_phase5_api.py b/tests/test_fund_phase5_api.py index 089f6162..a9ca1614 100644 --- a/tests/test_fund_phase5_api.py +++ b/tests/test_fund_phase5_api.py @@ -304,6 +304,40 @@ async def incomplete_scan(): self.assertEqual(response.json()["code"], "PARTIAL_RESULT") self.assertTrue(response.json()["retryable"]) + def test_a_refusal_leaves_one_bounded_payload_free_line(self): + """Astra R6: the Query problem handler records the refusal's typed reasons and detail digest.""" + import hashlib + import json + + router_module = importlib.import_module("qdl.api_v2.router") + from qdl.runtime.refusal_log import RefusalLog + + detail = "HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND: HOT_READER_BUSY" + + async def refused(): + raise QueryServiceError( + QueryProblem(CanonicalErrorCode.DATA_NOT_READY, detail, True, retry_after_ms=1_000), + request_id="r6-request", instrument_uid="u-r6", + ) + + lines = [] + log = RefusalLog(emit=lambda line: lines.append(json.loads(line)), replica="query-test") + with patch.object(self.service, "open_gaps_async", new=refused), \ + patch.object(router_module, "REFUSALS", log): + response = self.client.get("/v2/data-quality/gaps") + problem = response.json() + self.assertEqual((problem["code"], problem["detail"]), ("DATA_NOT_READY", detail)) # response unchanged + (line,) = lines + self.assertEqual( + {key: line[key] for key in ("origin", "code", "status", "retryable", "reasons", "request_id", + "instrument_uid", "route", "replica")}, + {"origin": "query", "code": "DATA_NOT_READY", "status": response.status_code, "retryable": True, + "reasons": ["HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND", "HOT_READER_BUSY"], + "request_id": "r6-request", "instrument_uid": "u-r6", "route": "/v2/data-quality/gaps", + "replica": "query-test"}, + ) + self.assertEqual(line["detail_sha256"], hashlib.sha256(detail.encode()).hexdigest()) + def test_query_routes_pass_authenticated_identity_to_owned_read_lanes(self): calls = [] snapshot_original = self.service.snapshot_async diff --git a/tests/test_kn_hot_view.py b/tests/test_kn_hot_view.py index 30cfee5c..cb036bea 100644 --- a/tests/test_kn_hot_view.py +++ b/tests/test_kn_hot_view.py @@ -135,7 +135,7 @@ def setUp(self): envelope.published_at_ns=self.now-7_000_000 self.backup=CanonicalHotView(self.lpk,(CacheRow(envelope.SerializeToString(),30),),SourceBoundary("topic",0,40)) self.calls=0 - def backup(*args): + def backup(*args,**_kwargs): self.calls+=1 if isinstance(self.backup,Exception):raise self.backup return self.backup diff --git a/tests/test_kn_mark_hot_memo.py b/tests/test_kn_mark_hot_memo.py index 13f12c10..da6821a3 100644 --- a/tests/test_kn_mark_hot_memo.py +++ b/tests/test_kn_mark_hot_memo.py @@ -16,7 +16,7 @@ def __init__(self, outcomes): self.outcomes = list(outcomes) self.calls = 0 - def latest(self, binding, key): + def latest(self, binding, key, **_kwargs): self.calls += 1 outcome = self.outcomes[min(self.calls, len(self.outcomes)) - 1] if isinstance(outcome, BaseException): @@ -94,9 +94,9 @@ def test_concurrent_readers_make_one_lookup(self): gate = threading.Event() class SlowHot(_Hot): - def latest(self, binding, key): + def latest(self, binding, key, **kwargs): gate.wait(1) - return super().latest(binding, key) + return super().latest(binding, key, **kwargs) hot = SlowHot(["v"]) backend = _backend(hot, clock) @@ -226,7 +226,7 @@ def primary(key): raise self.primary return self.primary - def latest(binding, key): + def latest(binding, key, **_kwargs): self.calls += 1 if isinstance(self.reply, Exception): raise self.reply @@ -379,6 +379,253 @@ def test_b_stale_record_on_the_backup_path_is_refused_primary_behind(self): stored, state = rc5.latest_stored_event(self.binding, prefer_hot=True) self.assertEqual((state, self.name_of(stored.event.payload)), ("OK", "H2")) + # (d) Astra R1: a hard hot refusal is never downgraded to the older cache view ------------- + + def view_read(self, backend): + """The execution MARK view over this backend (the TS at-use path), at the fixture clock.""" + import asyncio + + from qdl.reference.local_mark_index import CacheRefreshingMarkIndexView + + view = CacheRefreshingMarkIndexView.from_catalog(self.catalog).attach_cache( + backend=backend, relax_to_alpha=False) + return asyncio.run(view.read( + instrument_uid=self.binding.instrument.instrument_uid, + instrument_revision=self.binding.instrument.metadata_revision, + source_policy_id=self.binding.source_policy_id, max_freshness_ms=2000, gateway_epoch=1, + now_ns=self.now)) + + def hot_reply(self, name, *, topic="topic", partition=0, payload=None): + from dataclasses import replace + + hot = self.hot(name) + hot = replace(hot, boundary=replace(hot.boundary, topic_id=topic, partition=partition)) + if payload is not None: + hot = replace(hot, rows=(replace(hot.rows[0], canonical=payload),)) + return hot + + def test_d_hard_refusals_are_not_downgraded_to_the_cache_view(self): + """Before the fix every row below except the payload conflict returned (C10, OK).""" + cases = { + "gap": (HotViewGap("HOT_RECORD_GAPPED"), "INTEGRITY"), + "other partition": (lambda: self.hot_reply("H1", partition=1), "FENCED"), + # Unreachable through CanonicalHotClient (its reply check refuses another topic as the transient + # HOT_REPLY_INVALID, see test_d_real_client_*), still a hard refusal at the backend gate. + "other topic": (lambda: self.hot_reply("H1", topic="different-topic"), "FENCED"), + "same offset, other bytes": (lambda: self.hot_reply("C10", payload=self.payloads["H1"]), "INTEGRITY"), + "lineage invalid": (lambda: self.hot_reply("H1", payload=b"\x0a\x01x"), "INTEGRITY"), + } + for name, (reply, state) in cases.items(): + with self.subTest(name): + self.setUp() + self.reply = reply() if callable(reply) else reply + backend = self.backend() + self.assertEqual(backend.latest_stored_event(self.binding), (None, state)) + self.assertEqual(self.calls, 1) + self.assertNotIn(self.binding.binding_id, backend._hot_boundaries, "nothing recorded") + # The TS at-use path: refused with the layer's state, no second (prefer_hot) attempt that could + # heal it, and inside the memo window the shared refusal still holds. + result = self.view_read(backend) + self.assertIsNone(result.record) + self.assertEqual(result.reason, f"MARKET_CACHE_{state}") + self.assertEqual(self.calls, 1) + + def test_d_transient_outcomes_keep_the_valid_cache_view(self): + for name in ("HOT_READER_BUSY", "HOT_READ_CAPACITY", "HOT_READ_DEADLINE", "HOT_READER_CATCHING_UP", + "HOT_BROKER_UNCONFIRMED", "HOT_RECORD_MISSING", "HOT_REPLY_INVALID", + "HOT_AUTHORITY_OR_PROTOCOL_REFUSED"): + with self.subTest(name): + self.setUp() + self.reply = HotViewUnavailable(name) + backend = self.backend() + stored, state = backend.latest_stored_event(self.binding) + self.assertEqual((state, self.name_of(stored.event.payload)), ("OK", "C10")) + self.assertEqual(self.recorded(backend), 10) + result = self.view_read(backend) + self.assertIsNotNone(result.record, result.reason) + self.assertEqual(result.record.canonical, self.payloads["C10"]) + # A hot record older than the cache view is not newer: the cache view stands (HOT_BACKUP_BEHIND_PRIMARY). + self.setUp() + self.primary, self.reply = self.cache("H1"), self.hot("C10") + stored, state = self.backend().latest_stored_event(self.binding) + self.assertEqual((state, self.name_of(stored.event.payload)), ("OK", "H1")) + + def test_d_recovery_after_a_gap_refusal(self): + backend = self.backend() + self.reply = HotViewGap("HOT_RECORD_GAPPED") + self.assertEqual(backend.latest_stored_event(self.binding), (None, "INTEGRITY")) + # Inside the window the shared gap still refuses, even though the cache view did not change. + self.reply = self.hot("H2") + self.mono[0] += MARK_HOT_MEMO_NS - 1 + self.assertEqual(backend.latest_stored_event(self.binding), (None, "INTEGRITY")) + self.assertEqual(self.calls, 1) + # Window over: the newer clean committed record is served through every gate. + self.mono[0] += 1 + stored, state = backend.latest_stored_event(self.binding) + self.assertEqual((state, self.name_of(stored.event.payload), self.calls), ("OK", "H2", 2)) + self.assertEqual(self.recorded(backend), 50) + # The backup path (prefer_hot) keeps its existing hard mapping for the same refusal. + self.reply = HotViewGap("HOT_RECORD_GAPPED") + self.assertEqual(backend.latest_stored_event(self.binding, prefer_hot=True), (None, "INTEGRITY")) + + def test_d_real_client_refuses_another_topic_as_a_transient_reply_error(self): + """Through CanonicalHotClient a reply for another canonical topic never reaches the backend gate: the + client's identity check (kn_hot_view.py _decode; expectation source_topic_id is Query's own topic, + stable.py hot_client_from_environment) turns it into HOT_REPLY_INVALID. A reply that cannot be attributed + to this Query's authority says nothing about the cache view, which stands. The same reply for the right + topic is taken (newer).""" + import base64 + import json + + from google.protobuf.wrappers_pb2 import BytesValue + + from qdl.runtime.kn_hot_view import SCHEMA, CanonicalHotClient + + def client(topic): + body = dict(schema=SCHEMA, binding_id=self.binding.binding_id, product_key=self.lpk.encode(), + physical_key=self.binding.partition_key, source_topic_id=topic, source_partition=0, + source_offset=40, record_offset=30, + canonical=base64.b64encode(self.payloads["H1"]).decode()) + return CanonicalHotClient(calls=(lambda *a, **k: BytesValue(value=json.dumps(body).encode()),), + expectation={"source_topic_id": "topic"}, secret=b"x" * 32) + + for topic, expected in (("different-topic", "C10"), ("topic", "H1")): + with self.subTest(topic): + self.setUp() + backend = self.backend() + backend.hot_client = client(topic) + stored, state = backend.latest_stored_event(self.binding) + self.assertEqual((state, self.name_of(stored.event.payload)), ("OK", expected)) + + +MARK_BINDING = SimpleNamespace(binding_id="mark-a", feed=FeedType.MARK_INDEX_PRICE, interval=None) + + +class MarkHotBudgetTests(unittest.TestCase): + """Astra R3: one budget bounds the wait for a binding's lookup slot plus the lookup itself.""" + + def test_a_waiter_behind_a_slow_leader_is_refused_within_its_budget_and_not_memoised(self): + import time as _time + + clock = [0] + release = threading.Event() + entered = threading.Event() + + class SlowHot(_Hot): + timeout_seconds = 0.05 + + def latest(self, binding, key, **kwargs): + entered.set() + release.wait(2) + return super().latest(binding, key, **kwargs) + + hot = SlowHot(["v1", "v2"]) + backend = _backend(hot, clock) + leader_result = [] + leader = threading.Thread(target=lambda: leader_result.append(backend._hot_latest(MARK))) + leader.start() + self.assertTrue(entered.wait(2)) + started = _time.monotonic() + with self.assertRaisesRegex(HotViewUnavailable, "HOT_READ_DEADLINE"): + backend._hot_latest(MARK) + waited = _time.monotonic() - started + self.assertGreaterEqual(waited, 0.04) + self.assertLess(waited, 1.0, "bounded by the 50 ms budget, not by the leader") + release.set() + leader.join(2) + self.assertEqual(leader_result, ["v1"]) + self.assertEqual(hot.calls, 1) + # The waiter's own deadline was not memoised: the next reader inside the window gets the leader's value. + self.assertEqual(backend._hot_latest(MARK), "v1") + self.assertEqual(hot.calls, 1) + + def test_the_lookup_gets_only_the_remaining_budget(self): + clock = [0] + seen = [] + + class BudgetHot(_Hot): + timeout_seconds = 0.2 + + def latest(self, binding, key, **kwargs): + seen.append(kwargs.get("timeout_seconds")) + return super().latest(binding, key, **kwargs) + + backend = _backend(BudgetHot(["v"]), clock) + self.assertEqual(backend._hot_latest(MARK), "v") + self.assertEqual(len(seen), 1) + self.assertGreater(seen[0], 0) + self.assertLessEqual(seen[0], 0.2) + + def test_an_unavailable_outcome_expires_and_recovers(self): + clock = [0] + hot = _Hot([HotViewUnavailable("HOT_READER_BUSY"), "v"]) + backend = _backend(hot, clock) + with self.assertRaises(HotViewUnavailable): + backend._hot_latest(MARK) + clock[0] += MARK_HOT_MEMO_NS + self.assertEqual(backend._hot_latest(MARK), "v") + self.assertEqual(hot.calls, 2) + + def test_other_feeds_keep_the_direct_call(self): + clock = [0] + seen = [] + + class Direct(_Hot): + def latest(self, binding, key, **kwargs): + seen.append(kwargs) + return super().latest(binding, key) + + backend = _backend(Direct(["q"]), clock) + self.assertEqual(backend._hot_latest(QUOTE), "q") + self.assertEqual(seen, [{}]) + + +class HotClientBudgetTests(unittest.TestCase): + """CanonicalHotClient.latest(timeout_seconds=...) can only shorten its configured deadline.""" + + def client(self, call, timeout=0.25): + from qdl.runtime.kn_hot_view import CanonicalHotClient + + return CanonicalHotClient(calls=(call,), expectation={}, secret=b"x" * 32, timeout_seconds=timeout) + + def test_an_exhausted_budget_never_calls_the_reader(self): + def call(*args, **kwargs): + raise AssertionError("no RPC without budget") + + client = self.client(call) + self.assertEqual(client.timeout_seconds, 0.25) + for remaining in (0, -0.1): + with self.assertRaisesRegex(HotViewUnavailable, "HOT_READ_DEADLINE"): + client.latest(MARK_BINDING, "k", timeout_seconds=remaining) + self.assertTrue(client._slots.acquire(False)) + client._slots.release() + + def test_the_rpc_timeout_is_the_smaller_budget(self): + import grpc + + timeouts = [] + + class Busy(grpc.RpcError): + def code(self): + return grpc.StatusCode.RESOURCE_EXHAUSTED + + def details(self): + return "HOT_READER_BUSY" + + def call(body, *, timeout, metadata, wait_for_ready): + timeouts.append(timeout) + raise Busy() + + client = self.client(call) + for budget in (0.05, 5.0, None): + with self.assertRaisesRegex(HotViewUnavailable, "HOT_READER_BUSY"): + client.latest(MARK_BINDING, "k", timeout_seconds=budget) + self.assertLessEqual(timeouts[0], 0.05) + self.assertLessEqual(timeouts[1], 0.25) + self.assertGreater(timeouts[1], 0.05) + self.assertLessEqual(timeouts[2], 0.25) + self.assertGreater(timeouts[2], 0.05) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_kn_query_backend.py b/tests/test_kn_query_backend.py index 826dcd11..96a7e348 100644 --- a/tests/test_kn_query_backend.py +++ b/tests/test_kn_query_backend.py @@ -924,7 +924,7 @@ def timed(age): primary = self.reader.latest(self.lpk(binding)) candidate = replace(candidate, boundary=replace(candidate.boundary, partition=primary.boundary.partition)) replies = [candidate] - def hot(*args): + def hot(*args, **_kwargs): if isinstance(replies[0], Exception): raise replies[0] return replies[0] backend = KnMarketCacheQueryBackend(self.reader, self.catalog, schema_digest=DIGEST, @@ -976,7 +976,7 @@ def hot_view(payload_bytes, offset, mark): return CanonicalHotView(self.lpk(binding), (CacheRow(payload_bytes, offset),), SourceBoundary(TOPIC_ID, primary.boundary.partition, mark)) replies = [hot_view(newer, 30, 40)] - def hot(*args): + def hot(*args, **_kwargs): if isinstance(replies[0], Exception): raise replies[0] return replies[0] backend = KnMarketCacheQueryBackend(self.reader, self.catalog, schema_digest=DIGEST, @@ -1008,7 +1008,7 @@ def read(): def test_non_mark_feeds_never_consult_the_hot_view_for_a_valid_cache_view(self): from types import SimpleNamespace - def hot(*args): + def hot(*args, **_kwargs): raise AssertionError("hot view consulted for a non-MARK feed") backend = KnMarketCacheQueryBackend(self.reader, self.catalog, schema_digest=DIGEST, topic_id=TOPIC_ID, hot_client=SimpleNamespace(latest=hot)) diff --git a/tests/test_phase3_consumer_load_driver.py b/tests/test_phase3_consumer_load_driver.py index bb95ff57..649d7744 100644 --- a/tests/test_phase3_consumer_load_driver.py +++ b/tests/test_phase3_consumer_load_driver.py @@ -593,6 +593,18 @@ def test_a_refusal_records_its_typed_reason_and_bounded_server_diagnostics(self) plain = _MODULE._safe_error(RuntimeError("boom")) self.assertNotIn("reason", plain) self.assertNotIn("diagnostics", plain) + self.assertNotIn("reason_tokens", plain) + + def test_a_refusal_keeps_every_typed_code_of_its_detail(self): + """Astra R6: stage-100 rows had only the detail hash for a nested hot-path reason.""" + from qdl.runtime.refusal_log import reason_tokens + from qdl_sdk.errors import DataLayerError + + detail = "HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND: HOT_READER_BUSY" + evidence = _MODULE._safe_error(DataLayerError("DATA_NOT_READY", detail, retryable=True)) + self.assertEqual(evidence["reason_tokens"], ["HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND", "HOT_READER_BUSY"]) + self.assertEqual(tuple(evidence["reason_tokens"]), reason_tokens(detail)) # same vocabulary as Query's line + self.assertNotIn("reason", evidence) class Kn4FixedWindowHandoffVerdictTests(unittest.TestCase): diff --git a/tests/test_refusal_log.py b/tests/test_refusal_log.py new file mode 100644 index 00000000..73ba4d71 --- /dev/null +++ b/tests/test_refusal_log.py @@ -0,0 +1,117 @@ +"""Astra R6: bounded, payload-free refusal lines at the layer that refused (qdl/runtime/refusal_log.py).""" + +from __future__ import annotations + +import hashlib +import json +import unittest + +from qdl.runtime.refusal_log import RefusalLog, reason_tokens + +DETAIL = "HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND: HOT_READER_BUSY (price 64123.5 for BNB-USDT-SWAP)" + + +class ReasonTokenTests(unittest.TestCase): + def test_typed_codes_only_in_order_deduplicated_and_bounded(self): + self.assertEqual(reason_tokens(DETAIL), ("HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND", "HOT_READER_BUSY")) + self.assertEqual(reason_tokens("required data exceeds its freshness policy (SESSION_STATE)"), + ("SESSION_STATE",)) + self.assertEqual(reason_tokens("plain words, BTC USDT 12_3"), ()) + self.assertEqual(reason_tokens(None), ()) + many = " ".join(f"CODE_{i}" for i in range(10)) + " CODE_0" + self.assertEqual(reason_tokens(many), ("CODE_0", "CODE_1", "CODE_2", "CODE_3")) + self.assertEqual(reason_tokens("A_" + "B" * 80), ()) + + +class RefusalLogTests(unittest.TestCase): + def log(self, **kwargs): + self.clock = [0.0] + self.lines: list[dict] = [] + return RefusalLog(clock=lambda: self.clock[0], emit=lambda line: self.lines.append(json.loads(line)), + replica="query-1", **kwargs) + + def test_a_line_is_payload_free_and_joins_the_harness_digest(self): + log = self.log() + self.assertTrue(log.record(origin="query", code="DATA_NOT_READY", detail=DETAIL, status=503, + retryable=True, request_id="r-1", instrument_uid="u-1", + route="/v2/market-data/{instrument_uid}/snapshot")) + (line,) = self.lines + self.assertEqual(line, { + "event": "qdl_refusal", "origin": "query", "code": "DATA_NOT_READY", "status": 503, + "retryable": True, "reasons": ["HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND", "HOT_READER_BUSY"], + "detail_sha256": hashlib.sha256(DETAIL.encode()).hexdigest(), "request_id": "r-1", + "instrument_uid": "u-1", "route": "/v2/market-data/{instrument_uid}/snapshot", "replica": "query-1", + }) + self.assertNotIn("64123.5", json.dumps(line)) + + def test_volume_is_bounded_per_key_and_the_suppressed_count_is_reported_later(self): + log = self.log(burst=2, window_seconds=10) + emitted = [log.record(origin="query", code="DATA_NOT_READY", detail=DETAIL) for _ in range(7)] + self.assertEqual(emitted, [True, True, False, False, False, False, False]) + # Another reason is another key with its own budget. + self.assertTrue(log.record(origin="query", code="DATA_NOT_READY", detail="SOURCE_BOUNDARY_UNKNOWN: x")) + self.clock[0] = 10.0 + self.assertTrue(log.record(origin="query", code="DATA_NOT_READY", detail=DETAIL)) + self.assertEqual(self.lines[-1]["suppressed_before"], 5) + self.assertTrue(log.record(origin="query", code="DATA_NOT_READY", detail=DETAIL)) + self.assertNotIn("suppressed_before", self.lines[-1]) + self.assertEqual(len(self.lines), 5) + + def test_key_cardinality_is_bounded(self): + log = self.log(burst=1, max_keys=2) + for index in range(5): + log.record(origin="query", code="DATA_NOT_READY", detail=f"REASON_{index}") + self.assertEqual([line["reasons"] for line in self.lines], [["REASON_0"], ["REASON_1"], ["REASON_2"]]) + self.assertEqual(len(log._counts), 3) # two keys + the shared overflow key + self.assertEqual(log._counts[("query", "DATA_NOT_READY", "OVERFLOW")], 3) + + def test_a_failing_sink_never_raises(self): + def broken(_line): + raise OSError("stderr closed") + + log = RefusalLog(emit=broken, replica="q") + self.assertFalse(log.record(origin="query", code="X", detail=DETAIL)) + + def test_default_sink_is_the_qdl_refusal_warning_logger(self): + with self.assertLogs("qdl.refusal", level="WARNING") as captured: + RefusalLog(replica="q").record(origin="query", code="DATA_STALE", detail="x (SESSION_STATE)") + self.assertEqual(json.loads(captured.records[0].getMessage())["reasons"], ["SESSION_STATE"]) + + +class ExecutionMarkIndexEdgeTests(unittest.TestCase): + """The private MARK/INDEX edge keeps the view's own reason, which the reference reader normalises away.""" + + def test_an_unavailable_view_leaves_its_reason(self): + from types import SimpleNamespace + from unittest.mock import patch + + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from qdl.runtime import execution_mark_index as edge + from qdl.runtime.internal_auth import stable_hmac_signature + + class View: + async def read(self, **_kwargs): + return edge.ExecutionMarkIndexRead(None, "MARKET_CACHE_BACKUP_UNAVAILABLE") + + secret = b"s" * 32 + app = FastAPI() + edge.install_execution_mark_index_read( + app, gateway=SimpleNamespace(assert_active=lambda epoch=None: 1), view=View(), secret=secret) + body = json.dumps({"schema": edge._LEGACY_REQUEST_SCHEMA, "instrument_uid": "u-1", + "instrument_revision": 1, "source_policy_id": "p", "max_freshness_ms": 2000}).encode() + lines: list[dict] = [] + log = RefusalLog(emit=lambda line: lines.append(json.loads(line)), replica="q") + with patch.object(edge, "REFUSALS", log): + response = TestClient(app).post("/internal/v2/execution/mark-index/latest", content=body, + headers={"X-QDL-Stable-Signature": stable_hmac_signature(secret, body)}) + self.assertEqual(response.status_code, 409) + (line,) = lines + self.assertEqual((line["origin"], line["reasons"], line["instrument_uid"]), + ("execution_mark_index", ["MARKET_CACHE_BACKUP_UNAVAILABLE"], "u-1")) + self.assertEqual(line["detail_sha256"], hashlib.sha256(response.json()["detail"].encode()).hexdigest()) + + +if __name__ == "__main__": + unittest.main() From 4188bdef7dcd4b2706edc71279c40303d53dd64c Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 07:04:28 +0000 Subject: [PATCH 099/147] docs(plan): stage 100f with fixed harness; G5 attribution same-replica queue; Astra R1/R3/R6 merged --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index e558f7e2..88a32efe 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71802,3 +71802,33 @@ No broad cleanup or disk reclamation claimed; retain active/rollback artifacts while Claude works. Only this plan and advisory changed; no push/merge/rollout. Remaining decision: Claude assesses each recommendation and selects or disproves options against existing gates; this review does not certify release readiness. + +### D3 stage 100f (fixed harness, rc5): 28/32; R5 attribution measured; rc5 ordering tests; Astra R1/R3/R6 - 2026-10-10 (Claude) + +Run 06:30-07:03Z under supervisor v2 (process-group scoped; guard and watcher target the driver PID). Driver 713aa90 ++ 7f974c3, profile with core/kafka monitored, Query rc5. Guard: 1 non-READY sample. MARK watcher: 28 minutes with 0 +errors, 5 with 1, 0 UNKNOWN. +- PASS: streams 179 / 0 errors; latency classes; TS v2_error / auth / disconnect-rate gates. +- ts:ready_60_every_sample FAIL, now scored on all 154 samples (0 unreadable, expected 154): 3 not ready. The TS + writer logged DATA_NOT_READY 12 and SOURCE_UNAVAILABLE 10 in the run. +- requests:no_failures FAIL: 3 DATA_NOT_READY. The detail hash matches `HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND: + HOT_READER_BUSY` (Astra R2, Rust hot reader contention). +- startup:bounded FAIL: s20 RATE_LIMITED after 208 retries in 74.7 s. +- g5:COLD_2500:BINANCE FAIL 7315.9 ms. Attribution (R5) from 7f974c3: + - both 2500-row reads went to query_v2_1; + - OKX was sent 0.15 s earlier: transport 3469 ms, client 242 ms; + - Binance: transport 7031 ms, client 285 ms; + - the 5000-row reads both went to query_v2_2 (6004 / 7729 ms transport). + The cause is same-replica queueing on the one-active large-warmup lane at about 3.5 s per 2500-row read under load. + Client validation is not the cause. The worst case (same replica) must pass, without relying on random placement. +- Also merged: + - 4e19966: rc5 real-ordering tests (Codex point 4); + - 57881a2 (from 6b0abdb), Astra R1/R3/R6, tested locally, not deployed: + - R1: hard hot refusals in the IDR-084 branch map to FENCED/INTEGRITY, never to an older cache OK. + - R3: memo lock wait and lookup share one budget (HOT_READ_DEADLINE is transient and not memoised). + - R6: payload-free, rate-bounded `qdl_refusal` log lines carrying the same detail_sha256 as the receipts; + receipts gain reason_tokens. + - Tests: 183 OK (33 Redis skips); after the merge, 9 modules OK. +- Next (rc6): QDL_QUERY_COLD_SLICE_MS 4 -> 8 (the one tuned variable; Astra R5 option 2) to cut per-read service time + below ~3.1 s for the same-replica worst case, scored together with the hot tails. Plus the R2 Rust hot-reader fix + (Stream image) and R1/R3/R6 (Query image). Then stage 100g. From f861663fd0243762d4107a5115aa7dec9990d78c Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 06:50:15 +0000 Subject: [PATCH 100/147] fix(ingestor): OKX make-before-break on notice 64008, immediate reconnect, session lineage Stage 100e refused OKX BNB-USDT-SWAP TRADE (DATA_STALE, SOURCE_SESSION_UNKNOWN, event recency LIVE): the lane liveness file was replaced by generation 168, so the last trade of generation 167 matched no session record. Ingestor (OKX lanes): - On event=notice (64008, sent 60 s before a service upgrade) open and subscribe a standby session while the old socket keeps publishing; hand over once every subscription is ACKed. The standby never publishes before the old generation's inflight is drained, so both generation fences still see N strictly before N+1. - A session that delivered data reconnects without the fixed 400 ms backoff; repeated failures still back off. One per-process pacer keeps connection requests under OKX's 3 per second per IP. - Keepalive: a frame arriving while a ping is outstanding no longer tears the session down and is no longer dropped; text "pong" outside the wait no longer fails the runtime. - Disconnect log carries lane, session, cause (NOTICE_HANDOVER, NOTICE, CLOSE_FRAME, EOF, READ_ERROR, WRITE_ERROR, HEARTBEAT_TIMEOUT, BOOK_RESYNC), close code/reason, last_frame_to_detected_ms and drain_ms; a new qdl_native_session_ready event records reconnect timings. - Overlapped successions are written to lineage/.lineage (qdl.provider-session-lineage.v1), reset by any gap reconnect. Query reader: an event whose session is no longer the lane record is judged by the chain's current session only when a lineage chain proves overlap; flag SOURCE_SESSION_CONTINUED. Every doubtful case fails closed. Old readers never list the subdirectory and the healthcheck matches only *.json, so reader and writer roll out in either order. Tests: ingestor bin 36 (28 + 8 new), session liveness 36 incl. 10 new lineage tests, mark-index live view 35. --- qdl/runtime/session_liveness.py | 150 +- .../src/bin/qdl-native-raw-ingestor.rs | 1447 +++++++++++++---- tests/test_session_liveness_lineage.py | 156 ++ 3 files changed, 1420 insertions(+), 333 deletions(-) create mode 100644 tests/test_session_liveness_lineage.py diff --git a/qdl/runtime/session_liveness.py b/qdl/runtime/session_liveness.py index 976bc093..6acf95f9 100644 --- a/qdl/runtime/session_liveness.py +++ b/qdl/runtime/session_liveness.py @@ -19,9 +19,14 @@ SESSION_LIVENESS_SCHEMA = "qdl.provider-session-liveness.v1" +SESSION_LINEAGE_SCHEMA = "qdl.provider-session-lineage.v1" +SESSION_CONTINUED_FLAG = "SOURCE_SESSION_CONTINUED" _SESSION_STATES = frozenset({"LIVE", "DISCONNECTED"}) _SAFE_FILE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,119}\.json$") +_SAFE_LINEAGE_FILE = re.compile(r"^[a-z0-9][a-z0-9-]{0,119}\.lineage$") +_LINEAGE_DIRECTORY = "lineage" _MAX_STATE_FILES = 256 +_MAX_LINEAGE_LINKS = 64 _MAX_SESSION_ID_LENGTH = 256 @@ -87,6 +92,71 @@ def from_mapping(cls, value: object) -> "ProviderSessionLiveness": ) +@dataclass(frozen=True, slots=True) +class ProviderSessionLineage: + """Sessions of one lane that succeeded each other with no uncovered window. + + The Rust ingestor writes ``lineage/.lineage`` only for a + make-before-break handover (OKX service-upgrade notice): every link was + subscribed while its predecessor still read, and the predecessor stopped + only after that. While the chain's last session is live, the last event an + earlier link published for a binding is still that binding's latest event; + anything newer would have come from a later link. A reconnect with a gap + starts a new chain, so it never vouches for an older session. + """ + + lane: str + chain: tuple[tuple[str, int], ...] + config_revision: int + + @classmethod + def from_mapping(cls, value: object) -> "ProviderSessionLineage": + if not isinstance(value, dict) or set(value) != { + "schema", + "lane", + "chain", + "updated_at_ns", + "config_revision", + }: + raise ValueError("provider session lineage schema is invalid") + if value["schema"] != SESSION_LINEAGE_SCHEMA: + raise ValueError("provider session lineage schema is unsupported") + chain = value["chain"] + revision = value["config_revision"] + if ( + not isinstance(value["lane"], str) + or not value["lane"] + or not isinstance(chain, list) + or not 1 <= len(chain) <= _MAX_LINEAGE_LINKS + or isinstance(revision, bool) + or not isinstance(revision, int) + or revision < 1 + ): + raise ValueError("provider session lineage fields are invalid") + links: list[tuple[str, int]] = [] + for link in chain: + if not isinstance(link, dict) or set(link) != { + "source_session_id", + "connection_generation", + }: + raise ValueError("provider session lineage link is invalid") + session = link["source_session_id"] + generation = link["connection_generation"] + if ( + not isinstance(session, str) + or not session + or len(session) > _MAX_SESSION_ID_LENGTH + or isinstance(generation, bool) + or not isinstance(generation, int) + or generation < 1 + # Each successor was allocated after its predecessor. + or (links and generation <= links[-1][1]) + ): + raise ValueError("provider session lineage link is invalid") + links.append((session, generation)) + return cls(lane=value["lane"], chain=tuple(links), config_revision=revision) + + @dataclass(frozen=True, slots=True) class ProviderSessionStatus: """Lookup result deliberately carries no raw provider payload.""" @@ -114,11 +184,14 @@ def __init__(self, root: str | Path, *, clock_ns: Callable[[], int] | None = Non # Parsed file contents by directory and name, keyed by the file's # (inode, mtime, size). Only content is kept - never a verdict: age, # skew and the session match are judged on every call. - self._parsed: dict[str, dict[str, tuple[tuple[int, int, int], ProviderSessionLiveness | None]]] = {} + self._parsed: dict[str, dict[str, tuple[tuple[int, int, int], object | None]]] = {} def _records( - self, directory: Path, names: list[str] - ) -> list[ProviderSessionLiveness | None]: + self, + directory: Path, + names: list[str], + parse: Callable[[object], object] = ProviderSessionLiveness.from_mapping, + ) -> list: """The records of ``names``, each re-read only when its file changed. The ingestor replaces a record by rename (a new inode), so the stat key @@ -128,8 +201,8 @@ def _records( """ previous = self._parsed.get(str(directory), {}) - current: dict[str, tuple[tuple[int, int, int], ProviderSessionLiveness | None]] = {} - records: list[ProviderSessionLiveness | None] = [] + current: dict[str, tuple[tuple[int, int, int], object | None]] = {} + records: list = [] for name in names: path = directory / name try: @@ -143,9 +216,7 @@ def _records( record = cached[1] else: try: - record = ProviderSessionLiveness.from_mapping( - json.loads(path.read_text(encoding="utf-8")) - ) + record = parse(json.loads(path.read_text(encoding="utf-8"))) except (OSError, UnicodeError, json.JSONDecodeError, ValueError): record = None current[name] = (key, record) @@ -202,7 +273,8 @@ def status( return ProviderSessionStatus("UNKNOWN", None, ("SOURCE_SESSION_UNAVAILABLE",)) match: ProviderSessionLiveness | None = None malformed = False - for record in self._records(directory, candidates): + records = self._records(directory, candidates) + for record in records: if record is None: malformed = True continue @@ -213,6 +285,15 @@ def status( if match is not None: return ProviderSessionStatus("UNKNOWN", None, ("SOURCE_SESSION_AMBIGUOUS",)) match = record + flags: tuple[str, ...] = () + if match is None: + match = self._successor( + directory, + [record for record in records if record is not None], + source_session_id, + connection_generation, + ) + flags = (SESSION_CONTINUED_FLAG,) if match is None: flag = "SOURCE_SESSION_MALFORMED" if malformed else "SOURCE_SESSION_UNAVAILABLE" return ProviderSessionStatus("UNKNOWN", None, (flag,)) @@ -228,4 +309,53 @@ def status( if match.last_transport_at_ns > evaluated_ns: return ProviderSessionStatus("UNKNOWN", None, ("SOURCE_SESSION_CLOCK_SKEW",)) age_ms = (evaluated_ns - match.last_transport_at_ns) // 1_000_000 - return ProviderSessionStatus(match.state, int(age_ms)) + return ProviderSessionStatus(match.state, int(age_ms), flags) + + def _successor( + self, + directory: Path, + records: list[ProviderSessionLiveness], + source_session_id: str, + connection_generation: int, + ) -> ProviderSessionLiveness | None: + """The live record that continues an older session without a gap. + + Only a lineage chain proves continuity, and only when its last link is + exactly one current session record of the same config revision. Any + doubt (unreadable lineage, two chains, no or two successor records) + returns None, so the caller fails closed as before. + """ + + lineage_directory = directory / _LINEAGE_DIRECTORY + try: + names = [] + with os.scandir(lineage_directory) as entries: + for entry in entries: + if entry.is_file() and _SAFE_LINEAGE_FILE.fullmatch(entry.name): + names.append(entry.name) + if len(names) > _MAX_STATE_FILES: + return None + names.sort() + except OSError: + return None + if not names: + self._parsed.pop(str(lineage_directory), None) + return None + target = (source_session_id, connection_generation) + successors: list[ProviderSessionLiveness] = [] + for lineage in self._records( + lineage_directory, names, ProviderSessionLineage.from_mapping + ): + if lineage is None or target not in lineage.chain[:-1]: + continue + last = lineage.chain[-1] + current = [ + record + for record in records + if (record.source_session_id, record.connection_generation) == last + and record.config_revision == lineage.config_revision + ] + if len(current) != 1: + return None + successors.append(current[0]) + return successors[0] if len(successors) == 1 else None diff --git a/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs b/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs index 4f8cffe1..d2256453 100644 --- a/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs +++ b/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs @@ -608,6 +608,16 @@ impl SessionLivenessWriter { self.write(session_id, generation, "LIVE", transport_at_ns, false) } + /// First record of a new session: written at once, never throttled. + fn begin( + &mut self, + session_id: &str, + generation: u64, + transport_at_ns: i64, + ) -> Result<(), Box> { + self.write(session_id, generation, "LIVE", transport_at_ns, true) + } + fn disconnected( &mut self, session_id: &str, @@ -645,11 +655,6 @@ impl SessionLivenessWriter { { return Ok(()); } - let parent = self - .path - .parent() - .ok_or("session liveness path has no parent")?; - fs::create_dir_all(parent)?; let payload = serde_json::to_vec(&ProviderSessionLiveness { schema: SESSION_LIVENESS_SCHEMA, source_session_id: session_id, @@ -659,22 +664,134 @@ impl SessionLivenessWriter { updated_at_ns: now_ns()?, config_revision: self.config_revision, })?; - let temporary = - self.path - .with_extension(format!("tmp-{}-{}", std::process::id(), now_ns()?)); - let mut file = OpenOptions::new() - .create_new(true) - .write(true) - .open(&temporary)?; - file.write_all(&payload)?; - file.sync_all()?; - fs::rename(&temporary, &self.path)?; - File::open(parent)?.sync_all()?; + write_state_atomically(&self.path, &payload)?; self.last_written_ns = Some(transport_at_ns); Ok(()) } } +/// Replace a small state file by rename, so a reader sees the old or the new +/// content, never a partial one. +fn write_state_atomically( + path: &Path, + payload: &[u8], +) -> Result<(), Box> { + let parent = path.parent().ok_or("state path has no parent")?; + fs::create_dir_all(parent)?; + let temporary = path.with_extension(format!("tmp-{}-{}", std::process::id(), now_ns()?)); + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&temporary)?; + file.write_all(payload)?; + file.sync_all()?; + fs::rename(&temporary, path)?; + File::open(parent)?.sync_all()?; + Ok(()) +} + +const SESSION_LINEAGE_SCHEMA: &str = "qdl.provider-session-lineage.v1"; +const MAX_SESSION_LINEAGE_LINKS: usize = 32; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct SessionLink { + source_session_id: String, + connection_generation: u64, +} + +#[derive(Serialize)] +struct ProviderSessionLineage<'a> { + schema: &'static str, + lane: &'a str, + chain: Vec, + updated_at_ns: i64, + config_revision: u64, +} + +/// Per-lane chain of sessions that succeeded each other with no uncovered +/// window (make-before-break on an OKX service-upgrade notice). +/// +/// Each link was subscribed and ACKed while its predecessor was still +/// reading, and the predecessor stopped reading only after that. So the last +/// event a predecessor published for a binding stays that binding's latest +/// event for as long as the chain's last session is live: anything newer was +/// published by a later link. A reconnect that left a gap starts a new chain. +/// The file is `lineage/.lineage` under the liveness directory: +/// liveness readers that predate it list only regular files of that +/// directory, and the container healthcheck (`find -name '*.json'`, which +/// does recurse) does not match the extension. +struct SessionLineageWriter { + path: PathBuf, + lane: String, + config_revision: u64, + chain: VecDeque, +} + +impl SessionLineageWriter { + fn new( + config: &IngestorConfig, + lane: &str, + ) -> Result> { + Self::at( + Path::new(&config.session_liveness_dir), + lane, + config.config_revision, + ) + } + + fn at( + liveness_dir: &Path, + lane: &str, + config_revision: u64, + ) -> Result> { + if lane.is_empty() + || lane.len() > 120 + || !lane.chars().all(|character| { + character.is_ascii_lowercase() || character.is_ascii_digit() || character == '-' + }) + { + return Err("session lineage lane is invalid".into()); + } + Ok(Self { + path: liveness_dir.join("lineage").join(format!("{lane}.lineage")), + lane: lane.into(), + config_revision, + chain: VecDeque::new(), + }) + } + + /// Record a new session; `continued` only when it took over from the + /// chain's last session with overlap. + fn begin( + &mut self, + session_id: &str, + generation: u64, + continued: bool, + ) -> Result<(), Box> { + if session_id.is_empty() || session_id.len() > 256 || generation == 0 { + return Err("provider session lineage identity is invalid".into()); + } + if !continued { + self.chain.clear(); + } + self.chain.push_back(SessionLink { + source_session_id: session_id.into(), + connection_generation: generation, + }); + while self.chain.len() > MAX_SESSION_LINEAGE_LINKS { + self.chain.pop_front(); + } + let payload = serde_json::to_vec(&ProviderSessionLineage { + schema: SESSION_LINEAGE_SCHEMA, + lane: &self.lane, + chain: self.chain.iter().cloned().collect(), + updated_at_ns: now_ns()?, + config_revision: self.config_revision, + })?; + write_state_atomically(&self.path, &payload) + } +} + fn capture_id(session: &str, generation: u64, received_at_ns: i64, frame: &[u8]) -> Vec { let mut digest = Sha256::new(); digest.update(session.as_bytes()); @@ -2211,13 +2328,264 @@ struct OkxServiceShard { bindings: Vec, } +type BoxError = Box; +type OkxSocket = + tokio_tungstenite::WebSocketStream>; +type OkxWriter = futures_util::stream::SplitSink; +type OkxReader = futures_util::stream::SplitStream; +type OkxStandby = Pin> + Send>>; + +/// OKX documents at most three WebSocket connection requests per second per +/// IP. Every OKX lane of this process connects through one pacer, so an +/// immediate reconnect of several lanes at once cannot exceed that limit. +const OKX_CONNECT_SPACING: Duration = Duration::from_millis(350); + +#[derive(Clone, Default)] +struct OkxConnectPacer { + next: Arc>>, +} + +impl OkxConnectPacer { + async fn admit(&self) { + let mut next = self.next.lock().await; + if let Some(at) = *next { + tokio::time::sleep_until(at).await; + } + *next = Some(Instant::now() + OKX_CONNECT_SPACING); + } +} + +/// Why an OKX session ended; logged with every disconnect so the cause of a +/// stale window is evidence, not inference. +#[derive(Clone, Debug, Eq, PartialEq)] +enum OkxSessionEnd { + /// `event: notice` (64008: service upgrade). The standby session took over + /// before this one stopped reading; no frame window was left uncovered. + NoticeHandover { + code: String, + }, + /// `event: notice`, but no standby session could be established before + /// the old connection ended. + Notice { + code: String, + }, + Closed { + code: Option, + reason: String, + }, + Eof, + ReadError(String), + WriteError(String), + HeartbeatTimeout, + BookResync, +} + +impl OkxSessionEnd { + fn cause(&self) -> &'static str { + match self { + Self::NoticeHandover { .. } => "NOTICE_HANDOVER", + Self::Notice { .. } => "NOTICE", + Self::Closed { .. } => "CLOSE_FRAME", + Self::Eof => "EOF", + Self::ReadError(_) => "READ_ERROR", + Self::WriteError(_) => "WRITE_ERROR", + Self::HeartbeatTimeout => "HEARTBEAT_TIMEOUT", + Self::BookResync => "BOOK_RESYNC", + } + } + + fn detail(&self) -> Value { + match self { + Self::NoticeHandover { code } | Self::Notice { code } => json!({"code": code}), + Self::Closed { code, reason } => json!({"code": code, "reason": reason}), + Self::ReadError(error) | Self::WriteError(error) => json!({"error": error}), + Self::Eof | Self::HeartbeatTimeout | Self::BookResync => Value::Null, + } + } + + fn handed_over(&self) -> bool { + matches!(self, Self::NoticeHandover { .. }) + } +} + +/// The OKX service-upgrade notice code, or None for any other payload. +/// +/// OKX pushes `{"event":"notice","code":"64008",...}` 60 seconds before a +/// service upgrade closes the connection (public, private and, since +/// 2026-06-11, business endpoints). +fn okx_notice_code(payload: &Value) -> Option { + (payload.get("event").and_then(Value::as_str) == Some("notice")).then(|| { + payload + .get("code") + .and_then(Value::as_str) + .unwrap_or("missing") + .to_owned() + }) +} + +/// Delay before reconnecting after a session ended. +/// +/// `failures` counts consecutive session ends and connect failures and is +/// reset by any durable publish. A session that delivered data reconnects at +/// once (`failures == 1`): a fixed backoff there is pure stale window. A +/// session that ended again without delivering anything backs off as before. +fn okx_reconnect_delay_ms(backoff: BackoffPolicy, failures: u32) -> u64 { + if failures <= 1 { + 0 + } else { + backoff.delay_ms(failures, failures.min(10_000) as u16) + } +} + +/// Idle-read outcome of the OKX keepalive: OKX closes a connection that has +/// pushed nothing for 30 seconds; a client sends the text `ping` after N < 30 +/// idle seconds and reconnects when no `pong` follows within N seconds. +#[derive(Debug, Eq, PartialEq)] +enum OkxKeepalive { + SendPing, + Dead, +} + +fn okx_keepalive_on_idle(ping_outstanding: bool) -> OkxKeepalive { + if ping_outstanding { + OkxKeepalive::Dead + } else { + OkxKeepalive::SendPing + } +} + +struct OkxEstablished { + writer: OkxWriter, + reader: OkxReader, + generation: u64, + session_id: String, + pre_ack_frames: VecDeque, + connect_started_ns: i64, + ready_at_ns: i64, + /// A service-upgrade notice arrived while subscribing; this session must + /// be replaced as soon as it is live. + notice_code: Option, +} + +#[derive(Debug)] +enum OkxEstablishError { + Connect(BoxError), + Subscribe(BoxError), +} + +impl std::fmt::Display for OkxEstablishError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Connect(error) => write!(formatter, "connect: {error}"), + Self::Subscribe(error) => write!(formatter, "subscribe: {error}"), + } + } +} + +/// Connect one OKX session and wait until every subscription is ACKed. +/// +/// It never publishes: frames that arrive before the last ACK are returned in +/// `pre_ack_frames`, stamped with this session's identity, for the caller to +/// publish only after the previous generation has stopped publishing. That +/// keeps the connection-generation fences (ordering tracker and L2 book core) +/// seeing generation N strictly before N+1 even when the two sockets overlap. +async fn okx_establish( + url: String, + subscriptions: Arc>, + bindings: Arc>, + session_prefix: String, + generation: u64, + max_pre_ack_frames: usize, + pacer: OkxConnectPacer, +) -> Result { + pacer.admit().await; + let connect_started_ns = now_ns().map_err(OkxEstablishError::Connect)?; + let (socket, _) = connect_async(&url) + .await + .map_err(|error| OkxEstablishError::Connect(error.into()))?; + let (mut writer, mut reader) = socket.split(); + let subscribed: Result<_, BoxError> = async { + let command_id = generation.to_string(); + writer + .send(Message::Text(okx_subscription_command( + &command_id, + &subscriptions, + )?)) + .await?; + let session_id = format!("{session_prefix}-{generation}-{}", now_ns()?); + let mut pending = subscriptions + .iter() + .map(|item| (item.channel.clone(), item.inst_id.clone())) + .collect::>(); + let mut pre_ack_frames = VecDeque::new(); + let mut notice_code = None; + while !pending.is_empty() { + let message = tokio::time::timeout(Duration::from_secs(10), reader.next()) + .await + .map_err(|_| "OKX subscription ACK timed out")? + .ok_or("OKX socket closed before subscription ACK")??; + match message { + Message::Ping(payload) => { + writer.send(Message::Pong(payload)).await?; + } + Message::Pong(_) => {} + Message::Text(payload) => { + let raw_text = payload.to_string(); + if raw_text == "pong" { + continue; + } + let decoded: Value = serde_json::from_str(&raw_text)?; + if let Some(code) = okx_notice_code(&decoded) { + notice_code = Some(code); + continue; + } + if parse_okx_subscription_ack(&decoded, &command_id, &mut pending)? { + continue; + } + if pre_ack_frames.len() >= max_pre_ack_frames { + return Err( + "OKX pre-ACK frame buffer exceeded durable publish bound".into() + ); + } + pre_ack_frames.push_back(pending_okx_frame( + &bindings, + &session_id, + generation, + raw_text, + now_ns()?, + )?); + } + Message::Close(_) => { + return Err("OKX socket closed before subscription ACK".into()); + } + _ => {} + } + } + Ok::<_, BoxError>((session_id, pre_ack_frames, notice_code)) + } + .await; + let (session_id, pre_ack_frames, notice_code) = + subscribed.map_err(OkxEstablishError::Subscribe)?; + Ok(OkxEstablished { + writer, + reader, + generation, + session_id, + pre_ack_frames, + connect_started_ns, + ready_at_ns: now_ns().map_err(OkxEstablishError::Subscribe)?, + notice_code, + }) +} + async fn run_okx_service( shard: OkxServiceShard, config: Arc, accepted: Arc, coalesced_latest: Arc, stopped: Arc, -) -> Result<(), Box> { + pacer: OkxConnectPacer, +) -> Result<(), BoxError> { let OkxServiceShard { service, shard_index, @@ -2227,38 +2595,28 @@ async fn run_okx_service( if bindings.is_empty() { return Ok(()); } - let publisher = RawPublisher::new( - &config, - &format!( - "{}-{shard_index:03}", - match service { - OkxService::Public => "okx-public", - OkxService::Business => "okx-business", - } - ), - )?; - let subscriptions: Vec = bindings - .iter() - .map(|binding| OkxSubscription { - channel: binding.native_channel.clone(), - inst_id: binding.native_symbol.clone(), - }) - .collect(); - let binding_map: HashMap = bindings - .into_iter() - .map(|binding| (binding.key(), binding)) - .collect(); - let mut resync_listener = book_resync_listener( - &config, - &format!( - "okx-{}-{shard_index:03}", - match service { - OkxService::Public => "public", - OkxService::Business => "business", - } - ), - &binding_map, - )?; + let service_name = match service { + OkxService::Public => "public", + OkxService::Business => "business", + }; + let lane = format!("okx-{service_name}-{shard_index:03}"); + let publisher = RawPublisher::new(&config, &lane)?; + let subscriptions: Arc> = Arc::new( + bindings + .iter() + .map(|binding| OkxSubscription { + channel: binding.native_channel.clone(), + inst_id: binding.native_symbol.clone(), + }) + .collect(), + ); + let binding_map: Arc> = Arc::new( + bindings + .into_iter() + .map(|binding| (binding.key(), binding)) + .collect(), + ); + let mut resync_listener = book_resync_listener(&config, &lane, &binding_map)?; let backoff = BackoffPolicy { initial_ms: 250, maximum_ms: 30_000, @@ -2267,259 +2625,202 @@ async fn run_okx_service( } .validate()?; let expires = deadline(config.max_runtime_seconds); - let service_name = match service { - OkxService::Public => "public", - OkxService::Business => "business", - }; - let generation_path = format!( - "{}.okx-{service_name}-{shard_index:03}", - config.generation_state_path - ); + let generation_path = format!("{}.{lane}", config.generation_state_path); let mut failures = 0_u32; let mut budget = ControlRequestBudget::default(); + let mut liveness = SessionLivenessWriter::new(&config, &lane)?; + let mut lineage = SessionLineageWriter::new(&config, &lane)?; + // A standby session that was already established when the previous one + // handed over; it is used instead of opening a new connection. + let mut handover: Option = None; + // When the previous session ended, for the reconnect-time evidence. + let mut previous_end: Option<(String, u64, i64, i64)> = None; + let establish = |generation: u64| { + okx_establish( + url.clone(), + subscriptions.clone(), + binding_map.clone(), + lane.clone(), + generation, + config.max_inflight_publishes, + pacer.clone(), + ) + }; while !should_stop(&stopped, &accepted, config.max_events, expires) { - let generation = next_connection_generation(Path::new(&generation_path))?; - match connect_async(&url).await { - Ok((socket, _)) => { - let (mut writer, mut reader) = socket.split(); - let command_id = generation.to_string(); + let continued = handover.is_some(); + let established = match handover.take() { + Some(established) => established, + None => { + let generation = next_connection_generation(Path::new(&generation_path))?; budget.permit(now_ns()?)?; - writer - .send(Message::Text(okx_subscription_command( - &command_id, - &subscriptions, - )?)) - .await?; - let session_id = format!( - "okx-{}-{shard_index:03}-{generation}-{}", - match service { - OkxService::Public => "public", - OkxService::Business => "business", - }, - now_ns()? - ); - let mut pending = subscriptions - .iter() - .map(|item| (item.channel.clone(), item.inst_id.clone())) - .collect::>(); - let mut pre_ack_frames = VecDeque::new(); - while !pending.is_empty() { - let message = tokio::time::timeout(Duration::from_secs(10), reader.next()) - .await - .map_err(|_| "OKX subscription ACK timed out")? - .ok_or("OKX socket closed before subscription ACK")??; - match message { - Message::Ping(payload) => { - writer.send(Message::Pong(payload)).await?; - } - Message::Pong(_) => {} - Message::Text(payload) => { - let raw_text = payload.to_string(); - let decoded: Value = serde_json::from_str(&raw_text)?; - if parse_okx_subscription_ack(&decoded, &command_id, &mut pending)? { - continue; - } - if pre_ack_frames.len() >= config.max_inflight_publishes { - return Err( - "OKX pre-ACK frame buffer exceeded durable publish bound" - .into(), - ); - } - pre_ack_frames.push_back(pending_okx_frame( - &binding_map, - &session_id, - generation, - raw_text, - now_ns()?, - )?); - } - Message::Close(_) => { - return Err("OKX socket closed before subscription ACK".into()); - } - _ => {} + match establish(generation).await { + Ok(established) => established, + Err(OkxEstablishError::Subscribe(error)) => return Err(error), + Err(OkxEstablishError::Connect(error)) => { + failures = failures.saturating_add(1); + eprintln!( + "{}", + serde_json::to_string(&json!({ + "event": "qdl_native_connect_failed", + "runtime": "OKX", + "service": format!("{:?}", service).to_ascii_uppercase(), + "lane": lane, + "attempt": failures, + "error": error.to_string(), + }))? + ); + tokio::time::sleep(Duration::from_millis( + backoff.delay_ms(failures, failures.min(10_000) as u16), + )) + .await; + continue; } } - let mut liveness = SessionLivenessWriter::new( - &config, - &format!("okx-{service_name}-{shard_index:03}"), - )?; - liveness.live(&session_id, generation, now_ns()?)?; - let mut inflight = FuturesUnordered::::new(); - let mut latest = LatestStateBuffer::default(); - let mut latest_tick = - tokio::time::interval(Duration::from_millis(config.latest_state_flush_ms)); - latest_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - latest_tick.tick().await; - match book_session_bootstrap(ProviderRuntime::Okx, &binding_map) { - Some(BookSessionBootstrap::InitialSnapshotAndGapResync) | None => {} - Some(BookSessionBootstrap::PeriodicRestAnchor(_)) => { - return Err("OKX BOOK lane cannot use a periodic REST-anchor policy".into()) + } + }; + let OkxEstablished { + mut writer, + mut reader, + generation, + session_id, + mut pre_ack_frames, + connect_started_ns, + ready_at_ns, + notice_code: early_notice, + } = established; + // Lineage first, then the session record: a reader that sees the new + // record can always resolve the predecessor through the chain. + lineage.begin(&session_id, generation, continued)?; + liveness.begin(&session_id, generation, ready_at_ns)?; + eprintln!( + "{}", + serde_json::to_string(&json!({ + "event": "qdl_native_session_ready", + "runtime": "OKX", + "service": format!("{:?}", service).to_ascii_uppercase(), + "lane": lane, + "generation": generation, + "session": session_id, + "continued": continued, + "connect_to_ready_ms": (ready_at_ns - connect_started_ns) / 1_000_000, + "previous_session": previous_end.as_ref().map(|value| value.0.clone()), + "previous_generation": previous_end.as_ref().map(|value| value.1), + "previous_last_frame_to_ready_ms": previous_end + .as_ref() + .map(|value| (ready_at_ns - value.2) / 1_000_000), + "previous_end_to_ready_ms": previous_end + .as_ref() + .map(|value| (ready_at_ns - value.3) / 1_000_000), + }))? + ); + previous_end = None; + let mut inflight = FuturesUnordered::::new(); + let mut latest = LatestStateBuffer::default(); + let mut latest_tick = + tokio::time::interval(Duration::from_millis(config.latest_state_flush_ms)); + latest_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + latest_tick.tick().await; + match book_session_bootstrap(ProviderRuntime::Okx, &binding_map) { + Some(BookSessionBootstrap::InitialSnapshotAndGapResync) | None => {} + Some(BookSessionBootstrap::PeriodicRestAnchor(_)) => { + return Err("OKX BOOK lane cannot use a periodic REST-anchor policy".into()) + } + } + let mut ended: Option = None; + let mut publish_error = None; + let mut exhausted = false; + let mut last_frame_at_ns = ready_at_ns; + let mut ping_outstanding = false; + let mut standby: Option = None; + let mut notice: Option = None; + if let Some(code) = early_notice { + let generation = next_connection_generation(Path::new(&generation_path))?; + budget.permit(now_ns()?)?; + standby = Some(Box::pin(establish(generation))); + notice = Some(code); + } + while let Some(frame) = pre_ack_frames.pop_front() { + if !publish_pending_frame( + frame, + &mut PendingPublishWindow { + publisher: &publisher, + latest: &mut latest, + inflight: &mut inflight, + accepted: &accepted, + coalesced_latest: &coalesced_latest, + max_events: config.max_events, + max_inflight: config.max_inflight_publishes, + stopped: &stopped, + }, + ) + .await? + { + exhausted = true; + break; + } + } + while !exhausted && !should_stop(&stopped, &accepted, config.max_events, expires) { + if inflight.len() >= config.max_inflight_publishes { + match inflight.next().await { + Some(Ok(())) => { + failures = 0; + continue; + } + Some(Err(error)) => { + publish_error = Some(error); + break; } + None => return Err("lossless publish window became inconsistent".into()), } - let mut disconnected = false; - let mut publish_error = None; - let mut exhausted = false; - while let Some(frame) = pre_ack_frames.pop_front() { - if !publish_pending_frame( - frame, - &mut PendingPublishWindow { - publisher: &publisher, - latest: &mut latest, - inflight: &mut inflight, - accepted: &accepted, - coalesced_latest: &coalesced_latest, - max_events: config.max_events, - max_inflight: config.max_inflight_publishes, - stopped: &stopped, - }, - ) - .await? - { - exhausted = true; + } + let read = + tokio::time::timeout(Duration::from_secs(config.heartbeat_seconds), reader.next()); + tokio::pin!(read); + // Control backlog must not take priority over socket and publish progress. + let outcome = tokio::select! { + requested = book_resync_requested(&mut resync_listener, &config, &binding_map, &session_id, generation) => { + if requested? { + eprintln!("{{\"event\":\"qdl_native_book_resync_requested\",\"session\":\"{}\",\"generation\":{}}}", session_id, generation); + ended = Some(OkxSessionEnd::BookResync); break; } + continue; } - while !exhausted && !should_stop(&stopped, &accepted, config.max_events, expires) { - if inflight.len() >= config.max_inflight_publishes { - match inflight.next().await { - Some(Ok(())) => { - failures = 0; - continue; - } - Some(Err(error)) => { - publish_error = Some(error); - break; - } - None => { - return Err("lossless publish window became inconsistent".into()) - } - } + ready = async { + match standby.as_mut() { + Some(future) => future.await, + None => std::future::pending().await, } - let read = tokio::time::timeout( - Duration::from_secs(config.heartbeat_seconds), - reader.next(), - ); - tokio::pin!(read); - // Control backlog must not take priority over socket and publish progress. - let outcome = tokio::select! { - requested = book_resync_requested(&mut resync_listener, &config, &binding_map, &session_id, generation) => { - if requested? { - eprintln!("{{\"event\":\"qdl_native_book_resync_requested\",\"session\":\"{}\",\"generation\":{}}}", session_id, generation); - disconnected = true; - break; - } - continue; - } - _ = latest_tick.tick(), if !latest.is_empty() => { - if !flush_latest_concurrent( - &mut latest, - &publisher, - &mut inflight, - &accepted, - config.max_events, - config.max_inflight_publishes, - &stopped, - ).await? { - break; - } - failures = 0; - continue; - } - result = &mut read => Some(result), - completed = inflight.next(), if !inflight.is_empty() => { - match completed { - Some(Ok(())) => { - failures = 0; - continue; - } - Some(Err(error)) => { - publish_error = Some(error); - break; - } - None => { - return Err("lossless publish window became inconsistent".into()) - } - } - } - }; - let message = match outcome.expect("OKX read outcome is present") { - Ok(Some(Ok(message))) => { - if !matches!(message, Message::Close(_)) { - liveness.live(&session_id, generation, now_ns()?)?; - } - message - } - Ok(Some(Err(_))) | Ok(None) => { - liveness.disconnected(&session_id, generation, now_ns()?)?; - disconnected = true; + } => { + standby = None; + match ready { + Ok(next) => { + // Make-before-break: the successor is subscribed + // while this socket still delivers, so stopping + // here leaves no uncovered window. + handover = Some(next); + ended = Some(OkxSessionEnd::NoticeHandover { + code: notice.clone().unwrap_or_default(), + }); break; } - Err(_) => { - writer.send(Message::Text("ping".into())).await?; - match tokio::time::timeout( - Duration::from_secs(config.heartbeat_seconds), - reader.next(), - ) - .await - { - Ok(Some(Ok(message))) - if message.is_text() && message.to_text()? == "pong" => - { - liveness.live(&session_id, generation, now_ns()?)?; - continue; - } - _ => { - liveness.disconnected(&session_id, generation, now_ns()?)?; - disconnected = true; - break; - } - } + Err(error) => { + eprintln!( + "{}", + serde_json::to_string(&json!({ + "event": "qdl_native_standby_failed", + "runtime": "OKX", + "lane": lane, + "generation": generation, + "error": error.to_string(), + }))? + ); + continue; } - }; - if let Message::Ping(payload) = message { - writer.send(Message::Pong(payload)).await?; - continue; - } - if !message.is_text() { - continue; - } - let raw_text = message.to_text()?.to_owned(); - let payload: Value = serde_json::from_str(&raw_text)?; - if payload.get("event").and_then(Value::as_str) == Some("notice") { - liveness.disconnected(&session_id, generation, now_ns()?)?; - disconnected = true; - break; - } - if payload.get("event").is_some() { - return Err("unexpected OKX control event after subscription".into()); - } - let frame = pending_okx_frame( - &binding_map, - &session_id, - generation, - raw_text, - now_ns()?, - )?; - if !publish_pending_frame( - frame, - &mut PendingPublishWindow { - publisher: &publisher, - latest: &mut latest, - inflight: &mut inflight, - accepted: &accepted, - coalesced_latest: &coalesced_latest, - max_events: config.max_events, - max_inflight: config.max_inflight_publishes, - stopped: &stopped, - }, - ) - .await? - { - break; } } - if publish_error.is_none() { - if let Err(error) = flush_latest_concurrent( + _ = latest_tick.tick(), if !latest.is_empty() => { + if !flush_latest_concurrent( &mut latest, &publisher, &mut inflight, @@ -2527,59 +2828,235 @@ async fn run_okx_service( config.max_events, config.max_inflight_publishes, &stopped, - ) - .await - { - publish_error = Some(error); + ).await? { + break; } + failures = 0; + continue; } - while let Some(result) = inflight.next().await { - match result { - Ok(()) => failures = 0, - Err(error) if publish_error.is_none() => publish_error = Some(error), - Err(_) => {} + result = &mut read => result, + completed = inflight.next(), if !inflight.is_empty() => { + match completed { + Some(Ok(())) => { + failures = 0; + continue; + } + Some(Err(error)) => { + publish_error = Some(error); + break; + } + None => { + return Err("lossless publish window became inconsistent".into()) + } } } - liveness.disconnected(&session_id, generation, now_ns()?)?; - if let Some(error) = publish_error { - return Err(error.into()); + }; + let message = match outcome { + Ok(Some(Ok(message))) => { + if let Message::Close(frame) = &message { + ended = Some(OkxSessionEnd::Closed { + code: frame.as_ref().map(|value| u16::from(value.code)), + reason: frame + .as_ref() + .map(|value| value.reason.to_string()) + .unwrap_or_default(), + }); + break; + } + let read_at_ns = now_ns()?; + last_frame_at_ns = read_at_ns; + ping_outstanding = false; + liveness.live(&session_id, generation, read_at_ns)?; + message } - if disconnected && !should_stop(&stopped, &accepted, config.max_events, expires) { - failures = failures.saturating_add(1); - eprintln!( - "{}", - serde_json::to_string(&json!({ - "event": "qdl_native_session_disconnected", - "runtime": "OKX", - "service": format!("{:?}", service).to_ascii_uppercase(), - "attempt": failures, - "generation": generation, - }))? - ); - tokio::time::sleep(Duration::from_millis( - backoff.delay_ms(failures, failures.min(10_000) as u16), - )) - .await; + Ok(Some(Err(error))) => { + ended = Some(OkxSessionEnd::ReadError(error.to_string())); + break; + } + Ok(None) => { + ended = Some(OkxSessionEnd::Eof); + break; + } + Err(_) => match okx_keepalive_on_idle(ping_outstanding) { + OkxKeepalive::Dead => { + ended = Some(OkxSessionEnd::HeartbeatTimeout); + break; + } + OkxKeepalive::SendPing => { + if let Err(error) = writer.send(Message::Text("ping".into())).await { + ended = Some(OkxSessionEnd::WriteError(error.to_string())); + break; + } + ping_outstanding = true; + continue; + } + }, + }; + if let Message::Ping(payload) = message { + if let Err(error) = writer.send(Message::Pong(payload)).await { + ended = Some(OkxSessionEnd::WriteError(error.to_string())); + break; } + continue; } - Err(error) => { - failures = failures.saturating_add(1); + if !message.is_text() { + continue; + } + let raw_text = message.to_text()?.to_owned(); + if raw_text == "pong" { + continue; + } + let payload: Value = serde_json::from_str(&raw_text)?; + if let Some(code) = okx_notice_code(&payload) { + // Keep publishing from this connection until the successor is + // subscribed; OKX keeps it open for 60 seconds after the notice. + // A repeated notice retries a standby that failed earlier. + if standby.is_none() { + let next_generation = next_connection_generation(Path::new(&generation_path))?; + budget.permit(now_ns()?)?; + standby = Some(Box::pin(establish(next_generation))); + } eprintln!( "{}", serde_json::to_string(&json!({ - "event": "qdl_native_connect_failed", + "event": "qdl_native_session_notice", "runtime": "OKX", - "service": format!("{:?}", service).to_ascii_uppercase(), - "attempt": failures, - "error": error.to_string(), + "lane": lane, + "generation": generation, + "code": code, }))? ); - tokio::time::sleep(Duration::from_millis( - backoff.delay_ms(failures, failures.min(10_000) as u16), - )) - .await; + notice = Some(code); + continue; + } + if payload.get("event").is_some() { + return Err("unexpected OKX control event after subscription".into()); + } + let frame = pending_okx_frame( + &binding_map, + &session_id, + generation, + raw_text, + last_frame_at_ns, + )?; + if !publish_pending_frame( + frame, + &mut PendingPublishWindow { + publisher: &publisher, + latest: &mut latest, + inflight: &mut inflight, + accepted: &accepted, + coalesced_latest: &coalesced_latest, + max_events: config.max_events, + max_inflight: config.max_inflight_publishes, + stopped: &stopped, + }, + ) + .await? + { + break; + } + } + // A notice whose standby never became ready is still the cause. + if let (Some(code), Some(end)) = (notice.as_ref(), ended.as_ref()) { + if !end.handed_over() { + eprintln!( + "{}", + serde_json::to_string(&json!({ + "event": "qdl_native_notice_without_handover", + "runtime": "OKX", + "lane": lane, + "generation": generation, + "code": code, + "cause": end.cause(), + }))? + ); + if !matches!(end, OkxSessionEnd::BookResync) { + ended = Some(OkxSessionEnd::Notice { code: code.clone() }); + } + } + } + // An unfinished standby is dropped here; its generation number stays + // consumed (generations only need to increase). + drop(standby); + let detected_at_ns = now_ns()?; + let handed_over = ended.as_ref().is_some_and(OkxSessionEnd::handed_over); + if ended.is_some() && !handed_over { + // Not covered by a successor: mark this session down at once, + // before the drain below, so readers stop trusting it. + liveness.disconnected(&session_id, generation, detected_at_ns)?; + } + if publish_error.is_none() { + if let Err(error) = flush_latest_concurrent( + &mut latest, + &publisher, + &mut inflight, + &accepted, + config.max_events, + config.max_inflight_publishes, + &stopped, + ) + .await + { + publish_error = Some(error); + } + } + while let Some(result) = inflight.next().await { + match result { + Ok(()) => failures = 0, + Err(error) if publish_error.is_none() => publish_error = Some(error), + Err(_) => {} } } + let stopping = should_stop(&stopped, &accepted, config.max_events, expires); + // A handed-over session stays LIVE until its successor's record + // replaces it, unless that successor will not be used after all. + if !handed_over || stopping || publish_error.is_some() { + liveness.disconnected(&session_id, generation, now_ns()?)?; + } + if let Some(error) = publish_error { + return Err(error.into()); + } + let Some(end) = ended else { + // Stopped, or the event budget is spent. + continue; + }; + if stopping { + continue; + } + failures = failures.saturating_add(1); + let drained_at_ns = now_ns()?; + eprintln!( + "{}", + serde_json::to_string(&json!({ + "event": "qdl_native_session_disconnected", + "runtime": "OKX", + "service": format!("{:?}", service).to_ascii_uppercase(), + "lane": lane, + "attempt": failures, + "generation": generation, + "session": session_id, + "cause": end.cause(), + "detail": end.detail(), + "last_frame_to_detected_ms": (detected_at_ns - last_frame_at_ns) / 1_000_000, + "drain_ms": (drained_at_ns - detected_at_ns) / 1_000_000, + }))? + ); + previous_end = Some(( + session_id.clone(), + generation, + last_frame_at_ns, + detected_at_ns, + )); + if handed_over { + // The successor is live already; nothing to wait for. + failures = 0; + continue; + } + let delay = okx_reconnect_delay_ms(backoff, failures); + if delay > 0 { + tokio::time::sleep(Duration::from_millis(delay)).await; + } } Ok(()) } @@ -2620,6 +3097,7 @@ async fn run_okx( .business_websocket_url .clone() .ok_or("OKX business WebSocket URL is required")?; + let pacer = OkxConnectPacer::default(); let mut futures = Vec::new(); for (index, bindings) in partition_okx_bindings(&public, config.max_subscriptions_per_connection) @@ -2637,6 +3115,7 @@ async fn run_okx( accepted.clone(), coalesced_latest.clone(), stopped.clone(), + pacer.clone(), )); } for (index, bindings) in @@ -2655,6 +3134,7 @@ async fn run_okx( accepted.clone(), coalesced_latest.clone(), stopped.clone(), + pacer.clone(), )); } try_join_all(futures).await?; @@ -3825,3 +4305,324 @@ mod resync_tests { .is_err()); } } + +#[cfg(test)] +mod okx_reconnect_tests { + use super::{ + okx_establish, okx_keepalive_on_idle, okx_notice_code, okx_reconnect_delay_ms, + BackoffPolicy, DeliveryClass, OkxConnectPacer, OkxEstablishError, OkxKeepalive, + OkxSessionEnd, OkxSubscription, RawBinding, RawFeed, SessionLineageWriter, + OKX_CONNECT_SPACING, + }; + use futures_util::{SinkExt, StreamExt}; + use serde_json::{json, Value}; + use std::collections::HashMap; + use std::path::PathBuf; + use std::sync::Arc; + use std::time::Duration; + use tokio::net::TcpListener; + use tokio::time::Instant; + use tokio_tungstenite::tungstenite::Message; + + fn okx_trade_binding() -> RawBinding { + RawBinding { + provider: "OKX_DIRECT".into(), + venue: "OKX".into(), + market: "SWAP".into(), + product_type: "PERPETUAL".into(), + native_symbol: "BTC-USDT-SWAP".into(), + native_channel: "trades".into(), + subscription_id: "test-source".into(), + adapter_version: "test/2.0.0".into(), + instrument_catalog_revision: 1, + feed: RawFeed::Trade, + delivery_class: DeliveryClass::Lossless, + l2: None, + mark_index_target: None, + } + } + + fn backoff() -> BackoffPolicy { + BackoffPolicy { + initial_ms: 250, + maximum_ms: 30_000, + multiplier: 2, + jitter_bps: 2_000, + } + .validate() + .unwrap() + } + + fn scratch_dir(label: &str) -> PathBuf { + let path = std::env::temp_dir().join(format!( + "qdl-okx-lineage-{label}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&path).unwrap(); + path + } + + #[test] + fn a_session_that_delivered_reconnects_without_backoff() { + // Production evidence 2026-10-10: every OKX reconnect took ~630 ms + // from the disconnect log to the new session id, 400 ms of it this + // fixed sleep (delay_ms(1, 1) of the 250 ms policy, jitter basis 1). + assert_eq!(backoff().delay_ms(1, 1), 400); + assert_eq!(okx_reconnect_delay_ms(backoff(), 1), 0); + // A session that ended again before delivering anything still backs off. + assert!(okx_reconnect_delay_ms(backoff(), 2) >= 800); + assert!((24_000..=30_000).contains(&okx_reconnect_delay_ms(backoff(), 30))); + } + + #[test] + fn idle_read_pings_once_then_declares_the_connection_dead() { + assert_eq!(okx_keepalive_on_idle(false), OkxKeepalive::SendPing); + assert_eq!(okx_keepalive_on_idle(true), OkxKeepalive::Dead); + } + + #[test] + fn only_the_okx_notice_event_is_a_service_upgrade_notice() { + let notice = json!({ + "event": "notice", + "code": "64008", + "msg": "The connection will soon be closed for a service upgrade. Please reconnect.", + "connId": "a4d3ae55", + }); + assert_eq!(okx_notice_code(¬ice).as_deref(), Some("64008")); + assert_eq!( + okx_notice_code(&json!({"event": "notice"})).as_deref(), + Some("missing") + ); + assert_eq!( + okx_notice_code(&json!({"event": "error", "code": "60012"})), + None + ); + assert_eq!( + okx_notice_code(&json!({"arg": {"channel": "trades"}, "data": []})), + None + ); + } + + #[test] + fn only_a_notice_handover_counts_as_covered() { + let covered = OkxSessionEnd::NoticeHandover { + code: "64008".into(), + }; + assert!(covered.handed_over()); + assert_eq!(covered.cause(), "NOTICE_HANDOVER"); + for end in [ + OkxSessionEnd::Notice { + code: "64008".into(), + }, + OkxSessionEnd::Closed { + code: Some(1001), + reason: "going away".into(), + }, + OkxSessionEnd::Eof, + OkxSessionEnd::ReadError("reset".into()), + OkxSessionEnd::WriteError("broken pipe".into()), + OkxSessionEnd::HeartbeatTimeout, + OkxSessionEnd::BookResync, + ] { + assert!(!end.handed_over(), "{end:?}"); + } + assert_eq!( + OkxSessionEnd::Closed { + code: Some(1001), + reason: "going away".into() + } + .detail(), + json!({"code": 1001, "reason": "going away"}) + ); + } + + #[test] + fn lineage_chain_grows_only_on_overlap_and_resets_after_a_gap() { + let dir = scratch_dir("chain"); + let mut writer = SessionLineageWriter::at(&dir, "okx-public-002", 20).unwrap(); + let path = dir.join("lineage").join("okx-public-002.lineage"); + let read = || -> Value { serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap() }; + let chain = |value: &Value| -> Vec<(String, u64)> { + value["chain"] + .as_array() + .unwrap() + .iter() + .map(|link| { + ( + link["source_session_id"].as_str().unwrap().to_owned(), + link["connection_generation"].as_u64().unwrap(), + ) + }) + .collect() + }; + + writer.begin("okx-public-002-167-1", 167, false).unwrap(); + writer.begin("okx-public-002-168-2", 168, true).unwrap(); + let value = read(); + assert_eq!(value["schema"], "qdl.provider-session-lineage.v1"); + assert_eq!(value["lane"], "okx-public-002"); + assert_eq!(value["config_revision"], 20); + assert_eq!( + chain(&value), + vec![ + ("okx-public-002-167-1".into(), 167), + ("okx-public-002-168-2".into(), 168) + ] + ); + + // A reconnect after a gap must not vouch for any earlier session. + writer.begin("okx-public-002-169-3", 169, false).unwrap(); + assert_eq!(chain(&read()), vec![("okx-public-002-169-3".into(), 169)]); + + for generation in 170..250_u64 { + writer + .begin(&format!("okx-public-002-{generation}-9"), generation, true) + .unwrap(); + } + let links = chain(&read()); + assert_eq!(links.len(), 32); + assert_eq!(links.last().unwrap().1, 249); + // The healthcheck matches '*.json' recursively; the lineage file must + // never be mistaken for a session record. + assert!(std::fs::read_dir(dir.join("lineage")) + .unwrap() + .all(|entry| !entry + .unwrap() + .file_name() + .to_string_lossy() + .ends_with(".json"))); + assert!(SessionLineageWriter::at(&dir, "OKX_PUBLIC", 20).is_err()); + assert!(writer.begin("", 1, true).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[tokio::test(start_paused = true)] + async fn connect_pacer_keeps_okx_connection_requests_under_three_per_second() { + let pacer = OkxConnectPacer::default(); + let start = Instant::now(); + for _ in 0..4 { + pacer.admit().await; + } + // Four admissions need three full spacings: fewer than 3 per second. + assert_eq!(start.elapsed(), OKX_CONNECT_SPACING * 3); + assert!(OKX_CONNECT_SPACING * 3 > Duration::from_secs(1)); + } + + /// One fake OKX endpoint: sends `script` before (optionally) ACKing. + async fn fake_okx(script: Vec, ack: bool) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap(); + let command = socket.next().await.unwrap().unwrap(); + let command: Value = serde_json::from_str(command.to_text().unwrap()).unwrap(); + assert_eq!(command["op"], "subscribe"); + for message in script { + socket + .send(Message::Text(message.to_string())) + .await + .unwrap(); + } + if ack { + for arg in command["args"].as_array().unwrap() { + socket + .send(Message::Text( + json!({"event": "subscribe", "arg": arg, "id": command["id"]}) + .to_string(), + )) + .await + .unwrap(); + } + // Hold the connection open, like a live venue. + while socket.next().await.is_some() {} + } else { + let _ = socket.close(None).await; + } + }); + format!("ws://{address}/ws/v5/public") + } + + fn subscriptions() -> (Arc>, Arc>) { + let binding = okx_trade_binding(); + ( + Arc::new(vec![OkxSubscription { + channel: binding.native_channel.clone(), + inst_id: binding.native_symbol.clone(), + }]), + Arc::new(HashMap::from([(binding.key(), binding)])), + ) + } + + #[tokio::test] + async fn establish_buffers_pre_ack_frames_under_the_new_generation_and_reports_a_notice() { + let trade = json!({ + "arg": {"channel": "trades", "instId": "BTC-USDT-SWAP"}, + "data": [{"instId": "BTC-USDT-SWAP", "tradeId": "42", "px": "1", "sz": "1", "side": "buy", "ts": "1"}], + }); + let notice = json!({"event": "notice", "code": "64008", "msg": "upgrade"}); + let url = fake_okx(vec![trade, notice], true).await; + let (subscriptions, bindings) = subscriptions(); + let established = okx_establish( + url, + subscriptions, + bindings, + "okx-public-002".into(), + 168, + 16, + OkxConnectPacer::default(), + ) + .await + .unwrap(); + assert_eq!(established.generation, 168); + assert!(established.session_id.starts_with("okx-public-002-168-")); + assert!(established.ready_at_ns >= established.connect_started_ns); + assert_eq!(established.notice_code.as_deref(), Some("64008")); + // Nothing is published here: the caller publishes these only after the + // previous generation stopped, so the generation fences see N before N+1. + assert_eq!(established.pre_ack_frames.len(), 1); + let frame = &established.pre_ack_frames[0]; + assert_eq!(frame.generation, 168); + assert_eq!(frame.session_id, established.session_id); + } + + #[tokio::test] + async fn establish_failures_are_typed_so_a_standby_failure_spares_the_live_session() { + let url = fake_okx(vec![], false).await; + let (subscriptions, bindings) = subscriptions(); + let error = okx_establish( + url, + subscriptions.clone(), + bindings.clone(), + "okx-public-002".into(), + 169, + 16, + OkxConnectPacer::default(), + ) + .await + .err() + .unwrap(); + assert!(matches!(error, OkxEstablishError::Subscribe(_)), "{error}"); + + let unused = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = unused.local_addr().unwrap(); + drop(unused); + let error = okx_establish( + format!("ws://{address}/ws/v5/public"), + subscriptions, + bindings, + "okx-public-002".into(), + 170, + 16, + OkxConnectPacer::default(), + ) + .await + .err() + .unwrap(); + assert!(matches!(error, OkxEstablishError::Connect(_)), "{error}"); + } +} diff --git a/tests/test_session_liveness_lineage.py b/tests/test_session_liveness_lineage.py new file mode 100644 index 00000000..68d067f5 --- /dev/null +++ b/tests/test_session_liveness_lineage.py @@ -0,0 +1,156 @@ +"""Session lineage: an event from a session that handed over without a gap. + +Stage 100e (2026-10-10) refused OKX BNB-USDT-SWAP TRADE with DATA_STALE / +SOURCE_SESSION_UNKNOWN although its event recency was LIVE: the lane's +liveness file had been replaced by the next session's record, so the last +trade, published by generation 167, matched no record. When the OKX ingestor +replaces a session make-before-break (service-upgrade notice 64008), it writes +``lineage/.lineage`` with the chain of sessions that overlapped. These +tests pin that the reader then judges such an event by the chain's current +session, and that every doubtful case still fails closed: a chain reset by a +gap, a successor that is down, ambiguous chains, a config mismatch and a +malformed lineage file. Readers that predate lineage never list the +subdirectory, so the rollout order of reader and writer does not matter. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest + +from qdl.runtime.session_liveness import ( + SESSION_CONTINUED_FLAG, + SESSION_LINEAGE_SCHEMA, + SESSION_LIVENESS_SCHEMA, + StableSessionLivenessReader, +) + +OLD = ("okx-public-002-167-1791612000000000000", 167) +NEW = ("okx-public-002-168-1791612713096478415", 168) +NOW = 1_791_612_714_551_564_264 + + +class SessionLineageTests(unittest.TestCase): + def setUp(self) -> None: + self._dir = tempfile.TemporaryDirectory() + self.directory = Path(self._dir.name) / "okx-swap" + self.directory.mkdir() + self.reader = StableSessionLivenessReader(Path(self._dir.name)) + + def tearDown(self) -> None: + self._dir.cleanup() + + def record(self, session: tuple[str, int], *, state: str = "LIVE", + revision: int = 20, lane: str = "okx-public-002") -> None: + (self.directory / f"{lane}.json").write_text(json.dumps({ + "schema": SESSION_LIVENESS_SCHEMA, "source_session_id": session[0], + "connection_generation": session[1], "state": state, + "last_transport_at_ns": NOW - 3_000_000, "updated_at_ns": NOW - 3_000_000, + "config_revision": revision}), encoding="utf-8") + + def lineage(self, *chain: tuple[str, int], lane: str = "okx-public-002", + revision: int = 20, body: object | None = None) -> None: + directory = self.directory / "lineage" + directory.mkdir(exist_ok=True) + payload = body if body is not None else { + "schema": SESSION_LINEAGE_SCHEMA, "lane": lane, + "chain": [{"source_session_id": session, "connection_generation": generation} + for session, generation in chain], + "updated_at_ns": NOW - 3_000_000, "config_revision": revision} + (directory / f"{lane}.lineage").write_text(json.dumps(payload), encoding="utf-8") + + def status(self, session: tuple[str, int] = OLD, revision: int = 20): + return self.reader.status(venue="OKX", market="SWAP", source_session_id=session[0], + connection_generation=session[1], + config_revision=revision, now_ns=NOW) + + def test_without_lineage_an_event_of_a_replaced_session_is_unknown(self) -> None: + # The stage 100e refusal, reproduced. + self.record(NEW) + result = self.status() + self.assertEqual((result.state, result.flags), ("UNKNOWN", ("SOURCE_SESSION_UNAVAILABLE",))) + + def test_an_overlapped_predecessor_is_judged_by_its_live_successor(self) -> None: + self.record(NEW) + self.lineage(OLD, NEW) + result = self.status() + self.assertEqual((result.state, result.liveness_ms, result.flags), + ("LIVE", 3, (SESSION_CONTINUED_FLAG,))) + # The current session itself still matches directly, with no flag. + self.assertEqual(self.status(NEW).flags, ()) + + def test_a_chain_reset_by_a_gap_never_vouches_for_the_old_session(self) -> None: + self.record(NEW) + self.lineage(NEW) + self.assertEqual(self.status().state, "UNKNOWN") + + def test_a_successor_that_is_down_is_not_live_evidence(self) -> None: + self.record(NEW, state="DISCONNECTED") + self.lineage(OLD, NEW) + result = self.status() + self.assertEqual((result.state, result.flags), ("DISCONNECTED", (SESSION_CONTINUED_FLAG,))) + + def test_a_chain_whose_last_link_is_not_the_current_record_is_ignored(self) -> None: + # The lineage was written for 168, but the record already names 169. + self.record(("okx-public-002-169-1", 169)) + self.lineage(OLD, NEW) + self.assertEqual(self.status().state, "UNKNOWN") + + def test_config_revision_must_match_on_lineage_successor_and_event(self) -> None: + self.record(NEW) + self.lineage(OLD, NEW, revision=21) + self.assertEqual(self.status().state, "UNKNOWN") + self.lineage(OLD, NEW) + result = self.status(revision=19) + self.assertEqual((result.state, result.flags), ("UNKNOWN", ("SOURCE_SESSION_CONFIG_MISMATCH",))) + + def test_two_chains_claiming_one_session_fail_closed(self) -> None: + other = ("okx-public-003-175-1", 175) + self.record(NEW) + self.record(other, lane="okx-public-003") + self.lineage(OLD, NEW) + self.lineage(OLD, other, lane="okx-public-003") + self.assertEqual(self.status().state, "UNKNOWN") + + def test_malformed_lineage_fails_closed(self) -> None: + self.record(NEW) + for body in ( + {"schema": SESSION_LINEAGE_SCHEMA, "lane": "okx-public-002", "chain": [], + "updated_at_ns": 1, "config_revision": 20}, + {"schema": "qdl.provider-session-lineage.v0", "lane": "x", + "chain": [{"source_session_id": OLD[0], "connection_generation": OLD[1]}], + "updated_at_ns": 1, "config_revision": 20}, + # Generations must increase along the chain. + {"schema": SESSION_LINEAGE_SCHEMA, "lane": "okx-public-002", + "chain": [{"source_session_id": NEW[0], "connection_generation": NEW[1]}, + {"source_session_id": OLD[0], "connection_generation": OLD[1]}], + "updated_at_ns": 1, "config_revision": 20}, + {"schema": SESSION_LINEAGE_SCHEMA, "lane": "okx-public-002", + "chain": [{"source_session_id": OLD[0], "connection_generation": True}, + {"source_session_id": NEW[0], "connection_generation": NEW[1]}], + "updated_at_ns": 1, "config_revision": 20, "extra": 1}, + ): + self.lineage(body=body) + self.assertEqual(self.status().state, "UNKNOWN", body) + + def test_lineage_change_is_seen_on_the_next_call(self) -> None: + self.record(NEW) + self.lineage(OLD, NEW) + self.assertEqual(self.status().state, "LIVE") + self.lineage(NEW) # reset after a gap: same name, new content + self.assertEqual(self.status().state, "UNKNOWN") + + def test_the_lineage_directory_is_not_a_session_record(self) -> None: + # Only the subdirectory exists next to the record; the direct lookup is + # unchanged and the healthcheck-visible '*.json' set is untouched. + self.record(NEW) + self.lineage(OLD, NEW) + self.assertEqual(sorted(path.name for path in self.directory.glob("**/*.json")), + ["okx-public-002.json"]) + self.assertEqual(self.status(NEW).state, "LIVE") + + +if __name__ == "__main__": + unittest.main() From d4f5ec98202ed650252b83c89ac2b92631067c7c Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 07:03:26 +0000 Subject: [PATCH 101/147] fix(ingestor): absolute inbound deadline bounds silent-socket detection (Astra R4) Both read loops built a fresh relative timeout(heartbeat, socket.next()) on every select iteration. Any other branch that won and continued (Binance book-liveness proof tick at 250..5000 ms, latest-state flush, resync listener, snapshot renewal, publish completion) discarded it, so a socket that sent neither data nor pong was never declared dead while those kept firing; the proof's own ping succeeds on a half-open socket. InboundDeadline is absolute and moves only on a message actually read (data, ping, pong, OKX text pong). Idle for heartbeat -> one ping; nothing by 2 x heartbeat after the last inbound message -> dead (Binance qdl_native_session_heartbeat_failed, OKX cause HEARTBEAT_TIMEOUT). Proof ticks, pings sent and publish completions never move it. The Binance pong now arrives through the same select instead of a nested 15 s wait that stalled every other branch. BookLivenessProof is still stamped only from received transport. Tests (ingestor bin 39): deadline state machine; a silent fake venue (no frames, no pongs) is declared dead at exactly 30 s with the proof tick on and off; the pre-R4 relative-timeout shape is still "alive" after 150 s with the proof ticking (reproduces R4); a venue that answers pings stays alive with the proof on and off. --- .../src/bin/qdl-native-raw-ingestor.rs | 301 ++++++++++++++---- 1 file changed, 231 insertions(+), 70 deletions(-) diff --git a/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs b/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs index d2256453..0fbb3618 100644 --- a/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs +++ b/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs @@ -1865,6 +1865,10 @@ async fn run_binance_connection( tick.tick().await; } let mut ping_due = false; + let mut inbound = InboundDeadline::new( + Duration::from_secs(config.heartbeat_seconds), + Instant::now(), + ); let mut inflight = FuturesUnordered::::new(); let mut latest = LatestStateBuffer::default(); let mut latest_tick = @@ -1967,10 +1971,9 @@ async fn run_binance_connection( ping_due = false; socket.send(Message::Ping(Vec::new())).await?; } - let read = tokio::time::timeout( - Duration::from_secs(config.heartbeat_seconds), - socket.next(), - ); + // Absolute deadline: the proof tick or any other branch + // winning never extends it (R4). + let read = tokio::time::timeout_at(inbound.deadline(), socket.next()); tokio::pin!(read); // Control backlog must not take priority over socket and publish progress. let outcome = tokio::select! { @@ -2098,6 +2101,7 @@ async fn run_binance_connection( Ok(Some(Ok(message))) => { if !matches!(message, Message::Close(_)) { let read_at_ns = now_ns()?; + inbound.inbound(Instant::now()); liveness.live(&session_id, generation, read_at_ns)?; if let Some(proof) = book_proof.as_mut() { proof.transport(read_at_ns); @@ -2137,49 +2141,30 @@ async fn run_binance_connection( disconnected = true; break; } - Err(_) => { - socket.send(Message::Ping(Vec::new())).await?; - match tokio::time::timeout( - Duration::from_secs(config.heartbeat_seconds), - socket.next(), - ) - .await - { - Ok(Some(Ok(Message::Pong(_)))) => { - let read_at_ns = now_ns()?; - liveness.live(&session_id, generation, read_at_ns)?; - if let Some(proof) = book_proof.as_mut() { - proof.transport(read_at_ns); - } - continue; - } - Ok(Some(Ok(message))) => { - if !matches!(message, Message::Close(_)) { - let read_at_ns = now_ns()?; - liveness.live(&session_id, generation, read_at_ns)?; - if let Some(proof) = book_proof.as_mut() { - proof.transport(read_at_ns); - } - } - message - } - _ => { - failures = failures.saturating_add(1); - eprintln!( - "{}", - serde_json::to_string(&json!({ - "event": "qdl_native_session_heartbeat_failed", - "runtime": "BINANCE", - "attempt": failures, - "generation": generation, - }))? - ); - liveness.disconnected(&session_id, generation, now_ns()?)?; - disconnected = true; - break; - } + // The pong (or any frame) arrives through the same + // select, so other branches keep running meanwhile; the + // ping itself never moves the deadline. + Err(_) => match inbound.expired() { + Keepalive::SendPing => { + socket.send(Message::Ping(Vec::new())).await?; + continue; } - } + Keepalive::Dead => { + failures = failures.saturating_add(1); + eprintln!( + "{}", + serde_json::to_string(&json!({ + "event": "qdl_native_session_heartbeat_failed", + "runtime": "BINANCE", + "attempt": failures, + "generation": generation, + }))? + ); + liveness.disconnected(&session_id, generation, now_ns()?)?; + disconnected = true; + break; + } + }, }; if let Message::Ping(payload) = message { socket.send(Message::Pong(payload)).await?; @@ -2437,20 +2422,61 @@ fn okx_reconnect_delay_ms(backoff: BackoffPolicy, failures: u32) -> u64 { } } -/// Idle-read outcome of the OKX keepalive: OKX closes a connection that has -/// pushed nothing for 30 seconds; a client sends the text `ping` after N < 30 -/// idle seconds and reconnects when no `pong` follows within N seconds. #[derive(Debug, Eq, PartialEq)] -enum OkxKeepalive { +enum Keepalive { SendPing, Dead, } -fn okx_keepalive_on_idle(ping_outstanding: bool) -> OkxKeepalive { - if ping_outstanding { - OkxKeepalive::Dead - } else { - OkxKeepalive::SendPing +/// Hard bound on detecting a silent socket (Astra R4). +/// +/// The read loops also select on timers (book-liveness proof, latest-state +/// flush, resync listener, snapshot renewal, publish completion). A read +/// timeout created afresh on every iteration is discarded whenever one of +/// those wins, so a socket that sends neither data nor pong was never +/// declared dead while any of them kept firing. This deadline is absolute: it +/// moves only on a message actually read from the socket (data, ping, pong or +/// OKX text `pong`). Sending a ping, a timer tick or a publish completion never +/// moves it. Idle for `idle` -> one ping; still nothing by `2 * idle` after the +/// last inbound message -> dead. That matches OKX's keepalive rule (send +/// `ping` after N < 30 idle seconds, reconnect if no `pong` within N). +#[derive(Debug)] +struct InboundDeadline { + idle: Duration, + last_inbound: Instant, + ping_sent: bool, +} + +impl InboundDeadline { + fn new(idle: Duration, now: Instant) -> Self { + Self { + idle, + last_inbound: now, + ping_sent: false, + } + } + + fn inbound(&mut self, now: Instant) { + self.last_inbound = now; + self.ping_sent = false; + } + + fn deadline(&self) -> Instant { + if self.ping_sent { + self.last_inbound + self.idle * 2 + } else { + self.last_inbound + self.idle + } + } + + /// Called when `deadline()` has passed with nothing read. + fn expired(&mut self) -> Keepalive { + if self.ping_sent { + Keepalive::Dead + } else { + self.ping_sent = true; + Keepalive::SendPing + } } } @@ -2730,7 +2756,10 @@ async fn run_okx_service( let mut publish_error = None; let mut exhausted = false; let mut last_frame_at_ns = ready_at_ns; - let mut ping_outstanding = false; + let mut inbound = InboundDeadline::new( + Duration::from_secs(config.heartbeat_seconds), + Instant::now(), + ); let mut standby: Option = None; let mut notice: Option = None; if let Some(code) = early_notice { @@ -2773,8 +2802,8 @@ async fn run_okx_service( None => return Err("lossless publish window became inconsistent".into()), } } - let read = - tokio::time::timeout(Duration::from_secs(config.heartbeat_seconds), reader.next()); + // Absolute deadline: other branches winning never extend it (R4). + let read = tokio::time::timeout_at(inbound.deadline(), reader.next()); tokio::pin!(read); // Control backlog must not take priority over socket and publish progress. let outcome = tokio::select! { @@ -2865,7 +2894,7 @@ async fn run_okx_service( } let read_at_ns = now_ns()?; last_frame_at_ns = read_at_ns; - ping_outstanding = false; + inbound.inbound(Instant::now()); liveness.live(&session_id, generation, read_at_ns)?; message } @@ -2877,17 +2906,16 @@ async fn run_okx_service( ended = Some(OkxSessionEnd::Eof); break; } - Err(_) => match okx_keepalive_on_idle(ping_outstanding) { - OkxKeepalive::Dead => { + Err(_) => match inbound.expired() { + Keepalive::Dead => { ended = Some(OkxSessionEnd::HeartbeatTimeout); break; } - OkxKeepalive::SendPing => { + Keepalive::SendPing => { if let Err(error) = writer.send(Message::Text("ping".into())).await { ended = Some(OkxSessionEnd::WriteError(error.to_string())); break; } - ping_outstanding = true; continue; } }, @@ -4309,10 +4337,9 @@ mod resync_tests { #[cfg(test)] mod okx_reconnect_tests { use super::{ - okx_establish, okx_keepalive_on_idle, okx_notice_code, okx_reconnect_delay_ms, - BackoffPolicy, DeliveryClass, OkxConnectPacer, OkxEstablishError, OkxKeepalive, - OkxSessionEnd, OkxSubscription, RawBinding, RawFeed, SessionLineageWriter, - OKX_CONNECT_SPACING, + okx_establish, okx_notice_code, okx_reconnect_delay_ms, BackoffPolicy, DeliveryClass, + InboundDeadline, Keepalive, OkxConnectPacer, OkxEstablishError, OkxSessionEnd, + OkxSubscription, RawBinding, RawFeed, SessionLineageWriter, OKX_CONNECT_SPACING, }; use futures_util::{SinkExt, StreamExt}; use serde_json::{json, Value}; @@ -4379,9 +4406,143 @@ mod okx_reconnect_tests { } #[test] - fn idle_read_pings_once_then_declares_the_connection_dead() { - assert_eq!(okx_keepalive_on_idle(false), OkxKeepalive::SendPing); - assert_eq!(okx_keepalive_on_idle(true), OkxKeepalive::Dead); + fn inbound_deadline_pings_once_then_declares_the_connection_dead() { + let idle = Duration::from_secs(15); + let start = Instant::now(); + let mut deadline = InboundDeadline::new(idle, start); + assert_eq!(deadline.deadline(), start + idle); + assert_eq!(deadline.expired(), Keepalive::SendPing); + // Sending the ping opens exactly one pong window; nothing renews it. + assert_eq!(deadline.deadline(), start + idle * 2); + assert_eq!(deadline.expired(), Keepalive::Dead); + assert_eq!(deadline.deadline(), start + idle * 2); + // Only a message read from the socket moves it. + let later = start + Duration::from_secs(20); + deadline.inbound(later); + assert_eq!(deadline.deadline(), later + idle); + assert_eq!(deadline.expired(), Keepalive::SendPing); + } + + /// A venue socket that stays open and sends nothing: no frames, no pongs + /// (it never reads, so tungstenite never answers our pings). With + /// `answer_pings`, it reads and auto-answers every ping instead. + async fn silent_venue(answer_pings: bool) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap(); + if answer_pings { + while socket.next().await.is_some() {} + } else { + std::future::pending::<()>().await; + } + drop(socket); + }); + format!("ws://{address}/ws") + } + + /// The production read-loop shape: a book-liveness proof tick (flag on) + /// competes with the socket read in one select and `continue`s, sending a + /// proof ping like `BookLivenessProof::due` asks for. Returns when the + /// connection is declared dead, or None if it is still considered alive + /// at `give_up`. + async fn detect_dead( + url: String, + proof_tick: Option, + absolute_deadline: bool, + give_up: Duration, + ) -> Option { + let idle = Duration::from_secs(15); + let (mut socket, _) = tokio_tungstenite::connect_async(&url).await.unwrap(); + let start = Instant::now(); + let mut inbound = InboundDeadline::new(idle, start); + let mut tick = proof_tick.map(tokio::time::interval); + if let Some(tick) = tick.as_mut() { + tick.tick().await; + } + while start.elapsed() < give_up { + let read = if absolute_deadline { + tokio::time::timeout_at(inbound.deadline(), socket.next()) + } else { + // The pre-R4 shape: a fresh relative timeout every iteration. + tokio::time::timeout_at(Instant::now() + idle, socket.next()) + }; + tokio::pin!(read); + tokio::select! { + _ = async { + match tick.as_mut() { + Some(tick) => { tick.tick().await; } + None => std::future::pending::<()>().await, + } + }, if tick.is_some() => { + socket.send(Message::Ping(Vec::new())).await.unwrap(); + continue; + } + result = &mut read => match result { + Ok(Some(Ok(_))) => inbound.inbound(Instant::now()), + Ok(_) => panic!("the fake venue never closes"), + Err(_) => match inbound.expired() { + Keepalive::SendPing => { + socket.send(Message::Ping(Vec::new())).await.unwrap(); + } + Keepalive::Dead => return Some(start.elapsed()), + }, + }, + } + } + None + } + + #[tokio::test(start_paused = true)] + async fn a_silent_socket_is_declared_dead_within_two_idle_periods_with_proof_on_and_off() { + let bound = Duration::from_secs(30); + for proof_tick in [None, Some(Duration::from_millis(250))] { + let detected = detect_dead( + silent_venue(false).await, + proof_tick, + true, + Duration::from_secs(600), + ) + .await + .unwrap_or_else(|| panic!("never detected, proof tick {proof_tick:?}")); + assert!( + detected >= Duration::from_secs(30) && detected <= bound + Duration::from_millis(1), + "proof tick {proof_tick:?}: detected after {detected:?}" + ); + } + } + + #[tokio::test(start_paused = true)] + async fn the_pre_r4_relative_timeout_never_detects_a_silent_socket_while_the_proof_ticks() { + // Reproduces Astra R4 on the old shape: 10 idle periods, still "alive". + assert_eq!( + detect_dead( + silent_venue(false).await, + Some(Duration::from_millis(250)), + false, + Duration::from_secs(150), + ) + .await, + None + ); + } + + #[tokio::test(start_paused = true)] + async fn a_socket_that_answers_pings_stays_alive_with_proof_on_and_off() { + for proof_tick in [None, Some(Duration::from_millis(250))] { + assert_eq!( + detect_dead( + silent_venue(true).await, + proof_tick, + true, + Duration::from_secs(120), + ) + .await, + None, + "proof tick {proof_tick:?}" + ); + } } #[test] From fdd379e2993855dfe8a417ff7b8744a36d944846 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 07:08:44 +0000 Subject: [PATCH 102/147] fix(stream): hot lookup touches only its key's partition (Astra R2) The canonical hot reader try_locked every partition, scanned each ring backwards with decoding under the lock, and try_locked the reader-progress and broker-head mutexes the live reader takes every poll. Concurrent lookups of unrelated products refused each other HOT_READER_BUSY and the scans starved dispatch. - Dispatch keeps, under the same partition lock, a per-key view of the ring (evicted with it) and a key -> partition directory (changed only when a key enters/leaves a ring). - A lookup copies the key's newest Arc refs (chunks of 256), the dispatch boundary and the broker head proof in one critical section; decode and all gap/generation/sibling/product checks run outside it, unchanged. - Broker head proof lives in the partition state (recorded only when the reader reached it), read with the boundary; reader progress is atomic. - Locks are waited for until the request deadline, then HOT_READER_BUSY; poison is HOT_READER_UNAVAILABLE. Tests: 4 new hub unit tests, 3 updated; new tests/hot_lookup_contention.rs (8 threads, 6 partitions, 20k rec/s): 1fb68ed 82-84% BUSY and dispatch ~2.5k/s; this commit 0 refusals, dispatch 20k/s. --- rust/qdl-stream-gateway/src/hub.rs | 613 ++++++++++++++---- .../tests/hot_lookup_contention.rs | 291 +++++++++ 2 files changed, 776 insertions(+), 128 deletions(-) create mode 100644 rust/qdl-stream-gateway/tests/hot_lookup_contention.rs diff --git a/rust/qdl-stream-gateway/src/hub.rs b/rust/qdl-stream-gateway/src/hub.rs index bf54225a..7d96f61b 100644 --- a/rust/qdl-stream-gateway/src/hub.rs +++ b/rust/qdl-stream-gateway/src/hub.rs @@ -13,13 +13,23 @@ //! replayed (from the ring when it still covers the range, else by a separate //! bounded reader), every record at or above it goes to the subscriber's //! queue. The live reader is never sought. +//! +//! Hot read (R2, Astra advisory 2026-10-10): a canonical hot lookup touches +//! only the partition(s) that hold its key. Dispatch keeps, under the same +//! partition lock, a per-key view of the ring (`by_key`) and a directory of +//! the partitions whose ring holds each key (`key_home`). A lookup copies the +//! key's `Arc` references, the dispatch boundary and the broker head proof in +//! one short critical section and decodes/validates outside it. Locks are +//! acquired with a bounded wait against the request deadline instead of a +//! single `try_lock`, and reader progress is an atomic, so concurrent lookups +//! of different (or the same) products no longer refuse each other. use crate::generated::marketdata_v2::EventEnvelope; use crate::subscription::Subscription; use prost::Message as _; use std::collections::{HashMap, VecDeque}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, MutexGuard, TryLockError}; use std::time::{Duration, Instant}; /// Accounting overhead per queued/ringed record beyond key and payload. @@ -186,6 +196,53 @@ struct PartitionState { /// The ring holds every dispatched record in `[ring_start, next_offset)`. ring_start: i64, subscribers: HashMap, Vec>>, + /// The ring's records per key, in ring (offset) order. Same entries as + /// `ring`, so it is bounded by the ring and evicted with it. + by_key: HashMap, VecDeque<(Instant, SharedRecord)>>, + /// Last broker head this partition had caught up to, and when the broker + /// confirmed it. Read under the same lock as `next_offset`. + broker_head: Option<(i64, Instant)>, +} + +/// Ring entries a hot lookup copies per critical section. Non-book keys +/// resolve on the newest entry; a BOOK_SNAPSHOT lookup behind many deltas +/// continues with further chunks, each one short lock. +const HOT_SCAN_CHUNK: usize = 256; + +/// What a hot lookup copies out of one partition in one critical section. +struct KeySnapshot { + partition: i32, + /// The key's newest ring entries (at most [`HOT_SCAN_CHUNK`]), newest first. + entries: Vec<(Instant, SharedRecord)>, + boundary_offset: i64, + broker_head: Option<(i64, Instant)>, +} + +/// Acquire `mutex` without blocking past `deadline`. A lock still held at the +/// deadline is `HOT_READER_BUSY`; a poisoned one means a reader-side panic. +fn lock_until(mutex: &Mutex, deadline: Instant) -> Result, String> { + let mut spins = 0u32; + loop { + match mutex.try_lock() { + Ok(guard) => return Ok(guard), + Err(TryLockError::Poisoned(_)) => return Err("HOT_READER_UNAVAILABLE".into()), + Err(TryLockError::WouldBlock) => { + if Instant::now() >= deadline { + return Err("HOT_READER_BUSY".into()); + } + // Holders keep these locks for O(record) work; spin briefly, + // then back off so a waiter never burns a core. + if spins < 64 { + std::hint::spin_loop(); + } else if spins < 128 { + std::thread::yield_now(); + } else { + std::thread::sleep(Duration::from_micros(20)); + } + spins = spins.saturating_add(1); + } + } + } } pub struct Hub { @@ -194,8 +251,13 @@ pub struct Hub { pub metrics: HubMetrics, stop: AtomicBool, reader_failed: AtomicBool, - reader_progress: Mutex>, - broker_heads: Mutex>, + /// Last reader poll, in ns after `origin` plus one; 0 = never. + reader_progress: AtomicU64, + origin: Instant, + /// Partitions whose ring currently holds each key. Changed only when a + /// key enters or leaves a partition's ring (lock order: partition, then + /// this); lookups hold it alone, for a map read. + key_home: Mutex, Vec>>, /// Records produced before this replica started (the warm range, a /// restart backlog) are not live-path lag and stay out of the histogram. started_ms: i64, @@ -222,6 +284,8 @@ impl Hub { ring_bytes: 0, ring_start: start, subscribers: HashMap::new(), + by_key: HashMap::new(), + broker_head: None, }), ) }) @@ -232,8 +296,12 @@ impl Hub { metrics: HubMetrics::default(), stop: AtomicBool::new(false), reader_failed: AtomicBool::new(true), - reader_progress: Mutex::new(None), - broker_heads: Mutex::new(HashMap::new()), + reader_progress: AtomicU64::new(0), + // Before any reader instant, so an old progress mark is representable. + origin: Instant::now() + .checked_sub(Duration::from_secs(3_600)) + .unwrap_or_else(Instant::now), + key_home: Mutex::new(HashMap::new()), started_ms: now_ns() / 1_000_000, } } @@ -250,7 +318,7 @@ impl Hub { .and_then(|state| state.lock().ok().map(|state| state.next_offset)) } - fn evict(&self, state: &mut PartitionState, now: Instant) { + fn evict(&self, partition: i32, state: &mut PartitionState, now: Instant) { while let Some((at, front)) = state.ring.front() { let aged = now.duration_since(*at) > self.config.ring_max_age; if state.ring_bytes <= self.config.ring_max_bytes && !aged { @@ -259,11 +327,43 @@ impl Hub { let weight = front.weight(); state.ring_start = front.offset + 1; state.ring_bytes -= weight; - state.ring.pop_front(); + let Some((_, front)) = state.ring.pop_front() else { + break; + }; + // The ring and every per-key view are in offset order, so the + // evicted record is the front of its key's view. + let emptied = state.by_key.get_mut(&front.key).is_some_and(|entries| { + entries.pop_front(); + entries.is_empty() + }); + if emptied { + state.by_key.remove(&front.key); + self.leave_home(&front.key, partition); + } self.metrics.ring_evictions.fetch_add(1, Ordering::Relaxed); } } + fn enter_home(&self, key: &[u8], partition: i32) { + if let Ok(mut homes) = self.key_home.lock() { + let entry = homes.entry(key.to_vec()).or_default(); + if !entry.contains(&partition) { + entry.push(partition); + } + } + } + + fn leave_home(&self, key: &[u8], partition: i32) { + if let Ok(mut homes) = self.key_home.lock() { + if let Some(entry) = homes.get_mut(key) { + entry.retain(|home| *home != partition); + if entry.is_empty() { + homes.remove(key); + } + } + } + } + /// Dispatch one committed record. A record at or below an already /// dispatched offset is a duplicate transport delivery and is ignored. pub fn dispatch(&self, record: RawRecord) { @@ -290,8 +390,16 @@ impl Hub { let now = Instant::now(); state.ring_bytes += record.weight(); state.ring.push_back((now, record.clone())); + if let Some(entries) = state.by_key.get_mut(&record.key) { + entries.push_back((now, record.clone())); + } else { + state + .by_key + .insert(record.key.clone(), VecDeque::from([(now, record.clone())])); + self.enter_home(&record.key, record.partition); + } state.next_offset = record.offset + 1; - self.evict(&mut state, now); + self.evict(record.partition, &mut state, now); let Some(subscribers) = state.subscribers.get(&record.key) else { return; }; @@ -323,13 +431,65 @@ impl Hub { state.ring_start = position; } } - self.evict(&mut state, Instant::now()); + self.evict(partition, &mut state, Instant::now()); } } } + /// Copy `key`'s newest ring entries, the dispatch boundary and the broker + /// head proof of one partition in a single critical section. + fn key_snapshot( + &self, + partition: i32, + key: &[u8], + deadline: Instant, + ) -> Result, String> { + let Some(state) = self.partitions.get(&partition) else { + return Ok(None); + }; + let state = lock_until(state, deadline)?; + let Some(entries) = state.by_key.get(key) else { + return Ok(None); + }; + Ok(Some(KeySnapshot { + partition, + entries: entries.iter().rev().take(HOT_SCAN_CHUNK).cloned().collect(), + boundary_offset: state.next_offset - 1, + broker_head: state.broker_head, + })) + } + + /// The next chunk of `key`'s ring entries strictly below `before`, newest + /// first. Records are immutable and only appended at the back or evicted + /// at the front, so this continues the first snapshot exactly; an + /// entry evicted meanwhile is simply absent (the scan then finds less). + fn key_entries_before( + &self, + partition: i32, + key: &[u8], + before: i64, + deadline: Instant, + ) -> Result, String> { + let Some(state) = self.partitions.get(&partition) else { + return Ok(Vec::new()); + }; + let state = lock_until(state, deadline)?; + let Some(entries) = state.by_key.get(key) else { + return Ok(Vec::new()); + }; + let end = entries.partition_point(|(_, record)| record.offset < before); + Ok(entries + .range(end.saturating_sub(HOT_SCAN_CHUNK)..end) + .rev() + .cloned() + .collect()) + } + /// A bounded read of committed canonical bytes, not an execution-quality verdict. /// No Redis, Query, new Kafka reader, offset commit, or retained decoded cache. + /// + /// Touches only the partition(s) whose ring holds `key`; the critical + /// section copies references, decoding and validation run outside it. pub fn latest_hot( &self, key: &[u8], @@ -345,123 +505,174 @@ impl Hub { return Err("HOT_FEED_UNSUPPORTED".into()); } self.check_hot_reader()?; + let homes = lock_until(&self.key_home, deadline)? + .get(key) + .cloned() + .unwrap_or_default(); + // A key with unexpired records in two partitions breaks the + // partition plan; records past the ring age do not count. + let mut live = None; + for partition in homes { + let Some(snapshot) = self.key_snapshot(partition, key, deadline)? else { + continue; + }; + let now = Instant::now(); + if snapshot + .entries + .first() + .is_none_or(|(at, _)| now.duration_since(*at) > self.config.ring_max_age) + { + continue; + } + if live.is_some() { + return Err("HOT_PARTITION_MISMATCH".into()); + } + live = Some(snapshot); + } let mut selected = None; - let mut key_partition = None; - for (partition, state) in &self.partitions { - let state = state.try_lock().map_err(|_| "HOT_READER_BUSY")?; + if let Some(snapshot) = live { let mut newest_scope = None; - for (at, raw) in state.ring.iter().rev() { - let now = Instant::now(); - if now >= deadline { - return Err("HOT_READ_DEADLINE".into()); - } - if now.duration_since(*at) > self.config.ring_max_age { - break; - } - if raw.key != key { - continue; - } - if key_partition.is_some_and(|prior| prior != *partition) { - return Err("HOT_PARTITION_MISMATCH".into()); - } - key_partition = Some(*partition); - if raw.offset < 0 || raw.payload.len() > 256 * 1024 { - return Err("HOT_RECORD_INVALID".into()); - } - let envelope = EventEnvelope::decode(raw.payload.as_slice()) - .map_err(|_| "HOT_RECORD_INVALID")?; - // A newer unsafe sibling (snapshot/delta share a physical key) - // must not be hidden by returning an older healthy payload. - if envelope.quality_flags.iter().any(|flag| { - matches!( - QualityFlag::try_from(*flag), - Ok(QualityFlag::SequenceGapBefore - | QualityFlag::OutOfOrder - | QualityFlag::ResyncRequired) - ) - }) { - return Err("HOT_RECORD_GAPPED".into()); - } - let book_generation = match &envelope.payload { - Some( - crate::generated::marketdata_v2::event_envelope::Payload::BookSnapshot( - book, - ), - ) => { - if !book.sequence_verified || book.book_generation == 0 { - return Err("HOT_BOOK_UNVERIFIED".into()); - } - book.book_generation + let mut chunk = snapshot.entries; + 'scan: loop { + for (at, raw) in &chunk { + let now = Instant::now(); + if now >= deadline { + return Err("HOT_READ_DEADLINE".into()); + } + if now.duration_since(*at) > self.config.ring_max_age { + break 'scan; } - Some(crate::generated::marketdata_v2::event_envelope::Payload::BookDelta( - book, - )) => { - if !book.sequence_verified || book.book_generation == 0 || book.reset { - return Err("HOT_BOOK_UNVERIFIED".into()); + if raw.offset < 0 || raw.payload.len() > 256 * 1024 { + return Err("HOT_RECORD_INVALID".into()); + } + let envelope = EventEnvelope::decode(raw.payload.as_slice()) + .map_err(|_| "HOT_RECORD_INVALID")?; + // A newer unsafe sibling (snapshot/delta share a physical key) + // must not be hidden by returning an older healthy payload. + if envelope.quality_flags.iter().any(|flag| { + matches!( + QualityFlag::try_from(*flag), + Ok(QualityFlag::SequenceGapBefore + | QualityFlag::OutOfOrder + | QualityFlag::ResyncRequired) + ) + }) { + return Err("HOT_RECORD_GAPPED".into()); + } + let book_generation = match &envelope.payload { + Some( + crate::generated::marketdata_v2::event_envelope::Payload::BookSnapshot( + book, + ), + ) => { + if !book.sequence_verified || book.book_generation == 0 { + return Err("HOT_BOOK_UNVERIFIED".into()); + } + book.book_generation + } + Some( + crate::generated::marketdata_v2::event_envelope::Payload::BookDelta( + book, + ), + ) => { + if !book.sequence_verified || book.book_generation == 0 || book.reset { + return Err("HOT_BOOK_UNVERIFIED".into()); + } + book.book_generation } - book.book_generation + _ => 0, + }; + let scope = ( + envelope.source_session_id.clone(), + envelope.connection_generation, + envelope.authority_revision, + envelope.config_revision, + envelope.lease_epoch, + book_generation, + ); + if newest_scope.as_ref().is_some_and(|newest| newest != &scope) { + return Err("HOT_RECORD_GENERATION_MISMATCH".into()); + } + newest_scope = Some(scope); + if is_product(feed, None, &envelope) { + selected = Some(( + HotRecord { + raw: raw.clone(), + boundary_offset: snapshot.boundary_offset, + }, + snapshot.broker_head, + )); + break 'scan; + } + if !(matches!(feed, "BOOK_SNAPSHOT" | "BOOK_DELTA") + && (is_product("BOOK_SNAPSHOT", None, &envelope) + || is_product("BOOK_DELTA", None, &envelope))) + { + return Err("HOT_RECORD_PRODUCT_MISMATCH".into()); } - _ => 0, - }; - let scope = ( - envelope.source_session_id.clone(), - envelope.connection_generation, - envelope.authority_revision, - envelope.config_revision, - envelope.lease_epoch, - book_generation, - ); - if newest_scope.as_ref().is_some_and(|newest| newest != &scope) { - return Err("HOT_RECORD_GENERATION_MISMATCH".into()); } - newest_scope = Some(scope); - if is_product(feed, None, &envelope) { - selected = Some(HotRecord { - raw: raw.clone(), - boundary_offset: state.next_offset - 1, - }); + let Some((_, oldest)) = chunk.last() else { + break; + }; + if chunk.len() < HOT_SCAN_CHUNK { break; } - if !(matches!(feed, "BOOK_SNAPSHOT" | "BOOK_DELTA") - && (is_product("BOOK_SNAPSHOT", None, &envelope) - || is_product("BOOK_DELTA", None, &envelope))) - { - return Err("HOT_RECORD_PRODUCT_MISMATCH".into()); - } + let before = oldest.offset; + chunk = self.key_entries_before(snapshot.partition, key, before, deadline)?; } } self.check_hot_reader()?; - let selected = selected.ok_or_else(|| "HOT_RECORD_MISSING".to_owned())?; - let heads = self - .broker_heads - .try_lock() - .map_err(|_| "HOT_READER_BUSY")?; - let (head, confirmed) = heads - .get(&selected.raw.partition) - .ok_or("HOT_BROKER_UNCONFIRMED")?; + let (selected, head) = selected.ok_or_else(|| "HOT_RECORD_MISSING".to_owned())?; + // The head proof was copied with the boundary, under one lock. + let (head, confirmed) = head.ok_or("HOT_BROKER_UNCONFIRMED")?; if confirmed.elapsed() > Duration::from_secs(1) { return Err("HOT_BROKER_UNCONFIRMED".into()); } - if selected.boundary_offset + 1 < *head { + if selected.boundary_offset + 1 < head { return Err("HOT_READER_CATCHING_UP".into()); } Ok(selected) } + fn mark_progress(&self, at: Instant) { + let since = at.saturating_duration_since(self.origin).as_nanos(); + self.reader_progress.store( + u64::try_from(since).unwrap_or(u64::MAX - 1) + 1, + Ordering::Release, + ); + } + fn check_hot_reader(&self) -> Result<(), String> { if self.stop.load(Ordering::Acquire) || self.reader_failed.load(Ordering::Acquire) { return Err("HOT_READER_UNAVAILABLE".into()); } - let progress = self - .reader_progress - .try_lock() - .map_err(|_| "HOT_READER_BUSY")?; - if progress.is_none_or(|at| at.elapsed() > Duration::from_secs(1)) { + let progress = self.reader_progress.load(Ordering::Acquire); + let stalled = progress == 0 + || (self.origin + Duration::from_nanos(progress - 1)).elapsed() + > Duration::from_secs(1); + if stalled { return Err("HOT_READER_STALLED".into()); } Ok(()) } + /// Record a positive broker head for `partition` if the reader has + /// dispatched up to it; returns whether it was recorded. A head the + /// reader has not reached yet never replaces the last caught-up proof. + fn confirm_broker_head(&self, partition: i32, head: i64, confirmed: Instant) -> bool { + let Some(state) = self.partitions.get(&partition) else { + return false; + }; + let Ok(mut state) = state.lock() else { + return false; + }; + if state.next_offset < head { + return false; + } + state.broker_head = Some((head, confirmed)); + true + } + /// Register `subscriber` for `key` on `partition`, resuming after /// `after`. Atomic with dispatch on that partition (D3). pub fn register( @@ -548,14 +759,10 @@ impl Hub { Ok(Some(record)) => { hub.dispatch(record); hub.reader_failed.store(false, Ordering::Release); - if let Ok(mut progress) = hub.reader_progress.lock() { - *progress = Some(Instant::now()); - } + hub.mark_progress(Instant::now()); } Ok(None) => { - if let Ok(mut progress) = hub.reader_progress.lock() { - *progress = Some(Instant::now()); - } + hub.mark_progress(Instant::now()); for (partition, position) in source.positions() { hub.advance(partition, position); } @@ -572,9 +779,8 @@ impl Hub { let proofs: Vec<_> = source .broker_heads() .into_iter() - .filter(|(partition, head, _)| { - hub.next_offset(*partition) - .is_some_and(|position| position >= *head) + .filter(|(partition, head, confirmed)| { + hub.confirm_broker_head(*partition, *head, *confirmed) }) .collect(); if failed_at.is_some_and(|failed| proofs.iter().any(|(_, _, at)| *at > failed)) @@ -582,11 +788,6 @@ impl Hub { hub.reader_failed.store(false, Ordering::Release); failed_at = None; } - if let Ok(mut heads) = hub.broker_heads.lock() { - for (partition, offset, confirmed) in proofs { - heads.insert(partition, (offset, confirmed)); - } - } } }) .expect("spawn live reader thread") @@ -620,15 +821,17 @@ mod tests { } fn healthy(hub: &Hub) { hub.reader_failed.store(false, Ordering::Release); - *hub.reader_progress.lock().unwrap() = Some(Instant::now()); + hub.mark_progress(Instant::now()); for partition in hub.partitions.keys() { - hub.broker_heads - .lock() - .unwrap() - .insert(*partition, (0, Instant::now())); + set_head(hub, *partition, Some((0, Instant::now()))); } } + /// Install a head proof directly, bypassing the caught-up filter. + fn set_head(hub: &Hub, partition: i32, head: Option<(i64, Instant)>) { + hub.partitions[&partition].lock().unwrap().broker_head = head; + } + fn hot_record( partition: i32, offset: i64, @@ -693,18 +896,15 @@ mod tests { let hub = Hub::new(&[(0, 0)], HubConfig::default()); hub.dispatch(hot_record(0, 0, "QUOTE", 1, vec![])); healthy(&hub); - hub.broker_heads.lock().unwrap().clear(); + set_head(&hub, 0, None); assert_eq!(latest(&hub, "QUOTE").unwrap_err(), "HOT_BROKER_UNCONFIRMED"); - hub.broker_heads - .lock() - .unwrap() - .insert(0, (1, Instant::now() - Duration::from_secs(2))); + set_head(&hub, 0, Some((1, Instant::now() - Duration::from_secs(2)))); assert_eq!(latest(&hub, "QUOTE").unwrap_err(), "HOT_BROKER_UNCONFIRMED"); - hub.broker_heads - .lock() - .unwrap() - .insert(0, (2, Instant::now())); + set_head(&hub, 0, Some((2, Instant::now()))); assert_eq!(latest(&hub, "QUOTE").unwrap_err(), "HOT_READER_CATCHING_UP"); + // The reader path records only a head it has reached. + assert!(!hub.confirm_broker_head(0, 3, Instant::now())); + assert!(hub.confirm_broker_head(0, 1, Instant::now())); hub.advance(0, 2); assert_eq!(latest(&hub, "QUOTE").unwrap().raw.offset, 0); } @@ -798,7 +998,7 @@ mod tests { hub.dispatch(hot_record(0, 0, "QUOTE", 1, vec![])); assert_eq!(latest(&hub, "QUOTE").unwrap_err(), "HOT_READER_UNAVAILABLE"); healthy(&hub); - *hub.reader_progress.lock().unwrap() = Some(Instant::now() - Duration::from_secs(2)); + hub.mark_progress(Instant::now() - Duration::from_secs(2)); assert_eq!(latest(&hub, "QUOTE").unwrap_err(), "HOT_READER_STALLED"); healthy(&hub); assert_eq!( @@ -832,7 +1032,164 @@ mod tests { healthy(&hub); assert_eq!(hub.ring_bytes(), 0); assert_eq!(latest(&hub, "QUOTE").unwrap_err(), "HOT_RECORD_MISSING"); + // The per-key index and the directory are evicted with the ring. + assert!(hub.partitions[&0].lock().unwrap().by_key.is_empty()); + assert!(hub.key_home.lock().unwrap().is_empty()); + let hub = Hub::new(&[(0, 0)], HubConfig::default()); + hub.dispatch(hot_record(0, 0, "QUOTE", 1, vec![])); + healthy(&hub); + // The key's own partition held past the deadline: BUSY at the deadline. let _lock = hub.partitions[&0].lock().unwrap(); - assert_eq!(latest(&hub, "QUOTE").unwrap_err(), "HOT_READER_BUSY"); + let started = Instant::now(); + let budget = Duration::from_millis(30); + assert_eq!( + hub.latest_hot(b"hot", "QUOTE", started + budget) + .unwrap_err(), + "HOT_READER_BUSY" + ); + let waited = started.elapsed(); + assert!(waited >= budget, "{waited:?}"); + assert!(waited < budget + Duration::from_millis(50), "{waited:?}"); + } + + fn keyed(partition: i32, offset: i64, key: &[u8]) -> RawRecord { + RawRecord { + key: key.to_vec(), + ..hot_record(partition, offset, "QUOTE", 1, vec![]) + } + } + + #[test] + fn hot_read_ignores_unrelated_partition_lock_and_quiet_product_skips_busy_ones() { + let hub = Hub::new(&[(0, 0), (1, 0)], HubConfig::default()); + hub.dispatch(keyed(0, 0, b"quiet")); + for offset in 1..5_001 { + hub.dispatch(keyed(0, offset, b"busy")); + } + hub.dispatch(keyed(1, 0, b"other")); + healthy(&hub); + let quiet = hub + .latest_hot(b"quiet", "QUOTE", Instant::now() + Duration::from_secs(1)) + .unwrap(); + assert_eq!((quiet.raw.offset, quiet.boundary_offset), (0, 5_000)); + // An unrelated partition held by someone else does not refuse. + let _held = hub.partitions[&1].lock().unwrap(); + let busy = hub + .latest_hot(b"busy", "QUOTE", Instant::now() + Duration::from_millis(50)) + .unwrap(); + assert_eq!(busy.raw.offset, 5_000); + } + + #[test] + fn book_snapshot_behind_many_deltas_is_found_across_chunks_and_gaps_still_refuse() { + for deltas in [ + HOT_SCAN_CHUNK - 1, + HOT_SCAN_CHUNK, + HOT_SCAN_CHUNK + 1, + 3 * HOT_SCAN_CHUNK + 7, + ] { + let hub = Hub::new(&[(0, 0)], HubConfig::default()); + hub.dispatch(hot_record(0, 0, "BOOK_SNAPSHOT", 1, vec![])); + for offset in 1..=deltas as i64 { + hub.dispatch(hot_record(0, offset, "BOOK_DELTA", 1, vec![])); + } + healthy(&hub); + let view = latest(&hub, "BOOK_SNAPSHOT").unwrap(); + assert_eq!((view.raw.offset, view.boundary_offset), (0, deltas as i64)); + assert_eq!( + latest(&hub, "BOOK_DELTA").unwrap().raw.offset, + deltas as i64 + ); + } + // An unsafe sibling in a later chunk still refuses the old snapshot. + let hub = Hub::new(&[(0, 0)], HubConfig::default()); + hub.dispatch(hot_record(0, 0, "BOOK_SNAPSHOT", 1, vec![])); + hub.dispatch(hot_record(0, 1, "BOOK_DELTA", 1, vec![16])); + for offset in 2..(2 * HOT_SCAN_CHUNK as i64) { + hub.dispatch(hot_record(0, offset, "BOOK_DELTA", 1, vec![])); + } + healthy(&hub); + assert_eq!( + latest(&hub, "BOOK_SNAPSHOT").unwrap_err(), + "HOT_RECORD_GAPPED" + ); + } + + #[test] + fn key_moving_partition_after_ring_expiry_is_not_a_mismatch() { + let hub = Hub::new( + &[(0, 0), (1, 0)], + HubConfig { + ring_max_age: Duration::from_millis(30), + ..Default::default() + }, + ); + hub.dispatch(keyed(0, 0, b"hot")); + healthy(&hub); + std::thread::sleep(Duration::from_millis(40)); + // Partition 0 still holds the aged record (no eviction ran there). + hub.dispatch(keyed(1, 0, b"hot")); + assert_eq!(hub.key_home.lock().unwrap()[&b"hot".to_vec()].len(), 2); + assert_eq!(latest(&hub, "QUOTE").unwrap().raw.partition, 1); + hub.advance(0, 1); + assert_eq!(hub.key_home.lock().unwrap()[&b"hot".to_vec()], vec![1]); + hub.dispatch(keyed(0, 1, b"hot")); + assert_eq!(latest(&hub, "QUOTE").unwrap_err(), "HOT_PARTITION_MISMATCH"); + } + + #[test] + fn hot_read_snapshot_is_one_critical_section_under_concurrent_dispatch() { + // Lookups race a dispatcher that confirms the broker head right after + // each record. Every reply must be a dispatched record of the key at + // or below its boundary, the boundary must cover the head proof, and + // per-thread results never go backwards. No reply may be BUSY. + let hub = Arc::new(Hub::new(&[(0, 0), (1, 0)], HubConfig::default())); + hub.dispatch(keyed(0, 0, b"k0")); + hub.dispatch(keyed(1, 0, b"k1")); + healthy(&hub); + let stop = Arc::new(AtomicBool::new(false)); + let writer = { + let (hub, stop) = (hub.clone(), stop.clone()); + std::thread::spawn(move || { + let mut offset = 1; + while !stop.load(Ordering::Relaxed) { + for partition in [0, 1] { + let key = if partition == 0 { b"k0" } else { b"k1" }; + hub.dispatch(keyed(partition, offset, key)); + assert!(hub.confirm_broker_head(partition, offset + 1, Instant::now())); + } + hub.mark_progress(Instant::now()); + offset += 1; + } + offset + }) + }; + let readers: Vec<_> = (0..8) + .map(|index| { + let hub = hub.clone(); + std::thread::spawn(move || { + let key: &[u8] = if index % 2 == 0 { b"k0" } else { b"k1" }; + let (mut ok, mut last) = (0, -1); + let until = Instant::now() + Duration::from_millis(300); + while Instant::now() < until { + let deadline = Instant::now() + Duration::from_millis(100); + let view = hub.latest_hot(key, "QUOTE", deadline).unwrap(); + assert_eq!(view.raw.key, key); + assert!(view.raw.offset <= view.boundary_offset); + assert!(view.raw.offset >= last, "went backwards"); + assert!( + view.boundary_offset < hub.next_offset(view.raw.partition).unwrap() + ); + last = view.raw.offset; + ok += 1; + } + ok + }) + }) + .collect(); + let total: usize = readers.into_iter().map(|r| r.join().unwrap()).sum(); + stop.store(true, Ordering::Relaxed); + assert!(writer.join().unwrap() > 1); + assert!(total > 0); } } diff --git a/rust/qdl-stream-gateway/tests/hot_lookup_contention.rs b/rust/qdl-stream-gateway/tests/hot_lookup_contention.rs new file mode 100644 index 00000000..9379b271 --- /dev/null +++ b/rust/qdl-stream-gateway/tests/hot_lookup_contention.rs @@ -0,0 +1,291 @@ +//! R2 (Astra advisory 2026-10-10): concurrent canonical hot lookups of +//! different products must not refuse each other while the live reader +//! dispatches. In-process workload over the public `Hub` API only (a paced +//! fake committed log driven by `Hub::run`), so the same file measures the +//! pre-fix hub as the counterfactual. Prints lookup latency, refusal counts +//! and the hub's commit -> dispatch lag; run with `--nocapture` to see them. + +use prost::Message as _; +use qdl_stream_gateway::generated::marketdata_v2::{ + event_envelope::Payload, EventEnvelope, OrderBookDelta, OrderBookSnapshot, Quote, +}; +use qdl_stream_gateway::hub::{Hub, HubConfig, LogSource, RawRecord}; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const PARTITIONS: i32 = 6; +/// Busy QUOTE products, one per (partition, index). +const BUSY: i32 = 48; +/// BOOK keys (snapshot every `SNAPSHOT_EVERY` records, deltas between). +const BOOKS: i32 = 6; +const SNAPSHOT_EVERY: u64 = 100; +/// Records per second offered by the fake log, over all partitions. +const RATE: u64 = 20_000; +const LOOKUP_THREADS: usize = 8; + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as i64 +} + +fn envelope(payload: Payload) -> Vec { + EventEnvelope { + payload: Some(payload), + connection_generation: 1, + ..Default::default() + } + .encode_to_vec() +} + +fn quote() -> Vec { + envelope(Payload::Quote(Quote::default())) +} + +fn book(snapshot: bool) -> Vec { + envelope(if snapshot { + Payload::BookSnapshot(OrderBookSnapshot { + book_generation: 1, + sequence_verified: true, + ..Default::default() + }) + } else { + Payload::BookDelta(OrderBookDelta { + book_generation: 1, + sequence_verified: true, + ..Default::default() + }) + }) +} + +/// Paced committed log: a few quiet records first, then busy QUOTE and BOOK +/// traffic round-robin. Broker heads are what was handed to the reader, so +/// the hub confirms each one right after dispatching it. +struct PacedLog { + next: BTreeMap, + sent: u64, + started: Instant, + pending: Vec<(i32, Vec, Vec)>, +} + +impl PacedLog { + fn new() -> Self { + let mut pending = Vec::new(); + for partition in 0..PARTITIONS { + pending.push(( + partition, + format!("quiet-{partition}").into_bytes(), + quote(), + )); + } + Self { + next: (0..PARTITIONS).map(|p| (p, 0)).collect(), + sent: 0, + started: Instant::now(), + pending, + } + } + + fn next_record(&mut self) -> (i32, Vec, Vec) { + if let Some(record) = self.pending.pop() { + return record; + } + let slot = (self.sent % (BUSY + BOOKS) as u64) as i32; + let round = self.sent / (BUSY + BOOKS) as u64; + if slot < BUSY { + ( + slot % PARTITIONS, + format!("busy-{slot}").into_bytes(), + quote(), + ) + } else { + let index = slot - BUSY; + let snapshot = round % SNAPSHOT_EVERY == 0; + ( + index % PARTITIONS, + format!("book-{index}").into_bytes(), + book(snapshot), + ) + } + } +} + +impl LogSource for PacedLog { + fn poll(&mut self, timeout: Duration) -> Result, String> { + let due = self.started + Duration::from_nanos(self.sent * 1_000_000_000 / RATE); + let now = Instant::now(); + if due > now { + let wait = (due - now).min(timeout); + std::thread::sleep(wait); + if Instant::now() < due { + return Ok(None); + } + } + let (partition, key, payload) = self.next_record(); + let offset = self.next[&partition]; + self.next.insert(partition, offset + 1); + self.sent += 1; + Ok(Some(RawRecord { + partition, + offset, + key, + payload, + timestamp_ms: now_ms(), + })) + } + + fn positions(&self) -> Vec<(i32, i64)> { + self.next.iter().map(|(p, o)| (*p, *o)).collect() + } + + fn broker_heads(&self) -> Vec<(i32, i64, Instant)> { + let now = Instant::now(); + self.next.iter().map(|(p, o)| (*p, *o, now)).collect() + } +} + +#[derive(Default)] +struct Outcome { + latencies_us: Vec, + refusals: BTreeMap, + ok: u64, +} + +fn quantile(sorted: &[u64], q: f64) -> u64 { + if sorted.is_empty() { + return 0; + } + sorted[((sorted.len() as f64 * q).ceil() as usize).clamp(1, sorted.len()) - 1] +} + +/// Each lookup thread owns distinct products: QUOTE busy keys, quiet keys +/// (behind every busy record of their partition) and BOOK_SNAPSHOT keys +/// (behind up to `SNAPSHOT_EVERY` deltas). +fn lookup_targets(thread: usize) -> Vec<(Vec, &'static str)> { + let mut targets = Vec::new(); + for slot in 0..BUSY { + if slot as usize % LOOKUP_THREADS == thread { + targets.push((format!("busy-{slot}").into_bytes(), "QUOTE")); + } + } + for partition in 0..PARTITIONS { + if partition as usize % LOOKUP_THREADS == thread { + targets.push((format!("quiet-{partition}").into_bytes(), "QUOTE")); + } + } + for index in 0..BOOKS { + if (index as usize + 3) % LOOKUP_THREADS == thread { + targets.push((format!("book-{index}").into_bytes(), "BOOK_SNAPSHOT")); + } + } + targets +} + +fn run_workload(duration: Duration) -> (Outcome, serde_json::Value, u64) { + let hub = Arc::new(Hub::new( + &(0..PARTITIONS).map(|p| (p, 0)).collect::>(), + HubConfig::default(), + )); + let reader = hub.run(Box::new(PacedLog::new())); + // Warm: every product (including the first BOOK snapshot) is readable. + let warm = Instant::now() + Duration::from_secs(10); + for thread in 0..LOOKUP_THREADS { + for (key, feed) in lookup_targets(thread) { + while hub + .latest_hot(&key, feed, Instant::now() + Duration::from_millis(100)) + .is_err() + { + assert!(Instant::now() < warm, "warm-up timed out"); + std::thread::sleep(Duration::from_millis(5)); + } + } + } + let records_before = hub.metrics.records.load(Ordering::Relaxed); + let stop = Arc::new(AtomicBool::new(false)); + let workers: Vec<_> = (0..LOOKUP_THREADS) + .map(|thread| { + let (hub, stop) = (hub.clone(), stop.clone()); + std::thread::spawn(move || { + let targets = lookup_targets(thread); + let mut outcome = Outcome::default(); + let mut index = 0; + while !stop.load(Ordering::Relaxed) { + let (key, feed) = &targets[index % targets.len()]; + index += 1; + let started = Instant::now(); + let result = hub.latest_hot(key, feed, started + Duration::from_millis(100)); + outcome + .latencies_us + .push(started.elapsed().as_micros() as u64); + match result { + Ok(view) => { + assert_eq!(&view.raw.key, key); + assert!(view.raw.offset <= view.boundary_offset); + outcome.ok += 1; + } + Err(code) => *outcome.refusals.entry(code).or_default() += 1, + } + // A request rate, not a spin: the RPC layer admits 8 at once. + std::thread::sleep(Duration::from_micros(50)); + } + outcome + }) + }) + .collect(); + std::thread::sleep(duration); + stop.store(true, Ordering::Relaxed); + let mut total = Outcome::default(); + for worker in workers { + let outcome = worker.join().unwrap(); + total.ok += outcome.ok; + total.latencies_us.extend(outcome.latencies_us); + for (code, count) in outcome.refusals { + *total.refusals.entry(code).or_default() += count; + } + } + let dispatched = hub.metrics.records.load(Ordering::Relaxed) - records_before; + let lag = hub.metrics.dispatch_lag.summary(); + hub.stop(); + reader.join().unwrap(); + total.latencies_us.sort_unstable(); + (total, lag, dispatched) +} + +#[test] +fn concurrent_lookups_of_different_products_do_not_refuse_each_other() { + let duration = Duration::from_secs(2); + let (outcome, lag, dispatched) = run_workload(duration); + let lookups = outcome.latencies_us.len() as u64; + let busy = outcome + .refusals + .get("HOT_READER_BUSY") + .copied() + .unwrap_or(0); + println!( + "R2_BENCH lookups={lookups} ok={} busy={busy} refusals={:?} \ + latency_us p50={} p95={} p99={} max={} dispatched={dispatched} \ + dispatch_rate_per_s={} dispatch_lag_ms={}", + outcome.ok, + outcome.refusals, + quantile(&outcome.latencies_us, 0.50), + quantile(&outcome.latencies_us, 0.95), + quantile(&outcome.latencies_us, 0.99), + outcome.latencies_us.last().copied().unwrap_or(0), + dispatched / duration.as_secs(), + lag, + ); + assert!(lookups > 1_000, "workload too small: {lookups}"); + assert_eq!( + busy, 0, + "lookups refused each other: {:?}", + outcome.refusals + ); + assert_eq!( + outcome.ok, lookups, + "unexpected refusals: {:?}", + outcome.refusals + ); +} From 10f9f5c42685ba65a4ec9f3b135c6d3870060546 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 07:10:06 +0000 Subject: [PATCH 103/147] docs(plan): Astra R1-R7 response table, OKX reconnect, R2 hot reader --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 38 +++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 88a32efe..db4c2d9c 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71832,3 +71832,41 @@ errors, 5 with 1, 0 UNKNOWN. - Next (rc6): QDL_QUERY_COLD_SLICE_MS 4 -> 8 (the one tuned variable; Astra R5 option 2) to cut per-read service time below ~3.1 s for the same-replica worst case, scored together with the hot tails. Plus the R2 Rust hot-reader fix (Stream image) and R1/R3/R6 (Query image). Then stage 100g. + +### Astra advisory R1-R7 response, OKX reconnect, R2 hot reader - 2026-10-10 (Claude, subagents) + +Advisory: upgrade/evidence/p184-astra-architecture-advisory-20261010.md (1fb68ed). Every item was checked against the +code before acting. All are tested locally and not deployed. + +| ID | Response | Selected option | Test / receipt | Runtime | +| --- | --- | --- | --- | --- | +| R1 | Accept (gap, partition/generation, lineage). The topic case is unreachable through the real client (`_decode` expectation); the backend gate is hardened anyway. Splitting DATA_LOSS details is staged | Hard refusals in the IDR-084 branch map to FENCED/INTEGRITY like the backup path; transient HotViewUnavailable keeps the validated cache view | 57881a2; RealOrderingChecks (d); counterfactual fails on 1fb68ed | not deployed | +| R2 | Accept (reproduced: 82-84 % BUSY, dispatch ~2.5k of 20k rec/s at 1fb68ed) | Per-key ring view and key->partition directory maintained by dispatch under the partition lock; Arc copy in one short section, decode outside; head proof read with the boundary; atomic reader progress; lock wait bounded by the 100 ms deadline. Fence/gap/sibling/generation checks unchanged | fdd379e (from d1167fe); stream-gateway lib 43/0, hot_lookup_contention 1/0 (0 BUSY, 20k rec/s, p99 ~300 us, debug in-process), native_stream 37/0, clippy clean | not deployed | +| R3 | Accept | One budget covers memo lock wait and lookup; HOT_READ_DEADLINE is transient and not memoised | 57881a2; MarkHotBudgetTests, HotClientBudgetTests | not deployed | +| R4 | Accept, wider than reported: every competing select branch reset the per-pass read timeout, in both loops, and the nested pong wait stalled the loop 15 s | Absolute InboundDeadline moved only by a message actually read; ping at 15 s of silence; dead at 2 x heartbeat; proof stamped only from received data | d4f5ec9 (from beeaea4); silent-socket test proof on/off (dead at 30 s), pre-R4 shape reproduced (still alive at 150 s) | not deployed | +| R5 | Accept; attribution measured in stage 100f (same-replica queue, client 242-285 ms) | Option 2: tune QDL_QUERY_COLD_SLICE_MS on the same workload, worst case same replica, with hot tails watched | stage100f receipt (7f974c3 split) | env on rc6 | +| R6 | Accept | Rate-bounded, payload-free `qdl_refusal` lines matched by detail_sha256; receipts carry reason_tokens | 57881a2; tests/test_refusal_log.py | not deployed | +| R7 | Accept as release discipline | Option B: one immutable packet with the IDR-040 proof flag OFF; check the final SDK wheel and image, not a mounted checkout; G12 scope stated as isolated BOOK; G4 unobserved rows NOT_MEASURED; historical failures and skips categorised | D5 packet | release | + +OKX reconnect (f861663 from 34dbdf9, d4f5ec9 from beeaea4): +- The stage 100e refusal was a session-identity refusal, not an event-age one. The trade came from generation 167 after + the lane's liveness record became 168. +- 37 OKX disconnects since 2026-10-09 00:00Z (1.22/h), all first attempts, clustered across lanes. This is consistent + with rolling upgrades but unproven. +- Reconnect took 620-634 ms (n=5), of which 400 ms was a fixed backoff. +- OKX documents notice 64008 60 s before a service upgrade. +- Fix: + - make-before-break handover on 64008, with the generation fences intact; + - a per-lane lineage chain; Query accepts a predecessor's event only with proven continuity + (SOURCE_SESSION_CONTINUED); + - abrupt closes reconnect with no backoff through a shared <= 3/s pacer; + - a frame arriving during the pong wait is no longer dropped; + - disconnect logs carry cause, close code, last-frame-to-detection and drain time. +- Tests: ingestor 39/0; session liveness 36 OK. After the cherry-pick, 7 related Python modules OK, including + test_phaseb_stable_edge and test_phase105_identity_acceptance. +- Open: + - silent-socket detection under 5 s needs heartbeat_seconds <= 2 (deployment config); + - OKX ends port 8443 on 2026-10-31. Our URLs are in qdl/adapters/okx/client.py, qdl/runtime/l2_demand.py, + qdl/runtime/production_catalog.py and config/v2/stable-acquisition-bindings.yaml. The bindings change alters the + acquisition plan revision (C.19/C.32 bar-edge checkpoint coupling), so it needs its own packet before + 2026-10-31 and is not mixed into rc6. From 47bec0401cff26e10e9005d5acb57da64a59374e Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 07:32:27 +0000 Subject: [PATCH 104/147] feat(query,sdk): opt-in replica redirect for a busy large-warmup lane (P18.4 G5) Stage 100f: two 2,500-row cold warmups sent 0.15 s apart both landed on query_v2_1 and ran one after the other on its single-active large-warmup lane (transport 3469 / 7031 ms, client ~250 ms) while query_v2_2 was idle. Query: a fully-local warmup that needs the large lane and carries X-QDL-Replica-Redirect: 1 is refused at once when the lane would make it wait (active job, earlier waiter, or full queue): RATE_LIMITED, retryable, retry_after_ms 0, detail "REPLICA_LANE_BUSY: ...", response headers X-QDL-Refusal-Reason: REPLICA_LANE_BUSY and Retry-After: 0. Nothing is reserved or read. Without the header, with 0 or any other value, small warmups and mixed/provider batches queue exactly as before. SDK 2.2.0: ReplicatedRestQueryTransport.warmup/warmup_batch send the header (1) when another candidate remains; on exactly that refusal the same request goes once to the next replica with header 0 (queues, no ping-pong). Other typed errors stay authoritative. DataLayerError.reason carries X-QDL-Refusal-Reason. Each attempt consumes one quota unit. OpenAPI snapshot regenerated (additive optional header on warmup and warmup:batch). test_phaseb_stable_release now expects SDK 2.2.0 (stale 2.1.0 assertion since the 2.2.0 bump). --- contracts/v2/openapi.snapshot.json | 36 +++ qdl/api_v2/router.py | 50 +++- qdl/query/lanes.py | 43 +++ qdl/query/service.py | 57 +++- qdl_sdk/CHANGELOG.md | 11 + qdl_sdk/errors.py | 4 + qdl_sdk/transport.py | 73 ++++- tests/test_phaseb_stable_release.py | 2 +- tests/test_qdl_sdk_replica_read_transport.py | 154 ++++++++++ tests/test_query_replica_lane_redirect.py | 287 +++++++++++++++++++ 10 files changed, 705 insertions(+), 12 deletions(-) create mode 100644 tests/test_query_replica_lane_redirect.py diff --git a/contracts/v2/openapi.snapshot.json b/contracts/v2/openapi.snapshot.json index 71c79473..83039342 100644 --- a/contracts/v2/openapi.snapshot.json +++ b/contracts/v2/openapi.snapshot.json @@ -3715,6 +3715,24 @@ "title": "X-Qdl-Purpose", "type": "string" } + }, + { + "description": "Optional. `1`: if this replica's large-warmup lane would make the request wait, refuse it at once with retryable RATE_LIMITED (retry_after_ms 0, detail and `X-QDL-Refusal-Reason` REPLICA_LANE_BUSY) so the client can retry once on another replica. Absent, `0` or any other value: queue as before.", + "in": "header", + "name": "X-QDL-Replica-Redirect", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional. `1`: if this replica's large-warmup lane would make the request wait, refuse it at once with retryable RATE_LIMITED (retry_after_ms 0, detail and `X-QDL-Refusal-Reason` REPLICA_LANE_BUSY) so the client can retry once on another replica. Absent, `0` or any other value: queue as before.", + "title": "X-Qdl-Replica-Redirect" + } } ], "requestBody": { @@ -4516,6 +4534,24 @@ "title": "X-Qdl-Purpose", "type": "string" } + }, + { + "description": "Optional. `1`: if this replica's large-warmup lane would make the request wait, refuse it at once with retryable RATE_LIMITED (retry_after_ms 0, detail and `X-QDL-Refusal-Reason` REPLICA_LANE_BUSY) so the client can retry once on another replica. Absent, `0` or any other value: queue as before.", + "in": "header", + "name": "X-QDL-Replica-Redirect", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional. `1`: if this replica's large-warmup lane would make the request wait, refuse it at once with retryable RATE_LIMITED (retry_after_ms 0, detail and `X-QDL-Refusal-Reason` REPLICA_LANE_BUSY) so the client can retry once on another replica. Absent, `0` or any other value: queue as before.", + "title": "X-Qdl-Replica-Redirect" + } } ], "responses": { diff --git a/qdl/api_v2/router.py b/qdl/api_v2/router.py index 748b854a..c1781750 100644 --- a/qdl/api_v2/router.py +++ b/qdl/api_v2/router.py @@ -129,6 +129,27 @@ def _purpose(value: Annotated[str, Header(alias="X-QDL-Purpose")]): ) from error +_REPLICA_REDIRECT_HEADER = "X-QDL-Replica-Redirect" + + +def _replica_redirect( + value: Annotated[ + str | None, + Header( + alias=_REPLICA_REDIRECT_HEADER, + description=( + "Optional. `1`: if this replica's large-warmup lane would make the " + "request wait, refuse it at once with retryable RATE_LIMITED " + "(retry_after_ms 0, detail and `X-QDL-Refusal-Reason` " + "REPLICA_LANE_BUSY) so the client can retry once on another " + "replica. Absent, `0` or any other value: queue as before." + ), + ), + ] = None, +) -> bool: + return value is not None and value.strip() == "1" + + def _requirement(model) -> DataRequirement: return DataRequirement.from_mapping(model.model_dump()) @@ -666,7 +687,9 @@ async def _warmup_json_off_loop(build) -> Response: return Response(content=body, media_type="application/json") -async def _single_warmup_response(request, access, service, requirement, purpose): +async def _single_warmup_response( + request, access, service, requirement, purpose, *, redirect: bool = False +): """One warmup, rendered inside the local lease exactly like ``warmup:batch``. Holding the lease through rendering keeps at most one large response being @@ -691,10 +714,13 @@ async def render(batch_result): complete = getattr(service, "warmup_batch_completed_async", None) if callable(complete): + # The opt-in is passed only when requested, so service doubles that + # predate it keep working for every request without the header. return await complete( BatchRequirement(access.consumer_id, (requirement,), require_all=True), purpose=purpose, completion=render, + **({"redirect_when_busy": True} if redirect else {}), ) # Service doubles predating the completion hook keep the original path. result = await service.warmup_async( @@ -1062,6 +1088,7 @@ async def warmup( purpose: AccessPurpose = Depends(_purpose), service: V2QueryService = Depends(_service), access: DataPlaneAccess = Depends(_data_access), + redirect: bool = Depends(_replica_redirect), ): try: warmup_spec = _warmup_specification( @@ -1088,7 +1115,9 @@ async def warmup( access.require_permission(DataPlanePermission.HISTORY_READ) access.require_purpose(purpose) access.require_requirement(requirement) - return await _single_warmup_response(request, access, service, requirement, purpose) + return await _single_warmup_response( + request, access, service, requirement, purpose, redirect=redirect + ) @router.get("/market-data/{instrument_uid}/history", response_model=WarmupResponse) @@ -1157,6 +1186,7 @@ async def warmup_batch( purpose: AccessPurpose = Depends(_purpose), service: V2QueryService = Depends(_service), access: DataPlaneAccess = Depends(_data_access), + redirect: bool = Depends(_replica_redirect), ): access.require_consumer(body.consumer_id) access.require_permission(DataPlanePermission.HISTORY_READ) @@ -1179,7 +1209,12 @@ async def render(result): complete = getattr(service, "warmup_batch_completed_async", None) if callable(complete): - return await complete(batch, purpose=purpose, completion=render) + return await complete( + batch, + purpose=purpose, + completion=render, + **({"redirect_when_busy": True} if redirect else {}), + ) # Focused service doubles predating the internal completion hook retain the # old public service protocol. Production V2QueryService always uses it. return await render(await service.warmup_batch_async(batch, purpose=purpose)) @@ -1448,10 +1483,19 @@ async def query_error_handler(_request: Request, error: QueryServiceError): REFUSALS.record(origin="query", code=problem.code, detail=problem.detail, status=problem.status, retryable=problem.retryable, request_id=problem.request_id, instrument_uid=problem.instrument_uid, route=_route_of(_request)) + headers = None + reason = getattr(error, "reason", None) + if reason: + # Additive response headers for a typed refusal reason (P18.4 G5: + # REPLICA_LANE_BUSY); the problem body schema is unchanged. + headers = {"X-QDL-Refusal-Reason": reason} + if problem.retry_after_ms is not None: + headers["Retry-After"] = str(-(-problem.retry_after_ms // 1000)) return JSONResponse( status_code=problem.status, content=problem.model_dump(exclude_none=True), media_type="application/problem+json", + headers=headers, ) @app.exception_handler(RequestValidationError) diff --git a/qdl/query/lanes.py b/qdl/query/lanes.py index 16c779d7..392dc367 100644 --- a/qdl/query/lanes.py +++ b/qdl/query/lanes.py @@ -21,6 +21,15 @@ class ReadLaneRejected(RuntimeError): """A finite in-process read lane cannot accept more work.""" +class ReadLaneBusy(ReadLaneRejected): + """The lane would queue this request; the caller asked not to wait here. + + Raised only when ``run(..., refuse_if_queued=True)``: the request could + not start at once (another job holds the active slot, an earlier waiter is + ahead, or the busy lane's queue is full). Nothing was reserved or queued. + """ + + @dataclass(frozen=True, slots=True) class ReadLanePolicy: """One bounded fair lane with an optional reserved consumer share.""" @@ -102,6 +111,7 @@ def __init__(self, policy: ReadLanePolicy) -> None: self._rejected = 0 self._rejected_bytes = 0 self._queue_wait_timeouts = 0 + self._busy_refusals = 0 async def run( self, @@ -110,6 +120,7 @@ async def run( consumer_id: str, reserved_bytes: int, wait_timeout_ms: int | None = None, + refuse_if_queued: bool = False, ) -> T: if not consumer_id.strip(): raise ValueError("read lane consumer_id is required") @@ -122,6 +133,17 @@ async def run( entry: _Entry | None = None active = False async with self._condition: + if ( + refuse_if_queued + and (self._active or self._waiting) + and reserved_bytes <= self.policy.max_pending_bytes + and not self._admittable(consumer_id, reserved_bytes) + ): + # A full queue on a busy lane is also "busy here": the request + # could run on another replica, so say that instead of a plain + # capacity refusal the client must not redirect. + self._busy_refusals += 1 + raise ReadLaneBusy("read lane is busy; its queue is full") self._reject_if_unadmittable(consumer_id, reserved_bytes) entry = _Entry(consumer_id, reserved_bytes, self._sequence) self._sequence += 1 @@ -130,6 +152,14 @@ async def run( self._pending_by_consumer.get(consumer_id, 0) + 1 ) self._pending_bytes += reserved_bytes + if refuse_if_queued and not self._can_start(entry): + # Admissible, but it would wait behind active or earlier work. + # The caller can run it on another replica instead; withdraw + # before any waiter is notified, so nothing observes it. + self._waiting.remove(entry) + self._withdraw(entry) + self._busy_refusals += 1 + raise ReadLaneBusy("read lane is busy; the request would queue") self._condition.notify_all() deadline = ( loop.time() + wait_timeout_ms / 1_000 @@ -195,8 +225,21 @@ def stats(self) -> dict[str, int]: "rejected": self._rejected, "rejected_bytes": self._rejected_bytes, "queue_wait_timeouts": self._queue_wait_timeouts, + "busy_refusals": self._busy_refusals, } + def _admittable(self, consumer_id: str, reserved_bytes: int) -> bool: + """``_reject_if_unadmittable`` as a predicate, without counting.""" + + pending = sum(self._pending_by_consumer.values()) + return ( + reserved_bytes <= self.policy.max_pending_bytes + and pending < self._class_pending_limit(consumer_id) + and self._pending_bytes + reserved_bytes <= self.policy.max_pending_bytes + and self._pending_by_consumer.get(consumer_id, 0) + < self._consumer_pending_limit(consumer_id) + ) + def _reject_if_unadmittable(self, consumer_id: str, reserved_bytes: int) -> None: pending = sum(self._pending_by_consumer.values()) if reserved_bytes > self.policy.max_pending_bytes: diff --git a/qdl/query/service.py b/qdl/query/service.py index 4e348bdd..4f44c7ba 100644 --- a/qdl/query/service.py +++ b/qdl/query/service.py @@ -33,7 +33,7 @@ ) from qdl.query.lifecycle import BarLifecycle from qdl.query.cold_work import await_in_thread, cold_yield -from qdl.query.lanes import BoundedReadLane, ReadLanePolicy, ReadLaneRejected +from qdl.query.lanes import BoundedReadLane, ReadLaneBusy, ReadLanePolicy, ReadLaneRejected from qdl.query.entitlement import AccessPurpose, DataProduct, EntitlementPolicy from qdl.query.reference import ( ReferenceBatchRequirement, @@ -98,6 +98,7 @@ def __init__( instrument_uid: str | None = None, quality_state: str | None = None, diagnostics: dict[str, Any] | None = None, + reason: str | None = None, ) -> None: super().__init__(problem.detail) self.problem = problem @@ -105,6 +106,9 @@ def __init__( self.instrument_uid = instrument_uid self.quality_state = quality_state self.diagnostics = diagnostics + # A typed refusal reason a transport may expose next to the problem + # (REST: ``X-QDL-Refusal-Reason``); None for every ordinary refusal. + self.reason = reason @dataclass(frozen=True) @@ -244,6 +248,17 @@ class _LocalBatchAdmissionRejected(ReadLaneRejected): """A bounded canonical-cache batch lane cannot accept more queued work.""" +class _LocalBatchLaneBusy(_LocalBatchAdmissionRejected): + """The large-warmup lane would queue a request that opted into redirect.""" + + +# Typed reason for a large local warmup refused because this replica's +# single-active large-warmup lane is busy and the client opted into redirect +# (REST header ``X-QDL-Replica-Redirect: 1``). Replica-local admission only: +# it says nothing about the data, so another replica may serve the request. +REPLICA_LANE_BUSY_REASON = "REPLICA_LANE_BUSY" + + _SMALL_LOCAL_WARMUP_ROWS = 2 @@ -358,6 +373,7 @@ async def run( consumer_id: str | None = None, reserved_bytes: int = _LOCAL_HISTORY_BASE_BYTES, wait_timeout_ms: int | None = None, + refuse_if_queued: bool = False, ): if consumer_id is None: # Focused legacy callers did not carry an authenticated consumer @@ -371,7 +387,12 @@ async def run( consumer_id=consumer_id, reserved_bytes=reserved_bytes, wait_timeout_ms=wait_timeout_ms, + refuse_if_queued=refuse_if_queued, ) + except ReadLaneBusy as error: + raise _LocalBatchLaneBusy( + f"local canonical-cache batch admission {error}" + ) from error except ReadLaneRejected as error: raise _LocalBatchAdmissionRejected( f"local canonical-cache batch admission {error}" @@ -819,6 +840,7 @@ async def warmup_batch_completed_async( purpose: AccessPurpose, completion: Callable[[BatchQueryResult], Awaitable[_BatchCompletion]], request_id: str | None = None, + redirect_when_busy: bool = False, ) -> _BatchCompletion: """Run an internal response completion under a fully-local batch lease. @@ -827,6 +849,15 @@ async def warmup_batch_completed_async( its public JSON before this finite canonical-cache lane is released. That prevents a second large local HTTP response from overlapping the first response's CPU-heavy conversion after the cache read is done. + + ``redirect_when_busy`` (the client's opt-in, P18.4 G5): a fully-local + batch that needs the single-active large-warmup lane while that lane + would make it wait is refused at once with retryable ``RATE_LIMITED`` + (``retry_after_ms=0``, reason ``REPLICA_LANE_BUSY``) instead of + queuing, so the client can run it on an idle replica. Nothing is + reserved or read. Small warmups, mixed/provider batches and requests + without the opt-in keep the queue. Stage 100f: two 2,500-row reads + on one replica took 3.5 s and 7.0 s while the other replica was idle. """ request_id = request_id or self.request_id() executor_before = self.warmup_executor.stats() @@ -1052,6 +1083,9 @@ async def complete(executions) -> _BatchCompletion: if fully_local_batch: admission = self._local_batch_admission_for(local_requirements) + refuse_if_queued = bool(redirect_when_busy) and admission is not getattr( + self, "_small_local_warmup_admission", None + ) async def execute_whole_local_batch() -> _BatchCompletion: nonlocal prefetched_local_histories @@ -1071,6 +1105,7 @@ async def execute_whole_local_batch() -> _BatchCompletion: consumer_id=batch.consumer_id, reserved_bytes=local_reserved_bytes, wait_timeout_ms=local_admission_wait_ms, + refuse_if_queued=refuse_if_queued, ) ) try: @@ -1081,6 +1116,26 @@ async def execute_whole_local_batch() -> _BatchCompletion: # CPU budget. Keep the request-local lease until it drains. local_batch_task.add_done_callback(self._consume_detached_local_batch) raise + except _LocalBatchLaneBusy as error: + # Whole-request refusal: the batch never entered the lane, so + # the client may send it unchanged to another replica. + raise QueryServiceError( + QueryProblem( + CanonicalErrorCode.RATE_LIMITED, + f"{REPLICA_LANE_BUSY_REASON}: this replica's large-warmup lane is busy; " + "retry once on another replica without redirect " + f"({error})", + True, + 0, + ), + request_id=request_id, + instrument_uid=( + batch.requirements[0].instrument_uid + if len(batch.requirements) == 1 + else None + ), + reason=REPLICA_LANE_BUSY_REASON, + ) from error except _LocalBatchAdmissionRejected as error: # A bounded queued lane must fail typed before it can create # cache work or borrow an external-provider budget. diff --git a/qdl_sdk/CHANGELOG.md b/qdl_sdk/CHANGELOG.md index 71f76375..0a677e62 100644 --- a/qdl_sdk/CHANGELOG.md +++ b/qdl_sdk/CHANGELOG.md @@ -6,6 +6,17 @@ instance instead of always replica 0. A new keyword `start_index` pins the start (tests, diagnostics). Additive; no request or response change. +Replica redirect for a busy large-warmup lane (P18.4 G5): `ReplicatedRestQueryTransport` +`warmup`/`warmup_batch` send `X-QDL-Replica-Redirect: 1` on the first attempt when another +candidate replica remains. On exactly RATE_LIMITED with `X-QDL-Refusal-Reason: +REPLICA_LANE_BUSY` (Retry-After 0) the same request goes once to the next replica with the +header `0`, which queues there (no ping-pong); every other typed error stays authoritative and +a busy replica is not marked unhealthy. Each attempt is one HTTP request and consumes one unit +of the identity's per-minute quota. `DataLayerError.reason` (optional, default `None`) carries +the response's `X-QDL-Refusal-Reason`; `RestQueryTransport.warmup`/`warmup_batch` take an +optional `replica_redirect` keyword (None = no header, the pre-2.2.0 request). Servers without +the header support, and requests without it (SDK 2.1.0), queue exactly as before. + `MarketDataView.event_id` (optional, default `None`): lowercase hex of the canonical `EventEnvelope.event_id`, rendered by the v2.2.3 server on Query snapshot/latest/warmup rows and set by `market_data_view_from_stream` from the delivered envelope, so a diff --git a/qdl_sdk/errors.py b/qdl_sdk/errors.py index cbbda888..dc5bc132 100644 --- a/qdl_sdk/errors.py +++ b/qdl_sdk/errors.py @@ -10,6 +10,7 @@ def __init__( retryable: bool = False, retry_after_ms: int | None = None, diagnostics: dict | None = None, + reason: str | None = None, ) -> None: super().__init__(detail) self.code = code @@ -18,6 +19,9 @@ def __init__( self.retry_after_ms = retry_after_ms # Query's quality at the refusal, when the server sent it. self.diagnostics = diagnostics + # Typed refusal reason from the ``X-QDL-Refusal-Reason`` response + # header (2.2.0; e.g. ``REPLICA_LANE_BUSY``), None when absent. + self.reason = reason class ContinuityError(DataLayerError): diff --git a/qdl_sdk/transport.py b/qdl_sdk/transport.py index ec91fc8c..7fabb86c 100644 --- a/qdl_sdk/transport.py +++ b/qdl_sdk/transport.py @@ -19,6 +19,17 @@ from qdl_sdk.tls import WorkloadTlsConfig +REPLICA_REDIRECT_HEADER = "X-QDL-Replica-Redirect" +REFUSAL_REASON_HEADER = "X-QDL-Refusal-Reason" +REPLICA_LANE_BUSY = "REPLICA_LANE_BUSY" + + +def _redirect_headers(headers: dict[str, str], replica_redirect: bool | None) -> dict[str, str]: + if replica_redirect is not None: + headers[REPLICA_REDIRECT_HEADER] = "1" if replica_redirect else "0" + return headers + + class RestQueryTransport: def __init__( self, @@ -57,8 +68,19 @@ async def _read_request(self, method: str, path: str, **kwargs) -> httpx.Respons raise return await request(path, timeout=remaining, **kwargs) - async def warmup(self, requirement: DataRequirement, *, consumer_id: str) -> dict: - headers = await self._headers(requirement, consumer_id) + async def warmup( + self, + requirement: DataRequirement, + *, + consumer_id: str, + replica_redirect: bool | None = None, + ) -> dict: + """``replica_redirect``: None sends no header (queue, as before 2.2.0); + True asks a busy large-warmup lane to refuse with REPLICA_LANE_BUSY; + False explicitly queues (the redirected retry).""" + headers = _redirect_headers( + await self._headers(requirement, consumer_id), replica_redirect + ) response = await self._read_request("get", f"/v2/market-data/{requirement.instrument_uid}/warmup", params=requirement.query_params(), @@ -72,6 +94,7 @@ async def warmup_batch( *, consumer_id: str, require_all: bool, + replica_redirect: bool | None = None, ) -> dict: values = tuple(requirements) if not values: @@ -79,7 +102,10 @@ async def warmup_batch( grades = {item.consumer_grade for item in values} if len(grades) != 1: raise ValueError("one warmup batch cannot mix consumer grades") - headers = await self._identity_headers(next(iter(grades)), consumer_id) + headers = _redirect_headers( + await self._identity_headers(next(iter(grades)), consumer_id), + replica_redirect, + ) response = await self._read_request("post", "/v2/market-data/warmup:batch", json={ @@ -234,6 +260,7 @@ def _decode(response: httpx.Response) -> dict: if isinstance(payload.get("diagnostics"), dict) else None ), + reason=response.headers.get(REFUSAL_REASON_HEADER) or None, ) @@ -251,6 +278,16 @@ class ReplicatedRestQueryTransport: response exists. A data-quality, entitlement or schema error remains the selected replica's authoritative result; trying another reader to find a green answer would weaken the public contract. + + One replica-local admission signal is the exception (2.2.0, P18.4 G5). + ``warmup``/``warmup_batch`` ask the first replica to refuse instead of + queuing behind its single-active large-warmup lane + (``X-QDL-Replica-Redirect: 1``). On exactly that refusal - RATE_LIMITED + with reason ``REPLICA_LANE_BUSY``, which says nothing about the data - the + same request goes once to the next replica with the header ``0``, which + queues there, so a redirect never ping-pongs. The header is sent only + when another candidate remains. Each attempt is a separate HTTP request + and consumes one unit of the identity's per-minute request quota. """ def __init__( @@ -285,7 +322,9 @@ def __init__( self._next = random.randrange(len(values)) if start_index is None else start_index async def warmup(self, requirement: DataRequirement, *, consumer_id: str) -> dict: - return await self._read("warmup", requirement, consumer_id=consumer_id) + return await self._read( + "warmup", requirement, consumer_id=consumer_id, _redirectable=True + ) async def warmup_batch( self, @@ -299,6 +338,7 @@ async def warmup_batch( requirements, consumer_id=consumer_id, require_all=require_all, + _redirectable=True, ) async def reference_batch( @@ -368,13 +408,32 @@ def stats(self) -> tuple[dict[str, int | str], ...]: for replica, health in zip(self.replicas, self._health, strict=True) ) - async def _read(self, method: str, *args, **kwargs) -> dict: + async def _read( + self, method: str, *args, _redirectable: bool = False, **kwargs + ) -> dict: selected = self._candidate_indexes() last_error: httpx.TransportError | None = None - for index in selected: + redirected = False + for position, index in enumerate(selected): replica = self.replicas[index] + call_kwargs = kwargs + if _redirectable and (redirected or position + 1 < len(selected)): + # First attempt with a fallback left: may be refused as busy. + # After a busy refusal: explicitly queue (no ping-pong). + call_kwargs = {**kwargs, "replica_redirect": not redirected} try: - result = await getattr(replica, method)(*args, **kwargs) + result = await getattr(replica, method)(*args, **call_kwargs) + except DataLayerError as error: + if ( + call_kwargs.get("replica_redirect") is True + and error.code == "RATE_LIMITED" + and error.reason == REPLICA_LANE_BUSY + ): + # Replica-local admission, not a data outcome: the replica + # is healthy, so its health record is left unchanged. + redirected = True + continue + raise except httpx.TransportError as error: last_error = error health = self._health[index] diff --git a/tests/test_phaseb_stable_release.py b/tests/test_phaseb_stable_release.py index 98a6ebaa..bf5bdd1f 100644 --- a/tests/test_phaseb_stable_release.py +++ b/tests/test_phaseb_stable_release.py @@ -317,7 +317,7 @@ def test_service_openapi_and_sdk_versions_are_explicit(self): ) generated = build_openapi() self.assertEqual(package["project"]["version"], "2.0.0") - self.assertEqual(qdl_sdk.__version__, "2.1.0") + self.assertEqual(qdl_sdk.__version__, "2.2.0") self.assertEqual(generated["info"]["version"], "2.0.0") self.assertEqual(snapshot, generated) # ``reference:batch`` is a governed V2 public path in the checked-in diff --git a/tests/test_qdl_sdk_replica_read_transport.py b/tests/test_qdl_sdk_replica_read_transport.py index 9f2915b9..0f6abdd8 100644 --- a/tests/test_qdl_sdk_replica_read_transport.py +++ b/tests/test_qdl_sdk_replica_read_transport.py @@ -1,5 +1,6 @@ from __future__ import annotations +from types import SimpleNamespace import unittest import httpx @@ -89,6 +90,159 @@ async def test_close_closes_every_owned_replica(self): self.assertTrue(second.closed) +def _lane_busy() -> DataLayerError: + return DataLayerError( + "RATE_LIMITED", "REPLICA_LANE_BUSY: lane busy", retryable=True, + retry_after_ms=0, reason="REPLICA_LANE_BUSY", + ) + + +class _WarmReplica(_Replica): + """Records the redirect opt-in each warmup attempt carried.""" + + def __init__(self, base_url: str, outcomes) -> None: + super().__init__(base_url, outcomes) + self.redirects: list[object] = [] + self.args: list[tuple] = [] + + async def _attempt(self, args, replica_redirect): + self.calls += 1 + self.redirects.append(replica_redirect) + self.args.append(args) + outcome = self.outcomes.pop(0) + if isinstance(outcome, BaseException): + raise outcome + return outcome + + async def warmup(self, requirement, *, consumer_id: str, replica_redirect=None): + return await self._attempt((requirement, consumer_id), replica_redirect) + + async def warmup_batch(self, requirements, *, consumer_id: str, require_all: bool, + replica_redirect=None): + return await self._attempt((requirements, consumer_id, require_all), replica_redirect) + + +class ReplicaLaneBusyRedirectTests(unittest.IsolatedAsyncioTestCase): + """2.2.0 (P18.4 G5): one retry on the next replica, only for REPLICA_LANE_BUSY.""" + + async def test_lane_busy_retries_once_on_the_next_replica_with_redirect_off(self): + for method, args, kwargs in ( + ("warmup", ("req",), {"consumer_id": "alpha"}), + ("warmup_batch", (("r1", "r2"),), {"consumer_id": "alpha", "require_all": True}), + ): + first = _WarmReplica("https://query-a", (_lane_busy(),)) + second = _WarmReplica("https://query-b", ({"replica": "b"},)) + transport = ReplicatedRestQueryTransport((first, second), start_index=0) + result = await getattr(transport, method)(*args, **kwargs) + self.assertEqual(result, {"replica": "b"}) + self.assertEqual((first.redirects, second.redirects), ([True], [False])) + # The same request (requirements, identity) goes to the second replica. + self.assertEqual(first.args, second.args) + # A busy lane is not a replica failure. + self.assertEqual([item["failures"] for item in transport.stats()], [0, 0]) + + async def test_both_replicas_busy_queues_on_the_second_no_ping_pong(self): + first = _WarmReplica("https://query-a", (_lane_busy(), {"replica": "a"})) + second = _WarmReplica("https://query-b", ({"replica": "b-queued"},)) + transport = ReplicatedRestQueryTransport((first, second), start_index=0) + # The second replica is busy too, but redirect=0 makes it queue there. + self.assertEqual(await transport.warmup("req", consumer_id="alpha"), {"replica": "b-queued"}) + self.assertEqual((first.calls, second.calls), (1, 1)) + self.assertEqual(second.redirects, [False]) + + async def test_a_second_busy_refusal_is_not_redirected_again(self): + # A server must queue on redirect=0; if a refusal still comes back it + # is returned to the caller, never sent around again. + first = _WarmReplica("https://query-a", (_lane_busy(),)) + second = _WarmReplica("https://query-b", (_lane_busy(),)) + transport = ReplicatedRestQueryTransport((first, second), start_index=0) + with self.assertRaises(DataLayerError) as raised: + await transport.warmup("req", consumer_id="alpha") + self.assertEqual(raised.exception.reason, "REPLICA_LANE_BUSY") + self.assertEqual((first.calls, second.calls), (1, 1)) + + async def test_other_typed_errors_stay_authoritative(self): + for error in ( + DataLayerError("RATE_LIMITED", "consumer request quota is exhausted", retryable=True), + DataLayerError("RATE_LIMITED", "read lane is full", retryable=True, reason="OTHER"), + DataLayerError("DATA_STALE", "REPLICA_LANE_BUSY", retryable=True, reason="REPLICA_LANE_BUSY"), + DataLayerError("SOURCE_NOT_ALLOWED", "entitlement", retryable=False), + ): + first = _WarmReplica("https://query-a", (error,)) + second = _WarmReplica("https://query-b", ({"replica": "b"},)) + transport = ReplicatedRestQueryTransport((first, second), start_index=0) + with self.assertRaises(DataLayerError): + await transport.warmup("req", consumer_id="alpha") + self.assertEqual((first.calls, second.calls), (1, 0), error.detail) + + async def test_no_opt_in_without_a_fallback_candidate(self): + # Transport failure consumed the first candidate: the last attempt + # must queue (no header), never be refused as busy with nowhere to go. + first = _WarmReplica("https://query-a", (_connect_error(),)) + second = _WarmReplica("https://query-b", ({"replica": "b"},)) + transport = ReplicatedRestQueryTransport((first, second), start_index=0) + self.assertEqual(await transport.warmup("req", consumer_id="alpha"), {"replica": "b"}) + self.assertEqual((first.redirects, second.redirects), ([True], [None])) + single = _WarmReplica("https://query-c", ({"replica": "c"},)) + other = _WarmReplica("https://query-d", ()) + limited = ReplicatedRestQueryTransport((single, other), max_attempts=1, start_index=0) + self.assertEqual(await limited.warmup("req", consumer_id="alpha"), {"replica": "c"}) + self.assertEqual(single.redirects, [None]) + + async def test_snapshot_never_carries_the_opt_in(self): + first = _Replica("https://query-a", ({"replica": "a"},)) + second = _Replica("https://query-b", ()) + transport = ReplicatedRestQueryTransport((first, second), start_index=0) + # _Replica.snapshot takes no replica_redirect keyword; a TypeError would fail here. + self.assertEqual(await transport.snapshot("req", consumer_id="alpha"), {"replica": "a"}) + + +class RestRedirectHeaderTests(unittest.IsolatedAsyncioTestCase): + async def test_header_values_and_refusal_reason_decode(self): + from qdl_sdk.models import Grade + from qdl_sdk.transport import RestQueryTransport + + seen: list[str | None] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request.headers.get("X-QDL-Replica-Redirect")) + return httpx.Response( + 429, + json={"code": "RATE_LIMITED", "detail": "REPLICA_LANE_BUSY: busy", + "retryable": True, "retry_after_ms": 0}, + headers={"X-QDL-Refusal-Reason": "REPLICA_LANE_BUSY", "Retry-After": "0"}, + ) + + class Credential: + async def get_token(self): + return "token" + + client = httpx.AsyncClient(base_url="https://qdl", transport=httpx.MockTransport(handler)) + rest = RestQueryTransport("https://qdl", client=client, credential_provider=Credential()) + requirement = SimpleNamespace( + instrument_uid="inst", consumer_grade=Grade.ALPHA, + query_params=lambda: {"feed": "BAR"}, to_mapping=lambda: {"instrument_uid": "inst"}, + ) + for value in (None, True, False): + with self.assertRaises(DataLayerError) as raised: + await rest.warmup(requirement, consumer_id="alpha", replica_redirect=value) + self.assertEqual(raised.exception.reason, "REPLICA_LANE_BUSY") + self.assertEqual(raised.exception.retry_after_ms, 0) + with self.assertRaises(DataLayerError): + await rest.warmup_batch((requirement,), consumer_id="alpha", require_all=True, + replica_redirect=True) + self.assertEqual(seen, [None, "1", "0", "1"]) + await client.aclose() + + def test_reason_is_none_without_the_header(self): + from qdl_sdk.transport import RestQueryTransport + + with self.assertRaises(DataLayerError) as raised: + RestQueryTransport._decode(httpx.Response( + 429, json={"code": "RATE_LIMITED", "detail": "quota", "retryable": True})) + self.assertIsNone(raised.exception.reason) + + class RefusalDiagnosticsDecodeTests(unittest.TestCase): def test_a_refusal_keeps_the_servers_quality_diagnostics(self) -> None: from qdl_sdk.transport import RestQueryTransport diff --git a/tests/test_query_replica_lane_redirect.py b/tests/test_query_replica_lane_redirect.py new file mode 100644 index 00000000..4ab5bb85 --- /dev/null +++ b/tests/test_query_replica_lane_redirect.py @@ -0,0 +1,287 @@ +"""Opt-in replica redirect for a busy large-warmup lane (P18.4 G5, stage 100f). + +Two 2,500-row cold warmups that landed on one Query replica ran one after the +other on its single-active large-warmup lane (3.5 s and 7.0 s) while the other +replica was idle. A client that sends ``X-QDL-Replica-Redirect: 1`` is refused +at once with retryable RATE_LIMITED / REPLICA_LANE_BUSY instead of queuing; +without the header (older clients) the request still queues. +""" + +from __future__ import annotations + +import asyncio +import importlib +import threading +from types import SimpleNamespace +import unittest + +from fastapi.responses import JSONResponse +from fastapi.testclient import TestClient + +from qdl.api_v2 import create_v2_app +from qdl.query import ( + AccessPurpose, + BatchRequirement, + CanonicalErrorCode, + ConsumerGrade, + DataRequirement, + FeedType, + QueryProblem, + V2QueryService, +) +from qdl.query.lanes import BoundedReadLane, ReadLaneBusy, ReadLaneRejected +from qdl.query.service import ( + REPLICA_LANE_BUSY_REASON, + QueryServiceError, + _LocalBatchAdmission, + _queued_local_batch_lane_policy, +) +from qdl.warmup import WarmupSpecification +from qdl.warmup.executor import WarmupExecution + +# ``qdl.api_v2.router`` the attribute is the APIRouter; the module holds the dependencies. +router_module = importlib.import_module("qdl.api_v2.router") + + +class LaneRefuseIfQueuedTests(unittest.IsolatedAsyncioTestCase): + async def test_idle_lane_runs_busy_lane_refuses_without_reserving(self): + lane = BoundedReadLane(_queued_local_batch_lane_policy()) + self.assertEqual( + await lane.run(_ok, consumer_id="a", reserved_bytes=1, refuse_if_queued=True), "ok" + ) + hold = asyncio.Event() + first = asyncio.create_task(lane.run(hold.wait, consumer_id="a", reserved_bytes=1)) + while lane.stats()["active"] < 1: + await asyncio.sleep(0) + with self.assertRaises(ReadLaneBusy): + await lane.run(_ok, consumer_id="b", reserved_bytes=1, refuse_if_queued=True) + stats = lane.stats() + self.assertEqual((stats["pending"], stats["pending_bytes"]), (1, 1)) + self.assertEqual((stats["busy_refusals"], stats["rejected"]), (1, 0)) + # Without the flag the same request queues and completes (unchanged). + queued = asyncio.create_task(lane.run(_ok, consumer_id="b", reserved_bytes=1)) + while lane.stats()["pending"] < 2: + await asyncio.sleep(0) + hold.set() + self.assertEqual(await queued, "ok") + await first + + async def test_full_queue_on_a_busy_lane_is_busy_not_a_plain_refusal(self): + lane = BoundedReadLane(_queued_local_batch_lane_policy()) + hold = asyncio.Event() + held = [asyncio.create_task(lane.run(hold.wait, consumer_id="a", reserved_bytes=1)) + for _ in range(8)] + while lane.stats()["pending"] < 8: + await asyncio.sleep(0) + with self.assertRaises(ReadLaneBusy): + await lane.run(_ok, consumer_id="a", reserved_bytes=1, refuse_if_queued=True) + with self.assertRaises(ReadLaneRejected) as plain: + await lane.run(_ok, consumer_id="a", reserved_bytes=1) + self.assertNotIsInstance(plain.exception, ReadLaneBusy) + hold.set() + await asyncio.gather(*held) + + +async def _ok(): + return "ok" + + +def _service(*, small_lane: bool = True): + class Backend: + def __init__(self): + self.large_started = threading.Event() + self.gate = threading.Event() + self.calls = 0 + + @staticmethod + def warmup_is_local(_requirement): + return True + + def history_many(self, requirements): + self.calls += 1 + if any(_rows(item) > 2 for item in requirements): + self.large_started.set() + self.gate.wait(5) + return {requirement: "history" for requirement in requirements} + + class Executor: + @staticmethod + def stats(): + return {} + + async def execute(self, items, *, work, identity, provider, deadline_ms): + del identity, provider, deadline_ms + return tuple( + [WarmupExecution(item, await work(item), None, 1, False, 0.0) for item in items] + ) + + class Service(V2QueryService): + def __init__(self): + self.backend = Backend() + self.instruments = SimpleNamespace() + self.warmup_executor = Executor() + self.last_batch_evidence = {} + self._small_local_warmup_lane = small_lane + self._queued_local_batch_lane = True + self._local_batch_admission = _LocalBatchAdmission( + policy=_queued_local_batch_lane_policy() + ) + + def _warmup_from_history(self, requirement, history, *, purpose, request_id): + del requirement, purpose, request_id + return history + + return Service() + + +def _requirement(uid: str, rows: int): + return DataRequirement( + instrument_uid=uid, feed=FeedType.BAR, consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", interval="1m", + warmup=WarmupSpecification.for_rows(rows, deadline_ms=5_000), + ) + + +def _rows(requirement) -> int: + specification = requirement.warmup_specification + return 1 if specification is None or specification.rows is None else specification.rows + + +async def _warm(service, consumer, *requirements, **kwargs): + async def completion(result): + return result + return await service.warmup_batch_completed_async( + BatchRequirement(consumer_id=consumer, requirements=requirements), + purpose=AccessPurpose.INTERNAL_ALPHA, completion=completion, **kwargs, + ) + + +class ServiceLaneBusyRedirectTests(unittest.IsolatedAsyncioTestCase): + async def _hold_large(self, service): + large = asyncio.create_task(_warm(service, "alpha.okx", _requirement("okx", 2500))) + await asyncio.wait_for(asyncio.to_thread(service.backend.large_started.wait, 2), 3) + return large + + async def test_opt_in_large_warmup_is_refused_at_once_typed_and_retryable(self): + service = _service() + large = await self._hold_large(service) + with self.assertRaises(QueryServiceError) as raised: + await asyncio.wait_for( + _warm(service, "alpha.binance", _requirement("bin", 2500), redirect_when_busy=True), 1 + ) + error = raised.exception + self.assertEqual(error.problem.code, CanonicalErrorCode.RATE_LIMITED) + self.assertTrue(error.problem.retryable) + self.assertEqual(error.problem.retry_after_ms, 0) + self.assertEqual(error.reason, REPLICA_LANE_BUSY_REASON) + self.assertTrue(error.problem.detail.startswith("REPLICA_LANE_BUSY:")) + self.assertEqual(error.instrument_uid, "bin") + self.assertEqual(service.backend.calls, 1) # nothing read for the refused one + lane = service._local_batch_admission_for((_requirement("bin", 2500),)) + self.assertEqual(lane.stats()["pending"], 1) + service.backend.gate.set() + self.assertEqual([item.status for item in (await large).results], ["OK"]) + + async def test_without_opt_in_the_request_queues_as_before(self): + service = _service() + large = await self._hold_large(service) + queued = asyncio.create_task(_warm(service, "alpha.binance", _requirement("bin", 2500))) + lane = service._local_batch_admission_for((_requirement("bin", 2500),)) + while lane.stats()["pending"] < 2: + await asyncio.sleep(0) + service.backend.gate.set() + results = await asyncio.gather(large, queued) + self.assertEqual([r.results[0].status for r in results], ["OK", "OK"]) + self.assertEqual(lane.stats()["busy_refusals"], 0) + + async def test_opt_in_on_an_idle_lane_and_small_warmups_are_served(self): + service = _service() + idle = await _warm(service, "alpha.binance", _requirement("bin", 2500), redirect_when_busy=True) + self.assertEqual([item.status for item in idle.results], ["OK"]) + service.backend.large_started.clear() + large = await self._hold_large(service) + small = await asyncio.wait_for( + _warm(service, "alpha.binance", _requirement("handoff", 1), redirect_when_busy=True), 2 + ) + self.assertEqual([item.status for item in small.results], ["OK"]) + service.backend.gate.set() + await large + + +class _Access(SimpleNamespace): + def __getattr__(self, name): + if name.startswith("require_"): + return lambda *args, **kwargs: None + raise AttributeError(name) + + +class _RecordingService: + """Records the opt-in the router passes; refuses like Query when it is set.""" + + def __init__(self): + self.calls: list[dict] = [] + + @staticmethod + def request_id(): + return "req-1" + + async def warmup_batch_completed_async(self, batch, *, purpose, completion, **kwargs): + del purpose, completion + self.calls.append(kwargs) + if kwargs.get("redirect_when_busy"): + raise QueryServiceError( + QueryProblem(CanonicalErrorCode.RATE_LIMITED, + "REPLICA_LANE_BUSY: this replica's large-warmup lane is busy", True, 0), + request_id="req-1", instrument_uid=batch.requirements[0].instrument_uid, + reason=REPLICA_LANE_BUSY_REASON, + ) + return JSONResponse({"queued": True}) + + +class RouterReplicaRedirectHeaderTests(unittest.TestCase): + def setUp(self): + self.service = _RecordingService() + app = create_v2_app(self.service) # type: ignore[arg-type] + app.dependency_overrides[router_module._data_access] = lambda: _Access(consumer_id="alpha") + self.client = TestClient(app) + + def _get(self, headers): + return self.client.get( + "/v2/market-data/inst-1/warmup", + params={"feed": "BAR", "source_policy_id": "crypto_primary_v2", + "interval": "1m", "limit": 2500}, + headers={"X-QDL-Purpose": "INTERNAL_ALPHA", **headers}, + ) + + def test_header_one_refuses_with_typed_reason_and_retry_after_zero(self): + response = self._get({"X-QDL-Replica-Redirect": "1"}) + self.assertEqual(response.status_code, 429, response.text) + self.assertEqual(response.headers["X-QDL-Refusal-Reason"], "REPLICA_LANE_BUSY") + self.assertEqual(response.headers["Retry-After"], "0") + body = response.json() + self.assertEqual((body["code"], body["retryable"], body["retry_after_ms"]), + ("RATE_LIMITED", True, 0)) + self.assertEqual(self.service.calls, [{"redirect_when_busy": True}]) + + def test_absent_zero_or_other_values_queue_exactly_as_before(self): + for headers in ({}, {"X-QDL-Replica-Redirect": "0"}, {"X-QDL-Replica-Redirect": "yes"}): + response = self._get(headers) + self.assertEqual(response.status_code, 200, response.text) + self.assertNotIn("X-QDL-Refusal-Reason", response.headers) + self.assertEqual(self.service.calls, [{}, {}, {}]) + + def test_ordinary_refusal_carries_no_reason_header(self): + async def refuse(batch, *, purpose, completion, **kwargs): + raise QueryServiceError( + QueryProblem(CanonicalErrorCode.RATE_LIMITED, "read lane is full", True), + request_id="req-2", + ) + self.service.warmup_batch_completed_async = refuse + response = self._get({"X-QDL-Replica-Redirect": "1"}) + self.assertEqual(response.status_code, 429) + self.assertNotIn("X-QDL-Refusal-Reason", response.headers) + self.assertNotIn("Retry-After", response.headers) + + +if __name__ == "__main__": + unittest.main() From 6674d0b65706833f8f56a604ab75ac94fdb4f742 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 07:46:37 +0000 Subject: [PATCH 105/147] fix(okx): move WebSocket URLs off port 8443 before the 2026-10-31 cutover OKX stops accepting WebSocket connections on port 8443 on 2026-10-31 (changelog "WebSocket port 8443 discontinuation", updated 2026-09-30). - qdl/adapters/okx_endpoints.py is the single owner of the OKX WS URLs and refuses :8443; client, production catalog and L2 demand import it. - The acquisition plan drops the port on all 212 OKX URLs and keeps revision 19, so bar edge checkpoints are not stranded (C.19/C.32). StableAcquisitionPlan.identity_digest() defines identity without transport endpoints; the old and new plans are identical under it. - scripts/migrate_okx_ws_port.py rewrites the two URL fields of a live OKX ingestor runtime config (dry run by default, backup, atomic replace, identity check, rollback). - Packet: upgrade/evidence/p184-okx-port-migration-packet.md. Tests: 17 new; 24 related modules 325 OK, 1 skip. --- config/v2/stable-acquisition-bindings.yaml | 424 +++++++++--------- qdl/adapters/okx/client.py | 4 +- qdl/adapters/okx_endpoints.py | 74 +++ qdl/runtime/l2_demand.py | 4 +- qdl/runtime/production_catalog.py | 6 +- qdl/runtime/stable_deployment.py | 41 +- scripts/execution_view_diagnostics.py | 2 +- scripts/migrate_okx_ws_port.py | 201 +++++++++ scripts/phase114_l2_real_provider_capture.py | 2 +- scripts/phasea_real_provider_certification.py | 4 +- tests/test_crypto_demand_manifest.py | 4 +- tests/test_dlv2_r127_binance_ws_route.py | 6 +- tests/test_okx_ws_port_migration.py | 285 ++++++++++++ tests/test_phase112_provider_admission.py | 2 +- tests/test_phase112_universal_realtime.py | 2 +- ...st_phase115c_native_bar_materialization.py | 4 +- tests/test_phaseb_stable_deployment.py | 4 +- tests/test_phasec36_liquid_crypto_features.py | 2 +- ...X_MARKET_DATA_V5_GUIDE_QUANT_DATA_LAYER.md | 12 +- .../p184-okx-port-migration-packet.md | 163 +++++++ 20 files changed, 1005 insertions(+), 241 deletions(-) create mode 100644 qdl/adapters/okx_endpoints.py create mode 100644 scripts/migrate_okx_ws_port.py create mode 100644 tests/test_okx_ws_port_migration.py create mode 100644 upgrade/evidence/p184-okx-port-migration-packet.md diff --git a/config/v2/stable-acquisition-bindings.yaml b/config/v2/stable-acquisition-bindings.yaml index 42c086b6..43f85d3e 100644 --- a/config/v2/stable-acquisition-bindings.yaml +++ b/config/v2/stable-acquisition-bindings.yaml @@ -3323,8 +3323,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -3337,8 +3337,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -3351,8 +3351,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -3365,8 +3365,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -3379,8 +3379,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -3393,8 +3393,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -3407,8 +3407,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -3421,8 +3421,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -3435,8 +3435,8 @@ bindings: provider_kind: okx_bar native_channel: candle1m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business enabled: false market_websocket_url: null - binding_id: okx-spot-btcusdt-quote @@ -3445,8 +3445,8 @@ bindings: provider_kind: okx_bbo native_channel: bbo-tbt sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business enabled: false market_websocket_url: null - binding_id: okx-spot-btcusdt-trade @@ -3455,8 +3455,8 @@ bindings: provider_kind: okx_trade native_channel: trades sequence_policy: MONOTONIC - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business enabled: false market_websocket_url: null - binding_id: okx-swap-0g-usdt-swap-bar-1d @@ -3870,8 +3870,8 @@ bindings: provider_kind: okx_bar native_channel: candle12Hutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-15m mode: RUST_NATIVE @@ -3879,8 +3879,8 @@ bindings: provider_kind: okx_bar native_channel: candle15m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-1d mode: RUST_NATIVE @@ -3888,8 +3888,8 @@ bindings: provider_kind: okx_bar native_channel: candle1Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-1h mode: RUST_NATIVE @@ -3897,8 +3897,8 @@ bindings: provider_kind: okx_bar native_channel: candle1H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-1m mode: RUST_NATIVE @@ -3906,8 +3906,8 @@ bindings: provider_kind: okx_bar native_channel: candle1m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-1w mode: RUST_NATIVE @@ -3915,8 +3915,8 @@ bindings: provider_kind: okx_bar native_channel: candle1Wutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-2d mode: RUST_NATIVE @@ -3924,8 +3924,8 @@ bindings: provider_kind: okx_bar native_channel: candle2Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-2h mode: RUST_NATIVE @@ -3933,8 +3933,8 @@ bindings: provider_kind: okx_bar native_channel: candle2H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-30m mode: RUST_NATIVE @@ -3942,8 +3942,8 @@ bindings: provider_kind: okx_bar native_channel: candle30m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-3d mode: RUST_NATIVE @@ -3951,8 +3951,8 @@ bindings: provider_kind: okx_bar native_channel: candle3Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-3m mode: RUST_NATIVE @@ -3960,8 +3960,8 @@ bindings: provider_kind: okx_bar native_channel: candle3m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-4h mode: RUST_NATIVE @@ -3969,8 +3969,8 @@ bindings: provider_kind: okx_bar native_channel: candle4H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-5m mode: RUST_NATIVE @@ -3978,8 +3978,8 @@ bindings: provider_kind: okx_bar native_channel: candle5m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-6h mode: RUST_NATIVE @@ -3987,8 +3987,8 @@ bindings: provider_kind: okx_bar native_channel: candle6Hutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-book_delta mode: RUST_NATIVE @@ -3996,8 +3996,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -4011,8 +4011,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -4026,8 +4026,8 @@ bindings: provider_kind: okx_mark_index native_channel: mark-price sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null mark_index: provider_protocol: OKX_MARK_INDEX @@ -4041,8 +4041,8 @@ bindings: provider_kind: okx_bbo native_channel: bbo-tbt sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-trade mode: RUST_NATIVE @@ -4050,8 +4050,8 @@ bindings: provider_kind: okx_trade native_channel: trades sequence_policy: MONOTONIC - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-bome-usdt-swap-bar-1d mode: PYTHON_REST @@ -4104,8 +4104,8 @@ bindings: provider_kind: okx_bar native_channel: candle12Hutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-bar-15m mode: RUST_NATIVE @@ -4113,8 +4113,8 @@ bindings: provider_kind: okx_bar native_channel: candle15m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-bar-1d mode: RUST_NATIVE @@ -4122,8 +4122,8 @@ bindings: provider_kind: okx_bar native_channel: candle1Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-bar-1h mode: RUST_NATIVE @@ -4131,8 +4131,8 @@ bindings: provider_kind: okx_bar native_channel: candle1H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-bar-1w mode: RUST_NATIVE @@ -4140,8 +4140,8 @@ bindings: provider_kind: okx_bar native_channel: candle1Wutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-bar-2d mode: RUST_NATIVE @@ -4149,8 +4149,8 @@ bindings: provider_kind: okx_bar native_channel: candle2Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-bar-2h mode: RUST_NATIVE @@ -4158,8 +4158,8 @@ bindings: provider_kind: okx_bar native_channel: candle2H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-bar-30m mode: RUST_NATIVE @@ -4167,8 +4167,8 @@ bindings: provider_kind: okx_bar native_channel: candle30m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-bar-3d mode: RUST_NATIVE @@ -4176,8 +4176,8 @@ bindings: provider_kind: okx_bar native_channel: candle3Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-bar-3m mode: RUST_NATIVE @@ -4185,8 +4185,8 @@ bindings: provider_kind: okx_bar native_channel: candle3m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-bar-4h mode: RUST_NATIVE @@ -4194,8 +4194,8 @@ bindings: provider_kind: okx_bar native_channel: candle4H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-bar-5m mode: RUST_NATIVE @@ -4203,8 +4203,8 @@ bindings: provider_kind: okx_bar native_channel: candle5m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-bar-6h mode: RUST_NATIVE @@ -4212,8 +4212,8 @@ bindings: provider_kind: okx_bar native_channel: candle6Hutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-book_delta mode: RUST_NATIVE @@ -4221,8 +4221,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -4236,8 +4236,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -4251,8 +4251,8 @@ bindings: provider_kind: okx_mark_index native_channel: mark-price sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null mark_index: provider_protocol: OKX_MARK_INDEX @@ -4266,8 +4266,8 @@ bindings: provider_kind: okx_bar native_channel: candle1m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btcusdt-quote mode: RUST_NATIVE @@ -4275,8 +4275,8 @@ bindings: provider_kind: okx_bbo native_channel: bbo-tbt sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-btcusdt-trade mode: RUST_NATIVE @@ -4284,8 +4284,8 @@ bindings: provider_kind: okx_trade native_channel: trades sequence_policy: MONOTONIC - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-cap-usdt-swap-bar-1d mode: PYTHON_REST @@ -4392,8 +4392,8 @@ bindings: provider_kind: okx_bar native_channel: candle12Hutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-15m mode: RUST_NATIVE @@ -4401,8 +4401,8 @@ bindings: provider_kind: okx_bar native_channel: candle15m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-1d mode: RUST_NATIVE @@ -4410,8 +4410,8 @@ bindings: provider_kind: okx_bar native_channel: candle1Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-1h mode: RUST_NATIVE @@ -4419,8 +4419,8 @@ bindings: provider_kind: okx_bar native_channel: candle1H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-1m mode: RUST_NATIVE @@ -4428,8 +4428,8 @@ bindings: provider_kind: okx_bar native_channel: candle1m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-1w mode: RUST_NATIVE @@ -4437,8 +4437,8 @@ bindings: provider_kind: okx_bar native_channel: candle1Wutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-2d mode: RUST_NATIVE @@ -4446,8 +4446,8 @@ bindings: provider_kind: okx_bar native_channel: candle2Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-2h mode: RUST_NATIVE @@ -4455,8 +4455,8 @@ bindings: provider_kind: okx_bar native_channel: candle2H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-30m mode: RUST_NATIVE @@ -4464,8 +4464,8 @@ bindings: provider_kind: okx_bar native_channel: candle30m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-3d mode: RUST_NATIVE @@ -4473,8 +4473,8 @@ bindings: provider_kind: okx_bar native_channel: candle3Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-3m mode: RUST_NATIVE @@ -4482,8 +4482,8 @@ bindings: provider_kind: okx_bar native_channel: candle3m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-4h mode: RUST_NATIVE @@ -4491,8 +4491,8 @@ bindings: provider_kind: okx_bar native_channel: candle4H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-5m mode: RUST_NATIVE @@ -4500,8 +4500,8 @@ bindings: provider_kind: okx_bar native_channel: candle5m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-bar-6h mode: RUST_NATIVE @@ -4509,8 +4509,8 @@ bindings: provider_kind: okx_bar native_channel: candle6Hutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-book_delta mode: RUST_NATIVE @@ -4518,8 +4518,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -4533,8 +4533,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -4548,8 +4548,8 @@ bindings: provider_kind: okx_mark_index native_channel: mark-price sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null mark_index: provider_protocol: OKX_MARK_INDEX @@ -4563,8 +4563,8 @@ bindings: provider_kind: okx_bbo native_channel: bbo-tbt sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-trade mode: RUST_NATIVE @@ -4572,8 +4572,8 @@ bindings: provider_kind: okx_trade native_channel: trades sequence_policy: MONOTONIC - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-dood-usdt-swap-bar-1d mode: PYTHON_REST @@ -4707,8 +4707,8 @@ bindings: provider_kind: okx_bar native_channel: candle12Hutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-15m mode: RUST_NATIVE @@ -4716,8 +4716,8 @@ bindings: provider_kind: okx_bar native_channel: candle15m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-1d mode: RUST_NATIVE @@ -4725,8 +4725,8 @@ bindings: provider_kind: okx_bar native_channel: candle1Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-1h mode: RUST_NATIVE @@ -4734,8 +4734,8 @@ bindings: provider_kind: okx_bar native_channel: candle1H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-1m mode: RUST_NATIVE @@ -4743,8 +4743,8 @@ bindings: provider_kind: okx_bar native_channel: candle1m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-1w mode: RUST_NATIVE @@ -4752,8 +4752,8 @@ bindings: provider_kind: okx_bar native_channel: candle1Wutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-2d mode: RUST_NATIVE @@ -4761,8 +4761,8 @@ bindings: provider_kind: okx_bar native_channel: candle2Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-2h mode: RUST_NATIVE @@ -4770,8 +4770,8 @@ bindings: provider_kind: okx_bar native_channel: candle2H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-30m mode: RUST_NATIVE @@ -4779,8 +4779,8 @@ bindings: provider_kind: okx_bar native_channel: candle30m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-3d mode: RUST_NATIVE @@ -4788,8 +4788,8 @@ bindings: provider_kind: okx_bar native_channel: candle3Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-3m mode: RUST_NATIVE @@ -4797,8 +4797,8 @@ bindings: provider_kind: okx_bar native_channel: candle3m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-4h mode: RUST_NATIVE @@ -4806,8 +4806,8 @@ bindings: provider_kind: okx_bar native_channel: candle4H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-5m mode: RUST_NATIVE @@ -4815,8 +4815,8 @@ bindings: provider_kind: okx_bar native_channel: candle5m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-bar-6h mode: RUST_NATIVE @@ -4824,8 +4824,8 @@ bindings: provider_kind: okx_bar native_channel: candle6Hutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-book_delta mode: RUST_NATIVE @@ -4833,8 +4833,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -4848,8 +4848,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -4863,8 +4863,8 @@ bindings: provider_kind: okx_mark_index native_channel: mark-price sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null mark_index: provider_protocol: OKX_MARK_INDEX @@ -4878,8 +4878,8 @@ bindings: provider_kind: okx_bbo native_channel: bbo-tbt sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-trade mode: RUST_NATIVE @@ -4887,8 +4887,8 @@ bindings: provider_kind: okx_trade native_channel: trades sequence_policy: MONOTONIC - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-ethfi-usdt-swap-bar-1d mode: PYTHON_REST @@ -5976,8 +5976,8 @@ bindings: provider_kind: okx_bar native_channel: candle12Hutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-15m mode: RUST_NATIVE @@ -5985,8 +5985,8 @@ bindings: provider_kind: okx_bar native_channel: candle15m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-1d mode: RUST_NATIVE @@ -5994,8 +5994,8 @@ bindings: provider_kind: okx_bar native_channel: candle1Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-1h mode: RUST_NATIVE @@ -6003,8 +6003,8 @@ bindings: provider_kind: okx_bar native_channel: candle1H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-1m mode: RUST_NATIVE @@ -6012,8 +6012,8 @@ bindings: provider_kind: okx_bar native_channel: candle1m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-1w mode: RUST_NATIVE @@ -6021,8 +6021,8 @@ bindings: provider_kind: okx_bar native_channel: candle1Wutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-2d mode: RUST_NATIVE @@ -6030,8 +6030,8 @@ bindings: provider_kind: okx_bar native_channel: candle2Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-2h mode: RUST_NATIVE @@ -6039,8 +6039,8 @@ bindings: provider_kind: okx_bar native_channel: candle2H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-30m mode: RUST_NATIVE @@ -6048,8 +6048,8 @@ bindings: provider_kind: okx_bar native_channel: candle30m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-3d mode: RUST_NATIVE @@ -6057,8 +6057,8 @@ bindings: provider_kind: okx_bar native_channel: candle3Dutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-3m mode: RUST_NATIVE @@ -6066,8 +6066,8 @@ bindings: provider_kind: okx_bar native_channel: candle3m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-4h mode: RUST_NATIVE @@ -6075,8 +6075,8 @@ bindings: provider_kind: okx_bar native_channel: candle4H sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-5m mode: RUST_NATIVE @@ -6084,8 +6084,8 @@ bindings: provider_kind: okx_bar native_channel: candle5m sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-bar-6h mode: RUST_NATIVE @@ -6093,8 +6093,8 @@ bindings: provider_kind: okx_bar native_channel: candle6Hutc sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-book_delta mode: RUST_NATIVE @@ -6102,8 +6102,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -6117,8 +6117,8 @@ bindings: provider_kind: okx_book native_channel: books sequence_policy: CONTIGUOUS - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 @@ -6132,8 +6132,8 @@ bindings: provider_kind: okx_mark_index native_channel: mark-price sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null mark_index: provider_protocol: OKX_MARK_INDEX @@ -6147,8 +6147,8 @@ bindings: provider_kind: okx_bbo native_channel: bbo-tbt sequence_policy: NONE - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-trade mode: RUST_NATIVE @@ -6156,8 +6156,8 @@ bindings: provider_kind: okx_trade native_channel: trades sequence_policy: MONOTONIC - websocket_url: wss://ws.okx.com:8443/ws/v5/public - business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + websocket_url: wss://ws.okx.com/ws/v5/public + business_websocket_url: wss://ws.okx.com/ws/v5/business market_websocket_url: null - binding_id: okx-swap-soon-usdt-swap-bar-1d mode: PYTHON_REST diff --git a/qdl/adapters/okx/client.py b/qdl/adapters/okx/client.py index c62ad044..520d94a1 100644 --- a/qdl/adapters/okx/client.py +++ b/qdl/adapters/okx/client.py @@ -14,6 +14,8 @@ import requests +# WebSocket URLs are re-exported from their single owner (port 8443 retirement). +from qdl.adapters.okx_endpoints import OKX_BUSINESS_WS, OKX_PUBLIC_WS from qdl.reference.contracts import ReferenceProviderExhausted @@ -51,8 +53,6 @@ def statistics_retry_after_ms(response: Any) -> int | None: OKX_REST_BASE = "https://www.okx.com" -OKX_PUBLIC_WS = "wss://ws.okx.com:8443/ws/v5/public" -OKX_BUSINESS_WS = "wss://ws.okx.com:8443/ws/v5/business" async def _receive_until_stop(socket: Any, stop: asyncio.Event, timeout: float) -> Any | None: diff --git a/qdl/adapters/okx_endpoints.py b/qdl/adapters/okx_endpoints.py new file mode 100644 index 00000000..59e353ab --- /dev/null +++ b/qdl/adapters/okx_endpoints.py @@ -0,0 +1,74 @@ +"""OKX WebSocket endpoints: the one owner of the URLs and of their admission rule. + +Purpose. Every Data Layer component that names an OKX WebSocket service takes +the URL from here: the production catalog generator, the L2 demand projection, +the Python supervisor used by certification scripts, and the acquisition-plan +validator. Before this module the same literal was copied into four places. + +Boundary. These are *transport* endpoints. They are not part of any data +identity: no catalog revision, acquisition revision, binding ID, checkpoint or +canonical record is derived from them. Moving an endpoint must therefore never +require an identity change (see ``StableAcquisitionPlan.identity_digest``). + +Venue fact (OKX API changelog, "WebSocket port 8443 discontinuation", last +updated 2026-09-30): OKX stops accepting WebSocket connections on port 8443 on +2026-10-31. Port 443 already serves every affected domain; the migration is to +drop ``:8443`` and keep host and path. The rule covers ``/ws/v5/public``, +``/ws/v5/private``, ``/ws/v5/business`` and ``/ws/v5/public-sbe``. +""" + +from __future__ import annotations + +from urllib.parse import urlsplit, urlunsplit + +OKX_PUBLIC_WS = "wss://ws.okx.com/ws/v5/public" +OKX_BUSINESS_WS = "wss://ws.okx.com/ws/v5/business" + +OKX_RETIRED_WS_PORT = 8443 +OKX_RETIRED_WS_PORT_CUTOVER = "2026-10-31" +_SERVICES = frozenset({"public", "private", "business", "public-sbe"}) + + +def require_okx_websocket_url(value: str | None, service: str) -> str: + """Admit an OKX WebSocket URL for ``service`` or raise ``ValueError``. + + A URL that still names port 8443 is refused rather than tolerated: after + the cutover it fails at connect time, after a role has already been + recreated, which is the costliest place to discover it. + """ + + if service not in _SERVICES: + raise ValueError(f"unknown OKX WebSocket service: {service}") + parsed = urlsplit(value or "") + host = parsed.hostname or "" + try: + port = parsed.port + except ValueError as error: + raise ValueError("OKX WebSocket URL has an invalid port") from error + if parsed.scheme != "wss" or not (host == "okx.com" or host.endswith(".okx.com")): + raise ValueError("OKX WebSocket URL must be wss on an okx.com host") + if port == OKX_RETIRED_WS_PORT: + raise ValueError( + "OKX WebSocket port 8443 is discontinued on " + f"{OKX_RETIRED_WS_PORT_CUTOVER}; drop the port (wss defaults to 443)" + ) + if port not in (None, 443): + raise ValueError("OKX WebSocket URL must use the default wss port") + if parsed.path.rstrip("/") != f"/ws/v5/{service}" or parsed.query or parsed.fragment: + raise ValueError(f"OKX WebSocket URL must be the /ws/v5/{service} endpoint") + return str(value) + + +def without_retired_port(value: str) -> str: + """Return ``value`` with an explicit ``:8443`` removed; host and path kept. + + Any other URL is returned unchanged, so applying this twice is a no-op. + """ + + parsed = urlsplit(value) + if parsed.port != OKX_RETIRED_WS_PORT or parsed.hostname is None: + return value + netloc = parsed.hostname + if parsed.username is not None or parsed.password is not None: + raise ValueError("OKX WebSocket URL must not carry credentials") + return urlunsplit((parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment)) diff --git a/qdl/runtime/l2_demand.py b/qdl/runtime/l2_demand.py index 43a236a8..977fdfa9 100644 --- a/qdl/runtime/l2_demand.py +++ b/qdl/runtime/l2_demand.py @@ -13,6 +13,7 @@ import re from typing import Iterable, Mapping +from qdl.adapters.okx_endpoints import OKX_PUBLIC_WS from qdl.demand import ( ActiveDemandConvergence, ActiveDemandInventory, @@ -41,7 +42,6 @@ "USDM": "https://fapi.binance.com/fapi/v1/depth", "SPOT": "https://api.binance.com/api/v3/depth", } -_OKX_PUBLIC_WS = "wss://ws.okx.com:8443/ws/v5/public" @dataclass(frozen=True, slots=True) @@ -339,5 +339,5 @@ def _provider_edge( _BINANCE_REST_DEPTH[market], ) if venue == "OKX" and market in {"SWAP", "FUTURES", "SPOT"}: - return "OKX_PUBLIC_BOOKS", "books", _OKX_PUBLIC_WS, None + return "OKX_PUBLIC_BOOKS", "books", OKX_PUBLIC_WS, None raise InventoryError(f"L2 provider edge is not certified: {venue}/{market}") diff --git a/qdl/runtime/production_catalog.py b/qdl/runtime/production_catalog.py index 6f16d313..32ac0c92 100644 --- a/qdl/runtime/production_catalog.py +++ b/qdl/runtime/production_catalog.py @@ -17,6 +17,7 @@ okx_candle_channel, ) from qdl.adapters.okx.instruments import parse_public_instrument +from qdl.adapters.okx_endpoints import OKX_BUSINESS_WS, OKX_PUBLIC_WS from qdl.domain.instrument import InstrumentRecord, InstrumentStatus from qdl.query import ConsumerGrade, FeedType from qdl.runtime.stable_catalog import StableSourceCatalog @@ -73,7 +74,6 @@ "USDM": "https://fapi.binance.com/fapi/v1/depth", "SPOT": "https://api.binance.com/api/v3/depth", } -_OKX_PUBLIC_WS = "wss://ws.okx.com:8443/ws/v5/public" # Provider-declared quiet cadence is compiled into the signed acquisition # bundle, never inferred by a reader at runtime. The margin is deliberately # finite: it covers normal provider cadence plus transport jitter, while the @@ -814,8 +814,8 @@ def _acquisition( "PYTHON_REST", "okx_bar", okx_candle_channel(item.interval or "1m"), "NONE", ) - websocket = _OKX_PUBLIC_WS if mode == "RUST_NATIVE" else None - business = "wss://ws.okx.com:8443/ws/v5/business" if mode == "RUST_NATIVE" else None + websocket = OKX_PUBLIC_WS if mode == "RUST_NATIVE" else None + business = OKX_BUSINESS_WS if mode == "RUST_NATIVE" else None # OKX routes by service, not by Binance's route groups. market_ws = None result = { diff --git a/qdl/runtime/stable_deployment.py b/qdl/runtime/stable_deployment.py index ecd02cef..4528bf4d 100644 --- a/qdl/runtime/stable_deployment.py +++ b/qdl/runtime/stable_deployment.py @@ -2,13 +2,14 @@ import hashlib import json -from dataclasses import dataclass +from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, Iterable, Mapping from urllib.parse import urlsplit import yaml +from qdl.adapters.okx_endpoints import require_okx_websocket_url from qdl.query import FeedType from qdl.runtime.stable_catalog import StableSourceBinding, StableSourceCatalog @@ -454,7 +455,10 @@ def validate(self, source: StableSourceBinding) -> None: raise ValueError("Rust native acquisition cannot use a REST BAR provider kind") self._require_wss(self.websocket_url) if self.runtime == "OKX": - self._require_wss(self.business_websocket_url) + # Refuse port 8443, which OKX discontinues on 2026-10-31, here + # rather than at connect time after a role was recreated. + require_okx_websocket_url(self.websocket_url, "public") + require_okx_websocket_url(self.business_websocket_url, "business") if self.market_websocket_url is not None: raise ValueError( "OKX does not use a Binance /market routed base" @@ -552,6 +556,39 @@ def __post_init__(self) -> None: if not identities or len(identities) != len(set(identities)): raise ValueError("stable acquisition binding IDs must be non-empty and unique") + # Where a binding connects, not what it is. Venues move endpoints (Binance + # routed USD-M on 2026-04-23, OKX retires port 8443 on 2026-10-31); such a + # move must not be an identity change, because the revision is pinned by + # edge checkpoints and a bump strands them (plan C.19/C.32). + TRANSPORT_FIELDS = ( + "websocket_url", "business_websocket_url", "market_websocket_url", + ) + + def identity_digest(self) -> str: + """SHA-256 of everything that defines the plan except transport endpoints. + + Two plans with the same ``revision`` must have the same identity + digest. A change that moves this digest is an identity change and needs + a revision bump plus the checkpoint handling recorded in C.19; a change + that leaves it equal (an endpoint move) needs neither. + """ + + bindings = [] + for item in sorted(self.bindings, key=lambda value: value.binding_id): + payload = asdict(item) + for field in self.TRANSPORT_FIELDS: + payload.pop(field) + bindings.append(payload) + payload = { + "schema": self.schema, + "revision": self.revision, + "topics": [self.raw_topic, self.canonical_topic, self.quarantine_topic], + "bindings": bindings, + } + return hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + @classmethod def load( cls, path: str | Path, *, catalog: StableSourceCatalog diff --git a/scripts/execution_view_diagnostics.py b/scripts/execution_view_diagnostics.py index 1be556c0..a7f84880 100644 --- a/scripts/execution_view_diagnostics.py +++ b/scripts/execution_view_diagnostics.py @@ -37,7 +37,7 @@ async def provider_capture(stop): symbols = ("BTC", "ETH", "SOL", "DOGE", "BNB") async def venue_capture(venue, route): - url = ("wss://ws.okx.com:8443/ws/v5/public" if venue == "OKX" else + url = ("wss://ws.okx.com/ws/v5/public" if venue == "OKX" else f"wss://fstream.binance.com/{route}/ws") async with aiohttp.ClientSession() as session: async with session.ws_connect(url, heartbeat=15, timeout=10) as ws: diff --git a/scripts/migrate_okx_ws_port.py b/scripts/migrate_okx_ws_port.py new file mode 100644 index 00000000..8b88d0ce --- /dev/null +++ b/scripts/migrate_okx_ws_port.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Move live OKX native-ingestor configs off WebSocket port 8443, transport only. + +Purpose. OKX stops accepting WebSocket connections on port 8443 on 2026-10-31 +(OKX changelog, "WebSocket port 8443 discontinuation"). The running OKX ingestor +reads its URLs from a bind-mounted runtime JSON. This tool rewrites only the +two OKX URL values in such a file, so the role can be restarted on port 443 +without regenerating the bundle. + +Boundary. It changes nothing but ``websocket_url`` and +``business_websocket_url`` and proves it: the file with those keys removed must +be identical before and after, so ``config_revision``, ``authority``, +``bindings`` and every catalog revision stay byte-for-byte the same and no edge +checkpoint is stranded (plan C.19/C.32). It never regenerates a bundle, never +touches ``stable.env`` or identities, and never starts or stops a container. + +Dry run by default. ``--apply`` writes a sibling backup, then atomically +replaces the file, keeping its owner and mode. A running container keeps the +inode it started with until it is restarted. ``--rollback-from BACKUP`` +restores a backup after checking that its identity equals the current file's. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from qdl.adapters.okx_endpoints import ( # noqa: E402 + require_okx_websocket_url, + without_retired_port, +) + +SERVICE_BY_FIELD = {"websocket_url": "public", "business_websocket_url": "business"} +TRANSPORT_FIELDS = ("websocket_url", "business_websocket_url", "market_websocket_url") +BACKUP_SUFFIX = ".pre-okx-port443-" + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _decode(data: bytes, path: Path) -> dict: + try: + payload = json.loads(data) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError(f"{path}: not JSON") from error + if not isinstance(payload, dict) or payload.get("runtime") != "OKX": + raise ValueError(f"{path}: not an OKX native ingestor config") + if not all(field in payload for field in TRANSPORT_FIELDS): + raise ValueError(f"{path}: transport fields are incomplete") + return payload + + +def identity_sha256(payload: dict) -> str: + """Digest of the config with transport endpoints removed.""" + identity = {key: value for key, value in payload.items() if key not in TRANSPORT_FIELDS} + return _sha256(json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()) + + +def migrate_bytes(data: bytes, path: Path) -> tuple[bytes, dict[str, dict[str, str]]]: + """Return the migrated bytes and the per-field change list. + + The rewrite is textual so the file keeps its formatting and every other + byte; the parsed result is then checked independently. + """ + before = _decode(data, path) + text = data.decode("utf-8") + changes: dict[str, dict[str, str]] = {} + for field, service in SERVICE_BY_FIELD.items(): + old = before[field] + if not isinstance(old, str): + raise ValueError(f"{path}: {field} is not a URL") + new = without_retired_port(old) + require_okx_websocket_url(new, service) + if new == old: + continue + pattern = re.compile( + "(" + re.escape(json.dumps(field)) + r"\s*:\s*)" + re.escape(json.dumps(old)) + ) + text, count = pattern.subn(lambda match: match.group(1) + json.dumps(new), text) + if count != 1: + raise ValueError(f"{path}: expected exactly one {field} value, found {count}") + changes[field] = {"from": old, "to": new} + migrated = text.encode("utf-8") + after = _decode(migrated, path) + if identity_sha256(after) != identity_sha256(before): + raise RuntimeError(f"{path}: rewrite changed a non-transport field") + for field, service in SERVICE_BY_FIELD.items(): + require_okx_websocket_url(after[field], service) + if after["market_websocket_url"] != before["market_websocket_url"]: + raise RuntimeError(f"{path}: rewrite changed market_websocket_url") + return migrated, changes + + +def _atomic_replace(path: Path, data: bytes, like: os.stat_result) -> None: + staging = path.with_name(f".{path.name}.okx443-staging-{time.time_ns()}") + descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL, like.st_mode & 0o7777) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + if os.geteuid() == 0: + os.chown(staging, like.st_uid, like.st_gid) + os.chmod(staging, like.st_mode & 0o7777) + os.replace(staging, path) + finally: + if staging.exists(): + staging.unlink() + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def migrate(path: Path, *, apply: bool, clock=time.time_ns) -> dict[str, object]: + data = path.read_bytes() + stat = path.stat() + migrated, changes = migrate_bytes(data, path) + before = _decode(data, path) + report: dict[str, object] = { + "path": str(path), + "status": "NO_CHANGE" if not changes else ("APPLIED" if apply else "DRY_RUN"), + "changes": changes, + "sha256_before": _sha256(data), + "sha256_after": _sha256(migrated), + "identity_sha256": identity_sha256(before), + "config_revision": before.get("config_revision"), + "backup": None, + } + if not changes or not apply: + return report + backup = path.with_name(f"{path.name}{BACKUP_SUFFIX}{clock()}") + shutil.copy2(path, backup) + if os.geteuid() == 0: + os.chown(backup, stat.st_uid, stat.st_gid) + if backup.read_bytes() != data: + raise RuntimeError(f"{backup}: backup differs from the original") + _atomic_replace(path, migrated, stat) + written = path.read_bytes() + if written != migrated or identity_sha256(_decode(written, path)) != report["identity_sha256"]: + raise RuntimeError(f"{path}: verification after write failed; restore {backup}") + report["backup"] = str(backup) + return report + + +def rollback(path: Path, backup: Path) -> dict[str, object]: + if not backup.name.startswith(path.name + BACKUP_SUFFIX) or backup.parent != path.parent: + raise ValueError("rollback source is not a backup written for this config") + current = path.read_bytes() + restored = backup.read_bytes() + if identity_sha256(_decode(current, path)) != identity_sha256(_decode(restored, backup)): + raise ValueError("backup identity differs from the current config; refusing") + _atomic_replace(path, restored, path.stat()) + if path.read_bytes() != restored: + raise RuntimeError(f"{path}: rollback verification failed") + return { + "path": str(path), + "status": "ROLLED_BACK", + "sha256_before": _sha256(current), + "sha256_after": _sha256(restored), + "from_backup": str(backup), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, action="append", required=True, + help="OKX native ingestor runtime JSON; repeatable") + parser.add_argument("--apply", action="store_true", + help="write the change; omit to report only") + parser.add_argument("--rollback-from", type=Path, + help="restore this backup into the single --config path") + args = parser.parse_args() + if args.rollback_from is not None: + if len(args.config) != 1 or args.apply: + parser.error("--rollback-from takes exactly one --config and no --apply") + result = [rollback(args.config[0], args.rollback_from)] + else: + # Validate every file before writing any, so a bad file cannot leave + # the set half migrated. + for path in args.config: + migrate(path, apply=False) + result = [migrate(path, apply=args.apply) for path in args.config] + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/phase114_l2_real_provider_capture.py b/scripts/phase114_l2_real_provider_capture.py index 620eec5e..7254eacf 100644 --- a/scripts/phase114_l2_real_provider_capture.py +++ b/scripts/phase114_l2_real_provider_capture.py @@ -27,7 +27,7 @@ DEFAULT_OUTPUT = ROOT / "upgrade/evidence/phase114-l2-real-provider-capture.json" BINANCE_WS = "wss://fstream.binance.com/ws" BINANCE_DEPTH = "https://fapi.binance.com/fapi/v1/depth" -OKX_WS = "wss://ws.okx.com:8443/ws/v5/public" +OKX_WS = "wss://ws.okx.com/ws/v5/public" class ProviderCaptureError(RuntimeError): diff --git a/scripts/phasea_real_provider_certification.py b/scripts/phasea_real_provider_certification.py index 37b3b539..c0a6297b 100755 --- a/scripts/phasea_real_provider_certification.py +++ b/scripts/phasea_real_provider_certification.py @@ -146,10 +146,10 @@ def crypto_specs(authority_record: dict): "wss://stream.binance.com:9443/stream", None, (("trade", "binance_spot_trade", 3), ("bookTicker", "binance_spot_bbo", 2))), ("OKX", "SWAP", "PERPETUAL", "BTC-USDT-SWAP", "OKX_DIRECT", - "wss://ws.okx.com:8443/ws/v5/public", "wss://ws.okx.com:8443/ws/v5/business", + "wss://ws.okx.com/ws/v5/public", "wss://ws.okx.com/ws/v5/business", (("trades", "okx_trade", 3), ("bbo-tbt", "okx_bbo", 2), ("candle1m", "okx_bar", 1))), ("OKX", "SPOT", "SPOT", "BTC-USDT", "OKX_DIRECT", - "wss://ws.okx.com:8443/ws/v5/public", "wss://ws.okx.com:8443/ws/v5/business", + "wss://ws.okx.com/ws/v5/public", "wss://ws.okx.com/ws/v5/business", (("trades", "okx_trade", 3), ("bbo-tbt", "okx_bbo", 2), ("candle1m", "okx_bar", 1))), ) specs = [] diff --git a/tests/test_crypto_demand_manifest.py b/tests/test_crypto_demand_manifest.py index 28ea26ed..3330fb09 100644 --- a/tests/test_crypto_demand_manifest.py +++ b/tests/test_crypto_demand_manifest.py @@ -199,10 +199,10 @@ def test_okx_swap_bar_uses_native_business_ws_and_preserves_interval(self): self.assertEqual(minute["mode"], "RUST_NATIVE") self.assertEqual(hour["mode"], "RUST_NATIVE") self.assertEqual(minute["provider_kind"], "okx_bar") - self.assertEqual(minute["websocket_url"], "wss://ws.okx.com:8443/ws/v5/public") + self.assertEqual(minute["websocket_url"], "wss://ws.okx.com/ws/v5/public") self.assertEqual( minute["business_websocket_url"], - "wss://ws.okx.com:8443/ws/v5/business", + "wss://ws.okx.com/ws/v5/business", ) self.assertEqual(minute["native_channel"], "candle1m") self.assertEqual(hour["native_channel"], "candle1H") diff --git a/tests/test_dlv2_r127_binance_ws_route.py b/tests/test_dlv2_r127_binance_ws_route.py index cfa239b1..da22ec6e 100644 --- a/tests/test_dlv2_r127_binance_ws_route.py +++ b/tests/test_dlv2_r127_binance_ws_route.py @@ -288,7 +288,7 @@ def test_binance_may_not_carry_an_okx_business_service(self): with self.assertRaises(ValueError): self._reload_with( lambda binding: binding.update( - {"business_websocket_url": "wss://ws.okx.com:8443/ws/v5/business"} + {"business_websocket_url": "wss://ws.okx.com/ws/v5/business"} ) ) @@ -312,9 +312,9 @@ def test_okx_keeps_public_and_business_and_declares_no_market_base(self): ] self.assertTrue(okx) for item in okx: - self.assertEqual(item.websocket_url, "wss://ws.okx.com:8443/ws/v5/public") + self.assertEqual(item.websocket_url, "wss://ws.okx.com/ws/v5/public") self.assertEqual( - item.business_websocket_url, "wss://ws.okx.com:8443/ws/v5/business" + item.business_websocket_url, "wss://ws.okx.com/ws/v5/business" ) self.assertIsNone(item.market_websocket_url) diff --git a/tests/test_okx_ws_port_migration.py b/tests/test_okx_ws_port_migration.py new file mode 100644 index 00000000..1ed2e845 --- /dev/null +++ b/tests/test_okx_ws_port_migration.py @@ -0,0 +1,285 @@ +"""OKX WebSocket port 8443 retirement (OKX changelog, cutover 2026-10-31). + +Covers the endpoint owner, the acquisition-plan guard, the rule that an +endpoint move is not an identity change, and the transport-only migration of a +live ingestor runtime config. +""" + +from __future__ import annotations + +import dataclasses +import json +import os +import tempfile +import unittest +from pathlib import Path + +import yaml + +from qdl.adapters.okx import client as okx_client +from qdl.adapters.okx_endpoints import ( + OKX_BUSINESS_WS, + OKX_PUBLIC_WS, + require_okx_websocket_url, + without_retired_port, +) +from qdl.runtime import l2_demand, production_catalog +from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_deployment import StableAcquisitionPlan, stable_authority_record +from scripts import migrate_okx_ws_port as tool + +ROOT = Path(__file__).resolve().parents[1] +CATALOG = ROOT / "config/v2/stable-source-bindings.yaml" +ACQUISITION = ROOT / "config/v2/stable-acquisition-bindings.yaml" +OLD_PUBLIC = "wss://ws.okx.com:8443/ws/v5/public" +OLD_BUSINESS = "wss://ws.okx.com:8443/ws/v5/business" + + +class EndpointOwnerTests(unittest.TestCase): + def test_owner_urls_carry_no_port(self): + self.assertEqual(OKX_PUBLIC_WS, "wss://ws.okx.com/ws/v5/public") + self.assertEqual(OKX_BUSINESS_WS, "wss://ws.okx.com/ws/v5/business") + + def test_every_consumer_uses_the_owner(self): + self.assertIs(okx_client.OKX_PUBLIC_WS, OKX_PUBLIC_WS) + self.assertIs(okx_client.OKX_BUSINESS_WS, OKX_BUSINESS_WS) + self.assertIs(production_catalog.OKX_PUBLIC_WS, OKX_PUBLIC_WS) + self.assertIs(production_catalog.OKX_BUSINESS_WS, OKX_BUSINESS_WS) + self.assertIs(l2_demand.OKX_PUBLIC_WS, OKX_PUBLIC_WS) + + def test_retired_port_is_refused_with_the_cutover_date(self): + with self.assertRaisesRegex(ValueError, "2026-10-31"): + require_okx_websocket_url(OLD_PUBLIC, "public") + with self.assertRaisesRegex(ValueError, "2026-10-31"): + require_okx_websocket_url(OLD_BUSINESS, "business") + + def test_admission_rule(self): + for url, service in ( + (OKX_PUBLIC_WS, "public"), + (OKX_BUSINESS_WS, "business"), + ("wss://ws.okx.com:443/ws/v5/public", "public"), + ("wss://wspap.okx.com/ws/v5/private", "private"), + ): + with self.subTest(url=url): + self.assertEqual(require_okx_websocket_url(url, service), url) + for url, service in ( + ("ws://ws.okx.com/ws/v5/public", "public"), + ("wss://ws.okx.com.evil.test/ws/v5/public", "public"), + ("wss://ws.okx.com:9443/ws/v5/public", "public"), + (OKX_PUBLIC_WS, "business"), + ("wss://ws.okx.com/ws/v5/public?x=1", "public"), + (None, "public"), + ): + with self.subTest(url=url, service=service): + with self.assertRaises(ValueError): + require_okx_websocket_url(url, service) + + def test_port_removal_keeps_host_and_path_and_is_idempotent(self): + self.assertEqual(without_retired_port(OLD_PUBLIC), OKX_PUBLIC_WS) + self.assertEqual( + without_retired_port("wss://wspap.okx.com:8443/ws/v5/business"), + "wss://wspap.okx.com/ws/v5/business", + ) + self.assertEqual(without_retired_port(OKX_PUBLIC_WS), OKX_PUBLIC_WS) + + +class AcquisitionPlanTests(unittest.TestCase): + def setUp(self): + self.catalog = StableSourceCatalog.load(CATALOG) + self.plan = StableAcquisitionPlan.load(ACQUISITION, catalog=self.catalog) + + def test_committed_plan_uses_port_443_for_every_okx_native_binding(self): + self.assertNotIn(b":8443", ACQUISITION.read_bytes()) + okx = [ + item for item in self.plan.bindings + if item.runtime == "OKX" and item.mode == "RUST_NATIVE" + ] + self.assertTrue(okx) + for item in okx: + self.assertEqual(item.websocket_url, OKX_PUBLIC_WS) + self.assertEqual(item.business_websocket_url, OKX_BUSINESS_WS) + + def test_a_plan_that_still_names_port_8443_does_not_load(self): + payload = yaml.safe_load(ACQUISITION.read_text(encoding="utf-8")) + target = next( + item for item in payload["bindings"] + if item["runtime"] == "OKX" and item["mode"] == "RUST_NATIVE" + ) + target["business_websocket_url"] = OLD_BUSINESS + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "acquisition.yaml" + path.write_text(yaml.safe_dump(payload, sort_keys=True), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "8443"): + StableAcquisitionPlan.load(path, catalog=self.catalog) + + def test_an_endpoint_move_is_not_an_identity_change(self): + moved = dataclasses.replace( + self.plan, + bindings=tuple( + dataclasses.replace( + item, + websocket_url=OLD_PUBLIC if item.websocket_url else None, + business_websocket_url=OLD_BUSINESS if item.business_websocket_url else None, + ) + if item.runtime == "OKX" else item + for item in self.plan.bindings + ), + ) + self.assertNotEqual(moved.bindings, self.plan.bindings) + self.assertEqual(moved.revision, self.plan.revision) + self.assertEqual(moved.identity_digest(), self.plan.identity_digest()) + + def test_identity_changes_are_still_detected(self): + first = self.plan.bindings[0] + renamed = dataclasses.replace( + self.plan, + bindings=(dataclasses.replace(first, native_channel=first.native_channel + "x"),) + + self.plan.bindings[1:], + ) + disabled = dataclasses.replace( + self.plan, + bindings=(dataclasses.replace(first, enabled=not first.enabled),) + + self.plan.bindings[1:], + ) + bumped = dataclasses.replace(self.plan, revision=self.plan.revision + 1) + digests = { + self.plan.identity_digest(), renamed.identity_digest(), + disabled.identity_digest(), bumped.identity_digest(), + } + self.assertEqual(len(digests), 4) + + def test_generated_okx_ingestor_config_carries_the_new_urls_only(self): + authority = stable_authority_record( + rust_image_digest="sha256:" + "a" * 64, + capability_manifest=ROOT / "config/v2/stable-capabilities.yaml", + contract=ROOT / "contracts/proto/qdl/marketdata/v2/market_data.proto", + partition_plan=ACQUISITION.read_bytes(), + effective_at_ns=1, + ) + configs = self.plan.native_ingestor_configs(catalog=self.catalog, authority=authority) + okx = {key: value for key, value in configs.items() if value["runtime"] == "OKX"} + self.assertTrue(okx) + for value in okx.values(): + self.assertEqual(value["websocket_url"], OKX_PUBLIC_WS) + self.assertEqual(value["business_websocket_url"], OKX_BUSINESS_WS) + self.assertEqual(value["config_revision"], self.plan.revision) + for value in configs.values(): + self.assertNotIn(":8443", json.dumps(value)) + + +def _live_like_config() -> dict: + """An OKX ingestor config shaped like the live one, still on port 8443.""" + return { + "runtime": "OKX", + "websocket_url": OLD_PUBLIC, + "business_websocket_url": OLD_BUSINESS, + "market_websocket_url": None, + "config_revision": 20, + "shard_id": "qdl-v2-stable-okx-swap", + "authority": {"revision": 1, "partition_plan_digest": "b" * 64}, + "bindings": [ + {"native_channel": "trades", "native_symbol": "BTC-USDT-SWAP", + "instrument_catalog_revision": 11, "subscription_id": "okx:x"}, + ], + } + + +def _encode(payload: dict) -> bytes: + return (json.dumps(payload, indent=2, sort_keys=True, separators=(",", ": ")) + "\n").encode() + + +class RuntimeConfigMigrationTests(unittest.TestCase): + def setUp(self): + self._directory = tempfile.TemporaryDirectory() + self.root = Path(self._directory.name) + self.path = self.root / "ingestor_okx_swap.json" + self.original = _encode(_live_like_config()) + self.path.write_bytes(self.original) + self.path.chmod(0o640) + + def tearDown(self): + self._directory.cleanup() + + def _backups(self): + return sorted(self.root.glob(self.path.name + tool.BACKUP_SUFFIX + "*")) + + def test_dry_run_writes_nothing(self): + report = tool.migrate(self.path, apply=False) + self.assertEqual(report["status"], "DRY_RUN") + self.assertEqual(self.path.read_bytes(), self.original) + self.assertEqual(self._backups(), []) + self.assertEqual(sorted(report["changes"]), ["business_websocket_url", "websocket_url"]) + + def test_apply_changes_only_the_two_urls_and_keeps_a_backup(self): + report = tool.migrate(self.path, apply=True, clock=lambda: 7) + self.assertEqual(report["status"], "APPLIED") + after = json.loads(self.path.read_bytes()) + before = json.loads(self.original) + self.assertEqual(after["websocket_url"], OKX_PUBLIC_WS) + self.assertEqual(after["business_websocket_url"], OKX_BUSINESS_WS) + for key in before: + if key not in ("websocket_url", "business_websocket_url"): + self.assertEqual(after[key], before[key], key) + self.assertEqual(tool.identity_sha256(after), tool.identity_sha256(before)) + # Byte-level: exactly the two URL lines differ. + diff = [ + (a, b) for a, b in zip(self.original.splitlines(), self.path.read_bytes().splitlines()) + if a != b + ] + self.assertEqual(len(diff), 2) + self.assertEqual(len(self.original.splitlines()), len(self.path.read_bytes().splitlines())) + self.assertEqual(self.path.stat().st_mode & 0o777, 0o640) + (backup,) = self._backups() + self.assertEqual(backup.read_bytes(), self.original) + self.assertEqual(report["backup"], str(backup)) + self.assertEqual(list(self.root.glob(".*staging*")), []) + + def test_second_run_is_a_no_op(self): + tool.migrate(self.path, apply=True, clock=lambda: 1) + migrated = self.path.read_bytes() + report = tool.migrate(self.path, apply=True, clock=lambda: 2) + self.assertEqual(report["status"], "NO_CHANGE") + self.assertEqual(self.path.read_bytes(), migrated) + self.assertEqual(len(self._backups()), 1) + + def test_rollback_restores_the_exact_bytes(self): + report = tool.migrate(self.path, apply=True, clock=lambda: 3) + result = tool.rollback(self.path, Path(report["backup"])) + self.assertEqual(result["status"], "ROLLED_BACK") + self.assertEqual(self.path.read_bytes(), self.original) + self.assertEqual(self.path.stat().st_mode & 0o777, 0o640) + + def test_rollback_refuses_a_backup_with_a_different_identity(self): + report = tool.migrate(self.path, apply=True, clock=lambda: 4) + backup = Path(report["backup"]) + tampered = _live_like_config() + tampered["config_revision"] = 21 + backup.write_bytes(_encode(tampered)) + with self.assertRaisesRegex(ValueError, "identity"): + tool.rollback(self.path, backup) + with self.assertRaisesRegex(ValueError, "not a backup"): + tool.rollback(self.path, self.root / "other.json") + + def test_refuses_non_okx_and_foreign_hosts(self): + binance = _live_like_config() | {"runtime": "BINANCE"} + self.path.write_bytes(_encode(binance)) + with self.assertRaisesRegex(ValueError, "not an OKX"): + tool.migrate(self.path, apply=False) + foreign = _live_like_config() | {"websocket_url": "wss://example.test:8443/ws/v5/public"} + self.path.write_bytes(_encode(foreign)) + with self.assertRaises(ValueError): + tool.migrate(self.path, apply=False) + self.assertEqual(self._backups(), []) + + def test_compact_json_is_rewritten_textually(self): + compact = json.dumps(_live_like_config(), separators=(",", ":")).encode() + self.path.write_bytes(compact) + tool.migrate(self.path, apply=True, clock=lambda: 5) + self.assertEqual( + self.path.read_bytes(), + compact.replace(b":8443", b""), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_phase112_provider_admission.py b/tests/test_phase112_provider_admission.py index b8975893..2d09955e 100644 --- a/tests/test_phase112_provider_admission.py +++ b/tests/test_phase112_provider_admission.py @@ -47,7 +47,7 @@ def _binding( ) ) websocket_url = ( - "wss://ws.okx.com:8443/ws/v5/public" + "wss://ws.okx.com/ws/v5/public" if is_okx else ( "wss://stream.binance.com:9443/ws" diff --git a/tests/test_phase112_universal_realtime.py b/tests/test_phase112_universal_realtime.py index a278fbab..0a21a402 100644 --- a/tests/test_phase112_universal_realtime.py +++ b/tests/test_phase112_universal_realtime.py @@ -356,7 +356,7 @@ def test_provider_projection_covers_every_admitted_binding_without_symbol_fallba if item.feed is FeedType.BAR and item.venue == "OKX" and item.market == "SWAP" ) self.assertEqual(okx_native_bar.native_channel, "candle1H") - self.assertEqual(okx_native_bar.business_websocket_url, "wss://ws.okx.com:8443/ws/v5/business") + self.assertEqual(okx_native_bar.business_websocket_url, "wss://ws.okx.com/ws/v5/business") dated = next(item for item in bars if item.product_type == "FUTURE") self.assertEqual(dated.native_symbol, "BTCUSDT_270326") self.assertIsNone(dated.websocket_url) diff --git a/tests/test_phase115c_native_bar_materialization.py b/tests/test_phase115c_native_bar_materialization.py index fa8710d0..7977a6f3 100644 --- a/tests/test_phase115c_native_bar_materialization.py +++ b/tests/test_phase115c_native_bar_materialization.py @@ -348,8 +348,8 @@ def test_every_active_bar_uses_only_the_admitted_venue_acquisition_lane(self) -> self.assertEqual(item.provider_kind, "okx_bar") self.assertEqual(item.mode, "RUST_NATIVE") self.assertEqual(item.native_channel, f"candle{self.tool.okx_candle_channel(source.interval)[6:]}") - self.assertEqual(item.websocket_url, "wss://ws.okx.com:8443/ws/v5/public") - self.assertEqual(item.business_websocket_url, "wss://ws.okx.com:8443/ws/v5/business") + self.assertEqual(item.websocket_url, "wss://ws.okx.com/ws/v5/public") + self.assertEqual(item.business_websocket_url, "wss://ws.okx.com/ws/v5/business") def test_okx_native_bar_policy_refresh_is_revisioned_and_idempotent(self) -> None: before_acquisition = deepcopy(self.current_acquisition) diff --git a/tests/test_phaseb_stable_deployment.py b/tests/test_phaseb_stable_deployment.py index 3db4c1ef..17c454e9 100644 --- a/tests/test_phaseb_stable_deployment.py +++ b/tests/test_phaseb_stable_deployment.py @@ -127,8 +127,8 @@ def native(item): ) else: provider_kind = "okx_bar" - websocket_url = "wss://ws.okx.com:8443/ws/v5/public" - business_websocket_url = "wss://ws.okx.com:8443/ws/v5/business" + websocket_url = "wss://ws.okx.com/ws/v5/public" + business_websocket_url = "wss://ws.okx.com/ws/v5/business" native_channel = f"candle{source.interval}" return replace( item, diff --git a/tests/test_phasec36_liquid_crypto_features.py b/tests/test_phasec36_liquid_crypto_features.py index 22148992..5956a01d 100644 --- a/tests/test_phasec36_liquid_crypto_features.py +++ b/tests/test_phasec36_liquid_crypto_features.py @@ -286,7 +286,7 @@ def test_okx_futures_uses_shared_public_books_binding_not_a_worker(self): execution_grade=False, provider_protocol="OKX_PUBLIC_BOOKS", native_channel="books", - websocket_url="wss://ws.okx.com:8443/ws/v5/public", + websocket_url="wss://ws.okx.com/ws/v5/public", rest_snapshot_url=None, ) self.assertEqual(binding.subscription.market, "FUTURES") diff --git a/upgrade/OKX_MARKET_DATA_V5_GUIDE_QUANT_DATA_LAYER.md b/upgrade/OKX_MARKET_DATA_V5_GUIDE_QUANT_DATA_LAYER.md index 658bf89f..cd31d437 100644 --- a/upgrade/OKX_MARKET_DATA_V5_GUIDE_QUANT_DATA_LAYER.md +++ b/upgrade/OKX_MARKET_DATA_V5_GUIDE_QUANT_DATA_LAYER.md @@ -201,8 +201,8 @@ OKX có tài liệu và endpoint host khác nhau theo pháp nhân/khu vực. Age OKX_ENABLED=true OKX_REGION_PROFILE=global OKX_REST_BASE_URL=https://www.okx.com -OKX_WS_PUBLIC_URL=wss://ws.okx.com:8443/ws/v5/public -OKX_WS_BUSINESS_URL=wss://ws.okx.com:8443/ws/v5/business +OKX_WS_PUBLIC_URL=wss://ws.okx.com/ws/v5/public +OKX_WS_BUSINESS_URL=wss://ws.okx.com/ws/v5/business OKX_DEMO=false ``` @@ -214,13 +214,17 @@ OKX_REST_BASE_URL=https://openapi.okx.com Nhưng lựa chọn domain phải là config/deployment decision, không rải hard-code trong client. +Note (OKX changelog, "WebSocket port 8443 discontinuation", updated 2026-09-30): port 8443 +stops accepting WebSocket connections on 2026-10-31. WebSocket URLs carry no port (wss = 443). +The code owner of the URLs is `qdl/adapters/okx_endpoints.py`, which refuses `:8443`. + Ví dụ profile EEA có thể là: ```env OKX_REGION_PROFILE=eea OKX_REST_BASE_URL=https://eea.okx.com -OKX_WS_PUBLIC_URL=wss://wseea.okx.com:8443/ws/v5/public -OKX_WS_BUSINESS_URL=wss://wseea.okx.com:8443/ws/v5/business +OKX_WS_PUBLIC_URL=wss://wseea.okx.com/ws/v5/public +OKX_WS_BUSINESS_URL=wss://wseea.okx.com/ws/v5/business ``` ### 3.2 Demo trading diff --git a/upgrade/evidence/p184-okx-port-migration-packet.md b/upgrade/evidence/p184-okx-port-migration-packet.md new file mode 100644 index 00000000..634f6559 --- /dev/null +++ b/upgrade/evidence/p184-okx-port-migration-packet.md @@ -0,0 +1,163 @@ +# P18.4 Packet: OKX WebSocket Port 8443 Migration + +Status: `IMPLEMENTED / TESTED LOCALLY / LIVE ROLLOUT PENDING APPROVAL`. +Deadline: **before 2026-10-31**. After that date, OKX refuses connections on port 8443. + +## 1. Venue fact + +The OKX API changelog has an entry under "Upcoming Changes" titled +"WebSocket port 8443 discontinuation", last updated 2026-09-30 +(https://www.okx.com/docs-v5/log_en/). It says: + +- "OKX will stop accepting WebSocket connections on port 8443 on October 31, 2026". + Port 443 already works on every affected domain, and both ports work until the cutover. +- Under "Breaking change": "After October 31, 2026, connections to port 8443 will fail." + The fix is to remove `:8443`. Host and path stay the same, and `wss://` defaults to 443. +- The rule applies to `/ws/v5/public`, `/ws/v5/private` and `/ws/v5/business`, plus + `/ws/v5/public-sbe` on the Global site. REST and Colo are not affected. +- Example: `wss://ws.okx.com:8443/ws/v5/public` becomes `wss://ws.okx.com/ws/v5/public`. + For demo: `wss://wspap.okx.com:8443/...` becomes `wss://wspap.okx.com/...`. + +The "Production Trading Services" section of the main API docs (https://www.okx.com/docs-v5/en/) +already lists `wss://ws.okx.com/ws/v5/{public,private,business}` with no port. + +Live probe on 2026-10-10. It ran in `qdl-v2-python:2.2.3-rc6-10f9f5c` with `--cpus 1`. +Both port-less URLs connected on remote port 443 and delivered data: + +| URL | open | first data | n | +|---|---:|---:|---:| +| `wss://ws.okx.com/ws/v5/public` (`tickers` BTC-USDT-SWAP) | 217.6 ms | 273.8 ms | 1 | +| `wss://ws.okx.com/ws/v5/business` (`candle1m` BTC-USDT-SWAP) | 182.1 ms | 234.7 ms | 1 | + +This was a connectivity check, not an endpoint benchmark. It does not report the four latency quantities. + +The deployed Rust ingestor (`qdl-v2-rust:2.2.1-855c914`) uses tokio-tungstenite 0.24.0 +(per `Cargo.lock` at `855c914`). In `connect.rs:84`, `"wss" => 443`, so a URL with no port +connects on 443. The ingestor only checks that the URL starts with `wss://` +(`qdl-native-raw-ingestor.rs:380,425` at `855c914`). **No Rust rebuild is needed.** + +## 2. Where port 8443 was used (inventory) + +| Location | Kind | Action | +|---|---|---| +| `qdl/adapters/okx/client.py:54-55` | Python supervisor constants (used by scripts/tests only) | re-exported from the new owner | +| `qdl/runtime/production_catalog.py:76,818` | catalog generator | uses the owner | +| `qdl/runtime/l2_demand.py:44` | L2 plan projection (report field only) | uses the owner | +| `config/v2/stable-acquisition-bindings.yaml` | 106 public + 106 business URLs | port removed, **revision stays 19** | +| `scripts/phasea_real_provider_certification.py`, `phase114_l2_real_provider_capture.py`, `execution_view_diagnostics.py` | harness literals | port removed | +| 7 test files | fixtures and assertions | port removed | +| `upgrade/OKX_MARKET_DATA_V5_GUIDE_QUANT_DATA_LAYER.md` | guide (including `wseea`) | port removed, note added | +| Live `/home/bobby/.local/state/qdl-v2/releases/v2.2.1-0e00c81-recovery/ingestor_okx_swap.json` | **the only live consumer**; mounted by `qdl_v2_stable_candidate-ingestor_okx_swap-1` | migrate with the script (section 4) | +| V1 `app/providers/okx/rest.py` | REST only (`https://www.okx.com`) | none | +| Rust sources | no OKX URL defaults | none | +| `CERTIFICATION_LEDGER.md:605`, the plan, `upgrade/evidence/*.json` | historical records | left as recorded | + +Of the generated runtime bundle files, only `ingestor-okx-swap.json` carries an OKX URL. +The other ten (`core*.json`, `production-core-*.json`, the manifest, `authority.json`, +`ingestor-binance-usdm.json`) do not. This was checked by generating the bundle from the +committed plan. + +Outside this repo (not changed here; the owning teams must act): +`/home/bobby/.worktrees/trading-system-next-upgrade/services/venues/okx/endpoint_profiles.py:31-40` +still uses `ws.okx.com:8443` and `wspap.okx.com:8443`, including **private** and **demo**. + +## 3. What an identity change would cost, and why this move has none + +| Identity | Derived from | Effect of this change | +|---|---|---| +| `catalog_revision` (`stable-source-bindings.yaml`) | the source catalog, which has no URLs | unchanged (11) | +| acquisition `revision` | a hand-maintained integer in the YAML | **unchanged (19)** | +| Bar edge checkpoint (`stable_bar_edge.py:468-482`, checked at `:560-567`) | slice, authority revision, catalog revision, acquisition revision, warmup rows, binding IDs | unchanged, so no checkpoint is stranded | +| Ingestor `config_revision` | `plan.revision` (`stable_deployment.py:1086`) | unchanged | +| `authority.partition_plan_digest` | sha256 of the acquisition YAML bytes (`stable_deployment.py:1247`) | changes only if authority is **re-issued**; runtime only checks its shape (`stable_deployment.py:282-321`, Rust `authority.rs:57`). Live authority is not re-issued (section 4) | +| Production-catalog `acquisition_plan_sha256` | generator provenance (`production_catalog.py:489`) | changes; report only | +| Consumer manifest / JWT revision, Query schema, contract digests | `consumers/`, proto, capabilities | unchanged (no URLs) | + +Old and new YAML were compared with the three transport fields removed. They are identical: +712 bindings, revision 19 in both. File sha256 moved from `8ed0de04…5950a` to `1b586549…3656a`. + +The bar edge running image `qdl-v2-python:2.2.2-d6d2637` pins acquisition 19 and catalog 11, +the same as this branch. So a later Python release built from this commit does not strand +its checkpoint. + +**Design.** A WebSocket endpoint is transport configuration, not data identity. + +- The single owner is `qdl/adapters/okx_endpoints.py`. It is dependency-free, so importing it + from `stable_deployment` cannot create an import cycle. +- `StableAcquisitionBinding.validate` refuses an OKX URL on port 8443 at load time. +- `StableAcquisitionPlan.TRANSPORT_FIELDS` and `identity_digest()` define identity as the + plan without endpoints. Tests prove that moving an endpoint keeps the digest, while channel, + `enabled` or revision changes move it. + +Making `partition_plan_digest` use `identity_digest` was considered and **deferred**. It is +written by 6 producers and compared by the rollover and cutover tooling. Changing its meaning +before the deadline would mix two definitions across live authority records. It is not needed +here because the authority is preserved. + +A checkpoint migration script is **not needed**, because no revision moves. + +## 4. Live rollout (one role) + +Recreate: **none.** Restart: `qdl_v2_stable_candidate-ingestor_okx_swap-1` only, with +`docker restart`. Its image stays `qdl-v2-rust:2.2.1-855c914` +(`sha256:0f6876e16e51600419e1f172aae596bed9c2d369d1afcef93d1da30c18f262ec`). + +Do not use compose: the container's compose working directory is +`kafka-client-recovery-20261001`. The bind mount is the single file above. A local test +confirmed that a running container keeps the old inode, and that `docker restart` rebinds +the path to the new file. + +0. Check that no stage-100 run is active: `ps -eo args | grep -E '/run_stage100.sh [a-z] ' | grep -v grep` must be empty. +1. Dry run (read-only mount). Expected: `sha256_before 0c5d181c…ce47`, `sha256_after d4bd4bae…7ba7`, + `identity_sha256 056207ce…79c3`, `config_revision 20`: + ``` + docker run --rm --cpus 1 --network none -e PYTHONDONTWRITEBYTECODE=1 \ + -v :/src:ro \ + -v /home/bobby/.local/state/qdl-v2/releases/v2.2.1-0e00c81-recovery:/live \ + --entrypoint python3 qdl-v2-python:2.2.3-rc6-10f9f5c -B \ + /src/scripts/migrate_okx_ws_port.py --config /live/ingestor_okx_swap.json + ``` +2. Apply: run the same command with `--apply`. This writes the backup + `ingestor_okx_swap.json.pre-okx-port443-` next to the file, does an atomic replace that + keeps the owner and mode, then reads the file back to verify. +3. `docker restart qdl_v2_stable_candidate-ingestor_okx_swap-1`. +4. Verify within 5 minutes: + - the container is healthy and its log shows OKX public and business sessions connected; + - `ss -tnp`/conntrack shows the remote port is 443, not 8443; + - OKX TRADE, QUOTE, BOOK and MARK_INDEX projection keys advance in the V2 Redis across two + samples about 10 s apart; + - the OKX Swap 1m BAR arrives at the next boundary; + - quarantine counters on `rust_core*` stay flat; + - the TS market cache for BTC-USDT-SWAP advances. + +Expected outage: one OKX reconnect, a few seconds. This is the same as the routine +reconnects already logged (plan, Astra R1-R7). + +**Rollback, only on failure:** run the script with +`--config /live/ingestor_okx_swap.json --rollback-from /live/ingestor_okx_swap.json.pre-okx-port443-`, +then `docker restart` the same container. The script refuses a backup whose identity differs. +Rollback is only valid until 2026-10-31. + +**Not touched:** +- Binance ingestor, all three `rust_core`, the projectors, query/stream KN, `binance_bar_edge` + and its checkpoint, Kafka, `stable_redis`, `market_cache` +- `authority.json`, `stable.env`, identities, consumer manifests, V1 `data_layer_service` +- the Trading System + +**Coupling:** `scripts/refresh_v2_native_ingestor_runtime.py` compares non-binding fields, +URLs included. Before this migration, it would refuse a refresh from this commit +("changes a non-binding runtime field"). Run the port migration first. + +## 5. Evidence + +- `tests/test_okx_ws_port_migration.py`: 17 tests. They cover the owner, the guard, + `identity_digest` invariance, generated configs, and dry run / apply / no-op / rollback / + refusal on a live-shaped config. +- 24 related modules: 325 tests passed, 1 environment skip, run network-off at `--cpus 1` in + `qdl-v2-python:2.2.3-rc6-10f9f5c`. They include `test_phaseb_stable_edge`, + `test_phase105_identity_acceptance`, `test_stable_runtime_refresh`, + `test_refresh_v2_native_ingestor_runtime`, `test_catalog_regeneration`, + `test_phasec40_authority_bootstrap`/`_cutover` and the 7 modules whose URL assertions were updated. +- Live dry run (read-only mount, no write) on the live OKX ingestor config: + 2 fields change, `config_revision` 20 is kept, identity `056207ce…79c3` is unchanged. +- Status: `tested locally`. The live rollout is not done. From 56a4f9746700c1948590d56065383351fca23b4d Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 08:10:30 +0000 Subject: [PATCH 106/147] docs(plan): stage 100g 31/32 (G5/startup/TS readiness PASS), OKX port 443 live --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index db4c2d9c..2e71f3c4 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71870,3 +71870,32 @@ OKX reconnect (f861663 from 34dbdf9, d4f5ec9 from beeaea4): qdl/runtime/production_catalog.py and config/v2/stable-acquisition-bindings.yaml. The bindings change alters the acquisition plan revision (C.19/C.32 bar-edge checkpoint coupling), so it needs its own packet before 2026-10-31 and is not mixed into rc6. + +### Stage 100g on Query rc7 + Stream rc6: 31/32 (G5, startup and TS readiness PASS); OKX port 443 live - 2026-10-10 (Claude) + +Stack: +- Query query_kn_1/2 on qdl-v2-python:2.2.3-rc7-47bec04 (sha256:4fcf724b6b36): R1/R3/R6 and the G5 replica redirect, + cold slice at the DEFAULT 4 ms (tuning reverted, so the run tests the structural fix alone). +- Stream stream_kn_1/2 on qdl-v2-rust:2.2.3-rc6-10f9f5c (sha256:3dd3c33d48b4): R2. +- Rolled one role at a time with rollout_rc6.py; each healthy, writer READY 64/64 after each. +Run 07:36-08:08Z under supervisor v2; guard all READY 64/64; MARK watcher 0 errors in 32 of 32 minutes. +- PASS: + - G5 cold: 2500 rows B 4849.5 ms (query_v2_1), O 3813.6 ms (query_v2_2); 5000 rows B 7356.7, O 6365.7 ms. + Client side 240-727 ms. + - The redirect fired 63 times (REPLICA_LANE_BUSY: 31 on query_kn_1, 32 on query_kn_2). + - startup:bounded (0 failures, 74.2 s). + - ts:ready_60_every_sample: 154/154 samples, 0 unreadable. + - streams 180, 0 errors; every latency class. +- FAIL requests:no_failures: 1 PARTIAL_RESULT, REFERENCE_BATCH, FUNDING_RATE OKX SOL-USDT-SWAP, query_v2_2, about + 07:46:47Z. + - The hash matches the SDK's "required reference batch contains one or more explicit failures". + - The item's own code was not recorded (an R6 gap at the batch level). No OKX disconnect occurred in the run. + - Per-item diagnostics and root cause are in progress. +- OKX port 8443 retirement (6674d0b packet), done live: + - scripts/migrate_okx_ws_port.py dry-run and then --apply on + releases/v2.2.1-0e00c81-recovery/ingestor_okx_swap.json, through a root --rm container; + - exactly 2 URL fields changed; config_revision 20; identity 056207ce... unchanged; + - sha 0c5d181c... -> d4bd4bae...; backup ingestor_okx_swap.json.pre-okx-port443-1791619322166921472. + - `docker restart` of ingestor_okx_swap at 08:09:07Z (after the run, same image): restarts 0, 5 established + sockets to remote port 443 (0x01BB), 0 to 8443. + - TS endpoint_profiles.py moved too (TS 8b03ff2, pushed). From e04db5b6e84cb7303b681eb5f32fb7581932d607 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 08:12:25 +0000 Subject: [PATCH 107/147] tooling(p184): G13 inherit/rerun matrix and final DR runner (plan only, not run) DR01-DR08 on the final DL + writer digests: 0 of 65 cells inherit (no final-provenance receipt; every DR crosses a role changed since its newest rehearsal). scripts/p184_g13_final.py runs only the existing harnesses, sequentially with hard timeouts, receipts under ~/.local/state/qdl-v2/ p184-g13-/, removes only its own new containers by ID, and refuses on harness image pins (rc2 constants in p184_dr08_isolated.py), checkout drift, deployed-digest drift or missing runtime approval. 16 unit tests. --- scripts/p184_g13_final.py | 579 ++++++++++++++++++ tests/test_p184_g13_final.py | 194 ++++++ .../evidence/p184-g13-inherit-rerun-matrix.md | 134 ++++ 3 files changed, 907 insertions(+) create mode 100644 scripts/p184_g13_final.py create mode 100644 tests/test_p184_g13_final.py create mode 100644 upgrade/evidence/p184-g13-inherit-rerun-matrix.md diff --git a/scripts/p184_g13_final.py b/scripts/p184_g13_final.py new file mode 100644 index 00000000..e573f5f6 --- /dev/null +++ b/scripts/p184_g13_final.py @@ -0,0 +1,579 @@ +#!/usr/bin/env python3 +"""P18.4 gate G13: run the DR01-DR08 steps that cannot be inherited, once, on the final DL + TS writer digests. + +Purpose. G13 (TS upgrade/p184-phase-d-gates.md) closes every DR cell of the frozen R2.5 case matrix +(contracts/baseline/rust-first-v2/p18/p184-r2-case-matrix.v1.json in the TS repo, 65 DR cells) on the final +Data Layer python/rust images and the final TS writer image. The inherit/rerun decision per DR is recorded in +upgrade/evidence/p184-g13-inherit-rerun-matrix.md; DECISIONS below is the same table in code. This runner calls +ONLY the existing harnesses, sequentially, each under a hard timeout: + ISOLATED phase (disposable namespaces, nothing shared is written): + dr05-upstream DL scripts/p184_dr05_dl_fault.py + p184_dr05_reset.py + p184_dr05_channel.py, TS + scripts/p184_dr05_upstream.py (GAP_REFUSAL / GENERATION_CHANGE_FENCED, both venues) + dr08-isolated DL scripts/p184_dr08_isolated.py --orchestrate (QUALIFICATION mode, >= 5 restart samples) + RUNTIME phase (reads the shared stack after the final rollout; needs the owner-approved runtime packet): + dr01-deployed DL scripts/p184_dr01_deployed.py prepare/probe/record (7 DL cells) + dr02-dr03-alpha TS scripts/p184_dr_alpha_reads.py probe (inside each alpha runtime) + record + dr04-dr06-views TS scripts/p184_dr_views.py (inside the TS writer image) + record; also DR05.QUIET_TYPED + dr05-transport TS scripts/p184_dr05_isolated.py (writer clone, transport cut at the clone only) + dr07-reference TS scripts/p184_dr07_reference.py +Receipts: a NEW directory ~/.local/state/qdl-v2/p184-g13-TZ/ (exclusive create) with plan.json, +one /receipt.json per step (argv, cwd, times, exit, timeout, output sha256, cleanup) and summary.json. + +Boundary. Never builds an image, never starts/stops/recreates a shared container, never writes shared Redis/Kafka/ +Postgres, never pushes. It refuses to start a step when (a) a harness still pins other image IDs than the final +ones (the harness containers re-validate their own constants, so a host-side override cannot qualify), (b) the +DL checkout that the harness containers mount at /app is not the release commit or has tracked edits, (c) for +the RUNTIME phase, the deployed roles do not run the final digests or no approval packet is given. Cleanup: only +containers whose name starts with the step's own prefix AND that did not exist before the step are removed, by +exact ID; nothing else (no prune, no image/volume/network removal). The harnesses keep their own cleanup receipts. +A step result is the harness's own rows; this runner never writes a case row and never relabels a rehearsal. + +usage: p184_g13_final.py --python-image sha256:.. --rust-image sha256:.. --python-image-ref qdl-v2-python:2.2.3- + --writer-image sha256:.. --ts-source-sha <40hex> --dl-source-sha <40hex> [--phase isolated|runtime|all] + [--only STEP ...] [--inputs inputs.json] [--runtime-approval PACKET] [--dry-run] [--receipts DIR] +""" +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import signal +import subprocess +import sys +import time +from datetime import datetime, timezone + +DL_ROOT = Path(__file__).resolve().parents[1] +TS_ROOT_DEFAULT = Path("/home/bobby/.worktrees/trading-system-next-upgrade") +STATE_ROOT = Path.home() / ".local/state/qdl-v2" +VENUES = ("BINANCE", "OKX") +DIGEST = re.compile(r"sha256:[0-9a-f]{64}") +SHA = re.compile(r"[0-9a-f]{40}") +STACK = "qdl_v2_stable_candidate-" +WRITER = "market_data_service_p184w" +# Deployed role -> which final image it must run (D5 rollout packet inventory, 2026-10-10). +ROLE_IMAGE = { + "query_kn_1": "python", "query_kn_2": "python", "binance_bar_edge": "python", + "ingestor_binance_usdm": "rust", "ingestor_okx_swap": "rust", + "rust_core": "rust", "rust_core_2": "rust", "rust_core_3": "rust", + "market_projector_1": "rust", "market_projector_2": "rust", + "stream_kn_1": "rust", "stream_kn_2": "rust", +} +# Harness image pins that run inside its containers (scripts/p184_dr08_isolated.py IMAGES; dr05_dl_fault imports it). +PIN_ROLES = {"query": "python", "python": "python", "core": "rust", "stream": "rust"} + + +def cells(case, names, venues=VENUES): + return tuple(f"{case}.{name}@{venue}" for name in names for venue in venues) + + +DR_CELLS = { + "DR01": cells("DR01", ("BINDING_COMPILE", "WRONG_ACCOUNT_DENIED", "WRONG_REALM_DENIED", "WRONG_REVISION_DENIED")) + + ("DR01.NO_GLOBAL_GRANT@ANY",), + "DR02": cells("DR02", ("HISTORY_LIMIT_BOUNDED_BUFFER", "MIXED_ITEM_OUTCOMES_VISIBLE", + "WARMUP_SINGLE_SYMBOL_EXACT_CLOSED_ROWS", "WARMUP_UNIVERSE_CONTEXT_BATCH")), + "DR03": cells("DR03", ("APPEND_1M_NO_FUTURE_SKIP_DUPLICATE", "NO_FULL_HISTORY_RELOAD_PER_CYCLE", + "RESTART_SAME_CANDLE_NO_DUPLICATE_DECISION")), + "DR04": cells("DR04", ("EXPLICIT_LAST", "L2_SNAPSHOT_DELTA_SEQUENCE_DEPTH", "MARK_INDEX_COMPONENTS", "QUOTE_BBO", + "RISK_REREADS_OWN_AUTHORITY")), + "DR05": cells("DR05", ("APPROVED_FALLBACK_AND_RETURN_NO_FORBIDDEN_V1", "DISCONNECT_RECONNECT", "GAP_REFUSAL", + "GENERATION_CHANGE_FENCED", "QUIET_TYPED", "STALE_TYPED")), + "DR06": cells("DR06", ("EXPLICIT_LAST_L2_MARK_STILL_FAIL_WHEN_INVALID", + "UNRELATED_STALE_TRADE_DOES_NOT_BLOCK_MARKET_LIMIT")), + "DR07": cells("DR07", ("FUNDING_RATE", "INSTRUMENT_METADATA_PROVENANCE", "LONG_SHORT_RATIO", "OPEN_INTEREST", + "TAKER_FLOW")) + ("DR07.FX_PROOF@ANY", "DR07.REFERENCE_NEVER_EXECUTION_AUTHORITY@ANY"), + "DR08": tuple(f"DR08.{n}@ANY" for n in ("ALERT_PATH_OBSERVABLE", "CONSUMER_RESTART_RECOVERY_MEASURED", + "NO_CROSS_CONSUMER_STARVATION", "SLOW_CONSUMER_BOUNDED_QUEUE")), +} +# Only the CONTROLLED_SIM p184_dr01_runtime.py emits these, which never closes DR01; the TS Gateway runner that +# should does not exist yet. Listed, never faked. +NO_PRODUCER = cells("DR01", ("WRONG_ACCOUNT_DENIED",)) + +# DR -> (decision, reason). Same table as upgrade/evidence/p184-g13-inherit-rerun-matrix.md. +DECISIONS = { + "DR01": ("RERUN", "no deployed-probe receipt (only CONTROLLED_SIM rehearsal, zero provenance); " + "qdl/api_v2/router.py changed since rc2 43bbcf0"), + "DR02": ("RERUN", "no receipt; warmup/batch path changed (65e0757, 76950c8, 47bec04, SDK 5c81c25/359c9ee)"), + "DR03": ("RERUN", "no receipt; BAR history/cold path changed (76950c8, 65e0757)"), + "DR04": ("RERUN", "no receipt; MARK/INDEX + public view changed (922a6ec, bd74faa, ec89a31, 57881a2, 616342c, " + "1a2daf1) and the writer digest is new"), + "DR05": ("RERUN", "GAP/GENERATION r16 is an ISOLATED_FAULT rehearsal on core overlay baafcb48 (d68fb2f), rc2 " + "python, stream 1f13408f, writer 411a6a8; core 1a2daf1, query 57881a2/922a6ec, stream " + "0b374bc/fdd379e changed since; STALE/DISCONNECT/FALLBACK/QUIET never produced"), + "DR06": ("RERUN", "no receipt; production validator reads the changed MARK/L2 views; new writer digest"), + "DR07": ("RERUN", "no R2 DR07 rows exist (10-07 'DR07' was runtime hardening, a different label); " + "router.py changed"), + "DR08": ("RERUN", "DEVELOPMENT rows only (n=1, zero TS provenance, stream binary overlay); stream fdd379e " + "(hub.rs) changed after the overlay; final mode needs >= 5 restart samples"), +} + +STEP_SPEC = ( + # id, phase, cells, timeout_s, est_minutes, owned container prefixes + ("dr05-upstream", "isolated", tuple(c for c in DR_CELLS["DR05"] if "GAP_" in c or "GENERATION_" in c), 1800, 5, + ("p184-dr05-",)), + ("dr08-isolated", "isolated", DR_CELLS["DR08"], 1500, 12, ("p184-dr08-",)), + ("dr01-deployed", "runtime", tuple(c for c in DR_CELLS["DR01"] if c not in NO_PRODUCER), 600, 3, ()), + ("dr02-dr03-alpha", "runtime", DR_CELLS["DR02"] + DR_CELLS["DR03"], 1200, 10, ()), + ("dr04-dr06-views", "runtime", DR_CELLS["DR04"] + DR_CELLS["DR06"] + cells("DR05", ("QUIET_TYPED",)), 1200, 12, + ()), + ("dr05-transport", "runtime", cells("DR05", ("STALE_TYPED", "DISCONNECT_RECONNECT", + "APPROVED_FALLBACK_AND_RETURN_NO_FORBIDDEN_V1")), 1500, 8, + ("p184-dr05-",)), + ("dr07-reference", "runtime", DR_CELLS["DR07"], 600, 2, ()), +) +STEPS = tuple(s[0] for s in STEP_SPEC) + + +class Blocked(Exception): + """A typed refusal before any step side effect.""" + + +# ---------------------------------------------------------------- pure plan logic (unit-tested) + +def coverage(): + """Every DR cell is either produced by exactly one step or listed in NO_PRODUCER.""" + produced = [c for s in STEP_SPEC for c in s[2]] + every = [c for case in sorted(DR_CELLS) for c in DR_CELLS[case]] + duplicates = sorted({c for c in produced if produced.count(c) > 1}) + missing = sorted(set(every) - set(produced) - set(NO_PRODUCER)) + extra = sorted(set(produced) - set(every)) + return {"cells": len(every), "produced": len(set(produced)), "no_producer": list(NO_PRODUCER), + "duplicates": duplicates, "missing": missing, "extra": extra} + + +def harness_pins(source): + """Image IDs the DR05/DR08 harness containers validate against, read from source without importing it.""" + tree = ast.parse(source) + consts, images = {}, None + for node in tree.body: + if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + name = node.targets[0].id + if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): + consts[name] = node.value.value + elif name == "IMAGES" and isinstance(node.value, ast.Dict): + images = {} + for key, value in zip(node.value.keys, node.value.values): + images[key.value] = consts.get(value.id) if isinstance(value, ast.Name) else value.value + if images is None: + raise Blocked("HARNESS_PINS_UNREADABLE: no IMAGES dict in scripts/p184_dr08_isolated.py") + return {"images": images, "query_image_ref": consts.get("QUERY_IMAGE_REF"), + "query_image_digest": consts.get("QUERY_IMAGE_DIGEST")} + + +def pin_mismatches(pins, final): + """Empty when the harness pins equal the final python/rust images and python image ref.""" + found = [] + for role, kind in PIN_ROLES.items(): + if pins["images"].get(role) != final[kind]: + found.append(f"IMAGES[{role!r}]={pins['images'].get(role)} != final {kind} {final[kind]}") + if pins["query_image_digest"] != final["python"]: + found.append(f"QUERY_IMAGE_DIGEST={pins['query_image_digest']} != final python {final['python']}") + if pins["query_image_ref"] != final["python_ref"]: + found.append(f"QUERY_IMAGE_REF={pins['query_image_ref']} != {final['python_ref']}") + return found + + +def validate_final(final): + for key in ("python", "rust", "writer"): + value = final.get(key) or "" + if not DIGEST.fullmatch(value) or len(set(value[7:])) == 1: + raise Blocked(f"FINAL_DIGEST_REQUIRED: --{key}-image must be a real sha256 image ID") + for key in ("ts_sha", "dl_sha"): + value = final.get(key) or "" + if not SHA.fullmatch(value) or len(set(value)) == 1: + raise Blocked(f"FINAL_SHA_REQUIRED: {key} must be a full 40-hex commit") + if final["python"] == final["rust"]: + raise Blocked("FINAL_DIGEST_REQUIRED: python and rust images must differ") + if not re.fullmatch(r"qdl-v2-python:\S+", final.get("python_ref") or ""): + raise Blocked("FINAL_REF_REQUIRED: --python-image-ref qdl-v2-python:") + + +def deployed_mismatches(inventory, final): + """inventory: {container name: image ID}. Empty when every listed role and the writer run the final digests.""" + found = [] + for role, kind in ROLE_IMAGE.items(): + name = STACK + role + "-1" + if inventory.get(name) != final[kind]: + found.append(f"{name} runs {inventory.get(name)} != final {kind} {final[kind]}") + if inventory.get(WRITER) != final["writer"]: + found.append(f"{WRITER} runs {inventory.get(WRITER)} != final writer {final['writer']}") + return found + + +def select(phase, only=()): + """Ordered step ids: isolated before runtime, matrix order otherwise; --only narrows, never reorders.""" + unknown = sorted(set(only) - set(STEPS)) + if unknown: + raise Blocked("UNKNOWN_STEP: " + ", ".join(unknown)) + phases = {"isolated": ("isolated",), "runtime": ("runtime",), "all": ("isolated", "runtime")}[phase] + chosen = [s[0] for p in phases for s in STEP_SPEC if s[1] == p + and DECISIONS[s[2][0].split(".")[0]][0] == "RERUN" and (not only or s[0] in only)] + if not chosen: + raise Blocked("NOTHING_SELECTED") + return chosen + + +def required_inputs(step): + return { + "dr05-upstream": ("dr05_public_export", "redis_image", "postgres_image"), + "dr08-isolated": (), + "dr01-deployed": ("dr01.inventory", "dr01.deployments.BINANCE", "dr01.deployments.OKX", "dr01.compose", + "dr01.alpha_service"), + "dr02-dr03-alpha": ("alpha.containers.BINANCE", "alpha.containers.OKX", "alpha.symbols.BINANCE", + "alpha.symbols.OKX"), + "dr04-dr06-views": ("ts_container",), + "dr05-transport": ("redis_image", "postgres_image"), + "dr07-reference": ("dr07.connections", "dr07.fx_account"), + }[step] + + +def lookup(inputs, dotted): + value = inputs + for part in dotted.split("."): + if not isinstance(value, dict) or part not in value: + return None + value = value[part] + return value + + +def missing_inputs(steps, inputs): + return {s: [k for k in required_inputs(s) if lookup(inputs, k) in (None, "")] for s in steps + if any(lookup(inputs, k) in (None, "") for k in required_inputs(s))} + + +def cmd(argv, *, cwd, stdin=None, stdout=None): + return {"argv": [str(a) for a in argv], "cwd": str(cwd), "stdin": stdin and str(stdin), + "stdout": stdout and str(stdout)} + + +def commands(step, final, inputs, work, ts_root): + """The exact harness invocations of one step (dr05-upstream's channel/watcher are built at run time).""" + py = [sys.executable, "-B"] + ts_ev = ["--source-sha", final["ts_sha"], "--image-digest", final["writer"]] + alpha_record = ts_root / "scripts/p184_dr_alpha_reads.py" + if step == "dr05-upstream": + w = work / "work" + return [cmd(py + ["scripts/p184_dr05_dl_fault.py", "--init", w, "--core-config", inputs["dr05_public_export"]], + cwd=DL_ROOT), + cmd(py + ["scripts/p184_dr05_dl_fault.py", "--up", w], cwd=DL_ROOT), + cmd(py + ["scripts/p184_dr05_dl_fault.py", "--ready", w], cwd=DL_ROOT), + cmd(py + ["scripts/p184_dr05_upstream.py", "--work", w, "--namespace", "", + "--dl-channel", "", "--image", final["writer"], + "--redis-image", inputs["redis_image"], "--postgres-image", inputs["postgres_image"], + *ts_ev, "--output", work / "dr05-upstream-rows.jsonl", "--dl-root", DL_ROOT], cwd=ts_root), + cmd(py + ["scripts/p184_dr05_dl_fault.py", "--down", w], cwd=DL_ROOT)] + if step == "dr08-isolated": + return [cmd(py + ["scripts/p184_dr08_isolated.py", "--orchestrate", work / "work", "--ts-source-sha", + final["ts_sha"], "--ts-image-digest", final["writer"], "--restart-samples", + str(inputs.get("dr08_restart_samples", 5))], cwd=DL_ROOT)] + if step == "dr01-deployed": + d = inputs["dr01"] + out = [] + for venue in VENUES: + packet = work / f"dr01-{venue.lower()}-packet.json" + prepare = ["scripts/p184_dr01_deployed.py", "prepare", "--root", DL_ROOT, "--inventory", d["inventory"], + "--deployment", d["deployments"][venue], "--venue", venue, "--output", packet] + if d.get("catalog"): + prepare += ["--catalog", d["catalog"]] + out.append(cmd(py + prepare, cwd=DL_ROOT)) + out.append(cmd(["docker", "compose", "-f", d["compose"], "run", "--rm", "--no-deps", "-T", "--entrypoint", + "python", "-v", f"{packet}:/run/p184/dr01-packet.json:ro", d["alpha_service"], "-B", "-", + "probe", "--packet", "/run/p184/dr01-packet.json"], + cwd=d.get("compose_cwd", "/home/bobby/execution_alpha"), + stdin=DL_ROOT / "scripts/p184_dr01_deployed.py", + stdout=work / f"dr01-{venue.lower()}-rows.jsonl")) + record = ["scripts/p184_dr01_deployed.py", "record", "--rows", work / "dr01-binance-rows.jsonl", + work / "dr01-okx-rows.jsonl", "--source-sha", final["ts_sha"], "--image-digest", final["writer"], + "--output", work / "dr01-dl.jsonl"] + if d.get("ci_confirmed") is True: + record.append("--ci-confirmed") + out.append(cmd(py + record, cwd=DL_ROOT)) + return out + if step == "dr02-dr03-alpha": + a = inputs["alpha"] + out = [] + for venue in VENUES: + probe = ["docker", "exec", "-i", a["containers"][venue], "python", "-B", "-", "probe", "--venue", + venue.lower(), "--symbol", a["symbols"][venue]] + if a.get("universe", {}).get(venue): + probe += ["--universe", a["universe"][venue]] + out.append(cmd(probe, cwd=ts_root, stdin=alpha_record, stdout=work / f"alpha-{venue.lower()}.rows")) + out.append(cmd(py + [alpha_record, "record", "--rows", work / "alpha.rows", *ts_ev, "--output", + work / "dr02-dr03.jsonl"], cwd=ts_root)) + return out + if step == "dr04-dr06-views": + out = [cmd(["docker", "exec", "-i", inputs["ts_container"], "python", "-B", "-", "--venue", venue.lower()], + cwd=ts_root, stdin=ts_root / "scripts/p184_dr_views.py", stdout=work / f"views-{venue.lower()}.rows") + for venue in VENUES] + out.append(cmd(py + [alpha_record, "record", "--rows", work / "views.rows", *ts_ev, "--output", + work / "dr04-dr06.jsonl"], cwd=ts_root)) + return out + if step == "dr05-transport": + return [cmd(py + ["scripts/p184_dr05_isolated.py", "--image", final["writer"], "--redis-image", + inputs["redis_image"], "--postgres-image", inputs["postgres_image"], *ts_ev, + "--output", work / "dr05-transport.jsonl"], cwd=ts_root)] + if step == "dr07-reference": + d = inputs["dr07"] + return [cmd(py + ["scripts/p184_dr07_reference.py", "--connections", d["connections"], *ts_ev, "--output", + work / "dr07.jsonl", "--fx-account", d["fx_account"], "--dsn-env", + d.get("dsn_env", "POSTGRES_DSN")], cwd=ts_root)] + raise Blocked("UNKNOWN_STEP: " + step) + + +# Rows printed by the two in-container probes are concatenated per step before `record` (it takes one file). +CONCAT = {"dr02-dr03-alpha": ("alpha.rows", ("alpha-binance.rows", "alpha-okx.rows")), + "dr04-dr06-views": ("views.rows", ("views-binance.rows", "views-okx.rows"))} + + +def dr05_channel_argv(helper_command, python_image, ns, work): + """The JSONL channel inside the DR05 broker namespace (same construction as the 2026-10-09 joint runs).""" + argv = helper_command(ns, work, ["--channel", "/work"], network="container:" + ns + "-kafka") + argv.insert(2, "-i") + argv[argv.index("scripts/p184_dr05_dl_fault.py")] = "scripts/p184_dr05_channel.py" + pos = argv.index(python_image) + argv[pos:pos] = ["-e", "P184_DR05_ISOLATED_NAMESPACE=" + ns] + return argv + + +def new_owned(before, after, prefixes): + """Containers the step created and left behind: owned prefix, absent before the step. {id: name}.""" + return {cid: name for cid, name in after.items() + if cid not in before and any(name.startswith(p) for p in prefixes)} + + +def receipt_root(base, now): + return Path(base) / ("p184-g13-" + now.strftime("%Y%m%dT%H%M%SZ")) + + +def plan(final, inputs, phase, only, ts_root, base, now, pins): + steps = select(phase, only) + spec = {s[0]: s for s in STEP_SPEC} + root = receipt_root(base, now) + problems = {} + mismatch = pin_mismatches(pins, final) + missing = missing_inputs(steps, inputs) + out = [] + for step in steps: + _, ph, step_cells, timeout, est, prefixes = spec[step] + blocked = [] + if step in ("dr05-upstream", "dr08-isolated") and mismatch: + blocked.append("HARNESS_IMAGE_PIN: " + "; ".join(mismatch)) + if step in missing: + blocked.append("MISSING_INPUTS: " + ", ".join(missing[step])) + if ph == "runtime" and not inputs.get("_runtime_approval"): + blocked.append("RUNTIME_APPROVAL_REQUIRED: --runtime-approval ") + entry = {"step": step, "phase": ph, "cells": list(step_cells), "timeout_s": timeout, "est_minutes": est, + "owned_prefixes": list(prefixes), "blocked": blocked} + if not blocked: + entry["commands"] = commands(step, final, inputs, root / step, ts_root) + out.append(entry) + if blocked: + problems[step] = blocked + return {"schema": "p184.g13-final-plan.v1", "receipts": str(root), "final": final, + "decisions": {k: {"decision": v[0], "reason": v[1]} for k, v in DECISIONS.items()}, + "coverage": coverage(), "steps": out, "blocked": problems, + "est_minutes_total": sum(e["est_minutes"] for e in out), + "timeout_minutes_total": sum(e["timeout_s"] for e in out) // 60} + + +# ---------------------------------------------------------------- side effects (only without --dry-run) + +def sh(argv, timeout=60): + return subprocess.run(argv, check=True, text=True, capture_output=True, timeout=timeout).stdout + + +def containers(): + rows = sh(["docker", "ps", "-a", "--no-trunc", "--format", "{{.ID}} {{.Names}}"]).split("\n") + return dict(r.split(" ", 1) for r in rows if " " in r) + + +def deployed_inventory(): + names = [STACK + r + "-1" for r in ROLE_IMAGE] + [WRITER] + out = {} + for name in names: + try: + out[name] = sh(["docker", "inspect", "--format", "{{.Image}}", name]).strip() + except subprocess.CalledProcessError: + out[name] = None + return out + + +def checkout_problems(dl_sha): + head = sh(["git", "-C", str(DL_ROOT), "rev-parse", "HEAD"]).strip() + dirty = sh(["git", "-C", str(DL_ROOT), "status", "--porcelain", "--untracked-files=no", "--", "qdl", "rust", + "scripts", "generated", "consumers", "config", "contracts"]).strip() + found = [] + if head != dl_sha: + found.append(f"DL checkout HEAD {head} != --dl-source-sha {dl_sha} (harness containers mount it at /app)") + if dirty: + found.append("DL checkout has tracked edits under mounted paths:\n" + dirty) + return found + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + h.update(block) + return h.hexdigest() + + +def bounded(c, deadline, log): + """One harness command in its own process group; killed (TERM, then KILL) at the step deadline.""" + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("step deadline") + stdin = open(c["stdin"], "rb") if c["stdin"] else subprocess.DEVNULL + stdout = open(c["stdout"], "xb") if c["stdout"] else log + try: + proc = subprocess.Popen(c["argv"], cwd=c["cwd"], stdin=stdin, stdout=stdout, stderr=log, + start_new_session=True) + try: + return proc.wait(remaining) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(15) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait(15) + raise TimeoutError("step deadline: " + shlex.join(c["argv"][:4])) + finally: + if c["stdin"]: + stdin.close() + if c["stdout"]: + stdout.close() + + +def run_dr05_upstream(entry, final, inputs, work, deadline, log, ts_root): + """init -> up -> ready -> reset watcher -> TS adapter over the channel; --down always runs.""" + sys.path.insert(0, str(DL_ROOT)) + from scripts import p184_dr05_dl_fault as dr # host-side helpers only (stdlib imports) + init, up, ready, adapter, down = entry["commands"] + watcher, codes = None, [] + try: + for c in (init, up, ready): + codes.append(bounded(c, deadline, log)) + if codes[-1]: + raise RuntimeError("DR05 DL stack step failed: " + c["argv"][3]) + ns = json.loads((work / "work/handoff.json").read_text())["namespace"] + watcher = subprocess.Popen([sys.executable, "-B", "scripts/p184_dr05_reset.py", "--serve-reset-host", + str(work / "work"), "--duration", str(int(max(60, deadline - time.monotonic())))], + cwd=DL_ROOT, stdout=log, stderr=log, start_new_session=True) + channel = shlex.join(dr05_channel_argv(dr.helper_command, dr.IMAGES["python"], ns, work / "work")) + argv = [ns if a == "" else channel if a == "" else a + for a in adapter["argv"]] + codes.append(bounded({**adapter, "argv": argv}, deadline, log)) + finally: + if watcher is not None and watcher.poll() is None: + os.killpg(watcher.pid, signal.SIGTERM) + try: + watcher.wait(30) + except subprocess.TimeoutExpired: + os.killpg(watcher.pid, signal.SIGKILL) + if (work / "work/handoff.json").exists(): + codes.append(bounded(down, time.monotonic() + 600, log)) + return codes + + +def run_step(entry, final, inputs, ts_root): + work = Path(inputs["_root"]) / entry["step"] + work.mkdir(mode=0o700) + before = containers() + clash = {cid: n for cid, n in before.items() if any(n.startswith(p) for p in entry["owned_prefixes"])} + receipt = {"step": entry["step"], "cells": entry["cells"], "commands": entry["commands"], + "started_utc": datetime.now(timezone.utc).isoformat(), "preexisting_owned": clash} + t0 = time.monotonic() + deadline = t0 + entry["timeout_s"] + codes, error = [], None + with open(work / "harness.log", "ab") as log: + try: + if clash: + raise Blocked("OWNED_PREFIX_IN_USE: " + ", ".join(sorted(clash.values()))) + if entry["step"] == "dr05-upstream": + codes = run_dr05_upstream(entry, final, inputs, work, deadline, log, ts_root) + else: + for c in entry["commands"]: + if c["argv"][2:3] == [str(ts_root / "scripts/p184_dr_alpha_reads.py")] and entry["step"] in CONCAT: + target, parts = CONCAT[entry["step"]] + with open(work / target, "xb") as joined: + for part in parts: + joined.write((work / part).read_bytes()) + codes.append(bounded(c, deadline, log)) + if codes[-1]: + break + except (Blocked, TimeoutError, RuntimeError, OSError, subprocess.SubprocessError) as exc: + error = f"{type(exc).__name__}: {exc}" + removed = {} + for cid, name in new_owned(before, containers(), entry["owned_prefixes"]).items(): + subprocess.run(["docker", "rm", "-f", cid], capture_output=True, text=True, timeout=60) + removed[cid] = name + outputs = {p.name: sha256(p) for p in sorted(work.iterdir()) if p.is_file() and p.name != "receipt.json"} + receipt.update(ended_utc=datetime.now(timezone.utc).isoformat(), duration_s=round(time.monotonic() - t0, 1), + exit_codes=codes, error=error, outputs_sha256=outputs, removed_leftovers=removed, + harness_ok=error is None and bool(codes) and not any(codes)) + (work / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + return receipt + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--python-image", required=True) + p.add_argument("--rust-image", required=True) + p.add_argument("--python-image-ref", required=True) + p.add_argument("--writer-image", required=True) + p.add_argument("--ts-source-sha", required=True) + p.add_argument("--dl-source-sha", required=True) + p.add_argument("--phase", choices=("isolated", "runtime", "all"), default="all") + p.add_argument("--only", nargs="+", default=()) + p.add_argument("--inputs", type=Path) + p.add_argument("--runtime-approval", type=Path) + p.add_argument("--ts-root", type=Path, default=TS_ROOT_DEFAULT) + p.add_argument("--receipts", type=Path, default=STATE_ROOT) + p.add_argument("--dry-run", action="store_true") + a = p.parse_args(argv) + final = {"python": a.python_image, "rust": a.rust_image, "python_ref": a.python_image_ref, + "writer": a.writer_image, "ts_sha": a.ts_source_sha, "dl_sha": a.dl_source_sha} + inputs = json.loads(a.inputs.read_text()) if a.inputs else {} + if a.runtime_approval: + inputs["_runtime_approval"] = {"path": str(a.runtime_approval), "sha256": sha256(a.runtime_approval)} + try: + validate_final(final) + pins = harness_pins((DL_ROOT / "scripts/p184_dr08_isolated.py").read_text()) + result = plan(final, inputs, a.phase, tuple(a.only), a.ts_root, a.receipts, datetime.now(timezone.utc), pins) + except Blocked as exc: + print(json.dumps({"blocked": str(exc)})) + return 2 + if a.dry_run: + print(json.dumps(result, indent=2, default=str)) + return 0 if not result["blocked"] else 3 + extra = checkout_problems(final["dl_sha"]) + if any(e["phase"] == "runtime" for e in result["steps"]) or "dr08-isolated" in [e["step"] for e in result["steps"]]: + # DR08 copies the deployed core config and refuses a core image other than its pin. + extra += deployed_mismatches(deployed_inventory(), final) + if result["blocked"] or extra: + print(json.dumps({"blocked": result["blocked"], "preflight": extra}, indent=2)) + return 3 + root = Path(result["receipts"]) + root.mkdir(mode=0o700, parents=False, exist_ok=False) + inputs["_root"] = str(root) + (root / "plan.json").write_text(json.dumps(result, indent=2, default=str) + "\n") + receipts = [] + for entry in result["steps"]: + receipts.append(run_step(entry, final, inputs, a.ts_root)) + print(json.dumps({k: receipts[-1][k] for k in ("step", "duration_s", "exit_codes", "error", "harness_ok")}), + flush=True) + summary = {"schema": "p184.g13-final-summary.v1", "final": final, "steps": [ + {k: r[k] for k in ("step", "cells", "duration_s", "exit_codes", "error", "harness_ok", "removed_leftovers")} + for r in receipts], "note": "harness_ok is the harness exit status; cells close only via the TS matrix check"} + (root / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + return 0 if all(r["harness_ok"] for r in receipts) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_p184_g13_final.py b/tests/test_p184_g13_final.py new file mode 100644 index 00000000..49bb1851 --- /dev/null +++ b/tests/test_p184_g13_final.py @@ -0,0 +1,194 @@ +"""Plan logic of scripts/p184_g13_final.py: selection, coverage, image-pin and deployment refusals, argv, cleanup.""" +from datetime import datetime, timezone +import io +import json +from pathlib import Path +import tempfile +import unittest +from contextlib import redirect_stdout + +from scripts import p184_g13_final as g13 + +PY = "sha256:" + "a1" * 32 +RUST = "sha256:" + "b2" * 32 +WRITER = "sha256:" + "c3" * 32 +FINAL = {"python": PY, "rust": RUST, "python_ref": "qdl-v2-python:2.2.3-abcdef1", "writer": WRITER, + "ts_sha": "1" * 39 + "2", "dl_sha": "3" * 39 + "4"} +PINS_OK = {"images": {"query": PY, "python": PY, "core": RUST, "stream": RUST, "kafka": "sha256:" + "d" * 64}, + "query_image_ref": FINAL["python_ref"], "query_image_digest": PY} +INPUTS = {"dr05_public_export": "/x/export.json", "redis_image": "redis:7.2-alpine", "postgres_image": "pg@sha256:1", + "dr01": {"inventory": "/x/inv.json", "deployments": {"BINANCE": "db", "OKX": "do"}, "compose": "/x/c.json", + "alpha_service": "alpha"}, + "alpha": {"containers": {"BINANCE": "ab", "OKX": "ao"}, "symbols": {"BINANCE": "ETHUSDT", + "OKX": "ETH-USDT-SWAP"}}, + "ts_container": "market_data_service_p184w", + "dr07": {"connections": "/x/conn.json", "fx_account": "acct"}, + "_runtime_approval": {"path": "/x/packet.md", "sha256": "e" * 64}} +NOW = datetime(2026, 10, 10, 9, 0, 0, tzinfo=timezone.utc) + + +def make_plan(phase="all", only=(), inputs=None, pins=None): + return g13.plan(FINAL, INPUTS if inputs is None else inputs, phase, only, Path("/ts"), Path("/state"), NOW, + PINS_OK if pins is None else pins) + + +class CoverageTests(unittest.TestCase): + def test_every_dr_cell_has_one_producer_or_is_listed_open(self): + cov = g13.coverage() + self.assertEqual(cov["cells"], 65) # frozen TS matrix p184-r2-case-matrix.v1: 65 DR cells + self.assertEqual((cov["duplicates"], cov["missing"], cov["extra"]), ([], [], [])) + self.assertEqual(cov["produced"] + len(cov["no_producer"]), 65) + self.assertEqual(cov["no_producer"], ["DR01.WRONG_ACCOUNT_DENIED@BINANCE", "DR01.WRONG_ACCOUNT_DENIED@OKX"]) + + def test_every_dr_has_a_decision(self): + self.assertEqual(sorted(g13.DECISIONS), sorted(g13.DR_CELLS)) + + +class SelectionTests(unittest.TestCase): + def test_isolated_runs_before_runtime_in_matrix_order(self): + self.assertEqual(g13.select("all"), list(g13.STEPS)) + self.assertEqual(g13.select("isolated"), ["dr05-upstream", "dr08-isolated"]) + self.assertEqual(g13.select("all", ("dr07-reference", "dr08-isolated")), ["dr08-isolated", "dr07-reference"]) + + def test_unknown_or_empty_selection_refused(self): + with self.assertRaises(g13.Blocked): + g13.select("all", ("dr09",)) + with self.assertRaises(g13.Blocked): + g13.select("isolated", ("dr07-reference",)) + + def test_inherited_dr_is_not_selected(self): + original = g13.DECISIONS["DR07"] + g13.DECISIONS["DR07"] = ("INHERIT", "test") + try: + self.assertNotIn("dr07-reference", g13.select("all")) + finally: + g13.DECISIONS["DR07"] = original + + +class FinalIdentityTests(unittest.TestCase): + def test_development_zero_digests_and_shas_refused(self): + for key, value in (("python", "sha256:" + "0" * 64), ("writer", "latest"), ("ts_sha", "0" * 40), + ("dl_sha", "abc"), ("rust", PY), ("python_ref", "2.2.3")): + with self.subTest(key=key), self.assertRaises(g13.Blocked): + g13.validate_final({**FINAL, key: value}) + g13.validate_final(FINAL) + + def test_harness_pins_read_without_import(self): + source = ('QUERY_IMAGE_REF = "qdl-v2-python:2.2.3-rc2-43bbcf0"\nQUERY_IMAGE_DIGEST = "sha256:77"\n' + 'IMAGES = {"query": QUERY_IMAGE_DIGEST, "core": "sha256:65", "stream": "sha256:1f",' + ' "python": "sha256:77"}\nimport not_installed_module\n') + pins = g13.harness_pins(source) + self.assertEqual(pins["images"]["query"], "sha256:77") + self.assertEqual(pins["query_image_ref"], "qdl-v2-python:2.2.3-rc2-43bbcf0") + self.assertEqual(len(g13.pin_mismatches(pins, FINAL)), 6) + with self.assertRaises(g13.Blocked): + g13.harness_pins("X = 1\n") + + def test_current_harness_pins_are_rc2_and_block_isolated_steps(self): + pins = g13.harness_pins((g13.DL_ROOT / "scripts/p184_dr08_isolated.py").read_text()) + self.assertTrue(g13.pin_mismatches(pins, FINAL)) + result = make_plan("isolated", pins=pins) + self.assertEqual(sorted(result["blocked"]), ["dr05-upstream", "dr08-isolated"]) + self.assertTrue(all("commands" not in e for e in result["steps"])) + self.assertTrue(all(b[0].startswith("HARNESS_IMAGE_PIN") for b in result["blocked"].values())) + + def test_deployed_inventory_must_match_final(self): + good = {g13.STACK + r + "-1": (PY if k == "python" else RUST) for r, k in g13.ROLE_IMAGE.items()} + good[g13.WRITER] = WRITER + self.assertEqual(g13.deployed_mismatches(good, FINAL), []) + bad = {**good, g13.STACK + "rust_core_2-1": "sha256:" + "6" * 64, g13.WRITER: None} + self.assertEqual(len(g13.deployed_mismatches(bad, FINAL)), 2) + + +class PlanTests(unittest.TestCase): + def test_full_plan_binds_final_identity_everywhere(self): + result = make_plan() + self.assertEqual(result["blocked"], {}) + self.assertEqual(result["receipts"], "/state/p184-g13-20261010T090000Z") + self.assertEqual(result["est_minutes_total"], sum(s[4] for s in g13.STEP_SPEC)) + steps = {e["step"]: e for e in result["steps"]} + dr08 = steps["dr08-isolated"]["commands"][0]["argv"] + self.assertNotIn("--development", dr08) + self.assertNotIn("--stream-binary", dr08) + self.assertEqual(dr08[dr08.index("--restart-samples") + 1], "5") + self.assertEqual(dr08[dr08.index("--ts-image-digest") + 1], WRITER) + upstream = steps["dr05-upstream"]["commands"] + self.assertEqual([c["argv"][3] for c in upstream[:3] + upstream[4:]], ["--init", "--up", "--ready", "--down"]) + self.assertNotIn("--core-binary", sum((c["argv"] for c in upstream), [])) + adapter = upstream[3]["argv"] + self.assertEqual(adapter[adapter.index("--image") + 1], WRITER) + self.assertEqual(adapter[adapter.index("--source-sha") + 1], FINAL["ts_sha"]) + for step in ("dr02-dr03-alpha", "dr04-dr06-views", "dr05-transport", "dr07-reference"): + argv = steps[step]["commands"][-1]["argv"] + self.assertEqual(argv[argv.index("--image-digest") + 1], WRITER, step) + dr01 = steps["dr01-deployed"]["commands"] + self.assertEqual(len(dr01), 5) + self.assertEqual(dr01[1]["cwd"], "/home/bobby/execution_alpha") + self.assertNotIn("--ci-confirmed", dr01[-1]["argv"]) + self.assertTrue(all(c["argv"][-1] != "-" for c in dr01)) + for e in result["steps"]: + for c in e["commands"]: + outputs = [a for a in c["argv"] if a.startswith(result["receipts"])] + [c["stdout"] or ""] + self.assertTrue(all(o == "" or o.startswith(result["receipts"] + "/" + e["step"]) for o in outputs)) + + def test_runtime_requires_approval_and_inputs(self): + inputs = {k: v for k, v in INPUTS.items() if k not in ("_runtime_approval", "dr07")} + result = make_plan("runtime", inputs=inputs) + self.assertEqual(sorted(result["blocked"]), sorted(s for s in g13.STEPS if s not in g13.select("isolated"))) + self.assertIn("MISSING_INPUTS: dr07.connections, dr07.fx_account", result["blocked"]["dr07-reference"]) + isolated = make_plan("isolated", inputs={k: v for k, v in INPUTS.items() if k != "_runtime_approval"}) + self.assertEqual(isolated["blocked"], {}) + + def test_concatenation_targets_match_record_rows(self): + steps = {e["step"]: e for e in make_plan()["steps"]} + for step, (target, parts) in g13.CONCAT.items(): + commands = steps[step]["commands"] + self.assertEqual([Path(c["stdout"]).name for c in commands[:2]], list(parts)) + record = commands[-1]["argv"] + self.assertEqual(Path(record[record.index("--rows") + 1]).name, target) + + +class HelperTests(unittest.TestCase): + def test_channel_argv_matches_joint_construction(self): + def helper_command(ns, work, args, *, network): + return ["docker", "run", "--rm", "--network", network, "--entrypoint", "python", "IMG", "-B", + "scripts/p184_dr05_dl_fault.py", *args] + argv = g13.dr05_channel_argv(helper_command, "IMG", "p184-dr05-dl-0123456789abcdef", Path("/w")) + self.assertEqual(argv[:3], ["docker", "run", "-i"]) + self.assertIn("scripts/p184_dr05_channel.py", argv) + self.assertNotIn("scripts/p184_dr05_dl_fault.py", argv) + self.assertEqual(argv[argv.index("IMG") - 2:argv.index("IMG")], + ["-e", "P184_DR05_ISOLATED_NAMESPACE=p184-dr05-dl-0123456789abcdef"]) + self.assertIn("container:p184-dr05-dl-0123456789abcdef-kafka", argv) + + def test_cleanup_only_new_owned_containers(self): + before = {"1": "p184-dr05-old", "2": "qdl_v2_stable_candidate-rust_core-1"} + after = {**before, "3": "p184-dr05-writer-x", "4": "unrelated", "5": "p184-dr08-y"} + self.assertEqual(g13.new_owned(before, after, ("p184-dr05-",)), {"3": "p184-dr05-writer-x"}) + self.assertEqual(g13.new_owned(before, after, ()), {}) + + def test_dry_run_prints_plan_and_creates_nothing(self): + with tempfile.TemporaryDirectory() as tmp: + out = io.StringIO() + with redirect_stdout(out): + code = g13.main(["--python-image", PY, "--rust-image", RUST, "--python-image-ref", + FINAL["python_ref"], "--writer-image", WRITER, "--ts-source-sha", FINAL["ts_sha"], + "--dl-source-sha", FINAL["dl_sha"], "--phase", "isolated", "--receipts", tmp, + "--dry-run"]) + result = json.loads(out.getvalue()) + self.assertEqual(code, 3) # the checked-in harness still pins rc2 images + self.assertEqual(sorted(result["blocked"]), ["dr05-upstream", "dr08-isolated"]) + self.assertEqual(list(Path(tmp).iterdir()), []) + + def test_bad_identity_exits_before_plan(self): + out = io.StringIO() + with redirect_stdout(out): + code = g13.main(["--python-image", "sha256:" + "0" * 64, "--rust-image", RUST, "--python-image-ref", + "x", "--writer-image", WRITER, "--ts-source-sha", FINAL["ts_sha"], "--dl-source-sha", + FINAL["dl_sha"], "--dry-run"]) + self.assertEqual(code, 2) + self.assertIn("FINAL_DIGEST_REQUIRED", out.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/upgrade/evidence/p184-g13-inherit-rerun-matrix.md b/upgrade/evidence/p184-g13-inherit-rerun-matrix.md new file mode 100644 index 00000000..4c3ecb66 --- /dev/null +++ b/upgrade/evidence/p184-g13-inherit-rerun-matrix.md @@ -0,0 +1,134 @@ +# P18.4 G13 - DR01-DR08 inherit / rerun matrix (final DL + writer digests) + +Status: PLAN / NOT RUN (2026-10-10, Claude coordinator, read-only planning). No DR, build or runtime action was +taken to produce this. Runner: `scripts/p184_g13_final.py` (tested locally, 16 unit tests; never executed against +Docker). Gate: G13 in TS `upgrade/p184-phase-d-gates.md` - "DR01-DR08 (incl. DR05 typed gap/generation fence, +inverse feeds): every cell closed once on the final DL + writer digests", frozen cells in TS +`contracts/baseline/rust-first-v2/p18/p184-r2-case-matrix.v1.json` (65 DR cells: DR01 9, DR02 8, DR03 6, DR04 10, +DR05 12, DR06 4, DR07 12, DR08 4; counted with a Python read of that file). + +## Result in one line + +**Inherit 0 of 65 cells.** No DR cell has a receipt on final-qualifying provenance, and every DR exercises at least +one path changed after its newest receipt. What can be inherited are inputs, not cells (see "Inherited inputs"). +The minimum is therefore seven harness steps, ~52 min of estimated wall time run serially (hard timeouts sum to +140 min), plus the two DR01 WRONG_ACCOUNT_DENIED cells that have no producer anywhere. + +## Receipts that exist (all found with `grep -rl` over `~/.local/state` for `DR0n.` cell ids) + +| DR | Newest receipt | Provenance | Why it cannot close a G13 cell | +|---|---|---|---| +| DR01 | `~/.local/state/qdl-v2/p184-dr01-20261008-agent/dr01.rehearsal.jsonl` (10-08 05:06Z) | `p184_dr01_runtime`, `development: true`, source/image all zeroes | Runbook `p184-dr-runtime-runbook.md`: CONTROLLED_SIM must NEVER close a DR01 runtime cell; the deployed probe (`p184_dr01_deployed.py`) never ran | +| DR02, DR03, DR04, DR06 | none | - | No rows exist (runners `p184_dr_alpha_reads.py`, `p184_dr_views.py` only dry-run, TS plan "DR runners DR02-DR04/DR06", "not evidence") | +| DR05 GAP/GENERATION | `~/.local/state/trading-system/p184-dr05-joint-20261009/r16/dr05-rows.jsonl` (10-09 12:27Z), 4/4 ok at ISOLATED_FAULT | writer `411a6a8` / `sha256:b5274bb9...`; DL python `77682eea` (rc2), core image `658a9570` + binary overlay `baafcb48` (built from `d68fb2f`), stream `1f13408f` (`r16/work/handoff.json`) | Rehearsal on non-final DL and writer; never relabelled (runbook) | +| DR05 STALE/DISCONNECT/FALLBACK/QUIET | none | - | `p184_dr05_isolated.py` / `p184_dr_views.py` quiet watch never produced rows | +| DR07 | none for R2 DR07 | - | `~/.local/state/qdl-v2/dr07-*-20261007` is the runtime-hardening "DR07", a different label (TS plan, "DR label warning") | +| DR08 | `~/.local/state/qdl-v2/dr08-continuity-20261008/candidate-wire-02/evidence.jsonl` (10-08 13:46Z), dev attempts 1-12 | `mode: DEVELOPMENT`, TS zeroes, `stream_binary_override` `b5e1c2c4`, n=1 restart | Development rows deliberately emit `ok=false` (`p184_dr08_isolated.py` `evidence_rows`); final mode needs >= 5 restarts | + +## DL changes after the receipts (DL `fix/p184-dl-closure`, HEAD `6674d0b`) + +Commands: `git log --oneline 43bbcf0..HEAD -- qdl rust qdl_sdk consumers config contracts Dockerfile*` +(rc2 source `43bbcf0` = IDR-062 top-change materialization, already inside every rc2-based receipt), +`git log --oneline d68fb2f..HEAD -- rust/qdl-core rust/qdl-realtime-core rust/qdl-kafka/src/bin/qdl-production-core.rs`, +`git log --oneline 0b374bc..HEAD -- rust/qdl-stream-gateway`. + +| Item | Commit(s) | Changed paths (role) | +|---|---|---| +| IDR-040 fix 1 | `4eb08dc` (prerequisite, before rc2) | in all receipts | +| IDR-040 fix 2 (proof flag OFF) | `1a2daf1` | core `rust/qdl-core/src/{canonical,l2_adapter}.rs`, `rust/qdl-realtime-core/src/lib.rs`, ingestor, query `qdl/query/{results,service}.py`, `qdl/runtime/stable_source.py`, SDK models/projection | +| IDR-062 core snapshot-on-top-change | `43bbcf0`, `63238bf` (l2_book copies) | core; `63238bf` is after rc2 | +| IDR-082 event_id view field | `616342c` | query router/results/stable_source, SDK | +| IDR-084 hot MARK | `922a6ec`, `bd74faa`, `ec89a31` | query `qdl/runtime/kn_query_backend.py` | +| Astra R1/R3/R6 | `57881a2` | query `kn_hot_view`, `kn_query_backend`, `kn_read_view`, `execution_mark_index`, router | +| Astra R2 | `fdd379e` | stream `rust/qdl-stream-gateway/src/hub.rs` | +| Astra R4 | `d4f5ec9` | ingestor `qdl-native-raw-ingestor.rs` | +| OKX make-before-break + session lineage | `f861663` | ingestor, `qdl/runtime/session_liveness.py` | +| G5 replica redirect | `47bec04` | query lanes/service/router, SDK transport | +| OKX port 443 | `6674d0b` | `config/v2/stable-acquisition-bindings.yaml` (URLs only, plan revision 19 kept), OKX client, production catalog | +| G5 warmup/BAR perf | `65e0757`, `76950c8`, `5c81c25` | query row cache / cold BAR, SDK transport | +| DR08 terminal status | `0b374bc` | stream `service.rs` | +| SDK 2.2.0 | `359c9ee` | SDK package metadata | +| heartbeat liveness | not in HEAD `6674d0b` (in progress) | if it lands before the build it changes ingestor/session paths; covered by the RUNTIME steps, no extra step, but it must be inside the final digests | + +Unchanged since rc2 (checked, empty `git log`): `qdl/security`, `qdl/consumer`, `consumers/`, +`qdl/reference`, `qdl/adapters/okx/reference.py`, `qdl/canonical/reference.py`, `qdl/adapters/binance`. +They do not rescue any DR, because each DR also crosses a changed role (router, query backend, core or stream) +and has no final-provenance receipt to inherit from. + +## Matrix + +| DR | Cells | Exercised paths (roles) | Receipt | Decision | Reason | Step / est. min | +|---|---|---|---|---|---|---| +| DR01 | 9 (7 DL + 2 WRONG_ACCOUNT) | alpha SDK credentials -> Query HTTP `qdl/api_v2/router.py`, `qdl/security/data_plane.py` (JWT/realm/revision/entitlement), binding compiler `qdl/consumer/universal_release.py`, `consumers/stable/alpha-*-paper.yaml` | rehearsal only | RERUN | no deployed-probe receipt; router changed (`616342c`, `65e0757`, `76950c8`, `57881a2`, `47bec04`) | `dr01-deployed` / 3 | +| DR02 | 8 | alpha `AlphaDataLayerV2` -> SDK transport -> Query warmup/batch, row cache, cold BAR (`qdl/query/*`, `kn_query_backend`, `stable_source`) | none | RERUN | warmup/batch path and SDK transport changed | `dr02-dr03-alpha` / 10 | +| DR03 | 6 | same as DR02 plus 1m append (Query BAR, projector-fed spool) | none | RERUN | same | (same step) | +| DR04 | 10 | Stream/Query -> TS writer -> TS Redis -> `MarketCacheReader` + Risk validator; MARK/INDEX hot path, L2 view | none | RERUN | IDR-084/Astra MARK, IDR-082, IDR-040 fix 2 changed; writer digest new | `dr04-dr06-views` / 12 | +| DR05 | 12 | GAP/GEN: raw -> core fence (`canonical.rs`, `realtime-core/lib.rs`, `qdl-production-core.rs`) -> stream/query `kn_hot_view` -> writer clone -> production `_validate_book`; STALE/DISCONNECT/FALLBACK: writer clone transport; QUIET: TS views | r16 rehearsal (4 cells) | RERUN | core `1a2daf1`, query `57881a2`/`922a6ec`, stream `0b374bc`/`fdd379e`, writer all changed after r16; 8 cells never produced | `dr05-upstream` / 5, `dr05-transport` / 8, QUIET inside `dr04-dr06-views` | +| DR06 | 4 | production `services.risk_engine.execution_market_context` on a copy of the live context (writer views) | none | RERUN | reads changed MARK/L2 views; writer digest new | (in `dr04-dr06-views`) | +| DR07 | 12 | alpha `alpha_sdk.reference` -> Query reference routes (`router.py`) -> `qdl/reference/*`; FX proof from TS DB | none | RERUN | no R2 DR07 rows; router changed | `dr07-reference` / 2 | +| DR08 | 4 | isolated kafka -> core -> stream `service.rs`/`subscription.rs`/`hub.rs` -> query GetSnapshot | development only | RERUN | stream `fdd379e` after the overlay; final mode never ran | `dr08-isolated` / 12 | + +Estimated minutes: from measured runs where they exist (DR05 joint r16: 236 s init-to-down from +`r16/orchestrator.jsonl`; DR08 dev ~3 min for n=1, driver cap 660 s for n=5) and otherwise from harness defaults +(DR02/03 `--append-seconds 150` + `--restart-seconds 90` per venue; views `--quiet-seconds 300` per venue; DR05 +transport `--ready-seconds 180` + 2 x (20 s cut + <= 120 s recover)). These are estimates, not measurements. + +## Inherited inputs (not cells) + +- DR05 sanitized core export `~/.local/state/qdl-v2/dr05-observer-20261008/public-export.json` (SHA256 `9cf857a2...`, + BINANCE ETHUSDT and OKX ETH-USDT-SWAP BOOK bindings, no transport). `6674d0b` changed only OKX transport URLs and + `consumers/` is unchanged, so the export stays valid; TLS/JWT material is regenerated per run by the harness. +- DR05/DR08 harnesses are reused as they are at the release commit (latest DR05 harness changes: `522cf3e` D6 + reset stderr, `75673bc` G12 helper); no new harness is written for G13. +- Unit tests of the harnesses are not rerun for G13; they gate the build CI (`scripts/p184_local_ci.py`). + +## Blockers before the final run (none is fixed here) + +1. **Harness image pins.** `scripts/p184_dr08_isolated.py` pins `IMAGES` (rc2 python `77682eea`, core `658a9570`, + stream `1f13408f`) and `QUERY_IMAGE_REF/DIGEST` (2.2.3-rc2-43bbcf0); `p184_dr05_dl_fault.py` imports the same + `IMAGES`. The containers re-validate against those constants (`validate_query_provenance`, readiness image + check), so the runner cannot override them from the host. Needs an additive, reviewed change in the harness + (final image IDs as arguments, constants as defaults) before `dr05-upstream`/`dr08-isolated` can run. The + runner refuses with `HARNESS_IMAGE_PIN` until then (its dry run shows this today). +2. **DR08 reads the deployed core.** `orchestrate` copies `/runtime/core.json` from `qdl_v2_stable_candidate-rust_core-1` + and refuses a core image other than its pin; with final pins DR08 runs only after the final rust image is + deployed. The runner therefore checks the deployed inventory before DR08 too. +3. **Python code is mounted from the checkout** (`-v
:/app:ro`, `PYTHONPATH=/app`) in DR05/DR08: the + checkout must sit at the release commit. The runner refuses if HEAD differs from `--dl-source-sha` or tracked + files under `qdl rust scripts generated consumers config contracts` are modified. +4. **DR01.WRONG_ACCOUNT_DENIED@BINANCE/OKX have no qualifying producer.** `grep -rln WRONG_ACCOUNT` finds only + the CONTROLLED_SIM `scripts/p184_dr01_runtime.py` (DL), which the runbook forbids for closing DR01, and no TS + script outside the case matrix. The runbook assigns them to a TS Gateway runner that does not exist yet. +5. **Inverse feeds.** G13 names inverse feeds, but the frozen matrix v1 has no inverse-scoped DR cell and + `p184_dr_views.py` `IDENTITY` is linear only (ETHUSDT / ETH-USDT-SWAP). Either a matrix revision adds inverse DR + cells (gates file line 119: "Inverse gets its own cells; linear is not inverse evidence") or G13's wording is + narrowed by the owner. Owner decision; not silently dropped. +6. **DR01 record needs CI confirmation** (`--ci-confirmed` only when the inputs file states `dr01.ci_confirmed: true` + for the TS source SHA). +7. RUNTIME steps read the shared stack and open extra Paper sessions: they need the owner-approved runtime packet + (`--runtime-approval`) after the D5 rollout and the final writer roll. + +## Runner + +```text +# plan only (no Docker, nothing created) +python3 -B scripts/p184_g13_final.py --python-image sha256: --rust-image sha256: \ + --python-image-ref qdl-v2-python:2.2.3- --writer-image sha256: \ + --ts-source-sha <40hex> --dl-source-sha <40hex> --phase all --inputs --dry-run +# isolated phase (after blocker 1; DR08 also after the rust image is deployed) +... --phase isolated +# runtime phase (after rollout + writer roll, approved packet) +... --phase runtime --inputs --runtime-approval +# one failed step again, fresh receipt dir +... --only dr05-transport +``` + +Inputs JSON keys: `dr05_public_export`, `redis_image`, `postgres_image`, `dr01.{inventory, deployments.BINANCE, +deployments.OKX, compose, alpha_service, compose_cwd?, catalog?, ci_confirmed?}`, `alpha.{containers.*, symbols.*, +universe.*?}`, `ts_container`, `dr07.{connections, fx_account, dsn_env?}`, `dr08_restart_samples?` (default 5). + +Receipts: `~/.local/state/qdl-v2/p184-g13-/` (exclusive create): `plan.json`, +`/receipt.json` (argv, cwd, UTC start/end, exit codes, timeout/error, sha256 of every output, +removed leftovers), harness rows and logs, `summary.json`. Cleanup: only containers with the step's own prefix +(`p184-dr05-`, `p184-dr08-`) that did not exist before the step, removed by exact ID; harnesses keep their own +`cleanup.json`/`--down`. `harness_ok` is the harness exit status; cells close only through the TS matrix check. From 0f30e170cf38dfb63030aee111be2a5d3f0857a2 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 08:14:25 +0000 Subject: [PATCH 108/147] fix(ingestor): sub-second liveness knob detects a silent socket in < 5 s d4f5ec9 bounded silent-socket detection at 2 x heartbeat_seconds (15 s -> 30 s); heartbeat_seconds is whole seconds and parser-bounded to 1..29, so it cannot express a < 5 s bound without overloading its meaning. New optional ingestor fields liveness_ping_ms / liveness_dead_ms (LivenessPolicy). Both absent or 0 keeps the heartbeat rule unchanged. Set together they must satisfy ping >= 500 ms, dead - ping >= 1000 ms, dead <= 29000 ms (OKX closes after 30 s idle and asks for N < 30 s); anything else fails config validation. Old binaries deny unknown fields, so the binary must ship before a config that carries them. InboundDeadline now records when the ping was actually sent and gives it its full pong window (dead - ping) from that moment, so a loop held by a REST snapshot fetch cannot declare a healthy socket dead before the venue could answer. Data read still resets the deadline: a busy socket never pings. Generation/lane identity is untouched. The stable renderer emits 1500 / 4000 ms: worst-case detection 4.0 s after the last frame; a quiet socket sends <= 0.67 pings/s (Binance USD-M limit: 10 incoming messages/s per connection; OKX limits only subscribe/ unsubscribe/login, 480/h). Tests: ingestor bin 44/0 (+5: sub-second state machine, late ping keeps its pong window, bounds, silent socket dead at 4.0 s proof on/off, busy socket never pings; answering socket alive 60 s with fast policy); fmt + clippy -D warnings clean; Python deployment/refresh/converge modules 79 OK. --- qdl/runtime/stable_deployment.py | 9 + .../src/bin/qdl-native-raw-ingestor.rs | 364 +++++++++++++++--- tests/test_phaseb_stable_deployment.py | 11 + 3 files changed, 331 insertions(+), 53 deletions(-) diff --git a/qdl/runtime/stable_deployment.py b/qdl/runtime/stable_deployment.py index 4528bf4d..9d16c180 100644 --- a/qdl/runtime/stable_deployment.py +++ b/qdl/runtime/stable_deployment.py @@ -1085,6 +1085,15 @@ def native_ingestor_configs( "partition_plan_epoch": 1, "config_revision": self.revision, "heartbeat_seconds": 15, + # Silent-socket detection < 5 s (ingestor LivenessPolicy): + # ping after 1.5 s of inbound silence, dead 4 s after the last + # message read. A socket with data never pings; a quiet one + # sends <= 0.67 pings/s, under Binance USD-M's 10 incoming + # messages/s per connection; OKX limits only subscribe, + # unsubscribe and login. Requires an ingestor binary that + # knows these fields (older ones deny unknown fields). + "liveness_ping_ms": 1500, + "liveness_dead_ms": 4000, "max_events": max_events, "max_runtime_seconds": max_runtime_seconds, "metrics_every_events": 1000, diff --git a/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs b/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs index 0fbb3618..36a67ce6 100644 --- a/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs +++ b/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs @@ -250,6 +250,18 @@ struct IngestorConfig { /// The realtime core that reads the marker must be deployed first. #[serde(default)] book_liveness_proof_ms: u64, + /// Silent-socket detection at sub-second resolution. Both absent (or + /// zero), the default, keeps the `heartbeat_seconds` rule: ping after + /// `heartbeat_seconds` of inbound silence, dead at twice that. Set + /// together: ping after `liveness_ping_ms` of inbound silence, dead once + /// `liveness_dead_ms` has passed since the last message read and the ping + /// has had at least `liveness_dead_ms - liveness_ping_ms` to be answered. + /// A socket that delivers data never pings. Old binaries reject these + /// fields (`deny_unknown_fields`), so the binary must ship first. + #[serde(default)] + liveness_ping_ms: u64, + #[serde(default)] + liveness_dead_ms: u64, authority: AuthorityRecord, bindings: Vec, #[serde(default)] @@ -402,6 +414,7 @@ impl IngestorConfig { || self.latest_state_flush_ms > 1_000 || (self.book_liveness_proof_ms != 0 && !(250..=5_000).contains(&self.book_liveness_proof_ms)) + || self.liveness().is_none() || self.bindings.is_empty() { return Err( @@ -1795,6 +1808,9 @@ async fn run_binance_connection( }; let liveness_proof_interval = (snapshot_period.is_some() && config.book_liveness_proof_ms > 0) .then(|| Duration::from_millis(config.book_liveness_proof_ms)); + let liveness_policy = config + .liveness() + .ok_or("native raw ingestor liveness bounds are invalid")?; let backoff = BackoffPolicy { initial_ms: 250, maximum_ms: 30_000, @@ -1865,10 +1881,7 @@ async fn run_binance_connection( tick.tick().await; } let mut ping_due = false; - let mut inbound = InboundDeadline::new( - Duration::from_secs(config.heartbeat_seconds), - Instant::now(), - ); + let mut inbound = InboundDeadline::new(liveness_policy, Instant::now()); let mut inflight = FuturesUnordered::::new(); let mut latest = LatestStateBuffer::default(); let mut latest_tick = @@ -2144,7 +2157,7 @@ async fn run_binance_connection( // The pong (or any frame) arrives through the same // select, so other branches keep running meanwhile; the // ping itself never moves the deadline. - Err(_) => match inbound.expired() { + Err(_) => match inbound.expired(Instant::now()) { Keepalive::SendPing => { socket.send(Message::Ping(Vec::new())).await?; continue; @@ -2428,6 +2441,70 @@ enum Keepalive { Dead, } +/// Smallest silence before a liveness ping. Venue limits checked 2026-10-10: +/// Binance USD-M allows 10 incoming messages per second per connection and +/// OKX lists no ping limit (only subscribe/unsubscribe/login, 480 per hour). +/// One ping per 500 ms of silence is at most 2 messages per second. +const LIVENESS_PING_MIN_MS: u64 = 500; +/// Smallest time a ping is given to be answered before the socket is dead. +const LIVENESS_PONG_WINDOW_MIN_MS: u64 = 1_000; +/// OKX closes a socket after 30 s without data and asks for a client timer +/// N < 30 s, so no liveness bound may reach it. +const LIVENESS_DEAD_MAX_MS: u64 = 29_000; + +/// When a silent connection is pinged and when it is declared dead. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct LivenessPolicy { + /// Inbound silence after which one ping is sent. + ping_after: Duration, + /// Inbound silence after which the connection is dead, provided the ping + /// has had at least `dead_after - ping_after` to be answered. + dead_after: Duration, +} + +impl LivenessPolicy { + /// The pre-knob rule: ping after `heartbeat`, dead at twice that. + fn heartbeat(heartbeat: Duration) -> Self { + Self { + ping_after: heartbeat, + dead_after: heartbeat * 2, + } + } + + /// Validated sub-second policy; both zero selects the heartbeat rule. + fn from_config(heartbeat_seconds: u64, ping_ms: u64, dead_ms: u64) -> Option { + match (ping_ms, dead_ms) { + (0, 0) => Some(Self::heartbeat(Duration::from_secs(heartbeat_seconds))), + (ping, dead) + if ping >= LIVENESS_PING_MIN_MS + && dead <= LIVENESS_DEAD_MAX_MS + && dead.checked_sub(ping)? >= LIVENESS_PONG_WINDOW_MIN_MS => + { + Some(Self { + ping_after: Duration::from_millis(ping), + dead_after: Duration::from_millis(dead), + }) + } + _ => None, + } + } + + fn pong_window(&self) -> Duration { + self.dead_after.saturating_sub(self.ping_after) + } +} + +impl IngestorConfig { + /// None when the liveness fields are out of bounds (validation refuses). + fn liveness(&self) -> Option { + LivenessPolicy::from_config( + self.heartbeat_seconds, + self.liveness_ping_ms, + self.liveness_dead_ms, + ) + } +} + /// Hard bound on detecting a silent socket (Astra R4). /// /// The read loops also select on timers (book-liveness proof, latest-state @@ -2437,44 +2514,52 @@ enum Keepalive { /// declared dead while any of them kept firing. This deadline is absolute: it /// moves only on a message actually read from the socket (data, ping, pong or /// OKX text `pong`). Sending a ping, a timer tick or a publish completion never -/// moves it. Idle for `idle` -> one ping; still nothing by `2 * idle` after the -/// last inbound message -> dead. That matches OKX's keepalive rule (send +/// moves it. Idle for `ping_after` -> one ping; still nothing by `dead_after` +/// after the last inbound message -> dead. With the heartbeat rule +/// (`dead_after = 2 * ping_after`) that matches OKX's keepalive rule (send /// `ping` after N < 30 idle seconds, reconnect if no `pong` within N). +/// +/// The ping is always given its full pong window from the moment it was +/// actually sent: if the loop was held past `ping_after` (a REST snapshot +/// fetch inside a select branch can take up to 5 s), the dead deadline moves +/// to `sent + (dead_after - ping_after)` instead of firing before the venue +/// could have answered. A busy socket never pings: every frame read resets it. #[derive(Debug)] struct InboundDeadline { - idle: Duration, + policy: LivenessPolicy, last_inbound: Instant, - ping_sent: bool, + ping_sent_at: Option, } impl InboundDeadline { - fn new(idle: Duration, now: Instant) -> Self { + fn new(policy: LivenessPolicy, now: Instant) -> Self { Self { - idle, + policy, last_inbound: now, - ping_sent: false, + ping_sent_at: None, } } fn inbound(&mut self, now: Instant) { self.last_inbound = now; - self.ping_sent = false; + self.ping_sent_at = None; } fn deadline(&self) -> Instant { - if self.ping_sent { - self.last_inbound + self.idle * 2 - } else { - self.last_inbound + self.idle + match self.ping_sent_at { + None => self.last_inbound + self.policy.ping_after, + Some(sent) => { + (self.last_inbound + self.policy.dead_after).max(sent + self.policy.pong_window()) + } } } /// Called when `deadline()` has passed with nothing read. - fn expired(&mut self) -> Keepalive { - if self.ping_sent { + fn expired(&mut self, now: Instant) -> Keepalive { + if self.ping_sent_at.is_some() { Keepalive::Dead } else { - self.ping_sent = true; + self.ping_sent_at = Some(now); Keepalive::SendPing } } @@ -2643,6 +2728,9 @@ async fn run_okx_service( .collect(), ); let mut resync_listener = book_resync_listener(&config, &lane, &binding_map)?; + let liveness_policy = config + .liveness() + .ok_or("native raw ingestor liveness bounds are invalid")?; let backoff = BackoffPolicy { initial_ms: 250, maximum_ms: 30_000, @@ -2756,10 +2844,7 @@ async fn run_okx_service( let mut publish_error = None; let mut exhausted = false; let mut last_frame_at_ns = ready_at_ns; - let mut inbound = InboundDeadline::new( - Duration::from_secs(config.heartbeat_seconds), - Instant::now(), - ); + let mut inbound = InboundDeadline::new(liveness_policy, Instant::now()); let mut standby: Option = None; let mut notice: Option = None; if let Some(code) = early_notice { @@ -2906,7 +2991,7 @@ async fn run_okx_service( ended = Some(OkxSessionEnd::Eof); break; } - Err(_) => match inbound.expired() { + Err(_) => match inbound.expired(Instant::now()) { Keepalive::Dead => { ended = Some(OkxSessionEnd::HeartbeatTimeout); break; @@ -4099,6 +4184,8 @@ mod resync_tests { max_subscriptions_per_connection: 100, latest_state_flush_ms: 50, book_liveness_proof_ms: 0, + liveness_ping_ms: 0, + liveness_dead_ms: 0, authority: AuthorityRecord { schema: "qdl.authority-record.v1".into(), slice_id: "unit".into(), @@ -4338,8 +4425,9 @@ mod resync_tests { mod okx_reconnect_tests { use super::{ okx_establish, okx_notice_code, okx_reconnect_delay_ms, BackoffPolicy, DeliveryClass, - InboundDeadline, Keepalive, OkxConnectPacer, OkxEstablishError, OkxSessionEnd, - OkxSubscription, RawBinding, RawFeed, SessionLineageWriter, OKX_CONNECT_SPACING, + InboundDeadline, Keepalive, LivenessPolicy, OkxConnectPacer, OkxEstablishError, + OkxSessionEnd, OkxSubscription, RawBinding, RawFeed, SessionLineageWriter, + OKX_CONNECT_SPACING, }; use futures_util::{SinkExt, StreamExt}; use serde_json::{json, Value}; @@ -4405,22 +4493,98 @@ mod okx_reconnect_tests { assert!((24_000..=30_000).contains(&okx_reconnect_delay_ms(backoff(), 30))); } + fn heartbeat_15s() -> LivenessPolicy { + LivenessPolicy::heartbeat(Duration::from_secs(15)) + } + + /// The values the stable renderer emits: ping after 1.5 s of silence, + /// dead 4 s after the last message read. + fn fast() -> LivenessPolicy { + LivenessPolicy::from_config(15, 1_500, 4_000).unwrap() + } + #[test] fn inbound_deadline_pings_once_then_declares_the_connection_dead() { let idle = Duration::from_secs(15); let start = Instant::now(); - let mut deadline = InboundDeadline::new(idle, start); + let mut deadline = InboundDeadline::new(heartbeat_15s(), start); assert_eq!(deadline.deadline(), start + idle); - assert_eq!(deadline.expired(), Keepalive::SendPing); + assert_eq!(deadline.expired(start + idle), Keepalive::SendPing); // Sending the ping opens exactly one pong window; nothing renews it. assert_eq!(deadline.deadline(), start + idle * 2); - assert_eq!(deadline.expired(), Keepalive::Dead); + assert_eq!(deadline.expired(start + idle * 2), Keepalive::Dead); assert_eq!(deadline.deadline(), start + idle * 2); // Only a message read from the socket moves it. let later = start + Duration::from_secs(20); deadline.inbound(later); assert_eq!(deadline.deadline(), later + idle); - assert_eq!(deadline.expired(), Keepalive::SendPing); + assert_eq!(deadline.expired(later + idle), Keepalive::SendPing); + } + + #[test] + fn sub_second_policy_pings_at_ping_after_and_is_dead_at_dead_after() { + let start = Instant::now(); + let mut deadline = InboundDeadline::new(fast(), start); + assert_eq!(deadline.deadline(), start + Duration::from_millis(1_500)); + assert_eq!( + deadline.expired(start + Duration::from_millis(1_500)), + Keepalive::SendPing + ); + assert_eq!(deadline.deadline(), start + Duration::from_millis(4_000)); + assert_eq!( + deadline.expired(start + Duration::from_millis(4_000)), + Keepalive::Dead + ); + } + + #[test] + fn a_late_ping_still_gets_its_full_pong_window() { + // The loop was held 5 s (a REST snapshot fetch inside a select + // branch) and only then saw the expired ping deadline. Declaring the + // socket dead at last_inbound + 4 s would be immediate, before the + // venue could answer; the pong gets dead_after - ping_after = 2.5 s. + let start = Instant::now(); + let mut deadline = InboundDeadline::new(fast(), start); + let late = start + Duration::from_secs(5); + assert_eq!(deadline.expired(late), Keepalive::SendPing); + assert_eq!(deadline.deadline(), late + Duration::from_millis(2_500)); + // The same holds for the heartbeat rule. + let mut deadline = InboundDeadline::new(heartbeat_15s(), start); + let late = start + Duration::from_secs(25); + assert_eq!(deadline.expired(late), Keepalive::SendPing); + assert_eq!(deadline.deadline(), late + Duration::from_secs(15)); + } + + #[test] + fn liveness_fields_are_bounded_and_absent_keeps_the_heartbeat_rule() { + // Absent (serde default 0/0): today's behaviour, 15 s ping, 30 s dead. + assert_eq!(LivenessPolicy::from_config(15, 0, 0), Some(heartbeat_15s())); + assert_eq!( + LivenessPolicy::from_config(15, 1_500, 4_000), + Some(LivenessPolicy { + ping_after: Duration::from_millis(1_500), + dead_after: Duration::from_millis(4_000), + }) + ); + // Edges that are accepted. + assert!(LivenessPolicy::from_config(15, 500, 1_500).is_some()); + assert!(LivenessPolicy::from_config(15, 28_000, 29_000).is_some()); + for (ping, dead) in [ + (1_500, 0), // only one field set + (0, 4_000), // only one field set + (499, 4_000), // ping faster than one per 500 ms of silence + (1_500, 2_499), // pong window under 1 s + (4_000, 1_500), // dead before the ping + (4_000, 4_000), // no pong window at all + (1_500, 29_001), // reaches OKX's 30 s idle close + (u64::MAX, 4_000), + ] { + assert_eq!( + LivenessPolicy::from_config(15, ping, dead), + None, + "ping {ping} dead {dead}" + ); + } } /// A venue socket that stays open and sends nothing: no frames, no pongs @@ -4442,21 +4606,49 @@ mod okx_reconnect_tests { format!("ws://{address}/ws") } + /// A venue that pushes a data frame every `every` and never reads (so it + /// never answers a ping either). + async fn busy_venue(every: Duration) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap(); + let mut tick = tokio::time::interval(every); + loop { + tick.tick().await; + if socket.send(Message::Text("{}".into())).await.is_err() { + break; + } + } + }); + format!("ws://{address}/ws") + } + + #[derive(Debug)] + struct Detection { + /// When the connection was declared dead, or None if still alive. + dead_after: Option, + /// Pings sent by the liveness deadline (not by the proof tick). + liveness_pings: u32, + } + /// The production read-loop shape: a book-liveness proof tick (flag on) /// competes with the socket read in one select and `continue`s, sending a /// proof ping like `BookLivenessProof::due` asks for. Returns when the - /// connection is declared dead, or None if it is still considered alive - /// at `give_up`. + /// connection is declared dead, or with `dead_after` None if it is still + /// considered alive at `give_up`. async fn detect_dead( url: String, + policy: LivenessPolicy, proof_tick: Option, absolute_deadline: bool, give_up: Duration, - ) -> Option { - let idle = Duration::from_secs(15); + ) -> Detection { let (mut socket, _) = tokio_tungstenite::connect_async(&url).await.unwrap(); let start = Instant::now(); - let mut inbound = InboundDeadline::new(idle, start); + let mut inbound = InboundDeadline::new(policy, start); + let mut liveness_pings = 0_u32; let mut tick = proof_tick.map(tokio::time::interval); if let Some(tick) = tick.as_mut() { tick.tick().await; @@ -4466,7 +4658,7 @@ mod okx_reconnect_tests { tokio::time::timeout_at(inbound.deadline(), socket.next()) } else { // The pre-R4 shape: a fresh relative timeout every iteration. - tokio::time::timeout_at(Instant::now() + idle, socket.next()) + tokio::time::timeout_at(Instant::now() + policy.ping_after, socket.next()) }; tokio::pin!(read); tokio::select! { @@ -4482,16 +4674,25 @@ mod okx_reconnect_tests { result = &mut read => match result { Ok(Some(Ok(_))) => inbound.inbound(Instant::now()), Ok(_) => panic!("the fake venue never closes"), - Err(_) => match inbound.expired() { + Err(_) => match inbound.expired(Instant::now()) { Keepalive::SendPing => { + liveness_pings += 1; socket.send(Message::Ping(Vec::new())).await.unwrap(); } - Keepalive::Dead => return Some(start.elapsed()), + Keepalive::Dead => { + return Detection { + dead_after: Some(start.elapsed()), + liveness_pings, + } + } }, }, } } - None + Detection { + dead_after: None, + liveness_pings, + } } #[tokio::test(start_paused = true)] @@ -4500,11 +4701,13 @@ mod okx_reconnect_tests { for proof_tick in [None, Some(Duration::from_millis(250))] { let detected = detect_dead( silent_venue(false).await, + heartbeat_15s(), proof_tick, true, Duration::from_secs(600), ) .await + .dead_after .unwrap_or_else(|| panic!("never detected, proof tick {proof_tick:?}")); assert!( detected >= Duration::from_secs(30) && detected <= bound + Duration::from_millis(1), @@ -4513,38 +4716,93 @@ mod okx_reconnect_tests { } } + #[tokio::test(start_paused = true)] + async fn a_silent_socket_is_declared_dead_in_under_five_seconds_with_the_fast_policy() { + for proof_tick in [None, Some(Duration::from_millis(250))] { + let detection = detect_dead( + silent_venue(false).await, + fast(), + proof_tick, + true, + Duration::from_secs(60), + ) + .await; + let detected = detection + .dead_after + .unwrap_or_else(|| panic!("never detected, proof tick {proof_tick:?}")); + assert!( + detected >= Duration::from_millis(4_000) + && detected <= Duration::from_millis(4_001) + && detected < Duration::from_secs(5), + "proof tick {proof_tick:?}: detected after {detected:?}" + ); + assert_eq!(detection.liveness_pings, 1, "proof tick {proof_tick:?}"); + } + } + #[tokio::test(start_paused = true)] async fn the_pre_r4_relative_timeout_never_detects_a_silent_socket_while_the_proof_ticks() { // Reproduces Astra R4 on the old shape: 10 idle periods, still "alive". assert_eq!( detect_dead( silent_venue(false).await, + heartbeat_15s(), Some(Duration::from_millis(250)), false, Duration::from_secs(150), ) - .await, + .await + .dead_after, None ); } #[tokio::test(start_paused = true)] async fn a_socket_that_answers_pings_stays_alive_with_proof_on_and_off() { - for proof_tick in [None, Some(Duration::from_millis(250))] { - assert_eq!( - detect_dead( - silent_venue(true).await, - proof_tick, - true, - Duration::from_secs(120), - ) - .await, - None, - "proof tick {proof_tick:?}" - ); + for policy in [heartbeat_15s(), fast()] { + for proof_tick in [None, Some(Duration::from_millis(250))] { + let give_up = if policy == fast() { + Duration::from_secs(60) + } else { + Duration::from_secs(120) + }; + let detection = + detect_dead(silent_venue(true).await, policy, proof_tick, true, give_up).await; + assert_eq!( + detection.dead_after, None, + "policy {policy:?} proof tick {proof_tick:?}" + ); + // At most one liveness ping per ping_after of silence: with the + // fast policy that is <= 0.67/s, far under Binance's 10 + // incoming messages per second per connection. + let ceiling = (give_up.as_millis() / policy.ping_after.as_millis()) as u32 + 1; + assert!( + detection.liveness_pings <= ceiling, + "policy {policy:?} proof tick {proof_tick:?}: {} pings", + detection.liveness_pings + ); + if proof_tick.is_none() && policy == fast() { + // A quiet-but-answering socket is pinged about every 1.5 s. + assert!(detection.liveness_pings >= 30, "{detection:?}"); + } + } } } + #[tokio::test(start_paused = true)] + async fn a_busy_socket_never_sends_a_liveness_ping() { + let detection = detect_dead( + busy_venue(Duration::from_millis(200)).await, + fast(), + None, + true, + Duration::from_secs(60), + ) + .await; + assert_eq!(detection.dead_after, None); + assert_eq!(detection.liveness_pings, 0); + } + #[test] fn only_the_okx_notice_event_is_a_service_upgrade_notice() { let notice = json!({ diff --git a/tests/test_phaseb_stable_deployment.py b/tests/test_phaseb_stable_deployment.py index 17c454e9..3a8cd67f 100644 --- a/tests/test_phaseb_stable_deployment.py +++ b/tests/test_phaseb_stable_deployment.py @@ -510,6 +510,17 @@ def test_all_catalog_bindings_have_one_capability_truthful_acquisition(self): self.assertEqual( {item["latest_state_flush_ms"] for item in native.values()}, {50} ) + self.assertEqual( + { + ( + item["heartbeat_seconds"], + item["liveness_ping_ms"], + item["liveness_dead_ms"], + ) + for item in native.values() + }, + {(15, 1500, 4000)}, + ) delivery_by_feed = { binding["feed"]: binding["delivery_class"] for item in native.values() From afe19f7caf6001643f83c1f8f94750b400222d54 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 08:29:49 +0000 Subject: [PATCH 109/147] tooling(p184): G13 harness images as parameters, TS matrix v2 inverse cells, DR01 wrong-account step DR08/DR05 harnesses: IMAGES/QUERY_IMAGE_REF become defaults. p184_dr08_isolated.py --orchestrate takes --image ROLE=sha256:ID and --query-image-ref, attests them to every container (P184_DR08_IMAGES, P184_DR08_QUERY_IMAGE_REF) and the in-container checks compare against the attested set. p184_dr05_dl_fault.py --init/--prepare --image seals the set in handoff.json; later steps, reset, channel and --material read it via image_set(work). Unchanged without flags. p184_g13_final.py: passes the final python/rust images (HARNESS_IMAGE_PIN refusal removed); binds DR_CELLS to TS p184-r2-case-matrix.v2 (76 DR cells, + 11 OKX_INVERSE; CASE_MATRIX_MISMATCH otherwise); views step adds okx_inverse; new Gateway-dependent step dr01-wrong-account (TS 6c44bf2 runner), deferred as GATEWAY_NOT_RUNNING without failing other steps; 5 inverse DR05 cells listed without producer. --- scripts/p184_dr05_case.py | 3 +- scripts/p184_dr05_channel_smoke.py | 2 +- scripts/p184_dr05_dl_fault.py | 70 ++++--- scripts/p184_dr05_observe_stack.py | 2 +- scripts/p184_dr05_reset.py | 2 +- scripts/p184_dr08_isolated.py | 116 +++++++++--- scripts/p184_g13_final.py | 178 +++++++++++------- tests/test_p184_dr05_dl_fault.py | 67 +++++++ tests/test_p184_dr08_isolated.py | 123 ++++++++++++ tests/test_p184_g13_final.py | 131 ++++++++++--- .../evidence/p184-g13-inherit-rerun-matrix.md | 72 ++++--- 11 files changed, 601 insertions(+), 165 deletions(-) diff --git a/scripts/p184_dr05_case.py b/scripts/p184_dr05_case.py index 52e5719a..7f055d90 100644 --- a/scripts/p184_dr05_case.py +++ b/scripts/p184_dr05_case.py @@ -111,6 +111,7 @@ def _packet(work): def _reset_preflight(work, ns, call, core_binary=None, *, allow_channel=False): core_binary = dr.core_override(core_binary) + images = dr.image_set(work) rows = {role: dr.owned(ns, role, call) for role in (*dr.ROLES, "helper", "tsvalidator")} if rows["tsvalidator"] or (rows["helper"] and not allow_channel): raise ValueError("stop helpers and TS clones before reset") @@ -137,7 +138,7 @@ def _reset_preflight(work, ns, call, core_binary=None, *, allow_channel=False): expected_binds["/usr/local/bin/qdl-realtime-core"] = core_binary["path"] if any(m.get("RW") is not False for m in mounts if m["Destination"] == "/usr/local/bin/qdl-realtime-core"): raise ValueError("candidate core binary mount must be read-only") - if (row["Image"] != dr.IMAGES[{"quota": "redis", "helper": "python"}.get(role, role)] + if (row["Image"] != images[{"quota": "redis", "helper": "python"}.get(role, role)] or host.get("AutoRemove") is not True or host.get("PortBindings") or host.get("NetworkMode") not in (expected, "container:" + ns + "-kafka" if role != "kafka" else expected) or host.get("ReadonlyRootfs") is not True diff --git a/scripts/p184_dr05_channel_smoke.py b/scripts/p184_dr05_channel_smoke.py index 9d8f0d1a..c96e429e 100644 --- a/scripts/p184_dr05_channel_smoke.py +++ b/scripts/p184_dr05_channel_smoke.py @@ -34,7 +34,7 @@ def exercise(output, exported, core_binary): argv = dr.helper_command(ns, work, ["--channel", "/work"], network="container:" + ns + "-kafka") argv.insert(2, "-i") argv[argv.index("scripts/p184_dr05_dl_fault.py")] = "scripts/p184_dr05_channel.py" - pos = argv.index(dr.IMAGES["python"]) + pos = argv.index(dr.image_set(work)["python"]) argv[pos:pos] = ["-e", "P184_DR05_ISOLATED_NAMESPACE=" + ns] channel = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=errors, text=True, bufsize=1) diff --git a/scripts/p184_dr05_dl_fault.py b/scripts/p184_dr05_dl_fault.py index e5a90c2e..883ca4c6 100644 --- a/scripts/p184_dr05_dl_fault.py +++ b/scripts/p184_dr05_dl_fault.py @@ -4,6 +4,11 @@ --prepare writes a NEW external directory with topology, fault schedules and nonqualifying verifier rows. --readiness/--cleanup are owner stage3 utilities. The publisher/TS adapter handoff is deliberately explicit, not a status mock. + +Image set: --init/--prepare accept --image ROLE=sha256: (repeatable, roles of +scripts/p184_dr08_isolated.py IMAGES, which stay the defaults). The resolved set is sealed +in handoff.json "images"; every later step (--up/--ready/--down, reset, channel, the +in-container --material) reads it from there via image_set(work), never from the constants. """ from __future__ import annotations @@ -23,7 +28,8 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from scripts.p184_dr08_isolated import IMAGES # immutable image inventory only +from scripts.p184_dr08_isolated import IMAGES # immutable image inventory only: the DEFAULT set +from scripts.p184_dr08_isolated import parse_image_overrides, resolve_images VENUES = ("BINANCE", "OKX") FAULTS = ("GAP_REFUSAL", "GENERATION_CHANGE_FENCED") @@ -137,7 +143,19 @@ def core_override(value): return {"path": str(path), "sha256": digest} -def topology(ns, work, topic_id="", *, core_binary=None): +def image_set(work=None): + """This run's sealed image set (handoff.json "images"), else the pinned defaults.""" + handoff = Path(work) / "handoff.json" if work is not None else None + if handoff is not None and handoff.is_file(): + images = json.loads(handoff.read_text()).get("images") + if images is not None: + if not isinstance(images, dict) or set(images) != set(IMAGES): + raise ValueError("handoff image set must name exactly the harness roles") + return resolve_images(images) + return dict(IMAGES) + + +def topology(ns, work, topic_id="", *, core_binary=None, images=None): """Generate executable Docker argv, but do not execute or discover config. Material preparation is a stage3 gate; topic UUID comes from the new broker. @@ -145,6 +163,7 @@ def topology(ns, work, topic_id="", *, core_binary=None): """ namespace(ns) core_binary = core_override(core_binary) + images = images or image_set(work) work = Path(work).resolve() if work == ROOT or ROOT in work.parents or ":" in str(work) or "," in str(work): raise ValueError("external unambiguous work directory required") @@ -197,7 +216,7 @@ def topology(ns, work, topic_id="", *, core_binary=None): argv += ["-v", core_binary["path"] + ":/usr/local/bin/qdl-realtime-core:ro"] for key, value in env.items(): argv += ["-e", key + "=" + value] - result[role] = argv + ["--entrypoint", command[0], IMAGES[image], *command[1:]] + result[role] = argv + ["--entrypoint", command[0], images[image], *command[1:]] return result @@ -228,12 +247,13 @@ def owned(ns, role, call=sh, *, allow_disappearance=True): return row -def readiness(ns, call=sh): +def readiness(ns, call=sh, images=None): + images = images or IMAGES rows = {role: owned(ns, role, call) for role in ROLES} broker = rows["kafka"] checks = {} for role, row in rows.items(): - expected_image = IMAGES[{"quota": "redis"}.get(role, role)] + expected_image = images[{"quota": "redis"}.get(role, role)] network = "none" if role == "kafka" else "container:" + (broker or {}).get("Id", "MISSING") checks[role] = bool(row and row["State"]["Running"] and row["Image"] == expected_image and row["HostConfig"]["NetworkMode"] in @@ -289,13 +309,14 @@ def cleanup(ns, call=sh): return {"removed": removed, "scope": ns} -def prepare(work): +def prepare(work, images=None): ns = "p184-dr05-dl-" + uuid.uuid4().hex[:16] - topology(ns, work) # Validate path before writing. + images = resolve_images(images) + topology(ns, work, images=images) # Validate path before writing. work = Path(work).resolve() work.mkdir(mode=0o700, parents=True, exist_ok=False) packet = {"namespace": ns, "status": "PREPARED_HANDOFF_NOT_RUN", "provenance": "TEST_ONLY", - "level": "ISOLATED_FAULT", "images": IMAGES, + "level": "ISOLATED_FAULT", "images": images, "deadline_seconds": 600, "acceptance": False, "material_ready": False, "adapter_ready": False, "schedules": {f"DR05.{f}@{v}": schedule(v, f) for v in VENUES for f in FAULTS}} @@ -304,9 +325,10 @@ def prepare(work): return packet -def isolated_config(exported, ns): +def isolated_config(exported, ns, images=None): """Take only public binding fields; never copy transport, credentials or authority.""" namespace(ns) + images = images or IMAGES required = {"provider", "venue", "market", "product_type", "native_symbol", "native_channel", "provider_kind", "instrument_uid", "instrument_id", "instrument_revision", "instrument_catalog_revision", "source_id", "source_role", "normalizer_version", @@ -334,7 +356,7 @@ def isolated_config(exported, ns): "batch_size": 25, "batch_wait_ms": 10, "max_events": 0, "metrics_every_batches": 1, "strict_subscription_scope": True, "book_resync_quarantine": True, "authority": {"schema": "qdl.authority-record.v1", "slice_id": ns, "revision": 1, - "mode": "RUST_SHADOW", "candidate_image_digest": IMAGES["core"], + "mode": "RUST_SHADOW", "candidate_image_digest": images["core"], "capability_manifest_digest": digest, "contract_digest": digest, "partition_plan_digest": digest, "public_write_allowed": False, "legacy_write_allowed": False, "approved_by": "TEST_ONLY ISOLATED_FAULT", @@ -359,7 +381,7 @@ def material(work, exported): work = Path(work) packet = json.loads((work / "handoff.json").read_text()) namespace = packet["namespace"] - cfg = isolated_config(exported, namespace) + cfg = isolated_config(exported, namespace, image_set(work)) def save(name, value, private=False): with (work / name).open("x") as handle: @@ -504,7 +526,7 @@ def helper_command(ns, work, args, *, network="none", exported=None): "-e", "PYTHONDONTWRITEBYTECODE=1"] if exported: argv += ["-v", str(Path(exported).resolve()) + ":/export.json:ro"] - return argv + ["--entrypoint", "python", IMAGES["python"], "-B", "scripts/p184_dr05_dl_fault.py", *args] + return argv + ["--entrypoint", "python", image_set(work)["python"], "-B", "scripts/p184_dr05_dl_fault.py", *args] def bounded(argv, timeout=120): @@ -525,9 +547,9 @@ def erase_private(work): (work / "handoff.json").write_text(json.dumps(packet, indent=2) + "\n") -def init(work, export_path): +def init(work, export_path, images=None): # Runtime-free initialization in the already-local dependency image. - packet = prepare(work) + packet = prepare(work, images) ns = packet["namespace"] try: bounded(helper_command(ns, work, ["--material", "/work", "--core-config", "/export.json"], exported=export_path)) @@ -560,19 +582,20 @@ def run(argv, limit=120): if selected_core: packet["core_binary"] = selected_core (work / "handoff.json").write_text(json.dumps(packet, indent=2) + "\n") - for digest in set(IMAGES.values()): + images = image_set(work) + for digest in set(images.values()): if run(["docker", "image", "inspect", digest, "--format", "{{.Id}} "]).strip() != digest: raise ValueError("local immutable image mismatch") try: - run(topology(ns, work, core_binary=selected_core)["kafka"]) + run(topology(ns, work, core_binary=selected_core, images=images)["kafka"]) run(helper_command(ns, work, ["--topics", "/work"], network="container:" + ns + "-kafka")) topic = json.loads((work / "packet.json").read_text())["slice"]["topic_id"] - for role, argv in topology(ns, work, topic, core_binary=selected_core).items(): + for role, argv in topology(ns, work, topic, core_binary=selected_core, images=images).items(): if role != "kafka": run(argv) deadline = min(deadline, time.monotonic() + 45) while True: - result = readiness(ns, run) + result = readiness(ns, run, images) if result["process_isolation_ready"]: try: run(helper_command(ns, work, ["--probe", "/work"], network="container:" + ns + "-kafka"), 25) @@ -635,12 +658,17 @@ def dispatch(argv=None): mode.add_argument("--readiness") mode.add_argument("--cleanup") parser.add_argument("--core-config", type=Path) + parser.add_argument("--image", action="append", default=[], metavar="ROLE=sha256:ID", + help="--init/--prepare image override (repeatable); sealed in handoff.json") args = parser.parse_args(argv) + if args.image and not (args.init or args.prepare): + parser.error("--image only with --init/--prepare; later steps read handoff.json") + images = resolve_images(parse_image_overrides(args.image)) if args.init or args.material: if args.core_config is None: parser.error("--core-config sanitized exported JSON required") if args.init: - result = init(args.init, args.core_config) + result = init(args.init, args.core_config, images) else: result = material(args.material, json.loads(args.core_config.read_text())) result = {k: result[k] for k in ("namespace", "status", "material_ready")} @@ -655,7 +683,7 @@ def dispatch(argv=None): finally: erase_private(work) else: - result = readiness(ns) + result = readiness(ns, images=image_set(work)) if result["process_isolation_ready"]: result.update(json.loads(bounded(helper_command(ns, work, ["--probe", "/work"], network="container:" + ns + "-kafka")))) @@ -666,7 +694,7 @@ def dispatch(argv=None): elif args.probe: result = probe(args.probe) elif args.prepare: - packet = prepare(args.prepare) + packet = prepare(args.prepare, images) result = {k: packet[k] for k in ("namespace", "status", "acceptance")} else: result = readiness(args.readiness) if args.readiness else cleanup(args.cleanup) diff --git a/scripts/p184_dr05_observe_stack.py b/scripts/p184_dr05_observe_stack.py index d3d8a567..f74b36bd 100644 --- a/scripts/p184_dr05_observe_stack.py +++ b/scripts/p184_dr05_observe_stack.py @@ -47,7 +47,7 @@ def exercise(output, exported, venue, fault, core_binary=None): "--entrypoint", "python", ts_image, "-B", "/probe.py", "--serve", "--namespace", ns] dr.bounded(live_args) argv = dr.helper_command(ns, work, [], network="container:" + ns + "-kafka") - index = argv.index(dr.IMAGES["python"]) + index = argv.index(dr.image_set(work)["python"]) argv[index:index] = ["-e", "P184_DR05_ISOLATED_NAMESPACE=" + ns] argv[-1:] = ["scripts/p184_dr05_observer.py", "--work", "/work", "--venue", venue, "--fault", fault] receipt["observation"] = json.loads(dr.bounded(argv, 120)) diff --git a/scripts/p184_dr05_reset.py b/scripts/p184_dr05_reset.py index 8432aa33..fcc2e4bc 100644 --- a/scripts/p184_dr05_reset.py +++ b/scripts/p184_dr05_reset.py @@ -114,7 +114,7 @@ def inside(script, args): else: argv = dr.helper_command(ns, work, args, network="container:" + ns + "-kafka") argv[argv.index("scripts/p184_dr05_dl_fault.py")] = script - pos = argv.index(dr.IMAGES["python"]) + pos = argv.index(dr.image_set(work)["python"]) argv[pos:pos] = ["-e", "P184_DR05_ISOLATED_NAMESPACE=" + ns] return call(argv) diff --git a/scripts/p184_dr08_isolated.py b/scripts/p184_dr08_isolated.py index d228e222..cf115a07 100644 --- a/scripts/p184_dr08_isolated.py +++ b/scripts/p184_dr08_isolated.py @@ -22,6 +22,12 @@ Stream must retain max_replay_events=10000, report metrics every second, and use a queue byte budget <=32MiB. No production credentials or mounts are allowed. +Image set: IMAGES / QUERY_IMAGE_REF below are DEFAULTS only. --orchestrate accepts +--image ROLE=sha256: (repeatable; roles query/core/stream/python/kafka/redis) and +--query-image-ref qdl-v2-python:; the parent passes the resolved set into every +container as P184_DR08_IMAGES (JSON) and P184_DR08_QUERY_IMAGE_REF, and the in-container +checks compare the packet against those attested values (defaults when absent). + Parent bounds: Kafka 1CPU/1536MiB; core .5CPU/256MiB; Stream .5CPU/512MiB; Python .75CPU/512MiB; quota Redis .1CPU/64MiB; Query .5CPU/512MiB. --rm, no published ports, no shared mounts, tmpfs/bounded logs, ten-minute deadline. The orchestrator validates labels and cleans only its owned names. @@ -66,12 +72,61 @@ def restart_count(packet): return count -def validate_query_provenance(packet): +ENV_IMAGES = "P184_DR08_IMAGES" +ENV_QUERY_IMAGE_REF = "P184_DR08_QUERY_IMAGE_REF" +IMAGE_ROLES = ("query", "core", "stream", "python", "kafka", "redis") +IMAGE_ID = re.compile(r"sha256:[0-9a-f]{64}") +QUERY_REF = re.compile(r"qdl-v2-python:[A-Za-z0-9][A-Za-z0-9_.-]{0,127}") + + +def parse_image_overrides(values): + """--image ROLE=sha256:<64hex> (repeatable) -> {role: id}; unknown roles and duplicates refused.""" + out = {} + for value in values or (): + role, sep, image = str(value).partition("=") + if not sep or role not in IMAGE_ROLES or role in out or not IMAGE_ID.fullmatch(image): + raise ValueError(f"--image ROLE=sha256:<64hex> with ROLE in {IMAGE_ROLES}, once each: {value!r}") + out[role] = image + return out + + +def resolve_images(overrides=None): + """The pinned IMAGES with explicit overrides applied; every role must be an immutable image ID.""" + images = dict(IMAGES) + for role, image in (overrides or {}).items(): + if role not in IMAGE_ROLES: + raise ValueError("unknown image role: " + str(role)) + images[role] = image + for role in IMAGE_ROLES: + if not isinstance(images.get(role), str) or not IMAGE_ID.fullmatch(images[role]): + raise ValueError(f"immutable {role} image ID required") + return images + + +def resolve_query_ref(images, query_image_ref=None): + """The Query image label must name the Query digest it runs: a changed digest needs its own ref.""" + ref = query_image_ref or QUERY_IMAGE_REF + if not QUERY_REF.fullmatch(ref): + raise ValueError("--query-image-ref must be qdl-v2-python:") + if (images["query"] == QUERY_IMAGE_DIGEST) != (ref == QUERY_IMAGE_REF): + raise ValueError("query image digest and --query-image-ref must change together") + return ref + + +def expected_query(environ=None): + """(image_ref, digest) the parent attested via env, else the pinned defaults.""" + environ = os.environ if environ is None else environ + images = json.loads(environ[ENV_IMAGES]) if environ.get(ENV_IMAGES) else IMAGES + return environ.get(ENV_QUERY_IMAGE_REF) or QUERY_IMAGE_REF, images["query"] + + +def validate_query_provenance(packet, expected=None): + ref, digest = expected or expected_query() query = packet.get("query_provenance", {}) - if (not isinstance(query, dict) or packet.get("images", {}).get("query") != QUERY_IMAGE_DIGEST - or query.get("image_ref") != QUERY_IMAGE_REF - or query.get("actual_image_digest") != QUERY_IMAGE_DIGEST): - raise ValueError("Query must be verified against candidate 2.2.3-rc2-43bbcf0, not production v2.2.2") + if (not isinstance(query, dict) or packet.get("images", {}).get("query") != digest + or query.get("image_ref") != ref + or query.get("actual_image_digest") != digest): + raise ValueError(f"Query must be verified against candidate {ref.split(':', 1)[-1]}, not production v2.2.2") def recovery_summary(samples): @@ -136,6 +191,8 @@ def validate_packet(packet, *, development): for role in ("kafka", "core", "stream", "python"): if not re.fullmatch(r"sha256:[0-9a-f]{64}", packet.get("images", {}).get(role, "")): raise ValueError(f"immutable {role} image ID required") + if os.environ.get(ENV_IMAGES) and packet.get("images") != json.loads(os.environ[ENV_IMAGES]): + raise ValueError("packet images differ from the parent-attested image set") restart_count(packet) validate_query_provenance(packet) if packet.get("stream_binary_override"): @@ -591,7 +648,8 @@ def start(consumer, token, **kwargs): "range_recovery_measurement_n": 1, "ack_measured": False, "fresh_snapshot_rpc": "MarketDataStreamService/GetSnapshot", - "snapshot_path": "isolated candidate Query 2.2.3-rc2-43bbcf0 to canonical hot read; not production v2.2.2 or projector recovery proof"} + "snapshot_path": "isolated candidate Query " + packet["query_provenance"]["image_ref"].split(":", 1)[-1] + + " to canonical hot read; not production v2.2.2 or projector recovery proof"} facts[CELLS[2]] = {"observed": True, "passed": fast_end["code"] == "OK" and fast_end["count"] == REPLAY + 1 and fast_end["last"] == baseline_id + REPLAY + 1 and bool(progress_samples) and slow_resumed, @@ -919,11 +977,17 @@ def candidate_stream_provenance(path, *, development): "container_path": "/work/candidate-stream", "scope": "development binary overlay; base image unchanged"} -def orchestrate(work, *, development, source_sha, image_digest, restart_samples=None, stream_binary=None): - """Adapt measure_l2_top_materialization's isolated broker/core packet.""" +def orchestrate(work, *, development, source_sha, image_digest, restart_samples=None, stream_binary=None, + images=None, query_image_ref=None): + """Adapt measure_l2_top_materialization's isolated broker/core packet. + + images: optional {role: sha256 id} overrides of IMAGES; query_image_ref names the Query image.""" import shutil import subprocess root = Path(__file__).resolve().parents[1] + run_images = resolve_images(images) + query_ref = resolve_query_ref(run_images, query_image_ref) + attested = {ENV_IMAGES: json.dumps(run_images, sort_keys=True), ENV_QUERY_IMAGE_REF: query_ref} if not re.fullmatch(r"[0-9a-f]{40}", source_sha) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_digest): raise ValueError("TS provenance must use full SHA/digest syntax even in development") if not development: @@ -950,12 +1014,12 @@ def inventory(): or r["Name"] == "/market_data_service_p184w"} before = inventory() try: - for image in IMAGES.values(): + for image in run_images.values(): actual = command(["docker", "image", "inspect", image, "--format", "{{.Id}}"]) if actual != image: raise ValueError("local image digest mismatch") runtime = json.loads(command(["docker", "inspect", "qdl_v2_stable_candidate-rust_core-1"]))[0] - if runtime["Image"] != IMAGES["core"]: + if runtime["Image"] != run_images["core"]: raise ValueError("deployed core image drift") cfg = json.loads(command(["docker", "exec", runtime["Id"], "cat", "/runtime/core.json"])) selected = [b for b in cfg["core"]["bindings"] if b["venue"] == "BINANCE" and b["market"] == "USDM" @@ -969,8 +1033,8 @@ def inventory(): cfg["authority"].update(mode="RUST_SHADOW", slice_id=namespace, approved_by="isolated DR08 test") (work / "core.json").write_text(json.dumps(cfg)) packet = {"namespace": namespace, "development": development, "ts_source_sha": source_sha, - "ts_image_digest": image_digest, "images": IMAGES, "bootstrap": "localhost:9092", - "restart_samples": count, "query_provenance": {"image_ref": QUERY_IMAGE_REF, + "ts_image_digest": image_digest, "images": run_images, "bootstrap": "localhost:9092", + "restart_samples": count, "query_provenance": {"image_ref": query_ref, "scope": "isolated candidate Query; not production v2.2.2"}} if candidate: shutil.copyfile(stream_binary, work / "candidate-stream") @@ -1009,29 +1073,29 @@ def launch(role, image, args, *, memory, cpus, network, env=None, detach=True, e (work / (role + "-command-error.log")).write_text(diagnostic[-16384:]) raise penv = {"PYTHONPATH": "/app:/app/generated/python", "PYTHONDONTWRITEBYTECODE": "1", - "P184_DR08_ISOLATED_NAMESPACE": namespace} + "P184_DR08_ISOLATED_NAMESPACE": namespace, **attested} script = ["python", "-B", "scripts/p184_dr08_isolated.py"] - launch("prepare", IMAGES["python"], script + ["--prepare-material", "/work"], + launch("prepare", run_images["python"], script + ["--prepare-material", "/work"], memory="512m", cpus=.5, network="none", env=penv, detach=False) - broker = launch("kafka", IMAGES["kafka"], ["/bin/sh", "-c", + broker = launch("kafka", run_images["kafka"], ["/bin/sh", "-c", "/opt/kafka/bin/kafka-storage.sh format -t MkU3OEVBNTcwNTJENDM2Qk -c /work/broker.properties" " && exec /opt/kafka/bin/kafka-server-start.sh /work/broker.properties"], memory="1536m", cpus=1, network="none", env={"KAFKA_HEAP_OPTS": "-Xms256m -Xmx512m", "LOG_DIR": "/tmp/kafka-logs"}, extra=("--hostname", "localhost", "--tmpfs", "/data:rw,size=512m,mode=1777")) net = "container:" + broker - launch("topics", IMAGES["python"], script + ["--prepare-topics", "/work"], + launch("topics", run_images["python"], script + ["--prepare-topics", "/work"], memory="256m", cpus=.5, network=net, env=penv, detach=False) packet = json.loads((work / "packet.json").read_text()) - launch("quota", IMAGES["redis"], ["redis-server", "--save", "", "--appendonly", "no", + launch("quota", run_images["redis"], ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "48mb", "--maxmemory-policy", "noeviction"], memory="64m", cpus=.1, network=net) common = {"QDL_ENVIRONMENT": "paper", "QDL_KN_CURSOR_KEYS_FILE": "/work/cursor-keys.json", "QDL_KN_CURSOR_ACTIVE_KEY_ID": "dr08", "QDL_KN_TOPIC_ID": packet["slice"]["topic_id"], "QDL_KN_ROUTE_GENERATION": namespace, "QDL_KN_READ_VIEW_SECRET_FILE": "/work/read-view.secret"} - launch("core", IMAGES["core"], ["/usr/local/bin/qdl-realtime-core", "/work/core.json"], + launch("core", run_images["core"], ["/usr/local/bin/qdl-realtime-core", "/work/core.json"], memory="256m", cpus=.5, network=net, env={"QDL_KAFKA_CERT_ROOT": "/work", "QDL_KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", "QDL_KAFKA_CLIENT_ID": namespace, "QDL_KAFKA_GROUP_ID": namespace, "QDL_PROVIDER_ADMISSION_ENABLED": "false"}) - stream = launch("stream", IMAGES["stream"], ["/work/candidate-stream" if candidate else "/usr/local/bin/qdl-stream-gateway", "serve"], + stream = launch("stream", run_images["stream"], ["/work/candidate-stream" if candidate else "/usr/local/bin/qdl-stream-gateway", "serve"], memory="512m", cpus=.5, network=net, env={**common, "QDL_KN_BUNDLE_FILE": "/work/bundle.json", "QDL_KN_JWT_CONFIG_FILE": "/work/jwt-config.json", "QDL_KN_QUOTA_REDIS_URL": "redis://localhost:6379/0", "QDL_KN_QUOTA_PREFIX": "qdl:stable:v2:paper:" + namespace, @@ -1043,11 +1107,11 @@ def launch(role, image, args, *, memory, cpus, network, env=None, detach=True, e "QDL_KN_READ_VIEW_URLS": "https://localhost:8200", "QDL_KN_READ_VIEW_CA_FILE": "/work/ca.crt", "QDL_KN_HOT_READ_ENABLED": "true", "QDL_KN_METRICS_SECONDS": "1", "QDL_KN_MAX_REPLAY_EVENTS": str(REPLAY), "QDL_KN_QUEUE_BYTES_TOTAL": str(QUEUE_BYTES)}) - query = launch("query", IMAGES["query"], script + ["--query", "/work"], memory="512m", cpus=.5, + query = launch("query", run_images["query"], script + ["--query", "/work"], memory="512m", cpus=.5, network=net, env={**penv, **common}) query_state = json.loads(command(["docker", "inspect", query]))[0] packet["query_provenance"]["actual_image_digest"] = query_state["Image"] - validate_query_provenance(packet) + validate_query_provenance(packet, expected=(query_ref, run_images["query"])) # Query may be reading its startup packet; publish the attestation atomically. attested_packet = work / "packet.attested.json" attested_packet.write_text(json.dumps(packet)) @@ -1074,7 +1138,7 @@ def launch(role, image, args, *, memory, cpus, network, env=None, detach=True, e driver_args = script + ["--packet", "/work/packet.json", "--output", "/work/evidence.jsonl"] if development: driver_args += ["--development"] - launch("driver", IMAGES["python"], driver_args, memory="512m", cpus=.75, network=net, env=penv, detach=False) + launch("driver", run_images["python"], driver_args, memory="512m", cpus=.75, network=net, env=penv, detach=False) finally: if log_process is not None: log_process.terminate() @@ -1124,6 +1188,9 @@ def main(argv=None): parser.add_argument("--stream-binary", type=Path, help="development-only isolated candidate executable") parser.add_argument("--restart-samples", type=int, help="orchestrator restart count: development 1-10, final 5-10") + parser.add_argument("--image", action="append", default=[], metavar="ROLE=sha256:ID", + help="orchestrator image override (repeatable); IMAGES are the defaults") + parser.add_argument("--query-image-ref", help="orchestrator Query image label, required with a new query digest") args = parser.parse_args(argv) if args.orchestrate: def orchestration_deadline(_signum, _frame): @@ -1131,7 +1198,8 @@ def orchestration_deadline(_signum, _frame): previous = signal.signal(signal.SIGALRM, orchestration_deadline) signal.alarm(600) try: - orchestrate(args.orchestrate, development=args.development, source_sha=args.ts_source_sha, image_digest=args.ts_image_digest, restart_samples=args.restart_samples, stream_binary=args.stream_binary) + orchestrate(args.orchestrate, development=args.development, source_sha=args.ts_source_sha, image_digest=args.ts_image_digest, restart_samples=args.restart_samples, stream_binary=args.stream_binary, + images=parse_image_overrides(args.image), query_image_ref=args.query_image_ref) finally: signal.alarm(0) signal.signal(signal.SIGALRM, previous) @@ -1140,6 +1208,8 @@ def orchestration_deadline(_signum, _frame): parser.error("--stream-binary requires --orchestrate") if args.restart_samples is not None: parser.error("--restart-samples requires --orchestrate; inner driver uses its sealed packet") + if args.image or args.query_image_ref: + parser.error("--image/--query-image-ref require --orchestrate; inner containers read the attested env") for directory, function in ((args.prepare_material, prepare_material), (args.prepare_topics, prepare_topics), (args.query, serve_query)): if directory is not None: function(directory) diff --git a/scripts/p184_g13_final.py b/scripts/p184_g13_final.py index e573f5f6..bf982d8e 100644 --- a/scripts/p184_g13_final.py +++ b/scripts/p184_g13_final.py @@ -2,7 +2,8 @@ """P18.4 gate G13: run the DR01-DR08 steps that cannot be inherited, once, on the final DL + TS writer digests. Purpose. G13 (TS upgrade/p184-phase-d-gates.md) closes every DR cell of the frozen R2.5 case matrix -(contracts/baseline/rust-first-v2/p18/p184-r2-case-matrix.v1.json in the TS repo, 65 DR cells) on the final +(contracts/baseline/rust-first-v2/p18/p184-r2-case-matrix.v2.json in the TS repo: 76 DR cells = the 65 of v1 plus +11 inverse cells scoped OKX_INVERSE; v1 stays readable) on the final Data Layer python/rust images and the final TS writer image. The inherit/rerun decision per DR is recorded in upgrade/evidence/p184-g13-inherit-rerun-matrix.md; DECISIONS below is the same table in code. This runner calls ONLY the existing harnesses, sequentially, each under a hard timeout: @@ -12,16 +13,21 @@ dr08-isolated DL scripts/p184_dr08_isolated.py --orchestrate (QUALIFICATION mode, >= 5 restart samples) RUNTIME phase (reads the shared stack after the final rollout; needs the owner-approved runtime packet): dr01-deployed DL scripts/p184_dr01_deployed.py prepare/probe/record (7 DL cells) + dr01-wrong-account TS scripts/p184_dr01_wrong_account.py probe (inside the deployed Gateway) + record; + GATEWAY-DEPENDENT: refused as GATEWAY_NOT_RUNNING (other steps still run) unless the Gateway + container runs the declared gateway image; it closes with the TS release (IDR-085 Gateway) dr02-dr03-alpha TS scripts/p184_dr_alpha_reads.py probe (inside each alpha runtime) + record - dr04-dr06-views TS scripts/p184_dr_views.py (inside the TS writer image) + record; also DR05.QUIET_TYPED + dr04-dr06-views TS scripts/p184_dr_views.py (inside the TS writer image; binance, okx, okx_inverse) + record; + also DR05.QUIET_TYPED (linear only: QUIET is a TRADE state, TRADE is not granted for inverse) dr05-transport TS scripts/p184_dr05_isolated.py (writer clone, transport cut at the clone only) dr07-reference TS scripts/p184_dr07_reference.py Receipts: a NEW directory ~/.local/state/qdl-v2/p184-g13-TZ/ (exclusive create) with plan.json, one /receipt.json per step (argv, cwd, times, exit, timeout, output sha256, cleanup) and summary.json. Boundary. Never builds an image, never starts/stops/recreates a shared container, never writes shared Redis/Kafka/ -Postgres, never pushes. It refuses to start a step when (a) a harness still pins other image IDs than the final -ones (the harness containers re-validate their own constants, so a host-side override cannot qualify), (b) the +Postgres, never pushes. The DR05/DR08 harnesses get the final images as explicit arguments (--image ROLE=ID, +DR08 also --query-image-ref; their in-container checks compare against these attested values; kafka/redis keep the +harness defaults). It refuses to start a step when (a) the TS case matrix v2 DR cells differ from DR_CELLS, (b) the DL checkout that the harness containers mount at /app is not the release commit or has tracked edits, (c) for the RUNTIME phase, the deployed roles do not run the final digests or no approval packet is given. Cleanup: only containers whose name starts with the step's own prefix AND that did not exist before the step are removed, by @@ -64,14 +70,25 @@ "market_projector_1": "rust", "market_projector_2": "rust", "stream_kn_1": "rust", "stream_kn_2": "rust", } -# Harness image pins that run inside its containers (scripts/p184_dr08_isolated.py IMAGES; dr05_dl_fault imports it). -PIN_ROLES = {"query": "python", "python": "python", "core": "rust", "stream": "rust"} +# Harness image roles set to the final images (scripts/p184_dr08_isolated.py IMAGES stay the defaults for the rest). +HARNESS_ROLES = {"query": "python", "python": "python", "core": "rust", "stream": "rust"} +MATRIX_V2 = "contracts/baseline/rust-first-v2/p18/p184-r2-case-matrix.v2.json" +GATEWAY_STEPS = ("dr01-wrong-account",) def cells(case, names, venues=VENUES): return tuple(f"{case}.{name}@{venue}" for name in names for venue in venues) +INVERSE = ("OKX_INVERSE",) # TS case matrix v2: OKX BTC-USD-SWAP, binding r23 QUOTE/BOOK/MARK_INDEX only +INV_DR04 = cells("DR04", ("L2_SNAPSHOT_DELTA_SEQUENCE_DEPTH", "MARK_INDEX_COMPONENTS", "QUOTE_BBO", + "RISK_REREADS_OWN_AUTHORITY"), INVERSE) +INV_DR05 = cells("DR05", ("APPROVED_FALLBACK_AND_RETURN_NO_FORBIDDEN_V1", "DISCONNECT_RECONNECT", "GAP_REFUSAL", + "GENERATION_CHANGE_FENCED", "STALE_TYPED"), INVERSE) +INV_DR06 = cells("DR06", ("EXPLICIT_LAST_L2_MARK_STILL_FAIL_WHEN_INVALID", + "UNRELATED_STALE_TRADE_DOES_NOT_BLOCK_MARKET_LIMIT"), INVERSE) + + DR_CELLS = { "DR01": cells("DR01", ("BINDING_COMPILE", "WRONG_ACCOUNT_DENIED", "WRONG_REALM_DENIED", "WRONG_REVISION_DENIED")) + ("DR01.NO_GLOBAL_GRANT@ANY",), @@ -80,19 +97,19 @@ def cells(case, names, venues=VENUES): "DR03": cells("DR03", ("APPEND_1M_NO_FUTURE_SKIP_DUPLICATE", "NO_FULL_HISTORY_RELOAD_PER_CYCLE", "RESTART_SAME_CANDLE_NO_DUPLICATE_DECISION")), "DR04": cells("DR04", ("EXPLICIT_LAST", "L2_SNAPSHOT_DELTA_SEQUENCE_DEPTH", "MARK_INDEX_COMPONENTS", "QUOTE_BBO", - "RISK_REREADS_OWN_AUTHORITY")), + "RISK_REREADS_OWN_AUTHORITY")) + INV_DR04, "DR05": cells("DR05", ("APPROVED_FALLBACK_AND_RETURN_NO_FORBIDDEN_V1", "DISCONNECT_RECONNECT", "GAP_REFUSAL", - "GENERATION_CHANGE_FENCED", "QUIET_TYPED", "STALE_TYPED")), + "GENERATION_CHANGE_FENCED", "QUIET_TYPED", "STALE_TYPED")) + INV_DR05, "DR06": cells("DR06", ("EXPLICIT_LAST_L2_MARK_STILL_FAIL_WHEN_INVALID", - "UNRELATED_STALE_TRADE_DOES_NOT_BLOCK_MARKET_LIMIT")), + "UNRELATED_STALE_TRADE_DOES_NOT_BLOCK_MARKET_LIMIT")) + INV_DR06, "DR07": cells("DR07", ("FUNDING_RATE", "INSTRUMENT_METADATA_PROVENANCE", "LONG_SHORT_RATIO", "OPEN_INTEREST", "TAKER_FLOW")) + ("DR07.FX_PROOF@ANY", "DR07.REFERENCE_NEVER_EXECUTION_AUTHORITY@ANY"), "DR08": tuple(f"DR08.{n}@ANY" for n in ("ALERT_PATH_OBSERVABLE", "CONSUMER_RESTART_RECOVERY_MEASURED", "NO_CROSS_CONSUMER_STARVATION", "SLOW_CONSUMER_BOUNDED_QUEUE")), } -# Only the CONTROLLED_SIM p184_dr01_runtime.py emits these, which never closes DR01; the TS Gateway runner that -# should does not exist yet. Listed, never faked. -NO_PRODUCER = cells("DR01", ("WRONG_ACCOUNT_DENIED",)) +# Inverse DR05 faults: the DL fault harness (p184_dr05_dl_fault.py isolated_config: ETH BOOK bindings only) and the +# TS writer clone (p184_dr05_isolated.py SYMBOLS: ETH only) have no BTC-USD-SWAP scope yet. Listed open, never faked. +NO_PRODUCER = INV_DR05 # DR -> (decision, reason). Same table as upgrade/evidence/p184-g13-inherit-rerun-matrix.md. DECISIONS = { @@ -114,12 +131,13 @@ def cells(case, names, venues=VENUES): STEP_SPEC = ( # id, phase, cells, timeout_s, est_minutes, owned container prefixes - ("dr05-upstream", "isolated", tuple(c for c in DR_CELLS["DR05"] if "GAP_" in c or "GENERATION_" in c), 1800, 5, + ("dr05-upstream", "isolated", cells("DR05", ("GAP_REFUSAL", "GENERATION_CHANGE_FENCED")), 1800, 5, ("p184-dr05-",)), ("dr08-isolated", "isolated", DR_CELLS["DR08"], 1500, 12, ("p184-dr08-",)), - ("dr01-deployed", "runtime", tuple(c for c in DR_CELLS["DR01"] if c not in NO_PRODUCER), 600, 3, ()), + ("dr01-deployed", "runtime", tuple(c for c in DR_CELLS["DR01"] if "WRONG_ACCOUNT" not in c), 600, 3, ()), + ("dr01-wrong-account", "runtime", cells("DR01", ("WRONG_ACCOUNT_DENIED",)), 600, 2, ()), ("dr02-dr03-alpha", "runtime", DR_CELLS["DR02"] + DR_CELLS["DR03"], 1200, 10, ()), - ("dr04-dr06-views", "runtime", DR_CELLS["DR04"] + DR_CELLS["DR06"] + cells("DR05", ("QUIET_TYPED",)), 1200, 12, + ("dr04-dr06-views", "runtime", DR_CELLS["DR04"] + DR_CELLS["DR06"] + cells("DR05", ("QUIET_TYPED",)), 1200, 14, ()), ("dr05-transport", "runtime", cells("DR05", ("STALE_TYPED", "DISCONNECT_RECONNECT", "APPROVED_FALLBACK_AND_RETURN_NO_FORBIDDEN_V1")), 1500, 8, @@ -143,39 +161,30 @@ def coverage(): missing = sorted(set(every) - set(produced) - set(NO_PRODUCER)) extra = sorted(set(produced) - set(every)) return {"cells": len(every), "produced": len(set(produced)), "no_producer": list(NO_PRODUCER), - "duplicates": duplicates, "missing": missing, "extra": extra} - - -def harness_pins(source): - """Image IDs the DR05/DR08 harness containers validate against, read from source without importing it.""" - tree = ast.parse(source) - consts, images = {}, None - for node in tree.body: - if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): - name = node.targets[0].id - if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): - consts[name] = node.value.value - elif name == "IMAGES" and isinstance(node.value, ast.Dict): - images = {} - for key, value in zip(node.value.keys, node.value.values): - images[key.value] = consts.get(value.id) if isinstance(value, ast.Name) else value.value - if images is None: - raise Blocked("HARNESS_PINS_UNREADABLE: no IMAGES dict in scripts/p184_dr08_isolated.py") - return {"images": images, "query_image_ref": consts.get("QUERY_IMAGE_REF"), - "query_image_digest": consts.get("QUERY_IMAGE_DIGEST")} - - -def pin_mismatches(pins, final): - """Empty when the harness pins equal the final python/rust images and python image ref.""" - found = [] - for role, kind in PIN_ROLES.items(): - if pins["images"].get(role) != final[kind]: - found.append(f"IMAGES[{role!r}]={pins['images'].get(role)} != final {kind} {final[kind]}") - if pins["query_image_digest"] != final["python"]: - found.append(f"QUERY_IMAGE_DIGEST={pins['query_image_digest']} != final python {final['python']}") - if pins["query_image_ref"] != final["python_ref"]: - found.append(f"QUERY_IMAGE_REF={pins['query_image_ref']} != {final['python_ref']}") - return found + "duplicates": duplicates + sorted(set(produced) & set(NO_PRODUCER)), "missing": missing, "extra": extra} + + +def harness_image_args(final): + """--image ROLE=ID for every harness role that must run a final image (both harnesses accept the flag).""" + out = [] + for role, kind in HARNESS_ROLES.items(): + out += ["--image", f"{role}={final[kind]}"] + return out + + +def matrix_dr_cells(matrix): + """DR cell ids of a loaded TS case matrix (v1 or v2); unknown schemas refused.""" + if matrix.get("schema") not in ("p184.r2-case-matrix.v1", "p184.r2-case-matrix.v2"): + raise Blocked(f"CASE_MATRIX_SCHEMA: {matrix.get('schema')!r}") + return sorted(c["id"] for c in matrix["cells"] if c["case"].startswith("DR")) + + +def matrix_mismatch(matrix): + """Empty when DR_CELLS equals the TS matrix's DR cells exactly.""" + ours = sorted(c for case in DR_CELLS.values() for c in case) + theirs = matrix_dr_cells(matrix) + return {"missing_here": sorted(set(theirs) - set(ours)), "extra_here": sorted(set(ours) - set(theirs))} \ + if ours != theirs else {} def validate_final(final): @@ -224,6 +233,7 @@ def required_inputs(step): "dr08-isolated": (), "dr01-deployed": ("dr01.inventory", "dr01.deployments.BINANCE", "dr01.deployments.OKX", "dr01.compose", "dr01.alpha_service"), + "dr01-wrong-account": ("gateway_container", "gateway_image_digest"), "dr02-dr03-alpha": ("alpha.containers.BINANCE", "alpha.containers.OKX", "alpha.symbols.BINANCE", "alpha.symbols.OKX"), "dr04-dr06-views": ("ts_container",), @@ -258,8 +268,8 @@ def commands(step, final, inputs, work, ts_root): alpha_record = ts_root / "scripts/p184_dr_alpha_reads.py" if step == "dr05-upstream": w = work / "work" - return [cmd(py + ["scripts/p184_dr05_dl_fault.py", "--init", w, "--core-config", inputs["dr05_public_export"]], - cwd=DL_ROOT), + return [cmd(py + ["scripts/p184_dr05_dl_fault.py", "--init", w, "--core-config", inputs["dr05_public_export"], + *harness_image_args(final)], cwd=DL_ROOT), cmd(py + ["scripts/p184_dr05_dl_fault.py", "--up", w], cwd=DL_ROOT), cmd(py + ["scripts/p184_dr05_dl_fault.py", "--ready", w], cwd=DL_ROOT), cmd(py + ["scripts/p184_dr05_upstream.py", "--work", w, "--namespace", "", @@ -270,7 +280,8 @@ def commands(step, final, inputs, work, ts_root): if step == "dr08-isolated": return [cmd(py + ["scripts/p184_dr08_isolated.py", "--orchestrate", work / "work", "--ts-source-sha", final["ts_sha"], "--ts-image-digest", final["writer"], "--restart-samples", - str(inputs.get("dr08_restart_samples", 5))], cwd=DL_ROOT)] + str(inputs.get("dr08_restart_samples", 5)), *harness_image_args(final), + "--query-image-ref", final["python_ref"]], cwd=DL_ROOT)] if step == "dr01-deployed": d = inputs["dr01"] out = [] @@ -294,6 +305,13 @@ def commands(step, final, inputs, work, ts_root): record.append("--ci-confirmed") out.append(cmd(py + record, cwd=DL_ROOT)) return out + if step == "dr01-wrong-account": + script = ts_root / "scripts/p184_dr01_wrong_account.py" + return [cmd(["docker", "exec", "-i", inputs["gateway_container"], "python", "-B", "-", "probe"], cwd=ts_root, + stdin=script, stdout=work / "dr01-wrong-account.rows"), + cmd(py + [script, "record", "--rows", work / "dr01-wrong-account.rows", *ts_ev, + "--gateway-image-digest", inputs["gateway_image_digest"], "--output", + work / "dr01-wrong-account.jsonl"], cwd=ts_root)] if step == "dr02-dr03-alpha": a = inputs["alpha"] out = [] @@ -309,7 +327,7 @@ def commands(step, final, inputs, work, ts_root): if step == "dr04-dr06-views": out = [cmd(["docker", "exec", "-i", inputs["ts_container"], "python", "-B", "-", "--venue", venue.lower()], cwd=ts_root, stdin=ts_root / "scripts/p184_dr_views.py", stdout=work / f"views-{venue.lower()}.rows") - for venue in VENUES] + for venue in VIEW_SCOPES] out.append(cmd(py + [alpha_record, "record", "--rows", work / "views.rows", *ts_ev, "--output", work / "dr04-dr06.jsonl"], cwd=ts_root)) return out @@ -326,8 +344,9 @@ def commands(step, final, inputs, work, ts_root): # Rows printed by the two in-container probes are concatenated per step before `record` (it takes one file). +VIEW_SCOPES = ("BINANCE", "OKX", "OKX_INVERSE") # p184_dr_views.py --venue binance|okx|okx_inverse CONCAT = {"dr02-dr03-alpha": ("alpha.rows", ("alpha-binance.rows", "alpha-okx.rows")), - "dr04-dr06-views": ("views.rows", ("views-binance.rows", "views-okx.rows"))} + "dr04-dr06-views": ("views.rows", ("views-binance.rows", "views-okx.rows", "views-okx_inverse.rows"))} def dr05_channel_argv(helper_command, python_image, ns, work): @@ -350,31 +369,33 @@ def receipt_root(base, now): return Path(base) / ("p184-g13-" + now.strftime("%Y%m%dT%H%M%SZ")) -def plan(final, inputs, phase, only, ts_root, base, now, pins): +def plan(final, inputs, phase, only, ts_root, base, now, matrix=None): + """matrix: the loaded TS case matrix (v2); when given, its DR cells must equal DR_CELLS for any step to run.""" steps = select(phase, only) spec = {s[0]: s for s in STEP_SPEC} root = receipt_root(base, now) problems = {} - mismatch = pin_mismatches(pins, final) + drift = matrix_mismatch(matrix) if matrix is not None else {} missing = missing_inputs(steps, inputs) out = [] for step in steps: _, ph, step_cells, timeout, est, prefixes = spec[step] blocked = [] - if step in ("dr05-upstream", "dr08-isolated") and mismatch: - blocked.append("HARNESS_IMAGE_PIN: " + "; ".join(mismatch)) + if drift: + blocked.append("CASE_MATRIX_MISMATCH: " + json.dumps(drift, sort_keys=True)) if step in missing: blocked.append("MISSING_INPUTS: " + ", ".join(missing[step])) if ph == "runtime" and not inputs.get("_runtime_approval"): blocked.append("RUNTIME_APPROVAL_REQUIRED: --runtime-approval ") entry = {"step": step, "phase": ph, "cells": list(step_cells), "timeout_s": timeout, "est_minutes": est, - "owned_prefixes": list(prefixes), "blocked": blocked} + "owned_prefixes": list(prefixes), "blocked": blocked, "gateway_dependent": step in GATEWAY_STEPS} if not blocked: entry["commands"] = commands(step, final, inputs, root / step, ts_root) out.append(entry) if blocked: problems[step] = blocked return {"schema": "p184.g13-final-plan.v1", "receipts": str(root), "final": final, + "case_matrix": {"schema": (matrix or {}).get("schema"), "sha256": (matrix or {}).get("sha256")}, "decisions": {k: {"decision": v[0], "reason": v[1]} for k, v in DECISIONS.items()}, "coverage": coverage(), "steps": out, "blocked": problems, "est_minutes_total": sum(e["est_minutes"] for e in out), @@ -403,6 +424,26 @@ def deployed_inventory(): return out +def gateway_problem(state, expected_image): + """None when the Gateway container runs the declared image; else the GATEWAY_NOT_RUNNING reason. + state: {"running": bool, "image": id} or None when the container does not exist.""" + if not state: + return "GATEWAY_NOT_RUNNING: container absent" + if not state.get("running"): + return "GATEWAY_NOT_RUNNING: container not running" + if state.get("image") != expected_image: + return f"GATEWAY_NOT_RUNNING: runs {state.get('image')} != declared gateway image {expected_image}" + return None + + +def gateway_state(name): + try: + row = json.loads(sh(["docker", "inspect", name]))[0] + except (subprocess.CalledProcessError, json.JSONDecodeError, IndexError): + return None + return {"running": bool(row["State"]["Running"]), "image": row["Image"]} + + def checkout_problems(dl_sha): head = sh(["git", "-C", str(DL_ROOT), "rev-parse", "HEAD"]).strip() dirty = sh(["git", "-C", str(DL_ROOT), "status", "--porcelain", "--untracked-files=no", "--", "qdl", "rust", @@ -465,7 +506,8 @@ def run_dr05_upstream(entry, final, inputs, work, deadline, log, ts_root): watcher = subprocess.Popen([sys.executable, "-B", "scripts/p184_dr05_reset.py", "--serve-reset-host", str(work / "work"), "--duration", str(int(max(60, deadline - time.monotonic())))], cwd=DL_ROOT, stdout=log, stderr=log, start_new_session=True) - channel = shlex.join(dr05_channel_argv(dr.helper_command, dr.IMAGES["python"], ns, work / "work")) + channel = shlex.join(dr05_channel_argv(dr.helper_command, dr.image_set(work / "work")["python"], ns, + work / "work")) argv = [ns if a == "" else channel if a == "" else a for a in adapter["argv"]] codes.append(bounded({**adapter, "argv": argv}, deadline, log)) @@ -495,6 +537,11 @@ def run_step(entry, final, inputs, ts_root): try: if clash: raise Blocked("OWNED_PREFIX_IN_USE: " + ", ".join(sorted(clash.values()))) + if entry.get("gateway_dependent"): + problem = gateway_problem(gateway_state(inputs["gateway_container"]), inputs["gateway_image_digest"]) + if problem: + receipt["deferred"] = problem # closes with the TS release; the other steps still run + raise Blocked(problem) if entry["step"] == "dr05-upstream": codes = run_dr05_upstream(entry, final, inputs, work, deadline, log, ts_root) else: @@ -544,8 +591,11 @@ def main(argv=None): inputs["_runtime_approval"] = {"path": str(a.runtime_approval), "sha256": sha256(a.runtime_approval)} try: validate_final(final) - pins = harness_pins((DL_ROOT / "scripts/p184_dr08_isolated.py").read_text()) - result = plan(final, inputs, a.phase, tuple(a.only), a.ts_root, a.receipts, datetime.now(timezone.utc), pins) + matrix_path = a.ts_root / MATRIX_V2 + if not matrix_path.is_file(): + raise Blocked(f"CASE_MATRIX_MISSING: {matrix_path}") + matrix = json.loads(matrix_path.read_text()) + result = plan(final, inputs, a.phase, tuple(a.only), a.ts_root, a.receipts, datetime.now(timezone.utc), matrix) except Blocked as exc: print(json.dumps({"blocked": str(exc)})) return 2 @@ -570,9 +620,11 @@ def main(argv=None): flush=True) summary = {"schema": "p184.g13-final-summary.v1", "final": final, "steps": [ {k: r[k] for k in ("step", "cells", "duration_s", "exit_codes", "error", "harness_ok", "removed_leftovers")} - for r in receipts], "note": "harness_ok is the harness exit status; cells close only via the TS matrix check"} + for r in receipts], "deferred": {r["step"]: r["deferred"] for r in receipts if r.get("deferred")}, + "note": "harness_ok is the harness exit status; cells close only via the TS matrix check; deferred steps " + "(Gateway not on the declared image) stay open until the TS release"} (root / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") - return 0 if all(r["harness_ok"] for r in receipts) else 1 + return 0 if all(r["harness_ok"] or r.get("deferred") for r in receipts) else 1 if __name__ == "__main__": diff --git a/tests/test_p184_dr05_dl_fault.py b/tests/test_p184_dr05_dl_fault.py index ada97d6a..8ae423f6 100644 --- a/tests/test_p184_dr05_dl_fault.py +++ b/tests/test_p184_dr05_dl_fault.py @@ -121,6 +121,73 @@ def fresh_case(self, *args, **kwargs): dr.drive(Adapter(), "OKX", "GAP_REFUSAL") +FINAL_PY, FINAL_RUST = "sha256:" + "a1" * 32, "sha256:" + "b2" * 32 +OVERRIDES = {"python": FINAL_PY, "query": FINAL_PY, "core": FINAL_RUST, "stream": FINAL_RUST} + + +class ImageSetTests(unittest.TestCase): + """--image overrides are sealed in handoff.json; every later step reads them, defaults stay the pins.""" + + def test_defaults_unchanged_without_overrides(self): + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "case" + packet = dr.prepare(work) + self.assertEqual(packet["images"], dr.IMAGES) + self.assertEqual(dr.image_set(work), dr.IMAGES) + self.assertEqual(dr.image_set(None), dr.IMAGES) + self.assertEqual(dr.image_set(Path(tmp) / "absent"), dr.IMAGES) + + def test_overrides_sealed_and_used_by_every_later_step(self): + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "case" + packet = dr.prepare(work, OVERRIDES) + sealed = json.loads((work / "handoff.json").read_text())["images"] + self.assertEqual(sealed, packet["images"]) + self.assertEqual({k: sealed[k] for k in OVERRIDES}, OVERRIDES) + self.assertEqual(sealed["kafka"], dr.IMAGES["kafka"]) # roles not overridden keep the defaults + commands = dr.topology(packet["namespace"], work) + self.assertIn(FINAL_RUST, commands["core"]) + self.assertIn(FINAL_RUST, commands["stream"]) + self.assertIn(FINAL_PY, commands["query"]) + self.assertIn(dr.IMAGES["kafka"], commands["kafka"]) + self.assertNotIn(dr.IMAGES["core"], commands["core"]) + helper = dr.helper_command(packet["namespace"], work, ["--topics", "/work"]) + self.assertIn(FINAL_PY, helper) + cfg = dr.isolated_config(MaterialTests.exported(), packet["namespace"], dr.image_set(work)) + self.assertEqual(cfg["authority"]["candidate_image_digest"], FINAL_RUST) + + def test_tampered_or_partial_handoff_image_set_refused(self): + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "case" + dr.prepare(work, OVERRIDES) + handoff = json.loads((work / "handoff.json").read_text()) + for images in ({"core": FINAL_RUST}, {**handoff["images"], "core": "latest"}, "x"): + (work / "handoff.json").write_text(json.dumps({**handoff, "images": images})) + with self.subTest(images=images), self.assertRaises(ValueError): + dr.image_set(work) + + def test_readiness_checks_the_sealed_images(self): + rows = LifecycleTests() + rows.setUp() + images = dr.resolve_images(OVERRIDES) + self.assertTrue(dr.readiness(NS, rows.call)["process_isolation_ready"]) # rows run the defaults + self.assertFalse(dr.readiness(NS, rows.call, images)["process_isolation_ready"]) + for role in dr.ROLES: + rows.rows[role]["Image"] = images[{"quota": "redis"}.get(role, role)] + self.assertTrue(dr.readiness(NS, rows.call, images)["process_isolation_ready"]) + + def test_cli_image_flag_only_at_init_or_prepare(self): + with tempfile.TemporaryDirectory() as tmp, patch("sys.stdout"), patch("sys.stderr"): + work = Path(tmp) / "case" + self.assertEqual(dr.dispatch(["--prepare", str(work), "--image", "core=" + FINAL_RUST]), 0) + self.assertEqual(dr.image_set(work)["core"], FINAL_RUST) + for argv in (["--up", str(work), "--image", "core=" + FINAL_RUST], + ["--prepare", str(Path(tmp) / "b"), "--image", "core=latest"], + ["--prepare", str(Path(tmp) / "c"), "--image", "nope=" + FINAL_RUST]): + with self.subTest(argv=argv), self.assertRaises((SystemExit, ValueError)): + dr.dispatch(argv) + + class LifecycleTests(unittest.TestCase): def setUp(self): self.rows = {} diff --git a/tests/test_p184_dr08_isolated.py b/tests/test_p184_dr08_isolated.py index 4376c492..58a2c910 100644 --- a/tests/test_p184_dr08_isolated.py +++ b/tests/test_p184_dr08_isolated.py @@ -512,5 +512,128 @@ def run(args, **kwargs): +FINAL_PY, FINAL_RUST = "sha256:" + "a1" * 32, "sha256:" + "b2" * 32 +FINAL_REF = "qdl-v2-python:2.2.3-abcdef1" +OVERRIDES = {"python": FINAL_PY, "query": FINAL_PY, "core": FINAL_RUST, "stream": FINAL_RUST} + + +class ImageParameters(unittest.TestCase): + """The image set is a parameter; the pinned constants are defaults; containers check the attested values.""" + + def test_parse_and_resolve(self): + flags = [f"{k}={v}" for k, v in OVERRIDES.items()] + self.assertEqual(R.parse_image_overrides(flags), OVERRIDES) + self.assertEqual(R.resolve_images(), R.IMAGES) + resolved = R.resolve_images(OVERRIDES) + self.assertEqual({k: resolved[k] for k in OVERRIDES}, OVERRIDES) + self.assertEqual(resolved["kafka"], R.IMAGES["kafka"]) + for bad in (["core"], ["core=latest"], ["nope=" + FINAL_RUST], ["core=" + FINAL_RUST, "core=" + FINAL_RUST]): + with self.subTest(bad=bad), self.assertRaises(ValueError): + R.parse_image_overrides(bad) + + def test_query_ref_and_digest_change_together(self): + self.assertEqual(R.resolve_query_ref(R.IMAGES), R.QUERY_IMAGE_REF) + self.assertEqual(R.resolve_query_ref(R.resolve_images(OVERRIDES), FINAL_REF), FINAL_REF) + for images, ref in ((R.resolve_images(OVERRIDES), None), (R.IMAGES, FINAL_REF), + (R.resolve_images(OVERRIDES), "final")): + with self.subTest(ref=ref), self.assertRaises(ValueError): + R.resolve_query_ref(images, ref) + + def _final_packet(self): + p = packet() + p["images"] = R.resolve_images(OVERRIDES) + p["query_provenance"] = {"image_ref": FINAL_REF, "actual_image_digest": FINAL_PY} + return p + + def test_in_container_check_uses_the_attested_env(self): + env = {R.ENV_IMAGES: json.dumps(R.resolve_images(OVERRIDES)), R.ENV_QUERY_IMAGE_REF: FINAL_REF} + final = self._final_packet() + with patch.dict(os.environ, env): + R.validate_query_provenance(final) + with self.assertRaises(ValueError): + R.validate_query_provenance(packet()) # the rc2 packet is not what the parent attested + with patch.dict(os.environ, {}, clear=False): + os.environ.pop(R.ENV_IMAGES, None) + os.environ.pop(R.ENV_QUERY_IMAGE_REF, None) + R.validate_query_provenance(packet()) # no attestation: the pinned defaults, as before + with self.assertRaises(ValueError): + R.validate_query_provenance(final) + + def test_packet_images_must_equal_the_attested_set(self): + p = self._final_packet() + env = {"P184_DR08_ISOLATED_NAMESPACE": p["namespace"], R.ENV_IMAGES: json.dumps(p["images"]), + R.ENV_QUERY_IMAGE_REF: FINAL_REF} + with patch.dict(os.environ, env): + R.validate_packet(p, development=True) + p["images"] = {**p["images"], "stream": "sha256:" + "c" * 64} + with self.assertRaisesRegex(ValueError, "parent-attested"): + R.validate_packet(p, development=True) + + def test_flags_only_with_orchestrate(self): + for argv in (["--image", "core=" + FINAL_RUST], ["--query-image-ref", FINAL_REF]): + with self.subTest(argv=argv), self.assertRaises(SystemExit), patch("sys.stderr"): + R.main(["--packet", "p", "--output", "o", *argv]) + + def test_orchestrate_launches_and_attests_the_passed_images(self): + import subprocess + from contextlib import redirect_stdout + import io + calls = [] + cfg = {"core": {"bindings": [{"venue": "BINANCE", "market": "USDM", + "native_symbol": "BTCUSDT", "native_channel": "btcusdt@trade"}]}, "authority": {}} + final_images = R.resolve_images(OVERRIDES) + + def run(args, **kwargs): + calls.append(args) + out = "" + if args[:3] == ["docker", "image", "inspect"]: + out = args[3] + elif args[:3] == ["docker", "inspect", "qdl_v2_stable_candidate-rust_core-1"]: + out = json.dumps([{"Image": FINAL_RUST, "Id": "shared-core"}]) + elif args[:3] == ["docker", "exec", "shared-core"]: + out = json.dumps(cfg) + elif args[:2] == ["docker", "run"]: + raise subprocess.TimeoutExpired(args, 100) # stop at the first launch + elif args[:2] == ["docker", "inspect"]: + raise subprocess.CalledProcessError(1, args) + return subprocess.CompletedProcess(args, 0, stdout=out, stderr="") + + with tempfile.TemporaryDirectory() as d, patch("subprocess.run", side_effect=run), \ + redirect_stdout(io.StringIO()): + target = Path(d) / "attempt" + with self.assertRaises(subprocess.TimeoutExpired): + R.orchestrate(target, development=True, source_sha="0" * 40, image_digest="sha256:" + "0" * 64, + images=OVERRIDES, query_image_ref=FINAL_REF) + written = json.loads((target / "packet.json").read_text()) + inspected = {a[3] for a in calls if a[:3] == ["docker", "image", "inspect"]} + self.assertEqual(inspected, set(final_images.values())) + self.assertEqual(written["images"], final_images) + self.assertEqual(written["query_provenance"]["image_ref"], FINAL_REF) + launch = next(a for a in calls if a[:2] == ["docker", "run"]) + self.assertIn(FINAL_PY, launch) # the prepare role runs the final python image + self.assertIn(R.ENV_IMAGES + "=" + json.dumps(final_images, sort_keys=True), launch) + self.assertIn(R.ENV_QUERY_IMAGE_REF + "=" + FINAL_REF, launch) + + def test_orchestrate_refuses_a_deployed_core_other_than_the_passed_one(self): + import subprocess + from contextlib import redirect_stdout + import io + + def run(args, **kwargs): + out = "" + if args[:3] == ["docker", "image", "inspect"]: + out = args[3] + elif args[:3] == ["docker", "inspect", "qdl_v2_stable_candidate-rust_core-1"]: + out = json.dumps([{"Image": R.IMAGES["core"], "Id": "shared-core"}]) + self.assertNotEqual(args[:2], ["docker", "run"]) + return subprocess.CompletedProcess(args, 0, stdout=out, stderr="") + + with tempfile.TemporaryDirectory() as d, patch("subprocess.run", side_effect=run), \ + redirect_stdout(io.StringIO()): + with self.assertRaisesRegex(ValueError, "deployed core image drift"): + R.orchestrate(Path(d) / "attempt", development=True, source_sha="0" * 40, + image_digest="sha256:" + "0" * 64, images=OVERRIDES, query_image_ref=FINAL_REF) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_p184_g13_final.py b/tests/test_p184_g13_final.py index 49bb1851..e7641ebb 100644 --- a/tests/test_p184_g13_final.py +++ b/tests/test_p184_g13_final.py @@ -14,31 +14,57 @@ WRITER = "sha256:" + "c3" * 32 FINAL = {"python": PY, "rust": RUST, "python_ref": "qdl-v2-python:2.2.3-abcdef1", "writer": WRITER, "ts_sha": "1" * 39 + "2", "dl_sha": "3" * 39 + "4"} -PINS_OK = {"images": {"query": PY, "python": PY, "core": RUST, "stream": RUST, "kafka": "sha256:" + "d" * 64}, - "query_image_ref": FINAL["python_ref"], "query_image_digest": PY} +TS_MATRIX_V2 = Path("/home/bobby/.worktrees/trading-system-next-upgrade") / g13.MATRIX_V2 + + +def matrix_of(dr_cells): + return {"schema": "p184.r2-case-matrix.v2", "sha256": "f" * 64, + "cells": [{"id": c, "case": c.split(".")[0]} for c in dr_cells] + [{"id": "OR01.X@OKX", "case": "OR01"}]} + + +MATRIX_OK = matrix_of([c for case in g13.DR_CELLS.values() for c in case]) INPUTS = {"dr05_public_export": "/x/export.json", "redis_image": "redis:7.2-alpine", "postgres_image": "pg@sha256:1", "dr01": {"inventory": "/x/inv.json", "deployments": {"BINANCE": "db", "OKX": "do"}, "compose": "/x/c.json", "alpha_service": "alpha"}, "alpha": {"containers": {"BINANCE": "ab", "OKX": "ao"}, "symbols": {"BINANCE": "ETHUSDT", "OKX": "ETH-USDT-SWAP"}}, "ts_container": "market_data_service_p184w", + "gateway_container": "gateway_service", "gateway_image_digest": "sha256:" + "e" * 64, "dr07": {"connections": "/x/conn.json", "fx_account": "acct"}, "_runtime_approval": {"path": "/x/packet.md", "sha256": "e" * 64}} NOW = datetime(2026, 10, 10, 9, 0, 0, tzinfo=timezone.utc) -def make_plan(phase="all", only=(), inputs=None, pins=None): +def make_plan(phase="all", only=(), inputs=None, matrix=None): return g13.plan(FINAL, INPUTS if inputs is None else inputs, phase, only, Path("/ts"), Path("/state"), NOW, - PINS_OK if pins is None else pins) + MATRIX_OK if matrix is None else matrix) class CoverageTests(unittest.TestCase): def test_every_dr_cell_has_one_producer_or_is_listed_open(self): cov = g13.coverage() - self.assertEqual(cov["cells"], 65) # frozen TS matrix p184-r2-case-matrix.v1: 65 DR cells + self.assertEqual(cov["cells"], 76) # TS matrix v2: 65 DR cells of v1 + 11 OKX_INVERSE self.assertEqual((cov["duplicates"], cov["missing"], cov["extra"]), ([], [], [])) - self.assertEqual(cov["produced"] + len(cov["no_producer"]), 65) - self.assertEqual(cov["no_producer"], ["DR01.WRONG_ACCOUNT_DENIED@BINANCE", "DR01.WRONG_ACCOUNT_DENIED@OKX"]) + self.assertEqual(cov["produced"] + len(cov["no_producer"]), 76) + # Only the inverse DR05 faults lack a producer (harnesses are ETH-only); WRONG_ACCOUNT now has its TS runner. + self.assertEqual(sorted(cov["no_producer"]), sorted(g13.INV_DR05)) + self.assertEqual(len(g13.INV_DR04 + g13.INV_DR05 + g13.INV_DR06), 11) + + def test_dr_cells_equal_the_committed_ts_matrix_v2(self): + if not TS_MATRIX_V2.is_file(): + self.skipTest("TS worktree not mounted") + matrix = json.loads(TS_MATRIX_V2.read_text()) + self.assertEqual(matrix["schema"], "p184.r2-case-matrix.v2") + self.assertEqual(g13.matrix_mismatch(matrix), {}) + + def test_matrix_mismatch_blocks_every_step(self): + drifted = matrix_of([c for case in g13.DR_CELLS.values() for c in case if c not in g13.INV_DR04] + + ["DR09.NEW@ANY"]) + result = make_plan("all", matrix=drifted) + self.assertEqual(sorted(result["blocked"]), sorted(g13.select("all"))) + self.assertTrue(all(b[0].startswith("CASE_MATRIX_MISMATCH") for b in result["blocked"].values())) + with self.assertRaises(g13.Blocked): + g13.matrix_dr_cells({"schema": "p184.r2-case-matrix.v9", "cells": []}) def test_every_dr_has_a_decision(self): self.assertEqual(sorted(g13.DECISIONS), sorted(g13.DR_CELLS)) @@ -48,6 +74,7 @@ class SelectionTests(unittest.TestCase): def test_isolated_runs_before_runtime_in_matrix_order(self): self.assertEqual(g13.select("all"), list(g13.STEPS)) self.assertEqual(g13.select("isolated"), ["dr05-upstream", "dr08-isolated"]) + self.assertIn("dr01-wrong-account", g13.select("runtime")) self.assertEqual(g13.select("all", ("dr07-reference", "dr08-isolated")), ["dr08-isolated", "dr07-reference"]) def test_unknown_or_empty_selection_refused(self): @@ -73,24 +100,27 @@ def test_development_zero_digests_and_shas_refused(self): g13.validate_final({**FINAL, key: value}) g13.validate_final(FINAL) - def test_harness_pins_read_without_import(self): - source = ('QUERY_IMAGE_REF = "qdl-v2-python:2.2.3-rc2-43bbcf0"\nQUERY_IMAGE_DIGEST = "sha256:77"\n' - 'IMAGES = {"query": QUERY_IMAGE_DIGEST, "core": "sha256:65", "stream": "sha256:1f",' - ' "python": "sha256:77"}\nimport not_installed_module\n') - pins = g13.harness_pins(source) - self.assertEqual(pins["images"]["query"], "sha256:77") - self.assertEqual(pins["query_image_ref"], "qdl-v2-python:2.2.3-rc2-43bbcf0") - self.assertEqual(len(g13.pin_mismatches(pins, FINAL)), 6) - with self.assertRaises(g13.Blocked): - g13.harness_pins("X = 1\n") + def test_harness_images_are_the_final_images_not_the_rc2_pins(self): + args = g13.harness_image_args(FINAL) + pairs = dict(a.split("=", 1) for a in args[1::2]) + self.assertEqual(args[0::2], ["--image"] * 4) + self.assertEqual(pairs, {"query": PY, "python": PY, "core": RUST, "stream": RUST}) + result = make_plan("isolated") + self.assertEqual(result["blocked"], {}) + steps = {e["step"]: e for e in result["steps"]} + init = steps["dr05-upstream"]["commands"][0]["argv"] + dr08 = steps["dr08-isolated"]["commands"][0]["argv"] + for argv in (init, dr08): + self.assertIn("python=" + PY, argv) + self.assertIn("core=" + RUST, argv) + self.assertEqual(dr08[dr08.index("--query-image-ref") + 1], FINAL["python_ref"]) + self.assertNotIn("--image", steps["dr05-upstream"]["commands"][1]["argv"]) # --up reads handoff.json - def test_current_harness_pins_are_rc2_and_block_isolated_steps(self): - pins = g13.harness_pins((g13.DL_ROOT / "scripts/p184_dr08_isolated.py").read_text()) - self.assertTrue(g13.pin_mismatches(pins, FINAL)) - result = make_plan("isolated", pins=pins) - self.assertEqual(sorted(result["blocked"]), ["dr05-upstream", "dr08-isolated"]) - self.assertTrue(all("commands" not in e for e in result["steps"])) - self.assertTrue(all(b[0].startswith("HARNESS_IMAGE_PIN") for b in result["blocked"].values())) + def test_harness_clis_accept_the_runner_flags(self): + from scripts import p184_dr05_dl_fault as dr, p184_dr08_isolated as r + self.assertEqual(r.parse_image_overrides(g13.harness_image_args(FINAL)[1::2]), + {"query": PY, "python": PY, "core": RUST, "stream": RUST}) + self.assertEqual(dr.resolve_images(r.parse_image_overrides(g13.harness_image_args(FINAL)[1::2]))["core"], RUST) def test_deployed_inventory_must_match_final(self): good = {g13.STACK + r + "-1": (PY if k == "python" else RUST) for r, k in g13.ROLE_IMAGE.items()} @@ -143,11 +173,52 @@ def test_concatenation_targets_match_record_rows(self): steps = {e["step"]: e for e in make_plan()["steps"]} for step, (target, parts) in g13.CONCAT.items(): commands = steps[step]["commands"] - self.assertEqual([Path(c["stdout"]).name for c in commands[:2]], list(parts)) + self.assertEqual([Path(c["stdout"]).name for c in commands[:len(parts)]], list(parts)) record = commands[-1]["argv"] self.assertEqual(Path(record[record.index("--rows") + 1]).name, target) +class GatewayStepTests(unittest.TestCase): + def test_wrong_account_step_probes_inside_the_gateway_and_records_with_its_image(self): + steps = {e["step"]: e for e in make_plan("runtime")["steps"]} + entry = steps["dr01-wrong-account"] + self.assertTrue(entry["gateway_dependent"]) + self.assertEqual(entry["cells"], ["DR01.WRONG_ACCOUNT_DENIED@BINANCE", "DR01.WRONG_ACCOUNT_DENIED@OKX"]) + probe, record = entry["commands"] + self.assertEqual(probe["argv"][:4], ["docker", "exec", "-i", "gateway_service"]) + self.assertEqual(probe["argv"][-2:], ["-", "probe"]) + self.assertTrue(probe["stdin"].endswith("scripts/p184_dr01_wrong_account.py")) + argv = record["argv"] + self.assertEqual(argv[argv.index("--gateway-image-digest") + 1], INPUTS["gateway_image_digest"]) + self.assertEqual(argv[argv.index("--source-sha") + 1], FINAL["ts_sha"]) + self.assertEqual(argv[argv.index("--rows") + 1], probe["stdout"]) + self.assertFalse(steps["dr01-deployed"]["gateway_dependent"]) + + def test_missing_gateway_inputs_block_only_that_step(self): + inputs = {k: v for k, v in INPUTS.items() if k != "gateway_container"} + result = make_plan("runtime", inputs=inputs) + self.assertEqual(list(result["blocked"]), ["dr01-wrong-account"]) + + def test_gateway_not_running_is_typed(self): + image = INPUTS["gateway_image_digest"] + self.assertIsNone(g13.gateway_problem({"running": True, "image": image}, image)) + for state in (None, {"running": False, "image": image}, {"running": True, "image": PY}): + self.assertTrue(g13.gateway_problem(state, image).startswith("GATEWAY_NOT_RUNNING")) + + def test_absent_gateway_defers_the_step_without_failing_the_run(self): + from unittest import mock + entry = {e["step"]: e for e in make_plan("runtime")["steps"]}["dr01-wrong-account"] + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch.object(g13, "containers", return_value={}), \ + mock.patch.object(g13, "gateway_state", return_value=None), \ + mock.patch.object(g13, "bounded") as bounded: + receipt = g13.run_step(entry, FINAL, {**INPUTS, "_root": tmp}, Path("/ts")) + bounded.assert_not_called() + self.assertFalse(receipt["harness_ok"]) + self.assertTrue(receipt["deferred"].startswith("GATEWAY_NOT_RUNNING")) + self.assertIn("GATEWAY_NOT_RUNNING", receipt["error"]) + + class HelperTests(unittest.TestCase): def test_channel_argv_matches_joint_construction(self): def helper_command(ns, work, args, *, network): @@ -174,10 +245,14 @@ def test_dry_run_prints_plan_and_creates_nothing(self): code = g13.main(["--python-image", PY, "--rust-image", RUST, "--python-image-ref", FINAL["python_ref"], "--writer-image", WRITER, "--ts-source-sha", FINAL["ts_sha"], "--dl-source-sha", FINAL["dl_sha"], "--phase", "isolated", "--receipts", tmp, - "--dry-run"]) + "--only", "dr08-isolated", "--dry-run", "--ts-root", str(TS_MATRIX_V2.parents[4])]) result = json.loads(out.getvalue()) - self.assertEqual(code, 3) # the checked-in harness still pins rc2 images - self.assertEqual(sorted(result["blocked"]), ["dr05-upstream", "dr08-isolated"]) + if not TS_MATRIX_V2.is_file(): + self.assertEqual(code, 2) # no TS matrix v2 to bind DR_CELLS to + return + self.assertEqual(code, 0) # final images are passed to the harnesses; nothing pins rc2 any more + self.assertEqual(result["blocked"], {}) + self.assertEqual(result["case_matrix"]["schema"], "p184.r2-case-matrix.v2") self.assertEqual(list(Path(tmp).iterdir()), []) def test_bad_identity_exits_before_plan(self): diff --git a/upgrade/evidence/p184-g13-inherit-rerun-matrix.md b/upgrade/evidence/p184-g13-inherit-rerun-matrix.md index 4c3ecb66..464944ce 100644 --- a/upgrade/evidence/p184-g13-inherit-rerun-matrix.md +++ b/upgrade/evidence/p184-g13-inherit-rerun-matrix.md @@ -1,18 +1,25 @@ # P18.4 G13 - DR01-DR08 inherit / rerun matrix (final DL + writer digests) Status: PLAN / NOT RUN (2026-10-10, Claude coordinator, read-only planning). No DR, build or runtime action was -taken to produce this. Runner: `scripts/p184_g13_final.py` (tested locally, 16 unit tests; never executed against +taken to produce this. Runner: `scripts/p184_g13_final.py` (tested locally, 22 unit tests; never executed against Docker). Gate: G13 in TS `upgrade/p184-phase-d-gates.md` - "DR01-DR08 (incl. DR05 typed gap/generation fence, -inverse feeds): every cell closed once on the final DL + writer digests", frozen cells in TS -`contracts/baseline/rust-first-v2/p18/p184-r2-case-matrix.v1.json` (65 DR cells: DR01 9, DR02 8, DR03 6, DR04 10, -DR05 12, DR06 4, DR07 12, DR08 4; counted with a Python read of that file). +inverse feeds): every cell closed once on the final DL + writer digests", cells in TS +`contracts/baseline/rust-first-v2/p18/p184-r2-case-matrix.v2.json` (TS `3538b48`; v1 kept unchanged and readable): +76 DR cells = the 65 of v1 (DR01 9, DR02 8, DR03 6, DR04 10, DR05 12, DR06 4, DR07 12, DR08 4) + 11 inverse cells +scoped `OKX_INVERSE` (DR04 4, DR05 5, DR06 2). Counted with a Python read of the file; the runner refuses +(`CASE_MATRIX_MISMATCH`) if its `DR_CELLS` differ from the matrix. + +Update 2026-10-10 (later): blockers 1, 4 and 5 below are resolved in code (harness image parameters; TS +WRONG_ACCOUNT Gateway runner `6c44bf2` wired as step `dr01-wrong-account`; matrix v2 inverse cells). One new +open item: the 5 inverse DR05 cells have no producer yet (see blocker 5). ## Result in one line -**Inherit 0 of 65 cells.** No DR cell has a receipt on final-qualifying provenance, and every DR exercises at least +**Inherit 0 of 76 cells.** No DR cell has a receipt on final-qualifying provenance, and every DR exercises at least one path changed after its newest receipt. What can be inherited are inputs, not cells (see "Inherited inputs"). -The minimum is therefore seven harness steps, ~52 min of estimated wall time run serially (hard timeouts sum to -140 min), plus the two DR01 WRONG_ACCOUNT_DENIED cells that have no producer anywhere. +The minimum is therefore eight harness steps, ~56 min of estimated wall time run serially (hard timeouts sum to +150 min). `dr01-wrong-account` is Gateway-dependent and closes with the TS release; the 5 inverse DR05 cells +have no producer yet. ## Receipts that exist (all found with `grep -rl` over `~/.local/state` for `DR0n.` cell ids) @@ -59,12 +66,12 @@ and has no final-provenance receipt to inherit from. | DR | Cells | Exercised paths (roles) | Receipt | Decision | Reason | Step / est. min | |---|---|---|---|---|---|---| -| DR01 | 9 (7 DL + 2 WRONG_ACCOUNT) | alpha SDK credentials -> Query HTTP `qdl/api_v2/router.py`, `qdl/security/data_plane.py` (JWT/realm/revision/entitlement), binding compiler `qdl/consumer/universal_release.py`, `consumers/stable/alpha-*-paper.yaml` | rehearsal only | RERUN | no deployed-probe receipt; router changed (`616342c`, `65e0757`, `76950c8`, `57881a2`, `47bec04`) | `dr01-deployed` / 3 | +| DR01 | 9 (7 DL + 2 WRONG_ACCOUNT via TS Gateway runner) | alpha SDK credentials -> Query HTTP `qdl/api_v2/router.py`, `qdl/security/data_plane.py` (JWT/realm/revision/entitlement), binding compiler `qdl/consumer/universal_release.py`, `consumers/stable/alpha-*-paper.yaml` | rehearsal only | RERUN | no deployed-probe receipt; router changed (`616342c`, `65e0757`, `76950c8`, `57881a2`, `47bec04`) | `dr01-deployed` / 3, `dr01-wrong-account` / 2 (Gateway-dependent) | | DR02 | 8 | alpha `AlphaDataLayerV2` -> SDK transport -> Query warmup/batch, row cache, cold BAR (`qdl/query/*`, `kn_query_backend`, `stable_source`) | none | RERUN | warmup/batch path and SDK transport changed | `dr02-dr03-alpha` / 10 | | DR03 | 6 | same as DR02 plus 1m append (Query BAR, projector-fed spool) | none | RERUN | same | (same step) | -| DR04 | 10 | Stream/Query -> TS writer -> TS Redis -> `MarketCacheReader` + Risk validator; MARK/INDEX hot path, L2 view | none | RERUN | IDR-084/Astra MARK, IDR-082, IDR-040 fix 2 changed; writer digest new | `dr04-dr06-views` / 12 | -| DR05 | 12 | GAP/GEN: raw -> core fence (`canonical.rs`, `realtime-core/lib.rs`, `qdl-production-core.rs`) -> stream/query `kn_hot_view` -> writer clone -> production `_validate_book`; STALE/DISCONNECT/FALLBACK: writer clone transport; QUIET: TS views | r16 rehearsal (4 cells) | RERUN | core `1a2daf1`, query `57881a2`/`922a6ec`, stream `0b374bc`/`fdd379e`, writer all changed after r16; 8 cells never produced | `dr05-upstream` / 5, `dr05-transport` / 8, QUIET inside `dr04-dr06-views` | -| DR06 | 4 | production `services.risk_engine.execution_market_context` on a copy of the live context (writer views) | none | RERUN | reads changed MARK/L2 views; writer digest new | (in `dr04-dr06-views`) | +| DR04 | 10 + 4 inverse | Stream/Query -> TS writer -> TS Redis -> `MarketCacheReader` + Risk validator; MARK/INDEX hot path, L2 view | none | RERUN | IDR-084/Astra MARK, IDR-082, IDR-040 fix 2 changed; writer digest new | `dr04-dr06-views` / 14 (binance, okx, okx_inverse) | +| DR05 | 12 + 5 inverse (no producer) | GAP/GEN: raw -> core fence (`canonical.rs`, `realtime-core/lib.rs`, `qdl-production-core.rs`) -> stream/query `kn_hot_view` -> writer clone -> production `_validate_book`; STALE/DISCONNECT/FALLBACK: writer clone transport; QUIET: TS views | r16 rehearsal (4 cells) | RERUN | core `1a2daf1`, query `57881a2`/`922a6ec`, stream `0b374bc`/`fdd379e`, writer all changed after r16; 8 cells never produced | `dr05-upstream` / 5, `dr05-transport` / 8, QUIET inside `dr04-dr06-views` | +| DR06 | 4 + 2 inverse | production `services.risk_engine.execution_market_context` on a copy of the live context (writer views) | none | RERUN | reads changed MARK/L2 views; writer digest new | (in `dr04-dr06-views`) | | DR07 | 12 | alpha `alpha_sdk.reference` -> Query reference routes (`router.py`) -> `qdl/reference/*`; FX proof from TS DB | none | RERUN | no R2 DR07 rows; router changed | `dr07-reference` / 2 | | DR08 | 4 | isolated kafka -> core -> stream `service.rs`/`subscription.rs`/`hub.rs` -> query GetSnapshot | development only | RERUN | stream `fdd379e` after the overlay; final mode never ran | `dr08-isolated` / 12 | @@ -84,25 +91,37 @@ transport `--ready-seconds 180` + 2 x (20 s cut + <= 120 s recover)). These are ## Blockers before the final run (none is fixed here) -1. **Harness image pins.** `scripts/p184_dr08_isolated.py` pins `IMAGES` (rc2 python `77682eea`, core `658a9570`, - stream `1f13408f`) and `QUERY_IMAGE_REF/DIGEST` (2.2.3-rc2-43bbcf0); `p184_dr05_dl_fault.py` imports the same - `IMAGES`. The containers re-validate against those constants (`validate_query_provenance`, readiness image - check), so the runner cannot override them from the host. Needs an additive, reviewed change in the harness - (final image IDs as arguments, constants as defaults) before `dr05-upstream`/`dr08-isolated` can run. The - runner refuses with `HARNESS_IMAGE_PIN` until then (its dry run shows this today). +1. **Harness image pins - RESOLVED (code, tested locally).** `scripts/p184_dr08_isolated.py` `IMAGES` / + `QUERY_IMAGE_REF` are now defaults only: `--orchestrate` takes `--image ROLE=sha256:ID` (repeatable) and + `--query-image-ref`, passes the resolved set to every container as `P184_DR08_IMAGES` / `P184_DR08_QUERY_IMAGE_REF`, + and `validate_query_provenance` / `validate_packet` compare against those attested values (pinned defaults when + absent, so behaviour without flags is unchanged). `p184_dr05_dl_fault.py --init/--prepare` takes the same + `--image` flag and seals the set in `handoff.json` `images`; `--up/--ready/--down`, reset, channel, readiness and + the in-container `--material` read it via `image_set(work)`. The runner passes the final python (query, python) + and rust (core, stream) images; kafka/redis keep the harness defaults. `HARNESS_IMAGE_PIN` refusal removed. 2. **DR08 reads the deployed core.** `orchestrate` copies `/runtime/core.json` from `qdl_v2_stable_candidate-rust_core-1` and refuses a core image other than its pin; with final pins DR08 runs only after the final rust image is deployed. The runner therefore checks the deployed inventory before DR08 too. 3. **Python code is mounted from the checkout** (`-v
:/app:ro`, `PYTHONPATH=/app`) in DR05/DR08: the checkout must sit at the release commit. The runner refuses if HEAD differs from `--dl-source-sha` or tracked files under `qdl rust scripts generated consumers config contracts` are modified. -4. **DR01.WRONG_ACCOUNT_DENIED@BINANCE/OKX have no qualifying producer.** `grep -rln WRONG_ACCOUNT` finds only - the CONTROLLED_SIM `scripts/p184_dr01_runtime.py` (DL), which the runbook forbids for closing DR01, and no TS - script outside the case matrix. The runbook assigns them to a TS Gateway runner that does not exist yet. -5. **Inverse feeds.** G13 names inverse feeds, but the frozen matrix v1 has no inverse-scoped DR cell and - `p184_dr_views.py` `IDENTITY` is linear only (ETHUSDT / ETH-USDT-SWAP). Either a matrix revision adds inverse DR - cells (gates file line 119: "Inverse gets its own cells; linear is not inverse evidence") or G13's wording is - narrowed by the owner. Owner decision; not silently dropped. +4. **DR01.WRONG_ACCOUNT_DENIED@BINANCE/OKX - producer exists, Gateway-dependent.** TS `6c44bf2` + `scripts/p184_dr01_wrong_account.py` (probe inside the deployed Gateway, record on the host). Runner step + `dr01-wrong-account` (runtime; inputs `gateway_container`, `gateway_image_digest`): probe via + `docker exec -i python -B - probe`, then `record --rows .. --source-sha .. --image-digest + --gateway-image-digest ..`. At run time it is refused as `GATEWAY_NOT_RUNNING` (receipt `deferred`, other steps + still run, overall exit not failed by it) unless the container is running the declared gateway image; it closes + with the TS release, when the Gateway runs the final image incl. IDR-085. +5. **Inverse feeds - cells added (TS matrix v2), 5 without producer.** Deployed writer binding (r23, manifest + revision 11, `market_data_service_p184w` mount) grants OKX BTC-USD-SWAP exactly QUOTE, BOOK_SNAPSHOT, BOOK_DELTA, + MARK_INDEX_PRICE (TRADE/BAR route BLOCK, no V1 fallback). Added `@OKX_INVERSE`: DR04 QUOTE_BBO, + L2_SNAPSHOT_DELTA_SEQUENCE_DEPTH, MARK_INDEX_COMPONENTS, RISK_REREADS_OWN_AUTHORITY; DR06 both cells; DR05 + STALE_TYPED, DISCONNECT_RECONNECT, GAP_REFUSAL, GENERATION_CHANGE_FENCED, APPROVED_FALLBACK_AND_RETURN_NO_FORBIDDEN_V1. + Not added (reasons in the matrix `inverse_excluded`): DR01 (no alpha grant), DR02/DR03 (BAR), DR04.EXPLICIT_LAST + and DR05.QUIET_TYPED (TRADE), DR07 (no reference grant), DR08 (@ANY). `p184_dr_views.py` reads the inverse scope + (`--venue okx_inverse`), so DR04/DR06 inverse run inside `dr04-dr06-views`. **Open:** the 5 inverse DR05 cells + have no producer - `p184_dr05_dl_fault.py isolated_config` exports ETH BOOK bindings only and TS + `p184_dr05_isolated.py` `SYMBOLS` is ETH only; both need a BTC-USD-SWAP scope (runner lists them `no_producer`). 6. **DR01 record needs CI confirmation** (`--ci-confirmed` only when the inputs file states `dr01.ci_confirmed: true` for the TS source SHA). 7. RUNTIME steps read the shared stack and open extra Paper sessions: they need the owner-approved runtime packet @@ -115,7 +134,7 @@ transport `--ready-seconds 180` + 2 x (20 s cut + <= 120 s recover)). These are python3 -B scripts/p184_g13_final.py --python-image sha256: --rust-image sha256: \ --python-image-ref qdl-v2-python:2.2.3- --writer-image sha256: \ --ts-source-sha <40hex> --dl-source-sha <40hex> --phase all --inputs --dry-run -# isolated phase (after blocker 1; DR08 also after the rust image is deployed) +# isolated phase (DR08 after the final rust image is deployed; --ts-root must hold the TS matrix v2) ... --phase isolated # runtime phase (after rollout + writer roll, approved packet) ... --phase runtime --inputs --runtime-approval @@ -125,7 +144,8 @@ python3 -B scripts/p184_g13_final.py --python-image sha256: --rust-image sha Inputs JSON keys: `dr05_public_export`, `redis_image`, `postgres_image`, `dr01.{inventory, deployments.BINANCE, deployments.OKX, compose, alpha_service, compose_cwd?, catalog?, ci_confirmed?}`, `alpha.{containers.*, symbols.*, -universe.*?}`, `ts_container`, `dr07.{connections, fx_account, dsn_env?}`, `dr08_restart_samples?` (default 5). +universe.*?}`, `ts_container`, `gateway_container`, `gateway_image_digest`, `dr07.{connections, fx_account, +dsn_env?}`, `dr08_restart_samples?` (default 5). Receipts: `~/.local/state/qdl-v2/p184-g13-/` (exclusive create): `plan.json`, `/receipt.json` (argv, cwd, UTC start/end, exit codes, timeout/error, sha256 of every output, From e4dfc3cf4d6810ec7a109b4c59ac798f57a634ca Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 08:45:27 +0000 Subject: [PATCH 110/147] tooling(p184): DR05 fault harness scopes incl. OKX inverse BTC-USD-SWAP; G13 runs every DR cell p184_dr05_dl_fault.py: a case scope is a venue and one BOOK instrument (SCOPES: BINANCE, OKX linear ETH; OKX_INVERSE = OKX BTC-USD-SWAP). --init/--prepare --scope (default the two linear scopes, unchanged) and --catalog (source catalog defining the scopes, copied into the work dir, sha256 sealed) are recorded in handoff.json; isolated_config takes each scope's exported BOOK binding, material keeps exactly each scope's BOOK_SNAPSHOT/BOOK_DELTA, one consumer/manifest per scope (okx_inverse), OKX books frames for BTC-USD-SWAP. Publisher (scope-keyed roster, own session for the inverse), observer/Query roster, channel and reset are scope-aware. p184_g13_final.py: both DR05 steps run BINANCE, OKX, OKX_INVERSE (new input dr05_catalog); NO_PRODUCER is empty (76/76 DR cells have a producer). Evidence note lists the fresh export it needs. --- scripts/p184_dr05_case.py | 10 +- scripts/p184_dr05_channel.py | 4 +- scripts/p184_dr05_dl_fault.py | 119 ++++++-- scripts/p184_dr05_observer.py | 21 +- scripts/p184_dr05_raw_publisher.py | 20 +- scripts/p184_dr05_reset.py | 4 +- scripts/p184_g13_final.py | 26 +- tests/test_p184_dr05_inverse_scope.py | 261 ++++++++++++++++++ tests/test_p184_g13_final.py | 40 ++- .../evidence/p184-g13-inherit-rerun-matrix.md | 35 ++- 10 files changed, 465 insertions(+), 75 deletions(-) create mode 100644 tests/test_p184_dr05_inverse_scope.py diff --git a/scripts/p184_dr05_case.py b/scripts/p184_dr05_case.py index 7f055d90..5cb16278 100644 --- a/scripts/p184_dr05_case.py +++ b/scripts/p184_dr05_case.py @@ -30,8 +30,9 @@ def consumer_packet(manifest_path, catalog, venue, issuer, audience): label = venue.lower() cid = manifest.consumer_id dr.namespace(cid.removesuffix("." + label)) - if venue not in dr.VENUES or not cid.endswith("." + label): + if venue not in dr.SCOPES or not cid.endswith("." + label): raise ValueError("isolated venue consumer required") + scope_venue, scope_market, scope_symbol = dr.SCOPES[venue] # `venue` is the case scope (e.g. OKX_INVERSE) policy = UniversalReleasePolicy.load(dr.ROOT / "config/v2/universal-release-policy.yaml", manifest_root=dr.ROOT) # A test-only routing generation, not a production release certificate. digest = hashlib.sha256(Path(manifest_path).read_bytes()).hexdigest() @@ -39,13 +40,14 @@ def consumer_packet(manifest_path, catalog, venue, issuer, audience): for req in manifest.requirements: instrument = catalog.instrument_for(req.instrument_uid) identity = instrument.identity - if identity.venue != venue or req.feed.value not in ("BOOK_SNAPSHOT", "BOOK_DELTA"): - raise ValueError("isolated venue BOOK requirements only") + if ((identity.venue, identity.market, instrument.native_symbol) != (scope_venue, scope_market, scope_symbol) + or req.feed.value not in ("BOOK_SNAPSHOT", "BOOK_DELTA")): + raise ValueError("isolated scope BOOK requirements only") rid = hashlib.sha256(f"{cid}:{requirement_key(req)}".encode()).hexdigest() products.append(UniversalReleaseProduct( consumer_id=cid, consumer_class=UniversalConsumerClass.TRADING_SYSTEM, requirement_id=rid, instrument_uid=identity.instrument_uid, - instrument_id=identity.instrument_id, venue=venue, market=identity.market, + instrument_id=identity.instrument_id, venue=identity.venue, market=identity.market, product_type=identity.product_type.value, native_symbol=instrument.native_symbol, feed=req.feed.value, interval=req.interval, source_policy_id=req.source_policy_id, provider_plane="L2", max_freshness_ms=req.max_freshness_ms, diff --git a/scripts/p184_dr05_channel.py b/scripts/p184_dr05_channel.py index a46421bd..bccca7d3 100644 --- a/scripts/p184_dr05_channel.py +++ b/scripts/p184_dr05_channel.py @@ -24,7 +24,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from scripts.p184_dr05_dl_fault import FAULTS, VENUES +from scripts.p184_dr05_dl_fault import FAULTS, SCOPES, VENUES, scope_set MAX_BYTES = 4 * 1024 * 1024 MAX_RECEIPTS = 64 @@ -156,7 +156,7 @@ def close(self): self.lock.close() def reset(self, venue, fault, deadline): - if venue not in VENUES or fault not in FAULTS: + if venue not in SCOPES or venue not in scope_set(self.work) or fault not in FAULTS: raise ValueError("unknown cell") request_id = uuid.uuid4().hex request = self.control / "reset-request.json" diff --git a/scripts/p184_dr05_dl_fault.py b/scripts/p184_dr05_dl_fault.py index 883ca4c6..ff7bb32d 100644 --- a/scripts/p184_dr05_dl_fault.py +++ b/scripts/p184_dr05_dl_fault.py @@ -9,6 +9,13 @@ scripts/p184_dr08_isolated.py IMAGES, which stay the defaults). The resolved set is sealed in handoff.json "images"; every later step (--up/--ready/--down, reset, channel, the in-container --material) reads it from there via image_set(work), never from the constants. + +Scopes: a case scope is a venue AND one BOOK instrument (SCOPES). The default scopes are the two +linear ETH books (VENUES, unchanged). --init/--prepare --scope S (repeatable) selects them, e.g. +also OKX_INVERSE (OKX BTC-USD-SWAP, the inverse instrument whose BOOK feeds the TS writer binding +r23 grants); --catalog PATH supplies the source catalog that defines it (copied into the work +directory, sha256 sealed in handoff.json "source_catalog"; default config/v2/stable-source-bindings.yaml). +The sealed "scopes" drive material, manifests, consumers, frames and resets via scope_set(work). """ from __future__ import annotations @@ -31,11 +38,52 @@ from scripts.p184_dr08_isolated import IMAGES # immutable image inventory only: the DEFAULT set from scripts.p184_dr08_isolated import parse_image_overrides, resolve_images -VENUES = ("BINANCE", "OKX") +VENUES = ("BINANCE", "OKX") # default scopes (linear ETH BOOK); a scope id equal to its venue +# scope -> (venue, market, native BOOK symbol). OKX_INVERSE: binding r23 grants QUOTE, BOOK_SNAPSHOT, BOOK_DELTA, +# MARK_INDEX_PRICE for BTC-USD-SWAP; this harness exercises its BOOK pair (the DR05 GAP/GENERATION fault path). +SCOPES = {"BINANCE": ("BINANCE", "USDM", "ETHUSDT"), "OKX": ("OKX", "SWAP", "ETH-USDT-SWAP"), + "OKX_INVERSE": ("OKX", "SWAP", "BTC-USD-SWAP")} FAULTS = ("GAP_REFUSAL", "GENERATION_CHANGE_FENCED") LABEL = "p184.dr05.dl" ROLES = ("kafka", "quota", "core", "stream", "query") CELLS = tuple(f"DR05.{fault}@{venue}" for venue in VENUES for fault in FAULTS) +SOURCE_CATALOG = "source-catalog.yaml" + + +def cells_for(scopes): + return tuple(f"DR05.{fault}@{scope}" for scope in scopes for fault in FAULTS) + + +def validate_scopes(scopes): + """Ordered (SCOPES order), unique, non-empty, known scopes; None -> the default linear pair.""" + if scopes is None: + return VENUES + scopes = tuple(scopes) + if not scopes or len(set(scopes)) != len(scopes) or set(scopes) - set(SCOPES): + raise ValueError("known, unique DR05 scopes required: " + ",".join(SCOPES)) + return tuple(s for s in SCOPES if s in scopes) + + +def scope_set(work=None): + """This run's sealed scopes (handoff.json "scopes"), else the default linear pair.""" + handoff = Path(work) / "handoff.json" if work is not None else None + if handoff is not None and handoff.is_file(): + scopes = json.loads(handoff.read_text()).get("scopes") + if scopes is not None: + if not isinstance(scopes, list) or validate_scopes(scopes) != tuple(scopes): + raise ValueError("handoff scopes must be ordered known scopes") + return tuple(scopes) + return VENUES + + +def scope_of(venue, native_symbol): + """The scope of one exported binding / frame identity, or None.""" + return next((s for s, (v, _, sym) in SCOPES.items() if (v, sym) == (venue, native_symbol)), None) + + +def source_catalog_path(work): + copied = Path(work) / SOURCE_CATALOG + return copied if copied.is_file() else ROOT / "config/v2/stable-source-bindings.yaml" def namespace(value): @@ -46,12 +94,12 @@ def namespace(value): return value -def pending_rows(): +def pending_rows(scopes=VENUES): # No provenance header: this file must not be submitted as runtime evidence. return [{"cell": cell, "level": "ISOLATED_FAULT", "ok": False, "outcome": "PREPARED_NOT_RUN", "provenance": "TEST_ONLY", "detail": {"gate": "stage3 real DL path AND TS adapter observations required"}} - for cell in CELLS] + for cell in cells_for(scopes)] def frame(venue, snapshot, sequence, previous, generation, quantity, at_ns): @@ -60,9 +108,9 @@ def frame(venue, snapshot, sequence, previous, generation, quantity, at_ns): Stamp at publish time, not preparation time. Same lane across generations; a lane rename tests a different fence and is not this case. """ - if venue not in VENUES or type(at_ns) is not int or at_ns <= 0: + if venue not in SCOPES or type(at_ns) is not int or at_ns <= 0: raise ValueError("venue and positive integer timestamp required") - symbol = "ETHUSDT" if venue == "BINANCE" else "ETH-USDT-SWAP" + venue, _, symbol = SCOPES[venue] # a scope selects the instrument; the frame names the real venue ms = at_ns // 1_000_000 if venue == "OKX": payload = {"arg": {"channel": "books", "instId": symbol}, @@ -84,7 +132,7 @@ def frame(venue, snapshot, sequence, previous, generation, quantity, at_ns): def schedule(venue, fault): - if venue not in VENUES or fault not in FAULTS: + if venue not in SCOPES or fault not in FAULTS: raise ValueError("unknown DR05 cell") # tuples: phase, snapshot, seq, previous, generation, bid quantity steps = [("positive_anchor", True, 100, 0, 7, "1"), @@ -96,7 +144,7 @@ def schedule(venue, fault): steps += [("new_generation_unanchored", False, 201, 200, 8, "3")] # Binance replays the buffered 201 delta when snapshot200 arrives. # OKX rejects the unanchored delta and starts from snapshot200 instead. - return_previous = 201 if venue == "BINANCE" and fault == "GENERATION_CHANGE_FENCED" else 200 + return_previous = 201 if SCOPES[venue][0] == "BINANCE" and fault == "GENERATION_CHANGE_FENCED" else 200 steps += [("return_anchor", True, 200, 0, 8, "5"), ("return_control", False, 202, return_previous, 8, "6"), ("old_generation", True, 300, 0, 7, "99"), @@ -309,40 +357,50 @@ def cleanup(ns, call=sh): return {"removed": removed, "scope": ns} -def prepare(work, images=None): +def prepare(work, images=None, scopes=None, catalog=None): ns = "p184-dr05-dl-" + uuid.uuid4().hex[:16] images = resolve_images(images) + scopes = validate_scopes(scopes) topology(ns, work, images=images) # Validate path before writing. + source = None + if catalog is not None: + catalog = Path(catalog).resolve(strict=True) + source = {"from": str(catalog), "sha256": hashlib.sha256(catalog.read_bytes()).hexdigest()} work = Path(work).resolve() work.mkdir(mode=0o700, parents=True, exist_ok=False) + if catalog is not None: + (work / SOURCE_CATALOG).write_bytes(catalog.read_bytes()) packet = {"namespace": ns, "status": "PREPARED_HANDOFF_NOT_RUN", "provenance": "TEST_ONLY", - "level": "ISOLATED_FAULT", "images": images, + "level": "ISOLATED_FAULT", "images": images, "scopes": list(scopes), + "source_catalog": source, "deadline_seconds": 600, "acceptance": False, "material_ready": False, "adapter_ready": False, - "schedules": {f"DR05.{f}@{v}": schedule(v, f) for v in VENUES for f in FAULTS}} + "schedules": {f"DR05.{f}@{v}": schedule(v, f) for v in scopes for f in FAULTS}} (work / "handoff.json").write_text(json.dumps(packet, indent=2) + "\n") - (work / "pending-cells.jsonl").write_text("".join(json.dumps(r) + "\n" for r in pending_rows())) + (work / "pending-cells.jsonl").write_text("".join(json.dumps(r) + "\n" for r in pending_rows(scopes))) return packet -def isolated_config(exported, ns, images=None): +def isolated_config(exported, ns, images=None, scopes=VENUES): """Take only public binding fields; never copy transport, credentials or authority.""" namespace(ns) images = images or IMAGES + scopes = validate_scopes(scopes) required = {"provider", "venue", "market", "product_type", "native_symbol", "native_channel", "provider_kind", "instrument_uid", "instrument_id", "instrument_revision", "instrument_catalog_revision", "source_id", "source_role", "normalizer_version", "sequence_policy", "l2"} optional = {"physical_native_symbol", "physical_native_channel", "require_final_bar"} selected = [] - for venue, market, symbol in (("BINANCE", "USDM", "ETHUSDT"), ("OKX", "SWAP", "ETH-USDT-SWAP")): + for scope in scopes: + venue, market, symbol = SCOPES[scope] matches = [b for b in exported["core"]["bindings"] if b.get("l2") and (b.get("venue"), b.get("market"), b.get("native_symbol")) == (venue, market, symbol)] if len(matches) != 1 or not required <= matches[0].keys(): - raise ValueError("exactly one complete exported ETH BOOK binding required: " + venue) + raise ValueError("exactly one complete exported BOOK binding required: " + scope) binding = {k: copy.deepcopy(v) for k, v in matches[0].items() if k in required | optional} if binding["product_type"] != "PERPETUAL" or binding["source_role"] != "PRIMARY": - raise ValueError("ETH primary perpetual BOOK required") + raise ValueError("primary perpetual BOOK required: " + scope) allowed_l2 = {"provider_protocol", "depth_per_side", "snapshot_refresh_seconds", "materialized_snapshot_interval_ms"} if set(binding["l2"]) - allowed_l2: @@ -381,20 +439,21 @@ def material(work, exported): work = Path(work) packet = json.loads((work / "handoff.json").read_text()) namespace = packet["namespace"] - cfg = isolated_config(exported, namespace, image_set(work)) + scopes = scope_set(work) + cfg = isolated_config(exported, namespace, image_set(work), scopes) def save(name, value, private=False): with (work / name).open("x") as handle: os.chmod(work / name, 0o600 if private else 0o644) handle.write(value if isinstance(value, str) else json.dumps(value)) - catalog = yaml.safe_load((ROOT / "config/v2/stable-source-bindings.yaml").read_text()) + catalog = yaml.safe_load(source_catalog_path(work).read_text()) catalog["canonical_stream"] = cfg["core"]["canonical_stream"] ids = {b["source_id"] for b in cfg["core"]["bindings"]} catalog["bindings"] = [b for b in catalog["bindings"] if b["source"]["source_id"] in ids and b["feed"] in ("BOOK_SNAPSHOT", "BOOK_DELTA")] - if len(catalog["bindings"]) != 4: - raise ValueError("both ETH BOOK products must match source catalog") + if len(catalog["bindings"]) != 2 * len(scopes): + raise ValueError("every scope's BOOK snapshot/delta must match the source catalog") for binding in catalog["bindings"]: binding["source"]["adapter_version"] = "p184-synthetic-test-only" save("catalog.yaml", yaml.safe_dump(catalog)) @@ -463,7 +522,8 @@ def save(name, value, private=False): manifests, identities, keys, subjects = [], [], {}, {} for b in cfg["core"]["bindings"]: - label = b["venue"].lower() + scope = scope_of(b["venue"], b["native_symbol"]) + label = scope.lower() cid = namespace + "." + label subject = "spiffe://qdl/paper/" + cid requirements = [] @@ -480,7 +540,7 @@ def save(name, value, private=False): mapping["spec"]["quotas"]["max_buffer_events"] = 1000 save(label + ".yaml", yaml.safe_dump(mapping)) from scripts.p184_dr05_case import consumer_packet - consumer, route = consumer_packet(work / (label + ".yaml"), loaded, b["venue"], ISSUER, AUDIENCE) + consumer, route = consumer_packet(work / (label + ".yaml"), loaded, scope, ISSUER, AUDIENCE) save(label + ".binding.json", route) save("ts-consumer." + label + ".json", consumer) manifests.append(work / (label + ".yaml")) @@ -500,7 +560,7 @@ def save(name, value, private=False): save("probes.json", []) runtime_packet = {"namespace": namespace, "bootstrap": "localhost:9092", "provenance": "TEST_ONLY", "client_tls": {"ca": "/work/ca.crt", "cert": "/work/client.crt", "key": "/work/client.key"}, - "isolated_manifest_paths": ["/work/binance.yaml", "/work/okx.yaml"], + "isolated_manifest_paths": ["/work/" + scope.lower() + ".yaml" for scope in scopes], "slice": {"bundle": "/work/bundle.json", "cursor_keys": "/work/cursor-keys.json", "profile": "/work/profile.json", "probes": "/work/probes.json", "route_generation": namespace, "target": "localhost:8210"}} @@ -547,9 +607,9 @@ def erase_private(work): (work / "handoff.json").write_text(json.dumps(packet, indent=2) + "\n") -def init(work, export_path, images=None): +def init(work, export_path, images=None, scopes=None, catalog=None): # Runtime-free initialization in the already-local dependency image. - packet = prepare(work, images) + packet = prepare(work, images, scopes, catalog) ns = packet["namespace"] try: bounded(helper_command(ns, work, ["--material", "/work", "--core-config", "/export.json"], exported=export_path)) @@ -660,15 +720,18 @@ def dispatch(argv=None): parser.add_argument("--core-config", type=Path) parser.add_argument("--image", action="append", default=[], metavar="ROLE=sha256:ID", help="--init/--prepare image override (repeatable); sealed in handoff.json") + parser.add_argument("--scope", action="append", choices=tuple(SCOPES), + help="--init/--prepare case scope (repeatable); default BINANCE and OKX (linear ETH)") + parser.add_argument("--catalog", type=Path, help="--init/--prepare source catalog defining every scope") args = parser.parse_args(argv) - if args.image and not (args.init or args.prepare): - parser.error("--image only with --init/--prepare; later steps read handoff.json") + if (args.image or args.scope or args.catalog) and not (args.init or args.prepare): + parser.error("--image/--scope/--catalog only with --init/--prepare; later steps read handoff.json") images = resolve_images(parse_image_overrides(args.image)) if args.init or args.material: if args.core_config is None: parser.error("--core-config sanitized exported JSON required") if args.init: - result = init(args.init, args.core_config, images) + result = init(args.init, args.core_config, images, args.scope, args.catalog) else: result = material(args.material, json.loads(args.core_config.read_text())) result = {k: result[k] for k in ("namespace", "status", "material_ready")} @@ -694,7 +757,7 @@ def dispatch(argv=None): elif args.probe: result = probe(args.probe) elif args.prepare: - packet = prepare(args.prepare, images) + packet = prepare(args.prepare, images, args.scope, args.catalog) result = {k: packet[k] for k in ("namespace", "status", "acceptance")} else: result = readiness(args.readiness) if args.readiness else cleanup(args.cleanup) diff --git a/scripts/p184_dr05_observer.py b/scripts/p184_dr05_observer.py index 6826c6aa..7714fd7b 100644 --- a/scripts/p184_dr05_observer.py +++ b/scripts/p184_dr05_observer.py @@ -19,7 +19,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from scripts.p184_dr05_dl_fault import frame, namespace, schedule +from scripts.p184_dr05_dl_fault import SCOPES, frame, namespace, schedule, scope_of, scope_set from scripts.p184_dr05_raw_publisher import RawPublisher @@ -105,10 +105,14 @@ def reader_harness(packet): from scripts.kn_native_slice_probe import Slice from qdl.consumer.manifest import ConsumerManifestLoader + # One manifest per sealed scope (both linear venues, optionally the inverse), checked before any I/O. + expected = {packet["namespace"] + "." + Path(p).stem for p in packet["isolated_manifest_paths"]} + if not {packet["namespace"] + ".binance", packet["namespace"] + ".okx"} <= expected or any( + Path(p).stem.upper() not in SCOPES for p in packet["isolated_manifest_paths"]): + raise ValueError("isolated manifest/identity roster mismatch") runner = Slice(SimpleNamespace(**packet["slice"])) runner.manifests = {m.consumer_id: m for m in ( ConsumerManifestLoader.load(Path(path)) for path in packet["isolated_manifest_paths"])} - expected = {packet["namespace"] + ".binance", packet["namespace"] + ".okx"} if set(runner.manifests) != expected or set(runner.identities) != expected: raise ValueError("isolated manifest/identity roster mismatch") return runner @@ -121,7 +125,7 @@ async def read_views(runner, packet, binding): from qdl.certification.phase103_consumer_acceptance import sdk_requirement from scripts.p184_dr08_isolated import channel_for - cid = packet["namespace"] + "." + binding["venue"].lower() + cid = packet["namespace"] + "." + scope_of(binding["venue"], binding["native_symbol"]).lower() channel = await channel_for(runner, cid) result = {} try: @@ -164,7 +168,7 @@ async def run(work, venue, fault): producer = RawPublisher(packet, config) observer = CanonicalObserver(packet, config) runner = reader_harness(packet) - binding = producer.bindings[venue] + binding = producer.bindings[venue] # keyed by scope packet_base = ts_packet(work, runner, packet, binding, []) rows = [] try: @@ -199,15 +203,14 @@ async def run(work, venue, fault): def ts_packet(work, runner, packet, binding, rows, base=None): - venue = binding["venue"] - cid = packet["namespace"] + "." + venue.lower() + scope = scope_of(binding["venue"], binding["native_symbol"]) + cid = packet["namespace"] + "." + scope.lower() if base is None: from dataclasses import asdict from qdl.runtime.stable_catalog import StableSourceCatalog from qdl.certification.phase103_consumer_acceptance import sdk_requirement catalog = StableSourceCatalog.load(work / "catalog.yaml") instrument = catalog.instrument_for(binding["instrument_uid"]) - cid = packet["namespace"] + "." + venue.lower() requirements = {} for feed in ("BOOK_SNAPSHOT", "BOOK_DELTA"): sdk = sdk_requirement(SimpleNamespace(requirement=runner.requirement(cid, binding["instrument_uid"], feed))) @@ -243,7 +246,7 @@ def public_query_app(work, service, issuer): if packet.get("provenance") != "TEST_ONLY": raise ValueError("isolated public Query requires TEST_ONLY packet") paths = packet["isolated_manifest_paths"] - if paths != ["/work/binance.yaml", "/work/okx.yaml"]: + if paths != ["/work/" + scope.lower() + ".yaml" for scope in scope_set(work)]: raise ValueError("unexpected isolated manifest paths") manifests = ConsumerManifestRegistry(tuple( ConsumerManifestLoader.load(work / Path(path).name) for path in paths)) @@ -285,7 +288,7 @@ def main(): return parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--work", type=Path, required=True) - parser.add_argument("--venue", choices=("BINANCE", "OKX"), required=True) + parser.add_argument("--venue", choices=tuple(SCOPES), required=True) parser.add_argument("--fault", choices=("GAP_REFUSAL", "GENERATION_CHANGE_FENCED"), required=True) args = parser.parse_args() print(json.dumps(asyncio.run(run(args.work, args.venue, args.fault)))) diff --git a/scripts/p184_dr05_raw_publisher.py b/scripts/p184_dr05_raw_publisher.py index 7dcd8779..5b757ff9 100644 --- a/scripts/p184_dr05_raw_publisher.py +++ b/scripts/p184_dr05_raw_publisher.py @@ -7,7 +7,7 @@ import time import uuid -from scripts.p184_dr05_dl_fault import namespace +from scripts.p184_dr05_dl_fault import VENUES, namespace, scope_of def raw_envelope(binding, frame, ns): @@ -29,7 +29,7 @@ def raw_envelope(binding, frame, ns): capture_id=uuid.uuid4().bytes, provider=binding["provider"], venue=binding["venue"], market=binding["market"], product_type=binding["product_type"], native_symbol=binding["native_symbol"], native_channel=binding["native_channel"], - subscription_id=binding["source_id"], source_session_id=ns + "." + binding["venue"], + subscription_id=binding["source_id"], source_session_id=ns + "." + _session_label(binding), connection_generation=frame["connection_generation"], lease_epoch=1, authority_revision=1, partition_plan_epoch=1, received_at_ns=frame["received_at_ns"], transport_protocol=2 if frame["transport"] == "HTTP" else 1, @@ -39,6 +39,12 @@ def raw_envelope(binding, frame, ns): correlation_id=str(uuid.uuid4()), test_provenance=True) +def _session_label(binding): + """Linear scopes keep their venue label; another scope on the same venue gets its own session.""" + scope = scope_of(binding["venue"], binding["native_symbol"]) + return binding["venue"] if scope in VENUES else scope + + class RawPublisher: """Single-flight, one attempt; never retry an ambiguous delivery automatically. @@ -58,9 +64,11 @@ def __init__(self, packet, config): if config.get("raw_topics") != [self.ns + ".raw"] or config["core"].get("allow_test_provenance") is not True: raise ValueError("isolated test-only raw topic required") self.topic = config["raw_topics"][0] - self.bindings = {b["venue"]: b for b in config["core"]["bindings"]} - if len(config["core"]["bindings"]) != 2 or set(self.bindings) != {"BINANCE", "OKX"}: - raise ValueError("exact two-venue binding roster required") + bindings = config["core"]["bindings"] + self.bindings = {scope_of(b["venue"], b["native_symbol"]): b for b in bindings} + if (len(bindings) != len(self.bindings) or None in self.bindings + or not set(VENUES) <= set(self.bindings)): + raise ValueError("exact scope binding roster required (both linear venues, optional inverse)") self.producer = Producer({"bootstrap.servers": "localhost:9092", "security.protocol": "SSL", "ssl.ca.location": "/work/ca.crt", "ssl.certificate.location": "/work/client.crt", "ssl.key.location": "/work/client.key", "enable.idempotence": True, "acks": "all", @@ -73,7 +81,7 @@ def publish(self, frame, *, timeout=5): raise RuntimeError("publisher fenced after unsuccessful/ambiguous delivery") if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not 0 < timeout <= 5: raise ValueError("publication deadline must be in (0, 5] seconds") - raw = raw_envelope(self.bindings[frame["venue"]], frame, self.ns) + raw = raw_envelope(self.bindings[scope_of(frame["venue"], frame["native_symbol"])], frame, self.ns) wire = raw.SerializeToString() delivered = [] start = time.monotonic() diff --git a/scripts/p184_dr05_reset.py b/scripts/p184_dr05_reset.py index fcc2e4bc..fb1e2224 100644 --- a/scripts/p184_dr05_reset.py +++ b/scripts/p184_dr05_reset.py @@ -72,9 +72,11 @@ def __init__(self, stage, error): def reset_preserving_broker(work, venue, timeout=30): deadline = _deadline(timeout) - if venue not in dr.VENUES: + if venue not in dr.SCOPES: raise ValueError("unknown venue") work, ns, packet, _ = _packet(work) + if venue not in dr.scope_set(work): + raise ValueError("scope not prepared in this case") selected = dr.core_override(json.loads((work / "handoff.json").read_text()).get("core_binary")) def call(argv): diff --git a/scripts/p184_g13_final.py b/scripts/p184_g13_final.py index bf982d8e..256b5f74 100644 --- a/scripts/p184_g13_final.py +++ b/scripts/p184_g13_final.py @@ -9,7 +9,8 @@ ONLY the existing harnesses, sequentially, each under a hard timeout: ISOLATED phase (disposable namespaces, nothing shared is written): dr05-upstream DL scripts/p184_dr05_dl_fault.py + p184_dr05_reset.py + p184_dr05_channel.py, TS - scripts/p184_dr05_upstream.py (GAP_REFUSAL / GENERATION_CHANGE_FENCED, both venues) + scripts/p184_dr05_upstream.py (GAP_REFUSAL / GENERATION_CHANGE_FENCED; scopes BINANCE, OKX and + OKX_INVERSE = BTC-USD-SWAP BOOK, defined by the --catalog input `dr05_catalog`) dr08-isolated DL scripts/p184_dr08_isolated.py --orchestrate (QUALIFICATION mode, >= 5 restart samples) RUNTIME phase (reads the shared stack after the final rollout; needs the owner-approved runtime packet): dr01-deployed DL scripts/p184_dr01_deployed.py prepare/probe/record (7 DL cells) @@ -19,7 +20,7 @@ dr02-dr03-alpha TS scripts/p184_dr_alpha_reads.py probe (inside each alpha runtime) + record dr04-dr06-views TS scripts/p184_dr_views.py (inside the TS writer image; binance, okx, okx_inverse) + record; also DR05.QUIET_TYPED (linear only: QUIET is a TRADE state, TRADE is not granted for inverse) - dr05-transport TS scripts/p184_dr05_isolated.py (writer clone, transport cut at the clone only) + dr05-transport TS scripts/p184_dr05_isolated.py (writer clone, transport cut at the clone only; same 3 scopes) dr07-reference TS scripts/p184_dr07_reference.py Receipts: a NEW directory ~/.local/state/qdl-v2/p184-g13-TZ/ (exclusive create) with plan.json, one /receipt.json per step (argv, cwd, times, exit, timeout, output sha256, cleanup) and summary.json. @@ -107,9 +108,10 @@ def cells(case, names, venues=VENUES): "DR08": tuple(f"DR08.{n}@ANY" for n in ("ALERT_PATH_OBSERVABLE", "CONSUMER_RESTART_RECOVERY_MEASURED", "NO_CROSS_CONSUMER_STARVATION", "SLOW_CONSUMER_BOUNDED_QUEUE")), } -# Inverse DR05 faults: the DL fault harness (p184_dr05_dl_fault.py isolated_config: ETH BOOK bindings only) and the -# TS writer clone (p184_dr05_isolated.py SYMBOLS: ETH only) have no BTC-USD-SWAP scope yet. Listed open, never faked. -NO_PRODUCER = INV_DR05 +# Every DR cell has a producer: the inverse DR05 faults run in the DL fault harness (--scope OKX_INVERSE with a +# source catalog defining BTC-USD-SWAP) and the TS writer clone (--venues ... OKX_INVERSE). Kept for listing open cells. +NO_PRODUCER = () +DR05_SCOPES = VENUES + INVERSE # p184_dr05_dl_fault.py --scope / TS p184_dr05_upstream.py, p184_dr05_isolated.py --venues # DR -> (decision, reason). Same table as upgrade/evidence/p184-g13-inherit-rerun-matrix.md. DECISIONS = { @@ -131,7 +133,7 @@ def cells(case, names, venues=VENUES): STEP_SPEC = ( # id, phase, cells, timeout_s, est_minutes, owned container prefixes - ("dr05-upstream", "isolated", cells("DR05", ("GAP_REFUSAL", "GENERATION_CHANGE_FENCED")), 1800, 5, + ("dr05-upstream", "isolated", cells("DR05", ("GAP_REFUSAL", "GENERATION_CHANGE_FENCED"), DR05_SCOPES), 1800, 6, ("p184-dr05-",)), ("dr08-isolated", "isolated", DR_CELLS["DR08"], 1500, 12, ("p184-dr08-",)), ("dr01-deployed", "runtime", tuple(c for c in DR_CELLS["DR01"] if "WRONG_ACCOUNT" not in c), 600, 3, ()), @@ -140,7 +142,7 @@ def cells(case, names, venues=VENUES): ("dr04-dr06-views", "runtime", DR_CELLS["DR04"] + DR_CELLS["DR06"] + cells("DR05", ("QUIET_TYPED",)), 1200, 14, ()), ("dr05-transport", "runtime", cells("DR05", ("STALE_TYPED", "DISCONNECT_RECONNECT", - "APPROVED_FALLBACK_AND_RETURN_NO_FORBIDDEN_V1")), 1500, 8, + "APPROVED_FALLBACK_AND_RETURN_NO_FORBIDDEN_V1"), DR05_SCOPES), 1500, 8, ("p184-dr05-",)), ("dr07-reference", "runtime", DR_CELLS["DR07"], 600, 2, ()), ) @@ -229,7 +231,7 @@ def select(phase, only=()): def required_inputs(step): return { - "dr05-upstream": ("dr05_public_export", "redis_image", "postgres_image"), + "dr05-upstream": ("dr05_public_export", "dr05_catalog", "redis_image", "postgres_image"), "dr08-isolated": (), "dr01-deployed": ("dr01.inventory", "dr01.deployments.BINANCE", "dr01.deployments.OKX", "dr01.compose", "dr01.alpha_service"), @@ -268,14 +270,16 @@ def commands(step, final, inputs, work, ts_root): alpha_record = ts_root / "scripts/p184_dr_alpha_reads.py" if step == "dr05-upstream": w = work / "work" + scopes = [a for scope in DR05_SCOPES for a in ("--scope", scope)] return [cmd(py + ["scripts/p184_dr05_dl_fault.py", "--init", w, "--core-config", inputs["dr05_public_export"], - *harness_image_args(final)], cwd=DL_ROOT), + *harness_image_args(final), *scopes, "--catalog", inputs["dr05_catalog"]], cwd=DL_ROOT), cmd(py + ["scripts/p184_dr05_dl_fault.py", "--up", w], cwd=DL_ROOT), cmd(py + ["scripts/p184_dr05_dl_fault.py", "--ready", w], cwd=DL_ROOT), cmd(py + ["scripts/p184_dr05_upstream.py", "--work", w, "--namespace", "", "--dl-channel", "", "--image", final["writer"], "--redis-image", inputs["redis_image"], "--postgres-image", inputs["postgres_image"], - *ts_ev, "--output", work / "dr05-upstream-rows.jsonl", "--dl-root", DL_ROOT], cwd=ts_root), + *ts_ev, "--output", work / "dr05-upstream-rows.jsonl", "--dl-root", DL_ROOT, + "--venues", *DR05_SCOPES], cwd=ts_root), cmd(py + ["scripts/p184_dr05_dl_fault.py", "--down", w], cwd=DL_ROOT)] if step == "dr08-isolated": return [cmd(py + ["scripts/p184_dr08_isolated.py", "--orchestrate", work / "work", "--ts-source-sha", @@ -334,7 +338,7 @@ def commands(step, final, inputs, work, ts_root): if step == "dr05-transport": return [cmd(py + ["scripts/p184_dr05_isolated.py", "--image", final["writer"], "--redis-image", inputs["redis_image"], "--postgres-image", inputs["postgres_image"], *ts_ev, - "--output", work / "dr05-transport.jsonl"], cwd=ts_root)] + "--output", work / "dr05-transport.jsonl", "--venues", *DR05_SCOPES], cwd=ts_root)] if step == "dr07-reference": d = inputs["dr07"] return [cmd(py + ["scripts/p184_dr07_reference.py", "--connections", d["connections"], *ts_ev, "--output", diff --git a/tests/test_p184_dr05_inverse_scope.py b/tests/test_p184_dr05_inverse_scope.py new file mode 100644 index 00000000..7a98d992 --- /dev/null +++ b/tests/test_p184_dr05_inverse_scope.py @@ -0,0 +1,261 @@ +"""DR05 OKX inverse scope (BTC-USD-SWAP BOOK): scopes, frames, export/catalog filter, consumers, publisher, resets. + +Offline only (no Docker). The inverse catalog fixture is the repository catalog plus the deployed inverse instrument +and its BOOK snapshot/delta bindings (identity as in the rev-12 paper catalog that binding r23 was compiled from).""" +import copy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import yaml + +from scripts import p184_dr05_dl_fault as dr + +NS = "p184-dr05-dl-0123456789abcdef" +ALL = ("BINANCE", "OKX", "OKX_INVERSE") +INV_UID = "8482b54b-6d9d-5aad-abe8-e22447d183a1" +INV_SOURCE = "okx-swap-btc-usd-swap-book-primary-v2" + + +def inverse_catalog(path): + """Write the repo catalog + the inverse instrument and its two BOOK bindings; return the path.""" + catalog = yaml.safe_load((dr.ROOT / "config/v2/stable-source-bindings.yaml").read_text()) + catalog["catalog_revision"] += 1 + catalog["instruments"].append({ + "instrument_uid": INV_UID, "instrument_id": "OKX.SWAP.PERPETUAL.BTC-USD", "metadata_revision": 1, + "venue": "OKX", "market": "SWAP", "product_type": "PERPETUAL", "canonical_symbol": "BTC-USD", + "native_symbol": "BTC-USD-SWAP", "asset_class": "DERIVATIVE", "base_asset": "BTC", "quote_asset": "USD", + "settlement_asset": "BTC", "price_tick": "0.1", "quantity_step": "0.1", "contract_multiplier": "100", + "session_calendar_id": "CRYPTO_24X7", "attributes": {"ctMult": "1", "ctType": "inverse", "ctVal": "100", + "ctValCcy": "USD", "groupId": "4", "instFamily": "BTC-USD", "instIdCode": "10458", "uly": "BTC-USD"}}) + for feed in ("BOOK_DELTA", "BOOK_SNAPSHOT"): + catalog["bindings"].append({ + "binding_id": "okx-swap-btc-usd-swap-" + feed.lower(), "instrument_uid": INV_UID, "feed": feed, + "interval": None, "source": {"provider": "OKX_DIRECT", "source_id": INV_SOURCE, "source_role": "PRIMARY", + "source_policy_id": "crypto_liquid_v2", "authoritative": True, "adapter_version": "okx-v5/2.0.0", + "normalizer_version": "qdl-rust-core/2.0.0"}, + "quality": {"stale_after_ms": 60000, "require_final_bar": False, "continuous_calendar": True}, + "v1_compatibility": "NONE"}) + path.write_text(yaml.safe_dump(catalog)) + return path + + +def exported(with_inverse=True): + from tests.test_p184_dr05_dl_fault import MaterialTests + out = MaterialTests.exported() + if with_inverse: + okx = next(b for b in out["core"]["bindings"] if b["venue"] == "OKX") + inverse = copy.deepcopy(okx) + inverse.update(native_symbol="BTC-USD-SWAP", instrument_uid=INV_UID, + instrument_id="OKX.SWAP.PERPETUAL.BTC-USD", instrument_catalog_revision=12, + source_id=INV_SOURCE) + out["core"]["bindings"].append(inverse) + return out + + +class ScopeTests(unittest.TestCase): + def test_default_scopes_and_cells_unchanged(self): + self.assertEqual(dr.validate_scopes(None), dr.VENUES) + self.assertEqual(dr.cells_for(dr.VENUES), dr.CELLS) + self.assertEqual(dr.scope_set(None), dr.VENUES) + + def test_scope_validation_orders_and_refuses(self): + self.assertEqual(dr.validate_scopes(["OKX_INVERSE", "BINANCE", "OKX"]), ALL) + for bad in ([], ["OKX", "OKX"], ["OKX_LINEAR"], ["BINANCE_INVERSE"]): + with self.subTest(bad=bad), self.assertRaises(ValueError): + dr.validate_scopes(bad) + + def test_prepare_seals_inverse_scope_cells_schedules_and_catalog(self): + with tempfile.TemporaryDirectory() as tmp: + catalog = inverse_catalog(Path(tmp) / "catalog.yaml") + work = Path(tmp) / "case" + packet = dr.prepare(work, scopes=["OKX_INVERSE", "BINANCE", "OKX"], catalog=catalog) + self.assertEqual(packet["scopes"], list(ALL)) + self.assertEqual(dr.scope_set(work), ALL) + self.assertIn("DR05.GAP_REFUSAL@OKX_INVERSE", packet["schedules"]) + self.assertIn("DR05.GENERATION_CHANGE_FENCED@OKX_INVERSE", packet["schedules"]) + pending = [json.loads(line)["cell"] for line in (work / "pending-cells.jsonl").read_text().splitlines()] + self.assertEqual(len(pending), 6) + self.assertEqual((work / dr.SOURCE_CATALOG).read_bytes(), catalog.read_bytes()) + self.assertEqual(dr.source_catalog_path(work), work / dr.SOURCE_CATALOG) + self.assertEqual(len(packet["source_catalog"]["sha256"]), 64) + handoff = json.loads((work / "handoff.json").read_text()) + (work / "handoff.json").write_text(json.dumps({**handoff, "scopes": ["OKX", "BINANCE"]})) + with self.assertRaises(ValueError): # unordered/tampered scopes refused + dr.scope_set(work) + + +class FrameTests(unittest.TestCase): + def test_inverse_frame_is_okx_books_for_btc_usd_swap(self): + value = dr.frame("OKX_INVERSE", False, 101, 100, 7, "2", 123_000_000) + self.assertEqual((value["venue"], value["native_symbol"], value["transport"]), + ("OKX", "BTC-USD-SWAP", "WEBSOCKET")) + self.assertEqual(value["payload"]["arg"], {"channel": "books", "instId": "BTC-USD-SWAP"}) + self.assertEqual(value["payload"]["data"][0]["prevSeqId"], "100") + snapshot = dr.frame("OKX_INVERSE", True, 100, 0, 7, "1", 123_000_000) + self.assertEqual((snapshot["payload"]["action"], snapshot["transport"]), ("snapshot", "WEBSOCKET")) + self.assertEqual(dr.scope_of(value["venue"], value["native_symbol"]), "OKX_INVERSE") + + def test_inverse_schedule_follows_okx_semantics(self): + for fault in dr.FAULTS: + inverse = dr.schedule("OKX_INVERSE", fault) + linear = dr.schedule("OKX", fault) + self.assertEqual([{k: v for k, v in s.items() if k != "venue"} for s in inverse], + [{k: v for k, v in s.items() if k != "venue"} for s in linear]) + self.assertTrue(all(s["venue"] == "OKX_INVERSE" for s in inverse)) + + +class ConfigTests(unittest.TestCase): + def test_isolated_config_selects_the_inverse_book_binding(self): + cfg = dr.isolated_config(exported(), NS, scopes=ALL) + self.assertEqual([(b["venue"], b["native_symbol"]) for b in cfg["core"]["bindings"]], + [("BINANCE", "ETHUSDT"), ("OKX", "ETH-USDT-SWAP"), ("OKX", "BTC-USD-SWAP")]) + self.assertEqual(len(dr.isolated_config(exported(), NS)["core"]["bindings"]), 2) # default linear only + with self.assertRaisesRegex(ValueError, "OKX_INVERSE"): + dr.isolated_config(exported(with_inverse=False), NS, scopes=ALL) + + def test_material_filters_catalog_to_granted_inverse_book_feeds(self): + from qdl.consumer.manifest import ConsumerManifestLoader + from qdl.runtime.stable_catalog import StableSourceCatalog + from scripts.kn_gateway_bundle import compile_bundle + with tempfile.TemporaryDirectory() as tmp: + catalog = inverse_catalog(Path(tmp) / "catalog.yaml") + work = Path(tmp) / "case" + packet = dr.prepare(work, scopes=ALL, catalog=catalog) + dr.material(work, exported()) + loaded = StableSourceCatalog.load(work / "catalog.yaml") + inverse = [b for b in loaded.bindings if b.instrument.instrument_uid == INV_UID] + self.assertEqual(sorted(b.feed.value for b in inverse), ["BOOK_DELTA", "BOOK_SNAPSHOT"]) + self.assertEqual(len(loaded.bindings), 6) + manifest = ConsumerManifestLoader.load(work / "okx_inverse.yaml") + self.assertEqual(manifest.consumer_id, packet["namespace"] + ".okx_inverse") + self.assertEqual({(r.instrument_uid, r.feed.value) for r in manifest.requirements}, + {(INV_UID, "BOOK_SNAPSHOT"), (INV_UID, "BOOK_DELTA")}) + consumer = json.loads((work / "ts-consumer.okx_inverse.json").read_text()) + self.assertEqual((consumer["venue"], consumer["jwt_key_id"]), ("OKX_INVERSE", "okx_inverse")) + route = json.loads((work / "okx_inverse.binding.json").read_text()) + self.assertEqual({(p["venue"], p["native_symbol"], p["feed"]) for p in route["products"]}, + {("OKX", "BTC-USD-SWAP", "BOOK_SNAPSHOT"), ("OKX", "BTC-USD-SWAP", "BOOK_DELTA")}) + runtime = json.loads((work / "packet.json").read_text()) + self.assertEqual(runtime["isolated_manifest_paths"], + ["/work/binance.yaml", "/work/okx.yaml", "/work/okx_inverse.yaml"]) + manifests = [work / Path(p).name for p in runtime["isolated_manifest_paths"]] + self.assertEqual(json.loads((work / "bundle.json").read_text()), compile_bundle( + environment="paper", catalog_path=work / "catalog.yaml", manifest_paths=manifests)) + self.assertEqual((work / "okx_inverse.key").stat().st_mode & 0o777, 0o600) + + def test_inverse_scope_without_a_catalog_that_defines_it_is_refused(self): + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "case" + dr.prepare(work, scopes=ALL) # repo catalog has no BTC-USD-SWAP BOOK + with self.assertRaises(ValueError): + dr.material(work, exported()) + + def test_cli_scope_and_catalog_only_at_init_or_prepare(self): + with tempfile.TemporaryDirectory() as tmp, patch("sys.stdout"), patch("sys.stderr"): + catalog = inverse_catalog(Path(tmp) / "catalog.yaml") + work = Path(tmp) / "case" + self.assertEqual(dr.dispatch(["--prepare", str(work), "--scope", "BINANCE", "--scope", "OKX", + "--scope", "OKX_INVERSE", "--catalog", str(catalog)]), 0) + self.assertEqual(dr.scope_set(work), ALL) + for argv in (["--up", str(work), "--scope", "OKX_INVERSE"], ["--up", str(work), "--catalog", str(catalog)], + ["--prepare", str(Path(tmp) / "b"), "--scope", "BINANCE_INVERSE"]): + with self.subTest(argv=argv), self.assertRaises(SystemExit): + dr.dispatch(argv) + + +class ConsumerAndPublisherTests(unittest.TestCase): + def test_consumer_packet_refuses_a_linear_manifest_for_the_inverse_scope(self): + from qdl.runtime.stable_catalog import StableSourceCatalog + from scripts.p184_dr05_case import consumer_packet + from scripts.kn_native_slice_probe import ISSUER, AUDIENCE + with tempfile.TemporaryDirectory() as tmp: + catalog = inverse_catalog(Path(tmp) / "catalog.yaml") + work = Path(tmp) / "case" + packet = dr.prepare(work, scopes=ALL, catalog=catalog) + dr.material(work, exported()) + loaded = StableSourceCatalog.load(work / "catalog.yaml") + linear = yaml.safe_load((work / "okx.yaml").read_text()) + linear["metadata"]["consumer_id"] = packet["namespace"] + ".okx_inverse" + forged = work / "forged.yaml" + forged.write_text(yaml.safe_dump(linear)) + with self.assertRaises(ValueError): + consumer_packet(forged, loaded, "OKX_INVERSE", ISSUER, AUDIENCE) + + def test_publisher_roster_and_inverse_envelope(self): + from qdl.raw.envelope import validate_raw_envelope + from scripts.p184_dr05_raw_publisher import RawPublisher, raw_envelope + from tests.test_p184_dr05_raw_publisher import Producer + bindings = dr.isolated_config(exported(), NS, scopes=ALL)["core"]["bindings"] + inverse = bindings[2] + value = raw_envelope(inverse, dr.frame("OKX_INVERSE", True, 100, 0, 7, "1", 123000000), NS) + validate_raw_envelope(value) + self.assertEqual((value.native_symbol, value.source_session_id), ("BTC-USD-SWAP", NS + ".OKX_INVERSE")) + linear = raw_envelope(bindings[1], dr.frame("OKX", True, 100, 0, 7, "1", 123000000), NS) + self.assertEqual(linear.source_session_id, NS + ".OKX") # linear sessions unchanged + with self.assertRaises(ValueError): + raw_envelope(bindings[1], dr.frame("OKX_INVERSE", True, 100, 0, 7, "1", 123000000), NS) + packet = {"namespace": NS, "bootstrap": "localhost:9092"} + config = {"raw_topics": [NS + ".raw"], "core": {"allow_test_provenance": True, "bindings": bindings}} + with patch.dict("os.environ", {"P184_DR05_ISOLATED_NAMESPACE": NS}), \ + patch("confluent_kafka.Producer", Producer): + publisher = RawPublisher(packet, config) + self.assertEqual(set(publisher.bindings), set(ALL)) + publisher.publish(dr.frame("OKX_INVERSE", True, 100, 0, 7, "1", 123000000)) + self.assertEqual(publisher.producer.calls[-1][1]["key"], INV_SOURCE.encode()) + for broken in (bindings[:1] + bindings[2:], bindings + bindings[2:]): + with self.subTest(n=len(broken)), self.assertRaises(ValueError): + RawPublisher(packet, {**config, "core": {**config["core"], "bindings": broken}}) + + +class ResetScopeTests(unittest.TestCase): + def test_reset_and_channel_refuse_a_scope_not_prepared(self): + from scripts import p184_dr05_reset as reset + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "case" + dr.prepare(work) # default linear scopes + with patch("scripts.p184_dr05_reset._packet", return_value=(work, NS, {}, {})), \ + self.assertRaisesRegex(ValueError, "not prepared"): + reset.reset_preserving_broker(work, "OKX_INVERSE", 5) + with self.assertRaisesRegex(ValueError, "unknown venue"): + reset.reset_preserving_broker(work, "BINANCE_INVERSE", 5) + + def test_channel_reset_checks_sealed_scopes(self): + from scripts import p184_dr05_channel as ch + from unittest.mock import Mock + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "handoff.json").write_text(json.dumps({"scopes": ["BINANCE", "OKX"]})) + loader = Mock(return_value=(work, NS, {"namespace": NS, "slice": {"topic_id": "t"}}, + {"raw_topics": [NS + ".raw"]})) + channel = ch.Channel(work, loader=loader, publisher_factory=Mock(), observer_factory=Mock()) + try: + with self.assertRaisesRegex(ValueError, "unknown cell"): + channel.reset("OKX_INVERSE", "GAP_REFUSAL", 10 ** 12) + finally: + channel.close() + + +class QueryRosterTests(unittest.TestCase): + def test_reader_roster_accepts_the_sealed_inverse_manifest_only(self): + from scripts import p184_dr05_observer as obs + paths = ["/work/binance.yaml", "/work/okx.yaml", "/work/okx_inverse.yaml"] + expected = {NS + "." + Path(p).stem for p in paths} + + class Slice: + def __init__(self, _args): + self.identities = {k: None for k in expected} + + manifests = [type("M", (), {"consumer_id": cid})() for cid in sorted(expected)] + with patch("scripts.kn_native_slice_probe.Slice", Slice), \ + patch("qdl.consumer.manifest.ConsumerManifestLoader.load", side_effect=manifests): + runner = obs.reader_harness({"namespace": NS, "slice": {}, "isolated_manifest_paths": paths}) + self.assertEqual(set(runner.manifests), expected) + with self.assertRaises(ValueError): + obs.reader_harness({"namespace": NS, "slice": {}, "isolated_manifest_paths": ["/work/okx_linear.yaml"]}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p184_g13_final.py b/tests/test_p184_g13_final.py index e7641ebb..0d2c1d75 100644 --- a/tests/test_p184_g13_final.py +++ b/tests/test_p184_g13_final.py @@ -23,7 +23,7 @@ def matrix_of(dr_cells): MATRIX_OK = matrix_of([c for case in g13.DR_CELLS.values() for c in case]) -INPUTS = {"dr05_public_export": "/x/export.json", "redis_image": "redis:7.2-alpine", "postgres_image": "pg@sha256:1", +INPUTS = {"dr05_public_export": "/x/export.json", "dr05_catalog": "/x/catalog-r12.yaml", "redis_image": "redis:7.2-alpine", "postgres_image": "pg@sha256:1", "dr01": {"inventory": "/x/inv.json", "deployments": {"BINANCE": "db", "OKX": "do"}, "compose": "/x/c.json", "alpha_service": "alpha"}, "alpha": {"containers": {"BINANCE": "ab", "OKX": "ao"}, "symbols": {"BINANCE": "ETHUSDT", @@ -46,8 +46,9 @@ def test_every_dr_cell_has_one_producer_or_is_listed_open(self): self.assertEqual(cov["cells"], 76) # TS matrix v2: 65 DR cells of v1 + 11 OKX_INVERSE self.assertEqual((cov["duplicates"], cov["missing"], cov["extra"]), ([], [], [])) self.assertEqual(cov["produced"] + len(cov["no_producer"]), 76) - # Only the inverse DR05 faults lack a producer (harnesses are ETH-only); WRONG_ACCOUNT now has its TS runner. - self.assertEqual(sorted(cov["no_producer"]), sorted(g13.INV_DR05)) + # Every cell has a producer: WRONG_ACCOUNT via the TS Gateway runner, inverse DR05 via the scoped harnesses. + self.assertEqual(cov["no_producer"], []) + self.assertEqual(cov["produced"], 76) self.assertEqual(len(g13.INV_DR04 + g13.INV_DR05 + g13.INV_DR06), 11) def test_dr_cells_equal_the_committed_ts_matrix_v2(self): @@ -178,6 +179,39 @@ def test_concatenation_targets_match_record_rows(self): self.assertEqual(Path(record[record.index("--rows") + 1]).name, target) +class InverseDr05Tests(unittest.TestCase): + def test_dr05_steps_run_every_scope_including_the_inverse(self): + steps = {e["step"]: e for e in make_plan("all")["steps"]} + self.assertEqual(set(g13.INV_DR05) - set(steps["dr05-upstream"]["cells"]) - set(steps["dr05-transport"]["cells"]), + set()) + self.assertIn("DR05.GAP_REFUSAL@OKX_INVERSE", steps["dr05-upstream"]["cells"]) + self.assertIn("DR05.STALE_TYPED@OKX_INVERSE", steps["dr05-transport"]["cells"]) + init = steps["dr05-upstream"]["commands"][0]["argv"] + self.assertEqual([init[i + 1] for i, a in enumerate(init) if a == "--scope"], ["BINANCE", "OKX", "OKX_INVERSE"]) + self.assertEqual(init[init.index("--catalog") + 1], INPUTS["dr05_catalog"]) + for argv in (steps["dr05-upstream"]["commands"][3]["argv"], steps["dr05-transport"]["commands"][0]["argv"]): + self.assertEqual(argv[argv.index("--venues") + 1:argv.index("--venues") + 4], ["BINANCE", "OKX", "OKX_INVERSE"]) + + def test_dr05_upstream_needs_the_catalog_input(self): + inputs = {k: v for k, v in INPUTS.items() if k != "dr05_catalog"} + self.assertIn("MISSING_INPUTS: dr05_catalog", make_plan("isolated", inputs=inputs)["blocked"]["dr05-upstream"]) + + def test_g13_init_argv_is_accepted_by_the_dl_fault_cli(self): + from scripts import p184_dr05_dl_fault as dr + init = {e["step"]: e for e in make_plan("isolated")["steps"]}["dr05-upstream"]["commands"][0]["argv"] + with tempfile.TemporaryDirectory() as tmp: + catalog = Path(tmp) / "c.yaml" + catalog.write_text("x: 1\n") + argv = [str(catalog) if a == INPUTS["dr05_catalog"] else a for a in init[3:]] + argv[argv.index("--init")] = "--prepare" + argv[argv.index("--prepare") + 1] = str(Path(tmp) / "case") + del argv[argv.index("--core-config"):argv.index("--core-config") + 2] + with redirect_stdout(io.StringIO()): + self.assertEqual(dr.dispatch(argv), 0) + self.assertEqual(dr.scope_set(Path(tmp) / "case"), g13.DR05_SCOPES) + self.assertEqual(dr.image_set(Path(tmp) / "case")["core"], RUST) + + class GatewayStepTests(unittest.TestCase): def test_wrong_account_step_probes_inside_the_gateway_and_records_with_its_image(self): steps = {e["step"]: e for e in make_plan("runtime")["steps"]} diff --git a/upgrade/evidence/p184-g13-inherit-rerun-matrix.md b/upgrade/evidence/p184-g13-inherit-rerun-matrix.md index 464944ce..5698e78a 100644 --- a/upgrade/evidence/p184-g13-inherit-rerun-matrix.md +++ b/upgrade/evidence/p184-g13-inherit-rerun-matrix.md @@ -10,16 +10,17 @@ scoped `OKX_INVERSE` (DR04 4, DR05 5, DR06 2). Counted with a Python read of the (`CASE_MATRIX_MISMATCH`) if its `DR_CELLS` differ from the matrix. Update 2026-10-10 (later): blockers 1, 4 and 5 below are resolved in code (harness image parameters; TS -WRONG_ACCOUNT Gateway runner `6c44bf2` wired as step `dr01-wrong-account`; matrix v2 inverse cells). One new -open item: the 5 inverse DR05 cells have no producer yet (see blocker 5). +WRONG_ACCOUNT Gateway runner `6c44bf2` wired as step `dr01-wrong-account`; matrix v2 inverse cells; DR05 +harnesses scoped for OKX_INVERSE, TS `acce72d`). Every one of the 76 DR cells now has a producer step. +New run inputs for the inverse DR05 faults: `dr05_catalog` and an export that carries the BTC-USD-SWAP BOOK +binding (blocker 5). ## Result in one line **Inherit 0 of 76 cells.** No DR cell has a receipt on final-qualifying provenance, and every DR exercises at least one path changed after its newest receipt. What can be inherited are inputs, not cells (see "Inherited inputs"). -The minimum is therefore eight harness steps, ~56 min of estimated wall time run serially (hard timeouts sum to -150 min). `dr01-wrong-account` is Gateway-dependent and closes with the TS release; the 5 inverse DR05 cells -have no producer yet. +The minimum is therefore eight harness steps, ~57 min of estimated wall time run serially (hard timeouts sum to +150 min). `dr01-wrong-account` is Gateway-dependent and closes with the TS release. ## Receipts that exist (all found with `grep -rl` over `~/.local/state` for `DR0n.` cell ids) @@ -70,7 +71,7 @@ and has no final-provenance receipt to inherit from. | DR02 | 8 | alpha `AlphaDataLayerV2` -> SDK transport -> Query warmup/batch, row cache, cold BAR (`qdl/query/*`, `kn_query_backend`, `stable_source`) | none | RERUN | warmup/batch path and SDK transport changed | `dr02-dr03-alpha` / 10 | | DR03 | 6 | same as DR02 plus 1m append (Query BAR, projector-fed spool) | none | RERUN | same | (same step) | | DR04 | 10 + 4 inverse | Stream/Query -> TS writer -> TS Redis -> `MarketCacheReader` + Risk validator; MARK/INDEX hot path, L2 view | none | RERUN | IDR-084/Astra MARK, IDR-082, IDR-040 fix 2 changed; writer digest new | `dr04-dr06-views` / 14 (binance, okx, okx_inverse) | -| DR05 | 12 + 5 inverse (no producer) | GAP/GEN: raw -> core fence (`canonical.rs`, `realtime-core/lib.rs`, `qdl-production-core.rs`) -> stream/query `kn_hot_view` -> writer clone -> production `_validate_book`; STALE/DISCONNECT/FALLBACK: writer clone transport; QUIET: TS views | r16 rehearsal (4 cells) | RERUN | core `1a2daf1`, query `57881a2`/`922a6ec`, stream `0b374bc`/`fdd379e`, writer all changed after r16; 8 cells never produced | `dr05-upstream` / 5, `dr05-transport` / 8, QUIET inside `dr04-dr06-views` | +| DR05 | 12 + 5 inverse | GAP/GEN: raw -> core fence (`canonical.rs`, `realtime-core/lib.rs`, `qdl-production-core.rs`) -> stream/query `kn_hot_view` -> writer clone -> production `_validate_book`; STALE/DISCONNECT/FALLBACK: writer clone transport; QUIET: TS views | r16 rehearsal (4 cells) | RERUN | core `1a2daf1`, query `57881a2`/`922a6ec`, stream `0b374bc`/`fdd379e`, writer all changed after r16; 8 cells never produced | `dr05-upstream` / 6 (3 scopes), `dr05-transport` / 8 (3 scopes), QUIET inside `dr04-dr06-views` | | DR06 | 4 + 2 inverse | production `services.risk_engine.execution_market_context` on a copy of the live context (writer views) | none | RERUN | reads changed MARK/L2 views; writer digest new | (in `dr04-dr06-views`) | | DR07 | 12 | alpha `alpha_sdk.reference` -> Query reference routes (`router.py`) -> `qdl/reference/*`; FX proof from TS DB | none | RERUN | no R2 DR07 rows; router changed | `dr07-reference` / 2 | | DR08 | 4 | isolated kafka -> core -> stream `service.rs`/`subscription.rs`/`hub.rs` -> query GetSnapshot | development only | RERUN | stream `fdd379e` after the overlay; final mode never ran | `dr08-isolated` / 12 | @@ -84,7 +85,9 @@ transport `--ready-seconds 180` + 2 x (20 s cut + <= 120 s recover)). These are - DR05 sanitized core export `~/.local/state/qdl-v2/dr05-observer-20261008/public-export.json` (SHA256 `9cf857a2...`, BINANCE ETHUSDT and OKX ETH-USDT-SWAP BOOK bindings, no transport). `6674d0b` changed only OKX transport URLs and - `consumers/` is unchanged, so the export stays valid; TLS/JWT material is regenerated per run by the harness. + `consumers/` is unchanged, so the export stays valid for the two linear scopes; TLS/JWT material is regenerated per + run by the harness. It does NOT cover the OKX_INVERSE scope, so G13 needs a fresh export with the inverse BOOK + binding as well (blocker 5). - DR05/DR08 harnesses are reused as they are at the release commit (latest DR05 harness changes: `522cf3e` D6 reset stderr, `75673bc` G12 helper); no new harness is written for G13. - Unit tests of the harnesses are not rerun for G13; they gate the build CI (`scripts/p184_local_ci.py`). @@ -119,9 +122,19 @@ transport `--ready-seconds 180` + 2 x (20 s cut + <= 120 s recover)). These are STALE_TYPED, DISCONNECT_RECONNECT, GAP_REFUSAL, GENERATION_CHANGE_FENCED, APPROVED_FALLBACK_AND_RETURN_NO_FORBIDDEN_V1. Not added (reasons in the matrix `inverse_excluded`): DR01 (no alpha grant), DR02/DR03 (BAR), DR04.EXPLICIT_LAST and DR05.QUIET_TYPED (TRADE), DR07 (no reference grant), DR08 (@ANY). `p184_dr_views.py` reads the inverse scope - (`--venue okx_inverse`), so DR04/DR06 inverse run inside `dr04-dr06-views`. **Open:** the 5 inverse DR05 cells - have no producer - `p184_dr05_dl_fault.py isolated_config` exports ETH BOOK bindings only and TS - `p184_dr05_isolated.py` `SYMBOLS` is ETH only; both need a BTC-USD-SWAP scope (runner lists them `no_producer`). + (`--venue okx_inverse`), so DR04/DR06 inverse run inside `dr04-dr06-views`. Inverse DR05 producers (code, + tested locally): DL `p184_dr05_dl_fault.py` scopes (`SCOPES`; `--init --scope BINANCE --scope OKX --scope + OKX_INVERSE --catalog `): `isolated_config` takes the exported BTC-USD-SWAP BOOK binding, the + material catalog filter keeps exactly each scope's BOOK_SNAPSHOT/BOOK_DELTA (no QUOTE/MARK/TRADE), consumer + `.okx_inverse`, OKX `books` frames for `BTC-USD-SWAP`; publisher, observer, channel and reset are scope-aware. + TS `acce72d`: `p184_dr05_upstream.py --venues ... OKX_INVERSE` and `p184_dr05_isolated.py --venues` (default all + three). The runner passes all three scopes to both DR05 steps. + **Run inputs this adds (not checked here, read-only session):** `dr05_catalog` = the deployed source catalog that + defines BTC-USD-SWAP (the repo `config/v2/stable-source-bindings.yaml` is revision 11 without it; the rev-12 paper + catalog copy is `~/.local/state/qdl-v2/p184-d1-rev11-20261009/base-catalog.yaml`; the deployed + `/runtime/stable-source-bindings.yaml` is root-only and was not read), and a fresh `dr05_public_export` that + includes the deployed core's BTC-USD-SWAP BOOK binding (the 10-08 export `9cf857a2...` has the two ETH bindings + only; `--init` refuses with `exactly one complete exported BOOK binding required: OKX_INVERSE`). 6. **DR01 record needs CI confirmation** (`--ci-confirmed` only when the inputs file states `dr01.ci_confirmed: true` for the TS source SHA). 7. RUNTIME steps read the shared stack and open extra Paper sessions: they need the owner-approved runtime packet @@ -142,7 +155,7 @@ python3 -B scripts/p184_g13_final.py --python-image sha256: --rust-image sha ... --only dr05-transport ``` -Inputs JSON keys: `dr05_public_export`, `redis_image`, `postgres_image`, `dr01.{inventory, deployments.BINANCE, +Inputs JSON keys: `dr05_public_export` (must include the BTC-USD-SWAP BOOK binding), `dr05_catalog`, `redis_image`, `postgres_image`, `dr01.{inventory, deployments.BINANCE, deployments.OKX, compose, alpha_service, compose_cwd?, catalog?, ci_confirmed?}`, `alpha.{containers.*, symbols.*, universe.*?}`, `ts_container`, `gateway_container`, `gateway_image_digest`, `dr07.{connections, fx_account, dsn_env?}`, `dr08_restart_samples?` (default 5). From a2184f4474a7068de51a57d5abc5a1a2b1b73d97 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 08:55:52 +0000 Subject: [PATCH 111/147] fix(query,sdk): batch item refusal lines and funding settlement publication allowance (P18.4 stage 100g) Stage 100g failed requests:no_failures on one REFERENCE_BATCH PARTIAL_RESULT (OKX SOL-USDT-SWAP FUNDING_RATE, query_v2_2). sent_ns 1791619207142576054 is 08:00:07.142Z, 7.1 s after the 08:00Z funding settlement; stage100g is the only stage-100 run whose window crossed a settlement and that was its only failure. Root cause (most probable; the item code was not recorded): the alpha manifests declare max_freshness_ms 28800000 for 8 h funding, Query measured age from the newest settlement with no allowance, and the driver window right edge moves to the new settlement at the boundary. Until OKX publishes the just-due row (settState=processing), the newest row is 00:00Z, aged 8h00m07s > 8h -> DATA_STALE, and the 30 s history cache keeps that series. Fix (Query internal, no contract change): - FUNDING_RATE history freshness allows _FUNDING_SETTLEMENT_PUBLICATION_GRACE_MS (60 s, the existing funding boundary tolerance); later it stays DATA_STALE. - A cache-served FUNDING_RATE series that is stale gets the existing single cache-bypassing re-read; a second stale observation stays terminal. Diagnostics (Astra R6 extension, additive): - Query: per-item problems in reference:batch / warmup:batch emit the bounded payload-free qdl_refusal line (origin reference_batch_item/warmup_batch_item, feed, typed provider tokens such as PROVIDER_PROTOCOL / terminal reason). - SDK 2.2.0: require_all PARTIAL_RESULT sets diagnostics.item_problems (instrument_uid, feed, code, reason_tokens, retryable, retry_after_ms, request_id, detail_sha256); code/detail/retryable unchanged. - Driver receipt records item_problems and item_problem_count. --- qdl/api_v2/router.py | 51 +++++++ qdl/query/service.py | 35 ++++- qdl/runtime/refusal_log.py | 20 ++- qdl_sdk/CHANGELOG.md | 9 ++ qdl_sdk/client.py | 45 ++++++ scripts/phase3_consumer_load_acceptance.py | 17 +++ tests/test_fund_phase5_api.py | 33 ++++ tests/test_kn_pre5_sdk.py | 16 ++ tests/test_phase113_reference_v2.py | 167 ++++++++++++++++++++- tests/test_phase3_consumer_load_driver.py | 21 +++ tests/test_refusal_log.py | 16 ++ 11 files changed, 423 insertions(+), 7 deletions(-) diff --git a/qdl/api_v2/router.py b/qdl/api_v2/router.py index c1781750..6cac1465 100644 --- a/qdl/api_v2/router.py +++ b/qdl/api_v2/router.py @@ -779,6 +779,39 @@ def _warmup_batch_response( ) +_WARMUP_BATCH_ROUTE = "/v2/market-data/warmup:batch" +_REFERENCE_BATCH_ROUTE = "/v2/market-data/reference:batch" + + +def _record_batch_item_refusal( + origin: str, + problem: ProblemDetails, + *, + feed: object, + route: str, + extra_reasons: tuple[object, ...] = (), +) -> None: + """Astra R6 extension: a per-item failure inside a 200 batch leaves the same line. + + The batch envelope succeeds, so the Query problem handler never sees these + item problems; without this line a ``require_all`` client's PARTIAL_RESULT + (stage 100g, 2026-10-10) could not be joined to the item that failed. + """ + + REFUSALS.record( + origin=origin, + code=problem.code, + detail=problem.detail, + status=problem.status, + retryable=problem.retryable, + request_id=problem.request_id, + instrument_uid=problem.instrument_uid, + route=route, + feed=str(getattr(feed, "value", feed)) if feed is not None else None, + extra_reasons=extra_reasons, + ) + + def _render_warmup_batch(request, access, result, requirements) -> bytes: """Render bounded row chunks per item, preserving the existing batch bytes.""" def encode(value): @@ -790,6 +823,10 @@ def encode(value): cold_yield() problem = None if item.problem is None else _problem(QueryServiceError( item.problem, request_id=result.request_id, instrument_uid=item.instrument_uid)) + if problem is not None: + _record_batch_item_refusal("warmup_batch_item", problem, + feed=getattr(requirement, "feed", None), + route=_WARMUP_BATCH_ROUTE) item_model = BatchItemResponse(instrument_uid=item.instrument_uid, status=item.status, data=None, problem=problem) item_models.append(item_model) @@ -1259,6 +1296,20 @@ async def reference_data_batch( instrument_uid=item.requirement.instrument_uid, ) ) + provider = item.result + _record_batch_item_refusal( + "reference_batch_item", + problem, + feed=item.requirement.product, + route=_REFERENCE_BATCH_ROUTE, + # Typed provider evidence only: the adapter's error code and + # the coverage terminal reason (e.g. PROVIDER_PROTOCOL, + # REACHED_REQUEST_START), never observations or prices. + extra_reasons=( + getattr(provider, "error_code", None), + getattr(getattr(provider, "coverage", None), "terminal_reason", None), + ) if provider is not None else (), + ) items.append( { "instrument_uid": item.requirement.instrument_uid, diff --git a/qdl/query/service.py b/qdl/query/service.py index 4f44c7ba..813d9d22 100644 --- a/qdl/query/service.py +++ b/qdl/query/service.py @@ -63,6 +63,17 @@ _EXECUTION_MARK_INDEX_LIVE_ENDPOINT = ( "qdl://stable-stream/internal/v2/execution/mark-index/latest" ) +# A funding series is settlement-cadenced: its newest record is exactly one +# interval old at the instant the next settlement falls due, and the venue only +# publishes that next record after it has settled (OKX reports +# ``settState=processing`` meanwhile). A freshness bound equal to the cadence +# (the alpha manifests declare 8 h for 8 h funding) therefore refused a healthy +# source at every settlement: stage 100g, 2026-10-10, OKX SOL-USDT-SWAP at +# 08:00:07Z. The allowance is the same 60 s funding-boundary tolerance the +# Binance adapter (_FUNDING_BOUNDARY_TOLERANCE_MS) and the acceptance window +# (_FUNDING_SETTLEMENT_JITTER_NS) already use; it applies to FUNDING_RATE +# history only, and a settlement still unpublished after it stays DATA_STALE. +_FUNDING_SETTLEMENT_PUBLICATION_GRACE_MS = 60_000 _BatchCompletion = TypeVar("_BatchCompletion") @@ -1600,13 +1611,24 @@ def _reference_snapshot_requires_refresh( completes. Separately, exchanges can expose a briefly delayed mark or index timestamp even though the same declared provider lane is current on the immediate next read. Both cases get one cache-bypassing re-read - through the same bounded admission path. This deliberately excludes - history and every other reference product: no freshness bound is - relaxed and a second stale observation remains terminal. + through the same bounded admission path. The only history included is + a cache-served FUNDING_RATE series (see below); no freshness bound is + relaxed by the re-read and a second stale observation remains terminal. """ if self._is_execution_mark_index_live_result(result): return False + if ( + request.product is ReferenceProduct.FUNDING_RATE + and request.is_history + and result.status is ReferenceStatus.OK + and result.cache_hit + ): + # A cached funding series cannot contain a settlement the venue + # published after it was cached; one cache-bypassing re-read + # through the same admission lane picks it up. A second stale + # observation remains terminal. + return True if self._reference_snapshot_was_current_at_receipt( requirement, request, @@ -1735,7 +1757,12 @@ def _reference_problem( 0, ((self._clock_ns() if at_ns is None else at_ns) - observed_ns) // 1_000_000, ) - if freshness_ms > requirement.max_freshness_ms: + allowance_ms = ( + _FUNDING_SETTLEMENT_PUBLICATION_GRACE_MS + if request.product is ReferenceProduct.FUNDING_RATE and request.is_history + else 0 + ) + if freshness_ms > requirement.max_freshness_ms + allowance_ms: return QueryProblem( CanonicalErrorCode.DATA_STALE, "reference provider result exceeds the declared freshness bound", diff --git a/qdl/runtime/refusal_log.py b/qdl/runtime/refusal_log.py index 2255edc7..6fb1dfa5 100644 --- a/qdl/runtime/refusal_log.py +++ b/qdl/runtime/refusal_log.py @@ -18,6 +18,10 @@ the first line of a later window, so an incident costs a fixed log rate and its size is still recorded. At most ``max_keys`` keys per window (excess keys share one overflow key); +* per-item refusals inside a 200 batch response (``reference_batch_item`` / + ``warmup_batch_item``) use the same line with the item's ``feed`` and any + typed provider tokens (``extra_reasons``, e.g. a provider error code or a + coverage terminal reason) appended after the detail's own tokens; * WARNING level on logger ``qdl.refusal``: the Query role installs no handler of its own (``serve_stable_query``), so the line reaches stderr through the logging module's last-resort handler without changing any other log volume. @@ -102,10 +106,22 @@ def record( request_id: str | None = None, instrument_uid: str | None = None, route: str | None = None, + feed: str | None = None, + extra_reasons: tuple[object, ...] = (), ) -> bool: - """Emit one line for this refusal unless its key is over budget; True when emitted.""" + """Emit one line for this refusal unless its key is over budget; True when emitted. + + ``feed`` and ``extra_reasons`` are additive: the line gains a ``feed`` + key only when one is given, and an extra reason is kept only when it is + itself one typed UPPER_SNAKE token (never free text). + """ reasons = reason_tokens(detail) + for extra in extra_reasons: + token = str(extra or "") + if (len(reasons) < MAX_REASONS and len(token) <= _MAX_TOKEN + and _REASON.fullmatch(token) and token not in reasons): + reasons = (*reasons, token) key = (str(origin), str(code), reasons[0] if reasons else "") with self._lock: now = self._clock() @@ -138,6 +154,8 @@ def record( "route": _bounded(route), "replica": _bounded(self._replica, 128), } + if feed is not None: + line["feed"] = _bounded(feed, 64) if suppressed: line["suppressed_before"] = suppressed try: diff --git a/qdl_sdk/CHANGELOG.md b/qdl_sdk/CHANGELOG.md index 0a677e62..57a08960 100644 --- a/qdl_sdk/CHANGELOG.md +++ b/qdl_sdk/CHANGELOG.md @@ -32,6 +32,15 @@ and never changes `freshness_ms`, `state` or `execution_eligible`. The server emits the key only when a proof exists (absent otherwise), so 2.0.x strict readers are unaffected until quiet proofs are enabled; 2.1.0+ readers accept it. +Required-batch item diagnostics (P18.4 stage 100g): the `PARTIAL_RESULT` +`DataLayerError` raised by `reference_batch`/`warmup_batch` with `require_all=True` now +sets `diagnostics = {"origin": "sdk_reference_batch" | "sdk_warmup_batch", +"item_problem_count", "item_problems": [...]}`; each item (first 32) carries +`instrument_uid`, `feed`, `code`, `reason_tokens` (typed UPPER_SNAKE tokens of the item +detail), `retryable`, `retry_after_ms`, `request_id` and `detail_sha256` (the digest +Query's `qdl_refusal` item line carries), never the detail text or data. `code`, +`detail`, `retryable` are unchanged; `diagnostics` was `None` on this path before. + ## 2.1.0 Response models, including nested market/reference payloads and error diagnostics, diff --git a/qdl_sdk/client.py b/qdl_sdk/client.py index 43e33b24..08b1e82e 100644 --- a/qdl_sdk/client.py +++ b/qdl_sdk/client.py @@ -1,6 +1,8 @@ from __future__ import annotations import asyncio +import hashlib +import re from collections import OrderedDict from contextlib import asynccontextmanager from typing import AsyncIterator, Protocol @@ -146,6 +148,43 @@ def _view_refusal(row, code: str, detail: str) -> ContinuityError: }) +_REASON_TOKEN = re.compile(r"\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b") +_MAX_ITEM_PROBLEMS = 32 + + +def _batch_item_problems(origin: str, results, feed_of) -> dict: + """Payload-free per-item failures behind a ``require_all`` PARTIAL_RESULT. + + Additive (2.2.0): the code, typed UPPER_SNAKE reason tokens, retryability + and the sha256 of the server's item detail (the digest Query's + ``qdl_refusal`` line carries), never the detail text, data or cursors. + Bounded to the first 32 failing items; ``item_problem_count`` is exact. + """ + + problems = [(index, item) for index, item in enumerate(results) if item.problem is not None] + rows = [] + for index, item in problems[:_MAX_ITEM_PROBLEMS]: + problem = item.problem + detail = str(problem.detail or "") + tokens: list[str] = [] + for token in _REASON_TOKEN.findall(detail[:4096]): + if len(token) <= 64 and token not in tokens: + tokens.append(token) + if len(tokens) == 4: + break + rows.append({ + "instrument_uid": item.instrument_uid, + "feed": feed_of(index, item), + "code": problem.code, + "reason_tokens": tokens, + "retryable": problem.retryable, + "retry_after_ms": problem.retry_after_ms, + "detail_sha256": hashlib.sha256(detail.encode()).hexdigest(), + "request_id": problem.request_id, + }) + return {"origin": origin, "item_problem_count": len(problems), "item_problems": rows} + + def _validate_query_payload( requirement: DataRequirement, payload: dict | WarmupResponse | SnapshotResponse, *, warmup: bool ) -> SnapshotResponse | WarmupResponse: @@ -657,6 +696,9 @@ def retry_batch(checked): "required warmup batch contains one or more explicit failures", retryable=any(item.problem is not None and item.problem.retryable for item in response.results), + diagnostics=_batch_item_problems( + "sdk_warmup_batch", response.results, + lambda index, _item: chunk[index].feed.value), ) self._record_batch(response) yield response @@ -755,6 +797,9 @@ async def reference_batch( item.problem is not None and item.problem.retryable for item in response.results ), + diagnostics=_batch_item_problems( + "sdk_reference_batch", response.results, + lambda _index, item: item.product.value), ) return response diff --git a/scripts/phase3_consumer_load_acceptance.py b/scripts/phase3_consumer_load_acceptance.py index 630c345c..c5fcfec3 100644 --- a/scripts/phase3_consumer_load_acceptance.py +++ b/scripts/phase3_consumer_load_acceptance.py @@ -64,6 +64,12 @@ def _sha256(value: bytes) -> str: ) +_ITEM_PROBLEM_FIELDS = ( + "instrument_uid", "feed", "code", "reason_tokens", "retryable", "retry_after_ms", + "detail_sha256", "request_id", +) + + def _safe_error(error: BaseException) -> dict[str, object]: code = getattr(error, "code", None) detail = str(getattr(error, "detail", error)) @@ -96,6 +102,17 @@ def _safe_error(error: BaseException) -> dict[str, object]: key: (list(diagnostics[key])[:32] if key == "reason_codes" else diagnostics[key]) for key in _DIAGNOSTIC_FIELDS if key in diagnostics } + # SDK 2.2.0 require_all batch PARTIAL_RESULT: which item failed and why + # (stage 100g had only the batch-level hash). Known keys only, bounded. + items = diagnostics.get("item_problems") + if isinstance(items, list): + out["item_problems"] = [ + {key: item.get(key) for key in _ITEM_PROBLEM_FIELDS if key in item} + for item in items[:32] if isinstance(item, dict) + ] + count = diagnostics.get("item_problem_count") + if isinstance(count, int): + out["item_problem_count"] = count return out diff --git a/tests/test_fund_phase5_api.py b/tests/test_fund_phase5_api.py index a9ca1614..9b816e74 100644 --- a/tests/test_fund_phase5_api.py +++ b/tests/test_fund_phase5_api.py @@ -338,6 +338,39 @@ async def refused(): ) self.assertEqual(line["detail_sha256"], hashlib.sha256(detail.encode()).hexdigest()) + def test_a_warmup_batch_item_failure_leaves_one_line(self): + """Astra R6 extension: item problems inside a 200 warmup batch are not silent.""" + import hashlib + import json + from dataclasses import replace + from qdl.query.service import BatchItemResult, BatchQueryResult, WarmupResult + from qdl.runtime.refusal_log import RefusalLog + + router_module = importlib.import_module("qdl.api_v2.router") + history = self.backend.history(self.requirement) + detail = "HOT_READER_BUSY on the cold lane" + result = BatchQueryResult("batch-r6", ( + BatchItemResult(self.binance.instrument_uid, "READY", WarmupResult("row", history)), + BatchItemResult(self.okx.instrument_uid, "DATA_NOT_READY", problem=QueryProblem( + CanonicalErrorCode.DATA_NOT_READY, detail, True)), + )) + requirements = (self.requirement, replace(self.requirement, instrument_uid=self.okx.instrument_uid)) + lines = [] + log = RefusalLog(emit=lambda line: lines.append(json.loads(line)), replica="query-test") + with patch.object(router_module, "_bind_history_cursor", side_effect=lambda _r, _a, _q, h: h), \ + patch.object(router_module, "REFUSALS", log): + router_module._render_warmup_batch(None, None, result, requirements) + (line,) = lines + self.assertEqual( + {key: line[key] for key in ("origin", "code", "retryable", "reasons", "instrument_uid", "feed", + "route", "request_id")}, + {"origin": "warmup_batch_item", "code": "DATA_NOT_READY", "retryable": True, + "reasons": ["HOT_READER_BUSY"], "instrument_uid": self.okx.instrument_uid, + "feed": self.requirement.feed.value, "route": "/v2/market-data/warmup:batch", + "request_id": "batch-r6"}, + ) + self.assertEqual(line["detail_sha256"], hashlib.sha256(detail.encode()).hexdigest()) + def test_query_routes_pass_authenticated_identity_to_owned_read_lanes(self): calls = [] snapshot_original = self.service.snapshot_async diff --git a/tests/test_kn_pre5_sdk.py b/tests/test_kn_pre5_sdk.py index b1542b03..5d28d33a 100644 --- a/tests/test_kn_pre5_sdk.py +++ b/tests/test_kn_pre5_sdk.py @@ -82,6 +82,22 @@ async def broken(*args, **kwargs): with self.subTest(mutation=mutation), self.assertRaises(ContinuityError): await client(query).warmup_batch(requirements(2), require_all=False) + async def test_required_batch_failure_names_the_failing_item(self): + """Stage 100g: a require_all PARTIAL_RESULT must say which item failed, payload-free.""" + import hashlib + with self.assertRaises(DataLayerError) as captured: + await client(_SdkBatchTransport(fail_uids={"uid-1"})).warmup_batch(requirements(3)) + error = captured.exception + self.assertEqual((error.code, error.retryable), ("PARTIAL_RESULT", True)) # unchanged fields + self.assertEqual(error.detail, "required warmup batch contains one or more explicit failures") + self.assertEqual(error.diagnostics["origin"], "sdk_warmup_batch") + self.assertEqual(error.diagnostics["item_problem_count"], 1) + self.assertEqual(error.diagnostics["item_problems"], [{ + "instrument_uid": "uid-1", "feed": "BAR", "code": "SOURCE_UNAVAILABLE", "reason_tokens": [], + "retryable": True, "retry_after_ms": None, "request_id": "batch-request", + "detail_sha256": hashlib.sha256(b"injected provider failure").hexdigest(), + }]) + async def test_cancel_inflight_does_not_launch_next_chunk(self): entered, canceled = asyncio.Event(), asyncio.Event() query = _SdkBatchTransport() diff --git a/tests/test_phase113_reference_v2.py b/tests/test_phase113_reference_v2.py index 46cf2c47..b3e3d503 100644 --- a/tests/test_phase113_reference_v2.py +++ b/tests/test_phase113_reference_v2.py @@ -43,6 +43,7 @@ ReferenceLineage, ReferenceObservation, ReferenceProduct, + ReferenceProviderError, ReferenceProviderExhausted, ReferenceRequest, ReferenceUnavailable, @@ -50,12 +51,13 @@ ) from qdl.warmup.executor import BoundedWarmupExecutor, ProviderBudgetPolicy from qdl_sdk import AsyncDataLayerClient, Grade -from qdl_sdk.errors import ContinuityError +from qdl_sdk.errors import ContinuityError, DataLayerError from qdl_sdk.reference import ReferenceRequirement from tests.phase7_support import make_identity, make_token, manifest_mapping NOW_NS = 1_800_000_000_000_000_000 +_FUNDING_8H_MS = 28_800_000 def record(*, venue: str, market: str, symbol: str, base: str) -> InstrumentRecord: @@ -470,7 +472,9 @@ async def test_cached_reference_result_matches_by_value_not_python_identity(self async def test_stale_reference_result_fails_closed(self): registry = InstrumentRegistry() registry.register(self.binance, []) - adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - 2_000_000_000) + # Beyond the declared bound plus the 60 s funding settlement + # publication allowance (stage 100g, 2026-10-10). + adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - 60_101_000_000) stale = V2QueryService( instruments=InstrumentQuery(registry), backend=MemoryMarketDataBackend(), @@ -490,6 +494,64 @@ async def test_stale_reference_result_fails_closed(self): self.assertEqual(result.results[0].problem.code.value, "DATA_STALE") self.assertEqual(adapter.calls, 1) + def _funding_service(self, adapter, clock, monotonic=lambda: 0.0): + registry = InstrumentRegistry() + registry.register(self.okx, []) + return V2QueryService( + instruments=InstrumentQuery(registry), + backend=MemoryMarketDataBackend(), + entitlements=grants(), + reference_batch=ReferenceBatch( + {("OKX", "SWAP"): adapter}, + clock_ns=lambda: clock["ns"], + monotonic=monotonic, + ), + reference_source_id=lambda item: f"{item.identity.venue}_DIRECT", + clock_ns=lambda: clock["ns"], + ) + + async def _funding_read(self, service): + return await service.reference_data_batch_async( + ReferenceBatchRequirement( + "phase113-alpha", + (self.funding_requirement(self.okx.instrument_uid, freshness_ms=_FUNDING_8H_MS),), + require_all=False, + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + ) + + async def test_funding_settlement_due_but_unpublished_is_not_stale(self): + """Stage 100g (2026-10-10): an 8 h bound on 8 h OKX funding refused SOL-USDT-SWAP at + 08:00:07Z, seven seconds after the settlement fell due and before the venue published it.""" + clock = {"ns": NOW_NS} + adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - (_FUNDING_8H_MS + 7_000) * 1_000_000) + result = await self._funding_read(self._funding_service(adapter, clock)) + self.assertIsNone(result.results[0].problem) + # The allowance is bounded: a settlement still unpublished after it stays fail-closed. + late = FixtureReferenceAdapter(observed_at_ns=NOW_NS - (_FUNDING_8H_MS + 60_001) * 1_000_000) + result = await self._funding_read(self._funding_service(late, clock)) + self.assertEqual(result.results[0].problem.code.value, "DATA_STALE") + self.assertTrue(result.results[0].problem.retryable) + self.assertEqual(late.calls, 1) # a provider-fresh stale series is not re-read + + async def test_a_cached_funding_series_is_re_read_once_when_it_crosses_its_bound(self): + clock = {"ns": NOW_NS} + adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - (_FUNDING_8H_MS - 10_000) * 1_000_000) + service = self._funding_service(adapter, clock) # frozen monotonic: the entry stays cached + self.assertIsNone((await self._funding_read(service)).results[0].problem) + clock["ns"] += 75_000_000_000 + adapter.observed_at_ns = clock["ns"] - 5_000_000_000 # the venue has published the next settlement + result = await self._funding_read(service) + self.assertIsNone(result.results[0].problem) + self.assertFalse(result.results[0].result.cache_hit) + self.assertEqual(adapter.calls, 2) + # A cached series that is still stale after its single re-read is terminal. + clock["ns"] += (_FUNDING_8H_MS + 65_000) * 1_000_000 + adapter.observed_at_ns = NOW_NS + result = await self._funding_read(service) + self.assertEqual(result.results[0].problem.code.value, "DATA_STALE") + self.assertEqual(adapter.calls, 3) + async def test_execution_mark_refreshes_a_cached_value_that_crossed_its_bound(self): clock = {"ns": NOW_NS} adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - 1_500_000_000) @@ -1040,6 +1102,70 @@ async def test_rest_api_checks_manifest_and_serializes_exact_decimals(self): self.assertEqual(denied.json()["code"], "INVALID_ARGUMENT") +class ProtocolFailureReferenceAdapter(FixtureReferenceAdapter): + async def fetch(self, request, *, capability, received_at_ns): + del request, capability, received_at_ns + self.calls += 1 + raise ReferenceProviderError("fixture funding row has no numeric fields 0.000125") + + +class ReferenceBatchItemRefusalLineTests(unittest.TestCase): + """Astra R6 extension: an item failure inside a 200 reference batch leaves a qdl_refusal line.""" + + def test_item_failure_leaves_one_payload_free_line(self): + import hashlib + import importlib + import json + from unittest.mock import patch + + from qdl.runtime.refusal_log import RefusalLog + + router_module = importlib.import_module("qdl.api_v2.router") + binance = record(venue="BINANCE", market="USDM", symbol="BTCUSDT", base="BTC") + registry = InstrumentRegistry() + registry.register(binance, []) + service = V2QueryService( + instruments=InstrumentQuery(registry), + backend=MemoryMarketDataBackend(), + entitlements=grants(), + reference_batch=ReferenceBatch({("BINANCE", "USDM"): ProtocolFailureReferenceAdapter()}, + clock_ns=lambda: NOW_NS), + reference_source_id=lambda item: f"{item.identity.venue}_DIRECT", + clock_ns=lambda: NOW_NS, + ) + consumer_id, subject = "phase113-api-alpha", "spiffe://qdl/paper/phase113-api-alpha" + payload = manifest_mapping(consumer_id=consumer_id, subject=subject, instrument_uid=binance.instrument_uid, + feed="FUNDING_RATE", interval=None, source_policy_id="reference_primary_v2") + payload["spec"]["requirements"][0].update({"warmup_limit": 500, "require_final_bars": False}) + loader = __import__("qdl.consumer", fromlist=["ConsumerManifestLoader"]).ConsumerManifestLoader + client = TestClient(create_v2_app(service, identity_service=make_identity(loader.from_mapping(payload)))) + lines: list[dict] = [] + log = RefusalLog(emit=lambda line: lines.append(json.loads(line)), replica="query-test") + with patch.object(router_module, "REFUSALS", log): + response = client.post( + "/v2/market-data/reference:batch", + headers={"Authorization": f"Bearer {make_token(subject)}", "X-QDL-Consumer-ID": consumer_id, + "X-QDL-Purpose": "INTERNAL_ALPHA"}, + json={"consumer_id": consumer_id, "require_all": True, "requirements": [{ + "instrument_uid": binance.instrument_uid, "product": "FUNDING_RATE", + "consumer_grade": "ALPHA", "source_policy_id": "reference_primary_v2", + "start_time_ns": NOW_NS - 86_400_000_000_000, "end_time_ns": NOW_NS, "limit": 10}]}, + ) + self.assertEqual(response.status_code, 200, response.text) # response unchanged: a partial batch + problem = response.json()["results"][0]["problem"] + (line,) = lines + self.assertEqual( + {key: line[key] for key in ("origin", "code", "retryable", "reasons", "instrument_uid", "feed", + "route", "replica", "request_id")}, + {"origin": "reference_batch_item", "code": "SOURCE_UNAVAILABLE", "retryable": False, + "reasons": ["PROVIDER_PROTOCOL", "PROVIDER_ERROR"], "instrument_uid": binance.instrument_uid, + "feed": "FUNDING_RATE", "route": "/v2/market-data/reference:batch", "replica": "query-test", + "request_id": problem["request_id"]}, + ) + self.assertEqual(line["detail_sha256"], hashlib.sha256(problem["detail"].encode()).hexdigest()) + self.assertNotIn("0.000125", json.dumps(line)) + + class _NoopStreamTransport: async def close(self): return None @@ -1139,6 +1265,43 @@ async def test_sdk_chunks_typed_reference_batches_without_cross_mixing(self): self.assertEqual(transport.calls, [100, 1]) self.assertEqual(len(result.results), 101) + async def test_required_reference_batch_failure_names_the_failing_item(self): + """Stage 100g (2026-10-10): the PARTIAL_RESULT carried only the batch-level detail hash.""" + import hashlib + + transport = _ReferenceTransport() + original = transport.reference_batch + detail = "reference provider result exceeds the declared freshness bound (FUNDING_SETTLEMENT_PENDING)" + + async def one_failed(requirements, **kwargs): + body = await original(requirements, **kwargs) + body["results"][1].update(status="DATA_STALE", data=None, problem={ + "type": "urn:qdl:error:data-stale", "title": "Data Stale", "status": 409, + "code": "DATA_STALE", "detail": detail, "request_id": "r-100g", "retryable": True, + "instrument_uid": "phase113-1"}) + body.update(partial=True, success_count=1, error_count=1) + return body + + transport.reference_batch = one_failed + client = AsyncDataLayerClient(query_transport=transport, stream_transport=_NoopStreamTransport(), + consumer_id="phase113-sdk-alpha") + with self.assertRaises(DataLayerError) as captured: + await client.reference_batch(self.requirements(2), require_all=True) + error = captured.exception + self.assertEqual((error.code, error.retryable), ("PARTIAL_RESULT", True)) + self.assertEqual(error.detail, "required reference batch contains one or more explicit failures") + self.assertEqual(error.diagnostics, { + "origin": "sdk_reference_batch", "item_problem_count": 1, + "item_problems": [{ + "instrument_uid": "phase113-1", "feed": "FUNDING_RATE", "code": "DATA_STALE", + "reason_tokens": ["FUNDING_SETTLEMENT_PENDING"], "retryable": True, "retry_after_ms": None, + "detail_sha256": hashlib.sha256(detail.encode()).hexdigest(), "request_id": "r-100g", + }], + }) + # Not raised when the caller accepts partial results. + partial = await client.reference_batch(self.requirements(2), require_all=False) + self.assertTrue(partial.partial) + async def test_sdk_rejects_reference_identity_corruption(self): client = AsyncDataLayerClient( query_transport=_ReferenceTransport(corrupt=True), diff --git a/tests/test_phase3_consumer_load_driver.py b/tests/test_phase3_consumer_load_driver.py index 649d7744..4728fd45 100644 --- a/tests/test_phase3_consumer_load_driver.py +++ b/tests/test_phase3_consumer_load_driver.py @@ -607,6 +607,27 @@ def test_a_refusal_keeps_every_typed_code_of_its_detail(self): self.assertNotIn("reason", evidence) +class BatchItemProblemEvidenceTests(unittest.TestCase): + def test_a_required_batch_failure_records_which_item_failed(self): + """Stage 100g: the receipt row held only the batch-level PARTIAL_RESULT hash.""" + from qdl_sdk.errors import DataLayerError + + item = {"instrument_uid": "u-sol", "feed": "FUNDING_RATE", "code": "DATA_STALE", + "reason_tokens": ["FUNDING_SETTLEMENT_PENDING"], "retryable": True, "retry_after_ms": None, + "detail_sha256": "d" * 64, "request_id": "r-1", "detail": "never copied", "price": "1.5"} + error = DataLayerError("PARTIAL_RESULT", "required reference batch contains one or more explicit failures", + retryable=True, diagnostics={"origin": "sdk_reference_batch", "item_problem_count": 40, + "item_problems": [item] * 40}) + evidence = _MODULE._safe_error(error) + self.assertEqual(evidence["code"], "PARTIAL_RESULT") + self.assertEqual(evidence["item_problem_count"], 40) + self.assertEqual(len(evidence["item_problems"]), 32) + self.assertEqual(evidence["item_problems"][0], { + key: value for key, value in item.items() if key not in {"detail", "price"}}) + self.assertEqual(evidence["diagnostics"], {}) # no quality fields: unchanged behaviour + self.assertNotIn("item_problems", _MODULE._safe_error(DataLayerError("DATA_STALE", "x"))) + + class Kn4FixedWindowHandoffVerdictTests(unittest.TestCase): """KN-4 D41: the handoff is judged over one fixed window against the canonical log, with the KN-2 rules, from real golden TRADE records.""" diff --git a/tests/test_refusal_log.py b/tests/test_refusal_log.py index 73ba4d71..be68c962 100644 --- a/tests/test_refusal_log.py +++ b/tests/test_refusal_log.py @@ -44,6 +44,22 @@ def test_a_line_is_payload_free_and_joins_the_harness_digest(self): }) self.assertNotIn("64123.5", json.dumps(line)) + def test_batch_item_line_adds_feed_and_only_typed_extra_reasons(self): + log = self.log() + log.record(origin="reference_batch_item", code="SOURCE_UNAVAILABLE", detail="provider said 1.25 (X)", + feed="FUNDING_RATE", extra_reasons=("PROVIDER_PROTOCOL", "free text", None, "PROVIDER_PROTOCOL", + "REACHED_REQUEST_START")) + (line,) = self.lines + self.assertEqual(line["feed"], "FUNDING_RATE") + self.assertEqual(line["reasons"], ["PROVIDER_PROTOCOL", "REACHED_REQUEST_START"]) + # Existing callers keep the original line shape (no feed key). + log.record(origin="query", code="DATA_STALE", detail="x") + self.assertNotIn("feed", self.lines[-1]) + # Extra reasons never exceed the bound and come after the detail's own tokens. + log.record(origin="reference_batch_item", code="DATA_STALE", detail="A_1 B_2 C_3", + extra_reasons=("D_4", "E_5")) + self.assertEqual(self.lines[-1]["reasons"], ["A_1", "B_2", "C_3", "D_4"]) + def test_volume_is_bounded_per_key_and_the_suppressed_count_is_reported_later(self): log = self.log(burst=2, window_seconds=10) emitted = [log.record(origin="query", code="DATA_NOT_READY", detail=DETAIL) for _ in range(7)] From dacde949ab729d61410e36767d5cdae08a2e4b5c Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 08:59:38 +0000 Subject: [PATCH 112/147] docs(plan): pre-rc7 closure (heartbeat, funding boundary, item diagnostics, G13 producers and inputs) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 44 +++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 2e71f3c4..13830f19 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71899,3 +71899,47 @@ Run 07:36-08:08Z under supervisor v2; guard all READY 64/64; MARK watcher 0 erro - `docker restart` of ingestor_okx_swap at 08:09:07Z (after the run, same image): restarts 0, 5 established sockets to remote port 443 (0x01BB), 0 to 8443. - TS endpoint_profiles.py moved too (TS 8b03ff2, pushed). + +### Pre-rc7 closure: heartbeat liveness, funding settlement boundary, per-item diagnostics, G13 producers - 2026-10-10 (Claude, subagents) + +Owner approved the full remaining sequence (rc7 build + CI, rollout, final stage 100 + G2, G13, G1-G14, release, +cleanup). All items below are tested locally and not deployed. +- Silent-socket liveness (0f30e17 from 40b7a3c): + - New optional ingestor fields liveness_ping_ms / liveness_dead_ms (absent = the old 15/30 s heartbeat rule; + bounds: ping >= 500, dead - ping >= 1000, dead <= 29000). + - A ping gets its full pong window from when it was actually sent. Busy sockets never ping. + - Renderer emits 1500/4000: worst-case detection 4.0 s after the last frame; quiet sockets send <= 0.67 pings/s + (Binance USD-M allows 10 incoming messages/s; OKX limits only subscribe/unsubscribe/login). + - Tests: ingestor 44/0, fmt and clippy clean. + - Rollout: the binary first (deny_unknown_fields), then the config from the new release dir + releases/v2.2.3-rc7-ingestor-liveness/ (prepared 2026-10-10 via a root --rm container: exactly the two keys + added; OKX URL 443; config_revision OKX 20 / Binance 19). +- Stage 100g PARTIAL_RESULT root cause (a2184f4 from ec9abe9; probable, the item code was not recorded): + - The request was at 08:00:07Z, 7.1 s after an 8 h OKX funding settlement. stage100g is the only stage-100 run that + crossed a settlement, and this was its only funding failure. + - The alpha manifests set FUNDING_RATE max_freshness_ms 28,800,000, equal to the cadence. Query measured age from + the newest fundingTime with no allowance, so before OKX publishes the due row a healthy series is DATA_STALE. + - Fix: a 60 s settlement-publication allowance (the existing funding boundary tolerance) and one cache-bypassing + re-read of a cache-served stale funding series. + - Per-item diagnostics (R6 extension): `qdl_refusal` lines for reference_batch_item / warmup_batch_item; SDK + 2.2.0 DataLayerError.diagnostics.item_problems on a require_all PARTIAL_RESULT; driver receipt item_problems. + - OKX publication latency after settlement is unmeasured; a read-only poll at a settlement will confirm the 60 s. +- G13: + - e04db5b runner and inherit/rerun matrix: 0/65 cells inheritable (no final-provenance receipts). + - afe19f7 harness image set as a parameter. + - e4dfc3c DR05 scopes BINANCE/OKX/OKX_INVERSE. + - TS 3538b48 matrix v2: 325 + 11 inverse cells, exclusions with reasons. TS acce72d inverse DR05 producers. TS + aeb63f1 regenerated the W01 roster: the ROSTER_DRIFT root cause was that 6c44bf2 added a test naming 9 routes + and the roster was not regenerated. + - TS 6c44bf2 DR01 WRONG_ACCOUNT runner. It found IDR-085 (gateway scope lookup before the API key, i.e. account + enumeration), fixed in TS be21d49. The WRONG_ACCOUNT cell is deferred to the TS release (no gateway runs now). + - All 76 cells have producers. + - Runtime inputs prepared: + - catalog = the deployed /runtime/stable-source-bindings.yaml, sha fc3a1e1e..., identical to + p184-d1-rev11-20261009/base-catalog.yaml (catalog_revision 12, includes BTC-USD-SWAP); + - DR05 export = ~/.local/state/qdl-v2/dr05-export-20261010/public-export.json: 17 l2 bindings, public fields + only, from rust_core /runtime/core.json sha 91b19310...; validated by isolated_config for all three scopes. +- Note for the release: the repo catalog stays revision 11, baked into the image for the bar edge checkpoint + (C.19). Query reads the runtime-mounted revision 12. This split is kept deliberately and stated in the release + notes. +- Tests after merging everything: 18 modules, 347 OK (34 Redis skips). From f69353bcb2189976299d99f50be31a43940156da Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 09:26:13 +0000 Subject: [PATCH 113/147] fix(deployment): refuse OKX port 8443 when rendering ingestor configs, not when readers load the plan (rc7 Query crash) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 20 ++++++++++++++ qdl/runtime/stable_deployment.py | 14 +++++++--- tests/test_okx_ws_port_migration.py | 33 +++++++++++++++++------ 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 13830f19..67bf4a95 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71943,3 +71943,23 @@ cleanup). All items below are tested locally and not deployed. (C.19). Query reads the runtime-mounted revision 12. This split is kept deliberately and stated in the release notes. - Tests after merging everything: 18 modules, 347 OK (34 Redis skips). + +### rc7 rollout stopped at query_kn_1: plan-load 8443 refusal crashed Query; fixed - 2026-10-10 (Claude) + +- rc7 (dacde94; python sha256:c816fedbd1b6, rust sha256:53c74bf236a2) was rolled one role at a time. + - query_kn_1 restarted during startup. + - The script rolled it back within 12 s to rc7-47bec04 (sha256:4fcf724b6b36), healthy. No other role was touched. +- Cause: 6674d0b made StableAcquisitionPlan validation refuse OKX `:8443` URLs at LOAD. + - The deployed runtime bundle mounted by Query and the bar edge (releases/v2.2.0-02cd827/runtime) still names + 8443: 220 lines, plan sha 09c895f1.... + - Isolated import of the Query entrypoint was clean; the refusal is in plan loading. + - Readers that never connect to OKX must not die on a transport URL. +- Fix (this commit): + - The 8443 refusal moved to `native_ingestor_configs`, where ingestor configs are rendered (the only consumer that + connects). Plan load accepts the old URL. The live ingestor config is already on 443. + - Tests: a deployed plan naming 8443 loads (identity unchanged); rendering an ingestor config from it is refused. + - With the fix, the real deployed runtime (catalog 12, plan revision 20, 716 bindings) loads. +- Lesson: check every load-time validator against the deployed runtime files before a rollout, not only against + repo files. +- rc7 is rebuilt (Python and Rust) from this commit. CI on dacde94 continues for early signal; the certifying CI runs + on the final commit. diff --git a/qdl/runtime/stable_deployment.py b/qdl/runtime/stable_deployment.py index 9d16c180..00f8561e 100644 --- a/qdl/runtime/stable_deployment.py +++ b/qdl/runtime/stable_deployment.py @@ -455,10 +455,11 @@ def validate(self, source: StableSourceBinding) -> None: raise ValueError("Rust native acquisition cannot use a REST BAR provider kind") self._require_wss(self.websocket_url) if self.runtime == "OKX": - # Refuse port 8443, which OKX discontinues on 2026-10-31, here - # rather than at connect time after a role was recreated. - require_okx_websocket_url(self.websocket_url, "public") - require_okx_websocket_url(self.business_websocket_url, "business") + # Port 8443 (discontinued by OKX on 2026-10-31) is refused where an + # ingestor config is rendered (native_ingestor_configs), not here: + # Query and the bar edge only read the plan, and a deployed runtime + # plan may still carry the old transport URL (rc7 rollout 2026-10-10: + # a load-time refusal crashed Query on an unchanged runtime bundle). if self.market_websocket_url is not None: raise ValueError( "OKX does not use a Binance /market routed base" @@ -1074,6 +1075,11 @@ def native_ingestor_configs( binding["mark_index_target"] = source.instrument.instrument_uid bindings.append(binding) key = f"{runtime.lower()}-{market.lower()}" + if runtime == "OKX": + # The rendered config is what an ingestor connects with: refuse the + # retired port here so it can never reach a role. + require_okx_websocket_url(first.websocket_url, "public") + require_okx_websocket_url(first.business_websocket_url, "business") result[key] = { "runtime": runtime, "websocket_url": first.websocket_url, diff --git a/tests/test_okx_ws_port_migration.py b/tests/test_okx_ws_port_migration.py index 1ed2e845..b47c15e7 100644 --- a/tests/test_okx_ws_port_migration.py +++ b/tests/test_okx_ws_port_migration.py @@ -99,18 +99,35 @@ def test_committed_plan_uses_port_443_for_every_okx_native_binding(self): self.assertEqual(item.websocket_url, OKX_PUBLIC_WS) self.assertEqual(item.business_websocket_url, OKX_BUSINESS_WS) - def test_a_plan_that_still_names_port_8443_does_not_load(self): + def _plan_with_port_8443(self, directory): payload = yaml.safe_load(ACQUISITION.read_text(encoding="utf-8")) - target = next( - item for item in payload["bindings"] - if item["runtime"] == "OKX" and item["mode"] == "RUST_NATIVE" + for item in payload["bindings"]: + if item["runtime"] == "OKX" and item["mode"] == "RUST_NATIVE": + item["websocket_url"] = OLD_PUBLIC + item["business_websocket_url"] = OLD_BUSINESS + path = Path(directory) / "acquisition.yaml" + path.write_text(yaml.safe_dump(payload, sort_keys=True), encoding="utf-8") + return path + + def test_a_deployed_plan_still_naming_port_8443_loads_for_readers(self): + # rc7 rollout 2026-10-10: Query and the bar edge only read the plan; a load-time refusal crashed Query on the + # unchanged runtime bundle (releases/v2.2.0-02cd827/runtime still names 8443). Loading must succeed. + with tempfile.TemporaryDirectory() as directory: + plan = StableAcquisitionPlan.load(self._plan_with_port_8443(directory), catalog=self.catalog) + self.assertEqual(plan.identity_digest(), self.plan.identity_digest()) + + def test_an_ingestor_config_is_never_rendered_with_port_8443(self): + authority = stable_authority_record( + rust_image_digest="sha256:" + "a" * 64, + capability_manifest=ROOT / "config/v2/stable-capabilities.yaml", + contract=ROOT / "contracts/proto/qdl/marketdata/v2/market_data.proto", + partition_plan=ACQUISITION.read_bytes(), + effective_at_ns=1, ) - target["business_websocket_url"] = OLD_BUSINESS with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "acquisition.yaml" - path.write_text(yaml.safe_dump(payload, sort_keys=True), encoding="utf-8") + plan = StableAcquisitionPlan.load(self._plan_with_port_8443(directory), catalog=self.catalog) with self.assertRaisesRegex(ValueError, "8443"): - StableAcquisitionPlan.load(path, catalog=self.catalog) + plan.native_ingestor_configs(catalog=self.catalog, authority=authority) def test_an_endpoint_move_is_not_an_identity_change(self): moved = dataclasses.replace( From 530a406e6c2097cef1b4f6d313b516f4070328b0 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 09:34:30 +0000 Subject: [PATCH 114/147] revert(query): no funding settlement allowance until the OKX publication delay is measured (Codex review); keep item diagnostics and the cache re-read --- qdl/query/service.py | 10 +++------- tests/test_phase113_reference_v2.py | 25 ++++++++++--------------- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/qdl/query/service.py b/qdl/query/service.py index 813d9d22..e3e23e0f 100644 --- a/qdl/query/service.py +++ b/qdl/query/service.py @@ -73,7 +73,6 @@ # Binance adapter (_FUNDING_BOUNDARY_TOLERANCE_MS) and the acceptance window # (_FUNDING_SETTLEMENT_JITTER_NS) already use; it applies to FUNDING_RATE # history only, and a settlement still unpublished after it stays DATA_STALE. -_FUNDING_SETTLEMENT_PUBLICATION_GRACE_MS = 60_000 _BatchCompletion = TypeVar("_BatchCompletion") @@ -1757,12 +1756,9 @@ def _reference_problem( 0, ((self._clock_ns() if at_ns is None else at_ns) - observed_ns) // 1_000_000, ) - allowance_ms = ( - _FUNDING_SETTLEMENT_PUBLICATION_GRACE_MS - if request.product is ReferenceProduct.FUNDING_RATE and request.is_history - else 0 - ) - if freshness_ms > requirement.max_freshness_ms + allowance_ms: + # No funding settlement allowance until the OKX publication delay is measured and the refused item code + # is recorded (Codex review 2026-10-10: the stage-100g cause is probable, not proven). + if freshness_ms > requirement.max_freshness_ms: return QueryProblem( CanonicalErrorCode.DATA_STALE, "reference provider result exceeds the declared freshness bound", diff --git a/tests/test_phase113_reference_v2.py b/tests/test_phase113_reference_v2.py index b3e3d503..8b1c251a 100644 --- a/tests/test_phase113_reference_v2.py +++ b/tests/test_phase113_reference_v2.py @@ -472,9 +472,7 @@ async def test_cached_reference_result_matches_by_value_not_python_identity(self async def test_stale_reference_result_fails_closed(self): registry = InstrumentRegistry() registry.register(self.binance, []) - # Beyond the declared bound plus the 60 s funding settlement - # publication allowance (stage 100g, 2026-10-10). - adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - 60_101_000_000) + adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - 101_000_000) stale = V2QueryService( instruments=InstrumentQuery(registry), backend=MemoryMarketDataBackend(), @@ -520,19 +518,16 @@ async def _funding_read(self, service): purpose=AccessPurpose.INTERNAL_ALPHA, ) - async def test_funding_settlement_due_but_unpublished_is_not_stale(self): - """Stage 100g (2026-10-10): an 8 h bound on 8 h OKX funding refused SOL-USDT-SWAP at - 08:00:07Z, seven seconds after the settlement fell due and before the venue published it.""" + async def test_funding_bound_stays_strict_until_publication_delay_is_measured(self): + """Stage 100g (2026-10-10) refused OKX SOL-USDT-SWAP funding 7 s after the 08:00Z settlement. The cause is + probable, not proven (Codex review), so no settlement allowance is granted: the declared bound is exact.""" clock = {"ns": NOW_NS} - adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - (_FUNDING_8H_MS + 7_000) * 1_000_000) - result = await self._funding_read(self._funding_service(adapter, clock)) - self.assertIsNone(result.results[0].problem) - # The allowance is bounded: a settlement still unpublished after it stays fail-closed. - late = FixtureReferenceAdapter(observed_at_ns=NOW_NS - (_FUNDING_8H_MS + 60_001) * 1_000_000) - result = await self._funding_read(self._funding_service(late, clock)) - self.assertEqual(result.results[0].problem.code.value, "DATA_STALE") - self.assertTrue(result.results[0].problem.retryable) - self.assertEqual(late.calls, 1) # a provider-fresh stale series is not re-read + for past_ms in (7_000, 60_001): + adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - (_FUNDING_8H_MS + past_ms) * 1_000_000) + result = await self._funding_read(self._funding_service(adapter, clock)) + self.assertEqual(result.results[0].problem.code.value, "DATA_STALE") + self.assertTrue(result.results[0].problem.retryable) + self.assertEqual(adapter.calls, 1) # a provider-fresh stale series is not re-read async def test_a_cached_funding_series_is_re_read_once_when_it_crosses_its_bound(self): clock = {"ns": NOW_NS} From 54460ac4f347728d2b28124145a8def87cbba6ef Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 09:37:52 +0000 Subject: [PATCH 115/147] fix(v1): make the vnstock poller opt-in; KN images exclude its SDK The V1 app defaulted DATA_LAYER_VNSTOCK_POLLER_ENABLED to true while the vnstock SDK has been excluded from pyproject/KN images since 4fe7e92, so every default startup of data-layer:v0.1.0 (incl. the CI unit-tests compose dependency data_layer) failed in lifespan with VNSTOCK_UNAVAILABLE and crash-looped. Default is now false; an explicit true still fails closed. Regression test drives lifespan with vnstock absent. --- .env.example | 3 ++- app/runtime_source_config.py | 5 ++++- tests/test_runtime_source_config.py | 7 ++++++- tests/test_runtime_source_lifespan.py | 28 +++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 908af1fd..4cab594e 100644 --- a/.env.example +++ b/.env.example @@ -39,7 +39,8 @@ BINANCE_WS_BATCH_SIZE=100 BINANCE_WS_QUEUE_MAXSIZE=10000 DATA_LAYER_BINANCE_SOURCES=binance_futures_trade,binance_futures_kline DATA_LAYER_DNSE_STREAM_ENABLED=true -DATA_LAYER_VNSTOCK_POLLER_ENABLED=true +# vnstock SDK is excluded from KN images; true there fails startup (VNSTOCK_UNAVAILABLE). +DATA_LAYER_VNSTOCK_POLLER_ENABLED=false DATA_LAYER_PRELOAD_WATCHDOG_ENABLED=true BINANCE_WS_MAX_CONNS_PER_SOURCE=0 STREAM_STALE_SECONDS=180 diff --git a/app/runtime_source_config.py b/app/runtime_source_config.py index 83469221..36bf02e2 100644 --- a/app/runtime_source_config.py +++ b/app/runtime_source_config.py @@ -61,9 +61,12 @@ def from_env(cls, values: Mapping[str, str] | None = None) -> "RuntimeSourceConf default=True, name="DATA_LAYER_DNSE_STREAM_ENABLED", ), + # The vnstock SDK is not a dependency of this source (pyproject) and + # is excluded from KN images, so the poller is opt-in. An explicit + # true on such an image still fails closed with VNSTOCK_UNAVAILABLE. vnstock_poller_enabled=_parse_bool( env.get("DATA_LAYER_VNSTOCK_POLLER_ENABLED"), - default=True, + default=False, name="DATA_LAYER_VNSTOCK_POLLER_ENABLED", ), preload_watchdog_enabled=_parse_bool( diff --git a/tests/test_runtime_source_config.py b/tests/test_runtime_source_config.py index d314ae0c..108fb9c4 100644 --- a/tests/test_runtime_source_config.py +++ b/tests/test_runtime_source_config.py @@ -11,9 +11,14 @@ def test_defaults_disable_undeclared_spot_and_keep_usdm(self): self.assertFalse(config.spot_enabled) self.assertTrue(config.usdm_enabled) self.assertTrue(config.dnse_stream_enabled) - self.assertTrue(config.vnstock_poller_enabled) + self.assertFalse(config.vnstock_poller_enabled) self.assertTrue(config.preload_watchdog_enabled) + def test_vnstock_poller_is_opt_in(self): + config = RuntimeSourceConfig.from_env({"DATA_LAYER_VNSTOCK_POLLER_ENABLED": "true"}) + + self.assertTrue(config.vnstock_poller_enabled) + def test_spot_can_be_disabled_without_disabling_usdm(self): config = RuntimeSourceConfig.from_env( { diff --git a/tests/test_runtime_source_lifespan.py b/tests/test_runtime_source_lifespan.py index 96c9336b..8d45fbc9 100644 --- a/tests/test_runtime_source_lifespan.py +++ b/tests/test_runtime_source_lifespan.py @@ -42,6 +42,34 @@ async def idle_stream(*args, **kwargs): self.assertFalse(any(source.startswith("binance_spot") for source in sources)) self.assertIsNone(app_main.dnse_stream_manager) + async def test_default_env_starts_without_the_excluded_vnstock_sdk(self): + """KN images exclude vnstock; default startup must not reach its poller.""" + async def idle_stream(*args, **kwargs): + await asyncio.Event().wait() + + env = { + key: value + for key, value in os.environ.items() + if key != "DATA_LAYER_VNSTOCK_POLLER_ENABLED" + } + env.update( + { + "DATA_LAYER_DNSE_STREAM_ENABLED": "false", + "DATA_LAYER_PRELOAD_WATCHDOG_ENABLED": "false", + } + ) + with ( + mock.patch.dict(os.environ, env, clear=True), + mock.patch.dict("sys.modules", {"vnstock": None}), + mock.patch.object(app_main.redis_cache, "init_ping", new=mock.AsyncMock()), + mock.patch.object(app_main, "start_stream", side_effect=idle_stream), + mock.patch.object(app_main, "load_vn_symbols", return_value=["VN30F1M"]), + mock.patch.object(app_main, "VnstockPoller") as poller, + ): + async with app_main.lifespan(app_main.app): + await asyncio.sleep(0) + poller.assert_not_called() + if __name__ == "__main__": unittest.main() From d6fb34c4f8d17d0a347fcfa0b8c45c9e7495c3d7 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 10:44:12 +0000 Subject: [PATCH 116/147] style(rust): apply cargo fmt (rustfmt 1.82.0) so CI fmt --check passes Formatting only; no semantic change. Verified offline in qdl-rust-builder:r134-test: cargo fmt --check, clippy --workspace --all-targets --locked -D warnings, cargo test --workspace --locked --offline (381 passed, 0 failed, 66 ignored). --- rust/qdl-core/src/canonical.rs | 22 ++++-- rust/qdl-core/src/l2_adapter.rs | 16 ++++- rust/qdl-projector/src/products.rs | 34 ++++++--- rust/qdl-realtime-core/src/lib.rs | 107 ++++++++++++++++++++++------- 4 files changed, 139 insertions(+), 40 deletions(-) diff --git a/rust/qdl-core/src/canonical.rs b/rust/qdl-core/src/canonical.rs index c4b3406b..48a3429e 100644 --- a/rust/qdl-core/src/canonical.rs +++ b/rust/qdl-core/src/canonical.rs @@ -922,23 +922,35 @@ pub fn canonicalize_l2_invalidation(fixture: &TradeFixture) -> Result ProcessBatch { // A transport liveness marker never starts, changes or fences a book. let liveness_marker = l2_transport_liveness(&payload).is_some(); - let mut batch = self.process_l2_inner(binding_key, binding, &raw, payload, - processing_at_ns, transport_offset); - let Some(adapter) = self.l2_adapters.get(binding_key) else { return batch; }; + let mut batch = self.process_l2_inner( + binding_key, + binding, + &raw, + payload, + processing_at_ns, + transport_offset, + ); + let Some(adapter) = self.l2_adapters.get(binding_key) else { + return batch; + }; if adapter.core().status() == BookStatus::Ready { if !batch.canonical.is_empty() { self.l2_invalidated.remove(binding_key); @@ -1181,20 +1189,35 @@ impl RealtimeCore { | Some(L2_LIVENESS_UNPROVEN | L2_LIVENESS_NOT_DUE) ) || liveness_marker - { return batch; } + { + return batch; + } let scope = (raw.source_session_id.clone(), raw.connection_generation); - if self.l2_invalidated.get(binding_key) == Some(&scope) { return batch; } + if self.l2_invalidated.get(binding_key) == Some(&scope) { + return batch; + } let Some(sequence) = self.next_l2_partition_sequence(binding, transport_offset) else { return batch; }; - let Ok(payload) = serde_json::from_slice(&raw.raw_frame_bytes) else { return batch; }; + let Ok(payload) = serde_json::from_slice(&raw.raw_frame_bytes) else { + return batch; + }; let fixture = l2_fixture(binding, &raw, payload, sequence); - let Ok(events) = canonicalize_l2_invalidation(&fixture) else { return batch; }; + let Ok(events) = canonicalize_l2_invalidation(&fixture) else { + return batch; + }; for event in events { - if self.seen_ids.contains(&event.event_id) { continue; } - self.partition_sequences.insert(l2_partition_key(binding), event.partition_sequence); + if self.seen_ids.contains(&event.event_id) { + continue; + } + self.partition_sequences + .insert(l2_partition_key(binding), event.partition_sequence); self.remember(event.event_id.clone()); - batch.canonical.push(canonical_record(&self.config.canonical_stream, event, raw.received_at_ns)); + batch.canonical.push(canonical_record( + &self.config.canonical_stream, + event, + raw.received_at_ns, + )); } self.l2_invalidated.insert(binding_key.to_owned(), scope); self.l2_snapshot_observations.remove(binding_key); @@ -2175,14 +2198,27 @@ mod tests { fn assert_book_fence(batch: &ProcessBatch) { use qdl_contracts::qdl::common::v1::QualityFlag; assert_eq!(batch.canonical.len(), 2); - let events: Vec<_> = batch.canonical.iter().map(|record| - EventEnvelope::decode(record.payload.as_slice()).unwrap()).collect(); - assert_eq!(events[1].partition_sequence, events[0].partition_sequence + 1); + let events: Vec<_> = batch + .canonical + .iter() + .map(|record| EventEnvelope::decode(record.payload.as_slice()).unwrap()) + .collect(); + assert_eq!( + events[1].partition_sequence, + events[0].partition_sequence + 1 + ); for event in &events { - assert!(event.quality_flags.contains(&(QualityFlag::SequenceGapBefore as i32))); - assert!(event.quality_flags.contains(&(QualityFlag::ResyncRequired as i32))); + assert!(event + .quality_flags + .contains(&(QualityFlag::SequenceGapBefore as i32))); + assert!(event + .quality_flags + .contains(&(QualityFlag::ResyncRequired as i32))); assert!(!event.raw_capture_id.is_empty()); - assert_eq!(event.source_event_time_ns, event.received_at_ns / 1_000_000 * 1_000_000); + assert_eq!( + event.source_event_time_ns, + event.received_at_ns / 1_000_000 * 1_000_000 + ); match event.payload.as_ref().unwrap() { event_envelope::Payload::BookSnapshot(book) => { assert!(book.levels.is_empty()); @@ -3567,15 +3603,34 @@ mod tests { let binding = okx_book_binding(); let mut engine = core(binding.clone(), true); let at = 1_786_352_400_000_000_000; - let original = engine.process_at_transport_offset( - delayed_book_frame(&binding, true, 100, 0, 7, at, "1"), at, 0).unwrap(); - let gap = engine.process_at_transport_offset( - delayed_book_frame(&binding, false, 110, 109, 7, at + 1_000_000, "2"), at, 1).unwrap(); + let original = engine + .process_at_transport_offset( + delayed_book_frame(&binding, true, 100, 0, 7, at, "1"), + at, + 0, + ) + .unwrap(); + let gap = engine + .process_at_transport_offset( + delayed_book_frame(&binding, false, 110, 109, 7, at + 1_000_000, "2"), + at, + 1, + ) + .unwrap(); assert_book_fence(&gap); let recovery = delayed_book_frame(&binding, true, 100, 0, 7, at + 2_000_000, "1"); - let restored = engine.process_at_transport_offset(recovery.clone(), at, 2).unwrap(); - assert_eq!(restored.canonical.len(), 1, "recovery anchor must replace fence"); - assert_ne!(restored.canonical[0].event_id, original.canonical[0].event_id); + let restored = engine + .process_at_transport_offset(recovery.clone(), at, 2) + .unwrap(); + assert_eq!( + restored.canonical.len(), + 1, + "recovery anchor must replace fence" + ); + assert_ne!( + restored.canonical[0].event_id, + original.canonical[0].event_id + ); let event = EventEnvelope::decode(restored.canonical[0].payload.as_slice()).unwrap(); match event.payload.unwrap() { event_envelope::Payload::BookSnapshot(book) => { @@ -3585,7 +3640,11 @@ mod tests { } _ => panic!("recovery requires snapshot"), } - assert!(engine.process_at_transport_offset(recovery, at, 2).unwrap().canonical.is_empty()); + assert!(engine + .process_at_transport_offset(recovery, at, 2) + .unwrap() + .canonical + .is_empty()); } #[test] From 2c84e2fa6090583de95f316e0203c47c83bef1b2 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 10:55:00 +0000 Subject: [PATCH 117/147] test(ci): skip TS-bound DR05 tests in DL-only checkouts; align phase7 and DR05 with SDK 2.1+/IDR-040 - test_p184_dr05_ts_validate: TS application code (adapters.*, services.*) is resolved from the TS image (PYTHONPATH=/app) or QDL_TRADING_SYSTEM_SOURCE_ROOT. A DL-only checkout skips with an explicit reason; a configured root without TS code, or any TS import failure other than an absent top-level package, errors. Helper subprocesses receive the TS root on PYTHONPATH. - p184_dr05_ts_validate: GetFeedStatus quality must carry every QualityView field except proof_observed_at_ns, which the server emits only when a quiet-book proof exists (CHANGELOG IDR-040 fix 2); unknown keys are still refused. - test_fund_phase7_contract_security: SDK 2.1.0 response models ignore unknown additive fields (CHANGELOG 2.1.0, ResponseModel extra="ignore"); assert the provider-native field is dropped from the typed view instead of raising. --- scripts/p184_dr05_ts_validate.py | 7 ++- tests/test_fund_phase7_contract_security.py | 10 +++- tests/test_p184_dr05_ts_validate.py | 65 ++++++++++++++++++++- 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/scripts/p184_dr05_ts_validate.py b/scripts/p184_dr05_ts_validate.py index 9481ebb2..6c559deb 100644 --- a/scripts/p184_dr05_ts_validate.py +++ b/scripts/p184_dr05_ts_validate.py @@ -29,6 +29,8 @@ MAX_INPUT = 4 * 1024 * 1024 IMAGE = "sha256:a6548ee36aeb0242c1734c071678c525c1506d05c10a70b533eb7110e7eaf16a" ENDPOINT = "/qdl.query.v2.MarketDataStreamService/" +# QualityView keys the server omits when unset (qdl_sdk/CHANGELOG.md, IDR-040 fix 2). +EMITTED_ONLY_WHEN_SET = frozenset({"proof_observed_at_ns"}) def requirement_from_json(raw): @@ -113,8 +115,11 @@ def validate(packet, *, live=None): if requirement.instrument_uid != instrument.instrument_uid: raise ValueError("requirement identity mismatch") # Require every captured status field: model defaults cannot create proof. + # Exception: SDK fields the server emits only when set (IDR-040 + # proof_observed_at_ns is absent without a quiet-book proof). fields = set(QualityView.model_fields) - if set(status["status"]) != fields: + keys = set(status["status"]) + if not fields - EMITTED_ONLY_WHEN_SET <= keys <= fields: raise ValueError("complete actual GetFeedStatus quality required") quality = QualityView.model_validate(status["status"]) if snapshot["cursor_valid"] is not True: diff --git a/tests/test_fund_phase7_contract_security.py b/tests/test_fund_phase7_contract_security.py index 9744f22b..de77dbcf 100644 --- a/tests/test_fund_phase7_contract_security.py +++ b/tests/test_fund_phase7_contract_security.py @@ -217,9 +217,15 @@ def test_typed_contract_rejects_provider_fallthrough_and_feed_mismatch(self): headers=self.fixture.headers(), ).json()["data"] response["payload"]["provider_native_secret"] = "must-not-leak" - with self.assertRaisesRegex(ValueError, "Extra inputs"): - MarketDataView.model_validate(response) + # SDK 2.1.0+ response readers ignore unknown additive fields + # (qdl_sdk/CHANGELOG.md "2.1.0"; ResponseModel extra="ignore"). A + # provider-native field must still never fall through into the typed + # view: it is dropped, not carried. + view = MarketDataView.model_validate(response) + self.assertFalse(hasattr(view.payload, "provider_native_secret")) + self.assertNotIn("must-not-leak", view.model_dump_json()) response["payload"].pop("provider_native_secret") + self.assertEqual(view, MarketDataView.model_validate(response)) response["feed"] = "TRADE" with self.assertRaises(ValueError): MarketDataView.model_validate(response) diff --git a/tests/test_p184_dr05_ts_validate.py b/tests/test_p184_dr05_ts_validate.py index 248a96b4..01c50b30 100644 --- a/tests/test_p184_dr05_ts_validate.py +++ b/tests/test_p184_dr05_ts_validate.py @@ -5,6 +5,7 @@ import importlib.util import json import os +import sys import unittest from qdl.common.v1 import common_pb2 @@ -17,6 +18,52 @@ else: from scripts import p184_dr05_ts_validate as helper +# The helper validates through Trading System application code (adapters.*, +# services.*). Inside the TS image it resolves from PYTHONPATH=/app. From a DL +# checkout, point QDL_TRADING_SYSTEM_SOURCE_ROOT at a TS checkout; the root is +# appended so DL packages keep precedence. A DL-only checkout (DL CI) skips the +# TS-dependent tests explicitly; an explicitly configured root that lacks the +# TS code is an error, never a skip. +TS_SOURCE_ROOT_ENV = "QDL_TRADING_SYSTEM_SOURCE_ROOT" +TS_SOURCE_ROOT = os.environ.get(TS_SOURCE_ROOT_ENV) +TS_PROBE_MODULE = "adapters.market_data.data_layer_client" +if TS_SOURCE_ROOT: + TS_SOURCE_ROOT = os.path.abspath(TS_SOURCE_ROOT) + if not os.path.isfile(os.path.join(TS_SOURCE_ROOT, *TS_PROBE_MODULE.split(".")) + ".py"): + raise RuntimeError(f"{TS_SOURCE_ROOT_ENV}={TS_SOURCE_ROOT} has no {TS_PROBE_MODULE}") + if TS_SOURCE_ROOT not in sys.path: + sys.path.append(TS_SOURCE_ROOT) + + +def _ts_source_available(): + # Only an absent top-level TS package means "DL-only checkout". Any other + # import failure inside TS code propagates so it cannot hide as a skip. + try: + return importlib.util.find_spec(TS_PROBE_MODULE) is not None + except ModuleNotFoundError as error: + if TS_SOURCE_ROOT or error.name != TS_PROBE_MODULE.split(".")[0]: + raise + return False + + +TS_SOURCE_AVAILABLE = _ts_source_available() +if TS_SOURCE_ROOT and not TS_SOURCE_AVAILABLE: + raise RuntimeError(f"{TS_PROBE_MODULE} not importable from {TS_SOURCE_ROOT_ENV}={TS_SOURCE_ROOT}") +REQUIRES_TS = unittest.skipUnless( + TS_SOURCE_AVAILABLE, + f"Trading System source ({TS_PROBE_MODULE}) not importable: run inside the TS image " + f"or set {TS_SOURCE_ROOT_ENV} to a TS checkout", +) + + +def helper_subprocess_env(): + env = dict(os.environ) + if TS_SOURCE_ROOT: + env["PYTHONPATH"] = os.pathsep.join( + part for part in (env.get("PYTHONPATH"), TS_SOURCE_ROOT) if part) + return env + + NOW = 1_800_000_000_000_000_000 UID = "11111111-1111-5111-8111-111111111111" @@ -84,6 +131,7 @@ def change_event(packet, callback): target["events_b64"] = [base64.b64encode(event.SerializeToString()).decode()] +@REQUIRES_TS class ValidationTests(unittest.TestCase): def result(self, packet): return helper.validate(packet)["results"][0] @@ -146,6 +194,17 @@ def test_missing_quality_never_uses_model_defaults(self): del row(packet)["GetFeedStatus"]["status"]["provider_session_state"] self.assertEqual(self.result(packet)["outcome"], "INPUT_ERROR") + def test_quality_keys_emitted_only_when_set_are_optional_not_open(self): + from qdl_sdk.models import QualityView + packet = fixture() + row(packet)["GetFeedStatus"]["status"]["not_a_quality_field"] = 1 + self.assertEqual(self.result(packet)["outcome"], "INPUT_ERROR") + if "proof_observed_at_ns" not in QualityView.model_fields: + self.skipTest("SDK before IDR-040 fix 2 has no proof_observed_at_ns") + packet = fixture() + row(packet)["GetFeedStatus"]["status"]["proof_observed_at_ns"] = NOW + self.assertEqual(self.result(packet)["outcome"], "ACCEPTED") + def test_missing_handoff_does_not_invent_authority(self): packet = fixture() packet["handoffs"] = {} @@ -196,7 +255,8 @@ def test_stdio_contract(self): import subprocess import sys process = subprocess.run([sys.executable, "-B", helper.__file__], - input=json.dumps(fixture()), text=True, capture_output=True, timeout=10) + input=json.dumps(fixture()), text=True, capture_output=True, timeout=10, + env=helper_subprocess_env()) self.assertEqual(process.returncode, 0, process.stderr) result = json.loads(process.stdout) self.assertEqual(result["results"][0]["outcome"], "ACCEPTED") @@ -234,6 +294,7 @@ def test_never_claims_full_acceptance(self): self.assertNotIn("/helper.py", result["application_modules"].values()) +@REQUIRES_TS @unittest.skipUnless(os.environ.get("P184_DR05_LIVE_TEST") == "YES", "isolated Redis DB2 required") class LiveTests(unittest.TestCase): @classmethod @@ -244,7 +305,7 @@ def setUpClass(cls): import urllib.request cls.ns = "p184-dr05-dl-0123456789abcdef" cls.server = subprocess.Popen([sys.executable, "-B", helper.__file__, "--serve"], - env={**os.environ, "P184_DR05_ISOLATED_NAMESPACE": cls.ns}, + env={**helper_subprocess_env(), "P184_DR05_ISOLATED_NAMESPACE": cls.ns}, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) try: for _ in range(60): From bbfb3be6b3f1b7f83b44022e77439daeb4999a65 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 11:11:17 +0000 Subject: [PATCH 118/147] docs(plan): rc7c rollout, writer ed02064, stage 100j PASS 32/32, G4 window-1 note, CI failures --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 39 +++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 67bf4a95..d4bba831 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -71963,3 +71963,42 @@ cleanup). All items below are tested locally and not deployed. repo files. - rc7 is rebuilt (Python and Rust) from this commit. CI on dacde94 continues for early signal; the certifying CI runs on the final commit. + +### rc7c rollout (all 13 DL roles on 54460ac), writer ed02064, stage 100j PASS 32/32 - 2026-10-10 (Claude) + +- Pre-rollout checks: + - C.19: old bar edge image and rc7c both have catalog_revision 11, acquisition revision 19, 712 bindings, catalog + sha 9d4ab51e4880. + - Isolated startup smoke (live env and mounts read-only, --network none) for Query, bar edge, Stream, projector, + core and ingestor. Every role passed config load; the only stops were network ones (the bar edge stops the same + way on the old image). + - The rc7c ingestor accepts the new liveness config: started with OKX 103 / Binance 27 bindings. +- Rollout (rollout_rc6.py, one role at a time, health and writer READY 64/64 after each), rc7c = python + sha256:55e493786748 / rust sha256:206ae872a5fe (54460ac): + - order: query_kn_1/2, binance_bar_edge, stream_kn_1/2, market_projector_1/2, rust_core x3, ingestor_okx_swap, + ingestor_binance_usdm; + - ingestors moved to releases/v2.2.3-rc7-ingestor-liveness/ (liveness 1500/4000, OKX on 443). + - All 12 steps ok, 0 restarts. IDR-039 closed at runtime: every application role runs revision 54460ac. + - Kafka, Redis and the one-shots untouched. The OKX ingestor holds 5 sockets to port 443. +- TS writer market_data_service_p184w: 15524c0b (9b86ed3) -> sha256:34202753bbe8 (TS ed02064). + - Image only; env and mounts verified identical. Guard all READY 64/64. + - The new DATA_LAYER_V2_BOOK_FAST_RETRY_RATE_PER_SECOND defaults to 0.5 (production pace unchanged); only the + isolated DR05 clone uses 2.0. + - Root cause of the G13 dr05-upstream 4/6 FAIL: the single-frame synthetic venue vs the 4 s writer retry spacing + since 12be5f9. DL was correct in every phase. +- Stage 100j (pre-certification; a cargo fmt/clippy/test job ran concurrently): PASS 32/32. + - requests 0 failures; streams 180 / 0 errors. + - G5 cold 2500 B/O 4023/3482 ms; 5000 B/O 6986/6502 ms. + - startup 77.2 s, 0 failures; ts:ready 154/154 samples, 0 unreadable; guard 0 bad samples. + - MARK watcher: 24 minutes with 0 errors, 1 minute with 5. +- G4 on 100j: + - window 1 (10:39:40-10:41:40Z, just after the fleet startup) is above the limits on 10 rows by 1-15 %; + - windows 2-4 are the best measured so far (trade B 107-117 ms vs 160-190 on rc5/rc7); + - venue->DL is flat in every window. + The same one-window pattern appeared before (100b w1, 100f w3), consistent with host CPU contention. The quiet-host + certification run decides whether the startup warmup burst itself delays core/Kafka (a G14 resource fix). +- Certifying CI on 54460ac FAILED 2 steps, both pre-existing: + - cargo fmt --check (15 diffs); + - unit tests 3151: 2 failures, 23 errors. TS-bound test_p184_dr05_ts_validate in a DL-only checkout, and phase7 + vs the SDK 2.1 tolerant response models. + Fix in progress (d6fb34c fmt, 2c84e2f tests); rc7d and a certifying CI follow. From 259aeb99a3437e07344d39a7091bc535fd7c3372 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 12:45:37 +0000 Subject: [PATCH 119/147] fix(ci-local): pass trivy ignore-unfixed when YAML gives a boolean and mount the repo .trivyignore, as trivy-action does --- scripts/p184_local_ci.py | 12 +++++++++--- tests/test_p184_local_ci.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/scripts/p184_local_ci.py b/scripts/p184_local_ci.py index 0011c90f..06f42993 100644 --- a/scripts/p184_local_ci.py +++ b/scripts/p184_local_ci.py @@ -573,9 +573,15 @@ def trivy_command(step, image, source): if with_.get("scan-type") == "fs": return ["docker", "run", "--rm", "-v", f"{source}:/scan:ro", image, "fs", *common, "--scanners", with_.get("scanners", "vuln"), "/scan"] - extra = ["--ignore-unfixed"] if with_.get("ignore-unfixed") == "true" else [] - return ["docker", "run", "--rm", "-v", "/var/run/docker.sock:/var/run/docker.sock:ro", image, "image", - *common, *extra, with_["image-ref"]] + # YAML parses `ignore-unfixed: true` as a boolean; accept both spellings, as trivy-action does (a string-only + # comparison silently dropped the flag and scanned 44 unfixed Debian HIGHs, 2026-10-10). + extra = ["--ignore-unfixed"] if str(with_.get("ignore-unfixed", "")).lower() == "true" else [] + mounts = ["-v", "/var/run/docker.sock:/var/run/docker.sock:ro"] + if (Path(source) / ".trivyignore").is_file(): + # trivy-action reads the repository's .trivyignore from the workspace; mount it the same way. + mounts += ["-v", f"{Path(source) / '.trivyignore'}:/tmp/.trivyignore:ro"] + extra += ["--ignorefile", "/tmp/.trivyignore"] + return ["docker", "run", "--rm", *mounts, image, "image", *common, *extra, with_["image-ref"]] def predict(steps, decisions): diff --git a/tests/test_p184_local_ci.py b/tests/test_p184_local_ci.py index 65b0263e..61b71d67 100644 --- a/tests/test_p184_local_ci.py +++ b/tests/test_p184_local_ci.py @@ -184,3 +184,20 @@ def test_disk_floor_is_forty_gigabytes(self): if __name__ == "__main__": unittest.main() + + +class TrivyCommandFlagsTests(unittest.TestCase): + def test_boolean_ignore_unfixed_and_repo_ignorefile_are_passed(self): + import tempfile + from types import SimpleNamespace + from pathlib import Path as _P + import scripts.p184_local_ci as ci + with tempfile.TemporaryDirectory() as src: + (_P(src) / ".trivyignore").write_text("CVE-0000-0000\n") + for value in (True, "true", "True"): + step = SimpleNamespace(with_={"image-ref": "img", "ignore-unfixed": value, "exit-code": "1"}) + cmd = ci.trivy_command(step, "aquasec/trivy:x", src) + self.assertIn("--ignore-unfixed", cmd) + self.assertIn("--ignorefile", cmd) + step = SimpleNamespace(with_={"image-ref": "img", "ignore-unfixed": False}) + self.assertNotIn("--ignore-unfixed", ci.trivy_command(step, "aquasec/trivy:x", src)) From 163191c840b78793ee9b3db8af170be9e43b4074 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 13:30:58 +0000 Subject: [PATCH 120/147] docs(plan): certifying stage 100m PASS 32/32 + G4, G14 table, G2 first window --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index d4bba831..204cfe04 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72002,3 +72002,30 @@ cleanup). All items below are tested locally and not deployed. - unit tests 3151: 2 failures, 23 errors. TS-bound test_p184_dr05_ts_validate in a DL-only checkout, and phase7 vs the SDK 2.1 tolerant response models. Fix in progress (d6fb34c fmt, 2c84e2f tests); rc7d and a certifying CI follow. + +### Certifying stage 100m PASS 32/32 + G4 on the final digests; G14; G2 first window - 2026-10-10 (Claude) + +Final digests: the DL application roles all on rc7d bbfb3be (python sha256:e0252f4c96da, rust sha256:64468c72ce5a; +rolled one role at a time, writer READY 64/64 after every step), TS writer sha256:34202753bbe8 (ed02064). Quiet host: +no build, CI or subagent job running. +- Stage 100m (12:45-13:17Z, supervisor v2, fixed harness): PASS 32/32. + - requests 0 failures; G5 cold 2500 B/O 3124/3727 ms, 5000 B/O 5299/6178 ms; startup 72.3 s, 0 failures. + - ts:ready 154/154, 0 unreadable; guard 0 bad samples; MARK watcher 0 errors in 32 of 32 minutes. + - G4 PASS on every measured row (16/17; the unmeasured one had no event in the windows). The 100j window-1 excess + did not recur on the quiet host, so it was the concurrent cargo job. +- G14, per-role CPU/RAM from 152 samples (stage100m/g14-resources.txt), mean CPU (p95): + - kafka1/2/3: 77/95/98 % (87/107/110); + - query_kn_1/2: 68/48 % (101/78); + - stream_kn_1/2: 22/7 %; projectors: 18/8 %; cores: 9/11/8 %; ingestors: 11/12 %; market_cache: 8 %; + - DL total about 5.0 vCPU mean; + - TS writer 124 % mean. + - Max RSS is well inside every limit: Query 761 MiB / 1.5 GiB, Kafka <= 1019 MiB / 1.5-2 GiB, cores <= 47 MiB / + 256 MiB, Stream <= 158 MiB / 512 MiB. +- G2 MARK, first 590 s trace window (13:18-13:28Z, writer recreated with trace env only; + scripts/p184_g2_mark_trace_transit.py): + - OKX new-event transit n 6385, p50 215, p95 321, max 399 ms: PASS (was p95 1426 / max 2326 before IDR-084). + - Binance n 2667, p50 430, p95 500.7, max 776 ms: max PASS, p95 0.7 ms over the 500 ms bound. + - Binance has a structural floor (min 370 ms) from the 250 ms MARK wake read delay, which was set to avoid racing + the DL latest view before IDR-084. + - One-variable tuning window started: Binance MARK wake read delay 250 -> 100 ms via the per-venue override + (parser bounds 0..500 ms). From e7b20281c2bf438ba1f4f587c6979283cd91c41a Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 14:32:15 +0000 Subject: [PATCH 121/147] docs(plan): G2 PASS after Binance read-delay tune, G13 41/41 DL cells on final digests --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 204cfe04..96fac556 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72029,3 +72029,28 @@ no build, CI or subagent job running. the DL latest view before IDR-084. - One-variable tuning window started: Binance MARK wake read delay 250 -> 100 ms via the per-venue override (parser bounds 0..500 ms). + +### G2 PASS after one-variable tune; G13 41/41 DL cells PASS on the final digests - 2026-10-10 (Claude) + +- G2 MARK, second 590 s trace window (13:30-13:40Z; writer env DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE=BINANCE=100, + parser bounds 0..500 ms, the only tuned variable; image unchanged): + - Binance new-event transit n 2671, p50 278.5, p95 339.3, max 708 ms: PASS (was p95 500.7); + - OKX n 6390, p50 212.3, p95 317.9, max 495 ms: PASS; + - writer READY in every guard sample. + The writer keeps BINANCE=100; the rollback compose is g2-tune/rollback-*.compose.json. +- G13 on DL bbfb3be + writer ed02064, runner run from a DL worktree at bbfb3be. The first attempt was refused by the + runner because the DL checkout HEAD differed from the release sha; receipts are kept. + - dr05-upstream 6/6 PASS (BINANCE, OKX, OKX_INVERSE GAP_REFUSAL and GENERATION_CHANGE_FENCED; receipt + p184-g13-20261010T134219Z). + - dr04-dr06-views 22/22 PASS, the inverse scope included (same receipt). + - dr05-transport 9/9 PASS (STALE_TYPED, DISCONNECT_RECONNECT recovered in 6765 ms, APPROVED_FALLBACK_AND_RETURN + with no V1; receipt p184-g13-20261010T142735Z). + - Root cause of the first failure: the TS harness probe read QUOTE only while the production policy requires + MARK for USD_M/SWAP; fixed in TS 7d7fd0e (probe uses execution_required_feeds; validator unchanged). + - Provenance: writer image ed02064 (deployed); harness script 7d7fd0e. + - dr08-isolated 4/4 PASS in QUALIFICATION mode, 5 restart samples (receipt p184-g13-20261010T142914Z). + - Deferred to the TS release because the TS stack (gateway, alpha, risk) is not running: + - dr01-deployed and dr01-wrong-account (the runner and the IDR-085 gateway fix exist); + - dr02-dr03-alpha; + - dr07-reference. + These cells stay OPEN; they are not claimed. From 59e725a9084362e201ed21846bc43480f0e8463b Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 15:17:22 +0000 Subject: [PATCH 122/147] release(v2.2.3): G1-G14 scorecard, final notes and preparation record (PREPARED_NOT_PUBLISHED) Scorecard on source bbfb3be, DL python e0252f4c / rust 64468c72, writer 34202753: PASS G5; FAIL G2 (TRADE BINANCE p95 539.1 ms in trace window 1); DEFERRED G13; NOT_MEASURED G1 G3 G4 G6 G7 G8 G9 G10 G11 G12 G14. SDK 2.2.0 wheel built twice from git archive bbfb3be, byte-identical, sha256 d8226f50. Adds scripts/p184_trace_feed_transit.py (G1/G2/G7 from writer delivery traces) with tests. No runtime change, no push, no tag. --- scripts/p184_trace_feed_transit.py | 181 ++ tests/test_p184_trace_feed_transit.py | 104 + .../evidence/releases/v2.2.3/RELEASE_NOTES.md | 279 ++- .../releases/v2.2.3/release-preparation.json | 586 +++++- .../evidence/releases/v2.2.3/scorecard.json | 1667 +++++++++++++++++ 5 files changed, 2723 insertions(+), 94 deletions(-) create mode 100644 scripts/p184_trace_feed_transit.py create mode 100644 tests/test_p184_trace_feed_transit.py create mode 100644 upgrade/evidence/releases/v2.2.3/scorecard.json diff --git a/scripts/p184_trace_feed_transit.py b/scripts/p184_trace_feed_transit.py new file mode 100644 index 00000000..9e43ca32 --- /dev/null +++ b/scripts/p184_trace_feed_transit.py @@ -0,0 +1,181 @@ +"""P18.4 phase D gates G1, G2 (TRADE/QUOTE) and G7 from the TS writer delivery trace files. + +Purpose: summarise, per feed and venue, the end-to-end time from the venue event to the TS execution Redis APPLY reply +(G1 BOOK_DELTA/BOOK_SNAPSHOT, G2 TRADE/QUOTE new-event transit) and the two TS consumer-path segments of G7, read from +the files that TS services/market_data/delivery_trace.py exports (delivery---[-pNN].json). The MARK +row of G2 stays with scripts/p184_g2_mark_trace_transit.py (component clocks, newest-component rule); BAR (G3) is not +traced by the writer, so this script reports it as absent instead of deriving it from another clock. + +Gates (TS upgrade/p184-phase-d-gates.md, frozen 2026-10-09): +* G1 BOOK_DELTA, BOOK_SNAPSHOT: p95 <= 500 ms, p99 <= 1000 ms (only when n >= 100), max < 2000 ms; +* G2 TRADE: p95 <= 500 ms, max < 1500 ms; QUOTE: p95 <= 500 ms, max < 1000 ms (new-event transit only); +* G7 per feed: SDK return -> writer entry p95 <= 100 ms; writer entry -> Redis APPLY reply p95 <= 50 ms. + +Definitions: +* venue clock = source_event_time_ns of the SDK stream envelope; only SDK_STREAM_RETURN records carry it. Records of + other bases (warmup, query, no basis) have no venue clock and are counted, never given a substitute clock; +* new-event transit = redis_reply_wall_ns - source_event_time_ns for the first APPLIED record of each event identity + (event_id_sha256; when absent, instrument + feed + source_event_time_ns). REFRESHED and DUPLICATE outcomes and a + repeated identity are the same state re-published (IDR-082 quiet republish) and are counted, not gated; +* transit split (attribution only, not gated), for gated records whose clocks are all present: venue_to_ingest = + envelope_received_at_ns - source_event_time_ns (ingestor receive; the envelope's received/normalized/published + stamps are one ingest clock, not a Kafka commit), ingest_to_sdk = sdk_receive_wall_ns - envelope_received_at_ns + (Data Layer pipeline + Stream + SDK), sdk_to_reply = redis_reply_wall_ns - sdk_receive_wall_ns (TS writer); +* G7 segments use the process monotonic clocks of the same worker: sdk_to_writer = writer_started_monotonic_ns - + sdk_receive_monotonic_ns; writer_to_redis = redis_reply_monotonic_ns - writer_started_monotonic_ns, over every record + with a Redis reply (any outcome). TRADE/QUOTE rows of one atomic APPLY batch share the batch start as writer entry. +Negative durations are counted as invalid, never clamped. Quantiles are nearest-rank as in the G2 MARK script. + +Boundary: read-only over the given files; no Redis, Docker or network access; it never enables tracing. Output is JSON +on stdout, or --out (created exclusively). +usage: p184_trace_feed_transit.py FILE_OR_GLOB [...] [--skip-seconds 40] [--from-ns N] [--to-ns N] [--out PATH] +""" +from __future__ import annotations + +import argparse +import collections +import glob +import json +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.p184_g2_mark_trace_transit import as_int, load, quantiles, venue_of # noqa: E402 + +TRANSIT_GATES = { + "BOOK_DELTA": {"gate": "G1", "p95_le_ms": 500.0, "p99_le_ms": 1000.0, "max_lt_ms": 2000.0}, + "BOOK_SNAPSHOT": {"gate": "G1", "p95_le_ms": 500.0, "p99_le_ms": 1000.0, "max_lt_ms": 2000.0}, + "TRADE": {"gate": "G2", "p95_le_ms": 500.0, "max_lt_ms": 1500.0}, + "QUOTE": {"gate": "G2", "p95_le_ms": 500.0, "max_lt_ms": 1000.0}, +} +G7_GATES = {"sdk_to_writer_ms": 100.0, "writer_to_redis_ms": 50.0} +STREAM_BASIS = "SDK_STREAM_RETURN" + + +def transit_verdict(block, gate): + if not block.get("n"): + return "NO_DATA" + inside = block["p95"] <= gate["p95_le_ms"] and block["max"] < gate["max_lt_ms"] + if "p99_le_ms" in gate and block.get("p99") is not None: + inside = inside and block["p99"] <= gate["p99_le_ms"] + return "INSIDE" if inside else "OUTSIDE" + + +def summarise(records, *, skip_seconds=40.0, from_ns=None, to_ns=None): + starts = [r["writer_started_wall_ns"] for r in records if as_int(r.get("writer_started_wall_ns"))] + if not starts: + return {"error": "no records with writer_started_wall_ns"} + lower = from_ns if from_ns is not None else min(starts) + int(skip_seconds * 1e9) + upper = to_ns if to_ns is not None else max(starts) + rows = [r for r in records if as_int(r.get("writer_started_wall_ns")) and lower <= r["writer_started_wall_ns"] <= upper] + rows.sort(key=lambda r: (r["writer_started_wall_ns"], r.get("redis_reply_wall_ns") or 0)) + outcomes = collections.Counter() + excluded = collections.Counter() + invalid = collections.Counter() + transit = collections.defaultdict(list) + split = collections.defaultdict(lambda: collections.defaultdict(list)) + segments = collections.defaultdict(lambda: collections.defaultdict(list)) + seen = set() + for r in rows: + feed = r.get("feed", "?") + venue = venue_of(r.get("canonical_instrument_id", "?")) + outcomes[(feed, venue, r.get("outcome"), r.get("sdk_receive_basis"))] += 1 + receive, started, reply = (as_int(r.get(k)) for k in + ("sdk_receive_monotonic_ns", "writer_started_monotonic_ns", "redis_reply_monotonic_ns")) + if started is not None and receive is not None: + value = (started - receive) / 1e6 + if value < 0: + invalid[(feed, venue, "negative_sdk_to_writer")] += 1 + else: + segments[feed]["sdk_to_writer_ms"].append(value) + segments[f"{feed}|{venue}"]["sdk_to_writer_ms"].append(value) + if started is not None and reply is not None: + value = (reply - started) / 1e6 + if value < 0: + invalid[(feed, venue, "negative_writer_to_redis")] += 1 + else: + segments[feed]["writer_to_redis_ms"].append(value) + segments[f"{feed}|{venue}"]["writer_to_redis_ms"].append(value) + if feed not in TRANSIT_GATES: + continue + if r.get("outcome") != "APPLIED": + excluded[(feed, venue, f"outcome_{r.get('outcome')}")] += 1 + continue + event_time = as_int(r.get("source_event_time_ns")) + reply_wall = as_int(r.get("redis_reply_wall_ns")) + if r.get("sdk_receive_basis") != STREAM_BASIS or event_time is None: + excluded[(feed, venue, f"no_venue_clock_{r.get('sdk_receive_basis')}")] += 1 + continue + if reply_wall is None: + invalid[(feed, venue, "missing_reply_clock")] += 1 + continue + digest = r.get("event_id_sha256") + identity = (digest if isinstance(digest, str) and len(digest) == 64 + else (r.get("canonical_instrument_id"), feed, event_time)) + if (feed, identity) in seen: + excluded[(feed, venue, "repeated_event_identity")] += 1 + continue + seen.add((feed, identity)) + value = (reply_wall - event_time) / 1e6 + if value < 0: + invalid[(feed, venue, "negative_transit")] += 1 + continue + transit[f"{feed}|{venue}"].append(value) + ingest, sdk_wall = as_int(r.get("envelope_received_at_ns")), as_int(r.get("sdk_receive_wall_ns")) + if ingest is not None and sdk_wall is not None and event_time <= ingest <= sdk_wall <= reply_wall: + parts = split[f"{feed}|{venue}"] + parts["venue_to_ingest_ms"].append((ingest - event_time) / 1e6) + parts["ingest_to_sdk_ms"].append((sdk_wall - ingest) / 1e6) + parts["sdk_to_reply_ms"].append((reply_wall - sdk_wall) / 1e6) + else: + invalid[(feed, venue, "split_unavailable")] += 1 + result = {"window": {"from_ns": lower, "to_ns": upper, "seconds": round((upper - lower) / 1e9, 1)}, + "gates": {"transit": TRANSIT_GATES, "g7_p95_le_ms": G7_GATES}, "transit": {}, "g7": {}} + for key in sorted(transit): + feed = key.split("|", 1)[0] + block = quantiles(transit[key]) + block["verdict"] = transit_verdict(block, TRANSIT_GATES[feed]) + block["split"] = {name: quantiles(values) for name, values in sorted(split[key].items())} + result["transit"][key] = block + for key in sorted(segments): + block = {name: quantiles(values) for name, values in sorted(segments[key].items())} + if "|" not in key: + block["verdict"] = {name: ("NO_DATA" if not block.get(name, {"n": 0})["n"] else + "INSIDE" if block[name]["p95"] <= limit else "OUTSIDE") + for name, limit in G7_GATES.items()} + result["g7"][key] = block + result["bar_records"] = sum(1 for r in rows if r.get("feed") == "BAR") + result["outcomes"] = {"|".join(str(p) for p in k): n for k, n in sorted(outcomes.items(), key=str)} + result["excluded_from_transit"] = {"|".join(k): n for k, n in sorted(excluded.items())} + result["invalid"] = {"|".join(k): n for k, n in sorted(invalid.items())} + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("inputs", nargs="+", help="trace files or quoted globs") + parser.add_argument("--skip-seconds", type=float, default=40.0, + help="drop the first N s after the first record (writer start-up); ignored with --from-ns") + parser.add_argument("--from-ns", type=int) + parser.add_argument("--to-ns", type=int) + parser.add_argument("--out") + args = parser.parse_args(argv) + paths = sorted({path for pattern in args.inputs for path in (glob.glob(pattern) or [pattern])}) + files, records = load(paths) + report = {"schema": "qdl.p184.trace-feed-transit.v1", "files": len(files), "records": len(records), + "dropped_at_export_total": sum(f["dropped_at_export"] or 0 for f in files), + "stop_reasons": dict(collections.Counter(f["stop_reason"] for f in files)), + **summarise(records, skip_seconds=args.skip_seconds, from_ns=args.from_ns, to_ns=args.to_ns)} + text = json.dumps(report, indent=1, sort_keys=True) + if args.out: + with open(args.out, "x") as handle: + handle.write(text + "\n") + print(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_p184_trace_feed_transit.py b/tests/test_p184_trace_feed_transit.py new file mode 100644 index 00000000..1b253779 --- /dev/null +++ b/tests/test_p184_trace_feed_transit.py @@ -0,0 +1,104 @@ +"""Offline tests for scripts/p184_trace_feed_transit.py (synthetic trace rows, test-only provenance).""" + +import contextlib +import io +import json +import os +import tempfile +import unittest + +from scripts import p184_trace_feed_transit as transit + +T0 = 1_791_600_000_000_000_000 +MONO0 = 2_000_000_000_000_000 +MS = 1_000_000 + + +def row(feed, instrument, start_ms, event_ms, reply_ms, *, receive_ms=None, outcome="APPLIED", + basis="SDK_STREAM_RETURN", event_id=None): + receive_ms = start_ms - 5 if receive_ms is None else receive_ms + return {"feed": feed, "canonical_instrument_id": instrument, "outcome": outcome, "sdk_receive_basis": basis, + "writer_started_wall_ns": T0 + start_ms * MS, "redis_reply_wall_ns": T0 + reply_ms * MS, + "source_event_time_ns": T0 + event_ms * MS if basis == "SDK_STREAM_RETURN" else "NOT_MEASURED", + "writer_started_monotonic_ns": MONO0 + start_ms * MS, "redis_reply_monotonic_ns": MONO0 + reply_ms * MS, + "sdk_receive_monotonic_ns": MONO0 + receive_ms * MS, + "event_id_sha256": event_id or "NOT_MEASURED", + "envelope_received_at_ns": T0 + (event_ms + 20) * MS, "sdk_receive_wall_ns": T0 + receive_ms * MS} + + +class TransitTests(unittest.TestCase): + def test_first_apply_per_event_identity_is_gated_repeats_are_counted(self): + rows = [row("TRADE", "BINANCE:USD_M:BTCUSDT", 100, 0, 120, event_id="a" * 64), + row("TRADE", "BINANCE:USD_M:BTCUSDT", 900, 0, 950, event_id="a" * 64), # same trade again + row("TRADE", "BINANCE:USD_M:BTCUSDT", 1000, 900, 1010, outcome="REFRESHED"), + row("TRADE", "BINANCE:USD_M:BTCUSDT", 1100, 1000, 1300, event_id="b" * 64)] + result = transit.summarise(rows, skip_seconds=0) + block = result["transit"]["TRADE|BINANCE"] + self.assertEqual(block["n"], 2) + self.assertEqual(block["max"], 300.0) + self.assertEqual(result["excluded_from_transit"]["TRADE|BINANCE|repeated_event_identity"], 1) + self.assertEqual(result["excluded_from_transit"]["TRADE|BINANCE|outcome_REFRESHED"], 1) + split = block["split"] + self.assertEqual(split["venue_to_ingest_ms"]["max"], 20.0) + self.assertEqual(split["ingest_to_sdk_ms"]["max"], 75.0) # ingest event+20, SDK return start-5 + self.assertEqual(split["sdk_to_reply_ms"]["max"], 205.0) + + def test_records_without_a_venue_clock_are_counted_not_substituted(self): + rows = [row("BOOK_DELTA", "OKX:SWAP:BTC-USDT-SWAP", 100, 0, 150, basis="SDK_WARMUP_RETURN"), + row("BOOK_DELTA", "OKX:SWAP:BTC-USDT-SWAP", 200, 100, 260, event_id="c" * 64)] + result = transit.summarise(rows, skip_seconds=0) + self.assertEqual(result["transit"]["BOOK_DELTA|OKX"]["n"], 1) + self.assertEqual(result["excluded_from_transit"]["BOOK_DELTA|OKX|no_venue_clock_SDK_WARMUP_RETURN"], 1) + + def test_negative_transit_is_invalid_never_clamped(self): + rows = [row("QUOTE", "OKX:SWAP:ETH-USDT-SWAP", 100, 500, 120, event_id="d" * 64)] + result = transit.summarise(rows, skip_seconds=0) + self.assertNotIn("QUOTE|OKX", result["transit"]) + self.assertEqual(result["invalid"]["QUOTE|OKX|negative_transit"], 1) + + def test_verdicts_follow_the_frozen_rows(self): + quote = [row("QUOTE", "OKX:SWAP:ETH-USDT-SWAP", i * 10, i * 10, i * 10 + 200, event_id=f"{i:064x}") + for i in range(1, 120)] + quote.append(row("QUOTE", "OKX:SWAP:ETH-USDT-SWAP", 5000, 5000, 6000, event_id="e" * 64)) # max 1000 ms + result = transit.summarise(quote, skip_seconds=0) + self.assertEqual(result["transit"]["QUOTE|OKX"]["verdict"], "OUTSIDE") # QUOTE max must stay < 1000 + trade = [dict(r, feed="TRADE") for r in quote] + self.assertEqual(transit.summarise(trade, skip_seconds=0)["transit"]["TRADE|OKX"]["verdict"], "INSIDE") + book = [row("BOOK_SNAPSHOT", "BINANCE:USD_M:BTCUSDT", i * 10, i * 10, i * 10 + (1200 if i > 115 else 100), + event_id=f"{i:064x}") for i in range(1, 120)] + block = transit.summarise(book, skip_seconds=0)["transit"]["BOOK_SNAPSHOT|BINANCE"] + self.assertLessEqual(block["p95"], 500.0) + self.assertGreater(block["p99"], 1000.0) + self.assertEqual(block["verdict"], "OUTSIDE") # p99 bound applies when n >= 100 + + def test_g7_segments_use_monotonic_clocks_for_every_reply(self): + rows = [row("MARK_INDEX_PRICE", "BINANCE:USD_M:BTCUSDT", 100, 0, 104, receive_ms=99, + basis="ADAPTER_REFERENCE_RETURN"), + row("QUOTE", "BINANCE:USD_M:BTCUSDT", 200, 150, 290, receive_ms=80, outcome="DUPLICATE")] + result = transit.summarise(rows, skip_seconds=0) + self.assertEqual(result["g7"]["MARK_INDEX_PRICE"]["sdk_to_writer_ms"]["max"], 1.0) + self.assertEqual(result["g7"]["MARK_INDEX_PRICE"]["writer_to_redis_ms"]["max"], 4.0) + self.assertEqual(result["g7"]["QUOTE"]["sdk_to_writer_ms"]["max"], 120.0) + self.assertEqual(result["g7"]["QUOTE"]["verdict"], + {"sdk_to_writer_ms": "OUTSIDE", "writer_to_redis_ms": "OUTSIDE"}) + self.assertEqual(result["bar_records"], 0) + + def test_cli_reads_files_and_writes_out_exclusively(self): + with tempfile.TemporaryDirectory() as directory: + trace = os.path.join(directory, "delivery-w-1-1.json") + with open(trace, "w") as handle: + json.dump({"worker": "w", "part": 0, "stop_reason": "EXPIRY", "recorded": 1, "dropped_at_export": 0, + "records": [row("TRADE", "OKX:SWAP:BTC-USDT-SWAP", 100, 50, 130, event_id="f" * 64)]}, + handle) + out = os.path.join(directory, "out.json") + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(transit.main([trace, "--skip-seconds", "0", "--out", out]), 0) + with open(out) as handle: + report = json.load(handle) + self.assertEqual(report["transit"]["TRADE|OKX"]["max"], 80.0) + with self.assertRaises(FileExistsError): + transit.main([trace, "--skip-seconds", "0", "--out", out]) + + +if __name__ == "__main__": + unittest.main() diff --git a/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md b/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md index 8a9e578e..b40de8a7 100644 --- a/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md +++ b/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md @@ -1,10 +1,279 @@ -# v2.2.3 Preparation: Not Published +# Quant Data Layer v2.2.3 + +Status: `PREPARED_NOT_PUBLISHED`. This is not a published release, a Git tag or an owner publication decision. +Publication needs the owner to push and tag. Remote CI must also run on the final head. The published stable +release is still v2.2.2. + +Scorecard: [scorecard.json](scorecard.json). Inventory, hashes and the SDK build: [release-preparation.json](release-preparation.json). + +## Scope + +This release closes the P18.4 phase D Data Layer work on the Kafka-native stack. It contains: +- one-commit images for every application role (IDR-039); +- execution-view correctness fixes (IDR-062, IDR-040, IDR-084, Astra R1/R3); +- hot-path contention fixes (Astra R2/R3), the OKX session fixes and the ingestor liveness knob; +- the G5 warmup fixes; +- additive public fields: `event_id` (IDR-082) and `proof_observed_at_ns` (IDR-040 fix 2), both in SDK 2.2.0. + +Public endpoints keep their versions. Every change is additive, and readers on SDK 2.1.0 ignore the new fields. + +This is not a Trading System order, accounting or alpha certification. It is not HA or independent-failure-domain +evidence, because every run used the same host. + +## Source And Images + +- Release source: `bbfb3be6b3f1b7f83b44022e77439daeb4999a65` on `fix/p184-dl-closure`. Its code tree equals + 2c84e2f plus the plan journal. +- Later commits on the branch are journal or tooling only: + - 259aeb9 is the local CI runner; + - 163191c and e7b2028 are the plan journal; + - this commit adds the trace analysis script, its test and the release evidence. + - None of them is in an image. + +Images were checked with `docker image inspect`. Each image's revision label equals the release source. The +version label reads `2.2.3-rc7d`, because a tag never changes a digest. + +| Image | Digest | Tag | Roles | +| --- | --- | --- | --- | +| Python | `sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a` | `qdl-v2-python:2.2.3-rc7d-bbfb3be` | query_kn_1/2, binance_bar_edge | +| Rust | `sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7` | `qdl-v2-rust:2.2.3-rc7d-bbfb3be` | stream_kn_1/2, market_projector_1/2, rust_core x3, ingestor_okx_swap, ingestor_binance_usdm | + +All 12 application roles run these two digests: restart count 0, OOM false (`docker inspect`, 2026-10-10 15:0xZ). +The following were not changed by this release: +- kafka1-3 (`sha256:9516fb7634ba`); +- market_cache and stable_redis (`sha256:dfa18828cbc0`); +- the one-shots stable_state_init and stable_tls_init (`sha256:bd0163fd76b0`). + +TS writer `market_data_service_p184w`: +- image `sha256:34202753bbe8b74d50e45afed1cf2e7b400141324c550168462b30db220edb31`, TS `ed02064`, SDK 2.1.0; +- env `DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE=BINANCE=100` is the one tuned variable of G2; +- the `TS_DELIVERY_TRACE_*` env from the G2 windows is still set but expired (until 1791639603470577388). The trace + refuses to rearm after its expiry. + +Ingestor configs come from `~/.local/state/qdl-v2/releases/v2.2.3-rc7-ingestor-liveness/`. They are mounted +read-only at `/runtime/config.json` (checked with `docker inspect`) and read through a `--network none` container: + +| Ingestor | Config revision | Liveness ping/dead | sha256 | +| --- | --- | --- | --- | +| OKX | 20 | 1500/4000 ms | `eb982fac...` | +| Binance | 19 | 1500/4000 ms | `5833bc4e...` | + +Neither config contains `book_liveness_proof_ms`, so IDR-040 fix 2 is OFF. + +## Gate Scorecard (G1-G14) + +Gates come from the frozen TS table `upgrade/p184-phase-d-gates.md` (TS 31cf663, sha256 `c67bf69b...`) and the +frozen budget `config/v2/p184-stage100-budget.json` (sha256 `ff85335b...`). + +Scoring rule: a PASS needs the table's measurement rule, which is steady state for at least 30 min at the stage-100 +workload plus a burst window. A measurement outside that rule can FAIL a gate but cannot PASS it. "Observed" gives +such a value against the threshold. A NOT_MEASURED is never upgraded. + +Summary: +- PASS: G5. +- FAIL: G2. +- DEFERRED: G13. +- NOT_MEASURED: G1, G3, G4, G6, G7, G8, G9, G10, G11, G12, G14. + +Paths under `~/.local/state/`; sha256 prefixes, full hashes in scorecard.json. + +| Gate | Rule | Measured | Source (sha256) | Verdict | +| --- | --- | --- | --- | --- | +| G1 BOOK e2e to TS cache | p95 <= 500, p99 <= 1000, max < 2000 ms | Window 2: DELTA B/O p95 126.5/130.0, p99 163.0/170.5, max 510.9/501.8 (n 24664/26105); SNAPSHOT B/O p95 152.4/155.5, max 294.0/353.1. Window 1 similar (max 528.1). Observed INSIDE | `qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json` (e9e82309), `...-1791638906937534999.json` (d8d0d6c3) | NOT_MEASURED: writer-only 2 x 535 s, no stage-100 load | +| G2 TRADE/QUOTE/MARK new-event transit | TRADE p95 <= 500, max < 1500; QUOTE/MARK p95 <= 500, max < 1000 ms | Window 1, final images: **TRADE BINANCE p95 539.1, max 1078.7 (n 21983)**. Window 2 (deployed config): TRADE B/O p95 269.7/158.9, max 623.9/394.0; QUOTE B/O p95 152.2/154.0, max 516.9/513.4; MARK B/O p95 339.3/317.9, max 708.0/495.2 | same two files; `p184-g2-20261010/g2-mark-*.json` (292256e8, 3da349c4); stage100m receipt.json (9dfce5d9) | **FAIL** | +| G3 BAR e2e from close | p95 <= 5 s, max < 60 s | The trace has no BAR records. Related, alpha SDK streams (not the TS cache): BAR 1m close -> usable max 3095 ms Binance, 1511 ms OKX | stage100m receipt.json (9dfce5d9) | NOT_MEASURED | +| G4 venue -> canonical | candidate p95 <= baseline p95 x 1.2 per row | 16/17 frozen rows PASS in all 4 windows (e.g. trade B/O 105.7/110.0 vs 269.3/257.4; mark OKX 1453.1 vs 1586.6). bar BINANCE 30m: no close in any window. 5 observed rows have no baseline. Inverse is pooled into OKX | `p184-d3-capacity-20261009/g4m-{1..4}-summary.json` (2c98891f, c6f321b9, b349e8c4, 6955061d); budget (ff85335b) | NOT_MEASURED: missing coverage blocks the gate | +| G5 request latency | certified p95/p99 per class and venue; COLD_2500 <= 6265.1, COLD_5000 <= 15267.0 ms | All 10 class/venue rows PASS (worst p95 89.4 OKX BAR_LATEST, p99 257.6 OKX L2). Cold B/O 2500 3124/3727 ms, 5000 5299/6178 ms (n=1 each). 1800 s stage 100m, 32/32 driver gates | `p184-d3-capacity-20261009/stage100m/gates.json` (c65db308), receipt.json (9dfce5d9) | **PASS** | +| G6 durable-view event age + quality | per-binding D3 age thresholds; zero unsafe eligibility | Not recorded per binding. Related: 182709 requests with 0 failures or refusals; 180 streams with 0 errors | stage100m gates.json (c65db308) | NOT_MEASURED | +| G7 TS consumer path | SDK -> writer p95 <= 100; writer -> Redis p95 <= 50 ms per feed | Window 2, SDK -> writer p95: BOOK_DELTA 29.5, BOOK_SNAPSHOT 37.8, QUOTE 28.6, TRADE 41.5, MARK 0.6. Writer -> Redis p95: 8.9, 13.3, 4.0, 26.2, 6.8. Window 1 TRADE writer -> Redis p95 36.9. Observed INSIDE | trace-transit files (e9e82309, d8d0d6c3) | NOT_MEASURED: writer-only windows | +| G8 TS execution Redis | commandstats <= 60 % core per 10 min; SLOWLOG 0; PING p99 <= 5 ms | One read-only 600 s window, 15:06-15:16Z, writer-only traffic: commandstats 0.065 core (EVALSHA 23.5 s, EVAL 9.7 s); SLOWLOG new entries 0 (threshold 10 ms); PING p50/p95/p99/max 0.318/0.528/0.942/5.279 ms (n 5971); 0 MONITOR clients. Observed INSIDE | `qdl-v2/releases/v2.2.3/g8-redis/g8-window-600s.json` (fbc269fd) | NOT_MEASURED: not under full mixed traffic | +| G9 readiness | demanded == session-ready; 0 data-path eligibility losses; v1 fallback 0 | ts:ready 154/154 samples, 0 unreadable, READY 64/64; v1_fallback_calls 0; writer MARK errors 0 in 32/32 min. Eligibility losses by cause: not counted | stage100m gates.json (c65db308), `guard-s100m.log` (cef88fa8), `mark-watch-s100m.log` (f905c281) | NOT_MEASURED | +| G10 TRADE integrity | 0 gaps, 0 dup applied, 0 replay-expired, 0 overflow | No production counters read on the final digests. DR08 isolated bounded queue closes at 1000 events with typed replay | g13 dr08 evidence.jsonl (a7bf8d01) | NOT_MEASURED | +| G11 writer restart recovery | first valid apply <= 30 s; caught up <= 60 s | Four timestamps not recorded for the writer. DR08 DL consumer restart -> first event 5.07-5.14 s, n=5 (not the TS writer) | g13 dr08 evidence.jsonl (a7bf8d01) | NOT_MEASURED | +| G12 broker/projector/Stream recovery | caught up <= 600 s, then G1-G3 | The isolated PASS (broker 0.202 s, projector 0.256 s, Stream 2.355 s; BOOK only) ran the predecessor core/projector/Stream/Query/writer images. Not inheritable, see Known Limits | `p184-g12-20261010/evidence/verdict.json` (7e98572c) | NOT_MEASURED on final digests | +| G13 DR01-DR08 | every cell once on the final digests | 41/41 DL-side cells harness-ok (2 of them NOT_OBSERVED quiet cells). 4 steps deferred to TS | `p184-g13-20261010T134219Z/summary.json` (b64f35e1), `...T142735Z/summary.json` (24d659c5), `...T142914Z/summary.json` (eb6ae4e2) | DEFERRED | +| G14 resources | CPU/RAM per role at stage 100, with headroom | 16 roles from 152 samples. Mean CPU: kafka 77/95/98 %, query 68/48 %, writer 124 %; DL total about 5.0 vCPU. Max RSS inside every limit. binance_bar_edge and stable_redis not sampled | `stage100m/g14-resources.txt` (6b1fdcab) | NOT_MEASURED: role coverage incomplete | + +Findings behind the verdicts: +- **G2 FAIL.** Both windows ran the final DL and writer images. The only difference between them is the Binance MARK + read delay (250 ms in window 1), which does not act on TRADE. + - The TRADE BINANCE excess is in the ingest -> SDK return segment: p95 434.5 ms, max 1018 ms. + - The other two segments are small: venue -> ingest p95 36.9 ms; writer, SDK -> Redis reply, p95 85.0 ms. + - G4 puts venue -> canonical visibility p95 at about 105 ms, so the tail sits between canonical visibility and + the SDK return. That is Stream dispatch/queue or SDK receive; not yet attributed. + - Under stage-100 load the alpha SDK streams show the same shape: TRADE BINANCE per-stream p95 up to 872.5 ms, max + 1402 ms. + - Not a threshold question: the gate is frozen. +- **G1/G7** are inside both windows by a wide margin, but only writer-only traffic was traced. The trace caps one + activation at 600 s, so a 30-min stage-100 measurement needs several activations during the run. +- G4 bar latency for OKX is measured from bar open (the OKX source time is the open). The same metric was used for + the baseline. + +## Items Closed (commit) + +| Item | Commit(s) | State | +| --- | --- | --- | +| IDR-039 one-commit images for every application role | rc7d rollout of bbfb3be | runtime: 12/12 roles on bbfb3be | +| IDR-062 BOOK_SNAPSHOT on top-of-book change | 43bbcf0, 63238bf | in images | +| IDR-040 fix 1 REST refresh ahead of stream (`REST_AHEAD_OF_STREAM`) | 4eb08dc | in images | +| IDR-040 fix 2 in-band quiet-book proof (`proof_observed_at_ns`) | 1a2daf1 | in images, ingestor flag OFF | +| IDR-002 binding V3 sealed request policy; IDR-029 quota log | 4eb08dc | in images | +| D2 tolerant SDK response models (SDK 2.1.0) | eb7c649 | SDK | +| IDR-075 OKX inverse BTC-USD-SWAP Paper entitlement | 704552d, 76db472; D1 rev11 applied on readers (plan 8c90ddc) | runtime: TS manifest rev 11 | +| IDR-076 coherence acceptance on the TS product path | 1cd24e3 (decision) | TS-owned | +| IDR-082 canonical `event_id` on the public view | 616342c | in images, SDK 2.2.0 | +| IDR-084 execution MARK takes the newer committed hot record | 922a6ec; bd74faa; rc4 regression fix ec89a31; real-ordering tests 4e19966 | in images | +| Astra R1 hard hot refusals not downgraded; R3 one MARK hot budget; R6 bounded refusal lines | 57881a2 | in images | +| Astra R2 Stream hot lookup per key partition | fdd379e | in images | +| Astra R4 absolute inbound deadline (silent socket) | d4f5ec9; liveness knob 0f30e17 | in images, config 1500/4000 | +| Astra R5 G5 same-replica queue | 5c81c25 (SDK random start), 65e0757/76950c8 (cold read), 7f974c3 (attribution), 47bec04 (replica redirect) | in images, G5 PASS | +| Astra R7 release discipline | this record | see Known Limits | +| Codex review: TS readiness sampled on the configured writer; run-scoped supervisor | 713aa90 | tooling | +| Codex review: rc5 ordering test | 4e19966 | tests | +| Codex review: no funding settlement allowance before measurement | 530a406 (reverts the allowance of a2184f4; keeps item diagnostics and the cache re-read) | in images | +| OKX make-before-break on 64008, session lineage | f861663 | in images | +| OKX WebSocket port 8443 -> 443; plan load accepts old URL | 6674d0b, f69353b | in images, config on 443 | +| Stream terminal errors across stalled handoffs; per-subscription counters | 0b374bc, b6a0119 | in images | +| Cursor keeps standard-base64 Kafka topic UUID | 47d906e | in images | +| DR05 invalid-book fence | d68fb2f | in images | +| CI convergence (cargo fmt, DL-only test skips, vnstock opt-in) | d6fb34c, 2c84e2f, 54460ac | in images | +| SDK 2.2.0 release (start_index, event_id, proof_observed_at_ns, item diagnostics) | 359c9ee, a2184f4 | wheel below | + +## SDK 2.2.0 Wheel + +Built by `scripts/build_qdl_sdk_release.py`. The builder image was `sha256:e0252f4c...`, run with +`--network none --cpus 1` as a non-root user. The source was a `git archive` of bbfb3be, extracted separately for +each build. + +| Field | Value | +| --- | --- | +| wheel | `qdl_sdk-2.2.0-py3-none-any.whl` | +| wheel sha256 | `d8226f509bce4ae408cacfa30e12f6160726e4040751d2e4f7379d3dc71c3997` | +| reproducible | two builds byte-identical (`cmp`) | +| sdk_source_digest | `63e230312cec966b51bfa20aedd3e73c51c838c1a52b778c24574f61b7399488` | +| generated_contract_digest | `ae77dc42074fd7cc896a46bde3282837a51b667acee23c4957d37565690856e9` | + +All 21 wheel files are byte-identical to the deployed Python image (`/app/qdl_sdk`, `/app/generated/python/qdl`). +The artifacts are under `~/.local/state/qdl-v2/releases/v2.2.3/sdk-build-{1,2}/`, outside Git. + +## CI + +Local exact-source CI (`scripts/p184_local_ci.py`, archive of bbfb3be), first run: +- summary sha256 `23a49251...`; +- contract-tests, sdk-python310, kn-native-integration and consolidated-source-preflight PASS; +- unit-tests FAIL on one step only, the trivy image scan, because of a runner flag bug fixed in 259aeb9 (boolean + `ignore-unfixed`, repo `.trivyignore` mount). The secret scan and SBOM steps after it were skipped. + +Rerun of the unit-tests job with the fixed runner: +- summary sha256 `07df25eb...`; +- 21 steps PASS, including the unit tests, the trivy image scan, the secret/misconfiguration scan and the SBOM + rehearsal; +- 1 SKIPPED: "Prepare non-root runtime bind mounts" needs sudo; host preparation is reported, never escalated; +- receipt status `INCOMPLETE` only because of that skip. + +Remote GitHub CI did NOT run, because the DL repository is not pushed. Publication requires the required jobs on +the final head. + +## Runtime Changes Performed (2026-10-08 .. 2026-10-10) + +Each change was owner-approved and recorded in `DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md`. Each role was +recreated one at a time. + +| Change | Rollback reference | +| --- | --- | +| Bar edge to `514506122111` (2026-10-08, see appendix) | prior Codex evidence dir | +| D1 rev11 on readers: TS Paper manifest rev 11, bundle f04ba6dc, 64 products incl. inverse | plan D1 entries | +| D3 window B: alpha rev15 quotas (4950 RPM / 300 streams); Stream `QDL_KN_MAX_SUBSCRIPTIONS` 384 -> 1024 | `p184-d3-capacity-20261009/readers/*-rollback.compose.json` | +| Query/Stream release candidates rc3-rc7 (rc4 rolled back to rc3 after a regression) | plan entries per candidate | +| OKX ingestor config to port 443 (config_revision 20) | backup `ingestor_okx_swap.json.pre-okx-port443-1791619322166921472` | +| rc7c all roles + ingestor liveness configs | as below, one step back | +| **rc7d (bbfb3be), all 12 application roles, 12:02-12:08Z 2026-10-10**, writer READY 64/64 after every step | `p184-d3-capacity-20261009/rollout-rc6/-rollback.compose.json` -> rc7c python `sha256:55e493786748...` / rust `sha256:206ae872a5fe...` (54460ac); log `rollout-rc7d.log` | +| TS writer image 15524c0b (9b86ed3) -> 34202753 (ed02064) | previous image `sha256:15524c0bea8c...`, same env/mounts | +| TS writer `DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE=BINANCE=100` (13:30Z) | `trading-system/p184-rev11-window/g2-tune/rollback-20261010T133013Z.compose.json` (a7299021) | + +Rollback runs in reverse role order. Kafka, Redis, realm dirs and offsets need no data migration. Rollback is for +failure only. + +## Known Limits + +- **G12 is isolated and not inheritable.** The run was same-host, single projector/Stream replica, TEST_ONLY BOOK + at 4 Hz, with G2/G3 NOT_MEASURED. It ran predecessor images: + - core 658a9570 (0e00c81); + - projector a696cfba and Stream 1f13408f (c8015ca); + - Query 672503b7 (76950c8); + - writer 15524c0b. + + Code changed since on every recovery path it exercised (`git diff --shortstat bbfb3be`): + - core +959/-30 (IDR-062, IDR-040 fix 2, the invalid-book fence); + - Stream +1020/-147 (Astra R2, terminal errors across stalled handoffs); + - projector +39 (invalid-book fence); + - Query. + + G12 on the final digests needs a rerun with the image set changed. In `scripts/p184_g12_isolated_fault.py` it is + a module constant. +- **IDR-040 fix 2 is OFF.** The ingestor flag `book_liveness_proof_ms` is unset and the renderer does not emit it. + A quiet Binance book stays stale, never falsely fresh. Ping cadence and marker volume are not measured. +- **Catalog revision split (C.19).** The repo and image catalog is revision 11, which the bar edge checkpoint reads + from the image. Query reads the runtime-mounted revision 12 (`releases/v2.2.0-02cd827/runtime/stable-source-bindings.yaml`, + sha256 `fc3a1e1e...`, includes BTC-USD-SWAP). This split is deliberate. Unifying it changes the acquisition plan + revision and needs its own packet. +- **Funding settlement allowance: PENDING evidence.** + - The stage-100g PARTIAL_RESULT (OKX SOL funding 7.1 s after an 8 h settlement) has a probable root cause: age + measured from the newest fundingTime before OKX publishes the due row. + - The 60 s allowance was reverted (530a406); the release ships without it. + - A read-only publication-latency poll (15:58-16:04Z, `~/.local/state/qdl-v2/funding-publication-20261010/`) is + PENDING. The coordinator appends its result. + - Until then a funding read right after settlement can be refused DATA_STALE. That refusal is typed, never a + stale success. +- **Cells deferred to the TS release:** dr01-deployed, dr01-wrong-account (IDR-085 fix in TS be21d49), + dr02-dr03-alpha and dr07-reference. Gateway, alpha and risk are not running. These cells stay OPEN. +- **G4 rows not measured:** + - bar BINANCE 30m (no close in the windows); + - BAR BINANCE 5m/1h and OKX 5m/15m/1h have no frozen baseline; + - OKX inverse has no separate cells. +- **G13 quiet cells:** DR05.QUIET_TYPED BINANCE/OKX are ok by the cell rule but NOT_OBSERVED, because no book was + quiet in the 300 s window. They are not a positive quiet-state proof. +- **G2 trace coverage:** the writer trace dropped 158 and 87 records at export, and records without a venue clock + are counted and excluded: + - TRADE/QUOTE from Query/warmup reads; + - the first records after a resubscribe. +- Historical failures stay recorded in the plan: stage 100/100b/100c/100e/100f/100g/100j, the rc4 regression, the + rc7 Query crash and the first G13 attempts. None is erased by a later PASS. +- Same host throughout: no HA, no independent failure domain, no mainnet or funded claim. + +## Consumer Hand-Off + +- **Alpha callers** mint JWTs for manifest revision **15** (alpha.binance/okx.paper.stable rev 15, 4950 RPM / 300 + streams). The readers run rev 15, so an older JWT revision gets 401. +- **Trading System**: + - Paper manifest/JWT revision 11 (64 products incl. OKX BTC-USD-SWAP) is live. + - Adopt SDK **2.2.0** (wheel sha256 `d8226f50...`). The writer runs 2.1.0 today, which ignores the new fields. + - Phase E uses `event_id` (IDR-082) for trade identity. + - The IDR-085 gateway fix (TS be21d49, scope lookup after the API key) ships with the TS release, which closes + the deferred DR01 cells. + - The writer keeps `BINANCE=100` for the MARK wake read delay. +- The G2 TRADE tail between canonical visibility and SDK return is open. It must be attributed before G2 can PASS. + +## Publication Requires + +1. The owner resolves the G2 FAIL and the NOT_MEASURED gates, or explicitly accepts them as release limits. This + record does not accept them. +2. The funding publication-latency result is appended. +3. Push, remote CI (contract-tests, sdk-python310, unit-tests, kn-native-integration) on the final head, and merge + to dev and main. +4. An annotated tag `v2.2.3` that links the final CI runs and the hashes of scorecard.json and this file. + +No push, merge or tag was performed. + +## Appendix: Preparation History + +### 2026-10-08 HOLD draft (verbatim, superseded by the record above) Status: `PREPARED_RELEASE_HOLD_NOT_PUBLISHED`. This record is not a certificate, Git tag, deploy authorization or proof that the feature HEAD runs on production. The published stable release remains v2.2.2. -## Inherited Runtime Change +#### Inherited Runtime Change Owner-approved bar-edge-only recreate at 2026-10-08 02:17:33 UTC: `f7351c3b` -> `514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9` @@ -18,7 +287,7 @@ bounded product read path (IDR-076). Prior paired core trial remains historical FAIL/rollback; core candidate `0d8da150` is explicitly excluded from this release proposal. No new core is needed merely to pass the revised coherence gate. -## Busy-Window Evidence +#### Busy-Window Evidence 2026-10-08 03:21:03-03:26:03 UTC, 17 DL roles, 31 samples/30 intervals over 299.992s: mean **4.882532**, p95 **5.564052**, max **5.870669 vCPU**. The existing @@ -37,7 +306,7 @@ records. This does not establish data loss or an endless loop, but it precludes claiming fully qualified burst recovery. One TS Binance BTC MARK timeout also appears in that window. Passive Stream counters remain unchanged. -## IDR-075 Preparation +#### IDR-075 Preparation Pure preparation reuses deployed catalog identities and preserves all60 existing V2 products, all prior manifest requirements, quotas and fail-closed policies. @@ -51,7 +320,7 @@ binding hash and environment projection are in `release-preparation.json`. Runtime remains manifest10/bindingV2r22/JWT10. Prepared files are outside runtime; no token, mount, reader, writer or account was changed. No live/sandbox grants. -## Publication Gates +#### Publication Gates DR01-DR08 and broker delta receipts remain mandatory, independently of Paper feature completion. Retain the new overflow/replay finding for affected recovery diff --git a/upgrade/evidence/releases/v2.2.3/release-preparation.json b/upgrade/evidence/releases/v2.2.3/release-preparation.json index dd44a88f..59e135ad 100644 --- a/upgrade/evidence/releases/v2.2.3/release-preparation.json +++ b/upgrade/evidence/releases/v2.2.3/release-preparation.json @@ -1,186 +1,594 @@ { - "active_runtime_roles": [ + "branch": "fix/p184-dl-closure", + "catalog": { + "note": "deliberate split (C.19 bar edge checkpoint reads the image catalog); unifying needs its own packet", + "repo_and_image_catalog_revision": 11, + "runtime_catalog": { + "path": "~/.local/state/qdl-v2/releases/v2.2.0-02cd827/runtime/stable-source-bindings.yaml", + "sha256": "fc3a1e1ec283e6c49b671f55cc270bd8f0241d595b241e6c7f5c87f5dc686cc5" + }, + "runtime_catalog_revision": 12 + }, + "ci": { + "local_full_run": { + "failure": "unit-tests step 19 trivy image scan only: local runner flag bug (boolean ignore-unfixed, .trivyignore mount), fixed in 259aeb9; steps 20-21 skipped after it", + "jobs": { + "consolidated-source-preflight": "PASS", + "contract-tests": "PASS", + "kn-native-integration": "PASS", + "sdk-python310": "PASS", + "unit-tests": "FAIL" + }, + "sha256": "23a49251af3f853f4fd8a5f640085c58f1475d12841a5f827c4db35c93759e19", + "status": "FAIL", + "summary": "~/.local/state/qdl-v2/p184-local-ci-bbfb3be/summary.json" + }, + "local_unit_tests_rerun": { + "passed": 21, + "runner": "259aeb9 on archive of bbfb3be", + "sha256": "07df25eb2578f7e68f08fba930e48262bf6cfbaea30d1bd4b3704e70d46d4071", + "skip": "Prepare non-root runtime bind mounts (requires sudo; reported, never escalated)", + "skipped": 1, + "status": "INCOMPLETE", + "summary": "~/.local/state/qdl-v2/p184-local-ci-bbfb3be-r2/summary.json" + }, + "remote_github_ci": "NOT_RUN (DL repository not pushed)" + }, + "commits_after_source": { + "163191c": "plan journal", + "259aeb9": "local CI runner trivy flag fix (tooling, not in images)", + "e7b2028": "plan journal", + "this_commit": "scripts/p184_trace_feed_transit.py + test, release evidence (not in images)" + }, + "consumer_revisions": { + "alpha_manifest_jwt_revision": 15, + "ts_paper_manifest_jwt_revision": 11, + "ts_paper_products": 64 + }, + "deferred_to_ts_release": [ + "dr01-deployed", + "dr01-wrong-account", + "dr02-dr03-alpha", + "dr07-reference" + ], + "funding_settlement_allowance": { + "evidence_dir": "~/.local/state/qdl-v2/funding-publication-20261010/", + "note": "coordinator appends the result", + "poll_window_utc": "2026-10-10T15:58Z-16:04Z", + "shipped": "no allowance (530a406 revert)", + "status": "PENDING" + }, + "images": [ { - "image": "sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9", + "digest": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "roles": [ + "query_kn_1", + "query_kn_2", + "binance_bar_edge" + ], + "tag": "qdl-v2-python:2.2.3-rc7d-bbfb3be", + "version_label": "2.2.3-rc7d" + }, + { + "digest": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "roles": [ + "stream_kn_1", + "stream_kn_2", + "market_projector_1", + "market_projector_2", + "rust_core", + "rust_core_2", + "rust_core_3", + "ingestor_okx_swap", + "ingestor_binance_usdm" + ], + "tag": "qdl-v2-rust:2.2.3-rc7d-bbfb3be", + "version_label": "2.2.3-rc7d" + } + ], + "ingestor_configs": { + "book_liveness_proof_ms": "ABSENT (IDR-040 fix 2 OFF)", + "dir": "~/.local/state/qdl-v2/releases/v2.2.3-rc7-ingestor-liveness", + "files": { + "ingestor_binance_usdm.json": { + "bindings": 27, + "config_revision": 19, + "liveness_dead_ms": 4000, + "liveness_ping_ms": 1500, + "sha256": "5833bc4ea35cb4558a931288d34b61a94e99d260f1ba6b0e56596423995930ed" + }, + "ingestor_okx_swap.json": { + "bindings": 103, + "config_revision": 20, + "liveness_dead_ms": 4000, + "liveness_ping_ms": 1500, + "sha256": "eb982facfccce78a825c835e5d7ecdebce5dba6e457252a946da69e808ed0460" + } + }, + "mounted_read_only_at": "/runtime/config.json" + }, + "not_a_release_certificate": true, + "preparation_history": { + "2026-10-08": { + "active_runtime_roles": [ + { + "image": "sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9", + "name": "qdl_v2_stable_candidate-binance_bar_edge-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-08T02:17:35.393509241Z" + }, + { + "image": "sha256:0f6876e16e51600419e1f172aae596bed9c2d369d1afcef93d1da30c18f262ec", + "name": "qdl_v2_stable_candidate-ingestor_binance_usdm-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-02T12:24:58.369037654Z" + }, + { + "image": "sha256:0f6876e16e51600419e1f172aae596bed9c2d369d1afcef93d1da30c18f262ec", + "name": "qdl_v2_stable_candidate-ingestor_okx_swap-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T05:51:24.840836395Z" + }, + { + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "name": "qdl_v2_stable_candidate-kafka1-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T03:56:13.899839854Z" + }, + { + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "name": "qdl_v2_stable_candidate-kafka2-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T03:57:34.401411155Z" + }, + { + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "name": "qdl_v2_stable_candidate-kafka3-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T03:58:43.963843285Z" + }, + { + "image": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", + "name": "qdl_v2_stable_candidate-market_cache-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-09-27T05:29:58.000340326Z" + }, + { + "image": "sha256:a696cfba6ab383173607f96413c814f74c7abcc98a4c41f57c9c6e78a41bb70b", + "name": "qdl_v2_stable_candidate-market_projector_1-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T12:16:20.78036445Z" + }, + { + "image": "sha256:a696cfba6ab383173607f96413c814f74c7abcc98a4c41f57c9c6e78a41bb70b", + "name": "qdl_v2_stable_candidate-market_projector_2-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T12:15:13.766866129Z" + }, + { + "image": "sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9", + "name": "qdl_v2_stable_candidate-query_kn_1-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-02T11:13:49.101943789Z" + }, + { + "image": "sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9", + "name": "qdl_v2_stable_candidate-query_kn_2-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-02T11:12:13.488673234Z" + }, + { + "image": "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023", + "name": "qdl_v2_stable_candidate-rust_core-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-07T17:56:52.401430746Z" + }, + { + "image": "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023", + "name": "qdl_v2_stable_candidate-rust_core_2-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-07T17:56:25.704876381Z" + }, + { + "image": "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023", + "name": "qdl_v2_stable_candidate-rust_core_3-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-07T17:56:10.228521642Z" + }, + { + "image": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", + "name": "qdl_v2_stable_candidate-stable_redis-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-09-16T06:38:55.588316335Z" + }, + { + "image": "sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9", + "name": "qdl_v2_stable_candidate-stream_kn_1-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T12:10:50.691735492Z" + }, + { + "image": "sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9", + "name": "qdl_v2_stable_candidate-stream_kn_2-1", + "oom": false, + "restart_count": 0, + "started_at": "2026-10-01T12:07:24.322915897Z" + } + ], + "bar_edge": { + "image": "sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9", + "owner_rollout_at": "2026-10-08T02:17:33Z", + "rollback": "sha256:f7351c3bda080a6d1266d49480dc65d7e590375f178252f858f87a04b1a4d685", + "source": "d6d2637557c3759f829421d2174e21a44762f7fc" + }, + "busy_window": { + "cpu": { + "max": 5.870669239231109, + "mean": 4.882531768271189, + "n": 30, + "p95_nearest_rank": 5.564052047081731 + }, + "duration_ms": 299992.35502200003, + "mean_lte5": true, + "not_peak_capacity": true, + "rates": { + "basis": "LOG_END_OFFSETS_INCLUDES_TRANSACTION_MARKERS_NOT_PAYLOAD_COUNT", + "covers_cpu_window_with_small_snapshot_setup_margin": true, + "offset_increments": { + "md.canonical.v2": 287754, + "md.raw.realtime.v2": 249628 + }, + "partitions": 12, + "rate": { + "md.canonical.v2": 952.4805598220488, + "md.raw.realtime.v2": 826.2815362679872 + }, + "seconds": 302.110102965, + "topics": 2 + }, + "receipt_sha256": "e1434fe1c534a1c20946779a89b5a53fd9013cc5378f0b88b923058bc6a24e80", + "stream_overflow_delta": 2, + "stream_replayed_delta": 15901 + }, + "excluded_core_candidate": "sha256:0d8da150820ed684bda084e92f714537cb40bd81e2e3a6bc93fa9a4b33538411", + "idr075": { + "binding_sha256": "5a9a19385ae34d34213284f06aac8e5c7eb687068702f493590f2664dc0accdf", + "existing_linear_index_preserved": true, + "new_binding_revision": 23, + "new_manifest_revision": 11, + "old_binding_revision": 22, + "old_manifest_revision": 10, + "paired_environment": { + "DATA_LAYER_V2_JWT_MANIFEST_REVISION": "11", + "DATA_LAYER_V2_RELEASE_MANIFEST_SHA256": "6d35f2cf26d9a348d3000ffbf80dc5266dbbe6e8c7cbc0fdc0675345f501ad27" + }, + "products": 64, + "status": "PREPARED_NOT_ACTIVATED" + }, + "previous_published_release": "v2.2.2", + "release_gates_open": [ + "DR01-DR08 indexed receipts", + "broker delta receipt", + "TRADE overflow/replay affected recovery proof", + "IDR075 approved paired runtime packet and actual Paper identity preflight", + "release source/image/config attestation and CI/approved publication" + ], + "runtime_changed_this_slice": false, + "schema": "qdl.release-preparation.v1", + "source_review_base": "1cd24e367bc9b3216318da740d0e7ab3f3ddba54", + "status": "PREPARED_RELEASE_HOLD_NOT_PUBLISHED", + "version": "2.2.3" + } + }, + "previous_published_release": "v2.2.2", + "publication_requires": [ + "owner decision on G2 FAIL and NOT_MEASURED gates (not accepted by this record)", + "funding publication-latency result", + "push; remote CI (contract-tests, sdk-python310, unit-tests, kn-native-integration) on the final head; merge to dev and main", + "annotated tag v2.2.3 linking CI runs and scorecard/notes hashes" + ], + "rollback": { + "dl_roles": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/-rollback.compose.json, reverse role order, failure only", + "not_touched": [ + "kafka1-3", + "market_cache", + "stable_redis", + "stable_state_init", + "stable_tls_init", + "Kafka offsets", + "Redis data", + "realm dirs" + ], + "python": "sha256:55e4937867489510b8fe07ce744c5d998af1f02e2bbaa31c40b371f73f85b723 (qdl-v2-python:2.2.3-rc7c-54460ac)", + "rollout_log": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc7d.log", + "sha256": "d8ab536105f6a6e71fb57a3a0287d9ff48bfc14bf4bdef3125a33a61fd76224a" + }, + "rust": "sha256:206ae872a5fed1dc127da9c3f70367b1ba79d34cbe6705c96b3f5bf16f082ef8 (qdl-v2-rust:2.2.3-rc7c-54460ac)" + }, + "runtime_changed_this_slice": false, + "runtime_inventory": [ + { + "image": "sha256:34202753bbe8b74d50e45afed1cf2e7b400141324c550168462b30db220edb31", + "name": "market_data_service_p184w", + "oom": false, + "restart_count": 0, + "revision_label": "ed020648286336170886043697cb229a3c5290b7", + "started_at": "2026-10-10T13:30:24.199934182Z", + "status": "running" + }, + { + "image": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", "name": "qdl_v2_stable_candidate-binance_bar_edge-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-08T02:17:35.393509241Z" + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:03:12.888708065Z", + "status": "running" }, { - "image": "sha256:0f6876e16e51600419e1f172aae596bed9c2d369d1afcef93d1da30c18f262ec", + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", "name": "qdl_v2_stable_candidate-ingestor_binance_usdm-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-02T12:24:58.369037654Z" + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:08:18.446850444Z", + "status": "running" }, { - "image": "sha256:0f6876e16e51600419e1f172aae596bed9c2d369d1afcef93d1da30c18f262ec", + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", "name": "qdl_v2_stable_candidate-ingestor_okx_swap-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-01T05:51:24.840836395Z" + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:08:10.243755542Z", + "status": "running" }, { "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", "name": "qdl_v2_stable_candidate-kafka1-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-01T03:56:13.899839854Z" + "revision_label": null, + "started_at": "2026-10-08T16:23:33.912058555Z", + "status": "running" }, { "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", "name": "qdl_v2_stable_candidate-kafka2-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-01T03:57:34.401411155Z" + "revision_label": null, + "started_at": "2026-10-08T16:23:34.12626004Z", + "status": "running" }, { "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", "name": "qdl_v2_stable_candidate-kafka3-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-01T03:58:43.963843285Z" + "revision_label": null, + "started_at": "2026-10-08T16:23:34.326288421Z", + "status": "running" }, { "image": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", "name": "qdl_v2_stable_candidate-market_cache-1", "oom": false, "restart_count": 0, - "started_at": "2026-09-27T05:29:58.000340326Z" + "revision_label": null, + "started_at": "2026-09-27T05:29:58.000340326Z", + "status": "running" }, { - "image": "sha256:a696cfba6ab383173607f96413c814f74c7abcc98a4c41f57c9c6e78a41bb70b", + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", "name": "qdl_v2_stable_candidate-market_projector_1-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-01T12:16:20.78036445Z" + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:05:32.09955379Z", + "status": "running" }, { - "image": "sha256:a696cfba6ab383173607f96413c814f74c7abcc98a4c41f57c9c6e78a41bb70b", + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", "name": "qdl_v2_stable_candidate-market_projector_2-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-01T12:15:13.766866129Z" + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:06:03.85847914Z", + "status": "running" }, { - "image": "sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9", + "image": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", "name": "qdl_v2_stable_candidate-query_kn_1-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-02T11:13:49.101943789Z" + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:02:32.131029796Z", + "status": "running" }, { - "image": "sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9", + "image": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", "name": "qdl_v2_stable_candidate-query_kn_2-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-02T11:12:13.488673234Z" + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:02:52.533684862Z", + "status": "running" }, { - "image": "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023", + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", "name": "qdl_v2_stable_candidate-rust_core-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-07T17:56:52.401430746Z" + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:06:35.398630448Z", + "status": "running" }, { - "image": "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023", + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", "name": "qdl_v2_stable_candidate-rust_core_2-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-07T17:56:25.704876381Z" + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:07:06.932177193Z", + "status": "running" }, { - "image": "sha256:658a9570c5fc23f4906413aa2460f4d82e21772cc63c9ed900363d4d13d54023", + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", "name": "qdl_v2_stable_candidate-rust_core_3-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-07T17:56:10.228521642Z" + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:07:38.503831129Z", + "status": "running" }, { "image": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", "name": "qdl_v2_stable_candidate-stable_redis-1", "oom": false, "restart_count": 0, - "started_at": "2026-09-16T06:38:55.588316335Z" + "revision_label": null, + "started_at": "2026-09-16T06:38:55.588316335Z", + "status": "running" + }, + { + "image": "sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d", + "name": "qdl_v2_stable_candidate-stable_state_init-1", + "oom": false, + "restart_count": 0, + "revision_label": "8ba4165", + "started_at": "2026-09-03T12:36:17.69711623Z", + "status": "exited" + }, + { + "image": "sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d", + "name": "qdl_v2_stable_candidate-stable_tls_init-1", + "oom": false, + "restart_count": 0, + "revision_label": "8ba4165", + "started_at": "2026-09-03T12:36:17.691555089Z", + "status": "exited" }, { - "image": "sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9", + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", "name": "qdl_v2_stable_candidate-stream_kn_1-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-01T12:10:50.691735492Z" + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:03:22.758338124Z", + "status": "running" }, { - "image": "sha256:1f13408fe94698a45f18e59e61c6554e979308dcebfe21514a5d0534609a51e9", + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", "name": "qdl_v2_stable_candidate-stream_kn_2-1", "oom": false, "restart_count": 0, - "started_at": "2026-10-01T12:07:24.322915897Z" + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:04:33.817947904Z", + "status": "running" } ], - "bar_edge": { - "image": "sha256:514506122111df8992a5dfac9dc9a3db7ccae202a398424f4dd5a033d2ddf9a9", - "owner_rollout_at": "2026-10-08T02:17:33Z", - "rollback": "sha256:f7351c3bda080a6d1266d49480dc65d7e590375f178252f858f87a04b1a4d685", - "source": "d6d2637557c3759f829421d2174e21a44762f7fc" + "runtime_inventory_basis": "docker inspect, read-only, at assembly time", + "schema": "qdl.release-preparation.v1", + "scorecard": { + "path": "upgrade/evidence/releases/v2.2.3/scorecard.json", + "sha256": "d4035ccd460838cfcb130800660014f8dcfd22079f034390d44fa3fe9971c714", + "summary": { + "DEFERRED": [ + "G13" + ], + "FAIL": [ + "G2" + ], + "NOT_MEASURED": [ + "G1", + "G3", + "G4", + "G6", + "G7", + "G8", + "G9", + "G10", + "G11", + "G12", + "G14" + ], + "PASS": [ + "G5" + ] + } }, - "busy_window": { - "cpu": { - "max": 5.870669239231109, - "mean": 4.882531768271189, - "n": 30, - "p95_nearest_rank": 5.564052047081731 - }, - "duration_ms": 299992.35502200003, - "mean_lte5": true, - "not_peak_capacity": true, - "rates": { - "basis": "LOG_END_OFFSETS_INCLUDES_TRANSACTION_MARKERS_NOT_PAYLOAD_COUNT", - "covers_cpu_window_with_small_snapshot_setup_margin": true, - "offset_increments": { - "md.canonical.v2": 287754, - "md.raw.realtime.v2": 249628 - }, - "partitions": 12, - "rate": { - "md.canonical.v2": 952.4805598220488, - "md.raw.realtime.v2": 826.2815362679872 + "sdk": { + "builder": "scripts/build_qdl_sdk_release.py from `git archive bbfb3be` (fresh extraction per build), docker run --rm --network none --cpus 1, non-root", + "builder_image": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "builds": [ + { + "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk-build-1", + "release_json_sha256": "240176ad449913c54b6ce6972b7114d6ea87e2982f69caa63a57369bce0a5bc3", + "sbom_sha256": "64bcb8491945da16c5400cda5bbcfdb292537a5e4eab0a9f681ff23c37e5930c", + "wheel_sha256": "d8226f509bce4ae408cacfa30e12f6160726e4040751d2e4f7379d3dc71c3997" }, - "seconds": 302.110102965, - "topics": 2 - }, - "receipt_sha256": "e1434fe1c534a1c20946779a89b5a53fd9013cc5378f0b88b923058bc6a24e80", - "stream_overflow_delta": 2, - "stream_replayed_delta": 15901 + { + "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk-build-2", + "release_json_sha256": "240176ad449913c54b6ce6972b7114d6ea87e2982f69caa63a57369bce0a5bc3", + "sbom_sha256": "64bcb8491945da16c5400cda5bbcfdb292537a5e4eab0a9f681ff23c37e5930c", + "wheel_sha256": "d8226f509bce4ae408cacfa30e12f6160726e4040751d2e4f7379d3dc71c3997" + } + ], + "contains_service_internals": false, + "dependencies": [ + "grpcio>=1.70.0,<2.0.0", + "httpx>=0.28.0,<1.0.0", + "protobuf>=6.31.1,<7.0.0", + "pydantic>=2.0.0,<3.0.0", + "PyJWT[crypto]>=2.15.1,<3.0.0" + ], + "deployed_image_comparison": "21/21 wheel files byte-identical to /app/qdl_sdk and /app/generated/python/qdl in the deployed python image", + "generated_contract_digest": "ae77dc42074fd7cc896a46bde3282837a51b667acee23c4957d37565690856e9", + "name": "qdl-sdk", + "python_requires": ">=3.10", + "reproducible": true, + "reproducible_timestamp": "2020-01-01T00:00:00Z", + "schema": "qdl.sdk.release.v1", + "sdk_source_digest": "63e230312cec966b51bfa20aedd3e73c51c838c1a52b778c24574f61b7399488", + "ts_writer_sdk_today": "2.1.0", + "version": "2.2.0", + "wheel": "qdl_sdk-2.2.0-py3-none-any.whl", + "wheel_sha256": "d8226f509bce4ae408cacfa30e12f6160726e4040751d2e4f7379d3dc71c3997" }, - "excluded_core_candidate": "sha256:0d8da150820ed684bda084e92f714537cb40bd81e2e3a6bc93fa9a4b33538411", - "idr075": { - "binding_sha256": "5a9a19385ae34d34213284f06aac8e5c7eb687068702f493590f2664dc0accdf", - "existing_linear_index_preserved": true, - "new_binding_revision": 23, - "new_manifest_revision": 11, - "old_binding_revision": 22, - "old_manifest_revision": 10, - "paired_environment": { - "DATA_LAYER_V2_JWT_MANIFEST_REVISION": "11", - "DATA_LAYER_V2_RELEASE_MANIFEST_SHA256": "6d35f2cf26d9a348d3000ffbf80dc5266dbbe6e8c7cbc0fdc0675345f501ad27" - }, - "products": 64, - "status": "PREPARED_NOT_ACTIVATED" + "source_commit": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "status": "PREPARED_NOT_PUBLISHED", + "ts_writer": { + "container": "market_data_service_p184w", + "expired_trace_env_present": true, + "image": "sha256:34202753bbe8b74d50e45afed1cf2e7b400141324c550168462b30db220edb31", + "rollback_env": { + "path": "~/.local/state/trading-system/p184-rev11-window/g2-tune/rollback-20261010T133013Z.compose.json", + "sha256": "a729902166e31749a2e411bab01d5b55b7126c58384237e8ddfb916f1ce827bd" + }, + "rollback_image": "sha256:15524c0bea8c51949d7ced79291d4c72722292fea2f0a5d4f8e9c7fd61dbb5ca", + "source": "ed020648286336170886043697cb229a3c5290b7", + "tuned_env": "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE=BINANCE=100" }, - "previous_published_release": "v2.2.2", - "release_gates_open": [ - "DR01-DR08 indexed receipts", - "broker delta receipt", - "TRADE overflow/replay affected recovery proof", - "IDR075 approved paired runtime packet and actual Paper identity preflight", - "release source/image/config attestation and CI/approved publication" - ], - "runtime_changed_this_slice": false, - "schema": "qdl.release-preparation.v1", - "source_review_base": "1cd24e367bc9b3216318da740d0e7ab3f3ddba54", - "status": "PREPARED_RELEASE_HOLD_NOT_PUBLISHED", - "version": "2.2.3" + "version": "2.2.3", + "worktree": "/home/bobby/.worktrees/data-layer-p184-closure" } diff --git a/upgrade/evidence/releases/v2.2.3/scorecard.json b/upgrade/evidence/releases/v2.2.3/scorecard.json new file mode 100644 index 00000000..edbf8826 --- /dev/null +++ b/upgrade/evidence/releases/v2.2.3/scorecard.json @@ -0,0 +1,1667 @@ +{ + "schema": "qdl.release-scorecard.v1", + "release": "2.2.3", + "status": "PREPARED_NOT_PUBLISHED", + "source_commit": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "gate_table": { + "path": "trading-system upgrade/p184-phase-d-gates.md (FROZEN 2026-10-09)", + "ts_commit": "31cf663f5c7c5384ec3dd296061582dee2463c41", + "sha256": "c67bf69bb921c992964eb8ff8b2e1bb17085a943fdf756d55526b6269641d5c4" + }, + "frozen_budget": { + "path": "config/v2/p184-stage100-budget.json", + "sha256": "ff85335b3c8762d9acd8e7230f3c81ee1ec8403aff34cf2fb8f67714521c2541" + }, + "scoring_rule": "steady state >= 30 min at the stage-100 workload plus one burst window (gate table 'Measurement rules'); a measurement outside this rule can FAIL a gate but cannot PASS it. Verdicts: PASS, FAIL, NOT_MEASURED, DEFERRED; a NOT_MEASURED is never upgraded. 'observed' records the value against the threshold when the measurement does not meet the protocol.", + "runtime_identity": { + "dl_python": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "dl_rust": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "dl_application_roles_on_release_digests": 12, + "ts_writer": "sha256:34202753bbe8b74d50e45afed1cf2e7b400141324c550168462b30db220edb31", + "ts_writer_source": "ed020648286336170886043697cb229a3c5290b7", + "ts_writer_tuned_env": "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE=BINANCE=100" + }, + "summary": { + "NOT_MEASURED": [ + "G1", + "G3", + "G4", + "G6", + "G7", + "G8", + "G9", + "G10", + "G11", + "G12", + "G14" + ], + "FAIL": [ + "G2" + ], + "PASS": [ + "G5" + ], + "DEFERRED": [ + "G13" + ] + }, + "gates": [ + { + "id": "G1", + "quantity": "end-to-end to TS cache (venue event -> TS Redis APPLY reply)", + "scope": "BOOK_DELTA, BOOK_SNAPSHOT", + "rule": "p95 <= 500 ms, p99 <= 1000 ms (n >= 100), max < 2000 ms per feed/venue", + "verdict": "NOT_MEASURED", + "observed": "INSIDE in both trace windows", + "reason": "Values come from writer-only trace windows, not the frozen stage-100 protocol.", + "measured": [ + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "BOOK_DELTA|BINANCE", + "n": 24553, + "p50_ms": 100.0, + "p95_ms": 126.5, + "p99_ms": 167.1, + "max_ms": 527.6, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 68.4, + "sdk_to_reply_ms": 36.7, + "venue_to_ingest_ms": 29.3 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "BOOK_DELTA|OKX", + "n": 26481, + "p50_ms": 104.3, + "p95_ms": 128.4, + "p99_ms": 173.4, + "max_ms": 528.1, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 67.6, + "sdk_to_reply_ms": 37.9, + "venue_to_ingest_ms": 33.0 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "BOOK_SNAPSHOT|BINANCE", + "n": 2647, + "p50_ms": 108.8, + "p95_ms": 154.4, + "p99_ms": 220.4, + "max_ms": 407.2, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 70.8, + "sdk_to_reply_ms": 58.3, + "venue_to_ingest_ms": 29.3 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "BOOK_SNAPSHOT|OKX", + "n": 3223, + "p50_ms": 113.6, + "p95_ms": 156.8, + "p99_ms": 206.3, + "max_ms": 336.1, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 69.2, + "sdk_to_reply_ms": 59.9, + "venue_to_ingest_ms": 33.1 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "BOOK_DELTA|BINANCE", + "n": 24664, + "p50_ms": 99.9, + "p95_ms": 126.5, + "p99_ms": 163.0, + "max_ms": 510.9, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 68.7, + "sdk_to_reply_ms": 36.4, + "venue_to_ingest_ms": 29.4 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "BOOK_DELTA|OKX", + "n": 26105, + "p50_ms": 105.1, + "p95_ms": 130.0, + "p99_ms": 170.5, + "max_ms": 501.8, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 67.8, + "sdk_to_reply_ms": 37.7, + "venue_to_ingest_ms": 37.9 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "BOOK_SNAPSHOT|BINANCE", + "n": 2647, + "p50_ms": 109.5, + "p95_ms": 152.4, + "p99_ms": 192.1, + "max_ms": 294.0, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 71.5, + "sdk_to_reply_ms": 53.5, + "venue_to_ingest_ms": 28.6 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "BOOK_SNAPSHOT|OKX", + "n": 3182, + "p50_ms": 114.3, + "p95_ms": 155.5, + "p99_ms": 188.9, + "max_ms": 353.1, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 68.5, + "sdk_to_reply_ms": 54.6, + "venue_to_ingest_ms": 37.4 + } + } + ], + "measurement_scope": "TS writer delivery trace, two 535 s analysis windows (13:19-13:28Z and 13:31-13:40Z, 40 s writer start-up skipped), writer-only production traffic AFTER stage 100m ended (13:17Z): no stage-100 alpha load, no burst window", + "would_measure": "the same trace analysis (scripts/p184_trace_feed_transit.py) over >= 30 min of trace activations taken DURING a stage-100 run, including its burst window (one activation is capped at 600 s and 16 x 4096 records per worker)", + "sources": [ + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791638906937534999.json", + "sha256": "d8d0d6c37dc37ccd1c22b3d9690d42041d09cc532ddd0008499d5a3d7b32526a" + }, + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json", + "sha256": "e9e823094e73782166d8275647fed3abbf6000c20bc18abd695b8bb1686bf72d" + } + ] + }, + { + "id": "G2", + "quantity": "end-to-end to TS cache, new-event transit", + "scope": "TRADE / QUOTE / MARK_INDEX_PRICE", + "rule": "TRADE p95 <= 500 ms and max < 1500 ms; QUOTE and MARK p95 <= 500 ms and max < 1000 ms (quiet-state age reported separately, not gated)", + "verdict": "FAIL", + "reason": "TRADE BINANCE p95 539.1 ms > 500 ms (max 1078.7 ms, n 21983) in trace window 1 on the final DL images and final writer image; window 2 (deployed writer config) is inside at p95 269.7 ms. The excess sits in the ingest -> SDK return segment (split p95 434.5 ms), not in the writer (sdk -> reply p95 85.0 ms) or the venue -> ingest leg (36.9 ms). Window 1 differs from the deployed config only by the Binance MARK wake read delay (250 vs 100 ms), which does not act on TRADE. Corroboration under stage-100 load: alpha SDK stream TRADE BINANCE source -> usable per-stream p95 up to 872.5 ms, max 1402.3 ms (stage100m receipt stream_latency). MARK and QUOTE are inside in the deployed-config window but are not PASS by protocol.", + "measured": [ + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "QUOTE|BINANCE", + "n": 25600, + "p50_ms": 116.2, + "p95_ms": 152.3, + "p99_ms": 179.0, + "max_ms": 434.1, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 98.5, + "sdk_to_reply_ms": 32.0, + "venue_to_ingest_ms": 30.8 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "QUOTE|OKX", + "n": 23103, + "p50_ms": 117.5, + "p95_ms": 153.9, + "p99_ms": 188.5, + "max_ms": 421.4, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 98.0, + "sdk_to_reply_ms": 32.6, + "venue_to_ingest_ms": 31.6 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "TRADE|BINANCE", + "n": 21983, + "p50_ms": 118.8, + "p95_ms": 539.1, + "p99_ms": 989.7, + "max_ms": 1078.7, + "observed": "OUTSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 434.5, + "sdk_to_reply_ms": 85.0, + "venue_to_ingest_ms": 36.9 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "TRADE|OKX", + "n": 5035, + "p50_ms": 107.7, + "p95_ms": 246.5, + "p99_ms": 558.3, + "max_ms": 731.1, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 107.5, + "sdk_to_reply_ms": 97.4, + "venue_to_ingest_ms": 37.9 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "QUOTE|BINANCE", + "n": 24134, + "p50_ms": 117.2, + "p95_ms": 152.2, + "p99_ms": 176.2, + "max_ms": 516.9, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 98.8, + "sdk_to_reply_ms": 32.1, + "venue_to_ingest_ms": 32.0 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "QUOTE|OKX", + "n": 22916, + "p50_ms": 118.3, + "p95_ms": 154.0, + "p99_ms": 181.7, + "max_ms": 513.4, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 98.2, + "sdk_to_reply_ms": 32.6, + "venue_to_ingest_ms": 31.9 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "TRADE|BINANCE", + "n": 14928, + "p50_ms": 116.8, + "p95_ms": 269.7, + "p99_ms": 545.5, + "max_ms": 623.9, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 191.3, + "sdk_to_reply_ms": 66.1, + "venue_to_ingest_ms": 36.0 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "TRADE|OKX", + "n": 4845, + "p50_ms": 106.6, + "p95_ms": 158.9, + "p99_ms": 216.5, + "max_ms": 394.0, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 80.3, + "sdk_to_reply_ms": 60.8, + "venue_to_ingest_ms": 35.3 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "MARK_INDEX_PRICE|BINANCE", + "n": 2667, + "p50_ms": 429.5, + "p95_ms": 500.7, + "p99_ms": 540.2, + "max_ms": 776.2, + "observed": "OUTSIDE" + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "MARK_INDEX_PRICE|OKX", + "n": 6385, + "p50_ms": 214.9, + "p95_ms": 320.9, + "p99_ms": 348.6, + "max_ms": 398.6, + "observed": "INSIDE" + }, + { + "window": "window2_deployed_config", + "feed_venue": "MARK_INDEX_PRICE|BINANCE", + "n": 2671, + "p50_ms": 278.5, + "p95_ms": 339.3, + "p99_ms": 379.5, + "max_ms": 708.0, + "observed": "INSIDE" + }, + { + "window": "window2_deployed_config", + "feed_venue": "MARK_INDEX_PRICE|OKX", + "n": 6390, + "p50_ms": 212.3, + "p95_ms": 317.9, + "p99_ms": 347.8, + "max_ms": 495.2, + "observed": "INSIDE" + } + ], + "measurement_scope": "TS writer delivery trace, two 535 s analysis windows (13:19-13:28Z and 13:31-13:40Z, 40 s writer start-up skipped), writer-only production traffic AFTER stage 100m ended (13:17Z): no stage-100 alpha load, no burst window", + "would_measure": "attribute the TRADE ingest -> SDK return tail (Stream dispatch/queue vs SDK client receive) and re-measure >= 30 min of trace during a stage-100 run", + "sources": [ + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791638906937534999.json", + "sha256": "d8d0d6c37dc37ccd1c22b3d9690d42041d09cc532ddd0008499d5a3d7b32526a" + }, + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json", + "sha256": "e9e823094e73782166d8275647fed3abbf6000c20bc18abd695b8bb1686bf72d" + }, + { + "path": "~/.local/state/qdl-v2/p184-g2-20261010/g2-mark-1791638906937534999.json", + "sha256": "292256e814110d6805efab9660667939699a228d99c1d1283cb5ce61e35f476c" + }, + { + "path": "~/.local/state/qdl-v2/p184-g2-20261010/g2-mark-1791639603470577388.json", + "sha256": "3da349c4f54c5125c23b8a6eb3c6e8f080c1a2023dd8b300ffa1732f2b09ea68" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/receipt.json", + "sha256": "9dfce5d983f25cdeb831eee6dbcd151864fa1cdbf567c9dd871f5866f67684a8" + } + ] + }, + { + "id": "G3", + "quantity": "end-to-end to TS cache, BAR from close", + "scope": "BAR", + "rule": "p95 <= 5 s, max < 60 s", + "verdict": "NOT_MEASURED", + "reason": "The writer delivery trace does not record BAR (0 BAR records in both windows); no other receipt measures BAR close -> TS Redis on the final digests.", + "related_observation": "stage100m alpha SDK streams (not the TS cache): BAR 1m close -> usable BINANCE per-stream p95 1135-3095 ms, max 3095 ms; OKX p95 1248-1307 ms, max 1511 ms", + "would_measure": "a read-only TS Redis notification probe (the 2026-10-09 transit_probe.py method, BAR from close) or BAR tracing in the writer, during a stage-100 run", + "sources": [ + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791638906937534999.json", + "sha256": "d8d0d6c37dc37ccd1c22b3d9690d42041d09cc532ddd0008499d5a3d7b32526a" + }, + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json", + "sha256": "e9e823094e73782166d8275647fed3abbf6000c20bc18abd695b8bb1686bf72d" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/receipt.json", + "sha256": "9dfce5d983f25cdeb831eee6dbcd151864fa1cdbf567c9dd871f5866f67684a8" + } + ] + }, + { + "id": "G4", + "quantity": "delivery lag, venue -> canonical (read_committed visibility, upper bound)", + "scope": "every feed, venue and interval", + "rule": "candidate p95 <= frozen baseline p95 x 1.2 per row (owner amendment 2026-10-09); missing coverage blocks the gate", + "verdict": "NOT_MEASURED", + "reason": "16/17 frozen rows PASS in all four stage-100m windows; bar|BINANCE|30m had no close in any window; five observed rows (BAR BINANCE 5m/1h, OKX 5m/15m/1h) have no frozen baseline; OKX inverse is pooled into OKX rows, not its own cells. Missing coverage blocks the gate by its own rule.", + "measured": [ + { + "row": "bar|BINANCE|15m", + "p95_ms_per_window": [ + null, + null, + 12533.0, + null + ], + "n_per_window": [ + 0, + 0, + 3, + 0 + ], + "limit_p95_ms": 40728.7, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|BINANCE|1h", + "p95_ms_per_window": [ + null, + null, + 14999.3, + null + ], + "n_per_window": [ + 0, + 0, + 3, + 0 + ], + "limit_p95_ms": null, + "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|BINANCE|1m", + "p95_ms_per_window": [ + 116.4, + 108.3, + 117.7, + 140.8 + ], + "n_per_window": [ + 10, + 10, + 10, + 10 + ], + "limit_p95_ms": 257.6, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|BINANCE|30m", + "p95_ms_per_window": [ + null, + null, + null, + null + ], + "n_per_window": [ + 0, + 0, + 0, + 0 + ], + "limit_p95_ms": 25836.6, + "verdict": "NOT_MEASURED_NOT_OBSERVED", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|BINANCE|3m", + "p95_ms_per_window": [ + 7992.0, + 8016.0, + null, + 8001.2 + ], + "n_per_window": [ + 5, + 5, + 0, + 5 + ], + "limit_p95_ms": 27273.2, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|BINANCE|5m", + "p95_ms_per_window": [ + null, + 8169.9, + 16186.8, + null + ], + "n_per_window": [ + 0, + 5, + 4, + 0 + ], + "limit_p95_ms": null, + "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|OKX|15m", + "p95_ms_per_window": [ + null, + null, + 901184.4, + null + ], + "n_per_window": [ + 0, + 0, + 1, + 0 + ], + "limit_p95_ms": null, + "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|OKX|1h", + "p95_ms_per_window": [ + null, + null, + 3600671.8, + null + ], + "n_per_window": [ + 0, + 0, + 1, + 0 + ], + "limit_p95_ms": null, + "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|OKX|1m", + "p95_ms_per_window": [ + 61304.5, + 60741.8, + 61188.9, + 61107.0 + ], + "n_per_window": [ + 7, + 1, + 6, + 8 + ], + "limit_p95_ms": 73817.2, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|OKX|30m", + "p95_ms_per_window": [ + null, + null, + 1800470.9, + null + ], + "n_per_window": [ + 0, + 0, + 1, + 0 + ], + "limit_p95_ms": 2160788.8, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|OKX|3m", + "p95_ms_per_window": [ + 181311.9, + 181132.3, + 180480.3, + 181110.8 + ], + "n_per_window": [ + 3, + 4, + 1, + 2 + ], + "limit_p95_ms": 217582.1, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|OKX|5m", + "p95_ms_per_window": [ + null, + 300978.9, + null, + null + ], + "n_per_window": [ + 0, + 5, + 0, + 0 + ], + "limit_p95_ms": null, + "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "book_delta|BINANCE|", + "p95_ms_per_window": [ + 97.2, + 98.0, + 100.0, + 101.9 + ], + "n_per_window": [ + 756, + 755, + 760, + 785 + ], + "limit_p95_ms": 263.3, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "book_delta|OKX|", + "p95_ms_per_window": [ + 103.1, + 100.3, + 103.3, + 114.8 + ], + "n_per_window": [ + 1061, + 1069, + 978, + 1047 + ], + "limit_p95_ms": 289.1, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "book_snapshot|BINANCE|", + "p95_ms_per_window": [ + 99.6, + 96.2, + 100.3, + 102.4 + ], + "n_per_window": [ + 867, + 752, + 859, + 928 + ], + "limit_p95_ms": 260.0, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "book_snapshot|OKX|", + "p95_ms_per_window": [ + 101.5, + 97.8, + 102.6, + 112.1 + ], + "n_per_window": [ + 1618, + 1688, + 1086, + 1538 + ], + "limit_p95_ms": 270.1, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "mark_index_price|BINANCE|", + "p95_ms_per_window": [ + 185.8, + 190.2, + 195.2, + 193.2 + ], + "n_per_window": [ + 377, + 376, + 384, + 379 + ], + "limit_p95_ms": 370.9, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "mark_index_price|OKX|", + "p95_ms_per_window": [ + 1447.1, + 1361.9, + 1453.1, + 1252.2 + ], + "n_per_window": [ + 665, + 664, + 665, + 662 + ], + "limit_p95_ms": 1586.6, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "quote|BINANCE|", + "p95_ms_per_window": [ + 126.5, + 129.1, + 130.0, + 134.5 + ], + "n_per_window": [ + 546, + 555, + 548, + 560 + ], + "limit_p95_ms": 292.7, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "quote|OKX|", + "p95_ms_per_window": [ + 129.8, + 129.3, + 132.4, + 141.6 + ], + "n_per_window": [ + 591, + 584, + 566, + 598 + ], + "limit_p95_ms": 283.3, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "trade|BINANCE|", + "p95_ms_per_window": [ + 100.5, + 100.4, + 104.7, + 105.7 + ], + "n_per_window": [ + 344, + 312, + 340, + 397 + ], + "limit_p95_ms": 269.3, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "trade|OKX|", + "p95_ms_per_window": [ + 104.1, + 104.6, + 105.0, + 110.0 + ], + "n_per_window": [ + 250, + 251, + 273, + 305 + ], + "limit_p95_ms": 257.4, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + } + ], + "windows": "4 x 120 s, 12:47-13:07Z, inside stage 100m (12:45-13:17Z), observer assigned read_committed, no join/commit", + "would_measure": "a G4 observer window long enough to see a BINANCE 30m close, frozen baselines for the five unbaselined rows, and inverse-separated rows", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/g4m-1-summary.json", + "sha256": "2c98891f2c9e25a09a106b665573b4ff4424edcafbcd22d01fc084affe2fe904" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/g4m-2-summary.json", + "sha256": "c6f321b95d05ca5aa27d353bae191449fad181d7a4bd2df0e2f9834830b2b287" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/g4m-3-summary.json", + "sha256": "b349e8c4f00ed31cbba341807597fbdab61e0b2a1cfa85dc718b528c8eabc5d4" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/g4m-4-summary.json", + "sha256": "6955061d7bf23c91e34fe885dd4a7917cce98d092b88ceffed9acbf8333a92fb" + }, + { + "path": "config/v2/p184-stage100-budget.json", + "sha256": "ff85335b3c8762d9acd8e7230f3c81ee1ec8403aff34cf2fb8f67714521c2541" + } + ] + }, + { + "id": "G5", + "quantity": "request latency", + "scope": "Query snapshot/reference classes + cold warmup shapes", + "rule": "certified stage-50 p95/p99 targets per class and venue; COLD_2500 <= 6265.1 ms, COLD_5000 <= 15267.0 ms (frozen budget)", + "verdict": "PASS", + "reason": "Every latency class and cold shape PASS in the 1800 s stage-100m run on the final digests (32/32 driver gates).", + "measured": [ + { + "gate": "latency:BAR_LATEST:BINANCE", + "count": 355, + "p50": 16.52, + "p95": 30.506, + "p99": 54.274, + "max": 291.955, + "target_p95": 1000, + "target_p99": 2000, + "status": "PASS" + }, + { + "gate": "latency:BAR_LATEST:OKX", + "count": 355, + "p50": 27.136, + "p95": 89.426, + "p99": 202.213, + "max": 340.742, + "target_p95": 1000, + "target_p99": 2000, + "status": "PASS" + }, + { + "gate": "latency:L2_SNAPSHOT:BINANCE", + "count": 355, + "p50": 40.123, + "p95": 80.944, + "p99": 229.67, + "max": 381.528, + "target_p95": 300, + "target_p99": 750, + "status": "PASS" + }, + { + "gate": "latency:L2_SNAPSHOT:OKX", + "count": 355, + "p50": 28.744, + "p95": 53.634, + "p99": 257.597, + "max": 445.148, + "target_p95": 300, + "target_p99": 750, + "status": "PASS" + }, + { + "gate": "latency:MARK_INDEX_REFERENCE:BINANCE", + "count": 53250, + "p50": 19.348, + "p95": 58.191, + "p99": 80.126, + "max": 536.599, + "target_p95": 250, + "target_p99": 500, + "status": "PASS" + }, + { + "gate": "latency:MARK_INDEX_REFERENCE:OKX", + "count": 53250, + "p50": 21.293, + "p95": 57.023, + "p99": 74.992, + "max": 455.591, + "target_p95": 250, + "target_p99": 500, + "status": "PASS" + }, + { + "gate": "latency:QUOTE_SNAPSHOT:BINANCE", + "count": 35500, + "p50": 13.349, + "p95": 37.49, + "p99": 74.924, + "max": 539.242, + "target_p95": 100, + "target_p99": 250, + "status": "PASS" + }, + { + "gate": "latency:QUOTE_SNAPSHOT:OKX", + "count": 35500, + "p50": 11.764, + "p95": 35.185, + "p99": 64.916, + "max": 464.515, + "target_p95": 100, + "target_p99": 250, + "status": "PASS" + }, + { + "gate": "latency:TRADE_SNAPSHOT:BINANCE", + "count": 354, + "p50": 10.332, + "p95": 34.75, + "p99": 57.465, + "max": 416.772, + "target_p95": 100, + "target_p99": 250, + "status": "PASS" + }, + { + "gate": "latency:TRADE_SNAPSHOT:OKX", + "count": 355, + "p50": 14.745, + "p95": 29.366, + "p99": 48.66, + "max": 237.098, + "target_p95": 100, + "target_p99": 250, + "status": "PASS" + }, + { + "gate": "g5:COLD_2500:BINANCE", + "count": 1, + "max_ms": 3124.007, + "limit_ms": 6265.1, + "status": "PASS" + }, + { + "gate": "g5:COLD_2500:OKX", + "count": 1, + "max_ms": 3726.708, + "limit_ms": 6265.1, + "status": "PASS" + }, + { + "gate": "g5:COLD_5000:BINANCE", + "count": 1, + "max_ms": 5298.549, + "limit_ms": 15267.0, + "status": "PASS" + }, + { + "gate": "g5:COLD_5000:OKX", + "count": 1, + "max_ms": 6177.615, + "limit_ms": 15267.0, + "status": "PASS" + } + ], + "limits": "cold shapes are n=1 per venue (max within limit); MARK_INDEX_REFERENCE is the measured reference-batch shape, not an arbitrary 50-item batch", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/gates.json", + "sha256": "c65db308bdfbf453d3ea1027b0dec73aa00b66c1d043ca72b3ec2e1aad8bb079" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/receipt.json", + "sha256": "9dfce5d983f25cdeb831eee6dbcd151864fa1cdbf567c9dd871f5866f67684a8" + }, + { + "path": "config/v2/p184-stage100-budget.json", + "sha256": "ff85335b3c8762d9acd8e7230f3c81ee1ec8403aff34cf2fb8f67714521c2541" + } + ] + }, + { + "id": "G6", + "quantity": "durable-view event age and typed execution quality at the endpoint", + "scope": "every requested feed/venue/interval", + "rule": "per-binding D3 thresholds (TRADE 3000, QUOTE/MARK/BOOK_DELTA 2000, BOOK_SNAPSHOT 60000, BAR1m 180000 ms) plus stricter TS use-time policy; zero unsafe eligibility", + "verdict": "NOT_MEASURED", + "reason": "The stage-100 receipt records no per-binding event age or eligibility of the returned views.", + "related_observation": "stage100m: 182709 offered requests, 0 failures (0 typed refusals of any code), 180 streams with 0 errors", + "would_measure": "driver receipt fields for per-binding view event age (SDK freshness) and execution eligibility of every returned view, including refusals", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/gates.json", + "sha256": "c65db308bdfbf453d3ea1027b0dec73aa00b66c1d043ca72b3ec2e1aad8bb079" + } + ] + }, + { + "id": "G7", + "quantity": "TS consumer path", + "scope": "SDK return -> writer entry; writer entry -> Redis APPLY reply", + "rule": "p95 <= 100 ms and p95 <= 50 ms per feed", + "verdict": "NOT_MEASURED", + "observed": "INSIDE for every feed in both trace windows", + "reason": "Writer-only trace windows, not the frozen stage-100 protocol.", + "measured": [ + { + "window": "window1_mark_delay_binance_250", + "feed": "BOOK_DELTA", + "sdk_to_writer_ms": { + "n": 51429, + "p50_ms": 22.3, + "p95_ms": 29.7, + "p99_ms": 58.6, + "max_ms": 372.2 + }, + "writer_to_redis_ms": { + "n": 51058, + "p50_ms": 2.1, + "p95_ms": 9.1, + "p99_ms": 16.2, + "max_ms": 192.3 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed": "BOOK_SNAPSHOT", + "sdk_to_writer_ms": { + "n": 5870, + "p50_ms": 28.8, + "p95_ms": 39.5, + "p99_ms": 92.9, + "max_ms": 190.7 + }, + "writer_to_redis_ms": { + "n": 5874, + "p50_ms": 4.0, + "p95_ms": 14.0, + "p99_ms": 40.5, + "max_ms": 192.2 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed": "MARK_INDEX_PRICE", + "sdk_to_writer_ms": { + "n": 9052, + "p50_ms": 0.4, + "p95_ms": 0.6, + "p99_ms": 0.6, + "max_ms": 1.2 + }, + "writer_to_redis_ms": { + "n": 9052, + "p50_ms": 2.1, + "p95_ms": 7.0, + "p99_ms": 13.3, + "max_ms": 94.5 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed": "QUOTE", + "sdk_to_writer_ms": { + "n": 48707, + "p50_ms": 22.1, + "p95_ms": 28.7, + "p99_ms": 39.6, + "max_ms": 175.3 + }, + "writer_to_redis_ms": { + "n": 54515, + "p50_ms": 1.1, + "p95_ms": 4.1, + "p99_ms": 10.5, + "max_ms": 140.1 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed": "TRADE", + "sdk_to_writer_ms": { + "n": 27066, + "p50_ms": 22.9, + "p95_ms": 43.2, + "p99_ms": 95.7, + "max_ms": 207.2 + }, + "writer_to_redis_ms": { + "n": 27398, + "p50_ms": 5.1, + "p95_ms": 36.9, + "p99_ms": 98.9, + "max_ms": 120.2 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window2_deployed_config", + "feed": "BOOK_DELTA", + "sdk_to_writer_ms": { + "n": 51162, + "p50_ms": 22.3, + "p95_ms": 29.5, + "p99_ms": 48.6, + "max_ms": 218.7 + }, + "writer_to_redis_ms": { + "n": 50788, + "p50_ms": 2.1, + "p95_ms": 8.9, + "p99_ms": 16.0, + "max_ms": 150.1 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window2_deployed_config", + "feed": "BOOK_SNAPSHOT", + "sdk_to_writer_ms": { + "n": 5829, + "p50_ms": 28.8, + "p95_ms": 37.8, + "p99_ms": 90.8, + "max_ms": 151.2 + }, + "writer_to_redis_ms": { + "n": 5836, + "p50_ms": 4.0, + "p95_ms": 13.3, + "p99_ms": 32.4, + "max_ms": 85.2 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window2_deployed_config", + "feed": "MARK_INDEX_PRICE", + "sdk_to_writer_ms": { + "n": 9061, + "p50_ms": 0.4, + "p95_ms": 0.6, + "p99_ms": 0.6, + "max_ms": 1.2 + }, + "writer_to_redis_ms": { + "n": 9061, + "p50_ms": 2.2, + "p95_ms": 6.8, + "p99_ms": 13.3, + "max_ms": 85.9 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window2_deployed_config", + "feed": "QUOTE", + "sdk_to_writer_ms": { + "n": 47053, + "p50_ms": 22.1, + "p95_ms": 28.6, + "p99_ms": 38.1, + "max_ms": 206.9 + }, + "writer_to_redis_ms": { + "n": 52863, + "p50_ms": 1.1, + "p95_ms": 4.0, + "p99_ms": 10.4, + "max_ms": 100.5 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window2_deployed_config", + "feed": "TRADE", + "sdk_to_writer_ms": { + "n": 19820, + "p50_ms": 22.9, + "p95_ms": 41.5, + "p99_ms": 88.4, + "max_ms": 157.5 + }, + "writer_to_redis_ms": { + "n": 20131, + "p50_ms": 4.6, + "p95_ms": 26.2, + "p99_ms": 52.4, + "max_ms": 94.5 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + } + ], + "measurement_scope": "TS writer delivery trace, two 535 s analysis windows (13:19-13:28Z and 13:31-13:40Z, 40 s writer start-up skipped), writer-only production traffic AFTER stage 100m ended (13:17Z): no stage-100 alpha load, no burst window", + "would_measure": "the same analysis over >= 30 min of trace during a stage-100 run", + "sources": [ + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791638906937534999.json", + "sha256": "d8d0d6c37dc37ccd1c22b3d9690d42041d09cc532ddd0008499d5a3d7b32526a" + }, + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json", + "sha256": "e9e823094e73782166d8275647fed3abbf6000c20bc18abd695b8bb1686bf72d" + } + ] + }, + { + "id": "G8", + "quantity": "TS execution Redis (redis_marketdata) command CPU, SLOWLOG, RTT", + "scope": "INFO commandstats usec delta, SLOWLOG count (10 ms), PING RTT", + "rule": "commandstats <= 60 % of one core per 10-min window; SLOWLOG 0 in steady state; PING RTT p99 <= 5 ms under full mixed traffic", + "verdict": "NOT_MEASURED", + "reason": "One read-only 600 s window on writer-only traffic, after the stage-100 run; not 'full mixed traffic'.", + "measured": { + "seconds": 600.011, + "core_fraction": 0.0651, + "slowlog_new_entries": 0, + "slowlog_new_by_command": {}, + "slowlog_new_max_duration_us": null, + "slowlog_threshold_us": 10000, + "ping_rtt_ms": { + "errors": 0, + "max": 5.279, + "n": 5971, + "p50": 0.318, + "p95": 0.528, + "p99": 0.942 + }, + "monitor_clients": 0, + "top_commands": { + "cmdstat_eval": { + "calls": 76968, + "usec": 9700807 + }, + "cmdstat_evalsha": { + "calls": 87009, + "usec": 23497977 + }, + "cmdstat_get": { + "calls": 263762, + "usec": 644174 + }, + "cmdstat_mget": { + "calls": 95034, + "usec": 558002 + }, + "cmdstat_publish": { + "calls": 891655, + "usec": 679478 + }, + "cmdstat_scan": { + "calls": 4470, + "usec": 196198 + }, + "cmdstat_set": { + "calls": 613529, + "usec": 1067763 + }, + "cmdstat_setex": { + "calls": 1021990, + "usec": 2681454 + } + } + }, + "window_utc": [ + 1791644816.3761318, + 1791645416.3872285 + ], + "would_measure": "the same read-only probe over every 10-min window of a stage-100 run", + "sources": [ + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/g8-redis/g8-window-600s.json", + "sha256": "fbc269fd1303e8bac13c4894089fc43d2368850d16859b1bde03ed497e8c528b" + } + ] + }, + { + "id": "G9", + "quantity": "readiness", + "scope": "demanded / session-ready / intent-eligible, final manifest denominator", + "rule": "demanded == session-ready for the whole window; 0 data-path-caused losses of execution eligibility; v1_fallback 0", + "verdict": "NOT_MEASURED", + "reason": "The session clause is met (ts:ready_60_every_sample 154/154 samples, 0 unreadable, writer READY 64/64; v1_fallback_calls 0), but losses of execution eligibility by cause were not counted.", + "measured": { + "ts_ready": { + "expected": 154, + "gate": "ts:ready_60_every_sample", + "not_ready": 0, + "samples": 154, + "status": "PASS", + "unreadable": 0 + }, + "v1_fallback_calls": 0, + "mark_watcher": "0 writer MARK errors in 32 of 32 minutes (mark-watch-s100m.log)" + }, + "would_measure": "per-slice intent-eligibility samples with refusal cause (quiet TRADE/LAST counted separately) over the stage-100 window", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/gates.json", + "sha256": "c65db308bdfbf453d3ea1027b0dec73aa00b66c1d043ca72b3ec2e1aad8bb079" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/guard-s100m.log", + "sha256": "cef88fa86bda376c1991618b6e5734cbb69603bc7a8ecb9c9c0812c9dbb7a91b" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/mark-watch-s100m.log", + "sha256": "f905c2810f7d74c7290865f18a3adb1dc4106c9181b5130cd5a3762a44c29083" + } + ] + }, + { + "id": "G10", + "quantity": "integrity", + "scope": "TRADE (LOSSLESS) at the TS writer", + "rule": "0 gaps, 0 duplicates applied, 0 replay-expired, 0 overflow", + "verdict": "NOT_MEASURED", + "reason": "No production gap/overflow/replay-expiry counters were read for the stage-100 window on the final digests.", + "related_observation": "DR08 isolated (QUALIFICATION, final DL digests): SLOW_CONSUMER_BOUNDED_QUEUE closes at 1000 queued events with RESOURCE_EXHAUSTED and replay; trace windows show DUPLICATE TRADE outcomes refused by the writer (not applied)", + "would_measure": "Stream overflow/replay counters and writer gap/duplicate counters before/after a stage-100 run", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T142914Z/dr08-isolated/work/evidence.jsonl", + "sha256": "a7bf8d013af4e011741f8013926993500140575d626916d148caa9b207ef4fc6" + } + ] + }, + { + "id": "G11", + "quantity": "recovery, TS writer restart", + "scope": "fault cleared -> first valid apply per slice; -> caught up to live", + "rule": "<= 30 s and <= 60 s; no rewind; cursor never ahead of applied state", + "verdict": "NOT_MEASURED", + "reason": "No receipt records the four timestamps for a writer restart on the final digests.", + "related_observation": "DR08 isolated CONSUMER_RESTART_RECOVERY_MEASURED (a DL stream consumer process, not the TS writer): restart -> first event 5067-5144 ms, n=5, ack_measured false", + "would_measure": "a writer restart with outage start, fault cleared, first valid apply and caught-up recorded per slice (the delivery trace started at writer start can supply first apply)", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T142914Z/dr08-isolated/work/evidence.jsonl", + "sha256": "a7bf8d013af4e011741f8013926993500140575d626916d148caa9b207ef4fc6" + } + ] + }, + { + "id": "G12", + "quantity": "recovery, Kafka broker / projector / Stream restart", + "scope": "fault cleared -> caught up to live, then G1-G3", + "rule": "<= 600 s (owner bound)", + "verdict": "NOT_MEASURED", + "reason": "The isolated PASS (75673bc/2a5e809; broker 0.202 s, projector 0.256 s, Stream 2.355 s caught up; BOOK only, G2/G3 NOT_MEASURED) ran core 658a9570 (0e00c81), projector a696cfba and Stream 1f13408f (c8015ca), Query 672503b7 (76950c8) and writer 15524c0b (9b86ed3). Since then the core (+959/-30 lines: IDR-062 top-change materialization, IDR-040 fix 2), Stream (+1020/-147: Astra R2 hot lookup, terminal errors across stalled handoffs), projector (+39: invalid-book fence) and Query changed, so the result is not inheritable by the final digests.", + "related_observation": "The rc7d rollout restarted Stream, projectors and cores one at a time with writer READY 64/64 after each step (rollout-rc7d.log); timestamps for caught-up were not recorded.", + "would_measure": "scripts/p184_g12_isolated_fault.py with its IMAGES set to python e0252f4c / rust 64468c72 / writer 34202753 (the image set is a module constant today)", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-g12-20261010/evidence/verdict.json", + "sha256": "7e98572c80ed6746234c7a7d7f2fbc9cae5d0b617d60a07ab8fabf48d70a8057" + }, + { + "path": "~/.local/state/qdl-v2/p184-g12-20261010/evidence/verdict.reanalyzed.json", + "sha256": "4bf70d72c10212dd2da407788691138cd2da048c2f9ec458f0c587e1366ce1b6" + } + ] + }, + { + "id": "G13", + "quantity": "acceptance", + "scope": "DR01-DR08 incl. DR05 typed gap/generation fence and inverse", + "rule": "every cell closed once on the final DL + writer digests", + "verdict": "DEFERRED", + "reason": "41/41 DL-side cells harness-ok on the final digests; dr01-deployed, dr01-wrong-account, dr02-dr03-alpha and dr07-reference are deferred to the TS release (gateway/alpha/risk not running) and stay OPEN.", + "measured": [ + { + "step": "dr05-upstream", + "cells": 6, + "ok": 6, + "outcome": "6 OBSERVED (ISOLATED_FAULT)", + "receipt": "p184-g13-20261010T134219Z" + }, + { + "step": "dr04-dr06-views", + "cells": 22, + "ok": 22, + "outcome": "14 OBSERVED RUNTIME, 6 OBSERVED ISOLATED_FAULT, 2 NOT_OBSERVED RUNTIME (DR05.QUIET_TYPED BINANCE/OKX: no quiet book in the 300 s window; ok by the cell rule, not a positive quiet proof)", + "receipt": "p184-g13-20261010T134219Z" + }, + { + "step": "dr05-transport", + "cells": 9, + "ok": 9, + "outcome": "9 OBSERVED (ISOLATED_FAULT); harness TS 7d7fd0e, writer image ed02064", + "receipt": "p184-g13-20261010T142735Z", + "superseded_attempt": "p184-g13-20261010T134219Z/dr05-transport harness exit 1 (probe read QUOTE only; fixed TS 7d7fd0e)" + }, + { + "step": "dr08-isolated", + "cells": 4, + "ok": 4, + "outcome": "4 OBSERVED (ISOLATED_FAULT, QUALIFICATION mode)", + "receipt": "p184-g13-20261010T142914Z" + } + ], + "deferred_cells": [ + "dr01-deployed", + "dr01-wrong-account", + "dr02-dr03-alpha", + "dr07-reference" + ], + "would_measure": "the TS release runs the four deferred steps with gateway/alpha/risk on their declared images", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T134219Z/summary.json", + "sha256": "b64f35e19ed5ad5fd283f11fa5d58af0e52ef7309440ff94a5e1df7deab63345" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T134219Z/dr05-upstream/dr05-upstream-rows.jsonl", + "sha256": "6dfe552ed9a116271561affce5e6836841bcc3a85cb9d1e23fda9325feca61c5" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T134219Z/dr04-dr06-views/dr04-dr06.jsonl", + "sha256": "11db0fc93c41ed4fc699d125781628ca81a0ad546e9b8026e872a93750e0d51f" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T142735Z/summary.json", + "sha256": "24d659c53f86e8cfdb9f6ea88fca7020a3d7c89e5d7771697e3560498537ccca" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T142735Z/dr05-transport/dr05-transport.jsonl", + "sha256": "42064f1393b6f7c6a615686295b7ffaba46a8a7f29b367e76a28e1cb7b69925b" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T142914Z/summary.json", + "sha256": "eb6ae4e299e7f3ee91593d2327b74daf653d1d2b8e43e496b49d74d573a9d85d" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T142914Z/dr08-isolated/work/evidence.jsonl", + "sha256": "a7bf8d013af4e011741f8013926993500140575d626916d148caa9b207ef4fc6" + } + ] + }, + { + "id": "G14", + "quantity": "resources", + "scope": "every DL role and the TS writer at stage 100", + "rule": "CPU/RAM recorded per role with headroom; budget set by the owner from the measurement", + "verdict": "NOT_MEASURED", + "reason": "16 roles recorded from 152 samples (memory max inside every limit; DL total about 5.0 vCPU mean); binance_bar_edge and stable_redis were not sampled, so 'every DL role' is incomplete. The owner budget decision is still open.", + "measured": [ + { + "role": "ingestor_binance_usdm", + "cpu_mean_pct": 10.6, + "cpu_p95_pct": 13.8, + "cpu_max_pct": 17.3, + "mem_max_mib": 29.9, + "mem_limit": "256MiB" + }, + { + "role": "ingestor_okx_swap", + "cpu_mean_pct": 11.5, + "cpu_p95_pct": 14.5, + "cpu_max_pct": 16.2, + "mem_max_mib": 28.4, + "mem_limit": "256MiB" + }, + { + "role": "kafka1", + "cpu_mean_pct": 77.0, + "cpu_p95_pct": 86.5, + "cpu_max_pct": 94.4, + "mem_max_mib": 968.0, + "mem_limit": "1.5GiB" + }, + { + "role": "kafka2", + "cpu_mean_pct": 95.4, + "cpu_p95_pct": 107.2, + "cpu_max_pct": 117.2, + "mem_max_mib": 921.1, + "mem_limit": "2GiB" + }, + { + "role": "kafka3", + "cpu_mean_pct": 98.2, + "cpu_p95_pct": 109.6, + "cpu_max_pct": 126.1, + "mem_max_mib": 1019.0, + "mem_limit": "1.5GiB" + }, + { + "role": "market_cache", + "cpu_mean_pct": 7.8, + "cpu_p95_pct": 10.7, + "cpu_max_pct": 12.7, + "mem_max_mib": 1162.2, + "mem_limit": "9GiB" + }, + { + "role": "market_data_service_p184w", + "cpu_mean_pct": 123.7, + "cpu_p95_pct": 149.2, + "cpu_max_pct": 198.2, + "mem_max_mib": 639.5, + "mem_limit": "30.64GiB" + }, + { + "role": "market_projector_1", + "cpu_mean_pct": 17.9, + "cpu_p95_pct": 20.0, + "cpu_max_pct": 22.5, + "mem_max_mib": 45.1, + "mem_limit": "512MiB" + }, + { + "role": "market_projector_2", + "cpu_mean_pct": 8.0, + "cpu_p95_pct": 8.8, + "cpu_max_pct": 9.6, + "mem_max_mib": 34.1, + "mem_limit": "512MiB" + }, + { + "role": "query_kn_1", + "cpu_mean_pct": 67.9, + "cpu_p95_pct": 100.6, + "cpu_max_pct": 124.6, + "mem_max_mib": 651.2, + "mem_limit": "1.5GiB" + }, + { + "role": "query_kn_2", + "cpu_mean_pct": 48.3, + "cpu_p95_pct": 77.7, + "cpu_max_pct": 109.8, + "mem_max_mib": 761.4, + "mem_limit": "1.5GiB" + }, + { + "role": "rust_core", + "cpu_mean_pct": 9.2, + "cpu_p95_pct": 10.8, + "cpu_max_pct": 12.9, + "mem_max_mib": 44.6, + "mem_limit": "256MiB" + }, + { + "role": "rust_core_2", + "cpu_mean_pct": 11.4, + "cpu_p95_pct": 13.1, + "cpu_max_pct": 15.6, + "mem_max_mib": 46.5, + "mem_limit": "256MiB" + }, + { + "role": "rust_core_3", + "cpu_mean_pct": 7.8, + "cpu_p95_pct": 9.0, + "cpu_max_pct": 10.2, + "mem_max_mib": 37.2, + "mem_limit": "256MiB" + }, + { + "role": "stream_kn_1", + "cpu_mean_pct": 21.5, + "cpu_p95_pct": 25.8, + "cpu_max_pct": 34.9, + "mem_max_mib": 158.0, + "mem_limit": "512MiB" + }, + { + "role": "stream_kn_2", + "cpu_mean_pct": 7.1, + "cpu_p95_pct": 7.8, + "cpu_max_pct": 8.2, + "mem_max_mib": 93.5, + "mem_limit": "512MiB" + } + ], + "would_measure": "the same sampler including binance_bar_edge and stable_redis over a stage-100 run", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/g14-resources.txt", + "sha256": "6b1fdcab6e110e27832dd878f4bae92f042e261bda4fca6aeac7a433d9925792" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/host.json", + "sha256": "1483bd2c800bfec9d0c9faf96ec446fed87c13d659dc526e572ecfdc324e58fc" + } + ] + } + ] +} From 627f19da4e77d4f36504a25bb3fe29a5b1a1ca77 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 15:24:31 +0000 Subject: [PATCH 123/147] feat(p184-g12): image set is a parameter (--image-set/--image, sealed and attested, per-container provenance), --query-code image; final-image re-run packet --- scripts/p184_g12_isolated_fault.py | 201 +++++++++++++++--- tests/test_p184_g12_isolated_fault.py | 96 +++++++++ .../p184-g12-isolated-fault-packet.md | 45 ++++ 3 files changed, 310 insertions(+), 32 deletions(-) diff --git a/scripts/p184_g12_isolated_fault.py b/scripts/p184_g12_isolated_fault.py index f1079a1f..ba40f2b5 100644 --- a/scripts/p184_g12_isolated_fault.py +++ b/scripts/p184_g12_isolated_fault.py @@ -28,6 +28,18 @@ Host modes (stdlib only): ``--run BASE`` (orchestrate, absolute deadline, teardown in ``finally``), ``--down BASE`` (exact-name, label-checked teardown), ``--analyze BASE`` (recompute the verdict from the JSONL evidence). + +Image set: ``IMAGES`` below are DEFAULTS only. ``--run`` accepts ``--image-set FILE`` +(a JSON object role -> sha256:<64hex>, any subset of the roles) and ``--image +ROLE=sha256:<64hex>`` (repeatable, applied last). The resolved set is checked against +the local image IDs, sealed in ``work/handoff.json`` ("g12_images", plus "images" in +the DR05 role names so the in-container ``--material`` step writes the core digest +from it), passed to every in-namespace helper as ``P184_G12_IMAGES`` (checked against +the seal by ``_packet``), and every container this run starts (including restarts and +the WriterClone containers) is ``docker inspect``-ed: its image ID must equal the +passed value for its role (``evidence/provenance.jsonl``). ``--query-code image`` +runs Query from the image's own /app (no worktree mount); the default ``worktree`` +keeps the earlier read-only worktree mount. In-container modes (python image, inside the namespace): ``--topics``, ``--publish``, ``--monitor``; each bounded by ``--seconds`` / its own deadline. """ @@ -76,11 +88,56 @@ LIMITS = {"disk_free_bytes": 20 * 10**9, "mem_available_bytes": 3 * 2**30, "deadline_s": 3600, "holder_s": 5400, "broker_outage_s": 60, "role_return_s": 120, "clone_ready_s": 300, "baseline_s": 300} +IMAGE_ID = re.compile(r"sha256:[0-9a-f]{64}") +QUERY_CODE = ("worktree", "image") +# G12 role -> DR05/DR08 image role (handoff "images"; dr05 image_set() requires exactly these keys). +DR05_IMAGE_ROLES = {"query": "python", "core": "core", "stream": "stream", "python": "python", + "kafka": "kafka", "redis": "redis"} PRIVATE_FILES = ("client.key", "server.p12", "binance.key", "okx.key", "cursor-keys.json", "read-view.secret", "broker.properties", "broker-1.properties", "broker-2.properties", "broker-3.properties") +def resolve_image_set(image_set=None, overrides=()): + """IMAGES (defaults) <- JSON file/object <- ROLE=ID overrides; every role an immutable image ID.""" + images = dict(IMAGES) + layers = [] + if image_set is not None: + loaded = image_set if isinstance(image_set, dict) else json.loads(Path(image_set).read_text()) + if not isinstance(loaded, dict): + raise ValueError("image set file must be a JSON object role -> sha256:<64hex>") + layers.append(loaded) + parsed = {} + for value in overrides or (): + role, sep, image = str(value).partition("=") + if not sep or role in parsed: + raise ValueError(f"--image ROLE=sha256:<64hex>, once per role: {value!r}") + parsed[role] = image + layers.append(parsed) + for layer in layers: + for role, image in layer.items(): + if role not in IMAGES: + raise ValueError(f"unknown image role {role!r}; roles: {sorted(IMAGES)}") + images[role] = image + for role, image in images.items(): + if not isinstance(image, str) or not IMAGE_ID.fullmatch(image): + raise ValueError(f"immutable {role} image ID required (sha256:<64hex>)") + return images + + +def dr05_images(images): + """The sealed set in DR05's role names (handoff.json "images").""" + return {role: images[g12_role] for role, g12_role in DR05_IMAGE_ROLES.items()} + + +def role_image_key(role): + """Which image-set role a container role runs.""" + base = re.sub(r"-(binance|okx)$", "", role) + return {"net": "holder", "quota": "redis", "query": "python", "helper": "python", "monitor": "python", + "publisher": "python", "sampler": "ts", "tswriter": "ts", "tsredis": "redis", + "tspg": "postgres"}.get(base, "kafka" if base.startswith("kafka-") else base) + + def namespace(value): if not isinstance(value, str) or not re.fullmatch(NS_PATTERN, value): raise ValueError("exact disposable G12 namespace required") @@ -164,10 +221,13 @@ def _env(argv, env): return argv -def topology(ns, work, topic_id="", *, uid=None): +def topology(ns, work, topic_id="", *, uid=None, images=None, query_code="worktree"): """Docker argv per role (adapted from p184_dr05_dl_fault.topology: 3 brokers, a projector, one holder-owned loopback namespace). Executes nothing.""" namespace(ns) + images = images or IMAGES + if query_code not in QUERY_CODE: + raise ValueError("query_code must be one of " + ",".join(QUERY_CODE)) work = Path(work).resolve() if work == ROOT or ROOT in work.parents or any(c in str(work) for c in (":", ",")): raise ValueError("external unambiguous work directory required") @@ -181,7 +241,7 @@ def topology(ns, work, topic_id="", *, uid=None): mount = ["-v", f"{work}:/work"] # The holder's hostname is the namespace's: Kafka/log4j resolve it (DR05 kafka: --hostname localhost). roles = {"net": _base(ns, "net", network="none", memory="16m", cpus="0.05") - + ["--hostname", "localhost", "--entrypoint", "sleep", IMAGES["holder"], str(LIMITS["holder_s"])]} + + ["--hostname", "localhost", "--entrypoint", "sleep", images["holder"], str(LIMITS["holder_s"])]} for node in BROKERS: argv = _base(ns, f"kafka-{node}", network=net, memory="1024m", cpus="0.75", uid=uid) argv += ["--tmpfs", "/tmp:rw,exec,size=64m,mode=1777", "-v", f"{work}/kafka-{node}:/data", @@ -190,17 +250,17 @@ def topology(ns, work, topic_id="", *, uid=None): argv += ["--tmpfs", target + ":rw,size=1m,mode=1777"] # no anonymous volumes _env(argv, {"KAFKA_HEAP_OPTS": "-Xms256m -Xmx512m", "LOG_DIR": "/tmp/kafka-logs"}) props = f"/work/broker-{node}.properties" - roles[f"kafka-{node}"] = argv + ["--entrypoint", "/bin/sh", IMAGES["kafka"], "-ec", + roles[f"kafka-{node}"] = argv + ["--entrypoint", "/bin/sh", images["kafka"], "-ec", f"/opt/kafka/bin/kafka-storage.sh format --ignore-formatted -t {CLUSTER_ID} -c {props}" f" && exec /opt/kafka/bin/kafka-server-start.sh {props}"] roles["quota"] = _base(ns, "quota", network=net, memory="192m", cpus="0.1", uid=uid) + [ - "--tmpfs", "/data:rw,size=1m,mode=1777", "--entrypoint", "redis-server", IMAGES["redis"], + "--tmpfs", "/data:rw,size=1m,mode=1777", "--entrypoint", "redis-server", images["redis"], "--save", "", "--appendonly", "no", "--maxmemory", "128mb", "--maxmemory-policy", "noeviction"] roles["core"] = _env(_base(ns, "core", network=net, memory="256m", cpus="0.5", uid=uid) + ["--tmpfs", "/tmp:rw,size=64m,mode=1777"] + mount, {"QDL_KAFKA_CERT_ROOT": "/work", "QDL_KAFKA_BOOTSTRAP_SERVERS": BOOTSTRAP, "QDL_KAFKA_CLIENT_ID": ns, "QDL_KAFKA_GROUP_ID": ns, "QDL_PROVIDER_ADMISSION_ENABLED": "false"}) + [ - "--entrypoint", "/usr/local/bin/qdl-realtime-core", IMAGES["core"], "/work/core.json"] + "--entrypoint", "/usr/local/bin/qdl-realtime-core", images["core"], "/work/core.json"] roles["projector"] = _env(_base(ns, "projector", network=net, memory="256m", cpus="0.5", uid=uid) + ["--tmpfs", "/tmp:rw,size=64m,mode=1777"] + mount, {**tls, "QDL_KN_KAFKA_BOOTSTRAP": BOOTSTRAP, "QDL_KN_BUNDLE_PATH": "/work/bundle.json", @@ -209,7 +269,7 @@ def topology(ns, work, topic_id="", *, uid=None): "QDL_KN_STATUS_PATH": "/work/g12-projector-status.json", "QDL_KN_STATUS_INTERVAL_S": "5", "QDL_KN_CANONICAL_TOPIC": ns + ".canonical", "QDL_KN_LATEST_TOPIC": ns + ".latest", "QDL_KN_BARS_TOPIC": ns + ".bars"}) + [ - "--entrypoint", "/usr/local/bin/qdl-projector", IMAGES["projector"], "run"] + "--entrypoint", "/usr/local/bin/qdl-projector", images["projector"], "run"] roles["stream"] = _env(_base(ns, "stream", network=net, memory="512m", cpus="0.5", uid=uid) + ["--tmpfs", "/tmp:rw,size=64m,mode=1777"] + mount, {**common, **tls, "QDL_KN_BUNDLE_FILE": "/work/bundle.json", @@ -222,11 +282,13 @@ def topology(ns, work, topic_id="", *, uid=None): "QDL_KN_READ_VIEW_CA_FILE": "/work/ca.crt", "QDL_KN_HOT_READ_ENABLED": "true", "QDL_KN_METRICS_SECONDS": "10", "QDL_KN_MAX_REPLAY_EVENTS": "10000", "QDL_KN_QUEUE_BYTES_TOTAL": "33554432"}) + [ - "--entrypoint", "/usr/local/bin/qdl-stream-gateway", IMAGES["stream"], "serve"] + "--entrypoint", "/usr/local/bin/qdl-stream-gateway", images["stream"], "serve"] + # "image": the Query code is the image's own /app (the release under test); "worktree": read-only mount. + code = ["-w", "/app"] if query_code == "image" else ["-v", f"{ROOT}:/app:ro", "-w", "/app"] roles["query"] = _env(_base(ns, "query", network=net, memory="512m", cpus="0.5", uid=uid) - + ["--tmpfs", "/tmp:rw,size=64m,mode=1777"] + mount + ["-v", f"{ROOT}:/app:ro", "-w", "/app"], + + ["--tmpfs", "/tmp:rw,size=64m,mode=1777"] + mount + code, {**common, "PYTHONPATH": "/app:/app/generated/python", "PYTHONDONTWRITEBYTECODE": "1"}) + [ - "--entrypoint", "python", IMAGES["python"], "-B", "scripts/p184_dr05_observer.py", "--query", "/work"] + "--entrypoint", "python", images["python"], "-B", "scripts/p184_dr05_observer.py", "--query", "/work"] return roles @@ -236,18 +298,20 @@ def topology(ns, work, topic_id="", *, uid=None): def helper_argv(ns, work, args, *, role="helper", detach=False, evidence=None, network=None, - cpus="0.5", memory="512m", script="scripts/p184_g12_isolated_fault.py", extra=()): + cpus="0.5", memory="512m", script="scripts/p184_g12_isolated_fault.py", extra=(), images=None): namespace(ns) + images = images or IMAGES argv = _base(ns, role, network=network or "container:" + ns + "-net", memory=memory, cpus=cpus, uid=f"{os.getuid()}:{os.getgid()}") if not detach: argv.remove("-d") argv += ["--tmpfs", "/tmp:rw,size=64m,mode=1777", "-v", f"{Path(work).resolve()}:/work", "-v", f"{ROOT}:/app:ro", "-w", "/app", "-e", "PYTHONPATH=/app:/app/generated/python", - "-e", "PYTHONDONTWRITEBYTECODE=1", "-e", "P184_G12_NAMESPACE=" + ns, *extra] + "-e", "PYTHONDONTWRITEBYTECODE=1", "-e", "P184_G12_NAMESPACE=" + ns, + "-e", "P184_G12_IMAGES=" + json.dumps(images, sort_keys=True, separators=(",", ":")), *extra] if evidence is not None: argv += ["-v", f"{Path(evidence).resolve()}:/evidence"] - return argv + ["--entrypoint", "python", IMAGES["python"], "-B", script, *args] + return argv + ["--entrypoint", "python", images["python"], "-B", script, *args] SAMPLER = r''' @@ -319,12 +383,13 @@ async def main(out, seconds, interval_ms, targets): ''' -def sampler_argv(ns, evidence, seconds, interval_ms=50): +def sampler_argv(ns, evidence, seconds, interval_ms=50, images=None): + images = images or IMAGES targets = ",".join(f"{v}=redis://localhost:{CLONE_PORTS[v][0]}/1" for v in VENUES) argv = _base(ns, "sampler", network="container:" + ns + "-net", memory="384m", cpus="0.25", uid=f"{os.getuid()}:{os.getgid()}") return argv + ["--tmpfs", "/tmp:rw,size=16m,mode=1777", "-v", f"{Path(evidence).resolve()}:/evidence", - "-e", "PYTHONDONTWRITEBYTECODE=1", "--entrypoint", "python", IMAGES["ts"], "-B", "-c", + "-e", "PYTHONDONTWRITEBYTECODE=1", "--entrypoint", "python", images["ts"], "-B", "-c", SAMPLER, "/evidence/sampler.jsonl", str(int(seconds)), str(interval_ms), targets] @@ -541,9 +606,21 @@ def _packet(work): ns = namespace(packet["namespace"]) if os.environ.get("P184_G12_NAMESPACE") != ns or packet.get("bootstrap") != BOOTSTRAP: raise ValueError("G12 namespace attestation and isolated bootstrap required") + check_sealed_images(work, os.environ.get("P184_G12_IMAGES")) return packet +def check_sealed_images(work, attested): + """In-container: the parent's passed image set must equal the run's sealed set.""" + handoff = Path(work) / "handoff.json" + sealed = json.loads(handoff.read_text()).get("g12_images") if handoff.is_file() else None + if sealed is None and attested is None: + return None # a run sealed before the image set was a parameter + if sealed is None or attested is None or resolve_image_set(json.loads(attested)) != resolve_image_set(sealed): + raise ValueError("G12 image set attestation does not match the sealed run image set") + return sealed + + def _security(packet): from scripts.p184_dr08_isolated import kafka_security return kafka_security(packet) @@ -811,6 +888,22 @@ def host_resources(): return {"disk_free_bytes": free, "mem_available_bytes": available} +def container_image(name): + """The image ID a container actually runs (read-only docker inspect).""" + return sh(["docker", "inspect", "--format", "{{.Image}}", name], timeout=30).strip() + + +def image_record(image): + """Local image ID, tags and OCI revision/version labels (provenance, read-only).""" + rows = json.loads(sh(["docker", "image", "inspect", image], timeout=30) or "[]") + if len(rows) != 1: + raise RuntimeError("local image missing " + image) + labels = (rows[0].get("Config") or {}).get("Labels") or {} + return {"id": rows[0]["Id"], "tags": rows[0].get("RepoTags") or [], + "revision": labels.get("org.opencontainers.image.revision"), + "version": labels.get("org.opencontainers.image.version")} + + def load_writer_clone(): import importlib.util path = TS_ROOT / "scripts/p184_dr05_isolated.py" @@ -838,7 +931,11 @@ def run(*args, check=True, capture=True, timeout=120): class Run: - def __init__(self, base, export, ns): + def __init__(self, base, export, ns, images=None, query_code="worktree"): + self.images = resolve_image_set(images) + if query_code not in QUERY_CODE: + raise ValueError("query_code must be one of " + ",".join(QUERY_CODE)) + self.query_code = query_code self.base = Path(base).resolve() self.work, self.evidence = self.base / "work", self.base / "evidence" self.export = Path(export).resolve() @@ -857,6 +954,23 @@ def note(self, step, **fields): handle.write(json.dumps(row, default=str) + "\n") print(step, json.dumps(fields, default=str)[:400], flush=True) + def verify_image(self, role): + """Provenance: the started container runs exactly the passed image for its role.""" + name, expected = f"{self.ns}-{role}", self.images[role_image_key(role)] + actual = container_image(name) + row = {"t": time.time_ns(), "role": role, "image_role": role_image_key(role), + "expected": expected, "actual": actual, "match": actual == expected} + with (self.evidence / "provenance.jsonl").open("a") as handle: + handle.write(json.dumps(row) + "\n") + if actual != expected: + raise RuntimeError(f"STOP provenance: {name} runs {actual}, expected {expected}") + + def topology(self, topic_id=""): + return topology(self.ns, self.work, topic_id, images=self.images, query_code=self.query_code) + + def helper(self, *args, **kwargs): + return helper_argv(*args, images=self.images, **kwargs) + def fault_row(self, row): with (self.evidence / "faults.jsonl").open("a") as handle: handle.write(json.dumps(row) + "\n") @@ -904,9 +1018,10 @@ def wait(self, seconds, faulted=(), until=None, step=5.0): return until() if until is not None else None def start(self, role, topic_id): - argv = topology(self.ns, self.work, topic_id)[role] + argv = self.topology(topic_id)[role] sh(argv, timeout=60) self.expected.add(role) + self.verify_image(role) # -- phases def prepare(self): @@ -917,9 +1032,16 @@ def prepare(self): self.note("preflight", **resources) if resources["disk_free_bytes"] < LIMITS["disk_free_bytes"]: raise RuntimeError("disk below 20 GB before start") - for image in IMAGES.values(): - if sh(["docker", "image", "inspect", image, "--format", "{{.Id}}"]).strip() != image: + records = {} + for role, image in self.images.items(): + records[role] = image_record(image) + if records[role]["id"] != image: raise RuntimeError("local image mismatch " + image) + (self.evidence / "images.json").write_text(json.dumps( + {"images": self.images, "local": records, "query_code": self.query_code, + "defaults_overridden": sorted(r for r in self.images if self.images[r] != IMAGES[r])}, + indent=2) + "\n") + self.note("image_set", images=self.images, query_code=self.query_code) if owned_names(self.ns) or sh(["docker", "ps", "-aq", "--filter", "name=" + self.ns]).strip(): raise RuntimeError("namespace already in use") self.guard0 = production_guard() @@ -928,8 +1050,10 @@ def prepare(self): self.networks0 = len(sh(["docker", "network", "ls", "-q"]).split()) (self.work / "handoff.json").write_text(json.dumps({ "namespace": self.ns, "status": "PREPARED_HANDOFF_NOT_RUN", "provenance": "TEST_ONLY", - "level": "ISOLATED_FAULT", "acceptance": False, "material_ready": False}, indent=2) + "\n") - sh(helper_argv(self.ns, self.work, ["--material", "/work", "--core-config", "/export.json"], + "level": "ISOLATED_FAULT", "acceptance": False, "material_ready": False, + "images": dr05_images(self.images), "g12_images": self.images, + "query_code": self.query_code}, indent=2) + "\n") + sh(self.helper(self.ns, self.work, ["--material", "/work", "--core-config", "/export.json"], network="none", script="scripts/p184_dr05_dl_fault.py", extra=["-v", f"{self.export}:/export.json:ro"]), timeout=180) packet = json.loads((self.work / "packet.json").read_text()) @@ -946,11 +1070,12 @@ def prepare(self): networks_before=self.networks0) def up(self): - sh(topology(self.ns, self.work)["net"]) + sh(self.topology()["net"]) self.expected.add("net") + self.verify_image("net") for node in BROKERS: self.start(f"kafka-{node}", "") - result = sh(helper_argv(self.ns, self.work, ["--topics", "/work"]), timeout=180) + result = sh(self.helper(self.ns, self.work, ["--topics", "/work"]), timeout=180) self.note("topics", result=result.strip()[-400:]) topic_id = json.loads((self.work / "packet.json").read_text())["slice"]["topic_id"] self.topic_id = topic_id @@ -959,7 +1084,7 @@ def up(self): limit = time.monotonic() + LIMITS["role_return_s"] while True: self.check() - probe = subprocess.run(helper_argv(self.ns, self.work, ["--probe", "/work"], + probe = subprocess.run(self.helper(self.ns, self.work, ["--probe", "/work"], script="scripts/p184_dr05_dl_fault.py"), capture_output=True, text=True, timeout=60) if probe.returncode == 0: @@ -969,14 +1094,16 @@ def up(self): raise RuntimeError("services not reachable: " + probe.stderr.strip()[-400:]) time.sleep(2) remaining = LIMITS["deadline_s"] - (time.monotonic() - self.started) - 60 - sh(helper_argv(self.ns, self.work, ["--monitor", "/work", "--out", "/evidence/monitor.jsonl", + sh(self.helper(self.ns, self.work, ["--monitor", "/work", "--out", "/evidence/monitor.jsonl", "--seconds", str(int(remaining))], role="monitor", detach=True, evidence=self.evidence, cpus="0.25", memory="256m")) self.expected.add("monitor") - sh(helper_argv(self.ns, self.work, ["--publish", "/work", "--out", "/evidence/publish.jsonl", + self.verify_image("monitor") + sh(self.helper(self.ns, self.work, ["--publish", "/work", "--out", "/evidence/publish.jsonl", "--seconds", str(int(remaining)), "--hz", "4"], role="publisher", detach=True, evidence=self.evidence, cpus="0.25", memory="256m")) self.expected.add("publisher") + self.verify_image("publisher") self.note("publisher_monitor_started", seconds=int(remaining)) iso = load_writer_clone() import tempfile @@ -987,7 +1114,8 @@ def up(self): for venue in VENUES: redis_port, pg_port = CLONE_PORTS[venue] spec = json.loads((self.work / f"ts-consumer.{venue.lower()}.json").read_text()) - clone = iso.WriterClone(IMAGES["ts"], redis_image=IMAGES["redis"], postgres_image=IMAGES["postgres"], + clone = iso.WriterClone(self.images["ts"], redis_image=self.images["redis"], + postgres_image=self.images["postgres"], venues=(venue,), network="container:" + self.ns + "-net", redis_port=redis_port, pg_port=pg_port, consumer=spec, dl_work=str(self.work), run=run, tag=venue.lower()) @@ -999,9 +1127,12 @@ def up(self): self.clones[venue] = clone clone.__enter__() self.expected |= {f"tswriter-{label}", f"tsredis-{label}", f"tspg-{label}"} + for kind in ("tswriter", "tsredis", "tspg"): + self.verify_image(f"{kind}-{label}") self.note("ts_clones_started", names=[c.name for c in self.clones.values()]) - sh(sampler_argv(self.ns, self.evidence, remaining - 30)) + sh(sampler_argv(self.ns, self.evidence, remaining - 30, images=self.images)) self.expected.add("sampler") + self.verify_image("sampler") started = time.time_ns() def ready(): @@ -1037,8 +1168,9 @@ def fault(self, name, kind): if kind == "broker": self.wait(LIMITS["broker_outage_s"], faulted=(role,)) restart = time.time_ns() - sh(topology(self.ns, self.work, self.topic_id)[role], timeout=60) + sh(self.topology(self.topic_id)[role], timeout=60) self.note("restart_issued", fault=name, role=role) + self.verify_image(role) row = {"k": "fault", "name": name, "kind": kind, "node": node, "role": role, "injected_ns": injected, "restart_ns": restart} self.fault_row(row) @@ -1139,8 +1271,8 @@ def erase_private(work): shutil.rmtree(work / "tslogs", ignore_errors=True) -def orchestrate(base, export, ns): - run = Run(base, export, ns) +def orchestrate(base, export, ns, images=None, query_code="worktree"): + run = Run(base, export, ns, images, query_code) def expire(_signum, _frame): raise TimeoutError("G12 orchestrator absolute deadline") @@ -1165,7 +1297,8 @@ def expire(_signum, _frame): try: if run.evidence.exists(): verdict = evaluate(run.evidence) - verdict.update(status=status, error=error, namespace=ns) + verdict.update(status=status, error=error, namespace=ns, images=run.images, + query_code=run.query_code) (run.evidence / "verdict.json").write_text(json.dumps(verdict, indent=2) + "\n") finally: run.down() if run.evidence.exists() else teardown(ns) @@ -1183,11 +1316,15 @@ def main(argv=None): parser.add_argument("--out") parser.add_argument("--seconds", type=float) parser.add_argument("--hz", type=float, default=4.0) + parser.add_argument("--image-set", type=Path, help="JSON object role -> sha256:<64hex> (defaults: IMAGES)") + parser.add_argument("--image", action="append", default=[], metavar="ROLE=sha256:<64hex>") + parser.add_argument("--query-code", choices=QUERY_CODE, default="worktree") args = parser.parse_args(argv) if args.run: if args.export is None or args.namespace is None: parser.error("--run needs --export and --namespace") - return orchestrate(args.run, args.export, args.namespace) + images = resolve_image_set(args.image_set, args.image) + return orchestrate(args.run, args.export, args.namespace, images, args.query_code) if args.down: ns = namespace(json.loads((args.down / "work" / "handoff.json").read_text())["namespace"]) report = teardown(ns) diff --git a/tests/test_p184_g12_isolated_fault.py b/tests/test_p184_g12_isolated_fault.py index b7be17e1..23db97f5 100644 --- a/tests/test_p184_g12_isolated_fault.py +++ b/tests/test_p184_g12_isolated_fault.py @@ -208,5 +208,101 @@ def test_teardown_refuses_foreign_names(self): g12.owned_names(NS) +FINAL = {"python": "sha256:" + "e" * 64, "core": "sha256:" + "6" * 64, "projector": "sha256:" + "6" * 64, + "stream": "sha256:" + "6" * 64, "ts": "sha256:" + "3" * 64} + + +class ImageSetTests(unittest.TestCase): + def test_defaults_file_and_overrides(self): + import json + import os + import tempfile + self.assertEqual(g12.resolve_image_set(), g12.IMAGES) + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as handle: + json.dump(FINAL, handle) + try: + images = g12.resolve_image_set(handle.name, ["ts=sha256:" + "4" * 64]) + finally: + os.unlink(handle.name) + self.assertEqual(images["core"], FINAL["core"]) + self.assertEqual(images["ts"], "sha256:" + "4" * 64) # --image applies last + self.assertEqual(images["kafka"], g12.IMAGES["kafka"]) # unnamed roles keep the default + self.assertEqual(set(images), set(g12.IMAGES)) + + def test_rejects_unknown_role_mutable_ref_and_duplicates(self): + for bad in ({"querry": FINAL["python"]}, {"python": "qdl-v2-python:latest"}, {"core": "sha256:abc"}): + with self.subTest(bad=bad), self.assertRaises(ValueError): + g12.resolve_image_set(bad) + with self.assertRaises(ValueError): + g12.resolve_image_set(None, ["core=" + FINAL["core"], "core=" + FINAL["core"]]) + with self.assertRaises(ValueError): + g12.resolve_image_set(None, ["core"]) + + def test_topology_helpers_and_sampler_use_the_passed_set(self): + images = g12.resolve_image_set(FINAL) + roles = g12.topology(NS, "/home/bobby/x/work", "T", uid="1:1", images=images) + for role, argv in roles.items(): + with self.subTest(role=role): + self.assertIn(images[g12.role_image_key(role)], argv) + helper = g12.helper_argv(NS, "/home/bobby/x/work", ["--topics", "/work"], images=images) + self.assertIn(images["python"], helper) + env = [a for a in helper if a.startswith("P184_G12_IMAGES=")] + self.assertEqual(len(env), 1) + self.assertEqual(g12.json.loads(env[0].split("=", 1)[1]), images) + self.assertIn(images["ts"], g12.sampler_argv(NS, "/home/bobby/x/ev", 10, images=images)) + + def test_query_code_from_image_has_no_worktree_mount(self): + mounted = g12.topology(NS, "/home/bobby/x/work", uid="1:1")["query"] + self.assertIn(f"{g12.ROOT}:/app:ro", mounted) + own = g12.topology(NS, "/home/bobby/x/work", uid="1:1", query_code="image")["query"] + volumes = [own[i + 1] for i, a in enumerate(own) if a == "-v"] + self.assertFalse(any(v.split(":")[1] == "/app" for v in volumes), volumes) # image's own /app + self.assertEqual(own[own.index("-w") + 1], "/app") + with self.assertRaises(ValueError): + g12.topology(NS, "/home/bobby/x/work", uid="1:1", query_code="host") + + def test_role_image_key_and_dr05_roles(self): + from scripts import p184_dr08_isolated as dr08 + expected = {"net": "holder", "kafka-2": "kafka", "quota": "redis", "core": "core", "projector": "projector", + "stream": "stream", "query": "python", "monitor": "python", "publisher": "python", + "sampler": "ts", "tswriter-okx": "ts", "tsredis-binance": "redis", "tspg-okx": "postgres"} + for role, key in expected.items(): + self.assertEqual(g12.role_image_key(role), key, role) + images = g12.resolve_image_set(FINAL) + sealed = g12.dr05_images(images) + self.assertEqual(set(sealed), set(dr08.IMAGES)) # dr05 image_set() requires exactly these roles + self.assertEqual(sealed["core"], FINAL["core"]) + self.assertEqual(sealed["query"], FINAL["python"]) + + def test_in_container_check_compares_passed_and_sealed(self): + import json + import tempfile + from pathlib import Path + images = g12.resolve_image_set(FINAL) + with tempfile.TemporaryDirectory() as work: + handoff = Path(work) / "handoff.json" + handoff.write_text(json.dumps({"namespace": NS})) + self.assertIsNone(g12.check_sealed_images(work, None)) # sealed before the parameter existed + with self.assertRaises(ValueError): + g12.check_sealed_images(work, json.dumps(images)) + handoff.write_text(json.dumps({"namespace": NS, "g12_images": images})) + self.assertEqual(g12.check_sealed_images(work, json.dumps(images)), images) + for attested in (None, json.dumps(g12.IMAGES)): + with self.subTest(attested=attested), self.assertRaises(ValueError): + g12.check_sealed_images(work, attested) + + def test_cli_passes_the_image_set_to_the_orchestrator(self): + with patch.object(g12, "orchestrate", return_value=0) as orchestrate: + g12.main(["--run", "/home/bobby/x", "--export", "/home/bobby/e.json", "--namespace", NS, + "--image", "core=" + FINAL["core"], "--query-code", "image"]) + images, query_code = orchestrate.call_args.args[3:5] + self.assertEqual(images["core"], FINAL["core"]) + self.assertEqual(images["python"], g12.IMAGES["python"]) + self.assertEqual(query_code, "image") + with patch.object(g12, "orchestrate", return_value=0) as orchestrate: + g12.main(["--run", "/home/bobby/x", "--export", "/home/bobby/e.json", "--namespace", NS]) + self.assertEqual(orchestrate.call_args.args[3:5], (g12.IMAGES, "worktree")) + + if __name__ == "__main__": unittest.main() diff --git a/upgrade/evidence/p184-g12-isolated-fault-packet.md b/upgrade/evidence/p184-g12-isolated-fault-packet.md index e1504dca..4c83f33b 100644 --- a/upgrade/evidence/p184-g12-isolated-fault-packet.md +++ b/upgrade/evidence/p184-g12-isolated-fault-packet.md @@ -163,3 +163,48 @@ breaches (query_kn_1/2 were recreated by another operator at 03:43-03:44, before Limits: same-host loopback rehearsal, one projector and one Stream replica, synthetic TEST_ONLY BOOK at 4 Hz/venue (not production load), sampler resolution 50 ms. The monitor's "projector lag 0" predicate was wrong (committed offset vs LSO keeps 1 for the commit marker); the re-analysis uses <= 1, no G12 number changed. + +## 12. Final-image re-run packet - 2026-10-10 (written before the run) + +Same gate, faults, measurements, pass rule, stop conditions, deadlines and teardown as sections 1 and 3-10; only +the image set and the Query code source change. Base: `fix/p184-dl-closure` tip `59e725a` plus the harness +change below. Namespace `p184-g12-dl-00e555b46f67d7c0`; evidence `~/.local/state/qdl-v2/p184-g12-final-20261010/`. + +Harness change (additive): the image set is a parameter. `IMAGES` stay the defaults. +- `--image-set FILE` (JSON object `role -> sha256:<64hex>`, any subset of the roles kafka/core/projector/stream/ + python/redis/ts/postgres/holder) and `--image ROLE=sha256:<64hex>` (repeatable, applied last). +- The resolved set must match the local image IDs. It is written to `evidence/images.json` (ID, tags, OCI revision) + and sealed in `work/handoff.json`: `g12_images`, plus `images` in DR05 role names, so the in-container + `--material` step writes the final core digest into `core.json`. +- It is passed to every in-namespace helper as `P184_G12_IMAGES`, and `_packet` refuses a mismatch with the seal. +- Every container the run starts (restarts and the six WriterClone containers included) is `docker inspect`-ed. + Its image ID must equal the passed value for its role, or the run stops. Rows go to `evidence/provenance.jsonl`. +- `--query-code image` runs Query from the image's own `/app` (no worktree mount). The default `worktree` keeps + the section-2 behaviour. + +Final image set (`image-set.json` in the evidence base): + +| Role | Image ID | Tag / revision | +|---|---|---| +| core, projector, stream | `sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7` | qdl-v2-rust:2.2.3-rc7d-bbfb3be (= rust_core*, market_projector_1/2, stream_kn_1/2 now) | +| python (query; helper/publisher/monitor) | `sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a` | qdl-v2-python:2.2.3-rc7d-bbfb3be (= query_kn_1/2 now) | +| ts (writer clones, sampler) | `sha256:34202753bbe8b74d50e45afed1cf2e7b400141324c550168462b30db220edb31` | local/trading-system:p184-writer-ed02064 (= market_data_service_p184w now) | +| kafka, redis, postgres, holder | section-2 defaults | unchanged | + +Query runs the image's code (`--query-code image`). Helper, publisher and monitor are harness tooling: they run in +the final python image with the worktree harness mounted read-only. `git diff bbfb3be HEAD -- qdl generated app +scripts/p184_dr05_observer.py scripts/p184_dr08_isolated.py` is empty. + +Invocation (host, stdlib only): + +``` +python3 -B scripts/p184_g12_isolated_fault.py --run ~/.local/state/qdl-v2/p184-g12-final-20261010 \ + --export ~/.local/state/qdl-v2/dr05-export-20261010/public-export.json \ + --namespace p184-g12-dl-00e555b46f67d7c0 \ + --image-set ~/.local/state/qdl-v2/p184-g12-final-20261010/image-set.json --query-code image +``` + +Before starting: +- `ps -eo args | grep -E '/run_stage100.sh [a-z] '` must be empty. If a load run is active, wait and re-check every + 2 min; give up after 60 min. +- `df /` must show >= 20 GB free. From 2e0de95674c68d8258ef6517c50641ef7648b374 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 16:06:13 +0000 Subject: [PATCH 124/147] fix(query): funding settlement publication allowance 60 s, now backed by measurement (OKX +10 s, Binance +5..+24 s at 16:00Z) --- qdl/query/service.py | 14 +++++++++++--- tests/test_phase113_reference_v2.py | 25 +++++++++++++++---------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/qdl/query/service.py b/qdl/query/service.py index e3e23e0f..d072690b 100644 --- a/qdl/query/service.py +++ b/qdl/query/service.py @@ -73,6 +73,11 @@ # Binance adapter (_FUNDING_BOUNDARY_TOLERANCE_MS) and the acceptance window # (_FUNDING_SETTLEMENT_JITTER_NS) already use; it applies to FUNDING_RATE # history only, and a settlement still unpublished after it stays DATA_STALE. +# Funding settlement publication allowance. Measured 2026-10-10 16:00Z (read-only public API poll, ~4.6 s cadence): +# the settled row appeared in history OKX BTC/ETH/SOL-USDT-SWAP +10 s (settState processing at +5 s), Binance +# BTC/SOL/ETH +5/+10/+24 s. 60 s = the existing Binance funding boundary tolerance, ~2.5x the observed max. +# Evidence: ~/.local/state/qdl-v2/funding-publication-20261010/. +_FUNDING_SETTLEMENT_PUBLICATION_GRACE_MS = 60_000 _BatchCompletion = TypeVar("_BatchCompletion") @@ -1756,9 +1761,12 @@ def _reference_problem( 0, ((self._clock_ns() if at_ns is None else at_ns) - observed_ns) // 1_000_000, ) - # No funding settlement allowance until the OKX publication delay is measured and the refused item code - # is recorded (Codex review 2026-10-10: the stage-100g cause is probable, not proven). - if freshness_ms > requirement.max_freshness_ms: + allowance_ms = ( + _FUNDING_SETTLEMENT_PUBLICATION_GRACE_MS + if request.product is ReferenceProduct.FUNDING_RATE and request.is_history + else 0 + ) + if freshness_ms > requirement.max_freshness_ms + allowance_ms: return QueryProblem( CanonicalErrorCode.DATA_STALE, "reference provider result exceeds the declared freshness bound", diff --git a/tests/test_phase113_reference_v2.py b/tests/test_phase113_reference_v2.py index 8b1c251a..b3e3d503 100644 --- a/tests/test_phase113_reference_v2.py +++ b/tests/test_phase113_reference_v2.py @@ -472,7 +472,9 @@ async def test_cached_reference_result_matches_by_value_not_python_identity(self async def test_stale_reference_result_fails_closed(self): registry = InstrumentRegistry() registry.register(self.binance, []) - adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - 101_000_000) + # Beyond the declared bound plus the 60 s funding settlement + # publication allowance (stage 100g, 2026-10-10). + adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - 60_101_000_000) stale = V2QueryService( instruments=InstrumentQuery(registry), backend=MemoryMarketDataBackend(), @@ -518,16 +520,19 @@ async def _funding_read(self, service): purpose=AccessPurpose.INTERNAL_ALPHA, ) - async def test_funding_bound_stays_strict_until_publication_delay_is_measured(self): - """Stage 100g (2026-10-10) refused OKX SOL-USDT-SWAP funding 7 s after the 08:00Z settlement. The cause is - probable, not proven (Codex review), so no settlement allowance is granted: the declared bound is exact.""" + async def test_funding_settlement_due_but_unpublished_is_not_stale(self): + """Stage 100g (2026-10-10): an 8 h bound on 8 h OKX funding refused SOL-USDT-SWAP at + 08:00:07Z, seven seconds after the settlement fell due and before the venue published it.""" clock = {"ns": NOW_NS} - for past_ms in (7_000, 60_001): - adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - (_FUNDING_8H_MS + past_ms) * 1_000_000) - result = await self._funding_read(self._funding_service(adapter, clock)) - self.assertEqual(result.results[0].problem.code.value, "DATA_STALE") - self.assertTrue(result.results[0].problem.retryable) - self.assertEqual(adapter.calls, 1) # a provider-fresh stale series is not re-read + adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - (_FUNDING_8H_MS + 7_000) * 1_000_000) + result = await self._funding_read(self._funding_service(adapter, clock)) + self.assertIsNone(result.results[0].problem) + # The allowance is bounded: a settlement still unpublished after it stays fail-closed. + late = FixtureReferenceAdapter(observed_at_ns=NOW_NS - (_FUNDING_8H_MS + 60_001) * 1_000_000) + result = await self._funding_read(self._funding_service(late, clock)) + self.assertEqual(result.results[0].problem.code.value, "DATA_STALE") + self.assertTrue(result.results[0].problem.retryable) + self.assertEqual(late.calls, 1) # a provider-fresh stale series is not re-read async def test_a_cached_funding_series_is_re_read_once_when_it_crosses_its_bound(self): clock = {"ns": NOW_NS} From 7188b8bdff3c05a076eb9293ffa1277d7cf78205 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 16:07:42 +0000 Subject: [PATCH 125/147] fix(p184-g12): bounded retry of the post-create topic metadata race; SIGTERM/SIGHUP/SIGINT run evidence and teardown --- scripts/p184_g12_isolated_fault.py | 23 ++++++++++- tests/test_p184_g12_isolated_fault.py | 55 +++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/scripts/p184_g12_isolated_fault.py b/scripts/p184_g12_isolated_fault.py index ba40f2b5..b3499fa1 100644 --- a/scripts/p184_g12_isolated_fault.py +++ b/scripts/p184_g12_isolated_fault.py @@ -647,12 +647,26 @@ def create_topics(work): for future in futures.values(): future.result(40) canonical = packet["namespace"] + ".canonical" - described = admin.describe_topics(TopicCollection([canonical]))[canonical].result(20) + described = describe_until_known(lambda: admin.describe_topics(TopicCollection([canonical]))[canonical] + .result(20)) packet["slice"]["topic_id"] = str(described.topic_id) (Path(work) / "packet.json").write_text(json.dumps(packet)) return {"topics": sorted(wanted), "replication_factor": len(BROKERS), "min_isr": 2} +def describe_until_known(describe, seconds=30.0, pause=1.0): + """A topic just created may not be in every broker's metadata yet (final-image run + 2026-10-10, attempt 1: UNKNOWN_TOPIC_OR_PART right after create); retry only that, bounded.""" + deadline = time.monotonic() + seconds + while True: + try: + return describe() + except Exception as error: # noqa: BLE001 - only the propagation race is retried + if "UNKNOWN_TOPIC_OR_PART" not in str(error) or time.monotonic() >= deadline: + raise + time.sleep(pause) + + class _Stop: def __init__(self): self.requested = False @@ -1276,7 +1290,12 @@ def orchestrate(base, export, ns, images=None, query_code="worktree"): def expire(_signum, _frame): raise TimeoutError("G12 orchestrator absolute deadline") + def terminated(signum, _frame): + # A session end / kill (SIGTERM, SIGHUP, SIGINT) still runs evidence + teardown in finally + # (final-image attempt 2, 2026-10-10: the orchestrator died with the session, 18 roles orphaned). + raise SystemExit(f"G12 orchestrator received signal {signum}") previous = signal.signal(signal.SIGALRM, expire) + previous_term = {sig: signal.signal(sig, terminated) for sig in (signal.SIGTERM, signal.SIGHUP, signal.SIGINT)} signal.alarm(LIMITS["deadline_s"]) status, error = "FAIL", None try: @@ -1303,6 +1322,8 @@ def expire(_signum, _frame): finally: run.down() if run.evidence.exists() else teardown(ns) signal.signal(signal.SIGALRM, previous) + for sig, handler in previous_term.items(): + signal.signal(sig, handler) return 0 if status == "COMPLETED" else 1 diff --git a/tests/test_p184_g12_isolated_fault.py b/tests/test_p184_g12_isolated_fault.py index 23db97f5..d2a77a25 100644 --- a/tests/test_p184_g12_isolated_fault.py +++ b/tests/test_p184_g12_isolated_fault.py @@ -304,5 +304,60 @@ def test_cli_passes_the_image_set_to_the_orchestrator(self): self.assertEqual(orchestrate.call_args.args[3:5], (g12.IMAGES, "worktree")) +class TopicReadinessTests(unittest.TestCase): + def test_describe_retries_only_the_metadata_race(self): + calls = [] + + def flaky(): + calls.append(1) + if len(calls) < 3: + raise RuntimeError('KafkaError{code=UNKNOWN_TOPIC_OR_PART,val=3}') + return "described" + self.assertEqual(g12.describe_until_known(flaky, seconds=5, pause=0), "described") + self.assertEqual(len(calls), 3) + + def other(): + raise RuntimeError("KafkaError{code=TOPIC_AUTHORIZATION_FAILED}") + with self.assertRaises(RuntimeError): + g12.describe_until_known(other, seconds=5, pause=0) + + def never(): + raise RuntimeError("UNKNOWN_TOPIC_OR_PART") + with self.assertRaises(RuntimeError): + g12.describe_until_known(never, seconds=0, pause=0) + + + +class OrchestratorSignalTests(unittest.TestCase): + def test_sigterm_still_writes_verdict_and_tears_down(self): + import os + import signal + import tempfile + from pathlib import Path + calls = [] + + class FakeRun: + def __init__(self, base, *_): + self.evidence = Path(base) / "evidence" + self.evidence.mkdir(parents=True) + self.images, self.query_code = g12.IMAGES, "worktree" + + def prepare(self): + os.kill(os.getpid(), signal.SIGTERM) + + def note(self, step, **fields): + calls.append(step) + + def down(self): + calls.append("down") + before = signal.getsignal(signal.SIGTERM) + with tempfile.TemporaryDirectory() as base, patch.object(g12, "Run", FakeRun): + self.assertEqual(g12.orchestrate(base, "/x", NS), 1) + verdict = g12.json.loads((Path(base) / "evidence" / "verdict.json").read_text()) + self.assertIn("signal", verdict["error"]) + self.assertEqual(calls, ["stopped", "down"]) + self.assertIs(signal.getsignal(signal.SIGTERM), before) # handlers restored + + if __name__ == "__main__": unittest.main() From 8bbcaae188b2e4bf8691ec0412760986d8f2158b Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 16:16:24 +0000 Subject: [PATCH 126/147] fix(p184-g12): detached in-container roles keep stderr and faulthandler dumps beside their evidence; monitor records its exit reason --- scripts/p184_g12_isolated_fault.py | 20 +++++++++++++++++++- tests/test_p184_g12_isolated_fault.py | 20 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/scripts/p184_g12_isolated_fault.py b/scripts/p184_g12_isolated_fault.py index b3499fa1..7db870e1 100644 --- a/scripts/p184_g12_isolated_fault.py +++ b/scripts/p184_g12_isolated_fault.py @@ -815,6 +815,8 @@ def monitor(work, out, seconds, period=1.0): row["projector_status_mtime"] = None log.write(json.dumps(row, default=list) + "\n") time.sleep(max(0.0, period - (time.monotonic() - started))) + log.write(json.dumps({"k": "end", "t": time.time_ns(), + "reason": "SIGTERM" if stop.requested else "seconds"}) + "\n") return {"stopped": True} @@ -1327,6 +1329,20 @@ def terminated(signum, _frame): return 0 if status == "COMPLETED" else 1 +def keep_diagnostics(out): + """Detached --rm roles lose their stderr when they exit: write it (and a native + crash's faulthandler dump) next to the evidence file (final-image attempt 3, + 2026-10-10: the monitor exited during the broker fault with no log left).""" + if not out: + return None + import faulthandler + stream = open(str(out) + ".stderr", "a", buffering=1) + sys.stderr = stream + faulthandler.enable(file=stream, all_threads=True) + stream.write(json.dumps({"k": "start", "t": time.time_ns(), "pid": os.getpid()}) + "\n") + return stream + + def main(argv=None): parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) mode = parser.add_mutually_exclusive_group(required=True) @@ -1360,7 +1376,9 @@ def main(argv=None): parser.error("--seconds required") if args.topics: print(json.dumps(create_topics(args.topics))) - elif args.publish: + return 0 + keep_diagnostics(args.out) + if args.publish: print(json.dumps(publish(args.publish, args.out, args.seconds, args.hz))) else: print(json.dumps(monitor(args.monitor, args.out, args.seconds))) diff --git a/tests/test_p184_g12_isolated_fault.py b/tests/test_p184_g12_isolated_fault.py index d2a77a25..3ebf0ddf 100644 --- a/tests/test_p184_g12_isolated_fault.py +++ b/tests/test_p184_g12_isolated_fault.py @@ -358,6 +358,26 @@ def down(self): self.assertEqual(calls, ["stopped", "down"]) self.assertIs(signal.getsignal(signal.SIGTERM), before) # handlers restored + def test_detached_roles_keep_stderr_next_to_evidence(self): + import sys + import tempfile + from pathlib import Path + self.assertIsNone(g12.keep_diagnostics(None)) + saved = sys.stderr + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "monitor.jsonl" + stream = g12.keep_diagnostics(out) + try: + sys.stderr.write("boom\n") + finally: + import faulthandler + faulthandler.disable() + sys.stderr = saved + stream.close() + text = (Path(tmp) / "monitor.jsonl.stderr").read_text() + self.assertIn('"k": "start"', text) + self.assertIn("boom", text) + if __name__ == "__main__": unittest.main() From 848d5d2089a59f10694b7db06ac2fc3bc8f9c034 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 16:24:51 +0000 Subject: [PATCH 127/147] feat(p184): driver records G6 view age/quality per binding and wall-clock load windows; trace analysis counts duplicate applies and loads slim G6: every view the Query endpoint returns to the stage-100 driver is classified BEFORE client validation against the frozen per-feed policy (TRADE 3000, QUOTE/MARK/BOOK_DELTA 2000, BOOK_SNAPSHOT 60000, BAR1m 180000 ms; other BAR intervals their sealed requirement): ELIGIBLE, ELIGIBLE_QUIET (OBSERVE quiet continuity with a live session), NOT_ELIGIBLE or UNSAFE (eligible although the policy rejects it). Exact counts/max per binding, bounded reservoir for p50/p95, merged across workers into receipt g6_view_quality. No change to the request mix, the validation or the gates. Receipt adds load_window_unix_s and BURST/RECONNECT wall-clock windows so host probes score the same window. Trace analysis: duplicate_applied_event_id (G10, canonical event_id APPLIED twice) and a field-projected loader so a 40-min campaign (~250 parts) fits in memory; the MARK script loads only MARK records. --- scripts/p184_g2_mark_trace_transit.py | 10 +- scripts/p184_trace_feed_transit.py | 12 +- scripts/phase3_consumer_load_acceptance.py | 207 ++++++++++++++++++++- tests/test_p184_g6_view_quality.py | 136 ++++++++++++++ tests/test_p184_trace_feed_transit.py | 20 ++ 5 files changed, 381 insertions(+), 4 deletions(-) create mode 100644 tests/test_p184_g6_view_quality.py diff --git a/scripts/p184_g2_mark_trace_transit.py b/scripts/p184_g2_mark_trace_transit.py index e22e3ed9..9896b033 100644 --- a/scripts/p184_g2_mark_trace_transit.py +++ b/scripts/p184_g2_mark_trace_transit.py @@ -51,7 +51,9 @@ def venue_of(instrument): return instrument.replace(".", ":").split(":", 1)[0].upper() -def load(paths): +def load(paths, *, feeds=None, fields=None): + """Read export files; feeds/fields (optional) keep only those records/keys so a 40-min campaign of ~250 parts + fits in memory (every record still counts in the per-file metadata).""" files, records = [], [] for path in paths: with open(path) as handle: @@ -62,6 +64,10 @@ def load(paths): "stop_reason": report.get("stop_reason"), "recorded": report.get("recorded"), "dropped_at_export": report.get("dropped_at_export"), "first_start_ns": min(starts) if starts else None, "last_start_ns": max(starts) if starts else None}) + if feeds is not None: + rows = [r for r in rows if r.get("feed") in feeds] + if fields is not None: + rows = [{k: r[k] for k in fields if k in r} for r in rows] records.extend(rows) return files, records @@ -141,7 +147,7 @@ def main(argv=None): parser.add_argument("--out") args = parser.parse_args(argv) paths = sorted({path for pattern in args.inputs for path in (glob.glob(pattern) or [pattern])}) - files, records = load(paths) + files, records = load(paths, feeds={FEED}) report = {"schema": "qdl.p184.g2-mark-trace-transit.v1", "files": len(files), "records": len(records), "dropped_at_export_total": sum(f["dropped_at_export"] or 0 for f in files), "stop_reasons": dict(collections.Counter(f["stop_reason"] for f in files)), diff --git a/scripts/p184_trace_feed_transit.py b/scripts/p184_trace_feed_transit.py index 9e43ca32..f7583859 100644 --- a/scripts/p184_trace_feed_transit.py +++ b/scripts/p184_trace_feed_transit.py @@ -14,6 +14,8 @@ Definitions: * venue clock = source_event_time_ns of the SDK stream envelope; only SDK_STREAM_RETURN records carry it. Records of other bases (warmup, query, no basis) have no venue clock and are counted, never given a substitute clock; +* duplicate_applied_event_id (G10) = APPLIED records whose canonical event_id digest was already APPLIED earlier in the + window, per feed and venue (records without a digest are not counted, their fallback identity is not unique); * new-event transit = redis_reply_wall_ns - source_event_time_ns for the first APPLIED record of each event identity (event_id_sha256; when absent, instrument + feed + source_event_time_ns). REFRESHED and DUPLICATE outcomes and a repeated identity are the same state re-published (IDR-082 quiet republish) and are counted, not gated; @@ -53,6 +55,10 @@ } G7_GATES = {"sdk_to_writer_ms": 100.0, "writer_to_redis_ms": 50.0} STREAM_BASIS = "SDK_STREAM_RETURN" +# Keys summarise() reads; loading only these keeps a full campaign (~250 parts x 4096 records) in memory. +FIELDS = ("feed", "canonical_instrument_id", "outcome", "sdk_receive_basis", "sdk_receive_monotonic_ns", + "writer_started_monotonic_ns", "redis_reply_monotonic_ns", "writer_started_wall_ns", "redis_reply_wall_ns", + "source_event_time_ns", "event_id_sha256", "envelope_received_at_ns", "sdk_receive_wall_ns") def transit_verdict(block, gate): @@ -79,6 +85,7 @@ def summarise(records, *, skip_seconds=40.0, from_ns=None, to_ns=None): split = collections.defaultdict(lambda: collections.defaultdict(list)) segments = collections.defaultdict(lambda: collections.defaultdict(list)) seen = set() + duplicates = collections.Counter() for r in rows: feed = r.get("feed", "?") venue = venue_of(r.get("canonical_instrument_id", "?")) @@ -117,6 +124,8 @@ def summarise(records, *, skip_seconds=40.0, from_ns=None, to_ns=None): else (r.get("canonical_instrument_id"), feed, event_time)) if (feed, identity) in seen: excluded[(feed, venue, "repeated_event_identity")] += 1 + if identity is digest: # G10: the same canonical event_id APPLIED twice (fallback identities excluded) + duplicates[(feed, venue)] += 1 continue seen.add((feed, identity)) value = (reply_wall - event_time) / 1e6 @@ -148,6 +157,7 @@ def summarise(records, *, skip_seconds=40.0, from_ns=None, to_ns=None): for name, limit in G7_GATES.items()} result["g7"][key] = block result["bar_records"] = sum(1 for r in rows if r.get("feed") == "BAR") + result["duplicate_applied_event_id"] = {"|".join(k): n for k, n in sorted(duplicates.items())} result["outcomes"] = {"|".join(str(p) for p in k): n for k, n in sorted(outcomes.items(), key=str)} result["excluded_from_transit"] = {"|".join(k): n for k, n in sorted(excluded.items())} result["invalid"] = {"|".join(k): n for k, n in sorted(invalid.items())} @@ -164,7 +174,7 @@ def main(argv=None): parser.add_argument("--out") args = parser.parse_args(argv) paths = sorted({path for pattern in args.inputs for path in (glob.glob(pattern) or [pattern])}) - files, records = load(paths) + files, records = load(paths, fields=FIELDS) report = {"schema": "qdl.p184.trace-feed-transit.v1", "files": len(files), "records": len(records), "dropped_at_export_total": sum(f["dropped_at_export"] or 0 for f in files), "stop_reasons": dict(collections.Counter(f["stop_reason"] for f in files)), diff --git a/scripts/phase3_consumer_load_acceptance.py b/scripts/phase3_consumer_load_acceptance.py index c5fcfec3..257940b1 100644 --- a/scripts/phase3_consumer_load_acceptance.py +++ b/scripts/phase3_consumer_load_acceptance.py @@ -1779,6 +1779,174 @@ async def main(): """ +# P18.4 gate G6 (frozen TS gate table): durable-view event age and typed execution quality of every view the Query +# endpoint returns, per binding, against the per-feed policy, BEFORE client validation so that an eligible view the +# policy would reject is counted as UNSAFE instead of disappearing into a refusal count. BAR intervals other than 1m +# use their own sealed requirement (max_freshness_ms). Event age is the server's quality.freshness_ms for snapshot +# views and (now - newest observation) for MARK reference results. Bounded: exact counts and max per binding, a +# reservoir of at most _G6_RESERVOIR ages per binding and worker for p50/p95. +_G6_POLICY_MS = {"TRADE": 3000, "QUOTE": 2000, "MARK_INDEX_PRICE": 2000, "BOOK_DELTA": 2000, + "BOOK_SNAPSHOT": 60000, "BAR|1m": 180000} +_G6_RESERVOIR = 2048 +_G6_VIEW: ContextVar[list | None] = ContextVar("qdl_phase3_g6_view", default=None) + + +def g6_policy_ms(feed: str, interval: str | None, requirement_max_ms: int | None) -> int | None: + if feed == "BAR": + return _G6_POLICY_MS.get(f"BAR|{interval}", requirement_max_ms) + return _G6_POLICY_MS.get(feed, requirement_max_ms) + + +def g6_classify(*, feed: str, freshness_ms: int | None, eligible: bool, state: str | None, gap_open: bool, + complete: bool, event_recency_state: str | None, provider_session_state: str | None, + session_liveness_ms: int | None, max_session_liveness_ms: int | None, observe: bool, + on_change: bool, policy_ms: int | None) -> tuple[str, list[str]]: + """ELIGIBLE / ELIGIBLE_QUIET (beyond the event threshold with a proven live session under OBSERVE) / + NOT_ELIGIBLE / UNSAFE (eligible although the policy rejects it). Pure: unit-tested.""" + beyond = freshness_ms is not None and policy_ms is not None and freshness_ms > policy_ms + session_ok = (provider_session_state == "LIVE" + and (max_session_liveness_ms is None + or (session_liveness_ms is not None and session_liveness_ms <= max_session_liveness_ms))) + quiet_proven = (observe and (feed in {"TRADE", "BOOK_DELTA"} or (feed == "QUOTE" and on_change)) + and event_recency_state == "STALE" and session_ok) + if not eligible: + return "NOT_ELIGIBLE", [] + reasons = [] + if gap_open: + reasons.append("GAP_OPEN") + if not complete: + reasons.append("INCOMPLETE") + if state not in (None, "LIVE"): + reasons.append(f"STATE_{state}") + if freshness_ms is None: + reasons.append("AGE_UNKNOWN") + elif beyond and not quiet_proven: + reasons.append("AGE_BEYOND_POLICY") + if (max_session_liveness_ms is not None and provider_session_state not in (None, "NOT_APPLICABLE") + and not session_ok): + reasons.append("SESSION_BEYOND_POLICY") + if reasons: + return "UNSAFE", reasons + return ("ELIGIBLE_QUIET" if beyond else "ELIGIBLE"), [] + + +def _g6_key(product) -> str: + return f"{product.feed.value}|{product.venue}|{product.interval or ''}" + + +def _g6_snapshot_observation(product, requirement, view) -> dict[str, object]: + quality = view.quality + policy = getattr(requirement, "effective_event_recency_policy", None) + policy_ms = g6_policy_ms(product.feed.value, product.interval, getattr(requirement, "max_freshness_ms", None)) + verdict, reasons = g6_classify( + feed=product.feed.value, freshness_ms=quality.freshness_ms, eligible=bool(quality.execution_eligible), + state=quality.state, gap_open=bool(quality.gap_open), complete=bool(quality.complete), + event_recency_state=quality.event_recency_state, provider_session_state=quality.provider_session_state, + session_liveness_ms=quality.provider_session_liveness_ms, + max_session_liveness_ms=getattr(requirement, "max_session_liveness_ms", None), + observe=getattr(policy, "value", policy) == "OBSERVE", + on_change="DELIVERY_ON_CHANGE" in (quality.flags or []), policy_ms=policy_ms) + return {"key": _g6_key(product), "age_ms": quality.freshness_ms, "policy_ms": policy_ms, "class": verdict, + "reasons": reasons, "state": quality.state, "recency": quality.event_recency_state} + + +def _g6_reference_observations(references, response, *, now_ns: int) -> list[dict[str, object]]: + from qdl.certification.reference_l2_acceptance import _quiet_execution_mark_index_evidence + + out = [] + for reference, item in zip(references, response.results): + product = reference + key = f"{product.requirement.feed.value}|{product.venue}|" + policy_ms = g6_policy_ms(product.requirement.feed.value, None, product.sdk_requirement.max_freshness_ms) + data = getattr(item, "data", None) + observations = getattr(data, "observations", None) or [] + if getattr(item, "status", None) != "OK" or not observations: + code = getattr(getattr(item, "problem", None), "code", None) or "NO_OBSERVATION" + out.append({"key": key, "age_ms": None, "policy_ms": policy_ms, "class": "NOT_ELIGIBLE", + "reasons": [str(code)[:64]]}) + continue + age_ms = max(0, (now_ns - max(o.observed_at_ns for o in observations)) // 1_000_000) + try: + quiet = _quiet_execution_mark_index_evidence(product, item, observed_at_ns=now_ns) is not None + except Exception: + quiet = False + beyond = policy_ms is not None and age_ms > policy_ms + verdict = "UNSAFE" if beyond and not quiet else "ELIGIBLE_QUIET" if beyond else "ELIGIBLE" + out.append({"key": key, "age_ms": age_ms, "policy_ms": policy_ms, "class": verdict, + "reasons": ["STALE_SERVED_WITHOUT_QUIET_PROOF"] if verdict == "UNSAFE" else []}) + return out + + +class _G6Recorder: + def __init__(self, reservoir: int = _G6_RESERVOIR) -> None: + self.reservoir = reservoir + self.rows: dict[str, dict[str, object]] = {} + + def _row(self, key: str) -> dict[str, object]: + return self.rows.setdefault(key, {"n": 0, "seen_ages": 0, "ages": [], "max_age_ms": None, "policy_ms": None, + "classes": Counter(), "reasons": Counter(), "refusals": Counter(), + "windows": Counter(), "unsafe_examples": []}) + + def view(self, observation: dict[str, object], window: str) -> None: + row = self._row(observation["key"]) + row["n"] += 1 + row["windows"][window] += 1 + row["policy_ms"] = observation.get("policy_ms") + row["classes"][observation["class"]] += 1 + for reason in observation.get("reasons") or []: + row["reasons"][reason] += 1 + age = observation.get("age_ms") + if isinstance(age, int): + row["seen_ages"] += 1 + row["max_age_ms"] = age if row["max_age_ms"] is None else max(row["max_age_ms"], age) + if len(row["ages"]) < self.reservoir: + row["ages"].append(age) + else: + slot = random.randrange(row["seen_ages"]) + if slot < self.reservoir: + row["ages"][slot] = age + if observation["class"] == "UNSAFE" and len(row["unsafe_examples"]) < 8: + row["unsafe_examples"].append({k: observation.get(k) for k in ("age_ms", "reasons", "state", "recency")} + | {"window": window, "at_ns": time.time_ns()}) + + def refusal(self, key: str, code: str, window: str) -> None: + row = self._row(key) + row["refusals"][code] += 1 + row["windows"][window] += 1 + + def evidence(self) -> dict[str, object]: + return {key: {**row, "classes": dict(row["classes"]), "reasons": dict(row["reasons"]), + "refusals": dict(row["refusals"]), "windows": dict(row["windows"])} + for key, row in self.rows.items()} + + +def merge_g6(blocks: list[dict[str, object]]) -> dict[str, object]: + """Merge per-worker G6 evidence: exact counts/max, pooled bounded reservoirs for p50/p95.""" + merged: dict[str, dict[str, object]] = {} + for block in blocks: + for key, row in (block or {}).items(): + out = merged.setdefault(key, {"n": 0, "ages": [], "max_age_ms": None, "policy_ms": row.get("policy_ms"), + "classes": Counter(), "reasons": Counter(), "refusals": Counter(), + "windows": Counter(), "unsafe_examples": []}) + out["n"] += row["n"] + out["ages"].extend(row["ages"]) + if row.get("max_age_ms") is not None: + out["max_age_ms"] = (row["max_age_ms"] if out["max_age_ms"] is None + else max(out["max_age_ms"], row["max_age_ms"])) + for name in ("classes", "reasons", "refusals", "windows"): + out[name].update(row[name]) + out["unsafe_examples"].extend(row["unsafe_examples"][:8]) + result = {} + for key, row in sorted(merged.items()): + ages = row.pop("ages") + result[key] = {**{k: (dict(v) if isinstance(v, Counter) else v) for k, v in row.items()}, + "age_ms": {**_percentiles([float(a) for a in ages]), + "basis": "reservoir sample per worker (exact n and max above)"}, + "unsafe": row["classes"].get("UNSAFE", 0)} + result[key]["unsafe_examples"] = result[key]["unsafe_examples"][:16] + return result + + def target_worker_count(sessions: int) -> int: return min(4, max(1, math.ceil(sessions / 13))) @@ -1859,6 +2027,10 @@ async def _target_read(client, operation: str, products) -> object: if operation == "REFERENCE_BATCH": references = [_reference_product(product, now_ns=time.time_ns()) for product in products] response = await client.reference_batch([item.sdk_requirement for item in references], require_all=True) + try: + _G6_VIEW.set(_g6_reference_observations(references, response, now_ns=time.time_ns())) + except Exception: + _G6_VIEW.set(None) # diagnostics never change the read result if response.partial or len(response.results) != len(references): raise ValueError("PARTIAL_RESULT") for reference, result in zip(references, response.results, strict=True): @@ -1873,6 +2045,10 @@ async def _target_read(client, operation: str, products) -> object: # reads, and would make every latest read judge a 10,000-row horizon. requirement = replace(requirement, warmup_limit=0) response = await client.snapshot(requirement) + try: + _G6_VIEW.set([_g6_snapshot_observation(product, product.requirement, response.data)]) + except Exception: + _G6_VIEW.set(None) # diagnostics never change the read result validate_product_view(product, response.data, require_current_quality=True) return response @@ -1925,6 +2101,7 @@ def __init__(self, *, windows: dict[str, tuple[float, float]]) -> None: # Bounded timeline of slow reads and scheduler lag, in wall-clock # milliseconds, so a tail can be matched to server-side samples. self.outliers: list[list[object]] = [] + self.g6 = _G6Recorder() def outlier(self, kind: str, value_ms: float, label: str) -> None: if len(self.outliers) < 400: @@ -1953,6 +2130,19 @@ def read(self, *, operation: str, products, elapsed_ms: float, window: str) -> N self.groups["|".join((name, product.venue, symbols, _SERVED_BY.get() or "?", window))].append(value) +def _g6_record(recorder, products, window: str, error_code: str | None) -> None: + """Count every returned view (also one the client then refused); a read with no view is a typed refusal.""" + try: + observations = _G6_VIEW.get() + if observations: + for item in observations: + recorder.g6.view(item, window) + elif error_code is not None: + recorder.g6.refusal(_g6_key(products[0]), error_code, window) + except Exception: + pass + + async def _target_poll_loop(*, client, label, operation, products, period, phase, start, end, recorder, stop=None) -> None: from qdl.certification.phase3_consumer_load import DeclaredRateTicker, PollLedger @@ -1981,6 +2171,7 @@ async def _target_poll_loop(*, client, label, operation, products, period, phase ledger.sent += 1 window = recorder.window(sent) _SERVED_BY.set(None) + _G6_VIEW.set(None) sent_ns = time.time_ns() try: await _target_read(client, operation, products) @@ -1989,12 +2180,14 @@ async def _target_poll_loop(*, client, label, operation, products, period, phase except Exception as error: failed_ns = time.time_ns() ledger.record_failure(_error_code(error)) + _g6_record(recorder, products, window, _error_code(error)) recorder.error(operation=operation, poll=label, window=window, product=_product_evidence(products[0]), error=_safe_error(error), served_by=_SERVED_BY.get(), sent_ns=sent_ns, failed_ns=failed_ns, elapsed_ms=round((failed_ns - sent_ns) / 1e6, 3)) continue done = time.monotonic() + _g6_record(recorder, products, window, None) ledger.completed += 1 if done > due + period: ledger.late += 1 @@ -2384,6 +2577,7 @@ async def establish(session) -> None: poll_tasks: list[asyncio.Task] = [] setup_seconds: float | None = None start: float | None = None + go_unix: float | None = None try: results = await asyncio.gather(*(establish(item) for item in mine), return_exceptions=True) for session, result in zip(mine, results, strict=True): @@ -2411,7 +2605,8 @@ async def establish(session) -> None: line = await asyncio.get_running_loop().run_in_executor(None, sys.stdin.readline) if not line.startswith("GO "): raise RuntimeError("target worker did not receive a start signal") - start = time.monotonic() + (float(line.split()[1]) - time.time()) + go_unix = float(line.split()[1]) + start = time.monotonic() + (go_unix - time.time()) end = start + duration if final: burst = final_cfg["burst"] @@ -2503,12 +2698,18 @@ async def reconnect_later(): for key, value in series_by_session.items()}, "cold": recorder.cold, "outliers": recorder.outliers, + "g6_view_quality": recorder.g6.evidence(), "counters": dict(recorder.counters), "errors": recorder.errors, "error_count": recorder.error_count, "leaked_tasks": len(leaked), "fault_windows": ({} if start is None else {name: [round(a - start, 3), round(b - start, 3)] for name, (a, b) in windows.items()}), + # Host-comparable wall clock of the load window and fault windows (same host clock as the campaign probes). + "load_window_unix_s": None if go_unix is None else [go_unix, go_unix + duration], + "fault_windows_unix_s": ({} if go_unix is None else + {name: [round(go_unix + a - start, 3), round(go_unix + b - start, 3)] + for name, (a, b) in windows.items()}), "cpu_seconds": round(usage.ru_utime + usage.ru_stime, 3), "max_rss_kib": usage.ru_maxrss, } @@ -2536,6 +2737,8 @@ def _merge_target(results: list[dict[str, object]], *, plan, final: bool) -> dic worker0 = next((item for item in results if item["worker"] == 0), {}) windows = { "BURST": worker0.get("fault_windows", {}).get("BURST"), + "BURST_unix_s": worker0.get("fault_windows_unix_s", {}).get("BURST"), + "RECONNECT_unix_s": worker0.get("fault_windows_unix_s", {}).get("RECONNECT"), "SLOW_READER": sum(item["counters"].get("slow_reader_pauses", 0) for item in results), "RECONNECT": sum(item["reconnects"] for item in streams), "reconnect_restored": sum(item["restored"] for item in streams), @@ -2590,6 +2793,8 @@ def _merge_target(results: list[dict[str, object]], *, plan, final: bool) -> dic "bar_series": {key: value for result in results for key, value in result["bar_series"].items()}, "cold": [item for result in results for item in result["cold"]], "outliers": sorted(entry for result in results for entry in result.get("outliers", [])), + "g6_view_quality": merge_g6([result.get("g6_view_quality") for result in results]), + "load_window_unix_s": next((item.get("load_window_unix_s") for item in results if item["worker"] == 0), None), "counters": dict(sum((Counter(item["counters"]) for item in results), Counter())), "errors": [item for result in results for item in result["errors"]][:40], "error_count": sum(item["error_count"] for item in results), diff --git a/tests/test_p184_g6_view_quality.py b/tests/test_p184_g6_view_quality.py new file mode 100644 index 00000000..39105815 --- /dev/null +++ b/tests/test_p184_g6_view_quality.py @@ -0,0 +1,136 @@ +"""P18.4 G6 instrumentation of the stage-100 driver: per-binding view age and typed eligibility (no network). + +Synthetic views only (test provenance). Pins: the policy table, the UNSAFE rule (eligible although the policy +rejects it), quiet-continuity exceptions, bounded per-worker evidence, merging, and that the poll loop records a view +the client later refused exactly once. +""" +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +import importlib.util +from pathlib import Path +import sys +import time +import unittest +from unittest.mock import patch + +_ROOT = Path(__file__).resolve().parents[1] +_SPEC = importlib.util.spec_from_file_location("phase3_driver_g6", _ROOT / "scripts/phase3_consumer_load_acceptance.py") +_DRIVER = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _DRIVER +_SPEC.loader.exec_module(_DRIVER) + + +def classify(**overrides): + base = dict(feed="QUOTE", freshness_ms=100, eligible=True, state="LIVE", gap_open=False, complete=True, + event_recency_state="LIVE", provider_session_state="LIVE", session_liveness_ms=500, + max_session_liveness_ms=2000, observe=True, on_change=False, policy_ms=2000) + base.update(overrides) + return _DRIVER.g6_classify(**base) + + +class G6ClassifyTests(unittest.TestCase): + def test_policy_table(self): + self.assertEqual(_DRIVER.g6_policy_ms("TRADE", None, 9), 3000) + self.assertEqual(_DRIVER.g6_policy_ms("BOOK_SNAPSHOT", None, 9), 60000) + self.assertEqual(_DRIVER.g6_policy_ms("BAR", "1m", 9), 180000) + self.assertEqual(_DRIVER.g6_policy_ms("BAR", "5m", 600000), 600000) # own sealed requirement + self.assertEqual(_DRIVER.g6_policy_ms("FUNDING_RATE", None, None), None) + + def test_fresh_eligible_and_not_eligible(self): + self.assertEqual(classify(), ("ELIGIBLE", [])) + self.assertEqual(classify(eligible=False, freshness_ms=99999), ("NOT_ELIGIBLE", [])) + + def test_eligible_but_stale_without_quiet_proof_is_unsafe(self): + self.assertEqual(classify(freshness_ms=2500), ("UNSAFE", ["AGE_BEYOND_POLICY"])) + state, reasons = classify(gap_open=True, state="GAPPED") + self.assertEqual(state, "UNSAFE") + self.assertEqual(set(reasons), {"GAP_OPEN", "STATE_GAPPED"}) + self.assertEqual(classify(session_liveness_ms=5000)[1], ["SESSION_BEYOND_POLICY"]) + + def test_quiet_continuity_under_observe_is_eligible_quiet(self): + trade = classify(feed="TRADE", freshness_ms=9000, policy_ms=3000, event_recency_state="STALE") + self.assertEqual(trade, ("ELIGIBLE_QUIET", [])) + quote = classify(freshness_ms=9000, event_recency_state="STALE", on_change=True) + self.assertEqual(quote, ("ELIGIBLE_QUIET", [])) + # A generic stale QUOTE cannot self-upgrade; neither can a quiet TRADE on a dead session. + self.assertEqual(classify(freshness_ms=9000, event_recency_state="STALE")[0], "UNSAFE") + self.assertEqual(classify(feed="TRADE", freshness_ms=9000, policy_ms=3000, event_recency_state="STALE", + provider_session_state="DISCONNECTED")[0], "UNSAFE") + + +class G6RecorderTests(unittest.TestCase): + def test_bounded_reservoir_exact_counts_and_merge(self): + one, two = _DRIVER._G6Recorder(reservoir=8), _DRIVER._G6Recorder(reservoir=8) + for age in range(100): + one.view({"key": "QUOTE|OKX|", "age_ms": age, "policy_ms": 2000, "class": "ELIGIBLE", "reasons": []}, + "STEADY") + two.view({"key": "QUOTE|OKX|", "age_ms": 5000, "policy_ms": 2000, "class": "UNSAFE", + "reasons": ["AGE_BEYOND_POLICY"]}, "BURST") + two.refusal("TRADE|OKX|", "DATA_STALE", "STEADY") + self.assertEqual(len(one.rows["QUOTE|OKX|"]["ages"]), 8) + merged = _DRIVER.merge_g6([one.evidence(), two.evidence()]) + row = merged["QUOTE|OKX|"] + self.assertEqual((row["n"], row["max_age_ms"], row["unsafe"]), (101, 5000, 1)) + self.assertEqual(row["windows"], {"STEADY": 100, "BURST": 1}) + self.assertEqual(row["age_ms"]["n"], 9) + self.assertEqual(merged["TRADE|OKX|"]["refusals"], {"DATA_STALE": 1}) + + +@dataclass(frozen=True) +class _Feed: + value: str + + +@dataclass(frozen=True) +class _Product: + venue: str = "OKX" + native_symbol: str = "BTC-USDT-SWAP" + feed: _Feed = _Feed("QUOTE") + interval: str | None = None + consumer_id: str = "alpha.okx.paper.stable" + delivery: _Feed = _Feed("DURABLE") + source_policy_id: str = "crypto_primary_v2" + + +class G6PollLoopTests(unittest.TestCase): + def _run(self, read): + recorder = _DRIVER._TargetRecorder(windows={}) + + async def scenario(): + start = time.monotonic() + 0.02 + await _DRIVER._target_poll_loop(client=None, label="s1", operation="SNAPSHOT", products=(_Product(),), + period=0.05, phase=0.0, start=start, end=start + 0.2, recorder=recorder) + with patch.object(_DRIVER, "_target_read", read): + asyncio.run(scenario()) + return recorder.g6.evidence() + + def test_a_refused_unsafe_view_is_counted_once_as_unsafe(self): + async def read(client, operation, products): + _DRIVER._G6_VIEW.set([{"key": "QUOTE|OKX|", "age_ms": 4000, "policy_ms": 2000, "class": "UNSAFE", + "reasons": ["AGE_BEYOND_POLICY"]}]) + raise ValueError("V2 receipt exceeds the governed freshness bound") + row = self._run(read)["QUOTE|OKX|"] + self.assertEqual(row["classes"], {"UNSAFE": 4}) + self.assertEqual(row["refusals"], {}) + + def test_a_read_without_a_view_is_a_typed_refusal(self): + class Rejected(Exception): + code = "RATE_LIMITED" + + async def read(client, operation, products): + raise Rejected("x") + row = self._run(read)["QUOTE|OKX|"] + self.assertEqual((row["n"], row["refusals"]), (0, {"RATE_LIMITED": 4})) + + def test_success_records_the_view(self): + async def read(client, operation, products): + _DRIVER._G6_VIEW.set([{"key": "QUOTE|OKX|", "age_ms": 120, "policy_ms": 2000, "class": "ELIGIBLE", + "reasons": []}]) + row = self._run(read)["QUOTE|OKX|"] + self.assertEqual((row["n"], row["classes"], row["max_age_ms"]), (4, {"ELIGIBLE": 4}, 120)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p184_trace_feed_transit.py b/tests/test_p184_trace_feed_transit.py index 1b253779..9d0c8a7e 100644 --- a/tests/test_p184_trace_feed_transit.py +++ b/tests/test_p184_trace_feed_transit.py @@ -99,6 +99,26 @@ def test_cli_reads_files_and_writes_out_exclusively(self): with self.assertRaises(FileExistsError): transit.main([trace, "--skip-seconds", "0", "--out", out]) + def test_duplicate_applied_counts_only_canonical_event_ids(self): + digest = "a" * 64 + rows = [row("TRADE", "OKX:SWAP:BTC-USDT-SWAP", 100, 50, 110, event_id=digest), + row("TRADE", "OKX:SWAP:BTC-USDT-SWAP", 200, 50, 210, event_id=digest), + row("TRADE", "OKX:SWAP:BTC-USDT-SWAP", 300, 60, 310), + row("TRADE", "OKX:SWAP:BTC-USDT-SWAP", 400, 60, 410)] + result = transit.summarise(rows, skip_seconds=0) + self.assertEqual(result["duplicate_applied_event_id"], {"TRADE|OKX": 1}) + self.assertEqual(result["excluded_from_transit"]["TRADE|OKX|repeated_event_identity"], 2) + + def test_slim_loader_keeps_only_the_fields_summarise_reads(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "delivery-w-1-2.json") + record = {**row("TRADE", "OKX:SWAP:X", 100, 50, 110), "mark_source_event_time_ns": 1, "payload_extra": 2} + with open(path, "w") as handle: + json.dump({"records": [record], "stop_reason": "EXPIRY", "dropped_at_export": 0}, handle) + files, records = transit.load([path], fields=transit.FIELDS) + self.assertEqual(set(records[0]) - set(transit.FIELDS), set()) + self.assertEqual(files[0]["stop_reason"], "EXPIRY") + if __name__ == "__main__": unittest.main() From 65ada379ca18befdfb2c67bed0f41b6039986e7a Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 16:25:01 +0000 Subject: [PATCH 128/147] feat(p184): final gate-campaign tooling - G3 BAR cache poller, G8 Redis sampler, G9/G10 writer counters, G11 restart probe, G14 table, runner and G1-G14 scorer One campaign run of the frozen protocol (stage-100 final run with BURST, SLOW_READER and RECONNECT windows, >= 30 min) with every probe the v2.2.3 scorecard listed as missing: - G3: MGET poll of the TS last_bar keys, close -> first visibility per slice; expected closes never seen are counted with their lower bound. - G8: PING RTT, INFO commandstats deltas per 10-min window, new SLOWLOG entries, MONITOR client count, on the Redis the writer projects into (TRADING_MARKET_REDIS_URL = redis_marketdata DB1, read from the writer). - G9/G10: writer heartbeat (demanded/session/execution, purpose not_ready rows by reason; quiet TRADE excluded) and typed TRADE log signals (CURSOR_EXPIRED, SlowConsumerError, continuity/SNAPSHOT_REPLACED). No dedicated TRADE integrity counter exists in the writer or SDK. - G11: waits for a coordinator-triggered writer restart; outage start, fault cleared, first valid apply and caught-up per slice; rewind and cursor-not-ahead checks (cursor offsets read without tokens). - G14: per-role mean (cgroup) / p95 / max CPU and max RSS vs limit; the campaign profile adds binance_bar_edge and stable_redis. - p184_gate_campaign.py run|score: read-only preflight (refuses without a long writer trace), PID-scoped children with deadlines and SIGTERM summaries, G4 windows incl. a UTC :00/:30 boundary window, trace analysis in a --rm --network none container, scorecard with PASS/FAIL/NOT_MEASURED where a measurement outside the protocol never passes. All Redis access is an allowlisted read-only RESP client (no MONITOR, no writes). --- scripts/p184_g11_writer_restart_probe.py | 390 ++++++++++++ scripts/p184_g14_resource_table.py | 152 +++++ scripts/p184_g3_bar_cache_probe.py | 205 ++++++ scripts/p184_g8_redis_sampler.py | 240 +++++++ scripts/p184_g9_g10_writer_counters.py | 315 ++++++++++ scripts/p184_gate_campaign.py | 758 +++++++++++++++++++++++ scripts/p184_resp_readonly.py | 221 +++++++ tests/test_p184_gate_campaign.py | 413 ++++++++++++ 8 files changed, 2694 insertions(+) create mode 100644 scripts/p184_g11_writer_restart_probe.py create mode 100644 scripts/p184_g14_resource_table.py create mode 100644 scripts/p184_g3_bar_cache_probe.py create mode 100644 scripts/p184_g8_redis_sampler.py create mode 100644 scripts/p184_g9_g10_writer_counters.py create mode 100644 scripts/p184_gate_campaign.py create mode 100644 scripts/p184_resp_readonly.py create mode 100644 tests/test_p184_gate_campaign.py diff --git a/scripts/p184_g11_writer_restart_probe.py b/scripts/p184_g11_writer_restart_probe.py new file mode 100644 index 00000000..72d0feea --- /dev/null +++ b/scripts/p184_g11_writer_restart_probe.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +"""P18.4 gate G11: TS writer restart recovery, measured around a restart the coordinator triggers (read-only). + +The probe never restarts anything. It arms, then waits (<= --wait-seconds) for the writer container's +State.StartedAt to change (`docker restart`; a recreate with a new container id is also detected by name). It records +the four G11 timestamps separately (gate table rev 2, Codex review 3): +* outage_start = State.FinishedAt of the old process (Docker's exit time), cross-checked by the last poll that saw + the old StartedAt; +* fault_cleared = the new State.StartedAt with State.Running true; +* first_valid_apply per slice = first poll after fault_cleared at which the slice's TS cache entry was rewritten + (value digest changed, or the key TTL jumped back up) with a parsable event time not older than the last + pre-outage event time; +* caught_up per slice = first poll after fault_cleared whose cached event age is inside the live bound (TRADE 3000 ms, + QUOTE/MARK/BOOK 2000 ms, BAR: newest closed bar closed <= interval + 5 s ago). +and checks: no rewind (event time and, for TRADE/QUOTE, the applied offset of the same generation never go backwards +across the restart) and cursor never ahead (each committed SDK cursor offset of a TRADE/QUOTE slice <= the applied +offset in TS Redis read after it; the cursor files are read with a read-only `docker exec ... python -c` that prints +only key -> offset, never the resume tokens). + +Keys per slice (TS services/market_data/cache_keys.py; the market Redis is TRADING_MARKET_REDIS_URL, default +redis_marketdata DB 1): TRADE/QUOTE `execution_trade|execution_quote:` (+ `applied:trade|quote:` offset and +generation); BOOK_SNAPSHOT/BOOK_DELTA/MARK `::ordering-v1` marker (observed_at_ms) plus PTTL of the +`:` view (the ~40 KB book body is never read); BAR `last_bar::`. +Gate (NEW owner bound, frozen table): fault cleared -> first valid apply per slice <= 30 s; -> caught up <= 60 s; +no rewind; cursor never ahead. + +Boundary: GET/MGET/PTTL/SCAN only; docker inspect (read-only) and one read-only docker exec per cursor sample. +usage: p184_g11_writer_restart_probe.py --binding FILE --out DIR [--wait-seconds 900] [--observe-seconds 180] +""" +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.p184_resp_readonly import ( # noqa: E402 + install_stop_signals, pause, stopped, + BoundedJsonl, ReadOnlyRedis, container_ip, quantiles, write_json_exclusive, +) + +GATE = {"first_valid_apply_le_s": 30.0, "caught_up_le_s": 60.0} +LIVE_BOUND_MS = {"TRADE": 3000, "QUOTE": 2000, "MARK_INDEX_PRICE": 2000, "BOOK_SNAPSHOT": 2000, "BOOK_DELTA": 2000} +KIND = {"TRADE": "execution_trade", "QUOTE": "execution_quote", "BOOK_SNAPSHOT": "execution_book_snapshot", + "BOOK_DELTA": "execution_book_delta", "MARK_INDEX_PRICE": "execution_mark_index_price"} +UNIT_MS = {"m": 60_000, "h": 3_600_000, "d": 86_400_000} +CURSOR_DUMP = ( + "import glob,json\n" + "out={}\n" + "for p in glob.glob('/app/state/data-layer-v2/cursors/fixed-workers-v1/count-*-worker-*/*.json'):\n" + " try:\n" + " for k,v in json.load(open(p)).get('items',{}).items(): out[k]=int(v['offset'])\n" + " except Exception: pass\n" + "print(json.dumps(out))\n" +) + + +def parse_docker_time(value: str) -> int | None: + """Docker RFC3339 with nanoseconds -> unix ms; the zero time means 'never'.""" + if not value or value.startswith("0001-"): + return None + head, _, frac = value.rstrip("Z").partition(".") + base = dt.datetime.strptime(head, "%Y-%m-%dT%H:%M:%S").replace(tzinfo=dt.timezone.utc) + return int(base.timestamp() * 1000) + int((frac + "000")[:3] or 0) + + +def demanded_slices(binding: dict) -> list[dict]: + return sorted(({"venue": p["venue"], "native": p["native_symbol"], "feed": p["feed"], + "interval": p.get("interval"), "uid": p["instrument_uid"]} + for p in binding.get("products", [])), key=lambda r: (r["venue"], r["native"], r["feed"])) + + +def identities_from_keys(keys) -> dict[tuple[str, str], str]: + """(venue, venue_symbol) -> canonical id 'VENUE:PRODUCT:venue_symbol', from execution/last_bar key names.""" + out = {} + for key in keys: + parts = key.split(":") + if (len(parts) >= 7 and parts[:3] == ["cache", "market", "v2"] + and (parts[3].startswith("execution_") or parts[3] == "last_bar")): + venue, product, symbol = parts[4], parts[5], parts[6] + out[(venue, symbol)] = f"{venue}:{product}:{symbol}" + return out + + +def slice_keys(item: dict, identity: str) -> dict: + name = "|".join(str(item[k]) for k in ("venue", "native", "feed", "interval")) + if item["feed"] == "BAR": + key = f"cache:market:v2:last_bar:{identity}:{item['interval']}" + return {"name": name, "value": key, "ttl": key, "applied": None} + kind = KIND[item["feed"]] + if item["feed"] in ("TRADE", "QUOTE"): + key = f"cache:market:v2:{kind}:{identity}" + return {"name": name, "value": key, "ttl": key, + "applied": f"cache:market:v2:applied:{item['feed'].lower()}:{identity}"} + return {"name": name, "value": f"cache:market:v2:{kind}:{identity}:ordering-v1", + "ttl": f"cache:market:v2:{kind}:{identity}", "applied": None} + + +def observe(feed: str, raw, applied_raw, pttl, interval: str | None) -> dict: + """Reduce one slice read to the fields the analysis needs (no payload values kept).""" + obs = {"digest": hashlib.sha256(raw).hexdigest()[:16] if raw else None, "pttl": pttl, + "event_ms": None, "offset": None, "generation": None, "closed": None} + if raw: + try: + value = json.loads(raw) + if feed == "BAR": + obs["event_ms"] = int(value["timestamp_ms"]) + int(interval[:-1]) * UNIT_MS[interval[-1]] + obs["closed"] = value.get("is_closed") is True + else: + obs["event_ms"] = int(value["observed_at_ms"]) + except (ValueError, KeyError, TypeError): + pass + if applied_raw: + try: + applied = json.loads(applied_raw) + obs["offset"], obs["generation"] = int(applied["offset"]), str(applied["generation"])[:16] + except (ValueError, KeyError, TypeError): + pass + return obs + + +class SliceRecovery: + def __init__(self, name: str, feed: str, interval: str | None, poll_ms: int): + self.name, self.feed, self.interval, self.poll_ms = name, feed, interval, poll_ms + self.pre = None # last observation before the outage + self.prev = None # previous observation + self.first_apply_ms = None + self.caught_up_ms = None + self.rewinds = [] + + def _rewritten(self, obs) -> bool: + if self.prev is None: + return False + if obs["digest"] != self.prev["digest"]: + return True + return (obs["pttl"] is not None and self.prev["pttl"] is not None and obs["pttl"] >= 0 + and obs["pttl"] > self.prev["pttl"] + self.poll_ms + 500) + + def _live(self, obs, now_ms) -> bool: + if obs["event_ms"] is None: + return False + if self.feed == "BAR": + step = int(self.interval[:-1]) * UNIT_MS[self.interval[-1]] + return bool(obs["closed"]) and now_ms - obs["event_ms"] <= step + 5000 + return now_ms - obs["event_ms"] <= LIVE_BOUND_MS[self.feed] + + def update(self, now_ms: int, obs: dict, *, phase: str) -> None: + """phase: PRE (before the outage), DOWN (outage, not cleared), POST (after fault cleared).""" + if phase == "PRE": + self.pre = obs + elif phase == "POST": + base = self.pre or {} + if base.get("event_ms") is not None and obs["event_ms"] is not None and obs["event_ms"] < base["event_ms"]: + self.rewinds.append({"at_ms": now_ms, "kind": "event_time", "before": base["event_ms"], + "after": obs["event_ms"]}) + if (base.get("offset") is not None and obs["offset"] is not None + and base.get("generation") == obs["generation"] and obs["offset"] < base["offset"]): + self.rewinds.append({"at_ms": now_ms, "kind": "applied_offset", "before": base["offset"], + "after": obs["offset"]}) + if self.first_apply_ms is None and self._rewritten(obs) and obs["event_ms"] is not None and ( + base.get("event_ms") is None or obs["event_ms"] >= base["event_ms"]): + self.first_apply_ms = now_ms + if self.caught_up_ms is None and self.first_apply_ms is not None and self._live(obs, now_ms): + self.caught_up_ms = now_ms + self.prev = obs + + +def cursor_check(cursors: dict[str, int], applied: dict[str, int], uid_slice: dict[str, str]) -> dict: + """cursors: 'consumer|uid|FEED|interval|policy' -> offset; applied: slice name -> applied offset (read after).""" + ahead, compared, unmatched = [], 0, 0 + for key, offset in cursors.items(): + parts = key.split("|") + if len(parts) < 3 or parts[2] not in ("TRADE", "QUOTE"): + continue + name = uid_slice.get(f"{parts[1]}|{parts[2]}") + if name is None or applied.get(name) is None: + unmatched += 1 + continue + compared += 1 + if offset > applied[name]: + ahead.append({"slice": name, "cursor": offset, "applied": applied[name]}) + return {"compared": compared, "unmatched": unmatched, "ahead": ahead} + + +def recovery_summary(slices: dict, cleared_ms: int | None) -> dict: + first = {n: (s.first_apply_ms - cleared_ms) / 1000 for n, s in slices.items() + if cleared_ms and s.first_apply_ms is not None} + caught = {n: (s.caught_up_ms - cleared_ms) / 1000 for n, s in slices.items() + if cleared_ms and s.caught_up_ms is not None} + return {"first_valid_apply_s": quantiles(list(first.values())), + "caught_up_s": quantiles(list(caught.values())), + "slices_without_first_apply": sorted(n for n in slices if n not in first), + "slices_not_caught_up": sorted(n for n in slices if n not in caught), + "per_slice": {n: {"first_valid_apply_s": first.get(n), "caught_up_s": caught.get(n), + "rewinds": s.rewinds[:5]} for n, s in sorted(slices.items())}, + "rewinds_total": sum(len(s.rewinds) for s in slices.values())} + + +def verdict(report: dict) -> tuple[str, list[str]]: + if report.get("restart") is None: + return "NOT_MEASURED", ["no writer restart observed inside the wait window"] + summary = report["recovery"] + reasons = [] + if summary["first_valid_apply_s"].get("n") and summary["first_valid_apply_s"]["max"] > GATE["first_valid_apply_le_s"]: + reasons.append(f"first valid apply max {summary['first_valid_apply_s']['max']} s > 30 s") + if summary["caught_up_s"].get("n") and summary["caught_up_s"]["max"] > GATE["caught_up_le_s"]: + reasons.append(f"caught up max {summary['caught_up_s']['max']} s > 60 s") + if summary["rewinds_total"]: + reasons.append(f"{summary['rewinds_total']} rewinds") + if any(sample["ahead"] for sample in report["cursor_samples"]): + reasons.append("committed cursor ahead of applied state") + observed = report["observe_seconds"] + late = summary["slices_without_first_apply"] + summary["slices_not_caught_up"] + if observed >= GATE["caught_up_le_s"] and summary["slices_not_caught_up"]: + reasons.append(f"{len(summary['slices_not_caught_up'])} slices not caught up within {observed} s " + "(includes quiet TRADE slices; see per_slice)") + if reasons: + return "FAIL", reasons + gaps = [] + if late: + gaps.append("slices without a measured first apply/caught-up") + if not report["cursor_samples"] or not any(s["compared"] for s in report["cursor_samples"]): + gaps.append("cursor-not-ahead not compared") + return ("NOT_MEASURED", gaps) if gaps else ("PASS", []) + + +def inspect_writer(name: str) -> dict | None: + result = subprocess.run(["docker", "inspect", name, "--format", + '{"id":{{json .Id}},"started":{{json .State.StartedAt}},"finished":{{json .State.FinishedAt}},' + '"running":{{json .State.Running}},"restarts":{{.RestartCount}}}'], + capture_output=True, text=True, timeout=20) + return json.loads(result.stdout) if result.returncode == 0 and result.stdout.strip() else None + + +def read_cursors(writer: str) -> dict[str, int] | None: + result = subprocess.run(["docker", "exec", writer, "python", "-B", "-c", CURSOR_DUMP], + capture_output=True, text=True, timeout=30) + try: + return {str(k): int(v) for k, v in json.loads(result.stdout).items()} if result.returncode == 0 else None + except (ValueError, AttributeError): + return None + + +def run(args) -> dict: + binding = json.loads(Path(args.binding).read_text()) + items = demanded_slices(binding) + host = args.host or container_ip(args.redis_container, args.network) + redis = ReadOnlyRedis(host, args.port, args.db) + ids = identities_from_keys(redis.scan("cache:market:v2:*", limit=20000)) + plan, slices, uid_slice, unresolved = [], {}, {}, [] + for item in items: + identity = ids.get((item["venue"], item["native"])) + if identity is None: + unresolved.append(f"{item['venue']}|{item['native']}|{item['feed']}") + continue + keys = slice_keys(item, identity) + plan.append((item, keys)) + slices[keys["name"]] = SliceRecovery(keys["name"], item["feed"], item["interval"], args.poll_ms) + uid_slice[f"{item['uid']}|{item['feed']}"] = keys["name"] + out = Path(args.out) + out.mkdir(parents=True, exist_ok=False) + evidence = BoundedJsonl(out / "g11-observations.jsonl", args.max_bytes) + value_keys = [k["value"] for _, k in plan] + applied_keys = [k["applied"] for _, k in plan if k["applied"]] + + def poll(): + commands = [("MGET", *value_keys)] + [("PTTL", k["ttl"]) for _, k in plan] + if applied_keys: + commands.append(("MGET", *applied_keys)) + replies = redis.pipeline(commands) + values, ttls = replies[0], replies[1:1 + len(plan)] + applied = dict(zip(applied_keys, replies[-1])) if applied_keys else {} + now_ms = time.time_ns() // 1_000_000 + return now_ms, {k["name"]: observe(item["feed"], raw, applied.get(k["applied"]), ttl, item["interval"]) + for (item, k), raw, ttl in zip(plan, values, ttls)} + + base = inspect_writer(args.writer) + if base is None or not base["running"]: + raise SystemExit("writer is not running at arm time; the probe measures a restart from a running writer") + evidence.write({"kind": "armed", "at_ms": time.time_ns() // 1_000_000, "writer": base, + "slices": len(plan), "unresolved": unresolved}) + print(json.dumps({"kind": "G11_ARMED", "writer": args.writer, "wait_seconds": args.wait_seconds}), flush=True) + restart, phase, last_old_seen_ms = None, "PRE", None + cleared_ms = None + deadline = time.monotonic() + args.wait_seconds + observe_until = None + next_inspect = time.monotonic() + cursor_samples, read_errors = [], 0 + try: + while not stopped(): + now = time.monotonic() + if observe_until is None and now >= deadline: + break + if observe_until is not None and now >= observe_until: + break + if now >= next_inspect: + next_inspect = now + 1.0 + state = inspect_writer(args.writer) + if restart is None: + if state and state["started"] == base["started"] and state["id"] == base["id"]: + last_old_seen_ms = time.time_ns() // 1_000_000 + else: + phase = "DOWN" + restart = {"detected_ms": time.time_ns() // 1_000_000, "last_old_seen_ms": last_old_seen_ms} + if restart is not None and cleared_ms is None and state and state["running"] and ( + state["started"] != base["started"] or state["id"] != base["id"]): + restart.update(new=state, outage_start_ms=parse_docker_time(state["finished"]) + if state["id"] == base["id"] else last_old_seen_ms, + fault_cleared_ms=parse_docker_time(state["started"])) + cleared_ms = restart["fault_cleared_ms"] + phase = "POST" + observe_until = time.monotonic() + args.observe_seconds + evidence.write({"kind": "restart", **restart}) + try: + now_ms, observed = poll() + except Exception as error: + read_errors += 1 + evidence.write({"kind": "read_error", "at_ms": time.time_ns() // 1_000_000, + "error": type(error).__name__}) + pause(args.poll_ms / 1000) + continue + effective = "POST" if phase == "POST" and cleared_ms is not None and now_ms >= cleared_ms else ( + "DOWN" if phase != "PRE" else "PRE") + for name, obs in observed.items(): + before = slices[name].first_apply_ms, slices[name].caught_up_ms + slices[name].update(now_ms, obs, phase=effective) + if effective == "POST" and (slices[name].first_apply_ms, slices[name].caught_up_ms) != before: + evidence.write({"kind": "slice_progress", "slice": name, "at_ms": now_ms, + "first_apply_ms": slices[name].first_apply_ms, + "caught_up_ms": slices[name].caught_up_ms, "event_ms": obs["event_ms"]}) + if (effective == "POST" and len(cursor_samples) < args.cursor_samples + and now_ms - cleared_ms >= 20_000 * (len(cursor_samples) + 1)): + cursors = read_cursors(args.writer) # cursor first, then the applied offsets it must not exceed + _, after = poll() + applied = {n: o["offset"] for n, o in after.items() if o["offset"] is not None} + sample = (cursor_check(cursors, applied, uid_slice) if cursors is not None + else {"compared": 0, "unmatched": 0, "ahead": [], "error": "cursor read failed"}) + sample["at_ms"] = now_ms + cursor_samples.append(sample) + evidence.write({"kind": "cursor_check", **sample}) + pause(max(0.0, args.poll_ms / 1000 - (time.monotonic() - now))) + finally: + redis.close() + evidence.close() + report = {"schema": "qdl.p184.g11-writer-restart-probe.v1", "gate": GATE, "writer": args.writer, + "redis": {"container": args.redis_container, "host": host, "db": args.db}, "poll_ms": args.poll_ms, + "measuring_point": "host poll of TS cache keys; first apply/caught-up are upper bounds by one poll", + "observe_seconds": args.observe_seconds, "unresolved_slices": unresolved, "read_errors": read_errors, + "restart": restart, "recovery": recovery_summary(slices, cleared_ms), "cursor_samples": cursor_samples} + if restart and restart.get("outage_start_ms") and cleared_ms: + report["outage_seconds"] = round((cleared_ms - restart["outage_start_ms"]) / 1000, 3) + report["verdict"], report["reasons"] = verdict(report) + write_json_exclusive(out / "g11-summary.json", report) + return report + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--binding", required=True) + parser.add_argument("--writer", default="market_data_service_p184w") + parser.add_argument("--redis-container", default="redis_marketdata") + parser.add_argument("--network", default="executor_network") + parser.add_argument("--host") + parser.add_argument("--port", type=int, default=6379) + parser.add_argument("--db", type=int, default=1) + parser.add_argument("--wait-seconds", type=float, default=900) + parser.add_argument("--observe-seconds", type=float, default=180) + parser.add_argument("--poll-ms", type=int, default=500) + parser.add_argument("--cursor-samples", type=int, default=3) + parser.add_argument("--max-bytes", type=int, default=16 * 1024 * 1024) + parser.add_argument("--out", required=True) + args = parser.parse_args(argv) + install_stop_signals() + if not 100 <= args.poll_ms <= 5000 or not 0 < args.wait_seconds <= 3600 or not 60 <= args.observe_seconds <= 900: + parser.error("poll-ms 100..5000, wait-seconds 0..3600, observe-seconds 60..900") + report = run(args) + print(json.dumps({"verdict": report["verdict"], "reasons": report["reasons"]})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/p184_g14_resource_table.py b/scripts/p184_g14_resource_table.py new file mode 100644 index 00000000..d6ba1ebe --- /dev/null +++ b/scripts/p184_g14_resource_table.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""P18.4 gate G14: per-role CPU and memory table from a stage-100 driver run (offline, read-only). + +Input: the driver's host.json (scripts/phase3_consumer_load_acceptance.py run_target_host), whose +`runtime_observation` holds one sample about every 10 s with `docker stats --no-stream` rows and the cumulative cgroup +`cpu_throttle` counters of every monitored container (profile `monitored_containers`; the final campaign adds +binance_bar_edge and stable_redis, which the v2.2.3 scorecard listed as not sampled). + +Per role: cpu_mean% = cgroup usage_usec delta / wall delta between the first and last sample that carry it (exact +mean; 100 % = one core), cpu_p95% / cpu_max% from the instantaneous docker stats CPUPerc samples (nearest rank), +mem_max_MiB from MemUsage, the memory limit from MemUsage's denominator, the CPU limit (NanoCpus) and throttled time +share when --inspect is given (read-only docker inspect). Roles missing from the samples are listed, never filled. + +G14 is a recording gate: the owner sets the budget from this measurement. The table is PASS-eligible only when every +expected role has samples, every max RSS is under its limit and the run met the protocol (decided by the scorer). +usage: p184_g14_resource_table.py HOST_JSON [--expect name,...] [--inspect] [--out PATH] +""" +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +PREFIX = "qdl_v2_stable_candidate-" +UNITS = {"B": 1 / 2**20, "KiB": 1 / 1024, "MiB": 1.0, "GiB": 1024.0, "TiB": 1024.0 ** 2, + "kB": 1000 / 2**20, "MB": 1e6 / 2**20, "GB": 1e9 / 2**20} + + +def role_name(container: str) -> str: + name = container[len(PREFIX):] if container.startswith(PREFIX) else container + return name[:-2] if name.endswith("-1") else name + + +def mib(text: str) -> float | None: + match = re.fullmatch(r"\s*([0-9.]+)\s*([A-Za-z]+)\s*", text or "") + if not match or match.group(2) not in UNITS: + return None + return float(match.group(1)) * UNITS[match.group(2)] + + +def percent(text: str) -> float | None: + try: + return float(str(text).strip().rstrip("%")) + except ValueError: + return None + + +def nearest(values, q): + ordered = sorted(values) + return ordered[min(len(ordered) - 1, int(q * len(ordered)))] if ordered else None + + +def table(host: dict, *, expect=(), limits=None) -> dict: + samples = host.get("runtime_observation") or [] + cpu, mem, mem_limit, usage = {}, {}, {}, {} + for sample in samples: + for row in sample.get("stats", []): + name = row.get("Name") or row.get("Container") + if not name: + continue + value = percent(row.get("CPUPerc")) + if value is not None: + cpu.setdefault(name, []).append(value) + used, _, limit = str(row.get("MemUsage", "")).partition("/") + if mib(used) is not None: + mem.setdefault(name, []).append(mib(used)) + if mib(limit) is not None: + mem_limit[name] = mib(limit) + for name, counters in (sample.get("cpu_throttle") or {}).items(): + usage.setdefault(name, []).append((sample["at_ns"], counters)) + roles = {} + for name in sorted(set(cpu) | set(mem) | set(usage)): + series = usage.get(name, []) + mean = throttled = None + if len(series) >= 2: + (t0, c0), (t1, c1) = series[0], series[-1] + wall_us = (t1 - t0) / 1000 + if wall_us > 0: + mean = round(100.0 * (c1["usage_usec"] - c0["usage_usec"]) / wall_us, 1) + throttled = round(100.0 * (c1["throttled_usec"] - c0["throttled_usec"]) / wall_us, 1) + row = {"container": name, "samples": len(cpu.get(name, [])), "cpu_mean_pct": mean, + "cpu_p95_pct": nearest(cpu.get(name, []), .95), "cpu_max_pct": max(cpu.get(name, []), default=None), + "mem_max_mib": round(max(mem.get(name, [])), 1) if mem.get(name) else None, + "mem_limit_mib": round(mem_limit[name], 1) if name in mem_limit else None, + "throttled_pct_of_wall": throttled} + if limits and name in limits: + row.update(limits[name]) + row["mem_headroom_pct"] = (round(100.0 * (1 - row["mem_max_mib"] / row["mem_limit_mib"]), 1) + if row["mem_max_mib"] and row["mem_limit_mib"] else None) + roles[role_name(name)] = row + expected = {role_name(name) for name in (expect or host.get("monitored_containers") or [])} + missing = sorted(expected - {r for r, v in roles.items() if v["samples"]}) + over = sorted(r for r, v in roles.items() if v["mem_max_mib"] and v["mem_limit_mib"] + and v["mem_max_mib"] >= v["mem_limit_mib"]) + means = [v["cpu_mean_pct"] for r, v in roles.items() if v["cpu_mean_pct"] is not None + and not r.startswith("market_data_service")] + return {"roles": roles, "missing_roles": missing, "mem_at_or_over_limit": over, + "samples": len(samples), "dl_cpu_mean_sum_vcpu": round(sum(means) / 100.0, 2) if means else None} + + +def inspect_limits(containers) -> dict: + result = subprocess.run(["docker", "inspect", "--format", + '{"name":{{json .Name}},"nano_cpus":{{.HostConfig.NanoCpus}},"memory":{{.HostConfig.Memory}}}', + *containers], capture_output=True, text=True, timeout=30) + out = {} + for line in result.stdout.splitlines(): + value = json.loads(line) + out[value["name"].lstrip("/")] = {"cpu_limit_vcpu": value["nano_cpus"] / 1e9 or None, + "mem_limit_bytes_inspect": value["memory"] or None} + return out + + +def render(result: dict) -> str: + lines = [f"{'role':<28}{'cpu_mean%':>10}{'cpu_p95%':>9}{'cpu_max%':>9}{'thr%':>6}{'mem_max_MiB':>12}" + f"{'limit_MiB':>11}{'headroom%':>10}"] + fmt = lambda v, w: f"{'-' if v is None else v:>{w}}" # noqa: E731 + for role, row in sorted(result["roles"].items()): + lines.append(f"{role:<28}{fmt(row['cpu_mean_pct'], 10)}{fmt(row['cpu_p95_pct'], 9)}" + f"{fmt(row['cpu_max_pct'], 9)}{fmt(row['throttled_pct_of_wall'], 6)}" + f"{fmt(row['mem_max_mib'], 12)}{fmt(row['mem_limit_mib'], 11)}{fmt(row['mem_headroom_pct'], 10)}") + lines.append(f"samples {result['samples']}; DL CPU mean sum {result['dl_cpu_mean_sum_vcpu']} vCPU (writer " + f"excluded); missing roles {result['missing_roles'] or 'none'}; at/over memory limit " + f"{result['mem_at_or_over_limit'] or 'none'}") + return "\n".join(lines) + "\n" + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("host_json", type=Path) + parser.add_argument("--expect", default="", help="comma list of containers that must have samples") + parser.add_argument("--inspect", action="store_true", help="add CPU limits via read-only docker inspect") + parser.add_argument("--out", type=Path, help="write JSON here (exclusive) and the text table beside it") + args = parser.parse_args(argv) + host = json.loads(args.host_json.read_text()) + expect = [x for x in args.expect.split(",") if x] + limits = inspect_limits(expect or host.get("monitored_containers") or []) if args.inspect else None + result = table(host, expect=expect, limits=limits) + text = render(result) + if args.out: + with args.out.open("x") as handle: + handle.write(json.dumps(result, indent=1, sort_keys=True) + "\n") + with args.out.with_suffix(".txt").open("x") as handle: + handle.write(text) + print(text, end="") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/p184_g3_bar_cache_probe.py b/scripts/p184_g3_bar_cache_probe.py new file mode 100644 index 00000000..3ad467aa --- /dev/null +++ b/scripts/p184_g3_bar_cache_probe.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""P18.4 gate G3: BAR close -> first visibility of the closed bar in the TS execution cache, per demanded BAR slice. + +Purpose: the writer delivery trace carries no BAR records (v2.2.3 scorecard G3 NOT_MEASURED), so this read-only poller +reads the TS market cache keys the writer projects BAR into (TS services/market_data/cache_projector.py _bar_operations: +`cache:market:v2:last_bar::::`, Redis TRADING_MARKET_REDIS_URL, default +redis_marketdata DB 1) with one MGET per tick, and records for each slice the first tick at which a closed bar +(is_closed true) with a new open time is visible. + +Quantity: transit_ms = first_seen_wall_ms - close_ms, close_ms = timestamp_ms (bar open) + interval. The poll period is +an upper-bound bias (a bar is seen at most one period + one RTT late); it is recorded in the summary. Bars already in +the cache at the first tick are not measured. Every close boundary C with start + 5 s < C <= end - 60 s is +expected per slice; an expected close never seen is counted with its lower bound (end - C, which is >= 60 s) and FAILs +the max rule, never dropped. + +Gate (TS upgrade/p184-phase-d-gates.md, frozen): p95 <= 5000 ms, max < 60000 ms. + +Boundary: read-only (allowlisted MGET/SCAN through scripts/p184_resp_readonly.py), one MGET per tick of <= 64 keys of +~1 KB, bounded by --seconds and --max-bytes. No writes anywhere except its own exclusive output files. +usage: p184_g3_bar_cache_probe.py --binding FILE --seconds N --out DIR [--poll-ms 250] +""" +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.p184_resp_readonly import ( # noqa: E402 + install_stop_signals, pause, stopped, + BoundedJsonl, ReadOnlyRedis, container_ip, quantiles, write_json_exclusive, +) + +GATE = {"p95_le_ms": 5000.0, "max_lt_ms": 60000.0} +UNIT_MS = {"m": 60_000, "h": 3_600_000, "d": 86_400_000} +EXPECT_GRACE_MS = 60_000 + + +def interval_ms(interval: str) -> int: + return int(interval[:-1]) * UNIT_MS[interval[-1]] + + +def bar_slices(binding: dict) -> list[dict]: + rows = [] + for product in binding.get("products", []): + if product.get("feed") == "BAR" and product.get("interval"): + rows.append({"venue": product["venue"], "native_symbol": product["native_symbol"], + "interval": product["interval"]}) + return sorted(rows, key=lambda r: (r["venue"], r["native_symbol"], r["interval"])) + + +def resolve_keys(slices, keys) -> dict[str, str | None]: + """Map 'VENUE|native|interval' -> the last_bar key (VENUE:::), or None.""" + out = {} + for item in slices: + name = f"{item['venue']}|{item['native_symbol']}|{item['interval']}" + match = [k for k in keys if k.startswith(f"cache:market:v2:last_bar:{item['venue']}:") + and k.endswith(f":{item['native_symbol']}:{item['interval']}")] + out[name] = match[0] if len(match) == 1 else None + return out + + +class BarTracker: + def __init__(self, slices: list[str], intervals: dict[str, int]): + self.intervals = intervals + self.last_open = {name: None for name in slices} + self.primed = set() + self.seen = {name: {} for name in slices} # open_ms -> record + self.unparsed = 0 + + def observe(self, name: str, raw, now_ms: int): + """Feed one GET value; returns a record when a new closed bar becomes visible, else None.""" + if raw is None: + return None + try: + value = json.loads(raw) + open_ms, closed = int(value["timestamp_ms"]), value.get("is_closed") is True + except (ValueError, KeyError, TypeError): + self.unparsed += 1 + return None + first_tick = name not in self.primed + self.primed.add(name) + if not closed: + return None + if first_tick: + self.last_open[name] = open_ms # already in the cache at start: not measured + return None + if open_ms in self.seen[name] or (self.last_open[name] is not None and open_ms <= self.last_open[name]): + return None + self.last_open[name] = open_ms + close_ms = open_ms + self.intervals[name] + record = {"slice": name, "open_ms": open_ms, "close_ms": close_ms, "seen_ms": now_ms, + "transit_ms": now_ms - close_ms} + self.seen[name][open_ms] = record + return record + + def finish(self, start_ms: int, end_ms: int) -> dict: + per_slice, all_values, unseen = {}, {}, [] + for name, records in self.seen.items(): + step = self.intervals[name] + first = (start_ms // step + 1) * step + expected = [c for c in range(first, end_ms - EXPECT_GRACE_MS + 1, step) if c > start_ms + 5000] + seen_closes = {r["close_ms"] for r in records.values()} + missing = [c for c in expected if c not in seen_closes] + values = [r["transit_ms"] for r in records.values()] + for close in missing: + unseen.append({"slice": name, "close_ms": close, "lower_bound_ms": end_ms - close}) + bound = values + [end_ms - c for c in missing] + per_slice[name] = {**quantiles(values), "expected_closes": len(expected), "unseen": len(missing)} + venue = name.split("|", 1)[0] + all_values.setdefault(venue, []).extend(bound) + venues = {venue: quantiles(values) for venue, values in sorted(all_values.items())} + pooled = [v for values in all_values.values() for v in values] + return {"per_slice": per_slice, "per_venue": venues, "pooled": quantiles(pooled), "unseen": unseen[:200], + "unseen_total": len(unseen), "unparsed": self.unparsed} + + +def verdict(summary: dict, *, unresolved: int) -> str: + pooled = summary["pooled"] + if not pooled.get("n"): + return "NOT_MEASURED" + for block in summary["per_venue"].values(): + if block.get("n") and (block["p95"] > GATE["p95_le_ms"] or block["max"] >= GATE["max_lt_ms"]): + return "FAIL" + if unresolved or any(not s.get("n") and not s["unseen"] for s in summary["per_slice"].values()): + return "NOT_MEASURED" + return "PASS" + + +def run(args) -> dict: + binding = json.loads(Path(args.binding).read_text()) + slices = bar_slices(binding) + host = args.host or container_ip(args.container, args.network) + redis = ReadOnlyRedis(host, args.port, args.db) + keys = resolve_keys(slices, redis.scan("cache:market:v2:last_bar:*")) + names = [name for name, key in keys.items() if key] + intervals = {f"{s['venue']}|{s['native_symbol']}|{s['interval']}": interval_ms(s["interval"]) for s in slices} + tracker = BarTracker(names, {n: intervals[n] for n in names}) + out = Path(args.out) + out.mkdir(parents=True, exist_ok=False) + evidence = BoundedJsonl(out / "g3-bar-observations.jsonl", args.max_bytes) + errors = 0 + start_ms = time.time_ns() // 1_000_000 + deadline = time.monotonic() + args.seconds + period = args.poll_ms / 1000.0 + try: + while time.monotonic() < deadline and not stopped(): + tick = time.monotonic() + try: + values = redis.call("MGET", *[keys[n] for n in names]) if names else [] + except Exception as error: # a failed read is counted; it never fabricates a visibility time + errors += 1 + evidence.write({"kind": "read_error", "at_ms": time.time_ns() // 1_000_000, + "error": type(error).__name__}) + values = None + now_ms = time.time_ns() // 1_000_000 + for name, raw in zip(names, values or []): + record = tracker.observe(name, raw, now_ms) + if record: + evidence.write({"kind": "bar_visible", **record}) + pause(max(0.0, period - (time.monotonic() - tick))) + finally: + redis.close() + evidence.close() + end_ms = time.time_ns() // 1_000_000 + summary = tracker.finish(start_ms, end_ms) + unresolved = sorted(name for name, key in keys.items() if key is None) + report = {"schema": "qdl.p184.g3-bar-cache-probe.v1", "gate": GATE, "window_ms": [start_ms, end_ms], + "poll_ms": args.poll_ms, "measuring_point": "host MGET reply wall time; upper bound by one poll period", + "redis": {"container": args.container, "host": host, "db": args.db}, + "demanded_bar_slices": len(slices), "unresolved_slices": unresolved, "read_errors": errors, + "evidence_capped": evidence.capped, **summary} + report["verdict"] = verdict(summary, unresolved=len(unresolved)) + write_json_exclusive(out / "g3-summary.json", report) + return report + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--binding", required=True, help="TS consumer binding JSON (products[] with feed/interval)") + parser.add_argument("--container", default="redis_marketdata") + parser.add_argument("--network", default="executor_network") + parser.add_argument("--host", help="override the resolved container IP") + parser.add_argument("--port", type=int, default=6379) + parser.add_argument("--db", type=int, default=1) + parser.add_argument("--seconds", type=float, required=True) + parser.add_argument("--poll-ms", type=int, default=250) + parser.add_argument("--max-bytes", type=int, default=8 * 1024 * 1024) + parser.add_argument("--out", required=True) + args = parser.parse_args(argv) + install_stop_signals() + if not 50 <= args.poll_ms <= 5000 or not 0 < args.seconds <= 4 * 3600: + parser.error("poll-ms must be 50..5000 and seconds 0..14400") + report = run(args) + print(json.dumps({k: report[k] for k in ("verdict", "pooled", "unseen_total", "read_errors")}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/p184_g8_redis_sampler.py b/scripts/p184_g8_redis_sampler.py new file mode 100644 index 00000000..bd9ca903 --- /dev/null +++ b/scripts/p184_g8_redis_sampler.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""P18.4 gate G8: TS execution Redis command CPU, SLOWLOG count and PING RTT under full mixed traffic (read-only). + +Which Redis: the writer projects the execution market cache through TRADING_MARKET_REDIS_URL (TS shared/config.py, +default redis://redis_marketdata:6379/1; MARKET_REDIS_ROLLOUT=EPHEMERAL_ONLY writes only there, TS +services/market_data/redis_plane.py write_targets). TRADING_REDIS_URL (redis_service) carries heartbeats and the +control plane, not the APPLY scripts. The campaign runner records the writer's resolved URL beside this evidence. + +Samples (allowlisted read-only commands only, never MONITOR, never SLOWLOG RESET or CONFIG SET): +* PING every --ping-ms: client-side round trip (host -> container IP), ms; +* every --stats-seconds: INFO commandstats (cumulative usec/calls per command) and SLOWLOG GET 128 (id, start time, + duration, command name only - no arguments are kept) plus SLOWLOG threshold (CONFIG GET slowlog-log-slower-than); +* every 60 s: CLIENT LIST, reduced to the count of clients whose flags contain 'O' (MONITOR). + +Gate (frozen table, rev 2): sum of INFO commandstats usec deltas <= 60 % of one core over every 10-min window; SLOWLOG +(10 ms threshold) new-entry count 0 in steady state; PING p99 <= 5 ms under full mixed traffic. summarize() is shared +with the campaign scorer, which passes the steady window; a commandstats counter that goes backwards (Redis restart) +invalidates that window instead of producing a negative delta. +usage: p184_g8_redis_sampler.py --seconds N --out DIR [--container redis_marketdata] +""" +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.p184_resp_readonly import ( # noqa: E402 + install_stop_signals, pause, stopped, + BoundedJsonl, ReadOnlyRedis, container_ip, quantiles, read_jsonl, write_json_exclusive, +) + +GATE = {"commandstats_core_fraction_le": 0.60, "window_seconds": 600, "slowlog_steady_count": 0, + "ping_p99_le_ms": 5.0, "slowlog_threshold_us": 10000} + + +def parse_commandstats(text: str) -> dict[str, dict[str, int]]: + out = {} + for line in text.splitlines(): + if not line.startswith("cmdstat_"): + continue + name, _, fields = line.partition(":") + values = dict(item.split("=", 1) for item in fields.split(",") if "=" in item) + try: + out[name[len("cmdstat_"):]] = {"calls": int(values["calls"]), "usec": int(values["usec"])} + except (KeyError, ValueError): + continue + return out + + +def slowlog_rows(reply) -> list[dict]: + rows = [] + for entry in reply or []: + try: + command = entry[3][0].decode(errors="replace")[:32] if entry[3] else "?" + rows.append({"id": int(entry[0]), "start_s": int(entry[1]), "duration_us": int(entry[2]), + "command": command.upper()}) + except (IndexError, TypeError, ValueError): + continue + return rows + + +def monitor_clients(text: str) -> int: + count = 0 + for line in text.splitlines(): + for field in line.split(): + if field.startswith("flags=") and "O" in field[len("flags="):]: + count += 1 + return count + + +def summarize(samples, *, steady_from_ns=None, steady_to_ns=None, window_seconds=600) -> dict: + """Window the raw samples. Windows are consecutive window_seconds spans from the first stats sample; a window + shorter than 90 % of window_seconds (the tail) is reported but not judged.""" + stats = [s for s in samples if s.get("kind") == "stats"] + pings = [s for s in samples if s.get("kind") == "ping"] + slow, seen_ids = [], set() + baseline_ids = None + for s in stats: + ids = {row["id"] for row in s.get("slowlog", [])} + if baseline_ids is None: + baseline_ids = ids # entries already in the log at start are history, not this run + continue + for row in s.get("slowlog", []): + if row["id"] not in baseline_ids and row["id"] not in seen_ids: + seen_ids.add(row["id"]) + slow.append(row) + windows = [] + if len(stats) >= 2: + origin = stats[0]["at_ns"] + span = window_seconds * 1_000_000_000 + index = 0 + while True: + lo, hi = origin + index * span, origin + (index + 1) * span + inside = [s for s in stats if lo <= s["at_ns"] <= hi] + if len(inside) < 2: + break + first, last = inside[0], inside[-1] + seconds = (last["at_ns"] - first["at_ns"]) / 1e9 + deltas, valid = {}, True + for name, value in last["commandstats"].items(): + before = first["commandstats"].get(name, {"usec": 0, "calls": 0}) + delta = value["usec"] - before["usec"] + if delta < 0: + valid = False + deltas[name] = delta + total = sum(deltas.values()) + fraction = round(total / (seconds * 1e6), 4) if seconds > 0 and valid else None + ping = quantiles([p["rtt_ms"] for p in pings if lo <= p["at_ns"] < hi and p.get("rtt_ms") is not None]) + steady = (steady_from_ns is not None and steady_to_ns is not None + and lo >= steady_from_ns and hi <= steady_to_ns) + windows.append({ + "from_ns": first["at_ns"], "to_ns": last["at_ns"], "seconds": round(seconds, 1), + "judged": seconds >= 0.9 * window_seconds and valid, + "valid": valid, "steady": steady, "core_fraction": fraction, + "top_commands_usec": dict(sorted(deltas.items(), key=lambda kv: -kv[1])[:5]), + "slowlog_new": sum(1 for r in slow if lo // 10**9 <= r["start_s"] < hi // 10**9), + "ping_ms": ping, + }) + index += 1 + in_steady = lambda ns: (steady_from_ns is not None and steady_to_ns is not None # noqa: E731 + and steady_from_ns <= ns <= steady_to_ns) + steady_pings = [p["rtt_ms"] for p in pings if p.get("rtt_ms") is not None and in_steady(p["at_ns"])] + steady_slow = [r for r in slow if in_steady(r["start_s"] * 10**9)] + ping_failures = sum(1 for p in pings if p.get("rtt_ms") is None) + return { + "windows": windows, + "slowlog_new_total": len(slow), + "slowlog_new_steady": len(steady_slow), + "slowlog_steady_rows": steady_slow[:50], + "ping_all_ms": quantiles([p["rtt_ms"] for p in pings if p.get("rtt_ms") is not None]), + "ping_steady_ms": quantiles(steady_pings), + "ping_failures": ping_failures, + "monitor_clients_max": max((s.get("monitor_clients", 0) for s in stats if "monitor_clients" in s), default=None), + "slowlog_threshold_us": next((s["slowlog_threshold_us"] for s in stats if "slowlog_threshold_us" in s), None), + } + + +def verdict(summary: dict) -> tuple[str, list[str]]: + reasons = [] + judged = [w for w in summary["windows"] if w["judged"]] + for w in judged: + if w["core_fraction"] is not None and w["core_fraction"] > GATE["commandstats_core_fraction_le"]: + reasons.append(f"commandstats {w['core_fraction']} core > 0.60 in window from {w['from_ns']}") + if summary["slowlog_new_steady"] > 0: + reasons.append(f"SLOWLOG {summary['slowlog_new_steady']} new entries in steady state") + steady_ping = summary["ping_steady_ms"] + if steady_ping.get("p99") is not None and steady_ping["p99"] > GATE["ping_p99_le_ms"]: + reasons.append(f"PING p99 {steady_ping['p99']} ms > 5 ms") + if reasons: + return "FAIL", reasons + missing = [] + if not any(w["steady"] and w["judged"] for w in summary["windows"]): + missing.append("no judged 10-min window inside the steady state") + if steady_ping.get("p99") is None: + missing.append("steady PING n < 100") + if summary["slowlog_threshold_us"] != GATE["slowlog_threshold_us"]: + missing.append(f"SLOWLOG threshold is {summary['slowlog_threshold_us']} us, gate assumes 10000") + if summary["monitor_clients_max"]: + missing.append("MONITOR clients attached during the run (their cost is in commandstats)") + return ("NOT_MEASURED", missing) if missing else ("PASS", []) + + +def run(args) -> Path: + host = args.host or container_ip(args.container, args.network) + redis = ReadOnlyRedis(host, args.port, 0) + out = Path(args.out) + out.mkdir(parents=True, exist_ok=False) + evidence = BoundedJsonl(out / "g8-samples.jsonl", args.max_bytes) + evidence.write({"kind": "scope", "container": args.container, "host": host, "ping_ms": args.ping_ms, + "stats_seconds": args.stats_seconds, "commands": "PING, INFO commandstats, SLOWLOG GET 128, " + "CONFIG GET slowlog-log-slower-than, CLIENT LIST (MONITOR flag count only)"}) + deadline = time.monotonic() + args.seconds + next_stats = next_clients = time.monotonic() + try: + while time.monotonic() < deadline and not stopped(): + tick = time.monotonic() + try: + started = time.perf_counter_ns() + redis.call("PING") + evidence.write({"kind": "ping", "at_ns": time.time_ns(), + "rtt_ms": round((time.perf_counter_ns() - started) / 1e6, 3)}) + except Exception as error: + evidence.write({"kind": "ping", "at_ns": time.time_ns(), "rtt_ms": None, + "error": type(error).__name__}) + if tick >= next_stats: + next_stats += args.stats_seconds + try: + info, slowlog, threshold = redis.pipeline([("INFO", "commandstats"), ("SLOWLOG", "GET", 128), + ("CONFIG", "GET", "slowlog-log-slower-than")]) + sample = {"kind": "stats", "at_ns": time.time_ns(), + "commandstats": parse_commandstats(info.decode()), + "slowlog": slowlog_rows(slowlog), + "slowlog_threshold_us": int(threshold[1])} + if tick >= next_clients: + next_clients += 60 + sample["monitor_clients"] = monitor_clients(redis.call("CLIENT", "LIST").decode()) + evidence.write(sample) + except Exception as error: + evidence.write({"kind": "stats_error", "at_ns": time.time_ns(), "error": type(error).__name__}) + pause(max(0.0, args.ping_ms / 1000.0 - (time.monotonic() - tick))) + finally: + redis.close() + evidence.close() + summary = summarize(list(read_jsonl(out / "g8-samples.jsonl"))) + state, reasons = verdict(summary) + write_json_exclusive(out / "g8-summary.json", {"schema": "qdl.p184.g8-redis-sampler.v1", "gate": GATE, + "redis": {"container": args.container, "host": host}, + "steady_window": "not applied (whole run)", **summary, + "self_verdict_whole_run": state, "reasons": reasons}) + return out + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--container", default="redis_marketdata") + parser.add_argument("--network", default="executor_network") + parser.add_argument("--host") + parser.add_argument("--port", type=int, default=6379) + parser.add_argument("--seconds", type=float, required=True) + parser.add_argument("--ping-ms", type=int, default=500) + parser.add_argument("--stats-seconds", type=float, default=10.0) + parser.add_argument("--max-bytes", type=int, default=32 * 1024 * 1024) + parser.add_argument("--out", required=True) + args = parser.parse_args(argv) + install_stop_signals() + if not 100 <= args.ping_ms <= 10000 or not 0 < args.seconds <= 4 * 3600: + parser.error("ping-ms must be 100..10000 and seconds 0..14400") + out = run(args) + print(json.dumps({"output": str(out)})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/p184_g9_g10_writer_counters.py b/scripts/p184_g9_g10_writer_counters.py new file mode 100644 index 00000000..0741ceab --- /dev/null +++ b/scripts/p184_g9_g10_writer_counters.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +"""P18.4 gates G9 (readiness) and G10 (TRADE integrity): writer heartbeat sampler plus writer log counters (read-only). + +G9 source: the writer heartbeat `service:heartbeat:market_data:*` on TRADING_REDIS_URL (redis_service DB 0), sampled +every --sample-seconds. Each readable sample records demanded / session-ready / execution-ready / quiet / unhealthy +counts, v1_fallback_count, v2_blocked/v2_error, the reported unhealthy rows (state, error_code) and, when the writer +image carries TS fb09a1b (purpose readiness), pricing/continuity/signal counts and the bounded `not_ready` rows with +their reason_code. An unreadable or stale heartbeat (last_seen older than 45 s; +the writer publishes about every 30 s, stage100m max age 30.0 s) is counted UNKNOWN, never READY. +A heartbeat key that changes (writer restart: new instance id) is re-scanned. + +Eligibility losses (G9): a slice that is in `not_ready` in a sample but was not in the previous readable sample is +one loss event, classified by reason_code. TRADE_EVENT_STALE (a quiet TRADE whose provider session is live) is the +only legitimate quiet refusal; it is counted and reported, not gated. Every other reason is a data-path loss. Without +purpose fields (writer older than fb09a1b) the loss count is NOT_MEASURED; execution_ready drops are reported only as +a diagnostic. + +G10 source: there is no dedicated TRADE integrity counter in the writer or SDK (checked: TS +adapters/market_data/data_layer_v2.py, services/market_data/*, qdl_sdk). The typed signals that do exist are its log +lines, read incrementally from the host-mounted writer log files (market_data_worker-*.log, market_data_service.log): +* `Data Layer V2 slice disconnected ... feed=TRADE ... reason= code=`: + CursorExpiredError / CURSOR_EXPIRED / CURSOR_INVALID -> replay_expired; SlowConsumerError (gRPC RESOURCE_EXHAUSTED, + qdl_sdk/transport.py) -> overflow; ContinuityError or a GAP / OPEN_SEQUENCE code -> gap; anything else -> other; +* `data layer slice control code= ... feed=TRADE`: SNAPSHOT_REPLACED -> gap (state rebuilt, continuity broken); + RECONNECTED -> resumed (committed-cursor resume, not a gap; counted). +Duplicates applied come from the delivery trace (scripts/p184_trace_feed_transit.py duplicate_applied_event_id), not +from here. A silent offset gap that the Data Layer does not signal is not observable from these sources; the proposed +minimal additive counter is in the campaign report. + +Boundary: read-only GET/SCAN on the heartbeat Redis; reads log files the host already owns read access to; bounded by +--seconds and --max-bytes. usage: p184_g9_g10_writer_counters.py --seconds N --log-dir DIR --out DIR +""" +from __future__ import annotations + +import argparse +import json +import re +import sys +import time +from collections import Counter +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.p184_resp_readonly import ( # noqa: E402 + install_stop_signals, pause, stopped, + BoundedJsonl, ReadOnlyRedis, container_ip, read_jsonl, write_json_exclusive, +) + +LEGIT_QUIET = frozenset({"TRADE_EVENT_STALE"}) +HEARTBEAT_PATTERN = "service:heartbeat:market_data:*" +COUNT_FIELDS = ("demanded_v2_slices", "session_ready_v2_slices", "execution_ready_v2_slices", + "quiet_session_v2_slices", "unhealthy_v2_slices", "v1_fallback_count", "v2_blocked_count", + "v2_error_count", "continuity_valid", "continuity_required", "pricing_eligible", "pricing_required", + "signal_data_usable", "signal_required") +LOG_STAMP = re.compile(r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}),(\d{3}) ") +DISCONNECT = re.compile(r"slice disconnected venue=(\S+) symbol=(\S+) feed=(\S+) interval=(\S+) reason=(\w+)" + r"(?: code=([A-Z_]+))?") +CONTROL = re.compile(r"slice control code=([A-Z_]+) venue=(\S+) symbol=(\S+) feed=(\S+)") + + +def heartbeat_sample(payload: dict | None, now_s: float) -> dict: + if not isinstance(payload, dict) or not isinstance(payload.get("details"), dict): + return {"readable": False} + details = payload["details"] + sample = {"readable": True, "status": payload.get("status"), + "age_s": round(now_s - float(payload.get("last_seen_unix") or 0), 1), + **{key: details.get(key) for key in COUNT_FIELDS if key in details}, + "unhealthy": [{k: row.get(k) for k in ("provider", "symbol", "feed", "interval", "state", "error_code")} + for row in details.get("reported_unhealthy_slices", [])[:64] if isinstance(row, dict)], + "purpose": "purpose_readiness_schema" in details and "not_ready" in details} + if sample["purpose"]: + sample["not_ready"] = [{k: row.get(k) for k in ("provider", "symbol", "feed", "interval", "reason_code", + "component_age_ms")} + for row in details.get("not_ready", [])[:64] if isinstance(row, dict)] + sample["not_ready_unreported"] = details.get("not_ready_unreported") + if "purpose_readiness_error" in details: + sample["purpose_error"] = details["purpose_readiness_error"] + return sample + + +def slice_key(row: dict) -> str: + return "|".join(str(row.get(k)) for k in ("provider", "symbol", "feed", "interval")) + + +def summarize_g9(samples, *, from_ns=None, to_ns=None, stale_after_s=45.0) -> dict: + rows = [s for s in samples if s.get("kind") == "heartbeat" + and (from_ns is None or s["at_ns"] >= from_ns) and (to_ns is None or s["at_ns"] <= to_ns)] + unknown = [s for s in rows if not s["sample"]["readable"] or s["sample"]["age_s"] > stale_after_s] + good = [s["sample"] for s in rows if s not in unknown] + mismatch = [s for s in good if s.get("demanded_v2_slices") != s.get("session_ready_v2_slices")] + session_causes = Counter(f"{row['feed']}:{row['state']}:{row['error_code']}" + for s in mismatch for row in s["unhealthy"]) + purpose = bool(good) and all(s["purpose"] for s in good) + losses, legit, previous = Counter(), Counter(), None + loss_rows = [] + for s in good: + if not s["purpose"]: + previous = None + continue + current = {slice_key(row): row for row in s.get("not_ready", [])} + if previous is not None: + for key, row in current.items(): + if key not in previous: + cause = f"{row.get('feed')}:{row.get('reason_code')}" + if row.get("reason_code") in LEGIT_QUIET: + legit[cause] += 1 + else: + losses[cause] += 1 + if len(loss_rows) < 100: + loss_rows.append(row) + previous = current + fallback = [s.get("v1_fallback_count") for s in good if isinstance(s.get("v1_fallback_count"), int)] + demanded = sorted({s.get("demanded_v2_slices") for s in good}) + exec_ready = [s.get("execution_ready_v2_slices") for s in good if isinstance(s.get("execution_ready_v2_slices"), int)] + return { + "samples": len(rows), "unknown_samples": len(unknown), "readable_samples": len(good), + "demanded_values": demanded, + "demanded_ne_session_ready_samples": len(mismatch), + "session_loss_causes": dict(session_causes), + "purpose_fields_present": purpose, + "eligibility_losses_data_path": sum(losses.values()) if purpose else None, + "eligibility_loss_causes": dict(losses) if purpose else None, + "eligibility_loss_rows": loss_rows, + "legit_quiet_refusals": dict(legit) if purpose else None, + "v1_fallback_delta": (max(fallback) - min(fallback)) if fallback else None, + "v1_fallback_max": max(fallback) if fallback else None, + "execution_ready_min": min(exec_ready) if exec_ready else None, + "execution_ready_max": max(exec_ready) if exec_ready else None, + "purpose_errors": sum(1 for s in good if "purpose_error" in s), + } + + +def verdict_g9(summary: dict, *, expected_demanded: int | None) -> tuple[str, list[str]]: + reasons = [] + if summary["demanded_ne_session_ready_samples"]: + reasons.append(f"demanded != session-ready in {summary['demanded_ne_session_ready_samples']} samples") + if summary["eligibility_losses_data_path"]: + reasons.append(f"{summary['eligibility_losses_data_path']} data-path eligibility losses") + if summary["v1_fallback_max"]: + reasons.append(f"v1_fallback_count {summary['v1_fallback_max']}") + if reasons: + return "FAIL", reasons + gaps = [] + if not summary["readable_samples"]: + gaps.append("no readable heartbeat sample") + if summary["unknown_samples"]: + gaps.append(f"{summary['unknown_samples']} unreadable/stale heartbeat samples") + if not summary["purpose_fields_present"]: + gaps.append("heartbeat has no purpose readiness (writer older than TS fb09a1b): losses not countable") + if expected_demanded is not None and summary["demanded_values"] != [expected_demanded]: + gaps.append(f"demanded {summary['demanded_values']} != final manifest {expected_demanded}") + if summary["purpose_errors"]: + gaps.append(f"{summary['purpose_errors']} purpose readiness evaluation errors") + return ("NOT_MEASURED", gaps) if gaps else ("PASS", []) + + +def classify_line(line: str): + """Return (stamp 'YYYY-MM-DD HH:MM:SS', category, detail) for a TRADE integrity line, else None.""" + stamp = LOG_STAMP.match(line) + if not stamp: + return None + match = DISCONNECT.search(line) + if match: + feed, reason, code = match.group(3), match.group(5), match.group(6) or "" + if feed != "TRADE": + return stamp.group(1), "other_feed_disconnect", f"{feed}:{reason}:{code}" + if reason == "CursorExpiredError" or code in {"CURSOR_EXPIRED", "CURSOR_INVALID"}: + category = "replay_expired" + elif reason == "SlowConsumerError": + category = "overflow" + elif reason == "ContinuityError" or "GAP" in code or "OPEN_SEQUENCE" in code: + category = "gap" + else: + category = "trade_disconnect_other" + return stamp.group(1), category, f"{match.group(1)}:{match.group(2)}:{reason}:{code}" + match = CONTROL.search(line) + if match and match.group(4) == "TRADE": + code = match.group(1) + category = "gap" if code == "SNAPSHOT_REPLACED" else "resumed" if code == "RECONNECTED" else "control_other" + return stamp.group(1), category, f"{match.group(2)}:{match.group(3)}:{code}" + return None + + +class LogTail: + """Incremental reader over a fixed set of files, starting at their size at construction.""" + + def __init__(self, paths): + self.offsets = {} + for path in paths: + try: + self.offsets[Path(path)] = Path(path).stat().st_size + except OSError: + self.offsets[Path(path)] = None + self.unreadable = 0 + + def read_new(self): + lines = [] + for path, offset in list(self.offsets.items()): + try: + size = path.stat().st_size + if offset is None or size < offset: + offset = 0 # created or truncated after start + with path.open("rb") as handle: + handle.seek(offset) + data = handle.read(min(size - offset, 64 * 1024 * 1024)) + # Only complete lines; a partial last line is read next time. + cut = data.rfind(b"\n") + 1 + self.offsets[path] = offset + cut + lines.extend(data[:cut].decode(errors="replace").splitlines()) + except OSError: + self.unreadable += 1 + return lines + + +def summarize_g10(samples, *, from_s=None, to_s=None) -> dict: + counts, codes, unreadable, polls = Counter(), Counter(), 0, 0 + examples = [] + for s in samples: + if s.get("kind") == "log_poll": + polls += 1 + unreadable += s.get("unreadable", 0) + if s.get("kind") != "log_event": + continue + if from_s is not None and s["stamp"] < from_s or to_s is not None and s["stamp"] > to_s: + continue + counts[s["category"]] += 1 + codes[f"{s['category']}|{s['detail'].split(':', 2)[-1]}"] += 1 + if s["category"] in {"gap", "replay_expired", "overflow"} and len(examples) < 50: + examples.append(s) + return {"log_polls": polls, "log_unreadable": unreadable, + "trade_gap": counts["gap"], "trade_replay_expired": counts["replay_expired"], + "trade_overflow": counts["overflow"], "trade_resumed": counts["resumed"], + "trade_disconnect_other": counts["trade_disconnect_other"], "other_feed_disconnects": counts[ + "other_feed_disconnect"], "codes": dict(codes), "examples": examples} + + +def run(args) -> Path: + out = Path(args.out) + out.mkdir(parents=True, exist_ok=False) + evidence = BoundedJsonl(out / "g9-g10-samples.jsonl", args.max_bytes) + host = args.host or container_ip(args.container, args.network) + redis = ReadOnlyRedis(host, args.port, args.db) + log_dir = Path(args.log_dir) + tail = LogTail(sorted(log_dir.glob("market_data_worker-*.log")) + [log_dir / "market_data_service.log"]) + evidence.write({"kind": "scope", "heartbeat_redis": {"container": args.container, "host": host, "db": args.db}, + "log_dir": str(log_dir), "files": [str(p) for p in tail.offsets]}) + key = None + deadline = time.monotonic() + args.seconds + next_log = time.monotonic() + try: + while time.monotonic() < deadline and not stopped(): + tick = time.monotonic() + payload = None + try: + if key is None: + keys = redis.scan(HEARTBEAT_PATTERN, limit=16) + key = keys[0] if len(keys) == 1 else None + if len(keys) > 1: + evidence.write({"kind": "heartbeat_keys_ambiguous", "count": len(keys)}) + raw = redis.call("GET", key) if key else None + payload = json.loads(raw) if raw else None + if raw is None: + key = None # instance changed (writer restart): re-scan next tick + except Exception as error: + key = None + evidence.write({"kind": "heartbeat_error", "at_ns": time.time_ns(), "error": type(error).__name__}) + evidence.write({"kind": "heartbeat", "at_ns": time.time_ns(), "key_tail": (key or "")[-24:], + "sample": heartbeat_sample(payload, time.time())}) + if tick >= next_log: + next_log += args.log_seconds + before = tail.unreadable + for line in tail.read_new(): + item = classify_line(line) + if item: + stamp, category, detail = item + evidence.write({"kind": "log_event", "stamp": stamp, "category": category, "detail": detail}) + evidence.write({"kind": "log_poll", "at_ns": time.time_ns(), "unreadable": tail.unreadable - before}) + pause(max(0.0, args.sample_seconds - (time.monotonic() - tick))) + finally: + redis.close() + evidence.close() + samples = list(read_jsonl(out / "g9-g10-samples.jsonl")) + g9 = summarize_g9(samples) + write_json_exclusive(out / "g9-g10-summary.json", {"schema": "qdl.p184.g9-g10-writer-counters.v1", + "g9": g9, "g9_verdict_whole_run": verdict_g9(g9, expected_demanded=None), + "g10": summarize_g10(samples)}) + return out + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--container", default="redis_service", help="TRADING_REDIS_URL host (heartbeats)") + parser.add_argument("--network", default="executor_network") + parser.add_argument("--host") + parser.add_argument("--port", type=int, default=6379) + parser.add_argument("--db", type=int, default=0) + parser.add_argument("--log-dir", required=True, help="host path mounted at the writer's /app/logs") + parser.add_argument("--seconds", type=float, required=True) + parser.add_argument("--sample-seconds", type=float, default=10.0) + parser.add_argument("--log-seconds", type=float, default=15.0) + parser.add_argument("--max-bytes", type=int, default=32 * 1024 * 1024) + parser.add_argument("--out", required=True) + args = parser.parse_args(argv) + install_stop_signals() + if not 1 <= args.sample_seconds <= 60 or not 0 < args.seconds <= 4 * 3600: + parser.error("sample-seconds must be 1..60 and seconds 0..14400") + print(json.dumps({"output": str(run(args))})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/p184_gate_campaign.py b/scripts/p184_gate_campaign.py new file mode 100644 index 00000000..fc3243cf --- /dev/null +++ b/scripts/p184_gate_campaign.py @@ -0,0 +1,758 @@ +#!/usr/bin/env python3 +"""P18.4 phase D final gate-measurement campaign: one run, every probe, one G1-G14 scorecard. + +Purpose: run the frozen measurement protocol of the TS gate table (upgrade/p184-phase-d-gates.md, FROZEN 2026-10-09: +steady state >= 30 min at the stage-100 workload plus one burst window and the fault windows) ONCE, with every +probe the v2.2.3 scorecard listed as missing, and score G1-G14 from that single run. + +What `run` starts (all children of this process, tracked by PID, each with its own deadline; nothing matched by name): +* preflight (read-only): writer running, its delivery trace armed long enough (TS_DELIVERY_TRACE_UNTIL_UNIX_NS covers + the load window; the writer must already have been recreated with the long-window trace, TS b642e0d), the market + Redis the writer actually projects into (TRADING_MARKET_REDIS_URL read from the writer's settings), host disk; +* the stage-100 FINAL driver run (scripts/phase3_consumer_load_acceptance.py --mode target --sessions 100 + --duration-seconds 1800 with the frozen budget: BURST, SLOW_READER and RECONNECT fault windows), with the existing + guard_v2.sh / mark_watch_v2.sh stop conditions (they SIGINT the driver PID only), and a profile derived from the base + profile plus binance_bar_edge and stable_redis as monitored containers (G14); +* G4 observer windows (scripts/p184_g4_observer.py), four 120 s windows plus one 180 s window around a UTC :00/:30 + boundary when one falls inside the load, so a 30m close can be seen; +* G3 BAR cache poller, G8 Redis sampler, G9/G10 writer counters for the whole load (stopped by SIGTERM after it); +* G11 (optional, after the load because any monitored-container restart aborts the driver): the restart probe arms and + waits for the COORDINATOR to restart the writer; this tool never restarts anything. +Then `score` (also callable alone on a finished campaign dir) runs the trace analyses (G1, G2, G7, G10 duplicates) in a +disposable `--network none` container of the already present DL python image, and writes scorecard.json. + +Scoring rule (v2.2.3 scorecard, unchanged): verdicts PASS / FAIL / NOT_MEASURED; a measurement outside the protocol +can FAIL a gate but cannot PASS it; missing coverage is NOT_MEASURED; nothing is ever upgraded. G12 and G13 are not part +of this campaign and are reported NOT_MEASURED with that reason. + +Boundary: no image builds, no container create/recreate/restart, no Redis/Kafka writes. Children: the driver's own +disposable client container (removed by the driver), the G4 observer (docker exec of a read-only consumer, as before) +and the trace analysis container (--rm, --network none, read-only mounts). +usage: p184_gate_campaign.py run [--dry-run] [--g11] ... | p184_gate_campaign.py score --campaign-dir DIR +""" +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +import shutil +import signal +import subprocess +import sys +import threading +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts import p184_g8_redis_sampler as g8 # noqa: E402 +from scripts import p184_g9_g10_writer_counters as g910 # noqa: E402 +from scripts import p184_g14_resource_table as g14 # noqa: E402 +from scripts.p184_resp_readonly import parse_redis_url, read_jsonl # noqa: E402 + +STATE = Path.home() / ".local/state/qdl-v2" +TOOLS = STATE / "p184-d3-capacity-20261009" +WRITER = "market_data_service_p184w" +EXTRA_MONITORED = ("qdl_v2_stable_candidate-binance_bar_edge-1", "qdl_v2_stable_candidate-stable_redis-1") +LOAD_SECONDS = 1800 +DRIVER_TIMEOUT = 2400 +G4_OFFSETS = (120, 480, 840, 1200) +G4_SECONDS = 120 +G4_BOUNDARY_SECONDS = 180 +MIN_DISK_GB = 20 +TRACE_FEEDS_G1 = ("BOOK_DELTA", "BOOK_SNAPSHOT") +TRACE_FEEDS_G2 = ("TRADE", "QUOTE") +G7_FEEDS = ("BOOK_DELTA", "BOOK_SNAPSHOT", "QUOTE", "TRADE", "MARK_INDEX_PRICE") +VENUES = ("BINANCE", "OKX") +G6_REQUIRED = tuple(f"{feed}|{venue}|{interval}" for feed, interval in + (("TRADE", ""), ("QUOTE", ""), ("MARK_INDEX_PRICE", ""), ("BOOK_SNAPSHOT", ""), + ("BOOK_DELTA", ""), ("BAR", "1m")) for venue in VENUES) +G6_GATED_FEEDS = ("TRADE", "QUOTE", "MARK_INDEX_PRICE", "BOOK_SNAPSHOT", "BOOK_DELTA", "BAR") + + +# ---------------------------------------------------------------- plan helpers (pure, unit-tested) + +def g4_schedule(launch_unix: float, *, offsets=G4_OFFSETS, seconds=G4_SECONDS, boundary_seconds=G4_BOUNDARY_SECONDS, + load_end_offset: float = LOAD_SECONDS + 80) -> list[dict]: + """Observer windows as (offset from launch, seconds). Adds a window from 30 s before a UTC :00/:30 boundary for + boundary_seconds when one fits inside the load; default windows that would overlap it are dropped.""" + windows = [{"offset": float(o), "seconds": float(seconds), "boundary": None} for o in offsets] + first = launch_unix + 150 + 30 + boundary = (int(first) // 1800 + 1) * 1800 + if boundary - 30 - launch_unix + boundary_seconds <= load_end_offset: + start = boundary - 30 - launch_unix + extra = {"offset": round(start, 3), "seconds": float(boundary_seconds), "boundary": boundary} + windows = [w for w in windows if w["offset"] + w["seconds"] + 5 <= start + or w["offset"] >= start + boundary_seconds + 5] + windows.append(extra) + return sorted(windows, key=lambda w: w["offset"]) + + +def derive_profile(base: dict, extra=EXTRA_MONITORED) -> dict: + profile = json.loads(json.dumps(base)) + monitored = list(profile.get("monitored_containers") or []) + for name in extra: + if name not in monitored: + monitored.append(name) + profile["monitored_containers"] = monitored + return profile + + +def mount_source(inspect: dict, destination: str) -> str | None: + for mount in inspect.get("Mounts", []): + if mount.get("Destination") == destination: + return mount.get("Source") + return None + + +def host_path(inspect: dict, container_path: str) -> str | None: + """Map a container path to its host path through the longest matching bind mount.""" + best = None + for mount in inspect.get("Mounts", []): + dest = mount.get("Destination", "") + if container_path == dest or container_path.startswith(dest.rstrip("/") + "/"): + if best is None or len(dest) > len(best["Destination"]): + best = mount + if best is None: + return None + return best["Source"] + container_path[len(best["Destination"]):] + + +def env_of(inspect: dict) -> dict[str, str]: + return dict(item.split("=", 1) for item in inspect.get("Config", {}).get("Env", []) if "=" in item) + + +# ---------------------------------------------------------------- scoring (pure, unit-tested) + +def _protocol(campaign: dict, receipt: dict | None, gates: dict | None) -> tuple[bool, list[str]]: + problems = [] + if receipt is None: + problems.append("no driver receipt") + else: + window = receipt.get("load_window_unix_s") + if not window or window[1] - window[0] < LOAD_SECONDS: + problems.append("load window < 1800 s or unknown") + if not (receipt.get("fault_windows") or {}).get("BURST"): + problems.append("no BURST window") + if receipt.get("stage") != 100 or not receipt.get("final"): + problems.append("not the stage-100 final run") + if gates is None or gates.get("status") != "PASS": + problems.append("driver acceptance not PASS: " + ",".join((gates or {}).get("failed_gates", ["no gates"]))[:300]) + return not problems, problems + + +def _gate(gid, quantity, rule, verdict, reason, protocol_ok, protocol_problems, **extra): + if verdict == "PASS" and not protocol_ok: + verdict, reason = "NOT_MEASURED", f"inside the rule but outside the protocol: {'; '.join(protocol_problems)}" + return {"id": gid, "quantity": quantity, "rule": rule, "verdict": verdict, "reason": reason, **extra} + + +def _trace_coverage(transit: dict | None, load_ns, until_ns: int | None) -> list[str]: + """Trace-derived gates need the trace armed through the whole load and no worker capped (a CAP stop means the + last allowed part filled and later deliveries were not recorded).""" + if transit is None: + return ["no trace analysis"] + gaps = [] + if transit.get("error"): + gaps.append(f"trace analysis: {transit['error']}") + if not transit.get("records"): + gaps.append("no trace records") + if transit.get("dropped_at_export_total"): + gaps.append(f"{transit['dropped_at_export_total']} trace records dropped at export") + if (transit.get("stop_reasons") or {}).get("CAP"): + gaps.append(f"{transit['stop_reasons']['CAP']} trace activations stopped at their file cap") + if load_ns and (not until_ns or until_ns < load_ns[1]): + gaps.append("trace expiry is before the end of the load window") + return gaps + + +def score_transit_rows(transit: dict | None, feeds) -> tuple[str, list, list[str]]: + rows, missing, outside = [], [], [] + for feed in feeds: + for venue in VENUES: + block = ((transit or {}).get("transit") or {}).get(f"{feed}|{venue}") + if not block or not block.get("n"): + missing.append(f"{feed}|{venue}") + continue + rows.append({"feed_venue": f"{feed}|{venue}", **{k: block.get(k) for k in ("n", "p50", "p95", "p99", "max")}, + "observed": block.get("verdict"), "split_p95_ms": {k: v.get("p95") for k, v in + (block.get("split") or {}).items()}}) + if block.get("verdict") == "OUTSIDE": + outside.append(f"{feed}|{venue}") + if outside: + return "FAIL", rows, [f"outside: {', '.join(outside)}"] + if missing: + return "NOT_MEASURED", rows, [f"no data: {', '.join(missing)}"] + return "PASS", rows, [] + + +def score_mark(mark: dict | None) -> tuple[str, list, list[str]]: + rows, missing, outside = [], [], [] + for venue in VENUES: + block = ((mark or {}).get("venues") or {}).get(venue) + gated = (block or {}).get("new_event_transit_ms") or {} + if not gated.get("n"): + missing.append(f"MARK|{venue}") + continue + rows.append({"feed_venue": f"MARK_INDEX_PRICE|{venue}", **{k: gated.get(k) for k in ("n", "p50", "p95", "p99", "max")}, + "observed": block.get("g2_verdict")}) + if block.get("g2_verdict") == "OUTSIDE": + outside.append(f"MARK|{venue}") + if outside: + return "FAIL", rows, [f"outside: {', '.join(outside)}"] + if missing: + return "NOT_MEASURED", rows, [f"no data: {', '.join(missing)}"] + return "PASS", rows, [] + + +def score_g7(transit: dict | None) -> tuple[str, dict, list[str]]: + g7 = (transit or {}).get("g7") or {} + rows, outside, missing = {}, [], [] + for feed in G7_FEEDS: + block = g7.get(feed) + if not block or "verdict" not in block: + missing.append(feed) + continue + rows[feed] = {name: block.get(name) for name in ("sdk_to_writer_ms", "writer_to_redis_ms")} + for name, state in block["verdict"].items(): + if state == "OUTSIDE": + outside.append(f"{feed}:{name}") + elif state == "NO_DATA" and not (feed == "MARK_INDEX_PRICE" and name == "sdk_to_writer_ms"): + missing.append(f"{feed}:{name}") + if outside: + return "FAIL", rows, [f"outside: {', '.join(outside)}"] + if missing: + return "NOT_MEASURED", rows, [f"no data: {', '.join(missing)}"] + return "PASS", rows, [] + + +def score_g4(windows: list[dict], table: dict) -> tuple[str, list, list[str]]: + """windows: [{'summary': {row: {'visibility_ms': {...}}}, 'inverse_rows': {...}}]; table: budget rows.""" + rows, failed, uncovered = [], [], [] + for key, limit in sorted(table.items()): + values = [((w.get("summary") or {}).get(key) or {}).get("visibility_ms", {}) for w in windows] + p95 = [v.get("p95") if v.get("n") else None for v in values] + n = [v.get("n", 0) for v in values] + verdict = ("NOT_MEASURED" if not any(n) else + "FAIL" if any(x is not None and x > limit["candidate_p95_limit_ms"] for x in p95) else "PASS") + rows.append({"row": key, "p95_ms_per_window": p95, "n_per_window": n, + "limit_p95_ms": limit["candidate_p95_limit_ms"], "verdict": verdict}) + if verdict == "FAIL": + failed.append(key) + elif verdict == "NOT_MEASURED": + uncovered.append(key) + observed = {key for w in windows for key in (w.get("summary") or {})} + inverse = {key for w in windows for key in (w.get("inverse_rows") or {})} + unbaselined = sorted((observed | inverse) - set(table)) + reasons = [] + if failed: + reasons.append(f"over limit: {', '.join(failed)}") + if uncovered: + reasons.append(f"no sample in any window: {', '.join(uncovered)}") + if unbaselined: + reasons.append(f"observed rows without a frozen baseline: {', '.join(unbaselined)}") + if failed: + return "FAIL", rows, reasons + if uncovered or unbaselined: + return "NOT_MEASURED", rows, reasons + return "PASS", rows, [] + + +def g4_inverse_rows(observations) -> dict: + """Separate OKX inverse (native symbol *-USD-SWAP) visibility from the pooled OKX rows; same quantity.""" + groups = {} + for item in observations: + if item.get("kind") != "canonical_observation": + continue + symbol = (item.get("identity") or {}).get("native_symbol") or "" + if not symbol.endswith("-USD-SWAP"): + continue + source, seen = item.get("source_event_ns"), item.get("observer_receipt_wall_ns") + if source and seen: + key = f"{item['feed']}|OKX_INVERSE|{item.get('interval') or ''}" + groups.setdefault(key, []).append((seen - source) / 1e6) + out = {} + for key, values in sorted(groups.items()): + ordered = sorted(values) + out[key] = {"n": len(ordered), "p95": round(ordered[min(len(ordered) - 1, int(.95 * len(ordered)))], 1), + "max": round(ordered[-1], 1)} + return out + + +def score_g6(quality: dict | None) -> tuple[str, dict, list[str]]: + quality = quality or {} + unsafe = {key: row.get("unsafe", 0) for key, row in quality.items() + if key.split("|", 1)[0] in G6_GATED_FEEDS and row.get("unsafe")} + rows = {key: {k: row.get(k) for k in ("n", "policy_ms", "age_ms", "max_age_ms", "classes", "refusals", "reasons", + "unsafe")} for key, row in quality.items()} + if unsafe: + return "FAIL", rows, [f"unsafe eligibility: {unsafe}"] + missing = [key for key in G6_REQUIRED if not (quality.get(key) or {}).get("n")] + if missing: + return "NOT_MEASURED", rows, [f"no returned view for: {', '.join(missing)} (BOOK_DELTA is not a Query shape of " + "the frozen stage-100 workload)"] + return "PASS", rows, [] + + +def score_g10(g10: dict, duplicates: dict | None, trace_gaps: list[str]) -> tuple[str, dict, list[str]]: + dup = {k: v for k, v in (duplicates or {}).items() if k.startswith("TRADE|")} + counters = {"gaps": g10.get("trade_gap"), "duplicates_applied": sum(dup.values()) if duplicates is not None else None, + "replay_expired": g10.get("trade_replay_expired"), "overflow": g10.get("trade_overflow"), + "resumed_reconnects": g10.get("trade_resumed"), "log_polls": g10.get("log_polls"), + "log_unreadable": g10.get("log_unreadable"), "codes": g10.get("codes")} + bad = {k: counters[k] for k in ("gaps", "duplicates_applied", "replay_expired", "overflow") if counters[k]} + if bad: + return "FAIL", counters, [f"non-zero: {bad}"] + gaps = [] + if not g10.get("log_polls") or g10.get("log_unreadable"): + gaps.append("writer logs not read for the whole window") + if duplicates is None or trace_gaps: + gaps.append("duplicates applied need the delivery trace: " + "; ".join(trace_gaps or ["no trace"])) + return ("NOT_MEASURED", counters, gaps) if gaps else ("PASS", counters, []) + + +def utc_stamp(unix_s: float) -> str: + return dt.datetime.fromtimestamp(unix_s, dt.timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + +def load_json(path: Path): + try: + return json.loads(Path(path).read_text()) + except (OSError, ValueError): + return None + + +def sha256_file(path: Path) -> str | None: + try: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + except OSError: + return None + + +def build_scorecard(campaign_dir: Path) -> dict: + d = Path(campaign_dir) + campaign = load_json(d / "campaign.json") or {} + receipt = load_json(d / "stage100" / "receipt.json") + gates = load_json(d / "stage100" / "gates.json") + host = load_json(d / "stage100" / "host.json") + ok, problems = _protocol(campaign, receipt, gates) + load_s = (receipt or {}).get("load_window_unix_s") + load_ns = [int(load_s[0] * 1e9), int(load_s[1] * 1e9)] if load_s else None + transit = load_json(d / "trace" / "transit-load.json") + mark = load_json(d / "trace" / "mark-load.json") + trace_gaps = _trace_coverage(transit, load_ns, int((campaign.get("trace") or {}).get("until_unix_ns") or 0)) + out = [] + + def add(gid, quantity, rule, verdict, reason, **extra): + out.append(_gate(gid, quantity, rule, verdict, reason, ok, problems, **extra)) + + trace_src = {"transit": sha256_file(d / "trace" / "transit-load.json"), + "mark": sha256_file(d / "trace" / "mark-load.json")} + v, rows, why = score_transit_rows(transit, TRACE_FEEDS_G1) + if trace_gaps and v != "FAIL": + v, why = "NOT_MEASURED", why + trace_gaps + add("G1", "end-to-end to TS cache (venue event -> TS Redis APPLY reply)", "p95 <= 500 ms, p99 <= 1000 ms (n >= 100), " + "max < 2000 ms per feed/venue", v, "; ".join(why) or "every BOOK row inside", measured=rows, sources=trace_src) + v1, rows1, why1 = score_transit_rows(transit, TRACE_FEEDS_G2) + v2, rows2, why2 = score_mark(mark) + verdicts = {v1, v2} + v = "FAIL" if "FAIL" in verdicts else "NOT_MEASURED" if "NOT_MEASURED" in verdicts else "PASS" + if trace_gaps and v != "FAIL": + v = "NOT_MEASURED" + add("G2", "end-to-end to TS cache, new-event transit", "TRADE p95 <= 500 ms and max < 1500 ms; QUOTE and MARK " + "p95 <= 500 ms and max < 1000 ms", v, "; ".join(why1 + why2 + (trace_gaps if v == "NOT_MEASURED" else [])) + or "every row inside", measured=rows1 + rows2, sources=trace_src) + g3 = load_json(d / "g3" / "g3-summary.json") + add("G3", "end-to-end to TS cache, BAR from close (first visibility of the closed bar)", "p95 <= 5 s, max < 60 s", + (g3 or {}).get("verdict", "NOT_MEASURED"), "g3 probe" if g3 else "no G3 summary", + measured={k: (g3 or {}).get(k) for k in ("per_venue", "pooled", "unseen_total", "unresolved_slices", + "read_errors", "poll_ms")}, + sources={"g3": sha256_file(d / "g3" / "g3-summary.json")}) + budget = load_json(campaign.get("budget_path", "")) if campaign.get("budget_path") else None + table = ((budget or {}).get("g4_delivery_visibility") or {}).get("table") or {} + g4_windows = [] + for item in campaign.get("g4_windows", []): + summary = load_json(d / f"g4-{item['index']}-summary.json") + obs = d / f"g4-{item['index']}" / "observations.jsonl" + inverse = g4_inverse_rows(read_jsonl(obs)) if obs.exists() else {} + g4_windows.append({"summary": summary, "inverse_rows": inverse}) + v, rows, why = score_g4(g4_windows, table) if table else ("NOT_MEASURED", [], ["no frozen G4 baseline table"]) + add("G4", "delivery lag, venue -> canonical (read_committed visibility, upper bound)", "candidate p95 <= frozen " + "baseline p95 x 1.2 per row; missing coverage blocks the gate", v, "; ".join(why) or "every row inside", + measured=rows, inverse_rows=[w["inverse_rows"] for w in g4_windows]) + latency = [g for g in (gates or {}).get("gates", []) if str(g.get("gate", "")).startswith(("latency:", "cold"))] + failed_latency = [g["gate"] for g in latency if g.get("status") != "PASS"] + v = "FAIL" if failed_latency else "PASS" if latency else "NOT_MEASURED" + add("G5", "request latency", "certified stage-50 p95/p99 per class and venue; frozen cold-shape limits", v, + f"failed: {failed_latency}" if failed_latency else f"{len(latency)} latency/cold gates" if latency else + "no driver gates", measured=latency) + v, rows, why = score_g6((receipt or {}).get("g6_view_quality")) + add("G6", "durable-view event age and typed execution quality at the Query endpoint", "per-binding policy (TRADE " + "3000, QUOTE/MARK/BOOK_DELTA 2000, BOOK_SNAPSHOT 60000, BAR1m 180000 ms); zero unsafe eligibility", v, + "; ".join(why) or "every required binding measured, 0 unsafe", measured=rows) + v, rows, why = score_g7(transit) + if trace_gaps and v != "FAIL": + v, why = "NOT_MEASURED", why + trace_gaps + add("G7", "TS consumer path", "SDK -> writer p95 <= 100 ms; writer -> Redis reply p95 <= 50 ms per feed", v, + "; ".join(why) or "every feed inside", measured=rows) + samples8 = list(read_jsonl(d / "g8" / "g8-samples.jsonl")) if (d / "g8" / "g8-samples.jsonl").exists() else [] + s8 = g8.summarize(samples8, steady_from_ns=load_ns[0] if load_ns else None, + steady_to_ns=load_ns[1] if load_ns else None) if samples8 else None + v, why = g8.verdict(s8) if s8 else ("NOT_MEASURED", ["no G8 samples"]) + add("G8", f"TS execution Redis ({campaign.get('market_redis', {}).get('container', '?')}) command CPU, SLOWLOG, RTT", + "commandstats <= 60 % of one core per 10-min window; SLOWLOG (10 ms) 0 in steady state; PING p99 <= 5 ms", v, + "; ".join(why) or "inside", measured=s8) + path910 = d / "g9g10" / "g9-g10-samples.jsonl" + samples910 = list(read_jsonl(path910)) if path910.exists() else [] + s9 = g910.summarize_g9(samples910, from_ns=load_ns[0] if load_ns else None, + to_ns=load_ns[1] if load_ns else None) if samples910 else None + v, why = g910.verdict_g9(s9, expected_demanded=campaign.get("demanded_slices")) if s9 else ( + "NOT_MEASURED", ["no heartbeat samples"]) + add("G9", "readiness: demanded, session-ready, intent-eligible kept separate", "demanded == session-ready; 0 " + "data-path eligibility losses (legitimate quiet TRADE refusals counted, excluded); v1 fallback 0", v, + "; ".join(why) or "inside", measured=s9) + s10 = g910.summarize_g10(samples910, from_s=utc_stamp(load_s[0]) if load_s else None, + to_s=utc_stamp(load_s[1]) if load_s else None) if samples910 else {} + v, counters, why = score_g10(s10, (transit or {}).get("duplicate_applied_event_id") if transit else None, trace_gaps) + add("G10", "TRADE integrity", "0 gaps, 0 duplicates applied, 0 replay-expired, 0 overflow", v, + "; ".join(why) or "all zero", measured=counters, + basis="writer log typed signals (DL-signalled gaps, CURSOR_EXPIRED, SlowConsumerError) + trace duplicate " + "event_id; a silent offset gap the DL does not signal is not observable (see proposed counter)") + g11 = load_json(d / "g11" / "g11-summary.json") + v = (g11 or {}).get("verdict", "NOT_MEASURED") + out.append({"id": "G11", "quantity": "writer restart recovery", "rule": "fault cleared -> first valid apply per " + "slice <= 30 s, -> caught up <= 60 s; no rewind; cursor never ahead", "verdict": v, + "reason": "; ".join((g11 or {}).get("reasons", [])) or ("restart probe" if g11 else "G11 not run"), + "measured": {k: (g11 or {}).get(k) for k in ("restart", "outage_seconds", "recovery", "cursor_samples")} + if g11 else None, + "note": "a fault window after the load (a restart of a monitored container aborts the driver); its " + "verdict does not depend on the steady-state protocol"}) + out.append({"id": "G12", "quantity": "Kafka broker / projector / Stream restart recovery", "verdict": "NOT_MEASURED", + "reason": "not part of this campaign (isolated G12 tooling, scripts/p184_g12_isolated_fault.py)"}) + out.append({"id": "G13", "quantity": "DR01-DR08 acceptance", "verdict": "NOT_MEASURED", + "reason": "not part of this campaign (G13 evidence is separate)"}) + expect = (host or {}).get("monitored_containers") or campaign.get("monitored_containers") or [] + table14 = g14.table(host, expect=expect) if host else None + if table14 is None: + v, why = "NOT_MEASURED", ["no driver host.json"] + elif table14["mem_at_or_over_limit"]: + v, why = "FAIL", [f"memory at/over limit: {table14['mem_at_or_over_limit']}"] + elif table14["missing_roles"]: + v, why = "NOT_MEASURED", [f"roles without samples: {table14['missing_roles']}"] + else: + v, why = "PASS", [] + add("G14", "resources per role at stage 100", "CPU/RAM per role recorded with headroom; budget set by the owner", + v, "; ".join(why) or f"{len(table14['roles'])} roles recorded", measured=table14) + summary = {} + for gate in out: + summary.setdefault(gate["verdict"], []).append(gate["id"]) + return {"schema": "qdl.p184.gate-campaign-scorecard.v1", "campaign_dir": str(d), + "scoring_rule": "steady state >= 30 min at the stage-100 workload plus one burst window; a measurement " + "outside the protocol can FAIL a gate but cannot PASS it; NOT_MEASURED is never upgraded", + "protocol_met": ok, "protocol_problems": problems, "load_window_unix_s": load_s, + "runtime_identity": campaign.get("runtime_identity"), "summary": summary, "gates": out, + "sources": {"receipt": sha256_file(d / "stage100" / "receipt.json"), + "gates": sha256_file(d / "stage100" / "gates.json"), + "host": sha256_file(d / "stage100" / "host.json")}} + + +# ---------------------------------------------------------------- runtime (read-only inspection + children) + +def docker_inspect(name: str) -> dict | None: + result = subprocess.run(["docker", "inspect", name], capture_output=True, text=True, timeout=30) + return json.loads(result.stdout)[0] if result.returncode == 0 else None + + +def writer_market_redis(writer: str) -> str | None: + code = "from shared.config import settings; print(settings.TRADING_MARKET_REDIS_URL)" + result = subprocess.run(["docker", "exec", writer, "python", "-B", "-c", code], capture_output=True, text=True, + timeout=60) + return result.stdout.strip() or None if result.returncode == 0 else None + + +def preflight(args) -> dict: + plan = {"writer": args.writer, "problems": [], "warnings": []} + inspect = docker_inspect(args.writer) + if inspect is None or not inspect["State"]["Running"]: + plan["problems"].append("writer not running") + return plan + env = env_of(inspect) + plan["runtime_identity"] = {"writer_image": inspect["Image"], "writer_started": inspect["State"]["StartedAt"], + "writer_restarts": inspect["RestartCount"]} + until = int(env.get("TS_DELIVERY_TRACE_UNTIL_UNIX_NS") or 0) + needed = time.time_ns() + int((args.load_timeout + 60) * 1e9) + plan["trace"] = {"dir_in_container": env.get("TS_DELIVERY_TRACE_DIR"), + "dir_on_host": host_path(inspect, env.get("TS_DELIVERY_TRACE_DIR", "")) if env.get( + "TS_DELIVERY_TRACE_DIR") else None, + "until_unix_ns": until, "max_files": env.get("TS_DELIVERY_TRACE_MAX_FILES"), + "max_records": env.get("TS_DELIVERY_TRACE_MAX_RECORDS"), + "covers_load": until >= needed} + if not plan["trace"]["covers_load"]: + (plan["warnings"] if args.allow_no_trace else plan["problems"]).append( + "delivery trace does not cover the load window (recreate the writer with the long trace first); " + "G1/G2/G7/G10 would be NOT_MEASURED") + plan["binding"] = mount_source(inspect, "/run/secrets/qdl-v2-binding/trading-system.paper.stable.json") + plan["writer_log_dir"] = host_path(inspect, "/app/logs") + binding = load_json(plan["binding"]) if plan["binding"] else None + plan["demanded_slices"] = len((binding or {}).get("products", [])) or None + resolved = writer_market_redis(args.writer) + if resolved is None: + plan["warnings"].append("could not read TRADING_MARKET_REDIS_URL from the writer; using the TS default") + host, _, db = parse_redis_url(resolved or "redis://redis_marketdata:6379/1") + plan["market_redis"] = {"url_host": host, "db": db, "container": args.market_redis or host, + "resolved_from": "writer shared.config settings" if resolved else "TS default"} + if args.market_redis and args.market_redis != host: + plan["warnings"].append(f"--market-redis {args.market_redis} differs from the writer's {host}") + heartbeat_host, _, heartbeat_db = parse_redis_url(env.get("TRADING_REDIS_URL", "redis://redis_service:6379/0")) + plan["heartbeat_redis"] = {"container": heartbeat_host, "db": heartbeat_db} + free = shutil.disk_usage("/").free / 2**30 + plan["disk_free_gb"] = round(free, 1) + if free < MIN_DISK_GB + 3: + plan["problems"].append(f"disk {free:.1f} GB free < {MIN_DISK_GB + 3} GB") + for name in EXTRA_MONITORED: + state = docker_inspect(name) + if state is None or not state["State"]["Running"]: + plan["problems"].append(f"{name} not running (monitored for G14)") + for tool in ("guard_v2.sh", "mark_watch_v2.sh"): + if not (Path(args.state_tools) / tool).exists(): + plan["problems"].append(f"missing {tool} in {args.state_tools}") + return plan + + +class Children: + """Every child is tracked by its Popen; cleanup signals exact PIDs only.""" + + def __init__(self): + self.items: list[tuple[str, subprocess.Popen]] = [] + self.lock = threading.Lock() + + def start(self, name, command, *, log: Path, cwd=None): + handle = log.open("x") + process = subprocess.Popen(command, cwd=cwd, stdout=handle, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL) + with self.lock: + self.items.append((name, process)) + return process + + def stop(self, process, *, grace=90): + if process.poll() is None: + process.send_signal(signal.SIGTERM) + try: + process.wait(timeout=grace) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + + def stop_all(self): + with self.lock: + items = list(self.items) + for _, process in items: + self.stop(process, grace=30) + + +def log_event(d: Path, kind: str, **fields): + with (d / "timeline.jsonl").open("a") as handle: + handle.write(json.dumps({"kind": kind, "at_ns": time.time_ns(), **fields}, sort_keys=True) + "\n") + + +def run_trace_analysis(campaign: dict, d: Path, *, image: str) -> None: + """Trace files are root 0600 inside the writer's log mount: read them in a disposable, network-less container of + an image already on the host, with read-only mounts; output is JSON on stdout.""" + trace = campaign.get("trace") or {} + load_s = (load_json(d / "stage100" / "receipt.json") or {}).get("load_window_unix_s") + out = d / "trace" + out.mkdir(exist_ok=True) + if not trace.get("dir_on_host") or not load_s: + (out / "transit-load.json").write_text(json.dumps({"error": "no trace dir or load window"}) + "\n") + return + pattern = f"/trace/delivery-*-{trace['until_unix_ns']}*.json" + window = ["--from-ns", str(int(load_s[0] * 1e9)), "--to-ns", str(int(load_s[1] * 1e9))] + base = ["docker", "run", "--rm", "--network", "none", "--cpus", "1", "--memory", "2500m", "--user", "0:0", + "--read-only", "--security-opt", "no-new-privileges", "-e", "PYTHONDONTWRITEBYTECODE=1", + "-v", f"{ROOT}:/w:ro", "-v", f"{trace['dir_on_host']}:/trace:ro", "-w", "/w", "--entrypoint", "python", + image, "-B"] + for name, script in (("transit-load.json", "scripts/p184_trace_feed_transit.py"), + ("mark-load.json", "scripts/p184_g2_mark_trace_transit.py")): + result = subprocess.run([*base, script, pattern, *window], capture_output=True, text=True, timeout=1800) + try: + value = json.loads(result.stdout) + except ValueError: + value = {"error": f"analysis exit {result.returncode}", "stderr_tail": result.stderr[-400:]} + value.pop("per_file", None) + (out / name).write_text(json.dumps(value, indent=1, sort_keys=True) + "\n") + + +def command_plan(args, plan: dict, d: Path, launch_unix: float) -> dict: + tools = Path(args.state_tools) + py = [sys.executable, "-B"] + sampler_seconds = str(args.load_timeout + 60) + return { + "driver": ["timeout", str(args.load_timeout), *py, "scripts/phase3_consumer_load_acceptance.py", + "--profile", str(d / "profile.json"), "--budget", str(Path(args.budget).resolve()), + "--output", str(d / "stage100"), "--mode", "target", "--sessions", "100", + "--duration-seconds", str(LOAD_SECONDS)], + "guard": [str(tools / "guard_v2.sh"), str(args.load_timeout), ""], + "mark_watch": [str(tools / "mark_watch_v2.sh"), str(args.load_timeout), ""], + "g3": [*py, "scripts/p184_g3_bar_cache_probe.py", "--binding", str(plan.get("binding")), + "--container", plan["market_redis"]["container"], "--db", str(plan["market_redis"]["db"]), + "--seconds", sampler_seconds, "--out", str(d / "g3")], + "g8": [*py, "scripts/p184_g8_redis_sampler.py", "--container", plan["market_redis"]["container"], + "--seconds", sampler_seconds, "--out", str(d / "g8")], + "g9g10": [*py, "scripts/p184_g9_g10_writer_counters.py", "--container", plan["heartbeat_redis"]["container"], + "--db", str(plan["heartbeat_redis"]["db"]), "--log-dir", str(plan.get("writer_log_dir")), + "--seconds", sampler_seconds, "--out", str(d / "g9g10")], + "g4_windows": [{"index": i + 1, **w, + "command": [*py, "scripts/p184_g4_observer.py", "--launch", "--mode", "canonical", + "--seconds", str(int(w["seconds"])), "--tail-records", "0", "--max-records", + "10000", "--sample-ms", "1000", "--max-output-bytes", "16777216", + "--output", str(d / f"g4-{i + 1}")]} + for i, w in enumerate(g4_schedule(launch_unix))], + "g11": ([*py, "scripts/p184_g11_writer_restart_probe.py", "--binding", str(plan.get("binding")), + "--writer", args.writer, "--redis-container", plan["market_redis"]["container"], + "--db", str(plan["market_redis"]["db"]), "--wait-seconds", str(args.g11_wait), + "--observe-seconds", "180", "--out", str(d / "g11")] if args.g11 else None), + } + + +def run(args) -> int: + plan = preflight(args) + launch_unix = time.time() + 5 + stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + d = Path(args.campaign_dir or STATE / f"p184-final-campaign-{stamp}").resolve() + commands = command_plan(args, plan, d, launch_unix) + expected = (args.load_timeout + 60 + (args.g11_wait + 180 if args.g11 else 0) + 600) + if args.dry_run: + print(json.dumps({"campaign_dir": str(d), "preflight": plan, "commands": commands, + "expected_wall_seconds_max": expected, + "order": ["preflight (read-only)", "samplers G3/G8/G9-G10 start", "driver + guard + MARK " + "watch", "G4 windows on schedule", "driver exit -> samplers SIGTERM", + "G11 probe arms; COORDINATOR restarts the writer" if args.g11 else "G11 skipped", + "trace analysis (--rm, --network none)", "scorecard.json"]}, + indent=1, sort_keys=True, default=str)) + return 0 if not plan["problems"] else 2 + if plan["problems"]: + print(json.dumps({"refused": plan["problems"]}), file=sys.stderr) + return 2 + d.mkdir(parents=True, exist_ok=False) + base = load_json(Path(args.base_profile)) + profile = derive_profile(base) + (d / "profile.json").write_text(json.dumps(profile, indent=1, sort_keys=True) + "\n") + os.chmod(d / "profile.json", 0o600) + campaign = {"schema": "qdl.p184.gate-campaign.v1", "started_unix": time.time(), "preflight": plan, + "runtime_identity": plan.get("runtime_identity"), "trace": plan.get("trace"), + "market_redis": plan.get("market_redis"), "demanded_slices": plan.get("demanded_slices"), + "budget_path": str(Path(args.budget).resolve()), "budget_sha256": sha256_file(Path(args.budget)), + "monitored_containers": profile["monitored_containers"], + "g4_windows": [{k: v for k, v in w.items() if k != "command"} for w in commands["g4_windows"]], + "tool_sha256": sha256_file(Path(__file__)), "source_sha": subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip()} + (d / "campaign.json").write_text(json.dumps(campaign, indent=1, sort_keys=True, default=str) + "\n") + children = Children() + hard_deadline = time.monotonic() + expected + 600 + watchdog_fired = threading.Event() + + def watchdog(): + while not watchdog_fired.wait(15): + if time.monotonic() > hard_deadline: + log_event(d, "DEADLINE") + children.stop_all() + return + threading.Thread(target=watchdog, daemon=True).start() + try: + samplers = {name: children.start(name, commands[name], log=d / f"{name}.log", cwd=ROOT) + for name in ("g3", "g8", "g9g10")} + time.sleep(max(0.0, launch_unix - time.time())) + driver = children.start("driver", commands["driver"], log=d / "stage100.log", cwd=ROOT) + log_event(d, "driver_started", pid=driver.pid) + (d / "driver.pid").write_text(f"{driver.pid}\n") + for name in ("guard", "mark_watch"): + command = [str(driver.pid) if part == "" else part for part in commands[name]] + children.start(name, command, log=d / f"{name}.log") + + def g4_runner(): + for window in commands["g4_windows"]: + delay = launch_unix + window["offset"] - time.time() + if delay > 0 and driver.poll() is None: + time.sleep(delay) + if driver.poll() is not None: + log_event(d, "g4_skipped", index=window["index"]) + continue + log_event(d, "g4_start", index=window["index"]) + with (d / f"g4-{window['index']}.log").open("x") as handle: + subprocess.run(window["command"], cwd=ROOT, stdout=handle, stderr=subprocess.STDOUT, + timeout=window["seconds"] + 180) + obs = d / f"g4-{window['index']}" / "observations.jsonl" + if obs.exists(): + summary = subprocess.run([sys.executable, "-B", "scripts/p184_g4_observer.py", "--summarize", + str(obs)], cwd=ROOT, capture_output=True, text=True, timeout=300) + (d / f"g4-{window['index']}-summary.json").write_text(summary.stdout) + log_event(d, "g4_end", index=window["index"]) + g4_thread = threading.Thread(target=g4_runner, daemon=True) + g4_thread.start() + code = driver.wait() + log_event(d, "driver_exit", code=code) + g4_thread.join(timeout=420) + time.sleep(30) + for process in samplers.values(): + children.stop(process) + log_event(d, "samplers_stopped") + if commands["g11"]: + probe = children.start("g11", commands["g11"], log=d / "g11.log", cwd=ROOT) + time.sleep(15) + message = (f"G11_ARMED: restart the writer now (coordinator action, e.g. `docker restart {args.writer}`); " + f"the probe waits up to {args.g11_wait} s") + log_event(d, "g11_armed", pid=probe.pid) + print(message, flush=True) + (d / "G11_ARMED").write_text(message + "\n") + probe.wait() + log_event(d, "g11_exit", code=probe.returncode) + finally: + watchdog_fired.set() + children.stop_all() + run_trace_analysis(campaign, d, image=args.analysis_image or profile["image"]) + card = build_scorecard(d) + (d / "scorecard.json").write_text(json.dumps(card, indent=1, sort_keys=True, default=str) + "\n") + print(json.dumps({"campaign_dir": str(d), "summary": card["summary"], "protocol_met": card["protocol_met"]})) + return 0 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + r = sub.add_parser("run") + r.add_argument("--dry-run", action="store_true") + r.add_argument("--campaign-dir") + r.add_argument("--base-profile", default=str(TOOLS / "profile-stage100-rc4.json")) + r.add_argument("--budget", default=str(ROOT / "config/v2/p184-stage100-budget.json")) + r.add_argument("--state-tools", default=str(TOOLS)) + r.add_argument("--writer", default=WRITER) + r.add_argument("--market-redis", help="container of TRADING_MARKET_REDIS_URL (default: read from the writer)") + r.add_argument("--analysis-image", help="image for the trace analysis (default: the profile image)") + r.add_argument("--load-timeout", type=int, default=DRIVER_TIMEOUT) + r.add_argument("--g11", action="store_true", help="arm the G11 probe after the load (coordinator restarts)") + r.add_argument("--g11-wait", type=int, default=900) + r.add_argument("--allow-no-trace", action="store_true") + s = sub.add_parser("score") + s.add_argument("--campaign-dir", required=True) + s.add_argument("--analysis-image") + s.add_argument("--rerun-trace", action="store_true") + args = parser.parse_args(argv) + if args.command == "run": + return run(args) + d = Path(args.campaign_dir) + if args.rerun_trace or not (d / "trace" / "transit-load.json").exists(): + campaign = load_json(d / "campaign.json") or {} + image = args.analysis_image or (load_json(d / "profile.json") or {}).get("image") + run_trace_analysis(campaign, d, image=image) + card = build_scorecard(d) + path = d / f"scorecard-{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}.json" + path.write_text(json.dumps(card, indent=1, sort_keys=True, default=str) + "\n") + print(json.dumps({"scorecard": str(path), "summary": card["summary"]})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/p184_resp_readonly.py b/scripts/p184_resp_readonly.py new file mode 100644 index 00000000..1d9e5c66 --- /dev/null +++ b/scripts/p184_resp_readonly.py @@ -0,0 +1,221 @@ +"""Read-only Redis access and shared helpers for the P18.4 final gate-campaign probes (G3, G8, G9, G11). + +Purpose: one small stdlib RESP2 client so the campaign probes run on the host without a Redis library, without +`docker exec` per sample and without a new container. It speaks only an allowlist of read-only commands (PING, GET, +MGET, PTTL, SCAN, INFO, SLOWLOG GET/LEN, CLIENT LIST, CONFIG GET, SELECT of a DB index for the connection); MONITOR, +writes, FLUSH*, CLIENT KILL, SLOWLOG RESET and CONFIG SET are refused before anything is sent. + +Also: a stop flag set by SIGTERM/SIGINT (probes finish their summary instead of dying), container IP resolution (read-only `docker inspect`), nearest-rank quantiles (the same rule as +scripts/p184_g2_mark_trace_transit.py: sorted[min(n-1, int(q*n))], p99 only when n >= 100), and a bounded JSONL writer +that stops writing (and says so) at its byte cap instead of growing without limit. + +Boundary: no Redis mutation, no Docker mutation, no secrets read. Values read from Redis are parsed by the callers and +never written whole to evidence. +""" +from __future__ import annotations + +import json +import signal +import socket +import subprocess +import threading +import time +from pathlib import Path + +READ_ONLY = { + ("PING",), ("GET",), ("MGET",), ("PTTL",), ("SCAN",), ("INFO",), ("SLOWLOG", "GET"), ("SLOWLOG", "LEN"), + ("CLIENT", "LIST"), ("CONFIG", "GET"), ("SELECT",), +} + + +class RespError(RuntimeError): + pass + + +def allowed(args) -> bool: + if not args: + return False + head = str(args[0]).upper() + if (head,) in READ_ONLY: + return True + return len(args) > 1 and (head, str(args[1]).upper()) in READ_ONLY + + +def encode(args) -> bytes: + parts = [b"*%d\r\n" % len(args)] + for arg in args: + data = arg if isinstance(arg, bytes) else str(arg).encode() + parts.append(b"$%d\r\n%s\r\n" % (len(data), data)) + return b"".join(parts) + + +def parse(reader): + """Parse one RESP2 reply from a binary file-like object (readline/read).""" + line = reader.readline() + if not line.endswith(b"\r\n"): + raise RespError("connection closed") + kind, rest = line[:1], line[1:-2] + if kind == b"+": + return rest.decode() + if kind == b"-": + raise RespError(rest.decode(errors="replace")[:200]) + if kind == b":": + return int(rest) + if kind == b"$": + size = int(rest) + if size < 0: + return None + data = reader.read(size + 2) + return data[:-2] + if kind == b"*": + size = int(rest) + return None if size < 0 else [parse(reader) for _ in range(size)] + raise RespError("unknown RESP type") + + +class ReadOnlyRedis: + def __init__(self, host: str, port: int = 6379, db: int = 0, timeout: float = 2.0): + self.host, self.port, self.db, self.timeout = host, port, db, timeout + self._sock = None + self._reader = None + + def _connect(self): + self._sock = socket.create_connection((self.host, self.port), timeout=self.timeout) + self._reader = self._sock.makefile("rb") + if self.db: + self._call(("SELECT", self.db)) + + def close(self): + if self._sock is not None: + try: + self._sock.close() + finally: + self._sock = self._reader = None + + def _call(self, args): + self._sock.sendall(encode(args)) + return parse(self._reader) + + def call(self, *args): + if not allowed(args): + raise PermissionError(f"command not in the read-only allowlist: {args[:2]}") + if self._sock is None: + self._connect() + try: + return self._call(args) + except (OSError, RespError): + self.close() + raise + + def pipeline(self, commands): + """Send several allowlisted commands in one round trip; returns replies in order.""" + for args in commands: + if not allowed(args): + raise PermissionError(f"command not in the read-only allowlist: {args[:2]}") + if self._sock is None: + self._connect() + try: + self._sock.sendall(b"".join(encode(args) for args in commands)) + return [parse(self._reader) for _ in commands] + except (OSError, RespError): + self.close() + raise + + def scan(self, pattern: str, count: int = 1000, limit: int = 10000): + keys, cursor = [], b"0" + while True: + cursor, batch = self.call("SCAN", cursor, "MATCH", pattern, "COUNT", count) + keys.extend(item.decode() for item in batch) + if cursor in (b"0", "0") or len(keys) >= limit: + return sorted(set(keys))[:limit] + + +def container_ip(name: str, network: str | None = None) -> str: + """Read-only `docker inspect` of a container's IP (the named network, else the first one).""" + out = subprocess.run(["docker", "inspect", name, "--format", "{{json .NetworkSettings.Networks}}"], + capture_output=True, text=True, timeout=20, check=True).stdout + networks = json.loads(out) + if network is not None: + return networks[network]["IPAddress"] + for value in networks.values(): + if value.get("IPAddress"): + return value["IPAddress"] + raise RuntimeError(f"no IP for {name}") + + +def parse_redis_url(url: str) -> tuple[str, int, int]: + """redis://host[:port][/db] -> (host, port, db). Credentials are refused (none are configured on this host).""" + if not url.startswith("redis://"): + raise ValueError("only redis:// URLs") + rest = url[len("redis://"):] + if "@" in rest: + raise ValueError("credentials in a Redis URL are not handled by the read-only probes") + hostport, _, db = rest.partition("/") + host, _, port = hostport.partition(":") + return host, int(port or 6379), int(db or 0) + + +def quantiles(values): + if not values: + return {"n": 0} + ordered = sorted(values) + n = len(ordered) + pick = lambda q: round(ordered[min(n - 1, int(q * n))], 3) # noqa: E731 + return {"n": n, "p50": pick(.5), "p95": pick(.95), "p99": pick(.99) if n >= 100 else None, + "max": round(ordered[-1], 3), "min": round(ordered[0], 3)} + + +class BoundedJsonl: + """Append-only JSONL evidence created exclusively; stops at max_bytes and records that it stopped.""" + + def __init__(self, path: Path, max_bytes: int): + self.path, self.max_bytes, self.written, self.capped = Path(path), max_bytes, 0, False + self._handle = self.path.open("x", encoding="utf-8") + + def write(self, value: dict) -> bool: + if self.capped: + return False + line = json.dumps(value, separators=(",", ":"), sort_keys=True) + "\n" + if self.written + len(line) > self.max_bytes: + self.capped = True + self._handle.write(json.dumps({"kind": "BYTE_CAP_REACHED", "at_ns": time.time_ns()}) + "\n") + self._handle.flush() + return False + self._handle.write(line) + self._handle.flush() + self.written += len(line) + return True + + def close(self): + self._handle.close() + + +def read_jsonl(path: Path): + with Path(path).open(encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if line: + yield json.loads(line) + + +def write_json_exclusive(path: Path, value) -> None: + with Path(path).open("x", encoding="utf-8") as handle: + handle.write(json.dumps(value, indent=1, sort_keys=True) + "\n") + + +_STOP = threading.Event() + + +def install_stop_signals() -> None: + """SIGTERM/SIGINT end a probe's loop early; the probe still writes its summary (clean exit, no lost evidence).""" + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: _STOP.set()) + + +def stopped() -> bool: + return _STOP.is_set() + + +def pause(seconds: float) -> None: + if seconds > 0: + _STOP.wait(seconds) diff --git a/tests/test_p184_gate_campaign.py b/tests/test_p184_gate_campaign.py new file mode 100644 index 00000000..710a1eb6 --- /dev/null +++ b/tests/test_p184_gate_campaign.py @@ -0,0 +1,413 @@ +"""Offline tests for the P18.4 final gate-campaign probes and scorer (synthetic inputs, test-only provenance). + +No Redis, Docker or network: the RESP client is exercised on bytes, the probes' analysis parts on synthetic samples. +""" +from __future__ import annotations + +import io +import json +import tempfile +import unittest +from pathlib import Path + +from scripts import p184_g3_bar_cache_probe as g3 +from scripts import p184_g8_redis_sampler as g8 +from scripts import p184_g9_g10_writer_counters as g910 +from scripts import p184_g11_writer_restart_probe as g11 +from scripts import p184_g14_resource_table as g14 +from scripts import p184_gate_campaign as campaign +from scripts import p184_resp_readonly as resp + +S = 1_000_000_000 + + +class RespTests(unittest.TestCase): + def test_only_read_commands_are_allowed(self): + for args in (("PING",), ("GET", "k"), ("MGET", "a", "b"), ("INFO", "commandstats"), ("SLOWLOG", "GET", 128), + ("CONFIG", "GET", "x"), ("CLIENT", "LIST"), ("scan", "0")): + self.assertTrue(resp.allowed(args), args) + for args in (("MONITOR",), ("SET", "k", "v"), ("FLUSHALL",), ("SLOWLOG", "RESET"), ("CONFIG", "SET", "a", "b"), + ("CLIENT", "KILL", "x"), ("EVAL", "x", "0"), ()): + self.assertFalse(resp.allowed(args), args) + + def test_refused_command_never_touches_the_socket(self): + client = resp.ReadOnlyRedis("203.0.113.1") # TEST-NET, never connected + with self.assertRaises(PermissionError): + client.call("MONITOR") + with self.assertRaises(PermissionError): + client.pipeline([("PING",), ("FLUSHDB",)]) + self.assertIsNone(client._sock) + + def test_encode_and_parse_roundtrip(self): + self.assertEqual(resp.encode(("GET", "k")), b"*2\r\n$3\r\nGET\r\n$1\r\nk\r\n") + reader = io.BytesIO(b"*3\r\n$3\r\nabc\r\n$-1\r\n:42\r\n+OK\r\n-ERR bad\r\n") + self.assertEqual(resp.parse(reader), [b"abc", None, 42]) + self.assertEqual(resp.parse(reader), "OK") + with self.assertRaises(resp.RespError): + resp.parse(reader) + + def test_redis_url_and_quantiles(self): + self.assertEqual(resp.parse_redis_url("redis://redis_marketdata:6379/1"), ("redis_marketdata", 6379, 1)) + with self.assertRaises(ValueError): + resp.parse_redis_url("redis://user:pw@host:6379/0") + q = resp.quantiles(list(range(1, 101))) + self.assertEqual((q["p50"], q["p95"], q["p99"], q["max"]), (51, 96, 100, 100)) + self.assertIsNone(resp.quantiles([1, 2])["p99"]) + + def test_bounded_jsonl_stops_at_cap(self): + with tempfile.TemporaryDirectory() as tmp: + out = resp.BoundedJsonl(Path(tmp) / "e.jsonl", 60) + self.assertTrue(out.write({"a": 1})) + self.assertFalse(out.write({"b": "x" * 100})) + out.close() + rows = list(resp.read_jsonl(Path(tmp) / "e.jsonl")) + self.assertEqual(rows[-1]["kind"], "BYTE_CAP_REACHED") + + +def bar(open_ms, closed=True): + return json.dumps({"timestamp_ms": open_ms, "is_closed": closed}).encode() + + +class G3Tests(unittest.TestCase): + def test_first_closed_bar_after_start_is_measured_from_close(self): + name = "OKX|BTC-USDT-SWAP|1m" + tracker = g3.BarTracker([name], {name: 60_000}) + self.assertIsNone(tracker.observe(name, bar(0), 59_000)) # present at start: not measured + self.assertIsNone(tracker.observe(name, bar(60_000, closed=False), 100_000)) + record = tracker.observe(name, bar(60_000), 121_500) + self.assertEqual(record["transit_ms"], 1_500) + self.assertIsNone(tracker.observe(name, bar(60_000), 122_000)) # revision of the same bar: first only + summary = tracker.finish(start_ms=59_000, end_ms=200_000) + self.assertEqual(summary["per_slice"][name]["n"], 1) + self.assertEqual(summary["unseen_total"], 0) + self.assertEqual(g3.verdict(summary, unresolved=0), "PASS") + + def test_unseen_expected_close_fails_with_its_lower_bound(self): + name = "BINANCE|BTCUSDT|1m" + tracker = g3.BarTracker([name], {name: 60_000}) + tracker.observe(name, bar(0), 59_000) + summary = tracker.finish(start_ms=59_000, end_ms=200_000) # closes at 120 s expected, never seen + self.assertEqual(summary["unseen_total"], 1) + self.assertEqual(summary["unseen"][0]["lower_bound_ms"], 80_000) + self.assertEqual(g3.verdict(summary, unresolved=0), "FAIL") + + def test_slow_p95_fails_and_unresolved_slice_blocks_pass(self): + name = "OKX|ETH-USDT-SWAP|1m" + tracker = g3.BarTracker([name], {name: 60_000}) + tracker.observe(name, bar(0), 59_000) + tracker.observe(name, bar(60_000), 127_000) + summary = tracker.finish(start_ms=59_000, end_ms=200_000) + self.assertEqual(g3.verdict(summary, unresolved=0), "FAIL") + tracker = g3.BarTracker([name], {name: 60_000}) + tracker.observe(name, bar(0), 59_000) + tracker.observe(name, bar(60_000), 121_000) + self.assertEqual(g3.verdict(tracker.finish(start_ms=59_000, end_ms=200_000), unresolved=1), "NOT_MEASURED") + + def test_slices_and_keys_come_from_the_binding_and_cache(self): + binding = {"products": [{"feed": "BAR", "interval": "1m", "venue": "OKX", "native_symbol": "BTC-USDT-SWAP"}, + {"feed": "TRADE", "interval": None, "venue": "OKX", "native_symbol": "BTC-USDT-SWAP"}]} + slices = g3.bar_slices(binding) + keys = g3.resolve_keys(slices, ["cache:market:v2:last_bar:OKX:SWAP:BTC-USDT-SWAP:1m", + "cache:market:v2:last_bar:OKX:SWAP:ETH-USDT-SWAP:1m"]) + self.assertEqual(keys, {"OKX|BTC-USDT-SWAP|1m": "cache:market:v2:last_bar:OKX:SWAP:BTC-USDT-SWAP:1m"}) + + +def stats(at_s, evalsha_usec, slow=()): + return {"kind": "stats", "at_ns": at_s * S, "commandstats": {"evalsha": {"calls": 1, "usec": evalsha_usec}}, + "slowlog": list(slow), "slowlog_threshold_us": 10000, "monitor_clients": 0} + + +class G8Tests(unittest.TestCase): + def test_parsers_keep_no_arguments(self): + info = "# Commandstats\r\ncmdstat_evalsha:calls=10,usec=2500,usec_per_call=250.00\r\ncmdstat_ping:calls=1,usec=1\r\n" + self.assertEqual(g8.parse_commandstats(info)["evalsha"], {"calls": 10, "usec": 2500}) + rows = g8.slowlog_rows([[7, 1000, 12000, [b"EVALSHA", b"secret-arg"], b"1.2.3.4:5", b""]]) + self.assertEqual(rows, [{"id": 7, "start_s": 1000, "duration_us": 12000, "command": "EVALSHA"}]) + self.assertEqual(g8.monitor_clients("id=1 flags=N\nid=2 flags=O\n"), 1) + + def test_window_core_fraction_and_steady_slowlog(self): + samples = [stats(t, int(t * 0.3e6)) for t in range(0, 1201, 10)] # 0.3 core + samples += [{"kind": "ping", "at_ns": t * S, "rtt_ms": 0.5} for t in range(0, 1200)] + summary = g8.summarize(samples, steady_from_ns=0, steady_to_ns=1200 * S) + self.assertEqual([w["core_fraction"] for w in summary["windows"]], [0.3, 0.3]) + self.assertEqual(g8.verdict(summary)[0], "PASS") + hot = [stats(t, int(t * 0.7e6)) for t in range(0, 601, 10)] + self.assertEqual(g8.verdict(g8.summarize(hot + samples[121:], steady_from_ns=0, steady_to_ns=600 * S))[0], + "FAIL") + + def test_new_slowlog_entries_in_steady_state_fail_history_does_not(self): + old = {"id": 1, "start_s": 1, "duration_us": 20000, "command": "EVAL"} + new = {"id": 2, "start_s": 300, "duration_us": 15000, "command": "EVALSHA"} + samples = [stats(0, 0, [old])] + [stats(t, t * 1000, [old, new] if t >= 300 else [old]) + for t in range(10, 611, 10)] + samples += [{"kind": "ping", "at_ns": t * S, "rtt_ms": 0.5} for t in range(0, 600)] + summary = g8.summarize(samples, steady_from_ns=0, steady_to_ns=610 * S) + self.assertEqual((summary["slowlog_new_total"], summary["slowlog_new_steady"]), (1, 1)) + self.assertEqual(g8.verdict(summary)[0], "FAIL") + + def test_counter_reset_invalidates_the_window_and_short_run_is_not_measured(self): + samples = [stats(0, 10**9), stats(10, 5)] + summary = g8.summarize(samples, steady_from_ns=0, steady_to_ns=10 * S) + self.assertFalse(summary["windows"][0]["valid"]) + self.assertEqual(g8.verdict(summary)[0], "NOT_MEASURED") + + +def beat(at_s, *, demanded=64, session=64, not_ready=(), purpose=True, fallback=0, last_seen=None): + details = {"demanded_v2_slices": demanded, "session_ready_v2_slices": session, "execution_ready_v2_slices": 50, + "v1_fallback_count": fallback, "reported_unhealthy_slices": []} + if purpose: + details.update(purpose_readiness_schema="ts.market-data.purpose-readiness.v1", not_ready=list(not_ready)) + sample = g910.heartbeat_sample({"status": "READY", "details": details, + "last_seen_unix": last_seen if last_seen is not None else at_s}, at_s) + return {"kind": "heartbeat", "at_ns": at_s * S, "sample": sample} + + +def row(feed, reason, symbol="BTCUSDT"): + return {"provider": "BINANCE", "symbol": symbol, "feed": feed, "interval": None, "reason_code": reason} + + +class G9Tests(unittest.TestCase): + def test_new_not_ready_rows_are_losses_quiet_trade_is_excluded(self): + samples = [beat(0), beat(10, not_ready=[row("TRADE", "TRADE_EVENT_STALE")]), + beat(20, not_ready=[row("TRADE", "TRADE_EVENT_STALE"), row("QUOTE", "QUOTE_STALE")]), + beat(30, not_ready=[row("QUOTE", "QUOTE_STALE")])] + summary = g910.summarize_g9(samples) + self.assertEqual(summary["eligibility_losses_data_path"], 1) + self.assertEqual(summary["eligibility_loss_causes"], {"QUOTE:QUOTE_STALE": 1}) + self.assertEqual(summary["legit_quiet_refusals"], {"TRADE:TRADE_EVENT_STALE": 1}) + self.assertEqual(g910.verdict_g9(summary, expected_demanded=64)[0], "FAIL") + + def test_clean_run_passes_and_old_writer_is_not_measured(self): + clean = [beat(t, not_ready=[row("TRADE", "TRADE_EVENT_STALE")]) for t in range(0, 100, 10)] + self.assertEqual(g910.verdict_g9(g910.summarize_g9(clean), expected_demanded=64), ("PASS", [])) + old = [beat(t, purpose=False) for t in range(0, 100, 10)] + state, reasons = g910.verdict_g9(g910.summarize_g9(old), expected_demanded=64) + self.assertEqual(state, "NOT_MEASURED") + self.assertIn("purpose readiness", reasons[0]) + + def test_unknown_and_stale_samples_never_count_as_ready(self): + samples = [beat(0), {"kind": "heartbeat", "at_ns": 10 * S, "sample": {"readable": False}}, + beat(100, last_seen=0)] + summary = g910.summarize_g9(samples) + self.assertEqual((summary["readable_samples"], summary["unknown_samples"]), (1, 2)) + self.assertEqual(g910.verdict_g9(summary, expected_demanded=64)[0], "NOT_MEASURED") + + def test_session_mismatch_and_fallback_fail(self): + self.assertEqual(g910.verdict_g9(g910.summarize_g9([beat(0, session=63)]), expected_demanded=64)[0], "FAIL") + self.assertEqual(g910.verdict_g9(g910.summarize_g9([beat(0, fallback=1)]), expected_demanded=64)[0], "FAIL") + + +LINES = [ + "2026-10-10 15:00:01,000 - market_data_worker-0 - services.market_data.data_layer_bridge - WARNING - Data Layer V2 " + "slice disconnected venue=OKX symbol=BTC-USDT-SWAP feed=TRADE interval=None reason=SlowConsumerError " + "code=RATE_LIMITED retryable=True", + "2026-10-10 15:00:02,000 - market_data_worker-0 - x - WARNING - Data Layer V2 slice disconnected venue=OKX " + "symbol=BTC-USDT-SWAP feed=TRADE interval=None reason=CursorExpiredError code=CURSOR_EXPIRED", + "2026-10-10 15:00:03,000 - market_data_worker-1 - x - INFO - data layer slice control code=SNAPSHOT_REPLACED " + "venue=BINANCE symbol=BTCUSDT feed=TRADE generation=2", + "2026-10-10 15:00:04,000 - market_data_worker-1 - x - INFO - data layer slice control code=RECONNECTED " + "venue=BINANCE symbol=BTCUSDT feed=TRADE generation=3", + "2026-10-10 15:00:05,000 - market_data_worker-1 - x - WARNING - Data Layer V2 slice disconnected venue=OKX " + "symbol=BTC-USDT-SWAP feed=QUOTE interval=None reason=DataLayerError code=DATA_STALE", + "2026-10-10 15:00:06,000 - market_data_worker-1 - qdl_sdk.cursor - INFO - qdl_cursor_persistence {}", +] + + +class G10Tests(unittest.TestCase): + def test_trade_lines_are_classified_by_typed_signal(self): + got = [g910.classify_line(line) for line in LINES] + self.assertEqual([item[1] if item else None for item in got], + ["overflow", "replay_expired", "gap", "resumed", "other_feed_disconnect", None]) + + def test_log_tail_reads_only_new_complete_lines_and_survives_truncation(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "market_data_worker-0.log" + path.write_text("old line\n") + tail = g910.LogTail([path]) + with path.open("a") as handle: + handle.write(LINES[0] + "\npartial") + self.assertEqual(tail.read_new(), [LINES[0]]) + with path.open("a") as handle: + handle.write(" end\n") + self.assertEqual(tail.read_new(), ["partial end"]) + path.write_text("x\n") # truncated/rotated + self.assertEqual(tail.read_new(), ["x"]) + + def test_summary_windows_by_log_stamp(self): + samples = [{"kind": "log_poll", "unreadable": 0}] + for line in LINES: + item = g910.classify_line(line) + if item: + samples.append({"kind": "log_event", "stamp": item[0], "category": item[1], "detail": item[2]}) + summary = g910.summarize_g10(samples, from_s="2026-10-10 15:00:02", to_s="2026-10-10 15:00:09") + self.assertEqual((summary["trade_overflow"], summary["trade_replay_expired"], summary["trade_gap"], + summary["trade_resumed"]), (0, 1, 1, 1)) + + +def obs(event_ms, digest, pttl=100_000, offset=None, generation="g1", closed=None): + return {"digest": digest, "pttl": pttl, "event_ms": event_ms, "offset": offset, "generation": generation, + "closed": closed} + + +class G11Tests(unittest.TestCase): + def test_first_apply_and_caught_up_after_fault_cleared(self): + s = g11.SliceRecovery("OKX|BTC-USDT-SWAP|TRADE|None", "TRADE", None, 500) + s.update(1_000, obs(900, "a", offset=10), phase="PRE") + s.update(5_000, obs(900, "a", pttl=96_000, offset=10), phase="DOWN") + s.update(20_000, obs(900, "a", pttl=81_000, offset=10), phase="POST") + self.assertIsNone(s.first_apply_ms) + s.update(21_000, obs(15_000, "b", offset=11), phase="POST") # rewritten, older than live bound + self.assertEqual((s.first_apply_ms, s.caught_up_ms), (21_000, None)) + s.update(22_000, obs(21_500, "c", offset=12), phase="POST") + self.assertEqual(s.caught_up_ms, 22_000) + summary = g11.recovery_summary({s.name: s}, cleared_ms=19_000) + self.assertEqual(summary["first_valid_apply_s"]["max"], 2.0) + self.assertEqual(summary["caught_up_s"]["max"], 3.0) + self.assertEqual(summary["rewinds_total"], 0) + + def test_ttl_jump_counts_as_a_rewrite_and_rewind_is_detected(self): + s = g11.SliceRecovery("B", "BOOK_SNAPSHOT", None, 500) + s.update(0, obs(1_000, "a", pttl=60_000), phase="PRE") + s.update(10_000, obs(1_000, "a", pttl=50_000), phase="POST") + s.update(10_500, obs(10_400, "a", pttl=119_900), phase="POST") + self.assertEqual(s.first_apply_ms, 10_500) + t = g11.SliceRecovery("T", "TRADE", None, 500) + t.update(0, obs(5_000, "a", offset=100), phase="PRE") + t.update(1_000, obs(4_000, "b", offset=90), phase="POST") + self.assertEqual({r["kind"] for r in t.rewinds}, {"event_time", "applied_offset"}) + self.assertIsNone(t.first_apply_ms) # an older event is never a valid first apply + + def test_bar_caught_up_by_close_age(self): + s = g11.SliceRecovery("BAR", "BAR", "1m", 500) + s.update(0, obs(60_000, "a", closed=True), phase="PRE") + s.update(130_000, obs(120_000, "b", closed=True), phase="POST") + self.assertEqual((s.first_apply_ms, s.caught_up_ms), (130_000, 130_000)) + + def test_cursor_check_and_helpers(self): + result = g11.cursor_check({"c|uid1|TRADE||p": 10, "c|uid1|QUOTE||p": 30, "c|uid1|BAR|1m|p": 99, + "c|uid9|TRADE||p": 1}, {"T": 12, "Q": 20}, {"uid1|TRADE": "T", "uid1|QUOTE": "Q"}) + self.assertEqual((result["compared"], result["unmatched"]), (2, 1)) + self.assertEqual(result["ahead"], [{"slice": "Q", "cursor": 30, "applied": 20}]) + self.assertEqual(g11.parse_docker_time("2026-10-10T13:30:24.199934182Z") % 1000, 199) + self.assertIsNone(g11.parse_docker_time("0001-01-01T00:00:00Z")) + ids = g11.identities_from_keys(["cache:market:v2:execution_trade:OKX:SWAP:BTC-USDT-SWAP", + "cache:market:v2:applied:trade:OKX:SWAP:BTC-USDT-SWAP"]) + self.assertEqual(ids, {("OKX", "BTC-USDT-SWAP"): "OKX:SWAP:BTC-USDT-SWAP"}) + keys = g11.slice_keys({"venue": "OKX", "native": "X", "feed": "BOOK_DELTA", "interval": None}, "OKX:SWAP:X") + self.assertEqual(keys["value"], "cache:market:v2:execution_book_delta:OKX:SWAP:X:ordering-v1") + + def test_verdict(self): + s = g11.SliceRecovery("T", "TRADE", None, 500) + s.update(0, obs(0, "a"), phase="PRE") + s.update(5_000, obs(4_500, "b"), phase="POST") + report = {"restart": {"fault_cleared_ms": 1_000}, "observe_seconds": 180, + "recovery": g11.recovery_summary({"T": s}, 1_000), + "cursor_samples": [{"compared": 2, "ahead": []}]} + self.assertEqual(g11.verdict(report), ("PASS", [])) + report["cursor_samples"] = [{"compared": 2, "ahead": [{"slice": "T"}]}] + self.assertEqual(g11.verdict(report)[0], "FAIL") + self.assertEqual(g11.verdict({"restart": None})[0], "NOT_MEASURED") + + +def host_json(): + def sample(at_s, usage_s, cpu, mem): + return {"at_ns": at_s * S, "stats": [ + {"Name": "qdl_v2_stable_candidate-stable_redis-1", "CPUPerc": f"{cpu}%", "MemUsage": f"{mem}MiB / 512MiB"}], + "cpu_throttle": {"qdl_v2_stable_candidate-stable_redis-1": { + "usage_usec": usage_s * 10**6, "throttled_usec": 0, "nr_periods": 0, "nr_throttled": 0}}} + return {"monitored_containers": ["qdl_v2_stable_candidate-stable_redis-1", + "qdl_v2_stable_candidate-binance_bar_edge-1"], + "runtime_observation": [sample(0, 0, 10, 100), sample(100, 25, 30, 120)]} + + +class G14Tests(unittest.TestCase): + def test_roles_mean_from_cgroup_and_missing_roles_listed(self): + result = g14.table(host_json()) + row = result["roles"]["stable_redis"] + self.assertEqual((row["cpu_mean_pct"], row["cpu_max_pct"], row["mem_max_mib"], row["mem_limit_mib"]), + (25.0, 30.0, 120.0, 512.0)) + self.assertEqual(result["missing_roles"], ["binance_bar_edge"]) + self.assertIn("stable_redis", g14.render(result)) + self.assertAlmostEqual(g14.mib("1.5GiB"), 1536.0) + + +class CampaignTests(unittest.TestCase): + def test_g4_schedule_adds_a_boundary_window_without_overlap(self): + launch = 1_791_649_000.0 # boundary at 1_791_649_800 (UTC :30) + windows = campaign.g4_schedule(launch) + boundary = [w for w in windows if w["boundary"]] + self.assertEqual(len(boundary), 1) + self.assertEqual(boundary[0]["offset"], 1_791_649_800 - 30 - launch) + ends = [(w["offset"], w["offset"] + w["seconds"]) for w in windows] + self.assertTrue(all(a[1] <= b[0] for a, b in zip(ends, ends[1:]))) + late = campaign.g4_schedule(1_791_649_799.0 - 1700) # next boundary falls after the load + self.assertTrue(all(w["boundary"] is None or w["offset"] + w["seconds"] <= 1880 for w in late)) + + def test_profile_paths_and_env(self): + profile = campaign.derive_profile({"monitored_containers": ["a"], "image": "i"}) + self.assertEqual(profile["monitored_containers"], ["a", *campaign.EXTRA_MONITORED]) + inspect = {"Mounts": [{"Source": "/h/logs", "Destination": "/app/logs"}, + {"Source": "/h/b.json", "Destination": "/run/b.json"}], + "Config": {"Env": ["A=1", "B=x=y"]}} + self.assertEqual(campaign.host_path(inspect, "/app/logs/dr07"), "/h/logs/dr07") + self.assertEqual(campaign.mount_source(inspect, "/run/b.json"), "/h/b.json") + self.assertEqual(campaign.env_of(inspect), {"A": "1", "B": "x=y"}) + + def test_g4_rows_need_baseline_coverage(self): + table = {"trade|OKX|": {"candidate_p95_limit_ms": 100.0}} + inside = [{"summary": {"trade|OKX|": {"visibility_ms": {"n": 5, "p95": 90.0}}}, "inverse_rows": {}}] + self.assertEqual(campaign.score_g4(inside, table)[0], "PASS") + over = [{"summary": {"trade|OKX|": {"visibility_ms": {"n": 5, "p95": 120.0}}}, "inverse_rows": {}}] + self.assertEqual(campaign.score_g4(over, table)[0], "FAIL") + inverse = [{**inside[0], "inverse_rows": {"trade|OKX_INVERSE|": {"n": 3}}}] + state, _, why = campaign.score_g4(inverse, table) + self.assertEqual(state, "NOT_MEASURED") + self.assertIn("OKX_INVERSE", why[0]) + rows = campaign.g4_inverse_rows([{"kind": "canonical_observation", "feed": "trade", "interval": None, + "identity": {"native_symbol": "BTC-USD-SWAP"}, "source_event_ns": 1 * S, + "observer_receipt_wall_ns": 1 * S + 50_000_000}]) + self.assertEqual(rows, {"trade|OKX_INVERSE|": {"n": 1, "p95": 50.0, "max": 50.0}}) + + def test_g6_unsafe_fails_missing_binding_not_measured(self): + full = {key: {"n": 5, "unsafe": 0} for key in campaign.G6_REQUIRED} + self.assertEqual(campaign.score_g6(full)[0], "PASS") + self.assertEqual(campaign.score_g6({**full, "TRADE|OKX|": {"n": 5, "unsafe": 1}})[0], "FAIL") + partial = {k: v for k, v in full.items() if not k.startswith("BOOK_DELTA")} + self.assertEqual(campaign.score_g6(partial)[0], "NOT_MEASURED") + + def test_g10_needs_trace_for_duplicates(self): + g10 = {"trade_gap": 0, "trade_replay_expired": 0, "trade_overflow": 0, "log_polls": 10, "log_unreadable": 0} + self.assertEqual(campaign.score_g10(g10, {}, [])[0], "PASS") + self.assertEqual(campaign.score_g10(g10, {"TRADE|OKX": 1}, [])[0], "FAIL") + self.assertEqual(campaign.score_g10(g10, None, ["no trace"])[0], "NOT_MEASURED") + self.assertEqual(campaign.score_g10({**g10, "trade_overflow": 2}, None, ["no trace"])[0], "FAIL") + + def test_trace_coverage_flags_cap_and_early_expiry(self): + ok = {"records": 10, "dropped_at_export_total": 0, "stop_reasons": {"ROTATED": 3, "EXPIRY": 4}} + self.assertEqual(campaign._trace_coverage(ok, [0, 10 * S], 11 * S), []) + self.assertTrue(campaign._trace_coverage({**ok, "stop_reasons": {"CAP": 1}}, [0, 10 * S], 11 * S)) + self.assertTrue(campaign._trace_coverage(ok, [0, 10 * S], 9 * S)) + + def test_scorecard_never_passes_outside_the_protocol(self): + with tempfile.TemporaryDirectory() as tmp: + d = Path(tmp) + (d / "stage100").mkdir() + (d / "campaign.json").write_text(json.dumps({"trace": {"until_unix_ns": 10**19}, "g4_windows": []})) + (d / "stage100" / "receipt.json").write_text(json.dumps({ + "stage": 100, "final": True, "load_window_unix_s": [1000.0, 2000.0], # 1000 s < 1800 s + "fault_windows": {"BURST": [720, 730]}, "g6_view_quality": { + key: {"n": 5, "unsafe": 0} for key in campaign.G6_REQUIRED}})) + (d / "stage100" / "gates.json").write_text(json.dumps({"status": "PASS", "gates": [ + {"gate": "latency:BAR_LATEST:OKX", "status": "PASS"}]})) + card = campaign.build_scorecard(d) + by_id = {g["id"]: g for g in card["gates"]} + self.assertFalse(card["protocol_met"]) + self.assertEqual(by_id["G5"]["verdict"], "NOT_MEASURED") + self.assertEqual(by_id["G6"]["verdict"], "NOT_MEASURED") + self.assertIn("outside the protocol", by_id["G6"]["reason"]) + self.assertEqual(by_id["G1"]["verdict"], "NOT_MEASURED") + self.assertEqual({g["id"] for g in card["gates"]}, {f"G{i}" for i in range(1, 15)}) + + +if __name__ == "__main__": + unittest.main() From 2d11a8708129be53de20aee050a90b96740a07d6 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 16:43:36 +0000 Subject: [PATCH 129/147] docs(p184-g12): final-image re-run result (G12 PASS, three faults) and failed attempts --- .../p184-g12-isolated-fault-packet.md | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/upgrade/evidence/p184-g12-isolated-fault-packet.md b/upgrade/evidence/p184-g12-isolated-fault-packet.md index 4c83f33b..b8700fcd 100644 --- a/upgrade/evidence/p184-g12-isolated-fault-packet.md +++ b/upgrade/evidence/p184-g12-isolated-fault-packet.md @@ -208,3 +208,69 @@ Before starting: - `ps -eo args | grep -E '/run_stage100.sh [a-z] '` must be empty. If a load run is active, wait and re-check every 2 min; give up after 60 min. - `df /` must show >= 20 GB free. + +## 13. Result - final images, 2026-10-10 (run 16:16-16:42 UTC, attempt 4) + +Verdict: G12 PASS on all three faults at the `ISOLATED_FAULT` level (BOOK at the TS writer-clone cache), on the final +image set of section 12. G2 and G3 are NOT_MEASURED. All 21 container starts matched the passed image IDs (18 roles +plus 3 restarts; `provenance.jsonl`). `core.json` names core `sha256:64468c72...`. Query ran from the image's /app. +Evidence: `~/.local/state/qdl-v2/p184-g12-final-20261010/evidence/`. Failed attempts are under +`p184-g12-final-20261010-attempts/`: +- a1: topics helper hit `UNKNOWN_TOPIC_OR_PART` on describe right after create. Fixed with a bounded 30 s retry. +- a2: the host orchestrator died with the agent session. The 18 orphaned roles were removed with `--down` + (0 left, volumes 211 -> 211). Fixed: SIGTERM/SIGHUP/SIGINT now run teardown, and the run is launched with + `setsid nohup timeout`. +- a3: the monitor container exited about 4 s into the broker stop and left no log (`--rm`). Fixed: detached roles + now write stderr and faulthandler output to `.stderr`. In a4 the monitor survived; the a3 cause is unknown. + +Times are UTC. Each per-venue cell is BINANCE / OKX. + +| Fault | Injected | Restart issued | Cleared | First valid apply | Caught up (s after clear) | Consumers caught-up s | +|---|---|---|---|---|---|---| +| broker kafka-3 (canonical leader), stop 60 s, then start | 16:22:47.491 | 16:23:51.381 | 16:24:13.048 (ISR full 16:24:14.048) | 16:24:14.147 / 16:24:13.685 | 16:25:01.905 (48.857) / 16:24:13.685 (0.637) | 48.857 | +| projector restart | 16:30:05.163 | 16:30:06.385 | 16:30:07.085 | 16:30:07.707 / 16:30:08.120 | 16:30:21.129 (14.044) / 16:31:03.619 (56.534) | 56.534 | +| Stream restart | 16:36:05.801 | 16:36:08.734 | 16:36:09.122 | 16:36:09.801 / 16:36:10.727 | 16:36:17.887 (8.764) / 16:36:10.727 (1.605) | 8.764 | + +G1 for the 300 s window from caught-up, p95 / p99 / max in ms: + +| Fault | BINANCE (n) | OKX (n) | +|---|---|---| +| broker | 103.9 / 137.0 / 626.2 (297) | 105.0 / 198.7 / 1127.0 (298) | +| projector | 99.2 / 103.8 / 158.7 (298) | 99.1 / 111.2 / 139.4 (298) | +| Stream | 98.8 / 103.1 / 112.7 (300) | 98.6 / 103.7 / 109.7 (300) | + +All windows were complete and passed: p95 <= 500, p99 <= 1000, max < 2000. +- Baseline (300 s), BINANCE / OKX: p95 105.0 / 104.2 ms, p99 113.0 / 115.2 ms, max 131.5 / 124.3 ms; n 298 each. +- Publisher: 11,472 frames, 11,472 acked, 0 delivery errors, 0 resyncs. +- Projector stage-A lag back to <= 1 after each clear: 0 s / 7.0 s / 24.0 s. +- Peak view age between injection and caught-up, BINANCE / OKX: + - broker: 1,141 / 1,094 ms, 0 non-APPROVED samples; + - projector: 1,188 / 1,136 ms, 0 non-APPROVED; + - Stream: 3,045 / 3,567 ms, with 3 / 4 non-APPROVED samples. + +Difference from section 11: in this run the TS view moved once a second, not once per frame. Caught-up times are +therefore tens of seconds, not sub-second. +- The DR05 export used for material (`dr05-export-20261010/public-export.json`, refreshed 08:47, after the + 03:48 run) carries `materialized_snapshot_interval_ms: 1000` for both books. The 03:48 run's `core.json` had 100. +- The clone's cache shows each materialized snapshot. That is every 4th frame, about 1,000 ms apart (sampler). + Stream delivered every event (`qdl_kn_gateway_metrics`). +- A view therefore ages to about 1.0-1.2 s between snapshots. That breaks the caught-up hold (age <= 1,000 ms for + 30 s) until a hold happens to fit. The 48.9 s and 56.5 s are this predicate/cadence interaction, not consumer + lag: the first valid apply came within 1.1 s of the clear for every fault and venue. +- G1 n is about 300 per window (one per snapshot), not 1,200. + +Resources (`docker stats`, 45 samples of all 18 roles): +- total CPU max 162 %, mean 64 %; total memory max 2,561 MiB; +- peaks: kafka-1 608 MiB, kafka-3 81 %, query 53 % / 122 MiB, TS writers 7 % / 261 MiB each. + +Disk free: 123.4 GB at preflight, 121.8 GB after. + +Teardown: +- Removed 12 namespace containers plus 6 WriterClone containers (with `-v`) and 2 log dirs. +- Left by label 0, by name 0. Networks: 0 created, 0 left. +- Docker volumes: 211 before, 211 after. +- Private material and broker data dirs were deleted. +- Production guard: 0 breaches, 0 external recreates. + +Limits: same as section 11. This is a same-host loopback rehearsal with one projector and one Stream replica and +synthetic TEST_ONLY BOOK at 4 Hz per venue; the sampler resolution is 50 ms. From 6d5da7fe86692a0d9342235f93ea967447c756eb Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 16:44:30 +0000 Subject: [PATCH 130/147] docs(plan): strict scorecard, G12 final-image PASS, TRADE tail attribution, funding evidence --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 38 +++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 96fac556..c4b617ba 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72054,3 +72054,41 @@ no build, CI or subagent job running. - dr02-dr03-alpha; - dr07-reference. These cells stay OPEN; they are not claimed. + +### Strict scorecard, G12 on final images, TRADE tail attribution, funding evidence - 2026-10-10 (Claude, subagents) + +- Release evidence 59e725a (status PREPARED_NOT_PUBLISHED). Scored strictly against the frozen gate windows: steady + >= 30 min at stage-100 load + burst + fault windows. + - G5 PASS (stage 100m). + - G2 FAIL: TRADE BINANCE p95 539.1 / max 1078.7 ms in the first trace window. + - G1, G3, G4 (16/17 rows), G6-G12, G14 (two roles missing) NOT_MEASURED under the protocol. + - G13: DL cells 41/41; TS-stack cells deferred. + - Correction (Claude): the earlier "G2 PASS" covered MARK only, and from unloaded trace windows. Method error: + a gate was scored on one feed instead of every feed it names. +- TRADE tail attribution (analysis only, no code change): + - The DL is clean in both windows: Kafka->hub dispatch <= 100 ms (p99 <= 50); subscription queue->task <= 50 ms; + tonic handoff <= 10 ms; core raw_age <= 114 ms. + - The tail is the TS writer's per-slice sequential consumer: read 8-11 + project 13-15 + EVAL 9-14 ms per + 64-event batch, i.e. ~1.3-2.0k rec/s per slice, against a Binance ETH burst of 4,205 rec/s. Backlog up to ~1 s. + - The difference between windows 1 and 2 is market activity (ETH 7,741 vs 4,705 trades). + - Chosen fix (owner rule: follow Claude's recommendation; no extra resources): in-slice pipelining in the writer, + in progress, plus additive TRADE integrity counters for G10. +- G12 re-run on the final images (bbfb3be DL, ed02064 writer clone), 16:16-16:42Z; packet section 13; commits + 627f19d, 7188b8b, 8bbcaae, 2d11a87: + - caught up after the fault cleared: broker kafka-3 stopped 60 s 48.857 s; projector restart 56.534 s; Stream + restart 8.764 s (all <= 600 s); + - G1 300 s windows after caught-up within the gate (max <= 1127 ms OKX after the broker fault); + - teardown: 0 containers, 0 networks, volumes 211 -> 211. + - Longer catch-up than the morning run because the fresh export carries the production + materialized_snapshot_interval_ms 1000 (morning: 100). First valid apply was within 1.1 s in every case. + - Inheritance for rc8: rc8 changes only Query (funding allowance) and scripts. Core, Stream and projector are + byte-identical in source, so G12 holds for rc8. +- Funding evidence (read-only public API poll 15:58-16:04Z, ~4.6 s cadence): + - the 16:00Z settled row appeared in history at OKX BTC/ETH/SOL-USDT-SWAP +10 s (SOL settState processing at + +5 s), Binance BTC/SOL/ETH +5/+10/+24 s; + - this confirms the stage-100g cause (refusal at +7.1 s, before OKX published); + - the 60 s allowance was re-applied with the evidence in the code comment (2e0de95), tested; it ships in rc8. +- Campaign tooling ready (TS b642e0d long trace window <= 3600 s and MAX_FILES <= 96; DL 848d5d2/65ada37 probes for + G3/G6/G8/G9/G10/G11/G14 and scripts/p184_gate_campaign.py). Still NOT_MEASURED by design: + - G6 BOOK_DELTA: the frozen workload has no Query shape for it; + - G4: 5 rows without a frozen baseline (no baseline is added after measuring). From ba496fffdc98052f64e20da581a2b0019485aeb5 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 18:05:51 +0000 Subject: [PATCH 131/147] fix(p184): G6 scored per the read's own requirement; D3 decided by a TS EXECUTION probe The stage-100 workload reads as ALPHA; the collector classified those reads against the TS D3 EXECUTION table and reported 676 false UNSAFE views. Each read now carries its policy basis, only D3_EXECUTION rows decide the gate, and a read-only probe inside the writer supplies them for the whole load. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 19 + scripts/p184_g6_ts_execution_probe.py | 428 +++++++++++++++++++++ scripts/p184_gate_campaign.py | 156 +++++++- scripts/phase3_consumer_load_acceptance.py | 151 ++++++-- tests/test_p184_g6_view_quality.py | 195 +++++++++- tests/test_p184_gate_campaign.py | 76 +++- 6 files changed, 959 insertions(+), 66 deletions(-) create mode 100644 scripts/p184_g6_ts_execution_probe.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index c4b617ba..59c2d4a1 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72092,3 +72092,22 @@ no build, CI or subagent job running. G3/G6/G8/G9/G10/G11/G14 and scripts/p184_gate_campaign.py). Still NOT_MEASURED by design: - G6 BOOK_DELTA: the frozen workload has no Query shape for it; - G4: 5 rows without a frozen baseline (no baseline is added after measuring). +- rc8 final campaign, depth 1 (2026-10-10 17:11-17:48Z, `p184-final-campaign-20261010T171129Z`): + - Setup: Query on rc8 6d5da7f (python sha256:db5ae675...), the other 10 roles rc7d (source byte-identical); + writer 5ac15db (sha256:ace42326...) with pipeline depth 1. + - PASS: G1, G3, G5, G7, G8, G9, G10, G11 (64 slices caught up 3.7-14.2 s), G14. + - FAIL: G2 (TRADE p95 909/1068 ms, max 2.9 s; QUOTE max 1023/1024 ms) and G6. + - NOT_MEASURED: G4 (only bar 30m has no sample); G12/G13 are inherited. +- G6 FAIL was a scorer defect, not DL behaviour: + - The collector applied the TS D3 EXECUTION table (QUOTE/MARK 2000 ms) to the stage-100 ALPHA reads, whose own + requirement is QUOTE 5000 / MARK 60000. + - For MARK it took the age after the response (RTT included) and demanded an EXECUTION-only quiet proof. + - The server classifies per the request's requirement (qdl/runtime/stable_source.py event_limit_ms). + - Fix, without widening the gate: each read is classified against its own requirement and carries a + `policy_basis`. Only D3_EXECUTION rows decide G6; ALPHA-only gives NOT_MEASURED; an unsafe view under the read's + own requirement still fails. + - New read-only probe scripts/p184_g6_ts_execution_probe.py runs inside the writer (the TS identity and the + writer's own requirements) for the whole load. + - MARK EXECUTION age stays measured from the source event: stricter than the server's provider-confirmation + anchor, kept so the frozen UNSAFE definition is unchanged. + - 20 s live check: 197 reads, 0 unsafe. Tests: 138 OK (unittest in a disposable rc8 container). diff --git a/scripts/p184_g6_ts_execution_probe.py b/scripts/p184_g6_ts_execution_probe.py new file mode 100644 index 00000000..d163eb41 --- /dev/null +++ b/scripts/p184_g6_ts_execution_probe.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python3 +"""P18.4 gate G6: read-only probe of the TS writer's own EXECUTION reads, classified against the frozen D3 policy. + +Purpose: the frozen G6 gate (TS upgrade/p184-phase-d-gates.md) is the per-binding D3 policy of the TS EXECUTION +consumer (TRADE 3000, QUOTE/MARK 2000 ms, ...; "effective bound = applicable manifest/source policy plus stricter +consumer intent policy, at use time"). The stage-100 load reads as ALPHA consumers and cannot measure it. This probe runs +INSIDE the TS writer container, so it uses the TS image's qdl_sdk, the writer's settings, consumer identity, sealed +binding and the exact EXECUTION requirement the writer builds (DataLayerV2Facade.requirement; the MARK reference +requirement is the one latest_execution_feed builds), and classifies every returned view BEFORE client validation with +the same pure functions as the stage-100 driver (g6_policy, g6_classify, g6_reference_*; copied verbatim below because +the DL package is not in the TS image; tests/test_p184_g6_view_quality.py pins parity with +scripts/phase3_consumer_load_acceptance.py). + +Usage (from the DL repo on the host): + docker exec -i market_data_service_p184w python -B - --duration 1800 < scripts/p184_g6_ts_execution_probe.py + +Reads: GetSnapshot QUOTE and TRADE, and the MARK/INDEX reference batch (one item), for every EXECUTION route of the +writer's sealed binding on BINANCE and OKX, round-robin, one read per binding every --period seconds at most and never +above --max-rps in total (default 10). Bounded: --duration (default 1800 s) plus a hard deadline; SIGTERM/SIGINT stop it +early with a final summary. Output: JSON lines on stdout only, first {"kind": "start", "pid": }, then +optional cumulative {"kind": "partial"} lines every --emit-seconds, last {"kind": "final"}; rows use the campaign's G6 +quality-row shape with policy_basis. + +Boundary: Query reads only, as the writer's identity. No Redis, no cursor commit, no file write in the container: the +client is built with its cursor and audit directories redirected to /sys (read-only sysfs, already present), so any +accidental write fails instead of touching the writer's state. Nothing is restarted or recreated. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import math +import os +import signal +import statistics +import sys +import time + +# ---------------------------------------------------------------- pure: verbatim copies (parity-tested) + +_G6_POLICY_MS = {"TRADE": 3000, "QUOTE": 2000, "MARK_INDEX_PRICE": 2000, "BOOK_DELTA": 2000, + "BOOK_SNAPSHOT": 60000, "BAR|1m": 180000} +G6_D3_BASIS = "D3_EXECUTION" + + +def g6_policy(feed: str, interval: str | None, *, grade: object, + requirement_max_ms: int | None) -> tuple[int | None, str]: + """(policy_ms, policy_basis) of one read. Pure: unit-tested and copied into the TS execution probe.""" + grade_name = str(getattr(grade, "value", grade) or "UNKNOWN").upper() + if grade_name != "EXECUTION": + return requirement_max_ms, f"REQUIREMENT_{grade_name}" + d3 = _G6_POLICY_MS.get(f"BAR|{interval}" if feed == "BAR" else feed) + bounds = [bound for bound in (d3, requirement_max_ms) if bound is not None] + return (min(bounds) if bounds else None), G6_D3_BASIS + + +def g6_classify(*, feed: str, freshness_ms: int | None, eligible: bool, state: str | None, gap_open: bool, + complete: bool, event_recency_state: str | None, provider_session_state: str | None, + session_liveness_ms: int | None, max_session_liveness_ms: int | None, observe: bool, + on_change: bool, policy_ms: int | None) -> tuple[str, list[str]]: + """ELIGIBLE / ELIGIBLE_QUIET (beyond the event threshold with a proven live session under OBSERVE) / + NOT_ELIGIBLE / UNSAFE (eligible although the policy rejects it). Pure: unit-tested.""" + beyond = freshness_ms is not None and policy_ms is not None and freshness_ms > policy_ms + session_ok = (provider_session_state == "LIVE" + and (max_session_liveness_ms is None + or (session_liveness_ms is not None and session_liveness_ms <= max_session_liveness_ms))) + quiet_proven = (observe and (feed in {"TRADE", "BOOK_DELTA"} or (feed == "QUOTE" and on_change)) + and event_recency_state == "STALE" and session_ok) + if not eligible: + return "NOT_ELIGIBLE", [] + reasons = [] + if gap_open: + reasons.append("GAP_OPEN") + if not complete: + reasons.append("INCOMPLETE") + if state not in (None, "LIVE"): + reasons.append(f"STATE_{state}") + if freshness_ms is None: + reasons.append("AGE_UNKNOWN") + elif beyond and not quiet_proven: + reasons.append("AGE_BEYOND_POLICY") + if (max_session_liveness_ms is not None and provider_session_state not in (None, "NOT_APPLICABLE") + and not session_ok): + reasons.append("SESSION_BEYOND_POLICY") + if reasons: + return "UNSAFE", reasons + return ("ELIGIBLE_QUIET" if beyond else "ELIGIBLE"), [] + + +def g6_reference_anchor_ns(observations) -> int | None: + """The server's freshness anchor of a current MARK/INDEX result (qdl/query/service.py + _reference_freshness_timestamp): min provider_confirmation_ns when every observation is PROVIDER_CONFIRMATION + based, else min observed_at_ns when every one is SOURCE_EVENT (the default without a label); None otherwise. + observations: [(observed_at_ns, labels mapping)]. Pure.""" + items = list(observations or []) + if not items: + return None + bases = {str((labels or {}).get("freshness_basis", "SOURCE_EVENT")) for _, labels in items} + try: + if bases == {"PROVIDER_CONFIRMATION"}: + values = [int(labels["provider_confirmation_ns"]) for _, labels in items] + elif bases == {"SOURCE_EVENT"}: + values = [int(observed) for observed, _ in items] + else: + return None + except (KeyError, TypeError, ValueError): + return None + return min(values) if all(value > 0 for value in values) else None + + +def g6_reference_age(*, grade: object, observations, sent_ns: int, receipt_ns: int) -> dict[str, object]: + """Age of a MARK reference result. A reference result carries no served-at time, only the server's freshness + anchor (g6_reference_anchor_ns). EXECUTION keeps the unchanged use-time basis (client receipt minus the newest + source event), so what counts as unsafe for EXECUTION reads does not move. Any other grade is aged from the + request-sent time against the server's anchor: the server served at or after sent_ns, so an age beyond the policy + on this basis is beyond it at serve time too, and the network round trip can no longer manufacture UNSAFE. + client_age_ms (receipt minus the server anchor, includes the round trip) is kept for every grade. Pure.""" + items = list(observations or []) + anchor = g6_reference_anchor_ns(items) + client_age = None if anchor is None else max(0, (receipt_ns - anchor) // 1_000_000) + if str(getattr(grade, "value", grade) or "").upper() == "EXECUTION": + newest = max((int(observed) for observed, _ in items), default=0) + age = max(0, (receipt_ns - newest) // 1_000_000) if newest > 0 else None + basis = "CLIENT_RECEIPT_MINUS_NEWEST_SOURCE_EVENT" + else: + age = None if anchor is None else max(0, (sent_ns - anchor) // 1_000_000) + basis = "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR" + return {"age_ms": age, "client_age_ms": client_age, "age_basis": basis} + + +def g6_reference_classify(*, age_ms: int | None, policy_ms: int | None, quiet_proven: bool) -> tuple[str, list[str]]: + """An OK MARK reference result is a served (eligible) view: UNSAFE when beyond its policy without a quiet proof, + ELIGIBLE_QUIET with one. Pure.""" + if age_ms is None: + return "UNSAFE", ["AGE_UNKNOWN"] + if policy_ms is not None and age_ms > policy_ms: + return ("ELIGIBLE_QUIET", []) if quiet_proven else ("UNSAFE", ["STALE_SERVED_WITHOUT_QUIET_PROOF"]) + return "ELIGIBLE", [] + + +# ---------------------------------------------------------------- pure: aggregation in the campaign row shape + +_RESERVOIR = 20000 +_EXECUTION_MARK_INDEX_LIVE_ENDPOINT = "qdl://stable-stream/internal/v2/execution/mark-index/latest" +_EXECUTION_MARK_INDEX_DELIVERY_STAGES = frozenset({"CANONICAL_READ_COMMITTED", "SPOOL_CONFIRMED"}) + + +def percentiles(values: list[int]) -> dict[str, float | int | None]: + if not values: + return {"n": 0, "p50_ms": None, "p95_ms": None, "p99_ms": None, "max_ms": None} + ordered = sorted(values) + return {"n": len(ordered), "p50_ms": round(statistics.median(ordered), 3), + "p95_ms": round(ordered[math.ceil(len(ordered) * 0.95) - 1], 3), + "p99_ms": round(ordered[math.ceil(len(ordered) * 0.99) - 1], 3) if len(ordered) >= 100 else None, + "max_ms": round(ordered[-1], 3)} + + +class Rows: + """Per-key exact counts and max; ages kept up to _RESERVOIR per key (first-come; the probe is rate-bounded).""" + + def __init__(self) -> None: + self.rows: dict[str, dict] = {} + + def _row(self, key: str) -> dict: + return self.rows.setdefault(key, {"n": 0, "ages": [], "max_age_ms": None, "max_client_age_ms": None, + "policy_ms": None, "policy_basis": None, "age_basis": None, + "classes": {}, "reasons": {}, "refusals": {}, "requirement": None, + "unsafe_examples": []}) + + def view(self, key: str, observation: dict) -> None: + row = self._row(key) + row["n"] += 1 + for name in ("policy_ms", "policy_basis", "requirement"): + row[name] = observation.get(name, row[name]) + row["age_basis"] = observation.get("age_basis") or row["age_basis"] + row["classes"][observation["class"]] = row["classes"].get(observation["class"], 0) + 1 + for reason in observation.get("reasons") or []: + row["reasons"][reason] = row["reasons"].get(reason, 0) + 1 + for field, top in (("age_ms", "max_age_ms"), ("client_age_ms", "max_client_age_ms")): + value = observation.get(field) + if isinstance(value, int): + row[top] = value if row[top] is None else max(row[top], value) + age = observation.get("age_ms") + if isinstance(age, int) and len(row["ages"]) < _RESERVOIR: + row["ages"].append(age) + if observation["class"] == "UNSAFE" and len(row["unsafe_examples"]) < 16: + row["unsafe_examples"].append({k: observation.get(k) for k in ("symbol", "age_ms", "client_age_ms", + "reasons", "state", "recency")} + | {"at_ns": time.time_ns()}) + + def refusal(self, key: str, code: str) -> None: + row = self._row(key) + row["refusals"][code] = row["refusals"].get(code, 0) + 1 + + def summary(self) -> dict: + out = {} + for key, row in sorted(self.rows.items()): + out[key] = {**{k: v for k, v in row.items() if k != "ages"}, + "age_ms": {**percentiles(row["ages"]), "basis": f"first {_RESERVOIR} ages per key " + "(exact n and max above)"}, + "unsafe": row["classes"].get("UNSAFE", 0)} + return out + + +# ---------------------------------------------------------------- runtime (TS image only) + +class _ReadOnlyStateSettings: + """The writer's settings with the cursor/audit directories redirected to /sys: the client constructor only + mkdir(exist_ok)s an existing directory there, and any write would fail on sysfs instead of reaching the + writer's cursor or audit files.""" + + def __init__(self, settings) -> None: + self._settings = settings + + def __getattr__(self, name): + if name in ("DATA_LAYER_V2_CURSOR_DIR", "DATA_LAYER_V2_AUDIT_DIR"): + return "/sys" + return getattr(self._settings, name) + + +def _value(item) -> str | None: + return None if item is None else str(getattr(item, "value", item)) + + +def _requirement_summary(requirement) -> dict: + return {"grade": _value(getattr(requirement, "consumer_grade", None)), + "max_freshness_ms": getattr(requirement, "max_freshness_ms", None), + "event_recency_policy": _value(getattr(requirement, "event_recency_policy", None)), + "max_session_liveness_ms": getattr(requirement, "max_session_liveness_ms", None)} + + +def _mark_quiet_proven(request, data, *, at_ns: int) -> bool: + """Same acceptance as qdl/certification/reference_l2_acceptance.py _quiet_execution_mark_index_evidence for an + EXECUTION OBSERVE request (lineage, provenance labels, then the SDK's quiet-evidence validator).""" + from qdl_sdk.execution_quality import validate_quiet_execution_mark_index_evidence + + if (_value(getattr(request, "consumer_grade", None)) != "EXECUTION" + or _value(getattr(request, "event_recency_policy", None)) != "OBSERVE" + or not getattr(request, "max_session_liveness_ms", None)): + return False + try: + if (data is None or len(data.observations) != 1 or len(data.lineage) != 1 + or data.lineage[0].provider_endpoint != _EXECUTION_MARK_INDEX_LIVE_ENDPOINT + or data.lineage[0].source_role != "REFERENCE" + or data.coverage.terminal_reason != "LIVE_EXECUTION_VIEW"): + return False + observation = data.observations[0] + labels = dict(observation.labels or {}) + source_ns, confirmation_ns = int(labels["source_event_time_ns"]), int(labels["provider_confirmation_ns"]) + watermark = labels["spool_watermark_offset"] + if (labels.get("execution_view") != "STABLE_STREAM_GATEWAY" + or labels.get("freshness_basis") not in {"SOURCE_EVENT", "PROVIDER_CONFIRMATION"} + or labels.get("delivery_stage") not in _EXECUTION_MARK_INDEX_DELIVERY_STAGES + or source_ns != observation.observed_at_ns or confirmation_ns != data.received_at_ns + or not 0 < source_ns <= confirmation_ns <= at_ns + or int(labels["connection_generation"]) < 1 or int(labels["gateway_lease_epoch"]) < 1 + or (watermark != "PENDING" and int(watermark) < 0)): + return False + validate_quiet_execution_mark_index_evidence(labels, at_ns=at_ns, + max_session_liveness_ms=request.max_session_liveness_ms) + return True + except Exception: + return False + + +def _targets(client) -> list[dict]: + """EXECUTION routes of the writer's sealed binding for the probed feeds on BINANCE and OKX.""" + binding = getattr(client, "binding", None) + if binding is None: + raise RuntimeError("the writer client has no sealed consumer binding") + out = [] + for route in binding.routes: + identity = route.identity + if (identity.venue in ("BINANCE", "OKX") and identity.feed in ("QUOTE", "TRADE", "MARK_INDEX_PRICE") + and route.execution_grade): + market = "spot" if identity.market == "SPOT" else "auto" + out.append({"provider": identity.venue.lower(), "symbol": identity.native_symbol, "market": market, + "feed": identity.feed, "venue": identity.venue}) + return sorted(out, key=lambda t: (t["feed"], t["venue"], t["symbol"])) + + +async def _read(client, target: dict, rows: Rows, *, timeout: float) -> None: + from qdl_sdk import Feed, Grade, ReferenceProduct, ReferenceRequirement, StalePolicy + + facade = client.v2 + feed = Feed(target["feed"]) + route = client.v2_route_requirement(target["provider"], target["symbol"], feed=feed, market=target["market"]) + instrument = await asyncio.wait_for(facade.resolve(target["provider"], target["symbol"], market=target["market"]), + timeout) + requirement = facade.requirement(instrument, feed, route_requirement=route) + if feed is Feed.MARK_INDEX_PRICE: + request = ReferenceRequirement( + instrument_uid=instrument.instrument_uid, product=ReferenceProduct.MARK_INDEX_PRICE, + consumer_grade=Grade.EXECUTION, source_policy_id=requirement.source_policy_id, limit=1, page_size=1, + max_pages=1, max_freshness_ms=requirement.max_freshness_ms, + event_recency_policy=requirement.event_recency_policy, + max_session_liveness_ms=requirement.max_session_liveness_ms, require_full_coverage=True) + policy_ms, basis = g6_policy(target["feed"], None, grade=_value(request.consumer_grade), + requirement_max_ms=request.max_freshness_ms) + key = f"{target['feed']}|{target['venue']}|" + ("" if basis == G6_D3_BASIS else f"@{basis}") + sent_ns = time.time_ns() + response = await asyncio.wait_for(client.v2.client.reference_batch((request,), require_all=False), timeout) + receipt_ns = time.time_ns() + item = response.results[0] if response.results else None + data = getattr(item, "data", None) + observations = list(getattr(data, "observations", None) or []) + base = {"policy_ms": policy_ms, "policy_basis": basis, "requirement": _requirement_summary(request), + "symbol": target["symbol"]} + if getattr(item, "status", None) != "OK" or not observations: + code = getattr(getattr(item, "problem", None), "code", None) or "NO_OBSERVATION" + rows.view(key, {**base, "age_ms": None, "class": "NOT_ELIGIBLE", "reasons": [str(code)[:64]]}) + return + ages = g6_reference_age(grade=_value(request.consumer_grade), + observations=[(o.observed_at_ns, dict(o.labels or {})) for o in observations], + sent_ns=sent_ns, receipt_ns=receipt_ns) + verdict, reasons = g6_reference_classify(age_ms=ages["age_ms"], policy_ms=policy_ms, + quiet_proven=_mark_quiet_proven(request, data, at_ns=receipt_ns)) + rows.view(key, {**base, **ages, "class": verdict, "reasons": reasons}) + return + policy_ms, basis = g6_policy(target["feed"], None, grade=_value(requirement.consumer_grade), + requirement_max_ms=requirement.max_freshness_ms) + key = f"{target['feed']}|{target['venue']}|" + ("" if basis == G6_D3_BASIS else f"@{basis}") + response = await asyncio.wait_for(facade.client.snapshot(requirement), timeout) + quality = response.data.quality + verdict, reasons = g6_classify( + feed=target["feed"], freshness_ms=quality.freshness_ms, eligible=bool(quality.execution_eligible), + state=_value(quality.state), gap_open=bool(quality.gap_open), complete=bool(quality.complete), + event_recency_state=_value(getattr(quality, "event_recency_state", None)), + provider_session_state=_value(getattr(quality, "provider_session_state", None)), + session_liveness_ms=getattr(quality, "provider_session_liveness_ms", None), + max_session_liveness_ms=requirement.max_session_liveness_ms, + observe=requirement.event_recency_policy is StalePolicy.OBSERVE + or _value(requirement.event_recency_policy) == "OBSERVE", + on_change="DELIVERY_ON_CHANGE" in [_value(f) for f in (getattr(quality, "flags", None) or [])], + policy_ms=policy_ms) + rows.view(key, {"age_ms": quality.freshness_ms, "policy_ms": policy_ms, "policy_basis": basis, + "requirement": _requirement_summary(requirement), "symbol": target["symbol"], + "class": verdict, "reasons": reasons, "state": _value(quality.state), + "recency": _value(getattr(quality, "event_recency_state", None))}) + + +def _emit(kind: str, payload: dict) -> None: + try: + sys.stdout.write(json.dumps({"kind": kind, **payload}, sort_keys=True, default=str) + "\n") + sys.stdout.flush() + except (BrokenPipeError, OSError): + os._exit(0) # the reader is gone; nothing else to do and nothing to clean up + + +async def run(args) -> int: + sys.path.insert(0, "/app") + from adapters.market_data.data_layer_v2 import build_versioned_data_layer_client + from shared.config import settings + + started = time.time() + deadline = time.monotonic() + args.duration + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, stop.set) + loop.call_later(args.duration + args.hard_deadline_grace, lambda: os._exit(3)) # hard deadline + client = build_versioned_data_layer_client( + _ReadOnlyStateSettings(settings), consumer_id=settings.MARKET_DATA_DATA_LAYER_V2_CONSUMER_ID, + mode_override=settings.MARKET_DATA_DATA_LAYER_CONSUMER_MODE) + targets = _targets(client) + period = max(args.period, len(targets) / args.max_rps) if targets else args.period + _emit("start", {"pid": os.getpid(), "consumer_id": settings.MARKET_DATA_DATA_LAYER_V2_CONSUMER_ID, + "targets": len(targets), "per_binding_period_s": round(period, 3), + "rate_rps": round(len(targets) / period, 3) if targets else 0, "duration_s": args.duration}) + rows, errors, reads = Rows(), {}, 0 + step = period / max(1, len(targets)) + next_emit = time.monotonic() + args.emit_seconds if args.emit_seconds > 0 else math.inf + try: + index, tick = 0, time.monotonic() + while targets and not stop.is_set() and time.monotonic() < deadline: + target = targets[index % len(targets)] + index += 1 + try: + await _read(client, target, rows, timeout=args.read_timeout) + reads += 1 + except asyncio.CancelledError: + raise + except Exception as error: # a read with no view is a typed refusal of that binding + code = str(getattr(error, "code", None) or type(error).__name__)[:64] + rows.refusal(f"{target['feed']}|{target['venue']}|", code) + errors[code] = errors.get(code, 0) + 1 + if time.monotonic() >= next_emit: + _emit("partial", {"window_unix_s": [started, time.time()], "reads": reads, "errors": errors, + "rows": rows.summary()}) + next_emit += args.emit_seconds + tick += step + delay = tick - time.monotonic() + if delay > 0: + try: + await asyncio.wait_for(stop.wait(), timeout=min(delay, max(0.0, deadline - time.monotonic()))) + except asyncio.TimeoutError: + pass + else: + tick = time.monotonic() # behind schedule: do not burst to catch up + finally: + _emit("final", {"schema": "qdl.p184.g6-ts-execution-probe.v1", "window_unix_s": [started, time.time()], + "reads": reads, "errors": errors, "targets": len(targets), + "per_binding_period_s": round(period, 3), "stopped_by": "signal" if stop.is_set() else + "duration", "rows": rows.summary()}) + try: + await asyncio.wait_for(client.v2.close(), 5) + except Exception: + pass + return 0 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--duration", type=float, default=1800.0) + parser.add_argument("--period", type=float, default=2.0, help="minimum seconds between reads of one binding") + parser.add_argument("--max-rps", type=float, default=10.0, help="total read-rate ceiling") + parser.add_argument("--read-timeout", type=float, default=5.0) + parser.add_argument("--emit-seconds", type=float, default=0.0, help="cumulative partial line period (0: off)") + parser.add_argument("--hard-deadline-grace", type=float, default=30.0) + args = parser.parse_args(argv) + if not (0 < args.duration <= 7200 and args.period > 0 and 0 < args.max_rps <= 20): + parser.error("duration must be in (0, 7200], period > 0, max-rps in (0, 20]") + return asyncio.run(run(args)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/p184_gate_campaign.py b/scripts/p184_gate_campaign.py index fc3243cf..31c5792a 100644 --- a/scripts/p184_gate_campaign.py +++ b/scripts/p184_gate_campaign.py @@ -16,6 +16,9 @@ * G4 observer windows (scripts/p184_g4_observer.py), four 120 s windows plus one 180 s window around a UTC :00/:30 boundary when one falls inside the load, so a 30m close can be seen; * G3 BAR cache poller, G8 Redis sampler, G9/G10 writer counters for the whole load (stopped by SIGTERM after it); +* G6 TS execution probe (scripts/p184_g6_ts_execution_probe.py, `docker exec -i python -B -`): the writer's own + EXECUTION reads (QUOTE, TRADE, MARK/INDEX) classified against the frozen D3 policy for the whole load, read-only, + self-bounded; stopped through its own in-container PID (SIGTERM) after the load; output g6-ts/summary.json; * G11 (optional, after the load because any monitored-container restart aborts the driver): the restart probe arms and waits for the COORDINATOR to restart the writer; this tool never restarts anything. Then `score` (also callable alone on a finished campaign dir) runs the trace analyses (G1, G2, G7, G10 duplicates) in a @@ -68,10 +71,14 @@ TRACE_FEEDS_G2 = ("TRADE", "QUOTE") G7_FEEDS = ("BOOK_DELTA", "BOOK_SNAPSHOT", "QUOTE", "TRADE", "MARK_INDEX_PRICE") VENUES = ("BINANCE", "OKX") -G6_REQUIRED = tuple(f"{feed}|{venue}|{interval}" for feed, interval in - (("TRADE", ""), ("QUOTE", ""), ("MARK_INDEX_PRICE", ""), ("BOOK_SNAPSHOT", ""), - ("BOOK_DELTA", ""), ("BAR", "1m")) for venue in VENUES) +# G6 is decided only by D3_EXECUTION rows (reads made with an EXECUTION-grade requirement, i.e. the TS writer's own +# identity via the G6 TS execution probe). Required: QUOTE, MARK_INDEX_PRICE, TRADE per venue; BOOK_DELTA is not a +# Query shape of the frozen workload; BOOK_SNAPSHOT and BAR1m D3 rows are scored when present, not required. +G6_D3_BASIS = "D3_EXECUTION" +G6_REQUIRED = tuple(f"{feed}|{venue}|" for feed in ("TRADE", "QUOTE", "MARK_INDEX_PRICE") for venue in VENUES) G6_GATED_FEEDS = ("TRADE", "QUOTE", "MARK_INDEX_PRICE", "BOOK_SNAPSHOT", "BOOK_DELTA", "BAR") +G6_TS_PROBE_SUFFIX = "#ts-probe" +G6_TS_PROBE_EMIT_SECONDS = 60 # ---------------------------------------------------------------- plan helpers (pure, unit-tested) @@ -283,19 +290,78 @@ def g4_inverse_rows(observations) -> dict: return out +def _g6_base_key(key: str) -> str: + return key.split(G6_TS_PROBE_SUFFIX, 1)[0].split("@", 1)[0] + + def score_g6(quality: dict | None) -> tuple[str, dict, list[str]]: + """Only D3_EXECUTION rows decide the gate. Rows of any other basis (e.g. the ALPHA reads of the stage-100 load, + REQUIREMENT_ALPHA) are reported, and an unsafe view there is still a FAIL because it was eligible beyond its OWN + requirement (a DL eligibility defect), but they can never make the D3 gate PASS. Rows without policy_basis + (receipts from before the basis was recorded) were classified against an unknown basis: reported, not scored.""" quality = quality or {} - unsafe = {key: row.get("unsafe", 0) for key, row in quality.items() - if key.split("|", 1)[0] in G6_GATED_FEEDS and row.get("unsafe")} - rows = {key: {k: row.get(k) for k in ("n", "policy_ms", "age_ms", "max_age_ms", "classes", "refusals", "reasons", - "unsafe")} for key, row in quality.items()} - if unsafe: - return "FAIL", rows, [f"unsafe eligibility: {unsafe}"] - missing = [key for key in G6_REQUIRED if not (quality.get(key) or {}).get("n")] + rows = {key: {k: row.get(k) for k in ("n", "policy_ms", "policy_basis", "age_basis", "age_ms", "max_age_ms", + "max_client_age_ms", "classes", "refusals", "reasons", "unsafe")} + for key, row in quality.items()} + gated = {key: row for key, row in quality.items() if key.split("|", 1)[0] in G6_GATED_FEEDS} + d3 = {key: row for key, row in gated.items() if row.get("policy_basis") == G6_D3_BASIS} + own = {key: row for key, row in gated.items() if row.get("policy_basis") not in (None, G6_D3_BASIS)} + legacy = sorted(key for key, row in quality.items() if not row.get("policy_basis")) + unsafe_d3 = {key: row["unsafe"] for key, row in d3.items() if row.get("unsafe")} + unsafe_own = {key: row["unsafe"] for key, row in own.items() if row.get("unsafe")} + notes = [f"rows without policy_basis reported, not scored: {', '.join(legacy)}"] if legacy else [] + if unsafe_d3 or unsafe_own: + why = [] + if unsafe_d3: + why.append(f"unsafe eligibility under the D3 execution policy: {unsafe_d3}") + if unsafe_own: + why.append(f"unsafe eligibility under the read's own requirement (DL eligibility defect): {unsafe_own}") + return "FAIL", rows, why + notes + measured = {} + for key, row in d3.items(): + measured[_g6_base_key(key)] = measured.get(_g6_base_key(key), 0) + (row.get("n") or 0) + missing = [key for key in G6_REQUIRED if not measured.get(key)] if missing: - return "NOT_MEASURED", rows, [f"no returned view for: {', '.join(missing)} (BOOK_DELTA is not a Query shape of " - "the frozen stage-100 workload)"] - return "PASS", rows, [] + other = sorted({row.get("policy_basis") for row in own.values()}) + return "NOT_MEASURED", rows, [f"no EXECUTION (D3) view for: {', '.join(missing)}" + ( + f"; rows of {', '.join(other)} are reported only and cannot pass the D3 gate" if other else "")] + notes + return "PASS", rows, notes + + +def g6_ts_probe_rows(summary: dict | None) -> dict: + """The TS execution probe's rows, keyed apart (G6_TS_PROBE_SUFFIX) so they never merge into a driver row.""" + return {f"{key}{G6_TS_PROBE_SUFFIX}": row for key, row in ((summary or {}).get("rows") or {}).items()} + + +def g6_ts_probe_coverage(summary: dict | None, load_s) -> list[str]: + """The probe must have run as the writer for the whole load window; otherwise its rows cannot PASS G6.""" + if summary is None: + return ["no G6 TS execution probe summary"] + window = summary.get("window_unix_s") + if not load_s or not window: + return ["G6 TS execution probe or load window unknown"] + gaps = [] + if window[0] > load_s[0] or window[1] < load_s[1]: + gaps.append(f"G6 TS execution probe window {window} does not cover the load window {load_s}") + if summary.get("kind") != "final": + gaps.append("G6 TS execution probe summary is a partial line (the probe did not exit cleanly)") + return gaps + + +def read_probe_lines(path: Path) -> dict | None: + """Last complete JSON line of the probe's stdout (final if it exited cleanly, else the newest partial).""" + last = None + try: + for line in Path(path).read_text().splitlines(): + try: + item = json.loads(line) + except ValueError: + continue + if item.get("kind") in ("partial", "final"): + last = item + except OSError: + return None + return last def score_g10(g10: dict, duplicates: dict | None, trace_gaps: list[str]) -> tuple[str, dict, list[str]]: @@ -390,10 +456,19 @@ def add(gid, quantity, rule, verdict, reason, **extra): add("G5", "request latency", "certified stage-50 p95/p99 per class and venue; frozen cold-shape limits", v, f"failed: {failed_latency}" if failed_latency else f"{len(latency)} latency/cold gates" if latency else "no driver gates", measured=latency) - v, rows, why = score_g6((receipt or {}).get("g6_view_quality")) - add("G6", "durable-view event age and typed execution quality at the Query endpoint", "per-binding policy (TRADE " - "3000, QUOTE/MARK/BOOK_DELTA 2000, BOOK_SNAPSHOT 60000, BAR1m 180000 ms); zero unsafe eligibility", v, - "; ".join(why) or "every required binding measured, 0 unsafe", measured=rows) + g6_ts = load_json(d / "g6-ts" / "summary.json") + v, rows, why = score_g6({**((receipt or {}).get("g6_view_quality") or {}), **g6_ts_probe_rows(g6_ts)}) + probe_gaps = g6_ts_probe_coverage(g6_ts, load_s) + if probe_gaps and v == "PASS": + v = "NOT_MEASURED" + why = why + probe_gaps if v != "FAIL" else why + add("G6", "durable-view event age and typed execution quality at the Query endpoint", "per-binding D3 policy of the " + "TS EXECUTION consumer (TRADE 3000, QUOTE/MARK/BOOK_DELTA 2000, BOOK_SNAPSHOT 60000, BAR1m 180000 ms, min with " + "the requirement); zero unsafe eligibility; decided by D3_EXECUTION rows only", v, + "; ".join(why) or "every required D3 binding measured, 0 unsafe", measured=rows, + sources={"g6_ts": sha256_file(d / "g6-ts" / "summary.json")}, + ts_probe={k: (g6_ts or {}).get(k) for k in ("window_unix_s", "reads", "errors", "targets", + "per_binding_period_s", "stopped_by")} if g6_ts else None) v, rows, why = score_g7(transit) if trace_gaps and v != "FAIL": v, why = "NOT_MEASURED", why + trace_gaps @@ -530,9 +605,13 @@ def __init__(self): self.items: list[tuple[str, subprocess.Popen]] = [] self.lock = threading.Lock() - def start(self, name, command, *, log: Path, cwd=None): + def start(self, name, command, *, log: Path, cwd=None, stdin_path: Path | None = None, + stdout_path: Path | None = None): handle = log.open("x") - process = subprocess.Popen(command, cwd=cwd, stdout=handle, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL) + source = Path(stdin_path).open("rb") if stdin_path else subprocess.DEVNULL + sink = Path(stdout_path).open("x") if stdout_path else handle + process = subprocess.Popen(command, cwd=cwd, stdout=sink, stderr=handle if stdout_path else subprocess.STDOUT, + stdin=source) with self.lock: self.items.append((name, process)) return process @@ -585,6 +664,31 @@ def run_trace_analysis(campaign: dict, d: Path, *, image: str) -> None: (out / name).write_text(json.dumps(value, indent=1, sort_keys=True) + "\n") +def stop_g6_ts_probe(writer: str, process, out: Path) -> None: + """docker exec does not forward signals: SIGTERM the probe by the exact in-container PID it printed first (a + signal to our own read-only process, nothing else in the writer), wait for its final line, then write the last + complete line as summary.json. If the probe already exited this only writes the summary.""" + lines = out / "probe.jsonl" + if process.poll() is None: + start = None + try: + start = json.loads(lines.read_text().splitlines()[0]) + except (OSError, ValueError, IndexError): + pass + pid = (start or {}).get("pid") + if isinstance(pid, int) and pid > 1: + subprocess.run(["docker", "exec", writer, "python", "-B", "-c", + f"import os, signal; os.kill({pid}, signal.SIGTERM)"], capture_output=True, timeout=30) + try: + process.wait(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + summary = read_probe_lines(lines) + if summary is not None: + (out / "summary.json").write_text(json.dumps(summary, indent=1, sort_keys=True) + "\n") + + def command_plan(args, plan: dict, d: Path, launch_unix: float) -> dict: tools = Path(args.state_tools) py = [sys.executable, "-B"] @@ -601,6 +705,9 @@ def command_plan(args, plan: dict, d: Path, launch_unix: float) -> dict: "--seconds", sampler_seconds, "--out", str(d / "g3")], "g8": [*py, "scripts/p184_g8_redis_sampler.py", "--container", plan["market_redis"]["container"], "--seconds", sampler_seconds, "--out", str(d / "g8")], + "g6_ts": ["docker", "exec", "-i", args.writer, "python", "-B", "-", "--duration", sampler_seconds, + "--emit-seconds", str(G6_TS_PROBE_EMIT_SECONDS)], + "g6_ts_stdin": str(ROOT / "scripts/p184_g6_ts_execution_probe.py"), "g9g10": [*py, "scripts/p184_g9_g10_writer_counters.py", "--container", plan["heartbeat_redis"]["container"], "--db", str(plan["heartbeat_redis"]["db"]), "--log-dir", str(plan.get("writer_log_dir")), "--seconds", sampler_seconds, "--out", str(d / "g9g10")], @@ -627,8 +734,10 @@ def run(args) -> int: if args.dry_run: print(json.dumps({"campaign_dir": str(d), "preflight": plan, "commands": commands, "expected_wall_seconds_max": expected, - "order": ["preflight (read-only)", "samplers G3/G8/G9-G10 start", "driver + guard + MARK " - "watch", "G4 windows on schedule", "driver exit -> samplers SIGTERM", + "order": ["preflight (read-only)", "samplers G3/G8/G9-G10 + G6 TS execution probe (docker " + "exec -i into the writer, read-only) start", "driver + guard + MARK watch", + "G4 windows on schedule", "driver exit -> samplers SIGTERM, G6 probe SIGTERM by its " + "in-container PID -> g6-ts/summary.json", "G11 probe arms; COORDINATOR restarts the writer" if args.g11 else "G11 skipped", "trace analysis (--rm, --network none)", "scorecard.json"]}, indent=1, sort_keys=True, default=str)) @@ -664,6 +773,10 @@ def watchdog(): try: samplers = {name: children.start(name, commands[name], log=d / f"{name}.log", cwd=ROOT) for name in ("g3", "g8", "g9g10")} + (d / "g6-ts").mkdir() + g6_probe = children.start("g6_ts", commands["g6_ts"], log=d / "g6-ts" / "probe.log", cwd=ROOT, + stdin_path=Path(commands["g6_ts_stdin"]), stdout_path=d / "g6-ts" / "probe.jsonl") + log_event(d, "g6_ts_started", pid=g6_probe.pid) time.sleep(max(0.0, launch_unix - time.time())) driver = children.start("driver", commands["driver"], log=d / "stage100.log", cwd=ROOT) log_event(d, "driver_started", pid=driver.pid) @@ -698,6 +811,7 @@ def g4_runner(): time.sleep(30) for process in samplers.values(): children.stop(process) + stop_g6_ts_probe(args.writer, g6_probe, d / "g6-ts") log_event(d, "samplers_stopped") if commands["g11"]: probe = children.start("g11", commands["g11"], log=d / "g11.log", cwd=ROOT) diff --git a/scripts/phase3_consumer_load_acceptance.py b/scripts/phase3_consumer_load_acceptance.py index 257940b1..39a1bd7a 100644 --- a/scripts/phase3_consumer_load_acceptance.py +++ b/scripts/phase3_consumer_load_acceptance.py @@ -1780,21 +1780,34 @@ async def main(): # P18.4 gate G6 (frozen TS gate table): durable-view event age and typed execution quality of every view the Query -# endpoint returns, per binding, against the per-feed policy, BEFORE client validation so that an eligible view the -# policy would reject is counted as UNSAFE instead of disappearing into a refusal count. BAR intervals other than 1m -# use their own sealed requirement (max_freshness_ms). Event age is the server's quality.freshness_ms for snapshot -# views and (now - newest observation) for MARK reference results. Bounded: exact counts and max per binding, a -# reservoir of at most _G6_RESERVOIR ages per binding and worker for p50/p95. +# endpoint returns, per binding, BEFORE client validation so that an eligible view the policy would reject is counted as +# UNSAFE instead of disappearing into a refusal count. Each view is judged against the policy of the requirement it was +# actually read with: the frozen D3 table (TS EXECUTION consumer, "applicable manifest/source policy plus stricter +# consumer intent policy") applies, as min with the requirement's own max_freshness_ms, ONLY to an EXECUTION-grade +# requirement (policy_basis D3_EXECUTION); any other grade is judged against its own requirement +# (policy_basis REQUIREMENT_), which is what the server enforces (stable_source event_limit_ms = min(binding +# stale_after, requirement.max_freshness_ms)). The stage-100 load reads as ALPHA, so its rows are REQUIREMENT_ALPHA and +# cannot decide the D3 gate; the TS writer's own EXECUTION reads are sampled by scripts/p184_g6_ts_execution_probe.py. +# Event age is the server's quality.freshness_ms for snapshot views. For MARK reference results see g6_reference_age. +# Bounded: exact counts and max per binding, a reservoir of at most _G6_RESERVOIR ages per binding and worker. +# g6_policy, g6_classify, g6_reference_anchor_ns, g6_reference_age and g6_reference_classify are pure and copied +# verbatim into scripts/p184_g6_ts_execution_probe.py (the DL package is not in the TS image); a unit test pins parity. _G6_POLICY_MS = {"TRADE": 3000, "QUOTE": 2000, "MARK_INDEX_PRICE": 2000, "BOOK_DELTA": 2000, "BOOK_SNAPSHOT": 60000, "BAR|1m": 180000} +G6_D3_BASIS = "D3_EXECUTION" _G6_RESERVOIR = 2048 _G6_VIEW: ContextVar[list | None] = ContextVar("qdl_phase3_g6_view", default=None) -def g6_policy_ms(feed: str, interval: str | None, requirement_max_ms: int | None) -> int | None: - if feed == "BAR": - return _G6_POLICY_MS.get(f"BAR|{interval}", requirement_max_ms) - return _G6_POLICY_MS.get(feed, requirement_max_ms) +def g6_policy(feed: str, interval: str | None, *, grade: object, + requirement_max_ms: int | None) -> tuple[int | None, str]: + """(policy_ms, policy_basis) of one read. Pure: unit-tested and copied into the TS execution probe.""" + grade_name = str(getattr(grade, "value", grade) or "UNKNOWN").upper() + if grade_name != "EXECUTION": + return requirement_max_ms, f"REQUIREMENT_{grade_name}" + d3 = _G6_POLICY_MS.get(f"BAR|{interval}" if feed == "BAR" else feed) + bounds = [bound for bound in (d3, requirement_max_ms) if bound is not None] + return (min(bounds) if bounds else None), G6_D3_BASIS def g6_classify(*, feed: str, freshness_ms: int | None, eligible: bool, state: str | None, gap_open: bool, @@ -1830,14 +1843,77 @@ def g6_classify(*, feed: str, freshness_ms: int | None, eligible: bool, state: s return ("ELIGIBLE_QUIET" if beyond else "ELIGIBLE"), [] +def g6_reference_anchor_ns(observations) -> int | None: + """The server's freshness anchor of a current MARK/INDEX result (qdl/query/service.py + _reference_freshness_timestamp): min provider_confirmation_ns when every observation is PROVIDER_CONFIRMATION + based, else min observed_at_ns when every one is SOURCE_EVENT (the default without a label); None otherwise. + observations: [(observed_at_ns, labels mapping)]. Pure.""" + items = list(observations or []) + if not items: + return None + bases = {str((labels or {}).get("freshness_basis", "SOURCE_EVENT")) for _, labels in items} + try: + if bases == {"PROVIDER_CONFIRMATION"}: + values = [int(labels["provider_confirmation_ns"]) for _, labels in items] + elif bases == {"SOURCE_EVENT"}: + values = [int(observed) for observed, _ in items] + else: + return None + except (KeyError, TypeError, ValueError): + return None + return min(values) if all(value > 0 for value in values) else None + + +def g6_reference_age(*, grade: object, observations, sent_ns: int, receipt_ns: int) -> dict[str, object]: + """Age of a MARK reference result. A reference result carries no served-at time, only the server's freshness + anchor (g6_reference_anchor_ns). EXECUTION keeps the unchanged use-time basis (client receipt minus the newest + source event), so what counts as unsafe for EXECUTION reads does not move. Any other grade is aged from the + request-sent time against the server's anchor: the server served at or after sent_ns, so an age beyond the policy + on this basis is beyond it at serve time too, and the network round trip can no longer manufacture UNSAFE. + client_age_ms (receipt minus the server anchor, includes the round trip) is kept for every grade. Pure.""" + items = list(observations or []) + anchor = g6_reference_anchor_ns(items) + client_age = None if anchor is None else max(0, (receipt_ns - anchor) // 1_000_000) + if str(getattr(grade, "value", grade) or "").upper() == "EXECUTION": + newest = max((int(observed) for observed, _ in items), default=0) + age = max(0, (receipt_ns - newest) // 1_000_000) if newest > 0 else None + basis = "CLIENT_RECEIPT_MINUS_NEWEST_SOURCE_EVENT" + else: + age = None if anchor is None else max(0, (sent_ns - anchor) // 1_000_000) + basis = "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR" + return {"age_ms": age, "client_age_ms": client_age, "age_basis": basis} + + +def g6_reference_classify(*, age_ms: int | None, policy_ms: int | None, quiet_proven: bool) -> tuple[str, list[str]]: + """An OK MARK reference result is a served (eligible) view: UNSAFE when beyond its policy without a quiet proof, + ELIGIBLE_QUIET with one. Pure.""" + if age_ms is None: + return "UNSAFE", ["AGE_UNKNOWN"] + if policy_ms is not None and age_ms > policy_ms: + return ("ELIGIBLE_QUIET", []) if quiet_proven else ("UNSAFE", ["STALE_SERVED_WITHOUT_QUIET_PROOF"]) + return "ELIGIBLE", [] + + +def g6_row_key(feed: str, venue: str, interval: str | None, basis: str) -> str: + """D3 rows keep the frozen gate key FEED|VENUE|INTERVAL; any other basis is kept apart with an @basis suffix.""" + key = f"{feed}|{venue}|{interval or ''}" + return key if basis == G6_D3_BASIS else f"{key}@{basis}" + + def _g6_key(product) -> str: - return f"{product.feed.value}|{product.venue}|{product.interval or ''}" + requirement = getattr(product, "requirement", None) + _policy, basis = g6_policy(product.feed.value, product.interval, + grade=getattr(requirement, "consumer_grade", None), + requirement_max_ms=getattr(requirement, "max_freshness_ms", None)) + return g6_row_key(product.feed.value, product.venue, product.interval, basis) def _g6_snapshot_observation(product, requirement, view) -> dict[str, object]: quality = view.quality policy = getattr(requirement, "effective_event_recency_policy", None) - policy_ms = g6_policy_ms(product.feed.value, product.interval, getattr(requirement, "max_freshness_ms", None)) + policy_ms, basis = g6_policy(product.feed.value, product.interval, + grade=getattr(requirement, "consumer_grade", None), + requirement_max_ms=getattr(requirement, "max_freshness_ms", None)) verdict, reasons = g6_classify( feed=product.feed.value, freshness_ms=quality.freshness_ms, eligible=bool(quality.execution_eligible), state=quality.state, gap_open=bool(quality.gap_open), complete=bool(quality.complete), @@ -1846,34 +1922,39 @@ def _g6_snapshot_observation(product, requirement, view) -> dict[str, object]: max_session_liveness_ms=getattr(requirement, "max_session_liveness_ms", None), observe=getattr(policy, "value", policy) == "OBSERVE", on_change="DELIVERY_ON_CHANGE" in (quality.flags or []), policy_ms=policy_ms) - return {"key": _g6_key(product), "age_ms": quality.freshness_ms, "policy_ms": policy_ms, "class": verdict, - "reasons": reasons, "state": quality.state, "recency": quality.event_recency_state} + return {"key": g6_row_key(product.feed.value, product.venue, product.interval, basis), "age_ms": + quality.freshness_ms, "policy_ms": policy_ms, "policy_basis": basis, "class": verdict, "reasons": reasons, + "state": quality.state, "recency": quality.event_recency_state} -def _g6_reference_observations(references, response, *, now_ns: int) -> list[dict[str, object]]: +def _g6_reference_observations(references, response, *, sent_ns: int, now_ns: int) -> list[dict[str, object]]: from qdl.certification.reference_l2_acceptance import _quiet_execution_mark_index_evidence out = [] for reference, item in zip(references, response.results): product = reference - key = f"{product.requirement.feed.value}|{product.venue}|" - policy_ms = g6_policy_ms(product.requirement.feed.value, None, product.sdk_requirement.max_freshness_ms) + feed = product.requirement.feed.value + grade = product.requirement.consumer_grade + policy_ms, basis = g6_policy(feed, None, grade=grade, + requirement_max_ms=product.sdk_requirement.max_freshness_ms) + key = g6_row_key(feed, product.venue, None, basis) data = getattr(item, "data", None) observations = getattr(data, "observations", None) or [] if getattr(item, "status", None) != "OK" or not observations: code = getattr(getattr(item, "problem", None), "code", None) or "NO_OBSERVATION" - out.append({"key": key, "age_ms": None, "policy_ms": policy_ms, "class": "NOT_ELIGIBLE", - "reasons": [str(code)[:64]]}) + out.append({"key": key, "age_ms": None, "policy_ms": policy_ms, "policy_basis": basis, + "class": "NOT_ELIGIBLE", "reasons": [str(code)[:64]]}) continue - age_ms = max(0, (now_ns - max(o.observed_at_ns for o in observations)) // 1_000_000) + ages = g6_reference_age(grade=grade, observations=[(o.observed_at_ns, dict(o.labels or {})) + for o in observations], + sent_ns=sent_ns, receipt_ns=now_ns) try: quiet = _quiet_execution_mark_index_evidence(product, item, observed_at_ns=now_ns) is not None except Exception: quiet = False - beyond = policy_ms is not None and age_ms > policy_ms - verdict = "UNSAFE" if beyond and not quiet else "ELIGIBLE_QUIET" if beyond else "ELIGIBLE" - out.append({"key": key, "age_ms": age_ms, "policy_ms": policy_ms, "class": verdict, - "reasons": ["STALE_SERVED_WITHOUT_QUIET_PROOF"] if verdict == "UNSAFE" else []}) + verdict, reasons = g6_reference_classify(age_ms=ages["age_ms"], policy_ms=policy_ms, quiet_proven=quiet) + out.append({"key": key, **ages, "policy_ms": policy_ms, "policy_basis": basis, "class": verdict, + "reasons": reasons}) return out @@ -1883,7 +1964,9 @@ def __init__(self, reservoir: int = _G6_RESERVOIR) -> None: self.rows: dict[str, dict[str, object]] = {} def _row(self, key: str) -> dict[str, object]: + basis = key.split("@", 1)[1] if "@" in key else G6_D3_BASIS return self.rows.setdefault(key, {"n": 0, "seen_ages": 0, "ages": [], "max_age_ms": None, "policy_ms": None, + "policy_basis": basis, "age_basis": None, "max_client_age_ms": None, "classes": Counter(), "reasons": Counter(), "refusals": Counter(), "windows": Counter(), "unsafe_examples": []}) @@ -1892,6 +1975,13 @@ def view(self, observation: dict[str, object], window: str) -> None: row["n"] += 1 row["windows"][window] += 1 row["policy_ms"] = observation.get("policy_ms") + row["policy_basis"] = observation.get("policy_basis") or row["policy_basis"] + if observation.get("age_basis"): + row["age_basis"] = observation["age_basis"] + client_age = observation.get("client_age_ms") + if isinstance(client_age, int): + row["max_client_age_ms"] = (client_age if row["max_client_age_ms"] is None + else max(row["max_client_age_ms"], client_age)) row["classes"][observation["class"]] += 1 for reason in observation.get("reasons") or []: row["reasons"][reason] += 1 @@ -1906,7 +1996,8 @@ def view(self, observation: dict[str, object], window: str) -> None: if slot < self.reservoir: row["ages"][slot] = age if observation["class"] == "UNSAFE" and len(row["unsafe_examples"]) < 8: - row["unsafe_examples"].append({k: observation.get(k) for k in ("age_ms", "reasons", "state", "recency")} + row["unsafe_examples"].append({k: observation.get(k) for k in ("age_ms", "client_age_ms", "reasons", + "state", "recency")} | {"window": window, "at_ns": time.time_ns()}) def refusal(self, key: str, code: str, window: str) -> None: @@ -1926,13 +2017,16 @@ def merge_g6(blocks: list[dict[str, object]]) -> dict[str, object]: for block in blocks: for key, row in (block or {}).items(): out = merged.setdefault(key, {"n": 0, "ages": [], "max_age_ms": None, "policy_ms": row.get("policy_ms"), + "policy_basis": row.get("policy_basis"), "age_basis": row.get("age_basis"), + "max_client_age_ms": None, "classes": Counter(), "reasons": Counter(), "refusals": Counter(), "windows": Counter(), "unsafe_examples": []}) out["n"] += row["n"] out["ages"].extend(row["ages"]) - if row.get("max_age_ms") is not None: - out["max_age_ms"] = (row["max_age_ms"] if out["max_age_ms"] is None - else max(out["max_age_ms"], row["max_age_ms"])) + out["age_basis"] = out["age_basis"] or row.get("age_basis") + for name in ("max_age_ms", "max_client_age_ms"): + if row.get(name) is not None: + out[name] = row[name] if out[name] is None else max(out[name], row[name]) for name in ("classes", "reasons", "refusals", "windows"): out[name].update(row[name]) out["unsafe_examples"].extend(row["unsafe_examples"][:8]) @@ -2026,9 +2120,10 @@ async def _target_read(client, operation: str, products) -> object: if operation == "REFERENCE_BATCH": references = [_reference_product(product, now_ns=time.time_ns()) for product in products] + sent_ns = time.time_ns() response = await client.reference_batch([item.sdk_requirement for item in references], require_all=True) try: - _G6_VIEW.set(_g6_reference_observations(references, response, now_ns=time.time_ns())) + _G6_VIEW.set(_g6_reference_observations(references, response, sent_ns=sent_ns, now_ns=time.time_ns())) except Exception: _G6_VIEW.set(None) # diagnostics never change the read result if response.partial or len(response.results) != len(references): diff --git a/tests/test_p184_g6_view_quality.py b/tests/test_p184_g6_view_quality.py index 39105815..4c785486 100644 --- a/tests/test_p184_g6_view_quality.py +++ b/tests/test_p184_g6_view_quality.py @@ -1,8 +1,10 @@ """P18.4 G6 instrumentation of the stage-100 driver: per-binding view age and typed eligibility (no network). -Synthetic views only (test provenance). Pins: the policy table, the UNSAFE rule (eligible although the policy -rejects it), quiet-continuity exceptions, bounded per-worker evidence, merging, and that the poll loop records a view -the client later refused exactly once. +Synthetic views only (test provenance). Pins: the policy of the requirement a view was read with (D3 table only for an +EXECUTION requirement, as min with it; any other grade its own requirement), the UNSAFE rule (eligible although the +policy rejects it), quiet-continuity exceptions, the MARK reference age basis, bounded per-worker evidence, merging, +that the poll loop records a view the client later refused exactly once, and parity of the pure functions with their +verbatim copy in the TS execution probe (scripts/p184_g6_ts_execution_probe.py). """ from __future__ import annotations @@ -20,6 +22,10 @@ _DRIVER = importlib.util.module_from_spec(_SPEC) sys.modules[_SPEC.name] = _DRIVER _SPEC.loader.exec_module(_DRIVER) +_PROBE_SPEC = importlib.util.spec_from_file_location("p184_g6_ts_probe", _ROOT / "scripts/p184_g6_ts_execution_probe.py") +_PROBE = importlib.util.module_from_spec(_PROBE_SPEC) +sys.modules[_PROBE_SPEC.name] = _PROBE +_PROBE_SPEC.loader.exec_module(_PROBE) def classify(**overrides): @@ -31,12 +37,32 @@ def classify(**overrides): class G6ClassifyTests(unittest.TestCase): - def test_policy_table(self): - self.assertEqual(_DRIVER.g6_policy_ms("TRADE", None, 9), 3000) - self.assertEqual(_DRIVER.g6_policy_ms("BOOK_SNAPSHOT", None, 9), 60000) - self.assertEqual(_DRIVER.g6_policy_ms("BAR", "1m", 9), 180000) - self.assertEqual(_DRIVER.g6_policy_ms("BAR", "5m", 600000), 600000) # own sealed requirement - self.assertEqual(_DRIVER.g6_policy_ms("FUNDING_RATE", None, None), None) + def test_policy_is_the_requirement_it_was_read_with(self): + policy = _DRIVER.g6_policy + # EXECUTION: the frozen D3 table, as min with the requirement (stricter consumer intent wins). + self.assertEqual(policy("TRADE", None, grade="EXECUTION", requirement_max_ms=3000), (3000, "D3_EXECUTION")) + self.assertEqual(policy("QUOTE", None, grade="EXECUTION", requirement_max_ms=15000), (2000, "D3_EXECUTION")) + self.assertEqual(policy("TRADE", None, grade="EXECUTION", requirement_max_ms=900), (900, "D3_EXECUTION")) + self.assertEqual(policy("BOOK_SNAPSHOT", None, grade="EXECUTION", requirement_max_ms=None), + (60000, "D3_EXECUTION")) + self.assertEqual(policy("BAR", "1m", grade="EXECUTION", requirement_max_ms=None), (180000, "D3_EXECUTION")) + self.assertEqual(policy("BAR", "5m", grade="EXECUTION", requirement_max_ms=600000), (600000, "D3_EXECUTION")) + # Any other grade: only its own requirement; the D3 table never applies. + self.assertEqual(policy("QUOTE", None, grade="ALPHA", requirement_max_ms=5000), (5000, "REQUIREMENT_ALPHA")) + self.assertEqual(policy("MARK_INDEX_PRICE", None, grade=_Feed("ALPHA"), requirement_max_ms=60000), + (60000, "REQUIREMENT_ALPHA")) + self.assertEqual(policy("FUNDING_RATE", None, grade=None, requirement_max_ms=None), + (None, "REQUIREMENT_UNKNOWN")) + + def test_alpha_quote_at_3000_ms_is_eligible_not_unsafe(self): + policy_ms, basis = _DRIVER.g6_policy("QUOTE", None, grade="ALPHA", requirement_max_ms=5000) + self.assertEqual(classify(freshness_ms=3000, policy_ms=policy_ms), ("ELIGIBLE", [])) + self.assertEqual(basis, "REQUIREMENT_ALPHA") + + def test_execution_quote_at_2500_ms_without_quiet_proof_is_unsafe(self): + policy_ms, basis = _DRIVER.g6_policy("QUOTE", None, grade="EXECUTION", requirement_max_ms=2000) + self.assertEqual(classify(freshness_ms=2500, policy_ms=policy_ms), ("UNSAFE", ["AGE_BEYOND_POLICY"])) + self.assertEqual(basis, "D3_EXECUTION") def test_fresh_eligible_and_not_eligible(self): self.assertEqual(classify(), ("ELIGIBLE", [])) @@ -83,6 +109,13 @@ class _Feed: value: str +@dataclass(frozen=True) +class _Requirement: + feed: _Feed = _Feed("QUOTE") + consumer_grade: _Feed = _Feed("ALPHA") + max_freshness_ms: int | None = 5000 + + @dataclass(frozen=True) class _Product: venue: str = "OKX" @@ -92,6 +125,145 @@ class _Product: consumer_id: str = "alpha.okx.paper.stable" delivery: _Feed = _Feed("DURABLE") source_policy_id: str = "crypto_primary_v2" + requirement: _Requirement = _Requirement() + + +@dataclass(frozen=True) +class _SdkReference: + max_freshness_ms: int | None = 60000 + + +@dataclass(frozen=True) +class _Reference: + venue: str = "BINANCE" + requirement: _Requirement = _Requirement(feed=_Feed("MARK_INDEX_PRICE"), max_freshness_ms=60000) + sdk_requirement: _SdkReference = _SdkReference() + + +@dataclass(frozen=True) +class _Observation: + observed_at_ns: int + labels: dict + + +@dataclass(frozen=True) +class _Data: + observations: list + + +@dataclass(frozen=True) +class _Item: + data: _Data + status: str = "OK" + + +@dataclass(frozen=True) +class _Response: + results: list + + +MS = 1_000_000 + + +class G6ReferenceTests(unittest.TestCase): + def _mark(self, reference, *, observed_ns, labels, sent_ns, now_ns): + response = _Response([_Item(_Data([_Observation(observed_ns, labels)]))]) + (row,) = _DRIVER._g6_reference_observations([reference], response, sent_ns=sent_ns, now_ns=now_ns) + return row + + def test_alpha_mark_at_2300_ms_is_not_unsafe_and_is_aged_from_request_sent(self): + sent = 10_000 * MS + labels = {"freshness_basis": "PROVIDER_CONFIRMATION", "provider_confirmation_ns": str(sent - 2300 * MS)} + row = self._mark(_Reference(), observed_ns=sent - 2600 * MS, labels=labels, sent_ns=sent, + now_ns=sent + 400 * MS) + self.assertEqual((row["class"], row["policy_ms"], row["policy_basis"]), ("ELIGIBLE", 60000, + "REQUIREMENT_ALPHA")) + self.assertEqual((row["age_ms"], row["client_age_ms"]), (2300, 2700)) + self.assertEqual(row["age_basis"], "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR") + self.assertEqual(row["key"], "MARK_INDEX_PRICE|BINANCE|@REQUIREMENT_ALPHA") + + def test_execution_mark_keeps_its_use_time_basis_and_d3_policy(self): + execution = _Reference(requirement=_Requirement(feed=_Feed("MARK_INDEX_PRICE"), consumer_grade=_Feed( + "EXECUTION"), max_freshness_ms=2000), sdk_requirement=_SdkReference(2000)) + sent = 10_000 * MS + row = self._mark(execution, observed_ns=sent - 2300 * MS, labels={}, sent_ns=sent, now_ns=sent + 100 * MS) + self.assertEqual((row["class"], row["reasons"]), ("UNSAFE", ["STALE_SERVED_WITHOUT_QUIET_PROOF"])) + self.assertEqual((row["age_ms"], row["policy_ms"], row["key"]), (2400, 2000, "MARK_INDEX_PRICE|BINANCE|")) + + def test_anchor_follows_the_server_rule(self): + anchor = _DRIVER.g6_reference_anchor_ns + self.assertEqual(anchor([(5, {}), (7, {})]), 5) + self.assertEqual(anchor([(5, {"freshness_basis": "PROVIDER_CONFIRMATION", "provider_confirmation_ns": "9"})]), + 9) + self.assertIsNone(anchor([(5, {"freshness_basis": "PROVIDER_CONFIRMATION"})])) + self.assertIsNone(anchor([(5, {"freshness_basis": "SOURCE_EVENT"}), + (6, {"freshness_basis": "PROVIDER_CONFIRMATION", "provider_confirmation_ns": "9"})])) + self.assertIsNone(anchor([])) + + +class G6ProbeParityTests(unittest.TestCase): + """The TS execution probe carries verbatim copies of the pure functions; both must agree on every case.""" + + def test_policy_parity(self): + cases = [(feed, interval, grade, req) for feed, interval in (("TRADE", None), ("QUOTE", None), + ("MARK_INDEX_PRICE", None), ("BOOK_DELTA", None), + ("BOOK_SNAPSHOT", None), ("BAR", "1m"), + ("BAR", "5m"), ("FUNDING_RATE", None)) + for grade in ("EXECUTION", "ALPHA", None, _Feed("EXECUTION")) for req in (None, 900, 2000, 60000)] + for feed, interval, grade, req in cases: + self.assertEqual(_DRIVER.g6_policy(feed, interval, grade=grade, requirement_max_ms=req), + _PROBE.g6_policy(feed, interval, grade=grade, requirement_max_ms=req)) + self.assertEqual(_DRIVER._G6_POLICY_MS, _PROBE._G6_POLICY_MS) + self.assertEqual(_DRIVER.G6_D3_BASIS, _PROBE.G6_D3_BASIS) + + def test_classify_parity(self): + import itertools + axes = dict(feed=("QUOTE", "TRADE", "BOOK_DELTA", "MARK_INDEX_PRICE"), freshness_ms=(None, 100, 2500, 9000), + eligible=(True, False), state=(None, "LIVE", "GAPPED"), gap_open=(False, True), + complete=(True, False), event_recency_state=("LIVE", "STALE", None), + provider_session_state=("LIVE", "DISCONNECTED", "NOT_APPLICABLE", None), + session_liveness_ms=(None, 500, 5000), max_session_liveness_ms=(None, 2000), + observe=(True, False), on_change=(True, False), policy_ms=(None, 2000, 3000)) + names = list(axes) + count = 0 + for values in itertools.product(*axes.values()): + case = dict(zip(names, values)) + self.assertEqual(_DRIVER.g6_classify(**case), _PROBE.g6_classify(**case), case) + count += 1 + self.assertGreater(count, 50000) + + def test_reference_parity(self): + label_sets = [{}, {"freshness_basis": "PROVIDER_CONFIRMATION", "provider_confirmation_ns": str(9_500 * MS)}, + {"freshness_basis": "PROVIDER_CONFIRMATION"}, {"freshness_basis": "SOURCE_EVENT"}] + for grade in ("EXECUTION", "ALPHA", None): + for labels in label_sets: + for observed in (1, 7_000 * MS, 9_000 * MS): + observations = [(observed, labels)] + self.assertEqual(_DRIVER.g6_reference_anchor_ns(observations), + _PROBE.g6_reference_anchor_ns(observations)) + kwargs = dict(grade=grade, observations=observations, sent_ns=10_000 * MS, + receipt_ns=10_300 * MS) + self.assertEqual(_DRIVER.g6_reference_age(**kwargs), _PROBE.g6_reference_age(**kwargs)) + for age in (None, 100, 2300, 70000): + for policy in (None, 2000, 60000): + for quiet in (True, False): + kwargs = dict(age_ms=age, policy_ms=policy, quiet_proven=quiet) + self.assertEqual(_DRIVER.g6_reference_classify(**kwargs), _PROBE.g6_reference_classify(**kwargs)) + + def test_probe_rows_have_the_campaign_shape(self): + rows = _PROBE.Rows() + rows.view("QUOTE|OKX|", {"age_ms": 300, "policy_ms": 2000, "policy_basis": "D3_EXECUTION", "class": "ELIGIBLE", + "reasons": []}) + rows.view("QUOTE|OKX|", {"age_ms": 2500, "policy_ms": 2000, "policy_basis": "D3_EXECUTION", "class": "UNSAFE", + "reasons": ["AGE_BEYOND_POLICY"]}) + rows.refusal("TRADE|OKX|", "DATA_STALE") + summary = rows.summary() + quote = summary["QUOTE|OKX|"] + self.assertEqual((quote["n"], quote["unsafe"], quote["max_age_ms"], quote["policy_basis"]), + (2, 1, 2500, "D3_EXECUTION")) + self.assertEqual(quote["classes"], {"ELIGIBLE": 1, "UNSAFE": 1}) + self.assertEqual(quote["age_ms"]["n"], 2) + self.assertEqual(summary["TRADE|OKX|"]["refusals"], {"DATA_STALE": 1}) class G6PollLoopTests(unittest.TestCase): @@ -121,8 +293,9 @@ class Rejected(Exception): async def read(client, operation, products): raise Rejected("x") - row = self._run(read)["QUOTE|OKX|"] - self.assertEqual((row["n"], row["refusals"]), (0, {"RATE_LIMITED": 4})) + row = self._run(read)["QUOTE|OKX|@REQUIREMENT_ALPHA"] + self.assertEqual((row["n"], row["refusals"], row["policy_basis"]), (0, {"RATE_LIMITED": 4}, + "REQUIREMENT_ALPHA")) def test_success_records_the_view(self): async def read(client, operation, products): diff --git a/tests/test_p184_gate_campaign.py b/tests/test_p184_gate_campaign.py index 710a1eb6..9bed2a82 100644 --- a/tests/test_p184_gate_campaign.py +++ b/tests/test_p184_gate_campaign.py @@ -368,12 +368,71 @@ def test_g4_rows_need_baseline_coverage(self): "observer_receipt_wall_ns": 1 * S + 50_000_000}]) self.assertEqual(rows, {"trade|OKX_INVERSE|": {"n": 1, "p95": 50.0, "max": 50.0}}) - def test_g6_unsafe_fails_missing_binding_not_measured(self): - full = {key: {"n": 5, "unsafe": 0} for key in campaign.G6_REQUIRED} - self.assertEqual(campaign.score_g6(full)[0], "PASS") - self.assertEqual(campaign.score_g6({**full, "TRADE|OKX|": {"n": 5, "unsafe": 1}})[0], "FAIL") - partial = {k: v for k, v in full.items() if not k.startswith("BOOK_DELTA")} + D3 = "D3_EXECUTION" + + def _d3(self, **extra): + return {key: {"n": 5, "unsafe": 0, "policy_basis": self.D3, **extra} for key in campaign.G6_REQUIRED} + + def _alpha(self, unsafe=0): + return {f"{key}@REQUIREMENT_ALPHA": {"n": 50, "unsafe": unsafe, "policy_basis": "REQUIREMENT_ALPHA"} + for key in campaign.G6_REQUIRED} + + def test_g6_only_alpha_rows_is_not_measured(self): + state, _, why = campaign.score_g6(self._alpha()) + self.assertEqual(state, "NOT_MEASURED") + self.assertIn("no EXECUTION (D3) view", why[0]) + self.assertIn("REQUIREMENT_ALPHA", why[0]) + + def test_g6_clean_d3_rows_pass(self): + self.assertEqual(campaign.score_g6({**self._d3(), **self._alpha()})[0], "PASS") + # Probe rows (keyed apart with the probe suffix) count for their base binding. + probe = campaign.g6_ts_probe_rows({"rows": {key: {"n": 3, "unsafe": 0, "policy_basis": self.D3} + for key in campaign.G6_REQUIRED}}) + self.assertEqual(campaign.score_g6({**probe, **self._alpha()})[0], "PASS") + + def test_g6_d3_unsafe_fails(self): + quality = {**self._d3(), "TRADE|OKX|": {"n": 5, "unsafe": 1, "policy_basis": self.D3}} + state, _, why = campaign.score_g6(quality) + self.assertEqual(state, "FAIL") + self.assertIn("D3 execution policy", why[0]) + + def test_g6_alpha_unsafe_under_its_own_requirement_fails(self): + state, _, why = campaign.score_g6({**self._d3(), **self._alpha(unsafe=1)}) + self.assertEqual(state, "FAIL") + self.assertIn("own requirement", why[0]) + self.assertEqual(campaign.score_g6(self._alpha(unsafe=1))[0], "FAIL") + + def test_g6_missing_d3_binding_and_legacy_rows(self): + partial = {k: v for k, v in self._d3().items() if not k.startswith("MARK_INDEX_PRICE|OKX")} self.assertEqual(campaign.score_g6(partial)[0], "NOT_MEASURED") + # Rows without policy_basis (older receipts) are reported, not scored: never PASS, never FAIL by themselves. + legacy = {key: {"n": 5, "unsafe": 3} for key in campaign.G6_REQUIRED} + state, _, why = campaign.score_g6(legacy) + self.assertEqual(state, "NOT_MEASURED") + self.assertTrue(any("without policy_basis" in item for item in why)) + + def test_g6_probe_window_and_last_line(self): + load = [1000.0, 2800.0] + self.assertEqual(campaign.g6_ts_probe_coverage({"kind": "final", "window_unix_s": [990.0, 2810.0]}, load), []) + self.assertTrue(campaign.g6_ts_probe_coverage({"kind": "final", "window_unix_s": [990.0, 2500.0]}, load)) + self.assertTrue(campaign.g6_ts_probe_coverage({"kind": "partial", "window_unix_s": [990.0, 2810.0]}, load)) + self.assertTrue(campaign.g6_ts_probe_coverage(None, load)) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "probe.jsonl" + path.write_text('{"kind": "start", "pid": 7}\n{"kind": "partial", "reads": 1}\n{"kind": "final", ' + '"reads": 2}\n{"kind": "fin') + self.assertEqual(campaign.read_probe_lines(path), {"kind": "final", "reads": 2}) + + def test_g6_probe_is_planned_as_a_read_only_docker_exec_into_the_writer(self): + import argparse + args = argparse.Namespace(state_tools="/tools", load_timeout=2400, budget="b.json", writer="w1", g11=False, + g11_wait=900) + plan = {"binding": "/b.json", "market_redis": {"container": "r", "db": 1}, + "heartbeat_redis": {"container": "h", "db": 0}} + commands = campaign.command_plan(args, plan, Path("/c"), 0.0) + self.assertEqual(commands["g6_ts"][:7], ["docker", "exec", "-i", "w1", "python", "-B", "-"]) + self.assertEqual(commands["g6_ts"][commands["g6_ts"].index("--duration") + 1], "2460") + self.assertTrue(commands["g6_ts_stdin"].endswith("scripts/p184_g6_ts_execution_probe.py")) def test_g10_needs_trace_for_duplicates(self): g10 = {"trade_gap": 0, "trade_replay_expired": 0, "trade_overflow": 0, "log_polls": 10, "log_unreadable": 0} @@ -396,7 +455,12 @@ def test_scorecard_never_passes_outside_the_protocol(self): (d / "stage100" / "receipt.json").write_text(json.dumps({ "stage": 100, "final": True, "load_window_unix_s": [1000.0, 2000.0], # 1000 s < 1800 s "fault_windows": {"BURST": [720, 730]}, "g6_view_quality": { - key: {"n": 5, "unsafe": 0} for key in campaign.G6_REQUIRED}})) + f"{key}@REQUIREMENT_ALPHA": {"n": 5, "unsafe": 0, "policy_basis": "REQUIREMENT_ALPHA"} + for key in campaign.G6_REQUIRED}})) + (d / "g6-ts").mkdir() + (d / "g6-ts" / "summary.json").write_text(json.dumps({ + "kind": "final", "window_unix_s": [990.0, 2010.0], "rows": { + key: {"n": 5, "unsafe": 0, "policy_basis": "D3_EXECUTION"} for key in campaign.G6_REQUIRED}})) (d / "stage100" / "gates.json").write_text(json.dumps({"status": "PASS", "gates": [ {"gate": "latency:BAR_LATEST:OKX", "status": "PASS"}]})) card = campaign.build_scorecard(d) From dadb9c837e7a1e0a378da44383107d3cd554a9ac Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 18:21:48 +0000 Subject: [PATCH 132/147] fix(stream): charge a shared decoded record once against the replica budget The live budget was charged the full decoded weight per subscriber although dispatch shares one Arc; a short stall exhausted the 32 MiB budget and 84 streams with empty queues were closed at once (2026-10-10 17:22:48Z). The default per-subscription cap becomes half of the replica total. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 17 ++ contracts/v2/kn-v220-contracts.md | 10 + rust/qdl-stream-gateway/src/hub.rs | 249 +++++++++++++++++++- rust/qdl-stream-gateway/src/main.rs | 18 +- rust/qdl-stream-gateway/src/memory.rs | 27 ++- rust/qdl-stream-gateway/src/service.rs | 49 +++- rust/qdl-stream-gateway/src/subscription.rs | 53 ++++- 7 files changed, 407 insertions(+), 16 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 59c2d4a1..d82f4ae5 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72111,3 +72111,20 @@ no build, CI or subagent job running. - MARK EXECUTION age stays measured from the source event: stricter than the server's provider-confirmation anchor, kept so the frozen UNSAFE definition is unchanged. - 20 s live check: 197 reads, 0 unsafe. Tests: 138 OK (unittest in a disposable rc8 container). +- Stream gateway budget defect, behind the 17:22:48Z G2 QUOTE/BOOK max: + - Incident: 84 streams on stream_kn_1 (all feeds, both venues, 24 TS writer + 60 alpha) closed with + BACKPRESSURE in 37 ms, each with an EMPTY own queue. + - Cause: the replica-wide 32 MiB ByteBudget was charged the full decoded weight per subscriber, while hub + dispatch shares one Arc (counted ~fanout x, ~4 here). A 1-2 s delivery stall exhausted it. + Separately, the per-subscription cap equalled the replica total. + - Fix (rust/qdl-stream-gateway; no endpoint/proto/manifest change): + - the decoded record is charged once at dispatch, released with the last Arc; + - each queue entry adds 512 B, and each transport copy its full weight (handoff_charge waits up to + slow_consumer_after before typed backpressure); + - a refused dispatch charge falls back to the old per-subscriber charge; + - default QDL_KN_QUEUE_BYTES_PER_SUBSCRIPTION = total/2 (16 MiB). Not /8: the TS writer legitimately held + 15.85 MB of BOOK_DELTA in one subscription at 14:28Z; + - new metric hub_live_budget_refused. + - Tests: fmt, clippy -D warnings, cargo test 49+1+37 pass (qdl-rust-builder:r134-test). Status: tested locally. +- Found alongside: stream_kn_2 served 0 of 244 streams. The SDK GrpcStreamTransport always starts at target 0; + 5c81c25 randomised only the REST Query transport. Fixed in the SDK (next slice). diff --git a/contracts/v2/kn-v220-contracts.md b/contracts/v2/kn-v220-contracts.md index c3268005..c64c7be6 100644 --- a/contracts/v2/kn-v220-contracts.md +++ b/contracts/v2/kn-v220-contracts.md @@ -229,6 +229,16 @@ away. The earlier Python stream's coalescing of BOOK_SNAPSHOT is not ported. - **Every held record is inside one replica byte budget**, charged once (`raw + 8 x payload`, an empirical decoded-size allowance) from the moment it enters a queue, replay channel or ring replay until the transport takes it. + A live record is decoded once per replica and shared by every subscriber + of its key, so its decoded copy is charged once at dispatch (released when + the last holder drops it); each queue entry adds a fixed 512-byte overhead + and each transport copy its own full weight. If the dispatch-time charge + is refused, every subscriber charges the full weight itself (lossless + subscribers then end with typed backpressure, as before). One subscription + may reference at most `QDL_KN_QUEUE_BYTES_PER_SUBSCRIPTION` bytes (full + weight); unset, it defaults to `QDL_KN_QUEUE_BYTES_TOTAL / 2` (16 MiB of the + default 32 MiB), so one slow subscriber cannot hold the whole live budget + and overflow the others. Concurrent Replay RPCs are admitted per replica and per consumer (manifest `max_streams`); refusal is `RESOURCE_EXHAUSTED RATE_LIMITED`. The factor eight exceeds the largest measured ratio (7.28) in the retained diff --git a/rust/qdl-stream-gateway/src/hub.rs b/rust/qdl-stream-gateway/src/hub.rs index 7d96f61b..74520d6a 100644 --- a/rust/qdl-stream-gateway/src/hub.rs +++ b/rust/qdl-stream-gateway/src/hub.rs @@ -25,11 +25,11 @@ //! of different (or the same) products no longer refuse each other. use crate::generated::marketdata_v2::EventEnvelope; -use crate::subscription::Subscription; +use crate::subscription::{ByteBudget, Charge, Subscription}; use prost::Message as _; use std::collections::{HashMap, VecDeque}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, MutexGuard, TryLockError}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock, TryLockError}; use std::time::{Duration, Instant}; /// Accounting overhead per queued/ringed record beyond key and payload. @@ -66,6 +66,13 @@ pub struct HotRecord { pub struct LiveRecord { pub raw: SharedRecord, pub envelope: EventEnvelope, + /// The live budget charge for this one decoded copy, taken once at + /// dispatch and released exactly once, when the last `Arc` (a queue + /// entry or a subscription task) drops. Subscribers then charge the + /// replica budget only their queue entry overhead. `None` for replay/ring decodes (charged per record to the + /// replay budget by their owner) and when the dispatch-time charge was + /// refused: each subscriber then charges the full weight itself. + pub shared_charge: Option, } /// Empirical decoded-size allowance per payload byte: measured @@ -87,10 +94,25 @@ impl LiveRecord { live_weight(&self.raw) } + /// Decode without a shared charge (the caller accounts for it). pub fn decode(raw: SharedRecord) -> Result, String> { + Self::decode_charged(raw, None) + } + + /// Decode once and charge the decoded copy to `budget` once, for every + /// holder together. A refused charge leaves `shared_charge` empty. + pub fn decode_charged( + raw: SharedRecord, + budget: Option<&Arc>, + ) -> Result, String> { let envelope = EventEnvelope::decode(raw.payload.as_slice()) .map_err(|error| format!("canonical record failed to decode: {error}"))?; - Ok(Arc::new(Self { raw, envelope })) + let shared_charge = budget.and_then(|budget| budget.charge(live_weight(&raw))); + Ok(Arc::new(Self { + raw, + envelope, + shared_charge, + })) } } @@ -175,6 +197,9 @@ pub struct HubMetrics { pub duplicates: AtomicU64, pub decode_failures: AtomicU64, pub offers: AtomicU64, + /// Dispatches whose shared live-budget charge was refused (the replica + /// live budget is really full); their subscribers charged in full. + pub live_budget_refused: AtomicU64, pub ring_evictions: AtomicU64, pub reader_errors: AtomicU64, } @@ -261,6 +286,9 @@ pub struct Hub { /// Records produced before this replica started (the warm range, a /// restart backlog) are not live-path lag and stay out of the histogram. started_ms: i64, + /// The replica live budget a dispatched record is charged to once + /// ([`Hub::attach_live_budget`]); unset, subscribers charge in full. + live_budget: OnceLock>, } fn now_ns() -> i64 { @@ -303,9 +331,17 @@ impl Hub { .unwrap_or_else(Instant::now), key_home: Mutex::new(HashMap::new()), started_ms: now_ns() / 1_000_000, + live_budget: OnceLock::new(), } } + /// Charge each decoded live record to `budget` once, at dispatch, + /// instead of once per subscriber. The first attachment wins; returns + /// whether this one was attached. + pub fn attach_live_budget(&self, budget: Arc) -> bool { + self.live_budget.set(budget).is_ok() + } + pub fn partitions(&self) -> Vec { let mut ids: Vec = self.partitions.keys().copied().collect(); ids.sort_unstable(); @@ -403,8 +439,14 @@ impl Hub { let Some(subscribers) = state.subscribers.get(&record.key) else { return; }; - match LiveRecord::decode(record) { + let budget = self.live_budget.get(); + match LiveRecord::decode_charged(record, budget) { Ok(live) => { + if budget.is_some() && live.shared_charge.is_none() { + self.metrics + .live_budget_refused + .fetch_add(1, Ordering::Relaxed); + } let at = now_ns(); for subscriber in subscribers { self.metrics.offers.fetch_add(1, Ordering::Relaxed); @@ -1192,4 +1234,203 @@ mod tests { assert!(writer.join().unwrap() > 1); assert!(total > 0); } + + // ----------------------------------------------- live budget charge-once + + use crate::generated::query_v2 as query; + use crate::requirement::StreamRequirement; + use crate::subscription::{Next, QUEUE_ENTRY_OVERHEAD}; + + fn trade_requirement() -> StreamRequirement { + StreamRequirement::from_proto(&query::DataRequirement { + instrument_uid: "u".into(), + source_policy_id: "p".into(), + require_full_coverage: true, + require_final_bars: true, + feed_type: query::FeedType::Trade as i32, + grade: query::ConsumerGrade::Alpha as i32, + stale_policy_type: query::StalePolicy::Block as i32, + gap_policy_type: query::GapPolicy::Block as i32, + recovery_policy: query::RecoveryPolicy::SnapshotAndReplay as i32, + revision_policy: query::BarRevisionPolicy::Latest as i32, + ..Default::default() + }) + .expect("requirement") + } + + /// A lossless TRADE whose decoded weight dwarfs the queue entry overhead. + fn big_trade(key: &[u8], offset: i64) -> RawRecord { + use crate::generated::marketdata_v2::{event_envelope::Payload, Trade}; + RawRecord { + partition: 0, + offset, + key: key.to_vec(), + timestamp_ms: 1, + payload: EventEnvelope { + payload: Some(Payload::Trade(Trade { + native_trade_id: "x".repeat(4_000), + ..Default::default() + })), + ..Default::default() + } + .encode_to_vec(), + } + } + + fn big_weight() -> usize { + live_weight(&big_trade(b"fan", 0)) + } + + fn subscribe( + hub: &Hub, + key: &[u8], + id: u64, + depth: usize, + max_bytes: usize, + budget: &Arc, + ) -> Arc { + let subscription = Arc::new(Subscription::new( + id, + "c".into(), + trade_requirement(), + depth, + max_bytes, + budget.clone(), + -1, + )); + hub.register(0, key, subscription.clone(), -1).unwrap(); + subscription + } + + #[tokio::test] + async fn a_record_fanned_out_to_four_subscribers_is_charged_once() { + let weight = big_weight(); + assert!(weight > 8 * QUEUE_ENTRY_OVERHEAD, "{weight}"); + let budget = ByteBudget::new(1 << 20); + let hub = Hub::new(&[(0, 0)], HubConfig::default()); + assert!(hub.attach_live_budget(budget.clone())); + assert!(!hub.attach_live_budget(ByteBudget::new(1)), "first wins"); + let subscribers: Vec<_> = (0..4) + .map(|id| subscribe(&hub, b"fan", id, 16, 1 << 20, &budget)) + .collect(); + hub.dispatch(big_trade(b"fan", 0)); + assert_eq!(budget.used(), weight + 4 * QUEUE_ENTRY_OVERHEAD); + for subscriber in &subscribers { + // The per-subscription cap still counts the full weight. + assert_eq!(subscriber.queued(), (1, weight)); + } + assert_eq!(hub.metrics.live_budget_refused.load(Ordering::Relaxed), 0); + // Three subscribers hand their copy to the transport and finish. + for subscriber in &subscribers[..3] { + match subscriber.next().await { + Next::Record(record, _, charge) => { + assert_eq!(charge.bytes(), QUEUE_ENTRY_OVERHEAD); + assert!(record.shared_charge.is_some()); + } + other => panic!("{other:?}"), + } + } + assert_eq!(budget.used(), weight + QUEUE_ENTRY_OVERHEAD); + // The last holder keeps the shared charge until it drops the record. + let (record, charge) = match subscribers[3].next().await { + Next::Record(record, _, charge) => (record, charge), + other => panic!("{other:?}"), + }; + drop(charge); + assert_eq!(budget.used(), weight); + drop(record); + assert_eq!(budget.used(), 0, "released exactly once"); + } + + #[tokio::test] + async fn a_budget_for_one_decoded_record_delivers_it_to_all_four() { + let weight = big_weight(); + let total = weight + 4 * QUEUE_ENTRY_OVERHEAD; + assert!(total < 2 * weight, "fits one decoded copy, not four"); + let budget = ByteBudget::new(total); + let hub = Hub::new(&[(0, 0)], HubConfig::default()); + hub.attach_live_budget(budget.clone()); + let subscribers: Vec<_> = (0..4) + .map(|id| subscribe(&hub, b"fan", id, 16, 1 << 20, &budget)) + .collect(); + hub.dispatch(big_trade(b"fan", 0)); + assert_eq!(budget.used(), total); + for subscriber in &subscribers { + assert!(matches!(subscriber.next().await, Next::Record(..))); + } + assert_eq!(budget.used(), 0); + + // Before: every subscriber charged the full weight; three of four + // overflowed with an empty queue (the 2026-10-10 17:22:48Z shape). + let budget = ByteBudget::new(total); + let hub = Hub::new(&[(0, 0)], HubConfig::default()); + let subscribers: Vec<_> = (0..4) + .map(|id| subscribe(&hub, b"fan", id, 16, 1 << 20, &budget)) + .collect(); + hub.dispatch(big_trade(b"fan", 0)); + let mut overflowed = 0; + for subscriber in &subscribers { + if matches!(subscriber.next().await, Next::Overflow) { + overflowed += 1; + assert_eq!(subscriber.queued(), (0, 0)); + } + } + assert_eq!(overflowed, 3); + } + + #[tokio::test] + async fn a_refused_dispatch_charge_falls_back_to_full_per_subscriber_charges() { + // Real exhaustion: the decoded copy itself does not fit. Lossless + // subscribers overflow typed, as before; nothing leaks. + let budget = ByteBudget::new(4 * QUEUE_ENTRY_OVERHEAD); + let hub = Hub::new(&[(0, 0)], HubConfig::default()); + hub.attach_live_budget(budget.clone()); + let subscribers: Vec<_> = (0..2) + .map(|id| subscribe(&hub, b"fan", id, 16, 1 << 20, &budget)) + .collect(); + hub.dispatch(big_trade(b"fan", 0)); + assert_eq!(hub.metrics.live_budget_refused.load(Ordering::Relaxed), 1); + for subscriber in &subscribers { + assert!(matches!(subscriber.next().await, Next::Overflow)); + } + assert_eq!(budget.used(), 0); + // A fitting fallback record carries its full weight per subscriber. + let budget = ByteBudget::new(4 * big_weight()); + let hub = Hub::new(&[(0, 0)], HubConfig::default()); + hub.attach_live_budget(ByteBudget::new(1)); + let subscriber = subscribe(&hub, b"fan", 0, 16, 1 << 20, &budget); + hub.dispatch(big_trade(b"fan", 0)); + assert_eq!(budget.used(), big_weight()); + subscriber.close(); + assert_eq!(budget.used(), 0); + } + + #[tokio::test] + async fn a_slow_lossless_subscriber_at_its_cap_cannot_overflow_an_idle_one() { + let weight = big_weight(); + let total = 16 * weight; + // `idle` is offered one record after `slow` filled up and stalled. + async fn run(total: usize, max_bytes: usize) -> (bool, bool) { + let budget = ByteBudget::new(total); + let hub = Hub::new(&[(0, 0)], HubConfig::default()); + hub.attach_live_budget(budget.clone()); + let slow = subscribe(&hub, b"slow", 0, 1_000, max_bytes, &budget); + let idle = subscribe(&hub, b"idle", 1, 1_000, max_bytes, &budget); + for offset in 0..64 { + hub.dispatch(big_trade(b"slow", offset)); + } + hub.dispatch(big_trade(b"idle", 64)); + assert!(budget.used() <= total); + let slow_overflowed = matches!(slow.next().await, Next::Overflow); + let idle_delivered = matches!(idle.next().await, Next::Record(..)); + (slow_overflowed, idle_delivered) + } + // A fair share (the new default, queue_total / 2): the slow + // subscriber overflows on its own cap; the idle one is untouched. + let share = crate::memory::default_queue_bytes_per_subscription(total as u64) as usize; + assert_eq!(run(total, share).await, (true, true)); + // A per-subscription cap equal to the whole budget (the old default) + // lets the slow subscriber take it all and overflow the idle one. + assert_eq!(run(total, total).await, (true, false)); + } } diff --git a/rust/qdl-stream-gateway/src/main.rs b/rust/qdl-stream-gateway/src/main.rs index 965c8e4f..13159b1c 100644 --- a/rust/qdl-stream-gateway/src/main.rs +++ b/rust/qdl-stream-gateway/src/main.rs @@ -14,7 +14,9 @@ use qdl_stream_gateway::bundle::Bundle; use qdl_stream_gateway::generated::marketdata_v2::EventEnvelope; use qdl_stream_gateway::generated::query_v2::market_data_stream_service_server::MarketDataStreamServiceServer; use qdl_stream_gateway::hub::{Hub, HubConfig}; -use qdl_stream_gateway::memory::{container_memory_limit, MemoryPlan}; +use qdl_stream_gateway::memory::{ + container_memory_limit, default_queue_bytes_per_subscription, MemoryPlan, +}; use qdl_stream_gateway::query_view::QueryReadView; use qdl_stream_gateway::reader::{latest_for_key, KafkaLogSource, KafkaRangeSource, KafkaSettings}; use qdl_stream_gateway::readview::{NotReadyReadView, ReadView}; @@ -199,7 +201,8 @@ fn process_usage() -> (u64, f64) { (rss, cpu) } -fn limits() -> Result { +fn limits(queue_total: u64) -> Result { + let per_subscription = default_queue_bytes_per_subscription(queue_total).to_string(); Ok(StreamLimits { cursor_ttl_seconds: parsed("QDL_KN_CURSOR_TTL_SECONDS", "3600")?, replay: ReplayLimits { @@ -210,7 +213,10 @@ fn limits() -> Result { }, catchup_deadline: Duration::from_millis(parsed("QDL_KN_CATCHUP_DEADLINE_MS", "10000")?), slow_consumer_after: Duration::from_millis(parsed("QDL_KN_SLOW_CONSUMER_MS", "10000")?), - queue_bytes_per_subscription: parsed("QDL_KN_QUEUE_BYTES_PER_SUBSCRIPTION", "33554432")?, + queue_bytes_per_subscription: parsed( + "QDL_KN_QUEUE_BYTES_PER_SUBSCRIPTION", + &per_subscription, + )?, max_subscriptions: parsed("QDL_KN_MAX_SUBSCRIPTIONS", "384")?, max_replay_rpcs: parsed("QDL_KN_MAX_REPLAY_RPCS", "32")?, replay_page_default: 1_000, @@ -224,11 +230,12 @@ async fn serve() -> Result<(), String> { &env("QDL_KN_QUOTA_PREFIX")?, )?; let readers: usize = parsed("QDL_KN_REPLAY_READERS", "4")?; - let limits = limits()?; + let queue_total: u64 = parsed("QDL_KN_QUEUE_BYTES_TOTAL", "33554432")?; + let limits = limits(queue_total)?; // Defaults fit a 256 MiB replica (K2-T08 sizing); checked, fail closed. let memory = MemoryPlan { ring_total: parsed("QDL_KN_RING_BYTES_TOTAL", "50331648")?, - queue_total: parsed("QDL_KN_QUEUE_BYTES_TOTAL", "33554432")?, + queue_total, replay_total: parsed("QDL_KN_REPLAY_BYTES_TOTAL", "33554432")?, replay_readers: readers as u64, streams: (limits.max_subscriptions + limits.max_replay_rpcs) as u64, @@ -335,6 +342,7 @@ async fn serve() -> Result<(), String> { "hub_duplicates": load(&hub.duplicates), "hub_decode_failures": load(&hub.decode_failures), "hub_offers": load(&hub.offers), + "hub_live_budget_refused": load(&hub.live_budget_refused), "hub_reader_errors": load(&hub.reader_errors), "ring_bytes": reporter.hub.ring_bytes(), "ring_evictions": load(&hub.ring_evictions), diff --git a/rust/qdl-stream-gateway/src/memory.rs b/rust/qdl-stream-gateway/src/memory.rs index 7d3d8f34..4c657ad3 100644 --- a/rust/qdl-stream-gateway/src/memory.rs +++ b/rust/qdl-stream-gateway/src/memory.rs @@ -8,7 +8,11 @@ //! - two byte budgets for everything holding decoded records (charged //! `raw + 8 x payload`, see `hub::DECODED_FACTOR`, until the transport takes //! the record): live subscriber queues, and replay (channels and ring-replay -//! references) - separate, so replay can never starve live delivery; +//! references) - separate, so replay can never starve live delivery. A +//! live record decoded once for N subscribers is charged once, plus a +//! small overhead per queue entry and a full charge per transport copy; +//! one subscription may reference at most a fair share of the live +//! budget ([`default_queue_bytes_per_subscription`]); //! - librdkafka's prefetch per consumer; //! - the transport of each admitted stream (Subscribe or Replay): tonic's //! encode buffer yields at 32 KiB and hyper holds one chunk until the peer's @@ -26,6 +30,20 @@ pub const REPLAY_QUEUE_KBYTES: u64 = 4_096; /// message (canonical book max measured 56,495 B, plus its resume token). pub const TRANSPORT_STREAM_BYTES: u64 = 96 * 1024; +/// The default per-subscription byte cap is this fraction of the live +/// budget, so one slow lossless subscriber cannot hold all of it and make +/// every other subscriber overflow (2026-10-10 17:22:48Z: 84 streams with +/// empty queues closed at once). Half, not less: the TS writer legitimately +/// held 15.85 MB of BOOK_DELTA in one subscription during a stall +/// (2026-10-10 14:28Z), so a tighter cap would close a healthy consumer. +pub const SUBSCRIPTION_SHARE_DIVISOR: u64 = 2; + +/// `QDL_KN_QUEUE_BYTES_PER_SUBSCRIPTION` when unset: a fair share of the +/// replica live budget (16 MiB of the default 32 MiB). +pub fn default_queue_bytes_per_subscription(queue_total: u64) -> u64 { + (queue_total / SUBSCRIPTION_SHARE_DIVISOR).max(1) +} + /// Where the container memory limit is read (cgroup v2). pub const CGROUP_MEMORY_MAX: &str = "/sys/fs/cgroup/memory.max"; @@ -170,6 +188,13 @@ mod tests { assert!(MemoryPlan { limit: None, ..old }.check().is_ok()); } + #[test] + fn one_subscription_defaults_to_a_fair_share_of_the_live_budget() { + assert_eq!(default_queue_bytes_per_subscription(32 * MIB), 16 * MIB); + assert!(default_queue_bytes_per_subscription(32 * MIB) < plan(None).queue_total); + assert_eq!(default_queue_bytes_per_subscription(0), 1); + } + #[test] fn the_ring_is_split_evenly_and_memory_max_parses() { assert_eq!(plan(None).ring_per_partition(6), (48 * MIB / 6) as usize); diff --git a/rust/qdl-stream-gateway/src/service.rs b/rust/qdl-stream-gateway/src/service.rs index 7fe13da7..c6dff231 100644 --- a/rust/qdl-stream-gateway/src/service.rs +++ b/rust/qdl-stream-gateway/src/service.rs @@ -43,6 +43,9 @@ use tonic::{Request, Response, Status}; /// queue (DL-V2 R1.8) and hide a slow client from the overflow signal. pub const OUTBOUND_HANDOFF: usize = 2; +/// How often a send waiting for live budget to copy a record retries. +const HANDOFF_BUDGET_RETRY: Duration = Duration::from_millis(5); + #[derive(Clone, Debug)] pub struct StreamLimits { pub cursor_ttl_seconds: u64, @@ -66,7 +69,8 @@ impl Default for StreamLimits { replay: ReplayLimits::default(), catchup_deadline: Duration::from_secs(10), slow_consumer_after: Duration::from_secs(10), - queue_bytes_per_subscription: 32 << 20, + // A fair share of the default 32 MiB live budget (memory.rs). + queue_bytes_per_subscription: 16 << 20, max_subscriptions: 384, max_replay_rpcs: 32, replay_page_default: 1_000, @@ -124,6 +128,8 @@ impl GatewayState { budget: Arc, limits: StreamLimits, ) -> Self { + // Dispatch charges each decoded record to the live budget once. + hub.attach_live_budget(budget.clone()); Self { authority, quota, @@ -1108,6 +1114,46 @@ impl SubscriptionTask { } } + /// Top a dequeued entry's charge up to the record's full weight before + /// the record is copied for the transport: the response owns a private + /// decoded copy until tonic takes it (F4). The wait is this stream's own + /// backpressure, bounded like a blocked send; `None` ends the stream. + async fn handoff_charge(&self, record: &LiveRecord, mut charge: Charge) -> Option { + let missing = record.weight().saturating_sub(charge.bytes()); + let fresh = delivery_decision( + &self.subscription.requirement, + &record.envelope, + now_ns() as i64, + ) == Delivery::Deliver; + // A record that will be skipped is never copied: no top-up. + if missing == 0 || !fresh { + return Some(charge); + } + let deadline = Instant::now() + self.state.limits.slow_consumer_after; + loop { + if let Some(extra) = self.state.budget.charge(missing) { + charge.absorb(extra); + return Some(charge); + } + if Instant::now() >= deadline { + self.backpressure( + record.raw.offset - 1, + "replica live memory budget exhausted at handoff; replay is required", + ) + .await; + return None; + } + tokio::select! { + biased; + status = self.auth.revoked() => { + self.revoke(status).await; + return None; + } + _ = tokio::time::sleep(HANDOFF_BUDGET_RETRY) => {} + } + } + } + /// [`Self::deliver`] for a live record, recording its send-path waits. async fn deliver_timed( &self, @@ -1116,6 +1162,7 @@ impl SubscriptionTask { charge: Charge, ) -> Option { let taken = Instant::now(); + let charge = self.handoff_charge(record, charge).await?; let result = self.deliver(record, Some(charge)).await; let metrics = &self.state.metrics; metrics diff --git a/rust/qdl-stream-gateway/src/subscription.rs b/rust/qdl-stream-gateway/src/subscription.rs index 79901394..00fba0fe 100644 --- a/rust/qdl-stream-gateway/src/subscription.rs +++ b/rust/qdl-stream-gateway/src/subscription.rs @@ -1,9 +1,14 @@ //! One subscriber's bounded queue (KN-2 K2.2/K2.4, D1/D5). //! //! The live reader offers records without ever blocking on a subscriber: the -//! queue is bounded by items and bytes, and bytes are also charged to a -//! replica-wide budget, so total queued memory is bounded whatever the -//! number of slow clients. When the queue is full, a record is dropped only if +//! queue is bounded by items and by the full weight of what it references, +//! and a replica-wide budget bounds what is really held, so total queued +//! memory is bounded whatever the number of slow clients. A record decoded +//! once by the hub carries one shared budget charge for that copy +//! ([`LiveRecord::shared_charge`]); a queue entry then charges the budget +//! only [`QUEUE_ENTRY_OVERHEAD`], never the decoded weight once per +//! subscriber. Without a shared charge it charges the full weight, as +//! before. When the queue is full, a record is dropped only if //! the record **right after it** has the same lifecycle key and signature //! (`qdl_contracts::delivery`): a coalesced run keeps its latest record and //! every transition between runs survives (A B A B stays four records). @@ -25,6 +30,10 @@ use tokio::sync::Notify; /// `LATEST_STATE_BUFFER_EVENTS`): a deep FIFO of quotes only ages them out. pub const LATEST_STATE_BUFFER_EVENTS: usize = 8; +/// Replica budget bytes of one queue entry beyond the shared record: the +/// `Queued` slot (~200 B) plus the lifecycle signature's strings. +pub const QUEUE_ENTRY_OVERHEAD: usize = 512; + /// Replica-wide byte budget shared by every subscriber queue. #[derive(Debug)] pub struct ByteBudget { @@ -77,6 +86,20 @@ pub struct Charge { } impl Charge { + pub fn bytes(&self) -> usize { + self.bytes + } + + /// Fold `other` into this charge (one release later, no budget change). + /// Both must hold the same budget. + pub fn absorb(&mut self, mut other: Charge) { + assert!( + Arc::ptr_eq(&self.budget, &other.budget), + "charges of different budgets cannot be merged" + ); + self.bytes += std::mem::take(&mut other.bytes); + } + /// Move up to `bytes` of this charge into a new one (no budget change). pub fn split(&mut self, bytes: usize) -> Charge { let moved = bytes.min(self.bytes); @@ -106,7 +129,9 @@ pub struct Queued { #[derive(Debug)] pub enum Next { /// A record, when it was queued (the send-path queue wait) and its - /// budget charge, which the caller hands on to the transport. + /// budget charge, which the caller hands on to the transport. The charge + /// is only the entry overhead when the record carries a shared charge; + /// the caller tops it up before it copies the record for the transport. Record(Arc, Instant, Charge), Overflow, Failed(String), @@ -217,11 +242,18 @@ impl Subscription { } self.counters.offered.fetch_add(1, Ordering::Relaxed); let lifecycle = RecordLifecycle::of(&self.requirement.delivery.feed, &record.envelope); + // The per-subscription cap counts the full weight this queue keeps + // alive; the replica budget only what this entry adds to memory. let weight = record.weight(); + let held = if record.shared_charge.is_some() { + QUEUE_ENTRY_OVERHEAD + } else { + weight + }; let charge = loop { let fits = state.queue.len() < self.depth && state.bytes + weight <= self.max_bytes; if fits { - if let Some(charge) = self.budget.charge(weight) { + if let Some(charge) = self.budget.charge(held) { break charge; } } @@ -366,6 +398,7 @@ mod tests { timestamp_ms: 0, }), envelope, + shared_charge: None, }) } @@ -395,6 +428,16 @@ mod tests { ) } + #[test] + fn the_queue_entry_overhead_covers_an_entry() { + // The slot itself; the lifecycle signature's short ids live in the rest. + assert!( + std::mem::size_of::() * 2 <= QUEUE_ENTRY_OVERHEAD, + "{}", + std::mem::size_of::() + ); + } + #[tokio::test] async fn counters_count_eligible_attempts_and_local_pops_only() { let mut subscription = queue(query::FeedType::Trade, 2); From 70bef7d2ef7699d4517fe25519e55edf1709a042 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 18:25:01 +0000 Subject: [PATCH 133/147] fix(sdk): stream transport starts on a random replica per instance (2.2.1) A fixed start put every client's streams on the first target: in the rc8 campaign stream_kn_1 served all 244 streams and stream_kn_2 none, so one replica's live budget carried the whole load. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 7 +++++++ qdl_sdk/CHANGELOG.md | 7 +++++++ qdl_sdk/README.md | 2 +- qdl_sdk/__init__.py | 2 +- qdl_sdk/transport.py | 13 ++++++++++-- scripts/build_qdl_sdk_release.py | 2 +- tests/test_fund_phase5_stream_sdk.py | 1 + tests/test_phaseb_stable_release.py | 2 +- tests/test_phasec2_transport_security.py | 25 +++++++++++++++++++++++ tests/test_qdl_sdk_release.py | 4 ++-- 10 files changed, 57 insertions(+), 8 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index d82f4ae5..809d00c0 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72128,3 +72128,10 @@ no build, CI or subagent job running. - Tests: fmt, clippy -D warnings, cargo test 49+1+37 pass (qdl-rust-builder:r134-test). Status: tested locally. - Found alongside: stream_kn_2 served 0 of 244 streams. The SDK GrpcStreamTransport always starts at target 0; 5c81c25 randomised only the REST Query transport. Fixed in the SDK (next slice). +- SDK 2.2.1: + - GrpcStreamTransport starts on a random target per instance; keyword start_index pins it. Failover on + UNAVAILABLE is unchanged. Additive, no wire change. + - SDK 2.2.0 was never published (DL not pushed), but its wheel hash is recorded in the release evidence, so the + version is bumped rather than reused. + - Tests: 57 OK (transport security, stream SDK incl. failover with a pinned start, replica read, release + wheel). diff --git a/qdl_sdk/CHANGELOG.md b/qdl_sdk/CHANGELOG.md index 57a08960..6a611bf8 100644 --- a/qdl_sdk/CHANGELOG.md +++ b/qdl_sdk/CHANGELOG.md @@ -1,5 +1,12 @@ # SDK Changelog +## 2.2.1 (v2.2.3 Data Layer) + +`GrpcStreamTransport` starts on a random target per instance instead of always +the first one, so the stream replicas share the subscriptions; failover on +UNAVAILABLE is unchanged. A new keyword `start_index` pins the start (tests, +diagnostics). Additive; no request or response change. + ## 2.2.0 (v2.2.3 Data Layer) `ReplicatedRestQueryTransport` starts its round-robin on a random replica per diff --git a/qdl_sdk/README.md b/qdl_sdk/README.md index ec978b0d..d41d81ff 100644 --- a/qdl_sdk/README.md +++ b/qdl_sdk/README.md @@ -146,7 +146,7 @@ Build the standalone artifact with: python scripts/build_qdl_sdk_release.py --output-dir dist/qdl-sdk ``` -The output contains a reproducible `qdl_sdk-2.2.0-py3-none-any.whl`, a release +The output contains a reproducible `qdl_sdk-2.2.1-py3-none-any.whl`, a release manifest with the wheel/source/generated-contract SHA-256 digests, and a CycloneDX SBOM. The wheel contains only the public SDK plus generated Protobuf contracts; it does not package `qdl.api_v2`, runtime adapters, provider code or diff --git a/qdl_sdk/__init__.py b/qdl_sdk/__init__.py index c8186dc7..9a5303eb 100644 --- a/qdl_sdk/__init__.py +++ b/qdl_sdk/__init__.py @@ -1,6 +1,6 @@ """Stable Python SDK V2 with typed public models and generated transports.""" -__version__ = "2.2.0" +__version__ = "2.2.1" from qdl_sdk.client import AsyncDataLayerClient, DataLayerClientV2, WarmupStreamSession from qdl_sdk.credentials import ( diff --git a/qdl_sdk/transport.py b/qdl_sdk/transport.py index 7fabb86c..7a1b8873 100644 --- a/qdl_sdk/transport.py +++ b/qdl_sdk/transport.py @@ -472,6 +472,7 @@ def __init__( tls: WorkloadTlsConfig | None = None, allow_insecure_loopback: bool = False, credential_provider: CredentialProvider | None = None, + start_index: int | None = None, ) -> None: raw_targets = (target,) if isinstance(target, str) else tuple(target) targets = tuple( @@ -482,9 +483,17 @@ def __init__( ) if not targets or len(set(targets)) != len(targets): raise ValueError("gRPC stream targets must be non-empty and unique") + if start_index is not None and not 0 <= start_index < len(targets): + raise ValueError("gRPC stream start_index is outside target bounds") self.targets = targets - self.target = targets[0] - self._target_index = 0 + # Each instance starts on a random replica, like the replicated Query + # transport. A fixed start put every client's streams on target 0 + # (P18.4 rc8 campaign: stream_kn_1 served 244 streams, stream_kn_2 + # none), so one replica's live byte budget carried the whole load. + self._target_index = ( + random.randrange(len(targets)) if start_index is None else start_index + ) + self.target = targets[self._target_index] self._credential_provider = credential_provider if credentials is not None and tls is not None: raise ValueError("provide either credentials or workload TLS, not both") diff --git a/scripts/build_qdl_sdk_release.py b/scripts/build_qdl_sdk_release.py index 500a3c6a..92321e3c 100755 --- a/scripts/build_qdl_sdk_release.py +++ b/scripts/build_qdl_sdk_release.py @@ -15,7 +15,7 @@ ROOT = Path(__file__).resolve().parents[1] NAME = "qdl-sdk" NORMALIZED_NAME = "qdl_sdk" -VERSION = "2.2.0" +VERSION = "2.2.1" DIST_INFO = f"{NORMALIZED_NAME}-{VERSION}.dist-info" DEPENDENCIES = ( "grpcio>=1.70.0,<2.0.0", diff --git a/tests/test_fund_phase5_stream_sdk.py b/tests/test_fund_phase5_stream_sdk.py index 16b5d840..8bd99dde 100644 --- a/tests/test_fund_phase5_stream_sdk.py +++ b/tests/test_fund_phase5_stream_sdk.py @@ -878,6 +878,7 @@ def assert_active(self, expected_epoch=None): (f"127.0.0.1:{standby_port}", f"127.0.0.1:{active_port}"), allow_insecure_loopback=True, credential_provider=self.credential, + start_index=0, ) requirement = DataRequirement( self.record.instrument_uid, diff --git a/tests/test_phaseb_stable_release.py b/tests/test_phaseb_stable_release.py index bf5bdd1f..1daee762 100644 --- a/tests/test_phaseb_stable_release.py +++ b/tests/test_phaseb_stable_release.py @@ -317,7 +317,7 @@ def test_service_openapi_and_sdk_versions_are_explicit(self): ) generated = build_openapi() self.assertEqual(package["project"]["version"], "2.0.0") - self.assertEqual(qdl_sdk.__version__, "2.2.0") + self.assertEqual(qdl_sdk.__version__, "2.2.1") self.assertEqual(generated["info"]["version"], "2.0.0") self.assertEqual(snapshot, generated) # ``reference:batch`` is a governed V2 public path in the checked-in diff --git a/tests/test_phasec2_transport_security.py b/tests/test_phasec2_transport_security.py index fa5d306b..7422a223 100644 --- a/tests/test_phasec2_transport_security.py +++ b/tests/test_phasec2_transport_security.py @@ -51,6 +51,7 @@ async def test_rest_and_grpc_share_one_workload_identity(self): stream = GrpcStreamTransport( ("qdl-v2-stream-a:8210", "qdl-v2-stream-b:8210"), tls=tls, + start_index=0, ) try: self.assertEqual( @@ -136,6 +137,30 @@ async def test_ambiguous_tls_clients_and_duplicate_targets_are_rejected(self): allow_insecure_loopback=True, ) + async def test_stream_targets_start_on_a_random_replica_per_instance(self): + targets = ("127.0.0.1:18220", "127.0.0.1:18221") + seen = set() + for _ in range(64): + stream = GrpcStreamTransport(targets, allow_insecure_loopback=True) + try: + self.assertEqual(stream.target, targets[stream._target_index]) + seen.add(stream.target) + finally: + await stream.close() + self.assertEqual(seen, set(targets)) + pinned = GrpcStreamTransport( + targets, allow_insecure_loopback=True, start_index=1 + ) + try: + self.assertEqual(pinned.target, targets[1]) + finally: + await pinned.close() + for bad in (-1, 2): + with self.assertRaisesRegex(ValueError, "start_index"): + GrpcStreamTransport( + targets, allow_insecure_loopback=True, start_index=bad + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_qdl_sdk_release.py b/tests/test_qdl_sdk_release.py index a064adc3..c93e687a 100644 --- a/tests/test_qdl_sdk_release.py +++ b/tests/test_qdl_sdk_release.py @@ -41,11 +41,11 @@ def test_wheel_is_reproducible_self_contained_and_recorded(self): expected = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()).rstrip(b"=").decode() self.assertEqual(encoded, "sha256=" + expected) self.assertEqual(int(size), len(payload)) - self.assertIn("qdl_sdk-2.2.0-py3-none-any.whl", archive.read("qdl_sdk/README.md").decode()) + self.assertIn("qdl_sdk-2.2.1-py3-none-any.whl", archive.read("qdl_sdk/README.md").decode()) release = json.loads(Path(first_result["manifest"]).read_text()) self.assertFalse(release["contains_service_internals"]) - self.assertEqual(release["version"], "2.2.0") + self.assertEqual(release["version"], "2.2.1") self.assertEqual(len(release["generated_contract_digest"]), 64) sbom = json.loads(Path(first_result["sbom"]).read_text()) self.assertEqual(sbom["bomFormat"], "CycloneDX") From 413c5b914f8c6fd0f67429fdddb39451589e9745 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 18:39:51 +0000 Subject: [PATCH 134/147] fix(admission): poll an inflight-capacity deferral instead of sleeping the lease expiry The INFLIGHT_CAPACITY retry hint is the earliest lease's maximum expiry (30 s). The bar edge slept it, so an hour-boundary cycle stalled ~30 s and most bindings failed their deadline at once (G4, 2026-10-10 18:00Z). --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 15 +++++++++++ qdl/admission/edge.py | 9 +++++++ tests/test_kn_provider_admission_edge.py | 33 +++++++++++++++++++++-- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 809d00c0..973042d4 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72135,3 +72135,18 @@ no build, CI or subagent job running. version is bumped rather than reused. - Tests: 57 OK (transport security, stream SDK incl. failover with a pinned start, replica read, release wheel). +- Depth-0 campaign (17:49-18:25Z, `p184-final-campaign-20261010T174944Z`, writer depth 0): + - G2 TRADE p95 579/568 ms (max 2146/1854); QUOTE inside (max 678/745); no overflow incident this time. + - G4 FAIL: bar|BINANCE 3m/30m at the 18:00Z boundary, 36-41 s against limits of ~26-27 s. + - G8 FAIL: one writer APPLY EVALSHA of 10.15 ms in the steady window. The same script has taken 10-40 ms for + 1-row calls today, so it is not batch-size bound (inferred: host scheduling noise). +- G4 cause, a DL defect in admission: + - An INFLIGHT_CAPACITY deferral returns retry_after = the earliest lease's MAXIMUM expiry (max_lease_ns 30 s). + - qdl/admission/edge.py slept exactly that hint. Callers that saw it first failed their deadline at once + (28 of 35 bindings); the next ones slept ~30 s although the lease was released 0.1-0.3 s later. + - run_cycle publishes after every future, so one sleeper held the 7 hour-boundary bars. + - Seen at 14:30, 16:00 and 18:00Z today (first ACK +31-32 s, against +6.7-7.3 s at the other boundaries). + - No venue 429/418 since 12:00Z. + - Fix: an INFLIGHT_CAPACITY wait polls jittered 50-200 ms, never longer than the hint, still deadline-bound. + Other defer reasons keep the hint. + - Tests: 285 OK across the admission and bar-edge suites. The frozen G4 limits are unchanged. diff --git a/qdl/admission/edge.py b/qdl/admission/edge.py index 95a0ea76..847a2b33 100644 --- a/qdl/admission/edge.py +++ b/qdl/admission/edge.py @@ -17,6 +17,7 @@ import asyncio from contextlib import contextmanager import logging +import random import threading import time import uuid @@ -24,6 +25,7 @@ from qdl.admission.contracts import ( AdmissionDecision, + AdmissionDeferReason, AdmissionDisposition, AdmissionPriority, AdmissionRequest, @@ -34,6 +36,11 @@ logger = logging.getLogger(__name__) _CALL_TIMEOUT_S = 5.0 +# An INFLIGHT_CAPACITY deferral's retry_after is the earliest lease's MAXIMUM expiry (max_lease_ns, 30 s), not the +# moment it is released (typically 0.1-0.3 s later). Sleeping the hint held a whole Binance bar-edge cycle ~30 s at +# hour boundaries (2026-10-10 14:30/16:00/18:00Z, G4) and made callers that saw it first fail their deadline at once. +# A capacity wait polls instead, jittered, never longer than the hint and still bounded by the deadline. +_INFLIGHT_POLL_S = (0.05, 0.2) class AdmissionDeadlineExceeded(RuntimeError): @@ -103,6 +110,8 @@ def lease(self, lane: ProviderLane, request_prefix: str, *, priority: AdmissionP if decision.disposition is AdmissionDisposition.GRANTED: break wait_s = max(0.01, (decision.retry_after_ms or 100) / 1000.0) + if decision.defer_reason is AdmissionDeferReason.INFLIGHT_CAPACITY: + wait_s = min(wait_s, random.uniform(*_INFLIGHT_POLL_S)) if self._clock() + wait_s > deadline: raise AdmissionDeadlineExceeded(lane, decision) self._sleep(wait_s) diff --git a/tests/test_kn_provider_admission_edge.py b/tests/test_kn_provider_admission_edge.py index f2ac491a..d6146107 100644 --- a/tests/test_kn_provider_admission_edge.py +++ b/tests/test_kn_provider_admission_edge.py @@ -30,9 +30,11 @@ class FakeRuntime: """The async surface of ``RustHttpProviderAdmission``, scripted.""" - def __init__(self, defers: int = 0, retry_after_ms: int = 250) -> None: + def __init__(self, defers: int = 0, retry_after_ms: int = 250, + reason: AdmissionDeferReason = AdmissionDeferReason.TOKEN_BUDGET) -> None: self.defers = defers self.retry_after_ms = retry_after_ms + self.reason = reason self.admits: list = [] self.completed: list = [] self.rate_limits: list = [] @@ -46,7 +48,7 @@ async def admit(self, request): return AdmissionDecision( lane=request.lane, request_id=request.request_id, disposition=AdmissionDisposition.DEFERRED if deferred else AdmissionDisposition.GRANTED, - defer_reason=AdmissionDeferReason.TOKEN_BUDGET if deferred else None, + defer_reason=self.reason if deferred else None, retry_after_ms=self.retry_after_ms if deferred else None, lease_expires_at_ns=None if deferred else 1, coalesced=False) @@ -113,6 +115,33 @@ def test_a_deferred_request_waits_for_the_rust_hint_then_completes_once(self): self.assertEqual({item.request_id for item in runtime.admits}, {runtime.admits[0].request_id}) self.assertEqual(len(runtime.completed), 1) + def test_an_inflight_capacity_wait_polls_instead_of_sleeping_the_lease_expiry(self): + # The INFLIGHT_CAPACITY hint is the earliest lease's maximum expiry (30 s), not its release + # (2026-10-10 18:00Z: a bar-edge cycle held ~30 s; callers seeing it first failed their deadline). + runtime = FakeRuntime(defers=3, retry_after_ms=29_900, reason=AdmissionDeferReason.INFLIGHT_CAPACITY) + admission, sleeps = self.facade(runtime, max_wait_s=30.0) + lane = ProviderLane("BINANCE", "USDM", "KLINES") + with admission.lease(lane, "klines:SOLUSDT:3m", priority=AdmissionPriority.BATCH, token_cost=1): + pass + self.assertEqual(len(sleeps), 3) + self.assertTrue(all(0.05 <= value <= 0.2 for value in sleeps), sleeps) + self.assertEqual(len(runtime.completed), 1) + + def test_an_inflight_capacity_poll_never_exceeds_a_shorter_hint_or_the_deadline(self): + runtime = FakeRuntime(defers=1, retry_after_ms=20, reason=AdmissionDeferReason.INFLIGHT_CAPACITY) + admission, sleeps = self.facade(runtime) + lane = ProviderLane("BINANCE", "USDM", "KLINES") + with admission.lease(lane, "klines:BTCUSDT:5m", priority=AdmissionPriority.BATCH, token_cost=1): + pass + self.assertEqual(sleeps, [0.02]) + runtime = FakeRuntime(defers=10_000, retry_after_ms=29_900, reason=AdmissionDeferReason.INFLIGHT_CAPACITY) + admission, sleeps = self.facade(runtime, max_wait_s=1.0) + with self.assertRaises(AdmissionDeadlineExceeded): + with admission.lease(lane, "klines:BTCUSDT:5m", priority=AdmissionPriority.BATCH, token_cost=1): + pass + self.assertLessEqual(sum(sleeps), 1.0) + self.assertEqual(runtime.completed, []) + def test_the_deadline_is_typed_and_no_provider_call_happens(self): runtime = FakeRuntime(defers=10_000, retry_after_ms=1_000) admission, _sleeps = self.facade(runtime, max_wait_s=2.5) From 2f489b186d8d590ffa59b6dae2e41caa2fa400ad Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sat, 10 Oct 2026 19:31:52 +0000 Subject: [PATCH 135/147] fix(p184): G6 reference anchor follows the server rule per product and grade Funding was anchored at its oldest settlement row and every alpha funding read scored UNSAFE; non-MARK products use the newest observation, and only an EXECUTION MARK read uses provider confirmation, as the Query service does. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 9 ++++ scripts/p184_g6_ts_execution_probe.py | 30 ++++++++----- scripts/phase3_consumer_load_acceptance.py | 32 ++++++++------ tests/test_p184_g6_view_quality.py | 50 +++++++++++++++------- 4 files changed, 83 insertions(+), 38 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 973042d4..c1b8a1e3 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72150,3 +72150,12 @@ no build, CI or subagent job running. - Fix: an INFLIGHT_CAPACITY wait polls jittered 50-200 ms, never longer than the hint, still deadline-bound. Other defer reasons keep the hint. - Tests: 285 OK across the admission and bar-edge suites. The frozen G4 limits are unchanged. +- Correction (Claude) to ba496ff: + - Fault: the G6 reference anchor always took min over observations. For FUNDING_RATE, a settlement history, that + is the oldest row, so the final campaign flagged 150/150 alpha funding reads UNSAFE (age ~11 h). + - Method error: the server rule was copied for the MARK branch only. + - The anchor now mirrors qdl/query/service.py _reference_freshness_timestamp: + - MARK EXECUTION read with all PROVIDER_CONFIRMATION: min provider_confirmation_ns; + - other MARK reads: min observed_at; + - every other product: max observed_at. + - Driver and probe copies stay in parity. Tests: 140 OK. diff --git a/scripts/p184_g6_ts_execution_probe.py b/scripts/p184_g6_ts_execution_probe.py index d163eb41..c5ea978a 100644 --- a/scripts/p184_g6_ts_execution_probe.py +++ b/scripts/p184_g6_ts_execution_probe.py @@ -88,20 +88,27 @@ def g6_classify(*, feed: str, freshness_ms: int | None, eligible: bool, state: s return ("ELIGIBLE_QUIET" if beyond else "ELIGIBLE"), [] -def g6_reference_anchor_ns(observations) -> int | None: - """The server's freshness anchor of a current MARK/INDEX result (qdl/query/service.py - _reference_freshness_timestamp): min provider_confirmation_ns when every observation is PROVIDER_CONFIRMATION - based, else min observed_at_ns when every one is SOURCE_EVENT (the default without a label); None otherwise. - observations: [(observed_at_ns, labels mapping)]. Pure.""" +def g6_reference_anchor_ns(observations, *, feed: str = "MARK_INDEX_PRICE", grade: object = None) -> int | None: + """The server's freshness anchor of a current reference result (qdl/query/service.py + _reference_freshness_timestamp). MARK/INDEX: an EXECUTION read whose observations are all PROVIDER_CONFIRMATION + based is anchored at min provider_confirmation_ns; otherwise min observed_at_ns (a pair is only as current as its + oldest component; mixed bases give None). Every other product (FUNDING_RATE) is anchored at max observed_at_ns: + how recently the series was updated, not its oldest settlement row (the min anchor aged funding by its whole + history and flagged every funding read UNSAFE, 2026-10-10 final campaign). observations: [(observed_at_ns, + labels mapping)]. Pure.""" items = list(observations or []) if not items: return None - bases = {str((labels or {}).get("freshness_basis", "SOURCE_EVENT")) for _, labels in items} try: - if bases == {"PROVIDER_CONFIRMATION"}: + observed = [int(value) for value, _ in items] + if str(getattr(feed, "value", feed)) != "MARK_INDEX_PRICE": + return max(observed) if max(observed) > 0 else None + bases = {str((labels or {}).get("freshness_basis", "SOURCE_EVENT")) for _, labels in items} + execution = str(getattr(grade, "value", grade) or "").upper() == "EXECUTION" + if execution and bases == {"PROVIDER_CONFIRMATION"}: values = [int(labels["provider_confirmation_ns"]) for _, labels in items] - elif bases == {"SOURCE_EVENT"}: - values = [int(observed) for observed, _ in items] + elif bases <= {"SOURCE_EVENT", "PROVIDER_CONFIRMATION"} and (bases == {"SOURCE_EVENT"} or not execution): + values = observed else: return None except (KeyError, TypeError, ValueError): @@ -109,7 +116,8 @@ def g6_reference_anchor_ns(observations) -> int | None: return min(values) if all(value > 0 for value in values) else None -def g6_reference_age(*, grade: object, observations, sent_ns: int, receipt_ns: int) -> dict[str, object]: +def g6_reference_age(*, grade: object, observations, sent_ns: int, receipt_ns: int, + feed: str = "MARK_INDEX_PRICE") -> dict[str, object]: """Age of a MARK reference result. A reference result carries no served-at time, only the server's freshness anchor (g6_reference_anchor_ns). EXECUTION keeps the unchanged use-time basis (client receipt minus the newest source event), so what counts as unsafe for EXECUTION reads does not move. Any other grade is aged from the @@ -117,7 +125,7 @@ def g6_reference_age(*, grade: object, observations, sent_ns: int, receipt_ns: i on this basis is beyond it at serve time too, and the network round trip can no longer manufacture UNSAFE. client_age_ms (receipt minus the server anchor, includes the round trip) is kept for every grade. Pure.""" items = list(observations or []) - anchor = g6_reference_anchor_ns(items) + anchor = g6_reference_anchor_ns(items, feed=feed, grade=grade) client_age = None if anchor is None else max(0, (receipt_ns - anchor) // 1_000_000) if str(getattr(grade, "value", grade) or "").upper() == "EXECUTION": newest = max((int(observed) for observed, _ in items), default=0) diff --git a/scripts/phase3_consumer_load_acceptance.py b/scripts/phase3_consumer_load_acceptance.py index 39a1bd7a..ee20e42a 100644 --- a/scripts/phase3_consumer_load_acceptance.py +++ b/scripts/phase3_consumer_load_acceptance.py @@ -1843,20 +1843,27 @@ def g6_classify(*, feed: str, freshness_ms: int | None, eligible: bool, state: s return ("ELIGIBLE_QUIET" if beyond else "ELIGIBLE"), [] -def g6_reference_anchor_ns(observations) -> int | None: - """The server's freshness anchor of a current MARK/INDEX result (qdl/query/service.py - _reference_freshness_timestamp): min provider_confirmation_ns when every observation is PROVIDER_CONFIRMATION - based, else min observed_at_ns when every one is SOURCE_EVENT (the default without a label); None otherwise. - observations: [(observed_at_ns, labels mapping)]. Pure.""" +def g6_reference_anchor_ns(observations, *, feed: str = "MARK_INDEX_PRICE", grade: object = None) -> int | None: + """The server's freshness anchor of a current reference result (qdl/query/service.py + _reference_freshness_timestamp). MARK/INDEX: an EXECUTION read whose observations are all PROVIDER_CONFIRMATION + based is anchored at min provider_confirmation_ns; otherwise min observed_at_ns (a pair is only as current as its + oldest component; mixed bases give None). Every other product (FUNDING_RATE) is anchored at max observed_at_ns: + how recently the series was updated, not its oldest settlement row (the min anchor aged funding by its whole + history and flagged every funding read UNSAFE, 2026-10-10 final campaign). observations: [(observed_at_ns, + labels mapping)]. Pure.""" items = list(observations or []) if not items: return None - bases = {str((labels or {}).get("freshness_basis", "SOURCE_EVENT")) for _, labels in items} try: - if bases == {"PROVIDER_CONFIRMATION"}: + observed = [int(value) for value, _ in items] + if str(getattr(feed, "value", feed)) != "MARK_INDEX_PRICE": + return max(observed) if max(observed) > 0 else None + bases = {str((labels or {}).get("freshness_basis", "SOURCE_EVENT")) for _, labels in items} + execution = str(getattr(grade, "value", grade) or "").upper() == "EXECUTION" + if execution and bases == {"PROVIDER_CONFIRMATION"}: values = [int(labels["provider_confirmation_ns"]) for _, labels in items] - elif bases == {"SOURCE_EVENT"}: - values = [int(observed) for observed, _ in items] + elif bases <= {"SOURCE_EVENT", "PROVIDER_CONFIRMATION"} and (bases == {"SOURCE_EVENT"} or not execution): + values = observed else: return None except (KeyError, TypeError, ValueError): @@ -1864,7 +1871,8 @@ def g6_reference_anchor_ns(observations) -> int | None: return min(values) if all(value > 0 for value in values) else None -def g6_reference_age(*, grade: object, observations, sent_ns: int, receipt_ns: int) -> dict[str, object]: +def g6_reference_age(*, grade: object, observations, sent_ns: int, receipt_ns: int, + feed: str = "MARK_INDEX_PRICE") -> dict[str, object]: """Age of a MARK reference result. A reference result carries no served-at time, only the server's freshness anchor (g6_reference_anchor_ns). EXECUTION keeps the unchanged use-time basis (client receipt minus the newest source event), so what counts as unsafe for EXECUTION reads does not move. Any other grade is aged from the @@ -1872,7 +1880,7 @@ def g6_reference_age(*, grade: object, observations, sent_ns: int, receipt_ns: i on this basis is beyond it at serve time too, and the network round trip can no longer manufacture UNSAFE. client_age_ms (receipt minus the server anchor, includes the round trip) is kept for every grade. Pure.""" items = list(observations or []) - anchor = g6_reference_anchor_ns(items) + anchor = g6_reference_anchor_ns(items, feed=feed, grade=grade) client_age = None if anchor is None else max(0, (receipt_ns - anchor) // 1_000_000) if str(getattr(grade, "value", grade) or "").upper() == "EXECUTION": newest = max((int(observed) for observed, _ in items), default=0) @@ -1947,7 +1955,7 @@ def _g6_reference_observations(references, response, *, sent_ns: int, now_ns: in continue ages = g6_reference_age(grade=grade, observations=[(o.observed_at_ns, dict(o.labels or {})) for o in observations], - sent_ns=sent_ns, receipt_ns=now_ns) + sent_ns=sent_ns, receipt_ns=now_ns, feed=feed) try: quiet = _quiet_execution_mark_index_evidence(product, item, observed_at_ns=now_ns) is not None except Exception: diff --git a/tests/test_p184_g6_view_quality.py b/tests/test_p184_g6_view_quality.py index 4c785486..18637df0 100644 --- a/tests/test_p184_g6_view_quality.py +++ b/tests/test_p184_g6_view_quality.py @@ -178,7 +178,8 @@ def test_alpha_mark_at_2300_ms_is_not_unsafe_and_is_aged_from_request_sent(self) now_ns=sent + 400 * MS) self.assertEqual((row["class"], row["policy_ms"], row["policy_basis"]), ("ELIGIBLE", 60000, "REQUIREMENT_ALPHA")) - self.assertEqual((row["age_ms"], row["client_age_ms"]), (2300, 2700)) + # Outside EXECUTION the server anchors a MARK pair at its oldest observed_at, not provider confirmation. + self.assertEqual((row["age_ms"], row["client_age_ms"]), (2600, 3000)) self.assertEqual(row["age_basis"], "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR") self.assertEqual(row["key"], "MARK_INDEX_PRICE|BINANCE|@REQUIREMENT_ALPHA") @@ -193,11 +194,12 @@ def test_execution_mark_keeps_its_use_time_basis_and_d3_policy(self): def test_anchor_follows_the_server_rule(self): anchor = _DRIVER.g6_reference_anchor_ns self.assertEqual(anchor([(5, {}), (7, {})]), 5) - self.assertEqual(anchor([(5, {"freshness_basis": "PROVIDER_CONFIRMATION", "provider_confirmation_ns": "9"})]), - 9) - self.assertIsNone(anchor([(5, {"freshness_basis": "PROVIDER_CONFIRMATION"})])) - self.assertIsNone(anchor([(5, {"freshness_basis": "SOURCE_EVENT"}), - (6, {"freshness_basis": "PROVIDER_CONFIRMATION", "provider_confirmation_ns": "9"})])) + confirmed = {"freshness_basis": "PROVIDER_CONFIRMATION", "provider_confirmation_ns": "9"} + self.assertEqual(anchor([(5, confirmed)], grade="EXECUTION"), 9) + self.assertEqual(anchor([(5, confirmed)], grade="ALPHA"), 5) + self.assertIsNone(anchor([(5, {"freshness_basis": "PROVIDER_CONFIRMATION"})], grade="EXECUTION")) + self.assertIsNone(anchor([(5, {"freshness_basis": "SOURCE_EVENT"}), (6, confirmed)], grade="EXECUTION")) + self.assertEqual(anchor([(5, {}), (7, {})], feed="FUNDING_RATE"), 7) self.assertIsNone(anchor([])) @@ -232,18 +234,36 @@ def test_classify_parity(self): count += 1 self.assertGreater(count, 50000) + def test_funding_is_anchored_at_its_newest_row_like_the_server(self): + # A funding series returns its settlement history; the server ages it by the newest row (max), so the + # 8-hourly history never makes a current funding read look 11 hours old. + rows = [(1_000 * MS, {}), (9_000 * MS, {}), (5_000 * MS, {})] + self.assertEqual(_DRIVER.g6_reference_anchor_ns(rows, feed="FUNDING_RATE", grade="ALPHA"), 9_000 * MS) + age = _DRIVER.g6_reference_age(grade="ALPHA", observations=rows, sent_ns=10_000 * MS, receipt_ns=10_300 * MS, + feed="FUNDING_RATE") + self.assertEqual(age["age_ms"], 1_000) + + def test_an_alpha_mark_pair_is_anchored_at_its_oldest_component(self): + # Outside an EXECUTION read the server does not use provider confirmation: min observed_at. + labels = {"freshness_basis": "PROVIDER_CONFIRMATION", "provider_confirmation_ns": str(9_900 * MS)} + rows = [(9_000 * MS, labels), (8_000 * MS, labels)] + self.assertEqual(_DRIVER.g6_reference_anchor_ns(rows, feed="MARK_INDEX_PRICE", grade="ALPHA"), 8_000 * MS) + self.assertEqual(_DRIVER.g6_reference_anchor_ns(rows, feed="MARK_INDEX_PRICE", grade="EXECUTION"), + 9_900 * MS) + def test_reference_parity(self): label_sets = [{}, {"freshness_basis": "PROVIDER_CONFIRMATION", "provider_confirmation_ns": str(9_500 * MS)}, {"freshness_basis": "PROVIDER_CONFIRMATION"}, {"freshness_basis": "SOURCE_EVENT"}] - for grade in ("EXECUTION", "ALPHA", None): - for labels in label_sets: - for observed in (1, 7_000 * MS, 9_000 * MS): - observations = [(observed, labels)] - self.assertEqual(_DRIVER.g6_reference_anchor_ns(observations), - _PROBE.g6_reference_anchor_ns(observations)) - kwargs = dict(grade=grade, observations=observations, sent_ns=10_000 * MS, - receipt_ns=10_300 * MS) - self.assertEqual(_DRIVER.g6_reference_age(**kwargs), _PROBE.g6_reference_age(**kwargs)) + for feed in ("MARK_INDEX_PRICE", "FUNDING_RATE"): + for grade in ("EXECUTION", "ALPHA", None): + for labels in label_sets: + for observed in (1, 7_000 * MS, 9_000 * MS): + observations = [(observed, labels), (observed // 2 or 1, labels)] + self.assertEqual(_DRIVER.g6_reference_anchor_ns(observations, feed=feed, grade=grade), + _PROBE.g6_reference_anchor_ns(observations, feed=feed, grade=grade)) + kwargs = dict(grade=grade, observations=observations, sent_ns=10_000 * MS, + receipt_ns=10_300 * MS, feed=feed) + self.assertEqual(_DRIVER.g6_reference_age(**kwargs), _PROBE.g6_reference_age(**kwargs)) for age in (None, 100, 2300, 70000): for policy in (None, 2000, 60000): for quiet in (True, False): From cb87370f8efb9949215670d25f958af0dfb0de1f Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 11 Oct 2026 06:32:06 +0000 Subject: [PATCH 136/147] docs(release): v2.2.3 evidence on the one-commit rc10 images (413c5b9) All 12 roles run python a0c3649d / rust 96a2c912 (IDR-039 closed). Final campaign 4: PASS G1 G3 G5 G6 G7 G9 G10 G11 G14; FAIL G2 G8 (writer TRADE burst throughput, owner moved it to TS phase E); NOT_MEASURED G4 G12; G13 41/41 DL cells, TS groups deferred. Kafka cleaner throttle, SDK 2.2.1, local CI and every earlier campaign recorded. Status PREPARED_NOT_PUBLISHED. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 36 + .../evidence/releases/v2.2.3/RELEASE_NOTES.md | 347 +- .../releases/v2.2.3/release-preparation.json | 896 ++- .../evidence/releases/v2.2.3/scorecard.json | 5899 ++++++++++++----- 4 files changed, 5431 insertions(+), 1747 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index c1b8a1e3..2506484a 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72159,3 +72159,39 @@ no build, CI or subagent job running. - other MARK reads: min observed_at; - every other product: max observed_at. - Driver and probe copies stay in parity. Tests: 140 OK. + +### Final campaign 4, Kafka cleaner throttle, rc10 on all roles, G12/G13 on rc10, v2.2.3 evidence - 2026-10-11 (Claude, subagents) + +- Kafka (owner-approved question, ~03:40Z): cluster dynamic default log.cleaner.io.max.bytes.per.second=8388608, + applied 03:40:56Z, no broker restart. Cause: md.latest.v2 compaction on kafka1 (1.25 vCPU) stalled every + rust_core commit ~2.06 s (2026-10-10 14:12, 15:29, 19:09Z). Receipt kafka-cleaner-throttle-20261010/; rollback + --delete-config. Campaign 20261011T033641Z was aborted to apply it. +- Earlier campaign 20261010T185053Z (writer bae2509f, depth 0): protocol not met (startup:bounded, s89 exhausted 10 + retries); G1/G2/G4 excess from the 19:09Z stall. +- Final campaign 4 (20261011T034233Z, protocol met, writer bae2509f 8b567eb depth 1, pinned OKX workers; DL mix + Query rc8, Stream rc9, bar edge rc10, rest rc7d): + - PASS G1 G3 G5 G6 G7 G9 G10 G11 G14; FAIL G2 (TRADE B/O p95 1562.1/2084.2, max 5338.8/4301.8; QUOTE max + 1439/1341; MARK max 1125/1228 ms) and G8 (6 SLOWLOG EVALSHA 10.0-19.7 ms); G4 NOT_MEASURED (17/17 baselined + rows PASS, 10 rows without a frozen baseline). + - Attribution (read-only per-second trace analysis): 1699/1801 s max e2e < 500 ms; 28 s > 1500 ms, 25 of them + TRADE-only; bursts up to 4274 TRADE rec in one source second on one slice/worker, <= 1138 applied rec/s per + worker; G4 TRADE canonical p95 108-137 ms; 0 overflow. TRADE volume ~2.2x earlier runs (n 160328). All 6 SLOWLOG + entries sit in TRADE burst seconds. + - Owner decision (AskUserQuestion): release DL; writer throughput (SDK batch decode, Rust projection, EVAL + chunking) is the first TS phase E item; gates not widened. +- Writer set to depth 0 after the campaign (04:24:53Z; rollback campaign-roll/rollback-20261011T042442Z). +- rc10 (413c5b9) rolled on all 12 application roles 04:30-04:35Z, writer READY 64/64 after every step: python + sha256:a0c3649d9a47, rust sha256:96a2c9120ef4; IDR-039 closed. Rollback per role in rollout-rc6/. +- G12 isolated on rc10 (p184-g12-rc10-20261011-a2): NOT_MEASURED by the strict rule (G2/G3 after recovery not measured on a BOOK-only source); caught up broker 22.334 s, projector 0.935 s, Stream 2.294 s; G1 + windows inside (max 1970 ms OKX); teardown 12 removed, 0 left, volumes 217 -> 217. Attempt 1 FAILED on a harness + STOP (in-namespace monitor exited, no traceback/OOM kept); duplicate -r2 stopped. Open: keep monitor exit + code/OOM state. +- G13 on rc10: 41/41 DL cells ok (043821Z: dr05-upstream 6, dr08 4, dr05-transport 9; 045533Z: views 22). First + views run refused DR04.EXPLICIT_LAST@BINANCE EXECUTION_CONTEXT_STALE in a quiet market (correct typed refusal). + Four TS steps deferred to the TS release. +- SDK 2.2.1 wheel 27db3971... reproducible over two builds; 2.2.0 superseded, never published. +- Local CI on 2f489b1: all jobs except unit-tests PASS or INCOMPLETE only by skips (contract-tests #10 needs a + download, PR-only steps n/a); unit-tests 19 PASS, 3 SKIPPED (#4 sudo, #19/#20 no trivy image); re-run with + aquasec/trivy:0.58.1 (unit-r2, f6b6b5e7): every step PASS except #4 sudo. Remote CI not run. +- Release evidence updated (upgrade/evidence/releases/v2.2.3/): status PREPARED_NOT_PUBLISHED; the 2026-10-10 + record moved into the history. Publication needs owner push, tag and remote CI. diff --git a/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md b/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md index b40de8a7..24589b57 100644 --- a/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md +++ b/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md @@ -4,10 +4,333 @@ Status: `PREPARED_NOT_PUBLISHED`. This is not a published release, a Git tag or Publication needs the owner to push and tag. Remote CI must also run on the final head. The published stable release is still v2.2.2. -Scorecard: [scorecard.json](scorecard.json). Inventory, hashes and the SDK build: [release-preparation.json](release-preparation.json). +Scorecard: [scorecard.json](scorecard.json). Inventory, hashes, the SDK build and CI: [release-preparation.json](release-preparation.json). ## Scope +This release closes the P18.4 phase D Data Layer work on the Kafka-native stack. It contains: +- one-commit images for every application role (IDR-039, closed on 413c5b9); +- execution-view correctness fixes (IDR-062, IDR-040, IDR-084, Astra R1/R3); +- hot-path contention fixes (Astra R2/R3), the OKX session fixes and the ingestor liveness knob; +- the G5 warmup fixes; +- the Stream replica byte budget fix (a shared decoded record is charged once) and the provider admission + INFLIGHT_CAPACITY wait fix; +- the funding settlement publication allowance (60 s, now backed by a measurement); +- additive public fields: `event_id` (IDR-082) and `proof_observed_at_ns` (IDR-040 fix 2), in SDK 2.2.1. + +Public endpoints keep their versions. Every change is additive, and readers on SDK 2.1.0 ignore the new fields. + +This is not a Trading System order, accounting or alpha certification. It is not HA or independent-failure-domain +evidence, because every run used the same host. + +## Source And Images + +- Release source: `413c5b914f8c6fd0f67429fdddb39451589e9745` on `fix/p184-dl-closure`. +- The only later commit, 2f489b1, changes scripts, tests and the plan journal (`git diff --stat 413c5b9 2f489b1`: + `scripts/p184_g6_ts_execution_probe.py`, `scripts/phase3_consumer_load_acceptance.py`, + `tests/test_p184_g6_view_quality.py`, the plan). It is not in an image. +- Commits since the previous evidence source bbfb3be (`git log --oneline bbfb3be..2f489b1`): + - runtime code: 2e0de95 (Query funding allowance re-applied with measurement), dadb9c8 (Stream charge-once + budget), 70bef7d (SDK 2.2.1 random stream replica start), 413c5b9 (admission INFLIGHT_CAPACITY poll); + - tooling: 259aeb9 (local CI runner), 627f19d, 7188b8b, 8bbcaae (G12 harness), 848d5d2, 65ada37 (campaign tooling + and G1-G14 scorer), ba496ff (G6 scorer and TS EXECUTION probe), 2f489b1 (G6 anchor correction); + - journal and evidence: 163191c, e7b2028, 59e725a (previous release evidence), 2d11a87, 6d5da7f. + +Images were checked with `docker image inspect`. Both revision labels equal 413c5b9, which closes IDR-039. The +version label reads `2.2.3-rc10`, because a tag never changes a digest. + +| Image | Digest | Tag | Roles | +| --- | --- | --- | --- | +| Python | `sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb` | `qdl-v2-python:2.2.3-rc10-413c5b9` | query_kn_1/2, binance_bar_edge | +| Rust | `sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5` | `qdl-v2-rust:2.2.3-rc10-413c5b9` | stream_kn_1/2, market_projector_1/2, rust_core x3, ingestor_okx_swap, ingestor_binance_usdm | + +All 12 application roles run these two digests: restart count 0, OOM false (`docker inspect`, 2026-10-11). They were +rolled one role at a time between 04:30:01Z and 04:35:15Z on 2026-10-11, with the writer READY 64/64 after every +step (receipts `p184-d3-capacity-20261009/rollout-rc6/-receipt-20261011T*.json`). The bar edge already ran the +rc10 Python image since 2026-10-10T18:46Z. + +The following were not changed by this release: +- kafka1-3 (`sha256:9516fb7634ba`), apart from the dynamic broker config below; +- market_cache and stable_redis (`sha256:dfa18828cbc0`); +- the one-shots stable_state_init and stable_tls_init (`sha256:bd0163fd76b0`). + +Ingestor configs are unchanged from the previous record: OKX revision 20, Binance revision 19, liveness 1500/4000 ms, +no `book_liveness_proof_ms`, so IDR-040 fix 2 is OFF. + +TS writer `market_data_service_p184w` (TS repo `trading-system-next-upgrade`, pushed): +- image `sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c`, TS `8b567eb`, SDK 2.1.0; +- env during campaign 4: `DATA_LAYER_V2_STREAM_PIPELINE_DEPTH=1`, `MARKET_DATA_FIXED_WORKER_PINS=OKX:BTCUSDT=2,OKX:ETHUSDT=0`, + `DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE=BINANCE=100` (from the writer inspect taken before the post-campaign + roll, `trading-system/p184-rev11-window/campaign-roll/inspect-20261011T042442Z.json`, sha256 `ee6c5ff4...`); +- since 2026-10-11T04:24:53Z the writer runs depth 0 with the same pins and MARK delay (`docker inspect`). Depth 0 + was the coordinator's choice from two comparisons; that comparison is not re-verified in this record. + +## Runtime Config Change: Kafka Log-Cleaner Throttle + +- Change: cluster-wide dynamic default `log.cleaner.io.max.bytes.per.second=8388608` (8 MiB/s), applied + 2026-10-11T03:40:56Z without a broker restart. The owner approved it through a question at about 03:40Z. +- Cause: compaction of `md.latest.v2` on kafka1 (CPU limit 1.25 vCPU, `docker inspect` NanoCpus) pinned the + broker. While it ran, every rust_core transaction commit stalled about 2.06 s, at 14:12, 15:29 and 19:09Z on + 2026-10-10. The 19:09Z stall is behind the G1/G2 max of campaign 20261010T185053Z. +- Receipt: `~/.local/state/qdl-v2/kafka-cleaner-throttle-20261010/` (README.md `72cbcf87...`, alter.out + `a59f5deb...`, after.out `8a635a9b...`, applied_at.txt `5f94630c...`). +- Rollback, failure only: `kafka-configs.sh --alter --delete-config log.cleaner.io.max.bytes.per.second`. + +## Gate Scorecard (G1-G14) + +Gates come from the frozen TS table `upgrade/p184-phase-d-gates.md` (TS 31cf663, sha256 `c67bf69b...`) and the +frozen budget `config/v2/p184-stage100-budget.json` (sha256 `ff85335b...`). + +Scoring rule: a PASS needs the table's measurement rule, which is steady state for at least 30 min at the stage-100 +workload plus a burst window. A measurement outside that rule can FAIL a gate but cannot PASS it. A NOT_MEASURED is +never upgraded. No frozen gate was widened. + +The gate scorecard is final campaign 4, `~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/` (scorecard +sha256 `01336708...`, `protocol_met: true`, no protocol problems, load 03:44:05-04:14:05Z, 64 demanded slices, writer +restarts 0). G12 and G13 are scored from their own receipts on the release digests. + +Summary: +- PASS: G1, G3, G5, G6, G7, G9, G10, G11, G14. +- FAIL: G2, G8. +- NOT_MEASURED: G4, G12. +- DEFERRED: G13. + +Paths under `~/.local/state/qdl-v2/`; sha256 prefixes, full hashes in scorecard.json. + +| Gate | Rule | Measured | Source (sha256) | Verdict | +| --- | --- | --- | --- | --- | +| G1 BOOK e2e to TS cache | p95 <= 500, p99 <= 1000, max < 2000 ms | DELTA B/O p95 177.4/191.5, p99 334.1/380.8, max 1736.7/1844.2 (n 83994/97565); SNAPSHOT B/O p95 245.6/236.7, max 1211.4/1144.5 | campaign `trace/transit-load.json` (d1376db1), `trace/mark-load.json` (7ac870af) | **PASS** | +| G2 TRADE/QUOTE/MARK new-event transit | TRADE p95 <= 500, max < 1500; QUOTE/MARK p95 <= 500, max < 1000 ms | **TRADE BINANCE p95 1562.1, max 5338.8 (n 160328); TRADE OKX p95 2084.2, max 4301.8 (n 39292)**; QUOTE B/O p95 188.0/207.1, **max 1439.2/1340.6**; MARK B/O p95 396.7/348.5, **max 1125.4/1227.8** | same two files | **FAIL** | +| G3 BAR e2e from close | p95 <= 5 s, max < 60 s | pooled n 320, p95 1535, max 3007 ms; 0 unseen, 0 read errors | `g3/g3-summary.json` (a15a368f) | **PASS** | +| G4 venue -> canonical | candidate p95 <= baseline p95 x 1.2 per row | 17/17 baselined rows PASS in every window they were sampled (e.g. trade B/O 108.2-133.2 / 110.8-136.6 vs limits 269.3/257.4). 10 observed rows have no frozen baseline: bar BINANCE 2h/4h/5m, bar OKX 15m/4h/5m, OKX_INVERSE book_delta/book_snapshot/mark/quote | `g4-{1..5}-summary.json` (02a554c5, 42aa78a1, 013e6929, 86494b65, 0111526b) | NOT_MEASURED: rows without baseline | +| G5 request latency | certified p95/p99 per class and venue; frozen cold limits | All 10 class/venue rows PASS (worst p95 95.0 OKX L2, worst p99 293.8 BINANCE L2); cold 2500/5000 overlap 4 runs, 0 bad | `stage100/gates.json` (63f12e8c), `receipt.json` (a79c9c9d) | **PASS** | +| G6 durable-view event age + quality | per-binding D3 policy of the TS EXECUTION consumer; zero unsafe | TS EXECUTION probe 19121 reads over 32 targets: every D3 binding measured, 0 unsafe (e.g. MARK OKX 3563 ELIGIBLE + 25 ELIGIBLE_QUIET; QUOTE OKX 3515 + 68; TRADE NOT_ELIGIBLE B/O 49/463, typed). Alpha reads: 0 unsafe under their own requirement | `g6-ts/summary.json` (229e0380) | **PASS** | +| G7 TS consumer path | SDK -> writer p95 <= 100; writer -> Redis p95 <= 50 ms per feed | SDK -> writer p95: BOOK_DELTA 51.0, BOOK_SNAPSHOT 79.3, QUOTE 40.7, TRADE 83.0, MARK 0.6. Writer -> Redis p95: 18.4, 41.2, 10.5, 46.7, 20.2 | `trace/transit-load.json` (d1376db1) | **PASS** | +| G8 TS execution Redis | commandstats <= 60 % core per 10 min; SLOWLOG 0 in steady state; PING p99 <= 5 ms | commandstats <= 0.0868 core; PING p99 <= 1.675 ms per window (n 3850, 0 failures); **SLOWLOG 6 new EVALSHA entries, 10.0-19.7 ms**; 0 MONITOR clients | `g8/g8-summary.json` (2b3375cf) | **FAIL** | +| G9 readiness | demanded == session-ready; 0 data-path eligibility losses; v1 fallback 0 | 180/180 samples readable, demanded 64 == session-ready in every sample, 0 data-path losses, 24 legitimate quiet TRADE refusals, v1 fallback 0 | `g9g10/g9-g10-summary.json` (3015c2db) | **PASS** | +| G10 TRADE integrity | 0 gaps, 0 dup applied, 0 replay-expired, 0 overflow | all 0; 2 RECONNECTED resumes; 129 log polls, 0 unreadable | same file and trace | **PASS** | +| G11 writer restart recovery | first valid apply <= 30 s; caught up <= 60 s | outage 0.121 s; 64/64 slices caught up, p50 3.919, max 18.923 s; 0 rewinds; cursor never ahead | `g11/g11-summary.json` (c59cccf3) | **PASS** | +| G12 broker/projector/Stream recovery | caught up <= 600 s, then G1-G3 | Isolated, release digests: broker stop/start 22.334 s, projector restart 0.935 s, Stream restart 2.294 s; G1 300 s windows after each inside (max 1970.0 ms OKX after the broker fault). G2/G3 after recovery: NOT_MEASURED (BOOK-only source) | `p184-g12-rc10-20261011-a2/evidence/verdict.json` (05c0652d), `teardown.json` (f17de5ca) | **NOT_MEASURED** (catch-up and G1 inside; G2/G3 after recovery not measured) | +| G13 DR01-DR08 | every cell once on the final digests | 41/41 DL cells ok on the release digests: dr05-upstream 6, dr08 4, dr05-transport 9, views 22 (2 quiet cells NOT_OBSERVED). 4 TS steps deferred | `p184-g13-20261011T043821Z/summary.json` (3a3a34e2), `...T045533Z/summary.json` (309199f1) | DEFERRED | +| G14 resources | CPU/RAM per role at stage 100, with headroom | 18/18 roles, 151 samples, none at or over its memory limit. Mean CPU: kafka 81/98/102 %, query 57/66 %, writer 151 %; DL mean sum 5.37 vCPU. Lowest memory headroom query_kn_1 14.3 % | `stage100/host.json` (06eab7ba) | **PASS** | + +Measurement identity. Campaign 4 ran before the rc10 roll, on this mix: +- query_kn_1/2: rc8 `sha256:db5ae6752280` (6d5da7f). `git diff --stat 6d5da7f 413c5b9`, without scripts, tests and + journal: `qdl/admission/edge.py` +9, `qdl_sdk` (2.2.1 version string and stream transport), the Stream gateway + Rust source (not run by Query) and a contracts doc. Service code imports only `qdl_sdk.models` and + `qdl_sdk.reference`. **Not source-identical**: reads that call a provider through `BlockingProviderAdmission` + (OKX admitted REST and reference, Binance admitted klines) differ when an INFLIGHT_CAPACITY deferral occurs. The + other Query paths are source-identical. +- stream_kn_1/2: rc9 `sha256:d32e354bd662` (70bef7d). The only diff to 413c5b9 is `qdl/admission/edge.py`, which + is not in the Rust build. Rust tree, `generated/rust`, Cargo files and config are unchanged: source-identical. +- binance_bar_edge: the rc10 release image itself. +- rust_core x3, projectors and ingestors: rc7d `sha256:64468c72ce5a` (bbfb3be). The Rust diff to 413c5b9 is + `rust/qdl-stream-gateway` only. No other crate depends on it, and Cargo files, `generated/rust`, config and + Dockerfile are unchanged: source-equivalent for these binaries (rebuilt, not byte-compared). + +G12 and G13 ran on the release digests themselves. + +Findings behind the verdicts: +- **G2 FAIL: writer throughput, not the DL canonical path.** Read-only per-second analysis of the campaign-4 + delivery trace (214 parts, the scorecard load window): + - 1699 of 1801 seconds have max end-to-end < 500 ms; 28 seconds exceed 1500 ms; + - in 25 of those 28 seconds every record over 1500 ms is TRADE; the other three also hold BOOK_DELTA records over + 1500 ms (G1 max stays below 2000 ms); + - the largest burst is 4274 TRADE records in one source second on one slice (BINANCE ETHUSDT), all on one writer + worker; the most a single worker applied in one second is 1138 TRADE records; + - TRADE BINANCE segment p95: venue -> ingest 39.6, ingest -> SDK return 1430.7, SDK -> Redis reply 118.2 ms; + - G4 in the same campaign puts venue -> canonical TRADE p95 at 108.2-136.6 ms in all five windows, and G10 has 0 + overflow, so the tail sits behind canonical visibility, in single-slice TRADE bursts queued at one writer + worker; + - TRADE volume was about 2.2x the earlier campaigns (TRADE BINANCE n 160328 against 69966 and 74061). + - The Stream-side dispatch and queue metrics for this campaign were not kept in a file, so they are not cited. +- **G8 FAIL**, same cause. All six SLOWLOG entries are writer APPLY EVALSHA calls in TRADE burst seconds. The + TRADE volume peak within +-1 s of each entry is at or above the 93rd percentile of per-second TRADE volume (median + 46, p95 428 applied records per second). Command CPU and PING stay inside. +- **Owner decision, 2026-10-11 (AskUserQuestion):** release the DL. Writer throughput (SDK batch decode, Rust + projection, EVAL chunking) is the first item of TS phase E. The gates are not widened, and the FAILs stay FAILs. +- G4 stays NOT_MEASURED only because of rows without a frozen baseline. No baseline is added after measuring. +- G4 bar latency for OKX is measured from bar open (the OKX source time is the open). The baseline used the same + metric. + +## Earlier Final Campaigns (2026-10-10 .. 2026-10-11) + +Paths under `~/.local/state/qdl-v2/`. None of these results is erased by campaign 4. + +| Campaign | Setup | Result | What it led to | +| --- | --- | --- | --- | +| `p184-final-campaign-20261010T171129Z` (94f30f85) | writer `sha256:ace42326` depth 1; Query rc8, rest rc7d | protocol met; FAIL G2, G6 | Stream replica byte-budget overflow at 17:22:48Z (84 streams closed BACKPRESSURE with empty own queues) -> dadb9c8. G6 FAIL was a scorer defect -> ba496ff | +| `p184-final-campaign-20261010T174944Z` (1c6a5d30) | same, writer depth 0 | protocol met; FAIL G2, G4, G6, G8 | G4 bar BINANCE 3m/30m at 18:00Z: admission slept the INFLIGHT_CAPACITY lease-expiry hint -> 413c5b9 | +| `p184-final-campaign-20261010T185053Z` (ac0b3f66) | writer bae2509f depth 0; Query rc8, Stream rc9, bar edge rc10, rest rc7d | protocol NOT met (startup:bounded FAIL); FAIL G1, G2, G4 | Compaction stall at 19:09Z -> Kafka cleaner throttle. G6 funding anchor took the oldest settlement row -> 2f489b1 | +| `p184-final-campaign-20261011T033641Z` (c3535323) | writer bae2509f depth 1 | aborted about 3 min into the load (driver SIGINT; the coordinator's watcher restarted the writer once when G11 armed during the shutdown) | stopped to apply the Kafka cleaner throttle (03:40:56Z) | + +`startup:bounded` across the five stage-100 runs (stage 100m and the four driver runs above): one FAIL, in +20261010T185053Z, where session s89 exhausted the frozen 10 retries (`retry_max_attempts` 10, retry code +RATE_LIMITED; gates.json `8a0db47a...`). The frozen budget is kept. + +## G12 And G13 On The Release Digests + +G12, isolated (owner approved), `p184-g12-rc10-20261011-a2`: +- `status COMPLETED`, `pass true`. Image set: core/projector/Stream rust `96a2c912`, python `a0c3649d`, writer + `bae2509f`. +- Caught up after the fault cleared: broker stop/start 22.334 s (OKX 2.177 s), projector restart 0.935 s, Stream + restart 2.294 s, all <= 600 s. +- G1 in the 300 s window after each fault: inside; the largest max is 1970.0 ms (OKX, after the broker fault). +- Teardown: 12 containers removed, 0 left by label or name, volumes 217 -> 217. +- Scope: same host, TEST_ONLY BOOK source, single projector/Stream replica; G2/G3 after recovery NOT_MEASURED. + +Failed and stopped G12 attempts, kept: +- `p184-g12-rc10-20261011` FAILED on a harness STOP: `RuntimeError: STOP role exited: ...-monitor`. The monitor's + last record is 3.9 s before the first fault injection (kafka-2 stop), and the STOP fired 27 ms after the + injection. No traceback or OOM was retained, the same symptom as attempt a3 on 2026-10-10 (verdict `728b460d...`; + teardown 10 removed, 0 left, volumes 217 -> 217). +- `p184-g12-rc10-20261011-r2` was a duplicate run started alongside a2. The coordinator stopped it with SIGTERM. + Its teardown removed 5 containers and left 0 by label or name. It read volumes 217 -> 219 while a2 held its own + volumes; a2's teardown later read 217. +- Open harness item: keep the monitor container's exit code and OOM state in the evidence. + +G13, on DL 413c5b9 and writer bae2509f (TS 8b567eb): +- `p184-g13-20261011T043821Z`: dr05-upstream 6/6, dr08-isolated 4/4 (QUALIFICATION), dr05-transport 9/9 ok. +- Its dr04-dr06-views step failed on DR04.EXPLICIT_LAST@BINANCE with `EXECUTION_CONTEXT_STALE`. The live LAST was + older than the probe limit in a quiet Sunday market, so the refusal is correct and typed. The age sample behind + this reading was not kept in a file. The rows are kept (`views-binance.rows`, `22e943b9...`). +- `p184-g13-20261011T045533Z`: dr04-dr06-views 22/22 ok. DR05.QUIET_TYPED BINANCE/OKX are ok but NOT_OBSERVED, + because no book was quiet in the 300 s window. +- Runtime approval: the owner's AskUserQuestion answer of 2026-10-11. The file + `p184-g13-inputs/runtime-approval-rc10.md` (`55189088...`) was written by an agent from that relayed approval. +- Deferred to the TS release: dr01-deployed, dr01-wrong-account, dr02-dr03-alpha, dr07-reference. + +## Items Closed (commit) + +| Item | Commit(s) | State | +| --- | --- | --- | +| IDR-039 one-commit images for every application role | rc10 rollout of 413c5b9 | **closed**: 12/12 roles on 413c5b9 | +| Stream replica byte budget: shared decoded record charged once; per-subscription cap total/2 | dadb9c8 | in images | +| Provider admission: INFLIGHT_CAPACITY deferral polls 50-200 ms instead of sleeping the lease expiry | 413c5b9 | in images | +| Funding settlement publication allowance 60 s, backed by the 2026-10-10 16:00Z poll | 2e0de95 | in images | +| SDK 2.2.1: stream transport starts on a random replica per instance | 70bef7d | wheel below | +| G6 scorer: read-own-requirement, D3 by a TS EXECUTION probe; anchor per product and grade | ba496ff, 2f489b1 | tooling | +| Gate-campaign tooling G3/G8/G9/G10/G11/G14 and the G1-G14 scorer | 848d5d2, 65ada37 | tooling | +| G12 harness: image set as a parameter, topic metadata retry, signal teardown, in-container stderr | 627f19d, 7188b8b, 8bbcaae | tooling | +| Kafka log-cleaner I/O throttle 8 MiB/s | runtime config, receipt above | runtime | +| Items closed in the 2026-10-10 record (IDR-062, IDR-040 fixes 1-2, IDR-002/029, D2, IDR-075, IDR-076, IDR-082, IDR-084, Astra R1-R7, Codex review items, OKX fixes, Stream terminal errors, cursor base64, DR05 fence, CI convergence) | see appendix | in images | + +The funding allowance revert 530a406 listed in the 2026-10-10 record is superseded by 2e0de95. + +## SDK 2.2.1 Wheel + +SDK 2.2.0 is superseded. It was never published, but its wheel hash is in the 2026-10-10 record, so the version was +bumped rather than reused. + +| Field | Value | +| --- | --- | +| wheel | `qdl_sdk-2.2.1-py3-none-any.whl` | +| wheel sha256 | `27db39715c58988128dfec3364919a3ef3bb9425e9bdb4cb7655829ee3f1076f` | +| reproducible | two builds byte-identical (`cmp`); wheel, SBOM and release json hashes equal | +| sdk_source_digest | `fcbd832c1a3e89c66ad625e488ae2a4f80f7baa895abb7b8904febfcb54aa217` | +| generated_contract_digest | `ae77dc42074fd7cc896a46bde3282837a51b667acee23c4957d37565690856e9` (unchanged from 2.2.0) | +| release json sha256 | `d2ae9ef67e334540e3cef4d0e1e1b17896762262263e4d6d4fadfeb518c894e6` | +| SBOM sha256 | `e80dcec6ce588d715e7e5d6bf3aa5449426c80d999f8d69647081047398d9111` | + +The artifacts are under `~/.local/state/qdl-v2/releases/v2.2.3/sdk221-build-{1,2}/`, outside Git. The TS writer runs +SDK 2.1.0. + +## CI + +Local exact-source CI (`scripts/p184_local_ci.py` replaying `.github/workflows/ci.yml` on an archive of 2f489b1): +- all jobs except unit-tests, summary `p184-local-ci-2f489b1/summary.json` (sha256 `f00a9423...`, receipt status + `INCOMPLETE`): + - consolidated-source-preflight, kn-native-integration and sdk-python310 PASS; + - contract-tests 7 PASS, 3 SKIPPED: #2 and #5 are pull-request-only steps; #10 (Rust dependency, license and + advisory policy) needs a host download, which the runner does not do; +- unit-tests, summary `p184-local-ci-2f489b1-unit/summary.json` (sha256 `98c6c157...`, status `INCOMPLETE`): + 19 PASS, 3 SKIPPED: + - #4 needs sudo (reported, never escalated); + - #19 and #20 (trivy image scan, trivy secret/misconfiguration scan) had no local trivy image; +- the unit-tests re-run with `--trivy-image aquasec/trivy:0.58.1` (`p184-local-ci-2f489b1-unit-r2`, summary sha256 + `f6b6b5e7`): every step PASS, including #19 and #20 (trivy scans); only #4 (sudo) SKIPPED. + +Remote GitHub CI did NOT run, because the DL repository is not pushed. Publication requires the required jobs on +the final head. + +## Runtime Changes Performed (2026-10-10 .. 2026-10-11) + +Each change was owner-approved. Each role was recreated one at a time. + +| Change | Rollback reference | +| --- | --- | +| Query rc8 (6d5da7f, funding allowance) on query_kn_1/2, 2026-10-10 17:09Z | rc7d python `sha256:e0252f4c96da` | +| Stream rc9 (70bef7d, charge-once budget) on stream_kn_1/2, 18:44-18:45Z | rc7d rust `sha256:64468c72ce5a` | +| bar edge rc10 python (413c5b9), 18:46Z | rc7d python `sha256:e0252f4c96da` | +| TS writer images for the campaigns (ace42326, then bae2509f 8b567eb) and env depth/pins | `trading-system/p184-rev11-window/campaign-roll/rollback-*.compose.json` | +| Kafka dynamic default `log.cleaner.io.max.bytes.per.second=8388608`, 2026-10-11T03:40:56Z | `--delete-config log.cleaner.io.max.bytes.per.second` | +| TS writer depth 1 -> 0 after campaign 4, 04:24:53Z | `campaign-roll/rollback-20261011T042442Z.compose.json` (e9c22775) | +| **rc10 (413c5b9), all 12 application roles, 04:30-04:35Z 2026-10-11**, writer READY 64/64 after every step | `p184-d3-capacity-20261009/rollout-rc6/-rollback.compose.json`: Query -> rc8 `db5ae6752280`, Stream -> rc9 `d32e354bd662`, projectors/cores/ingestors -> rc7d `64468c72ce5a`, bar edge -> rc10 (unchanged) | + +Rollback runs in reverse role order. Kafka data, Redis, realm dirs and offsets need no data migration. Rollback is +for failure only. The 2026-10-08 .. 2026-10-10 changes are listed in the appendix. + +## Known Limits + +- **G2 TRADE and G8 SLOWLOG FAIL: writer throughput under TRADE bursts.** Owner decision: TS phase E, first item. + The gates are not widened. +- **DL CPU above the owner's 5 vCPU budget:** G14 passes its own per-role headroom rule, but the DL mean sum at + stage 100 was 5.37 vCPU in campaign 4 (`stage100/host.json`), above the owner's DL budget of 5 vCPU. Kafka is the + largest share. Tuning it is open: one variable at a time, measured against G1/G2. +- **Single host:** every run used the same host. No HA and no independent failure domains. No mainnet or funded + claim. +- **G4 rows without a frozen baseline** (bar BINANCE 2h/4h/5m, bar OKX 15m/4h/5m, OKX_INVERSE book_delta, + book_snapshot, mark and quote) keep G4 NOT_MEASURED. No baseline is added after measuring. +- **G6 BOOK_DELTA** is not a Query shape in the frozen workload, so it has no G6 row. +- **startup:bounded** is a stochastic tail of the warmup lane: one session exhausted its 10 retries in one of five + stage-100 runs. The frozen budget is kept. +- **G12 scope:** isolated, TEST_ONLY BOOK, single replica. G2/G3 after recovery are not measured. The harness does + not yet keep the monitor's exit code and OOM state. +- **G13:** the four TS-stack steps stay OPEN. The two DR05.QUIET_TYPED cells are NOT_OBSERVED, not a positive + quiet-state proof. +- **Measurement identity:** campaign 4 ran the pre-rc10 mix. Query is not source-identical to 413c5b9 on provider + admission waits (see above). +- **IDR-040 fix 2 is OFF** (`book_liveness_proof_ms` unset). A quiet Binance book stays stale, never falsely fresh. +- **Catalog revision split (C.19)** is unchanged: image catalog revision 11, runtime-mounted revision 12. +- **CI:** local only; contract-tests #10 was skipped (needs a download), #4 needs sudo, and remote CI has not run. +- Historical failures stay recorded: the four earlier final campaigns above, the failed G12 attempt, the first G13 + views run, and every failure listed in the appendix. None is erased by a later PASS. + +## Consumer Hand-Off (TS phase E) + +- Adopt SDK **2.2.1** (wheel sha256 `27db3971...`). The writer runs 2.1.0 today, which ignores the new fields. +- Alpha callers mint JWTs for manifest revision **15** (4950 RPM / 300 streams). An older JWT revision gets 401. +- TS Paper manifest/JWT revision 11 (64 products incl. OKX BTC-USD-SWAP) is live. +- Deploy the IDR-085 gateway fix (TS be21d49) with the TS release. That closes the deferred DR01 cells. +- IDR-040 fix 2: activate the in-band quiet-book proof flag through its own packet when TS needs it. +- Use `event_id` (IDR-082) for trade identity. +- Close the four deferred G13 steps: dr01-deployed, dr01-wrong-account, dr02-dr03-alpha, dr07-reference. +- Writer throughput, first item: SDK batch decode, Rust projection, EVAL chunking. G2 and G8 are re-scored after it + against the same frozen gates. +- The writer keeps `BINANCE=100` for the MARK wake read delay and the OKX worker pins. + +## Publication Requires + +1. The owner's publication decision. The 2026-10-11 release decision records G2/G8 as known limits; it does not turn + them into PASS. +2. Push, remote CI (contract-tests, sdk-python310, unit-tests, kn-native-integration) on the final head, and merge + to dev and main. +3. An annotated tag `v2.2.3` that links the final CI runs and the hashes of scorecard.json and this file. + +No push, merge or tag was performed. + +## Appendix: Preparation History + +### 2026-10-10 record at bbfb3be (verbatim, superseded by the record above) + +Status: `PREPARED_NOT_PUBLISHED`. This is not a published release, a Git tag or an owner publication decision. +Publication needs the owner to push and tag. Remote CI must also run on the final head. The published stable +release is still v2.2.2. + +Scorecard: [scorecard.json](scorecard.json). Inventory, hashes and the SDK build: [release-preparation.json](release-preparation.json). + +#### Scope + This release closes the P18.4 phase D Data Layer work on the Kafka-native stack. It contains: - one-commit images for every application role (IDR-039); - execution-view correctness fixes (IDR-062, IDR-040, IDR-084, Astra R1/R3); @@ -20,7 +343,7 @@ Public endpoints keep their versions. Every change is additive, and readers on S This is not a Trading System order, accounting or alpha certification. It is not HA or independent-failure-domain evidence, because every run used the same host. -## Source And Images +#### Source And Images - Release source: `bbfb3be6b3f1b7f83b44022e77439daeb4999a65` on `fix/p184-dl-closure`. Its code tree equals 2c84e2f plus the plan journal. @@ -60,7 +383,7 @@ read-only at `/runtime/config.json` (checked with `docker inspect`) and read thr Neither config contains `book_liveness_proof_ms`, so IDR-040 fix 2 is OFF. -## Gate Scorecard (G1-G14) +#### Gate Scorecard (G1-G14) Gates come from the frozen TS table `upgrade/p184-phase-d-gates.md` (TS 31cf663, sha256 `c67bf69b...`) and the frozen budget `config/v2/p184-stage100-budget.json` (sha256 `ff85335b...`). @@ -109,7 +432,7 @@ Findings behind the verdicts: - G4 bar latency for OKX is measured from bar open (the OKX source time is the open). The same metric was used for the baseline. -## Items Closed (commit) +#### Items Closed (commit) | Item | Commit(s) | State | | --- | --- | --- | @@ -139,7 +462,7 @@ Findings behind the verdicts: | CI convergence (cargo fmt, DL-only test skips, vnstock opt-in) | d6fb34c, 2c84e2f, 54460ac | in images | | SDK 2.2.0 release (start_index, event_id, proof_observed_at_ns, item diagnostics) | 359c9ee, a2184f4 | wheel below | -## SDK 2.2.0 Wheel +#### SDK 2.2.0 Wheel Built by `scripts/build_qdl_sdk_release.py`. The builder image was `sha256:e0252f4c...`, run with `--network none --cpus 1` as a non-root user. The source was a `git archive` of bbfb3be, extracted separately for @@ -156,7 +479,7 @@ each build. All 21 wheel files are byte-identical to the deployed Python image (`/app/qdl_sdk`, `/app/generated/python/qdl`). The artifacts are under `~/.local/state/qdl-v2/releases/v2.2.3/sdk-build-{1,2}/`, outside Git. -## CI +#### CI Local exact-source CI (`scripts/p184_local_ci.py`, archive of bbfb3be), first run: - summary sha256 `23a49251...`; @@ -174,7 +497,7 @@ Rerun of the unit-tests job with the fixed runner: Remote GitHub CI did NOT run, because the DL repository is not pushed. Publication requires the required jobs on the final head. -## Runtime Changes Performed (2026-10-08 .. 2026-10-10) +#### Runtime Changes Performed (2026-10-08 .. 2026-10-10) Each change was owner-approved and recorded in `DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md`. Each role was recreated one at a time. @@ -194,7 +517,7 @@ recreated one at a time. Rollback runs in reverse role order. Kafka, Redis, realm dirs and offsets need no data migration. Rollback is for failure only. -## Known Limits +#### Known Limits - **G12 is isolated and not inheritable.** The run was same-host, single projector/Stream replica, TEST_ONLY BOOK at 4 Hz, with G2/G3 NOT_MEASURED. It ran predecessor images: @@ -241,7 +564,7 @@ failure only. rc7 Query crash and the first G13 attempts. None is erased by a later PASS. - Same host throughout: no HA, no independent failure domain, no mainnet or funded claim. -## Consumer Hand-Off +#### Consumer Hand-Off - **Alpha callers** mint JWTs for manifest revision **15** (alpha.binance/okx.paper.stable rev 15, 4950 RPM / 300 streams). The readers run rev 15, so an older JWT revision gets 401. @@ -254,7 +577,7 @@ failure only. - The writer keeps `BINANCE=100` for the MARK wake read delay. - The G2 TRADE tail between canonical visibility and SDK return is open. It must be attributed before G2 can PASS. -## Publication Requires +#### Publication Requires 1. The owner resolves the G2 FAIL and the NOT_MEASURED gates, or explicitly accepts them as release limits. This record does not accept them. @@ -265,9 +588,7 @@ failure only. No push, merge or tag was performed. -## Appendix: Preparation History - -### 2026-10-08 HOLD draft (verbatim, superseded by the record above) +### 2026-10-08 HOLD draft (verbatim, superseded by the records above) Status: `PREPARED_RELEASE_HOLD_NOT_PUBLISHED`. This record is not a certificate, Git tag, deploy authorization or proof that the feature HEAD runs on production. diff --git a/upgrade/evidence/releases/v2.2.3/release-preparation.json b/upgrade/evidence/releases/v2.2.3/release-preparation.json index 59e135ad..a130f0a1 100644 --- a/upgrade/evidence/releases/v2.2.3/release-preparation.json +++ b/upgrade/evidence/releases/v2.2.3/release-preparation.json @@ -11,34 +11,63 @@ }, "ci": { "local_full_run": { - "failure": "unit-tests step 19 trivy image scan only: local runner flag bug (boolean ignore-unfixed, .trivyignore mount), fixed in 259aeb9; steps 20-21 skipped after it", + "contract_tests_skips": [ + "#2 and #5 pull-request-only steps (condition false for a push event)", + "#10 Rust dependency/license/advisory policy: downloads a tool on the host (--allow-download not given)" + ], "jobs": { "consolidated-source-preflight": "PASS", - "contract-tests": "PASS", + "contract-tests": "INCOMPLETE (7 PASS, 3 SKIPPED)", "kn-native-integration": "PASS", "sdk-python310": "PASS", - "unit-tests": "FAIL" + "unit-tests": "INCOMPLETE (run separately, below)" }, - "sha256": "23a49251af3f853f4fd8a5f640085c58f1475d12841a5f827c4db35c93759e19", - "status": "FAIL", - "summary": "~/.local/state/qdl-v2/p184-local-ci-bbfb3be/summary.json" - }, - "local_unit_tests_rerun": { - "passed": 21, - "runner": "259aeb9 on archive of bbfb3be", - "sha256": "07df25eb2578f7e68f08fba930e48262bf6cfbaea30d1bd4b3704e70d46d4071", - "skip": "Prepare non-root runtime bind mounts (requires sudo; reported, never escalated)", - "skipped": 1, + "sha": "2f489b1", + "sha256": "f00a94234118ecd74a82f821ab16c378d04f2b484473bc674ed91976291db802", + "status": "INCOMPLETE", + "summary": "~/.local/state/qdl-v2/p184-local-ci-2f489b1/summary.json" + }, + "local_unit_tests": { + "passed": 19, + "sha256": "98c6c157e9c3f446dee7161f43dbba1576d0915591a8217b3bb1a874be7ceaed", + "skipped": 3, + "skips": [ + "#4 Prepare non-root runtime bind mounts (requires sudo; reported, never escalated)", + "#19 trivy image scan and #20 trivy secret/misconfiguration scan: no local trivy image" + ], "status": "INCOMPLETE", - "summary": "~/.local/state/qdl-v2/p184-local-ci-bbfb3be-r2/summary.json" + "summary": "~/.local/state/qdl-v2/p184-local-ci-2f489b1-unit/summary.json" }, - "remote_github_ci": "NOT_RUN (DL repository not pushed)" + "local_unit_tests_trivy_rerun": { + "status": "PASS except #4 SKIPPED (requires_sudo)", + "trivy_image": "aquasec/trivy:0.58.1", + "summary": "~/.local/state/qdl-v2/p184-local-ci-2f489b1-unit-r2/summary.json", + "summary_sha256": "f6b6b5e772902541c0171557cb35835583a67d98ca7e2ac645ca239cb8ca49ea" + }, + "remote_github_ci": "NOT_RUN (DL repository not pushed)", + "runner": "scripts/p184_local_ci.py replaying .github/workflows/ci.yml on an archive of 2f489b1" }, "commits_after_source": { + "2f489b1": "G6 reference anchor per product/grade (scripts, tests, plan journal; not in images)" + }, + "commits_since_previous_evidence_source_bbfb3be": { "163191c": "plan journal", - "259aeb9": "local CI runner trivy flag fix (tooling, not in images)", - "e7b2028": "plan journal", - "this_commit": "scripts/p184_trace_feed_transit.py + test, release evidence (not in images)" + "259aeb9": "local CI runner trivy flag fix (tooling)", + "2d11a87": "G12 packet journal", + "2e0de95": "Query: funding settlement publication allowance 60 s re-applied with measurement (runtime: Query)", + "2f489b1": "G6 anchor correction (tooling)", + "413c5b9": "admission: poll an INFLIGHT_CAPACITY deferral (runtime: provider admission in Query/bar edge)", + "59e725a": "previous release evidence + trace transit script", + "627f19d": "G12 harness image set as a parameter (tooling)", + "65ada37": "final gate-campaign tooling G3/G8/G9/G10/G11/G14 + G1-G14 scorer (tooling)", + "6d5da7f": "plan journal (rc8 Query image built from this tree)", + "70bef7d": "SDK 2.2.1: stream transport random replica start (SDK)", + "7188b8b": "G12 harness topic metadata retry, signal teardown (tooling)", + "848d5d2": "driver G6 view age/quality per binding; trace duplicate applies (tooling)", + "8bbcaae": "G12 harness keeps in-container stderr/faulthandler; monitor exit reason (tooling)", + "ba496ff": "G6 scored per the read's own requirement; TS EXECUTION probe (tooling)", + "dadb9c8": "Stream: charge a shared decoded record once against the replica budget (runtime: Stream)", + "e7b2028": "plan journal" }, "consumer_revisions": { "alpha_manifest_jwt_revision": 15, @@ -52,27 +81,68 @@ "dr07-reference" ], "funding_settlement_allowance": { + "commit": "2e0de95 (60 s, FUNDING_RATE history only)", + "evidence": { + "binance.jsonl": "1c6a51a6b3246e105d4f25b76ff6bf682760eaa41a216c28273719290f0d54c0", + "okx.jsonl": "66213ff359aaa7259d3a646259071589dda5cac25e3d7410b66dfd1e9ad7111d" + }, "evidence_dir": "~/.local/state/qdl-v2/funding-publication-20261010/", - "note": "coordinator appends the result", - "poll_window_utc": "2026-10-10T15:58Z-16:04Z", - "shipped": "no allowance (530a406 revert)", - "status": "PENDING" + "measured": "16:00Z settled row visible OKX BTC/ETH/SOL +10 s, Binance BTC/SOL/ETH +5/+10/+24 s (code comment of 2e0de95)", + "status": "SHIPPED" + }, + "g12": { + "evidence": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/evidence/verdict.json", + "sha256": "05c0652d320929a13073a2ff1eb57a319f5b4a5817f7dd4f917ca8ae914a48b8" + }, + "failed_attempt": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011/evidence/verdict.json", + "sha256": "728b460d8244a36260a428542e93fb6bfec44cd3cae107e684000a8464e1573e" + }, + "stopped_duplicate": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-r2/evidence/verdict.json", + "sha256": "603fe82c761b2633c326fee6b56ad0578710ac4599d0b27f95a9a059846a1ebf" + }, + "teardown": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/evidence/teardown.json", + "sha256": "f17de5ca9539848e581972fd276809d8b9e5fd3fb5972f848c795f60bd722bb8" + }, + "verdict": "PASS (isolated, release digests)" + }, + "g13": { + "approval_file": { + "path": "~/.local/state/qdl-v2/p184-g13-inputs/runtime-approval-rc10.md", + "sha256": "551890885673bb569119dc65c218deb0b561c59fec8071bea48a7699eb22d2ce" + }, + "approval_note": "written by an agent from the owner's relayed AskUserQuestion answer of 2026-10-11", + "runs": [ + { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/summary.json", + "sha256": "3a3a34e29715a4b67406b2b5e790bed0c9b8d7f0551498d43d0cd9f9c610a09d" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T045533Z/summary.json", + "sha256": "309199f1a24c8625ed98d13d572a79f2cbff754359c8f88ce9a21e9c175c3472" + } + ], + "verdict": "DEFERRED (41/41 DL cells ok; 4 TS steps deferred)" }, + "idr_039": "CLOSED: all 12 application roles run images whose revision label is 413c5b9", "images": [ { - "digest": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "digest": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", "roles": [ "query_kn_1", "query_kn_2", "binance_bar_edge" ], - "tag": "qdl-v2-python:2.2.3-rc7d-bbfb3be", - "version_label": "2.2.3-rc7d" + "tag": "qdl-v2-python:2.2.3-rc10-413c5b9", + "version_label": "2.2.3-rc10" }, { - "digest": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "digest": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", "roles": [ "stream_kn_1", "stream_kn_2", @@ -84,8 +154,8 @@ "ingestor_okx_swap", "ingestor_binance_usdm" ], - "tag": "qdl-v2-rust:2.2.3-rc7d-bbfb3be", - "version_label": "2.2.3-rc7d" + "tag": "qdl-v2-rust:2.2.3-rc10-413c5b9", + "version_label": "2.2.3-rc10" } ], "ingestor_configs": { @@ -109,7 +179,22 @@ }, "mounted_read_only_at": "/runtime/config.json" }, + "kafka_runtime_config": { + "applied_at": "2026-10-11T03:40:56Z", + "approval": "owner AskUserQuestion, 2026-10-11 ~03:40Z", + "cause": "md.latest.v2 compaction on kafka1 (NanoCpus 1.25e9) stalled every rust_core commit ~2.06 s at 14:12, 15:29 and 19:09Z on 2026-10-10", + "change": "cluster-wide dynamic default log.cleaner.io.max.bytes.per.second=8388608 (8 MiB/s); no broker restart", + "receipt": { + "README.md": "72cbcf8725fa14df710b571e88149089bf36b1d320fe494e81ed3479102621b7", + "after.out": "8a635a9bd5df3efc07906db02a0fdb45cc799ac502929b46d9680b8ee884302c", + "alter.out": "a59f5deb54f0117897bc08637af9a82ee9b04577953061225471e2e827c0fdc6", + "applied_at.txt": "5f94630c679ebaf17bc248b71b617e00082277a03862ed3ad4e32fab89e76223" + }, + "receipt_dir": "~/.local/state/qdl-v2/kafka-cleaner-throttle-20261010/", + "rollback": "kafka-configs.sh --alter --delete-config log.cleaner.io.max.bytes.per.second (failure only)" + }, "not_a_release_certificate": true, + "owner_decision": "2026-10-11 (AskUserQuestion): release DL; G2/G8 writer throughput to TS phase E first item; gates not widened", "preparation_history": { "2026-10-08": { "active_runtime_roles": [ @@ -296,17 +381,424 @@ "source_review_base": "1cd24e367bc9b3216318da740d0e7ab3f3ddba54", "status": "PREPARED_RELEASE_HOLD_NOT_PUBLISHED", "version": "2.2.3" + }, + "2026-10-10": { + "branch": "fix/p184-dl-closure", + "catalog": { + "note": "deliberate split (C.19 bar edge checkpoint reads the image catalog); unifying needs its own packet", + "repo_and_image_catalog_revision": 11, + "runtime_catalog": { + "path": "~/.local/state/qdl-v2/releases/v2.2.0-02cd827/runtime/stable-source-bindings.yaml", + "sha256": "fc3a1e1ec283e6c49b671f55cc270bd8f0241d595b241e6c7f5c87f5dc686cc5" + }, + "runtime_catalog_revision": 12 + }, + "ci": { + "local_full_run": { + "failure": "unit-tests step 19 trivy image scan only: local runner flag bug (boolean ignore-unfixed, .trivyignore mount), fixed in 259aeb9; steps 20-21 skipped after it", + "jobs": { + "consolidated-source-preflight": "PASS", + "contract-tests": "PASS", + "kn-native-integration": "PASS", + "sdk-python310": "PASS", + "unit-tests": "FAIL" + }, + "sha256": "23a49251af3f853f4fd8a5f640085c58f1475d12841a5f827c4db35c93759e19", + "status": "FAIL", + "summary": "~/.local/state/qdl-v2/p184-local-ci-bbfb3be/summary.json" + }, + "local_unit_tests_rerun": { + "passed": 21, + "runner": "259aeb9 on archive of bbfb3be", + "sha256": "07df25eb2578f7e68f08fba930e48262bf6cfbaea30d1bd4b3704e70d46d4071", + "skip": "Prepare non-root runtime bind mounts (requires sudo; reported, never escalated)", + "skipped": 1, + "status": "INCOMPLETE", + "summary": "~/.local/state/qdl-v2/p184-local-ci-bbfb3be-r2/summary.json" + }, + "remote_github_ci": "NOT_RUN (DL repository not pushed)" + }, + "commits_after_source": { + "163191c": "plan journal", + "259aeb9": "local CI runner trivy flag fix (tooling, not in images)", + "e7b2028": "plan journal", + "this_commit": "scripts/p184_trace_feed_transit.py + test, release evidence (not in images)" + }, + "consumer_revisions": { + "alpha_manifest_jwt_revision": 15, + "ts_paper_manifest_jwt_revision": 11, + "ts_paper_products": 64 + }, + "deferred_to_ts_release": [ + "dr01-deployed", + "dr01-wrong-account", + "dr02-dr03-alpha", + "dr07-reference" + ], + "funding_settlement_allowance": { + "evidence_dir": "~/.local/state/qdl-v2/funding-publication-20261010/", + "note": "coordinator appends the result", + "poll_window_utc": "2026-10-10T15:58Z-16:04Z", + "shipped": "no allowance (530a406 revert)", + "status": "PENDING" + }, + "images": [ + { + "digest": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "roles": [ + "query_kn_1", + "query_kn_2", + "binance_bar_edge" + ], + "tag": "qdl-v2-python:2.2.3-rc7d-bbfb3be", + "version_label": "2.2.3-rc7d" + }, + { + "digest": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "roles": [ + "stream_kn_1", + "stream_kn_2", + "market_projector_1", + "market_projector_2", + "rust_core", + "rust_core_2", + "rust_core_3", + "ingestor_okx_swap", + "ingestor_binance_usdm" + ], + "tag": "qdl-v2-rust:2.2.3-rc7d-bbfb3be", + "version_label": "2.2.3-rc7d" + } + ], + "ingestor_configs": { + "book_liveness_proof_ms": "ABSENT (IDR-040 fix 2 OFF)", + "dir": "~/.local/state/qdl-v2/releases/v2.2.3-rc7-ingestor-liveness", + "files": { + "ingestor_binance_usdm.json": { + "bindings": 27, + "config_revision": 19, + "liveness_dead_ms": 4000, + "liveness_ping_ms": 1500, + "sha256": "5833bc4ea35cb4558a931288d34b61a94e99d260f1ba6b0e56596423995930ed" + }, + "ingestor_okx_swap.json": { + "bindings": 103, + "config_revision": 20, + "liveness_dead_ms": 4000, + "liveness_ping_ms": 1500, + "sha256": "eb982facfccce78a825c835e5d7ecdebce5dba6e457252a946da69e808ed0460" + } + }, + "mounted_read_only_at": "/runtime/config.json" + }, + "not_a_release_certificate": true, + "previous_published_release": "v2.2.2", + "publication_requires": [ + "owner decision on G2 FAIL and NOT_MEASURED gates (not accepted by this record)", + "funding publication-latency result", + "push; remote CI (contract-tests, sdk-python310, unit-tests, kn-native-integration) on the final head; merge to dev and main", + "annotated tag v2.2.3 linking CI runs and scorecard/notes hashes" + ], + "rollback": { + "dl_roles": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/-rollback.compose.json, reverse role order, failure only", + "not_touched": [ + "kafka1-3", + "market_cache", + "stable_redis", + "stable_state_init", + "stable_tls_init", + "Kafka offsets", + "Redis data", + "realm dirs" + ], + "python": "sha256:55e4937867489510b8fe07ce744c5d998af1f02e2bbaa31c40b371f73f85b723 (qdl-v2-python:2.2.3-rc7c-54460ac)", + "rollout_log": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc7d.log", + "sha256": "d8ab536105f6a6e71fb57a3a0287d9ff48bfc14bf4bdef3125a33a61fd76224a" + }, + "rust": "sha256:206ae872a5fed1dc127da9c3f70367b1ba79d34cbe6705c96b3f5bf16f082ef8 (qdl-v2-rust:2.2.3-rc7c-54460ac)" + }, + "runtime_changed_this_slice": false, + "runtime_inventory": [ + { + "image": "sha256:34202753bbe8b74d50e45afed1cf2e7b400141324c550168462b30db220edb31", + "name": "market_data_service_p184w", + "oom": false, + "restart_count": 0, + "revision_label": "ed020648286336170886043697cb229a3c5290b7", + "started_at": "2026-10-10T13:30:24.199934182Z", + "status": "running" + }, + { + "image": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "name": "qdl_v2_stable_candidate-binance_bar_edge-1", + "oom": false, + "restart_count": 0, + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:03:12.888708065Z", + "status": "running" + }, + { + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "name": "qdl_v2_stable_candidate-ingestor_binance_usdm-1", + "oom": false, + "restart_count": 0, + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:08:18.446850444Z", + "status": "running" + }, + { + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "name": "qdl_v2_stable_candidate-ingestor_okx_swap-1", + "oom": false, + "restart_count": 0, + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:08:10.243755542Z", + "status": "running" + }, + { + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "name": "qdl_v2_stable_candidate-kafka1-1", + "oom": false, + "restart_count": 0, + "revision_label": null, + "started_at": "2026-10-08T16:23:33.912058555Z", + "status": "running" + }, + { + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "name": "qdl_v2_stable_candidate-kafka2-1", + "oom": false, + "restart_count": 0, + "revision_label": null, + "started_at": "2026-10-08T16:23:34.12626004Z", + "status": "running" + }, + { + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "name": "qdl_v2_stable_candidate-kafka3-1", + "oom": false, + "restart_count": 0, + "revision_label": null, + "started_at": "2026-10-08T16:23:34.326288421Z", + "status": "running" + }, + { + "image": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", + "name": "qdl_v2_stable_candidate-market_cache-1", + "oom": false, + "restart_count": 0, + "revision_label": null, + "started_at": "2026-09-27T05:29:58.000340326Z", + "status": "running" + }, + { + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "name": "qdl_v2_stable_candidate-market_projector_1-1", + "oom": false, + "restart_count": 0, + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:05:32.09955379Z", + "status": "running" + }, + { + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "name": "qdl_v2_stable_candidate-market_projector_2-1", + "oom": false, + "restart_count": 0, + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:06:03.85847914Z", + "status": "running" + }, + { + "image": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "name": "qdl_v2_stable_candidate-query_kn_1-1", + "oom": false, + "restart_count": 0, + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:02:32.131029796Z", + "status": "running" + }, + { + "image": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "name": "qdl_v2_stable_candidate-query_kn_2-1", + "oom": false, + "restart_count": 0, + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:02:52.533684862Z", + "status": "running" + }, + { + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "name": "qdl_v2_stable_candidate-rust_core-1", + "oom": false, + "restart_count": 0, + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:06:35.398630448Z", + "status": "running" + }, + { + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "name": "qdl_v2_stable_candidate-rust_core_2-1", + "oom": false, + "restart_count": 0, + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:07:06.932177193Z", + "status": "running" + }, + { + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "name": "qdl_v2_stable_candidate-rust_core_3-1", + "oom": false, + "restart_count": 0, + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:07:38.503831129Z", + "status": "running" + }, + { + "image": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", + "name": "qdl_v2_stable_candidate-stable_redis-1", + "oom": false, + "restart_count": 0, + "revision_label": null, + "started_at": "2026-09-16T06:38:55.588316335Z", + "status": "running" + }, + { + "image": "sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d", + "name": "qdl_v2_stable_candidate-stable_state_init-1", + "oom": false, + "restart_count": 0, + "revision_label": "8ba4165", + "started_at": "2026-09-03T12:36:17.69711623Z", + "status": "exited" + }, + { + "image": "sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d", + "name": "qdl_v2_stable_candidate-stable_tls_init-1", + "oom": false, + "restart_count": 0, + "revision_label": "8ba4165", + "started_at": "2026-09-03T12:36:17.691555089Z", + "status": "exited" + }, + { + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "name": "qdl_v2_stable_candidate-stream_kn_1-1", + "oom": false, + "restart_count": 0, + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:03:22.758338124Z", + "status": "running" + }, + { + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "name": "qdl_v2_stable_candidate-stream_kn_2-1", + "oom": false, + "restart_count": 0, + "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "started_at": "2026-10-10T12:04:33.817947904Z", + "status": "running" + } + ], + "runtime_inventory_basis": "docker inspect, read-only, at assembly time", + "schema": "qdl.release-preparation.v1", + "scorecard": { + "path": "upgrade/evidence/releases/v2.2.3/scorecard.json", + "sha256": "d4035ccd460838cfcb130800660014f8dcfd22079f034390d44fa3fe9971c714", + "summary": { + "DEFERRED": [ + "G13" + ], + "FAIL": [ + "G2" + ], + "NOT_MEASURED": [ + "G1", + "G3", + "G4", + "G6", + "G7", + "G8", + "G9", + "G10", + "G11", + "G12", + "G14" + ], + "PASS": [ + "G5" + ] + } + }, + "sdk": { + "builder": "scripts/build_qdl_sdk_release.py from `git archive bbfb3be` (fresh extraction per build), docker run --rm --network none --cpus 1, non-root", + "builder_image": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "builds": [ + { + "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk-build-1", + "release_json_sha256": "240176ad449913c54b6ce6972b7114d6ea87e2982f69caa63a57369bce0a5bc3", + "sbom_sha256": "64bcb8491945da16c5400cda5bbcfdb292537a5e4eab0a9f681ff23c37e5930c", + "wheel_sha256": "d8226f509bce4ae408cacfa30e12f6160726e4040751d2e4f7379d3dc71c3997" + }, + { + "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk-build-2", + "release_json_sha256": "240176ad449913c54b6ce6972b7114d6ea87e2982f69caa63a57369bce0a5bc3", + "sbom_sha256": "64bcb8491945da16c5400cda5bbcfdb292537a5e4eab0a9f681ff23c37e5930c", + "wheel_sha256": "d8226f509bce4ae408cacfa30e12f6160726e4040751d2e4f7379d3dc71c3997" + } + ], + "contains_service_internals": false, + "dependencies": [ + "grpcio>=1.70.0,<2.0.0", + "httpx>=0.28.0,<1.0.0", + "protobuf>=6.31.1,<7.0.0", + "pydantic>=2.0.0,<3.0.0", + "PyJWT[crypto]>=2.15.1,<3.0.0" + ], + "deployed_image_comparison": "21/21 wheel files byte-identical to /app/qdl_sdk and /app/generated/python/qdl in the deployed python image", + "generated_contract_digest": "ae77dc42074fd7cc896a46bde3282837a51b667acee23c4957d37565690856e9", + "name": "qdl-sdk", + "python_requires": ">=3.10", + "reproducible": true, + "reproducible_timestamp": "2020-01-01T00:00:00Z", + "schema": "qdl.sdk.release.v1", + "sdk_source_digest": "63e230312cec966b51bfa20aedd3e73c51c838c1a52b778c24574f61b7399488", + "ts_writer_sdk_today": "2.1.0", + "version": "2.2.0", + "wheel": "qdl_sdk-2.2.0-py3-none-any.whl", + "wheel_sha256": "d8226f509bce4ae408cacfa30e12f6160726e4040751d2e4f7379d3dc71c3997" + }, + "source_commit": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "status": "PREPARED_NOT_PUBLISHED", + "superseded_note": "record at bbfb3be/rc7d, committed in 59e725a (file sha256 288044e576ac8ab686f6694dffa1cc4b26e0e936d037a89a40608d051f40f5bd)", + "ts_writer": { + "container": "market_data_service_p184w", + "expired_trace_env_present": true, + "image": "sha256:34202753bbe8b74d50e45afed1cf2e7b400141324c550168462b30db220edb31", + "rollback_env": { + "path": "~/.local/state/trading-system/p184-rev11-window/g2-tune/rollback-20261010T133013Z.compose.json", + "sha256": "a729902166e31749a2e411bab01d5b55b7126c58384237e8ddfb916f1ce827bd" + }, + "rollback_image": "sha256:15524c0bea8c51949d7ced79291d4c72722292fea2f0a5d4f8e9c7fd61dbb5ca", + "source": "ed020648286336170886043697cb229a3c5290b7", + "tuned_env": "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE=BINANCE=100" + }, + "version": "2.2.3", + "worktree": "/home/bobby/.worktrees/data-layer-p184-closure" } }, "previous_published_release": "v2.2.2", "publication_requires": [ - "owner decision on G2 FAIL and NOT_MEASURED gates (not accepted by this record)", - "funding publication-latency result", - "push; remote CI (contract-tests, sdk-python310, unit-tests, kn-native-integration) on the final head; merge to dev and main", - "annotated tag v2.2.3 linking CI runs and scorecard/notes hashes" + "push fix/p184-dl-closure; remote CI (contract-tests, sdk-python310, unit-tests, kn-native-integration) on the final head; merge to dev and main", + "annotated tag v2.2.3 linking the CI runs and the scorecard/notes hashes", + "owner publication decision (the 2026-10-11 release decision records G2/G8 as known limits; it does not turn them into PASS)" ], "rollback": { "dl_roles": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/-rollback.compose.json, reverse role order, failure only", + "kafka_config": "--delete-config log.cleaner.io.max.bytes.per.second", "not_touched": [ "kafka1-3", "market_cache", @@ -317,49 +809,223 @@ "Redis data", "realm dirs" ], - "python": "sha256:55e4937867489510b8fe07ce744c5d998af1f02e2bbaa31c40b371f73f85b723 (qdl-v2-python:2.2.3-rc7c-54460ac)", - "rollout_log": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc7d.log", - "sha256": "d8ab536105f6a6e71fb57a3a0287d9ff48bfc14bf4bdef3125a33a61fd76224a" + "per_role_previous_image": { + "binance_bar_edge": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb (already rc10 since 2026-10-10T18:46Z; its earlier image was rc7d python sha256:e0252f4c96da)", + "projectors, cores, ingestors": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7 (qdl-v2-rust:2.2.3-rc7d-bbfb3be)", + "query_kn_1/2": "sha256:db5ae6752280765d8b85f6df11136d2cc42bcd7e9dba94b41b0897b681b65736 (qdl-v2-python:2.2.3-rc8-6d5da7f)", + "stream_kn_1/2": "sha256:d32e354bd6628cf465a27f6691cadaa91ce4f70f8f3311f0d454e48c8d1a6b40 (qdl-v2-rust:2.2.3-rc9-70bef7d)" }, - "rust": "sha256:206ae872a5fed1dc127da9c3f70367b1ba79d34cbe6705c96b3f5bf16f082ef8 (qdl-v2-rust:2.2.3-rc7c-54460ac)" + "rc10_receipts": { + "binance_bar_edge": { + "new_image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "ok": true, + "old_image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/binance_bar_edge-receipt-20261011T043042Z.json", + "sha256": "45e5e45fa93a5449a8b8c9cc32d9ffbd453e806aab5298f679cc780b9655864c" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/binance_bar_edge-rollback.compose.json", + "sha256": "4ea4c00e13ac641b1ab6f5592f6d52f2a4cbbc646b86b3b7eb59b7b7a9f1c4bc" + }, + "writer": "READY 64 64" + }, + "ingestor_binance_usdm": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_binance_usdm-receipt-20261011T043515Z.json", + "sha256": "b0adf2ba3b97987591ea1200ca04843d77e00af56600997fbcba8b6313fd53af" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_binance_usdm-rollback.compose.json", + "sha256": "b99fca03be6c60e543fdb2eaa20e0862d3f33e0e5e5841fc1d37ed6059cfbac4" + }, + "writer": "READY 64 64" + }, + "ingestor_okx_swap": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_okx_swap-receipt-20261011T043507Z.json", + "sha256": "dd63d5b677ee20905f21e3f064731b83b0288765872bb85607a96858e9e7e16e" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_okx_swap-rollback.compose.json", + "sha256": "9510b426c5b1716df8e8cab2a6d7538b7f3a3367c55f8ee77ea5e0e350aa3576" + }, + "writer": "READY 64 64" + }, + "market_projector_1": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_1-receipt-20261011T043229Z.json", + "sha256": "3bf2a877f73a8be58be71f15ac1acef4a9a57f30ae2857423b4f7ba41425d9f8" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_1-rollback.compose.json", + "sha256": "3641dffc1de3f068a85745922e6825a3e02e983b7f7c358da2b2ea37ac1017e7" + }, + "writer": "READY 64 64" + }, + "market_projector_2": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_2-receipt-20261011T043301Z.json", + "sha256": "912aa21423170838b750c8f9dc2bbb61f5641ce67f4e10f26e5b9f0a9f135235" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_2-rollback.compose.json", + "sha256": "2370b4394858092d85716d64896788951fa85f8978d4e3daa02f26d7945dabb3" + }, + "writer": "READY 64 64" + }, + "query_kn_1": { + "new_image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "ok": true, + "old_image": "sha256:db5ae6752280765d8b85f6df11136d2cc42bcd7e9dba94b41b0897b681b65736", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_1-receipt-20261011T043001Z.json", + "sha256": "596a0c6fda90089ceb80b0dc9c14d757125ad4e45d2c579719649e7406f7c752" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_1-rollback.compose.json", + "sha256": "f0858eae514a8a65b84cd87d69d7140c0be0ed5f182cb29fb143de4659111d59" + }, + "writer": "READY 64 64" + }, + "query_kn_2": { + "new_image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "ok": true, + "old_image": "sha256:db5ae6752280765d8b85f6df11136d2cc42bcd7e9dba94b41b0897b681b65736", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_2-receipt-20261011T043019Z.json", + "sha256": "847f690f6909ea8fb4e311970a1beb6b821971fd94439e8c9d7a80aedac55b88" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_2-rollback.compose.json", + "sha256": "f4d5dfe4c19608fb018b5b8df29f1f3691aca2f51b5a12b64607fec921939707" + }, + "writer": "READY 64 64" + }, + "rust_core": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core-receipt-20261011T043332Z.json", + "sha256": "78c0aecdcbcc25071f3a6831788c56f541a33a8b72bafc656af92c91ba8c7e49" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core-rollback.compose.json", + "sha256": "fa91883ec8a7ca99faa4e8712e237b3710f5a5e96b216fddc1cf230c3b8489de" + }, + "writer": "READY 64 64" + }, + "rust_core_2": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_2-receipt-20261011T043404Z.json", + "sha256": "0d04e1c7b13ce7b8069fea6a17122fb39f956f638fd7d15cc7f3ec1e7e8e38eb" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_2-rollback.compose.json", + "sha256": "8a52e413347ba8fda3d0bf61993eb8bafc2a0e9ae56d78d063734e71884b6092" + }, + "writer": "READY 64 64" + }, + "rust_core_3": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_3-receipt-20261011T043435Z.json", + "sha256": "eb9fe530eae55107a429a548f2f944783a096f6b96e423126ee82ffa43f852a2" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_3-rollback.compose.json", + "sha256": "20a3bb453625dfa5e3c5cd4aa6acece2efba1c4a010c7d98265ac93cf6922c7f" + }, + "writer": "READY 64 64" + }, + "stream_kn_1": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:d32e354bd6628cf465a27f6691cadaa91ce4f70f8f3311f0d454e48c8d1a6b40", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_1-receipt-20261011T043052Z.json", + "sha256": "6136fd868c7eee7bc7a31681c962160a916a2479a234d040d738009287a549db" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_1-rollback.compose.json", + "sha256": "03b2efe1108b653235c992e9eb7ffee7b3ede2e34f75638fff45a71972300119" + }, + "writer": "READY 64 64" + }, + "stream_kn_2": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:d32e354bd6628cf465a27f6691cadaa91ce4f70f8f3311f0d454e48c8d1a6b40", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_2-receipt-20261011T043127Z.json", + "sha256": "735a9da30c3fcc63179ea1d17b4a05d5c8c4e69b0850daae4397bace86b4e1d6" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_2-rollback.compose.json", + "sha256": "ae9e514dc24794fedb5815e44123823a2536f8692949c1ce946cf7dc7db7cae1" + }, + "writer": "READY 64 64" + } + }, + "ts_writer": { + "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/rollback-20261011T042442Z.compose.json", + "sha256": "e9c227757c662e3214447b2c01415e44c900ad63de2435966e34a88d6543795b" + } }, "runtime_changed_this_slice": false, "runtime_inventory": [ { - "image": "sha256:34202753bbe8b74d50e45afed1cf2e7b400141324c550168462b30db220edb31", + "image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", "name": "market_data_service_p184w", "oom": false, "restart_count": 0, - "revision_label": "ed020648286336170886043697cb229a3c5290b7", - "started_at": "2026-10-10T13:30:24.199934182Z", + "revision_label": "8b567eb07c9036dbbb42079ed1da90db7e0f5f25", + "started_at": "2026-10-11T04:24:53.613267377Z", "status": "running" }, { - "image": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", "name": "qdl_v2_stable_candidate-binance_bar_edge-1", "oom": false, "restart_count": 0, - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "started_at": "2026-10-10T12:03:12.888708065Z", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:30:45.289811072Z", "status": "running" }, { - "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", "name": "qdl_v2_stable_candidate-ingestor_binance_usdm-1", "oom": false, "restart_count": 0, - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "started_at": "2026-10-10T12:08:18.446850444Z", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:35:16.489289377Z", "status": "running" }, { - "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", "name": "qdl_v2_stable_candidate-ingestor_okx_swap-1", "oom": false, "restart_count": 0, - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "started_at": "2026-10-10T12:08:10.243755542Z", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:35:08.381807086Z", "status": "running" }, { @@ -399,66 +1065,66 @@ "status": "running" }, { - "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", "name": "qdl_v2_stable_candidate-market_projector_1-1", "oom": false, "restart_count": 0, - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "started_at": "2026-10-10T12:05:32.09955379Z", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:32:30.621173851Z", "status": "running" }, { - "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", "name": "qdl_v2_stable_candidate-market_projector_2-1", "oom": false, "restart_count": 0, - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "started_at": "2026-10-10T12:06:03.85847914Z", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:33:02.29275674Z", "status": "running" }, { - "image": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", "name": "qdl_v2_stable_candidate-query_kn_1-1", "oom": false, "restart_count": 0, - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "started_at": "2026-10-10T12:02:32.131029796Z", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:30:02.858025475Z", "status": "running" }, { - "image": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", "name": "qdl_v2_stable_candidate-query_kn_2-1", "oom": false, "restart_count": 0, - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "started_at": "2026-10-10T12:02:52.533684862Z", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:30:23.042515056Z", "status": "running" }, { - "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", "name": "qdl_v2_stable_candidate-rust_core-1", "oom": false, "restart_count": 0, - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "started_at": "2026-10-10T12:06:35.398630448Z", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:33:33.780053453Z", "status": "running" }, { - "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", "name": "qdl_v2_stable_candidate-rust_core_2-1", "oom": false, "restart_count": 0, - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "started_at": "2026-10-10T12:07:06.932177193Z", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:34:05.235527067Z", "status": "running" }, { - "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", "name": "qdl_v2_stable_candidate-rust_core_3-1", "oom": false, "restart_count": 0, - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "started_at": "2026-10-10T12:07:38.503831129Z", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:34:36.604572262Z", "status": "running" }, { @@ -489,21 +1155,21 @@ "status": "exited" }, { - "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", "name": "qdl_v2_stable_candidate-stream_kn_1-1", "oom": false, "restart_count": 0, - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "started_at": "2026-10-10T12:03:22.758338124Z", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:30:56.645942409Z", "status": "running" }, { - "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", "name": "qdl_v2_stable_candidate-stream_kn_2-1", "oom": false, "restart_count": 0, - "revision_label": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "started_at": "2026-10-10T12:04:33.817947904Z", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:31:32.097345347Z", "status": "running" } ], @@ -511,47 +1177,46 @@ "schema": "qdl.release-preparation.v1", "scorecard": { "path": "upgrade/evidence/releases/v2.2.3/scorecard.json", - "sha256": "d4035ccd460838cfcb130800660014f8dcfd22079f034390d44fa3fe9971c714", + "sha256": "bc1d2ded655e65fa73085fff68989d31eeaa8d52ce9255aa8b289ff0d28e323d", "summary": { "DEFERRED": [ "G13" ], "FAIL": [ - "G2" + "G2", + "G8" ], "NOT_MEASURED": [ + "G4", + "G12" + ], + "PASS": [ "G1", "G3", - "G4", + "G5", "G6", "G7", - "G8", "G9", "G10", "G11", - "G12", "G14" - ], - "PASS": [ - "G5" ] } }, "sdk": { - "builder": "scripts/build_qdl_sdk_release.py from `git archive bbfb3be` (fresh extraction per build), docker run --rm --network none --cpus 1, non-root", - "builder_image": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "builder": "scripts/build_qdl_sdk_release.py, two builds", "builds": [ { - "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk-build-1", - "release_json_sha256": "240176ad449913c54b6ce6972b7114d6ea87e2982f69caa63a57369bce0a5bc3", - "sbom_sha256": "64bcb8491945da16c5400cda5bbcfdb292537a5e4eab0a9f681ff23c37e5930c", - "wheel_sha256": "d8226f509bce4ae408cacfa30e12f6160726e4040751d2e4f7379d3dc71c3997" + "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk221-build-1", + "release_json_sha256": "d2ae9ef67e334540e3cef4d0e1e1b17896762262263e4d6d4fadfeb518c894e6", + "sbom_sha256": "e80dcec6ce588d715e7e5d6bf3aa5449426c80d999f8d69647081047398d9111", + "wheel_sha256": "27db39715c58988128dfec3364919a3ef3bb9425e9bdb4cb7655829ee3f1076f" }, { - "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk-build-2", - "release_json_sha256": "240176ad449913c54b6ce6972b7114d6ea87e2982f69caa63a57369bce0a5bc3", - "sbom_sha256": "64bcb8491945da16c5400cda5bbcfdb292537a5e4eab0a9f681ff23c37e5930c", - "wheel_sha256": "d8226f509bce4ae408cacfa30e12f6160726e4040751d2e4f7379d3dc71c3997" + "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk221-build-2", + "release_json_sha256": "d2ae9ef67e334540e3cef4d0e1e1b17896762262263e4d6d4fadfeb518c894e6", + "sbom_sha256": "e80dcec6ce588d715e7e5d6bf3aa5449426c80d999f8d69647081047398d9111", + "wheel_sha256": "27db39715c58988128dfec3364919a3ef3bb9425e9bdb4cb7655829ee3f1076f" } ], "contains_service_internals": false, @@ -562,32 +1227,41 @@ "pydantic>=2.0.0,<3.0.0", "PyJWT[crypto]>=2.15.1,<3.0.0" ], - "deployed_image_comparison": "21/21 wheel files byte-identical to /app/qdl_sdk and /app/generated/python/qdl in the deployed python image", "generated_contract_digest": "ae77dc42074fd7cc896a46bde3282837a51b667acee23c4957d37565690856e9", "name": "qdl-sdk", "python_requires": ">=3.10", "reproducible": true, "reproducible_timestamp": "2020-01-01T00:00:00Z", "schema": "qdl.sdk.release.v1", - "sdk_source_digest": "63e230312cec966b51bfa20aedd3e73c51c838c1a52b778c24574f61b7399488", + "sdk_source_digest": "fcbd832c1a3e89c66ad625e488ae2a4f80f7baa895abb7b8904febfcb54aa217", + "supersedes": "2.2.0 (wheel d8226f50..., never published; version bumped, not reused)", "ts_writer_sdk_today": "2.1.0", - "version": "2.2.0", - "wheel": "qdl_sdk-2.2.0-py3-none-any.whl", - "wheel_sha256": "d8226f509bce4ae408cacfa30e12f6160726e4040751d2e4f7379d3dc71c3997" + "version": "2.2.1", + "wheel": "qdl_sdk-2.2.1-py3-none-any.whl", + "wheel_sha256": "27db39715c58988128dfec3364919a3ef3bb9425e9bdb4cb7655829ee3f1076f" }, - "source_commit": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "source_commit": "413c5b914f8c6fd0f67429fdddb39451589e9745", "status": "PREPARED_NOT_PUBLISHED", "ts_writer": { "container": "market_data_service_p184w", - "expired_trace_env_present": true, - "image": "sha256:34202753bbe8b74d50e45afed1cf2e7b400141324c550168462b30db220edb31", - "rollback_env": { - "path": "~/.local/state/trading-system/p184-rev11-window/g2-tune/rollback-20261010T133013Z.compose.json", - "sha256": "a729902166e31749a2e411bab01d5b55b7126c58384237e8ddfb916f1ce827bd" - }, - "rollback_image": "sha256:15524c0bea8c51949d7ced79291d4c72722292fea2f0a5d4f8e9c7fd61dbb5ca", - "source": "ed020648286336170886043697cb229a3c5290b7", - "tuned_env": "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE=BINANCE=100" + "depth_note": "depth 0 after the campaign was the coordinator's choice from two comparisons; that comparison is not re-verified here", + "env_campaign_4": { + "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH": "1", + "source": { + "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/inspect-20261011T042442Z.json", + "sha256": "ee6c5ff47f9d4372beb8a8ab52c48cb64f4c9573d8c4837adf875abc5ed7421d" + } + }, + "env_now": { + "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE": "BINANCE=100", + "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH": "0", + "MARKET_DATA_FIXED_WORKER_PINS": "OKX:BTCUSDT=2,OKX:ETHUSDT=0" + }, + "image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", + "image_tag": "local/trading-system:p184-writer-8b567eb", + "sdk": "2.1.0", + "source": "8b567eb07c9036dbbb42079ed1da90db7e0f5f25 (pushed: origin/feat/v2-rust-first-okx-demo contains it)", + "trace_env_present_until_unix_ns": 1791695682881050430 }, "version": "2.2.3", "worktree": "/home/bobby/.worktrees/data-layer-p184-closure" diff --git a/upgrade/evidence/releases/v2.2.3/scorecard.json b/upgrade/evidence/releases/v2.2.3/scorecard.json index edbf8826..67b889fc 100644 --- a/upgrade/evidence/releases/v2.2.3/scorecard.json +++ b/upgrade/evidence/releases/v2.2.3/scorecard.json @@ -1,1667 +1,4320 @@ { - "schema": "qdl.release-scorecard.v1", - "release": "2.2.3", - "status": "PREPARED_NOT_PUBLISHED", - "source_commit": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", - "gate_table": { - "path": "trading-system upgrade/p184-phase-d-gates.md (FROZEN 2026-10-09)", - "ts_commit": "31cf663f5c7c5384ec3dd296061582dee2463c41", - "sha256": "c67bf69bb921c992964eb8ff8b2e1bb17085a943fdf756d55526b6269641d5c4" - }, - "frozen_budget": { - "path": "config/v2/p184-stage100-budget.json", - "sha256": "ff85335b3c8762d9acd8e7230f3c81ee1ec8403aff34cf2fb8f67714521c2541" - }, - "scoring_rule": "steady state >= 30 min at the stage-100 workload plus one burst window (gate table 'Measurement rules'); a measurement outside this rule can FAIL a gate but cannot PASS it. Verdicts: PASS, FAIL, NOT_MEASURED, DEFERRED; a NOT_MEASURED is never upgraded. 'observed' records the value against the threshold when the measurement does not meet the protocol.", - "runtime_identity": { - "dl_python": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", - "dl_rust": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", - "dl_application_roles_on_release_digests": 12, - "ts_writer": "sha256:34202753bbe8b74d50e45afed1cf2e7b400141324c550168462b30db220edb31", - "ts_writer_source": "ed020648286336170886043697cb229a3c5290b7", - "ts_writer_tuned_env": "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE=BINANCE=100" - }, - "summary": { - "NOT_MEASURED": [ - "G1", - "G3", - "G4", - "G6", - "G7", - "G8", - "G9", - "G10", - "G11", - "G12", - "G14" - ], - "FAIL": [ - "G2" - ], - "PASS": [ - "G5" - ], - "DEFERRED": [ - "G13" - ] - }, - "gates": [ - { - "id": "G1", - "quantity": "end-to-end to TS cache (venue event -> TS Redis APPLY reply)", - "scope": "BOOK_DELTA, BOOK_SNAPSHOT", - "rule": "p95 <= 500 ms, p99 <= 1000 ms (n >= 100), max < 2000 ms per feed/venue", - "verdict": "NOT_MEASURED", - "observed": "INSIDE in both trace windows", - "reason": "Values come from writer-only trace windows, not the frozen stage-100 protocol.", - "measured": [ - { - "window": "window1_mark_delay_binance_250", - "feed_venue": "BOOK_DELTA|BINANCE", - "n": 24553, - "p50_ms": 100.0, - "p95_ms": 126.5, - "p99_ms": 167.1, - "max_ms": 527.6, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 68.4, - "sdk_to_reply_ms": 36.7, - "venue_to_ingest_ms": 29.3 - } - }, - { - "window": "window1_mark_delay_binance_250", - "feed_venue": "BOOK_DELTA|OKX", - "n": 26481, - "p50_ms": 104.3, - "p95_ms": 128.4, - "p99_ms": 173.4, - "max_ms": 528.1, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 67.6, - "sdk_to_reply_ms": 37.9, - "venue_to_ingest_ms": 33.0 - } - }, - { - "window": "window1_mark_delay_binance_250", - "feed_venue": "BOOK_SNAPSHOT|BINANCE", - "n": 2647, - "p50_ms": 108.8, - "p95_ms": 154.4, - "p99_ms": 220.4, - "max_ms": 407.2, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 70.8, - "sdk_to_reply_ms": 58.3, - "venue_to_ingest_ms": 29.3 - } - }, - { - "window": "window1_mark_delay_binance_250", - "feed_venue": "BOOK_SNAPSHOT|OKX", - "n": 3223, - "p50_ms": 113.6, - "p95_ms": 156.8, - "p99_ms": 206.3, - "max_ms": 336.1, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 69.2, - "sdk_to_reply_ms": 59.9, - "venue_to_ingest_ms": 33.1 - } - }, - { - "window": "window2_deployed_config", - "feed_venue": "BOOK_DELTA|BINANCE", - "n": 24664, - "p50_ms": 99.9, - "p95_ms": 126.5, - "p99_ms": 163.0, - "max_ms": 510.9, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 68.7, - "sdk_to_reply_ms": 36.4, - "venue_to_ingest_ms": 29.4 - } - }, - { - "window": "window2_deployed_config", - "feed_venue": "BOOK_DELTA|OKX", - "n": 26105, - "p50_ms": 105.1, - "p95_ms": 130.0, - "p99_ms": 170.5, - "max_ms": 501.8, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 67.8, - "sdk_to_reply_ms": 37.7, - "venue_to_ingest_ms": 37.9 - } - }, - { - "window": "window2_deployed_config", - "feed_venue": "BOOK_SNAPSHOT|BINANCE", - "n": 2647, - "p50_ms": 109.5, - "p95_ms": 152.4, - "p99_ms": 192.1, - "max_ms": 294.0, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 71.5, - "sdk_to_reply_ms": 53.5, - "venue_to_ingest_ms": 28.6 - } - }, - { - "window": "window2_deployed_config", - "feed_venue": "BOOK_SNAPSHOT|OKX", - "n": 3182, - "p50_ms": 114.3, - "p95_ms": 155.5, - "p99_ms": 188.9, - "max_ms": 353.1, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 68.5, - "sdk_to_reply_ms": 54.6, - "venue_to_ingest_ms": 37.4 - } - } - ], - "measurement_scope": "TS writer delivery trace, two 535 s analysis windows (13:19-13:28Z and 13:31-13:40Z, 40 s writer start-up skipped), writer-only production traffic AFTER stage 100m ended (13:17Z): no stage-100 alpha load, no burst window", - "would_measure": "the same trace analysis (scripts/p184_trace_feed_transit.py) over >= 30 min of trace activations taken DURING a stage-100 run, including its burst window (one activation is capped at 600 s and 16 x 4096 records per worker)", - "sources": [ - { - "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791638906937534999.json", - "sha256": "d8d0d6c37dc37ccd1c22b3d9690d42041d09cc532ddd0008499d5a3d7b32526a" - }, - { - "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json", - "sha256": "e9e823094e73782166d8275647fed3abbf6000c20bc18abd695b8bb1686bf72d" - } - ] - }, - { - "id": "G2", - "quantity": "end-to-end to TS cache, new-event transit", - "scope": "TRADE / QUOTE / MARK_INDEX_PRICE", - "rule": "TRADE p95 <= 500 ms and max < 1500 ms; QUOTE and MARK p95 <= 500 ms and max < 1000 ms (quiet-state age reported separately, not gated)", - "verdict": "FAIL", - "reason": "TRADE BINANCE p95 539.1 ms > 500 ms (max 1078.7 ms, n 21983) in trace window 1 on the final DL images and final writer image; window 2 (deployed writer config) is inside at p95 269.7 ms. The excess sits in the ingest -> SDK return segment (split p95 434.5 ms), not in the writer (sdk -> reply p95 85.0 ms) or the venue -> ingest leg (36.9 ms). Window 1 differs from the deployed config only by the Binance MARK wake read delay (250 vs 100 ms), which does not act on TRADE. Corroboration under stage-100 load: alpha SDK stream TRADE BINANCE source -> usable per-stream p95 up to 872.5 ms, max 1402.3 ms (stage100m receipt stream_latency). MARK and QUOTE are inside in the deployed-config window but are not PASS by protocol.", - "measured": [ - { - "window": "window1_mark_delay_binance_250", - "feed_venue": "QUOTE|BINANCE", - "n": 25600, - "p50_ms": 116.2, - "p95_ms": 152.3, - "p99_ms": 179.0, - "max_ms": 434.1, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 98.5, - "sdk_to_reply_ms": 32.0, - "venue_to_ingest_ms": 30.8 - } - }, - { - "window": "window1_mark_delay_binance_250", - "feed_venue": "QUOTE|OKX", - "n": 23103, - "p50_ms": 117.5, - "p95_ms": 153.9, - "p99_ms": 188.5, - "max_ms": 421.4, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 98.0, - "sdk_to_reply_ms": 32.6, - "venue_to_ingest_ms": 31.6 - } - }, - { - "window": "window1_mark_delay_binance_250", - "feed_venue": "TRADE|BINANCE", - "n": 21983, - "p50_ms": 118.8, - "p95_ms": 539.1, - "p99_ms": 989.7, - "max_ms": 1078.7, - "observed": "OUTSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 434.5, - "sdk_to_reply_ms": 85.0, - "venue_to_ingest_ms": 36.9 - } - }, - { - "window": "window1_mark_delay_binance_250", - "feed_venue": "TRADE|OKX", - "n": 5035, - "p50_ms": 107.7, - "p95_ms": 246.5, - "p99_ms": 558.3, - "max_ms": 731.1, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 107.5, - "sdk_to_reply_ms": 97.4, - "venue_to_ingest_ms": 37.9 - } - }, - { - "window": "window2_deployed_config", - "feed_venue": "QUOTE|BINANCE", - "n": 24134, - "p50_ms": 117.2, - "p95_ms": 152.2, - "p99_ms": 176.2, - "max_ms": 516.9, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 98.8, - "sdk_to_reply_ms": 32.1, - "venue_to_ingest_ms": 32.0 - } - }, - { - "window": "window2_deployed_config", - "feed_venue": "QUOTE|OKX", - "n": 22916, - "p50_ms": 118.3, - "p95_ms": 154.0, - "p99_ms": 181.7, - "max_ms": 513.4, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 98.2, - "sdk_to_reply_ms": 32.6, - "venue_to_ingest_ms": 31.9 - } - }, - { - "window": "window2_deployed_config", - "feed_venue": "TRADE|BINANCE", - "n": 14928, - "p50_ms": 116.8, - "p95_ms": 269.7, - "p99_ms": 545.5, - "max_ms": 623.9, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 191.3, - "sdk_to_reply_ms": 66.1, - "venue_to_ingest_ms": 36.0 - } - }, - { - "window": "window2_deployed_config", - "feed_venue": "TRADE|OKX", - "n": 4845, - "p50_ms": 106.6, - "p95_ms": 158.9, - "p99_ms": 216.5, - "max_ms": 394.0, - "observed": "INSIDE", - "split_p95_ms": { - "ingest_to_sdk_ms": 80.3, - "sdk_to_reply_ms": 60.8, - "venue_to_ingest_ms": 35.3 - } - }, - { - "window": "window1_mark_delay_binance_250", - "feed_venue": "MARK_INDEX_PRICE|BINANCE", - "n": 2667, - "p50_ms": 429.5, - "p95_ms": 500.7, - "p99_ms": 540.2, - "max_ms": 776.2, - "observed": "OUTSIDE" - }, - { - "window": "window1_mark_delay_binance_250", - "feed_venue": "MARK_INDEX_PRICE|OKX", - "n": 6385, - "p50_ms": 214.9, - "p95_ms": 320.9, - "p99_ms": 348.6, - "max_ms": 398.6, - "observed": "INSIDE" - }, - { - "window": "window2_deployed_config", - "feed_venue": "MARK_INDEX_PRICE|BINANCE", - "n": 2671, - "p50_ms": 278.5, - "p95_ms": 339.3, - "p99_ms": 379.5, - "max_ms": 708.0, - "observed": "INSIDE" - }, - { - "window": "window2_deployed_config", - "feed_venue": "MARK_INDEX_PRICE|OKX", - "n": 6390, - "p50_ms": 212.3, - "p95_ms": 317.9, - "p99_ms": 347.8, - "max_ms": 495.2, - "observed": "INSIDE" - } - ], - "measurement_scope": "TS writer delivery trace, two 535 s analysis windows (13:19-13:28Z and 13:31-13:40Z, 40 s writer start-up skipped), writer-only production traffic AFTER stage 100m ended (13:17Z): no stage-100 alpha load, no burst window", - "would_measure": "attribute the TRADE ingest -> SDK return tail (Stream dispatch/queue vs SDK client receive) and re-measure >= 30 min of trace during a stage-100 run", - "sources": [ - { - "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791638906937534999.json", - "sha256": "d8d0d6c37dc37ccd1c22b3d9690d42041d09cc532ddd0008499d5a3d7b32526a" - }, - { - "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json", - "sha256": "e9e823094e73782166d8275647fed3abbf6000c20bc18abd695b8bb1686bf72d" - }, - { - "path": "~/.local/state/qdl-v2/p184-g2-20261010/g2-mark-1791638906937534999.json", - "sha256": "292256e814110d6805efab9660667939699a228d99c1d1283cb5ce61e35f476c" - }, - { - "path": "~/.local/state/qdl-v2/p184-g2-20261010/g2-mark-1791639603470577388.json", - "sha256": "3da349c4f54c5125c23b8a6eb3c6e8f080c1a2023dd8b300ffa1732f2b09ea68" - }, - { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/receipt.json", - "sha256": "9dfce5d983f25cdeb831eee6dbcd151864fa1cdbf567c9dd871f5866f67684a8" - } - ] + "schema": "qdl.release-scorecard.v1", + "release": "2.2.3", + "status": "PREPARED_NOT_PUBLISHED", + "source_commit": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "gate_table": { + "path": "trading-system upgrade/p184-phase-d-gates.md (FROZEN 2026-10-09)", + "ts_commit": "31cf663f5c7c5384ec3dd296061582dee2463c41", + "sha256": "c67bf69bb921c992964eb8ff8b2e1bb17085a943fdf756d55526b6269641d5c4" }, - { - "id": "G3", - "quantity": "end-to-end to TS cache, BAR from close", - "scope": "BAR", - "rule": "p95 <= 5 s, max < 60 s", - "verdict": "NOT_MEASURED", - "reason": "The writer delivery trace does not record BAR (0 BAR records in both windows); no other receipt measures BAR close -> TS Redis on the final digests.", - "related_observation": "stage100m alpha SDK streams (not the TS cache): BAR 1m close -> usable BINANCE per-stream p95 1135-3095 ms, max 3095 ms; OKX p95 1248-1307 ms, max 1511 ms", - "would_measure": "a read-only TS Redis notification probe (the 2026-10-09 transit_probe.py method, BAR from close) or BAR tracing in the writer, during a stage-100 run", - "sources": [ - { - "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791638906937534999.json", - "sha256": "d8d0d6c37dc37ccd1c22b3d9690d42041d09cc532ddd0008499d5a3d7b32526a" - }, - { - "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json", - "sha256": "e9e823094e73782166d8275647fed3abbf6000c20bc18abd695b8bb1686bf72d" - }, - { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/receipt.json", - "sha256": "9dfce5d983f25cdeb831eee6dbcd151864fa1cdbf567c9dd871f5866f67684a8" - } - ] + "frozen_budget": { + "path": "config/v2/p184-stage100-budget.json", + "sha256": "ff85335b3c8762d9acd8e7230f3c81ee1ec8403aff34cf2fb8f67714521c2541" }, - { - "id": "G4", - "quantity": "delivery lag, venue -> canonical (read_committed visibility, upper bound)", - "scope": "every feed, venue and interval", - "rule": "candidate p95 <= frozen baseline p95 x 1.2 per row (owner amendment 2026-10-09); missing coverage blocks the gate", - "verdict": "NOT_MEASURED", - "reason": "16/17 frozen rows PASS in all four stage-100m windows; bar|BINANCE|30m had no close in any window; five observed rows (BAR BINANCE 5m/1h, OKX 5m/15m/1h) have no frozen baseline; OKX inverse is pooled into OKX rows, not its own cells. Missing coverage blocks the gate by its own rule.", - "measured": [ - { - "row": "bar|BINANCE|15m", - "p95_ms_per_window": [ - null, - null, - 12533.0, - null - ], - "n_per_window": [ - 0, - 0, - 3, - 0 - ], - "limit_p95_ms": 40728.7, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 + "scoring_rule": "steady state >= 30 min at the stage-100 workload plus one burst window (gate table 'Measurement rules'); a measurement outside this rule can FAIL a gate but cannot PASS it. Verdicts: PASS, FAIL, NOT_MEASURED, DEFERRED; a NOT_MEASURED is never upgraded. 'observed' records the value against the threshold when the measurement does not meet the protocol.", + "runtime_identity": { + "release_images": { + "python": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb (qdl-v2-python:2.2.3-rc10-413c5b9)", + "rust": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5 (qdl-v2-rust:2.2.3-rc10-413c5b9)", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "dl_application_roles_on_release_digests": 12, + "rolled": "2026-10-11T04:30:01Z-04:35:15Z, one role at a time, writer READY 64/64 after every step" }, - { - "row": "bar|BINANCE|1h", - "p95_ms_per_window": [ - null, - null, - 14999.3, - null - ], - "n_per_window": [ - 0, - 0, - 3, - 0 - ], - "limit_p95_ms": null, - "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "bar|BINANCE|1m", - "p95_ms_per_window": [ - 116.4, - 108.3, - 117.7, - 140.8 - ], - "n_per_window": [ - 10, - 10, - 10, - 10 - ], - "limit_p95_ms": 257.6, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 + "campaign_4_measurement_identity": { + "note": "campaign 4 ran on the pre-rc10 mix; G12 and G13 ran on the release digests", + "query_kn_1/2": { + "image": "sha256:db5ae6752280765d8b85f6df11136d2cc42bcd7e9dba94b41b0897b681b65736", + "tag": "qdl-v2-python:2.2.3-rc8-6d5da7f", + "source": "6d5da7f", + "diff_to_413c5b9": "qdl/admission/edge.py +9 (INFLIGHT_CAPACITY wait polls); qdl_sdk 2.2.1 (version string, GrpcStreamTransport random start; service code imports only qdl_sdk.models/reference); contracts doc; the Rust gateway source in the tree is not run by Query", + "equivalence": "NOT source-identical: provider-admitted reads (OKX admitted REST and reference, Binance admitted klines) differ when an INFLIGHT_CAPACITY deferral occurs; other Query paths are source-identical" + }, + "stream_kn_1/2": { + "image": "sha256:d32e354bd6628cf465a27f6691cadaa91ce4f70f8f3311f0d454e48c8d1a6b40", + "tag": "qdl-v2-rust:2.2.3-rc9-70bef7d", + "source": "70bef7d", + "diff_to_413c5b9": "qdl/admission/edge.py only (not in the Rust build); rust/, generated/rust, Cargo.toml/lock and config unchanged", + "equivalence": "source-identical for the Stream binary" + }, + "binance_bar_edge": { + "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "tag": "qdl-v2-python:2.2.3-rc10-413c5b9", + "source": "413c5b9", + "equivalence": "the release image itself (rolled 2026-10-10T18:46Z)" + }, + "rust_core x3, market_projector_1/2, ingestor_okx_swap, ingestor_binance_usdm": { + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "tag": "qdl-v2-rust:2.2.3-rc7d-bbfb3be", + "source": "bbfb3be", + "diff_to_413c5b9": "rust/qdl-stream-gateway only (dadb9c8), a workspace member no other crate depends on; Cargo.toml/lock, generated/rust, config and Dockerfile unchanged", + "equivalence": "source-equivalent for the core, projector and ingestor binaries (rebuilt, not byte-compared)" + } }, - { - "row": "bar|BINANCE|30m", - "p95_ms_per_window": [ - null, - null, - null, - null - ], - "n_per_window": [ - 0, - 0, - 0, - 0 - ], - "limit_p95_ms": 25836.6, - "verdict": "NOT_MEASURED_NOT_OBSERVED", - "negative_samples": 0, - "missing_clock": 0 + "ts_writer": { + "container": "market_data_service_p184w", + "image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", + "source": "8b567eb07c9036dbbb42079ed1da90db7e0f5f25", + "sdk": "2.1.0", + "env_campaign_4": { + "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH": "1", + "MARKET_DATA_FIXED_WORKER_PINS": "OKX:BTCUSDT=2,OKX:ETHUSDT=0", + "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE": "BINANCE=100" + }, + "env_source": { + "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/inspect-20261011T042442Z.json", + "sha256": "ee6c5ff47f9d4372beb8a8ab52c48cb64f4c9573d8c4837adf875abc5ed7421d" + }, + "after_campaign": "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH=0 since 2026-10-11T04:24:53Z (docker inspect)" }, - { - "row": "bar|BINANCE|3m", - "p95_ms_per_window": [ - 7992.0, - 8016.0, - null, - 8001.2 - ], - "n_per_window": [ - 5, - 5, - 0, - 5 - ], - "limit_p95_ms": 27273.2, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "bar|BINANCE|5m", - "p95_ms_per_window": [ - null, - 8169.9, - 16186.8, - null - ], - "n_per_window": [ - 0, - 5, - 4, - 0 - ], - "limit_p95_ms": null, - "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "bar|OKX|15m", - "p95_ms_per_window": [ - null, - null, - 901184.4, - null - ], - "n_per_window": [ - 0, - 0, - 1, - 0 - ], - "limit_p95_ms": null, - "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "bar|OKX|1h", - "p95_ms_per_window": [ - null, - null, - 3600671.8, - null - ], - "n_per_window": [ - 0, - 0, - 1, - 0 - ], - "limit_p95_ms": null, - "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "bar|OKX|1m", - "p95_ms_per_window": [ - 61304.5, - 60741.8, - 61188.9, - 61107.0 - ], - "n_per_window": [ - 7, - 1, - 6, - 8 - ], - "limit_p95_ms": 73817.2, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "bar|OKX|30m", - "p95_ms_per_window": [ - null, - null, - 1800470.9, - null - ], - "n_per_window": [ - 0, - 0, - 1, - 0 - ], - "limit_p95_ms": 2160788.8, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "bar|OKX|3m", - "p95_ms_per_window": [ - 181311.9, - 181132.3, - 180480.3, - 181110.8 - ], - "n_per_window": [ - 3, - 4, - 1, - 2 - ], - "limit_p95_ms": 217582.1, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "bar|OKX|5m", - "p95_ms_per_window": [ - null, - 300978.9, - null, - null - ], - "n_per_window": [ - 0, - 5, - 0, - 0 - ], - "limit_p95_ms": null, - "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "book_delta|BINANCE|", - "p95_ms_per_window": [ - 97.2, - 98.0, - 100.0, - 101.9 - ], - "n_per_window": [ - 756, - 755, - 760, - 785 - ], - "limit_p95_ms": 263.3, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "book_delta|OKX|", - "p95_ms_per_window": [ - 103.1, - 100.3, - 103.3, - 114.8 - ], - "n_per_window": [ - 1061, - 1069, - 978, - 1047 - ], - "limit_p95_ms": 289.1, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "book_snapshot|BINANCE|", - "p95_ms_per_window": [ - 99.6, - 96.2, - 100.3, - 102.4 - ], - "n_per_window": [ - 867, - 752, - 859, - 928 - ], - "limit_p95_ms": 260.0, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "book_snapshot|OKX|", - "p95_ms_per_window": [ - 101.5, - 97.8, - 102.6, - 112.1 - ], - "n_per_window": [ - 1618, - 1688, - 1086, - 1538 - ], - "limit_p95_ms": 270.1, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "mark_index_price|BINANCE|", - "p95_ms_per_window": [ - 185.8, - 190.2, - 195.2, - 193.2 - ], - "n_per_window": [ - 377, - 376, - 384, - 379 - ], - "limit_p95_ms": 370.9, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "mark_index_price|OKX|", - "p95_ms_per_window": [ - 1447.1, - 1361.9, - 1453.1, - 1252.2 - ], - "n_per_window": [ - 665, - 664, - 665, - 662 - ], - "limit_p95_ms": 1586.6, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "quote|BINANCE|", - "p95_ms_per_window": [ - 126.5, - 129.1, - 130.0, - 134.5 - ], - "n_per_window": [ - 546, - 555, - 548, - 560 - ], - "limit_p95_ms": 292.7, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "quote|OKX|", - "p95_ms_per_window": [ - 129.8, - 129.3, - 132.4, - 141.6 - ], - "n_per_window": [ - 591, - 584, - 566, - 598 - ], - "limit_p95_ms": 283.3, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "trade|BINANCE|", - "p95_ms_per_window": [ - 100.5, - 100.4, - 104.7, - 105.7 - ], - "n_per_window": [ - 344, - 312, - 340, - 397 - ], - "limit_p95_ms": 269.3, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - }, - { - "row": "trade|OKX|", - "p95_ms_per_window": [ - 104.1, - 104.6, - 105.0, - 110.0 - ], - "n_per_window": [ - 250, - 251, - 273, - 305 - ], - "limit_p95_ms": 257.4, - "verdict": "PASS", - "negative_samples": 0, - "missing_clock": 0 - } - ], - "windows": "4 x 120 s, 12:47-13:07Z, inside stage 100m (12:45-13:17Z), observer assigned read_committed, no join/commit", - "would_measure": "a G4 observer window long enough to see a BINANCE 30m close, frozen baselines for the five unbaselined rows, and inverse-separated rows", - "sources": [ - { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/g4m-1-summary.json", - "sha256": "2c98891f2c9e25a09a106b665573b4ff4424edcafbcd22d01fc084affe2fe904" - }, - { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/g4m-2-summary.json", - "sha256": "c6f321b95d05ca5aa27d353bae191449fad181d7a4bd2df0e2f9834830b2b287" - }, - { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/g4m-3-summary.json", - "sha256": "b349e8c4f00ed31cbba341807597fbdab61e0b2a1cfa85dc718b528c8eabc5d4" - }, - { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/g4m-4-summary.json", - "sha256": "6955061d7bf23c91e34fe885dd4a7917cce98d092b88ceffed9acbf8333a92fb" - }, - { - "path": "config/v2/p184-stage100-budget.json", - "sha256": "ff85335b3c8762d9acd8e7230f3c81ee1ec8403aff34cf2fb8f67714521c2541" + "kafka_runtime_config": { + "change": "cluster dynamic default log.cleaner.io.max.bytes.per.second=8388608", + "applied_at": "2026-10-11T03:40:56Z (before campaign 4)", + "receipt": { + "path": "~/.local/state/qdl-v2/kafka-cleaner-throttle-20261010/after.out", + "sha256": "8a635a9bd5df3efc07906db02a0fdb45cc799ac502929b46d9680b8ee884302c" + } } - ] }, - { - "id": "G5", - "quantity": "request latency", - "scope": "Query snapshot/reference classes + cold warmup shapes", - "rule": "certified stage-50 p95/p99 targets per class and venue; COLD_2500 <= 6265.1 ms, COLD_5000 <= 15267.0 ms (frozen budget)", - "verdict": "PASS", - "reason": "Every latency class and cold shape PASS in the 1800 s stage-100m run on the final digests (32/32 driver gates).", - "measured": [ - { - "gate": "latency:BAR_LATEST:BINANCE", - "count": 355, - "p50": 16.52, - "p95": 30.506, - "p99": 54.274, - "max": 291.955, - "target_p95": 1000, - "target_p99": 2000, - "status": "PASS" - }, - { - "gate": "latency:BAR_LATEST:OKX", - "count": 355, - "p50": 27.136, - "p95": 89.426, - "p99": 202.213, - "max": 340.742, - "target_p95": 1000, - "target_p99": 2000, - "status": "PASS" - }, - { - "gate": "latency:L2_SNAPSHOT:BINANCE", - "count": 355, - "p50": 40.123, - "p95": 80.944, - "p99": 229.67, - "max": 381.528, - "target_p95": 300, - "target_p99": 750, - "status": "PASS" - }, - { - "gate": "latency:L2_SNAPSHOT:OKX", - "count": 355, - "p50": 28.744, - "p95": 53.634, - "p99": 257.597, - "max": 445.148, - "target_p95": 300, - "target_p99": 750, - "status": "PASS" + "campaign": { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z", + "scorecard": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/scorecard.json", + "sha256": "0133670885374c1a280a8fcd4c34f6dd704e29904d5f492bbb08853398eb12e4" }, - { - "gate": "latency:MARK_INDEX_REFERENCE:BINANCE", - "count": 53250, - "p50": 19.348, - "p95": 58.191, - "p99": 80.126, - "max": 536.599, - "target_p95": 250, - "target_p99": 500, - "status": "PASS" + "campaign_json": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/campaign.json", + "sha256": "a7bbf80073b1796d55fdcffdcd87476de28bde4622559ab55b3ee536c35b64e9" }, - { - "gate": "latency:MARK_INDEX_REFERENCE:OKX", - "count": 53250, - "p50": 21.293, - "p95": 57.023, - "p99": 74.992, - "max": 455.591, - "target_p95": 250, - "target_p99": 500, - "status": "PASS" - }, - { - "gate": "latency:QUOTE_SNAPSHOT:BINANCE", - "count": 35500, - "p50": 13.349, - "p95": 37.49, - "p99": 74.924, - "max": 539.242, - "target_p95": 100, - "target_p99": 250, - "status": "PASS" - }, - { - "gate": "latency:QUOTE_SNAPSHOT:OKX", - "count": 35500, - "p50": 11.764, - "p95": 35.185, - "p99": 64.916, - "max": 464.515, - "target_p95": 100, - "target_p99": 250, - "status": "PASS" - }, - { - "gate": "latency:TRADE_SNAPSHOT:BINANCE", - "count": 354, - "p50": 10.332, - "p95": 34.75, - "p99": 57.465, - "max": 416.772, - "target_p95": 100, - "target_p99": 250, - "status": "PASS" - }, - { - "gate": "latency:TRADE_SNAPSHOT:OKX", - "count": 355, - "p50": 14.745, - "p95": 29.366, - "p99": 48.66, - "max": 237.098, - "target_p95": 100, - "target_p99": 250, - "status": "PASS" - }, - { - "gate": "g5:COLD_2500:BINANCE", - "count": 1, - "max_ms": 3124.007, - "limit_ms": 6265.1, - "status": "PASS" - }, - { - "gate": "g5:COLD_2500:OKX", - "count": 1, - "max_ms": 3726.708, - "limit_ms": 6265.1, - "status": "PASS" - }, - { - "gate": "g5:COLD_5000:BINANCE", - "count": 1, - "max_ms": 5298.549, - "limit_ms": 15267.0, - "status": "PASS" - }, - { - "gate": "g5:COLD_5000:OKX", - "count": 1, - "max_ms": 6177.615, - "limit_ms": 15267.0, - "status": "PASS" - } - ], - "limits": "cold shapes are n=1 per venue (max within limit); MARK_INDEX_REFERENCE is the measured reference-batch shape, not an arbitrary 50-item batch", - "sources": [ - { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/gates.json", - "sha256": "c65db308bdfbf453d3ea1027b0dec73aa00b66c1d043ca72b3ec2e1aad8bb079" - }, - { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/receipt.json", - "sha256": "9dfce5d983f25cdeb831eee6dbcd151864fa1cdbf567c9dd871f5866f67684a8" + "protocol_met": true, + "protocol_problems": [], + "load_window_unix_s": [ + 1791690245.4681032, + 1791692045.4681032 + ], + "tooling_source": "2f489b186d8d590ffa59b6dae2e41caa2fa400ad", + "tool_sha256": "39c22d78d37dbacac0042c85a6bfafafdd3dc70a9b21179c053f30a9d6cc19f3", + "demanded_slices": 64, + "writer_restarts": 0, + "campaign_summary": { + "FAIL": [ + "G2", + "G8" + ], + "NOT_MEASURED": [ + "G4", + "G12", + "G13" + ], + "PASS": [ + "G1", + "G3", + "G5", + "G6", + "G7", + "G9", + "G10", + "G11", + "G14" + ] }, - { - "path": "config/v2/p184-stage100-budget.json", - "sha256": "ff85335b3c8762d9acd8e7230f3c81ee1ec8403aff34cf2fb8f67714521c2541" - } - ] + "g12_g13": "the campaign scorer marks G12/G13 NOT_MEASURED by design; they are scored from their own receipts below" }, - { - "id": "G6", - "quantity": "durable-view event age and typed execution quality at the endpoint", - "scope": "every requested feed/venue/interval", - "rule": "per-binding D3 thresholds (TRADE 3000, QUOTE/MARK/BOOK_DELTA 2000, BOOK_SNAPSHOT 60000, BAR1m 180000 ms) plus stricter TS use-time policy; zero unsafe eligibility", - "verdict": "NOT_MEASURED", - "reason": "The stage-100 receipt records no per-binding event age or eligibility of the returned views.", - "related_observation": "stage100m: 182709 offered requests, 0 failures (0 typed refusals of any code), 180 streams with 0 errors", - "would_measure": "driver receipt fields for per-binding view event age (SDK freshness) and execution eligibility of every returned view, including refusals", - "sources": [ - { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/gates.json", - "sha256": "c65db308bdfbf453d3ea1027b0dec73aa00b66c1d043ca72b3ec2e1aad8bb079" - } - ] + "summary": { + "PASS": [ + "G1", + "G3", + "G5", + "G6", + "G7", + "G9", + "G10", + "G11", + "G14" + ], + "FAIL": [ + "G2", + "G8" + ], + "NOT_MEASURED": [ + "G4", + "G12" + ], + "DEFERRED": [ + "G13" + ] }, - { - "id": "G7", - "quantity": "TS consumer path", - "scope": "SDK return -> writer entry; writer entry -> Redis APPLY reply", - "rule": "p95 <= 100 ms and p95 <= 50 ms per feed", - "verdict": "NOT_MEASURED", - "observed": "INSIDE for every feed in both trace windows", - "reason": "Writer-only trace windows, not the frozen stage-100 protocol.", - "measured": [ - { - "window": "window1_mark_delay_binance_250", - "feed": "BOOK_DELTA", - "sdk_to_writer_ms": { - "n": 51429, - "p50_ms": 22.3, - "p95_ms": 29.7, - "p99_ms": 58.6, - "max_ms": 372.2 - }, - "writer_to_redis_ms": { - "n": 51058, - "p50_ms": 2.1, - "p95_ms": 9.1, - "p99_ms": 16.2, - "max_ms": 192.3 - }, - "observed": { - "sdk_to_writer_ms": "INSIDE", - "writer_to_redis_ms": "INSIDE" - } - }, - { - "window": "window1_mark_delay_binance_250", - "feed": "BOOK_SNAPSHOT", - "sdk_to_writer_ms": { - "n": 5870, - "p50_ms": 28.8, - "p95_ms": 39.5, - "p99_ms": 92.9, - "max_ms": 190.7 - }, - "writer_to_redis_ms": { - "n": 5874, - "p50_ms": 4.0, - "p95_ms": 14.0, - "p99_ms": 40.5, - "max_ms": 192.2 - }, - "observed": { - "sdk_to_writer_ms": "INSIDE", - "writer_to_redis_ms": "INSIDE" - } - }, + "owner_decision": "2026-10-11 (AskUserQuestion): release the DL with G2 and G8 FAIL recorded; writer throughput (SDK batch decode, Rust projection, EVAL chunking) is the first item of TS phase E; no gate is widened. This is not a PASS and does not change any verdict", + "gates": [ { - "window": "window1_mark_delay_binance_250", - "feed": "MARK_INDEX_PRICE", - "sdk_to_writer_ms": { - "n": 9052, - "p50_ms": 0.4, - "p95_ms": 0.6, - "p99_ms": 0.6, - "max_ms": 1.2 - }, - "writer_to_redis_ms": { - "n": 9052, - "p50_ms": 2.1, - "p95_ms": 7.0, - "p99_ms": 13.3, - "max_ms": 94.5 - }, - "observed": { - "sdk_to_writer_ms": "INSIDE", - "writer_to_redis_ms": "INSIDE" - } + "id": "G1", + "measured": [ + { + "feed_venue": "BOOK_DELTA|BINANCE", + "max": 1736.7, + "n": 83994, + "observed": "INSIDE", + "p50": 108.4, + "p95": 177.4, + "p99": 334.1, + "split_p95_ms": { + "ingest_to_sdk_ms": 90.8, + "sdk_to_reply_ms": 62.6, + "venue_to_ingest_ms": 30.1 + } + }, + { + "feed_venue": "BOOK_DELTA|OKX", + "max": 1844.2, + "n": 97565, + "observed": "INSIDE", + "p50": 115.3, + "p95": 191.5, + "p99": 380.8, + "split_p95_ms": { + "ingest_to_sdk_ms": 93.7, + "sdk_to_reply_ms": 72.5, + "venue_to_ingest_ms": 34.3 + } + }, + { + "feed_venue": "BOOK_SNAPSHOT|BINANCE", + "max": 1211.4, + "n": 9667, + "observed": "INSIDE", + "p50": 123.0, + "p95": 245.6, + "p99": 423.9, + "split_p95_ms": { + "ingest_to_sdk_ms": 114.7, + "sdk_to_reply_ms": 117.7, + "venue_to_ingest_ms": 31.6 + } + }, + { + "feed_venue": "BOOK_SNAPSHOT|OKX", + "max": 1144.5, + "n": 11562, + "observed": "INSIDE", + "p50": 127.7, + "p95": 236.7, + "p99": 419.0, + "split_p95_ms": { + "ingest_to_sdk_ms": 105.9, + "sdk_to_reply_ms": 121.7, + "venue_to_ingest_ms": 34.6 + } + } + ], + "quantity": "end-to-end to TS cache (venue event -> TS Redis APPLY reply)", + "reason": "every BOOK row inside", + "rule": "p95 <= 500 ms, p99 <= 1000 ms (n >= 100), max < 2000 ms per feed/venue", + "sources": { + "transit": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/transit-load.json", + "sha256": "d1376db1a50db991df5e9e9e80212463ad095640ee00a67ab673e473145d73f6" + }, + "mark": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/mark-load.json", + "sha256": "7ac870af08414f19f852af5851928a3e3030fd5898ade38b1ff8e417ec6850a6" + } + }, + "verdict": "PASS" }, { - "window": "window1_mark_delay_binance_250", - "feed": "QUOTE", - "sdk_to_writer_ms": { - "n": 48707, - "p50_ms": 22.1, - "p95_ms": 28.7, - "p99_ms": 39.6, - "max_ms": 175.3 - }, - "writer_to_redis_ms": { - "n": 54515, - "p50_ms": 1.1, - "p95_ms": 4.1, - "p99_ms": 10.5, - "max_ms": 140.1 - }, - "observed": { - "sdk_to_writer_ms": "INSIDE", - "writer_to_redis_ms": "INSIDE" - } + "id": "G2", + "measured": [ + { + "feed_venue": "TRADE|BINANCE", + "max": 5338.8, + "n": 160328, + "observed": "OUTSIDE", + "p50": 153.7, + "p95": 1562.1, + "p99": 4326.4, + "split_p95_ms": { + "ingest_to_sdk_ms": 1430.7, + "sdk_to_reply_ms": 118.2, + "venue_to_ingest_ms": 39.6 + } + }, + { + "feed_venue": "TRADE|OKX", + "max": 4301.8, + "n": 39292, + "observed": "OUTSIDE", + "p50": 138.3, + "p95": 2084.2, + "p99": 3782.9, + "split_p95_ms": { + "ingest_to_sdk_ms": 1866.1, + "sdk_to_reply_ms": 152.8, + "venue_to_ingest_ms": 72.6 + } + }, + { + "feed_venue": "QUOTE|BINANCE", + "max": 1439.2, + "n": 116367, + "observed": "OUTSIDE", + "p50": 125.9, + "p95": 188.0, + "p99": 300.2, + "split_p95_ms": { + "ingest_to_sdk_ms": 120.0, + "sdk_to_reply_ms": 46.9, + "venue_to_ingest_ms": 29.7 + } + }, + { + "feed_venue": "QUOTE|OKX", + "max": 1340.6, + "n": 98174, + "observed": "OUTSIDE", + "p50": 127.2, + "p95": 207.1, + "p99": 339.3, + "split_p95_ms": { + "ingest_to_sdk_ms": 125.0, + "sdk_to_reply_ms": 58.4, + "venue_to_ingest_ms": 30.5 + } + }, + { + "feed_venue": "MARK_INDEX_PRICE|BINANCE", + "max": 1125.4, + "n": 8989, + "observed": "OUTSIDE", + "p50": 302.0, + "p95": 396.7, + "p99": 537.2 + }, + { + "feed_venue": "MARK_INDEX_PRICE|OKX", + "max": 1227.8, + "n": 21437, + "observed": "OUTSIDE", + "p50": 234.9, + "p95": 348.5, + "p99": 411.0 + } + ], + "quantity": "end-to-end to TS cache, new-event transit", + "reason": "outside: TRADE|BINANCE, TRADE|OKX, QUOTE|BINANCE, QUOTE|OKX; outside: MARK|BINANCE, MARK|OKX", + "rule": "TRADE p95 <= 500 ms and max < 1500 ms; QUOTE and MARK p95 <= 500 ms and max < 1000 ms", + "sources": { + "transit": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/transit-load.json", + "sha256": "d1376db1a50db991df5e9e9e80212463ad095640ee00a67ab673e473145d73f6" + }, + "mark": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/mark-load.json", + "sha256": "7ac870af08414f19f852af5851928a3e3030fd5898ade38b1ff8e417ec6850a6" + } + }, + "verdict": "FAIL", + "attribution": { + "basis": "read-only per-second analysis of the campaign-4 delivery trace (214 trace parts, load window of the campaign scorecard), records APPLIED with SDK_STREAM_RETURN and a venue clock; run in a disposable --network none container with the trace mounted read-only", + "seconds_in_window": 1801, + "seconds_max_e2e_lt_500_ms": 1699, + "seconds_max_e2e_gt_1500_ms": 28, + "seconds_gt_1500_where_every_record_over_1500_is_TRADE": 25, + "seconds_gt_1500_that_also_hold_BOOK_DELTA_over_1500": "3 BINANCE / 2 OKX second-feed hits (G1 max still < 2000 ms)", + "largest_TRADE_burst": "4274 TRADE records in one source second on one slice (BINANCE:USD_M:ETHUSDT) on one writer worker", + "max_applied_TRADE_per_worker_second": 1138, + "TRADE_records_over_1500_ms_by_slice": { + "BINANCE:USD_M:ETHUSDT": 4844, + "BINANCE:USD_M:SOLUSDT": 2287, + "OKX:SWAP:ETH-USDT-SWAP": 2211, + "BINANCE:USD_M:BNBUSDT": 642, + "BINANCE:USD_M:BTCUSDT": 572, + "OKX:SWAP:SOL-USDT-SWAP": 469 + }, + "segments_p95_ms_TRADE_BINANCE": { + "ingest_to_sdk_ms": 1430.7, + "sdk_to_reply_ms": 118.2, + "venue_to_ingest_ms": 39.6 + }, + "canonical_visibility": "G4 venue -> canonical TRADE p95 108.2-136.6 ms in all 5 windows of the same campaign", + "TRADE_volume": "TRADE BINANCE n 160328 against 69966 / 74061 in the 2026-10-10 campaigns 171129Z / 174944Z (about 2.2x)", + "integrity": "G10 in the same campaign: 0 gaps, 0 duplicates applied, 0 replay-expired, 0 overflow", + "reading": "the tail is single-slice TRADE bursts queued behind one writer worker (arrival up to ~4.3k rec/s vs at most ~1.1k applied rec/s per worker); the DL canonical path stays inside G4. Stream-side dispatch/queue metrics for this campaign were not retained in a file and are not cited here.", + "owner_decision": "2026-10-11 (AskUserQuestion): release DL; writer throughput (SDK batch decode, Rust projection, EVAL chunking) moves to TS phase E as its first item; the gate is not widened" + } }, { - "window": "window1_mark_delay_binance_250", - "feed": "TRADE", - "sdk_to_writer_ms": { - "n": 27066, - "p50_ms": 22.9, - "p95_ms": 43.2, - "p99_ms": 95.7, - "max_ms": 207.2 - }, - "writer_to_redis_ms": { - "n": 27398, - "p50_ms": 5.1, - "p95_ms": 36.9, - "p99_ms": 98.9, - "max_ms": 120.2 - }, - "observed": { - "sdk_to_writer_ms": "INSIDE", - "writer_to_redis_ms": "INSIDE" - } + "id": "G3", + "measured": { + "per_venue": { + "BINANCE": { + "max": 3007, + "min": 109, + "n": 160, + "p50": 468, + "p95": 1754, + "p99": 2861 + }, + "OKX": { + "max": 1822, + "min": 316, + "n": 160, + "p50": 1034, + "p95": 1535, + "p99": 1822 + } + }, + "poll_ms": 250, + "pooled": { + "max": 3007, + "min": 109, + "n": 320, + "p50": 758, + "p95": 1535, + "p99": 2435 + }, + "read_errors": 0, + "unresolved_slices": [], + "unseen_total": 0 + }, + "quantity": "end-to-end to TS cache, BAR from close (first visibility of the closed bar)", + "reason": "g3 probe", + "rule": "p95 <= 5 s, max < 60 s", + "sources": { + "g3": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g3/g3-summary.json", + "sha256": "a15a368f3c05dc0de7f0d072bf2518574fea0168d3c3ca73b21558575ae13574" + } + }, + "verdict": "PASS" }, { - "window": "window2_deployed_config", - "feed": "BOOK_DELTA", - "sdk_to_writer_ms": { - "n": 51162, - "p50_ms": 22.3, - "p95_ms": 29.5, - "p99_ms": 48.6, - "max_ms": 218.7 - }, - "writer_to_redis_ms": { - "n": 50788, - "p50_ms": 2.1, - "p95_ms": 8.9, - "p99_ms": 16.0, - "max_ms": 150.1 - }, - "observed": { - "sdk_to_writer_ms": "INSIDE", - "writer_to_redis_ms": "INSIDE" - } + "id": "G4", + "inverse_rows": [ + { + "book_delta|OKX_INVERSE|": { + "max": 108.9, + "n": 106, + "p95": 101.6 + }, + "book_snapshot|OKX_INVERSE|": { + "max": 156.4, + "n": 112, + "p95": 102.3 + }, + "mark_index_price|OKX_INVERSE|": { + "max": 1793.0, + "n": 110, + "p95": 1310.1 + }, + "quote|OKX_INVERSE|": { + "max": 147.5, + "n": 64, + "p95": 139.0 + } + }, + { + "book_delta|OKX_INVERSE|": { + "max": 198.5, + "n": 112, + "p95": 121.0 + }, + "book_snapshot|OKX_INVERSE|": { + "max": 198.7, + "n": 124, + "p95": 143.3 + }, + "mark_index_price|OKX_INVERSE|": { + "max": 2041.9, + "n": 109, + "p95": 1248.0 + }, + "quote|OKX_INVERSE|": { + "max": 298.2, + "n": 96, + "p95": 154.8 + } + }, + { + "book_delta|OKX_INVERSE|": { + "max": 126.2, + "n": 112, + "p95": 107.2 + }, + "book_snapshot|OKX_INVERSE|": { + "max": 147.9, + "n": 120, + "p95": 109.6 + }, + "mark_index_price|OKX_INVERSE|": { + "max": 1610.9, + "n": 110, + "p95": 1076.3 + }, + "quote|OKX_INVERSE|": { + "max": 217.1, + "n": 70, + "p95": 145.4 + } + }, + { + "book_delta|OKX_INVERSE|": { + "max": 232.3, + "n": 125, + "p95": 121.9 + }, + "book_snapshot|OKX_INVERSE|": { + "max": 199.3, + "n": 142, + "p95": 126.8 + }, + "mark_index_price|OKX_INVERSE|": { + "max": 1932.8, + "n": 123, + "p95": 1079.1 + }, + "quote|OKX_INVERSE|": { + "max": 289.6, + "n": 99, + "p95": 162.8 + } + }, + { + "book_delta|OKX_INVERSE|": { + "max": 340.5, + "n": 113, + "p95": 118.7 + }, + "book_snapshot|OKX_INVERSE|": { + "max": 510.7, + "n": 128, + "p95": 141.4 + }, + "mark_index_price|OKX_INVERSE|": { + "max": 2032.2, + "n": 111, + "p95": 1166.9 + }, + "quote|OKX_INVERSE|": { + "max": 385.5, + "n": 81, + "p95": 148.6 + } + } + ], + "measured": [ + { + "limit_p95_ms": 40728.7, + "n_per_window": [ + 3, + 0, + 0, + 0, + 0 + ], + "p95_ms_per_window": [ + 7106.7, + null, + null, + null, + null + ], + "row": "bar|BINANCE|15m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 257.6, + "n_per_window": [ + 10, + 10, + 10, + 10, + 10 + ], + "p95_ms_per_window": [ + 127.5, + 106.6, + 116.9, + 140.6, + 94.3 + ], + "row": "bar|BINANCE|1m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 25836.6, + "n_per_window": [ + 0, + 0, + 0, + 1, + 0 + ], + "p95_ms_per_window": [ + null, + null, + null, + 11641.8, + null + ], + "row": "bar|BINANCE|30m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 27273.2, + "n_per_window": [ + 1, + 5, + 5, + 0, + 5 + ], + "p95_ms_per_window": [ + 7099.6, + 6853.7, + 6838.9, + null, + 6840.1 + ], + "row": "bar|BINANCE|3m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 73817.2, + "n_per_window": [ + 6, + 8, + 7, + 6, + 9 + ], + "p95_ms_per_window": [ + 61364.0, + 61099.9, + 61105.6, + 61223.5, + 61238.3 + ], + "row": "bar|OKX|1m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 2160788.8, + "n_per_window": [ + 0, + 0, + 0, + 1, + 0 + ], + "p95_ms_per_window": [ + null, + null, + null, + 1801148.6, + null + ], + "row": "bar|OKX|30m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 217582.1, + "n_per_window": [ + 3, + 2, + 3, + 1, + 1 + ], + "p95_ms_per_window": [ + 181246.0, + 181058.9, + 180978.3, + 180970.0, + 181286.0 + ], + "row": "bar|OKX|3m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 263.3, + "n_per_window": [ + 763, + 790, + 791, + 880, + 783 + ], + "p95_ms_per_window": [ + 101.4, + 130.3, + 106.6, + 128.7, + 124.1 + ], + "row": "book_delta|BINANCE|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 289.1, + "n_per_window": [ + 1085, + 1076, + 1081, + 1210, + 1077 + ], + "p95_ms_per_window": [ + 106.1, + 132.9, + 112.1, + 132.7, + 125.6 + ], + "row": "book_delta|OKX|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 260.0, + "n_per_window": [ + 1000, + 1329, + 1200, + 1687, + 1258 + ], + "p95_ms_per_window": [ + 103.4, + 155.1, + 108.4, + 140.5, + 148.4 + ], + "row": "book_snapshot|BINANCE|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 270.1, + "n_per_window": [ + 1713, + 2375, + 2293, + 2712, + 2434 + ], + "p95_ms_per_window": [ + 107.2, + 152.5, + 113.4, + 142.6, + 140.5 + ], + "row": "book_snapshot|OKX|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 370.9, + "n_per_window": [ + 374, + 390, + 379, + 435, + 382 + ], + "p95_ms_per_window": [ + 198.4, + 220.2, + 202.8, + 234.8, + 207.7 + ], + "row": "mark_index_price|BINANCE|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 1586.6, + "n_per_window": [ + 662, + 660, + 662, + 739, + 662 + ], + "p95_ms_per_window": [ + 1432.2, + 1281.4, + 1372.2, + 1109.2, + 1155.3 + ], + "row": "mark_index_price|OKX|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 292.7, + "n_per_window": [ + 554, + 578, + 564, + 640, + 571 + ], + "p95_ms_per_window": [ + 131.6, + 152.5, + 137.5, + 158.3, + 152.0 + ], + "row": "quote|BINANCE|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 283.3, + "n_per_window": [ + 601, + 652, + 625, + 727, + 634 + ], + "p95_ms_per_window": [ + 134.8, + 155.5, + 135.6, + 155.2, + 152.0 + ], + "row": "quote|OKX|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 269.3, + "n_per_window": [ + 319, + 439, + 401, + 522, + 447 + ], + "p95_ms_per_window": [ + 108.2, + 133.2, + 109.0, + 125.8, + 118.9 + ], + "row": "trade|BINANCE|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 257.4, + "n_per_window": [ + 217, + 353, + 295, + 423, + 351 + ], + "p95_ms_per_window": [ + 111.5, + 125.4, + 110.8, + 136.6, + 129.6 + ], + "row": "trade|OKX|", + "verdict": "PASS" + } + ], + "quantity": "delivery lag, venue -> canonical (read_committed visibility, upper bound)", + "reason": "observed rows without a frozen baseline: bar|BINANCE|2h, bar|BINANCE|4h, bar|BINANCE|5m, bar|OKX|15m, bar|OKX|4h, bar|OKX|5m, book_delta|OKX_INVERSE|, book_snapshot|OKX_INVERSE|, mark_index_price|OKX_INVERSE|, quote|OKX_INVERSE|", + "rule": "candidate p95 <= frozen baseline p95 x 1.2 per row; missing coverage blocks the gate", + "verdict": "NOT_MEASURED", + "sources": { + "g4": [ + { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-1-summary.json", + "sha256": "02a554c56d9215c9e48678df5389305f1acd1a108da1d34bf46bca455a581e90" + }, + { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-2-summary.json", + "sha256": "42aa78a1bb21566c65510e53afe6b4db0afd0af52d9abc6b21b4e538af88a587" + }, + { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-3-summary.json", + "sha256": "013e6929f3bb978d16acb37e6e64e0d4961971c7c7cf43e9dbf5f52cc617e57e" + }, + { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-4-summary.json", + "sha256": "86494b6575b97c101459cb6e98257f3c3d4f1a98fae62b2482d69d940079288d" + }, + { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-5-summary.json", + "sha256": "0111526b3ba62b40c8bfd5c4dd30cb5d9f6dd7a4ea14db0043beebbeb54ca2fe" + } + ] + }, + "baselined_rows": { + "rows": 17, + "pass": 17 + }, + "note": "every row with a frozen baseline PASS in every window it was sampled; the gate stays NOT_MEASURED because observed rows have no frozen baseline and no baseline is added after measuring" }, { - "window": "window2_deployed_config", - "feed": "BOOK_SNAPSHOT", - "sdk_to_writer_ms": { - "n": 5829, - "p50_ms": 28.8, - "p95_ms": 37.8, - "p99_ms": 90.8, - "max_ms": 151.2 - }, - "writer_to_redis_ms": { - "n": 5836, - "p50_ms": 4.0, - "p95_ms": 13.3, - "p99_ms": 32.4, - "max_ms": 85.2 - }, - "observed": { - "sdk_to_writer_ms": "INSIDE", - "writer_to_redis_ms": "INSIDE" - } + "id": "G5", + "measured": [ + { + "count": 355, + "gate": "latency:BAR_LATEST:BINANCE", + "max": 396.156, + "p50": 21.767, + "p95": 45.867, + "p99": 237.583, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 1000, + "target_p99": 2000 + }, + { + "count": 355, + "gate": "latency:BAR_LATEST:OKX", + "max": 622.885, + "p50": 28.809, + "p95": 59.219, + "p99": 205.444, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 1000, + "target_p99": 2000 + }, + { + "count": 355, + "gate": "latency:L2_SNAPSHOT:BINANCE", + "max": 417.651, + "p50": 26.463, + "p95": 65.632, + "p99": 293.751, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 300, + "target_p99": 750 + }, + { + "count": 355, + "gate": "latency:L2_SNAPSHOT:OKX", + "max": 322.334, + "p50": 25.784, + "p95": 94.998, + "p99": 230.694, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 300, + "target_p99": 750 + }, + { + "count": 53250, + "gate": "latency:MARK_INDEX_REFERENCE:BINANCE", + "max": 698.315, + "p50": 23.67, + "p95": 61.139, + "p99": 103.998, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 250, + "target_p99": 500 + }, + { + "count": 53250, + "gate": "latency:MARK_INDEX_REFERENCE:OKX", + "max": 716.352, + "p50": 22.85, + "p95": 62.527, + "p99": 93.194, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 250, + "target_p99": 500 + }, + { + "count": 35500, + "gate": "latency:QUOTE_SNAPSHOT:BINANCE", + "max": 626.853, + "p50": 13.151, + "p95": 34.995, + "p99": 89.424, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 100, + "target_p99": 250 + }, + { + "count": 35500, + "gate": "latency:QUOTE_SNAPSHOT:OKX", + "max": 675.312, + "p50": 13.823, + "p95": 39.259, + "p99": 80.913, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 100, + "target_p99": 250 + }, + { + "count": 354, + "gate": "latency:TRADE_SNAPSHOT:BINANCE", + "max": 234.675, + "p50": 10.968, + "p95": 29.074, + "p99": 84.744, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 100, + "target_p99": 250 + }, + { + "count": 355, + "gate": "latency:TRADE_SNAPSHOT:OKX", + "max": 329.802, + "p50": 20.276, + "p95": 41.388, + "p99": 198.375, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 100, + "target_p99": 250 + }, + { + "bad": 0, + "gate": "cold:2500_5000_overlap_per_venue", + "runs": 4, + "status": "PASS" + } + ], + "quantity": "request latency", + "reason": "11 latency/cold gates", + "rule": "certified stage-50 p95/p99 per class and venue; frozen cold-shape limits", + "verdict": "PASS", + "sources": { + "gates": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/stage100/gates.json", + "sha256": "63f12e8c6df961b0505fd75ab8df9fe2a34e4fa7b2d985cb025e17a6785bcc43" + }, + "receipt": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/stage100/receipt.json", + "sha256": "a79c9c9d5989f3befb770449badac8cdc9330ec30365b659b96c61cb7e54a0c3" + } + } }, { - "window": "window2_deployed_config", - "feed": "MARK_INDEX_PRICE", - "sdk_to_writer_ms": { - "n": 9061, - "p50_ms": 0.4, - "p95_ms": 0.6, - "p99_ms": 0.6, - "max_ms": 1.2 - }, - "writer_to_redis_ms": { - "n": 9061, - "p50_ms": 2.2, - "p95_ms": 6.8, - "p99_ms": 13.3, - "max_ms": 85.9 - }, - "observed": { - "sdk_to_writer_ms": "INSIDE", - "writer_to_redis_ms": "INSIDE" - } + "id": "G6", + "measured": { + "FUNDING_RATE|BINANCE|@REQUIREMENT_ALPHA": { + "age_basis": "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR", + "age_ms": { + "basis": "reservoir sample per worker (exact n and max above)", + "max_ms": 15242197.0, + "n": 150, + "p50_ms": 14352510.5, + "p95_ms": 15156931.0, + "p99_ms": 15241597.0 + }, + "classes": { + "ELIGIBLE": 150 + }, + "max_age_ms": 15242197, + "max_client_age_ms": 15242303, + "n": 150, + "policy_basis": "REQUIREMENT_ALPHA", + "policy_ms": 32400000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "FUNDING_RATE|OKX|@REQUIREMENT_ALPHA": { + "age_basis": "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR", + "age_ms": { + "basis": "reservoir sample per worker (exact n and max above)", + "max_ms": 15239191.0, + "n": 150, + "p50_ms": 14342900.0, + "p95_ms": 15141540.0, + "p99_ms": 15223646.0 + }, + "classes": { + "ELIGIBLE": 150 + }, + "max_age_ms": 15239191, + "max_client_age_ms": 15239300, + "n": 150, + "policy_basis": "REQUIREMENT_ALPHA", + "policy_ms": 28800000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "MARK_INDEX_PRICE|BINANCE|#ts-probe": { + "age_basis": "CLIENT_RECEIPT_MINUS_NEWEST_SOURCE_EVENT", + "age_ms": { + "basis": "first 20000 ages per key (exact n and max above)", + "max_ms": 1582, + "n": 2990, + "p50_ms": 670.0, + "p95_ms": 1118, + "p99_ms": 1176 + }, + "classes": { + "ELIGIBLE": 2990 + }, + "max_age_ms": 1582, + "max_client_age_ms": 1501, + "n": 2990, + "policy_basis": "D3_EXECUTION", + "policy_ms": 2000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "MARK_INDEX_PRICE|BINANCE|@REQUIREMENT_ALPHA": { + "age_basis": "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR", + "age_ms": { + "basis": "reservoir sample per worker (exact n and max above)", + "max_ms": 1184.0, + "n": 4096, + "p50_ms": 669.0, + "p95_ms": 1048.0, + "p99_ms": 1143.0 + }, + "classes": { + "ELIGIBLE": 63000 + }, + "max_age_ms": 1668, + "max_client_age_ms": 1780, + "n": 63000, + "policy_basis": "REQUIREMENT_ALPHA", + "policy_ms": 60000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "MARK_INDEX_PRICE|OKX|#ts-probe": { + "age_basis": "CLIENT_RECEIPT_MINUS_NEWEST_SOURCE_EVENT", + "age_ms": { + "basis": "first 20000 ages per key (exact n and max above)", + "max_ms": 2177, + "n": 3588, + "p50_ms": 502.0, + "p95_ms": 1331, + "p99_ms": 1923 + }, + "classes": { + "ELIGIBLE": 3563, + "ELIGIBLE_QUIET": 25 + }, + "max_age_ms": 2177, + "max_client_age_ms": 2109, + "n": 3588, + "policy_basis": "D3_EXECUTION", + "policy_ms": 2000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "MARK_INDEX_PRICE|OKX|@REQUIREMENT_ALPHA": { + "age_basis": "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR", + "age_ms": { + "basis": "reservoir sample per worker (exact n and max above)", + "max_ms": 2286.0, + "n": 4096, + "p50_ms": 444.5, + "p95_ms": 1265.0, + "p99_ms": 1866.0 + }, + "classes": { + "ELIGIBLE": 63180 + }, + "max_age_ms": 2291, + "max_client_age_ms": 2401, + "n": 63180, + "policy_basis": "REQUIREMENT_ALPHA", + "policy_ms": 60000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "QUOTE|BINANCE|#ts-probe": { + "age_basis": null, + "age_ms": { + "basis": "first 20000 ages per key (exact n and max above)", + "max_ms": 1237, + "n": 2990, + "p50_ms": 324.0, + "p95_ms": 530, + "p99_ms": 739 + }, + "classes": { + "ELIGIBLE": 2990 + }, + "max_age_ms": 1237, + "max_client_age_ms": null, + "n": 2990, + "policy_basis": "D3_EXECUTION", + "policy_ms": 2000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "QUOTE|BINANCE|@REQUIREMENT_ALPHA": { + "age_basis": null, + "age_ms": { + "basis": "reservoir sample per worker (exact n and max above)", + "max_ms": 1277.0, + "n": 4096, + "p50_ms": 321.0, + "p95_ms": 522.0, + "p99_ms": 708.0 + }, + "classes": { + "ELIGIBLE": 36000 + }, + "max_age_ms": 1580, + "max_client_age_ms": null, + "n": 36000, + "policy_basis": "REQUIREMENT_ALPHA", + "policy_ms": 5000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "QUOTE|OKX|#ts-probe": { + "age_basis": null, + "age_ms": { + "basis": "first 20000 ages per key (exact n and max above)", + "max_ms": 13027, + "n": 3583, + "p50_ms": 353, + "p95_ms": 1083, + "p99_ms": 3074 + }, + "classes": { + "ELIGIBLE": 3515, + "ELIGIBLE_QUIET": 68 + }, + "max_age_ms": 13027, + "max_client_age_ms": null, + "n": 3583, + "policy_basis": "D3_EXECUTION", + "policy_ms": 2000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "QUOTE|OKX|@REQUIREMENT_ALPHA": { + "age_basis": null, + "age_ms": { + "basis": "reservoir sample per worker (exact n and max above)", + "max_ms": 3889.0, + "n": 4096, + "p50_ms": 338.0, + "p95_ms": 675.0, + "p99_ms": 1015.0 + }, + "classes": { + "ELIGIBLE": 36100 + }, + "max_age_ms": 3990, + "max_client_age_ms": null, + "n": 36100, + "policy_basis": "REQUIREMENT_ALPHA", + "policy_ms": 5000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "TRADE|BINANCE|#ts-probe": { + "age_basis": null, + "age_ms": { + "basis": "first 20000 ages per key (exact n and max above)", + "max_ms": 6770, + "n": 2985, + "p50_ms": 558, + "p95_ms": 2195, + "p99_ms": 3565 + }, + "classes": { + "ELIGIBLE": 2936, + "NOT_ELIGIBLE": 49 + }, + "max_age_ms": 6770, + "max_client_age_ms": null, + "n": 2985, + "policy_basis": "D3_EXECUTION", + "policy_ms": 3000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "TRADE|OKX|#ts-probe": { + "age_basis": null, + "age_ms": { + "basis": "first 20000 ages per key (exact n and max above)", + "max_ms": 40706, + "n": 2985, + "p50_ms": 807, + "p95_ms": 7967, + "p99_ms": 19634 + }, + "classes": { + "ELIGIBLE": 2522, + "NOT_ELIGIBLE": 463 + }, + "max_age_ms": 40706, + "max_client_age_ms": null, + "n": 2985, + "policy_basis": "D3_EXECUTION", + "policy_ms": 3000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + } + }, + "quantity": "durable-view event age and typed execution quality at the Query endpoint", + "reason": "every required D3 binding measured, 0 unsafe", + "rule": "per-binding D3 policy of the TS EXECUTION consumer (TRADE 3000, QUOTE/MARK/BOOK_DELTA 2000, BOOK_SNAPSHOT 60000, BAR1m 180000 ms, min with the requirement); zero unsafe eligibility; decided by D3_EXECUTION rows only", + "sources": { + "g6_ts": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g6-ts/summary.json", + "sha256": "229e038090ec1000c19ceca41eb805bcda1b4a7827e2e0b41803f4cc60aeb659" + } + }, + "ts_probe": { + "errors": {}, + "per_binding_period_s": 3.2, + "reads": 19121, + "stopped_by": "signal", + "targets": 32, + "window_unix_s": [ + 1791690153.9782138, + 1791692078.7164075 + ] + }, + "verdict": "PASS", + "limits": [ + "BOOK_DELTA has no Query shape in the frozen workload, so it is not a G6 row", + "scorer: ba496ff (read-own-requirement, D3 by TS EXECUTION probe) + 2f489b1 (anchor per product)" + ] }, { - "window": "window2_deployed_config", - "feed": "QUOTE", - "sdk_to_writer_ms": { - "n": 47053, - "p50_ms": 22.1, - "p95_ms": 28.6, - "p99_ms": 38.1, - "max_ms": 206.9 - }, - "writer_to_redis_ms": { - "n": 52863, - "p50_ms": 1.1, - "p95_ms": 4.0, - "p99_ms": 10.4, - "max_ms": 100.5 - }, - "observed": { - "sdk_to_writer_ms": "INSIDE", - "writer_to_redis_ms": "INSIDE" - } + "id": "G7", + "measured": { + "BOOK_DELTA": { + "sdk_to_writer_ms": { + "max": 1311.7, + "min": 0.3, + "n": 182930, + "p50": 23.0, + "p95": 51.0, + "p99": 135.5 + }, + "writer_to_redis_ms": { + "max": 530.3, + "min": 0.5, + "n": 181572, + "p50": 3.4, + "p95": 18.4, + "p99": 52.0 + } + }, + "BOOK_SNAPSHOT": { + "sdk_to_writer_ms": { + "max": 616.3, + "min": 12.3, + "n": 21229, + "p50": 31.0, + "p95": 79.3, + "p99": 141.0 + }, + "writer_to_redis_ms": { + "max": 379.4, + "min": 1.5, + "n": 21229, + "p50": 7.3, + "p95": 41.2, + "p99": 128.4 + } + }, + "MARK_INDEX_PRICE": { + "sdk_to_writer_ms": { + "max": 8.1, + "min": 0.2, + "n": 30441, + "p50": 0.5, + "p95": 0.6, + "p99": 0.9 + }, + "writer_to_redis_ms": { + "max": 520.7, + "min": 1.1, + "n": 30441, + "p50": 4.0, + "p95": 20.2, + "p99": 79.3 + } + }, + "QUOTE": { + "sdk_to_writer_ms": { + "max": 661.6, + "min": 1.3, + "n": 214601, + "p50": 22.6, + "p95": 40.7, + "p99": 99.3 + }, + "writer_to_redis_ms": { + "max": 473.8, + "min": 0.5, + "n": 233880, + "p50": 1.4, + "p95": 10.5, + "p99": 48.6 + } + }, + "TRADE": { + "sdk_to_writer_ms": { + "max": 501.5, + "min": 0.3, + "n": 203657, + "p50": 26.9, + "p95": 83.0, + "p99": 148.1 + }, + "writer_to_redis_ms": { + "max": 438.0, + "min": 0.5, + "n": 204778, + "p50": 7.7, + "p95": 46.7, + "p99": 101.9 + } + } + }, + "quantity": "TS consumer path", + "reason": "every feed inside", + "rule": "SDK -> writer p95 <= 100 ms; writer -> Redis reply p95 <= 50 ms per feed", + "verdict": "PASS", + "sources": { + "transit": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/transit-load.json", + "sha256": "d1376db1a50db991df5e9e9e80212463ad095640ee00a67ab673e473145d73f6" + } + } }, { - "window": "window2_deployed_config", - "feed": "TRADE", - "sdk_to_writer_ms": { - "n": 19820, - "p50_ms": 22.9, - "p95_ms": 41.5, - "p99_ms": 88.4, - "max_ms": 157.5 - }, - "writer_to_redis_ms": { - "n": 20131, - "p50_ms": 4.6, - "p95_ms": 26.2, - "p99_ms": 52.4, - "max_ms": 94.5 - }, - "observed": { - "sdk_to_writer_ms": "INSIDE", - "writer_to_redis_ms": "INSIDE" - } - } - ], - "measurement_scope": "TS writer delivery trace, two 535 s analysis windows (13:19-13:28Z and 13:31-13:40Z, 40 s writer start-up skipped), writer-only production traffic AFTER stage 100m ended (13:17Z): no stage-100 alpha load, no burst window", - "would_measure": "the same analysis over >= 30 min of trace during a stage-100 run", - "sources": [ - { - "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791638906937534999.json", - "sha256": "d8d0d6c37dc37ccd1c22b3d9690d42041d09cc532ddd0008499d5a3d7b32526a" + "id": "G8", + "measured": { + "monitor_clients_max": 0, + "ping_all_ms": { + "max": 5.869, + "min": 0.1, + "n": 3850, + "p50": 0.154, + "p95": 0.51, + "p99": 1.275 + }, + "ping_failures": 0, + "ping_steady_ms": { + "max": 5.869, + "min": 0.1, + "n": 3599, + "p50": 0.155, + "p95": 0.516, + "p99": 1.274 + }, + "slowlog_new_steady": 6, + "slowlog_new_total": 6, + "slowlog_steady_rows": [ + { + "command": "EVALSHA", + "duration_us": 10016, + "id": 976628, + "start_s": 1791690302 + }, + { + "command": "EVALSHA", + "duration_us": 16120, + "id": 976629, + "start_s": 1791690605 + }, + { + "command": "EVALSHA", + "duration_us": 13258, + "id": 976630, + "start_s": 1791690763 + }, + { + "command": "EVALSHA", + "duration_us": 12964, + "id": 976631, + "start_s": 1791690842 + }, + { + "command": "EVALSHA", + "duration_us": 18431, + "id": 976632, + "start_s": 1791691251 + }, + { + "command": "EVALSHA", + "duration_us": 19732, + "id": 976633, + "start_s": 1791691386 + } + ], + "slowlog_threshold_us": 10000, + "windows": [ + { + "core_fraction": 0.0687, + "from_ns": 1791690153422572495, + "judged": true, + "ping_ms": { + "max": 2.9, + "min": 0.1, + "n": 1199, + "p50": 0.154, + "p95": 0.476, + "p99": 1.245 + }, + "seconds": 590.2, + "slowlog_new": 2, + "steady": false, + "to_ns": 1791690743584489495, + "top_commands_usec": { + "eval": 10145176, + "evalsha": 24499749, + "publish": 701412, + "set": 979302, + "setex": 2558834 + }, + "valid": true + }, + { + "core_fraction": 0.0868, + "from_ns": 1791690753588790195, + "judged": true, + "ping_ms": { + "max": 5.869, + "min": 0.1, + "n": 1200, + "p50": 0.158, + "p95": 0.715, + "p99": 1.675 + }, + "seconds": 590.2, + "slowlog_new": 3, + "steady": true, + "to_ns": 1791691343765594831, + "top_commands_usec": { + "eval": 11868395, + "evalsha": 32166251, + "publish": 896227, + "set": 1288739, + "setex": 3237803 + }, + "valid": true + }, + { + "core_fraction": 0.0741, + "from_ns": 1791691353768453016, + "judged": true, + "ping_ms": { + "max": 3.248, + "min": 0.1, + "n": 1199, + "p50": 0.152, + "p95": 0.467, + "p99": 1.039 + }, + "seconds": 589.7, + "slowlog_new": 1, + "steady": true, + "to_ns": 1791691943439104168, + "top_commands_usec": { + "eval": 10480850, + "evalsha": 26212845, + "publish": 839100, + "set": 1309821, + "setex": 3165549 + }, + "valid": true + }, + { + "core_fraction": 0.0627, + "from_ns": 1791691953441892544, + "judged": false, + "ping_ms": { + "max": 2.198, + "min": 0.1, + "n": 251, + "p50": 0.146, + "p95": 0.352, + "p99": 0.815 + }, + "seconds": 120.0, + "slowlog_new": 0, + "steady": false, + "to_ns": 1791692073476356936, + "top_commands_usec": { + "eval": 1933282, + "evalsha": 4387436, + "publish": 141285, + "set": 214685, + "setex": 530274 + }, + "valid": true + } + ] + }, + "quantity": "TS execution Redis (redis_marketdata) command CPU, SLOWLOG, RTT", + "reason": "SLOWLOG 6 new entries in steady state", + "rule": "commandstats <= 60 % of one core per 10-min window; SLOWLOG (10 ms) 0 in steady state; PING p99 <= 5 ms", + "verdict": "FAIL", + "sources": { + "g8": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g8/g8-summary.json", + "sha256": "2b3375cffc57c87e9e85ac21fe6b70f3897c2ebf2b80eea3f01b1180eb2d5671" + } + }, + "attribution": { + "basis": "the 6 SLOWLOG start seconds compared with applied TRADE records per writer-start second in the same trace", + "per_second_applied_TRADE": { + "median": 46, + "p95": 428, + "seconds": 1801 + }, + "slowlog_seconds": [ + { + "start_s": 1791690302, + "EVALSHA_us": 10016, + "TRADE_s-1..s+1": [ + 138, + 320, + 379 + ], + "percentile_of_peak": 93.2 + }, + { + "start_s": 1791690605, + "EVALSHA_us": 16120, + "TRADE_s-1..s+1": [ + 42, + 709, + 320 + ], + "percentile_of_peak": 98.1 + }, + { + "start_s": 1791690763, + "EVALSHA_us": 13258, + "TRADE_s-1..s+1": [ + 778, + 1114, + 1062 + ], + "percentile_of_peak": 99.4 + }, + { + "start_s": 1791690842, + "EVALSHA_us": 12964, + "TRADE_s-1..s+1": [ + 677, + 1068, + 625 + ], + "percentile_of_peak": 99.2 + }, + { + "start_s": 1791691251, + "EVALSHA_us": 18431, + "TRADE_s-1..s+1": [ + 296, + 691, + 284 + ], + "percentile_of_peak": 98.0 + }, + { + "start_s": 1791691386, + "EVALSHA_us": 19732, + "TRADE_s-1..s+1": [ + 174, + 372, + 974 + ], + "percentile_of_peak": 99.1 + } + ], + "reading": "every SLOWLOG entry is a writer APPLY EVALSHA in a TRADE burst second (peak within +-1 s at or above the 93rd percentile of per-second TRADE volume); commandstats <= 0.0868 core and PING p99 <= 1.675 ms stay inside. Owner decision as G2: EVAL chunking moves to TS phase E" + } }, { - "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json", - "sha256": "e9e823094e73782166d8275647fed3abbf6000c20bc18abd695b8bb1686bf72d" - } - ] - }, - { - "id": "G8", - "quantity": "TS execution Redis (redis_marketdata) command CPU, SLOWLOG, RTT", - "scope": "INFO commandstats usec delta, SLOWLOG count (10 ms), PING RTT", - "rule": "commandstats <= 60 % of one core per 10-min window; SLOWLOG 0 in steady state; PING RTT p99 <= 5 ms under full mixed traffic", - "verdict": "NOT_MEASURED", - "reason": "One read-only 600 s window on writer-only traffic, after the stage-100 run; not 'full mixed traffic'.", - "measured": { - "seconds": 600.011, - "core_fraction": 0.0651, - "slowlog_new_entries": 0, - "slowlog_new_by_command": {}, - "slowlog_new_max_duration_us": null, - "slowlog_threshold_us": 10000, - "ping_rtt_ms": { - "errors": 0, - "max": 5.279, - "n": 5971, - "p50": 0.318, - "p95": 0.528, - "p99": 0.942 + "id": "G9", + "measured": { + "demanded_ne_session_ready_samples": 0, + "demanded_values": [ + 64 + ], + "eligibility_loss_causes": {}, + "eligibility_loss_rows": [], + "eligibility_losses_data_path": 0, + "execution_ready_max": 53, + "execution_ready_min": 50, + "legit_quiet_refusals": { + "TRADE:TRADE_EVENT_STALE": 24 + }, + "purpose_errors": 0, + "purpose_fields_present": true, + "readable_samples": 180, + "samples": 180, + "session_loss_causes": {}, + "unknown_samples": 0, + "v1_fallback_delta": 0, + "v1_fallback_max": 0 + }, + "quantity": "readiness: demanded, session-ready, intent-eligible kept separate", + "reason": "inside", + "rule": "demanded == session-ready; 0 data-path eligibility losses (legitimate quiet TRADE refusals counted, excluded); v1 fallback 0", + "verdict": "PASS", + "sources": { + "g9g10": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g9g10/g9-g10-summary.json", + "sha256": "3015c2db3820c172a984af38043443622e97065f59fbdd8054dbfff13abf117d" + }, + "gates": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/stage100/gates.json", + "sha256": "63f12e8c6df961b0505fd75ab8df9fe2a34e4fa7b2d985cb025e17a6785bcc43" + } + } }, - "monitor_clients": 0, - "top_commands": { - "cmdstat_eval": { - "calls": 76968, - "usec": 9700807 - }, - "cmdstat_evalsha": { - "calls": 87009, - "usec": 23497977 - }, - "cmdstat_get": { - "calls": 263762, - "usec": 644174 - }, - "cmdstat_mget": { - "calls": 95034, - "usec": 558002 - }, - "cmdstat_publish": { - "calls": 891655, - "usec": 679478 - }, - "cmdstat_scan": { - "calls": 4470, - "usec": 196198 - }, - "cmdstat_set": { - "calls": 613529, - "usec": 1067763 - }, - "cmdstat_setex": { - "calls": 1021990, - "usec": 2681454 - } - } - }, - "window_utc": [ - 1791644816.3761318, - 1791645416.3872285 - ], - "would_measure": "the same read-only probe over every 10-min window of a stage-100 run", - "sources": [ { - "path": "~/.local/state/qdl-v2/releases/v2.2.3/g8-redis/g8-window-600s.json", - "sha256": "fbc269fd1303e8bac13c4894089fc43d2368850d16859b1bde03ed497e8c528b" - } - ] - }, - { - "id": "G9", - "quantity": "readiness", - "scope": "demanded / session-ready / intent-eligible, final manifest denominator", - "rule": "demanded == session-ready for the whole window; 0 data-path-caused losses of execution eligibility; v1_fallback 0", - "verdict": "NOT_MEASURED", - "reason": "The session clause is met (ts:ready_60_every_sample 154/154 samples, 0 unreadable, writer READY 64/64; v1_fallback_calls 0), but losses of execution eligibility by cause were not counted.", - "measured": { - "ts_ready": { - "expected": 154, - "gate": "ts:ready_60_every_sample", - "not_ready": 0, - "samples": 154, - "status": "PASS", - "unreadable": 0 + "basis": "writer log typed signals (DL-signalled gaps, CURSOR_EXPIRED, SlowConsumerError) + trace duplicate event_id; a silent offset gap the DL does not signal is not observable (see proposed counter)", + "id": "G10", + "measured": { + "codes": { + "resumed|RECONNECTED": 2 + }, + "duplicates_applied": 0, + "gaps": 0, + "log_polls": 129, + "log_unreadable": 0, + "overflow": 0, + "replay_expired": 0, + "resumed_reconnects": 2 + }, + "quantity": "TRADE integrity", + "reason": "all zero", + "rule": "0 gaps, 0 duplicates applied, 0 replay-expired, 0 overflow", + "verdict": "PASS", + "sources": { + "g9g10": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g9g10/g9-g10-summary.json", + "sha256": "3015c2db3820c172a984af38043443622e97065f59fbdd8054dbfff13abf117d" + }, + "transit": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/transit-load.json", + "sha256": "d1376db1a50db991df5e9e9e80212463ad095640ee00a67ab673e473145d73f6" + } + } }, - "v1_fallback_calls": 0, - "mark_watcher": "0 writer MARK errors in 32 of 32 minutes (mark-watch-s100m.log)" - }, - "would_measure": "per-slice intent-eligibility samples with refusal cause (quiet TRADE/LAST counted separately) over the stage-100 window", - "sources": [ { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/gates.json", - "sha256": "c65db308bdfbf453d3ea1027b0dec73aa00b66c1d043ca72b3ec2e1aad8bb079" + "id": "G11", + "measured": { + "cursor_samples": [ + { + "ahead": [], + "at_ms": 1791692127022, + "compared": 21, + "unmatched": 0 + }, + { + "ahead": [], + "at_ms": 1791692147026, + "compared": 21, + "unmatched": 0 + }, + { + "ahead": [], + "at_ms": 1791692167033, + "compared": 21, + "unmatched": 0 + } + ], + "outage_seconds": 0.121, + "recovery": { + "caught_up_s": { + "max": 18.923, + "min": 3.44, + "n": 64, + "p50": 3.919, + "p95": 4.437, + "p99": null + }, + "first_valid_apply_s": { + "max": 18.923, + "min": 3.44, + "n": 64, + "p50": 3.919, + "p95": 4.437, + "p99": null + }, + "per_slice": { + "BINANCE|BNBUSDT|BAR|1m": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BNBUSDT|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BNBUSDT|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BNBUSDT|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BNBUSDT|QUOTE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "BINANCE|BNBUSDT|TRADE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "BINANCE|BTCUSDT|BAR|1m": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "BINANCE|BTCUSDT|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BTCUSDT|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BTCUSDT|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BTCUSDT|QUOTE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BTCUSDT|TRADE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|DOGEUSDT|BAR|1m": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|DOGEUSDT|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|DOGEUSDT|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|DOGEUSDT|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|DOGEUSDT|QUOTE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|DOGEUSDT|TRADE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|ETHUSDT|BAR|1m": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "BINANCE|ETHUSDT|BOOK_DELTA|None": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "BINANCE|ETHUSDT|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|ETHUSDT|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "BINANCE|ETHUSDT|QUOTE|None": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "BINANCE|ETHUSDT|TRADE|None": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "BINANCE|SOLUSDT|BAR|1m": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|SOLUSDT|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|SOLUSDT|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|SOLUSDT|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|SOLUSDT|QUOTE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "BINANCE|SOLUSDT|TRADE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|BNB-USDT-SWAP|BAR|1m": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|BNB-USDT-SWAP|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BNB-USDT-SWAP|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BNB-USDT-SWAP|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BNB-USDT-SWAP|QUOTE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|BNB-USDT-SWAP|TRADE|None": { + "caught_up_s": 18.923, + "first_valid_apply_s": 18.923, + "rewinds": [] + }, + "OKX|BTC-USD-SWAP|BOOK_DELTA|None": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "OKX|BTC-USD-SWAP|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USD-SWAP|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USD-SWAP|QUOTE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USDT-SWAP|BAR|1m": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USDT-SWAP|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USDT-SWAP|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USDT-SWAP|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USDT-SWAP|QUOTE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USDT-SWAP|TRADE|None": { + "caught_up_s": 4.92, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|DOGE-USDT-SWAP|BAR|1m": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|DOGE-USDT-SWAP|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|DOGE-USDT-SWAP|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|DOGE-USDT-SWAP|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|DOGE-USDT-SWAP|QUOTE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|DOGE-USDT-SWAP|TRADE|None": { + "caught_up_s": 4.92, + "first_valid_apply_s": 4.92, + "rewinds": [] + }, + "OKX|ETH-USDT-SWAP|BAR|1m": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|ETH-USDT-SWAP|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|ETH-USDT-SWAP|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|ETH-USDT-SWAP|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|ETH-USDT-SWAP|QUOTE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|ETH-USDT-SWAP|TRADE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|SOL-USDT-SWAP|BAR|1m": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|SOL-USDT-SWAP|BOOK_DELTA|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|SOL-USDT-SWAP|BOOK_SNAPSHOT|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|SOL-USDT-SWAP|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|SOL-USDT-SWAP|QUOTE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|SOL-USDT-SWAP|TRADE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + } + }, + "rewinds_total": 0, + "slices_not_caught_up": [], + "slices_without_first_apply": [] + }, + "restart": { + "detected_ms": 1791692107015, + "fault_cleared_ms": 1791692106579, + "last_old_seen_ms": 1791692106016, + "new": { + "finished": "2026-10-11T04:15:06.458006215Z", + "id": "c9ded16d4ff019b16d3dbc577edfceaaea6fe2282baf0a65a1708b2c4c0b8cbd", + "restarts": 0, + "running": true, + "started": "2026-10-11T04:15:06.579836652Z" + }, + "outage_start_ms": 1791692106458 + } + }, + "note": "a fault window after the load (a restart of a monitored container aborts the driver); its verdict does not depend on the steady-state protocol", + "quantity": "writer restart recovery", + "reason": "restart probe", + "rule": "fault cleared -> first valid apply per slice <= 30 s, -> caught up <= 60 s; no rewind; cursor never ahead", + "verdict": "PASS", + "sources": { + "g11": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g11/g11-summary.json", + "sha256": "c59cccf3447dbe67b98c4f4842903601352234600de4b6d60b741e94cb98ed8f" + } + } }, { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/guard-s100m.log", - "sha256": "cef88fa86bda376c1991618b6e5734cbb69603bc7a8ecb9c9c0812c9dbb7a91b" + "id": "G12", + "quantity": "Kafka broker / projector / Stream restart recovery", + "rule": "fault cleared -> consumers caught up to live <= 600 s, then G1-G3 inside gate; the same four timestamps as G11", + "verdict": "NOT_MEASURED", + "reason": "isolated run on the release digests (python a0c3649d, rust 96a2c912, writer bae2509f): all three faults caught up <= 600 s and G1 inside in the 300 s window after each; G2/G3 after recovery are NOT_MEASURED (BOOK-only isolated source), and the rule requires G1-G3, so the gate is NOT_MEASURED, not PASS", + "level": "ISOLATED_FAULT", + "provenance": "TEST_ONLY", + "measured": [ + { + "fault": "broker_stop_start", + "caught_up_s": 22.334, + "per_venue_caught_up_s": { + "BINANCE": 22.334, + "OKX": 2.177 + }, + "g1_after": { + "BINANCE": { + "n": 297, + "p95_ms": 108.227, + "p99_ms": 160.586, + "max_ms": 1424.895 + }, + "OKX": { + "n": 298, + "p95_ms": 116.204, + "p99_ms": 170.137, + "max_ms": 1970.011 + } + } + }, + { + "fault": "projector_restart", + "caught_up_s": 0.935, + "per_venue_caught_up_s": { + "BINANCE": 0.934, + "OKX": 0.935 + }, + "g1_after": { + "BINANCE": { + "n": 297, + "p95_ms": 100.731, + "p99_ms": 113.233, + "max_ms": 127.018 + }, + "OKX": { + "n": 299, + "p95_ms": 101.251, + "p99_ms": 113.503, + "max_ms": 196.204 + } + } + }, + { + "fault": "stream_restart", + "caught_up_s": 2.294, + "per_venue_caught_up_s": { + "BINANCE": 2.293, + "OKX": 2.294 + }, + "g1_after": { + "BINANCE": { + "n": 298, + "p95_ms": 98.764, + "p99_ms": 103.427, + "max_ms": 111.131 + }, + "OKX": { + "n": 300, + "p95_ms": 98.576, + "p99_ms": 109.594, + "max_ms": 122.954 + } + } + } + ], + "status": "COMPLETED", + "pass": true, + "limits": [ + "isolated namespace, same host, TEST_ONLY BOOK source, single projector/Stream replica", + "G2 and G3 after recovery: NOT_MEASURED (BOOK-only isolated source) / NOT_MEASURED (BOOK-only isolated source)" + ], + "teardown": { + "removed_containers": 12, + "left_by_label": 0, + "left_by_name": 0, + "volumes_before": 217, + "volumes_after": 217 + }, + "attempts": [ + { + "dir": "~/.local/state/qdl-v2/p184-g12-rc10-20261011", + "status": "FAIL", + "error": "RuntimeError: STOP role exited: monitor (in-namespace)", + "detail": "the last monitor record precedes the first fault injection (broker_stop_start kafka-2) by 3.9 s; the STOP fired 27 ms after the injection; no traceback or OOM retained (same symptom as 10-10 attempt a3)", + "verdict_sha256": "728b460d8244a36260a428542e93fb6bfec44cd3cae107e684000a8464e1573e", + "teardown": "10 removed, 0 left, volumes 217 -> 217" + }, + { + "dir": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-r2", + "status": "STOPPED", + "error": "SystemExit: G12 orchestrator received signal 15 (duplicate run, stopped by the coordinator)", + "verdict_sha256": "603fe82c761b2633c326fee6b56ad0578710ac4599d0b27f95a9a059846a1ebf", + "teardown": "5 removed, 0 left by label/name; volumes 217 -> 219 at its teardown while the concurrent a2 run held its own volumes; a2's teardown later read 217" + } + ], + "open_harness_item": "keep the monitor container's exit code and OOM state in the evidence", + "sources": { + "verdict": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/evidence/verdict.json", + "sha256": "05c0652d320929a13073a2ff1eb57a319f5b4a5817f7dd4f917ca8ae914a48b8" + }, + "teardown": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/evidence/teardown.json", + "sha256": "f17de5ca9539848e581972fd276809d8b9e5fd3fb5972f848c795f60bd722bb8" + }, + "image_set": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/image-set.json", + "sha256": "7576125751ef20235bad4c139188fe8b31bd10ee3113efea7cd153f11282c87d" + } + }, + "observed": "caught up and G1 INSIDE for all three faults" }, { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/mark-watch-s100m.log", - "sha256": "f905c2810f7d74c7290865f18a3adb1dc4106c9181b5130cd5a3762a44c29083" - } - ] - }, - { - "id": "G10", - "quantity": "integrity", - "scope": "TRADE (LOSSLESS) at the TS writer", - "rule": "0 gaps, 0 duplicates applied, 0 replay-expired, 0 overflow", - "verdict": "NOT_MEASURED", - "reason": "No production gap/overflow/replay-expiry counters were read for the stage-100 window on the final digests.", - "related_observation": "DR08 isolated (QUALIFICATION, final DL digests): SLOW_CONSUMER_BOUNDED_QUEUE closes at 1000 queued events with RESOURCE_EXHAUSTED and replay; trace windows show DUPLICATE TRADE outcomes refused by the writer (not applied)", - "would_measure": "Stream overflow/replay counters and writer gap/duplicate counters before/after a stage-100 run", - "sources": [ - { - "path": "~/.local/state/qdl-v2/p184-g13-20261010T142914Z/dr08-isolated/work/evidence.jsonl", - "sha256": "a7bf8d013af4e011741f8013926993500140575d626916d148caa9b207ef4fc6" - } - ] - }, - { - "id": "G11", - "quantity": "recovery, TS writer restart", - "scope": "fault cleared -> first valid apply per slice; -> caught up to live", - "rule": "<= 30 s and <= 60 s; no rewind; cursor never ahead of applied state", - "verdict": "NOT_MEASURED", - "reason": "No receipt records the four timestamps for a writer restart on the final digests.", - "related_observation": "DR08 isolated CONSUMER_RESTART_RECOVERY_MEASURED (a DL stream consumer process, not the TS writer): restart -> first event 5067-5144 ms, n=5, ack_measured false", - "would_measure": "a writer restart with outage start, fault cleared, first valid apply and caught-up recorded per slice (the delivery trace started at writer start can supply first apply)", - "sources": [ - { - "path": "~/.local/state/qdl-v2/p184-g13-20261010T142914Z/dr08-isolated/work/evidence.jsonl", - "sha256": "a7bf8d013af4e011741f8013926993500140575d626916d148caa9b207ef4fc6" - } - ] - }, - { - "id": "G12", - "quantity": "recovery, Kafka broker / projector / Stream restart", - "scope": "fault cleared -> caught up to live, then G1-G3", - "rule": "<= 600 s (owner bound)", - "verdict": "NOT_MEASURED", - "reason": "The isolated PASS (75673bc/2a5e809; broker 0.202 s, projector 0.256 s, Stream 2.355 s caught up; BOOK only, G2/G3 NOT_MEASURED) ran core 658a9570 (0e00c81), projector a696cfba and Stream 1f13408f (c8015ca), Query 672503b7 (76950c8) and writer 15524c0b (9b86ed3). Since then the core (+959/-30 lines: IDR-062 top-change materialization, IDR-040 fix 2), Stream (+1020/-147: Astra R2 hot lookup, terminal errors across stalled handoffs), projector (+39: invalid-book fence) and Query changed, so the result is not inheritable by the final digests.", - "related_observation": "The rc7d rollout restarted Stream, projectors and cores one at a time with writer READY 64/64 after each step (rollout-rc7d.log); timestamps for caught-up were not recorded.", - "would_measure": "scripts/p184_g12_isolated_fault.py with its IMAGES set to python e0252f4c / rust 64468c72 / writer 34202753 (the image set is a module constant today)", - "sources": [ - { - "path": "~/.local/state/qdl-v2/p184-g12-20261010/evidence/verdict.json", - "sha256": "7e98572c80ed6746234c7a7d7f2fbc9cae5d0b617d60a07ab8fabf48d70a8057" + "id": "G13", + "quantity": "DR01-DR08 acceptance", + "rule": "every cell closed once on the final DL + writer digests", + "verdict": "DEFERRED", + "reason": "41/41 DL-side cells ok on the release digests; 4 TS-stack steps deferred to the TS release", + "measured": { + "dr05-upstream": "6/6 ok", + "dr08-isolated": "4/4 ok (QUALIFICATION)", + "dr05-transport": "9/9 ok", + "dr04-dr06-views": "22/22 ok (DR05.QUIET_TYPED BINANCE/OKX NOT_OBSERVED)" + }, + "digests": { + "dl_sha": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "python": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "rust": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "writer": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", + "ts_sha": "8b567eb07c9036dbbb42079ed1da90db7e0f5f25" + }, + "failed_attempt": "first dr04-dr06-views in 043821Z: DR04.EXPLICIT_LAST@BINANCE refused EXECUTION_CONTEXT_STALE (live LAST older than the probe limit in a quiet Sunday market), a correct typed refusal; re-run 045533Z 22/22 ok. The age sample behind this reading was not retained in a file", + "deferred_cells": [ + "dr01-deployed", + "dr01-wrong-account", + "dr02-dr03-alpha", + "dr07-reference" + ], + "approval": "owner AskUserQuestion answer 2026-10-11, relayed; p184-g13-inputs/runtime-approval-rc10.md was written by an agent from that relayed approval", + "sources": { + "run_1": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/summary.json", + "sha256": "3a3a34e29715a4b67406b2b5e790bed0c9b8d7f0551498d43d0cd9f9c610a09d" + }, + "run_2_views": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T045533Z/summary.json", + "sha256": "309199f1a24c8625ed98d13d572a79f2cbff754359c8f88ce9a21e9c175c3472" + }, + "dr04_dr06_rows": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T045533Z/dr04-dr06-views/dr04-dr06.jsonl", + "sha256": "2ac4622ce54cff91b4e59ee423067d211460aeb475e8901c6eab155130f9d222" + }, + "dr05_upstream_rows": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/dr05-upstream/dr05-upstream-rows.jsonl", + "sha256": "700721727d38dbd3d3acbe6ffeeead32e925104d35526a04f2b2ebeceba35ed7" + }, + "dr05_transport_rows": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/dr05-transport/dr05-transport.jsonl", + "sha256": "415538c65c0b6a1d0146264176b5e8ab0f8fece7967f2152ff8a5fd119f4b626" + }, + "dr08_evidence": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/dr08-isolated/work/evidence.jsonl", + "sha256": "e5b14dde2964f79e52607049fdbeff37461f6964818a1a92884a78cbce83d1d3" + }, + "approval": { + "path": "~/.local/state/qdl-v2/p184-g13-inputs/runtime-approval-rc10.md", + "sha256": "551890885673bb569119dc65c218deb0b561c59fec8071bea48a7699eb22d2ce" + } + } }, { - "path": "~/.local/state/qdl-v2/p184-g12-20261010/evidence/verdict.reanalyzed.json", - "sha256": "4bf70d72c10212dd2da407788691138cd2da048c2f9ec458f0c587e1366ce1b6" + "id": "G14", + "measured": { + "dl_cpu_mean_sum_vcpu": 5.37, + "mem_at_or_over_limit": [], + "missing_roles": [], + "roles": { + "binance_bar_edge": { + "container": "qdl_v2_stable_candidate-binance_bar_edge-1", + "cpu_max_pct": 76.32, + "cpu_mean_pct": 1.9, + "cpu_p95_pct": 29.25, + "mem_headroom_pct": 71.7, + "mem_limit_mib": 512.0, + "mem_max_mib": 145.0, + "samples": 151, + "throttled_pct_of_wall": 0.3 + }, + "ingestor_binance_usdm": { + "container": "qdl_v2_stable_candidate-ingestor_binance_usdm-1", + "cpu_max_pct": 22.47, + "cpu_mean_pct": 12.4, + "cpu_p95_pct": 17.85, + "mem_headroom_pct": 86.2, + "mem_limit_mib": 256.0, + "mem_max_mib": 35.3, + "samples": 151, + "throttled_pct_of_wall": 0.1 + }, + "ingestor_okx_swap": { + "container": "qdl_v2_stable_candidate-ingestor_okx_swap-1", + "cpu_max_pct": 18.38, + "cpu_mean_pct": 12.1, + "cpu_p95_pct": 15.81, + "mem_headroom_pct": 87.1, + "mem_limit_mib": 256.0, + "mem_max_mib": 32.9, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "kafka1": { + "container": "qdl_v2_stable_candidate-kafka1-1", + "cpu_max_pct": 99.17, + "cpu_mean_pct": 80.8, + "cpu_p95_pct": 96.04, + "mem_headroom_pct": 35.7, + "mem_limit_mib": 1536.0, + "mem_max_mib": 987.9, + "samples": 151, + "throttled_pct_of_wall": 0.4 + }, + "kafka2": { + "container": "qdl_v2_stable_candidate-kafka2-1", + "cpu_max_pct": 122.07, + "cpu_mean_pct": 98.3, + "cpu_p95_pct": 111.79, + "mem_headroom_pct": 54.5, + "mem_limit_mib": 2048.0, + "mem_max_mib": 931.5, + "samples": 151, + "throttled_pct_of_wall": 0.3 + }, + "kafka3": { + "container": "qdl_v2_stable_candidate-kafka3-1", + "cpu_max_pct": 131.35, + "cpu_mean_pct": 102.2, + "cpu_p95_pct": 118.61, + "mem_headroom_pct": 33.5, + "mem_limit_mib": 1536.0, + "mem_max_mib": 1022.0, + "samples": 151, + "throttled_pct_of_wall": 0.5 + }, + "market_cache": { + "container": "qdl_v2_stable_candidate-market_cache-1", + "cpu_max_pct": 12.98, + "cpu_mean_pct": 8.3, + "cpu_p95_pct": 11.87, + "mem_headroom_pct": 87.4, + "mem_limit_mib": 9216.0, + "mem_max_mib": 1163.3, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "market_data_service_p184w": { + "container": "market_data_service_p184w", + "cpu_max_pct": 270.39, + "cpu_mean_pct": 150.7, + "cpu_p95_pct": 212.82, + "mem_headroom_pct": 97.9, + "mem_limit_mib": 31375.4, + "mem_max_mib": 657.1, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "market_projector_1": { + "container": "qdl_v2_stable_candidate-market_projector_1-1", + "cpu_max_pct": 28.42, + "cpu_mean_pct": 18.8, + "cpu_p95_pct": 23.68, + "mem_headroom_pct": 88.4, + "mem_limit_mib": 512.0, + "mem_max_mib": 59.6, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "market_projector_2": { + "container": "qdl_v2_stable_candidate-market_projector_2-1", + "cpu_max_pct": 10.47, + "cpu_mean_pct": 8.3, + "cpu_p95_pct": 9.87, + "mem_headroom_pct": 80.7, + "mem_limit_mib": 512.0, + "mem_max_mib": 98.6, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "query_kn_1": { + "container": "qdl_v2_stable_candidate-query_kn_1-1", + "cpu_max_pct": 129.77, + "cpu_mean_pct": 57.1, + "cpu_p95_pct": 92.01, + "mem_headroom_pct": 14.3, + "mem_limit_mib": 1536.0, + "mem_max_mib": 1316.9, + "samples": 151, + "throttled_pct_of_wall": 0.7 + }, + "query_kn_2": { + "container": "qdl_v2_stable_candidate-query_kn_2-1", + "cpu_max_pct": 130.11, + "cpu_mean_pct": 66.1, + "cpu_p95_pct": 105.09, + "mem_headroom_pct": 22.0, + "mem_limit_mib": 1536.0, + "mem_max_mib": 1198.1, + "samples": 151, + "throttled_pct_of_wall": 1.2 + }, + "rust_core": { + "container": "qdl_v2_stable_candidate-rust_core-1", + "cpu_max_pct": 25.96, + "cpu_mean_pct": 10.9, + "cpu_p95_pct": 15.09, + "mem_headroom_pct": 79.1, + "mem_limit_mib": 256.0, + "mem_max_mib": 53.5, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "rust_core_2": { + "container": "qdl_v2_stable_candidate-rust_core_2-1", + "cpu_max_pct": 23.11, + "cpu_mean_pct": 13.0, + "cpu_p95_pct": 17.57, + "mem_headroom_pct": 76.4, + "mem_limit_mib": 256.0, + "mem_max_mib": 60.4, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "rust_core_3": { + "container": "qdl_v2_stable_candidate-rust_core_3-1", + "cpu_max_pct": 14.14, + "cpu_mean_pct": 8.5, + "cpu_p95_pct": 11.13, + "mem_headroom_pct": 83.9, + "mem_limit_mib": 256.0, + "mem_max_mib": 41.3, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "stable_redis": { + "container": "qdl_v2_stable_candidate-stable_redis-1", + "cpu_max_pct": 9.38, + "cpu_mean_pct": 3.3, + "cpu_p95_pct": 5.44, + "mem_headroom_pct": 93.5, + "mem_limit_mib": 160.0, + "mem_max_mib": 10.4, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "stream_kn_1": { + "container": "qdl_v2_stable_candidate-stream_kn_1-1", + "cpu_max_pct": 48.75, + "cpu_mean_pct": 20.8, + "cpu_p95_pct": 29.86, + "mem_headroom_pct": 66.5, + "mem_limit_mib": 512.0, + "mem_max_mib": 171.4, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "stream_kn_2": { + "container": "qdl_v2_stable_candidate-stream_kn_2-1", + "cpu_max_pct": 33.95, + "cpu_mean_pct": 14.1, + "cpu_p95_pct": 20.12, + "mem_headroom_pct": 78.8, + "mem_limit_mib": 512.0, + "mem_max_mib": 108.8, + "samples": 151, + "throttled_pct_of_wall": 0.0 + } + }, + "samples": 151 + }, + "quantity": "resources per role at stage 100", + "reason": "18 roles recorded", + "rule": "CPU/RAM per role recorded with headroom; budget set by the owner", + "verdict": "PASS", + "sources": { + "host": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/stage100/host.json", + "sha256": "06eab7bae9b385f8742630ea26cd91571e7e03f9910f7798f97f909b50f2ea1d" + } + }, + "note": "DL mean CPU sum 5.37 vCPU (17 DL roles, writer excluded); stage 100m on 2026-10-10 was about 5.0" } - ] - }, - { - "id": "G13", - "quantity": "acceptance", - "scope": "DR01-DR08 incl. DR05 typed gap/generation fence and inverse", - "rule": "every cell closed once on the final DL + writer digests", - "verdict": "DEFERRED", - "reason": "41/41 DL-side cells harness-ok on the final digests; dr01-deployed, dr01-wrong-account, dr02-dr03-alpha and dr07-reference are deferred to the TS release (gateway/alpha/risk not running) and stay OPEN.", - "measured": [ - { - "step": "dr05-upstream", - "cells": 6, - "ok": 6, - "outcome": "6 OBSERVED (ISOLATED_FAULT)", - "receipt": "p184-g13-20261010T134219Z" - }, - { - "step": "dr04-dr06-views", - "cells": 22, - "ok": 22, - "outcome": "14 OBSERVED RUNTIME, 6 OBSERVED ISOLATED_FAULT, 2 NOT_OBSERVED RUNTIME (DR05.QUIET_TYPED BINANCE/OKX: no quiet book in the 300 s window; ok by the cell rule, not a positive quiet proof)", - "receipt": "p184-g13-20261010T134219Z" - }, - { - "step": "dr05-transport", - "cells": 9, - "ok": 9, - "outcome": "9 OBSERVED (ISOLATED_FAULT); harness TS 7d7fd0e, writer image ed02064", - "receipt": "p184-g13-20261010T142735Z", - "superseded_attempt": "p184-g13-20261010T134219Z/dr05-transport harness exit 1 (probe read QUOTE only; fixed TS 7d7fd0e)" - }, - { - "step": "dr08-isolated", - "cells": 4, - "ok": 4, - "outcome": "4 OBSERVED (ISOLATED_FAULT, QUALIFICATION mode)", - "receipt": "p184-g13-20261010T142914Z" - } - ], - "deferred_cells": [ - "dr01-deployed", - "dr01-wrong-account", - "dr02-dr03-alpha", - "dr07-reference" - ], - "would_measure": "the TS release runs the four deferred steps with gateway/alpha/risk on their declared images", - "sources": [ - { - "path": "~/.local/state/qdl-v2/p184-g13-20261010T134219Z/summary.json", - "sha256": "b64f35e19ed5ad5fd283f11fa5d58af0e52ef7309440ff94a5e1df7deab63345" - }, - { - "path": "~/.local/state/qdl-v2/p184-g13-20261010T134219Z/dr05-upstream/dr05-upstream-rows.jsonl", - "sha256": "6dfe552ed9a116271561affce5e6836841bcc3a85cb9d1e23fda9325feca61c5" - }, - { - "path": "~/.local/state/qdl-v2/p184-g13-20261010T134219Z/dr04-dr06-views/dr04-dr06.jsonl", - "sha256": "11db0fc93c41ed4fc699d125781628ca81a0ad546e9b8026e872a93750e0d51f" - }, - { - "path": "~/.local/state/qdl-v2/p184-g13-20261010T142735Z/summary.json", - "sha256": "24d659c53f86e8cfdb9f6ea88fca7020a3d7c89e5d7771697e3560498537ccca" - }, - { - "path": "~/.local/state/qdl-v2/p184-g13-20261010T142735Z/dr05-transport/dr05-transport.jsonl", - "sha256": "42064f1393b6f7c6a615686295b7ffaba46a8a7f29b367e76a28e1cb7b69925b" - }, - { - "path": "~/.local/state/qdl-v2/p184-g13-20261010T142914Z/summary.json", - "sha256": "eb6ae4e299e7f3ee91593d2327b74daf653d1d2b8e43e496b49d74d573a9d85d" - }, - { - "path": "~/.local/state/qdl-v2/p184-g13-20261010T142914Z/dr08-isolated/work/evidence.jsonl", - "sha256": "a7bf8d013af4e011741f8013926993500140575d626916d148caa9b207ef4fc6" - } - ] - }, - { - "id": "G14", - "quantity": "resources", - "scope": "every DL role and the TS writer at stage 100", - "rule": "CPU/RAM recorded per role with headroom; budget set by the owner from the measurement", - "verdict": "NOT_MEASURED", - "reason": "16 roles recorded from 152 samples (memory max inside every limit; DL total about 5.0 vCPU mean); binance_bar_edge and stable_redis were not sampled, so 'every DL role' is incomplete. The owner budget decision is still open.", - "measured": [ - { - "role": "ingestor_binance_usdm", - "cpu_mean_pct": 10.6, - "cpu_p95_pct": 13.8, - "cpu_max_pct": 17.3, - "mem_max_mib": 29.9, - "mem_limit": "256MiB" - }, - { - "role": "ingestor_okx_swap", - "cpu_mean_pct": 11.5, - "cpu_p95_pct": 14.5, - "cpu_max_pct": 16.2, - "mem_max_mib": 28.4, - "mem_limit": "256MiB" - }, - { - "role": "kafka1", - "cpu_mean_pct": 77.0, - "cpu_p95_pct": 86.5, - "cpu_max_pct": 94.4, - "mem_max_mib": 968.0, - "mem_limit": "1.5GiB" - }, - { - "role": "kafka2", - "cpu_mean_pct": 95.4, - "cpu_p95_pct": 107.2, - "cpu_max_pct": 117.2, - "mem_max_mib": 921.1, - "mem_limit": "2GiB" - }, - { - "role": "kafka3", - "cpu_mean_pct": 98.2, - "cpu_p95_pct": 109.6, - "cpu_max_pct": 126.1, - "mem_max_mib": 1019.0, - "mem_limit": "1.5GiB" - }, - { - "role": "market_cache", - "cpu_mean_pct": 7.8, - "cpu_p95_pct": 10.7, - "cpu_max_pct": 12.7, - "mem_max_mib": 1162.2, - "mem_limit": "9GiB" - }, - { - "role": "market_data_service_p184w", - "cpu_mean_pct": 123.7, - "cpu_p95_pct": 149.2, - "cpu_max_pct": 198.2, - "mem_max_mib": 639.5, - "mem_limit": "30.64GiB" - }, - { - "role": "market_projector_1", - "cpu_mean_pct": 17.9, - "cpu_p95_pct": 20.0, - "cpu_max_pct": 22.5, - "mem_max_mib": 45.1, - "mem_limit": "512MiB" - }, - { - "role": "market_projector_2", - "cpu_mean_pct": 8.0, - "cpu_p95_pct": 8.8, - "cpu_max_pct": 9.6, - "mem_max_mib": 34.1, - "mem_limit": "512MiB" - }, - { - "role": "query_kn_1", - "cpu_mean_pct": 67.9, - "cpu_p95_pct": 100.6, - "cpu_max_pct": 124.6, - "mem_max_mib": 651.2, - "mem_limit": "1.5GiB" - }, - { - "role": "query_kn_2", - "cpu_mean_pct": 48.3, - "cpu_p95_pct": 77.7, - "cpu_max_pct": 109.8, - "mem_max_mib": 761.4, - "mem_limit": "1.5GiB" - }, - { - "role": "rust_core", - "cpu_mean_pct": 9.2, - "cpu_p95_pct": 10.8, - "cpu_max_pct": 12.9, - "mem_max_mib": 44.6, - "mem_limit": "256MiB" - }, - { - "role": "rust_core_2", - "cpu_mean_pct": 11.4, - "cpu_p95_pct": 13.1, - "cpu_max_pct": 15.6, - "mem_max_mib": 46.5, - "mem_limit": "256MiB" - }, - { - "role": "rust_core_3", - "cpu_mean_pct": 7.8, - "cpu_p95_pct": 9.0, - "cpu_max_pct": 10.2, - "mem_max_mib": 37.2, - "mem_limit": "256MiB" - }, - { - "role": "stream_kn_1", - "cpu_mean_pct": 21.5, - "cpu_p95_pct": 25.8, - "cpu_max_pct": 34.9, - "mem_max_mib": 158.0, - "mem_limit": "512MiB" - }, - { - "role": "stream_kn_2", - "cpu_mean_pct": 7.1, - "cpu_p95_pct": 7.8, - "cpu_max_pct": 8.2, - "mem_max_mib": 93.5, - "mem_limit": "512MiB" - } - ], - "would_measure": "the same sampler including binance_bar_edge and stable_redis over a stage-100 run", - "sources": [ - { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/g14-resources.txt", - "sha256": "6b1fdcab6e110e27832dd878f4bae92f042e261bda4fca6aeac7a433d9925792" + ], + "history": { + "2026-10-10_bbfb3be_scorecard": { + "superseded_by": "campaign 20261011T034233Z + isolated G12/G13 on rc10", + "sha256_of_committed_file": "d4035ccd460838cfcb130800660014f8dcfd22079f034390d44fa3fe9971c714", + "record": { + "schema": "qdl.release-scorecard.v1", + "release": "2.2.3", + "status": "PREPARED_NOT_PUBLISHED", + "source_commit": "bbfb3be6b3f1b7f83b44022e77439daeb4999a65", + "gate_table": { + "path": "trading-system upgrade/p184-phase-d-gates.md (FROZEN 2026-10-09)", + "ts_commit": "31cf663f5c7c5384ec3dd296061582dee2463c41", + "sha256": "c67bf69bb921c992964eb8ff8b2e1bb17085a943fdf756d55526b6269641d5c4" + }, + "frozen_budget": { + "path": "config/v2/p184-stage100-budget.json", + "sha256": "ff85335b3c8762d9acd8e7230f3c81ee1ec8403aff34cf2fb8f67714521c2541" + }, + "scoring_rule": "steady state >= 30 min at the stage-100 workload plus one burst window (gate table 'Measurement rules'); a measurement outside this rule can FAIL a gate but cannot PASS it. Verdicts: PASS, FAIL, NOT_MEASURED, DEFERRED; a NOT_MEASURED is never upgraded. 'observed' records the value against the threshold when the measurement does not meet the protocol.", + "runtime_identity": { + "dl_python": "sha256:e0252f4c96da70791b0c0fd031538bff2dd7249cdf6e0fbd4399c90e2e527a2a", + "dl_rust": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "dl_application_roles_on_release_digests": 12, + "ts_writer": "sha256:34202753bbe8b74d50e45afed1cf2e7b400141324c550168462b30db220edb31", + "ts_writer_source": "ed020648286336170886043697cb229a3c5290b7", + "ts_writer_tuned_env": "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE=BINANCE=100" + }, + "summary": { + "NOT_MEASURED": [ + "G1", + "G3", + "G4", + "G6", + "G7", + "G8", + "G9", + "G10", + "G11", + "G12", + "G14" + ], + "FAIL": [ + "G2" + ], + "PASS": [ + "G5" + ], + "DEFERRED": [ + "G13" + ] + }, + "gates": [ + { + "id": "G1", + "quantity": "end-to-end to TS cache (venue event -> TS Redis APPLY reply)", + "scope": "BOOK_DELTA, BOOK_SNAPSHOT", + "rule": "p95 <= 500 ms, p99 <= 1000 ms (n >= 100), max < 2000 ms per feed/venue", + "verdict": "NOT_MEASURED", + "observed": "INSIDE in both trace windows", + "reason": "Values come from writer-only trace windows, not the frozen stage-100 protocol.", + "measured": [ + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "BOOK_DELTA|BINANCE", + "n": 24553, + "p50_ms": 100.0, + "p95_ms": 126.5, + "p99_ms": 167.1, + "max_ms": 527.6, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 68.4, + "sdk_to_reply_ms": 36.7, + "venue_to_ingest_ms": 29.3 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "BOOK_DELTA|OKX", + "n": 26481, + "p50_ms": 104.3, + "p95_ms": 128.4, + "p99_ms": 173.4, + "max_ms": 528.1, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 67.6, + "sdk_to_reply_ms": 37.9, + "venue_to_ingest_ms": 33.0 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "BOOK_SNAPSHOT|BINANCE", + "n": 2647, + "p50_ms": 108.8, + "p95_ms": 154.4, + "p99_ms": 220.4, + "max_ms": 407.2, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 70.8, + "sdk_to_reply_ms": 58.3, + "venue_to_ingest_ms": 29.3 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "BOOK_SNAPSHOT|OKX", + "n": 3223, + "p50_ms": 113.6, + "p95_ms": 156.8, + "p99_ms": 206.3, + "max_ms": 336.1, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 69.2, + "sdk_to_reply_ms": 59.9, + "venue_to_ingest_ms": 33.1 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "BOOK_DELTA|BINANCE", + "n": 24664, + "p50_ms": 99.9, + "p95_ms": 126.5, + "p99_ms": 163.0, + "max_ms": 510.9, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 68.7, + "sdk_to_reply_ms": 36.4, + "venue_to_ingest_ms": 29.4 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "BOOK_DELTA|OKX", + "n": 26105, + "p50_ms": 105.1, + "p95_ms": 130.0, + "p99_ms": 170.5, + "max_ms": 501.8, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 67.8, + "sdk_to_reply_ms": 37.7, + "venue_to_ingest_ms": 37.9 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "BOOK_SNAPSHOT|BINANCE", + "n": 2647, + "p50_ms": 109.5, + "p95_ms": 152.4, + "p99_ms": 192.1, + "max_ms": 294.0, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 71.5, + "sdk_to_reply_ms": 53.5, + "venue_to_ingest_ms": 28.6 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "BOOK_SNAPSHOT|OKX", + "n": 3182, + "p50_ms": 114.3, + "p95_ms": 155.5, + "p99_ms": 188.9, + "max_ms": 353.1, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 68.5, + "sdk_to_reply_ms": 54.6, + "venue_to_ingest_ms": 37.4 + } + } + ], + "measurement_scope": "TS writer delivery trace, two 535 s analysis windows (13:19-13:28Z and 13:31-13:40Z, 40 s writer start-up skipped), writer-only production traffic AFTER stage 100m ended (13:17Z): no stage-100 alpha load, no burst window", + "would_measure": "the same trace analysis (scripts/p184_trace_feed_transit.py) over >= 30 min of trace activations taken DURING a stage-100 run, including its burst window (one activation is capped at 600 s and 16 x 4096 records per worker)", + "sources": [ + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791638906937534999.json", + "sha256": "d8d0d6c37dc37ccd1c22b3d9690d42041d09cc532ddd0008499d5a3d7b32526a" + }, + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json", + "sha256": "e9e823094e73782166d8275647fed3abbf6000c20bc18abd695b8bb1686bf72d" + } + ] + }, + { + "id": "G2", + "quantity": "end-to-end to TS cache, new-event transit", + "scope": "TRADE / QUOTE / MARK_INDEX_PRICE", + "rule": "TRADE p95 <= 500 ms and max < 1500 ms; QUOTE and MARK p95 <= 500 ms and max < 1000 ms (quiet-state age reported separately, not gated)", + "verdict": "FAIL", + "reason": "TRADE BINANCE p95 539.1 ms > 500 ms (max 1078.7 ms, n 21983) in trace window 1 on the final DL images and final writer image; window 2 (deployed writer config) is inside at p95 269.7 ms. The excess sits in the ingest -> SDK return segment (split p95 434.5 ms), not in the writer (sdk -> reply p95 85.0 ms) or the venue -> ingest leg (36.9 ms). Window 1 differs from the deployed config only by the Binance MARK wake read delay (250 vs 100 ms), which does not act on TRADE. Corroboration under stage-100 load: alpha SDK stream TRADE BINANCE source -> usable per-stream p95 up to 872.5 ms, max 1402.3 ms (stage100m receipt stream_latency). MARK and QUOTE are inside in the deployed-config window but are not PASS by protocol.", + "measured": [ + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "QUOTE|BINANCE", + "n": 25600, + "p50_ms": 116.2, + "p95_ms": 152.3, + "p99_ms": 179.0, + "max_ms": 434.1, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 98.5, + "sdk_to_reply_ms": 32.0, + "venue_to_ingest_ms": 30.8 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "QUOTE|OKX", + "n": 23103, + "p50_ms": 117.5, + "p95_ms": 153.9, + "p99_ms": 188.5, + "max_ms": 421.4, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 98.0, + "sdk_to_reply_ms": 32.6, + "venue_to_ingest_ms": 31.6 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "TRADE|BINANCE", + "n": 21983, + "p50_ms": 118.8, + "p95_ms": 539.1, + "p99_ms": 989.7, + "max_ms": 1078.7, + "observed": "OUTSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 434.5, + "sdk_to_reply_ms": 85.0, + "venue_to_ingest_ms": 36.9 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "TRADE|OKX", + "n": 5035, + "p50_ms": 107.7, + "p95_ms": 246.5, + "p99_ms": 558.3, + "max_ms": 731.1, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 107.5, + "sdk_to_reply_ms": 97.4, + "venue_to_ingest_ms": 37.9 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "QUOTE|BINANCE", + "n": 24134, + "p50_ms": 117.2, + "p95_ms": 152.2, + "p99_ms": 176.2, + "max_ms": 516.9, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 98.8, + "sdk_to_reply_ms": 32.1, + "venue_to_ingest_ms": 32.0 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "QUOTE|OKX", + "n": 22916, + "p50_ms": 118.3, + "p95_ms": 154.0, + "p99_ms": 181.7, + "max_ms": 513.4, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 98.2, + "sdk_to_reply_ms": 32.6, + "venue_to_ingest_ms": 31.9 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "TRADE|BINANCE", + "n": 14928, + "p50_ms": 116.8, + "p95_ms": 269.7, + "p99_ms": 545.5, + "max_ms": 623.9, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 191.3, + "sdk_to_reply_ms": 66.1, + "venue_to_ingest_ms": 36.0 + } + }, + { + "window": "window2_deployed_config", + "feed_venue": "TRADE|OKX", + "n": 4845, + "p50_ms": 106.6, + "p95_ms": 158.9, + "p99_ms": 216.5, + "max_ms": 394.0, + "observed": "INSIDE", + "split_p95_ms": { + "ingest_to_sdk_ms": 80.3, + "sdk_to_reply_ms": 60.8, + "venue_to_ingest_ms": 35.3 + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "MARK_INDEX_PRICE|BINANCE", + "n": 2667, + "p50_ms": 429.5, + "p95_ms": 500.7, + "p99_ms": 540.2, + "max_ms": 776.2, + "observed": "OUTSIDE" + }, + { + "window": "window1_mark_delay_binance_250", + "feed_venue": "MARK_INDEX_PRICE|OKX", + "n": 6385, + "p50_ms": 214.9, + "p95_ms": 320.9, + "p99_ms": 348.6, + "max_ms": 398.6, + "observed": "INSIDE" + }, + { + "window": "window2_deployed_config", + "feed_venue": "MARK_INDEX_PRICE|BINANCE", + "n": 2671, + "p50_ms": 278.5, + "p95_ms": 339.3, + "p99_ms": 379.5, + "max_ms": 708.0, + "observed": "INSIDE" + }, + { + "window": "window2_deployed_config", + "feed_venue": "MARK_INDEX_PRICE|OKX", + "n": 6390, + "p50_ms": 212.3, + "p95_ms": 317.9, + "p99_ms": 347.8, + "max_ms": 495.2, + "observed": "INSIDE" + } + ], + "measurement_scope": "TS writer delivery trace, two 535 s analysis windows (13:19-13:28Z and 13:31-13:40Z, 40 s writer start-up skipped), writer-only production traffic AFTER stage 100m ended (13:17Z): no stage-100 alpha load, no burst window", + "would_measure": "attribute the TRADE ingest -> SDK return tail (Stream dispatch/queue vs SDK client receive) and re-measure >= 30 min of trace during a stage-100 run", + "sources": [ + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791638906937534999.json", + "sha256": "d8d0d6c37dc37ccd1c22b3d9690d42041d09cc532ddd0008499d5a3d7b32526a" + }, + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json", + "sha256": "e9e823094e73782166d8275647fed3abbf6000c20bc18abd695b8bb1686bf72d" + }, + { + "path": "~/.local/state/qdl-v2/p184-g2-20261010/g2-mark-1791638906937534999.json", + "sha256": "292256e814110d6805efab9660667939699a228d99c1d1283cb5ce61e35f476c" + }, + { + "path": "~/.local/state/qdl-v2/p184-g2-20261010/g2-mark-1791639603470577388.json", + "sha256": "3da349c4f54c5125c23b8a6eb3c6e8f080c1a2023dd8b300ffa1732f2b09ea68" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/receipt.json", + "sha256": "9dfce5d983f25cdeb831eee6dbcd151864fa1cdbf567c9dd871f5866f67684a8" + } + ] + }, + { + "id": "G3", + "quantity": "end-to-end to TS cache, BAR from close", + "scope": "BAR", + "rule": "p95 <= 5 s, max < 60 s", + "verdict": "NOT_MEASURED", + "reason": "The writer delivery trace does not record BAR (0 BAR records in both windows); no other receipt measures BAR close -> TS Redis on the final digests.", + "related_observation": "stage100m alpha SDK streams (not the TS cache): BAR 1m close -> usable BINANCE per-stream p95 1135-3095 ms, max 3095 ms; OKX p95 1248-1307 ms, max 1511 ms", + "would_measure": "a read-only TS Redis notification probe (the 2026-10-09 transit_probe.py method, BAR from close) or BAR tracing in the writer, during a stage-100 run", + "sources": [ + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791638906937534999.json", + "sha256": "d8d0d6c37dc37ccd1c22b3d9690d42041d09cc532ddd0008499d5a3d7b32526a" + }, + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json", + "sha256": "e9e823094e73782166d8275647fed3abbf6000c20bc18abd695b8bb1686bf72d" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/receipt.json", + "sha256": "9dfce5d983f25cdeb831eee6dbcd151864fa1cdbf567c9dd871f5866f67684a8" + } + ] + }, + { + "id": "G4", + "quantity": "delivery lag, venue -> canonical (read_committed visibility, upper bound)", + "scope": "every feed, venue and interval", + "rule": "candidate p95 <= frozen baseline p95 x 1.2 per row (owner amendment 2026-10-09); missing coverage blocks the gate", + "verdict": "NOT_MEASURED", + "reason": "16/17 frozen rows PASS in all four stage-100m windows; bar|BINANCE|30m had no close in any window; five observed rows (BAR BINANCE 5m/1h, OKX 5m/15m/1h) have no frozen baseline; OKX inverse is pooled into OKX rows, not its own cells. Missing coverage blocks the gate by its own rule.", + "measured": [ + { + "row": "bar|BINANCE|15m", + "p95_ms_per_window": [ + null, + null, + 12533.0, + null + ], + "n_per_window": [ + 0, + 0, + 3, + 0 + ], + "limit_p95_ms": 40728.7, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|BINANCE|1h", + "p95_ms_per_window": [ + null, + null, + 14999.3, + null + ], + "n_per_window": [ + 0, + 0, + 3, + 0 + ], + "limit_p95_ms": null, + "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|BINANCE|1m", + "p95_ms_per_window": [ + 116.4, + 108.3, + 117.7, + 140.8 + ], + "n_per_window": [ + 10, + 10, + 10, + 10 + ], + "limit_p95_ms": 257.6, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|BINANCE|30m", + "p95_ms_per_window": [ + null, + null, + null, + null + ], + "n_per_window": [ + 0, + 0, + 0, + 0 + ], + "limit_p95_ms": 25836.6, + "verdict": "NOT_MEASURED_NOT_OBSERVED", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|BINANCE|3m", + "p95_ms_per_window": [ + 7992.0, + 8016.0, + null, + 8001.2 + ], + "n_per_window": [ + 5, + 5, + 0, + 5 + ], + "limit_p95_ms": 27273.2, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|BINANCE|5m", + "p95_ms_per_window": [ + null, + 8169.9, + 16186.8, + null + ], + "n_per_window": [ + 0, + 5, + 4, + 0 + ], + "limit_p95_ms": null, + "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|OKX|15m", + "p95_ms_per_window": [ + null, + null, + 901184.4, + null + ], + "n_per_window": [ + 0, + 0, + 1, + 0 + ], + "limit_p95_ms": null, + "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|OKX|1h", + "p95_ms_per_window": [ + null, + null, + 3600671.8, + null + ], + "n_per_window": [ + 0, + 0, + 1, + 0 + ], + "limit_p95_ms": null, + "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|OKX|1m", + "p95_ms_per_window": [ + 61304.5, + 60741.8, + 61188.9, + 61107.0 + ], + "n_per_window": [ + 7, + 1, + 6, + 8 + ], + "limit_p95_ms": 73817.2, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|OKX|30m", + "p95_ms_per_window": [ + null, + null, + 1800470.9, + null + ], + "n_per_window": [ + 0, + 0, + 1, + 0 + ], + "limit_p95_ms": 2160788.8, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|OKX|3m", + "p95_ms_per_window": [ + 181311.9, + 181132.3, + 180480.3, + 181110.8 + ], + "n_per_window": [ + 3, + 4, + 1, + 2 + ], + "limit_p95_ms": 217582.1, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "bar|OKX|5m", + "p95_ms_per_window": [ + null, + 300978.9, + null, + null + ], + "n_per_window": [ + 0, + 5, + 0, + 0 + ], + "limit_p95_ms": null, + "verdict": "NOT_MEASURED_NO_FROZEN_BASELINE", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "book_delta|BINANCE|", + "p95_ms_per_window": [ + 97.2, + 98.0, + 100.0, + 101.9 + ], + "n_per_window": [ + 756, + 755, + 760, + 785 + ], + "limit_p95_ms": 263.3, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "book_delta|OKX|", + "p95_ms_per_window": [ + 103.1, + 100.3, + 103.3, + 114.8 + ], + "n_per_window": [ + 1061, + 1069, + 978, + 1047 + ], + "limit_p95_ms": 289.1, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "book_snapshot|BINANCE|", + "p95_ms_per_window": [ + 99.6, + 96.2, + 100.3, + 102.4 + ], + "n_per_window": [ + 867, + 752, + 859, + 928 + ], + "limit_p95_ms": 260.0, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "book_snapshot|OKX|", + "p95_ms_per_window": [ + 101.5, + 97.8, + 102.6, + 112.1 + ], + "n_per_window": [ + 1618, + 1688, + 1086, + 1538 + ], + "limit_p95_ms": 270.1, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "mark_index_price|BINANCE|", + "p95_ms_per_window": [ + 185.8, + 190.2, + 195.2, + 193.2 + ], + "n_per_window": [ + 377, + 376, + 384, + 379 + ], + "limit_p95_ms": 370.9, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "mark_index_price|OKX|", + "p95_ms_per_window": [ + 1447.1, + 1361.9, + 1453.1, + 1252.2 + ], + "n_per_window": [ + 665, + 664, + 665, + 662 + ], + "limit_p95_ms": 1586.6, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "quote|BINANCE|", + "p95_ms_per_window": [ + 126.5, + 129.1, + 130.0, + 134.5 + ], + "n_per_window": [ + 546, + 555, + 548, + 560 + ], + "limit_p95_ms": 292.7, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "quote|OKX|", + "p95_ms_per_window": [ + 129.8, + 129.3, + 132.4, + 141.6 + ], + "n_per_window": [ + 591, + 584, + 566, + 598 + ], + "limit_p95_ms": 283.3, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "trade|BINANCE|", + "p95_ms_per_window": [ + 100.5, + 100.4, + 104.7, + 105.7 + ], + "n_per_window": [ + 344, + 312, + 340, + 397 + ], + "limit_p95_ms": 269.3, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + }, + { + "row": "trade|OKX|", + "p95_ms_per_window": [ + 104.1, + 104.6, + 105.0, + 110.0 + ], + "n_per_window": [ + 250, + 251, + 273, + 305 + ], + "limit_p95_ms": 257.4, + "verdict": "PASS", + "negative_samples": 0, + "missing_clock": 0 + } + ], + "windows": "4 x 120 s, 12:47-13:07Z, inside stage 100m (12:45-13:17Z), observer assigned read_committed, no join/commit", + "would_measure": "a G4 observer window long enough to see a BINANCE 30m close, frozen baselines for the five unbaselined rows, and inverse-separated rows", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/g4m-1-summary.json", + "sha256": "2c98891f2c9e25a09a106b665573b4ff4424edcafbcd22d01fc084affe2fe904" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/g4m-2-summary.json", + "sha256": "c6f321b95d05ca5aa27d353bae191449fad181d7a4bd2df0e2f9834830b2b287" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/g4m-3-summary.json", + "sha256": "b349e8c4f00ed31cbba341807597fbdab61e0b2a1cfa85dc718b528c8eabc5d4" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/g4m-4-summary.json", + "sha256": "6955061d7bf23c91e34fe885dd4a7917cce98d092b88ceffed9acbf8333a92fb" + }, + { + "path": "config/v2/p184-stage100-budget.json", + "sha256": "ff85335b3c8762d9acd8e7230f3c81ee1ec8403aff34cf2fb8f67714521c2541" + } + ] + }, + { + "id": "G5", + "quantity": "request latency", + "scope": "Query snapshot/reference classes + cold warmup shapes", + "rule": "certified stage-50 p95/p99 targets per class and venue; COLD_2500 <= 6265.1 ms, COLD_5000 <= 15267.0 ms (frozen budget)", + "verdict": "PASS", + "reason": "Every latency class and cold shape PASS in the 1800 s stage-100m run on the final digests (32/32 driver gates).", + "measured": [ + { + "gate": "latency:BAR_LATEST:BINANCE", + "count": 355, + "p50": 16.52, + "p95": 30.506, + "p99": 54.274, + "max": 291.955, + "target_p95": 1000, + "target_p99": 2000, + "status": "PASS" + }, + { + "gate": "latency:BAR_LATEST:OKX", + "count": 355, + "p50": 27.136, + "p95": 89.426, + "p99": 202.213, + "max": 340.742, + "target_p95": 1000, + "target_p99": 2000, + "status": "PASS" + }, + { + "gate": "latency:L2_SNAPSHOT:BINANCE", + "count": 355, + "p50": 40.123, + "p95": 80.944, + "p99": 229.67, + "max": 381.528, + "target_p95": 300, + "target_p99": 750, + "status": "PASS" + }, + { + "gate": "latency:L2_SNAPSHOT:OKX", + "count": 355, + "p50": 28.744, + "p95": 53.634, + "p99": 257.597, + "max": 445.148, + "target_p95": 300, + "target_p99": 750, + "status": "PASS" + }, + { + "gate": "latency:MARK_INDEX_REFERENCE:BINANCE", + "count": 53250, + "p50": 19.348, + "p95": 58.191, + "p99": 80.126, + "max": 536.599, + "target_p95": 250, + "target_p99": 500, + "status": "PASS" + }, + { + "gate": "latency:MARK_INDEX_REFERENCE:OKX", + "count": 53250, + "p50": 21.293, + "p95": 57.023, + "p99": 74.992, + "max": 455.591, + "target_p95": 250, + "target_p99": 500, + "status": "PASS" + }, + { + "gate": "latency:QUOTE_SNAPSHOT:BINANCE", + "count": 35500, + "p50": 13.349, + "p95": 37.49, + "p99": 74.924, + "max": 539.242, + "target_p95": 100, + "target_p99": 250, + "status": "PASS" + }, + { + "gate": "latency:QUOTE_SNAPSHOT:OKX", + "count": 35500, + "p50": 11.764, + "p95": 35.185, + "p99": 64.916, + "max": 464.515, + "target_p95": 100, + "target_p99": 250, + "status": "PASS" + }, + { + "gate": "latency:TRADE_SNAPSHOT:BINANCE", + "count": 354, + "p50": 10.332, + "p95": 34.75, + "p99": 57.465, + "max": 416.772, + "target_p95": 100, + "target_p99": 250, + "status": "PASS" + }, + { + "gate": "latency:TRADE_SNAPSHOT:OKX", + "count": 355, + "p50": 14.745, + "p95": 29.366, + "p99": 48.66, + "max": 237.098, + "target_p95": 100, + "target_p99": 250, + "status": "PASS" + }, + { + "gate": "g5:COLD_2500:BINANCE", + "count": 1, + "max_ms": 3124.007, + "limit_ms": 6265.1, + "status": "PASS" + }, + { + "gate": "g5:COLD_2500:OKX", + "count": 1, + "max_ms": 3726.708, + "limit_ms": 6265.1, + "status": "PASS" + }, + { + "gate": "g5:COLD_5000:BINANCE", + "count": 1, + "max_ms": 5298.549, + "limit_ms": 15267.0, + "status": "PASS" + }, + { + "gate": "g5:COLD_5000:OKX", + "count": 1, + "max_ms": 6177.615, + "limit_ms": 15267.0, + "status": "PASS" + } + ], + "limits": "cold shapes are n=1 per venue (max within limit); MARK_INDEX_REFERENCE is the measured reference-batch shape, not an arbitrary 50-item batch", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/gates.json", + "sha256": "c65db308bdfbf453d3ea1027b0dec73aa00b66c1d043ca72b3ec2e1aad8bb079" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/receipt.json", + "sha256": "9dfce5d983f25cdeb831eee6dbcd151864fa1cdbf567c9dd871f5866f67684a8" + }, + { + "path": "config/v2/p184-stage100-budget.json", + "sha256": "ff85335b3c8762d9acd8e7230f3c81ee1ec8403aff34cf2fb8f67714521c2541" + } + ] + }, + { + "id": "G6", + "quantity": "durable-view event age and typed execution quality at the endpoint", + "scope": "every requested feed/venue/interval", + "rule": "per-binding D3 thresholds (TRADE 3000, QUOTE/MARK/BOOK_DELTA 2000, BOOK_SNAPSHOT 60000, BAR1m 180000 ms) plus stricter TS use-time policy; zero unsafe eligibility", + "verdict": "NOT_MEASURED", + "reason": "The stage-100 receipt records no per-binding event age or eligibility of the returned views.", + "related_observation": "stage100m: 182709 offered requests, 0 failures (0 typed refusals of any code), 180 streams with 0 errors", + "would_measure": "driver receipt fields for per-binding view event age (SDK freshness) and execution eligibility of every returned view, including refusals", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/gates.json", + "sha256": "c65db308bdfbf453d3ea1027b0dec73aa00b66c1d043ca72b3ec2e1aad8bb079" + } + ] + }, + { + "id": "G7", + "quantity": "TS consumer path", + "scope": "SDK return -> writer entry; writer entry -> Redis APPLY reply", + "rule": "p95 <= 100 ms and p95 <= 50 ms per feed", + "verdict": "NOT_MEASURED", + "observed": "INSIDE for every feed in both trace windows", + "reason": "Writer-only trace windows, not the frozen stage-100 protocol.", + "measured": [ + { + "window": "window1_mark_delay_binance_250", + "feed": "BOOK_DELTA", + "sdk_to_writer_ms": { + "n": 51429, + "p50_ms": 22.3, + "p95_ms": 29.7, + "p99_ms": 58.6, + "max_ms": 372.2 + }, + "writer_to_redis_ms": { + "n": 51058, + "p50_ms": 2.1, + "p95_ms": 9.1, + "p99_ms": 16.2, + "max_ms": 192.3 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed": "BOOK_SNAPSHOT", + "sdk_to_writer_ms": { + "n": 5870, + "p50_ms": 28.8, + "p95_ms": 39.5, + "p99_ms": 92.9, + "max_ms": 190.7 + }, + "writer_to_redis_ms": { + "n": 5874, + "p50_ms": 4.0, + "p95_ms": 14.0, + "p99_ms": 40.5, + "max_ms": 192.2 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed": "MARK_INDEX_PRICE", + "sdk_to_writer_ms": { + "n": 9052, + "p50_ms": 0.4, + "p95_ms": 0.6, + "p99_ms": 0.6, + "max_ms": 1.2 + }, + "writer_to_redis_ms": { + "n": 9052, + "p50_ms": 2.1, + "p95_ms": 7.0, + "p99_ms": 13.3, + "max_ms": 94.5 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed": "QUOTE", + "sdk_to_writer_ms": { + "n": 48707, + "p50_ms": 22.1, + "p95_ms": 28.7, + "p99_ms": 39.6, + "max_ms": 175.3 + }, + "writer_to_redis_ms": { + "n": 54515, + "p50_ms": 1.1, + "p95_ms": 4.1, + "p99_ms": 10.5, + "max_ms": 140.1 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window1_mark_delay_binance_250", + "feed": "TRADE", + "sdk_to_writer_ms": { + "n": 27066, + "p50_ms": 22.9, + "p95_ms": 43.2, + "p99_ms": 95.7, + "max_ms": 207.2 + }, + "writer_to_redis_ms": { + "n": 27398, + "p50_ms": 5.1, + "p95_ms": 36.9, + "p99_ms": 98.9, + "max_ms": 120.2 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window2_deployed_config", + "feed": "BOOK_DELTA", + "sdk_to_writer_ms": { + "n": 51162, + "p50_ms": 22.3, + "p95_ms": 29.5, + "p99_ms": 48.6, + "max_ms": 218.7 + }, + "writer_to_redis_ms": { + "n": 50788, + "p50_ms": 2.1, + "p95_ms": 8.9, + "p99_ms": 16.0, + "max_ms": 150.1 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window2_deployed_config", + "feed": "BOOK_SNAPSHOT", + "sdk_to_writer_ms": { + "n": 5829, + "p50_ms": 28.8, + "p95_ms": 37.8, + "p99_ms": 90.8, + "max_ms": 151.2 + }, + "writer_to_redis_ms": { + "n": 5836, + "p50_ms": 4.0, + "p95_ms": 13.3, + "p99_ms": 32.4, + "max_ms": 85.2 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window2_deployed_config", + "feed": "MARK_INDEX_PRICE", + "sdk_to_writer_ms": { + "n": 9061, + "p50_ms": 0.4, + "p95_ms": 0.6, + "p99_ms": 0.6, + "max_ms": 1.2 + }, + "writer_to_redis_ms": { + "n": 9061, + "p50_ms": 2.2, + "p95_ms": 6.8, + "p99_ms": 13.3, + "max_ms": 85.9 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window2_deployed_config", + "feed": "QUOTE", + "sdk_to_writer_ms": { + "n": 47053, + "p50_ms": 22.1, + "p95_ms": 28.6, + "p99_ms": 38.1, + "max_ms": 206.9 + }, + "writer_to_redis_ms": { + "n": 52863, + "p50_ms": 1.1, + "p95_ms": 4.0, + "p99_ms": 10.4, + "max_ms": 100.5 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + }, + { + "window": "window2_deployed_config", + "feed": "TRADE", + "sdk_to_writer_ms": { + "n": 19820, + "p50_ms": 22.9, + "p95_ms": 41.5, + "p99_ms": 88.4, + "max_ms": 157.5 + }, + "writer_to_redis_ms": { + "n": 20131, + "p50_ms": 4.6, + "p95_ms": 26.2, + "p99_ms": 52.4, + "max_ms": 94.5 + }, + "observed": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + } + } + ], + "measurement_scope": "TS writer delivery trace, two 535 s analysis windows (13:19-13:28Z and 13:31-13:40Z, 40 s writer start-up skipped), writer-only production traffic AFTER stage 100m ended (13:17Z): no stage-100 alpha load, no burst window", + "would_measure": "the same analysis over >= 30 min of trace during a stage-100 run", + "sources": [ + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791638906937534999.json", + "sha256": "d8d0d6c37dc37ccd1c22b3d9690d42041d09cc532ddd0008499d5a3d7b32526a" + }, + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/trace-transit/feed-transit-1791639603470577388.json", + "sha256": "e9e823094e73782166d8275647fed3abbf6000c20bc18abd695b8bb1686bf72d" + } + ] + }, + { + "id": "G8", + "quantity": "TS execution Redis (redis_marketdata) command CPU, SLOWLOG, RTT", + "scope": "INFO commandstats usec delta, SLOWLOG count (10 ms), PING RTT", + "rule": "commandstats <= 60 % of one core per 10-min window; SLOWLOG 0 in steady state; PING RTT p99 <= 5 ms under full mixed traffic", + "verdict": "NOT_MEASURED", + "reason": "One read-only 600 s window on writer-only traffic, after the stage-100 run; not 'full mixed traffic'.", + "measured": { + "seconds": 600.011, + "core_fraction": 0.0651, + "slowlog_new_entries": 0, + "slowlog_new_by_command": {}, + "slowlog_new_max_duration_us": null, + "slowlog_threshold_us": 10000, + "ping_rtt_ms": { + "errors": 0, + "max": 5.279, + "n": 5971, + "p50": 0.318, + "p95": 0.528, + "p99": 0.942 + }, + "monitor_clients": 0, + "top_commands": { + "cmdstat_eval": { + "calls": 76968, + "usec": 9700807 + }, + "cmdstat_evalsha": { + "calls": 87009, + "usec": 23497977 + }, + "cmdstat_get": { + "calls": 263762, + "usec": 644174 + }, + "cmdstat_mget": { + "calls": 95034, + "usec": 558002 + }, + "cmdstat_publish": { + "calls": 891655, + "usec": 679478 + }, + "cmdstat_scan": { + "calls": 4470, + "usec": 196198 + }, + "cmdstat_set": { + "calls": 613529, + "usec": 1067763 + }, + "cmdstat_setex": { + "calls": 1021990, + "usec": 2681454 + } + } + }, + "window_utc": [ + 1791644816.3761318, + 1791645416.3872285 + ], + "would_measure": "the same read-only probe over every 10-min window of a stage-100 run", + "sources": [ + { + "path": "~/.local/state/qdl-v2/releases/v2.2.3/g8-redis/g8-window-600s.json", + "sha256": "fbc269fd1303e8bac13c4894089fc43d2368850d16859b1bde03ed497e8c528b" + } + ] + }, + { + "id": "G9", + "quantity": "readiness", + "scope": "demanded / session-ready / intent-eligible, final manifest denominator", + "rule": "demanded == session-ready for the whole window; 0 data-path-caused losses of execution eligibility; v1_fallback 0", + "verdict": "NOT_MEASURED", + "reason": "The session clause is met (ts:ready_60_every_sample 154/154 samples, 0 unreadable, writer READY 64/64; v1_fallback_calls 0), but losses of execution eligibility by cause were not counted.", + "measured": { + "ts_ready": { + "expected": 154, + "gate": "ts:ready_60_every_sample", + "not_ready": 0, + "samples": 154, + "status": "PASS", + "unreadable": 0 + }, + "v1_fallback_calls": 0, + "mark_watcher": "0 writer MARK errors in 32 of 32 minutes (mark-watch-s100m.log)" + }, + "would_measure": "per-slice intent-eligibility samples with refusal cause (quiet TRADE/LAST counted separately) over the stage-100 window", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/gates.json", + "sha256": "c65db308bdfbf453d3ea1027b0dec73aa00b66c1d043ca72b3ec2e1aad8bb079" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/guard-s100m.log", + "sha256": "cef88fa86bda376c1991618b6e5734cbb69603bc7a8ecb9c9c0812c9dbb7a91b" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/mark-watch-s100m.log", + "sha256": "f905c2810f7d74c7290865f18a3adb1dc4106c9181b5130cd5a3762a44c29083" + } + ] + }, + { + "id": "G10", + "quantity": "integrity", + "scope": "TRADE (LOSSLESS) at the TS writer", + "rule": "0 gaps, 0 duplicates applied, 0 replay-expired, 0 overflow", + "verdict": "NOT_MEASURED", + "reason": "No production gap/overflow/replay-expiry counters were read for the stage-100 window on the final digests.", + "related_observation": "DR08 isolated (QUALIFICATION, final DL digests): SLOW_CONSUMER_BOUNDED_QUEUE closes at 1000 queued events with RESOURCE_EXHAUSTED and replay; trace windows show DUPLICATE TRADE outcomes refused by the writer (not applied)", + "would_measure": "Stream overflow/replay counters and writer gap/duplicate counters before/after a stage-100 run", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T142914Z/dr08-isolated/work/evidence.jsonl", + "sha256": "a7bf8d013af4e011741f8013926993500140575d626916d148caa9b207ef4fc6" + } + ] + }, + { + "id": "G11", + "quantity": "recovery, TS writer restart", + "scope": "fault cleared -> first valid apply per slice; -> caught up to live", + "rule": "<= 30 s and <= 60 s; no rewind; cursor never ahead of applied state", + "verdict": "NOT_MEASURED", + "reason": "No receipt records the four timestamps for a writer restart on the final digests.", + "related_observation": "DR08 isolated CONSUMER_RESTART_RECOVERY_MEASURED (a DL stream consumer process, not the TS writer): restart -> first event 5067-5144 ms, n=5, ack_measured false", + "would_measure": "a writer restart with outage start, fault cleared, first valid apply and caught-up recorded per slice (the delivery trace started at writer start can supply first apply)", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T142914Z/dr08-isolated/work/evidence.jsonl", + "sha256": "a7bf8d013af4e011741f8013926993500140575d626916d148caa9b207ef4fc6" + } + ] + }, + { + "id": "G12", + "quantity": "recovery, Kafka broker / projector / Stream restart", + "scope": "fault cleared -> caught up to live, then G1-G3", + "rule": "<= 600 s (owner bound)", + "verdict": "NOT_MEASURED", + "reason": "The isolated PASS (75673bc/2a5e809; broker 0.202 s, projector 0.256 s, Stream 2.355 s caught up; BOOK only, G2/G3 NOT_MEASURED) ran core 658a9570 (0e00c81), projector a696cfba and Stream 1f13408f (c8015ca), Query 672503b7 (76950c8) and writer 15524c0b (9b86ed3). Since then the core (+959/-30 lines: IDR-062 top-change materialization, IDR-040 fix 2), Stream (+1020/-147: Astra R2 hot lookup, terminal errors across stalled handoffs), projector (+39: invalid-book fence) and Query changed, so the result is not inheritable by the final digests.", + "related_observation": "The rc7d rollout restarted Stream, projectors and cores one at a time with writer READY 64/64 after each step (rollout-rc7d.log); timestamps for caught-up were not recorded.", + "would_measure": "scripts/p184_g12_isolated_fault.py with its IMAGES set to python e0252f4c / rust 64468c72 / writer 34202753 (the image set is a module constant today)", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-g12-20261010/evidence/verdict.json", + "sha256": "7e98572c80ed6746234c7a7d7f2fbc9cae5d0b617d60a07ab8fabf48d70a8057" + }, + { + "path": "~/.local/state/qdl-v2/p184-g12-20261010/evidence/verdict.reanalyzed.json", + "sha256": "4bf70d72c10212dd2da407788691138cd2da048c2f9ec458f0c587e1366ce1b6" + } + ] + }, + { + "id": "G13", + "quantity": "acceptance", + "scope": "DR01-DR08 incl. DR05 typed gap/generation fence and inverse", + "rule": "every cell closed once on the final DL + writer digests", + "verdict": "DEFERRED", + "reason": "41/41 DL-side cells harness-ok on the final digests; dr01-deployed, dr01-wrong-account, dr02-dr03-alpha and dr07-reference are deferred to the TS release (gateway/alpha/risk not running) and stay OPEN.", + "measured": [ + { + "step": "dr05-upstream", + "cells": 6, + "ok": 6, + "outcome": "6 OBSERVED (ISOLATED_FAULT)", + "receipt": "p184-g13-20261010T134219Z" + }, + { + "step": "dr04-dr06-views", + "cells": 22, + "ok": 22, + "outcome": "14 OBSERVED RUNTIME, 6 OBSERVED ISOLATED_FAULT, 2 NOT_OBSERVED RUNTIME (DR05.QUIET_TYPED BINANCE/OKX: no quiet book in the 300 s window; ok by the cell rule, not a positive quiet proof)", + "receipt": "p184-g13-20261010T134219Z" + }, + { + "step": "dr05-transport", + "cells": 9, + "ok": 9, + "outcome": "9 OBSERVED (ISOLATED_FAULT); harness TS 7d7fd0e, writer image ed02064", + "receipt": "p184-g13-20261010T142735Z", + "superseded_attempt": "p184-g13-20261010T134219Z/dr05-transport harness exit 1 (probe read QUOTE only; fixed TS 7d7fd0e)" + }, + { + "step": "dr08-isolated", + "cells": 4, + "ok": 4, + "outcome": "4 OBSERVED (ISOLATED_FAULT, QUALIFICATION mode)", + "receipt": "p184-g13-20261010T142914Z" + } + ], + "deferred_cells": [ + "dr01-deployed", + "dr01-wrong-account", + "dr02-dr03-alpha", + "dr07-reference" + ], + "would_measure": "the TS release runs the four deferred steps with gateway/alpha/risk on their declared images", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T134219Z/summary.json", + "sha256": "b64f35e19ed5ad5fd283f11fa5d58af0e52ef7309440ff94a5e1df7deab63345" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T134219Z/dr05-upstream/dr05-upstream-rows.jsonl", + "sha256": "6dfe552ed9a116271561affce5e6836841bcc3a85cb9d1e23fda9325feca61c5" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T134219Z/dr04-dr06-views/dr04-dr06.jsonl", + "sha256": "11db0fc93c41ed4fc699d125781628ca81a0ad546e9b8026e872a93750e0d51f" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T142735Z/summary.json", + "sha256": "24d659c53f86e8cfdb9f6ea88fca7020a3d7c89e5d7771697e3560498537ccca" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T142735Z/dr05-transport/dr05-transport.jsonl", + "sha256": "42064f1393b6f7c6a615686295b7ffaba46a8a7f29b367e76a28e1cb7b69925b" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T142914Z/summary.json", + "sha256": "eb6ae4e299e7f3ee91593d2327b74daf653d1d2b8e43e496b49d74d573a9d85d" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261010T142914Z/dr08-isolated/work/evidence.jsonl", + "sha256": "a7bf8d013af4e011741f8013926993500140575d626916d148caa9b207ef4fc6" + } + ] + }, + { + "id": "G14", + "quantity": "resources", + "scope": "every DL role and the TS writer at stage 100", + "rule": "CPU/RAM recorded per role with headroom; budget set by the owner from the measurement", + "verdict": "NOT_MEASURED", + "reason": "16 roles recorded from 152 samples (memory max inside every limit; DL total about 5.0 vCPU mean); binance_bar_edge and stable_redis were not sampled, so 'every DL role' is incomplete. The owner budget decision is still open.", + "measured": [ + { + "role": "ingestor_binance_usdm", + "cpu_mean_pct": 10.6, + "cpu_p95_pct": 13.8, + "cpu_max_pct": 17.3, + "mem_max_mib": 29.9, + "mem_limit": "256MiB" + }, + { + "role": "ingestor_okx_swap", + "cpu_mean_pct": 11.5, + "cpu_p95_pct": 14.5, + "cpu_max_pct": 16.2, + "mem_max_mib": 28.4, + "mem_limit": "256MiB" + }, + { + "role": "kafka1", + "cpu_mean_pct": 77.0, + "cpu_p95_pct": 86.5, + "cpu_max_pct": 94.4, + "mem_max_mib": 968.0, + "mem_limit": "1.5GiB" + }, + { + "role": "kafka2", + "cpu_mean_pct": 95.4, + "cpu_p95_pct": 107.2, + "cpu_max_pct": 117.2, + "mem_max_mib": 921.1, + "mem_limit": "2GiB" + }, + { + "role": "kafka3", + "cpu_mean_pct": 98.2, + "cpu_p95_pct": 109.6, + "cpu_max_pct": 126.1, + "mem_max_mib": 1019.0, + "mem_limit": "1.5GiB" + }, + { + "role": "market_cache", + "cpu_mean_pct": 7.8, + "cpu_p95_pct": 10.7, + "cpu_max_pct": 12.7, + "mem_max_mib": 1162.2, + "mem_limit": "9GiB" + }, + { + "role": "market_data_service_p184w", + "cpu_mean_pct": 123.7, + "cpu_p95_pct": 149.2, + "cpu_max_pct": 198.2, + "mem_max_mib": 639.5, + "mem_limit": "30.64GiB" + }, + { + "role": "market_projector_1", + "cpu_mean_pct": 17.9, + "cpu_p95_pct": 20.0, + "cpu_max_pct": 22.5, + "mem_max_mib": 45.1, + "mem_limit": "512MiB" + }, + { + "role": "market_projector_2", + "cpu_mean_pct": 8.0, + "cpu_p95_pct": 8.8, + "cpu_max_pct": 9.6, + "mem_max_mib": 34.1, + "mem_limit": "512MiB" + }, + { + "role": "query_kn_1", + "cpu_mean_pct": 67.9, + "cpu_p95_pct": 100.6, + "cpu_max_pct": 124.6, + "mem_max_mib": 651.2, + "mem_limit": "1.5GiB" + }, + { + "role": "query_kn_2", + "cpu_mean_pct": 48.3, + "cpu_p95_pct": 77.7, + "cpu_max_pct": 109.8, + "mem_max_mib": 761.4, + "mem_limit": "1.5GiB" + }, + { + "role": "rust_core", + "cpu_mean_pct": 9.2, + "cpu_p95_pct": 10.8, + "cpu_max_pct": 12.9, + "mem_max_mib": 44.6, + "mem_limit": "256MiB" + }, + { + "role": "rust_core_2", + "cpu_mean_pct": 11.4, + "cpu_p95_pct": 13.1, + "cpu_max_pct": 15.6, + "mem_max_mib": 46.5, + "mem_limit": "256MiB" + }, + { + "role": "rust_core_3", + "cpu_mean_pct": 7.8, + "cpu_p95_pct": 9.0, + "cpu_max_pct": 10.2, + "mem_max_mib": 37.2, + "mem_limit": "256MiB" + }, + { + "role": "stream_kn_1", + "cpu_mean_pct": 21.5, + "cpu_p95_pct": 25.8, + "cpu_max_pct": 34.9, + "mem_max_mib": 158.0, + "mem_limit": "512MiB" + }, + { + "role": "stream_kn_2", + "cpu_mean_pct": 7.1, + "cpu_p95_pct": 7.8, + "cpu_max_pct": 8.2, + "mem_max_mib": 93.5, + "mem_limit": "512MiB" + } + ], + "would_measure": "the same sampler including binance_bar_edge and stable_redis over a stage-100 run", + "sources": [ + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/g14-resources.txt", + "sha256": "6b1fdcab6e110e27832dd878f4bae92f042e261bda4fca6aeac7a433d9925792" + }, + { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/host.json", + "sha256": "1483bd2c800bfec9d0c9faf96ec446fed87c13d659dc526e572ecfdc324e58fc" + } + ] + } + ] + } }, - { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/stage100m/host.json", - "sha256": "1483bd2c800bfec9d0c9faf96ec446fed87c13d659dc526e572ecfdc324e58fc" - } - ] + "earlier_final_campaigns": [ + { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261010T171129Z", + "scorecard_sha256": "94f30f85cbe10e4a300da9db68360db1c0945457b210143a5d21da1f9572d55d", + "protocol_met": true, + "protocol_problems": [], + "summary": { + "FAIL": [ + "G2", + "G6" + ], + "NOT_MEASURED": [ + "G4", + "G12", + "G13" + ], + "PASS": [ + "G1", + "G3", + "G5", + "G7", + "G8", + "G9", + "G10", + "G11", + "G14" + ] + }, + "writer_image": "sha256:ace423263d6aba97c8e45e657cdb7470e57b994dbfbe8571f2ffbb023a8dc940", + "note": "depth 1, Query rc8 + rc7d rest, writer sha256:ace42326 (5ac15db); Stream replica byte budget overflow incident 17:22:48Z (84 streams BACKPRESSURE) -> stream budget fix dadb9c8; G6 FAIL was a scorer defect -> ba496ff" + }, + { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261010T174944Z", + "scorecard_sha256": "1c6a5d304a88713e53ed5b962b11cd335ef2079f6015c610765b2f143acc1850", + "protocol_met": true, + "protocol_problems": [], + "summary": { + "FAIL": [ + "G2", + "G4", + "G6", + "G8" + ], + "NOT_MEASURED": [ + "G11", + "G12", + "G13" + ], + "PASS": [ + "G1", + "G3", + "G5", + "G7", + "G9", + "G10", + "G14" + ] + }, + "writer_image": "sha256:ace423263d6aba97c8e45e657cdb7470e57b994dbfbe8571f2ffbb023a8dc940", + "note": "depth 0, same images; G4 FAIL bar BINANCE 3m/30m at 18:00Z = admission INFLIGHT_CAPACITY sleep defect -> 413c5b9; G8 one EVALSHA 10.15 ms" + }, + { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261010T185053Z", + "scorecard_sha256": "ac0b3f66cfd12057842f0642aa5d9255a8807ae211bde50b80d31fae4f0cccb3", + "protocol_met": false, + "protocol_problems": [ + "driver acceptance not PASS: startup:bounded" + ], + "summary": { + "FAIL": [ + "G1", + "G2", + "G4" + ], + "NOT_MEASURED": [ + "G3", + "G5", + "G6", + "G7", + "G8", + "G9", + "G10", + "G12", + "G13", + "G14" + ], + "PASS": [ + "G11" + ] + }, + "writer_image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", + "note": "writer bae2509f depth 0; protocol not met (startup:bounded FAIL, session s89 exhausted the 10 frozen retries); md.latest.v2 compaction stall on kafka1 at 19:09Z behind the G1/G2/G4 excess; G6 funding anchor scorer bug -> 2f489b1" + }, + { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T033641Z", + "scorecard_sha256": "c3535323b4a3c41030f9ee2466c922c1b15ce061eb108023c0f2063d4db5a576", + "protocol_met": false, + "protocol_problems": [ + "no driver receipt", + "driver acceptance not PASS: no gates" + ], + "summary": { + "NOT_MEASURED": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "G8", + "G9", + "G10", + "G12", + "G13", + "G14" + ], + "PASS": [ + "G11" + ] + }, + "writer_image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", + "note": "aborted before the driver ran to apply the Kafka log-cleaner throttle (03:40:56Z); no gate measured except the G11 probe" + } + ] } - ] } From 1a161f41556eb86c43dce40bd9f17eee75f62dfc Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 11 Oct 2026 07:26:03 +0000 Subject: [PATCH 137/147] test(p184): G12 isolated harness measures G2 (TRADE/QUOTE/MARK) and G3 after recovery The isolated source was BOOK-only, so G12 could only be NOT_MEASURED. Adds TEST_ONLY TRADE, QUOTE, BAR 1m and MARK/INDEX lanes built from the production acquisition mapping, scores every part in the post-catch-up window, and keeps container exit/OOM events as evidence. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 12 + scripts/p184_dr05_case.py | 33 +- scripts/p184_g12_isolated_fault.py | 833 +++++++++++++++++++++- tests/test_p184_g12_isolated_fault.py | 576 +++++++++++++++ 4 files changed, 1429 insertions(+), 25 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 2506484a..bb2d4f41 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72195,3 +72195,15 @@ no build, CI or subagent job running. aquasec/trivy:0.58.1 (unit-r2, f6b6b5e7): every step PASS except #4 sudo. Remote CI not run. - Release evidence updated (upgrade/evidence/releases/v2.2.3/): status PREPARED_NOT_PUBLISHED; the 2026-10-10 record moved into the history. Publication needs owner push, tag and remote CI. +- G12 isolated harness extended so G12 can be scored in full (owner chose "close G12, then phase E", 2026-10-11): + - New TEST_ONLY deterministic sources, each on its own lane at generation 1: TRADE (3/s plus bursts of 8 every + 15 s), QUOTE (2/s), BAR 1m closed, MARK/INDEX (Binance markPrice@1s; OKX mark-price + index-tickers pair). + - Core bindings are built from the production acquisition mapping; a guard refuses any entry that differs from + the production plan. + - The clones demand the feeds through their normal routes; the isolated Query adds the production KN + execution/alpha MARK readers (product code still from the image). + - G2 (TRADE/QUOTE/MARK, frozen thresholds, n >= 100 per venue) and G3 (BAR from close, n >= 3) are measured in the + same 300 s post-catch-up window as G1. A fault passes only if every part passes; NOT_MEASURED on any part blocks + a PASS. + - Container die/oom events are kept as evidence (open item from attempt 1). + - Tests: 99 + 157 OK (unittest in rc10). Status: tested locally; the live run follows. diff --git a/scripts/p184_dr05_case.py b/scripts/p184_dr05_case.py index 5cb16278..c0f3a935 100644 --- a/scripts/p184_dr05_case.py +++ b/scripts/p184_dr05_case.py @@ -18,7 +18,18 @@ from scripts import p184_dr05_dl_fault as dr -def consumer_packet(manifest_path, catalog, venue, issuer, audience): +BOOK_FEEDS = ("BOOK_SNAPSHOT", "BOOK_DELTA") +REFERENCE_FEEDS = ("MARK_INDEX_PRICE",) + + +def consumer_packet(manifest_path, catalog, venue, issuer, audience, *, feeds=BOOK_FEEDS): + """One isolated TS consumer (ts-consumer JSON) and its sealed V3 route binding. + + ``feeds`` is the sealed feed set; the default is the DR05 BOOK pair (unchanged output). The G12 + harness (scripts/p184_g12_isolated_fault.py) also seals TRADE/QUOTE/BAR 1m (REALTIME plane, their one + source policy -> ``crypto_source_policy_id``) and MARK_INDEX_PRICE (REFERENCE plane, its source policy -> + ``execution_reference_source_policy_id``). + """ from qdl.consumer.manifest import ConsumerManifestLoader from qdl.consumer.realtime_route import requirement_key from qdl.consumer.universal_release import ( @@ -41,8 +52,8 @@ def consumer_packet(manifest_path, catalog, venue, issuer, audience): instrument = catalog.instrument_for(req.instrument_uid) identity = instrument.identity if ((identity.venue, identity.market, instrument.native_symbol) != (scope_venue, scope_market, scope_symbol) - or req.feed.value not in ("BOOK_SNAPSHOT", "BOOK_DELTA")): - raise ValueError("isolated scope BOOK requirements only") + or req.feed.value not in feeds or not set(BOOK_FEEDS) <= set(feeds)): + raise ValueError("isolated scope requirements of the sealed feeds only (BOOK pair included)") rid = hashlib.sha256(f"{cid}:{requirement_key(req)}".encode()).hexdigest() products.append(UniversalReleaseProduct( consumer_id=cid, consumer_class=UniversalConsumerClass.TRADING_SYSTEM, @@ -50,7 +61,8 @@ def consumer_packet(manifest_path, catalog, venue, issuer, audience): instrument_id=identity.instrument_id, venue=identity.venue, market=identity.market, product_type=identity.product_type.value, native_symbol=instrument.native_symbol, feed=req.feed.value, interval=req.interval, source_policy_id=req.source_policy_id, - provider_plane="L2", max_freshness_ms=req.max_freshness_ms, + provider_plane=("L2" if req.feed.value in BOOK_FEEDS else "REFERENCE" if req.feed.value in REFERENCE_FEEDS + else "REALTIME"), max_freshness_ms=req.max_freshness_ms, require_final_bars=req.require_final_bars, require_live=True, execution_grade=req.consumer_grade.value == "EXECUTION", route="V2_PRIMARY", fallback="BLOCKED", fallback_rule_id=None, blocked_reason="TEST_ONLY_NO_V1")) @@ -66,9 +78,15 @@ def consumer_packet(manifest_path, catalog, venue, issuer, audience): products=tuple(sorted(products, key=lambda p: p.requirement_id)), consumer_manifest_revision=manifest.manifest_revision, request_policies=tuple(sorted(policies, key=lambda p: p.requirement_id))) - sources = {p.source_policy_id for p in products} - if len(products) != 2 or len(sources) != 1: + sources = {p.source_policy_id for p in products if p.feed in BOOK_FEEDS} + others = {p.source_policy_id for p in products if p.feed not in BOOK_FEEDS + REFERENCE_FEEDS} + references = {p.source_policy_id for p in products if p.feed in REFERENCE_FEEDS} + if (sum(p.feed in BOOK_FEEDS for p in products) != 2 or len(sources) != 1 or len(others) > 1 + or len(references) > 1 or {p.feed for p in products} != set(feeds)): raise ValueError("exact BOOK snapshot/delta source policy required") + extra = {"crypto_source_policy_id": others.pop()} if others else {} + if references: + extra["execution_reference_source_policy_id"] = references.pop() return {"schema": "p184.dr05.ts-consumer.v1", "venue": venue, "consumer_id": cid, "subject": "spiffe://qdl/paper/" + cid, "jwt_private_key_file": "/work/" + label + ".key", "jwt_key_id": label, @@ -77,7 +95,8 @@ def consumer_packet(manifest_path, catalog, venue, issuer, audience): "query_url": "https://localhost:8200", "grpc_target": "localhost:8210", "tls_ca_file": "/work/ca.crt", "tls_cert_file": "/work/client.crt", "tls_key_file": "/work/client.key", "binding_file": "/work/" + label + ".binding.json", - "release_manifest_sha256": digest, "execution_book_source_policy_id": sources.pop()}, binding.canonical_mapping() + "release_manifest_sha256": digest, "execution_book_source_policy_id": sources.pop(), **extra}, \ + binding.canonical_mapping() def _deadline(timeout): diff --git a/scripts/p184_g12_isolated_fault.py b/scripts/p184_g12_isolated_fault.py index 7db870e1..810f354c 100644 --- a/scripts/p184_g12_isolated_fault.py +++ b/scripts/p184_g12_isolated_fault.py @@ -3,12 +3,32 @@ Purpose: run the G12 gate (TS upgrade/p184-phase-d-gates.md) once, in a disposable namespace: three KRaft brokers (RF3, min ISR 2, mTLS), the production core, -projector, Stream and Query images, a test-provenance BOOK publisher, two TS -writer clones (the TS-owned ``WriterClone``, production entrypoint, 4 workers) -and a sampler that reads each clone's BOOK view through the TS at-use validator. +projector, Stream and Query images, a test-provenance publisher (BOOK, and +TRADE / QUOTE / closed BAR 1m / MARK-INDEX on their own provider lanes), two TS +writer clones (the TS-owned ``WriterClone``, production entrypoint, 4 workers) and a +sampler that reads each clone's BOOK view through the TS at-use validator and receives +every TRADE / QUOTE / MARK / BAR the clone applies (its own ``events.market.v2.*`` +notifications). Faults: stop/start one broker, restart the projector, restart Stream. Per fault it records the G11 four timestamps (injected, cleared, first valid apply, -caught up) and then G1 transit inside a fixed window. +caught up) and then, inside one fixed window, G1 (BOOK), G2 (TRADE, QUOTE and +MARK_INDEX_PRICE new-event transit) and G3 (BAR 1m from close). A fault passes only +when caught up in time and G1, every G2 feed and G3 pass; a part with too few samples +is NOT_MEASURED, never a pass. + +Feeds: the DR05 material is BOOK-only; the in-container ``--feeds`` step (network +none) seals TRADE, QUOTE, BAR 1m and MARK_INDEX_PRICE for the same two instruments +into the same material: core bindings from the production acquisition mapping +(``ProductionCatalogBuilder._acquisition``, checked equal to the production plan; +MARK/INDEX as the deployment's physical components), the source catalog's own +bindings, the TS paper manifest's requirement shape (event recency / session +liveness null like the DR05 BOOK requirements), the route binding and ts-consumer +via the DR05 ``consumer_packet`` and a recompiled bundle. The writer reads MARK +through Query reference reads woken by Stream: the Query role (``--query``) is the +DR05 Query of the code under test with the production KN execution MARK/INDEX +reader wired in (``qdl.runtime.stable`` kn3). Exits of every owned container +(``--rm`` removes it, so its own inspect is gone) are recorded from the daemon's +die/oom/kill events (``evidence/container-events.jsonl``, ``exit-states.json``). Reuse, not a new engine: material/frames/raw envelopes come from the DR05 tools (``p184_dr05_dl_fault``, ``p184_dr05_raw_publisher``), the Query read view is the @@ -41,11 +61,15 @@ runs Query from the image's own /app (no worktree mount); the default ``worktree`` keeps the earlier read-only worktree mount. In-container modes (python image, inside the namespace): ``--topics``, -``--publish``, ``--monitor``; each bounded by ``--seconds`` / its own deadline. +``--publish``, ``--monitor``; each bounded by ``--seconds`` / its own deadline; +``--feeds`` (network none, material only); ``--query`` (the Query role). """ from __future__ import annotations import argparse +import copy +import hashlib +import inspect import json import math import os @@ -87,7 +111,42 @@ "p95_ms": 500, "p99_ms": 1000, "max_ms": 2000, "p99_min_n": 100} LIMITS = {"disk_free_bytes": 20 * 10**9, "mem_available_bytes": 3 * 2**30, "deadline_s": 3600, "holder_s": 5400, "broker_outage_s": 60, - "role_return_s": 120, "clone_ready_s": 300, "baseline_s": 300} + "role_return_s": 120, "clone_ready_s": 300, "baseline_s": 300, "md_ready_s": 180} +# G2 / G3 (frozen TS gate rows; owner decision 2026-10-09). min_n: below it the part is NOT_MEASURED. +# G2 is new-event transit (publish -> first applied notification); G3 is BAR 1m from close. +GATE_G2_G3 = {"TRADE": {"p95_ms": 500, "max_ms": 1500, "min_n": 100}, + "QUOTE": {"p95_ms": 500, "max_ms": 1000, "min_n": 100}, + "MARK": {"p95_ms": 500, "max_ms": 1000, "min_n": 100}, + "BAR": {"p95_ms": 5000, "max_ms": 60000, "min_n": 3}} +G2_FEEDS = ("TRADE", "QUOTE", "MARK") # MARK = MARK_INDEX_PRICE +# An event not applied this long after its reference is lost (TRADE/BAR: a FAIL). QUOTE (ON_CHANGE) and MARK (a +# latest-state reference read per Stream wake) may legitimately skip a superseded value: only counted as unseen. +# Before that an event is pending and not sampled. +LOST_AFTER_S = {"TRADE": 10, "QUOTE": 10, "MARK": 10, "BAR": 60} +LATEST_STATE_FEEDS = ("QUOTE", "MARK") +MD_FEEDS = ("TRADE", "QUOTE", "BAR", "MARK") +CATALOG_FEED = {"TRADE": "TRADE", "QUOTE": "QUOTE", "BAR": "BAR", "MARK": "MARK_INDEX_PRICE"} +# The production acquisition plan: read-only, only for the OKX index symbol of a MARK_INDEX binding (never inferred). +PRODUCTION_ACQUISITION = "config/v2/stable-acquisition-bindings.yaml" +BAR_INTERVAL, BAR_MS = "1m", 60_000 +# Synthetic venue cadence per instrument: steady trades plus a short burst, on-change quotes, one final bar per +# minute shortly after its close (Binance's native lane measured 0.04-1.1 s), a few closed bars seeded at start +# (the bar edge's REST bootstrap role) so the writer's BAR warmup has a retained view, one MARK/INDEX observation +# per second (Binance @markPrice@1s; OKX mark-price + index-tickers pushed as one changed pair per second). +MD_RATE = {"trade_per_s": 3, "burst_every_s": 15, "burst_trades": 8, "quote_per_s": 2, + "bar_delay_ms": 200, "seed_bars": 5, "mark_per_s": 1} +TEST_ADAPTER = "p184-synthetic-test-only" +# Requirement shape of the deployed TS paper manifest (consumers/stable/trading-system-paper.yaml r10) for these feeds; +# event recency and session liveness stay null like the DR05 BOOK requirements (no isolated provider-session signal). +MD_REQUIREMENTS = {"TRADE": {"max_freshness_ms": 3000, "warmup_limit": 0, "require_final_bars": True}, + "QUOTE": {"max_freshness_ms": 2000, "warmup_limit": 0, "require_final_bars": True}, + "BAR": {"max_freshness_ms": 180000, "warmup_limit": 1000, "interval": BAR_INTERVAL, + "require_final_bars": True}, + "MARK": {"max_freshness_ms": 2000, "warmup_limit": 0, "require_final_bars": False, + "recovery": "FRESH_SNAPSHOT"}} +# provider kind suffix -> lane. MARK lanes: Binance one BOTH component; OKX a MARK and an INDEX component. +MD_KIND_FEED = {"_trade": "TRADE", "_bbo": "QUOTE", "_bar": "BAR", "_mark_index": "MARK", "_mark_price": "MARK", + "_index_price": "INDEX"} IMAGE_ID = re.compile(r"sha256:[0-9a-f]{64}") QUERY_CODE = ("worktree", "image") # G12 role -> DR05/DR08 image role (handoff "images"; dr05 image_set() requires exactly these keys). @@ -284,11 +343,18 @@ def topology(ns, work, topic_id="", *, uid=None, images=None, query_code="worktr "QDL_KN_QUEUE_BYTES_TOTAL": "33554432"}) + [ "--entrypoint", "/usr/local/bin/qdl-stream-gateway", images["stream"], "serve"] # "image": the Query code is the image's own /app (the release under test); "worktree": read-only mount. - code = ["-w", "/app"] if query_code == "image" else ["-v", f"{ROOT}:/app:ro", "-w", "/app"] + # Either way the entry is this harness's --query (serve_query): the DR05/DR08 Query of that code plus the + # execution MARK/INDEX reader; with "image" only this one harness file is mounted, never the worktree. + if query_code == "image": + code = ["-v", f"{ROOT / 'scripts/p184_g12_isolated_fault.py'}:/harness/p184_g12_isolated_fault.py:ro", + "-w", "/app"] + entry = "/harness/p184_g12_isolated_fault.py" + else: + code, entry = ["-v", f"{ROOT}:/app:ro", "-w", "/app"], "scripts/p184_g12_isolated_fault.py" roles["query"] = _env(_base(ns, "query", network=net, memory="512m", cpus="0.5", uid=uid) + ["--tmpfs", "/tmp:rw,size=64m,mode=1777"] + mount + code, {**common, "PYTHONPATH": "/app:/app/generated/python", "PYTHONDONTWRITEBYTECODE": "1"}) + [ - "--entrypoint", "python", images["python"], "-B", "scripts/p184_dr05_observer.py", "--query", "/work"] + "--entrypoint", "python", images["python"], "-B", entry, "--query", "/work"] return roles @@ -314,6 +380,34 @@ def helper_argv(ns, work, args, *, role="helper", detach=False, evidence=None, n return argv + ["--entrypoint", "python", images["python"], "-B", script, *args] +def md_event_row(channel, data): + """One applied TS notification -> evidence fields, or None (not a G2/G3 sample). + + Runs inside the sampler (its source is embedded in SAMPLER). TRADE/QUOTE: the + writer's per-event ``events.market.v2.execution..*`` publish (every applied + event; observed_at_ms = canonical source_event_time); id = native trade id, or the + quote's event ms. MARK: the ordered MARK_INDEX_PRICE state publish + (``execution.mark_index_price``, only when the CAS applies a newer view); id = the + pair's observed ms. BAR: ``events.market.v2.bar.*`` closed 1m rows; id = open ms. + """ + parts = str(channel).split(".") + try: + payload = json.loads(data) + except (TypeError, ValueError): + return None + if not isinstance(payload, dict) or parts[:3] != ["events", "market", "v2"] or len(parts) < 5: + return None + if parts[3] == "execution" and parts[4] in ("trade", "quote", "mark_index_price"): + feed = {"trade": "TRADE", "quote": "QUOTE", "mark_index_price": "MARK"}[parts[4]] + event_ms = payload.get("observed_at_ms") + ident = ((payload.get("payload") or {}).get("native_trade_id") if feed == "TRADE" else event_ms) + return {"feed": feed, "id": None if ident is None else str(ident), "ev_ms": event_ms} + if parts[3] == "bar" and payload.get("is_closed") is True and payload.get("interval") == "1m": + opened = payload.get("timestamp_ms") + return {"feed": "BAR", "id": None if opened is None else str(opened), "ev_ms": opened} + return None + + SAMPLER = r''' import asyncio, json, signal, sys, time from redis.asyncio import Redis @@ -348,6 +442,32 @@ async def beat(r): b = json.loads(await r.get(keys[0]) or "{}"); d = b.get("details") or {} return {"heartbeat": True, "status": b.get("status"), "demanded": d.get("demanded_v2_slices"), "session_ready": d.get("session_ready_v2_slices")} +MD_PATTERNS = ("events.market.v2.execution.trade.*", "events.market.v2.execution.quote.*", + "events.market.v2.execution.mark_index_price.*", "events.market.v2.bar.*") +async def listen(url, venue, f, end): + """PSUBSCRIBE only (read-only): receipt time of every applied TRADE/QUOTE/MARK/closed BAR of one clone.""" + while time.monotonic() < end and not STOP: + r = Redis.from_url(url.rsplit("/", 1)[0] + "/0", socket_connect_timeout=1, decode_responses=True) + p = r.pubsub(ignore_subscribe_messages=True) + try: + await p.psubscribe(*MD_PATTERNS) + while time.monotonic() < end and not STOP: + m = await p.get_message(ignore_subscribe_messages=True, timeout=0.5) + if not m or m.get("type") != "pmessage": + continue + t = time.time_ns() + row = md_event_row(m["channel"], m["data"]) + if row is not None: + f.write(json.dumps({"k": "ev", "t": t, "venue": venue, **row}) + "\n") + except Exception as e: + f.write(json.dumps({"k": "ev_error", "t": time.time_ns(), "venue": venue, "error": type(e).__name__}) + "\n") + await asyncio.sleep(1) + finally: + for close in (p.aclose, r.aclose): + try: + await close() + except Exception: + pass async def main(out, seconds, interval_ms, targets): views = {v: Redis.from_url(u, socket_timeout=1, socket_connect_timeout=1) for v, u in targets.items()} beats = {v: Redis.from_url(u.rsplit("/", 1)[0] + "/2", socket_timeout=1, socket_connect_timeout=1, @@ -355,6 +475,7 @@ async def main(out, seconds, interval_ms, targets): end = time.monotonic() + seconds last, last_t, last_beat = {}, {}, 0.0 with open(out, "a", buffering=1) as f: + listeners = [asyncio.create_task(listen(u, v, f, end)) for v, u in targets.items()] while time.monotonic() < end and not STOP: for venue, r in views.items(): t = time.time_ns() @@ -376,11 +497,16 @@ async def main(out, seconds, interval_ms, targets): row = {"heartbeat": None, "error": type(e).__name__} f.write(json.dumps({"k": "hb", "t": time.time_ns(), "venue": venue, **row}) + "\n") await asyncio.sleep(interval_ms / 1000) + await asyncio.wait(listeners, timeout=3) + for task in listeners: + task.cancel() for r in (*views.values(), *beats.values()): await r.aclose() asyncio.run(main(sys.argv[1], float(sys.argv[2]), float(sys.argv[3]), dict(item.split("=", 1) for item in sys.argv[4].split(",")))) ''' +# The sampler runs the same parser the unit tests exercise. +SAMPLER = SAMPLER.replace("STOP = []\n", "STOP = []\n" + inspect.getsource(md_event_row), 1) def sampler_argv(ns, evidence, seconds, interval_ms=50, images=None): @@ -509,6 +635,143 @@ def g1_stats(values): return {k: (round(v, 3) if isinstance(v, float) else v) for k, v in stats.items()} +def md_publish_index(rows): + """(feed, venue, id) -> the publisher's row of one TRADE/QUOTE/BAR source event (k == pub_md).""" + return {(r["feed"], r["venue"], str(r["id"])): r for r in rows if r.get("k") == "pub_md"} + + +def md_first_seen(rows): + """(feed, venue, id) -> first receipt ns of the clone's applied notification (k == ev).""" + seen = {} + for r in rows: + if r.get("k") == "ev" and r.get("id") is not None: + key = (r["feed"], r["venue"], str(r["id"])) + if key not in seen or r["t"] < seen[key]: + seen[key] = r["t"] + return seen + + +def md_reference_ns(row): + """Transit origin: the bar close for BAR (open + 1 m), else the publish stamp (= the frame's event time).""" + return (row["close_ms"] * 1_000_000) if row["feed"] == "BAR" else row["t"] + + +def md_sample(index, seen, feed, venue, start_ns, end_ns, covered_ns): + """New-event transit (ms) of every non-seed event of one feed/venue whose reference lies in [start, end). + + An event never applied is lost once the sampler covered reference + LOST_AFTER_S (QUOTE / MARK: unseen, + a latest-state feed may skip a superseded value), else pending (excluded: not yet decidable). + """ + values, counts = [], {"lost": 0, "unseen": 0, "pending": 0} + for key, row in index.items(): + if key[0] != feed or key[1] != venue or row.get("seed"): + continue + reference = md_reference_ns(row) + if not start_ns <= reference < end_ns: + continue + applied = seen.get(key) + if applied is not None: + values.append((applied - reference) / 1e6) + elif covered_ns >= reference + LOST_AFTER_S[feed] * 1_000_000_000: + counts["unseen" if feed in LATEST_STATE_FEEDS else "lost"] += 1 + else: + counts["pending"] += 1 + return values, counts + + +def md_stats(feed, values, counts=None): + """G2/G3 verdict of one feed/venue: PASS, FAIL, or NOT_MEASURED below the minimum sample count.""" + gate = GATE_G2_G3[feed] + values = sorted(values) + n = len(values) + stats = {"n": n, "p50_ms": quantile(values, .5), "p95_ms": quantile(values, .95), + "p99_ms": quantile(values, .99) if n >= 100 else None, "max_ms": values[-1] if values else None, + **(counts or {"lost": 0, "unseen": 0, "pending": 0})} + if stats["lost"]: + stats["status"] = "FAIL" + stats["note"] = "events never applied" + elif n < gate["min_n"]: + stats["status"] = "NOT_MEASURED" + stats["note"] = f"n < {gate['min_n']}" + else: + stats["status"] = ("PASS" if stats["p95_ms"] <= gate["p95_ms"] and stats["max_ms"] < gate["max_ms"] + else "FAIL") + stats["pass"] = stats["status"] == "PASS" + return {k: (round(v, 3) if isinstance(v, float) else v) for k, v in stats.items()} + + +def part_status(stats): + """One gate part over its cells: any FAIL -> FAIL, else any NOT_MEASURED -> NOT_MEASURED, else PASS.""" + statuses = [s["status"] for s in stats] + if not statuses: + return "NOT_MEASURED" + return "FAIL" if "FAIL" in statuses else "NOT_MEASURED" if "NOT_MEASURED" in statuses else "PASS" + + +def md_window(publish_rows, sampler_rows, start_ns, end_ns): + """G2 (TRADE, QUOTE, MARK_INDEX_PRICE) and G3 (BAR 1m) over one window, plus each part's status.""" + index, seen = md_publish_index(publish_rows), md_first_seen(sampler_rows) + covered = max((r["t"] for r in sampler_rows if r.get("k") in ("view", "ev")), default=0) + cell = lambda feed, venue: md_stats(feed, *md_sample(index, seen, feed, venue, start_ns, end_ns, covered)) + g2 = {feed: {venue: cell(feed, venue) for venue in VENUES} for feed in G2_FEEDS} + g3 = {"BAR_" + BAR_INTERVAL: {venue: cell("BAR", venue) for venue in VENUES}} + return {"g2": g2, "g3": g3, + "g2_feed_status": {feed: part_status(list(cells.values())) for feed, cells in g2.items()}, + "g2_status": part_status([s for cells in g2.values() for s in cells.values()]), + "g3_status": part_status([s for cells in g3.values() for s in cells.values()])} + + +def md_missing(sampler_rows): + """FEED@VENUE never applied by a clone yet (start-up gate before the baseline).""" + seen = {(key[0], key[1]) for key in md_first_seen(sampler_rows)} + return [f"{feed}@{venue}" for venue in VENUES for feed in MD_FEEDS if (feed, venue) not in seen] + + +def publisher_md_summary(publish_rows): + """Per feed: published source events (seed bars apart), broker ACKs and delivery errors.""" + out = {feed: {"published": 0, "seed": 0, "acked": 0, "delivery_errors": 0} for feed in MD_FEEDS} + for r in publish_rows: + feed = out.get(r.get("feed")) + if feed is None: + continue + if r.get("k") == "pub_md": + feed["seed" if r.get("seed") else "published"] += 1 + elif r.get("k") == "ack_md": + feed["delivery_errors" if "error" in r else "acked"] += 1 + return out + + +def exit_states(event_rows): + """role -> exit record of every owned container life, from the daemon's container events. + + ``--rm`` removes a container as it exits, so ``docker inspect`` has nothing left to read; + the die event carries exitCode, an oom event marks the OOM kill of that life, a kill event + the signal that was sent (docker stop / teardown). Attributes are the daemon's; never inferred. + """ + states = {} + pending = {} + for row in sorted(event_rows, key=lambda r: r.get("timeNano") or 0): + attrs = (row.get("Actor") or {}).get("Attributes") or {} + match = re.fullmatch(NS_PATTERN + r"-(.+)", str(attrs.get("name", ""))) + if not match: + continue + role, action = match.group(1), row.get("Action") or row.get("status") + life = pending.setdefault(role, {"oom_killed": False, "kill_signals": []}) + if action == "oom": + life["oom_killed"] = True + elif action == "kill": + life["kill_signals"].append(attrs.get("signal")) + elif action == "die": + code = attrs.get("exitCode") + life.update(exit_code=int(code) if str(code).lstrip("-").isdigit() else None, + died_ns=row.get("timeNano"), + oom_killed=life["oom_killed"] or str(attrs.get("oomKilled", "")).lower() == "true") + states.setdefault(role, {"lives": []})["lives"].append(pending.pop(role)) + for role, value in states.items(): + value.update({k: value["lives"][-1][k] for k in ("exit_code", "oom_killed", "died_ns")}) + return states + + def cleared_at(kind, monitor_rows, restart_ns, node=None): """Fault-cleared time from the monitor (packet section 5).""" for row in monitor_rows: @@ -568,8 +831,15 @@ def evaluate_fault(fault, publish_rows, sampler_rows, monitor_rows, *, window_s= monitor_rows, cleared, lambda r: r.get("projector_lag") is not None and r["projector_lag"] <= 1) result["isr_restored_ns"] = first_monitor(monitor_rows, cleared, lambda r: r.get("under_replicated") == 0) result["g12_caught_up_pass"] = result["caught_up_s"] <= GATE["caught_up_s"] + # G2/G3 over the same window as G1 (new events whose publish time / bar close lies in it). + result.update(md_window(publish_rows, sampler_rows, *window)) + result["parts"] = {"caught_up": result["g12_caught_up_pass"], "window_complete": complete, + "g1": "PASS" if all(result["g1"][v]["pass"] for v in VENUES) else "FAIL", + "g2": result["g2_status"], "g2_feeds": result["g2_feed_status"], + "g3": result["g3_status"]} result["pass"] = (result["g12_caught_up_pass"] and complete - and all(result["g1"][v]["pass"] for v in VENUES)) + and all(result["g1"][v]["pass"] for v in VENUES) + and result["g2_status"] == "PASS" and result["g3_status"] == "PASS") return result @@ -580,20 +850,25 @@ def evaluate(evidence): monitor_rows = sorted(load_jsonl(evidence / "monitor.jsonl"), key=lambda r: r.get("t", 0)) faults = [r for r in load_jsonl(evidence / "faults.jsonl") if r.get("k") == "fault"] index, series = publish_index(publish_rows), view_series(sampler_rows) - out = {"schema": "p184.g12.verdict.v1", "provenance": "TEST_ONLY", "level": "ISOLATED_FAULT", - "gate": GATE, "faults": [], "g2": "NOT_MEASURED (BOOK-only isolated source)", - "g3": "NOT_MEASURED (BOOK-only isolated source)"} + out = {"schema": "p184.g12.verdict.v2", "provenance": "TEST_ONLY", "level": "ISOLATED_FAULT", + "gate": GATE, "gate_g2_g3": GATE_G2_G3, "faults": [], + "g2": "per fault: TRADE, QUOTE and MARK (MARK_INDEX_PRICE) new-event transit " + "(publish -> first applied notification)", + "g3": "per fault: BAR 1m from close (open + 60 s -> first applied closed-bar notification)"} base = [r for r in load_jsonl(evidence / "faults.jsonl") if r.get("k") == "baseline"] if base: window = (base[0]["start_ns"], base[0]["end_ns"]) out["baseline_g1"] = {v: g1_stats(transit(series, index, v, *window)) for v in VENUES} + out["baseline_g2_g3"] = md_window(publish_rows, sampler_rows, *window) for fault in faults: out["faults"].append(evaluate_fault(fault, publish_rows, sampler_rows, monitor_rows)) acks = [r for r in publish_rows if r.get("k") == "ack"] out["publisher"] = {"frames": sum(1 for r in publish_rows if r.get("k") == "pub"), "acked": sum(1 for r in acks if "error" not in r), "delivery_errors": sum(1 for r in acks if "error" in r), - "resyncs": sum(1 for r in publish_rows if r.get("k") == "resync")} + "resyncs": sum(1 for r in publish_rows if r.get("k") == "resync"), + "market_data": publisher_md_summary(publish_rows)} + out["exit_states"] = exit_states(load_jsonl(evidence / "container-events.jsonl")) out["pass"] = bool(out["faults"]) and len(out["faults"]) == 3 and all(f["pass"] for f in out["faults"]) return out @@ -676,9 +951,419 @@ def _set(self, *_): self.requested = True +SYMBOLS = {"BINANCE": "ETHUSDT", "OKX": "ETH-USDT-SWAP"} # the DR05 linear scopes' native symbols + + +def _md_frame(venue, feed, payload, at_ns, lane=None): + return {"venue": venue, "native_symbol": SYMBOLS[venue], "feed": feed, "lane": lane or feed.lower(), + "transport": "WEBSOCKET", "received_at_ns": at_ns, "test_provenance": True, "provenance": "TEST_ONLY", + "adapter_version": TEST_ADAPTER, "payload": payload} + + +def _check_md(venue, at_ns): + if venue not in SYMBOLS or type(at_ns) is not int or at_ns <= 0: + raise ValueError("venue and positive integer timestamp required") + return at_ns // 1_000_000 + + +def trade_frame(venue, trade_id, at_ns): + """One provider-shaped trade (Binance ``@trade`` / OKX ``trades``), event time = the stamp; deterministic.""" + ms = _check_md(venue, at_ns) + if type(trade_id) is not int or trade_id <= 0: + raise ValueError("positive integer trade id required") + price = f"{60000 + (trade_id % 20) / 10:.1f}" + if venue == "BINANCE": + payload = {"e": "trade", "E": ms, "T": ms, "s": SYMBOLS[venue], "t": trade_id, "p": price, + "q": "0.010", "X": "MARKET", "m": trade_id % 2 == 0} + else: + payload = {"arg": {"channel": "trades", "instId": SYMBOLS[venue]}, "data": [{ + "instId": SYMBOLS[venue], "tradeId": str(trade_id), "px": price, "sz": "1", + "side": "sell" if trade_id % 2 == 0 else "buy", "ts": str(ms)}]} + return _md_frame(venue, "TRADE", payload, at_ns) + + +def quote_frame(venue, sequence, at_ns): + """One top-of-book quote (Binance ``@bookTicker`` / OKX ``bbo-tbt``); every quote differs from the previous + one (the bid size moves), so ON_CHANGE delivery has a change to deliver.""" + ms = _check_md(venue, at_ns) + if type(sequence) is not int or sequence <= 0: + raise ValueError("positive integer quote sequence required") + size = f"{1 + (sequence % 97) / 100:.2f}" + if venue == "BINANCE": + payload = {"e": "bookTicker", "u": sequence, "s": SYMBOLS[venue], "b": "60000.0", "B": size, + "a": "60001.0", "A": "1.00", "T": ms, "E": ms} + else: + payload = {"arg": {"channel": "bbo-tbt", "instId": SYMBOLS[venue]}, "data": [{ + "asks": [["60001.0", "1", "0", "1"]], "bids": [["60000.0", size, "0", "1"]], + "ts": str(ms), "seqId": sequence}]} + return _md_frame(venue, "QUOTE", payload, at_ns) + + +def bar_frame(venue, open_ms, at_ns, trades=0): + """One FINAL 1m bar (Binance closed kline x=true / OKX candle1m confirm=1), sent after its close.""" + ms = _check_md(venue, at_ns) + if type(open_ms) is not int or open_ms <= 0 or open_ms % BAR_MS or ms < open_ms + BAR_MS: + raise ValueError("a closed minute-aligned bar is required") + if venue == "BINANCE": + payload = {"e": "kline", "E": ms, "s": SYMBOLS[venue], "k": { + "t": open_ms, "T": open_ms + BAR_MS - 1, "s": SYMBOLS[venue], "i": BAR_INTERVAL, "f": -1, "L": -1, + "o": "60000.0", "c": "60001.0", "h": "60001.9", "l": "60000.0", "v": "1.000", "n": int(trades), + "x": True, "q": "60000.50", "V": "0.500", "Q": "30000.25"}} + else: + payload = {"arg": {"channel": "candle" + BAR_INTERVAL, "instId": SYMBOLS[venue]}, "data": [[ + str(open_ms), "60000.0", "60001.9", "60000.0", "60001.0", "10", "0.1", "6000.05", "1"]]} + return _md_frame(venue, "BAR", payload, at_ns) + + +INDEX_SYMBOLS = {"OKX": "ETH-USDT"} # default for frames only; the sealed binding's physical symbol is used at run + + +def mark_frames(venue, sequence, at_ns, index_symbol=None): + """One MARK/INDEX observation: Binance one ``@markPrice@1s`` frame (mark and index together, component BOTH); + OKX a ``mark-price`` frame and an ``index-tickers`` frame of the index symbol, same ms (the core pairs them on + one generation). Each frame's ``feed`` is its component lane (MARK / INDEX).""" + ms = _check_md(venue, at_ns) + if type(sequence) is not int or sequence <= 0: + raise ValueError("positive integer mark sequence required") + mark, index = f"{60000.5 + (sequence % 10) / 100:.2f}", f"{60000.4 + (sequence % 10) / 100:.2f}" + if venue == "BINANCE": + funding = (ms // 28_800_000 + 1) * 28_800_000 + return [_md_frame(venue, "MARK", {"e": "markPriceUpdate", "E": ms, "s": SYMBOLS[venue], "p": mark, + "P": mark, "i": index, "r": "0.00010000", "T": funding}, at_ns, "mark")] + index_symbol = index_symbol or INDEX_SYMBOLS[venue] + mark_frame = _md_frame(venue, "MARK", {"arg": {"channel": "mark-price", "instId": SYMBOLS[venue]}, "data": [{ + "instType": "SWAP", "instId": SYMBOLS[venue], "markPx": mark, "ts": str(ms)}]}, at_ns, "mark") + index_frame = _md_frame(venue, "INDEX", {"arg": {"channel": "index-tickers", "instId": index_symbol}, "data": [{ + "instId": index_symbol, "idxPx": index, "high24h": "60100.0", "low24h": "59900.0", "open24h": "60000.0", + "sodUtc0": "60000.0", "sodUtc8": "60000.0", "ts": str(ms)}]}, at_ns, "mark") + index_frame["native_symbol"] = index_symbol + return [mark_frame, index_frame] + + +def market_plan(tick, hz, rate=MD_RATE): + """Deterministic (trades, quotes, marks) to send at publisher tick ``tick`` of a ``hz`` loop: the per-second + rates spread over the ticks, plus a burst of trades every ``burst_every_s``.""" + def spread(per_s): + return math.floor((tick + 1) * per_s / hz) - math.floor(tick * per_s / hz) + every = max(1, round(rate["burst_every_s"] * hz)) + trades = spread(rate["trade_per_s"]) + (rate["burst_trades"] if tick and tick % every == 0 else 0) + return trades, spread(rate["quote_per_s"]), spread(rate["mark_per_s"]) + + +def seed_opens(now_ms, count=MD_RATE["seed_bars"]): + """Open times of the ``count`` closed bars just before the current minute (oldest first).""" + current = now_ms // BAR_MS * BAR_MS + return [current - (count - i) * BAR_MS for i in range(count)] + + +def bars_due(now_ms, last_open_ms, delay_ms=MD_RATE["bar_delay_ms"]): + """Opens of every bar after ``last_open_ms`` whose close + delay has passed, in order (no gap after a stall).""" + due, nxt = [], last_open_ms + BAR_MS + while nxt + BAR_MS + delay_ms <= now_ms: + due.append(nxt) + nxt += BAR_MS + return due + + +class MarketDataSource: + """The synthetic venue's TRADE / QUOTE / BAR / MARK events, per tick of the publisher loop (no I/O). + + Each event is {venue, feed, id, frame | frames, extra}: id is the identity the TS notification carries back + (trade id, quote event ms, bar open ms, mark observation ms); a frame is stamped by ``clock`` when it is + built (an OKX MARK event is its mark + index pair). Trade ids and quote / mark sequences only increase; bars + are contiguous from the seed on, each sent once after its close. + """ + + def __init__(self, venues=VENUES, rate=MD_RATE, clock=time.time_ns, index_symbols=None): + self.venues, self.rate, self.clock = tuple(venues), rate, clock + self.index_symbols = dict(INDEX_SYMBOLS if index_symbols is None else index_symbols) + self.mark_seq = {v: 0 for v in self.venues} + self.mark_ms = {v: 0 for v in self.venues} + self.trade_id = {v: 0 for v in self.venues} + self.quote_seq = {v: 0 for v in self.venues} + self.quote_ms = {v: 0 for v in self.venues} + self.last_open = {v: None for v in self.venues} + self.minute_trades = {v: {} for v in self.venues} + + def _bar(self, venue, open_ms, at_ns, seed): + frame = bar_frame(venue, open_ms, at_ns, self.minute_trades[venue].pop(open_ms, 0)) + self.last_open[venue] = open_ms + return {"venue": venue, "feed": "BAR", "id": str(open_ms), "frame": frame, + "extra": {"close_ms": open_ms + BAR_MS, "seed": seed}} + + def seed(self, now_ns): + """The closed bars just before the current minute (a retained view for the writer's BAR warmup).""" + return [self._bar(venue, open_ms, now_ns, True) for venue in self.venues + for open_ms in seed_opens(now_ns // 1_000_000, self.rate["seed_bars"])] + + def tick(self, tick, hz): + events = [] + trades, quotes, marks = market_plan(tick, hz, self.rate) + for venue in self.venues: + for _ in range(trades): + self.trade_id[venue] += 1 + at_ns = self.clock() + minute = at_ns // 1_000_000 // BAR_MS * BAR_MS + self.minute_trades[venue][minute] = self.minute_trades[venue].get(minute, 0) + 1 + events.append({"venue": venue, "feed": "TRADE", "id": str(self.trade_id[venue]), + "frame": trade_frame(venue, self.trade_id[venue], at_ns), "extra": {}}) + for _ in range(quotes): + at_ns = self.clock() + if at_ns // 1_000_000 <= self.quote_ms[venue]: + continue # the quote identity is its event ms: never two in one ms + self.quote_seq[venue] += 1 + self.quote_ms[venue] = at_ns // 1_000_000 + events.append({"venue": venue, "feed": "QUOTE", "id": str(self.quote_ms[venue]), + "frame": quote_frame(venue, self.quote_seq[venue], at_ns), "extra": {}}) + for _ in range(marks): + at_ns = self.clock() + if at_ns // 1_000_000 <= self.mark_ms[venue]: + continue # the mark identity is its observation ms + self.mark_seq[venue] += 1 + self.mark_ms[venue] = at_ns // 1_000_000 + events.append({"venue": venue, "feed": "MARK", "id": str(self.mark_ms[venue]), "extra": {}, + "frames": mark_frames(venue, self.mark_seq[venue], at_ns, + self.index_symbols.get(venue))}) + if self.last_open[venue] is not None: + for open_ms in bars_due(self.clock() // 1_000_000, self.last_open[venue], self.rate["bar_delay_ms"]): + events.append(self._bar(venue, open_ms, self.clock(), False)) + return events + + +def lane_feed(provider_kind): + """The lane of a sealed core binding by provider kind (TRADE/QUOTE/BAR/MARK/INDEX), None otherwise.""" + return next((f for suffix, f in MD_KIND_FEED.items() if provider_kind.endswith(suffix)), None) + + +def md_bindings(config): + """(venue, lane) -> the sealed core binding of each TRADE/QUOTE/BAR/MARK lane (and the OKX INDEX + component); exactly one each.""" + out = {} + for binding in config["core"]["bindings"]: + if binding.get("l2"): + continue + feed = lane_feed(binding["provider_kind"]) + if feed is None or (binding["venue"], feed) in out: + raise ValueError("unexpected or duplicate market-data binding " + binding["source_id"]) + out[(binding["venue"], feed)] = binding + expected = {(venue, feed) for venue in VENUES for feed in MD_FEEDS} | {(v, "INDEX") for v in INDEX_SYMBOLS} + if set(out) != expected: + raise ValueError("sealed TRADE/QUOTE/BAR/MARK lane per venue (and the OKX INDEX component) required") + return out + + +def md_envelope(binding, frame, ns): + """Raw envelope of a TRADE/QUOTE/BAR/MARK test frame on its own provider lane (session + ``..``, generation 1, never changed): the BOOK lane's resync generations never fence these + lanes. A MARK/INDEX component carries its physical symbol/channel (the core's binding key).""" + from qdl.raw.envelope import build_raw_envelope + import uuid + namespace(ns) + if frame.get("test_provenance") is not True or frame.get("provenance") != "TEST_ONLY" \ + or frame.get("adapter_version") != TEST_ADAPTER or frame.get("transport") != "WEBSOCKET": + raise ValueError("explicit test provenance and websocket transport required") + symbol = binding.get("physical_native_symbol") or binding["native_symbol"] + channel = binding.get("physical_native_channel") or binding["native_channel"] + if (binding["venue"], symbol) != (frame["venue"], frame["native_symbol"]) or \ + lane_feed(binding["provider_kind"]) != frame["feed"]: + raise ValueError("frame/binding identity mismatch") + payload = json.dumps(frame["payload"], separators=(",", ":"), allow_nan=False).encode() + return build_raw_envelope( + capture_id=uuid.uuid4().bytes, provider=binding["provider"], venue=binding["venue"], + market=binding["market"], product_type=binding["product_type"], + native_symbol=symbol, native_channel=channel, + subscription_id=binding["source_id"], source_session_id=f"{ns}.{binding['venue']}.{frame['lane']}", + connection_generation=1, lease_epoch=1, authority_revision=1, partition_plan_epoch=1, + received_at_ns=frame["received_at_ns"], transport_protocol=1, transport_compression=1, + capture_boundary=2, raw_frame_bytes=payload, adapter_version=TEST_ADAPTER, config_revision=1, + instrument_catalog_revision=binding["instrument_catalog_revision"], + correlation_id=str(uuid.uuid4()), test_provenance=True) + + +def md_core_binding(book, source, acquisition): + """Core binding of one TRADE/QUOTE/BAR lane: the BOOK binding's exported instrument identity, the source + catalog's source, the production acquisition mapping's provider kind / channel / sequence policy.""" + if acquisition.get("mode") != "RUST_NATIVE": + raise ValueError("native acquisition lane required: " + source["binding_id"]) + if source["instrument_uid"] != book["instrument_uid"] or source["source"]["provider"] != book["provider"]: + raise ValueError("source/instrument identity mismatch: " + source["binding_id"]) + binding = {key: book[key] for key in ("provider", "venue", "market", "product_type", "native_symbol", + "instrument_uid", "instrument_id", "instrument_revision", + "instrument_catalog_revision")} + binding.update(native_channel=acquisition["native_channel"], provider_kind=acquisition["provider_kind"], + source_id=source["source"]["source_id"], source_role=source["source"]["source_role"], + normalizer_version=source["source"]["normalizer_version"], + require_final_bar=bool(source["quality"]["require_final_bar"]), + sequence_policy=acquisition["sequence_policy"]) + return binding + + +def md_core_bindings(book, source, acquisition): + """All core bindings of one lane: one, or the MARK_INDEX physical components exactly as the production + deployment emits them (StableMarkIndexAcquisition.component_entries + the core mapping): Binance one BOTH + component, OKX a MARK component and an INDEX component on the explicit index symbol, each with its signed + quiet cadence.""" + binding = md_core_binding(book, source, acquisition) + mark_index = acquisition.get("mark_index") + if source["feed"] != "MARK_INDEX_PRICE": + if mark_index is not None: + raise ValueError("mark/index acquisition on a non-MARK lane") + return [binding] + cadence = mark_index["component_quiet_after_ms"] + if mark_index["provider_protocol"] == "BINANCE_MARK_PRICE": + entries = [(book["native_symbol"], acquisition["native_channel"], "binance_usdm_mark_index", "BOTH")] + elif mark_index["provider_protocol"] == "OKX_MARK_INDEX": + entries = [(book["native_symbol"], "mark-price", "okx_mark_price", "MARK"), + (str(mark_index["index_native_symbol"]), "index-tickers", "okx_index_price", "INDEX")] + else: + raise ValueError("MARK_INDEX provider protocol is not certified") + out = [] + for symbol, channel, kind, component in entries: + item = dict(binding, provider_kind=kind, physical_native_symbol=symbol, physical_native_channel=channel, + mark_index={"component": component}) + if cadence.get(component) is not None: + item["mark_index"]["quiet_after_ms"] = cadence[component] + out.append(item) + return out + + +def _rewrite(path, value): + """Replace one non-private material file in place (same 0644 mode the DR05 material used).""" + path = Path(path) + temporary = path.with_name(path.name + ".g12-tmp") + with temporary.open("x") as handle: + os.chmod(temporary, 0o644) + handle.write(value if isinstance(value, str) else json.dumps(value)) + os.replace(temporary, path) + + +def seal_market_feeds(work, ns): + """Add TRADE / QUOTE / BAR 1m / MARK_INDEX_PRICE of the two sealed instruments to the DR05 BOOK material + (offline, no I/O outside ``work`` and the read-only source catalog / production acquisition plan): core.json, + catalog.yaml, both manifests, route bindings, ts-consumers and bundle.json. Each generated acquisition must + equal the production plan's entry for that binding. Once per run (handoff "g12_feeds").""" + import yaml + from qdl.query import ConsumerGrade, FeedType + from qdl.runtime.production_catalog import ProductionCatalogBuilder, ProductionDemand + from qdl.runtime.stable_catalog import StableSourceCatalog + from scripts.kn_gateway_bundle import compile_bundle + from scripts.kn_native_slice_probe import ISSUER, AUDIENCE + from scripts.p184_dr05_case import BOOK_FEEDS, consumer_packet + from scripts.p184_dr05_dl_fault import source_catalog_path + + work = Path(work) + handoff = json.loads((work / "handoff.json").read_text()) + if namespace(handoff["namespace"]) != namespace(ns) or handoff.get("g12_feeds") is not None: + raise ValueError("unsealed G12 material of this namespace required") + config = json.loads((work / "core.json").read_text()) + books = config["core"]["bindings"] + if any(not b.get("l2") for b in books) or sorted(b["venue"] for b in books) != sorted(VENUES): + raise ValueError("the DR05 two-venue BOOK material is required") + source = yaml.safe_load(source_catalog_path(work).read_text()) + catalog = yaml.safe_load((work / "catalog.yaml").read_text()) + production = {item["binding_id"]: item for item in + yaml.safe_load((ROOT / PRODUCTION_ACQUISITION).read_text())["bindings"]} + lanes = [] + for book in books: + for feed in MD_FEEDS: + interval = BAR_INTERVAL if feed == "BAR" else None + matches = [b for b in source["bindings"] if b["instrument_uid"] == book["instrument_uid"] + and b["feed"] == CATALOG_FEED[feed] and b.get("interval") == interval + and b["source"]["source_role"] == "PRIMARY"] + if len(matches) != 1: + raise ValueError(f"exactly one primary {feed} source required for {book['instrument_id']}") + planned = production.get(matches[0]["binding_id"]) or {} + mark = ({"max_freshness_ms": MD_REQUIREMENTS["MARK"]["max_freshness_ms"], "require_live": True, + "index_native_symbol": (planned.get("mark_index") or {}).get("index_native_symbol")} + if feed == "MARK" else {}) + demand = ProductionDemand( + consumer_id=ns + "." + book["venue"].lower(), consumer_grade=ConsumerGrade.EXECUTION, + venue=book["venue"], market=book["market"], product_type=book["product_type"], + native_symbol=book["native_symbol"], feed=FeedType(CATALOG_FEED[feed]), interval=interval, + source_policy_id=matches[0]["source"]["source_policy_id"], **mark) + acquisition = ProductionCatalogBuilder._acquisition(matches[0]["binding_id"], demand) + fields = ("mode", "provider_kind", "native_channel", "sequence_policy", "mark_index") + if {k: acquisition.get(k) for k in fields} != {k: planned.get(k) for k in fields}: + raise ValueError("generated acquisition differs from the production plan: " + + matches[0]["binding_id"]) + entry = copy.deepcopy(matches[0]) + entry["source"]["adapter_version"] = TEST_ADAPTER + lanes.append((book, feed, entry, md_core_bindings(book, matches[0], acquisition))) + config["core"]["bindings"] = books + [item for lane in lanes for item in lane[3]] + digest = hashlib.sha256(json.dumps(config["core"]["bindings"], sort_keys=True).encode()).hexdigest() + config["authority"].update(capability_manifest_digest=digest, contract_digest=digest, + partition_plan_digest=digest) + catalog["bindings"] = catalog["bindings"] + [lane[2] for lane in lanes] + _rewrite(work / "core.json", config) + _rewrite(work / "catalog.yaml", yaml.safe_dump(catalog)) + loaded = StableSourceCatalog.load(work / "catalog.yaml") + manifests = [] + for book in books: + label = book["venue"].lower() # the linear scopes: scope id == venue + path = work / (label + ".yaml") + mapping = yaml.safe_load(path.read_text()) + template = next(r for r in mapping["spec"]["requirements"] if r["feed"] == "BOOK_SNAPSHOT") + for _, feed, entry, _ in (lane for lane in lanes if lane[0] is book): + requirement = {k: v for k, v in template.items() if k not in ("interval", "event_recency_policy", + "max_session_liveness_ms")} + requirement.update(feed=CATALOG_FEED[feed], source_policy_id=entry["source"]["source_policy_id"], + bar_revision_policy="LATEST", **MD_REQUIREMENTS[feed]) + mapping["spec"]["requirements"].append(requirement) + mapping["spec"]["quotas"]["max_streams"] = 50 # the TS paper manifest's; 6 slices per clone now + _rewrite(path, yaml.safe_dump(mapping)) + consumer, route = consumer_packet(path, loaded, book["venue"], ISSUER, AUDIENCE, + feeds=BOOK_FEEDS + tuple(CATALOG_FEED[f] for f in MD_FEEDS)) + _rewrite(work / (label + ".binding.json"), route) + _rewrite(work / ("ts-consumer." + label + ".json"), consumer) + manifests.append(path) + _rewrite(work / "bundle.json", compile_bundle(environment="paper", catalog_path=work / "catalog.yaml", + manifest_paths=manifests)) + handoff["g12_feeds"] = {"feeds": list(BOOK_FEEDS) + [CATALOG_FEED[f] for f in MD_FEEDS], + "bar_interval": BAR_INTERVAL, "source_ids": [lane[2]["source"]["source_id"] for lane in lanes], + "core_bindings_sha256": digest} + (work / "handoff.json").write_text(json.dumps(handoff, indent=2) + "\n") + return handoff["g12_feeds"] + + +def with_execution_mark_index(build): + """Wrap ``build_stable_query_stack`` with the production KN execution MARK/INDEX wiring of + ``qdl.runtime.stable`` (query backend kn3): reference data enabled, execution reads from the market cache + (``build_cache_execution_mark_index_reader``), alpha reads likewise. No acquisition quiet policy and no + session-liveness reader: the sealed requirement is event-age BLOCK, which needs neither.""" + from qdl.reference.local_mark_index import ( + build_cache_alpha_mark_index_reader, build_cache_execution_mark_index_reader) + + def stack(*, catalog, backend, **kwargs): + kwargs.update( + reference_data_enabled=True, + execution_mark_index_reader=build_cache_execution_mark_index_reader( + catalog=catalog, backend=backend, acquisition=None, session_liveness_reader=None), + alpha_mark_index_reader_factory=lambda: build_cache_alpha_mark_index_reader( + catalog=catalog, backend=backend)) + return build(catalog=catalog, backend=backend, **kwargs) + return stack + + +def serve_query(work): + """In-namespace Query (--query): the unchanged DR05 diagnostic Query (DR08 serve_query, public HTTP boundary, + KN market cache + hot view) of the code under test, with execution MARK/INDEX reads enabled.""" + import qdl.runtime.stable_source as stable_source + from scripts.p184_dr05_observer import diagnostic_query + stable_source.build_stable_query_stack = with_execution_mark_index(stable_source.build_stable_query_stack) + diagnostic_query(Path(work)) + + +def extend_material(work): + """In-container --feeds (network none): attested namespace / image set, then seal_market_feeds.""" + packet = _packet(work) + return seal_market_feeds(work, packet["namespace"]) + + def publish(work, out, seconds, hz): """Steady provider-shaped BOOK frames per venue; resync (new generation + - snapshot) only after a failed delivery, as a venue adapter would.""" + snapshot) only after a failed delivery, as a venue adapter would. TRADE / + QUOTE / closed BAR 1m / MARK-INDEX frames on their own lanes at the MD_RATE + cadence; their delivery errors are recorded, never a BOOK resync.""" from confluent_kafka import Producer from scripts.p184_dr05_dl_fault import frame from scripts.p184_dr05_raw_publisher import raw_envelope @@ -687,7 +1372,8 @@ def publish(work, out, seconds, hz): config = json.loads((Path(work) / "core.json").read_text()) if config.get("raw_topics") != [ns + ".raw"] or config["core"].get("allow_test_provenance") is not True: raise ValueError("isolated test-only raw topic required") - bindings = {b["venue"]: b for b in config["core"]["bindings"]} + bindings = {b["venue"]: b for b in config["core"]["bindings"] if b.get("l2")} + lanes = md_bindings(config) if set(bindings) != set(VENUES) or not 0 < hz <= 20 or not 0 < seconds <= LIMITS["deadline_s"]: raise ValueError("two-venue roster and bounded rate/duration required") producer = Producer({**_security(packet), "enable.idempotence": True, "acks": "all", @@ -711,8 +1397,43 @@ def callback(error, message): log.write(json.dumps(row) + "\n") return callback + def md_ack(event, sent_ns): + def callback(error, message): + row = {"k": "ack_md", "venue": event["venue"], "feed": event["feed"], "id": event["id"], + "ack_ms": round((time.time_ns() - sent_ns) / 1e6, 3)} + if error is not None: + row["error"] = str(error.code()) + else: + row["offset"] = message.offset() + log.write(json.dumps(row) + "\n") + return callback + + def send_md(event): + frames = event.get("frames") or [event["frame"]] # an OKX MARK event is its mark + index pair + at_ns = frames[0]["received_at_ns"] + for item in frames: + raw = md_envelope(lanes[(event["venue"], item["feed"])], item, ns) + try: + producer.produce(config["raw_topics"][0], partition=0, key=raw.subscription_id.encode(), + value=raw.SerializeToString(), headers=[("qdl-event-id", raw.capture_id)], + on_delivery=md_ack(event, at_ns)) + except BufferError: + log.write(json.dumps({"k": "ack_md", "venue": event["venue"], "feed": event["feed"], + "id": event["id"], "error": "LOCAL_QUEUE_FULL"}) + "\n") + return + log.write(json.dumps({"k": "pub_md", "t": at_ns, "venue": event["venue"], "feed": event["feed"], + "id": event["id"], **event["extra"]}) + "\n") + + source = MarketDataSource(index_symbols={venue: lane["physical_native_symbol"] + for (venue, feed), lane in lanes.items() if feed == "INDEX"}) + for event in source.seed(time.time_ns()): + send_md(event) + tick = 0 next_at = time.monotonic() while time.monotonic() < end and not stop.requested: + for event in source.tick(tick, hz): + send_md(event) + tick += 1 for venue in VENUES: s = state[venue] snapshot = s["snapshot"] @@ -920,6 +1641,17 @@ def image_record(image): "version": labels.get("org.opencontainers.image.version")} +def events_argv(ns, since_s, until_s): + """Bounded read-only ``docker events`` of this namespace's containers: die (exitCode), oom, kill (signal). + ``--until`` makes the follower exit by itself even if the orchestrator is killed.""" + namespace(ns) + if not 0 < since_s < until_s: + raise ValueError("bounded event window required") + return ["docker", "events", "--since", str(int(since_s)), "--until", str(int(until_s)), + "--filter", f"label={LABEL}={ns}", "--filter", "type=container", "--filter", "event=die", + "--filter", "event=oom", "--filter", "event=kill", "--format", "{{json .}}"] + + def load_writer_clone(): import importlib.util path = TS_ROOT / "scripts/p184_dr05_isolated.py" @@ -962,6 +1694,43 @@ def __init__(self, base, export, ns, images=None, query_code="worktree"): self.guard0 = None self.last_guard = 0.0 self.last_stats = 0.0 + self.events_proc = None + + # -- container exit states + def start_events(self): + """Follow this namespace's die/oom/kill events into the evidence for the whole run (bounded).""" + now = time.time() + out = (self.evidence / "container-events.jsonl").open("a") + err = (self.evidence / "container-events.stderr").open("a") + try: + self.events_proc = subprocess.Popen( + events_argv(self.ns, now - 1, now + LIMITS["deadline_s"] + 900), stdout=out, stderr=err, + stdin=subprocess.DEVNULL) + finally: + out.close() + err.close() + + def exit_states(self, roles=(), wait_s=3.0): + """Exit states recorded so far; waits (bounded) until each named role has a die event.""" + limit = time.monotonic() + wait_s + while True: + states = exit_states(load_jsonl(self.evidence / "container-events.jsonl")) + if all(role in states for role in roles) or time.monotonic() >= limit: + return states + time.sleep(.2) + + def stop_events(self): + proc, self.events_proc = self.events_proc, None + if proc is None: + return None + time.sleep(1.0) # the last teardown die events reach the follower + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + return proc.returncode # -- evidence def note(self, step, **fields): @@ -1000,7 +1769,12 @@ def check(self, faulted=()): raise RuntimeError(f"STOP memory available {resources['mem_available_bytes']} < 3 GiB") missing = {f"{self.ns}-{r}" for r in self.expected - set(faulted)} - running_names(self.ns) if missing: - raise RuntimeError("STOP role exited: " + ",".join(sorted(missing))) + roles = sorted(name[len(self.ns) + 1:] for name in missing) + states = self.exit_states(roles) + detail = {role: ({k: states[role][k] for k in ("exit_code", "oom_killed")} if role in states + else "NO_DIE_EVENT") for role in roles} + self.note("role_exited", roles=roles, exit_states={r: states.get(r) for r in roles}) + raise RuntimeError("STOP role exited: " + ",".join(sorted(missing)) + " " + json.dumps(detail)) now = time.monotonic() if now - self.last_guard >= 60: self.last_guard = now @@ -1062,6 +1836,7 @@ def prepare(self): raise RuntimeError("namespace already in use") self.guard0 = production_guard() self.note("production_guard_before", containers=len(self.guard0), guard=self.guard0) + self.start_events() self.volumes0 = len(sh(["docker", "volume", "ls", "-q"]).split()) self.networks0 = len(sh(["docker", "network", "ls", "-q"]).split()) (self.work / "handoff.json").write_text(json.dumps({ @@ -1075,6 +1850,8 @@ def prepare(self): packet = json.loads((self.work / "packet.json").read_text()) packet["bootstrap"] = BOOTSTRAP (self.work / "packet.json").write_text(json.dumps(packet)) + feeds = sh(self.helper(self.ns, self.work, ["--feeds", "/work"], network="none"), timeout=180) + self.note("feeds_sealed", result=feeds.strip()[-400:]) password = broker_password((self.work / "broker.properties").read_text()) for node in BROKERS: path = self.work / f"broker-{node}.properties" @@ -1159,6 +1936,12 @@ def ready(): if not self.wait(LIMITS["clone_ready_s"], until=ready): raise RuntimeError("STOP TS clones never showed a fresh APPROVED book view") self.note("ts_views_fresh") + # Fail fast instead of an hour of NOT_MEASURED: every clone applies TRADE, QUOTE and a closed BAR. + if not self.wait(LIMITS["md_ready_s"], until=lambda: not md_missing( + load_jsonl(self.evidence / "sampler.jsonl"))): + raise RuntimeError("STOP TS clones never applied: " + ",".join( + md_missing(load_jsonl(self.evidence / "sampler.jsonl")))) + self.note("ts_market_data_applied") def baseline(self): start = time.time_ns() @@ -1212,7 +1995,7 @@ def caught(): load_jsonl(self.evidence / "sampler.jsonl"), monitor_rows()) self.note("fault_verdict", fault=name, verdict=verdict) if not verdict["pass"]: - raise RuntimeError(f"STOP {name}: G12/G1 verdict FAIL") + raise RuntimeError(f"STOP {name}: G12 verdict not PASS: " + json.dumps(verdict.get("parts"))) def keep_logs(self): """Bounded log tails of every owned container (--rm loses them at stop).""" @@ -1243,6 +2026,14 @@ def down(self): report["networks_before"] = getattr(self, "networks0", None) report["volumes_after"] = len(sh(["docker", "volume", "ls", "-q"]).split()) report["networks_after"] = len(sh(["docker", "network", "ls", "-q"]).split()) + try: + report["events_follower_exit"] = self.stop_events() + states = exit_states(load_jsonl(self.evidence / "container-events.jsonl")) + (self.evidence / "exit-states.json").write_text(json.dumps(states, indent=2) + "\n") + report["exit_states"] = {role: {k: s[k] for k in ("exit_code", "oom_killed")} + for role, s in sorted(states.items())} + except Exception as error: # noqa: BLE001 - evidence is best effort, teardown is not + report["errors"].append("exit_states:" + type(error).__name__) erase_private(self.work) if self.guard0 is not None: breaches, recreated = guard_breaches(self.guard0, production_guard()) @@ -1346,7 +2137,7 @@ def keep_diagnostics(out): def main(argv=None): parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) mode = parser.add_mutually_exclusive_group(required=True) - for flag in ("run", "down", "analyze", "topics", "publish", "monitor"): + for flag in ("run", "down", "analyze", "topics", "publish", "monitor", "feeds", "query"): mode.add_argument("--" + flag, type=Path) parser.add_argument("--export", type=Path) parser.add_argument("--namespace") @@ -1372,6 +2163,12 @@ def main(argv=None): print(json.dumps(evaluate(args.analyze / "evidence" if (args.analyze / "evidence").exists() else args.analyze), indent=2)) return 0 + if args.feeds: + print(json.dumps(extend_material(args.feeds))) + return 0 + if args.query: + serve_query(args.query) + return 0 if args.seconds is None and not args.topics: parser.error("--seconds required") if args.topics: diff --git a/tests/test_p184_g12_isolated_fault.py b/tests/test_p184_g12_isolated_fault.py index 3ebf0ddf..ef8c1058 100644 --- a/tests/test_p184_g12_isolated_fault.py +++ b/tests/test_p184_g12_isolated_fault.py @@ -22,6 +22,37 @@ def stream_rows(venue, start, end, *, step_ms=250, transit_ms=100, approved=True return pubs, views +def md_rows(venue, start, end, *, trade_ms=100, quote_ms=100, bar_ms=800, mark_ms=300, step_ms=250, trades=True, + quotes=True, bars=True, marks=True): + """Publisher pub_md rows (a trade every step, a quote every other step, a mark every fourth step, a closed bar + per minute) and the clone's applied notifications (k == ev) the given transit later.""" + pubs, evs = [], [] + t, n = start, 0 + while t < end: + n += 1 + if trades: + pubs.append({"k": "pub_md", "t": t, "venue": venue, "feed": "TRADE", "id": str(n)}) + evs.append({"k": "ev", "t": t + trade_ms * MS, "venue": venue, "feed": "TRADE", "id": str(n), + "ev_ms": t // MS}) + if quotes and n % 2 == 0: + pubs.append({"k": "pub_md", "t": t, "venue": venue, "feed": "QUOTE", "id": str(t // MS)}) + evs.append({"k": "ev", "t": t + quote_ms * MS, "venue": venue, "feed": "QUOTE", "id": str(t // MS), + "ev_ms": t // MS}) + if marks and n % 4 == 0: + pubs.append({"k": "pub_md", "t": t, "venue": venue, "feed": "MARK", "id": str(t // MS)}) + evs.append({"k": "ev", "t": t + mark_ms * MS, "venue": venue, "feed": "MARK", "id": str(t // MS), + "ev_ms": t // MS}) + t += step_ms * MS + if bars: + for close_ms in range((start // MS // 60_000 + 1) * 60_000, end // MS, 60_000): + open_ms = close_ms - 60_000 + pubs.append({"k": "pub_md", "t": (close_ms + 200) * MS, "venue": venue, "feed": "BAR", + "id": str(open_ms), "close_ms": close_ms, "seed": False}) + evs.append({"k": "ev", "t": close_ms * MS + bar_ms * MS, "venue": venue, "feed": "BAR", + "id": str(open_ms), "ev_ms": open_ms}) + return pubs, evs + + class NamespaceAndTopologyTests(unittest.TestCase): def test_namespace_is_exact(self): self.assertEqual(g12.namespace(NS), NS) @@ -174,6 +205,9 @@ def test_evaluate_fault_end_to_end(self): p, v = stream_rows(venue, 0, 500 * S) pubs += p views += v + p, v = md_rows(venue, 0, 500 * S) + pubs += p + views += v monitor = [{"t": t * S, "ports": {"8210": t >= 62}, "projector_lag": 0, "under_replicated": 0} for t in range(0, 500)] fault = {"k": "fault", "name": "stream_restart", "kind": "stream", "injected_ns": 60 * S, @@ -185,6 +219,11 @@ def test_evaluate_fault_end_to_end(self): self.assertTrue(result["g1_window_complete"]) self.assertTrue(result["pass"]) self.assertEqual(result["g1"]["OKX"]["n"], 1200) + self.assertEqual(result["parts"], {"caught_up": True, "window_complete": True, "g1": "PASS", + "g2": "PASS", "g2_feeds": {"TRADE": "PASS", "QUOTE": "PASS", + "MARK": "PASS"}, "g3": "PASS"}) + self.assertEqual(result["g2"]["TRADE"]["BINANCE"]["n"], 1200) + self.assertEqual(result["g3"]["BAR_1m"]["OKX"]["n"], 5) # closes 120..360 s in [62.1, 362.1) s short = g12.evaluate_fault(fault, pubs, [v for v in views if v["t"] < 200 * S], monitor) self.assertFalse(short["g1_window_complete"]) self.assertFalse(short["pass"]) @@ -379,5 +418,542 @@ def test_detached_roles_keep_stderr_next_to_evidence(self): self.assertIn("boom", text) +class MarketDataFrameTests(unittest.TestCase): + AT = 1_791_696_000_123 * MS # a minute boundary + 123 ms + + def test_trade_frames_are_provider_shaped_and_deterministic(self): + b = g12.trade_frame("BINANCE", 7, self.AT) + self.assertEqual(b["payload"], {"e": "trade", "E": 1_791_696_000_123, "T": 1_791_696_000_123, "s": "ETHUSDT", + "t": 7, "p": "60000.7", "q": "0.010", "X": "MARKET", "m": False}) + o = g12.trade_frame("OKX", 8, self.AT) + self.assertEqual(o["payload"]["arg"], {"channel": "trades", "instId": "ETH-USDT-SWAP"}) + self.assertEqual(o["payload"]["data"], [{"instId": "ETH-USDT-SWAP", "tradeId": "8", "px": "60000.8", + "sz": "1", "side": "sell", "ts": "1791696000123"}]) + for frame in (b, o): + self.assertEqual((frame["provenance"], frame["test_provenance"], frame["adapter_version"], frame["lane"]), + ("TEST_ONLY", True, g12.TEST_ADAPTER, "trade")) + self.assertEqual(g12.trade_frame("OKX", 8, self.AT), o) + for bad in (("BINANCE", 0, self.AT), ("BINANCE", 1, 0), ("DERIBIT", 1, self.AT), ("OKX", "1", self.AT)): + with self.subTest(bad=bad), self.assertRaises(ValueError): + g12.trade_frame(*bad) + + def test_quotes_change_every_time(self): + b1, b2 = (g12.quote_frame("BINANCE", n, self.AT)["payload"] for n in (1, 2)) + self.assertEqual((b1["u"], b1["T"], b1["b"], b1["a"]), (1, 1_791_696_000_123, "60000.0", "60001.0")) + self.assertNotEqual(b1["B"], b2["B"]) + o = g12.quote_frame("OKX", 3, self.AT)["payload"] + self.assertEqual(o["arg"]["channel"], "bbo-tbt") + self.assertEqual((o["data"][0]["seqId"], o["data"][0]["ts"]), (3, "1791696000123")) + + def test_bar_frames_are_final_and_only_after_close(self): + opened = 1_791_695_940_000 + b = g12.bar_frame("BINANCE", opened, self.AT, trades=4)["payload"] + self.assertTrue(b["k"]["x"]) + self.assertEqual((b["k"]["t"], b["k"]["T"], b["k"]["i"], b["k"]["n"]), (opened, opened + 59_999, "1m", 4)) + o = g12.bar_frame("OKX", opened, self.AT)["payload"] + self.assertEqual(o["arg"]["channel"], "candle1m") + self.assertEqual((o["data"][0][0], o["data"][0][8]), (str(opened), "1")) + with self.assertRaises(ValueError): # not closed yet at the stamp + g12.bar_frame("OKX", opened + 60_000, self.AT) + with self.assertRaises(ValueError): # not minute aligned + g12.bar_frame("OKX", opened + 1, self.AT) + + def test_market_plan_rates_and_burst(self): + hz = 4 + plans = [g12.market_plan(tick, hz) for tick in range(60 * hz)] + trades = sum(t for t, _, _ in plans) + quotes = sum(q for _, q, _ in plans) + self.assertEqual(quotes, 2 * 60) + self.assertEqual(sum(m for _, _, m in plans), 60) # one MARK/INDEX observation per second + bursts = 60 // g12.MD_RATE["burst_every_s"] - 1 # tick 0 never bursts + self.assertEqual(trades, 3 * 60 + bursts * g12.MD_RATE["burst_trades"]) + base = (15 * hz + 1) * 3 // hz - 15 * hz * 3 // hz # the steady share of that tick + self.assertEqual(g12.market_plan(15 * hz, hz)[0], base + g12.MD_RATE["burst_trades"]) + self.assertEqual(plans, [g12.market_plan(tick, hz) for tick in range(60 * hz)]) # deterministic + + def test_bars_due_and_seed(self): + now = 1_791_696_000_150 + self.assertEqual(g12.seed_opens(now, 3), [1_791_695_820_000, 1_791_695_880_000, 1_791_695_940_000]) + last = 1_791_695_940_000 + self.assertEqual(g12.bars_due(now, last), []) # close + 200 ms not reached yet + self.assertEqual(g12.bars_due(1_791_696_060_200, last), [1_791_696_000_000]) + self.assertEqual(g12.bars_due(1_791_696_180_300, last), # a stalled loop catches up without a gap + [1_791_696_000_000, 1_791_696_060_000, 1_791_696_120_000]) + + def test_source_is_monotonic_contiguous_and_seeded(self): + clock = [1_791_696_000_000 * MS] + + def now(): + clock[0] += 1 * MS + return clock[0] + source = g12.MarketDataSource(clock=now) + seeds = source.seed(clock[0]) + self.assertEqual(len(seeds), 2 * g12.MD_RATE["seed_bars"]) + self.assertTrue(all(e["extra"]["seed"] for e in seeds)) + events = [] + for tick in range(4 * 200): # 200 s at 4 Hz + clock[0] += 250 * MS + events += source.tick(tick, 4) + for venue in g12.VENUES: + trades = [int(e["id"]) for e in events if e["venue"] == venue and e["feed"] == "TRADE"] + self.assertEqual(trades, list(range(1, len(trades) + 1))) + quotes = [int(e["id"]) for e in events if e["venue"] == venue and e["feed"] == "QUOTE"] + self.assertEqual(quotes, sorted(set(quotes))) + opens = [int(e["id"]) for e in seeds + events if e["venue"] == venue and e["feed"] == "BAR"] + self.assertEqual(opens, list(range(opens[0], opens[-1] + 1, 60_000))) # contiguous, each once + live = [e for e in events if e["venue"] == venue and e["feed"] == "BAR"] + self.assertEqual(len(live), 3) + for e in live: + self.assertFalse(e["extra"]["seed"]) + self.assertEqual(e["extra"]["close_ms"], int(e["id"]) + 60_000) + self.assertGreaterEqual(e["frame"]["received_at_ns"] // MS, e["extra"]["close_ms"] + 200) + if venue == "BINANCE": # a closed kline counts the trades of its minute + self.assertGreater(sum(e["frame"]["payload"]["k"]["n"] for e in live), 0) + + def test_mark_frames_binance_one_okx_pair(self): + (b,) = g12.mark_frames("BINANCE", 3, self.AT) + self.assertEqual((b["feed"], b["lane"], b["native_symbol"]), ("MARK", "mark", "ETHUSDT")) + self.assertEqual({k: b["payload"][k] for k in ("e", "E", "s", "p", "i")}, + {"e": "markPriceUpdate", "E": 1_791_696_000_123, "s": "ETHUSDT", "p": "60000.53", + "i": "60000.43"}) + self.assertEqual(b["payload"]["T"] % 28_800_000, 0) + mark, index = g12.mark_frames("OKX", 3, self.AT, "ETH-USDT") + self.assertEqual((mark["feed"], mark["native_symbol"], mark["payload"]["arg"]), + ("MARK", "ETH-USDT-SWAP", {"channel": "mark-price", "instId": "ETH-USDT-SWAP"})) + self.assertEqual((index["feed"], index["native_symbol"], index["payload"]["arg"]), + ("INDEX", "ETH-USDT", {"channel": "index-tickers", "instId": "ETH-USDT"})) + self.assertEqual(mark["payload"]["data"][0]["ts"], index["payload"]["data"][0]["ts"]) # one pair, one ms + self.assertEqual((mark["lane"], index["lane"]), ("mark", "mark")) # one session / generation + with self.assertRaises(ValueError): + g12.mark_frames("OKX", 0, self.AT) + + def test_source_marks_once_per_second(self): + clock = [1_791_696_000_000 * MS] + + def now(): + clock[0] += 1 * MS + return clock[0] + source = g12.MarketDataSource(clock=now) + events = [] + for tick in range(4 * 30): + clock[0] += 250 * MS + events += source.tick(tick, 4) + for venue, parts in (("BINANCE", 1), ("OKX", 2)): + marks = [e for e in events if e["venue"] == venue and e["feed"] == "MARK"] + self.assertEqual(len(marks), 30) + self.assertEqual([int(e["id"]) for e in marks], sorted({int(e["id"]) for e in marks})) + self.assertTrue(all(len(e["frames"]) == parts for e in marks)) + self.assertTrue(all(e["id"] == str(e["frames"][0]["received_at_ns"] // MS) for e in marks)) + + def test_query_role_runs_the_harness_query_entry(self): + image = g12.topology(NS, "/home/bobby/x/work", uid="1:1", query_code="image")["query"] + self.assertIn(f"{g12.ROOT / 'scripts/p184_g12_isolated_fault.py'}:/harness/p184_g12_isolated_fault.py:ro", + image) + self.assertEqual(image[-3:], ["/harness/p184_g12_isolated_fault.py", "--query", "/work"]) + worktree = g12.topology(NS, "/home/bobby/x/work", uid="1:1")["query"] + self.assertEqual(worktree[-3:], ["scripts/p184_g12_isolated_fault.py", "--query", "/work"]) + with patch.object(g12, "serve_query") as serve: + self.assertEqual(g12.main(["--query", "/work"]), 0) + serve.assert_called_once() + + def test_notification_parser(self): + trade = g12.json.dumps({"observed_at_ms": 5, "payload": {"native_trade_id": "42", "price": "1"}}) + self.assertEqual(g12.md_event_row("events.market.v2.execution.trade.OKX.SWAP.ETH-USDT-SWAP", trade), + {"feed": "TRADE", "id": "42", "ev_ms": 5}) + quote = g12.json.dumps({"observed_at_ms": 9, "payload": {"bid_price": "1"}}) + self.assertEqual(g12.md_event_row("events.market.v2.execution.quote.BINANCE.USD_M.ETHUSDT", quote), + {"feed": "QUOTE", "id": "9", "ev_ms": 9}) + mark = g12.json.dumps({"observed_at_ms": 11, "feed": "MARK_INDEX_PRICE", + "payload": {"mark_price": {}, "index_price": {}}}) + self.assertEqual(g12.md_event_row("events.market.v2.execution.mark_index_price.OKX.SWAP.ETH-USDT-SWAP", mark), + {"feed": "MARK", "id": "11", "ev_ms": 11}) + closed = g12.json.dumps({"timestamp_ms": 60_000, "is_closed": True, "interval": "1m"}) + self.assertEqual(g12.md_event_row("events.market.v2.bar.OKX.SWAP.ETH-USDT-SWAP.1m", closed), + {"feed": "BAR", "id": "60000", "ev_ms": 60_000}) + for channel, data in (("events.market.v2.bar.OKX.SWAP.X.1m", g12.json.dumps({"timestamp_ms": 1, + "is_closed": False, + "interval": "1m"})), + ("events.market.v2.bar.OKX.SWAP.X.5m", g12.json.dumps({"timestamp_ms": 1, + "is_closed": True, + "interval": "5m"})), + ("events.market.v2.execution.book_delta.OKX.SWAP.X", trade), + ("events.market.v2.execution.trade.OKX.SWAP.X", "not json"), + ("events.market.trade.OKX.X", trade)): + with self.subTest(channel=channel): + self.assertIsNone(g12.md_event_row(channel, data)) + self.assertIn(g12.inspect.getsource(g12.md_event_row), g12.SAMPLER) # the sampler runs this parser + + +class MarketDataWindowTests(unittest.TestCase): + def window(self, pubs, evs, start=100 * S, end=400 * S): + return g12.md_window(pubs, evs + [{"k": "view", "t": 500 * S, "venue": "OKX", "seq": 1, "v": "APPROVED"}], + start, end) + + def rows(self, **kwargs): + pubs, evs = [], [] + for venue in g12.VENUES: + p, e = md_rows(venue, 0, 500 * S, **kwargs) + pubs += p + evs += e + return pubs, evs + + def test_pass(self): + result = self.window(*self.rows()) + self.assertEqual((result["g2_status"], result["g3_status"]), ("PASS", "PASS")) + self.assertEqual(result["g2"]["TRADE"]["OKX"]["n"], 1200) + self.assertEqual(result["g2"]["QUOTE"]["OKX"]["n"], 600) + self.assertEqual(result["g2"]["TRADE"]["OKX"]["p95_ms"], 100.0) + self.assertEqual(result["g3"]["BAR_1m"]["BINANCE"]["n"], 5) # closes 120..360 s + self.assertEqual(result["g3"]["BAR_1m"]["BINANCE"]["max_ms"], 800.0) # from close, not from publish + + def test_fail_on_max_and_p95(self): + pubs, evs = self.rows() + slow = next(e for e in evs if e["feed"] == "TRADE" and e["venue"] == "OKX" and e["t"] > 200 * S) + slow["t"] += 1500 * MS # one trade 1.6 s: max >= 1500 + result = self.window(pubs, evs) + self.assertEqual(result["g2"]["TRADE"]["OKX"]["status"], "FAIL") + self.assertEqual(result["g2"]["TRADE"]["BINANCE"]["status"], "PASS") + self.assertEqual(result["g2_status"], "FAIL") + result = self.window(*self.rows(bar_ms=6000)) # every bar 6 s after close: p95 > 5 s + self.assertEqual(result["g3_status"], "FAIL") + result = self.window(*self.rows(quote_ms=1000)) # QUOTE max must stay < 1000 + self.assertEqual(result["g2"]["QUOTE"]["OKX"]["status"], "FAIL") + + def test_few_samples_are_not_measured_never_pass(self): + pubs, evs = self.rows() + short = self.window(pubs, evs, 100 * S, 190 * S) # one bar close (120 s), 360 trades + self.assertEqual(short["g3"]["BAR_1m"]["OKX"]["status"], "NOT_MEASURED") + self.assertEqual(short["g3_status"], "NOT_MEASURED") + self.assertEqual(short["g2_feed_status"], {"TRADE": "PASS", "QUOTE": "PASS", "MARK": "NOT_MEASURED"}) + self.assertEqual(short["g2"]["QUOTE"]["OKX"]["n"], 180) + self.assertEqual(short["g2"]["MARK"]["OKX"]["n"], 90) + self.assertEqual(short["g2_status"], "NOT_MEASURED") + sparse = self.window(*self.rows(step_ms=4000)) # 75 trades in 300 s + self.assertEqual(sparse["g2"]["TRADE"]["OKX"]["status"], "NOT_MEASURED") + none = self.window(*self.rows(trades=False, quotes=False, bars=False, marks=False)) + self.assertEqual((none["g2_status"], none["g3_status"]), ("NOT_MEASURED", "NOT_MEASURED")) + + def test_lost_trade_fails_unseen_quote_does_not_pending_excluded(self): + pubs, evs = self.rows() + lost = next(e for e in evs if e["feed"] == "TRADE" and e["venue"] == "OKX" and e["t"] > 200 * S) + conflated = next(e for e in evs if e["feed"] == "QUOTE" and e["venue"] == "OKX" and e["t"] > 200 * S) + evs = [e for e in evs if e is not lost and e is not conflated] + result = self.window(pubs, evs) + self.assertEqual((result["g2"]["TRADE"]["OKX"]["lost"], result["g2"]["TRADE"]["OKX"]["status"]), (1, "FAIL")) + self.assertEqual(result["g2"]["QUOTE"]["OKX"]["unseen"], 1) + self.assertEqual(result["g2"]["QUOTE"]["OKX"]["status"], "PASS") + # The sampler has not yet covered reference + 10 s: undecidable, excluded rather than lost. + early = g12.md_window(pubs, [e for e in evs if e["t"] < 205 * S], 100 * S, 400 * S) + self.assertEqual(early["g2"]["TRADE"]["OKX"]["lost"], 0) + self.assertGreater(early["g2"]["TRADE"]["OKX"]["pending"], 0) + + def test_mark_window_gate_and_unseen(self): + result = self.window(*self.rows()) + self.assertEqual(result["g2"]["MARK"]["OKX"]["n"], 300) + self.assertEqual(result["g2_feed_status"]["MARK"], "PASS") + slow = self.window(*self.rows(mark_ms=1000)) # max must stay < 1000 + self.assertEqual(slow["g2"]["MARK"]["BINANCE"]["status"], "FAIL") + self.assertEqual(slow["g2_status"], "FAIL") + pubs, evs = self.rows() + skipped = [e for e in evs if e["feed"] == "MARK" and e["venue"] == "OKX" and 200 * S < e["t"] < 210 * S] + evs = [e for e in evs if not any(e is x for x in skipped)] + result = self.window(pubs, evs) # a latest-state read may skip a superseded observation + self.assertEqual((result["g2"]["MARK"]["OKX"]["unseen"], result["g2"]["MARK"]["OKX"]["lost"], + result["g2"]["MARK"]["OKX"]["status"]), (len(skipped), 0, "PASS")) + sparse = self.window(*self.rows(marks=False)) + self.assertEqual(sparse["g2"]["MARK"]["OKX"]["status"], "NOT_MEASURED") + self.assertEqual(sparse["g2_status"], "NOT_MEASURED") + + def test_seed_bars_and_replays_do_not_count(self): + pubs, evs = self.rows() + for row in pubs: + if row["feed"] == "BAR": + row["seed"] = True + self.assertEqual(self.window(pubs, evs)["g3"]["BAR_1m"]["OKX"]["n"], 0) + pubs, evs = self.rows() + replay = [dict(e, t=e["t"] + 30 * S) for e in evs] # re-projected after a restart: first receipt counts + result = self.window(pubs, evs + replay) + self.assertEqual(result["g2"]["TRADE"]["OKX"]["max_ms"], 100.0) + + def test_fault_verdict_needs_g1_g2_and_g3(self): + monitor = [{"t": t * S, "ports": {"8210": t >= 62}, "projector_lag": 0, "under_replicated": 0} + for t in range(0, 500)] + fault = {"k": "fault", "name": "stream_restart", "kind": "stream", "injected_ns": 60 * S, + "restart_ns": 61 * S} + + def run(**kwargs): + pubs, views = [], [] + for venue in g12.VENUES: + p, v = stream_rows(venue, 0, 500 * S) + pubs += p + views += v + p, v = md_rows(venue, 0, 500 * S, **kwargs) + pubs += p + views += v + return g12.evaluate_fault(fault, pubs, views, monitor) + self.assertTrue(run()["pass"]) + no_bars = run(bars=False) + self.assertEqual(no_bars["parts"]["g1"], "PASS") + self.assertEqual(no_bars["parts"]["g3"], "NOT_MEASURED") + self.assertFalse(no_bars["pass"]) + slow = run(trade_ms=1600) + self.assertEqual(slow["parts"]["g2"], "FAIL") + self.assertFalse(slow["pass"]) + no_mark = run(marks=False) + self.assertEqual(no_mark["parts"]["g2_feeds"], {"TRADE": "PASS", "QUOTE": "PASS", "MARK": "NOT_MEASURED"}) + self.assertEqual(no_mark["parts"]["g2"], "NOT_MEASURED") + self.assertFalse(no_mark["pass"]) + slow_mark = run(mark_ms=1000) # MARK max must stay < 1000 ms + self.assertEqual(slow_mark["parts"]["g2_feeds"]["MARK"], "FAIL") + self.assertFalse(slow_mark["pass"]) + book_only = run(trades=False, quotes=False, bars=False, marks=False) # the 2026-10-11 rc10 evidence + self.assertEqual((book_only["parts"]["g2"], book_only["parts"]["g3"]), ("NOT_MEASURED", "NOT_MEASURED")) + self.assertFalse(book_only["pass"]) + + def test_readiness_and_publisher_summary(self): + pubs, evs = self.rows() + self.assertEqual(g12.md_missing(evs), []) + self.assertEqual(g12.md_missing([e for e in evs if e["feed"] != "BAR"]), ["BAR@BINANCE", "BAR@OKX"]) + self.assertEqual(g12.md_missing([e for e in evs if e["feed"] != "MARK"]), ["MARK@BINANCE", "MARK@OKX"]) + summary = g12.publisher_md_summary(pubs + [{"k": "ack_md", "feed": "TRADE", "venue": "OKX", "id": "1"}, + {"k": "ack_md", "feed": "TRADE", "venue": "OKX", "id": "2", + "error": "_MSG_TIMED_OUT"}]) + self.assertEqual(summary["TRADE"]["acked"], 1) + self.assertEqual(summary["TRADE"]["delivery_errors"], 1) + self.assertEqual(summary["BAR"]["published"], 2 * 8) + + +class SealedFeedMaterialTests(unittest.TestCase): + """Real offline DR05 material + the G12 feed seal (python image: qdl, yaml, cryptography).""" + + def test_seal_adds_trade_quote_bar_lanes_end_to_end(self): + import json + import tempfile + from pathlib import Path + from scripts import p184_dr05_dl_fault as dr + from scripts.kn_gateway_bundle import compile_bundle + from qdl.consumer.manifest import ConsumerManifestLoader + from qdl.runtime.stable_catalog import StableSourceCatalog + from tests.test_p184_dr05_dl_fault import MaterialTests + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "work" + work.mkdir() + (work / "handoff.json").write_text(json.dumps({"namespace": NS, "images": g12.dr05_images(g12.IMAGES), + "g12_images": g12.IMAGES})) + dr.material(work, MaterialTests.exported()) + sealed = g12.seal_market_feeds(work, NS) + self.assertEqual(sealed["feeds"], ["BOOK_SNAPSHOT", "BOOK_DELTA", "TRADE", "QUOTE", "BAR", + "MARK_INDEX_PRICE"]) + config = json.loads((work / "core.json").read_text()) + lanes = g12.md_bindings(config) + expected = {("BINANCE", "TRADE"): ("binance_usdm_trade", "ethusdt@trade", "MONOTONIC", False), + ("BINANCE", "QUOTE"): ("binance_usdm_bbo", "ethusdt@bookTicker", "MONOTONIC", False), + ("BINANCE", "BAR"): ("binance_usdm_bar", "ethusdt@kline_1m", "NONE", True), + ("OKX", "TRADE"): ("okx_trade", "trades", "MONOTONIC", False), + ("OKX", "QUOTE"): ("okx_bbo", "bbo-tbt", "NONE", False), + ("OKX", "BAR"): ("okx_bar", "candle1m", "NONE", True)} + for key, (kind, channel, policy, final) in expected.items(): + binding = lanes[key] + self.assertEqual((binding["provider_kind"], binding["native_channel"], binding["sequence_policy"], + binding["require_final_bar"]), (kind, channel, policy, final), key) + self.assertNotIn("l2", binding) + # MARK_INDEX_PRICE as the deployment's physical components (production cadences). + mark = lanes[("BINANCE", "MARK")] + self.assertEqual((mark["provider_kind"], mark["physical_native_symbol"], mark["physical_native_channel"], + mark["mark_index"], mark["sequence_policy"]), + ("binance_usdm_mark_index", "ETHUSDT", "ethusdt@markPrice@1s", + {"component": "BOTH", "quiet_after_ms": 5000}, "NONE")) + okx_mark, okx_index = lanes[("OKX", "MARK")], lanes[("OKX", "INDEX")] + self.assertEqual((okx_mark["provider_kind"], okx_mark["physical_native_symbol"], + okx_mark["physical_native_channel"], okx_mark["mark_index"]), + ("okx_mark_price", "ETH-USDT-SWAP", "mark-price", + {"component": "MARK", "quiet_after_ms": 15000})) + self.assertEqual((okx_index["provider_kind"], okx_index["physical_native_symbol"], + okx_index["physical_native_channel"], okx_index["mark_index"]), + ("okx_index_price", "ETH-USDT", "index-tickers", + {"component": "INDEX", "quiet_after_ms": 70000})) + self.assertEqual(okx_mark["source_id"], okx_index["source_id"]) # one logical source, two components + self.assertEqual((okx_mark["native_symbol"], okx_mark["native_channel"]), ("ETH-USDT-SWAP", "mark-price")) + books = [b for b in config["core"]["bindings"] if b.get("l2")] + self.assertEqual(len(books), 2) + self.assertEqual(len(config["core"]["bindings"]), 2 + 2 * 4 + 1) + self.assertEqual(len({b["source_id"] for b in config["core"]["bindings"]}), 10) + self.assertEqual(config["authority"]["contract_digest"], sealed["core_bindings_sha256"]) + catalog = StableSourceCatalog.load(work / "catalog.yaml") + self.assertEqual(len(catalog.bindings), 4 + 8) + manifests = [work / "binance.yaml", work / "okx.yaml"] + for path in manifests: + manifest = ConsumerManifestLoader.load(path) + self.assertEqual({r.feed.value for r in manifest.requirements}, + {"BOOK_SNAPSHOT", "BOOK_DELTA", "TRADE", "QUOTE", "BAR", "MARK_INDEX_PRICE"}) + ref = next(r for r in manifest.requirements if r.feed.value == "MARK_INDEX_PRICE") + self.assertEqual((ref.max_freshness_ms, ref.source_policy_id, ref.recovery.value, + ref.require_final_bars, ref.event_recency_policy), + (2000, "crypto_liquid_v2", "FRESH_SNAPSHOT", False, None)) + bar = next(r for r in manifest.requirements if r.feed.value == "BAR") + self.assertEqual((bar.interval, bar.max_freshness_ms, bar.require_final_bars), ("1m", 180000, True)) + trade = next(r for r in manifest.requirements if r.feed.value == "TRADE") + self.assertEqual((trade.max_freshness_ms, trade.source_policy_id, trade.event_recency_policy), + (3000, "crypto_primary_v2", None)) + for label in ("binance", "okx"): + route = json.loads((work / (label + ".binding.json")).read_text()) + planes = {p["feed"]: p["provider_plane"] for p in route["products"]} + self.assertEqual(planes, {"BOOK_SNAPSHOT": "L2", "BOOK_DELTA": "L2", "TRADE": "REALTIME", + "QUOTE": "REALTIME", "BAR": "REALTIME", "MARK_INDEX_PRICE": "REFERENCE"}) + self.assertEqual(next(p["interval"] for p in route["products"] if p["feed"] == "BAR"), "1m") + consumer = json.loads((work / ("ts-consumer." + label + ".json")).read_text()) + # TS WriterClone maps crypto_source_policy_id -> DATA_LAYER_V2_CRYPTO_SOURCE_POLICY_ID. + self.assertEqual((consumer["crypto_source_policy_id"], consumer["execution_book_source_policy_id"], + consumer["execution_reference_source_policy_id"]), + ("crypto_primary_v2", "crypto_liquid_v2", "crypto_liquid_v2")) + import hashlib + self.assertEqual(consumer["release_manifest_sha256"], + hashlib.sha256((work / (label + ".yaml")).read_bytes()).hexdigest()) + self.assertEqual(json.loads((work / "bundle.json").read_text()), compile_bundle( + environment="paper", catalog_path=work / "catalog.yaml", manifest_paths=manifests)) + self.assertFalse(list(work.glob("*.g12-tmp"))) + with self.assertRaises(ValueError): # sealed once + g12.seal_market_feeds(work, NS) + + # Raw envelopes of each lane: own session, generation 1, test provenance, exact binding identity. + from qdl.raw.envelope import validate_raw_envelope + at = 1_791_696_000_123 * MS + frames = {"TRADE": g12.trade_frame("OKX", 1, at), "QUOTE": g12.quote_frame("OKX", 1, at), + "BAR": g12.bar_frame("OKX", 1_791_695_940_000, at)} + for feed, frame in frames.items(): + raw = g12.md_envelope(lanes[("OKX", feed)], frame, NS) + validate_raw_envelope(raw) + self.assertEqual((raw.source_session_id, raw.connection_generation, raw.test_provenance), + (f"{NS}.OKX.{feed.lower()}", 1, True)) + self.assertEqual((raw.subscription_id, raw.native_channel), + (lanes[("OKX", feed)]["source_id"], lanes[("OKX", feed)]["native_channel"])) + with self.assertRaises(ValueError): # a trade frame on the bar lane + g12.md_envelope(lanes[("OKX", "BAR")], frames["TRADE"], NS) + okx_pair = g12.mark_frames("OKX", 1, at, okx_index["physical_native_symbol"]) + for part, lane in zip(okx_pair, (okx_mark, okx_index)): + raw = g12.md_envelope(lanes[("OKX", part["feed"])], part, NS) + validate_raw_envelope(raw) + self.assertEqual((raw.native_symbol, raw.native_channel, raw.source_session_id, + raw.connection_generation), + (lane["physical_native_symbol"], lane["physical_native_channel"], f"{NS}.OKX.mark", 1)) + (binance_mark,) = g12.mark_frames("BINANCE", 1, at) + raw = g12.md_envelope(mark, binance_mark, NS) + self.assertEqual((raw.native_symbol, raw.native_channel), ("ETHUSDT", "ethusdt@markPrice@1s")) + with self.assertRaises(ValueError): # an index frame on the mark component + g12.md_envelope(okx_mark, okx_pair[1], NS) + + # Query: the DR05 Query stack with the production KN execution MARK/INDEX reader wired in. + from qdl.reference.execution_live import HttpExecutionMarkIndexReader + captured = {} + + def build(**kwargs): + captured.update(kwargs) + return "stack" + backend = object() + stack = g12.with_execution_mark_index(build) + self.assertEqual(stack(spool=None, catalog=catalog, backend=backend, issuer="i"), "stack") + self.assertTrue(captured["reference_data_enabled"]) + self.assertIsInstance(captured["execution_mark_index_reader"], HttpExecutionMarkIndexReader) + self.assertIs(captured["backend"], backend) + self.assertEqual(captured["issuer"], "i") + self.assertIsNotNone(captured["alpha_mark_index_reader_factory"]) + with self.assertRaises(ValueError): + g12.md_envelope(lanes[("BINANCE", "TRADE")], frames["TRADE"], NS) + with self.assertRaises(ValueError): + g12.md_envelope(lanes[("OKX", "TRADE")], dict(frames["TRADE"], provenance="LIVE"), NS) + + def test_dr05_consumer_packet_default_is_unchanged(self): + import json + import tempfile + from pathlib import Path + from scripts import p184_dr05_dl_fault as dr + from tests.test_p184_dr05_dl_fault import MaterialTests + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "work" + dr.prepare(work) + dr.material(work, MaterialTests.exported()) + consumer = json.loads((work / "ts-consumer.okx.json").read_text()) + self.assertNotIn("crypto_source_policy_id", consumer) + route = json.loads((work / "okx.binding.json").read_text()) + self.assertEqual({p["provider_plane"] for p in route["products"]}, {"L2"}) + + +class ExitStateTests(unittest.TestCase): + @staticmethod + def event(action, role, t, **attrs): + return {"Type": "container", "Action": action, "status": action, "timeNano": t, + "Actor": {"ID": "x", "Attributes": {"name": f"{NS}-{role}", g12.LABEL: NS, **attrs}}} + + def test_events_argv_is_bounded_and_label_scoped(self): + argv = g12.events_argv(NS, 100, 200) + self.assertEqual(argv[:6], ["docker", "events", "--since", "100", "--until", "200"]) + self.assertIn(f"label={g12.LABEL}={NS}", argv) + for event in ("die", "oom", "kill"): + self.assertIn("event=" + event, argv) + for bad in ((NS, 0, 1), (NS, 5, 5), ("qdl_v2_stable_candidate", 1, 2)): + with self.subTest(bad=bad), self.assertRaises(ValueError): + g12.events_argv(*bad) + + def test_exit_states_per_life(self): + rows = [self.event("kill", "projector", 1, signal="15"), self.event("die", "projector", 2, exitCode="143"), + self.event("oom", "monitor", 5), self.event("die", "monitor", 6, exitCode="137"), + self.event("die", "tswriter-okx", 7, exitCode="1"), + self.event("die", "projector", 9, exitCode="0"), + {"Action": "die", "timeNano": 3, "Actor": {"Attributes": {"name": "qdl_v2_stable_candidate-x", + "exitCode": "1"}}}] + states = g12.exit_states(rows) + self.assertEqual(set(states), {"projector", "monitor", "tswriter-okx"}) + self.assertEqual((states["monitor"]["exit_code"], states["monitor"]["oom_killed"]), (137, True)) + self.assertEqual(len(states["projector"]["lives"]), 2) + self.assertEqual(states["projector"]["lives"][0]["kill_signals"], ["15"]) + self.assertEqual((states["projector"]["exit_code"], states["projector"]["oom_killed"]), (0, False)) + self.assertEqual(states["tswriter-okx"]["exit_code"], 1) + self.assertTrue(g12.exit_states([self.event("die", "publisher", 1, exitCode="137", + oomKilled="true")])["publisher"]["oom_killed"]) + + def test_check_stops_with_the_exit_state_of_a_vanished_role(self): + import json + import tempfile + from pathlib import Path + with tempfile.TemporaryDirectory() as base: + run = g12.Run(base, "/home/bobby/e.json", NS) + run.evidence.mkdir(parents=True) + with (run.evidence / "container-events.jsonl").open("w") as handle: + for row in (self.event("oom", "monitor", 1), self.event("die", "monitor", 2, exitCode="137")): + handle.write(json.dumps(row) + "\n") + run.expected = {"monitor", "publisher", "sampler"} + with patch.object(g12, "host_resources", return_value={"disk_free_bytes": 10**12, + "mem_available_bytes": 10**11}), \ + patch.object(g12, "running_names", return_value={NS + "-publisher"}), \ + self.assertRaises(RuntimeError) as stop: + run.check() + message = str(stop.exception) + self.assertIn("STOP role exited", message) + self.assertIn('"monitor": {"exit_code": 137, "oom_killed": true}', message) + self.assertIn('"sampler": "NO_DIE_EVENT"', message) + notes = [json.loads(line) for line in + (Path(base) / "evidence" / "orchestrator.jsonl").read_text().splitlines()] + self.assertEqual(notes[-1]["step"], "role_exited") + + def test_stop_events_terminates_the_follower(self): + import tempfile + + class Proc: + returncode = None + + def terminate(self): + self.returncode = -15 + + def wait(self, timeout): + return self.returncode + with tempfile.TemporaryDirectory() as base: + run = g12.Run(base, "/home/bobby/e.json", NS) + self.assertIsNone(run.stop_events()) + run.events_proc = Proc() + with patch.object(g12.time, "sleep"): + self.assertEqual(run.stop_events(), -15) + self.assertIsNone(run.events_proc) + + if __name__ == "__main__": unittest.main() From 22ca8df3184fef63584b60805c0681d4bb918f79 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 11 Oct 2026 08:10:40 +0000 Subject: [PATCH 138/147] test(p184): G12 isolated Query reads the namespace's own state topics The DR08 Query built its cache reader from production topic defaults, so every isolated BAR warmup missed its checkpoint and the TS clones never started BAR. Also keeps the writer clones' stderr in the evidence. --- scripts/p184_g12_isolated_fault.py | 37 ++++++++++++++++- tests/test_p184_g12_isolated_fault.py | 59 +++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/scripts/p184_g12_isolated_fault.py b/scripts/p184_g12_isolated_fault.py index 810f354c..969a57c9 100644 --- a/scripts/p184_g12_isolated_fault.py +++ b/scripts/p184_g12_isolated_fault.py @@ -1344,11 +1344,41 @@ def stack(*, catalog, backend, **kwargs): return stack +def isolated_state_topics(ns): + """The market-cache reader's state-topic settings of this namespace (the projector's own topics; it takes + their partition counts from the topic metadata, ``topics()`` creates them).""" + from qdl.runtime.kn_market_cache import ( + BARS_PARTITIONS_ENV, BARS_TOPIC_ENV, LATEST_PARTITIONS_ENV, LATEST_TOPIC_ENV) + wanted = topics(ns) + return {LATEST_TOPIC_ENV: ns + ".latest", LATEST_PARTITIONS_ENV: str(wanted[ns + ".latest"][0]), + BARS_TOPIC_ENV: ns + ".bars", BARS_PARTITIONS_ENV: str(wanted[ns + ".bars"][0])} + + +def with_isolated_state_topics(build_reader, ns): + """Wrap ``reader_from_environment`` so the Query reads this namespace's state topics. + + The DR08 ``serve_query`` passes a literal two-key environment, so the reader fell back to the production + defaults ``md.latest.v2`` / ``md.bars.v2`` with 6 partitions. A BAR read takes its source boundary from the + bars checkpoint ``ckpt::``; under the wrong topic that hash does not exist, every + BAR history is ``SOURCE_BOUNDARY_UNKNOWN`` (typed DATA_NOT_READY) and the writer's BAR slice never starts + (G12 run 2026-10-11 p184-g12-rc10-full: 22 warmup refusals, BAR never applied; latest feeds were unaffected + because a latest read also proves its boundary from the entry's own offset).""" + extra = isolated_state_topics(ns) + + def reader(environ): + return build_reader({**dict(environ), **extra}) + return reader + + def serve_query(work): """In-namespace Query (--query): the unchanged DR05 diagnostic Query (DR08 serve_query, public HTTP boundary, - KN market cache + hot view) of the code under test, with execution MARK/INDEX reads enabled.""" + KN market cache + hot view) of the code under test, with execution MARK/INDEX reads enabled and the cache + reader on this namespace's state topics.""" + import qdl.runtime.kn_market_cache as kn_market_cache import qdl.runtime.stable_source as stable_source from scripts.p184_dr05_observer import diagnostic_query + ns = namespace(json.loads((Path(work) / "packet.json").read_text())["namespace"]) + kn_market_cache.reader_from_environment = with_isolated_state_topics(kn_market_cache.reader_from_environment, ns) stable_source.build_stable_query_stack = with_execution_mark_index(stable_source.build_stable_query_stack) diagnostic_query(Path(work)) @@ -1998,7 +2028,8 @@ def caught(): raise RuntimeError(f"STOP {name}: G12 verdict not PASS: " + json.dumps(verdict.get("parts"))) def keep_logs(self): - """Bounded log tails of every owned container (--rm loses them at stop).""" + """Bounded log tails of every owned container (--rm loses them at stop), and each TS writer clone's own + stderr file (the writer logs there, not to docker logs; the file is erased with work/tslogs).""" logs = self.evidence / "logs" logs.mkdir(exist_ok=True) for name in owned_names(self.ns): @@ -2006,6 +2037,8 @@ def keep_logs(self): text=True, timeout=30) (logs / (name[len(self.ns) + 1:] + ".log")).write_text( (result.stdout + result.stderr)[-60000:]) + for venue, clone in self.clones.items(): + (logs / f"tswriter-{venue.lower()}.stderr.log").write_text(clone.stderr_tail(60000)) def down(self): removed, errors = [], [] diff --git a/tests/test_p184_g12_isolated_fault.py b/tests/test_p184_g12_isolated_fault.py index ef8c1058..e9bcb078 100644 --- a/tests/test_p184_g12_isolated_fault.py +++ b/tests/test_p184_g12_isolated_fault.py @@ -955,5 +955,64 @@ def wait(self, timeout): self.assertIsNone(run.events_proc) +class QueryStateTopicTests(unittest.TestCase): + """G12 run 2026-10-11 (p184-g12-rc10-full): BAR never reached the clones because the isolated Query read the + BAR source boundary from the production bars checkpoint (DR08 serve_query passes a literal two-key env).""" + + DR08_ENV = {"QDL_KN3_MARKET_CACHE_URL": "redis://localhost:6379/1", "QDL_KN3_ENVIRONMENT": "paper"} + + def test_reader_reads_this_namespace_state_topics(self): + from qdl.runtime import kn_market_cache + default = kn_market_cache.reader_from_environment(self.DR08_ENV) # the cause: production defaults + self.assertEqual((default.bars_topic, default.bars_partitions), ("md.bars.v2", 6)) + reader = g12.with_isolated_state_topics(kn_market_cache.reader_from_environment, NS)(self.DR08_ENV) + wanted = g12.topics(NS) + self.assertEqual((reader.bars_topic, reader.bars_partitions), (NS + ".bars", wanted[NS + ".bars"][0])) + self.assertEqual((reader.latest_topic, reader.latest_partitions), + (NS + ".latest", wanted[NS + ".latest"][0])) + # The BAR head reads its source watermark from the checkpoint the projector writes for its own topic. + self.assertEqual(reader.checkpoint_key(reader.bars_topic, 1), f"kn3:paper:ckpt:{NS}.bars:1") + self.assertEqual(g12.isolated_state_topics(NS)["QDL_KN_BARS_PARTITIONS"], "3") + + def test_serve_query_patches_reader_and_stack_then_runs_the_dr05_query(self): + import json + import tempfile + from pathlib import Path + from qdl.runtime import kn_market_cache, stable_source + from scripts import p184_dr05_observer + reader, stack = kn_market_cache.reader_from_environment, stable_source.build_stable_query_stack + seen = {} + + def fake_query(work): + seen["reader"] = kn_market_cache.reader_from_environment(self.DR08_ENV) + seen["stack"] = stable_source.build_stable_query_stack + seen["work"] = work + try: + with tempfile.TemporaryDirectory() as tmp, patch.object(p184_dr05_observer, "diagnostic_query", + side_effect=fake_query): + (Path(tmp) / "packet.json").write_text(json.dumps({"namespace": NS})) + g12.serve_query(tmp) + finally: + kn_market_cache.reader_from_environment, stable_source.build_stable_query_stack = reader, stack + self.assertEqual(seen["reader"].bars_topic, NS + ".bars") + self.assertIsNot(seen["stack"], stack) + self.assertEqual(str(seen["work"]), tmp) + + def test_keep_logs_keeps_the_writer_stderr(self): + import tempfile + from pathlib import Path + + class Clone: + def stderr_tail(self, limit): + return "Traceback: boom\n" + with tempfile.TemporaryDirectory() as base: + run = g12.Run(base, "/home/bobby/e.json", NS) + run.evidence.mkdir(parents=True) + run.clones = {"OKX": Clone()} + with patch.object(g12, "owned_names", return_value=[]): + run.keep_logs() + self.assertEqual((Path(base) / "evidence/logs/tswriter-okx.stderr.log").read_text(), "Traceback: boom\n") + + if __name__ == "__main__": unittest.main() From 139dafef3ae595cb31af4228b029f0a27a5afaed Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 11 Oct 2026 08:12:25 +0000 Subject: [PATCH 139/147] fix(query): a typed backend refusal in a warmup batch keeps its code and retryability KnMarketCacheQueryBackend.history_many stores QueryBackendError per item; the batch route mapped it to a non-retryable INTERNAL_ERROR, hiding retryable DATA_NOT_READY results (seen on isolated BAR warmup, 2026-10-11). --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 13 +++++ qdl/query/service.py | 7 ++- tests/test_phase10_universal_warmup.py | 61 ++++++++++++++++++++++- 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index bb2d4f41..e365da1d 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72207,3 +72207,16 @@ no build, CI or subagent job running. a PASS. - Container die/oom events are kept as evidence (open item from attempt 1). - Tests: 99 + 157 OK (unittest in rc10). Status: tested locally; the live run follows. +- DL defect fixed (found by the extended G12 run, 2026-10-11): the warmup batch route turned a typed per-item + QueryBackendError from the KN local batch (kn_query_backend.history_many) into a non-retryable INTERNAL_ERROR + 500 ("warmup batch item failed inside the bounded executor"). That hid every retryable DATA_NOT_READY BAR warmup, + in production too. + - qdl/query/service.py now keeps the backend problem (code and retryability), as the single-item path already did. + - Regression test added; 116 warmup/backend tests OK. + - Needs a Query image rebuild and roll (owner approval). +- Redis `redis_marketdata` read-only diagnosis (owner approved), then cpu-shares 4096 (owner approved): + - Found: no cgroup limit, no throttling, no eviction or fork, 5 MB used; 44.6 M involuntary context switches on a + CPU-pressured host. This is the likely cause of the 10-40 ms EVAL stalls (inferred). + - After the change, involuntary switches went 209 -> 160 per 60 s at similar load. G8 is re-measured in the next + campaign. + - Receipt: ~/.local/state/qdl-v2/redis-cpu-shares-20261011/. Rollback (failure only): --cpu-shares 0. diff --git a/qdl/query/service.py b/qdl/query/service.py index d072690b..3e36ee5d 100644 --- a/qdl/query/service.py +++ b/qdl/query/service.py @@ -1022,10 +1022,13 @@ def assemble_batch(executions) -> BatchQueryResult: continue error = execution.error if isinstance(error, RetryableWarmupError) and isinstance( - error.cause, QueryServiceError + error.cause, (QueryServiceError, QueryBackendError) ): problem = error.cause.problem - elif isinstance(error, QueryServiceError): + elif isinstance(error, (QueryServiceError, QueryBackendError)): + # A typed backend refusal (e.g. KN DATA_NOT_READY from the local + # batch) keeps its code and retryability, as on the single-item + # path; it is not an internal error. problem = error.problem elif isinstance(error, RetryableWarmupError): problem = QueryProblem( diff --git a/tests/test_phase10_universal_warmup.py b/tests/test_phase10_universal_warmup.py index 74d3b656..7f7721af 100644 --- a/tests/test_phase10_universal_warmup.py +++ b/tests/test_phase10_universal_warmup.py @@ -41,7 +41,7 @@ V2QueryService, ) from qdl.domain.calendar import trading_calendar_for_id -from qdl.query.results import MarketDataItem +from qdl.query.results import MarketDataItem, QueryBackendError from qdl.query.lanes import ReadLaneRejected from qdl.query.service import _LocalBatchAdmission from qdl.runtime.closed_bar_cache import ClosedBarWindowCache @@ -1684,6 +1684,65 @@ def requirement(instrument_uid: str) -> DataRequirement: self.assertEqual(service.last_batch_evidence["local_batch_items"], 2) self.assertEqual(service.warmup_executor._pending, {}) + async def test_query_local_batch_keeps_a_backend_refusal_typed_and_retryable(self): + # The KN backend stores QueryBackendError per item (kn_query_backend.history_many). The batch + # route turned it into a non-retryable INTERNAL_ERROR 500 and hid every retryable not-ready + # BAR warmup (P18.4 G12 isolated run, 2026-10-11). + class Backend: + @staticmethod + def warmup_is_local(_requirement): + return True + + def history_many(self, requirements): + return { + requirements[0]: QueryBackendError( + QueryProblem(CanonicalErrorCode.DATA_NOT_READY, "source boundary unknown", True) + ) + } + + class Service(V2QueryService): + def __init__(self): + self.backend = Backend() + self.instruments = SimpleNamespace() + self.warmup_executor = BoundedWarmupExecutor( + provider_policies={ + "LOCAL_CANONICAL_CACHE": ProviderBudgetPolicy( + max_concurrency=1, + requests_per_second=None, + max_attempts=1, + max_pending=2, + deadline_starts_after_admission=True, + max_batch_concurrency=1, + ) + } + ) + self.last_batch_evidence = {} + + def _warmup_from_history(self, requirement, history, *, purpose, request_id): + raise AssertionError("a refused item is never materialized") + + service = Service() + result = await service.warmup_batch_async( + BatchRequirement( + consumer_id="phase10-local-batch-backend-refusal", + requirements=( + DataRequirement( + instrument_uid="local-batch-backend-refusal", + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + warmup=WarmupSpecification.for_rows(1), + ), + ), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + ) + service._query_work_pools_for().close() + (item,) = result.results + self.assertEqual(item.status, "DATA_NOT_READY") + self.assertTrue(item.problem.retryable) + async def test_query_local_batch_snapshot_cancellation_releases_admission(self): class Backend: def __init__(self): From 2d4b88c876cc80895c5aae09a7d0029780352c7d Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 11 Oct 2026 08:30:10 +0000 Subject: [PATCH 140/147] test(p184): G12 monitor survives a librdkafka crash; monitor gaps make a fault NOT_MEASURED Each 1 s sample's Kafka admin calls run in a supervised child with a hard timeout; a SIGSEGV/hang becomes a typed sample_error row naming the call instead of stopping the run (exit 139 after a broker stop, 2026-10-11). A monitor gap over 5 s around catch-up scores the fault NOT_MEASURED. --- scripts/p184_g12_isolated_fault.py | 305 ++++++++++++++++++++------ tests/test_p184_g12_isolated_fault.py | 145 +++++++++++- 2 files changed, 382 insertions(+), 68 deletions(-) diff --git a/scripts/p184_g12_isolated_fault.py b/scripts/p184_g12_isolated_fault.py index 969a57c9..31301033 100644 --- a/scripts/p184_g12_isolated_fault.py +++ b/scripts/p184_g12_isolated_fault.py @@ -111,7 +111,11 @@ "p95_ms": 500, "p99_ms": 1000, "max_ms": 2000, "p99_min_n": 100} LIMITS = {"disk_free_bytes": 20 * 10**9, "mem_available_bytes": 3 * 2**30, "deadline_s": 3600, "holder_s": 5400, "broker_outage_s": 60, - "role_return_s": 120, "clone_ready_s": 300, "baseline_s": 300, "md_ready_s": 180} + "role_return_s": 120, "clone_ready_s": 300, "baseline_s": 300, "md_ready_s": 180, + "monitor_sample_s": 5} +# A fault whose clearance / catch-up interval has no valid monitor sample for longer than this is NOT_MEASURED: +# its fault-cleared time (the monitor's) would be uncertain by more than the gap (steady cadence: 1.0 s). +MONITOR_MAX_GAP_S = 5 # G2 / G3 (frozen TS gate rows; owner decision 2026-10-09). min_n: below it the part is NOT_MEASURED. # G2 is new-event transit (publish -> first applied notification); G3 is BAR 1m from close. GATE_G2_G3 = {"TRADE": {"p95_ms": 500, "max_ms": 1500, "min_n": 100}, @@ -772,10 +776,22 @@ def exit_states(event_rows): return states +def monitor_valid(monitor_rows): + """Monitor samples that observed: a ``sample_error`` (crashed / timed-out sample) or the end marker is + never evidence of anything.""" + return [row for row in monitor_rows if not row.get("k")] + + +def monitor_max_gap_s(monitor_rows, start_ns, end_ns): + """Longest interval (s) in [start, end] without a valid monitor sample, the edges included.""" + points = [start_ns, *sorted(r["t"] for r in monitor_valid(monitor_rows) if start_ns < r["t"] < end_ns), end_ns] + return round(max(b - a for a, b in zip(points, points[1:])) / 1e9, 3) if end_ns > start_ns else 0.0 + + def cleared_at(kind, monitor_rows, restart_ns, node=None): """Fault-cleared time from the monitor (packet section 5).""" for row in monitor_rows: - if row.get("t", 0) < restart_ns: + if row.get("t", 0) < restart_ns or row.get("k"): continue if kind == "broker" and node in ((row.get("cluster") or {}).get("broker_ids") or ()): return row["t"] @@ -788,7 +804,7 @@ def cleared_at(kind, monitor_rows, restart_ns, node=None): def first_monitor(monitor_rows, start_ns, predicate): for row in monitor_rows: - if row.get("t", 0) >= start_ns and predicate(row): + if row.get("t", 0) >= start_ns and not row.get("k") and predicate(row): return row["t"] return None @@ -831,13 +847,22 @@ def evaluate_fault(fault, publish_rows, sampler_rows, monitor_rows, *, window_s= monitor_rows, cleared, lambda r: r.get("projector_lag") is not None and r["projector_lag"] <= 1) result["isr_restored_ns"] = first_monitor(monitor_rows, cleared, lambda r: r.get("under_replicated") == 0) result["g12_caught_up_pass"] = result["caught_up_s"] <= GATE["caught_up_s"] + # The fault-cleared time is the monitor's: with no valid sample for longer than MONITOR_MAX_GAP_S between the + # restart and catch-up it is uncertain by that much, so the fault is NOT_MEASURED (never a PASS). + result["monitor_max_gap_s"] = monitor_max_gap_s(monitor_rows, fault["restart_ns"], caught) + result["monitor_sample_errors"] = [ + {k: r.get(k) for k in ("t", "error", "phase")} for r in monitor_rows + if r.get("k") == "sample_error" and fault["injected_ns"] <= r["t"] <= caught] + monitored = "PASS" if result["monitor_max_gap_s"] <= MONITOR_MAX_GAP_S else "NOT_MEASURED" + if monitored != "PASS": + result["reason"] = "MONITOR_GAP" # G2/G3 over the same window as G1 (new events whose publish time / bar close lies in it). result.update(md_window(publish_rows, sampler_rows, *window)) result["parts"] = {"caught_up": result["g12_caught_up_pass"], "window_complete": complete, "g1": "PASS" if all(result["g1"][v]["pass"] for v in VENUES) else "FAIL", "g2": result["g2_status"], "g2_feeds": result["g2_feed_status"], - "g3": result["g3_status"]} - result["pass"] = (result["g12_caught_up_pass"] and complete + "g3": result["g3_status"], "monitor": monitored} + result["pass"] = (monitored == "PASS" and result["g12_caught_up_pass"] and complete and all(result["g1"][v]["pass"] for v in VENUES) and result["g2_status"] == "PASS" and result["g3_status"] == "PASS") return result @@ -869,6 +894,10 @@ def evaluate(evidence): "resyncs": sum(1 for r in publish_rows if r.get("k") == "resync"), "market_data": publisher_md_summary(publish_rows)} out["exit_states"] = exit_states(load_jsonl(evidence / "container-events.jsonl")) + errors = [r for r in monitor_rows if r.get("k") == "sample_error"] + out["monitor"] = {"samples": len(monitor_valid(monitor_rows)), "sample_errors": len(errors), + "errors": sorted({(r.get("error"), r.get("phase")) for r in errors}, key=str), + "max_gap_rule_s": MONITOR_MAX_GAP_S} out["pass"] = bool(out["faults"]) and len(out["faults"]) == 3 and all(f["pass"] for f in out["faults"]) return out @@ -1496,78 +1525,220 @@ def send_md(event): return {"stopped": True} +def kafka_sample(source, admin, roster, rosters, groups, phase=lambda name: None): + """The Kafka part of one monitor sample (cluster/ISR, projector committed vs LSO, group membership). + ``phase(name)`` is announced before each admin call, so a native crash names the call it died in.""" + from scripts.kn_recovery_observe import Deadline + row = {} + phase("describe_cluster") + try: + row["cluster"] = source.describe_cluster(Deadline(3.0)) + except Exception as error: # noqa: BLE001 - recorded, never inferred + row["cluster_error"] = type(error).__name__ + phase("describe_topics") + try: + described = source.describe_topics(sorted(roster), Deadline(3.0)) + parts = [p for values in described.values() for p in values] + row["under_replicated"] = sum(1 for p in parts if len(p["isr"]) < len(p["replicas"])) + row["offline"] = sum(1 for p in parts if p["leader"] is None) + row["leaders"] = {f"{t}/{p['partition']}": p["leader"] for t, ps in described.items() for p in ps} + row["min_isr_size"] = min(len(p["isr"]) for p in parts) + except Exception as error: # noqa: BLE001 + row["topics_error"] = type(error).__name__ + try: + lag = {} + for group, group_roster in rosters.items(): + phase("list_consumer_group_offsets:" + group) + committed = source.committed_offsets(group, group_roster, Deadline(3.0)) + phase("list_offsets:" + group) + ends = source.log_end_offsets(sorted(committed), Deadline(3.0)) + lag[group] = sum(max(0, ends[tp] - max(committed[tp], 0)) for tp in committed) + row["lag"] = lag + row["projector_lag"] = lag["kn-projector-v3-a"] + except Exception as error: # noqa: BLE001 + row["lag_error"] = type(error).__name__ + phase("describe_consumer_groups") + try: + described = admin.describe_consumer_groups(list(groups), request_timeout=3) + row["groups"] = {} + for group, future in described.items(): + value = future.result(timeout=4) + row["groups"][group] = {"state": str(getattr(value, "state", "")).split(".")[-1], + "members": len(value.members or ())} + except Exception as error: # noqa: BLE001 + row["groups_error"] = type(error).__name__ + phase("done") + return row + + +def local_sample(work): + """The non-Kafka part of one monitor sample: Query/Stream ports, disk, projector status file.""" + import socket + row = {"ports": {}} + for port in (8200, 8210): + try: + with socket.create_connection(("localhost", port), timeout=.5): + row["ports"][str(port)] = True + except OSError: + row["ports"][str(port)] = False + row["disk_free_bytes"] = shutil.disk_usage(str(work)).free + try: + row["projector_status_mtime"] = (Path(work) / "g12-projector-status.json").stat().st_mtime + except FileNotFoundError: + row["projector_status_mtime"] = None + return row + + +def exit_reason(code): + """A child's exit as a typed reason: SIGSEGV / SIGKILL / ... for a signal, EXIT_ otherwise.""" + if code is not None and code < 0: + try: + return signal.Signals(-code).name + except ValueError: + return f"SIGNAL_{-code}" + return f"EXIT_{code}" + + +def _sample_child(conn, make_sampler): + """Forked sampler: builds its own client on first use, answers one sample per request.""" + sampler = None + while True: + try: + request = conn.recv() + except (EOFError, OSError): + return + if request is None: + return + try: + if sampler is None: + sampler = make_sampler() + fields = sampler(lambda name: conn.send(("phase", name))) + except Exception as error: # noqa: BLE001 - typed back to the parent, then this child ends + conn.send(("error", type(error).__name__ + ":" + str(error)[:200])) + return + conn.send(("row", fields)) + + +class SampleSupervisor: + """Runs each sample in a long-lived forked child with a hard per-sample timeout. + + A native crash (librdkafka SIGSEGV, G12 run 2026-10-11 full2: the monitor exited 139 one second into a + broker stop) or a hang loses only the sample in flight: it comes back as a typed error ({error, phase, + exitcode}) and the next sample starts a fresh child. The parent never loads the Kafka client (fork-safe). + """ + + def __init__(self, make_sampler, timeout_s, context=None): + import multiprocessing + self.make_sampler, self.timeout_s = make_sampler, timeout_s + self.context = context or multiprocessing.get_context("fork") + self.proc = self.conn = None + self.restarts, self.started = 0, False + + def _start(self): + parent, child = self.context.Pipe() + self.proc = self.context.Process(target=_sample_child, args=(child, self.make_sampler), daemon=True) + self.proc.start() + child.close() + self.conn = parent + + def _stop(self, kill): + proc, conn, self.proc, self.conn = self.proc, self.conn, None, None + if proc is None: + return None + if kill and proc.is_alive(): + proc.kill() + proc.join(5) + if conn is not None: + conn.close() + return proc.exitcode + + def sample(self): + """(fields, None) or (None, error).""" + if self.proc is None or not self.proc.is_alive(): + if self.proc is not None: + self._stop(kill=True) + if self.started: + self.restarts += 1 + self._start() + self.started = True + phase, limit = None, time.monotonic() + self.timeout_s + try: + self.conn.send("sample") + except (BrokenPipeError, OSError): + code = self._stop(kill=True) + return None, {"error": exit_reason(code), "phase": None, "exitcode": code} + while True: + left = limit - time.monotonic() + if left <= 0: + code = self._stop(kill=True) + return None, {"error": "TIMEOUT", "phase": phase, "exitcode": code} + try: + if self.conn.poll(min(left, .05)): + kind, value = self.conn.recv() + if kind == "phase": + phase = value + continue + if kind == "row": + return value, None + code = self._stop(kill=True) + return None, {"error": "EXCEPTION", "detail": value, "phase": phase, "exitcode": code} + except (EOFError, OSError): + pass + if not self.proc.is_alive(): + code = self._stop(kill=False) + return None, {"error": exit_reason(code), "phase": phase, "exitcode": code} + + def close(self): + if self.conn is not None: + try: + self.conn.send(None) + except (BrokenPipeError, OSError): + pass + self._stop(kill=True) + + def monitor(work, out, seconds, period=1.0): """Read-only 1 s observation: cluster/ISR, projector committed vs LSO, - group membership, Query/Stream ports, disk. No group join or commit.""" - import socket - from confluent_kafka.admin import AdminClient - from scripts.kn_recovery_observe import Deadline, KafkaAdminSource + group membership, Query/Stream ports, disk. No group join or commit. + + The Kafka admin calls run in a supervised child (SampleSupervisor): a crash or + hang of one sample is written as ``{"k": "sample_error", "error": "SIGSEGV" | + "TIMEOUT" | ..., "phase": }`` and never stops the monitor; the + G12 analysis excludes those rows and refuses a fault whose clearance or + catch-up falls in a monitor gap (MONITOR_MAX_GAP_S).""" packet = _packet(work) ns = packet["namespace"] - admin = AdminClient(_security(packet)) - source = KafkaAdminSource(admin) roster = topics(ns) rosters = {"kn-projector-v3-a": {ns + ".canonical": roster[ns + ".canonical"][0]}, "kn-projector-v3-b": {ns + ".latest": roster[ns + ".latest"][0], ns + ".bars": roster[ns + ".bars"][0]}} groups = (*PROJECTOR_GROUPS, "kn-" + ns, ns) + + def make_sampler(): + from confluent_kafka.admin import AdminClient + from scripts.kn_recovery_observe import KafkaAdminSource + admin = AdminClient(_security(packet)) + source = KafkaAdminSource(admin) + return lambda phase: kafka_sample(source, admin, roster, rosters, groups, phase) + + supervisor = SampleSupervisor(make_sampler, LIMITS["monitor_sample_s"]) stop, end = _Stop(), time.monotonic() + seconds - status_path = Path(work) / "g12-projector-status.json" - with open(out, "a", buffering=1) as log: - while time.monotonic() < end and not stop.requested: - started = time.monotonic() - row = {"t": time.time_ns()} - deadline = Deadline(3.0) - try: - row["cluster"] = source.describe_cluster(deadline) - except Exception as error: # noqa: BLE001 - recorded, never inferred - row["cluster_error"] = type(error).__name__ - try: - described = source.describe_topics(sorted(roster), Deadline(3.0)) - parts = [p for values in described.values() for p in values] - row["under_replicated"] = sum(1 for p in parts if len(p["isr"]) < len(p["replicas"])) - row["offline"] = sum(1 for p in parts if p["leader"] is None) - row["leaders"] = {f"{t}/{p['partition']}": p["leader"] for t, ps in described.items() for p in ps} - row["min_isr_size"] = min(len(p["isr"]) for p in parts) - except Exception as error: # noqa: BLE001 - row["topics_error"] = type(error).__name__ - try: - lag = {} - for group, group_roster in rosters.items(): - committed = source.committed_offsets(group, group_roster, Deadline(3.0)) - ends = source.log_end_offsets(sorted(committed), Deadline(3.0)) - lag[group] = sum(max(0, ends[tp] - max(committed[tp], 0)) for tp in committed) - row["lag"] = lag - row["projector_lag"] = lag["kn-projector-v3-a"] - except Exception as error: # noqa: BLE001 - row["lag_error"] = type(error).__name__ - try: - described = admin.describe_consumer_groups(list(groups), request_timeout=3) - row["groups"] = {} - for group, future in described.items(): - value = future.result(timeout=4) - row["groups"][group] = {"state": str(getattr(value, "state", "")).split(".")[-1], - "members": len(value.members or ())} - except Exception as error: # noqa: BLE001 - row["groups_error"] = type(error).__name__ - row["ports"] = {} - for port in (8200, 8210): - try: - with socket.create_connection(("localhost", port), timeout=.5): - row["ports"][str(port)] = True - except OSError: - row["ports"][str(port)] = False - usage = shutil.disk_usage("/work") - row["disk_free_bytes"] = usage.free - try: - stat = status_path.stat() - row["projector_status_mtime"] = stat.st_mtime - except FileNotFoundError: - row["projector_status_mtime"] = None - log.write(json.dumps(row, default=list) + "\n") - time.sleep(max(0.0, period - (time.monotonic() - started))) - log.write(json.dumps({"k": "end", "t": time.time_ns(), - "reason": "SIGTERM" if stop.requested else "seconds"}) + "\n") + try: + with open(out, "a", buffering=1) as log: + while time.monotonic() < end and not stop.requested: + started = time.monotonic() + t = time.time_ns() + fields, error = supervisor.sample() + if error is not None: + row = {"t": t, "k": "sample_error", **error, "restarts": supervisor.restarts} + else: + row = {"t": t, **fields, **local_sample(work)} + log.write(json.dumps(row, default=list) + "\n") + time.sleep(max(0.0, period - (time.monotonic() - started))) + log.write(json.dumps({"k": "end", "t": time.time_ns(), + "reason": "SIGTERM" if stop.requested else "seconds"}) + "\n") + finally: + supervisor.close() return {"stopped": True} diff --git a/tests/test_p184_g12_isolated_fault.py b/tests/test_p184_g12_isolated_fault.py index e9bcb078..3e1d2a18 100644 --- a/tests/test_p184_g12_isolated_fault.py +++ b/tests/test_p184_g12_isolated_fault.py @@ -221,7 +221,8 @@ def test_evaluate_fault_end_to_end(self): self.assertEqual(result["g1"]["OKX"]["n"], 1200) self.assertEqual(result["parts"], {"caught_up": True, "window_complete": True, "g1": "PASS", "g2": "PASS", "g2_feeds": {"TRADE": "PASS", "QUOTE": "PASS", - "MARK": "PASS"}, "g3": "PASS"}) + "MARK": "PASS"}, "g3": "PASS", + "monitor": "PASS"}) self.assertEqual(result["g2"]["TRADE"]["BINANCE"]["n"], 1200) self.assertEqual(result["g3"]["BAR_1m"]["OKX"]["n"], 5) # closes 120..360 s in [62.1, 362.1) s short = g12.evaluate_fault(fault, pubs, [v for v in views if v["t"] < 200 * S], monitor) @@ -1014,5 +1015,147 @@ def stderr_tail(self, limit): self.assertEqual((Path(base) / "evidence/logs/tswriter-okx.stderr.log").read_text(), "Traceback: boom\n") +def _crash_on_second(calls=[]): # noqa: B006 - per-process call counter of the forked child + def sampler(phase): + calls.append(1) + phase("list_offsets:kn-projector-v3-a") + if len(calls) == 2: + import os + import signal + os.kill(os.getpid(), signal.SIGSEGV) + return {"cluster": {"broker_ids": [1, 2, 3]}, "n": len(calls)} + return sampler + + +class MonitorSupervisorTests(unittest.TestCase): + """G12 run 2026-10-11 full2: the monitor (librdkafka admin calls) died with SIGSEGV (exit 139) one second + into a broker stop and the run stopped. A crash now costs one typed sample, never the monitor.""" + + def test_native_crash_is_a_typed_sample_error_and_the_next_sample_works(self): + supervisor = g12.SampleSupervisor(_crash_on_second, timeout_s=5) + try: + first, error = supervisor.sample() + self.assertEqual((first["n"], error), (1, None)) + fields, error = supervisor.sample() + self.assertIsNone(fields) + self.assertEqual((error["error"], error["phase"], error["exitcode"]), + ("SIGSEGV", "list_offsets:kn-projector-v3-a", -11)) + again, error = supervisor.sample() # a fresh child (its own counter) + self.assertEqual((again["n"], error, supervisor.restarts), (1, None, 1)) + finally: + supervisor.close() + + def test_hang_and_exception_are_bounded(self): + def hang(): + def sampler(phase): + phase("describe_consumer_groups") + import time as clock + clock.sleep(30) + return sampler + + def broken(): + raise RuntimeError("no TLS material") + supervisor = g12.SampleSupervisor(hang, timeout_s=0.5) + try: + started = g12.time.monotonic() + fields, error = supervisor.sample() + self.assertLess(g12.time.monotonic() - started, 5) + self.assertEqual((fields, error["error"], error["phase"]), (None, "TIMEOUT", "describe_consumer_groups")) + finally: + supervisor.close() + supervisor = g12.SampleSupervisor(broken, timeout_s=5) + try: + fields, error = supervisor.sample() + self.assertEqual((fields, error["error"]), (None, "EXCEPTION")) + self.assertIn("no TLS material", error["detail"]) + finally: + supervisor.close() + self.assertEqual([g12.exit_reason(c) for c in (-11, -9, 1, None)], + ["SIGSEGV", "SIGKILL", "EXIT_1", "EXIT_None"]) + + def test_monitor_writes_sample_error_rows_and_keeps_running(self): + import signal + import tempfile + from pathlib import Path + outcomes = [({"cluster": {"broker_ids": [1, 2, 3]}}, None), + (None, {"error": "SIGSEGV", "phase": "describe_cluster", "exitcode": -11}), + ({"cluster": {"broker_ids": [1, 2, 3]}}, None)] + + class Supervisor: + restarts = 1 + + def __init__(self, make_sampler, timeout_s): + self.timeout_s = timeout_s + + def sample(self): + return outcomes.pop(0) if outcomes else ({"cluster": {}}, None) + + def close(self): + pass + before = signal.getsignal(signal.SIGTERM) + try: + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(g12, "_packet", return_value={"namespace": NS}), \ + patch.object(g12, "SampleSupervisor", Supervisor), \ + patch.object(g12, "local_sample", return_value={"ports": {"8210": True}}): + out = Path(tmp) / "monitor.jsonl" + g12.monitor(tmp, out, seconds=0.35, period=0.1) + rows = g12.load_jsonl(out) + finally: + signal.signal(signal.SIGTERM, before) + self.assertEqual(rows[1]["k"], "sample_error") + self.assertEqual((rows[1]["error"], rows[1]["phase"], rows[1]["restarts"]), ("SIGSEGV", "describe_cluster", 1)) + self.assertNotIn("ports", rows[1]) # a crashed sample observes nothing + self.assertEqual(rows[2]["ports"], {"8210": True}) + self.assertEqual(rows[-1]["k"], "end") + + +class MonitorGapRuleTests(unittest.TestCase): + FAULT = {"k": "fault", "name": "stream_restart", "kind": "stream", "injected_ns": 60 * S, "restart_ns": 61 * S} + + def run_fault(self, monitor): + pubs, views = [], [] + for venue in g12.VENUES: + p, v = stream_rows(venue, 0, 500 * S) + pubs += p + views += v + p, v = md_rows(venue, 0, 500 * S) + pubs += p + views += v + return g12.evaluate_fault(self.FAULT, pubs, views, sorted(monitor, key=lambda r: r["t"])) + + @staticmethod + def sample(t_s, up): + return {"t": int(t_s * S), "ports": {"8210": up}, "projector_lag": 0, "under_replicated": 0} + + def test_crash_samples_are_excluded_and_a_short_hole_still_passes(self): + monitor = [self.sample(t, t >= 64) for t in range(0, 500) if not 62 <= t <= 63] + # Crashed samples in the hole: even carrying an "up" port they are never evidence. + monitor += [{"t": t * S, "k": "sample_error", "error": "SIGSEGV", "phase": "describe_cluster", + "ports": {"8210": True}} for t in (62, 63)] + result = self.run_fault(monitor) + self.assertEqual(result["fault_cleared_ns"], 64 * S) + self.assertEqual(result["monitor_max_gap_s"], 3.0) # 61 -> 64 s, restart to clearance + self.assertEqual(result["parts"]["monitor"], "PASS") + self.assertEqual([e["error"] for e in result["monitor_sample_errors"]], ["SIGSEGV", "SIGSEGV"]) + self.assertTrue(result["pass"]) + + def test_a_gap_over_five_seconds_at_clearance_is_not_measured(self): + monitor = [self.sample(t, t >= 62) for t in range(0, 500) if not 55 <= t <= 67] + monitor += [{"t": 61.5 * S, "k": "sample_error", "error": "SIGSEGV", "phase": "list_offsets:x", + "ports": {"8210": True}}] + result = self.run_fault(monitor) + self.assertEqual(result["fault_cleared_ns"], 68 * S) # the first valid sample after the hole + self.assertGreater(result["monitor_max_gap_s"], g12.MONITOR_MAX_GAP_S) + self.assertEqual((result["parts"]["monitor"], result["reason"]), ("NOT_MEASURED", "MONITOR_GAP")) + self.assertFalse(result["pass"]) + + def test_gap_helper_counts_edges(self): + rows = [{"t": 3 * S}, {"t": 4 * S, "k": "sample_error"}, {"t": 9 * S}] + self.assertEqual(g12.monitor_max_gap_s(rows, 0, 10 * S), 6.0) + self.assertEqual(g12.monitor_max_gap_s([], 0, 2 * S), 2.0) + self.assertEqual(g12.cleared_at("stream", [{"t": 5, "k": "sample_error", "ports": {"8210": True}}], 0), None) + + if __name__ == "__main__": unittest.main() From 4c779acdc6a04855f69616f075baae695cf83cb1 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 11 Oct 2026 09:20:04 +0000 Subject: [PATCH 141/147] feat(stream): batched Subscribe responses for lossless live feeds (SDK 2.3.0) Additive wire fields SubscribeRequest.max_records_per_response and SubscribeResponse.records; the gateway packs already-queued TRADE/BOOK records without waiting or crossing a control, and old clients see the unchanged single-record shape. Client gRPC cost per record drops ~13x at 64 per response. Phase E step 3. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 18 + contracts/proto/qdl/query/v2/query.proto | 14 + contracts/v2/kn-v220-contracts.md | 32 ++ generated/python/qdl/query/v2/query_pb2.py | 76 +-- generated/rust/qdl/query/v2/qdl.query.v2.rs | 16 + qdl_sdk/CHANGELOG.md | 19 + qdl_sdk/README.md | 8 +- qdl_sdk/__init__.py | 2 +- qdl_sdk/transport.py | 57 ++- rust/qdl-stream-gateway/src/service.rs | 305 ++++++++++-- rust/qdl-stream-gateway/src/subscription.rs | 50 ++ .../qdl-stream-gateway/tests/native_stream.rs | 452 +++++++++++++++++- scripts/build_qdl_sdk_release.py | 2 +- tests/test_phaseb_stable_release.py | 2 +- tests/test_qdl_sdk_batched_subscribe.py | 224 +++++++++ tests/test_qdl_sdk_release.py | 4 +- 16 files changed, 1171 insertions(+), 110 deletions(-) create mode 100644 tests/test_qdl_sdk_batched_subscribe.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index e365da1d..500a50ec 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72220,3 +72220,21 @@ no build, CI or subagent job running. - After the change, involuntary switches went 209 -> 160 per 60 s at similar load. G8 is re-measured in the next campaign. - Receipt: ~/.local/state/qdl-v2/redis-cpu-shares-20261011/. Rollback (failure only): --cpu-shares 0. +- Phase E step 3, batched Subscribe (additive, versioned): + - Wire: SubscribeRequest.max_records_per_response = 7 (0/absent/1 = one record per response, as before); + SubscribeResponse.records = 2 (repeated). Exactly one of record/records is set. + - Gateway: batches only lossless live feeds (TRADE, BOOK_SNAPSHOT, BOOK_DELTA) from records already queued. + - It never waits to fill a batch and never crosses an overflow, failure or close. + - Batch clamp 256 records / ~1 MiB. + - QUOTE/latest-state, BAR, controls and committed replay stay single. + - Per-record freshness, token and offset are kept; the merged budget charge is held until tonic takes the batch. + - SDK 2.3.0: GrpcStreamTransport(max_records_per_response=64, 1..256). At 1 the request bytes equal 2.2.x; the + SDK falls back on an older gateway. + - Tests: + - Rust: fmt, clippy, cargo test (lib 50, native_stream 41 incl. b01-b04); + - Python: 156 SDK/contract tests; new parity tests 6/6; + - buf lint and breaking against both frozen baselines pass. + - Bench, real gateway over loopback, 4274 TRADE: client gRPC 0.088-0.099 -> 0.0066-0.0076 ms/rec at 64 per + response. + - Old SDK 2.1.0 and 2.2.1 against the new gateway: unchanged shape (real gRPC). + - Status: tested locally; needs a gateway image and roll, then TS SDK 2.3.0 adoption. diff --git a/contracts/proto/qdl/query/v2/query.proto b/contracts/proto/qdl/query/v2/query.proto index d9df0b30..29a126df 100644 --- a/contracts/proto/qdl/query/v2/query.proto +++ b/contracts/proto/qdl/query/v2/query.proto @@ -112,6 +112,12 @@ message SubscribeRequest { DataRequirement requirement = 2; string cursor_token = 5; uint32 max_buffer_events = 6; + // Additive (SDK 2.3.0): the most stream records the client accepts in one + // SubscribeResponse. 0 (absent) and 1 mean one record per response, the + // pre-2.3.0 behavior. A server that knows the field may pack up to this many + // already-available event records into SubscribeResponse.records; it never + // waits to fill a batch and clamps the value to its own cap. + uint32 max_records_per_response = 7; } message ReplayRequest { @@ -148,7 +154,15 @@ message StreamRecord { } message SubscribeResponse { + // Exactly one of `record` or `records` is set. A response that carries one + // record (every control, and every record for a client that did not ask for + // batches) uses `record`, unchanged since 2.0. StreamRecord record = 1; + // Additive (SDK 2.3.0): two or more event records in delivery order, each + // with its own logical_offset and resume_token. Sent only to a client that + // set SubscribeRequest.max_records_per_response > 1; never contains a + // control. + repeated StreamRecord records = 2; } message ReplayResponse { diff --git a/contracts/v2/kn-v220-contracts.md b/contracts/v2/kn-v220-contracts.md index c64c7be6..1a73b6c5 100644 --- a/contracts/v2/kn-v220-contracts.md +++ b/contracts/v2/kn-v220-contracts.md @@ -251,6 +251,38 @@ away. The earlier Python stream's coalescing of BOOK_SNAPSHOT is not ported. payload/cursor semantics do not change; overload is typed and retryable, never silently truncated or reinterpreted as an empty successful page. +## 8. Batched Subscribe responses (additive, SDK 2.3.0) + +- `SubscribeRequest.max_records_per_response = 7` (uint32) and + `SubscribeResponse.records = 2` (repeated `StreamRecord`) are additive; no + field number is reused. 0/absent and 1 mean one record per response, the + 2.0-2.2 behavior, and such a client receives exactly the responses it did + before (`record` set, `records` empty). +- Exactly one of `record` or `records` is set. `records` carries two or more + event records in delivery order; a control (REPLAYING, LIVE, BACKPRESSURE) + is always a response of its own in `record`. Every record keeps its own + `logical_offset` and signed resume token, so ACK/resume per record is + unchanged. +- The gateway packs only records already queued for the subscription; it + never waits to fill a batch. It clamps the request to 256 records and stops + a batch at about 1 MiB of payload (a single larger record is still sent + alone). It batches lossless feeds only (TRADE, BOOK_SNAPSHOT, BOOK_DELTA); + latest-state and lifecycle-coalesced feeds keep one record per response so + they stay in the coalescing queue. Committed replay responses stay one + record each. +- A batch never crosses an overflow, failure or close of the subscription + queue: those are reported after the records gathered before them are handed + to the transport. Revocation is checked before every response, batched or + not, exactly as for a single record. A record that ages out while queued ends the current batch first, + so cursor movement and the aged-out accounting match one-record delivery. + A batch carries the merged budget charges of its records until the + transport takes it; a record that cannot top up its charge without waiting + ends the batch, and only then waits, holding nothing else. A slow client + gets BACKPRESSURE with the token before the batch's first record. +- Readers: 2.3.0+ read `records` when present and `record` otherwise, so they + work against older gateways (one record per response) unchanged. + `delivered` counters count records, not responses. + ### KN-5 Derived BAR Diagnostic Index diff --git a/generated/python/qdl/query/v2/query_pb2.py b/generated/python/qdl/query/v2/query_pb2.py index 04fc61ba..1530357d 100644 --- a/generated/python/qdl/query/v2/query_pb2.py +++ b/generated/python/qdl/query/v2/query_pb2.py @@ -25,7 +25,7 @@ from qdl.marketdata.v2 import market_data_pb2 as qdl_dot_marketdata_dot_v2_dot_market__data__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18qdl/query/v2/query.proto\x12\x0cqdl.query.v2\x1a#qdl/marketdata/v2/market_data.proto\"U\n\x0fWarmupTimeRange\x12\"\n\rstart_time_ns\x18\x01 \x01(\x03R\x0bstartTimeNs\x12\x1e\n\x0b\x65nd_time_ns\x18\x02 \x01(\x03R\tendTimeNs\"\x9a\x02\n\x13WarmupSpecification\x12\x14\n\x04rows\x18\x01 \x01(\rH\x00R\x04rows\x12>\n\ntime_range\x18\x02 \x01(\x0b\x32\x1d.qdl.query.v2.WarmupTimeRangeH\x00R\ttimeRange\x12X\n\x16interval_source_policy\x18\x03 \x01(\x0e\x32\".qdl.query.v2.IntervalSourcePolicyR\x14intervalSourcePolicy\x12\'\n\x10max_cache_age_ms\x18\x04 \x01(\rR\rmaxCacheAgeMs\x12\x1f\n\x0b\x64\x65\x61\x64line_ms\x18\x05 \x01(\rR\ndeadlineMsB\t\n\x07horizon\"\xce\x08\n\x0f\x44\x61taRequirement\x12%\n\x0einstrument_uid\x18\x01 \x01(\tR\rinstrumentUid\x12\x16\n\x04\x66\x65\x65\x64\x18\x02 \x01(\tB\x02\x18\x01R\x04\x66\x65\x65\x64\x12\x1a\n\x08interval\x18\x03 \x01(\tR\x08interval\x12)\n\x0e\x63onsumer_grade\x18\x04 \x01(\tB\x02\x18\x01R\rconsumerGrade\x12(\n\x10source_policy_id\x18\x05 \x01(\tR\x0esourcePolicyId\x12!\n\x0cwarmup_limit\x18\x06 \x01(\rR\x0bwarmupLimit\x12(\n\x10max_freshness_ms\x18\x07 \x01(\x04R\x0emaxFreshnessMs\x12\x32\n\x15require_full_coverage\x18\x08 \x01(\x08R\x13requireFullCoverage\x12,\n\x12require_final_bars\x18\t \x01(\x08R\x10requireFinalBars\x12%\n\x0cstale_policy\x18\n \x01(\tB\x02\x18\x01R\x0bstalePolicy\x12!\n\ngap_policy\x18\x0b \x01(\tB\x02\x18\x01R\tgapPolicy\x12\x1e\n\x08recovery\x18\x0c \x01(\tB\x02\x18\x01R\x08recovery\x12\x32\n\x13\x62\x61r_revision_policy\x18\r \x01(\tB\x02\x18\x01R\x11\x62\x61rRevisionPolicy\x12\x33\n\tfeed_type\x18\x14 \x01(\x0e\x32\x16.qdl.query.v2.FeedTypeR\x08\x66\x65\x65\x64Type\x12\x31\n\x05grade\x18\x15 \x01(\x0e\x32\x1b.qdl.query.v2.ConsumerGradeR\x05grade\x12\x45\n\x11stale_policy_type\x18\x16 \x01(\x0e\x32\x19.qdl.query.v2.StalePolicyR\x0fstalePolicyType\x12?\n\x0fgap_policy_type\x18\x17 \x01(\x0e\x32\x17.qdl.query.v2.GapPolicyR\rgapPolicyType\x12\x45\n\x0frecovery_policy\x18\x18 \x01(\x0e\x32\x1c.qdl.query.v2.RecoveryPolicyR\x0erecoveryPolicy\x12H\n\x0frevision_policy\x18\x19 \x01(\x0e\x32\x1f.qdl.query.v2.BarRevisionPolicyR\x0erevisionPolicy\x12\x39\n\x06warmup\x18\x1a \x01(\x0b\x32!.qdl.query.v2.WarmupSpecificationR\x06warmup\x12K\n\x14\x65vent_recency_policy\x18\x1b \x01(\x0e\x32\x19.qdl.query.v2.StalePolicyR\x12\x65ventRecencyPolicy\x12\x35\n\x17max_session_liveness_ms\x18\x1c \x01(\x04R\x14maxSessionLivenessMs\"\xe6\x01\n\x10SubscribeRequest\x12\x1f\n\x0b\x63onsumer_id\x18\x01 \x01(\tR\nconsumerId\x12?\n\x0brequirement\x18\x02 \x01(\x0b\x32\x1d.qdl.query.v2.DataRequirementR\x0brequirement\x12!\n\x0c\x63ursor_token\x18\x05 \x01(\tR\x0b\x63ursorToken\x12*\n\x11max_buffer_events\x18\x06 \x01(\rR\x0fmaxBufferEventsJ\x04\x08\x03\x10\x04J\x04\x08\x04\x10\x05R\x06streamR\rpartition_key\"\x8c\x01\n\rReplayRequest\x12\x1f\n\x0b\x63onsumer_id\x18\x01 \x01(\tR\nconsumerId\x12!\n\x0c\x63ursor_token\x18\x04 \x01(\tR\x0b\x63ursorToken\x12\x14\n\x05limit\x18\x05 \x01(\rR\x05limitJ\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04R\x06streamR\rpartition_key\"\x9a\x01\n\rStreamControl\x12\x36\n\x05state\x18\x01 \x01(\x0e\x32 .qdl.query.v2.StreamControlStateR\x05state\x12\x12\n\x04\x63ode\x18\x02 \x01(\tR\x04\x63ode\x12\x16\n\x06\x64\x65tail\x18\x03 \x01(\tR\x06\x64\x65tail\x12%\n\x0ehigh_watermark\x18\x04 \x01(\x04R\rhighWatermark\"\xd6\x01\n\x0cStreamRecord\x12%\n\x0elogical_offset\x18\x01 \x01(\x04R\rlogicalOffset\x12!\n\x0cresume_token\x18\x02 \x01(\tR\x0bresumeToken\x12\x38\n\x05\x65vent\x18\n \x01(\x0b\x32 .qdl.marketdata.v2.EventEnvelopeH\x00R\x05\x65vent\x12\x37\n\x07\x63ontrol\x18\x0b \x01(\x0b\x32\x1b.qdl.query.v2.StreamControlH\x00R\x07\x63ontrolB\t\n\x07payload\"G\n\x11SubscribeResponse\x12\x32\n\x06record\x18\x01 \x01(\x0b\x32\x1a.qdl.query.v2.StreamRecordR\x06record\"D\n\x0eReplayResponse\x12\x32\n\x06record\x18\x01 \x01(\x0b\x32\x1a.qdl.query.v2.StreamRecordR\x06record\"v\n\x12GetSnapshotRequest\x12\x1f\n\x0b\x63onsumer_id\x18\x01 \x01(\tR\nconsumerId\x12?\n\x0brequirement\x18\x02 \x01(\x0b\x32\x1d.qdl.query.v2.DataRequirementR\x0brequirement\"\x82\x02\n\x13GetSnapshotResponse\x12\x1d\n\nrequest_id\x18\x01 \x01(\tR\trequestId\x12\x1f\n\x0bsnapshot_id\x18\x02 \x01(\tR\nsnapshotId\x12#\n\rstream_cursor\x18\x03 \x01(\tR\x0cstreamCursor\x12!\n\rdata_as_of_ns\x18\x04 \x01(\x03R\ndataAsOfNs\x12)\n\x10watermark_offset\x18\x05 \x01(\x04R\x0fwatermarkOffset\x12\x38\n\x06\x65vents\x18\x06 \x03(\x0b\x32 .qdl.marketdata.v2.EventEnvelopeR\x06\x65vents\"x\n\x14GetFeedStatusRequest\x12?\n\x0brequirement\x18\x01 \x01(\x0b\x32\x1d.qdl.query.v2.DataRequirementR\x0brequirement\x12\x1f\n\x0b\x63onsumer_id\x18\x02 \x01(\tR\nconsumerId\"\x90\x03\n\x15GetFeedStatusResponse\x12\x14\n\x05state\x18\x01 \x01(\tR\x05state\x12!\n\x0c\x66reshness_ms\x18\x02 \x01(\x04R\x0b\x66reshnessMs\x12\x19\n\x08gap_open\x18\x03 \x01(\x08R\x07gapOpen\x12\x1a\n\x08\x63omplete\x18\x04 \x01(\x08R\x08\x63omplete\x12-\n\x12\x65xecution_eligible\x18\x05 \x01(\x08R\x11\x65xecutionEligible\x12\x1b\n\tpolicy_id\x18\x06 \x01(\tR\x08policyId\x12\x14\n\x05\x66lags\x18\x07 \x03(\tR\x05\x66lags\x12.\n\x13\x65vent_recency_state\x18\x08 \x01(\tR\x11\x65ventRecencyState\x12\x34\n\x16provider_session_state\x18\t \x01(\tR\x14providerSessionState\x12?\n\x1cprovider_session_liveness_ms\x18\n \x01(\x04R\x19providerSessionLivenessMs*\xf8\x02\n\x08\x46\x65\x65\x64Type\x12\x19\n\x15\x46\x45\x45\x44_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0f\x46\x45\x45\x44_TYPE_TRADE\x10\x01\x12\x13\n\x0f\x46\x45\x45\x44_TYPE_QUOTE\x10\x02\x12\x11\n\rFEED_TYPE_BAR\x10\x03\x12\x1b\n\x17\x46\x45\x45\x44_TYPE_BOOK_SNAPSHOT\x10\x04\x12\x18\n\x14\x46\x45\x45\x44_TYPE_BOOK_DELTA\x10\x05\x12\x1a\n\x16\x46\x45\x45\x44_TYPE_FUNDING_RATE\x10\x06\x12\x1b\n\x17\x46\x45\x45\x44_TYPE_OPEN_INTEREST\x10\x07\x12\x1e\n\x1a\x46\x45\x45\x44_TYPE_MARK_INDEX_PRICE\x10\x08\x12\x14\n\x10\x46\x45\x45\x44_TYPE_TICKER\x10\t\x12\x1e\n\x1a\x46\x45\x45\x44_TYPE_LONG_SHORT_RATIO\x10\n\x12\x18\n\x14\x46\x45\x45\x44_TYPE_TAKER_FLOW\x10\x0b\x12\x13\n\x0f\x46\x45\x45\x44_TYPE_BASIS\x10\x0c\x12\x1f\n\x1b\x46\x45\x45\x44_TYPE_CONTRACT_METADATA\x10\r*\x84\x01\n\rConsumerGrade\x12\x1e\n\x1a\x43ONSUMER_GRADE_UNSPECIFIED\x10\x00\x12\x1c\n\x18\x43ONSUMER_GRADE_EXECUTION\x10\x01\x12\x18\n\x14\x43ONSUMER_GRADE_ALPHA\x10\x02\x12\x1b\n\x17\x43ONSUMER_GRADE_RESEARCH\x10\x03*u\n\x0bStalePolicy\x12\x1c\n\x18STALE_POLICY_UNSPECIFIED\x10\x00\x12\x16\n\x12STALE_POLICY_BLOCK\x10\x01\x12\x16\n\x12STALE_POLICY_PAUSE\x10\x02\x12\x18\n\x14STALE_POLICY_OBSERVE\x10\x03*k\n\tGapPolicy\x12\x1a\n\x16GAP_POLICY_UNSPECIFIED\x10\x00\x12\x14\n\x10GAP_POLICY_BLOCK\x10\x01\x12\x14\n\x10GAP_POLICY_PAUSE\x10\x02\x12\x16\n\x12GAP_POLICY_OBSERVE\x10\x03*\x98\x01\n\x0eRecoveryPolicy\x12\x1f\n\x1bRECOVERY_POLICY_UNSPECIFIED\x10\x00\x12\'\n#RECOVERY_POLICY_SNAPSHOT_AND_REPLAY\x10\x01\x12\"\n\x1eRECOVERY_POLICY_FRESH_SNAPSHOT\x10\x02\x12\x18\n\x14RECOVERY_POLICY_NONE\x10\x03*\xa6\x01\n\x11\x42\x61rRevisionPolicy\x12#\n\x1f\x42\x41R_REVISION_POLICY_UNSPECIFIED\x10\x00\x12\x1e\n\x1a\x42\x41R_REVISION_POLICY_LATEST\x10\x01\x12$\n BAR_REVISION_POLICY_INITIAL_ONLY\x10\x02\x12&\n\"BAR_REVISION_POLICY_EMIT_REVISIONS\x10\x03*\x9b\x01\n\x14IntervalSourcePolicy\x12&\n\"INTERVAL_SOURCE_POLICY_UNSPECIFIED\x10\x00\x12&\n\"INTERVAL_SOURCE_POLICY_NATIVE_ONLY\x10\x01\x12\x33\n/INTERVAL_SOURCE_POLICY_NATIVE_OR_EXACT_RESAMPLE\x10\x02*\xd0\x01\n\x12StreamControlState\x12$\n STREAM_CONTROL_STATE_UNSPECIFIED\x10\x00\x12\"\n\x1eSTREAM_CONTROL_STATE_REPLAYING\x10\x01\x12\x1d\n\x19STREAM_CONTROL_STATE_LIVE\x10\x02\x12%\n!STREAM_CONTROL_STATE_BACKPRESSURE\x10\x03\x12*\n&STREAM_CONTROL_STATE_RECOVERY_REQUIRED\x10\x04\x32\xde\x02\n\x17MarketDataStreamService\x12N\n\tSubscribe\x12\x1e.qdl.query.v2.SubscribeRequest\x1a\x1f.qdl.query.v2.SubscribeResponse0\x01\x12\x45\n\x06Replay\x12\x1b.qdl.query.v2.ReplayRequest\x1a\x1c.qdl.query.v2.ReplayResponse0\x01\x12R\n\x0bGetSnapshot\x12 .qdl.query.v2.GetSnapshotRequest\x1a!.qdl.query.v2.GetSnapshotResponse\x12X\n\rGetFeedStatus\x12\".qdl.query.v2.GetFeedStatusRequest\x1a#.qdl.query.v2.GetFeedStatusResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18qdl/query/v2/query.proto\x12\x0cqdl.query.v2\x1a#qdl/marketdata/v2/market_data.proto\"U\n\x0fWarmupTimeRange\x12\"\n\rstart_time_ns\x18\x01 \x01(\x03R\x0bstartTimeNs\x12\x1e\n\x0b\x65nd_time_ns\x18\x02 \x01(\x03R\tendTimeNs\"\x9a\x02\n\x13WarmupSpecification\x12\x14\n\x04rows\x18\x01 \x01(\rH\x00R\x04rows\x12>\n\ntime_range\x18\x02 \x01(\x0b\x32\x1d.qdl.query.v2.WarmupTimeRangeH\x00R\ttimeRange\x12X\n\x16interval_source_policy\x18\x03 \x01(\x0e\x32\".qdl.query.v2.IntervalSourcePolicyR\x14intervalSourcePolicy\x12\'\n\x10max_cache_age_ms\x18\x04 \x01(\rR\rmaxCacheAgeMs\x12\x1f\n\x0b\x64\x65\x61\x64line_ms\x18\x05 \x01(\rR\ndeadlineMsB\t\n\x07horizon\"\xce\x08\n\x0f\x44\x61taRequirement\x12%\n\x0einstrument_uid\x18\x01 \x01(\tR\rinstrumentUid\x12\x16\n\x04\x66\x65\x65\x64\x18\x02 \x01(\tB\x02\x18\x01R\x04\x66\x65\x65\x64\x12\x1a\n\x08interval\x18\x03 \x01(\tR\x08interval\x12)\n\x0e\x63onsumer_grade\x18\x04 \x01(\tB\x02\x18\x01R\rconsumerGrade\x12(\n\x10source_policy_id\x18\x05 \x01(\tR\x0esourcePolicyId\x12!\n\x0cwarmup_limit\x18\x06 \x01(\rR\x0bwarmupLimit\x12(\n\x10max_freshness_ms\x18\x07 \x01(\x04R\x0emaxFreshnessMs\x12\x32\n\x15require_full_coverage\x18\x08 \x01(\x08R\x13requireFullCoverage\x12,\n\x12require_final_bars\x18\t \x01(\x08R\x10requireFinalBars\x12%\n\x0cstale_policy\x18\n \x01(\tB\x02\x18\x01R\x0bstalePolicy\x12!\n\ngap_policy\x18\x0b \x01(\tB\x02\x18\x01R\tgapPolicy\x12\x1e\n\x08recovery\x18\x0c \x01(\tB\x02\x18\x01R\x08recovery\x12\x32\n\x13\x62\x61r_revision_policy\x18\r \x01(\tB\x02\x18\x01R\x11\x62\x61rRevisionPolicy\x12\x33\n\tfeed_type\x18\x14 \x01(\x0e\x32\x16.qdl.query.v2.FeedTypeR\x08\x66\x65\x65\x64Type\x12\x31\n\x05grade\x18\x15 \x01(\x0e\x32\x1b.qdl.query.v2.ConsumerGradeR\x05grade\x12\x45\n\x11stale_policy_type\x18\x16 \x01(\x0e\x32\x19.qdl.query.v2.StalePolicyR\x0fstalePolicyType\x12?\n\x0fgap_policy_type\x18\x17 \x01(\x0e\x32\x17.qdl.query.v2.GapPolicyR\rgapPolicyType\x12\x45\n\x0frecovery_policy\x18\x18 \x01(\x0e\x32\x1c.qdl.query.v2.RecoveryPolicyR\x0erecoveryPolicy\x12H\n\x0frevision_policy\x18\x19 \x01(\x0e\x32\x1f.qdl.query.v2.BarRevisionPolicyR\x0erevisionPolicy\x12\x39\n\x06warmup\x18\x1a \x01(\x0b\x32!.qdl.query.v2.WarmupSpecificationR\x06warmup\x12K\n\x14\x65vent_recency_policy\x18\x1b \x01(\x0e\x32\x19.qdl.query.v2.StalePolicyR\x12\x65ventRecencyPolicy\x12\x35\n\x17max_session_liveness_ms\x18\x1c \x01(\x04R\x14maxSessionLivenessMs\"\x9f\x02\n\x10SubscribeRequest\x12\x1f\n\x0b\x63onsumer_id\x18\x01 \x01(\tR\nconsumerId\x12?\n\x0brequirement\x18\x02 \x01(\x0b\x32\x1d.qdl.query.v2.DataRequirementR\x0brequirement\x12!\n\x0c\x63ursor_token\x18\x05 \x01(\tR\x0b\x63ursorToken\x12*\n\x11max_buffer_events\x18\x06 \x01(\rR\x0fmaxBufferEvents\x12\x37\n\x18max_records_per_response\x18\x07 \x01(\rR\x15maxRecordsPerResponseJ\x04\x08\x03\x10\x04J\x04\x08\x04\x10\x05R\x06streamR\rpartition_key\"\x8c\x01\n\rReplayRequest\x12\x1f\n\x0b\x63onsumer_id\x18\x01 \x01(\tR\nconsumerId\x12!\n\x0c\x63ursor_token\x18\x04 \x01(\tR\x0b\x63ursorToken\x12\x14\n\x05limit\x18\x05 \x01(\rR\x05limitJ\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04R\x06streamR\rpartition_key\"\x9a\x01\n\rStreamControl\x12\x36\n\x05state\x18\x01 \x01(\x0e\x32 .qdl.query.v2.StreamControlStateR\x05state\x12\x12\n\x04\x63ode\x18\x02 \x01(\tR\x04\x63ode\x12\x16\n\x06\x64\x65tail\x18\x03 \x01(\tR\x06\x64\x65tail\x12%\n\x0ehigh_watermark\x18\x04 \x01(\x04R\rhighWatermark\"\xd6\x01\n\x0cStreamRecord\x12%\n\x0elogical_offset\x18\x01 \x01(\x04R\rlogicalOffset\x12!\n\x0cresume_token\x18\x02 \x01(\tR\x0bresumeToken\x12\x38\n\x05\x65vent\x18\n \x01(\x0b\x32 .qdl.marketdata.v2.EventEnvelopeH\x00R\x05\x65vent\x12\x37\n\x07\x63ontrol\x18\x0b \x01(\x0b\x32\x1b.qdl.query.v2.StreamControlH\x00R\x07\x63ontrolB\t\n\x07payload\"}\n\x11SubscribeResponse\x12\x32\n\x06record\x18\x01 \x01(\x0b\x32\x1a.qdl.query.v2.StreamRecordR\x06record\x12\x34\n\x07records\x18\x02 \x03(\x0b\x32\x1a.qdl.query.v2.StreamRecordR\x07records\"D\n\x0eReplayResponse\x12\x32\n\x06record\x18\x01 \x01(\x0b\x32\x1a.qdl.query.v2.StreamRecordR\x06record\"v\n\x12GetSnapshotRequest\x12\x1f\n\x0b\x63onsumer_id\x18\x01 \x01(\tR\nconsumerId\x12?\n\x0brequirement\x18\x02 \x01(\x0b\x32\x1d.qdl.query.v2.DataRequirementR\x0brequirement\"\x82\x02\n\x13GetSnapshotResponse\x12\x1d\n\nrequest_id\x18\x01 \x01(\tR\trequestId\x12\x1f\n\x0bsnapshot_id\x18\x02 \x01(\tR\nsnapshotId\x12#\n\rstream_cursor\x18\x03 \x01(\tR\x0cstreamCursor\x12!\n\rdata_as_of_ns\x18\x04 \x01(\x03R\ndataAsOfNs\x12)\n\x10watermark_offset\x18\x05 \x01(\x04R\x0fwatermarkOffset\x12\x38\n\x06\x65vents\x18\x06 \x03(\x0b\x32 .qdl.marketdata.v2.EventEnvelopeR\x06\x65vents\"x\n\x14GetFeedStatusRequest\x12?\n\x0brequirement\x18\x01 \x01(\x0b\x32\x1d.qdl.query.v2.DataRequirementR\x0brequirement\x12\x1f\n\x0b\x63onsumer_id\x18\x02 \x01(\tR\nconsumerId\"\x90\x03\n\x15GetFeedStatusResponse\x12\x14\n\x05state\x18\x01 \x01(\tR\x05state\x12!\n\x0c\x66reshness_ms\x18\x02 \x01(\x04R\x0b\x66reshnessMs\x12\x19\n\x08gap_open\x18\x03 \x01(\x08R\x07gapOpen\x12\x1a\n\x08\x63omplete\x18\x04 \x01(\x08R\x08\x63omplete\x12-\n\x12\x65xecution_eligible\x18\x05 \x01(\x08R\x11\x65xecutionEligible\x12\x1b\n\tpolicy_id\x18\x06 \x01(\tR\x08policyId\x12\x14\n\x05\x66lags\x18\x07 \x03(\tR\x05\x66lags\x12.\n\x13\x65vent_recency_state\x18\x08 \x01(\tR\x11\x65ventRecencyState\x12\x34\n\x16provider_session_state\x18\t \x01(\tR\x14providerSessionState\x12?\n\x1cprovider_session_liveness_ms\x18\n \x01(\x04R\x19providerSessionLivenessMs*\xf8\x02\n\x08\x46\x65\x65\x64Type\x12\x19\n\x15\x46\x45\x45\x44_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0f\x46\x45\x45\x44_TYPE_TRADE\x10\x01\x12\x13\n\x0f\x46\x45\x45\x44_TYPE_QUOTE\x10\x02\x12\x11\n\rFEED_TYPE_BAR\x10\x03\x12\x1b\n\x17\x46\x45\x45\x44_TYPE_BOOK_SNAPSHOT\x10\x04\x12\x18\n\x14\x46\x45\x45\x44_TYPE_BOOK_DELTA\x10\x05\x12\x1a\n\x16\x46\x45\x45\x44_TYPE_FUNDING_RATE\x10\x06\x12\x1b\n\x17\x46\x45\x45\x44_TYPE_OPEN_INTEREST\x10\x07\x12\x1e\n\x1a\x46\x45\x45\x44_TYPE_MARK_INDEX_PRICE\x10\x08\x12\x14\n\x10\x46\x45\x45\x44_TYPE_TICKER\x10\t\x12\x1e\n\x1a\x46\x45\x45\x44_TYPE_LONG_SHORT_RATIO\x10\n\x12\x18\n\x14\x46\x45\x45\x44_TYPE_TAKER_FLOW\x10\x0b\x12\x13\n\x0f\x46\x45\x45\x44_TYPE_BASIS\x10\x0c\x12\x1f\n\x1b\x46\x45\x45\x44_TYPE_CONTRACT_METADATA\x10\r*\x84\x01\n\rConsumerGrade\x12\x1e\n\x1a\x43ONSUMER_GRADE_UNSPECIFIED\x10\x00\x12\x1c\n\x18\x43ONSUMER_GRADE_EXECUTION\x10\x01\x12\x18\n\x14\x43ONSUMER_GRADE_ALPHA\x10\x02\x12\x1b\n\x17\x43ONSUMER_GRADE_RESEARCH\x10\x03*u\n\x0bStalePolicy\x12\x1c\n\x18STALE_POLICY_UNSPECIFIED\x10\x00\x12\x16\n\x12STALE_POLICY_BLOCK\x10\x01\x12\x16\n\x12STALE_POLICY_PAUSE\x10\x02\x12\x18\n\x14STALE_POLICY_OBSERVE\x10\x03*k\n\tGapPolicy\x12\x1a\n\x16GAP_POLICY_UNSPECIFIED\x10\x00\x12\x14\n\x10GAP_POLICY_BLOCK\x10\x01\x12\x14\n\x10GAP_POLICY_PAUSE\x10\x02\x12\x16\n\x12GAP_POLICY_OBSERVE\x10\x03*\x98\x01\n\x0eRecoveryPolicy\x12\x1f\n\x1bRECOVERY_POLICY_UNSPECIFIED\x10\x00\x12\'\n#RECOVERY_POLICY_SNAPSHOT_AND_REPLAY\x10\x01\x12\"\n\x1eRECOVERY_POLICY_FRESH_SNAPSHOT\x10\x02\x12\x18\n\x14RECOVERY_POLICY_NONE\x10\x03*\xa6\x01\n\x11\x42\x61rRevisionPolicy\x12#\n\x1f\x42\x41R_REVISION_POLICY_UNSPECIFIED\x10\x00\x12\x1e\n\x1a\x42\x41R_REVISION_POLICY_LATEST\x10\x01\x12$\n BAR_REVISION_POLICY_INITIAL_ONLY\x10\x02\x12&\n\"BAR_REVISION_POLICY_EMIT_REVISIONS\x10\x03*\x9b\x01\n\x14IntervalSourcePolicy\x12&\n\"INTERVAL_SOURCE_POLICY_UNSPECIFIED\x10\x00\x12&\n\"INTERVAL_SOURCE_POLICY_NATIVE_ONLY\x10\x01\x12\x33\n/INTERVAL_SOURCE_POLICY_NATIVE_OR_EXACT_RESAMPLE\x10\x02*\xd0\x01\n\x12StreamControlState\x12$\n STREAM_CONTROL_STATE_UNSPECIFIED\x10\x00\x12\"\n\x1eSTREAM_CONTROL_STATE_REPLAYING\x10\x01\x12\x1d\n\x19STREAM_CONTROL_STATE_LIVE\x10\x02\x12%\n!STREAM_CONTROL_STATE_BACKPRESSURE\x10\x03\x12*\n&STREAM_CONTROL_STATE_RECOVERY_REQUIRED\x10\x04\x32\xde\x02\n\x17MarketDataStreamService\x12N\n\tSubscribe\x12\x1e.qdl.query.v2.SubscribeRequest\x1a\x1f.qdl.query.v2.SubscribeResponse0\x01\x12\x45\n\x06Replay\x12\x1b.qdl.query.v2.ReplayRequest\x1a\x1c.qdl.query.v2.ReplayResponse0\x01\x12R\n\x0bGetSnapshot\x12 .qdl.query.v2.GetSnapshotRequest\x1a!.qdl.query.v2.GetSnapshotResponse\x12X\n\rGetFeedStatus\x12\".qdl.query.v2.GetFeedStatusRequest\x1a#.qdl.query.v2.GetFeedStatusResponseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -44,22 +44,22 @@ _globals['_DATAREQUIREMENT'].fields_by_name['recovery']._serialized_options = b'\030\001' _globals['_DATAREQUIREMENT'].fields_by_name['bar_revision_policy']._loaded_options = None _globals['_DATAREQUIREMENT'].fields_by_name['bar_revision_policy']._serialized_options = b'\030\001' - _globals['_FEEDTYPE']._serialized_start=3356 - _globals['_FEEDTYPE']._serialized_end=3732 - _globals['_CONSUMERGRADE']._serialized_start=3735 - _globals['_CONSUMERGRADE']._serialized_end=3867 - _globals['_STALEPOLICY']._serialized_start=3869 - _globals['_STALEPOLICY']._serialized_end=3986 - _globals['_GAPPOLICY']._serialized_start=3988 - _globals['_GAPPOLICY']._serialized_end=4095 - _globals['_RECOVERYPOLICY']._serialized_start=4098 - _globals['_RECOVERYPOLICY']._serialized_end=4250 - _globals['_BARREVISIONPOLICY']._serialized_start=4253 - _globals['_BARREVISIONPOLICY']._serialized_end=4419 - _globals['_INTERVALSOURCEPOLICY']._serialized_start=4422 - _globals['_INTERVALSOURCEPOLICY']._serialized_end=4577 - _globals['_STREAMCONTROLSTATE']._serialized_start=4580 - _globals['_STREAMCONTROLSTATE']._serialized_end=4788 + _globals['_FEEDTYPE']._serialized_start=3467 + _globals['_FEEDTYPE']._serialized_end=3843 + _globals['_CONSUMERGRADE']._serialized_start=3846 + _globals['_CONSUMERGRADE']._serialized_end=3978 + _globals['_STALEPOLICY']._serialized_start=3980 + _globals['_STALEPOLICY']._serialized_end=4097 + _globals['_GAPPOLICY']._serialized_start=4099 + _globals['_GAPPOLICY']._serialized_end=4206 + _globals['_RECOVERYPOLICY']._serialized_start=4209 + _globals['_RECOVERYPOLICY']._serialized_end=4361 + _globals['_BARREVISIONPOLICY']._serialized_start=4364 + _globals['_BARREVISIONPOLICY']._serialized_end=4530 + _globals['_INTERVALSOURCEPOLICY']._serialized_start=4533 + _globals['_INTERVALSOURCEPOLICY']._serialized_end=4688 + _globals['_STREAMCONTROLSTATE']._serialized_start=4691 + _globals['_STREAMCONTROLSTATE']._serialized_end=4899 _globals['_WARMUPTIMERANGE']._serialized_start=79 _globals['_WARMUPTIMERANGE']._serialized_end=164 _globals['_WARMUPSPECIFICATION']._serialized_start=167 @@ -67,25 +67,25 @@ _globals['_DATAREQUIREMENT']._serialized_start=452 _globals['_DATAREQUIREMENT']._serialized_end=1554 _globals['_SUBSCRIBEREQUEST']._serialized_start=1557 - _globals['_SUBSCRIBEREQUEST']._serialized_end=1787 - _globals['_REPLAYREQUEST']._serialized_start=1790 - _globals['_REPLAYREQUEST']._serialized_end=1930 - _globals['_STREAMCONTROL']._serialized_start=1933 - _globals['_STREAMCONTROL']._serialized_end=2087 - _globals['_STREAMRECORD']._serialized_start=2090 - _globals['_STREAMRECORD']._serialized_end=2304 - _globals['_SUBSCRIBERESPONSE']._serialized_start=2306 - _globals['_SUBSCRIBERESPONSE']._serialized_end=2377 - _globals['_REPLAYRESPONSE']._serialized_start=2379 - _globals['_REPLAYRESPONSE']._serialized_end=2447 - _globals['_GETSNAPSHOTREQUEST']._serialized_start=2449 - _globals['_GETSNAPSHOTREQUEST']._serialized_end=2567 - _globals['_GETSNAPSHOTRESPONSE']._serialized_start=2570 - _globals['_GETSNAPSHOTRESPONSE']._serialized_end=2828 - _globals['_GETFEEDSTATUSREQUEST']._serialized_start=2830 - _globals['_GETFEEDSTATUSREQUEST']._serialized_end=2950 - _globals['_GETFEEDSTATUSRESPONSE']._serialized_start=2953 - _globals['_GETFEEDSTATUSRESPONSE']._serialized_end=3353 - _globals['_MARKETDATASTREAMSERVICE']._serialized_start=4791 - _globals['_MARKETDATASTREAMSERVICE']._serialized_end=5141 + _globals['_SUBSCRIBEREQUEST']._serialized_end=1844 + _globals['_REPLAYREQUEST']._serialized_start=1847 + _globals['_REPLAYREQUEST']._serialized_end=1987 + _globals['_STREAMCONTROL']._serialized_start=1990 + _globals['_STREAMCONTROL']._serialized_end=2144 + _globals['_STREAMRECORD']._serialized_start=2147 + _globals['_STREAMRECORD']._serialized_end=2361 + _globals['_SUBSCRIBERESPONSE']._serialized_start=2363 + _globals['_SUBSCRIBERESPONSE']._serialized_end=2488 + _globals['_REPLAYRESPONSE']._serialized_start=2490 + _globals['_REPLAYRESPONSE']._serialized_end=2558 + _globals['_GETSNAPSHOTREQUEST']._serialized_start=2560 + _globals['_GETSNAPSHOTREQUEST']._serialized_end=2678 + _globals['_GETSNAPSHOTRESPONSE']._serialized_start=2681 + _globals['_GETSNAPSHOTRESPONSE']._serialized_end=2939 + _globals['_GETFEEDSTATUSREQUEST']._serialized_start=2941 + _globals['_GETFEEDSTATUSREQUEST']._serialized_end=3061 + _globals['_GETFEEDSTATUSRESPONSE']._serialized_start=3064 + _globals['_GETFEEDSTATUSRESPONSE']._serialized_end=3464 + _globals['_MARKETDATASTREAMSERVICE']._serialized_start=4902 + _globals['_MARKETDATASTREAMSERVICE']._serialized_end=5252 # @@protoc_insertion_point(module_scope) diff --git a/generated/rust/qdl/query/v2/qdl.query.v2.rs b/generated/rust/qdl/query/v2/qdl.query.v2.rs index b9b64d02..51a2fd62 100644 --- a/generated/rust/qdl/query/v2/qdl.query.v2.rs +++ b/generated/rust/qdl/query/v2/qdl.query.v2.rs @@ -93,6 +93,13 @@ pub struct SubscribeRequest { pub cursor_token: ::prost::alloc::string::String, #[prost(uint32, tag="6")] pub max_buffer_events: u32, + /// Additive (SDK 2.3.0): the most stream records the client accepts in one + /// SubscribeResponse. 0 (absent) and 1 mean one record per response, the + /// pre-2.3.0 behavior. A server that knows the field may pack up to this many + /// already-available event records into SubscribeResponse.records; it never + /// waits to fill a batch and clamps the value to its own cap. + #[prost(uint32, tag="7")] + pub max_records_per_response: u32, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReplayRequest { @@ -135,8 +142,17 @@ pub mod stream_record { } #[derive(Clone, PartialEq, ::prost::Message)] pub struct SubscribeResponse { + /// Exactly one of `record` or `records` is set. A response that carries one + /// record (every control, and every record for a client that did not ask for + /// batches) uses `record`, unchanged since 2.0. #[prost(message, optional, tag="1")] pub record: ::core::option::Option, + /// Additive (SDK 2.3.0): two or more event records in delivery order, each + /// with its own logical_offset and resume_token. Sent only to a client that + /// set SubscribeRequest.max_records_per_response > 1; never contains a + /// control. + #[prost(message, repeated, tag="2")] + pub records: ::prost::alloc::vec::Vec, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct ReplayResponse { diff --git a/qdl_sdk/CHANGELOG.md b/qdl_sdk/CHANGELOG.md index 6a611bf8..9f8591c3 100644 --- a/qdl_sdk/CHANGELOG.md +++ b/qdl_sdk/CHANGELOG.md @@ -1,5 +1,24 @@ # SDK Changelog +## 2.3.0 (batched Subscribe) + +`GrpcStreamTransport` asks the stream gateway for up to +`max_records_per_response` records per `SubscribeResponse` (new keyword, +default 64, 1..256; 1 sends the pre-2.3.0 request without the field). The +protobuf contract gains `SubscribeRequest.max_records_per_response = 7` and +`SubscribeResponse.records = 2` (repeated `StreamRecord`); both are additive. +A gateway that knows the field packs only records it already has queued for +a lossless feed (TRADE, BOOK_SNAPSHOT, BOOK_DELTA), never waits to fill a +batch, never puts a control in a batch and clamps the value to 256 records or +1 MiB; latest-state and lifecycle-coalesced feeds keep one record per response. +The transport reads `records` when present and `record` otherwise, so callers +get the identical sequence of per-record `StreamEvent`/`ControlEvent` objects, +each with its own logical offset and resume token, from a batching gateway, a +gateway without the field (one record per response) or with batching off. +`StreamTransport.subscribe`, `WarmupStreamSession` and acknowledgement are +unchanged. 2.1.x/2.2.x clients never set the field and are served exactly as +before by a 2.3.0 gateway. + ## 2.2.1 (v2.2.3 Data Layer) `GrpcStreamTransport` starts on a random target per instance instead of always diff --git a/qdl_sdk/README.md b/qdl_sdk/README.md index d41d81ff..a25d9312 100644 --- a/qdl_sdk/README.md +++ b/qdl_sdk/README.md @@ -64,6 +64,12 @@ async with client.warmup_then_stream( handle_control(item) ``` +`GrpcStreamTransport(..., max_records_per_response=64)` (SDK 2.3.0) lets a +gateway that supports it carry up to that many already-queued records in one +gRPC message. The session still yields one event per record with its own +resume token; against an older gateway, or with `max_records_per_response=1`, +every message carries one record as before. + ## Universal BAR warmup and closed-bar handoff BAR consumers declare either an exact row count or an exact half-open time @@ -146,7 +152,7 @@ Build the standalone artifact with: python scripts/build_qdl_sdk_release.py --output-dir dist/qdl-sdk ``` -The output contains a reproducible `qdl_sdk-2.2.1-py3-none-any.whl`, a release +The output contains a reproducible `qdl_sdk-2.3.0-py3-none-any.whl`, a release manifest with the wheel/source/generated-contract SHA-256 digests, and a CycloneDX SBOM. The wheel contains only the public SDK plus generated Protobuf contracts; it does not package `qdl.api_v2`, runtime adapters, provider code or diff --git a/qdl_sdk/__init__.py b/qdl_sdk/__init__.py index 9a5303eb..370da3ee 100644 --- a/qdl_sdk/__init__.py +++ b/qdl_sdk/__init__.py @@ -1,6 +1,6 @@ """Stable Python SDK V2 with typed public models and generated transports.""" -__version__ = "2.2.1" +__version__ = "2.3.0" from qdl_sdk.client import AsyncDataLayerClient, DataLayerClientV2, WarmupStreamSession from qdl_sdk.credentials import ( diff --git a/qdl_sdk/transport.py b/qdl_sdk/transport.py index 7a1b8873..561e317b 100644 --- a/qdl_sdk/transport.py +++ b/qdl_sdk/transport.py @@ -463,7 +463,18 @@ def _candidate_indexes(self) -> tuple[int, ...]: return selected[: self._max_attempts] +# Records the SDK accepts in one SubscribeResponse (SDK 2.3.0). A gateway that +# knows `SubscribeRequest.max_records_per_response` packs up to this many +# already-available lossless records into `SubscribeResponse.records`; an older +# gateway ignores the field and keeps sending one `record` per response. +DEFAULT_MAX_RECORDS_PER_RESPONSE = 64 +MAX_RECORDS_PER_RESPONSE = 256 + + class GrpcStreamTransport: + # Class default for instances built without __init__ (test doubles). + max_records_per_response: int = DEFAULT_MAX_RECORDS_PER_RESPONSE + def __init__( self, target: str | Sequence[str], @@ -473,7 +484,18 @@ def __init__( allow_insecure_loopback: bool = False, credential_provider: CredentialProvider | None = None, start_index: int | None = None, + max_records_per_response: int = DEFAULT_MAX_RECORDS_PER_RESPONSE, ) -> None: + if ( + type(max_records_per_response) is not int + or not 1 <= max_records_per_response <= MAX_RECORDS_PER_RESPONSE + ): + raise ValueError( + "max_records_per_response must be an int between 1 and " + f"{MAX_RECORDS_PER_RESPONSE}" + ) + # 1 sends the pre-2.3.0 request (field absent): one record per response. + self.max_records_per_response = max_records_per_response raw_targets = (target,) if isinstance(target, str) else tuple(target) targets = tuple( value.strip() @@ -531,6 +553,8 @@ async def subscribe( cursor_token=cursor_token, max_buffer_events=max_buffer_events, ) + if self.max_records_per_response > 1: + request.max_records_per_response = self.max_records_per_response if self._credential_provider is None: raise DataLayerError( "UNAUTHENTICATED", @@ -555,21 +579,24 @@ async def subscribe( call = subscribe(request, metadata=metadata) async for response in call: responses_seen = True - record = response.record - payload = record.WhichOneof("payload") - if payload == "control": - yield ControlEvent( - record.control.code, - record.control.detail, - {"high_watermark": record.control.high_watermark}, - ) - continue - if payload == "event": - yield StreamEvent( - record.logical_offset, - record.resume_token, - record.event, - ) + # A batched response carries `records` (2.3.0 gateway); + # every other response, and any older gateway, `record`. + # Callers see the same per-record events either way. + records = response.records or (response.record,) + for record in records: + payload = record.WhichOneof("payload") + if payload == "control": + yield ControlEvent( + record.control.code, + record.control.detail, + {"high_watermark": record.control.high_watermark}, + ) + elif payload == "event": + yield StreamEvent( + record.logical_offset, + record.resume_token, + record.event, + ) return except grpc.aio.AioRpcError as error: detail = error.details() or "gRPC stream failed" diff --git a/rust/qdl-stream-gateway/src/service.rs b/rust/qdl-stream-gateway/src/service.rs index c6dff231..bbce69bf 100644 --- a/rust/qdl-stream-gateway/src/service.rs +++ b/rust/qdl-stream-gateway/src/service.rs @@ -46,6 +46,29 @@ pub const OUTBOUND_HANDOFF: usize = 2; /// How often a send waiting for live budget to copy a record retries. const HANDOFF_BUDGET_RETRY: Duration = Duration::from_millis(5); +/// Most event records one batched `SubscribeResponse` carries, whatever the +/// client asks for (`SubscribeRequest.max_records_per_response`, SDK 2.3.0). +pub const MAX_RECORDS_PER_RESPONSE: usize = 256; + +/// A batch stops growing once its records reach this many encoded bytes, far +/// below the 4 MiB default receive limit of gRPC clients. A single record is +/// always sent whatever its size, as before batching. +pub const MAX_BATCH_BYTES: usize = 1 << 20; + +/// Records one response may carry for this request: 1 (the pre-2.3.0 shape) +/// unless the client asked for more and the feed is lossless. Latest-state +/// and lifecycle-coalesced feeds keep one record per response, so records +/// stay in the coalescing queue (DL-V2 R1.8) instead of the handoff. +pub fn response_batch_limit(requested: u32, feed: &str) -> usize { + let lossless = qdl_contracts::delivery::delivery_policy(feed, None) + == Ok(qdl_contracts::delivery::DeliveryPolicy::Lossless); + if lossless { + (requested as usize).clamp(1, MAX_RECORDS_PER_RESPONSE) + } else { + 1 + } +} + #[derive(Clone, Debug)] pub struct StreamLimits { pub cursor_ttl_seconds: u64, @@ -294,6 +317,7 @@ fn subscribe_control( ) -> query::SubscribeResponse { query::SubscribeResponse { record: Some(control(state as i32, code, detail, high, token)), + records: Vec::new(), } } @@ -484,6 +508,10 @@ impl MarketDataStreamService for Gateway { Entitlement::Requirement(Box::new(requirement.clone())), generation, ); + let batch_limit = response_batch_limit( + body.max_records_per_response, + &subscription.requirement.delivery.feed, + ); let task = SubscriptionTask { report: Report { outcome: Mutex::new("ended"), @@ -501,6 +529,7 @@ impl MarketDataStreamService for Gateway { partition, key, after, + batch_limit, }; tokio::spawn(task.run(registration.barrier, ring)); Ok(Response::new(Box::pin( @@ -957,6 +986,59 @@ struct SubscriptionTask { partition: i32, key: Vec, after: i64, + /// Records per live response (1 = one record per response, pre-2.3.0). + batch_limit: usize, +} + +/// Live event records gathered for one response, in delivery order, with +/// their budget charges merged: the charge travels with the response until +/// tonic takes it, exactly as a single record's does. +#[derive(Default)] +struct Batch { + records: Vec, + charge: Option, + bytes: usize, + /// Offset of the first and of the last record. + first: i64, + last: i64, + /// When the first record was taken from the queue. + started: Option, +} + +impl Batch { + fn push(&mut self, record: query::StreamRecord, offset: i64, bytes: usize, charge: Charge) { + if self.records.is_empty() { + self.first = offset; + } + self.last = offset; + self.bytes += bytes; + self.records.push(record); + match self.charge.as_mut() { + Some(held) => held.absorb(charge), + None => self.charge = Some(charge), + } + } + + /// One record goes in `record` (the shape every client reads), two or + /// more in `records` (only clients that asked for batches get these). + fn take_response(&mut self) -> (query::SubscribeResponse, Option, usize) { + let mut records = std::mem::take(&mut self.records); + let count = records.len(); + let response = if count == 1 { + query::SubscribeResponse { + record: records.pop(), + records: Vec::new(), + } + } else { + query::SubscribeResponse { + record: None, + records, + } + }; + self.bytes = 0; + self.started = None; + (response, self.charge.take(), count) + } } enum Sent { @@ -1099,6 +1181,7 @@ impl SubscriptionTask { }; let response = query::SubscribeResponse { record: Some(event_record(record, token)), + records: Vec::new(), }; match self.send(response, charge).await { Sent::Ok => Some(true), @@ -1154,24 +1237,176 @@ impl SubscriptionTask { } } - /// [`Self::deliver`] for a live record, recording its send-path waits. - async fn deliver_timed( + /// [`Self::handoff_charge`] without waiting: a record joining a batch + /// that already holds charges never waits for budget (no wait while + /// holding). `Err` hands the entry charge back. + fn try_handoff_charge( &self, record: &LiveRecord, - enqueued: Instant, - charge: Charge, - ) -> Option { - let taken = Instant::now(); - let charge = self.handoff_charge(record, charge).await?; - let result = self.deliver(record, Some(charge)).await; - let metrics = &self.state.metrics; - metrics - .queue_wait - .record(taken.duration_since(enqueued).as_millis() as i64); - metrics - .handoff_wait - .record(taken.elapsed().as_millis() as i64); - result + mut charge: Charge, + ) -> Result { + let missing = record.weight().saturating_sub(charge.bytes()); + let fresh = delivery_decision( + &self.subscription.requirement, + &record.envelope, + now_ns() as i64, + ) == Delivery::Deliver; + if missing == 0 || !fresh { + return Ok(charge); + } + match self.state.budget.charge(missing) { + Some(extra) => { + charge.absorb(extra); + Ok(charge) + } + None => Err(charge), + } + } + + /// Send the gathered records as one response. `false` ends the stream + /// (it already failed typed); a slow client gets BACKPRESSURE with the + /// token before the batch's first record, as for a single record. + async fn flush(&self, batch: &mut Batch, last: &mut i64) -> bool { + if batch.records.is_empty() { + return true; + } + let (first, newest, started) = (batch.first, batch.last, batch.started); + let (response, charge, count) = batch.take_response(); + let sent = self.send(response, charge).await; + if let Some(started) = started { + self.state + .metrics + .handoff_wait + .record(started.elapsed().as_millis() as i64); + } + match sent { + Sent::Ok => { + *last = newest; + let count = count as u64; + self.state + .metrics + .delivered + .fetch_add(count, Ordering::Relaxed); + self.report.delivered.fetch_add(count, Ordering::Relaxed); + true + } + Sent::Closed => false, + Sent::Slow => { + self.backpressure( + first - 1, + "slow consumer exceeded its stream buffer; replay is required", + ) + .await; + false + } + } + } + + /// Deliver a dequeued live record and, when the client asked for + /// batches, the records already queued behind it (never waiting for more) + /// in one response, in queue order. Every record keeps its own offset, + /// resume token, freshness check and charge; overflow, failure and close + /// are never crossed ([`Subscription::try_next`]), and controls are sent + /// alone elsewhere. `false` ends the stream (it already failed typed). + async fn deliver_live( + &self, + first: (Arc, Instant, Charge), + last: &mut i64, + filtered_since_delivery: &mut usize, + ) -> bool { + let mut batch = Batch::default(); + let mut item = Some(first); + while let Some((record, enqueued, charge)) = item.take() { + let taken = Instant::now(); + self.state + .metrics + .queue_wait + .record(taken.duration_since(enqueued).as_millis() as i64); + let charge = if batch.records.is_empty() { + match self.handoff_charge(&record, charge).await { + Some(charge) => charge, + None => return false, + } + } else { + match self.try_handoff_charge(&record, charge) { + Ok(charge) => charge, + Err(charge) => { + // Budget is short: send what is gathered, then wait + // for this record's top-up holding nothing else. + if !self.flush(&mut batch, last).await { + return false; + } + match self.handoff_charge(&record, charge).await { + Some(charge) => charge, + None => return false, + } + } + } + }; + match delivery_decision( + &self.subscription.requirement, + &record.envelope, + now_ns() as i64, + ) { + Delivery::Deliver => { + let token = match self.state.sign(&self.claims, record.raw.offset) { + Ok(token) => token, + Err(status) => { + self.fail(status).await; + return false; + } + }; + let bytes = record.raw.payload.len() + token.len() + 32; + let full = !batch.records.is_empty() && batch.bytes + bytes > MAX_BATCH_BYTES; + if full && !self.flush(&mut batch, last).await { + return false; + } + batch.started.get_or_insert(taken); + batch.push( + event_record(&record, token), + record.raw.offset, + bytes, + charge, + ); + *filtered_since_delivery = 0; + } + Delivery::OtherProduct | Delivery::TooOld => { + drop(charge); + // Delivered records ahead of it go first, so the cursor + // and the aged-out accounting match one-record delivery. + if !self.flush(&mut batch, last).await { + return false; + } + self.state + .metrics + .handoff_wait + .record(taken.elapsed().as_millis() as i64); + // Aged out while queued: the cursor moves past it, and + // a whole buffer of such records without a delivery + // is a slow consumer (DL-V2 R1.13). + *last = record.raw.offset; + *filtered_since_delivery += 1; + self.state + .metrics + .aged_out_at_read + .fetch_add(1, Ordering::Relaxed); + self.report.aged_out.fetch_add(1, Ordering::Relaxed); + if *filtered_since_delivery > self.subscription.depth { + self.backpressure( + *last, + "records aged out of the bounded buffer before delivery; \ + replay from the last confirmed token is required", + ) + .await; + return false; + } + } + } + if batch.records.len() < self.batch_limit && self.batch_limit > 1 { + item = self.subscription.try_next(); + } + } + self.flush(&mut batch, last).await } /// Replay one record; `false` ends the stream (it already failed typed). @@ -1343,35 +1578,15 @@ impl SubscriptionTask { }; match next { Next::Record(record, enqueued, charge) => { - match self.deliver_timed(&record, enqueued, charge).await { - Some(true) => { - filtered_since_delivery = 0; - last = record.raw.offset; - self.state.metrics.delivered.fetch_add(1, Ordering::Relaxed); - self.report.delivered.fetch_add(1, Ordering::Relaxed); - } - Some(false) => { - // Aged out while queued: the cursor moves past it, and - // a whole buffer of such records without a delivery - // is a slow consumer (DL-V2 R1.13). - last = record.raw.offset; - filtered_since_delivery += 1; - self.state - .metrics - .aged_out_at_read - .fetch_add(1, Ordering::Relaxed); - self.report.aged_out.fetch_add(1, Ordering::Relaxed); - if filtered_since_delivery > self.subscription.depth { - self.backpressure( - last, - "records aged out of the bounded buffer before delivery; \ - replay from the last confirmed token is required", - ) - .await; - return; - } - } - None => return, + if !self + .deliver_live( + (record, enqueued, charge), + &mut last, + &mut filtered_since_delivery, + ) + .await + { + return; } } Next::Overflow => { diff --git a/rust/qdl-stream-gateway/src/subscription.rs b/rust/qdl-stream-gateway/src/subscription.rs index 00fba0fe..9433ea83 100644 --- a/rust/qdl-stream-gateway/src/subscription.rs +++ b/rust/qdl-stream-gateway/src/subscription.rs @@ -333,6 +333,21 @@ impl Subscription { } } + /// The next queued record only if one is already available: never waits + /// (batched delivery packs what is queued, it does not wait to fill). + /// `None` also when the stream must end; [`Self::next`] then reports the + /// overflow, failure or close, so a batch never crosses that boundary. + pub fn try_next(&self) -> Option<(Arc, Instant, Charge)> { + let mut state = self.state.lock().ok()?; + if state.failed.is_some() || state.overflowed || state.closed { + return None; + } + let item = state.queue.pop_front()?; + state.bytes -= item.weight; + self.counters.dequeued.fetch_add(1, Ordering::Relaxed); + Some((item.record, item.enqueued, item.charge)) + } + pub fn queued(&self) -> (usize, usize) { self.state .lock() @@ -590,6 +605,41 @@ mod tests { assert_eq!(budget.used(), 0); } + #[tokio::test] + async fn try_next_takes_only_queued_records_and_never_crosses_an_end() { + let trade = |offset| { + record( + offset, + event_envelope::Payload::Trade(Trade::default()), + vec![], + ) + }; + let subscription = queue(query::FeedType::Trade, 8); + assert!(subscription.try_next().is_none(), "never waits"); + for offset in 1..=3 { + subscription.offer(&trade(offset), 0); + } + let (first, _, charge) = subscription.try_next().expect("queued"); + assert_eq!(first.raw.offset, 1); + assert_eq!(counts(&subscription), (3, 1)); + assert_eq!(subscription.queued().0, 2); + drop(charge); + // A failure ends batching even with records queued; next() reports it. + subscription.fail("test-only failure"); + assert!(subscription.try_next().is_none()); + assert!(matches!(subscription.next().await, Next::Failed(_))); + subscription.close(); + assert_eq!(subscription.budget.used(), 0); + + let overflow = queue(query::FeedType::Trade, 1); + overflow.offer(&trade(1), 0); + overflow.offer(&trade(2), 0); // Lossless: overflows, nothing dropped. + assert!(overflow.try_next().is_none()); + assert!(matches!(overflow.next().await, Next::Overflow)); + overflow.close(); + assert_eq!(overflow.budget.used(), 0); + } + #[tokio::test] async fn records_at_or_below_the_cursor_are_never_queued() { let subscription = Subscription::new( diff --git a/rust/qdl-stream-gateway/tests/native_stream.rs b/rust/qdl-stream-gateway/tests/native_stream.rs index 095b05e1..96d90795 100644 --- a/rust/qdl-stream-gateway/tests/native_stream.rs +++ b/rust/qdl-stream-gateway/tests/native_stream.rs @@ -127,18 +127,19 @@ impl Log { /// lock: a reader past the last record has also passed trailing markers. fn end_if_caught_up(&self, partition: i32, position: i64) -> Option { let inner = self.0.lock().unwrap(); + // Records are appended in offset order: only the last one can be pending. let pending = inner.records[&partition] - .iter() - .any(|record| record.offset >= position); + .last() + .is_some_and(|record| record.offset >= position); (!pending).then(|| inner.next[&partition]) } fn first_at_or_after(&self, partition: i32, from: i64) -> Option { let inner = self.0.lock().unwrap(); - inner.records[&partition] - .iter() - .find(|record| record.offset >= from) - .cloned() + // Offset order: a binary search keeps long benchmark logs linear. + let records = &inner.records[&partition]; + let index = records.partition_point(|record| record.offset < from); + records.get(index).cloned() } } @@ -697,6 +698,20 @@ impl Harness { interval: Option<&str>, after: i64, buffer: u32, + ) -> Result { + self.subscribe_batched(feed, interval, after, buffer, 0) + .await + } + + /// Subscribe asking for up to `batch` records per response (SDK 2.3.0; + /// 0 is a pre-2.3.0 client that does not know the field). + async fn subscribe_batched( + &self, + feed: &str, + interval: Option<&str>, + after: i64, + buffer: u32, + batch: u32, ) -> Result { let partition = 0; let request = self.authorized(query::SubscribeRequest { @@ -704,6 +719,7 @@ impl Harness { requirement: Some(requirement(feed, interval)), cursor_token: self.cursor(feed, interval, partition, after), max_buffer_events: buffer, + max_records_per_response: batch, }); let response = self.gateway.subscribe(request).await?; Ok(Stream(response.into_inner())) @@ -914,6 +930,7 @@ async fn t01_offset_zero_is_snapshot_state_not_a_genesis_or_skipped_event() { requirement: Some(requirement("TRADE", None)), cursor_token: snapshot.stream_cursor, max_buffer_events: 100, + max_records_per_response: 0, })) .await .unwrap(); @@ -1796,6 +1813,7 @@ async fn t07_every_rpc_refuses_missing_or_mismatched_identity() { requirement: Some(requirement("TRADE", None)), cursor_token: cursor.clone(), max_buffer_events: 10, + max_records_per_response: 0, }; let replay = query::ReplayRequest { consumer_id: CONSUMER.into(), @@ -2896,3 +2914,425 @@ async fn hot_read_real_grpc_framing_keeps_internal_auth_and_public_rpcs_separate stop.send(()).unwrap(); server.await.unwrap().unwrap(); } + +// ------------------------------------------- batched Subscribe (SDK 2.3.0) +// +// `SubscribeRequest.max_records_per_response` > 1 lets one response carry +// the records already queued, in `SubscribeResponse.records`. These tests pin +// that a client sees the same per-record sequence either way, that a client +// without the field is served exactly as before, that controls are never +// batched, and that budget and slots return to baseline. + +/// What one stream showed, response by response. +#[derive(Default)] +struct Shapes { + /// Records carried by each event response (1 for the single `record`). + sizes: Vec, + /// Responses that used `records` (the batched shape). + batched: usize, + events: Vec<(u64, String, EventEnvelope)>, + controls: Vec, + status: Option<(Code, String)>, +} + +impl Shapes { + fn offsets(&self) -> Vec { + self.events.iter().map(|(offset, _, _)| *offset).collect() + } +} + +/// Read raw responses until `count` events arrived or the stream ended. +async fn read_shapes(stream: &mut Stream, count: usize, shapes: &mut Shapes) { + while shapes.events.len() < count && shapes.status.is_none() { + let response = match tokio::time::timeout(Duration::from_secs(5), stream.0.next()).await { + Err(_) | Ok(None) => break, + Ok(Some(Err(status))) => { + shapes.status = Some((status.code(), status.message().to_owned())); + break; + } + Ok(Some(Ok(response))) => response, + }; + if response.records.is_empty() { + let record = response.record.expect("a response carries a record"); + match record.payload.expect("payload") { + query::stream_record::Payload::Control(control) => { + shapes.controls.push(control.code) + } + query::stream_record::Payload::Event(event) => { + shapes.sizes.push(1); + shapes + .events + .push((record.logical_offset, record.resume_token, event)); + } + } + continue; + } + assert!( + response.record.is_none(), + "exactly one of record or records is set" + ); + assert!( + response.records.len() >= 2, + "a batch has two or more records" + ); + shapes.batched += 1; + shapes.sizes.push(response.records.len()); + for record in response.records { + match record.payload.expect("payload") { + query::stream_record::Payload::Control(control) => { + panic!("a control was batched: {}", control.code) + } + query::stream_record::Payload::Event(event) => { + shapes + .events + .push((record.logical_offset, record.resume_token, event)); + } + } + } + } +} + +impl Harness { + /// Every resume token names exactly its own record. + fn assert_tokens(&self, feed: &str, shapes: &Shapes) { + let digest = DeliveryRequirement::from_proto(&requirement(feed, None)) + .unwrap() + .digest(); + for (offset, token, _) in &shapes.events { + let claims = self + .codec + .verify( + token, + CONSUMER, + "paper", + &digest, + &expectation(), + now_ns() as u64, + ) + .expect("a valid resume token"); + assert_eq!(claims.source_offset, *offset, "token of offset {offset}"); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn b01_batched_delivery_equals_one_record_delivery_and_old_clients_are_unchanged() { + let harness = Harness::new(Options::default()); + let cursor = harness.log.append(0, TRADE_KEY, &trade(1)); + harness.settle(); + // 0: a pre-2.3.0 client; 1: asks for one record; 64: batched. + let mut streams = Vec::new(); + for batch in [0u32, 1, 64] { + let mut stream = harness + .subscribe_batched("TRADE", None, cursor, 1_000, batch) + .await + .unwrap(); + assert!(stream.until_live().await.is_empty()); + streams.push((batch, stream)); + } + // A burst while nobody reads: it queues behind each stream's handoff. + for index in 0..300u64 { + harness.log.append(0, TRADE_KEY, &trade(10 + index)); + } + harness.settle(); + tokio::time::sleep(Duration::from_millis(100)).await; + let expected = as_offsets(&harness.log.records_of(TRADE_KEY, cursor)); + assert_eq!(expected.len(), 300); + let mut seen = Vec::new(); + for (batch, stream) in streams.iter_mut() { + let mut shapes = Shapes::default(); + read_shapes(stream, expected.len(), &mut shapes).await; + assert!(shapes.status.is_none(), "{:?}", shapes.status); + assert_eq!( + shapes.offsets(), + expected, + "batch {batch}: every record once, in order" + ); + harness.assert_tokens("TRADE", &shapes); + if *batch <= 1 { + assert_eq!(shapes.batched, 0, "batch {batch}: the pre-2.3.0 shape only"); + assert!(shapes.sizes.iter().all(|&size| size == 1)); + } else { + assert!(shapes.batched > 0, "queued records were packed"); + assert!( + shapes.sizes.iter().all(|&size| size <= 64), + "{:?}", + shapes.sizes + ); + } + seen.push(shapes); + } + for shapes in &seen[1..] { + let left: Vec<_> = seen[0].events.iter().map(|(o, _, e)| (o, e)).collect(); + let right: Vec<_> = shapes.events.iter().map(|(o, _, e)| (o, e)).collect(); + assert_eq!(left, right, "same offsets and envelopes whatever the batch"); + } + // One record available: a batching client gets the single `record` shape + // at once (never waits to fill a batch). + let next = harness.log.append(0, TRADE_KEY, &trade(999)) as u64; + for (batch, stream) in streams.iter_mut() { + let started = Instant::now(); + let mut shapes = Shapes::default(); + read_shapes(stream, 1, &mut shapes).await; + assert_eq!(shapes.offsets(), vec![next], "batch {batch}"); + assert_eq!(shapes.batched, 0); + assert!(started.elapsed() < Duration::from_secs(1)); + } + let metrics = &harness.state().metrics; + let deadline = Instant::now() + Duration::from_secs(2); + while metrics.delivered.load(Ordering::Relaxed) != 3 * 301 && Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!( + metrics.delivered.load(Ordering::Relaxed), + 3 * 301, + "delivered counts records, not responses" + ); + drop(streams); + harness.baseline().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn b02_latest_state_feeds_and_controls_are_never_batched() { + let harness = Harness::new(Options::default()); + let cursor = harness.log.append(0, QUOTE_KEY, "e(0, vec![])); + harness.settle(); + let mut stream = harness + .subscribe_batched("QUOTE", None, cursor, 1_000, 64) + .await + .unwrap(); + // REPLAYING and LIVE are single controls (`Stream::next` reads `record`). + stream.until_live().await; + let mut last = 0; + for index in 1..=40u32 { + let flags = if index >= 20 { vec![9] } else { vec![] }; + last = harness.log.append(0, QUOTE_KEY, "e(index, flags)); + } + harness.settle(); + tokio::time::sleep(Duration::from_millis(100)).await; + let mut shapes = Shapes::default(); + while shapes.offsets().last() != Some(&(last as u64)) { + let before = shapes.events.len(); + read_shapes(&mut stream, before + 1, &mut shapes).await; + assert!(shapes.status.is_none(), "{:?}", shapes.status); + assert!( + shapes.events.len() > before, + "ended after {:?}", + shapes.offsets() + ); + } + assert_eq!( + shapes.batched, 0, + "a latest-state feed keeps one record per response" + ); + assert!(shapes.controls.is_empty()); + // Coalescing still works: depth 8 + the 2-response handoff + one in flight. + assert!(shapes.events.len() <= 11, "{:?}", shapes.offsets()); + drop(stream); + harness.baseline().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn b03_a_slow_batched_reader_ends_typed_resumes_exactly_and_releases_budget() { + let harness = Harness::new(Options::default()); + let cursor = harness.log.append(0, TRADE_KEY, &trade(1)); + harness.settle(); + let mut stream = harness + .subscribe_batched("TRADE", None, cursor, 10, 64) + .await + .unwrap(); + assert!(stream.until_live().await.is_empty()); + // The client stops reading; lossless records overflow its bounded queue. + for index in 0..200u64 { + harness.log.append(0, TRADE_KEY, &trade(10 + index)); + } + harness.settle(); + let expected = as_offsets(&harness.log.records_of(TRADE_KEY, cursor)); + let mut shapes = Shapes::default(); + read_shapes(&mut stream, usize::MAX, &mut shapes).await; + let (code, message) = shapes.status.clone().expect("a typed terminal status"); + assert_eq!(code, Code::ResourceExhausted); + assert!(message.starts_with("RATE_LIMITED:"), "{message}"); + assert_eq!( + shapes.controls, + vec!["RATE_LIMITED"], + "BACKPRESSURE is a single control before the status" + ); + assert!( + expected.starts_with(&shapes.offsets()), + "delivered prefix in order" + ); + harness.assert_tokens("TRADE", &shapes); + drop(stream); + harness.released().await; + // Resume from the last delivered record's token: the rest, exactly. + let after = shapes + .offsets() + .last() + .map_or(cursor, |&offset| offset as i64); + let mut resumed = harness + .subscribe_batched("TRADE", None, after, 1_000, 64) + .await + .unwrap(); + let rest = resumed.until_live().await; + assert_eq!(rest, expected[shapes.events.len()..]); + drop(resumed); + harness.baseline().await; + assert!(harness.state().metrics.overflow.load(Ordering::Relaxed) >= 1); +} + +#[test] +fn b04_the_batch_limit_is_clamped_and_lossless_only() { + use qdl_stream_gateway::service::{response_batch_limit, MAX_RECORDS_PER_RESPONSE}; + assert_eq!(response_batch_limit(0, "TRADE"), 1); + assert_eq!(response_batch_limit(1, "TRADE"), 1); + assert_eq!(response_batch_limit(64, "TRADE"), 64); + assert_eq!(response_batch_limit(64, "BOOK_DELTA"), 64); + assert_eq!(response_batch_limit(64, "BOOK_SNAPSHOT"), 64); + assert_eq!( + response_batch_limit(u32::MAX, "TRADE"), + MAX_RECORDS_PER_RESPONSE + ); + for feed in ["QUOTE", "BAR", "MARK_INDEX_PRICE", "TICKER", "FUNDING_RATE"] { + assert_eq!(response_batch_limit(64, feed), 1, "{feed}"); + } +} + +/// Process CPU (user + system) in milliseconds, from `/proc/self/stat` +/// (clock ticks of 10 ms at the usual 100 Hz). +fn process_cpu_ms() -> f64 { + let stat = std::fs::read_to_string("/proc/self/stat").unwrap(); + let fields: Vec<&str> = stat.rsplit_once(") ").unwrap().1.split(' ').collect(); + let ticks: u64 = fields[11].parse::().unwrap() + fields[12].parse::().unwrap(); + ticks as f64 * 10.0 +} + +/// Benchmark server, not a contract test: serves this harness's native +/// Subscribe over real gRPC (tonic, loopback 127.0.0.1:18320, no TLS) and +/// feeds `QDL_BENCH_RECORDS` live TRADE records (the phase-2 golden +/// binance-usdm envelope, re-timed) per round to one client at a time, +/// keeping its queue busy but below the 1000-event buffer. Per round it +/// writes `round-.json` (cursor, bearer, requirement) for the client and +/// `server-.json` (gateway process CPU per record) to `QDL_BENCH_DIR`. +/// Run: `cargo test -p qdl-stream-gateway --test native_stream -- --ignored +/// bench_serve_live_trade_bursts_over_grpc`, with the client in the same +/// network namespace. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "benchmark server; run explicitly with a client"] +async fn bench_serve_live_trade_bursts_over_grpc() { + use qdl_stream_gateway::generated::query_v2::market_data_stream_service_server::MarketDataStreamServiceServer; + let dir = std::path::PathBuf::from(std::env::var("QDL_BENCH_DIR").expect("QDL_BENCH_DIR")); + let env = |name: &str, default: u64| { + std::env::var(name) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) + }; + let records = env("QDL_BENCH_RECORDS", 4_274); + let rounds = env("QDL_BENCH_ROUNDS", 6); + let harness = Harness::new(Options::default()); + let golden = EventEnvelope::decode( + &include_bytes!("../../../contracts/golden/phase2/binance-usdm-trade.bin")[..], + ) + .unwrap(); + let event = |id: u64| { + let mut envelope = golden.clone(); + envelope.instrument_uid = UID.into(); + let now = now_ns(); + envelope.source_event_time_ns = now; + envelope.received_at_ns = now; + envelope.normalized_at_ns = now; + envelope.published_at_ns = now; + envelope.partition_sequence = id; + if let Some(event_envelope::Payload::Trade(trade)) = envelope.payload.as_mut() { + trade.native_trade_id = id.to_string(); + } + envelope + }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:18320") + .await + .unwrap(); + let service = MarketDataStreamServiceServer::new(Gateway { + state: harness.state().clone(), + }); + let (stop, stopped) = tokio::sync::oneshot::channel::<()>(); + let server = tokio::spawn( + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + async { + let _ = stopped.await; + }, + ), + ); + let requirement_hex = + requirement("TRADE", None) + .encode_to_vec() + .iter() + .fold(String::new(), |mut hex, byte| { + use std::fmt::Write as _; + let _ = write!(hex, "{byte:02x}"); + hex + }); + let metrics = &harness.state().metrics; + async fn wait(what: &str, ready: &dyn Fn() -> bool) { + let deadline = Instant::now() + Duration::from_secs(120); + while !ready() { + assert!(Instant::now() < deadline, "timed out waiting for {what}"); + tokio::time::sleep(Duration::from_millis(1)).await; + } + } + let mut sequence = 1_000u64; + for round in 0..rounds { + let after = harness.log.end(0) - 1; + let bearer = bearer( + &harness.keys, + "k1", + &["stream_consumer", "market_data_reader", "historical_reader"], + 3, + ); + let ready = serde_json::json!({ + "round": round, + "target": "127.0.0.1:18320", + "bearer": bearer.trim_start_matches("Bearer "), + "cursor": harness.cursor("TRADE", None, 0, after), + "consumer_id": CONSUMER, + "requirement_hex": requirement_hex, + "records": records, + }); + std::fs::write(dir.join(format!("round-{round}.json")), ready.to_string()).unwrap(); + wait("the client stream", &|| harness.state().open_streams() == 1).await; + tokio::time::sleep(Duration::from_millis(300)).await; // REPLAYING, LIVE. + let delivered = || metrics.delivered.load(Ordering::Relaxed); + let start = delivered(); + let (cpu, started) = (process_cpu_ms(), Instant::now()); + let mut appended = 0u64; + while appended < records { + if appended - (delivered() - start) < 700 { + for _ in 0..256.min(records - appended) { + sequence += 1; + harness.log.append(0, TRADE_KEY, &event(sequence)); + appended += 1; + } + } else { + tokio::time::sleep(Duration::from_millis(1)).await; + } + } + wait("delivery", &|| delivered() - start >= records).await; + let (cpu, wall) = (process_cpu_ms() - cpu, started.elapsed()); + let result = serde_json::json!({ + "round": round, + "records": records, + "server_cpu_ms_per_record": cpu / records as f64, + "server_wall_ms": wall.as_secs_f64() * 1000.0, + }); + std::fs::write(dir.join(format!("server-{round}.json")), result.to_string()).unwrap(); + wait("the client to leave", &|| { + harness.state().open_streams() == 0 + }) + .await; + } + stop.send(()).unwrap(); + server.await.unwrap().unwrap(); +} diff --git a/scripts/build_qdl_sdk_release.py b/scripts/build_qdl_sdk_release.py index 92321e3c..bc702967 100755 --- a/scripts/build_qdl_sdk_release.py +++ b/scripts/build_qdl_sdk_release.py @@ -15,7 +15,7 @@ ROOT = Path(__file__).resolve().parents[1] NAME = "qdl-sdk" NORMALIZED_NAME = "qdl_sdk" -VERSION = "2.2.1" +VERSION = "2.3.0" DIST_INFO = f"{NORMALIZED_NAME}-{VERSION}.dist-info" DEPENDENCIES = ( "grpcio>=1.70.0,<2.0.0", diff --git a/tests/test_phaseb_stable_release.py b/tests/test_phaseb_stable_release.py index 1daee762..eefc300a 100644 --- a/tests/test_phaseb_stable_release.py +++ b/tests/test_phaseb_stable_release.py @@ -317,7 +317,7 @@ def test_service_openapi_and_sdk_versions_are_explicit(self): ) generated = build_openapi() self.assertEqual(package["project"]["version"], "2.0.0") - self.assertEqual(qdl_sdk.__version__, "2.2.1") + self.assertEqual(qdl_sdk.__version__, "2.3.0") self.assertEqual(generated["info"]["version"], "2.0.0") self.assertEqual(snapshot, generated) # ``reference:batch`` is a governed V2 public path in the checked-in diff --git a/tests/test_qdl_sdk_batched_subscribe.py b/tests/test_qdl_sdk_batched_subscribe.py new file mode 100644 index 00000000..14a742d9 --- /dev/null +++ b/tests/test_qdl_sdk_batched_subscribe.py @@ -0,0 +1,224 @@ +"""SDK 2.3.0 batched Subscribe: per-record parity against batching and older gateways. + +A loopback grpc.aio server plays the gateway: it either packs up to the requested +number of records into `SubscribeResponse.records` (2.3.0 shape) or ignores the +request field and sends one `record` per response (a pre-2.3.0 gateway). The SDK +must yield the identical StreamEvent/ControlEvent sequence, with each record's own +offset and resume token, in every combination. The Rust gateway's packing rules are +pinned in `rust/qdl-stream-gateway/tests/native_stream.rs` (b01-b04). +""" + +from __future__ import annotations + +import asyncio +import unittest + +import grpc + +from qdl.marketdata.v2 import market_data_pb2 +from qdl.query.v2 import query_pb2 +from qdl_sdk import DataRequirement, Feed, Grade, GrpcStreamTransport, StaticBearerCredential +from qdl_sdk.models import ControlEvent, StreamEvent +from qdl_sdk.transport import DEFAULT_MAX_RECORDS_PER_RESPONSE, MAX_RECORDS_PER_RESPONSE + +METHOD = "/qdl.query.v2.MarketDataStreamService/Subscribe" +RECORDS = 150 + + +def control(code: str, high: int) -> query_pb2.StreamRecord: + state = { + "REPLAYING": query_pb2.STREAM_CONTROL_STATE_REPLAYING, + "LIVE": query_pb2.STREAM_CONTROL_STATE_LIVE, + }[code] + return query_pb2.StreamRecord( + resume_token=f"tok-control-{code}", + control=query_pb2.StreamControl( + state=state, code=code, detail=f"{code} detail", high_watermark=high + ), + ) + + +def event(offset: int) -> query_pb2.StreamRecord: + return query_pb2.StreamRecord( + logical_offset=offset, + resume_token=f"tok-{offset}", + event=market_data_pb2.EventEnvelope( + event_id=offset.to_bytes(16, "big"), + instrument_uid="uid-test", + partition_sequence=offset, + trade=market_data_pb2.Trade(native_trade_id=f"t{offset}"), + ), + ) + + +class FakeGateway: + """`batching=False` is a pre-2.3.0 gateway: it never reads field 7.""" + + def __init__(self, *, batching: bool) -> None: + self.batching = batching + self.requests: list[bytes] = [] + self.response_sizes: list[int] = [] + + def responses(self, request: query_pb2.SubscribeRequest): + yield query_pb2.SubscribeResponse(record=control("REPLAYING", 10)) + yield query_pb2.SubscribeResponse(record=event(10)) # a replayed record + yield query_pb2.SubscribeResponse(record=control("LIVE", 10)) + limit = max(1, request.max_records_per_response) if self.batching else 1 + offsets = list(range(11, 11 + RECORDS)) + # Irregular availability: what a gateway finds queued varies. + chunks, index, pattern = [], 0, (1, 7, 64, 3, 200, 2) + while index < len(offsets): + size = min(pattern[len(chunks) % len(pattern)], limit) + chunks.append(offsets[index:index + size]) + index += size + for chunk in chunks: + if len(chunk) == 1: + yield query_pb2.SubscribeResponse(record=event(chunk[0])) + else: + yield query_pb2.SubscribeResponse(records=[event(o) for o in chunk]) + + async def subscribe(self, raw: bytes, context): + self.requests.append(raw) + request = query_pb2.SubscribeRequest.FromString(raw) + for response in self.responses(request): + self.response_sizes.append(max(1, len(response.records))) + yield response.SerializeToString() + + +async def serve(gateway: FakeGateway): + server = grpc.aio.server() + server.add_generic_rpc_handlers((grpc.method_handlers_generic_handler( + "qdl.query.v2.MarketDataStreamService", + {"Subscribe": grpc.unary_stream_rpc_method_handler( + gateway.subscribe, request_deserializer=None, response_serializer=None)}, + ),)) + port = server.add_insecure_port("127.0.0.1:0") + await server.start() + return server, port + + +def requirement() -> DataRequirement: + return DataRequirement("uid-test", Feed.TRADE, Grade.EXECUTION, "policy-test") + + +def key(item) -> tuple: + if isinstance(item, ControlEvent): + return ("control", item.code, item.detail, dict(item.snapshot)) + assert isinstance(item, StreamEvent) + return ("event", item.logical_offset, item.resume_token, item.event.SerializeToString()) + + +class BatchedSubscribeTests(unittest.IsolatedAsyncioTestCase): + async def run_stream(self, *, batching: bool, **kwargs): + gateway = FakeGateway(batching=batching) + server, port = await serve(gateway) + transport = GrpcStreamTransport( + f"127.0.0.1:{port}", + allow_insecure_loopback=True, + credential_provider=StaticBearerCredential("test-only"), + **kwargs, + ) + try: + items = [ + item + async for item in transport.subscribe( + requirement(), + consumer_id="consumer-test", + cursor_token="cursor-test", + max_buffer_events=1000, + ) + ] + finally: + await transport.close() + await server.stop(grace=0) + return items, gateway + + def assert_per_record(self, items) -> None: + events = [item for item in items if isinstance(item, StreamEvent)] + self.assertEqual([e.logical_offset for e in events], list(range(10, 11 + RECORDS))) + for e in events: + self.assertEqual(e.resume_token, f"tok-{e.logical_offset}") + self.assertEqual(e.event.partition_sequence, e.logical_offset) + self.assertEqual( + [item.code for item in items if isinstance(item, ControlEvent)], + ["REPLAYING", "LIVE"], + ) + self.assertEqual(items[2].snapshot, {"high_watermark": 10}) + + async def test_default_requests_batches_and_yields_one_event_per_record(self): + items, gateway = await self.run_stream(batching=True) + request = query_pb2.SubscribeRequest.FromString(gateway.requests[0]) + self.assertEqual(request.max_records_per_response, DEFAULT_MAX_RECORDS_PER_RESPONSE) + self.assertEqual(DEFAULT_MAX_RECORDS_PER_RESPONSE, 64) + self.assertGreater(max(gateway.response_sizes), 1, "the gateway did batch") + self.assertLessEqual(max(gateway.response_sizes), 64) + self.assert_per_record(items) + + async def test_batched_and_unbatched_sequences_are_identical(self): + reference, _ = await self.run_stream(batching=True, max_records_per_response=1) + self.assert_per_record(reference) + for limit in (2, 16, 64, MAX_RECORDS_PER_RESPONSE): + with self.subTest(limit=limit): + items, gateway = await self.run_stream( + batching=True, max_records_per_response=limit + ) + self.assertGreater(max(gateway.response_sizes), 1) + self.assertEqual([key(i) for i in items], [key(i) for i in reference]) + + async def test_one_record_per_response_sends_the_pre_2_3_request_bytes(self): + _, gateway = await self.run_stream(batching=True, max_records_per_response=1) + legacy = query_pb2.SubscribeRequest( + consumer_id="consumer-test", + requirement=requirement().to_proto(), + cursor_token="cursor-test", + max_buffer_events=1000, + ).SerializeToString() + self.assertEqual(gateway.requests[0], legacy, "field 7 absent, as in 2.2.x") + self.assertEqual(set(gateway.response_sizes), {1}) + + async def test_an_older_gateway_is_read_one_record_per_response(self): + reference, _ = await self.run_stream(batching=True, max_records_per_response=1) + items, gateway = await self.run_stream(batching=False) + self.assertEqual( + query_pb2.SubscribeRequest.FromString(gateway.requests[0]).max_records_per_response, + 64, + ) + self.assertEqual(set(gateway.response_sizes), {1}, "old gateway ignores the field") + self.assertEqual([key(i) for i in items], [key(i) for i in reference]) + + async def test_a_batch_can_be_abandoned_mid_way_and_resumed_by_record_token(self): + gateway = FakeGateway(batching=True) + server, port = await serve(gateway) + transport = GrpcStreamTransport( + f"127.0.0.1:{port}", + allow_insecure_loopback=True, + credential_provider=StaticBearerCredential("test-only"), + ) + stream = transport.subscribe( + requirement(), consumer_id="consumer-test", cursor_token="cursor-test" + ).__aiter__() + try: + seen = [] + while len([i for i in seen if isinstance(i, StreamEvent)]) < 12: + seen.append(await stream.__anext__()) + last = [i for i in seen if isinstance(i, StreamEvent)][-1] + # Stopping inside a 64-record batch still leaves an exact per-record + # resume point: the token of the last record the caller saw. + self.assertEqual((last.logical_offset, last.resume_token), (21, "tok-21")) + finally: + await stream.aclose() + await transport.close() + await server.stop(grace=0) + + def test_the_batch_size_is_validated(self): + for bad in (0, -1, MAX_RECORDS_PER_RESPONSE + 1, 1.5, "64", True): + with self.subTest(bad=bad), self.assertRaises(ValueError): + GrpcStreamTransport( + "127.0.0.1:1", + allow_insecure_loopback=True, + max_records_per_response=bad, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_qdl_sdk_release.py b/tests/test_qdl_sdk_release.py index c93e687a..224ef299 100644 --- a/tests/test_qdl_sdk_release.py +++ b/tests/test_qdl_sdk_release.py @@ -41,11 +41,11 @@ def test_wheel_is_reproducible_self_contained_and_recorded(self): expected = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()).rstrip(b"=").decode() self.assertEqual(encoded, "sha256=" + expected) self.assertEqual(int(size), len(payload)) - self.assertIn("qdl_sdk-2.2.1-py3-none-any.whl", archive.read("qdl_sdk/README.md").decode()) + self.assertIn("qdl_sdk-2.3.0-py3-none-any.whl", archive.read("qdl_sdk/README.md").decode()) release = json.loads(Path(first_result["manifest"]).read_text()) self.assertFalse(release["contains_service_internals"]) - self.assertEqual(release["version"], "2.2.1") + self.assertEqual(release["version"], "2.3.0") self.assertEqual(len(release["generated_contract_digest"]), 64) sbom = json.loads(Path(first_result["sbom"]).read_text()) self.assertEqual(sbom["bomFormat"], "CycloneDX") From 4704edd8ea9189db70cde8ebef3115f663a26f0f Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 11 Oct 2026 10:04:53 +0000 Subject: [PATCH 142/147] docs(release): v2.2.3 G12 PASS from the full isolated run (G1-G3 after every fault) --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 7 +++++ .../evidence/releases/v2.2.3/RELEASE_NOTES.md | 6 ++-- .../releases/v2.2.3/release-preparation.json | 4 +-- .../evidence/releases/v2.2.3/scorecard.json | 28 +++++++++++++++---- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 500a50ec..29b0c8f4 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72238,3 +72238,10 @@ no build, CI or subagent job running. response. - Old SDK 2.1.0 and 2.2.1 against the new gateway: unchanged shape (real gRPC). - Status: tested locally; needs a gateway image and roll, then TS SDK 2.3.0 adoption. +- G12 full run on the release digests, harness 2d4b88c, `p184-g12-rc10-full3-20261011` (verdict c7b8ea95): PASS. + - Caught up: broker 483.8 s, projector 405.8 s, Stream 275.4 s (<= 600). + - In every post-catch-up window: G1, G2 TRADE/QUOTE/MARK (max <= 96.8 ms, n 1060/600 per venue) and G3 BAR + 1m (max 477.6 ms) inside. + - Monitor: 2511 samples, 0 crashes. + - Teardown clean. + - The v2.2.3 scorecard moves G12 to PASS; the BOOK-only verdict is kept as superseded. diff --git a/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md b/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md index 24589b57..61bad470 100644 --- a/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md +++ b/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md @@ -90,9 +90,9 @@ sha256 `01336708...`, `protocol_met: true`, no protocol problems, load 03:44:05- restarts 0). G12 and G13 are scored from their own receipts on the release digests. Summary: -- PASS: G1, G3, G5, G6, G7, G9, G10, G11, G14. +- PASS: G1, G3, G5, G6, G7, G9, G10, G11, G12, G14. - FAIL: G2, G8. -- NOT_MEASURED: G4, G12. +- NOT_MEASURED: G4. - DEFERRED: G13. Paths under `~/.local/state/qdl-v2/`; sha256 prefixes, full hashes in scorecard.json. @@ -110,7 +110,7 @@ Paths under `~/.local/state/qdl-v2/`; sha256 prefixes, full hashes in scorecard. | G9 readiness | demanded == session-ready; 0 data-path eligibility losses; v1 fallback 0 | 180/180 samples readable, demanded 64 == session-ready in every sample, 0 data-path losses, 24 legitimate quiet TRADE refusals, v1 fallback 0 | `g9g10/g9-g10-summary.json` (3015c2db) | **PASS** | | G10 TRADE integrity | 0 gaps, 0 dup applied, 0 replay-expired, 0 overflow | all 0; 2 RECONNECTED resumes; 129 log polls, 0 unreadable | same file and trace | **PASS** | | G11 writer restart recovery | first valid apply <= 30 s; caught up <= 60 s | outage 0.121 s; 64/64 slices caught up, p50 3.919, max 18.923 s; 0 rewinds; cursor never ahead | `g11/g11-summary.json` (c59cccf3) | **PASS** | -| G12 broker/projector/Stream recovery | caught up <= 600 s, then G1-G3 | Isolated, release digests: broker stop/start 22.334 s, projector restart 0.935 s, Stream restart 2.294 s; G1 300 s windows after each inside (max 1970.0 ms OKX after the broker fault). G2/G3 after recovery: NOT_MEASURED (BOOK-only source) | `p184-g12-rc10-20261011-a2/evidence/verdict.json` (05c0652d), `teardown.json` (f17de5ca) | **NOT_MEASURED** (catch-up and G1 inside; G2/G3 after recovery not measured) | +| G12 broker/projector/Stream recovery | caught up <= 600 s, then G1-G3 | Isolated, release digests: broker stop/start 22.334 s, projector restart 0.935 s, Stream restart 2.294 s; G1 300 s windows after each inside (max 1970.0 ms OKX after the broker fault). G2/G3 after recovery: NOT_MEASURED (BOOK-only source) | `p184-g12-rc10-20261011-a2/evidence/verdict.json` (05c0652d), `teardown.json` (f17de5ca) | **PASS**: full run `p184-g12-rc10-full3-20261011` (verdict c7b8ea95, harness 2d4b88c): caught up 483.8 / 405.8 / 275.4 s; G1, G2 TRADE/QUOTE/MARK (max <= 96.8 ms), G3 BAR (max 477.6 ms) inside after every fault; the BOOK-only run above is superseded | | G13 DR01-DR08 | every cell once on the final digests | 41/41 DL cells ok on the release digests: dr05-upstream 6, dr08 4, dr05-transport 9, views 22 (2 quiet cells NOT_OBSERVED). 4 TS steps deferred | `p184-g13-20261011T043821Z/summary.json` (3a3a34e2), `...T045533Z/summary.json` (309199f1) | DEFERRED | | G14 resources | CPU/RAM per role at stage 100, with headroom | 18/18 roles, 151 samples, none at or over its memory limit. Mean CPU: kafka 81/98/102 %, query 57/66 %, writer 151 %; DL mean sum 5.37 vCPU. Lowest memory headroom query_kn_1 14.3 % | `stage100/host.json` (06eab7ba) | **PASS** | diff --git a/upgrade/evidence/releases/v2.2.3/release-preparation.json b/upgrade/evidence/releases/v2.2.3/release-preparation.json index a130f0a1..32fab872 100644 --- a/upgrade/evidence/releases/v2.2.3/release-preparation.json +++ b/upgrade/evidence/releases/v2.2.3/release-preparation.json @@ -1187,8 +1187,7 @@ "G8" ], "NOT_MEASURED": [ - "G4", - "G12" + "G4" ], "PASS": [ "G1", @@ -1199,6 +1198,7 @@ "G9", "G10", "G11", + "G12", "G14" ] } diff --git a/upgrade/evidence/releases/v2.2.3/scorecard.json b/upgrade/evidence/releases/v2.2.3/scorecard.json index 67b889fc..1f34f253 100644 --- a/upgrade/evidence/releases/v2.2.3/scorecard.json +++ b/upgrade/evidence/releases/v2.2.3/scorecard.json @@ -130,6 +130,7 @@ "G9", "G10", "G11", + "G12", "G14" ], "FAIL": [ @@ -137,8 +138,7 @@ "G8" ], "NOT_MEASURED": [ - "G4", - "G12" + "G4" ], "DEFERRED": [ "G13" @@ -2121,8 +2121,8 @@ "id": "G12", "quantity": "Kafka broker / projector / Stream restart recovery", "rule": "fault cleared -> consumers caught up to live <= 600 s, then G1-G3 inside gate; the same four timestamps as G11", - "verdict": "NOT_MEASURED", - "reason": "isolated run on the release digests (python a0c3649d, rust 96a2c912, writer bae2509f): all three faults caught up <= 600 s and G1 inside in the 300 s window after each; G2/G3 after recovery are NOT_MEASURED (BOOK-only isolated source), and the rule requires G1-G3, so the gate is NOT_MEASURED, not PASS", + "verdict": "PASS", + "reason": "full isolated run on the release digests (python a0c3649d, rust 96a2c912, writer bae2509f) with harness 2d4b88c (TEST_ONLY BOOK, TRADE, QUOTE, MARK/INDEX and BAR 1m sources): every fault caught up <= 600 s, then G1, G2 (TRADE, QUOTE, MARK) and G3 inside in the 300 s window; monitor 2511 samples, 0 sample errors, no gap", "level": "ISOLATED_FAULT", "provenance": "TEST_ONLY", "measured": [ @@ -2238,7 +2238,25 @@ "sha256": "7576125751ef20235bad4c139188fe8b31bd10ee3113efea7cd153f11282c87d" } }, - "observed": "caught up and G1 INSIDE for all three faults" + "observed": "caught up 483.8 / 405.8 / 275.4 s; all parts PASS", + "full_run": { + "dir": "~/.local/state/qdl-v2/p184-g12-rc10-full3-20261011", + "verdict_sha256": "c7b8ea951fcf2c008e3d97d15584162c064b17ecbbd54b865fbbfb66862cac39", + "teardown_sha256": "fc11efb3ca785a93df184b76caa1c44a0aa2408dd22742440248b87c27173e63", + "harness_commit": "2d4b88c", + "caught_up_s": { + "broker_stop_start": 483.833, + "projector_restart": 405.8, + "stream_restart": 275.425 + }, + "g2_max_ms": "TRADE <= 96.8, QUOTE <= 96.6 (n 1060 / 600 per venue per fault)", + "g3_bar_1m_max_ms": 477.6 + }, + "superseded_bookonly_verdict": { + "verdict": "NOT_MEASURED", + "reason": "isolated run on the release digests (python a0c3649d, rust 96a2c912, writer bae2509f): all three faults caught up <= 600 s and G1 inside in the 300 s window after each; G2/G3 after recovery are NOT_MEASURED (BOOK-only isolated source), and the rule requires G1-G3, so the gate is NOT_MEASURED, not PASS", + "observed": "caught up and G1 INSIDE for all three faults" + } }, { "id": "G13", From f1849e09e1c99a7bdd09b84dcc3d4f89e01dbd86 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 11 Oct 2026 11:15:44 +0000 Subject: [PATCH 143/147] test(p184): the load driver's source SDK carries its source generated contracts With sdk_from_source only qdl_sdk was mounted over a 2.2.2 image, so SDK 2.3.0 read SubscribeResponse.records from an old protobuf and every alpha stream failed (AttributeError, campaign 2026-10-11 10:37Z). --- scripts/phase3_consumer_load_acceptance.py | 6 +++++- tests/test_phase3_consumer_load_driver.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/scripts/phase3_consumer_load_acceptance.py b/scripts/phase3_consumer_load_acceptance.py index ee20e42a..c8807537 100644 --- a/scripts/phase3_consumer_load_acceptance.py +++ b/scripts/phase3_consumer_load_acceptance.py @@ -563,7 +563,11 @@ def docker_command( "--mount", f"type=bind,src={profile['runtime_dir']},dst=/runtime,readonly", *[arg for consumer, path in sorted((profile.get("manifest_overrides") or {}).items()) for arg in ("--mount", f"type=bind,src={path},dst={_PHASE3_CONSUMER_MANIFESTS[consumer]},readonly")], - *(("--mount", f"type=bind,src={ROOT / 'qdl_sdk'},dst=/app/qdl_sdk,readonly") + # The SDK from source carries its generated contracts from the same source, as its wheel does: an SDK + # newer than the image's contracts otherwise reads fields the old protobuf lacks (SDK 2.3.0 + # SubscribeResponse.records against a 2.2.2 image, 2026-10-11). + *(("--mount", f"type=bind,src={ROOT / 'qdl_sdk'},dst=/app/qdl_sdk,readonly", + "--mount", f"type=bind,src={ROOT / 'generated/python/qdl'},dst=/app/generated/python/qdl,readonly") if profile.get("sdk_from_source") else ()), "--mount", f"type=bind,src={ROOT / 'qdl/certification/phase3_consumer_load.py'},dst=/app/qdl/certification/phase3_consumer_load.py,readonly", "--mount", f"type=bind,src={Path(__file__).resolve()},dst=/driver/phase3_consumer_load_acceptance.py,readonly", diff --git a/tests/test_phase3_consumer_load_driver.py b/tests/test_phase3_consumer_load_driver.py index 4728fd45..1ba75083 100644 --- a/tests/test_phase3_consumer_load_driver.py +++ b/tests/test_phase3_consumer_load_driver.py @@ -84,6 +84,21 @@ def test_disposable_command_is_bounded_read_only_and_secret_paths_are_not_expose self.assertTrue(identity["tls"]["ca_file"].startswith("/tmp/identity/")) self.assertTrue(identity["jwt"]["private_key_file"].startswith("/tmp/identity/")) + def test_sdk_from_source_brings_its_generated_contracts(self): + # An SDK newer than the image's protobuf read SubscribeResponse.records from a 2.2.2 image and every + # stream failed with AttributeError (2026-10-11); the source SDK must carry the source contracts. + with tempfile.TemporaryDirectory() as temporary: + raw = self._profile(Path(temporary)) + for flag, expected in ((False, False), (True, True)): + raw["sdk_from_source"] = flag + command, _ = _MODULE.docker_command( + _MODULE.validate_profile(raw), name="qdl-phase3-load-unit", image_id="sha256:" + "a" * 64, + mode="load", sessions=1, duration_seconds=60) + mounts = [value for value in command if value.startswith("type=bind,")] + self.assertEqual(any(m.endswith("dst=/app/qdl_sdk,readonly") for m in mounts), expected) + self.assertEqual( + any(m.endswith("dst=/app/generated/python/qdl,readonly") for m in mounts), expected) + def test_profile_rejects_unknown_field_before_docker_is_called(self): with tempfile.TemporaryDirectory() as temporary: profile = self._profile(Path(temporary)) From 5ee94636a3b41d4d482eadbc9dd80f51e126cf84 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 11 Oct 2026 11:53:32 +0000 Subject: [PATCH 144/147] fix(query): serve a proven same hot record at its recorded boundary (IDR-088) A quiet key's state-partition watermark can trail the head an earlier ring read recorded; once the ring evicted the key, Query refused the very same record as PRIMARY_BEHIND and the TS writer's slice dropped (stage 100, 2026-10-11 11:37Z, OKX BNB TRADE quiet > 120 s). --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 13 ++++++++++++ qdl/runtime/kn_query_backend.py | 26 +++++++++++++++++++++++ tests/test_kn_hot_view.py | 23 ++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 29b0c8f4..984fd890 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72245,3 +72245,16 @@ no build, CI or subagent job running. - Monitor: 2511 samples, 0 crashes. - Teardown clean. - The v2.2.3 scorecard moves G12 to PASS; the BOOK-only verdict is kept as superseded. +- IDR-088, latent since rc10 and first triggered 2026-10-11 11:37Z (stage-100 run 111700Z stopped by its guard, + writer DEGRADED 63/64): + - Trigger: OKX BNB-USDT-SWAP TRADE was quiet for more than 120 s. + - The Stream ring evicted the key (ring_max_age 120 s). + - The primary held the same record, but its state-partition watermark only advances when that partition gets a + frame, so its boundary trailed the head an earlier hot read had recorded. + - Query refused with HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND / HOT_RECORD_MISSING although nothing newer existed. + - The writer's idle session probe treats that refusal as a disconnect. + - Fix (qdl/runtime/kn_query_backend.py): a primary that is the same record (source offset and payload digest) + an earlier hot read proved newest is served at the recorded boundary. The boundary stays monotonic, no gap is + invented, and the record-regression and payload-mismatch guards are unchanged. + - Regression tests added; 181 tests OK. + - Not caused by the writer or SDK 2.3.0 change, nor by Query rc11 (diffs checked). diff --git a/qdl/runtime/kn_query_backend.py b/qdl/runtime/kn_query_backend.py index 32a1ca68..26c3619f 100644 --- a/qdl/runtime/kn_query_backend.py +++ b/qdl/runtime/kn_query_backend.py @@ -279,6 +279,29 @@ def _hot_monotonic(self, binding, view, *, record=False) -> bool: self._hot_boundaries[binding.binding_id] = (view.boundary, record_offset, digest) return True + def _proven_same_record(self, binding, view): + """The primary view lifted to the recorded boundary when it is the same record an earlier hot read proved + newest up to that boundary (same source offset and payload digest); None otherwise. + + A quiet key's state partition watermark only advances when that partition receives a frame, so the primary + boundary can trail the head an earlier ring read recorded. Once the ring evicts the quiet key (ring_max_age), + the same record was refused HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND although nothing newer exists: a newer + record would carry a larger source offset. Serving it at the recorded boundary keeps the boundary monotonic + for the client and invents no gap (P18.4 stage 100, 2026-10-11 11:37Z, OKX BNB TRADE quiet > 120 s).""" + if view is None or len(view.rows) != 1 or view.rows[0].source_offset is None: + return None + digest = hashlib.sha256(view.rows[0].canonical).hexdigest() + with self._hot_boundary_lock: + previous = self._hot_boundaries.get(binding.binding_id) + if previous is None: + return None + boundary, prior_record_offset, prior_digest = previous + if ((view.boundary.topic_id, view.boundary.partition) != (boundary.topic_id, boundary.partition) + or view.rows[0].source_offset != prior_record_offset or digest != prior_digest + or view.boundary.offset >= boundary.offset): + return None + return replace(view, boundary=boundary) + def _hot_candidate(self, binding, primary=None, *, shared=False): """Authenticated committed bytes, validated by the same lineage oracle. @@ -412,6 +435,9 @@ def _latest_view(self, requirement, binding): if quiet_primary: raise _not_ready(f"HOT_QUIET_PRIMARY_UNVERIFIED: {backup_failure}") if primary is not None and not self._hot_monotonic(binding, primary): + lifted = self._proven_same_record(binding, primary) + if lifted is not None: + return lifted raise _not_ready(f"HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND: {backup_failure}") return primary diff --git a/tests/test_kn_hot_view.py b/tests/test_kn_hot_view.py index cb036bea..f33f4568 100644 --- a/tests/test_kn_hot_view.py +++ b/tests/test_kn_hot_view.py @@ -169,6 +169,29 @@ def test_switches_back_only_when_primary_catches_up_and_stays_valid(self): self.assertTrue(self.backend.latest(self.requirement).quality.execution_eligible) self.assertEqual(self.calls,1) + def test_proven_same_record_behind_a_lagging_watermark_is_served_at_the_recorded_boundary(self): + # A quiet key: an earlier hot read proved record 30 newest up to 40; the primary now holds that same + # record, but its state-partition watermark trails (35) and the ring has evicted the key. It was refused + # HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND although nothing newer exists (2026-10-11 11:37Z, OKX BNB TRADE). + from qdl.runtime.kn_market_cache import ProductView, SourceBoundary + from qdl.runtime.kn_query_backend import parse_placeholder + self.assertTrue(self.backend.latest(self.requirement).quality.execution_eligible) + self.primary = ProductView(self.lpk, 2, 3, self.backup.rows, SourceBoundary("topic", 0, 35)) + self.backup = HotViewUnavailable("HOT_RECORD_MISSING") + self.now += 10_000_000_000 + result = self.backend.latest(self.requirement) + self.assertFalse(result.quality.execution_eligible) + self.assertEqual(parse_placeholder(result.cursor), ("topic", 0, 40)) + + def test_a_different_record_behind_the_recorded_boundary_is_still_refused(self): + from qdl.runtime.kn_market_cache import CacheRow, ProductView, SourceBoundary + from qdl.query.results import QueryBackendError + self.backend.latest(self.requirement) + self.primary = ProductView(self.lpk, 2, 3, (CacheRow(self.original, 25),), SourceBoundary("topic", 0, 35)) + self.backup = HotViewUnavailable("HOT_RECORD_MISSING") + with self.assertRaisesRegex(QueryBackendError, "HOT_BACKUP_UNAVAILABLE_PRIMARY_BEHIND"): + self.backend.latest(self.requirement) + def test_stale_backup_does_not_turn_old_price_usable(self): self.now+=10_000_000_000 result=self.backend.latest(self.requirement) From 481fd6704b28d8fa63deee39d59eeba23d87f5a8 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 11 Oct 2026 14:23:34 +0000 Subject: [PATCH 145/147] fix(bar-edge): isolate the Kafka backlog probe in a supervised spawn child (IDR-087) AdminClient.list_consumer_group_offsets segfaulted inside librdkafka 2.15.0 during a broker stop (G12 monitor evidence); the bar edge called it in-process. A crash or hang now closes the history gate (fail-safe) instead of killing the bar edge, and the next call respawns the child. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 23 +++ qdl/runtime/history_backpressure.py | 50 +++++- qdl/runtime/isolated_probe.py | 141 +++++++++++++++++ tests/test_idr087_supervised_probe.py | 185 ++++++++++++++++++++++ tests/test_kn_history_fill.py | 5 +- 5 files changed, 396 insertions(+), 8 deletions(-) create mode 100644 qdl/runtime/isolated_probe.py create mode 100644 tests/test_idr087_supervised_probe.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 984fd890..a1e6c87a 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72258,3 +72258,26 @@ no build, CI or subagent job running. invented, and the record-regression and payload-mismatch guards are unchanged. - Regression tests added; 181 tests OK. - Not caused by the writer or SDK 2.3.0 change, nor by Query rc11 (diffs checked). +- IDR-087 confirmed and fixed: + - The G12 harness named the crashing call: SIGSEGV in confluent-kafka/librdkafka 2.15.0 + AdminClient.list_consumer_group_offsets during a broker stop (p184-g12-rc12-20261011 monitor.errors). + - The product bar edge used the same call in its history backpressure gate. + - Fix: qdl/runtime/isolated_probe.py SupervisedProbe. + - The Kafka offset reads run in one spawn-started child; spawn because the bar edge process already holds + librdkafka threads, so fork is unsafe. + - Hard timeout 15 s (QDL_STABLE_BAR_BACKPRESSURE_PROBE_TIMEOUT_S). + - A crash, hang or child error becomes ProbeUnavailable. The gate closes (fail-safe: history pauses, live + continues) and the next call respawns the child. + - Tests: 13 new (real SIGSEGV child, timeout, child error, parity with in-process numbers) plus 271 related. + - Status: tested locally; a real broker-stop check of the bar edge needs a live fault. +- G12 full on rc12 digests (python 3b585597, rust ebc7b794, writer 64347a94), `p184-g12-rc12-20261011` (verdict + 20f29671): PASS. + - Caught up: broker 224.7 s, projector 19.4 s, Stream 138.4 s. + - G1/G2/G3 inside after every fault. + - The monitor survived 2 SIGSEGV samples. +- G13 on rc12 digests with writer 44baacc0 (TS a37ff4d): 41/41 DL cells ok. + - dr05-upstream 6, dr08 4, dr05-transport 9 (`p184-g13-20261011T134532Z`); views 22 on the re-run + (`...T140432Z`). + - The first views run refused DR04.EXPLICIT_LAST@OKX for a stale LAST in a quiet market. + - DR05 DISCONNECT_RECONNECT now counts only controls after the cut. The 04:38Z PASS was a false positive from + startup quota controls; TS a37ff4d logs a bridge RECONNECTED control. diff --git a/qdl/runtime/history_backpressure.py b/qdl/runtime/history_backpressure.py index 0881f2ae..58c53f2f 100644 --- a/qdl/runtime/history_backpressure.py +++ b/qdl/runtime/history_backpressure.py @@ -10,7 +10,10 @@ backlog drains, and resumes from its checkpoint. Boundary: read-only Kafka metadata (group offsets, watermarks); never joins or -commits a group; no venue call. +commits a group; no venue call. In production the Kafka client lives in a +supervised spawned child (IDR-087: librdkafka 2.15.0 segfaulted in +``list_consumer_group_offsets`` during a broker stop); a crash or hang there is +an unknown backlog, which keeps the gate closed, never a dead edge. """ from __future__ import annotations @@ -18,6 +21,10 @@ from pathlib import Path from typing import Any, Callable, Mapping, Sequence +from qdl.runtime.isolated_probe import ProbeUnavailable, SupervisedProbe + +DEFAULT_PROBE_TIMEOUT_S = 15.0 + @dataclass(frozen=True) class Stage: @@ -35,15 +42,22 @@ def stage_backlog(committed: Mapping[int, int], ends: Mapping[int, int]) -> int: class KafkaStageBacklog: - def __init__(self, stages: Sequence[Stage], *, read_committed: Callable[[str, str], Mapping[int, int]], - read_ends: Callable[[str], Mapping[int, int]]) -> None: + def __init__(self, stages: Sequence[Stage], *, read_committed: Callable[[str, str], Mapping[int, int]] | None = None, + read_ends: Callable[[str], Mapping[int, int]] | None = None, + read_backlog: Callable[[], Mapping[str, int]] | None = None) -> None: if not stages or any(stage.limit_records < 1 for stage in stages): raise ValueError("backpressure needs stages with positive limits") + if (read_backlog is None) == (read_committed is None or read_ends is None): + raise ValueError("backpressure needs either per-stage readers or one backlog reader") self.stages = tuple(stages) self._read_committed = read_committed self._read_ends = read_ends + self._read_backlog = read_backlog def backlog(self) -> dict[str, int]: + if self._read_backlog is not None: + backlog = self._read_backlog() + return {stage.name: int(backlog[stage.name]) for stage in self.stages} return {stage.name: stage_backlog(self._read_committed(stage.group, stage.topic), self._read_ends(stage.topic)) for stage in self.stages} @@ -52,6 +66,8 @@ def __call__(self) -> tuple[bool, dict[str, Any]]: try: backlog = self.backlog() + except ProbeUnavailable as error: # crashed/hung/failed probe child: backlog unknown + return False, {"error": f"ProbeUnavailable: {error}"[:200], "unknown": error.reason} except Exception as error: # noqa: BLE001 - an unknown backlog never admits history return False, {"error": f"{type(error).__name__}: {error}"[:200]} over = {stage.name: backlog[stage.name] for stage in self.stages if backlog[stage.name] > stage.limit_records} @@ -71,12 +87,32 @@ def kafka_backlog_from_environment(environ: Mapping[str, str], stages: Sequence[ if not path.is_file(): raise ValueError(f"history backpressure TLS file unavailable: {path}") security[option] = str(path) - return kafka_backlog_from_config(environ["QDL_STABLE_BAR_BACKPRESSURE_BOOTSTRAP"], stages, security) + timeout_s = float(environ.get("QDL_STABLE_BAR_BACKPRESSURE_PROBE_TIMEOUT_S", "").strip() + or DEFAULT_PROBE_TIMEOUT_S) + return kafka_backlog_from_config(environ["QDL_STABLE_BAR_BACKPRESSURE_BOOTSTRAP"], stages, security, + timeout_s=timeout_s) def kafka_backlog_from_config(bootstrap: str, stages: Sequence[Stage], - security: Mapping[str, str] | None = None) -> KafkaStageBacklog: - """Wire the gate to a broker with confluent-kafka (admin group offsets + watermarks).""" + security: Mapping[str, str] | None = None, *, + timeout_s: float = DEFAULT_PROBE_TIMEOUT_S) -> KafkaStageBacklog: + """The gate with its Kafka reads in a supervised spawned child (IDR-087). + + The caller's process never loads librdkafka for this gate. Each gate call + asks the long-lived child for the whole backlog; a native crash, a hang past + ``timeout_s`` or a client error raises ``ProbeUnavailable`` -> gate closed, + and the next call starts a fresh child. + """ + + probe = SupervisedProbe(kafka_backlog_in_process, (bootstrap, tuple(stages), dict(security or {})), + timeout_s=timeout_s) + return KafkaStageBacklog(stages, read_backlog=probe) + + +def kafka_backlog_in_process(bootstrap: str, stages: Sequence[Stage], + security: Mapping[str, str] | None = None) -> Callable[[], dict[str, int]]: + """Backlog reader with confluent-kafka (admin group offsets + watermarks) in + THIS process. Production runs it only inside the supervised child.""" from confluent_kafka import Consumer, ConsumerGroupTopicPartitions, TopicPartition from confluent_kafka.admin import AdminClient @@ -100,4 +136,4 @@ def read_ends(topic: str) -> dict[int, int]: return {p: metadata.get_watermark_offsets(TopicPartition(topic, p), timeout=10)[1] for p in topic_partitions(topic)} - return KafkaStageBacklog(stages, read_committed=read_committed, read_ends=read_ends) + return KafkaStageBacklog(stages, read_committed=read_committed, read_ends=read_ends).backlog diff --git a/qdl/runtime/isolated_probe.py b/qdl/runtime/isolated_probe.py new file mode 100644 index 00000000..4bda8a88 --- /dev/null +++ b/qdl/runtime/isolated_probe.py @@ -0,0 +1,141 @@ +"""Run a native-client probe in a supervised child process (IDR-087). + +Purpose: librdkafka 2.15.0 (confluent-kafka 2.15.0) segfaulted inside +``AdminClient.list_consumer_group_offsets`` during a broker stop (G12 isolated +harness, 2026-10-11: monitor child SIGSEGV; earlier runs exit 139). A native +crash cannot be caught in-process, so a role that only needs a read-only +metadata answer runs the client in a child: a crash, a hang past the hard +timeout or a dead pipe loses only that answer, comes back to the caller as +``ProbeUnavailable`` (typed reason + exit code), and the next call starts a +fresh child. + +Start method: ``spawn`` (fresh interpreter via fork+exec). The parent may +already hold librdkafka clients and their threads (the BAR publisher); a plain +``fork`` would copy their locks in an arbitrary state into the child. With +spawn the child loads the native library itself and inherits nothing. + +Boundary: one long-lived child per probe; requests are serialised; the +parent never imports the probed client. No retries here - the caller decides +what an unknown answer means. +""" +from __future__ import annotations + +import multiprocessing +import signal +import threading +import time +from typing import Any, Callable + + +class ProbeUnavailable(RuntimeError): + """The probe produced no answer: the child crashed, hung, or failed.""" + + def __init__(self, reason: str, *, exitcode: int | None = None, detail: str = "") -> None: + self.reason, self.exitcode, self.detail = reason, exitcode, detail + super().__init__(f"{reason} exitcode={exitcode}{': ' + detail if detail else ''}") + + +def exit_reason(exitcode: int | None) -> str: + """SIGSEGV for -11, EXIT_ for a plain exit, NO_EXIT if still unknown.""" + + if exitcode is None: + return "NO_EXIT" + if exitcode < 0: + try: + return signal.Signals(-exitcode).name + except ValueError: + return f"SIGNAL_{-exitcode}" + return f"EXIT_{exitcode}" + + +def _probe_child(conn: Any, factory: Callable[..., Callable[[], Any]], args: tuple) -> None: + """Child loop: build the probe once, answer each request until the pipe closes.""" + + probe = None + while True: + try: + conn.recv() + except (EOFError, OSError): + return # the parent is gone + try: + if probe is None: + probe = factory(*args) + conn.send(("ok", probe())) + except Exception as error: # noqa: BLE001 - reported to the parent as a typed failure + conn.send(("error", f"{type(error).__name__}: {error}"[:300])) + + +class SupervisedProbe: + """``probe()`` returns ``factory(*args)()`` evaluated in a spawned child. + + ``factory`` and ``args`` must be picklable (a module-level function). Raises + ``ProbeUnavailable`` on crash (reason = signal name), timeout (``TIMEOUT``), + or an exception inside the child (``CHILD_ERROR``). + """ + + def __init__(self, factory: Callable[..., Callable[[], Any]], args: tuple = (), *, + timeout_s: float, context: Any = None) -> None: + if timeout_s <= 0: + raise ValueError("supervised probe needs a positive timeout") + self.factory, self.args, self.timeout_s = factory, tuple(args), float(timeout_s) + self.context = context or multiprocessing.get_context("spawn") + self.proc = self.conn = None + self.restarts = 0 + self._started = False + self._lock = threading.Lock() + + def _start(self) -> None: + parent, child = self.context.Pipe() + self.proc = self.context.Process(target=_probe_child, args=(child, self.factory, self.args), + daemon=True, name="qdl-supervised-probe") + self.proc.start() + child.close() + self.conn = parent + if self._started: + self.restarts += 1 + self._started = True + + def _stop(self, *, kill: bool) -> int | None: + proc, conn, self.proc, self.conn = self.proc, self.conn, None, None + if conn is not None: + conn.close() + if proc is None: + return None + if kill and proc.is_alive(): + proc.kill() + proc.join(5) + return proc.exitcode + + def close(self) -> None: + with self._lock: + self._stop(kill=True) + + def __call__(self) -> Any: + with self._lock: + if self.proc is not None and not self.proc.is_alive(): + self._stop(kill=True) + if self.proc is None: + self._start() + deadline = time.monotonic() + self.timeout_s + try: + self.conn.send("probe") + while True: + left = deadline - time.monotonic() + if left <= 0: + code = self._stop(kill=True) + raise ProbeUnavailable("TIMEOUT", exitcode=code) + if self.conn.poll(min(left, 0.05)): + kind, value = self.conn.recv() + if kind == "ok": + return value + raise ProbeUnavailable("CHILD_ERROR", detail=str(value)) + if not self.proc.is_alive(): + # A last answer may still sit in the pipe after a clean exit. + if self.conn.poll(0): + continue + code = self._stop(kill=True) + raise ProbeUnavailable(exit_reason(code), exitcode=code) + except (EOFError, BrokenPipeError, ConnectionResetError, OSError) as error: + code = self._stop(kill=True) + raise ProbeUnavailable(exit_reason(code), exitcode=code, + detail=type(error).__name__) from None diff --git a/tests/test_idr087_supervised_probe.py b/tests/test_idr087_supervised_probe.py new file mode 100644 index 00000000..6cecbe8b --- /dev/null +++ b/tests/test_idr087_supervised_probe.py @@ -0,0 +1,185 @@ +"""IDR-087: the history backpressure Kafka probe runs in a supervised child. + +librdkafka 2.15.0 segfaulted in ``AdminClient.list_consumer_group_offsets`` +during a broker stop (G12 isolated harness). These cases use fake probes in a +real spawned child (no broker): a SIGSEGV or a hang in the child becomes a +typed ``ProbeUnavailable``, the parent survives, the gate stays closed, and a +healthy child returns the same numbers as the in-process reader. +""" +from __future__ import annotations + +import os +import signal +import sys +import time +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +from qdl.runtime.history_backpressure import KafkaStageBacklog, Stage, kafka_backlog_from_config +from qdl.runtime.isolated_probe import ProbeUnavailable, SupervisedProbe, exit_reason + +_COMMITTED = {"core": {0: 100, 1: 7}, "a": {0: 0}} +_ENDS = {"raw": {0: 150, 1: 7, 2: 3}, "canonical": {0: 20}} +_STAGES = (Stage("core", "core", "raw", 100), Stage("a", "a", "canonical", 10)) + + +# Module-level factories: the spawned child imports them by name. +def fake_kafka_reader(stages): + """The production composition (per-stage readers -> backlog) with fake reads.""" + return KafkaStageBacklog(stages, read_committed=lambda group, _topic: _COMMITTED[group], + read_ends=lambda topic: _ENDS[topic]).backlog + + +def segv_probe(marker=None): + def probe(): + if marker is None or not Path(marker).exists(): + if marker is not None: + Path(marker).write_text("crashed once") + os.kill(os.getpid(), signal.SIGSEGV) + return {"core": 1, "a": 2} + return probe + + +def hang_probe(): + def probe(): + time.sleep(60) + return probe + + +def raising_probe(): + def probe(): + raise RuntimeError("KafkaException: broker transport failure") + return probe + + +def pid_probe(): + return os.getpid + + +class SupervisedProbeTests(unittest.TestCase): + def _probe(self, factory, args=(), timeout_s=20.0): + probe = SupervisedProbe(factory, args, timeout_s=timeout_s) + self.addCleanup(probe.close) + return probe + + def test_a_native_crash_in_the_child_is_typed_and_the_parent_survives(self): + probe = self._probe(segv_probe) + with self.assertRaises(ProbeUnavailable) as caught: + probe() + self.assertEqual((caught.exception.reason, caught.exception.exitcode), ("SIGSEGV", -signal.SIGSEGV)) + self.assertIsNone(probe.proc) + + def test_the_next_call_after_a_crash_starts_a_fresh_child(self): + with TemporaryDirectory() as directory: + probe = self._probe(segv_probe, (str(Path(directory) / "crashed"),)) + with self.assertRaises(ProbeUnavailable): + probe() + self.assertEqual(probe(), {"core": 1, "a": 2}) + self.assertEqual(probe.restarts, 1) + + def test_a_hang_is_killed_at_the_hard_timeout(self): + probe = self._probe(hang_probe, timeout_s=1.0) + started = time.monotonic() + with self.assertRaises(ProbeUnavailable) as caught: + probe() + self.assertEqual(caught.exception.reason, "TIMEOUT") + self.assertLess(time.monotonic() - started, 8.0) + self.assertIsNone(probe.proc) + + def test_a_client_error_in_the_child_is_typed_and_keeps_the_child(self): + probe = self._probe(raising_probe) + with self.assertRaises(ProbeUnavailable) as caught: + probe() + self.assertEqual(caught.exception.reason, "CHILD_ERROR") + self.assertIn("broker transport failure", caught.exception.detail) + self.assertTrue(probe.proc.is_alive()) + + def test_the_child_is_reused_and_is_not_the_parent(self): + probe = self._probe(pid_probe) + child = probe() + self.assertNotEqual(child, os.getpid()) + self.assertEqual(probe(), child) + self.assertEqual(probe.restarts, 0) + + def test_spawn_is_the_start_method(self): + # fork would copy the parent's librdkafka threads/locks (BAR publisher) into the child. + self.assertEqual(SupervisedProbe(pid_probe, timeout_s=1).context.get_start_method(), "spawn") + with self.assertRaises(ValueError): + SupervisedProbe(pid_probe, timeout_s=0) + + def test_exit_reasons(self): + self.assertEqual([exit_reason(code) for code in (-11, -9, 0, 3, None)], + ["SIGSEGV", "SIGKILL", "EXIT_0", "EXIT_3", "NO_EXIT"]) + + +class SupervisedBacklogGateTests(unittest.TestCase): + def test_a_healthy_child_returns_the_in_process_numbers(self): + in_process = KafkaStageBacklog(_STAGES, read_committed=lambda group, _topic: _COMMITTED[group], + read_ends=lambda topic: _ENDS[topic]) + probe = SupervisedProbe(fake_kafka_reader, (_STAGES,), timeout_s=20) + self.addCleanup(probe.close) + gate = KafkaStageBacklog(_STAGES, read_backlog=probe) + self.assertEqual(gate.backlog(), in_process.backlog()) + self.assertEqual(gate.backlog(), {"core": 53, "a": 20}) + self.assertEqual(gate(), in_process()) + self.assertEqual(gate(), (False, {"backlog": {"core": 53, "a": 20}, "over": {"a": 20}})) + + def test_a_crashed_or_hung_probe_keeps_the_gate_closed(self): + for factory, timeout_s, reason in ((segv_probe, 20.0, "SIGSEGV"), (hang_probe, 1.0, "TIMEOUT")): + with self.subTest(reason=reason): + probe = SupervisedProbe(factory, timeout_s=timeout_s) + self.addCleanup(probe.close) + admitted, detail = KafkaStageBacklog(_STAGES, read_backlog=probe)() + self.assertFalse(admitted) + self.assertEqual(detail["unknown"], reason) + self.assertIn("ProbeUnavailable", detail["error"]) + + def test_the_bar_edge_pauses_history_on_an_unknown_backlog(self): + from qdl.runtime.stable_bar_edge import StableBinanceBarEdge + + probe = SupervisedProbe(segv_probe, timeout_s=20) + self.addCleanup(probe.close) + edge = object.__new__(StableBinanceBarEdge) + edge.history_gate = KafkaStageBacklog(_STAGES, read_backlog=probe) + edge._history_gate_closed_at = None + edge.clock = lambda: 1000.0 + with self.assertLogs("qdl.runtime.stable_bar_edge", "WARNING") as logs: + self.assertFalse(edge._history_gate_open()) + self.assertEqual(edge._history_gate_closed_at, 1000.0) + self.assertIn("SIGSEGV", logs.output[0]) + + def test_production_wiring_isolates_the_kafka_client_and_starts_lazily(self): + from qdl.runtime import history_backpressure + + gate = kafka_backlog_from_config("broker:9092", _STAGES, {"security.protocol": "ssl"}, timeout_s=7) + probe = gate._read_backlog + self.assertIsInstance(probe, SupervisedProbe) + self.assertIs(probe.factory, history_backpressure.kafka_backlog_in_process) + self.assertEqual(probe.args, ("broker:9092", _STAGES, {"security.protocol": "ssl"})) + self.assertEqual(probe.timeout_s, 7.0) + self.assertIsNone(probe.proc) # no child, and no Kafka client, until the first gate call + + def test_the_gate_never_imports_the_kafka_client_in_the_parent(self): + # The bar edge itself does load confluent-kafka (BAR publisher); that is why the + # child is spawned, not forked. The gate adds no librdkafka client to the parent. + import subprocess + + code = ("import sys\n" + "from qdl.runtime.history_backpressure import Stage, kafka_backlog_from_config\n" + "kafka_backlog_from_config('broker:9092', [Stage('a', 'g', 't', 1)])\n" + "print(sorted(m for m in sys.modules if m.startswith('confluent_kafka')))\n") + result = subprocess.run([sys.executable, "-B", "-c", code], capture_output=True, text=True, timeout=60, + check=True) + self.assertEqual(result.stdout.strip(), "[]") + + def test_both_reader_shapes_are_exclusive(self): + with self.assertRaises(ValueError): + KafkaStageBacklog(_STAGES) + with self.assertRaises(ValueError): + KafkaStageBacklog(_STAGES, read_committed=lambda *_: {}, read_ends=lambda _t: {}, + read_backlog=lambda: {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_kn_history_fill.py b/tests/test_kn_history_fill.py index f025d1a4..3f7a6720 100644 --- a/tests/test_kn_history_fill.py +++ b/tests/test_kn_history_fill.py @@ -103,7 +103,10 @@ def test_production_backpressure_uses_the_existing_publisher_tls_identity(self): factory.assert_called_once_with("broker:9092", stages, { "security.protocol": "ssl", "ssl.ca.location": str(root / "ca.crt"), "ssl.certificate.location": str(root / "client.crt"), - "ssl.key.location": str(root / "client.key")}) + "ssl.key.location": str(root / "client.key")}, timeout_s=15.0) + factory.reset_mock() + kafka_backlog_from_environment({**env, "QDL_STABLE_BAR_BACKPRESSURE_PROBE_TIMEOUT_S": "4"}, stages) + self.assertEqual(factory.call_args.kwargs, {"timeout_s": 4.0}) factory.reset_mock() (root / "client.key").unlink() with self.assertRaisesRegex(ValueError, "TLS file unavailable"): From 63640c2fae12d34fdbd5675cfe7da4905944285d Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 11 Oct 2026 15:34:28 +0000 Subject: [PATCH 146/147] docs(release): v2.2.3 evidence on rc13 (481fd67): no FAIL gate All 12 roles run python ccbea12e / rust 6afdc155. Stage-100 campaign (2026-10-11T12:35Z): PASS G1 G2 G3 G5 G6 G7 G8 G9 G10 G11 G14; G12 PASS (isolated, every part); G4 NOT_MEASURED (rows without a frozen baseline); G13 41/41 DL cells, TS groups deferred. SDK 2.3.0, local CI and every earlier campaign recorded. Status PREPARED_NOT_PUBLISHED. --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 24 + .../evidence/releases/v2.2.3/RELEASE_NOTES.md | 382 +- .../releases/v2.2.3/release-preparation.json | 1321 +++- .../evidence/releases/v2.2.3/scorecard.json | 5298 +++++++++++++---- 4 files changed, 5498 insertions(+), 1527 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index a1e6c87a..1fad481c 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -72281,3 +72281,27 @@ no build, CI or subagent job running. - The first views run refused DR04.EXPLICIT_LAST@OKX for a stale LAST in a quiet market. - DR05 DISCONNECT_RECONNECT now counts only controls after the cut. The 04:38Z PASS was a false positive from startup quota controls; TS a37ff4d logs a bridge RECONNECTED control. + +### v2.2.3 final record: rc13 (481fd67) on all roles, phase E step-2 scorecard, SDK 2.3.0 - 2026-10-11 (Claude, subagent) + +- rc13 (481fd67, IDR-087) rolled on all 12 application roles 14:26-14:31Z from rc12, writer READY 64/64 after + every step: python sha256:ccbea12e87e4, rust sha256:6afdc155ba3e (labels 481fd67, IDR-039 closed again). Rollback + per role in rollout-rc6/ (back to rc12 3b585597 / ebc7b794). +- Earlier the same day: Query/Stream rc11 (10:33Z), Query rc12 (11:55Z), rc12 on all roles (13:12-13:18Z); writer + 64347a94 (a3035c8, SDK 2.3.0), then knob DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE=64 (12:34Z), then 44baacc0 + (a37ff4d, bridge reconnect control log only) at 13:44Z. +- Campaigns: 103749Z protocol not met (driver mounted SDK 2.3.0 over an old protobuf -> f1849e0); 111700Z stopped by + its guard (IDR-088 -> 5ee9463); step 1 115743Z, knob 1: protocol met, G2 FAIL (TRADE B/O p95 801/817, max + 3284/3209 ms), G8 PASS; step 2 123525Z, knob 64: protocol met, PASS G1-G3, G5-G11, G14, G4 NOT_MEASURED (17/17 + baselined rows PASS, 8 rows without baseline). Caveat: TRADE volume step 2 66303/12492 vs step 1 92415/26567. +- G12 PASS from p184-g12-rc12-20261011 (verdict 20f29671); G13 41/41 DL cells (134532Z + 140432Z), 4 TS steps + DEFERRED. rc13 differs from rc12 only by IDR-087 (bar edge, not in either stack). +- Measurement identity of step 2 vs 481fd67: Query rc12 and Stream rc11 source-identical on their paths; core, + projectors, ingestors rc10 source-equivalent (gateway crate + two additive prost fields no other crate uses); bar + edge rc10 not identical (IDR-087 only). +- SDK 2.3.0 wheel b9c74157... reproducible over two builds from 481fd67 (= TS vendored wheel); supersedes 2.2.1. +- Release evidence updated (upgrade/evidence/releases/v2.2.3/): final scorecard PASS 12, NOT_MEASURED G4, DEFERRED + G13, FAIL none; the rc10 record (campaign 4, G2/G8 FAIL) moved verbatim into the history. Status + PREPARED_NOT_PUBLISHED; local CI on 481fd67 41 PASS, 3 SKIPPED (2 PR-only, 1 sudo), summary 83e7dda9; remote CI, owner decision and tag open. +- Open: IDR-087 tested locally only (no live broker stop of the bar edge); DL CPU 5.08 vCPU > 5 budget; writer burst + capacity depends on knob 64. Not committed. diff --git a/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md b/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md index 61bad470..cbec9579 100644 --- a/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md +++ b/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md @@ -1,13 +1,367 @@ # Quant Data Layer v2.2.3 Status: `PREPARED_NOT_PUBLISHED`. This is not a published release, a Git tag or an owner publication decision. -Publication needs the owner to push and tag. Remote CI must also run on the final head. The published stable +Publication needs the owner's decision and a tag. Remote CI must also run on the final head. The published stable release is still v2.2.2. Scorecard: [scorecard.json](scorecard.json). Inventory, hashes, the SDK build and CI: [release-preparation.json](release-preparation.json). ## Scope +This release closes the P18.4 phase D Data Layer work on the Kafka-native stack. It contains: +- one-commit images for every application role (IDR-039, closed again on 481fd67); +- execution-view correctness fixes (IDR-062, IDR-040, IDR-084, IDR-088, Astra R1/R3); +- hot-path contention fixes (Astra R2/R3), the OKX session fixes and the ingestor liveness knob; +- the G5 warmup fixes and IDR-086 (a typed refusal in a warmup batch keeps its code and retryability); +- the Stream replica byte budget fix and the provider admission INFLIGHT_CAPACITY wait fix; +- the funding settlement publication allowance (60 s, backed by a measurement); +- batched Subscribe responses for lossless live feeds (additive wire fields, SDK 2.3.0); +- IDR-087: the bar edge's Kafka backlog probe runs in a supervised child, so a librdkafka crash closes the history + gate instead of killing the bar edge; +- additive public fields `event_id` (IDR-082) and `proof_observed_at_ns` (IDR-040 fix 2). + +Public endpoints keep their versions. Every change is additive. A client that does not ask for batches gets the +unchanged single-record Subscribe shape, and readers on SDK 2.1.0 ignore the new fields. + +This is not a Trading System order, accounting or alpha certification. It is not HA or independent-failure-domain +evidence, because every run used the same host. + +## Source And Images + +- Release source: `481fd6704b28d8fa63deee39d59eeba23d87f5a8` on `fix/p184-dl-closure`. The branch is pushed + (`origin/fix/p184-dl-closure` contains it). No later commit exists at assembly time. +- Commits since the previous evidence source 413c5b9 (`git log --oneline 413c5b9..481fd67`): + - runtime code: 139dafe (IDR-086, Query warmup batch typed refusal), 4c779ac (batched Subscribe, Stream gateway, + contracts, SDK 2.3.0), 5ee9463 (IDR-088, Query serves a proven same hot record at its recorded boundary), + 481fd67 (IDR-087, supervised bar-edge Kafka probe); + - harness and tooling: 1a161f4, 22ca8df and 2d4b88c (the full G12 harness), f1849e0 (the load driver mounts its + source generated contracts); + - evidence: 4704edd (the previous record) and cb87370 (before it). + +Images were checked with `docker image inspect`. Both revision labels equal 481fd67, which closes IDR-039 again. The +version label reads `2.2.3-rc13`, because a tag never changes a digest. + +| Image | Digest | Tag | Roles | +| --- | --- | --- | --- | +| Python | `sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677` | `qdl-v2-python:2.2.3-rc13-481fd67` | query_kn_1/2, binance_bar_edge | +| Rust | `sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad` | `qdl-v2-rust:2.2.3-rc13-481fd67` | stream_kn_1/2, market_projector_1/2, rust_core x3, ingestor_okx_swap, ingestor_binance_usdm | + +All 12 application roles run these two digests: restart count 0, OOM false (`docker inspect`, 2026-10-11 ~14:40Z). +They were rolled one role at a time between 14:26:16Z and 14:31:42Z on 2026-10-11, with the writer READY 64/64 after +every step (receipts `p184-d3-capacity-20261009/rollout-rc6/-receipt-20261011T14*.json`). Every role came +from rc12 (python `3b585597`, rust `ebc7b794`, 5ee9463), which is the rollback target. + +The following were not changed by this release: +- kafka1-3 (`sha256:9516fb7634ba`), apart from the dynamic broker config below; +- market_cache and stable_redis (`sha256:dfa18828cbc0`); +- the one-shots stable_state_init and stable_tls_init (`sha256:bd0163fd76b0`). + +Ingestor configs are unchanged and were re-read for this record through a `--network none` container: OKX revision +20 (`eb982fac...`), Binance revision 19 (`5833bc4e...`), liveness 1500/4000 ms, no `book_liveness_proof_ms`, so +IDR-040 fix 2 is OFF. + +TS writer `market_data_service_p184w` (TS repo `trading-system-next-upgrade`, pushed): +- image `sha256:44baacc0a3ff95f71029d230e48e1b7282627f26de9bd99e13d1148862e09068`, TS `a37ff4d`, SDK 2.3.0, running + since 2026-10-11T13:44:14Z, restart count 0; +- env (`docker inspect`, only these keys): `DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE=64`, + `DATA_LAYER_V2_STREAM_PIPELINE_DEPTH=0`, `MARKET_DATA_FIXED_WORKER_PINS=OKX:BTCUSDT=2,OKX:ETHUSDT=0`, + `DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE=BINANCE=100`; +- the gate campaign ran writer `64347a94` (TS a3035c8, SDK 2.3.0) with the same four values. `git diff --stat + a3035c8 a37ff4d` changes one runtime file, `services/market_data/data_layer_bridge.py` +17: a slice the bridge + reopened logs one `RECONNECTED origin=bridge` control. The rest is tests, the DR05 harness, the register and the + plan. + +## Runtime Config + +Kafka log-cleaner throttle (unchanged since the previous record, not re-read here): +- cluster-wide dynamic default `log.cleaner.io.max.bytes.per.second=8388608` (8 MiB/s), applied + 2026-10-11T03:40:56Z without a broker restart, owner-approved; +- cause: `md.latest.v2` compaction on kafka1 stalled every rust_core commit about 2.06 s on 2026-10-10; +- receipt `~/.local/state/qdl-v2/kafka-cleaner-throttle-20261010/` (README.md `72cbcf87...`, alter.out + `a59f5deb...`, after.out `8a635a9b...`, applied_at.txt `5f94630c...`); +- rollback, failure only: `kafka-configs.sh --alter --delete-config log.cleaner.io.max.bytes.per.second`. + +TS execution Redis `redis_marketdata`, cpu-shares 4096 (owner-approved, plan journal 2026-10-11): +- applied about 08:09Z on 2026-10-11, before both phase E campaigns; `docker inspect` reads `CpuShares 4096` now; +- receipt `~/.local/state/qdl-v2/redis-cpu-shares-20261011/receipt.txt` (`c1b5f23e...`): involuntary context + switches 209 -> 160 per 60 s at similar load; +- rollback, failure only: `docker update --cpu-shares 0 redis_marketdata`. + +## Gate Scorecard (G1-G14) + +Gates come from the frozen TS table `upgrade/p184-phase-d-gates.md` (TS 31cf663, sha256 `c67bf69b...`) and the +frozen budget `config/v2/p184-stage100-budget.json` (sha256 `ff85335b...`, unchanged at 481fd67). + +Scoring rule: a PASS needs the table's measurement rule, which is steady state for at least 30 min at the stage-100 +workload plus a burst window. A measurement outside that rule can FAIL a gate but cannot PASS it. A NOT_MEASURED is +never upgraded. No frozen gate was widened. + +The gate scorecard is the phase E step-2 campaign `~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/` +(scorecard sha256 `747e4764...`, `protocol_met: true`, no protocol problems, load 12:37:01-13:07:01Z, 64 demanded +slices, writer restarts 0, `startup:bounded` PASS). G12 and G13 are scored from their own receipts. + +Summary: +- PASS: G1, G2, G3, G5, G6, G7, G8, G9, G10, G11, G12, G14. +- NOT_MEASURED: G4. +- DEFERRED: G13. +- FAIL: none. + +Paths under `~/.local/state/qdl-v2/`; sha256 prefixes, full hashes in scorecard.json. + +| Gate | Rule | Measured | Source (sha256) | Verdict | +| --- | --- | --- | --- | --- | +| G1 BOOK e2e to TS cache | p95 <= 500, p99 <= 1000, max < 2000 ms | DELTA B/O p95 138.8/146.4, p99 190.3/199.5, max 570.4/585.9 (n 82692/94817); SNAPSHOT B/O p95 176.5/177.6, max 571.2/503.2 | campaign `trace/transit-load.json` (bd1ccac0), `trace/mark-load.json` (b6a51981) | **PASS** | +| G2 TRADE/QUOTE/MARK new-event transit | TRADE p95 <= 500, max < 1500; QUOTE/MARK p95 <= 500, max < 1000 ms | TRADE B p95 243.4, max 583.1 (n 66303); TRADE O p95 210.3, max 495.5 (n 12492); QUOTE B/O p95 162.2/169.1, max 531.5/532.0; MARK B/O p95 360.3/337.4, max 803.1/712.7 | same two files | **PASS** | +| G3 BAR e2e from close | p95 <= 5 s, max < 60 s | pooled n 320, p95 1914, max 3330 ms; 0 unseen, 0 read errors | `g3/g3-summary.json` (fafca1be) | **PASS** | +| G4 venue -> canonical | candidate p95 <= baseline p95 x 1.2 per row | 17/17 baselined rows PASS in every window they were sampled (e.g. trade B/O 99.8-105.3 / 102.1-111.9 vs limits 269.3/257.4). 8 observed rows have no frozen baseline: bar BINANCE 1h/5m, bar OKX 1h/5m, OKX_INVERSE book_delta/book_snapshot/mark/quote | `g4-{1..5}-summary.json` (717bf3b8, 3f080361, 21b72b59, 0b68e97f, eeaa9a68) | NOT_MEASURED: rows without baseline | +| G5 request latency | certified p95/p99 per class and venue; frozen cold limits | All 10 class/venue rows PASS (worst p95 77.6 OKX BAR_LATEST, worst p99 252.0 BINANCE L2); cold 2500/5000 overlap 4 runs, 0 bad | `stage100/gates.json` (86b74f26), `receipt.json` (fdf26baa) | **PASS** | +| G6 durable-view event age + quality | per-binding D3 policy of the TS EXECUTION consumer; zero unsafe | TS EXECUTION probe 19213 reads over 32 targets: every D3 binding measured, 0 unsafe (e.g. MARK OKX 3566 ELIGIBLE + 40 ELIGIBLE_QUIET; QUOTE OKX 3515 + 85; TRADE NOT_ELIGIBLE B/O 138/916, typed). Alpha reads: 0 unsafe under their own requirement | `g6-ts/summary.json` (5a9fc905) | **PASS** | +| G7 TS consumer path | SDK -> writer p95 <= 100; writer -> Redis p95 <= 50 ms per feed | SDK -> writer p95: BOOK_DELTA 33.8, BOOK_SNAPSHOT 44.8, QUOTE 30.7, TRADE 45.3, MARK 0.6. Writer -> Redis p95: 13.5, 20.5, 5.9, 33.3, 14.4 | `trace/transit-load.json` (bd1ccac0) | **PASS** | +| G8 TS execution Redis | commandstats <= 60 % core per 10 min; SLOWLOG 0 in steady state; PING p99 <= 5 ms | commandstats <= 0.0612 core; PING p99 <= 1.15 ms per window (n 3861, 0 failures); SLOWLOG 0 new entries; 0 MONITOR clients | `g8/g8-summary.json` (0314b552) | **PASS** | +| G9 readiness | demanded == session-ready; 0 data-path eligibility losses; v1 fallback 0 | 180/180 samples readable, demanded 64 == session-ready in every sample, 0 data-path losses, 48 legitimate quiet TRADE refusals, v1 fallback 0 | `g9g10/g9-g10-summary.json` (247c62e6) | **PASS** | +| G10 TRADE integrity | 0 gaps, 0 dup applied, 0 replay-expired, 0 overflow | all 0; 0 resumed reconnects; 129 log polls, 0 unreadable | same file and trace | **PASS** | +| G11 writer restart recovery | first valid apply <= 30 s; caught up <= 60 s | outage 0.132 s; 64/64 slices caught up, p50 0.158, max 7.511 s; 0 rewinds; cursor never ahead | `g11/g11-summary.json` (c85e4566) | **PASS** | +| G12 broker/projector/Stream recovery | caught up <= 600 s, then G1-G3 | Isolated, rc12 digests (python 3b585597, rust ebc7b794, writer 64347a94): caught up broker stop/start 224.704 s, projector restart 19.4 s, Stream restart 138.381 s; G1, G2 TRADE/QUOTE/MARK and G3 BAR 1m inside after every fault (largest max: G1 574.5 ms, G2 541.1 ms, both after the broker fault; G3 341.1 ms) | `p184-g12-rc12-20261011/evidence/verdict.json` (20f29671), `teardown.json` (a5fceec5) | **PASS** | +| G13 DR01-DR08 | every cell once on the final digests | 41/41 DL cells ok on the rc12 digests with the final writer 44baacc0: dr05-upstream 6, dr08 4, dr05-transport 9, views 22 (2 quiet cells NOT_OBSERVED). 4 TS steps deferred | `p184-g13-20261011T134532Z/summary.json` (445e05c9), `...T140432Z/summary.json` (f0b7b523) | DEFERRED | +| G14 resources | CPU/RAM per role at stage 100, with headroom | 18/18 roles, 152 samples, none at or over its memory limit. Mean CPU: kafka 78/94/97 %, query 64/56 %, writer 134 %; DL mean sum 5.08 vCPU. Lowest memory headroom query_kn_2 31.2 % | `stage100/host.json` (a5ba4fcf) | **PASS** | + +Measurement identity. The step-2 campaign ran before the rc12 (13:12-13:18Z) and rc13 (14:26-14:31Z) rolls. Diffs +below are `git diff --stat 481fd67` without tests, scripts, docs and `upgrade/`: +- query_kn_1/2: rc12 `sha256:3b585597570f` (5ee9463, rolled 11:55-11:56Z). The diff is + `qdl/runtime/history_backpressure.py` +50/-7 and `qdl/runtime/isolated_probe.py` +141 (IDR-087). Only + `qdl/runtime/stable_bar_edge.py` imports them: **source-identical for every Query path**. +- stream_kn_1/2: rc11 `sha256:ddb2ad19d610` (4704edd, rolled 10:34-10:35Z). The Rust tree, `generated/rust`, Cargo + files and config have an empty diff: **source-identical for the Stream binary**, batched Subscribe included. +- rust_core x3, projectors and ingestors: rc10 `sha256:96a2c9120ef4` (413c5b9). The Rust diff is + `rust/qdl-stream-gateway` (no other crate depends on it) and `generated/rust/qdl/query/v2/qdl.query.v2.rs` +16, + which adds two prost fields (`SubscribeRequest.max_records_per_response`, `SubscribeResponse.records`) to + `qdl-contracts`. Every crate depends on `qdl-contracts`, but no code outside the gateway references those two + messages (`git grep`): **source-equivalent** for these binaries (rebuilt, not byte-compared). +- binance_bar_edge: rc10 `sha256:a0c3649d9a47` (413c5b9). On its import path (checked one level deep) the only + diff is IDR-087. **Not source-identical:** the measured bar edge ran the in-process Kafka backlog probe. IDR-087 + changes behaviour only when that probe crashes or hangs. +- Writer: `64347a94` (a3035c8) with knob 64; the final writer `44baacc0` adds only the reconnect control log (see + above). + +G12 and G13 ran on the rc12 digests. The rc13 images differ from rc12 only by IDR-087: the Python diff is the two +bar-edge files above, and the Rust diff is empty. The bar edge is not part of the isolated G12 stack, and no G13 step +exercises it. + +### One-Variable Comparison: Batched Subscribe + +Phase E step 1 and step 2 ran on the same DL images (Query rc12, Stream rc11, rest rc10), the same writer image +`64347a94`, depth 0, the same pins and MARK delay, the Kafka throttle and the Redis cpu-shares. Only the writer +knob `DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE` differed. Step 1 had it unset, which is the default of 1 (writer +composes `trace-20261011T115631Z.compose.json` and `trace-20261011T123412Z.compose.json` in the TS campaign-roll +directory). + +| Campaign | Knob | G2 TRADE B/O p95 / max (ms) | QUOTE B/O max | G2 | G8 | Scorecard | +| --- | --- | --- | --- | --- | --- | --- | +| `p184-final-campaign-20261011T115743Z` (step 1, protocol met) | 1 | 801.2/817.2 / 3284.0/3208.8 | 1067.2/1283.3 | **FAIL** | PASS | `6ae2945d` | +| `p184-final-campaign-20261011T123525Z` (step 2, protocol met) | 64 | 243.4/210.3 / 583.1/495.5 | 531.5/532.0 | **PASS** | PASS | `747e4764` | + +Caveat: TRADE volume was not equal. TRADE BINANCE/OKX n was 66303/12492 in step 2, 92415/26567 in step 1 and +160328/39292 in campaign 4. Part of the improvement may come from the lower TRADE volume. The step-2 PASS is a +measurement at that volume. It is not a capacity claim at the campaign-4 volume. + +From campaign 4 (the previous record, G2/G8 FAIL) to step 2, more than one variable changed: +- the writer image (8b567eb -> a3035c8, adding SDK 2.3.0, a FLAT apply script and a latest-wins cursor commit); +- the Redis cpu-shares; +- the Query/Stream images. + +That comparison is not attributed to one cause. + +## Campaigns Since The Previous Record (2026-10-11) + +Paths under `~/.local/state/qdl-v2/`. Every result is kept, and none is erased by the step-2 PASS. + +| Campaign | Setup | Result | What it led to | +| --- | --- | --- | --- | +| `p184-final-campaign-20261011T103749Z` (c55e2e39) | Query rc11 b128de1e, Stream rc11 ddb2ad19, rest rc10, writer 64347a94 | protocol NOT met (driver acceptance FAIL: streams no_errors / every_live_stream_delivered / final_bar, fault windows); G2 FAIL | The load driver mounted SDK 2.3.0 over a 2.2.2 image's generated protobuf, so alpha streams failed with `AttributeError` on `SubscribeResponse.records` (driver error_count 180; all 40 listed errors are STREAM `AttributeError`) -> f1849e0 | +| `p184-final-campaign-20261011T111700Z` (2ec43e87) | same | stopped by its guard at 11:38Z (driver SIGINT); G9 FAIL (demanded != session-ready in 6 samples) | The writer went DEGRADED 63/64 at 11:37:40Z: OKX BNB-USDT-SWAP TRADE was quiet for more than 120 s, and Query refused the same record PRIMARY_BEHIND after ring eviction (IDR-088) -> 5ee9463 | +| `p184-final-campaign-20261011T115743Z` (6ae2945d) | Query rc12, Stream rc11, rest rc10, writer 64347a94, knob 1 | protocol met; FAIL G2; PASS G8; G4 NOT_MEASURED | the one-variable baseline above | +| `p184-final-campaign-20261011T123525Z` (747e4764) | same, knob 64 | protocol met; the gate scorecard | - | + +`startup:bounded` passed in both protocol-met step campaigns (212 and 193 retries). The frozen budget is kept. + +## G12 And G13 + +G12, isolated, `p184-g12-rc12-20261011`: +- `status COMPLETED`, `pass true`. Image set: core/projector/Stream rust `ebc7b794`, python `3b585597`, writer + `64347a94`. Harness: `scripts/p184_g12_isolated_fault.py` (sha256 `2fa21904...`). The file is identical at + 2d4b88c and 481fd67. The run receipt does not record the harness commit itself. +- Caught up after the fault cleared: broker stop/start 224.704 s (BINANCE 182.931 s), projector restart 19.4 s, + Stream restart 138.381 s, all <= 600 s. +- Every part passed after every fault: G1, G2 TRADE/QUOTE/MARK (n 1060/600/300 per venue) and G3 BAR 1m (n 5 per + venue). The monitor passed too. +- Monitor: 1718 samples, max gap 2.0 s. There were 2 typed `SIGSEGV` sample errors in + `list_consumer_group_offsets:kn-projector-v3-a` during the broker fault. They are the evidence for IDR-087: the + product bar edge called the same AdminClient method in-process. +- Teardown: 12 containers removed, 0 left by label or name, 0 volumes by label, no errors. +- Scope: same host, TEST_ONLY sources, single projector/Stream replica. +- The earlier rc10 full run `p184-g12-rc10-full3-20261011` (PASS, verdict `c7b8ea95`) and the rc10 attempts stay in + the appendix. + +G13, on DL 5ee9463 (rc12 digests) and the final writer 44baacc0 (TS a37ff4d): +- `p184-g13-20261011T134532Z`: dr05-upstream 6/6, dr08-isolated 4/4, dr05-transport 9/9 ok. +- Its dr04-dr06-views step failed on DR04.EXPLICIT_LAST@OKX with `EXECUTION_CONTEXT_STALE`. The LAST trigger was + stale in a quiet market, so the refusal is correct and typed (rows `views-okx.rows`, `efa7882a...`). +- `p184-g13-20261011T140432Z`: dr04-dr06-views 22/22 ok. DR05.QUIET_TYPED BINANCE/OKX are ok but NOT_OBSERVED. +- Finding from the earlier runs `p184-g13-20261011T131833Z` and `...T133254Z` (writer 64347a94): + - DR05.DISCONNECT_RECONNECT failed with 0 logged controls after the cut. Data recovered (validator APPROVED about + 6.2-6.4 s after the reconnect). + - A transport cut ends with the bridge reopening the slice in a fresh session, which yields no SDK RECONNECTED + control. + - The rc10 PASS of this cell (`043821Z`) had counted startup quota controls logged before the cut, so it was a + false positive. + - Fixed in TS a37ff4d: the bridge logs `RECONNECTED origin=bridge`, and the harness counts only controls after the + cut. +- Runtime approval: the owner's chat approval of 2026-10-11, written down by the coordinating session in + `p184-g13-inputs/runtime-approval-rc12.md` (`9e786bfd...`). The owner did not write the file. +- Deferred to the TS release: dr01-deployed, dr01-wrong-account, dr02-dr03-alpha, dr07-reference. + +## Items Closed (commit) + +| Item | Commit(s) | State | +| --- | --- | --- | +| IDR-039 one-commit images for every application role | rc13 rollout of 481fd67 | **closed**: 12/12 roles on 481fd67 | +| IDR-086 warmup batch keeps a typed backend refusal (code, retryability) | 139dafe | in images (Query); tested locally | +| IDR-088 Query serves a proven same hot record at its recorded boundary | 5ee9463 | in images (Query); in the step-2 campaign: 0 data-path eligibility losses | +| IDR-087 bar-edge Kafka backlog probe in a supervised spawn child (fail-safe gate) | 481fd67 | in images (bar edge); **tested locally only**, no live broker stop of the bar edge | +| Batched Subscribe (`max_records_per_response`, `records`), SDK 2.3.0 | 4c779ac | in images (Stream gateway); TS writer adopted it with knob 64 | +| G12 harness: G2/G3 after recovery, namespace state topics, crash-tolerant monitor | 1a161f4, 22ca8df, 2d4b88c | tooling | +| Load driver mounts its source generated contracts | f1849e0 | tooling | +| Redis `redis_marketdata` cpu-shares 4096 | runtime config, receipt above | runtime | +| Items closed in the rc10 record (dadb9c8, 413c5b9, 2e0de95, 70bef7d, Kafka cleaner throttle, tooling) and the 2026-10-10 record | see appendix | in images | + +The defect register lives in the TS repo, `INTEGRATION_DEFECT_REGISTER.md` (TS 68cd019, sha256 `e1c9a2fc...`). Its +status text for IDR-086 and IDR-088 still says that an image rebuild and roll are needed. Both rolls have since +happened (Query rc12 at 11:55Z, rc13 at 14:26Z). This DL record does not edit the TS register. + +## SDK 2.3.0 Wheel + +SDK 2.3.0 supersedes 2.2.1. Neither 2.2.1 nor 2.2.0 was published. + +| Field | Value | +| --- | --- | +| wheel | `qdl_sdk-2.3.0-py3-none-any.whl` | +| wheel sha256 | `b9c74157db2a2d147564d72b65bbbafc9a7d3d055bccee67645b831470ad38c3` | +| reproducible | two builds from a `git archive` of 481fd67 are byte-identical (`cmp`); wheel, SBOM and release json hashes are equal | +| builder | `scripts/build_qdl_sdk_release.py` in `qdl-v2-python:2.2.3-rc13-481fd67`, `--network none`, non-root, read-only source | +| sdk_source_digest | `e0e5d044c5edfffb1084bc84b69d4cf1898a4221004ca720c0c9dd02fb0d895b` | +| generated_contract_digest | `2c59805220f06caf1a19b6df5d4ec7bf44282a780d265e0dd667b9abd4d7c1d1` | +| release json sha256 | `218b45b0250c95c1b273b9b809afc234984ab5fffdefcfdf541ebd635111ea15` | +| SBOM sha256 | `4c69c98d56f29962d9981d1acd517cf44717aa4c224f9df5b2e3e913ceff488b` | + +`qdl_sdk`, `generated/python`, `contracts` and the builder have an empty diff from 4c779ac to 481fd67. The wheel is +byte-identical to the TS-vendored `vendor/qdl_sdk-2.3.0-py3-none-any.whl` (same sha256). Both writer images +(`64347a94`, `44baacc0`) report `qdl_sdk.__version__` 2.3.0. The artifacts are under +`~/.local/state/qdl-v2/releases/v2.2.3/sdk230-build-{1,2}/`, outside Git. + +## CI + +Local exact-source CI on 481fd67 (`scripts/p184_local_ci.py --sha 481fd67 --allow-pull --allow-download +--trivy-image aquasec/trivy:0.58.1`): summary `p184-local-ci-481fd67/summary.json` (sha256 `83e7dda9...`, status +`INCOMPLETE` only because of skips). Of 44 steps, 41 PASS and 3 SKIPPED: +- contract-tests #2 and #5 are pull-request-only steps, so their condition is false for a push event; +- unit-tests #4 (prepare non-root runtime bind mounts) needs sudo. Host preparation is reported, never escalated. + +Every other step passed. That includes consolidated-source-preflight, contract-tests #10 (Rust dependency, license +and advisory policy, run with the download allowed), sdk-python310, the unit tests, both trivy scans (#19, #20), the +SBOM rehearsal and kn-native-integration. + +The previous local CI on 2f489b1 is in the appendix record. Remote GitHub CI on the final head is not cited in this +record. Publication requires the required jobs on the final head. + +## Runtime Changes Performed (2026-10-11, after the rc10 record) + +Receipts were checked for every change below. Each role was recreated one at a time, and the writer was READY 64/64 +after every DL step. The approval wording for each roll was given to the coordinating session and is not re-verified +in this record. + +| Change | Rollback reference | +| --- | --- | +| Redis `redis_marketdata` cpu-shares 4096, ~08:09Z | `docker update --cpu-shares 0 redis_marketdata` | +| Query rc11 `b128de1e` and Stream rc11 `ddb2ad19` (4704edd), 10:33-10:35Z | superseded by the later rolls | +| TS writer `64347a94` (a3035c8, SDK 2.3.0), 10:36Z | TS campaign-roll `rollback-20261011T103624Z.compose.json` | +| Query rc12 `3b585597` (5ee9463), 11:55-11:56Z | superseded | +| TS writer knob `DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE=64`, 12:34Z | TS campaign-roll `rollback-20261011T123412Z.compose.json` | +| rc12 (5ee9463) on all 12 application roles, 13:12-13:18Z | superseded | +| TS writer `44baacc0` (a37ff4d), 13:44Z | TS campaign-roll `rollback-20261011T134403Z.compose.json` (`d673be45`, back to 64347a94 with knob 64) | +| **rc13 (481fd67), all 12 application roles, 14:26-14:31Z** | `p184-d3-capacity-20261009/rollout-rc6/-rollback.compose.json` -> python rc12 `3b585597`, rust rc12 `ebc7b794` | + +Rollback runs in reverse role order. Kafka data, Redis, realm dirs and offsets need no data migration. Rollback is +for failure only. The rc10 rollback composes cited in the appendix record were overwritten by these later rolls, so +their recorded sha256 no longer match the files on disk. + +## Known Limits + +- **Single host:** every run used the same host. No HA and no independent failure domains. No mainnet or funded + claim. +- **G4 rows without a frozen baseline** (bar BINANCE 1h/5m, bar OKX 1h/5m, OKX_INVERSE book_delta, book_snapshot, + mark and quote in the step-2 campaign) keep G4 NOT_MEASURED. No baseline is added after measuring. +- **G6 BOOK_DELTA** is not a Query shape in the frozen workload, so it has no G6 row. +- **startup:bounded** is a stochastic tail of the warmup lane. One session exhausted its 10 retries in one earlier + stage-100 run (20261010T185053Z). The frozen budget is kept. +- **DL CPU above the owner's 5 vCPU budget:** G14 passes its per-role headroom rule, but the DL mean sum at stage + 100 was 5.08 vCPU in the step-2 campaign (`stage100/host.json`, `a5ba4fcf...`; 5.18 in step 1, 5.37 in campaign 4). + Kafka is the largest share. Tuning it is open: one variable at a time, measured against G1/G2. +- **Writer burst capacity depends on batched Subscribe:** + - G2 and G8 pass with the writer knob at 64; at knob 1, G2 failed in step 1. + - TRADE volume in step 2 was lower than in step 1 and campaign 4, so the TRADE-burst capacity at the campaign-4 + volume is not shown. + - A writer that drops the knob loses the PASS. +- **IDR-087 is tested locally only.** A real SIGSEGV child, timeout and child error are covered by tests. The bar + edge itself has not been through a live broker stop. +- **Measurement identity:** the step-2 campaign ran the rc12/rc11/rc10 mix. Query, Stream and the other Rust roles + are source-identical or source-equivalent to 481fd67. The bar edge is not, by the IDR-087 change only. +- **G13:** the four TS-stack steps stay OPEN. The two DR05.QUIET_TYPED cells are NOT_OBSERVED, not a positive + quiet-state proof. +- **IDR-040 fix 2 is OFF** (`book_liveness_proof_ms` unset). A quiet Binance book stays stale, never falsely fresh. +- **Catalog revision split (C.19)** is unchanged: image catalog revision 11, runtime-mounted revision 12. +- **CI:** local only (41 PASS, 3 SKIPPED). Remote CI on the final head is not cited. +- Historical failures stay recorded: + - the campaigns above and in the appendix; + - the G12 attempts; + - the G13 view refusals and the DR05 false positive; + - every failure listed in the appendix. + + None is erased by a later PASS. + +## Consumer Hand-Off (TS phase E) + +- Writer adoption is live: image `44baacc0` (TS a37ff4d) with SDK 2.3.0 (wheel `b9c74157...`) and + `DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE=64`. Keep the knob, `BINANCE=100` for the MARK wake read delay, depth 0 and + the OKX worker pins. +- Remaining TS items: + - close the four deferred G13 steps: dr01-deployed, dr01-wrong-account, dr02-dr03-alpha, dr07-reference; + - alpha callers mint JWTs for manifest revision **15** (4950 RPM / 300 streams); an older JWT revision gets 401; + - deploy the IDR-085 gateway fix (TS be21d49) with the TS release, which closes the deferred DR01 cells; + - IDR-040 fix 2: activate the in-band quiet-book proof flag through its own packet when TS needs it; + - use `event_id` (IDR-082) for trade identity. +- TS Paper manifest/JWT revision 11 (64 products incl. OKX BTC-USD-SWAP) is live. + +## Publication Requires + +1. The owner's publication decision. +2. Remote CI (contract-tests, sdk-python310, unit-tests, kn-native-integration) on the final head, and a merge to dev + and main. +3. An annotated tag `v2.2.3` that links the final CI runs and the hashes of scorecard.json and this file. + +No merge or tag was performed. The branch was pushed before this record. This record is not committed. + +## Appendix: Preparation History + +### 2026-10-11 rc10 record at 413c5b9 (verbatim, superseded by the record above) + +Status: `PREPARED_NOT_PUBLISHED`. This is not a published release, a Git tag or an owner publication decision. +Publication needs the owner to push and tag. Remote CI must also run on the final head. The published stable +release is still v2.2.2. + +Scorecard: [scorecard.json](scorecard.json). Inventory, hashes, the SDK build and CI: [release-preparation.json](release-preparation.json). + +#### Scope + This release closes the P18.4 phase D Data Layer work on the Kafka-native stack. It contains: - one-commit images for every application role (IDR-039, closed on 413c5b9); - execution-view correctness fixes (IDR-062, IDR-040, IDR-084, Astra R1/R3); @@ -23,7 +377,7 @@ Public endpoints keep their versions. Every change is additive, and readers on S This is not a Trading System order, accounting or alpha certification. It is not HA or independent-failure-domain evidence, because every run used the same host. -## Source And Images +#### Source And Images - Release source: `413c5b914f8c6fd0f67429fdddb39451589e9745` on `fix/p184-dl-closure`. - The only later commit, 2f489b1, changes scripts, tests and the plan journal (`git diff --stat 413c5b9 2f489b1`: @@ -65,7 +419,7 @@ TS writer `market_data_service_p184w` (TS repo `trading-system-next-upgrade`, pu - since 2026-10-11T04:24:53Z the writer runs depth 0 with the same pins and MARK delay (`docker inspect`). Depth 0 was the coordinator's choice from two comparisons; that comparison is not re-verified in this record. -## Runtime Config Change: Kafka Log-Cleaner Throttle +#### Runtime Config Change: Kafka Log-Cleaner Throttle - Change: cluster-wide dynamic default `log.cleaner.io.max.bytes.per.second=8388608` (8 MiB/s), applied 2026-10-11T03:40:56Z without a broker restart. The owner approved it through a question at about 03:40Z. @@ -76,7 +430,7 @@ TS writer `market_data_service_p184w` (TS repo `trading-system-next-upgrade`, pu `a59f5deb...`, after.out `8a635a9b...`, applied_at.txt `5f94630c...`). - Rollback, failure only: `kafka-configs.sh --alter --delete-config log.cleaner.io.max.bytes.per.second`. -## Gate Scorecard (G1-G14) +#### Gate Scorecard (G1-G14) Gates come from the frozen TS table `upgrade/p184-phase-d-gates.md` (TS 31cf663, sha256 `c67bf69b...`) and the frozen budget `config/v2/p184-stage100-budget.json` (sha256 `ff85335b...`). @@ -153,7 +507,7 @@ Findings behind the verdicts: - G4 bar latency for OKX is measured from bar open (the OKX source time is the open). The baseline used the same metric. -## Earlier Final Campaigns (2026-10-10 .. 2026-10-11) +#### Earlier Final Campaigns (2026-10-10 .. 2026-10-11) Paths under `~/.local/state/qdl-v2/`. None of these results is erased by campaign 4. @@ -168,7 +522,7 @@ Paths under `~/.local/state/qdl-v2/`. None of these results is erased by campaig 20261010T185053Z, where session s89 exhausted the frozen 10 retries (`retry_max_attempts` 10, retry code RATE_LIMITED; gates.json `8a0db47a...`). The frozen budget is kept. -## G12 And G13 On The Release Digests +#### G12 And G13 On The Release Digests G12, isolated (owner approved), `p184-g12-rc10-20261011-a2`: - `status COMPLETED`, `pass true`. Image set: core/projector/Stream rust `96a2c912`, python `a0c3649d`, writer @@ -200,7 +554,7 @@ G13, on DL 413c5b9 and writer bae2509f (TS 8b567eb): `p184-g13-inputs/runtime-approval-rc10.md` (`55189088...`) was written by an agent from that relayed approval. - Deferred to the TS release: dr01-deployed, dr01-wrong-account, dr02-dr03-alpha, dr07-reference. -## Items Closed (commit) +#### Items Closed (commit) | Item | Commit(s) | State | | --- | --- | --- | @@ -217,7 +571,7 @@ G13, on DL 413c5b9 and writer bae2509f (TS 8b567eb): The funding allowance revert 530a406 listed in the 2026-10-10 record is superseded by 2e0de95. -## SDK 2.2.1 Wheel +#### SDK 2.2.1 Wheel SDK 2.2.0 is superseded. It was never published, but its wheel hash is in the 2026-10-10 record, so the version was bumped rather than reused. @@ -235,7 +589,7 @@ bumped rather than reused. The artifacts are under `~/.local/state/qdl-v2/releases/v2.2.3/sdk221-build-{1,2}/`, outside Git. The TS writer runs SDK 2.1.0. -## CI +#### CI Local exact-source CI (`scripts/p184_local_ci.py` replaying `.github/workflows/ci.yml` on an archive of 2f489b1): - all jobs except unit-tests, summary `p184-local-ci-2f489b1/summary.json` (sha256 `f00a9423...`, receipt status @@ -253,7 +607,7 @@ Local exact-source CI (`scripts/p184_local_ci.py` replaying `.github/workflows/c Remote GitHub CI did NOT run, because the DL repository is not pushed. Publication requires the required jobs on the final head. -## Runtime Changes Performed (2026-10-10 .. 2026-10-11) +#### Runtime Changes Performed (2026-10-10 .. 2026-10-11) Each change was owner-approved. Each role was recreated one at a time. @@ -270,7 +624,7 @@ Each change was owner-approved. Each role was recreated one at a time. Rollback runs in reverse role order. Kafka data, Redis, realm dirs and offsets need no data migration. Rollback is for failure only. The 2026-10-08 .. 2026-10-10 changes are listed in the appendix. -## Known Limits +#### Known Limits - **G2 TRADE and G8 SLOWLOG FAIL: writer throughput under TRADE bursts.** Owner decision: TS phase E, first item. The gates are not widened. @@ -296,7 +650,7 @@ for failure only. The 2026-10-08 .. 2026-10-10 changes are listed in the appendi - Historical failures stay recorded: the four earlier final campaigns above, the failed G12 attempt, the first G13 views run, and every failure listed in the appendix. None is erased by a later PASS. -## Consumer Hand-Off (TS phase E) +#### Consumer Hand-Off (TS phase E) - Adopt SDK **2.2.1** (wheel sha256 `27db3971...`). The writer runs 2.1.0 today, which ignores the new fields. - Alpha callers mint JWTs for manifest revision **15** (4950 RPM / 300 streams). An older JWT revision gets 401. @@ -309,7 +663,7 @@ for failure only. The 2026-10-08 .. 2026-10-10 changes are listed in the appendi against the same frozen gates. - The writer keeps `BINANCE=100` for the MARK wake read delay and the OKX worker pins. -## Publication Requires +#### Publication Requires 1. The owner's publication decision. The 2026-10-11 release decision records G2/G8 as known limits; it does not turn them into PASS. @@ -319,8 +673,6 @@ for failure only. The 2026-10-08 .. 2026-10-10 changes are listed in the appendi No push, merge or tag was performed. -## Appendix: Preparation History - ### 2026-10-10 record at bbfb3be (verbatim, superseded by the record above) Status: `PREPARED_NOT_PUBLISHED`. This is not a published release, a Git tag or an owner publication decision. diff --git a/upgrade/evidence/releases/v2.2.3/release-preparation.json b/upgrade/evidence/releases/v2.2.3/release-preparation.json index 32fab872..2af35987 100644 --- a/upgrade/evidence/releases/v2.2.3/release-preparation.json +++ b/upgrade/evidence/releases/v2.2.3/release-preparation.json @@ -1,5 +1,6 @@ { "branch": "fix/p184-dl-closure", + "branch_pushed": "origin/fix/p184-dl-closure contains 481fd67 (git branch -r --contains)", "catalog": { "note": "deliberate split (C.19 bar edge checkpoint reads the image catalog); unifying needs its own packet", "repo_and_image_catalog_revision": 11, @@ -10,70 +11,62 @@ "runtime_catalog_revision": 12 }, "ci": { - "local_full_run": { - "contract_tests_skips": [ - "#2 and #5 pull-request-only steps (condition false for a push event)", - "#10 Rust dependency/license/advisory policy: downloads a tool on the host (--allow-download not given)" - ], - "jobs": { - "consolidated-source-preflight": "PASS", - "contract-tests": "INCOMPLETE (7 PASS, 3 SKIPPED)", - "kn-native-integration": "PASS", - "sdk-python310": "PASS", - "unit-tests": "INCOMPLETE (run separately, below)" - }, - "sha": "2f489b1", - "sha256": "f00a94234118ecd74a82f821ab16c378d04f2b484473bc674ed91976291db802", - "status": "INCOMPLETE", - "summary": "~/.local/state/qdl-v2/p184-local-ci-2f489b1/summary.json" - }, - "local_unit_tests": { - "passed": 19, - "sha256": "98c6c157e9c3f446dee7161f43dbba1576d0915591a8217b3bb1a874be7ceaed", - "skipped": 3, - "skips": [ - "#4 Prepare non-root runtime bind mounts (requires sudo; reported, never escalated)", - "#19 trivy image scan and #20 trivy secret/misconfiguration scan: no local trivy image" - ], - "status": "INCOMPLETE", - "summary": "~/.local/state/qdl-v2/p184-local-ci-2f489b1-unit/summary.json" - }, - "local_unit_tests_trivy_rerun": { - "status": "PASS except #4 SKIPPED (requires_sudo)", - "trivy_image": "aquasec/trivy:0.58.1", - "summary": "~/.local/state/qdl-v2/p184-local-ci-2f489b1-unit-r2/summary.json", - "summary_sha256": "f6b6b5e772902541c0171557cb35835583a67d98ca7e2ac645ca239cb8ca49ea" + "sha": "481fd67", + "summary": { + "path": "~/.local/state/qdl-v2/p184-local-ci-481fd67/summary.json", + "sha256": "83e7dda9f3605b52b4dc94d63f51fbc5c7d1d7f06a119e920c20b4a64a47ca0a" }, - "remote_github_ci": "NOT_RUN (DL repository not pushed)", - "runner": "scripts/p184_local_ci.py replaying .github/workflows/ci.yml on an archive of 2f489b1" - }, - "commits_after_source": { - "2f489b1": "G6 reference anchor per product/grade (scripts, tests, plan journal; not in images)" + "status": "INCOMPLETE", + "non_pass_steps": [ + "contract-tests #2 SKIPPED: condition false for local push event: github.event_name == 'pull_request'", + "contract-tests #5 SKIPPED: condition false for local push event: github.event_name == 'pull_request'", + "unit-tests #4 SKIPPED: requires_sudo: not run (host preparation is reported, never escalated)" + ], + "runner": "scripts/p184_local_ci.py --sha 481fd67 --allow-pull --allow-download --trivy-image aquasec/trivy:0.58.1", + "remote_github_ci": "NOT verified: no remote CI run on the final head is cited in this record (the branch is pushed)" }, - "commits_since_previous_evidence_source_bbfb3be": { - "163191c": "plan journal", - "259aeb9": "local CI runner trivy flag fix (tooling)", - "2d11a87": "G12 packet journal", - "2e0de95": "Query: funding settlement publication allowance 60 s re-applied with measurement (runtime: Query)", - "2f489b1": "G6 anchor correction (tooling)", - "413c5b9": "admission: poll an INFLIGHT_CAPACITY deferral (runtime: provider admission in Query/bar edge)", - "59e725a": "previous release evidence + trace transit script", - "627f19d": "G12 harness image set as a parameter (tooling)", - "65ada37": "final gate-campaign tooling G3/G8/G9/G10/G11/G14 + G1-G14 scorer (tooling)", - "6d5da7f": "plan journal (rc8 Query image built from this tree)", - "70bef7d": "SDK 2.2.1: stream transport random replica start (SDK)", - "7188b8b": "G12 harness topic metadata retry, signal teardown (tooling)", - "848d5d2": "driver G6 view age/quality per binding; trace duplicate applies (tooling)", - "8bbcaae": "G12 harness keeps in-container stderr/faulthandler; monitor exit reason (tooling)", - "ba496ff": "G6 scored per the read's own requirement; TS EXECUTION probe (tooling)", - "dadb9c8": "Stream: charge a shared decoded record once against the replica budget (runtime: Stream)", - "e7b2028": "plan journal" + "commits_after_source": {}, + "commits_since_previous_evidence_source_413c5b9": { + "1a161f4": "G12 isolated harness measures G2 (TRADE/QUOTE/MARK) and G3 after recovery (tooling)", + "22ca8df": "G12 isolated Query reads the namespace's own state topics (tooling)", + "139dafe": "IDR-086: a typed backend refusal in a warmup batch keeps its code and retryability (runtime: Query)", + "2d4b88c": "G12 monitor survives a librdkafka crash; monitor gaps make a fault NOT_MEASURED (tooling, the full G12 harness)", + "4c779ac": "batched Subscribe responses for lossless live feeds, SDK 2.3.0 (runtime: Stream gateway, contracts, SDK)", + "4704edd": "release evidence: G12 PASS from the full rc10 isolated run (evidence)", + "f1849e0": "load driver's source SDK carries its source generated contracts (tooling)", + "5ee9463": "IDR-088: serve a proven same hot record at its recorded boundary (runtime: Query)", + "481fd67": "IDR-087: Kafka backlog probe in a supervised spawn child (runtime: bar edge)" }, "consumer_revisions": { "alpha_manifest_jwt_revision": 15, "ts_paper_manifest_jwt_revision": 11, "ts_paper_products": 64 }, + "defects_closed": { + "IDR-086": { + "commit": "139dafe", + "kind": "DL_DEFECT", + "in_images": "Query rc12+ (rc13 released)", + "status": "tested locally (116 tests); in the release Query image" + }, + "IDR-087": { + "commit": "481fd67", + "kind": "DL_RISK confirmed by the rc12 G12 monitor (2 SIGSEGV in list_consumer_group_offsets)", + "in_images": "bar edge rc13", + "status": "tested locally (13 new tests with a real SIGSEGV child + 271 related); no live broker stop of the bar edge" + }, + "IDR-088": { + "commit": "5ee9463", + "kind": "DL_DEFECT", + "in_images": "Query rc12+ (rc13 released)", + "status": "tested locally (181 tests); the step-2 campaign ran it with 0 data-path eligibility losses (G9)" + }, + "register": { + "path": "trading-system INTEGRATION_DEFECT_REGISTER.md (TS 68cd019)", + "sha256": "e1c9a2fcfbcdd078dce07bd9e4d1eed72661cab92b3baab3cd273bc25fbb9b92", + "note": "its status text for IDR-086/088 still says 'needs image rebuild and roll'; the roll happened (rc12 11:55Z Query, rc13 14:26Z all roles); the TS register is not edited by this DL record" + } + }, "deferred_to_ts_release": [ "dr01-deployed", "dr01-wrong-account", @@ -91,58 +84,71 @@ "status": "SHIPPED" }, "g12": { + "verdict": "PASS (isolated, rc12 digests; rc13 differs by IDR-087 only, bar edge not in the isolated stack)", "evidence": { - "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/evidence/verdict.json", - "sha256": "05c0652d320929a13073a2ff1eb57a319f5b4a5817f7dd4f917ca8ae914a48b8" + "path": "~/.local/state/qdl-v2/p184-g12-rc12-20261011/evidence/verdict.json", + "sha256": "20f296719c740dd53a2f32f20b0f18b673b7fa082532c54ee478d7166fbc5e3e" }, - "failed_attempt": { - "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011/evidence/verdict.json", - "sha256": "728b460d8244a36260a428542e93fb6bfec44cd3cae107e684000a8464e1573e" + "teardown": { + "path": "~/.local/state/qdl-v2/p184-g12-rc12-20261011/evidence/teardown.json", + "sha256": "a5fceec5d8f40626ce01dfc83d93514dbd378824957966f944c6920154a3e71b" }, - "stopped_duplicate": { - "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-r2/evidence/verdict.json", - "sha256": "603fe82c761b2633c326fee6b56ad0578710ac4599d0b27f95a9a059846a1ebf" + "image_set": { + "path": "~/.local/state/qdl-v2/p184-g12-rc12-20261011/image-set.json", + "sha256": "838c0503cc99229ac5b1b02393ec8df0fece3abfe07af1ef0fe91d69dc69b85a" }, - "teardown": { - "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/evidence/teardown.json", - "sha256": "f17de5ca9539848e581972fd276809d8b9e5fd3fb5972f848c795f60bd722bb8" + "caught_up_s": { + "broker_stop_start": 224.704, + "projector_restart": 19.4, + "stream_restart": 138.381 }, - "verdict": "PASS (isolated, release digests)" + "monitor": "1718 samples, 2 SIGSEGV sample errors (list_consumer_group_offsets), max gap 2.0 s", + "earlier": "rc10 full run p184-g12-rc10-full3-20261011 PASS (verdict c7b8ea95) and the rc10 attempts: preparation_history.2026-10-11_rc10_413c5b9" }, "g13": { + "verdict": "DEFERRED (41/41 DL cells ok on rc12 digests + final writer 44baacc0; 4 TS steps deferred)", "approval_file": { - "path": "~/.local/state/qdl-v2/p184-g13-inputs/runtime-approval-rc10.md", - "sha256": "551890885673bb569119dc65c218deb0b561c59fec8071bea48a7699eb22d2ce" + "path": "~/.local/state/qdl-v2/p184-g13-inputs/runtime-approval-rc12.md", + "sha256": "9e786bfd2205b5d69977f5fc547a2880d79450bb9856d2a70d1833bbcb638aa8" }, - "approval_note": "written by an agent from the owner's relayed AskUserQuestion answer of 2026-10-11", + "approval_note": "owner chat approval of 2026-10-11, recorded by the coordinating session (not written by the owner)", "runs": [ { - "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/summary.json", - "sha256": "3a3a34e29715a4b67406b2b5e790bed0c9b8d7f0551498d43d0cd9f9c610a09d" + "path": "~/.local/state/qdl-v2/p184-g13-20261011T134532Z/summary.json", + "sha256": "445e05c9de1a0074fb0f94acc950e25536e9a74342edea44e4142a8d6765e55c" }, { - "path": "~/.local/state/qdl-v2/p184-g13-20261011T045533Z/summary.json", - "sha256": "309199f1a24c8625ed98d13d572a79f2cbff754359c8f88ce9a21e9c175c3472" + "path": "~/.local/state/qdl-v2/p184-g13-20261011T140432Z/summary.json", + "sha256": "f0b7b523fa6af40503f009f97d4ebde9119dda228b7902644f93e0629de40a6c" } ], - "verdict": "DEFERRED (41/41 DL cells ok; 4 TS steps deferred)" + "failed_runs": [ + { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T131833Z/summary.json", + "sha256": "0ce63d394a55c0f96a21eee6cba672cbc1390528306474de0ac3fb22992605f8" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T133254Z/summary.json", + "sha256": "205c654d64f1a8d93e5212861f2e32ef14faf58465a47e0213e9a97adb916205" + } + ] }, - "idr_039": "CLOSED: all 12 application roles run images whose revision label is 413c5b9", + "idr_039": "CLOSED: all 12 application roles run images whose revision label is 481fd67", "images": [ { - "digest": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "digest": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", "roles": [ "query_kn_1", "query_kn_2", "binance_bar_edge" ], - "tag": "qdl-v2-python:2.2.3-rc10-413c5b9", - "version_label": "2.2.3-rc10" + "tag": "qdl-v2-python:2.2.3-rc13-481fd67", + "version_label": "2.2.3-rc13" }, { - "digest": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "digest": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", "roles": [ "stream_kn_1", "stream_kn_2", @@ -154,8 +160,8 @@ "ingestor_okx_swap", "ingestor_binance_usdm" ], - "tag": "qdl-v2-rust:2.2.3-rc10-413c5b9", - "version_label": "2.2.3-rc10" + "tag": "qdl-v2-rust:2.2.3-rc13-481fd67", + "version_label": "2.2.3-rc13" } ], "ingestor_configs": { @@ -177,7 +183,8 @@ "sha256": "eb982facfccce78a825c835e5d7ecdebce5dba6e457252a946da69e808ed0460" } }, - "mounted_read_only_at": "/runtime/config.json" + "mounted_read_only_at": "/runtime/config.json", + "reverified": "2026-10-11 ~14:40Z: same sha256 and revisions, read through a --network none alpine:3.20 container; both mounted read-only at /runtime/config.json" }, "kafka_runtime_config": { "applied_at": "2026-10-11T03:40:56Z", @@ -191,11 +198,686 @@ "applied_at.txt": "5f94630c679ebaf17bc248b71b617e00082277a03862ed3ad4e32fab89e76223" }, "receipt_dir": "~/.local/state/qdl-v2/kafka-cleaner-throttle-20261010/", - "rollback": "kafka-configs.sh --alter --delete-config log.cleaner.io.max.bytes.per.second (failure only)" + "rollback": "kafka-configs.sh --alter --delete-config log.cleaner.io.max.bytes.per.second (failure only)", + "reverified_now": false }, "not_a_release_certificate": true, - "owner_decision": "2026-10-11 (AskUserQuestion): release DL; G2/G8 writer throughput to TS phase E first item; gates not widened", + "owner_decision": "the 2026-10-11 decision (release the DL with G2 and G8 FAIL as known limits; writer throughput first in TS phase E) is kept in history. G2 and G8 now PASS in a protocol-met campaign after the writer adopted batched Subscribe; no gate was widened. Publication still needs the owner's decision", "preparation_history": { + "2026-10-11_rc10_413c5b9": { + "branch": "fix/p184-dl-closure", + "catalog": { + "note": "deliberate split (C.19 bar edge checkpoint reads the image catalog); unifying needs its own packet", + "repo_and_image_catalog_revision": 11, + "runtime_catalog": { + "path": "~/.local/state/qdl-v2/releases/v2.2.0-02cd827/runtime/stable-source-bindings.yaml", + "sha256": "fc3a1e1ec283e6c49b671f55cc270bd8f0241d595b241e6c7f5c87f5dc686cc5" + }, + "runtime_catalog_revision": 12 + }, + "ci": { + "local_full_run": { + "contract_tests_skips": [ + "#2 and #5 pull-request-only steps (condition false for a push event)", + "#10 Rust dependency/license/advisory policy: downloads a tool on the host (--allow-download not given)" + ], + "jobs": { + "consolidated-source-preflight": "PASS", + "contract-tests": "INCOMPLETE (7 PASS, 3 SKIPPED)", + "kn-native-integration": "PASS", + "sdk-python310": "PASS", + "unit-tests": "INCOMPLETE (run separately, below)" + }, + "sha": "2f489b1", + "sha256": "f00a94234118ecd74a82f821ab16c378d04f2b484473bc674ed91976291db802", + "status": "INCOMPLETE", + "summary": "~/.local/state/qdl-v2/p184-local-ci-2f489b1/summary.json" + }, + "local_unit_tests": { + "passed": 19, + "sha256": "98c6c157e9c3f446dee7161f43dbba1576d0915591a8217b3bb1a874be7ceaed", + "skipped": 3, + "skips": [ + "#4 Prepare non-root runtime bind mounts (requires sudo; reported, never escalated)", + "#19 trivy image scan and #20 trivy secret/misconfiguration scan: no local trivy image" + ], + "status": "INCOMPLETE", + "summary": "~/.local/state/qdl-v2/p184-local-ci-2f489b1-unit/summary.json" + }, + "local_unit_tests_trivy_rerun": { + "status": "PASS except #4 SKIPPED (requires_sudo)", + "trivy_image": "aquasec/trivy:0.58.1", + "summary": "~/.local/state/qdl-v2/p184-local-ci-2f489b1-unit-r2/summary.json", + "summary_sha256": "f6b6b5e772902541c0171557cb35835583a67d98ca7e2ac645ca239cb8ca49ea" + }, + "remote_github_ci": "NOT_RUN (DL repository not pushed)", + "runner": "scripts/p184_local_ci.py replaying .github/workflows/ci.yml on an archive of 2f489b1" + }, + "commits_after_source": { + "2f489b1": "G6 reference anchor per product/grade (scripts, tests, plan journal; not in images)" + }, + "commits_since_previous_evidence_source_bbfb3be": { + "163191c": "plan journal", + "259aeb9": "local CI runner trivy flag fix (tooling)", + "2d11a87": "G12 packet journal", + "2e0de95": "Query: funding settlement publication allowance 60 s re-applied with measurement (runtime: Query)", + "2f489b1": "G6 anchor correction (tooling)", + "413c5b9": "admission: poll an INFLIGHT_CAPACITY deferral (runtime: provider admission in Query/bar edge)", + "59e725a": "previous release evidence + trace transit script", + "627f19d": "G12 harness image set as a parameter (tooling)", + "65ada37": "final gate-campaign tooling G3/G8/G9/G10/G11/G14 + G1-G14 scorer (tooling)", + "6d5da7f": "plan journal (rc8 Query image built from this tree)", + "70bef7d": "SDK 2.2.1: stream transport random replica start (SDK)", + "7188b8b": "G12 harness topic metadata retry, signal teardown (tooling)", + "848d5d2": "driver G6 view age/quality per binding; trace duplicate applies (tooling)", + "8bbcaae": "G12 harness keeps in-container stderr/faulthandler; monitor exit reason (tooling)", + "ba496ff": "G6 scored per the read's own requirement; TS EXECUTION probe (tooling)", + "dadb9c8": "Stream: charge a shared decoded record once against the replica budget (runtime: Stream)", + "e7b2028": "plan journal" + }, + "consumer_revisions": { + "alpha_manifest_jwt_revision": 15, + "ts_paper_manifest_jwt_revision": 11, + "ts_paper_products": 64 + }, + "deferred_to_ts_release": [ + "dr01-deployed", + "dr01-wrong-account", + "dr02-dr03-alpha", + "dr07-reference" + ], + "funding_settlement_allowance": { + "commit": "2e0de95 (60 s, FUNDING_RATE history only)", + "evidence": { + "binance.jsonl": "1c6a51a6b3246e105d4f25b76ff6bf682760eaa41a216c28273719290f0d54c0", + "okx.jsonl": "66213ff359aaa7259d3a646259071589dda5cac25e3d7410b66dfd1e9ad7111d" + }, + "evidence_dir": "~/.local/state/qdl-v2/funding-publication-20261010/", + "measured": "16:00Z settled row visible OKX BTC/ETH/SOL +10 s, Binance BTC/SOL/ETH +5/+10/+24 s (code comment of 2e0de95)", + "status": "SHIPPED" + }, + "g12": { + "evidence": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/evidence/verdict.json", + "sha256": "05c0652d320929a13073a2ff1eb57a319f5b4a5817f7dd4f917ca8ae914a48b8" + }, + "failed_attempt": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011/evidence/verdict.json", + "sha256": "728b460d8244a36260a428542e93fb6bfec44cd3cae107e684000a8464e1573e" + }, + "stopped_duplicate": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-r2/evidence/verdict.json", + "sha256": "603fe82c761b2633c326fee6b56ad0578710ac4599d0b27f95a9a059846a1ebf" + }, + "teardown": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/evidence/teardown.json", + "sha256": "f17de5ca9539848e581972fd276809d8b9e5fd3fb5972f848c795f60bd722bb8" + }, + "verdict": "PASS (isolated, release digests)" + }, + "g13": { + "approval_file": { + "path": "~/.local/state/qdl-v2/p184-g13-inputs/runtime-approval-rc10.md", + "sha256": "551890885673bb569119dc65c218deb0b561c59fec8071bea48a7699eb22d2ce" + }, + "approval_note": "written by an agent from the owner's relayed AskUserQuestion answer of 2026-10-11", + "runs": [ + { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/summary.json", + "sha256": "3a3a34e29715a4b67406b2b5e790bed0c9b8d7f0551498d43d0cd9f9c610a09d" + }, + { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T045533Z/summary.json", + "sha256": "309199f1a24c8625ed98d13d572a79f2cbff754359c8f88ce9a21e9c175c3472" + } + ], + "verdict": "DEFERRED (41/41 DL cells ok; 4 TS steps deferred)" + }, + "idr_039": "CLOSED: all 12 application roles run images whose revision label is 413c5b9", + "images": [ + { + "digest": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "roles": [ + "query_kn_1", + "query_kn_2", + "binance_bar_edge" + ], + "tag": "qdl-v2-python:2.2.3-rc10-413c5b9", + "version_label": "2.2.3-rc10" + }, + { + "digest": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "roles": [ + "stream_kn_1", + "stream_kn_2", + "market_projector_1", + "market_projector_2", + "rust_core", + "rust_core_2", + "rust_core_3", + "ingestor_okx_swap", + "ingestor_binance_usdm" + ], + "tag": "qdl-v2-rust:2.2.3-rc10-413c5b9", + "version_label": "2.2.3-rc10" + } + ], + "ingestor_configs": { + "book_liveness_proof_ms": "ABSENT (IDR-040 fix 2 OFF)", + "dir": "~/.local/state/qdl-v2/releases/v2.2.3-rc7-ingestor-liveness", + "files": { + "ingestor_binance_usdm.json": { + "bindings": 27, + "config_revision": 19, + "liveness_dead_ms": 4000, + "liveness_ping_ms": 1500, + "sha256": "5833bc4ea35cb4558a931288d34b61a94e99d260f1ba6b0e56596423995930ed" + }, + "ingestor_okx_swap.json": { + "bindings": 103, + "config_revision": 20, + "liveness_dead_ms": 4000, + "liveness_ping_ms": 1500, + "sha256": "eb982facfccce78a825c835e5d7ecdebce5dba6e457252a946da69e808ed0460" + } + }, + "mounted_read_only_at": "/runtime/config.json" + }, + "kafka_runtime_config": { + "applied_at": "2026-10-11T03:40:56Z", + "approval": "owner AskUserQuestion, 2026-10-11 ~03:40Z", + "cause": "md.latest.v2 compaction on kafka1 (NanoCpus 1.25e9) stalled every rust_core commit ~2.06 s at 14:12, 15:29 and 19:09Z on 2026-10-10", + "change": "cluster-wide dynamic default log.cleaner.io.max.bytes.per.second=8388608 (8 MiB/s); no broker restart", + "receipt": { + "README.md": "72cbcf8725fa14df710b571e88149089bf36b1d320fe494e81ed3479102621b7", + "after.out": "8a635a9bd5df3efc07906db02a0fdb45cc799ac502929b46d9680b8ee884302c", + "alter.out": "a59f5deb54f0117897bc08637af9a82ee9b04577953061225471e2e827c0fdc6", + "applied_at.txt": "5f94630c679ebaf17bc248b71b617e00082277a03862ed3ad4e32fab89e76223" + }, + "receipt_dir": "~/.local/state/qdl-v2/kafka-cleaner-throttle-20261010/", + "rollback": "kafka-configs.sh --alter --delete-config log.cleaner.io.max.bytes.per.second (failure only)" + }, + "not_a_release_certificate": true, + "owner_decision": "2026-10-11 (AskUserQuestion): release DL; G2/G8 writer throughput to TS phase E first item; gates not widened", + "previous_published_release": "v2.2.2", + "publication_requires": [ + "push fix/p184-dl-closure; remote CI (contract-tests, sdk-python310, unit-tests, kn-native-integration) on the final head; merge to dev and main", + "annotated tag v2.2.3 linking the CI runs and the scorecard/notes hashes", + "owner publication decision (the 2026-10-11 release decision records G2/G8 as known limits; it does not turn them into PASS)" + ], + "rollback": { + "dl_roles": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/-rollback.compose.json, reverse role order, failure only", + "kafka_config": "--delete-config log.cleaner.io.max.bytes.per.second", + "not_touched": [ + "kafka1-3", + "market_cache", + "stable_redis", + "stable_state_init", + "stable_tls_init", + "Kafka offsets", + "Redis data", + "realm dirs" + ], + "per_role_previous_image": { + "binance_bar_edge": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb (already rc10 since 2026-10-10T18:46Z; its earlier image was rc7d python sha256:e0252f4c96da)", + "projectors, cores, ingestors": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7 (qdl-v2-rust:2.2.3-rc7d-bbfb3be)", + "query_kn_1/2": "sha256:db5ae6752280765d8b85f6df11136d2cc42bcd7e9dba94b41b0897b681b65736 (qdl-v2-python:2.2.3-rc8-6d5da7f)", + "stream_kn_1/2": "sha256:d32e354bd6628cf465a27f6691cadaa91ce4f70f8f3311f0d454e48c8d1a6b40 (qdl-v2-rust:2.2.3-rc9-70bef7d)" + }, + "rc10_receipts": { + "binance_bar_edge": { + "new_image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "ok": true, + "old_image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/binance_bar_edge-receipt-20261011T043042Z.json", + "sha256": "45e5e45fa93a5449a8b8c9cc32d9ffbd453e806aab5298f679cc780b9655864c" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/binance_bar_edge-rollback.compose.json", + "sha256": "4ea4c00e13ac641b1ab6f5592f6d52f2a4cbbc646b86b3b7eb59b7b7a9f1c4bc" + }, + "writer": "READY 64 64" + }, + "ingestor_binance_usdm": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_binance_usdm-receipt-20261011T043515Z.json", + "sha256": "b0adf2ba3b97987591ea1200ca04843d77e00af56600997fbcba8b6313fd53af" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_binance_usdm-rollback.compose.json", + "sha256": "b99fca03be6c60e543fdb2eaa20e0862d3f33e0e5e5841fc1d37ed6059cfbac4" + }, + "writer": "READY 64 64" + }, + "ingestor_okx_swap": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_okx_swap-receipt-20261011T043507Z.json", + "sha256": "dd63d5b677ee20905f21e3f064731b83b0288765872bb85607a96858e9e7e16e" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_okx_swap-rollback.compose.json", + "sha256": "9510b426c5b1716df8e8cab2a6d7538b7f3a3367c55f8ee77ea5e0e350aa3576" + }, + "writer": "READY 64 64" + }, + "market_projector_1": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_1-receipt-20261011T043229Z.json", + "sha256": "3bf2a877f73a8be58be71f15ac1acef4a9a57f30ae2857423b4f7ba41425d9f8" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_1-rollback.compose.json", + "sha256": "3641dffc1de3f068a85745922e6825a3e02e983b7f7c358da2b2ea37ac1017e7" + }, + "writer": "READY 64 64" + }, + "market_projector_2": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_2-receipt-20261011T043301Z.json", + "sha256": "912aa21423170838b750c8f9dc2bbb61f5641ce67f4e10f26e5b9f0a9f135235" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_2-rollback.compose.json", + "sha256": "2370b4394858092d85716d64896788951fa85f8978d4e3daa02f26d7945dabb3" + }, + "writer": "READY 64 64" + }, + "query_kn_1": { + "new_image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "ok": true, + "old_image": "sha256:db5ae6752280765d8b85f6df11136d2cc42bcd7e9dba94b41b0897b681b65736", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_1-receipt-20261011T043001Z.json", + "sha256": "596a0c6fda90089ceb80b0dc9c14d757125ad4e45d2c579719649e7406f7c752" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_1-rollback.compose.json", + "sha256": "f0858eae514a8a65b84cd87d69d7140c0be0ed5f182cb29fb143de4659111d59" + }, + "writer": "READY 64 64" + }, + "query_kn_2": { + "new_image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "ok": true, + "old_image": "sha256:db5ae6752280765d8b85f6df11136d2cc42bcd7e9dba94b41b0897b681b65736", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_2-receipt-20261011T043019Z.json", + "sha256": "847f690f6909ea8fb4e311970a1beb6b821971fd94439e8c9d7a80aedac55b88" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_2-rollback.compose.json", + "sha256": "f4d5dfe4c19608fb018b5b8df29f1f3691aca2f51b5a12b64607fec921939707" + }, + "writer": "READY 64 64" + }, + "rust_core": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core-receipt-20261011T043332Z.json", + "sha256": "78c0aecdcbcc25071f3a6831788c56f541a33a8b72bafc656af92c91ba8c7e49" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core-rollback.compose.json", + "sha256": "fa91883ec8a7ca99faa4e8712e237b3710f5a5e96b216fddc1cf230c3b8489de" + }, + "writer": "READY 64 64" + }, + "rust_core_2": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_2-receipt-20261011T043404Z.json", + "sha256": "0d04e1c7b13ce7b8069fea6a17122fb39f956f638fd7d15cc7f3ec1e7e8e38eb" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_2-rollback.compose.json", + "sha256": "8a52e413347ba8fda3d0bf61993eb8bafc2a0e9ae56d78d063734e71884b6092" + }, + "writer": "READY 64 64" + }, + "rust_core_3": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_3-receipt-20261011T043435Z.json", + "sha256": "eb9fe530eae55107a429a548f2f944783a096f6b96e423126ee82ffa43f852a2" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_3-rollback.compose.json", + "sha256": "20a3bb453625dfa5e3c5cd4aa6acece2efba1c4a010c7d98265ac93cf6922c7f" + }, + "writer": "READY 64 64" + }, + "stream_kn_1": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:d32e354bd6628cf465a27f6691cadaa91ce4f70f8f3311f0d454e48c8d1a6b40", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_1-receipt-20261011T043052Z.json", + "sha256": "6136fd868c7eee7bc7a31681c962160a916a2479a234d040d738009287a549db" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_1-rollback.compose.json", + "sha256": "03b2efe1108b653235c992e9eb7ffee7b3ede2e34f75638fff45a71972300119" + }, + "writer": "READY 64 64" + }, + "stream_kn_2": { + "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ok": true, + "old_image": "sha256:d32e354bd6628cf465a27f6691cadaa91ce4f70f8f3311f0d454e48c8d1a6b40", + "receipt": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_2-receipt-20261011T043127Z.json", + "sha256": "735a9da30c3fcc63179ea1d17b4a05d5c8c4e69b0850daae4397bace86b4e1d6" + }, + "rollback_compose": { + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_2-rollback.compose.json", + "sha256": "ae9e514dc24794fedb5815e44123823a2536f8692949c1ce946cf7dc7db7cae1" + }, + "writer": "READY 64 64" + } + }, + "ts_writer": { + "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/rollback-20261011T042442Z.compose.json", + "sha256": "e9c227757c662e3214447b2c01415e44c900ad63de2435966e34a88d6543795b" + } + }, + "runtime_changed_this_slice": false, + "runtime_inventory": [ + { + "image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", + "name": "market_data_service_p184w", + "oom": false, + "restart_count": 0, + "revision_label": "8b567eb07c9036dbbb42079ed1da90db7e0f5f25", + "started_at": "2026-10-11T04:24:53.613267377Z", + "status": "running" + }, + { + "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "name": "qdl_v2_stable_candidate-binance_bar_edge-1", + "oom": false, + "restart_count": 0, + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:30:45.289811072Z", + "status": "running" + }, + { + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "name": "qdl_v2_stable_candidate-ingestor_binance_usdm-1", + "oom": false, + "restart_count": 0, + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:35:16.489289377Z", + "status": "running" + }, + { + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "name": "qdl_v2_stable_candidate-ingestor_okx_swap-1", + "oom": false, + "restart_count": 0, + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:35:08.381807086Z", + "status": "running" + }, + { + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "name": "qdl_v2_stable_candidate-kafka1-1", + "oom": false, + "restart_count": 0, + "revision_label": null, + "started_at": "2026-10-08T16:23:33.912058555Z", + "status": "running" + }, + { + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "name": "qdl_v2_stable_candidate-kafka2-1", + "oom": false, + "restart_count": 0, + "revision_label": null, + "started_at": "2026-10-08T16:23:34.12626004Z", + "status": "running" + }, + { + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "name": "qdl_v2_stable_candidate-kafka3-1", + "oom": false, + "restart_count": 0, + "revision_label": null, + "started_at": "2026-10-08T16:23:34.326288421Z", + "status": "running" + }, + { + "image": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", + "name": "qdl_v2_stable_candidate-market_cache-1", + "oom": false, + "restart_count": 0, + "revision_label": null, + "started_at": "2026-09-27T05:29:58.000340326Z", + "status": "running" + }, + { + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "name": "qdl_v2_stable_candidate-market_projector_1-1", + "oom": false, + "restart_count": 0, + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:32:30.621173851Z", + "status": "running" + }, + { + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "name": "qdl_v2_stable_candidate-market_projector_2-1", + "oom": false, + "restart_count": 0, + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:33:02.29275674Z", + "status": "running" + }, + { + "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "name": "qdl_v2_stable_candidate-query_kn_1-1", + "oom": false, + "restart_count": 0, + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:30:02.858025475Z", + "status": "running" + }, + { + "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "name": "qdl_v2_stable_candidate-query_kn_2-1", + "oom": false, + "restart_count": 0, + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:30:23.042515056Z", + "status": "running" + }, + { + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "name": "qdl_v2_stable_candidate-rust_core-1", + "oom": false, + "restart_count": 0, + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:33:33.780053453Z", + "status": "running" + }, + { + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "name": "qdl_v2_stable_candidate-rust_core_2-1", + "oom": false, + "restart_count": 0, + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:34:05.235527067Z", + "status": "running" + }, + { + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "name": "qdl_v2_stable_candidate-rust_core_3-1", + "oom": false, + "restart_count": 0, + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:34:36.604572262Z", + "status": "running" + }, + { + "image": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", + "name": "qdl_v2_stable_candidate-stable_redis-1", + "oom": false, + "restart_count": 0, + "revision_label": null, + "started_at": "2026-09-16T06:38:55.588316335Z", + "status": "running" + }, + { + "image": "sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d", + "name": "qdl_v2_stable_candidate-stable_state_init-1", + "oom": false, + "restart_count": 0, + "revision_label": "8ba4165", + "started_at": "2026-09-03T12:36:17.69711623Z", + "status": "exited" + }, + { + "image": "sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d", + "name": "qdl_v2_stable_candidate-stable_tls_init-1", + "oom": false, + "restart_count": 0, + "revision_label": "8ba4165", + "started_at": "2026-09-03T12:36:17.691555089Z", + "status": "exited" + }, + { + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "name": "qdl_v2_stable_candidate-stream_kn_1-1", + "oom": false, + "restart_count": 0, + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:30:56.645942409Z", + "status": "running" + }, + { + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "name": "qdl_v2_stable_candidate-stream_kn_2-1", + "oom": false, + "restart_count": 0, + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "started_at": "2026-10-11T04:31:32.097345347Z", + "status": "running" + } + ], + "runtime_inventory_basis": "docker inspect, read-only, at assembly time", + "schema": "qdl.release-preparation.v1", + "scorecard": { + "path": "upgrade/evidence/releases/v2.2.3/scorecard.json", + "sha256": "bc1d2ded655e65fa73085fff68989d31eeaa8d52ce9255aa8b289ff0d28e323d", + "summary": { + "DEFERRED": [ + "G13" + ], + "FAIL": [ + "G2", + "G8" + ], + "NOT_MEASURED": [ + "G4" + ], + "PASS": [ + "G1", + "G3", + "G5", + "G6", + "G7", + "G9", + "G10", + "G11", + "G12", + "G14" + ] + } + }, + "sdk": { + "builder": "scripts/build_qdl_sdk_release.py, two builds", + "builds": [ + { + "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk221-build-1", + "release_json_sha256": "d2ae9ef67e334540e3cef4d0e1e1b17896762262263e4d6d4fadfeb518c894e6", + "sbom_sha256": "e80dcec6ce588d715e7e5d6bf3aa5449426c80d999f8d69647081047398d9111", + "wheel_sha256": "27db39715c58988128dfec3364919a3ef3bb9425e9bdb4cb7655829ee3f1076f" + }, + { + "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk221-build-2", + "release_json_sha256": "d2ae9ef67e334540e3cef4d0e1e1b17896762262263e4d6d4fadfeb518c894e6", + "sbom_sha256": "e80dcec6ce588d715e7e5d6bf3aa5449426c80d999f8d69647081047398d9111", + "wheel_sha256": "27db39715c58988128dfec3364919a3ef3bb9425e9bdb4cb7655829ee3f1076f" + } + ], + "contains_service_internals": false, + "dependencies": [ + "grpcio>=1.70.0,<2.0.0", + "httpx>=0.28.0,<1.0.0", + "protobuf>=6.31.1,<7.0.0", + "pydantic>=2.0.0,<3.0.0", + "PyJWT[crypto]>=2.15.1,<3.0.0" + ], + "generated_contract_digest": "ae77dc42074fd7cc896a46bde3282837a51b667acee23c4957d37565690856e9", + "name": "qdl-sdk", + "python_requires": ">=3.10", + "reproducible": true, + "reproducible_timestamp": "2020-01-01T00:00:00Z", + "schema": "qdl.sdk.release.v1", + "sdk_source_digest": "fcbd832c1a3e89c66ad625e488ae2a4f80f7baa895abb7b8904febfcb54aa217", + "supersedes": "2.2.0 (wheel d8226f50..., never published; version bumped, not reused)", + "ts_writer_sdk_today": "2.1.0", + "version": "2.2.1", + "wheel": "qdl_sdk-2.2.1-py3-none-any.whl", + "wheel_sha256": "27db39715c58988128dfec3364919a3ef3bb9425e9bdb4cb7655829ee3f1076f" + }, + "source_commit": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "status": "PREPARED_NOT_PUBLISHED", + "ts_writer": { + "container": "market_data_service_p184w", + "depth_note": "depth 0 after the campaign was the coordinator's choice from two comparisons; that comparison is not re-verified here", + "env_campaign_4": { + "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH": "1", + "source": { + "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/inspect-20261011T042442Z.json", + "sha256": "ee6c5ff47f9d4372beb8a8ab52c48cb64f4c9573d8c4837adf875abc5ed7421d" + } + }, + "env_now": { + "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE": "BINANCE=100", + "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH": "0", + "MARKET_DATA_FIXED_WORKER_PINS": "OKX:BTCUSDT=2,OKX:ETHUSDT=0" + }, + "image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", + "image_tag": "local/trading-system:p184-writer-8b567eb", + "sdk": "2.1.0", + "source": "8b567eb07c9036dbbb42079ed1da90db7e0f5f25 (pushed: origin/feat/v2-rust-first-okx-demo contains it)", + "trace_env_present_until_unix_ns": 1791695682881050430 + }, + "version": "2.2.3", + "worktree": "/home/bobby/.worktrees/data-layer-p184-closure", + "superseded_note": "superseded by the final record on 481fd67 (rc13 images, phase E step-2 campaign, G12/G13 on rc12); sha256 of the committed file e6da1e71b8051702501b9e70e24bc2201fb01382bbad38e5099c3b7f60f77a74 (committed in 4704edd). The rc10 rollback compose files it cites were overwritten by later rolls; their recorded sha256 no longer match the files on disk" + }, "2026-10-08": { "active_runtime_roles": [ { @@ -792,240 +1474,296 @@ }, "previous_published_release": "v2.2.2", "publication_requires": [ - "push fix/p184-dl-closure; remote CI (contract-tests, sdk-python310, unit-tests, kn-native-integration) on the final head; merge to dev and main", - "annotated tag v2.2.3 linking the CI runs and the scorecard/notes hashes", - "owner publication decision (the 2026-10-11 release decision records G2/G8 as known limits; it does not turn them into PASS)" + "owner publication decision", + "remote CI (contract-tests, sdk-python310, unit-tests, kn-native-integration) on the final head; merge to dev and main", + "annotated tag v2.2.3 linking the CI runs and the scorecard/notes hashes" ], + "redis_runtime_config": { + "container": "redis_marketdata (TS execution Redis)", + "change": "docker update --cpu-shares 4096 (cpu.weight 303)", + "applied_at": "2026-10-11T08:09Z", + "approval": "owner approved (plan journal entry of 2026-10-11; the receipt itself does not record the approval)", + "receipt": { + "path": "~/.local/state/qdl-v2/redis-cpu-shares-20261011/receipt.txt", + "sha256": "c1b5f23e3f1b42699f5a6b955647ba7a72add9555f1accf539d995bee2b386b1" + }, + "measured": "involuntary context switches 209 -> 160 per 60 s at similar load (receipt)", + "now": "HostConfig.CpuShares 4096, restart count 0 (docker inspect)", + "rollback": "docker update --cpu-shares 0 redis_marketdata (failure only)" + }, "rollback": { "dl_roles": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/-rollback.compose.json, reverse role order, failure only", - "kafka_config": "--delete-config log.cleaner.io.max.bytes.per.second", - "not_touched": [ - "kafka1-3", - "market_cache", - "stable_redis", - "stable_state_init", - "stable_tls_init", - "Kafka offsets", - "Redis data", - "realm dirs" - ], "per_role_previous_image": { - "binance_bar_edge": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb (already rc10 since 2026-10-10T18:46Z; its earlier image was rc7d python sha256:e0252f4c96da)", - "projectors, cores, ingestors": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7 (qdl-v2-rust:2.2.3-rc7d-bbfb3be)", - "query_kn_1/2": "sha256:db5ae6752280765d8b85f6df11136d2cc42bcd7e9dba94b41b0897b681b65736 (qdl-v2-python:2.2.3-rc8-6d5da7f)", - "stream_kn_1/2": "sha256:d32e354bd6628cf465a27f6691cadaa91ce4f70f8f3311f0d454e48c8d1a6b40 (qdl-v2-rust:2.2.3-rc9-70bef7d)" + "query_kn_1/2, binance_bar_edge": "sha256:3b585597570f3f7065c17b0a7f9228a2dad2f71224905e2f105a1754b9d4a75c (qdl-v2-python:2.2.3-rc12-5ee9463)", + "stream_kn_1/2, projectors, cores, ingestors": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f (qdl-v2-rust:2.2.3-rc12-5ee9463)" }, - "rc10_receipts": { - "binance_bar_edge": { - "new_image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "rc13_receipts": { + "query_kn_1": { + "old_image": "sha256:3b585597570f3f7065c17b0a7f9228a2dad2f71224905e2f105a1754b9d4a75c", + "new_image": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", "ok": true, - "old_image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "writer": "READY 64 64", "receipt": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/binance_bar_edge-receipt-20261011T043042Z.json", - "sha256": "45e5e45fa93a5449a8b8c9cc32d9ffbd453e806aab5298f679cc780b9655864c" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_1-receipt-20261011T142616Z.json", + "sha256": "2f52c5c0d59bdbe0f58fd29168c15f144c0e7384034c4c870a79310f3c588d7f" }, "rollback_compose": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/binance_bar_edge-rollback.compose.json", - "sha256": "4ea4c00e13ac641b1ab6f5592f6d52f2a4cbbc646b86b3b7eb59b7b7a9f1c4bc" - }, - "writer": "READY 64 64" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_1-rollback.compose.json", + "sha256": "9b0890cc4ede2af26f2f24b3c876035f1406df1b92060874cc624754d516c5d0" + } }, - "ingestor_binance_usdm": { - "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "query_kn_2": { + "old_image": "sha256:3b585597570f3f7065c17b0a7f9228a2dad2f71224905e2f105a1754b9d4a75c", + "new_image": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", "ok": true, - "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "writer": "READY 64 64", "receipt": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_binance_usdm-receipt-20261011T043515Z.json", - "sha256": "b0adf2ba3b97987591ea1200ca04843d77e00af56600997fbcba8b6313fd53af" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_2-receipt-20261011T142636Z.json", + "sha256": "3e426beecd699ed7c27871f53b0acb727d8e06c24fbeef5b9394b813471f51a8" }, "rollback_compose": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_binance_usdm-rollback.compose.json", - "sha256": "b99fca03be6c60e543fdb2eaa20e0862d3f33e0e5e5841fc1d37ed6059cfbac4" - }, - "writer": "READY 64 64" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_2-rollback.compose.json", + "sha256": "12f9e6888399ca47e34a44bd3e801dfb2d42451a3a16ddd0d922f63e607776b3" + } }, - "ingestor_okx_swap": { - "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "binance_bar_edge": { + "old_image": "sha256:3b585597570f3f7065c17b0a7f9228a2dad2f71224905e2f105a1754b9d4a75c", + "new_image": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", "ok": true, - "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "writer": "READY 64 64", "receipt": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_okx_swap-receipt-20261011T043507Z.json", - "sha256": "dd63d5b677ee20905f21e3f064731b83b0288765872bb85607a96858e9e7e16e" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/binance_bar_edge-receipt-20261011T142651Z.json", + "sha256": "b500719e493be645c0ccd055119fef7ee54eb56bf50f2dd2ad91a583fd260176" }, "rollback_compose": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_okx_swap-rollback.compose.json", - "sha256": "9510b426c5b1716df8e8cab2a6d7538b7f3a3367c55f8ee77ea5e0e350aa3576" - }, - "writer": "READY 64 64" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/binance_bar_edge-rollback.compose.json", + "sha256": "2a38cba97b6431e4aa0cd0b4f0ef438e0849dd565e202d2f0c129f93b5c7fbf5" + } }, - "market_projector_1": { - "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "stream_kn_1": { + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "ok": true, - "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "writer": "READY 64 64", "receipt": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_1-receipt-20261011T043229Z.json", - "sha256": "3bf2a877f73a8be58be71f15ac1acef4a9a57f30ae2857423b4f7ba41425d9f8" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_1-receipt-20261011T142658Z.json", + "sha256": "2e4cccbb41827a9cfb4302ef1fb1a3e1c11705eafe3de5a852ba5bc742b40a27" }, "rollback_compose": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_1-rollback.compose.json", - "sha256": "3641dffc1de3f068a85745922e6825a3e02e983b7f7c358da2b2ea37ac1017e7" - }, - "writer": "READY 64 64" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_1-rollback.compose.json", + "sha256": "d3e8343d7ef27cfc7109375f074b6ceb9994c1d3567f6a21e477b73067edabce" + } }, - "market_projector_2": { - "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "stream_kn_2": { + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "ok": true, - "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "writer": "READY 64 64", "receipt": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_2-receipt-20261011T043301Z.json", - "sha256": "912aa21423170838b750c8f9dc2bbb61f5641ce67f4e10f26e5b9f0a9f135235" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_2-receipt-20261011T142733Z.json", + "sha256": "23b2b1450fe6ab7e7c08a6441c008eac21ffdb21aec67337a957484486e239cc" }, "rollback_compose": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_2-rollback.compose.json", - "sha256": "2370b4394858092d85716d64896788951fa85f8978d4e3daa02f26d7945dabb3" - }, - "writer": "READY 64 64" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_2-rollback.compose.json", + "sha256": "83b98c51e1ac20580225bd34aa022931ea98bd0739cbb323351a09dda73e2953" + } }, - "query_kn_1": { - "new_image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "market_projector_1": { + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "ok": true, - "old_image": "sha256:db5ae6752280765d8b85f6df11136d2cc42bcd7e9dba94b41b0897b681b65736", + "writer": "READY 64 64", "receipt": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_1-receipt-20261011T043001Z.json", - "sha256": "596a0c6fda90089ceb80b0dc9c14d757125ad4e45d2c579719649e7406f7c752" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_1-receipt-20261011T142828Z.json", + "sha256": "bd260d07921c8e1cf26cc961caf40d4fa76fecc932971416240dc6042e0bc4f9" }, "rollback_compose": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_1-rollback.compose.json", - "sha256": "f0858eae514a8a65b84cd87d69d7140c0be0ed5f182cb29fb143de4659111d59" - }, - "writer": "READY 64 64" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_1-rollback.compose.json", + "sha256": "c957bcd8db42b34333c25a5544c3d72ba1ddbc260d6699dd818f110f4b72a5f3" + } }, - "query_kn_2": { - "new_image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "market_projector_2": { + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "ok": true, - "old_image": "sha256:db5ae6752280765d8b85f6df11136d2cc42bcd7e9dba94b41b0897b681b65736", + "writer": "READY 64 64", "receipt": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_2-receipt-20261011T043019Z.json", - "sha256": "847f690f6909ea8fb4e311970a1beb6b821971fd94439e8c9d7a80aedac55b88" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_2-receipt-20261011T142859Z.json", + "sha256": "c4eca21e6c6dda96690ff3f1acbb0a218eaeb43def65286796a76f20bf2f16f0" }, "rollback_compose": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/query_kn_2-rollback.compose.json", - "sha256": "f4d5dfe4c19608fb018b5b8df29f1f3691aca2f51b5a12b64607fec921939707" - }, - "writer": "READY 64 64" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/market_projector_2-rollback.compose.json", + "sha256": "0810b910c897c7b066efa21682d2120bc03bbbfc9598e39559fd15bd4ea8218d" + } }, "rust_core": { - "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "ok": true, - "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "writer": "READY 64 64", "receipt": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core-receipt-20261011T043332Z.json", - "sha256": "78c0aecdcbcc25071f3a6831788c56f541a33a8b72bafc656af92c91ba8c7e49" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core-receipt-20261011T142931Z.json", + "sha256": "1c6264a1f62c2d359d59db996e0067b8725c5a8bde705e9a02067abaa28616b1" }, "rollback_compose": { "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core-rollback.compose.json", - "sha256": "fa91883ec8a7ca99faa4e8712e237b3710f5a5e96b216fddc1cf230c3b8489de" - }, - "writer": "READY 64 64" + "sha256": "ccdc9b50198a86f277e1e37796c9110c744d9728881825bc9ef7a7aaa3ac2fef" + } }, "rust_core_2": { - "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "ok": true, - "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "writer": "READY 64 64", "receipt": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_2-receipt-20261011T043404Z.json", - "sha256": "0d04e1c7b13ce7b8069fea6a17122fb39f956f638fd7d15cc7f3ec1e7e8e38eb" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_2-receipt-20261011T143002Z.json", + "sha256": "50c9fabf26712102dc631979519efcaa32768c30e95bb20dd6e22400bb30b617" }, "rollback_compose": { "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_2-rollback.compose.json", - "sha256": "8a52e413347ba8fda3d0bf61993eb8bafc2a0e9ae56d78d063734e71884b6092" - }, - "writer": "READY 64 64" + "sha256": "b88cab5e8b718193549dab8848d6588bfc39877a39536e639ed6c2c3a4f3cde5" + } }, "rust_core_3": { - "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "ok": true, - "old_image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "writer": "READY 64 64", "receipt": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_3-receipt-20261011T043435Z.json", - "sha256": "eb9fe530eae55107a429a548f2f944783a096f6b96e423126ee82ffa43f852a2" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_3-receipt-20261011T143055Z.json", + "sha256": "470127e16b7c09676283d30b412b839a38fc1cd56fc4b10eb0058f102210d0a2" }, "rollback_compose": { "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/rust_core_3-rollback.compose.json", - "sha256": "20a3bb453625dfa5e3c5cd4aa6acece2efba1c4a010c7d98265ac93cf6922c7f" - }, - "writer": "READY 64 64" + "sha256": "d7b1ac8c128ea57446141eb49bf685359b223be6e9e2ae3001cc7fb4c6111b5f" + } }, - "stream_kn_1": { - "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ingestor_okx_swap": { + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "ok": true, - "old_image": "sha256:d32e354bd6628cf465a27f6691cadaa91ce4f70f8f3311f0d454e48c8d1a6b40", + "writer": "READY 64 64", "receipt": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_1-receipt-20261011T043052Z.json", - "sha256": "6136fd868c7eee7bc7a31681c962160a916a2479a234d040d738009287a549db" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_okx_swap-receipt-20261011T143126Z.json", + "sha256": "165650757049e537b5fa57ebd3be74e27a2617a4fde92a831dfcd41500095a8f" }, "rollback_compose": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_1-rollback.compose.json", - "sha256": "03b2efe1108b653235c992e9eb7ffee7b3ede2e34f75638fff45a71972300119" - }, - "writer": "READY 64 64" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_okx_swap-rollback.compose.json", + "sha256": "d101a5198bfcb93562580f90d2a02665811fe0efd4fe8be0e34fd8399e2c9d49" + } }, - "stream_kn_2": { - "new_image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "ingestor_binance_usdm": { + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "ok": true, - "old_image": "sha256:d32e354bd6628cf465a27f6691cadaa91ce4f70f8f3311f0d454e48c8d1a6b40", + "writer": "READY 64 64", "receipt": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_2-receipt-20261011T043127Z.json", - "sha256": "735a9da30c3fcc63179ea1d17b4a05d5c8c4e69b0850daae4397bace86b4e1d6" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_binance_usdm-receipt-20261011T143134Z.json", + "sha256": "42574d8a10dd2e2fc7d1d0c7013b942e89b03e23f595e52a03154a4aa9ef08e8" }, "rollback_compose": { - "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/stream_kn_2-rollback.compose.json", - "sha256": "ae9e514dc24794fedb5815e44123823a2536f8692949c1ce946cf7dc7db7cae1" - }, - "writer": "READY 64 64" + "path": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/ingestor_binance_usdm-rollback.compose.json", + "sha256": "fb8655b4e3efb7731bf1c5c747ad1052f23661cf282aa7df9aef97a079d2c30a" + } } }, + "kafka_config": "--delete-config log.cleaner.io.max.bytes.per.second", + "redis_cpu_shares": "docker update --cpu-shares 0 redis_marketdata", "ts_writer": { - "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/rollback-20261011T042442Z.compose.json", - "sha256": "e9c227757c662e3214447b2c01415e44c900ad63de2435966e34a88d6543795b" - } + "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/rollback-20261011T134403Z.compose.json", + "sha256": "d673be459f0bf8b6ba5b41cf361237f99116abfc018ccfbdceea3205d7415697" + }, + "ts_writer_note": "rolls the writer back to 64347a94 (a3035c8) with knob 64; TS owns this rollback", + "not_touched": [ + "kafka1-3", + "market_cache", + "stable_redis", + "stable_state_init", + "stable_tls_init", + "Kafka offsets", + "Redis data", + "realm dirs" + ] }, "runtime_changed_this_slice": false, + "runtime_changes_approval_note": "approvals for these rolls were given to the coordinating session; this record verifies the receipts and docker inspect, not the approval wording", + "runtime_changes_since_previous_record": [ + { + "at": "2026-10-11T08:09Z", + "change": "redis_marketdata cpu-shares 4096", + "receipt": "redis-cpu-shares-20261011/receipt.txt" + }, + { + "at": "2026-10-11T10:33:37Z-10:35:18Z", + "change": "Query rc11 b128de1e (4704edd), Stream rc11 ddb2ad19 (4704edd, batched Subscribe)", + "receipt": "rollout-rc6/{query_kn_*,stream_kn_*}-receipt-20261011T103*.json" + }, + { + "at": "2026-10-11T10:36Z", + "change": "TS writer 64347a94 (a3035c8, SDK 2.3.0)", + "receipt": "trading-system campaign-roll/receipt-20261011T103624Z.json" + }, + { + "at": "2026-10-11T11:55:46Z-11:56:04Z", + "change": "Query rc12 3b585597 (5ee9463, IDR-086 + IDR-088)", + "receipt": "rollout-rc6/query_kn_*-receipt-20261011T1155*.json" + }, + { + "at": "2026-10-11T12:34Z", + "change": "TS writer knob DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE=64 (same image)", + "receipt": "trading-system campaign-roll/receipt-20261011T123412Z.json" + }, + { + "at": "2026-10-11T13:12:54Z-13:18:03Z", + "change": "rc12 (5ee9463) on all 12 application roles: python 3b585597, rust ebc7b794", + "receipt": "rollout-rc6/-receipt-20261011T131*.json" + }, + { + "at": "2026-10-11T13:44Z", + "change": "TS writer 44baacc0 (a37ff4d), knob 64 kept", + "receipt": "trading-system campaign-roll/receipt-20261011T134403Z.json" + }, + { + "at": "2026-10-11T14:26:16Z-14:31:42Z", + "change": "rc13 (481fd67) on all 12 application roles, writer READY 64/64 after every step", + "receipt": "rollout-rc6/-receipt-20261011T14*.json" + } + ], "runtime_inventory": [ { - "image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", + "image": "sha256:44baacc0a3ff95f71029d230e48e1b7282627f26de9bd99e13d1148862e09068", "name": "market_data_service_p184w", "oom": false, "restart_count": 0, - "revision_label": "8b567eb07c9036dbbb42079ed1da90db7e0f5f25", - "started_at": "2026-10-11T04:24:53.613267377Z", + "revision_label": "a37ff4dcef418947b89578c1870726cb03db4fd3", + "started_at": "2026-10-11T13:44:14.494194887Z", "status": "running" }, { - "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "image": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", "name": "qdl_v2_stable_candidate-binance_bar_edge-1", "oom": false, "restart_count": 0, - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "started_at": "2026-10-11T04:30:45.289811072Z", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "started_at": "2026-10-11T14:26:52.382759721Z", "status": "running" }, { - "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "name": "qdl_v2_stable_candidate-ingestor_binance_usdm-1", "oom": false, "restart_count": 0, - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "started_at": "2026-10-11T04:35:16.489289377Z", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "started_at": "2026-10-11T14:31:35.59199243Z", "status": "running" }, { - "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "name": "qdl_v2_stable_candidate-ingestor_okx_swap-1", "oom": false, "restart_count": 0, - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "started_at": "2026-10-11T04:35:08.381807086Z", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "started_at": "2026-10-11T14:31:27.839139393Z", "status": "running" }, { @@ -1065,66 +1803,66 @@ "status": "running" }, { - "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "name": "qdl_v2_stable_candidate-market_projector_1-1", "oom": false, "restart_count": 0, - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "started_at": "2026-10-11T04:32:30.621173851Z", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "started_at": "2026-10-11T14:28:29.10000567Z", "status": "running" }, { - "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "name": "qdl_v2_stable_candidate-market_projector_2-1", "oom": false, "restart_count": 0, - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "started_at": "2026-10-11T04:33:02.29275674Z", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "started_at": "2026-10-11T14:29:00.699668563Z", "status": "running" }, { - "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "image": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", "name": "qdl_v2_stable_candidate-query_kn_1-1", "oom": false, "restart_count": 0, - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "started_at": "2026-10-11T04:30:02.858025475Z", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "started_at": "2026-10-11T14:26:17.881199252Z", "status": "running" }, { - "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "image": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", "name": "qdl_v2_stable_candidate-query_kn_2-1", "oom": false, "restart_count": 0, - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "started_at": "2026-10-11T04:30:23.042515056Z", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "started_at": "2026-10-11T14:26:38.472711377Z", "status": "running" }, { - "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "name": "qdl_v2_stable_candidate-rust_core-1", "oom": false, "restart_count": 0, - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "started_at": "2026-10-11T04:33:33.780053453Z", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "started_at": "2026-10-11T14:29:32.163633283Z", "status": "running" }, { - "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "name": "qdl_v2_stable_candidate-rust_core_2-1", "oom": false, "restart_count": 0, - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "started_at": "2026-10-11T04:34:05.235527067Z", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "started_at": "2026-10-11T14:30:03.591093881Z", "status": "running" }, { - "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "name": "qdl_v2_stable_candidate-rust_core_3-1", "oom": false, "restart_count": 0, - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "started_at": "2026-10-11T04:34:36.604572262Z", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "started_at": "2026-10-11T14:30:56.132123836Z", "status": "running" }, { @@ -1155,68 +1893,67 @@ "status": "exited" }, { - "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "name": "qdl_v2_stable_candidate-stream_kn_1-1", "oom": false, "restart_count": 0, - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "started_at": "2026-10-11T04:30:56.645942409Z", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "started_at": "2026-10-11T14:27:02.493271338Z", "status": "running" }, { - "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", "name": "qdl_v2_stable_candidate-stream_kn_2-1", "oom": false, "restart_count": 0, - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "started_at": "2026-10-11T04:31:32.097345347Z", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "started_at": "2026-10-11T14:27:36.229558337Z", "status": "running" } ], - "runtime_inventory_basis": "docker inspect, read-only, at assembly time", + "runtime_inventory_basis": "docker inspect, read-only, at assembly time (2026-10-11 ~14:40Z)", "schema": "qdl.release-preparation.v1", "scorecard": { "path": "upgrade/evidence/releases/v2.2.3/scorecard.json", - "sha256": "bc1d2ded655e65fa73085fff68989d31eeaa8d52ce9255aa8b289ff0d28e323d", + "sha256": "23b79a01d81d378074043050810e5cb34f01d01e919561366e63d0ccd1137a81", "summary": { - "DEFERRED": [ - "G13" - ], - "FAIL": [ - "G2", - "G8" - ], - "NOT_MEASURED": [ - "G4" - ], "PASS": [ "G1", + "G2", "G3", "G5", "G6", "G7", + "G8", "G9", "G10", "G11", "G12", "G14" + ], + "FAIL": [], + "NOT_MEASURED": [ + "G4" + ], + "DEFERRED": [ + "G13" ] } }, "sdk": { - "builder": "scripts/build_qdl_sdk_release.py, two builds", + "builder": "scripts/build_qdl_sdk_release.py from a git archive of 481fd67, two builds, image qdl-v2-python:2.2.3-rc13-481fd67, --network none, non-root, read-only source", "builds": [ { - "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk221-build-1", - "release_json_sha256": "d2ae9ef67e334540e3cef4d0e1e1b17896762262263e4d6d4fadfeb518c894e6", - "sbom_sha256": "e80dcec6ce588d715e7e5d6bf3aa5449426c80d999f8d69647081047398d9111", - "wheel_sha256": "27db39715c58988128dfec3364919a3ef3bb9425e9bdb4cb7655829ee3f1076f" + "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk230-build-1", + "wheel_sha256": "b9c74157db2a2d147564d72b65bbbafc9a7d3d055bccee67645b831470ad38c3", + "sbom_sha256": "4c69c98d56f29962d9981d1acd517cf44717aa4c224f9df5b2e3e913ceff488b", + "release_json_sha256": "218b45b0250c95c1b273b9b809afc234984ab5fffdefcfdf541ebd635111ea15" }, { - "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk221-build-2", - "release_json_sha256": "d2ae9ef67e334540e3cef4d0e1e1b17896762262263e4d6d4fadfeb518c894e6", - "sbom_sha256": "e80dcec6ce588d715e7e5d6bf3aa5449426c80d999f8d69647081047398d9111", - "wheel_sha256": "27db39715c58988128dfec3364919a3ef3bb9425e9bdb4cb7655829ee3f1076f" + "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk230-build-2", + "wheel_sha256": "b9c74157db2a2d147564d72b65bbbafc9a7d3d055bccee67645b831470ad38c3", + "sbom_sha256": "4c69c98d56f29962d9981d1acd517cf44717aa4c224f9df5b2e3e913ceff488b", + "release_json_sha256": "218b45b0250c95c1b273b9b809afc234984ab5fffdefcfdf541ebd635111ea15" } ], "contains_service_internals": false, @@ -1227,41 +1964,35 @@ "pydantic>=2.0.0,<3.0.0", "PyJWT[crypto]>=2.15.1,<3.0.0" ], - "generated_contract_digest": "ae77dc42074fd7cc896a46bde3282837a51b667acee23c4957d37565690856e9", + "generated_contract_digest": "2c59805220f06caf1a19b6df5d4ec7bf44282a780d265e0dd667b9abd4d7c1d1", "name": "qdl-sdk", "python_requires": ">=3.10", "reproducible": true, "reproducible_timestamp": "2020-01-01T00:00:00Z", "schema": "qdl.sdk.release.v1", - "sdk_source_digest": "fcbd832c1a3e89c66ad625e488ae2a4f80f7baa895abb7b8904febfcb54aa217", - "supersedes": "2.2.0 (wheel d8226f50..., never published; version bumped, not reused)", - "ts_writer_sdk_today": "2.1.0", - "version": "2.2.1", - "wheel": "qdl_sdk-2.2.1-py3-none-any.whl", - "wheel_sha256": "27db39715c58988128dfec3364919a3ef3bb9425e9bdb4cb7655829ee3f1076f" + "sdk_source_digest": "e0e5d044c5edfffb1084bc84b69d4cf1898a4221004ca720c0c9dd02fb0d895b", + "supersedes": "2.2.1 (wheel 27db3971..., never published; 2.2.0 before it)", + "source_unchanged_since": "4c779ac (git diff 4c779ac 481fd67 -- qdl_sdk generated/python contracts scripts/build_qdl_sdk_release.py: empty); the wheel equals the TS-vendored vendor/qdl_sdk-2.3.0-py3-none-any.whl (same sha256)", + "ts_writer_sdk": "2.3.0 (writer images 64347a94 and 44baacc0, qdl_sdk.__version__ read with --network none)", + "version": "2.3.0", + "wheel": "qdl_sdk-2.3.0-py3-none-any.whl", + "wheel_sha256": "b9c74157db2a2d147564d72b65bbbafc9a7d3d055bccee67645b831470ad38c3" }, - "source_commit": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "source_commit": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", "status": "PREPARED_NOT_PUBLISHED", "ts_writer": { "container": "market_data_service_p184w", - "depth_note": "depth 0 after the campaign was the coordinator's choice from two comparisons; that comparison is not re-verified here", - "env_campaign_4": { - "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH": "1", - "source": { - "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/inspect-20261011T042442Z.json", - "sha256": "ee6c5ff47f9d4372beb8a8ab52c48cb64f4c9573d8c4837adf875abc5ed7421d" - } - }, + "image": "sha256:44baacc0a3ff95f71029d230e48e1b7282627f26de9bd99e13d1148862e09068", + "image_tag": "local/trading-system:p184-writer-a37ff4d", + "source": "a37ff4dcef418947b89578c1870726cb03db4fd3 (pushed: origin/feat/v2-rust-first-okx-demo contains it)", + "sdk": "2.3.0", "env_now": { - "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE": "BINANCE=100", + "DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE": "64", "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH": "0", - "MARKET_DATA_FIXED_WORKER_PINS": "OKX:BTCUSDT=2,OKX:ETHUSDT=0" + "MARKET_DATA_FIXED_WORKER_PINS": "OKX:BTCUSDT=2,OKX:ETHUSDT=0", + "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE": "BINANCE=100" }, - "image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", - "image_tag": "local/trading-system:p184-writer-8b567eb", - "sdk": "2.1.0", - "source": "8b567eb07c9036dbbb42079ed1da90db7e0f5f25 (pushed: origin/feat/v2-rust-first-okx-demo contains it)", - "trace_env_present_until_unix_ns": 1791695682881050430 + "campaign_writer": "sha256:64347a94a3ed682d155690f3afaed5aaeba6d8feb8e8febcaeb7dc33dea488d9 (a3035c8); a37ff4d adds only the bridge reconnect control log in runtime code" }, "version": "2.2.3", "worktree": "/home/bobby/.worktrees/data-layer-p184-closure" diff --git a/upgrade/evidence/releases/v2.2.3/scorecard.json b/upgrade/evidence/releases/v2.2.3/scorecard.json index 1f34f253..609699e6 100644 --- a/upgrade/evidence/releases/v2.2.3/scorecard.json +++ b/upgrade/evidence/releases/v2.2.3/scorecard.json @@ -2,7 +2,7 @@ "schema": "qdl.release-scorecard.v1", "release": "2.2.3", "status": "PREPARED_NOT_PUBLISHED", - "source_commit": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "source_commit": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", "gate_table": { "path": "trading-system upgrade/p184-phase-d-gates.md (FROZEN 2026-10-09)", "ts_commit": "31cf663f5c7c5384ec3dd296061582dee2463c41", @@ -15,92 +15,117 @@ "scoring_rule": "steady state >= 30 min at the stage-100 workload plus one burst window (gate table 'Measurement rules'); a measurement outside this rule can FAIL a gate but cannot PASS it. Verdicts: PASS, FAIL, NOT_MEASURED, DEFERRED; a NOT_MEASURED is never upgraded. 'observed' records the value against the threshold when the measurement does not meet the protocol.", "runtime_identity": { "release_images": { - "python": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb (qdl-v2-python:2.2.3-rc10-413c5b9)", - "rust": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5 (qdl-v2-rust:2.2.3-rc10-413c5b9)", - "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "python": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677 (qdl-v2-python:2.2.3-rc13-481fd67)", + "rust": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad (qdl-v2-rust:2.2.3-rc13-481fd67)", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "version_label": "2.2.3-rc13", "dl_application_roles_on_release_digests": 12, - "rolled": "2026-10-11T04:30:01Z-04:35:15Z, one role at a time, writer READY 64/64 after every step" + "rolled": "2026-10-11T14:26:16Z-14:31:42Z, one role at a time, writer READY 64/64 after every step (rollout-rc6/-receipt-20261011T14*.json)" }, - "campaign_4_measurement_identity": { - "note": "campaign 4 ran on the pre-rc10 mix; G12 and G13 ran on the release digests", + "campaign_measurement_identity": { + "note": "the step-2 campaign ran 12:35-13:10Z on the mix below, before the rc12 (13:12-13:18Z) and rc13 (14:26-14:31Z) rolls; diffs are `git diff --stat 481fd67` without tests, scripts, docs and upgrade/", "query_kn_1/2": { - "image": "sha256:db5ae6752280765d8b85f6df11136d2cc42bcd7e9dba94b41b0897b681b65736", - "tag": "qdl-v2-python:2.2.3-rc8-6d5da7f", - "source": "6d5da7f", - "diff_to_413c5b9": "qdl/admission/edge.py +9 (INFLIGHT_CAPACITY wait polls); qdl_sdk 2.2.1 (version string, GrpcStreamTransport random start; service code imports only qdl_sdk.models/reference); contracts doc; the Rust gateway source in the tree is not run by Query", - "equivalence": "NOT source-identical: provider-admitted reads (OKX admitted REST and reference, Binance admitted klines) differ when an INFLIGHT_CAPACITY deferral occurs; other Query paths are source-identical" + "image": "sha256:3b585597570f3f7065c17b0a7f9228a2dad2f71224905e2f105a1754b9d4a75c", + "tag": "qdl-v2-python:2.2.3-rc12-5ee9463", + "source": "5ee9463", + "rolled": "11:55:46Z / 11:56:04Z", + "diff_to_481fd67": "qdl/runtime/history_backpressure.py +50/-7, qdl/runtime/isolated_probe.py +141 (IDR-087); imported only by qdl/runtime/stable_bar_edge.py", + "equivalence": "source-identical for every Query path" }, "stream_kn_1/2": { - "image": "sha256:d32e354bd6628cf465a27f6691cadaa91ce4f70f8f3311f0d454e48c8d1a6b40", - "tag": "qdl-v2-rust:2.2.3-rc9-70bef7d", - "source": "70bef7d", - "diff_to_413c5b9": "qdl/admission/edge.py only (not in the Rust build); rust/, generated/rust, Cargo.toml/lock and config unchanged", - "equivalence": "source-identical for the Stream binary" + "image": "sha256:ddb2ad19d6106baee8c35f1660f15e4dec8a220a26f91e52651fbd77a54b58fd", + "tag": "qdl-v2-rust:2.2.3-rc11-4704edd", + "source": "4704edd", + "rolled": "10:34:18Z / 10:35:18Z", + "diff_to_481fd67": "Python only (IDR-087 files, qdl/runtime/kn_query_backend.py IDR-088); rust/, generated/rust, Cargo files, config: empty diff", + "equivalence": "source-identical for the Stream binary (includes batched Subscribe 4c779ac)" }, "binance_bar_edge": { "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", "tag": "qdl-v2-python:2.2.3-rc10-413c5b9", "source": "413c5b9", - "equivalence": "the release image itself (rolled 2026-10-10T18:46Z)" + "diff_to_481fd67": "on its import path (stable_bar_edge.py and its direct qdl imports, checked one level): the IDR-087 files only; the other Python diffs (qdl/query/service.py IDR-086, kn_query_backend.py IDR-088, qdl_sdk 2.3.0, generated query_pb2) are not imported by it", + "equivalence": "NOT source-identical: the measured bar edge ran the in-process Kafka backlog probe that IDR-087 moves into a supervised child; the change acts only when the probe crashes or hangs" }, "rust_core x3, market_projector_1/2, ingestor_okx_swap, ingestor_binance_usdm": { - "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", - "tag": "qdl-v2-rust:2.2.3-rc7d-bbfb3be", - "source": "bbfb3be", - "diff_to_413c5b9": "rust/qdl-stream-gateway only (dadb9c8), a workspace member no other crate depends on; Cargo.toml/lock, generated/rust, config and Dockerfile unchanged", + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "tag": "qdl-v2-rust:2.2.3-rc10-413c5b9", + "source": "413c5b9", + "diff_to_481fd67": "rust/qdl-stream-gateway (no other crate depends on it) and generated/rust/qdl/query/v2/qdl.query.v2.rs +16 (two additive prost fields, SubscribeRequest.max_records_per_response and SubscribeResponse.records), included by rust/qdl-contracts, which every crate depends on; no code outside qdl-stream-gateway references SubscribeRequest/SubscribeResponse (git grep)", "equivalence": "source-equivalent for the core, projector and ingestor binaries (rebuilt, not byte-compared)" } }, - "ts_writer": { + "ts_writer_campaign": { "container": "market_data_service_p184w", - "image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", - "source": "8b567eb07c9036dbbb42079ed1da90db7e0f5f25", - "sdk": "2.1.0", - "env_campaign_4": { - "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH": "1", + "image": "sha256:64347a94a3ed682d155690f3afaed5aaeba6d8feb8e8febcaeb7dc33dea488d9", + "tag": "local/trading-system:p184-writer-a3035c8", + "source": "a3035c8cf257cd5b3935224612e55e93ba1f4bf5", + "sdk": "2.3.0", + "env": { + "DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE": "64", + "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH": "0", "MARKET_DATA_FIXED_WORKER_PINS": "OKX:BTCUSDT=2,OKX:ETHUSDT=0", "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE": "BINANCE=100" }, "env_source": { - "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/inspect-20261011T042442Z.json", - "sha256": "ee6c5ff47f9d4372beb8a8ab52c48cb64f4c9573d8c4837adf875abc5ed7421d" - }, - "after_campaign": "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH=0 since 2026-10-11T04:24:53Z (docker inspect)" + "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/trace-20261011T123412Z.compose.json", + "sha256": "8b28aeeba20abb9657b06176b509d8732c65d2b07cea47af17881048ba04b58c" + } + }, + "ts_writer_final": { + "image": "sha256:44baacc0a3ff95f71029d230e48e1b7282627f26de9bd99e13d1148862e09068", + "tag": "local/trading-system:p184-writer-a37ff4d", + "source": "a37ff4dcef418947b89578c1870726cb03db4fd3", + "sdk": "2.3.0", + "started": "2026-10-11T13:44:14Z (docker inspect)", + "diff_from_campaign_writer": "`git diff --stat a3035c8 a37ff4d`: runtime code services/market_data/data_layer_bridge.py +17 only (a bridge-reopened slice logs one RECONNECTED origin=bridge control); the rest is tests, scripts/p184_dr05_isolated.py, the register and the plan", + "env": "same four keys and values as the campaign writer (docker inspect)" }, - "kafka_runtime_config": { - "change": "cluster dynamic default log.cleaner.io.max.bytes.per.second=8388608", - "applied_at": "2026-10-11T03:40:56Z (before campaign 4)", - "receipt": { - "path": "~/.local/state/qdl-v2/kafka-cleaner-throttle-20261010/after.out", - "sha256": "8a635a9bd5df3efc07906db02a0fdb45cc799ac502929b46d9680b8ee884302c" + "runtime_config": { + "kafka": { + "change": "cluster dynamic default log.cleaner.io.max.bytes.per.second=8388608", + "applied_at": "2026-10-11T03:40:56Z", + "receipt": { + "path": "~/.local/state/qdl-v2/kafka-cleaner-throttle-20261010/after.out", + "sha256": "8a635a9bd5df3efc07906db02a0fdb45cc799ac502929b46d9680b8ee884302c" + }, + "reverified_now": false + }, + "redis_marketdata": { + "change": "cpu-shares 4096 (cpu.weight 303)", + "applied_at": "2026-10-11T08:09Z", + "receipt": { + "path": "~/.local/state/qdl-v2/redis-cpu-shares-20261011/receipt.txt", + "sha256": "c1b5f23e3f1b42699f5a6b955647ba7a72add9555f1accf539d995bee2b386b1" + }, + "now": "HostConfig.CpuShares 4096 (docker inspect)" } } }, "campaign": { - "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z", + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z", + "role": "phase E step 2 (batched Subscribe knob 64)", "scorecard": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/scorecard.json", - "sha256": "0133670885374c1a280a8fcd4c34f6dd704e29904d5f492bbb08853398eb12e4" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/scorecard.json", + "sha256": "747e476489a3ef9807e7ade9bd347b00a8e56226a5bc85a99b8fa03b5e7fa58e" }, "campaign_json": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/campaign.json", - "sha256": "a7bbf80073b1796d55fdcffdcd87476de28bde4622559ab55b3ee536c35b64e9" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/campaign.json", + "sha256": "b926a20e3acbe5dcdde965ec7147fe3ee46b1d5a6fe0ca95795d36614aa71c2b" }, "protocol_met": true, "protocol_problems": [], "load_window_unix_s": [ - 1791690245.4681032, - 1791692045.4681032 + 1791722221.9021854, + 1791724021.9021854 ], - "tooling_source": "2f489b186d8d590ffa59b6dae2e41caa2fa400ad", + "load_window_utc": "2026-10-11T12:37:01Z-13:07:01Z", + "tooling_source": "5ee94636a3b41d4d482eadbc9dd80f51e126cf84", "tool_sha256": "39c22d78d37dbacac0042c85a6bfafafdd3dc70a9b21179c053f30a9d6cc19f3", "demanded_slices": 64, "writer_restarts": 0, + "startup_bounded": "PASS (193 retries, 70.41 s)", "campaign_summary": { - "FAIL": [ - "G2", - "G8" - ], "NOT_MEASURED": [ "G4", "G12", @@ -108,35 +133,103 @@ ], "PASS": [ "G1", + "G2", "G3", "G5", "G6", "G7", + "G8", "G9", "G10", "G11", "G14" ] }, - "g12_g13": "the campaign scorer marks G12/G13 NOT_MEASURED by design; they are scored from their own receipts below" + "g12_g13": "the campaign scorer marks G12/G13 NOT_MEASURED by design; they are scored from their own receipts" + }, + "one_variable_comparison": { + "step_1": { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T115743Z", + "scorecard": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T115743Z/scorecard.json", + "sha256": "6ae2945de18852fc2fd0b514d5679f0c269ce444719285f4a5800adba71990d9" + }, + "protocol_met": true, + "summary": { + "FAIL": [ + "G2" + ], + "NOT_MEASURED": [ + "G4", + "G12", + "G13" + ], + "PASS": [ + "G1", + "G3", + "G5", + "G6", + "G7", + "G8", + "G9", + "G10", + "G11", + "G14" + ] + }, + "writer_knob": "DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE absent (default 1)", + "env_source": { + "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/trace-20261011T115631Z.compose.json", + "sha256": "df76c9748d103238171cf1b0bb53faac203324d62eb0c1c08c769213b50cf56a" + }, + "load_window_utc": "2026-10-11T11:59:15Z-12:29:15Z", + "G14_dl_cpu_mean_sum_vcpu": 5.18 + }, + "step_2": { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z", + "writer_knob": "64", + "summary": { + "NOT_MEASURED": [ + "G4", + "G12", + "G13" + ], + "PASS": [ + "G1", + "G2", + "G3", + "G5", + "G6", + "G7", + "G8", + "G9", + "G10", + "G11", + "G14" + ] + }, + "G14_dl_cpu_mean_sum_vcpu": 5.08 + }, + "same_in_both": "DL images (Query rc12, Stream rc11, rest rc10), writer image 64347a94, depth 0, pins, MARK delay, Kafka throttle, Redis cpu-shares", + "result": "G2 FAIL -> PASS (step 1 also outside on QUOTE max 1067.2/1283.3 ms); G8 PASS in both", + "caveat": "TRADE volume step 2 66303/12492 vs step 1 92415/26567 (TRADE BINANCE/OKX n)" }, "summary": { "PASS": [ "G1", + "G2", "G3", "G5", "G6", "G7", + "G8", "G9", "G10", "G11", "G12", "G14" ], - "FAIL": [ - "G2", - "G8" - ], + "FAIL": [], "NOT_MEASURED": [ "G4" ], @@ -144,65 +237,65 @@ "G13" ] }, - "owner_decision": "2026-10-11 (AskUserQuestion): release the DL with G2 and G8 FAIL recorded; writer throughput (SDK batch decode, Rust projection, EVAL chunking) is the first item of TS phase E; no gate is widened. This is not a PASS and does not change any verdict", + "owner_decision": "the 2026-10-11 decision (release the DL with G2 and G8 FAIL as known limits; writer throughput first in TS phase E) is kept in history. G2 and G8 now PASS in a protocol-met campaign after the writer adopted batched Subscribe; no gate was widened. Publication still needs the owner's decision", "gates": [ { "id": "G1", "measured": [ { "feed_venue": "BOOK_DELTA|BINANCE", - "max": 1736.7, - "n": 83994, + "max": 570.4, + "n": 82692, "observed": "INSIDE", - "p50": 108.4, - "p95": 177.4, - "p99": 334.1, + "p50": 103.9, + "p95": 138.8, + "p99": 190.3, "split_p95_ms": { - "ingest_to_sdk_ms": 90.8, - "sdk_to_reply_ms": 62.6, - "venue_to_ingest_ms": 30.1 + "ingest_to_sdk_ms": 76.4, + "sdk_to_reply_ms": 42.4, + "venue_to_ingest_ms": 29.3 } }, { "feed_venue": "BOOK_DELTA|OKX", - "max": 1844.2, - "n": 97565, + "max": 585.9, + "n": 94817, "observed": "INSIDE", - "p50": 115.3, - "p95": 191.5, - "p99": 380.8, + "p50": 106.2, + "p95": 146.4, + "p99": 199.5, "split_p95_ms": { - "ingest_to_sdk_ms": 93.7, - "sdk_to_reply_ms": 72.5, - "venue_to_ingest_ms": 34.3 + "ingest_to_sdk_ms": 76.7, + "sdk_to_reply_ms": 47.5, + "venue_to_ingest_ms": 35.1 } }, { "feed_venue": "BOOK_SNAPSHOT|BINANCE", - "max": 1211.4, - "n": 9667, + "max": 571.2, + "n": 8894, "observed": "INSIDE", - "p50": 123.0, - "p95": 245.6, - "p99": 423.9, + "p50": 115.1, + "p95": 176.5, + "p99": 242.4, "split_p95_ms": { - "ingest_to_sdk_ms": 114.7, - "sdk_to_reply_ms": 117.7, - "venue_to_ingest_ms": 31.6 + "ingest_to_sdk_ms": 81.1, + "sdk_to_reply_ms": 75.8, + "venue_to_ingest_ms": 28.8 } }, { "feed_venue": "BOOK_SNAPSHOT|OKX", - "max": 1144.5, - "n": 11562, + "max": 503.2, + "n": 10873, "observed": "INSIDE", - "p50": 127.7, - "p95": 236.7, - "p99": 419.0, + "p50": 118.5, + "p95": 177.6, + "p99": 237.8, "split_p95_ms": { - "ingest_to_sdk_ms": 105.9, - "sdk_to_reply_ms": 121.7, - "venue_to_ingest_ms": 34.6 + "ingest_to_sdk_ms": 80.3, + "sdk_to_reply_ms": 74.4, + "venue_to_ingest_ms": 35.5 } } ], @@ -211,12 +304,12 @@ "rule": "p95 <= 500 ms, p99 <= 1000 ms (n >= 100), max < 2000 ms per feed/venue", "sources": { "transit": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/transit-load.json", - "sha256": "d1376db1a50db991df5e9e9e80212463ad095640ee00a67ab673e473145d73f6" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/trace/transit-load.json", + "sha256": "bd1ccac05fad1728d73bbab833a629d43266756c3575e09082a498e1ee7c78c7" }, "mark": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/mark-load.json", - "sha256": "7ac870af08414f19f852af5851928a3e3030fd5898ade38b1ff8e417ec6850a6" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/trace/mark-load.json", + "sha256": "b6a5198116d7916e32b8b05f48e2bf45798b7863aeb64e32247b874c442f8c9f" } }, "verdict": "PASS" @@ -226,120 +319,168 @@ "measured": [ { "feed_venue": "TRADE|BINANCE", - "max": 5338.8, - "n": 160328, - "observed": "OUTSIDE", - "p50": 153.7, - "p95": 1562.1, - "p99": 4326.4, + "max": 583.1, + "n": 66303, + "observed": "INSIDE", + "p50": 118.2, + "p95": 243.4, + "p99": 381.4, "split_p95_ms": { - "ingest_to_sdk_ms": 1430.7, - "sdk_to_reply_ms": 118.2, - "venue_to_ingest_ms": 39.6 + "ingest_to_sdk_ms": 154.8, + "sdk_to_reply_ms": 77.6, + "venue_to_ingest_ms": 36.4 } }, { "feed_venue": "TRADE|OKX", - "max": 4301.8, - "n": 39292, - "observed": "OUTSIDE", - "p50": 138.3, - "p95": 2084.2, - "p99": 3782.9, + "max": 495.5, + "n": 12492, + "observed": "INSIDE", + "p50": 114.4, + "p95": 210.3, + "p99": 330.7, "split_p95_ms": { - "ingest_to_sdk_ms": 1866.1, - "sdk_to_reply_ms": 152.8, - "venue_to_ingest_ms": 72.6 + "ingest_to_sdk_ms": 115.1, + "sdk_to_reply_ms": 77.8, + "venue_to_ingest_ms": 37.1 } }, { "feed_venue": "QUOTE|BINANCE", - "max": 1439.2, - "n": 116367, - "observed": "OUTSIDE", - "p50": 125.9, - "p95": 188.0, - "p99": 300.2, + "max": 531.5, + "n": 103534, + "observed": "INSIDE", + "p50": 122.5, + "p95": 162.2, + "p99": 199.4, "split_p95_ms": { - "ingest_to_sdk_ms": 120.0, - "sdk_to_reply_ms": 46.9, - "venue_to_ingest_ms": 29.7 + "ingest_to_sdk_ms": 107.2, + "sdk_to_reply_ms": 34.5, + "venue_to_ingest_ms": 28.8 } }, { "feed_venue": "QUOTE|OKX", - "max": 1340.6, - "n": 98174, - "observed": "OUTSIDE", - "p50": 127.2, - "p95": 207.1, - "p99": 339.3, + "max": 532.0, + "n": 79722, + "observed": "INSIDE", + "p50": 126.0, + "p95": 169.1, + "p99": 217.2, "split_p95_ms": { - "ingest_to_sdk_ms": 125.0, - "sdk_to_reply_ms": 58.4, - "venue_to_ingest_ms": 30.5 + "ingest_to_sdk_ms": 110.4, + "sdk_to_reply_ms": 37.9, + "venue_to_ingest_ms": 29.8 } }, { "feed_venue": "MARK_INDEX_PRICE|BINANCE", - "max": 1125.4, - "n": 8989, - "observed": "OUTSIDE", - "p50": 302.0, - "p95": 396.7, - "p99": 537.2 + "max": 803.1, + "n": 9000, + "observed": "INSIDE", + "p50": 304.1, + "p95": 360.3, + "p99": 419.8 }, { "feed_venue": "MARK_INDEX_PRICE|OKX", - "max": 1227.8, - "n": 21437, - "observed": "OUTSIDE", - "p50": 234.9, - "p95": 348.5, - "p99": 411.0 + "max": 712.7, + "n": 21514, + "observed": "INSIDE", + "p50": 231.1, + "p95": 337.4, + "p99": 371.2 } ], "quantity": "end-to-end to TS cache, new-event transit", - "reason": "outside: TRADE|BINANCE, TRADE|OKX, QUOTE|BINANCE, QUOTE|OKX; outside: MARK|BINANCE, MARK|OKX", + "reason": "every row inside", "rule": "TRADE p95 <= 500 ms and max < 1500 ms; QUOTE and MARK p95 <= 500 ms and max < 1000 ms", "sources": { "transit": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/transit-load.json", - "sha256": "d1376db1a50db991df5e9e9e80212463ad095640ee00a67ab673e473145d73f6" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/trace/transit-load.json", + "sha256": "bd1ccac05fad1728d73bbab833a629d43266756c3575e09082a498e1ee7c78c7" }, "mark": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/mark-load.json", - "sha256": "7ac870af08414f19f852af5851928a3e3030fd5898ade38b1ff8e417ec6850a6" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/trace/mark-load.json", + "sha256": "b6a5198116d7916e32b8b05f48e2bf45798b7863aeb64e32247b874c442f8c9f" } }, - "verdict": "FAIL", - "attribution": { - "basis": "read-only per-second analysis of the campaign-4 delivery trace (214 trace parts, load window of the campaign scorecard), records APPLIED with SDK_STREAM_RETURN and a venue clock; run in a disposable --network none container with the trace mounted read-only", - "seconds_in_window": 1801, - "seconds_max_e2e_lt_500_ms": 1699, - "seconds_max_e2e_gt_1500_ms": 28, - "seconds_gt_1500_where_every_record_over_1500_is_TRADE": 25, - "seconds_gt_1500_that_also_hold_BOOK_DELTA_over_1500": "3 BINANCE / 2 OKX second-feed hits (G1 max still < 2000 ms)", - "largest_TRADE_burst": "4274 TRADE records in one source second on one slice (BINANCE:USD_M:ETHUSDT) on one writer worker", - "max_applied_TRADE_per_worker_second": 1138, - "TRADE_records_over_1500_ms_by_slice": { - "BINANCE:USD_M:ETHUSDT": 4844, - "BINANCE:USD_M:SOLUSDT": 2287, - "OKX:SWAP:ETH-USDT-SWAP": 2211, - "BINANCE:USD_M:BNBUSDT": 642, - "BINANCE:USD_M:BTCUSDT": 572, - "OKX:SWAP:SOL-USDT-SWAP": 469 + "verdict": "PASS", + "one_variable_comparison": { + "variable": "TS writer DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE (batched Subscribe, SDK 2.3.0): absent = default 1 in step 1, 64 in step 2; same DL images, writer image, pins, depth and MARK delay", + "step_1": { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T115743Z", + "verdict": "FAIL", + "rows": { + "TRADE|BINANCE": { + "n": 92415, + "p95": 801.2, + "max": 3284.0 + }, + "TRADE|OKX": { + "n": 26567, + "p95": 817.2, + "max": 3208.8 + }, + "QUOTE|BINANCE": { + "n": 108047, + "p95": 166.5, + "max": 1067.2 + }, + "QUOTE|OKX": { + "n": 94647, + "p95": 173.3, + "max": 1283.3 + }, + "MARK_INDEX_PRICE|BINANCE": { + "n": 8998, + "p95": 359.3, + "max": 801.0 + }, + "MARK_INDEX_PRICE|OKX": { + "n": 21505, + "p95": 338.5, + "max": 820.9 + } + } }, - "segments_p95_ms_TRADE_BINANCE": { - "ingest_to_sdk_ms": 1430.7, - "sdk_to_reply_ms": 118.2, - "venue_to_ingest_ms": 39.6 + "step_2": { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z", + "verdict": "PASS", + "rows": { + "TRADE|BINANCE": { + "n": 66303, + "p95": 243.4, + "max": 583.1 + }, + "TRADE|OKX": { + "n": 12492, + "p95": 210.3, + "max": 495.5 + }, + "QUOTE|BINANCE": { + "n": 103534, + "p95": 162.2, + "max": 531.5 + }, + "QUOTE|OKX": { + "n": 79722, + "p95": 169.1, + "max": 532.0 + }, + "MARK_INDEX_PRICE|BINANCE": { + "n": 9000, + "p95": 360.3, + "max": 803.1 + }, + "MARK_INDEX_PRICE|OKX": { + "n": 21514, + "p95": 337.4, + "max": 712.7 + } + } }, - "canonical_visibility": "G4 venue -> canonical TRADE p95 108.2-136.6 ms in all 5 windows of the same campaign", - "TRADE_volume": "TRADE BINANCE n 160328 against 69966 / 74061 in the 2026-10-10 campaigns 171129Z / 174944Z (about 2.2x)", - "integrity": "G10 in the same campaign: 0 gaps, 0 duplicates applied, 0 replay-expired, 0 overflow", - "reading": "the tail is single-slice TRADE bursts queued behind one writer worker (arrival up to ~4.3k rec/s vs at most ~1.1k applied rec/s per worker); the DL canonical path stays inside G4. Stream-side dispatch/queue metrics for this campaign were not retained in a file and are not cited here.", - "owner_decision": "2026-10-11 (AskUserQuestion): release DL; writer throughput (SDK batch decode, Rust projection, EVAL chunking) moves to TS phase E as its first item; the gate is not widened" + "caveat": "TRADE volume differed between the two runs: TRADE BINANCE/OKX n 66303/12492 in step 2 against 92415/26567 in step 1, and 160328/39292 in campaign 4. Part of the improvement may come from the lower TRADE volume; the step-2 PASS is a measurement at that volume, not a capacity claim at the campaign-4 volume" } }, { @@ -347,30 +488,30 @@ "measured": { "per_venue": { "BINANCE": { - "max": 3007, - "min": 109, + "max": 3330, + "min": 124, "n": 160, - "p50": 468, - "p95": 1754, - "p99": 2861 + "p50": 540, + "p95": 2680, + "p99": 3244 }, "OKX": { - "max": 1822, - "min": 316, + "max": 1546, + "min": 328, "n": 160, - "p50": 1034, - "p95": 1535, - "p99": 1822 + "p50": 1046, + "p95": 1493, + "p99": 1540 } }, "poll_ms": 250, "pooled": { - "max": 3007, - "min": 109, + "max": 3330, + "min": 124, "n": 320, - "p50": 758, - "p95": 1535, - "p99": 2435 + "p50": 799, + "p95": 1914, + "p99": 3134 }, "read_errors": 0, "unresolved_slices": [], @@ -381,8 +522,8 @@ "rule": "p95 <= 5 s, max < 60 s", "sources": { "g3": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g3/g3-summary.json", - "sha256": "a15a368f3c05dc0de7f0d072bf2518574fea0168d3c3ca73b21558575ae13574" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g3/g3-summary.json", + "sha256": "fafca1be177364c2032dd18afd5f7e2c8d67468403b2de3fde2658ee69274e07" } }, "verdict": "PASS" @@ -392,112 +533,112 @@ "inverse_rows": [ { "book_delta|OKX_INVERSE|": { - "max": 108.9, - "n": 106, - "p95": 101.6 + "max": 118.2, + "n": 114, + "p95": 106.2 }, "book_snapshot|OKX_INVERSE|": { - "max": 156.4, - "n": 112, - "p95": 102.3 + "max": 139.7, + "n": 126, + "p95": 111.3 }, "mark_index_price|OKX_INVERSE|": { - "max": 1793.0, + "max": 1908.8, "n": 110, - "p95": 1310.1 + "p95": 1270.6 }, "quote|OKX_INVERSE|": { - "max": 147.5, - "n": 64, - "p95": 139.0 + "max": 161.0, + "n": 112, + "p95": 132.6 } }, { "book_delta|OKX_INVERSE|": { - "max": 198.5, + "max": 113.2, "n": 112, - "p95": 121.0 + "p95": 103.5 }, "book_snapshot|OKX_INVERSE|": { - "max": 198.7, - "n": 124, - "p95": 143.3 + "max": 133.4, + "n": 118, + "p95": 105.0 }, "mark_index_price|OKX_INVERSE|": { - "max": 2041.9, + "max": 2085.7, "n": 109, - "p95": 1248.0 + "p95": 1279.6 }, "quote|OKX_INVERSE|": { - "max": 298.2, - "n": 96, - "p95": 154.8 + "max": 146.6, + "n": 83, + "p95": 134.6 } }, { "book_delta|OKX_INVERSE|": { - "max": 126.2, + "max": 119.1, "n": 112, - "p95": 107.2 + "p95": 99.7 }, "book_snapshot|OKX_INVERSE|": { - "max": 147.9, - "n": 120, - "p95": 109.6 + "max": 127.0, + "n": 119, + "p95": 106.1 }, "mark_index_price|OKX_INVERSE|": { - "max": 1610.9, + "max": 1940.5, "n": 110, - "p95": 1076.3 + "p95": 1353.8 }, "quote|OKX_INVERSE|": { - "max": 217.1, - "n": 70, - "p95": 145.4 + "max": 156.3, + "n": 101, + "p95": 137.3 } }, { "book_delta|OKX_INVERSE|": { - "max": 232.3, - "n": 125, - "p95": 121.9 + "max": 128.9, + "n": 111, + "p95": 106.8 }, "book_snapshot|OKX_INVERSE|": { - "max": 199.3, - "n": 142, - "p95": 126.8 + "max": 122.4, + "n": 119, + "p95": 108.2 }, "mark_index_price|OKX_INVERSE|": { - "max": 1932.8, - "n": 123, - "p95": 1079.1 + "max": 1860.3, + "n": 110, + "p95": 1301.1 }, "quote|OKX_INVERSE|": { - "max": 289.6, - "n": 99, - "p95": 162.8 + "max": 147.4, + "n": 56, + "p95": 135.2 } }, { "book_delta|OKX_INVERSE|": { - "max": 340.5, - "n": 113, - "p95": 118.7 + "max": 135.4, + "n": 161, + "p95": 105.6 }, "book_snapshot|OKX_INVERSE|": { - "max": 510.7, - "n": 128, - "p95": 141.4 + "max": 122.6, + "n": 171, + "p95": 106.5 }, "mark_index_price|OKX_INVERSE|": { - "max": 2032.2, - "n": 111, - "p95": 1166.9 + "max": 2095.4, + "n": 160, + "p95": 1333.4 }, "quote|OKX_INVERSE|": { - "max": 385.5, - "n": 81, - "p95": 148.6 + "max": 152.0, + "n": 106, + "p95": 136.9 } } ], @@ -505,15 +646,15 @@ { "limit_p95_ms": 40728.7, "n_per_window": [ - 3, 0, + 3, 0, 0, 0 ], "p95_ms_per_window": [ - 7106.7, null, + 7107.5, null, null, null @@ -528,14 +669,14 @@ 10, 10, 10, - 10 + 15 ], "p95_ms_per_window": [ - 127.5, - 106.6, - 116.9, - 140.6, - 94.3 + 116.7, + 112.2, + 101.3, + 136.1, + 106.9 ], "row": "bar|BINANCE|1m", "verdict": "PASS" @@ -546,15 +687,15 @@ 0, 0, 0, - 1, - 0 + 0, + 2 ], "p95_ms_per_window": [ null, null, null, - 11641.8, - null + null, + 9673.5 ], "row": "bar|BINANCE|30m", "verdict": "PASS" @@ -562,18 +703,18 @@ { "limit_p95_ms": 27273.2, "n_per_window": [ + 5, 1, 5, 5, - 0, - 5 + 2 ], "p95_ms_per_window": [ - 7099.6, - 6853.7, - 6838.9, - null, - 6840.1 + 6873.3, + 7062.6, + 6919.2, + 6856.1, + 9674.1 ], "row": "bar|BINANCE|3m", "verdict": "PASS" @@ -581,18 +722,18 @@ { "limit_p95_ms": 73817.2, "n_per_window": [ - 6, 8, 7, - 6, - 9 + 7, + 7, + 10 ], "p95_ms_per_window": [ - 61364.0, - 61099.9, - 61105.6, - 61223.5, - 61238.3 + 61317.8, + 61235.8, + 61077.2, + 61223.6, + 61281.1 ], "row": "bar|OKX|1m", "verdict": "PASS" @@ -603,15 +744,15 @@ 0, 0, 0, - 1, - 0 + 0, + 2 ], "p95_ms_per_window": [ null, null, null, - 1801148.6, - null + null, + 1801249.7 ], "row": "bar|OKX|30m", "verdict": "PASS" @@ -619,18 +760,18 @@ { "limit_p95_ms": 217582.1, "n_per_window": [ - 3, + 2, 2, 3, - 1, - 1 + 3, + 0 ], "p95_ms_per_window": [ - 181246.0, - 181058.9, - 180978.3, - 180970.0, - 181286.0 + 181285.1, + 180941.1, + 181203.5, + 181161.2, + null ], "row": "bar|OKX|3m", "verdict": "PASS" @@ -638,18 +779,18 @@ { "limit_p95_ms": 263.3, "n_per_window": [ - 763, - 790, - 791, - 880, - 783 + 747, + 754, + 744, + 753, + 1108 ], "p95_ms_per_window": [ - 101.4, - 130.3, - 106.6, - 128.7, - 124.1 + 104.1, + 100.3, + 103.1, + 100.7, + 103.7 ], "row": "book_delta|BINANCE|", "verdict": "PASS" @@ -657,18 +798,18 @@ { "limit_p95_ms": 289.1, "n_per_window": [ - 1085, - 1076, - 1081, - 1210, - 1077 + 1027, + 1017, + 974, + 996, + 1478 ], "p95_ms_per_window": [ - 106.1, - 132.9, - 112.1, - 132.7, - 125.6 + 110.8, + 107.8, + 108.2, + 103.8, + 109.3 ], "row": "book_delta|OKX|", "verdict": "PASS" @@ -676,18 +817,18 @@ { "limit_p95_ms": 260.0, "n_per_window": [ - 1000, - 1329, - 1200, - 1687, - 1258 + 878, + 823, + 829, + 759, + 1294 ], "p95_ms_per_window": [ - 103.4, - 155.1, - 108.4, - 140.5, - 148.4 + 108.8, + 107.8, + 105.8, + 100.4, + 107.6 ], "row": "book_snapshot|BINANCE|", "verdict": "PASS" @@ -695,18 +836,18 @@ { "limit_p95_ms": 270.1, "n_per_window": [ - 1713, - 2375, - 2293, - 2712, - 2434 + 1369, + 1039, + 937, + 837, + 2064 ], "p95_ms_per_window": [ - 107.2, - 152.5, - 113.4, - 142.6, - 140.5 + 118.7, + 112.1, + 111.9, + 105.4, + 112.2 ], "row": "book_snapshot|OKX|", "verdict": "PASS" @@ -714,18 +855,18 @@ { "limit_p95_ms": 370.9, "n_per_window": [ - 374, - 390, + 372, + 381, 379, - 435, - 382 + 380, + 539 ], "p95_ms_per_window": [ - 198.4, - 220.2, - 202.8, - 234.8, - 207.7 + 173.5, + 173.8, + 172.8, + 174.2, + 181.3 ], "row": "mark_index_price|BINANCE|", "verdict": "PASS" @@ -733,18 +874,18 @@ { "limit_p95_ms": 1586.6, "n_per_window": [ + 659, 662, - 660, - 662, - 739, - 662 + 661, + 663, + 959 ], "p95_ms_per_window": [ - 1432.2, - 1281.4, - 1372.2, - 1109.2, - 1155.3 + 1438.7, + 1520.7, + 1459.6, + 1457.9, + 1268.9 ], "row": "mark_index_price|OKX|", "verdict": "PASS" @@ -752,18 +893,18 @@ { "limit_p95_ms": 292.7, "n_per_window": [ + 561, + 558, + 558, 554, - 578, - 564, - 640, - 571 + 805 ], "p95_ms_per_window": [ - 131.6, - 152.5, - 137.5, - 158.3, - 152.0 + 137.1, + 132.9, + 135.6, + 130.1, + 135.4 ], "row": "quote|BINANCE|", "verdict": "PASS" @@ -771,18 +912,18 @@ { "limit_p95_ms": 283.3, "n_per_window": [ - 601, - 652, - 625, - 727, - 634 + 633, + 614, + 616, + 566, + 865 ], "p95_ms_per_window": [ - 134.8, - 155.5, - 135.6, - 155.2, - 152.0 + 136.4, + 134.7, + 138.3, + 133.8, + 133.6 ], "row": "quote|OKX|", "verdict": "PASS" @@ -790,18 +931,18 @@ { "limit_p95_ms": 269.3, "n_per_window": [ - 319, - 439, - 401, - 522, - 447 + 378, + 365, + 372, + 336, + 518 ], "p95_ms_per_window": [ - 108.2, - 133.2, - 109.0, - 125.8, - 118.9 + 100.9, + 102.9, + 104.9, + 99.8, + 105.3 ], "row": "trade|BINANCE|", "verdict": "PASS" @@ -809,56 +950,54 @@ { "limit_p95_ms": 257.4, "n_per_window": [ - 217, - 353, - 295, - 423, - 351 + 227, + 238, + 177, + 199, + 335 ], "p95_ms_per_window": [ - 111.5, - 125.4, - 110.8, - 136.6, - 129.6 + 109.8, + 106.2, + 111.9, + 102.1, + 107.0 ], "row": "trade|OKX|", "verdict": "PASS" } ], "quantity": "delivery lag, venue -> canonical (read_committed visibility, upper bound)", - "reason": "observed rows without a frozen baseline: bar|BINANCE|2h, bar|BINANCE|4h, bar|BINANCE|5m, bar|OKX|15m, bar|OKX|4h, bar|OKX|5m, book_delta|OKX_INVERSE|, book_snapshot|OKX_INVERSE|, mark_index_price|OKX_INVERSE|, quote|OKX_INVERSE|", + "reason": "observed rows without a frozen baseline: bar|BINANCE|1h, bar|BINANCE|5m, bar|OKX|1h, bar|OKX|5m, book_delta|OKX_INVERSE|, book_snapshot|OKX_INVERSE|, mark_index_price|OKX_INVERSE|, quote|OKX_INVERSE|", "rule": "candidate p95 <= frozen baseline p95 x 1.2 per row; missing coverage blocks the gate", "verdict": "NOT_MEASURED", "sources": { - "g4": [ - { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-1-summary.json", - "sha256": "02a554c56d9215c9e48678df5389305f1acd1a108da1d34bf46bca455a581e90" - }, - { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-2-summary.json", - "sha256": "42aa78a1bb21566c65510e53afe6b4db0afd0af52d9abc6b21b4e538af88a587" - }, - { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-3-summary.json", - "sha256": "013e6929f3bb978d16acb37e6e64e0d4961971c7c7cf43e9dbf5f52cc617e57e" - }, - { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-4-summary.json", - "sha256": "86494b6575b97c101459cb6e98257f3c3d4f1a98fae62b2482d69d940079288d" - }, - { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-5-summary.json", - "sha256": "0111526b3ba62b40c8bfd5c4dd30cb5d9f6dd7a4ea14db0043beebbeb54ca2fe" - } - ] + "g4_1": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g4-1-summary.json", + "sha256": "717bf3b8be882a4801468b61fdb4a094861265b135d53bade2c04661c12e3c3c" + }, + "g4_2": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g4-2-summary.json", + "sha256": "3f080361fb9b7d9d9fb3e4af204fd8e7c67a7e199f70aee9d97a849fc936e52a" + }, + "g4_3": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g4-3-summary.json", + "sha256": "21b72b595926f62e15d712e58a0f7bdacfed3a53bacb067ab66ce72f4a494f6b" + }, + "g4_4": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g4-4-summary.json", + "sha256": "0b68e97fb3744c57dba8b337f230c2e0d33fae5c9442a15d6b0809d6a6d6f322" + }, + "g4_5": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g4-5-summary.json", + "sha256": "eeaa9a683f48c73c1bb891bbc2abb188168d871865bb79e2a4517ccd6a9a619f" + } }, "baselined_rows": { "rows": 17, "pass": 17 }, - "note": "every row with a frozen baseline PASS in every window it was sampled; the gate stays NOT_MEASURED because observed rows have no frozen baseline and no baseline is added after measuring" + "note": "every row with a frozen baseline PASS in every window it was sampled (bar|BINANCE|15m and bar|BINANCE|30m and bar|OKX|30m were sampled in one window each); the gate stays NOT_MEASURED because observed rows have no frozen baseline and no baseline is added after measuring. OKX bar latency is measured from bar open, as in the baseline" }, { "id": "G5", @@ -866,10 +1005,10 @@ { "count": 355, "gate": "latency:BAR_LATEST:BINANCE", - "max": 396.156, - "p50": 21.767, - "p95": 45.867, - "p99": 237.583, + "max": 267.327, + "p50": 15.524, + "p95": 25.601, + "p99": 41.544, "rule": "p95 and p99", "status": "PASS", "target_p95": 1000, @@ -878,10 +1017,10 @@ { "count": 355, "gate": "latency:BAR_LATEST:OKX", - "max": 622.885, - "p50": 28.809, - "p95": 59.219, - "p99": 205.444, + "max": 354.887, + "p50": 27.242, + "p95": 77.565, + "p99": 234.588, "rule": "p95 and p99", "status": "PASS", "target_p95": 1000, @@ -890,10 +1029,10 @@ { "count": 355, "gate": "latency:L2_SNAPSHOT:BINANCE", - "max": 417.651, - "p50": 26.463, - "p95": 65.632, - "p99": 293.751, + "max": 479.697, + "p50": 40.045, + "p95": 69.358, + "p99": 251.959, "rule": "p95 and p99", "status": "PASS", "target_p95": 300, @@ -902,10 +1041,10 @@ { "count": 355, "gate": "latency:L2_SNAPSHOT:OKX", - "max": 322.334, - "p50": 25.784, - "p95": 94.998, - "p99": 230.694, + "max": 440.775, + "p50": 29.848, + "p95": 54.544, + "p99": 220.378, "rule": "p95 and p99", "status": "PASS", "target_p95": 300, @@ -914,10 +1053,10 @@ { "count": 53250, "gate": "latency:MARK_INDEX_REFERENCE:BINANCE", - "max": 698.315, - "p50": 23.67, - "p95": 61.139, - "p99": 103.998, + "max": 499.112, + "p50": 20.096, + "p95": 58.353, + "p99": 83.549, "rule": "p95 and p99", "status": "PASS", "target_p95": 250, @@ -926,10 +1065,10 @@ { "count": 53250, "gate": "latency:MARK_INDEX_REFERENCE:OKX", - "max": 716.352, - "p50": 22.85, - "p95": 62.527, - "p99": 93.194, + "max": 550.297, + "p50": 23.962, + "p95": 57.627, + "p99": 71.045, "rule": "p95 and p99", "status": "PASS", "target_p95": 250, @@ -938,10 +1077,10 @@ { "count": 35500, "gate": "latency:QUOTE_SNAPSHOT:BINANCE", - "max": 626.853, - "p50": 13.151, - "p95": 34.995, - "p99": 89.424, + "max": 613.678, + "p50": 12.77, + "p95": 31.472, + "p99": 65.693, "rule": "p95 and p99", "status": "PASS", "target_p95": 100, @@ -950,10 +1089,10 @@ { "count": 35500, "gate": "latency:QUOTE_SNAPSHOT:OKX", - "max": 675.312, - "p50": 13.823, - "p95": 39.259, - "p99": 80.913, + "max": 613.255, + "p50": 12.329, + "p95": 32.554, + "p99": 63.39, "rule": "p95 and p99", "status": "PASS", "target_p95": 100, @@ -962,10 +1101,10 @@ { "count": 354, "gate": "latency:TRADE_SNAPSHOT:BINANCE", - "max": 234.675, - "p50": 10.968, - "p95": 29.074, - "p99": 84.744, + "max": 285.234, + "p50": 11.533, + "p95": 28.489, + "p99": 50.097, "rule": "p95 and p99", "status": "PASS", "target_p95": 100, @@ -974,10 +1113,10 @@ { "count": 355, "gate": "latency:TRADE_SNAPSHOT:OKX", - "max": 329.802, - "p50": 20.276, - "p95": 41.388, - "p99": 198.375, + "max": 255.068, + "p50": 13.836, + "p95": 30.461, + "p99": 64.057, "rule": "p95 and p99", "status": "PASS", "target_p95": 100, @@ -996,12 +1135,12 @@ "verdict": "PASS", "sources": { "gates": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/stage100/gates.json", - "sha256": "63f12e8c6df961b0505fd75ab8df9fe2a34e4fa7b2d985cb025e17a6785bcc43" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/stage100/gates.json", + "sha256": "86b74f26c3f7dc68228d260becffa77247d080bfc5d6ce9736ebba495df81544" }, "receipt": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/stage100/receipt.json", - "sha256": "a79c9c9d5989f3befb770449badac8cdc9330ec30365b659b96c61cb7e54a0c3" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/stage100/receipt.json", + "sha256": "fdf26baa323115015501aa56b6450e51deaa83e9bf284d6ef8ec1574c20825eb" } } }, @@ -1012,17 +1151,17 @@ "age_basis": "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR", "age_ms": { "basis": "reservoir sample per worker (exact n and max above)", - "max_ms": 15242197.0, + "max_ms": 18418638.0, "n": 150, - "p50_ms": 14352510.5, - "p95_ms": 15156931.0, - "p99_ms": 15241597.0 + "p50_ms": 17528950.5, + "p95_ms": 18333372.0, + "p99_ms": 18418038.0 }, "classes": { "ELIGIBLE": 150 }, - "max_age_ms": 15242197, - "max_client_age_ms": 15242303, + "max_age_ms": 18418638, + "max_client_age_ms": 18418736, "n": 150, "policy_basis": "REQUIREMENT_ALPHA", "policy_ms": 32400000, @@ -1034,17 +1173,17 @@ "age_basis": "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR", "age_ms": { "basis": "reservoir sample per worker (exact n and max above)", - "max_ms": 15239191.0, + "max_ms": 18415626.0, "n": 150, - "p50_ms": 14342900.0, - "p95_ms": 15141540.0, - "p99_ms": 15223646.0 + "p50_ms": 17519334.0, + "p95_ms": 18317974.0, + "p99_ms": 18400080.0 }, "classes": { "ELIGIBLE": 150 }, - "max_age_ms": 15239191, - "max_client_age_ms": 15239300, + "max_age_ms": 18415626, + "max_client_age_ms": 18415735, "n": 150, "policy_basis": "REQUIREMENT_ALPHA", "policy_ms": 28800000, @@ -1056,18 +1195,18 @@ "age_basis": "CLIENT_RECEIPT_MINUS_NEWEST_SOURCE_EVENT", "age_ms": { "basis": "first 20000 ages per key (exact n and max above)", - "max_ms": 1582, - "n": 2990, - "p50_ms": 670.0, - "p95_ms": 1118, - "p99_ms": 1176 + "max_ms": 1250, + "n": 3005, + "p50_ms": 656, + "p95_ms": 1107, + "p99_ms": 1151 }, "classes": { - "ELIGIBLE": 2990 + "ELIGIBLE": 3005 }, - "max_age_ms": 1582, - "max_client_age_ms": 1501, - "n": 2990, + "max_age_ms": 1250, + "max_client_age_ms": 1171, + "n": 3005, "policy_basis": "D3_EXECUTION", "policy_ms": 2000, "reasons": {}, @@ -1078,17 +1217,17 @@ "age_basis": "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR", "age_ms": { "basis": "reservoir sample per worker (exact n and max above)", - "max_ms": 1184.0, + "max_ms": 1131.0, "n": 4096, - "p50_ms": 669.0, - "p95_ms": 1048.0, - "p99_ms": 1143.0 + "p50_ms": 579.0, + "p95_ms": 1065.0, + "p99_ms": 1106.0 }, "classes": { "ELIGIBLE": 63000 }, - "max_age_ms": 1668, - "max_client_age_ms": 1780, + "max_age_ms": 1192, + "max_client_age_ms": 1359, "n": 63000, "policy_basis": "REQUIREMENT_ALPHA", "policy_ms": 60000, @@ -1100,19 +1239,19 @@ "age_basis": "CLIENT_RECEIPT_MINUS_NEWEST_SOURCE_EVENT", "age_ms": { "basis": "first 20000 ages per key (exact n and max above)", - "max_ms": 2177, - "n": 3588, - "p50_ms": 502.0, - "p95_ms": 1331, - "p99_ms": 1923 + "max_ms": 2278, + "n": 3606, + "p50_ms": 567.5, + "p95_ms": 1527, + "p99_ms": 2016 }, "classes": { - "ELIGIBLE": 3563, - "ELIGIBLE_QUIET": 25 + "ELIGIBLE": 3566, + "ELIGIBLE_QUIET": 40 }, - "max_age_ms": 2177, - "max_client_age_ms": 2109, - "n": 3588, + "max_age_ms": 2278, + "max_client_age_ms": 2236, + "n": 3606, "policy_basis": "D3_EXECUTION", "policy_ms": 2000, "reasons": {}, @@ -1123,17 +1262,17 @@ "age_basis": "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR", "age_ms": { "basis": "reservoir sample per worker (exact n and max above)", - "max_ms": 2286.0, + "max_ms": 2227.0, "n": 4096, - "p50_ms": 444.5, - "p95_ms": 1265.0, - "p99_ms": 1866.0 + "p50_ms": 533.5, + "p95_ms": 1513.0, + "p99_ms": 1988.0 }, "classes": { "ELIGIBLE": 63180 }, - "max_age_ms": 2291, - "max_client_age_ms": 2401, + "max_age_ms": 2259, + "max_client_age_ms": 2305, "n": 63180, "policy_basis": "REQUIREMENT_ALPHA", "policy_ms": 60000, @@ -1145,18 +1284,18 @@ "age_basis": null, "age_ms": { "basis": "first 20000 ages per key (exact n and max above)", - "max_ms": 1237, - "n": 2990, + "max_ms": 1720, + "n": 3002, "p50_ms": 324.0, - "p95_ms": 530, - "p99_ms": 739 + "p95_ms": 591, + "p99_ms": 909 }, "classes": { - "ELIGIBLE": 2990 + "ELIGIBLE": 3002 }, - "max_age_ms": 1237, + "max_age_ms": 1720, "max_client_age_ms": null, - "n": 2990, + "n": 3002, "policy_basis": "D3_EXECUTION", "policy_ms": 2000, "reasons": {}, @@ -1167,16 +1306,16 @@ "age_basis": null, "age_ms": { "basis": "reservoir sample per worker (exact n and max above)", - "max_ms": 1277.0, + "max_ms": 1536.0, "n": 4096, - "p50_ms": 321.0, - "p95_ms": 522.0, - "p99_ms": 708.0 + "p50_ms": 328.5, + "p95_ms": 593.0, + "p99_ms": 892.0 }, "classes": { "ELIGIBLE": 36000 }, - "max_age_ms": 1580, + "max_age_ms": 1986, "max_client_age_ms": null, "n": 36000, "policy_basis": "REQUIREMENT_ALPHA", @@ -1189,19 +1328,19 @@ "age_basis": null, "age_ms": { "basis": "first 20000 ages per key (exact n and max above)", - "max_ms": 13027, - "n": 3583, - "p50_ms": 353, - "p95_ms": 1083, - "p99_ms": 3074 + "max_ms": 11131, + "n": 3600, + "p50_ms": 366.0, + "p95_ms": 1227, + "p99_ms": 3518 }, "classes": { "ELIGIBLE": 3515, - "ELIGIBLE_QUIET": 68 + "ELIGIBLE_QUIET": 85 }, - "max_age_ms": 13027, + "max_age_ms": 11131, "max_client_age_ms": null, - "n": 3583, + "n": 3600, "policy_basis": "D3_EXECUTION", "policy_ms": 2000, "reasons": {}, @@ -1212,16 +1351,17 @@ "age_basis": null, "age_ms": { "basis": "reservoir sample per worker (exact n and max above)", - "max_ms": 3889.0, + "max_ms": 5110.0, "n": 4096, - "p50_ms": 338.0, - "p95_ms": 675.0, - "p99_ms": 1015.0 + "p50_ms": 365.0, + "p95_ms": 932.0, + "p99_ms": 1785.0 }, "classes": { - "ELIGIBLE": 36100 + "ELIGIBLE": 36091, + "ELIGIBLE_QUIET": 9 }, - "max_age_ms": 3990, + "max_age_ms": 7103, "max_client_age_ms": null, "n": 36100, "policy_basis": "REQUIREMENT_ALPHA", @@ -1234,19 +1374,19 @@ "age_basis": null, "age_ms": { "basis": "first 20000 ages per key (exact n and max above)", - "max_ms": 6770, - "n": 2985, - "p50_ms": 558, - "p95_ms": 2195, - "p99_ms": 3565 + "max_ms": 10230, + "n": 3000, + "p50_ms": 634.0, + "p95_ms": 2930, + "p99_ms": 5369 }, "classes": { - "ELIGIBLE": 2936, - "NOT_ELIGIBLE": 49 + "ELIGIBLE": 2862, + "NOT_ELIGIBLE": 138 }, - "max_age_ms": 6770, + "max_age_ms": 10230, "max_client_age_ms": null, - "n": 2985, + "n": 3000, "policy_basis": "D3_EXECUTION", "policy_ms": 3000, "reasons": {}, @@ -1257,19 +1397,19 @@ "age_basis": null, "age_ms": { "basis": "first 20000 ages per key (exact n and max above)", - "max_ms": 40706, - "n": 2985, - "p50_ms": 807, - "p95_ms": 7967, - "p99_ms": 19634 + "max_ms": 139576, + "n": 3000, + "p50_ms": 1410.5, + "p95_ms": 25883, + "p99_ms": 64896 }, "classes": { - "ELIGIBLE": 2522, - "NOT_ELIGIBLE": 463 + "ELIGIBLE": 2084, + "NOT_ELIGIBLE": 916 }, - "max_age_ms": 40706, + "max_age_ms": 139576, "max_client_age_ms": null, - "n": 2985, + "n": 3000, "policy_basis": "D3_EXECUTION", "policy_ms": 3000, "reasons": {}, @@ -1282,19 +1422,19 @@ "rule": "per-binding D3 policy of the TS EXECUTION consumer (TRADE 3000, QUOTE/MARK/BOOK_DELTA 2000, BOOK_SNAPSHOT 60000, BAR1m 180000 ms, min with the requirement); zero unsafe eligibility; decided by D3_EXECUTION rows only", "sources": { "g6_ts": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g6-ts/summary.json", - "sha256": "229e038090ec1000c19ceca41eb805bcda1b4a7827e2e0b41803f4cc60aeb659" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g6-ts/summary.json", + "sha256": "5a9fc905a71d86e1d013bfeed463bee3bfb366545fd742189b6ddedc3ee1bede" } }, "ts_probe": { "errors": {}, "per_binding_period_s": 3.2, - "reads": 19121, + "reads": 19213, "stopped_by": "signal", "targets": 32, "window_unix_s": [ - 1791690153.9782138, - 1791692078.7164075 + 1791722125.7330813, + 1791724056.2921798 ] }, "verdict": "PASS", @@ -1308,92 +1448,92 @@ "measured": { "BOOK_DELTA": { "sdk_to_writer_ms": { - "max": 1311.7, - "min": 0.3, - "n": 182930, - "p50": 23.0, - "p95": 51.0, - "p99": 135.5 + "max": 350.1, + "min": 0.2, + "n": 178828, + "p50": 22.6, + "p95": 33.8, + "p99": 72.2 }, "writer_to_redis_ms": { - "max": 530.3, + "max": 195.5, "min": 0.5, - "n": 181572, - "p50": 3.4, - "p95": 18.4, - "p99": 52.0 + "n": 177511, + "p50": 2.7, + "p95": 13.5, + "p99": 24.6 } }, "BOOK_SNAPSHOT": { "sdk_to_writer_ms": { - "max": 616.3, - "min": 12.3, - "n": 21229, - "p50": 31.0, - "p95": 79.3, - "p99": 141.0 + "max": 361.7, + "min": 15.3, + "n": 19767, + "p50": 30.0, + "p95": 44.8, + "p99": 108.5 }, "writer_to_redis_ms": { - "max": 379.4, + "max": 284.2, "min": 1.5, - "n": 21229, - "p50": 7.3, - "p95": 41.2, - "p99": 128.4 + "n": 19767, + "p50": 5.5, + "p95": 20.5, + "p99": 61.0 } }, "MARK_INDEX_PRICE": { "sdk_to_writer_ms": { - "max": 8.1, + "max": 3.8, "min": 0.2, - "n": 30441, - "p50": 0.5, + "n": 30514, + "p50": 0.4, "p95": 0.6, - "p99": 0.9 + "p99": 0.7 }, "writer_to_redis_ms": { - "max": 520.7, + "max": 339.1, "min": 1.1, - "n": 30441, - "p50": 4.0, - "p95": 20.2, - "p99": 79.3 + "n": 30514, + "p50": 3.7, + "p95": 14.4, + "p99": 28.8 } }, "QUOTE": { "sdk_to_writer_ms": { - "max": 661.6, - "min": 1.3, - "n": 214601, - "p50": 22.6, - "p95": 40.7, - "p99": 99.3 + "max": 386.1, + "min": 1.1, + "n": 183275, + "p50": 22.1, + "p95": 30.7, + "p99": 47.4 }, "writer_to_redis_ms": { - "max": 473.8, + "max": 230.7, "min": 0.5, - "n": 233880, - "p50": 1.4, - "p95": 10.5, - "p99": 48.6 + "n": 202646, + "p50": 1.3, + "p95": 5.9, + "p99": 15.2 } }, "TRADE": { "sdk_to_writer_ms": { - "max": 501.5, - "min": 0.3, - "n": 203657, - "p50": 26.9, - "p95": 83.0, - "p99": 148.1 + "max": 228.7, + "min": 0.2, + "n": 79000, + "p50": 23.1, + "p95": 45.3, + "p99": 97.9 }, "writer_to_redis_ms": { - "max": 438.0, + "max": 247.0, "min": 0.5, - "n": 204778, - "p50": 7.7, - "p95": 46.7, - "p99": 101.9 + "n": 79994, + "p50": 4.9, + "p95": 33.3, + "p99": 77.0 } } }, @@ -1403,8 +1543,12 @@ "verdict": "PASS", "sources": { "transit": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/transit-load.json", - "sha256": "d1376db1a50db991df5e9e9e80212463ad095640ee00a67ab673e473145d73f6" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/trace/transit-load.json", + "sha256": "bd1ccac05fad1728d73bbab833a629d43266756c3575e09082a498e1ee7c78c7" + }, + "mark": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/trace/mark-load.json", + "sha256": "b6a5198116d7916e32b8b05f48e2bf45798b7863aeb64e32247b874c442f8c9f" } } }, @@ -1413,246 +1557,138 @@ "measured": { "monitor_clients_max": 0, "ping_all_ms": { - "max": 5.869, - "min": 0.1, - "n": 3850, - "p50": 0.154, - "p95": 0.51, - "p99": 1.275 + "max": 2.335, + "min": 0.088, + "n": 3861, + "p50": 0.151, + "p95": 0.344, + "p99": 0.779 }, "ping_failures": 0, "ping_steady_ms": { - "max": 5.869, - "min": 0.1, + "max": 1.788, + "min": 0.088, "n": 3599, - "p50": 0.155, - "p95": 0.516, - "p99": 1.274 + "p50": 0.151, + "p95": 0.344, + "p99": 0.771 }, - "slowlog_new_steady": 6, - "slowlog_new_total": 6, - "slowlog_steady_rows": [ - { - "command": "EVALSHA", - "duration_us": 10016, - "id": 976628, - "start_s": 1791690302 - }, - { - "command": "EVALSHA", - "duration_us": 16120, - "id": 976629, - "start_s": 1791690605 - }, - { - "command": "EVALSHA", - "duration_us": 13258, - "id": 976630, - "start_s": 1791690763 - }, - { - "command": "EVALSHA", - "duration_us": 12964, - "id": 976631, - "start_s": 1791690842 - }, - { - "command": "EVALSHA", - "duration_us": 18431, - "id": 976632, - "start_s": 1791691251 - }, - { - "command": "EVALSHA", - "duration_us": 19732, - "id": 976633, - "start_s": 1791691386 - } - ], + "slowlog_new_steady": 0, + "slowlog_new_total": 0, + "slowlog_steady_rows": [], "slowlog_threshold_us": 10000, "windows": [ { - "core_fraction": 0.0687, - "from_ns": 1791690153422572495, + "core_fraction": 0.0612, + "from_ns": 1791722125139191026, "judged": true, "ping_ms": { - "max": 2.9, - "min": 0.1, + "max": 2.116, + "min": 0.097, "n": 1199, - "p50": 0.154, - "p95": 0.476, - "p99": 1.245 + "p50": 0.152, + "p95": 0.411, + "p99": 1.15 }, "seconds": 590.2, - "slowlog_new": 2, + "slowlog_new": 0, "steady": false, - "to_ns": 1791690743584489495, + "to_ns": 1791722715299287754, "top_commands_usec": { - "eval": 10145176, - "evalsha": 24499749, - "publish": 701412, - "set": 979302, - "setex": 2558834 + "eval": 9735370, + "evalsha": 21061597, + "get": 669220, + "set": 813207, + "setex": 2310118 }, "valid": true }, { - "core_fraction": 0.0868, - "from_ns": 1791690753588790195, + "core_fraction": 0.059, + "from_ns": 1791722725302986730, "judged": true, "ping_ms": { - "max": 5.869, - "min": 0.1, + "max": 1.535, + "min": 0.097, "n": 1200, - "p50": 0.158, - "p95": 0.715, - "p99": 1.675 + "p50": 0.153, + "p95": 0.329, + "p99": 0.74 }, "seconds": 590.2, - "slowlog_new": 3, + "slowlog_new": 0, "steady": true, - "to_ns": 1791691343765594831, + "to_ns": 1791723315471310704, "top_commands_usec": { - "eval": 11868395, - "evalsha": 32166251, - "publish": 896227, - "set": 1288739, - "setex": 3237803 + "eval": 9501468, + "evalsha": 19777631, + "publish": 659205, + "set": 912393, + "setex": 2450256 }, "valid": true }, { - "core_fraction": 0.0741, - "from_ns": 1791691353768453016, + "core_fraction": 0.0583, + "from_ns": 1791723325473055404, "judged": true, "ping_ms": { - "max": 3.248, - "min": 0.1, - "n": 1199, - "p50": 0.152, - "p95": 0.467, - "p99": 1.039 + "max": 1.549, + "min": 0.088, + "n": 1200, + "p50": 0.148, + "p95": 0.302, + "p99": 0.602 }, - "seconds": 589.7, - "slowlog_new": 1, + "seconds": 599.7, + "slowlog_new": 0, "steady": true, - "to_ns": 1791691943439104168, + "to_ns": 1791723925137596179, "top_commands_usec": { - "eval": 10480850, - "evalsha": 26212845, - "publish": 839100, - "set": 1309821, - "setex": 3165549 + "eval": 9497541, + "evalsha": 19597255, + "publish": 694107, + "set": 1019249, + "setex": 2605077 }, "valid": true }, { - "core_fraction": 0.0627, - "from_ns": 1791691953441892544, + "core_fraction": 0.0594, + "from_ns": 1791723935140264489, "judged": false, "ping_ms": { - "max": 2.198, - "min": 0.1, - "n": 251, - "p50": 0.146, - "p95": 0.352, - "p99": 0.815 + "max": 1.782, + "min": 0.102, + "n": 261, + "p50": 0.15, + "p95": 0.308, + "p99": 0.707 }, "seconds": 120.0, "slowlog_new": 0, "steady": false, - "to_ns": 1791692073476356936, + "to_ns": 1791724055171136739, "top_commands_usec": { - "eval": 1933282, - "evalsha": 4387436, - "publish": 141285, - "set": 214685, - "setex": 530274 + "eval": 1909963, + "evalsha": 4090137, + "publish": 134593, + "set": 188050, + "setex": 497210 }, "valid": true } ] }, "quantity": "TS execution Redis (redis_marketdata) command CPU, SLOWLOG, RTT", - "reason": "SLOWLOG 6 new entries in steady state", + "reason": "inside", "rule": "commandstats <= 60 % of one core per 10-min window; SLOWLOG (10 ms) 0 in steady state; PING p99 <= 5 ms", - "verdict": "FAIL", + "verdict": "PASS", "sources": { "g8": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g8/g8-summary.json", - "sha256": "2b3375cffc57c87e9e85ac21fe6b70f3897c2ebf2b80eea3f01b1180eb2d5671" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g8/g8-summary.json", + "sha256": "0314b552d3f09b7d2143304ee77c9cd0c27a959fb10b764e2996865a2ffae7e5" } - }, - "attribution": { - "basis": "the 6 SLOWLOG start seconds compared with applied TRADE records per writer-start second in the same trace", - "per_second_applied_TRADE": { - "median": 46, - "p95": 428, - "seconds": 1801 - }, - "slowlog_seconds": [ - { - "start_s": 1791690302, - "EVALSHA_us": 10016, - "TRADE_s-1..s+1": [ - 138, - 320, - 379 - ], - "percentile_of_peak": 93.2 - }, - { - "start_s": 1791690605, - "EVALSHA_us": 16120, - "TRADE_s-1..s+1": [ - 42, - 709, - 320 - ], - "percentile_of_peak": 98.1 - }, - { - "start_s": 1791690763, - "EVALSHA_us": 13258, - "TRADE_s-1..s+1": [ - 778, - 1114, - 1062 - ], - "percentile_of_peak": 99.4 - }, - { - "start_s": 1791690842, - "EVALSHA_us": 12964, - "TRADE_s-1..s+1": [ - 677, - 1068, - 625 - ], - "percentile_of_peak": 99.2 - }, - { - "start_s": 1791691251, - "EVALSHA_us": 18431, - "TRADE_s-1..s+1": [ - 296, - 691, - 284 - ], - "percentile_of_peak": 98.0 - }, - { - "start_s": 1791691386, - "EVALSHA_us": 19732, - "TRADE_s-1..s+1": [ - 174, - 372, - 974 - ], - "percentile_of_peak": 99.1 - } - ], - "reading": "every SLOWLOG entry is a writer APPLY EVALSHA in a TRADE burst second (peak within +-1 s at or above the 93rd percentile of per-second TRADE volume); commandstats <= 0.0868 core and PING p99 <= 1.675 ms stay inside. Owner decision as G2: EVAL chunking moves to TS phase E" } }, { @@ -1666,9 +1702,9 @@ "eligibility_loss_rows": [], "eligibility_losses_data_path": 0, "execution_ready_max": 53, - "execution_ready_min": 50, + "execution_ready_min": 48, "legit_quiet_refusals": { - "TRADE:TRADE_EVENT_STALE": 24 + "TRADE:TRADE_EVENT_STALE": 48 }, "purpose_errors": 0, "purpose_fields_present": true, @@ -1685,12 +1721,8 @@ "verdict": "PASS", "sources": { "g9g10": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g9g10/g9-g10-summary.json", - "sha256": "3015c2db3820c172a984af38043443622e97065f59fbdd8054dbfff13abf117d" - }, - "gates": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/stage100/gates.json", - "sha256": "63f12e8c6df961b0505fd75ab8df9fe2a34e4fa7b2d985cb025e17a6785bcc43" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g9g10/g9-g10-summary.json", + "sha256": "247c62e6702c264efb4b7b8e1a3089b369a378a8baf21e81d525945b3d8bf5d0" } } }, @@ -1698,29 +1730,31 @@ "basis": "writer log typed signals (DL-signalled gaps, CURSOR_EXPIRED, SlowConsumerError) + trace duplicate event_id; a silent offset gap the DL does not signal is not observable (see proposed counter)", "id": "G10", "measured": { - "codes": { - "resumed|RECONNECTED": 2 - }, + "codes": {}, "duplicates_applied": 0, "gaps": 0, "log_polls": 129, "log_unreadable": 0, "overflow": 0, "replay_expired": 0, - "resumed_reconnects": 2 + "resumed_reconnects": 0 }, "quantity": "TRADE integrity", "reason": "all zero", "rule": "0 gaps, 0 duplicates applied, 0 replay-expired, 0 overflow", "verdict": "PASS", "sources": { - "g9g10": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g9g10/g9-g10-summary.json", - "sha256": "3015c2db3820c172a984af38043443622e97065f59fbdd8054dbfff13abf117d" - }, "transit": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/transit-load.json", - "sha256": "d1376db1a50db991df5e9e9e80212463ad095640ee00a67ab673e473145d73f6" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/trace/transit-load.json", + "sha256": "bd1ccac05fad1728d73bbab833a629d43266756c3575e09082a498e1ee7c78c7" + }, + "mark": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/trace/mark-load.json", + "sha256": "b6a5198116d7916e32b8b05f48e2bf45798b7863aeb64e32247b874c442f8c9f" + }, + "g9g10": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g9g10/g9-g10-summary.json", + "sha256": "247c62e6702c264efb4b7b8e1a3089b369a378a8baf21e81d525945b3d8bf5d0" } } }, @@ -1730,360 +1764,360 @@ "cursor_samples": [ { "ahead": [], - "at_ms": 1791692127022, + "at_ms": 1791724111600, "compared": 21, "unmatched": 0 }, { "ahead": [], - "at_ms": 1791692147026, + "at_ms": 1791724131603, "compared": 21, "unmatched": 0 }, { "ahead": [], - "at_ms": 1791692167033, + "at_ms": 1791724151611, "compared": 21, "unmatched": 0 } ], - "outage_seconds": 0.121, + "outage_seconds": 0.132, "recovery": { "caught_up_s": { - "max": 18.923, - "min": 3.44, + "max": 7.511, + "min": 0.158, "n": 64, - "p50": 3.919, - "p95": 4.437, + "p50": 0.158, + "p95": 4.51, "p99": null }, "first_valid_apply_s": { - "max": 18.923, - "min": 3.44, + "max": 7.511, + "min": 0.158, "n": 64, - "p50": 3.919, - "p95": 4.437, + "p50": 0.158, + "p95": 4.51, "p99": null }, "per_slice": { "BINANCE|BNBUSDT|BAR|1m": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "BINANCE|BNBUSDT|BOOK_DELTA|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|BNBUSDT|BOOK_SNAPSHOT|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "BINANCE|BNBUSDT|MARK_INDEX_PRICE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|BNBUSDT|QUOTE|None": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|BNBUSDT|TRADE|None": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "BINANCE|BTCUSDT|BAR|1m": { - "caught_up_s": 3.44, - "first_valid_apply_s": 3.44, + "caught_up_s": 4.033, + "first_valid_apply_s": 4.033, "rewinds": [] }, "BINANCE|BTCUSDT|BOOK_DELTA|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|BTCUSDT|BOOK_SNAPSHOT|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.033, + "first_valid_apply_s": 4.033, "rewinds": [] }, "BINANCE|BTCUSDT|MARK_INDEX_PRICE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|BTCUSDT|QUOTE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|BTCUSDT|TRADE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.033, + "first_valid_apply_s": 4.033, "rewinds": [] }, "BINANCE|DOGEUSDT|BAR|1m": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.033, + "first_valid_apply_s": 4.033, "rewinds": [] }, "BINANCE|DOGEUSDT|BOOK_DELTA|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|DOGEUSDT|BOOK_SNAPSHOT|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|DOGEUSDT|MARK_INDEX_PRICE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|DOGEUSDT|QUOTE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|DOGEUSDT|TRADE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|ETHUSDT|BAR|1m": { - "caught_up_s": 3.44, - "first_valid_apply_s": 3.44, + "caught_up_s": 4.033, + "first_valid_apply_s": 4.033, "rewinds": [] }, "BINANCE|ETHUSDT|BOOK_DELTA|None": { - "caught_up_s": 3.44, - "first_valid_apply_s": 3.44, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|ETHUSDT|BOOK_SNAPSHOT|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.033, + "first_valid_apply_s": 4.033, "rewinds": [] }, "BINANCE|ETHUSDT|MARK_INDEX_PRICE|None": { - "caught_up_s": 3.44, - "first_valid_apply_s": 3.44, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|ETHUSDT|QUOTE|None": { - "caught_up_s": 3.44, - "first_valid_apply_s": 3.44, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|ETHUSDT|TRADE|None": { - "caught_up_s": 3.44, - "first_valid_apply_s": 3.44, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|SOLUSDT|BAR|1m": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "BINANCE|SOLUSDT|BOOK_DELTA|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|SOLUSDT|BOOK_SNAPSHOT|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "BINANCE|SOLUSDT|MARK_INDEX_PRICE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|SOLUSDT|QUOTE|None": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "BINANCE|SOLUSDT|TRADE|None": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|BNB-USDT-SWAP|BAR|1m": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "OKX|BNB-USDT-SWAP|BOOK_DELTA|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|BNB-USDT-SWAP|BOOK_SNAPSHOT|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|BNB-USDT-SWAP|MARK_INDEX_PRICE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|BNB-USDT-SWAP|QUOTE|None": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "OKX|BNB-USDT-SWAP|TRADE|None": { - "caught_up_s": 18.923, - "first_valid_apply_s": 18.923, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "OKX|BTC-USD-SWAP|BOOK_DELTA|None": { - "caught_up_s": 3.44, - "first_valid_apply_s": 3.44, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|BTC-USD-SWAP|BOOK_SNAPSHOT|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|BTC-USD-SWAP|MARK_INDEX_PRICE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.033, + "first_valid_apply_s": 4.033, "rewinds": [] }, "OKX|BTC-USD-SWAP|QUOTE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|BTC-USDT-SWAP|BAR|1m": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.033, + "first_valid_apply_s": 4.033, "rewinds": [] }, "OKX|BTC-USDT-SWAP|BOOK_DELTA|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|BTC-USDT-SWAP|BOOK_SNAPSHOT|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|BTC-USDT-SWAP|MARK_INDEX_PRICE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|BTC-USDT-SWAP|QUOTE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|BTC-USDT-SWAP|TRADE|None": { - "caught_up_s": 4.92, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "OKX|DOGE-USDT-SWAP|BAR|1m": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.033, + "first_valid_apply_s": 4.033, "rewinds": [] }, "OKX|DOGE-USDT-SWAP|BOOK_DELTA|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|DOGE-USDT-SWAP|BOOK_SNAPSHOT|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.033, + "first_valid_apply_s": 4.033, "rewinds": [] }, "OKX|DOGE-USDT-SWAP|MARK_INDEX_PRICE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|DOGE-USDT-SWAP|QUOTE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|DOGE-USDT-SWAP|TRADE|None": { - "caught_up_s": 4.92, - "first_valid_apply_s": 4.92, + "caught_up_s": 7.511, + "first_valid_apply_s": 7.511, "rewinds": [] }, "OKX|ETH-USDT-SWAP|BAR|1m": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "OKX|ETH-USDT-SWAP|BOOK_DELTA|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|ETH-USDT-SWAP|BOOK_SNAPSHOT|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "OKX|ETH-USDT-SWAP|MARK_INDEX_PRICE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.51, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|ETH-USDT-SWAP|QUOTE|None": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|ETH-USDT-SWAP|TRADE|None": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|SOL-USDT-SWAP|BAR|1m": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "OKX|SOL-USDT-SWAP|BOOK_DELTA|None": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|SOL-USDT-SWAP|BOOK_SNAPSHOT|None": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 4.51, + "first_valid_apply_s": 4.51, "rewinds": [] }, "OKX|SOL-USDT-SWAP|MARK_INDEX_PRICE|None": { - "caught_up_s": 3.919, - "first_valid_apply_s": 3.919, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|SOL-USDT-SWAP|QUOTE|None": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] }, "OKX|SOL-USDT-SWAP|TRADE|None": { - "caught_up_s": 4.437, - "first_valid_apply_s": 4.437, + "caught_up_s": 0.158, + "first_valid_apply_s": 0.158, "rewinds": [] } }, @@ -2092,17 +2126,17 @@ "slices_without_first_apply": [] }, "restart": { - "detected_ms": 1791692107015, - "fault_cleared_ms": 1791692106579, - "last_old_seen_ms": 1791692106016, + "detected_ms": 1791724091721, + "fault_cleared_ms": 1791724091567, + "last_old_seen_ms": 1791724090592, "new": { - "finished": "2026-10-11T04:15:06.458006215Z", - "id": "c9ded16d4ff019b16d3dbc577edfceaaea6fe2282baf0a65a1708b2c4c0b8cbd", + "finished": "2026-10-11T13:08:11.435367212Z", + "id": "29c638798a99c3c792e9c417c17ab7fd08bd95db92252355a6a25fbe7fdcdc62", "restarts": 0, "running": true, - "started": "2026-10-11T04:15:06.579836652Z" + "started": "2026-10-11T13:08:11.567013847Z" }, - "outage_start_ms": 1791692106458 + "outage_start_ms": 1791724091435 } }, "note": "a fault window after the load (a restart of a monitored container aborts the driver); its verdict does not depend on the steady-state protocol", @@ -2112,8 +2146,8 @@ "verdict": "PASS", "sources": { "g11": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g11/g11-summary.json", - "sha256": "c59cccf3447dbe67b98c4f4842903601352234600de4b6d60b741e94cb98ed8f" + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g11/g11-summary.json", + "sha256": "c85e4566e327328b0bd4c5f5cc75a4825f1edcff1586a4bad3226e3233c5b764" } } }, @@ -2122,422 +2156,3127 @@ "quantity": "Kafka broker / projector / Stream restart recovery", "rule": "fault cleared -> consumers caught up to live <= 600 s, then G1-G3 inside gate; the same four timestamps as G11", "verdict": "PASS", - "reason": "full isolated run on the release digests (python a0c3649d, rust 96a2c912, writer bae2509f) with harness 2d4b88c (TEST_ONLY BOOK, TRADE, QUOTE, MARK/INDEX and BAR 1m sources): every fault caught up <= 600 s, then G1, G2 (TRADE, QUOTE, MARK) and G3 inside in the 300 s window; monitor 2511 samples, 0 sample errors, no gap", + "reason": "full isolated run on the rc12 digests (python 3b585597, rust ebc7b794, writer 64347a94) with the G12 harness of 2d4b88c (TEST_ONLY BOOK, TRADE, QUOTE, MARK/INDEX and BAR 1m sources): every fault caught up <= 600 s, then G1, G2 (TRADE, QUOTE, MARK) and G3 inside in the 300 s window; every part PASS; monitor 1718 samples, 2 typed SIGSEGV sample errors (list_consumer_group_offsets, the IDR-087 evidence), max gap 2.0 s <= 5 s", "level": "ISOLATED_FAULT", "provenance": "TEST_ONLY", + "status": "COMPLETED", + "pass": true, + "images": { + "kafka": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "core": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "projector": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "stream": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "python": "sha256:3b585597570f3f7065c17b0a7f9228a2dad2f71224905e2f105a1754b9d4a75c", + "redis": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", + "ts": "sha256:64347a94a3ed682d155690f3afaed5aaeba6d8feb8e8febcaeb7dc33dea488d9", + "postgres": "sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685", + "holder": "sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc" + }, "measured": [ { "fault": "broker_stop_start", - "caught_up_s": 22.334, + "caught_up_s": 224.704, "per_venue_caught_up_s": { - "BINANCE": 22.334, - "OKX": 2.177 + "BINANCE": 182.931, + "OKX": 224.704 }, - "g1_after": { - "BINANCE": { - "n": 297, - "p95_ms": 108.227, - "p99_ms": 160.586, - "max_ms": 1424.895 + "g1_after_max_ms": { + "BINANCE": 573.85, + "OKX": 574.526 + }, + "g2_after_max_ms": { + "TRADE": { + "BINANCE": 541.108, + "OKX": 540.965 + }, + "QUOTE": { + "BINANCE": 540.803, + "OKX": 540.546 }, - "OKX": { - "n": 298, - "p95_ms": 116.204, - "p99_ms": 170.137, - "max_ms": 1970.011 + "MARK": { + "BINANCE": 257.318, + "OKX": 409.072 } - } + }, + "g2_after_n": { + "TRADE": { + "BINANCE": 1060, + "OKX": 1060 + }, + "QUOTE": { + "BINANCE": 600, + "OKX": 600 + }, + "MARK": { + "BINANCE": 300, + "OKX": 300 + } + }, + "g3_bar_1m_after_max_ms": { + "BINANCE": 312.981, + "OKX": 313.66 + }, + "parts": { + "caught_up": true, + "window_complete": true, + "g1": "PASS", + "g2": "PASS", + "g2_feeds": { + "TRADE": "PASS", + "QUOTE": "PASS", + "MARK": "PASS" + }, + "g3": "PASS", + "monitor": "PASS" + }, + "monitor_max_gap_s": 2.0, + "monitor_sample_errors": [ + { + "t": 1791726068774921055, + "error": "SIGSEGV", + "phase": "list_consumer_group_offsets:kn-projector-v3-a" + }, + { + "t": 1791726303883223138, + "error": "SIGSEGV", + "phase": "list_consumer_group_offsets:kn-projector-v3-a" + } + ] }, { "fault": "projector_restart", - "caught_up_s": 0.935, + "caught_up_s": 19.4, "per_venue_caught_up_s": { - "BINANCE": 0.934, - "OKX": 0.935 + "BINANCE": 19.4, + "OKX": 12.403 }, - "g1_after": { - "BINANCE": { - "n": 297, - "p95_ms": 100.731, - "p99_ms": 113.233, - "max_ms": 127.018 + "g1_after_max_ms": { + "BINANCE": 128.675, + "OKX": 130.162 + }, + "g2_after_max_ms": { + "TRADE": { + "BINANCE": 95.377, + "OKX": 107.406 + }, + "QUOTE": { + "BINANCE": 89.969, + "OKX": 93.329 }, - "OKX": { - "n": 299, - "p95_ms": 101.251, - "p99_ms": 113.503, - "max_ms": 196.204 + "MARK": { + "BINANCE": 202.008, + "OKX": 376.883 } - } + }, + "g2_after_n": { + "TRADE": { + "BINANCE": 1060, + "OKX": 1060 + }, + "QUOTE": { + "BINANCE": 600, + "OKX": 600 + }, + "MARK": { + "BINANCE": 300, + "OKX": 300 + } + }, + "g3_bar_1m_after_max_ms": { + "BINANCE": 335.081, + "OKX": 341.066 + }, + "parts": { + "caught_up": true, + "window_complete": true, + "g1": "PASS", + "g2": "PASS", + "g2_feeds": { + "TRADE": "PASS", + "QUOTE": "PASS", + "MARK": "PASS" + }, + "g3": "PASS", + "monitor": "PASS" + }, + "monitor_max_gap_s": 1.0, + "monitor_sample_errors": [] }, { "fault": "stream_restart", - "caught_up_s": 2.294, + "caught_up_s": 138.381, "per_venue_caught_up_s": { - "BINANCE": 2.293, - "OKX": 2.294 + "BINANCE": 14.137, + "OKX": 138.381 + }, + "g1_after_max_ms": { + "BINANCE": 121.845, + "OKX": 126.928 }, - "g1_after": { - "BINANCE": { - "n": 298, - "p95_ms": 98.764, - "p99_ms": 103.427, - "max_ms": 111.131 + "g2_after_max_ms": { + "TRADE": { + "BINANCE": 93.962, + "OKX": 94.971 }, - "OKX": { - "n": 300, - "p95_ms": 98.576, - "p99_ms": 109.594, - "max_ms": 122.954 + "QUOTE": { + "BINANCE": 93.628, + "OKX": 94.69 + }, + "MARK": { + "BINANCE": 207.005, + "OKX": 353.751 } - } + }, + "g2_after_n": { + "TRADE": { + "BINANCE": 1060, + "OKX": 1060 + }, + "QUOTE": { + "BINANCE": 600, + "OKX": 600 + }, + "MARK": { + "BINANCE": 300, + "OKX": 300 + } + }, + "g3_bar_1m_after_max_ms": { + "BINANCE": 315.31, + "OKX": 318.669 + }, + "parts": { + "caught_up": true, + "window_complete": true, + "g1": "PASS", + "g2": "PASS", + "g2_feeds": { + "TRADE": "PASS", + "QUOTE": "PASS", + "MARK": "PASS" + }, + "g3": "PASS", + "monitor": "PASS" + }, + "monitor_max_gap_s": 1.0, + "monitor_sample_errors": [] } ], - "status": "COMPLETED", - "pass": true, - "limits": [ - "isolated namespace, same host, TEST_ONLY BOOK source, single projector/Stream replica", - "G2 and G3 after recovery: NOT_MEASURED (BOOK-only isolated source) / NOT_MEASURED (BOOK-only isolated source)" - ], + "monitor": { + "samples": 1718, + "sample_errors": 2, + "errors": [ + [ + "SIGSEGV", + "list_consumer_group_offsets:kn-projector-v3-a" + ] + ], + "max_gap_rule_s": 5 + }, + "equivalence_to_release": "rc13 (481fd67) differs from rc12 (5ee9463) only by IDR-087: `git diff --stat 5ee9463 481fd67 -- . ':!tests' ':!scripts' ':!*.md' ':!upgrade'` = qdl/runtime/history_backpressure.py, qdl/runtime/isolated_probe.py, imported only by qdl/runtime/stable_bar_edge.py; the bar edge is not in the isolated G12 stack; the Rust tree is identical (empty diff for rust, generated/rust, Cargo files, config)", + "harness": { + "script": "scripts/p184_g12_isolated_fault.py", + "sha256": "2fa21904d60593118d7fed8650ebf68e33d7a65b73abe1bcb4af41fb00e24917", + "note": "same file content at 2d4b88c and 481fd67 (git show); the run receipt does not record the harness commit itself" + }, "teardown": { "removed_containers": 12, "left_by_label": 0, "left_by_name": 0, - "volumes_before": 217, - "volumes_after": 217 + "volumes_by_label": 0, + "errors": [] }, - "attempts": [ - { - "dir": "~/.local/state/qdl-v2/p184-g12-rc10-20261011", - "status": "FAIL", - "error": "RuntimeError: STOP role exited: monitor (in-namespace)", - "detail": "the last monitor record precedes the first fault injection (broker_stop_start kafka-2) by 3.9 s; the STOP fired 27 ms after the injection; no traceback or OOM retained (same symptom as 10-10 attempt a3)", - "verdict_sha256": "728b460d8244a36260a428542e93fb6bfec44cd3cae107e684000a8464e1573e", - "teardown": "10 removed, 0 left, volumes 217 -> 217" - }, - { - "dir": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-r2", - "status": "STOPPED", - "error": "SystemExit: G12 orchestrator received signal 15 (duplicate run, stopped by the coordinator)", - "verdict_sha256": "603fe82c761b2633c326fee6b56ad0578710ac4599d0b27f95a9a059846a1ebf", - "teardown": "5 removed, 0 left by label/name; volumes 217 -> 219 at its teardown while the concurrent a2 run held its own volumes; a2's teardown later read 217" - } + "limits": [ + "isolated namespace, same host, TEST_ONLY sources, single projector/Stream replica" ], - "open_harness_item": "keep the monitor container's exit code and OOM state in the evidence", "sources": { "verdict": { - "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/evidence/verdict.json", - "sha256": "05c0652d320929a13073a2ff1eb57a319f5b4a5817f7dd4f917ca8ae914a48b8" + "path": "~/.local/state/qdl-v2/p184-g12-rc12-20261011/evidence/verdict.json", + "sha256": "20f296719c740dd53a2f32f20b0f18b673b7fa082532c54ee478d7166fbc5e3e" }, "teardown": { - "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/evidence/teardown.json", - "sha256": "f17de5ca9539848e581972fd276809d8b9e5fd3fb5972f848c795f60bd722bb8" + "path": "~/.local/state/qdl-v2/p184-g12-rc12-20261011/evidence/teardown.json", + "sha256": "a5fceec5d8f40626ce01dfc83d93514dbd378824957966f944c6920154a3e71b" }, "image_set": { - "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/image-set.json", - "sha256": "7576125751ef20235bad4c139188fe8b31bd10ee3113efea7cd153f11282c87d" - } - }, - "observed": "caught up 483.8 / 405.8 / 275.4 s; all parts PASS", - "full_run": { - "dir": "~/.local/state/qdl-v2/p184-g12-rc10-full3-20261011", - "verdict_sha256": "c7b8ea951fcf2c008e3d97d15584162c064b17ecbbd54b865fbbfb66862cac39", - "teardown_sha256": "fc11efb3ca785a93df184b76caa1c44a0aa2408dd22742440248b87c27173e63", - "harness_commit": "2d4b88c", - "caught_up_s": { - "broker_stop_start": 483.833, - "projector_restart": 405.8, - "stream_restart": 275.425 + "path": "~/.local/state/qdl-v2/p184-g12-rc12-20261011/image-set.json", + "sha256": "838c0503cc99229ac5b1b02393ec8df0fece3abfe07af1ef0fe91d69dc69b85a" + }, + "monitor": { + "path": "~/.local/state/qdl-v2/p184-g12-rc12-20261011/evidence/monitor.jsonl", + "sha256": "8aa135ab0bdaefa6df5cd6aa1f260164dd87f58ebe407341d884f324ba9ad063" }, - "g2_max_ms": "TRADE <= 96.8, QUOTE <= 96.6 (n 1060 / 600 per venue per fault)", - "g3_bar_1m_max_ms": 477.6 + "exit_states": { + "path": "~/.local/state/qdl-v2/p184-g12-rc12-20261011/evidence/exit-states.json", + "sha256": "52d7ce96025dc75a48fad58e64e724bac61acb4c30bd830dbc00767a66206a6d" + } }, - "superseded_bookonly_verdict": { - "verdict": "NOT_MEASURED", - "reason": "isolated run on the release digests (python a0c3649d, rust 96a2c912, writer bae2509f): all three faults caught up <= 600 s and G1 inside in the 300 s window after each; G2/G3 after recovery are NOT_MEASURED (BOOK-only isolated source), and the rule requires G1-G3, so the gate is NOT_MEASURED, not PASS", - "observed": "caught up and G1 INSIDE for all three faults" - } + "earlier_runs": "the rc10 full run p184-g12-rc10-full3-20261011 (PASS, verdict c7b8ea95) and the rc10 attempts stay in history.2026-10-11_rc10_413c5b9_scorecard" }, { "id": "G13", "quantity": "DR01-DR08 acceptance", "rule": "every cell closed once on the final DL + writer digests", "verdict": "DEFERRED", - "reason": "41/41 DL-side cells ok on the release digests; 4 TS-stack steps deferred to the TS release", + "reason": "41/41 DL-side cells ok on the rc12 digests with the final writer 44baacc0; 4 TS-stack steps deferred to the TS release", "measured": { - "dr05-upstream": "6/6 ok", - "dr08-isolated": "4/4 ok (QUALIFICATION)", - "dr05-transport": "9/9 ok", - "dr04-dr06-views": "22/22 ok (DR05.QUIET_TYPED BINANCE/OKX NOT_OBSERVED)" + "dr05-upstream": "6/6 ok (134532Z)", + "dr08-isolated": "4/4 ok (134532Z, ISOLATED_FAULT rows)", + "dr05-transport": "9/9 ok (134532Z; DISCONNECT_RECONNECT counts bridge RECONNECTED controls dated after the cut)", + "dr04-dr06-views": "22/22 ok (140432Z; DR05.QUIET_TYPED BINANCE/OKX ok but NOT_OBSERVED)" }, "digests": { - "dl_sha": "413c5b914f8c6fd0f67429fdddb39451589e9745", - "python": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", - "rust": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", - "writer": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", - "ts_sha": "8b567eb07c9036dbbb42079ed1da90db7e0f5f25" + "dl_sha": "5ee94636a3b41d4d482eadbc9dd80f51e126cf84", + "python": "sha256:3b585597570f3f7065c17b0a7f9228a2dad2f71224905e2f105a1754b9d4a75c", + "rust": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "writer": "sha256:44baacc0a3ff95f71029d230e48e1b7282627f26de9bd99e13d1148862e09068", + "ts_sha": "a37ff4dcef418947b89578c1870726cb03db4fd3" }, - "failed_attempt": "first dr04-dr06-views in 043821Z: DR04.EXPLICIT_LAST@BINANCE refused EXECUTION_CONTEXT_STALE (live LAST older than the probe limit in a quiet Sunday market), a correct typed refusal; re-run 045533Z 22/22 ok. The age sample behind this reading was not retained in a file", + "equivalence_to_release": "rc13 differs from rc12 only by IDR-087 (bar edge backlog probe), which no G13 step exercises; the writer is the final one", + "failed_attempts": [ + { + "dir": "~/.local/state/qdl-v2/p184-g13-20261011T134532Z", + "step": "dr04-dr06-views", + "detail": "DR04.EXPLICIT_LAST@OKX refused EXECUTION_CONTEXT_STALE (LAST trigger) in a quiet market: a correct typed refusal; the re-run 140432Z is 22/22 ok", + "rows": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T134532Z/dr04-dr06-views/views-okx.rows", + "sha256": "efa7882ac1117cf0a0e38a7cc5153b19fc41cf7c41fecaa4a5c7060ba4dc31f3" + } + }, + { + "dir": "~/.local/state/qdl-v2/p184-g13-20261011T131833Z", + "writer": "64347a94 (TS a3035c8)", + "detail": "dr05-transport FAIL: DR05.DISCONNECT_RECONNECT B/O/OKX_INVERSE ok=false with logged_control_events 0 (data recovered, validator APPROVED, ~6.4 s after reconnect); dr04-dr06-views FAIL on DR04.EXPLICIT_LAST@BINANCE EXECUTION_CONTEXT_STALE (quiet market)", + "summary": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T131833Z/summary.json", + "sha256": "0ce63d394a55c0f96a21eee6cba672cbc1390528306474de0ac3fb22992605f8" + }, + "rows": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T131833Z/dr05-transport/dr05-transport.jsonl", + "sha256": "2c9dd2eb5b7ba0afb1d5f191b867765355984ea7180eac5c06591276f59623b1" + } + }, + { + "dir": "~/.local/state/qdl-v2/p184-g13-20261011T133254Z", + "writer": "64347a94 (TS a3035c8)", + "detail": "dr05-transport re-run, same DISCONNECT_RECONNECT result (logged_control_events 0)", + "summary": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T133254Z/summary.json", + "sha256": "205c654d64f1a8d93e5212861f2e32ef14faf58465a47e0213e9a97adb916205" + }, + "rows": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T133254Z/dr05-transport/dr05-transport.jsonl", + "sha256": "78d980f45e37e97e0b653702d5cf225db6081afabd35f2cca364f6e390964743" + } + } + ], + "finding": "a transport cut ends in the bridge reopening the slice with a fresh session, which yields no SDK RECONNECTED control, so no control was logged; the earlier DR05.DISCONNECT_RECONNECT PASS (rc10 run 043821Z) counted startup quota controls logged before the cut, a false positive. TS a37ff4d: the bridge logs RECONNECTED origin=bridge once the reopened slice applies data, and the harness counts only controls dated after the cut", "deferred_cells": [ "dr01-deployed", "dr01-wrong-account", "dr02-dr03-alpha", "dr07-reference" ], - "approval": "owner AskUserQuestion answer 2026-10-11, relayed; p184-g13-inputs/runtime-approval-rc10.md was written by an agent from that relayed approval", + "approval": "owner chat approval 2026-10-11, recorded by the coordinating session in p184-g13-inputs/runtime-approval-rc12.md (not written by the owner)", "sources": { "run_1": { - "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/summary.json", - "sha256": "3a3a34e29715a4b67406b2b5e790bed0c9b8d7f0551498d43d0cd9f9c610a09d" + "path": "~/.local/state/qdl-v2/p184-g13-20261011T134532Z/summary.json", + "sha256": "445e05c9de1a0074fb0f94acc950e25536e9a74342edea44e4142a8d6765e55c" }, "run_2_views": { - "path": "~/.local/state/qdl-v2/p184-g13-20261011T045533Z/summary.json", - "sha256": "309199f1a24c8625ed98d13d572a79f2cbff754359c8f88ce9a21e9c175c3472" - }, - "dr04_dr06_rows": { - "path": "~/.local/state/qdl-v2/p184-g13-20261011T045533Z/dr04-dr06-views/dr04-dr06.jsonl", - "sha256": "2ac4622ce54cff91b4e59ee423067d211460aeb475e8901c6eab155130f9d222" + "path": "~/.local/state/qdl-v2/p184-g13-20261011T140432Z/summary.json", + "sha256": "f0b7b523fa6af40503f009f97d4ebde9119dda228b7902644f93e0629de40a6c" }, "dr05_upstream_rows": { - "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/dr05-upstream/dr05-upstream-rows.jsonl", - "sha256": "700721727d38dbd3d3acbe6ffeeead32e925104d35526a04f2b2ebeceba35ed7" + "path": "~/.local/state/qdl-v2/p184-g13-20261011T134532Z/dr05-upstream/dr05-upstream-rows.jsonl", + "sha256": "b01f3f4ad86bef3b4c982aa86a3cff2ae3f981e5ede5afc04cf5c8932de3825b" }, "dr05_transport_rows": { - "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/dr05-transport/dr05-transport.jsonl", - "sha256": "415538c65c0b6a1d0146264176b5e8ab0f8fece7967f2152ff8a5fd119f4b626" + "path": "~/.local/state/qdl-v2/p184-g13-20261011T134532Z/dr05-transport/dr05-transport.jsonl", + "sha256": "95a6540104b1c566773d8b52281304c15ff671b3e9795ba022a51c77d159ff93" }, "dr08_evidence": { - "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/dr08-isolated/work/evidence.jsonl", - "sha256": "e5b14dde2964f79e52607049fdbeff37461f6964818a1a92884a78cbce83d1d3" + "path": "~/.local/state/qdl-v2/p184-g13-20261011T134532Z/dr08-isolated/work/evidence.jsonl", + "sha256": "b996d55d41354975764fa9dcdd6ac5ae944e73f2497b0b9648dfa42d38f64489" + }, + "dr04_dr06_rows": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T140432Z/dr04-dr06-views/views.rows", + "sha256": "0fb613f61b14e0553103e64a47d040e1e24ab2fbd7f1ec766180170aa7a2ea10" }, "approval": { - "path": "~/.local/state/qdl-v2/p184-g13-inputs/runtime-approval-rc10.md", - "sha256": "551890885673bb569119dc65c218deb0b561c59fec8071bea48a7699eb22d2ce" + "path": "~/.local/state/qdl-v2/p184-g13-inputs/runtime-approval-rc12.md", + "sha256": "9e786bfd2205b5d69977f5fc547a2880d79450bb9856d2a70d1833bbcb638aa8" } } }, { "id": "G14", "measured": { - "dl_cpu_mean_sum_vcpu": 5.37, + "dl_cpu_mean_sum_vcpu": 5.08, "mem_at_or_over_limit": [], "missing_roles": [], "roles": { "binance_bar_edge": { "container": "qdl_v2_stable_candidate-binance_bar_edge-1", - "cpu_max_pct": 76.32, - "cpu_mean_pct": 1.9, - "cpu_p95_pct": 29.25, - "mem_headroom_pct": 71.7, + "cpu_max_pct": 40.35, + "cpu_mean_pct": 1.8, + "cpu_p95_pct": 5.63, + "mem_headroom_pct": 73.3, "mem_limit_mib": 512.0, - "mem_max_mib": 145.0, - "samples": 151, + "mem_max_mib": 136.8, + "samples": 152, "throttled_pct_of_wall": 0.3 }, "ingestor_binance_usdm": { "container": "qdl_v2_stable_candidate-ingestor_binance_usdm-1", - "cpu_max_pct": 22.47, - "cpu_mean_pct": 12.4, - "cpu_p95_pct": 17.85, - "mem_headroom_pct": 86.2, + "cpu_max_pct": 17.73, + "cpu_mean_pct": 11.0, + "cpu_p95_pct": 14.64, + "mem_headroom_pct": 87.4, "mem_limit_mib": 256.0, - "mem_max_mib": 35.3, - "samples": 151, - "throttled_pct_of_wall": 0.1 + "mem_max_mib": 32.2, + "samples": 152, + "throttled_pct_of_wall": 0.0 }, "ingestor_okx_swap": { "container": "qdl_v2_stable_candidate-ingestor_okx_swap-1", - "cpu_max_pct": 18.38, - "cpu_mean_pct": 12.1, - "cpu_p95_pct": 15.81, - "mem_headroom_pct": 87.1, + "cpu_max_pct": 17.58, + "cpu_mean_pct": 10.7, + "cpu_p95_pct": 13.55, + "mem_headroom_pct": 86.7, "mem_limit_mib": 256.0, - "mem_max_mib": 32.9, - "samples": 151, + "mem_max_mib": 34.0, + "samples": 152, "throttled_pct_of_wall": 0.0 }, "kafka1": { "container": "qdl_v2_stable_candidate-kafka1-1", - "cpu_max_pct": 99.17, - "cpu_mean_pct": 80.8, - "cpu_p95_pct": 96.04, - "mem_headroom_pct": 35.7, + "cpu_max_pct": 101.86, + "cpu_mean_pct": 78.0, + "cpu_p95_pct": 91.97, + "mem_headroom_pct": 35.9, "mem_limit_mib": 1536.0, - "mem_max_mib": 987.9, - "samples": 151, - "throttled_pct_of_wall": 0.4 + "mem_max_mib": 984.8, + "samples": 152, + "throttled_pct_of_wall": 0.7 }, "kafka2": { "container": "qdl_v2_stable_candidate-kafka2-1", - "cpu_max_pct": 122.07, - "cpu_mean_pct": 98.3, - "cpu_p95_pct": 111.79, + "cpu_max_pct": 120.88, + "cpu_mean_pct": 94.0, + "cpu_p95_pct": 109.43, "mem_headroom_pct": 54.5, "mem_limit_mib": 2048.0, - "mem_max_mib": 931.5, - "samples": 151, - "throttled_pct_of_wall": 0.3 + "mem_max_mib": 932.3, + "samples": 152, + "throttled_pct_of_wall": 0.2 }, "kafka3": { "container": "qdl_v2_stable_candidate-kafka3-1", - "cpu_max_pct": 131.35, - "cpu_mean_pct": 102.2, - "cpu_p95_pct": 118.61, - "mem_headroom_pct": 33.5, + "cpu_max_pct": 125.4, + "cpu_mean_pct": 97.4, + "cpu_p95_pct": 114.17, + "mem_headroom_pct": 33.9, "mem_limit_mib": 1536.0, - "mem_max_mib": 1022.0, - "samples": 151, - "throttled_pct_of_wall": 0.5 + "mem_max_mib": 1016.0, + "samples": 152, + "throttled_pct_of_wall": 0.3 }, "market_cache": { "container": "qdl_v2_stable_candidate-market_cache-1", - "cpu_max_pct": 12.98, - "cpu_mean_pct": 8.3, - "cpu_p95_pct": 11.87, + "cpu_max_pct": 12.11, + "cpu_mean_pct": 7.6, + "cpu_p95_pct": 11.09, "mem_headroom_pct": 87.4, "mem_limit_mib": 9216.0, "mem_max_mib": 1163.3, - "samples": 151, + "samples": 152, "throttled_pct_of_wall": 0.0 }, "market_data_service_p184w": { "container": "market_data_service_p184w", - "cpu_max_pct": 270.39, - "cpu_mean_pct": 150.7, - "cpu_p95_pct": 212.82, - "mem_headroom_pct": 97.9, + "cpu_max_pct": 221.73, + "cpu_mean_pct": 133.8, + "cpu_p95_pct": 174.45, + "mem_headroom_pct": 98.0, "mem_limit_mib": 31375.4, - "mem_max_mib": 657.1, - "samples": 151, + "mem_max_mib": 614.2, + "samples": 152, "throttled_pct_of_wall": 0.0 }, "market_projector_1": { "container": "qdl_v2_stable_candidate-market_projector_1-1", - "cpu_max_pct": 28.42, - "cpu_mean_pct": 18.8, - "cpu_p95_pct": 23.68, - "mem_headroom_pct": 88.4, + "cpu_max_pct": 24.66, + "cpu_mean_pct": 17.4, + "cpu_p95_pct": 20.44, + "mem_headroom_pct": 88.5, "mem_limit_mib": 512.0, - "mem_max_mib": 59.6, - "samples": 151, + "mem_max_mib": 59.1, + "samples": 152, "throttled_pct_of_wall": 0.0 }, "market_projector_2": { "container": "qdl_v2_stable_candidate-market_projector_2-1", - "cpu_max_pct": 10.47, - "cpu_mean_pct": 8.3, - "cpu_p95_pct": 9.87, - "mem_headroom_pct": 80.7, + "cpu_max_pct": 9.85, + "cpu_mean_pct": 7.9, + "cpu_p95_pct": 9.02, + "mem_headroom_pct": 82.4, "mem_limit_mib": 512.0, - "mem_max_mib": 98.6, - "samples": 151, + "mem_max_mib": 90.2, + "samples": 152, "throttled_pct_of_wall": 0.0 }, "query_kn_1": { "container": "qdl_v2_stable_candidate-query_kn_1-1", - "cpu_max_pct": 129.77, - "cpu_mean_pct": 57.1, - "cpu_p95_pct": 92.01, - "mem_headroom_pct": 14.3, + "cpu_max_pct": 120.85, + "cpu_mean_pct": 64.4, + "cpu_p95_pct": 102.37, + "mem_headroom_pct": 43.8, "mem_limit_mib": 1536.0, - "mem_max_mib": 1316.9, - "samples": 151, - "throttled_pct_of_wall": 0.7 + "mem_max_mib": 863.1, + "samples": 152, + "throttled_pct_of_wall": 1.2 }, "query_kn_2": { "container": "qdl_v2_stable_candidate-query_kn_2-1", - "cpu_max_pct": 130.11, - "cpu_mean_pct": 66.1, - "cpu_p95_pct": 105.09, - "mem_headroom_pct": 22.0, + "cpu_max_pct": 128.97, + "cpu_mean_pct": 56.0, + "cpu_p95_pct": 92.2, + "mem_headroom_pct": 31.2, "mem_limit_mib": 1536.0, - "mem_max_mib": 1198.1, - "samples": 151, - "throttled_pct_of_wall": 1.2 + "mem_max_mib": 1056.8, + "samples": 152, + "throttled_pct_of_wall": 0.6 }, "rust_core": { "container": "qdl_v2_stable_candidate-rust_core-1", - "cpu_max_pct": 25.96, - "cpu_mean_pct": 10.9, - "cpu_p95_pct": 15.09, - "mem_headroom_pct": 79.1, + "cpu_max_pct": 14.73, + "cpu_mean_pct": 9.1, + "cpu_p95_pct": 12.06, + "mem_headroom_pct": 79.8, + "mem_limit_mib": 256.0, + "mem_max_mib": 51.8, + "samples": 152, + "throttled_pct_of_wall": 0.0 + }, + "rust_core_2": { + "container": "qdl_v2_stable_candidate-rust_core_2-1", + "cpu_max_pct": 19.17, + "cpu_mean_pct": 11.4, + "cpu_p95_pct": 14.56, + "mem_headroom_pct": 80.2, "mem_limit_mib": 256.0, - "mem_max_mib": 53.5, - "samples": 151, + "mem_max_mib": 50.8, + "samples": 152, "throttled_pct_of_wall": 0.0 }, - "rust_core_2": { - "container": "qdl_v2_stable_candidate-rust_core_2-1", - "cpu_max_pct": 23.11, - "cpu_mean_pct": 13.0, - "cpu_p95_pct": 17.57, - "mem_headroom_pct": 76.4, - "mem_limit_mib": 256.0, - "mem_max_mib": 60.4, - "samples": 151, - "throttled_pct_of_wall": 0.0 + "rust_core_3": { + "container": "qdl_v2_stable_candidate-rust_core_3-1", + "cpu_max_pct": 10.7, + "cpu_mean_pct": 8.0, + "cpu_p95_pct": 9.4, + "mem_headroom_pct": 85.0, + "mem_limit_mib": 256.0, + "mem_max_mib": 38.4, + "samples": 152, + "throttled_pct_of_wall": 0.0 + }, + "stable_redis": { + "container": "qdl_v2_stable_candidate-stable_redis-1", + "cpu_max_pct": 5.81, + "cpu_mean_pct": 3.2, + "cpu_p95_pct": 5.26, + "mem_headroom_pct": 93.2, + "mem_limit_mib": 160.0, + "mem_max_mib": 10.8, + "samples": 152, + "throttled_pct_of_wall": 0.0 + }, + "stream_kn_1": { + "container": "qdl_v2_stable_candidate-stream_kn_1-1", + "cpu_max_pct": 23.66, + "cpu_mean_pct": 16.4, + "cpu_p95_pct": 19.82, + "mem_headroom_pct": 75.8, + "mem_limit_mib": 512.0, + "mem_max_mib": 124.0, + "samples": 152, + "throttled_pct_of_wall": 0.0 + }, + "stream_kn_2": { + "container": "qdl_v2_stable_candidate-stream_kn_2-1", + "cpu_max_pct": 21.27, + "cpu_mean_pct": 14.2, + "cpu_p95_pct": 17.87, + "mem_headroom_pct": 77.7, + "mem_limit_mib": 512.0, + "mem_max_mib": 114.4, + "samples": 152, + "throttled_pct_of_wall": 0.0 + } + }, + "samples": 152 + }, + "quantity": "resources per role at stage 100", + "reason": "18 roles recorded", + "rule": "CPU/RAM per role recorded with headroom; budget set by the owner", + "verdict": "PASS", + "sources": { + "host": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/stage100/host.json", + "sha256": "a5ba4fcf963ff2c003f4f6f211fa6a8210f453d56b6a37fd3e8fb19276c894e1" + } + }, + "note": "DL mean CPU sum 5.08 vCPU (17 DL roles, writer excluded; recomputed from the per-role means: 5.085), above the owner's 5 vCPU DL budget; step 1 5.18, campaign 4 5.37" + } + ], + "history": { + "2026-10-11_rc10_413c5b9_scorecard": { + "superseded_by": "phase E step-2 campaign 20261011T123525Z + isolated G12/G13 on rc12, release images rc13 (481fd67)", + "sha256_of_committed_file": "8d3902b649ab4c75cd595ede3c7e3df55f7799e27616c9666a2584d739c25936", + "committed_in": "4704edd", + "record": { + "schema": "qdl.release-scorecard.v1", + "release": "2.2.3", + "status": "PREPARED_NOT_PUBLISHED", + "source_commit": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "gate_table": { + "path": "trading-system upgrade/p184-phase-d-gates.md (FROZEN 2026-10-09)", + "ts_commit": "31cf663f5c7c5384ec3dd296061582dee2463c41", + "sha256": "c67bf69bb921c992964eb8ff8b2e1bb17085a943fdf756d55526b6269641d5c4" + }, + "frozen_budget": { + "path": "config/v2/p184-stage100-budget.json", + "sha256": "ff85335b3c8762d9acd8e7230f3c81ee1ec8403aff34cf2fb8f67714521c2541" + }, + "scoring_rule": "steady state >= 30 min at the stage-100 workload plus one burst window (gate table 'Measurement rules'); a measurement outside this rule can FAIL a gate but cannot PASS it. Verdicts: PASS, FAIL, NOT_MEASURED, DEFERRED; a NOT_MEASURED is never upgraded. 'observed' records the value against the threshold when the measurement does not meet the protocol.", + "runtime_identity": { + "release_images": { + "python": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb (qdl-v2-python:2.2.3-rc10-413c5b9)", + "rust": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5 (qdl-v2-rust:2.2.3-rc10-413c5b9)", + "revision_label": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "dl_application_roles_on_release_digests": 12, + "rolled": "2026-10-11T04:30:01Z-04:35:15Z, one role at a time, writer READY 64/64 after every step" + }, + "campaign_4_measurement_identity": { + "note": "campaign 4 ran on the pre-rc10 mix; G12 and G13 ran on the release digests", + "query_kn_1/2": { + "image": "sha256:db5ae6752280765d8b85f6df11136d2cc42bcd7e9dba94b41b0897b681b65736", + "tag": "qdl-v2-python:2.2.3-rc8-6d5da7f", + "source": "6d5da7f", + "diff_to_413c5b9": "qdl/admission/edge.py +9 (INFLIGHT_CAPACITY wait polls); qdl_sdk 2.2.1 (version string, GrpcStreamTransport random start; service code imports only qdl_sdk.models/reference); contracts doc; the Rust gateway source in the tree is not run by Query", + "equivalence": "NOT source-identical: provider-admitted reads (OKX admitted REST and reference, Binance admitted klines) differ when an INFLIGHT_CAPACITY deferral occurs; other Query paths are source-identical" + }, + "stream_kn_1/2": { + "image": "sha256:d32e354bd6628cf465a27f6691cadaa91ce4f70f8f3311f0d454e48c8d1a6b40", + "tag": "qdl-v2-rust:2.2.3-rc9-70bef7d", + "source": "70bef7d", + "diff_to_413c5b9": "qdl/admission/edge.py only (not in the Rust build); rust/, generated/rust, Cargo.toml/lock and config unchanged", + "equivalence": "source-identical for the Stream binary" + }, + "binance_bar_edge": { + "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "tag": "qdl-v2-python:2.2.3-rc10-413c5b9", + "source": "413c5b9", + "equivalence": "the release image itself (rolled 2026-10-10T18:46Z)" + }, + "rust_core x3, market_projector_1/2, ingestor_okx_swap, ingestor_binance_usdm": { + "image": "sha256:64468c72ce5ab36518107014bd4a3e311b96d47fad06e531f6627c46f532a1b7", + "tag": "qdl-v2-rust:2.2.3-rc7d-bbfb3be", + "source": "bbfb3be", + "diff_to_413c5b9": "rust/qdl-stream-gateway only (dadb9c8), a workspace member no other crate depends on; Cargo.toml/lock, generated/rust, config and Dockerfile unchanged", + "equivalence": "source-equivalent for the core, projector and ingestor binaries (rebuilt, not byte-compared)" + } + }, + "ts_writer": { + "container": "market_data_service_p184w", + "image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", + "source": "8b567eb07c9036dbbb42079ed1da90db7e0f5f25", + "sdk": "2.1.0", + "env_campaign_4": { + "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH": "1", + "MARKET_DATA_FIXED_WORKER_PINS": "OKX:BTCUSDT=2,OKX:ETHUSDT=0", + "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE": "BINANCE=100" + }, + "env_source": { + "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/inspect-20261011T042442Z.json", + "sha256": "ee6c5ff47f9d4372beb8a8ab52c48cb64f4c9573d8c4837adf875abc5ed7421d" + }, + "after_campaign": "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH=0 since 2026-10-11T04:24:53Z (docker inspect)" + }, + "kafka_runtime_config": { + "change": "cluster dynamic default log.cleaner.io.max.bytes.per.second=8388608", + "applied_at": "2026-10-11T03:40:56Z (before campaign 4)", + "receipt": { + "path": "~/.local/state/qdl-v2/kafka-cleaner-throttle-20261010/after.out", + "sha256": "8a635a9bd5df3efc07906db02a0fdb45cc799ac502929b46d9680b8ee884302c" + } + } + }, + "campaign": { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z", + "scorecard": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/scorecard.json", + "sha256": "0133670885374c1a280a8fcd4c34f6dd704e29904d5f492bbb08853398eb12e4" + }, + "campaign_json": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/campaign.json", + "sha256": "a7bbf80073b1796d55fdcffdcd87476de28bde4622559ab55b3ee536c35b64e9" + }, + "protocol_met": true, + "protocol_problems": [], + "load_window_unix_s": [ + 1791690245.4681032, + 1791692045.4681032 + ], + "tooling_source": "2f489b186d8d590ffa59b6dae2e41caa2fa400ad", + "tool_sha256": "39c22d78d37dbacac0042c85a6bfafafdd3dc70a9b21179c053f30a9d6cc19f3", + "demanded_slices": 64, + "writer_restarts": 0, + "campaign_summary": { + "FAIL": [ + "G2", + "G8" + ], + "NOT_MEASURED": [ + "G4", + "G12", + "G13" + ], + "PASS": [ + "G1", + "G3", + "G5", + "G6", + "G7", + "G9", + "G10", + "G11", + "G14" + ] + }, + "g12_g13": "the campaign scorer marks G12/G13 NOT_MEASURED by design; they are scored from their own receipts below" + }, + "summary": { + "PASS": [ + "G1", + "G3", + "G5", + "G6", + "G7", + "G9", + "G10", + "G11", + "G12", + "G14" + ], + "FAIL": [ + "G2", + "G8" + ], + "NOT_MEASURED": [ + "G4" + ], + "DEFERRED": [ + "G13" + ] + }, + "owner_decision": "2026-10-11 (AskUserQuestion): release the DL with G2 and G8 FAIL recorded; writer throughput (SDK batch decode, Rust projection, EVAL chunking) is the first item of TS phase E; no gate is widened. This is not a PASS and does not change any verdict", + "gates": [ + { + "id": "G1", + "measured": [ + { + "feed_venue": "BOOK_DELTA|BINANCE", + "max": 1736.7, + "n": 83994, + "observed": "INSIDE", + "p50": 108.4, + "p95": 177.4, + "p99": 334.1, + "split_p95_ms": { + "ingest_to_sdk_ms": 90.8, + "sdk_to_reply_ms": 62.6, + "venue_to_ingest_ms": 30.1 + } + }, + { + "feed_venue": "BOOK_DELTA|OKX", + "max": 1844.2, + "n": 97565, + "observed": "INSIDE", + "p50": 115.3, + "p95": 191.5, + "p99": 380.8, + "split_p95_ms": { + "ingest_to_sdk_ms": 93.7, + "sdk_to_reply_ms": 72.5, + "venue_to_ingest_ms": 34.3 + } + }, + { + "feed_venue": "BOOK_SNAPSHOT|BINANCE", + "max": 1211.4, + "n": 9667, + "observed": "INSIDE", + "p50": 123.0, + "p95": 245.6, + "p99": 423.9, + "split_p95_ms": { + "ingest_to_sdk_ms": 114.7, + "sdk_to_reply_ms": 117.7, + "venue_to_ingest_ms": 31.6 + } + }, + { + "feed_venue": "BOOK_SNAPSHOT|OKX", + "max": 1144.5, + "n": 11562, + "observed": "INSIDE", + "p50": 127.7, + "p95": 236.7, + "p99": 419.0, + "split_p95_ms": { + "ingest_to_sdk_ms": 105.9, + "sdk_to_reply_ms": 121.7, + "venue_to_ingest_ms": 34.6 + } + } + ], + "quantity": "end-to-end to TS cache (venue event -> TS Redis APPLY reply)", + "reason": "every BOOK row inside", + "rule": "p95 <= 500 ms, p99 <= 1000 ms (n >= 100), max < 2000 ms per feed/venue", + "sources": { + "transit": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/transit-load.json", + "sha256": "d1376db1a50db991df5e9e9e80212463ad095640ee00a67ab673e473145d73f6" + }, + "mark": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/mark-load.json", + "sha256": "7ac870af08414f19f852af5851928a3e3030fd5898ade38b1ff8e417ec6850a6" + } + }, + "verdict": "PASS" + }, + { + "id": "G2", + "measured": [ + { + "feed_venue": "TRADE|BINANCE", + "max": 5338.8, + "n": 160328, + "observed": "OUTSIDE", + "p50": 153.7, + "p95": 1562.1, + "p99": 4326.4, + "split_p95_ms": { + "ingest_to_sdk_ms": 1430.7, + "sdk_to_reply_ms": 118.2, + "venue_to_ingest_ms": 39.6 + } + }, + { + "feed_venue": "TRADE|OKX", + "max": 4301.8, + "n": 39292, + "observed": "OUTSIDE", + "p50": 138.3, + "p95": 2084.2, + "p99": 3782.9, + "split_p95_ms": { + "ingest_to_sdk_ms": 1866.1, + "sdk_to_reply_ms": 152.8, + "venue_to_ingest_ms": 72.6 + } + }, + { + "feed_venue": "QUOTE|BINANCE", + "max": 1439.2, + "n": 116367, + "observed": "OUTSIDE", + "p50": 125.9, + "p95": 188.0, + "p99": 300.2, + "split_p95_ms": { + "ingest_to_sdk_ms": 120.0, + "sdk_to_reply_ms": 46.9, + "venue_to_ingest_ms": 29.7 + } + }, + { + "feed_venue": "QUOTE|OKX", + "max": 1340.6, + "n": 98174, + "observed": "OUTSIDE", + "p50": 127.2, + "p95": 207.1, + "p99": 339.3, + "split_p95_ms": { + "ingest_to_sdk_ms": 125.0, + "sdk_to_reply_ms": 58.4, + "venue_to_ingest_ms": 30.5 + } + }, + { + "feed_venue": "MARK_INDEX_PRICE|BINANCE", + "max": 1125.4, + "n": 8989, + "observed": "OUTSIDE", + "p50": 302.0, + "p95": 396.7, + "p99": 537.2 + }, + { + "feed_venue": "MARK_INDEX_PRICE|OKX", + "max": 1227.8, + "n": 21437, + "observed": "OUTSIDE", + "p50": 234.9, + "p95": 348.5, + "p99": 411.0 + } + ], + "quantity": "end-to-end to TS cache, new-event transit", + "reason": "outside: TRADE|BINANCE, TRADE|OKX, QUOTE|BINANCE, QUOTE|OKX; outside: MARK|BINANCE, MARK|OKX", + "rule": "TRADE p95 <= 500 ms and max < 1500 ms; QUOTE and MARK p95 <= 500 ms and max < 1000 ms", + "sources": { + "transit": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/transit-load.json", + "sha256": "d1376db1a50db991df5e9e9e80212463ad095640ee00a67ab673e473145d73f6" + }, + "mark": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/mark-load.json", + "sha256": "7ac870af08414f19f852af5851928a3e3030fd5898ade38b1ff8e417ec6850a6" + } + }, + "verdict": "FAIL", + "attribution": { + "basis": "read-only per-second analysis of the campaign-4 delivery trace (214 trace parts, load window of the campaign scorecard), records APPLIED with SDK_STREAM_RETURN and a venue clock; run in a disposable --network none container with the trace mounted read-only", + "seconds_in_window": 1801, + "seconds_max_e2e_lt_500_ms": 1699, + "seconds_max_e2e_gt_1500_ms": 28, + "seconds_gt_1500_where_every_record_over_1500_is_TRADE": 25, + "seconds_gt_1500_that_also_hold_BOOK_DELTA_over_1500": "3 BINANCE / 2 OKX second-feed hits (G1 max still < 2000 ms)", + "largest_TRADE_burst": "4274 TRADE records in one source second on one slice (BINANCE:USD_M:ETHUSDT) on one writer worker", + "max_applied_TRADE_per_worker_second": 1138, + "TRADE_records_over_1500_ms_by_slice": { + "BINANCE:USD_M:ETHUSDT": 4844, + "BINANCE:USD_M:SOLUSDT": 2287, + "OKX:SWAP:ETH-USDT-SWAP": 2211, + "BINANCE:USD_M:BNBUSDT": 642, + "BINANCE:USD_M:BTCUSDT": 572, + "OKX:SWAP:SOL-USDT-SWAP": 469 + }, + "segments_p95_ms_TRADE_BINANCE": { + "ingest_to_sdk_ms": 1430.7, + "sdk_to_reply_ms": 118.2, + "venue_to_ingest_ms": 39.6 + }, + "canonical_visibility": "G4 venue -> canonical TRADE p95 108.2-136.6 ms in all 5 windows of the same campaign", + "TRADE_volume": "TRADE BINANCE n 160328 against 69966 / 74061 in the 2026-10-10 campaigns 171129Z / 174944Z (about 2.2x)", + "integrity": "G10 in the same campaign: 0 gaps, 0 duplicates applied, 0 replay-expired, 0 overflow", + "reading": "the tail is single-slice TRADE bursts queued behind one writer worker (arrival up to ~4.3k rec/s vs at most ~1.1k applied rec/s per worker); the DL canonical path stays inside G4. Stream-side dispatch/queue metrics for this campaign were not retained in a file and are not cited here.", + "owner_decision": "2026-10-11 (AskUserQuestion): release DL; writer throughput (SDK batch decode, Rust projection, EVAL chunking) moves to TS phase E as its first item; the gate is not widened" + } + }, + { + "id": "G3", + "measured": { + "per_venue": { + "BINANCE": { + "max": 3007, + "min": 109, + "n": 160, + "p50": 468, + "p95": 1754, + "p99": 2861 + }, + "OKX": { + "max": 1822, + "min": 316, + "n": 160, + "p50": 1034, + "p95": 1535, + "p99": 1822 + } + }, + "poll_ms": 250, + "pooled": { + "max": 3007, + "min": 109, + "n": 320, + "p50": 758, + "p95": 1535, + "p99": 2435 + }, + "read_errors": 0, + "unresolved_slices": [], + "unseen_total": 0 + }, + "quantity": "end-to-end to TS cache, BAR from close (first visibility of the closed bar)", + "reason": "g3 probe", + "rule": "p95 <= 5 s, max < 60 s", + "sources": { + "g3": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g3/g3-summary.json", + "sha256": "a15a368f3c05dc0de7f0d072bf2518574fea0168d3c3ca73b21558575ae13574" + } + }, + "verdict": "PASS" + }, + { + "id": "G4", + "inverse_rows": [ + { + "book_delta|OKX_INVERSE|": { + "max": 108.9, + "n": 106, + "p95": 101.6 + }, + "book_snapshot|OKX_INVERSE|": { + "max": 156.4, + "n": 112, + "p95": 102.3 + }, + "mark_index_price|OKX_INVERSE|": { + "max": 1793.0, + "n": 110, + "p95": 1310.1 + }, + "quote|OKX_INVERSE|": { + "max": 147.5, + "n": 64, + "p95": 139.0 + } + }, + { + "book_delta|OKX_INVERSE|": { + "max": 198.5, + "n": 112, + "p95": 121.0 + }, + "book_snapshot|OKX_INVERSE|": { + "max": 198.7, + "n": 124, + "p95": 143.3 + }, + "mark_index_price|OKX_INVERSE|": { + "max": 2041.9, + "n": 109, + "p95": 1248.0 + }, + "quote|OKX_INVERSE|": { + "max": 298.2, + "n": 96, + "p95": 154.8 + } + }, + { + "book_delta|OKX_INVERSE|": { + "max": 126.2, + "n": 112, + "p95": 107.2 + }, + "book_snapshot|OKX_INVERSE|": { + "max": 147.9, + "n": 120, + "p95": 109.6 + }, + "mark_index_price|OKX_INVERSE|": { + "max": 1610.9, + "n": 110, + "p95": 1076.3 + }, + "quote|OKX_INVERSE|": { + "max": 217.1, + "n": 70, + "p95": 145.4 + } + }, + { + "book_delta|OKX_INVERSE|": { + "max": 232.3, + "n": 125, + "p95": 121.9 + }, + "book_snapshot|OKX_INVERSE|": { + "max": 199.3, + "n": 142, + "p95": 126.8 + }, + "mark_index_price|OKX_INVERSE|": { + "max": 1932.8, + "n": 123, + "p95": 1079.1 + }, + "quote|OKX_INVERSE|": { + "max": 289.6, + "n": 99, + "p95": 162.8 + } + }, + { + "book_delta|OKX_INVERSE|": { + "max": 340.5, + "n": 113, + "p95": 118.7 + }, + "book_snapshot|OKX_INVERSE|": { + "max": 510.7, + "n": 128, + "p95": 141.4 + }, + "mark_index_price|OKX_INVERSE|": { + "max": 2032.2, + "n": 111, + "p95": 1166.9 + }, + "quote|OKX_INVERSE|": { + "max": 385.5, + "n": 81, + "p95": 148.6 + } + } + ], + "measured": [ + { + "limit_p95_ms": 40728.7, + "n_per_window": [ + 3, + 0, + 0, + 0, + 0 + ], + "p95_ms_per_window": [ + 7106.7, + null, + null, + null, + null + ], + "row": "bar|BINANCE|15m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 257.6, + "n_per_window": [ + 10, + 10, + 10, + 10, + 10 + ], + "p95_ms_per_window": [ + 127.5, + 106.6, + 116.9, + 140.6, + 94.3 + ], + "row": "bar|BINANCE|1m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 25836.6, + "n_per_window": [ + 0, + 0, + 0, + 1, + 0 + ], + "p95_ms_per_window": [ + null, + null, + null, + 11641.8, + null + ], + "row": "bar|BINANCE|30m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 27273.2, + "n_per_window": [ + 1, + 5, + 5, + 0, + 5 + ], + "p95_ms_per_window": [ + 7099.6, + 6853.7, + 6838.9, + null, + 6840.1 + ], + "row": "bar|BINANCE|3m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 73817.2, + "n_per_window": [ + 6, + 8, + 7, + 6, + 9 + ], + "p95_ms_per_window": [ + 61364.0, + 61099.9, + 61105.6, + 61223.5, + 61238.3 + ], + "row": "bar|OKX|1m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 2160788.8, + "n_per_window": [ + 0, + 0, + 0, + 1, + 0 + ], + "p95_ms_per_window": [ + null, + null, + null, + 1801148.6, + null + ], + "row": "bar|OKX|30m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 217582.1, + "n_per_window": [ + 3, + 2, + 3, + 1, + 1 + ], + "p95_ms_per_window": [ + 181246.0, + 181058.9, + 180978.3, + 180970.0, + 181286.0 + ], + "row": "bar|OKX|3m", + "verdict": "PASS" + }, + { + "limit_p95_ms": 263.3, + "n_per_window": [ + 763, + 790, + 791, + 880, + 783 + ], + "p95_ms_per_window": [ + 101.4, + 130.3, + 106.6, + 128.7, + 124.1 + ], + "row": "book_delta|BINANCE|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 289.1, + "n_per_window": [ + 1085, + 1076, + 1081, + 1210, + 1077 + ], + "p95_ms_per_window": [ + 106.1, + 132.9, + 112.1, + 132.7, + 125.6 + ], + "row": "book_delta|OKX|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 260.0, + "n_per_window": [ + 1000, + 1329, + 1200, + 1687, + 1258 + ], + "p95_ms_per_window": [ + 103.4, + 155.1, + 108.4, + 140.5, + 148.4 + ], + "row": "book_snapshot|BINANCE|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 270.1, + "n_per_window": [ + 1713, + 2375, + 2293, + 2712, + 2434 + ], + "p95_ms_per_window": [ + 107.2, + 152.5, + 113.4, + 142.6, + 140.5 + ], + "row": "book_snapshot|OKX|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 370.9, + "n_per_window": [ + 374, + 390, + 379, + 435, + 382 + ], + "p95_ms_per_window": [ + 198.4, + 220.2, + 202.8, + 234.8, + 207.7 + ], + "row": "mark_index_price|BINANCE|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 1586.6, + "n_per_window": [ + 662, + 660, + 662, + 739, + 662 + ], + "p95_ms_per_window": [ + 1432.2, + 1281.4, + 1372.2, + 1109.2, + 1155.3 + ], + "row": "mark_index_price|OKX|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 292.7, + "n_per_window": [ + 554, + 578, + 564, + 640, + 571 + ], + "p95_ms_per_window": [ + 131.6, + 152.5, + 137.5, + 158.3, + 152.0 + ], + "row": "quote|BINANCE|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 283.3, + "n_per_window": [ + 601, + 652, + 625, + 727, + 634 + ], + "p95_ms_per_window": [ + 134.8, + 155.5, + 135.6, + 155.2, + 152.0 + ], + "row": "quote|OKX|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 269.3, + "n_per_window": [ + 319, + 439, + 401, + 522, + 447 + ], + "p95_ms_per_window": [ + 108.2, + 133.2, + 109.0, + 125.8, + 118.9 + ], + "row": "trade|BINANCE|", + "verdict": "PASS" + }, + { + "limit_p95_ms": 257.4, + "n_per_window": [ + 217, + 353, + 295, + 423, + 351 + ], + "p95_ms_per_window": [ + 111.5, + 125.4, + 110.8, + 136.6, + 129.6 + ], + "row": "trade|OKX|", + "verdict": "PASS" + } + ], + "quantity": "delivery lag, venue -> canonical (read_committed visibility, upper bound)", + "reason": "observed rows without a frozen baseline: bar|BINANCE|2h, bar|BINANCE|4h, bar|BINANCE|5m, bar|OKX|15m, bar|OKX|4h, bar|OKX|5m, book_delta|OKX_INVERSE|, book_snapshot|OKX_INVERSE|, mark_index_price|OKX_INVERSE|, quote|OKX_INVERSE|", + "rule": "candidate p95 <= frozen baseline p95 x 1.2 per row; missing coverage blocks the gate", + "verdict": "NOT_MEASURED", + "sources": { + "g4": [ + { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-1-summary.json", + "sha256": "02a554c56d9215c9e48678df5389305f1acd1a108da1d34bf46bca455a581e90" + }, + { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-2-summary.json", + "sha256": "42aa78a1bb21566c65510e53afe6b4db0afd0af52d9abc6b21b4e538af88a587" + }, + { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-3-summary.json", + "sha256": "013e6929f3bb978d16acb37e6e64e0d4961971c7c7cf43e9dbf5f52cc617e57e" + }, + { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-4-summary.json", + "sha256": "86494b6575b97c101459cb6e98257f3c3d4f1a98fae62b2482d69d940079288d" + }, + { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g4-5-summary.json", + "sha256": "0111526b3ba62b40c8bfd5c4dd30cb5d9f6dd7a4ea14db0043beebbeb54ca2fe" + } + ] + }, + "baselined_rows": { + "rows": 17, + "pass": 17 + }, + "note": "every row with a frozen baseline PASS in every window it was sampled; the gate stays NOT_MEASURED because observed rows have no frozen baseline and no baseline is added after measuring" + }, + { + "id": "G5", + "measured": [ + { + "count": 355, + "gate": "latency:BAR_LATEST:BINANCE", + "max": 396.156, + "p50": 21.767, + "p95": 45.867, + "p99": 237.583, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 1000, + "target_p99": 2000 + }, + { + "count": 355, + "gate": "latency:BAR_LATEST:OKX", + "max": 622.885, + "p50": 28.809, + "p95": 59.219, + "p99": 205.444, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 1000, + "target_p99": 2000 + }, + { + "count": 355, + "gate": "latency:L2_SNAPSHOT:BINANCE", + "max": 417.651, + "p50": 26.463, + "p95": 65.632, + "p99": 293.751, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 300, + "target_p99": 750 + }, + { + "count": 355, + "gate": "latency:L2_SNAPSHOT:OKX", + "max": 322.334, + "p50": 25.784, + "p95": 94.998, + "p99": 230.694, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 300, + "target_p99": 750 + }, + { + "count": 53250, + "gate": "latency:MARK_INDEX_REFERENCE:BINANCE", + "max": 698.315, + "p50": 23.67, + "p95": 61.139, + "p99": 103.998, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 250, + "target_p99": 500 + }, + { + "count": 53250, + "gate": "latency:MARK_INDEX_REFERENCE:OKX", + "max": 716.352, + "p50": 22.85, + "p95": 62.527, + "p99": 93.194, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 250, + "target_p99": 500 + }, + { + "count": 35500, + "gate": "latency:QUOTE_SNAPSHOT:BINANCE", + "max": 626.853, + "p50": 13.151, + "p95": 34.995, + "p99": 89.424, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 100, + "target_p99": 250 + }, + { + "count": 35500, + "gate": "latency:QUOTE_SNAPSHOT:OKX", + "max": 675.312, + "p50": 13.823, + "p95": 39.259, + "p99": 80.913, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 100, + "target_p99": 250 + }, + { + "count": 354, + "gate": "latency:TRADE_SNAPSHOT:BINANCE", + "max": 234.675, + "p50": 10.968, + "p95": 29.074, + "p99": 84.744, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 100, + "target_p99": 250 + }, + { + "count": 355, + "gate": "latency:TRADE_SNAPSHOT:OKX", + "max": 329.802, + "p50": 20.276, + "p95": 41.388, + "p99": 198.375, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 100, + "target_p99": 250 + }, + { + "bad": 0, + "gate": "cold:2500_5000_overlap_per_venue", + "runs": 4, + "status": "PASS" + } + ], + "quantity": "request latency", + "reason": "11 latency/cold gates", + "rule": "certified stage-50 p95/p99 per class and venue; frozen cold-shape limits", + "verdict": "PASS", + "sources": { + "gates": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/stage100/gates.json", + "sha256": "63f12e8c6df961b0505fd75ab8df9fe2a34e4fa7b2d985cb025e17a6785bcc43" + }, + "receipt": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/stage100/receipt.json", + "sha256": "a79c9c9d5989f3befb770449badac8cdc9330ec30365b659b96c61cb7e54a0c3" + } + } + }, + { + "id": "G6", + "measured": { + "FUNDING_RATE|BINANCE|@REQUIREMENT_ALPHA": { + "age_basis": "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR", + "age_ms": { + "basis": "reservoir sample per worker (exact n and max above)", + "max_ms": 15242197.0, + "n": 150, + "p50_ms": 14352510.5, + "p95_ms": 15156931.0, + "p99_ms": 15241597.0 + }, + "classes": { + "ELIGIBLE": 150 + }, + "max_age_ms": 15242197, + "max_client_age_ms": 15242303, + "n": 150, + "policy_basis": "REQUIREMENT_ALPHA", + "policy_ms": 32400000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "FUNDING_RATE|OKX|@REQUIREMENT_ALPHA": { + "age_basis": "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR", + "age_ms": { + "basis": "reservoir sample per worker (exact n and max above)", + "max_ms": 15239191.0, + "n": 150, + "p50_ms": 14342900.0, + "p95_ms": 15141540.0, + "p99_ms": 15223646.0 + }, + "classes": { + "ELIGIBLE": 150 + }, + "max_age_ms": 15239191, + "max_client_age_ms": 15239300, + "n": 150, + "policy_basis": "REQUIREMENT_ALPHA", + "policy_ms": 28800000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "MARK_INDEX_PRICE|BINANCE|#ts-probe": { + "age_basis": "CLIENT_RECEIPT_MINUS_NEWEST_SOURCE_EVENT", + "age_ms": { + "basis": "first 20000 ages per key (exact n and max above)", + "max_ms": 1582, + "n": 2990, + "p50_ms": 670.0, + "p95_ms": 1118, + "p99_ms": 1176 + }, + "classes": { + "ELIGIBLE": 2990 + }, + "max_age_ms": 1582, + "max_client_age_ms": 1501, + "n": 2990, + "policy_basis": "D3_EXECUTION", + "policy_ms": 2000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "MARK_INDEX_PRICE|BINANCE|@REQUIREMENT_ALPHA": { + "age_basis": "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR", + "age_ms": { + "basis": "reservoir sample per worker (exact n and max above)", + "max_ms": 1184.0, + "n": 4096, + "p50_ms": 669.0, + "p95_ms": 1048.0, + "p99_ms": 1143.0 + }, + "classes": { + "ELIGIBLE": 63000 + }, + "max_age_ms": 1668, + "max_client_age_ms": 1780, + "n": 63000, + "policy_basis": "REQUIREMENT_ALPHA", + "policy_ms": 60000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "MARK_INDEX_PRICE|OKX|#ts-probe": { + "age_basis": "CLIENT_RECEIPT_MINUS_NEWEST_SOURCE_EVENT", + "age_ms": { + "basis": "first 20000 ages per key (exact n and max above)", + "max_ms": 2177, + "n": 3588, + "p50_ms": 502.0, + "p95_ms": 1331, + "p99_ms": 1923 + }, + "classes": { + "ELIGIBLE": 3563, + "ELIGIBLE_QUIET": 25 + }, + "max_age_ms": 2177, + "max_client_age_ms": 2109, + "n": 3588, + "policy_basis": "D3_EXECUTION", + "policy_ms": 2000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "MARK_INDEX_PRICE|OKX|@REQUIREMENT_ALPHA": { + "age_basis": "REQUEST_SENT_MINUS_SERVER_FRESHNESS_ANCHOR", + "age_ms": { + "basis": "reservoir sample per worker (exact n and max above)", + "max_ms": 2286.0, + "n": 4096, + "p50_ms": 444.5, + "p95_ms": 1265.0, + "p99_ms": 1866.0 + }, + "classes": { + "ELIGIBLE": 63180 + }, + "max_age_ms": 2291, + "max_client_age_ms": 2401, + "n": 63180, + "policy_basis": "REQUIREMENT_ALPHA", + "policy_ms": 60000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "QUOTE|BINANCE|#ts-probe": { + "age_basis": null, + "age_ms": { + "basis": "first 20000 ages per key (exact n and max above)", + "max_ms": 1237, + "n": 2990, + "p50_ms": 324.0, + "p95_ms": 530, + "p99_ms": 739 + }, + "classes": { + "ELIGIBLE": 2990 + }, + "max_age_ms": 1237, + "max_client_age_ms": null, + "n": 2990, + "policy_basis": "D3_EXECUTION", + "policy_ms": 2000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "QUOTE|BINANCE|@REQUIREMENT_ALPHA": { + "age_basis": null, + "age_ms": { + "basis": "reservoir sample per worker (exact n and max above)", + "max_ms": 1277.0, + "n": 4096, + "p50_ms": 321.0, + "p95_ms": 522.0, + "p99_ms": 708.0 + }, + "classes": { + "ELIGIBLE": 36000 + }, + "max_age_ms": 1580, + "max_client_age_ms": null, + "n": 36000, + "policy_basis": "REQUIREMENT_ALPHA", + "policy_ms": 5000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "QUOTE|OKX|#ts-probe": { + "age_basis": null, + "age_ms": { + "basis": "first 20000 ages per key (exact n and max above)", + "max_ms": 13027, + "n": 3583, + "p50_ms": 353, + "p95_ms": 1083, + "p99_ms": 3074 + }, + "classes": { + "ELIGIBLE": 3515, + "ELIGIBLE_QUIET": 68 + }, + "max_age_ms": 13027, + "max_client_age_ms": null, + "n": 3583, + "policy_basis": "D3_EXECUTION", + "policy_ms": 2000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "QUOTE|OKX|@REQUIREMENT_ALPHA": { + "age_basis": null, + "age_ms": { + "basis": "reservoir sample per worker (exact n and max above)", + "max_ms": 3889.0, + "n": 4096, + "p50_ms": 338.0, + "p95_ms": 675.0, + "p99_ms": 1015.0 + }, + "classes": { + "ELIGIBLE": 36100 + }, + "max_age_ms": 3990, + "max_client_age_ms": null, + "n": 36100, + "policy_basis": "REQUIREMENT_ALPHA", + "policy_ms": 5000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "TRADE|BINANCE|#ts-probe": { + "age_basis": null, + "age_ms": { + "basis": "first 20000 ages per key (exact n and max above)", + "max_ms": 6770, + "n": 2985, + "p50_ms": 558, + "p95_ms": 2195, + "p99_ms": 3565 + }, + "classes": { + "ELIGIBLE": 2936, + "NOT_ELIGIBLE": 49 + }, + "max_age_ms": 6770, + "max_client_age_ms": null, + "n": 2985, + "policy_basis": "D3_EXECUTION", + "policy_ms": 3000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + }, + "TRADE|OKX|#ts-probe": { + "age_basis": null, + "age_ms": { + "basis": "first 20000 ages per key (exact n and max above)", + "max_ms": 40706, + "n": 2985, + "p50_ms": 807, + "p95_ms": 7967, + "p99_ms": 19634 + }, + "classes": { + "ELIGIBLE": 2522, + "NOT_ELIGIBLE": 463 + }, + "max_age_ms": 40706, + "max_client_age_ms": null, + "n": 2985, + "policy_basis": "D3_EXECUTION", + "policy_ms": 3000, + "reasons": {}, + "refusals": {}, + "unsafe": 0 + } + }, + "quantity": "durable-view event age and typed execution quality at the Query endpoint", + "reason": "every required D3 binding measured, 0 unsafe", + "rule": "per-binding D3 policy of the TS EXECUTION consumer (TRADE 3000, QUOTE/MARK/BOOK_DELTA 2000, BOOK_SNAPSHOT 60000, BAR1m 180000 ms, min with the requirement); zero unsafe eligibility; decided by D3_EXECUTION rows only", + "sources": { + "g6_ts": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g6-ts/summary.json", + "sha256": "229e038090ec1000c19ceca41eb805bcda1b4a7827e2e0b41803f4cc60aeb659" + } + }, + "ts_probe": { + "errors": {}, + "per_binding_period_s": 3.2, + "reads": 19121, + "stopped_by": "signal", + "targets": 32, + "window_unix_s": [ + 1791690153.9782138, + 1791692078.7164075 + ] + }, + "verdict": "PASS", + "limits": [ + "BOOK_DELTA has no Query shape in the frozen workload, so it is not a G6 row", + "scorer: ba496ff (read-own-requirement, D3 by TS EXECUTION probe) + 2f489b1 (anchor per product)" + ] + }, + { + "id": "G7", + "measured": { + "BOOK_DELTA": { + "sdk_to_writer_ms": { + "max": 1311.7, + "min": 0.3, + "n": 182930, + "p50": 23.0, + "p95": 51.0, + "p99": 135.5 + }, + "writer_to_redis_ms": { + "max": 530.3, + "min": 0.5, + "n": 181572, + "p50": 3.4, + "p95": 18.4, + "p99": 52.0 + } + }, + "BOOK_SNAPSHOT": { + "sdk_to_writer_ms": { + "max": 616.3, + "min": 12.3, + "n": 21229, + "p50": 31.0, + "p95": 79.3, + "p99": 141.0 + }, + "writer_to_redis_ms": { + "max": 379.4, + "min": 1.5, + "n": 21229, + "p50": 7.3, + "p95": 41.2, + "p99": 128.4 + } + }, + "MARK_INDEX_PRICE": { + "sdk_to_writer_ms": { + "max": 8.1, + "min": 0.2, + "n": 30441, + "p50": 0.5, + "p95": 0.6, + "p99": 0.9 + }, + "writer_to_redis_ms": { + "max": 520.7, + "min": 1.1, + "n": 30441, + "p50": 4.0, + "p95": 20.2, + "p99": 79.3 + } + }, + "QUOTE": { + "sdk_to_writer_ms": { + "max": 661.6, + "min": 1.3, + "n": 214601, + "p50": 22.6, + "p95": 40.7, + "p99": 99.3 + }, + "writer_to_redis_ms": { + "max": 473.8, + "min": 0.5, + "n": 233880, + "p50": 1.4, + "p95": 10.5, + "p99": 48.6 + } + }, + "TRADE": { + "sdk_to_writer_ms": { + "max": 501.5, + "min": 0.3, + "n": 203657, + "p50": 26.9, + "p95": 83.0, + "p99": 148.1 + }, + "writer_to_redis_ms": { + "max": 438.0, + "min": 0.5, + "n": 204778, + "p50": 7.7, + "p95": 46.7, + "p99": 101.9 + } + } + }, + "quantity": "TS consumer path", + "reason": "every feed inside", + "rule": "SDK -> writer p95 <= 100 ms; writer -> Redis reply p95 <= 50 ms per feed", + "verdict": "PASS", + "sources": { + "transit": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/transit-load.json", + "sha256": "d1376db1a50db991df5e9e9e80212463ad095640ee00a67ab673e473145d73f6" + } + } + }, + { + "id": "G8", + "measured": { + "monitor_clients_max": 0, + "ping_all_ms": { + "max": 5.869, + "min": 0.1, + "n": 3850, + "p50": 0.154, + "p95": 0.51, + "p99": 1.275 + }, + "ping_failures": 0, + "ping_steady_ms": { + "max": 5.869, + "min": 0.1, + "n": 3599, + "p50": 0.155, + "p95": 0.516, + "p99": 1.274 + }, + "slowlog_new_steady": 6, + "slowlog_new_total": 6, + "slowlog_steady_rows": [ + { + "command": "EVALSHA", + "duration_us": 10016, + "id": 976628, + "start_s": 1791690302 + }, + { + "command": "EVALSHA", + "duration_us": 16120, + "id": 976629, + "start_s": 1791690605 + }, + { + "command": "EVALSHA", + "duration_us": 13258, + "id": 976630, + "start_s": 1791690763 + }, + { + "command": "EVALSHA", + "duration_us": 12964, + "id": 976631, + "start_s": 1791690842 + }, + { + "command": "EVALSHA", + "duration_us": 18431, + "id": 976632, + "start_s": 1791691251 + }, + { + "command": "EVALSHA", + "duration_us": 19732, + "id": 976633, + "start_s": 1791691386 + } + ], + "slowlog_threshold_us": 10000, + "windows": [ + { + "core_fraction": 0.0687, + "from_ns": 1791690153422572495, + "judged": true, + "ping_ms": { + "max": 2.9, + "min": 0.1, + "n": 1199, + "p50": 0.154, + "p95": 0.476, + "p99": 1.245 + }, + "seconds": 590.2, + "slowlog_new": 2, + "steady": false, + "to_ns": 1791690743584489495, + "top_commands_usec": { + "eval": 10145176, + "evalsha": 24499749, + "publish": 701412, + "set": 979302, + "setex": 2558834 + }, + "valid": true + }, + { + "core_fraction": 0.0868, + "from_ns": 1791690753588790195, + "judged": true, + "ping_ms": { + "max": 5.869, + "min": 0.1, + "n": 1200, + "p50": 0.158, + "p95": 0.715, + "p99": 1.675 + }, + "seconds": 590.2, + "slowlog_new": 3, + "steady": true, + "to_ns": 1791691343765594831, + "top_commands_usec": { + "eval": 11868395, + "evalsha": 32166251, + "publish": 896227, + "set": 1288739, + "setex": 3237803 + }, + "valid": true + }, + { + "core_fraction": 0.0741, + "from_ns": 1791691353768453016, + "judged": true, + "ping_ms": { + "max": 3.248, + "min": 0.1, + "n": 1199, + "p50": 0.152, + "p95": 0.467, + "p99": 1.039 + }, + "seconds": 589.7, + "slowlog_new": 1, + "steady": true, + "to_ns": 1791691943439104168, + "top_commands_usec": { + "eval": 10480850, + "evalsha": 26212845, + "publish": 839100, + "set": 1309821, + "setex": 3165549 + }, + "valid": true + }, + { + "core_fraction": 0.0627, + "from_ns": 1791691953441892544, + "judged": false, + "ping_ms": { + "max": 2.198, + "min": 0.1, + "n": 251, + "p50": 0.146, + "p95": 0.352, + "p99": 0.815 + }, + "seconds": 120.0, + "slowlog_new": 0, + "steady": false, + "to_ns": 1791692073476356936, + "top_commands_usec": { + "eval": 1933282, + "evalsha": 4387436, + "publish": 141285, + "set": 214685, + "setex": 530274 + }, + "valid": true + } + ] + }, + "quantity": "TS execution Redis (redis_marketdata) command CPU, SLOWLOG, RTT", + "reason": "SLOWLOG 6 new entries in steady state", + "rule": "commandstats <= 60 % of one core per 10-min window; SLOWLOG (10 ms) 0 in steady state; PING p99 <= 5 ms", + "verdict": "FAIL", + "sources": { + "g8": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g8/g8-summary.json", + "sha256": "2b3375cffc57c87e9e85ac21fe6b70f3897c2ebf2b80eea3f01b1180eb2d5671" + } + }, + "attribution": { + "basis": "the 6 SLOWLOG start seconds compared with applied TRADE records per writer-start second in the same trace", + "per_second_applied_TRADE": { + "median": 46, + "p95": 428, + "seconds": 1801 + }, + "slowlog_seconds": [ + { + "start_s": 1791690302, + "EVALSHA_us": 10016, + "TRADE_s-1..s+1": [ + 138, + 320, + 379 + ], + "percentile_of_peak": 93.2 + }, + { + "start_s": 1791690605, + "EVALSHA_us": 16120, + "TRADE_s-1..s+1": [ + 42, + 709, + 320 + ], + "percentile_of_peak": 98.1 + }, + { + "start_s": 1791690763, + "EVALSHA_us": 13258, + "TRADE_s-1..s+1": [ + 778, + 1114, + 1062 + ], + "percentile_of_peak": 99.4 + }, + { + "start_s": 1791690842, + "EVALSHA_us": 12964, + "TRADE_s-1..s+1": [ + 677, + 1068, + 625 + ], + "percentile_of_peak": 99.2 + }, + { + "start_s": 1791691251, + "EVALSHA_us": 18431, + "TRADE_s-1..s+1": [ + 296, + 691, + 284 + ], + "percentile_of_peak": 98.0 + }, + { + "start_s": 1791691386, + "EVALSHA_us": 19732, + "TRADE_s-1..s+1": [ + 174, + 372, + 974 + ], + "percentile_of_peak": 99.1 + } + ], + "reading": "every SLOWLOG entry is a writer APPLY EVALSHA in a TRADE burst second (peak within +-1 s at or above the 93rd percentile of per-second TRADE volume); commandstats <= 0.0868 core and PING p99 <= 1.675 ms stay inside. Owner decision as G2: EVAL chunking moves to TS phase E" + } + }, + { + "id": "G9", + "measured": { + "demanded_ne_session_ready_samples": 0, + "demanded_values": [ + 64 + ], + "eligibility_loss_causes": {}, + "eligibility_loss_rows": [], + "eligibility_losses_data_path": 0, + "execution_ready_max": 53, + "execution_ready_min": 50, + "legit_quiet_refusals": { + "TRADE:TRADE_EVENT_STALE": 24 + }, + "purpose_errors": 0, + "purpose_fields_present": true, + "readable_samples": 180, + "samples": 180, + "session_loss_causes": {}, + "unknown_samples": 0, + "v1_fallback_delta": 0, + "v1_fallback_max": 0 + }, + "quantity": "readiness: demanded, session-ready, intent-eligible kept separate", + "reason": "inside", + "rule": "demanded == session-ready; 0 data-path eligibility losses (legitimate quiet TRADE refusals counted, excluded); v1 fallback 0", + "verdict": "PASS", + "sources": { + "g9g10": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g9g10/g9-g10-summary.json", + "sha256": "3015c2db3820c172a984af38043443622e97065f59fbdd8054dbfff13abf117d" + }, + "gates": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/stage100/gates.json", + "sha256": "63f12e8c6df961b0505fd75ab8df9fe2a34e4fa7b2d985cb025e17a6785bcc43" + } + } }, - "rust_core_3": { - "container": "qdl_v2_stable_candidate-rust_core_3-1", - "cpu_max_pct": 14.14, - "cpu_mean_pct": 8.5, - "cpu_p95_pct": 11.13, - "mem_headroom_pct": 83.9, - "mem_limit_mib": 256.0, - "mem_max_mib": 41.3, - "samples": 151, - "throttled_pct_of_wall": 0.0 + { + "basis": "writer log typed signals (DL-signalled gaps, CURSOR_EXPIRED, SlowConsumerError) + trace duplicate event_id; a silent offset gap the DL does not signal is not observable (see proposed counter)", + "id": "G10", + "measured": { + "codes": { + "resumed|RECONNECTED": 2 + }, + "duplicates_applied": 0, + "gaps": 0, + "log_polls": 129, + "log_unreadable": 0, + "overflow": 0, + "replay_expired": 0, + "resumed_reconnects": 2 + }, + "quantity": "TRADE integrity", + "reason": "all zero", + "rule": "0 gaps, 0 duplicates applied, 0 replay-expired, 0 overflow", + "verdict": "PASS", + "sources": { + "g9g10": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g9g10/g9-g10-summary.json", + "sha256": "3015c2db3820c172a984af38043443622e97065f59fbdd8054dbfff13abf117d" + }, + "transit": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/trace/transit-load.json", + "sha256": "d1376db1a50db991df5e9e9e80212463ad095640ee00a67ab673e473145d73f6" + } + } }, - "stable_redis": { - "container": "qdl_v2_stable_candidate-stable_redis-1", - "cpu_max_pct": 9.38, - "cpu_mean_pct": 3.3, - "cpu_p95_pct": 5.44, - "mem_headroom_pct": 93.5, - "mem_limit_mib": 160.0, - "mem_max_mib": 10.4, - "samples": 151, - "throttled_pct_of_wall": 0.0 + { + "id": "G11", + "measured": { + "cursor_samples": [ + { + "ahead": [], + "at_ms": 1791692127022, + "compared": 21, + "unmatched": 0 + }, + { + "ahead": [], + "at_ms": 1791692147026, + "compared": 21, + "unmatched": 0 + }, + { + "ahead": [], + "at_ms": 1791692167033, + "compared": 21, + "unmatched": 0 + } + ], + "outage_seconds": 0.121, + "recovery": { + "caught_up_s": { + "max": 18.923, + "min": 3.44, + "n": 64, + "p50": 3.919, + "p95": 4.437, + "p99": null + }, + "first_valid_apply_s": { + "max": 18.923, + "min": 3.44, + "n": 64, + "p50": 3.919, + "p95": 4.437, + "p99": null + }, + "per_slice": { + "BINANCE|BNBUSDT|BAR|1m": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BNBUSDT|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BNBUSDT|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BNBUSDT|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BNBUSDT|QUOTE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "BINANCE|BNBUSDT|TRADE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "BINANCE|BTCUSDT|BAR|1m": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "BINANCE|BTCUSDT|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BTCUSDT|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BTCUSDT|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BTCUSDT|QUOTE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|BTCUSDT|TRADE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|DOGEUSDT|BAR|1m": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|DOGEUSDT|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|DOGEUSDT|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|DOGEUSDT|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|DOGEUSDT|QUOTE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|DOGEUSDT|TRADE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|ETHUSDT|BAR|1m": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "BINANCE|ETHUSDT|BOOK_DELTA|None": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "BINANCE|ETHUSDT|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|ETHUSDT|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "BINANCE|ETHUSDT|QUOTE|None": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "BINANCE|ETHUSDT|TRADE|None": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "BINANCE|SOLUSDT|BAR|1m": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|SOLUSDT|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|SOLUSDT|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|SOLUSDT|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "BINANCE|SOLUSDT|QUOTE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "BINANCE|SOLUSDT|TRADE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|BNB-USDT-SWAP|BAR|1m": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|BNB-USDT-SWAP|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BNB-USDT-SWAP|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BNB-USDT-SWAP|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BNB-USDT-SWAP|QUOTE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|BNB-USDT-SWAP|TRADE|None": { + "caught_up_s": 18.923, + "first_valid_apply_s": 18.923, + "rewinds": [] + }, + "OKX|BTC-USD-SWAP|BOOK_DELTA|None": { + "caught_up_s": 3.44, + "first_valid_apply_s": 3.44, + "rewinds": [] + }, + "OKX|BTC-USD-SWAP|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USD-SWAP|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USD-SWAP|QUOTE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USDT-SWAP|BAR|1m": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USDT-SWAP|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USDT-SWAP|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USDT-SWAP|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USDT-SWAP|QUOTE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|BTC-USDT-SWAP|TRADE|None": { + "caught_up_s": 4.92, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|DOGE-USDT-SWAP|BAR|1m": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|DOGE-USDT-SWAP|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|DOGE-USDT-SWAP|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|DOGE-USDT-SWAP|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|DOGE-USDT-SWAP|QUOTE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|DOGE-USDT-SWAP|TRADE|None": { + "caught_up_s": 4.92, + "first_valid_apply_s": 4.92, + "rewinds": [] + }, + "OKX|ETH-USDT-SWAP|BAR|1m": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|ETH-USDT-SWAP|BOOK_DELTA|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|ETH-USDT-SWAP|BOOK_SNAPSHOT|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|ETH-USDT-SWAP|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|ETH-USDT-SWAP|QUOTE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|ETH-USDT-SWAP|TRADE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|SOL-USDT-SWAP|BAR|1m": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|SOL-USDT-SWAP|BOOK_DELTA|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|SOL-USDT-SWAP|BOOK_SNAPSHOT|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|SOL-USDT-SWAP|MARK_INDEX_PRICE|None": { + "caught_up_s": 3.919, + "first_valid_apply_s": 3.919, + "rewinds": [] + }, + "OKX|SOL-USDT-SWAP|QUOTE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + }, + "OKX|SOL-USDT-SWAP|TRADE|None": { + "caught_up_s": 4.437, + "first_valid_apply_s": 4.437, + "rewinds": [] + } + }, + "rewinds_total": 0, + "slices_not_caught_up": [], + "slices_without_first_apply": [] + }, + "restart": { + "detected_ms": 1791692107015, + "fault_cleared_ms": 1791692106579, + "last_old_seen_ms": 1791692106016, + "new": { + "finished": "2026-10-11T04:15:06.458006215Z", + "id": "c9ded16d4ff019b16d3dbc577edfceaaea6fe2282baf0a65a1708b2c4c0b8cbd", + "restarts": 0, + "running": true, + "started": "2026-10-11T04:15:06.579836652Z" + }, + "outage_start_ms": 1791692106458 + } + }, + "note": "a fault window after the load (a restart of a monitored container aborts the driver); its verdict does not depend on the steady-state protocol", + "quantity": "writer restart recovery", + "reason": "restart probe", + "rule": "fault cleared -> first valid apply per slice <= 30 s, -> caught up <= 60 s; no rewind; cursor never ahead", + "verdict": "PASS", + "sources": { + "g11": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/g11/g11-summary.json", + "sha256": "c59cccf3447dbe67b98c4f4842903601352234600de4b6d60b741e94cb98ed8f" + } + } }, - "stream_kn_1": { - "container": "qdl_v2_stable_candidate-stream_kn_1-1", - "cpu_max_pct": 48.75, - "cpu_mean_pct": 20.8, - "cpu_p95_pct": 29.86, - "mem_headroom_pct": 66.5, - "mem_limit_mib": 512.0, - "mem_max_mib": 171.4, - "samples": 151, - "throttled_pct_of_wall": 0.0 + { + "id": "G12", + "quantity": "Kafka broker / projector / Stream restart recovery", + "rule": "fault cleared -> consumers caught up to live <= 600 s, then G1-G3 inside gate; the same four timestamps as G11", + "verdict": "PASS", + "reason": "full isolated run on the release digests (python a0c3649d, rust 96a2c912, writer bae2509f) with harness 2d4b88c (TEST_ONLY BOOK, TRADE, QUOTE, MARK/INDEX and BAR 1m sources): every fault caught up <= 600 s, then G1, G2 (TRADE, QUOTE, MARK) and G3 inside in the 300 s window; monitor 2511 samples, 0 sample errors, no gap", + "level": "ISOLATED_FAULT", + "provenance": "TEST_ONLY", + "measured": [ + { + "fault": "broker_stop_start", + "caught_up_s": 22.334, + "per_venue_caught_up_s": { + "BINANCE": 22.334, + "OKX": 2.177 + }, + "g1_after": { + "BINANCE": { + "n": 297, + "p95_ms": 108.227, + "p99_ms": 160.586, + "max_ms": 1424.895 + }, + "OKX": { + "n": 298, + "p95_ms": 116.204, + "p99_ms": 170.137, + "max_ms": 1970.011 + } + } + }, + { + "fault": "projector_restart", + "caught_up_s": 0.935, + "per_venue_caught_up_s": { + "BINANCE": 0.934, + "OKX": 0.935 + }, + "g1_after": { + "BINANCE": { + "n": 297, + "p95_ms": 100.731, + "p99_ms": 113.233, + "max_ms": 127.018 + }, + "OKX": { + "n": 299, + "p95_ms": 101.251, + "p99_ms": 113.503, + "max_ms": 196.204 + } + } + }, + { + "fault": "stream_restart", + "caught_up_s": 2.294, + "per_venue_caught_up_s": { + "BINANCE": 2.293, + "OKX": 2.294 + }, + "g1_after": { + "BINANCE": { + "n": 298, + "p95_ms": 98.764, + "p99_ms": 103.427, + "max_ms": 111.131 + }, + "OKX": { + "n": 300, + "p95_ms": 98.576, + "p99_ms": 109.594, + "max_ms": 122.954 + } + } + } + ], + "status": "COMPLETED", + "pass": true, + "limits": [ + "isolated namespace, same host, TEST_ONLY BOOK source, single projector/Stream replica", + "G2 and G3 after recovery: NOT_MEASURED (BOOK-only isolated source) / NOT_MEASURED (BOOK-only isolated source)" + ], + "teardown": { + "removed_containers": 12, + "left_by_label": 0, + "left_by_name": 0, + "volumes_before": 217, + "volumes_after": 217 + }, + "attempts": [ + { + "dir": "~/.local/state/qdl-v2/p184-g12-rc10-20261011", + "status": "FAIL", + "error": "RuntimeError: STOP role exited: monitor (in-namespace)", + "detail": "the last monitor record precedes the first fault injection (broker_stop_start kafka-2) by 3.9 s; the STOP fired 27 ms after the injection; no traceback or OOM retained (same symptom as 10-10 attempt a3)", + "verdict_sha256": "728b460d8244a36260a428542e93fb6bfec44cd3cae107e684000a8464e1573e", + "teardown": "10 removed, 0 left, volumes 217 -> 217" + }, + { + "dir": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-r2", + "status": "STOPPED", + "error": "SystemExit: G12 orchestrator received signal 15 (duplicate run, stopped by the coordinator)", + "verdict_sha256": "603fe82c761b2633c326fee6b56ad0578710ac4599d0b27f95a9a059846a1ebf", + "teardown": "5 removed, 0 left by label/name; volumes 217 -> 219 at its teardown while the concurrent a2 run held its own volumes; a2's teardown later read 217" + } + ], + "open_harness_item": "keep the monitor container's exit code and OOM state in the evidence", + "sources": { + "verdict": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/evidence/verdict.json", + "sha256": "05c0652d320929a13073a2ff1eb57a319f5b4a5817f7dd4f917ca8ae914a48b8" + }, + "teardown": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/evidence/teardown.json", + "sha256": "f17de5ca9539848e581972fd276809d8b9e5fd3fb5972f848c795f60bd722bb8" + }, + "image_set": { + "path": "~/.local/state/qdl-v2/p184-g12-rc10-20261011-a2/image-set.json", + "sha256": "7576125751ef20235bad4c139188fe8b31bd10ee3113efea7cd153f11282c87d" + } + }, + "observed": "caught up 483.8 / 405.8 / 275.4 s; all parts PASS", + "full_run": { + "dir": "~/.local/state/qdl-v2/p184-g12-rc10-full3-20261011", + "verdict_sha256": "c7b8ea951fcf2c008e3d97d15584162c064b17ecbbd54b865fbbfb66862cac39", + "teardown_sha256": "fc11efb3ca785a93df184b76caa1c44a0aa2408dd22742440248b87c27173e63", + "harness_commit": "2d4b88c", + "caught_up_s": { + "broker_stop_start": 483.833, + "projector_restart": 405.8, + "stream_restart": 275.425 + }, + "g2_max_ms": "TRADE <= 96.8, QUOTE <= 96.6 (n 1060 / 600 per venue per fault)", + "g3_bar_1m_max_ms": 477.6 + }, + "superseded_bookonly_verdict": { + "verdict": "NOT_MEASURED", + "reason": "isolated run on the release digests (python a0c3649d, rust 96a2c912, writer bae2509f): all three faults caught up <= 600 s and G1 inside in the 300 s window after each; G2/G3 after recovery are NOT_MEASURED (BOOK-only isolated source), and the rule requires G1-G3, so the gate is NOT_MEASURED, not PASS", + "observed": "caught up and G1 INSIDE for all three faults" + } }, - "stream_kn_2": { - "container": "qdl_v2_stable_candidate-stream_kn_2-1", - "cpu_max_pct": 33.95, - "cpu_mean_pct": 14.1, - "cpu_p95_pct": 20.12, - "mem_headroom_pct": 78.8, - "mem_limit_mib": 512.0, - "mem_max_mib": 108.8, - "samples": 151, - "throttled_pct_of_wall": 0.0 + { + "id": "G13", + "quantity": "DR01-DR08 acceptance", + "rule": "every cell closed once on the final DL + writer digests", + "verdict": "DEFERRED", + "reason": "41/41 DL-side cells ok on the release digests; 4 TS-stack steps deferred to the TS release", + "measured": { + "dr05-upstream": "6/6 ok", + "dr08-isolated": "4/4 ok (QUALIFICATION)", + "dr05-transport": "9/9 ok", + "dr04-dr06-views": "22/22 ok (DR05.QUIET_TYPED BINANCE/OKX NOT_OBSERVED)" + }, + "digests": { + "dl_sha": "413c5b914f8c6fd0f67429fdddb39451589e9745", + "python": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "rust": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "writer": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", + "ts_sha": "8b567eb07c9036dbbb42079ed1da90db7e0f5f25" + }, + "failed_attempt": "first dr04-dr06-views in 043821Z: DR04.EXPLICIT_LAST@BINANCE refused EXECUTION_CONTEXT_STALE (live LAST older than the probe limit in a quiet Sunday market), a correct typed refusal; re-run 045533Z 22/22 ok. The age sample behind this reading was not retained in a file", + "deferred_cells": [ + "dr01-deployed", + "dr01-wrong-account", + "dr02-dr03-alpha", + "dr07-reference" + ], + "approval": "owner AskUserQuestion answer 2026-10-11, relayed; p184-g13-inputs/runtime-approval-rc10.md was written by an agent from that relayed approval", + "sources": { + "run_1": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/summary.json", + "sha256": "3a3a34e29715a4b67406b2b5e790bed0c9b8d7f0551498d43d0cd9f9c610a09d" + }, + "run_2_views": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T045533Z/summary.json", + "sha256": "309199f1a24c8625ed98d13d572a79f2cbff754359c8f88ce9a21e9c175c3472" + }, + "dr04_dr06_rows": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T045533Z/dr04-dr06-views/dr04-dr06.jsonl", + "sha256": "2ac4622ce54cff91b4e59ee423067d211460aeb475e8901c6eab155130f9d222" + }, + "dr05_upstream_rows": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/dr05-upstream/dr05-upstream-rows.jsonl", + "sha256": "700721727d38dbd3d3acbe6ffeeead32e925104d35526a04f2b2ebeceba35ed7" + }, + "dr05_transport_rows": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/dr05-transport/dr05-transport.jsonl", + "sha256": "415538c65c0b6a1d0146264176b5e8ab0f8fece7967f2152ff8a5fd119f4b626" + }, + "dr08_evidence": { + "path": "~/.local/state/qdl-v2/p184-g13-20261011T043821Z/dr08-isolated/work/evidence.jsonl", + "sha256": "e5b14dde2964f79e52607049fdbeff37461f6964818a1a92884a78cbce83d1d3" + }, + "approval": { + "path": "~/.local/state/qdl-v2/p184-g13-inputs/runtime-approval-rc10.md", + "sha256": "551890885673bb569119dc65c218deb0b561c59fec8071bea48a7699eb22d2ce" + } + } + }, + { + "id": "G14", + "measured": { + "dl_cpu_mean_sum_vcpu": 5.37, + "mem_at_or_over_limit": [], + "missing_roles": [], + "roles": { + "binance_bar_edge": { + "container": "qdl_v2_stable_candidate-binance_bar_edge-1", + "cpu_max_pct": 76.32, + "cpu_mean_pct": 1.9, + "cpu_p95_pct": 29.25, + "mem_headroom_pct": 71.7, + "mem_limit_mib": 512.0, + "mem_max_mib": 145.0, + "samples": 151, + "throttled_pct_of_wall": 0.3 + }, + "ingestor_binance_usdm": { + "container": "qdl_v2_stable_candidate-ingestor_binance_usdm-1", + "cpu_max_pct": 22.47, + "cpu_mean_pct": 12.4, + "cpu_p95_pct": 17.85, + "mem_headroom_pct": 86.2, + "mem_limit_mib": 256.0, + "mem_max_mib": 35.3, + "samples": 151, + "throttled_pct_of_wall": 0.1 + }, + "ingestor_okx_swap": { + "container": "qdl_v2_stable_candidate-ingestor_okx_swap-1", + "cpu_max_pct": 18.38, + "cpu_mean_pct": 12.1, + "cpu_p95_pct": 15.81, + "mem_headroom_pct": 87.1, + "mem_limit_mib": 256.0, + "mem_max_mib": 32.9, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "kafka1": { + "container": "qdl_v2_stable_candidate-kafka1-1", + "cpu_max_pct": 99.17, + "cpu_mean_pct": 80.8, + "cpu_p95_pct": 96.04, + "mem_headroom_pct": 35.7, + "mem_limit_mib": 1536.0, + "mem_max_mib": 987.9, + "samples": 151, + "throttled_pct_of_wall": 0.4 + }, + "kafka2": { + "container": "qdl_v2_stable_candidate-kafka2-1", + "cpu_max_pct": 122.07, + "cpu_mean_pct": 98.3, + "cpu_p95_pct": 111.79, + "mem_headroom_pct": 54.5, + "mem_limit_mib": 2048.0, + "mem_max_mib": 931.5, + "samples": 151, + "throttled_pct_of_wall": 0.3 + }, + "kafka3": { + "container": "qdl_v2_stable_candidate-kafka3-1", + "cpu_max_pct": 131.35, + "cpu_mean_pct": 102.2, + "cpu_p95_pct": 118.61, + "mem_headroom_pct": 33.5, + "mem_limit_mib": 1536.0, + "mem_max_mib": 1022.0, + "samples": 151, + "throttled_pct_of_wall": 0.5 + }, + "market_cache": { + "container": "qdl_v2_stable_candidate-market_cache-1", + "cpu_max_pct": 12.98, + "cpu_mean_pct": 8.3, + "cpu_p95_pct": 11.87, + "mem_headroom_pct": 87.4, + "mem_limit_mib": 9216.0, + "mem_max_mib": 1163.3, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "market_data_service_p184w": { + "container": "market_data_service_p184w", + "cpu_max_pct": 270.39, + "cpu_mean_pct": 150.7, + "cpu_p95_pct": 212.82, + "mem_headroom_pct": 97.9, + "mem_limit_mib": 31375.4, + "mem_max_mib": 657.1, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "market_projector_1": { + "container": "qdl_v2_stable_candidate-market_projector_1-1", + "cpu_max_pct": 28.42, + "cpu_mean_pct": 18.8, + "cpu_p95_pct": 23.68, + "mem_headroom_pct": 88.4, + "mem_limit_mib": 512.0, + "mem_max_mib": 59.6, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "market_projector_2": { + "container": "qdl_v2_stable_candidate-market_projector_2-1", + "cpu_max_pct": 10.47, + "cpu_mean_pct": 8.3, + "cpu_p95_pct": 9.87, + "mem_headroom_pct": 80.7, + "mem_limit_mib": 512.0, + "mem_max_mib": 98.6, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "query_kn_1": { + "container": "qdl_v2_stable_candidate-query_kn_1-1", + "cpu_max_pct": 129.77, + "cpu_mean_pct": 57.1, + "cpu_p95_pct": 92.01, + "mem_headroom_pct": 14.3, + "mem_limit_mib": 1536.0, + "mem_max_mib": 1316.9, + "samples": 151, + "throttled_pct_of_wall": 0.7 + }, + "query_kn_2": { + "container": "qdl_v2_stable_candidate-query_kn_2-1", + "cpu_max_pct": 130.11, + "cpu_mean_pct": 66.1, + "cpu_p95_pct": 105.09, + "mem_headroom_pct": 22.0, + "mem_limit_mib": 1536.0, + "mem_max_mib": 1198.1, + "samples": 151, + "throttled_pct_of_wall": 1.2 + }, + "rust_core": { + "container": "qdl_v2_stable_candidate-rust_core-1", + "cpu_max_pct": 25.96, + "cpu_mean_pct": 10.9, + "cpu_p95_pct": 15.09, + "mem_headroom_pct": 79.1, + "mem_limit_mib": 256.0, + "mem_max_mib": 53.5, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "rust_core_2": { + "container": "qdl_v2_stable_candidate-rust_core_2-1", + "cpu_max_pct": 23.11, + "cpu_mean_pct": 13.0, + "cpu_p95_pct": 17.57, + "mem_headroom_pct": 76.4, + "mem_limit_mib": 256.0, + "mem_max_mib": 60.4, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "rust_core_3": { + "container": "qdl_v2_stable_candidate-rust_core_3-1", + "cpu_max_pct": 14.14, + "cpu_mean_pct": 8.5, + "cpu_p95_pct": 11.13, + "mem_headroom_pct": 83.9, + "mem_limit_mib": 256.0, + "mem_max_mib": 41.3, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "stable_redis": { + "container": "qdl_v2_stable_candidate-stable_redis-1", + "cpu_max_pct": 9.38, + "cpu_mean_pct": 3.3, + "cpu_p95_pct": 5.44, + "mem_headroom_pct": 93.5, + "mem_limit_mib": 160.0, + "mem_max_mib": 10.4, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "stream_kn_1": { + "container": "qdl_v2_stable_candidate-stream_kn_1-1", + "cpu_max_pct": 48.75, + "cpu_mean_pct": 20.8, + "cpu_p95_pct": 29.86, + "mem_headroom_pct": 66.5, + "mem_limit_mib": 512.0, + "mem_max_mib": 171.4, + "samples": 151, + "throttled_pct_of_wall": 0.0 + }, + "stream_kn_2": { + "container": "qdl_v2_stable_candidate-stream_kn_2-1", + "cpu_max_pct": 33.95, + "cpu_mean_pct": 14.1, + "cpu_p95_pct": 20.12, + "mem_headroom_pct": 78.8, + "mem_limit_mib": 512.0, + "mem_max_mib": 108.8, + "samples": 151, + "throttled_pct_of_wall": 0.0 + } + }, + "samples": 151 + }, + "quantity": "resources per role at stage 100", + "reason": "18 roles recorded", + "rule": "CPU/RAM per role recorded with headroom; budget set by the owner", + "verdict": "PASS", + "sources": { + "host": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/stage100/host.json", + "sha256": "06eab7bae9b385f8742630ea26cd91571e7e03f9910f7798f97f909b50f2ea1d" + } + }, + "note": "DL mean CPU sum 5.37 vCPU (17 DL roles, writer excluded); stage 100m on 2026-10-10 was about 5.0" } - }, - "samples": 151 - }, - "quantity": "resources per role at stage 100", - "reason": "18 roles recorded", - "rule": "CPU/RAM per role recorded with headroom; budget set by the owner", - "verdict": "PASS", - "sources": { - "host": { - "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z/stage100/host.json", - "sha256": "06eab7bae9b385f8742630ea26cd91571e7e03f9910f7798f97f909b50f2ea1d" - } + ] }, - "note": "DL mean CPU sum 5.37 vCPU (17 DL roles, writer excluded); stage 100m on 2026-10-10 was about 5.0" - } - ], - "history": { + "caveat": "the rc10 rollback compose files it cites (rollout-rc6/-rollback.compose.json) were overwritten by the rc11/rc12/rc13 rolls; their recorded sha256 no longer match the files on disk" + }, "2026-10-10_bbfb3be_scorecard": { "superseded_by": "campaign 20261011T034233Z + isolated G12/G13 on rc10", "sha256_of_committed_file": "d4035ccd460838cfcb130800660014f8dcfd22079f034390d44fa3fe9971c714", @@ -4332,6 +7071,131 @@ }, "writer_image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", "note": "aborted before the driver ran to apply the Kafka log-cleaner throttle (03:40:56Z); no gate measured except the G11 probe" + }, + { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T034233Z", + "scorecard_sha256": "0133670885374c1a280a8fcd4c34f6dd704e29904d5f492bbb08853398eb12e4", + "protocol_met": true, + "protocol_problems": [], + "summary": { + "FAIL": [ + "G2", + "G8" + ], + "NOT_MEASURED": [ + "G4", + "G12", + "G13" + ], + "PASS": [ + "G1", + "G3", + "G5", + "G6", + "G7", + "G9", + "G10", + "G11", + "G14" + ] + }, + "writer_image": "sha256:bae2509fac12c2ba00fc5815a9d89924d8adfe3ad482bbccc7d55ea6b1f6510c", + "note": "campaign 4 (the rc10 record's gate source): writer bae2509f 8b567eb depth 1, Query rc8, Stream rc9, bar edge rc10, rest rc7d; G2 TRADE B/O p95 1562.1/2084.2, max 5338.8/4301.8 (n 160328/39292); G8 6 SLOWLOG EVALSHA 10.0-19.7 ms" + }, + { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T103749Z", + "scorecard_sha256": "c55e2e39828a84bd7dd081bb8ec6bf47521eeb8d0e1bda39df2ef6825429da61", + "protocol_met": false, + "protocol_problems": [ + "driver acceptance not PASS: streams:no_errors,streams:every_live_stream_delivered,streams:final_bar_every_bar_stream,final:fault_windows_ran" + ], + "summary": { + "FAIL": [ + "G2" + ], + "NOT_MEASURED": [ + "G1", + "G3", + "G4", + "G5", + "G6", + "G7", + "G8", + "G9", + "G10", + "G12", + "G13", + "G14" + ], + "PASS": [ + "G11" + ] + }, + "writer_image": "sha256:64347a94a3ed682d155690f3afaed5aaeba6d8feb8e8febcaeb7dc33dea488d9", + "note": "Query rc11 b128de1e, Stream rc11 ddb2ad19, rest rc10, writer 64347a94 (a3035c8, SDK 2.3.0); protocol not met: the load driver mounted SDK 2.3.0 over a 2.2.2 image's generated protobuf, so alpha streams failed (AttributeError on SubscribeResponse.records; driver error_count 180, all 40 listed errors STREAM AttributeError) -> f1849e0" + }, + { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T111700Z", + "scorecard_sha256": "2ec43e8777627b8a274ce37eba222ed60d9b8b817d0a17c7409e8498d9895011", + "protocol_met": false, + "protocol_problems": [ + "no driver receipt", + "driver acceptance not PASS: no gates" + ], + "summary": { + "FAIL": [ + "G9" + ], + "NOT_MEASURED": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "G8", + "G10", + "G12", + "G13", + "G14" + ], + "PASS": [ + "G11" + ] + }, + "writer_image": "sha256:64347a94a3ed682d155690f3afaed5aaeba6d8feb8e8febcaeb7dc33dea488d9", + "note": "same images; stopped by its guard at 11:38Z after the writer went DEGRADED 63/64 at 11:37:40Z (OKX BNB-USDT-SWAP TRADE quiet > 120 s refused PRIMARY_BEHIND after ring eviction) = IDR-088 -> 5ee9463; G9 FAIL demanded != session-ready in 6 samples" + }, + { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T115743Z", + "scorecard_sha256": "6ae2945de18852fc2fd0b514d5679f0c269ce444719285f4a5800adba71990d9", + "protocol_met": true, + "protocol_problems": [], + "summary": { + "FAIL": [ + "G2" + ], + "NOT_MEASURED": [ + "G4", + "G12", + "G13" + ], + "PASS": [ + "G1", + "G3", + "G5", + "G6", + "G7", + "G8", + "G9", + "G10", + "G11", + "G14" + ] + }, + "writer_image": "sha256:64347a94a3ed682d155690f3afaed5aaeba6d8feb8e8febcaeb7dc33dea488d9", + "note": "phase E step 1 (knob 1): Query rc12 3b585597, Stream rc11, rest rc10, writer 64347a94; protocol met; G2 FAIL (TRADE B/O p95 801.2/817.2, max 3284.0/3208.8, n 92415/26567; QUOTE max 1067.2/1283.3), G8 PASS; the one-variable comparison for step 2" } ] } From c9f53d1c38d1a5458e25415efdad6f917e9bd53b Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 11 Oct 2026 15:41:20 +0000 Subject: [PATCH 147/147] docs(release): v2.2.3 certificate and endpoint report (scoped PASS on 481fd67) certificate.json (qdl.scoped-release-certificate.v2, status PASS under the stated limits) and endpoint-report.json from the stage-100 step-2 campaign; the release workflow's certificate and SDK checks pass locally (wheel b9c74157 reproducible across five Python environments). --- .../evidence/releases/v2.2.3/RELEASE_NOTES.md | 11 +- .../evidence/releases/v2.2.3/certificate.json | 503 +++++++ .../releases/v2.2.3/endpoint-report.json | 1165 +++++++++++++++++ 3 files changed, 1675 insertions(+), 4 deletions(-) create mode 100644 upgrade/evidence/releases/v2.2.3/certificate.json create mode 100644 upgrade/evidence/releases/v2.2.3/endpoint-report.json diff --git a/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md b/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md index cbec9579..5034ffef 100644 --- a/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md +++ b/upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md @@ -1,8 +1,11 @@ # Quant Data Layer v2.2.3 -Status: `PREPARED_NOT_PUBLISHED`. This is not a published release, a Git tag or an owner publication decision. -Publication needs the owner's decision and a tag. Remote CI must also run on the final head. The published stable -release is still v2.2.2. +Status: certified (scoped PASS, [certificate.json](certificate.json)). It is published by the tag workflow after the +owner's decision (2026-10-11). +- Publication path: required remote CI at the final head, feature -> dev (PR #31), release PR dev -> main, tag + `v2.2.3` on main. +- Scope: the Data Layer release under the limits below. Not a Trading System release; not HA (single host). +- Previous stable release: v2.2.2. Scorecard: [scorecard.json](scorecard.json). Inventory, hashes, the SDK build and CI: [release-preparation.json](release-preparation.json). @@ -348,7 +351,7 @@ their recorded sha256 no longer match the files on disk. and main. 3. An annotated tag `v2.2.3` that links the final CI runs and the hashes of scorecard.json and this file. -No merge or tag was performed. The branch was pushed before this record. This record is not committed. +This record was committed and pushed on fix/p184-dl-closure; the merge and tag follow the publication path above. ## Appendix: Preparation History diff --git a/upgrade/evidence/releases/v2.2.3/certificate.json b/upgrade/evidence/releases/v2.2.3/certificate.json new file mode 100644 index 00000000..69eb81b5 --- /dev/null +++ b/upgrade/evidence/releases/v2.2.3/certificate.json @@ -0,0 +1,503 @@ +{ + "ci": { + "head_sha": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "later_run_at_assembly": { + "head_sha": "63640c2fae12d34fdbd5675cfe7da4905944285d", + "id": 38151850803, + "passed": [ + "consolidated-source-preflight", + "sdk-python310" + ], + "pending": [ + "contract-tests", + "unit-tests", + "kn-native-integration" + ] + }, + "local_exact_source": { + "jobs": { + "consolidated-source-preflight": "PASS", + "contract-tests": "PASS", + "kn-native-integration": "PASS", + "sdk-python310": "PASS", + "unit-tests": "INCOMPLETE" + }, + "path": "~/.local/state/qdl-v2/p184-local-ci-481fd67/summary.json", + "sha": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "sha256": "83e7dda9f3605b52b4dc94d63f51fbc5c7d1d7f06a119e920c20b4a64a47ca0a", + "status": "INCOMPLETE", + "steps": "41 PASS, 3 SKIPPED (contract-tests #2/#5 pull-request-only; unit-tests #4 needs sudo)" + }, + "note": "Metadata is frozen before final-head CI; job conclusions below are a snapshot at assembly (2026-10-11T15:37Z, GitHub API), not a claim that pending checks passed.", + "pull_request": "https://github.com/BobbyAxerol/quant-data-layer/pull/31", + "release_metadata_head_requires_checks": true, + "required_jobs": [ + "consolidated-source-preflight", + "contract-tests", + "sdk-python310", + "unit-tests", + "kn-native-integration" + ], + "reviewed_source_run": { + "event": "pull_request", + "head_sha": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "id": 38151336521, + "passed": [ + "consolidated-source-preflight", + "contract-tests", + "sdk-python310", + "kn-native-integration" + ], + "pending": [ + "unit-tests" + ] + }, + "status": "REQUIRED_EXTERNAL_GATE" + }, + "classification": "SCOPED_DATA_LAYER_STAGE100_PASS_G4_NOT_MEASURED_G13_TS_DEFERRED_SINGLE_HOST_NOT_HA", + "classification_note": "PASS certifies only the Data Layer release under the limits below: 12 of 14 gates PASS in a protocol-met stage-100 campaign plus isolated G12, G4 NOT_MEASURED (rows without a frozen baseline; every baselined row PASS), G13 DEFERRED (41/41 DL-side cells ok; 4 TS-stack cells belong to the TS release). It is not a Trading System release, not HA, not uninterrupted availability and not a capacity claim beyond the measured volume.", + "endpoint_report_sha256": "8e09cda5838f1fb398e90358e2f3627dfadc56fe09cb9223de5e79ecd30378a6", + "evidence_sha256": { + "upgrade/evidence/releases/v2.2.3/RELEASE_NOTES.md": "84bb4c2387eff4e8f2fb487aa8f4b451b316e7c4e0469362154993cd2153dafb", + "upgrade/evidence/releases/v2.2.3/release-preparation.json": "a17f15e5ab48d09bdbd4ddf55ee760b5412825eaceb57a402cb20f9fca9fb1ee", + "upgrade/evidence/releases/v2.2.3/scorecard.json": "23b79a01d81d378074043050810e5cb34f01d01e919561366e63d0ccd1137a81", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/campaign.json": "b926a20e3acbe5dcdde965ec7147fe3ee46b1d5a6fe0ca95795d36614aa71c2b", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g11/g11-summary.json": "c85e4566e327328b0bd4c5f5cc75a4825f1edcff1586a4bad3226e3233c5b764", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g3/g3-summary.json": "fafca1be177364c2032dd18afd5f7e2c8d67468403b2de3fde2658ee69274e07", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g4-1-summary.json": "717bf3b8be882a4801468b61fdb4a094861265b135d53bade2c04661c12e3c3c", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g4-2-summary.json": "3f080361fb9b7d9d9fb3e4af204fd8e7c67a7e199f70aee9d97a849fc936e52a", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g4-3-summary.json": "21b72b595926f62e15d712e58a0f7bdacfed3a53bacb067ab66ce72f4a494f6b", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g4-4-summary.json": "0b68e97fb3744c57dba8b337f230c2e0d33fae5c9442a15d6b0809d6a6d6f322", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g4-5-summary.json": "eeaa9a683f48c73c1bb891bbc2abb188168d871865bb79e2a4517ccd6a9a619f", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g6-ts/summary.json": "5a9fc905a71d86e1d013bfeed463bee3bfb366545fd742189b6ddedc3ee1bede", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g8/g8-summary.json": "0314b552d3f09b7d2143304ee77c9cd0c27a959fb10b764e2996865a2ffae7e5", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g9g10/g9-g10-summary.json": "247c62e6702c264efb4b7b8e1a3089b369a378a8baf21e81d525945b3d8bf5d0", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/scorecard.json": "747e476489a3ef9807e7ade9bd347b00a8e56226a5bc85a99b8fa03b5e7fa58e", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/stage100/gates.json": "86b74f26c3f7dc68228d260becffa77247d080bfc5d6ce9736ebba495df81544", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/stage100/host.json": "a5ba4fcf963ff2c003f4f6f211fa6a8210f453d56b6a37fd3e8fb19276c894e1", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/stage100/receipt.json": "fdf26baa323115015501aa56b6450e51deaa83e9bf284d6ef8ec1574c20825eb", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/trace/mark-load.json": "b6a5198116d7916e32b8b05f48e2bf45798b7863aeb64e32247b874c442f8c9f", + "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/trace/transit-load.json": "bd1ccac05fad1728d73bbab833a629d43266756c3575e09082a498e1ee7c78c7", + "~/.local/state/qdl-v2/p184-g12-rc12-20261011/evidence/exit-states.json": "52d7ce96025dc75a48fad58e64e724bac61acb4c30bd830dbc00767a66206a6d", + "~/.local/state/qdl-v2/p184-g12-rc12-20261011/evidence/monitor.jsonl": "8aa135ab0bdaefa6df5cd6aa1f260164dd87f58ebe407341d884f324ba9ad063", + "~/.local/state/qdl-v2/p184-g12-rc12-20261011/evidence/teardown.json": "a5fceec5d8f40626ce01dfc83d93514dbd378824957966f944c6920154a3e71b", + "~/.local/state/qdl-v2/p184-g12-rc12-20261011/evidence/verdict.json": "20f296719c740dd53a2f32f20b0f18b673b7fa082532c54ee478d7166fbc5e3e", + "~/.local/state/qdl-v2/p184-g12-rc12-20261011/image-set.json": "838c0503cc99229ac5b1b02393ec8df0fece3abfe07af1ef0fe91d69dc69b85a", + "~/.local/state/qdl-v2/p184-g13-20261011T134532Z/dr05-transport/dr05-transport.jsonl": "95a6540104b1c566773d8b52281304c15ff671b3e9795ba022a51c77d159ff93", + "~/.local/state/qdl-v2/p184-g13-20261011T134532Z/dr05-upstream/dr05-upstream-rows.jsonl": "b01f3f4ad86bef3b4c982aa86a3cff2ae3f981e5ede5afc04cf5c8932de3825b", + "~/.local/state/qdl-v2/p184-g13-20261011T134532Z/dr08-isolated/work/evidence.jsonl": "b996d55d41354975764fa9dcdd6ac5ae944e73f2497b0b9648dfa42d38f64489", + "~/.local/state/qdl-v2/p184-g13-20261011T134532Z/summary.json": "445e05c9de1a0074fb0f94acc950e25536e9a74342edea44e4142a8d6765e55c", + "~/.local/state/qdl-v2/p184-g13-20261011T140432Z/dr04-dr06-views/views.rows": "0fb613f61b14e0553103e64a47d040e1e24ab2fbd7f1ec766180170aa7a2ea10", + "~/.local/state/qdl-v2/p184-g13-20261011T140432Z/summary.json": "f0b7b523fa6af40503f009f97d4ebde9119dda228b7902644f93e0629de40a6c", + "~/.local/state/qdl-v2/p184-g13-inputs/runtime-approval-rc12.md": "9e786bfd2205b5d69977f5fc547a2880d79450bb9856d2a70d1833bbcb638aa8", + "~/.local/state/qdl-v2/p184-local-ci-481fd67/summary.json": "83e7dda9f3605b52b4dc94d63f51fbc5c7d1d7f06a119e920c20b4a64a47ca0a" + }, + "gates": { + "G10_trade_integrity": "PASS 0 gaps, 0 duplicates applied, 0 replay-expired, 0 overflow", + "G11_writer_restart": "PASS outage 0.132 s, first valid apply/caught up max 7.511 s over 64 slices, 0 rewinds", + "G12_restart_recovery": "PASS isolated (TEST_ONLY, rc12 digests; rc13 differs by IDR-087 only): broker 224.704 s, projector 19.4 s, stream 138.381 s <= 600 s, then G1-G3 inside", + "G13_dr_acceptance": "DEFERRED 41/41 DL cells ok on rc12 + final writer 44baacc0; dr01-deployed, dr01-wrong-account, dr02-dr03-alpha, dr07-reference deferred to the TS release", + "G14_resources": "PASS 18 roles recorded with headroom; DL mean CPU sum 5.08 vCPU is above the owner's 5 vCPU budget (limit)", + "G1_book_transit": "PASS every BOOK row inside (p95 138.8-177.6 ms, max <= 585.9 ms; rule p95<=500, p99<=1000, max<2000)", + "G2_trade_quote_mark_transit": "PASS every row inside (TRADE p95 243.4/210.3, max 583.1/495.5; QUOTE max 531.5/532.0; MARK max 803.1/712.7 ms BINANCE/OKX) with writer knob 64; step 1 at knob 1 FAIL kept", + "G3_bar_from_close": "PASS BINANCE p95 2680 ms max 3330; OKX p95 1493 max 1546; 0 unseen", + "G4_delivery_lag_vs_baseline": "NOT_MEASURED 17/17 baselined rows PASS; 8 observed rows have no frozen baseline; none added after measuring", + "G5_request_latency": "PASS 10 latency class/venue gates + cold overlap 4 runs 0 bad; 0 failed, 0 missed of 182709 offered", + "G6_execution_quality": "PASS 6 D3 rows, 0 unsafe, 19213 probe reads, 0 errors", + "G7_ts_consumer_path": "PASS SDK->writer p95 <= 45.3 ms, writer->Redis p95 <= 33.3 ms", + "G8_ts_redis": "PASS ping p99 0.779 ms, 0 slowlog, core fraction <= 0.0612", + "G9_readiness": "PASS demanded 64 == session-ready in 180/180 scored samples, 0 data-path eligibility losses, v1 fallback 0", + "remote_ci": "PENDING_REQUIRED_CHECKS", + "summary": { + "DEFERRED": [ + "G13" + ], + "FAIL": [], + "NOT_MEASURED": [ + "G4" + ], + "PASS": [ + "G1", + "G2", + "G3", + "G5", + "G6", + "G7", + "G8", + "G9", + "G10", + "G11", + "G12", + "G14" + ] + } + }, + "limits": [ + "Scoped Data Layer certificate: not a Trading System order, accounting, alpha or DR01/DR02/DR03/DR07 certification.", + "Single host for every run: no HA, no independent failure domains, no zero-downtime, no mainnet or funded sizing claim.", + "G4 NOT_MEASURED: rows without a frozen baseline (bar BINANCE/OKX 1h and 5m, OKX_INVERSE book_delta/book_snapshot/mark/quote).", + "G13 DEFERRED: four TS-stack cells open; DR05.QUIET_TYPED BINANCE/OKX are NOT_OBSERVED, not a positive quiet-state proof.", + "G12 is ISOLATED_FAULT with TEST_ONLY sources on the rc12 digests, single projector/Stream replica, same host.", + "Latency gates were measured on the pre-rc13 mix (source-identical Query/Stream, source-equivalent Rust roles, bar edge differs by IDR-087); rc13 itself has readiness receipts only.", + "IDR-087 (bar edge backlog probe in a supervised child) is tested locally only; the bar edge has not been through a live broker stop.", + "G2/G8 PASS requires the TS writer batched Subscribe knob 64; TRADE volume in the PASS run was lower than earlier runs, so no capacity claim at higher TRADE volume.", + "DL mean CPU 5.08 vCPU at stage 100 exceeds the owner's 5 vCPU budget; Kafka is the largest share; tuning is open.", + "IDR-040 fix 2 is OFF (book_liveness_proof_ms unset): a quiet Binance book stays stale, never falsely fresh.", + "Catalog revision split (C.19) unchanged: image catalog 11, runtime-mounted 12.", + "Remote CI at the final head is not yet green at assembly time (see ci); historical failures, refusals and FAIL campaigns stay recorded and are not erased by this PASS." + ], + "previous_published_release": "v2.2.2", + "publication_requires": "Owner publication decision; all required remote CI jobs (consolidated-source-preflight, contract-tests, sdk-python310, unit-tests, kn-native-integration) green at the final head that contains this certificate (required at final head; run 38151336521 on 481fd67 and the run on 63640c2 do not cover the certificate commit); PR fix/p184-dl-closure -> dev (#31), then release PR dev -> main; annotated tag v2.2.3 on a main commit (the workflow checks main ancestry) that links the CI runs and the scorecard/notes hashes. statusPASS here certifies only the scoped DL release above.", + "publication_status": "REQUIRES_OWNER_DECISION_FINAL_HEAD_CI_DEV_MAIN_MERGE_AND_TAG", + "release": "v2.2.3", + "release_images": { + "python": { + "digest": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "tag": "qdl-v2-python:2.2.3-rc13-481fd67" + }, + "rust": { + "digest": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "tag": "qdl-v2-rust:2.2.3-rc13-481fd67" + } + }, + "rollback": { + "images": { + "query_kn_1/2, binance_bar_edge": "sha256:3b585597570f3f7065c17b0a7f9228a2dad2f71224905e2f105a1754b9d4a75c (qdl-v2-python:2.2.3-rc12-5ee9463)", + "stream_kn_1/2, projectors, cores, ingestors": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f (qdl-v2-rust:2.2.3-rc12-5ee9463)" + }, + "kafka_config": "--delete-config log.cleaner.io.max.bytes.per.second", + "not_touched": [ + "kafka1-3", + "market_cache", + "stable_redis", + "stable_state_init", + "stable_tls_init", + "Kafka offsets", + "Redis data", + "realm dirs" + ], + "order": "reverse role order, failure only", + "packets": "~/.local/state/qdl-v2/p184-d3-capacity-20261009/rollout-rc6/-rollback.compose.json", + "preserve": "same environment, TLS, mounts, state, groups, offsets, config and consumer identity; no data migration", + "redis_cpu_shares": "docker update --cpu-shares 0 redis_marketdata", + "roles": [ + "ingestor_binance_usdm", + "ingestor_okx_swap", + "rust_core_3", + "rust_core_2", + "rust_core", + "market_projector_2", + "market_projector_1", + "stream_kn_2", + "stream_kn_1", + "binance_bar_edge", + "query_kn_2", + "query_kn_1" + ], + "rollback_compose_sha256": { + "binance_bar_edge": "2a38cba97b6431e4aa0cd0b4f0ef438e0849dd565e202d2f0c129f93b5c7fbf5", + "ingestor_binance_usdm": "fb8655b4e3efb7731bf1c5c747ad1052f23661cf282aa7df9aef97a079d2c30a", + "ingestor_okx_swap": "d101a5198bfcb93562580f90d2a02665811fe0efd4fe8be0e34fd8399e2c9d49", + "market_projector_1": "c957bcd8db42b34333c25a5544c3d72ba1ddbc260d6699dd818f110f4b72a5f3", + "market_projector_2": "0810b910c897c7b066efa21682d2120bc03bbbfc9598e39559fd15bd4ea8218d", + "query_kn_1": "9b0890cc4ede2af26f2f24b3c876035f1406df1b92060874cc624754d516c5d0", + "query_kn_2": "12f9e6888399ca47e34a44bd3e801dfb2d42451a3a16ddd0d922f63e607776b3", + "rust_core": "ccdc9b50198a86f277e1e37796c9110c744d9728881825bc9ef7a7aaa3ac2fef", + "rust_core_2": "b88cab5e8b718193549dab8848d6588bfc39877a39536e639ed6c2c3a4f3cde5", + "rust_core_3": "d7b1ac8c128ea57446141eb49bf685359b223be6e9e2ae3001cc7fb4c6111b5f", + "stream_kn_1": "d3e8343d7ef27cfc7109375f074b6ceb9994c1d3567f6a21e477b73067edabce", + "stream_kn_2": "83b98c51e1ac20580225bd34aa022931ea98bd0739cbb323351a09dda73e2953" + }, + "ts_writer": { + "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/rollback-20261011T134403Z.compose.json", + "sha256": "d673be459f0bf8b6ba5b41cf361237f99116abfc018ccfbdceea3205d7415697" + } + }, + "runtime": [ + { + "health": null, + "image": "sha256:44baacc0a3ff95f71029d230e48e1b7282627f26de9bd99e13d1148862e09068", + "oom": false, + "restart": 0, + "revision_label": "a37ff4dcef418947b89578c1870726cb03db4fd3", + "service": "market_data_service_p184w", + "started": "2026-10-11T13:44:14.494194887Z", + "status": "running" + }, + { + "health": "healthy", + "image": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", + "oom": false, + "restart": 0, + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "service": "qdl_v2_stable_candidate-binance_bar_edge-1", + "started": "2026-10-11T14:26:52.382759721Z", + "status": "running" + }, + { + "health": "healthy", + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "oom": false, + "restart": 0, + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "service": "qdl_v2_stable_candidate-ingestor_binance_usdm-1", + "started": "2026-10-11T14:31:35.59199243Z", + "status": "running" + }, + { + "health": "healthy", + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "oom": false, + "restart": 0, + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "service": "qdl_v2_stable_candidate-ingestor_okx_swap-1", + "started": "2026-10-11T14:31:27.839139393Z", + "status": "running" + }, + { + "health": "healthy", + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "oom": false, + "restart": 0, + "revision_label": null, + "service": "qdl_v2_stable_candidate-kafka1-1", + "started": "2026-10-08T16:23:33.912058555Z", + "status": "running" + }, + { + "health": "healthy", + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "oom": false, + "restart": 0, + "revision_label": null, + "service": "qdl_v2_stable_candidate-kafka2-1", + "started": "2026-10-08T16:23:34.12626004Z", + "status": "running" + }, + { + "health": "healthy", + "image": "sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9", + "oom": false, + "restart": 0, + "revision_label": null, + "service": "qdl_v2_stable_candidate-kafka3-1", + "started": "2026-10-08T16:23:34.326288421Z", + "status": "running" + }, + { + "health": "healthy", + "image": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", + "oom": false, + "restart": 0, + "revision_label": null, + "service": "qdl_v2_stable_candidate-market_cache-1", + "started": "2026-09-27T05:29:58.000340326Z", + "status": "running" + }, + { + "health": null, + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "oom": false, + "restart": 0, + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "service": "qdl_v2_stable_candidate-market_projector_1-1", + "started": "2026-10-11T14:28:29.10000567Z", + "status": "running" + }, + { + "health": null, + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "oom": false, + "restart": 0, + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "service": "qdl_v2_stable_candidate-market_projector_2-1", + "started": "2026-10-11T14:29:00.699668563Z", + "status": "running" + }, + { + "health": "healthy", + "image": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", + "oom": false, + "restart": 0, + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "service": "qdl_v2_stable_candidate-query_kn_1-1", + "started": "2026-10-11T14:26:17.881199252Z", + "status": "running" + }, + { + "health": "healthy", + "image": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", + "oom": false, + "restart": 0, + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "service": "qdl_v2_stable_candidate-query_kn_2-1", + "started": "2026-10-11T14:26:38.472711377Z", + "status": "running" + }, + { + "health": null, + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "oom": false, + "restart": 0, + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "service": "qdl_v2_stable_candidate-rust_core-1", + "started": "2026-10-11T14:29:32.163633283Z", + "status": "running" + }, + { + "health": null, + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "oom": false, + "restart": 0, + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "service": "qdl_v2_stable_candidate-rust_core_2-1", + "started": "2026-10-11T14:30:03.591093881Z", + "status": "running" + }, + { + "health": null, + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "oom": false, + "restart": 0, + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "service": "qdl_v2_stable_candidate-rust_core_3-1", + "started": "2026-10-11T14:30:56.132123836Z", + "status": "running" + }, + { + "health": "healthy", + "image": "sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8", + "oom": false, + "restart": 0, + "revision_label": null, + "service": "qdl_v2_stable_candidate-stable_redis-1", + "started": "2026-09-16T06:38:55.588316335Z", + "status": "running" + }, + { + "health": null, + "image": "sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d", + "oom": false, + "restart": 0, + "revision_label": "8ba4165", + "service": "qdl_v2_stable_candidate-stable_state_init-1", + "started": "2026-09-03T12:36:17.69711623Z", + "status": "exited" + }, + { + "health": null, + "image": "sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d", + "oom": false, + "restart": 0, + "revision_label": "8ba4165", + "service": "qdl_v2_stable_candidate-stable_tls_init-1", + "started": "2026-09-03T12:36:17.691555089Z", + "status": "exited" + }, + { + "health": null, + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "oom": false, + "restart": 0, + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "service": "qdl_v2_stable_candidate-stream_kn_1-1", + "started": "2026-10-11T14:27:02.493271338Z", + "status": "running" + }, + { + "health": null, + "image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "oom": false, + "restart": 0, + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "service": "qdl_v2_stable_candidate-stream_kn_2-1", + "started": "2026-10-11T14:27:36.229558337Z", + "status": "running" + } + ], + "runtime_basis": "docker inspect, read-only, at certificate assembly (2026-10-11 ~15:38Z)", + "runtime_scope_note": "All 12 Data Layer application roles run the rc13 digests with revision label 481fd67 (IDR-039 closed); kafka1-3, market_cache, stable_redis and the two init one-shots are unchanged; market_data_service_p184w is the TS writer (TS a37ff4d, SDK 2.3.0), owned by TS.", + "runtime_scope_totals": { + "application_roles_on_release_digests": 12, + "python_roles": [ + "binance_bar_edge", + "query_kn_1", + "query_kn_2" + ], + "rust_roles": [ + "ingestor_binance_usdm", + "ingestor_okx_swap", + "market_projector_1", + "market_projector_2", + "rust_core", + "rust_core_2", + "rust_core_3", + "stream_kn_1", + "stream_kn_2" + ], + "unchanged_roles": [ + "kafka1", + "kafka2", + "kafka3", + "market_cache", + "stable_redis", + "stable_state_init", + "stable_tls_init" + ] + }, + "schema": "qdl.scoped-release-certificate.v2", + "sdk": { + "builder": "scripts/build_qdl_sdk_release.py from a git archive of 481fd67, two builds, image qdl-v2-python:2.2.3-rc13-481fd67, --network none, non-root, read-only source", + "builds": [ + { + "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk230-build-1", + "release_json_sha256": "218b45b0250c95c1b273b9b809afc234984ab5fffdefcfdf541ebd635111ea15", + "sbom_sha256": "4c69c98d56f29962d9981d1acd517cf44717aa4c224f9df5b2e3e913ceff488b", + "wheel_sha256": "b9c74157db2a2d147564d72b65bbbafc9a7d3d055bccee67645b831470ad38c3" + }, + { + "dir": "~/.local/state/qdl-v2/releases/v2.2.3/sdk230-build-2", + "release_json_sha256": "218b45b0250c95c1b273b9b809afc234984ab5fffdefcfdf541ebd635111ea15", + "sbom_sha256": "4c69c98d56f29962d9981d1acd517cf44717aa4c224f9df5b2e3e913ceff488b", + "wheel_sha256": "b9c74157db2a2d147564d72b65bbbafc9a7d3d055bccee67645b831470ad38c3" + } + ], + "contains_service_internals": false, + "dependencies": [ + "grpcio>=1.70.0,<2.0.0", + "httpx>=0.28.0,<1.0.0", + "protobuf>=6.31.1,<7.0.0", + "pydantic>=2.0.0,<3.0.0", + "PyJWT[crypto]>=2.15.1,<3.0.0" + ], + "generated_contract_digest": "2c59805220f06caf1a19b6df5d4ec7bf44282a780d265e0dd667b9abd4d7c1d1", + "name": "qdl-sdk", + "python_requires": ">=3.10", + "reproducibility_check_2026_10_11": { + "environments_byte_identical": [ + "qdl-v2-python:2.2.3-rc13-481fd67 (CPython 3.12.14, zlib 1.3.1), two builds, --network none, non-root, read-only source", + "python:3.10-slim (CPython 3.10.22, zlib 1.3.1)", + "python:3.12-slim (CPython 3.12.14, zlib 1.3.1)", + "host Ubuntu 24.04.4 /usr/bin/python3 (CPython 3.12.3, zlib 1.3), the same OS image family as GitHub ubuntu-latest" + ], + "residual_risk": "a runner whose python3 links zlib-ng or a future zlib with changed deflate output would produce a different wheel; the workflow then fails closed (hash mismatch), it cannot publish a different artifact", + "source": "git archive HEAD 63640c2 (qdl_sdk, generated/python, contracts, LICENSE and the builder have an empty diff to 481fd67)", + "wheel_sha256": "b9c74157db2a2d147564d72b65bbbafc9a7d3d055bccee67645b831470ad38c3", + "why": "fixed zip timestamps and modes, sorted entries, no environment or clock input; deflate output depends only on the zlib deflate implementation" + }, + "reproducible": true, + "reproducible_timestamp": "2020-01-01T00:00:00Z", + "schema": "qdl.sdk.release.v1", + "sdk_source_digest": "e0e5d044c5edfffb1084bc84b69d4cf1898a4221004ca720c0c9dd02fb0d895b", + "source_unchanged_since": "4c779ac (git diff 4c779ac 481fd67 -- qdl_sdk generated/python contracts scripts/build_qdl_sdk_release.py: empty); the wheel equals the TS-vendored vendor/qdl_sdk-2.3.0-py3-none-any.whl (same sha256)", + "supersedes": "2.2.1 (wheel 27db3971..., never published; 2.2.0 before it)", + "ts_writer_sdk": "2.3.0 (writer images 64347a94 and 44baacc0, qdl_sdk.__version__ read with --network none)", + "version": "2.3.0", + "wheel": "qdl_sdk-2.3.0-py3-none-any.whl", + "wheel_sha256": "b9c74157db2a2d147564d72b65bbbafc9a7d3d055bccee67645b831470ad38c3" + }, + "source_code_sha": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "status": "PASS" +} diff --git a/upgrade/evidence/releases/v2.2.3/endpoint-report.json b/upgrade/evidence/releases/v2.2.3/endpoint-report.json new file mode 100644 index 00000000..045b0fd4 --- /dev/null +++ b/upgrade/evidence/releases/v2.2.3/endpoint-report.json @@ -0,0 +1,1165 @@ +{ + "after_rollout": { + "latency_remeasured_after_rc13": false, + "note": "Each of the 12 application roles was recreated one at a time; the TS writer was READY 64/64 after every step. No latency campaign was run on the rc13 digests; the measured section applies to rc13 only through the source-identity/equivalence statements in before_rollout.", + "release_images": { + "python": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", + "revision_label": "481fd6704b28d8fa63deee39d59eeba23d87f5a8", + "rust": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad" + }, + "roles": [ + { + "health": "healthy", + "new_image": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", + "ok": true, + "old_image": "sha256:3b585597570f3f7065c17b0a7f9228a2dad2f71224905e2f105a1754b9d4a75c", + "receipt_sha256": "2f52c5c0d59bdbe0f58fd29168c15f144c0e7384034c4c870a79310f3c588d7f", + "role": "query_kn_1", + "writer": "READY 64 64", + "writer_ready": true + }, + { + "health": "healthy", + "new_image": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", + "ok": true, + "old_image": "sha256:3b585597570f3f7065c17b0a7f9228a2dad2f71224905e2f105a1754b9d4a75c", + "receipt_sha256": "3e426beecd699ed7c27871f53b0acb727d8e06c24fbeef5b9394b813471f51a8", + "role": "query_kn_2", + "writer": "READY 64 64", + "writer_ready": true + }, + { + "health": "healthy", + "new_image": "sha256:ccbea12e87e42273364b1a78847ea97f5659cc5f43b14c0b1b807f5aa6e00677", + "ok": true, + "old_image": "sha256:3b585597570f3f7065c17b0a7f9228a2dad2f71224905e2f105a1754b9d4a75c", + "receipt_sha256": "b500719e493be645c0ccd055119fef7ee54eb56bf50f2dd2ad91a583fd260176", + "role": "binance_bar_edge", + "writer": "READY 64 64", + "writer_ready": true + }, + { + "health": null, + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "ok": true, + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "receipt_sha256": "2e4cccbb41827a9cfb4302ef1fb1a3e1c11705eafe3de5a852ba5bc742b40a27", + "role": "stream_kn_1", + "writer": "READY 64 64", + "writer_ready": true + }, + { + "health": null, + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "ok": true, + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "receipt_sha256": "23b2b1450fe6ab7e7c08a6441c008eac21ffdb21aec67337a957484486e239cc", + "role": "stream_kn_2", + "writer": "READY 64 64", + "writer_ready": true + }, + { + "health": null, + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "ok": true, + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "receipt_sha256": "bd260d07921c8e1cf26cc961caf40d4fa76fecc932971416240dc6042e0bc4f9", + "role": "market_projector_1", + "writer": "READY 64 64", + "writer_ready": true + }, + { + "health": null, + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "ok": true, + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "receipt_sha256": "c4eca21e6c6dda96690ff3f1acbb0a218eaeb43def65286796a76f20bf2f16f0", + "role": "market_projector_2", + "writer": "READY 64 64", + "writer_ready": true + }, + { + "health": null, + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "ok": true, + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "receipt_sha256": "1c6264a1f62c2d359d59db996e0067b8725c5a8bde705e9a02067abaa28616b1", + "role": "rust_core", + "writer": "READY 64 64", + "writer_ready": true + }, + { + "health": null, + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "ok": true, + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "receipt_sha256": "50c9fabf26712102dc631979519efcaa32768c30e95bb20dd6e22400bb30b617", + "role": "rust_core_2", + "writer": "READY 64 64", + "writer_ready": true + }, + { + "health": null, + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "ok": true, + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "receipt_sha256": "470127e16b7c09676283d30b412b839a38fc1cd56fc4b10eb0058f102210d0a2", + "role": "rust_core_3", + "writer": "READY 64 64", + "writer_ready": true + }, + { + "health": "healthy", + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "ok": true, + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "receipt_sha256": "165650757049e537b5fa57ebd3be74e27a2617a4fde92a831dfcd41500095a8f", + "role": "ingestor_okx_swap", + "writer": "READY 64 64", + "writer_ready": true + }, + { + "health": "healthy", + "new_image": "sha256:6afdc155ba3e4e0e87216c209a23a2938897d235b30bc5d52e054c2e21cb65ad", + "ok": true, + "old_image": "sha256:ebc7b7944e57ab02d3d2bb3eb68b7ef4365c8e3b51e97a2f46ebc1817540dd5f", + "receipt_sha256": "42574d8a10dd2e2fc7d1d0c7013b942e89b03e23f595e52a03154a4aa9ef08e8", + "role": "ingestor_binance_usdm", + "writer": "READY 64 64", + "writer_ready": true + } + ], + "rolled": "2026-10-11T14:26:16Z-14:31:42Z, one role at a time, writer READY 64/64 after every step (rollout-rc6/-receipt-20261011T14*.json)", + "stage": "RC13_ROLL_READINESS_ONLY" + }, + "before_rollout": { + "measurement_identity": { + "binance_bar_edge": { + "diff_to_481fd67": "on its import path (stable_bar_edge.py and its direct qdl imports, checked one level): the IDR-087 files only; the other Python diffs (qdl/query/service.py IDR-086, kn_query_backend.py IDR-088, qdl_sdk 2.3.0, generated query_pb2) are not imported by it", + "equivalence": "NOT source-identical: the measured bar edge ran the in-process Kafka backlog probe that IDR-087 moves into a supervised child; the change acts only when the probe crashes or hangs", + "image": "sha256:a0c3649d9a47cf59300746cb521fd020b190fbeefb3ff0bda6851ea8033119eb", + "source": "413c5b9", + "tag": "qdl-v2-python:2.2.3-rc10-413c5b9" + }, + "note": "the step-2 campaign ran 12:35-13:10Z on the mix below, before the rc12 (13:12-13:18Z) and rc13 (14:26-14:31Z) rolls; diffs are `git diff --stat 481fd67` without tests, scripts, docs and upgrade/", + "query_kn_1/2": { + "diff_to_481fd67": "qdl/runtime/history_backpressure.py +50/-7, qdl/runtime/isolated_probe.py +141 (IDR-087); imported only by qdl/runtime/stable_bar_edge.py", + "equivalence": "source-identical for every Query path", + "image": "sha256:3b585597570f3f7065c17b0a7f9228a2dad2f71224905e2f105a1754b9d4a75c", + "rolled": "11:55:46Z / 11:56:04Z", + "source": "5ee9463", + "tag": "qdl-v2-python:2.2.3-rc12-5ee9463" + }, + "rust_core x3, market_projector_1/2, ingestor_okx_swap, ingestor_binance_usdm": { + "diff_to_481fd67": "rust/qdl-stream-gateway (no other crate depends on it) and generated/rust/qdl/query/v2/qdl.query.v2.rs +16 (two additive prost fields, SubscribeRequest.max_records_per_response and SubscribeResponse.records), included by rust/qdl-contracts, which every crate depends on; no code outside qdl-stream-gateway references SubscribeRequest/SubscribeResponse (git grep)", + "equivalence": "source-equivalent for the core, projector and ingestor binaries (rebuilt, not byte-compared)", + "image": "sha256:96a2c9120ef42892f56278df5d6c6dc5371cc83aa713b7b692902d657f74c3d5", + "source": "413c5b9", + "tag": "qdl-v2-rust:2.2.3-rc10-413c5b9" + }, + "stream_kn_1/2": { + "diff_to_481fd67": "Python only (IDR-087 files, qdl/runtime/kn_query_backend.py IDR-088); rust/, generated/rust, Cargo files, config: empty diff", + "equivalence": "source-identical for the Stream binary (includes batched Subscribe 4c779ac)", + "image": "sha256:ddb2ad19d6106baee8c35f1660f15e4dec8a220a26f91e52651fbd77a54b58fd", + "rolled": "10:34:18Z / 10:35:18Z", + "source": "4704edd", + "tag": "qdl-v2-rust:2.2.3-rc11-4704edd" + } + }, + "note": "The step-2 campaign (12:35-13:10Z) ran before the rc12 (13:12-13:18Z) and rc13 (14:26-14:31Z) rolls, on the image mix below. Query, Stream and the Rust core/projector/ingestor roles are source-identical or source-equivalent to 481fd67; the bar edge is not (IDR-087 only).", + "one_variable_comparison": { + "caveat": "TRADE volume differed between the two runs: TRADE BINANCE/OKX n 66303/12492 in step 2 against 92415/26567 in step 1, and 160328/39292 in campaign 4. Part of the improvement may come from the lower TRADE volume; the step-2 PASS is a measurement at that volume, not a capacity claim at the campaign-4 volume", + "step_1": { + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T115743Z", + "rows": { + "MARK_INDEX_PRICE|BINANCE": { + "max": 801.0, + "n": 8998, + "p95": 359.3 + }, + "MARK_INDEX_PRICE|OKX": { + "max": 820.9, + "n": 21505, + "p95": 338.5 + }, + "QUOTE|BINANCE": { + "max": 1067.2, + "n": 108047, + "p95": 166.5 + }, + "QUOTE|OKX": { + "max": 1283.3, + "n": 94647, + "p95": 173.3 + }, + "TRADE|BINANCE": { + "max": 3284.0, + "n": 92415, + "p95": 801.2 + }, + "TRADE|OKX": { + "max": 3208.8, + "n": 26567, + "p95": 817.2 + } + }, + "verdict": "FAIL" + }, + "step_1_scorecard": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T115743Z/scorecard.json", + "sha256": "6ae2945de18852fc2fd0b514d5679f0c269ce444719285f4a5800adba71990d9" + }, + "variable": "TS writer DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE (batched Subscribe, SDK 2.3.0): absent = default 1 in step 1, 64 in step 2; same DL images, writer image, pins, depth and MARK delay" + }, + "stage": "MEASURED_ON_PRE_RC13_MIX", + "ts_writer": { + "container": "market_data_service_p184w", + "env": { + "DATA_LAYER_V2_MARK_WAKE_READ_DELAY_BY_VENUE": "BINANCE=100", + "DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE": "64", + "DATA_LAYER_V2_STREAM_PIPELINE_DEPTH": "0", + "MARKET_DATA_FIXED_WORKER_PINS": "OKX:BTCUSDT=2,OKX:ETHUSDT=0" + }, + "env_source": { + "path": "~/.local/state/trading-system/p184-rev11-window/campaign-roll/trace-20261011T123412Z.compose.json", + "sha256": "8b28aeeba20abb9657b06176b509d8732c65d2b07cea47af17881048ba04b58c" + }, + "image": "sha256:64347a94a3ed682d155690f3afaed5aaeba6d8feb8e8febcaeb7dc33dea488d9", + "sdk": "2.3.0", + "source": "a3035c8cf257cd5b3935224612e55e93ba1f4bf5", + "tag": "local/trading-system:p184-writer-a3035c8" + } + }, + "campaign": { + "demanded_slices": 64, + "dir": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z", + "files": { + "campaign": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/campaign.json", + "sha256": "b926a20e3acbe5dcdde965ec7147fe3ee46b1d5a6fe0ca95795d36614aa71c2b" + }, + "campaign_scorecard": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/scorecard.json", + "sha256": "747e476489a3ef9807e7ade9bd347b00a8e56226a5bc85a99b8fa03b5e7fa58e" + }, + "g11": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g11/g11-summary.json", + "sha256": "c85e4566e327328b0bd4c5f5cc75a4825f1edcff1586a4bad3226e3233c5b764" + }, + "g3": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g3/g3-summary.json", + "sha256": "fafca1be177364c2032dd18afd5f7e2c8d67468403b2de3fde2658ee69274e07" + }, + "g6_ts": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g6-ts/summary.json", + "sha256": "5a9fc905a71d86e1d013bfeed463bee3bfb366545fd742189b6ddedc3ee1bede" + }, + "g8": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g8/g8-summary.json", + "sha256": "0314b552d3f09b7d2143304ee77c9cd0c27a959fb10b764e2996865a2ffae7e5" + }, + "g9g10": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/g9g10/g9-g10-summary.json", + "sha256": "247c62e6702c264efb4b7b8e1a3089b369a378a8baf21e81d525945b3d8bf5d0" + }, + "mark": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/trace/mark-load.json", + "sha256": "b6a5198116d7916e32b8b05f48e2bf45798b7863aeb64e32247b874c442f8c9f" + }, + "stage100_gates": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/stage100/gates.json", + "sha256": "86b74f26c3f7dc68228d260becffa77247d080bfc5d6ce9736ebba495df81544" + }, + "stage100_host": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/stage100/host.json", + "sha256": "a5ba4fcf963ff2c003f4f6f211fa6a8210f453d56b6a37fd3e8fb19276c894e1" + }, + "stage100_receipt": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/stage100/receipt.json", + "sha256": "fdf26baa323115015501aa56b6450e51deaa83e9bf284d6ef8ec1574c20825eb" + }, + "transit": { + "path": "~/.local/state/qdl-v2/p184-final-campaign-20261011T123525Z/trace/transit-load.json", + "sha256": "bd1ccac05fad1728d73bbab833a629d43266756c3575e09082a498e1ee7c78c7" + } + }, + "frozen_budget": { + "path": "config/v2/p184-stage100-budget.json", + "sha256": "ff85335b3c8762d9acd8e7230f3c81ee1ec8403aff34cf2fb8f67714521c2541" + }, + "load_window_unix_s": [ + 1791722221.9021854, + 1791724021.9021854 + ], + "load_window_utc": "2026-10-11T12:37:01Z-13:07:01Z", + "protocol_met": true, + "protocol_problems": [], + "scoring_rule": "steady state >= 30 min at the stage-100 workload plus one burst window (gate table 'Measurement rules'); a measurement outside this rule can FAIL a gate but cannot PASS it. Verdicts: PASS, FAIL, NOT_MEASURED, DEFERRED; a NOT_MEASURED is never upgraded. 'observed' records the value against the threshold when the measurement does not meet the protocol.", + "tool_sha256": "39c22d78d37dbacac0042c85a6bfafafdd3dc70a9b21179c053f30a9d6cc19f3", + "tooling_source": "5ee94636a3b41d4d482eadbc9dd80f51e126cf84", + "writer_restarts": 0 + }, + "limits": [ + "Single host for every run: no HA, no independent failure domains, no mainnet or funded claim.", + "Latency numbers were measured on the pre-rc13 image mix; rc13 is source-identical for Query/Stream and source-equivalent (rebuilt, not byte-compared) for core/projector/ingestors; the bar edge differs by IDR-087 only.", + "G2/G8 PASS depends on the TS writer using batched Subscribe (DATA_LAYER_V2_MAX_RECORDS_PER_RESPONSE=64); at knob 1 G2 failed in step 1.", + "TRADE volume in step 2 (BINANCE/OKX n 66303/12492) was lower than step 1 (92415/26567) and campaign 4 (160328/39292); the PASS is at that volume, not a capacity claim at the campaign-4 volume.", + "G4 stays NOT_MEASURED: observed rows without a frozen baseline (bar 1h/5m both venues, OKX_INVERSE book_delta/book_snapshot/mark/quote); every baselined row (17) PASS; no baseline added after measuring.", + "G6 has no BOOK_DELTA row (not a Query shape in the frozen workload); TRADE OKX probe ages reach 139576 ms max in quiet markets and are typed NOT_ELIGIBLE, never unsafe.", + "DL mean CPU sum at stage 100 was 5.08 vCPU, above the owner's 5 vCPU DL budget (G14 passes its per-role headroom rule only).", + "Timing for request latency (G5) is SDK request latency on the load driver; transit (G1/G2) ends at the TS Redis APPLY reply." + ], + "measured": { + "g6_execution_probe": { + "errors": {}, + "kind": "final", + "note": "TRADE NOT_ELIGIBLE rows are typed refusals in quiet markets (policy 3000 ms); unsafe 0 in every row", + "per_binding_period_s": 3.2, + "quantity": "durable-view event age and typed execution quality at the Query endpoint (TS EXECUTION probe)", + "reads": 19213, + "rows": [ + { + "age_basis": "CLIENT_RECEIPT_MINUS_NEWEST_SOURCE_EVENT", + "age_p50_ms": 656, + "age_p95_ms": 1107, + "age_p99_ms": 1151, + "classes": { + "ELIGIBLE": 3005 + }, + "feed": "MARK_INDEX_PRICE", + "max_age_ms": 1250, + "n": 3005, + "policy_basis": "D3_EXECUTION", + "policy_ms": 2000, + "refusals": {}, + "unsafe": 0, + "venue": "BINANCE" + }, + { + "age_basis": "CLIENT_RECEIPT_MINUS_NEWEST_SOURCE_EVENT", + "age_p50_ms": 567.5, + "age_p95_ms": 1527, + "age_p99_ms": 2016, + "classes": { + "ELIGIBLE": 3566, + "ELIGIBLE_QUIET": 40 + }, + "feed": "MARK_INDEX_PRICE", + "max_age_ms": 2278, + "n": 3606, + "policy_basis": "D3_EXECUTION", + "policy_ms": 2000, + "refusals": {}, + "unsafe": 0, + "venue": "OKX" + }, + { + "age_basis": null, + "age_p50_ms": 324.0, + "age_p95_ms": 591, + "age_p99_ms": 909, + "classes": { + "ELIGIBLE": 3002 + }, + "feed": "QUOTE", + "max_age_ms": 1720, + "n": 3002, + "policy_basis": "D3_EXECUTION", + "policy_ms": 2000, + "refusals": {}, + "unsafe": 0, + "venue": "BINANCE" + }, + { + "age_basis": null, + "age_p50_ms": 366.0, + "age_p95_ms": 1227, + "age_p99_ms": 3518, + "classes": { + "ELIGIBLE": 3515, + "ELIGIBLE_QUIET": 85 + }, + "feed": "QUOTE", + "max_age_ms": 11131, + "n": 3600, + "policy_basis": "D3_EXECUTION", + "policy_ms": 2000, + "refusals": {}, + "unsafe": 0, + "venue": "OKX" + }, + { + "age_basis": null, + "age_p50_ms": 634.0, + "age_p95_ms": 2930, + "age_p99_ms": 5369, + "classes": { + "ELIGIBLE": 2862, + "NOT_ELIGIBLE": 138 + }, + "feed": "TRADE", + "max_age_ms": 10230, + "n": 3000, + "policy_basis": "D3_EXECUTION", + "policy_ms": 3000, + "refusals": {}, + "unsafe": 0, + "venue": "BINANCE" + }, + { + "age_basis": null, + "age_p50_ms": 1410.5, + "age_p95_ms": 25883, + "age_p99_ms": 64896, + "classes": { + "ELIGIBLE": 2084, + "NOT_ELIGIBLE": 916 + }, + "feed": "TRADE", + "max_age_ms": 139576, + "n": 3000, + "policy_basis": "D3_EXECUTION", + "policy_ms": 3000, + "refusals": {}, + "unsafe": 0, + "venue": "OKX" + } + ], + "scorecard_verdict": "PASS", + "stopped_by": "signal", + "targets": 32, + "window_unix_s": [ + 1791722125.7330813, + 1791724056.2921798 + ] + }, + "readiness_g9": { + "g10_trade_integrity": { + "codes": {}, + "examples": [], + "log_polls": 129, + "log_unreadable": 0, + "other_feed_disconnects": 0, + "trade_disconnect_other": 0, + "trade_gap": 0, + "trade_overflow": 0, + "trade_replay_expired": 0, + "trade_resumed": 0 + }, + "note": "the file covers the whole run (194 samples); the scorecard scores the 30-min load window (180 samples)", + "scored_load_window": { + "demanded_ne_session_ready_samples": 0, + "demanded_values": [ + 64 + ], + "eligibility_loss_causes": {}, + "eligibility_loss_rows": [], + "eligibility_losses_data_path": 0, + "execution_ready_max": 53, + "execution_ready_min": 48, + "legit_quiet_refusals": { + "TRADE:TRADE_EVENT_STALE": 48 + }, + "purpose_errors": 0, + "purpose_fields_present": true, + "readable_samples": 180, + "samples": 180, + "session_loss_causes": {}, + "unknown_samples": 0, + "v1_fallback_delta": 0, + "v1_fallback_max": 0 + }, + "whole_run_file": { + "demanded_ne_session_ready_samples": 0, + "demanded_values": [ + 64 + ], + "eligibility_loss_causes": {}, + "eligibility_loss_rows": [], + "eligibility_losses_data_path": 0, + "execution_ready_max": 53, + "execution_ready_min": 48, + "legit_quiet_refusals": { + "TRADE:TRADE_EVENT_STALE": 50 + }, + "purpose_errors": 0, + "purpose_fields_present": true, + "readable_samples": 194, + "samples": 194, + "session_loss_causes": {}, + "unknown_samples": 0, + "v1_fallback_delta": 0, + "v1_fallback_max": 0 + }, + "whole_run_verdict": [ + "PASS", + [] + ] + }, + "request_latency_g5": { + "basis": "stage100/gates.json, SDK request latency per class and venue; targets are the frozen stage-50 p95/p99", + "cold": [ + { + "bad": 0, + "gate": "cold:2500_5000_overlap_per_venue", + "runs": 4, + "status": "PASS" + }, + { + "count": 1, + "gate": "g5:COLD_2500:BINANCE", + "limit_ms": 6265.1, + "max_ms": 3557.278, + "rule": "small sample: max within the frozen limit", + "status": "PASS" + }, + { + "count": 1, + "gate": "g5:COLD_2500:OKX", + "limit_ms": 6265.1, + "max_ms": 4074.877, + "rule": "small sample: max within the frozen limit", + "status": "PASS" + }, + { + "count": 1, + "gate": "g5:COLD_5000:BINANCE", + "limit_ms": 15267.0, + "max_ms": 6048.93, + "rule": "small sample: max within the frozen limit", + "status": "PASS" + }, + { + "count": 1, + "gate": "g5:COLD_5000:OKX", + "limit_ms": 15267.0, + "max_ms": 7011.598, + "rule": "small sample: max within the frozen limit", + "status": "PASS" + } + ], + "failed_gates": [], + "groups": [ + { + "count": 355, + "gate": "latency:BAR_LATEST:BINANCE", + "max": 267.327, + "p50": 15.524, + "p95": 25.601, + "p99": 41.544, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 1000, + "target_p99": 2000 + }, + { + "count": 355, + "gate": "latency:BAR_LATEST:OKX", + "max": 354.887, + "p50": 27.242, + "p95": 77.565, + "p99": 234.588, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 1000, + "target_p99": 2000 + }, + { + "count": 355, + "gate": "latency:L2_SNAPSHOT:BINANCE", + "max": 479.697, + "p50": 40.045, + "p95": 69.358, + "p99": 251.959, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 300, + "target_p99": 750 + }, + { + "count": 355, + "gate": "latency:L2_SNAPSHOT:OKX", + "max": 440.775, + "p50": 29.848, + "p95": 54.544, + "p99": 220.378, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 300, + "target_p99": 750 + }, + { + "count": 53250, + "gate": "latency:MARK_INDEX_REFERENCE:BINANCE", + "max": 499.112, + "p50": 20.096, + "p95": 58.353, + "p99": 83.549, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 250, + "target_p99": 500 + }, + { + "count": 53250, + "gate": "latency:MARK_INDEX_REFERENCE:OKX", + "max": 550.297, + "p50": 23.962, + "p95": 57.627, + "p99": 71.045, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 250, + "target_p99": 500 + }, + { + "count": 35500, + "gate": "latency:QUOTE_SNAPSHOT:BINANCE", + "max": 613.678, + "p50": 12.77, + "p95": 31.472, + "p99": 65.693, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 100, + "target_p99": 250 + }, + { + "count": 35500, + "gate": "latency:QUOTE_SNAPSHOT:OKX", + "max": 613.255, + "p50": 12.329, + "p95": 32.554, + "p99": 63.39, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 100, + "target_p99": 250 + }, + { + "count": 354, + "gate": "latency:TRADE_SNAPSHOT:BINANCE", + "max": 285.234, + "p50": 11.533, + "p95": 28.489, + "p99": 50.097, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 100, + "target_p99": 250 + }, + { + "count": 355, + "gate": "latency:TRADE_SNAPSHOT:OKX", + "max": 255.068, + "p50": 13.836, + "p95": 30.461, + "p99": 64.057, + "rule": "p95 and p99", + "status": "PASS", + "target_p95": 100, + "target_p99": 250 + } + ], + "request_and_stream_accounting": [ + { + "gate": "requests:offered_equals_sent_plus_missed", + "offered": 182709, + "status": "PASS", + "unbalanced": 0 + }, + { + "gate": "requests:no_missed_ticks", + "missed": 0, + "status": "PASS" + }, + { + "codes": {}, + "failed": 0, + "gate": "requests:no_failures", + "status": "PASS" + }, + { + "gate": "requests:no_starved_session", + "starved": 0, + "status": "PASS" + }, + { + "gate": "client:scheduler_lag_valid", + "p99_ms": 3.521, + "status": "PASS" + }, + { + "errors": 0, + "gate": "streams:no_errors", + "status": "PASS", + "streams": 180 + }, + { + "gate": "streams:every_live_stream_delivered", + "silent": [], + "status": "PASS" + }, + { + "bar_streams": 60, + "gate": "streams:final_bar_every_bar_stream", + "missing": [], + "status": "PASS" + }, + { + "failures": [], + "gate": "startup:bounded", + "retries": 193, + "seconds": 70.41, + "status": "PASS" + }, + { + "gate": "teardown:no_leaked_work", + "leaked": 0, + "status": "PASS" + }, + { + "expected": 154, + "gate": "ts:ready_60_every_sample", + "not_ready": 0, + "samples": 154, + "status": "PASS", + "unreadable": 0 + }, + { + "first": 0, + "gate": "ts:v2_error_not_increased", + "last": 0, + "status": "PASS" + }, + { + "forbidden": {}, + "gate": "ts:no_auth_or_manifest_disconnect", + "status": "PASS" + }, + { + "baseline_per_minute": 0.0, + "codes": {}, + "gate": "ts:disconnects_within_baseline", + "run_per_minute": 0.0, + "status": "PASS" + }, + { + "gate": "final:fault_windows_ran", + "status": "PASS", + "windows": { + "BURST": [ + 720.0, + 730.0 + ], + "BURST_unix_s": [ + 1791722941.902, + 1791722951.902 + ], + "RECONNECT": 20, + "RECONNECT_unix_s": [ + 1791723481.902, + 1791723496.902 + ], + "SLOW_READER": 1, + "reconnect_restored": 20 + } + }, + { + "count": 1264, + "gate": "final:burst_window_measured", + "status": "PASS" + }, + { + "count": 1520, + "gate": "final:reconnect_window_measured", + "status": "PASS" + } + ], + "stage": 100, + "status": "PASS" + }, + "transit_g1_g2": { + "dropped_at_export_total": 0, + "excluded_from_transit": { + "BOOK_DELTA|BINANCE|no_venue_clock_SDK_QUERY_RETURN": 1, + "BOOK_DELTA|BINANCE|outcome_DUPLICATE": 18, + "BOOK_DELTA|BINANCE|outcome_REJECTED": 581, + "BOOK_DELTA|OKX|no_venue_clock_SDK_QUERY_RETURN": 1, + "BOOK_DELTA|OKX|outcome_DUPLICATE": 76, + "BOOK_DELTA|OKX|outcome_REJECTED": 642, + "QUOTE|BINANCE|no_venue_clock_NOT_MEASURED": 536, + "QUOTE|BINANCE|outcome_DUPLICATE": 7337, + "QUOTE|BINANCE|outcome_REFRESHED": 933, + "QUOTE|OKX|no_venue_clock_NOT_MEASURED": 3011, + "QUOTE|OKX|outcome_DUPLICATE": 6762, + "QUOTE|OKX|outcome_REFRESHED": 811, + "TRADE|BINANCE|no_venue_clock_NOT_MEASURED": 299, + "TRADE|BINANCE|no_venue_clock_SDK_QUERY_RETURN": 26, + "TRADE|BINANCE|outcome_DUPLICATE": 178, + "TRADE|BINANCE|outcome_REFRESHED": 97, + "TRADE|OKX|no_venue_clock_NOT_MEASURED": 248, + "TRADE|OKX|no_venue_clock_SDK_QUERY_RETURN": 178, + "TRADE|OKX|outcome_DUPLICATE": 60, + "TRADE|OKX|outcome_REFRESHED": 113 + }, + "g7_consumer_path": { + "BOOK_DELTA": { + "sdk_to_writer_ms": { + "max": 350.1, + "min": 0.2, + "n": 178828, + "p50": 22.6, + "p95": 33.8, + "p99": 72.2 + }, + "verdict": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + }, + "writer_to_redis_ms": { + "max": 195.5, + "min": 0.5, + "n": 177511, + "p50": 2.7, + "p95": 13.5, + "p99": 24.6 + } + }, + "BOOK_SNAPSHOT": { + "sdk_to_writer_ms": { + "max": 361.7, + "min": 15.3, + "n": 19767, + "p50": 30.0, + "p95": 44.8, + "p99": 108.5 + }, + "verdict": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + }, + "writer_to_redis_ms": { + "max": 284.2, + "min": 1.5, + "n": 19767, + "p50": 5.5, + "p95": 20.5, + "p99": 61.0 + } + }, + "MARK_INDEX_PRICE": { + "sdk_to_writer_ms": { + "max": 3.8, + "min": 0.2, + "n": 30514, + "p50": 0.4, + "p95": 0.6, + "p99": 0.7 + }, + "verdict": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + }, + "writer_to_redis_ms": { + "max": 339.1, + "min": 1.1, + "n": 30514, + "p50": 3.7, + "p95": 14.4, + "p99": 28.8 + } + }, + "QUOTE": { + "sdk_to_writer_ms": { + "max": 386.1, + "min": 1.1, + "n": 183275, + "p50": 22.1, + "p95": 30.7, + "p99": 47.4 + }, + "verdict": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + }, + "writer_to_redis_ms": { + "max": 230.7, + "min": 0.5, + "n": 202646, + "p50": 1.3, + "p95": 5.9, + "p99": 15.2 + } + }, + "TRADE": { + "sdk_to_writer_ms": { + "max": 228.7, + "min": 0.2, + "n": 79000, + "p50": 23.1, + "p95": 45.3, + "p99": 97.9 + }, + "verdict": { + "sdk_to_writer_ms": "INSIDE", + "writer_to_redis_ms": "INSIDE" + }, + "writer_to_redis_ms": { + "max": 247.0, + "min": 0.5, + "n": 79994, + "p50": 4.9, + "p95": 33.3, + "p99": 77.0 + } + } + }, + "invalid": {}, + "mark": { + "records": 36537, + "rows": [ + { + "feed": "MARK_INDEX_PRICE", + "gate": "G2", + "index_age_max_ms": 803.1, + "index_age_p95_ms": 360.3, + "max": 803.1, + "min": 227.6, + "n": 9000, + "p50": 304.1, + "p95": 360.3, + "p99": 419.8, + "quantity": "new_event_transit_ms (first apply per mark event)", + "upstream_p95_ms": 350.7, + "venue": "BINANCE", + "verdict": "INSIDE", + "writer_p95_ms": 15.5 + }, + { + "feed": "MARK_INDEX_PRICE", + "gate": "G2", + "index_age_max_ms": 2332.2, + "index_age_p95_ms": 1512.3, + "max": 712.7, + "min": 77.3, + "n": 21514, + "p50": 231.1, + "p95": 337.4, + "p99": 371.2, + "quantity": "new_event_transit_ms (first apply per mark event)", + "upstream_p95_ms": 330.5, + "venue": "OKX", + "verdict": "INSIDE", + "writer_p95_ms": 14.6 + } + ], + "rule": { + "max_lt_ms": 1000.0, + "p95_le_ms": 500.0 + }, + "sdk_receive_basis": { + "BINANCE:ADAPTER_REFERENCE_RETURN": 9000, + "OKX:ADAPTER_REFERENCE_RETURN": 21514 + }, + "window": { + "from_ns": 1791722221902185472, + "seconds": 1800.0, + "to_ns": 1791724021902185472 + } + }, + "quantity": "venue event -> TS Redis APPLY reply (end-to-end to TS cache)", + "records": 618496, + "rows": [ + { + "feed": "BOOK_DELTA", + "gate": "G1", + "max": 570.4, + "min": 65.0, + "n": 82692, + "p50": 103.9, + "p95": 138.8, + "p99": 190.3, + "split_p95_ms": { + "ingest_to_sdk_ms": 76.4, + "sdk_to_reply_ms": 42.4, + "venue_to_ingest_ms": 29.3 + }, + "venue": "BINANCE", + "verdict": "INSIDE" + }, + { + "feed": "BOOK_DELTA", + "gate": "G1", + "max": 585.9, + "min": 69.2, + "n": 94817, + "p50": 106.2, + "p95": 146.4, + "p99": 199.5, + "split_p95_ms": { + "ingest_to_sdk_ms": 76.7, + "sdk_to_reply_ms": 47.5, + "venue_to_ingest_ms": 35.1 + }, + "venue": "OKX", + "verdict": "INSIDE" + }, + { + "feed": "BOOK_SNAPSHOT", + "gate": "G1", + "max": 571.2, + "min": 76.1, + "n": 8894, + "p50": 115.1, + "p95": 176.5, + "p99": 242.4, + "split_p95_ms": { + "ingest_to_sdk_ms": 81.1, + "sdk_to_reply_ms": 75.8, + "venue_to_ingest_ms": 28.8 + }, + "venue": "BINANCE", + "verdict": "INSIDE" + }, + { + "feed": "BOOK_SNAPSHOT", + "gate": "G1", + "max": 503.2, + "min": 80.2, + "n": 10873, + "p50": 118.5, + "p95": 177.6, + "p99": 237.8, + "split_p95_ms": { + "ingest_to_sdk_ms": 80.3, + "sdk_to_reply_ms": 74.4, + "venue_to_ingest_ms": 35.5 + }, + "venue": "OKX", + "verdict": "INSIDE" + }, + { + "feed": "QUOTE", + "gate": "G2", + "max": 531.5, + "min": 65.9, + "n": 103534, + "p50": 122.5, + "p95": 162.2, + "p99": 199.4, + "split_p95_ms": { + "ingest_to_sdk_ms": 107.2, + "sdk_to_reply_ms": 34.5, + "venue_to_ingest_ms": 28.8 + }, + "venue": "BINANCE", + "verdict": "INSIDE" + }, + { + "feed": "QUOTE", + "gate": "G2", + "max": 532.0, + "min": 63.9, + "n": 79722, + "p50": 126.0, + "p95": 169.1, + "p99": 217.2, + "split_p95_ms": { + "ingest_to_sdk_ms": 110.4, + "sdk_to_reply_ms": 37.9, + "venue_to_ingest_ms": 29.8 + }, + "venue": "OKX", + "verdict": "INSIDE" + }, + { + "feed": "TRADE", + "gate": "G2", + "max": 583.1, + "min": 68.2, + "n": 66303, + "p50": 118.2, + "p95": 243.4, + "p99": 381.4, + "split_p95_ms": { + "ingest_to_sdk_ms": 154.8, + "sdk_to_reply_ms": 77.6, + "venue_to_ingest_ms": 36.4 + }, + "venue": "BINANCE", + "verdict": "INSIDE" + }, + { + "feed": "TRADE", + "gate": "G2", + "max": 495.5, + "min": 71.5, + "n": 12492, + "p50": 114.4, + "p95": 210.3, + "p99": 330.7, + "split_p95_ms": { + "ingest_to_sdk_ms": 115.1, + "sdk_to_reply_ms": 77.8, + "venue_to_ingest_ms": 37.1 + }, + "venue": "OKX", + "verdict": "INSIDE" + } + ], + "rules": { + "BOOK_DELTA": { + "gate": "G1", + "max_lt_ms": 2000.0, + "p95_le_ms": 500.0, + "p99_le_ms": 1000.0 + }, + "BOOK_SNAPSHOT": { + "gate": "G1", + "max_lt_ms": 2000.0, + "p95_le_ms": 500.0, + "p99_le_ms": 1000.0 + }, + "QUOTE": { + "gate": "G2", + "max_lt_ms": 1000.0, + "p95_le_ms": 500.0 + }, + "TRADE": { + "gate": "G2", + "max_lt_ms": 1500.0, + "p95_le_ms": 500.0 + } + }, + "window": { + "from_ns": 1791722221902185472, + "seconds": 1800.0, + "to_ns": 1791724021902185472 + } + }, + "writer_restart_g11": { + "caught_up_s": { + "max": 7.511, + "min": 0.158, + "n": 64, + "p50": 0.158, + "p95": 4.51, + "p99": null + }, + "first_valid_apply_s": { + "max": 7.511, + "min": 0.158, + "n": 64, + "p50": 0.158, + "p95": 4.51, + "p99": null + }, + "outage_seconds": 0.132, + "rewinds_total": 0 + } + }, + "release": "v2.2.3", + "schema": "qdl.release-endpoint-report.v2", + "scope": "Data Layer P18.4 phase D closure on the Kafka-native stack at the stage-100 workload: 64 demanded TS Paper slices, BINANCE and OKX (incl. OKX_INVERSE), one TS writer on the same host. Numbers are copied from the phase E step-2 campaign files cited below; nothing is re-measured here. Not a TS order/accounting/alpha certification, not HA, not independent failure domains." +}