From 8dcbd555d7a068d87f7c6b883b1816a55f758f1b Mon Sep 17 00:00:00 2001 From: Dave Greer Date: Wed, 12 Aug 2026 14:56:44 +0000 Subject: [PATCH 1/3] validation: don't police a connector and model sync schedule together Remove the check which rejected a materialization configuring a connector-side `syncSchedule` alongside a model-level one, and with it `connector_config_has_sync_schedule` -- which sniffed the undecrypted endpoint config for the key, treating an empty object or all-null / all empty-string values as absent so that a UI clearing the schedule didn't leave a tripwire behind. Configuring both is not something we want users to do, but it doesn't break: the two pacing mechanisms just interleave chaotically, and we plan to migrate extant configs automatically. That isn't worth the complexity, nor the guesswork of deciding what a disabled connector schedule looks like. It also stood in the way of runtime-only commit pacing, where a model-level `syncSchedule` is the sole cadence control and the connector's is left at its documented all-zero off switch. The connector parses a Go duration, which accepts "0m0s" and a bare "0" as readily as "0s", so the check would have had to enumerate zero spellings to let that through. `SyncScheduleInvalid` stays: a malformed schedule is still an error. --- crates/validation/src/errors.rs | 4 - crates/validation/src/materialization.rs | 32 --- crates/validation/tests/scenario_tests.rs | 66 ------ ...configured_in_both_places_is_rejected.snap | 221 ------------------ 4 files changed, 323 deletions(-) delete mode 100644 crates/validation/tests/snapshots/scenario_tests__sync_schedule_configured_in_both_places_is_rejected.snap diff --git a/crates/validation/src/errors.rs b/crates/validation/src/errors.rs index 63fa9b2198f..d9ac9a63614 100644 --- a/crates/validation/src/errors.rs +++ b/crates/validation/src/errors.rs @@ -395,10 +395,6 @@ pub enum Error { #[error("invalid sync schedule: {detail}")] SyncScheduleInvalid { detail: String }, - #[error( - "materialization {materialization} configures a sync schedule both in its connector config (syncSchedule) and as a materialization sync schedule; remove one" - )] - SyncScheduleConflict { materialization: String }, #[error("raising an error because {this_entity} specifies `onIncompatibleSchemaChange: abort`")] AbortOnIncompatibleSchemaChange { diff --git a/crates/validation/src/materialization.rs b/crates/validation/src/materialization.rs index 50173ae77c5..2f36d4395fd 100644 --- a/crates/validation/src/materialization.rs +++ b/crates/validation/src/materialization.rs @@ -700,16 +700,6 @@ async fn walk_materialization( if let Err(detail) = sync_schedule.validate() { Error::SyncScheduleInvalid { detail }.push(scope, errors); } - // A connector-side sync schedule and a model-level one would fight - // over commit cadence; reject configuring both. - if let models::MaterializationEndpoint::Connector(config) = &endpoint - && connector_config_has_sync_schedule(&config.config) - { - Error::SyncScheduleConflict { - materialization: materialization.to_string(), - } - .push(scope, errors); - } bytes::Bytes::from( serde_json::to_vec(sync_schedule).expect("sync schedule must serialize"), ) @@ -1165,28 +1155,6 @@ fn temporary_group_by_migration( .collect() } -/// Whether a connector endpoint config carries a configured top-level -/// `syncSchedule`, i.e. the legacy connector-side sync schedule. Detected -/// without decryption: SOPS preserves object keys, and sync-schedule fields -/// are not secrets so their values remain plaintext. -/// -/// An empty `syncSchedule` object, or one whose values are all null or empty -/// strings, is NOT configured: the connector treats those identically to an -/// absent key, and a UI removing the schedule may leave such a remnant behind. -fn connector_config_has_sync_schedule(config: &models::RawValue) -> bool { - let Ok(serde_json::Value::Object(map)) = serde_json::from_str(config.get()) else { - return false; - }; - match map.get("syncSchedule") { - None | Some(serde_json::Value::Null) => false, - Some(serde_json::Value::Object(sched)) => sched - .values() - .any(|value| !value.is_null() && value.as_str() != Some("")), - // Any other shape is malformed, but conservatively "configured". - Some(_) => true, - } -} - fn validate_triggers( scope: Scope, triggers: &[models::TriggerConfig], diff --git a/crates/validation/tests/scenario_tests.rs b/crates/validation/tests/scenario_tests.rs index 0d0d9aa8dfb..05107ea2839 100644 --- a/crates/validation/tests/scenario_tests.rs +++ b/crates/validation/tests/scenario_tests.rs @@ -43,72 +43,6 @@ fn connector_validation_is_skipped_when_shards_are_disabled() { insta::assert_debug_snapshot!(outcome); } -#[test] -fn sync_schedule_configured_in_both_places_is_rejected() { - // A model-level sync schedule alongside a connector-config `syncSchedule` - // is an error: the two would fight over commit cadence. - let fixture = r##" -test://example/catalog.yaml: - materializations: - testing/materialization: - endpoint: - connector: - image: an/image - config: - syncSchedule: { syncFrequency: 30m } - syncSchedule: { baseInterval: 15m } - shards: { disable: true } - bindings: [] - -driver: - dataPlanes: - "1d:1d:1d:1d:1d:1d:1d:1d": {} -"##; - let outcome = common::run(fixture, "{}"); - insta::assert_debug_snapshot!(outcome); -} - -#[test] -fn sync_schedule_conflict_ignores_a_cleared_connector_schedule() { - // A UI that removes a connector-side sync schedule may leave behind an - // empty `syncSchedule: {}`, or one whose values are cleared to empty or - // null. The connector treats those identically to an absent key, so they - // must not conflict with a model-level schedule. - let fixture = r##" -test://example/catalog.yaml: - materializations: - testing/empty-object: - endpoint: - connector: - image: an/image - config: - syncSchedule: {} - syncSchedule: { baseInterval: 15m } - shards: { disable: true } - bindings: [] - testing/cleared-values: - endpoint: - connector: - image: an/image - config: - syncSchedule: { syncFrequency: "", timezone: null } - syncSchedule: { baseInterval: 15m } - shards: { disable: true } - bindings: [] - -driver: - dataPlanes: - "1d:1d:1d:1d:1d:1d:1d:1d": {} -"##; - let outcome = common::run(fixture, "{}"); - assert!( - outcome.errors.is_empty() && outcome.errors_draft.is_empty(), - "expected no errors, got: {:?} {:?}", - outcome.errors, - outcome.errors_draft, - ); -} - #[test] fn test_collection_schema_contains_flow_document() { let fixture = r##" diff --git a/crates/validation/tests/snapshots/scenario_tests__sync_schedule_configured_in_both_places_is_rejected.snap b/crates/validation/tests/snapshots/scenario_tests__sync_schedule_configured_in_both_places_is_rejected.snap deleted file mode 100644 index 20a63036b11..00000000000 --- a/crates/validation/tests/snapshots/scenario_tests__sync_schedule_configured_in_both_places_is_rejected.snap +++ /dev/null @@ -1,221 +0,0 @@ ---- -source: crates/validation/tests/scenario_tests.rs -expression: outcome ---- -Outcome { - built_captures: [], - built_collections: [], - built_materializations: [ - BuiltMaterialization { - materialization: testing/materialization, - scope: test://example/catalog.yaml#/materializations/testing~1materialization, - control_id: "0000000000000000", - data_plane_id: "1d1d1d1d1d1d1d1d", - expect_pub_id: "0000000000000000", - expect_build_id: "0000000000000000", - model: { - "endpoint": { - "connector": { - "image": "an/image", - "config": {"syncSchedule":{"syncFrequency":"30m"}} - } - }, - "bindings": [], - "shards": { - "disable": true - }, - "syncSchedule": { - "baseInterval": "15m" - } - }, - model_fixes: [], - validated: Validated { - bindings: [], - }, - spec: MaterializationSpec { - name: "testing/materialization", - connector_type: Image, - config_json: b"{\"image\":\"an/image\",\"config\":{\"syncSchedule\":{\"syncFrequency\":\"30m\"}}}", - bindings: [], - shard_template: Some( - ShardSpec { - id: "materialize/testing/materialization/2020202020202020", - sources: [], - recovery_log_prefix: "recovery", - hint_prefix: "/estuary/flow/hints", - hint_backups: 2, - max_txn_duration: Some( - Duration { - seconds: 1200, - nanos: 0, - }, - ), - min_txn_duration: Some( - Duration { - seconds: 0, - nanos: 0, - }, - ), - disable: true, - hot_standbys: 0, - labels: Some( - LabelSet { - labels: [ - Label { - name: "app.gazette.dev/managed-by", - value: "estuary.dev/flow", - prefix: false, - }, - Label { - name: "estuary.dev/build", - value: "2121212121212121", - prefix: false, - }, - Label { - name: "estuary.dev/log-level", - value: "info", - prefix: false, - }, - Label { - name: "estuary.dev/task-name", - value: "testing/materialization", - prefix: false, - }, - Label { - name: "estuary.dev/task-type", - value: "materialization", - prefix: false, - }, - ], - }, - ), - disable_wait_for_ack: false, - ring_buffer_size: 65536, - read_channel_size: 4096, - }, - ), - recovery_log_template: Some( - JournalSpec { - name: "recovery/materialize/testing/materialization/2020202020202020", - replication: 3, - labels: Some( - LabelSet { - labels: [ - Label { - name: "app.gazette.dev/managed-by", - value: "estuary.dev/flow", - prefix: false, - }, - Label { - name: "content-type", - value: "application/x-gazette-recoverylog", - prefix: false, - }, - Label { - name: "estuary.dev/build", - value: "2121212121212121", - prefix: false, - }, - Label { - name: "estuary.dev/task-name", - value: "testing/materialization", - prefix: false, - }, - Label { - name: "estuary.dev/task-type", - value: "materialization", - prefix: false, - }, - ], - }, - ), - fragment: Some( - Fragment { - length: 268435456, - compression_codec: Snappy, - stores: [ - "s3://a-bucket/", - ], - refresh_interval: Some( - Duration { - seconds: 300, - nanos: 0, - }, - ), - retention: None, - flush_interval: Some( - Duration { - seconds: 172800, - nanos: 0, - }, - ), - path_postfix_template: "", - }, - ), - flags: 4, - max_append_rate: 4194304, - suspend: None, - }, - ), - network_ports: [], - inactive_bindings: [], - triggers_json: b"", - created_at: "", - sync_schedule_json: b"{\"baseInterval\":\"15m\"}", - linked_collections: [], - }, - previous_spec: NULL, - is_touch: 0, - dependency_hash: NULL, - }, - ], - built_tests: [], - captures: [], - collections: [], - errors: [ - Error { - scope: test://example/catalog.yaml#/materializations/testing~1materialization/syncSchedule, - error: materialization testing/materialization configures a sync schedule both in its connector config (syncSchedule) and as a materialization sync schedule; remove one, - }, - ], - errors_draft: [], - fetches: [ - Fetch { - depth: 1, - resource: test://example/catalog.yaml, - }, - ], - imports: [], - materializations: [ - DraftMaterialization { - materialization: testing/materialization, - scope: test://example/catalog.yaml#/materializations/testing~1materialization, - expect_pub_id: NULL, - model: { - "endpoint": { - "connector": { - "image": "an/image", - "config": {"syncSchedule":{"syncFrequency":"30m"}} - } - }, - "bindings": [], - "shards": { - "disable": true - }, - "syncSchedule": { - "baseInterval": "15m" - } - }, - is_touch: 0, - }, - ], - resources: [ - Resource { - resource: test://example/catalog.yaml, - content_type: "CATALOG", - content: ".. binary ..", - content_dom: {"materializations":{"testing/materialization":{"bindings":[],"endpoint":{"connector":{"config":{"syncSchedule":{"syncFrequency":"30m"}},"image":"an/image"}},"shards":{"disable":true},"syncSchedule":{"baseInterval":"15m"}}}}, - }, - ], - tests: [], -} From 14c76464d32aae120d8059b0408979463b7d4e54 Mon Sep 17 00:00:00 2001 From: Dave Greer Date: Wed, 12 Aug 2026 14:56:53 +0000 Subject: [PATCH 2/3] protocol, runtime-next: serve TaskControl.SyncNow in the materialize leader Sync-now forces an immediate commit of a materialization's open transaction -- collapsing any sync-schedule hold -- and resolves once that transaction is fully acknowledged (committed and queryable in the destination). Define it as a new TaskControl gRPC service in runtime.proto, kept separate from Leader because its callers present ordinary gazette READ claims over the task's shards rather than the LEAD capability. The response stream is exactly one Ack, then zero or more Heartbeats, then exactly one Done. Its messages are structural and carry no payload: transaction statistics are recorded to the task's stats journal, which is where callers read them. There is no error for "nothing to do" -- a task with nothing to await Acks and its Done follows immediately. - The TaskControl tonic service registers beside Leader on the sidecar port, behind a gazette READ AuthN floor rather than LEAD. A live session is reachable through a per-session SyncNowHandle (task name -> delivery channel plus shard-zero ID, un-registered by guard with the session); the caller's claims are authorized against shard zero and the request is delivered into the Actor's select loop. Anything not addressable that way is NOT_FOUND, including a capture or derivation: only the reactor front door can distinguish those, because only it has the shard keyspace to resolve a task's type. - The Actor decides what a request awaits with a pure function over POD FSM state (sync_now::evaluate over fsm::sync_now_inputs): one Tail::Done per pipeline stage ahead of the caller, and the existing `close_requested` input armed when an open transaction's close decision is still ahead. - Waiters park on a target count of Tail::Done transitions -- the "committed and queryable" instant -- so N concurrent requests await the same commit. The Begin->Done stopping shortcut is excluded from that count: it defers connector acknowledgement to a future session rather than completing it. Parked waiters receive ~15s Heartbeats via the actor's existing wake machinery. - A session exiting with parked waiters errors them Unavailable, and closes the request channel before its final drain so that a request racing teardown is likewise answered Unavailable rather than accepted and discarded. Sync-now is idempotent and callers re-invoke. An e2e test pins the contract on the Rust side: a real tonic server hosting Leader + TaskControl behind armed AuthN interceptors (assembled exactly as the runtime sidecar serves them), a real materialize leader session, and real journal IO for the stats and ACK-intent writes that gate Tail::Done. The test plays shard zero itself over the wire, with a scripted connector side whose withheld Acknowledged is its control over when a transaction becomes fully acknowledged. Source checkpoints come from a fixture ShuffleSessionFactory fed synthetic Frontiers, so no journals are read. Fixture tasks cover both hold shapes: a 2h syncSchedule and a 300s min-transaction-duration floor. Covered: a held transaction collapses on SyncNow and Done arrives only after the shard-side ack, with an inline re-poke; N concurrent pokes resolve at one shared commit; a poke in the post-commit drain window awaits the same ack; a close request bypasses the min-duration floor; parked waiters receive Heartbeats (the one deliberately slow test -- the 15s cadence has no test knob on purpose); token scope and capability enforcement plus NOT_FOUND and InvalidArgument resolution over the real wire, including a capture, which this service answers NOT_FOUND because task types are resolved by the reactor front door; and a session exiting with parked waiters errors them Unavailable and un-registers its handle. A schedule hold is proven by the leader's silence -- absent a poke it asks the shard for nothing -- and the drain barrier by Done itself, which is what proves Tail::Done was reached. Held-transaction tests warm up with a first transaction (never schedule-held, by session_start) and guard the ~1-in-1e5 wall-clock flake of opening adjacent to a schedule grid instant with one bounded retry. Adds a rustls dev-dependency to install the process-level CryptoProvider that production mains install; pinned to 0.23 because the workspace's rustls = "0" re-resolves to the also-locked 0.21 (which lacks the aws_lc_rs feature) when named from this crate. --- Cargo.lock | 1 + crates/proto-flow/src/runtime.rs | 39 + crates/proto-flow/src/runtime.serde.rs | 438 ++++ crates/proto-grpc/src/runtime.rs | 324 +++ crates/runtime-next/Cargo.toml | 1 + crates/runtime-next/README.md | 23 + .../src/leader/materialize/actor.rs | 168 ++ .../src/leader/materialize/fsm.rs | 23 + .../src/leader/materialize/handler.rs | 13 +- .../src/leader/materialize/mod.rs | 2 + .../src/leader/materialize/sync_now.rs | 140 ++ crates/runtime-next/src/leader/service.rs | 132 ++ .../runtime-next/tests/sync_now_e2e.flow.yaml | 37 + crates/runtime-next/tests/sync_now_e2e.rs | 956 +++++++++ crates/runtime-sidecar/README.md | 18 +- crates/runtime-sidecar/src/lib.rs | 8 +- go/protocols/runtime/runtime.pb.go | 1821 +++++++++++++---- go/protocols/runtime/runtime.proto | 53 + 18 files changed, 3835 insertions(+), 362 deletions(-) create mode 100644 crates/runtime-next/src/leader/materialize/sync_now.rs create mode 100644 crates/runtime-next/tests/sync_now_e2e.flow.yaml create mode 100644 crates/runtime-next/tests/sync_now_e2e.rs diff --git a/Cargo.lock b/Cargo.lock index 3b17495a5aa..38b1fe2cc73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7401,6 +7401,7 @@ dependencies = [ "rand 0.9.2", "reqwest 0.12.23", "rocksdb", + "rustls 0.23.32", "schemars", "serde", "serde_json", diff --git a/crates/proto-flow/src/runtime.rs b/crates/proto-flow/src/runtime.rs index ec5d962982e..23c66acd828 100644 --- a/crates/proto-flow/src/runtime.rs +++ b/crates/proto-flow/src/runtime.rs @@ -1407,6 +1407,45 @@ pub mod derive { #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct StartedCommit {} } +/// SyncNowRequest is the request of a TaskControl.SyncNow RPC. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SyncNowRequest { + /// Name of the task to synchronize. + #[prost(string, tag = "1")] + pub task_name: ::prost::alloc::string::String, +} +/// SyncNowResponse is the streamed response of a TaskControl.SyncNow RPC. +/// Its messages are structural: they mark the caller's position in the +/// stream and carry no payload. Statistics of the awaited transaction are +/// recorded to the task's stats journal, which is where callers read them. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SyncNowResponse { + #[prost(oneof = "sync_now_response::Response", tags = "1, 2, 3")] + pub response: ::core::option::Option, +} +/// Nested message and enum types in `SyncNowResponse`. +pub mod sync_now_response { + /// Ack is sent exactly once, as the first message of the stream: the + /// request reached the task's leader and the commit has been forced. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct Ack {} + /// Heartbeat keeps a long-lived stream alive while the caller waits. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct Heartbeat {} + /// Done is sent exactly once, as the final message of the stream: the + /// awaited transaction is committed and queryable in the endpoint. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct Done {} + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Response { + #[prost(message, tag = "1")] + Ack(Ack), + #[prost(message, tag = "2")] + Heartbeat(Heartbeat), + #[prost(message, tag = "3")] + Done(Done), + } +} /// Plane describes the type of data plane in which the runtime is operating. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/crates/proto-flow/src/runtime.serde.rs b/crates/proto-flow/src/runtime.serde.rs index fe3bc873717..03a1bad365a 100644 --- a/crates/proto-flow/src/runtime.serde.rs +++ b/crates/proto-flow/src/runtime.serde.rs @@ -10571,6 +10571,444 @@ impl<'de> serde::Deserialize<'de> for Stopped { deserializer.deserialize_struct("runtime.Stopped", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for SyncNowRequest { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if !self.task_name.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("runtime.SyncNowRequest", len)?; + if !self.task_name.is_empty() { + struct_ser.serialize_field("taskName", &self.task_name)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for SyncNowRequest { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "task_name", + "taskName", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + TaskName, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "taskName" | "task_name" => Ok(GeneratedField::TaskName), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = SyncNowRequest; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.SyncNowRequest") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut task_name__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::TaskName => { + if task_name__.is_some() { + return Err(serde::de::Error::duplicate_field("taskName")); + } + task_name__ = Some(map_.next_value()?); + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(SyncNowRequest { + task_name: task_name__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("runtime.SyncNowRequest", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for SyncNowResponse { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.response.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("runtime.SyncNowResponse", len)?; + if let Some(v) = self.response.as_ref() { + match v { + sync_now_response::Response::Ack(v) => { + struct_ser.serialize_field("ack", v)?; + } + sync_now_response::Response::Heartbeat(v) => { + struct_ser.serialize_field("heartbeat", v)?; + } + sync_now_response::Response::Done(v) => { + struct_ser.serialize_field("done", v)?; + } + } + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for SyncNowResponse { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "ack", + "heartbeat", + "done", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Ack, + Heartbeat, + Done, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "ack" => Ok(GeneratedField::Ack), + "heartbeat" => Ok(GeneratedField::Heartbeat), + "done" => Ok(GeneratedField::Done), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = SyncNowResponse; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.SyncNowResponse") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut response__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Ack => { + if response__.is_some() { + return Err(serde::de::Error::duplicate_field("ack")); + } + response__ = map_.next_value::<::std::option::Option<_>>()?.map(sync_now_response::Response::Ack) +; + } + GeneratedField::Heartbeat => { + if response__.is_some() { + return Err(serde::de::Error::duplicate_field("heartbeat")); + } + response__ = map_.next_value::<::std::option::Option<_>>()?.map(sync_now_response::Response::Heartbeat) +; + } + GeneratedField::Done => { + if response__.is_some() { + return Err(serde::de::Error::duplicate_field("done")); + } + response__ = map_.next_value::<::std::option::Option<_>>()?.map(sync_now_response::Response::Done) +; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(SyncNowResponse { + response: response__, + }) + } + } + deserializer.deserialize_struct("runtime.SyncNowResponse", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for sync_now_response::Ack { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let len = 0; + let struct_ser = serializer.serialize_struct("runtime.SyncNowResponse.Ack", len)?; + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for sync_now_response::Ack { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + Ok(GeneratedField::__SkipField__) + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = sync_now_response::Ack; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.SyncNowResponse.Ack") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + while map_.next_key::()?.is_some() { + let _ = map_.next_value::()?; + } + Ok(sync_now_response::Ack { + }) + } + } + deserializer.deserialize_struct("runtime.SyncNowResponse.Ack", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for sync_now_response::Done { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let len = 0; + let struct_ser = serializer.serialize_struct("runtime.SyncNowResponse.Done", len)?; + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for sync_now_response::Done { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + Ok(GeneratedField::__SkipField__) + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = sync_now_response::Done; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.SyncNowResponse.Done") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + while map_.next_key::()?.is_some() { + let _ = map_.next_value::()?; + } + Ok(sync_now_response::Done { + }) + } + } + deserializer.deserialize_struct("runtime.SyncNowResponse.Done", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for sync_now_response::Heartbeat { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let len = 0; + let struct_ser = serializer.serialize_struct("runtime.SyncNowResponse.Heartbeat", len)?; + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for sync_now_response::Heartbeat { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + Ok(GeneratedField::__SkipField__) + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = sync_now_response::Heartbeat; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.SyncNowResponse.Heartbeat") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + while map_.next_key::()?.is_some() { + let _ = map_.next_value::()?; + } + Ok(sync_now_response::Heartbeat { + }) + } + } + deserializer.deserialize_struct("runtime.SyncNowResponse.Heartbeat", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for Task { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/crates/proto-grpc/src/runtime.rs b/crates/proto-grpc/src/runtime.rs index dc963b71d11..f03297c64ed 100644 --- a/crates/proto-grpc/src/runtime.rs +++ b/crates/proto-grpc/src/runtime.rs @@ -1774,3 +1774,327 @@ pub mod shard_server { const NAME: &'static str = SERVICE_NAME; } } +/// Generated client implementations. +#[cfg(feature = "runtime_client")] +pub mod task_control_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::http::Uri; + use tonic::codegen::*; + /// TaskControl is a user-facing control surface for running tasks. + /// It's distinct from the Leader service because it has a different + /// authorization model: callers present ordinary gazette READ claims + /// scoped to the task's shards, not the LEAD capability held by shards. + /// + /// Only the runtime-next sidecar serves TaskControl over gRPC. Users reach it + /// through the reactor front door, which relays to that sidecar but exposes + /// itself as HTTP/NDJSON rather than gRPC — see go/runtime/task_control_http.go. + #[derive(Debug, Clone)] + pub struct TaskControlClient { + inner: tonic::client::Grpc, + } + impl TaskControlClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl TaskControlClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> TaskControlClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + >>::Error: + Into + std::marker::Send + std::marker::Sync, + { + TaskControlClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + /// SyncNow asks a materialization to immediately commit its open + /// transaction, and resolves once that transaction is fully acknowledged: + /// committed and queryable in the endpoint. It's served by the + /// runtime-next sidecar hosting the task's leader session, and relayed + /// by the reactor front door. + /// + /// The stream is exactly one Ack, then zero or more Heartbeats while the + /// caller waits, then exactly one Done, after which the stream closes. + /// SyncNow is idempotent: concurrent calls await the same commit, and a + /// caller which hangs up early has still forced the commit. There is no + /// error for "nothing to do" — a task with nothing to await, including a + /// capture or derivation, Acks and its Done follows immediately. + pub async fn sync_now( + &mut self, + request: impl tonic::IntoRequest<::proto_flow::runtime::SyncNowRequest>, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/runtime.TaskControl/SyncNow"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("runtime.TaskControl", "SyncNow")); + self.inner.server_streaming(req, path, codec).await + } + } +} +/// Generated server implementations. +#[cfg(feature = "runtime_server")] +pub mod task_control_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with TaskControlServer. + #[async_trait] + pub trait TaskControl: std::marker::Send + std::marker::Sync + 'static { + /// Server streaming response type for the SyncNow method. + type SyncNowStream: tonic::codegen::tokio_stream::Stream< + Item = std::result::Result<::proto_flow::runtime::SyncNowResponse, tonic::Status>, + > + std::marker::Send + + 'static; + /// SyncNow asks a materialization to immediately commit its open + /// transaction, and resolves once that transaction is fully acknowledged: + /// committed and queryable in the endpoint. It's served by the + /// runtime-next sidecar hosting the task's leader session, and relayed + /// by the reactor front door. + /// + /// The stream is exactly one Ack, then zero or more Heartbeats while the + /// caller waits, then exactly one Done, after which the stream closes. + /// SyncNow is idempotent: concurrent calls await the same commit, and a + /// caller which hangs up early has still forced the commit. There is no + /// error for "nothing to do" — a task with nothing to await, including a + /// capture or derivation, Acks and its Done follows immediately. + async fn sync_now( + &self, + request: tonic::Request<::proto_flow::runtime::SyncNowRequest>, + ) -> std::result::Result, tonic::Status>; + } + /// TaskControl is a user-facing control surface for running tasks. + /// It's distinct from the Leader service because it has a different + /// authorization model: callers present ordinary gazette READ claims + /// scoped to the task's shards, not the LEAD capability held by shards. + /// + /// Only the runtime-next sidecar serves TaskControl over gRPC. Users reach it + /// through the reactor front door, which relays to that sidecar but exposes + /// itself as HTTP/NDJSON rather than gRPC — see go/runtime/task_control_http.go. + #[derive(Debug)] + pub struct TaskControlServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl TaskControlServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for TaskControlServer + where + T: TaskControl, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/runtime.TaskControl/SyncNow" => { + #[allow(non_camel_case_types)] + struct SyncNowSvc(pub Arc); + impl + tonic::server::ServerStreamingService<::proto_flow::runtime::SyncNowRequest> + for SyncNowSvc + { + type Response = ::proto_flow::runtime::SyncNowResponse; + type ResponseStream = T::SyncNowStream; + type Future = + BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request<::proto_flow::runtime::SyncNowRequest>, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = + async move { ::sync_now(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = SyncNowSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers.insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }), + } + } + } + impl Clone for TaskControlServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "runtime.TaskControl"; + impl tonic::server::NamedService for TaskControlServer { + const NAME: &'static str = SERVICE_NAME; + } +} diff --git a/crates/runtime-next/Cargo.toml b/crates/runtime-next/Cargo.toml index f7e05561555..0a6f0cee50d 100644 --- a/crates/runtime-next/Cargo.toml +++ b/crates/runtime-next/Cargo.toml @@ -88,4 +88,5 @@ build = { path = "../build" } e2e-support = { path = "../e2e-support" } insta = { workspace = true } quickcheck = { workspace = true } +rustls = { version = "0.23", features = ["aws_lc_rs"] } tokio = { workspace = true, features = ["test-util", "macros"] } diff --git a/crates/runtime-next/README.md b/crates/runtime-next/README.md index 0c677ea77e8..338a4acd081 100644 --- a/crates/runtime-next/README.md +++ b/crates/runtime-next/README.md @@ -94,6 +94,7 @@ src/ │ ├── fsm.rs # pipelined HeadFSM / TailFSM state machines │ ├── actor.rs # event loop driving open / commit / acknowledge / trigger │ ├── triggers.rs # webhook trigger delivery +│ ├── sync_now.rs # TaskControl.SyncNow decision evaluation │ ├── sync_schedule.rs # compiled sync-schedule evaluator (commit pacing) │ └── task.rs # Task: the leader's data model │ @@ -388,6 +389,28 @@ ceilings still force early commits — a backfill drains under memory pressure with no caught-up detection — and `CloseNow` bypasses a hold, so spec updates restart promptly. The first transaction of a leader session is never held. +## Sync-now (materialize) + +`TaskControl.SyncNow` is a user-facing RPC forcing an immediate commit of a +materialization's open transaction — collapsing any sync-schedule hold — and +resolving once that transaction is fully acknowledged (committed and +queryable in the destination). The tonic service lives beside `Leader` in +`leader/service.rs` but is registered separately, because callers present +ordinary gazette READ claims over the task's shards rather than LEAD. + +The service delivers the request to the task's live leader session via a +per-session `SyncNowHandle` registered in `ServiceImpl`. The Actor evaluates +a pure decision function (`sync_now::evaluate` over `fsm::sync_now_inputs`), +acks it, arms the FSM's existing `close_requested` input when an open +transaction can still be told to close, and parks the caller as a waiter on +a monotonic count of `Tail::Done` transitions — so N concurrent requests +await the same commit. Parked waiters receive ~15s heartbeats, then Done; a +session that exits with parked waiters errors them out (sync-now is +idempotent — callers re-invoke). The response messages are structural and +carry no payload: transaction statistics belong to the stats journal. +Derivations and captures are answered by the reactor front door; tasks +without a live session are `NOT_FOUND`. + ## Status - `leader::materialize` / `shard::materialize` and `leader::derive` / diff --git a/crates/runtime-next/src/leader/materialize/actor.rs b/crates/runtime-next/src/leader/materialize/actor.rs index 241788103d7..4c2bc749870 100644 --- a/crates/runtime-next/src/leader/materialize/actor.rs +++ b/crates/runtime-next/src/leader/materialize/actor.rs @@ -1,5 +1,6 @@ use super::{Task, fsm}; use crate::proto; +use crate::proto::sync_now_response; use anyhow::Context; use bytes::Bytes; use futures::stream::{BoxStream, FuturesUnordered}; @@ -37,6 +38,13 @@ pub struct Actor { shard_tx: Vec>>, // Future for an in-flight stats flush, if any, yielding ACK intents. stats_write_fut: Option)>>>, + // Clock of the last sync-now Progress heartbeat broadcast. + sync_now_heartbeat_at: uuid::Clock, + // Parked sync-now waiters, resolved as `tail_done_count` reaches their targets. + sync_now_waiters: Vec, + // Monotonic count of Tail arrivals at Done within this session: + // the anchor against which sync-now waiter targets are expressed. + tail_done_count: u64, // Task being executed by this actor. task: Task, // Leader-lifetime trigger debounce accumulator and last-fire times. @@ -45,6 +53,17 @@ pub struct Actor { trigger_fut: Option>>, } +/// A parked `TaskControl.SyncNow` caller, resolved when the actor's count of +/// Tail::Done transitions reaches its target. +struct SyncNowWaiter { + target: u64, + reply_tx: mpsc::UnboundedSender>, +} + +/// Cadence of heartbeats to parked sync-now waiters. Coarse: beats keep +/// hour-long streams alive through LB idle timeouts. +const SYNC_NOW_HEARTBEAT: Duration = Duration::from_secs(15); + impl Actor { pub fn new( backfill_begin: BTreeMap, @@ -69,6 +88,9 @@ impl Actor { pending_ack_intents: BTreeMap::new(), shard_tx, stats_write_fut: None, + sync_now_heartbeat_at: uuid::Clock::zero(), + sync_now_waiters: Vec::new(), + tail_done_count: 0, task, trigger_debounce: fsm::TriggerDebounce::default(), trigger_fut: None, @@ -77,11 +99,47 @@ impl Actor { #[tracing::instrument(level = "debug", err(Debug, level = "warn"), skip_all)] pub async fn serve( + &mut self, + head: fsm::Head, + tail: fsm::Tail, + session: S, + shard_rx: Vec>>, + mut sync_now_rx: mpsc::UnboundedReceiver, + ) -> anyhow::Result<()> { + let result = self + .serve_inner(head, tail, session, shard_rx, &mut sync_now_rx) + .await; + + // Parked waiters (and undelivered requests) cannot resolve once this + // session ends: the Tail::Done they await happens — if ever — in a + // future session. Error them out; sync-now is idempotent and callers + // re-invoke to await the next session. + // + // Close first, so that a request racing this drain fails to send and + // is answered Unavailable by the service, rather than landing in a + // channel nobody will read again. + sync_now_rx.close(); + + let status = tonic::Status::unavailable( + "leader session ended before the awaited transaction was fully acknowledged; retry", + ); + while let Ok(request) = sync_now_rx.try_recv() { + let _ = request.reply_tx.send(Err(status.clone())); + } + for waiter in self.sync_now_waiters.drain(..) { + let _ = waiter.reply_tx.send(Err(status.clone())); + } + + result + } + + async fn serve_inner( &mut self, mut head: fsm::Head, mut tail: fsm::Tail, mut session: S, shard_rx: Vec>>, + sync_now_rx: &mut mpsc::UnboundedReceiver, ) -> anyhow::Result<()> { service_kit::event!( tracing::Level::INFO, @@ -152,6 +210,7 @@ impl Actor { let mut action: fsm::Action; let prev_kind = tail.kind(); + let prev_resolves = !matches!(tail, fsm::Tail::Done(_) | fsm::Tail::Begin(_)); (action, tail) = tail.step( &self.trigger_debounce, self.intents_write_fut.is_none(), @@ -171,6 +230,15 @@ impl Actor { "transition", ); } + // Tail arriving at Done is the "committed and queryable" instant + // that sync-now waiters await. The Begin→Done stopping shortcut is + // excluded: it defers connector acknowledgement to a future + // session rather than completing it, so its waiters must not + // resolve (they instead error when this session exits). + if prev_resolves && matches!(tail, fsm::Tail::Done(_)) { + self.tail_done_count += 1; + self.resolve_sync_now_waiters(); + } self.merge_backfill_clocks(&mut action); let tail_wake_after = self.dispatch(action)?; @@ -229,6 +297,7 @@ impl Actor { } }; let wake_after = std::cmp::min(head_wake_after, tail_wake_after); + let wake_after = self.sync_now_heartbeat(now, wake_after); // If `head` and `tail` are awaiting IO and `ready_shard_rx` was not // consumed by either, then it was unexpected and is a protocol error. @@ -303,6 +372,14 @@ impl Actor { } shard_rx.push(next_shard_rx((shard_index, rx))); } + // Receive a sync-now request for this task. + Some(request) = sync_now_rx.recv() => { + // Resync before parking a waiter on this clock: `now` + // predates the park, which for a held transaction can be + // hours long. + now.update(now_clock()); + self.on_sync_now(request, &head, &tail, &mut close_requested, now); + } // Receive a requested NextCheckpoint frontier. result = session.recv_checkpoint(), if checkpoint_requested => { let frontier = result?; @@ -665,6 +742,97 @@ impl Actor { Ok(Some(msg)) } + /// Receive a sync-now request: acknowledge it, arm `close_requested` when + /// the decision calls for it, and either resolve immediately (nothing to + /// await) or park a waiter on its target count of Tail::Done transitions. + /// Concurrent requests over the same state share a target and resolve + /// together. + fn on_sync_now( + &mut self, + request: super::SyncNow, + head: &fsm::Head, + tail: &fsm::Tail, + close_requested: &mut bool, + now: uuid::Clock, + ) { + let decision = super::sync_now::evaluate(fsm::sync_now_inputs(head, tail)); + + service_kit::event!( + tracing::Level::INFO, + "leader", + set_close_requested = decision.set_close_requested, + await_dones = decision.await_dones, + "received sync-now request", + ); + + if decision.set_close_requested { + *close_requested = true; + } + let _ = request.reply_tx.send(Ok(proto::SyncNowResponse { + response: Some(sync_now_response::Response::Ack(sync_now_response::Ack {})), + })); + + if decision.await_dones == 0 { + let _ = request.reply_tx.send(Ok(proto::SyncNowResponse { + response: Some(sync_now_response::Response::Done( + sync_now_response::Done {}, + )), + })); + return; + } + + if self.sync_now_waiters.is_empty() { + self.sync_now_heartbeat_at = now; + } + self.sync_now_waiters.push(SyncNowWaiter { + target: self.tail_done_count + decision.await_dones, + reply_tx: request.reply_tx, + }); + } + + /// Resolve waiters whose target Tail::Done count has been reached. + fn resolve_sync_now_waiters(&mut self) { + let count = self.tail_done_count; + self.sync_now_waiters.retain(|waiter| { + if waiter.target > count { + return true; + } + let _ = waiter.reply_tx.send(Ok(proto::SyncNowResponse { + response: Some(sync_now_response::Response::Done( + sync_now_response::Done {}, + )), + })); + false + }); + } + + /// While sync-now waiters are parked, emit a periodic heartbeat to each + /// and bound the actor's sleep so the next beat isn't overslept. + /// A waiter that hung up is dropped silently: its request already landed. + fn sync_now_heartbeat(&mut self, now: uuid::Clock, wake_after: Duration) -> Duration { + if self.sync_now_waiters.is_empty() { + return wake_after; + } + let elapsed = uuid::Clock::delta(now, self.sync_now_heartbeat_at); + if elapsed < SYNC_NOW_HEARTBEAT { + return wake_after.min(SYNC_NOW_HEARTBEAT - elapsed); + } + + self.sync_now_waiters.retain(|waiter| { + waiter + .reply_tx + .send(Ok(proto::SyncNowResponse { + response: Some(sync_now_response::Response::Heartbeat( + sync_now_response::Heartbeat {}, + )), + })) + .is_ok() + }); + self.sync_now_heartbeat_at = now; + + wake_after.min(SYNC_NOW_HEARTBEAT) + } + /// Synchronously fan out a single leader message to every shard. fn broadcast(&self, msg: proto::Materialize) { let (head, tail) = self.shard_tx.split_first().unwrap(); diff --git a/crates/runtime-next/src/leader/materialize/fsm.rs b/crates/runtime-next/src/leader/materialize/fsm.rs index 1a5d840c5e3..0902d7e0992 100644 --- a/crates/runtime-next/src/leader/materialize/fsm.rs +++ b/crates/runtime-next/src/leader/materialize/fsm.rs @@ -1248,6 +1248,29 @@ fn compute_open_duration( hold..hold } +/// Gather the POD inputs of a sync-now decision from live FSM state. +/// Lives here (rather than with `sync_now::evaluate`) because it reads +/// private `Extents` fields. +pub(crate) fn sync_now_inputs(head: &Head, tail: &Tail) -> super::sync_now::Inputs { + let (head_open, head_deciding) = match head { + Head::Idle(s) => { + let is_open = s.extents.open != uuid::Clock::zero(); + (is_open, is_open) + } + Head::Extend(_) => (true, true), + Head::Stop => (false, false), + // Flush / Persist / Store / WriteStats / StartCommit: the close + // decision is behind us and the transaction is already closing. + _ => (true, false), + }; + + super::sync_now::Inputs { + head_open, + head_deciding, + tail_done: matches!(tail, Tail::Done(_)), + } +} + /// Leader-lifetime debounce state for materialization triggers. Accumulates /// per-transaction windows and gates firing to at most once per the task's /// configured trigger `interval`. diff --git a/crates/runtime-next/src/leader/materialize/handler.rs b/crates/runtime-next/src/leader/materialize/handler.rs index 85c70476e65..8fae7b8ef4d 100644 --- a/crates/runtime-next/src/leader/materialize/handler.rs +++ b/crates/runtime-next/src/leader/materialize/handler.rs @@ -130,6 +130,15 @@ where handler.set_phase("starting"); let metrics = super::Metrics::new(&slots[0].join.shards[0].id); + // Accept TaskControl.SyncNow requests for the life of this session; + // the guard un-registers on any exit from this scope. + let (sync_now_tx, sync_now_rx) = mpsc::unbounded_channel(); + let _sync_now_guard = service.register_sync_now_handle( + &task_name, + slots[0].join.shards[0].id.clone(), + sync_now_tx, + ); + service_kit::event!( tracing::Level::INFO, "leader", @@ -252,7 +261,9 @@ where task, ); handler.set_phase("running"); - actor.serve(head, tail, session, shard_rx).await + actor + .serve(head, tail, session, shard_rx, sync_now_rx) + .await } .await; diff --git a/crates/runtime-next/src/leader/materialize/mod.rs b/crates/runtime-next/src/leader/materialize/mod.rs index 87a0d168191..00e94eef150 100644 --- a/crates/runtime-next/src/leader/materialize/mod.rs +++ b/crates/runtime-next/src/leader/materialize/mod.rs @@ -2,6 +2,7 @@ mod actor; mod fsm; mod handler; mod startup; +pub(crate) mod sync_now; mod sync_schedule; mod task; mod triggers; @@ -9,6 +10,7 @@ mod triggers; use super::close_policy; pub(crate) use handler::serve; +pub(crate) use sync_now::SyncNow; #[derive(Clone)] pub(crate) struct Metrics { diff --git a/crates/runtime-next/src/leader/materialize/sync_now.rs b/crates/runtime-next/src/leader/materialize/sync_now.rs new file mode 100644 index 00000000000..6bed4a28d5e --- /dev/null +++ b/crates/runtime-next/src/leader/materialize/sync_now.rs @@ -0,0 +1,140 @@ +//! Sync-now evaluation: the pure decision at the heart of +//! `TaskControl.SyncNow`. The tonic service (`leader/service.rs`) delivers a +//! [`SyncNow`] request to the task's leader Actor, which gathers [`Inputs`] +//! from its live FSM state (`fsm::sync_now_inputs`) and applies [`evaluate`] +//! to decide what the caller's stream then awaits. + +use crate::proto; +use tokio::sync::mpsc; + +/// A sync-now request delivered to a materialize leader Actor, carrying the +/// reply channel that feeds the caller's SyncNow response stream. +pub(crate) struct SyncNow { + pub reply_tx: mpsc::UnboundedSender>, +} + +/// POD inputs of a sync-now decision. +#[derive(Debug, Copy, Clone)] +pub(crate) struct Inputs { + /// An in-flight transaction exists: opened, and its commit not yet fully + /// persisted. + pub head_open: bool, + /// The close decision is still ahead (Head is Idle or Extend), so setting + /// `close_requested` can shorten the transaction. + pub head_deciding: bool, + /// Tail is Done: the prior transaction is fully acknowledged. + pub tail_done: bool, +} + +/// What a sync-now request decided: whether to arm `close_requested`, and how +/// many future `Tail::Done` transitions the caller's stream awaits before Done +/// (zero resolves immediately). +#[derive(Debug, Copy, Clone, PartialEq)] +pub(crate) struct Decision { + pub set_close_requested: bool, + pub await_dones: u64, +} + +/// Decide a sync-now request from POD state. +/// +/// One wait rule covers every case: the caller awaits one `Tail::Done` per +/// pipeline stage that's ahead of it — the in-flight Head transaction (if any) +/// and the draining Tail transaction (if not already Done). A task which is +/// fully current awaits nothing and resolves immediately. +pub(crate) fn evaluate(inputs: Inputs) -> Decision { + let Inputs { + head_open, + head_deciding, + tail_done, + } = inputs; + + Decision { + // Arm `close_requested` only for an open transaction whose close + // decision is still ahead. Past that point the transaction is already + // closing and arming it would be harmless but pointless. + set_close_requested: head_open && head_deciding, + await_dones: head_open as u64 + !tail_done as u64, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Table-driven coverage of `evaluate`, in the style of + /// `close_policy_table`: every row of the design mapping, plus the + /// pipelined Head-open-while-Tail-drains cases. + #[test] + fn decision_table() { + struct Case { + name: &'static str, + inputs: Inputs, + want: Decision, + } + let mk = |head_open, head_deciding, tail_done| Inputs { + head_open, + head_deciding, + tail_done, + }; + + let cases = [ + Case { + name: "open transaction before its close decision: shorten it and wait", + inputs: mk(true, true, true), + want: Decision { + set_close_requested: true, + await_dones: 1, + }, + }, + Case { + name: "pipelined: Head extends while Tail drains, await both", + inputs: mk(true, true, false), + want: Decision { + set_close_requested: true, + await_dones: 2, + }, + }, + Case { + name: "Head past the close decision (flushing / committing): don't arm close", + inputs: mk(true, false, true), + want: Decision { + set_close_requested: false, + await_dones: 1, + }, + }, + Case { + name: "Head committing while Tail still drains: await both", + inputs: mk(true, false, false), + want: Decision { + set_close_requested: false, + await_dones: 2, + }, + }, + Case { + name: "no open transaction, Tail draining acknowledgement: wait, don't arm close", + inputs: mk(false, false, false), + want: Decision { + set_close_requested: false, + await_dones: 1, + }, + }, + Case { + name: "no open transaction, Tail done: fully current, nothing to await", + inputs: mk(false, false, true), + want: Decision { + set_close_requested: false, + await_dones: 0, + }, + }, + ]; + + for case in cases { + let got = evaluate(case.inputs); + assert_eq!( + got, case.want, + "case `{}` failed: inputs={:?}", + case.name, case.inputs, + ); + } + } +} diff --git a/crates/runtime-next/src/leader/service.rs b/crates/runtime-next/src/leader/service.rs index 81ffb335191..5918c781a34 100644 --- a/crates/runtime-next/src/leader/service.rs +++ b/crates/runtime-next/src/leader/service.rs @@ -42,6 +42,18 @@ pub struct ServiceImpl< pub(crate) registry: service_kit::Registry, /// When true, disarm AuthN+AuthZ enforcement (trusted local contexts only). pub(crate) disarm_auth: bool, + /// Sync-now handles of live Materialize leader sessions, keyed by task + /// name: the delivery points for TaskControl.SyncNow requests. + pub(crate) sync_now_handles: std::sync::Mutex>, +} + +/// A live Materialize session's sync-now delivery handle. +pub(crate) struct SyncNowHandle { + /// Shard-zero ID of the session: the concrete scope that a TaskControl + /// caller's claims must authorize. + shard_zero: String, + /// SyncNow delivery channel into the session's Actor. + sync_now_tx: mpsc::UnboundedSender, } impl @@ -63,6 +75,7 @@ impl proto_grpc::runtime::task_control_server::TaskControlServer { + proto_grpc::runtime::task_control_server::TaskControlServer::new(self) + .max_decoding_message_size(crate::MAX_MESSAGE_SIZE) + .max_encoding_message_size(usize::MAX) + } + + /// Register a live Materialize session's sync-now handle, replacing any + /// prior registration for the task. The returned guard un-registers it + /// when dropped — tie the guard to the session's serve scope. + pub(crate) fn register_sync_now_handle( + &self, + task_name: &str, + shard_zero: String, + sync_now_tx: mpsc::UnboundedSender, + ) -> SyncNowGuard { + self.sync_now_handles.lock().unwrap().insert( + task_name.to_string(), + SyncNowHandle { + shard_zero, + sync_now_tx: sync_now_tx.clone(), + }, + ); + SyncNowGuard { + service: self.clone(), + task_name: task_name.to_string(), + sync_now_tx, + } + } + pub fn spawn_derive( &self, authz: proto_grpc::Authorizer, @@ -160,3 +207,88 @@ impl { + service: Service, + task_name: String, + sync_now_tx: mpsc::UnboundedSender, +} + +impl Drop + for SyncNowGuard +{ + fn drop(&mut self) { + let mut guard = self.service.sync_now_handles.lock().unwrap(); + // Remove only our own registration: a replacement session for the + // same task may have re-registered while this one wound down. + if guard + .get(&self.task_name) + .is_some_and(|handle| handle.sync_now_tx.same_channel(&self.sync_now_tx)) + { + guard.remove(&self.task_name); + } + } +} + +#[tonic::async_trait] +impl + proto_grpc::runtime::task_control_server::TaskControl for Service +{ + type SyncNowStream = + tokio_stream::wrappers::UnboundedReceiverStream>; + + async fn sync_now( + &self, + mut request: tonic::Request, + ) -> tonic::Result> { + let authz = proto_grpc::Authorizer::from_request(&mut request, self.disarm_auth)?; + let proto::SyncNowRequest { task_name } = request.into_inner(); + + if task_name.is_empty() { + return Err(tonic::Status::invalid_argument("task_name is required")); + } + + // A live Materialize session? Authorize the caller against its + // concrete shard-zero ID and deliver the request; the Actor drives + // the response stream from there. + let handle = { + let guard = self.sync_now_handles.lock().unwrap(); + guard + .get(&task_name) + .map(|handle| (handle.shard_zero.clone(), handle.sync_now_tx.clone())) + }; + let Some((shard_zero, sync_now_tx)) = handle else { + // Not running here, not on the V2 runtime, or not a + // materialization at all: only the reactor front door can tell + // those apart, since only it has the shard keyspace. It answers + // for captures and derivations without dialing us. + return Err(not_found(&task_name)); + }; + let _authorized = authz.authorize_id(&shard_zero)?; + + let (reply_tx, reply_rx) = mpsc::unbounded_channel(); + if sync_now_tx + .send(super::materialize::SyncNow { reply_tx }) + .is_ok() + { + return Ok(tonic::Response::new( + tokio_stream::wrappers::UnboundedReceiverStream::new(reply_rx), + )); + } + // The session exited between lookup and delivery. + Err(tonic::Status::unavailable(format!( + "leader session of task {task_name} is winding down; retry" + ))) + } +} + +fn not_found(task_name: &str) -> tonic::Status { + tonic::Status::not_found(format!( + "task {task_name} has no live leader session here (it may not be running, or may not be on the V2 runtime)" + )) +} diff --git a/crates/runtime-next/tests/sync_now_e2e.flow.yaml b/crates/runtime-next/tests/sync_now_e2e.flow.yaml new file mode 100644 index 00000000000..1f444273b1a --- /dev/null +++ b/crates/runtime-next/tests/sync_now_e2e.flow.yaml @@ -0,0 +1,37 @@ +collections: + testing/source: + schema: + type: object + required: [id] + properties: + id: { type: string } + payload: { type: string } + key: [/id] + +materializations: + # Schedule-paced task: commits land on a 2h grid, so an open transaction + # (after the session's first, which is never schedule-held) is genuinely + # held until a SyncNow collapses the hold. + testing/sync-now: + endpoint: + connector: + image: test/image + config: {} + syncSchedule: { baseInterval: 2h } + shards: { flags: { enable-runtime-v2: "true" } } + bindings: + - source: testing/source + resource: { table: sync_now } + + # No schedule; a 300s minimum-transaction-duration floor holds the open + # transaction instead, exercising the CLOSE_REQUESTED outcome (which + # bypasses the floor). + testing/sync-now-floor: + endpoint: + connector: + image: test/image + config: {} + shards: { minTxnDuration: 300s, flags: { enable-runtime-v2: "true" } } + bindings: + - source: testing/source + resource: { table: sync_now_floor } diff --git a/crates/runtime-next/tests/sync_now_e2e.rs b/crates/runtime-next/tests/sync_now_e2e.rs new file mode 100644 index 00000000000..1b2723c08e4 --- /dev/null +++ b/crates/runtime-next/tests/sync_now_e2e.rs @@ -0,0 +1,956 @@ +//! End-to-end integration test of `TaskControl.SyncNow`: a real tonic server +//! hosting the `Leader` and `TaskControl` services behind armed AuthN +//! interceptors, a real materialize leader session, and real journal IO for +//! stats and ACK-intent writes — the IO that gates `Tail::Done`, the instant +//! sync-now waiters resolve at. +//! +//! The test plays the shard side itself: it acts as shard zero of a +//! single-shard topology, speaking the `Materialize` session protocol over +//! the Leader service (Join → Task → Recover → Apply → Open, then the +//! Load / Flush / Store / StartCommit / Acknowledge transaction flow) with a +//! scripted "connector-side" that acknowledges commits only when the test +//! says so. Withholding `Acknowledged` is the test's control over the moment +//! a transaction becomes fully acknowledged. +//! +//! Source checkpoints come from a fixture `ShuffleSessionFactory` fed by the +//! test — one synthetic `shuffle::Frontier` per transaction — so no journals +//! are read. Multi-shard session mechanics are deliberately NOT exercised: a +//! single-shard Join is immediate consensus, and sync-now waiter semantics +//! don't depend on shard count. +//! +//! Spawns real `etcd` (on PATH) and `~/go/bin/gazette` child processes via +//! `e2e_support::DataPlane`, exactly like the `shuffle` scenario tests that +//! run under `ci:nextest-run`. + +use prost::Message; +use proto_flow::flow; +use proto_gazette::{broker, uuid}; +use runtime_next::proto; +use runtime_next::proto::sync_now_response; +use std::time::Duration; +use tokio::sync::mpsc; + +/// The schedule-paced fixture task: its open transactions (after the +/// session's first) are held for up to the 2h `baseInterval`. +const SCHEDULED_TASK: &str = "testing/sync-now"; +/// The fixture task held by a 300s minimum-transaction-duration floor. +const FLOOR_TASK: &str = "testing/sync-now-floor"; + +/// Ops-stats journal that leader commits publish stats and ACK intents to. +/// Pre-created by the harness; carried in the Join's `stats_journal` labeling. +const OPS_STATS_JOURNAL: &str = "testing/ops/stats"; + +/// Source documents reported by each scripted `Loaded`. +const DOCS_PER_TXN: u64 = 3; + +/// Bound on every await of the driver and of sync-now streams. Also the +/// promptness assertion: a schedule-collapsed or floor-bypassed commit must +/// complete well within it (the alternative was a 2h hold / 300s floor). +const EXPECT_TIMEOUT: Duration = Duration::from_secs(30); + +/// Window over which a held transaction must leave the leader silent: absent +/// a poke, the shard is asked for nothing until the hold expires. +const HELD_QUIET_WINDOW: Duration = Duration::from_millis(500); + +const ISSUER: &str = "sync-now.test"; +const SECRET: &[u8] = b"sync-now-e2e-secret"; + +fn shard_zero_id(task_name: &str) -> String { + format!("materialize/{task_name}/0011223344556677/00000000-00000000") +} + +/// A fixture [`runtime_next::ShuffleSessionFactory`]: sessions relay the +/// Frontiers pushed by the test, reading no journals. The single receiver is +/// shared behind a mutex so a session owns it for its lifetime. +struct FixtureShuffleFactory { + frontier_rx: std::sync::Arc>>, +} + +impl runtime_next::ShuffleSessionFactory for FixtureShuffleFactory { + type Session = FixtureFrontiers; + + async fn open( + &self, + _task: shuffle::proto::Task, + _shards: Vec, + _resume: shuffle::Frontier, + ) -> anyhow::Result { + Ok(FixtureFrontiers { + frontier_rx: self.frontier_rx.clone().lock_owned().await, + }) + } +} + +/// A fixture [`runtime_next::ShuffleSession`]: yields one queued Frontier per +/// checkpoint request, and parks forever once the channel is idle or closed +/// (the leader keeps exactly one request in flight; an unanswered request is +/// simply an idle task). +struct FixtureFrontiers { + frontier_rx: tokio::sync::OwnedMutexGuard>, +} + +impl runtime_next::ShuffleSession for FixtureFrontiers { + fn request_checkpoint(&self) { + // No request protocol: `recv_checkpoint` pops the next queued frontier. + } + + async fn recv_checkpoint(&mut self) -> anyhow::Result { + match self.frontier_rx.recv().await { + Some(frontier) => Ok(frontier), + None => std::future::pending().await, + } + } + + async fn close(self) -> anyhow::Result<()> { + Ok(()) + } +} + +/// Synthetic checkpoint Frontier for the `seq`-th transaction: binding zero, +/// one producer whose committed Clock strictly increases and whose negative +/// `offset` (a closed span) grows in magnitude. `flushed_lsn` carries one +/// entry for the single shard; the scripted shard never reads segments, so +/// its value is inert. +fn synthetic_frontier(seq: u64) -> shuffle::Frontier { + shuffle::Frontier::new( + vec![shuffle::JournalFrontier { + journal: "testing/source/pivot=00".into(), + binding: 0, + producers: vec![shuffle::ProducerFrontier { + producer: uuid::Producer::from_bytes([7, 19, 83, 3, 3, 17]), + last_commit: uuid::Clock::from_unix(1_000_000 + seq, 0), + hinted_commit: uuid::Clock::from_u64(0), + offset: -((seq * 1_000) as i64), + }], + bytes_read_delta: (DOCS_PER_TXN * 100) as i64, + bytes_behind_delta: 0, + }], + vec![0], + ) + .expect("synthetic frontier is well-formed") +} + +type SyncNowStream = tonic::Streaming; + +/// One test's hermetic world: a DataPlane, the built fixture catalog, and an +/// in-process tonic server hosting `Leader` + `TaskControl` behind armed +/// authentication interceptors — assembled exactly as the runtime sidecar +/// serves them. +struct Harness { + data_plane: e2e_support::DataPlane, + endpoint: String, + server_task: tokio::task::JoinHandle>, + signer: proto_grpc::Signer, + frontier_tx: mpsc::UnboundedSender, + /// Built `MaterializationSpec` bytes, keyed by task name. + specs: std::collections::BTreeMap, + /// Monotonic ordinal of pushed frontiers, for strictly-increasing clocks. + txn_seq: std::sync::atomic::AtomicU64, +} + +impl Harness { + async fn start() -> Harness { + // The tonic/reqwest TLS stack requires a process-level provider, as + // installed by every production `main` (e.g. runtime-sidecar's). + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let data_plane = + e2e_support::DataPlane::start(e2e_support::DataPlaneArgs { broker_count: 1 }) + .await + .expect("DataPlane start"); + + let source = build::arg_source_to_url("./tests/sync_now_e2e.flow.yaml", false).unwrap(); + let build::Output { built, .. } = build::for_local_test(&source, true) + .await + .into_result() + .expect("fixture build"); + + let specs: std::collections::BTreeMap = built + .built_materializations + .iter() + .map(|row| { + let spec = row.spec.as_ref().expect("built materialization has a spec"); + (spec.name.clone(), spec.encode_to_vec().into()) + }) + .collect(); + + // Pre-create the ops-stats journal: commits publish stats and ACK + // intents to it, and appends require the journal to exist. A clone of + // the built collection's partition template carries a valid fragment + // spec; replication drops to the single test broker. + let mut ops_spec = built + .built_collections + .get_key(&models::Collection::new("testing/source")) + .expect("built source collection") + .spec + .as_ref() + .expect("collection spec") + .partition_template + .clone() + .expect("partition template"); + ops_spec.name = OPS_STATS_JOURNAL.to_string(); + ops_spec.replication = 1; + + data_plane + .journal_client + .apply(broker::ApplyRequest { + changes: vec![broker::apply_request::Change { + expect_mod_revision: 0, // Created by this Apply. + upsert: Some(ops_spec), + delete: String::new(), + }], + }) + .await + .expect("creating ops stats journal"); + + let (frontier_tx, frontier_rx) = mpsc::unbounded_channel(); + let shuffle_factory = FixtureShuffleFactory { + frontier_rx: std::sync::Arc::new(tokio::sync::Mutex::new(frontier_rx)), + }; + + let journal_client = data_plane.journal_client.clone(); + let publisher_factory = runtime_next::JournalPublisherFactory::new(std::sync::Arc::new( + move |_authz_sub, _authz_obj| journal_client.clone(), + )); + + let svc = runtime_next::Service::new( + shuffle_factory, + publisher_factory, + runtime_next::TracingLoggerFactory, + service_kit::Registry::new(), + false, // AuthN+AuthZ stays armed. + ); + + // Serve both services behind interceptors, exactly as the sidecar + // does: Leader requires LEAD; TaskControl requires gazette READ. + let authn = proto_grpc::Authenticator::new( + ISSUER.to_string(), + vec![tokens::jwt::DecodingKey::from_secret(SECRET)], + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("binding ephemeral listener"); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + + let server_task = tokio::spawn( + tonic::transport::Server::builder() + .add_service(tonic::service::interceptor::InterceptedService::new( + svc.clone().into_tonic_service(), + authn.clone().interceptor(proto_flow::capability::LEAD), + )) + .add_service(tonic::service::interceptor::InterceptedService::new( + svc.into_task_control_service(), + authn.interceptor(proto_gazette::capability::READ), + )) + .serve_with_incoming(tokio_stream::wrappers::TcpListenerStream::new(listener)), + ); + + let signer = proto_grpc::Signer::new( + ISSUER.to_string(), + tokens::jwt::EncodingKey::from_secret(SECRET), + ); + + Harness { + data_plane, + endpoint, + server_task, + signer, + frontier_tx, + specs, + txn_seq: std::sync::atomic::AtomicU64::new(0), + } + } + + async fn stop(self) { + self.server_task.abort(); + self.data_plane + .graceful_stop() + .await + .expect("graceful stop"); + } + + fn spec_bytes(&self, task_name: &str) -> bytes::Bytes { + self.specs + .get(task_name) + .unwrap_or_else(|| panic!("fixture has no built task {task_name}")) + .clone() + } + + /// Push one synthetic checkpoint Frontier, feeding the leader's next + /// transaction. + fn push_frontier(&self) { + let seq = self + .txn_seq + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + + 1; + self.frontier_tx + .send(synthetic_frontier(seq)) + .expect("frontier channel is open"); + } + + /// Mint a gazette-READ bearer scoped to `id_prefix`, as any + /// `/authorize/user/task` Read-level caller would hold. + fn read_token(&self, id_prefix: &str) -> proto_grpc::Metadata { + self.token(proto_gazette::capability::READ, id_prefix) + } + + fn token(&self, capability: u32, id_prefix: &str) -> proto_grpc::Metadata { + let token = self + .signer + .sign( + capability, + "test-caller".to_string(), + broker::LabelSelector { + include: Some(labels::build_set([("id:prefix", id_prefix)])), + exclude: None, + }, + tokens::TimeDelta::hours(1), + ) + .expect("signing test token"); + proto_grpc::Metadata::new() + .with_bearer_token(&token) + .expect("building bearer metadata") + } + + /// Open a SyncNow stream for `task_name` bearing `metadata`. + async fn sync_now( + &self, + metadata: proto_grpc::Metadata, + task_name: &str, + ) -> tonic::Result { + let channel = gazette::dial_channel(&self.endpoint).expect("dialing test endpoint"); + let response = + proto_grpc::runtime::task_control_client::TaskControlClient::with_interceptor( + channel, metadata, + ) + .sync_now(proto::SyncNowRequest { + task_name: task_name.to_string(), + }) + .await?; + Ok(response.into_inner()) + } + + /// SyncNow with a correctly-scoped READ token, consuming the Ack and + /// returning the still-open stream. + async fn poke(&self, task_name: &str) -> SyncNowStream { + let metadata = self.read_token(&format!("materialize/{task_name}/")); + let mut stream = self + .sync_now(metadata, task_name) + .await + .expect("SyncNow call"); + expect_ack(&mut stream).await; + stream + } + + /// Drain barrier: poke and await its Done, which proves the leader reached + /// `Tail::Done` — so the next transaction starts from a fully-drained + /// leader (deterministic `await_dones`). Idempotent re-poking is exercised + /// for free. Only valid while no transaction is open: an open transaction + /// would arm `close_requested` and park us until the shard side commits. + async fn poke_until_drained(&self, task_name: &str) { + let mut stream = self.poke(task_name).await; + expect_done(&mut stream).await; + } + + /// Act as shard zero of `task_name`: dial the Leader service with a LEAD + /// bearer and run session startup through the leader's first (recovered, + /// empty) acknowledgement cycle, so the session begins fully drained. + async fn join_session(&self, task_name: &str) -> ShardDriver { + let shuffle_dir = tempfile::tempdir().expect("shuffle tempdir"); + let shard_id = shard_zero_id(task_name); + + let channel = gazette::dial_channel(&self.endpoint).expect("dialing test endpoint"); + let metadata = self + .signer + .shard_bearer(proto_flow::capability::LEAD, &shard_id) + .expect("minting LEAD bearer"); + let mut client = + proto_grpc::runtime::leader_client::LeaderClient::with_interceptor(channel, metadata); + + let (to_leader, request_rx) = mpsc::unbounded_channel(); + let from_leader = client + .materialize(tokio_stream::wrappers::UnboundedReceiverStream::new( + request_rx, + )) + .await + .expect("opening leader Materialize stream") + .into_inner(); + + let join = proto::Join { + etcd_mod_revision: 1, + shards: vec![proto::join::Shard { + id: shard_id, + labeling: Some(ops::ShardLabeling { + build: "0011223344556677".to_string(), + range: Some(flow::RangeSpec { + key_begin: 0, + key_end: u32::MAX, + r_clock_begin: 0, + r_clock_end: u32::MAX, + }), + task_name: task_name.to_string(), + task_type: ops::TaskType::Materialization as i32, + stats_journal: OPS_STATS_JOURNAL.to_string(), + ..Default::default() + }), + reactor: Some(Default::default()), + etcd_create_revision: 1, + }], + shard_index: 0, + shuffle_directory: shuffle_dir.path().to_string_lossy().into_owned(), + shuffle_endpoint: "http://shuffle.invalid".to_string(), + leader_endpoint: String::new(), + }; + + let mut driver = ShardDriver { + to_leader, + from_leader, + _shuffle_dir: shuffle_dir, + }; + + driver.send(proto::Materialize { + join: Some(join), + ..Default::default() + }); + let msg = driver.expect("Joined").await; + let joined = msg + .joined + .clone() + .unwrap_or_else(|| panic!("expected Joined, got {msg:?}")); + assert_eq!( + joined.max_etcd_revision, 0, + "single-shard Join is immediate consensus", + ); + + driver.send(proto::Materialize { + task: Some(proto::Task { + spec: self.spec_bytes(task_name), + max_transactions: 0, + sqlite_vfs_uri: String::new(), + publisher_id: bytes::Bytes::copy_from_slice( + runtime_next::new_producer().as_bytes(), + ), + }), + ..Default::default() + }); + // A fresh task: nothing recovered from RocksDB. + driver.send(proto::Materialize { + recover: Some(proto::Recover::default()), + ..Default::default() + }); + + // Apply loop: no connector patches. The leader then persists + // `last_applied` (echoed transparently by `expect`) and Opens. + let msg = driver.expect("Apply").await; + assert!(msg.apply.is_some(), "expected Apply, got {msg:?}"); + driver.send(proto::Materialize { + applied: Some(proto::Applied { + action_description: String::new(), + connector_patches_json: bytes::Bytes::new(), + }), + ..Default::default() + }); + + // An empty connector checkpoint makes startup reconciliation a no-op + // fixed point: no rescan Persists. + let msg = driver.expect("Open").await; + assert!(msg.open.is_some(), "expected Open, got {msg:?}"); + driver.send(proto::Materialize { + opened: Some(proto::materialize::Opened { + container: None, + connector_checkpoint: None, + }), + ..Default::default() + }); + + // The actor's first act is acknowledging the recovered (empty) prior + // transaction; answer it so the session starts fully drained. + driver.expect_acknowledge().await; + driver.send_acknowledged(); + + driver + } +} + +/// The scripted shard side of one Leader.Materialize session. +struct ShardDriver { + to_leader: mpsc::UnboundedSender, + from_leader: tonic::Streaming, + _shuffle_dir: tempfile::TempDir, +} + +impl ShardDriver { + fn send(&self, msg: proto::Materialize) { + self.to_leader + .send(msg) + .expect("leader request stream is open"); + } + + /// Receive the next leader message, transparently echoing `Persisted` for + /// any `Persist`: the leader persists at multiple FSM points (hint before + /// Store, commit after StartedCommit, legacy-checkpoint fields included), + /// and answering them generically keeps this driver robust to + /// FSM-internal ordering. + async fn expect(&mut self, awaiting: &'static str) -> proto::Materialize { + loop { + let msg = tokio::time::timeout(EXPECT_TIMEOUT, self.from_leader.message()) + .await + .unwrap_or_else(|_| panic!("timed out awaiting {awaiting}")) + .unwrap_or_else(|err| panic!("leader stream error awaiting {awaiting}: {err}")) + .unwrap_or_else(|| panic!("unexpected leader EOF awaiting {awaiting}")); + + if let Some(persist) = &msg.persist { + assert!( + !persist.rescan, + "unexpected rescan Persist awaiting {awaiting}" + ); + self.send(proto::Materialize { + persisted: Some(proto::Persisted { + seq_no: persist.seq_no, + }), + ..Default::default() + }); + continue; + } + return msg; + } + } + + async fn expect_load(&mut self) { + let msg = self.expect("Load").await; + assert!(msg.load.is_some(), "expected Load, got {msg:?}"); + } + + async fn expect_acknowledge(&mut self) { + let msg = self.expect("Acknowledge").await; + assert!( + msg.acknowledge.is_some(), + "expected Acknowledge, got {msg:?}" + ); + } + + /// Assert the leader sends nothing further within `window`. + async fn assert_quiet(&mut self, window: Duration) { + match tokio::time::timeout(window, self.from_leader.message()).await { + Err(_) => (), + Ok(msg) => panic!("unexpected leader message: {msg:?}"), + } + } + + fn send_loaded(&self, sourced_docs: u64) { + self.send(proto::Materialize { + loaded: Some(proto::materialize::Loaded { + bindings: vec![proto::materialize::loaded::Binding { + index: 0, + min_source_clock: 0, + max_source_clock: 0, + sourced_docs_total: sourced_docs, + sourced_bytes_total: sourced_docs * 100, + max_key_delta: bytes::Bytes::new(), + }], + combiner_usage_bytes: 1 << 10, + }), + ..Default::default() + }); + } + + fn send_acknowledged(&self) { + self.send(proto::Materialize { + acknowledged: Some(proto::materialize::Acknowledged::default()), + ..Default::default() + }); + } + + /// Drive a closing transaction from Flush through StartedCommit, and park + /// at the leader's `Acknowledge` — the caller controls when to + /// `send_acknowledged`. Completing within the expect timeout is itself + /// the promptness assertion for collapsed holds and bypassed floors. + async fn run_close_script(&mut self) { + let msg = self.expect("Flush").await; + assert!(msg.flush.is_some(), "expected Flush, got {msg:?}"); + self.send(proto::Materialize { + flushed: Some(proto::materialize::Flushed::default()), + ..Default::default() + }); + + let msg = self.expect("Store").await; + assert!(msg.store.is_some(), "expected Store, got {msg:?}"); + self.send(proto::Materialize { + stored: Some(proto::materialize::Stored::default()), + ..Default::default() + }); + + let msg = self.expect("StartCommit").await; + assert!( + msg.start_commit.is_some(), + "expected StartCommit, got {msg:?}" + ); + self.send(proto::Materialize { + started_commit: Some(proto::materialize::StartedCommit::default()), + ..Default::default() + }); + + self.expect_acknowledge().await; + } +} + +// ---- SyncNow stream helpers ---- + +async fn recv_sync_now( + stream: &mut SyncNowStream, + awaiting: &'static str, + timeout: Duration, +) -> sync_now_response::Response { + tokio::time::timeout(timeout, stream.message()) + .await + .unwrap_or_else(|_| panic!("timed out awaiting SyncNow {awaiting}")) + .unwrap_or_else(|err| panic!("SyncNow stream error awaiting {awaiting}: {err}")) + .unwrap_or_else(|| panic!("unexpected SyncNow EOF awaiting {awaiting}")) + .response + .unwrap_or_else(|| panic!("SyncNow response with empty oneof awaiting {awaiting}")) +} + +async fn expect_ack(stream: &mut SyncNowStream) { + match recv_sync_now(stream, "Ack", EXPECT_TIMEOUT).await { + sync_now_response::Response::Ack(_) => (), + other => panic!("expected SyncNow Ack, got {other:?}"), + } +} + +/// Await Done, skipping any interleaved heartbeats. +async fn expect_done(stream: &mut SyncNowStream) { + loop { + match recv_sync_now(stream, "Done", EXPECT_TIMEOUT).await { + sync_now_response::Response::Done(_) => return, + sync_now_response::Response::Heartbeat(_) => continue, + other => panic!("expected SyncNow Done, got {other:?}"), + } + } +} + +async fn expect_heartbeat(stream: &mut SyncNowStream, timeout: Duration) { + match recv_sync_now(stream, "Heartbeat", timeout).await { + sync_now_response::Response::Heartbeat(_) => (), + other => panic!("expected SyncNow Heartbeat, got {other:?}"), + } +} + +/// Assert no response arrives on `stream` within `window` (bounded peek). +async fn assert_no_response(stream: &mut SyncNowStream, window: Duration) { + match tokio::time::timeout(window, stream.message()).await { + Err(_) => (), + Ok(result) => panic!("unexpected SyncNow response while ack withheld: {result:?}"), + } +} + +// ---- Scripted transaction helpers ---- + +/// Run one full transaction to completion, then drain via the IDLE barrier. +/// Used as the warm-up before every held-transaction test: the first +/// transaction of a session is never schedule-held (`session_start`), so the +/// NEXT transaction is the one a schedule genuinely holds. +async fn warm_up_txn(harness: &Harness, driver: &mut ShardDriver, task_name: &str) { + harness.push_frontier(); + driver.expect_load().await; + driver.send_loaded(DOCS_PER_TXN); + driver.run_close_script().await; + driver.send_acknowledged(); + harness.poke_until_drained(task_name).await; +} + +/// Open a transaction that the sync schedule holds, and deliver the SyncNow +/// that collapses it, returning the caller's stream. +/// +/// The hold is proven by the leader's silence: a held transaction asks the +/// shard for nothing, because absent our poke it would sit until the +/// schedule's next fire instant hours away. Wall-clock flake: the fire +/// instants sit on a fixed grid (`jitter(seed) + k * baseInterval`) which the +/// test cannot choose "now" against, so a transaction opening milliseconds +/// before an instant commits on its own and trips `assert_quiet` (~1-in-1e5 +/// odds with the 2h interval). +async fn open_held_txn(harness: &Harness, driver: &mut ShardDriver) -> SyncNowStream { + harness.push_frontier(); + driver.expect_load().await; + driver.send_loaded(DOCS_PER_TXN); + driver.assert_quiet(HELD_QUIET_WINDOW).await; + + harness.poke(SCHEDULED_TASK).await +} + +// ---- Tests ---- + +/// A schedule-held transaction collapses on SyncNow and resolves only once +/// the shard side acknowledges; a fully-drained task then resolves at once. +#[tokio::test(flavor = "multi_thread")] +async fn held_transaction_collapses_and_resolves_on_ack() { + let harness = Harness::start().await; + let mut driver = harness.join_session(SCHEDULED_TASK).await; + warm_up_txn(&harness, &mut driver, SCHEDULED_TASK).await; + + // Transaction #2 is genuinely held (a 2h hold, absent the poke). + let mut stream = open_held_txn(&harness, &mut driver).await; + + // The hold collapsed: the commit proceeds promptly rather than waiting + // out the 2h grid, and parks at Acknowledge. + driver.run_close_script().await; + + // Done must not arrive while the acknowledgement is withheld. + assert_no_response(&mut stream, Duration::from_secs(2)).await; + + driver.send_acknowledged(); + expect_done(&mut stream).await; + + // With everything drained and no new frontier, a fresh poke has nothing + // to await and its Done follows immediately. + let mut stream = harness.poke(SCHEDULED_TASK).await; + expect_done(&mut stream).await; + driver.assert_quiet(HELD_QUIET_WINDOW).await; + + drop(driver); + harness.stop().await; +} + +/// N concurrent SyncNow calls against the same open transaction all ack and +/// all resolve at the same single commit. +#[tokio::test(flavor = "multi_thread")] +async fn concurrent_pokes_share_one_commit() { + let harness = Harness::start().await; + let mut driver = harness.join_session(SCHEDULED_TASK).await; + warm_up_txn(&harness, &mut driver, SCHEDULED_TASK).await; + + // The first poke collapses the held transaction; three more land while + // the (single) commit is in flight, each acking wherever the actor is. + let first = open_held_txn(&harness, &mut driver).await; + let mut streams = vec![first]; + for _ in 0..3 { + streams.push(harness.poke(SCHEDULED_TASK).await); + } + + driver.run_close_script().await; + + // No stream resolves while the acknowledgement is withheld. + for stream in &mut streams { + assert_no_response(stream, Duration::from_millis(500)).await; + } + + driver.send_acknowledged(); + for stream in &mut streams { + expect_done(stream).await; + } + + // One transaction total: the leader is idle and asks nothing more of the + // shard once the shared commit drains. + harness.poke_until_drained(SCHEDULED_TASK).await; + driver.assert_quiet(Duration::from_millis(500)).await; + + drop(driver); + harness.stop().await; +} + +/// A poke arriving after commit, while the connector acknowledgement still +/// drains, resolves at that same acknowledgement release rather than awaiting +/// a further transaction. +#[tokio::test(flavor = "multi_thread")] +async fn poke_in_drain_window_awaits_the_same_ack() { + let harness = Harness::start().await; + let mut driver = harness.join_session(SCHEDULED_TASK).await; + warm_up_txn(&harness, &mut driver, SCHEDULED_TASK).await; + + let mut collapse_stream = open_held_txn(&harness, &mut driver).await; + + // Drive through StartedCommit and the commit Persist: the transaction has + // committed and its Tail drains the (withheld) acknowledgement. + driver.run_close_script().await; + + let mut drain_stream = harness.poke(SCHEDULED_TASK).await; + assert_no_response(&mut drain_stream, Duration::from_millis(500)).await; + + driver.send_acknowledged(); + expect_done(&mut collapse_stream).await; + expect_done(&mut drain_stream).await; + + drop(driver); + harness.stop().await; +} + +/// A transaction held by the 300s min-duration floor (no schedule) commits +/// immediately on SyncNow — the close request bypasses the floor. The floor +/// holds even the session's first transaction (`session_start` waives only +/// schedule holds), so no warm-up is needed. +#[tokio::test(flavor = "multi_thread")] +async fn sync_now_bypasses_the_min_duration_floor() { + let harness = Harness::start().await; + let mut driver = harness.join_session(FLOOR_TASK).await; + + harness.push_frontier(); + driver.expect_load().await; + driver.send_loaded(DOCS_PER_TXN); + driver.assert_quiet(HELD_QUIET_WINDOW).await; + + let mut stream = harness.poke(FLOOR_TASK).await; + + // The commit proceeds well before the 300s floor, and Done follows the + // released acknowledgement. + driver.run_close_script().await; + driver.send_acknowledged(); + expect_done(&mut stream).await; + + drop(driver); + harness.stop().await; +} + +/// A parked waiter receives heartbeats. The cadence is the production 15s +/// `SYNC_NOW_HEARTBEAT` constant, kept un-injectable on purpose, so this is +/// deliberately the one slow test: the first beat lands one full interval +/// after the waiter parks. +#[tokio::test(flavor = "multi_thread")] +async fn parked_waiter_receives_heartbeats() { + let harness = Harness::start().await; + let mut driver = harness.join_session(FLOOR_TASK).await; + + harness.push_frontier(); + driver.expect_load().await; + driver.send_loaded(DOCS_PER_TXN); + + let mut stream = harness.poke(FLOOR_TASK).await; + + // Commit, then park with the acknowledgement withheld. + driver.run_close_script().await; + + // One beat proves the machinery; don't wait for a second. + expect_heartbeat(&mut stream, Duration::from_secs(30)).await; + + driver.send_acknowledged(); + expect_done(&mut stream).await; + + drop(driver); + harness.stop().await; +} + +/// Token scope and capability checks over the real wire, and NOT_FOUND +/// resolution from the caller's shard-scope when no live session matches. +#[tokio::test(flavor = "multi_thread")] +async fn authz_and_task_resolution() { + let harness = Harness::start().await; + let driver = harness.join_session(SCHEDULED_TASK).await; + + // A correctly-scoped READ token is accepted, and with no open transaction + // resolves immediately. + harness.poke_until_drained(SCHEDULED_TASK).await; + + // A READ token scoped to a DIFFERENT task is denied against the live + // session's concrete shard-zero ID. + let err = harness + .sync_now( + harness.read_token("materialize/testing/other/0011223344556677/"), + SCHEDULED_TASK, + ) + .await + .expect_err("differently-scoped token is rejected"); + assert_eq!(err.code(), tonic::Code::PermissionDenied, "{err}"); + + // A token minted WITHOUT the READ capability bit fails at the + // interceptor, before any Ack. + let err = harness + .sync_now( + harness.token( + proto_flow::capability::LEAD, + &format!("materialize/{SCHEDULED_TASK}/"), + ), + SCHEDULED_TASK, + ) + .await + .expect_err("token without READ is rejected"); + assert!( + matches!( + err.code(), + tonic::Code::PermissionDenied | tonic::Code::Unauthenticated + ), + "{err}", + ); + + // A materialization the token names, but which isn't running here, is + // NOT_FOUND. + let err = harness + .sync_now( + harness.read_token("materialize/testing/not-running/"), + "testing/not-running", + ) + .await + .expect_err("not-running task is NOT_FOUND"); + assert_eq!(err.code(), tonic::Code::NotFound, "{err}"); + + // A capture or derivation is NOT_FOUND here too: this service hosts only + // Materialize leader sessions. A task with nothing to sync is resolved by + // the reactor front door, which has the shard keyspace to know its type. + let err = harness + .sync_now( + harness.read_token("capture/testing/some-capture/"), + "testing/some-capture", + ) + .await + .expect_err("a capture has no leader session here"); + assert_eq!(err.code(), tonic::Code::NotFound, "{err}"); + + // An empty task_name is an invalid argument. + let err = harness + .sync_now(harness.read_token("materialize/testing/"), "") + .await + .expect_err("empty task_name is rejected"); + assert_eq!(err.code(), tonic::Code::InvalidArgument, "{err}"); + + drop(driver); + harness.stop().await; +} + +/// A session that exits with waiters parked errors them out (they can't +/// resolve: the Tail::Done they await happens, if ever, in a future session), +/// and the SyncNowGuard un-registers the handle so follow-up pokes are +/// NOT_FOUND. +#[tokio::test(flavor = "multi_thread")] +async fn session_exit_errors_parked_waiters() { + let harness = Harness::start().await; + let mut driver = harness.join_session(FLOOR_TASK).await; + + // Park a waiter: a floor-held open transaction whose close script the + // driver never answers. + harness.push_frontier(); + driver.expect_load().await; + driver.send_loaded(DOCS_PER_TXN); + + let mut stream = harness.poke(FLOOR_TASK).await; + + // Kill the session: the leader errors on the shard stream's EOF. + drop(driver); + + let err = tokio::time::timeout(EXPECT_TIMEOUT, stream.message()) + .await + .expect("waiter resolves after session exit") + .expect_err("waiter errors after session exit"); + assert_eq!(err.code(), tonic::Code::Unavailable, "{err}"); + assert!(err.message().contains("retry"), "{err}"); + + // The guard un-registered the handle; a follow-up poke is NOT_FOUND. + // Guard drop races the client's next call, so retry briefly. + let deadline = tokio::time::Instant::now() + EXPECT_TIMEOUT; + loop { + match harness + .sync_now( + harness.read_token(&format!("materialize/{FLOOR_TASK}/")), + FLOOR_TASK, + ) + .await + { + Err(status) if status.code() == tonic::Code::NotFound => break, + other => { + assert!( + tokio::time::Instant::now() < deadline, + "timed out awaiting NOT_FOUND after session exit (last: {other:?})", + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + } + + harness.stop().await; +} diff --git a/crates/runtime-sidecar/README.md b/crates/runtime-sidecar/README.md index dcd1dd8e7db..81e461fa99b 100644 --- a/crates/runtime-sidecar/README.md +++ b/crates/runtime-sidecar/README.md @@ -2,11 +2,13 @@ Production sidecar process for the runtime-v2 architecture (`plans/runtime-v2/plan.md`). One per reactor machine, supervised by -systemd, hosting two gRPC services on a fixed fleet-wide port: +systemd, hosting three gRPC services on a fixed fleet-wide port: - **Shuffle Leader** — `runtime_next::leader::Service`, the per-task Join rendezvous and HeadFSM/TailFSM coordination for tasks whose shard zero is on this machine. +- **TaskControl** — the same `leader::Service`, wrapped separately: the + user-facing SyncNow RPC against tasks led from this machine. - **Shuffle** — `shuffle::Service`, the Session/Slice/Log RPCs. ## Listeners @@ -22,12 +24,14 @@ signs outbound requests; all keys verify inbound traffic (rotation). Inbound RPCs are authenticated and authorized: every request must carry a JWT issued by `--data-plane-fqdn` and bearing the service's -capability — `LEAD` for the Leader, `SHUFFLE` for Shuffle — or it is -rejected before its handler runs. Each handler additionally enforces a -scope check: the token's selector must authorize the shard the handler -operates on (shard zero of a Leader/Session join, or the hosted shard of -a Slice/Log). Because a task's shards share an `id` prefix and the bearer -is scoped to that prefix, this gates access at task granularity. +capability — `LEAD` for the Leader, gazette `READ` for TaskControl +(the token every `/authorize/user/task` caller holds), `SHUFFLE` for +Shuffle — or it is rejected before its handler runs. Each handler +additionally enforces a scope check: the token's selector must authorize +the shard the handler operates on (shard zero of a Leader/Session join +or a TaskControl SyncNow, or the hosted shard of a Slice/Log). Because a +task's shards share an `id` prefix and the bearer is scoped to that +prefix, this gates access at task granularity. The loopback admin surface is a separate server and is intentionally not authenticated (bound only to loopback). diff --git a/crates/runtime-sidecar/src/lib.rs b/crates/runtime-sidecar/src/lib.rs index 5d2f4d021e4..dc178dd0e89 100644 --- a/crates/runtime-sidecar/src/lib.rs +++ b/crates/runtime-sidecar/src/lib.rs @@ -186,9 +186,15 @@ pub async fn run(args: Args, registry: service_kit::Registry) -> anyhow::Result< } builder .add_service(tonic::service::interceptor::InterceptedService::new( - runtime_svc.into_tonic_service(), + runtime_svc.clone().into_tonic_service(), authn.clone().interceptor(proto_flow::capability::LEAD), )) + // TaskControl callers hold ordinary gazette READ claims over the + // task's shards (the /authorize/user/task token), not LEAD. + .add_service(tonic::service::interceptor::InterceptedService::new( + runtime_svc.into_task_control_service(), + authn.clone().interceptor(proto_gazette::capability::READ), + )) .add_service(tonic::service::interceptor::InterceptedService::new( shuffle_svc.into_tonic_service(), authn.interceptor(proto_flow::capability::SHUFFLE), diff --git a/go/protocols/runtime/runtime.pb.go b/go/protocols/runtime/runtime.pb.go index 388cfa9752b..17983f2899d 100644 --- a/go/protocols/runtime/runtime.pb.go +++ b/go/protocols/runtime/runtime.pb.go @@ -3811,6 +3811,275 @@ func (m *Derive_StartedCommit) XXX_DiscardUnknown() { var xxx_messageInfo_Derive_StartedCommit proto.InternalMessageInfo +// SyncNowRequest is the request of a TaskControl.SyncNow RPC. +type SyncNowRequest struct { + // Name of the task to synchronize. + TaskName string `protobuf:"bytes,1,opt,name=task_name,json=taskName,proto3" json:"task_name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SyncNowRequest) Reset() { *m = SyncNowRequest{} } +func (m *SyncNowRequest) String() string { return proto.CompactTextString(m) } +func (*SyncNowRequest) ProtoMessage() {} +func (*SyncNowRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{32} +} +func (m *SyncNowRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SyncNowRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SyncNowRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SyncNowRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncNowRequest.Merge(m, src) +} +func (m *SyncNowRequest) XXX_Size() int { + return m.ProtoSize() +} +func (m *SyncNowRequest) XXX_DiscardUnknown() { + xxx_messageInfo_SyncNowRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncNowRequest proto.InternalMessageInfo + +// SyncNowResponse is the streamed response of a TaskControl.SyncNow RPC. +// Its messages are structural: they mark the caller's position in the +// stream and carry no payload. Statistics of the awaited transaction are +// recorded to the task's stats journal, which is where callers read them. +type SyncNowResponse struct { + // Types that are valid to be assigned to Response: + // *SyncNowResponse_Ack_ + // *SyncNowResponse_Heartbeat_ + // *SyncNowResponse_Done_ + Response isSyncNowResponse_Response `protobuf_oneof:"response"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SyncNowResponse) Reset() { *m = SyncNowResponse{} } +func (m *SyncNowResponse) String() string { return proto.CompactTextString(m) } +func (*SyncNowResponse) ProtoMessage() {} +func (*SyncNowResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{33} +} +func (m *SyncNowResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SyncNowResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SyncNowResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SyncNowResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncNowResponse.Merge(m, src) +} +func (m *SyncNowResponse) XXX_Size() int { + return m.ProtoSize() +} +func (m *SyncNowResponse) XXX_DiscardUnknown() { + xxx_messageInfo_SyncNowResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncNowResponse proto.InternalMessageInfo + +type isSyncNowResponse_Response interface { + isSyncNowResponse_Response() + MarshalTo([]byte) (int, error) + ProtoSize() int +} + +type SyncNowResponse_Ack_ struct { + Ack *SyncNowResponse_Ack `protobuf:"bytes,1,opt,name=ack,proto3,oneof" json:"ack,omitempty"` +} +type SyncNowResponse_Heartbeat_ struct { + Heartbeat *SyncNowResponse_Heartbeat `protobuf:"bytes,2,opt,name=heartbeat,proto3,oneof" json:"heartbeat,omitempty"` +} +type SyncNowResponse_Done_ struct { + Done *SyncNowResponse_Done `protobuf:"bytes,3,opt,name=done,proto3,oneof" json:"done,omitempty"` +} + +func (*SyncNowResponse_Ack_) isSyncNowResponse_Response() {} +func (*SyncNowResponse_Heartbeat_) isSyncNowResponse_Response() {} +func (*SyncNowResponse_Done_) isSyncNowResponse_Response() {} + +func (m *SyncNowResponse) GetResponse() isSyncNowResponse_Response { + if m != nil { + return m.Response + } + return nil +} + +func (m *SyncNowResponse) GetAck() *SyncNowResponse_Ack { + if x, ok := m.GetResponse().(*SyncNowResponse_Ack_); ok { + return x.Ack + } + return nil +} + +func (m *SyncNowResponse) GetHeartbeat() *SyncNowResponse_Heartbeat { + if x, ok := m.GetResponse().(*SyncNowResponse_Heartbeat_); ok { + return x.Heartbeat + } + return nil +} + +func (m *SyncNowResponse) GetDone() *SyncNowResponse_Done { + if x, ok := m.GetResponse().(*SyncNowResponse_Done_); ok { + return x.Done + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*SyncNowResponse) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*SyncNowResponse_Ack_)(nil), + (*SyncNowResponse_Heartbeat_)(nil), + (*SyncNowResponse_Done_)(nil), + } +} + +// Ack is sent exactly once, as the first message of the stream: the +// request reached the task's leader and the commit has been forced. +type SyncNowResponse_Ack struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SyncNowResponse_Ack) Reset() { *m = SyncNowResponse_Ack{} } +func (m *SyncNowResponse_Ack) String() string { return proto.CompactTextString(m) } +func (*SyncNowResponse_Ack) ProtoMessage() {} +func (*SyncNowResponse_Ack) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{33, 0} +} +func (m *SyncNowResponse_Ack) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SyncNowResponse_Ack) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SyncNowResponse_Ack.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SyncNowResponse_Ack) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncNowResponse_Ack.Merge(m, src) +} +func (m *SyncNowResponse_Ack) XXX_Size() int { + return m.ProtoSize() +} +func (m *SyncNowResponse_Ack) XXX_DiscardUnknown() { + xxx_messageInfo_SyncNowResponse_Ack.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncNowResponse_Ack proto.InternalMessageInfo + +// Heartbeat keeps a long-lived stream alive while the caller waits. +type SyncNowResponse_Heartbeat struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SyncNowResponse_Heartbeat) Reset() { *m = SyncNowResponse_Heartbeat{} } +func (m *SyncNowResponse_Heartbeat) String() string { return proto.CompactTextString(m) } +func (*SyncNowResponse_Heartbeat) ProtoMessage() {} +func (*SyncNowResponse_Heartbeat) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{33, 1} +} +func (m *SyncNowResponse_Heartbeat) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SyncNowResponse_Heartbeat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SyncNowResponse_Heartbeat.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SyncNowResponse_Heartbeat) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncNowResponse_Heartbeat.Merge(m, src) +} +func (m *SyncNowResponse_Heartbeat) XXX_Size() int { + return m.ProtoSize() +} +func (m *SyncNowResponse_Heartbeat) XXX_DiscardUnknown() { + xxx_messageInfo_SyncNowResponse_Heartbeat.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncNowResponse_Heartbeat proto.InternalMessageInfo + +// Done is sent exactly once, as the final message of the stream: the +// awaited transaction is committed and queryable in the endpoint. +type SyncNowResponse_Done struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SyncNowResponse_Done) Reset() { *m = SyncNowResponse_Done{} } +func (m *SyncNowResponse_Done) String() string { return proto.CompactTextString(m) } +func (*SyncNowResponse_Done) ProtoMessage() {} +func (*SyncNowResponse_Done) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{33, 2} +} +func (m *SyncNowResponse_Done) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SyncNowResponse_Done) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SyncNowResponse_Done.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SyncNowResponse_Done) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncNowResponse_Done.Merge(m, src) +} +func (m *SyncNowResponse_Done) XXX_Size() int { + return m.ProtoSize() +} +func (m *SyncNowResponse_Done) XXX_DiscardUnknown() { + xxx_messageInfo_SyncNowResponse_Done.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncNowResponse_Done proto.InternalMessageInfo + func init() { proto.RegisterEnum("runtime.Plane", Plane_name, Plane_value) proto.RegisterEnum("runtime.CaptureResponseExt_PollResult", CaptureResponseExt_PollResult_name, CaptureResponseExt_PollResult_value) @@ -3894,6 +4163,11 @@ func init() { proto.RegisterType((*Derive_Stored_PublisherCommit)(nil), "runtime.Derive.Stored.PublisherCommit") proto.RegisterType((*Derive_StartCommit)(nil), "runtime.Derive.StartCommit") proto.RegisterType((*Derive_StartedCommit)(nil), "runtime.Derive.StartedCommit") + proto.RegisterType((*SyncNowRequest)(nil), "runtime.SyncNowRequest") + proto.RegisterType((*SyncNowResponse)(nil), "runtime.SyncNowResponse") + proto.RegisterType((*SyncNowResponse_Ack)(nil), "runtime.SyncNowResponse.Ack") + proto.RegisterType((*SyncNowResponse_Heartbeat)(nil), "runtime.SyncNowResponse.Heartbeat") + proto.RegisterType((*SyncNowResponse_Done)(nil), "runtime.SyncNowResponse.Done") } func init() { @@ -3901,289 +4175,298 @@ func init() { } var fileDescriptor_73af6e0737ce390c = []byte{ - // 4509 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7b, 0x4d, 0x6c, 0x1c, 0x47, - 0x76, 0xb0, 0xe6, 0x7f, 0xe6, 0xcd, 0x90, 0x9c, 0x29, 0x91, 0x54, 0xab, 0x65, 0x8b, 0xf4, 0xd8, - 0xfe, 0x4c, 0x8b, 0xd4, 0x90, 0xa6, 0xe5, 0x5d, 0x5b, 0x9f, 0xff, 0xf8, 0xa7, 0x15, 0xb5, 0x94, - 0x44, 0x17, 0x29, 0x21, 0xd9, 0x4b, 0xa3, 0xd9, 0x5d, 0x1c, 0xb6, 0xd8, 0xd3, 0xd5, 0xee, 0xee, - 0xa1, 0xc4, 0x3d, 0x05, 0xc8, 0x25, 0x40, 0x80, 0xe4, 0xb2, 0x97, 0x20, 0x97, 0xdc, 0x82, 0x04, - 0xc8, 0x21, 0xa7, 0x05, 0xf6, 0x90, 0x53, 0x0e, 0x46, 0x4e, 0x49, 0x0e, 0x41, 0x4e, 0x0a, 0xb2, - 0xb9, 0x26, 0xa7, 0x4d, 0x80, 0x44, 0xc8, 0x21, 0xa8, 0x9f, 0xfe, 0x9d, 0x1e, 0x8a, 0xa4, 0x8d, - 0xc4, 0x58, 0xec, 0x41, 0x62, 0xd7, 0xfb, 0xab, 0x57, 0x55, 0xef, 0xbd, 0x7a, 0xef, 0x75, 0x0f, - 0x74, 0xfb, 0x74, 0xd9, 0xf5, 0x68, 0x40, 0x0d, 0x6a, 0xfb, 0xcb, 0xde, 0xd0, 0x09, 0xac, 0x01, - 0x09, 0xff, 0xf6, 0x38, 0x06, 0xd5, 0xe4, 0x50, 0xbd, 0x79, 0xe0, 0xd1, 0x63, 0xe2, 0x45, 0x0c, - 0xd1, 0x83, 0x20, 0x54, 0xe7, 0x0d, 0xea, 0xf8, 0xc3, 0xc1, 0x19, 0x14, 0xe9, 0xe9, 0x0c, 0xdd, - 0x0d, 0x86, 0x1e, 0x09, 0xff, 0x86, 0x52, 0x52, 0x34, 0x26, 0xf1, 0xac, 0x13, 0x22, 0xff, 0x48, - 0x8a, 0x37, 0x52, 0x14, 0x87, 0x36, 0x7d, 0xce, 0xff, 0x93, 0xd8, 0x5b, 0x29, 0xec, 0x40, 0x0f, - 0x88, 0x67, 0xe9, 0xb6, 0xf5, 0x53, 0x92, 0x7c, 0x96, 0xb4, 0x6a, 0x8a, 0x96, 0xba, 0xfc, 0x5f, - 0xae, 0xae, 0xfe, 0xd1, 0xf0, 0xf0, 0xd0, 0x26, 0xe1, 0x5f, 0x49, 0x33, 0xdd, 0xa7, 0x7d, 0xca, - 0x1f, 0x97, 0xd9, 0x93, 0x80, 0x76, 0xff, 0xaa, 0x00, 0x9d, 0x7d, 0xdd, 0x3f, 0xde, 0x23, 0xde, - 0x89, 0x65, 0x90, 0x0d, 0xea, 0x1c, 0x5a, 0x7d, 0x74, 0x13, 0x9a, 0x36, 0xed, 0x6b, 0x87, 0x96, - 0x4d, 0xb4, 0x43, 0x53, 0x29, 0xcc, 0x17, 0x16, 0x2a, 0xb8, 0x61, 0xd3, 0xfe, 0x3d, 0xcb, 0x26, - 0xf7, 0x4c, 0x74, 0x03, 0x1a, 0x81, 0xee, 0x1f, 0x6b, 0x8e, 0x3e, 0x20, 0x4a, 0x71, 0xbe, 0xb0, - 0xd0, 0xc0, 0x75, 0x06, 0x78, 0xa4, 0x0f, 0x08, 0xba, 0x0e, 0xf5, 0xa1, 0xe9, 0x6b, 0xae, 0x1e, - 0x1c, 0x29, 0x25, 0x8e, 0xab, 0x0d, 0x4d, 0x7f, 0x57, 0x0f, 0x8e, 0xd0, 0x22, 0x74, 0x0c, 0xea, - 0x04, 0xba, 0xe5, 0x10, 0x4f, 0x73, 0x48, 0xf0, 0x9c, 0x7a, 0xc7, 0x4a, 0x99, 0xd3, 0xb4, 0x23, - 0xc4, 0x23, 0x01, 0x47, 0xef, 0x40, 0xc5, 0xb5, 0x75, 0x87, 0x28, 0xd5, 0xf9, 0xc2, 0xc2, 0xe4, - 0xea, 0x64, 0x2f, 0x3c, 0xea, 0x5d, 0x06, 0xc5, 0x02, 0xd9, 0xfd, 0xaf, 0x32, 0x4c, 0xee, 0x89, - 0x85, 0x62, 0xf2, 0xf5, 0x90, 0xf8, 0x01, 0xda, 0x86, 0xda, 0x33, 0x3a, 0xf4, 0x1c, 0xdd, 0xe6, - 0x9a, 0x37, 0xd6, 0x97, 0x5f, 0xbd, 0x9c, 0x5b, 0xec, 0xd3, 0x5e, 0x5f, 0xff, 0x29, 0x09, 0x02, - 0xd2, 0x33, 0xc9, 0xc9, 0xb2, 0x41, 0x3d, 0xb2, 0x9c, 0x31, 0x92, 0xde, 0x03, 0xc1, 0x86, 0x43, - 0x7e, 0x34, 0x0b, 0x55, 0x8f, 0xb8, 0xb6, 0x7e, 0xca, 0x57, 0x59, 0xc7, 0x72, 0xc4, 0xd6, 0x78, - 0x30, 0xb4, 0x6c, 0x53, 0xb3, 0xcc, 0x70, 0x8d, 0x7c, 0xbc, 0x6d, 0xa2, 0x7b, 0x50, 0xa5, 0x87, - 0x87, 0x3e, 0x09, 0xf8, 0xc2, 0x4a, 0xeb, 0xbd, 0x57, 0x2f, 0xe7, 0x6e, 0x9d, 0x67, 0xf2, 0xc7, - 0x9c, 0x0b, 0x4b, 0x6e, 0xf4, 0x10, 0x80, 0x38, 0xa6, 0x26, 0x65, 0x55, 0x2e, 0x25, 0xab, 0x41, - 0x1c, 0x53, 0x3c, 0xa2, 0x45, 0xa8, 0x78, 0xba, 0xd3, 0x17, 0xbb, 0xd9, 0x5c, 0x9d, 0xea, 0x71, - 0x33, 0xc4, 0x0c, 0xb4, 0xe7, 0x12, 0x63, 0xbd, 0xfc, 0xcd, 0xcb, 0xb9, 0x2b, 0x58, 0xd0, 0xa0, - 0x3d, 0x68, 0x1a, 0x94, 0x7a, 0xa6, 0xe5, 0xe8, 0x01, 0xf5, 0x94, 0x1a, 0xdf, 0xc5, 0x0f, 0x5e, - 0xbd, 0x9c, 0xbb, 0x9d, 0x37, 0xf9, 0x88, 0x2b, 0xf5, 0xf6, 0x8e, 0x74, 0xcf, 0xdc, 0xde, 0xc4, - 0x49, 0x29, 0x68, 0x05, 0xc0, 0x23, 0x3e, 0xb5, 0x87, 0x81, 0x45, 0x1d, 0xa5, 0xce, 0xd5, 0x68, - 0xf7, 0x22, 0x9e, 0xfb, 0x44, 0x37, 0x89, 0x87, 0x13, 0x34, 0xe8, 0x6d, 0x98, 0x90, 0x36, 0xac, - 0x59, 0x8e, 0x49, 0x5e, 0x28, 0x8d, 0xf9, 0xc2, 0xc2, 0x04, 0x6e, 0x49, 0xe0, 0x36, 0x83, 0xa1, - 0x3b, 0x00, 0xdc, 0xe3, 0x74, 0x2e, 0x16, 0xb8, 0xd8, 0x69, 0xb1, 0xba, 0x0d, 0x6a, 0xdb, 0xc4, - 0x60, 0x70, 0xb6, 0x44, 0x9c, 0xa0, 0x43, 0x1b, 0x30, 0x15, 0xbb, 0x98, 0x60, 0x6d, 0x72, 0xd6, - 0xeb, 0x82, 0xf5, 0x61, 0x1a, 0xc9, 0xf9, 0xb3, 0x1c, 0xdd, 0xbf, 0x2f, 0xc3, 0x54, 0x64, 0x7b, - 0xbe, 0x4b, 0x1d, 0x9f, 0xa0, 0x05, 0xa8, 0xfa, 0x81, 0x1e, 0x0c, 0x7d, 0x6e, 0x7b, 0x93, 0xab, - 0xed, 0x5e, 0xb8, 0x3d, 0xbd, 0x3d, 0x0e, 0xc7, 0x12, 0xcf, 0x28, 0x8f, 0xf8, 0x9a, 0xb9, 0x6d, - 0xe5, 0xed, 0x85, 0xc4, 0xa3, 0x77, 0x61, 0x32, 0x20, 0xde, 0xc0, 0x72, 0x74, 0x5b, 0x23, 0x9e, - 0x47, 0x3d, 0x69, 0x73, 0x13, 0x21, 0x74, 0x8b, 0x01, 0xd1, 0x57, 0xd0, 0xf2, 0x88, 0x6e, 0x6a, - 0xc1, 0x91, 0x47, 0x87, 0xfd, 0xa3, 0x4b, 0xda, 0x5f, 0x93, 0xc9, 0xd8, 0x17, 0x22, 0x98, 0x11, - 0x3e, 0xf7, 0xac, 0x80, 0x68, 0x4c, 0x93, 0xcb, 0x1a, 0x21, 0x97, 0xc0, 0x96, 0x84, 0xb6, 0xa1, - 0xa2, 0x7b, 0xc4, 0xd1, 0xb9, 0x11, 0xb6, 0xd6, 0x3f, 0x7c, 0xf5, 0x72, 0x6e, 0xb9, 0x6f, 0x05, - 0x47, 0xc3, 0x83, 0x9e, 0x41, 0x07, 0xcb, 0xc4, 0x0f, 0x86, 0xba, 0x77, 0x2a, 0xc2, 0xe4, 0x48, - 0xe0, 0xec, 0xad, 0x31, 0x56, 0x2c, 0x24, 0xa0, 0x77, 0xa1, 0x6c, 0x52, 0xc3, 0x57, 0x6a, 0xf3, - 0xa5, 0x85, 0xe6, 0x6a, 0x53, 0x9c, 0xda, 0x9e, 0x6d, 0x19, 0x44, 0x9a, 0x32, 0x47, 0xa3, 0xfb, - 0x50, 0x13, 0x1e, 0xe4, 0x2b, 0xf5, 0xf9, 0xd2, 0x25, 0xb4, 0x0f, 0xd9, 0x99, 0x9d, 0x0d, 0x87, - 0x96, 0xa9, 0xb9, 0xba, 0x17, 0xf8, 0x4a, 0x83, 0x4f, 0x2b, 0xbd, 0xe8, 0xc9, 0x93, 0xed, 0xcd, - 0x5d, 0x06, 0x96, 0x53, 0x37, 0x18, 0x21, 0x07, 0x30, 0xa3, 0x77, 0x75, 0xe3, 0x98, 0x98, 0xda, - 0x31, 0x39, 0x55, 0x60, 0x9c, 0xb2, 0x0d, 0x41, 0xf4, 0x63, 0x72, 0xda, 0x35, 0xa1, 0x83, 0xa9, - 0x71, 0xec, 0x6f, 0xae, 0x6f, 0x12, 0xdf, 0xf0, 0x2c, 0x97, 0xf9, 0xce, 0x12, 0x20, 0x8f, 0x01, - 0xcd, 0x03, 0x8d, 0x38, 0x27, 0xda, 0x80, 0x0c, 0xdc, 0xc0, 0xe3, 0x16, 0x56, 0xc5, 0x6d, 0x89, - 0xd9, 0x72, 0x4e, 0x1e, 0x72, 0x38, 0x7a, 0x0b, 0x5a, 0x21, 0x35, 0x8f, 0xc2, 0x22, 0x42, 0x37, - 0x25, 0x8c, 0x45, 0xe2, 0xee, 0xcf, 0x8a, 0xd0, 0xd8, 0x08, 0x23, 0x2e, 0xba, 0x06, 0x35, 0xcb, - 0xd5, 0x74, 0xd3, 0x14, 0x32, 0x1b, 0xb8, 0x6a, 0xb9, 0x6b, 0xa6, 0xe9, 0xa1, 0x1f, 0xc0, 0x84, - 0x0c, 0xd3, 0x9a, 0x4b, 0xd9, 0xba, 0x8b, 0x7c, 0x05, 0x1d, 0xb1, 0x02, 0x19, 0xa9, 0x77, 0xa9, - 0x17, 0xe0, 0x96, 0x13, 0x0f, 0x7c, 0xb4, 0x07, 0x9d, 0x81, 0xee, 0xba, 0xc4, 0xd4, 0x8e, 0xa8, - 0x1f, 0x48, 0xde, 0x12, 0xe7, 0x7d, 0x2f, 0x8a, 0xe3, 0xd1, 0xfc, 0xbd, 0x87, 0x9c, 0xf6, 0x3e, - 0xf5, 0x03, 0xce, 0xbe, 0xe5, 0x04, 0xde, 0x29, 0x73, 0xb7, 0x14, 0x14, 0xbd, 0x09, 0x30, 0xf4, - 0xf5, 0x3e, 0xd1, 0x3c, 0x3d, 0x20, 0xdc, 0xba, 0x8b, 0xb8, 0xc1, 0x21, 0x58, 0x0f, 0x88, 0xba, - 0x0e, 0xd3, 0x79, 0x72, 0x50, 0x1b, 0x4a, 0x6c, 0xef, 0x0b, 0x3c, 0x76, 0xb0, 0x47, 0x34, 0x0d, - 0x95, 0x13, 0xdd, 0x1e, 0x86, 0x57, 0x97, 0x18, 0xdc, 0x2d, 0x7e, 0x5c, 0xe8, 0xfe, 0x79, 0x11, - 0x3a, 0x1b, 0xe2, 0x8a, 0x97, 0xb7, 0xc9, 0xd6, 0x0b, 0x16, 0x3b, 0xd9, 0xdd, 0xa7, 0xd9, 0xe4, - 0x84, 0xd8, 0xd2, 0xad, 0x27, 0x7b, 0xec, 0xf6, 0xdd, 0xa1, 0xfd, 0xde, 0x0e, 0x83, 0xe2, 0xba, - 0x4d, 0xfb, 0xfc, 0x09, 0x6d, 0xc7, 0x47, 0x65, 0x46, 0x07, 0x28, 0x5d, 0x5c, 0x8d, 0xd6, 0x3e, - 0x72, 0xc4, 0xb8, 0x23, 0xb9, 0x12, 0xa7, 0xbe, 0x0d, 0x2d, 0x3f, 0xd0, 0xbd, 0x40, 0x33, 0xe8, - 0x60, 0x60, 0x05, 0xdc, 0xeb, 0x9b, 0xab, 0xff, 0x2f, 0xde, 0xc0, 0xac, 0xa6, 0x2c, 0xc4, 0x78, - 0xc1, 0x06, 0xa7, 0xc6, 0x4d, 0x3f, 0x1e, 0xa8, 0x18, 0x9a, 0x09, 0x1c, 0xda, 0x00, 0x24, 0x85, - 0x68, 0xc6, 0x11, 0x31, 0x8e, 0x5d, 0x6a, 0x39, 0x01, 0x5f, 0x1a, 0x0b, 0x9e, 0x51, 0xc4, 0xda, - 0x88, 0x70, 0xb8, 0x23, 0xe9, 0x63, 0x50, 0xf7, 0xbf, 0xcb, 0x80, 0x22, 0x15, 0x44, 0xf8, 0x63, - 0xbb, 0xb5, 0x02, 0x8d, 0xe8, 0x2e, 0x97, 0x22, 0xd1, 0xe8, 0x99, 0xe3, 0x98, 0x08, 0xdd, 0x85, - 0x2a, 0x75, 0x89, 0x43, 0x4c, 0xb9, 0x4d, 0xdd, 0xd1, 0x15, 0x46, 0xe2, 0x7b, 0x8f, 0x39, 0x25, - 0x96, 0x1c, 0xe8, 0x4b, 0xa8, 0xcb, 0x9c, 0xcc, 0x94, 0xfb, 0xf3, 0xce, 0x59, 0xdc, 0x12, 0x64, - 0xe2, 0x88, 0x0b, 0xdd, 0x03, 0x48, 0xec, 0x41, 0x79, 0xdc, 0x1e, 0x27, 0x64, 0xc4, 0xbb, 0x92, - 0xe0, 0x54, 0x1f, 0x42, 0x55, 0xe8, 0xf6, 0x9d, 0xec, 0xae, 0xfa, 0x14, 0xea, 0xa1, 0xb2, 0xcc, - 0xf2, 0x8f, 0xc9, 0xa9, 0x26, 0x82, 0x04, 0x17, 0xd4, 0xc2, 0x8d, 0x63, 0x72, 0xba, 0xcb, 0x01, - 0x2c, 0xad, 0x62, 0x51, 0xc9, 0x62, 0x97, 0x92, 0x1f, 0x52, 0x15, 0x39, 0x55, 0x3b, 0x46, 0x08, - 0x62, 0xf5, 0x39, 0x40, 0x3c, 0x0b, 0x9a, 0x87, 0x0a, 0xbb, 0x8e, 0x7c, 0xa9, 0x1d, 0x70, 0xb3, - 0x66, 0x17, 0x95, 0x8f, 0x05, 0x02, 0xfd, 0x08, 0x9a, 0x2e, 0xb5, 0x6d, 0xcd, 0x23, 0xfe, 0xd0, - 0x0e, 0xb8, 0xd8, 0xc9, 0xb3, 0xf7, 0x67, 0x97, 0xda, 0x36, 0xe6, 0xd4, 0x18, 0xdc, 0xe8, 0xb9, - 0xfb, 0x08, 0x20, 0xc6, 0xa0, 0x26, 0xd4, 0xb6, 0x1f, 0x3d, 0x5d, 0xdb, 0xd9, 0xde, 0x6c, 0x5f, - 0x41, 0x0d, 0xa8, 0xe0, 0xad, 0xb5, 0xcd, 0xdf, 0x6e, 0x17, 0xd0, 0x04, 0x34, 0x1e, 0x3d, 0xde, - 0xd7, 0xc4, 0xb0, 0x88, 0x5a, 0x50, 0xdf, 0x78, 0xfc, 0x78, 0x47, 0x7b, 0x7c, 0xef, 0x5e, 0xbb, - 0xc4, 0x98, 0xf0, 0xd6, 0xde, 0xfe, 0x1a, 0xde, 0x6f, 0x97, 0xbb, 0xff, 0x5a, 0x80, 0xf6, 0x26, - 0xcf, 0xb5, 0xbf, 0x07, 0xae, 0xba, 0x0a, 0x65, 0x66, 0x90, 0xd2, 0x04, 0x6f, 0x46, 0xcc, 0x59, - 0x05, 0xb9, 0xf9, 0x62, 0x4e, 0xab, 0x2e, 0x41, 0x99, 0x8d, 0xd0, 0x3b, 0x30, 0xe9, 0x7f, 0x6d, - 0xb3, 0x5b, 0xf6, 0xe4, 0xd0, 0xd7, 0x86, 0x9e, 0x25, 0x83, 0x70, 0x4b, 0x40, 0x9f, 0x1e, 0xfa, - 0x4f, 0x3c, 0xab, 0xfb, 0xef, 0x25, 0xe8, 0x84, 0xd2, 0xbe, 0x8d, 0xb3, 0x7d, 0x92, 0x71, 0xb6, - 0xb7, 0x46, 0x74, 0x1d, 0xeb, 0x6b, 0xeb, 0xd0, 0x70, 0x87, 0x07, 0xb6, 0xe5, 0x1f, 0xe5, 0x38, - 0xdb, 0x28, 0xf7, 0x6e, 0x48, 0x8b, 0x63, 0x36, 0xf4, 0x29, 0xd4, 0x0e, 0xed, 0x21, 0x97, 0x50, - 0xce, 0x38, 0xfb, 0xa8, 0x84, 0x7b, 0x82, 0x12, 0x87, 0x2c, 0xdf, 0xb5, 0x8f, 0x05, 0xd0, 0x88, - 0x94, 0x64, 0x45, 0xcd, 0x40, 0x7f, 0xa1, 0x19, 0x36, 0x35, 0x8e, 0xe5, 0xd5, 0x5a, 0x1f, 0xe8, - 0x2f, 0x36, 0xd8, 0x38, 0xe3, 0x81, 0xc5, 0x73, 0x79, 0x60, 0x69, 0x8c, 0x07, 0x2e, 0x42, 0x4d, - 0x2e, 0xec, 0xf5, 0xee, 0xd7, 0xfd, 0xc3, 0x02, 0xcc, 0xc4, 0xc9, 0xe8, 0xf7, 0xc0, 0xd4, 0xbb, - 0xbf, 0x28, 0xc0, 0x6c, 0x4a, 0xa3, 0x6f, 0x63, 0x8d, 0x6b, 0xb1, 0x39, 0x08, 0x65, 0xe2, 0xf4, - 0x20, 0x7f, 0x8e, 0x51, 0x9b, 0xb8, 0xd0, 0x76, 0xfe, 0xa2, 0x0c, 0x93, 0x1b, 0x74, 0x70, 0x60, - 0x39, 0x51, 0xb9, 0xb8, 0x22, 0x5d, 0x57, 0xf0, 0xbc, 0x91, 0xd0, 0x37, 0x49, 0x96, 0x70, 0x5c, - 0x74, 0x1b, 0x4a, 0xba, 0x19, 0x2a, 0x7c, 0x63, 0x1c, 0xc3, 0x9a, 0x69, 0x62, 0x46, 0xa7, 0xfe, - 0x43, 0x51, 0x3a, 0xfa, 0x97, 0x50, 0x3f, 0xb0, 0x1c, 0xd3, 0x72, 0xfa, 0x4c, 0xc3, 0x52, 0xfa, - 0xae, 0x1a, 0x9d, 0xad, 0xb7, 0x2e, 0x88, 0x71, 0xc4, 0xa5, 0xfe, 0x7e, 0x11, 0x6a, 0x12, 0x8a, - 0x10, 0x94, 0x0f, 0x87, 0xb6, 0x38, 0xfa, 0x3a, 0xe6, 0xcf, 0x61, 0xae, 0xc3, 0xb2, 0xb4, 0x86, - 0xc8, 0x75, 0x3e, 0x86, 0xa6, 0xeb, 0xd1, 0x67, 0xa2, 0x0c, 0x0a, 0x73, 0xb0, 0xb6, 0xc8, 0xdf, - 0x76, 0x23, 0x84, 0x4c, 0x43, 0x93, 0xa4, 0xe8, 0x33, 0x68, 0xfa, 0xc6, 0x11, 0x19, 0xe8, 0xda, - 0x33, 0x9f, 0x3a, 0xdc, 0x5b, 0x5b, 0xeb, 0x6f, 0xbc, 0x7a, 0x39, 0xa7, 0x10, 0xc7, 0xa0, 0x4c, - 0x85, 0x65, 0x86, 0xe8, 0x61, 0xfd, 0xf9, 0x43, 0xe2, 0xf3, 0x34, 0x0c, 0x04, 0xc3, 0x03, 0x9f, - 0x3a, 0xa8, 0x07, 0xe0, 0x13, 0x4f, 0x73, 0xa9, 0x6d, 0x19, 0xa7, 0xbc, 0x74, 0x88, 0xf2, 0xe5, - 0x3d, 0xe2, 0xed, 0x72, 0x30, 0x6e, 0xf8, 0xe1, 0x23, 0x6f, 0x1b, 0xf0, 0xfc, 0x3a, 0xf0, 0x78, - 0x79, 0xd0, 0xc0, 0x35, 0x9e, 0x46, 0x07, 0x1e, 0xab, 0xc2, 0x79, 0x8a, 0x26, 0xb2, 0xfd, 0x06, - 0x96, 0x23, 0xd5, 0x81, 0xd2, 0x9a, 0x69, 0x22, 0x05, 0x6a, 0x72, 0x83, 0x64, 0x92, 0x17, 0x0e, - 0xd1, 0x0f, 0xa1, 0x6e, 0x52, 0x43, 0xe8, 0x5f, 0x3c, 0x87, 0xfe, 0x35, 0x93, 0x1a, 0x5c, 0xf9, - 0x69, 0xa8, 0x1c, 0x7a, 0xd4, 0x11, 0x29, 0x57, 0x1d, 0x8b, 0x41, 0xf7, 0x1f, 0x0b, 0x30, 0x15, - 0x9d, 0x93, 0xac, 0xf7, 0xc6, 0x4f, 0xae, 0x40, 0xcd, 0x24, 0x36, 0x09, 0xa4, 0x69, 0xd7, 0x71, - 0x38, 0x4c, 0xa9, 0x55, 0xba, 0x94, 0x5a, 0xe5, 0x84, 0x5a, 0x99, 0xd8, 0x54, 0xc9, 0xc6, 0xa6, - 0xb7, 0x61, 0x42, 0xec, 0x57, 0x48, 0xc1, 0x8b, 0x2f, 0xdc, 0x12, 0x40, 0x41, 0xd4, 0xbd, 0x06, - 0x33, 0x1b, 0xd4, 0x71, 0x88, 0x11, 0x50, 0x6f, 0xd7, 0xa3, 0x2f, 0x4e, 0xa5, 0x21, 0x76, 0xff, - 0xb8, 0x00, 0xb3, 0x59, 0x8c, 0x5c, 0xfa, 0x03, 0xa8, 0xb1, 0x92, 0x81, 0xf8, 0xbe, 0xec, 0xb3, - 0xac, 0xbc, 0x7a, 0x39, 0xb7, 0x74, 0x9e, 0xda, 0x6a, 0xcb, 0x31, 0x45, 0x4c, 0x0e, 0x05, 0xb0, - 0xd3, 0x77, 0x99, 0x70, 0xcd, 0x32, 0x65, 0x56, 0x5e, 0xe3, 0xe3, 0x6d, 0x13, 0xa9, 0x50, 0xb2, - 0x69, 0x5f, 0xde, 0x37, 0xf5, 0x30, 0xc2, 0x61, 0x06, 0xec, 0xfe, 0x65, 0x09, 0xca, 0x0f, 0xa8, - 0xe5, 0xa0, 0x5b, 0xd0, 0x21, 0x81, 0x61, 0x6a, 0x03, 0x6a, 0x6a, 0x1e, 0x39, 0xb1, 0x7c, 0x56, - 0xd1, 0x33, 0xad, 0x4a, 0x78, 0x8a, 0x21, 0x1e, 0x52, 0x13, 0x4b, 0x30, 0x5a, 0x84, 0xaa, 0x7f, - 0xa4, 0x7b, 0x66, 0x58, 0xcd, 0x5c, 0x8d, 0x9c, 0x90, 0x89, 0x12, 0xcd, 0x0b, 0x2c, 0x49, 0xd0, - 0x1c, 0x34, 0xf9, 0x93, 0xec, 0x40, 0x94, 0xf8, 0x19, 0x03, 0x07, 0x89, 0xfe, 0xc3, 0x22, 0x74, - 0xc2, 0x26, 0x85, 0x69, 0x79, 0x7c, 0x9b, 0x4e, 0xc3, 0x9e, 0x96, 0x44, 0x6c, 0x86, 0x70, 0xf4, - 0x3e, 0x84, 0x30, 0x8d, 0xc8, 0x3d, 0xe0, 0x07, 0xd6, 0xc0, 0x53, 0x12, 0x1e, 0x6e, 0x0d, 0x7a, - 0x0f, 0xa6, 0x6c, 0x5e, 0xfe, 0xc7, 0x94, 0xc2, 0x2d, 0x26, 0x05, 0x38, 0x24, 0x54, 0xff, 0xa2, - 0x00, 0x15, 0xae, 0x33, 0x9a, 0x84, 0xa2, 0x65, 0xca, 0xe4, 0xa1, 0x68, 0x99, 0xa8, 0x07, 0x75, - 0x5b, 0x3f, 0x20, 0x36, 0x33, 0xce, 0xa2, 0x8c, 0xc6, 0x3c, 0x22, 0x32, 0xea, 0x1d, 0x89, 0xc1, - 0x11, 0x0d, 0x5a, 0x85, 0x9a, 0x47, 0x74, 0xa6, 0xa9, 0xdc, 0x6d, 0x25, 0x6e, 0x49, 0xec, 0x7a, - 0xd4, 0x20, 0xbe, 0xbf, 0xe7, 0x12, 0xa3, 0xb7, 0xbd, 0x89, 0x43, 0x42, 0xb4, 0x02, 0xd3, 0x7c, - 0xe3, 0x0d, 0x8f, 0xe8, 0x01, 0x89, 0xf7, 0x9e, 0x37, 0x1f, 0x30, 0x62, 0xb8, 0x0d, 0x8e, 0x0a, - 0xb7, 0xbf, 0x7b, 0x07, 0xaa, 0x6c, 0x9f, 0x89, 0xc9, 0x0e, 0x8d, 0xdd, 0xb8, 0x9c, 0x3f, 0x7b, - 0x68, 0x03, 0xfd, 0xc5, 0x56, 0x60, 0x44, 0x87, 0xd6, 0xfd, 0x59, 0x01, 0xca, 0xfb, 0xba, 0x7f, - 0xcc, 0xc2, 0x9e, 0xef, 0x12, 0x43, 0x66, 0xc1, 0xfc, 0x99, 0x6d, 0x2b, 0x13, 0x14, 0x78, 0xba, - 0xe3, 0xeb, 0x51, 0xa4, 0x63, 0x27, 0xc5, 0xe4, 0xec, 0x27, 0xc0, 0x39, 0xc9, 0x56, 0x79, 0x34, - 0xd9, 0x62, 0x15, 0x74, 0x98, 0xb2, 0x78, 0xcc, 0x24, 0x85, 0x53, 0x35, 0x23, 0xd8, 0xb6, 0xf9, - 0xa0, 0x5c, 0x2f, 0xb6, 0x4b, 0xdd, 0x9f, 0x57, 0xa1, 0x86, 0x89, 0x41, 0x4f, 0xf8, 0x5d, 0xd6, - 0xd4, 0x8d, 0x63, 0xcd, 0x72, 0x02, 0xe2, 0x04, 0x61, 0x84, 0x9f, 0x8f, 0x2f, 0x57, 0x41, 0xd6, - 0x5b, 0x33, 0x8e, 0xb7, 0x05, 0x89, 0xa8, 0x73, 0x41, 0x8f, 0x00, 0x68, 0x15, 0x66, 0x44, 0xad, - 0x17, 0x10, 0x93, 0x65, 0x22, 0x3e, 0x91, 0xf9, 0x48, 0x91, 0xe7, 0x23, 0x57, 0x23, 0xe4, 0x06, - 0xc3, 0x89, 0xd4, 0xe4, 0x4b, 0x40, 0x31, 0x0f, 0x8f, 0x08, 0x16, 0x09, 0x0f, 0xb0, 0xd3, 0x0b, - 0x9b, 0xc0, 0xf7, 0x24, 0x02, 0x77, 0x22, 0xe2, 0x10, 0x84, 0x96, 0x60, 0xda, 0x08, 0x5d, 0x5c, - 0x63, 0xf7, 0x24, 0x49, 0x84, 0x7c, 0x3c, 0x19, 0xe1, 0xd8, 0x4d, 0x4a, 0xd0, 0x12, 0xa0, 0x23, - 0xb6, 0xc6, 0xb4, 0x82, 0x15, 0xd1, 0x8b, 0x10, 0x98, 0x84, 0x76, 0x77, 0x61, 0x4a, 0x52, 0x47, - 0xaa, 0x55, 0xc7, 0xa9, 0x36, 0x29, 0x28, 0x23, 0xbd, 0xde, 0x82, 0x96, 0xad, 0xfb, 0x81, 0xa6, - 0xbb, 0xae, 0x6d, 0x11, 0x93, 0xf7, 0x21, 0x5b, 0xb8, 0xc9, 0x60, 0x6b, 0x02, 0x84, 0xd6, 0xa0, - 0x63, 0x93, 0xbe, 0x6e, 0x9c, 0x26, 0xb3, 0xc0, 0xfa, 0x19, 0x59, 0x60, 0x5b, 0x90, 0x27, 0x4a, - 0xa0, 0x8f, 0x81, 0xa5, 0x79, 0xda, 0x31, 0x39, 0x0d, 0xdb, 0x3a, 0x6f, 0x8e, 0x9c, 0xd9, 0x43, - 0xfd, 0xc5, 0x8f, 0xc9, 0xa9, 0x3c, 0xb0, 0xda, 0x40, 0x8c, 0xd0, 0x2d, 0xb8, 0x1a, 0x78, 0x56, - 0xbf, 0xcf, 0xae, 0x39, 0xdd, 0xd3, 0x07, 0xbe, 0xd8, 0x36, 0xe0, 0x6a, 0x4e, 0x48, 0xd4, 0x2e, - 0xc7, 0xa0, 0x5d, 0x68, 0x33, 0x13, 0x3c, 0x21, 0xda, 0x81, 0x6e, 0x1c, 0x1f, 0x5a, 0xb6, 0xed, - 0x2b, 0x4d, 0x3e, 0xdb, 0xbb, 0x39, 0x16, 0xc2, 0x08, 0xd7, 0x43, 0x3a, 0xd9, 0x0e, 0xd1, 0xd3, - 0x50, 0xf5, 0x33, 0x98, 0xca, 0x98, 0x52, 0xb2, 0xd5, 0xd1, 0xc8, 0x69, 0x75, 0xb4, 0x12, 0xad, - 0x0e, 0xf5, 0x2e, 0xb4, 0x92, 0xab, 0x7a, 0x5d, 0x9b, 0x24, 0xc5, 0xbb, 0x0e, 0xd3, 0x79, 0x3a, - 0xbe, 0x4e, 0x46, 0x35, 0xd9, 0x6a, 0xf9, 0xa7, 0x3a, 0xd4, 0x76, 0x89, 0xe7, 0x5b, 0x7e, 0x80, - 0x66, 0xa0, 0xea, 0x93, 0xaf, 0x35, 0x87, 0x72, 0xd6, 0x32, 0xae, 0xf8, 0xe4, 0xeb, 0x47, 0x94, - 0x59, 0x9a, 0xb8, 0x32, 0xb5, 0xa4, 0x5f, 0x89, 0xcb, 0xb4, 0x2d, 0x30, 0xf1, 0x0e, 0x64, 0xdd, - 0xaf, 0x94, 0x71, 0x3f, 0x39, 0xd7, 0xe5, 0xdc, 0xaf, 0x3c, 0xde, 0xfd, 0xee, 0xc2, 0x75, 0xa9, - 0x64, 0x8e, 0x17, 0x56, 0xb8, 0xae, 0xd7, 0x04, 0xc1, 0xc6, 0x88, 0xe3, 0xe5, 0xbb, 0x6e, 0xf5, - 0x02, 0xae, 0xbb, 0x02, 0xb3, 0xb1, 0xeb, 0xba, 0x7a, 0x60, 0x1c, 0x11, 0x69, 0x85, 0xc2, 0x59, - 0xda, 0x11, 0x76, 0x57, 0x20, 0xc7, 0xb8, 0x6f, 0x7d, 0x8c, 0xfb, 0xde, 0x81, 0x59, 0xb9, 0xba, - 0xac, 0x17, 0x37, 0xf8, 0xd2, 0xa6, 0x05, 0xf6, 0x7e, 0xda, 0x71, 0x73, 0x9c, 0x1e, 0x2e, 0xeb, - 0xf4, 0xcd, 0x51, 0xa7, 0xff, 0x18, 0x14, 0xa9, 0xd4, 0xa8, 0xef, 0xb7, 0xb8, 0x5a, 0x52, 0xe9, - 0x9d, 0xac, 0xaf, 0xe7, 0x86, 0x8b, 0x89, 0x4b, 0x87, 0x8b, 0xc9, 0x4c, 0xb8, 0x08, 0x6d, 0x2c, - 0x3f, 0x5c, 0xac, 0xc2, 0x8c, 0x54, 0x3b, 0x1d, 0x35, 0x94, 0x29, 0xae, 0xf3, 0x55, 0x81, 0xdc, - 0x4f, 0x85, 0x8d, 0x31, 0x21, 0xa6, 0x9d, 0x17, 0x62, 0xf8, 0xcb, 0x2a, 0xdf, 0xd0, 0x1d, 0xa5, - 0x13, 0xbe, 0xac, 0x62, 0x23, 0x74, 0x07, 0x2a, 0x07, 0xa4, 0x6f, 0x39, 0x0a, 0xca, 0x54, 0x38, - 0x69, 0x1f, 0x5e, 0x67, 0x34, 0xf7, 0xaf, 0x60, 0x41, 0x8c, 0x16, 0xa1, 0x6d, 0xd0, 0x81, 0xcb, - 0xf5, 0x0d, 0x33, 0xdc, 0xab, 0xcc, 0xb1, 0xef, 0x5f, 0xc1, 0x53, 0x21, 0x46, 0xd6, 0x22, 0xff, - 0x87, 0xb1, 0x68, 0x5d, 0x81, 0xd9, 0x4c, 0x60, 0xd5, 0x8c, 0x23, 0xdd, 0xe9, 0x93, 0x2e, 0x86, - 0xab, 0x39, 0x2b, 0x3c, 0x23, 0x63, 0x7f, 0x0b, 0x5a, 0x81, 0x37, 0x74, 0x0c, 0x9d, 0x59, 0xae, - 0x1e, 0xc8, 0x98, 0xd5, 0x8c, 0x60, 0x6b, 0x41, 0xb7, 0x0b, 0x0d, 0x79, 0xc8, 0xc4, 0x1c, 0x13, - 0xb6, 0xba, 0x7f, 0x5a, 0x80, 0x0a, 0x33, 0xd5, 0xd3, 0xdc, 0x5c, 0x45, 0x81, 0xda, 0x09, 0x93, - 0x20, 0x4b, 0x92, 0x06, 0x0e, 0x87, 0xe8, 0x06, 0x34, 0xb8, 0xe5, 0x73, 0x16, 0x71, 0xf7, 0xd6, - 0x19, 0x80, 0xe5, 0x5c, 0x91, 0x5b, 0x84, 0xbc, 0x22, 0x6b, 0xe4, 0x6e, 0xf1, 0x54, 0xf2, 0xaf, - 0x8c, 0xb9, 0xc6, 0x45, 0xbe, 0x8f, 0xd2, 0xd7, 0x38, 0xab, 0x27, 0xba, 0xcf, 0xa0, 0x16, 0xfa, - 0xd4, 0x6d, 0x40, 0x22, 0x45, 0x8a, 0xfa, 0x03, 0x61, 0x32, 0xd6, 0xc0, 0x1d, 0x81, 0xd9, 0x8c, - 0x11, 0x67, 0xc4, 0x9d, 0x62, 0x7e, 0xdc, 0xe9, 0xfe, 0xaa, 0x20, 0xab, 0xe0, 0x8b, 0x6d, 0xca, - 0xbb, 0xe1, 0x7b, 0xcb, 0x52, 0xee, 0x7b, 0xcb, 0xf0, 0x8d, 0xe5, 0xdb, 0x67, 0xa6, 0x30, 0xbc, - 0xf8, 0x27, 0xe8, 0xa3, 0x84, 0xeb, 0x56, 0xb8, 0xeb, 0xc6, 0xad, 0x0f, 0x5e, 0x70, 0xe7, 0xfa, - 0xed, 0xb7, 0xb1, 0xce, 0x2e, 0x40, 0x9d, 0x47, 0xd3, 0x47, 0xf4, 0x79, 0xb7, 0x0a, 0xe5, 0xbd, - 0x80, 0xba, 0xdd, 0x06, 0xd4, 0xd8, 0x5f, 0x97, 0x98, 0xdd, 0xdf, 0x82, 0xe6, 0x1e, 0xf1, 0xd9, - 0x42, 0x77, 0x28, 0x75, 0xc7, 0x74, 0x69, 0x0a, 0x97, 0xe9, 0xd2, 0xfc, 0x51, 0x15, 0x6a, 0xb2, - 0x37, 0x8b, 0xde, 0x4f, 0xec, 0x78, 0x73, 0x75, 0xa6, 0x17, 0x7e, 0xc4, 0x10, 0x36, 0x1b, 0xf8, - 0x46, 0x8a, 0x83, 0xf8, 0xff, 0x30, 0xc1, 0xfe, 0x6a, 0x9e, 0x2c, 0xf2, 0x64, 0xdd, 0x30, 0x9b, - 0xe0, 0x11, 0x08, 0xc1, 0xd4, 0x62, 0xc4, 0x51, 0x41, 0xf8, 0x11, 0xd4, 0x4d, 0xcb, 0xe7, 0x39, - 0x8c, 0x3c, 0xae, 0xeb, 0x23, 0x73, 0x6d, 0x4a, 0x02, 0x1c, 0x91, 0xa2, 0x4f, 0x01, 0xc2, 0xe7, - 0xa8, 0x2b, 0xf8, 0xc6, 0xe8, 0x84, 0x9b, 0x11, 0x0d, 0x4e, 0xd0, 0xb3, 0x49, 0x4f, 0x74, 0xdb, - 0x32, 0xf5, 0x80, 0xc8, 0x2e, 0xc3, 0xe8, 0xa4, 0x4f, 0x25, 0x01, 0x8e, 0x48, 0xd1, 0x27, 0xd0, - 0x08, 0x9f, 0x4d, 0x79, 0xe3, 0xde, 0x18, 0x9d, 0x33, 0x64, 0x34, 0x71, 0x4c, 0x9d, 0x6e, 0xbc, - 0x35, 0x5e, 0xd3, 0x78, 0xfb, 0x21, 0xb4, 0x7c, 0x71, 0xc2, 0x9a, 0x4d, 0xa9, 0xab, 0x4c, 0xcb, - 0xcb, 0x26, 0x3c, 0xcc, 0xc4, 0xf1, 0xe3, 0xa6, 0x9f, 0xb0, 0x85, 0xb7, 0xa0, 0xfc, 0x8c, 0x5a, - 0x8e, 0x32, 0xc3, 0x19, 0x26, 0x52, 0x35, 0x2a, 0xe6, 0x28, 0xf4, 0x1e, 0x54, 0x9f, 0xf1, 0x4a, - 0x4a, 0x99, 0x95, 0xce, 0x91, 0x24, 0x22, 0x26, 0x96, 0x68, 0x26, 0x2b, 0xd0, 0xfd, 0x63, 0xe5, - 0x5a, 0x46, 0x16, 0x2b, 0xa8, 0x30, 0x47, 0xa1, 0xe5, 0xa8, 0x2d, 0xac, 0x70, 0xa2, 0x6b, 0xd9, - 0x0e, 0x7f, 0xb6, 0x19, 0xdc, 0x83, 0x86, 0x48, 0x20, 0x1c, 0xfa, 0x5c, 0xb9, 0x2e, 0x6f, 0xf7, - 0x88, 0x47, 0xda, 0x3c, 0xae, 0x1b, 0xf2, 0x89, 0xe9, 0xe0, 0x07, 0xd4, 0x55, 0xd4, 0x8c, 0x0e, - 0xcc, 0x15, 0x30, 0x47, 0xa1, 0x5b, 0x50, 0xf3, 0x85, 0x63, 0x28, 0x37, 0xe4, 0x2b, 0xf1, 0x24, - 0x95, 0x4b, 0x4c, 0x1c, 0x12, 0xa8, 0x77, 0xa3, 0x4e, 0xf0, 0x85, 0x9b, 0x8e, 0xdd, 0x3f, 0xb8, - 0x0e, 0xcd, 0x44, 0x77, 0x11, 0xdd, 0x4e, 0xf9, 0xc7, 0xf5, 0x5e, 0xf2, 0xe3, 0x9b, 0x1c, 0x1f, - 0xf9, 0x22, 0xdf, 0x47, 0xd4, 0x0c, 0xdf, 0x78, 0x3f, 0xf9, 0x24, 0x61, 0xb2, 0xc2, 0x4f, 0xde, - 0xcc, 0x9d, 0x33, 0xc7, 0x6c, 0x3f, 0x4b, 0x9a, 0xad, 0x70, 0x95, 0xb9, 0xfc, 0x79, 0x5f, 0x6f, - 0xba, 0x95, 0x5f, 0x13, 0xd3, 0xbd, 0x05, 0x35, 0x4f, 0x54, 0x4e, 0xd2, 0x76, 0xdb, 0xd9, 0x8a, - 0x0a, 0x87, 0x04, 0xe8, 0x1d, 0xa8, 0xb0, 0xc4, 0xf2, 0x54, 0x5a, 0x6c, 0xfc, 0x51, 0x11, 0xbf, - 0xb0, 0xb1, 0x40, 0x32, 0x89, 0x61, 0xfa, 0xa9, 0x66, 0x24, 0xca, 0xfb, 0x12, 0x87, 0x04, 0x4c, - 0x41, 0xde, 0x3e, 0xbe, 0x91, 0x51, 0x30, 0xd1, 0x2f, 0xfe, 0x30, 0xf2, 0xad, 0x37, 0x32, 0x2d, - 0xe3, 0x84, 0x15, 0x66, 0xfd, 0xeb, 0x36, 0x94, 0x6d, 0xaa, 0x9b, 0xca, 0x82, 0x34, 0xca, 0x3c, - 0x96, 0x1d, 0xaa, 0x9b, 0x98, 0x93, 0xb1, 0x39, 0xd8, 0x5f, 0x62, 0x2a, 0xef, 0x9f, 0x31, 0xc7, - 0x0e, 0x27, 0xc1, 0x92, 0x14, 0xad, 0x40, 0x85, 0x77, 0xd1, 0x95, 0x5b, 0x99, 0x2b, 0x26, 0xc9, - 0xc3, 0x9b, 0xeb, 0x58, 0x10, 0xa2, 0x1f, 0xc4, 0xfd, 0xfa, 0xc5, 0x4c, 0x36, 0x39, 0xc2, 0x93, - 0x68, 0xd2, 0xb3, 0x99, 0xfc, 0x80, 0x7a, 0x44, 0x59, 0x3a, 0x63, 0xa6, 0x3d, 0x46, 0x81, 0x05, - 0x21, 0x5b, 0x10, 0x7f, 0x30, 0x95, 0xdb, 0x67, 0x2c, 0x88, 0xb3, 0x98, 0x58, 0x92, 0xa2, 0x8d, - 0xcc, 0x1b, 0xf3, 0x1e, 0x67, 0x9d, 0x1f, 0xc3, 0x9a, 0xff, 0xae, 0x1c, 0x6d, 0xc3, 0x24, 0x1f, - 0xb2, 0x12, 0x49, 0x88, 0x59, 0xce, 0xbc, 0xa9, 0x1a, 0x11, 0x43, 0x4c, 0x29, 0x68, 0xc2, 0x4f, - 0x0e, 0xd1, 0x3a, 0xaf, 0x49, 0x1d, 0xfa, 0xdc, 0x26, 0x66, 0x9f, 0x28, 0x2b, 0x67, 0xa8, 0xb3, - 0x16, 0xd3, 0xe1, 0x24, 0x13, 0xda, 0x82, 0x56, 0x62, 0x68, 0x2a, 0x1f, 0x64, 0x5e, 0xdb, 0x8d, - 0x11, 0x62, 0xe2, 0x14, 0x1b, 0xb3, 0x69, 0x57, 0x64, 0xae, 0xca, 0x6a, 0xc6, 0xa6, 0x65, 0x46, - 0x8b, 0x43, 0x02, 0x16, 0x52, 0xdd, 0x30, 0xcb, 0x55, 0x3e, 0xcc, 0x84, 0xd4, 0x28, 0xff, 0xc5, - 0x31, 0x51, 0xfa, 0x36, 0xb8, 0x73, 0xfe, 0xdb, 0xe0, 0xd3, 0x73, 0xdd, 0x06, 0x9f, 0xbd, 0xee, - 0x36, 0xf8, 0xdd, 0xc2, 0xe5, 0xaf, 0x03, 0xf4, 0xa3, 0x64, 0xee, 0x98, 0xa8, 0x0b, 0x8b, 0x67, - 0xd4, 0x85, 0x57, 0x23, 0x8e, 0xc4, 0xeb, 0xc4, 0x8f, 0xa0, 0xcc, 0x1c, 0x0c, 0xdd, 0x86, 0x7a, - 0x54, 0xf7, 0x16, 0xc6, 0xd5, 0xbd, 0x11, 0x89, 0xfa, 0xab, 0x22, 0x54, 0x85, 0x63, 0xa2, 0x2f, - 0x46, 0xde, 0x10, 0xbd, 0x7d, 0x86, 0x1f, 0x8f, 0xbe, 0x20, 0x12, 0x35, 0x00, 0x7f, 0x43, 0xe1, - 0x69, 0xe2, 0x63, 0x99, 0x83, 0xd3, 0x80, 0x88, 0xa6, 0x49, 0x99, 0xd5, 0x00, 0x02, 0xf7, 0x84, - 0xa1, 0xd6, 0x19, 0x46, 0xfd, 0x8f, 0x42, 0xfc, 0x4a, 0x69, 0x1a, 0x2a, 0xa2, 0xcd, 0x2d, 0x72, - 0x5b, 0x31, 0x40, 0x0b, 0xd0, 0x1e, 0x58, 0x8e, 0xe6, 0xd3, 0xa1, 0x67, 0xa4, 0xfb, 0x91, 0x93, - 0x03, 0xcb, 0xd9, 0xe3, 0x60, 0xd1, 0x2d, 0x58, 0x10, 0x7d, 0xd8, 0x14, 0x65, 0x49, 0x52, 0xea, - 0x2f, 0x92, 0x94, 0x4b, 0x80, 0x04, 0x95, 0xa9, 0x99, 0xd4, 0xf0, 0xb5, 0x80, 0x06, 0xba, 0xcd, - 0x2f, 0xb4, 0x32, 0x6e, 0x4b, 0xcc, 0x26, 0x35, 0xfc, 0x7d, 0x06, 0x47, 0x3d, 0xb8, 0x1a, 0x52, - 0xf3, 0xe5, 0x48, 0xf2, 0x0a, 0x27, 0xef, 0x48, 0x14, 0x5f, 0x8e, 0xa0, 0xef, 0xc2, 0x84, 0x4c, - 0xf4, 0x35, 0x93, 0xd8, 0x81, 0xfc, 0xde, 0x0c, 0x37, 0x45, 0x46, 0xbf, 0xc9, 0x40, 0xea, 0xbf, - 0x15, 0xa1, 0xc2, 0xc3, 0xd4, 0x19, 0xb5, 0x4c, 0x61, 0x4c, 0x0f, 0xe5, 0x2b, 0x98, 0x8a, 0x8a, - 0x4d, 0x5e, 0x2d, 0x87, 0xaf, 0x12, 0x16, 0xc6, 0x47, 0xc3, 0x5e, 0xaa, 0x0a, 0xc5, 0x93, 0x07, - 0xc9, 0xa1, 0x8f, 0x7e, 0x02, 0x28, 0xae, 0x5f, 0x65, 0x75, 0x1d, 0x36, 0xb1, 0x16, 0xcf, 0x21, - 0x75, 0x43, 0xf2, 0xe0, 0xce, 0x41, 0x06, 0xe2, 0xab, 0x5f, 0xc0, 0xc4, 0x79, 0x4b, 0xe0, 0x69, - 0xa8, 0x24, 0x0f, 0x58, 0x0c, 0xd4, 0x75, 0x68, 0x67, 0xe7, 0xb9, 0xb0, 0x8c, 0xff, 0x2c, 0xc4, - 0xef, 0x69, 0xcf, 0x7a, 0x11, 0x9a, 0x73, 0x8d, 0xe4, 0xda, 0xf9, 0x05, 0xeb, 0x4f, 0xf5, 0xf4, - 0x75, 0x66, 0x7e, 0x0b, 0x3a, 0xe2, 0x5a, 0x4c, 0x5a, 0xa4, 0xf0, 0x9b, 0x29, 0x81, 0x88, 0x0d, - 0x72, 0x09, 0x90, 0xa4, 0x4d, 0xda, 0x63, 0x49, 0x98, 0xaf, 0xc0, 0xc4, 0xe6, 0xa8, 0xd6, 0xa0, - 0xc2, 0xef, 0x29, 0xf5, 0xaf, 0x0b, 0x50, 0x15, 0x37, 0xd6, 0xb9, 0x3d, 0x5d, 0x90, 0xe7, 0xbc, - 0x0a, 0x3e, 0xcf, 0x7a, 0xc4, 0xad, 0x98, 0xb3, 0x1e, 0x81, 0x48, 0xad, 0x47, 0xd2, 0xe6, 0xac, - 0x47, 0x60, 0x12, 0xeb, 0xf9, 0xbd, 0x42, 0xfa, 0x6b, 0xb2, 0x8b, 0x3b, 0xd0, 0x77, 0x16, 0x72, - 0xd7, 0x60, 0x22, 0x75, 0x01, 0x5f, 0x5c, 0x17, 0xf5, 0x0b, 0x68, 0x26, 0xae, 0xcd, 0x4b, 0x08, - 0xf8, 0x12, 0x5a, 0xc9, 0x7b, 0xf7, 0xe2, 0x12, 0xba, 0x3f, 0x47, 0x50, 0x15, 0x5f, 0xbf, 0xa0, - 0x85, 0x54, 0x2d, 0x32, 0xdd, 0x93, 0xbf, 0x26, 0xc8, 0x29, 0x43, 0xee, 0xe6, 0x97, 0x21, 0x33, - 0x31, 0xcb, 0xf8, 0x0a, 0xe4, 0xce, 0x48, 0x05, 0xa2, 0x64, 0x67, 0xca, 0x29, 0x3e, 0x3e, 0x1e, - 0x2d, 0x3e, 0xd4, 0x91, 0xd9, 0x7e, 0x53, 0x77, 0xe4, 0xd5, 0x1d, 0xe7, 0xa8, 0x12, 0x7a, 0x99, - 0x2a, 0x61, 0x36, 0xf3, 0x61, 0x54, 0xb6, 0x40, 0x58, 0x48, 0x15, 0x08, 0xd3, 0x59, 0xea, 0x44, - 0x6d, 0xd0, 0xcb, 0xd4, 0x06, 0xb3, 0x79, 0xb4, 0x89, 0xb2, 0x60, 0x31, 0x5d, 0x16, 0xcc, 0x64, - 0xc9, 0x53, 0x15, 0xc1, 0x07, 0xd9, 0x8a, 0xe0, 0x5a, 0x2e, 0x79, 0xb2, 0x18, 0x58, 0x4c, 0x17, - 0x03, 0x23, 0xf2, 0x53, 0x75, 0x40, 0x2f, 0x53, 0x07, 0xcc, 0xe6, 0x52, 0xc7, 0x25, 0xc0, 0xe7, - 0xb9, 0x25, 0xc0, 0x8d, 0x51, 0xae, 0x31, 0xd9, 0xff, 0xe6, 0x98, 0xec, 0xff, 0xcd, 0x5c, 0x09, - 0xe3, 0x12, 0xff, 0xdf, 0x64, 0xdb, 0xdf, 0xd7, 0x6c, 0xfb, 0x6f, 0xe2, 0x6c, 0xfb, 0xee, 0xc8, - 0x1d, 0x7c, 0x33, 0xdf, 0x33, 0xbe, 0x93, 0x44, 0xfb, 0xef, 0x7e, 0xfd, 0x12, 0x6d, 0xf5, 0x93, - 0x4b, 0xe7, 0xd0, 0xea, 0xe3, 0x38, 0x1d, 0xbc, 0x78, 0xfe, 0x80, 0xa0, 0x3c, 0x60, 0x01, 0x44, - 0xbc, 0x0b, 0xe6, 0xcf, 0x71, 0x96, 0xf5, 0x3b, 0xa5, 0x28, 0xcb, 0x5a, 0x81, 0xe9, 0xe8, 0xd3, - 0xd3, 0xe4, 0xfa, 0xc5, 0xfb, 0x1a, 0x14, 0xe1, 0xe2, 0x1d, 0x58, 0x85, 0x99, 0x98, 0x23, 0xb9, - 0x07, 0xe2, 0x60, 0xaf, 0x46, 0xc8, 0x44, 0xb9, 0xb1, 0x04, 0xc8, 0xf4, 0x98, 0x75, 0xa7, 0xe6, - 0x90, 0xd9, 0x93, 0xc4, 0xa4, 0xf6, 0x38, 0xa4, 0x4e, 0xca, 0x17, 0x47, 0xd2, 0x91, 0xa8, 0x84, - 0xf4, 0xaf, 0xa0, 0x1d, 0x7f, 0x8b, 0x22, 0x43, 0x52, 0x25, 0xf3, 0x95, 0x7a, 0x2a, 0x14, 0x46, - 0x1f, 0xde, 0x7a, 0x32, 0x36, 0x4d, 0xb9, 0x69, 0x80, 0xaa, 0xc1, 0x54, 0x86, 0x06, 0xa9, 0xfc, - 0x03, 0x2c, 0x73, 0x68, 0x48, 0x2f, 0x6a, 0xe1, 0x68, 0x9c, 0x9f, 0xd0, 0x33, 0x0e, 0xf9, 0x33, - 0x39, 0x51, 0xa7, 0x34, 0x70, 0x34, 0x56, 0x9f, 0xa6, 0x13, 0xc4, 0x71, 0x3e, 0x5f, 0xb8, 0xa8, - 0xcf, 0x4f, 0x65, 0xd2, 0xbd, 0x5b, 0x8b, 0x50, 0xe1, 0x3f, 0x07, 0x44, 0x00, 0xd5, 0xdd, 0x27, - 0xeb, 0x3b, 0xdb, 0x1b, 0xed, 0x2b, 0xa8, 0x09, 0xb5, 0x5d, 0xbc, 0xfd, 0x74, 0x6d, 0x7f, 0xab, - 0x5d, 0x40, 0x0d, 0xa8, 0xec, 0x3c, 0xde, 0x58, 0xdb, 0x69, 0x17, 0x57, 0x1f, 0x40, 0x5d, 0xfe, - 0x5c, 0xcb, 0x43, 0x9f, 0x43, 0x4d, 0x3e, 0xa3, 0xf8, 0xc2, 0x4a, 0xff, 0x90, 0x50, 0x55, 0x46, - 0x11, 0x22, 0xc9, 0x59, 0x29, 0xac, 0xee, 0x40, 0x5d, 0x7e, 0x0a, 0xe8, 0xa1, 0x2f, 0xa1, 0x26, - 0x9f, 0x13, 0xb2, 0xd2, 0x1f, 0x74, 0x26, 0x64, 0x65, 0xbe, 0x20, 0x5c, 0x28, 0xac, 0x14, 0x56, - 0x8f, 0x60, 0x32, 0xfd, 0x91, 0x1d, 0x7a, 0x0a, 0x53, 0xfc, 0x21, 0x02, 0xfb, 0xe8, 0x66, 0x32, - 0xb0, 0x8e, 0x7e, 0xaa, 0xa7, 0xce, 0x8d, 0xc5, 0x27, 0x66, 0x7a, 0x0e, 0xd5, 0x1d, 0xf1, 0xab, - 0xb2, 0x5e, 0x94, 0x73, 0x4e, 0x65, 0xec, 0x48, 0xcd, 0x02, 0x18, 0x27, 0xfa, 0x2c, 0xdd, 0x34, - 0x9f, 0xce, 0x2b, 0x57, 0xd4, 0x5c, 0x28, 0x9f, 0xf8, 0xcf, 0xa2, 0xcf, 0xd4, 0x3e, 0x88, 0xdf, - 0x4c, 0xb5, 0xb3, 0x6f, 0x19, 0xd4, 0x11, 0x08, 0x9f, 0xfb, 0x7f, 0x57, 0xd7, 0xf5, 0xcf, 0xbf, - 0xf9, 0xe7, 0x9b, 0x57, 0xbe, 0xf9, 0xe5, 0xcd, 0xc2, 0xdf, 0xfe, 0xf2, 0x66, 0xe1, 0x4f, 0xfe, - 0xe5, 0x66, 0xe1, 0x27, 0x4b, 0xe7, 0xfa, 0x99, 0x9a, 0x94, 0x77, 0x50, 0xe5, 0xa0, 0x0f, 0xff, - 0x27, 0x00, 0x00, 0xff, 0xff, 0x5d, 0x3e, 0xa7, 0x21, 0xb3, 0x3c, 0x00, 0x00, + // 4643 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5b, 0x3d, 0x6c, 0x1c, 0x59, + 0x72, 0xd6, 0xfc, 0xcf, 0xd4, 0x0c, 0xc9, 0x99, 0x27, 0x8a, 0x6a, 0xb5, 0x76, 0x25, 0xed, 0xec, + 0xae, 0x8f, 0x2b, 0x52, 0x43, 0x2e, 0x57, 0x7b, 0xb7, 0x2b, 0xef, 0x1f, 0x87, 0xa4, 0x8e, 0xd4, + 0x51, 0x12, 0xf7, 0x91, 0x12, 0xec, 0x4b, 0x1a, 0xcd, 0xee, 0xc7, 0x61, 0x8b, 0x3d, 0xfd, 0x7a, + 0xbb, 0x7b, 0x28, 0xf1, 0x22, 0x03, 0x4e, 0x0c, 0x18, 0xb0, 0x93, 0x4b, 0x0c, 0x27, 0xce, 0x0c, + 0x1b, 0x70, 0xe0, 0xe8, 0x80, 0x0b, 0x1c, 0x39, 0x58, 0x38, 0xb2, 0x1d, 0x18, 0x8e, 0x64, 0xf8, + 0xec, 0xd0, 0x8e, 0xce, 0x06, 0x6c, 0xc1, 0x81, 0xf1, 0x7e, 0xfa, 0x77, 0x7a, 0x28, 0x92, 0xbb, + 0xb0, 0x17, 0x87, 0x0b, 0x24, 0xf6, 0xab, 0xfa, 0xea, 0xfd, 0x56, 0xd5, 0xab, 0xaa, 0xee, 0x81, + 0xee, 0x80, 0x2e, 0xb9, 0x1e, 0x0d, 0xa8, 0x41, 0x6d, 0x7f, 0xc9, 0x1b, 0x39, 0x81, 0x35, 0x24, + 0xe1, 0xdf, 0x1e, 0xe7, 0xa0, 0x9a, 0x6c, 0xaa, 0x37, 0xf6, 0x3d, 0x7a, 0x44, 0xbc, 0x48, 0x20, + 0x7a, 0x10, 0x40, 0xf5, 0x96, 0x41, 0x1d, 0x7f, 0x34, 0x3c, 0x05, 0x91, 0x1e, 0xce, 0xd0, 0xdd, + 0x60, 0xe4, 0x91, 0xf0, 0x6f, 0xd8, 0x4b, 0x0a, 0x63, 0x12, 0xcf, 0x3a, 0x26, 0xf2, 0x8f, 0x44, + 0xbc, 0x91, 0x42, 0x1c, 0xd8, 0xf4, 0x39, 0xff, 0x4f, 0x72, 0x6f, 0xa7, 0xb8, 0x43, 0x3d, 0x20, + 0x9e, 0xa5, 0xdb, 0xd6, 0x4f, 0x48, 0xf2, 0x59, 0x62, 0xd5, 0x14, 0x96, 0xba, 0xfc, 0x5f, 0xee, + 0x5c, 0xfd, 0xc3, 0xd1, 0xc1, 0x81, 0x4d, 0xc2, 0xbf, 0x12, 0x33, 0x3b, 0xa0, 0x03, 0xca, 0x1f, + 0x97, 0xd8, 0x93, 0xa0, 0x76, 0xff, 0xaa, 0x00, 0x9d, 0x3d, 0xdd, 0x3f, 0xda, 0x25, 0xde, 0xb1, + 0x65, 0x90, 0x35, 0xea, 0x1c, 0x58, 0x03, 0x74, 0x03, 0x9a, 0x36, 0x1d, 0x68, 0x07, 0x96, 0x4d, + 0xb4, 0x03, 0x53, 0x29, 0xdc, 0x2a, 0xcc, 0x57, 0x70, 0xc3, 0xa6, 0x83, 0xfb, 0x96, 0x4d, 0xee, + 0x9b, 0xe8, 0x3a, 0x34, 0x02, 0xdd, 0x3f, 0xd2, 0x1c, 0x7d, 0x48, 0x94, 0xe2, 0xad, 0xc2, 0x7c, + 0x03, 0xd7, 0x19, 0xe1, 0x91, 0x3e, 0x24, 0xe8, 0x1a, 0xd4, 0x47, 0xa6, 0xaf, 0xb9, 0x7a, 0x70, + 0xa8, 0x94, 0x38, 0xaf, 0x36, 0x32, 0xfd, 0x1d, 0x3d, 0x38, 0x44, 0x0b, 0xd0, 0x31, 0xa8, 0x13, + 0xe8, 0x96, 0x43, 0x3c, 0xcd, 0x21, 0xc1, 0x73, 0xea, 0x1d, 0x29, 0x65, 0x8e, 0x69, 0x47, 0x8c, + 0x47, 0x82, 0x8e, 0xde, 0x81, 0x8a, 0x6b, 0xeb, 0x0e, 0x51, 0xaa, 0xb7, 0x0a, 0xf3, 0xd3, 0x2b, + 0xd3, 0xbd, 0xf0, 0xa8, 0x77, 0x18, 0x15, 0x0b, 0x66, 0xf7, 0xbf, 0xcb, 0x30, 0xbd, 0x2b, 0x16, + 0x8a, 0xc9, 0x57, 0x23, 0xe2, 0x07, 0x68, 0x0b, 0x6a, 0xcf, 0xe8, 0xc8, 0x73, 0x74, 0x9b, 0xcf, + 0xbc, 0xd1, 0x5f, 0x7a, 0xf5, 0xf2, 0xe6, 0xc2, 0x80, 0xf6, 0x06, 0xfa, 0x4f, 0x48, 0x10, 0x90, + 0x9e, 0x49, 0x8e, 0x97, 0x0c, 0xea, 0x91, 0xa5, 0x8c, 0x92, 0xf4, 0x1e, 0x08, 0x31, 0x1c, 0xca, + 0xa3, 0x39, 0xa8, 0x7a, 0xc4, 0xb5, 0xf5, 0x13, 0xbe, 0xca, 0x3a, 0x96, 0x2d, 0xb6, 0xc6, 0xfd, + 0x91, 0x65, 0x9b, 0x9a, 0x65, 0x86, 0x6b, 0xe4, 0xed, 0x2d, 0x13, 0xdd, 0x87, 0x2a, 0x3d, 0x38, + 0xf0, 0x49, 0xc0, 0x17, 0x56, 0xea, 0xf7, 0x5e, 0xbd, 0xbc, 0x79, 0xfb, 0x2c, 0x83, 0x3f, 0xe6, + 0x52, 0x58, 0x4a, 0xa3, 0x87, 0x00, 0xc4, 0x31, 0x35, 0xd9, 0x57, 0xe5, 0x42, 0x7d, 0x35, 0x88, + 0x63, 0x8a, 0x47, 0xb4, 0x00, 0x15, 0x4f, 0x77, 0x06, 0x62, 0x37, 0x9b, 0x2b, 0x33, 0x3d, 0xae, + 0x86, 0x98, 0x91, 0x76, 0x5d, 0x62, 0xf4, 0xcb, 0x5f, 0xbf, 0xbc, 0x79, 0x09, 0x0b, 0x0c, 0xda, + 0x85, 0xa6, 0x41, 0xa9, 0x67, 0x5a, 0x8e, 0x1e, 0x50, 0x4f, 0xa9, 0xf1, 0x5d, 0x7c, 0xff, 0xd5, + 0xcb, 0x9b, 0x77, 0xf2, 0x06, 0x1f, 0x33, 0xa5, 0xde, 0xee, 0xa1, 0xee, 0x99, 0x5b, 0xeb, 0x38, + 0xd9, 0x0b, 0x5a, 0x06, 0xf0, 0x88, 0x4f, 0xed, 0x51, 0x60, 0x51, 0x47, 0xa9, 0xf3, 0x69, 0xb4, + 0x7b, 0x91, 0xcc, 0x26, 0xd1, 0x4d, 0xe2, 0xe1, 0x04, 0x06, 0xbd, 0x0d, 0x53, 0x52, 0x87, 0x35, + 0xcb, 0x31, 0xc9, 0x0b, 0xa5, 0x71, 0xab, 0x30, 0x3f, 0x85, 0x5b, 0x92, 0xb8, 0xc5, 0x68, 0xe8, + 0x2e, 0x00, 0xb7, 0x38, 0x9d, 0x77, 0x0b, 0xbc, 0xdb, 0x59, 0xb1, 0xba, 0x35, 0x6a, 0xdb, 0xc4, + 0x60, 0x74, 0xb6, 0x44, 0x9c, 0xc0, 0xa1, 0x35, 0x98, 0x89, 0x4d, 0x4c, 0x88, 0x36, 0xb9, 0xe8, + 0x35, 0x21, 0xfa, 0x30, 0xcd, 0xe4, 0xf2, 0x59, 0x89, 0xee, 0xdf, 0x97, 0x61, 0x26, 0xd2, 0x3d, + 0xdf, 0xa5, 0x8e, 0x4f, 0xd0, 0x3c, 0x54, 0xfd, 0x40, 0x0f, 0x46, 0x3e, 0xd7, 0xbd, 0xe9, 0x95, + 0x76, 0x2f, 0xdc, 0x9e, 0xde, 0x2e, 0xa7, 0x63, 0xc9, 0x67, 0xc8, 0x43, 0xbe, 0x66, 0xae, 0x5b, + 0x79, 0x7b, 0x21, 0xf9, 0xe8, 0x5d, 0x98, 0x0e, 0x88, 0x37, 0xb4, 0x1c, 0xdd, 0xd6, 0x88, 0xe7, + 0x51, 0x4f, 0xea, 0xdc, 0x54, 0x48, 0xdd, 0x60, 0x44, 0xf4, 0x25, 0xb4, 0x3c, 0xa2, 0x9b, 0x5a, + 0x70, 0xe8, 0xd1, 0xd1, 0xe0, 0xf0, 0x82, 0xfa, 0xd7, 0x64, 0x7d, 0xec, 0x89, 0x2e, 0x98, 0x12, + 0x3e, 0xf7, 0xac, 0x80, 0x68, 0x6c, 0x26, 0x17, 0x55, 0x42, 0xde, 0x03, 0x5b, 0x12, 0xda, 0x82, + 0x8a, 0xee, 0x11, 0x47, 0xe7, 0x4a, 0xd8, 0xea, 0x7f, 0xf0, 0xea, 0xe5, 0xcd, 0xa5, 0x81, 0x15, + 0x1c, 0x8e, 0xf6, 0x7b, 0x06, 0x1d, 0x2e, 0x11, 0x3f, 0x18, 0xe9, 0xde, 0x89, 0x70, 0x93, 0x63, + 0x8e, 0xb3, 0xb7, 0xca, 0x44, 0xb1, 0xe8, 0x01, 0xbd, 0x0b, 0x65, 0x93, 0x1a, 0xbe, 0x52, 0xbb, + 0x55, 0x9a, 0x6f, 0xae, 0x34, 0xc5, 0xa9, 0xed, 0xda, 0x96, 0x41, 0xa4, 0x2a, 0x73, 0x36, 0xda, + 0x84, 0x9a, 0xb0, 0x20, 0x5f, 0xa9, 0xdf, 0x2a, 0x5d, 0x60, 0xf6, 0xa1, 0x38, 0xd3, 0xb3, 0xd1, + 0xc8, 0x32, 0x35, 0x57, 0xf7, 0x02, 0x5f, 0x69, 0xf0, 0x61, 0xa5, 0x15, 0x3d, 0x79, 0xb2, 0xb5, + 0xbe, 0xc3, 0xc8, 0x72, 0xe8, 0x06, 0x03, 0x72, 0x02, 0x53, 0x7a, 0x57, 0x37, 0x8e, 0x88, 0xa9, + 0x1d, 0x91, 0x13, 0x05, 0x26, 0x4d, 0xb6, 0x21, 0x40, 0x3f, 0x22, 0x27, 0x5d, 0x13, 0x3a, 0x98, + 0x1a, 0x47, 0xfe, 0x7a, 0x7f, 0x9d, 0xf8, 0x86, 0x67, 0xb9, 0xcc, 0x76, 0x16, 0x01, 0x79, 0x8c, + 0x68, 0xee, 0x6b, 0xc4, 0x39, 0xd6, 0x86, 0x64, 0xe8, 0x06, 0x1e, 0xd7, 0xb0, 0x2a, 0x6e, 0x4b, + 0xce, 0x86, 0x73, 0xfc, 0x90, 0xd3, 0xd1, 0x5b, 0xd0, 0x0a, 0xd1, 0xdc, 0x0b, 0x0b, 0x0f, 0xdd, + 0x94, 0x34, 0xe6, 0x89, 0xbb, 0x3f, 0x2d, 0x42, 0x63, 0x2d, 0xf4, 0xb8, 0xe8, 0x2a, 0xd4, 0x2c, + 0x57, 0xd3, 0x4d, 0x53, 0xf4, 0xd9, 0xc0, 0x55, 0xcb, 0x5d, 0x35, 0x4d, 0x0f, 0x7d, 0x1f, 0xa6, + 0xa4, 0x9b, 0xd6, 0x5c, 0xca, 0xd6, 0x5d, 0xe4, 0x2b, 0xe8, 0x88, 0x15, 0x48, 0x4f, 0xbd, 0x43, + 0xbd, 0x00, 0xb7, 0x9c, 0xb8, 0xe1, 0xa3, 0x5d, 0xe8, 0x0c, 0x75, 0xd7, 0x25, 0xa6, 0x76, 0x48, + 0xfd, 0x40, 0xca, 0x96, 0xb8, 0xec, 0xf7, 0x22, 0x3f, 0x1e, 0x8d, 0xdf, 0x7b, 0xc8, 0xb1, 0x9b, + 0xd4, 0x0f, 0xb8, 0xf8, 0x86, 0x13, 0x78, 0x27, 0xcc, 0xdc, 0x52, 0x54, 0xf4, 0x26, 0xc0, 0xc8, + 0xd7, 0x07, 0x44, 0xf3, 0xf4, 0x80, 0x70, 0xed, 0x2e, 0xe2, 0x06, 0xa7, 0x60, 0x3d, 0x20, 0x6a, + 0x1f, 0x66, 0xf3, 0xfa, 0x41, 0x6d, 0x28, 0xb1, 0xbd, 0x2f, 0x70, 0xdf, 0xc1, 0x1e, 0xd1, 0x2c, + 0x54, 0x8e, 0x75, 0x7b, 0x14, 0x5e, 0x5d, 0xa2, 0x71, 0xaf, 0xf8, 0x51, 0xa1, 0xfb, 0xe7, 0x45, + 0xe8, 0xac, 0x89, 0x2b, 0x5e, 0xde, 0x26, 0x1b, 0x2f, 0x98, 0xef, 0x64, 0x77, 0x9f, 0x66, 0x93, + 0x63, 0x62, 0x4b, 0xb3, 0x9e, 0xee, 0xb1, 0xdb, 0x77, 0x9b, 0x0e, 0x7a, 0xdb, 0x8c, 0x8a, 0xeb, + 0x36, 0x1d, 0xf0, 0x27, 0xb4, 0x15, 0x1f, 0x95, 0x19, 0x1d, 0xa0, 0x34, 0x71, 0x35, 0x5a, 0xfb, + 0xd8, 0x11, 0xe3, 0x8e, 0x94, 0x4a, 0x9c, 0xfa, 0x16, 0xb4, 0xfc, 0x40, 0xf7, 0x02, 0xcd, 0xa0, + 0xc3, 0xa1, 0x15, 0x70, 0xab, 0x6f, 0xae, 0xfc, 0x46, 0xbc, 0x81, 0xd9, 0x99, 0x32, 0x17, 0xe3, + 0x05, 0x6b, 0x1c, 0x8d, 0x9b, 0x7e, 0xdc, 0x50, 0x31, 0x34, 0x13, 0x3c, 0xb4, 0x06, 0x48, 0x76, + 0xa2, 0x19, 0x87, 0xc4, 0x38, 0x72, 0xa9, 0xe5, 0x04, 0x7c, 0x69, 0xcc, 0x79, 0x46, 0x1e, 0x6b, + 0x2d, 0xe2, 0xe1, 0x8e, 0xc4, 0xc7, 0xa4, 0xee, 0xff, 0x94, 0x01, 0x45, 0x53, 0x10, 0xee, 0x8f, + 0xed, 0xd6, 0x32, 0x34, 0xa2, 0xbb, 0x5c, 0x76, 0x89, 0xc6, 0xcf, 0x1c, 0xc7, 0x20, 0x74, 0x0f, + 0xaa, 0xd4, 0x25, 0x0e, 0x31, 0xe5, 0x36, 0x75, 0xc7, 0x57, 0x18, 0x75, 0xdf, 0x7b, 0xcc, 0x91, + 0x58, 0x4a, 0xa0, 0x2f, 0xa0, 0x2e, 0x63, 0x32, 0x53, 0xee, 0xcf, 0x3b, 0xa7, 0x49, 0x4b, 0x92, + 0x89, 0x23, 0x29, 0x74, 0x1f, 0x20, 0xb1, 0x07, 0xe5, 0x49, 0x7b, 0x9c, 0xe8, 0x23, 0xde, 0x95, + 0x84, 0xa4, 0xfa, 0x10, 0xaa, 0x62, 0x6e, 0xdf, 0xca, 0xee, 0xaa, 0x4f, 0xa1, 0x1e, 0x4e, 0x96, + 0x69, 0xfe, 0x11, 0x39, 0xd1, 0x84, 0x93, 0xe0, 0x1d, 0xb5, 0x70, 0xe3, 0x88, 0x9c, 0xec, 0x70, + 0x02, 0x0b, 0xab, 0x98, 0x57, 0xb2, 0xd8, 0xa5, 0xe4, 0x87, 0xa8, 0x22, 0x47, 0xb5, 0x63, 0x86, + 0x00, 0xab, 0xcf, 0x01, 0xe2, 0x51, 0xd0, 0x2d, 0xa8, 0xb0, 0xeb, 0xc8, 0x97, 0xb3, 0x03, 0xae, + 0xd6, 0xec, 0xa2, 0xf2, 0xb1, 0x60, 0xa0, 0x1f, 0x42, 0xd3, 0xa5, 0xb6, 0xad, 0x79, 0xc4, 0x1f, + 0xd9, 0x01, 0xef, 0x76, 0xfa, 0xf4, 0xfd, 0xd9, 0xa1, 0xb6, 0x8d, 0x39, 0x1a, 0x83, 0x1b, 0x3d, + 0x77, 0x1f, 0x01, 0xc4, 0x1c, 0xd4, 0x84, 0xda, 0xd6, 0xa3, 0xa7, 0xab, 0xdb, 0x5b, 0xeb, 0xed, + 0x4b, 0xa8, 0x01, 0x15, 0xbc, 0xb1, 0xba, 0xfe, 0xdb, 0xed, 0x02, 0x9a, 0x82, 0xc6, 0xa3, 0xc7, + 0x7b, 0x9a, 0x68, 0x16, 0x51, 0x0b, 0xea, 0x6b, 0x8f, 0x1f, 0x6f, 0x6b, 0x8f, 0xef, 0xdf, 0x6f, + 0x97, 0x98, 0x10, 0xde, 0xd8, 0xdd, 0x5b, 0xc5, 0x7b, 0xed, 0x72, 0xf7, 0xdf, 0x0a, 0xd0, 0x5e, + 0xe7, 0xb1, 0xf6, 0x77, 0xc0, 0x54, 0x57, 0xa0, 0xcc, 0x14, 0x52, 0xaa, 0xe0, 0x8d, 0x48, 0x38, + 0x3b, 0x41, 0xae, 0xbe, 0x98, 0x63, 0xd5, 0x45, 0x28, 0xb3, 0x16, 0x7a, 0x07, 0xa6, 0xfd, 0xaf, + 0x6c, 0x76, 0xcb, 0x1e, 0x1f, 0xf8, 0xda, 0xc8, 0xb3, 0xa4, 0x13, 0x6e, 0x09, 0xea, 0xd3, 0x03, + 0xff, 0x89, 0x67, 0x75, 0xff, 0xa3, 0x04, 0x9d, 0xb0, 0xb7, 0x6f, 0x62, 0x6c, 0x1f, 0x67, 0x8c, + 0xed, 0xad, 0xb1, 0xb9, 0x4e, 0xb4, 0xb5, 0x3e, 0x34, 0xdc, 0xd1, 0xbe, 0x6d, 0xf9, 0x87, 0x39, + 0xc6, 0x36, 0x2e, 0xbd, 0x13, 0x62, 0x71, 0x2c, 0x86, 0x3e, 0x81, 0xda, 0x81, 0x3d, 0xe2, 0x3d, + 0x94, 0x33, 0xc6, 0x3e, 0xde, 0xc3, 0x7d, 0x81, 0xc4, 0xa1, 0xc8, 0xb7, 0x6d, 0x63, 0x01, 0x34, + 0xa2, 0x49, 0xb2, 0xa4, 0x66, 0xa8, 0xbf, 0xd0, 0x0c, 0x9b, 0x1a, 0x47, 0xf2, 0x6a, 0xad, 0x0f, + 0xf5, 0x17, 0x6b, 0xac, 0x9d, 0xb1, 0xc0, 0xe2, 0x99, 0x2c, 0xb0, 0x34, 0xc1, 0x02, 0x17, 0xa0, + 0x26, 0x17, 0xf6, 0x7a, 0xf3, 0xeb, 0xfe, 0x61, 0x01, 0xae, 0xc4, 0xc1, 0xe8, 0x77, 0x40, 0xd5, + 0xbb, 0x3f, 0x2f, 0xc0, 0x5c, 0x6a, 0x46, 0xdf, 0x44, 0x1b, 0x57, 0x63, 0x75, 0x10, 0x93, 0x89, + 0xc3, 0x83, 0xfc, 0x31, 0xc6, 0x75, 0xe2, 0x5c, 0xdb, 0xf9, 0xf3, 0x32, 0x4c, 0xaf, 0xd1, 0xe1, + 0xbe, 0xe5, 0x44, 0xe9, 0xe2, 0xb2, 0x34, 0x5d, 0x21, 0xf3, 0x46, 0x62, 0xbe, 0x49, 0x58, 0xc2, + 0x70, 0xd1, 0x1d, 0x28, 0xe9, 0x66, 0x38, 0xe1, 0xeb, 0x93, 0x04, 0x56, 0x4d, 0x13, 0x33, 0x9c, + 0xfa, 0x0f, 0x45, 0x69, 0xe8, 0x5f, 0x40, 0x7d, 0xdf, 0x72, 0x4c, 0xcb, 0x19, 0xb0, 0x19, 0x96, + 0xd2, 0x77, 0xd5, 0xf8, 0x68, 0xbd, 0xbe, 0x00, 0xe3, 0x48, 0x4a, 0xfd, 0xfd, 0x22, 0xd4, 0x24, + 0x15, 0x21, 0x28, 0x1f, 0x8c, 0x6c, 0x71, 0xf4, 0x75, 0xcc, 0x9f, 0xc3, 0x58, 0x87, 0x45, 0x69, + 0x0d, 0x11, 0xeb, 0x7c, 0x04, 0x4d, 0xd7, 0xa3, 0xcf, 0x44, 0x1a, 0x14, 0xc6, 0x60, 0x6d, 0x11, + 0xbf, 0xed, 0x44, 0x0c, 0x19, 0x86, 0x26, 0xa1, 0xe8, 0x53, 0x68, 0xfa, 0xc6, 0x21, 0x19, 0xea, + 0xda, 0x33, 0x9f, 0x3a, 0xdc, 0x5a, 0x5b, 0xfd, 0x37, 0x5e, 0xbd, 0xbc, 0xa9, 0x10, 0xc7, 0xa0, + 0x6c, 0x0a, 0x4b, 0x8c, 0xd1, 0xc3, 0xfa, 0xf3, 0x87, 0xc4, 0xe7, 0x61, 0x18, 0x08, 0x81, 0x07, + 0x3e, 0x75, 0x50, 0x0f, 0xc0, 0x27, 0x9e, 0xe6, 0x52, 0xdb, 0x32, 0x4e, 0x78, 0xea, 0x10, 0xc5, + 0xcb, 0xbb, 0xc4, 0xdb, 0xe1, 0x64, 0xdc, 0xf0, 0xc3, 0x47, 0x5e, 0x36, 0xe0, 0xf1, 0x75, 0xe0, + 0xf1, 0xf4, 0xa0, 0x81, 0x6b, 0x3c, 0x8c, 0x0e, 0x3c, 0x96, 0x85, 0xf3, 0x10, 0x4d, 0x44, 0xfb, + 0x0d, 0x2c, 0x5b, 0xaa, 0x03, 0xa5, 0x55, 0xd3, 0x44, 0x0a, 0xd4, 0xe4, 0x06, 0xc9, 0x20, 0x2f, + 0x6c, 0xa2, 0x1f, 0x40, 0xdd, 0xa4, 0x86, 0x98, 0x7f, 0xf1, 0x0c, 0xf3, 0xaf, 0x99, 0xd4, 0xe0, + 0x93, 0x9f, 0x85, 0xca, 0x81, 0x47, 0x1d, 0x11, 0x72, 0xd5, 0xb1, 0x68, 0x74, 0xff, 0xb1, 0x00, + 0x33, 0xd1, 0x39, 0xc9, 0x7c, 0x6f, 0xf2, 0xe0, 0x0a, 0xd4, 0x4c, 0x62, 0x93, 0x40, 0xaa, 0x76, + 0x1d, 0x87, 0xcd, 0xd4, 0xb4, 0x4a, 0x17, 0x9a, 0x56, 0x39, 0x31, 0xad, 0x8c, 0x6f, 0xaa, 0x64, + 0x7d, 0xd3, 0xdb, 0x30, 0x25, 0xf6, 0x2b, 0x44, 0xf0, 0xe4, 0x0b, 0xb7, 0x04, 0x51, 0x80, 0xba, + 0x57, 0xe1, 0xca, 0x1a, 0x75, 0x1c, 0x62, 0x04, 0xd4, 0xdb, 0xf1, 0xe8, 0x8b, 0x13, 0xa9, 0x88, + 0xdd, 0x3f, 0x2e, 0xc0, 0x5c, 0x96, 0x23, 0x97, 0xfe, 0x00, 0x6a, 0x2c, 0x65, 0x20, 0xbe, 0x2f, + 0xeb, 0x2c, 0xcb, 0xaf, 0x5e, 0xde, 0x5c, 0x3c, 0x4b, 0x6e, 0xb5, 0xe1, 0x98, 0xc2, 0x27, 0x87, + 0x1d, 0xb0, 0xd3, 0x77, 0x59, 0xe7, 0x9a, 0x65, 0xca, 0xa8, 0xbc, 0xc6, 0xdb, 0x5b, 0x26, 0x52, + 0xa1, 0x64, 0xd3, 0x81, 0xbc, 0x6f, 0xea, 0xa1, 0x87, 0xc3, 0x8c, 0xd8, 0xfd, 0xcb, 0x12, 0x94, + 0x1f, 0x50, 0xcb, 0x41, 0xb7, 0xa1, 0x43, 0x02, 0xc3, 0xd4, 0x86, 0xd4, 0xd4, 0x3c, 0x72, 0x6c, + 0xf9, 0x2c, 0xa3, 0x67, 0xb3, 0x2a, 0xe1, 0x19, 0xc6, 0x78, 0x48, 0x4d, 0x2c, 0xc9, 0x68, 0x01, + 0xaa, 0xfe, 0xa1, 0xee, 0x99, 0x61, 0x36, 0x73, 0x39, 0x32, 0x42, 0xd6, 0x95, 0x28, 0x5e, 0x60, + 0x09, 0x41, 0x37, 0xa1, 0xc9, 0x9f, 0x64, 0x05, 0xa2, 0xc4, 0xcf, 0x18, 0x38, 0x49, 0xd4, 0x1f, + 0x16, 0xa0, 0x13, 0x16, 0x29, 0x4c, 0xcb, 0xe3, 0xdb, 0x74, 0x12, 0xd6, 0xb4, 0x24, 0x63, 0x3d, + 0xa4, 0xa3, 0xf7, 0x20, 0xa4, 0x69, 0x44, 0xee, 0x01, 0x3f, 0xb0, 0x06, 0x9e, 0x91, 0xf4, 0x70, + 0x6b, 0xd0, 0xf7, 0x60, 0xc6, 0xe6, 0xe9, 0x7f, 0x8c, 0x14, 0x66, 0x31, 0x2d, 0xc8, 0x21, 0x50, + 0xfd, 0x8b, 0x02, 0x54, 0xf8, 0x9c, 0xd1, 0x34, 0x14, 0x2d, 0x53, 0x06, 0x0f, 0x45, 0xcb, 0x44, + 0x3d, 0xa8, 0xdb, 0xfa, 0x3e, 0xb1, 0x99, 0x72, 0x16, 0xa5, 0x37, 0xe6, 0x1e, 0x91, 0xa1, 0xb7, + 0x25, 0x07, 0x47, 0x18, 0xb4, 0x02, 0x35, 0x8f, 0xe8, 0x6c, 0xa6, 0x72, 0xb7, 0x95, 0xb8, 0x24, + 0xb1, 0xe3, 0x51, 0x83, 0xf8, 0xfe, 0xae, 0x4b, 0x8c, 0xde, 0xd6, 0x3a, 0x0e, 0x81, 0x68, 0x19, + 0x66, 0xf9, 0xc6, 0x1b, 0x1e, 0xd1, 0x03, 0x12, 0xef, 0x3d, 0x2f, 0x3e, 0x60, 0xc4, 0x78, 0x6b, + 0x9c, 0x15, 0x6e, 0x7f, 0xf7, 0x2e, 0x54, 0xd9, 0x3e, 0x13, 0x93, 0x1d, 0x1a, 0xbb, 0x71, 0xb9, + 0x7c, 0xf6, 0xd0, 0x86, 0xfa, 0x8b, 0x8d, 0xc0, 0x88, 0x0e, 0xad, 0xfb, 0xd3, 0x02, 0x94, 0xf7, + 0x74, 0xff, 0x88, 0xb9, 0x3d, 0xdf, 0x25, 0x86, 0x8c, 0x82, 0xf9, 0x33, 0xdb, 0x56, 0xd6, 0x51, + 0xe0, 0xe9, 0x8e, 0xaf, 0x47, 0x9e, 0x8e, 0x9d, 0x14, 0xeb, 0x67, 0x2f, 0x41, 0xce, 0x09, 0xb6, + 0xca, 0xe3, 0xc1, 0x16, 0xcb, 0xa0, 0xc3, 0x90, 0xc5, 0x63, 0x2a, 0x29, 0x8c, 0xaa, 0x19, 0xd1, + 0xb6, 0xcc, 0x07, 0xe5, 0x7a, 0xb1, 0x5d, 0xea, 0xfe, 0xac, 0x0a, 0x35, 0x4c, 0x0c, 0x7a, 0xcc, + 0xef, 0xb2, 0xa6, 0x6e, 0x1c, 0x69, 0x96, 0x13, 0x10, 0x27, 0x08, 0x3d, 0xfc, 0xad, 0xf8, 0x72, + 0x15, 0xb0, 0xde, 0xaa, 0x71, 0xb4, 0x25, 0x20, 0x22, 0xcf, 0x05, 0x3d, 0x22, 0xa0, 0x15, 0xb8, + 0x22, 0x72, 0xbd, 0x80, 0x98, 0x2c, 0x12, 0xf1, 0x89, 0x8c, 0x47, 0x8a, 0x3c, 0x1e, 0xb9, 0x1c, + 0x31, 0xd7, 0x18, 0x4f, 0x84, 0x26, 0x5f, 0x00, 0x8a, 0x65, 0xb8, 0x47, 0xb0, 0x48, 0x78, 0x80, + 0x9d, 0x5e, 0x58, 0x04, 0xbe, 0x2f, 0x19, 0xb8, 0x13, 0x81, 0x43, 0x12, 0x5a, 0x84, 0x59, 0x23, + 0x34, 0x71, 0x8d, 0xdd, 0x93, 0x24, 0xe1, 0xf2, 0xf1, 0x74, 0xc4, 0x63, 0x37, 0x29, 0x41, 0x8b, + 0x80, 0x0e, 0xd9, 0x1a, 0xd3, 0x13, 0xac, 0x88, 0x5a, 0x84, 0xe0, 0x24, 0x66, 0x77, 0x0f, 0x66, + 0x24, 0x3a, 0x9a, 0x5a, 0x75, 0xd2, 0xd4, 0xa6, 0x05, 0x32, 0x9a, 0xd7, 0x5b, 0xd0, 0xb2, 0x75, + 0x3f, 0xd0, 0x74, 0xd7, 0xb5, 0x2d, 0x62, 0xf2, 0x3a, 0x64, 0x0b, 0x37, 0x19, 0x6d, 0x55, 0x90, + 0xd0, 0x2a, 0x74, 0x6c, 0x32, 0xd0, 0x8d, 0x93, 0x64, 0x14, 0x58, 0x3f, 0x25, 0x0a, 0x6c, 0x0b, + 0x78, 0x22, 0x05, 0xfa, 0x08, 0x58, 0x98, 0xa7, 0x1d, 0x91, 0x93, 0xb0, 0xac, 0xf3, 0xe6, 0xd8, + 0x99, 0x3d, 0xd4, 0x5f, 0xfc, 0x88, 0x9c, 0xc8, 0x03, 0xab, 0x0d, 0x45, 0x0b, 0xdd, 0x86, 0xcb, + 0x81, 0x67, 0x0d, 0x06, 0xec, 0x9a, 0xd3, 0x3d, 0x7d, 0xe8, 0x8b, 0x6d, 0x03, 0x3e, 0xcd, 0x29, + 0xc9, 0xda, 0xe1, 0x1c, 0xb4, 0x03, 0x6d, 0xa6, 0x82, 0xc7, 0x44, 0xdb, 0xd7, 0x8d, 0xa3, 0x03, + 0xcb, 0xb6, 0x7d, 0xa5, 0xc9, 0x47, 0x7b, 0x37, 0x47, 0x43, 0x18, 0xb0, 0x1f, 0xe2, 0x64, 0x39, + 0x44, 0x4f, 0x53, 0xd5, 0x4f, 0x61, 0x26, 0xa3, 0x4a, 0xc9, 0x52, 0x47, 0x23, 0xa7, 0xd4, 0xd1, + 0x4a, 0x94, 0x3a, 0xd4, 0x7b, 0xd0, 0x4a, 0xae, 0xea, 0x75, 0x65, 0x92, 0x94, 0x6c, 0x1f, 0x66, + 0xf3, 0xe6, 0xf8, 0xba, 0x3e, 0xaa, 0xc9, 0x52, 0xcb, 0x3f, 0xd5, 0xa1, 0xb6, 0x43, 0x3c, 0xdf, + 0xf2, 0x03, 0x74, 0x05, 0xaa, 0x3e, 0xf9, 0x4a, 0x73, 0x28, 0x17, 0x2d, 0xe3, 0x8a, 0x4f, 0xbe, + 0x7a, 0x44, 0x99, 0xa6, 0x89, 0x2b, 0x53, 0x4b, 0xda, 0x95, 0xb8, 0x4c, 0xdb, 0x82, 0x13, 0xef, + 0x40, 0xd6, 0xfc, 0x4a, 0x19, 0xf3, 0x93, 0x63, 0x5d, 0xcc, 0xfc, 0xca, 0x93, 0xcd, 0xef, 0x1e, + 0x5c, 0x93, 0x93, 0xcc, 0xb1, 0xc2, 0x0a, 0x9f, 0xeb, 0x55, 0x01, 0x58, 0x1b, 0x33, 0xbc, 0x7c, + 0xd3, 0xad, 0x9e, 0xc3, 0x74, 0x97, 0x61, 0x2e, 0x36, 0x5d, 0x57, 0x0f, 0x8c, 0x43, 0x22, 0xb5, + 0x50, 0x18, 0x4b, 0x3b, 0xe2, 0xee, 0x08, 0xe6, 0x04, 0xf3, 0xad, 0x4f, 0x30, 0xdf, 0xbb, 0x30, + 0x27, 0x57, 0x97, 0xb5, 0xe2, 0x06, 0x5f, 0xda, 0xac, 0xe0, 0x6e, 0xa6, 0x0d, 0x37, 0xc7, 0xe8, + 0xe1, 0xa2, 0x46, 0xdf, 0x1c, 0x37, 0xfa, 0x8f, 0x40, 0x91, 0x93, 0x1a, 0xb7, 0xfd, 0x16, 0x9f, + 0x96, 0x9c, 0xf4, 0x76, 0xd6, 0xd6, 0x73, 0xdd, 0xc5, 0xd4, 0x85, 0xdd, 0xc5, 0x74, 0xc6, 0x5d, + 0x84, 0x3a, 0x96, 0xef, 0x2e, 0x56, 0xe0, 0x8a, 0x9c, 0x76, 0xda, 0x6b, 0x28, 0x33, 0x7c, 0xce, + 0x97, 0x05, 0x73, 0x2f, 0xe5, 0x36, 0x26, 0xb8, 0x98, 0x76, 0x9e, 0x8b, 0xe1, 0x2f, 0xab, 0x7c, + 0x43, 0x77, 0x94, 0x4e, 0xf8, 0xb2, 0x8a, 0xb5, 0xd0, 0x5d, 0xa8, 0xec, 0x93, 0x81, 0xe5, 0x28, + 0x28, 0x93, 0xe1, 0xa4, 0x6d, 0xb8, 0xcf, 0x30, 0x9b, 0x97, 0xb0, 0x00, 0xa3, 0x05, 0x68, 0x1b, + 0x74, 0xe8, 0xf2, 0xf9, 0x86, 0x11, 0xee, 0x65, 0x66, 0xd8, 0x9b, 0x97, 0xf0, 0x4c, 0xc8, 0x91, + 0xb9, 0xc8, 0xff, 0xa3, 0x2f, 0xea, 0x2b, 0x30, 0x97, 0x71, 0xac, 0x9a, 0x71, 0xa8, 0x3b, 0x03, + 0xd2, 0xc5, 0x70, 0x39, 0x67, 0x85, 0xa7, 0x44, 0xec, 0x6f, 0x41, 0x2b, 0xf0, 0x46, 0x8e, 0xa1, + 0x33, 0xcd, 0xd5, 0x03, 0xe9, 0xb3, 0x9a, 0x11, 0x6d, 0x35, 0xe8, 0x76, 0xa1, 0x21, 0x0f, 0x99, + 0x98, 0x13, 0xdc, 0x56, 0xf7, 0x4f, 0x0b, 0x50, 0x61, 0xaa, 0x7a, 0x92, 0x1b, 0xab, 0x28, 0x50, + 0x3b, 0x66, 0x3d, 0xc8, 0x94, 0xa4, 0x81, 0xc3, 0x26, 0xba, 0x0e, 0x0d, 0xae, 0xf9, 0x5c, 0x44, + 0xdc, 0xbd, 0x75, 0x46, 0x60, 0x31, 0x57, 0x64, 0x16, 0xa1, 0xac, 0x88, 0x1a, 0xb9, 0x59, 0x3c, + 0x95, 0xf2, 0xcb, 0x13, 0xae, 0x71, 0x11, 0xef, 0xa3, 0xf4, 0x35, 0xce, 0xf2, 0x89, 0xee, 0x33, + 0xa8, 0x85, 0x36, 0x75, 0x07, 0x90, 0x08, 0x91, 0xa2, 0xfa, 0x40, 0x18, 0x8c, 0x35, 0x70, 0x47, + 0x70, 0xd6, 0x63, 0xc6, 0x29, 0x7e, 0xa7, 0x98, 0xef, 0x77, 0xba, 0xbf, 0x2c, 0xc8, 0x2c, 0xf8, + 0x7c, 0x9b, 0xf2, 0x6e, 0xf8, 0xde, 0xb2, 0x94, 0xfb, 0xde, 0x32, 0x7c, 0x63, 0xf9, 0xf6, 0xa9, + 0x21, 0x0c, 0x4f, 0xfe, 0x09, 0xfa, 0x30, 0x61, 0xba, 0x15, 0x6e, 0xba, 0x71, 0xe9, 0x83, 0x27, + 0xdc, 0xb9, 0x76, 0xfb, 0x4d, 0xb4, 0xb3, 0x0b, 0x50, 0xe7, 0xde, 0xf4, 0x11, 0x7d, 0xde, 0xad, + 0x42, 0x79, 0x37, 0xa0, 0x6e, 0xb7, 0x01, 0x35, 0xf6, 0xd7, 0x25, 0x66, 0xf7, 0xb7, 0xa0, 0xb9, + 0x4b, 0x7c, 0xb6, 0xd0, 0x6d, 0x4a, 0xdd, 0x09, 0x55, 0x9a, 0xc2, 0x45, 0xaa, 0x34, 0x7f, 0x54, + 0x85, 0x9a, 0xac, 0xcd, 0xa2, 0xf7, 0x12, 0x3b, 0xde, 0x5c, 0xb9, 0xd2, 0x0b, 0x3f, 0x62, 0x08, + 0x8b, 0x0d, 0x7c, 0x23, 0xc5, 0x41, 0xfc, 0x26, 0x4c, 0xb1, 0xbf, 0x9a, 0x27, 0x93, 0x3c, 0x99, + 0x37, 0xcc, 0x25, 0x64, 0x04, 0x43, 0x08, 0xb5, 0x18, 0x38, 0x4a, 0x08, 0x3f, 0x84, 0xba, 0x69, + 0xf9, 0x3c, 0x86, 0x91, 0xc7, 0x75, 0x6d, 0x6c, 0xac, 0x75, 0x09, 0xc0, 0x11, 0x14, 0x7d, 0x02, + 0x10, 0x3e, 0x47, 0x55, 0xc1, 0x37, 0xc6, 0x07, 0x5c, 0x8f, 0x30, 0x38, 0x81, 0x67, 0x83, 0x1e, + 0xeb, 0xb6, 0x65, 0xea, 0x01, 0x91, 0x55, 0x86, 0xf1, 0x41, 0x9f, 0x4a, 0x00, 0x8e, 0xa0, 0xe8, + 0x63, 0x68, 0x84, 0xcf, 0xa6, 0xbc, 0x71, 0xaf, 0x8f, 0x8f, 0x19, 0x0a, 0x9a, 0x38, 0x46, 0xa7, + 0x0b, 0x6f, 0x8d, 0xd7, 0x14, 0xde, 0x7e, 0x00, 0x2d, 0x5f, 0x9c, 0xb0, 0x66, 0x53, 0xea, 0x2a, + 0xb3, 0xf2, 0xb2, 0x09, 0x0f, 0x33, 0x71, 0xfc, 0xb8, 0xe9, 0x27, 0x74, 0xe1, 0x2d, 0x28, 0x3f, + 0xa3, 0x96, 0xa3, 0x5c, 0xe1, 0x02, 0x53, 0xa9, 0x1c, 0x15, 0x73, 0x16, 0xfa, 0x1e, 0x54, 0x9f, + 0xf1, 0x4c, 0x4a, 0x99, 0x93, 0xc6, 0x91, 0x04, 0x11, 0x13, 0x4b, 0x36, 0xeb, 0x2b, 0xd0, 0xfd, + 0x23, 0xe5, 0x6a, 0xa6, 0x2f, 0x96, 0x50, 0x61, 0xce, 0x42, 0x4b, 0x51, 0x59, 0x58, 0xe1, 0xa0, + 0xab, 0xd9, 0x0a, 0x7f, 0xb6, 0x18, 0xdc, 0x83, 0x86, 0x08, 0x20, 0x1c, 0xfa, 0x5c, 0xb9, 0x26, + 0x6f, 0xf7, 0x48, 0x46, 0xea, 0x3c, 0xae, 0x1b, 0xf2, 0x89, 0xcd, 0xc1, 0x0f, 0xa8, 0xab, 0xa8, + 0x99, 0x39, 0x30, 0x53, 0xc0, 0x9c, 0x85, 0x6e, 0x43, 0xcd, 0x17, 0x86, 0xa1, 0x5c, 0x97, 0xaf, + 0xc4, 0x93, 0x28, 0x97, 0x98, 0x38, 0x04, 0xa8, 0xf7, 0xa2, 0x4a, 0xf0, 0xb9, 0x8b, 0x8e, 0xdd, + 0x3f, 0xb8, 0x06, 0xcd, 0x44, 0x75, 0x11, 0xdd, 0x49, 0xd9, 0xc7, 0xb5, 0x5e, 0xf2, 0xe3, 0x9b, + 0x1c, 0x1b, 0xf9, 0x3c, 0xdf, 0x46, 0xd4, 0x8c, 0xdc, 0x64, 0x3b, 0xf9, 0x38, 0xa1, 0xb2, 0xc2, + 0x4e, 0xde, 0xcc, 0x1d, 0x33, 0x47, 0x6d, 0x3f, 0x4d, 0xaa, 0xad, 0x30, 0x95, 0x9b, 0xf9, 0xe3, + 0xbe, 0x5e, 0x75, 0x2b, 0xbf, 0x22, 0xaa, 0x7b, 0x1b, 0x6a, 0x9e, 0xc8, 0x9c, 0xa4, 0xee, 0xb6, + 0xb3, 0x19, 0x15, 0x0e, 0x01, 0xe8, 0x1d, 0xa8, 0xb0, 0xc0, 0xf2, 0x44, 0x6a, 0x6c, 0xfc, 0x51, + 0x11, 0xbf, 0xb0, 0xb1, 0x60, 0xb2, 0x1e, 0xc3, 0xf0, 0x53, 0xcd, 0xf4, 0x28, 0xef, 0x4b, 0x1c, + 0x02, 0xd8, 0x04, 0x79, 0xf9, 0xf8, 0x7a, 0x66, 0x82, 0x89, 0x7a, 0xf1, 0x07, 0x91, 0x6d, 0xbd, + 0x91, 0x29, 0x19, 0x27, 0xb4, 0x30, 0x6b, 0x5f, 0x77, 0xa0, 0x6c, 0x53, 0xdd, 0x54, 0xe6, 0xa5, + 0x52, 0xe6, 0x89, 0x6c, 0x53, 0xdd, 0xc4, 0x1c, 0xc6, 0xc6, 0x60, 0x7f, 0x89, 0xa9, 0xbc, 0x77, + 0xca, 0x18, 0xdb, 0x1c, 0x82, 0x25, 0x14, 0x2d, 0x43, 0x85, 0x57, 0xd1, 0x95, 0xdb, 0x99, 0x2b, + 0x26, 0x29, 0xc3, 0x8b, 0xeb, 0x58, 0x00, 0xd1, 0xf7, 0xe3, 0x7a, 0xfd, 0x42, 0x26, 0x9a, 0x1c, + 0x93, 0x49, 0x14, 0xe9, 0xd9, 0x48, 0x7e, 0x40, 0x3d, 0xa2, 0x2c, 0x9e, 0x32, 0xd2, 0x2e, 0x43, + 0x60, 0x01, 0x64, 0x0b, 0xe2, 0x0f, 0xa6, 0x72, 0xe7, 0x94, 0x05, 0x71, 0x11, 0x13, 0x4b, 0x28, + 0x5a, 0xcb, 0xbc, 0x31, 0xef, 0x71, 0xd1, 0x5b, 0x13, 0x44, 0xf3, 0xdf, 0x95, 0xa3, 0x2d, 0x98, + 0xe6, 0x4d, 0x96, 0x22, 0x89, 0x6e, 0x96, 0x32, 0x6f, 0xaa, 0xc6, 0xba, 0x21, 0xa6, 0xec, 0x68, + 0xca, 0x4f, 0x36, 0x51, 0x9f, 0xe7, 0xa4, 0x0e, 0x7d, 0x6e, 0x13, 0x73, 0x40, 0x94, 0xe5, 0x53, + 0xa6, 0xb3, 0x1a, 0xe3, 0x70, 0x52, 0x08, 0x6d, 0x40, 0x2b, 0xd1, 0x34, 0x95, 0xf7, 0x33, 0xaf, + 0xed, 0x26, 0x74, 0x62, 0xe2, 0x94, 0x18, 0xd3, 0x69, 0x57, 0x44, 0xae, 0xca, 0x4a, 0x46, 0xa7, + 0x65, 0x44, 0x8b, 0x43, 0x00, 0x73, 0xa9, 0x6e, 0x18, 0xe5, 0x2a, 0x1f, 0x64, 0x5c, 0x6a, 0x14, + 0xff, 0xe2, 0x18, 0x94, 0xbe, 0x0d, 0xee, 0x9e, 0xfd, 0x36, 0xf8, 0xe4, 0x4c, 0xb7, 0xc1, 0xa7, + 0xaf, 0xbb, 0x0d, 0x7e, 0xb7, 0x70, 0xf1, 0xeb, 0x00, 0xfd, 0x30, 0x19, 0x3b, 0x26, 0xf2, 0xc2, + 0xe2, 0x29, 0x79, 0xe1, 0xe5, 0x48, 0x22, 0xf1, 0x3a, 0xf1, 0x43, 0x28, 0x33, 0x03, 0x43, 0x77, + 0xa0, 0x1e, 0xe5, 0xbd, 0x85, 0x49, 0x79, 0x6f, 0x04, 0x51, 0x7f, 0x59, 0x84, 0xaa, 0x30, 0x4c, + 0xf4, 0xf9, 0xd8, 0x1b, 0xa2, 0xb7, 0x4f, 0xb1, 0xe3, 0xf1, 0x17, 0x44, 0x22, 0x07, 0xe0, 0x6f, + 0x28, 0x3c, 0x4d, 0x7c, 0x2c, 0xb3, 0x7f, 0x12, 0x10, 0x51, 0x34, 0x29, 0xb3, 0x1c, 0x40, 0xf0, + 0x9e, 0x30, 0x56, 0x9f, 0x71, 0xd4, 0xff, 0x2c, 0xc4, 0xaf, 0x94, 0x66, 0xa1, 0x22, 0xca, 0xdc, + 0x22, 0xb6, 0x15, 0x0d, 0x34, 0x0f, 0xed, 0xa1, 0xe5, 0x68, 0x3e, 0x1d, 0x79, 0x46, 0xba, 0x1e, + 0x39, 0x3d, 0xb4, 0x9c, 0x5d, 0x4e, 0x16, 0xd5, 0x82, 0x79, 0x51, 0x87, 0x4d, 0x21, 0x4b, 0x12, + 0xa9, 0xbf, 0x48, 0x22, 0x17, 0x01, 0x09, 0x94, 0xa9, 0x99, 0xd4, 0xf0, 0xb5, 0x80, 0x06, 0xba, + 0xcd, 0x2f, 0xb4, 0x32, 0x6e, 0x4b, 0xce, 0x3a, 0x35, 0xfc, 0x3d, 0x46, 0x47, 0x3d, 0xb8, 0x1c, + 0xa2, 0xf9, 0x72, 0x24, 0xbc, 0xc2, 0xe1, 0x1d, 0xc9, 0xe2, 0xcb, 0x11, 0xf8, 0x2e, 0x4c, 0xc9, + 0x40, 0x5f, 0x33, 0x89, 0x1d, 0xc8, 0xef, 0xcd, 0x70, 0x53, 0x44, 0xf4, 0xeb, 0x8c, 0xa4, 0xfe, + 0x7b, 0x11, 0x2a, 0xdc, 0x4d, 0x9d, 0x92, 0xcb, 0x14, 0x26, 0xd4, 0x50, 0xbe, 0x84, 0x99, 0x28, + 0xd9, 0xe4, 0xd9, 0x72, 0xf8, 0x2a, 0x61, 0x7e, 0xb2, 0x37, 0xec, 0xa5, 0xb2, 0x50, 0x3c, 0xbd, + 0x9f, 0x6c, 0xfa, 0xe8, 0xc7, 0x80, 0xe2, 0xfc, 0x55, 0x66, 0xd7, 0x61, 0x11, 0x6b, 0xe1, 0x0c, + 0xbd, 0xae, 0x49, 0x19, 0xdc, 0xd9, 0xcf, 0x50, 0x7c, 0xf5, 0x73, 0x98, 0x3a, 0x6b, 0x0a, 0x3c, + 0x0b, 0x95, 0xe4, 0x01, 0x8b, 0x86, 0xda, 0x87, 0x76, 0x76, 0x9c, 0x73, 0xf7, 0xf1, 0x5f, 0x85, + 0xf8, 0x3d, 0xed, 0x69, 0x2f, 0x42, 0x73, 0xae, 0x91, 0x5c, 0x3d, 0x3f, 0x67, 0xfe, 0xa9, 0x9e, + 0xbc, 0x4e, 0xcd, 0x6f, 0x43, 0x47, 0x5c, 0x8b, 0x49, 0x8d, 0x14, 0x76, 0x33, 0x23, 0x18, 0xb1, + 0x42, 0x2e, 0x02, 0x92, 0xd8, 0xa4, 0x3e, 0x96, 0x84, 0xfa, 0x0a, 0x4e, 0xac, 0x8e, 0x6a, 0x0d, + 0x2a, 0xfc, 0x9e, 0x52, 0xff, 0xba, 0x00, 0x55, 0x71, 0x63, 0x9d, 0xd9, 0xd2, 0x05, 0x3c, 0xe7, + 0x55, 0xf0, 0x59, 0xd6, 0x23, 0x6e, 0xc5, 0x9c, 0xf5, 0x08, 0x46, 0x6a, 0x3d, 0x12, 0x9b, 0xb3, + 0x1e, 0xc1, 0x49, 0xac, 0xe7, 0xf7, 0x0a, 0xe9, 0xaf, 0xc9, 0xce, 0x6f, 0x40, 0xdf, 0x9a, 0xcb, + 0x5d, 0x85, 0xa9, 0xd4, 0x05, 0x7c, 0xfe, 0xb9, 0xa8, 0x9f, 0x43, 0x33, 0x71, 0x6d, 0x5e, 0xa0, + 0x83, 0x2f, 0xa0, 0x95, 0xbc, 0x77, 0xcf, 0xdf, 0x43, 0xf7, 0x67, 0x08, 0xaa, 0xe2, 0xeb, 0x17, + 0x34, 0x9f, 0xca, 0x45, 0x66, 0x7b, 0xf2, 0xd7, 0x04, 0x39, 0x69, 0xc8, 0xbd, 0xfc, 0x34, 0xe4, + 0x4a, 0x2c, 0x32, 0x39, 0x03, 0xb9, 0x3b, 0x96, 0x81, 0x28, 0xd9, 0x91, 0x72, 0x92, 0x8f, 0x8f, + 0xc6, 0x93, 0x0f, 0x75, 0x6c, 0xb4, 0x5f, 0xe7, 0x1d, 0x79, 0x79, 0xc7, 0x19, 0xb2, 0x84, 0x5e, + 0x26, 0x4b, 0x98, 0xcb, 0x7c, 0x18, 0x95, 0x4d, 0x10, 0xe6, 0x53, 0x09, 0xc2, 0x6c, 0x16, 0x9d, + 0xc8, 0x0d, 0x7a, 0x99, 0xdc, 0x60, 0x2e, 0x0f, 0x9b, 0x48, 0x0b, 0x16, 0xd2, 0x69, 0xc1, 0x95, + 0x2c, 0x3c, 0x95, 0x11, 0xbc, 0x9f, 0xcd, 0x08, 0xae, 0xe6, 0xc2, 0x93, 0xc9, 0xc0, 0x42, 0x3a, + 0x19, 0x18, 0xeb, 0x3f, 0x95, 0x07, 0xf4, 0x32, 0x79, 0xc0, 0x5c, 0x2e, 0x3a, 0x4e, 0x01, 0x3e, + 0xcb, 0x4d, 0x01, 0xae, 0x8f, 0x4b, 0x4d, 0x88, 0xfe, 0xd7, 0x27, 0x44, 0xff, 0x6f, 0xe6, 0xf6, + 0x30, 0x29, 0xf0, 0xff, 0x75, 0xb4, 0xfd, 0x5d, 0x8d, 0xb6, 0xff, 0x26, 0x8e, 0xb6, 0xef, 0x8d, + 0xdd, 0xc1, 0x37, 0xf2, 0x2d, 0xe3, 0x5b, 0x09, 0xb4, 0xff, 0xee, 0x57, 0x2f, 0xd0, 0x56, 0x3f, + 0xbe, 0x70, 0x0c, 0xad, 0x3e, 0x8e, 0xc3, 0xc1, 0xf3, 0xc7, 0x0f, 0x08, 0xca, 0x43, 0xe6, 0x40, + 0xc4, 0xbb, 0x60, 0xfe, 0x1c, 0x47, 0x59, 0xbf, 0x53, 0x8a, 0xa2, 0xac, 0x65, 0x98, 0x8d, 0x3e, + 0x3d, 0x4d, 0xae, 0x5f, 0xbc, 0xaf, 0x41, 0x11, 0x2f, 0xde, 0x81, 0x15, 0xb8, 0x12, 0x4b, 0x24, + 0xf7, 0x40, 0x1c, 0xec, 0xe5, 0x88, 0x99, 0x48, 0x37, 0x16, 0x01, 0x99, 0x1e, 0xd3, 0xee, 0xd4, + 0x18, 0x32, 0x7a, 0x92, 0x9c, 0xd4, 0x1e, 0x87, 0xe8, 0x64, 0xff, 0xe2, 0x48, 0x3a, 0x92, 0x95, + 0xe8, 0xfd, 0x4b, 0x68, 0xc7, 0xdf, 0xa2, 0x48, 0x97, 0x54, 0xc9, 0x7c, 0xa5, 0x9e, 0x72, 0x85, + 0xd1, 0x87, 0xb7, 0x9e, 0xf4, 0x4d, 0x33, 0x6e, 0x9a, 0xa0, 0x6a, 0x30, 0x93, 0xc1, 0x20, 0x95, + 0x7f, 0x80, 0x65, 0x8e, 0x0c, 0x69, 0x45, 0x2d, 0x1c, 0xb5, 0xf3, 0x03, 0x7a, 0x26, 0x21, 0x7f, + 0x26, 0x27, 0xf2, 0x94, 0x06, 0x8e, 0xda, 0xea, 0xd3, 0x74, 0x80, 0x38, 0xc9, 0xe6, 0x0b, 0xe7, + 0xb5, 0xf9, 0x99, 0x4c, 0xb8, 0xd7, 0xbd, 0x03, 0xd3, 0xbb, 0x27, 0x8e, 0xc1, 0xdc, 0x9d, 0xfc, + 0x9c, 0x33, 0xf5, 0xdb, 0xc4, 0x42, 0xfa, 0xb7, 0x89, 0xdd, 0x7f, 0x2d, 0xc0, 0x4c, 0x84, 0x97, + 0xb1, 0xd0, 0x32, 0x94, 0x74, 0xf9, 0xc5, 0x6f, 0xb2, 0x9c, 0x95, 0x81, 0xf5, 0x56, 0x8d, 0xa3, + 0xcd, 0x4b, 0x98, 0x41, 0x51, 0x1f, 0x1a, 0x87, 0x44, 0xf7, 0x82, 0x7d, 0xa2, 0x07, 0x63, 0x3f, + 0x59, 0xc8, 0xca, 0x6d, 0x86, 0xc8, 0xcd, 0x4b, 0x38, 0x16, 0x43, 0x1f, 0x40, 0xd9, 0xa4, 0x4e, + 0x5c, 0xff, 0x9d, 0x24, 0xbe, 0x4e, 0x1d, 0xb2, 0xc9, 0x7f, 0xcd, 0xe4, 0x10, 0xb5, 0x02, 0xa5, + 0x55, 0xe3, 0x48, 0x6d, 0x42, 0x23, 0xea, 0x55, 0xad, 0x42, 0x99, 0x61, 0xfa, 0x00, 0xf5, 0x30, + 0x12, 0xbc, 0xbd, 0x00, 0x15, 0xfe, 0x23, 0x49, 0x04, 0x50, 0xdd, 0x79, 0xd2, 0xdf, 0xde, 0x5a, + 0x6b, 0x5f, 0x42, 0x4d, 0xa8, 0xed, 0xe0, 0xad, 0xa7, 0xab, 0x7b, 0x1b, 0xed, 0x02, 0x6a, 0x40, + 0x65, 0xfb, 0xf1, 0xda, 0xea, 0x76, 0xbb, 0xb8, 0xf2, 0x00, 0xea, 0xf2, 0x47, 0x6c, 0x1e, 0xfa, + 0x0c, 0x6a, 0xf2, 0x19, 0xc5, 0xd7, 0x78, 0xfa, 0xe7, 0x95, 0xaa, 0x32, 0xce, 0x10, 0xc3, 0x2e, + 0x17, 0x56, 0xb6, 0xa1, 0x2e, 0x3f, 0x90, 0xf4, 0xd0, 0x17, 0x50, 0x93, 0xcf, 0x89, 0xbe, 0xd2, + 0x9f, 0xb9, 0x26, 0xfa, 0xca, 0x7c, 0x57, 0x39, 0x5f, 0x58, 0x2e, 0xac, 0x1c, 0xc2, 0x74, 0xfa, + 0xd3, 0x43, 0xf4, 0x14, 0x66, 0xf8, 0x43, 0x44, 0xf6, 0xd1, 0x8d, 0xe4, 0x75, 0x33, 0xfe, 0x01, + 0xa3, 0x7a, 0x73, 0x22, 0x3f, 0x31, 0xd2, 0x73, 0xa8, 0x6e, 0x8b, 0xdf, 0xda, 0xf5, 0xa2, 0x48, + 0x7c, 0x26, 0x63, 0x5d, 0x6a, 0x96, 0xc0, 0x24, 0xd1, 0xa7, 0xe9, 0x57, 0x09, 0xb3, 0x79, 0x49, + 0x9c, 0x9a, 0x4b, 0xe5, 0x03, 0xff, 0x59, 0xf4, 0xf1, 0xde, 0xfb, 0xf1, 0xfb, 0xba, 0x76, 0xf6, + 0xdd, 0x8b, 0x3a, 0x46, 0xe1, 0x63, 0xff, 0x1f, 0xcf, 0xf5, 0x21, 0x34, 0x59, 0xf4, 0xcb, 0x6e, + 0x75, 0x8f, 0xda, 0x5c, 0x57, 0x84, 0xb2, 0x26, 0x75, 0x25, 0x65, 0x8c, 0x49, 0x5d, 0x49, 0xeb, + 0xf5, 0x72, 0xa1, 0xff, 0xd9, 0xd7, 0xff, 0x7c, 0xe3, 0xd2, 0xd7, 0xbf, 0xb8, 0x51, 0xf8, 0xdb, + 0x5f, 0xdc, 0x28, 0xfc, 0xc9, 0xbf, 0xdc, 0x28, 0xfc, 0x78, 0xf1, 0x4c, 0xbf, 0x05, 0x94, 0x3d, + 0xee, 0x57, 0x39, 0xe9, 0x83, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x07, 0x1b, 0x00, 0xd0, 0x18, + 0x3e, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -4913,6 +5196,129 @@ var _Shard_serviceDesc = grpc.ServiceDesc{ Metadata: "go/protocols/runtime/runtime.proto", } +// TaskControlClient is the client API for TaskControl service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type TaskControlClient interface { + // SyncNow asks a materialization to immediately commit its open + // transaction, and resolves once that transaction is fully acknowledged: + // committed and queryable in the endpoint. It's served by the + // runtime-next sidecar hosting the task's leader session, and relayed + // by the reactor front door. + // + // The stream is exactly one Ack, then zero or more Heartbeats while the + // caller waits, then exactly one Done, after which the stream closes. + // SyncNow is idempotent: concurrent calls await the same commit, and a + // caller which hangs up early has still forced the commit. There is no + // error for "nothing to do" — a task with nothing to await, including a + // capture or derivation, Acks and its Done follows immediately. + SyncNow(ctx context.Context, in *SyncNowRequest, opts ...grpc.CallOption) (TaskControl_SyncNowClient, error) +} + +type taskControlClient struct { + cc *grpc.ClientConn +} + +func NewTaskControlClient(cc *grpc.ClientConn) TaskControlClient { + return &taskControlClient{cc} +} + +func (c *taskControlClient) SyncNow(ctx context.Context, in *SyncNowRequest, opts ...grpc.CallOption) (TaskControl_SyncNowClient, error) { + stream, err := c.cc.NewStream(ctx, &_TaskControl_serviceDesc.Streams[0], "/runtime.TaskControl/SyncNow", opts...) + if err != nil { + return nil, err + } + x := &taskControlSyncNowClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type TaskControl_SyncNowClient interface { + Recv() (*SyncNowResponse, error) + grpc.ClientStream +} + +type taskControlSyncNowClient struct { + grpc.ClientStream +} + +func (x *taskControlSyncNowClient) Recv() (*SyncNowResponse, error) { + m := new(SyncNowResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// TaskControlServer is the server API for TaskControl service. +type TaskControlServer interface { + // SyncNow asks a materialization to immediately commit its open + // transaction, and resolves once that transaction is fully acknowledged: + // committed and queryable in the endpoint. It's served by the + // runtime-next sidecar hosting the task's leader session, and relayed + // by the reactor front door. + // + // The stream is exactly one Ack, then zero or more Heartbeats while the + // caller waits, then exactly one Done, after which the stream closes. + // SyncNow is idempotent: concurrent calls await the same commit, and a + // caller which hangs up early has still forced the commit. There is no + // error for "nothing to do" — a task with nothing to await, including a + // capture or derivation, Acks and its Done follows immediately. + SyncNow(*SyncNowRequest, TaskControl_SyncNowServer) error +} + +// UnimplementedTaskControlServer can be embedded to have forward compatible implementations. +type UnimplementedTaskControlServer struct { +} + +func (*UnimplementedTaskControlServer) SyncNow(req *SyncNowRequest, srv TaskControl_SyncNowServer) error { + return status.Errorf(codes.Unimplemented, "method SyncNow not implemented") +} + +func RegisterTaskControlServer(s *grpc.Server, srv TaskControlServer) { + s.RegisterService(&_TaskControl_serviceDesc, srv) +} + +func _TaskControl_SyncNow_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SyncNowRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(TaskControlServer).SyncNow(m, &taskControlSyncNowServer{stream}) +} + +type TaskControl_SyncNowServer interface { + Send(*SyncNowResponse) error + grpc.ServerStream +} + +type taskControlSyncNowServer struct { + grpc.ServerStream +} + +func (x *taskControlSyncNowServer) Send(m *SyncNowResponse) error { + return x.ServerStream.SendMsg(m) +} + +var _TaskControl_serviceDesc = grpc.ServiceDesc{ + ServiceName: "runtime.TaskControl", + HandlerType: (*TaskControlServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "SyncNow", + Handler: _TaskControl_SyncNow_Handler, + ServerStreams: true, + }, + }, + Metadata: "go/protocols/runtime/runtime.proto", +} + func (m *TaskServiceConfig) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) @@ -9615,32 +10021,246 @@ func (m *Derive_StartedCommit) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func encodeVarintRuntime(dAtA []byte, offset int, v uint64) int { - offset -= sovRuntime(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ +func (m *SyncNowRequest) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err } - dAtA[offset] = uint8(v) - return base + return dAtA[:n], nil } -func (m *TaskServiceConfig) ProtoSize() (n int) { - if m == nil { - return 0 - } + +func (m *SyncNowRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i var l int _ = l - if m.LogFileFd != 0 { - n += 1 + sovRuntime(uint64(m.LogFileFd)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - l = len(m.TaskName) - if l > 0 { - n += 1 + l + sovRuntime(uint64(l)) + if len(m.TaskName) > 0 { + i -= len(m.TaskName) + copy(dAtA[i:], m.TaskName) + i = encodeVarintRuntime(dAtA, i, uint64(len(m.TaskName))) + i-- + dAtA[i] = 0xa } - l = len(m.UdsPath) - if l > 0 { + return len(dAtA) - i, nil +} + +func (m *SyncNowResponse) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SyncNowResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Response != nil { + { + size := m.Response.ProtoSize() + i -= size + if _, err := m.Response.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + } + } + return len(dAtA) - i, nil +} + +func (m *SyncNowResponse_Ack_) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse_Ack_) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Ack != nil { + { + size, err := m.Ack.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRuntime(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} +func (m *SyncNowResponse_Heartbeat_) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse_Heartbeat_) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Heartbeat != nil { + { + size, err := m.Heartbeat.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRuntime(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + return len(dAtA) - i, nil +} +func (m *SyncNowResponse_Done_) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse_Done_) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Done != nil { + { + size, err := m.Done.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRuntime(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + return len(dAtA) - i, nil +} +func (m *SyncNowResponse_Ack) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SyncNowResponse_Ack) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse_Ack) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + return len(dAtA) - i, nil +} + +func (m *SyncNowResponse_Heartbeat) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SyncNowResponse_Heartbeat) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse_Heartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + return len(dAtA) - i, nil +} + +func (m *SyncNowResponse_Done) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SyncNowResponse_Done) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse_Done) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + return len(dAtA) - i, nil +} + +func encodeVarintRuntime(dAtA []byte, offset int, v uint64) int { + offset -= sovRuntime(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *TaskServiceConfig) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.LogFileFd != 0 { + n += 1 + sovRuntime(uint64(m.LogFileFd)) + } + l = len(m.TaskName) + if l > 0 { + n += 1 + l + sovRuntime(uint64(l)) + } + l = len(m.UdsPath) + if l > 0 { n += 1 + l + sovRuntime(uint64(l)) } l = len(m.ContainerNetwork) @@ -11607,58 +12227,161 @@ func (m *Derive_StartedCommit) ProtoSize() (n int) { return n } -func sovRuntime(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 +func (m *SyncNowRequest) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.TaskName) + if l > 0 { + n += 1 + l + sovRuntime(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n } -func sozRuntime(x uint64) (n int) { - return sovRuntime(uint64((x << 1) ^ uint64((int64(x) >> 63)))) + +func (m *SyncNowResponse) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Response != nil { + n += m.Response.ProtoSize() + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n } -func (m *TaskServiceConfig) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRuntime - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: TaskServiceConfig: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: TaskServiceConfig: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field LogFileFd", wireType) - } - m.LogFileFd = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRuntime - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.LogFileFd |= int32(b&0x7F) << shift - if b < 0x80 { - break + +func (m *SyncNowResponse_Ack_) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Ack != nil { + l = m.Ack.ProtoSize() + n += 1 + l + sovRuntime(uint64(l)) + } + return n +} +func (m *SyncNowResponse_Heartbeat_) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Heartbeat != nil { + l = m.Heartbeat.ProtoSize() + n += 1 + l + sovRuntime(uint64(l)) + } + return n +} +func (m *SyncNowResponse_Done_) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Done != nil { + l = m.Done.ProtoSize() + n += 1 + l + sovRuntime(uint64(l)) + } + return n +} +func (m *SyncNowResponse_Ack) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *SyncNowResponse_Heartbeat) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *SyncNowResponse_Done) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovRuntime(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozRuntime(x uint64) (n int) { + return sovRuntime(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *TaskServiceConfig) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TaskServiceConfig: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TaskServiceConfig: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LogFileFd", wireType) + } + m.LogFileFd = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LogFileFd |= int32(b&0x7F) << shift + if b < 0x80 { + break } } case 2: @@ -24085,6 +24808,398 @@ func (m *Derive_StartedCommit) Unmarshal(dAtA []byte) error { } return nil } +func (m *SyncNowRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SyncNowRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SyncNowRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TaskName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRuntime + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRuntime + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TaskName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SyncNowResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SyncNowResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SyncNowResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ack", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRuntime + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRuntime + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &SyncNowResponse_Ack{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Response = &SyncNowResponse_Ack_{v} + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Heartbeat", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRuntime + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRuntime + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &SyncNowResponse_Heartbeat{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Response = &SyncNowResponse_Heartbeat_{v} + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Done", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRuntime + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRuntime + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &SyncNowResponse_Done{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Response = &SyncNowResponse_Done_{v} + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SyncNowResponse_Ack) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Ack: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Ack: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SyncNowResponse_Heartbeat) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Heartbeat: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Heartbeat: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SyncNowResponse_Done) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Done: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Done: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipRuntime(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/go/protocols/runtime/runtime.proto b/go/protocols/runtime/runtime.proto index 081026371a9..7ac5103ff27 100644 --- a/go/protocols/runtime/runtime.proto +++ b/go/protocols/runtime/runtime.proto @@ -1219,3 +1219,56 @@ message Derive { // each leg. Stopped stopped = 61; } + +// SyncNowRequest is the request of a TaskControl.SyncNow RPC. +message SyncNowRequest { + // Name of the task to synchronize. + string task_name = 1; +} + +// SyncNowResponse is the streamed response of a TaskControl.SyncNow RPC. +// Its messages are structural: they mark the caller's position in the +// stream and carry no payload. Statistics of the awaited transaction are +// recorded to the task's stats journal, which is where callers read them. +message SyncNowResponse { + // Ack is sent exactly once, as the first message of the stream: the + // request reached the task's leader and the commit has been forced. + message Ack {} + + // Heartbeat keeps a long-lived stream alive while the caller waits. + message Heartbeat {} + + // Done is sent exactly once, as the final message of the stream: the + // awaited transaction is committed and queryable in the endpoint. + message Done {} + + oneof response { + Ack ack = 1; + Heartbeat heartbeat = 2; + Done done = 3; + } +} + +// TaskControl is a user-facing control surface for running tasks. +// It's distinct from the Leader service because it has a different +// authorization model: callers present ordinary gazette READ claims +// scoped to the task's shards, not the LEAD capability held by shards. +// +// Only the runtime-next sidecar serves TaskControl over gRPC. Users reach it +// through the reactor front door, which relays to that sidecar but exposes +// itself as HTTP/NDJSON rather than gRPC — see go/runtime/task_control_http.go. +service TaskControl { + // SyncNow asks a materialization to immediately commit its open + // transaction, and resolves once that transaction is fully acknowledged: + // committed and queryable in the endpoint. It's served by the + // runtime-next sidecar hosting the task's leader session, and relayed + // by the reactor front door. + // + // The stream is exactly one Ack, then zero or more Heartbeats while the + // caller waits, then exactly one Done, after which the stream closes. + // SyncNow is idempotent: concurrent calls await the same commit, and a + // caller which hangs up early has still forced the commit. There is no + // error for "nothing to do" — a task with nothing to await, including a + // capture or derivation, Acks and its Done follows immediately. + rpc SyncNow(SyncNowRequest) returns (stream SyncNowResponse); +} From c71883f8b12dd146e4d257a58af66e87a68a0804 Mon Sep 17 00:00:00 2001 From: Dave Greer Date: Wed, 12 Aug 2026 14:57:12 +0000 Subject: [PATCH 3/3] go/runtime, flowctl: relay SyncNow at the front door and in the CLI Expose sync-now to users: the reactor front door relays TaskControl to the sidecar hosting a task's leader, over a hand-written REST/NDJSON endpoint which both the dashboard and flowctl call. gRPC is used only on the internal front-door-to-sidecar hop; the front door serves no TaskControl gRPC of its own. The relay core resolves the task's shard zero -- found by item-prefix scan over `//` with an exact task-name label match, since one task's name may prefix another's -- through the consumer framework's resolver with the caller's verified claims, so the token's label selector is enforced against the task's shards (AuthN floor is gazette READ; no new capability bit). The resolution is for routing and AuthZ only, and is released immediately after: an hour-long relay must not pin the shard's teardown, and a mid-relay primary move just breaks the stream (SyncNow is idempotent; callers re-invoke). Any front door can then dial any sidecar directly -- they listen on a fleet-wide port, and it's the same cross-host sidecar dial materialize_v2.go already makes for LeaderEndpoint -- so there's no gazette-style proxy hop to the primary's own front door. The caller's Authorization is forwarded verbatim for the sidecar to independently verify. One consequence of dialing the sidecar rather than proxying to the primary's front door: a non-primary front door doesn't inherit the primary's storeReadyCh wait, so a sync-now landing during shard recovery is likelier to see NOT_FOUND. That wait was never a real guarantee -- store-ready does not imply the sidecar's leader session has reached Join consensus, which is why flowctl retries a post-Ack NOT_FOUND at all. Because it resolved the shard, the front door also knows the task's type, and answers a capture or derivation itself rather than dialing the sidecar -- which sees only a task name. That keeps the contract uniform across task types: "not running here" is NOT_FOUND whatever the task is, and a task which simply holds no transaction to force Acks and is done. The HTTP endpoint `/v1/task-control/sync-now` mounts beside gazette's grpc-gateway `/v1/` mux with the same CORS wrapper, and reproduces the gateway's streaming dialect exactly -- `{"result": ...}` NDJSON lines via gogo/gateway's JSONPb with EmitDefaults, flushed per message, and terminal `{"error": {grpcCode, httpCode, ...}}` lines -- so the UI's existing stream parsing needs nothing new. The endpoint's doc comment is the UI contract for the dashboard's "Sync Now" button. Both deps were already pinned transitively and are promoted to direct. A Go test stands up two consumertest members over etcdtest with a stub sidecar TaskControl server, covering relay from the shard's own primary and from a front door which is not it, task name and bearer token arriving intact, claim rejection (missing / mis-scoped), NOT_FOUND passthrough, a capture answered without the sidecar being dialed at all, and a gated-flush assertion that each NDJSON line is readable before the stream completes. flowctl: `flowctl raw sync-now --task acmeCo/foo/materialize-bar` exits once the awaited transaction is fully acknowledged, so that `flowctl raw sync-now --task X && run-analytics.sh` sees the freshest data. It prints nothing: the exit status is the whole contract. Authorization is the ordinary Read-capability /authorize/user/task token every task reader already holds; no capability changes anywhere. dataplane.rs gains the pair `user_task_auth_watch` (starts the watch) and `reactor_front_door` (reads an address and token from it), because the caller must hold the watch: an hour-long wait can outlive its reactor token, and each attempt re-reads rather than reusing. The watch is now the single construction site for task authorizations in this module. The client reads SSL_CERT_FILE explicitly, because reqwest verifies against webpki's bundled roots rather than the OS roots that tonic's tls-native-roots uses; without it every private-CA data plane, including local stacks, would be unreachable. Failures are classified by gRPC code, not by how far the stream got. Unavailable is always a leader transient: the leader itself reports it when a session ends before its awaited transaction is acknowledged, and when a request races session teardown. DeadlineExceeded is the claims deadline which gazette's KeyedAuth.Verify binds the relay to -- reactor tokens live for a jittered 40 to 80 minutes, so every wait outliving one dies this way, and the next attempt's freshly-read token resolves it whether or not we had been acked. NOT_FOUND and EOF-before-Ack are the "not running here, or not on the V2 runtime" diagnostic only until some attempt has been acknowledged; afterwards they mean a leader restarted and its replacement is not addressable until it reaches Join consensus, which is exactly the state a reconnect lands in. So they are fatal on a first attempt and retryable after one. EOF before Done is never success, since a clean exit 0 without a Done would silently break the shell contract above. An unparsed response (a proxy or load balancer answering in the data plane's stead) is retryable iff 5xx, and a failed request or aborted body always is. Retries back off from 1s to 30s and continue indefinitely, because a caller who wants a deadline can bound the invocation itself and one who doesn't wants the sync. Seven tests drive the response handling against a stub front door over a real loopback socket: Done-waiting through heartbeats, a table over the retry classification, a mid-stream body abort, EOF before and after an Ack, a terminal error line, and an unparsed gateway response. The retry loop itself is not covered, as it needs a control-plane authorization watch. --- crates/flowctl/src/dataplane.rs | 41 ++- crates/flowctl/src/raw/mod.rs | 5 + crates/flowctl/src/raw/sync_now.rs | 546 +++++++++++++++++++++++++++++ go.mod | 4 +- go/runtime/flow_consumer.go | 11 + go/runtime/task_control.go | 187 ++++++++++ go/runtime/task_control_http.go | 143 ++++++++ go/runtime/task_control_test.go | 411 ++++++++++++++++++++++ 8 files changed, 1334 insertions(+), 14 deletions(-) create mode 100644 crates/flowctl/src/raw/sync_now.rs create mode 100644 go/runtime/task_control.go create mode 100644 go/runtime/task_control_http.go create mode 100644 go/runtime/task_control_test.go diff --git a/crates/flowctl/src/dataplane.rs b/crates/flowctl/src/dataplane.rs index c12a54b0910..7dec6cab0fb 100644 --- a/crates/flowctl/src/dataplane.rs +++ b/crates/flowctl/src/dataplane.rs @@ -57,12 +57,7 @@ pub async fn user_task_authorization( gazette::shard::Client, gazette::journal::Client, )> { - let watch = tokens::watch(workflows::UserTaskAuth { - client: rest.clone(), - user_tokens: user_tokens.clone(), - task: models::Name::new(task), - capability: models::Capability::Read, - }); + let watch = user_task_auth_watch(rest, user_tokens, task); let (shard_id_prefix, ops_logs_journal, ops_stats_journal) = { let ready = watch.ready().await.token(); @@ -90,6 +85,33 @@ pub async fn user_task_authorization( )) } +/// Start a live authorization watch for user access to a task. +/// Callers hold the watch and mint clients from it as needed, so that each +/// RPC bears a currently-valid data-plane token. +pub fn user_task_auth_watch( + rest: &flow_client_next::rest::Client, + user_tokens: &tokens::PendingWatch, + task: &str, +) -> tokens::PendingWatch { + tokens::watch(workflows::UserTaskAuth { + client: rest.clone(), + user_tokens: user_tokens.clone(), + task: models::Name::new(task), + capability: models::Capability::Read, + }) +} + +/// Await the reactor front-door address and bearer token of the data plane +/// hosting the task which `auth` authorizes. +pub async fn reactor_front_door( + auth: &tokens::PendingWatch, +) -> anyhow::Result<(String, String)> { + let ready = auth.ready().await.token(); + let model = ready.result()?; + + Ok((model.reactor_address.clone(), model.reactor_token.clone())) +} + /// Authorize the user for administrative operations over a task's shards /// and recovery logs, returning the task's ops journal names and /// Admin-capability shard + journal clients. @@ -106,12 +128,7 @@ pub async fn user_task_admin( gazette::journal::Client, )> { let (ops_logs_journal, ops_stats_journal) = { - let watch = tokens::watch(workflows::UserTaskAuth { - client: rest.clone(), - user_tokens: user_tokens.clone(), - task: models::Name::new(task), - capability: models::Capability::Read, - }); + let watch = user_task_auth_watch(rest, user_tokens, task); let ready = watch.ready().await.token(); let model = ready.result()?; diff --git a/crates/flowctl/src/raw/mod.rs b/crates/flowctl/src/raw/mod.rs index 3f5321e4186..f6b1cfb4af1 100644 --- a/crates/flowctl/src/raw/mod.rs +++ b/crates/flowctl/src/raw/mod.rs @@ -19,6 +19,7 @@ mod preview_next; mod shards; mod spec; mod split_shards; +mod sync_now; #[derive(Debug, clap::Args)] #[clap(rename_all = "kebab-case")] @@ -75,6 +76,9 @@ pub enum Command { ListShards(TaskSelector), /// Split each shard of a task on either shuffled key or rotated clock. SplitShards(split_shards::Split), + /// Force a materialization to immediately commit its open transaction, + /// and wait until that transaction is acknowledged by its endpoint. + SyncNow(sync_now::SyncNow), /// Print environment variables for working with a given data-plane /// and prefix using Gazette's `gazctl`. GazctlEnv(GazctlEnv), @@ -232,6 +236,7 @@ impl Advanced { Command::BearerLogs(bearer_logs) => bearer_logs.run(ctx).await, Command::ListShards(selector) => shards::do_list_shards(ctx, selector).await, Command::SplitShards(split) => split_shards::do_split(ctx, split).await, + Command::SyncNow(sync_now) => sync_now::do_sync_now(ctx, sync_now).await, Command::GazctlEnv(gazctl_env) => gazctl_env.run(ctx).await, Command::PreviewNext(preview) => preview.run(ctx).await, } diff --git a/crates/flowctl/src/raw/sync_now.rs b/crates/flowctl/src/raw/sync_now.rs new file mode 100644 index 00000000000..d187d8f6d50 --- /dev/null +++ b/crates/flowctl/src/raw/sync_now.rs @@ -0,0 +1,546 @@ +//! `flowctl raw sync-now`: force a materialization to immediately commit its +//! open transaction, and exit once that transaction is fully acknowledged -- +//! so that `flowctl raw sync-now --task X && run-analytics.sh` does what it +//! says. See the TaskControl service in `go/protocols/runtime/runtime.proto`. +//! +//! We call the reactor front door's HTTP/NDJSON endpoint, the same one the +//! dashboard uses. `go/runtime/task_control_http.go` documents its contract. + +use crate::CliContext; +use anyhow::Context; +use std::time::Duration; + +/// Path of TaskControl.SyncNow on a reactor front door. Mirrors +/// `TaskControlSyncNowPath` in `go/runtime/task_control_http.go`. +const SYNC_NOW_PATH: &str = "/v1/task-control/sync-now"; + +#[derive(Debug, clap::Args)] +#[clap(rename_all = "kebab-case")] +pub struct SyncNow { + #[clap(flatten)] + task: crate::ops::TaskSelector, +} + +/// Attempt SyncNow until it completes or a failure proves terminal. +/// +/// SyncNow is idempotent, and a retry's "everything as of this call" contract +/// still covers all data which preceded the first attempt. +pub async fn do_sync_now(ctx: &CliContext, args: &SyncNow) -> anyhow::Result<()> { + let auth = crate::dataplane::user_task_auth_watch(&ctx.rest, &ctx.user_tokens, &args.task.task); + let task = &args.task.task; + let client = new_http_client()?; + + let mut acked_ever = false; + let mut backoff = Duration::from_secs(1); + + loop { + // Re-read the front door on each attempt, as an hour-long wait may + // have outlived its reactor token. + let (address, token) = crate::dataplane::reactor_front_door(&auth).await?; + let url = url::Url::parse(&address) + .and_then(|url| url.join(SYNC_NOW_PATH)) + .with_context(|| format!("building a sync-now URL from reactor address {address}"))?; + + let failed = match attempt(&client, &url, &token, task).await { + Ok(()) => return Ok(()), + Err(failed) => failed, + }; + acked_ever |= failed.acked; + + if !failed.is_retryable(acked_ever) { + return Err(failed.into_error(task)); + } + tracing::warn!( + error = %failed.detail(), + retry_in = ?backoff, + "sync-now attempt failed; retrying", + ); + + tokio::time::sleep(backoff).await; + backoff = if failed.acked { + Duration::from_secs(1) // Progress was made; start over. + } else { + (backoff * 2).min(Duration::from_secs(30)) + }; + } +} + +/// Build the HTTP client which calls the reactor front door. +/// +/// `reqwest` verifies against webpki's bundled roots, and -- unlike tonic's +/// `tls-native-roots` -- ignores `SSL_CERT_FILE`. Honor it explicitly, so that +/// a data plane behind a private CA (such as a local stack) is reachable. +fn new_http_client() -> anyhow::Result { + let mut builder = reqwest::Client::builder(); + + if let Some(path) = std::env::var_os("SSL_CERT_FILE") { + let path = std::path::PathBuf::from(path); + let pem = std::fs::read(&path) + .with_context(|| format!("reading SSL_CERT_FILE {}", path.display()))?; + + for cert in reqwest::Certificate::from_pem_bundle(&pem) + .with_context(|| format!("parsing certificates of {}", path.display()))? + { + builder = builder.add_root_certificate(cert); + } + } + Ok(builder.build()?) +} + +/// One NDJSON line of a SyncNow response stream. Both fields are absent in a +/// line we don't understand. +#[derive(serde::Deserialize)] +struct Line { + #[serde(default)] + result: Option, + #[serde(default)] + error: Option, +} + +/// A terminal `{"error": ...}` line, in grpc-gateway's mid-stream convention. +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct ErrorLine { + grpc_code: i32, + message: String, +} + +/// Failure of a single SyncNow attempt. +#[derive(Debug)] +struct Failed { + /// The leader acknowledged this attempt before it failed, as happens on a + /// leader restart or shard reassignment mid-wait. + acked: bool, + kind: FailedKind, +} + +#[derive(Debug)] +enum FailedKind { + /// A terminal error line of the data plane, carrying its gRPC code. + Status { code: tonic::Code, message: String }, + /// A response which isn't a line of the documented protocol, as an + /// intervening proxy or load balancer returns in the data plane's stead. + Unparsed { + status: reqwest::StatusCode, + body: String, + }, + /// The request, or its response stream, failed in transport. + Transport(String), + /// The stream ended without its final Done response. + Eof, +} + +impl Failed { + /// Is this failure a transient condition of the task's leader, rather than + /// a verdict on the request? `acked_ever` distinguishes the two for + /// NotFound and EOF: before any acknowledgement they're the diagnostic + /// that the task isn't running here (or isn't on the V2 runtime), but + /// afterwards they're a leader restart racing our reconnect -- the + /// replacement session isn't addressable until it reaches Join consensus. + fn is_retryable(&self, acked_ever: bool) -> bool { + match &self.kind { + FailedKind::Status { code, .. } => match code { + // The leader itself reports Unavailable to ask for a retry, + // and DeadlineExceeded is our reactor token's claims deadline. + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded => true, + tonic::Code::NotFound => acked_ever, + _ => false, + }, + // A 5xx from an intermediary is transient; a 4xx is a verdict. + FailedKind::Unparsed { status, .. } => status.is_server_error(), + FailedKind::Transport(_) => true, + FailedKind::Eof => acked_ever, + } + } + + fn detail(&self) -> String { + match &self.kind { + FailedKind::Status { code, message } => format!("{code:?}: {message}"), + FailedKind::Unparsed { status, body } => format!("HTTP {status}: {body}"), + FailedKind::Transport(detail) => detail.clone(), + FailedKind::Eof => "the leader hung up without a Done response".to_string(), + } + } + + fn into_error(self, task: &str) -> anyhow::Error { + anyhow::anyhow!("sync-now of {task} did not complete: {}", self.detail()) + } +} + +/// Invoke SyncNow once, returning when the leader reports the awaited +/// transaction committed. Nothing is printed: the exit status is the whole +/// contract. +async fn attempt( + client: &reqwest::Client, + url: &url::Url, + token: &str, + task: &str, +) -> Result<(), Failed> { + let mut acked = false; + + let response = client + .post(url.clone()) + .bearer_auth(token) + .json(&proto_flow::runtime::SyncNowRequest { + task_name: task.to_string(), + }) + .send() + .await + .map_err(|err| Failed { + acked, + kind: FailedKind::Transport(err.to_string()), + })?; + + let status = response.status(); + let mut lines = tokio::io::AsyncBufReadExt::lines(tokio::io::BufReader::new( + tokio_util::io::StreamReader::new(futures::TryStreamExt::map_err( + response.bytes_stream(), + std::io::Error::other, + )), + )); + + loop { + let line = match lines.next_line().await { + Ok(Some(line)) if line.trim().is_empty() => continue, + Ok(Some(line)) => line, + // A well-formed stream always returns from its Done, below. + Ok(None) if status.is_success() => { + return Err(Failed { + acked, + kind: FailedKind::Eof, + }); + } + Ok(None) => { + return Err(Failed { + acked, + kind: FailedKind::Unparsed { + status, + body: String::new(), + }, + }); + } + Err(err) => { + return Err(Failed { + acked, + kind: FailedKind::Transport(err.to_string()), + }); + } + }; + + let Ok(Line { result, error }) = serde_json::from_str::(&line) else { + return Err(Failed { + acked, + kind: FailedKind::Unparsed { status, body: line }, + }); + }; + + if let Some(ErrorLine { grpc_code, message }) = error { + return Err(Failed { + acked, + kind: FailedKind::Status { + code: tonic::Code::from_i32(grpc_code), + message, + }, + }); + } + + use proto_flow::runtime::sync_now_response::Response; + match result.and_then(|result| result.response) { + // Our request reached the leader and the commit is forced. + Some(Response::Ack(_)) => acked = true, + // Consumed for liveness only. + Some(Response::Heartbeat(_)) => {} + Some(Response::Done(_)) => return Ok(()), + None => {} // A line, or response variant, we don't know about. + } + } +} + +#[cfg(test)] +mod test { + use super::{Failed, FailedKind, SYNC_NOW_PATH, attempt}; + use proto_flow::runtime::{SyncNowResponse, sync_now_response}; + + /// A full stream — ack, heartbeat, done — completes, and heartbeats + /// arriving mid-wait neither terminate it nor are mistaken for Done. + #[tokio::test] + async fn awaits_done_through_heartbeats() { + let outcome = run(Script::ok(vec![ack(), heartbeat(), done()])).await; + + outcome.expect("attempt completed"); + } + + /// Retry classification: NotFound and EOF are the "not running here" + /// diagnostic until an Ack proves otherwise, while Unavailable and + /// DeadlineExceeded are always transients of the leader or of our token. + #[test] + fn retry_classification() { + let cases = [ + ("not-found, never acked", not_found(), false, false), + ("not-found, acked earlier", not_found(), true, true), + ("unavailable, never acked", unavailable(), false, true), + ("unavailable, acked earlier", unavailable(), true, true), + ("deadline, never acked", deadline(), false, true), + ("deadline, acked earlier", deadline(), true, true), + ("permission denied", permission_denied(), true, false), + ("eof, never acked", eof(), false, false), + ("eof, acked earlier", eof(), true, true), + ("bad gateway", bad_gateway(), false, true), + ("not found by a proxy", proxied_not_found(), false, false), + ("transport", transport(), false, true), + ]; + + for (name, failed, acked_ever, want) in cases { + assert_eq!(failed.is_retryable(acked_ever), want, "case `{name}`"); + } + } + + #[tokio::test] + async fn eof_before_ack_is_fatal_on_a_first_attempt() { + let outcome = run(Script::ok(Vec::new())).await; + + let Err(failed) = outcome else { + panic!("expected a failure"); + }; + assert!(!failed.acked); + assert!(!failed.is_retryable(false)); + insta::assert_snapshot!( + format!("{:#}", failed.into_error("acmeCo/foo/materialize-bar")), + @"sync-now of acmeCo/foo/materialize-bar did not complete: the leader hung up without a Done response" + ); + } + + #[tokio::test] + async fn eof_after_ack_is_retryable() { + let outcome = run(Script::ok(vec![ack()])).await; + + let Err(failed) = outcome else { + panic!("expected a failure"); + }; + assert!(failed.acked); + assert!(failed.is_retryable(true)); + } + + /// A body which aborts mid-stream, as a leader restart or a dropped + /// connection produces, is always retryable — whether or not we managed to + /// read the Ack before the connection went away. + #[tokio::test] + async fn broken_stream_is_retryable() { + let outcome = run(Script { + status: reqwest::StatusCode::OK, + lines: vec![line(ack())], + abort: true, + }) + .await; + + let Err(failed) = outcome else { + panic!("expected a failure"); + }; + assert!( + matches!(failed.kind, FailedKind::Transport(_)), + "{failed:?}" + ); + assert!(failed.is_retryable(false)); + } + + /// A terminal error line renders as the data plane's own message. + #[tokio::test] + async fn terminal_error_line_is_contextualized() { + let outcome = run(Script { + status: reqwest::StatusCode::NOT_FOUND, + lines: vec![ + r#"{"error":{"grpcCode":5,"httpCode":404,"message":"task acmeCo/foo/materialize-bar has no live leader session here","httpStatus":"Not Found"}}"#.to_string(), + ], + abort: false, + }) + .await; + + let Err(failed) = outcome else { + panic!("expected a failure"); + }; + assert!(!failed.is_retryable(false)); + insta::assert_snapshot!( + format!("{:#}", failed.into_error("acmeCo/foo/materialize-bar")), + @"sync-now of acmeCo/foo/materialize-bar did not complete: NotFound: task acmeCo/foo/materialize-bar has no live leader session here" + ); + } + + /// A load balancer answering in the data plane's stead speaks neither + /// NDJSON nor gRPC codes, and its 5xx is retryable. + #[tokio::test] + async fn unparsed_gateway_response_is_retryable() { + let outcome = run(Script { + status: reqwest::StatusCode::BAD_GATEWAY, + lines: vec!["".to_string()], + abort: false, + }) + .await; + + let Err(failed) = outcome else { + panic!("expected a failure"); + }; + assert!(failed.is_retryable(false)); + insta::assert_snapshot!( + format!("{:#}", failed.into_error("acmeCo/foo/materialize-bar")), + @"sync-now of acmeCo/foo/materialize-bar did not complete: HTTP 502 Bad Gateway: " + ); + } + + fn status(code: tonic::Code) -> Failed { + Failed { + acked: false, + kind: FailedKind::Status { + code, + message: "as the leader or front door reports".to_string(), + }, + } + } + + fn not_found() -> Failed { + status(tonic::Code::NotFound) + } + + fn unavailable() -> Failed { + status(tonic::Code::Unavailable) + } + + // As the reactor front door reports when the claims deadline of a + // long-running relay elapses. + fn deadline() -> Failed { + status(tonic::Code::DeadlineExceeded) + } + + fn permission_denied() -> Failed { + status(tonic::Code::PermissionDenied) + } + + fn bad_gateway() -> Failed { + unparsed(reqwest::StatusCode::BAD_GATEWAY) + } + + // A 404 from an intermediary is a verdict on our URL, not on the task. + fn proxied_not_found() -> Failed { + unparsed(reqwest::StatusCode::NOT_FOUND) + } + + fn unparsed(status: reqwest::StatusCode) -> Failed { + Failed { + acked: false, + kind: FailedKind::Unparsed { + status, + body: "".to_string(), + }, + } + } + + fn eof() -> Failed { + Failed { + acked: false, + kind: FailedKind::Eof, + } + } + + fn transport() -> Failed { + Failed { + acked: false, + kind: FailedKind::Transport("connection reset".to_string()), + } + } + + fn ack() -> SyncNowResponse { + SyncNowResponse { + response: Some(sync_now_response::Response::Ack(sync_now_response::Ack {})), + } + } + + fn heartbeat() -> SyncNowResponse { + SyncNowResponse { + response: Some(sync_now_response::Response::Heartbeat( + sync_now_response::Heartbeat {}, + )), + } + } + + fn done() -> SyncNowResponse { + SyncNowResponse { + response: Some(sync_now_response::Response::Done( + sync_now_response::Done {}, + )), + } + } + + /// Render a response as the front door's `{"result": ...}` NDJSON line. + fn line(response: SyncNowResponse) -> String { + serde_json::json!({"result": response}).to_string() + } + + /// A scripted front-door response. + #[derive(Clone)] + struct Script { + status: reqwest::StatusCode, + /// NDJSON lines of the response body. + lines: Vec, + /// Abort the body after its lines, rather than ending it cleanly. + abort: bool, + } + + impl Script { + fn ok(responses: Vec) -> Self { + Self { + status: reqwest::StatusCode::OK, + lines: responses.into_iter().map(line).collect(), + abort: false, + } + } + } + + /// Drive `attempt` against a stub front door replaying `script`. + async fn run(script: Script) -> Result<(), Failed> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let app = axum::Router::new().route( + SYNC_NOW_PATH, + axum::routing::post(move |body: String| { + let script = script.clone(); + async move { + assert_eq!(body, r#"{"taskName":"acmeCo/foo/materialize-bar"}"#); + serve(script) + } + }), + ); + let server = tokio::spawn(async move { axum::serve(listener, app).await }); + + let url = url::Url::parse(&format!("http://{addr}{SYNC_NOW_PATH}")).unwrap(); + let outcome = attempt( + &reqwest::Client::new(), + &url, + "a-reactor-token", + "acmeCo/foo/materialize-bar", + ) + .await; + + server.abort(); + outcome + } + + /// Serve `script` as a chunk-per-line streamed body, so that a mid-stream + /// abort is distinguishable from a clean end. + fn serve(script: Script) -> axum::response::Response { + let Script { + status, + lines, + abort, + } = script; + + let chunks = lines + .into_iter() + .map(|line| Ok(format!("{line}\n"))) + .chain(abort.then(|| Err(std::io::Error::other("mid-stream abort")))); + + axum::response::IntoResponse::into_response(( + axum::http::StatusCode::from_u16(status.as_u16()).unwrap(), + axum::body::Body::from_stream(futures::stream::iter(chunks)), + )) + } +} diff --git a/go.mod b/go.mod index 10519c03e35..b802c00dded 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,11 @@ require ( github.com/bradleyjkemp/cupaloy v2.3.0+incompatible github.com/evanphx/json-patch/v5 v5.9.11 github.com/fatih/color v1.18.0 + github.com/gogo/gateway v1.1.0 github.com/gogo/protobuf v1.3.2 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jessevdk/go-flags v1.6.1 github.com/jgraettinger/gorocksdb v0.0.0-20250815051509-2d5c1f160b80 @@ -64,7 +66,6 @@ require ( github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/gogo/gateway v1.1.0 // indirect github.com/golang/glog v1.2.5 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v1.0.0 // indirect @@ -73,7 +74,6 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/gorilla/schema v1.4.1 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/jgraettinger/cockroach-encoding v1.1.0 // indirect diff --git a/go/runtime/flow_consumer.go b/go/runtime/flow_consumer.go index 31d5a5a11af..be5d32a16e4 100644 --- a/go/runtime/flow_consumer.go +++ b/go/runtime/flow_consumer.go @@ -325,6 +325,17 @@ func (f *FlowConsumer) InitApplication(args runconsumer.InitArgs) error { pr.RegisterShufflerServer(args.Server.GRPCServer, pr.NewVerifiedShufflerServer(shuffle.NewAPI(args.Service.Resolver), f.service.Verifier)) + // TaskControl is served only as a hand-written REST/NDJSON endpoint, + // sharing the CORS treatment of gazette's grpc-gateway `/v1/` mux (Go's + // ServeMux prefers the more specific pattern). + args.Server.HTTPMux.Handle(TaskControlSyncNowPath, config.Consumer.CORSWrapper(&taskControlHTTP{ + relay: &taskControl{ + service: args.Service, + sidecarEndpoint: config.SidecarEndpoint, + }, + verifier: f.service.Verifier, + })) + pf.RegisterNetworkProxyServer(args.Server.GRPCServer, pf.NewVerifiedNetworkProxyServer(&network.ProxyServer{Resolver: args.Service.Resolver}, f.service.Verifier)) diff --git a/go/runtime/task_control.go b/go/runtime/task_control.go new file mode 100644 index 00000000000..5c35cb08368 --- /dev/null +++ b/go/runtime/task_control.go @@ -0,0 +1,187 @@ +package runtime + +import ( + "context" + "io" + "strings" + + "github.com/estuary/flow/go/labels" + pr "github.com/estuary/flow/go/protocols/runtime" + "go.gazette.dev/core/allocator" + pb "go.gazette.dev/core/broker/protocol" + "go.gazette.dev/core/consumer" + pc "go.gazette.dev/core/consumer/protocol" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +// taskControl serves TaskControl.SyncNow on the reactor front door, as a +// semantics-free relay: it routes a caller's request to the runtime-next +// sidecar co-located with the task's shard-zero primary, which hosts the +// task's leader session and owns all SyncNow semantics. Callers reach it over +// HTTP/NDJSON; see task_control_http.go. +type taskControl struct { + service *consumer.Service + // sidecarEndpoint maps a reactor member endpoint to the dial-able URL of + // its co-located runtime-next sidecar (FlowConsumerConfig.SidecarEndpoint). + sidecarEndpoint func(reactor pb.Endpoint) (string, error) +} + +// syncNow resolves the task's shard-zero primary, dials its co-located +// sidecar, and relays response messages 1:1 until EOF or error. Any front door +// can dial any sidecar (they listen on a fleet-wide port), so there's no +// gazette-style proxy hop to the primary's own front door. The caller's +// Authorization is forwarded verbatim, for the sidecar to independently verify. +func (tc *taskControl) syncNow( + ctx context.Context, + claims pb.Claims, + req *pr.SyncNowRequest, + send func(*pr.SyncNowResponse) error, +) error { + if req.TaskName == "" { + return status.Error(codes.InvalidArgument, "task_name is required") + } + var shardZero, found = taskShardZero(tc.service.State, req.TaskName) + if !found { + return errTaskNotFound(req.TaskName) + } + + // MayProxy because we never serve locally: the sidecar does, whether or not + // we're the primary. + var res, err = tc.service.Resolver.Resolve(consumer.ResolveArgs{ + Context: ctx, + Claims: claims, + ShardID: shardZero, + MayProxy: true, + }) + if err != nil { + return err + } + // Resolution was needed only for routing. Release it now: a SyncNow relay + // can be open for an hour, and must not pin the local shard's teardown. + // If the primary moves mid-relay the sidecar stream breaks, and the + // caller re-invokes (SyncNow is idempotent). + if res.Done != nil { + res.Done() + } + + switch res.Status { + case pc.Status_OK: + // Pass. + case pc.Status_SHARD_NOT_FOUND: + // Also returned when the caller's claims don't cover the shard. + return errTaskNotFound(req.TaskName) + default: + return status.Errorf(codes.Unavailable, + "cannot resolve task shard %s to a ready primary (%s)", shardZero, res.Status) + } + + // Captures and derivations hold no open transaction, so there's nothing to + // force or await. Only this front door can answer that, because only it + // resolved the task's actual shard (and thus its type) from the keyspace. + // Resolve() above verified the caller's claims cover that shard. + if !strings.HasPrefix(shardZero.String(), "materialize/") { + if err = send(&pr.SyncNowResponse{ + Response: &pr.SyncNowResponse_Ack_{Ack: &pr.SyncNowResponse_Ack{}}, + }); err != nil { + return err + } + return send(&pr.SyncNowResponse{ + Response: &pr.SyncNowResponse_Done_{Done: &pr.SyncNowResponse_Done{}}, + }) + } + var sidecar string + if sidecar, err = tc.sidecarEndpoint(res.Header.Route.Endpoints[res.Header.Route.Primary]); err != nil { + return err + } + var conn *grpc.ClientConn + if conn, err = dialSidecar(ctx, sidecar); err != nil { + return err + } + defer conn.Close() + + var client pr.TaskControl_SyncNowClient + if client, err = pr.NewTaskControlClient(conn).SyncNow(forwardAuthorization(ctx), req); err != nil { + return err + } + + for { + var resp, err = client.Recv() + if err == io.EOF { + return nil + } else if err != nil { + return err // Terminal gRPC status passes through verbatim. + } else if err = send(resp); err != nil { + return err + } + } +} + +func errTaskNotFound(taskName string) error { + return status.Errorf(codes.NotFound, + "task %s was not found in this data plane (or your authorization does not cover it)", taskName) +} + +// taskShardZero returns the ID of the named task's shard zero: the shard with +// the lowest key / r-clock range. Shard IDs are `// +// /`, so a prefix scan finds the task's shards in +// range-ascending order — but one task's name may prefix another's, so require +// an exact task-name label match. +// +// A `reset` publication starts a new generation, and activation creates the new +// shards before deleting the old, so both can briefly appear here and we take +// the older. Sync-now during a reset is meaningless either way: the task is +// being backfilled from scratch. +func taskShardZero(state *allocator.State, taskName string) (pc.ShardID, bool) { + state.KS.Mu.RLock() + defer state.KS.Mu.RUnlock() + + for _, taskType := range []string{"capture", "derivation", "materialize"} { + var prefix = allocator.ItemKey(state.KS, taskType+"/"+taskName+"/") + for _, kv := range state.Items.Prefixed(prefix) { + var spec = kv.Decoded.(allocator.Item).ItemValue.(*pc.ShardSpec) + if spec.LabelSet.ValueOf(labels.TaskName) != taskName { + continue + } + return spec.Id, true + } + } + return "", false +} + +// dialSidecar dials the runtime-next sidecar at `endpoint`, using TLS +// or plaintext per its scheme (which mirrors the reactor's own). +// +// A nil TLS config means the process's ambient roots (honoring SSL_CERT_FILE): +// the same trust `gazette::dial_channel` applies to the sidecar's own leader +// and shuffle dials. The reactor's `--*-ca-file` flags configure gazette +// peers, not this hop. +func dialSidecar(ctx context.Context, endpoint string) (*grpc.ClientConn, error) { + var ep = pb.Endpoint(endpoint) + if err := ep.Validate(); err != nil { + return nil, err + } + var creds credentials.TransportCredentials + if ep.URL().Scheme == "https" { + creds = credentials.NewTLS(nil) + } else { + creds = insecure.NewCredentials() + } + return grpc.DialContext(ctx, ep.GRPCAddr(), grpc.WithTransportCredentials(creds)) +} + +// forwardAuthorization returns a Context which forwards the caller's incoming +// Authorization verbatim on outgoing RPCs, so that the sidecar independently +// verifies the caller's own token. +func forwardAuthorization(ctx context.Context) context.Context { + if md, ok := metadata.FromIncomingContext(ctx); ok { + if auth := md.Get("authorization"); len(auth) != 0 { + return metadata.AppendToOutgoingContext(ctx, "authorization", auth[len(auth)-1]) + } + } + return ctx +} diff --git a/go/runtime/task_control_http.go b/go/runtime/task_control_http.go new file mode 100644 index 00000000000..3197b47134b --- /dev/null +++ b/go/runtime/task_control_http.go @@ -0,0 +1,143 @@ +package runtime + +import ( + "encoding/json" + "errors" + "io" + "net/http" + + pr "github.com/estuary/flow/go/protocols/runtime" + "github.com/gogo/gateway" + gwruntime "github.com/grpc-ecosystem/grpc-gateway/runtime" + pb "go.gazette.dev/core/broker/protocol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +// TaskControlSyncNowPath is the reactor front-door HTTP endpoint of +// TaskControl.SyncNow. It's mounted alongside gazette's grpc-gateway +// `/v1/` mux with the same CORS treatment, and speaks the same +// fetch-streaming NDJSON dialect as gateway streaming RPCs +// (such as dashboard journal reads). +const TaskControlSyncNowPath = "/v1/task-control/sync-now" + +// Ceiling on a SyncNow request body, which carries only a task name. +const maxSyncNowRequestBytes = 4096 + +// taskControlHTTP is the reactor front door's only transport of +// TaskControl.SyncNow: both the dashboard and flowctl call it. +// +// This endpoint is the UI contract of the dashboard's "Sync Now" button: +// +// POST /v1/task-control/sync-now +// Authorization: Bearer -- a Read-level `/authorize/user/task` +// reactor token for the task. +// Content-Type: application/json +// +// {"taskName": "acmeCo/my/materialization"} +// +// The response is newline-delimited JSON, flushed per message, where each +// line wraps a runtime.SyncNowResponse exactly as grpc-gateway would: +// +// {"result": {"ack": {}}} +// {"result": {"heartbeat": {}}} +// {"result": {"done": {}}} +// +// The stream is exactly one `ack`, then zero or more `heartbeat` lines (one +// every ~15 seconds while awaiting the transaction, keeping idle +// load-balancer timeouts at bay), then exactly one `done`, after which the +// stream closes. The lines are structural and carry no payload: transaction +// statistics are recorded to the task's stats journal. A task with nothing to +// await — including a capture or derivation — sends `done` immediately after +// `ack`. See runtime.proto for full message shapes and semantics. +// +// A terminal error is a final `{"error": {"grpcCode": ..., "httpCode": ..., +// "message": ..., "httpStatus": ...}}` line (grpc-gateway's mid-stream error +// convention), with the HTTP status also set when no `result` line has been +// written yet. Notably `httpCode` 404 means the task isn't running in this +// data plane (or isn't on the V2 runtime, or the token doesn't cover it). +// +// SyncNow is idempotent: once `ack` is received the commit request has +// landed, hanging up early is harmless, and concurrent calls await the same +// commit. If the stream dies before `done` (network hiccup, task +// re-assignment), simply re-invoke. +type taskControlHTTP struct { + relay *taskControl + verifier pb.Verifier +} + +// jsonpbMarshaler matches the marshaling of gazette's grpc-gateway mux +// (see runconsumer.Main): camelCase names, enums as strings, zero-valued +// fields emitted. +var jsonpbMarshaler = &gateway.JSONPb{EmitDefaults: true} + +func (h *taskControlHTTP) ServeHTTP(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", jsonpbMarshaler.ContentType()) + + if req.Method != http.MethodPost { + writeStreamError(w, false, status.Error(codes.InvalidArgument, "expected POST")) + return + } + // Authenticate before reading the body: this endpoint is reachable by + // anyone, and the token is in the header. gazette's Verifier reads it from + // incoming gRPC metadata, so synthesize that from the HTTP header. The + // relay then forwards the token verbatim from this Context. + var ctx = metadata.NewIncomingContext(req.Context(), + metadata.Pairs("authorization", req.Header.Get("Authorization"))) + ctx, cancel, claims, err := h.verifier.Verify(ctx, pb.Capability_READ) + if err != nil { + writeStreamError(w, false, err) + return + } + defer cancel() + + // The body is one task name; cap it rather than decoding whatever arrives. + var syncNow = new(pr.SyncNowRequest) + if err := jsonpbMarshaler.NewDecoder(http.MaxBytesReader(w, req.Body, maxSyncNowRequestBytes)). + Decode(syncNow); err != nil && err != io.EOF { + writeStreamError(w, false, status.Errorf(codes.InvalidArgument, "parsing request body: %s", err)) + return + } + + var flusher, _ = w.(http.Flusher) + var wroteResult = false + err = h.relay.syncNow(ctx, claims, syncNow, func(resp *pr.SyncNowResponse) error { + var line, err = jsonpbMarshaler.Marshal(map[string]interface{}{"result": resp}) + if err != nil { + return err + } + if _, err = w.Write(append(line, '\n')); err != nil { + return err + } + wroteResult = true + if flusher != nil { + flusher.Flush() + } + return nil + }) + if err != nil && !errors.Is(err, req.Context().Err()) { + writeStreamError(w, wroteResult, err) + } +} + +// writeStreamError writes a terminal error in grpc-gateway's mid-stream +// convention: an `{"error": {...}}` NDJSON line, preceded by the mapped HTTP +// status code if the response header hasn't already been sent with a result. +func writeStreamError(w http.ResponseWriter, wroteResult bool, err error) { + var s = status.Convert(err) + var httpCode = gwruntime.HTTPStatusFromCode(s.Code()) + + if !wroteResult { + w.WriteHeader(httpCode) + } + var line, _ = json.Marshal(map[string]interface{}{ + "error": map[string]interface{}{ + "grpcCode": int32(s.Code()), + "httpCode": httpCode, + "message": s.Message(), + "httpStatus": http.StatusText(httpCode), + }, + }) + _, _ = w.Write(append(line, '\n')) +} diff --git a/go/runtime/task_control_test.go b/go/runtime/task_control_test.go new file mode 100644 index 00000000000..444148591cf --- /dev/null +++ b/go/runtime/task_control_test.go @@ -0,0 +1,411 @@ +package runtime + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/estuary/flow/go/labels" + pr "github.com/estuary/flow/go/protocols/runtime" + "github.com/stretchr/testify/require" + "go.gazette.dev/core/broker/client" + pb "go.gazette.dev/core/broker/protocol" + "go.gazette.dev/core/consumer" + pc "go.gazette.dev/core/consumer/protocol" + "go.gazette.dev/core/consumer/recoverylog" + "go.gazette.dev/core/consumertest" + "go.gazette.dev/core/etcdtest" + "go.gazette.dev/core/message" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +// Required by the gRPC loopback which consumertest.NewConsumer builds. +func init() { pb.RegisterGRPCDispatcher("local") } + +const ( + tcTestTask = "acmeCo/test/materialization" + tcTestShardID = "materialize/acmeCo/test/materialization/0011223344556677/00000000-00000000" + // tcTestScope is the `id:prefix` a Read-level /authorize/user/task + // token carries: the task's shard template prefix. + tcTestScope = "materialize/acmeCo/test/materialization/0011223344556677/" + + // A capture, which the front door answers as having nothing to await. + tcTestCaptureTask = "acmeCo/test/capture" + tcTestCaptureShardID = "capture/acmeCo/test/capture/0011223344556677/00000000-00000000" + tcTestCaptureScope = "capture/acmeCo/test/capture/0011223344556677/" +) + +func TestTaskControlSyncNow(t *testing.T) { + var etcd = etcdtest.TestClient() + defer etcdtest.Cleanup() + var ctx, cancel = context.WithCancel(context.Background()) + defer cancel() + + var stub = newStubSidecar(t) + defer stub.server.GracefulStop() + + // Two reactor members, so that a front door which is *not* shard zero's + // primary is exercised: it must reach the primary's sidecar all the same. + var members [2]*consumertest.Consumer + for i := range members { + var cmr = consumertest.NewConsumer(consumertest.Args{ + C: t, + Etcd: etcd, + Journals: nil, // Never read: the stub App produces no messages. + App: taskControlTestApp{}, + Suffix: fmt.Sprintf("member-%d", i), + }) + cmr.Server.HTTPMux.Handle(TaskControlSyncNowPath, &taskControlHTTP{ + relay: &taskControl{ + service: cmr.Service, + sidecarEndpoint: func(pb.Endpoint) (string, error) { return stub.endpoint, nil }, + }, + verifier: cmr.Service.Verifier, + }) + cmr.Tasks.GoRun() + members[i] = cmr + } + defer func() { + for _, cmr := range members { + cmr.Tasks.Cancel() + require.NoError(t, cmr.Tasks.Wait()) + } + }() + + consumertest.CreateShards(t, members[0], + &pc.ShardSpec{ + Id: tcTestShardID, + MaxTxnDuration: time.Minute, + LabelSet: pb.MustLabelSet(labels.TaskName, tcTestTask), + }, + &pc.ShardSpec{ + Id: tcTestCaptureShardID, + MaxTxnDuration: time.Minute, + LabelSet: pb.MustLabelSet(labels.TaskName, tcTestCaptureTask), + }, + ) + + // Determine the shard's primary member, and thereby which front door is + // co-located with the sidecar and which must reach across. + var route pb.Route + require.NoError(t, members[0].WaitForPrimary(ctx, tcTestShardID, &route)) + require.NoError(t, members[0].WaitForPrimary(ctx, tcTestCaptureShardID, nil)) + var primary, remote = members[0], members[1] + if route.Members[route.Primary].Suffix == "member-1" { + primary, remote = members[1], members[0] + } + + var token = mintTaskToken(t, primary, pb.MustLabelSet("id:prefix", tcTestScope)) + + t.Run("relay-flushes-per-message", func(t *testing.T) { + // Gate the stub between the heartbeat and Done, proving each NDJSON + // line is flushed and readable before the stream completes. + var gate = make(chan struct{}) + stub.script(scriptedOkStream(gate)) + + var resp = postSyncNow(t, primary, token, `{"taskName": "`+tcTestTask+`"}`) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "application/json", resp.Header.Get("Content-Type")) + + var lines = bufio.NewScanner(resp.Body) + require.Contains(t, readResultLine(t, lines), "ack") + require.Contains(t, readResultLine(t, lines), "heartbeat") + + close(gate) // Only now may the stub complete the stream. + require.Contains(t, readResultLine(t, lines), "done") + require.False(t, lines.Scan(), "expected EOF after done") + + var taskName, auth = stub.observed() + require.Equal(t, tcTestTask, taskName) + require.Equal(t, "Bearer "+token, auth) + }) + + t.Run("relay-from-a-non-primary-front-door", func(t *testing.T) { + stub.script(scriptedOkStream(nil)) + var resp = postSyncNow(t, remote, token, `{"taskName": "`+tcTestTask+`"}`) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + var lines = bufio.NewScanner(resp.Body) + require.Contains(t, readResultLine(t, lines), "ack") + require.Contains(t, readResultLine(t, lines), "heartbeat") + require.Contains(t, readResultLine(t, lines), "done") + + // The token is forwarded verbatim to the primary's sidecar. + var taskName, auth = stub.observed() + require.Equal(t, tcTestTask, taskName) + require.Equal(t, "Bearer "+token, auth) + }) + + t.Run("not-found-passthrough", func(t *testing.T) { + stub.script(func(*pr.SyncNowRequest, pr.TaskControl_SyncNowServer) error { + return status.Error(codes.NotFound, "no live leader session") + }) + var resp = postSyncNow(t, primary, token, `{"taskName": "`+tcTestTask+`"}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusNotFound, resp.StatusCode) + var errBody = readErrorLine(t, resp) + require.Equal(t, float64(codes.NotFound), errBody["grpcCode"]) + require.Equal(t, float64(http.StatusNotFound), errBody["httpCode"]) + require.Contains(t, errBody["message"], "no live leader session") + }) + + t.Run("rejects-missing-token", func(t *testing.T) { + var resp = postSyncNow(t, primary, "", `{"taskName": "`+tcTestTask+`"}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusUnauthorized, resp.StatusCode) + require.Equal(t, float64(codes.Unauthenticated), readErrorLine(t, resp)["grpcCode"]) + }) + + t.Run("rejects-missing-token-before-reading-the-body", func(t *testing.T) { + // An unauthenticated caller must not get us to decode its body: the + // endpoint is publicly reachable and the body is otherwise unbounded. + // A body which would itself be rejected proves the ordering, since it + // answers Unauthenticated rather than InvalidArgument. + var resp = postSyncNow(t, primary, "", `{"taskName": 42}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusUnauthorized, resp.StatusCode) + require.Equal(t, float64(codes.Unauthenticated), readErrorLine(t, resp)["grpcCode"]) + }) + + t.Run("rejects-an-oversized-body", func(t *testing.T) { + var resp = postSyncNow(t, primary, token, + `{"taskName": "`+strings.Repeat("x", maxSyncNowRequestBytes)+`"}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + }) + + t.Run("rejects-mis-scoped-token", func(t *testing.T) { + var misScoped = mintTaskToken(t, primary, + pb.MustLabelSet("id:prefix", "materialize/acmeCo/other/0011223344556677/")) + var resp = postSyncNow(t, primary, misScoped, `{"taskName": "`+tcTestTask+`"}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + + t.Run("rejects-bad-request", func(t *testing.T) { + var resp = postSyncNow(t, primary, token, `{"taskName": 42}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + }) + + t.Run("unknown-task", func(t *testing.T) { + stub.script(scriptedOkStream(nil)) + var resp = postSyncNow(t, primary, token, `{"taskName": "acmeCo/test/not-a-task"}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusNotFound, resp.StatusCode) + + // The sidecar is never consulted for a task we can't resolve. + var taskName, _ = stub.observed() + require.Empty(t, taskName) + }) + + t.Run("capture-has-nothing-to-await", func(t *testing.T) { + stub.script(scriptedOkStream(nil)) + var captureToken = mintTaskToken(t, primary, + pb.MustLabelSet("id:prefix", tcTestCaptureScope)) + var resp = postSyncNow(t, primary, captureToken, `{"taskName": "`+tcTestCaptureTask+`"}`) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + // Captures hold no open transaction: the front door answers from the + // task's shard type, without dialing the sidecar. + var lines = bufio.NewScanner(resp.Body) + require.Contains(t, readResultLine(t, lines), "ack") + require.Contains(t, readResultLine(t, lines), "done") + require.False(t, lines.Scan(), "expected EOF after done") + + var taskName, _ = stub.observed() + require.Empty(t, taskName) + }) + + t.Run("capture-rejects-mis-scoped-token", func(t *testing.T) { + // The capture's "nothing to await" answer comes only after the + // caller's claims are verified to cover its shard. + var resp = postSyncNow(t, primary, token, `{"taskName": "`+tcTestCaptureTask+`"}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) +} + +// scriptedOkStream returns a stub script sending the canonical happy-path +// sequence: Ack, Heartbeat, then Done. If `gate` is non-nil, Done is withheld +// until it closes. +func scriptedOkStream(gate <-chan struct{}) func(*pr.SyncNowRequest, pr.TaskControl_SyncNowServer) error { + return func(_ *pr.SyncNowRequest, stream pr.TaskControl_SyncNowServer) error { + if err := stream.Send(&pr.SyncNowResponse{ + Response: &pr.SyncNowResponse_Ack_{Ack: &pr.SyncNowResponse_Ack{}}, + }); err != nil { + return err + } + if err := stream.Send(&pr.SyncNowResponse{ + Response: &pr.SyncNowResponse_Heartbeat_{Heartbeat: &pr.SyncNowResponse_Heartbeat{}}, + }); err != nil { + return err + } + if gate != nil { + <-gate + } + return stream.Send(&pr.SyncNowResponse{ + Response: &pr.SyncNowResponse_Done_{Done: &pr.SyncNowResponse_Done{}}, + }) + } +} + +func postSyncNow(t *testing.T, cmr *consumertest.Consumer, token, body string) *http.Response { + var url = cmr.Server.Endpoint().URL().String() + TaskControlSyncNowPath + var req, err = http.NewRequest(http.MethodPost, url, strings.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + return resp +} + +// readResultLine scans one NDJSON line and returns its unwrapped "result". +func readResultLine(t *testing.T, lines *bufio.Scanner) map[string]interface{} { + require.True(t, lines.Scan(), "expected another NDJSON line") + var parsed map[string]map[string]interface{} + require.NoError(t, json.Unmarshal(lines.Bytes(), &parsed)) + require.Contains(t, parsed, "result") + return parsed["result"] +} + +// readErrorLine reads the response body as a single NDJSON error line and +// returns its unwrapped "error". +func readErrorLine(t *testing.T, resp *http.Response) map[string]interface{} { + var parsed map[string]map[string]interface{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&parsed)) + require.Contains(t, parsed, "error") + return parsed["error"] +} + +// mintTaskToken signs READ-capability Claims as the control plane's +// /authorize/user/task would, and returns the raw bearer token. +func mintTaskToken(t *testing.T, cmr *consumertest.Consumer, sel pb.LabelSet) string { + var ctx, err = cmr.Service.Authorizer.Authorize(context.Background(), pb.Claims{ + Capability: pb.Capability_READ, + Selector: pb.LabelSelector{Include: sel}, + }, time.Hour) + require.NoError(t, err) + + var md, _ = metadata.FromOutgoingContext(ctx) + return strings.TrimPrefix(md.Get("authorization")[0], "Bearer ") +} + +// stubSidecar is a scripted TaskControl gRPC server standing in for the +// runtime-next sidecar, recording what the relay delivers to it. +type stubSidecar struct { + endpoint string + server *grpc.Server + + mu sync.Mutex + serveFn func(*pr.SyncNowRequest, pr.TaskControl_SyncNowServer) error + taskName string + auth string +} + +func newStubSidecar(t *testing.T) *stubSidecar { + var listener, err = net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + var stub = &stubSidecar{ + endpoint: "http://" + listener.Addr().String(), + server: grpc.NewServer(), + } + pr.RegisterTaskControlServer(stub.server, stub) + go func() { _ = stub.server.Serve(listener) }() + return stub +} + +// script installs the SyncNow behavior for the next test case. +func (s *stubSidecar) script(fn func(*pr.SyncNowRequest, pr.TaskControl_SyncNowServer) error) { + s.mu.Lock() + defer s.mu.Unlock() + s.serveFn, s.taskName, s.auth = fn, "", "" +} + +// observed returns the task name and Authorization of the last request. +func (s *stubSidecar) observed() (taskName, auth string) { + s.mu.Lock() + defer s.mu.Unlock() + return s.taskName, s.auth +} + +func (s *stubSidecar) SyncNow(req *pr.SyncNowRequest, stream pr.TaskControl_SyncNowServer) error { + var md, _ = metadata.FromIncomingContext(stream.Context()) + + s.mu.Lock() + var fn = s.serveFn + s.taskName = req.TaskName + if auth := md.Get("authorization"); len(auth) != 0 { + s.auth = auth[0] + } + s.mu.Unlock() + + return fn(req, stream) +} + +// taskControlTestApp is a minimal consumer.Application whose shards become +// resolvable primaries without a broker, recovery log, or messages: as a +// MessageProducer it simply never produces, so shards idle. +type taskControlTestApp struct{} +type taskControlTestStore struct{} + +func (taskControlTestApp) NewStore(consumer.Shard, *recoverylog.Recorder) (consumer.Store, error) { + return taskControlTestStore{}, nil +} +func (taskControlTestApp) NewMessage(*pb.JournalSpec) (message.Message, error) { + panic("not called") +} +func (taskControlTestApp) ConsumeMessage(consumer.Shard, consumer.Store, message.Envelope, *message.Publisher) error { + panic("not called") +} +func (taskControlTestApp) FinalizeTxn(consumer.Shard, consumer.Store, *message.Publisher) error { + panic("not called") +} +func (taskControlTestApp) StartReadingMessages(shard consumer.Shard, _ consumer.Store, _ pc.Checkpoint, intoCh chan<- consumer.EnvelopeOrError) { + // Produce no messages, but deliver the shard's cancellation: the + // transaction loop of a MessageProducer application tears down only + // upon reading an error from this channel. + go func() { + <-shard.Context().Done() + intoCh <- consumer.EnvelopeOrError{Error: shard.Context().Err()} + }() +} +func (taskControlTestApp) ReplayRange(_ consumer.Shard, _ consumer.Store, _ pb.Journal, _, _ pb.Offset) message.Iterator { + panic("not called") +} +func (taskControlTestApp) ReadThrough(consumer.Shard, consumer.Store, consumer.ResolveArgs) (pb.Offsets, error) { + return nil, nil +} + +func (taskControlTestStore) StartCommit(consumer.Shard, pc.Checkpoint, consumer.OpFutures) consumer.OpFuture { + return client.FinishedOperation(nil) +} +func (taskControlTestStore) RestoreCheckpoint(consumer.Shard) (pc.Checkpoint, error) { + return pc.Checkpoint{}, nil +} +func (taskControlTestStore) Destroy() {}