diff --git a/Cargo.lock b/Cargo.lock index 3b17495a5aa..38b1fe2cc73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7401,6 +7401,7 @@ dependencies = [ "rand 0.9.2", "reqwest 0.12.23", "rocksdb", + "rustls 0.23.32", "schemars", "serde", "serde_json", diff --git a/crates/flowctl/src/dataplane.rs b/crates/flowctl/src/dataplane.rs index c12a54b0910..7dec6cab0fb 100644 --- a/crates/flowctl/src/dataplane.rs +++ b/crates/flowctl/src/dataplane.rs @@ -57,12 +57,7 @@ pub async fn user_task_authorization( gazette::shard::Client, gazette::journal::Client, )> { - let watch = tokens::watch(workflows::UserTaskAuth { - client: rest.clone(), - user_tokens: user_tokens.clone(), - task: models::Name::new(task), - capability: models::Capability::Read, - }); + let watch = user_task_auth_watch(rest, user_tokens, task); let (shard_id_prefix, ops_logs_journal, ops_stats_journal) = { let ready = watch.ready().await.token(); @@ -90,6 +85,33 @@ pub async fn user_task_authorization( )) } +/// Start a live authorization watch for user access to a task. +/// Callers hold the watch and mint clients from it as needed, so that each +/// RPC bears a currently-valid data-plane token. +pub fn user_task_auth_watch( + rest: &flow_client_next::rest::Client, + user_tokens: &tokens::PendingWatch, + task: &str, +) -> tokens::PendingWatch { + tokens::watch(workflows::UserTaskAuth { + client: rest.clone(), + user_tokens: user_tokens.clone(), + task: models::Name::new(task), + capability: models::Capability::Read, + }) +} + +/// Await the reactor front-door address and bearer token of the data plane +/// hosting the task which `auth` authorizes. +pub async fn reactor_front_door( + auth: &tokens::PendingWatch, +) -> anyhow::Result<(String, String)> { + let ready = auth.ready().await.token(); + let model = ready.result()?; + + Ok((model.reactor_address.clone(), model.reactor_token.clone())) +} + /// Authorize the user for administrative operations over a task's shards /// and recovery logs, returning the task's ops journal names and /// Admin-capability shard + journal clients. @@ -106,12 +128,7 @@ pub async fn user_task_admin( gazette::journal::Client, )> { let (ops_logs_journal, ops_stats_journal) = { - let watch = tokens::watch(workflows::UserTaskAuth { - client: rest.clone(), - user_tokens: user_tokens.clone(), - task: models::Name::new(task), - capability: models::Capability::Read, - }); + let watch = user_task_auth_watch(rest, user_tokens, task); let ready = watch.ready().await.token(); let model = ready.result()?; diff --git a/crates/flowctl/src/raw/mod.rs b/crates/flowctl/src/raw/mod.rs index 3f5321e4186..f6b1cfb4af1 100644 --- a/crates/flowctl/src/raw/mod.rs +++ b/crates/flowctl/src/raw/mod.rs @@ -19,6 +19,7 @@ mod preview_next; mod shards; mod spec; mod split_shards; +mod sync_now; #[derive(Debug, clap::Args)] #[clap(rename_all = "kebab-case")] @@ -75,6 +76,9 @@ pub enum Command { ListShards(TaskSelector), /// Split each shard of a task on either shuffled key or rotated clock. SplitShards(split_shards::Split), + /// Force a materialization to immediately commit its open transaction, + /// and wait until that transaction is acknowledged by its endpoint. + SyncNow(sync_now::SyncNow), /// Print environment variables for working with a given data-plane /// and prefix using Gazette's `gazctl`. GazctlEnv(GazctlEnv), @@ -232,6 +236,7 @@ impl Advanced { Command::BearerLogs(bearer_logs) => bearer_logs.run(ctx).await, Command::ListShards(selector) => shards::do_list_shards(ctx, selector).await, Command::SplitShards(split) => split_shards::do_split(ctx, split).await, + Command::SyncNow(sync_now) => sync_now::do_sync_now(ctx, sync_now).await, Command::GazctlEnv(gazctl_env) => gazctl_env.run(ctx).await, Command::PreviewNext(preview) => preview.run(ctx).await, } diff --git a/crates/flowctl/src/raw/sync_now.rs b/crates/flowctl/src/raw/sync_now.rs new file mode 100644 index 00000000000..d187d8f6d50 --- /dev/null +++ b/crates/flowctl/src/raw/sync_now.rs @@ -0,0 +1,546 @@ +//! `flowctl raw sync-now`: force a materialization to immediately commit its +//! open transaction, and exit once that transaction is fully acknowledged -- +//! so that `flowctl raw sync-now --task X && run-analytics.sh` does what it +//! says. See the TaskControl service in `go/protocols/runtime/runtime.proto`. +//! +//! We call the reactor front door's HTTP/NDJSON endpoint, the same one the +//! dashboard uses. `go/runtime/task_control_http.go` documents its contract. + +use crate::CliContext; +use anyhow::Context; +use std::time::Duration; + +/// Path of TaskControl.SyncNow on a reactor front door. Mirrors +/// `TaskControlSyncNowPath` in `go/runtime/task_control_http.go`. +const SYNC_NOW_PATH: &str = "/v1/task-control/sync-now"; + +#[derive(Debug, clap::Args)] +#[clap(rename_all = "kebab-case")] +pub struct SyncNow { + #[clap(flatten)] + task: crate::ops::TaskSelector, +} + +/// Attempt SyncNow until it completes or a failure proves terminal. +/// +/// SyncNow is idempotent, and a retry's "everything as of this call" contract +/// still covers all data which preceded the first attempt. +pub async fn do_sync_now(ctx: &CliContext, args: &SyncNow) -> anyhow::Result<()> { + let auth = crate::dataplane::user_task_auth_watch(&ctx.rest, &ctx.user_tokens, &args.task.task); + let task = &args.task.task; + let client = new_http_client()?; + + let mut acked_ever = false; + let mut backoff = Duration::from_secs(1); + + loop { + // Re-read the front door on each attempt, as an hour-long wait may + // have outlived its reactor token. + let (address, token) = crate::dataplane::reactor_front_door(&auth).await?; + let url = url::Url::parse(&address) + .and_then(|url| url.join(SYNC_NOW_PATH)) + .with_context(|| format!("building a sync-now URL from reactor address {address}"))?; + + let failed = match attempt(&client, &url, &token, task).await { + Ok(()) => return Ok(()), + Err(failed) => failed, + }; + acked_ever |= failed.acked; + + if !failed.is_retryable(acked_ever) { + return Err(failed.into_error(task)); + } + tracing::warn!( + error = %failed.detail(), + retry_in = ?backoff, + "sync-now attempt failed; retrying", + ); + + tokio::time::sleep(backoff).await; + backoff = if failed.acked { + Duration::from_secs(1) // Progress was made; start over. + } else { + (backoff * 2).min(Duration::from_secs(30)) + }; + } +} + +/// Build the HTTP client which calls the reactor front door. +/// +/// `reqwest` verifies against webpki's bundled roots, and -- unlike tonic's +/// `tls-native-roots` -- ignores `SSL_CERT_FILE`. Honor it explicitly, so that +/// a data plane behind a private CA (such as a local stack) is reachable. +fn new_http_client() -> anyhow::Result { + let mut builder = reqwest::Client::builder(); + + if let Some(path) = std::env::var_os("SSL_CERT_FILE") { + let path = std::path::PathBuf::from(path); + let pem = std::fs::read(&path) + .with_context(|| format!("reading SSL_CERT_FILE {}", path.display()))?; + + for cert in reqwest::Certificate::from_pem_bundle(&pem) + .with_context(|| format!("parsing certificates of {}", path.display()))? + { + builder = builder.add_root_certificate(cert); + } + } + Ok(builder.build()?) +} + +/// One NDJSON line of a SyncNow response stream. Both fields are absent in a +/// line we don't understand. +#[derive(serde::Deserialize)] +struct Line { + #[serde(default)] + result: Option, + #[serde(default)] + error: Option, +} + +/// A terminal `{"error": ...}` line, in grpc-gateway's mid-stream convention. +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct ErrorLine { + grpc_code: i32, + message: String, +} + +/// Failure of a single SyncNow attempt. +#[derive(Debug)] +struct Failed { + /// The leader acknowledged this attempt before it failed, as happens on a + /// leader restart or shard reassignment mid-wait. + acked: bool, + kind: FailedKind, +} + +#[derive(Debug)] +enum FailedKind { + /// A terminal error line of the data plane, carrying its gRPC code. + Status { code: tonic::Code, message: String }, + /// A response which isn't a line of the documented protocol, as an + /// intervening proxy or load balancer returns in the data plane's stead. + Unparsed { + status: reqwest::StatusCode, + body: String, + }, + /// The request, or its response stream, failed in transport. + Transport(String), + /// The stream ended without its final Done response. + Eof, +} + +impl Failed { + /// Is this failure a transient condition of the task's leader, rather than + /// a verdict on the request? `acked_ever` distinguishes the two for + /// NotFound and EOF: before any acknowledgement they're the diagnostic + /// that the task isn't running here (or isn't on the V2 runtime), but + /// afterwards they're a leader restart racing our reconnect -- the + /// replacement session isn't addressable until it reaches Join consensus. + fn is_retryable(&self, acked_ever: bool) -> bool { + match &self.kind { + FailedKind::Status { code, .. } => match code { + // The leader itself reports Unavailable to ask for a retry, + // and DeadlineExceeded is our reactor token's claims deadline. + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded => true, + tonic::Code::NotFound => acked_ever, + _ => false, + }, + // A 5xx from an intermediary is transient; a 4xx is a verdict. + FailedKind::Unparsed { status, .. } => status.is_server_error(), + FailedKind::Transport(_) => true, + FailedKind::Eof => acked_ever, + } + } + + fn detail(&self) -> String { + match &self.kind { + FailedKind::Status { code, message } => format!("{code:?}: {message}"), + FailedKind::Unparsed { status, body } => format!("HTTP {status}: {body}"), + FailedKind::Transport(detail) => detail.clone(), + FailedKind::Eof => "the leader hung up without a Done response".to_string(), + } + } + + fn into_error(self, task: &str) -> anyhow::Error { + anyhow::anyhow!("sync-now of {task} did not complete: {}", self.detail()) + } +} + +/// Invoke SyncNow once, returning when the leader reports the awaited +/// transaction committed. Nothing is printed: the exit status is the whole +/// contract. +async fn attempt( + client: &reqwest::Client, + url: &url::Url, + token: &str, + task: &str, +) -> Result<(), Failed> { + let mut acked = false; + + let response = client + .post(url.clone()) + .bearer_auth(token) + .json(&proto_flow::runtime::SyncNowRequest { + task_name: task.to_string(), + }) + .send() + .await + .map_err(|err| Failed { + acked, + kind: FailedKind::Transport(err.to_string()), + })?; + + let status = response.status(); + let mut lines = tokio::io::AsyncBufReadExt::lines(tokio::io::BufReader::new( + tokio_util::io::StreamReader::new(futures::TryStreamExt::map_err( + response.bytes_stream(), + std::io::Error::other, + )), + )); + + loop { + let line = match lines.next_line().await { + Ok(Some(line)) if line.trim().is_empty() => continue, + Ok(Some(line)) => line, + // A well-formed stream always returns from its Done, below. + Ok(None) if status.is_success() => { + return Err(Failed { + acked, + kind: FailedKind::Eof, + }); + } + Ok(None) => { + return Err(Failed { + acked, + kind: FailedKind::Unparsed { + status, + body: String::new(), + }, + }); + } + Err(err) => { + return Err(Failed { + acked, + kind: FailedKind::Transport(err.to_string()), + }); + } + }; + + let Ok(Line { result, error }) = serde_json::from_str::(&line) else { + return Err(Failed { + acked, + kind: FailedKind::Unparsed { status, body: line }, + }); + }; + + if let Some(ErrorLine { grpc_code, message }) = error { + return Err(Failed { + acked, + kind: FailedKind::Status { + code: tonic::Code::from_i32(grpc_code), + message, + }, + }); + } + + use proto_flow::runtime::sync_now_response::Response; + match result.and_then(|result| result.response) { + // Our request reached the leader and the commit is forced. + Some(Response::Ack(_)) => acked = true, + // Consumed for liveness only. + Some(Response::Heartbeat(_)) => {} + Some(Response::Done(_)) => return Ok(()), + None => {} // A line, or response variant, we don't know about. + } + } +} + +#[cfg(test)] +mod test { + use super::{Failed, FailedKind, SYNC_NOW_PATH, attempt}; + use proto_flow::runtime::{SyncNowResponse, sync_now_response}; + + /// A full stream — ack, heartbeat, done — completes, and heartbeats + /// arriving mid-wait neither terminate it nor are mistaken for Done. + #[tokio::test] + async fn awaits_done_through_heartbeats() { + let outcome = run(Script::ok(vec![ack(), heartbeat(), done()])).await; + + outcome.expect("attempt completed"); + } + + /// Retry classification: NotFound and EOF are the "not running here" + /// diagnostic until an Ack proves otherwise, while Unavailable and + /// DeadlineExceeded are always transients of the leader or of our token. + #[test] + fn retry_classification() { + let cases = [ + ("not-found, never acked", not_found(), false, false), + ("not-found, acked earlier", not_found(), true, true), + ("unavailable, never acked", unavailable(), false, true), + ("unavailable, acked earlier", unavailable(), true, true), + ("deadline, never acked", deadline(), false, true), + ("deadline, acked earlier", deadline(), true, true), + ("permission denied", permission_denied(), true, false), + ("eof, never acked", eof(), false, false), + ("eof, acked earlier", eof(), true, true), + ("bad gateway", bad_gateway(), false, true), + ("not found by a proxy", proxied_not_found(), false, false), + ("transport", transport(), false, true), + ]; + + for (name, failed, acked_ever, want) in cases { + assert_eq!(failed.is_retryable(acked_ever), want, "case `{name}`"); + } + } + + #[tokio::test] + async fn eof_before_ack_is_fatal_on_a_first_attempt() { + let outcome = run(Script::ok(Vec::new())).await; + + let Err(failed) = outcome else { + panic!("expected a failure"); + }; + assert!(!failed.acked); + assert!(!failed.is_retryable(false)); + insta::assert_snapshot!( + format!("{:#}", failed.into_error("acmeCo/foo/materialize-bar")), + @"sync-now of acmeCo/foo/materialize-bar did not complete: the leader hung up without a Done response" + ); + } + + #[tokio::test] + async fn eof_after_ack_is_retryable() { + let outcome = run(Script::ok(vec![ack()])).await; + + let Err(failed) = outcome else { + panic!("expected a failure"); + }; + assert!(failed.acked); + assert!(failed.is_retryable(true)); + } + + /// A body which aborts mid-stream, as a leader restart or a dropped + /// connection produces, is always retryable — whether or not we managed to + /// read the Ack before the connection went away. + #[tokio::test] + async fn broken_stream_is_retryable() { + let outcome = run(Script { + status: reqwest::StatusCode::OK, + lines: vec![line(ack())], + abort: true, + }) + .await; + + let Err(failed) = outcome else { + panic!("expected a failure"); + }; + assert!( + matches!(failed.kind, FailedKind::Transport(_)), + "{failed:?}" + ); + assert!(failed.is_retryable(false)); + } + + /// A terminal error line renders as the data plane's own message. + #[tokio::test] + async fn terminal_error_line_is_contextualized() { + let outcome = run(Script { + status: reqwest::StatusCode::NOT_FOUND, + lines: vec![ + r#"{"error":{"grpcCode":5,"httpCode":404,"message":"task acmeCo/foo/materialize-bar has no live leader session here","httpStatus":"Not Found"}}"#.to_string(), + ], + abort: false, + }) + .await; + + let Err(failed) = outcome else { + panic!("expected a failure"); + }; + assert!(!failed.is_retryable(false)); + insta::assert_snapshot!( + format!("{:#}", failed.into_error("acmeCo/foo/materialize-bar")), + @"sync-now of acmeCo/foo/materialize-bar did not complete: NotFound: task acmeCo/foo/materialize-bar has no live leader session here" + ); + } + + /// A load balancer answering in the data plane's stead speaks neither + /// NDJSON nor gRPC codes, and its 5xx is retryable. + #[tokio::test] + async fn unparsed_gateway_response_is_retryable() { + let outcome = run(Script { + status: reqwest::StatusCode::BAD_GATEWAY, + lines: vec!["".to_string()], + abort: false, + }) + .await; + + let Err(failed) = outcome else { + panic!("expected a failure"); + }; + assert!(failed.is_retryable(false)); + insta::assert_snapshot!( + format!("{:#}", failed.into_error("acmeCo/foo/materialize-bar")), + @"sync-now of acmeCo/foo/materialize-bar did not complete: HTTP 502 Bad Gateway: " + ); + } + + fn status(code: tonic::Code) -> Failed { + Failed { + acked: false, + kind: FailedKind::Status { + code, + message: "as the leader or front door reports".to_string(), + }, + } + } + + fn not_found() -> Failed { + status(tonic::Code::NotFound) + } + + fn unavailable() -> Failed { + status(tonic::Code::Unavailable) + } + + // As the reactor front door reports when the claims deadline of a + // long-running relay elapses. + fn deadline() -> Failed { + status(tonic::Code::DeadlineExceeded) + } + + fn permission_denied() -> Failed { + status(tonic::Code::PermissionDenied) + } + + fn bad_gateway() -> Failed { + unparsed(reqwest::StatusCode::BAD_GATEWAY) + } + + // A 404 from an intermediary is a verdict on our URL, not on the task. + fn proxied_not_found() -> Failed { + unparsed(reqwest::StatusCode::NOT_FOUND) + } + + fn unparsed(status: reqwest::StatusCode) -> Failed { + Failed { + acked: false, + kind: FailedKind::Unparsed { + status, + body: "".to_string(), + }, + } + } + + fn eof() -> Failed { + Failed { + acked: false, + kind: FailedKind::Eof, + } + } + + fn transport() -> Failed { + Failed { + acked: false, + kind: FailedKind::Transport("connection reset".to_string()), + } + } + + fn ack() -> SyncNowResponse { + SyncNowResponse { + response: Some(sync_now_response::Response::Ack(sync_now_response::Ack {})), + } + } + + fn heartbeat() -> SyncNowResponse { + SyncNowResponse { + response: Some(sync_now_response::Response::Heartbeat( + sync_now_response::Heartbeat {}, + )), + } + } + + fn done() -> SyncNowResponse { + SyncNowResponse { + response: Some(sync_now_response::Response::Done( + sync_now_response::Done {}, + )), + } + } + + /// Render a response as the front door's `{"result": ...}` NDJSON line. + fn line(response: SyncNowResponse) -> String { + serde_json::json!({"result": response}).to_string() + } + + /// A scripted front-door response. + #[derive(Clone)] + struct Script { + status: reqwest::StatusCode, + /// NDJSON lines of the response body. + lines: Vec, + /// Abort the body after its lines, rather than ending it cleanly. + abort: bool, + } + + impl Script { + fn ok(responses: Vec) -> Self { + Self { + status: reqwest::StatusCode::OK, + lines: responses.into_iter().map(line).collect(), + abort: false, + } + } + } + + /// Drive `attempt` against a stub front door replaying `script`. + async fn run(script: Script) -> Result<(), Failed> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let app = axum::Router::new().route( + SYNC_NOW_PATH, + axum::routing::post(move |body: String| { + let script = script.clone(); + async move { + assert_eq!(body, r#"{"taskName":"acmeCo/foo/materialize-bar"}"#); + serve(script) + } + }), + ); + let server = tokio::spawn(async move { axum::serve(listener, app).await }); + + let url = url::Url::parse(&format!("http://{addr}{SYNC_NOW_PATH}")).unwrap(); + let outcome = attempt( + &reqwest::Client::new(), + &url, + "a-reactor-token", + "acmeCo/foo/materialize-bar", + ) + .await; + + server.abort(); + outcome + } + + /// Serve `script` as a chunk-per-line streamed body, so that a mid-stream + /// abort is distinguishable from a clean end. + fn serve(script: Script) -> axum::response::Response { + let Script { + status, + lines, + abort, + } = script; + + let chunks = lines + .into_iter() + .map(|line| Ok(format!("{line}\n"))) + .chain(abort.then(|| Err(std::io::Error::other("mid-stream abort")))); + + axum::response::IntoResponse::into_response(( + axum::http::StatusCode::from_u16(status.as_u16()).unwrap(), + axum::body::Body::from_stream(futures::stream::iter(chunks)), + )) + } +} diff --git a/crates/proto-flow/src/runtime.rs b/crates/proto-flow/src/runtime.rs index ec5d962982e..23c66acd828 100644 --- a/crates/proto-flow/src/runtime.rs +++ b/crates/proto-flow/src/runtime.rs @@ -1407,6 +1407,45 @@ pub mod derive { #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct StartedCommit {} } +/// SyncNowRequest is the request of a TaskControl.SyncNow RPC. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SyncNowRequest { + /// Name of the task to synchronize. + #[prost(string, tag = "1")] + pub task_name: ::prost::alloc::string::String, +} +/// SyncNowResponse is the streamed response of a TaskControl.SyncNow RPC. +/// Its messages are structural: they mark the caller's position in the +/// stream and carry no payload. Statistics of the awaited transaction are +/// recorded to the task's stats journal, which is where callers read them. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SyncNowResponse { + #[prost(oneof = "sync_now_response::Response", tags = "1, 2, 3")] + pub response: ::core::option::Option, +} +/// Nested message and enum types in `SyncNowResponse`. +pub mod sync_now_response { + /// Ack is sent exactly once, as the first message of the stream: the + /// request reached the task's leader and the commit has been forced. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct Ack {} + /// Heartbeat keeps a long-lived stream alive while the caller waits. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct Heartbeat {} + /// Done is sent exactly once, as the final message of the stream: the + /// awaited transaction is committed and queryable in the endpoint. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct Done {} + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Response { + #[prost(message, tag = "1")] + Ack(Ack), + #[prost(message, tag = "2")] + Heartbeat(Heartbeat), + #[prost(message, tag = "3")] + Done(Done), + } +} /// Plane describes the type of data plane in which the runtime is operating. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/crates/proto-flow/src/runtime.serde.rs b/crates/proto-flow/src/runtime.serde.rs index fe3bc873717..03a1bad365a 100644 --- a/crates/proto-flow/src/runtime.serde.rs +++ b/crates/proto-flow/src/runtime.serde.rs @@ -10571,6 +10571,444 @@ impl<'de> serde::Deserialize<'de> for Stopped { deserializer.deserialize_struct("runtime.Stopped", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for SyncNowRequest { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if !self.task_name.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("runtime.SyncNowRequest", len)?; + if !self.task_name.is_empty() { + struct_ser.serialize_field("taskName", &self.task_name)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for SyncNowRequest { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "task_name", + "taskName", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + TaskName, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "taskName" | "task_name" => Ok(GeneratedField::TaskName), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = SyncNowRequest; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.SyncNowRequest") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut task_name__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::TaskName => { + if task_name__.is_some() { + return Err(serde::de::Error::duplicate_field("taskName")); + } + task_name__ = Some(map_.next_value()?); + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(SyncNowRequest { + task_name: task_name__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("runtime.SyncNowRequest", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for SyncNowResponse { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.response.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("runtime.SyncNowResponse", len)?; + if let Some(v) = self.response.as_ref() { + match v { + sync_now_response::Response::Ack(v) => { + struct_ser.serialize_field("ack", v)?; + } + sync_now_response::Response::Heartbeat(v) => { + struct_ser.serialize_field("heartbeat", v)?; + } + sync_now_response::Response::Done(v) => { + struct_ser.serialize_field("done", v)?; + } + } + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for SyncNowResponse { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "ack", + "heartbeat", + "done", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Ack, + Heartbeat, + Done, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "ack" => Ok(GeneratedField::Ack), + "heartbeat" => Ok(GeneratedField::Heartbeat), + "done" => Ok(GeneratedField::Done), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = SyncNowResponse; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.SyncNowResponse") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut response__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Ack => { + if response__.is_some() { + return Err(serde::de::Error::duplicate_field("ack")); + } + response__ = map_.next_value::<::std::option::Option<_>>()?.map(sync_now_response::Response::Ack) +; + } + GeneratedField::Heartbeat => { + if response__.is_some() { + return Err(serde::de::Error::duplicate_field("heartbeat")); + } + response__ = map_.next_value::<::std::option::Option<_>>()?.map(sync_now_response::Response::Heartbeat) +; + } + GeneratedField::Done => { + if response__.is_some() { + return Err(serde::de::Error::duplicate_field("done")); + } + response__ = map_.next_value::<::std::option::Option<_>>()?.map(sync_now_response::Response::Done) +; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(SyncNowResponse { + response: response__, + }) + } + } + deserializer.deserialize_struct("runtime.SyncNowResponse", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for sync_now_response::Ack { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let len = 0; + let struct_ser = serializer.serialize_struct("runtime.SyncNowResponse.Ack", len)?; + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for sync_now_response::Ack { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + Ok(GeneratedField::__SkipField__) + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = sync_now_response::Ack; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.SyncNowResponse.Ack") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + while map_.next_key::()?.is_some() { + let _ = map_.next_value::()?; + } + Ok(sync_now_response::Ack { + }) + } + } + deserializer.deserialize_struct("runtime.SyncNowResponse.Ack", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for sync_now_response::Done { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let len = 0; + let struct_ser = serializer.serialize_struct("runtime.SyncNowResponse.Done", len)?; + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for sync_now_response::Done { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + Ok(GeneratedField::__SkipField__) + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = sync_now_response::Done; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.SyncNowResponse.Done") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + while map_.next_key::()?.is_some() { + let _ = map_.next_value::()?; + } + Ok(sync_now_response::Done { + }) + } + } + deserializer.deserialize_struct("runtime.SyncNowResponse.Done", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for sync_now_response::Heartbeat { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let len = 0; + let struct_ser = serializer.serialize_struct("runtime.SyncNowResponse.Heartbeat", len)?; + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for sync_now_response::Heartbeat { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + Ok(GeneratedField::__SkipField__) + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = sync_now_response::Heartbeat; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.SyncNowResponse.Heartbeat") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + while map_.next_key::()?.is_some() { + let _ = map_.next_value::()?; + } + Ok(sync_now_response::Heartbeat { + }) + } + } + deserializer.deserialize_struct("runtime.SyncNowResponse.Heartbeat", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for Task { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/crates/proto-grpc/src/runtime.rs b/crates/proto-grpc/src/runtime.rs index dc963b71d11..f03297c64ed 100644 --- a/crates/proto-grpc/src/runtime.rs +++ b/crates/proto-grpc/src/runtime.rs @@ -1774,3 +1774,327 @@ pub mod shard_server { const NAME: &'static str = SERVICE_NAME; } } +/// Generated client implementations. +#[cfg(feature = "runtime_client")] +pub mod task_control_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::http::Uri; + use tonic::codegen::*; + /// TaskControl is a user-facing control surface for running tasks. + /// It's distinct from the Leader service because it has a different + /// authorization model: callers present ordinary gazette READ claims + /// scoped to the task's shards, not the LEAD capability held by shards. + /// + /// Only the runtime-next sidecar serves TaskControl over gRPC. Users reach it + /// through the reactor front door, which relays to that sidecar but exposes + /// itself as HTTP/NDJSON rather than gRPC — see go/runtime/task_control_http.go. + #[derive(Debug, Clone)] + pub struct TaskControlClient { + inner: tonic::client::Grpc, + } + impl TaskControlClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl TaskControlClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> TaskControlClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + >>::Error: + Into + std::marker::Send + std::marker::Sync, + { + TaskControlClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + /// SyncNow asks a materialization to immediately commit its open + /// transaction, and resolves once that transaction is fully acknowledged: + /// committed and queryable in the endpoint. It's served by the + /// runtime-next sidecar hosting the task's leader session, and relayed + /// by the reactor front door. + /// + /// The stream is exactly one Ack, then zero or more Heartbeats while the + /// caller waits, then exactly one Done, after which the stream closes. + /// SyncNow is idempotent: concurrent calls await the same commit, and a + /// caller which hangs up early has still forced the commit. There is no + /// error for "nothing to do" — a task with nothing to await, including a + /// capture or derivation, Acks and its Done follows immediately. + pub async fn sync_now( + &mut self, + request: impl tonic::IntoRequest<::proto_flow::runtime::SyncNowRequest>, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/runtime.TaskControl/SyncNow"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("runtime.TaskControl", "SyncNow")); + self.inner.server_streaming(req, path, codec).await + } + } +} +/// Generated server implementations. +#[cfg(feature = "runtime_server")] +pub mod task_control_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with TaskControlServer. + #[async_trait] + pub trait TaskControl: std::marker::Send + std::marker::Sync + 'static { + /// Server streaming response type for the SyncNow method. + type SyncNowStream: tonic::codegen::tokio_stream::Stream< + Item = std::result::Result<::proto_flow::runtime::SyncNowResponse, tonic::Status>, + > + std::marker::Send + + 'static; + /// SyncNow asks a materialization to immediately commit its open + /// transaction, and resolves once that transaction is fully acknowledged: + /// committed and queryable in the endpoint. It's served by the + /// runtime-next sidecar hosting the task's leader session, and relayed + /// by the reactor front door. + /// + /// The stream is exactly one Ack, then zero or more Heartbeats while the + /// caller waits, then exactly one Done, after which the stream closes. + /// SyncNow is idempotent: concurrent calls await the same commit, and a + /// caller which hangs up early has still forced the commit. There is no + /// error for "nothing to do" — a task with nothing to await, including a + /// capture or derivation, Acks and its Done follows immediately. + async fn sync_now( + &self, + request: tonic::Request<::proto_flow::runtime::SyncNowRequest>, + ) -> std::result::Result, tonic::Status>; + } + /// TaskControl is a user-facing control surface for running tasks. + /// It's distinct from the Leader service because it has a different + /// authorization model: callers present ordinary gazette READ claims + /// scoped to the task's shards, not the LEAD capability held by shards. + /// + /// Only the runtime-next sidecar serves TaskControl over gRPC. Users reach it + /// through the reactor front door, which relays to that sidecar but exposes + /// itself as HTTP/NDJSON rather than gRPC — see go/runtime/task_control_http.go. + #[derive(Debug)] + pub struct TaskControlServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl TaskControlServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for TaskControlServer + where + T: TaskControl, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/runtime.TaskControl/SyncNow" => { + #[allow(non_camel_case_types)] + struct SyncNowSvc(pub Arc); + impl + tonic::server::ServerStreamingService<::proto_flow::runtime::SyncNowRequest> + for SyncNowSvc + { + type Response = ::proto_flow::runtime::SyncNowResponse; + type ResponseStream = T::SyncNowStream; + type Future = + BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request<::proto_flow::runtime::SyncNowRequest>, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = + async move { ::sync_now(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = SyncNowSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers.insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }), + } + } + } + impl Clone for TaskControlServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "runtime.TaskControl"; + impl tonic::server::NamedService for TaskControlServer { + const NAME: &'static str = SERVICE_NAME; + } +} diff --git a/crates/runtime-next/Cargo.toml b/crates/runtime-next/Cargo.toml index f7e05561555..0a6f0cee50d 100644 --- a/crates/runtime-next/Cargo.toml +++ b/crates/runtime-next/Cargo.toml @@ -88,4 +88,5 @@ build = { path = "../build" } e2e-support = { path = "../e2e-support" } insta = { workspace = true } quickcheck = { workspace = true } +rustls = { version = "0.23", features = ["aws_lc_rs"] } tokio = { workspace = true, features = ["test-util", "macros"] } diff --git a/crates/runtime-next/README.md b/crates/runtime-next/README.md index 0c677ea77e8..338a4acd081 100644 --- a/crates/runtime-next/README.md +++ b/crates/runtime-next/README.md @@ -94,6 +94,7 @@ src/ │ ├── fsm.rs # pipelined HeadFSM / TailFSM state machines │ ├── actor.rs # event loop driving open / commit / acknowledge / trigger │ ├── triggers.rs # webhook trigger delivery +│ ├── sync_now.rs # TaskControl.SyncNow decision evaluation │ ├── sync_schedule.rs # compiled sync-schedule evaluator (commit pacing) │ └── task.rs # Task: the leader's data model │ @@ -388,6 +389,28 @@ ceilings still force early commits — a backfill drains under memory pressure with no caught-up detection — and `CloseNow` bypasses a hold, so spec updates restart promptly. The first transaction of a leader session is never held. +## Sync-now (materialize) + +`TaskControl.SyncNow` is a user-facing RPC forcing an immediate commit of a +materialization's open transaction — collapsing any sync-schedule hold — and +resolving once that transaction is fully acknowledged (committed and +queryable in the destination). The tonic service lives beside `Leader` in +`leader/service.rs` but is registered separately, because callers present +ordinary gazette READ claims over the task's shards rather than LEAD. + +The service delivers the request to the task's live leader session via a +per-session `SyncNowHandle` registered in `ServiceImpl`. The Actor evaluates +a pure decision function (`sync_now::evaluate` over `fsm::sync_now_inputs`), +acks it, arms the FSM's existing `close_requested` input when an open +transaction can still be told to close, and parks the caller as a waiter on +a monotonic count of `Tail::Done` transitions — so N concurrent requests +await the same commit. Parked waiters receive ~15s heartbeats, then Done; a +session that exits with parked waiters errors them out (sync-now is +idempotent — callers re-invoke). The response messages are structural and +carry no payload: transaction statistics belong to the stats journal. +Derivations and captures are answered by the reactor front door; tasks +without a live session are `NOT_FOUND`. + ## Status - `leader::materialize` / `shard::materialize` and `leader::derive` / diff --git a/crates/runtime-next/src/leader/materialize/actor.rs b/crates/runtime-next/src/leader/materialize/actor.rs index 241788103d7..4c2bc749870 100644 --- a/crates/runtime-next/src/leader/materialize/actor.rs +++ b/crates/runtime-next/src/leader/materialize/actor.rs @@ -1,5 +1,6 @@ use super::{Task, fsm}; use crate::proto; +use crate::proto::sync_now_response; use anyhow::Context; use bytes::Bytes; use futures::stream::{BoxStream, FuturesUnordered}; @@ -37,6 +38,13 @@ pub struct Actor { shard_tx: Vec>>, // Future for an in-flight stats flush, if any, yielding ACK intents. stats_write_fut: Option)>>>, + // Clock of the last sync-now Progress heartbeat broadcast. + sync_now_heartbeat_at: uuid::Clock, + // Parked sync-now waiters, resolved as `tail_done_count` reaches their targets. + sync_now_waiters: Vec, + // Monotonic count of Tail arrivals at Done within this session: + // the anchor against which sync-now waiter targets are expressed. + tail_done_count: u64, // Task being executed by this actor. task: Task, // Leader-lifetime trigger debounce accumulator and last-fire times. @@ -45,6 +53,17 @@ pub struct Actor { trigger_fut: Option>>, } +/// A parked `TaskControl.SyncNow` caller, resolved when the actor's count of +/// Tail::Done transitions reaches its target. +struct SyncNowWaiter { + target: u64, + reply_tx: mpsc::UnboundedSender>, +} + +/// Cadence of heartbeats to parked sync-now waiters. Coarse: beats keep +/// hour-long streams alive through LB idle timeouts. +const SYNC_NOW_HEARTBEAT: Duration = Duration::from_secs(15); + impl Actor { pub fn new( backfill_begin: BTreeMap, @@ -69,6 +88,9 @@ impl Actor { pending_ack_intents: BTreeMap::new(), shard_tx, stats_write_fut: None, + sync_now_heartbeat_at: uuid::Clock::zero(), + sync_now_waiters: Vec::new(), + tail_done_count: 0, task, trigger_debounce: fsm::TriggerDebounce::default(), trigger_fut: None, @@ -77,11 +99,47 @@ impl Actor { #[tracing::instrument(level = "debug", err(Debug, level = "warn"), skip_all)] pub async fn serve( + &mut self, + head: fsm::Head, + tail: fsm::Tail, + session: S, + shard_rx: Vec>>, + mut sync_now_rx: mpsc::UnboundedReceiver, + ) -> anyhow::Result<()> { + let result = self + .serve_inner(head, tail, session, shard_rx, &mut sync_now_rx) + .await; + + // Parked waiters (and undelivered requests) cannot resolve once this + // session ends: the Tail::Done they await happens — if ever — in a + // future session. Error them out; sync-now is idempotent and callers + // re-invoke to await the next session. + // + // Close first, so that a request racing this drain fails to send and + // is answered Unavailable by the service, rather than landing in a + // channel nobody will read again. + sync_now_rx.close(); + + let status = tonic::Status::unavailable( + "leader session ended before the awaited transaction was fully acknowledged; retry", + ); + while let Ok(request) = sync_now_rx.try_recv() { + let _ = request.reply_tx.send(Err(status.clone())); + } + for waiter in self.sync_now_waiters.drain(..) { + let _ = waiter.reply_tx.send(Err(status.clone())); + } + + result + } + + async fn serve_inner( &mut self, mut head: fsm::Head, mut tail: fsm::Tail, mut session: S, shard_rx: Vec>>, + sync_now_rx: &mut mpsc::UnboundedReceiver, ) -> anyhow::Result<()> { service_kit::event!( tracing::Level::INFO, @@ -152,6 +210,7 @@ impl Actor { let mut action: fsm::Action; let prev_kind = tail.kind(); + let prev_resolves = !matches!(tail, fsm::Tail::Done(_) | fsm::Tail::Begin(_)); (action, tail) = tail.step( &self.trigger_debounce, self.intents_write_fut.is_none(), @@ -171,6 +230,15 @@ impl Actor { "transition", ); } + // Tail arriving at Done is the "committed and queryable" instant + // that sync-now waiters await. The Begin→Done stopping shortcut is + // excluded: it defers connector acknowledgement to a future + // session rather than completing it, so its waiters must not + // resolve (they instead error when this session exits). + if prev_resolves && matches!(tail, fsm::Tail::Done(_)) { + self.tail_done_count += 1; + self.resolve_sync_now_waiters(); + } self.merge_backfill_clocks(&mut action); let tail_wake_after = self.dispatch(action)?; @@ -229,6 +297,7 @@ impl Actor { } }; let wake_after = std::cmp::min(head_wake_after, tail_wake_after); + let wake_after = self.sync_now_heartbeat(now, wake_after); // If `head` and `tail` are awaiting IO and `ready_shard_rx` was not // consumed by either, then it was unexpected and is a protocol error. @@ -303,6 +372,14 @@ impl Actor { } shard_rx.push(next_shard_rx((shard_index, rx))); } + // Receive a sync-now request for this task. + Some(request) = sync_now_rx.recv() => { + // Resync before parking a waiter on this clock: `now` + // predates the park, which for a held transaction can be + // hours long. + now.update(now_clock()); + self.on_sync_now(request, &head, &tail, &mut close_requested, now); + } // Receive a requested NextCheckpoint frontier. result = session.recv_checkpoint(), if checkpoint_requested => { let frontier = result?; @@ -665,6 +742,97 @@ impl Actor { Ok(Some(msg)) } + /// Receive a sync-now request: acknowledge it, arm `close_requested` when + /// the decision calls for it, and either resolve immediately (nothing to + /// await) or park a waiter on its target count of Tail::Done transitions. + /// Concurrent requests over the same state share a target and resolve + /// together. + fn on_sync_now( + &mut self, + request: super::SyncNow, + head: &fsm::Head, + tail: &fsm::Tail, + close_requested: &mut bool, + now: uuid::Clock, + ) { + let decision = super::sync_now::evaluate(fsm::sync_now_inputs(head, tail)); + + service_kit::event!( + tracing::Level::INFO, + "leader", + set_close_requested = decision.set_close_requested, + await_dones = decision.await_dones, + "received sync-now request", + ); + + if decision.set_close_requested { + *close_requested = true; + } + let _ = request.reply_tx.send(Ok(proto::SyncNowResponse { + response: Some(sync_now_response::Response::Ack(sync_now_response::Ack {})), + })); + + if decision.await_dones == 0 { + let _ = request.reply_tx.send(Ok(proto::SyncNowResponse { + response: Some(sync_now_response::Response::Done( + sync_now_response::Done {}, + )), + })); + return; + } + + if self.sync_now_waiters.is_empty() { + self.sync_now_heartbeat_at = now; + } + self.sync_now_waiters.push(SyncNowWaiter { + target: self.tail_done_count + decision.await_dones, + reply_tx: request.reply_tx, + }); + } + + /// Resolve waiters whose target Tail::Done count has been reached. + fn resolve_sync_now_waiters(&mut self) { + let count = self.tail_done_count; + self.sync_now_waiters.retain(|waiter| { + if waiter.target > count { + return true; + } + let _ = waiter.reply_tx.send(Ok(proto::SyncNowResponse { + response: Some(sync_now_response::Response::Done( + sync_now_response::Done {}, + )), + })); + false + }); + } + + /// While sync-now waiters are parked, emit a periodic heartbeat to each + /// and bound the actor's sleep so the next beat isn't overslept. + /// A waiter that hung up is dropped silently: its request already landed. + fn sync_now_heartbeat(&mut self, now: uuid::Clock, wake_after: Duration) -> Duration { + if self.sync_now_waiters.is_empty() { + return wake_after; + } + let elapsed = uuid::Clock::delta(now, self.sync_now_heartbeat_at); + if elapsed < SYNC_NOW_HEARTBEAT { + return wake_after.min(SYNC_NOW_HEARTBEAT - elapsed); + } + + self.sync_now_waiters.retain(|waiter| { + waiter + .reply_tx + .send(Ok(proto::SyncNowResponse { + response: Some(sync_now_response::Response::Heartbeat( + sync_now_response::Heartbeat {}, + )), + })) + .is_ok() + }); + self.sync_now_heartbeat_at = now; + + wake_after.min(SYNC_NOW_HEARTBEAT) + } + /// Synchronously fan out a single leader message to every shard. fn broadcast(&self, msg: proto::Materialize) { let (head, tail) = self.shard_tx.split_first().unwrap(); diff --git a/crates/runtime-next/src/leader/materialize/fsm.rs b/crates/runtime-next/src/leader/materialize/fsm.rs index 1a5d840c5e3..0902d7e0992 100644 --- a/crates/runtime-next/src/leader/materialize/fsm.rs +++ b/crates/runtime-next/src/leader/materialize/fsm.rs @@ -1248,6 +1248,29 @@ fn compute_open_duration( hold..hold } +/// Gather the POD inputs of a sync-now decision from live FSM state. +/// Lives here (rather than with `sync_now::evaluate`) because it reads +/// private `Extents` fields. +pub(crate) fn sync_now_inputs(head: &Head, tail: &Tail) -> super::sync_now::Inputs { + let (head_open, head_deciding) = match head { + Head::Idle(s) => { + let is_open = s.extents.open != uuid::Clock::zero(); + (is_open, is_open) + } + Head::Extend(_) => (true, true), + Head::Stop => (false, false), + // Flush / Persist / Store / WriteStats / StartCommit: the close + // decision is behind us and the transaction is already closing. + _ => (true, false), + }; + + super::sync_now::Inputs { + head_open, + head_deciding, + tail_done: matches!(tail, Tail::Done(_)), + } +} + /// Leader-lifetime debounce state for materialization triggers. Accumulates /// per-transaction windows and gates firing to at most once per the task's /// configured trigger `interval`. diff --git a/crates/runtime-next/src/leader/materialize/handler.rs b/crates/runtime-next/src/leader/materialize/handler.rs index 85c70476e65..8fae7b8ef4d 100644 --- a/crates/runtime-next/src/leader/materialize/handler.rs +++ b/crates/runtime-next/src/leader/materialize/handler.rs @@ -130,6 +130,15 @@ where handler.set_phase("starting"); let metrics = super::Metrics::new(&slots[0].join.shards[0].id); + // Accept TaskControl.SyncNow requests for the life of this session; + // the guard un-registers on any exit from this scope. + let (sync_now_tx, sync_now_rx) = mpsc::unbounded_channel(); + let _sync_now_guard = service.register_sync_now_handle( + &task_name, + slots[0].join.shards[0].id.clone(), + sync_now_tx, + ); + service_kit::event!( tracing::Level::INFO, "leader", @@ -252,7 +261,9 @@ where task, ); handler.set_phase("running"); - actor.serve(head, tail, session, shard_rx).await + actor + .serve(head, tail, session, shard_rx, sync_now_rx) + .await } .await; diff --git a/crates/runtime-next/src/leader/materialize/mod.rs b/crates/runtime-next/src/leader/materialize/mod.rs index 87a0d168191..00e94eef150 100644 --- a/crates/runtime-next/src/leader/materialize/mod.rs +++ b/crates/runtime-next/src/leader/materialize/mod.rs @@ -2,6 +2,7 @@ mod actor; mod fsm; mod handler; mod startup; +pub(crate) mod sync_now; mod sync_schedule; mod task; mod triggers; @@ -9,6 +10,7 @@ mod triggers; use super::close_policy; pub(crate) use handler::serve; +pub(crate) use sync_now::SyncNow; #[derive(Clone)] pub(crate) struct Metrics { diff --git a/crates/runtime-next/src/leader/materialize/sync_now.rs b/crates/runtime-next/src/leader/materialize/sync_now.rs new file mode 100644 index 00000000000..6bed4a28d5e --- /dev/null +++ b/crates/runtime-next/src/leader/materialize/sync_now.rs @@ -0,0 +1,140 @@ +//! Sync-now evaluation: the pure decision at the heart of +//! `TaskControl.SyncNow`. The tonic service (`leader/service.rs`) delivers a +//! [`SyncNow`] request to the task's leader Actor, which gathers [`Inputs`] +//! from its live FSM state (`fsm::sync_now_inputs`) and applies [`evaluate`] +//! to decide what the caller's stream then awaits. + +use crate::proto; +use tokio::sync::mpsc; + +/// A sync-now request delivered to a materialize leader Actor, carrying the +/// reply channel that feeds the caller's SyncNow response stream. +pub(crate) struct SyncNow { + pub reply_tx: mpsc::UnboundedSender>, +} + +/// POD inputs of a sync-now decision. +#[derive(Debug, Copy, Clone)] +pub(crate) struct Inputs { + /// An in-flight transaction exists: opened, and its commit not yet fully + /// persisted. + pub head_open: bool, + /// The close decision is still ahead (Head is Idle or Extend), so setting + /// `close_requested` can shorten the transaction. + pub head_deciding: bool, + /// Tail is Done: the prior transaction is fully acknowledged. + pub tail_done: bool, +} + +/// What a sync-now request decided: whether to arm `close_requested`, and how +/// many future `Tail::Done` transitions the caller's stream awaits before Done +/// (zero resolves immediately). +#[derive(Debug, Copy, Clone, PartialEq)] +pub(crate) struct Decision { + pub set_close_requested: bool, + pub await_dones: u64, +} + +/// Decide a sync-now request from POD state. +/// +/// One wait rule covers every case: the caller awaits one `Tail::Done` per +/// pipeline stage that's ahead of it — the in-flight Head transaction (if any) +/// and the draining Tail transaction (if not already Done). A task which is +/// fully current awaits nothing and resolves immediately. +pub(crate) fn evaluate(inputs: Inputs) -> Decision { + let Inputs { + head_open, + head_deciding, + tail_done, + } = inputs; + + Decision { + // Arm `close_requested` only for an open transaction whose close + // decision is still ahead. Past that point the transaction is already + // closing and arming it would be harmless but pointless. + set_close_requested: head_open && head_deciding, + await_dones: head_open as u64 + !tail_done as u64, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Table-driven coverage of `evaluate`, in the style of + /// `close_policy_table`: every row of the design mapping, plus the + /// pipelined Head-open-while-Tail-drains cases. + #[test] + fn decision_table() { + struct Case { + name: &'static str, + inputs: Inputs, + want: Decision, + } + let mk = |head_open, head_deciding, tail_done| Inputs { + head_open, + head_deciding, + tail_done, + }; + + let cases = [ + Case { + name: "open transaction before its close decision: shorten it and wait", + inputs: mk(true, true, true), + want: Decision { + set_close_requested: true, + await_dones: 1, + }, + }, + Case { + name: "pipelined: Head extends while Tail drains, await both", + inputs: mk(true, true, false), + want: Decision { + set_close_requested: true, + await_dones: 2, + }, + }, + Case { + name: "Head past the close decision (flushing / committing): don't arm close", + inputs: mk(true, false, true), + want: Decision { + set_close_requested: false, + await_dones: 1, + }, + }, + Case { + name: "Head committing while Tail still drains: await both", + inputs: mk(true, false, false), + want: Decision { + set_close_requested: false, + await_dones: 2, + }, + }, + Case { + name: "no open transaction, Tail draining acknowledgement: wait, don't arm close", + inputs: mk(false, false, false), + want: Decision { + set_close_requested: false, + await_dones: 1, + }, + }, + Case { + name: "no open transaction, Tail done: fully current, nothing to await", + inputs: mk(false, false, true), + want: Decision { + set_close_requested: false, + await_dones: 0, + }, + }, + ]; + + for case in cases { + let got = evaluate(case.inputs); + assert_eq!( + got, case.want, + "case `{}` failed: inputs={:?}", + case.name, case.inputs, + ); + } + } +} diff --git a/crates/runtime-next/src/leader/service.rs b/crates/runtime-next/src/leader/service.rs index 81ffb335191..5918c781a34 100644 --- a/crates/runtime-next/src/leader/service.rs +++ b/crates/runtime-next/src/leader/service.rs @@ -42,6 +42,18 @@ pub struct ServiceImpl< pub(crate) registry: service_kit::Registry, /// When true, disarm AuthN+AuthZ enforcement (trusted local contexts only). pub(crate) disarm_auth: bool, + /// Sync-now handles of live Materialize leader sessions, keyed by task + /// name: the delivery points for TaskControl.SyncNow requests. + pub(crate) sync_now_handles: std::sync::Mutex>, +} + +/// A live Materialize session's sync-now delivery handle. +pub(crate) struct SyncNowHandle { + /// Shard-zero ID of the session: the concrete scope that a TaskControl + /// caller's claims must authorize. + shard_zero: String, + /// SyncNow delivery channel into the session's Actor. + sync_now_tx: mpsc::UnboundedSender, } impl @@ -63,6 +75,7 @@ impl proto_grpc::runtime::task_control_server::TaskControlServer { + proto_grpc::runtime::task_control_server::TaskControlServer::new(self) + .max_decoding_message_size(crate::MAX_MESSAGE_SIZE) + .max_encoding_message_size(usize::MAX) + } + + /// Register a live Materialize session's sync-now handle, replacing any + /// prior registration for the task. The returned guard un-registers it + /// when dropped — tie the guard to the session's serve scope. + pub(crate) fn register_sync_now_handle( + &self, + task_name: &str, + shard_zero: String, + sync_now_tx: mpsc::UnboundedSender, + ) -> SyncNowGuard { + self.sync_now_handles.lock().unwrap().insert( + task_name.to_string(), + SyncNowHandle { + shard_zero, + sync_now_tx: sync_now_tx.clone(), + }, + ); + SyncNowGuard { + service: self.clone(), + task_name: task_name.to_string(), + sync_now_tx, + } + } + pub fn spawn_derive( &self, authz: proto_grpc::Authorizer, @@ -160,3 +207,88 @@ impl { + service: Service, + task_name: String, + sync_now_tx: mpsc::UnboundedSender, +} + +impl Drop + for SyncNowGuard +{ + fn drop(&mut self) { + let mut guard = self.service.sync_now_handles.lock().unwrap(); + // Remove only our own registration: a replacement session for the + // same task may have re-registered while this one wound down. + if guard + .get(&self.task_name) + .is_some_and(|handle| handle.sync_now_tx.same_channel(&self.sync_now_tx)) + { + guard.remove(&self.task_name); + } + } +} + +#[tonic::async_trait] +impl + proto_grpc::runtime::task_control_server::TaskControl for Service +{ + type SyncNowStream = + tokio_stream::wrappers::UnboundedReceiverStream>; + + async fn sync_now( + &self, + mut request: tonic::Request, + ) -> tonic::Result> { + let authz = proto_grpc::Authorizer::from_request(&mut request, self.disarm_auth)?; + let proto::SyncNowRequest { task_name } = request.into_inner(); + + if task_name.is_empty() { + return Err(tonic::Status::invalid_argument("task_name is required")); + } + + // A live Materialize session? Authorize the caller against its + // concrete shard-zero ID and deliver the request; the Actor drives + // the response stream from there. + let handle = { + let guard = self.sync_now_handles.lock().unwrap(); + guard + .get(&task_name) + .map(|handle| (handle.shard_zero.clone(), handle.sync_now_tx.clone())) + }; + let Some((shard_zero, sync_now_tx)) = handle else { + // Not running here, not on the V2 runtime, or not a + // materialization at all: only the reactor front door can tell + // those apart, since only it has the shard keyspace. It answers + // for captures and derivations without dialing us. + return Err(not_found(&task_name)); + }; + let _authorized = authz.authorize_id(&shard_zero)?; + + let (reply_tx, reply_rx) = mpsc::unbounded_channel(); + if sync_now_tx + .send(super::materialize::SyncNow { reply_tx }) + .is_ok() + { + return Ok(tonic::Response::new( + tokio_stream::wrappers::UnboundedReceiverStream::new(reply_rx), + )); + } + // The session exited between lookup and delivery. + Err(tonic::Status::unavailable(format!( + "leader session of task {task_name} is winding down; retry" + ))) + } +} + +fn not_found(task_name: &str) -> tonic::Status { + tonic::Status::not_found(format!( + "task {task_name} has no live leader session here (it may not be running, or may not be on the V2 runtime)" + )) +} diff --git a/crates/runtime-next/tests/sync_now_e2e.flow.yaml b/crates/runtime-next/tests/sync_now_e2e.flow.yaml new file mode 100644 index 00000000000..1f444273b1a --- /dev/null +++ b/crates/runtime-next/tests/sync_now_e2e.flow.yaml @@ -0,0 +1,37 @@ +collections: + testing/source: + schema: + type: object + required: [id] + properties: + id: { type: string } + payload: { type: string } + key: [/id] + +materializations: + # Schedule-paced task: commits land on a 2h grid, so an open transaction + # (after the session's first, which is never schedule-held) is genuinely + # held until a SyncNow collapses the hold. + testing/sync-now: + endpoint: + connector: + image: test/image + config: {} + syncSchedule: { baseInterval: 2h } + shards: { flags: { enable-runtime-v2: "true" } } + bindings: + - source: testing/source + resource: { table: sync_now } + + # No schedule; a 300s minimum-transaction-duration floor holds the open + # transaction instead, exercising the CLOSE_REQUESTED outcome (which + # bypasses the floor). + testing/sync-now-floor: + endpoint: + connector: + image: test/image + config: {} + shards: { minTxnDuration: 300s, flags: { enable-runtime-v2: "true" } } + bindings: + - source: testing/source + resource: { table: sync_now_floor } diff --git a/crates/runtime-next/tests/sync_now_e2e.rs b/crates/runtime-next/tests/sync_now_e2e.rs new file mode 100644 index 00000000000..1b2723c08e4 --- /dev/null +++ b/crates/runtime-next/tests/sync_now_e2e.rs @@ -0,0 +1,956 @@ +//! End-to-end integration test of `TaskControl.SyncNow`: a real tonic server +//! hosting the `Leader` and `TaskControl` services behind armed AuthN +//! interceptors, a real materialize leader session, and real journal IO for +//! stats and ACK-intent writes — the IO that gates `Tail::Done`, the instant +//! sync-now waiters resolve at. +//! +//! The test plays the shard side itself: it acts as shard zero of a +//! single-shard topology, speaking the `Materialize` session protocol over +//! the Leader service (Join → Task → Recover → Apply → Open, then the +//! Load / Flush / Store / StartCommit / Acknowledge transaction flow) with a +//! scripted "connector-side" that acknowledges commits only when the test +//! says so. Withholding `Acknowledged` is the test's control over the moment +//! a transaction becomes fully acknowledged. +//! +//! Source checkpoints come from a fixture `ShuffleSessionFactory` fed by the +//! test — one synthetic `shuffle::Frontier` per transaction — so no journals +//! are read. Multi-shard session mechanics are deliberately NOT exercised: a +//! single-shard Join is immediate consensus, and sync-now waiter semantics +//! don't depend on shard count. +//! +//! Spawns real `etcd` (on PATH) and `~/go/bin/gazette` child processes via +//! `e2e_support::DataPlane`, exactly like the `shuffle` scenario tests that +//! run under `ci:nextest-run`. + +use prost::Message; +use proto_flow::flow; +use proto_gazette::{broker, uuid}; +use runtime_next::proto; +use runtime_next::proto::sync_now_response; +use std::time::Duration; +use tokio::sync::mpsc; + +/// The schedule-paced fixture task: its open transactions (after the +/// session's first) are held for up to the 2h `baseInterval`. +const SCHEDULED_TASK: &str = "testing/sync-now"; +/// The fixture task held by a 300s minimum-transaction-duration floor. +const FLOOR_TASK: &str = "testing/sync-now-floor"; + +/// Ops-stats journal that leader commits publish stats and ACK intents to. +/// Pre-created by the harness; carried in the Join's `stats_journal` labeling. +const OPS_STATS_JOURNAL: &str = "testing/ops/stats"; + +/// Source documents reported by each scripted `Loaded`. +const DOCS_PER_TXN: u64 = 3; + +/// Bound on every await of the driver and of sync-now streams. Also the +/// promptness assertion: a schedule-collapsed or floor-bypassed commit must +/// complete well within it (the alternative was a 2h hold / 300s floor). +const EXPECT_TIMEOUT: Duration = Duration::from_secs(30); + +/// Window over which a held transaction must leave the leader silent: absent +/// a poke, the shard is asked for nothing until the hold expires. +const HELD_QUIET_WINDOW: Duration = Duration::from_millis(500); + +const ISSUER: &str = "sync-now.test"; +const SECRET: &[u8] = b"sync-now-e2e-secret"; + +fn shard_zero_id(task_name: &str) -> String { + format!("materialize/{task_name}/0011223344556677/00000000-00000000") +} + +/// A fixture [`runtime_next::ShuffleSessionFactory`]: sessions relay the +/// Frontiers pushed by the test, reading no journals. The single receiver is +/// shared behind a mutex so a session owns it for its lifetime. +struct FixtureShuffleFactory { + frontier_rx: std::sync::Arc>>, +} + +impl runtime_next::ShuffleSessionFactory for FixtureShuffleFactory { + type Session = FixtureFrontiers; + + async fn open( + &self, + _task: shuffle::proto::Task, + _shards: Vec, + _resume: shuffle::Frontier, + ) -> anyhow::Result { + Ok(FixtureFrontiers { + frontier_rx: self.frontier_rx.clone().lock_owned().await, + }) + } +} + +/// A fixture [`runtime_next::ShuffleSession`]: yields one queued Frontier per +/// checkpoint request, and parks forever once the channel is idle or closed +/// (the leader keeps exactly one request in flight; an unanswered request is +/// simply an idle task). +struct FixtureFrontiers { + frontier_rx: tokio::sync::OwnedMutexGuard>, +} + +impl runtime_next::ShuffleSession for FixtureFrontiers { + fn request_checkpoint(&self) { + // No request protocol: `recv_checkpoint` pops the next queued frontier. + } + + async fn recv_checkpoint(&mut self) -> anyhow::Result { + match self.frontier_rx.recv().await { + Some(frontier) => Ok(frontier), + None => std::future::pending().await, + } + } + + async fn close(self) -> anyhow::Result<()> { + Ok(()) + } +} + +/// Synthetic checkpoint Frontier for the `seq`-th transaction: binding zero, +/// one producer whose committed Clock strictly increases and whose negative +/// `offset` (a closed span) grows in magnitude. `flushed_lsn` carries one +/// entry for the single shard; the scripted shard never reads segments, so +/// its value is inert. +fn synthetic_frontier(seq: u64) -> shuffle::Frontier { + shuffle::Frontier::new( + vec![shuffle::JournalFrontier { + journal: "testing/source/pivot=00".into(), + binding: 0, + producers: vec![shuffle::ProducerFrontier { + producer: uuid::Producer::from_bytes([7, 19, 83, 3, 3, 17]), + last_commit: uuid::Clock::from_unix(1_000_000 + seq, 0), + hinted_commit: uuid::Clock::from_u64(0), + offset: -((seq * 1_000) as i64), + }], + bytes_read_delta: (DOCS_PER_TXN * 100) as i64, + bytes_behind_delta: 0, + }], + vec![0], + ) + .expect("synthetic frontier is well-formed") +} + +type SyncNowStream = tonic::Streaming; + +/// One test's hermetic world: a DataPlane, the built fixture catalog, and an +/// in-process tonic server hosting `Leader` + `TaskControl` behind armed +/// authentication interceptors — assembled exactly as the runtime sidecar +/// serves them. +struct Harness { + data_plane: e2e_support::DataPlane, + endpoint: String, + server_task: tokio::task::JoinHandle>, + signer: proto_grpc::Signer, + frontier_tx: mpsc::UnboundedSender, + /// Built `MaterializationSpec` bytes, keyed by task name. + specs: std::collections::BTreeMap, + /// Monotonic ordinal of pushed frontiers, for strictly-increasing clocks. + txn_seq: std::sync::atomic::AtomicU64, +} + +impl Harness { + async fn start() -> Harness { + // The tonic/reqwest TLS stack requires a process-level provider, as + // installed by every production `main` (e.g. runtime-sidecar's). + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let data_plane = + e2e_support::DataPlane::start(e2e_support::DataPlaneArgs { broker_count: 1 }) + .await + .expect("DataPlane start"); + + let source = build::arg_source_to_url("./tests/sync_now_e2e.flow.yaml", false).unwrap(); + let build::Output { built, .. } = build::for_local_test(&source, true) + .await + .into_result() + .expect("fixture build"); + + let specs: std::collections::BTreeMap = built + .built_materializations + .iter() + .map(|row| { + let spec = row.spec.as_ref().expect("built materialization has a spec"); + (spec.name.clone(), spec.encode_to_vec().into()) + }) + .collect(); + + // Pre-create the ops-stats journal: commits publish stats and ACK + // intents to it, and appends require the journal to exist. A clone of + // the built collection's partition template carries a valid fragment + // spec; replication drops to the single test broker. + let mut ops_spec = built + .built_collections + .get_key(&models::Collection::new("testing/source")) + .expect("built source collection") + .spec + .as_ref() + .expect("collection spec") + .partition_template + .clone() + .expect("partition template"); + ops_spec.name = OPS_STATS_JOURNAL.to_string(); + ops_spec.replication = 1; + + data_plane + .journal_client + .apply(broker::ApplyRequest { + changes: vec![broker::apply_request::Change { + expect_mod_revision: 0, // Created by this Apply. + upsert: Some(ops_spec), + delete: String::new(), + }], + }) + .await + .expect("creating ops stats journal"); + + let (frontier_tx, frontier_rx) = mpsc::unbounded_channel(); + let shuffle_factory = FixtureShuffleFactory { + frontier_rx: std::sync::Arc::new(tokio::sync::Mutex::new(frontier_rx)), + }; + + let journal_client = data_plane.journal_client.clone(); + let publisher_factory = runtime_next::JournalPublisherFactory::new(std::sync::Arc::new( + move |_authz_sub, _authz_obj| journal_client.clone(), + )); + + let svc = runtime_next::Service::new( + shuffle_factory, + publisher_factory, + runtime_next::TracingLoggerFactory, + service_kit::Registry::new(), + false, // AuthN+AuthZ stays armed. + ); + + // Serve both services behind interceptors, exactly as the sidecar + // does: Leader requires LEAD; TaskControl requires gazette READ. + let authn = proto_grpc::Authenticator::new( + ISSUER.to_string(), + vec![tokens::jwt::DecodingKey::from_secret(SECRET)], + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("binding ephemeral listener"); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + + let server_task = tokio::spawn( + tonic::transport::Server::builder() + .add_service(tonic::service::interceptor::InterceptedService::new( + svc.clone().into_tonic_service(), + authn.clone().interceptor(proto_flow::capability::LEAD), + )) + .add_service(tonic::service::interceptor::InterceptedService::new( + svc.into_task_control_service(), + authn.interceptor(proto_gazette::capability::READ), + )) + .serve_with_incoming(tokio_stream::wrappers::TcpListenerStream::new(listener)), + ); + + let signer = proto_grpc::Signer::new( + ISSUER.to_string(), + tokens::jwt::EncodingKey::from_secret(SECRET), + ); + + Harness { + data_plane, + endpoint, + server_task, + signer, + frontier_tx, + specs, + txn_seq: std::sync::atomic::AtomicU64::new(0), + } + } + + async fn stop(self) { + self.server_task.abort(); + self.data_plane + .graceful_stop() + .await + .expect("graceful stop"); + } + + fn spec_bytes(&self, task_name: &str) -> bytes::Bytes { + self.specs + .get(task_name) + .unwrap_or_else(|| panic!("fixture has no built task {task_name}")) + .clone() + } + + /// Push one synthetic checkpoint Frontier, feeding the leader's next + /// transaction. + fn push_frontier(&self) { + let seq = self + .txn_seq + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + + 1; + self.frontier_tx + .send(synthetic_frontier(seq)) + .expect("frontier channel is open"); + } + + /// Mint a gazette-READ bearer scoped to `id_prefix`, as any + /// `/authorize/user/task` Read-level caller would hold. + fn read_token(&self, id_prefix: &str) -> proto_grpc::Metadata { + self.token(proto_gazette::capability::READ, id_prefix) + } + + fn token(&self, capability: u32, id_prefix: &str) -> proto_grpc::Metadata { + let token = self + .signer + .sign( + capability, + "test-caller".to_string(), + broker::LabelSelector { + include: Some(labels::build_set([("id:prefix", id_prefix)])), + exclude: None, + }, + tokens::TimeDelta::hours(1), + ) + .expect("signing test token"); + proto_grpc::Metadata::new() + .with_bearer_token(&token) + .expect("building bearer metadata") + } + + /// Open a SyncNow stream for `task_name` bearing `metadata`. + async fn sync_now( + &self, + metadata: proto_grpc::Metadata, + task_name: &str, + ) -> tonic::Result { + let channel = gazette::dial_channel(&self.endpoint).expect("dialing test endpoint"); + let response = + proto_grpc::runtime::task_control_client::TaskControlClient::with_interceptor( + channel, metadata, + ) + .sync_now(proto::SyncNowRequest { + task_name: task_name.to_string(), + }) + .await?; + Ok(response.into_inner()) + } + + /// SyncNow with a correctly-scoped READ token, consuming the Ack and + /// returning the still-open stream. + async fn poke(&self, task_name: &str) -> SyncNowStream { + let metadata = self.read_token(&format!("materialize/{task_name}/")); + let mut stream = self + .sync_now(metadata, task_name) + .await + .expect("SyncNow call"); + expect_ack(&mut stream).await; + stream + } + + /// Drain barrier: poke and await its Done, which proves the leader reached + /// `Tail::Done` — so the next transaction starts from a fully-drained + /// leader (deterministic `await_dones`). Idempotent re-poking is exercised + /// for free. Only valid while no transaction is open: an open transaction + /// would arm `close_requested` and park us until the shard side commits. + async fn poke_until_drained(&self, task_name: &str) { + let mut stream = self.poke(task_name).await; + expect_done(&mut stream).await; + } + + /// Act as shard zero of `task_name`: dial the Leader service with a LEAD + /// bearer and run session startup through the leader's first (recovered, + /// empty) acknowledgement cycle, so the session begins fully drained. + async fn join_session(&self, task_name: &str) -> ShardDriver { + let shuffle_dir = tempfile::tempdir().expect("shuffle tempdir"); + let shard_id = shard_zero_id(task_name); + + let channel = gazette::dial_channel(&self.endpoint).expect("dialing test endpoint"); + let metadata = self + .signer + .shard_bearer(proto_flow::capability::LEAD, &shard_id) + .expect("minting LEAD bearer"); + let mut client = + proto_grpc::runtime::leader_client::LeaderClient::with_interceptor(channel, metadata); + + let (to_leader, request_rx) = mpsc::unbounded_channel(); + let from_leader = client + .materialize(tokio_stream::wrappers::UnboundedReceiverStream::new( + request_rx, + )) + .await + .expect("opening leader Materialize stream") + .into_inner(); + + let join = proto::Join { + etcd_mod_revision: 1, + shards: vec![proto::join::Shard { + id: shard_id, + labeling: Some(ops::ShardLabeling { + build: "0011223344556677".to_string(), + range: Some(flow::RangeSpec { + key_begin: 0, + key_end: u32::MAX, + r_clock_begin: 0, + r_clock_end: u32::MAX, + }), + task_name: task_name.to_string(), + task_type: ops::TaskType::Materialization as i32, + stats_journal: OPS_STATS_JOURNAL.to_string(), + ..Default::default() + }), + reactor: Some(Default::default()), + etcd_create_revision: 1, + }], + shard_index: 0, + shuffle_directory: shuffle_dir.path().to_string_lossy().into_owned(), + shuffle_endpoint: "http://shuffle.invalid".to_string(), + leader_endpoint: String::new(), + }; + + let mut driver = ShardDriver { + to_leader, + from_leader, + _shuffle_dir: shuffle_dir, + }; + + driver.send(proto::Materialize { + join: Some(join), + ..Default::default() + }); + let msg = driver.expect("Joined").await; + let joined = msg + .joined + .clone() + .unwrap_or_else(|| panic!("expected Joined, got {msg:?}")); + assert_eq!( + joined.max_etcd_revision, 0, + "single-shard Join is immediate consensus", + ); + + driver.send(proto::Materialize { + task: Some(proto::Task { + spec: self.spec_bytes(task_name), + max_transactions: 0, + sqlite_vfs_uri: String::new(), + publisher_id: bytes::Bytes::copy_from_slice( + runtime_next::new_producer().as_bytes(), + ), + }), + ..Default::default() + }); + // A fresh task: nothing recovered from RocksDB. + driver.send(proto::Materialize { + recover: Some(proto::Recover::default()), + ..Default::default() + }); + + // Apply loop: no connector patches. The leader then persists + // `last_applied` (echoed transparently by `expect`) and Opens. + let msg = driver.expect("Apply").await; + assert!(msg.apply.is_some(), "expected Apply, got {msg:?}"); + driver.send(proto::Materialize { + applied: Some(proto::Applied { + action_description: String::new(), + connector_patches_json: bytes::Bytes::new(), + }), + ..Default::default() + }); + + // An empty connector checkpoint makes startup reconciliation a no-op + // fixed point: no rescan Persists. + let msg = driver.expect("Open").await; + assert!(msg.open.is_some(), "expected Open, got {msg:?}"); + driver.send(proto::Materialize { + opened: Some(proto::materialize::Opened { + container: None, + connector_checkpoint: None, + }), + ..Default::default() + }); + + // The actor's first act is acknowledging the recovered (empty) prior + // transaction; answer it so the session starts fully drained. + driver.expect_acknowledge().await; + driver.send_acknowledged(); + + driver + } +} + +/// The scripted shard side of one Leader.Materialize session. +struct ShardDriver { + to_leader: mpsc::UnboundedSender, + from_leader: tonic::Streaming, + _shuffle_dir: tempfile::TempDir, +} + +impl ShardDriver { + fn send(&self, msg: proto::Materialize) { + self.to_leader + .send(msg) + .expect("leader request stream is open"); + } + + /// Receive the next leader message, transparently echoing `Persisted` for + /// any `Persist`: the leader persists at multiple FSM points (hint before + /// Store, commit after StartedCommit, legacy-checkpoint fields included), + /// and answering them generically keeps this driver robust to + /// FSM-internal ordering. + async fn expect(&mut self, awaiting: &'static str) -> proto::Materialize { + loop { + let msg = tokio::time::timeout(EXPECT_TIMEOUT, self.from_leader.message()) + .await + .unwrap_or_else(|_| panic!("timed out awaiting {awaiting}")) + .unwrap_or_else(|err| panic!("leader stream error awaiting {awaiting}: {err}")) + .unwrap_or_else(|| panic!("unexpected leader EOF awaiting {awaiting}")); + + if let Some(persist) = &msg.persist { + assert!( + !persist.rescan, + "unexpected rescan Persist awaiting {awaiting}" + ); + self.send(proto::Materialize { + persisted: Some(proto::Persisted { + seq_no: persist.seq_no, + }), + ..Default::default() + }); + continue; + } + return msg; + } + } + + async fn expect_load(&mut self) { + let msg = self.expect("Load").await; + assert!(msg.load.is_some(), "expected Load, got {msg:?}"); + } + + async fn expect_acknowledge(&mut self) { + let msg = self.expect("Acknowledge").await; + assert!( + msg.acknowledge.is_some(), + "expected Acknowledge, got {msg:?}" + ); + } + + /// Assert the leader sends nothing further within `window`. + async fn assert_quiet(&mut self, window: Duration) { + match tokio::time::timeout(window, self.from_leader.message()).await { + Err(_) => (), + Ok(msg) => panic!("unexpected leader message: {msg:?}"), + } + } + + fn send_loaded(&self, sourced_docs: u64) { + self.send(proto::Materialize { + loaded: Some(proto::materialize::Loaded { + bindings: vec![proto::materialize::loaded::Binding { + index: 0, + min_source_clock: 0, + max_source_clock: 0, + sourced_docs_total: sourced_docs, + sourced_bytes_total: sourced_docs * 100, + max_key_delta: bytes::Bytes::new(), + }], + combiner_usage_bytes: 1 << 10, + }), + ..Default::default() + }); + } + + fn send_acknowledged(&self) { + self.send(proto::Materialize { + acknowledged: Some(proto::materialize::Acknowledged::default()), + ..Default::default() + }); + } + + /// Drive a closing transaction from Flush through StartedCommit, and park + /// at the leader's `Acknowledge` — the caller controls when to + /// `send_acknowledged`. Completing within the expect timeout is itself + /// the promptness assertion for collapsed holds and bypassed floors. + async fn run_close_script(&mut self) { + let msg = self.expect("Flush").await; + assert!(msg.flush.is_some(), "expected Flush, got {msg:?}"); + self.send(proto::Materialize { + flushed: Some(proto::materialize::Flushed::default()), + ..Default::default() + }); + + let msg = self.expect("Store").await; + assert!(msg.store.is_some(), "expected Store, got {msg:?}"); + self.send(proto::Materialize { + stored: Some(proto::materialize::Stored::default()), + ..Default::default() + }); + + let msg = self.expect("StartCommit").await; + assert!( + msg.start_commit.is_some(), + "expected StartCommit, got {msg:?}" + ); + self.send(proto::Materialize { + started_commit: Some(proto::materialize::StartedCommit::default()), + ..Default::default() + }); + + self.expect_acknowledge().await; + } +} + +// ---- SyncNow stream helpers ---- + +async fn recv_sync_now( + stream: &mut SyncNowStream, + awaiting: &'static str, + timeout: Duration, +) -> sync_now_response::Response { + tokio::time::timeout(timeout, stream.message()) + .await + .unwrap_or_else(|_| panic!("timed out awaiting SyncNow {awaiting}")) + .unwrap_or_else(|err| panic!("SyncNow stream error awaiting {awaiting}: {err}")) + .unwrap_or_else(|| panic!("unexpected SyncNow EOF awaiting {awaiting}")) + .response + .unwrap_or_else(|| panic!("SyncNow response with empty oneof awaiting {awaiting}")) +} + +async fn expect_ack(stream: &mut SyncNowStream) { + match recv_sync_now(stream, "Ack", EXPECT_TIMEOUT).await { + sync_now_response::Response::Ack(_) => (), + other => panic!("expected SyncNow Ack, got {other:?}"), + } +} + +/// Await Done, skipping any interleaved heartbeats. +async fn expect_done(stream: &mut SyncNowStream) { + loop { + match recv_sync_now(stream, "Done", EXPECT_TIMEOUT).await { + sync_now_response::Response::Done(_) => return, + sync_now_response::Response::Heartbeat(_) => continue, + other => panic!("expected SyncNow Done, got {other:?}"), + } + } +} + +async fn expect_heartbeat(stream: &mut SyncNowStream, timeout: Duration) { + match recv_sync_now(stream, "Heartbeat", timeout).await { + sync_now_response::Response::Heartbeat(_) => (), + other => panic!("expected SyncNow Heartbeat, got {other:?}"), + } +} + +/// Assert no response arrives on `stream` within `window` (bounded peek). +async fn assert_no_response(stream: &mut SyncNowStream, window: Duration) { + match tokio::time::timeout(window, stream.message()).await { + Err(_) => (), + Ok(result) => panic!("unexpected SyncNow response while ack withheld: {result:?}"), + } +} + +// ---- Scripted transaction helpers ---- + +/// Run one full transaction to completion, then drain via the IDLE barrier. +/// Used as the warm-up before every held-transaction test: the first +/// transaction of a session is never schedule-held (`session_start`), so the +/// NEXT transaction is the one a schedule genuinely holds. +async fn warm_up_txn(harness: &Harness, driver: &mut ShardDriver, task_name: &str) { + harness.push_frontier(); + driver.expect_load().await; + driver.send_loaded(DOCS_PER_TXN); + driver.run_close_script().await; + driver.send_acknowledged(); + harness.poke_until_drained(task_name).await; +} + +/// Open a transaction that the sync schedule holds, and deliver the SyncNow +/// that collapses it, returning the caller's stream. +/// +/// The hold is proven by the leader's silence: a held transaction asks the +/// shard for nothing, because absent our poke it would sit until the +/// schedule's next fire instant hours away. Wall-clock flake: the fire +/// instants sit on a fixed grid (`jitter(seed) + k * baseInterval`) which the +/// test cannot choose "now" against, so a transaction opening milliseconds +/// before an instant commits on its own and trips `assert_quiet` (~1-in-1e5 +/// odds with the 2h interval). +async fn open_held_txn(harness: &Harness, driver: &mut ShardDriver) -> SyncNowStream { + harness.push_frontier(); + driver.expect_load().await; + driver.send_loaded(DOCS_PER_TXN); + driver.assert_quiet(HELD_QUIET_WINDOW).await; + + harness.poke(SCHEDULED_TASK).await +} + +// ---- Tests ---- + +/// A schedule-held transaction collapses on SyncNow and resolves only once +/// the shard side acknowledges; a fully-drained task then resolves at once. +#[tokio::test(flavor = "multi_thread")] +async fn held_transaction_collapses_and_resolves_on_ack() { + let harness = Harness::start().await; + let mut driver = harness.join_session(SCHEDULED_TASK).await; + warm_up_txn(&harness, &mut driver, SCHEDULED_TASK).await; + + // Transaction #2 is genuinely held (a 2h hold, absent the poke). + let mut stream = open_held_txn(&harness, &mut driver).await; + + // The hold collapsed: the commit proceeds promptly rather than waiting + // out the 2h grid, and parks at Acknowledge. + driver.run_close_script().await; + + // Done must not arrive while the acknowledgement is withheld. + assert_no_response(&mut stream, Duration::from_secs(2)).await; + + driver.send_acknowledged(); + expect_done(&mut stream).await; + + // With everything drained and no new frontier, a fresh poke has nothing + // to await and its Done follows immediately. + let mut stream = harness.poke(SCHEDULED_TASK).await; + expect_done(&mut stream).await; + driver.assert_quiet(HELD_QUIET_WINDOW).await; + + drop(driver); + harness.stop().await; +} + +/// N concurrent SyncNow calls against the same open transaction all ack and +/// all resolve at the same single commit. +#[tokio::test(flavor = "multi_thread")] +async fn concurrent_pokes_share_one_commit() { + let harness = Harness::start().await; + let mut driver = harness.join_session(SCHEDULED_TASK).await; + warm_up_txn(&harness, &mut driver, SCHEDULED_TASK).await; + + // The first poke collapses the held transaction; three more land while + // the (single) commit is in flight, each acking wherever the actor is. + let first = open_held_txn(&harness, &mut driver).await; + let mut streams = vec![first]; + for _ in 0..3 { + streams.push(harness.poke(SCHEDULED_TASK).await); + } + + driver.run_close_script().await; + + // No stream resolves while the acknowledgement is withheld. + for stream in &mut streams { + assert_no_response(stream, Duration::from_millis(500)).await; + } + + driver.send_acknowledged(); + for stream in &mut streams { + expect_done(stream).await; + } + + // One transaction total: the leader is idle and asks nothing more of the + // shard once the shared commit drains. + harness.poke_until_drained(SCHEDULED_TASK).await; + driver.assert_quiet(Duration::from_millis(500)).await; + + drop(driver); + harness.stop().await; +} + +/// A poke arriving after commit, while the connector acknowledgement still +/// drains, resolves at that same acknowledgement release rather than awaiting +/// a further transaction. +#[tokio::test(flavor = "multi_thread")] +async fn poke_in_drain_window_awaits_the_same_ack() { + let harness = Harness::start().await; + let mut driver = harness.join_session(SCHEDULED_TASK).await; + warm_up_txn(&harness, &mut driver, SCHEDULED_TASK).await; + + let mut collapse_stream = open_held_txn(&harness, &mut driver).await; + + // Drive through StartedCommit and the commit Persist: the transaction has + // committed and its Tail drains the (withheld) acknowledgement. + driver.run_close_script().await; + + let mut drain_stream = harness.poke(SCHEDULED_TASK).await; + assert_no_response(&mut drain_stream, Duration::from_millis(500)).await; + + driver.send_acknowledged(); + expect_done(&mut collapse_stream).await; + expect_done(&mut drain_stream).await; + + drop(driver); + harness.stop().await; +} + +/// A transaction held by the 300s min-duration floor (no schedule) commits +/// immediately on SyncNow — the close request bypasses the floor. The floor +/// holds even the session's first transaction (`session_start` waives only +/// schedule holds), so no warm-up is needed. +#[tokio::test(flavor = "multi_thread")] +async fn sync_now_bypasses_the_min_duration_floor() { + let harness = Harness::start().await; + let mut driver = harness.join_session(FLOOR_TASK).await; + + harness.push_frontier(); + driver.expect_load().await; + driver.send_loaded(DOCS_PER_TXN); + driver.assert_quiet(HELD_QUIET_WINDOW).await; + + let mut stream = harness.poke(FLOOR_TASK).await; + + // The commit proceeds well before the 300s floor, and Done follows the + // released acknowledgement. + driver.run_close_script().await; + driver.send_acknowledged(); + expect_done(&mut stream).await; + + drop(driver); + harness.stop().await; +} + +/// A parked waiter receives heartbeats. The cadence is the production 15s +/// `SYNC_NOW_HEARTBEAT` constant, kept un-injectable on purpose, so this is +/// deliberately the one slow test: the first beat lands one full interval +/// after the waiter parks. +#[tokio::test(flavor = "multi_thread")] +async fn parked_waiter_receives_heartbeats() { + let harness = Harness::start().await; + let mut driver = harness.join_session(FLOOR_TASK).await; + + harness.push_frontier(); + driver.expect_load().await; + driver.send_loaded(DOCS_PER_TXN); + + let mut stream = harness.poke(FLOOR_TASK).await; + + // Commit, then park with the acknowledgement withheld. + driver.run_close_script().await; + + // One beat proves the machinery; don't wait for a second. + expect_heartbeat(&mut stream, Duration::from_secs(30)).await; + + driver.send_acknowledged(); + expect_done(&mut stream).await; + + drop(driver); + harness.stop().await; +} + +/// Token scope and capability checks over the real wire, and NOT_FOUND +/// resolution from the caller's shard-scope when no live session matches. +#[tokio::test(flavor = "multi_thread")] +async fn authz_and_task_resolution() { + let harness = Harness::start().await; + let driver = harness.join_session(SCHEDULED_TASK).await; + + // A correctly-scoped READ token is accepted, and with no open transaction + // resolves immediately. + harness.poke_until_drained(SCHEDULED_TASK).await; + + // A READ token scoped to a DIFFERENT task is denied against the live + // session's concrete shard-zero ID. + let err = harness + .sync_now( + harness.read_token("materialize/testing/other/0011223344556677/"), + SCHEDULED_TASK, + ) + .await + .expect_err("differently-scoped token is rejected"); + assert_eq!(err.code(), tonic::Code::PermissionDenied, "{err}"); + + // A token minted WITHOUT the READ capability bit fails at the + // interceptor, before any Ack. + let err = harness + .sync_now( + harness.token( + proto_flow::capability::LEAD, + &format!("materialize/{SCHEDULED_TASK}/"), + ), + SCHEDULED_TASK, + ) + .await + .expect_err("token without READ is rejected"); + assert!( + matches!( + err.code(), + tonic::Code::PermissionDenied | tonic::Code::Unauthenticated + ), + "{err}", + ); + + // A materialization the token names, but which isn't running here, is + // NOT_FOUND. + let err = harness + .sync_now( + harness.read_token("materialize/testing/not-running/"), + "testing/not-running", + ) + .await + .expect_err("not-running task is NOT_FOUND"); + assert_eq!(err.code(), tonic::Code::NotFound, "{err}"); + + // A capture or derivation is NOT_FOUND here too: this service hosts only + // Materialize leader sessions. A task with nothing to sync is resolved by + // the reactor front door, which has the shard keyspace to know its type. + let err = harness + .sync_now( + harness.read_token("capture/testing/some-capture/"), + "testing/some-capture", + ) + .await + .expect_err("a capture has no leader session here"); + assert_eq!(err.code(), tonic::Code::NotFound, "{err}"); + + // An empty task_name is an invalid argument. + let err = harness + .sync_now(harness.read_token("materialize/testing/"), "") + .await + .expect_err("empty task_name is rejected"); + assert_eq!(err.code(), tonic::Code::InvalidArgument, "{err}"); + + drop(driver); + harness.stop().await; +} + +/// A session that exits with waiters parked errors them out (they can't +/// resolve: the Tail::Done they await happens, if ever, in a future session), +/// and the SyncNowGuard un-registers the handle so follow-up pokes are +/// NOT_FOUND. +#[tokio::test(flavor = "multi_thread")] +async fn session_exit_errors_parked_waiters() { + let harness = Harness::start().await; + let mut driver = harness.join_session(FLOOR_TASK).await; + + // Park a waiter: a floor-held open transaction whose close script the + // driver never answers. + harness.push_frontier(); + driver.expect_load().await; + driver.send_loaded(DOCS_PER_TXN); + + let mut stream = harness.poke(FLOOR_TASK).await; + + // Kill the session: the leader errors on the shard stream's EOF. + drop(driver); + + let err = tokio::time::timeout(EXPECT_TIMEOUT, stream.message()) + .await + .expect("waiter resolves after session exit") + .expect_err("waiter errors after session exit"); + assert_eq!(err.code(), tonic::Code::Unavailable, "{err}"); + assert!(err.message().contains("retry"), "{err}"); + + // The guard un-registered the handle; a follow-up poke is NOT_FOUND. + // Guard drop races the client's next call, so retry briefly. + let deadline = tokio::time::Instant::now() + EXPECT_TIMEOUT; + loop { + match harness + .sync_now( + harness.read_token(&format!("materialize/{FLOOR_TASK}/")), + FLOOR_TASK, + ) + .await + { + Err(status) if status.code() == tonic::Code::NotFound => break, + other => { + assert!( + tokio::time::Instant::now() < deadline, + "timed out awaiting NOT_FOUND after session exit (last: {other:?})", + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + } + + harness.stop().await; +} diff --git a/crates/runtime-sidecar/README.md b/crates/runtime-sidecar/README.md index dcd1dd8e7db..81e461fa99b 100644 --- a/crates/runtime-sidecar/README.md +++ b/crates/runtime-sidecar/README.md @@ -2,11 +2,13 @@ Production sidecar process for the runtime-v2 architecture (`plans/runtime-v2/plan.md`). One per reactor machine, supervised by -systemd, hosting two gRPC services on a fixed fleet-wide port: +systemd, hosting three gRPC services on a fixed fleet-wide port: - **Shuffle Leader** — `runtime_next::leader::Service`, the per-task Join rendezvous and HeadFSM/TailFSM coordination for tasks whose shard zero is on this machine. +- **TaskControl** — the same `leader::Service`, wrapped separately: the + user-facing SyncNow RPC against tasks led from this machine. - **Shuffle** — `shuffle::Service`, the Session/Slice/Log RPCs. ## Listeners @@ -22,12 +24,14 @@ signs outbound requests; all keys verify inbound traffic (rotation). Inbound RPCs are authenticated and authorized: every request must carry a JWT issued by `--data-plane-fqdn` and bearing the service's -capability — `LEAD` for the Leader, `SHUFFLE` for Shuffle — or it is -rejected before its handler runs. Each handler additionally enforces a -scope check: the token's selector must authorize the shard the handler -operates on (shard zero of a Leader/Session join, or the hosted shard of -a Slice/Log). Because a task's shards share an `id` prefix and the bearer -is scoped to that prefix, this gates access at task granularity. +capability — `LEAD` for the Leader, gazette `READ` for TaskControl +(the token every `/authorize/user/task` caller holds), `SHUFFLE` for +Shuffle — or it is rejected before its handler runs. Each handler +additionally enforces a scope check: the token's selector must authorize +the shard the handler operates on (shard zero of a Leader/Session join +or a TaskControl SyncNow, or the hosted shard of a Slice/Log). Because a +task's shards share an `id` prefix and the bearer is scoped to that +prefix, this gates access at task granularity. The loopback admin surface is a separate server and is intentionally not authenticated (bound only to loopback). diff --git a/crates/runtime-sidecar/src/lib.rs b/crates/runtime-sidecar/src/lib.rs index 5d2f4d021e4..dc178dd0e89 100644 --- a/crates/runtime-sidecar/src/lib.rs +++ b/crates/runtime-sidecar/src/lib.rs @@ -186,9 +186,15 @@ pub async fn run(args: Args, registry: service_kit::Registry) -> anyhow::Result< } builder .add_service(tonic::service::interceptor::InterceptedService::new( - runtime_svc.into_tonic_service(), + runtime_svc.clone().into_tonic_service(), authn.clone().interceptor(proto_flow::capability::LEAD), )) + // TaskControl callers hold ordinary gazette READ claims over the + // task's shards (the /authorize/user/task token), not LEAD. + .add_service(tonic::service::interceptor::InterceptedService::new( + runtime_svc.into_task_control_service(), + authn.clone().interceptor(proto_gazette::capability::READ), + )) .add_service(tonic::service::interceptor::InterceptedService::new( shuffle_svc.into_tonic_service(), authn.interceptor(proto_flow::capability::SHUFFLE), diff --git a/crates/validation/src/errors.rs b/crates/validation/src/errors.rs index 63fa9b2198f..d9ac9a63614 100644 --- a/crates/validation/src/errors.rs +++ b/crates/validation/src/errors.rs @@ -395,10 +395,6 @@ pub enum Error { #[error("invalid sync schedule: {detail}")] SyncScheduleInvalid { detail: String }, - #[error( - "materialization {materialization} configures a sync schedule both in its connector config (syncSchedule) and as a materialization sync schedule; remove one" - )] - SyncScheduleConflict { materialization: String }, #[error("raising an error because {this_entity} specifies `onIncompatibleSchemaChange: abort`")] AbortOnIncompatibleSchemaChange { diff --git a/crates/validation/src/materialization.rs b/crates/validation/src/materialization.rs index 50173ae77c5..2f36d4395fd 100644 --- a/crates/validation/src/materialization.rs +++ b/crates/validation/src/materialization.rs @@ -700,16 +700,6 @@ async fn walk_materialization( if let Err(detail) = sync_schedule.validate() { Error::SyncScheduleInvalid { detail }.push(scope, errors); } - // A connector-side sync schedule and a model-level one would fight - // over commit cadence; reject configuring both. - if let models::MaterializationEndpoint::Connector(config) = &endpoint - && connector_config_has_sync_schedule(&config.config) - { - Error::SyncScheduleConflict { - materialization: materialization.to_string(), - } - .push(scope, errors); - } bytes::Bytes::from( serde_json::to_vec(sync_schedule).expect("sync schedule must serialize"), ) @@ -1165,28 +1155,6 @@ fn temporary_group_by_migration( .collect() } -/// Whether a connector endpoint config carries a configured top-level -/// `syncSchedule`, i.e. the legacy connector-side sync schedule. Detected -/// without decryption: SOPS preserves object keys, and sync-schedule fields -/// are not secrets so their values remain plaintext. -/// -/// An empty `syncSchedule` object, or one whose values are all null or empty -/// strings, is NOT configured: the connector treats those identically to an -/// absent key, and a UI removing the schedule may leave such a remnant behind. -fn connector_config_has_sync_schedule(config: &models::RawValue) -> bool { - let Ok(serde_json::Value::Object(map)) = serde_json::from_str(config.get()) else { - return false; - }; - match map.get("syncSchedule") { - None | Some(serde_json::Value::Null) => false, - Some(serde_json::Value::Object(sched)) => sched - .values() - .any(|value| !value.is_null() && value.as_str() != Some("")), - // Any other shape is malformed, but conservatively "configured". - Some(_) => true, - } -} - fn validate_triggers( scope: Scope, triggers: &[models::TriggerConfig], diff --git a/crates/validation/tests/scenario_tests.rs b/crates/validation/tests/scenario_tests.rs index 0d0d9aa8dfb..05107ea2839 100644 --- a/crates/validation/tests/scenario_tests.rs +++ b/crates/validation/tests/scenario_tests.rs @@ -43,72 +43,6 @@ fn connector_validation_is_skipped_when_shards_are_disabled() { insta::assert_debug_snapshot!(outcome); } -#[test] -fn sync_schedule_configured_in_both_places_is_rejected() { - // A model-level sync schedule alongside a connector-config `syncSchedule` - // is an error: the two would fight over commit cadence. - let fixture = r##" -test://example/catalog.yaml: - materializations: - testing/materialization: - endpoint: - connector: - image: an/image - config: - syncSchedule: { syncFrequency: 30m } - syncSchedule: { baseInterval: 15m } - shards: { disable: true } - bindings: [] - -driver: - dataPlanes: - "1d:1d:1d:1d:1d:1d:1d:1d": {} -"##; - let outcome = common::run(fixture, "{}"); - insta::assert_debug_snapshot!(outcome); -} - -#[test] -fn sync_schedule_conflict_ignores_a_cleared_connector_schedule() { - // A UI that removes a connector-side sync schedule may leave behind an - // empty `syncSchedule: {}`, or one whose values are cleared to empty or - // null. The connector treats those identically to an absent key, so they - // must not conflict with a model-level schedule. - let fixture = r##" -test://example/catalog.yaml: - materializations: - testing/empty-object: - endpoint: - connector: - image: an/image - config: - syncSchedule: {} - syncSchedule: { baseInterval: 15m } - shards: { disable: true } - bindings: [] - testing/cleared-values: - endpoint: - connector: - image: an/image - config: - syncSchedule: { syncFrequency: "", timezone: null } - syncSchedule: { baseInterval: 15m } - shards: { disable: true } - bindings: [] - -driver: - dataPlanes: - "1d:1d:1d:1d:1d:1d:1d:1d": {} -"##; - let outcome = common::run(fixture, "{}"); - assert!( - outcome.errors.is_empty() && outcome.errors_draft.is_empty(), - "expected no errors, got: {:?} {:?}", - outcome.errors, - outcome.errors_draft, - ); -} - #[test] fn test_collection_schema_contains_flow_document() { let fixture = r##" diff --git a/crates/validation/tests/snapshots/scenario_tests__sync_schedule_configured_in_both_places_is_rejected.snap b/crates/validation/tests/snapshots/scenario_tests__sync_schedule_configured_in_both_places_is_rejected.snap deleted file mode 100644 index 20a63036b11..00000000000 --- a/crates/validation/tests/snapshots/scenario_tests__sync_schedule_configured_in_both_places_is_rejected.snap +++ /dev/null @@ -1,221 +0,0 @@ ---- -source: crates/validation/tests/scenario_tests.rs -expression: outcome ---- -Outcome { - built_captures: [], - built_collections: [], - built_materializations: [ - BuiltMaterialization { - materialization: testing/materialization, - scope: test://example/catalog.yaml#/materializations/testing~1materialization, - control_id: "0000000000000000", - data_plane_id: "1d1d1d1d1d1d1d1d", - expect_pub_id: "0000000000000000", - expect_build_id: "0000000000000000", - model: { - "endpoint": { - "connector": { - "image": "an/image", - "config": {"syncSchedule":{"syncFrequency":"30m"}} - } - }, - "bindings": [], - "shards": { - "disable": true - }, - "syncSchedule": { - "baseInterval": "15m" - } - }, - model_fixes: [], - validated: Validated { - bindings: [], - }, - spec: MaterializationSpec { - name: "testing/materialization", - connector_type: Image, - config_json: b"{\"image\":\"an/image\",\"config\":{\"syncSchedule\":{\"syncFrequency\":\"30m\"}}}", - bindings: [], - shard_template: Some( - ShardSpec { - id: "materialize/testing/materialization/2020202020202020", - sources: [], - recovery_log_prefix: "recovery", - hint_prefix: "/estuary/flow/hints", - hint_backups: 2, - max_txn_duration: Some( - Duration { - seconds: 1200, - nanos: 0, - }, - ), - min_txn_duration: Some( - Duration { - seconds: 0, - nanos: 0, - }, - ), - disable: true, - hot_standbys: 0, - labels: Some( - LabelSet { - labels: [ - Label { - name: "app.gazette.dev/managed-by", - value: "estuary.dev/flow", - prefix: false, - }, - Label { - name: "estuary.dev/build", - value: "2121212121212121", - prefix: false, - }, - Label { - name: "estuary.dev/log-level", - value: "info", - prefix: false, - }, - Label { - name: "estuary.dev/task-name", - value: "testing/materialization", - prefix: false, - }, - Label { - name: "estuary.dev/task-type", - value: "materialization", - prefix: false, - }, - ], - }, - ), - disable_wait_for_ack: false, - ring_buffer_size: 65536, - read_channel_size: 4096, - }, - ), - recovery_log_template: Some( - JournalSpec { - name: "recovery/materialize/testing/materialization/2020202020202020", - replication: 3, - labels: Some( - LabelSet { - labels: [ - Label { - name: "app.gazette.dev/managed-by", - value: "estuary.dev/flow", - prefix: false, - }, - Label { - name: "content-type", - value: "application/x-gazette-recoverylog", - prefix: false, - }, - Label { - name: "estuary.dev/build", - value: "2121212121212121", - prefix: false, - }, - Label { - name: "estuary.dev/task-name", - value: "testing/materialization", - prefix: false, - }, - Label { - name: "estuary.dev/task-type", - value: "materialization", - prefix: false, - }, - ], - }, - ), - fragment: Some( - Fragment { - length: 268435456, - compression_codec: Snappy, - stores: [ - "s3://a-bucket/", - ], - refresh_interval: Some( - Duration { - seconds: 300, - nanos: 0, - }, - ), - retention: None, - flush_interval: Some( - Duration { - seconds: 172800, - nanos: 0, - }, - ), - path_postfix_template: "", - }, - ), - flags: 4, - max_append_rate: 4194304, - suspend: None, - }, - ), - network_ports: [], - inactive_bindings: [], - triggers_json: b"", - created_at: "", - sync_schedule_json: b"{\"baseInterval\":\"15m\"}", - linked_collections: [], - }, - previous_spec: NULL, - is_touch: 0, - dependency_hash: NULL, - }, - ], - built_tests: [], - captures: [], - collections: [], - errors: [ - Error { - scope: test://example/catalog.yaml#/materializations/testing~1materialization/syncSchedule, - error: materialization testing/materialization configures a sync schedule both in its connector config (syncSchedule) and as a materialization sync schedule; remove one, - }, - ], - errors_draft: [], - fetches: [ - Fetch { - depth: 1, - resource: test://example/catalog.yaml, - }, - ], - imports: [], - materializations: [ - DraftMaterialization { - materialization: testing/materialization, - scope: test://example/catalog.yaml#/materializations/testing~1materialization, - expect_pub_id: NULL, - model: { - "endpoint": { - "connector": { - "image": "an/image", - "config": {"syncSchedule":{"syncFrequency":"30m"}} - } - }, - "bindings": [], - "shards": { - "disable": true - }, - "syncSchedule": { - "baseInterval": "15m" - } - }, - is_touch: 0, - }, - ], - resources: [ - Resource { - resource: test://example/catalog.yaml, - content_type: "CATALOG", - content: ".. binary ..", - content_dom: {"materializations":{"testing/materialization":{"bindings":[],"endpoint":{"connector":{"config":{"syncSchedule":{"syncFrequency":"30m"}},"image":"an/image"}},"shards":{"disable":true},"syncSchedule":{"baseInterval":"15m"}}}}, - }, - ], - tests: [], -} diff --git a/go.mod b/go.mod index 10519c03e35..b802c00dded 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,11 @@ require ( github.com/bradleyjkemp/cupaloy v2.3.0+incompatible github.com/evanphx/json-patch/v5 v5.9.11 github.com/fatih/color v1.18.0 + github.com/gogo/gateway v1.1.0 github.com/gogo/protobuf v1.3.2 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jessevdk/go-flags v1.6.1 github.com/jgraettinger/gorocksdb v0.0.0-20250815051509-2d5c1f160b80 @@ -64,7 +66,6 @@ require ( github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/gogo/gateway v1.1.0 // indirect github.com/golang/glog v1.2.5 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v1.0.0 // indirect @@ -73,7 +74,6 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/gorilla/schema v1.4.1 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/jgraettinger/cockroach-encoding v1.1.0 // indirect diff --git a/go/protocols/runtime/runtime.pb.go b/go/protocols/runtime/runtime.pb.go index 388cfa9752b..17983f2899d 100644 --- a/go/protocols/runtime/runtime.pb.go +++ b/go/protocols/runtime/runtime.pb.go @@ -3811,6 +3811,275 @@ func (m *Derive_StartedCommit) XXX_DiscardUnknown() { var xxx_messageInfo_Derive_StartedCommit proto.InternalMessageInfo +// SyncNowRequest is the request of a TaskControl.SyncNow RPC. +type SyncNowRequest struct { + // Name of the task to synchronize. + TaskName string `protobuf:"bytes,1,opt,name=task_name,json=taskName,proto3" json:"task_name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SyncNowRequest) Reset() { *m = SyncNowRequest{} } +func (m *SyncNowRequest) String() string { return proto.CompactTextString(m) } +func (*SyncNowRequest) ProtoMessage() {} +func (*SyncNowRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{32} +} +func (m *SyncNowRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SyncNowRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SyncNowRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SyncNowRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncNowRequest.Merge(m, src) +} +func (m *SyncNowRequest) XXX_Size() int { + return m.ProtoSize() +} +func (m *SyncNowRequest) XXX_DiscardUnknown() { + xxx_messageInfo_SyncNowRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncNowRequest proto.InternalMessageInfo + +// SyncNowResponse is the streamed response of a TaskControl.SyncNow RPC. +// Its messages are structural: they mark the caller's position in the +// stream and carry no payload. Statistics of the awaited transaction are +// recorded to the task's stats journal, which is where callers read them. +type SyncNowResponse struct { + // Types that are valid to be assigned to Response: + // *SyncNowResponse_Ack_ + // *SyncNowResponse_Heartbeat_ + // *SyncNowResponse_Done_ + Response isSyncNowResponse_Response `protobuf_oneof:"response"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SyncNowResponse) Reset() { *m = SyncNowResponse{} } +func (m *SyncNowResponse) String() string { return proto.CompactTextString(m) } +func (*SyncNowResponse) ProtoMessage() {} +func (*SyncNowResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{33} +} +func (m *SyncNowResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SyncNowResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SyncNowResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SyncNowResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncNowResponse.Merge(m, src) +} +func (m *SyncNowResponse) XXX_Size() int { + return m.ProtoSize() +} +func (m *SyncNowResponse) XXX_DiscardUnknown() { + xxx_messageInfo_SyncNowResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncNowResponse proto.InternalMessageInfo + +type isSyncNowResponse_Response interface { + isSyncNowResponse_Response() + MarshalTo([]byte) (int, error) + ProtoSize() int +} + +type SyncNowResponse_Ack_ struct { + Ack *SyncNowResponse_Ack `protobuf:"bytes,1,opt,name=ack,proto3,oneof" json:"ack,omitempty"` +} +type SyncNowResponse_Heartbeat_ struct { + Heartbeat *SyncNowResponse_Heartbeat `protobuf:"bytes,2,opt,name=heartbeat,proto3,oneof" json:"heartbeat,omitempty"` +} +type SyncNowResponse_Done_ struct { + Done *SyncNowResponse_Done `protobuf:"bytes,3,opt,name=done,proto3,oneof" json:"done,omitempty"` +} + +func (*SyncNowResponse_Ack_) isSyncNowResponse_Response() {} +func (*SyncNowResponse_Heartbeat_) isSyncNowResponse_Response() {} +func (*SyncNowResponse_Done_) isSyncNowResponse_Response() {} + +func (m *SyncNowResponse) GetResponse() isSyncNowResponse_Response { + if m != nil { + return m.Response + } + return nil +} + +func (m *SyncNowResponse) GetAck() *SyncNowResponse_Ack { + if x, ok := m.GetResponse().(*SyncNowResponse_Ack_); ok { + return x.Ack + } + return nil +} + +func (m *SyncNowResponse) GetHeartbeat() *SyncNowResponse_Heartbeat { + if x, ok := m.GetResponse().(*SyncNowResponse_Heartbeat_); ok { + return x.Heartbeat + } + return nil +} + +func (m *SyncNowResponse) GetDone() *SyncNowResponse_Done { + if x, ok := m.GetResponse().(*SyncNowResponse_Done_); ok { + return x.Done + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*SyncNowResponse) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*SyncNowResponse_Ack_)(nil), + (*SyncNowResponse_Heartbeat_)(nil), + (*SyncNowResponse_Done_)(nil), + } +} + +// Ack is sent exactly once, as the first message of the stream: the +// request reached the task's leader and the commit has been forced. +type SyncNowResponse_Ack struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SyncNowResponse_Ack) Reset() { *m = SyncNowResponse_Ack{} } +func (m *SyncNowResponse_Ack) String() string { return proto.CompactTextString(m) } +func (*SyncNowResponse_Ack) ProtoMessage() {} +func (*SyncNowResponse_Ack) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{33, 0} +} +func (m *SyncNowResponse_Ack) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SyncNowResponse_Ack) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SyncNowResponse_Ack.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SyncNowResponse_Ack) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncNowResponse_Ack.Merge(m, src) +} +func (m *SyncNowResponse_Ack) XXX_Size() int { + return m.ProtoSize() +} +func (m *SyncNowResponse_Ack) XXX_DiscardUnknown() { + xxx_messageInfo_SyncNowResponse_Ack.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncNowResponse_Ack proto.InternalMessageInfo + +// Heartbeat keeps a long-lived stream alive while the caller waits. +type SyncNowResponse_Heartbeat struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SyncNowResponse_Heartbeat) Reset() { *m = SyncNowResponse_Heartbeat{} } +func (m *SyncNowResponse_Heartbeat) String() string { return proto.CompactTextString(m) } +func (*SyncNowResponse_Heartbeat) ProtoMessage() {} +func (*SyncNowResponse_Heartbeat) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{33, 1} +} +func (m *SyncNowResponse_Heartbeat) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SyncNowResponse_Heartbeat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SyncNowResponse_Heartbeat.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SyncNowResponse_Heartbeat) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncNowResponse_Heartbeat.Merge(m, src) +} +func (m *SyncNowResponse_Heartbeat) XXX_Size() int { + return m.ProtoSize() +} +func (m *SyncNowResponse_Heartbeat) XXX_DiscardUnknown() { + xxx_messageInfo_SyncNowResponse_Heartbeat.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncNowResponse_Heartbeat proto.InternalMessageInfo + +// Done is sent exactly once, as the final message of the stream: the +// awaited transaction is committed and queryable in the endpoint. +type SyncNowResponse_Done struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SyncNowResponse_Done) Reset() { *m = SyncNowResponse_Done{} } +func (m *SyncNowResponse_Done) String() string { return proto.CompactTextString(m) } +func (*SyncNowResponse_Done) ProtoMessage() {} +func (*SyncNowResponse_Done) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{33, 2} +} +func (m *SyncNowResponse_Done) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SyncNowResponse_Done) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SyncNowResponse_Done.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SyncNowResponse_Done) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncNowResponse_Done.Merge(m, src) +} +func (m *SyncNowResponse_Done) XXX_Size() int { + return m.ProtoSize() +} +func (m *SyncNowResponse_Done) XXX_DiscardUnknown() { + xxx_messageInfo_SyncNowResponse_Done.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncNowResponse_Done proto.InternalMessageInfo + func init() { proto.RegisterEnum("runtime.Plane", Plane_name, Plane_value) proto.RegisterEnum("runtime.CaptureResponseExt_PollResult", CaptureResponseExt_PollResult_name, CaptureResponseExt_PollResult_value) @@ -3894,6 +4163,11 @@ func init() { proto.RegisterType((*Derive_Stored_PublisherCommit)(nil), "runtime.Derive.Stored.PublisherCommit") proto.RegisterType((*Derive_StartCommit)(nil), "runtime.Derive.StartCommit") proto.RegisterType((*Derive_StartedCommit)(nil), "runtime.Derive.StartedCommit") + proto.RegisterType((*SyncNowRequest)(nil), "runtime.SyncNowRequest") + proto.RegisterType((*SyncNowResponse)(nil), "runtime.SyncNowResponse") + proto.RegisterType((*SyncNowResponse_Ack)(nil), "runtime.SyncNowResponse.Ack") + proto.RegisterType((*SyncNowResponse_Heartbeat)(nil), "runtime.SyncNowResponse.Heartbeat") + proto.RegisterType((*SyncNowResponse_Done)(nil), "runtime.SyncNowResponse.Done") } func init() { @@ -3901,289 +4175,298 @@ func init() { } var fileDescriptor_73af6e0737ce390c = []byte{ - // 4509 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7b, 0x4d, 0x6c, 0x1c, 0x47, - 0x76, 0xb0, 0xe6, 0x7f, 0xe6, 0xcd, 0x90, 0x9c, 0x29, 0x91, 0x54, 0xab, 0x65, 0x8b, 0xf4, 0xd8, - 0xfe, 0x4c, 0x8b, 0xd4, 0x90, 0xa6, 0xe5, 0x5d, 0x5b, 0x9f, 0xff, 0xf8, 0xa7, 0x15, 0xb5, 0x94, - 0x44, 0x17, 0x29, 0x21, 0xd9, 0x4b, 0xa3, 0xd9, 0x5d, 0x1c, 0xb6, 0xd8, 0xd3, 0xd5, 0xee, 0xee, - 0xa1, 0xc4, 0x3d, 0x05, 0xc8, 0x25, 0x40, 0x80, 0xe4, 0xb2, 0x97, 0x20, 0x97, 0xdc, 0x82, 0x04, - 0xc8, 0x21, 0xa7, 0x05, 0xf6, 0x90, 0x53, 0x0e, 0x46, 0x4e, 0x49, 0x0e, 0x41, 0x4e, 0x0a, 0xb2, - 0xb9, 0x26, 0xa7, 0x4d, 0x80, 0x44, 0xc8, 0x21, 0xa8, 0x9f, 0xfe, 0x9d, 0x1e, 0x8a, 0xa4, 0x8d, - 0xc4, 0x58, 0xec, 0x41, 0x62, 0xd7, 0xfb, 0xab, 0x57, 0x55, 0xef, 0xbd, 0x7a, 0xef, 0x75, 0x0f, - 0x74, 0xfb, 0x74, 0xd9, 0xf5, 0x68, 0x40, 0x0d, 0x6a, 0xfb, 0xcb, 0xde, 0xd0, 0x09, 0xac, 0x01, - 0x09, 0xff, 0xf6, 0x38, 0x06, 0xd5, 0xe4, 0x50, 0xbd, 0x79, 0xe0, 0xd1, 0x63, 0xe2, 0x45, 0x0c, - 0xd1, 0x83, 0x20, 0x54, 0xe7, 0x0d, 0xea, 0xf8, 0xc3, 0xc1, 0x19, 0x14, 0xe9, 0xe9, 0x0c, 0xdd, - 0x0d, 0x86, 0x1e, 0x09, 0xff, 0x86, 0x52, 0x52, 0x34, 0x26, 0xf1, 0xac, 0x13, 0x22, 0xff, 0x48, - 0x8a, 0x37, 0x52, 0x14, 0x87, 0x36, 0x7d, 0xce, 0xff, 0x93, 0xd8, 0x5b, 0x29, 0xec, 0x40, 0x0f, - 0x88, 0x67, 0xe9, 0xb6, 0xf5, 0x53, 0x92, 0x7c, 0x96, 0xb4, 0x6a, 0x8a, 0x96, 0xba, 0xfc, 0x5f, - 0xae, 0xae, 0xfe, 0xd1, 0xf0, 0xf0, 0xd0, 0x26, 0xe1, 0x5f, 0x49, 0x33, 0xdd, 0xa7, 0x7d, 0xca, - 0x1f, 0x97, 0xd9, 0x93, 0x80, 0x76, 0xff, 0xaa, 0x00, 0x9d, 0x7d, 0xdd, 0x3f, 0xde, 0x23, 0xde, - 0x89, 0x65, 0x90, 0x0d, 0xea, 0x1c, 0x5a, 0x7d, 0x74, 0x13, 0x9a, 0x36, 0xed, 0x6b, 0x87, 0x96, - 0x4d, 0xb4, 0x43, 0x53, 0x29, 0xcc, 0x17, 0x16, 0x2a, 0xb8, 0x61, 0xd3, 0xfe, 0x3d, 0xcb, 0x26, - 0xf7, 0x4c, 0x74, 0x03, 0x1a, 0x81, 0xee, 0x1f, 0x6b, 0x8e, 0x3e, 0x20, 0x4a, 0x71, 0xbe, 0xb0, - 0xd0, 0xc0, 0x75, 0x06, 0x78, 0xa4, 0x0f, 0x08, 0xba, 0x0e, 0xf5, 0xa1, 0xe9, 0x6b, 0xae, 0x1e, - 0x1c, 0x29, 0x25, 0x8e, 0xab, 0x0d, 0x4d, 0x7f, 0x57, 0x0f, 0x8e, 0xd0, 0x22, 0x74, 0x0c, 0xea, - 0x04, 0xba, 0xe5, 0x10, 0x4f, 0x73, 0x48, 0xf0, 0x9c, 0x7a, 0xc7, 0x4a, 0x99, 0xd3, 0xb4, 0x23, - 0xc4, 0x23, 0x01, 0x47, 0xef, 0x40, 0xc5, 0xb5, 0x75, 0x87, 0x28, 0xd5, 0xf9, 0xc2, 0xc2, 0xe4, - 0xea, 0x64, 0x2f, 0x3c, 0xea, 0x5d, 0x06, 0xc5, 0x02, 0xd9, 0xfd, 0xaf, 0x32, 0x4c, 0xee, 0x89, - 0x85, 0x62, 0xf2, 0xf5, 0x90, 0xf8, 0x01, 0xda, 0x86, 0xda, 0x33, 0x3a, 0xf4, 0x1c, 0xdd, 0xe6, - 0x9a, 0x37, 0xd6, 0x97, 0x5f, 0xbd, 0x9c, 0x5b, 0xec, 0xd3, 0x5e, 0x5f, 0xff, 0x29, 0x09, 0x02, - 0xd2, 0x33, 0xc9, 0xc9, 0xb2, 0x41, 0x3d, 0xb2, 0x9c, 0x31, 0x92, 0xde, 0x03, 0xc1, 0x86, 0x43, - 0x7e, 0x34, 0x0b, 0x55, 0x8f, 0xb8, 0xb6, 0x7e, 0xca, 0x57, 0x59, 0xc7, 0x72, 0xc4, 0xd6, 0x78, - 0x30, 0xb4, 0x6c, 0x53, 0xb3, 0xcc, 0x70, 0x8d, 0x7c, 0xbc, 0x6d, 0xa2, 0x7b, 0x50, 0xa5, 0x87, - 0x87, 0x3e, 0x09, 0xf8, 0xc2, 0x4a, 0xeb, 0xbd, 0x57, 0x2f, 0xe7, 0x6e, 0x9d, 0x67, 0xf2, 0xc7, - 0x9c, 0x0b, 0x4b, 0x6e, 0xf4, 0x10, 0x80, 0x38, 0xa6, 0x26, 0x65, 0x55, 0x2e, 0x25, 0xab, 0x41, - 0x1c, 0x53, 0x3c, 0xa2, 0x45, 0xa8, 0x78, 0xba, 0xd3, 0x17, 0xbb, 0xd9, 0x5c, 0x9d, 0xea, 0x71, - 0x33, 0xc4, 0x0c, 0xb4, 0xe7, 0x12, 0x63, 0xbd, 0xfc, 0xcd, 0xcb, 0xb9, 0x2b, 0x58, 0xd0, 0xa0, - 0x3d, 0x68, 0x1a, 0x94, 0x7a, 0xa6, 0xe5, 0xe8, 0x01, 0xf5, 0x94, 0x1a, 0xdf, 0xc5, 0x0f, 0x5e, - 0xbd, 0x9c, 0xbb, 0x9d, 0x37, 0xf9, 0x88, 0x2b, 0xf5, 0xf6, 0x8e, 0x74, 0xcf, 0xdc, 0xde, 0xc4, - 0x49, 0x29, 0x68, 0x05, 0xc0, 0x23, 0x3e, 0xb5, 0x87, 0x81, 0x45, 0x1d, 0xa5, 0xce, 0xd5, 0x68, - 0xf7, 0x22, 0x9e, 0xfb, 0x44, 0x37, 0x89, 0x87, 0x13, 0x34, 0xe8, 0x6d, 0x98, 0x90, 0x36, 0xac, - 0x59, 0x8e, 0x49, 0x5e, 0x28, 0x8d, 0xf9, 0xc2, 0xc2, 0x04, 0x6e, 0x49, 0xe0, 0x36, 0x83, 0xa1, - 0x3b, 0x00, 0xdc, 0xe3, 0x74, 0x2e, 0x16, 0xb8, 0xd8, 0x69, 0xb1, 0xba, 0x0d, 0x6a, 0xdb, 0xc4, - 0x60, 0x70, 0xb6, 0x44, 0x9c, 0xa0, 0x43, 0x1b, 0x30, 0x15, 0xbb, 0x98, 0x60, 0x6d, 0x72, 0xd6, - 0xeb, 0x82, 0xf5, 0x61, 0x1a, 0xc9, 0xf9, 0xb3, 0x1c, 0xdd, 0xbf, 0x2f, 0xc3, 0x54, 0x64, 0x7b, - 0xbe, 0x4b, 0x1d, 0x9f, 0xa0, 0x05, 0xa8, 0xfa, 0x81, 0x1e, 0x0c, 0x7d, 0x6e, 0x7b, 0x93, 0xab, - 0xed, 0x5e, 0xb8, 0x3d, 0xbd, 0x3d, 0x0e, 0xc7, 0x12, 0xcf, 0x28, 0x8f, 0xf8, 0x9a, 0xb9, 0x6d, - 0xe5, 0xed, 0x85, 0xc4, 0xa3, 0x77, 0x61, 0x32, 0x20, 0xde, 0xc0, 0x72, 0x74, 0x5b, 0x23, 0x9e, - 0x47, 0x3d, 0x69, 0x73, 0x13, 0x21, 0x74, 0x8b, 0x01, 0xd1, 0x57, 0xd0, 0xf2, 0x88, 0x6e, 0x6a, - 0xc1, 0x91, 0x47, 0x87, 0xfd, 0xa3, 0x4b, 0xda, 0x5f, 0x93, 0xc9, 0xd8, 0x17, 0x22, 0x98, 0x11, - 0x3e, 0xf7, 0xac, 0x80, 0x68, 0x4c, 0x93, 0xcb, 0x1a, 0x21, 0x97, 0xc0, 0x96, 0x84, 0xb6, 0xa1, - 0xa2, 0x7b, 0xc4, 0xd1, 0xb9, 0x11, 0xb6, 0xd6, 0x3f, 0x7c, 0xf5, 0x72, 0x6e, 0xb9, 0x6f, 0x05, - 0x47, 0xc3, 0x83, 0x9e, 0x41, 0x07, 0xcb, 0xc4, 0x0f, 0x86, 0xba, 0x77, 0x2a, 0xc2, 0xe4, 0x48, - 0xe0, 0xec, 0xad, 0x31, 0x56, 0x2c, 0x24, 0xa0, 0x77, 0xa1, 0x6c, 0x52, 0xc3, 0x57, 0x6a, 0xf3, - 0xa5, 0x85, 0xe6, 0x6a, 0x53, 0x9c, 0xda, 0x9e, 0x6d, 0x19, 0x44, 0x9a, 0x32, 0x47, 0xa3, 0xfb, - 0x50, 0x13, 0x1e, 0xe4, 0x2b, 0xf5, 0xf9, 0xd2, 0x25, 0xb4, 0x0f, 0xd9, 0x99, 0x9d, 0x0d, 0x87, - 0x96, 0xa9, 0xb9, 0xba, 0x17, 0xf8, 0x4a, 0x83, 0x4f, 0x2b, 0xbd, 0xe8, 0xc9, 0x93, 0xed, 0xcd, - 0x5d, 0x06, 0x96, 0x53, 0x37, 0x18, 0x21, 0x07, 0x30, 0xa3, 0x77, 0x75, 0xe3, 0x98, 0x98, 0xda, - 0x31, 0x39, 0x55, 0x60, 0x9c, 0xb2, 0x0d, 0x41, 0xf4, 0x63, 0x72, 0xda, 0x35, 0xa1, 0x83, 0xa9, - 0x71, 0xec, 0x6f, 0xae, 0x6f, 0x12, 0xdf, 0xf0, 0x2c, 0x97, 0xf9, 0xce, 0x12, 0x20, 0x8f, 0x01, - 0xcd, 0x03, 0x8d, 0x38, 0x27, 0xda, 0x80, 0x0c, 0xdc, 0xc0, 0xe3, 0x16, 0x56, 0xc5, 0x6d, 0x89, - 0xd9, 0x72, 0x4e, 0x1e, 0x72, 0x38, 0x7a, 0x0b, 0x5a, 0x21, 0x35, 0x8f, 0xc2, 0x22, 0x42, 0x37, - 0x25, 0x8c, 0x45, 0xe2, 0xee, 0xcf, 0x8a, 0xd0, 0xd8, 0x08, 0x23, 0x2e, 0xba, 0x06, 0x35, 0xcb, - 0xd5, 0x74, 0xd3, 0x14, 0x32, 0x1b, 0xb8, 0x6a, 0xb9, 0x6b, 0xa6, 0xe9, 0xa1, 0x1f, 0xc0, 0x84, - 0x0c, 0xd3, 0x9a, 0x4b, 0xd9, 0xba, 0x8b, 0x7c, 0x05, 0x1d, 0xb1, 0x02, 0x19, 0xa9, 0x77, 0xa9, - 0x17, 0xe0, 0x96, 0x13, 0x0f, 0x7c, 0xb4, 0x07, 0x9d, 0x81, 0xee, 0xba, 0xc4, 0xd4, 0x8e, 0xa8, - 0x1f, 0x48, 0xde, 0x12, 0xe7, 0x7d, 0x2f, 0x8a, 0xe3, 0xd1, 0xfc, 0xbd, 0x87, 0x9c, 0xf6, 0x3e, - 0xf5, 0x03, 0xce, 0xbe, 0xe5, 0x04, 0xde, 0x29, 0x73, 0xb7, 0x14, 0x14, 0xbd, 0x09, 0x30, 0xf4, - 0xf5, 0x3e, 0xd1, 0x3c, 0x3d, 0x20, 0xdc, 0xba, 0x8b, 0xb8, 0xc1, 0x21, 0x58, 0x0f, 0x88, 0xba, - 0x0e, 0xd3, 0x79, 0x72, 0x50, 0x1b, 0x4a, 0x6c, 0xef, 0x0b, 0x3c, 0x76, 0xb0, 0x47, 0x34, 0x0d, - 0x95, 0x13, 0xdd, 0x1e, 0x86, 0x57, 0x97, 0x18, 0xdc, 0x2d, 0x7e, 0x5c, 0xe8, 0xfe, 0x79, 0x11, - 0x3a, 0x1b, 0xe2, 0x8a, 0x97, 0xb7, 0xc9, 0xd6, 0x0b, 0x16, 0x3b, 0xd9, 0xdd, 0xa7, 0xd9, 0xe4, - 0x84, 0xd8, 0xd2, 0xad, 0x27, 0x7b, 0xec, 0xf6, 0xdd, 0xa1, 0xfd, 0xde, 0x0e, 0x83, 0xe2, 0xba, - 0x4d, 0xfb, 0xfc, 0x09, 0x6d, 0xc7, 0x47, 0x65, 0x46, 0x07, 0x28, 0x5d, 0x5c, 0x8d, 0xd6, 0x3e, - 0x72, 0xc4, 0xb8, 0x23, 0xb9, 0x12, 0xa7, 0xbe, 0x0d, 0x2d, 0x3f, 0xd0, 0xbd, 0x40, 0x33, 0xe8, - 0x60, 0x60, 0x05, 0xdc, 0xeb, 0x9b, 0xab, 0xff, 0x2f, 0xde, 0xc0, 0xac, 0xa6, 0x2c, 0xc4, 0x78, - 0xc1, 0x06, 0xa7, 0xc6, 0x4d, 0x3f, 0x1e, 0xa8, 0x18, 0x9a, 0x09, 0x1c, 0xda, 0x00, 0x24, 0x85, - 0x68, 0xc6, 0x11, 0x31, 0x8e, 0x5d, 0x6a, 0x39, 0x01, 0x5f, 0x1a, 0x0b, 0x9e, 0x51, 0xc4, 0xda, - 0x88, 0x70, 0xb8, 0x23, 0xe9, 0x63, 0x50, 0xf7, 0xbf, 0xcb, 0x80, 0x22, 0x15, 0x44, 0xf8, 0x63, - 0xbb, 0xb5, 0x02, 0x8d, 0xe8, 0x2e, 0x97, 0x22, 0xd1, 0xe8, 0x99, 0xe3, 0x98, 0x08, 0xdd, 0x85, - 0x2a, 0x75, 0x89, 0x43, 0x4c, 0xb9, 0x4d, 0xdd, 0xd1, 0x15, 0x46, 0xe2, 0x7b, 0x8f, 0x39, 0x25, - 0x96, 0x1c, 0xe8, 0x4b, 0xa8, 0xcb, 0x9c, 0xcc, 0x94, 0xfb, 0xf3, 0xce, 0x59, 0xdc, 0x12, 0x64, - 0xe2, 0x88, 0x0b, 0xdd, 0x03, 0x48, 0xec, 0x41, 0x79, 0xdc, 0x1e, 0x27, 0x64, 0xc4, 0xbb, 0x92, - 0xe0, 0x54, 0x1f, 0x42, 0x55, 0xe8, 0xf6, 0x9d, 0xec, 0xae, 0xfa, 0x14, 0xea, 0xa1, 0xb2, 0xcc, - 0xf2, 0x8f, 0xc9, 0xa9, 0x26, 0x82, 0x04, 0x17, 0xd4, 0xc2, 0x8d, 0x63, 0x72, 0xba, 0xcb, 0x01, - 0x2c, 0xad, 0x62, 0x51, 0xc9, 0x62, 0x97, 0x92, 0x1f, 0x52, 0x15, 0x39, 0x55, 0x3b, 0x46, 0x08, - 0x62, 0xf5, 0x39, 0x40, 0x3c, 0x0b, 0x9a, 0x87, 0x0a, 0xbb, 0x8e, 0x7c, 0xa9, 0x1d, 0x70, 0xb3, - 0x66, 0x17, 0x95, 0x8f, 0x05, 0x02, 0xfd, 0x08, 0x9a, 0x2e, 0xb5, 0x6d, 0xcd, 0x23, 0xfe, 0xd0, - 0x0e, 0xb8, 0xd8, 0xc9, 0xb3, 0xf7, 0x67, 0x97, 0xda, 0x36, 0xe6, 0xd4, 0x18, 0xdc, 0xe8, 0xb9, - 0xfb, 0x08, 0x20, 0xc6, 0xa0, 0x26, 0xd4, 0xb6, 0x1f, 0x3d, 0x5d, 0xdb, 0xd9, 0xde, 0x6c, 0x5f, - 0x41, 0x0d, 0xa8, 0xe0, 0xad, 0xb5, 0xcd, 0xdf, 0x6e, 0x17, 0xd0, 0x04, 0x34, 0x1e, 0x3d, 0xde, - 0xd7, 0xc4, 0xb0, 0x88, 0x5a, 0x50, 0xdf, 0x78, 0xfc, 0x78, 0x47, 0x7b, 0x7c, 0xef, 0x5e, 0xbb, - 0xc4, 0x98, 0xf0, 0xd6, 0xde, 0xfe, 0x1a, 0xde, 0x6f, 0x97, 0xbb, 0xff, 0x5a, 0x80, 0xf6, 0x26, - 0xcf, 0xb5, 0xbf, 0x07, 0xae, 0xba, 0x0a, 0x65, 0x66, 0x90, 0xd2, 0x04, 0x6f, 0x46, 0xcc, 0x59, - 0x05, 0xb9, 0xf9, 0x62, 0x4e, 0xab, 0x2e, 0x41, 0x99, 0x8d, 0xd0, 0x3b, 0x30, 0xe9, 0x7f, 0x6d, - 0xb3, 0x5b, 0xf6, 0xe4, 0xd0, 0xd7, 0x86, 0x9e, 0x25, 0x83, 0x70, 0x4b, 0x40, 0x9f, 0x1e, 0xfa, - 0x4f, 0x3c, 0xab, 0xfb, 0xef, 0x25, 0xe8, 0x84, 0xd2, 0xbe, 0x8d, 0xb3, 0x7d, 0x92, 0x71, 0xb6, - 0xb7, 0x46, 0x74, 0x1d, 0xeb, 0x6b, 0xeb, 0xd0, 0x70, 0x87, 0x07, 0xb6, 0xe5, 0x1f, 0xe5, 0x38, - 0xdb, 0x28, 0xf7, 0x6e, 0x48, 0x8b, 0x63, 0x36, 0xf4, 0x29, 0xd4, 0x0e, 0xed, 0x21, 0x97, 0x50, - 0xce, 0x38, 0xfb, 0xa8, 0x84, 0x7b, 0x82, 0x12, 0x87, 0x2c, 0xdf, 0xb5, 0x8f, 0x05, 0xd0, 0x88, - 0x94, 0x64, 0x45, 0xcd, 0x40, 0x7f, 0xa1, 0x19, 0x36, 0x35, 0x8e, 0xe5, 0xd5, 0x5a, 0x1f, 0xe8, - 0x2f, 0x36, 0xd8, 0x38, 0xe3, 0x81, 0xc5, 0x73, 0x79, 0x60, 0x69, 0x8c, 0x07, 0x2e, 0x42, 0x4d, - 0x2e, 0xec, 0xf5, 0xee, 0xd7, 0xfd, 0xc3, 0x02, 0xcc, 0xc4, 0xc9, 0xe8, 0xf7, 0xc0, 0xd4, 0xbb, - 0xbf, 0x28, 0xc0, 0x6c, 0x4a, 0xa3, 0x6f, 0x63, 0x8d, 0x6b, 0xb1, 0x39, 0x08, 0x65, 0xe2, 0xf4, - 0x20, 0x7f, 0x8e, 0x51, 0x9b, 0xb8, 0xd0, 0x76, 0xfe, 0xa2, 0x0c, 0x93, 0x1b, 0x74, 0x70, 0x60, - 0x39, 0x51, 0xb9, 0xb8, 0x22, 0x5d, 0x57, 0xf0, 0xbc, 0x91, 0xd0, 0x37, 0x49, 0x96, 0x70, 0x5c, - 0x74, 0x1b, 0x4a, 0xba, 0x19, 0x2a, 0x7c, 0x63, 0x1c, 0xc3, 0x9a, 0x69, 0x62, 0x46, 0xa7, 0xfe, - 0x43, 0x51, 0x3a, 0xfa, 0x97, 0x50, 0x3f, 0xb0, 0x1c, 0xd3, 0x72, 0xfa, 0x4c, 0xc3, 0x52, 0xfa, - 0xae, 0x1a, 0x9d, 0xad, 0xb7, 0x2e, 0x88, 0x71, 0xc4, 0xa5, 0xfe, 0x7e, 0x11, 0x6a, 0x12, 0x8a, - 0x10, 0x94, 0x0f, 0x87, 0xb6, 0x38, 0xfa, 0x3a, 0xe6, 0xcf, 0x61, 0xae, 0xc3, 0xb2, 0xb4, 0x86, - 0xc8, 0x75, 0x3e, 0x86, 0xa6, 0xeb, 0xd1, 0x67, 0xa2, 0x0c, 0x0a, 0x73, 0xb0, 0xb6, 0xc8, 0xdf, - 0x76, 0x23, 0x84, 0x4c, 0x43, 0x93, 0xa4, 0xe8, 0x33, 0x68, 0xfa, 0xc6, 0x11, 0x19, 0xe8, 0xda, - 0x33, 0x9f, 0x3a, 0xdc, 0x5b, 0x5b, 0xeb, 0x6f, 0xbc, 0x7a, 0x39, 0xa7, 0x10, 0xc7, 0xa0, 0x4c, - 0x85, 0x65, 0x86, 0xe8, 0x61, 0xfd, 0xf9, 0x43, 0xe2, 0xf3, 0x34, 0x0c, 0x04, 0xc3, 0x03, 0x9f, - 0x3a, 0xa8, 0x07, 0xe0, 0x13, 0x4f, 0x73, 0xa9, 0x6d, 0x19, 0xa7, 0xbc, 0x74, 0x88, 0xf2, 0xe5, - 0x3d, 0xe2, 0xed, 0x72, 0x30, 0x6e, 0xf8, 0xe1, 0x23, 0x6f, 0x1b, 0xf0, 0xfc, 0x3a, 0xf0, 0x78, - 0x79, 0xd0, 0xc0, 0x35, 0x9e, 0x46, 0x07, 0x1e, 0xab, 0xc2, 0x79, 0x8a, 0x26, 0xb2, 0xfd, 0x06, - 0x96, 0x23, 0xd5, 0x81, 0xd2, 0x9a, 0x69, 0x22, 0x05, 0x6a, 0x72, 0x83, 0x64, 0x92, 0x17, 0x0e, - 0xd1, 0x0f, 0xa1, 0x6e, 0x52, 0x43, 0xe8, 0x5f, 0x3c, 0x87, 0xfe, 0x35, 0x93, 0x1a, 0x5c, 0xf9, - 0x69, 0xa8, 0x1c, 0x7a, 0xd4, 0x11, 0x29, 0x57, 0x1d, 0x8b, 0x41, 0xf7, 0x1f, 0x0b, 0x30, 0x15, - 0x9d, 0x93, 0xac, 0xf7, 0xc6, 0x4f, 0xae, 0x40, 0xcd, 0x24, 0x36, 0x09, 0xa4, 0x69, 0xd7, 0x71, - 0x38, 0x4c, 0xa9, 0x55, 0xba, 0x94, 0x5a, 0xe5, 0x84, 0x5a, 0x99, 0xd8, 0x54, 0xc9, 0xc6, 0xa6, - 0xb7, 0x61, 0x42, 0xec, 0x57, 0x48, 0xc1, 0x8b, 0x2f, 0xdc, 0x12, 0x40, 0x41, 0xd4, 0xbd, 0x06, - 0x33, 0x1b, 0xd4, 0x71, 0x88, 0x11, 0x50, 0x6f, 0xd7, 0xa3, 0x2f, 0x4e, 0xa5, 0x21, 0x76, 0xff, - 0xb8, 0x00, 0xb3, 0x59, 0x8c, 0x5c, 0xfa, 0x03, 0xa8, 0xb1, 0x92, 0x81, 0xf8, 0xbe, 0xec, 0xb3, - 0xac, 0xbc, 0x7a, 0x39, 0xb7, 0x74, 0x9e, 0xda, 0x6a, 0xcb, 0x31, 0x45, 0x4c, 0x0e, 0x05, 0xb0, - 0xd3, 0x77, 0x99, 0x70, 0xcd, 0x32, 0x65, 0x56, 0x5e, 0xe3, 0xe3, 0x6d, 0x13, 0xa9, 0x50, 0xb2, - 0x69, 0x5f, 0xde, 0x37, 0xf5, 0x30, 0xc2, 0x61, 0x06, 0xec, 0xfe, 0x65, 0x09, 0xca, 0x0f, 0xa8, - 0xe5, 0xa0, 0x5b, 0xd0, 0x21, 0x81, 0x61, 0x6a, 0x03, 0x6a, 0x6a, 0x1e, 0x39, 0xb1, 0x7c, 0x56, - 0xd1, 0x33, 0xad, 0x4a, 0x78, 0x8a, 0x21, 0x1e, 0x52, 0x13, 0x4b, 0x30, 0x5a, 0x84, 0xaa, 0x7f, - 0xa4, 0x7b, 0x66, 0x58, 0xcd, 0x5c, 0x8d, 0x9c, 0x90, 0x89, 0x12, 0xcd, 0x0b, 0x2c, 0x49, 0xd0, - 0x1c, 0x34, 0xf9, 0x93, 0xec, 0x40, 0x94, 0xf8, 0x19, 0x03, 0x07, 0x89, 0xfe, 0xc3, 0x22, 0x74, - 0xc2, 0x26, 0x85, 0x69, 0x79, 0x7c, 0x9b, 0x4e, 0xc3, 0x9e, 0x96, 0x44, 0x6c, 0x86, 0x70, 0xf4, - 0x3e, 0x84, 0x30, 0x8d, 0xc8, 0x3d, 0xe0, 0x07, 0xd6, 0xc0, 0x53, 0x12, 0x1e, 0x6e, 0x0d, 0x7a, - 0x0f, 0xa6, 0x6c, 0x5e, 0xfe, 0xc7, 0x94, 0xc2, 0x2d, 0x26, 0x05, 0x38, 0x24, 0x54, 0xff, 0xa2, - 0x00, 0x15, 0xae, 0x33, 0x9a, 0x84, 0xa2, 0x65, 0xca, 0xe4, 0xa1, 0x68, 0x99, 0xa8, 0x07, 0x75, - 0x5b, 0x3f, 0x20, 0x36, 0x33, 0xce, 0xa2, 0x8c, 0xc6, 0x3c, 0x22, 0x32, 0xea, 0x1d, 0x89, 0xc1, - 0x11, 0x0d, 0x5a, 0x85, 0x9a, 0x47, 0x74, 0xa6, 0xa9, 0xdc, 0x6d, 0x25, 0x6e, 0x49, 0xec, 0x7a, - 0xd4, 0x20, 0xbe, 0xbf, 0xe7, 0x12, 0xa3, 0xb7, 0xbd, 0x89, 0x43, 0x42, 0xb4, 0x02, 0xd3, 0x7c, - 0xe3, 0x0d, 0x8f, 0xe8, 0x01, 0x89, 0xf7, 0x9e, 0x37, 0x1f, 0x30, 0x62, 0xb8, 0x0d, 0x8e, 0x0a, - 0xb7, 0xbf, 0x7b, 0x07, 0xaa, 0x6c, 0x9f, 0x89, 0xc9, 0x0e, 0x8d, 0xdd, 0xb8, 0x9c, 0x3f, 0x7b, - 0x68, 0x03, 0xfd, 0xc5, 0x56, 0x60, 0x44, 0x87, 0xd6, 0xfd, 0x59, 0x01, 0xca, 0xfb, 0xba, 0x7f, - 0xcc, 0xc2, 0x9e, 0xef, 0x12, 0x43, 0x66, 0xc1, 0xfc, 0x99, 0x6d, 0x2b, 0x13, 0x14, 0x78, 0xba, - 0xe3, 0xeb, 0x51, 0xa4, 0x63, 0x27, 0xc5, 0xe4, 0xec, 0x27, 0xc0, 0x39, 0xc9, 0x56, 0x79, 0x34, - 0xd9, 0x62, 0x15, 0x74, 0x98, 0xb2, 0x78, 0xcc, 0x24, 0x85, 0x53, 0x35, 0x23, 0xd8, 0xb6, 0xf9, - 0xa0, 0x5c, 0x2f, 0xb6, 0x4b, 0xdd, 0x9f, 0x57, 0xa1, 0x86, 0x89, 0x41, 0x4f, 0xf8, 0x5d, 0xd6, - 0xd4, 0x8d, 0x63, 0xcd, 0x72, 0x02, 0xe2, 0x04, 0x61, 0x84, 0x9f, 0x8f, 0x2f, 0x57, 0x41, 0xd6, - 0x5b, 0x33, 0x8e, 0xb7, 0x05, 0x89, 0xa8, 0x73, 0x41, 0x8f, 0x00, 0x68, 0x15, 0x66, 0x44, 0xad, - 0x17, 0x10, 0x93, 0x65, 0x22, 0x3e, 0x91, 0xf9, 0x48, 0x91, 0xe7, 0x23, 0x57, 0x23, 0xe4, 0x06, - 0xc3, 0x89, 0xd4, 0xe4, 0x4b, 0x40, 0x31, 0x0f, 0x8f, 0x08, 0x16, 0x09, 0x0f, 0xb0, 0xd3, 0x0b, - 0x9b, 0xc0, 0xf7, 0x24, 0x02, 0x77, 0x22, 0xe2, 0x10, 0x84, 0x96, 0x60, 0xda, 0x08, 0x5d, 0x5c, - 0x63, 0xf7, 0x24, 0x49, 0x84, 0x7c, 0x3c, 0x19, 0xe1, 0xd8, 0x4d, 0x4a, 0xd0, 0x12, 0xa0, 0x23, - 0xb6, 0xc6, 0xb4, 0x82, 0x15, 0xd1, 0x8b, 0x10, 0x98, 0x84, 0x76, 0x77, 0x61, 0x4a, 0x52, 0x47, - 0xaa, 0x55, 0xc7, 0xa9, 0x36, 0x29, 0x28, 0x23, 0xbd, 0xde, 0x82, 0x96, 0xad, 0xfb, 0x81, 0xa6, - 0xbb, 0xae, 0x6d, 0x11, 0x93, 0xf7, 0x21, 0x5b, 0xb8, 0xc9, 0x60, 0x6b, 0x02, 0x84, 0xd6, 0xa0, - 0x63, 0x93, 0xbe, 0x6e, 0x9c, 0x26, 0xb3, 0xc0, 0xfa, 0x19, 0x59, 0x60, 0x5b, 0x90, 0x27, 0x4a, - 0xa0, 0x8f, 0x81, 0xa5, 0x79, 0xda, 0x31, 0x39, 0x0d, 0xdb, 0x3a, 0x6f, 0x8e, 0x9c, 0xd9, 0x43, - 0xfd, 0xc5, 0x8f, 0xc9, 0xa9, 0x3c, 0xb0, 0xda, 0x40, 0x8c, 0xd0, 0x2d, 0xb8, 0x1a, 0x78, 0x56, - 0xbf, 0xcf, 0xae, 0x39, 0xdd, 0xd3, 0x07, 0xbe, 0xd8, 0x36, 0xe0, 0x6a, 0x4e, 0x48, 0xd4, 0x2e, - 0xc7, 0xa0, 0x5d, 0x68, 0x33, 0x13, 0x3c, 0x21, 0xda, 0x81, 0x6e, 0x1c, 0x1f, 0x5a, 0xb6, 0xed, - 0x2b, 0x4d, 0x3e, 0xdb, 0xbb, 0x39, 0x16, 0xc2, 0x08, 0xd7, 0x43, 0x3a, 0xd9, 0x0e, 0xd1, 0xd3, - 0x50, 0xf5, 0x33, 0x98, 0xca, 0x98, 0x52, 0xb2, 0xd5, 0xd1, 0xc8, 0x69, 0x75, 0xb4, 0x12, 0xad, - 0x0e, 0xf5, 0x2e, 0xb4, 0x92, 0xab, 0x7a, 0x5d, 0x9b, 0x24, 0xc5, 0xbb, 0x0e, 0xd3, 0x79, 0x3a, - 0xbe, 0x4e, 0x46, 0x35, 0xd9, 0x6a, 0xf9, 0xa7, 0x3a, 0xd4, 0x76, 0x89, 0xe7, 0x5b, 0x7e, 0x80, - 0x66, 0xa0, 0xea, 0x93, 0xaf, 0x35, 0x87, 0x72, 0xd6, 0x32, 0xae, 0xf8, 0xe4, 0xeb, 0x47, 0x94, - 0x59, 0x9a, 0xb8, 0x32, 0xb5, 0xa4, 0x5f, 0x89, 0xcb, 0xb4, 0x2d, 0x30, 0xf1, 0x0e, 0x64, 0xdd, - 0xaf, 0x94, 0x71, 0x3f, 0x39, 0xd7, 0xe5, 0xdc, 0xaf, 0x3c, 0xde, 0xfd, 0xee, 0xc2, 0x75, 0xa9, - 0x64, 0x8e, 0x17, 0x56, 0xb8, 0xae, 0xd7, 0x04, 0xc1, 0xc6, 0x88, 0xe3, 0xe5, 0xbb, 0x6e, 0xf5, - 0x02, 0xae, 0xbb, 0x02, 0xb3, 0xb1, 0xeb, 0xba, 0x7a, 0x60, 0x1c, 0x11, 0x69, 0x85, 0xc2, 0x59, - 0xda, 0x11, 0x76, 0x57, 0x20, 0xc7, 0xb8, 0x6f, 0x7d, 0x8c, 0xfb, 0xde, 0x81, 0x59, 0xb9, 0xba, - 0xac, 0x17, 0x37, 0xf8, 0xd2, 0xa6, 0x05, 0xf6, 0x7e, 0xda, 0x71, 0x73, 0x9c, 0x1e, 0x2e, 0xeb, - 0xf4, 0xcd, 0x51, 0xa7, 0xff, 0x18, 0x14, 0xa9, 0xd4, 0xa8, 0xef, 0xb7, 0xb8, 0x5a, 0x52, 0xe9, - 0x9d, 0xac, 0xaf, 0xe7, 0x86, 0x8b, 0x89, 0x4b, 0x87, 0x8b, 0xc9, 0x4c, 0xb8, 0x08, 0x6d, 0x2c, - 0x3f, 0x5c, 0xac, 0xc2, 0x8c, 0x54, 0x3b, 0x1d, 0x35, 0x94, 0x29, 0xae, 0xf3, 0x55, 0x81, 0xdc, - 0x4f, 0x85, 0x8d, 0x31, 0x21, 0xa6, 0x9d, 0x17, 0x62, 0xf8, 0xcb, 0x2a, 0xdf, 0xd0, 0x1d, 0xa5, - 0x13, 0xbe, 0xac, 0x62, 0x23, 0x74, 0x07, 0x2a, 0x07, 0xa4, 0x6f, 0x39, 0x0a, 0xca, 0x54, 0x38, - 0x69, 0x1f, 0x5e, 0x67, 0x34, 0xf7, 0xaf, 0x60, 0x41, 0x8c, 0x16, 0xa1, 0x6d, 0xd0, 0x81, 0xcb, - 0xf5, 0x0d, 0x33, 0xdc, 0xab, 0xcc, 0xb1, 0xef, 0x5f, 0xc1, 0x53, 0x21, 0x46, 0xd6, 0x22, 0xff, - 0x87, 0xb1, 0x68, 0x5d, 0x81, 0xd9, 0x4c, 0x60, 0xd5, 0x8c, 0x23, 0xdd, 0xe9, 0x93, 0x2e, 0x86, - 0xab, 0x39, 0x2b, 0x3c, 0x23, 0x63, 0x7f, 0x0b, 0x5a, 0x81, 0x37, 0x74, 0x0c, 0x9d, 0x59, 0xae, - 0x1e, 0xc8, 0x98, 0xd5, 0x8c, 0x60, 0x6b, 0x41, 0xb7, 0x0b, 0x0d, 0x79, 0xc8, 0xc4, 0x1c, 0x13, - 0xb6, 0xba, 0x7f, 0x5a, 0x80, 0x0a, 0x33, 0xd5, 0xd3, 0xdc, 0x5c, 0x45, 0x81, 0xda, 0x09, 0x93, - 0x20, 0x4b, 0x92, 0x06, 0x0e, 0x87, 0xe8, 0x06, 0x34, 0xb8, 0xe5, 0x73, 0x16, 0x71, 0xf7, 0xd6, - 0x19, 0x80, 0xe5, 0x5c, 0x91, 0x5b, 0x84, 0xbc, 0x22, 0x6b, 0xe4, 0x6e, 0xf1, 0x54, 0xf2, 0xaf, - 0x8c, 0xb9, 0xc6, 0x45, 0xbe, 0x8f, 0xd2, 0xd7, 0x38, 0xab, 0x27, 0xba, 0xcf, 0xa0, 0x16, 0xfa, - 0xd4, 0x6d, 0x40, 0x22, 0x45, 0x8a, 0xfa, 0x03, 0x61, 0x32, 0xd6, 0xc0, 0x1d, 0x81, 0xd9, 0x8c, - 0x11, 0x67, 0xc4, 0x9d, 0x62, 0x7e, 0xdc, 0xe9, 0xfe, 0xaa, 0x20, 0xab, 0xe0, 0x8b, 0x6d, 0xca, - 0xbb, 0xe1, 0x7b, 0xcb, 0x52, 0xee, 0x7b, 0xcb, 0xf0, 0x8d, 0xe5, 0xdb, 0x67, 0xa6, 0x30, 0xbc, - 0xf8, 0x27, 0xe8, 0xa3, 0x84, 0xeb, 0x56, 0xb8, 0xeb, 0xc6, 0xad, 0x0f, 0x5e, 0x70, 0xe7, 0xfa, - 0xed, 0xb7, 0xb1, 0xce, 0x2e, 0x40, 0x9d, 0x47, 0xd3, 0x47, 0xf4, 0x79, 0xb7, 0x0a, 0xe5, 0xbd, - 0x80, 0xba, 0xdd, 0x06, 0xd4, 0xd8, 0x5f, 0x97, 0x98, 0xdd, 0xdf, 0x82, 0xe6, 0x1e, 0xf1, 0xd9, - 0x42, 0x77, 0x28, 0x75, 0xc7, 0x74, 0x69, 0x0a, 0x97, 0xe9, 0xd2, 0xfc, 0x51, 0x15, 0x6a, 0xb2, - 0x37, 0x8b, 0xde, 0x4f, 0xec, 0x78, 0x73, 0x75, 0xa6, 0x17, 0x7e, 0xc4, 0x10, 0x36, 0x1b, 0xf8, - 0x46, 0x8a, 0x83, 0xf8, 0xff, 0x30, 0xc1, 0xfe, 0x6a, 0x9e, 0x2c, 0xf2, 0x64, 0xdd, 0x30, 0x9b, - 0xe0, 0x11, 0x08, 0xc1, 0xd4, 0x62, 0xc4, 0x51, 0x41, 0xf8, 0x11, 0xd4, 0x4d, 0xcb, 0xe7, 0x39, - 0x8c, 0x3c, 0xae, 0xeb, 0x23, 0x73, 0x6d, 0x4a, 0x02, 0x1c, 0x91, 0xa2, 0x4f, 0x01, 0xc2, 0xe7, - 0xa8, 0x2b, 0xf8, 0xc6, 0xe8, 0x84, 0x9b, 0x11, 0x0d, 0x4e, 0xd0, 0xb3, 0x49, 0x4f, 0x74, 0xdb, - 0x32, 0xf5, 0x80, 0xc8, 0x2e, 0xc3, 0xe8, 0xa4, 0x4f, 0x25, 0x01, 0x8e, 0x48, 0xd1, 0x27, 0xd0, - 0x08, 0x9f, 0x4d, 0x79, 0xe3, 0xde, 0x18, 0x9d, 0x33, 0x64, 0x34, 0x71, 0x4c, 0x9d, 0x6e, 0xbc, - 0x35, 0x5e, 0xd3, 0x78, 0xfb, 0x21, 0xb4, 0x7c, 0x71, 0xc2, 0x9a, 0x4d, 0xa9, 0xab, 0x4c, 0xcb, - 0xcb, 0x26, 0x3c, 0xcc, 0xc4, 0xf1, 0xe3, 0xa6, 0x9f, 0xb0, 0x85, 0xb7, 0xa0, 0xfc, 0x8c, 0x5a, - 0x8e, 0x32, 0xc3, 0x19, 0x26, 0x52, 0x35, 0x2a, 0xe6, 0x28, 0xf4, 0x1e, 0x54, 0x9f, 0xf1, 0x4a, - 0x4a, 0x99, 0x95, 0xce, 0x91, 0x24, 0x22, 0x26, 0x96, 0x68, 0x26, 0x2b, 0xd0, 0xfd, 0x63, 0xe5, - 0x5a, 0x46, 0x16, 0x2b, 0xa8, 0x30, 0x47, 0xa1, 0xe5, 0xa8, 0x2d, 0xac, 0x70, 0xa2, 0x6b, 0xd9, - 0x0e, 0x7f, 0xb6, 0x19, 0xdc, 0x83, 0x86, 0x48, 0x20, 0x1c, 0xfa, 0x5c, 0xb9, 0x2e, 0x6f, 0xf7, - 0x88, 0x47, 0xda, 0x3c, 0xae, 0x1b, 0xf2, 0x89, 0xe9, 0xe0, 0x07, 0xd4, 0x55, 0xd4, 0x8c, 0x0e, - 0xcc, 0x15, 0x30, 0x47, 0xa1, 0x5b, 0x50, 0xf3, 0x85, 0x63, 0x28, 0x37, 0xe4, 0x2b, 0xf1, 0x24, - 0x95, 0x4b, 0x4c, 0x1c, 0x12, 0xa8, 0x77, 0xa3, 0x4e, 0xf0, 0x85, 0x9b, 0x8e, 0xdd, 0x3f, 0xb8, - 0x0e, 0xcd, 0x44, 0x77, 0x11, 0xdd, 0x4e, 0xf9, 0xc7, 0xf5, 0x5e, 0xf2, 0xe3, 0x9b, 0x1c, 0x1f, - 0xf9, 0x22, 0xdf, 0x47, 0xd4, 0x0c, 0xdf, 0x78, 0x3f, 0xf9, 0x24, 0x61, 0xb2, 0xc2, 0x4f, 0xde, - 0xcc, 0x9d, 0x33, 0xc7, 0x6c, 0x3f, 0x4b, 0x9a, 0xad, 0x70, 0x95, 0xb9, 0xfc, 0x79, 0x5f, 0x6f, - 0xba, 0x95, 0x5f, 0x13, 0xd3, 0xbd, 0x05, 0x35, 0x4f, 0x54, 0x4e, 0xd2, 0x76, 0xdb, 0xd9, 0x8a, - 0x0a, 0x87, 0x04, 0xe8, 0x1d, 0xa8, 0xb0, 0xc4, 0xf2, 0x54, 0x5a, 0x6c, 0xfc, 0x51, 0x11, 0xbf, - 0xb0, 0xb1, 0x40, 0x32, 0x89, 0x61, 0xfa, 0xa9, 0x66, 0x24, 0xca, 0xfb, 0x12, 0x87, 0x04, 0x4c, - 0x41, 0xde, 0x3e, 0xbe, 0x91, 0x51, 0x30, 0xd1, 0x2f, 0xfe, 0x30, 0xf2, 0xad, 0x37, 0x32, 0x2d, - 0xe3, 0x84, 0x15, 0x66, 0xfd, 0xeb, 0x36, 0x94, 0x6d, 0xaa, 0x9b, 0xca, 0x82, 0x34, 0xca, 0x3c, - 0x96, 0x1d, 0xaa, 0x9b, 0x98, 0x93, 0xb1, 0x39, 0xd8, 0x5f, 0x62, 0x2a, 0xef, 0x9f, 0x31, 0xc7, - 0x0e, 0x27, 0xc1, 0x92, 0x14, 0xad, 0x40, 0x85, 0x77, 0xd1, 0x95, 0x5b, 0x99, 0x2b, 0x26, 0xc9, - 0xc3, 0x9b, 0xeb, 0x58, 0x10, 0xa2, 0x1f, 0xc4, 0xfd, 0xfa, 0xc5, 0x4c, 0x36, 0x39, 0xc2, 0x93, - 0x68, 0xd2, 0xb3, 0x99, 0xfc, 0x80, 0x7a, 0x44, 0x59, 0x3a, 0x63, 0xa6, 0x3d, 0x46, 0x81, 0x05, - 0x21, 0x5b, 0x10, 0x7f, 0x30, 0x95, 0xdb, 0x67, 0x2c, 0x88, 0xb3, 0x98, 0x58, 0x92, 0xa2, 0x8d, - 0xcc, 0x1b, 0xf3, 0x1e, 0x67, 0x9d, 0x1f, 0xc3, 0x9a, 0xff, 0xae, 0x1c, 0x6d, 0xc3, 0x24, 0x1f, - 0xb2, 0x12, 0x49, 0x88, 0x59, 0xce, 0xbc, 0xa9, 0x1a, 0x11, 0x43, 0x4c, 0x29, 0x68, 0xc2, 0x4f, - 0x0e, 0xd1, 0x3a, 0xaf, 0x49, 0x1d, 0xfa, 0xdc, 0x26, 0x66, 0x9f, 0x28, 0x2b, 0x67, 0xa8, 0xb3, - 0x16, 0xd3, 0xe1, 0x24, 0x13, 0xda, 0x82, 0x56, 0x62, 0x68, 0x2a, 0x1f, 0x64, 0x5e, 0xdb, 0x8d, - 0x11, 0x62, 0xe2, 0x14, 0x1b, 0xb3, 0x69, 0x57, 0x64, 0xae, 0xca, 0x6a, 0xc6, 0xa6, 0x65, 0x46, - 0x8b, 0x43, 0x02, 0x16, 0x52, 0xdd, 0x30, 0xcb, 0x55, 0x3e, 0xcc, 0x84, 0xd4, 0x28, 0xff, 0xc5, - 0x31, 0x51, 0xfa, 0x36, 0xb8, 0x73, 0xfe, 0xdb, 0xe0, 0xd3, 0x73, 0xdd, 0x06, 0x9f, 0xbd, 0xee, - 0x36, 0xf8, 0xdd, 0xc2, 0xe5, 0xaf, 0x03, 0xf4, 0xa3, 0x64, 0xee, 0x98, 0xa8, 0x0b, 0x8b, 0x67, - 0xd4, 0x85, 0x57, 0x23, 0x8e, 0xc4, 0xeb, 0xc4, 0x8f, 0xa0, 0xcc, 0x1c, 0x0c, 0xdd, 0x86, 0x7a, - 0x54, 0xf7, 0x16, 0xc6, 0xd5, 0xbd, 0x11, 0x89, 0xfa, 0xab, 0x22, 0x54, 0x85, 0x63, 0xa2, 0x2f, - 0x46, 0xde, 0x10, 0xbd, 0x7d, 0x86, 0x1f, 0x8f, 0xbe, 0x20, 0x12, 0x35, 0x00, 0x7f, 0x43, 0xe1, - 0x69, 0xe2, 0x63, 0x99, 0x83, 0xd3, 0x80, 0x88, 0xa6, 0x49, 0x99, 0xd5, 0x00, 0x02, 0xf7, 0x84, - 0xa1, 0xd6, 0x19, 0x46, 0xfd, 0x8f, 0x42, 0xfc, 0x4a, 0x69, 0x1a, 0x2a, 0xa2, 0xcd, 0x2d, 0x72, - 0x5b, 0x31, 0x40, 0x0b, 0xd0, 0x1e, 0x58, 0x8e, 0xe6, 0xd3, 0xa1, 0x67, 0xa4, 0xfb, 0x91, 0x93, - 0x03, 0xcb, 0xd9, 0xe3, 0x60, 0xd1, 0x2d, 0x58, 0x10, 0x7d, 0xd8, 0x14, 0x65, 0x49, 0x52, 0xea, - 0x2f, 0x92, 0x94, 0x4b, 0x80, 0x04, 0x95, 0xa9, 0x99, 0xd4, 0xf0, 0xb5, 0x80, 0x06, 0xba, 0xcd, - 0x2f, 0xb4, 0x32, 0x6e, 0x4b, 0xcc, 0x26, 0x35, 0xfc, 0x7d, 0x06, 0x47, 0x3d, 0xb8, 0x1a, 0x52, - 0xf3, 0xe5, 0x48, 0xf2, 0x0a, 0x27, 0xef, 0x48, 0x14, 0x5f, 0x8e, 0xa0, 0xef, 0xc2, 0x84, 0x4c, - 0xf4, 0x35, 0x93, 0xd8, 0x81, 0xfc, 0xde, 0x0c, 0x37, 0x45, 0x46, 0xbf, 0xc9, 0x40, 0xea, 0xbf, - 0x15, 0xa1, 0xc2, 0xc3, 0xd4, 0x19, 0xb5, 0x4c, 0x61, 0x4c, 0x0f, 0xe5, 0x2b, 0x98, 0x8a, 0x8a, - 0x4d, 0x5e, 0x2d, 0x87, 0xaf, 0x12, 0x16, 0xc6, 0x47, 0xc3, 0x5e, 0xaa, 0x0a, 0xc5, 0x93, 0x07, - 0xc9, 0xa1, 0x8f, 0x7e, 0x02, 0x28, 0xae, 0x5f, 0x65, 0x75, 0x1d, 0x36, 0xb1, 0x16, 0xcf, 0x21, - 0x75, 0x43, 0xf2, 0xe0, 0xce, 0x41, 0x06, 0xe2, 0xab, 0x5f, 0xc0, 0xc4, 0x79, 0x4b, 0xe0, 0x69, - 0xa8, 0x24, 0x0f, 0x58, 0x0c, 0xd4, 0x75, 0x68, 0x67, 0xe7, 0xb9, 0xb0, 0x8c, 0xff, 0x2c, 0xc4, - 0xef, 0x69, 0xcf, 0x7a, 0x11, 0x9a, 0x73, 0x8d, 0xe4, 0xda, 0xf9, 0x05, 0xeb, 0x4f, 0xf5, 0xf4, - 0x75, 0x66, 0x7e, 0x0b, 0x3a, 0xe2, 0x5a, 0x4c, 0x5a, 0xa4, 0xf0, 0x9b, 0x29, 0x81, 0x88, 0x0d, - 0x72, 0x09, 0x90, 0xa4, 0x4d, 0xda, 0x63, 0x49, 0x98, 0xaf, 0xc0, 0xc4, 0xe6, 0xa8, 0xd6, 0xa0, - 0xc2, 0xef, 0x29, 0xf5, 0xaf, 0x0b, 0x50, 0x15, 0x37, 0xd6, 0xb9, 0x3d, 0x5d, 0x90, 0xe7, 0xbc, - 0x0a, 0x3e, 0xcf, 0x7a, 0xc4, 0xad, 0x98, 0xb3, 0x1e, 0x81, 0x48, 0xad, 0x47, 0xd2, 0xe6, 0xac, - 0x47, 0x60, 0x12, 0xeb, 0xf9, 0xbd, 0x42, 0xfa, 0x6b, 0xb2, 0x8b, 0x3b, 0xd0, 0x77, 0x16, 0x72, - 0xd7, 0x60, 0x22, 0x75, 0x01, 0x5f, 0x5c, 0x17, 0xf5, 0x0b, 0x68, 0x26, 0xae, 0xcd, 0x4b, 0x08, - 0xf8, 0x12, 0x5a, 0xc9, 0x7b, 0xf7, 0xe2, 0x12, 0xba, 0x3f, 0x47, 0x50, 0x15, 0x5f, 0xbf, 0xa0, - 0x85, 0x54, 0x2d, 0x32, 0xdd, 0x93, 0xbf, 0x26, 0xc8, 0x29, 0x43, 0xee, 0xe6, 0x97, 0x21, 0x33, - 0x31, 0xcb, 0xf8, 0x0a, 0xe4, 0xce, 0x48, 0x05, 0xa2, 0x64, 0x67, 0xca, 0x29, 0x3e, 0x3e, 0x1e, - 0x2d, 0x3e, 0xd4, 0x91, 0xd9, 0x7e, 0x53, 0x77, 0xe4, 0xd5, 0x1d, 0xe7, 0xa8, 0x12, 0x7a, 0x99, - 0x2a, 0x61, 0x36, 0xf3, 0x61, 0x54, 0xb6, 0x40, 0x58, 0x48, 0x15, 0x08, 0xd3, 0x59, 0xea, 0x44, - 0x6d, 0xd0, 0xcb, 0xd4, 0x06, 0xb3, 0x79, 0xb4, 0x89, 0xb2, 0x60, 0x31, 0x5d, 0x16, 0xcc, 0x64, - 0xc9, 0x53, 0x15, 0xc1, 0x07, 0xd9, 0x8a, 0xe0, 0x5a, 0x2e, 0x79, 0xb2, 0x18, 0x58, 0x4c, 0x17, - 0x03, 0x23, 0xf2, 0x53, 0x75, 0x40, 0x2f, 0x53, 0x07, 0xcc, 0xe6, 0x52, 0xc7, 0x25, 0xc0, 0xe7, - 0xb9, 0x25, 0xc0, 0x8d, 0x51, 0xae, 0x31, 0xd9, 0xff, 0xe6, 0x98, 0xec, 0xff, 0xcd, 0x5c, 0x09, - 0xe3, 0x12, 0xff, 0xdf, 0x64, 0xdb, 0xdf, 0xd7, 0x6c, 0xfb, 0x6f, 0xe2, 0x6c, 0xfb, 0xee, 0xc8, - 0x1d, 0x7c, 0x33, 0xdf, 0x33, 0xbe, 0x93, 0x44, 0xfb, 0xef, 0x7e, 0xfd, 0x12, 0x6d, 0xf5, 0x93, - 0x4b, 0xe7, 0xd0, 0xea, 0xe3, 0x38, 0x1d, 0xbc, 0x78, 0xfe, 0x80, 0xa0, 0x3c, 0x60, 0x01, 0x44, - 0xbc, 0x0b, 0xe6, 0xcf, 0x71, 0x96, 0xf5, 0x3b, 0xa5, 0x28, 0xcb, 0x5a, 0x81, 0xe9, 0xe8, 0xd3, - 0xd3, 0xe4, 0xfa, 0xc5, 0xfb, 0x1a, 0x14, 0xe1, 0xe2, 0x1d, 0x58, 0x85, 0x99, 0x98, 0x23, 0xb9, - 0x07, 0xe2, 0x60, 0xaf, 0x46, 0xc8, 0x44, 0xb9, 0xb1, 0x04, 0xc8, 0xf4, 0x98, 0x75, 0xa7, 0xe6, - 0x90, 0xd9, 0x93, 0xc4, 0xa4, 0xf6, 0x38, 0xa4, 0x4e, 0xca, 0x17, 0x47, 0xd2, 0x91, 0xa8, 0x84, - 0xf4, 0xaf, 0xa0, 0x1d, 0x7f, 0x8b, 0x22, 0x43, 0x52, 0x25, 0xf3, 0x95, 0x7a, 0x2a, 0x14, 0x46, - 0x1f, 0xde, 0x7a, 0x32, 0x36, 0x4d, 0xb9, 0x69, 0x80, 0xaa, 0xc1, 0x54, 0x86, 0x06, 0xa9, 0xfc, - 0x03, 0x2c, 0x73, 0x68, 0x48, 0x2f, 0x6a, 0xe1, 0x68, 0x9c, 0x9f, 0xd0, 0x33, 0x0e, 0xf9, 0x33, - 0x39, 0x51, 0xa7, 0x34, 0x70, 0x34, 0x56, 0x9f, 0xa6, 0x13, 0xc4, 0x71, 0x3e, 0x5f, 0xb8, 0xa8, - 0xcf, 0x4f, 0x65, 0xd2, 0xbd, 0x5b, 0x8b, 0x50, 0xe1, 0x3f, 0x07, 0x44, 0x00, 0xd5, 0xdd, 0x27, - 0xeb, 0x3b, 0xdb, 0x1b, 0xed, 0x2b, 0xa8, 0x09, 0xb5, 0x5d, 0xbc, 0xfd, 0x74, 0x6d, 0x7f, 0xab, - 0x5d, 0x40, 0x0d, 0xa8, 0xec, 0x3c, 0xde, 0x58, 0xdb, 0x69, 0x17, 0x57, 0x1f, 0x40, 0x5d, 0xfe, - 0x5c, 0xcb, 0x43, 0x9f, 0x43, 0x4d, 0x3e, 0xa3, 0xf8, 0xc2, 0x4a, 0xff, 0x90, 0x50, 0x55, 0x46, - 0x11, 0x22, 0xc9, 0x59, 0x29, 0xac, 0xee, 0x40, 0x5d, 0x7e, 0x0a, 0xe8, 0xa1, 0x2f, 0xa1, 0x26, - 0x9f, 0x13, 0xb2, 0xd2, 0x1f, 0x74, 0x26, 0x64, 0x65, 0xbe, 0x20, 0x5c, 0x28, 0xac, 0x14, 0x56, - 0x8f, 0x60, 0x32, 0xfd, 0x91, 0x1d, 0x7a, 0x0a, 0x53, 0xfc, 0x21, 0x02, 0xfb, 0xe8, 0x66, 0x32, - 0xb0, 0x8e, 0x7e, 0xaa, 0xa7, 0xce, 0x8d, 0xc5, 0x27, 0x66, 0x7a, 0x0e, 0xd5, 0x1d, 0xf1, 0xab, - 0xb2, 0x5e, 0x94, 0x73, 0x4e, 0x65, 0xec, 0x48, 0xcd, 0x02, 0x18, 0x27, 0xfa, 0x2c, 0xdd, 0x34, - 0x9f, 0xce, 0x2b, 0x57, 0xd4, 0x5c, 0x28, 0x9f, 0xf8, 0xcf, 0xa2, 0xcf, 0xd4, 0x3e, 0x88, 0xdf, - 0x4c, 0xb5, 0xb3, 0x6f, 0x19, 0xd4, 0x11, 0x08, 0x9f, 0xfb, 0x7f, 0x57, 0xd7, 0xf5, 0xcf, 0xbf, - 0xf9, 0xe7, 0x9b, 0x57, 0xbe, 0xf9, 0xe5, 0xcd, 0xc2, 0xdf, 0xfe, 0xf2, 0x66, 0xe1, 0x4f, 0xfe, - 0xe5, 0x66, 0xe1, 0x27, 0x4b, 0xe7, 0xfa, 0x99, 0x9a, 0x94, 0x77, 0x50, 0xe5, 0xa0, 0x0f, 0xff, - 0x27, 0x00, 0x00, 0xff, 0xff, 0x5d, 0x3e, 0xa7, 0x21, 0xb3, 0x3c, 0x00, 0x00, + // 4643 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5b, 0x3d, 0x6c, 0x1c, 0x59, + 0x72, 0xd6, 0xfc, 0xcf, 0xd4, 0x0c, 0xc9, 0x99, 0x27, 0x8a, 0x6a, 0xb5, 0x76, 0x25, 0xed, 0xec, + 0xae, 0x8f, 0x2b, 0x52, 0x43, 0x2e, 0x57, 0x7b, 0xb7, 0x2b, 0xef, 0x1f, 0x87, 0xa4, 0x8e, 0xd4, + 0x51, 0x12, 0xf7, 0x91, 0x12, 0xec, 0x4b, 0x1a, 0xcd, 0xee, 0xc7, 0x61, 0x8b, 0x3d, 0xfd, 0x7a, + 0xbb, 0x7b, 0x28, 0xf1, 0x22, 0x03, 0x4e, 0x0c, 0x18, 0xb0, 0x93, 0x4b, 0x0c, 0x27, 0xce, 0x0c, + 0x1b, 0x70, 0xe0, 0xe8, 0x80, 0x0b, 0x1c, 0x39, 0x58, 0x38, 0xb2, 0x1d, 0x18, 0x8e, 0x64, 0xf8, + 0xec, 0xd0, 0x8e, 0xce, 0x06, 0x6c, 0xc1, 0x81, 0xf1, 0x7e, 0xfa, 0x77, 0x7a, 0x28, 0x92, 0xbb, + 0xb0, 0x17, 0x87, 0x0b, 0x24, 0xf6, 0xab, 0xfa, 0xea, 0xfd, 0x56, 0xd5, 0xab, 0xaa, 0xee, 0x81, + 0xee, 0x80, 0x2e, 0xb9, 0x1e, 0x0d, 0xa8, 0x41, 0x6d, 0x7f, 0xc9, 0x1b, 0x39, 0x81, 0x35, 0x24, + 0xe1, 0xdf, 0x1e, 0xe7, 0xa0, 0x9a, 0x6c, 0xaa, 0x37, 0xf6, 0x3d, 0x7a, 0x44, 0xbc, 0x48, 0x20, + 0x7a, 0x10, 0x40, 0xf5, 0x96, 0x41, 0x1d, 0x7f, 0x34, 0x3c, 0x05, 0x91, 0x1e, 0xce, 0xd0, 0xdd, + 0x60, 0xe4, 0x91, 0xf0, 0x6f, 0xd8, 0x4b, 0x0a, 0x63, 0x12, 0xcf, 0x3a, 0x26, 0xf2, 0x8f, 0x44, + 0xbc, 0x91, 0x42, 0x1c, 0xd8, 0xf4, 0x39, 0xff, 0x4f, 0x72, 0x6f, 0xa7, 0xb8, 0x43, 0x3d, 0x20, + 0x9e, 0xa5, 0xdb, 0xd6, 0x4f, 0x48, 0xf2, 0x59, 0x62, 0xd5, 0x14, 0x96, 0xba, 0xfc, 0x5f, 0xee, + 0x5c, 0xfd, 0xc3, 0xd1, 0xc1, 0x81, 0x4d, 0xc2, 0xbf, 0x12, 0x33, 0x3b, 0xa0, 0x03, 0xca, 0x1f, + 0x97, 0xd8, 0x93, 0xa0, 0x76, 0xff, 0xaa, 0x00, 0x9d, 0x3d, 0xdd, 0x3f, 0xda, 0x25, 0xde, 0xb1, + 0x65, 0x90, 0x35, 0xea, 0x1c, 0x58, 0x03, 0x74, 0x03, 0x9a, 0x36, 0x1d, 0x68, 0x07, 0x96, 0x4d, + 0xb4, 0x03, 0x53, 0x29, 0xdc, 0x2a, 0xcc, 0x57, 0x70, 0xc3, 0xa6, 0x83, 0xfb, 0x96, 0x4d, 0xee, + 0x9b, 0xe8, 0x3a, 0x34, 0x02, 0xdd, 0x3f, 0xd2, 0x1c, 0x7d, 0x48, 0x94, 0xe2, 0xad, 0xc2, 0x7c, + 0x03, 0xd7, 0x19, 0xe1, 0x91, 0x3e, 0x24, 0xe8, 0x1a, 0xd4, 0x47, 0xa6, 0xaf, 0xb9, 0x7a, 0x70, + 0xa8, 0x94, 0x38, 0xaf, 0x36, 0x32, 0xfd, 0x1d, 0x3d, 0x38, 0x44, 0x0b, 0xd0, 0x31, 0xa8, 0x13, + 0xe8, 0x96, 0x43, 0x3c, 0xcd, 0x21, 0xc1, 0x73, 0xea, 0x1d, 0x29, 0x65, 0x8e, 0x69, 0x47, 0x8c, + 0x47, 0x82, 0x8e, 0xde, 0x81, 0x8a, 0x6b, 0xeb, 0x0e, 0x51, 0xaa, 0xb7, 0x0a, 0xf3, 0xd3, 0x2b, + 0xd3, 0xbd, 0xf0, 0xa8, 0x77, 0x18, 0x15, 0x0b, 0x66, 0xf7, 0xbf, 0xcb, 0x30, 0xbd, 0x2b, 0x16, + 0x8a, 0xc9, 0x57, 0x23, 0xe2, 0x07, 0x68, 0x0b, 0x6a, 0xcf, 0xe8, 0xc8, 0x73, 0x74, 0x9b, 0xcf, + 0xbc, 0xd1, 0x5f, 0x7a, 0xf5, 0xf2, 0xe6, 0xc2, 0x80, 0xf6, 0x06, 0xfa, 0x4f, 0x48, 0x10, 0x90, + 0x9e, 0x49, 0x8e, 0x97, 0x0c, 0xea, 0x91, 0xa5, 0x8c, 0x92, 0xf4, 0x1e, 0x08, 0x31, 0x1c, 0xca, + 0xa3, 0x39, 0xa8, 0x7a, 0xc4, 0xb5, 0xf5, 0x13, 0xbe, 0xca, 0x3a, 0x96, 0x2d, 0xb6, 0xc6, 0xfd, + 0x91, 0x65, 0x9b, 0x9a, 0x65, 0x86, 0x6b, 0xe4, 0xed, 0x2d, 0x13, 0xdd, 0x87, 0x2a, 0x3d, 0x38, + 0xf0, 0x49, 0xc0, 0x17, 0x56, 0xea, 0xf7, 0x5e, 0xbd, 0xbc, 0x79, 0xfb, 0x2c, 0x83, 0x3f, 0xe6, + 0x52, 0x58, 0x4a, 0xa3, 0x87, 0x00, 0xc4, 0x31, 0x35, 0xd9, 0x57, 0xe5, 0x42, 0x7d, 0x35, 0x88, + 0x63, 0x8a, 0x47, 0xb4, 0x00, 0x15, 0x4f, 0x77, 0x06, 0x62, 0x37, 0x9b, 0x2b, 0x33, 0x3d, 0xae, + 0x86, 0x98, 0x91, 0x76, 0x5d, 0x62, 0xf4, 0xcb, 0x5f, 0xbf, 0xbc, 0x79, 0x09, 0x0b, 0x0c, 0xda, + 0x85, 0xa6, 0x41, 0xa9, 0x67, 0x5a, 0x8e, 0x1e, 0x50, 0x4f, 0xa9, 0xf1, 0x5d, 0x7c, 0xff, 0xd5, + 0xcb, 0x9b, 0x77, 0xf2, 0x06, 0x1f, 0x33, 0xa5, 0xde, 0xee, 0xa1, 0xee, 0x99, 0x5b, 0xeb, 0x38, + 0xd9, 0x0b, 0x5a, 0x06, 0xf0, 0x88, 0x4f, 0xed, 0x51, 0x60, 0x51, 0x47, 0xa9, 0xf3, 0x69, 0xb4, + 0x7b, 0x91, 0xcc, 0x26, 0xd1, 0x4d, 0xe2, 0xe1, 0x04, 0x06, 0xbd, 0x0d, 0x53, 0x52, 0x87, 0x35, + 0xcb, 0x31, 0xc9, 0x0b, 0xa5, 0x71, 0xab, 0x30, 0x3f, 0x85, 0x5b, 0x92, 0xb8, 0xc5, 0x68, 0xe8, + 0x2e, 0x00, 0xb7, 0x38, 0x9d, 0x77, 0x0b, 0xbc, 0xdb, 0x59, 0xb1, 0xba, 0x35, 0x6a, 0xdb, 0xc4, + 0x60, 0x74, 0xb6, 0x44, 0x9c, 0xc0, 0xa1, 0x35, 0x98, 0x89, 0x4d, 0x4c, 0x88, 0x36, 0xb9, 0xe8, + 0x35, 0x21, 0xfa, 0x30, 0xcd, 0xe4, 0xf2, 0x59, 0x89, 0xee, 0xdf, 0x97, 0x61, 0x26, 0xd2, 0x3d, + 0xdf, 0xa5, 0x8e, 0x4f, 0xd0, 0x3c, 0x54, 0xfd, 0x40, 0x0f, 0x46, 0x3e, 0xd7, 0xbd, 0xe9, 0x95, + 0x76, 0x2f, 0xdc, 0x9e, 0xde, 0x2e, 0xa7, 0x63, 0xc9, 0x67, 0xc8, 0x43, 0xbe, 0x66, 0xae, 0x5b, + 0x79, 0x7b, 0x21, 0xf9, 0xe8, 0x5d, 0x98, 0x0e, 0x88, 0x37, 0xb4, 0x1c, 0xdd, 0xd6, 0x88, 0xe7, + 0x51, 0x4f, 0xea, 0xdc, 0x54, 0x48, 0xdd, 0x60, 0x44, 0xf4, 0x25, 0xb4, 0x3c, 0xa2, 0x9b, 0x5a, + 0x70, 0xe8, 0xd1, 0xd1, 0xe0, 0xf0, 0x82, 0xfa, 0xd7, 0x64, 0x7d, 0xec, 0x89, 0x2e, 0x98, 0x12, + 0x3e, 0xf7, 0xac, 0x80, 0x68, 0x6c, 0x26, 0x17, 0x55, 0x42, 0xde, 0x03, 0x5b, 0x12, 0xda, 0x82, + 0x8a, 0xee, 0x11, 0x47, 0xe7, 0x4a, 0xd8, 0xea, 0x7f, 0xf0, 0xea, 0xe5, 0xcd, 0xa5, 0x81, 0x15, + 0x1c, 0x8e, 0xf6, 0x7b, 0x06, 0x1d, 0x2e, 0x11, 0x3f, 0x18, 0xe9, 0xde, 0x89, 0x70, 0x93, 0x63, + 0x8e, 0xb3, 0xb7, 0xca, 0x44, 0xb1, 0xe8, 0x01, 0xbd, 0x0b, 0x65, 0x93, 0x1a, 0xbe, 0x52, 0xbb, + 0x55, 0x9a, 0x6f, 0xae, 0x34, 0xc5, 0xa9, 0xed, 0xda, 0x96, 0x41, 0xa4, 0x2a, 0x73, 0x36, 0xda, + 0x84, 0x9a, 0xb0, 0x20, 0x5f, 0xa9, 0xdf, 0x2a, 0x5d, 0x60, 0xf6, 0xa1, 0x38, 0xd3, 0xb3, 0xd1, + 0xc8, 0x32, 0x35, 0x57, 0xf7, 0x02, 0x5f, 0x69, 0xf0, 0x61, 0xa5, 0x15, 0x3d, 0x79, 0xb2, 0xb5, + 0xbe, 0xc3, 0xc8, 0x72, 0xe8, 0x06, 0x03, 0x72, 0x02, 0x53, 0x7a, 0x57, 0x37, 0x8e, 0x88, 0xa9, + 0x1d, 0x91, 0x13, 0x05, 0x26, 0x4d, 0xb6, 0x21, 0x40, 0x3f, 0x22, 0x27, 0x5d, 0x13, 0x3a, 0x98, + 0x1a, 0x47, 0xfe, 0x7a, 0x7f, 0x9d, 0xf8, 0x86, 0x67, 0xb9, 0xcc, 0x76, 0x16, 0x01, 0x79, 0x8c, + 0x68, 0xee, 0x6b, 0xc4, 0x39, 0xd6, 0x86, 0x64, 0xe8, 0x06, 0x1e, 0xd7, 0xb0, 0x2a, 0x6e, 0x4b, + 0xce, 0x86, 0x73, 0xfc, 0x90, 0xd3, 0xd1, 0x5b, 0xd0, 0x0a, 0xd1, 0xdc, 0x0b, 0x0b, 0x0f, 0xdd, + 0x94, 0x34, 0xe6, 0x89, 0xbb, 0x3f, 0x2d, 0x42, 0x63, 0x2d, 0xf4, 0xb8, 0xe8, 0x2a, 0xd4, 0x2c, + 0x57, 0xd3, 0x4d, 0x53, 0xf4, 0xd9, 0xc0, 0x55, 0xcb, 0x5d, 0x35, 0x4d, 0x0f, 0x7d, 0x1f, 0xa6, + 0xa4, 0x9b, 0xd6, 0x5c, 0xca, 0xd6, 0x5d, 0xe4, 0x2b, 0xe8, 0x88, 0x15, 0x48, 0x4f, 0xbd, 0x43, + 0xbd, 0x00, 0xb7, 0x9c, 0xb8, 0xe1, 0xa3, 0x5d, 0xe8, 0x0c, 0x75, 0xd7, 0x25, 0xa6, 0x76, 0x48, + 0xfd, 0x40, 0xca, 0x96, 0xb8, 0xec, 0xf7, 0x22, 0x3f, 0x1e, 0x8d, 0xdf, 0x7b, 0xc8, 0xb1, 0x9b, + 0xd4, 0x0f, 0xb8, 0xf8, 0x86, 0x13, 0x78, 0x27, 0xcc, 0xdc, 0x52, 0x54, 0xf4, 0x26, 0xc0, 0xc8, + 0xd7, 0x07, 0x44, 0xf3, 0xf4, 0x80, 0x70, 0xed, 0x2e, 0xe2, 0x06, 0xa7, 0x60, 0x3d, 0x20, 0x6a, + 0x1f, 0x66, 0xf3, 0xfa, 0x41, 0x6d, 0x28, 0xb1, 0xbd, 0x2f, 0x70, 0xdf, 0xc1, 0x1e, 0xd1, 0x2c, + 0x54, 0x8e, 0x75, 0x7b, 0x14, 0x5e, 0x5d, 0xa2, 0x71, 0xaf, 0xf8, 0x51, 0xa1, 0xfb, 0xe7, 0x45, + 0xe8, 0xac, 0x89, 0x2b, 0x5e, 0xde, 0x26, 0x1b, 0x2f, 0x98, 0xef, 0x64, 0x77, 0x9f, 0x66, 0x93, + 0x63, 0x62, 0x4b, 0xb3, 0x9e, 0xee, 0xb1, 0xdb, 0x77, 0x9b, 0x0e, 0x7a, 0xdb, 0x8c, 0x8a, 0xeb, + 0x36, 0x1d, 0xf0, 0x27, 0xb4, 0x15, 0x1f, 0x95, 0x19, 0x1d, 0xa0, 0x34, 0x71, 0x35, 0x5a, 0xfb, + 0xd8, 0x11, 0xe3, 0x8e, 0x94, 0x4a, 0x9c, 0xfa, 0x16, 0xb4, 0xfc, 0x40, 0xf7, 0x02, 0xcd, 0xa0, + 0xc3, 0xa1, 0x15, 0x70, 0xab, 0x6f, 0xae, 0xfc, 0x46, 0xbc, 0x81, 0xd9, 0x99, 0x32, 0x17, 0xe3, + 0x05, 0x6b, 0x1c, 0x8d, 0x9b, 0x7e, 0xdc, 0x50, 0x31, 0x34, 0x13, 0x3c, 0xb4, 0x06, 0x48, 0x76, + 0xa2, 0x19, 0x87, 0xc4, 0x38, 0x72, 0xa9, 0xe5, 0x04, 0x7c, 0x69, 0xcc, 0x79, 0x46, 0x1e, 0x6b, + 0x2d, 0xe2, 0xe1, 0x8e, 0xc4, 0xc7, 0xa4, 0xee, 0xff, 0x94, 0x01, 0x45, 0x53, 0x10, 0xee, 0x8f, + 0xed, 0xd6, 0x32, 0x34, 0xa2, 0xbb, 0x5c, 0x76, 0x89, 0xc6, 0xcf, 0x1c, 0xc7, 0x20, 0x74, 0x0f, + 0xaa, 0xd4, 0x25, 0x0e, 0x31, 0xe5, 0x36, 0x75, 0xc7, 0x57, 0x18, 0x75, 0xdf, 0x7b, 0xcc, 0x91, + 0x58, 0x4a, 0xa0, 0x2f, 0xa0, 0x2e, 0x63, 0x32, 0x53, 0xee, 0xcf, 0x3b, 0xa7, 0x49, 0x4b, 0x92, + 0x89, 0x23, 0x29, 0x74, 0x1f, 0x20, 0xb1, 0x07, 0xe5, 0x49, 0x7b, 0x9c, 0xe8, 0x23, 0xde, 0x95, + 0x84, 0xa4, 0xfa, 0x10, 0xaa, 0x62, 0x6e, 0xdf, 0xca, 0xee, 0xaa, 0x4f, 0xa1, 0x1e, 0x4e, 0x96, + 0x69, 0xfe, 0x11, 0x39, 0xd1, 0x84, 0x93, 0xe0, 0x1d, 0xb5, 0x70, 0xe3, 0x88, 0x9c, 0xec, 0x70, + 0x02, 0x0b, 0xab, 0x98, 0x57, 0xb2, 0xd8, 0xa5, 0xe4, 0x87, 0xa8, 0x22, 0x47, 0xb5, 0x63, 0x86, + 0x00, 0xab, 0xcf, 0x01, 0xe2, 0x51, 0xd0, 0x2d, 0xa8, 0xb0, 0xeb, 0xc8, 0x97, 0xb3, 0x03, 0xae, + 0xd6, 0xec, 0xa2, 0xf2, 0xb1, 0x60, 0xa0, 0x1f, 0x42, 0xd3, 0xa5, 0xb6, 0xad, 0x79, 0xc4, 0x1f, + 0xd9, 0x01, 0xef, 0x76, 0xfa, 0xf4, 0xfd, 0xd9, 0xa1, 0xb6, 0x8d, 0x39, 0x1a, 0x83, 0x1b, 0x3d, + 0x77, 0x1f, 0x01, 0xc4, 0x1c, 0xd4, 0x84, 0xda, 0xd6, 0xa3, 0xa7, 0xab, 0xdb, 0x5b, 0xeb, 0xed, + 0x4b, 0xa8, 0x01, 0x15, 0xbc, 0xb1, 0xba, 0xfe, 0xdb, 0xed, 0x02, 0x9a, 0x82, 0xc6, 0xa3, 0xc7, + 0x7b, 0x9a, 0x68, 0x16, 0x51, 0x0b, 0xea, 0x6b, 0x8f, 0x1f, 0x6f, 0x6b, 0x8f, 0xef, 0xdf, 0x6f, + 0x97, 0x98, 0x10, 0xde, 0xd8, 0xdd, 0x5b, 0xc5, 0x7b, 0xed, 0x72, 0xf7, 0xdf, 0x0a, 0xd0, 0x5e, + 0xe7, 0xb1, 0xf6, 0x77, 0xc0, 0x54, 0x57, 0xa0, 0xcc, 0x14, 0x52, 0xaa, 0xe0, 0x8d, 0x48, 0x38, + 0x3b, 0x41, 0xae, 0xbe, 0x98, 0x63, 0xd5, 0x45, 0x28, 0xb3, 0x16, 0x7a, 0x07, 0xa6, 0xfd, 0xaf, + 0x6c, 0x76, 0xcb, 0x1e, 0x1f, 0xf8, 0xda, 0xc8, 0xb3, 0xa4, 0x13, 0x6e, 0x09, 0xea, 0xd3, 0x03, + 0xff, 0x89, 0x67, 0x75, 0xff, 0xa3, 0x04, 0x9d, 0xb0, 0xb7, 0x6f, 0x62, 0x6c, 0x1f, 0x67, 0x8c, + 0xed, 0xad, 0xb1, 0xb9, 0x4e, 0xb4, 0xb5, 0x3e, 0x34, 0xdc, 0xd1, 0xbe, 0x6d, 0xf9, 0x87, 0x39, + 0xc6, 0x36, 0x2e, 0xbd, 0x13, 0x62, 0x71, 0x2c, 0x86, 0x3e, 0x81, 0xda, 0x81, 0x3d, 0xe2, 0x3d, + 0x94, 0x33, 0xc6, 0x3e, 0xde, 0xc3, 0x7d, 0x81, 0xc4, 0xa1, 0xc8, 0xb7, 0x6d, 0x63, 0x01, 0x34, + 0xa2, 0x49, 0xb2, 0xa4, 0x66, 0xa8, 0xbf, 0xd0, 0x0c, 0x9b, 0x1a, 0x47, 0xf2, 0x6a, 0xad, 0x0f, + 0xf5, 0x17, 0x6b, 0xac, 0x9d, 0xb1, 0xc0, 0xe2, 0x99, 0x2c, 0xb0, 0x34, 0xc1, 0x02, 0x17, 0xa0, + 0x26, 0x17, 0xf6, 0x7a, 0xf3, 0xeb, 0xfe, 0x61, 0x01, 0xae, 0xc4, 0xc1, 0xe8, 0x77, 0x40, 0xd5, + 0xbb, 0x3f, 0x2f, 0xc0, 0x5c, 0x6a, 0x46, 0xdf, 0x44, 0x1b, 0x57, 0x63, 0x75, 0x10, 0x93, 0x89, + 0xc3, 0x83, 0xfc, 0x31, 0xc6, 0x75, 0xe2, 0x5c, 0xdb, 0xf9, 0xf3, 0x32, 0x4c, 0xaf, 0xd1, 0xe1, + 0xbe, 0xe5, 0x44, 0xe9, 0xe2, 0xb2, 0x34, 0x5d, 0x21, 0xf3, 0x46, 0x62, 0xbe, 0x49, 0x58, 0xc2, + 0x70, 0xd1, 0x1d, 0x28, 0xe9, 0x66, 0x38, 0xe1, 0xeb, 0x93, 0x04, 0x56, 0x4d, 0x13, 0x33, 0x9c, + 0xfa, 0x0f, 0x45, 0x69, 0xe8, 0x5f, 0x40, 0x7d, 0xdf, 0x72, 0x4c, 0xcb, 0x19, 0xb0, 0x19, 0x96, + 0xd2, 0x77, 0xd5, 0xf8, 0x68, 0xbd, 0xbe, 0x00, 0xe3, 0x48, 0x4a, 0xfd, 0xfd, 0x22, 0xd4, 0x24, + 0x15, 0x21, 0x28, 0x1f, 0x8c, 0x6c, 0x71, 0xf4, 0x75, 0xcc, 0x9f, 0xc3, 0x58, 0x87, 0x45, 0x69, + 0x0d, 0x11, 0xeb, 0x7c, 0x04, 0x4d, 0xd7, 0xa3, 0xcf, 0x44, 0x1a, 0x14, 0xc6, 0x60, 0x6d, 0x11, + 0xbf, 0xed, 0x44, 0x0c, 0x19, 0x86, 0x26, 0xa1, 0xe8, 0x53, 0x68, 0xfa, 0xc6, 0x21, 0x19, 0xea, + 0xda, 0x33, 0x9f, 0x3a, 0xdc, 0x5a, 0x5b, 0xfd, 0x37, 0x5e, 0xbd, 0xbc, 0xa9, 0x10, 0xc7, 0xa0, + 0x6c, 0x0a, 0x4b, 0x8c, 0xd1, 0xc3, 0xfa, 0xf3, 0x87, 0xc4, 0xe7, 0x61, 0x18, 0x08, 0x81, 0x07, + 0x3e, 0x75, 0x50, 0x0f, 0xc0, 0x27, 0x9e, 0xe6, 0x52, 0xdb, 0x32, 0x4e, 0x78, 0xea, 0x10, 0xc5, + 0xcb, 0xbb, 0xc4, 0xdb, 0xe1, 0x64, 0xdc, 0xf0, 0xc3, 0x47, 0x5e, 0x36, 0xe0, 0xf1, 0x75, 0xe0, + 0xf1, 0xf4, 0xa0, 0x81, 0x6b, 0x3c, 0x8c, 0x0e, 0x3c, 0x96, 0x85, 0xf3, 0x10, 0x4d, 0x44, 0xfb, + 0x0d, 0x2c, 0x5b, 0xaa, 0x03, 0xa5, 0x55, 0xd3, 0x44, 0x0a, 0xd4, 0xe4, 0x06, 0xc9, 0x20, 0x2f, + 0x6c, 0xa2, 0x1f, 0x40, 0xdd, 0xa4, 0x86, 0x98, 0x7f, 0xf1, 0x0c, 0xf3, 0xaf, 0x99, 0xd4, 0xe0, + 0x93, 0x9f, 0x85, 0xca, 0x81, 0x47, 0x1d, 0x11, 0x72, 0xd5, 0xb1, 0x68, 0x74, 0xff, 0xb1, 0x00, + 0x33, 0xd1, 0x39, 0xc9, 0x7c, 0x6f, 0xf2, 0xe0, 0x0a, 0xd4, 0x4c, 0x62, 0x93, 0x40, 0xaa, 0x76, + 0x1d, 0x87, 0xcd, 0xd4, 0xb4, 0x4a, 0x17, 0x9a, 0x56, 0x39, 0x31, 0xad, 0x8c, 0x6f, 0xaa, 0x64, + 0x7d, 0xd3, 0xdb, 0x30, 0x25, 0xf6, 0x2b, 0x44, 0xf0, 0xe4, 0x0b, 0xb7, 0x04, 0x51, 0x80, 0xba, + 0x57, 0xe1, 0xca, 0x1a, 0x75, 0x1c, 0x62, 0x04, 0xd4, 0xdb, 0xf1, 0xe8, 0x8b, 0x13, 0xa9, 0x88, + 0xdd, 0x3f, 0x2e, 0xc0, 0x5c, 0x96, 0x23, 0x97, 0xfe, 0x00, 0x6a, 0x2c, 0x65, 0x20, 0xbe, 0x2f, + 0xeb, 0x2c, 0xcb, 0xaf, 0x5e, 0xde, 0x5c, 0x3c, 0x4b, 0x6e, 0xb5, 0xe1, 0x98, 0xc2, 0x27, 0x87, + 0x1d, 0xb0, 0xd3, 0x77, 0x59, 0xe7, 0x9a, 0x65, 0xca, 0xa8, 0xbc, 0xc6, 0xdb, 0x5b, 0x26, 0x52, + 0xa1, 0x64, 0xd3, 0x81, 0xbc, 0x6f, 0xea, 0xa1, 0x87, 0xc3, 0x8c, 0xd8, 0xfd, 0xcb, 0x12, 0x94, + 0x1f, 0x50, 0xcb, 0x41, 0xb7, 0xa1, 0x43, 0x02, 0xc3, 0xd4, 0x86, 0xd4, 0xd4, 0x3c, 0x72, 0x6c, + 0xf9, 0x2c, 0xa3, 0x67, 0xb3, 0x2a, 0xe1, 0x19, 0xc6, 0x78, 0x48, 0x4d, 0x2c, 0xc9, 0x68, 0x01, + 0xaa, 0xfe, 0xa1, 0xee, 0x99, 0x61, 0x36, 0x73, 0x39, 0x32, 0x42, 0xd6, 0x95, 0x28, 0x5e, 0x60, + 0x09, 0x41, 0x37, 0xa1, 0xc9, 0x9f, 0x64, 0x05, 0xa2, 0xc4, 0xcf, 0x18, 0x38, 0x49, 0xd4, 0x1f, + 0x16, 0xa0, 0x13, 0x16, 0x29, 0x4c, 0xcb, 0xe3, 0xdb, 0x74, 0x12, 0xd6, 0xb4, 0x24, 0x63, 0x3d, + 0xa4, 0xa3, 0xf7, 0x20, 0xa4, 0x69, 0x44, 0xee, 0x01, 0x3f, 0xb0, 0x06, 0x9e, 0x91, 0xf4, 0x70, + 0x6b, 0xd0, 0xf7, 0x60, 0xc6, 0xe6, 0xe9, 0x7f, 0x8c, 0x14, 0x66, 0x31, 0x2d, 0xc8, 0x21, 0x50, + 0xfd, 0x8b, 0x02, 0x54, 0xf8, 0x9c, 0xd1, 0x34, 0x14, 0x2d, 0x53, 0x06, 0x0f, 0x45, 0xcb, 0x44, + 0x3d, 0xa8, 0xdb, 0xfa, 0x3e, 0xb1, 0x99, 0x72, 0x16, 0xa5, 0x37, 0xe6, 0x1e, 0x91, 0xa1, 0xb7, + 0x25, 0x07, 0x47, 0x18, 0xb4, 0x02, 0x35, 0x8f, 0xe8, 0x6c, 0xa6, 0x72, 0xb7, 0x95, 0xb8, 0x24, + 0xb1, 0xe3, 0x51, 0x83, 0xf8, 0xfe, 0xae, 0x4b, 0x8c, 0xde, 0xd6, 0x3a, 0x0e, 0x81, 0x68, 0x19, + 0x66, 0xf9, 0xc6, 0x1b, 0x1e, 0xd1, 0x03, 0x12, 0xef, 0x3d, 0x2f, 0x3e, 0x60, 0xc4, 0x78, 0x6b, + 0x9c, 0x15, 0x6e, 0x7f, 0xf7, 0x2e, 0x54, 0xd9, 0x3e, 0x13, 0x93, 0x1d, 0x1a, 0xbb, 0x71, 0xb9, + 0x7c, 0xf6, 0xd0, 0x86, 0xfa, 0x8b, 0x8d, 0xc0, 0x88, 0x0e, 0xad, 0xfb, 0xd3, 0x02, 0x94, 0xf7, + 0x74, 0xff, 0x88, 0xb9, 0x3d, 0xdf, 0x25, 0x86, 0x8c, 0x82, 0xf9, 0x33, 0xdb, 0x56, 0xd6, 0x51, + 0xe0, 0xe9, 0x8e, 0xaf, 0x47, 0x9e, 0x8e, 0x9d, 0x14, 0xeb, 0x67, 0x2f, 0x41, 0xce, 0x09, 0xb6, + 0xca, 0xe3, 0xc1, 0x16, 0xcb, 0xa0, 0xc3, 0x90, 0xc5, 0x63, 0x2a, 0x29, 0x8c, 0xaa, 0x19, 0xd1, + 0xb6, 0xcc, 0x07, 0xe5, 0x7a, 0xb1, 0x5d, 0xea, 0xfe, 0xac, 0x0a, 0x35, 0x4c, 0x0c, 0x7a, 0xcc, + 0xef, 0xb2, 0xa6, 0x6e, 0x1c, 0x69, 0x96, 0x13, 0x10, 0x27, 0x08, 0x3d, 0xfc, 0xad, 0xf8, 0x72, + 0x15, 0xb0, 0xde, 0xaa, 0x71, 0xb4, 0x25, 0x20, 0x22, 0xcf, 0x05, 0x3d, 0x22, 0xa0, 0x15, 0xb8, + 0x22, 0x72, 0xbd, 0x80, 0x98, 0x2c, 0x12, 0xf1, 0x89, 0x8c, 0x47, 0x8a, 0x3c, 0x1e, 0xb9, 0x1c, + 0x31, 0xd7, 0x18, 0x4f, 0x84, 0x26, 0x5f, 0x00, 0x8a, 0x65, 0xb8, 0x47, 0xb0, 0x48, 0x78, 0x80, + 0x9d, 0x5e, 0x58, 0x04, 0xbe, 0x2f, 0x19, 0xb8, 0x13, 0x81, 0x43, 0x12, 0x5a, 0x84, 0x59, 0x23, + 0x34, 0x71, 0x8d, 0xdd, 0x93, 0x24, 0xe1, 0xf2, 0xf1, 0x74, 0xc4, 0x63, 0x37, 0x29, 0x41, 0x8b, + 0x80, 0x0e, 0xd9, 0x1a, 0xd3, 0x13, 0xac, 0x88, 0x5a, 0x84, 0xe0, 0x24, 0x66, 0x77, 0x0f, 0x66, + 0x24, 0x3a, 0x9a, 0x5a, 0x75, 0xd2, 0xd4, 0xa6, 0x05, 0x32, 0x9a, 0xd7, 0x5b, 0xd0, 0xb2, 0x75, + 0x3f, 0xd0, 0x74, 0xd7, 0xb5, 0x2d, 0x62, 0xf2, 0x3a, 0x64, 0x0b, 0x37, 0x19, 0x6d, 0x55, 0x90, + 0xd0, 0x2a, 0x74, 0x6c, 0x32, 0xd0, 0x8d, 0x93, 0x64, 0x14, 0x58, 0x3f, 0x25, 0x0a, 0x6c, 0x0b, + 0x78, 0x22, 0x05, 0xfa, 0x08, 0x58, 0x98, 0xa7, 0x1d, 0x91, 0x93, 0xb0, 0xac, 0xf3, 0xe6, 0xd8, + 0x99, 0x3d, 0xd4, 0x5f, 0xfc, 0x88, 0x9c, 0xc8, 0x03, 0xab, 0x0d, 0x45, 0x0b, 0xdd, 0x86, 0xcb, + 0x81, 0x67, 0x0d, 0x06, 0xec, 0x9a, 0xd3, 0x3d, 0x7d, 0xe8, 0x8b, 0x6d, 0x03, 0x3e, 0xcd, 0x29, + 0xc9, 0xda, 0xe1, 0x1c, 0xb4, 0x03, 0x6d, 0xa6, 0x82, 0xc7, 0x44, 0xdb, 0xd7, 0x8d, 0xa3, 0x03, + 0xcb, 0xb6, 0x7d, 0xa5, 0xc9, 0x47, 0x7b, 0x37, 0x47, 0x43, 0x18, 0xb0, 0x1f, 0xe2, 0x64, 0x39, + 0x44, 0x4f, 0x53, 0xd5, 0x4f, 0x61, 0x26, 0xa3, 0x4a, 0xc9, 0x52, 0x47, 0x23, 0xa7, 0xd4, 0xd1, + 0x4a, 0x94, 0x3a, 0xd4, 0x7b, 0xd0, 0x4a, 0xae, 0xea, 0x75, 0x65, 0x92, 0x94, 0x6c, 0x1f, 0x66, + 0xf3, 0xe6, 0xf8, 0xba, 0x3e, 0xaa, 0xc9, 0x52, 0xcb, 0x3f, 0xd5, 0xa1, 0xb6, 0x43, 0x3c, 0xdf, + 0xf2, 0x03, 0x74, 0x05, 0xaa, 0x3e, 0xf9, 0x4a, 0x73, 0x28, 0x17, 0x2d, 0xe3, 0x8a, 0x4f, 0xbe, + 0x7a, 0x44, 0x99, 0xa6, 0x89, 0x2b, 0x53, 0x4b, 0xda, 0x95, 0xb8, 0x4c, 0xdb, 0x82, 0x13, 0xef, + 0x40, 0xd6, 0xfc, 0x4a, 0x19, 0xf3, 0x93, 0x63, 0x5d, 0xcc, 0xfc, 0xca, 0x93, 0xcd, 0xef, 0x1e, + 0x5c, 0x93, 0x93, 0xcc, 0xb1, 0xc2, 0x0a, 0x9f, 0xeb, 0x55, 0x01, 0x58, 0x1b, 0x33, 0xbc, 0x7c, + 0xd3, 0xad, 0x9e, 0xc3, 0x74, 0x97, 0x61, 0x2e, 0x36, 0x5d, 0x57, 0x0f, 0x8c, 0x43, 0x22, 0xb5, + 0x50, 0x18, 0x4b, 0x3b, 0xe2, 0xee, 0x08, 0xe6, 0x04, 0xf3, 0xad, 0x4f, 0x30, 0xdf, 0xbb, 0x30, + 0x27, 0x57, 0x97, 0xb5, 0xe2, 0x06, 0x5f, 0xda, 0xac, 0xe0, 0x6e, 0xa6, 0x0d, 0x37, 0xc7, 0xe8, + 0xe1, 0xa2, 0x46, 0xdf, 0x1c, 0x37, 0xfa, 0x8f, 0x40, 0x91, 0x93, 0x1a, 0xb7, 0xfd, 0x16, 0x9f, + 0x96, 0x9c, 0xf4, 0x76, 0xd6, 0xd6, 0x73, 0xdd, 0xc5, 0xd4, 0x85, 0xdd, 0xc5, 0x74, 0xc6, 0x5d, + 0x84, 0x3a, 0x96, 0xef, 0x2e, 0x56, 0xe0, 0x8a, 0x9c, 0x76, 0xda, 0x6b, 0x28, 0x33, 0x7c, 0xce, + 0x97, 0x05, 0x73, 0x2f, 0xe5, 0x36, 0x26, 0xb8, 0x98, 0x76, 0x9e, 0x8b, 0xe1, 0x2f, 0xab, 0x7c, + 0x43, 0x77, 0x94, 0x4e, 0xf8, 0xb2, 0x8a, 0xb5, 0xd0, 0x5d, 0xa8, 0xec, 0x93, 0x81, 0xe5, 0x28, + 0x28, 0x93, 0xe1, 0xa4, 0x6d, 0xb8, 0xcf, 0x30, 0x9b, 0x97, 0xb0, 0x00, 0xa3, 0x05, 0x68, 0x1b, + 0x74, 0xe8, 0xf2, 0xf9, 0x86, 0x11, 0xee, 0x65, 0x66, 0xd8, 0x9b, 0x97, 0xf0, 0x4c, 0xc8, 0x91, + 0xb9, 0xc8, 0xff, 0xa3, 0x2f, 0xea, 0x2b, 0x30, 0x97, 0x71, 0xac, 0x9a, 0x71, 0xa8, 0x3b, 0x03, + 0xd2, 0xc5, 0x70, 0x39, 0x67, 0x85, 0xa7, 0x44, 0xec, 0x6f, 0x41, 0x2b, 0xf0, 0x46, 0x8e, 0xa1, + 0x33, 0xcd, 0xd5, 0x03, 0xe9, 0xb3, 0x9a, 0x11, 0x6d, 0x35, 0xe8, 0x76, 0xa1, 0x21, 0x0f, 0x99, + 0x98, 0x13, 0xdc, 0x56, 0xf7, 0x4f, 0x0b, 0x50, 0x61, 0xaa, 0x7a, 0x92, 0x1b, 0xab, 0x28, 0x50, + 0x3b, 0x66, 0x3d, 0xc8, 0x94, 0xa4, 0x81, 0xc3, 0x26, 0xba, 0x0e, 0x0d, 0xae, 0xf9, 0x5c, 0x44, + 0xdc, 0xbd, 0x75, 0x46, 0x60, 0x31, 0x57, 0x64, 0x16, 0xa1, 0xac, 0x88, 0x1a, 0xb9, 0x59, 0x3c, + 0x95, 0xf2, 0xcb, 0x13, 0xae, 0x71, 0x11, 0xef, 0xa3, 0xf4, 0x35, 0xce, 0xf2, 0x89, 0xee, 0x33, + 0xa8, 0x85, 0x36, 0x75, 0x07, 0x90, 0x08, 0x91, 0xa2, 0xfa, 0x40, 0x18, 0x8c, 0x35, 0x70, 0x47, + 0x70, 0xd6, 0x63, 0xc6, 0x29, 0x7e, 0xa7, 0x98, 0xef, 0x77, 0xba, 0xbf, 0x2c, 0xc8, 0x2c, 0xf8, + 0x7c, 0x9b, 0xf2, 0x6e, 0xf8, 0xde, 0xb2, 0x94, 0xfb, 0xde, 0x32, 0x7c, 0x63, 0xf9, 0xf6, 0xa9, + 0x21, 0x0c, 0x4f, 0xfe, 0x09, 0xfa, 0x30, 0x61, 0xba, 0x15, 0x6e, 0xba, 0x71, 0xe9, 0x83, 0x27, + 0xdc, 0xb9, 0x76, 0xfb, 0x4d, 0xb4, 0xb3, 0x0b, 0x50, 0xe7, 0xde, 0xf4, 0x11, 0x7d, 0xde, 0xad, + 0x42, 0x79, 0x37, 0xa0, 0x6e, 0xb7, 0x01, 0x35, 0xf6, 0xd7, 0x25, 0x66, 0xf7, 0xb7, 0xa0, 0xb9, + 0x4b, 0x7c, 0xb6, 0xd0, 0x6d, 0x4a, 0xdd, 0x09, 0x55, 0x9a, 0xc2, 0x45, 0xaa, 0x34, 0x7f, 0x54, + 0x85, 0x9a, 0xac, 0xcd, 0xa2, 0xf7, 0x12, 0x3b, 0xde, 0x5c, 0xb9, 0xd2, 0x0b, 0x3f, 0x62, 0x08, + 0x8b, 0x0d, 0x7c, 0x23, 0xc5, 0x41, 0xfc, 0x26, 0x4c, 0xb1, 0xbf, 0x9a, 0x27, 0x93, 0x3c, 0x99, + 0x37, 0xcc, 0x25, 0x64, 0x04, 0x43, 0x08, 0xb5, 0x18, 0x38, 0x4a, 0x08, 0x3f, 0x84, 0xba, 0x69, + 0xf9, 0x3c, 0x86, 0x91, 0xc7, 0x75, 0x6d, 0x6c, 0xac, 0x75, 0x09, 0xc0, 0x11, 0x14, 0x7d, 0x02, + 0x10, 0x3e, 0x47, 0x55, 0xc1, 0x37, 0xc6, 0x07, 0x5c, 0x8f, 0x30, 0x38, 0x81, 0x67, 0x83, 0x1e, + 0xeb, 0xb6, 0x65, 0xea, 0x01, 0x91, 0x55, 0x86, 0xf1, 0x41, 0x9f, 0x4a, 0x00, 0x8e, 0xa0, 0xe8, + 0x63, 0x68, 0x84, 0xcf, 0xa6, 0xbc, 0x71, 0xaf, 0x8f, 0x8f, 0x19, 0x0a, 0x9a, 0x38, 0x46, 0xa7, + 0x0b, 0x6f, 0x8d, 0xd7, 0x14, 0xde, 0x7e, 0x00, 0x2d, 0x5f, 0x9c, 0xb0, 0x66, 0x53, 0xea, 0x2a, + 0xb3, 0xf2, 0xb2, 0x09, 0x0f, 0x33, 0x71, 0xfc, 0xb8, 0xe9, 0x27, 0x74, 0xe1, 0x2d, 0x28, 0x3f, + 0xa3, 0x96, 0xa3, 0x5c, 0xe1, 0x02, 0x53, 0xa9, 0x1c, 0x15, 0x73, 0x16, 0xfa, 0x1e, 0x54, 0x9f, + 0xf1, 0x4c, 0x4a, 0x99, 0x93, 0xc6, 0x91, 0x04, 0x11, 0x13, 0x4b, 0x36, 0xeb, 0x2b, 0xd0, 0xfd, + 0x23, 0xe5, 0x6a, 0xa6, 0x2f, 0x96, 0x50, 0x61, 0xce, 0x42, 0x4b, 0x51, 0x59, 0x58, 0xe1, 0xa0, + 0xab, 0xd9, 0x0a, 0x7f, 0xb6, 0x18, 0xdc, 0x83, 0x86, 0x08, 0x20, 0x1c, 0xfa, 0x5c, 0xb9, 0x26, + 0x6f, 0xf7, 0x48, 0x46, 0xea, 0x3c, 0xae, 0x1b, 0xf2, 0x89, 0xcd, 0xc1, 0x0f, 0xa8, 0xab, 0xa8, + 0x99, 0x39, 0x30, 0x53, 0xc0, 0x9c, 0x85, 0x6e, 0x43, 0xcd, 0x17, 0x86, 0xa1, 0x5c, 0x97, 0xaf, + 0xc4, 0x93, 0x28, 0x97, 0x98, 0x38, 0x04, 0xa8, 0xf7, 0xa2, 0x4a, 0xf0, 0xb9, 0x8b, 0x8e, 0xdd, + 0x3f, 0xb8, 0x06, 0xcd, 0x44, 0x75, 0x11, 0xdd, 0x49, 0xd9, 0xc7, 0xb5, 0x5e, 0xf2, 0xe3, 0x9b, + 0x1c, 0x1b, 0xf9, 0x3c, 0xdf, 0x46, 0xd4, 0x8c, 0xdc, 0x64, 0x3b, 0xf9, 0x38, 0xa1, 0xb2, 0xc2, + 0x4e, 0xde, 0xcc, 0x1d, 0x33, 0x47, 0x6d, 0x3f, 0x4d, 0xaa, 0xad, 0x30, 0x95, 0x9b, 0xf9, 0xe3, + 0xbe, 0x5e, 0x75, 0x2b, 0xbf, 0x22, 0xaa, 0x7b, 0x1b, 0x6a, 0x9e, 0xc8, 0x9c, 0xa4, 0xee, 0xb6, + 0xb3, 0x19, 0x15, 0x0e, 0x01, 0xe8, 0x1d, 0xa8, 0xb0, 0xc0, 0xf2, 0x44, 0x6a, 0x6c, 0xfc, 0x51, + 0x11, 0xbf, 0xb0, 0xb1, 0x60, 0xb2, 0x1e, 0xc3, 0xf0, 0x53, 0xcd, 0xf4, 0x28, 0xef, 0x4b, 0x1c, + 0x02, 0xd8, 0x04, 0x79, 0xf9, 0xf8, 0x7a, 0x66, 0x82, 0x89, 0x7a, 0xf1, 0x07, 0x91, 0x6d, 0xbd, + 0x91, 0x29, 0x19, 0x27, 0xb4, 0x30, 0x6b, 0x5f, 0x77, 0xa0, 0x6c, 0x53, 0xdd, 0x54, 0xe6, 0xa5, + 0x52, 0xe6, 0x89, 0x6c, 0x53, 0xdd, 0xc4, 0x1c, 0xc6, 0xc6, 0x60, 0x7f, 0x89, 0xa9, 0xbc, 0x77, + 0xca, 0x18, 0xdb, 0x1c, 0x82, 0x25, 0x14, 0x2d, 0x43, 0x85, 0x57, 0xd1, 0x95, 0xdb, 0x99, 0x2b, + 0x26, 0x29, 0xc3, 0x8b, 0xeb, 0x58, 0x00, 0xd1, 0xf7, 0xe3, 0x7a, 0xfd, 0x42, 0x26, 0x9a, 0x1c, + 0x93, 0x49, 0x14, 0xe9, 0xd9, 0x48, 0x7e, 0x40, 0x3d, 0xa2, 0x2c, 0x9e, 0x32, 0xd2, 0x2e, 0x43, + 0x60, 0x01, 0x64, 0x0b, 0xe2, 0x0f, 0xa6, 0x72, 0xe7, 0x94, 0x05, 0x71, 0x11, 0x13, 0x4b, 0x28, + 0x5a, 0xcb, 0xbc, 0x31, 0xef, 0x71, 0xd1, 0x5b, 0x13, 0x44, 0xf3, 0xdf, 0x95, 0xa3, 0x2d, 0x98, + 0xe6, 0x4d, 0x96, 0x22, 0x89, 0x6e, 0x96, 0x32, 0x6f, 0xaa, 0xc6, 0xba, 0x21, 0xa6, 0xec, 0x68, + 0xca, 0x4f, 0x36, 0x51, 0x9f, 0xe7, 0xa4, 0x0e, 0x7d, 0x6e, 0x13, 0x73, 0x40, 0x94, 0xe5, 0x53, + 0xa6, 0xb3, 0x1a, 0xe3, 0x70, 0x52, 0x08, 0x6d, 0x40, 0x2b, 0xd1, 0x34, 0x95, 0xf7, 0x33, 0xaf, + 0xed, 0x26, 0x74, 0x62, 0xe2, 0x94, 0x18, 0xd3, 0x69, 0x57, 0x44, 0xae, 0xca, 0x4a, 0x46, 0xa7, + 0x65, 0x44, 0x8b, 0x43, 0x00, 0x73, 0xa9, 0x6e, 0x18, 0xe5, 0x2a, 0x1f, 0x64, 0x5c, 0x6a, 0x14, + 0xff, 0xe2, 0x18, 0x94, 0xbe, 0x0d, 0xee, 0x9e, 0xfd, 0x36, 0xf8, 0xe4, 0x4c, 0xb7, 0xc1, 0xa7, + 0xaf, 0xbb, 0x0d, 0x7e, 0xb7, 0x70, 0xf1, 0xeb, 0x00, 0xfd, 0x30, 0x19, 0x3b, 0x26, 0xf2, 0xc2, + 0xe2, 0x29, 0x79, 0xe1, 0xe5, 0x48, 0x22, 0xf1, 0x3a, 0xf1, 0x43, 0x28, 0x33, 0x03, 0x43, 0x77, + 0xa0, 0x1e, 0xe5, 0xbd, 0x85, 0x49, 0x79, 0x6f, 0x04, 0x51, 0x7f, 0x59, 0x84, 0xaa, 0x30, 0x4c, + 0xf4, 0xf9, 0xd8, 0x1b, 0xa2, 0xb7, 0x4f, 0xb1, 0xe3, 0xf1, 0x17, 0x44, 0x22, 0x07, 0xe0, 0x6f, + 0x28, 0x3c, 0x4d, 0x7c, 0x2c, 0xb3, 0x7f, 0x12, 0x10, 0x51, 0x34, 0x29, 0xb3, 0x1c, 0x40, 0xf0, + 0x9e, 0x30, 0x56, 0x9f, 0x71, 0xd4, 0xff, 0x2c, 0xc4, 0xaf, 0x94, 0x66, 0xa1, 0x22, 0xca, 0xdc, + 0x22, 0xb6, 0x15, 0x0d, 0x34, 0x0f, 0xed, 0xa1, 0xe5, 0x68, 0x3e, 0x1d, 0x79, 0x46, 0xba, 0x1e, + 0x39, 0x3d, 0xb4, 0x9c, 0x5d, 0x4e, 0x16, 0xd5, 0x82, 0x79, 0x51, 0x87, 0x4d, 0x21, 0x4b, 0x12, + 0xa9, 0xbf, 0x48, 0x22, 0x17, 0x01, 0x09, 0x94, 0xa9, 0x99, 0xd4, 0xf0, 0xb5, 0x80, 0x06, 0xba, + 0xcd, 0x2f, 0xb4, 0x32, 0x6e, 0x4b, 0xce, 0x3a, 0x35, 0xfc, 0x3d, 0x46, 0x47, 0x3d, 0xb8, 0x1c, + 0xa2, 0xf9, 0x72, 0x24, 0xbc, 0xc2, 0xe1, 0x1d, 0xc9, 0xe2, 0xcb, 0x11, 0xf8, 0x2e, 0x4c, 0xc9, + 0x40, 0x5f, 0x33, 0x89, 0x1d, 0xc8, 0xef, 0xcd, 0x70, 0x53, 0x44, 0xf4, 0xeb, 0x8c, 0xa4, 0xfe, + 0x7b, 0x11, 0x2a, 0xdc, 0x4d, 0x9d, 0x92, 0xcb, 0x14, 0x26, 0xd4, 0x50, 0xbe, 0x84, 0x99, 0x28, + 0xd9, 0xe4, 0xd9, 0x72, 0xf8, 0x2a, 0x61, 0x7e, 0xb2, 0x37, 0xec, 0xa5, 0xb2, 0x50, 0x3c, 0xbd, + 0x9f, 0x6c, 0xfa, 0xe8, 0xc7, 0x80, 0xe2, 0xfc, 0x55, 0x66, 0xd7, 0x61, 0x11, 0x6b, 0xe1, 0x0c, + 0xbd, 0xae, 0x49, 0x19, 0xdc, 0xd9, 0xcf, 0x50, 0x7c, 0xf5, 0x73, 0x98, 0x3a, 0x6b, 0x0a, 0x3c, + 0x0b, 0x95, 0xe4, 0x01, 0x8b, 0x86, 0xda, 0x87, 0x76, 0x76, 0x9c, 0x73, 0xf7, 0xf1, 0x5f, 0x85, + 0xf8, 0x3d, 0xed, 0x69, 0x2f, 0x42, 0x73, 0xae, 0x91, 0x5c, 0x3d, 0x3f, 0x67, 0xfe, 0xa9, 0x9e, + 0xbc, 0x4e, 0xcd, 0x6f, 0x43, 0x47, 0x5c, 0x8b, 0x49, 0x8d, 0x14, 0x76, 0x33, 0x23, 0x18, 0xb1, + 0x42, 0x2e, 0x02, 0x92, 0xd8, 0xa4, 0x3e, 0x96, 0x84, 0xfa, 0x0a, 0x4e, 0xac, 0x8e, 0x6a, 0x0d, + 0x2a, 0xfc, 0x9e, 0x52, 0xff, 0xba, 0x00, 0x55, 0x71, 0x63, 0x9d, 0xd9, 0xd2, 0x05, 0x3c, 0xe7, + 0x55, 0xf0, 0x59, 0xd6, 0x23, 0x6e, 0xc5, 0x9c, 0xf5, 0x08, 0x46, 0x6a, 0x3d, 0x12, 0x9b, 0xb3, + 0x1e, 0xc1, 0x49, 0xac, 0xe7, 0xf7, 0x0a, 0xe9, 0xaf, 0xc9, 0xce, 0x6f, 0x40, 0xdf, 0x9a, 0xcb, + 0x5d, 0x85, 0xa9, 0xd4, 0x05, 0x7c, 0xfe, 0xb9, 0xa8, 0x9f, 0x43, 0x33, 0x71, 0x6d, 0x5e, 0xa0, + 0x83, 0x2f, 0xa0, 0x95, 0xbc, 0x77, 0xcf, 0xdf, 0x43, 0xf7, 0x67, 0x08, 0xaa, 0xe2, 0xeb, 0x17, + 0x34, 0x9f, 0xca, 0x45, 0x66, 0x7b, 0xf2, 0xd7, 0x04, 0x39, 0x69, 0xc8, 0xbd, 0xfc, 0x34, 0xe4, + 0x4a, 0x2c, 0x32, 0x39, 0x03, 0xb9, 0x3b, 0x96, 0x81, 0x28, 0xd9, 0x91, 0x72, 0x92, 0x8f, 0x8f, + 0xc6, 0x93, 0x0f, 0x75, 0x6c, 0xb4, 0x5f, 0xe7, 0x1d, 0x79, 0x79, 0xc7, 0x19, 0xb2, 0x84, 0x5e, + 0x26, 0x4b, 0x98, 0xcb, 0x7c, 0x18, 0x95, 0x4d, 0x10, 0xe6, 0x53, 0x09, 0xc2, 0x6c, 0x16, 0x9d, + 0xc8, 0x0d, 0x7a, 0x99, 0xdc, 0x60, 0x2e, 0x0f, 0x9b, 0x48, 0x0b, 0x16, 0xd2, 0x69, 0xc1, 0x95, + 0x2c, 0x3c, 0x95, 0x11, 0xbc, 0x9f, 0xcd, 0x08, 0xae, 0xe6, 0xc2, 0x93, 0xc9, 0xc0, 0x42, 0x3a, + 0x19, 0x18, 0xeb, 0x3f, 0x95, 0x07, 0xf4, 0x32, 0x79, 0xc0, 0x5c, 0x2e, 0x3a, 0x4e, 0x01, 0x3e, + 0xcb, 0x4d, 0x01, 0xae, 0x8f, 0x4b, 0x4d, 0x88, 0xfe, 0xd7, 0x27, 0x44, 0xff, 0x6f, 0xe6, 0xf6, + 0x30, 0x29, 0xf0, 0xff, 0x75, 0xb4, 0xfd, 0x5d, 0x8d, 0xb6, 0xff, 0x26, 0x8e, 0xb6, 0xef, 0x8d, + 0xdd, 0xc1, 0x37, 0xf2, 0x2d, 0xe3, 0x5b, 0x09, 0xb4, 0xff, 0xee, 0x57, 0x2f, 0xd0, 0x56, 0x3f, + 0xbe, 0x70, 0x0c, 0xad, 0x3e, 0x8e, 0xc3, 0xc1, 0xf3, 0xc7, 0x0f, 0x08, 0xca, 0x43, 0xe6, 0x40, + 0xc4, 0xbb, 0x60, 0xfe, 0x1c, 0x47, 0x59, 0xbf, 0x53, 0x8a, 0xa2, 0xac, 0x65, 0x98, 0x8d, 0x3e, + 0x3d, 0x4d, 0xae, 0x5f, 0xbc, 0xaf, 0x41, 0x11, 0x2f, 0xde, 0x81, 0x15, 0xb8, 0x12, 0x4b, 0x24, + 0xf7, 0x40, 0x1c, 0xec, 0xe5, 0x88, 0x99, 0x48, 0x37, 0x16, 0x01, 0x99, 0x1e, 0xd3, 0xee, 0xd4, + 0x18, 0x32, 0x7a, 0x92, 0x9c, 0xd4, 0x1e, 0x87, 0xe8, 0x64, 0xff, 0xe2, 0x48, 0x3a, 0x92, 0x95, + 0xe8, 0xfd, 0x4b, 0x68, 0xc7, 0xdf, 0xa2, 0x48, 0x97, 0x54, 0xc9, 0x7c, 0xa5, 0x9e, 0x72, 0x85, + 0xd1, 0x87, 0xb7, 0x9e, 0xf4, 0x4d, 0x33, 0x6e, 0x9a, 0xa0, 0x6a, 0x30, 0x93, 0xc1, 0x20, 0x95, + 0x7f, 0x80, 0x65, 0x8e, 0x0c, 0x69, 0x45, 0x2d, 0x1c, 0xb5, 0xf3, 0x03, 0x7a, 0x26, 0x21, 0x7f, + 0x26, 0x27, 0xf2, 0x94, 0x06, 0x8e, 0xda, 0xea, 0xd3, 0x74, 0x80, 0x38, 0xc9, 0xe6, 0x0b, 0xe7, + 0xb5, 0xf9, 0x99, 0x4c, 0xb8, 0xd7, 0xbd, 0x03, 0xd3, 0xbb, 0x27, 0x8e, 0xc1, 0xdc, 0x9d, 0xfc, + 0x9c, 0x33, 0xf5, 0xdb, 0xc4, 0x42, 0xfa, 0xb7, 0x89, 0xdd, 0x7f, 0x2d, 0xc0, 0x4c, 0x84, 0x97, + 0xb1, 0xd0, 0x32, 0x94, 0x74, 0xf9, 0xc5, 0x6f, 0xb2, 0x9c, 0x95, 0x81, 0xf5, 0x56, 0x8d, 0xa3, + 0xcd, 0x4b, 0x98, 0x41, 0x51, 0x1f, 0x1a, 0x87, 0x44, 0xf7, 0x82, 0x7d, 0xa2, 0x07, 0x63, 0x3f, + 0x59, 0xc8, 0xca, 0x6d, 0x86, 0xc8, 0xcd, 0x4b, 0x38, 0x16, 0x43, 0x1f, 0x40, 0xd9, 0xa4, 0x4e, + 0x5c, 0xff, 0x9d, 0x24, 0xbe, 0x4e, 0x1d, 0xb2, 0xc9, 0x7f, 0xcd, 0xe4, 0x10, 0xb5, 0x02, 0xa5, + 0x55, 0xe3, 0x48, 0x6d, 0x42, 0x23, 0xea, 0x55, 0xad, 0x42, 0x99, 0x61, 0xfa, 0x00, 0xf5, 0x30, + 0x12, 0xbc, 0xbd, 0x00, 0x15, 0xfe, 0x23, 0x49, 0x04, 0x50, 0xdd, 0x79, 0xd2, 0xdf, 0xde, 0x5a, + 0x6b, 0x5f, 0x42, 0x4d, 0xa8, 0xed, 0xe0, 0xad, 0xa7, 0xab, 0x7b, 0x1b, 0xed, 0x02, 0x6a, 0x40, + 0x65, 0xfb, 0xf1, 0xda, 0xea, 0x76, 0xbb, 0xb8, 0xf2, 0x00, 0xea, 0xf2, 0x47, 0x6c, 0x1e, 0xfa, + 0x0c, 0x6a, 0xf2, 0x19, 0xc5, 0xd7, 0x78, 0xfa, 0xe7, 0x95, 0xaa, 0x32, 0xce, 0x10, 0xc3, 0x2e, + 0x17, 0x56, 0xb6, 0xa1, 0x2e, 0x3f, 0x90, 0xf4, 0xd0, 0x17, 0x50, 0x93, 0xcf, 0x89, 0xbe, 0xd2, + 0x9f, 0xb9, 0x26, 0xfa, 0xca, 0x7c, 0x57, 0x39, 0x5f, 0x58, 0x2e, 0xac, 0x1c, 0xc2, 0x74, 0xfa, + 0xd3, 0x43, 0xf4, 0x14, 0x66, 0xf8, 0x43, 0x44, 0xf6, 0xd1, 0x8d, 0xe4, 0x75, 0x33, 0xfe, 0x01, + 0xa3, 0x7a, 0x73, 0x22, 0x3f, 0x31, 0xd2, 0x73, 0xa8, 0x6e, 0x8b, 0xdf, 0xda, 0xf5, 0xa2, 0x48, + 0x7c, 0x26, 0x63, 0x5d, 0x6a, 0x96, 0xc0, 0x24, 0xd1, 0xa7, 0xe9, 0x57, 0x09, 0xb3, 0x79, 0x49, + 0x9c, 0x9a, 0x4b, 0xe5, 0x03, 0xff, 0x59, 0xf4, 0xf1, 0xde, 0xfb, 0xf1, 0xfb, 0xba, 0x76, 0xf6, + 0xdd, 0x8b, 0x3a, 0x46, 0xe1, 0x63, 0xff, 0x1f, 0xcf, 0xf5, 0x21, 0x34, 0x59, 0xf4, 0xcb, 0x6e, + 0x75, 0x8f, 0xda, 0x5c, 0x57, 0x84, 0xb2, 0x26, 0x75, 0x25, 0x65, 0x8c, 0x49, 0x5d, 0x49, 0xeb, + 0xf5, 0x72, 0xa1, 0xff, 0xd9, 0xd7, 0xff, 0x7c, 0xe3, 0xd2, 0xd7, 0xbf, 0xb8, 0x51, 0xf8, 0xdb, + 0x5f, 0xdc, 0x28, 0xfc, 0xc9, 0xbf, 0xdc, 0x28, 0xfc, 0x78, 0xf1, 0x4c, 0xbf, 0x05, 0x94, 0x3d, + 0xee, 0x57, 0x39, 0xe9, 0x83, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x07, 0x1b, 0x00, 0xd0, 0x18, + 0x3e, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -4913,6 +5196,129 @@ var _Shard_serviceDesc = grpc.ServiceDesc{ Metadata: "go/protocols/runtime/runtime.proto", } +// TaskControlClient is the client API for TaskControl service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type TaskControlClient interface { + // SyncNow asks a materialization to immediately commit its open + // transaction, and resolves once that transaction is fully acknowledged: + // committed and queryable in the endpoint. It's served by the + // runtime-next sidecar hosting the task's leader session, and relayed + // by the reactor front door. + // + // The stream is exactly one Ack, then zero or more Heartbeats while the + // caller waits, then exactly one Done, after which the stream closes. + // SyncNow is idempotent: concurrent calls await the same commit, and a + // caller which hangs up early has still forced the commit. There is no + // error for "nothing to do" — a task with nothing to await, including a + // capture or derivation, Acks and its Done follows immediately. + SyncNow(ctx context.Context, in *SyncNowRequest, opts ...grpc.CallOption) (TaskControl_SyncNowClient, error) +} + +type taskControlClient struct { + cc *grpc.ClientConn +} + +func NewTaskControlClient(cc *grpc.ClientConn) TaskControlClient { + return &taskControlClient{cc} +} + +func (c *taskControlClient) SyncNow(ctx context.Context, in *SyncNowRequest, opts ...grpc.CallOption) (TaskControl_SyncNowClient, error) { + stream, err := c.cc.NewStream(ctx, &_TaskControl_serviceDesc.Streams[0], "/runtime.TaskControl/SyncNow", opts...) + if err != nil { + return nil, err + } + x := &taskControlSyncNowClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type TaskControl_SyncNowClient interface { + Recv() (*SyncNowResponse, error) + grpc.ClientStream +} + +type taskControlSyncNowClient struct { + grpc.ClientStream +} + +func (x *taskControlSyncNowClient) Recv() (*SyncNowResponse, error) { + m := new(SyncNowResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// TaskControlServer is the server API for TaskControl service. +type TaskControlServer interface { + // SyncNow asks a materialization to immediately commit its open + // transaction, and resolves once that transaction is fully acknowledged: + // committed and queryable in the endpoint. It's served by the + // runtime-next sidecar hosting the task's leader session, and relayed + // by the reactor front door. + // + // The stream is exactly one Ack, then zero or more Heartbeats while the + // caller waits, then exactly one Done, after which the stream closes. + // SyncNow is idempotent: concurrent calls await the same commit, and a + // caller which hangs up early has still forced the commit. There is no + // error for "nothing to do" — a task with nothing to await, including a + // capture or derivation, Acks and its Done follows immediately. + SyncNow(*SyncNowRequest, TaskControl_SyncNowServer) error +} + +// UnimplementedTaskControlServer can be embedded to have forward compatible implementations. +type UnimplementedTaskControlServer struct { +} + +func (*UnimplementedTaskControlServer) SyncNow(req *SyncNowRequest, srv TaskControl_SyncNowServer) error { + return status.Errorf(codes.Unimplemented, "method SyncNow not implemented") +} + +func RegisterTaskControlServer(s *grpc.Server, srv TaskControlServer) { + s.RegisterService(&_TaskControl_serviceDesc, srv) +} + +func _TaskControl_SyncNow_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SyncNowRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(TaskControlServer).SyncNow(m, &taskControlSyncNowServer{stream}) +} + +type TaskControl_SyncNowServer interface { + Send(*SyncNowResponse) error + grpc.ServerStream +} + +type taskControlSyncNowServer struct { + grpc.ServerStream +} + +func (x *taskControlSyncNowServer) Send(m *SyncNowResponse) error { + return x.ServerStream.SendMsg(m) +} + +var _TaskControl_serviceDesc = grpc.ServiceDesc{ + ServiceName: "runtime.TaskControl", + HandlerType: (*TaskControlServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "SyncNow", + Handler: _TaskControl_SyncNow_Handler, + ServerStreams: true, + }, + }, + Metadata: "go/protocols/runtime/runtime.proto", +} + func (m *TaskServiceConfig) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) @@ -9615,32 +10021,246 @@ func (m *Derive_StartedCommit) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func encodeVarintRuntime(dAtA []byte, offset int, v uint64) int { - offset -= sovRuntime(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ +func (m *SyncNowRequest) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err } - dAtA[offset] = uint8(v) - return base + return dAtA[:n], nil } -func (m *TaskServiceConfig) ProtoSize() (n int) { - if m == nil { - return 0 - } + +func (m *SyncNowRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i var l int _ = l - if m.LogFileFd != 0 { - n += 1 + sovRuntime(uint64(m.LogFileFd)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - l = len(m.TaskName) - if l > 0 { - n += 1 + l + sovRuntime(uint64(l)) + if len(m.TaskName) > 0 { + i -= len(m.TaskName) + copy(dAtA[i:], m.TaskName) + i = encodeVarintRuntime(dAtA, i, uint64(len(m.TaskName))) + i-- + dAtA[i] = 0xa } - l = len(m.UdsPath) - if l > 0 { + return len(dAtA) - i, nil +} + +func (m *SyncNowResponse) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SyncNowResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Response != nil { + { + size := m.Response.ProtoSize() + i -= size + if _, err := m.Response.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + } + } + return len(dAtA) - i, nil +} + +func (m *SyncNowResponse_Ack_) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse_Ack_) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Ack != nil { + { + size, err := m.Ack.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRuntime(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} +func (m *SyncNowResponse_Heartbeat_) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse_Heartbeat_) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Heartbeat != nil { + { + size, err := m.Heartbeat.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRuntime(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + return len(dAtA) - i, nil +} +func (m *SyncNowResponse_Done_) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse_Done_) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Done != nil { + { + size, err := m.Done.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRuntime(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + return len(dAtA) - i, nil +} +func (m *SyncNowResponse_Ack) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SyncNowResponse_Ack) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse_Ack) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + return len(dAtA) - i, nil +} + +func (m *SyncNowResponse_Heartbeat) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SyncNowResponse_Heartbeat) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse_Heartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + return len(dAtA) - i, nil +} + +func (m *SyncNowResponse_Done) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SyncNowResponse_Done) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncNowResponse_Done) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + return len(dAtA) - i, nil +} + +func encodeVarintRuntime(dAtA []byte, offset int, v uint64) int { + offset -= sovRuntime(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *TaskServiceConfig) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.LogFileFd != 0 { + n += 1 + sovRuntime(uint64(m.LogFileFd)) + } + l = len(m.TaskName) + if l > 0 { + n += 1 + l + sovRuntime(uint64(l)) + } + l = len(m.UdsPath) + if l > 0 { n += 1 + l + sovRuntime(uint64(l)) } l = len(m.ContainerNetwork) @@ -11607,58 +12227,161 @@ func (m *Derive_StartedCommit) ProtoSize() (n int) { return n } -func sovRuntime(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 +func (m *SyncNowRequest) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.TaskName) + if l > 0 { + n += 1 + l + sovRuntime(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n } -func sozRuntime(x uint64) (n int) { - return sovRuntime(uint64((x << 1) ^ uint64((int64(x) >> 63)))) + +func (m *SyncNowResponse) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Response != nil { + n += m.Response.ProtoSize() + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n } -func (m *TaskServiceConfig) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRuntime - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: TaskServiceConfig: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: TaskServiceConfig: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field LogFileFd", wireType) - } - m.LogFileFd = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRuntime - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.LogFileFd |= int32(b&0x7F) << shift - if b < 0x80 { - break + +func (m *SyncNowResponse_Ack_) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Ack != nil { + l = m.Ack.ProtoSize() + n += 1 + l + sovRuntime(uint64(l)) + } + return n +} +func (m *SyncNowResponse_Heartbeat_) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Heartbeat != nil { + l = m.Heartbeat.ProtoSize() + n += 1 + l + sovRuntime(uint64(l)) + } + return n +} +func (m *SyncNowResponse_Done_) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Done != nil { + l = m.Done.ProtoSize() + n += 1 + l + sovRuntime(uint64(l)) + } + return n +} +func (m *SyncNowResponse_Ack) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *SyncNowResponse_Heartbeat) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *SyncNowResponse_Done) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovRuntime(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozRuntime(x uint64) (n int) { + return sovRuntime(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *TaskServiceConfig) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TaskServiceConfig: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TaskServiceConfig: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LogFileFd", wireType) + } + m.LogFileFd = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LogFileFd |= int32(b&0x7F) << shift + if b < 0x80 { + break } } case 2: @@ -24085,6 +24808,398 @@ func (m *Derive_StartedCommit) Unmarshal(dAtA []byte) error { } return nil } +func (m *SyncNowRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SyncNowRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SyncNowRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TaskName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRuntime + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRuntime + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TaskName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SyncNowResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SyncNowResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SyncNowResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ack", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRuntime + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRuntime + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &SyncNowResponse_Ack{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Response = &SyncNowResponse_Ack_{v} + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Heartbeat", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRuntime + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRuntime + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &SyncNowResponse_Heartbeat{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Response = &SyncNowResponse_Heartbeat_{v} + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Done", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRuntime + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRuntime + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &SyncNowResponse_Done{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Response = &SyncNowResponse_Done_{v} + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SyncNowResponse_Ack) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Ack: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Ack: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SyncNowResponse_Heartbeat) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Heartbeat: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Heartbeat: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SyncNowResponse_Done) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Done: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Done: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipRuntime(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/go/protocols/runtime/runtime.proto b/go/protocols/runtime/runtime.proto index 081026371a9..7ac5103ff27 100644 --- a/go/protocols/runtime/runtime.proto +++ b/go/protocols/runtime/runtime.proto @@ -1219,3 +1219,56 @@ message Derive { // each leg. Stopped stopped = 61; } + +// SyncNowRequest is the request of a TaskControl.SyncNow RPC. +message SyncNowRequest { + // Name of the task to synchronize. + string task_name = 1; +} + +// SyncNowResponse is the streamed response of a TaskControl.SyncNow RPC. +// Its messages are structural: they mark the caller's position in the +// stream and carry no payload. Statistics of the awaited transaction are +// recorded to the task's stats journal, which is where callers read them. +message SyncNowResponse { + // Ack is sent exactly once, as the first message of the stream: the + // request reached the task's leader and the commit has been forced. + message Ack {} + + // Heartbeat keeps a long-lived stream alive while the caller waits. + message Heartbeat {} + + // Done is sent exactly once, as the final message of the stream: the + // awaited transaction is committed and queryable in the endpoint. + message Done {} + + oneof response { + Ack ack = 1; + Heartbeat heartbeat = 2; + Done done = 3; + } +} + +// TaskControl is a user-facing control surface for running tasks. +// It's distinct from the Leader service because it has a different +// authorization model: callers present ordinary gazette READ claims +// scoped to the task's shards, not the LEAD capability held by shards. +// +// Only the runtime-next sidecar serves TaskControl over gRPC. Users reach it +// through the reactor front door, which relays to that sidecar but exposes +// itself as HTTP/NDJSON rather than gRPC — see go/runtime/task_control_http.go. +service TaskControl { + // SyncNow asks a materialization to immediately commit its open + // transaction, and resolves once that transaction is fully acknowledged: + // committed and queryable in the endpoint. It's served by the + // runtime-next sidecar hosting the task's leader session, and relayed + // by the reactor front door. + // + // The stream is exactly one Ack, then zero or more Heartbeats while the + // caller waits, then exactly one Done, after which the stream closes. + // SyncNow is idempotent: concurrent calls await the same commit, and a + // caller which hangs up early has still forced the commit. There is no + // error for "nothing to do" — a task with nothing to await, including a + // capture or derivation, Acks and its Done follows immediately. + rpc SyncNow(SyncNowRequest) returns (stream SyncNowResponse); +} diff --git a/go/runtime/flow_consumer.go b/go/runtime/flow_consumer.go index 31d5a5a11af..be5d32a16e4 100644 --- a/go/runtime/flow_consumer.go +++ b/go/runtime/flow_consumer.go @@ -325,6 +325,17 @@ func (f *FlowConsumer) InitApplication(args runconsumer.InitArgs) error { pr.RegisterShufflerServer(args.Server.GRPCServer, pr.NewVerifiedShufflerServer(shuffle.NewAPI(args.Service.Resolver), f.service.Verifier)) + // TaskControl is served only as a hand-written REST/NDJSON endpoint, + // sharing the CORS treatment of gazette's grpc-gateway `/v1/` mux (Go's + // ServeMux prefers the more specific pattern). + args.Server.HTTPMux.Handle(TaskControlSyncNowPath, config.Consumer.CORSWrapper(&taskControlHTTP{ + relay: &taskControl{ + service: args.Service, + sidecarEndpoint: config.SidecarEndpoint, + }, + verifier: f.service.Verifier, + })) + pf.RegisterNetworkProxyServer(args.Server.GRPCServer, pf.NewVerifiedNetworkProxyServer(&network.ProxyServer{Resolver: args.Service.Resolver}, f.service.Verifier)) diff --git a/go/runtime/task_control.go b/go/runtime/task_control.go new file mode 100644 index 00000000000..5c35cb08368 --- /dev/null +++ b/go/runtime/task_control.go @@ -0,0 +1,187 @@ +package runtime + +import ( + "context" + "io" + "strings" + + "github.com/estuary/flow/go/labels" + pr "github.com/estuary/flow/go/protocols/runtime" + "go.gazette.dev/core/allocator" + pb "go.gazette.dev/core/broker/protocol" + "go.gazette.dev/core/consumer" + pc "go.gazette.dev/core/consumer/protocol" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +// taskControl serves TaskControl.SyncNow on the reactor front door, as a +// semantics-free relay: it routes a caller's request to the runtime-next +// sidecar co-located with the task's shard-zero primary, which hosts the +// task's leader session and owns all SyncNow semantics. Callers reach it over +// HTTP/NDJSON; see task_control_http.go. +type taskControl struct { + service *consumer.Service + // sidecarEndpoint maps a reactor member endpoint to the dial-able URL of + // its co-located runtime-next sidecar (FlowConsumerConfig.SidecarEndpoint). + sidecarEndpoint func(reactor pb.Endpoint) (string, error) +} + +// syncNow resolves the task's shard-zero primary, dials its co-located +// sidecar, and relays response messages 1:1 until EOF or error. Any front door +// can dial any sidecar (they listen on a fleet-wide port), so there's no +// gazette-style proxy hop to the primary's own front door. The caller's +// Authorization is forwarded verbatim, for the sidecar to independently verify. +func (tc *taskControl) syncNow( + ctx context.Context, + claims pb.Claims, + req *pr.SyncNowRequest, + send func(*pr.SyncNowResponse) error, +) error { + if req.TaskName == "" { + return status.Error(codes.InvalidArgument, "task_name is required") + } + var shardZero, found = taskShardZero(tc.service.State, req.TaskName) + if !found { + return errTaskNotFound(req.TaskName) + } + + // MayProxy because we never serve locally: the sidecar does, whether or not + // we're the primary. + var res, err = tc.service.Resolver.Resolve(consumer.ResolveArgs{ + Context: ctx, + Claims: claims, + ShardID: shardZero, + MayProxy: true, + }) + if err != nil { + return err + } + // Resolution was needed only for routing. Release it now: a SyncNow relay + // can be open for an hour, and must not pin the local shard's teardown. + // If the primary moves mid-relay the sidecar stream breaks, and the + // caller re-invokes (SyncNow is idempotent). + if res.Done != nil { + res.Done() + } + + switch res.Status { + case pc.Status_OK: + // Pass. + case pc.Status_SHARD_NOT_FOUND: + // Also returned when the caller's claims don't cover the shard. + return errTaskNotFound(req.TaskName) + default: + return status.Errorf(codes.Unavailable, + "cannot resolve task shard %s to a ready primary (%s)", shardZero, res.Status) + } + + // Captures and derivations hold no open transaction, so there's nothing to + // force or await. Only this front door can answer that, because only it + // resolved the task's actual shard (and thus its type) from the keyspace. + // Resolve() above verified the caller's claims cover that shard. + if !strings.HasPrefix(shardZero.String(), "materialize/") { + if err = send(&pr.SyncNowResponse{ + Response: &pr.SyncNowResponse_Ack_{Ack: &pr.SyncNowResponse_Ack{}}, + }); err != nil { + return err + } + return send(&pr.SyncNowResponse{ + Response: &pr.SyncNowResponse_Done_{Done: &pr.SyncNowResponse_Done{}}, + }) + } + var sidecar string + if sidecar, err = tc.sidecarEndpoint(res.Header.Route.Endpoints[res.Header.Route.Primary]); err != nil { + return err + } + var conn *grpc.ClientConn + if conn, err = dialSidecar(ctx, sidecar); err != nil { + return err + } + defer conn.Close() + + var client pr.TaskControl_SyncNowClient + if client, err = pr.NewTaskControlClient(conn).SyncNow(forwardAuthorization(ctx), req); err != nil { + return err + } + + for { + var resp, err = client.Recv() + if err == io.EOF { + return nil + } else if err != nil { + return err // Terminal gRPC status passes through verbatim. + } else if err = send(resp); err != nil { + return err + } + } +} + +func errTaskNotFound(taskName string) error { + return status.Errorf(codes.NotFound, + "task %s was not found in this data plane (or your authorization does not cover it)", taskName) +} + +// taskShardZero returns the ID of the named task's shard zero: the shard with +// the lowest key / r-clock range. Shard IDs are `// +// /`, so a prefix scan finds the task's shards in +// range-ascending order — but one task's name may prefix another's, so require +// an exact task-name label match. +// +// A `reset` publication starts a new generation, and activation creates the new +// shards before deleting the old, so both can briefly appear here and we take +// the older. Sync-now during a reset is meaningless either way: the task is +// being backfilled from scratch. +func taskShardZero(state *allocator.State, taskName string) (pc.ShardID, bool) { + state.KS.Mu.RLock() + defer state.KS.Mu.RUnlock() + + for _, taskType := range []string{"capture", "derivation", "materialize"} { + var prefix = allocator.ItemKey(state.KS, taskType+"/"+taskName+"/") + for _, kv := range state.Items.Prefixed(prefix) { + var spec = kv.Decoded.(allocator.Item).ItemValue.(*pc.ShardSpec) + if spec.LabelSet.ValueOf(labels.TaskName) != taskName { + continue + } + return spec.Id, true + } + } + return "", false +} + +// dialSidecar dials the runtime-next sidecar at `endpoint`, using TLS +// or plaintext per its scheme (which mirrors the reactor's own). +// +// A nil TLS config means the process's ambient roots (honoring SSL_CERT_FILE): +// the same trust `gazette::dial_channel` applies to the sidecar's own leader +// and shuffle dials. The reactor's `--*-ca-file` flags configure gazette +// peers, not this hop. +func dialSidecar(ctx context.Context, endpoint string) (*grpc.ClientConn, error) { + var ep = pb.Endpoint(endpoint) + if err := ep.Validate(); err != nil { + return nil, err + } + var creds credentials.TransportCredentials + if ep.URL().Scheme == "https" { + creds = credentials.NewTLS(nil) + } else { + creds = insecure.NewCredentials() + } + return grpc.DialContext(ctx, ep.GRPCAddr(), grpc.WithTransportCredentials(creds)) +} + +// forwardAuthorization returns a Context which forwards the caller's incoming +// Authorization verbatim on outgoing RPCs, so that the sidecar independently +// verifies the caller's own token. +func forwardAuthorization(ctx context.Context) context.Context { + if md, ok := metadata.FromIncomingContext(ctx); ok { + if auth := md.Get("authorization"); len(auth) != 0 { + return metadata.AppendToOutgoingContext(ctx, "authorization", auth[len(auth)-1]) + } + } + return ctx +} diff --git a/go/runtime/task_control_http.go b/go/runtime/task_control_http.go new file mode 100644 index 00000000000..3197b47134b --- /dev/null +++ b/go/runtime/task_control_http.go @@ -0,0 +1,143 @@ +package runtime + +import ( + "encoding/json" + "errors" + "io" + "net/http" + + pr "github.com/estuary/flow/go/protocols/runtime" + "github.com/gogo/gateway" + gwruntime "github.com/grpc-ecosystem/grpc-gateway/runtime" + pb "go.gazette.dev/core/broker/protocol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +// TaskControlSyncNowPath is the reactor front-door HTTP endpoint of +// TaskControl.SyncNow. It's mounted alongside gazette's grpc-gateway +// `/v1/` mux with the same CORS treatment, and speaks the same +// fetch-streaming NDJSON dialect as gateway streaming RPCs +// (such as dashboard journal reads). +const TaskControlSyncNowPath = "/v1/task-control/sync-now" + +// Ceiling on a SyncNow request body, which carries only a task name. +const maxSyncNowRequestBytes = 4096 + +// taskControlHTTP is the reactor front door's only transport of +// TaskControl.SyncNow: both the dashboard and flowctl call it. +// +// This endpoint is the UI contract of the dashboard's "Sync Now" button: +// +// POST /v1/task-control/sync-now +// Authorization: Bearer -- a Read-level `/authorize/user/task` +// reactor token for the task. +// Content-Type: application/json +// +// {"taskName": "acmeCo/my/materialization"} +// +// The response is newline-delimited JSON, flushed per message, where each +// line wraps a runtime.SyncNowResponse exactly as grpc-gateway would: +// +// {"result": {"ack": {}}} +// {"result": {"heartbeat": {}}} +// {"result": {"done": {}}} +// +// The stream is exactly one `ack`, then zero or more `heartbeat` lines (one +// every ~15 seconds while awaiting the transaction, keeping idle +// load-balancer timeouts at bay), then exactly one `done`, after which the +// stream closes. The lines are structural and carry no payload: transaction +// statistics are recorded to the task's stats journal. A task with nothing to +// await — including a capture or derivation — sends `done` immediately after +// `ack`. See runtime.proto for full message shapes and semantics. +// +// A terminal error is a final `{"error": {"grpcCode": ..., "httpCode": ..., +// "message": ..., "httpStatus": ...}}` line (grpc-gateway's mid-stream error +// convention), with the HTTP status also set when no `result` line has been +// written yet. Notably `httpCode` 404 means the task isn't running in this +// data plane (or isn't on the V2 runtime, or the token doesn't cover it). +// +// SyncNow is idempotent: once `ack` is received the commit request has +// landed, hanging up early is harmless, and concurrent calls await the same +// commit. If the stream dies before `done` (network hiccup, task +// re-assignment), simply re-invoke. +type taskControlHTTP struct { + relay *taskControl + verifier pb.Verifier +} + +// jsonpbMarshaler matches the marshaling of gazette's grpc-gateway mux +// (see runconsumer.Main): camelCase names, enums as strings, zero-valued +// fields emitted. +var jsonpbMarshaler = &gateway.JSONPb{EmitDefaults: true} + +func (h *taskControlHTTP) ServeHTTP(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", jsonpbMarshaler.ContentType()) + + if req.Method != http.MethodPost { + writeStreamError(w, false, status.Error(codes.InvalidArgument, "expected POST")) + return + } + // Authenticate before reading the body: this endpoint is reachable by + // anyone, and the token is in the header. gazette's Verifier reads it from + // incoming gRPC metadata, so synthesize that from the HTTP header. The + // relay then forwards the token verbatim from this Context. + var ctx = metadata.NewIncomingContext(req.Context(), + metadata.Pairs("authorization", req.Header.Get("Authorization"))) + ctx, cancel, claims, err := h.verifier.Verify(ctx, pb.Capability_READ) + if err != nil { + writeStreamError(w, false, err) + return + } + defer cancel() + + // The body is one task name; cap it rather than decoding whatever arrives. + var syncNow = new(pr.SyncNowRequest) + if err := jsonpbMarshaler.NewDecoder(http.MaxBytesReader(w, req.Body, maxSyncNowRequestBytes)). + Decode(syncNow); err != nil && err != io.EOF { + writeStreamError(w, false, status.Errorf(codes.InvalidArgument, "parsing request body: %s", err)) + return + } + + var flusher, _ = w.(http.Flusher) + var wroteResult = false + err = h.relay.syncNow(ctx, claims, syncNow, func(resp *pr.SyncNowResponse) error { + var line, err = jsonpbMarshaler.Marshal(map[string]interface{}{"result": resp}) + if err != nil { + return err + } + if _, err = w.Write(append(line, '\n')); err != nil { + return err + } + wroteResult = true + if flusher != nil { + flusher.Flush() + } + return nil + }) + if err != nil && !errors.Is(err, req.Context().Err()) { + writeStreamError(w, wroteResult, err) + } +} + +// writeStreamError writes a terminal error in grpc-gateway's mid-stream +// convention: an `{"error": {...}}` NDJSON line, preceded by the mapped HTTP +// status code if the response header hasn't already been sent with a result. +func writeStreamError(w http.ResponseWriter, wroteResult bool, err error) { + var s = status.Convert(err) + var httpCode = gwruntime.HTTPStatusFromCode(s.Code()) + + if !wroteResult { + w.WriteHeader(httpCode) + } + var line, _ = json.Marshal(map[string]interface{}{ + "error": map[string]interface{}{ + "grpcCode": int32(s.Code()), + "httpCode": httpCode, + "message": s.Message(), + "httpStatus": http.StatusText(httpCode), + }, + }) + _, _ = w.Write(append(line, '\n')) +} diff --git a/go/runtime/task_control_test.go b/go/runtime/task_control_test.go new file mode 100644 index 00000000000..444148591cf --- /dev/null +++ b/go/runtime/task_control_test.go @@ -0,0 +1,411 @@ +package runtime + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/estuary/flow/go/labels" + pr "github.com/estuary/flow/go/protocols/runtime" + "github.com/stretchr/testify/require" + "go.gazette.dev/core/broker/client" + pb "go.gazette.dev/core/broker/protocol" + "go.gazette.dev/core/consumer" + pc "go.gazette.dev/core/consumer/protocol" + "go.gazette.dev/core/consumer/recoverylog" + "go.gazette.dev/core/consumertest" + "go.gazette.dev/core/etcdtest" + "go.gazette.dev/core/message" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +// Required by the gRPC loopback which consumertest.NewConsumer builds. +func init() { pb.RegisterGRPCDispatcher("local") } + +const ( + tcTestTask = "acmeCo/test/materialization" + tcTestShardID = "materialize/acmeCo/test/materialization/0011223344556677/00000000-00000000" + // tcTestScope is the `id:prefix` a Read-level /authorize/user/task + // token carries: the task's shard template prefix. + tcTestScope = "materialize/acmeCo/test/materialization/0011223344556677/" + + // A capture, which the front door answers as having nothing to await. + tcTestCaptureTask = "acmeCo/test/capture" + tcTestCaptureShardID = "capture/acmeCo/test/capture/0011223344556677/00000000-00000000" + tcTestCaptureScope = "capture/acmeCo/test/capture/0011223344556677/" +) + +func TestTaskControlSyncNow(t *testing.T) { + var etcd = etcdtest.TestClient() + defer etcdtest.Cleanup() + var ctx, cancel = context.WithCancel(context.Background()) + defer cancel() + + var stub = newStubSidecar(t) + defer stub.server.GracefulStop() + + // Two reactor members, so that a front door which is *not* shard zero's + // primary is exercised: it must reach the primary's sidecar all the same. + var members [2]*consumertest.Consumer + for i := range members { + var cmr = consumertest.NewConsumer(consumertest.Args{ + C: t, + Etcd: etcd, + Journals: nil, // Never read: the stub App produces no messages. + App: taskControlTestApp{}, + Suffix: fmt.Sprintf("member-%d", i), + }) + cmr.Server.HTTPMux.Handle(TaskControlSyncNowPath, &taskControlHTTP{ + relay: &taskControl{ + service: cmr.Service, + sidecarEndpoint: func(pb.Endpoint) (string, error) { return stub.endpoint, nil }, + }, + verifier: cmr.Service.Verifier, + }) + cmr.Tasks.GoRun() + members[i] = cmr + } + defer func() { + for _, cmr := range members { + cmr.Tasks.Cancel() + require.NoError(t, cmr.Tasks.Wait()) + } + }() + + consumertest.CreateShards(t, members[0], + &pc.ShardSpec{ + Id: tcTestShardID, + MaxTxnDuration: time.Minute, + LabelSet: pb.MustLabelSet(labels.TaskName, tcTestTask), + }, + &pc.ShardSpec{ + Id: tcTestCaptureShardID, + MaxTxnDuration: time.Minute, + LabelSet: pb.MustLabelSet(labels.TaskName, tcTestCaptureTask), + }, + ) + + // Determine the shard's primary member, and thereby which front door is + // co-located with the sidecar and which must reach across. + var route pb.Route + require.NoError(t, members[0].WaitForPrimary(ctx, tcTestShardID, &route)) + require.NoError(t, members[0].WaitForPrimary(ctx, tcTestCaptureShardID, nil)) + var primary, remote = members[0], members[1] + if route.Members[route.Primary].Suffix == "member-1" { + primary, remote = members[1], members[0] + } + + var token = mintTaskToken(t, primary, pb.MustLabelSet("id:prefix", tcTestScope)) + + t.Run("relay-flushes-per-message", func(t *testing.T) { + // Gate the stub between the heartbeat and Done, proving each NDJSON + // line is flushed and readable before the stream completes. + var gate = make(chan struct{}) + stub.script(scriptedOkStream(gate)) + + var resp = postSyncNow(t, primary, token, `{"taskName": "`+tcTestTask+`"}`) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "application/json", resp.Header.Get("Content-Type")) + + var lines = bufio.NewScanner(resp.Body) + require.Contains(t, readResultLine(t, lines), "ack") + require.Contains(t, readResultLine(t, lines), "heartbeat") + + close(gate) // Only now may the stub complete the stream. + require.Contains(t, readResultLine(t, lines), "done") + require.False(t, lines.Scan(), "expected EOF after done") + + var taskName, auth = stub.observed() + require.Equal(t, tcTestTask, taskName) + require.Equal(t, "Bearer "+token, auth) + }) + + t.Run("relay-from-a-non-primary-front-door", func(t *testing.T) { + stub.script(scriptedOkStream(nil)) + var resp = postSyncNow(t, remote, token, `{"taskName": "`+tcTestTask+`"}`) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + var lines = bufio.NewScanner(resp.Body) + require.Contains(t, readResultLine(t, lines), "ack") + require.Contains(t, readResultLine(t, lines), "heartbeat") + require.Contains(t, readResultLine(t, lines), "done") + + // The token is forwarded verbatim to the primary's sidecar. + var taskName, auth = stub.observed() + require.Equal(t, tcTestTask, taskName) + require.Equal(t, "Bearer "+token, auth) + }) + + t.Run("not-found-passthrough", func(t *testing.T) { + stub.script(func(*pr.SyncNowRequest, pr.TaskControl_SyncNowServer) error { + return status.Error(codes.NotFound, "no live leader session") + }) + var resp = postSyncNow(t, primary, token, `{"taskName": "`+tcTestTask+`"}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusNotFound, resp.StatusCode) + var errBody = readErrorLine(t, resp) + require.Equal(t, float64(codes.NotFound), errBody["grpcCode"]) + require.Equal(t, float64(http.StatusNotFound), errBody["httpCode"]) + require.Contains(t, errBody["message"], "no live leader session") + }) + + t.Run("rejects-missing-token", func(t *testing.T) { + var resp = postSyncNow(t, primary, "", `{"taskName": "`+tcTestTask+`"}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusUnauthorized, resp.StatusCode) + require.Equal(t, float64(codes.Unauthenticated), readErrorLine(t, resp)["grpcCode"]) + }) + + t.Run("rejects-missing-token-before-reading-the-body", func(t *testing.T) { + // An unauthenticated caller must not get us to decode its body: the + // endpoint is publicly reachable and the body is otherwise unbounded. + // A body which would itself be rejected proves the ordering, since it + // answers Unauthenticated rather than InvalidArgument. + var resp = postSyncNow(t, primary, "", `{"taskName": 42}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusUnauthorized, resp.StatusCode) + require.Equal(t, float64(codes.Unauthenticated), readErrorLine(t, resp)["grpcCode"]) + }) + + t.Run("rejects-an-oversized-body", func(t *testing.T) { + var resp = postSyncNow(t, primary, token, + `{"taskName": "`+strings.Repeat("x", maxSyncNowRequestBytes)+`"}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + }) + + t.Run("rejects-mis-scoped-token", func(t *testing.T) { + var misScoped = mintTaskToken(t, primary, + pb.MustLabelSet("id:prefix", "materialize/acmeCo/other/0011223344556677/")) + var resp = postSyncNow(t, primary, misScoped, `{"taskName": "`+tcTestTask+`"}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + + t.Run("rejects-bad-request", func(t *testing.T) { + var resp = postSyncNow(t, primary, token, `{"taskName": 42}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + }) + + t.Run("unknown-task", func(t *testing.T) { + stub.script(scriptedOkStream(nil)) + var resp = postSyncNow(t, primary, token, `{"taskName": "acmeCo/test/not-a-task"}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusNotFound, resp.StatusCode) + + // The sidecar is never consulted for a task we can't resolve. + var taskName, _ = stub.observed() + require.Empty(t, taskName) + }) + + t.Run("capture-has-nothing-to-await", func(t *testing.T) { + stub.script(scriptedOkStream(nil)) + var captureToken = mintTaskToken(t, primary, + pb.MustLabelSet("id:prefix", tcTestCaptureScope)) + var resp = postSyncNow(t, primary, captureToken, `{"taskName": "`+tcTestCaptureTask+`"}`) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + // Captures hold no open transaction: the front door answers from the + // task's shard type, without dialing the sidecar. + var lines = bufio.NewScanner(resp.Body) + require.Contains(t, readResultLine(t, lines), "ack") + require.Contains(t, readResultLine(t, lines), "done") + require.False(t, lines.Scan(), "expected EOF after done") + + var taskName, _ = stub.observed() + require.Empty(t, taskName) + }) + + t.Run("capture-rejects-mis-scoped-token", func(t *testing.T) { + // The capture's "nothing to await" answer comes only after the + // caller's claims are verified to cover its shard. + var resp = postSyncNow(t, primary, token, `{"taskName": "`+tcTestCaptureTask+`"}`) + defer resp.Body.Close() + + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) +} + +// scriptedOkStream returns a stub script sending the canonical happy-path +// sequence: Ack, Heartbeat, then Done. If `gate` is non-nil, Done is withheld +// until it closes. +func scriptedOkStream(gate <-chan struct{}) func(*pr.SyncNowRequest, pr.TaskControl_SyncNowServer) error { + return func(_ *pr.SyncNowRequest, stream pr.TaskControl_SyncNowServer) error { + if err := stream.Send(&pr.SyncNowResponse{ + Response: &pr.SyncNowResponse_Ack_{Ack: &pr.SyncNowResponse_Ack{}}, + }); err != nil { + return err + } + if err := stream.Send(&pr.SyncNowResponse{ + Response: &pr.SyncNowResponse_Heartbeat_{Heartbeat: &pr.SyncNowResponse_Heartbeat{}}, + }); err != nil { + return err + } + if gate != nil { + <-gate + } + return stream.Send(&pr.SyncNowResponse{ + Response: &pr.SyncNowResponse_Done_{Done: &pr.SyncNowResponse_Done{}}, + }) + } +} + +func postSyncNow(t *testing.T, cmr *consumertest.Consumer, token, body string) *http.Response { + var url = cmr.Server.Endpoint().URL().String() + TaskControlSyncNowPath + var req, err = http.NewRequest(http.MethodPost, url, strings.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + return resp +} + +// readResultLine scans one NDJSON line and returns its unwrapped "result". +func readResultLine(t *testing.T, lines *bufio.Scanner) map[string]interface{} { + require.True(t, lines.Scan(), "expected another NDJSON line") + var parsed map[string]map[string]interface{} + require.NoError(t, json.Unmarshal(lines.Bytes(), &parsed)) + require.Contains(t, parsed, "result") + return parsed["result"] +} + +// readErrorLine reads the response body as a single NDJSON error line and +// returns its unwrapped "error". +func readErrorLine(t *testing.T, resp *http.Response) map[string]interface{} { + var parsed map[string]map[string]interface{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&parsed)) + require.Contains(t, parsed, "error") + return parsed["error"] +} + +// mintTaskToken signs READ-capability Claims as the control plane's +// /authorize/user/task would, and returns the raw bearer token. +func mintTaskToken(t *testing.T, cmr *consumertest.Consumer, sel pb.LabelSet) string { + var ctx, err = cmr.Service.Authorizer.Authorize(context.Background(), pb.Claims{ + Capability: pb.Capability_READ, + Selector: pb.LabelSelector{Include: sel}, + }, time.Hour) + require.NoError(t, err) + + var md, _ = metadata.FromOutgoingContext(ctx) + return strings.TrimPrefix(md.Get("authorization")[0], "Bearer ") +} + +// stubSidecar is a scripted TaskControl gRPC server standing in for the +// runtime-next sidecar, recording what the relay delivers to it. +type stubSidecar struct { + endpoint string + server *grpc.Server + + mu sync.Mutex + serveFn func(*pr.SyncNowRequest, pr.TaskControl_SyncNowServer) error + taskName string + auth string +} + +func newStubSidecar(t *testing.T) *stubSidecar { + var listener, err = net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + var stub = &stubSidecar{ + endpoint: "http://" + listener.Addr().String(), + server: grpc.NewServer(), + } + pr.RegisterTaskControlServer(stub.server, stub) + go func() { _ = stub.server.Serve(listener) }() + return stub +} + +// script installs the SyncNow behavior for the next test case. +func (s *stubSidecar) script(fn func(*pr.SyncNowRequest, pr.TaskControl_SyncNowServer) error) { + s.mu.Lock() + defer s.mu.Unlock() + s.serveFn, s.taskName, s.auth = fn, "", "" +} + +// observed returns the task name and Authorization of the last request. +func (s *stubSidecar) observed() (taskName, auth string) { + s.mu.Lock() + defer s.mu.Unlock() + return s.taskName, s.auth +} + +func (s *stubSidecar) SyncNow(req *pr.SyncNowRequest, stream pr.TaskControl_SyncNowServer) error { + var md, _ = metadata.FromIncomingContext(stream.Context()) + + s.mu.Lock() + var fn = s.serveFn + s.taskName = req.TaskName + if auth := md.Get("authorization"); len(auth) != 0 { + s.auth = auth[0] + } + s.mu.Unlock() + + return fn(req, stream) +} + +// taskControlTestApp is a minimal consumer.Application whose shards become +// resolvable primaries without a broker, recovery log, or messages: as a +// MessageProducer it simply never produces, so shards idle. +type taskControlTestApp struct{} +type taskControlTestStore struct{} + +func (taskControlTestApp) NewStore(consumer.Shard, *recoverylog.Recorder) (consumer.Store, error) { + return taskControlTestStore{}, nil +} +func (taskControlTestApp) NewMessage(*pb.JournalSpec) (message.Message, error) { + panic("not called") +} +func (taskControlTestApp) ConsumeMessage(consumer.Shard, consumer.Store, message.Envelope, *message.Publisher) error { + panic("not called") +} +func (taskControlTestApp) FinalizeTxn(consumer.Shard, consumer.Store, *message.Publisher) error { + panic("not called") +} +func (taskControlTestApp) StartReadingMessages(shard consumer.Shard, _ consumer.Store, _ pc.Checkpoint, intoCh chan<- consumer.EnvelopeOrError) { + // Produce no messages, but deliver the shard's cancellation: the + // transaction loop of a MessageProducer application tears down only + // upon reading an error from this channel. + go func() { + <-shard.Context().Done() + intoCh <- consumer.EnvelopeOrError{Error: shard.Context().Err()} + }() +} +func (taskControlTestApp) ReplayRange(_ consumer.Shard, _ consumer.Store, _ pb.Journal, _, _ pb.Offset) message.Iterator { + panic("not called") +} +func (taskControlTestApp) ReadThrough(consumer.Shard, consumer.Store, consumer.ResolveArgs) (pb.Offsets, error) { + return nil, nil +} + +func (taskControlTestStore) StartCommit(consumer.Shard, pc.Checkpoint, consumer.OpFutures) consumer.OpFuture { + return client.FinishedOperation(nil) +} +func (taskControlTestStore) RestoreCheckpoint(consumer.Shard) (pc.Checkpoint, error) { + return pc.Checkpoint{}, nil +} +func (taskControlTestStore) Destroy() {}