diff --git a/AGENTS.md b/AGENTS.md index 88316df7704..ff7a27cf8e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,7 @@ go build -tags libsqlite3 ./go/bindings # Regenerate checked-in protobuf (required after .proto changes) mise run build:go-protobufs mise run build:rust-protobufs +cargo fmt --all # remove format-only churn from codegen # Run pgTAP SQL Tests mise run ci:sql-tap diff --git a/crates/dekaf/src/topology.rs b/crates/dekaf/src/topology.rs index c15881cb600..85610459af3 100644 --- a/crates/dekaf/src/topology.rs +++ b/crates/dekaf/src/topology.rs @@ -261,7 +261,7 @@ impl Collection { let key_schema = avro::key_to_avro(&key_ptr, collection_schema_shape); - let (not_before, not_after) = ( + let (mut not_before, not_after) = ( binding.not_before.map(|b| { uuid::Clock::from_unix(b.seconds.try_into().unwrap(), b.nanos.try_into().unwrap()) }), @@ -270,6 +270,23 @@ impl Collection { }), ); + // Honor truncated-at journal labels: if any partition carries a + // truncated-at label, use the max across all partitions and the + // binding's not_before as the effective not_before. + for partition in &partitions { + if let Some(truncated_at_str) = partition + .spec + .labels + .as_ref() + .and_then(|ls| ls.labels.iter().find(|l| l.name == ::labels::TRUNCATED_AT)) + .map(|l| &l.value) + { + let truncated_clock = + uuid::Clock::from_u64(::labels::parse_truncated_at(truncated_at_str)?); + not_before = Some(not_before.map_or(truncated_clock, |nb| nb.max(truncated_clock))); + } + } + tracing::debug!( collection_name, partitions = partitions.len(), diff --git a/crates/doc/src/combine/memtable.rs b/crates/doc/src/combine/memtable.rs index 797332645b4..d6536f3f01b 100644 --- a/crates/doc/src/combine/memtable.rs +++ b/crates/doc/src/combine/memtable.rs @@ -66,10 +66,12 @@ impl Entries { } fn compact(&mut self, alloc: &'static Bump) -> Result<(), Error> { - // `sort_ord` orders over (binding, key, !front): - // For each (binding, key), we take front() entries first, and further - // rely on sort preserving the order in which entries were added. - // This maintains the left-to-right associative ordering of reductions. + // `sort_ord` orders over (binding, key, stale, !front): + // For each (binding, key), stale entries sort first, then front() + // entries, and we further rely on sort preserving the order in which + // entries were added. This maintains the left-to-right associative + // ordering of reductions. Stale is a group boundary, so compaction + // never reduces a stale entry with a fresh one. // // `meta` contains a packed structure that's order-preserving over // (binding, key), so we first test it for inequality. @@ -80,6 +82,7 @@ impl Entries { // Cold path: Meta prefix was equal, so compare the full key. compare_root_keys(&self.spec.keys[l.meta.binding()], &l.root, &r.root) }) + .then_with(|| l.meta.stale().cmp(&r.meta.stale()).reverse()) .then_with(|| l.meta.front().cmp(&r.meta.front()).reverse()) }; let validators = &mut self.spec.validators; @@ -87,6 +90,11 @@ impl Entries { // Closure which attempts an associative reduction of `index` into `index-1`. // If the reduction succeeds then the item at `index` is removed. let mut maybe_reduce = |next: &mut Vec>, index: usize| -> Result<(), Error> { + // Stale content is known-dead: never validate or reduce it. Groups + // are uniformly stale or fresh (per `sort_ord`), so one test suffices. + if next[index].meta.stale() { + return Ok(()); + } let rhs = &next[index]; let rhs_outcomes = validate_root( @@ -234,6 +242,24 @@ impl MemTable { /// Add the document to the MemTable. pub fn add<'s>(&'s self, binding: u16, root: HeapNode<'s>, front: bool) -> Result<(), Error> { + self.add_inner(binding, root, front, false) + } + + /// Add a Loaded document the caller has classified as stale. Like `add` with + /// `front=true`, but flagged stale: its value is never reduced or emitted, + /// while its existence transfers onto the fresh entry of the same + /// (binding, key). See [`super::Accumulator::truncate`]. + pub fn add_stale_front<'s>(&'s self, binding: u16, root: HeapNode<'s>) -> Result<(), Error> { + self.add_inner(binding, root, true, true) + } + + fn add_inner<'s>( + &'s self, + binding: u16, + root: HeapNode<'s>, + front: bool, + stale: bool, + ) -> Result<(), Error> { // Safety: mutable borrow does not escape this function. let entries = unsafe { &mut *self.entries.get() }; let root = unsafe { std::mem::transmute::, HeapNode<'static>>(root) }; @@ -245,12 +271,15 @@ impl MemTable { &mut entries.scratch, None, ); - let meta = Meta::new( + let mut meta = Meta::new( binding, &entries.scratch, front, false, // `known_valid` ); + if stale { + meta.set_stale(); + } entries.queued.push(HeapEntry { meta, @@ -265,6 +294,30 @@ impl MemTable { } } + /// Truncate `binding` within this MemTable in a single pass: drop its + /// `!front()` entries (pre-boundary sources carry no existence) and flag its + /// `front()` entries stale in place (value dead, but the body is retained so + /// drain can match its key against fresh entries). Other bindings untouched. + pub fn truncate(&self, binding: u16) { + // Safety: mutable borrow does not escape this function. + let entries = unsafe { &mut *self.entries.get() }; + + let retain = |entry: &mut HeapEntry| -> bool { + if entry.meta.binding() != binding as usize { + return true; + } else if !entry.meta.front() { + return false; + } + entry.meta.set_stale(); + true + }; + + // `sorted` stays sorted: `retain_mut` preserves order, and the binding's + // survivors are homogeneous stale fronts, which sort ahead of fresh. + entries.queued.retain_mut(retain); + entries.sorted.retain_mut(retain); + } + /// Add a pre-serialized ArchivedEmbedded document from the shuffle reader. /// The packed_key_prefix is the first 16 bytes of the packed key of the document. pub fn add_embedded<'s>( @@ -432,6 +485,34 @@ impl MemDrainer { let Some(HeapEntry { mut meta, mut root }) = self.it.next() else { return Ok(None); }; + + // Advance past stale entries to the first fresh entry of their key, + // ORing their front() existence onto it. A stale run with no fresh + // successor emits nothing. + let mut stale_front = false; + while meta.stale() { + stale_front |= meta.front(); + + let Some(next) = self.it.next() else { + return Ok(None); // Trailing stale run: nothing to emit. + }; + + let same_key = meta.0 == next.meta.0 + && compare_root_keys(&self.spec.keys[meta.binding()], &root, &next.root).is_eq(); + + HeapEntry { meta, root } = next; + self.in_group = false; + + if !same_key { + // Orphaned stale run: drop its existence (next may differ in binding). + stale_front = false; + } + } + + if stale_front { + meta.set_front(); // Transfer stale existence onto the fresh output. + } + let is_full = self.spec.is_full[meta.binding()]; let keys = self.spec.keys[meta.binding()].as_ref(); let name = &self.spec.names[meta.binding()]; @@ -645,6 +726,469 @@ mod test { .unwrap(); } + /// A full-reduction Spec over `n_bindings` bindings keyed on `/key`, whose + /// `v` array reduces by append. A closure (not `vec![…; N]`) because + /// `Validator` isn't `Clone`. + fn append_merge_spec(n_bindings: usize) -> Spec { + let binding = || { + let schema = build_schema( + &url::Url::parse("http://example/schema").unwrap(), + &json!({ + "properties": { "v": { "type": "array", "reduce": { "strategy": "append" } } }, + "reduce": { "strategy": "merge" } + }), + ) + .unwrap(); + ( + true, // Full reduction. + vec![Extractor::with_default( + "/key", + &SerPolicy::noop(), + json!("def"), + )], + "test", + Validator::new(schema).unwrap(), + ) + }; + Spec::with_bindings(std::iter::repeat_with(binding).take(n_bindings), Vec::new()) + } + + /// Force the Accumulator's current MemTable into a new spill segment, + /// leaving a fresh empty MemTable behind. + fn force_spill(acc: &mut crate::combine::Accumulator) { + let spec = acc + .memtable + .take() + .unwrap() + .spill(&mut acc.spill, CHUNK_TARGET_SIZE) + .unwrap(); + acc.memtable = Some(MemTable::new(spec)); + } + + fn project(doc: DrainedDoc) -> (usize, serde_json::Value, bool) { + ( + doc.meta.binding(), + serde_json::to_value(SerPolicy::noop().on_owned(&doc.root)).unwrap(), + doc.meta.front(), + ) + } + + #[test] + fn test_truncate_partition() { + // Backfill truncation over two bindings: binding 0 is truncated (its + // pre-boundary sources dropped, stale Loaded rows kept as existence-only + // fronts) while binding 1 is untouched. The drain output must be + // identical whether the fixture stays in memory (MemTable::truncate) or + // is forced through a spill file whose pre-boundary segment is fenced by + // ordinal (Accumulator::truncate). + let doc = |key: &str, v: &str| json!({ "key": key, "v": [v] }); + + let expected = vec![ + // Two stale Loaded fronts transfer existence once onto the fresh source. + (0, json!({"key": "exist_once", "v": ["s2"]}), true), + // Pre-boundary sources dropped; only the fresh source stores. + (0, json!({"key": "multi", "v": ["a2"]}), false), + // A dropped pre-boundary source fabricates no existence. + (0, json!({"key": "no_exist", "v": ["x2"]}), false), + // A fresh Loaded front reduces with a fresh source; pre-boundary dropped. + (0, json!({"key": "reduce2", "v": ["r_load", "r_src"]}), true), + // "orphan" and "zzz_cross" are stale-only and emit nothing; binding 1 + // (never truncated) drains normally. + (1, json!({"key": "b", "v": ["b0"]}), false), + ]; + + // Add post-boundary arrivals: stale Loaded rows (add_stale_front), one + // fresh Loaded front, then fresh sources, and an untouched binding 1. + let add_post = |mt: &MemTable| { + for (key, v) in [ + ("exist_once", "L0"), + ("exist_once", "L1"), + ("orphan", "gone"), + ("zzz_cross", "gone"), + ] { + mt.add_stale_front(0, HeapNode::from_node(&doc(key, v), mt.alloc())) + .unwrap(); + } + mt.add( + 0, + HeapNode::from_node(&doc("reduce2", "r_load"), mt.alloc()), + true, + ) + .unwrap(); + for (key, v) in [ + ("exist_once", "s2"), + ("multi", "a2"), + ("no_exist", "x2"), + ("reduce2", "r_src"), + ] { + mt.add(0, HeapNode::from_node(&doc(key, v), mt.alloc()), false) + .unwrap(); + } + mt.add(1, HeapNode::from_node(&doc("b", "b0"), mt.alloc()), false) + .unwrap(); + }; + // Pre-boundary source documents of binding 0 (dropped/fenced on truncate). + let pre = [("multi", "a0"), ("no_exist", "x0"), ("reduce2", "r0")]; + + // In-memory variant: MemTable::truncate directly between adds. + { + let memtable = MemTable::new(append_merge_spec(2)); + for (key, v) in pre { + memtable + .add( + 0, + HeapNode::from_node(&doc(key, v), memtable.alloc()), + false, + ) + .unwrap(); + } + memtable.truncate(0); + add_post(&memtable); + + let in_memory = memtable + .try_into_drainer() + .unwrap() + .map_ok(project) + .collect::, _>>() + .unwrap(); + assert_eq!(in_memory, expected, "in-memory drain"); + } + + // Spill variant: an Accumulator spills the pre-boundary sources into a + // segment, Accumulator::truncate fences that segment by ordinal, and + // post-boundary arrivals (including stale fronts carrying their + // persisted STALE flag) land in the final segment. + { + let mut acc = crate::combine::Accumulator::new( + append_merge_spec(2), + tempfile::tempfile().unwrap(), + ) + .unwrap(); + { + let mt = acc.memtable().unwrap(); + for (key, v) in pre { + mt.add(0, HeapNode::from_node(&doc(key, v), mt.alloc()), false) + .unwrap(); + } + } + force_spill(&mut acc); // Pre-boundary sources become segment 0. + acc.truncate(0); // Fence segment 0 for binding 0. + add_post(acc.memtable().unwrap()); + + let spilled = acc + .into_drainer() + .unwrap() + .map_ok(project) + .collect::, _>>() + .unwrap(); + assert_eq!(spilled, expected, "spill drain"); + } + } + + #[test] + fn test_truncate_purges_and_flags() { + // truncate() drops the binding's !front entries and flags its front + // entries stale in place, across both the compacted `sorted` vec and the + // uncompacted `queued` vec, leaving other bindings untouched. + let memtable = MemTable::new(append_merge_spec(2)); + let add = |binding: u16, key: &str, front: bool| { + let node = HeapNode::from_node(&json!({"key": key, "v": ["x"]}), memtable.alloc()); + memtable.add(binding, node, front).unwrap(); + }; + + add(0, "s_front", true); + add(0, "s_drop", false); + memtable.compact().unwrap(); // Move the above into `sorted`. + add(0, "q_front", true); + add(0, "q_drop", false); + add(1, "other", false); // Untouched second binding. + + memtable.truncate(0); + + // Safety: no references to `entries` are lent out. + let entries = unsafe { &*memtable.entries.get() }; + let key_of = |e: &HeapEntry| -> String { + serde_json::to_value(SerPolicy::noop().on(&e.root.access().unwrap())).unwrap()["key"] + .as_str() + .unwrap() + .to_string() + }; + + // Binding 0: only the two front entries survive, both now stale. + let mut b0: Vec<(String, bool, bool)> = entries + .sorted + .iter() + .chain(entries.queued.iter()) + .filter(|e| e.meta.binding() == 0) + .map(|e| (key_of(e), e.meta.front(), e.meta.stale())) + .collect(); + b0.sort(); + assert_eq!( + b0, + vec![ + ("q_front".to_string(), true, true), + ("s_front".to_string(), true, true), + ], + ); + + // Binding 1: untouched (fresh, not stale). + let b1: Vec<(String, bool, bool)> = entries + .sorted + .iter() + .chain(entries.queued.iter()) + .filter(|e| e.meta.binding() == 1) + .map(|e| (key_of(e), e.meta.front(), e.meta.stale())) + .collect(); + assert_eq!(b1, vec![("other".to_string(), false, false)]); + } + + #[test] + fn test_stale_front_spilled_after_truncate() { + // A Loaded row flagged stale via add_stale_front AFTER truncate lands in + // a segment at/above the cutoff, so its staleness rides the persisted + // flag byte (not the ordinal fence) and it's still discarded on drain, + // transferring only existence onto the fresh source. + let mut acc = + crate::combine::Accumulator::new(append_merge_spec(1), tempfile::tempfile().unwrap()) + .unwrap(); + { + let mt = acc.memtable().unwrap(); + mt.add( + 0, + HeapNode::from_node(&json!({"key": "k", "v": ["pre"]}), mt.alloc()), + false, + ) + .unwrap(); + } + force_spill(&mut acc); // "pre" becomes fenced segment 0. + acc.truncate(0); + { + let mt = acc.memtable().unwrap(); + mt.add_stale_front( + 0, + HeapNode::from_node(&json!({"key": "k", "v": ["stale_load"]}), mt.alloc()), + ) + .unwrap(); + mt.add( + 0, + HeapNode::from_node(&json!({"key": "k", "v": ["fresh"]}), mt.alloc()), + false, + ) + .unwrap(); + } + + let out = acc + .into_drainer() + .unwrap() + .map_ok(|d| { + ( + serde_json::to_value(SerPolicy::noop().on_owned(&d.root)).unwrap(), + d.meta.front(), + ) + }) + .collect::, _>>() + .unwrap(); + + assert_eq!(out, vec![(json!({"key": "k", "v": ["fresh"]}), true)]); + } + + #[test] + fn test_multiple_truncates() { + // Truncating one binding repeatedly within a single Accumulator keeps + // dropping each generation's pre-boundary source, so only the final + // generation's data drains. + let mut acc = + crate::combine::Accumulator::new(append_merge_spec(1), tempfile::tempfile().unwrap()) + .unwrap(); + let add = |acc: &mut crate::combine::Accumulator, v: &str, front: bool| { + let mt = acc.memtable().unwrap(); + let node = HeapNode::from_node(&json!({"key": "k", "v": [v]}), mt.alloc()); + mt.add(0, node, front).unwrap(); + }; + + add(&mut acc, "v0", false); + acc.truncate(0); // Drops v0. + add(&mut acc, "v1", false); + acc.truncate(0); // Drops v1. + add(&mut acc, "v2", false); + add(&mut acc, "load", true); + + let out = acc + .into_drainer() + .unwrap() + .map_ok(|d| { + ( + serde_json::to_value(SerPolicy::noop().on_owned(&d.root)).unwrap(), + d.meta.front(), + ) + }) + .collect::, _>>() + .unwrap(); + + assert_eq!(out, vec![(json!({"key": "k", "v": ["load", "v2"]}), true)]); + } + + #[test] + fn test_multiple_truncates_across_segments() { + // The ordinal cutoff ratchets across real spilled segments: truncate, + // spill, truncate again. Every pre-boundary segment stays fenced, and a + // stale front in the earliest (still-fenced) segment transfers its + // existence onto the final fresh source. + let mut acc = + crate::combine::Accumulator::new(append_merge_spec(1), tempfile::tempfile().unwrap()) + .unwrap(); + let add = |acc: &mut crate::combine::Accumulator, v: &str, front: bool| { + let mt = acc.memtable().unwrap(); + let node = HeapNode::from_node(&json!({"key": "k", "v": [v]}), mt.alloc()); + mt.add(0, node, front).unwrap(); + }; + + add(&mut acc, "load", true); // A Loaded front, pre-boundary. + add(&mut acc, "v0", false); + force_spill(&mut acc); // Segment 0: [load(front), v0]. + acc.truncate(0); // cutoffs[0] = 1, fencing segment 0. + + add(&mut acc, "v1", false); + force_spill(&mut acc); // Segment 1: [v1]. + acc.truncate(0); // cutoffs[0] = 2, fencing segments 0 and 1. + + add(&mut acc, "v2", false); // Fresh source in the final segment. + + let out = acc + .into_drainer() + .unwrap() + .map_ok(|d| { + ( + serde_json::to_value(SerPolicy::noop().on_owned(&d.root)).unwrap(), + d.meta.front(), + ) + }) + .collect::, _>>() + .unwrap(); + + // v0 and v1 (fenced sources) drop; `load`'s existence (fenced segment 0) + // transfers onto v2. + assert_eq!(out, vec![(json!({"key": "k", "v": ["v2"]}), true)]); + } + + #[test] + fn test_existence_transfer_across_locations() { + // Two stale fronts for one key reach drain by different routes — one + // fenced by an ordinal cutoff, one carrying a persisted STALE flag — and + // together transfer existence exactly once onto the fresh source. + let mut acc = + crate::combine::Accumulator::new(append_merge_spec(1), tempfile::tempfile().unwrap()) + .unwrap(); + + { + let mt = acc.memtable().unwrap(); + mt.add( + 0, + HeapNode::from_node(&json!({"key": "k", "v": ["load_a"]}), mt.alloc()), + true, + ) + .unwrap(); + } + force_spill(&mut acc); // Segment 0: [load_a(front)]. + acc.truncate(0); // Fences segment 0 → load_a stale by ordinal. + + { + let mt = acc.memtable().unwrap(); + // A Loaded row classified stale on arrival, after the truncate. + mt.add_stale_front( + 0, + HeapNode::from_node(&json!({"key": "k", "v": ["load_b"]}), mt.alloc()), + ) + .unwrap(); + mt.add( + 0, + HeapNode::from_node(&json!({"key": "k", "v": ["fresh"]}), mt.alloc()), + false, + ) + .unwrap(); + } + + let out = acc + .into_drainer() + .unwrap() + .map_ok(|d| { + ( + serde_json::to_value(SerPolicy::noop().on_owned(&d.root)).unwrap(), + d.meta.front(), + ) + }) + .collect::, _>>() + .unwrap(); + + // load_a (fenced) and load_b (persisted flag) drop; existence transfers + // once onto `fresh`. + assert_eq!(out, vec![(json!({"key": "k", "v": ["fresh"]}), true)]); + } + + #[test] + fn test_truncate_associative() { + // Truncation composes with associative (non-full) reduction across a + // spill: fenced stale entries are skipped, existence transfers onto the + // first fresh entry, and the associative drain emits the leftmost alone. + let schema = build_schema( + &url::Url::parse("http://example/schema").unwrap(), + &json!({ + "properties": { "v": { "type": "array", "reduce": { "strategy": "append" } } }, + "reduce": { "strategy": "merge" } + }), + ) + .unwrap(); + let spec = Spec::with_bindings( + [( + false, // Associative (not full) reduction. + vec![Extractor::with_default( + "/key", + &SerPolicy::noop(), + json!("def"), + )], + "test", + Validator::new(schema).unwrap(), + )], + Vec::new(), + ); + let mut acc = + crate::combine::Accumulator::new(spec, tempfile::tempfile().unwrap()).unwrap(); + let add = |acc: &mut crate::combine::Accumulator, v: &str, front: bool| { + let mt = acc.memtable().unwrap(); + let node = HeapNode::from_node(&json!({"key": "k", "v": [v]}), mt.alloc()); + mt.add(0, node, front).unwrap(); + }; + + add(&mut acc, "load", true); // Loaded front, pre-boundary. + add(&mut acc, "s", false); // Pre-boundary source. + force_spill(&mut acc); // Segment 0: [load(front), s]. + acc.truncate(0); // Fences segment 0. + add(&mut acc, "a", false); // Fresh sources, post-boundary. + add(&mut acc, "b", false); + + let out = acc + .into_drainer() + .unwrap() + .map_ok(|d| { + ( + serde_json::to_value(SerPolicy::noop().on_owned(&d.root)).unwrap(), + d.meta.front(), + ) + }) + .collect::, _>>() + .unwrap(); + + // `load`/`s` (fenced) are gone; existence transfers onto the first fresh + // entry `a`. Associative drain emits the leftmost alone, then `b`. + assert_eq!( + out, + vec![ + (json!({"key": "k", "v": ["a"]}), true), + (json!({"key": "k", "v": ["b"]}), false), + ], + ); + } + #[test] fn test_memtable_combine_reduce_sequence() { let key = vec![Extractor::with_default( @@ -1375,7 +1919,8 @@ mod test { // Read back all spilled documents and verify redaction let (spill, ranges) = spill.into_parts(); - let drainer = crate::combine::SpillDrainer::new(spec, spill, &ranges).unwrap(); + let drainer = + crate::combine::SpillDrainer::new(spec, spill, &ranges, Vec::new().into()).unwrap(); let docs: String = drainer .map(|doc| { diff --git a/crates/doc/src/combine/mod.rs b/crates/doc/src/combine/mod.rs index 9b187eb9b1f..177868c5f0d 100644 --- a/crates/doc/src/combine/mod.rs +++ b/crates/doc/src/combine/mod.rs @@ -125,16 +125,45 @@ pub use spill::{SpillDrainer, SpillWriter}; pub struct Accumulator { memtable: Option, spill: SpillWriter, + // Per-binding spill-segment cutoff: an entry in a segment whose ordinal is + // below `cutoffs[binding]` is stale. Accumulator-local; zeroed when the + // spill file is, so ordinals and fences restart together. + cutoffs: Box<[u32]>, } impl Accumulator { pub fn new(spec: Spec, spill: std::fs::File) -> Result { + let cutoffs = vec![0u32; spec.keys.len()].into_boxed_slice(); Ok(Self { memtable: Some(MemTable::new(spec)), spill: SpillWriter::new(spill)?, + cutoffs, }) } + /// Truncate `binding`'s backfill boundary: everything the Accumulator holds + /// for `binding` becomes stale — already-spilled segments are fenced by + /// ordinal, and the live MemTable drops its pre-boundary sources while + /// flagging its Loaded fronts stale (see [`MemTable::truncate`]). + /// + /// Callers MUST ensure every later `add` for `binding` is post-boundary, or + /// is explicitly stale via [`MemTable::add_stale_front`]; `truncate` only + /// reclassifies what is already present. + pub fn truncate(&mut self, binding: usize) { + let Self { + memtable: Some(memtable), + spill, + cutoffs, + } = self + else { + unreachable!("memtable is always Some"); + }; + + // Fence every segment written so far (all ordinals < the segment count). + cutoffs[binding] = spill.segment_ranges().len() as u32; + memtable.truncate(binding as u16); + } + /// Obtain an MemTable with available capacity. /// If the held MemTable is already over-capacity, it is first spilled and /// then replaced with a new instance, which is then returned. @@ -142,6 +171,7 @@ impl Accumulator { let Self { memtable: Some(memtable), spill, + .. } = self else { unreachable!("memtable is always Some"); @@ -169,16 +199,21 @@ impl Accumulator { /// Map this combine Accumulator into a Drainer, which will drain directly /// from the inner MemTable (if no spill occurred) or from an inner SpillDrainer. + /// Stale entries (flagged, or fenced by a segment cutoff) are dropped on + /// drain, transferring only their `front()` existence onto the fresh entry. pub fn into_drainer(self) -> Result { let Self { memtable: Some(memtable), mut spill, + cutoffs, } = self else { unreachable!("memtable must be Some"); }; if spill.segment_ranges().is_empty() { + // No segments spilled: cutoffs cannot fence anything, so staleness + // is entirely carried by the STALE flag. let (spill, _ranges) = spill.into_parts(); Ok(Drainer::Mem { @@ -190,8 +225,15 @@ impl Accumulator { let spec = memtable.spill(&mut spill, CHUNK_TARGET_SIZE)?; let (spill, ranges) = spill.into_parts(); + // Empty when nothing was truncated, so segment stamping short-circuits. + let cutoffs: Arc<[u32]> = if cutoffs.iter().all(|&c| c == 0) { + Vec::new().into() + } else { + cutoffs.into() + }; + Ok(Drainer::Spill { - drainer: SpillDrainer::new(spec, spill, &ranges)?, + drainer: SpillDrainer::new(spec, spill, &ranges, cutoffs)?, }) } } @@ -330,6 +372,15 @@ impl Meta { self.1 & META_FLAG_FRONT != 0 } + /// Is this entry stale (superseded by a backfill truncation)? A stale entry + /// is never validated or reduced; on drain it's discarded, transferring only + /// its `front()` existence onto the first fresh entry of the shared + /// (binding, key). + #[inline] + pub fn stale(&self) -> bool { + self.1 & META_FLAG_STALE != 0 + } + /// Is this entry known to be valid? Known-valid entries skip validation /// during spill/drain, and are assumed to not need redaction (validation /// drives redaction). @@ -377,6 +428,16 @@ impl Meta { fn set_not_associative(&mut self) { self.1 |= META_FLAG_NOT_ASSOCIATIVE; } + + #[inline] + fn set_front(&mut self) { + self.1 |= META_FLAG_FRONT; + } + + #[inline] + fn set_stale(&mut self) { + self.1 |= META_FLAG_STALE; + } } impl std::fmt::Debug for Meta { @@ -396,6 +457,9 @@ impl std::fmt::Debug for Meta { if self.known_valid() { s.field(&"V"); } + if self.stale() { + s.field(&"S"); + } s.finish() } } @@ -408,6 +472,8 @@ const META_FLAG_NOT_ASSOCIATIVE: u8 = 0x02; const META_FLAG_DELETED: u8 = 0x04; // Flag marking this entry is known to be valid against its schema. const META_FLAG_KNOWN_VALID: u8 = 0x08; +// Flag marking this entry is stale (superseded by a backfill truncation). +const META_FLAG_STALE: u8 = 0x10; // The number of used bytes within a Bump allocator. fn bump_mem_used(alloc: &bumpalo::Bump) -> usize { diff --git a/crates/doc/src/combine/spill.rs b/crates/doc/src/combine/spill.rs index e207058834a..049bbd16c6b 100644 --- a/crates/doc/src/combine/spill.rs +++ b/crates/doc/src/combine/spill.rs @@ -197,12 +197,31 @@ struct Segment { keys: Arc<[Box<[Extractor]>]>, // Keys for comparing Entries across Segments. next: Range, // Next chunk of this Segment. tail: bytes::Bytes, // Remainder of the current chunk. + ordinal: usize, // Index of this segment within the spill `ranges`. + cutoffs: Arc<[u32]>, // Per-binding spill-segment cutoffs (see Accumulator). } impl Segment { + /// Parse the next Entry and stamp it stale when this segment's `ordinal` is + /// below its binding's cutoff (the segment predates the binding's truncate). + /// Centralized so no parse site forgets to fence. + fn parse_entry( + ordinal: usize, + cutoffs: &[u32], + chunk: bytes::Bytes, + ) -> Result<(Entry, bytes::Bytes), io::Error> { + let (mut entry, rest) = Entry::parse(chunk)?; + if ordinal < cutoffs.get(entry.meta.binding()).copied().unwrap_or(0) as usize { + entry.meta.set_stale(); + } + Ok((entry, rest)) + } + /// Build a new Segment covering the given range of the spill file. fn new( keys: Arc<[Box<[Extractor]>]>, + cutoffs: Arc<[u32]>, + ordinal: usize, r: &mut R, range: Range, ) -> Result { @@ -250,13 +269,15 @@ impl Segment { } let chunk: bytes::Bytes = raw_buf.into_vec().into(); - let (head, tail) = Entry::parse(chunk)?; + let (head, tail) = Self::parse_entry(ordinal, &cutoffs, chunk)?; Ok(Self { head, keys, next, tail, + ordinal, + cutoffs, }) } @@ -271,10 +292,12 @@ impl Segment { keys, next, tail, + ordinal, + cutoffs, } = self; if !tail.is_empty() { - let (head, tail) = Entry::parse(tail)?; + let (head, tail) = Self::parse_entry(ordinal, &cutoffs, tail)?; Ok(( popped, @@ -283,10 +306,12 @@ impl Segment { keys, next, tail, + ordinal, + cutoffs, }), )) } else if !next.is_empty() { - Ok((popped, Some(Self::new(keys, r, next)?))) + Ok((popped, Some(Self::new(keys, cutoffs, ordinal, r, next)?))) } else { Ok((popped, None)) } @@ -297,10 +322,11 @@ impl Ord for Segment { fn cmp(&self, other: &Self) -> cmp::Ordering { let (l, r) = (&self.head, &other.head); - // Order entries on (binding, key, !front, spill-order): - // For each (binding, key), we take front() entries first, and then - // take the Segment which was produced into the spill file first. - // This maintains the left-to-right associative ordering of reductions. + // Order entries on (binding, key, stale, !front, spill-order): + // For each (binding, key), stale entries sort first and then front() + // entries, and then we take the Segment which was produced into the + // spill file first. This maintains the left-to-right associative + // ordering of reductions. // // `meta` contains a packed structure that's order-preserving over // (binding, key), so we first test it for inequality. @@ -309,6 +335,7 @@ impl Ord for Segment { .then_with(|| { Extractor::compare_key(&self.keys[l.meta.binding()], l.root.get(), r.root.get()) }) + .then_with(|| l.meta.stale().cmp(&r.meta.stale()).reverse()) .then_with(|| l.meta.front().cmp(&r.meta.front()).reverse()) .then_with(|| self.next.start.cmp(&other.next.start)) } @@ -351,7 +378,44 @@ impl SpillDrainer { self.heap.push(cmp::Reverse(segment)); } - let Entry { mut meta, root } = entry; + let Entry { mut meta, mut root } = entry; + + // Advance past stale entries to the first fresh entry of their key, + // ORing their front() existence onto it. A stale run with no fresh + // successor emits nothing. + let mut stale_front = false; + while meta.stale() { + stale_front |= meta.front(); + + let Some(cmp::Reverse(segment)) = self.heap.pop() else { + return Ok(None); // Trailing stale run: nothing to emit. + }; + let (next, segment) = segment.pop_head(&mut self.spill)?; + if let Some(segment) = segment { + self.heap.push(cmp::Reverse(segment)); + } + + let same_key = meta.0 == next.meta.0 + && Extractor::compare_key( + &self.spec.keys[meta.binding()], + root.get(), + next.root.get(), + ) + .is_eq(); + + Entry { meta, root } = next; + self.in_group = false; + + if !same_key { + // Orphaned stale run: drop its existence (next may differ in binding). + stale_front = false; + } + } + + if stale_front { + meta.set_front(); // Transfer stale existence onto the fresh output. + } + let is_full = self.spec.is_full[meta.binding()]; let key = self.spec.keys[meta.binding()].as_ref(); let validator = &mut self.spec.validators[meta.binding()]; @@ -478,12 +542,25 @@ impl Iterator for SpillDrainer { impl SpillDrainer { /// Build a new SpillDrainer which drains the given segment ranges previously - /// written to the spill file. - pub fn new(spec: Spec, mut spill: F, ranges: &[Range]) -> Result { + /// written to the spill file. `cutoffs` fences segments by ordinal: an entry + /// in the segment at ordinal `i` is stamped stale when `i < cutoffs[binding]`; + /// an empty `cutoffs` fences nothing. + pub fn new( + spec: Spec, + mut spill: F, + ranges: &[Range], + cutoffs: Arc<[u32]>, + ) -> Result { let mut heap = BinaryHeap::with_capacity(ranges.len()); - for range in ranges { - let segment = Segment::new(spec.keys.clone(), &mut spill, range.clone())?; + for (ordinal, range) in ranges.iter().enumerate() { + let segment = Segment::new( + spec.keys.clone(), + cutoffs.clone(), + ordinal, + &mut spill, + range.clone(), + )?; heap.push(cmp::Reverse(segment)); } @@ -555,7 +632,8 @@ mod test { "); // Parse the region as a Segment. - let mut segment = Segment::new(keys, &mut spill, ranges[0].clone()).unwrap(); + let mut segment = + Segment::new(keys, Vec::new().into(), 0, &mut spill, ranges[0].clone()).unwrap(); // First chunk has two documents. assert_eq!(segment.head.meta.binding(), 0); @@ -672,7 +750,7 @@ mod test { // Map from SpillWriter => SpillDrainer. let (spill, ranges) = spill.into_parts(); - let drainer = SpillDrainer::new(spec, spill, &ranges).unwrap(); + let drainer = SpillDrainer::new(spec, spill, &ranges, Vec::new().into()).unwrap(); let actual = drainer .map_ok(|doc| { @@ -808,7 +886,7 @@ mod test { spill.write_segment(&segment, CHUNK_TARGET_SIZE).unwrap(); } let (spill, ranges) = spill.into_parts(); - let mut drainer = SpillDrainer::new(spec, spill, &ranges).unwrap(); + let mut drainer = SpillDrainer::new(spec, spill, &ranges, Vec::new().into()).unwrap(); // "aaa" is front() & validated, and matches the schema. assert!(matches!( @@ -939,7 +1017,7 @@ mod test { // Read back through SpillDrainer and verify ordering let (spill, _) = spill.into_parts(); - let mut drainer = SpillDrainer::new(spec, spill, &ranges).unwrap(); + let mut drainer = SpillDrainer::new(spec, spill, &ranges, Vec::new().into()).unwrap(); let all_keys: Vec = std::iter::from_fn(|| drainer.next()) .map(|doc| { @@ -1018,7 +1096,7 @@ mod test { } let (spill, ranges) = spill.into_parts(); - let drainer = SpillDrainer::new(spec, spill, &ranges).unwrap(); + let drainer = SpillDrainer::new(spec, spill, &ranges, Vec::new().into()).unwrap(); let actual = drainer .map_ok(|doc| { @@ -1078,6 +1156,97 @@ mod test { "###); } + #[test] + fn test_stale_flag_header_roundtrip() { + // A STALE flag set in memory must survive the 24-byte entry header's + // persisted flags byte, independent of any ordinal fence. + let alloc = Bump::new(); + let mut meta = Meta::new(0, &[], false, true); + meta.set_stale(); + let entries = vec![HeapEntry { + meta, + root: HeapRoot::from_heap_node(HeapNode::from_node(&json!({"key": "k"}), &alloc)), + }]; + + let mut spill = SpillWriter::new(io::Cursor::new(Vec::new())).unwrap(); + spill.write_segment(&entries, CHUNK_TARGET_SIZE).unwrap(); + let (mut spill, ranges) = spill.into_parts(); + + // Empty cutoffs fence nothing, so staleness can only come from the + // persisted flag. + let keys: Arc<[Box<[Extractor]>]> = Vec::new().into(); + let segment = + Segment::new(keys, Vec::new().into(), 0, &mut spill, ranges[0].clone()).unwrap(); + assert!(segment.head.meta.stale()); + assert_eq!(segment.head.meta.binding(), 0); + assert!(segment.head.meta.known_valid()); + } + + #[test] + fn test_cross_segment_cutoff() { + // A front Loaded and a fresh source of one key, split across two spill + // segments, still partition on drain when a cutoff fences the older + // segment by ordinal; a stale-only key emits nothing. + let schema = json::schema::build( + &url::Url::parse("http://example/schema").unwrap(), + &json!({ + "properties": { "v": { "type": "array", "reduce": { "strategy": "append" } } }, + "reduce": { "strategy": "merge" } + }), + ) + .unwrap(); + let spec = Spec::with_one_binding( + true, + vec![Extractor::with_default( + "/key", + &SerPolicy::noop(), + json!("def"), + )], + "source", + Vec::new(), + Validator::new(schema).unwrap(), + ); + + let alloc = Bump::new(); + // Segment 0 (fenced by the cutoff) holds a front Loaded for "k" and a + // stale-only key "z"; segment 1 holds the fresh source for "k". + let fixtures = vec![ + segment_fixture( + &[ + (0, json!({"key": "k", "v": ["stale"]}), true), + (0, json!({"key": "z", "v": ["orphan"]}), true), + ], + &alloc, + ), + segment_fixture(&[(0, json!({"key": "k", "v": ["fresh"]}), false)], &alloc), + ]; + + let mut spill = SpillWriter::new(io::Cursor::new(Vec::new())).unwrap(); + for segment in fixtures { + spill.write_segment(&segment, CHUNK_TARGET_SIZE).unwrap(); + } + let (spill, ranges) = spill.into_parts(); + + // cutoffs[0] = 1 fences segment 0 (ordinal 0 < 1), stamping its entries + // stale; segment 1 (ordinal 1) is unfenced. + let drained = SpillDrainer::new(spec, spill, &ranges, vec![1u32].into()) + .unwrap() + .map_ok(|doc| { + ( + serde_json::to_value(SerPolicy::noop().on_owned(&doc.root)).unwrap(), + doc.meta.front(), + ) + }) + .collect::, _>>() + .unwrap(); + + assert_eq!( + drained, + vec![(json!({"key": "k", "v": ["fresh"]}), true)], + "stale content dropped, existence transferred; orphan 'z' emits nothing", + ); + } + fn to_hex(b: &[u8]) -> String { hexdump::hexdump_iter(b) .map(|line| format!("{line}")) diff --git a/crates/doc/tests/merge_patch_fuzz.rs b/crates/doc/tests/merge_patch_fuzz.rs index 927efeca323..55398019880 100644 --- a/crates/doc/tests/merge_patch_fuzz.rs +++ b/crates/doc/tests/merge_patch_fuzz.rs @@ -134,7 +134,8 @@ fn reduce_combiner(input: Vec) -> bool { let spec = memtable_2.spill(&mut spill, 1 << 18).unwrap(); let (mut spill, ranges) = spill.into_parts(); - let mut spill_drainer = SpillDrainer::new(spec, &mut spill, &ranges).unwrap(); + let mut spill_drainer = + SpillDrainer::new(spec, &mut spill, &ranges, Vec::new().into()).unwrap(); let mut actual_associative = None; let mut actual_full = json!(null); diff --git a/crates/doc/tests/spill_merge_fuzz.rs b/crates/doc/tests/spill_merge_fuzz.rs index f05aababfa2..cec6ffa5601 100644 --- a/crates/doc/tests/spill_merge_fuzz.rs +++ b/crates/doc/tests/spill_merge_fuzz.rs @@ -182,7 +182,7 @@ fn run_sequence(seq: Vec<(u8, u8, bool, bool)>) -> Result<(), FuzzError> { // Spill the final memtable_spill and drain via SpillDrainer. let spec = memtable_spill.spill(&mut spill, chunk_target).unwrap(); let (spill, ranges) = spill.into_parts(); - let mut spill_drainer = combine::SpillDrainer::new(spec, spill, &ranges)?; + let mut spill_drainer = combine::SpillDrainer::new(spec, spill, &ranges, Vec::new().into())?; let mut expect_it = expect_full.into_iter(); diff --git a/crates/e2e-support/tests/hello_world.rs b/crates/e2e-support/tests/hello_world.rs index 54547d551d5..d269f0cf3b4 100644 --- a/crates/e2e-support/tests/hello_world.rs +++ b/crates/e2e-support/tests/hello_world.rs @@ -110,7 +110,7 @@ async fn hello_world(build: Arc, journal_client: gazette::journal // Build and write ACK intent documents. let (producer, commit_clock, journals) = publisher.commit_intents(); let journal_acks = - publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)]); + publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)], None); publisher .write_intents(journal_acks) diff --git a/crates/flowctl/src/raw/preview_next/publish.rs b/crates/flowctl/src/raw/preview_next/publish.rs index 9b0c5000114..5d2224d839d 100644 --- a/crates/flowctl/src/raw/preview_next/publish.rs +++ b/crates/flowctl/src/raw/preview_next/publish.rs @@ -98,6 +98,28 @@ impl runtime_next::Publisher for PreviewPublisher { Ok(()) } + async fn marker_commit( + &mut self, + _binding_index: usize, + ) -> tonic::Result< + Option<( + proto_gazette::uuid::Producer, + proto_gazette::uuid::Clock, + Vec, + )>, + > { + // No journal IO: no backfill marker is broadcast. + Ok(None) + } + + async fn apply_truncated_at_labels( + &mut self, + _active_backfills: &BTreeMap, + ) -> tonic::Result<()> { + // No journal IO: there are no `truncated-at` journal labels to apply. + Ok(()) + } + fn commit_intents( &mut self, ) -> Option<( diff --git a/crates/labels/src/lib.rs b/crates/labels/src/lib.rs index 970aab140c6..6f27eb81671 100644 --- a/crates/labels/src/lib.rs +++ b/crates/labels/src/lib.rs @@ -18,6 +18,7 @@ pub const KEY_BEGIN_MIN: &str = "00000000"; pub const KEY_END: &str = "estuary.dev/key-end"; pub const KEY_END_MAX: &str = "ffffffff"; pub const MANAGED_BY_FLOW: &str = "estuary.dev/flow"; +pub const TRUNCATED_AT: &str = "estuary.dev/truncated-at"; // ShardSpec labels. pub const TASK_NAME: &str = "estuary.dev/task-name"; @@ -187,10 +188,12 @@ pub fn is_data_plane_label(label: &str) -> bool { return true; } match label { - // Key and R-Clock splits are performed within the data-plane. - CORDON | KEY_BEGIN | KEY_END | RCLOCK_BEGIN | RCLOCK_END | SPLIT_SOURCE | SPLIT_TARGET => { - true - } + // Labels the data-plane runtime applies to live journals/shards — key, + // r-clock, and shard splits, cordoning, and the backfill truncation + // boundary — which activation and partition splits must preserve rather + // than rebuild away. + CORDON | KEY_BEGIN | KEY_END | RCLOCK_BEGIN | RCLOCK_END | SPLIT_SOURCE | SPLIT_TARGET + | TRUNCATED_AT => true, _ => false, } } @@ -218,6 +221,21 @@ pub fn expect_one_u32(set: &LabelSet, name: &str) -> Result { Ok(parsed) } +/// Format a Gazette message clock (`u64`) as a [`TRUNCATED_AT`] label value: a +/// fixed-width, 16-character lowercase hex string. +pub fn truncated_at_value(clock: u64) -> String { + format!("{clock:016x}") +} + +/// Parse a [`TRUNCATED_AT`] label value produced by [`truncated_at_value`] back +/// into its `u64` clock. +pub fn parse_truncated_at(value: &str) -> Result { + u64::from_str_radix(value, 16).map_err(|_| Error::InvalidValue { + name: TRUNCATED_AT.to_string(), + value: value.to_string(), + }) +} + pub fn expect_one<'s>(set: &'s LabelSet, name: &str) -> Result<&'s str, Error> { let labels = values(set, name); @@ -634,4 +652,32 @@ mod test { .unwrap_err(); assert!(matches!(err, Error::NotSorted(..))); } + + #[test] + fn truncated_at_value_roundtrips_and_sorts() { + // Always 16 lowercase hex chars, and round-trips. + for clock in [0u64, 1, 0xdead_beef, u64::MAX, 1_750_000_000 << 4] { + let v = super::truncated_at_value(clock); + assert_eq!(v.len(), 16, "value {v:?} is not fixed-width"); + assert_eq!(super::parse_truncated_at(&v).unwrap(), clock); + } + // Lexical (string) order matches clock (numeric) order. + let mut by_value = [3u64, 1 << 40, 2, u64::MAX, 1 << 4]; + let mut numeric = by_value; + by_value.sort_by_key(|c| super::truncated_at_value(*c)); + numeric.sort(); + assert_eq!(by_value, numeric); + + // Unparseable values error (a stale 19-digit decimal that overflows + // u64, non-hex, empty). + assert!(super::parse_truncated_at("1234605616436508552").is_err()); + assert!(super::parse_truncated_at("zzzzzzzzzzzzzzzz").is_err()); + assert!(super::parse_truncated_at("").is_err()); + // We trust our own encoding, so non-canonical-but-parseable forms + // (uppercase hex here) are accepted rather than rejected. + assert_eq!( + super::parse_truncated_at("DEADBEEF00000001").unwrap(), + 0xDEAD_BEEF_0000_0001, + ); + } } diff --git a/crates/proto-flow/src/capture.rs b/crates/proto-flow/src/capture.rs index c891a81aa55..8ef59a53476 100644 --- a/crates/proto-flow/src/capture.rs +++ b/crates/proto-flow/src/capture.rs @@ -194,6 +194,10 @@ pub struct Response { pub sourced_schema: ::core::option::Option, #[prost(message, optional, tag = "7")] pub checkpoint: ::core::option::Option, + #[prost(message, optional, tag = "9")] + pub backfill_begin: ::core::option::Option, + #[prost(message, optional, tag = "10")] + pub backfill_complete: ::core::option::Option, /// Reserved for internal use. #[prost(bytes = "bytes", tag = "100")] pub internal: ::prost::bytes::Bytes, @@ -384,4 +388,24 @@ pub mod response { #[prost(message, optional, tag = "1")] pub state: ::core::option::Option, } + /// Signals the start of a backfill for a binding. + /// + /// A backfill message (BackfillBegin or BackfillComplete) must stand alone + /// in its connector checkpoint: the checkpoint must contain only the + /// backfill message followed by the terminating Checkpoint response, with + /// no Captured, SourcedSchema, or other backfill messages. The runtime + /// enforces this rule and will fail the session on violation. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct BackfillBegin { + #[prost(uint32, tag = "1")] + pub binding: u32, + } + /// Signals the end of a backfill for a binding. + /// + /// See BackfillBegin for the "stands alone in its checkpoint" rule. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct BackfillComplete { + #[prost(uint32, tag = "1")] + pub binding: u32, + } } diff --git a/crates/proto-flow/src/capture.serde.rs b/crates/proto-flow/src/capture.serde.rs index 9b7c83db4a0..d4110912e3d 100644 --- a/crates/proto-flow/src/capture.serde.rs +++ b/crates/proto-flow/src/capture.serde.rs @@ -1273,6 +1273,12 @@ impl serde::Serialize for Response { if self.checkpoint.is_some() { len += 1; } + if self.backfill_begin.is_some() { + len += 1; + } + if self.backfill_complete.is_some() { + len += 1; + } if !self.internal.is_empty() { len += 1; } @@ -1301,6 +1307,12 @@ impl serde::Serialize for Response { if let Some(v) = self.checkpoint.as_ref() { struct_ser.serialize_field("checkpoint", v)?; } + if let Some(v) = self.backfill_begin.as_ref() { + struct_ser.serialize_field("backfillBegin", v)?; + } + if let Some(v) = self.backfill_complete.as_ref() { + struct_ser.serialize_field("backfillComplete", v)?; + } if !self.internal.is_empty() { #[allow(clippy::needless_borrow)] #[allow(clippy::needless_borrows_for_generic_args)] @@ -1325,6 +1337,10 @@ impl<'de> serde::Deserialize<'de> for Response { "sourced_schema", "sourcedSchema", "checkpoint", + "backfill_begin", + "backfillBegin", + "backfill_complete", + "backfillComplete", "internal", "$internal", ]; @@ -1339,6 +1355,8 @@ impl<'de> serde::Deserialize<'de> for Response { Captured, SourcedSchema, Checkpoint, + BackfillBegin, + BackfillComplete, Internal, __SkipField__, } @@ -1370,6 +1388,8 @@ impl<'de> serde::Deserialize<'de> for Response { "captured" => Ok(GeneratedField::Captured), "sourcedSchema" | "sourced_schema" => Ok(GeneratedField::SourcedSchema), "checkpoint" => Ok(GeneratedField::Checkpoint), + "backfillBegin" | "backfill_begin" => Ok(GeneratedField::BackfillBegin), + "backfillComplete" | "backfill_complete" => Ok(GeneratedField::BackfillComplete), "$internal" | "internal" => Ok(GeneratedField::Internal), _ => Ok(GeneratedField::__SkipField__), } @@ -1398,6 +1418,8 @@ impl<'de> serde::Deserialize<'de> for Response { let mut captured__ = None; let mut sourced_schema__ = None; let mut checkpoint__ = None; + let mut backfill_begin__ = None; + let mut backfill_complete__ = None; let mut internal__ = None; while let Some(k) = map_.next_key()? { match k { @@ -1449,6 +1471,18 @@ impl<'de> serde::Deserialize<'de> for Response { } checkpoint__ = map_.next_value()?; } + GeneratedField::BackfillBegin => { + if backfill_begin__.is_some() { + return Err(serde::de::Error::duplicate_field("backfillBegin")); + } + backfill_begin__ = map_.next_value()?; + } + GeneratedField::BackfillComplete => { + if backfill_complete__.is_some() { + return Err(serde::de::Error::duplicate_field("backfillComplete")); + } + backfill_complete__ = map_.next_value()?; + } GeneratedField::Internal => { if internal__.is_some() { return Err(serde::de::Error::duplicate_field("$internal")); @@ -1471,6 +1505,8 @@ impl<'de> serde::Deserialize<'de> for Response { captured: captured__, sourced_schema: sourced_schema__, checkpoint: checkpoint__, + backfill_begin: backfill_begin__, + backfill_complete: backfill_complete__, internal: internal__.unwrap_or_default(), }) } @@ -1591,6 +1627,200 @@ impl<'de> serde::Deserialize<'de> for response::Applied { deserializer.deserialize_struct("capture.Response.Applied", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for response::BackfillBegin { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.binding != 0 { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("capture.Response.BackfillBegin", len)?; + if self.binding != 0 { + struct_ser.serialize_field("binding", &self.binding)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for response::BackfillBegin { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "binding", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Binding, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "binding" => Ok(GeneratedField::Binding), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = response::BackfillBegin; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct capture.Response.BackfillBegin") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut binding__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Binding => { + if binding__.is_some() { + return Err(serde::de::Error::duplicate_field("binding")); + } + binding__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(response::BackfillBegin { + binding: binding__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("capture.Response.BackfillBegin", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for response::BackfillComplete { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.binding != 0 { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("capture.Response.BackfillComplete", len)?; + if self.binding != 0 { + struct_ser.serialize_field("binding", &self.binding)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for response::BackfillComplete { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "binding", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Binding, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "binding" => Ok(GeneratedField::Binding), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = response::BackfillComplete; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct capture.Response.BackfillComplete") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut binding__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Binding => { + if binding__.is_some() { + return Err(serde::de::Error::duplicate_field("binding")); + } + binding__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(response::BackfillComplete { + binding: binding__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("capture.Response.BackfillComplete", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for response::Captured { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/crates/proto-flow/src/materialize.rs b/crates/proto-flow/src/materialize.rs index 4bc629a2acc..ddef2104ef3 100644 --- a/crates/proto-flow/src/materialize.rs +++ b/crates/proto-flow/src/materialize.rs @@ -191,7 +191,7 @@ pub mod request { /// Flush loads. No further Loads will be sent in this transaction, /// and the runtime will await the connectors's remaining Loaded /// responses followed by one Flushed response. - #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] + #[derive(Clone, PartialEq, ::prost::Message)] pub struct Flush { /// Aggregated state patches from all shards' prior-transaction Acknowledged /// responses, as a tab-delimited JSON array. Includes this shard's own @@ -200,6 +200,39 @@ pub mod request { /// cooperative multi-shard strategies use this to observe peers' state. #[prost(bytes = "bytes", tag = "1")] pub state_patches_json: ::prost::bytes::Bytes, + #[prost(message, repeated, tag = "2")] + pub backfill_begins: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "3")] + pub backfill_completes: ::prost::alloc::vec::Vec, + } + /// Nested message and enum types in `Flush`. + pub mod flush { + /// Backfill-begin signals observed during this transaction. A connector + /// acts on those relevant to its key range. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct BackfillBegin { + #[prost(uint32, tag = "1")] + pub binding: u32, + /// Truncation boundary of the binding's backfill: documents published at or + /// after this time are current, while earlier ones were superseded by the + /// backfill. It equals the begin's own publication time (flow_published_at), + /// and is carried on both begin and complete so connectors need not track it + /// across transactions. + #[prost(message, optional, tag = "2")] + pub timestamp: ::core::option::Option<::pbjson_types::Timestamp>, + } + /// Backfill-complete signals observed during this transaction. See + /// BackfillBegin. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct BackfillComplete { + #[prost(uint32, tag = "1")] + pub binding: u32, + /// Truncation boundary of the completed backfill (see BackfillBegin.timestamp): + /// the connector may delete destination rows whose flow_published_at predates + /// this time, as they were superseded by the backfill. + #[prost(message, optional, tag = "2")] + pub timestamp: ::core::option::Option<::pbjson_types::Timestamp>, + } } /// Store documents updated by the current transaction. /// diff --git a/crates/proto-flow/src/materialize.serde.rs b/crates/proto-flow/src/materialize.serde.rs index 4b78b655139..4ba697a6446 100644 --- a/crates/proto-flow/src/materialize.serde.rs +++ b/crates/proto-flow/src/materialize.serde.rs @@ -833,12 +833,24 @@ impl serde::Serialize for request::Flush { if !self.state_patches_json.is_empty() { len += 1; } + if !self.backfill_begins.is_empty() { + len += 1; + } + if !self.backfill_completes.is_empty() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("materialize.Request.Flush", len)?; if !self.state_patches_json.is_empty() { #[allow(clippy::needless_borrow)] #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("statePatches", &crate::as_raw_json(&self.state_patches_json)?)?; } + if !self.backfill_begins.is_empty() { + struct_ser.serialize_field("backfillBegins", &self.backfill_begins)?; + } + if !self.backfill_completes.is_empty() { + struct_ser.serialize_field("backfillCompletes", &self.backfill_completes)?; + } struct_ser.end() } } @@ -851,11 +863,17 @@ impl<'de> serde::Deserialize<'de> for request::Flush { const FIELDS: &[&str] = &[ "state_patches_json", "statePatches", + "backfill_begins", + "backfillBegins", + "backfill_completes", + "backfillCompletes", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { StatePatchesJson, + BackfillBegins, + BackfillCompletes, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -879,6 +897,8 @@ impl<'de> serde::Deserialize<'de> for request::Flush { { match value { "statePatches" | "state_patches_json" => Ok(GeneratedField::StatePatchesJson), + "backfillBegins" | "backfill_begins" => Ok(GeneratedField::BackfillBegins), + "backfillCompletes" | "backfill_completes" => Ok(GeneratedField::BackfillCompletes), _ => Ok(GeneratedField::__SkipField__), } } @@ -899,6 +919,8 @@ impl<'de> serde::Deserialize<'de> for request::Flush { V: serde::de::MapAccess<'de>, { let mut state_patches_json__ = None; + let mut backfill_begins__ = None; + let mut backfill_completes__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::StatePatchesJson => { @@ -909,6 +931,18 @@ impl<'de> serde::Deserialize<'de> for request::Flush { Some(map_.next_value::()?.0) ; } + GeneratedField::BackfillBegins => { + if backfill_begins__.is_some() { + return Err(serde::de::Error::duplicate_field("backfillBegins")); + } + backfill_begins__ = Some(map_.next_value()?); + } + GeneratedField::BackfillCompletes => { + if backfill_completes__.is_some() { + return Err(serde::de::Error::duplicate_field("backfillCompletes")); + } + backfill_completes__ = Some(map_.next_value()?); + } GeneratedField::__SkipField__ => { let _ = map_.next_value::()?; } @@ -916,12 +950,242 @@ impl<'de> serde::Deserialize<'de> for request::Flush { } Ok(request::Flush { state_patches_json: state_patches_json__.unwrap_or_default(), + backfill_begins: backfill_begins__.unwrap_or_default(), + backfill_completes: backfill_completes__.unwrap_or_default(), }) } } deserializer.deserialize_struct("materialize.Request.Flush", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for request::flush::BackfillBegin { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.binding != 0 { + len += 1; + } + if self.timestamp.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("materialize.Request.Flush.BackfillBegin", len)?; + if self.binding != 0 { + struct_ser.serialize_field("binding", &self.binding)?; + } + if let Some(v) = self.timestamp.as_ref() { + struct_ser.serialize_field("timestamp", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for request::flush::BackfillBegin { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "binding", + "timestamp", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Binding, + Timestamp, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "binding" => Ok(GeneratedField::Binding), + "timestamp" => Ok(GeneratedField::Timestamp), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = request::flush::BackfillBegin; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct materialize.Request.Flush.BackfillBegin") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut binding__ = None; + let mut timestamp__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Binding => { + if binding__.is_some() { + return Err(serde::de::Error::duplicate_field("binding")); + } + binding__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::Timestamp => { + if timestamp__.is_some() { + return Err(serde::de::Error::duplicate_field("timestamp")); + } + timestamp__ = map_.next_value()?; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(request::flush::BackfillBegin { + binding: binding__.unwrap_or_default(), + timestamp: timestamp__, + }) + } + } + deserializer.deserialize_struct("materialize.Request.Flush.BackfillBegin", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for request::flush::BackfillComplete { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.binding != 0 { + len += 1; + } + if self.timestamp.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("materialize.Request.Flush.BackfillComplete", len)?; + if self.binding != 0 { + struct_ser.serialize_field("binding", &self.binding)?; + } + if let Some(v) = self.timestamp.as_ref() { + struct_ser.serialize_field("timestamp", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for request::flush::BackfillComplete { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "binding", + "timestamp", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Binding, + Timestamp, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "binding" => Ok(GeneratedField::Binding), + "timestamp" => Ok(GeneratedField::Timestamp), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = request::flush::BackfillComplete; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct materialize.Request.Flush.BackfillComplete") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut binding__ = None; + let mut timestamp__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Binding => { + if binding__.is_some() { + return Err(serde::de::Error::duplicate_field("binding")); + } + binding__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::Timestamp => { + if timestamp__.is_some() { + return Err(serde::de::Error::duplicate_field("timestamp")); + } + timestamp__ = map_.next_value()?; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(request::flush::BackfillComplete { + binding: binding__.unwrap_or_default(), + timestamp: timestamp__, + }) + } + } + deserializer.deserialize_struct("materialize.Request.Flush.BackfillComplete", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for request::Load { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/crates/proto-flow/src/runtime.rs b/crates/proto-flow/src/runtime.rs index d5abb24ac04..ec5d962982e 100644 --- a/crates/proto-flow/src/runtime.rs +++ b/crates/proto-flow/src/runtime.rs @@ -578,6 +578,12 @@ pub struct Recover { /// Persisted trigger parameters (materialize only), or empty. #[prost(bytes = "bytes", tag = "10")] pub trigger_params_json: ::prost::bytes::Bytes, + /// Active-backfill begin clocks, keyed by binding index. Restored so the + /// capture runtime can re-apply truncated-at journal labels on startup and + /// resolve a BackfillComplete's truncated_at. Resolved from "AB:{state_key}" + /// keys by the scan. + #[prost(btree_map = "uint32, fixed64", tag = "11")] + pub active_backfills: ::prost::alloc::collections::BTreeMap, } /// Persist is sent by the leader to shard zero when state must be durably /// written. Each field maps to a contractual WriteBatch effect on shard @@ -668,6 +674,35 @@ pub struct Persist { /// Effect: after the WriteBatch commits, scan and reply `Recover` not `Persisted`. #[prost(bool, tag = "17")] pub rescan: bool, + /// The active-backfill change this transaction observed, if any. At most one + /// per commit — a backfill control signal stands alone in its transaction. + #[prost(oneof = "persist::ActiveBackfillChange", tags = "18, 19")] + pub active_backfill_change: ::core::option::Option, +} +/// Nested message and enum types in `Persist`. +pub mod persist { + /// The active-backfill change this transaction observed, if any. At most one + /// per commit — a backfill control signal stands alone in its transaction. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum ActiveBackfillChange { + /// BackfillBegin: record the binding's begin clock. + /// Effect: Put fixed64-LE under "AB:{state_key}" (state_key resolved by the encoder). + #[prost(message, tag = "18")] + Begin(super::ActiveBackfillBegin), + /// BackfillComplete: clear the binding's active-backfill entry. + /// Effect: Delete "AB:{state_key}" (state_key resolved by the encoder). + #[prost(uint32, tag = "19")] + CompleteBinding(u32), + } +} +/// ActiveBackfillBegin records a binding's backfill begin clock — its +/// authoritative truncated_at — staged by a committing Persist. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ActiveBackfillBegin { + #[prost(uint32, tag = "1")] + pub binding: u32, + #[prost(fixed64, tag = "2")] + pub truncated_at: u64, } /// Persisted is sent by shard zero to the leader after the state is durable /// in the recovery log. @@ -996,12 +1031,48 @@ pub mod materialize { } } /// Leader → Shards. Signals end of Load phase. - #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] + #[derive(Clone, PartialEq, ::prost::Message)] pub struct Flush { /// Prior transaction's aggregated C:Acknowledged state patches. /// State Update Wire Format. #[prost(bytes = "bytes", tag = "1")] pub connector_patches_json: ::prost::bytes::Bytes, + /// Backfill-begin markers observed during this transaction (the leader's + /// per-transaction delta). Each shard forwards them to its connector as a + /// C:Flush notification. The shuffle reads fold each marker exactly once per + /// committed generation, so the set is already a delta — no leader-side + /// deduplication. + #[prost(message, repeated, tag = "2")] + pub backfill_begins: ::prost::alloc::vec::Vec, + /// Backfill-complete markers observed during this transaction. Forwarded like + /// `backfill_begins`. + #[prost(message, repeated, tag = "3")] + pub backfill_completes: ::prost::alloc::vec::Vec, + } + /// Nested message and enum types in `Flush`. + pub mod flush { + /// A backfill-begin marker: a binding index and the begin clock (the + /// truncation boundary). + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct BackfillBegin { + /// Binding index. + #[prost(uint32, tag = "1")] + pub binding: u32, + /// Begin clock: the backfill's truncation boundary. + #[prost(fixed64, tag = "2")] + pub clock: u64, + } + /// A backfill-complete marker; same shape as BackfillBegin, where `clock` is + /// the completed backfill's begin (truncation) boundary. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct BackfillComplete { + /// Binding index. + #[prost(uint32, tag = "1")] + pub binding: u32, + /// Begin clock the completed backfill reported (its truncation boundary). + #[prost(fixed64, tag = "2")] + pub clock: u64, + } } /// Shard → Leader. Flush phase complete. /// Reports connector state patches and max-key deltas from C:Flushed. diff --git a/crates/proto-flow/src/runtime.serde.rs b/crates/proto-flow/src/runtime.serde.rs index 0215a7d1467..fe3bc873717 100644 --- a/crates/proto-flow/src/runtime.serde.rs +++ b/crates/proto-flow/src/runtime.serde.rs @@ -1,3 +1,122 @@ +impl serde::Serialize for ActiveBackfillBegin { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.binding != 0 { + len += 1; + } + if self.truncated_at != 0 { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("runtime.ActiveBackfillBegin", len)?; + if self.binding != 0 { + struct_ser.serialize_field("binding", &self.binding)?; + } + if self.truncated_at != 0 { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("truncatedAt", ToString::to_string(&self.truncated_at).as_str())?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for ActiveBackfillBegin { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "binding", + "truncated_at", + "truncatedAt", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Binding, + TruncatedAt, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "binding" => Ok(GeneratedField::Binding), + "truncatedAt" | "truncated_at" => Ok(GeneratedField::TruncatedAt), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = ActiveBackfillBegin; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.ActiveBackfillBegin") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut binding__ = None; + let mut truncated_at__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Binding => { + if binding__.is_some() { + return Err(serde::de::Error::duplicate_field("binding")); + } + binding__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::TruncatedAt => { + if truncated_at__.is_some() { + return Err(serde::de::Error::duplicate_field("truncatedAt")); + } + truncated_at__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(ActiveBackfillBegin { + binding: binding__.unwrap_or_default(), + truncated_at: truncated_at__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("runtime.ActiveBackfillBegin", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for Applied { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result @@ -6431,12 +6550,24 @@ impl serde::Serialize for materialize::Flush { if !self.connector_patches_json.is_empty() { len += 1; } + if !self.backfill_begins.is_empty() { + len += 1; + } + if !self.backfill_completes.is_empty() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("runtime.Materialize.Flush", len)?; if !self.connector_patches_json.is_empty() { #[allow(clippy::needless_borrow)] #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("connectorPatches", &crate::as_raw_json(&self.connector_patches_json)?)?; } + if !self.backfill_begins.is_empty() { + struct_ser.serialize_field("backfillBegins", &self.backfill_begins)?; + } + if !self.backfill_completes.is_empty() { + struct_ser.serialize_field("backfillCompletes", &self.backfill_completes)?; + } struct_ser.end() } } @@ -6449,11 +6580,17 @@ impl<'de> serde::Deserialize<'de> for materialize::Flush { const FIELDS: &[&str] = &[ "connector_patches_json", "connectorPatches", + "backfill_begins", + "backfillBegins", + "backfill_completes", + "backfillCompletes", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { ConnectorPatchesJson, + BackfillBegins, + BackfillCompletes, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -6477,6 +6614,8 @@ impl<'de> serde::Deserialize<'de> for materialize::Flush { { match value { "connectorPatches" | "connector_patches_json" => Ok(GeneratedField::ConnectorPatchesJson), + "backfillBegins" | "backfill_begins" => Ok(GeneratedField::BackfillBegins), + "backfillCompletes" | "backfill_completes" => Ok(GeneratedField::BackfillCompletes), _ => Ok(GeneratedField::__SkipField__), } } @@ -6497,6 +6636,8 @@ impl<'de> serde::Deserialize<'de> for materialize::Flush { V: serde::de::MapAccess<'de>, { let mut connector_patches_json__ = None; + let mut backfill_begins__ = None; + let mut backfill_completes__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::ConnectorPatchesJson => { @@ -6507,6 +6648,18 @@ impl<'de> serde::Deserialize<'de> for materialize::Flush { Some(map_.next_value::()?.0) ; } + GeneratedField::BackfillBegins => { + if backfill_begins__.is_some() { + return Err(serde::de::Error::duplicate_field("backfillBegins")); + } + backfill_begins__ = Some(map_.next_value()?); + } + GeneratedField::BackfillCompletes => { + if backfill_completes__.is_some() { + return Err(serde::de::Error::duplicate_field("backfillCompletes")); + } + backfill_completes__ = Some(map_.next_value()?); + } GeneratedField::__SkipField__ => { let _ = map_.next_value::()?; } @@ -6514,12 +6667,250 @@ impl<'de> serde::Deserialize<'de> for materialize::Flush { } Ok(materialize::Flush { connector_patches_json: connector_patches_json__.unwrap_or_default(), + backfill_begins: backfill_begins__.unwrap_or_default(), + backfill_completes: backfill_completes__.unwrap_or_default(), }) } } deserializer.deserialize_struct("runtime.Materialize.Flush", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for materialize::flush::BackfillBegin { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.binding != 0 { + len += 1; + } + if self.clock != 0 { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("runtime.Materialize.Flush.BackfillBegin", len)?; + if self.binding != 0 { + struct_ser.serialize_field("binding", &self.binding)?; + } + if self.clock != 0 { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("clock", ToString::to_string(&self.clock).as_str())?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for materialize::flush::BackfillBegin { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "binding", + "clock", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Binding, + Clock, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "binding" => Ok(GeneratedField::Binding), + "clock" => Ok(GeneratedField::Clock), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = materialize::flush::BackfillBegin; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.Materialize.Flush.BackfillBegin") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut binding__ = None; + let mut clock__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Binding => { + if binding__.is_some() { + return Err(serde::de::Error::duplicate_field("binding")); + } + binding__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::Clock => { + if clock__.is_some() { + return Err(serde::de::Error::duplicate_field("clock")); + } + clock__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(materialize::flush::BackfillBegin { + binding: binding__.unwrap_or_default(), + clock: clock__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("runtime.Materialize.Flush.BackfillBegin", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for materialize::flush::BackfillComplete { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.binding != 0 { + len += 1; + } + if self.clock != 0 { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("runtime.Materialize.Flush.BackfillComplete", len)?; + if self.binding != 0 { + struct_ser.serialize_field("binding", &self.binding)?; + } + if self.clock != 0 { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("clock", ToString::to_string(&self.clock).as_str())?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for materialize::flush::BackfillComplete { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "binding", + "clock", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Binding, + Clock, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "binding" => Ok(GeneratedField::Binding), + "clock" => Ok(GeneratedField::Clock), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = materialize::flush::BackfillComplete; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct runtime.Materialize.Flush.BackfillComplete") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut binding__ = None; + let mut clock__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Binding => { + if binding__.is_some() { + return Err(serde::de::Error::duplicate_field("binding")); + } + binding__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::Clock => { + if clock__.is_some() { + return Err(serde::de::Error::duplicate_field("clock")); + } + clock__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(materialize::flush::BackfillComplete { + binding: binding__.unwrap_or_default(), + clock: clock__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("runtime.Materialize.Flush.BackfillComplete", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for materialize::Flushed { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result @@ -8396,6 +8787,9 @@ impl serde::Serialize for Persist { if self.rescan { len += 1; } + if self.active_backfill_change.is_some() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("runtime.Persist", len)?; if self.seq_no != 0 { #[allow(clippy::needless_borrow)] @@ -8464,6 +8858,16 @@ impl serde::Serialize for Persist { if self.rescan { struct_ser.serialize_field("rescan", &self.rescan)?; } + if let Some(v) = self.active_backfill_change.as_ref() { + match v { + persist::ActiveBackfillChange::Begin(v) => { + struct_ser.serialize_field("begin", v)?; + } + persist::ActiveBackfillChange::CompleteBinding(v) => { + struct_ser.serialize_field("completeBinding", v)?; + } + } + } struct_ser.end() } } @@ -8507,6 +8911,9 @@ impl<'de> serde::Deserialize<'de> for Persist { "trigger_params_json", "triggerParams", "rescan", + "begin", + "complete_binding", + "completeBinding", ]; #[allow(clippy::enum_variant_names)] @@ -8528,6 +8935,8 @@ impl<'de> serde::Deserialize<'de> for Persist { DeleteTriggerParams, TriggerParamsJson, Rescan, + Begin, + CompleteBinding, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -8567,6 +8976,8 @@ impl<'de> serde::Deserialize<'de> for Persist { "deleteTriggerParams" | "delete_trigger_params" => Ok(GeneratedField::DeleteTriggerParams), "triggerParams" | "trigger_params_json" => Ok(GeneratedField::TriggerParamsJson), "rescan" => Ok(GeneratedField::Rescan), + "begin" => Ok(GeneratedField::Begin), + "completeBinding" | "complete_binding" => Ok(GeneratedField::CompleteBinding), _ => Ok(GeneratedField::__SkipField__), } } @@ -8603,6 +9014,7 @@ impl<'de> serde::Deserialize<'de> for Persist { let mut delete_trigger_params__ = None; let mut trigger_params_json__ = None; let mut rescan__ = None; + let mut active_backfill_change__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::SeqNo => { @@ -8725,6 +9137,19 @@ impl<'de> serde::Deserialize<'de> for Persist { } rescan__ = Some(map_.next_value()?); } + GeneratedField::Begin => { + if active_backfill_change__.is_some() { + return Err(serde::de::Error::duplicate_field("begin")); + } + active_backfill_change__ = map_.next_value::<::std::option::Option<_>>()?.map(persist::ActiveBackfillChange::Begin) +; + } + GeneratedField::CompleteBinding => { + if active_backfill_change__.is_some() { + return Err(serde::de::Error::duplicate_field("completeBinding")); + } + active_backfill_change__ = map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| persist::ActiveBackfillChange::CompleteBinding(x.0)); + } GeneratedField::__SkipField__ => { let _ = map_.next_value::()?; } @@ -8748,6 +9173,7 @@ impl<'de> serde::Deserialize<'de> for Persist { delete_trigger_params: delete_trigger_params__.unwrap_or_default(), trigger_params_json: trigger_params_json__.unwrap_or_default(), rescan: rescan__.unwrap_or_default(), + active_backfill_change: active_backfill_change__, }) } } @@ -8966,6 +9392,9 @@ impl serde::Serialize for Recover { if !self.trigger_params_json.is_empty() { len += 1; } + if !self.active_backfills.is_empty() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("runtime.Recover", len)?; if !self.ack_intents.is_empty() { let v: std::collections::HashMap<_, _> = self.ack_intents.iter() @@ -9011,6 +9440,11 @@ impl serde::Serialize for Recover { #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("triggerParams", &crate::as_raw_json(&self.trigger_params_json)?)?; } + if !self.active_backfills.is_empty() { + let v: std::collections::HashMap<_, _> = self.active_backfills.iter() + .map(|(k, v)| (k, v.to_string())).collect(); + struct_ser.serialize_field("activeBackfills", &v)?; + } struct_ser.end() } } @@ -9041,6 +9475,8 @@ impl<'de> serde::Deserialize<'de> for Recover { "maxKeys", "trigger_params_json", "triggerParams", + "active_backfills", + "activeBackfills", ]; #[allow(clippy::enum_variant_names)] @@ -9055,6 +9491,7 @@ impl<'de> serde::Deserialize<'de> for Recover { LegacyCheckpoint, MaxKeys, TriggerParamsJson, + ActiveBackfills, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -9087,6 +9524,7 @@ impl<'de> serde::Deserialize<'de> for Recover { "legacyCheckpoint" | "legacy_checkpoint" => Ok(GeneratedField::LegacyCheckpoint), "maxKeys" | "max_keys" => Ok(GeneratedField::MaxKeys), "triggerParams" | "trigger_params_json" => Ok(GeneratedField::TriggerParamsJson), + "activeBackfills" | "active_backfills" => Ok(GeneratedField::ActiveBackfills), _ => Ok(GeneratedField::__SkipField__), } } @@ -9116,6 +9554,7 @@ impl<'de> serde::Deserialize<'de> for Recover { let mut legacy_checkpoint__ = None; let mut max_keys__ = None; let mut trigger_params_json__ = None; + let mut active_backfills__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::AckIntents => { @@ -9194,6 +9633,15 @@ impl<'de> serde::Deserialize<'de> for Recover { Some(map_.next_value::()?.0) ; } + GeneratedField::ActiveBackfills => { + if active_backfills__.is_some() { + return Err(serde::de::Error::duplicate_field("activeBackfills")); + } + active_backfills__ = Some( + map_.next_value::, ::pbjson::private::NumberDeserialize>>()? + .into_iter().map(|(k,v)| (k.0, v.0)).collect() + ); + } GeneratedField::__SkipField__ => { let _ = map_.next_value::()?; } @@ -9210,6 +9658,7 @@ impl<'de> serde::Deserialize<'de> for Recover { legacy_checkpoint: legacy_checkpoint__, max_keys: max_keys__.unwrap_or_default(), trigger_params_json: trigger_params_json__.unwrap_or_default(), + active_backfills: active_backfills__.unwrap_or_default(), }) } } diff --git a/crates/proto-flow/src/shuffle.rs b/crates/proto-flow/src/shuffle.rs index c7400d91440..9195aa436da 100644 --- a/crates/proto-flow/src/shuffle.rs +++ b/crates/proto-flow/src/shuffle.rs @@ -102,7 +102,7 @@ pub struct JournalFrontier { pub journal_name_suffix: ::prost::alloc::string::String, /// Binding index under which the journal is read. /// When persisting across sessions, this should be mapped via the task binding's - /// `journal_read_suffix` to ensure stability across task versions. + /// `state_key` to ensure stability across task versions. #[prost(uint32, tag = "3")] pub binding: u32, /// Delta of journal bytes read since the last checkpoint. @@ -128,6 +128,41 @@ pub struct Frontier { /// Per-shard flushed LSN, indexed by shard_index. #[prost(uint64, repeated, tag = "2")] pub flushed_lsn: ::prost::alloc::vec::Vec, + /// Latest backfill-begin clock for each binding in the checkpoint delta. + /// Populated only on a terminal (empty-journals) frontier of a Progressed + /// or NextCheckpoint sequence. Empty otherwise. + #[prost(message, repeated, tag = "3")] + pub latest_backfill_begin: ::prost::alloc::vec::Vec, + /// Latest backfill-complete clock for each binding in the checkpoint delta. + /// Populated only on a terminal (empty-journals) frontier of a Progressed + /// or NextCheckpoint sequence. Empty otherwise. + #[prost(message, repeated, tag = "4")] + pub latest_backfill_complete: ::prost::alloc::vec::Vec, +} +/// Nested message and enum types in `Frontier`. +pub mod frontier { + /// BackfillBegin is a binding's latest backfill-begin clock, keyed by binding + /// index. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct BackfillBegin { + /// Binding index. + #[prost(uint32, tag = "1")] + pub binding: u32, + /// Clock of the binding's most recent backfill-begin. + #[prost(fixed64, tag = "2")] + pub clock: u64, + } + /// BackfillComplete reports a binding's most recent backfill-complete event, + /// keyed by binding index. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] + pub struct BackfillComplete { + /// Binding index. + #[prost(uint32, tag = "1")] + pub binding: u32, + /// Truncation boundary of the completed backfill: the backfill's begin clock. + #[prost(fixed64, tag = "2")] + pub clock: u64, + } } /// SessionRequest is sent by the Coordinator to manage the shuffle session. #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/crates/proto-flow/src/shuffle.serde.rs b/crates/proto-flow/src/shuffle.serde.rs index b52f9b873c1..6b5fb80db7c 100644 --- a/crates/proto-flow/src/shuffle.serde.rs +++ b/crates/proto-flow/src/shuffle.serde.rs @@ -161,6 +161,12 @@ impl serde::Serialize for Frontier { if !self.flushed_lsn.is_empty() { len += 1; } + if !self.latest_backfill_begin.is_empty() { + len += 1; + } + if !self.latest_backfill_complete.is_empty() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("shuffle.Frontier", len)?; if !self.journals.is_empty() { struct_ser.serialize_field("journals", &self.journals)?; @@ -168,6 +174,12 @@ impl serde::Serialize for Frontier { if !self.flushed_lsn.is_empty() { struct_ser.serialize_field("flushedLsn", &self.flushed_lsn.iter().map(ToString::to_string).collect::>())?; } + if !self.latest_backfill_begin.is_empty() { + struct_ser.serialize_field("latestBackfillBegin", &self.latest_backfill_begin)?; + } + if !self.latest_backfill_complete.is_empty() { + struct_ser.serialize_field("latestBackfillComplete", &self.latest_backfill_complete)?; + } struct_ser.end() } } @@ -181,12 +193,18 @@ impl<'de> serde::Deserialize<'de> for Frontier { "journals", "flushed_lsn", "flushedLsn", + "latest_backfill_begin", + "latestBackfillBegin", + "latest_backfill_complete", + "latestBackfillComplete", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { Journals, FlushedLsn, + LatestBackfillBegin, + LatestBackfillComplete, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -211,6 +229,8 @@ impl<'de> serde::Deserialize<'de> for Frontier { match value { "journals" => Ok(GeneratedField::Journals), "flushedLsn" | "flushed_lsn" => Ok(GeneratedField::FlushedLsn), + "latestBackfillBegin" | "latest_backfill_begin" => Ok(GeneratedField::LatestBackfillBegin), + "latestBackfillComplete" | "latest_backfill_complete" => Ok(GeneratedField::LatestBackfillComplete), _ => Ok(GeneratedField::__SkipField__), } } @@ -232,6 +252,8 @@ impl<'de> serde::Deserialize<'de> for Frontier { { let mut journals__ = None; let mut flushed_lsn__ = None; + let mut latest_backfill_begin__ = None; + let mut latest_backfill_complete__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::Journals => { @@ -249,6 +271,18 @@ impl<'de> serde::Deserialize<'de> for Frontier { .into_iter().map(|x| x.0).collect()) ; } + GeneratedField::LatestBackfillBegin => { + if latest_backfill_begin__.is_some() { + return Err(serde::de::Error::duplicate_field("latestBackfillBegin")); + } + latest_backfill_begin__ = Some(map_.next_value()?); + } + GeneratedField::LatestBackfillComplete => { + if latest_backfill_complete__.is_some() { + return Err(serde::de::Error::duplicate_field("latestBackfillComplete")); + } + latest_backfill_complete__ = Some(map_.next_value()?); + } GeneratedField::__SkipField__ => { let _ = map_.next_value::()?; } @@ -257,12 +291,250 @@ impl<'de> serde::Deserialize<'de> for Frontier { Ok(Frontier { journals: journals__.unwrap_or_default(), flushed_lsn: flushed_lsn__.unwrap_or_default(), + latest_backfill_begin: latest_backfill_begin__.unwrap_or_default(), + latest_backfill_complete: latest_backfill_complete__.unwrap_or_default(), }) } } deserializer.deserialize_struct("shuffle.Frontier", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for frontier::BackfillBegin { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.binding != 0 { + len += 1; + } + if self.clock != 0 { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("shuffle.Frontier.BackfillBegin", len)?; + if self.binding != 0 { + struct_ser.serialize_field("binding", &self.binding)?; + } + if self.clock != 0 { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("clock", ToString::to_string(&self.clock).as_str())?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for frontier::BackfillBegin { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "binding", + "clock", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Binding, + Clock, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "binding" => Ok(GeneratedField::Binding), + "clock" => Ok(GeneratedField::Clock), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = frontier::BackfillBegin; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct shuffle.Frontier.BackfillBegin") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut binding__ = None; + let mut clock__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Binding => { + if binding__.is_some() { + return Err(serde::de::Error::duplicate_field("binding")); + } + binding__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::Clock => { + if clock__.is_some() { + return Err(serde::de::Error::duplicate_field("clock")); + } + clock__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(frontier::BackfillBegin { + binding: binding__.unwrap_or_default(), + clock: clock__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("shuffle.Frontier.BackfillBegin", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for frontier::BackfillComplete { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.binding != 0 { + len += 1; + } + if self.clock != 0 { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("shuffle.Frontier.BackfillComplete", len)?; + if self.binding != 0 { + struct_ser.serialize_field("binding", &self.binding)?; + } + if self.clock != 0 { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("clock", ToString::to_string(&self.clock).as_str())?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for frontier::BackfillComplete { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "binding", + "clock", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Binding, + Clock, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "binding" => Ok(GeneratedField::Binding), + "clock" => Ok(GeneratedField::Clock), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = frontier::BackfillComplete; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct shuffle.Frontier.BackfillComplete") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut binding__ = None; + let mut clock__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Binding => { + if binding__.is_some() { + return Err(serde::de::Error::duplicate_field("binding")); + } + binding__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::Clock => { + if clock__.is_some() { + return Err(serde::de::Error::duplicate_field("clock")); + } + clock__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(frontier::BackfillComplete { + binding: binding__.unwrap_or_default(), + clock: clock__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("shuffle.Frontier.BackfillComplete", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for JournalFrontier { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/crates/proto-flow/tests/regression.rs b/crates/proto-flow/tests/regression.rs index cc26692f5c7..dbce41e05c9 100644 --- a/crates/proto-flow/tests/regression.rs +++ b/crates/proto-flow/tests/regression.rs @@ -498,6 +498,8 @@ fn ex_capture_response() -> capture::Response { checkpoint: Some(capture::response::Checkpoint { state: Some(ex_connector_state()), }), + backfill_begin: Some(capture::response::BackfillBegin { binding: 4 }), + backfill_complete: Some(capture::response::BackfillComplete { binding: 5 }), internal: ex_internal(), } } @@ -645,6 +647,20 @@ fn ex_materialize_request() -> materialize::Request { }), flush: Some(materialize::request::Flush { state_patches_json: json!([{"flushed": 1}]).to_string().into(), + backfill_begins: vec![materialize::request::flush::BackfillBegin { + binding: 2, + timestamp: Some(pbjson_types::Timestamp { + seconds: 1700000000, + nanos: 0, + }), + }], + backfill_completes: vec![materialize::request::flush::BackfillComplete { + binding: 3, + timestamp: Some(pbjson_types::Timestamp { + seconds: 1700000500, + nanos: 123000000, + }), + }], }), store: Some(materialize::request::Store { binding: 3, diff --git a/crates/proto-flow/tests/snapshots/regression__capture_response_json.snap b/crates/proto-flow/tests/snapshots/regression__capture_response_json.snap index 8f107d3a2ca..07de71d375b 100644 --- a/crates/proto-flow/tests/snapshots/regression__capture_response_json.snap +++ b/crates/proto-flow/tests/snapshots/regression__capture_response_json.snap @@ -86,5 +86,11 @@ expression: json_test(msg) "mergePatch": true } }, + "backfillBegin": { + "binding": 4 + }, + "backfillComplete": { + "binding": 5 + }, "$internal": "EgJIaRgB" } diff --git a/crates/proto-flow/tests/snapshots/regression__capture_response_proto.snap b/crates/proto-flow/tests/snapshots/regression__capture_response_proto.snap index 0e7c084d579..1f295e4410a 100644 --- a/crates/proto-flow/tests/snapshots/regression__capture_response_proto.snap +++ b/crates/proto-flow/tests/snapshots/regression__capture_response_proto.snap @@ -36,5 +36,6 @@ expression: proto_test(msg) |22757064 61746522 7d100142 2a080312| "update"}..B*... 000001f0 |267b2266 6f726d61 74223a22 64617465| &{"format":"date 00000200 |2d74696d 65222c22 74797065 223a2273| -time","type":"s 00000210 -|7472696e 67227da2 06061202 48691801| tring"}.....Hi.. 00000220 - 00000230 +|7472696e 67227d4a 02080452 020805a2| tring"}J...R.... 00000220 +|06061202 48691801| ....Hi.. 00000230 + 00000238 diff --git a/crates/proto-flow/tests/snapshots/regression__materialize_request_json.snap b/crates/proto-flow/tests/snapshots/regression__materialize_request_json.snap index ea3e8bc0ebb..f978c503ddf 100644 --- a/crates/proto-flow/tests/snapshots/regression__materialize_request_json.snap +++ b/crates/proto-flow/tests/snapshots/regression__materialize_request_json.snap @@ -526,7 +526,19 @@ expression: json_test(msg) "keyPacked": "VkseCQ==" }, "flush": { - "statePatches": [{"flushed":1}] + "statePatches": [{"flushed":1}], + "backfillBegins": [ + { + "binding": 2, + "timestamp": "2023-11-14T22:13:20+00:00" + } + ], + "backfillCompletes": [ + { + "binding": 3, + "timestamp": "2023-11-14T22:21:40.123+00:00" + } + ] }, "store": { "binding": 3, diff --git a/crates/proto-flow/tests/snapshots/regression__materialize_request_proto.snap b/crates/proto-flow/tests/snapshots/regression__materialize_request_proto.snap index 7b448a9e82d..ac23c8f4c57 100644 --- a/crates/proto-flow/tests/snapshots/regression__materialize_request_proto.snap +++ b/crates/proto-flow/tests/snapshots/regression__materialize_request_proto.snap @@ -200,20 +200,22 @@ expression: proto_test(msg) |3a226332 566a636d 5630222c 22736f70| :"c2VjcmV0","sop 00000c30 |73223a7b 226d6163 223a2261 6263227d| s":{"mac":"abc"} 00000c40 |7d2a1308 0c12095b 34322c22 6869225d| }*.....[42,"hi"] 00000c50 -|1a04564b 1e093211 0a0f5b7b 22666c75| ..VK..2...[{"flu 00000c60 -|73686564 223a317d 5d3a4508 03120b5b| shed":1}]:E....[ 00000c70 -|74727565 2c6e756c 6c5d1a03 5a150022| true,null]..Z.." 00000c80 -|125b332e 31343135 392c2266 69656c64| .[3.14159,"field 00000c90 -|21225d2a 023c5b32 137b2266 756c6c22| !"]*.<[2.{"full" 00000ca0 -|3a22646f 63756d65 6e74227d 38014001| :"document"}8.@. 00000cb0 -|427e0a64 0a4a0a15 612f7265 61642f6a| B~.d.J..a/read/j 00000cc0 -|6f75726e 616c3b73 75666669 78123108| ournal;suffix.1. 00000cd0 -|b9601215 0a050309 08050712 0c09e321| .`.............! 00000ce0 -|00000000 000010d7 0812150a 05070c66| ...............f 00000cf0 -|2b1d120c 09350100 00000000 0010ae11| +....5.......... 00000d00 -|12160a0e 616e2f61 636b2f6a 6f75726e| ....an/ack/journ 00000d10 -|616c1204 03040205 12165b7b 22737461| al........[{"sta 00000d20 -|72746564 223a2263 6f6d6d69 74227d5d| rted":"commit"}] 00000d30 -|4a120a10 5b7b2261 636b6564 223a7472| J...[{"acked":tr 00000d40 -|75657d5d a2060612 02486918 01| ue}].....Hi.. 00000d50 - 00000d5d +|1a04564b 1e09322e 0a0f5b7b 22666c75| ..VK..2...[{"flu 00000c60 +|73686564 223a317d 5d120a08 02120608| shed":1}]....... 00000c70 +|80e2cfaa 061a0f08 03120b08 f4e5cfaa| ................ 00000c80 +|0610c0a9 d33a3a45 0803120b 5b747275| .....::E....[tru 00000c90 +|652c6e75 6c6c5d1a 035a1500 22125b33| e,null]..Z..".[3 00000ca0 +|2e313431 35392c22 6669656c 6421225d| .14159,"field!"] 00000cb0 +|2a023c5b 32137b22 66756c6c 223a2264| *.<[2.{"full":"d 00000cc0 +|6f63756d 656e7422 7d380140 01427e0a| ocument"}8.@.B~. 00000cd0 +|640a4a0a 15612f72 6561642f 6a6f7572| d.J..a/read/jour 00000ce0 +|6e616c3b 73756666 69781231 08b96012| nal;suffix.1..`. 00000cf0 +|150a0503 09080507 120c09e3 21000000| ............!... 00000d00 +|00000010 d7081215 0a05070c 662b1d12| ............f+.. 00000d10 +|0c093501 00000000 000010ae 1112160a| ..5............. 00000d20 +|0e616e2f 61636b2f 6a6f7572 6e616c12| .an/ack/journal. 00000d30 +|04030402 0512165b 7b227374 61727465| .......[{"starte 00000d40 +|64223a22 636f6d6d 6974227d 5d4a120a| d":"commit"}]J.. 00000d50 +|105b7b22 61636b65 64223a74 7275657d| .[{"acked":true} 00000d60 +|5da20606 12024869 1801| ].....Hi.. 00000d70 + 00000d7a diff --git a/crates/publisher/Cargo.toml b/crates/publisher/Cargo.toml index 5aea18d3310..e6c0b05e33f 100644 --- a/crates/publisher/Cargo.toml +++ b/crates/publisher/Cargo.toml @@ -39,4 +39,5 @@ proto-grpc = { path = "../proto-grpc" } tables = { path = "../tables" } insta = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } url = { workspace = true } diff --git a/crates/publisher/src/intents.rs b/crates/publisher/src/intents.rs index e55ab1a5755..163138393cb 100644 --- a/crates/publisher/src/intents.rs +++ b/crates/publisher/src/intents.rs @@ -3,6 +3,13 @@ use itertools::Itertools; use proto_gazette::uuid; use std::collections::BTreeMap; +/// A backfill marker carried as body fields on the ACK documents of a marker +/// broadcast transaction. +pub enum BackfillMarker { + Begin, + Complete { truncated_at: u64 }, +} + /// Build per-journal ACK intent documents for a committed transaction. /// /// Each element of `transaction` is a (producer, clock, journals) tuple @@ -15,6 +22,7 @@ use std::collections::BTreeMap; /// hints as they'd be redundant. pub fn build_transaction_intents( transaction: &[(uuid::Producer, uuid::Clock, Vec)], + marker: Option<&BackfillMarker>, ) -> BTreeMap { // Flatten and index on journal, then producer. let mut flattened: Vec<(&str, uuid::Producer, uuid::Clock)> = transaction @@ -82,26 +90,24 @@ pub fn build_transaction_intents( let this_uuid = uuid::build(*this_producer, *this_commit, uuid::Flags::ACK_TXN); let mut buf = Vec::new(); - write_ndjson( - &mut buf, - &serde_json::json!({ - "_meta": { "uuid": this_uuid }, - "is_ack": true, - "hints": hinted_journals, - }), - ); + let mut doc = serde_json::json!({ + "_meta": { "uuid": this_uuid }, + "is_ack": true, + "hints": hinted_journals, + }); + apply_marker(&mut doc, marker); + write_ndjson(&mut buf, &doc); // Remainder of `these_producers` also need ACK documents. // They were already hinted by the first ACK of this journal, and don't carry hints themselves. for (_this_journal, this_producer, this_commit) in these_producers { let this_uuid = uuid::build(*this_producer, *this_commit, uuid::Flags::ACK_TXN); - write_ndjson( - &mut buf, - &serde_json::json!({ - "_meta": { "uuid": this_uuid }, - "is_ack": true, - }), - ); + let mut doc = serde_json::json!({ + "_meta": { "uuid": this_uuid }, + "is_ack": true, + }); + apply_marker(&mut doc, marker); + write_ndjson(&mut buf, &doc); } journal_acks.insert(this_journal.to_string(), bytes::Bytes::from(buf)); @@ -115,6 +121,29 @@ fn write_ndjson(buf: &mut Vec, doc: &serde_json::Value) { buf.push(b'\n'); } +fn apply_marker(doc: &mut serde_json::Value, marker: Option<&BackfillMarker>) { + let Some(marker) = marker else { return }; + let obj = doc + .as_object_mut() + .expect("ACK document is a JSON object literal"); + + match marker { + BackfillMarker::Begin => { + obj.insert("backfillBegin".to_string(), serde_json::Value::Bool(true)); + } + BackfillMarker::Complete { truncated_at } => { + obj.insert( + "backfillComplete".to_string(), + serde_json::Value::Bool(true), + ); + obj.insert( + "truncatedAt".to_string(), + serde_json::Value::String(labels::truncated_at_value(*truncated_at)), + ); + } + } +} + /// Decode causal hints embedded in an ACK document. /// /// Returns a `HintIter` that yields `(hinted_journal, hinted_producer, hinted_clock)` @@ -329,14 +358,14 @@ mod test { #[test] fn test_empty_transaction() { - let result = parse_intents(build_transaction_intents(&[])); + let result = parse_intents(build_transaction_intents(&[], None)); insta::assert_json_snapshot!(result); } #[test] fn test_single_producer_single_journal() { let txn = vec![(P1, clock(100), js(&["acmeCo/anvils/part=a/pivot=00"]))]; - insta::assert_json_snapshot!(parse_intents(build_transaction_intents(&txn))); + insta::assert_json_snapshot!(parse_intents(build_transaction_intents(&txn, None))); } #[test] @@ -349,7 +378,7 @@ mod test { "acmeCo/anvils/part=b/pivot=00", ]), )]; - insta::assert_json_snapshot!(parse_intents(build_transaction_intents(&txn))); + insta::assert_json_snapshot!(parse_intents(build_transaction_intents(&txn, None))); } #[test] @@ -358,7 +387,7 @@ mod test { (P1, clock(100), js(&["acmeCo/anvils/part=a/pivot=00"])), (P2, clock(200), js(&["acmeCo/anvils/part=a/pivot=00"])), ]; - insta::assert_json_snapshot!(parse_intents(build_transaction_intents(&txn))); + insta::assert_json_snapshot!(parse_intents(build_transaction_intents(&txn, None))); } // Three producers across four journals with overlapping membership. @@ -394,7 +423,41 @@ mod test { ]), ), ]; - insta::assert_json_snapshot!(parse_intents(build_transaction_intents(&txn))); + insta::assert_json_snapshot!(parse_intents(build_transaction_intents(&txn, None))); + } + + #[test] + fn test_marker_begin_broadcast() { + let txn = vec![( + P1, + clock(0x1122334455667788), + js(&[ + "acmeCo/anvils/part=a/pivot=00", + "acmeCo/anvils/part=b/pivot=00", + "acmeCo/anvils/part=c/pivot=00", + ]), + )]; + insta::assert_json_snapshot!(parse_intents(build_transaction_intents( + &txn, + Some(&BackfillMarker::Begin), + ))); + } + + #[test] + fn test_marker_complete_broadcast() { + let truncated_at = uuid::Clock::from_unix(1_700_000_000, 0).as_u64(); + let txn = vec![( + P1, + clock(0x1122334455667788), + js(&[ + "acmeCo/anvils/part=a/pivot=00", + "acmeCo/anvils/part=b/pivot=00", + ]), + )]; + insta::assert_json_snapshot!(parse_intents(build_transaction_intents( + &txn, + Some(&BackfillMarker::Complete { truncated_at }), + ))); } // --- decode_transaction_hints tests --- @@ -414,7 +477,7 @@ mod test { /// no hints. fn assert_round_trip(txn: &[(uuid::Producer, uuid::Clock, Vec)]) { let expected = flatten_transaction(txn); - let journal_acks = parse_intents(build_transaction_intents(txn)); + let journal_acks = parse_intents(build_transaction_intents(txn, None)); for (journal, acks) in &journal_acks { // First ACK carries hints. diff --git a/crates/publisher/src/publisher.rs b/crates/publisher/src/publisher.rs index cf060ac9f4a..77921496e74 100644 --- a/crates/publisher/src/publisher.rs +++ b/crates/publisher/src/publisher.rs @@ -1,5 +1,6 @@ use bytes::BufMut; -use proto_gazette::uuid; +use futures::StreamExt; +use proto_gazette::{broker, uuid}; /// Publisher is responsible for transactional publishing of documents to /// journal partitions, creating partitions on-demand and as needed. @@ -236,6 +237,40 @@ impl Publisher { (self.producer, clock, journal_names) } + /// Snapshot a backfill marker broadcast: every partition journal of a Mapped + /// collection binding, plus a ticked commit clock and producer identity. Unlike + /// [`Self::commit_intents`] (only appended journals), a marker must reach *every* + /// partition so a reader observes it regardless of its selector. + pub async fn marker_commit( + &mut self, + binding_idx: usize, + ) -> tonic::Result<(uuid::Producer, uuid::Clock, Vec)> { + // Snapshot the journals to broadcast to, releasing the partitions watch + // borrow before returning. + let journals: Vec = match &self.bindings[binding_idx] { + super::Binding::Mapped(_) => { + let super::LazyBindingClient::Mapped(lazy) = &self.binding_clients[binding_idx] + else { + unreachable!("Mapped binding has Mapped lazy client"); + }; + let (_client, partitions) = &(**lazy); + let partitions = partitions.ready().await; + let refresh = partitions.token(); + refresh + .result()? + .iter() + .map(|split| split.name.to_string()) + .collect() + } + super::Binding::Fixed(_) => { + unreachable!("backfill markers are only broadcast to Mapped collection bindings") + } + }; + + let clock = self.clock.tick(); + Ok((self.producer, clock, journals)) + } + /// Write pre-serialized ACK intent documents to their journals. /// /// Takes the output of `intents::build_transaction_intents()` — per-journal @@ -359,6 +394,58 @@ impl Publisher { ) } + /// Apply the `estuary.dev/truncated-at` journal label to the partitions of + /// each active backfill. Journals already at the target value are skipped. + pub async fn apply_truncated_at_labels( + &mut self, + active_backfills: &std::collections::BTreeMap, + ) -> tonic::Result<()> { + for (&index, &clock) in active_backfills { + let target = labels::truncated_at_value(clock); + + let super::Binding::Mapped(binding) = &self.bindings[index] else { + return Err(tonic::Status::internal(format!( + "binding {index} has an active backfill but is not a Mapped collection binding" + ))); + }; + let client = self.binding_clients[index].client(); + + // Watch the partition listing: the watch handles transient-error + // backoff and restates the journals after every change, so a lost CAS + // race (`false`) is retried on the snapshot that the racing writer's + // own change delivers. + let watch = client.clone().list_watch(broker::ListRequest { + selector: Some(broker::LabelSelector { + include: Some(labels::build_set([( + "name:prefix", + binding.partitions_prefix.as_str(), + )])), + exclude: None, + }), + watch: true, + ..Default::default() + }); + let mut watch = std::pin::pin!(watch); + + loop { + match watch.next().await { + Some(Ok(listing)) => { + if advance_truncated_at_labels(client, listing, &target).await? { + break; + } + } + // Transient — retried on the next poll. + Some(Err(gazette::RetryError { inner, .. })) if inner.is_transient() => {} + Some(Err(gazette::RetryError { inner, .. })) => { + return Err(status_from_gazette(inner)); + } + None => break, + } + } + } + Ok(()) + } + /// Access the lazy Client and partitions watch for the Mapped binding at /// `index`. Panics if the binding is Fixed. Primarily used by tests. pub fn mapped_binding_client( @@ -376,3 +463,250 @@ impl Publisher { } } } + +async fn advance_truncated_at_labels( + client: &gazette::journal::Client, + listing: broker::ListResponse, + target: &str, +) -> tonic::Result { + for journal in listing.journals { + let Some(change) = truncated_at_label_change(journal, target)? else { + continue; + }; + + match retry_transient("apply truncated-at label", || { + client.apply(broker::ApplyRequest { + changes: vec![change.clone()], + }) + }) + .await + { + Ok(_) => {} + Err(gazette::Error::BrokerStatus(broker::Status::EtcdTransactionFailed)) => { + return Ok(false); + } + Err(err) => return Err(status_from_gazette(err)), + } + } + Ok(true) +} + +/// Convert a gazette client Error into a tonic::Status, preserving a gRPC status +/// when present and otherwise wrapping the error as Internal. +fn status_from_gazette(err: gazette::Error) -> tonic::Status { + match err { + gazette::Error::Grpc(status) => status, + other => tonic::Status::internal(other.to_string()), + } +} + +async fn retry_transient(what: &'static str, mut op: F) -> gazette::Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut attempt: u32 = 0; + loop { + let err = match op().await { + ok @ Ok(_) => return ok, + Err(err) => err, + }; + attempt += 1; + + if !err.is_transient() || attempt == 8 { + return Err(err); + } + + // Exponential backoff from a 100ms base, capped at 10 seconds. + let backoff = std::time::Duration::from_millis(100) + .saturating_mul(2u32.saturating_pow(attempt - 1)) + .min(std::time::Duration::from_secs(10)); + tracing::warn!(what, attempt, %err, "gazette RPC failed; retrying after backoff"); + tokio::time::sleep(backoff).await; + } +} + +/// Build the apply Change that advances `journal`'s `estuary.dev/truncated-at` +/// label to `target`, or `None` if the journal is already at or beyond it. The +/// fixed-width hex encoding sorts lexically by clock, so the skip covers both an +/// equal label (idempotent) and a newer one (a later backfill already applied, +/// or clock skew across a restart) -- the label only ever advances. +fn truncated_at_label_change( + journal: broker::list_response::Journal, + target: &str, +) -> tonic::Result> { + let Some(mut spec) = journal.spec else { + return Err(tonic::Status::internal( + "list response journal is missing its spec", + )); + }; + let current = spec + .labels + .as_ref() + .and_then(|set| labels::maybe_one(set, labels::TRUNCATED_AT).ok()) + .unwrap_or(""); + + if current >= target { + return Ok(None); + } + + spec.labels = Some(labels::set_value( + spec.labels.take().unwrap_or_default(), + labels::TRUNCATED_AT, + target, + )); + + Ok(Some(broker::apply_request::Change { + expect_mod_revision: journal.mod_revision, + upsert: Some(spec), + delete: String::new(), + })) +} + +#[cfg(test)] +mod test { + use super::*; + + // A list-response journal carrying `truncated_at` (when Some) plus an + // unrelated label, used to check the advance decision and that other labels + // survive the upsert. + fn journal(mod_revision: i64, truncated_at: Option<&str>) -> broker::list_response::Journal { + let mut set = labels::build_set([("estuary.dev/collection", "the/collection")]); + if let Some(value) = truncated_at { + set = labels::set_value(set, labels::TRUNCATED_AT, value); + } + broker::list_response::Journal { + spec: Some(broker::JournalSpec { + name: "the/collection/pivot=00".to_string(), + labels: Some(set), + ..Default::default() + }), + mod_revision, + ..Default::default() + } + } + + #[test] + fn test_truncated_at_label_change_advances_only() { + let target = labels::truncated_at_value(0x20); + + // No label yet, an older label, and an equal-then-newer label: the + // decision is "advance only" -- skip unless strictly older than target. + let absent = truncated_at_label_change(journal(1, None), &target).unwrap(); + assert!(absent.is_some(), "absent label advances"); + + let older = labels::truncated_at_value(0x10); + let older = truncated_at_label_change(journal(1, Some(&older)), &target).unwrap(); + assert!(older.is_some(), "older label advances"); + + let equal = truncated_at_label_change(journal(1, Some(&target)), &target).unwrap(); + assert!(equal.is_none(), "equal label is skipped (idempotent)"); + + let newer = labels::truncated_at_value(0x30); + let newer = truncated_at_label_change(journal(1, Some(&newer)), &target).unwrap(); + assert!(newer.is_none(), "newer label is never regressed"); + } + + #[test] + fn test_truncated_at_label_change_builds_upsert() { + let target = labels::truncated_at_value(0x20); + let prior = labels::truncated_at_value(0x10); + + let change = truncated_at_label_change(journal(42, Some(&prior)), &target) + .unwrap() + .expect("older label advances"); + + // The CAS guard carries the listed revision, and the upsert sets the + // target label while preserving the journal's unrelated labels. + assert_eq!(change.expect_mod_revision, 42); + let set = change.upsert.unwrap().labels.unwrap(); + assert_eq!( + labels::maybe_one(&set, labels::TRUNCATED_AT).unwrap(), + target + ); + assert_eq!( + labels::maybe_one(&set, "estuary.dev/collection").unwrap(), + "the/collection" + ); + } + + #[test] + fn test_truncated_at_label_change_requires_spec() { + let target = labels::truncated_at_value(0x20); + let no_spec = broker::list_response::Journal { + spec: None, + mod_revision: 1, + ..Default::default() + }; + assert!(truncated_at_label_change(no_spec, &target).is_err()); + } + + #[tokio::test(start_paused = true)] + async fn test_retry_transient_budget_and_passthrough() { + use std::sync::atomic::{AtomicU32, Ordering}; + + struct Case { + name: &'static str, + fail_times: u32, + transient: bool, + expect_ok: bool, + expect_calls: u32, + } + let cases = [ + Case { + name: "immediate success", + fail_times: 0, + transient: true, + expect_ok: true, + expect_calls: 1, + }, + Case { + name: "transient then success", + fail_times: 3, + transient: true, + expect_ok: true, + expect_calls: 4, + }, + Case { + name: "terminal surfaces at once", + fail_times: 99, + transient: false, + expect_ok: false, + expect_calls: 1, + }, + Case { + name: "transient exhausts budget", + fail_times: 99, + transient: true, + expect_ok: false, + expect_calls: 8, // mirrors retry_transient's attempt budget + }, + ]; + + for case in cases { + let calls = AtomicU32::new(0); + let result = retry_transient("test", || { + let n = calls.fetch_add(1, Ordering::SeqCst); + let out: gazette::Result = if n < case.fail_times { + Err(if case.transient { + gazette::Error::Grpc(tonic::Status::unavailable("transient")) + } else { + gazette::Error::BrokerStatus(broker::Status::EtcdTransactionFailed) + }) + } else { + Ok(n) + }; + std::future::ready(out) + }) + .await; + + assert_eq!(result.is_ok(), case.expect_ok, "{}", case.name); + assert_eq!( + calls.load(Ordering::SeqCst), + case.expect_calls, + "{}", + case.name + ); + } + } +} diff --git a/crates/publisher/src/snapshots/publisher__intents__test__marker_begin_broadcast.snap b/crates/publisher/src/snapshots/publisher__intents__test__marker_begin_broadcast.snap new file mode 100644 index 00000000000..15cc279777d --- /dev/null +++ b/crates/publisher/src/snapshots/publisher__intents__test__marker_begin_broadcast.snap @@ -0,0 +1,102 @@ +--- +source: crates/publisher/src/intents.rs +expression: "parse_intents(build_transaction_intents(&txn, Some(&BackfillMarker::Begin),))" +--- +[ + [ + "acmeCo/anvils/part=a/pivot=00", + [ + { + "_meta": { + "uuid": "45566778-2334-1112-a002-010000000001" + }, + "backfillBegin": true, + "hints": [ + { + "j": [ + 10, + "b/pivot=00" + ], + "p": [ + {} + ] + }, + { + "j": [ + 10, + "c/pivot=00" + ], + "p": [ + {} + ] + } + ], + "is_ack": true + } + ] + ], + [ + "acmeCo/anvils/part=b/pivot=00", + [ + { + "_meta": { + "uuid": "45566778-2334-1112-a002-010000000001" + }, + "backfillBegin": true, + "hints": [ + { + "j": [ + 10, + "a/pivot=00" + ], + "p": [ + {} + ] + }, + { + "j": [ + 10, + "c/pivot=00" + ], + "p": [ + {} + ] + } + ], + "is_ack": true + } + ] + ], + [ + "acmeCo/anvils/part=c/pivot=00", + [ + { + "_meta": { + "uuid": "45566778-2334-1112-a002-010000000001" + }, + "backfillBegin": true, + "hints": [ + { + "j": [ + 10, + "a/pivot=00" + ], + "p": [ + {} + ] + }, + { + "j": [ + 10, + "b/pivot=00" + ], + "p": [ + {} + ] + } + ], + "is_ack": true + } + ] + ] +] diff --git a/crates/publisher/src/snapshots/publisher__intents__test__marker_complete_broadcast.snap b/crates/publisher/src/snapshots/publisher__intents__test__marker_complete_broadcast.snap new file mode 100644 index 00000000000..941c18ebb62 --- /dev/null +++ b/crates/publisher/src/snapshots/publisher__intents__test__marker_complete_broadcast.snap @@ -0,0 +1,54 @@ +--- +source: crates/publisher/src/intents.rs +expression: "parse_intents(build_transaction_intents(&txn,\nSome(&BackfillMarker::Complete { truncated_at }),))" +--- +[ + [ + "acmeCo/anvils/part=a/pivot=00", + [ + { + "_meta": { + "uuid": "45566778-2334-1112-a002-010000000001" + }, + "backfillComplete": true, + "hints": [ + { + "j": [ + 10, + "b/pivot=00" + ], + "p": [ + {} + ] + } + ], + "is_ack": true, + "truncatedAt": "1ee833b04afc0000" + } + ] + ], + [ + "acmeCo/anvils/part=b/pivot=00", + [ + { + "_meta": { + "uuid": "45566778-2334-1112-a002-010000000001" + }, + "backfillComplete": true, + "hints": [ + { + "j": [ + 10, + "a/pivot=00" + ], + "p": [ + {} + ] + } + ], + "is_ack": true, + "truncatedAt": "1ee833b04afc0000" + } + ] + ] +] diff --git a/crates/runtime-next/README.md b/crates/runtime-next/README.md index 5b8ac279bec..8c3c478724b 100644 --- a/crates/runtime-next/README.md +++ b/crates/runtime-next/README.md @@ -181,6 +181,78 @@ key. An authoritative (unmarked) checkpoint implies no V2 transaction has committed, so clearing `FC:` loses no V2 state. The transaction loop then only ever writes `FC:` deltas. +## Backfill truncation (materialize) + +When a source collection is backfill-truncated, documents a materialization +sourced before the truncation boundary are superseded and must not combine +with — or reduce forward into — documents at or above it. The shard actor +(`shard/materialize/boundaries.rs`) tracks per-binding the latest observed +truncation boundary (`Begin`) clock; boundaries classify ingress rather than +tagging combiner entries. + +The combiner (`doc::combine`) marks superseded entries with a one-bit **STALE** +flag. A stale entry is never validated or reduced; on drain it is discarded, +transferring only its `front()` existence onto the first fresh entry of a +shared `(binding, key)` — so a truncated row's destination presence is +preserved while its value is not. Staleness reaches an entry three ways, all +collapsing to the same flag: + +- **`truncate(binding)`** at the moment a boundary is learned reclassifies + everything the accumulator already holds: the live MemTable drops the + binding's pre-boundary source documents outright (they carry no existence) + and flags its Loaded fronts stale in place, while every spill segment already + written is fenced by a per-binding **ordinal cutoff** (`cutoffs[binding]` = + segment count). At drain, an entry in a segment below its binding's cutoff is + stamped stale. +- **`add_stale_front`** flags a Loaded row classified stale on arrival. +- The **persisted flag** rides the spill entry header, so an entry flagged in + memory and then spilled (under memory pressure, into a segment at or above the + cutoff) is still stale at drain — staleness is `persisted-flag OR ordinal + fence`. + +The split is **exhaustive** because `observe_begin` has a single call site — at +L:Load receipt, before the scan — so every combiner add is unambiguously either +*before* it (reclassified by `truncate`) or *after* it (self-classifying at +ingress): the scan drops pre-boundary source documents by their shuffle clock, +and Loaded rows split into fresh fronts vs. stale fronts by their embedded +document-UUID clock. Loaded rows classify by the document UUID (not message +timing) because staleness is a property of when the row was last *stored*: a row +can load stale many transactions after the one that truncated its binding. + +Cutoffs are **accumulator-local** and die with the accumulator — a recycled +(drained) accumulator starts with zeroed cutoffs at the same moment its spill +file is truncated to length 0, so segment ordinals and their fences restart +together. The per-binding boundary **clocks**, by contrast, are the persistent +runtime state that outlives any single accumulator. + +Consequences and requirements: + +- **Once a binding has observed a `Begin`, its Loaded rows must expose a + parseable document UUID** (at the binding's configured pointer, typically + `/_meta/uuid`) so each row can be classified against the boundary; a missing + or malformed UUID then fails the transaction. A binding that has never + truncated has no boundary, so its Loaded rows are fresh regardless of clock + and need no UUID — this spares the many pre-existing materializations, + unrelated to truncation, whose rows carry none. Delta-update bindings never + load and are unaffected. +- **Boundaries must be visible before the documents they fence.** A `Begin` + rides eagerly on unresolved shuffle peeks (see `crates/shuffle`), so the shard + applies it before scanning any document at or above its clock, and each + document then classifies against the current boundary. This assumes a single + writer per truncating collection; concurrent writers are undefined. +- **Markers are latest-state, not an event log.** The leader keeps + session-cumulative per-binding `Begin`/`Complete` maps; each connector + `Flush` projects the transaction's latest observed clocks. An eager + (unresolved-peek) `Begin` is used only to stamp outgoing `Load` frontiers + for shard classification — it never enters transaction extents, the + connector `Flush`, or durable `Persist` state until its causal hints + resolve and it rides a fully-resolved frontier. +- **No persisted combiner state.** Neither the STALE flags nor the segment + cutoffs survive a session: cutoffs are accumulator-local and reset when the + accumulator is recycled. The per-binding boundary clocks live in the shard + session, and on recovery the shard reconstructs them from the leader's + cumulative `Begin` (committed ∪ hinted) delivered on the first `L:Load`. + ## Status - `leader::materialize` / `shard::materialize` and `leader::derive` / diff --git a/crates/runtime-next/src/leader/capture/fsm.rs b/crates/runtime-next/src/leader/capture/fsm.rs index 28383542e59..66c6b06821a 100644 --- a/crates/runtime-next/src/leader/capture/fsm.rs +++ b/crates/runtime-next/src/leader/capture/fsm.rs @@ -43,6 +43,15 @@ use proto_gazette::uuid; use std::collections::BTreeMap; use std::time::Duration; +/// A backfill message observed within an isolated connector checkpoint. +#[derive(Debug, Clone, Copy)] +pub enum BackfillMessage { + /// A backfill is beginning for `binding`. + BackfillBegin { binding: u32 }, + /// A backfill has completed for `binding`. + BackfillComplete { binding: u32 }, +} + /// Per-transaction aggregated state threaded through Head/Tail FSMs. #[derive(Debug, Default, Clone)] pub struct Extents { @@ -63,6 +72,8 @@ pub struct Extents { sourced_schemas: BTreeMap, // Was a synthetic checkpoint injected due to hard-bound violation? synthetic_checkpoint: bool, + // A backfill message observed in this transaction. + backfill: Option, } #[derive(Debug, Default, Clone)] @@ -85,6 +96,8 @@ pub enum ConnectorRx { Checkpoint(capture::response::Checkpoint), /// Connector emitted a SourcedSchema. SourcedSchema { binding: u32, shape: doc::Shape }, + /// Connector signaled a backfill message (begin or complete). + Backfill(BackfillMessage), /// Connector closed its output stream. Eof, } @@ -96,6 +109,8 @@ impl ConnectorRx { Self::Captured(_) => "Captured", Self::Checkpoint(_) => "Checkpoint", Self::SourcedSchema { .. } => "SourcedSchema", + Self::Backfill(BackfillMessage::BackfillBegin { .. }) => "BackfillBegin", + Self::Backfill(BackfillMessage::BackfillComplete { .. }) => "BackfillComplete", Self::Eof => "Eof", } } @@ -117,6 +132,7 @@ pub enum Tail { Recover(TailRecover), Acknowledge(TailAcknowledge), WriteIntents(TailWriteIntents), + ApplyLabels(TailApplyLabels), Done(TailDone), } @@ -142,8 +158,14 @@ pub enum Action { // Per-binding shapes folded from transaction SourcedSchema messages. sourced_schemas: BTreeMap, }, - /// Publish a stats document as CONTINUE_TXN to the ops stats journal. - WriteStats { stats: ops::proto::Stats }, + /// Publish a stats document as CONTINUE_TXN to the ops stats journal, then + /// snapshot this transaction's ACK intents. When `backfill` is Some, the + /// intents are a backfill marker broadcast instead of the ordinary commit + /// set. + WriteStats { + stats: ops::proto::Stats, + backfill: Option, + }, /// Persist one `proto::Persist` WriteBatch to RocksDB. /// This is the transaction's single, committing Persist. Persist { persist: proto::Persist }, @@ -153,6 +175,8 @@ pub enum Action { WriteIntents { ack_intents: BTreeMap, }, + /// Apply journal truncation labels for the shard's active backfills. + ApplyTruncatedLabels, /// Rotate accumulating and draining combiners. Rotate { extents: Extents }, /// Emit an error. @@ -172,6 +196,7 @@ impl Action { Self::Persist { .. } => "Persist", Self::Acknowledge { .. } => "Acknowledge", Self::WriteIntents { .. } => "WriteIntents", + Self::ApplyTruncatedLabels => "ApplyTruncatedLabels", Self::Rotate { .. } => "Rotate", Self::Error(_) => "Error", } @@ -219,19 +244,22 @@ impl Tail { acknowledge_done: bool, drain_finished: &mut Option, intents_write_idle: bool, + labels_apply_idle: bool, now: uuid::Clock, persist_done: bool, task: &Task, stats_write_idle: Option<&mut BTreeMap>, + active_backfill_change: &mut Option, ) -> (Action, Tail) { match self { Self::Begin(s) => s.step(), Self::Drain(s) => s.step(drain_finished, task), - Self::WriteStats(s) => s.step(now, stats_write_idle), + Self::WriteStats(s) => s.step(now, stats_write_idle, active_backfill_change), Self::Persist(s) => s.step(persist_done), Self::Recover(s) => s.step(task), Self::Acknowledge(s) => s.step(acknowledge_done), Self::WriteIntents(s) => s.step(intents_write_idle), + Self::ApplyLabels(s) => s.step(labels_apply_idle), Self::Done(_) => (Action::Idle, self), } } @@ -245,6 +273,7 @@ impl Tail { Self::Recover(_) => "Recover", Self::Acknowledge(_) => "Acknowledge", Self::WriteIntents(_) => "WriteIntents", + Self::ApplyLabels(_) => "ApplyLabels", Self::Done(_) => "Done", } } @@ -300,6 +329,15 @@ impl HeadIdle { Duration::ZERO }; + let ready_is_backfill = matches!(ready, ConnectorRx::Backfill(_)); + // Force a prompt close to keep a backfill message isolated: flush an open + // data transaction ahead of an incoming backfill message, then close the + // marker transaction itself (it never reaches the size trigger, so it + // would otherwise idle to the age timeout). + if (ready_is_backfill && is_open) || self.extents.backfill.is_some() { + *close_requested = true; + } + let close_policy::Decision { may_close, may_extend, @@ -318,8 +356,9 @@ impl HeadIdle { }); // Should we extend with a ready next connector checkpoint sequence? - if self.extents.synthetic_checkpoint { - // Don't extend transactions after a synthetic checkpoint. + if self.extents.synthetic_checkpoint || self.extents.backfill.is_some() { + // Don't extend once a synthetic checkpoint was injected or a backfill + // message isolated this transaction as a hard boundary. } else if may_extend && matches!( ready, @@ -340,6 +379,21 @@ impl HeadIdle { ); } + // A pending backfill message with no open transaction opens a fresh one + // (it stands alone). Unlike the data-extend path above this is not gated + // on `may_extend`: a backfill message is always admitted into its own + // isolated transaction immediately. + if ready_is_backfill && !is_open { + self.extents.open = now; + return ( + Action::PollAgain, + Head::Extend(HeadExtend { + inner: self, + sequence_bytes: 0, + }), + ); + } + // Should we begin to close the transaction? if !is_open { return (Action::Idle, Head::Idle(self)); @@ -409,6 +463,16 @@ impl HeadExtend { match std::mem::take(ready) { ConnectorRx::Pending => (Action::Idle, Head::Extend(self)), ConnectorRx::Captured(captured) => { + if self.inner.extents.backfill.is_some() { + return ( + Action::Error(anyhow::anyhow!( + "capture connector emitted a document after a backfill \ + message in the same checkpoint; backfill \ + messages must stand alone" + )), + Head::Stop, + ); + } let extents = &mut self.inner.extents; let extent = extents.bindings.entry(captured.binding).or_default(); @@ -422,6 +486,16 @@ impl HeadExtend { (Action::Captured { captured }, Head::Extend(self)) } ConnectorRx::SourcedSchema { binding, shape } => { + if self.inner.extents.backfill.is_some() { + return ( + Action::Error(anyhow::anyhow!( + "capture connector emitted a SourcedSchema after a backfill \ + message in the same checkpoint; backfill \ + messages must stand alone" + )), + Head::Stop, + ); + } let extents = &mut self.inner.extents; let entry = extents @@ -432,6 +506,24 @@ impl HeadExtend { (Action::Idle, Head::Extend(self)) } + ConnectorRx::Backfill(ctrl) => { + let extents = &mut self.inner.extents; + if extents.captured_docs != 0 + || !extents.sourced_schemas.is_empty() + || extents.backfill.is_some() + { + return ( + Action::Error(anyhow::anyhow!( + "capture connector backfill message must stand \ + alone in its checkpoint, with no other documents" + )), + Head::Stop, + ); + } + extents.backfill = Some(ctrl); + // Await the terminating Checkpoint of this isolated sequence. + (Action::Idle, Head::Extend(self)) + } ConnectorRx::Checkpoint(checkpoint) => { let Self { mut inner, @@ -469,6 +561,8 @@ impl TailBegin { let Self { mut extents } = self; // Sourced shapes belong to the drain, not to stats; lift them out of // Extents into the Drain action and let the rest of Extents flow on. + // Any backfill message stays in `extents` and is lifted at the WriteStats + // step, where the marker ACK broadcast is built. let sourced_schemas = std::mem::take(&mut extents.sourced_schemas); ( Action::Drain { sourced_schemas }, @@ -502,8 +596,12 @@ impl TailDrain { } let stats = build_stats_doc(task, &extents); + // Lift any backfill message into the WriteStats action: the actor builds + // the marker ACK broadcast there, alongside the ordinary commit intents. + let backfill = extents.backfill.take(); + ( - Action::WriteStats { stats }, + Action::WriteStats { stats, backfill }, Tail::WriteStats(TailWriteStats { connector_patches, extents, @@ -523,8 +621,10 @@ impl TailWriteStats { self, now: uuid::Clock, stats_write_idle: Option<&mut BTreeMap>, + active_backfill_change: &mut Option, ) -> (Action, Tail) { - // The stats write yields this transaction's ACK intents; hold until it + // The stats write yields this transaction's ACK intents (and, for a + // marker transaction, its resolved active-backfill change); hold until it // completes and they're available. let Some(ack_intents) = stats_write_idle else { return (Action::Idle, Tail::WriteStats(self)); @@ -535,6 +635,9 @@ impl TailWriteStats { extents, } = self; let ack_intents = std::mem::take(ack_intents); + // The change was resolved at intent-build time (it needs the marker's + // commit clock), and is staged alongside the ACK intents. + let active_backfill_change = active_backfill_change.take(); let seq_no = now.as_u64(); // The transaction's single, committing Persist. It records the ACK @@ -542,11 +645,14 @@ impl TailWriteStats { // connector-state patches. `delete_ack_intents` first clears the prior // transaction's per-journal intents — a journal written last transaction // but not this one would otherwise leave a stale entry. + // `active_backfill_change` stages the binding's change atomically with + // the commit (and with the marker ACKs those intents carry). let persist = proto::Persist { seq_no, ack_intents: ack_intents.clone(), connector_patches_json: connector_patches, delete_ack_intents: true, + active_backfill_change, ..Default::default() }; @@ -661,6 +767,27 @@ impl TailWriteIntents { // follow-up Persist clears them from RocksDB: the next transaction's // commit Persist overwrites them, and an idle capture simply re-writes // the same idempotent intents on its next recovery. + // + // Post-commit, re-apply truncated-at journal labels for the shard's + // active backfills. The actor skips the IO when no labels are dirty, + // so this hop is cheap for ordinary transactions. + ( + Action::ApplyTruncatedLabels, + Tail::ApplyLabels(TailApplyLabels {}), + ) + } +} + +/// TailApplyLabels awaits the post-commit truncated-at journal-label apply, +/// then completes the transaction. +#[derive(Debug)] +pub struct TailApplyLabels {} + +impl TailApplyLabels { + pub fn step(self, labels_apply_idle: bool) -> (Action, Tail) { + if !labels_apply_idle { + return (Action::Idle, Tail::ApplyLabels(self)); + } (Action::Idle, Tail::Done(TailDone {})) } } @@ -722,8 +849,10 @@ mod tests { combiner_bytes: u64, drain_finished: Option, intents_idle: bool, + labels_idle: bool, now: uuid::Clock, pending_ack_intents: BTreeMap, + pending_active_backfill_change: Option, persist_done: bool, ready: ConnectorRx, stats_idle: bool, @@ -751,10 +880,12 @@ mod tests { self.acknowledge_done, &mut self.drain_finished, self.intents_idle, + self.labels_idle, self.now, self.persist_done, &self.task, self.stats_idle.then_some(&mut self.pending_ack_intents), + &mut self.pending_active_backfill_change, ) } } @@ -767,8 +898,10 @@ mod tests { combiner_bytes: 0, drain_finished: None, intents_idle: true, + labels_idle: true, now: uuid::Clock::from_unix(1_700_000_000, 0), pending_ack_intents: BTreeMap::new(), + pending_active_backfill_change: None, persist_done: true, ready: ConnectorRx::Pending, stats_idle: true, @@ -873,6 +1006,11 @@ mod tests { ctx.intents_idle = true; let (action, t) = ctx.step_tail(tail); tail = t; + // Post-commit: a no-op ApplyLabels hop (no active backfills), then Done. + assert!(matches!(action, Action::ApplyTruncatedLabels)); + assert!(matches!(tail, Tail::ApplyLabels(_))); + let (action, t) = ctx.step_tail(tail); + tail = t; assert!(matches!(action, Action::Idle)); assert!(matches!(tail, Tail::Done(_))); @@ -1009,7 +1147,7 @@ mod tests { let (action, t) = ctx.step_tail(tail); tail = t; let stats = match action { - Action::WriteStats { stats } => stats, + Action::WriteStats { stats, .. } => stats, other => panic!("expected WriteStats, got {other:?}"), }; assert!(matches!(tail, Tail::WriteStats(_))); @@ -1017,7 +1155,7 @@ mod tests { { "_meta": {}, "shard": {}, - "ts": "2023-11-14T22:13:20.000004+00:00", + "ts": "2023-11-14T22:13:20.000005+00:00", "openSecondsTotal": 0.000008, "txnCount": 1, "capture": { @@ -1030,7 +1168,7 @@ mod tests { "docsTotal": 2, "bytesTotal": 50 }, - "lastPublishedAt": "2023-11-14T22:13:20.000012+00:00" + "lastPublishedAt": "2023-11-14T22:13:20.000013+00:00" }, "test/collectionB": { "right": { @@ -1041,7 +1179,7 @@ mod tests { "docsTotal": 1, "bytesTotal": 25 }, - "lastPublishedAt": "2023-11-14T22:13:20.000012+00:00" + "lastPublishedAt": "2023-11-14T22:13:20.000013+00:00" } } } @@ -1133,6 +1271,11 @@ mod tests { ctx.intents_idle = true; let (action, t) = ctx.step_tail(tail); tail = t; + // Post-commit ApplyLabels hop (no active backfills), then Done. + assert!(matches!(action, Action::ApplyTruncatedLabels)); + assert!(matches!(tail, Tail::ApplyLabels(_))); + let (action, t) = ctx.step_tail(tail); + tail = t; assert!(matches!(action, Action::Idle)); assert!(matches!(tail, Tail::Done(_))); @@ -1175,6 +1318,11 @@ mod tests { assert!(matches!(action, Action::WriteIntents { .. })); let (action, t) = ctx.step_tail(tail); tail = t; + // Post-commit ApplyLabels hop (no active backfills), then Done. + assert!(matches!(action, Action::ApplyTruncatedLabels)); + assert!(matches!(tail, Tail::ApplyLabels(_))); + let (action, t) = ctx.step_tail(tail); + tail = t; assert!(matches!(action, Action::Idle)); assert!(matches!(tail, Tail::Done(_))); @@ -1310,13 +1458,21 @@ mod tests { // the connector already being Pending) and only rarely `Eof`, so traces // spend their time accumulating and committing rather than stopping early. fn random_connector_rx(rng: &mut SmallRng) -> ConnectorRx { - match rng.random_range(0..12) { + match rng.random_range(0..13) { 0..=4 => captured(rng.random_range(0..3), b"{\"v\":1}"), 5..=8 => checkpoint(), 9..=10 => ConnectorRx::SourcedSchema { binding: rng.random_range(0..3), shape: doc::Shape::nothing(), }, + 11 if rng.random_bool(0.5) => { + ConnectorRx::Backfill(BackfillMessage::BackfillBegin { + binding: rng.random_range(0..3), + }) + } + 11 => ConnectorRx::Backfill(BackfillMessage::BackfillComplete { + binding: rng.random_range(0..3), + }), _ => ConnectorRx::Eof, } } @@ -1369,6 +1525,22 @@ mod tests { }); } + // Independently of the trace's actual control flow, sometimes stage an + // active-backfill change so it threads WriteStats → Persist. + if rng.random_bool(0.20) { + ctx.pending_active_backfill_change = match rng.random_range(0..2) { + 0 => Some(proto::persist::ActiveBackfillChange::Begin( + proto::ActiveBackfillBegin { + binding: rng.random_range(0..3), + truncated_at: rng.random_range(1..1_000_000), + }, + )), + _ => Some(proto::persist::ActiveBackfillChange::CompleteBinding( + rng.random_range(0..3), + )), + }; + } + // Occasionally add an ACK intent; WriteStats drains them into Persist. if rng.random_bool(0.10) { ctx.pending_ack_intents.insert( @@ -1428,4 +1600,260 @@ mod tests { .max_tests(400) .quickcheck(prop as fn(u64) -> bool); } + + /// A backfill marker transaction, end to end. + #[test] + fn marker_transaction_commits_active_backfill() { + let mut ctx = mk_ctx(mk_task(false)); + let mut head = Head::Idle(HeadIdle::default()); + // Tail is Done, so Head may rotate the moment the marker txn seals. + let tail = Tail::Done(TailDone::default()); + + // A BackfillBegin with no open transaction opens its own isolated one. + ctx.ready = ConnectorRx::Backfill(BackfillMessage::BackfillBegin { binding: 0 }); + let (action, h) = ctx.step_head(head, &tail); + head = h; + assert!(matches!(action, Action::PollAgain)); + assert!(matches!(head, Head::Extend(_))); + + // HeadExtend folds the backfill message into `extents.backfill`. + let (action, h) = ctx.step_head(head, &tail); + head = h; + assert!(matches!(action, Action::Idle)); + let Head::Extend(extend) = &head else { + panic!("expected Extend, got {}", head.kind()); + }; + assert!(matches!( + extend.inner.extents.backfill, + Some(BackfillMessage::BackfillBegin { binding: 0 }), + )); + + // The terminating Checkpoint completes the isolated sequence. + ctx.ready = checkpoint(); + let (action, h) = ctx.step_head(head, &tail); + head = h; + assert!(matches!(action, Action::Checkpoint { .. })); + assert!(matches!(head, Head::Idle(_))); + + // A document now waits behind the sealed marker transaction. It can + // neither extend the (refuse-extend) marker txn nor open its own. + ctx.ready = captured(0, b"{}"); + let (action, _head) = ctx.step_head(head, &tail); + assert!( + ctx.close_requested, + "the sealed marker transaction requests a prompt close", + ); + let extents = match action { + Action::Rotate { extents } => extents, + other => panic!("expected Rotate, got {other:?}"), + }; + + // Begin no longer lifts the backfill message — the drain publishes no marker. + let mut tail = Tail::Begin(TailBegin { extents }); + let (action, t) = ctx.step_tail(tail); + tail = t; + assert!(matches!(action, Action::Drain { .. })); + + // The (empty) drain completes; TailDrain lifts the backfill message into + // the WriteStats action, where the actor builds the marker ACK broadcast. + ctx.drain_finished = Some(DrainedCapture { + connector_patches: Bytes::new(), + bindings: BTreeMap::new(), + }); + let (action, t) = ctx.step_tail(tail); + tail = t; + match action { + Action::WriteStats { backfill, .. } => assert!(matches!( + backfill, + Some(BackfillMessage::BackfillBegin { binding: 0 }), + )), + other => panic!("expected WriteStats, got {other:?}"), + } + + // The stats write resolves the marker's broadcast clock into the + // active-backfill change (staged directly here); WriteStats folds it into + // the committing Persist verbatim. + ctx.pending_active_backfill_change = Some(proto::persist::ActiveBackfillChange::Begin( + proto::ActiveBackfillBegin { + binding: 0, + truncated_at: 0xABCD, + }, + )); + let (action, t) = ctx.step_tail(tail); + tail = t; + let persist = match action { + Action::Persist { persist } => persist, + other => panic!("expected Persist, got {other:?}"), + }; + assert_eq!( + persist.active_backfill_change, + Some(proto::persist::ActiveBackfillChange::Begin( + proto::ActiveBackfillBegin { + binding: 0, + truncated_at: 0xABCD, + }, + )), + ); + + // The remaining commit drains to Done: Persist → Recover → WriteIntents + // (no explicit acks) → ApplyLabels → Done. + let (action, t) = ctx.step_tail(tail); + tail = t; + assert!(matches!(action, Action::PollAgain)); + let (action, t) = ctx.step_tail(tail); + tail = t; + assert!(matches!(action, Action::WriteIntents { .. })); + let (action, t) = ctx.step_tail(tail); + tail = t; + assert!(matches!(action, Action::ApplyTruncatedLabels)); + assert!(matches!(tail, Tail::ApplyLabels(_))); + + // A marker transaction makes the active-backfill set dirty, so the + // post-commit label apply actually runs: hold in ApplyLabels while that + // IO is in flight, then complete to Done once it lands. + ctx.labels_idle = false; + let (action, t) = ctx.step_tail(tail); + tail = t; + assert!(matches!(action, Action::Idle)); + assert!(matches!(tail, Tail::ApplyLabels(_))); + + ctx.labels_idle = true; + let (action, t) = ctx.step_tail(tail); + tail = t; + assert!(matches!(action, Action::Idle)); + assert!(matches!(tail, Tail::Done(_))); + } + + /// A backfill message opens its own transaction immediately, ungated by the + /// close policy: even when `may_extend` is false (so a data message would not + /// open one), a BackfillBegin is admitted into its own isolated transaction. + #[test] + fn backfill_message_opens_transaction_ignoring_close_policy() { + let mut ctx = mk_ctx(mk_task(false)); + // Narrow the policy and load the combiner so `may_extend` is false. + ctx.task.close_policy.combiner_usage_bytes = 0..10_000; + ctx.combiner_bytes = 1_000_000; + let tail = Tail::Done(TailDone::default()); + + // A Captured into a closed Head does NOT open a transaction. + ctx.ready = captured(0, b"{}"); + let (action, head) = ctx.step_head(Head::Idle(HeadIdle::default()), &tail); + assert!( + !matches!(action, Action::PollAgain) && matches!(head, Head::Idle(_)), + "a data message must not open while may_extend is false, got {action:?}", + ); + + // A BackfillBegin DOES open its own isolated transaction. + ctx.ready = ConnectorRx::Backfill(BackfillMessage::BackfillBegin { binding: 1 }); + let (action, head) = ctx.step_head(Head::Idle(HeadIdle::default()), &tail); + assert!(matches!(action, Action::PollAgain)); + assert!(matches!(head, Head::Extend(_))); + } + + /// A backfill message arriving while a *data* transaction is open forces that + /// transaction to close first, so the backfill message lands in its own isolated + /// transaction rather than mixing into the open one. + #[test] + fn incoming_backfill_message_closes_open_data_transaction() { + let mut ctx = mk_ctx(mk_task(false)); + let head = Head::Idle(HeadIdle { + extents: Extents { + checkpoints: 1, // is_open + captured_docs: 3, + ..Default::default() + }, + last_close: ctx.now, + }); + // A BackfillBegin waits behind the open data transaction. + ctx.ready = ConnectorRx::Backfill(BackfillMessage::BackfillBegin { binding: 0 }); + + let (action, head) = ctx.step_head(head, &Tail::Done(TailDone::default())); + assert!( + ctx.close_requested, + "an incoming backfill message requests the open transaction's close", + ); + let extents = match action { + Action::Rotate { extents } => extents, + other => panic!("expected Rotate, got {other:?}"), + }; + // The rotated transaction carries the data; the backfill message is still queued in + // `ready`, to stand alone in the next transaction. + assert!(extents.backfill.is_none()); + assert!(matches!(head, Head::Idle(_))); + } + + /// The "a backfill message stands alone in its checkpoint" invariant: mixing a + /// backfill message with documents or schemas in the same checkpoint sequence, + /// in either order, fails the task. + #[test] + fn backfill_message_must_stand_alone() { + // (label, pre-accrued extents, the violating ready message) + let cases: Vec<(&str, Extents, ConnectorRx)> = vec![ + ( + "captured after backfill message", + Extents { + backfill: Some(BackfillMessage::BackfillBegin { binding: 0 }), + ..Default::default() + }, + captured(0, b"{}"), + ), + ( + "sourced_schema after backfill message", + Extents { + backfill: Some(BackfillMessage::BackfillBegin { binding: 0 }), + ..Default::default() + }, + ConnectorRx::SourcedSchema { + binding: 0, + shape: doc::Shape::nothing(), + }, + ), + ( + "backfill message after captured", + Extents { + captured_docs: 1, + ..Default::default() + }, + ConnectorRx::Backfill(BackfillMessage::BackfillBegin { binding: 0 }), + ), + ( + "backfill message after sourced_schema", + Extents { + sourced_schemas: BTreeMap::from([(0, doc::Shape::nothing())]), + ..Default::default() + }, + ConnectorRx::Backfill(BackfillMessage::BackfillBegin { binding: 0 }), + ), + ( + "backfill message after backfill message", + Extents { + backfill: Some(BackfillMessage::BackfillBegin { binding: 0 }), + ..Default::default() + }, + ConnectorRx::Backfill(BackfillMessage::BackfillComplete { binding: 1 }), + ), + ]; + + for (label, extents, ready) in cases { + let mut ctx = mk_ctx(mk_task(false)); + ctx.ready = ready; + let head = Head::Extend(HeadExtend { + inner: HeadIdle { + extents, + last_close: ctx.now, + }, + sequence_bytes: 0, + }); + + let (action, head) = ctx.step_head(head, &Tail::Done(TailDone::default())); + let Action::Error(error) = action else { + panic!("{label}: expected Error, got {action:?}"); + }; + assert!( + format!("{error:?}").contains("stand alone"), + "{label}: {error:?}", + ); + assert!(matches!(head, Head::Stop), "{label}"); + } + } } diff --git a/crates/runtime-next/src/leader/derive/actor.rs b/crates/runtime-next/src/leader/derive/actor.rs index 6cf49b42b0c..7ebffda0191 100644 --- a/crates/runtime-next/src/leader/derive/actor.rs +++ b/crates/runtime-next/src/leader/derive/actor.rs @@ -415,7 +415,8 @@ impl Actor { all_commits.push(leader_commit); } - let intents = publisher::intents::build_transaction_intents(&all_commits); + let intents = + publisher::intents::build_transaction_intents(&all_commits, None); Ok((publisher, intents)) } diff --git a/crates/runtime-next/src/leader/derive/startup.rs b/crates/runtime-next/src/leader/derive/startup.rs index e19fb8c92c9..ff1456181ca 100644 --- a/crates/runtime-next/src/leader/derive/startup.rs +++ b/crates/runtime-next/src/leader/derive/startup.rs @@ -256,6 +256,7 @@ impl Baseline { connector_state_json: _, max_keys, trigger_params_json: _, + active_backfills: _, // capture-only state } = recover; // Derivations never track max-keys or a hinted frontier. diff --git a/crates/runtime-next/src/leader/materialize/actor.rs b/crates/runtime-next/src/leader/materialize/actor.rs index 06299ad06b6..0b8d5f2cab8 100644 --- a/crates/runtime-next/src/leader/materialize/actor.rs +++ b/crates/runtime-next/src/leader/materialize/actor.rs @@ -11,6 +11,12 @@ use tokio::sync::mpsc; /// Actor leads transactions of an established materialization task session. pub struct Actor { + // Cumulative per-binding backfill-begin clock, accumulated across all + // transactions of the session. + backfill_begin: BTreeMap, + // Cumulative per-binding backfill-complete clock, accumulated across all + // transactions of the session. + backfill_complete: BTreeMap, // Client used for trigger dispatch. http_client: reqwest::Client, // Future for an in-flight ACK intents write, if any. @@ -41,6 +47,8 @@ pub struct Actor { impl Actor { pub fn new( + backfill_begin: BTreeMap, + backfill_complete: BTreeMap, http_client: reqwest::Client, legacy_checkpoint: Option, metrics: super::Metrics, @@ -50,6 +58,8 @@ impl Actor { task: Task, ) -> Self { Self { + backfill_begin, + backfill_complete, http_client, intents_write_fut: None, legacy_checkpoint: legacy_checkpoint.map(|f| (f, consumer::Checkpoint::default())), @@ -127,7 +137,7 @@ impl Actor { "leader materialize Actor::serve iteration" ); - let action: fsm::Action; + let mut action: fsm::Action; let prev_kind = tail.kind(); (action, tail) = tail.step( &self.trigger_debounce, @@ -148,6 +158,7 @@ impl Actor { "transition", ); } + self.merge_backfill_clocks(&mut action); let tail_wake_after = self.dispatch(action)?; let action: fsm::Action; @@ -199,7 +210,10 @@ impl Actor { Duration::ZERO } - action => self.dispatch(action)?, + mut action => { + self.merge_backfill_clocks(&mut action); + self.dispatch(action)? + } }; let wake_after = std::cmp::min(head_wake_after, tail_wake_after); @@ -333,6 +347,75 @@ impl Actor { Ok(()) } + /// Stamp the session-cumulative backfill boundary onto outgoing frontiers: + /// `Load` stamps begin only (shards classify on begin; the connector's + /// Complete rides `Flush`), `Persist` stamps both maps as durable state. + fn merge_backfill_clocks(&mut self, action: &mut fsm::Action) { + match action { + fsm::Action::Load { frontier } if frontier.unresolved_hints != 0 => { + // Eager peek: fold the cumulative begin into the frontier's own + // eager begin (a union) without advancing the cumulative maps. + for (binding, clock) in &self.backfill_begin { + let entry = frontier + .latest_backfill_begin + .entry(*binding) + .or_insert(uuid::Clock::zero()); + *entry = (*entry).max(*clock); + } + } + fsm::Action::Load { frontier } => { + // Resolved Load: fold the begin and complete deltas into the + // cumulative maps (complete feeds Persist), then stamp begin. + for (binding, clock) in &frontier.latest_backfill_begin { + let entry = self + .backfill_begin + .entry(*binding) + .or_insert(uuid::Clock::zero()); + *entry = (*entry).max(*clock); + } + for (binding, clock) in &frontier.latest_backfill_complete { + let entry = self + .backfill_complete + .entry(*binding) + .or_insert(uuid::Clock::zero()); + *entry = (*entry).max(*clock); + } + frontier.latest_backfill_begin = self.backfill_begin.clone(); + } + fsm::Action::Persist { persist } => { + let begin: Vec<_> = self + .backfill_begin + .iter() + .map(|(binding, clock)| shuffle::proto::frontier::BackfillBegin { + binding: *binding as u32, + clock: clock.as_u64(), + }) + .collect(); + let complete: Vec<_> = self + .backfill_complete + .iter() + .map( + |(binding, clock)| shuffle::proto::frontier::BackfillComplete { + binding: *binding as u32, + clock: clock.as_u64(), + }, + ) + .collect(); + for frontier in [ + &mut persist.committed_frontier, + &mut persist.hinted_frontier, + ] + .into_iter() + .flatten() + { + frontier.latest_backfill_begin = begin.clone(); + frontier.latest_backfill_complete = complete.clone(); + } + } + _ => {} + } + } + /// Execute the outgoing-IO primitive for an Action. #[tracing::instrument(level = "trace", fields(action = ?action), skip_all)] fn dispatch(&mut self, action: fsm::Action) -> anyhow::Result { @@ -354,11 +437,35 @@ impl Actor { }); } - fsm::Action::Flush { connector_patches } => { + fsm::Action::Flush { + connector_patches, + backfill_begins, + backfill_completes, + } => { service_kit::event!(tracing::Level::DEBUG, "shard", "broadcasting L:Flush"); + let backfill_begins = backfill_begins + .into_iter() + .map( + |(binding, clock)| proto::materialize::flush::BackfillBegin { + binding: binding as u32, + clock: clock.as_u64(), + }, + ) + .collect(); + let backfill_completes = backfill_completes + .into_iter() + .map( + |(binding, clock)| proto::materialize::flush::BackfillComplete { + binding: binding as u32, + clock: clock.as_u64(), + }, + ) + .collect(); self.broadcast(proto::Materialize { flush: Some(proto::materialize::Flush { connector_patches_json: connector_patches, + backfill_begins, + backfill_completes, }), ..Default::default() }); @@ -428,7 +535,7 @@ impl Actor { let intents = match publisher.commit_intents() { Some(commit) => { - publisher::intents::build_transaction_intents(&[commit]) + publisher::intents::build_transaction_intents(&[commit], None) } None => BTreeMap::new(), }; @@ -608,6 +715,8 @@ mod tests { triggers: None, }; let actor = Actor::new( + BTreeMap::new(), + BTreeMap::new(), reqwest::Client::new(), None, super::super::Metrics::new("test/task/shard"), @@ -729,4 +838,229 @@ mod tests { } } } + + #[test] + fn merge_backfill_clocks_load_advances_and_stamps() { + let (mut actor, _rxs) = mk_actor(1); + actor.backfill_begin = BTreeMap::from([(0, uuid::Clock::from_u64(5))]); + actor.backfill_complete = BTreeMap::from([(0, uuid::Clock::from_u64(4))]); + + // Incoming Load delta: an older binding 0 (must not regress the + // cumulative) and a fresh binding 1. + let mut action = fsm::Action::Load { + frontier: shuffle::Frontier { + latest_backfill_begin: BTreeMap::from([ + (0, uuid::Clock::from_u64(3)), + (1, uuid::Clock::from_u64(7)), + ]), + latest_backfill_complete: BTreeMap::from([(1, uuid::Clock::from_u64(6))]), + ..Default::default() + }, + }; + actor.merge_backfill_clocks(&mut action); + + let want_begin = BTreeMap::from([ + (0, uuid::Clock::from_u64(5)), // kept 5, not regressed to 3 + (1, uuid::Clock::from_u64(7)), + ]); + let want_complete = + BTreeMap::from([(0, uuid::Clock::from_u64(4)), (1, uuid::Clock::from_u64(6))]); + + // The cumulative maps advance (max-fold), never regress. + assert_eq!(actor.backfill_begin, want_begin); + assert_eq!(actor.backfill_complete, want_complete); + + // The outgoing frontier carries the full cumulative begin, not just the + // delta. (Complete isn't stamped on a Load; only the fold above matters.) + let fsm::Action::Load { frontier } = &action else { + panic!("expected Load"); + }; + assert_eq!(frontier.latest_backfill_begin, want_begin); + } + + // An eager (unresolved-peek) Load stamps cumulative ∪ eager begin but must + // NOT advance the cumulative maps; a following Persist stamps only the + // pre-eager resolved state. + #[test] + fn merge_backfill_clocks_eager_load_does_not_advance_cumulative() { + let (mut actor, _rxs) = mk_actor(1); + actor.backfill_begin = BTreeMap::from([(0, uuid::Clock::from_u64(5))]); + actor.backfill_complete = BTreeMap::from([(0, uuid::Clock::from_u64(4))]); + + // An unresolved peek carrying eager markers: a higher begin for binding + // 0, a fresh binding 1, and a complete. + let mut action = fsm::Action::Load { + frontier: shuffle::Frontier { + latest_backfill_begin: BTreeMap::from([ + (0, uuid::Clock::from_u64(9)), + (1, uuid::Clock::from_u64(7)), + ]), + latest_backfill_complete: BTreeMap::from([(0, uuid::Clock::from_u64(8))]), + unresolved_hints: 1, + ..Default::default() + }, + }; + actor.merge_backfill_clocks(&mut action); + + // The outgoing frontier carries cumulative ∪ eager begin. + let fsm::Action::Load { frontier } = &action else { + panic!("expected Load"); + }; + assert_eq!( + frontier.latest_backfill_begin, + BTreeMap::from([(0, uuid::Clock::from_u64(9)), (1, uuid::Clock::from_u64(7))]), + ); + + // Neither cumulative map advanced — the eager markers are not durable. + assert_eq!( + actor.backfill_begin, + BTreeMap::from([(0, uuid::Clock::from_u64(5))]), + "eager begin must not advance the cumulative map", + ); + assert_eq!( + actor.backfill_complete, + BTreeMap::from([(0, uuid::Clock::from_u64(4))]), + "eager complete must not advance the cumulative map", + ); + + // A subsequent Persist stamps only the pre-eager cumulative begin (5). + let mut persist = fsm::Action::Persist { + persist: proto::Persist { + committed_frontier: Some(shuffle::proto::Frontier::default()), + ..Default::default() + }, + }; + actor.merge_backfill_clocks(&mut persist); + let fsm::Action::Persist { persist } = &persist else { + panic!("expected Persist"); + }; + assert_eq!( + persist + .committed_frontier + .as_ref() + .unwrap() + .latest_backfill_begin, + vec![shuffle::proto::frontier::BackfillBegin { + binding: 0, + clock: 5, + }], + "Persist stamps only durable resolved state, never the eager begin", + ); + } + + #[test] + fn merge_backfill_clocks_persist_stamps() { + let (mut actor, _rxs) = mk_actor(1); + actor.backfill_begin = BTreeMap::from([(0, uuid::Clock::from_u64(5))]); + actor.backfill_complete = BTreeMap::from([(0, uuid::Clock::from_u64(4))]); + + let mut action = fsm::Action::Persist { + persist: proto::Persist { + committed_frontier: Some(shuffle::proto::Frontier::default()), + ..Default::default() + }, + }; + actor.merge_backfill_clocks(&mut action); + + let fsm::Action::Persist { persist } = &action else { + panic!("expected Persist"); + }; + let frontier = persist.committed_frontier.as_ref().unwrap(); + assert_eq!( + frontier.latest_backfill_begin, + vec![shuffle::proto::frontier::BackfillBegin { + binding: 0, + clock: 5, + }], + ); + assert_eq!( + frontier.latest_backfill_complete, + vec![shuffle::proto::frontier::BackfillComplete { + binding: 0, + clock: 4, + }], + ); + } + + // A hint Persist carries a `hinted_frontier`, not a `committed_frontier`; it + // must be stamped too. The hinted boundary is persisted durably (HB:/HC: keys, + // see recovery.rs) and reduced into committed on a remote-authoritative + // recovery, so a marker observed in the hinted transaction isn't lost. + #[test] + fn merge_backfill_clocks_stamps_hinted_frontier() { + let (mut actor, _rxs) = mk_actor(1); + actor.backfill_begin = BTreeMap::from([(0, uuid::Clock::from_u64(5))]); + + let mut action = fsm::Action::Persist { + persist: proto::Persist { + hinted_frontier: Some(shuffle::proto::Frontier::default()), + ..Default::default() + }, + }; + actor.merge_backfill_clocks(&mut action); + + let fsm::Action::Persist { persist } = &action else { + panic!("expected Persist"); + }; + assert_eq!( + persist + .hinted_frontier + .as_ref() + .unwrap() + .latest_backfill_begin, + vec![shuffle::proto::frontier::BackfillBegin { + binding: 0, + clock: 5, + }], + ); + } + + #[test] + fn merge_backfill_clocks_noop_cases() { + let (mut actor, _rxs) = mk_actor(1); + actor.backfill_begin = BTreeMap::from([(0, uuid::Clock::from_u64(5))]); + + // A Persist without a committed frontier has nothing to stamp... + let mut persist = fsm::Action::Persist { + persist: proto::Persist::default(), + }; + actor.merge_backfill_clocks(&mut persist); + + // ...and a non-Load/Persist action is ignored. + let mut store = fsm::Action::Store; + actor.merge_backfill_clocks(&mut store); + + assert_eq!( + actor.backfill_begin, + BTreeMap::from([(0, uuid::Clock::from_u64(5))]), + ); + } + + // Ordinary delivery: the FSM's per-transaction marker delta (carried on + // `Action::Flush`) is serialized onto the broadcast L:Flush, so each shard + // can notify its connector. + #[test] + fn flush_action_serializes_backfill_delta() { + let (mut actor, mut rxs) = mk_actor(2); + + actor + .dispatch(fsm::Action::Flush { + connector_patches: bytes::Bytes::new(), + backfill_begins: BTreeMap::from([(0, uuid::Clock::from_u64(42))]), + backfill_completes: BTreeMap::new(), + }) + .unwrap(); + + for rx in &mut rxs { + let flush = rx.try_recv().unwrap().unwrap().flush.unwrap(); + assert_eq!( + flush.backfill_begins, + vec![proto::materialize::flush::BackfillBegin { + binding: 0, + clock: 42, + }], + ); + assert!(flush.backfill_completes.is_empty()); + } + } } diff --git a/crates/runtime-next/src/leader/materialize/fsm.rs b/crates/runtime-next/src/leader/materialize/fsm.rs index d975751e1f0..438aa284d07 100644 --- a/crates/runtime-next/src/leader/materialize/fsm.rs +++ b/crates/runtime-next/src/leader/materialize/fsm.rs @@ -105,6 +105,11 @@ pub enum Action { Flush { // Prior transaction's C:Acknowledged patches. connector_patches: bytes::Bytes, + // Backfill-begin markers observed this transaction, forwarded to each + // shard's connector as a notification. + backfill_begins: BTreeMap, + // Backfill-complete markers observed this transaction. + backfill_completes: BTreeMap, }, /// Broadcast `L:Store`. Store, @@ -327,7 +332,16 @@ impl HeadIdle { self.extents.open = now; self.combiner_usage_bytes = vec![0; task.n_shards]; } - self.extents.frontier = self.extents.frontier.reduce(frontier.clone()); + + // Extents fold in the full frontier minus an unresolved peek's + // backfill markers — they must not reach Flush/durable state until + // they resolve. + let mut extents_delta = frontier.clone(); + if extents_delta.unresolved_hints != 0 { + extents_delta.latest_backfill_begin = Default::default(); + extents_delta.latest_backfill_complete = Default::default(); + } + self.extents.frontier = self.extents.frontier.reduce(extents_delta); return ( Action::Load { frontier }, @@ -391,7 +405,11 @@ impl HeadIdle { }; return ( - Action::Flush { connector_patches }, + Action::Flush { + connector_patches, + backfill_begins: extents.frontier.latest_backfill_begin.clone(), + backfill_completes: extents.frontier.latest_backfill_complete.clone(), + }, Head::Flush(HeadFlush { extents, pending, diff --git a/crates/runtime-next/src/leader/materialize/handler.rs b/crates/runtime-next/src/leader/materialize/handler.rs index cd7eb638822..5fec5e7d972 100644 --- a/crates/runtime-next/src/leader/materialize/handler.rs +++ b/crates/runtime-next/src/leader/materialize/handler.rs @@ -203,6 +203,8 @@ where publisher, session, task, + backfill_begin, + backfill_complete, } = startup::run( build, drop_v1_rollback, @@ -238,6 +240,8 @@ where }; let mut actor = actor::Actor::new( + backfill_begin, + backfill_complete, service.http_client.clone(), legacy_checkpoint, metrics, diff --git a/crates/runtime-next/src/leader/materialize/startup.rs b/crates/runtime-next/src/leader/materialize/startup.rs index 730260ccbf1..ac229158d7b 100644 --- a/crates/runtime-next/src/leader/materialize/startup.rs +++ b/crates/runtime-next/src/leader/materialize/startup.rs @@ -30,6 +30,10 @@ pub(super) struct Startup, + // Leader's initial cumulative backfill-complete boundary (committed ∪ hinted). + pub backfill_complete: BTreeMap, } #[tracing::instrument( @@ -124,6 +128,7 @@ pub(super) async fn run< legacy_checkpoint, max_keys, trigger_params_json: pending_trigger_params, + active_backfills: _, // capture-only state } = recv_recovers(shard_rx, &task.peers) .await .context("receiving Recover fan-in")?; @@ -232,6 +237,8 @@ pub(super) async fn run< committed_close, committed_frontier, pending_ack_intents, + backfill_begin, + backfill_complete, ) = scanned.into_projected_parts(); // Open the shuffle Session with the recovered resume Frontier. @@ -254,6 +261,8 @@ pub(super) async fn run< publisher, session, task, + backfill_begin, + backfill_complete, }) } @@ -295,6 +304,7 @@ impl Baseline { connector_state_json: _, max_keys: _, trigger_params_json: _, + active_backfills: _, // capture-only state } = recover; let baseline = Baseline { @@ -322,6 +332,8 @@ impl Baseline { uuid::Clock, shuffle::Frontier, BTreeMap, + BTreeMap, + BTreeMap, ) { let Baseline { committed_close, @@ -330,14 +342,47 @@ impl Baseline { ack_intents, .. } = self; - let (resume_frontier, idempotent_replay) = + + // Markers the hinted transaction adds over committed — the delta the + // replay re-notifies (not the whole cumulative, which would re-notify + // every backfill). + let delta_begin = backfill_delta( + &hinted_frontier.latest_backfill_begin, + &committed_frontier.latest_backfill_begin, + ); + let delta_complete = backfill_delta( + &hinted_frontier.latest_backfill_complete, + &committed_frontier.latest_backfill_complete, + ); + // Leader's cumulative = committed ∪ hinted, so a hinted marker is present + // for prior-gen load classification on every replay Load (peek rounds + // included). The committed boundary survives a checkpoint adopt (only + // `FC:` is cleared) and the hinted boundary is dropped with the hints + // (`delete_hinted_frontier` clears `HB:`/`HC:`), so the scanned Baseline + // already reflects the reconciled truncation state. + let backfill_begin = backfill_union( + committed_frontier.latest_backfill_begin.clone(), + &hinted_frontier.latest_backfill_begin, + ); + let backfill_complete = backfill_union( + committed_frontier.latest_backfill_complete.clone(), + &hinted_frontier.latest_backfill_complete, + ); + + let (mut resume_frontier, idempotent_replay) = Self::resume_frontier(hinted_frontier, committed_frontier.clone()); + // The session replay carries only the marker delta. + resume_frontier.latest_backfill_begin = delta_begin; + resume_frontier.latest_backfill_complete = delta_complete; + ( resume_frontier, idempotent_replay, committed_close, committed_frontier, ack_intents, + backfill_begin, + backfill_complete, ) } @@ -762,6 +807,34 @@ async fn recv_opened( Ok(openeds.swap_remove(0)) } +fn backfill_union( + mut a: BTreeMap, + b: &BTreeMap, +) -> BTreeMap { + for (binding, clock) in b { + let entry = a.entry(*binding).or_insert(uuid::Clock::zero()); + *entry = (*entry).max(*clock); + } + a +} + +fn backfill_delta( + hinted: &BTreeMap, + committed: &BTreeMap, +) -> BTreeMap { + hinted + .iter() + .filter(|(binding, clock)| { + **clock + > committed + .get(*binding) + .copied() + .unwrap_or(uuid::Clock::zero()) + }) + .map(|(binding, clock)| (*binding, *clock)) + .collect() +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/runtime-next/src/lib.rs b/crates/runtime-next/src/lib.rs index 6ff5911ffc5..26e6166b0a1 100644 --- a/crates/runtime-next/src/lib.rs +++ b/crates/runtime-next/src/lib.rs @@ -226,6 +226,16 @@ impl Accumulator { Ok((memtable, alloc, self.1.parse_one(doc_bytes, alloc)?)) } + /// Truncate `binding`'s backfill boundary: pre-boundary documents the + /// combiner holds become stale (existence-only) and already-spilled segments + /// are fenced. See [`doc::combine::Accumulator::truncate`]. + pub fn truncate(&mut self, binding: usize) { + self.0.truncate(binding) + } + + /// Drain the combiner. Stale entries — flagged or fenced by a backfill + /// truncation — are discarded, transferring only their existence onto the + /// fresh entry of a shared (binding, key). pub fn into_drainer( self, ) -> Result<(doc::combine::Drainer, simd_doc::Parser), doc::combine::Error> { diff --git a/crates/runtime-next/src/publish.rs b/crates/runtime-next/src/publish.rs index e7efc7f8ff2..8615562cf4d 100644 --- a/crates/runtime-next/src/publish.rs +++ b/crates/runtime-next/src/publish.rs @@ -59,6 +59,26 @@ pub trait Publisher: Send + 'static { /// Flush all currently buffered documents. fn flush(&mut self) -> impl std::future::Future> + Send; + /// Snapshot a backfill marker broadcast for `binding_index`'s collection: + /// every partition journal, a single ticked commit clock, and the producer + /// identity — fed to `publisher::intents::build_transaction_intents` with a + /// marker. For a `Begin`, the returned clock is the authoritative + /// `truncated_at`. `None` when the publisher performs no real journal IO. + fn marker_commit( + &mut self, + binding_index: usize, + ) -> impl std::future::Future< + Output = tonic::Result)>>, + > + Send; + + /// Apply (or re-apply) the `estuary.dev/truncated-at` journal label for the + /// shard's active backfills. `active_backfills` is keyed by task-binding + /// index; the value is the backfill's begin-clock. + fn apply_truncated_at_labels( + &mut self, + active_backfills: &BTreeMap, + ) -> impl std::future::Future> + Send; + /// Snapshot this publisher's contribution to the current transaction's /// ACK intents, or `None` when no real publishes happened (so there are no /// commit positions to encode). @@ -215,6 +235,26 @@ impl Publisher for JournalPublisher { self.0.flush().await } + async fn marker_commit( + &mut self, + binding_index: usize, + ) -> tonic::Result)>> { + // Task-binding index `i` maps to publisher binding `i + 1` (binding 0 is + // the fixed ops-stats journal), mirroring `publish_doc`. + Ok(Some(self.0.marker_commit(binding_index + 1).await?)) + } + + async fn apply_truncated_at_labels( + &mut self, + active_backfills: &BTreeMap, + ) -> tonic::Result<()> { + let mapped: BTreeMap = active_backfills + .iter() + .map(|(&binding, &clock)| (binding as usize + 1, clock)) + .collect(); + self.0.apply_truncated_at_labels(&mapped).await + } + fn commit_intents(&mut self) -> Option<(uuid::Producer, uuid::Clock, Vec)> { Some(self.0.commit_intents()) } @@ -370,6 +410,20 @@ impl Publisher for NoopPublisher { Ok(()) } + async fn marker_commit( + &mut self, + _binding_index: usize, + ) -> tonic::Result)>> { + Ok(None) + } + + async fn apply_truncated_at_labels( + &mut self, + _active_backfills: &BTreeMap, + ) -> tonic::Result<()> { + Ok(()) + } + fn commit_intents(&mut self) -> Option<(uuid::Producer, uuid::Clock, Vec)> { None } diff --git a/crates/runtime-next/src/shard/capture/actor.rs b/crates/runtime-next/src/shard/capture/actor.rs index 02fd70b615c..bb533f97c48 100644 --- a/crates/runtime-next/src/shard/capture/actor.rs +++ b/crates/runtime-next/src/shard/capture/actor.rs @@ -12,6 +12,23 @@ use std::collections::BTreeMap; use std::time::Duration; use tokio::sync::mpsc; +type PersistFut = BoxFuture< + 'static, + anyhow::Result<( + (crate::shard::RocksDB, Vec), + Option, + )>, +>; +type StatsWriteFut

= BoxFuture< + 'static, + tonic::Result<( + P, + BTreeMap, + Option, + )>, +>; +type LabelsApplyFut

= BoxFuture<'static, (P, BTreeMap, tonic::Result<()>)>; + /// Shard-side capture transaction loop for one connector session. /// /// The actor drives the [`fsm::Head`] and [`fsm::Tail`] state machines: it polls @@ -34,6 +51,10 @@ pub(super) struct Actor { token_restart_at: Option, // Logger of task-centric state changes and events. logger: L, + // True only for shard zero (origin of the key and r-clock ranges). Gates + // backfill truncation: only shard zero emits backfill messages and manages + // truncated-at labels, since it alone sees each backfill's full lifecycle. + is_shard_zero: bool, // --- Parked resources: `Some` unless borrowed by an in-flight future. --- // RocksDB is parked with its per-binding state keys. @@ -52,15 +73,26 @@ pub(super) struct Actor { acknowledge_fut: Option>>, drain_fut: Option>>>, intents_write_fut: Option>>, - persist_fut: Option)>>>, + labels_apply_fut: Option>, + persist_fut: Option, split_fut: Option, - stats_write_fut: Option)>>>, + stats_write_fut: Option>, + + // `truncated_at` clock of each binding with an in-progress backfill — + // present from its BackfillBegin until its BackfillComplete commits. + active_backfills: BTreeMap, + // True when `active_backfills` differs from what's reflected in journal + // labels. + labels_dirty: bool, // --- Hand-offs staged between FSM steps. --- // Drain output, staged for `TailDrain`. drain_finished: Option, // ACK intents from a completed stats write, staged for `TailWriteStats`. pending_ack_intents: BTreeMap, + // Active-backfill change resolved by the stats write (needs the marker's + // commit clock), staged for `TailWriteStats` to fold into its Persist. + pending_active_backfill_change: Option, } /// Drain inputs staged by a Rotate, handed to [`drain::drain_and_publish`] @@ -72,9 +104,11 @@ struct DrainInput { impl Actor { pub fn new( + active_backfills: BTreeMap, binding_state_keys: Vec, connector_tx: mpsc::Sender, db: crate::shard::RocksDB, + is_shard_zero: bool, metrics: super::Metrics, logger: L, publisher: P, @@ -90,12 +124,14 @@ impl Actor { tokio::time::Instant::now() + delay }); + let labels_dirty = !active_backfills.is_empty(); Self { task, connector_tx, metrics, token_restart_at, logger, + is_shard_zero, db: Some((db, binding_state_keys)), publisher: Some(publisher), shapes: Some(shapes), @@ -104,11 +140,15 @@ impl Actor { acknowledge_fut: None, drain_fut: None, intents_write_fut: None, + labels_apply_fut: None, persist_fut: None, split_fut: None, stats_write_fut: None, + active_backfills, + labels_dirty, drain_finished: None, pending_ack_intents: BTreeMap::new(), + pending_active_backfill_change: None, } } @@ -168,12 +208,14 @@ impl Actor { self.acknowledge_fut.is_none(), &mut self.drain_finished, self.intents_write_fut.is_none(), + self.labels_apply_fut.is_none(), now, self.persist_fut.is_none(), &self.task, self.stats_write_fut .is_none() .then_some(&mut self.pending_ack_intents), + &mut self.pending_active_backfill_change, ); if prev_kind != tail.kind() { service_kit::event!( @@ -254,10 +296,11 @@ impl Actor { self.drain_fut = None; } Some(result) = maybe_fut(&mut self.stats_write_fut) => { - let (publisher, ack_intents) = result.map_err(crate::status_to_anyhow) + let (publisher, ack_intents, change) = result.map_err(crate::status_to_anyhow) .context("writing capture ops stats document")?; self.publisher = Some(publisher); self.pending_ack_intents = ack_intents; + self.pending_active_backfill_change = change; self.stats_write_fut = None; // WriteStats flushed this transaction's collection appends, so @@ -265,8 +308,23 @@ impl Actor { self.observe_throttle(); } Some(result) = maybe_fut(&mut self.persist_fut) => { - self.db = Some(result?); + let (db, change) = result?; + self.db = Some(db); self.persist_fut = None; + match change { + Some(proto::persist::ActiveBackfillChange::Begin(begin)) => { + self.active_backfills.insert(begin.binding, begin.truncated_at); + self.labels_dirty = true; + } + Some(proto::persist::ActiveBackfillChange::CompleteBinding(binding)) => { + self.active_backfills.remove(&binding); + // Re-apply remaining backfills' labels; if this was the + // last one, an empty map has nothing to apply, so don't + // strand `labels_dirty` at true. + self.labels_dirty = !self.active_backfills.is_empty(); + } + None => {} + } } Some(result) = maybe_fut(&mut self.acknowledge_fut) => { result?; @@ -287,6 +345,13 @@ impl Actor { ); self.split_fut = None; } + Some((publisher, active_backfills, result)) = maybe_fut(&mut self.labels_apply_fut) => { + result.context("applying truncated-at journal labels")?; + self.publisher = Some(publisher); + self.active_backfills = active_backfills; + self.labels_apply_fut = None; + self.labels_dirty = false; + } // Process controller messages next. msg = controller_rx.next() => { Self::on_controller_rx(msg, &mut close_requested, &mut stopping)?; @@ -444,8 +509,17 @@ impl Actor { true } - fsm::Action::WriteStats { stats } => { + fsm::Action::WriteStats { stats, backfill } => { let mut publisher = self.publisher.take().context("missing capture publisher")?; + // A BackfillComplete truncates to its matching begin's clock, + // recovered from the shard's active-backfill state; snapshot it + // before the future moves `publisher`. + let active_backfill_begin = match &backfill { + Some(fsm::BackfillMessage::BackfillComplete { binding }) => { + self.active_backfills.get(binding).copied() + } + _ => None, + }; self.stats_write_fut = Some( async move { if !stats.capture.is_empty() { @@ -453,14 +527,11 @@ impl Actor { } publisher.flush().await?; - let intents = match publisher.commit_intents() { - Some(commit) => { - publisher::intents::build_transaction_intents(&[commit]) - } - None => BTreeMap::new(), - }; + let (intents, change) = + build_write_intents(&mut publisher, backfill, active_backfill_begin) + .await?; - Ok((publisher, intents)) + Ok((publisher, intents, change)) } .boxed(), ); @@ -479,7 +550,7 @@ impl Actor { .persist(&persist, &binding_state_keys) .await .context("Persisting capture state")?; - Ok((db, binding_state_keys)) + Ok(((db, binding_state_keys), persist.active_backfill_change)) } .boxed(), ); @@ -515,6 +586,28 @@ impl Actor { true } + fsm::Action::ApplyTruncatedLabels => { + // Only shard zero manages truncated-at labels: a non-zero shard + // that inherited `active_backfills` (e.g. a mid-backfill split) + // can't clear them — BackfillComplete reaches only shard zero. + if !self.is_shard_zero || !self.labels_dirty || self.active_backfills.is_empty() { + false + } else { + let mut publisher = + self.publisher.take().context("missing capture publisher")?; + let active_backfills = std::mem::take(&mut self.active_backfills); + self.labels_apply_fut = Some( + async move { + let result = + publisher.apply_truncated_at_labels(&active_backfills).await; + (publisher, active_backfills, result) + } + .boxed(), + ); + true + } + } + fsm::Action::Error(error) => return Err(error), }; @@ -560,7 +653,7 @@ impl Actor { ) -> anyhow::Result<()> { let verify = crate::verify( "Capture", - "Captured, SourcedSchema, or Checkpoint", + "Captured, SourcedSchema, Checkpoint, BackfillBegin, or BackfillComplete", "connector", ); let Some(response) = msg else { @@ -588,6 +681,42 @@ impl Actor { "received Checkpoint from connector", ); fsm::ConnectorRx::Checkpoint(checkpoint) + } else if let Some(response::BackfillBegin { binding }) = response.backfill_begin { + if !self.is_shard_zero { + anyhow::bail!( + "connector emitted BackfillBegin for binding {binding}, but \ + only shard zero manages backfill truncation" + ); + } + if binding as usize >= self.task.bindings.len() { + anyhow::bail!("connector emitted BackfillBegin for out-of-range binding {binding}"); + } + service_kit::event!( + tracing::Level::INFO, + "connector", + binding, + "received BackfillBegin from connector", + ); + fsm::ConnectorRx::Backfill(fsm::BackfillMessage::BackfillBegin { binding }) + } else if let Some(response::BackfillComplete { binding }) = response.backfill_complete { + if !self.is_shard_zero { + anyhow::bail!( + "connector emitted BackfillComplete for binding {binding}, but \ + only shard zero manages backfill truncation" + ); + } + if binding as usize >= self.task.bindings.len() { + anyhow::bail!( + "connector emitted BackfillComplete for out-of-range binding {binding}" + ); + } + service_kit::event!( + tracing::Level::INFO, + "connector", + binding, + "received BackfillComplete from connector", + ); + fsm::ConnectorRx::Backfill(fsm::BackfillMessage::BackfillComplete { binding }) } else { return Err(verify.fail_msg(response)); }; @@ -595,6 +724,91 @@ impl Actor { } } +/// Snapshot this transaction's ACK intents, plus its resolved +/// [`proto::persist::ActiveBackfillChange`] for a marker transaction. An ordinary +/// transaction (`backfill` is `None`) ACKs only journals it appended; a marker +/// broadcasts across *every* partition via [`crate::Publisher::marker_commit`]. +async fn build_write_intents( + publisher: &mut impl crate::Publisher, + backfill: Option, + active_backfill_begin: Option, +) -> tonic::Result<( + BTreeMap, + Option, +)> { + match backfill { + None => { + let intents = match publisher.commit_intents() { + Some(commit) => publisher::intents::build_transaction_intents(&[commit], None), + None => BTreeMap::new(), + }; + Ok((intents, None)) + } + Some(fsm::BackfillMessage::BackfillBegin { binding }) => { + let Some((producer, clock, journals)) = + publisher.marker_commit(binding as usize).await? + else { + // Preview only: no journal IO, so no broadcast clock. Stage the + // Begin with a zero (inert) boundary so preview state transitions + // like a real run — a `truncated_at` of 0 suppresses nothing, + // since real document clocks are always > 0. + return Ok(( + BTreeMap::new(), + Some(proto::persist::ActiveBackfillChange::Begin( + proto::ActiveBackfillBegin { + binding, + truncated_at: 0, + }, + )), + )); + }; + // The marker's single broadcast clock is the authoritative boundary. + let truncated_at = clock.as_u64(); + let intents = publisher::intents::build_transaction_intents( + &[(producer, clock, journals)], + Some(&publisher::intents::BackfillMarker::Begin), + ); + Ok(( + intents, + Some(proto::persist::ActiveBackfillChange::Begin( + proto::ActiveBackfillBegin { + binding, + truncated_at, + }, + )), + )) + } + Some(fsm::BackfillMessage::BackfillComplete { binding }) => { + let Some(truncated_at) = active_backfill_begin else { + // Orphaned complete (no active backfill, e.g. a begin was never + // observed): publish nothing, change nothing. Unexpected — a + // connector shouldn't complete a backfill it never began — so + // surface it rather than swallowing it silently. + service_kit::event!( + tracing::Level::WARN, + "connector", + binding, + "ignoring a BackfillComplete with no matching active backfill (orphaned complete)", + ); + return Ok((BTreeMap::new(), None)); + }; + let change = Some(proto::persist::ActiveBackfillChange::CompleteBinding( + binding, + )); + let Some((producer, clock, journals)) = + publisher.marker_commit(binding as usize).await? + else { + return Ok((BTreeMap::new(), change)); + }; + let intents = publisher::intents::build_transaction_intents( + &[(producer, clock, journals)], + Some(&publisher::intents::BackfillMarker::Complete { truncated_at }), + ); + Ok((intents, change)) + } + } +} + /// Parse and validate a connector `SourcedSchema` into its target binding /// index and inferred write-shape. All schema parsing and error checking lives /// here so the HeadFSM's per-binding shape fold stays infallible. @@ -740,9 +954,11 @@ mod tests { let shapes = task.binding_shapes_by_index(Default::default()); let actor = Actor::new( + BTreeMap::new(), vec!["stateA".to_string(), "stateB".to_string()], connector_tx, crate::shard::RocksDB::open(None).await.unwrap(), + true, super::super::Metrics::new("test/shard"), crate::TracingLogger, crate::publish::NoopPublisher, @@ -826,9 +1042,11 @@ mod tests { let shapes = task.binding_shapes_by_index(Default::default()); let mut actor = Actor::new( + BTreeMap::new(), vec!["stateA".to_string()], connector_tx, crate::shard::RocksDB::open(None).await.unwrap(), + true, super::super::Metrics::new("test/shard"), crate::TracingLogger, publisher, @@ -888,6 +1106,241 @@ mod tests { assert!(actor.split_fut.is_none()); } + #[tokio::test] + async fn backfill_message_rejects_out_of_range_binding() { + // `mk_task(true)` has two bindings (indices 0 and 1); index 2 is out of + // range. An out-of-range binding from the connector must surface as a + // clean error rather than panicking downstream in publisher indexing. + let (connector_tx, _conn_rx) = mpsc::channel::(crate::CHANNEL_BUFFER); + let db = crate::shard::RocksDB::open(None).await.unwrap(); + let task = std::sync::Arc::new(mk_task(true)); + let publisher = crate::publish::NoopPublisher; + let shapes = task.binding_shapes_by_index(Default::default()); + + let actor = Actor::new( + BTreeMap::new(), + vec!["stateA".to_string(), "stateB".to_string()], + connector_tx, + db, + true, // is_shard_zero + super::super::Metrics::new("test/shard"), + crate::TracingLogger, + publisher, + shapes, + task, + None, // token_restart_at + ); + + let mut ready = fsm::ConnectorRx::Eof; + for response in [ + Response { + backfill_begin: Some(response::BackfillBegin { binding: 2 }), + ..Default::default() + }, + Response { + backfill_complete: Some(response::BackfillComplete { binding: 2 }), + ..Default::default() + }, + ] { + let err = actor + .on_connector_rx(&mut ready, Some(Ok(response))) + .unwrap_err(); + assert!( + err.to_string().contains("out-of-range binding 2"), + "unexpected error: {err}", + ); + } + } + + /// Backfill lifecycle across a restart: a Begin persists the active backfill, a + /// fresh session recovers it, a Complete removes it, and an orphaned Complete + /// (never-begun binding) is a no-op. `truncated_at` is 0 — the preview + /// publisher's no-op marker clock. Each backfill message is sealed by its own + /// terminating Checkpoint. + #[tokio::test] + async fn serve_backfill_lifecycle() { + // One capture session over `db`: feed `responses`, drain `expect_acks` + // Acknowledges (one per committed marker transaction, so each commits + // before Stop), then Stop and return the db. + async fn run_capture_session( + db: crate::shard::RocksDB, + active_backfills: BTreeMap, + responses: Vec>, + expect_acks: usize, + ) -> crate::shard::RocksDB { + let (connector_tx, mut actor_to_conn_rx) = + mpsc::channel::(crate::CHANNEL_BUFFER); + let (conn_resp_tx, conn_resp_rx) = + mpsc::channel::>(crate::CHANNEL_BUFFER); + let (controller_tx, controller_rx) = + mpsc::unbounded_channel::>(); + + let task = std::sync::Arc::new(mk_task(true)); + let publisher = crate::publish::NoopPublisher; + let shapes = task.binding_shapes_by_index(Default::default()); + + let actor = Actor::new( + active_backfills, + vec!["stateA".to_string(), "stateB".to_string()], + connector_tx, + db, + true, + super::super::Metrics::new("test/shard"), + crate::TracingLogger, + publisher, + shapes, + task, + None, // token_restart_at + ); + + let serve = tokio::spawn(async move { + let mut controller_rx = UnboundedReceiverStream::new(controller_rx); + actor + .serve( + ReceiverStream::new(conn_resp_rx), + &mut controller_rx, + fsm::Head::Idle(fsm::HeadIdle::default()), + fsm::Tail::Recover(fsm::TailRecover { + checkpoints: 0, + ack_intents: BTreeMap::new(), + }), + ) + .await + }); + + for response in responses { + conn_resp_tx.send(response).await.unwrap(); + } + for _ in 0..expect_acks { + assert!(actor_to_conn_rx.recv().await.unwrap().acknowledge.is_some()); + } + + controller_tx + .send(Ok(proto::Capture { + stop: Some(proto::Stop {}), + ..Default::default() + })) + .unwrap(); + let (db, _shapes) = serve.await.unwrap().unwrap(); + db + } + + let state_keys = || vec!["stateA".to_string(), "stateB".to_string()]; + + let db = run_capture_session( + crate::shard::RocksDB::open(None).await.unwrap(), + BTreeMap::new(), + vec![ + Ok(Response { + backfill_begin: Some(response::BackfillBegin { binding: 0 }), + ..Default::default() + }), + checkpoint(br#"{"cursor":"1"}"#), + ], + 1, + ) + .await; + let (db, recover) = db.scan(state_keys()).await.unwrap(); + assert_eq!( + recover.active_backfills, + BTreeMap::from([(0u32, 0u64)]), + "begin persisted the active backfill", + ); + + let db = run_capture_session( + db, + recover.active_backfills, + vec![ + Ok(Response { + backfill_complete: Some(response::BackfillComplete { binding: 0 }), + ..Default::default() + }), + checkpoint(br#"{"cursor":"2"}"#), + Ok(Response { + backfill_complete: Some(response::BackfillComplete { binding: 1 }), + ..Default::default() + }), + checkpoint(br#"{"cursor":"3"}"#), + ], + 2, + ) + .await; + let (_db, recover) = db.scan(state_keys()).await.unwrap(); + assert_eq!( + recover.active_backfills, + BTreeMap::new(), + "complete removed binding 0; orphaned complete for binding 1 was a no-op", + ); + } + + /// A shard recovered mid-backfill (non-empty `active_backfills`) re-applies its + /// truncated-at labels on the first `ApplyTruncatedLabels` rather than skipping + /// — the restart case a false `labels_dirty` seed would silently break. + #[tokio::test] + async fn recovered_active_backfills_reapply_labels() { + let (connector_tx, _connector_rx) = mpsc::channel::(crate::CHANNEL_BUFFER); + let task = std::sync::Arc::new(mk_task(true)); + let publisher = crate::publish::NoopPublisher; + let shapes = task.binding_shapes_by_index(Default::default()); + + let mut actor = Actor::new( + BTreeMap::from([(0u32, 5u64)]), // recovered mid-backfill + vec!["stateA".to_string(), "stateB".to_string()], + connector_tx, + crate::shard::RocksDB::open(None).await.unwrap(), + true, + super::super::Metrics::new("test/shard"), + crate::TracingLogger, + publisher, + shapes, + task.clone(), + None, // token_restart_at + ); + + let mut accumulator = crate::Accumulator::new(task.combine_spec().unwrap()).unwrap(); + actor + .dispatch(fsm::Action::ApplyTruncatedLabels, &mut accumulator) + .unwrap(); + assert!( + actor.labels_apply_fut.is_some(), + "recovered active backfills must re-apply labels, not skip", + ); + } + + /// A non-shard-zero shard that inherited `active_backfills` via a mid-backfill + /// split must NOT apply truncated-at labels: it never sees the BackfillComplete + /// that would clear them. + #[tokio::test] + async fn non_shard_zero_skips_label_apply() { + let (connector_tx, _connector_rx) = mpsc::channel::(crate::CHANNEL_BUFFER); + let task = std::sync::Arc::new(mk_task(true)); + let publisher = crate::publish::NoopPublisher; + let shapes = task.binding_shapes_by_index(Default::default()); + + let mut actor = Actor::new( + BTreeMap::from([(0u32, 5u64)]), // inherited mid-backfill via a split + vec!["stateA".to_string(), "stateB".to_string()], + connector_tx, + crate::shard::RocksDB::open(None).await.unwrap(), + false, // not shard zero + super::super::Metrics::new("test/shard"), + crate::TracingLogger, + publisher, + shapes, + task.clone(), + None, // token_restart_at + ); + + let mut accumulator = crate::Accumulator::new(task.combine_spec().unwrap()).unwrap(); + actor + .dispatch(fsm::Action::ApplyTruncatedLabels, &mut accumulator) + .unwrap(); + assert!( + actor.labels_apply_fut.is_none(), + "a non-shard-zero shard must not apply truncated-at labels", + ); + } + /// `parse_sourced_schema` resolves a valid closed schema to its binding and /// inferred shape, and rejects an out-of-range binding index. #[test] diff --git a/crates/runtime-next/src/shard/capture/handler.rs b/crates/runtime-next/src/shard/capture/handler.rs index 9ec040caab9..1e118842837 100644 --- a/crates/runtime-next/src/shard/capture/handler.rs +++ b/crates/runtime-next/src/shard/capture/handler.rs @@ -297,6 +297,7 @@ where db = db.seed_connector_state(&mut recover).await?; let proto::Recover { ack_intents, + active_backfills, mut connector_state_json, last_applied, .. @@ -389,10 +390,16 @@ where // binding layout, and stow the session's final shapes back when it ends. let shapes = task.binding_shapes_by_index(std::mem::take(shapes_by_key)); + // Only shard zero drives backfill truncation: it owns the origin of the key + // and r-clock ranges, so it sees each backfill's full lifecycle even when split. + let is_shard_zero = range.key_begin == 0 && range.r_clock_begin == 0; + let (db, shapes) = super::actor::Actor::new( + active_backfills, binding_state_keys, connector_tx, db, + is_shard_zero, metrics, logger, publisher, diff --git a/crates/runtime-next/src/shard/materialize/actor.rs b/crates/runtime-next/src/shard/materialize/actor.rs index 76863d88572..3c7028f3525 100644 --- a/crates/runtime-next/src/shard/materialize/actor.rs +++ b/crates/runtime-next/src/shard/materialize/actor.rs @@ -1,4 +1,4 @@ -use super::{Binding, LoadKeys, drain, scan}; +use super::{Binding, LoadKeys, boundaries::Boundaries, drain, scan}; use crate::{patches, proto}; use anyhow::Context; use bytes::Bytes; @@ -24,6 +24,10 @@ pub(super) enum Phase { pub(super) struct Actor { // Task binding specifications. bindings: Vec, + // Per-binding backfill-truncation boundaries. Both the scanner and + // asynchronous Loaded handling classify ingress documents against them, + // and an advancing boundary truncates the accumulator at L:Load receipt. + boundaries: Boundaries, // FIFO of outbound connector requests, drained head-first into // `connector_tx` as channel capacity permits. connector_pending: Vec, @@ -84,8 +88,10 @@ impl Actor { tokio::time::Instant::now() + delay }); + let l = bindings.len(); Self { bindings, + boundaries: Boundaries::new(l), connector_pending: Vec::new(), connector_tx, db: Some((db, binding_state_keys)), @@ -153,6 +159,7 @@ impl Actor { } else if let Phase::Scanning(mut scanner) = phase { if scanner.step( &self.bindings, + &self.boundaries, &mut self.load_keys, &mut self.max_keys, self.disable_load_optimization, @@ -361,7 +368,7 @@ impl Actor { shuffle::Frontier::decode(proto).context("invalid Frontier on L:Load")?; let Phase::Idle { - accumulator, + mut accumulator, shuffle_reader, shuffle_remainders, } = phase @@ -369,16 +376,33 @@ impl Actor { anyhow::bail!("L:Load received while actor is not idle"); }; + // When a boundary first advances, truncate the accumulator to + // reclassify everything it already holds (MemTable purge/flag plus a + // spilled-segment fence). Adds after this point self-classify: the + // scan drops pre-boundary sources, Loaded rows split by UUID clock. + for (binding, clock) in &frontier.latest_backfill_begin { + if self.boundaries.observe_begin(*binding as usize, *clock) { + accumulator.truncate(*binding as usize); + } + } + let scanner = scan::Scanner::new(accumulator, frontier, shuffle_reader, shuffle_remainders)?; return Ok((Phase::Scanning(scanner), false)); } else if let Some(proto::materialize::Flush { connector_patches_json, + backfill_begins, + backfill_completes, }) = msg.flush { + // Forward the markers; the connector self-selects whether to act, + // per its key range. self.connector_pending.push(materialize::Request { flush: Some(materialize::request::Flush { state_patches_json: connector_patches_json, + backfill_begins: Self::project_backfill_begins(backfill_begins), + backfill_completes: Self::project_backfill_completes(backfill_completes), + ..Default::default() }), ..Default::default() }); @@ -471,6 +495,36 @@ impl Actor { Ok((phase, false)) } + /// Project the leader's forwarded begin markers into connector-facing + /// notifications, converting each clock — the truncation boundary — to a + /// wall-clock Timestamp. + fn project_backfill_begins( + events: Vec, + ) -> Vec { + events + .into_iter() + .map(|e| materialize::request::flush::BackfillBegin { + binding: e.binding, + timestamp: proto_gazette::uuid::Clock::from_u64(e.clock).to_pb_json_timestamp(), + }) + .collect() + } + + /// Project the leader's forwarded complete markers into connector-facing + /// notifications. See [`Self::project_backfill_begins`]; the clock is the + /// completed backfill's begin (truncation) boundary. + fn project_backfill_completes( + events: Vec, + ) -> Vec { + events + .into_iter() + .map(|e| materialize::request::flush::BackfillComplete { + binding: e.binding, + timestamp: proto_gazette::uuid::Clock::from_u64(e.clock).to_pb_json_timestamp(), + }) + .collect() + } + fn on_connector_response( &mut self, phase: &mut Phase, @@ -521,7 +575,37 @@ impl Actor { accumulator.parse_json_doc(&doc_json).with_context(|| { format!("parsing loaded doc for {}", binding_spec.collection_name) })?; - memtable.add(binding_index as u16, doc, true)?; + + // Classify by the row's embedded document-UUID clock, not message + // timing: staleness reflects when the row was last STORED, and a row + // can load stale many transactions after its binding was truncated. + // A never-truncated binding needs no classification (nor a UUID). + let stale = if !self.boundaries.has_boundary(binding_index) { + false + } else if let Some(doc::HeapNode::String(uuid)) = + binding_spec.document_uuid_ptr.query(&doc) + { + let (_, clock, _) = proto_gazette::uuid::parse_str(uuid).with_context(|| { + format!( + "loaded doc for {} has an unparseable document UUID {uuid:?}", + binding_spec.collection_name, + ) + })?; + self.boundaries.is_stale(binding_index, clock) + } else { + anyhow::bail!( + "loaded doc for {} is being backfill-truncated but has no document UUID \ + at {}; the materialization must store the root document \ + (flow_document) or reconstruct its UUID", + binding_spec.collection_name, + binding_spec.document_uuid_ptr, + ); + }; + if stale { + memtable.add_stale_front(binding_index as u16, doc)?; + } else { + memtable.add(binding_index as u16, doc, true)?; + } } else if let Some(materialize::response::Flushed { state }) = resp.flushed { let bindings = std::mem::take(&mut self.flushed).into_values().collect(); _ = self.leader_tx.send(proto::Materialize { @@ -623,6 +707,7 @@ mod tests { ( Actor { bindings: Vec::new(), + boundaries: Boundaries::new(0), connector_pending: Vec::new(), connector_tx, db: None, @@ -721,6 +806,7 @@ mod tests { let actor = Actor { bindings: Vec::new(), + boundaries: Boundaries::new(0), connector_pending: Vec::new(), connector_tx: actor_to_conn_tx, db: Some((db, Vec::new())), @@ -792,6 +878,7 @@ mod tests { .send(Ok(proto::Materialize { flush: Some(proto::materialize::Flush { connector_patches_json: Bytes::from_static(br#"[{"f":1}]"#), + ..Default::default() }), ..Default::default() })) @@ -914,4 +1001,276 @@ mod tests { let (_db, recover) = db.scan(Vec::<&str>::new()).await.unwrap(); assert_eq!(recover.last_applied.as_ref(), b"persisted-spec-bytes"); } + + // A full-reduction binding storing the root document, keyed on /key, whose + // `v` array reduces by append. `document_uuid_ptr` lets the shard read each + // loaded row's UUID to classify it against the backfill boundary. + fn backfill_binding() -> Binding { + Binding { + collection_name: "test/collection".to_string(), + delta_updates: false, + document_uuid_ptr: json::Pointer::from("/_meta/uuid"), + key_extractors: vec![doc::Extractor::with_default( + "/key", + &doc::SerPolicy::noop(), + serde_json::json!(""), + )], + read_schema_json: bytes::Bytes::from_static( + br#"{ + "type": "object", + "properties": { + "key": { "type": "string" }, + "v": { "type": "array", "reduce": { "strategy": "append" } } + }, + "reduce": { "strategy": "merge" } + }"#, + ), + ser_policy: doc::SerPolicy::noop(), + state_key: "test/collection".to_string(), + store_document: true, + value_plan: doc::ExtractorPlan::new(&[]), + } + } + + // Build an L:Load message whose Frontier carries `truncated_at` as binding + // 0's backfill begin. + fn backfill_load(truncated_at: proto_gazette::uuid::Clock) -> proto::Materialize { + let mut frontier = shuffle::Frontier::new(Vec::new(), vec![0u64]).unwrap(); + frontier.latest_backfill_begin.insert(0, truncated_at); + proto::Materialize { + load: Some(proto::materialize::Load { + frontier: Some(frontier.encode()), + }), + ..Default::default() + } + } + + #[tokio::test] + async fn backfill_load_classifies_loaded_docs_through_drain() { + let producer = proto_gazette::uuid::Producer::from_bytes([0x01, 0, 0, 0, 0, 0]); + let flags = proto_gazette::uuid::Flags(0); + let mk_uuid = |clock| proto_gazette::uuid::build(producer, clock, flags).to_string(); + // The boundary, plus a row clock below it (stale) and above it (fresh). + let truncated_at = proto_gazette::uuid::Clock::from_unix(1_700_000_000, 0); + let stale = mk_uuid(proto_gazette::uuid::Clock::from_unix(1_699_999_999, 0)); + let fresh = mk_uuid(proto_gazette::uuid::Clock::from_unix(1_700_000_001, 0)); + + let (mut actor, _leader_rx, _connector_rx) = make_actor(); + actor.bindings = vec![backfill_binding()]; + actor.boundaries = Boundaries::new(1); + + // Seed the accumulator with a pre-boundary source document BEFORE the + // truncating L:Load. The truncate() at L:Load receipt must purge it (a + // pre-boundary source carries no existence), so it never drains. + let mut accumulator = crate::Accumulator::new( + super::super::task::combine_spec(&[backfill_binding()]).unwrap(), + ) + .unwrap(); + { + let mt = accumulator.memtable().unwrap(); + let node = doc::HeapNode::from_node( + &serde_json::json!({"key": "purged", "v": ["old"]}), + mt.alloc(), + ); + mt.add(0, node, false).unwrap(); + } + let shuffle_dir = tempfile::tempdir().unwrap(); + let shuffle_reader = shuffle::log::Reader::new(shuffle_dir.path(), 0); + let idle = Phase::Idle { + accumulator, + shuffle_reader, + shuffle_remainders: VecDeque::new(), + }; + + // L:Load establishes binding 0's boundary and truncates the accumulator. + let (mut phase, _stop) = actor + .on_leader_message(idle, Some(Ok(backfill_load(truncated_at)))) + .unwrap(); + + // The boundary is applied: clocks below it are stale, at/above are fresh. + assert!(actor.boundaries.has_boundary(0)); + assert!( + actor + .boundaries + .is_stale(0, proto_gazette::uuid::Clock::from_unix(1_699_999_999, 0)) + ); + assert!(!actor.boundaries.is_stale(0, truncated_at)); + assert!( + matches!(phase, Phase::Scanning(_)), + "L:Load enters the scan" + ); + + // C:Loaded rows, split by their document UUID as the actor receives them. + let loaded = |key: &str, v: &str, uuid: &str| materialize::Response { + loaded: Some(materialize::response::Loaded { + binding: 0, + doc_json: Bytes::from( + serde_json::to_vec(&serde_json::json!({ + "key": key, "v": [v], "_meta": {"uuid": uuid}, + })) + .unwrap(), + ), + }), + ..Default::default() + }; + for resp in [ + loaded("straddle", "stale", &stale), // below the boundary → stale front + loaded("normal", "loaded", &fresh), // at/above → fresh front + ] { + actor + .on_connector_response(&mut phase, Some(Ok(resp))) + .unwrap(); + } + + // Inject the post-boundary source documents the scan would surface, + // pairing one against each loaded row (added fresh, as the scan does). + let Phase::Scanning(mut scanner) = phase else { + panic!("expected Scanning phase after L:Load"); + }; + { + let memtable = scanner.accumulator().memtable().unwrap(); + for (key, v) in [("straddle", "fresh"), ("normal", "src")] { + let doc = serde_json::json!({"key": key, "v": [v]}); + let node = doc::HeapNode::from_node(&doc, memtable.alloc()); + memtable.add(0, node, false).unwrap(); + } + } + + // Drain and collect each stored (key, v, exists). + let (accumulator, shuffle_reader, shuffle_remainders, _active) = scanner.into_parts(); + let mut drainer = + drain::Drainer::new(accumulator, shuffle_reader, shuffle_remainders).unwrap(); + + let mut stores = Vec::new(); + while let Some(req) = drainer + .step(&actor.bindings, connector_init::Codec::Json) + .unwrap() + { + let store = req.store.expect("drained request is a Store"); + let doc: serde_json::Value = serde_json::from_slice(&store.doc_json).unwrap(); + stores.push(( + doc.get("key").and_then(|k| k.as_str()).unwrap().to_string(), + doc.get("v").cloned().unwrap(), + store.exists, + )); + } + + // Drained in key order. "purged" (pre-boundary source) was dropped by + // truncate() and never appears. "straddle"'s loaded row is stale: its + // ["stale"] is dropped (not reduced), only the fresh source ["fresh"] + // stores with existence transferred. "normal" (at/above the boundary) + // loads normally, reducing its loaded value forward. + assert_eq!( + stores, + vec![ + ( + "normal".to_string(), + serde_json::json!(["loaded", "src"]), + true + ), + ("straddle".to_string(), serde_json::json!(["fresh"]), true), + ], + ); + } + + #[tokio::test] + async fn repeated_begin_does_not_retruncate() { + // A begin that doesn't advance the boundary must not truncate again, or + // it would purge freshly-accumulated post-boundary sources. + let (mut actor, _leader_rx, _connector_rx) = make_actor(); + actor.bindings = vec![backfill_binding()]; + actor.boundaries = Boundaries::new(1); + let truncated_at = proto_gazette::uuid::Clock::from_unix(1_700_000_000, 0); + + let accumulator = crate::Accumulator::new( + super::super::task::combine_spec(&[backfill_binding()]).unwrap(), + ) + .unwrap(); + let shuffle_dir = tempfile::tempdir().unwrap(); + let idle = Phase::Idle { + accumulator, + shuffle_reader: shuffle::log::Reader::new(shuffle_dir.path(), 0), + shuffle_remainders: VecDeque::new(), + }; + + // First L:Load establishes the boundary (an advance → truncate). + let (phase, _stop) = actor + .on_leader_message(idle, Some(Ok(backfill_load(truncated_at)))) + .unwrap(); + let Phase::Scanning(mut scanner) = phase else { + panic!("expected Scanning after first L:Load"); + }; + + // Accumulate a fresh post-boundary source, then return to Idle. + { + let mt = scanner.accumulator().memtable().unwrap(); + let node = doc::HeapNode::from_node( + &serde_json::json!({"key": "keep", "v": ["v"]}), + mt.alloc(), + ); + mt.add(0, node, false).unwrap(); + } + let (accumulator, shuffle_reader, shuffle_remainders, _active) = scanner.into_parts(); + let idle = Phase::Idle { + accumulator, + shuffle_reader, + shuffle_remainders, + }; + + // A second L:Load with the same begin does not advance, so it must not + // truncate — "keep" survives to drain. + let (phase, _stop) = actor + .on_leader_message(idle, Some(Ok(backfill_load(truncated_at)))) + .unwrap(); + let Phase::Scanning(scanner) = phase else { + panic!("expected Scanning after second L:Load"); + }; + let (accumulator, shuffle_reader, shuffle_remainders, _active) = scanner.into_parts(); + let mut drainer = + drain::Drainer::new(accumulator, shuffle_reader, shuffle_remainders).unwrap(); + + let mut keys = Vec::new(); + while let Some(req) = drainer + .step(&actor.bindings, connector_init::Codec::Json) + .unwrap() + { + let doc: serde_json::Value = + serde_json::from_slice(&req.store.unwrap().doc_json).unwrap(); + keys.push(doc["key"].as_str().unwrap().to_string()); + } + assert_eq!( + keys, + vec!["keep".to_string()], + "no re-truncate purged 'keep'" + ); + } + + #[tokio::test] + async fn loaded_doc_with_corrupt_uuid_errors() { + let (mut actor, _leader_rx, _connector_rx) = make_actor(); + actor.bindings = vec![backfill_binding()]; + actor.boundaries = Boundaries::new(1); + // The binding is truncating, so a loaded row's clock is required. + actor + .boundaries + .observe_begin(0, proto_gazette::uuid::Clock::from_u64(10)); + + // /_meta/uuid is present and a string, but not a valid v1 UUID. + let doc = serde_json::json!({"key": "k", "v": ["x"], "_meta": {"uuid": "not-a-uuid"}}); + let mut phase = make_idle_phase(); + let result = actor.on_connector_response( + &mut phase, + Some(Ok(materialize::Response { + loaded: Some(materialize::response::Loaded { + binding: 0, + doc_json: Bytes::from(serde_json::to_vec(&doc).unwrap()), + }), + ..Default::default() + })), + ); + assert!( + result.is_err(), + "a corrupt document UUID fails a truncating binding's transaction" + ); + } } diff --git a/crates/runtime-next/src/shard/materialize/boundaries.rs b/crates/runtime-next/src/shard/materialize/boundaries.rs new file mode 100644 index 00000000000..bf9ae445f92 --- /dev/null +++ b/crates/runtime-next/src/shard/materialize/boundaries.rs @@ -0,0 +1,95 @@ +//! Per-binding backfill-truncation boundaries: each binding tracks its latest +//! observed truncation boundary clock. +//! +//! Boundaries classify ingress rather than tagging combiner entries: the scan +//! drops source documents below the boundary, Loaded rows split into fresh vs. +//! stale existence-only fronts, and a boundary advance triggers +//! `Accumulator::truncate` to reclassify what's already accumulated. + +use proto_gazette::uuid; + +#[derive(Debug)] +pub(super) struct Boundaries(Vec>); + +impl Boundaries { + pub(super) fn new(n_bindings: usize) -> Self { + Self(vec![None; n_bindings]) + } + + /// Observe `binding`'s backfill boundary, returning whether it genuinely + /// advanced. Begins are re-delivered on every Load and only ever move + /// forward, so a duplicate or older begin returns `false` and is a no-op. + pub(super) fn observe_begin(&mut self, binding: usize, begin: uuid::Clock) -> bool { + let latest = &mut self.0[binding]; + if latest.is_some_and(|l| begin <= l) { + return false; + } + *latest = Some(begin); + true + } + + /// Whether `binding` has observed a truncation boundary. Without one no + /// Loaded row needs classification, so a Loaded row's UUID clock is consulted + /// only once its binding is truncating. + pub(super) fn has_boundary(&self, binding: usize) -> bool { + self.0[binding].is_some() + } + + /// Whether `clock` is stale for `binding`: true iff a boundary exists and + /// `clock` is below it. Without a boundary, nothing is stale. + pub(super) fn is_stale(&self, binding: usize, clock: uuid::Clock) -> bool { + matches!(self.0[binding], Some(boundary) if clock < boundary) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn clock(v: u64) -> uuid::Clock { + uuid::Clock::from_u64(v) + } + + #[test] + fn no_boundary_is_never_stale() { + let b = Boundaries::new(1); + assert!(!b.has_boundary(0)); + for c in [1, 100, u64::MAX] { + assert!(!b.is_stale(0, clock(c))); + } + } + + #[test] + fn boundary_classifies_by_clock() { + let mut b = Boundaries::new(1); + assert!(b.observe_begin(0, clock(500))); // advanced + assert!(b.has_boundary(0)); + assert!(b.is_stale(0, clock(499))); // below → stale + assert!(!b.is_stale(0, clock(500))); // at boundary → fresh + assert!(!b.is_stale(0, clock(999))); + + // Advancing the boundary reclassifies the prior boundary clock as stale. + assert!(b.observe_begin(0, clock(800))); + assert!(b.is_stale(0, clock(500))); + assert!(b.is_stale(0, clock(799))); + assert!(!b.is_stale(0, clock(800))); + } + + #[test] + fn repeated_or_older_begin_does_not_advance() { + let mut b = Boundaries::new(1); + assert!(b.observe_begin(0, clock(500))); + assert!(!b.observe_begin(0, clock(500))); // duplicate + assert!(!b.observe_begin(0, clock(300))); // older + assert!(!b.is_stale(0, clock(500))); + } + + #[test] + fn bindings_advance_independently() { + let mut b = Boundaries::new(2); + assert!(b.observe_begin(0, clock(500))); + assert!(b.is_stale(0, clock(499))); + assert!(!b.has_boundary(1)); + assert!(!b.is_stale(1, clock(1))); // binding 1 has no boundary + } +} diff --git a/crates/runtime-next/src/shard/materialize/mod.rs b/crates/runtime-next/src/shard/materialize/mod.rs index c52371becc6..a2a6ad0b559 100644 --- a/crates/runtime-next/src/shard/materialize/mod.rs +++ b/crates/runtime-next/src/shard/materialize/mod.rs @@ -1,4 +1,5 @@ mod actor; +mod boundaries; mod connector; mod drain; mod handler; @@ -83,6 +84,7 @@ impl Metrics { struct Binding { collection_name: String, // Source collection. delta_updates: bool, // Delta updates, or standard? + document_uuid_ptr: json::Pointer, // Document UUID pointer (often /_meta/uuid). key_extractors: Vec, // Key extractors for this collection. read_schema_json: bytes::Bytes, // Read JSON-Schema of collection documents. ser_policy: doc::SerPolicy, // Serialization policy for this source. diff --git a/crates/runtime-next/src/shard/materialize/scan.rs b/crates/runtime-next/src/shard/materialize/scan.rs index 0f056700f1c..04e472a4b48 100644 --- a/crates/runtime-next/src/shard/materialize/scan.rs +++ b/crates/runtime-next/src/shard/materialize/scan.rs @@ -1,8 +1,9 @@ -use super::{Binding, LoadKeys}; +use super::{Binding, LoadKeys, boundaries::Boundaries}; use anyhow::Context; use bytes::Buf; use bytes::{BufMut, Bytes}; use proto_flow::materialize; +use proto_gazette::uuid; use std::collections::{HashMap, VecDeque}; use crate::proto::materialize::loaded::Binding as LoadedBinding; @@ -49,6 +50,7 @@ impl Scanner { pub fn step( &mut self, bindings: &[Binding], + boundaries: &Boundaries, load_keys: &mut LoadKeys, max_keys: &mut [(Bytes, Bytes)], disable_load_optimization: bool, @@ -84,6 +86,30 @@ impl Scanner { .get(meta.binding.to_native() as usize) .context("scan entry has invalid meta.binding")?; + let active = self.active.entry(binding_index).or_default(); + + // Accumulate metrics for active bindings of the scan. A stale doc + // still counts here (it was read) before the drop below. + let clock = meta.clock.to_native(); + if active.sourced_docs_total == 0 { + active.index = binding_index; + active.max_source_clock = clock; + active.min_source_clock = clock; + } else { + active.max_source_clock = active.max_source_clock.max(clock); + active.min_source_clock = active.min_source_clock.min(clock); + } + active.sourced_docs_total += 1; + active.sourced_bytes_total += doc.source_byte_length.to_native() as u64; + + // Drop source documents below the binding's truncation boundary: + // superseded, never stored, so they must not add to the combiner, + // ratchet `next_max`, or emit a Load. Correct only for + // post-observation arrivals; earlier ones were handled by truncate(). + if boundaries.is_stale(binding_index as usize, uuid::Clock::from_u64(clock)) { + continue; + } + memtable .add_embedded( meta.binding.to_native(), @@ -125,21 +151,6 @@ impl Scanner { let gt_prev_max = key_packed > *prev_max; let gt_next_max = gt_prev_max && key_packed > *next_max; - let active = self.active.entry(binding_index).or_default(); - - // Accumulate metrics for active bindings of the scan. - let clock = meta.clock.to_native(); - if active.sourced_docs_total == 0 { - active.index = binding_index; - active.max_source_clock = clock; - active.min_source_clock = clock; - } else { - active.max_source_clock = active.max_source_clock.max(clock); - active.min_source_clock = active.min_source_clock.min(clock); - } - active.sourced_docs_total += 1; - active.sourced_bytes_total += doc.source_byte_length.to_native() as u64; - // Is `key_packed` larger than the largest key previously stored // to the connector? If so, then it cannot possibly exist. // We still track the max key even when the optimization is disabled. diff --git a/crates/runtime-next/src/shard/materialize/task.rs b/crates/runtime-next/src/shard/materialize/task.rs index 6ebd43d73dd..6d15c9971da 100644 --- a/crates/runtime-next/src/shard/materialize/task.rs +++ b/crates/runtime-next/src/shard/materialize/task.rs @@ -119,7 +119,7 @@ fn build_binding( partition_template: _, projections, read_schema_json, - uuid_ptr: _, + uuid_ptr, write_schema_json, } = collection.as_ref().context("missing collection")?; @@ -166,6 +166,7 @@ fn build_binding( Ok(Binding { collection_name: collection_name.clone(), delta_updates: *delta_updates, + document_uuid_ptr: json::Pointer::from(uuid_ptr.as_str()), key_extractors, read_schema_json, ser_policy, diff --git a/crates/runtime-next/src/shard/recovery.rs b/crates/runtime-next/src/shard/recovery.rs index 36e1e6ce1bc..61cdbcef753 100644 --- a/crates/runtime-next/src/shard/recovery.rs +++ b/crates/runtime-next/src/shard/recovery.rs @@ -18,12 +18,21 @@ //! between the binding indices used in the leader protocol and the //! `state_key` strings used in RocksDB keys. //! +//! Three keys carry backfill-truncation state, split by task type: `AB:` is +//! capture state — the in-progress active backfills that drive the +//! `estuary.dev/truncated-at` journal label — while `BB:`/`BC:` are +//! materialization state — the cumulative begin/complete clocks of the durable +//! truncation boundary used for stale source/loaded handling. +//! //! | Prefix | Key tail | Value | //! |--------------|------------------------------------------|----------------------------------| //! | `FH:` | `{journal}\0{state_key}\0{producer[6]}` | proto `shuffle.ProducerFrontier` | //! | `FC:` | `{journal}\0{state_key}\0{producer[6]}` | proto `shuffle.ProducerFrontier` | //! | `AI:` | `{journal}` | raw ACK intent bytes | //! | `MK-v2:` | `{state_key}` | `tuple::pack` packed key | +//! | `AB:` | `{state_key}` | fixed64 little-endian clock | +//! | `BB:` | `{state_key}` | fixed64 little-endian clock | +//! | `BC:` | `{state_key}` | fixed64 little-endian clock | //! | (singleton) | `checkpoint` | legacy `consumer.Checkpoint` | //! | (singleton) | `committed-close` | fixed64 little-endian clock | //! | (singleton) | `connector-state` | reduced JSON merge-patch | @@ -52,6 +61,22 @@ pub const PREFIX_ACK_INTENT: &[u8] = b"AI:"; pub const PREFIX_ACK_INTENT_END: &[u8] = b"AI;"; /// Key prefix for per-binding max-key entries: `MK-v2:{state_key}`. pub const PREFIX_MAX_KEY: &[u8] = b"MK-v2:"; +/// Capture state. Per-binding active-backfill begin clock. +pub const PREFIX_ACTIVE_BACKFILL: &[u8] = b"AB:"; +/// Materialization state. Per-binding cumulative backfill-begin clock. +pub const PREFIX_BACKFILL_BEGIN: &[u8] = b"BB:"; +/// Materialization state. Per-binding cumulative backfill-complete clock. +pub const PREFIX_BACKFILL_COMPLETE: &[u8] = b"BC:"; +/// Materialization state. Backfill-begin clocks of the *hinted* frontier — +/// persisted so a not-yet-committed marker survives recovery (both the +/// remote-authoritative reduce and the local idempotent replay rely on it). +pub const PREFIX_HINTED_BACKFILL_BEGIN: &[u8] = b"HB:"; +/// Exclusive upper bound used for `DeleteRange` over `PREFIX_HINTED_BACKFILL_BEGIN`. +pub const PREFIX_HINTED_BACKFILL_BEGIN_END: &[u8] = b"HB;"; +/// Materialization state. Backfill-complete clocks of the *hinted* frontier. +pub const PREFIX_HINTED_BACKFILL_COMPLETE: &[u8] = b"HC:"; +/// Exclusive upper bound used for `DeleteRange` over `PREFIX_HINTED_BACKFILL_COMPLETE`. +pub const PREFIX_HINTED_BACKFILL_COMPLETE_END: &[u8] = b"HC;"; /// Legacy checkpoint. pub const KEY_LEGACY_CHECKPOINT: &[u8] = b"checkpoint"; /// Clock at which the last-committed transaction closed. @@ -189,6 +214,20 @@ pub fn encode_persist>( &mut emit, &mut buf, )?; + + // Cumulative backfill clocks ride the committed Frontier (keyed by + // state_key), persisting the durable truncation boundary so it survives + // transaction rotation and leader restart. Kept out of encode_frontier + // (a separate keyspace) so the hinted frontier can carry its own + // boundary under distinct prefixes; see the hinted path below. + encode_backfill_clocks( + frontier, + binding_state_keys, + PREFIX_BACKFILL_BEGIN, + PREFIX_BACKFILL_COMPLETE, + &mut emit, + &mut buf, + )?; } for patch in crate::patches::split_state_patches(&persist.connector_patches_json)? { @@ -206,10 +245,24 @@ pub fn encode_persist>( } if persist.delete_hinted_frontier { - emit(KeyOp::DeleteRange { - from: Bytes::from_static(PREFIX_HINTED_FRONTIER), - to: Bytes::from_static(PREFIX_HINTED_FRONTIER_END), - }); + // Clear the hinted frontier and its backfill boundary together — each + // hint Persist rewrites both, so stale per-suffix entries must not linger. + for (from, to) in [ + (PREFIX_HINTED_FRONTIER, PREFIX_HINTED_FRONTIER_END), + ( + PREFIX_HINTED_BACKFILL_BEGIN, + PREFIX_HINTED_BACKFILL_BEGIN_END, + ), + ( + PREFIX_HINTED_BACKFILL_COMPLETE, + PREFIX_HINTED_BACKFILL_COMPLETE_END, + ), + ] { + emit(KeyOp::DeleteRange { + from: Bytes::from_static(from), + to: Bytes::from_static(to), + }); + } } if let Some(frontier) = &persist.hinted_frontier { encode_frontier( @@ -219,6 +272,16 @@ pub fn encode_persist>( &mut emit, &mut buf, )?; + // Persist the hinted frontier's backfill boundary too, under its own + // prefixes (see PREFIX_HINTED_BACKFILL_BEGIN for why). + encode_backfill_clocks( + frontier, + binding_state_keys, + PREFIX_HINTED_BACKFILL_BEGIN, + PREFIX_HINTED_BACKFILL_COMPLETE, + &mut emit, + &mut buf, + )?; } if !persist.last_applied.is_empty() { @@ -260,6 +323,36 @@ pub fn encode_persist>( }); } + // Active-backfill change: at most one per commit. Begin records the + // binding's truncated_at clock; Complete clears it. + if let Some(change) = &persist.active_backfill_change { + let (binding, truncated_at) = match change { + proto::persist::ActiveBackfillChange::Begin(begin) => { + (begin.binding, Some(begin.truncated_at)) + } + proto::persist::ActiveBackfillChange::CompleteBinding(binding) => (*binding, None), + }; + let state_key = binding_state_keys + .get(binding as usize) + .ok_or(EncodeError::UnknownBinding { + binding, + num_bindings: binding_state_keys.len(), + })? + .as_ref(); + + buf.extend_from_slice(PREFIX_ACTIVE_BACKFILL); + buf.extend_from_slice(state_key.as_bytes()); + let key = buf.split().freeze(); + + emit(match truncated_at { + Some(clock) => KeyOp::Put { + key, + value: Bytes::copy_from_slice(&clock.to_le_bytes()), + }, + None => KeyOp::Delete { key }, + }); + } + if persist.delete_trigger_params { emit(KeyOp::Delete { key: Bytes::from_static(KEY_TRIGGER_PARAMS), @@ -322,6 +415,47 @@ fn encode_frontier>( Ok(()) } +fn encode_backfill_clocks>( + frontier: &shuffle::proto::Frontier, + binding_state_keys: &[S], + begin_prefix: &[u8], + complete_prefix: &[u8], + emit: &mut impl FnMut(KeyOp), + buf: &mut BytesMut, +) -> Result<(), EncodeError> { + for entry in &frontier.latest_backfill_begin { + let state_key = binding_state_keys + .get(entry.binding as usize) + .ok_or(EncodeError::UnknownBinding { + binding: entry.binding, + num_bindings: binding_state_keys.len(), + })? + .as_ref(); + buf.extend_from_slice(begin_prefix); + buf.extend_from_slice(state_key.as_bytes()); + emit(KeyOp::Put { + key: buf.split().freeze(), + value: Bytes::copy_from_slice(&entry.clock.to_le_bytes()), + }); + } + for entry in &frontier.latest_backfill_complete { + let state_key = binding_state_keys + .get(entry.binding as usize) + .ok_or(EncodeError::UnknownBinding { + binding: entry.binding, + num_bindings: binding_state_keys.len(), + })? + .as_ref(); + buf.extend_from_slice(complete_prefix); + buf.extend_from_slice(state_key.as_bytes()); + emit(KeyOp::Put { + key: buf.split().freeze(), + value: Bytes::copy_from_slice(&entry.clock.to_le_bytes()), + }); + } + Ok(()) +} + fn append_frontier_key( out: &mut BytesMut, prefix: &[u8], @@ -359,14 +493,18 @@ pub fn committed_frontier_key(journal: &str, state_key: &str, producer: &uuid::P /// Decode one RocksDB `(key, value)` pair into recovery accumulators. /// /// `binding_state_keys` is a slice of `(state_key, binding_index)` tuples -/// sorted on `state_key`, used to translate persisted `state_key`s in -/// `FH:`/`FC:`/`MK-v2:` keys into their current binding indices. Entries -/// whose `state_key` does not appear in the slice are silently dropped: they -/// belong to bindings that have been removed or backfilled. +/// sorted on `state_key`, used to translate persisted `state_key`s into their +/// current binding indices. Entries whose `state_key` does not appear in the +/// slice are silently dropped: they belong to bindings that have been removed +/// or backfilled. pub fn decode_recover_key_value( recover: &mut proto::Recover, committed_frontier: &mut Vec, hinted_frontier: &mut Vec, + committed_backfill_begin: &mut std::collections::BTreeMap, + committed_backfill_complete: &mut std::collections::BTreeMap, + hinted_backfill_begin: &mut std::collections::BTreeMap, + hinted_backfill_complete: &mut std::collections::BTreeMap, key: &[u8], value: &[u8], binding_state_keys: &[(String, u32)], @@ -389,6 +527,39 @@ pub fn decode_recover_key_value( .insert(binding, Bytes::copy_from_slice(value)); } Ok(()) + } else if let Some(rest) = key.strip_prefix(PREFIX_ACTIVE_BACKFILL) { + let state_key = std::str::from_utf8(rest).map_err(DecodeError::InvalidUtf8)?; + if let Some(binding) = lookup_binding(binding_state_keys, state_key) { + recover + .active_backfills + .insert(binding, decode_clock(value, "active-backfill")?); + } + Ok(()) + } else if let Some(rest) = key.strip_prefix(PREFIX_BACKFILL_BEGIN) { + let state_key = std::str::from_utf8(rest).map_err(DecodeError::InvalidUtf8)?; + if let Some(binding) = lookup_binding(binding_state_keys, state_key) { + committed_backfill_begin.insert(binding, decode_clock(value, "backfill-begin")?); + } + Ok(()) + } else if let Some(rest) = key.strip_prefix(PREFIX_BACKFILL_COMPLETE) { + let state_key = std::str::from_utf8(rest).map_err(DecodeError::InvalidUtf8)?; + if let Some(binding) = lookup_binding(binding_state_keys, state_key) { + committed_backfill_complete.insert(binding, decode_clock(value, "backfill-complete")?); + } + Ok(()) + } else if let Some(rest) = key.strip_prefix(PREFIX_HINTED_BACKFILL_BEGIN) { + let state_key = std::str::from_utf8(rest).map_err(DecodeError::InvalidUtf8)?; + if let Some(binding) = lookup_binding(binding_state_keys, state_key) { + hinted_backfill_begin.insert(binding, decode_clock(value, "hinted-backfill-begin")?); + } + Ok(()) + } else if let Some(rest) = key.strip_prefix(PREFIX_HINTED_BACKFILL_COMPLETE) { + let state_key = std::str::from_utf8(rest).map_err(DecodeError::InvalidUtf8)?; + if let Some(binding) = lookup_binding(binding_state_keys, state_key) { + hinted_backfill_complete + .insert(binding, decode_clock(value, "hinted-backfill-complete")?); + } + Ok(()) } else if key == KEY_COMMITTED_CLOSE { recover.committed_close_clock = decode_clock(value, "committed-close-clock")?; Ok(()) @@ -408,6 +579,50 @@ pub fn decode_recover_key_value( } } +/// Restore backfill clocks recovered from the `BB:`/`BC:` (committed) and +/// `HB:`/`HC:` (hinted) keys onto their respective Frontiers, advancing the +/// durable truncation boundary. The hinted boundary lets a marker observed in a +/// hinted-but-not-committed transaction survive a remote-authoritative recovery. +pub fn restore_backfill_clocks( + recover: &mut proto::Recover, + committed_backfill_begin: std::collections::BTreeMap, + committed_backfill_complete: std::collections::BTreeMap, + hinted_backfill_begin: std::collections::BTreeMap, + hinted_backfill_complete: std::collections::BTreeMap, +) { + stamp_backfill( + &mut recover.committed_frontier, + committed_backfill_begin, + committed_backfill_complete, + ); + stamp_backfill( + &mut recover.hinted_frontier, + hinted_backfill_begin, + hinted_backfill_complete, + ); +} + +/// Stamp per-binding backfill clocks onto an optional Frontier, creating it only +/// when there's a boundary to record. +fn stamp_backfill( + frontier: &mut Option, + begin: std::collections::BTreeMap, + complete: std::collections::BTreeMap, +) { + if begin.is_empty() && complete.is_empty() { + return; + } + let frontier = frontier.get_or_insert_default(); + frontier.latest_backfill_begin = begin + .into_iter() + .map(|(binding, clock)| shuffle::proto::frontier::BackfillBegin { binding, clock }) + .collect(); + frontier.latest_backfill_complete = complete + .into_iter() + .map(|(binding, clock)| shuffle::proto::frontier::BackfillComplete { binding, clock }) + .collect(); +} + fn decode_clock(value: &[u8], kind: &'static str) -> Result { let bytes: [u8; 8] = value .try_into() @@ -693,16 +908,31 @@ mod test { let mut recover = proto::Recover::default(); let mut committed_frontier = Vec::new(); let mut hinted_frontier = Vec::new(); + let mut committed_backfill_begin = std::collections::BTreeMap::new(); + let mut committed_backfill_complete = std::collections::BTreeMap::new(); + let mut hinted_backfill_begin = std::collections::BTreeMap::new(); + let mut hinted_backfill_complete = std::collections::BTreeMap::new(); for (k, v) in pairs { decode_recover_key_value( &mut recover, &mut committed_frontier, &mut hinted_frontier, + &mut committed_backfill_begin, + &mut committed_backfill_complete, + &mut hinted_backfill_begin, + &mut hinted_backfill_complete, &k, &v, binding_state_keys, )?; } + restore_backfill_clocks( + &mut recover, + committed_backfill_begin, + committed_backfill_complete, + hinted_backfill_begin, + hinted_backfill_complete, + ); Ok(DecodedRecover { recover, committed_frontier, @@ -868,6 +1098,210 @@ mod test { insta::assert_debug_snapshot!(decode_pairs(store, &mapping).unwrap()); } + #[test] + fn committed_backfill_clocks_roundtrip() { + let persist = proto::Persist { + committed_frontier: Some(shuffle::proto::Frontier { + latest_backfill_begin: vec![ + shuffle::proto::frontier::BackfillBegin { + binding: 0, + clock: 111, + }, + shuffle::proto::frontier::BackfillBegin { + binding: 1, + clock: 222, + }, + ], + latest_backfill_complete: vec![shuffle::proto::frontier::BackfillComplete { + binding: 0, + clock: 110, + }], + ..Default::default() + }), + ..Default::default() + }; + + let binding_state_keys = &["materialize/mat/t1", "materialize/mat/t2"]; + let mut store: Vec<(Bytes, Bytes)> = Vec::new(); + encode_persist(&persist, binding_state_keys, |op| apply_op(&mut store, op)).unwrap(); + store.sort_by(|a, b| a.0.cmp(&b.0)); + + let mapping = state_key_index(&[("materialize/mat/t1", 0), ("materialize/mat/t2", 1)]); + let recovered = decode_pairs(store, &mapping) + .unwrap() + .recover + .committed_frontier + .expect("committed frontier recovered"); + + assert_eq!(recovered.latest_backfill_begin, persist_frontier_begin()); + assert_eq!( + recovered.latest_backfill_complete, + vec![shuffle::proto::frontier::BackfillComplete { + binding: 0, + clock: 110, + }], + ); + } + + #[test] + fn hinted_backfill_clocks_roundtrip() { + // The hinted boundary must survive encode→decode via the HB:/HC: keys — + // the durability a remote-authoritative recovery relies on. + let persist = proto::Persist { + delete_hinted_frontier: true, + hinted_frontier: Some(shuffle::proto::Frontier { + latest_backfill_begin: vec![shuffle::proto::frontier::BackfillBegin { + binding: 0, + clock: 111, + }], + latest_backfill_complete: vec![shuffle::proto::frontier::BackfillComplete { + binding: 0, + clock: 110, + }], + ..Default::default() + }), + ..Default::default() + }; + + let binding_state_keys = &["materialize/mat/t1", "materialize/mat/t2"]; + let mut store: Vec<(Bytes, Bytes)> = Vec::new(); + encode_persist(&persist, binding_state_keys, |op| apply_op(&mut store, op)).unwrap(); + store.sort_by(|a, b| a.0.cmp(&b.0)); + + let mapping = state_key_index(&[("materialize/mat/t1", 0), ("materialize/mat/t2", 1)]); + let recovered = decode_pairs(store, &mapping) + .unwrap() + .recover + .hinted_frontier + .expect("hinted frontier recovered from HB:/HC: keys"); + + assert_eq!( + recovered.latest_backfill_begin, + vec![shuffle::proto::frontier::BackfillBegin { + binding: 0, + clock: 111, + }], + ); + assert_eq!( + recovered.latest_backfill_complete, + vec![shuffle::proto::frontier::BackfillComplete { + binding: 0, + clock: 110, + }], + ); + } + + #[test] + fn active_backfill_change_roundtrip() { + // Capture-side AB: keys. A BackfillBegin records the binding's + // truncated_at clock; a later BackfillComplete deletes it. Bindings are + // keyed by stable state_key, so binding 1 resolves to "cap/c/t1". + let binding_state_keys = &["cap/c/t0", "cap/c/t1"]; + let mapping = state_key_index(&[("cap/c/t0", 0), ("cap/c/t1", 1)]); + + let begin = proto::Persist { + active_backfill_change: Some(proto::persist::ActiveBackfillChange::Begin( + proto::ActiveBackfillBegin { + binding: 1, + truncated_at: 0xABCD, + }, + )), + ..Default::default() + }; + let mut store: Vec<(Bytes, Bytes)> = Vec::new(); + encode_persist(&begin, binding_state_keys, |op| apply_op(&mut store, op)).unwrap(); + + let recovered = decode_pairs(store.clone(), &mapping).unwrap().recover; + assert_eq!( + recovered.active_backfills, + std::collections::BTreeMap::from([(1, 0xABCD)]), + "begin records binding 1's clock under its state_key", + ); + + // Completing binding 1 deletes the AB: key seeded above. + let complete = proto::Persist { + active_backfill_change: Some(proto::persist::ActiveBackfillChange::CompleteBinding(1)), + ..Default::default() + }; + encode_persist(&complete, binding_state_keys, |op| apply_op(&mut store, op)).unwrap(); + + let recovered = decode_pairs(store, &mapping).unwrap().recover; + assert!( + recovered.active_backfills.is_empty(), + "complete clears the binding's active backfill", + ); + } + + fn persist_frontier_begin() -> Vec { + vec![ + shuffle::proto::frontier::BackfillBegin { + binding: 0, + clock: 111, + }, + shuffle::proto::frontier::BackfillBegin { + binding: 1, + clock: 222, + }, + ] + } + + #[test] + fn restore_backfill_clocks_preserves_existing_journals() { + // The common case: a materialization with committed read offsets that has + // also observed a backfill. The clocks attach to the journal-bearing + // committed Frontier rather than replacing it. + let mut recover = proto::Recover { + committed_frontier: Some(frontier_fixture()), + ..Default::default() + }; + let journals = recover + .committed_frontier + .as_ref() + .unwrap() + .journals + .clone(); + + restore_backfill_clocks( + &mut recover, + std::collections::BTreeMap::from([(0u32, 111u64)]), + std::collections::BTreeMap::from([(0u32, 110u64)]), + std::collections::BTreeMap::new(), + std::collections::BTreeMap::new(), + ); + + let frontier = recover.committed_frontier.unwrap(); + assert_eq!(frontier.journals, journals, "journals are untouched"); + assert_eq!( + frontier.latest_backfill_begin, + vec![shuffle::proto::frontier::BackfillBegin { + binding: 0, + clock: 111, + }], + ); + assert_eq!( + frontier.latest_backfill_complete, + vec![shuffle::proto::frontier::BackfillComplete { + binding: 0, + clock: 110, + }], + ); + } + + #[test] + fn restore_backfill_clocks_without_clocks_is_noop() { + // No clocks must not materialize a committed Frontier: it stays `None`, + // matching the hinted-frontier "None when empty" invariant. + let mut recover = proto::Recover::default(); + restore_backfill_clocks( + &mut recover, + std::collections::BTreeMap::new(), + std::collections::BTreeMap::new(), + std::collections::BTreeMap::new(), + std::collections::BTreeMap::new(), + ); + assert!(recover.committed_frontier.is_none()); + } + #[test] fn delete_committed_frontier_clears_stale_keys() { // A prior session left a stale/partial committed Frontier in `FC:`. diff --git a/crates/runtime-next/src/shard/rocksdb.rs b/crates/runtime-next/src/shard/rocksdb.rs index 81cb435de09..1d9273e3601 100644 --- a/crates/runtime-next/src/shard/rocksdb.rs +++ b/crates/runtime-next/src/shard/rocksdb.rs @@ -175,6 +175,10 @@ impl RocksDB { let mut recover = proto::Recover::default(); let mut committed_frontier: Vec = Vec::new(); let mut hinted_frontier: Vec = Vec::new(); + let mut committed_backfill_begin = std::collections::BTreeMap::new(); + let mut committed_backfill_complete = std::collections::BTreeMap::new(); + let mut hinted_backfill_begin = std::collections::BTreeMap::new(); + let mut hinted_backfill_complete = std::collections::BTreeMap::new(); let mut it = self.db.raw_iterator(); it.seek_to_first(); @@ -184,6 +188,10 @@ impl RocksDB { &mut recover, &mut committed_frontier, &mut hinted_frontier, + &mut committed_backfill_begin, + &mut committed_backfill_complete, + &mut hinted_backfill_begin, + &mut hinted_backfill_complete, key, value, &index, @@ -238,6 +246,16 @@ impl RocksDB { .then(|| shuffle::JournalFrontier::encode(&frontier)); } + // Fold the persisted backfill clocks onto the recovered committed + // and hinted Frontiers (keyed by binding index). + recovery::restore_backfill_clocks( + &mut recover, + committed_backfill_begin, + committed_backfill_complete, + hinted_backfill_begin, + hinted_backfill_complete, + ); + Ok((self, recover)) }) .await diff --git a/crates/runtime-next/src/shard/snapshots/runtime_next__shard__recovery__test__decode_recover_classifies_ranges.snap b/crates/runtime-next/src/shard/snapshots/runtime_next__shard__recovery__test__decode_recover_classifies_ranges.snap index cca32411692..48d5c07c41f 100644 --- a/crates/runtime-next/src/shard/snapshots/runtime_next__shard__recovery__test__decode_recover_classifies_ranges.snap +++ b/crates/runtime-next/src/shard/snapshots/runtime_next__shard__recovery__test__decode_recover_classifies_ranges.snap @@ -1,6 +1,5 @@ --- source: crates/runtime-next/src/shard/recovery.rs -assertion_line: 947 expression: "decode_pairs(pairs, &mapping).unwrap()" --- DecodedRecover { @@ -26,6 +25,7 @@ DecodedRecover { 0: b"pk", }, trigger_params_json: b"{\"run_id\":\"r\"}", + active_backfills: {}, }, committed_frontier: [ JournalFrontier { diff --git a/crates/runtime-next/src/shard/snapshots/runtime_next__shard__recovery__test__encode_persist_hinted_then_committed_roundtrip.snap b/crates/runtime-next/src/shard/snapshots/runtime_next__shard__recovery__test__encode_persist_hinted_then_committed_roundtrip.snap index 04127c05bd1..8e709b339a4 100644 --- a/crates/runtime-next/src/shard/snapshots/runtime_next__shard__recovery__test__encode_persist_hinted_then_committed_roundtrip.snap +++ b/crates/runtime-next/src/shard/snapshots/runtime_next__shard__recovery__test__encode_persist_hinted_then_committed_roundtrip.snap @@ -1,6 +1,5 @@ --- source: crates/runtime-next/src/shard/recovery.rs -assertion_line: 797 expression: "decode_pairs(store, &mapping).unwrap()" --- DecodedRecover { @@ -20,6 +19,7 @@ DecodedRecover { 0: b"mk-v1", }, trigger_params_json: b"", + active_backfills: {}, }, committed_frontier: [ JournalFrontier { diff --git a/crates/runtime-next/src/shard/snapshots/runtime_next__shard__recovery__test__encode_persist_snapshots.snap b/crates/runtime-next/src/shard/snapshots/runtime_next__shard__recovery__test__encode_persist_snapshots.snap index c77437870d9..0729b43662a 100644 --- a/crates/runtime-next/src/shard/snapshots/runtime_next__shard__recovery__test__encode_persist_snapshots.snap +++ b/crates/runtime-next/src/shard/snapshots/runtime_next__shard__recovery__test__encode_persist_snapshots.snap @@ -38,6 +38,14 @@ expression: snapshot from: b"FH:", to: b"FH;", }, + DeleteRange { + from: b"HB:", + to: b"HB;", + }, + DeleteRange { + from: b"HC:", + to: b"HC;", + }, Put { key: b"FH:acme/events/000\0materialize/mat/t1\0\x01\xaa\0\0\0\0", value: b"\x11d\0\0\0\0\0\0\0 \xfa\x01", diff --git a/crates/runtime/src/harness/materialize.rs b/crates/runtime/src/harness/materialize.rs index 5e0ddad4d52..61ea8657161 100644 --- a/crates/runtime/src/harness/materialize.rs +++ b/crates/runtime/src/harness/materialize.rs @@ -238,6 +238,7 @@ async fn run_session( let flush = Request { flush: Some(request::Flush { state_patches_json: bytes::Bytes::new(), // Not implemented. + ..Default::default() }), ..Default::default() }; diff --git a/crates/shuffle/README.md b/crates/shuffle/README.md index 13db8044755..a93b331d168 100644 --- a/crates/shuffle/README.md +++ b/crates/shuffle/README.md @@ -343,6 +343,16 @@ The same "did `unresolved` make progress?" signal disarms the `on_tick` stall timeout: it fires only when no progress at all occurs between two consecutive ticks. +A peek also carries `latest_backfill_begin` eagerly (cloned from +`unresolved`, which retains it for the eventual resolved `ready`), as +scan-classification metadata: a downstream materialization must observe a +backfill-truncation boundary before it scans any source or Loaded document +at or above that boundary's clock, so documents on opposite sides of the +boundary are never combined. The begin clock only becomes durable +checkpoint state once its causal hints resolve and it rides a fully-resolved +`ready`. `latest_backfill_complete` is surfaced the same way — eagerly on a peek +and durably on a resolved `ready` — but plays no part in classification. + ### 12. Coordinator Receives Checkpoint The coordinator receives `NextCheckpoint` chunks and reassembles a diff --git a/crates/shuffle/src/frontier.rs b/crates/shuffle/src/frontier.rs index 5775cdaf804..2132fa0f27f 100644 --- a/crates/shuffle/src/frontier.rs +++ b/crates/shuffle/src/frontier.rs @@ -1,6 +1,7 @@ use crate::log; use proto_flow::shuffle; use proto_gazette::uuid::{Clock, Producer}; +use std::collections::BTreeMap; /// Lower-bound synthetic `last_commit` for a producer observed through journal /// reads, distinguishing it from a hint-only producer. All actual Clock values @@ -281,6 +282,7 @@ impl JournalFrontier { shuffle::Frontier { journals, flushed_lsn: vec![], + ..Default::default() } } } @@ -303,6 +305,13 @@ pub struct Frontier { /// Per-shard flushed LSN (log read-through barrier), indexed by shard_index. /// Empty when not applicable (e.g. resume checkpoints). pub flushed_lsn: Vec, + /// Latest committed backfill-begin clock of the checkpoint delta, keyed by + /// binding index. Folded from immediately-committed CONTROL documents; + /// does not participate in causal-hint sequencing. + pub latest_backfill_begin: BTreeMap, + /// Latest committed backfill-complete clock of the checkpoint delta, keyed + /// by binding index. See `latest_backfill_begin`. + pub latest_backfill_complete: BTreeMap, /// Count of `ProducerFrontier` entries with `hinted_commit > last_commit`. /// A Frontier with a non-zero count is "partial": readable for processing /// (e.g. log scanning), but NOT a transactional boundary. @@ -414,10 +423,24 @@ impl Frontier { Ok(Self { journals, flushed_lsn, + latest_backfill_begin: BTreeMap::new(), + latest_backfill_complete: BTreeMap::new(), unresolved_hints, }) } + fn merge_backfill_clocks( + mut a: BTreeMap, + b: BTreeMap, + ) -> BTreeMap { + for (binding, clock) in b { + a.entry(binding) + .and_modify(|current: &mut Clock| *current = (*current).max(clock)) + .or_insert(clock); + } + a + } + /// Element-wise max of two per-shard `flushed_lsn` vectors. /// Extends the shorter vector with zeros. pub fn merge_flushed_lsn(a: Vec, b: Vec) -> Vec { @@ -444,15 +467,25 @@ impl Frontier { /// Both inputs may contain non-unique keys, which are reduced to single entries. pub fn reduce(self, other: Self) -> Self { let flushed_lsn = Self::merge_flushed_lsn(self.flushed_lsn, other.flushed_lsn); + let latest_backfill_begin = + Self::merge_backfill_clocks(self.latest_backfill_begin, other.latest_backfill_begin); + let latest_backfill_complete = Self::merge_backfill_clocks( + self.latest_backfill_complete, + other.latest_backfill_complete, + ); if self.journals.is_empty() { return Self { flushed_lsn, + latest_backfill_begin, + latest_backfill_complete, ..other }; } else if other.journals.is_empty() { return Self { flushed_lsn, + latest_backfill_begin, + latest_backfill_complete, ..self }; } @@ -495,6 +528,8 @@ impl Frontier { Self { journals: merged, flushed_lsn, + latest_backfill_begin, + latest_backfill_complete, unresolved_hints, } } @@ -566,14 +601,42 @@ impl Frontier { pub fn encode(&self) -> shuffle::Frontier { let mut proto = JournalFrontier::encode(&self.journals); proto.flushed_lsn = self.flushed_lsn.iter().map(|lsn| lsn.as_u64()).collect(); + proto.latest_backfill_begin = self + .latest_backfill_begin + .iter() + .map(|(binding, clock)| shuffle::frontier::BackfillBegin { + binding: *binding as u32, + clock: clock.as_u64(), + }) + .collect(); + proto.latest_backfill_complete = self + .latest_backfill_complete + .iter() + .map(|(binding, clock)| shuffle::frontier::BackfillComplete { + binding: *binding as u32, + clock: clock.as_u64(), + }) + .collect(); proto } /// Decode a proto `shuffle::Frontier` into a validated `Frontier`. pub fn decode(mut proto: shuffle::Frontier) -> Result { let flushed_lsn = std::mem::take(&mut proto.flushed_lsn); + let latest_backfill_begin = std::mem::take(&mut proto.latest_backfill_begin) + .into_iter() + .map(|e| (e.binding as u16, Clock::from_u64(e.clock))) + .collect(); + let latest_backfill_complete = std::mem::take(&mut proto.latest_backfill_complete) + .into_iter() + .map(|e| (e.binding as u16, Clock::from_u64(e.clock))) + .collect(); + let journals: Vec = JournalFrontier::decode(proto).collect(); - Self::new(journals, flushed_lsn) + let mut frontier = Self::new(journals, flushed_lsn)?; + frontier.latest_backfill_begin = latest_backfill_begin; + frontier.latest_backfill_complete = latest_backfill_complete; + Ok(frontier) } /// Extract producers with unresolved causal hints (`hinted_commit > last_commit`) @@ -609,6 +672,8 @@ impl Frontier { Frontier { journals, flushed_lsn: vec![], + latest_backfill_begin: self.latest_backfill_begin.clone(), + latest_backfill_complete: self.latest_backfill_complete.clone(), unresolved_hints, } } @@ -691,6 +756,7 @@ mod test { use super::*; use crate::testing::{jf, jf_with_bytes, pf, pf_tuple}; use log::Lsn; + use std::collections::BTreeMap; #[test] fn test_producer_frontier_reduce() { @@ -778,11 +844,13 @@ mod test { )], flushed_lsn: vec![], unresolved_hints: 1, + ..Default::default() }; let progressed = Frontier { journals: vec![jf("journal/A", 0, vec![pf(0x01, 250, 0, -800)])], flushed_lsn: vec![], unresolved_hints: 0, + ..Default::default() }; let (advanced, resolved) = pending.resolve_hints(&progressed); @@ -813,11 +881,13 @@ mod test { )], flushed_lsn: vec![], unresolved_hints: 1, + ..Default::default() }; let progressed = Frontier { journals: vec![jf("journal/A", 0, vec![pf(0x01, 250, 0, -900)])], flushed_lsn: vec![], unresolved_hints: 0, + ..Default::default() }; let (advanced, resolved) = pending.resolve_hints(&progressed); assert_eq!((advanced, resolved), (1, 1)); @@ -853,6 +923,8 @@ mod test { ), ], flushed_lsn: vec![Lsn::from_u64(10), Lsn::from_u64(50), Lsn::from_u64(3)], + latest_backfill_begin: BTreeMap::from([(0, Clock::from_u64(100))]), + latest_backfill_complete: BTreeMap::from([(1, Clock::from_u64(140))]), unresolved_hints: 0, }; let hints = Frontier { @@ -861,6 +933,8 @@ mod test { jf("journal/C", 1, vec![pf(0x03, 0, 300, 0)]), ], flushed_lsn: vec![Lsn::from_u64(40), Lsn::from_u64(20), Lsn::from_u64(30)], + latest_backfill_begin: BTreeMap::from([(0, Clock::from_u64(120))]), + latest_backfill_complete: BTreeMap::from([(0, Clock::from_u64(130))]), unresolved_hints: 2, }; let r = reads.reduce(hints); @@ -926,19 +1000,38 @@ mod test { vec![Lsn::from_u64(40), Lsn::from_u64(50), Lsn::from_u64(30)], "element-wise max of flushed_lsn" ); + // Per-binding max of backfill clocks across both inputs. + assert_eq!(r.latest_backfill_begin.get(&0), Some(&Clock::from_u64(120))); + assert_eq!( + r.latest_backfill_complete.get(&0), + Some(&Clock::from_u64(130)) + ); + assert_eq!( + r.latest_backfill_complete.get(&1), + Some(&Clock::from_u64(140)) + ); - // Identity: empty reduces are no-ops and preserve flushed_lsn. + // Identity: reducing with an empty frontier preserves all fields. let f = Frontier { journals: vec![jf("j", 0, vec![pf(0x01, 1, 0, -1)])], flushed_lsn: vec![Lsn::from_u64(10), Lsn::from_u64(20)], + latest_backfill_begin: BTreeMap::from([(0, Clock::from_u64(100))]), + latest_backfill_complete: BTreeMap::from([(0, Clock::from_u64(200))]), unresolved_hints: 0, }; let r = f.clone().reduce(Frontier::default()); assert_eq!(r.journals.len(), 1); assert_eq!(r.flushed_lsn, vec![Lsn::from_u64(10), Lsn::from_u64(20)]); + assert_eq!(r.latest_backfill_begin, f.latest_backfill_begin); + assert_eq!(r.latest_backfill_complete, f.latest_backfill_complete); let r = Frontier::default().reduce(f); assert_eq!(r.journals.len(), 1); assert_eq!(r.flushed_lsn, vec![Lsn::from_u64(10), Lsn::from_u64(20)]); + assert_eq!(r.latest_backfill_begin.get(&0), Some(&Clock::from_u64(100))); + assert_eq!( + r.latest_backfill_complete.get(&0), + Some(&Clock::from_u64(200)) + ); assert!( Frontier::default() .reduce(Frontier::default()) @@ -1134,6 +1227,8 @@ mod test { ), ], flushed_lsn: vec![], + latest_backfill_begin: BTreeMap::new(), + latest_backfill_complete: BTreeMap::new(), unresolved_hints: 2, }; @@ -1145,6 +1240,8 @@ mod test { jf("journal/B", 0, vec![pf(0x03, 250, 0, -600)]), ], flushed_lsn: vec![], + latest_backfill_begin: BTreeMap::new(), + latest_backfill_complete: BTreeMap::new(), unresolved_hints: 0, }; @@ -1178,6 +1275,8 @@ mod test { let progressed2 = Frontier { journals: vec![jf("journal/B", 0, vec![pf(0x03, 400, 0, -900)])], flushed_lsn: vec![], + latest_backfill_begin: BTreeMap::new(), + latest_backfill_complete: BTreeMap::new(), unresolved_hints: 0, }; let (advanced2, resolved2) = pending.resolve_hints(&progressed2); @@ -1219,6 +1318,7 @@ mod test { ], flushed_lsn: vec![], unresolved_hints: 2, + ..Default::default() }; let progressed = Frontier { journals: vec![ @@ -1227,6 +1327,7 @@ mod test { ], flushed_lsn: vec![], unresolved_hints: 0, + ..Default::default() }; let (advanced, resolved) = pending.resolve_hints(&progressed); @@ -1254,11 +1355,15 @@ mod test { let mut pending = Frontier { journals: vec![jf("journal/X", 1, vec![pf(0x01, 0, 100, 0)])], flushed_lsn: vec![], + latest_backfill_begin: BTreeMap::new(), + latest_backfill_complete: BTreeMap::new(), unresolved_hints: 1, }; let progressed = Frontier { journals: vec![jf("journal/X", 0, vec![pf(0x01, 200, 0, -500)])], flushed_lsn: vec![], + latest_backfill_begin: BTreeMap::new(), + latest_backfill_complete: BTreeMap::new(), unresolved_hints: 0, }; assert_eq!(pending.resolve_hints(&progressed), (0, 0)); @@ -1308,11 +1413,21 @@ mod test { jf("journal/C", 1, vec![pf(0x07, 0, 300, 0)]), // unresolved ], flushed_lsn: vec![], + latest_backfill_begin: BTreeMap::from([(0, Clock::from_u64(9))]), + latest_backfill_complete: BTreeMap::from([(0, Clock::from_u64(8))]), unresolved_hints: 2, }; let projected = f.project_unresolved_hints(); + // The projection preserves the input's backfill boundary verbatim — the + // leader controls what it seeds onto the resume frontier (see startup.rs). + assert_eq!(projected.latest_backfill_begin, f.latest_backfill_begin); + assert_eq!( + projected.latest_backfill_complete, + f.latest_backfill_complete + ); + // journal/A: only P1 (unresolved). journal/B: filtered out (no hints). // journal/C: P7 (unresolved). insta::assert_debug_snapshot!(projected.journals.iter().map(|j| { @@ -1349,6 +1464,8 @@ mod test { let no_hints = Frontier { journals: vec![jf("journal/A", 0, vec![pf(0x01, 100, 0, -200)])], flushed_lsn: vec![], + latest_backfill_begin: BTreeMap::new(), + latest_backfill_complete: BTreeMap::new(), unresolved_hints: 0, }; assert!(no_hints.project_unresolved_hints().journals.is_empty()); @@ -1412,6 +1529,7 @@ mod test { journals, flushed_lsn: vec![], unresolved_hints: count, + ..Default::default() }; let desc = f.describe_unresolved(); @@ -1435,6 +1553,7 @@ mod test { ], flushed_lsn: vec![], unresolved_hints: 1, + ..Default::default() }; let desc = f.describe_unresolved(); diff --git a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_hint_resolved.snap b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_hint_resolved.snap index b616281392a..e8b8d69608d 100644 --- a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_hint_resolved.snap +++ b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_hint_resolved.snap @@ -24,11 +24,15 @@ PipelineSnapshot { flushed_lsn: [ 0/1000, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved_count: 0, @@ -66,6 +70,8 @@ PipelineSnapshot { flushed_lsn: [ 0/1000, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, } diff --git a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_normal_resumes.snap b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_normal_resumes.snap index 94c82c2d0d9..b8c6c901f03 100644 --- a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_normal_resumes.snap +++ b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_normal_resumes.snap @@ -66,17 +66,23 @@ PipelineSnapshot { flushed_lsn: [ 0/3000, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved_count: 0, progressed: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, } diff --git a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_second_progressed.snap b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_second_progressed.snap index 1e31e21427b..f68eca95878 100644 --- a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_second_progressed.snap +++ b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_second_progressed.snap @@ -24,11 +24,15 @@ PipelineSnapshot { flushed_lsn: [ 0/1000, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved_count: 0, @@ -80,6 +84,8 @@ PipelineSnapshot { flushed_lsn: [ 0/2000, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, } diff --git a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_take.snap b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_take.snap index 7d1c940fc28..d23da601dbe 100644 --- a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_take.snap +++ b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__after_take.snap @@ -52,17 +52,23 @@ PipelineSnapshot { flushed_lsn: [ 0/2000, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved_count: 0, progressed: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, } diff --git a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__checkpoint_progression.snap b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__checkpoint_progression.snap index c52296646c6..9f8742cf87c 100644 --- a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__checkpoint_progression.snap +++ b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__checkpoint_progression.snap @@ -34,5 +34,7 @@ Frontier { }, ], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, } diff --git a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__cross_cohort_hint_preserved.snap b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__cross_cohort_hint_preserved.snap index 8170eb5ef92..2f669dbd77d 100644 --- a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__cross_cohort_hint_preserved.snap +++ b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__cross_cohort_hint_preserved.snap @@ -7,6 +7,8 @@ PipelineSnapshot { ready: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved: Frontier { @@ -27,12 +29,16 @@ PipelineSnapshot { }, ], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 1, }, unresolved_count: 1, progressed: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, } diff --git a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__frontier_hint_not_filtered.snap b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__frontier_hint_not_filtered.snap index 5a7be161947..14b2944f47e 100644 --- a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__frontier_hint_not_filtered.snap +++ b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__frontier_hint_not_filtered.snap @@ -42,6 +42,8 @@ PipelineSnapshot { }, ], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved: Frontier { @@ -62,12 +64,16 @@ PipelineSnapshot { }, ], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 1, }, unresolved_count: 1, progressed: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, } diff --git a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__no_recovery.snap b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__no_recovery.snap index 8f6e1c8d530..75b102871f7 100644 --- a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__no_recovery.snap +++ b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__no_recovery.snap @@ -22,17 +22,23 @@ PipelineSnapshot { }, ], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved_count: 0, progressed: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, } diff --git a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__recovery_hint_survives_completed_clocks.snap b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__recovery_hint_survives_completed_clocks.snap index 96dbd7b184c..e4e48f11e9b 100644 --- a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__recovery_hint_survives_completed_clocks.snap +++ b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__recovery_hint_survives_completed_clocks.snap @@ -7,6 +7,8 @@ PipelineSnapshot { ready: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved: Frontier { @@ -27,12 +29,16 @@ PipelineSnapshot { }, ], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 1, }, unresolved_count: 1, progressed: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, } diff --git a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__stale_hint_filtered.snap b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__stale_hint_filtered.snap index 9af81710fbc..67495e40c92 100644 --- a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__stale_hint_filtered.snap +++ b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__stale_hint_filtered.snap @@ -42,17 +42,23 @@ PipelineSnapshot { }, ], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, unresolved_count: 0, progressed: Frontier { journals: [], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, } diff --git a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__taken_recovery.snap b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__taken_recovery.snap index f70e7423843..bbb16481bad 100644 --- a/crates/shuffle/src/session/snapshots/shuffle__session__state__test__taken_recovery.snap +++ b/crates/shuffle/src/session/snapshots/shuffle__session__state__test__taken_recovery.snap @@ -22,5 +22,7 @@ Frontier { flushed_lsn: [ 0/1000, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, } diff --git a/crates/shuffle/src/session/state.rs b/crates/shuffle/src/session/state.rs index 22d068e5bc5..ebcd2d1f2a0 100644 --- a/crates/shuffle/src/session/state.rs +++ b/crates/shuffle/src/session/state.rs @@ -370,6 +370,8 @@ impl CheckpointPipeline { journals: peek_journals, flushed_lsn: self.unresolved.flushed_lsn.clone(), unresolved_hints: self.unresolved.unresolved_hints, + latest_backfill_begin: self.unresolved.latest_backfill_begin.clone(), + latest_backfill_complete: self.unresolved.latest_backfill_complete.clone(), }; self.floor_flushed_lsn(&mut peek); @@ -782,6 +784,30 @@ mod test { pipeline.on_progressed(0, proto).unwrap(); } + fn ingest_progressed_with_backfill( + pipeline: &mut CheckpointPipeline, + journals: Vec, + begin: &[(u16, u64)], + complete: &[(u16, u64)], + ) { + let mut proto = crate::JournalFrontier::encode(&journals); + proto.latest_backfill_begin = begin + .iter() + .map(|(binding, clock)| shuffle::frontier::BackfillBegin { + binding: *binding as u32, + clock: *clock, + }) + .collect(); + proto.latest_backfill_complete = complete + .iter() + .map(|(binding, clock)| shuffle::frontier::BackfillComplete { + binding: *binding as u32, + clock: *clock, + }) + .collect(); + pipeline.on_progressed(0, proto).unwrap(); + } + // --- Tests --- #[test] @@ -1242,6 +1268,8 @@ mod test { journals: vec![jf("journal/A", 0, vec![pf(0x01, 10, 100, -50)])], flushed_lsn: vec![], unresolved_hints: 1, + latest_backfill_begin: Default::default(), + latest_backfill_complete: Default::default(), }, vec![0], ); @@ -1363,6 +1391,8 @@ mod test { journals: vec![jf("journal/A", 0, vec![pf(0x01, 50, 200, -100)])], flushed_lsn: vec![], unresolved_hints: 0, + latest_backfill_begin: Default::default(), + latest_backfill_complete: Default::default(), }, vec![0], ); @@ -1567,6 +1597,8 @@ mod test { journals: vec![jf("test/journal/F", 0, vec![pf(0x01, 100, 200, -500)])], flushed_lsn: vec![], unresolved_hints: 0, + latest_backfill_begin: Default::default(), + latest_backfill_complete: Default::default(), }; // Checkpoint found in resume_checkpoint. @@ -1859,6 +1891,8 @@ mod test { ], flushed_lsn: vec![], unresolved_hints: 0, + latest_backfill_begin: Default::default(), + latest_backfill_complete: Default::default(), }; let mut pipeline = CheckpointPipeline::new(&resume, vec![0, 0]); @@ -1890,4 +1924,238 @@ mod test { assert_eq!(pipeline.unresolved.unresolved_hints, 0, "hint resolved"); assert!(!pipeline.ready.journals.is_empty(), "promoted to ready"); } + + // A backfill marker broadcast to journals A and B: reading A's ACK commits A + // and hints B. Both marker clocks ride peeks eagerly but stay gated in + // `unresolved` — never durable — until B's ACK resolves them, then ride + // exactly one resolved `ready`. + #[test] + fn test_backfill_marker_gated_on_hint_resolution() { + let mut pipeline = test_pipeline(); + let c = 100u64; + let cc = 50u64; // A prior backfill's completion clock. + + // Read A's marker ACK: A committed at `c`; B is hinted at `c`. The begin + // and complete metadata ride in with the (still unresolved) frontier. + ingest_progressed_with_backfill( + &mut pipeline, + vec![ + jf("journal/A", 0, vec![pf(0x01, c, 0, -100)]), + jf("journal/B", 0, vec![pf(0x01, 0, c, 0)]), + ], + &[(0, c)], + &[(0, cc)], + ); + assert_eq!(pipeline.unresolved.unresolved_hints, 1, "B's hint holds it"); + + // A peek carries both eager marker clocks, gated in `unresolved`. + pipeline.request().unwrap(); + let peek = pipeline.take_ready().expect("peek of unresolved"); + assert_ne!( + peek.unresolved_hints, 0, + "it is a peek, not a resolved ready" + ); + assert_eq!( + peek.latest_backfill_begin.get(&0), + Some(&uuid::Clock::from_u64(c)), + "peek carries the eager begin clock" + ); + assert_eq!( + peek.latest_backfill_complete.get(&0), + Some(&uuid::Clock::from_u64(cc)), + "peek carries the eager complete clock too" + ); + + // Partial progress on B's hint (toward but below `c`) re-arms the peek + // without resolving it: the eager markers re-surface. + ingest_progressed_with_backfill( + &mut pipeline, + vec![jf("journal/B", 0, vec![pf(0x01, c / 2, 0, -50)])], + &[(0, c)], + &[(0, cc)], + ); + assert_eq!( + pipeline.unresolved.unresolved_hints, 1, + "B's hint still holds (advanced below `c`)" + ); + pipeline.request().unwrap(); + let peek2 = pipeline.take_ready().expect("re-armed peek"); + assert_eq!( + peek2.latest_backfill_begin.get(&0), + Some(&uuid::Clock::from_u64(c)), + "eager begin re-surfaces on the re-armed peek" + ); + assert_eq!( + peek2.latest_backfill_complete.get(&0), + Some(&uuid::Clock::from_u64(cc)), + "eager complete re-surfaces too" + ); + + // Read B's marker ACK: B commits at `c`, resolving the hold-back hint. + ingest_progressed_with_backfill( + &mut pipeline, + vec![jf("journal/B", 0, vec![pf(0x01, c, 0, -200)])], + &[(0, c)], + &[(0, cc)], + ); + assert_eq!(pipeline.unresolved.unresolved_hints, 0, "resolved"); + + // Both clocks now ride exactly one fully-resolved checkpoint. + pipeline.request().unwrap(); + let ready = pipeline.take_ready().expect("resolved checkpoint"); + assert_eq!(ready.unresolved_hints, 0); + assert_eq!( + ready.latest_backfill_begin.get(&0), + Some(&uuid::Clock::from_u64(c)), + "begin delivered on the resolved frontier" + ); + assert_eq!( + ready.latest_backfill_complete.get(&0), + Some(&uuid::Clock::from_u64(cc)), + "complete delivered on the resolved frontier" + ); + + // A subsequent checkpoint does not re-deliver it. + pipeline.request().unwrap(); + assert!( + pipeline.take_ready().is_none(), + "marker delivered exactly once" + ); + } + + // Recovery counterpart to the gating test: the replay carries the marker in its + // OWN checkpoint. Startup seeds the resume frontier with the hinted−committed + // marker delta (see startup.rs) and project_unresolved_hints preserves it, so + // the delta rides `unresolved` → the recovery-resolved `ready`. + #[test] + fn test_recovery_replay_carries_marker_on_recovery_ready() { + // Resume with an unresolved hint for P1 on journal/A, plus the seeded marker + // delta (as startup computes it) → recovery_pending. + let mut pipeline = CheckpointPipeline::new( + &crate::Frontier { + journals: vec![jf("journal/A", 0, vec![pf(0x01, 50, 200, -100)])], + flushed_lsn: vec![], + unresolved_hints: 1, + latest_backfill_begin: std::collections::BTreeMap::from([( + 0, + uuid::Clock::from_u64(42), + )]), + latest_backfill_complete: Default::default(), + }, + vec![0], + ); + assert!(pipeline.recovery_pending); + + // Replaying to the hinted commit resolves the hint. + pipeline.request().unwrap(); + ingest_progressed( + &mut pipeline, + vec![jf("journal/A", 0, vec![pf(0x01, 200, 0, -300)])], + vec![], + ); + + // The replay's own recovery checkpoint carries the marker — the seeded delta + // rode `unresolved` → `ready`, not one transaction late. + let recovery = pipeline.take_ready().expect("recovery ready"); + assert_eq!(recovery.unresolved_hints, 0, "recovery is fully resolved"); + assert_eq!( + recovery.latest_backfill_begin.get(&0), + Some(&uuid::Clock::from_u64(42)), + "replay's own checkpoint carries the marker", + ); + } + + // Two backfill generations in flight: gen-1's begin (clock C1) is unresolved + // when gen-2's begin (clock C2 > C1) arrives and parks in `progressed`. The + // generations must deliver in order, each exactly once. + #[test] + fn test_backfill_markers_two_generations_in_order() { + let mut pipeline = test_pipeline(); + let (c1, c2) = (100u64, 200u64); + + // Gen-1: read A's marker (A committed @C1, B hinted @C1). → unresolved. + ingest_progressed_with_backfill( + &mut pipeline, + vec![ + jf("journal/A", 0, vec![pf(0x01, c1, 0, -100)]), + jf("journal/B", 0, vec![pf(0x01, 0, c1, 0)]), + ], + &[(0, c1)], + &[], + ); + + // Gen-2: read A's later marker (A committed @C2, B hinted @C2). `unresolved` + // is occupied by gen-1, so this parks in `progressed` carrying begin @C2. + ingest_progressed_with_backfill( + &mut pipeline, + vec![ + jf("journal/A", 0, vec![pf(0x01, c2, 0, -300)]), + jf("journal/B", 0, vec![pf(0x01, 0, c2, 0)]), + ], + &[(0, c2)], + &[], + ); + assert_eq!(pipeline.unresolved.unresolved_hints, 1, "gen-1 still held"); + + // Resolve gen-1: read B's marker @C1. gen-1 promotes to ready; gen-2 then + // promotes from `progressed` into `unresolved` (B hinted @C2, held). + ingest_progressed_with_backfill( + &mut pipeline, + vec![jf("journal/B", 0, vec![pf(0x01, c1, 0, -200)])], + &[(0, c1)], + &[], + ); + + pipeline.request().unwrap(); + let gen1 = pipeline.take_ready().expect("gen-1 resolved"); + assert_eq!( + gen1.latest_backfill_begin.get(&0), + Some(&uuid::Clock::from_u64(c1)), + "first generation delivers C1" + ); + assert_eq!(pipeline.unresolved.unresolved_hints, 1, "gen-2 now held"); + + // Resolve gen-2: read B's marker @C2. + ingest_progressed_with_backfill( + &mut pipeline, + vec![jf("journal/B", 0, vec![pf(0x01, c2, 0, -400)])], + &[(0, c2)], + &[], + ); + + pipeline.request().unwrap(); + let gen2 = pipeline.take_ready().expect("gen-2 resolved"); + assert_eq!( + gen2.latest_backfill_begin.get(&0), + Some(&uuid::Clock::from_u64(c2)), + "second generation delivers C2" + ); + } + + // A single-partition collection: a marker ACK carries no hints, so its + // frontier is immediately fully-resolved and the marker delivers at once. + #[test] + fn test_backfill_marker_single_partition_immediate() { + let mut pipeline = test_pipeline(); + + ingest_progressed_with_backfill( + &mut pipeline, + vec![jf("journal/A", 0, vec![pf(0x01, 100, 0, -100)])], + &[(0, 100)], + &[(0, 100)], + ); + assert_eq!(pipeline.unresolved.unresolved_hints, 0); + + pipeline.request().unwrap(); + let ready = pipeline.take_ready().expect("immediate delivery"); + assert_eq!(ready.unresolved_hints, 0); + assert_eq!( + ready.latest_backfill_begin.get(&0), + Some(&uuid::Clock::from_u64(100)) + ); + assert_eq!( + ready.latest_backfill_complete.get(&0), + Some(&uuid::Clock::from_u64(100)) + ); + } } diff --git a/crates/shuffle/src/slice/actor.rs b/crates/shuffle/src/slice/actor.rs index b541ad0628f..c9cccdca949 100644 --- a/crates/shuffle/src/slice/actor.rs +++ b/crates/shuffle/src/slice/actor.rs @@ -345,6 +345,17 @@ impl SliceActor { let binding_state_key = binding.state_key().to_string(); let client = (*self.topology.journal_clients[binding.index as usize]).clone(); let spec = spec.context("StartRead missing spec")?; + + let truncated_at = match spec + .labels + .as_ref() + .map_or(Ok(""), |set| labels::maybe_one(set, labels::TRUNCATED_AT))? + { + "" => uuid::Clock::default(), + value => uuid::Clock::from_u64(labels::parse_truncated_at(value)?), + }; + let effective_not_before = binding.not_before.max(truncated_at); + let journal = spec.name.into_boxed_str(); let read_id = self.reads.len() as u32; @@ -356,7 +367,7 @@ impl SliceActor { // This helps identify the sources of reads from the perspective of a gazette broker. journal: format!("{journal};{}", binding.journal_read_suffix), - begin_mod_time: binding.not_before.to_unix().0 as i64, + begin_mod_time: effective_not_before.to_unix().0 as i64, block: true, do_not_proxy: true, end_offset: 0, // No end offset. @@ -388,6 +399,7 @@ impl SliceActor { self.reads.push(ReadState::recovered( binding_index as u16, journal, + truncated_at, producers, )); @@ -758,8 +770,13 @@ impl SliceActor { } let producer_state = read_state.producer_state(meta.producer); - let sequenced = - state::sequence_producer(producer_state, &read_state.journal, binding, meta)?; + let sequenced = state::sequence_producer( + producer_state, + &read_state.journal, + read_state.truncated_at, + binding, + meta, + )?; // If this document awakens a gapped producer, then we must pause // here (leaving this document in the heap) while we replay the @@ -812,7 +829,7 @@ impl SliceActor { read_state.read_offset = end_offset; if sequenced.is_commit { - if flags == uuid::Flags::ACK_TXN { + if flags.is_ack() { // This ACK is (binding, journal)-scoped: it commits only // this producer's documents in this binding's read of this journal. // But, it may contain causal hints of *other* journals which @@ -833,7 +850,12 @@ impl SliceActor { self.flush.set_ready(); } - // Step producer state forward to reflect the append. + // Fold any committed backfill control clock into this journal's + // per-flush backfill state, then step producer state forward. + read_state.backfill_begin = read_state.backfill_begin.max(sequenced.backfill_begin); + read_state.backfill_complete = read_state + .backfill_complete + .max(sequenced.backfill_complete); _ = read_state .pending .insert(producer, sequenced.producer_state); diff --git a/crates/shuffle/src/slice/producer.rs b/crates/shuffle/src/slice/producer.rs index 3bd51ca760e..3e258da3ff6 100644 --- a/crates/shuffle/src/slice/producer.rs +++ b/crates/shuffle/src/slice/producer.rs @@ -1,4 +1,5 @@ use proto_gazette::uuid::{Clock, Producer}; +use std::collections::BTreeMap; /// Per-producer sequencing state. /// @@ -68,6 +69,8 @@ pub fn build_flush_frontier( ) -> crate::Frontier { // Walk all journal reads to build their JournalFrontier. let mut journals: Vec = Vec::new(); + let mut latest_backfill_begin = BTreeMap::::new(); + let mut latest_backfill_complete = BTreeMap::::new(); for read_state in reads.iter_mut() { if read_state.pending.is_empty() { @@ -78,6 +81,26 @@ pub fn build_flush_frontier( // even if offsets advanced meanwhile. continue; } + + // Backfill clocks are per-binding metadata. Drain this journal's clocks + // into the checkpoint maps; a non-zero clock implies the read has + // pending work. Multiple journals of one binding fold to their max. + let binding = read_state.binding_index; + let backfill_begin = std::mem::take(&mut read_state.backfill_begin); + if backfill_begin != Clock::zero() { + latest_backfill_begin + .entry(binding) + .and_modify(|c| *c = (*c).max(backfill_begin)) + .or_insert(backfill_begin); + } + let backfill_complete = std::mem::take(&mut read_state.backfill_complete); + if backfill_complete != Clock::zero() { + latest_backfill_complete + .entry(binding) + .and_modify(|c| *c = (*c).max(backfill_complete)) + .or_insert(backfill_complete); + } + let mut producers: Vec<_> = read_state .pending .iter() @@ -119,6 +142,8 @@ pub fn build_flush_frontier( unresolved_hints: 0, // By construction: only `last_commit` set. journals, flushed_lsn: vec![crate::log::Lsn::ZERO; shard_count], + latest_backfill_begin, + latest_backfill_complete, }; // Build a Frontier from causal hints via single-pass iteration. @@ -162,6 +187,8 @@ pub fn build_flush_frontier( unresolved_hints, journals: hint_journals, flushed_lsn: vec![], + latest_backfill_begin: Default::default(), + latest_backfill_complete: Default::default(), }) } @@ -205,6 +232,9 @@ mod test { super::super::read::ReadState { binding_index: binding, journal: journal.into(), + truncated_at: Clock::zero(), + backfill_begin: Clock::zero(), + backfill_complete: Clock::zero(), settled: ProducerMap::default(), pending: map, read_offset, @@ -370,4 +400,38 @@ mod test { insta::assert_debug_snapshot!(snap); } + + #[test] + fn test_build_flush_frontier_drains_backfill_clocks_from_reads() { + // Two journals of one binding (shared `suffix/0`). The drain folds both + // journals' clocks per-binding via `max` and clears each read's fields. + // journal/A is processed first with the smaller begin; journal/B carries + // the larger begin, so the fold must `max`-upgrade to it rather than keep + // the first-seen value. Only journal/A carries a complete. + let mut has_both = read_state("journal/A", 0, &[(0x01, 100, -500)]); + has_both.backfill_begin = Clock::from_u64(80); + has_both.backfill_complete = Clock::from_u64(100); + + let mut begin_only = read_state("journal/B", 0, &[(0x03, 0, 700)]); + begin_only.backfill_begin = Clock::from_u64(90); + + let mut reads = vec![has_both, begin_only]; + + let frontier = build_flush_frontier(&mut reads, std::iter::empty(), 2); + + // Per-binding max across both journals of binding 0. + assert_eq!( + frontier.latest_backfill_begin.get(&0), + Some(&Clock::from_u64(90)) + ); + assert_eq!( + frontier.latest_backfill_complete.get(&0), + Some(&Clock::from_u64(100)) + ); + + // The per-journal clocks are drained (reset to zero). + assert_eq!(reads[0].backfill_begin, Clock::zero()); + assert_eq!(reads[0].backfill_complete, Clock::zero()); + assert_eq!(reads[1].backfill_begin, Clock::zero()); + } } diff --git a/crates/shuffle/src/slice/read.rs b/crates/shuffle/src/slice/read.rs index 41c8256a211..6bd6f2bed8d 100644 --- a/crates/shuffle/src/slice/read.rs +++ b/crates/shuffle/src/slice/read.rs @@ -2,6 +2,14 @@ use super::producer::ProducerState; use crate::ProducerMap; use proto_gazette::{broker, uuid}; +/// A backfill begin/complete event, parsed from the top-level fields of an +/// ACK_TXN document. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BackfillEvent { + BackfillBegin, + BackfillComplete { truncated_at: uuid::Clock }, +} + /// State about an active read, indexed by its `read.id()`. /// /// Each ReadState represents one (journal, binding) pair and is the @@ -14,6 +22,16 @@ pub struct ReadState { pub binding_index: u16, /// The journal name (canonical, without the `;suffix` read metadata). pub journal: Box, + /// Truncation boundary for this journal: the `estuary.dev/truncated-at` + /// label clock, or zero when absent. Documents published before it were + /// superseded by a backfill, so `sequence_document` suppresses their append. + pub truncated_at: uuid::Clock, + /// Clock of the latest `BackfillBegin` folded from a committing ACK on this + /// journal, awaiting the next flush (zero = none). + pub backfill_begin: uuid::Clock, + /// Truncation boundary of the latest `BackfillComplete` folded from a + /// committing ACK on this journal, awaiting the next flush (zero = none). + pub backfill_complete: uuid::Clock, /// Producers whose state is settled: either from the initial checkpoint /// or drained from `pending` at the start of a flush cycle. pub settled: ProducerMap, @@ -36,11 +54,15 @@ impl ReadState { pub fn recovered( binding_index: u16, journal: Box, + truncated_at: uuid::Clock, settled: ProducerMap, ) -> Self { Self { binding_index, journal, + truncated_at, + backfill_begin: uuid::Clock::zero(), + backfill_complete: uuid::Clock::zero(), settled, pending: Default::default(), read_offset: 0, @@ -92,6 +114,8 @@ pub struct Meta { pub flags: uuid::Flags, /// Publication Producer of `doc` (extracted from its UUID). pub producer: uuid::Producer, + /// Parsed backfill event, when this ACK document carries backfill fields. + pub backfill_event: Option, } /// ReadyRead is a ReadLines which has one or more parsed documents. @@ -135,7 +159,37 @@ pub fn extract_metas( ) })?; - let flags = if flags != uuid::Flags::ACK_TXN && validator.is_valid(archived) { + let backfill_event = if !flags.is_ack() { + None + } else { + match json::AsNode::as_node(archived) { + json::Node::Object(fields) => { + if json::Fields::get(fields, "backfillBegin").is_some() { + Some(BackfillEvent::BackfillBegin) + } else if json::Fields::get(fields, "backfillComplete").is_some() { + let truncated_at = json::Fields::get(fields, "truncatedAt") + .map(|field| json::Field::value(&field)) + .and_then(|node| match node { + doc::ArchivedNode::String(s) => labels::parse_truncated_at(s).ok(), + _ => None, + }) + .map(uuid::Clock::from_u64) + .ok_or_else(|| { + anyhow::anyhow!( + "journal {journal} offset {begin_offset}: \ + BackfillComplete ACK is missing a valid truncatedAt" + ) + })?; + Some(BackfillEvent::BackfillComplete { truncated_at }) + } else { + None + } + } + _ => None, + } + }; + + let flags = if !flags.is_ack() && validator.is_valid(archived) { uuid::Flags(flags.0 | crate::FLAGS_SCHEMA_VALID) } else { flags @@ -147,6 +201,7 @@ pub fn extract_metas( clock, flags, producer, + backfill_event, }; begin_offset = end_offset; @@ -370,7 +425,8 @@ mod test { #[test] fn test_extract_metas() { - // Schema requires "required_field", exercising both valid and invalid paths. + // Schema requires "required_field", exercising valid, invalid, ACK bypass, + // and backfill marker (begin / complete) ACK paths. let schema = br#"{"type":"object","required":["required_field"]}"#; let bundle = doc::validation::build_bundle(schema).unwrap(); let mut validator = doc::Validator::new(bundle).unwrap(); @@ -380,22 +436,39 @@ mod test { let c1 = clock.tick(); let c2 = clock.tick(); let c3 = clock.tick(); + let c4 = clock.tick(); + let c5 = clock.tick(); - // Three docs exercise: non-zero base offset, offset chaining, - // OUTSIDE_TXN/CONTINUE_TXN/ACK_TXN flags, valid + invalid schema, ACK bypass. let json = [ + // Valid schema, OUTSIDE_TXN. format!( r#"{{"_meta":{{"uuid":"{}"}},"required_field":"present"}}"#, make_uuid_str(p1, c1, uuid::Flags::OUTSIDE_TXN), ), + // Invalid schema, CONTINUE_TXN. format!( r#"{{"_meta":{{"uuid":"{}"}},"other":"value"}}"#, make_uuid_str(p1, c2, uuid::Flags::CONTINUE_TXN), ), + // ACK_TXN (skips validation). format!( r#"{{"_meta":{{"uuid":"{}"}}}}"#, make_uuid_str(p1, c3, uuid::Flags::ACK_TXN), ), + // Backfill begin marker: an ACK_TXN carrying a top-level + // `backfillBegin` field (alongside `is_ack`). Skips validation like + // any ACK; its own clock is the truncation boundary. + format!( + r#"{{"_meta":{{"uuid":"{}"}},"is_ack":true,"backfillBegin":true}}"#, + make_uuid_str(p1, c4, uuid::Flags::ACK_TXN), + ), + // Backfill complete marker: an ACK_TXN carrying the begin boundary it + // completed as a top-level hex-encoded-clock `truncatedAt`. + format!( + r#"{{"_meta":{{"uuid":"{}"}},"is_ack":true,"backfillComplete":true,"truncatedAt":"{}"}}"#, + make_uuid_str(p1, c5, uuid::Flags::ACK_TXN), + labels::truncated_at_value(uuid::Clock::from_unix(1_700_000_000, 0).as_u64()), + ), ] .join("\n") + "\n"; diff --git a/crates/shuffle/src/slice/replay.rs b/crates/shuffle/src/slice/replay.rs index fdb36109359..cd6ded9ee8d 100644 --- a/crates/shuffle/src/slice/replay.rs +++ b/crates/shuffle/src/slice/replay.rs @@ -199,6 +199,7 @@ impl SliceActor { let sequenced = state::sequence_producer( producer_state, &read_state.journal, + read_state.truncated_at, binding, &ready_read.meta, )?; diff --git a/crates/shuffle/src/slice/snapshots/shuffle__slice__producer__test__build_flush_frontier.snap b/crates/shuffle/src/slice/snapshots/shuffle__slice__producer__test__build_flush_frontier.snap index fd073594436..f5c07c86dda 100644 --- a/crates/shuffle/src/slice/snapshots/shuffle__slice__producer__test__build_flush_frontier.snap +++ b/crates/shuffle/src/slice/snapshots/shuffle__slice__producer__test__build_flush_frontier.snap @@ -1,6 +1,5 @@ --- source: crates/shuffle/src/slice/producer.rs -assertion_line: 400 expression: snap --- [ @@ -13,6 +12,8 @@ expression: snap 0/0, 0/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, [], @@ -55,12 +56,17 @@ expression: snap 0/0, 0/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, [ ReadState { binding_index: 0, journal: "journal/B", + truncated_at: Clock(0s 0ns), + backfill_begin: Clock(0s 0ns), + backfill_complete: Clock(0s 0ns), settled: { Producer(03:00:00:00:00:00): ProducerState { last_commit: Clock(0s 32ns), @@ -77,6 +83,9 @@ expression: snap ReadState { binding_index: 0, journal: "journal/A", + truncated_at: Clock(0s 0ns), + backfill_begin: Clock(0s 0ns), + backfill_complete: Clock(0s 0ns), settled: { Producer(01:00:00:00:00:00): ProducerState { last_commit: Clock(0s 16ns), @@ -93,6 +102,9 @@ expression: snap ReadState { binding_index: 0, journal: "journal/C", + truncated_at: Clock(0s 0ns), + backfill_begin: Clock(0s 0ns), + backfill_complete: Clock(0s 0ns), settled: {}, pending: {}, read_offset: 123, @@ -140,6 +152,8 @@ expression: snap 0/0, 0/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 2, }, [], @@ -168,12 +182,17 @@ expression: snap 0/0, 0/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, [ ReadState { binding_index: 0, journal: "journal/A", + truncated_at: Clock(0s 0ns), + backfill_begin: Clock(0s 0ns), + backfill_complete: Clock(0s 0ns), settled: { Producer(01:00:00:00:00:00): ProducerState { last_commit: Clock(0s 16ns), @@ -190,6 +209,9 @@ expression: snap ReadState { binding_index: 0, journal: "journal/B", + truncated_at: Clock(0s 0ns), + backfill_begin: Clock(0s 0ns), + backfill_complete: Clock(0s 0ns), settled: {}, pending: {}, read_offset: 0, @@ -257,12 +279,17 @@ expression: snap 0/0, 0/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 2, }, [ ReadState { binding_index: 0, journal: "journal/A", + truncated_at: Clock(0s 0ns), + backfill_begin: Clock(0s 0ns), + backfill_complete: Clock(0s 0ns), settled: { Producer(01:00:00:00:00:00): ProducerState { last_commit: Clock(0s 16ns), @@ -279,6 +306,9 @@ expression: snap ReadState { binding_index: 0, journal: "journal/B", + truncated_at: Clock(0s 0ns), + backfill_begin: Clock(0s 0ns), + backfill_complete: Clock(0s 0ns), settled: { Producer(03:00:00:00:00:00): ProducerState { last_commit: Clock(0s 32ns), @@ -351,12 +381,17 @@ expression: snap 0/0, 0/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 1, }, [ ReadState { binding_index: 2, journal: "journal/X", + truncated_at: Clock(0s 0ns), + backfill_begin: Clock(0s 0ns), + backfill_complete: Clock(0s 0ns), settled: { Producer(01:00:00:00:00:00): ProducerState { last_commit: Clock(0s 16ns), @@ -373,6 +408,9 @@ expression: snap ReadState { binding_index: 0, journal: "journal/X", + truncated_at: Clock(0s 0ns), + backfill_begin: Clock(0s 0ns), + backfill_complete: Clock(0s 0ns), settled: { Producer(03:00:00:00:00:00): ProducerState { last_commit: Clock(0s 8ns), @@ -412,6 +450,8 @@ expression: snap 0/0, 0/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 1, }, [], @@ -440,12 +480,17 @@ expression: snap 0/0, 0/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 1, }, [ ReadState { binding_index: 0, journal: "journal/A", + truncated_at: Clock(0s 0ns), + backfill_begin: Clock(0s 0ns), + backfill_complete: Clock(0s 0ns), settled: { Producer(01:00:00:00:00:00): ProducerState { last_commit: Clock(0s 8ns), @@ -511,12 +556,17 @@ expression: snap 0/0, 0/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 1, }, [ ReadState { binding_index: 0, journal: "journal/A", + truncated_at: Clock(0s 0ns), + backfill_begin: Clock(0s 0ns), + backfill_complete: Clock(0s 0ns), settled: { Producer(01:00:00:00:00:00): ProducerState { last_commit: Clock(0s 0ns), diff --git a/crates/shuffle/src/slice/snapshots/shuffle__slice__read__test__extract_metas.snap b/crates/shuffle/src/slice/snapshots/shuffle__slice__read__test__extract_metas.snap index 3a0c885fa03..bd32799f160 100644 --- a/crates/shuffle/src/slice/snapshots/shuffle__slice__read__test__extract_metas.snap +++ b/crates/shuffle/src/slice/snapshots/shuffle__slice__read__test__extract_metas.snap @@ -9,6 +9,7 @@ expression: metas clock: Clock(1000s 1000ns), flags: Flags(8000 (OUTSIDE_TXN)), producer: Producer(01:00:00:00:00:00), + backfill_event: None, }, Meta { begin_offset: 12430, @@ -16,6 +17,7 @@ expression: metas clock: Clock(1000s 2000ns), flags: Flags(1 (CONTINUE_TXN)), producer: Producer(01:00:00:00:00:00), + backfill_event: None, }, Meta { begin_offset: 12504, @@ -23,5 +25,28 @@ expression: metas clock: Clock(1000s 3000ns), flags: Flags(2 (ACK_TXN)), producer: Producer(01:00:00:00:00:00), + backfill_event: None, + }, + Meta { + begin_offset: 12562, + end_offset: 12655, + clock: Clock(1000s 4000ns), + flags: Flags(2 (ACK_TXN)), + producer: Producer(01:00:00:00:00:00), + backfill_event: Some( + BackfillBegin, + ), + }, + Meta { + begin_offset: 12655, + end_offset: 12784, + clock: Clock(1000s 5000ns), + flags: Flags(2 (ACK_TXN)), + producer: Producer(01:00:00:00:00:00), + backfill_event: Some( + BackfillComplete { + truncated_at: Clock(1700000000s 0ns), + }, + ), }, ] diff --git a/crates/shuffle/src/slice/snapshots/shuffle__slice__state__test__flush_state_machine.snap b/crates/shuffle/src/slice/snapshots/shuffle__slice__state__test__flush_state_machine.snap index b234fb561c9..ef8de17ecbd 100644 --- a/crates/shuffle/src/slice/snapshots/shuffle__slice__state__test__flush_state_machine.snap +++ b/crates/shuffle/src/slice/snapshots/shuffle__slice__state__test__flush_state_machine.snap @@ -30,5 +30,7 @@ Frontier { 0/200, 0/300, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, } diff --git a/crates/shuffle/src/slice/state.rs b/crates/shuffle/src/slice/state.rs index ae86ee31e14..d6d1632a569 100644 --- a/crates/shuffle/src/slice/state.rs +++ b/crates/shuffle/src/slice/state.rs @@ -217,6 +217,10 @@ pub struct SequencedDoc { pub replay: bool, /// Updated producer state to commit after processing this document. pub producer_state: ProducerState, + /// Backfill-begin control-doc clock this document committed (zero = none). + pub backfill_begin: uuid::Clock, + /// Backfill-complete control-doc clock this document committed (zero = none). + pub backfill_complete: uuid::Clock, } /// Gate on `adjusted_clock` relative to `now`: if the clock is in the future, @@ -329,6 +333,7 @@ pub fn resolve_checkpoint(checkpoint: Vec) -> Resolve pub fn sequence_producer( mut producer_state: ProducerState, journal: &str, + truncated_at: uuid::Clock, binding: &crate::Binding, meta: &super::read::Meta, ) -> anyhow::Result { @@ -336,6 +341,7 @@ pub fn sequence_producer( producer, clock, flags, + backfill_event: marker, begin_offset, end_offset, } = meta; @@ -366,6 +372,9 @@ pub fn sequence_producer( uuid::SequenceOutcome::ContinueExtendSpan | uuid::SequenceOutcome::AckCommit ); + let mut backfill_begin = uuid::Clock::zero(); + let mut backfill_complete = uuid::Clock::zero(); + // Match over `outcome` to update `producer_state` and determine append/commit. let (is_append, is_commit) = match outcome { uuid::SequenceOutcome::OutsideCommit => { @@ -398,15 +407,32 @@ pub fn sequence_producer( // CONTINUE_TXN documents in this journal only. Cross-journal // visibility for the same producer transaction is handled separately // via `extract_causal_hints`. + // + // A committing ACK also folds any backfill marker it carries. Markers + // are broadcast on ACKs and delivered exactly once per committed + // generation; a conservative re-read that replays the ACK as + // `AckDuplicate` (or an `AckDeepRollback`) must NOT re-fold, which is + // why the fold lives on this arm alone. + match marker { + Some(super::read::BackfillEvent::BackfillBegin) => backfill_begin = *clock, + Some(super::read::BackfillEvent::BackfillComplete { truncated_at }) => { + backfill_complete = *truncated_at + } + None => {} + } producer_state.offset = -*end_offset; (false, true) } uuid::SequenceOutcome::AckDuplicate => (false, false), }; - // A `notBefore` or `notAfter` suppresses document append, but doesn't impact - // the propagation of flush and progress reporting. - let is_append = is_append && *clock >= binding.not_before && *clock < binding.not_after; + // A `notBefore`/`notAfter` window and the journal's `truncated_at` + // truncation boundary all suppress document append, but don't impact the + // propagation of flush and progress reporting. + let is_append = is_append + && *clock >= binding.not_before + && *clock >= truncated_at + && *clock < binding.not_after; tracing::trace!( %journal, @@ -426,6 +452,8 @@ pub fn sequence_producer( is_commit, replay, producer_state, + backfill_begin, + backfill_complete, }) } @@ -624,6 +652,7 @@ mod test { flags, begin_offset, end_offset, + backfill_event: None, } } @@ -640,7 +669,13 @@ mod test { meta: &Meta, ) -> anyhow::Result { let producer_state = read_state.producer_state(meta.producer); - sequence_producer(producer_state, &read_state.journal, binding, meta) + sequence_producer( + producer_state, + &read_state.journal, + read_state.truncated_at, + binding, + meta, + ) } /// Build a checkpoint entry for a producer with zero clocks and a committed offset. @@ -672,9 +707,10 @@ mod test { impl TestState { fn commit(&mut self, read_id: usize, producer: Producer, seq: SequencedDoc) { - _ = self.reads[read_id] - .pending - .insert(producer, seq.producer_state); + let read = &mut self.reads[read_id]; + read.backfill_begin = read.backfill_begin.max(seq.backfill_begin); + read.backfill_complete = read.backfill_complete.max(seq.backfill_complete); + _ = read.pending.insert(producer, seq.producer_state); if seq.is_commit { self.flush.set_ready(); } @@ -818,6 +854,7 @@ mod test { let s = sequence_producer( state, "test/journal", + Clock::zero(), &binding, &meta(p1, Clock::from_u64(150), OUTSIDE, 500, 600), ) @@ -831,6 +868,7 @@ mod test { let s = sequence_producer( state, "test/journal", + Clock::zero(), &binding, &meta(p1, Clock::from_u64(150), CONTINUE, 500, 600), ) @@ -840,6 +878,7 @@ mod test { let s = sequence_producer( s.producer_state, "test/journal", + Clock::zero(), &binding, &meta(p1, Clock::from_u64(150), ACK, 600, 700), ) @@ -872,6 +911,7 @@ mod test { sequence_producer( gapped, journal, + Clock::zero(), &binding, &meta(p, Clock::from_u64(clock), flags, 400, 500), ) @@ -991,6 +1031,7 @@ mod test { let s = sequence_producer( committed, journal, + Clock::zero(), &binding, &meta(p, Clock::from_u64(1_001), CONTINUE, 900, 950), ) @@ -1014,6 +1055,7 @@ mod test { let s = sequence_producer( s.producer_state, journal, + Clock::zero(), &binding, &meta(p, Clock::from_u64(1_002), CONTINUE, 950, 1_000), ) @@ -1025,6 +1067,7 @@ mod test { let s = sequence_producer( s.producer_state, journal, + Clock::zero(), &binding, &meta(p, Clock::from_u64(1_003), ACK, 1_000, 1_050), ) @@ -1071,6 +1114,9 @@ mod test { s.reads.push(ReadState { binding_index: 0, journal: "test/journal/A".into(), + truncated_at: Clock::zero(), + backfill_begin: Clock::zero(), + backfill_complete: Clock::zero(), settled: producers, pending: Default::default(), read_offset: 0, @@ -1125,6 +1171,9 @@ mod test { s.reads.push(ReadState { binding_index: 0, journal: "test/journal/A".into(), + truncated_at: Clock::zero(), + backfill_begin: Clock::zero(), + backfill_complete: Clock::zero(), settled: producers, pending: Default::default(), read_offset: 0, @@ -1229,6 +1278,7 @@ mod test { }], flushed_lsn: vec![], unresolved_hints: 0, + ..Default::default() }; // Request + flushed → has_progressed gates, take_progressed consumes. @@ -1285,6 +1335,9 @@ mod test { s.reads.push(ReadState { binding_index: 0, journal: "test/journal/A".into(), + truncated_at: Clock::zero(), + backfill_begin: Clock::zero(), + backfill_complete: Clock::zero(), settled: producers, pending: Default::default(), read_offset: 0, @@ -1380,6 +1433,135 @@ mod test { ); } + #[test] + fn test_sequence_ack_backfill_markers_fold_on_commit() { + let bindings = vec![test_binding(0, true, None, "/suffix")]; + let mut s = test_state(bindings); + + let p1 = producer(0x01); + + let ResolvedCheckpoint { producers, .. } = + resolve_checkpoint(vec![checkpoint_entry(&p1, 0)]); + s.reads.push(ReadState { + binding_index: 0, + journal: "test/journal/A".into(), + truncated_at: Clock::zero(), + backfill_begin: Clock::zero(), + backfill_complete: Clock::zero(), + settled: producers, + pending: Default::default(), + read_offset: 0, + prev_read_offset: 0, + write_head: 0, + prev_write_head: 0, + }); + + // A BackfillBegin marker rides on an ACK. In a journal the producer never + // wrote to (no pending span), it sequences as AckEmpty: it commits, folds + // its clock into backfill_begin, and is never appended (ACKs never append). + let seq = sequence( + &s.reads[0], + &s.bindings[0], + &Meta { + producer: p1, + clock: Clock::from_unix(10, 0), + flags: ACK, + begin_offset: 100, + end_offset: 150, + backfill_event: Some(super::super::read::BackfillEvent::BackfillBegin), + }, + ) + .unwrap(); + assert!(!seq.is_append, "ACKs never append"); + assert!(seq.is_commit, "AckEmpty commits"); + assert_eq!( + seq.backfill_begin, + Clock::from_unix(10, 0), + "reports its clock" + ); + assert_eq!(seq.backfill_complete, Clock::zero()); + s.commit(0, p1, seq); + assert_eq!(s.reads[0].backfill_begin, Clock::from_unix(10, 0)); + + // A BackfillComplete marker at a later clock carries the begin boundary it + // completed as `truncated_at` (so a reader need not have seen the begin). + let seq = sequence( + &s.reads[0], + &s.bindings[0], + &Meta { + producer: p1, + clock: Clock::from_unix(20, 0), + flags: ACK, + begin_offset: 150, + end_offset: 200, + backfill_event: Some(super::super::read::BackfillEvent::BackfillComplete { + truncated_at: Clock::from_unix(10, 0), + }), + }, + ) + .unwrap(); + assert!(!seq.is_append); + assert!(seq.is_commit); + assert_eq!(seq.backfill_begin, Clock::zero()); + assert_eq!(seq.backfill_complete, Clock::from_unix(10, 0)); + s.commit(0, p1, seq); + assert_eq!(s.reads[0].backfill_begin, Clock::from_unix(10, 0)); + assert_eq!(s.reads[0].backfill_complete, Clock::from_unix(10, 0)); + + // A conservative re-read replays the same complete ACK (same clock). It + // sequences as AckDuplicate: no commit, and crucially NO re-fold — the + // marker must not be re-delivered within an already-resolved checkpoint. + let seq = sequence( + &s.reads[0], + &s.bindings[0], + &Meta { + producer: p1, + clock: Clock::from_unix(20, 0), + flags: ACK, + begin_offset: 200, + end_offset: 250, + backfill_event: Some(super::super::read::BackfillEvent::BackfillComplete { + truncated_at: Clock::from_unix(10, 0), + }), + }, + ) + .unwrap(); + assert!(!seq.is_append); + assert!(!seq.is_commit, "AckDuplicate suppresses flush"); + assert_eq!( + seq.backfill_complete, + Clock::zero(), + "duplicate does not re-fold" + ); + s.commit(0, p1, seq); + assert_eq!( + s.reads[0].backfill_complete, + Clock::from_unix(10, 0), + "unchanged on duplicate" + ); + + // A fresh BackfillBegin ACK at a later clock supersedes the prior begin + // via the commit-time max fold into the read. + let seq = sequence( + &s.reads[0], + &s.bindings[0], + &Meta { + producer: p1, + clock: Clock::from_unix(30, 0), + flags: ACK, + begin_offset: 250, + end_offset: 300, + backfill_event: Some(super::super::read::BackfillEvent::BackfillBegin), + }, + ) + .unwrap(); + assert!(seq.is_commit); + assert_eq!(seq.backfill_begin, Clock::from_unix(30, 0)); + s.commit(0, p1, seq); + assert_eq!(s.reads[0].backfill_begin, Clock::from_unix(30, 0)); + assert_eq!(s.reads[0].backfill_complete, Clock::from_unix(10, 0)); + } + /// Build a passthrough PartitionFilter that accepts any value for the given fields. /// The field count must match the partition field segments in the journal suffix. fn passthrough_filter(fields: &[&str]) -> PartitionFilter { @@ -1543,7 +1725,7 @@ mod test { ), ]; - let journal_acks = publisher::intents::build_transaction_intents(&txn); + let journal_acks = publisher::intents::build_transaction_intents(&txn, None); // For each journal's first ACK, extract hints into causal_hints. // build_transaction_intents returns NDJSON bytes per journal; diff --git a/crates/shuffle/tests/scenario_fixtures.rs b/crates/shuffle/tests/scenario_fixtures.rs index 0fd46fd67b7..dbd45122b17 100644 --- a/crates/shuffle/tests/scenario_fixtures.rs +++ b/crates/shuffle/tests/scenario_fixtures.rs @@ -330,6 +330,36 @@ async fn shuffle_scenarios() { .await; data_plane.reset().await.expect("reset"); + control_docs_are_metadata_only( + &materialization_spec, + &capture_spec, + &data_plane.journal_client, + &service, + log_dir.path(), + ) + .await; + data_plane.reset().await.expect("reset"); + + control_docs_reach_all_partitions( + &materialization_spec, + &capture_spec, + &data_plane.journal_client, + &service, + log_dir.path(), + ) + .await; + data_plane.reset().await.expect("reset"); + + resume_with_backfill_metadata( + &materialization_spec, + &capture_spec, + &data_plane.journal_client, + &service, + log_dir.path(), + ) + .await; + data_plane.reset().await.expect("reset"); + gapped_replay( &materialization_spec, &capture_spec, @@ -488,7 +518,7 @@ async fn continue_then_ack( // Commit the transaction. let (producer, commit_clock, journals) = pub_.commit_intents(); let journal_acks = - publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)]); + publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)], None); pub_.write_intents(journal_acks).await.unwrap(); let mut session = shuffle::SessionClient::open( @@ -659,7 +689,7 @@ async fn multiple_producers( // Now commit P2's transaction. let (producer, commit_clock, journals) = pub2.commit_intents(); let journal_acks = - publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)]); + publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)], None); pub2.write_intents(journal_acks).await.unwrap(); // Second checkpoint: P2 now committed. Reader yields P2's doc. @@ -774,8 +804,10 @@ async fn resume_from_checkpoint( // Commit with ACK spanning both journals. let (producer_id, commit_clock, journals) = pub_.commit_intents(); - let journal_acks = - publisher::intents::build_transaction_intents(&[(producer_id, commit_clock, journals)]); + let journal_acks = publisher::intents::build_transaction_intents( + &[(producer_id, commit_clock, journals)], + None, + ); pub_.write_intents(journal_acks).await.unwrap(); let mut session = shuffle::SessionClient::open( @@ -929,7 +961,7 @@ async fn multi_partition_transaction( // Commit with ACK intents spanning both journals. let (producer, commit_clock, journals) = pub_.commit_intents(); let journal_acks = - publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)]); + publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)], None); pub_.write_intents(journal_acks).await.unwrap(); let mut session = shuffle::SessionClient::open( @@ -1031,7 +1063,7 @@ async fn partition_filtered_hints( // Commit with ACK intents spanning all three partition journals. let (producer, commit_clock, journals) = pub_.commit_intents(); let journal_acks = - publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)]); + publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)], None); pub_.write_intents(journal_acks).await.unwrap(); let mut session = shuffle::SessionClient::open( @@ -1128,7 +1160,7 @@ async fn clock_window_filtering( // Commit with ACK intents spanning both journals. let (producer, commit_clock, journals) = pub_.commit_intents(); let journal_acks = - publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)]); + publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)], None); pub_.write_intents(journal_acks).await.unwrap(); let mut session = shuffle::SessionClient::open( @@ -1235,11 +1267,10 @@ async fn rollback( .unwrap(); let (p2_id, commit_clock_p2, p2_journals) = pub2.commit_intents(); - let p2_acks = publisher::intents::build_transaction_intents(&[( - p2_id, - commit_clock_p2, - p2_journals.clone(), - )]); + let p2_acks = publisher::intents::build_transaction_intents( + &[(p2_id, commit_clock_p2, p2_journals.clone())], + None, + ); pub2.write_intents(p2_acks).await.unwrap(); // P1 commits OUTSIDE_TXN docs. @@ -1519,7 +1550,8 @@ async fn gapped_replay( // (its first newer document), triggers a replay of [F, ack.begin), and on // completion re-presents the ACK to commit and deliver P2's span. let (producer, commit_clock, journals) = pub2.commit_intents(); - let acks = publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)]); + let acks = + publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)], None); pub2.write_intents(acks).await.unwrap(); let frontier2 = next_resolved_checkpoint(&mut resumed, "gapped replay").await; @@ -1654,7 +1686,8 @@ async fn gapped_continue_trigger( // P2 commits. Its ACK is read by the (now un-gapped) main read after the // replay completes, committing both of P2's CONTINUE documents together. let (producer, commit_clock, journals) = pub2.commit_intents(); - let acks = publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)]); + let acks = + publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)], None); pub2.write_intents(acks).await.unwrap(); let frontier2 = next_resolved_checkpoint(&mut resumed, "gapped continue replay").await; @@ -1769,7 +1802,8 @@ async fn gapped_replay_blocks_other_journal( // Slice: it is ready while the apples replay is in flight, and the blocking // gate must hold it until the replay completes, then let it flow. let (producer, commit_clock, journals) = pub2.commit_intents(); - let acks = publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)]); + let acks = + publisher::intents::build_transaction_intents(&[(producer, commit_clock, journals)], None); pub2.write_intents(acks).await.unwrap(); pub3.enqueue( @@ -1995,15 +2029,18 @@ async fn hint_elevated_offset_flip( // P2 commits: its ACK's end offset M becomes apples' maximum offset (M > O). let (p2_id, p2_commit, p2_journals) = pub2.commit_intents(); - let p2_acks = publisher::intents::build_transaction_intents(&[(p2_id, p2_commit, p2_journals)]); + let p2_acks = + publisher::intents::build_transaction_intents(&[(p2_id, p2_commit, p2_journals)], None); pub2.write_intents(p2_acks).await.unwrap(); // T1 commits at H1 — but only bananas' ACK is written (P1 "crashed" before // writing apples'). The apples read can never observe T1's commit directly; // only the causal hint (apples, P1) @ H1 from bananas' ACK covers the span. let (p1_id, t1_commit, t1_journals) = pub1.commit_intents(); - let mut t1_acks = - publisher::intents::build_transaction_intents(&[(p1_id, t1_commit, t1_journals.clone())]); + let mut t1_acks = publisher::intents::build_transaction_intents( + &[(p1_id, t1_commit, t1_journals.clone())], + None, + ); t1_acks.retain(|journal, _| journal.contains("bananas")); assert_eq!(t1_acks.len(), 1, "T1 spans apples and bananas"); pub1.write_intents(t1_acks).await.unwrap(); @@ -2083,8 +2120,10 @@ async fn hint_elevated_offset_flip( pub1.flush().await.unwrap(); let (_, t2_commit, _) = pub1.commit_intents(); - let mut t2_acks = - publisher::intents::build_transaction_intents(&[(p1_id, t2_commit, t1_journals.clone())]); + let mut t2_acks = publisher::intents::build_transaction_intents( + &[(p1_id, t2_commit, t1_journals.clone())], + None, + ); let ack_b2: Vec<(String, bytes::Bytes)> = t2_acks .iter() .filter(|(journal, _)| journal.contains("bananas")) @@ -2291,3 +2330,378 @@ async fn gapped_outside_violation( ); // The session tore down on error; there is no clean close to perform. } + +/// Broadcast a standalone backfill marker as ACK intents across `journals` +/// (models `marker_commit` + `build_transaction_intents` + `write_intents`). +/// `clock` must exceed each journal's last committed clock so the ACK sequences +/// as `AckEmpty`; the marker folds into the frontier as metadata, never a doc. +async fn broadcast_marker( + pub_: &mut publisher::Publisher, + producer: uuid::Producer, + clock: uuid::Clock, + journals: &[String], + marker: &publisher::intents::BackfillMarker, +) { + let acks = publisher::intents::build_transaction_intents( + &[(producer, clock, journals.to_vec())], + Some(marker), + ); + pub_.write_intents(acks).await.unwrap(); +} + +/// Publish backfill marker ACKs and a regular document. Verify the markers fold +/// into checkpoint metadata but never appear as read documents (they ride on +/// ACKs, which are never appended to shuffle logs). +async fn control_docs_are_metadata_only( + materialization_spec: &flow::MaterializationSpec, + capture_spec: &flow::CaptureSpec, + journal_client: &gazette::journal::Client, + service: &shuffle::Service, + log_dir: &std::path::Path, +) { + let scenario_dir = log_dir.join("control_docs_are_metadata_only"); + std::fs::create_dir_all(&scenario_dir).unwrap(); + + let producer = uuid::Producer::from_bytes([0x01, 0x00, 0x00, 0x00, 0x00, 0x11]); + let mut pub_ = make_publisher(capture_spec, journal_client, producer); + + // Write one ordinary data document and commit it. This establishes the + // collection's physical partition (so a marker broadcast can reach it) and + // yields the partition journal name for the broadcast. + pub_.enqueue( + |uuid| { + Ok(( + 0, + serde_json::json!({ + "_meta": {"uuid": uuid.to_string()}, + "id": "data", + "category": "alpha", + "value": 7, + }), + )) + }, + uuid::Flags::CONTINUE_TXN, + ) + .await + .unwrap(); + let (producer, data_commit, journals) = pub_.commit_intents(); + let partitions = journals.clone(); + let data_acks = + publisher::intents::build_transaction_intents(&[(producer, data_commit, journals)], None); + pub_.write_intents(data_acks).await.unwrap(); + + // Broadcast a standalone BackfillBegin marker ACK. `commit_intents` ticks a + // fresh clock past the data commit; that clock is the truncation boundary. + let (producer, begin_clock, _) = pub_.commit_intents(); + broadcast_marker( + &mut pub_, + producer, + begin_clock, + &partitions, + &publisher::intents::BackfillMarker::Begin, + ) + .await; + + // Broadcast a standalone BackfillComplete marker ACK carrying the begin + // clock as its `truncated_at` boundary. + let (producer, complete_clock, _) = pub_.commit_intents(); + broadcast_marker( + &mut pub_, + producer, + complete_clock, + &partitions, + &publisher::intents::BackfillMarker::Complete { + truncated_at: begin_clock.as_u64(), + }, + ) + .await; + + let mut session = shuffle::SessionClient::open( + service, + build_task(materialization_spec), + build_shards(1, service.peer_endpoint(), &scenario_dir), + Default::default(), + ) + .await + .expect("SessionClient::open"); + + // The begin and complete markers ride separate ACKs and may land in separate + // checkpoint flushes; aggregate across checkpoints until both are visible. + let binding_0: u16 = 0; + let mut frontier = session.next_checkpoint().await.expect("next_checkpoint"); + let mut shard_state: ShardState = (0..1).map(|_| None).collect(); + let mut read = collect_read_entries(&frontier, &scenario_dir, &mut shard_state); + while frontier.latest_backfill_begin.get(&binding_0) != Some(&begin_clock) + || frontier.latest_backfill_complete.get(&binding_0) != Some(&begin_clock) + { + let next = session.next_checkpoint().await.expect("next_checkpoint"); + read.extend(collect_read_entries(&next, &scenario_dir, &mut shard_state)); + frontier = frontier.reduce(next); + } + assert_eq!( + frontier.latest_backfill_begin.get(&binding_0), + Some(&begin_clock) + ); + assert_eq!( + frontier.latest_backfill_complete.get(&binding_0), + Some(&begin_clock) + ); + + insta::assert_debug_snapshot!( + "control_docs_are_metadata_only", + Checkpoint { + frontier: &frontier, + read, + } + ); + + session.close().await.expect("close"); +} + +/// A backfill marker is broadcast as an ACK to every partition journal of the +/// collection, so a reader observes it regardless of its partition selector, +/// and its pairwise causal hints hold the checkpoint back until every *read* +/// journal's marker ACK is seen. +/// +/// Seed the alpha, beta, and gamma partitions, then broadcast a BackfillBegin +/// marker ACK to all three. The reader's selector excludes beta, yet it must +/// still observe the backfill begin via the included alpha/gamma partitions — +/// which only holds because the marker ACK reached those journals too. The +/// beta-directed hints are dropped by the reader's partition filter, so the +/// hold-back spans exactly the read subset (alpha + gamma). +async fn control_docs_reach_all_partitions( + materialization_spec: &flow::MaterializationSpec, + capture_spec: &flow::CaptureSpec, + journal_client: &gazette::journal::Client, + service: &shuffle::Service, + log_dir: &std::path::Path, +) { + let scenario_dir = log_dir.join("control_docs_reach_all_partitions"); + std::fs::create_dir_all(&scenario_dir).unwrap(); + + let producer = uuid::Producer::from_bytes([0x01, 0x00, 0x00, 0x00, 0x00, 0x08]); + let mut pub_ = make_publisher(capture_spec, journal_client, producer); + + // Seed one document into each of alpha/beta/gamma so all three physical + // partitions exist and are listable, then commit them. + for category in ["alpha", "beta", "gamma"] { + pub_.enqueue( + |uuid| { + Ok(( + 0, + serde_json::json!({ + "_meta": {"uuid": uuid.to_string()}, + "id": format!("seed-{category}"), + "category": category, + "value": 0, + }), + )) + }, + uuid::Flags::CONTINUE_TXN, + ) + .await + .unwrap(); + } + let (producer_id, commit_clock, journals) = pub_.commit_intents(); + let all_partitions = journals.clone(); + let journal_acks = publisher::intents::build_transaction_intents( + &[(producer_id, commit_clock, journals)], + None, + ); + pub_.write_intents(journal_acks).await.unwrap(); + + // Broadcast a BackfillBegin marker ACK to every partition (including the + // excluded beta). `commit_intents` ticks a fresh clock past the seed commit; + // the marker's pairwise hints span all three journals. + let (producer_id, begin_clock, _) = pub_.commit_intents(); + broadcast_marker( + &mut pub_, + producer_id, + begin_clock, + &all_partitions, + &publisher::intents::BackfillMarker::Begin, + ) + .await; + + let mut session = shuffle::SessionClient::open( + service, + build_task(materialization_spec), + build_shards(1, service.peer_endpoint(), &scenario_dir), + Default::default(), + ) + .await + .expect("SessionClient::open"); + + // Aggregate checkpoints until the (terminal) frontier carries the backfill + // begin. The reader excludes beta, so observing it at all proves the control + // doc reached the included alpha/gamma partitions. + let binding_0: u16 = 0; + let mut frontier = next_resolved_checkpoint(&mut session, "next_checkpoint").await; + while frontier.latest_backfill_begin.get(&binding_0).is_none() { + let next = next_resolved_checkpoint(&mut session, "next_checkpoint").await; + frontier = frontier.reduce(next); + } + assert!( + frontier.latest_backfill_begin.get(&binding_0).is_some(), + "a selector that excludes beta still observed the broadcast backfill begin", + ); + + session.close().await.expect("close"); +} + +/// Verify that `latest_backfill_begin` and `latest_backfill_complete` survive +/// the checkpoint→wire→resume round-trip through the Session handler. +/// +/// Phase 1: Publish control docs + data in a committed transaction. Capture +/// a checkpoint whose frontier carries non-empty backfill maps. +/// Phase 2: Reopen a new session using that frontier as the resume +/// checkpoint. Write additional data and poll a checkpoint. The resumed +/// session must have accepted the backfill metadata from the resume +/// frontier — the session would error if the resume checkpoint were +/// malformed, and new progress comes back correctly layered on top. +async fn resume_with_backfill_metadata( + materialization_spec: &flow::MaterializationSpec, + capture_spec: &flow::CaptureSpec, + journal_client: &gazette::journal::Client, + service: &shuffle::Service, + log_dir: &std::path::Path, +) { + let phase1_dir = log_dir.join("resume_backfill_p1"); + let phase2_dir = log_dir.join("resume_backfill_p2"); + std::fs::create_dir_all(&phase1_dir).unwrap(); + std::fs::create_dir_all(&phase2_dir).unwrap(); + + let producer = uuid::Producer::from_bytes([0x01, 0x00, 0x00, 0x00, 0x00, 0x21]); + let mut pub_ = make_publisher(capture_spec, journal_client, producer); + + // ---- Phase 1: Commit data + marker ACKs, capture a checkpoint. ---- + + // Write a data document and commit it, establishing the partition and + // yielding its journal name for the marker broadcast. + pub_.enqueue( + |uuid| { + Ok(( + 0, + serde_json::json!({ + "_meta": {"uuid": uuid.to_string()}, + "id": "rb-data", + "category": "alpha", + "value": 42, + }), + )) + }, + uuid::Flags::CONTINUE_TXN, + ) + .await + .unwrap(); + let (producer_id, commit_clock, journals) = pub_.commit_intents(); + let partitions = journals.clone(); + let journal_acks = publisher::intents::build_transaction_intents( + &[(producer_id, commit_clock, journals)], + None, + ); + pub_.write_intents(journal_acks).await.unwrap(); + + // Broadcast a standalone BackfillBegin marker ACK; its clock is the boundary. + let (producer_id, begin_clock, _) = pub_.commit_intents(); + broadcast_marker( + &mut pub_, + producer_id, + begin_clock, + &partitions, + &publisher::intents::BackfillMarker::Begin, + ) + .await; + + // Broadcast a standalone BackfillComplete marker ACK carrying the begin clock. + let (producer_id, complete_clock, _) = pub_.commit_intents(); + broadcast_marker( + &mut pub_, + producer_id, + complete_clock, + &partitions, + &publisher::intents::BackfillMarker::Complete { + truncated_at: begin_clock.as_u64(), + }, + ) + .await; + + let mut session = shuffle::SessionClient::open( + service, + build_task(materialization_spec), + build_shards(1, service.peer_endpoint(), &phase1_dir), + Default::default(), + ) + .await + .expect("SessionClient::open phase 1"); + + // Aggregate checkpoint deltas until both the backfill begin clock (from + // the OUTSIDE_TXN BackfillBegin commit) and the backfill complete clock + // (from the OUTSIDE_TXN BackfillComplete commit after the span's ACK) are + // visible. The two control docs commit in separate flush cycles. + let binding_0: u16 = 0; + let mut phase1_frontier = session.next_checkpoint().await.expect("phase 1 checkpoint"); + while phase1_frontier.latest_backfill_begin.get(&binding_0) != Some(&begin_clock) + || phase1_frontier.latest_backfill_complete.get(&binding_0) != Some(&begin_clock) + { + let next = session + .next_checkpoint() + .await + .expect("phase 1 next checkpoint"); + phase1_frontier = phase1_frontier.reduce(next); + } + assert_eq!( + phase1_frontier.latest_backfill_begin.get(&binding_0), + Some(&begin_clock), + "phase 1 frontier should carry backfill begin" + ); + assert_eq!( + phase1_frontier.latest_backfill_complete.get(&binding_0), + Some(&begin_clock), + "phase 1 frontier should carry the backfill begin clock as its truncation boundary" + ); + session.close().await.expect("close phase 1"); + + // ---- Phase 2: Resume from phase1_frontier, write more data. ---- + + pub_.enqueue( + |uuid| { + Ok(( + 0, + serde_json::json!({ + "_meta": {"uuid": uuid.to_string()}, + "id": "rb-new", + "category": "alpha", + "value": 99, + }), + )) + }, + uuid::Flags::OUTSIDE_TXN, + ) + .await + .unwrap(); + pub_.flush().await.unwrap(); + + let mut session = shuffle::SessionClient::open( + service, + build_task(materialization_spec), + build_shards(1, service.peer_endpoint(), &phase2_dir), + phase1_frontier.clone(), + ) + .await + .expect("SessionClient::open phase 2 (resume with backfill metadata)"); + + let phase2_frontier = next_resolved_checkpoint(&mut session, "phase 2 checkpoint").await; + + let mut phase2_shard_state: ShardState = (0..1).map(|_| None).collect(); + let phase2_read = collect_read_entries(&phase2_frontier, &phase2_dir, &mut phase2_shard_state); + insta::assert_debug_snapshot!( + "resume_with_backfill_metadata", + Checkpoint { + frontier: &phase2_frontier, + read: phase2_read, + } + ); + + session.close().await.expect("close phase 2"); +} diff --git a/crates/shuffle/tests/scenario_fuzz.rs b/crates/shuffle/tests/scenario_fuzz.rs index f70ddc74d8b..c902343c97c 100644 --- a/crates/shuffle/tests/scenario_fuzz.rs +++ b/crates/shuffle/tests/scenario_fuzz.rs @@ -557,11 +557,10 @@ async fn write_actions( } state.publisher.flush().await.unwrap(); let (producer_id, commit_clock, journals) = state.publisher.commit_intents(); - let intents = publisher::intents::build_transaction_intents(&[( - producer_id, - commit_clock, - journals, - )]); + let intents = publisher::intents::build_transaction_intents( + &[(producer_id, commit_clock, journals)], + None, + ); for (journal, _) in &intents { state .journal_committed_clocks @@ -601,11 +600,10 @@ async fn write_actions( Action::CommitOpen => { // ACK the previously-opened span. No new CONTINUEs. let (producer_id, commit_clock, journals) = state.publisher.commit_intents(); - let intents = publisher::intents::build_transaction_intents(&[( - producer_id, - commit_clock, - journals, - )]); + let intents = publisher::intents::build_transaction_intents( + &[(producer_id, commit_clock, journals)], + None, + ); for (journal, _) in &intents { state .journal_committed_clocks @@ -791,6 +789,7 @@ fn project_hints( unresolved_hints, journals, flushed_lsn: vec![], + ..Default::default() } } @@ -1038,6 +1037,7 @@ async fn run_test_case_inner( journals: vec![], flushed_lsn: recovery.flushed_lsn.clone(), unresolved_hints: 0, + ..Default::default() }; // STEP 3: POLL CHECKPOINTS. @@ -1133,6 +1133,7 @@ async fn run_test_case_inner( journals: vec![], flushed_lsn: recovery.flushed_lsn.clone(), unresolved_hints: 0, + ..Default::default() }; if !commit_clocks.is_empty() || recovery.unresolved_hints != 0 { diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__clock_window_filtering.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__clock_window_filtering.snap index 262f6eb29b5..cdb60f9b150 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__clock_window_filtering.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__clock_window_filtering.snap @@ -37,6 +37,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__continue_then_ack.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__continue_then_ack.snap index da2035582b5..e51feec44e2 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__continue_then_ack.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__continue_then_ack.snap @@ -23,6 +23,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__control_docs_are_metadata_only.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__control_docs_are_metadata_only.snap new file mode 100644 index 00000000000..287632fbce1 --- /dev/null +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__control_docs_are_metadata_only.snap @@ -0,0 +1,48 @@ +--- +source: crates/shuffle/tests/scenario_fixtures.rs +expression: "Checkpoint { frontier: &frontier, read, }" +--- +Checkpoint { + frontier: Frontier { + journals: [ + JournalFrontier { + journal: "testing/apples/2020202020202020/category=alpha/pivot=00", + binding: 0, + producers: [ + ProducerFrontier { + producer: Producer(01:00:00:00:00:11), + last_commit: Clock(0s 4000ns), + hinted_commit: Clock(0s 0ns), + offset: -426, + }, + ], + bytes_read_delta: 426, + bytes_behind_delta: 0, + }, + ], + flushed_lsn: [ + 1/0, + ], + latest_backfill_begin: { + 0: Clock(0s 3000ns), + }, + latest_backfill_complete: { + 0: Clock(0s 3000ns), + }, + unresolved_hints: 0, + }, + read: [ + ReadEntry { + binding: 0, + journal: "testing/apples/2020202020202020/category=alpha/pivot=00", + doc: Object { + "_meta": Object { + "uuid": String("1381400a-1dd2-11b2-8001-010000000011"), + }, + "category": String("alpha"), + "id": String("data"), + "value": Number(7), + }, + }, + ], +} diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__gapped_continue_trigger_resumed.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__gapped_continue_trigger_resumed.snap index 8ec19dbce16..b0be39df169 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__gapped_continue_trigger_resumed.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__gapped_continue_trigger_resumed.snap @@ -23,6 +23,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__gapped_replay_checkpoint1.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__gapped_replay_checkpoint1.snap index a1d1d0cb85f..6ce35cc05e9 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__gapped_replay_checkpoint1.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__gapped_replay_checkpoint1.snap @@ -1,6 +1,5 @@ --- source: crates/shuffle/tests/scenario_fixtures.rs -assertion_line: 1489 expression: "Checkpoint { frontier: &frontier1, read: read1, }" --- Checkpoint { @@ -30,6 +29,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__gapped_replay_resumed.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__gapped_replay_resumed.snap index 46d324e4214..4a10f723abd 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__gapped_replay_resumed.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__gapped_replay_resumed.snap @@ -23,6 +23,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__hint_elevated_offset_checkpoint1.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__hint_elevated_offset_checkpoint1.snap index d067917d6b5..29f2371c395 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__hint_elevated_offset_checkpoint1.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__hint_elevated_offset_checkpoint1.snap @@ -1,6 +1,5 @@ --- source: crates/shuffle/tests/scenario_fixtures.rs -assertion_line: 2056 expression: "Checkpoint { frontier: &scrub_measures(&base), read: read1, }" --- Checkpoint { @@ -42,6 +41,8 @@ Checkpoint { }, ], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 1, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__hint_elevated_offset_checkpoint2.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__hint_elevated_offset_checkpoint2.snap index 777926aa7cb..2f898c84fbc 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__hint_elevated_offset_checkpoint2.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__hint_elevated_offset_checkpoint2.snap @@ -1,6 +1,5 @@ --- source: crates/shuffle/tests/scenario_fixtures.rs -assertion_line: 2126 expression: "Checkpoint { frontier: &scrub_measures(&cp2), read: read2, }" --- Checkpoint { @@ -42,6 +41,8 @@ Checkpoint { }, ], flushed_lsn: [], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__hint_elevated_offset_resumed.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__hint_elevated_offset_resumed.snap index 6edcc53eca4..92a653f0e12 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__hint_elevated_offset_resumed.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__hint_elevated_offset_resumed.snap @@ -1,6 +1,5 @@ --- source: crates/shuffle/tests/scenario_fixtures.rs -assertion_line: 2163 expression: "Checkpoint { frontier: &frontier3, read: read3, }" --- Checkpoint { @@ -38,6 +37,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__multi_partition_transaction.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__multi_partition_transaction.snap index 23ba62716d5..067cabb1f4d 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__multi_partition_transaction.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__multi_partition_transaction.snap @@ -37,6 +37,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__multi_shard_routing.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__multi_shard_routing.snap index eac11e31fd2..f2151fbc29c 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__multi_shard_routing.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__multi_shard_routing.snap @@ -25,6 +25,8 @@ Checkpoint { 1/0, 0/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__multiple_producers_checkpoint1.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__multiple_producers_checkpoint1.snap index a8cf34a119f..70d2f6c5037 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__multiple_producers_checkpoint1.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__multiple_producers_checkpoint1.snap @@ -30,6 +30,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__multiple_producers_checkpoint2.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__multiple_producers_checkpoint2.snap index 3ea5c9e2b5d..057eacb50d4 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__multiple_producers_checkpoint2.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__multiple_producers_checkpoint2.snap @@ -23,6 +23,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__multiple_producers_resumed.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__multiple_producers_resumed.snap index 9cac1255831..5695e886d8f 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__multiple_producers_resumed.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__multiple_producers_resumed.snap @@ -23,6 +23,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__partition_filtered_hints.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__partition_filtered_hints.snap index 52889a25480..0501e729535 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__partition_filtered_hints.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__partition_filtered_hints.snap @@ -37,6 +37,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__resume_from_checkpoint_progress.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__resume_from_checkpoint_progress.snap index 0d37b29f9cb..92974fe3355 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__resume_from_checkpoint_progress.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__resume_from_checkpoint_progress.snap @@ -37,6 +37,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__resume_from_checkpoint_recovery.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__resume_from_checkpoint_recovery.snap index 5d1b7d54303..68aea96e1d6 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__resume_from_checkpoint_recovery.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__resume_from_checkpoint_recovery.snap @@ -37,6 +37,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__resume_with_backfill_metadata.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__resume_with_backfill_metadata.snap new file mode 100644 index 00000000000..564061f77f9 --- /dev/null +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__resume_with_backfill_metadata.snap @@ -0,0 +1,44 @@ +--- +source: crates/shuffle/tests/scenario_fixtures.rs +expression: "Checkpoint { frontier: &phase2_frontier, read: phase2_read, }" +--- +Checkpoint { + frontier: Frontier { + journals: [ + JournalFrontier { + journal: "testing/apples/2020202020202020/category=alpha/pivot=00", + binding: 0, + producers: [ + ProducerFrontier { + producer: Producer(01:00:00:00:00:21), + last_commit: Clock(0s 5000ns), + hinted_commit: Clock(0s 0ns), + offset: -532, + }, + ], + bytes_read_delta: 102, + bytes_behind_delta: 0, + }, + ], + flushed_lsn: [ + 1/0, + ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, + unresolved_hints: 0, + }, + read: [ + ReadEntry { + binding: 0, + journal: "testing/apples/2020202020202020/category=alpha/pivot=00", + doc: Object { + "_meta": Object { + "uuid": String("13814032-1dd2-11b2-8000-010000000021"), + }, + "category": String("alpha"), + "id": String("rb-new"), + "value": Number(99), + }, + }, + ], +} diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase1_committed.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase1_committed.snap index 9cfe9f52df1..0dff3e4c092 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase1_committed.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase1_committed.snap @@ -29,6 +29,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase2_clean_rollback.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase2_clean_rollback.snap index 6ae9dfa411e..b4447a0891e 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase2_clean_rollback.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase2_clean_rollback.snap @@ -23,6 +23,8 @@ Checkpoint { flushed_lsn: [ 1/1, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [], diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase3_p1_continues.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase3_p1_continues.snap index 41ec7867f23..e9b83deecd0 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase3_p1_continues.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase3_p1_continues.snap @@ -23,6 +23,8 @@ Checkpoint { flushed_lsn: [ 1/2, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase4_deep_rollback.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase4_deep_rollback.snap index 17d80fd3bac..b933c4a3698 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase4_deep_rollback.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__rollback_phase4_deep_rollback.snap @@ -23,6 +23,8 @@ Checkpoint { flushed_lsn: [ 1/3, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [], diff --git a/crates/shuffle/tests/snapshots/scenario_fixtures__single_producer_outside_txn.snap b/crates/shuffle/tests/snapshots/scenario_fixtures__single_producer_outside_txn.snap index e4957eb6325..25ecbb31504 100644 --- a/crates/shuffle/tests/snapshots/scenario_fixtures__single_producer_outside_txn.snap +++ b/crates/shuffle/tests/snapshots/scenario_fixtures__single_producer_outside_txn.snap @@ -23,6 +23,8 @@ Checkpoint { flushed_lsn: [ 1/0, ], + latest_backfill_begin: {}, + latest_backfill_complete: {}, unresolved_hints: 0, }, read: [ diff --git a/go/labels/labels.go b/go/labels/labels.go index 8120a0f2275..17cd71a1360 100644 --- a/go/labels/labels.go +++ b/go/labels/labels.go @@ -36,6 +36,13 @@ const ( KeyEndMax = "ffffffff" // ManagedByFlow is a value for the Gazette labels.ManagedBy label. ManagedByFlow = "estuary.dev/flow" + // TruncatedAt is the publication timestamp after which journal data is current, + // determined by the last backfill-begin timestamp of the collection's capture. + // + // Its value is a fixed-width, 16-character hex encoding of a uint64 Gazette + // message.Clock. A reader raises its effective not_before to this clock to + // skip the stale pre-backfill prefix. + TruncatedAt = "estuary.dev/truncated-at" ) // ShardSpec labels. @@ -112,7 +119,10 @@ func IsRuntimeLabel(label string) bool { // R-Clock splits are performed dynamically by the runtime. RClockBegin, RClockEnd, // Shard splits are performed dynamically by the runtime. - SplitTarget, SplitSource: + SplitTarget, SplitSource, + // The backfill truncation boundary is applied to live journals by the + // capture runtime, so convergence must preserve it (not rebuild it away). + TruncatedAt: return true default: return false diff --git a/go/labels/labels_test.go b/go/labels/labels_test.go index a431a600ba1..508f924a690 100644 --- a/go/labels/labels_test.go +++ b/go/labels/labels_test.go @@ -24,6 +24,7 @@ func TestRuntimeLabels(t *testing.T) { {RClockEnd, true}, {SplitSource, true}, {SplitTarget, true}, + {TruncatedAt, true}, {TaskName, false}, {TaskType, false}, {labels.ContentType, false}, diff --git a/go/protocols/capture/capture.pb.go b/go/protocols/capture/capture.pb.go index 831139d80df..0449de01214 100644 --- a/go/protocols/capture/capture.pb.go +++ b/go/protocols/capture/capture.pb.go @@ -446,14 +446,16 @@ func (m *Request_Acknowledge) XXX_DiscardUnknown() { var xxx_messageInfo_Request_Acknowledge proto.InternalMessageInfo type Response struct { - Spec *Response_Spec `protobuf:"bytes,1,opt,name=spec,proto3" json:"spec,omitempty"` - Discovered *Response_Discovered `protobuf:"bytes,2,opt,name=discovered,proto3" json:"discovered,omitempty"` - Validated *Response_Validated `protobuf:"bytes,3,opt,name=validated,proto3" json:"validated,omitempty"` - Applied *Response_Applied `protobuf:"bytes,4,opt,name=applied,proto3" json:"applied,omitempty"` - Opened *Response_Opened `protobuf:"bytes,5,opt,name=opened,proto3" json:"opened,omitempty"` - Captured *Response_Captured `protobuf:"bytes,6,opt,name=captured,proto3" json:"captured,omitempty"` - SourcedSchema *Response_SourcedSchema `protobuf:"bytes,8,opt,name=sourced_schema,json=sourcedSchema,proto3" json:"sourced_schema,omitempty"` - Checkpoint *Response_Checkpoint `protobuf:"bytes,7,opt,name=checkpoint,proto3" json:"checkpoint,omitempty"` + Spec *Response_Spec `protobuf:"bytes,1,opt,name=spec,proto3" json:"spec,omitempty"` + Discovered *Response_Discovered `protobuf:"bytes,2,opt,name=discovered,proto3" json:"discovered,omitempty"` + Validated *Response_Validated `protobuf:"bytes,3,opt,name=validated,proto3" json:"validated,omitempty"` + Applied *Response_Applied `protobuf:"bytes,4,opt,name=applied,proto3" json:"applied,omitempty"` + Opened *Response_Opened `protobuf:"bytes,5,opt,name=opened,proto3" json:"opened,omitempty"` + Captured *Response_Captured `protobuf:"bytes,6,opt,name=captured,proto3" json:"captured,omitempty"` + SourcedSchema *Response_SourcedSchema `protobuf:"bytes,8,opt,name=sourced_schema,json=sourcedSchema,proto3" json:"sourced_schema,omitempty"` + Checkpoint *Response_Checkpoint `protobuf:"bytes,7,opt,name=checkpoint,proto3" json:"checkpoint,omitempty"` + BackfillBegin *Response_BackfillBegin `protobuf:"bytes,9,opt,name=backfill_begin,json=backfillBegin,proto3" json:"backfill_begin,omitempty"` + BackfillComplete *Response_BackfillComplete `protobuf:"bytes,10,opt,name=backfill_complete,json=backfillComplete,proto3" json:"backfill_complete,omitempty"` // Reserved for internal use. Internal []byte `protobuf:"bytes,100,opt,name=internal,json=$internal,proto3" json:"internal,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1004,6 +1006,96 @@ func (m *Response_Checkpoint) XXX_DiscardUnknown() { var xxx_messageInfo_Response_Checkpoint proto.InternalMessageInfo +// Signals the start of a backfill for a binding. +// +// A backfill message (BackfillBegin or BackfillComplete) must stand alone +// in its connector checkpoint: the checkpoint must contain only the +// backfill message followed by the terminating Checkpoint response, with +// no Captured, SourcedSchema, or other backfill messages. The runtime +// enforces this rule and will fail the session on violation. +type Response_BackfillBegin struct { + Binding uint32 `protobuf:"varint,1,opt,name=binding,proto3" json:"binding,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Response_BackfillBegin) Reset() { *m = Response_BackfillBegin{} } +func (m *Response_BackfillBegin) String() string { return proto.CompactTextString(m) } +func (*Response_BackfillBegin) ProtoMessage() {} +func (*Response_BackfillBegin) Descriptor() ([]byte, []int) { + return fileDescriptor_841a70e6e6288f13, []int{1, 8} +} +func (m *Response_BackfillBegin) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Response_BackfillBegin) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Response_BackfillBegin.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Response_BackfillBegin) XXX_Merge(src proto.Message) { + xxx_messageInfo_Response_BackfillBegin.Merge(m, src) +} +func (m *Response_BackfillBegin) XXX_Size() int { + return m.ProtoSize() +} +func (m *Response_BackfillBegin) XXX_DiscardUnknown() { + xxx_messageInfo_Response_BackfillBegin.DiscardUnknown(m) +} + +var xxx_messageInfo_Response_BackfillBegin proto.InternalMessageInfo + +// Signals the end of a backfill for a binding. +// +// See BackfillBegin for the "stands alone in its checkpoint" rule. +type Response_BackfillComplete struct { + Binding uint32 `protobuf:"varint,1,opt,name=binding,proto3" json:"binding,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Response_BackfillComplete) Reset() { *m = Response_BackfillComplete{} } +func (m *Response_BackfillComplete) String() string { return proto.CompactTextString(m) } +func (*Response_BackfillComplete) ProtoMessage() {} +func (*Response_BackfillComplete) Descriptor() ([]byte, []int) { + return fileDescriptor_841a70e6e6288f13, []int{1, 9} +} +func (m *Response_BackfillComplete) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Response_BackfillComplete) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Response_BackfillComplete.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Response_BackfillComplete) XXX_Merge(src proto.Message) { + xxx_messageInfo_Response_BackfillComplete.Merge(m, src) +} +func (m *Response_BackfillComplete) XXX_Size() int { + return m.ProtoSize() +} +func (m *Response_BackfillComplete) XXX_DiscardUnknown() { + xxx_messageInfo_Response_BackfillComplete.DiscardUnknown(m) +} + +var xxx_messageInfo_Response_BackfillComplete proto.InternalMessageInfo + func init() { proto.RegisterType((*Request)(nil), "capture.Request") proto.RegisterType((*Request_Spec)(nil), "capture.Request.Spec") @@ -1024,6 +1116,8 @@ func init() { proto.RegisterType((*Response_Captured)(nil), "capture.Response.Captured") proto.RegisterType((*Response_SourcedSchema)(nil), "capture.Response.SourcedSchema") proto.RegisterType((*Response_Checkpoint)(nil), "capture.Response.Checkpoint") + proto.RegisterType((*Response_BackfillBegin)(nil), "capture.Response.BackfillBegin") + proto.RegisterType((*Response_BackfillComplete)(nil), "capture.Response.BackfillComplete") } func init() { @@ -1031,88 +1125,92 @@ func init() { } var fileDescriptor_841a70e6e6288f13 = []byte{ - // 1289 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x56, 0xcb, 0x6e, 0x1c, 0x45, - 0x17, 0x4e, 0x7b, 0x6e, 0x3d, 0x67, 0x66, 0x1c, 0xbb, 0xe4, 0x3f, 0xea, 0x74, 0xfc, 0xdb, 0xce, - 0x05, 0xe4, 0x5c, 0x18, 0x87, 0x49, 0x02, 0x84, 0x4b, 0x82, 0xed, 0x10, 0x89, 0x20, 0x48, 0x54, - 0x81, 0x20, 0xd8, 0xb4, 0xca, 0xd5, 0xe5, 0x71, 0xe3, 0x9e, 0xae, 0xa6, 0xbb, 0x27, 0x61, 0x24, - 0xf6, 0x88, 0x05, 0x5b, 0xd6, 0x79, 0x01, 0x1e, 0x03, 0x29, 0x0b, 0x16, 0x3c, 0x41, 0x04, 0xe1, - 0x19, 0x10, 0x52, 0x56, 0xa8, 0x6e, 0x3d, 0x3d, 0xd3, 0xb1, 0x99, 0x44, 0x41, 0x62, 0x63, 0x4f, - 0x9d, 0xf3, 0x9d, 0xea, 0x3a, 0x97, 0xfa, 0xbe, 0x82, 0x53, 0x7d, 0xbe, 0x11, 0x27, 0x3c, 0xe3, - 0x94, 0x87, 0xe9, 0x06, 0x25, 0x71, 0x36, 0x4c, 0x98, 0xf9, 0xdf, 0x95, 0x1e, 0xd4, 0xd0, 0x4b, - 0x77, 0x79, 0x02, 0xbc, 0x1b, 0xf2, 0x07, 0xf2, 0x8f, 0x82, 0xb9, 0x4b, 0x7d, 0xde, 0xe7, 0xf2, - 0xe7, 0x86, 0xf8, 0xa5, 0xac, 0xa7, 0x7e, 0x98, 0x87, 0x06, 0x66, 0x5f, 0x0f, 0x59, 0x9a, 0xa1, - 0xb3, 0x50, 0x4d, 0x63, 0x46, 0x1d, 0x6b, 0xcd, 0x5a, 0x6f, 0xf5, 0xfe, 0xd7, 0x35, 0x9f, 0xd1, - 0xfe, 0xee, 0xdd, 0x98, 0x51, 0x2c, 0x21, 0xe8, 0x0a, 0xd8, 0x7e, 0x90, 0x52, 0x7e, 0x9f, 0x25, - 0xce, 0x9c, 0x84, 0x1f, 0x2f, 0xc1, 0x6f, 0x68, 0x00, 0xce, 0xa1, 0x22, 0xec, 0x3e, 0x09, 0x03, - 0x9f, 0x64, 0xcc, 0xa9, 0x1c, 0x10, 0x76, 0x4f, 0x03, 0x70, 0x0e, 0x45, 0x17, 0xa0, 0x46, 0xe2, - 0x38, 0x1c, 0x39, 0x55, 0x19, 0x73, 0xac, 0x14, 0xb3, 0x29, 0xbc, 0x58, 0x81, 0x44, 0x1a, 0x3c, - 0x66, 0x91, 0x53, 0x3b, 0x20, 0x8d, 0xdb, 0x31, 0x8b, 0xb0, 0x84, 0xa0, 0x6b, 0xd0, 0x22, 0x74, - 0x3f, 0xe2, 0x0f, 0x42, 0xe6, 0xf7, 0x99, 0x53, 0x97, 0x11, 0xcb, 0xe5, 0xed, 0xc7, 0x18, 0x5c, - 0x0c, 0x40, 0x27, 0xc0, 0x0e, 0xa2, 0x8c, 0x25, 0x11, 0x09, 0x1d, 0x7f, 0xcd, 0x5a, 0x6f, 0xe3, - 0xe6, 0x19, 0x63, 0x70, 0xbf, 0xb7, 0xa0, 0x2a, 0x4a, 0x86, 0x6e, 0xc2, 0x3c, 0xe5, 0x51, 0xc4, - 0x68, 0xc6, 0x13, 0x2f, 0x1b, 0xc5, 0x4c, 0x56, 0x78, 0xbe, 0xb7, 0xda, 0x95, 0xed, 0xd9, 0x56, - 0x5f, 0x13, 0xd0, 0xee, 0xb6, 0xc1, 0x7d, 0x3a, 0x8a, 0x19, 0xee, 0xd0, 0xe2, 0x12, 0x5d, 0x85, - 0x16, 0xe5, 0xd1, 0x6e, 0xd0, 0xf7, 0xbe, 0x4a, 0x79, 0x24, 0xeb, 0xde, 0xde, 0x5a, 0x7e, 0xfa, - 0x78, 0xd5, 0x61, 0x11, 0xe5, 0x7e, 0x10, 0xf5, 0x37, 0x84, 0xa3, 0x8b, 0xc9, 0x83, 0x8f, 0x59, - 0x9a, 0x92, 0x3e, 0xc3, 0x75, 0x15, 0xe0, 0xfe, 0x65, 0x81, 0x6d, 0xfa, 0x81, 0x3e, 0x84, 0x6a, - 0x44, 0x06, 0xaa, 0x03, 0xcd, 0xad, 0x2b, 0x4f, 0x1f, 0xaf, 0xbe, 0xde, 0x0f, 0xb2, 0xbd, 0xe1, - 0x4e, 0x97, 0xf2, 0xc1, 0x06, 0x4b, 0xb3, 0x21, 0x49, 0x46, 0x6a, 0x7e, 0x4a, 0x13, 0x65, 0x4e, - 0x8b, 0xe5, 0x16, 0xff, 0x81, 0xd4, 0xd0, 0xff, 0x01, 0x68, 0xc2, 0x48, 0xc6, 0x7c, 0x8f, 0x64, - 0x72, 0x42, 0x9a, 0xb8, 0xa9, 0x2d, 0x9b, 0x99, 0xfb, 0xb0, 0x0a, 0xb6, 0x19, 0xa9, 0x3c, 0x73, - 0xeb, 0xdf, 0xc8, 0x7c, 0xee, 0x65, 0x64, 0x5e, 0x79, 0x8e, 0xcc, 0xdf, 0x03, 0x7b, 0x27, 0x88, - 0x04, 0x24, 0x75, 0xaa, 0x6b, 0x95, 0xf5, 0x56, 0xef, 0xe4, 0x81, 0xb7, 0xa9, 0xbb, 0xa5, 0x90, - 0x38, 0x0f, 0x41, 0x97, 0xa1, 0x1d, 0x92, 0x34, 0xf3, 0x74, 0x88, 0xbe, 0x2f, 0x8b, 0xa5, 0xf3, - 0xe3, 0x96, 0x80, 0x69, 0x03, 0x3a, 0xa9, 0xa3, 0xee, 0xb3, 0x24, 0x0d, 0x78, 0x24, 0xef, 0x4c, - 0x53, 0x41, 0xee, 0x29, 0x93, 0xfb, 0x93, 0x05, 0x0d, 0xfd, 0x39, 0x74, 0x0b, 0x96, 0x12, 0x96, - 0xf2, 0x61, 0x42, 0x99, 0x57, 0xcc, 0xd3, 0x9a, 0x21, 0xcf, 0x79, 0x13, 0xb9, 0xad, 0xf2, 0x7d, - 0x1b, 0x80, 0xf2, 0x30, 0x64, 0x34, 0x0b, 0xf4, 0x8c, 0xb4, 0x7a, 0x4b, 0xfa, 0xb8, 0xb9, 0x5d, - 0x9c, 0x78, 0xab, 0xfa, 0xe8, 0xf1, 0xea, 0x11, 0x5c, 0x40, 0x23, 0x17, 0xec, 0x1d, 0x42, 0xf7, - 0x77, 0x83, 0x30, 0x94, 0x35, 0xee, 0xe0, 0x7c, 0xed, 0xfe, 0x66, 0x41, 0x4d, 0x32, 0x08, 0x3a, - 0x0f, 0x86, 0x4c, 0x35, 0x09, 0x3e, 0xa3, 0x1a, 0x06, 0x81, 0x1c, 0x68, 0x98, 0x22, 0xcc, 0xc9, - 0x22, 0x98, 0x65, 0xa9, 0xb2, 0xd5, 0x17, 0xaa, 0x6c, 0xad, 0x54, 0x59, 0xf4, 0x26, 0x40, 0x9a, - 0x91, 0x8c, 0xa9, 0x1a, 0xd6, 0x67, 0xa8, 0x61, 0x4d, 0xe2, 0xdd, 0x3f, 0x2d, 0xa8, 0x0a, 0xde, - 0x7b, 0x59, 0x19, 0xbe, 0x02, 0xb5, 0x84, 0x44, 0x7d, 0xc3, 0xe2, 0x47, 0xd5, 0x26, 0x58, 0x98, - 0xe4, 0x16, 0xca, 0x3b, 0x75, 0xde, 0xea, 0xcc, 0xe7, 0x45, 0x37, 0x01, 0xa5, 0x8c, 0x84, 0xcc, - 0x9f, 0x18, 0x9a, 0xda, 0x0c, 0x1b, 0xb4, 0x55, 0x9c, 0x1a, 0x19, 0x77, 0x03, 0x5a, 0x05, 0xf2, - 0x46, 0x6b, 0xd0, 0xa2, 0x7b, 0x8c, 0xee, 0xc7, 0x3c, 0x88, 0xb2, 0x54, 0x56, 0xa0, 0x83, 0x8b, - 0xa6, 0x53, 0xdf, 0xcd, 0x83, 0x8d, 0x59, 0x1a, 0xf3, 0x28, 0x65, 0xe8, 0xdc, 0x84, 0x20, 0x16, - 0x65, 0x47, 0x01, 0x8a, 0x8a, 0xf8, 0x2e, 0x80, 0x91, 0x39, 0xe6, 0xeb, 0xe1, 0x5c, 0x2e, 0x47, - 0xdc, 0xc8, 0x31, 0xb8, 0x80, 0x47, 0x57, 0xa1, 0x69, 0xd4, 0xce, 0xd7, 0x35, 0x3d, 0x51, 0x0e, - 0x36, 0x97, 0xd9, 0xc7, 0x63, 0x34, 0xba, 0x04, 0x0d, 0xa1, 0x7b, 0x01, 0xf3, 0xf5, 0x9c, 0x1d, - 0x2f, 0x07, 0x6e, 0x2a, 0x00, 0x36, 0x48, 0x74, 0x11, 0xea, 0x42, 0x00, 0x99, 0xaf, 0x6f, 0xbd, - 0x53, 0x8e, 0xb9, 0x2d, 0xfd, 0x58, 0xe3, 0xd0, 0x1b, 0x60, 0x6b, 0x88, 0xaf, 0x75, 0xd2, 0x2d, - 0xc7, 0xe8, 0x29, 0xf2, 0x71, 0x8e, 0x15, 0x3c, 0xa9, 0x2e, 0xb1, 0xef, 0xa5, 0x74, 0x8f, 0x0d, - 0x88, 0x63, 0xcb, 0xe8, 0xd5, 0x67, 0x54, 0x53, 0xe1, 0xee, 0x4a, 0x18, 0xee, 0xa4, 0xc5, 0xa5, - 0xa8, 0xef, 0xb8, 0x4f, 0x4e, 0xe3, 0xa0, 0xfa, 0x6e, 0xe7, 0x18, 0x5c, 0xc0, 0x1f, 0x2e, 0xd4, - 0xbf, 0xcc, 0x69, 0xa1, 0x76, 0xc1, 0x36, 0x9c, 0xaf, 0x67, 0x23, 0x5f, 0x8b, 0x89, 0xd4, 0xa3, - 0xa8, 0xd2, 0x98, 0x5d, 0xa8, 0xda, 0x2a, 0x4e, 0xe7, 0xf1, 0x39, 0x9c, 0x98, 0x26, 0xc4, 0xe2, - 0x86, 0xb3, 0xf0, 0xff, 0xd2, 0x24, 0x2f, 0xea, 0x8d, 0xcf, 0xc3, 0xa2, 0xcf, 0xe9, 0x70, 0xc0, - 0xa2, 0x8c, 0x08, 0xca, 0xf3, 0x86, 0x49, 0xa8, 0xe5, 0x70, 0x61, 0xc2, 0xf1, 0x59, 0x12, 0xa2, - 0x33, 0x50, 0xe7, 0x64, 0x98, 0xed, 0xf5, 0x74, 0xff, 0xdb, 0xea, 0x02, 0xdf, 0xde, 0x14, 0x36, - 0xac, 0x7d, 0xe8, 0x32, 0x1c, 0xcb, 0xcf, 0x1a, 0x93, 0x6c, 0xcf, 0x93, 0xc5, 0x64, 0x49, 0xea, - 0xd4, 0xd7, 0x2a, 0xeb, 0xcd, 0xf1, 0x41, 0xee, 0x90, 0x6c, 0xef, 0x8e, 0xf6, 0xb9, 0x3f, 0x56, - 0x00, 0xc6, 0x63, 0x8e, 0xde, 0x2f, 0xa8, 0x94, 0x25, 0x55, 0xea, 0xcc, 0x61, 0xd7, 0xa2, 0x2c, - 0x54, 0xee, 0xcf, 0x73, 0x63, 0x3d, 0x39, 0x0b, 0x0b, 0x09, 0xa3, 0x7c, 0x30, 0x60, 0x91, 0xcf, - 0x7c, 0x6f, 0xac, 0xe6, 0xf8, 0x68, 0xc1, 0xfe, 0x89, 0x50, 0xe8, 0x83, 0xa4, 0x67, 0xee, 0x05, - 0xa4, 0xe7, 0x16, 0x2c, 0x99, 0x1a, 0x3e, 0x77, 0xbb, 0xe6, 0x4d, 0xa4, 0x6e, 0xd4, 0x02, 0x54, - 0xf6, 0xd9, 0x48, 0x2a, 0x76, 0x13, 0x8b, 0x9f, 0x82, 0x67, 0xfd, 0x20, 0x25, 0x3b, 0xa1, 0x12, - 0x61, 0x1b, 0x9b, 0x25, 0x3a, 0x0d, 0x9d, 0x89, 0x0e, 0xe8, 0xc2, 0xb7, 0x8b, 0x85, 0x47, 0xaf, - 0xc2, 0xd1, 0x20, 0xf5, 0x76, 0x49, 0x18, 0x0a, 0x49, 0xf3, 0xc4, 0xe6, 0x0d, 0xb9, 0x4d, 0x27, - 0x48, 0x6f, 0x6a, 0xeb, 0x47, 0x6c, 0xe4, 0x7e, 0x0b, 0xcd, 0x9c, 0x41, 0xd0, 0xf5, 0x52, 0x5b, - 0x4e, 0x1f, 0x42, 0x38, 0xcf, 0xe8, 0x4a, 0x77, 0xdc, 0x94, 0xd2, 0x29, 0xad, 0xf2, 0x29, 0x5d, - 0x1f, 0x1a, 0x9a, 0x86, 0xd0, 0x6b, 0x80, 0x88, 0x94, 0x65, 0xcf, 0x67, 0x29, 0x4d, 0x82, 0x58, - 0x0a, 0xba, 0x6a, 0xe3, 0xa2, 0xf2, 0xdc, 0x18, 0x3b, 0xd0, 0x39, 0x50, 0xaa, 0x30, 0x2d, 0xf9, - 0xfa, 0x19, 0x75, 0x57, 0xf8, 0x8c, 0xd0, 0x7d, 0x00, 0x75, 0x45, 0x5c, 0xe8, 0x1d, 0x38, 0xce, - 0xbe, 0x89, 0xc3, 0x80, 0x06, 0x99, 0x57, 0x78, 0xb3, 0x8b, 0x46, 0x28, 0xe6, 0xb7, 0xb1, 0x63, - 0x00, 0x9b, 0x53, 0x7e, 0xf7, 0x0b, 0xb0, 0x0d, 0x97, 0x89, 0xee, 0xe8, 0xa4, 0x35, 0x29, 0x98, - 0x25, 0xba, 0x04, 0xb6, 0xcf, 0xe9, 0xec, 0x53, 0x55, 0xf1, 0x39, 0x75, 0x43, 0xe8, 0x4c, 0x10, - 0xdd, 0x21, 0xfb, 0x6f, 0x42, 0xeb, 0x79, 0xc9, 0x66, 0x6a, 0xd8, 0xdc, 0xb7, 0x00, 0xc6, 0x94, - 0x38, 0xae, 0xa4, 0xf5, 0x8f, 0x95, 0xec, 0x5d, 0x87, 0x66, 0xee, 0x40, 0x3d, 0x68, 0x98, 0x67, - 0xca, 0xc2, 0xf4, 0x1b, 0xd3, 0x5d, 0x2c, 0x0d, 0xce, 0xba, 0x75, 0xd1, 0xda, 0xba, 0xf6, 0xe8, - 0xf7, 0x95, 0x23, 0x8f, 0x9e, 0xac, 0x58, 0xbf, 0x3e, 0x59, 0xb1, 0x1e, 0xfe, 0xb1, 0x62, 0x7d, - 0x79, 0x61, 0xa6, 0xc7, 0xb6, 0xde, 0x6c, 0xa7, 0x2e, 0x4d, 0x97, 0xfe, 0x0e, 0x00, 0x00, 0xff, - 0xff, 0x84, 0xc4, 0x00, 0x21, 0x04, 0x0f, 0x00, 0x00, + // 1357 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x57, 0xdd, 0x6e, 0x1b, 0xd5, + 0x13, 0xef, 0xc6, 0x5f, 0xeb, 0xb1, 0x9d, 0x8f, 0xa3, 0xfc, 0xab, 0xed, 0x36, 0xff, 0x24, 0x4d, + 0x0b, 0x4a, 0x3f, 0x70, 0x8a, 0xdb, 0x02, 0xe5, 0xa3, 0x25, 0x4e, 0xa9, 0x44, 0x11, 0xa4, 0x3a, + 0x85, 0x22, 0xb8, 0x59, 0xad, 0xcf, 0x9e, 0x38, 0x4b, 0xd6, 0x7b, 0x96, 0xdd, 0x75, 0x4b, 0x24, + 0x5e, 0x80, 0x0b, 0x6e, 0xb9, 0xee, 0x0b, 0xf0, 0x18, 0x48, 0xbd, 0xe0, 0x82, 0x27, 0xa8, 0xa0, + 0x3c, 0x03, 0x42, 0xf4, 0x0a, 0x9d, 0xaf, 0xf5, 0xda, 0x5b, 0x07, 0xa7, 0x2a, 0x12, 0x37, 0xad, + 0xcf, 0xcc, 0x6f, 0x66, 0xe7, 0xfc, 0x66, 0xce, 0xcc, 0x04, 0x36, 0xfa, 0x6c, 0x2b, 0x8a, 0x59, + 0xca, 0x08, 0x0b, 0x92, 0x2d, 0xe2, 0x46, 0xe9, 0x30, 0xa6, 0xfa, 0xff, 0xb6, 0xd0, 0xa0, 0x9a, + 0x3a, 0xda, 0x2b, 0x63, 0xe0, 0xbd, 0x80, 0x3d, 0x14, 0xff, 0x48, 0x98, 0xbd, 0xdc, 0x67, 0x7d, + 0x26, 0x7e, 0x6e, 0xf1, 0x5f, 0x52, 0xba, 0xf1, 0xfd, 0x3c, 0xd4, 0x30, 0xfd, 0x7a, 0x48, 0x93, + 0x14, 0x9d, 0x87, 0x72, 0x12, 0x51, 0x62, 0x19, 0xeb, 0xc6, 0x66, 0xa3, 0xf3, 0xbf, 0xb6, 0xfe, + 0x8c, 0xd2, 0xb7, 0xef, 0x45, 0x94, 0x60, 0x01, 0x41, 0xd7, 0xc0, 0xf4, 0xfc, 0x84, 0xb0, 0x07, + 0x34, 0xb6, 0xe6, 0x04, 0xfc, 0x54, 0x01, 0x7e, 0x4b, 0x01, 0x70, 0x06, 0xe5, 0x66, 0x0f, 0xdc, + 0xc0, 0xf7, 0xdc, 0x94, 0x5a, 0xa5, 0x29, 0x66, 0xf7, 0x15, 0x00, 0x67, 0x50, 0x74, 0x09, 0x2a, + 0x6e, 0x14, 0x05, 0x87, 0x56, 0x59, 0xd8, 0x9c, 0x2c, 0xd8, 0x6c, 0x73, 0x2d, 0x96, 0x20, 0x7e, + 0x0d, 0x16, 0xd1, 0xd0, 0xaa, 0x4c, 0xb9, 0xc6, 0x6e, 0x44, 0x43, 0x2c, 0x20, 0xe8, 0x06, 0x34, + 0x5c, 0x72, 0x10, 0xb2, 0x87, 0x01, 0xf5, 0xfa, 0xd4, 0xaa, 0x0a, 0x8b, 0x95, 0xa2, 0xfb, 0x11, + 0x06, 0xe7, 0x0d, 0xd0, 0x69, 0x30, 0xfd, 0x30, 0xa5, 0x71, 0xe8, 0x06, 0x96, 0xb7, 0x6e, 0x6c, + 0x36, 0x71, 0xfd, 0x9c, 0x16, 0xd8, 0xdf, 0x19, 0x50, 0xe6, 0x94, 0xa1, 0xdb, 0x30, 0x4f, 0x58, + 0x18, 0x52, 0x92, 0xb2, 0xd8, 0x49, 0x0f, 0x23, 0x2a, 0x18, 0x9e, 0xef, 0xac, 0xb5, 0x45, 0x7a, + 0x76, 0xe4, 0xd7, 0x38, 0xb4, 0xbd, 0xa3, 0x71, 0x9f, 0x1e, 0x46, 0x14, 0xb7, 0x48, 0xfe, 0x88, + 0xae, 0x43, 0x83, 0xb0, 0x70, 0xcf, 0xef, 0x3b, 0x5f, 0x25, 0x2c, 0x14, 0xbc, 0x37, 0xbb, 0x2b, + 0xcf, 0x9e, 0xac, 0x59, 0x34, 0x24, 0xcc, 0xf3, 0xc3, 0xfe, 0x16, 0x57, 0xb4, 0xb1, 0xfb, 0xf0, + 0x63, 0x9a, 0x24, 0x6e, 0x9f, 0xe2, 0xaa, 0x34, 0xb0, 0xff, 0x34, 0xc0, 0xd4, 0xf9, 0x40, 0x1f, + 0x42, 0x39, 0x74, 0x07, 0x32, 0x03, 0xf5, 0xee, 0xb5, 0x67, 0x4f, 0xd6, 0x5e, 0xef, 0xfb, 0xe9, + 0xfe, 0xb0, 0xd7, 0x26, 0x6c, 0xb0, 0x45, 0x93, 0x74, 0xe8, 0xc6, 0x87, 0xb2, 0x7e, 0x0a, 0x15, + 0xa5, 0xa3, 0xc5, 0xc2, 0xc5, 0x7f, 0xe0, 0x6a, 0xe8, 0xff, 0x00, 0x24, 0xa6, 0x6e, 0x4a, 0x3d, + 0xc7, 0x4d, 0x45, 0x85, 0xd4, 0x71, 0x5d, 0x49, 0xb6, 0x53, 0xfb, 0x51, 0x19, 0x4c, 0x5d, 0x52, + 0xd9, 0xcd, 0x8d, 0x7f, 0xe3, 0xe6, 0x73, 0x2f, 0xe3, 0xe6, 0xa5, 0x63, 0xdc, 0xfc, 0x3d, 0x30, + 0x7b, 0x7e, 0xc8, 0x21, 0x89, 0x55, 0x5e, 0x2f, 0x6d, 0x36, 0x3a, 0x67, 0xa6, 0xbe, 0xa6, 0x76, + 0x57, 0x22, 0x71, 0x66, 0x82, 0xae, 0x42, 0x33, 0x70, 0x93, 0xd4, 0x51, 0x26, 0xea, 0xbd, 0x2c, + 0x15, 0xe2, 0xc7, 0x0d, 0x0e, 0x53, 0x02, 0x74, 0x46, 0x59, 0x3d, 0xa0, 0x71, 0xe2, 0xb3, 0x50, + 0xbc, 0x99, 0xba, 0x84, 0xdc, 0x97, 0x22, 0xfb, 0x47, 0x03, 0x6a, 0xea, 0x73, 0xe8, 0x0e, 0x2c, + 0xc7, 0x34, 0x61, 0xc3, 0x98, 0x50, 0x27, 0x7f, 0x4f, 0x63, 0x86, 0x7b, 0xce, 0x6b, 0xcb, 0x1d, + 0x79, 0xdf, 0xb7, 0x01, 0x08, 0x0b, 0x02, 0x4a, 0x52, 0x5f, 0xd5, 0x48, 0xa3, 0xb3, 0xac, 0xc2, + 0xcd, 0xe4, 0x3c, 0xe2, 0x6e, 0xf9, 0xf1, 0x93, 0xb5, 0x13, 0x38, 0x87, 0x46, 0x36, 0x98, 0x3d, + 0x97, 0x1c, 0xec, 0xf9, 0x41, 0x20, 0x38, 0x6e, 0xe1, 0xec, 0x6c, 0xff, 0x6a, 0x40, 0x45, 0x74, + 0x10, 0x74, 0x11, 0x74, 0x33, 0x55, 0x4d, 0xf0, 0x39, 0x6c, 0x68, 0x04, 0xb2, 0xa0, 0xa6, 0x49, + 0x98, 0x13, 0x24, 0xe8, 0x63, 0x81, 0xd9, 0xf2, 0x0b, 0x31, 0x5b, 0x29, 0x30, 0x8b, 0xde, 0x04, + 0x48, 0x52, 0x37, 0xa5, 0x92, 0xc3, 0xea, 0x0c, 0x1c, 0x56, 0x04, 0xde, 0xfe, 0xc3, 0x80, 0x32, + 0xef, 0x7b, 0x2f, 0xeb, 0x86, 0xaf, 0x40, 0x25, 0x76, 0xc3, 0xbe, 0xee, 0xe2, 0x0b, 0xd2, 0x09, + 0xe6, 0x22, 0xe1, 0x42, 0x6a, 0x27, 0xe2, 0x2d, 0xcf, 0x1c, 0x2f, 0xba, 0x0d, 0x28, 0xa1, 0x6e, + 0x40, 0xbd, 0xb1, 0xa2, 0xa9, 0xcc, 0xe0, 0xa0, 0x29, 0xed, 0x64, 0xc9, 0xd8, 0x5b, 0xd0, 0xc8, + 0x35, 0x6f, 0xb4, 0x0e, 0x0d, 0xb2, 0x4f, 0xc9, 0x41, 0xc4, 0xfc, 0x30, 0x4d, 0x04, 0x03, 0x2d, + 0x9c, 0x17, 0x6d, 0xfc, 0xb5, 0x00, 0x26, 0xa6, 0x49, 0xc4, 0xc2, 0x84, 0xa2, 0x0b, 0x63, 0x03, + 0x31, 0x3f, 0x76, 0x24, 0x20, 0x3f, 0x11, 0xdf, 0x05, 0xd0, 0x63, 0x8e, 0x7a, 0xaa, 0x38, 0x57, + 0x8a, 0x16, 0xb7, 0x32, 0x0c, 0xce, 0xe1, 0xd1, 0x75, 0xa8, 0xeb, 0x69, 0xe7, 0x29, 0x4e, 0x4f, + 0x17, 0x8d, 0xf5, 0x63, 0xf6, 0xf0, 0x08, 0x8d, 0xae, 0x40, 0x8d, 0xcf, 0x3d, 0x9f, 0x7a, 0xaa, + 0xce, 0x4e, 0x15, 0x0d, 0xb7, 0x25, 0x00, 0x6b, 0x24, 0xba, 0x0c, 0x55, 0x3e, 0x00, 0xa9, 0xa7, + 0x5e, 0xbd, 0x55, 0xb4, 0xd9, 0x15, 0x7a, 0xac, 0x70, 0xe8, 0x0d, 0x30, 0x15, 0xc4, 0x53, 0x73, + 0xd2, 0x2e, 0xda, 0xa8, 0x2a, 0xf2, 0x70, 0x86, 0xe5, 0x7d, 0x52, 0x3e, 0x62, 0xcf, 0x49, 0xc8, + 0x3e, 0x1d, 0xb8, 0x96, 0x29, 0xac, 0xd7, 0x9e, 0xc3, 0xa6, 0xc4, 0xdd, 0x13, 0x30, 0xdc, 0x4a, + 0xf2, 0x47, 0xce, 0xef, 0x28, 0x4f, 0x56, 0x6d, 0x1a, 0xbf, 0x3b, 0x19, 0x06, 0xe7, 0xf0, 0x3c, + 0x0a, 0xfd, 0xdc, 0x9d, 0x1e, 0xed, 0xfb, 0xa1, 0x55, 0x9f, 0x16, 0x45, 0x57, 0xe1, 0xba, 0x1c, + 0x86, 0x5b, 0xbd, 0xfc, 0x11, 0xed, 0xc2, 0x52, 0xe6, 0x87, 0xb0, 0x41, 0x14, 0xd0, 0x94, 0x5a, + 0x20, 0x5c, 0x6d, 0x4c, 0x77, 0xb5, 0xa3, 0x90, 0x78, 0xb1, 0x37, 0x21, 0x39, 0x7a, 0x83, 0xf8, + 0x79, 0x4e, 0x6d, 0x10, 0x36, 0x98, 0x7a, 0x18, 0xa9, 0xa2, 0xcd, 0xce, 0xfc, 0xa9, 0xa8, 0x37, + 0x22, 0xf9, 0x9d, 0x7d, 0x82, 0x36, 0xa5, 0x9d, 0x22, 0xf8, 0x73, 0x38, 0x3d, 0xd9, 0xa9, 0xf3, + 0x0e, 0x67, 0x19, 0x4c, 0xcb, 0xe3, 0x0d, 0x5b, 0x39, 0xbe, 0x08, 0x4b, 0x1e, 0x23, 0xc3, 0x01, + 0x0d, 0x53, 0x97, 0xf7, 0x62, 0x67, 0x18, 0x07, 0x6a, 0x4e, 0x2f, 0x8e, 0x29, 0x3e, 0x8b, 0x03, + 0x74, 0x0e, 0xaa, 0xcc, 0x1d, 0xa6, 0xfb, 0x1d, 0x55, 0x98, 0x4d, 0xd9, 0x59, 0x76, 0xb7, 0xb9, + 0x0c, 0x2b, 0x1d, 0xba, 0x0a, 0x27, 0xb3, 0x58, 0x23, 0x37, 0xdd, 0x77, 0x44, 0x96, 0x69, 0x9c, + 0x58, 0xd5, 0xf5, 0xd2, 0x66, 0x7d, 0x14, 0xc8, 0x5d, 0x37, 0xdd, 0xbf, 0xab, 0x74, 0xf6, 0x0f, + 0x25, 0x80, 0xd1, 0xfb, 0x43, 0xef, 0xe7, 0xc6, 0xa7, 0x21, 0xc6, 0xe7, 0xb9, 0xa3, 0xde, 0x6b, + 0x71, 0x82, 0xda, 0x3f, 0xcd, 0x8d, 0x06, 0xdd, 0x79, 0x58, 0x8c, 0x29, 0x61, 0x83, 0x01, 0x0d, + 0x3d, 0xea, 0x39, 0xa3, 0x35, 0x03, 0x2f, 0xe4, 0xe4, 0x9f, 0xf0, 0xd5, 0x61, 0xda, 0x4c, 0x9c, + 0x7b, 0x81, 0x99, 0x78, 0x07, 0x96, 0x35, 0x87, 0xc7, 0x4e, 0xd7, 0xbc, 0xb6, 0x54, 0x89, 0x5a, + 0x84, 0xd2, 0x01, 0x3d, 0x14, 0xab, 0x44, 0x1d, 0xf3, 0x9f, 0x7c, 0x00, 0x78, 0x7e, 0xe2, 0xf6, + 0x02, 0xb9, 0x1d, 0x98, 0x58, 0x1f, 0xd1, 0x59, 0x68, 0x8d, 0x65, 0x40, 0x11, 0xdf, 0xcc, 0x13, + 0x8f, 0x5e, 0x85, 0x05, 0x3f, 0x71, 0xf6, 0xdc, 0x20, 0xe0, 0x75, 0xef, 0x70, 0xe7, 0x35, 0xe1, + 0xa6, 0xe5, 0x27, 0xb7, 0x95, 0xf4, 0x23, 0x7a, 0x68, 0x7f, 0x0b, 0xf5, 0xac, 0xb5, 0xa1, 0x9b, + 0x85, 0xb4, 0x9c, 0x3d, 0xa2, 0x13, 0x3e, 0x27, 0x2b, 0xed, 0x51, 0x52, 0x0a, 0x51, 0x1a, 0xc5, + 0x28, 0x6d, 0x0f, 0x6a, 0xaa, 0x3f, 0xa2, 0xd7, 0x00, 0xb9, 0x62, 0x5f, 0x70, 0x3c, 0x9a, 0x90, + 0xd8, 0x8f, 0xc4, 0xa6, 0x21, 0xd3, 0xb8, 0x24, 0x35, 0xb7, 0x46, 0x0a, 0x74, 0x01, 0xe4, 0xb8, + 0x9a, 0xdc, 0x45, 0xd4, 0x7e, 0x77, 0x8f, 0xeb, 0xf4, 0x04, 0xfe, 0x00, 0xaa, 0xb2, 0xa3, 0xa2, + 0x77, 0xe0, 0x14, 0xfd, 0x26, 0x0a, 0x7c, 0xe2, 0xa7, 0x4e, 0xee, 0x8f, 0x09, 0x9e, 0x08, 0x39, + 0x92, 0x4c, 0x6c, 0x69, 0xc0, 0xf6, 0x84, 0xde, 0xfe, 0x02, 0x4c, 0xdd, 0x64, 0x79, 0x76, 0xd4, + 0xa5, 0x55, 0x53, 0xd0, 0x47, 0x74, 0x05, 0x4c, 0x8f, 0x91, 0xd9, 0xab, 0xaa, 0xe4, 0x31, 0x62, + 0x07, 0xd0, 0x1a, 0xeb, 0xc0, 0x47, 0xf8, 0xdf, 0x86, 0xc6, 0x71, 0x9b, 0xcd, 0x44, 0xb1, 0xd9, + 0x6f, 0x01, 0x8c, 0x7a, 0xf5, 0x88, 0x49, 0xe3, 0x9f, 0x99, 0x3c, 0x0f, 0xad, 0xb1, 0x1e, 0x3d, + 0x3d, 0x4e, 0xfb, 0x12, 0x2c, 0x4e, 0xf6, 0xe0, 0xe9, 0xe8, 0xce, 0x4d, 0xa8, 0x67, 0x5f, 0x44, + 0x1d, 0xa8, 0xe9, 0xc5, 0x6c, 0x71, 0x72, 0xab, 0xb6, 0x97, 0x0a, 0x15, 0xb9, 0x69, 0x5c, 0x36, + 0xba, 0x37, 0x1e, 0xff, 0xb6, 0x7a, 0xe2, 0xf1, 0xd3, 0x55, 0xe3, 0x97, 0xa7, 0xab, 0xc6, 0xa3, + 0xdf, 0x57, 0x8d, 0x2f, 0x2f, 0xcd, 0xf4, 0xe7, 0x85, 0x72, 0xd6, 0xab, 0x0a, 0xd1, 0x95, 0xbf, + 0x03, 0x00, 0x00, 0xff, 0xff, 0x48, 0x46, 0xa9, 0xce, 0xf6, 0x0f, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1764,6 +1862,30 @@ func (m *Response) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0xa2 } + if m.BackfillComplete != nil { + { + size, err := m.BackfillComplete.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintCapture(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x52 + } + if m.BackfillBegin != nil { + { + size, err := m.BackfillBegin.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintCapture(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x4a + } if m.SourcedSchema != nil { { size, err := m.SourcedSchema.MarshalToSizedBuffer(dAtA[:i]) @@ -2341,6 +2463,70 @@ func (m *Response_Checkpoint) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *Response_BackfillBegin) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Response_BackfillBegin) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Response_BackfillBegin) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Binding != 0 { + i = encodeVarintCapture(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *Response_BackfillComplete) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Response_BackfillComplete) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Response_BackfillComplete) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Binding != 0 { + i = encodeVarintCapture(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func encodeVarintCapture(dAtA []byte, offset int, v uint64) int { offset -= sovCapture(v) base := offset @@ -2613,6 +2799,14 @@ func (m *Response) ProtoSize() (n int) { l = m.SourcedSchema.ProtoSize() n += 1 + l + sovCapture(uint64(l)) } + if m.BackfillBegin != nil { + l = m.BackfillBegin.ProtoSize() + n += 1 + l + sovCapture(uint64(l)) + } + if m.BackfillComplete != nil { + l = m.BackfillComplete.ProtoSize() + n += 1 + l + sovCapture(uint64(l)) + } l = len(m.Internal) if l > 0 { n += 2 + l + sovCapture(uint64(l)) @@ -2845,6 +3039,36 @@ func (m *Response_Checkpoint) ProtoSize() (n int) { return n } +func (m *Response_BackfillBegin) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Binding != 0 { + n += 1 + sovCapture(uint64(m.Binding)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Response_BackfillComplete) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Binding != 0 { + n += 1 + sovCapture(uint64(m.Binding)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovCapture(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -4630,6 +4854,78 @@ func (m *Response) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BackfillBegin", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCapture + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthCapture + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthCapture + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.BackfillBegin == nil { + m.BackfillBegin = &Response_BackfillBegin{} + } + if err := m.BackfillBegin.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 10: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BackfillComplete", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCapture + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthCapture + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthCapture + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.BackfillComplete == nil { + m.BackfillComplete = &Response_BackfillComplete{} + } + if err := m.BackfillComplete.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex case 100: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Internal", wireType) @@ -5917,6 +6213,146 @@ func (m *Response_Checkpoint) Unmarshal(dAtA []byte) error { } return nil } +func (m *Response_BackfillBegin) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCapture + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BackfillBegin: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BackfillBegin: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCapture + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipCapture(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCapture + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Response_BackfillComplete) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCapture + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BackfillComplete: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BackfillComplete: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCapture + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipCapture(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCapture + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipCapture(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/go/protocols/capture/capture.proto b/go/protocols/capture/capture.proto index 945b7181c46..66d24813c22 100644 --- a/go/protocols/capture/capture.proto +++ b/go/protocols/capture/capture.proto @@ -393,6 +393,26 @@ message Response { } Checkpoint checkpoint = 7; + // Signals the start of a backfill for a binding. + // + // A backfill message (BackfillBegin or BackfillComplete) must stand alone + // in its connector checkpoint: the checkpoint must contain only the + // backfill message followed by the terminating Checkpoint response, with + // no Captured, SourcedSchema, or other backfill messages. The runtime + // enforces this rule and will fail the session on violation. + message BackfillBegin { + uint32 binding = 1; + } + BackfillBegin backfill_begin = 9; + + // Signals the end of a backfill for a binding. + // + // See BackfillBegin for the "stands alone in its checkpoint" rule. + message BackfillComplete { + uint32 binding = 1; + } + BackfillComplete backfill_complete = 10; + // Reserved for internal use. bytes internal = 100 [ json_name = "$internal" ]; } diff --git a/go/protocols/materialize/materialize.pb.go b/go/protocols/materialize/materialize.pb.go index c95bc1268eb..b96d8ba2c82 100644 --- a/go/protocols/materialize/materialize.pb.go +++ b/go/protocols/materialize/materialize.pb.go @@ -11,6 +11,7 @@ import ( github_com_estuary_flow_go_protocols_flow "github.com/estuary/flow/go/protocols/flow" _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" + types "github.com/gogo/protobuf/types" protocol "go.gazette.dev/core/consumer/protocol" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" @@ -513,10 +514,12 @@ type Request_Flush struct { // patches (the runtime feeds the shard's contribution back to it for // symmetry with the scaled-out case). Connectors participating in // cooperative multi-shard strategies use this to observe peers' state. - StatePatchesJson encoding_json.RawMessage `protobuf:"bytes,1,opt,name=state_patches_json,json=statePatches,proto3,casttype=encoding/json.RawMessage" json:"state_patches_json,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + StatePatchesJson encoding_json.RawMessage `protobuf:"bytes,1,opt,name=state_patches_json,json=statePatches,proto3,casttype=encoding/json.RawMessage" json:"state_patches_json,omitempty"` + BackfillBegins []*Request_Flush_BackfillBegin `protobuf:"bytes,2,rep,name=backfill_begins,json=backfillBegins,proto3" json:"backfill_begins,omitempty"` + BackfillCompletes []*Request_Flush_BackfillComplete `protobuf:"bytes,3,rep,name=backfill_completes,json=backfillCompletes,proto3" json:"backfill_completes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Request_Flush) Reset() { *m = Request_Flush{} } @@ -552,6 +555,100 @@ func (m *Request_Flush) XXX_DiscardUnknown() { var xxx_messageInfo_Request_Flush proto.InternalMessageInfo +// Backfill-begin signals observed during this transaction. A connector +// acts on those relevant to its key range. +type Request_Flush_BackfillBegin struct { + Binding uint32 `protobuf:"varint,1,opt,name=binding,proto3" json:"binding,omitempty"` + // Truncation boundary of the binding's backfill: documents published at or + // after this time are current, while earlier ones were superseded by the + // backfill. It equals the begin's own publication time (flow_published_at), + // and is carried on both begin and complete so connectors need not track it + // across transactions. + Timestamp *types.Timestamp `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Request_Flush_BackfillBegin) Reset() { *m = Request_Flush_BackfillBegin{} } +func (m *Request_Flush_BackfillBegin) String() string { return proto.CompactTextString(m) } +func (*Request_Flush_BackfillBegin) ProtoMessage() {} +func (*Request_Flush_BackfillBegin) Descriptor() ([]byte, []int) { + return fileDescriptor_3e8b62b327f34bc6, []int{0, 5, 0} +} +func (m *Request_Flush_BackfillBegin) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Request_Flush_BackfillBegin) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Request_Flush_BackfillBegin.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Request_Flush_BackfillBegin) XXX_Merge(src proto.Message) { + xxx_messageInfo_Request_Flush_BackfillBegin.Merge(m, src) +} +func (m *Request_Flush_BackfillBegin) XXX_Size() int { + return m.ProtoSize() +} +func (m *Request_Flush_BackfillBegin) XXX_DiscardUnknown() { + xxx_messageInfo_Request_Flush_BackfillBegin.DiscardUnknown(m) +} + +var xxx_messageInfo_Request_Flush_BackfillBegin proto.InternalMessageInfo + +// Backfill-complete signals observed during this transaction. See +// BackfillBegin. +type Request_Flush_BackfillComplete struct { + Binding uint32 `protobuf:"varint,1,opt,name=binding,proto3" json:"binding,omitempty"` + // Truncation boundary of the completed backfill (see BackfillBegin.timestamp): + // the connector may delete destination rows whose flow_published_at predates + // this time, as they were superseded by the backfill. + Timestamp *types.Timestamp `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Request_Flush_BackfillComplete) Reset() { *m = Request_Flush_BackfillComplete{} } +func (m *Request_Flush_BackfillComplete) String() string { return proto.CompactTextString(m) } +func (*Request_Flush_BackfillComplete) ProtoMessage() {} +func (*Request_Flush_BackfillComplete) Descriptor() ([]byte, []int) { + return fileDescriptor_3e8b62b327f34bc6, []int{0, 5, 1} +} +func (m *Request_Flush_BackfillComplete) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Request_Flush_BackfillComplete) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Request_Flush_BackfillComplete.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Request_Flush_BackfillComplete) XXX_Merge(src proto.Message) { + xxx_messageInfo_Request_Flush_BackfillComplete.Merge(m, src) +} +func (m *Request_Flush_BackfillComplete) XXX_Size() int { + return m.ProtoSize() +} +func (m *Request_Flush_BackfillComplete) XXX_DiscardUnknown() { + xxx_messageInfo_Request_Flush_BackfillComplete.DiscardUnknown(m) +} + +var xxx_messageInfo_Request_Flush_BackfillComplete proto.InternalMessageInfo + // Store documents updated by the current transaction. // // The runtime populates exactly one of the JSON encodings (`key_json`, @@ -1485,6 +1582,8 @@ func init() { proto.RegisterType((*Request_Open)(nil), "materialize.Request.Open") proto.RegisterType((*Request_Load)(nil), "materialize.Request.Load") proto.RegisterType((*Request_Flush)(nil), "materialize.Request.Flush") + proto.RegisterType((*Request_Flush_BackfillBegin)(nil), "materialize.Request.Flush.BackfillBegin") + proto.RegisterType((*Request_Flush_BackfillComplete)(nil), "materialize.Request.Flush.BackfillComplete") proto.RegisterType((*Request_Store)(nil), "materialize.Request.Store") proto.RegisterType((*Request_StartCommit)(nil), "materialize.Request.StartCommit") proto.RegisterType((*Request_Acknowledge)(nil), "materialize.Request.Acknowledge") @@ -1512,129 +1611,136 @@ func init() { } var fileDescriptor_3e8b62b327f34bc6 = []byte{ - // 1942 bytes of a gzipped FileDescriptorProto + // 2055 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x58, 0x4f, 0x6f, 0x1b, 0xc7, - 0x15, 0xf7, 0x52, 0xfc, 0xfb, 0x48, 0x59, 0xd4, 0x84, 0x8a, 0xe9, 0x8d, 0x63, 0xcb, 0x4a, 0x82, - 0x08, 0x2e, 0x42, 0x19, 0x72, 0xda, 0xc4, 0x0e, 0x5c, 0x94, 0xa4, 0x28, 0x80, 0xaa, 0x24, 0x2a, - 0x23, 0xcb, 0x01, 0x72, 0x21, 0x46, 0xbb, 0x23, 0x6a, 0xad, 0xe5, 0xce, 0x76, 0x67, 0x69, 0x9b, - 0xbd, 0x17, 0x45, 0x0b, 0x14, 0x68, 0x8f, 0x0d, 0x7a, 0xe8, 0xa9, 0xdf, 0xa0, 0x40, 0xd1, 0x6b, - 0x2f, 0x3e, 0xf6, 0x13, 0xb8, 0x68, 0xfa, 0x05, 0x7a, 0x29, 0x0a, 0xe4, 0x54, 0xcc, 0x9f, 0x5d, - 0x2e, 0x69, 0x92, 0xa2, 0x00, 0x37, 0x17, 0x62, 0xe7, 0xcd, 0xef, 0xf7, 0x76, 0xde, 0xcc, 0x9b, - 0xf7, 0x7e, 0x4b, 0xb8, 0xd7, 0x63, 0x5b, 0x7e, 0xc0, 0x42, 0x66, 0x31, 0x97, 0x6f, 0xf5, 0x49, - 0x48, 0x03, 0x87, 0xb8, 0xce, 0xcf, 0x69, 0xf2, 0xb9, 0x26, 0x11, 0xa8, 0x98, 0x30, 0x99, 0xeb, - 0x16, 0xf3, 0xf8, 0xa0, 0x4f, 0x83, 0x98, 0x1e, 0x3f, 0x28, 0xb8, 0x79, 0x6b, 0xcc, 0xf5, 0x99, - 0xcb, 0x5e, 0xc8, 0x1f, 0x3d, 0x5b, 0xe9, 0xb1, 0x1e, 0x93, 0x8f, 0x5b, 0xe2, 0x49, 0x59, 0x37, - 0xfe, 0xbb, 0x06, 0x39, 0x4c, 0x7f, 0x36, 0xa0, 0x3c, 0x44, 0x9f, 0x40, 0x9a, 0xfb, 0xd4, 0xaa, - 0x1a, 0xeb, 0xc6, 0x66, 0x71, 0xfb, 0x66, 0x2d, 0xb9, 0x20, 0x8d, 0xa9, 0x1d, 0xfb, 0xd4, 0xc2, - 0x12, 0x86, 0x1e, 0x42, 0xfe, 0x39, 0x71, 0x1d, 0x9b, 0x84, 0xb4, 0x9a, 0x92, 0x94, 0xf7, 0xa7, - 0x52, 0x9e, 0x6a, 0x10, 0x8e, 0xe1, 0xe8, 0x3e, 0x64, 0x88, 0xef, 0xbb, 0xc3, 0xea, 0x92, 0xe4, - 0x99, 0x53, 0x79, 0x75, 0x81, 0xc0, 0x0a, 0x28, 0xd6, 0xc6, 0x7c, 0xea, 0x55, 0xd3, 0x73, 0xd6, - 0xd6, 0xf1, 0xa9, 0x87, 0x25, 0x4c, 0xc0, 0x5d, 0x46, 0xec, 0x6a, 0x66, 0x0e, 0x7c, 0x9f, 0x11, - 0x1b, 0x4b, 0x98, 0x58, 0xcf, 0x99, 0x3b, 0xe0, 0xe7, 0xd5, 0xec, 0x9c, 0xf5, 0xec, 0x0a, 0x04, - 0x56, 0x40, 0xc1, 0xe0, 0x21, 0x0b, 0x68, 0x35, 0x37, 0x87, 0x71, 0x2c, 0x10, 0x58, 0x01, 0x51, - 0x13, 0x4a, 0x3c, 0x24, 0x41, 0xd8, 0xb5, 0x58, 0xbf, 0xef, 0x84, 0xd5, 0xbc, 0x24, 0xae, 0xcf, - 0x20, 0x92, 0x20, 0x6c, 0x4a, 0x1c, 0x2e, 0xf2, 0xd1, 0x00, 0x35, 0xa0, 0x48, 0xac, 0x0b, 0x8f, - 0xbd, 0x70, 0xa9, 0xdd, 0xa3, 0xd5, 0xc2, 0x1c, 0x1f, 0xf5, 0x11, 0x0e, 0x27, 0x49, 0xe8, 0x3d, - 0xc8, 0x3b, 0x5e, 0x48, 0x03, 0x8f, 0xb8, 0x55, 0x7b, 0xdd, 0xd8, 0x2c, 0xe1, 0xc2, 0x87, 0x91, - 0xc1, 0xfc, 0x9d, 0x01, 0x69, 0x71, 0xc6, 0xe8, 0x10, 0xae, 0x5b, 0xcc, 0xf3, 0xa8, 0x15, 0xb2, - 0xa0, 0x1b, 0x0e, 0x7d, 0x2a, 0xd3, 0xe2, 0xfa, 0xf6, 0xc7, 0x35, 0x99, 0x53, 0x07, 0xf1, 0x1b, - 0x49, 0xe8, 0x30, 0x4f, 0x50, 0x6a, 0xcd, 0x08, 0xff, 0x64, 0xe8, 0x53, 0xbc, 0x6c, 0x25, 0x87, - 0xe8, 0x21, 0x14, 0x2d, 0xe6, 0x9d, 0x39, 0xbd, 0xee, 0x33, 0xce, 0x3c, 0x99, 0x30, 0xa5, 0xc6, - 0xad, 0xef, 0x5e, 0xdf, 0xa9, 0x52, 0xcf, 0x62, 0xb6, 0xe3, 0xf5, 0xb6, 0xc4, 0x44, 0x0d, 0x93, - 0x17, 0x07, 0x94, 0x73, 0xd2, 0xa3, 0x38, 0xab, 0x08, 0xe6, 0x5f, 0xb2, 0x90, 0x8f, 0x92, 0x08, - 0x7d, 0x09, 0x69, 0x8f, 0xf4, 0xd5, 0x6a, 0x0a, 0x8d, 0xc7, 0xdf, 0xbd, 0xbe, 0xf3, 0xb0, 0xe7, - 0x84, 0xe7, 0x83, 0xd3, 0x9a, 0xc5, 0xfa, 0x5b, 0x94, 0x87, 0x03, 0x12, 0x0c, 0x55, 0xf2, 0xbf, - 0x71, 0x1d, 0x26, 0x57, 0x8d, 0xa5, 0xab, 0x29, 0xa1, 0xa6, 0xde, 0x66, 0xa8, 0x4b, 0x8b, 0x87, - 0x8a, 0xea, 0x90, 0x3f, 0x75, 0x3c, 0x01, 0xe1, 0xd5, 0xf4, 0xfa, 0xd2, 0x66, 0x71, 0xfb, 0xa3, - 0xb9, 0x77, 0xaa, 0xd6, 0x50, 0x68, 0x1c, 0xd3, 0xd0, 0x3e, 0x54, 0x5c, 0xc2, 0xc3, 0x6e, 0x7f, - 0x7c, 0xd9, 0xf1, 0x55, 0x98, 0x15, 0x13, 0x7e, 0x47, 0xd0, 0x26, 0x26, 0xd0, 0x5d, 0x28, 0x49, - 0x6f, 0xcf, 0x69, 0xc0, 0x85, 0x17, 0x71, 0x41, 0x0a, 0xb8, 0x28, 0x6c, 0x4f, 0x95, 0xc9, 0xfc, - 0xfd, 0x12, 0xe4, 0xf4, 0x32, 0xd0, 0x1e, 0x54, 0x02, 0xca, 0xd9, 0x20, 0xb0, 0x68, 0x37, 0xb9, - 0x07, 0xc6, 0x02, 0x7b, 0x70, 0x3d, 0x62, 0x36, 0xd5, 0x5e, 0x3c, 0x02, 0xb0, 0x98, 0xeb, 0x52, - 0x4b, 0x2e, 0x5f, 0x55, 0x98, 0x8a, 0x5a, 0x7e, 0x33, 0xb6, 0x8b, 0x95, 0x37, 0xd2, 0xaf, 0x5e, - 0xdf, 0xb9, 0x86, 0x13, 0x68, 0xf4, 0x4b, 0x03, 0xd6, 0xce, 0x1c, 0xea, 0xda, 0xc9, 0x55, 0x74, - 0xfb, 0xc4, 0xaf, 0x2e, 0xc9, 0x5d, 0x7d, 0xbc, 0xd0, 0xae, 0xd6, 0x76, 0x85, 0x0b, 0xb5, 0x9c, - 0x3d, 0xce, 0xbc, 0x03, 0xe2, 0xb7, 0xbc, 0x30, 0x18, 0x36, 0x6e, 0xfd, 0xfa, 0x1f, 0x73, 0x02, - 0x29, 0x9e, 0x8d, 0x68, 0xc8, 0x84, 0xfc, 0x29, 0xb1, 0x2e, 0xce, 0x1c, 0xd7, 0x95, 0xc5, 0x6b, - 0x19, 0xc7, 0x63, 0x74, 0x13, 0xf2, 0xbd, 0x80, 0x0d, 0xfc, 0xee, 0xe9, 0xb0, 0x9a, 0x59, 0x5f, - 0xda, 0x2c, 0xe0, 0x9c, 0x1c, 0x37, 0x86, 0x66, 0x0b, 0x6e, 0xcc, 0x78, 0x39, 0x2a, 0xc3, 0xd2, - 0x05, 0x1d, 0xaa, 0x0b, 0x80, 0xc5, 0x23, 0xaa, 0x40, 0xe6, 0x39, 0x71, 0x07, 0x2a, 0x6f, 0x4b, - 0x58, 0x0d, 0x1e, 0xa5, 0x3e, 0x37, 0xcc, 0xdf, 0xa6, 0x20, 0x23, 0xeb, 0x28, 0x6a, 0xc2, 0xca, - 0x64, 0x46, 0x18, 0x97, 0x65, 0xc4, 0x24, 0x03, 0x55, 0x21, 0x17, 0x25, 0x42, 0x4a, 0xbe, 0x3e, - 0x1a, 0xce, 0xcc, 0xba, 0xf4, 0x5b, 0xc9, 0xba, 0xcc, 0x1b, 0x59, 0x87, 0x3e, 0x03, 0xe0, 0x21, - 0x09, 0xa9, 0xca, 0xaf, 0xec, 0x02, 0xf9, 0x95, 0x91, 0x78, 0xf3, 0x37, 0x29, 0x48, 0x8b, 0x4e, - 0xf1, 0xff, 0xde, 0x91, 0x8f, 0x20, 0x13, 0x10, 0xaf, 0x47, 0x75, 0x8f, 0x5b, 0x51, 0x4e, 0xb1, - 0x30, 0x49, 0x57, 0x6a, 0x76, 0x22, 0x8e, 0xf4, 0xc2, 0x71, 0xa0, 0x5d, 0x40, 0x9c, 0x12, 0x97, - 0x8e, 0xa5, 0xb8, 0xdc, 0xa9, 0xcb, 0x1c, 0x94, 0x14, 0x4f, 0xa5, 0x96, 0x19, 0x42, 0x5a, 0x74, - 0x42, 0x11, 0x89, 0xae, 0x21, 0x72, 0x1b, 0x96, 0x71, 0x34, 0x44, 0x0f, 0x20, 0x7f, 0x41, 0x87, - 0x8b, 0xd7, 0x6d, 0x99, 0x93, 0xef, 0x03, 0x08, 0x92, 0x4f, 0xac, 0x0b, 0x6a, 0xab, 0x1a, 0x88, - 0x0b, 0x17, 0x74, 0x78, 0x24, 0x0d, 0x66, 0x07, 0x32, 0xb2, 0x9f, 0xca, 0x30, 0x64, 0xfc, 0x3e, - 0x09, 0xad, 0x73, 0xca, 0x17, 0xaf, 0x17, 0x25, 0xc9, 0x3b, 0x52, 0x34, 0xf3, 0xaf, 0x29, 0xc8, - 0xc8, 0x7e, 0xfb, 0xfd, 0x06, 0x22, 0x8a, 0xbd, 0xbc, 0x6e, 0x7c, 0xf1, 0x03, 0xcc, 0x2a, 0x02, - 0xfa, 0x00, 0x96, 0x35, 0x55, 0x3b, 0x97, 0x87, 0x87, 0x4b, 0xca, 0xa8, 0xfd, 0x3f, 0x80, 0xbc, - 0xcd, 0xac, 0xc5, 0xb3, 0x7c, 0xc9, 0x66, 0x16, 0x7a, 0x17, 0xb2, 0xf4, 0xa5, 0xc3, 0x43, 0x2e, - 0xe5, 0x49, 0x1e, 0xeb, 0x91, 0xb0, 0xdb, 0xd4, 0xa5, 0x21, 0x95, 0xea, 0x23, 0x8f, 0xf5, 0xc8, - 0xfc, 0xc6, 0x80, 0x62, 0x42, 0x73, 0xa0, 0x26, 0xa0, 0x60, 0xe0, 0x85, 0x4e, 0x9f, 0x76, 0xad, - 0x73, 0x6a, 0x5d, 0xf8, 0xcc, 0xf1, 0x42, 0x7d, 0x3b, 0x2a, 0xb5, 0x48, 0x88, 0xd6, 0x9a, 0xf1, - 0x1c, 0x5e, 0xd5, 0xf8, 0x91, 0x69, 0xc6, 0xc9, 0xa6, 0xae, 0x7c, 0xb2, 0x27, 0x50, 0x4c, 0x68, - 0x99, 0xb7, 0x95, 0x30, 0x1b, 0xdf, 0x20, 0xc8, 0x63, 0xca, 0x7d, 0xe6, 0x71, 0x8a, 0x6a, 0x63, - 0xd2, 0x77, 0x52, 0xcd, 0x29, 0x50, 0x52, 0xfb, 0x3e, 0x86, 0x42, 0x24, 0x66, 0x6d, 0xdd, 0x9a, - 0xee, 0x4c, 0x27, 0x45, 0x3d, 0xc5, 0xc6, 0x23, 0x06, 0xfa, 0x0c, 0x72, 0x42, 0xd6, 0x3a, 0x3a, - 0xa1, 0xde, 0x54, 0xce, 0x9a, 0x5c, 0x57, 0x20, 0x1c, 0xa1, 0xd1, 0xa7, 0x90, 0x15, 0xfa, 0x96, - 0xda, 0xba, 0xb0, 0xde, 0x9a, 0xce, 0xeb, 0x48, 0x0c, 0xd6, 0x58, 0xc1, 0x12, 0x32, 0x97, 0x46, - 0x7a, 0x78, 0x06, 0x6b, 0x5f, 0x62, 0xb0, 0xc6, 0x8a, 0x45, 0x4a, 0xad, 0x4b, 0x6d, 0x2d, 0x8b, - 0x67, 0x2c, 0x72, 0x57, 0x81, 0x70, 0x84, 0x46, 0x7b, 0x70, 0x5d, 0x6a, 0x56, 0x59, 0x9a, 0xa4, - 0xd6, 0x55, 0x22, 0xf9, 0x83, 0x19, 0xdb, 0xaa, 0xb0, 0x5a, 0xee, 0x2e, 0xf3, 0xe4, 0x10, 0xed, - 0x42, 0x29, 0xa1, 0x5d, 0x6d, 0xad, 0x9a, 0x37, 0x66, 0x6c, 0x57, 0x02, 0x89, 0xc7, 0x78, 0xf3, - 0x45, 0xef, 0x1f, 0x52, 0x5a, 0xf4, 0x9a, 0x90, 0x8f, 0x14, 0xa3, 0xae, 0x1d, 0xf1, 0x58, 0xe4, - 0x9d, 0x2e, 0xb4, 0xdc, 0x3a, 0xa7, 0x7d, 0x72, 0x85, 0x74, 0x56, 0xbc, 0x63, 0x49, 0x43, 0x5f, - 0xc1, 0x7b, 0x93, 0x12, 0x29, 0xe9, 0x70, 0x11, 0xb5, 0x58, 0x19, 0x57, 0x4a, 0xda, 0xf1, 0x0f, - 0x60, 0xd5, 0x66, 0xd6, 0xa0, 0x4f, 0xbd, 0x50, 0xf6, 0xa6, 0xee, 0x20, 0x50, 0x92, 0xa3, 0x80, - 0xcb, 0x63, 0x13, 0x27, 0x81, 0x8b, 0x3e, 0x84, 0x2c, 0x23, 0x83, 0xf0, 0x7c, 0x5b, 0xa7, 0x44, - 0x49, 0xb5, 0xa7, 0x4e, 0x5d, 0xd8, 0xb0, 0x9e, 0xdb, 0x4b, 0xe7, 0xb3, 0xe5, 0x9c, 0xf9, 0x9f, - 0x1c, 0x14, 0xe2, 0x34, 0x46, 0xcd, 0x84, 0x44, 0x35, 0xa4, 0x98, 0xfa, 0xf8, 0x92, 0xcc, 0x7f, - 0x53, 0xa4, 0x9a, 0x2f, 0xa1, 0x72, 0x14, 0xb0, 0x67, 0x4a, 0xad, 0x35, 0x99, 0xc7, 0xc3, 0x80, - 0x88, 0x9a, 0x51, 0x81, 0x8c, 0x14, 0x4f, 0x5a, 0xdd, 0xa8, 0x01, 0xda, 0x13, 0x4a, 0x30, 0xc2, - 0xe8, 0xeb, 0x76, 0xef, 0xb2, 0x97, 0x8e, 0xbc, 0xe2, 0x04, 0xdb, 0xfc, 0x73, 0x0a, 0x20, 0xf1, - 0xc2, 0x26, 0xa4, 0x13, 0x8a, 0x7f, 0x6b, 0x71, 0xa7, 0x35, 0xa9, 0xfc, 0x25, 0x59, 0x94, 0xd5, - 0x80, 0x92, 0xe8, 0xf4, 0x0a, 0x58, 0x8f, 0x84, 0x8c, 0x39, 0x63, 0xae, 0x4d, 0xed, 0xae, 0x0a, - 0x4a, 0x1d, 0x46, 0x51, 0xd9, 0xa4, 0xbc, 0xdb, 0xf8, 0x93, 0x01, 0x69, 0xf9, 0xd1, 0x50, 0x84, - 0x5c, 0xfb, 0xf0, 0x69, 0x7d, 0xbf, 0xbd, 0x53, 0xbe, 0x86, 0x10, 0x5c, 0xdf, 0x6d, 0xb7, 0xf6, - 0x77, 0xba, 0xb8, 0xf5, 0xe5, 0x49, 0x1b, 0xb7, 0x76, 0xca, 0x06, 0x5a, 0x83, 0xd5, 0xfd, 0x4e, - 0xb3, 0xfe, 0xa4, 0xdd, 0x39, 0x1c, 0x99, 0x53, 0xa8, 0x0a, 0x95, 0x84, 0xb9, 0xd9, 0x39, 0x38, - 0x68, 0x1d, 0xee, 0xb4, 0x76, 0xca, 0x4b, 0x23, 0x27, 0x9d, 0x23, 0x31, 0x5b, 0xdf, 0x2f, 0xa7, - 0xd1, 0x3b, 0xb0, 0xa2, 0x6c, 0xbb, 0x1d, 0xdc, 0x68, 0xef, 0xec, 0xb4, 0x0e, 0xcb, 0x19, 0x54, - 0x86, 0x52, 0xfb, 0xb0, 0xd9, 0x39, 0x38, 0xaa, 0x3f, 0x69, 0x37, 0xf6, 0x5b, 0xe5, 0x2c, 0x5a, - 0x85, 0xe5, 0x93, 0xc3, 0xe3, 0xfa, 0x93, 0xf6, 0xf1, 0x6e, 0xbb, 0x2e, 0x4c, 0x39, 0xf3, 0xdf, - 0x09, 0x95, 0xff, 0x23, 0xb8, 0x61, 0x11, 0x4e, 0xbb, 0x8e, 0xc7, 0xa9, 0xc7, 0x9d, 0xd0, 0x79, - 0x4e, 0x55, 0x84, 0x5c, 0x66, 0x53, 0x1e, 0xaf, 0x89, 0xe9, 0xf6, 0x68, 0x56, 0xc6, 0xca, 0xd1, - 0xd7, 0xf2, 0xc3, 0x48, 0x6f, 0x60, 0x94, 0x3d, 0x9f, 0x2f, 0x98, 0x3d, 0x89, 0xbd, 0xe7, 0x52, - 0x08, 0xe3, 0xa4, 0x33, 0xd1, 0x4c, 0xe3, 0x6b, 0xe5, 0x93, 0xf0, 0xbc, 0x9a, 0x92, 0x82, 0xba, - 0x14, 0x19, 0x8f, 0x48, 0x78, 0x2e, 0x40, 0x36, 0x75, 0x43, 0xd2, 0x1d, 0xf8, 0xc2, 0x37, 0x97, - 0xe7, 0x95, 0xc7, 0x25, 0x69, 0x3c, 0x51, 0x36, 0x54, 0x03, 0xe0, 0x34, 0xe8, 0xfa, 0xcc, 0x75, - 0xac, 0xa1, 0xae, 0xb3, 0x5a, 0xbd, 0x1d, 0xd3, 0xe0, 0x48, 0x9a, 0x71, 0x81, 0x47, 0x8f, 0xe8, - 0x02, 0xde, 0xf5, 0xe3, 0x5c, 0xee, 0x26, 0x03, 0xcc, 0xca, 0x00, 0x3f, 0xbd, 0x2c, 0xc0, 0x69, - 0x37, 0x01, 0xaf, 0xf9, 0x53, 0xac, 0xdc, 0x7c, 0x06, 0xe5, 0xc9, 0x7d, 0x98, 0xf2, 0x41, 0xf0, - 0x93, 0xe4, 0x07, 0xc1, 0xd5, 0xee, 0x4a, 0xe2, 0xe3, 0xc1, 0x86, 0x9c, 0x6e, 0x40, 0xe8, 0x13, - 0x40, 0x44, 0xc5, 0x67, 0x53, 0x6e, 0x05, 0x8e, 0x1f, 0xcb, 0xe5, 0x02, 0x5e, 0x55, 0x33, 0x3b, - 0xa3, 0x09, 0x74, 0x0f, 0x94, 0x48, 0x9d, 0xfc, 0x6a, 0xd3, 0x5f, 0xc9, 0xc7, 0x62, 0x2e, 0xd2, - 0xe3, 0xbf, 0x32, 0x20, 0xab, 0xfa, 0xd5, 0xdb, 0x91, 0x1d, 0x8f, 0xe0, 0xa6, 0xed, 0x70, 0x72, - 0xea, 0xd2, 0xae, 0x68, 0x64, 0x5d, 0xe6, 0x87, 0x4e, 0x3f, 0x12, 0xf8, 0x29, 0x79, 0xde, 0x37, - 0x34, 0x40, 0x34, 0xbc, 0x4e, 0x62, 0xda, 0xfc, 0x0a, 0xb2, 0xaa, 0x09, 0xce, 0x17, 0x91, 0xb1, - 0x20, 0x4b, 0x2d, 0x28, 0xc8, 0xcc, 0x1f, 0x42, 0x4e, 0xb7, 0xc9, 0xd1, 0xde, 0x18, 0x97, 0xef, - 0xcd, 0x17, 0xb0, 0x3c, 0xd6, 0x1d, 0xaf, 0x44, 0x7e, 0x04, 0xa5, 0x64, 0x43, 0xbc, 0x0a, 0x77, - 0xe3, 0x17, 0x69, 0xc8, 0xb4, 0x5e, 0x86, 0x01, 0x31, 0xff, 0x66, 0xc0, 0xdd, 0x28, 0x51, 0x5a, - 0x42, 0x45, 0x3a, 0x5e, 0x6f, 0x94, 0xb0, 0xd1, 0x5f, 0x87, 0xfb, 0x50, 0xa6, 0x7a, 0xb2, 0x9b, - 0xdc, 0xb7, 0xe2, 0xf6, 0xdd, 0xd9, 0x7f, 0xa2, 0x44, 0x6d, 0x61, 0x25, 0xa2, 0x46, 0xf5, 0xe5, - 0x08, 0xca, 0x7e, 0xc0, 0x7c, 0xc6, 0xa9, 0x1d, 0x7b, 0x53, 0x99, 0xb4, 0xe0, 0xbf, 0x21, 0x2b, - 0x11, 0x5d, 0x1b, 0x44, 0xd5, 0x8f, 0xa3, 0xd0, 0xb6, 0x7a, 0x8f, 0x38, 0x1e, 0x0f, 0x13, 0xb7, - 0x09, 0x7d, 0x31, 0x7e, 0xe8, 0x0b, 0x2d, 0x3e, 0xce, 0x8b, 0xde, 0x78, 0x71, 0x4b, 0xc9, 0xbb, - 0xdf, 0x1a, 0x5b, 0xaf, 0xdc, 0xd1, 0xda, 0xa5, 0xeb, 0x98, 0x5f, 0xe9, 0xbe, 0xcf, 0x12, 0xb0, - 0xfd, 0x53, 0x28, 0xc4, 0x09, 0x82, 0x7e, 0x0c, 0xc5, 0xd1, 0x4e, 0x50, 0x54, 0x99, 0x76, 0x16, - 0xe6, 0xda, 0xd4, 0x17, 0x6d, 0x1a, 0xf7, 0x8d, 0x46, 0xe3, 0xd5, 0x3f, 0x6f, 0x5f, 0x7b, 0xf5, - 0xed, 0x6d, 0xe3, 0xef, 0xdf, 0xde, 0x36, 0xfe, 0xf8, 0xaf, 0xdb, 0xc6, 0xd7, 0xf7, 0x17, 0xfa, - 0xeb, 0x2e, 0xe1, 0xf0, 0x34, 0x2b, 0xcd, 0x0f, 0xfe, 0x17, 0x00, 0x00, 0xff, 0xff, 0x70, 0x5e, - 0xcc, 0xaf, 0x47, 0x17, 0x00, 0x00, + 0x15, 0xf7, 0xf2, 0x3f, 0x1f, 0x29, 0x89, 0x9a, 0xd0, 0x31, 0xbd, 0x71, 0x2c, 0x59, 0x49, 0x10, + 0xc1, 0x41, 0x28, 0x43, 0x4e, 0x1b, 0xdb, 0x81, 0x8b, 0x92, 0x14, 0x05, 0x50, 0x95, 0x44, 0x79, + 0x24, 0x39, 0x40, 0x2e, 0xc4, 0x72, 0x77, 0x44, 0xad, 0xb5, 0xdc, 0xd9, 0xee, 0x2c, 0x6d, 0xb3, + 0xf7, 0xa2, 0x68, 0x81, 0x02, 0xed, 0xb1, 0x41, 0x0f, 0x3d, 0xf5, 0x1b, 0x14, 0x28, 0x7a, 0xed, + 0xc5, 0x40, 0x2f, 0xfd, 0x04, 0x2e, 0x9a, 0x7e, 0x81, 0x1e, 0xda, 0x4b, 0x4e, 0xc1, 0xfc, 0xd9, + 0xe5, 0x92, 0x26, 0x29, 0x0a, 0x70, 0x72, 0x21, 0x76, 0xde, 0xfc, 0x7e, 0x6f, 0xdf, 0xbc, 0x7d, + 0xf3, 0xfe, 0x10, 0xee, 0xf6, 0xe8, 0x96, 0xe7, 0xd3, 0x80, 0x9a, 0xd4, 0x61, 0x5b, 0x7d, 0x23, + 0x20, 0xbe, 0x6d, 0x38, 0xf6, 0x2f, 0x48, 0xfc, 0xb9, 0x2a, 0x10, 0xa8, 0x10, 0x13, 0xe9, 0xeb, + 0x26, 0x75, 0xd9, 0xa0, 0x4f, 0xfc, 0x88, 0x1e, 0x3d, 0x48, 0xb8, 0x7e, 0x6b, 0x4c, 0xf5, 0x99, + 0x43, 0x5f, 0x88, 0x1f, 0xb5, 0x5b, 0xee, 0xd1, 0x1e, 0x15, 0x8f, 0x5b, 0xfc, 0x49, 0x49, 0xd7, + 0x7a, 0x94, 0xf6, 0x1c, 0x22, 0x79, 0xdd, 0xc1, 0xd9, 0x56, 0x60, 0xf7, 0x09, 0x0b, 0x8c, 0xbe, + 0x27, 0x01, 0x1b, 0xff, 0xab, 0x40, 0x16, 0x93, 0x9f, 0x0f, 0x08, 0x0b, 0xd0, 0xa7, 0x90, 0x62, + 0x1e, 0x31, 0x2b, 0xda, 0xba, 0xb6, 0x59, 0xd8, 0xbe, 0x59, 0x8d, 0x5b, 0xac, 0x30, 0xd5, 0x63, + 0x8f, 0x98, 0x58, 0xc0, 0xd0, 0x43, 0xc8, 0x3d, 0x37, 0x1c, 0xdb, 0x32, 0x02, 0x52, 0x49, 0x08, + 0xca, 0xfb, 0x53, 0x29, 0x4f, 0x15, 0x08, 0x47, 0x70, 0x74, 0x0f, 0xd2, 0x86, 0xe7, 0x39, 0xc3, + 0x4a, 0x52, 0xf0, 0xf4, 0xa9, 0xbc, 0x1a, 0x47, 0x60, 0x09, 0xe4, 0xb6, 0x51, 0x8f, 0xb8, 0x95, + 0xd4, 0x1c, 0xdb, 0xda, 0x1e, 0x71, 0xb1, 0x80, 0x71, 0xb8, 0x43, 0x0d, 0xab, 0x92, 0x9e, 0x03, + 0xdf, 0xa7, 0x86, 0x85, 0x05, 0x8c, 0xdb, 0x73, 0xe6, 0x0c, 0xd8, 0x79, 0x25, 0x33, 0xc7, 0x9e, + 0x5d, 0x8e, 0xc0, 0x12, 0xc8, 0x19, 0x2c, 0xa0, 0x3e, 0xa9, 0x64, 0xe7, 0x30, 0x8e, 0x39, 0x02, + 0x4b, 0x20, 0x6a, 0x40, 0x91, 0x05, 0x86, 0x1f, 0x74, 0x4c, 0xda, 0xef, 0xdb, 0x41, 0x25, 0x27, + 0x88, 0xeb, 0x33, 0x88, 0x86, 0x1f, 0x34, 0x04, 0x0e, 0x17, 0xd8, 0x68, 0x81, 0xea, 0x50, 0x30, + 0xcc, 0x0b, 0x97, 0xbe, 0x70, 0x88, 0xd5, 0x23, 0x95, 0xfc, 0x1c, 0x1d, 0xb5, 0x11, 0x0e, 0xc7, + 0x49, 0xe8, 0x3d, 0xc8, 0xd9, 0x6e, 0x40, 0x7c, 0xd7, 0x70, 0x2a, 0xd6, 0xba, 0xb6, 0x59, 0xc4, + 0xf9, 0x0f, 0x43, 0x81, 0xfe, 0x7b, 0x0d, 0x52, 0xfc, 0x1b, 0xa3, 0x43, 0x58, 0x36, 0xa9, 0xeb, + 0x12, 0x33, 0xa0, 0x7e, 0x27, 0x18, 0x7a, 0x44, 0x84, 0xc5, 0xf2, 0xf6, 0xc7, 0x55, 0x11, 0x74, + 0x07, 0xd1, 0x1b, 0x8d, 0xc0, 0xa6, 0x2e, 0xa7, 0x54, 0x1b, 0x21, 0xfe, 0x64, 0xe8, 0x11, 0xbc, + 0x64, 0xc6, 0x97, 0xe8, 0x21, 0x14, 0x4c, 0xea, 0x9e, 0xd9, 0xbd, 0xce, 0x33, 0x46, 0x5d, 0x11, + 0x30, 0xc5, 0xfa, 0xad, 0x6f, 0x5f, 0xaf, 0x55, 0x88, 0x6b, 0x52, 0xcb, 0x76, 0x7b, 0x5b, 0x7c, + 0xa3, 0x8a, 0x8d, 0x17, 0x07, 0x84, 0x31, 0xa3, 0x47, 0x70, 0x46, 0x12, 0xf4, 0xbf, 0x66, 0x20, + 0x17, 0x06, 0x11, 0x7a, 0x02, 0x29, 0xd7, 0xe8, 0x4b, 0x6b, 0xf2, 0xf5, 0xc7, 0xdf, 0xbe, 0x5e, + 0x7b, 0xd8, 0xb3, 0x83, 0xf3, 0x41, 0xb7, 0x6a, 0xd2, 0xfe, 0x16, 0x61, 0xc1, 0xc0, 0xf0, 0x87, + 0xf2, 0x76, 0xbc, 0x71, 0x5f, 0x26, 0xad, 0xc6, 0x42, 0xd5, 0x94, 0xa3, 0x26, 0xde, 0xe6, 0x51, + 0x93, 0x8b, 0x1f, 0x15, 0xd5, 0x20, 0xd7, 0xb5, 0x5d, 0x0e, 0x61, 0x95, 0xd4, 0x7a, 0x72, 0xb3, + 0xb0, 0xfd, 0xd1, 0xdc, 0x3b, 0x55, 0xad, 0x4b, 0x34, 0x8e, 0x68, 0x68, 0x1f, 0xca, 0x8e, 0xc1, + 0x82, 0x4e, 0x7f, 0xdc, 0xec, 0xe8, 0x2a, 0xcc, 0x3a, 0x13, 0x7e, 0x87, 0xd3, 0x26, 0x36, 0xd0, + 0x1d, 0x28, 0x0a, 0x6d, 0xcf, 0x89, 0xcf, 0xb8, 0x16, 0x7e, 0x41, 0xf2, 0xb8, 0xc0, 0x65, 0x4f, + 0xa5, 0x48, 0xff, 0x43, 0x12, 0xb2, 0xca, 0x0c, 0xb4, 0x07, 0x65, 0x9f, 0x30, 0x3a, 0xf0, 0x4d, + 0xd2, 0x89, 0xfb, 0x40, 0x5b, 0xc0, 0x07, 0xcb, 0x21, 0xb3, 0x21, 0x7d, 0xf1, 0x08, 0xc0, 0xa4, + 0x8e, 0x43, 0x4c, 0x61, 0xbe, 0xcc, 0x30, 0x65, 0x69, 0x7e, 0x23, 0x92, 0x73, 0xcb, 0xeb, 0xa9, + 0x57, 0xaf, 0xd7, 0xae, 0xe1, 0x18, 0x1a, 0xfd, 0x4a, 0x83, 0xeb, 0x67, 0x36, 0x71, 0xac, 0xb8, + 0x15, 0x9d, 0xbe, 0xe1, 0x55, 0x92, 0xc2, 0xab, 0x8f, 0x17, 0xf2, 0x6a, 0x75, 0x97, 0xab, 0x90, + 0xe6, 0xec, 0x31, 0xea, 0x1e, 0x18, 0x5e, 0xd3, 0x0d, 0xfc, 0x61, 0xfd, 0xd6, 0x6f, 0xfe, 0x35, + 0xe7, 0x20, 0x85, 0xb3, 0x11, 0x0d, 0xe9, 0x90, 0xeb, 0x1a, 0xe6, 0xc5, 0x99, 0xed, 0x38, 0x22, + 0x79, 0x2d, 0xe1, 0x68, 0x8d, 0x6e, 0x42, 0xae, 0xe7, 0xd3, 0x81, 0xd7, 0xe9, 0x0e, 0x2b, 0xe9, + 0xf5, 0xe4, 0x66, 0x1e, 0x67, 0xc5, 0xba, 0x3e, 0xd4, 0x9b, 0x70, 0x63, 0xc6, 0xcb, 0x51, 0x09, + 0x92, 0x17, 0x64, 0x28, 0x2f, 0x00, 0xe6, 0x8f, 0xa8, 0x0c, 0xe9, 0xe7, 0x86, 0x33, 0x90, 0x71, + 0x5b, 0xc4, 0x72, 0xf1, 0x28, 0xf1, 0x40, 0xd3, 0x7f, 0x97, 0x80, 0xb4, 0xc8, 0xa3, 0xa8, 0x01, + 0x2b, 0x93, 0x11, 0xa1, 0x5d, 0x16, 0x11, 0x93, 0x0c, 0x54, 0x81, 0x6c, 0x18, 0x08, 0x09, 0xf1, + 0xfa, 0x70, 0x39, 0x33, 0xea, 0x52, 0x6f, 0x25, 0xea, 0xd2, 0x6f, 0x44, 0x1d, 0xfa, 0x1c, 0x80, + 0x05, 0x46, 0x40, 0x64, 0x7c, 0x65, 0x16, 0x88, 0xaf, 0xb4, 0xc0, 0xeb, 0xbf, 0x4d, 0x40, 0x8a, + 0x57, 0x8a, 0xef, 0xdb, 0x23, 0x1f, 0x41, 0xda, 0x37, 0xdc, 0x1e, 0x51, 0x35, 0x6e, 0x45, 0x2a, + 0xc5, 0x5c, 0x24, 0x54, 0xc9, 0xdd, 0x89, 0x73, 0xa4, 0x16, 0x3e, 0x07, 0xda, 0x05, 0xc4, 0x88, + 0xe1, 0x90, 0xb1, 0x10, 0x17, 0x9e, 0xba, 0x4c, 0x41, 0x51, 0xf2, 0x64, 0x68, 0xe9, 0x01, 0xa4, + 0x78, 0x25, 0xe4, 0x27, 0x51, 0x39, 0x44, 0xb8, 0x61, 0x09, 0x87, 0x4b, 0x74, 0x1f, 0x72, 0x17, + 0x64, 0xb8, 0x78, 0xde, 0x16, 0x31, 0xf9, 0x3e, 0x00, 0x27, 0x79, 0x86, 0x79, 0x41, 0x2c, 0x99, + 0x03, 0x71, 0xfe, 0x82, 0x0c, 0x8f, 0x84, 0x40, 0xff, 0x47, 0x12, 0xd2, 0xa2, 0xa0, 0x8a, 0x73, + 0x08, 0x07, 0x78, 0x46, 0x60, 0x9e, 0x13, 0xb6, 0x78, 0xc2, 0x28, 0x0a, 0xde, 0x91, 0xa4, 0xa1, + 0x27, 0xb0, 0x12, 0x5e, 0xac, 0x4e, 0x97, 0xf4, 0x6c, 0x97, 0x55, 0x12, 0xe2, 0xae, 0x6f, 0xce, + 0xae, 0xe6, 0xd5, 0xba, 0x62, 0xd4, 0x39, 0x01, 0x2f, 0x77, 0xe3, 0x4b, 0x86, 0xbe, 0x02, 0x14, + 0xa9, 0x34, 0x69, 0xdf, 0x73, 0x48, 0x40, 0x98, 0xca, 0x20, 0x9f, 0x2c, 0xa0, 0xb5, 0xa1, 0x38, + 0x78, 0xb5, 0x3b, 0x21, 0x61, 0xba, 0x09, 0x4b, 0x63, 0x2f, 0x9f, 0xe3, 0xff, 0x07, 0x90, 0x8f, + 0xda, 0x36, 0x95, 0x07, 0xf5, 0xaa, 0x6c, 0xec, 0xaa, 0x61, 0x63, 0x57, 0x3d, 0x09, 0x11, 0x78, + 0x04, 0xd6, 0xcf, 0xa0, 0x34, 0x69, 0xcb, 0xf7, 0xf2, 0x9e, 0xbf, 0x25, 0x20, 0x2d, 0x9a, 0x9d, + 0x1f, 0x36, 0x8a, 0x78, 0xa5, 0x15, 0xb9, 0x8e, 0x2d, 0x7e, 0x7b, 0x32, 0x92, 0x80, 0x3e, 0x80, + 0x25, 0x45, 0x55, 0xca, 0xc5, 0xcd, 0xc1, 0x45, 0x29, 0x54, 0xfa, 0xef, 0x43, 0xce, 0xa2, 0xe6, + 0xe2, 0x29, 0x26, 0x69, 0x51, 0x13, 0xbd, 0x0b, 0x19, 0xf2, 0xd2, 0x66, 0x01, 0x13, 0xbd, 0x61, + 0x0e, 0xab, 0x15, 0x97, 0x5b, 0x84, 0x7f, 0x02, 0xd1, 0xfa, 0xe5, 0xb0, 0x5a, 0xe9, 0x5f, 0x6b, + 0x50, 0x88, 0x35, 0x7c, 0xa8, 0x01, 0xc8, 0x1f, 0xb8, 0xdc, 0xb9, 0x1d, 0xf3, 0x9c, 0x98, 0x17, + 0x1e, 0xb5, 0xdd, 0x40, 0xa5, 0xa6, 0x72, 0x35, 0x1c, 0x13, 0xaa, 0x8d, 0x68, 0x0f, 0xaf, 0x2a, + 0xfc, 0x48, 0x34, 0xe3, 0x56, 0x25, 0xae, 0x7a, 0xab, 0xf4, 0x53, 0x28, 0xc4, 0x1a, 0xc9, 0xb7, + 0x75, 0x59, 0x37, 0xbe, 0x46, 0x90, 0xc3, 0x84, 0x79, 0xd4, 0x65, 0x04, 0x55, 0xc7, 0xe6, 0x8e, + 0xc9, 0x56, 0x5a, 0x82, 0xe2, 0x83, 0xc7, 0x63, 0xc8, 0x87, 0x93, 0x84, 0xa5, 0xe2, 0x74, 0x6d, + 0x3a, 0x29, 0x2c, 0xe8, 0x16, 0x1e, 0x31, 0xd0, 0xe7, 0x90, 0xe5, 0x33, 0x85, 0xad, 0x02, 0xea, + 0xcd, 0xb1, 0x45, 0x91, 0x6b, 0x12, 0x84, 0x43, 0x34, 0xfa, 0x0c, 0x32, 0x7c, 0xb8, 0x20, 0x96, + 0xaa, 0x6a, 0xb7, 0xa6, 0xf3, 0xda, 0x02, 0x83, 0x15, 0x96, 0xb3, 0xf8, 0x8c, 0x41, 0xc2, 0x61, + 0x64, 0x06, 0x6b, 0x5f, 0x60, 0xb0, 0xc2, 0x72, 0x23, 0xc5, 0xa0, 0x41, 0x2c, 0x35, 0x93, 0xcc, + 0x30, 0x72, 0x57, 0x82, 0x70, 0x88, 0x46, 0x7b, 0xb0, 0x2c, 0x06, 0x06, 0x51, 0x17, 0xc4, 0xa0, + 0x21, 0x27, 0x94, 0x0f, 0x66, 0xb8, 0x55, 0x62, 0xd5, 0xac, 0xb1, 0xc4, 0xe2, 0x4b, 0xb4, 0x0b, + 0xc5, 0xd8, 0xe0, 0x60, 0xa9, 0x91, 0x65, 0x63, 0x86, 0xbb, 0x62, 0x48, 0x3c, 0xc6, 0x9b, 0x3f, + 0x71, 0xfc, 0x31, 0xa1, 0x26, 0x0e, 0x1d, 0x72, 0x61, 0xbb, 0xae, 0x72, 0x47, 0xb4, 0xe6, 0x71, + 0xa7, 0xaa, 0x1c, 0x33, 0xcf, 0x49, 0xdf, 0xb8, 0x42, 0x38, 0x4b, 0xde, 0xb1, 0xa0, 0xa1, 0x2f, + 0xe1, 0xbd, 0xc9, 0xfe, 0x34, 0xae, 0x70, 0x91, 0x56, 0xbd, 0x3c, 0xde, 0xa6, 0x2a, 0xc5, 0x9f, + 0xc0, 0xaa, 0x45, 0xcd, 0x41, 0x9f, 0xb8, 0x81, 0x68, 0x0c, 0x3a, 0x03, 0x5f, 0xf6, 0x7b, 0x79, + 0x5c, 0x1a, 0xdb, 0x38, 0xf5, 0x1d, 0xf4, 0x21, 0x64, 0xa8, 0x31, 0x08, 0xce, 0xb7, 0x55, 0x48, + 0x14, 0x65, 0x6f, 0xd0, 0xae, 0x71, 0x19, 0x56, 0x7b, 0x7b, 0xa9, 0x5c, 0xa6, 0x94, 0xd5, 0xff, + 0x9f, 0x85, 0x7c, 0x14, 0xc6, 0xa8, 0x11, 0x9b, 0x0f, 0x34, 0x51, 0x87, 0x3e, 0xbe, 0x24, 0xf2, + 0xdf, 0x9c, 0x10, 0xf4, 0x97, 0x50, 0x3e, 0xf2, 0xe9, 0x33, 0xd9, 0x2a, 0x37, 0xa8, 0xcb, 0x02, + 0xdf, 0xe0, 0x39, 0xa3, 0x0c, 0x69, 0xd1, 0xb9, 0xaa, 0xd6, 0x52, 0x2e, 0xd0, 0x1e, 0x6f, 0xc3, + 0x43, 0x8c, 0xba, 0x6e, 0x77, 0x2f, 0x7b, 0xe9, 0x48, 0x2b, 0x8e, 0xb1, 0xf5, 0xbf, 0x24, 0x00, + 0x62, 0x2f, 0x6c, 0x40, 0x2a, 0x36, 0x6e, 0x6d, 0x2d, 0xae, 0xb4, 0x2a, 0xc6, 0x2e, 0x41, 0xe6, + 0x69, 0xd5, 0x27, 0x46, 0xf8, 0xf5, 0xf2, 0x58, 0xad, 0x78, 0x0f, 0x79, 0x46, 0x1d, 0x8b, 0x58, + 0x1d, 0x79, 0x28, 0xf9, 0x31, 0x0a, 0x52, 0x26, 0x7a, 0xeb, 0x8d, 0x3f, 0x6b, 0x90, 0x12, 0x13, + 0x5b, 0x01, 0xb2, 0xad, 0xc3, 0xa7, 0xb5, 0xfd, 0xd6, 0x4e, 0xe9, 0x1a, 0x42, 0xb0, 0xbc, 0xdb, + 0x6a, 0xee, 0xef, 0x74, 0x70, 0xf3, 0xc9, 0x69, 0x0b, 0x37, 0x77, 0x4a, 0x1a, 0xba, 0x0e, 0xab, + 0xfb, 0xed, 0x46, 0xed, 0xa4, 0xd5, 0x3e, 0x1c, 0x89, 0x13, 0xa8, 0x02, 0xe5, 0x98, 0xb8, 0xd1, + 0x3e, 0x38, 0x68, 0x1e, 0xee, 0x34, 0x77, 0x4a, 0xc9, 0x91, 0x92, 0xf6, 0x11, 0xdf, 0xad, 0xed, + 0x97, 0x52, 0xe8, 0x1d, 0x58, 0x91, 0xb2, 0xdd, 0x36, 0xae, 0xb7, 0x76, 0x76, 0x9a, 0x87, 0xa5, + 0x34, 0x2a, 0x41, 0xb1, 0x75, 0xd8, 0x68, 0x1f, 0x1c, 0xd5, 0x4e, 0x5a, 0xf5, 0xfd, 0x66, 0x29, + 0x83, 0x56, 0x61, 0xe9, 0xf4, 0xf0, 0xb8, 0x76, 0xd2, 0x3a, 0xde, 0x6d, 0xd5, 0xb8, 0x28, 0xab, + 0xff, 0x37, 0x36, 0x62, 0xfd, 0x18, 0x6e, 0x98, 0x06, 0x23, 0x1d, 0xdb, 0x65, 0xc4, 0x65, 0x76, + 0x60, 0x3f, 0x27, 0xf2, 0x84, 0x4c, 0x44, 0x53, 0x0e, 0x5f, 0xe7, 0xdb, 0xad, 0xd1, 0xae, 0x38, + 0x2b, 0x6f, 0x66, 0x0a, 0xa3, 0x2f, 0x11, 0x46, 0xcf, 0x83, 0x05, 0xa3, 0x27, 0xe6, 0x7b, 0x26, + 0xa6, 0x10, 0x1c, 0x57, 0xc6, 0x8b, 0x69, 0x74, 0xad, 0x3c, 0x23, 0x38, 0x17, 0x9d, 0x57, 0x1e, + 0x17, 0x43, 0xe1, 0x91, 0x11, 0x9c, 0x73, 0x90, 0x45, 0x9c, 0xc0, 0xe8, 0x0c, 0x3c, 0xae, 0x9b, + 0x89, 0xef, 0x95, 0xc3, 0x45, 0x21, 0x3c, 0x95, 0x32, 0x54, 0x05, 0x60, 0xc4, 0xef, 0x78, 0xd4, + 0xb1, 0xcd, 0xa1, 0xca, 0xb3, 0xaa, 0x75, 0x3e, 0x26, 0xfe, 0x91, 0x10, 0xe3, 0x3c, 0x0b, 0x1f, + 0xd1, 0x05, 0xbc, 0xeb, 0x45, 0xb1, 0xdc, 0x89, 0x1f, 0x30, 0x23, 0x0e, 0xf8, 0xd9, 0x65, 0x07, + 0x9c, 0x76, 0x13, 0xf0, 0x75, 0x6f, 0x8a, 0x94, 0xe9, 0xcf, 0xa0, 0x34, 0xe9, 0x87, 0x29, 0xd3, + 0xd8, 0x4f, 0xe3, 0xd3, 0xd8, 0xd5, 0xee, 0x4a, 0x6c, 0x72, 0xb3, 0x20, 0xab, 0x0a, 0x10, 0xfa, + 0x14, 0x90, 0x21, 0xcf, 0x67, 0x11, 0x66, 0xfa, 0xb6, 0x17, 0xcd, 0x2a, 0x79, 0xbc, 0x2a, 0x77, + 0x76, 0x46, 0x1b, 0xe8, 0x2e, 0xc8, 0x09, 0x61, 0x72, 0x64, 0x56, 0x7f, 0x51, 0x1c, 0xf3, 0xbd, + 0x70, 0x18, 0xfa, 0xb5, 0x06, 0x19, 0x59, 0xaf, 0xde, 0x4e, 0xdb, 0xf1, 0x08, 0x6e, 0x5a, 0x36, + 0x33, 0xba, 0x0e, 0xe9, 0xf0, 0x42, 0xd6, 0xa1, 0x5e, 0x60, 0xf7, 0xc3, 0xe9, 0x2a, 0x21, 0xbe, + 0xf7, 0x0d, 0x05, 0xe0, 0x05, 0xaf, 0x1d, 0xdb, 0xd6, 0xbf, 0x84, 0x8c, 0x2c, 0x82, 0xf3, 0x9b, + 0xc8, 0xa8, 0x21, 0x4b, 0x2c, 0xd8, 0x90, 0xe9, 0x3f, 0x82, 0xac, 0x2a, 0x93, 0x23, 0xdf, 0x68, + 0x97, 0xfb, 0xe6, 0x0b, 0x58, 0x1a, 0xab, 0x8e, 0x57, 0x22, 0x3f, 0x82, 0x62, 0xbc, 0x20, 0x5e, + 0x85, 0xbb, 0xf1, 0xcb, 0x14, 0xa4, 0x9b, 0x2f, 0x03, 0xdf, 0xd0, 0xff, 0xae, 0xc1, 0x9d, 0x30, + 0x50, 0x9a, 0xbc, 0x8b, 0xb4, 0xdd, 0xde, 0x28, 0x60, 0xc3, 0xff, 0x6d, 0xf7, 0xa1, 0x44, 0xd4, + 0x66, 0x27, 0xee, 0xb7, 0xc2, 0xf6, 0x9d, 0xd9, 0xff, 0x60, 0x85, 0x65, 0x61, 0x25, 0xa4, 0x86, + 0xf9, 0xe5, 0x08, 0x4a, 0x9e, 0x4f, 0x3d, 0xca, 0x88, 0x15, 0x69, 0x93, 0x91, 0xb4, 0xe0, 0x5f, + 0x51, 0x2b, 0x21, 0x5d, 0x09, 0x78, 0xd6, 0x8f, 0x4e, 0xa1, 0x64, 0xb5, 0x9e, 0x61, 0xbb, 0x2c, + 0x88, 0xdd, 0x26, 0xf4, 0xc5, 0xf8, 0x47, 0x5f, 0xc8, 0xf8, 0x28, 0x2e, 0x7a, 0xe3, 0xc9, 0x4d, + 0x0e, 0x7e, 0xcd, 0x31, 0x7b, 0x85, 0x47, 0xab, 0x97, 0xda, 0x31, 0x3f, 0xd3, 0xfd, 0x90, 0x29, + 0x60, 0xfb, 0x67, 0x90, 0x8f, 0x02, 0x04, 0xfd, 0x04, 0x0a, 0x23, 0x4f, 0x10, 0x54, 0x9e, 0xf6, + 0x2d, 0xf4, 0xeb, 0x53, 0x5f, 0xb4, 0xa9, 0xdd, 0xd3, 0xea, 0xf5, 0x57, 0xff, 0xbe, 0x7d, 0xed, + 0xd5, 0x37, 0xb7, 0xb5, 0x7f, 0x7e, 0x73, 0x5b, 0xfb, 0xd3, 0x7f, 0x6e, 0x6b, 0x5f, 0xdd, 0x5b, + 0xe8, 0x7f, 0xd3, 0x98, 0xc2, 0x6e, 0x46, 0x88, 0xef, 0x7f, 0x17, 0x00, 0x00, 0xff, 0xff, 0x66, + 0x00, 0x64, 0x75, 0xe5, 0x18, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -2304,6 +2410,34 @@ func (m *Request_Flush) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.BackfillCompletes) > 0 { + for iNdEx := len(m.BackfillCompletes) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.BackfillCompletes[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMaterialize(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + } + if len(m.BackfillBegins) > 0 { + for iNdEx := len(m.BackfillBegins) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.BackfillBegins[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMaterialize(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } if len(m.StatePatchesJson) > 0 { i -= len(m.StatePatchesJson) copy(dAtA[i:], m.StatePatchesJson) @@ -2314,6 +2448,94 @@ func (m *Request_Flush) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *Request_Flush_BackfillBegin) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Request_Flush_BackfillBegin) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Request_Flush_BackfillBegin) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Timestamp != nil { + { + size, err := m.Timestamp.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMaterialize(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + if m.Binding != 0 { + i = encodeVarintMaterialize(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *Request_Flush_BackfillComplete) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Request_Flush_BackfillComplete) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Request_Flush_BackfillComplete) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Timestamp != nil { + { + size, err := m.Timestamp.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMaterialize(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + if m.Binding != 0 { + i = encodeVarintMaterialize(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *Request_Store) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) @@ -3568,6 +3790,56 @@ func (m *Request_Flush) ProtoSize() (n int) { if l > 0 { n += 1 + l + sovMaterialize(uint64(l)) } + if len(m.BackfillBegins) > 0 { + for _, e := range m.BackfillBegins { + l = e.ProtoSize() + n += 1 + l + sovMaterialize(uint64(l)) + } + } + if len(m.BackfillCompletes) > 0 { + for _, e := range m.BackfillCompletes { + l = e.ProtoSize() + n += 1 + l + sovMaterialize(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Request_Flush_BackfillBegin) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Binding != 0 { + n += 1 + sovMaterialize(uint64(m.Binding)) + } + if m.Timestamp != nil { + l = m.Timestamp.ProtoSize() + n += 1 + l + sovMaterialize(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Request_Flush_BackfillComplete) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Binding != 0 { + n += 1 + sovMaterialize(uint64(m.Binding)) + } + if m.Timestamp != nil { + l = m.Timestamp.ProtoSize() + n += 1 + l + sovMaterialize(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5704,6 +5976,286 @@ func (m *Request_Flush) Unmarshal(dAtA []byte) error { m.StatePatchesJson = []byte{} } iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BackfillBegins", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMaterialize + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMaterialize + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthMaterialize + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BackfillBegins = append(m.BackfillBegins, &Request_Flush_BackfillBegin{}) + if err := m.BackfillBegins[len(m.BackfillBegins)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BackfillCompletes", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMaterialize + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMaterialize + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthMaterialize + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BackfillCompletes = append(m.BackfillCompletes, &Request_Flush_BackfillComplete{}) + if err := m.BackfillCompletes[len(m.BackfillCompletes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMaterialize(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthMaterialize + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Request_Flush_BackfillBegin) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMaterialize + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BackfillBegin: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BackfillBegin: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMaterialize + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMaterialize + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMaterialize + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthMaterialize + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Timestamp == nil { + m.Timestamp = &types.Timestamp{} + } + if err := m.Timestamp.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMaterialize(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthMaterialize + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Request_Flush_BackfillComplete) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMaterialize + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BackfillComplete: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BackfillComplete: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMaterialize + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMaterialize + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMaterialize + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthMaterialize + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Timestamp == nil { + m.Timestamp = &types.Timestamp{} + } + if err := m.Timestamp.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipMaterialize(dAtA[iNdEx:]) diff --git a/go/protocols/materialize/materialize.proto b/go/protocols/materialize/materialize.proto index 3786cccc08c..9908d7a9139 100644 --- a/go/protocols/materialize/materialize.proto +++ b/go/protocols/materialize/materialize.proto @@ -6,6 +6,7 @@ option go_package = "github.com/estuary/flow/go/protocols/materialize"; import "consumer/protocol/protocol.proto"; import "go/protocols/flow/flow.proto"; import "gogoproto/gogo.proto"; +import "google/protobuf/timestamp.proto"; option (gogoproto.marshaler_all) = true; option (gogoproto.protosizer_all) = true; @@ -185,6 +186,30 @@ message Request { (gogoproto.casttype) = "encoding/json.RawMessage", json_name = "statePatches" ]; + + // Backfill-begin signals observed during this transaction. A connector + // acts on those relevant to its key range. + message BackfillBegin { + uint32 binding = 1; + // Truncation boundary of the binding's backfill: documents published at or + // after this time are current, while earlier ones were superseded by the + // backfill. It equals the begin's own publication time (flow_published_at), + // and is carried on both begin and complete so connectors need not track it + // across transactions. + google.protobuf.Timestamp timestamp = 2; + } + repeated BackfillBegin backfill_begins = 2; + + // Backfill-complete signals observed during this transaction. See + // BackfillBegin. + message BackfillComplete { + uint32 binding = 1; + // Truncation boundary of the completed backfill (see BackfillBegin.timestamp): + // the connector may delete destination rows whose flow_published_at predates + // this time, as they were superseded by the backfill. + google.protobuf.Timestamp timestamp = 2; + } + repeated BackfillComplete backfill_completes = 3; } Flush flush = 6; diff --git a/go/protocols/runtime/runtime.pb.go b/go/protocols/runtime/runtime.pb.go index b11da3112db..388cfa9752b 100644 --- a/go/protocols/runtime/runtime.pb.go +++ b/go/protocols/runtime/runtime.pb.go @@ -1622,10 +1622,15 @@ type Recover struct { // Key: binding index; Value: packed composite key tuple. MaxKeys map[uint32][]byte `protobuf:"bytes,9,rep,name=max_keys,json=maxKeys,proto3" json:"max_keys,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // Persisted trigger parameters (materialize only), or empty. - TriggerParamsJson []byte `protobuf:"bytes,10,opt,name=trigger_params_json,json=triggerParams,proto3" json:"trigger_params_json,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + TriggerParamsJson []byte `protobuf:"bytes,10,opt,name=trigger_params_json,json=triggerParams,proto3" json:"trigger_params_json,omitempty"` + // Active-backfill begin clocks, keyed by binding index. Restored so the + // capture runtime can re-apply truncated-at journal labels on startup and + // resolve a BackfillComplete's truncated_at. Resolved from "AB:{state_key}" + // keys by the scan. + ActiveBackfills map[uint32]uint64 `protobuf:"bytes,11,rep,name=active_backfills,json=activeBackfills,proto3" json:"active_backfills,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"fixed64,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Recover) Reset() { *m = Recover{} } @@ -1728,10 +1733,17 @@ type Persist struct { // Persists the incremental step of a reconciliation trigger and re-scans, // stopping once no trigger fires against the freshly-scanned state. // Effect: after the WriteBatch commits, scan and reply `Recover` not `Persisted`. - Rescan bool `protobuf:"varint,17,opt,name=rescan,proto3" json:"rescan,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Rescan bool `protobuf:"varint,17,opt,name=rescan,proto3" json:"rescan,omitempty"` + // The active-backfill change this transaction observed, if any. At most one + // per commit — a backfill control signal stands alone in its transaction. + // + // Types that are valid to be assigned to ActiveBackfillChange: + // *Persist_Begin + // *Persist_CompleteBinding + ActiveBackfillChange isPersist_ActiveBackfillChange `protobuf_oneof:"active_backfill_change"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Persist) Reset() { *m = Persist{} } @@ -1767,6 +1779,94 @@ func (m *Persist) XXX_DiscardUnknown() { var xxx_messageInfo_Persist proto.InternalMessageInfo +type isPersist_ActiveBackfillChange interface { + isPersist_ActiveBackfillChange() + MarshalTo([]byte) (int, error) + ProtoSize() int +} + +type Persist_Begin struct { + Begin *ActiveBackfillBegin `protobuf:"bytes,18,opt,name=begin,proto3,oneof" json:"begin,omitempty"` +} +type Persist_CompleteBinding struct { + CompleteBinding uint32 `protobuf:"varint,19,opt,name=complete_binding,json=completeBinding,proto3,oneof" json:"complete_binding,omitempty"` +} + +func (*Persist_Begin) isPersist_ActiveBackfillChange() {} +func (*Persist_CompleteBinding) isPersist_ActiveBackfillChange() {} + +func (m *Persist) GetActiveBackfillChange() isPersist_ActiveBackfillChange { + if m != nil { + return m.ActiveBackfillChange + } + return nil +} + +func (m *Persist) GetBegin() *ActiveBackfillBegin { + if x, ok := m.GetActiveBackfillChange().(*Persist_Begin); ok { + return x.Begin + } + return nil +} + +func (m *Persist) GetCompleteBinding() uint32 { + if x, ok := m.GetActiveBackfillChange().(*Persist_CompleteBinding); ok { + return x.CompleteBinding + } + return 0 +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*Persist) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*Persist_Begin)(nil), + (*Persist_CompleteBinding)(nil), + } +} + +// ActiveBackfillBegin records a binding's backfill begin clock — its +// authoritative truncated_at — staged by a committing Persist. +type ActiveBackfillBegin struct { + Binding uint32 `protobuf:"varint,1,opt,name=binding,proto3" json:"binding,omitempty"` + TruncatedAt uint64 `protobuf:"fixed64,2,opt,name=truncated_at,json=truncatedAt,proto3" json:"truncated_at,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ActiveBackfillBegin) Reset() { *m = ActiveBackfillBegin{} } +func (m *ActiveBackfillBegin) String() string { return proto.CompactTextString(m) } +func (*ActiveBackfillBegin) ProtoMessage() {} +func (*ActiveBackfillBegin) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{20} +} +func (m *ActiveBackfillBegin) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ActiveBackfillBegin) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ActiveBackfillBegin.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ActiveBackfillBegin) XXX_Merge(src proto.Message) { + xxx_messageInfo_ActiveBackfillBegin.Merge(m, src) +} +func (m *ActiveBackfillBegin) XXX_Size() int { + return m.ProtoSize() +} +func (m *ActiveBackfillBegin) XXX_DiscardUnknown() { + xxx_messageInfo_ActiveBackfillBegin.DiscardUnknown(m) +} + +var xxx_messageInfo_ActiveBackfillBegin proto.InternalMessageInfo + // Persisted is sent by shard zero to the leader after the state is durable // in the recovery log. type Persisted struct { @@ -1781,7 +1881,7 @@ func (m *Persisted) Reset() { *m = Persisted{} } func (m *Persisted) String() string { return proto.CompactTextString(m) } func (*Persisted) ProtoMessage() {} func (*Persisted) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{20} + return fileDescriptor_73af6e0737ce390c, []int{21} } func (m *Persisted) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1838,7 +1938,7 @@ func (m *Apply) Reset() { *m = Apply{} } func (m *Apply) String() string { return proto.CompactTextString(m) } func (*Apply) ProtoMessage() {} func (*Apply) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{21} + return fileDescriptor_73af6e0737ce390c, []int{22} } func (m *Apply) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1882,7 +1982,7 @@ func (m *Applied) Reset() { *m = Applied{} } func (m *Applied) String() string { return proto.CompactTextString(m) } func (*Applied) ProtoMessage() {} func (*Applied) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{22} + return fileDescriptor_73af6e0737ce390c, []int{23} } func (m *Applied) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1932,7 +2032,7 @@ func (m *Open) Reset() { *m = Open{} } func (m *Open) String() string { return proto.CompactTextString(m) } func (*Open) ProtoMessage() {} func (*Open) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{23} + return fileDescriptor_73af6e0737ce390c, []int{24} } func (m *Open) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1973,7 +2073,7 @@ func (m *CloseNow) Reset() { *m = CloseNow{} } func (m *CloseNow) String() string { return proto.CompactTextString(m) } func (*CloseNow) ProtoMessage() {} func (*CloseNow) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{24} + return fileDescriptor_73af6e0737ce390c, []int{25} } func (m *CloseNow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2013,7 +2113,7 @@ func (m *Stop) Reset() { *m = Stop{} } func (m *Stop) String() string { return proto.CompactTextString(m) } func (*Stop) ProtoMessage() {} func (*Stop) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{25} + return fileDescriptor_73af6e0737ce390c, []int{26} } func (m *Stop) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2055,7 +2155,7 @@ func (m *Stopped) Reset() { *m = Stopped{} } func (m *Stopped) String() string { return proto.CompactTextString(m) } func (*Stopped) ProtoMessage() {} func (*Stopped) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{26} + return fileDescriptor_73af6e0737ce390c, []int{27} } func (m *Stopped) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2102,7 +2202,7 @@ func (m *SessionLoop) Reset() { *m = SessionLoop{} } func (m *SessionLoop) String() string { return proto.CompactTextString(m) } func (*SessionLoop) ProtoMessage() {} func (*SessionLoop) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{27} + return fileDescriptor_73af6e0737ce390c, []int{28} } func (m *SessionLoop) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2179,7 +2279,7 @@ func (m *Capture) Reset() { *m = Capture{} } func (m *Capture) String() string { return proto.CompactTextString(m) } func (*Capture) ProtoMessage() {} func (*Capture) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{28} + return fileDescriptor_73af6e0737ce390c, []int{29} } func (m *Capture) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2221,7 +2321,7 @@ func (m *Capture_Opened) Reset() { *m = Capture_Opened{} } func (m *Capture_Opened) String() string { return proto.CompactTextString(m) } func (*Capture_Opened) ProtoMessage() {} func (*Capture_Opened) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{28, 0} + return fileDescriptor_73af6e0737ce390c, []int{29, 0} } func (m *Capture_Opened) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2320,7 +2420,7 @@ func (m *Materialize) Reset() { *m = Materialize{} } func (m *Materialize) String() string { return proto.CompactTextString(m) } func (*Materialize) ProtoMessage() {} func (*Materialize) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29} + return fileDescriptor_73af6e0737ce390c, []int{30} } func (m *Materialize) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2386,7 +2486,7 @@ func (m *Materialize_Opened) Reset() { *m = Materialize_Opened{} } func (m *Materialize_Opened) String() string { return proto.CompactTextString(m) } func (*Materialize_Opened) ProtoMessage() {} func (*Materialize_Opened) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 0} + return fileDescriptor_73af6e0737ce390c, []int{30, 0} } func (m *Materialize_Opened) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2430,7 +2530,7 @@ func (m *Materialize_Load) Reset() { *m = Materialize_Load{} } func (m *Materialize_Load) String() string { return proto.CompactTextString(m) } func (*Materialize_Load) ProtoMessage() {} func (*Materialize_Load) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 1} + return fileDescriptor_73af6e0737ce390c, []int{30, 1} } func (m *Materialize_Load) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2474,7 +2574,7 @@ func (m *Materialize_Loaded) Reset() { *m = Materialize_Loaded{} } func (m *Materialize_Loaded) String() string { return proto.CompactTextString(m) } func (*Materialize_Loaded) ProtoMessage() {} func (*Materialize_Loaded) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 2} + return fileDescriptor_73af6e0737ce390c, []int{30, 2} } func (m *Materialize_Loaded) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2526,7 +2626,7 @@ func (m *Materialize_Loaded_Binding) Reset() { *m = Materialize_Loaded_B func (m *Materialize_Loaded_Binding) String() string { return proto.CompactTextString(m) } func (*Materialize_Loaded_Binding) ProtoMessage() {} func (*Materialize_Loaded_Binding) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 2, 0} + return fileDescriptor_73af6e0737ce390c, []int{30, 2, 0} } func (m *Materialize_Loaded_Binding) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2559,17 +2659,26 @@ var xxx_messageInfo_Materialize_Loaded_Binding proto.InternalMessageInfo type Materialize_Flush struct { // Prior transaction's aggregated C:Acknowledged state patches. // State Update Wire Format. - ConnectorPatchesJson []byte `protobuf:"bytes,1,opt,name=connector_patches_json,json=connectorPatches,proto3" json:"connector_patches_json,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ConnectorPatchesJson []byte `protobuf:"bytes,1,opt,name=connector_patches_json,json=connectorPatches,proto3" json:"connector_patches_json,omitempty"` + // Backfill-begin markers observed during this transaction (the leader's + // per-transaction delta). Each shard forwards them to its connector as a + // C:Flush notification. The shuffle reads fold each marker exactly once per + // committed generation, so the set is already a delta — no leader-side + // deduplication. + BackfillBegins []*Materialize_Flush_BackfillBegin `protobuf:"bytes,2,rep,name=backfill_begins,json=backfillBegins,proto3" json:"backfill_begins,omitempty"` + // Backfill-complete markers observed during this transaction. Forwarded like + // `backfill_begins`. + BackfillCompletes []*Materialize_Flush_BackfillComplete `protobuf:"bytes,3,rep,name=backfill_completes,json=backfillCompletes,proto3" json:"backfill_completes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Materialize_Flush) Reset() { *m = Materialize_Flush{} } func (m *Materialize_Flush) String() string { return proto.CompactTextString(m) } func (*Materialize_Flush) ProtoMessage() {} func (*Materialize_Flush) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 3} + return fileDescriptor_73af6e0737ce390c, []int{30, 3} } func (m *Materialize_Flush) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2598,6 +2707,96 @@ func (m *Materialize_Flush) XXX_DiscardUnknown() { var xxx_messageInfo_Materialize_Flush proto.InternalMessageInfo +// A backfill-begin marker: a binding index and the begin clock (the +// truncation boundary). +type Materialize_Flush_BackfillBegin struct { + // Binding index. + Binding uint32 `protobuf:"varint,1,opt,name=binding,proto3" json:"binding,omitempty"` + // Begin clock: the backfill's truncation boundary. + Clock uint64 `protobuf:"fixed64,2,opt,name=clock,proto3" json:"clock,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Materialize_Flush_BackfillBegin) Reset() { *m = Materialize_Flush_BackfillBegin{} } +func (m *Materialize_Flush_BackfillBegin) String() string { return proto.CompactTextString(m) } +func (*Materialize_Flush_BackfillBegin) ProtoMessage() {} +func (*Materialize_Flush_BackfillBegin) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{30, 3, 0} +} +func (m *Materialize_Flush_BackfillBegin) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Materialize_Flush_BackfillBegin) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Materialize_Flush_BackfillBegin.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Materialize_Flush_BackfillBegin) XXX_Merge(src proto.Message) { + xxx_messageInfo_Materialize_Flush_BackfillBegin.Merge(m, src) +} +func (m *Materialize_Flush_BackfillBegin) XXX_Size() int { + return m.ProtoSize() +} +func (m *Materialize_Flush_BackfillBegin) XXX_DiscardUnknown() { + xxx_messageInfo_Materialize_Flush_BackfillBegin.DiscardUnknown(m) +} + +var xxx_messageInfo_Materialize_Flush_BackfillBegin proto.InternalMessageInfo + +// A backfill-complete marker; same shape as BackfillBegin, where `clock` is +// the completed backfill's begin (truncation) boundary. +type Materialize_Flush_BackfillComplete struct { + // Binding index. + Binding uint32 `protobuf:"varint,1,opt,name=binding,proto3" json:"binding,omitempty"` + // Begin clock the completed backfill reported (its truncation boundary). + Clock uint64 `protobuf:"fixed64,2,opt,name=clock,proto3" json:"clock,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Materialize_Flush_BackfillComplete) Reset() { *m = Materialize_Flush_BackfillComplete{} } +func (m *Materialize_Flush_BackfillComplete) String() string { return proto.CompactTextString(m) } +func (*Materialize_Flush_BackfillComplete) ProtoMessage() {} +func (*Materialize_Flush_BackfillComplete) Descriptor() ([]byte, []int) { + return fileDescriptor_73af6e0737ce390c, []int{30, 3, 1} +} +func (m *Materialize_Flush_BackfillComplete) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Materialize_Flush_BackfillComplete) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Materialize_Flush_BackfillComplete.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Materialize_Flush_BackfillComplete) XXX_Merge(src proto.Message) { + xxx_messageInfo_Materialize_Flush_BackfillComplete.Merge(m, src) +} +func (m *Materialize_Flush_BackfillComplete) XXX_Size() int { + return m.ProtoSize() +} +func (m *Materialize_Flush_BackfillComplete) XXX_DiscardUnknown() { + xxx_messageInfo_Materialize_Flush_BackfillComplete.DiscardUnknown(m) +} + +var xxx_messageInfo_Materialize_Flush_BackfillComplete proto.InternalMessageInfo + // Shard → Leader. Flush phase complete. // Reports connector state patches and max-key deltas from C:Flushed. type Materialize_Flushed struct { @@ -2614,7 +2813,7 @@ func (m *Materialize_Flushed) Reset() { *m = Materialize_Flushed{} } func (m *Materialize_Flushed) String() string { return proto.CompactTextString(m) } func (*Materialize_Flushed) ProtoMessage() {} func (*Materialize_Flushed) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 4} + return fileDescriptor_73af6e0737ce390c, []int{30, 4} } func (m *Materialize_Flushed) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2662,7 +2861,7 @@ func (m *Materialize_Flushed_Binding) Reset() { *m = Materialize_Flushed func (m *Materialize_Flushed_Binding) String() string { return proto.CompactTextString(m) } func (*Materialize_Flushed_Binding) ProtoMessage() {} func (*Materialize_Flushed_Binding) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 4, 0} + return fileDescriptor_73af6e0737ce390c, []int{30, 4, 0} } func (m *Materialize_Flushed_Binding) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2703,7 +2902,7 @@ func (m *Materialize_Store) Reset() { *m = Materialize_Store{} } func (m *Materialize_Store) String() string { return proto.CompactTextString(m) } func (*Materialize_Store) ProtoMessage() {} func (*Materialize_Store) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 5} + return fileDescriptor_73af6e0737ce390c, []int{30, 5} } func (m *Materialize_Store) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2745,7 +2944,7 @@ func (m *Materialize_Stored) Reset() { *m = Materialize_Stored{} } func (m *Materialize_Stored) String() string { return proto.CompactTextString(m) } func (*Materialize_Stored) ProtoMessage() {} func (*Materialize_Stored) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 6} + return fileDescriptor_73af6e0737ce390c, []int{30, 6} } func (m *Materialize_Stored) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2791,7 +2990,7 @@ func (m *Materialize_Stored_Binding) Reset() { *m = Materialize_Stored_B func (m *Materialize_Stored_Binding) String() string { return proto.CompactTextString(m) } func (*Materialize_Stored_Binding) ProtoMessage() {} func (*Materialize_Stored_Binding) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 6, 0} + return fileDescriptor_73af6e0737ce390c, []int{30, 6, 0} } func (m *Materialize_Stored_Binding) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2839,7 +3038,7 @@ func (m *Materialize_StartCommit) Reset() { *m = Materialize_StartCommit func (m *Materialize_StartCommit) String() string { return proto.CompactTextString(m) } func (*Materialize_StartCommit) ProtoMessage() {} func (*Materialize_StartCommit) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 7} + return fileDescriptor_73af6e0737ce390c, []int{30, 7} } func (m *Materialize_StartCommit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2883,7 +3082,7 @@ func (m *Materialize_StartedCommit) Reset() { *m = Materialize_StartedCo func (m *Materialize_StartedCommit) String() string { return proto.CompactTextString(m) } func (*Materialize_StartedCommit) ProtoMessage() {} func (*Materialize_StartedCommit) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 8} + return fileDescriptor_73af6e0737ce390c, []int{30, 8} } func (m *Materialize_StartedCommit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2926,7 +3125,7 @@ func (m *Materialize_Acknowledge) Reset() { *m = Materialize_Acknowledge func (m *Materialize_Acknowledge) String() string { return proto.CompactTextString(m) } func (*Materialize_Acknowledge) ProtoMessage() {} func (*Materialize_Acknowledge) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 9} + return fileDescriptor_73af6e0737ce390c, []int{30, 9} } func (m *Materialize_Acknowledge) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2970,7 +3169,7 @@ func (m *Materialize_Acknowledged) Reset() { *m = Materialize_Acknowledg func (m *Materialize_Acknowledged) String() string { return proto.CompactTextString(m) } func (*Materialize_Acknowledged) ProtoMessage() {} func (*Materialize_Acknowledged) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{29, 10} + return fileDescriptor_73af6e0737ce390c, []int{30, 10} } func (m *Materialize_Acknowledged) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3058,7 +3257,7 @@ func (m *Derive) Reset() { *m = Derive{} } func (m *Derive) String() string { return proto.CompactTextString(m) } func (*Derive) ProtoMessage() {} func (*Derive) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{30} + return fileDescriptor_73af6e0737ce390c, []int{31} } func (m *Derive) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3113,7 +3312,7 @@ func (m *Derive_Opened) Reset() { *m = Derive_Opened{} } func (m *Derive_Opened) String() string { return proto.CompactTextString(m) } func (*Derive_Opened) ProtoMessage() {} func (*Derive_Opened) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{30, 0} + return fileDescriptor_73af6e0737ce390c, []int{31, 0} } func (m *Derive_Opened) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3157,7 +3356,7 @@ func (m *Derive_Load) Reset() { *m = Derive_Load{} } func (m *Derive_Load) String() string { return proto.CompactTextString(m) } func (*Derive_Load) ProtoMessage() {} func (*Derive_Load) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{30, 1} + return fileDescriptor_73af6e0737ce390c, []int{31, 1} } func (m *Derive_Load) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3202,7 +3401,7 @@ func (m *Derive_Loaded) Reset() { *m = Derive_Loaded{} } func (m *Derive_Loaded) String() string { return proto.CompactTextString(m) } func (*Derive_Loaded) ProtoMessage() {} func (*Derive_Loaded) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{30, 2} + return fileDescriptor_73af6e0737ce390c, []int{31, 2} } func (m *Derive_Loaded) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3252,7 +3451,7 @@ func (m *Derive_Loaded_Binding) Reset() { *m = Derive_Loaded_Binding{} } func (m *Derive_Loaded_Binding) String() string { return proto.CompactTextString(m) } func (*Derive_Loaded_Binding) ProtoMessage() {} func (*Derive_Loaded_Binding) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{30, 2, 0} + return fileDescriptor_73af6e0737ce390c, []int{31, 2, 0} } func (m *Derive_Loaded_Binding) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3300,7 +3499,7 @@ func (m *Derive_Flush) Reset() { *m = Derive_Flush{} } func (m *Derive_Flush) String() string { return proto.CompactTextString(m) } func (*Derive_Flush) ProtoMessage() {} func (*Derive_Flush) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{30, 3} + return fileDescriptor_73af6e0737ce390c, []int{31, 3} } func (m *Derive_Flush) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3350,7 +3549,7 @@ func (m *Derive_Flushed) Reset() { *m = Derive_Flushed{} } func (m *Derive_Flushed) String() string { return proto.CompactTextString(m) } func (*Derive_Flushed) ProtoMessage() {} func (*Derive_Flushed) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{30, 4} + return fileDescriptor_73af6e0737ce390c, []int{31, 4} } func (m *Derive_Flushed) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3391,7 +3590,7 @@ func (m *Derive_Store) Reset() { *m = Derive_Store{} } func (m *Derive_Store) String() string { return proto.CompactTextString(m) } func (*Derive_Store) ProtoMessage() {} func (*Derive_Store) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{30, 5} + return fileDescriptor_73af6e0737ce390c, []int{31, 5} } func (m *Derive_Store) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3446,7 +3645,7 @@ func (m *Derive_Stored) Reset() { *m = Derive_Stored{} } func (m *Derive_Stored) String() string { return proto.CompactTextString(m) } func (*Derive_Stored) ProtoMessage() {} func (*Derive_Stored) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{30, 6} + return fileDescriptor_73af6e0737ce390c, []int{31, 6} } func (m *Derive_Stored) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3496,7 +3695,7 @@ func (m *Derive_Stored_PublisherCommit) Reset() { *m = Derive_Stored_Pub func (m *Derive_Stored_PublisherCommit) String() string { return proto.CompactTextString(m) } func (*Derive_Stored_PublisherCommit) ProtoMessage() {} func (*Derive_Stored_PublisherCommit) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{30, 6, 0} + return fileDescriptor_73af6e0737ce390c, []int{31, 6, 0} } func (m *Derive_Stored_PublisherCommit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3541,7 +3740,7 @@ func (m *Derive_StartCommit) Reset() { *m = Derive_StartCommit{} } func (m *Derive_StartCommit) String() string { return proto.CompactTextString(m) } func (*Derive_StartCommit) ProtoMessage() {} func (*Derive_StartCommit) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{30, 7} + return fileDescriptor_73af6e0737ce390c, []int{31, 7} } func (m *Derive_StartCommit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3583,7 +3782,7 @@ func (m *Derive_StartedCommit) Reset() { *m = Derive_StartedCommit{} } func (m *Derive_StartedCommit) String() string { return proto.CompactTextString(m) } func (*Derive_StartedCommit) ProtoMessage() {} func (*Derive_StartedCommit) Descriptor() ([]byte, []int) { - return fileDescriptor_73af6e0737ce390c, []int{30, 8} + return fileDescriptor_73af6e0737ce390c, []int{31, 8} } func (m *Derive_StartedCommit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3649,10 +3848,12 @@ func init() { proto.RegisterType((*Task)(nil), "runtime.Task") proto.RegisterType((*Recover)(nil), "runtime.Recover") proto.RegisterMapType((map[string][]byte)(nil), "runtime.Recover.AckIntentsEntry") + proto.RegisterMapType((map[uint32]uint64)(nil), "runtime.Recover.ActiveBackfillsEntry") proto.RegisterMapType((map[uint32][]byte)(nil), "runtime.Recover.MaxKeysEntry") proto.RegisterType((*Persist)(nil), "runtime.Persist") proto.RegisterMapType((map[string][]byte)(nil), "runtime.Persist.AckIntentsEntry") proto.RegisterMapType((map[uint32][]byte)(nil), "runtime.Persist.MaxKeysEntry") + proto.RegisterType((*ActiveBackfillBegin)(nil), "runtime.ActiveBackfillBegin") proto.RegisterType((*Persisted)(nil), "runtime.Persisted") proto.RegisterType((*Apply)(nil), "runtime.Apply") proto.RegisterType((*Applied)(nil), "runtime.Applied") @@ -3670,6 +3871,8 @@ func init() { proto.RegisterType((*Materialize_Loaded)(nil), "runtime.Materialize.Loaded") proto.RegisterType((*Materialize_Loaded_Binding)(nil), "runtime.Materialize.Loaded.Binding") proto.RegisterType((*Materialize_Flush)(nil), "runtime.Materialize.Flush") + proto.RegisterType((*Materialize_Flush_BackfillBegin)(nil), "runtime.Materialize.Flush.BackfillBegin") + proto.RegisterType((*Materialize_Flush_BackfillComplete)(nil), "runtime.Materialize.Flush.BackfillComplete") proto.RegisterType((*Materialize_Flushed)(nil), "runtime.Materialize.Flushed") proto.RegisterType((*Materialize_Flushed_Binding)(nil), "runtime.Materialize.Flushed.Binding") proto.RegisterType((*Materialize_Store)(nil), "runtime.Materialize.Store") @@ -3698,277 +3901,289 @@ func init() { } var fileDescriptor_73af6e0737ce390c = []byte{ - // 4313 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5b, 0x4b, 0x6c, 0x1c, 0x47, - 0x7a, 0xf6, 0xbc, 0x67, 0xfe, 0x19, 0x92, 0x33, 0x25, 0x92, 0x6a, 0xb5, 0x64, 0x91, 0x1e, 0xdb, - 0x31, 0x2d, 0x52, 0x43, 0x9a, 0x96, 0x77, 0x65, 0xc5, 0xb2, 0xc5, 0x97, 0xd6, 0xd4, 0x52, 0x12, - 0xb7, 0x48, 0x09, 0x49, 0x2e, 0x8d, 0x66, 0x77, 0x71, 0xd8, 0x62, 0x4f, 0x57, 0xbb, 0xbb, 0x87, - 0x12, 0xf7, 0x14, 0x20, 0x97, 0x00, 0x39, 0xe4, 0xb2, 0x97, 0x20, 0x97, 0xe4, 0x14, 0x24, 0x40, - 0x0e, 0x39, 0x05, 0xd8, 0x43, 0x4e, 0x39, 0x18, 0x39, 0x25, 0x39, 0x04, 0xc9, 0x45, 0x40, 0x9c, - 0x6b, 0x6e, 0x9b, 0x00, 0x89, 0x90, 0x43, 0x50, 0x8f, 0x7e, 0x4e, 0x0f, 0x45, 0xd1, 0x46, 0x62, - 0x2c, 0x7c, 0x90, 0xa6, 0xeb, 0x7f, 0x54, 0xfd, 0x55, 0xf5, 0x3f, 0xbe, 0xaa, 0x6e, 0x42, 0xb7, - 0x4f, 0x97, 0x5d, 0x8f, 0x06, 0xd4, 0xa0, 0xb6, 0xbf, 0xec, 0x0d, 0x9d, 0xc0, 0x1a, 0x90, 0xf0, - 0xb7, 0xc7, 0x39, 0xa8, 0x26, 0x9b, 0xea, 0xf5, 0x03, 0x8f, 0x1e, 0x13, 0x2f, 0x52, 0x88, 0x1e, - 0x84, 0xa0, 0x3a, 0x6f, 0x50, 0xc7, 0x1f, 0x0e, 0xce, 0x90, 0x48, 0x0f, 0x67, 0xe8, 0x6e, 0x30, - 0xf4, 0x48, 0xf8, 0x1b, 0xf6, 0x92, 0x92, 0x31, 0x89, 0x67, 0x9d, 0x10, 0xf9, 0x23, 0x25, 0xae, - 0xa5, 0x24, 0x0e, 0x6d, 0xfa, 0x9c, 0xff, 0x27, 0xb9, 0x37, 0x52, 0xdc, 0x81, 0x1e, 0x10, 0xcf, - 0xd2, 0x6d, 0xeb, 0xe7, 0x24, 0xf9, 0x2c, 0x65, 0xd5, 0x94, 0x2c, 0x75, 0xf9, 0xbf, 0x5c, 0x5b, - 0xfd, 0xa3, 0xe1, 0xe1, 0xa1, 0x4d, 0xc2, 0x5f, 0x29, 0x33, 0xdd, 0xa7, 0x7d, 0xca, 0x1f, 0x97, - 0xd9, 0x93, 0xa0, 0x76, 0xff, 0xa6, 0x00, 0x9d, 0x7d, 0xdd, 0x3f, 0xde, 0x23, 0xde, 0x89, 0x65, - 0x90, 0x0d, 0xea, 0x1c, 0x5a, 0x7d, 0x74, 0x1d, 0x9a, 0x36, 0xed, 0x6b, 0x87, 0x96, 0x4d, 0xb4, - 0x43, 0x53, 0x29, 0xcc, 0x17, 0x16, 0x2a, 0xb8, 0x61, 0xd3, 0xfe, 0x7d, 0xcb, 0x26, 0xf7, 0x4d, - 0x74, 0x15, 0x1a, 0x81, 0xee, 0x1f, 0x6b, 0x8e, 0x3e, 0x20, 0x4a, 0x71, 0xbe, 0xb0, 0xd0, 0xc0, - 0x75, 0x46, 0x78, 0xa4, 0x0f, 0x08, 0xba, 0x02, 0xf5, 0xa1, 0xe9, 0x6b, 0xae, 0x1e, 0x1c, 0x29, - 0x25, 0xce, 0xab, 0x0d, 0x4d, 0x7f, 0x57, 0x0f, 0x8e, 0xd0, 0x22, 0x74, 0x0c, 0xea, 0x04, 0xba, - 0xe5, 0x10, 0x4f, 0x73, 0x48, 0xf0, 0x9c, 0x7a, 0xc7, 0x4a, 0x99, 0xcb, 0xb4, 0x23, 0xc6, 0x23, - 0x41, 0x47, 0xef, 0x41, 0xc5, 0xb5, 0x75, 0x87, 0x28, 0xd5, 0xf9, 0xc2, 0xc2, 0xe4, 0xea, 0x64, - 0x2f, 0xdc, 0xea, 0x5d, 0x46, 0xc5, 0x82, 0xd9, 0xfd, 0xef, 0x32, 0x4c, 0xee, 0x89, 0x89, 0x62, - 0xf2, 0xd5, 0x90, 0xf8, 0x01, 0xda, 0x86, 0xda, 0x33, 0x3a, 0xf4, 0x1c, 0xdd, 0xe6, 0x96, 0x37, - 0xd6, 0x97, 0x5f, 0xbd, 0x9c, 0x5b, 0xec, 0xd3, 0x5e, 0x5f, 0xff, 0x39, 0x09, 0x02, 0xd2, 0x33, - 0xc9, 0xc9, 0xb2, 0x41, 0x3d, 0xb2, 0x9c, 0x71, 0x92, 0xde, 0x03, 0xa1, 0x86, 0x43, 0x7d, 0x34, - 0x0b, 0x55, 0x8f, 0xb8, 0xb6, 0x7e, 0xca, 0x67, 0x59, 0xc7, 0xb2, 0xc5, 0xe6, 0x78, 0x30, 0xb4, - 0x6c, 0x53, 0xb3, 0xcc, 0x70, 0x8e, 0xbc, 0xbd, 0x6d, 0xa2, 0xfb, 0x50, 0xa5, 0x87, 0x87, 0x3e, - 0x09, 0xf8, 0xc4, 0x4a, 0xeb, 0xbd, 0x57, 0x2f, 0xe7, 0x6e, 0x9c, 0x67, 0xf0, 0xc7, 0x5c, 0x0b, - 0x4b, 0x6d, 0xf4, 0x10, 0x80, 0x38, 0xa6, 0x26, 0xfb, 0xaa, 0x5c, 0xa8, 0xaf, 0x06, 0x71, 0x4c, - 0xf1, 0x88, 0x16, 0xa1, 0xe2, 0xe9, 0x4e, 0x5f, 0xac, 0x66, 0x73, 0x75, 0xaa, 0xc7, 0xdd, 0x10, - 0x33, 0xd2, 0x9e, 0x4b, 0x8c, 0xf5, 0xf2, 0xd7, 0x2f, 0xe7, 0xde, 0xc2, 0x42, 0x06, 0xed, 0x41, - 0xd3, 0xa0, 0xd4, 0x33, 0x2d, 0x47, 0x0f, 0xa8, 0xa7, 0xd4, 0xf8, 0x2a, 0x7e, 0xf4, 0xea, 0xe5, - 0xdc, 0xcd, 0xbc, 0xc1, 0x47, 0x42, 0xa9, 0xb7, 0x77, 0xa4, 0x7b, 0xe6, 0xf6, 0x26, 0x4e, 0xf6, - 0x82, 0x56, 0x00, 0x3c, 0xe2, 0x53, 0x7b, 0x18, 0x58, 0xd4, 0x51, 0xea, 0xdc, 0x8c, 0x76, 0x2f, - 0xd2, 0xf9, 0x92, 0xe8, 0x26, 0xf1, 0x70, 0x42, 0x06, 0xbd, 0x0b, 0x13, 0xd2, 0x87, 0x35, 0xcb, - 0x31, 0xc9, 0x0b, 0xa5, 0x31, 0x5f, 0x58, 0x98, 0xc0, 0x2d, 0x49, 0xdc, 0x66, 0x34, 0x74, 0x0b, - 0x80, 0x47, 0x9c, 0xce, 0xbb, 0x05, 0xde, 0xed, 0xb4, 0x98, 0xdd, 0x06, 0xb5, 0x6d, 0x62, 0x30, - 0x3a, 0x9b, 0x22, 0x4e, 0xc8, 0xa1, 0x0d, 0x98, 0x8a, 0x43, 0x4c, 0xa8, 0x36, 0xb9, 0xea, 0x15, - 0xa1, 0xfa, 0x30, 0xcd, 0xe4, 0xfa, 0x59, 0x8d, 0xee, 0x3f, 0x96, 0x61, 0x2a, 0xf2, 0x3d, 0xdf, - 0xa5, 0x8e, 0x4f, 0xd0, 0x02, 0x54, 0xfd, 0x40, 0x0f, 0x86, 0x3e, 0xf7, 0xbd, 0xc9, 0xd5, 0x76, - 0x2f, 0x5c, 0x9e, 0xde, 0x1e, 0xa7, 0x63, 0xc9, 0x67, 0x92, 0x47, 0x7c, 0xce, 0xdc, 0xb7, 0xf2, - 0xd6, 0x42, 0xf2, 0xd1, 0xfb, 0x30, 0x19, 0x10, 0x6f, 0x60, 0x39, 0xba, 0xad, 0x11, 0xcf, 0xa3, - 0x9e, 0xf4, 0xb9, 0x89, 0x90, 0xba, 0xc5, 0x88, 0xe8, 0x67, 0xd0, 0xf2, 0x88, 0x6e, 0x6a, 0xc1, - 0x91, 0x47, 0x87, 0xfd, 0xa3, 0x0b, 0xfa, 0x5f, 0x93, 0xf5, 0xb1, 0x2f, 0xba, 0x60, 0x4e, 0xf8, - 0xdc, 0xb3, 0x02, 0xa2, 0x31, 0x4b, 0x2e, 0xea, 0x84, 0xbc, 0x07, 0x36, 0x25, 0xb4, 0x0d, 0x15, - 0xdd, 0x23, 0x8e, 0xce, 0x9d, 0xb0, 0xb5, 0xfe, 0xf1, 0xab, 0x97, 0x73, 0xcb, 0x7d, 0x2b, 0x38, - 0x1a, 0x1e, 0xf4, 0x0c, 0x3a, 0x58, 0x26, 0x7e, 0x30, 0xd4, 0xbd, 0x53, 0x91, 0x26, 0x47, 0x12, - 0x67, 0x6f, 0x8d, 0xa9, 0x62, 0xd1, 0x03, 0x7a, 0x1f, 0xca, 0x26, 0x35, 0x7c, 0xa5, 0x36, 0x5f, - 0x5a, 0x68, 0xae, 0x36, 0xc5, 0xae, 0xed, 0xd9, 0x96, 0x41, 0xa4, 0x2b, 0x73, 0x36, 0xfa, 0x12, - 0x6a, 0x22, 0x82, 0x7c, 0xa5, 0x3e, 0x5f, 0xba, 0x80, 0xf5, 0xa1, 0x3a, 0xf3, 0xb3, 0xe1, 0xd0, - 0x32, 0x35, 0x57, 0xf7, 0x02, 0x5f, 0x69, 0xf0, 0x61, 0x65, 0x14, 0x3d, 0x79, 0xb2, 0xbd, 0xb9, - 0xcb, 0xc8, 0x72, 0xe8, 0x06, 0x13, 0xe4, 0x04, 0xe6, 0xf4, 0xae, 0x6e, 0x1c, 0x13, 0x53, 0x3b, - 0x26, 0xa7, 0x0a, 0x8c, 0x33, 0xb6, 0x21, 0x84, 0x7e, 0x4a, 0x4e, 0xbb, 0x26, 0x74, 0x30, 0x35, - 0x8e, 0xfd, 0xcd, 0xf5, 0x4d, 0xe2, 0x1b, 0x9e, 0xe5, 0xb2, 0xd8, 0x59, 0x02, 0xe4, 0x31, 0xa2, - 0x79, 0xa0, 0x11, 0xe7, 0x44, 0x1b, 0x90, 0x81, 0x1b, 0x78, 0xdc, 0xc3, 0xaa, 0xb8, 0x2d, 0x39, - 0x5b, 0xce, 0xc9, 0x43, 0x4e, 0x47, 0xef, 0x40, 0x2b, 0x94, 0xe6, 0x59, 0x58, 0x64, 0xe8, 0xa6, - 0xa4, 0xb1, 0x4c, 0xdc, 0xfd, 0x45, 0x11, 0x1a, 0x1b, 0x61, 0xc6, 0x45, 0x97, 0xa1, 0x66, 0xb9, - 0x9a, 0x6e, 0x9a, 0xa2, 0xcf, 0x06, 0xae, 0x5a, 0xee, 0x9a, 0x69, 0x7a, 0xe8, 0x47, 0x30, 0x21, - 0xd3, 0xb4, 0xe6, 0x52, 0x36, 0xef, 0x22, 0x9f, 0x41, 0x47, 0xcc, 0x40, 0x66, 0xea, 0x5d, 0xea, - 0x05, 0xb8, 0xe5, 0xc4, 0x0d, 0x1f, 0xed, 0x41, 0x67, 0xa0, 0xbb, 0x2e, 0x31, 0xb5, 0x23, 0xea, - 0x07, 0x52, 0xb7, 0xc4, 0x75, 0x3f, 0x88, 0xf2, 0x78, 0x34, 0x7e, 0xef, 0x21, 0x97, 0xfd, 0x92, - 0xfa, 0x01, 0x57, 0xdf, 0x72, 0x02, 0xef, 0x94, 0x85, 0x5b, 0x8a, 0x8a, 0xde, 0x06, 0x18, 0xfa, - 0x7a, 0x9f, 0x68, 0x9e, 0x1e, 0x10, 0xee, 0xdd, 0x45, 0xdc, 0xe0, 0x14, 0xac, 0x07, 0x44, 0x5d, - 0x87, 0xe9, 0xbc, 0x7e, 0x50, 0x1b, 0x4a, 0x6c, 0xed, 0x0b, 0x3c, 0x77, 0xb0, 0x47, 0x34, 0x0d, - 0x95, 0x13, 0xdd, 0x1e, 0x86, 0xa5, 0x4b, 0x34, 0xee, 0x14, 0x6f, 0x17, 0xba, 0x7f, 0x51, 0x84, - 0xce, 0x86, 0x28, 0xf1, 0xb2, 0x9a, 0x6c, 0xbd, 0x60, 0xb9, 0x93, 0xd5, 0x3e, 0xcd, 0x26, 0x27, - 0xc4, 0x96, 0x61, 0x3d, 0xd9, 0x63, 0xd5, 0x77, 0x87, 0xf6, 0x7b, 0x3b, 0x8c, 0x8a, 0xeb, 0x36, - 0xed, 0xf3, 0x27, 0xb4, 0x1d, 0x6f, 0x95, 0x19, 0x6d, 0xa0, 0x0c, 0x71, 0x35, 0x9a, 0xfb, 0xc8, - 0x16, 0xe3, 0x8e, 0xd4, 0x4a, 0xec, 0xfa, 0x36, 0xb4, 0xfc, 0x40, 0xf7, 0x02, 0xcd, 0xa0, 0x83, - 0x81, 0x15, 0xf0, 0xa8, 0x6f, 0xae, 0xfe, 0x46, 0xbc, 0x80, 0x59, 0x4b, 0x59, 0x8a, 0xf1, 0x82, - 0x0d, 0x2e, 0x8d, 0x9b, 0x7e, 0xdc, 0x50, 0x31, 0x34, 0x13, 0x3c, 0xb4, 0x01, 0x48, 0x76, 0xa2, - 0x19, 0x47, 0xc4, 0x38, 0x76, 0xa9, 0xe5, 0x04, 0x7c, 0x6a, 0x2c, 0x79, 0x46, 0x19, 0x6b, 0x23, - 0xe2, 0xe1, 0x8e, 0x94, 0x8f, 0x49, 0xdd, 0xff, 0x29, 0x03, 0x8a, 0x4c, 0x10, 0xe9, 0x8f, 0xad, - 0xd6, 0x0a, 0x34, 0xa2, 0x5a, 0x2e, 0xbb, 0x44, 0xa3, 0x7b, 0x8e, 0x63, 0x21, 0x74, 0x07, 0xaa, - 0xd4, 0x25, 0x0e, 0x31, 0xe5, 0x32, 0x75, 0x47, 0x67, 0x18, 0x75, 0xdf, 0x7b, 0xcc, 0x25, 0xb1, - 0xd4, 0x40, 0xf7, 0xa0, 0x2e, 0x31, 0x99, 0x29, 0xd7, 0xe7, 0xbd, 0xb3, 0xb4, 0x25, 0xc9, 0xc4, - 0x91, 0x16, 0xba, 0x0f, 0x90, 0x58, 0x83, 0xf2, 0xb8, 0x35, 0x4e, 0xf4, 0x11, 0xaf, 0x4a, 0x42, - 0x53, 0x7d, 0x08, 0x55, 0x61, 0xdb, 0x77, 0xb2, 0xba, 0xea, 0x53, 0xa8, 0x87, 0xc6, 0x32, 0xcf, - 0x3f, 0x26, 0xa7, 0x9a, 0x48, 0x12, 0xbc, 0xa3, 0x16, 0x6e, 0x1c, 0x93, 0xd3, 0x5d, 0x4e, 0x60, - 0xb0, 0x8a, 0x65, 0x25, 0x8b, 0x15, 0x25, 0x3f, 0x94, 0x2a, 0x72, 0xa9, 0x76, 0xcc, 0x10, 0xc2, - 0xea, 0x73, 0x80, 0x78, 0x14, 0x34, 0x0f, 0x15, 0x56, 0x8e, 0x7c, 0x69, 0x1d, 0x70, 0xb7, 0x66, - 0x85, 0xca, 0xc7, 0x82, 0x81, 0x7e, 0x02, 0x4d, 0x97, 0xda, 0xb6, 0xe6, 0x11, 0x7f, 0x68, 0x07, - 0xbc, 0xdb, 0xc9, 0xb3, 0xd7, 0x67, 0x97, 0xda, 0x36, 0xe6, 0xd2, 0x18, 0xdc, 0xe8, 0xb9, 0xfb, - 0x08, 0x20, 0xe6, 0xa0, 0x26, 0xd4, 0xb6, 0x1f, 0x3d, 0x5d, 0xdb, 0xd9, 0xde, 0x6c, 0xbf, 0x85, - 0x1a, 0x50, 0xc1, 0x5b, 0x6b, 0x9b, 0xbf, 0xdd, 0x2e, 0xa0, 0x09, 0x68, 0x3c, 0x7a, 0xbc, 0xaf, - 0x89, 0x66, 0x11, 0xb5, 0xa0, 0xbe, 0xf1, 0xf8, 0xf1, 0x8e, 0xf6, 0xf8, 0xfe, 0xfd, 0x76, 0x89, - 0x29, 0xe1, 0xad, 0xbd, 0xfd, 0x35, 0xbc, 0xdf, 0x2e, 0x77, 0xff, 0xbd, 0x00, 0xed, 0x4d, 0x8e, - 0xb5, 0xbf, 0x07, 0xa1, 0xba, 0x0a, 0x65, 0xe6, 0x90, 0xd2, 0x05, 0xaf, 0x47, 0xca, 0x59, 0x03, - 0xb9, 0xfb, 0x62, 0x2e, 0xab, 0x2e, 0x41, 0x99, 0xb5, 0xd0, 0x7b, 0x30, 0xe9, 0x7f, 0x65, 0xb3, - 0x2a, 0x7b, 0x72, 0xe8, 0x6b, 0x43, 0xcf, 0x92, 0x49, 0xb8, 0x25, 0xa8, 0x4f, 0x0f, 0xfd, 0x27, - 0x9e, 0xd5, 0xfd, 0x8f, 0x12, 0x74, 0xc2, 0xde, 0xbe, 0x4d, 0xb0, 0x7d, 0x9a, 0x09, 0xb6, 0x77, - 0x46, 0x6c, 0x1d, 0x1b, 0x6b, 0xeb, 0xd0, 0x70, 0x87, 0x07, 0xb6, 0xe5, 0x1f, 0xe5, 0x04, 0xdb, - 0xa8, 0xf6, 0x6e, 0x28, 0x8b, 0x63, 0x35, 0xf4, 0x19, 0xd4, 0x0e, 0xed, 0x21, 0xef, 0xa1, 0x9c, - 0x09, 0xf6, 0xd1, 0x1e, 0xee, 0x0b, 0x49, 0x1c, 0xaa, 0x7c, 0xd7, 0x31, 0x16, 0x40, 0x23, 0x32, - 0x92, 0x1d, 0x6a, 0x06, 0xfa, 0x0b, 0xcd, 0xb0, 0xa9, 0x71, 0x2c, 0x4b, 0x6b, 0x7d, 0xa0, 0xbf, - 0xd8, 0x60, 0xed, 0x4c, 0x04, 0x16, 0xcf, 0x15, 0x81, 0xa5, 0x31, 0x11, 0xb8, 0x08, 0x35, 0x39, - 0xb1, 0xd7, 0x87, 0x5f, 0xf7, 0x0f, 0x0b, 0x30, 0x13, 0x83, 0xd1, 0xef, 0x81, 0xab, 0x77, 0x7f, - 0x59, 0x80, 0xd9, 0x94, 0x45, 0xdf, 0xc6, 0x1b, 0xd7, 0x62, 0x77, 0x10, 0xc6, 0xc4, 0xf0, 0x20, - 0x7f, 0x8c, 0x51, 0x9f, 0x78, 0xa3, 0xe5, 0xfc, 0x65, 0x19, 0x26, 0x37, 0xe8, 0xe0, 0xc0, 0x72, - 0xa2, 0xe3, 0xe2, 0x8a, 0x0c, 0x5d, 0xa1, 0x73, 0x2d, 0x61, 0x6f, 0x52, 0x2c, 0x11, 0xb8, 0xe8, - 0x26, 0x94, 0x74, 0x33, 0x34, 0xf8, 0xea, 0x38, 0x85, 0x35, 0xd3, 0xc4, 0x4c, 0x4e, 0xfd, 0xa7, - 0xa2, 0x0c, 0xf4, 0x7b, 0x50, 0x3f, 0xb0, 0x1c, 0xd3, 0x72, 0xfa, 0xcc, 0xc2, 0x52, 0xba, 0x56, - 0x8d, 0x8e, 0xd6, 0x5b, 0x17, 0xc2, 0x38, 0xd2, 0x52, 0xff, 0xa0, 0x08, 0x35, 0x49, 0x45, 0x08, - 0xca, 0x87, 0x43, 0x5b, 0x6c, 0x7d, 0x1d, 0xf3, 0xe7, 0x10, 0xeb, 0x30, 0x94, 0xd6, 0x10, 0x58, - 0xe7, 0x36, 0x34, 0x5d, 0x8f, 0x3e, 0x13, 0xc7, 0xa0, 0x10, 0x83, 0xb5, 0x05, 0x7e, 0xdb, 0x8d, - 0x18, 0x12, 0x86, 0x26, 0x45, 0xd1, 0x5d, 0x68, 0xfa, 0xc6, 0x11, 0x19, 0xe8, 0xda, 0x33, 0x9f, - 0x3a, 0x3c, 0x5a, 0x5b, 0xeb, 0xd7, 0x5e, 0xbd, 0x9c, 0x53, 0x88, 0x63, 0x50, 0x66, 0xc2, 0x32, - 0x63, 0xf4, 0xb0, 0xfe, 0xfc, 0x21, 0xf1, 0x39, 0x0c, 0x03, 0xa1, 0xf0, 0xc0, 0xa7, 0x0e, 0xea, - 0x01, 0xf8, 0xc4, 0xd3, 0x5c, 0x6a, 0x5b, 0xc6, 0x29, 0x3f, 0x3a, 0x44, 0x78, 0x79, 0x8f, 0x78, - 0xbb, 0x9c, 0x8c, 0x1b, 0x7e, 0xf8, 0xc8, 0xaf, 0x0d, 0x38, 0xbe, 0x0e, 0x3c, 0x7e, 0x3c, 0x68, - 0xe0, 0x1a, 0x87, 0xd1, 0x81, 0xc7, 0x4e, 0xe1, 0x1c, 0xa2, 0x09, 0xb4, 0xdf, 0xc0, 0xb2, 0xa5, - 0x3a, 0x50, 0x5a, 0x33, 0x4d, 0xa4, 0x40, 0x4d, 0x2e, 0x90, 0x04, 0x79, 0x61, 0x13, 0xfd, 0x18, - 0xea, 0x26, 0x35, 0x84, 0xfd, 0xc5, 0x73, 0xd8, 0x5f, 0x33, 0xa9, 0xc1, 0x8d, 0x9f, 0x86, 0xca, - 0xa1, 0x47, 0x1d, 0x01, 0xb9, 0xea, 0x58, 0x34, 0xba, 0xff, 0x5c, 0x80, 0xa9, 0x68, 0x9f, 0xe4, - 0x79, 0x6f, 0xfc, 0xe0, 0x0a, 0xd4, 0x4c, 0x62, 0x93, 0x40, 0xba, 0x76, 0x1d, 0x87, 0xcd, 0x94, - 0x59, 0xa5, 0x0b, 0x99, 0x55, 0x4e, 0x98, 0x95, 0xc9, 0x4d, 0x95, 0x6c, 0x6e, 0x7a, 0x17, 0x26, - 0xc4, 0x7a, 0x85, 0x12, 0xfc, 0xf0, 0x85, 0x5b, 0x82, 0x28, 0x84, 0xba, 0x97, 0x61, 0x66, 0x83, - 0x3a, 0x0e, 0x31, 0x02, 0xea, 0xed, 0x7a, 0xf4, 0xc5, 0xa9, 0x74, 0xc4, 0xee, 0x1f, 0x17, 0x60, - 0x36, 0xcb, 0x91, 0x53, 0x7f, 0x00, 0x35, 0x76, 0x64, 0x20, 0xbe, 0x2f, 0xef, 0x59, 0x56, 0x5e, - 0xbd, 0x9c, 0x5b, 0x3a, 0xcf, 0xd9, 0x6a, 0xcb, 0x31, 0x45, 0x4e, 0x0e, 0x3b, 0x60, 0xbb, 0xef, - 0xb2, 0xce, 0x35, 0xcb, 0x94, 0xa8, 0xbc, 0xc6, 0xdb, 0xdb, 0x26, 0x52, 0xa1, 0x64, 0xd3, 0xbe, - 0xac, 0x37, 0xf5, 0x30, 0xc3, 0x61, 0x46, 0xec, 0xfe, 0x55, 0x09, 0xca, 0x0f, 0xa8, 0xe5, 0xa0, - 0x1b, 0xd0, 0x21, 0x81, 0x61, 0x6a, 0x03, 0x6a, 0x6a, 0x1e, 0x39, 0xb1, 0x7c, 0x76, 0xa2, 0x67, - 0x56, 0x95, 0xf0, 0x14, 0x63, 0x3c, 0xa4, 0x26, 0x96, 0x64, 0xb4, 0x08, 0x55, 0xff, 0x48, 0xf7, - 0xcc, 0xf0, 0x34, 0x73, 0x29, 0x0a, 0x42, 0xd6, 0x95, 0xb8, 0xbc, 0xc0, 0x52, 0x04, 0xcd, 0x41, - 0x93, 0x3f, 0xc9, 0x1b, 0x88, 0x12, 0xdf, 0x63, 0xe0, 0x24, 0x71, 0xff, 0xb0, 0x08, 0x9d, 0xf0, - 0x92, 0xc2, 0xb4, 0x3c, 0xbe, 0x4c, 0xa7, 0xe1, 0x9d, 0x96, 0x64, 0x6c, 0x86, 0x74, 0xf4, 0x21, - 0x84, 0x34, 0x8d, 0xc8, 0x35, 0xe0, 0x1b, 0xd6, 0xc0, 0x53, 0x92, 0x1e, 0x2e, 0x0d, 0xfa, 0x00, - 0xa6, 0x6c, 0x7e, 0xfc, 0x8f, 0x25, 0x45, 0x58, 0x4c, 0x0a, 0x72, 0x28, 0xa8, 0xfe, 0x65, 0x01, - 0x2a, 0xdc, 0x66, 0x34, 0x09, 0x45, 0xcb, 0x94, 0xe0, 0xa1, 0x68, 0x99, 0xa8, 0x07, 0x75, 0x5b, - 0x3f, 0x20, 0x36, 0x73, 0xce, 0xa2, 0xcc, 0xc6, 0x3c, 0x23, 0x32, 0xe9, 0x1d, 0xc9, 0xc1, 0x91, - 0x0c, 0x5a, 0x85, 0x9a, 0x47, 0x74, 0x66, 0xa9, 0x5c, 0x6d, 0x25, 0xbe, 0x92, 0xd8, 0xf5, 0xa8, - 0x41, 0x7c, 0x7f, 0xcf, 0x25, 0x46, 0x6f, 0x7b, 0x13, 0x87, 0x82, 0x68, 0x05, 0xa6, 0xf9, 0xc2, - 0x1b, 0x1e, 0xd1, 0x03, 0x12, 0xaf, 0x3d, 0xbf, 0x7c, 0xc0, 0x88, 0xf1, 0x36, 0x38, 0x2b, 0x5c, - 0xfe, 0xee, 0x2d, 0xa8, 0xb2, 0x75, 0x26, 0x26, 0xdb, 0x34, 0x56, 0x71, 0xb9, 0x7e, 0x76, 0xd3, - 0x06, 0xfa, 0x8b, 0xad, 0xc0, 0x88, 0x36, 0xad, 0xfb, 0x8b, 0x02, 0x94, 0xf7, 0x75, 0xff, 0x98, - 0xa5, 0x3d, 0xdf, 0x25, 0x86, 0x44, 0xc1, 0xfc, 0x99, 0x2d, 0x2b, 0xeb, 0x28, 0xf0, 0x74, 0xc7, - 0xd7, 0xa3, 0x4c, 0xc7, 0x76, 0x8a, 0xf5, 0xb3, 0x9f, 0x20, 0xe7, 0x80, 0xad, 0xf2, 0x28, 0xd8, - 0x62, 0x27, 0xe8, 0x10, 0xb2, 0x78, 0xcc, 0x25, 0x45, 0x50, 0x35, 0x23, 0xda, 0xb6, 0xf9, 0xa0, - 0x5c, 0x2f, 0xb6, 0x4b, 0xdd, 0x3f, 0xad, 0x40, 0x0d, 0x13, 0x83, 0x9e, 0xf0, 0x5a, 0xd6, 0xd4, - 0x8d, 0x63, 0xcd, 0x72, 0x02, 0xe2, 0x04, 0x61, 0x86, 0x9f, 0x8f, 0x8b, 0xab, 0x10, 0xeb, 0xad, - 0x19, 0xc7, 0xdb, 0x42, 0x44, 0x9c, 0x73, 0x41, 0x8f, 0x08, 0x68, 0x15, 0x66, 0xc4, 0x59, 0x2f, - 0x20, 0x26, 0x43, 0x22, 0x3e, 0x91, 0x78, 0xa4, 0xc8, 0xf1, 0xc8, 0xa5, 0x88, 0xb9, 0xc1, 0x78, - 0x02, 0x9a, 0xdc, 0x03, 0x14, 0xeb, 0xf0, 0x8c, 0x60, 0x91, 0x70, 0x03, 0x3b, 0xbd, 0xf0, 0x12, - 0xf8, 0xbe, 0x64, 0xe0, 0x4e, 0x24, 0x1c, 0x92, 0xd0, 0x12, 0x4c, 0x1b, 0x61, 0x88, 0x6b, 0xac, - 0x4e, 0x92, 0x44, 0xca, 0xc7, 0x93, 0x11, 0x8f, 0x55, 0x52, 0x82, 0x96, 0x00, 0x1d, 0xb1, 0x39, - 0xa6, 0x0d, 0xac, 0x88, 0xbb, 0x08, 0xc1, 0x49, 0x58, 0x77, 0x07, 0xa6, 0xa4, 0x74, 0x64, 0x5a, - 0x75, 0x9c, 0x69, 0x93, 0x42, 0x32, 0xb2, 0xeb, 0x1d, 0x68, 0xd9, 0xba, 0x1f, 0x68, 0xba, 0xeb, - 0xda, 0x16, 0x31, 0xf9, 0x3d, 0x64, 0x0b, 0x37, 0x19, 0x6d, 0x4d, 0x90, 0xd0, 0x1a, 0x74, 0x6c, - 0xd2, 0xd7, 0x8d, 0xd3, 0x24, 0x0a, 0xac, 0x9f, 0x81, 0x02, 0xdb, 0x42, 0x3c, 0x71, 0x04, 0xba, - 0x0d, 0x0c, 0xe6, 0x69, 0xc7, 0xe4, 0x34, 0xbc, 0xd6, 0x79, 0x7b, 0x64, 0xcf, 0x1e, 0xea, 0x2f, - 0x7e, 0x4a, 0x4e, 0xe5, 0x86, 0xd5, 0x06, 0xa2, 0x85, 0x6e, 0xc0, 0xa5, 0xc0, 0xb3, 0xfa, 0x7d, - 0x56, 0xe6, 0x74, 0x4f, 0x1f, 0xf8, 0x62, 0xd9, 0x80, 0x9b, 0x39, 0x21, 0x59, 0xbb, 0x9c, 0xa3, - 0xde, 0x85, 0xa9, 0xcc, 0xc6, 0x27, 0x2f, 0x26, 0x1a, 0x39, 0x17, 0x13, 0xad, 0xc4, 0xc5, 0x84, - 0x7a, 0x07, 0x5a, 0x49, 0x1b, 0x5e, 0x77, 0xa9, 0x91, 0xd4, 0xed, 0x7e, 0x53, 0x83, 0xda, 0x2e, - 0xf1, 0x7c, 0xcb, 0x0f, 0xd0, 0x0c, 0x54, 0x7d, 0xf2, 0x95, 0xe6, 0x50, 0xae, 0x5a, 0xc6, 0x15, - 0x9f, 0x7c, 0xf5, 0x88, 0xb2, 0x3d, 0x15, 0xc5, 0x49, 0x4b, 0x7a, 0xb0, 0x28, 0x5b, 0x6d, 0xc1, - 0x89, 0xad, 0xcf, 0x3a, 0x7a, 0x29, 0xe3, 0xe8, 0x72, 0xac, 0x8b, 0x39, 0x7a, 0x79, 0xbc, 0xa3, - 0xdf, 0x81, 0x2b, 0xd2, 0xc8, 0x1c, 0x7f, 0xaf, 0x70, 0x5b, 0x2f, 0x0b, 0x81, 0x8d, 0x11, 0x17, - 0xcf, 0x0f, 0x92, 0xea, 0x1b, 0x04, 0xc9, 0x0a, 0xcc, 0xc6, 0x41, 0xe2, 0xea, 0x81, 0x71, 0x44, - 0xe4, 0x7e, 0x0b, 0xb7, 0x6c, 0x47, 0xdc, 0x5d, 0xc1, 0x1c, 0x13, 0x28, 0xf5, 0x31, 0x81, 0x72, - 0x0b, 0x66, 0xe5, 0xec, 0xb2, 0xf1, 0xd2, 0xe0, 0x53, 0x9b, 0x16, 0xdc, 0x2f, 0xd3, 0x21, 0x92, - 0x13, 0x5e, 0x70, 0xd1, 0xf0, 0x6a, 0x8e, 0x86, 0xd7, 0x6d, 0x50, 0xa4, 0x51, 0xa3, 0x51, 0xd6, - 0xe2, 0x66, 0x49, 0xa3, 0x77, 0xb2, 0x51, 0x95, 0x1b, 0x98, 0x13, 0x17, 0x0e, 0xcc, 0xc9, 0x4c, - 0x60, 0x86, 0x3e, 0x96, 0x1f, 0x98, 0xab, 0x30, 0x23, 0xcd, 0x4e, 0xc7, 0xa7, 0x32, 0xc5, 0x6d, - 0xbe, 0x24, 0x98, 0xfb, 0xc9, 0x00, 0x1d, 0x17, 0xcc, 0xed, 0x9c, 0x60, 0x16, 0xaf, 0x85, 0x7c, - 0x43, 0x77, 0x94, 0x4e, 0xf8, 0x5a, 0x88, 0xb5, 0xfe, 0x3f, 0x83, 0xbc, 0x0b, 0x0d, 0xb9, 0x26, - 0xc4, 0x1c, 0x13, 0xe5, 0xdd, 0x3f, 0x2b, 0x40, 0x85, 0xed, 0xec, 0x69, 0x6e, 0x11, 0x55, 0xa0, - 0x76, 0xc2, 0x7a, 0x90, 0x58, 0xb9, 0x81, 0xc3, 0x26, 0x3b, 0x19, 0x73, 0x47, 0xe1, 0x2a, 0xa2, - 0x28, 0xd4, 0x19, 0x81, 0x81, 0x81, 0xc8, 0x8b, 0x42, 0x5d, 0x01, 0x67, 0xb8, 0x17, 0x3d, 0x95, - 0xfa, 0x2b, 0x63, 0xea, 0x8b, 0x00, 0xa2, 0x28, 0x5d, 0x5f, 0x18, 0xd0, 0xed, 0x3e, 0x83, 0x5a, - 0xe8, 0x82, 0x37, 0x01, 0x89, 0xda, 0x1d, 0x1d, 0x5c, 0x43, 0x94, 0xd0, 0xc0, 0x1d, 0xc1, 0xd9, - 0x8c, 0x19, 0x67, 0x84, 0x69, 0x31, 0x3f, 0x4c, 0xbb, 0xbf, 0x2a, 0xc8, 0xe3, 0xd9, 0x9b, 0x2d, - 0xca, 0xfb, 0xe1, 0x0b, 0xb5, 0x52, 0xee, 0x0b, 0xb5, 0xf0, 0x55, 0xda, 0xbb, 0x67, 0xd6, 0x56, - 0x7e, 0x2a, 0x25, 0xe8, 0x93, 0x84, 0xa7, 0x57, 0xb8, 0xa7, 0xc7, 0x67, 0x72, 0x7e, 0x12, 0xcc, - 0x75, 0xf3, 0x6f, 0xe5, 0x2f, 0x00, 0x75, 0x9e, 0x7c, 0x1e, 0xd1, 0xe7, 0xdd, 0x2a, 0x94, 0xf7, - 0x02, 0xea, 0x76, 0x1b, 0x50, 0x63, 0xbf, 0x2e, 0x31, 0xbb, 0xbf, 0x05, 0xcd, 0x3d, 0xe2, 0xb3, - 0x89, 0xee, 0x50, 0xea, 0x8e, 0xb9, 0x3e, 0x28, 0x5c, 0xe4, 0xfa, 0xe0, 0x8f, 0xaa, 0x50, 0x93, - 0x97, 0x86, 0xe8, 0xc3, 0xc4, 0x8a, 0x37, 0x57, 0x67, 0x7a, 0xe1, 0xdb, 0xf5, 0xf0, 0x14, 0xcc, - 0x17, 0x52, 0x6c, 0xc4, 0x6f, 0xc2, 0x04, 0xfb, 0xd5, 0x3c, 0x79, 0xfa, 0x90, 0x80, 0x76, 0x36, - 0xa1, 0x23, 0x18, 0x42, 0xa9, 0xc5, 0x84, 0xa3, 0x93, 0xca, 0x27, 0x50, 0x37, 0x2d, 0x9f, 0x97, - 0x72, 0xb9, 0x5d, 0x57, 0x46, 0xc6, 0xda, 0x94, 0x02, 0x38, 0x12, 0x45, 0x9f, 0x01, 0x84, 0xcf, - 0xd1, 0x75, 0xd5, 0xb5, 0xd1, 0x01, 0x37, 0x23, 0x19, 0x9c, 0x90, 0x67, 0x83, 0x9e, 0xe8, 0xb6, - 0x65, 0xea, 0x01, 0x91, 0xc7, 0xdf, 0xd1, 0x41, 0x9f, 0x4a, 0x01, 0x1c, 0x89, 0xa2, 0x4f, 0xa1, - 0x11, 0x3e, 0x9b, 0xb2, 0x40, 0x5d, 0x1d, 0x1d, 0x33, 0x54, 0x34, 0x71, 0x2c, 0x9d, 0xbe, 0x11, - 0x6a, 0xbc, 0xe6, 0x46, 0xe8, 0xc7, 0xd0, 0xf2, 0xc5, 0x0e, 0x6b, 0x36, 0xa5, 0xae, 0x32, 0x2d, - 0x73, 0x73, 0xb8, 0x99, 0x89, 0xed, 0xc7, 0x4d, 0x3f, 0xe1, 0x0b, 0xef, 0x40, 0xf9, 0x19, 0xb5, - 0x1c, 0x65, 0x86, 0x2b, 0x4c, 0xa4, 0x0e, 0x4f, 0x98, 0xb3, 0xd0, 0x07, 0x50, 0x7d, 0xc6, 0x21, - 0xbe, 0x32, 0x2b, 0x83, 0x23, 0x29, 0x44, 0x4c, 0x2c, 0xd9, 0xac, 0xaf, 0x40, 0xf7, 0x8f, 0x95, - 0xcb, 0x99, 0xbe, 0x18, 0xd2, 0xc7, 0x9c, 0x85, 0x96, 0xa3, 0xfb, 0x4a, 0x85, 0x0b, 0x5d, 0xce, - 0x5e, 0x3d, 0x67, 0x6f, 0x29, 0x7b, 0xd0, 0x10, 0xf5, 0xd6, 0xa1, 0xcf, 0x95, 0x2b, 0xb2, 0x18, - 0x46, 0x3a, 0xd2, 0xe7, 0x71, 0xdd, 0x90, 0x4f, 0xcc, 0x06, 0x3f, 0xa0, 0xae, 0xa2, 0x66, 0x6c, - 0x60, 0xa1, 0x80, 0x39, 0x0b, 0xdd, 0x80, 0x9a, 0x2f, 0x02, 0x43, 0xb9, 0x2a, 0xdf, 0xd5, 0x26, - 0xa5, 0x5c, 0x62, 0xe2, 0x50, 0x40, 0xbd, 0x13, 0x5d, 0x51, 0xbe, 0xf1, 0x6d, 0x58, 0xf7, 0x5f, - 0x66, 0xa1, 0x99, 0xb8, 0xf6, 0x42, 0x37, 0x53, 0xf1, 0x71, 0xa5, 0x97, 0xfc, 0x2a, 0x24, 0x27, - 0x46, 0xbe, 0xc8, 0x8f, 0x11, 0x35, 0xa3, 0x37, 0x3e, 0x4e, 0x3e, 0x4d, 0xb8, 0xac, 0x88, 0x93, - 0xb7, 0x73, 0xc7, 0xcc, 0x71, 0xdb, 0xbb, 0x49, 0xb7, 0x15, 0xa1, 0x32, 0x97, 0x3f, 0xee, 0xeb, - 0x5d, 0xb7, 0xf2, 0x6b, 0xe2, 0xba, 0x37, 0xd8, 0x79, 0x5a, 0x64, 0x1d, 0x25, 0xe3, 0x36, 0xf2, - 0x60, 0x81, 0x43, 0x01, 0xf4, 0x1e, 0x54, 0x18, 0x0e, 0x3b, 0x95, 0x1e, 0x1b, 0x7f, 0xed, 0xc2, - 0x0b, 0x36, 0x16, 0x4c, 0xd6, 0x63, 0x88, 0xd6, 0xd4, 0x4c, 0x8f, 0xb2, 0x5e, 0xe2, 0x50, 0x80, - 0x19, 0xc8, 0xef, 0x35, 0xaf, 0x66, 0x0c, 0x4c, 0x5c, 0x64, 0x7e, 0x1c, 0xc5, 0xd6, 0xb5, 0xcc, - 0x5d, 0x66, 0xc2, 0x0b, 0xb3, 0xf1, 0x75, 0x13, 0xca, 0x36, 0xd5, 0x4d, 0x65, 0x41, 0x3a, 0x65, - 0x9e, 0xca, 0x0e, 0xd5, 0x4d, 0xcc, 0xc5, 0xd8, 0x18, 0xec, 0x97, 0x98, 0xca, 0x87, 0x67, 0x8c, - 0xb1, 0xc3, 0x45, 0xb0, 0x14, 0x45, 0x2b, 0x50, 0xe1, 0xd7, 0xbb, 0xca, 0x8d, 0x4c, 0x89, 0x49, - 0xea, 0xf0, 0x5b, 0x5f, 0x2c, 0x04, 0xd1, 0x8f, 0xe2, 0x8b, 0xe4, 0xc5, 0xcc, 0x45, 0xee, 0x88, - 0x4e, 0xe2, 0xf6, 0x98, 0x8d, 0xe4, 0x07, 0xd4, 0x23, 0xca, 0xd2, 0x19, 0x23, 0xed, 0x31, 0x09, - 0x2c, 0x04, 0xd9, 0x84, 0xf8, 0x83, 0xa9, 0xdc, 0x3c, 0x63, 0x42, 0x5c, 0xc5, 0xc4, 0x52, 0x14, - 0x6d, 0x64, 0x5e, 0xe5, 0xf6, 0xb8, 0xea, 0xfc, 0x18, 0xd5, 0xfc, 0x97, 0xb8, 0x68, 0x1b, 0x26, - 0x79, 0x93, 0x9d, 0x28, 0x44, 0x37, 0xcb, 0x99, 0x57, 0x28, 0x23, 0xdd, 0x10, 0x53, 0x76, 0x34, - 0xe1, 0x27, 0x9b, 0x68, 0x9d, 0x1f, 0xe1, 0x1c, 0xfa, 0xdc, 0x26, 0x66, 0x9f, 0x28, 0x2b, 0x67, - 0x98, 0xb3, 0x16, 0xcb, 0xe1, 0xa4, 0x12, 0xda, 0x82, 0x56, 0xa2, 0x69, 0x2a, 0x1f, 0x65, 0xde, - 0x27, 0x8d, 0xe9, 0xc4, 0xc4, 0x29, 0x35, 0xe6, 0xd3, 0xae, 0x40, 0xae, 0xca, 0x6a, 0xc6, 0xa7, - 0x25, 0xa2, 0xc5, 0xa1, 0x00, 0x4b, 0xa9, 0x6e, 0x88, 0x72, 0x95, 0x8f, 0x33, 0x29, 0x35, 0xc2, - 0xbf, 0x38, 0x16, 0x4a, 0x57, 0x83, 0x5b, 0xe7, 0xaf, 0x06, 0x9f, 0x9d, 0xab, 0x1a, 0xdc, 0x7d, - 0x5d, 0x35, 0xf8, 0xbd, 0xc2, 0xc5, 0xcb, 0x01, 0xfa, 0x49, 0x12, 0x3b, 0x26, 0x8e, 0x51, 0xc5, - 0x33, 0x8e, 0x51, 0x97, 0x22, 0x8d, 0xc4, 0x7b, 0xae, 0x4f, 0xa0, 0xcc, 0x02, 0x0c, 0xdd, 0x84, - 0x7a, 0x74, 0x4c, 0x2c, 0x8c, 0x3b, 0x26, 0x46, 0x22, 0xea, 0xaf, 0x8a, 0x50, 0x15, 0x81, 0x89, - 0xbe, 0x18, 0x79, 0x75, 0xf1, 0xee, 0x19, 0x71, 0x3c, 0xfa, 0xe6, 0x42, 0x9c, 0x01, 0xf8, 0xd5, - 0xb9, 0xa7, 0x89, 0xaf, 0x38, 0x0e, 0x4e, 0x03, 0x22, 0xee, 0x18, 0xca, 0xec, 0x0c, 0x20, 0x78, - 0x4f, 0x18, 0x6b, 0x9d, 0x71, 0xd4, 0xff, 0x2c, 0xc4, 0xef, 0x3a, 0xa6, 0xa1, 0x22, 0xee, 0x5f, - 0x05, 0xb6, 0x15, 0x0d, 0xb4, 0x00, 0xed, 0x81, 0xe5, 0x68, 0x3e, 0x1d, 0x7a, 0x46, 0xfa, 0xa2, - 0x6c, 0x72, 0x60, 0x39, 0x7b, 0x9c, 0x2c, 0x0e, 0xd7, 0x0b, 0xe2, 0x82, 0x30, 0x25, 0x59, 0x92, - 0x92, 0xfa, 0x8b, 0xa4, 0xe4, 0x12, 0x20, 0x21, 0x65, 0x6a, 0x26, 0x35, 0x7c, 0x2d, 0xa0, 0x81, - 0x6e, 0xf3, 0x82, 0x56, 0xc6, 0x6d, 0xc9, 0xd9, 0xa4, 0x86, 0xbf, 0xcf, 0xe8, 0xa8, 0x07, 0x97, - 0x42, 0x69, 0x3e, 0x1d, 0x29, 0x5e, 0xe1, 0xe2, 0x1d, 0xc9, 0xe2, 0xd3, 0x11, 0xf2, 0x5d, 0x98, - 0x90, 0x40, 0x5f, 0x33, 0x89, 0x1d, 0xc8, 0x0f, 0xa1, 0x70, 0x53, 0x20, 0xfa, 0x4d, 0x46, 0x52, - 0x3f, 0x85, 0x0a, 0xcf, 0x52, 0x67, 0x1c, 0x65, 0x0a, 0xf9, 0x47, 0x19, 0xf5, 0xbf, 0x0a, 0xf1, - 0xbb, 0xb0, 0xb3, 0x5e, 0x36, 0xe5, 0x64, 0xc4, 0xdc, 0x2d, 0x7b, 0xc3, 0xa3, 0x94, 0x7a, 0xfa, - 0xba, 0x1d, 0xbb, 0x01, 0x1d, 0x91, 0xe1, 0x93, 0x8b, 0x2b, 0x5c, 0x60, 0x4a, 0x30, 0xe2, 0xb5, - 0x5d, 0x02, 0x24, 0x65, 0x93, 0x4b, 0x5b, 0x12, 0x3b, 0x21, 0x38, 0xf1, 0xca, 0xaa, 0x35, 0xa8, - 0xf0, 0x94, 0xab, 0xfe, 0x6d, 0x01, 0xaa, 0x22, 0xf9, 0x9e, 0xdb, 0x69, 0x85, 0x78, 0xce, 0xeb, - 0xb6, 0xf3, 0xcc, 0x47, 0x24, 0xf8, 0x9c, 0xf9, 0x08, 0x46, 0x6a, 0x3e, 0x52, 0x36, 0x67, 0x3e, - 0x82, 0x93, 0x98, 0xcf, 0xef, 0x17, 0xd2, 0x5f, 0xec, 0xbc, 0xb1, 0x33, 0x7c, 0x77, 0xd9, 0x63, - 0x0d, 0x26, 0x52, 0xb5, 0xe4, 0x02, 0x8e, 0xf9, 0x05, 0x34, 0x13, 0x15, 0xe0, 0x02, 0x1d, 0xdc, - 0x83, 0x56, 0xb2, 0x84, 0xbc, 0x79, 0x0f, 0xdd, 0xbf, 0x46, 0x50, 0x15, 0x5f, 0x18, 0xa0, 0x85, - 0x14, 0xac, 0x9e, 0xee, 0xc9, 0x2f, 0xb6, 0x73, 0x10, 0xf5, 0x9d, 0x7c, 0x44, 0x3d, 0x13, 0xab, - 0x8c, 0x07, 0xd3, 0xb7, 0x46, 0xc0, 0xb4, 0x92, 0x1d, 0x29, 0x07, 0x47, 0xdf, 0x1e, 0xc5, 0xd1, - 0xea, 0xc8, 0x68, 0x3f, 0x40, 0xe8, 0x3c, 0x08, 0x7d, 0x0e, 0xc0, 0xdb, 0xcb, 0x00, 0xde, 0xd9, - 0xcc, 0xc7, 0x27, 0x59, 0xac, 0xbb, 0x90, 0xc2, 0xba, 0xd3, 0x59, 0xe9, 0x04, 0xcc, 0xed, 0x65, - 0x60, 0xee, 0x6c, 0x9e, 0x6c, 0x02, 0xe1, 0x2e, 0xa6, 0x11, 0xee, 0x4c, 0x56, 0x3c, 0x05, 0x6e, - 0x3f, 0xca, 0x82, 0xdb, 0xcb, 0xb9, 0xe2, 0x49, 0x5c, 0xbb, 0x98, 0xc6, 0xb5, 0x23, 0xfd, 0xa7, - 0x20, 0x6d, 0x2f, 0x03, 0x69, 0x67, 0x73, 0xa5, 0x63, 0x34, 0xfb, 0x79, 0x2e, 0x9a, 0xbd, 0x3a, - 0xaa, 0x35, 0x06, 0xc8, 0x6e, 0x8e, 0x01, 0xb2, 0x6f, 0xe7, 0xf6, 0x30, 0x0e, 0xc3, 0xfe, 0x00, - 0x1c, 0xbf, 0xaf, 0xc0, 0xf1, 0xef, 0x62, 0xe0, 0x78, 0x67, 0xa4, 0x06, 0x5f, 0xcf, 0x8f, 0x8c, - 0xef, 0x04, 0x33, 0xfe, 0xc3, 0xaf, 0x1f, 0x66, 0xfc, 0x36, 0x78, 0xf0, 0x71, 0x0c, 0x07, 0xdf, - 0x1c, 0x3f, 0x20, 0x28, 0x0f, 0x58, 0x02, 0x11, 0x6f, 0x01, 0xf9, 0x73, 0x8c, 0xb2, 0x7e, 0xb7, - 0x14, 0xa1, 0xac, 0x15, 0x98, 0x8e, 0x3e, 0xef, 0x4b, 0xce, 0x5f, 0xbc, 0x7a, 0x40, 0x11, 0x2f, - 0x5e, 0x81, 0x55, 0x98, 0x89, 0x35, 0x92, 0x6b, 0x20, 0x36, 0xf6, 0x52, 0xc4, 0x4c, 0x20, 0xe7, - 0x25, 0x40, 0xa6, 0xc7, 0xbc, 0x3b, 0x35, 0x86, 0x44, 0x4f, 0x92, 0x93, 0x5a, 0xe3, 0x50, 0x3a, - 0xd9, 0xbf, 0xd8, 0x92, 0x8e, 0x64, 0x25, 0x7a, 0xff, 0x19, 0xb4, 0xe3, 0xf7, 0xfd, 0x32, 0x25, - 0x55, 0x32, 0x5f, 0x02, 0xa7, 0x52, 0x61, 0xf4, 0x71, 0xa3, 0x27, 0x73, 0xd3, 0x94, 0x9b, 0x26, - 0xa8, 0x1a, 0x4c, 0x65, 0x64, 0x90, 0xca, 0x3f, 0x72, 0x31, 0x87, 0x86, 0x8c, 0xa2, 0x16, 0x8e, - 0xda, 0xcc, 0x5b, 0x93, 0xce, 0x28, 0x1a, 0x4c, 0x43, 0xfe, 0x29, 0x92, 0x78, 0xcd, 0xda, 0xc0, - 0x51, 0x5b, 0x7d, 0x9a, 0x06, 0x88, 0xe3, 0x62, 0xbe, 0xf0, 0xa6, 0x31, 0x3f, 0x95, 0x81, 0x7b, - 0x37, 0x16, 0xa1, 0xc2, 0xff, 0xe4, 0x0a, 0x01, 0x54, 0x77, 0x9f, 0xac, 0xef, 0x6c, 0x6f, 0xb4, - 0xdf, 0x42, 0x4d, 0xa8, 0xed, 0xe2, 0xed, 0xa7, 0x6b, 0xfb, 0x5b, 0xed, 0x02, 0x6a, 0x40, 0x65, - 0xe7, 0xf1, 0xc6, 0xda, 0x4e, 0xbb, 0xb8, 0xfa, 0x00, 0xea, 0xf2, 0x4f, 0x62, 0x3c, 0xf4, 0x39, - 0xd4, 0xe4, 0x33, 0x8a, 0x0b, 0x56, 0xfa, 0x8f, 0xb5, 0x54, 0x65, 0x94, 0x21, 0x40, 0xce, 0x4a, - 0x61, 0x75, 0x07, 0xea, 0xf2, 0x73, 0x2b, 0x0f, 0xdd, 0x83, 0x9a, 0x7c, 0x4e, 0xf4, 0x95, 0xfe, - 0x68, 0x2e, 0xd1, 0x57, 0xe6, 0x2b, 0xad, 0x85, 0xc2, 0x4a, 0x61, 0xf5, 0x08, 0x26, 0xd3, 0x1f, - 0x32, 0xa1, 0xa7, 0x30, 0xc5, 0x1f, 0x22, 0xb2, 0x8f, 0xae, 0x27, 0x13, 0xeb, 0xe8, 0xe7, 0x50, - 0xea, 0xdc, 0x58, 0x7e, 0x62, 0xa4, 0xe7, 0x50, 0xdd, 0x11, 0x7f, 0xb9, 0xd3, 0x8b, 0x30, 0xe7, - 0x54, 0xc6, 0x8f, 0xd4, 0x2c, 0x81, 0x69, 0xa2, 0xbb, 0xe9, 0xfb, 0xdf, 0xe9, 0xbc, 0xe3, 0x8a, - 0x9a, 0x4b, 0xe5, 0x03, 0xff, 0x79, 0xf4, 0x29, 0xd0, 0x47, 0xf1, 0x4b, 0x96, 0x76, 0xf6, 0xc2, - 0x5c, 0x1d, 0xa1, 0xf0, 0xb1, 0xff, 0x6f, 0x6d, 0x5d, 0xff, 0xfc, 0xeb, 0x7f, 0xbd, 0xfe, 0xd6, - 0xd7, 0xdf, 0x5c, 0x2f, 0xfc, 0xfd, 0x37, 0xd7, 0x0b, 0x7f, 0xf2, 0x6f, 0xd7, 0x0b, 0xbf, 0xb3, - 0x74, 0xae, 0x3f, 0x05, 0x92, 0xfd, 0x1d, 0x54, 0x39, 0xe9, 0xe3, 0xff, 0x0d, 0x00, 0x00, 0xff, - 0xff, 0xa7, 0x5b, 0xa9, 0x97, 0x17, 0x3a, 0x00, 0x00, + // 4509 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7b, 0x4d, 0x6c, 0x1c, 0x47, + 0x76, 0xb0, 0xe6, 0x7f, 0xe6, 0xcd, 0x90, 0x9c, 0x29, 0x91, 0x54, 0xab, 0x65, 0x8b, 0xf4, 0xd8, + 0xfe, 0x4c, 0x8b, 0xd4, 0x90, 0xa6, 0xe5, 0x5d, 0x5b, 0x9f, 0xff, 0xf8, 0xa7, 0x15, 0xb5, 0x94, + 0x44, 0x17, 0x29, 0x21, 0xd9, 0x4b, 0xa3, 0xd9, 0x5d, 0x1c, 0xb6, 0xd8, 0xd3, 0xd5, 0xee, 0xee, + 0xa1, 0xc4, 0x3d, 0x05, 0xc8, 0x25, 0x40, 0x80, 0xe4, 0xb2, 0x97, 0x20, 0x97, 0xdc, 0x82, 0x04, + 0xc8, 0x21, 0xa7, 0x05, 0xf6, 0x90, 0x53, 0x0e, 0x46, 0x4e, 0x49, 0x0e, 0x41, 0x4e, 0x0a, 0xb2, + 0xb9, 0x26, 0xa7, 0x4d, 0x80, 0x44, 0xc8, 0x21, 0xa8, 0x9f, 0xfe, 0x9d, 0x1e, 0x8a, 0xa4, 0x8d, + 0xc4, 0x58, 0xec, 0x41, 0x62, 0xd7, 0xfb, 0xab, 0x57, 0x55, 0xef, 0xbd, 0x7a, 0xef, 0x75, 0x0f, + 0x74, 0xfb, 0x74, 0xd9, 0xf5, 0x68, 0x40, 0x0d, 0x6a, 0xfb, 0xcb, 0xde, 0xd0, 0x09, 0xac, 0x01, + 0x09, 0xff, 0xf6, 0x38, 0x06, 0xd5, 0xe4, 0x50, 0xbd, 0x79, 0xe0, 0xd1, 0x63, 0xe2, 0x45, 0x0c, + 0xd1, 0x83, 0x20, 0x54, 0xe7, 0x0d, 0xea, 0xf8, 0xc3, 0xc1, 0x19, 0x14, 0xe9, 0xe9, 0x0c, 0xdd, + 0x0d, 0x86, 0x1e, 0x09, 0xff, 0x86, 0x52, 0x52, 0x34, 0x26, 0xf1, 0xac, 0x13, 0x22, 0xff, 0x48, + 0x8a, 0x37, 0x52, 0x14, 0x87, 0x36, 0x7d, 0xce, 0xff, 0x93, 0xd8, 0x5b, 0x29, 0xec, 0x40, 0x0f, + 0x88, 0x67, 0xe9, 0xb6, 0xf5, 0x53, 0x92, 0x7c, 0x96, 0xb4, 0x6a, 0x8a, 0x96, 0xba, 0xfc, 0x5f, + 0xae, 0xae, 0xfe, 0xd1, 0xf0, 0xf0, 0xd0, 0x26, 0xe1, 0x5f, 0x49, 0x33, 0xdd, 0xa7, 0x7d, 0xca, + 0x1f, 0x97, 0xd9, 0x93, 0x80, 0x76, 0xff, 0xaa, 0x00, 0x9d, 0x7d, 0xdd, 0x3f, 0xde, 0x23, 0xde, + 0x89, 0x65, 0x90, 0x0d, 0xea, 0x1c, 0x5a, 0x7d, 0x74, 0x13, 0x9a, 0x36, 0xed, 0x6b, 0x87, 0x96, + 0x4d, 0xb4, 0x43, 0x53, 0x29, 0xcc, 0x17, 0x16, 0x2a, 0xb8, 0x61, 0xd3, 0xfe, 0x3d, 0xcb, 0x26, + 0xf7, 0x4c, 0x74, 0x03, 0x1a, 0x81, 0xee, 0x1f, 0x6b, 0x8e, 0x3e, 0x20, 0x4a, 0x71, 0xbe, 0xb0, + 0xd0, 0xc0, 0x75, 0x06, 0x78, 0xa4, 0x0f, 0x08, 0xba, 0x0e, 0xf5, 0xa1, 0xe9, 0x6b, 0xae, 0x1e, + 0x1c, 0x29, 0x25, 0x8e, 0xab, 0x0d, 0x4d, 0x7f, 0x57, 0x0f, 0x8e, 0xd0, 0x22, 0x74, 0x0c, 0xea, + 0x04, 0xba, 0xe5, 0x10, 0x4f, 0x73, 0x48, 0xf0, 0x9c, 0x7a, 0xc7, 0x4a, 0x99, 0xd3, 0xb4, 0x23, + 0xc4, 0x23, 0x01, 0x47, 0xef, 0x40, 0xc5, 0xb5, 0x75, 0x87, 0x28, 0xd5, 0xf9, 0xc2, 0xc2, 0xe4, + 0xea, 0x64, 0x2f, 0x3c, 0xea, 0x5d, 0x06, 0xc5, 0x02, 0xd9, 0xfd, 0xaf, 0x32, 0x4c, 0xee, 0x89, + 0x85, 0x62, 0xf2, 0xf5, 0x90, 0xf8, 0x01, 0xda, 0x86, 0xda, 0x33, 0x3a, 0xf4, 0x1c, 0xdd, 0xe6, + 0x9a, 0x37, 0xd6, 0x97, 0x5f, 0xbd, 0x9c, 0x5b, 0xec, 0xd3, 0x5e, 0x5f, 0xff, 0x29, 0x09, 0x02, + 0xd2, 0x33, 0xc9, 0xc9, 0xb2, 0x41, 0x3d, 0xb2, 0x9c, 0x31, 0x92, 0xde, 0x03, 0xc1, 0x86, 0x43, + 0x7e, 0x34, 0x0b, 0x55, 0x8f, 0xb8, 0xb6, 0x7e, 0xca, 0x57, 0x59, 0xc7, 0x72, 0xc4, 0xd6, 0x78, + 0x30, 0xb4, 0x6c, 0x53, 0xb3, 0xcc, 0x70, 0x8d, 0x7c, 0xbc, 0x6d, 0xa2, 0x7b, 0x50, 0xa5, 0x87, + 0x87, 0x3e, 0x09, 0xf8, 0xc2, 0x4a, 0xeb, 0xbd, 0x57, 0x2f, 0xe7, 0x6e, 0x9d, 0x67, 0xf2, 0xc7, + 0x9c, 0x0b, 0x4b, 0x6e, 0xf4, 0x10, 0x80, 0x38, 0xa6, 0x26, 0x65, 0x55, 0x2e, 0x25, 0xab, 0x41, + 0x1c, 0x53, 0x3c, 0xa2, 0x45, 0xa8, 0x78, 0xba, 0xd3, 0x17, 0xbb, 0xd9, 0x5c, 0x9d, 0xea, 0x71, + 0x33, 0xc4, 0x0c, 0xb4, 0xe7, 0x12, 0x63, 0xbd, 0xfc, 0xcd, 0xcb, 0xb9, 0x2b, 0x58, 0xd0, 0xa0, + 0x3d, 0x68, 0x1a, 0x94, 0x7a, 0xa6, 0xe5, 0xe8, 0x01, 0xf5, 0x94, 0x1a, 0xdf, 0xc5, 0x0f, 0x5e, + 0xbd, 0x9c, 0xbb, 0x9d, 0x37, 0xf9, 0x88, 0x2b, 0xf5, 0xf6, 0x8e, 0x74, 0xcf, 0xdc, 0xde, 0xc4, + 0x49, 0x29, 0x68, 0x05, 0xc0, 0x23, 0x3e, 0xb5, 0x87, 0x81, 0x45, 0x1d, 0xa5, 0xce, 0xd5, 0x68, + 0xf7, 0x22, 0x9e, 0xfb, 0x44, 0x37, 0x89, 0x87, 0x13, 0x34, 0xe8, 0x6d, 0x98, 0x90, 0x36, 0xac, + 0x59, 0x8e, 0x49, 0x5e, 0x28, 0x8d, 0xf9, 0xc2, 0xc2, 0x04, 0x6e, 0x49, 0xe0, 0x36, 0x83, 0xa1, + 0x3b, 0x00, 0xdc, 0xe3, 0x74, 0x2e, 0x16, 0xb8, 0xd8, 0x69, 0xb1, 0xba, 0x0d, 0x6a, 0xdb, 0xc4, + 0x60, 0x70, 0xb6, 0x44, 0x9c, 0xa0, 0x43, 0x1b, 0x30, 0x15, 0xbb, 0x98, 0x60, 0x6d, 0x72, 0xd6, + 0xeb, 0x82, 0xf5, 0x61, 0x1a, 0xc9, 0xf9, 0xb3, 0x1c, 0xdd, 0xbf, 0x2f, 0xc3, 0x54, 0x64, 0x7b, + 0xbe, 0x4b, 0x1d, 0x9f, 0xa0, 0x05, 0xa8, 0xfa, 0x81, 0x1e, 0x0c, 0x7d, 0x6e, 0x7b, 0x93, 0xab, + 0xed, 0x5e, 0xb8, 0x3d, 0xbd, 0x3d, 0x0e, 0xc7, 0x12, 0xcf, 0x28, 0x8f, 0xf8, 0x9a, 0xb9, 0x6d, + 0xe5, 0xed, 0x85, 0xc4, 0xa3, 0x77, 0x61, 0x32, 0x20, 0xde, 0xc0, 0x72, 0x74, 0x5b, 0x23, 0x9e, + 0x47, 0x3d, 0x69, 0x73, 0x13, 0x21, 0x74, 0x8b, 0x01, 0xd1, 0x57, 0xd0, 0xf2, 0x88, 0x6e, 0x6a, + 0xc1, 0x91, 0x47, 0x87, 0xfd, 0xa3, 0x4b, 0xda, 0x5f, 0x93, 0xc9, 0xd8, 0x17, 0x22, 0x98, 0x11, + 0x3e, 0xf7, 0xac, 0x80, 0x68, 0x4c, 0x93, 0xcb, 0x1a, 0x21, 0x97, 0xc0, 0x96, 0x84, 0xb6, 0xa1, + 0xa2, 0x7b, 0xc4, 0xd1, 0xb9, 0x11, 0xb6, 0xd6, 0x3f, 0x7c, 0xf5, 0x72, 0x6e, 0xb9, 0x6f, 0x05, + 0x47, 0xc3, 0x83, 0x9e, 0x41, 0x07, 0xcb, 0xc4, 0x0f, 0x86, 0xba, 0x77, 0x2a, 0xc2, 0xe4, 0x48, + 0xe0, 0xec, 0xad, 0x31, 0x56, 0x2c, 0x24, 0xa0, 0x77, 0xa1, 0x6c, 0x52, 0xc3, 0x57, 0x6a, 0xf3, + 0xa5, 0x85, 0xe6, 0x6a, 0x53, 0x9c, 0xda, 0x9e, 0x6d, 0x19, 0x44, 0x9a, 0x32, 0x47, 0xa3, 0xfb, + 0x50, 0x13, 0x1e, 0xe4, 0x2b, 0xf5, 0xf9, 0xd2, 0x25, 0xb4, 0x0f, 0xd9, 0x99, 0x9d, 0x0d, 0x87, + 0x96, 0xa9, 0xb9, 0xba, 0x17, 0xf8, 0x4a, 0x83, 0x4f, 0x2b, 0xbd, 0xe8, 0xc9, 0x93, 0xed, 0xcd, + 0x5d, 0x06, 0x96, 0x53, 0x37, 0x18, 0x21, 0x07, 0x30, 0xa3, 0x77, 0x75, 0xe3, 0x98, 0x98, 0xda, + 0x31, 0x39, 0x55, 0x60, 0x9c, 0xb2, 0x0d, 0x41, 0xf4, 0x63, 0x72, 0xda, 0x35, 0xa1, 0x83, 0xa9, + 0x71, 0xec, 0x6f, 0xae, 0x6f, 0x12, 0xdf, 0xf0, 0x2c, 0x97, 0xf9, 0xce, 0x12, 0x20, 0x8f, 0x01, + 0xcd, 0x03, 0x8d, 0x38, 0x27, 0xda, 0x80, 0x0c, 0xdc, 0xc0, 0xe3, 0x16, 0x56, 0xc5, 0x6d, 0x89, + 0xd9, 0x72, 0x4e, 0x1e, 0x72, 0x38, 0x7a, 0x0b, 0x5a, 0x21, 0x35, 0x8f, 0xc2, 0x22, 0x42, 0x37, + 0x25, 0x8c, 0x45, 0xe2, 0xee, 0xcf, 0x8a, 0xd0, 0xd8, 0x08, 0x23, 0x2e, 0xba, 0x06, 0x35, 0xcb, + 0xd5, 0x74, 0xd3, 0x14, 0x32, 0x1b, 0xb8, 0x6a, 0xb9, 0x6b, 0xa6, 0xe9, 0xa1, 0x1f, 0xc0, 0x84, + 0x0c, 0xd3, 0x9a, 0x4b, 0xd9, 0xba, 0x8b, 0x7c, 0x05, 0x1d, 0xb1, 0x02, 0x19, 0xa9, 0x77, 0xa9, + 0x17, 0xe0, 0x96, 0x13, 0x0f, 0x7c, 0xb4, 0x07, 0x9d, 0x81, 0xee, 0xba, 0xc4, 0xd4, 0x8e, 0xa8, + 0x1f, 0x48, 0xde, 0x12, 0xe7, 0x7d, 0x2f, 0x8a, 0xe3, 0xd1, 0xfc, 0xbd, 0x87, 0x9c, 0xf6, 0x3e, + 0xf5, 0x03, 0xce, 0xbe, 0xe5, 0x04, 0xde, 0x29, 0x73, 0xb7, 0x14, 0x14, 0xbd, 0x09, 0x30, 0xf4, + 0xf5, 0x3e, 0xd1, 0x3c, 0x3d, 0x20, 0xdc, 0xba, 0x8b, 0xb8, 0xc1, 0x21, 0x58, 0x0f, 0x88, 0xba, + 0x0e, 0xd3, 0x79, 0x72, 0x50, 0x1b, 0x4a, 0x6c, 0xef, 0x0b, 0x3c, 0x76, 0xb0, 0x47, 0x34, 0x0d, + 0x95, 0x13, 0xdd, 0x1e, 0x86, 0x57, 0x97, 0x18, 0xdc, 0x2d, 0x7e, 0x5c, 0xe8, 0xfe, 0x79, 0x11, + 0x3a, 0x1b, 0xe2, 0x8a, 0x97, 0xb7, 0xc9, 0xd6, 0x0b, 0x16, 0x3b, 0xd9, 0xdd, 0xa7, 0xd9, 0xe4, + 0x84, 0xd8, 0xd2, 0xad, 0x27, 0x7b, 0xec, 0xf6, 0xdd, 0xa1, 0xfd, 0xde, 0x0e, 0x83, 0xe2, 0xba, + 0x4d, 0xfb, 0xfc, 0x09, 0x6d, 0xc7, 0x47, 0x65, 0x46, 0x07, 0x28, 0x5d, 0x5c, 0x8d, 0xd6, 0x3e, + 0x72, 0xc4, 0xb8, 0x23, 0xb9, 0x12, 0xa7, 0xbe, 0x0d, 0x2d, 0x3f, 0xd0, 0xbd, 0x40, 0x33, 0xe8, + 0x60, 0x60, 0x05, 0xdc, 0xeb, 0x9b, 0xab, 0xff, 0x2f, 0xde, 0xc0, 0xac, 0xa6, 0x2c, 0xc4, 0x78, + 0xc1, 0x06, 0xa7, 0xc6, 0x4d, 0x3f, 0x1e, 0xa8, 0x18, 0x9a, 0x09, 0x1c, 0xda, 0x00, 0x24, 0x85, + 0x68, 0xc6, 0x11, 0x31, 0x8e, 0x5d, 0x6a, 0x39, 0x01, 0x5f, 0x1a, 0x0b, 0x9e, 0x51, 0xc4, 0xda, + 0x88, 0x70, 0xb8, 0x23, 0xe9, 0x63, 0x50, 0xf7, 0xbf, 0xcb, 0x80, 0x22, 0x15, 0x44, 0xf8, 0x63, + 0xbb, 0xb5, 0x02, 0x8d, 0xe8, 0x2e, 0x97, 0x22, 0xd1, 0xe8, 0x99, 0xe3, 0x98, 0x08, 0xdd, 0x85, + 0x2a, 0x75, 0x89, 0x43, 0x4c, 0xb9, 0x4d, 0xdd, 0xd1, 0x15, 0x46, 0xe2, 0x7b, 0x8f, 0x39, 0x25, + 0x96, 0x1c, 0xe8, 0x4b, 0xa8, 0xcb, 0x9c, 0xcc, 0x94, 0xfb, 0xf3, 0xce, 0x59, 0xdc, 0x12, 0x64, + 0xe2, 0x88, 0x0b, 0xdd, 0x03, 0x48, 0xec, 0x41, 0x79, 0xdc, 0x1e, 0x27, 0x64, 0xc4, 0xbb, 0x92, + 0xe0, 0x54, 0x1f, 0x42, 0x55, 0xe8, 0xf6, 0x9d, 0xec, 0xae, 0xfa, 0x14, 0xea, 0xa1, 0xb2, 0xcc, + 0xf2, 0x8f, 0xc9, 0xa9, 0x26, 0x82, 0x04, 0x17, 0xd4, 0xc2, 0x8d, 0x63, 0x72, 0xba, 0xcb, 0x01, + 0x2c, 0xad, 0x62, 0x51, 0xc9, 0x62, 0x97, 0x92, 0x1f, 0x52, 0x15, 0x39, 0x55, 0x3b, 0x46, 0x08, + 0x62, 0xf5, 0x39, 0x40, 0x3c, 0x0b, 0x9a, 0x87, 0x0a, 0xbb, 0x8e, 0x7c, 0xa9, 0x1d, 0x70, 0xb3, + 0x66, 0x17, 0x95, 0x8f, 0x05, 0x02, 0xfd, 0x08, 0x9a, 0x2e, 0xb5, 0x6d, 0xcd, 0x23, 0xfe, 0xd0, + 0x0e, 0xb8, 0xd8, 0xc9, 0xb3, 0xf7, 0x67, 0x97, 0xda, 0x36, 0xe6, 0xd4, 0x18, 0xdc, 0xe8, 0xb9, + 0xfb, 0x08, 0x20, 0xc6, 0xa0, 0x26, 0xd4, 0xb6, 0x1f, 0x3d, 0x5d, 0xdb, 0xd9, 0xde, 0x6c, 0x5f, + 0x41, 0x0d, 0xa8, 0xe0, 0xad, 0xb5, 0xcd, 0xdf, 0x6e, 0x17, 0xd0, 0x04, 0x34, 0x1e, 0x3d, 0xde, + 0xd7, 0xc4, 0xb0, 0x88, 0x5a, 0x50, 0xdf, 0x78, 0xfc, 0x78, 0x47, 0x7b, 0x7c, 0xef, 0x5e, 0xbb, + 0xc4, 0x98, 0xf0, 0xd6, 0xde, 0xfe, 0x1a, 0xde, 0x6f, 0x97, 0xbb, 0xff, 0x5a, 0x80, 0xf6, 0x26, + 0xcf, 0xb5, 0xbf, 0x07, 0xae, 0xba, 0x0a, 0x65, 0x66, 0x90, 0xd2, 0x04, 0x6f, 0x46, 0xcc, 0x59, + 0x05, 0xb9, 0xf9, 0x62, 0x4e, 0xab, 0x2e, 0x41, 0x99, 0x8d, 0xd0, 0x3b, 0x30, 0xe9, 0x7f, 0x6d, + 0xb3, 0x5b, 0xf6, 0xe4, 0xd0, 0xd7, 0x86, 0x9e, 0x25, 0x83, 0x70, 0x4b, 0x40, 0x9f, 0x1e, 0xfa, + 0x4f, 0x3c, 0xab, 0xfb, 0xef, 0x25, 0xe8, 0x84, 0xd2, 0xbe, 0x8d, 0xb3, 0x7d, 0x92, 0x71, 0xb6, + 0xb7, 0x46, 0x74, 0x1d, 0xeb, 0x6b, 0xeb, 0xd0, 0x70, 0x87, 0x07, 0xb6, 0xe5, 0x1f, 0xe5, 0x38, + 0xdb, 0x28, 0xf7, 0x6e, 0x48, 0x8b, 0x63, 0x36, 0xf4, 0x29, 0xd4, 0x0e, 0xed, 0x21, 0x97, 0x50, + 0xce, 0x38, 0xfb, 0xa8, 0x84, 0x7b, 0x82, 0x12, 0x87, 0x2c, 0xdf, 0xb5, 0x8f, 0x05, 0xd0, 0x88, + 0x94, 0x64, 0x45, 0xcd, 0x40, 0x7f, 0xa1, 0x19, 0x36, 0x35, 0x8e, 0xe5, 0xd5, 0x5a, 0x1f, 0xe8, + 0x2f, 0x36, 0xd8, 0x38, 0xe3, 0x81, 0xc5, 0x73, 0x79, 0x60, 0x69, 0x8c, 0x07, 0x2e, 0x42, 0x4d, + 0x2e, 0xec, 0xf5, 0xee, 0xd7, 0xfd, 0xc3, 0x02, 0xcc, 0xc4, 0xc9, 0xe8, 0xf7, 0xc0, 0xd4, 0xbb, + 0xbf, 0x28, 0xc0, 0x6c, 0x4a, 0xa3, 0x6f, 0x63, 0x8d, 0x6b, 0xb1, 0x39, 0x08, 0x65, 0xe2, 0xf4, + 0x20, 0x7f, 0x8e, 0x51, 0x9b, 0xb8, 0xd0, 0x76, 0xfe, 0xa2, 0x0c, 0x93, 0x1b, 0x74, 0x70, 0x60, + 0x39, 0x51, 0xb9, 0xb8, 0x22, 0x5d, 0x57, 0xf0, 0xbc, 0x91, 0xd0, 0x37, 0x49, 0x96, 0x70, 0x5c, + 0x74, 0x1b, 0x4a, 0xba, 0x19, 0x2a, 0x7c, 0x63, 0x1c, 0xc3, 0x9a, 0x69, 0x62, 0x46, 0xa7, 0xfe, + 0x43, 0x51, 0x3a, 0xfa, 0x97, 0x50, 0x3f, 0xb0, 0x1c, 0xd3, 0x72, 0xfa, 0x4c, 0xc3, 0x52, 0xfa, + 0xae, 0x1a, 0x9d, 0xad, 0xb7, 0x2e, 0x88, 0x71, 0xc4, 0xa5, 0xfe, 0x7e, 0x11, 0x6a, 0x12, 0x8a, + 0x10, 0x94, 0x0f, 0x87, 0xb6, 0x38, 0xfa, 0x3a, 0xe6, 0xcf, 0x61, 0xae, 0xc3, 0xb2, 0xb4, 0x86, + 0xc8, 0x75, 0x3e, 0x86, 0xa6, 0xeb, 0xd1, 0x67, 0xa2, 0x0c, 0x0a, 0x73, 0xb0, 0xb6, 0xc8, 0xdf, + 0x76, 0x23, 0x84, 0x4c, 0x43, 0x93, 0xa4, 0xe8, 0x33, 0x68, 0xfa, 0xc6, 0x11, 0x19, 0xe8, 0xda, + 0x33, 0x9f, 0x3a, 0xdc, 0x5b, 0x5b, 0xeb, 0x6f, 0xbc, 0x7a, 0x39, 0xa7, 0x10, 0xc7, 0xa0, 0x4c, + 0x85, 0x65, 0x86, 0xe8, 0x61, 0xfd, 0xf9, 0x43, 0xe2, 0xf3, 0x34, 0x0c, 0x04, 0xc3, 0x03, 0x9f, + 0x3a, 0xa8, 0x07, 0xe0, 0x13, 0x4f, 0x73, 0xa9, 0x6d, 0x19, 0xa7, 0xbc, 0x74, 0x88, 0xf2, 0xe5, + 0x3d, 0xe2, 0xed, 0x72, 0x30, 0x6e, 0xf8, 0xe1, 0x23, 0x6f, 0x1b, 0xf0, 0xfc, 0x3a, 0xf0, 0x78, + 0x79, 0xd0, 0xc0, 0x35, 0x9e, 0x46, 0x07, 0x1e, 0xab, 0xc2, 0x79, 0x8a, 0x26, 0xb2, 0xfd, 0x06, + 0x96, 0x23, 0xd5, 0x81, 0xd2, 0x9a, 0x69, 0x22, 0x05, 0x6a, 0x72, 0x83, 0x64, 0x92, 0x17, 0x0e, + 0xd1, 0x0f, 0xa1, 0x6e, 0x52, 0x43, 0xe8, 0x5f, 0x3c, 0x87, 0xfe, 0x35, 0x93, 0x1a, 0x5c, 0xf9, + 0x69, 0xa8, 0x1c, 0x7a, 0xd4, 0x11, 0x29, 0x57, 0x1d, 0x8b, 0x41, 0xf7, 0x1f, 0x0b, 0x30, 0x15, + 0x9d, 0x93, 0xac, 0xf7, 0xc6, 0x4f, 0xae, 0x40, 0xcd, 0x24, 0x36, 0x09, 0xa4, 0x69, 0xd7, 0x71, + 0x38, 0x4c, 0xa9, 0x55, 0xba, 0x94, 0x5a, 0xe5, 0x84, 0x5a, 0x99, 0xd8, 0x54, 0xc9, 0xc6, 0xa6, + 0xb7, 0x61, 0x42, 0xec, 0x57, 0x48, 0xc1, 0x8b, 0x2f, 0xdc, 0x12, 0x40, 0x41, 0xd4, 0xbd, 0x06, + 0x33, 0x1b, 0xd4, 0x71, 0x88, 0x11, 0x50, 0x6f, 0xd7, 0xa3, 0x2f, 0x4e, 0xa5, 0x21, 0x76, 0xff, + 0xb8, 0x00, 0xb3, 0x59, 0x8c, 0x5c, 0xfa, 0x03, 0xa8, 0xb1, 0x92, 0x81, 0xf8, 0xbe, 0xec, 0xb3, + 0xac, 0xbc, 0x7a, 0x39, 0xb7, 0x74, 0x9e, 0xda, 0x6a, 0xcb, 0x31, 0x45, 0x4c, 0x0e, 0x05, 0xb0, + 0xd3, 0x77, 0x99, 0x70, 0xcd, 0x32, 0x65, 0x56, 0x5e, 0xe3, 0xe3, 0x6d, 0x13, 0xa9, 0x50, 0xb2, + 0x69, 0x5f, 0xde, 0x37, 0xf5, 0x30, 0xc2, 0x61, 0x06, 0xec, 0xfe, 0x65, 0x09, 0xca, 0x0f, 0xa8, + 0xe5, 0xa0, 0x5b, 0xd0, 0x21, 0x81, 0x61, 0x6a, 0x03, 0x6a, 0x6a, 0x1e, 0x39, 0xb1, 0x7c, 0x56, + 0xd1, 0x33, 0xad, 0x4a, 0x78, 0x8a, 0x21, 0x1e, 0x52, 0x13, 0x4b, 0x30, 0x5a, 0x84, 0xaa, 0x7f, + 0xa4, 0x7b, 0x66, 0x58, 0xcd, 0x5c, 0x8d, 0x9c, 0x90, 0x89, 0x12, 0xcd, 0x0b, 0x2c, 0x49, 0xd0, + 0x1c, 0x34, 0xf9, 0x93, 0xec, 0x40, 0x94, 0xf8, 0x19, 0x03, 0x07, 0x89, 0xfe, 0xc3, 0x22, 0x74, + 0xc2, 0x26, 0x85, 0x69, 0x79, 0x7c, 0x9b, 0x4e, 0xc3, 0x9e, 0x96, 0x44, 0x6c, 0x86, 0x70, 0xf4, + 0x3e, 0x84, 0x30, 0x8d, 0xc8, 0x3d, 0xe0, 0x07, 0xd6, 0xc0, 0x53, 0x12, 0x1e, 0x6e, 0x0d, 0x7a, + 0x0f, 0xa6, 0x6c, 0x5e, 0xfe, 0xc7, 0x94, 0xc2, 0x2d, 0x26, 0x05, 0x38, 0x24, 0x54, 0xff, 0xa2, + 0x00, 0x15, 0xae, 0x33, 0x9a, 0x84, 0xa2, 0x65, 0xca, 0xe4, 0xa1, 0x68, 0x99, 0xa8, 0x07, 0x75, + 0x5b, 0x3f, 0x20, 0x36, 0x33, 0xce, 0xa2, 0x8c, 0xc6, 0x3c, 0x22, 0x32, 0xea, 0x1d, 0x89, 0xc1, + 0x11, 0x0d, 0x5a, 0x85, 0x9a, 0x47, 0x74, 0xa6, 0xa9, 0xdc, 0x6d, 0x25, 0x6e, 0x49, 0xec, 0x7a, + 0xd4, 0x20, 0xbe, 0xbf, 0xe7, 0x12, 0xa3, 0xb7, 0xbd, 0x89, 0x43, 0x42, 0xb4, 0x02, 0xd3, 0x7c, + 0xe3, 0x0d, 0x8f, 0xe8, 0x01, 0x89, 0xf7, 0x9e, 0x37, 0x1f, 0x30, 0x62, 0xb8, 0x0d, 0x8e, 0x0a, + 0xb7, 0xbf, 0x7b, 0x07, 0xaa, 0x6c, 0x9f, 0x89, 0xc9, 0x0e, 0x8d, 0xdd, 0xb8, 0x9c, 0x3f, 0x7b, + 0x68, 0x03, 0xfd, 0xc5, 0x56, 0x60, 0x44, 0x87, 0xd6, 0xfd, 0x59, 0x01, 0xca, 0xfb, 0xba, 0x7f, + 0xcc, 0xc2, 0x9e, 0xef, 0x12, 0x43, 0x66, 0xc1, 0xfc, 0x99, 0x6d, 0x2b, 0x13, 0x14, 0x78, 0xba, + 0xe3, 0xeb, 0x51, 0xa4, 0x63, 0x27, 0xc5, 0xe4, 0xec, 0x27, 0xc0, 0x39, 0xc9, 0x56, 0x79, 0x34, + 0xd9, 0x62, 0x15, 0x74, 0x98, 0xb2, 0x78, 0xcc, 0x24, 0x85, 0x53, 0x35, 0x23, 0xd8, 0xb6, 0xf9, + 0xa0, 0x5c, 0x2f, 0xb6, 0x4b, 0xdd, 0x9f, 0x57, 0xa1, 0x86, 0x89, 0x41, 0x4f, 0xf8, 0x5d, 0xd6, + 0xd4, 0x8d, 0x63, 0xcd, 0x72, 0x02, 0xe2, 0x04, 0x61, 0x84, 0x9f, 0x8f, 0x2f, 0x57, 0x41, 0xd6, + 0x5b, 0x33, 0x8e, 0xb7, 0x05, 0x89, 0xa8, 0x73, 0x41, 0x8f, 0x00, 0x68, 0x15, 0x66, 0x44, 0xad, + 0x17, 0x10, 0x93, 0x65, 0x22, 0x3e, 0x91, 0xf9, 0x48, 0x91, 0xe7, 0x23, 0x57, 0x23, 0xe4, 0x06, + 0xc3, 0x89, 0xd4, 0xe4, 0x4b, 0x40, 0x31, 0x0f, 0x8f, 0x08, 0x16, 0x09, 0x0f, 0xb0, 0xd3, 0x0b, + 0x9b, 0xc0, 0xf7, 0x24, 0x02, 0x77, 0x22, 0xe2, 0x10, 0x84, 0x96, 0x60, 0xda, 0x08, 0x5d, 0x5c, + 0x63, 0xf7, 0x24, 0x49, 0x84, 0x7c, 0x3c, 0x19, 0xe1, 0xd8, 0x4d, 0x4a, 0xd0, 0x12, 0xa0, 0x23, + 0xb6, 0xc6, 0xb4, 0x82, 0x15, 0xd1, 0x8b, 0x10, 0x98, 0x84, 0x76, 0x77, 0x61, 0x4a, 0x52, 0x47, + 0xaa, 0x55, 0xc7, 0xa9, 0x36, 0x29, 0x28, 0x23, 0xbd, 0xde, 0x82, 0x96, 0xad, 0xfb, 0x81, 0xa6, + 0xbb, 0xae, 0x6d, 0x11, 0x93, 0xf7, 0x21, 0x5b, 0xb8, 0xc9, 0x60, 0x6b, 0x02, 0x84, 0xd6, 0xa0, + 0x63, 0x93, 0xbe, 0x6e, 0x9c, 0x26, 0xb3, 0xc0, 0xfa, 0x19, 0x59, 0x60, 0x5b, 0x90, 0x27, 0x4a, + 0xa0, 0x8f, 0x81, 0xa5, 0x79, 0xda, 0x31, 0x39, 0x0d, 0xdb, 0x3a, 0x6f, 0x8e, 0x9c, 0xd9, 0x43, + 0xfd, 0xc5, 0x8f, 0xc9, 0xa9, 0x3c, 0xb0, 0xda, 0x40, 0x8c, 0xd0, 0x2d, 0xb8, 0x1a, 0x78, 0x56, + 0xbf, 0xcf, 0xae, 0x39, 0xdd, 0xd3, 0x07, 0xbe, 0xd8, 0x36, 0xe0, 0x6a, 0x4e, 0x48, 0xd4, 0x2e, + 0xc7, 0xa0, 0x5d, 0x68, 0x33, 0x13, 0x3c, 0x21, 0xda, 0x81, 0x6e, 0x1c, 0x1f, 0x5a, 0xb6, 0xed, + 0x2b, 0x4d, 0x3e, 0xdb, 0xbb, 0x39, 0x16, 0xc2, 0x08, 0xd7, 0x43, 0x3a, 0xd9, 0x0e, 0xd1, 0xd3, + 0x50, 0xf5, 0x33, 0x98, 0xca, 0x98, 0x52, 0xb2, 0xd5, 0xd1, 0xc8, 0x69, 0x75, 0xb4, 0x12, 0xad, + 0x0e, 0xf5, 0x2e, 0xb4, 0x92, 0xab, 0x7a, 0x5d, 0x9b, 0x24, 0xc5, 0xbb, 0x0e, 0xd3, 0x79, 0x3a, + 0xbe, 0x4e, 0x46, 0x35, 0xd9, 0x6a, 0xf9, 0xa7, 0x3a, 0xd4, 0x76, 0x89, 0xe7, 0x5b, 0x7e, 0x80, + 0x66, 0xa0, 0xea, 0x93, 0xaf, 0x35, 0x87, 0x72, 0xd6, 0x32, 0xae, 0xf8, 0xe4, 0xeb, 0x47, 0x94, + 0x59, 0x9a, 0xb8, 0x32, 0xb5, 0xa4, 0x5f, 0x89, 0xcb, 0xb4, 0x2d, 0x30, 0xf1, 0x0e, 0x64, 0xdd, + 0xaf, 0x94, 0x71, 0x3f, 0x39, 0xd7, 0xe5, 0xdc, 0xaf, 0x3c, 0xde, 0xfd, 0xee, 0xc2, 0x75, 0xa9, + 0x64, 0x8e, 0x17, 0x56, 0xb8, 0xae, 0xd7, 0x04, 0xc1, 0xc6, 0x88, 0xe3, 0xe5, 0xbb, 0x6e, 0xf5, + 0x02, 0xae, 0xbb, 0x02, 0xb3, 0xb1, 0xeb, 0xba, 0x7a, 0x60, 0x1c, 0x11, 0x69, 0x85, 0xc2, 0x59, + 0xda, 0x11, 0x76, 0x57, 0x20, 0xc7, 0xb8, 0x6f, 0x7d, 0x8c, 0xfb, 0xde, 0x81, 0x59, 0xb9, 0xba, + 0xac, 0x17, 0x37, 0xf8, 0xd2, 0xa6, 0x05, 0xf6, 0x7e, 0xda, 0x71, 0x73, 0x9c, 0x1e, 0x2e, 0xeb, + 0xf4, 0xcd, 0x51, 0xa7, 0xff, 0x18, 0x14, 0xa9, 0xd4, 0xa8, 0xef, 0xb7, 0xb8, 0x5a, 0x52, 0xe9, + 0x9d, 0xac, 0xaf, 0xe7, 0x86, 0x8b, 0x89, 0x4b, 0x87, 0x8b, 0xc9, 0x4c, 0xb8, 0x08, 0x6d, 0x2c, + 0x3f, 0x5c, 0xac, 0xc2, 0x8c, 0x54, 0x3b, 0x1d, 0x35, 0x94, 0x29, 0xae, 0xf3, 0x55, 0x81, 0xdc, + 0x4f, 0x85, 0x8d, 0x31, 0x21, 0xa6, 0x9d, 0x17, 0x62, 0xf8, 0xcb, 0x2a, 0xdf, 0xd0, 0x1d, 0xa5, + 0x13, 0xbe, 0xac, 0x62, 0x23, 0x74, 0x07, 0x2a, 0x07, 0xa4, 0x6f, 0x39, 0x0a, 0xca, 0x54, 0x38, + 0x69, 0x1f, 0x5e, 0x67, 0x34, 0xf7, 0xaf, 0x60, 0x41, 0x8c, 0x16, 0xa1, 0x6d, 0xd0, 0x81, 0xcb, + 0xf5, 0x0d, 0x33, 0xdc, 0xab, 0xcc, 0xb1, 0xef, 0x5f, 0xc1, 0x53, 0x21, 0x46, 0xd6, 0x22, 0xff, + 0x87, 0xb1, 0x68, 0x5d, 0x81, 0xd9, 0x4c, 0x60, 0xd5, 0x8c, 0x23, 0xdd, 0xe9, 0x93, 0x2e, 0x86, + 0xab, 0x39, 0x2b, 0x3c, 0x23, 0x63, 0x7f, 0x0b, 0x5a, 0x81, 0x37, 0x74, 0x0c, 0x9d, 0x59, 0xae, + 0x1e, 0xc8, 0x98, 0xd5, 0x8c, 0x60, 0x6b, 0x41, 0xb7, 0x0b, 0x0d, 0x79, 0xc8, 0xc4, 0x1c, 0x13, + 0xb6, 0xba, 0x7f, 0x5a, 0x80, 0x0a, 0x33, 0xd5, 0xd3, 0xdc, 0x5c, 0x45, 0x81, 0xda, 0x09, 0x93, + 0x20, 0x4b, 0x92, 0x06, 0x0e, 0x87, 0xe8, 0x06, 0x34, 0xb8, 0xe5, 0x73, 0x16, 0x71, 0xf7, 0xd6, + 0x19, 0x80, 0xe5, 0x5c, 0x91, 0x5b, 0x84, 0xbc, 0x22, 0x6b, 0xe4, 0x6e, 0xf1, 0x54, 0xf2, 0xaf, + 0x8c, 0xb9, 0xc6, 0x45, 0xbe, 0x8f, 0xd2, 0xd7, 0x38, 0xab, 0x27, 0xba, 0xcf, 0xa0, 0x16, 0xfa, + 0xd4, 0x6d, 0x40, 0x22, 0x45, 0x8a, 0xfa, 0x03, 0x61, 0x32, 0xd6, 0xc0, 0x1d, 0x81, 0xd9, 0x8c, + 0x11, 0x67, 0xc4, 0x9d, 0x62, 0x7e, 0xdc, 0xe9, 0xfe, 0xaa, 0x20, 0xab, 0xe0, 0x8b, 0x6d, 0xca, + 0xbb, 0xe1, 0x7b, 0xcb, 0x52, 0xee, 0x7b, 0xcb, 0xf0, 0x8d, 0xe5, 0xdb, 0x67, 0xa6, 0x30, 0xbc, + 0xf8, 0x27, 0xe8, 0xa3, 0x84, 0xeb, 0x56, 0xb8, 0xeb, 0xc6, 0xad, 0x0f, 0x5e, 0x70, 0xe7, 0xfa, + 0xed, 0xb7, 0xb1, 0xce, 0x2e, 0x40, 0x9d, 0x47, 0xd3, 0x47, 0xf4, 0x79, 0xb7, 0x0a, 0xe5, 0xbd, + 0x80, 0xba, 0xdd, 0x06, 0xd4, 0xd8, 0x5f, 0x97, 0x98, 0xdd, 0xdf, 0x82, 0xe6, 0x1e, 0xf1, 0xd9, + 0x42, 0x77, 0x28, 0x75, 0xc7, 0x74, 0x69, 0x0a, 0x97, 0xe9, 0xd2, 0xfc, 0x51, 0x15, 0x6a, 0xb2, + 0x37, 0x8b, 0xde, 0x4f, 0xec, 0x78, 0x73, 0x75, 0xa6, 0x17, 0x7e, 0xc4, 0x10, 0x36, 0x1b, 0xf8, + 0x46, 0x8a, 0x83, 0xf8, 0xff, 0x30, 0xc1, 0xfe, 0x6a, 0x9e, 0x2c, 0xf2, 0x64, 0xdd, 0x30, 0x9b, + 0xe0, 0x11, 0x08, 0xc1, 0xd4, 0x62, 0xc4, 0x51, 0x41, 0xf8, 0x11, 0xd4, 0x4d, 0xcb, 0xe7, 0x39, + 0x8c, 0x3c, 0xae, 0xeb, 0x23, 0x73, 0x6d, 0x4a, 0x02, 0x1c, 0x91, 0xa2, 0x4f, 0x01, 0xc2, 0xe7, + 0xa8, 0x2b, 0xf8, 0xc6, 0xe8, 0x84, 0x9b, 0x11, 0x0d, 0x4e, 0xd0, 0xb3, 0x49, 0x4f, 0x74, 0xdb, + 0x32, 0xf5, 0x80, 0xc8, 0x2e, 0xc3, 0xe8, 0xa4, 0x4f, 0x25, 0x01, 0x8e, 0x48, 0xd1, 0x27, 0xd0, + 0x08, 0x9f, 0x4d, 0x79, 0xe3, 0xde, 0x18, 0x9d, 0x33, 0x64, 0x34, 0x71, 0x4c, 0x9d, 0x6e, 0xbc, + 0x35, 0x5e, 0xd3, 0x78, 0xfb, 0x21, 0xb4, 0x7c, 0x71, 0xc2, 0x9a, 0x4d, 0xa9, 0xab, 0x4c, 0xcb, + 0xcb, 0x26, 0x3c, 0xcc, 0xc4, 0xf1, 0xe3, 0xa6, 0x9f, 0xb0, 0x85, 0xb7, 0xa0, 0xfc, 0x8c, 0x5a, + 0x8e, 0x32, 0xc3, 0x19, 0x26, 0x52, 0x35, 0x2a, 0xe6, 0x28, 0xf4, 0x1e, 0x54, 0x9f, 0xf1, 0x4a, + 0x4a, 0x99, 0x95, 0xce, 0x91, 0x24, 0x22, 0x26, 0x96, 0x68, 0x26, 0x2b, 0xd0, 0xfd, 0x63, 0xe5, + 0x5a, 0x46, 0x16, 0x2b, 0xa8, 0x30, 0x47, 0xa1, 0xe5, 0xa8, 0x2d, 0xac, 0x70, 0xa2, 0x6b, 0xd9, + 0x0e, 0x7f, 0xb6, 0x19, 0xdc, 0x83, 0x86, 0x48, 0x20, 0x1c, 0xfa, 0x5c, 0xb9, 0x2e, 0x6f, 0xf7, + 0x88, 0x47, 0xda, 0x3c, 0xae, 0x1b, 0xf2, 0x89, 0xe9, 0xe0, 0x07, 0xd4, 0x55, 0xd4, 0x8c, 0x0e, + 0xcc, 0x15, 0x30, 0x47, 0xa1, 0x5b, 0x50, 0xf3, 0x85, 0x63, 0x28, 0x37, 0xe4, 0x2b, 0xf1, 0x24, + 0x95, 0x4b, 0x4c, 0x1c, 0x12, 0xa8, 0x77, 0xa3, 0x4e, 0xf0, 0x85, 0x9b, 0x8e, 0xdd, 0x3f, 0xb8, + 0x0e, 0xcd, 0x44, 0x77, 0x11, 0xdd, 0x4e, 0xf9, 0xc7, 0xf5, 0x5e, 0xf2, 0xe3, 0x9b, 0x1c, 0x1f, + 0xf9, 0x22, 0xdf, 0x47, 0xd4, 0x0c, 0xdf, 0x78, 0x3f, 0xf9, 0x24, 0x61, 0xb2, 0xc2, 0x4f, 0xde, + 0xcc, 0x9d, 0x33, 0xc7, 0x6c, 0x3f, 0x4b, 0x9a, 0xad, 0x70, 0x95, 0xb9, 0xfc, 0x79, 0x5f, 0x6f, + 0xba, 0x95, 0x5f, 0x13, 0xd3, 0xbd, 0x05, 0x35, 0x4f, 0x54, 0x4e, 0xd2, 0x76, 0xdb, 0xd9, 0x8a, + 0x0a, 0x87, 0x04, 0xe8, 0x1d, 0xa8, 0xb0, 0xc4, 0xf2, 0x54, 0x5a, 0x6c, 0xfc, 0x51, 0x11, 0xbf, + 0xb0, 0xb1, 0x40, 0x32, 0x89, 0x61, 0xfa, 0xa9, 0x66, 0x24, 0xca, 0xfb, 0x12, 0x87, 0x04, 0x4c, + 0x41, 0xde, 0x3e, 0xbe, 0x91, 0x51, 0x30, 0xd1, 0x2f, 0xfe, 0x30, 0xf2, 0xad, 0x37, 0x32, 0x2d, + 0xe3, 0x84, 0x15, 0x66, 0xfd, 0xeb, 0x36, 0x94, 0x6d, 0xaa, 0x9b, 0xca, 0x82, 0x34, 0xca, 0x3c, + 0x96, 0x1d, 0xaa, 0x9b, 0x98, 0x93, 0xb1, 0x39, 0xd8, 0x5f, 0x62, 0x2a, 0xef, 0x9f, 0x31, 0xc7, + 0x0e, 0x27, 0xc1, 0x92, 0x14, 0xad, 0x40, 0x85, 0x77, 0xd1, 0x95, 0x5b, 0x99, 0x2b, 0x26, 0xc9, + 0xc3, 0x9b, 0xeb, 0x58, 0x10, 0xa2, 0x1f, 0xc4, 0xfd, 0xfa, 0xc5, 0x4c, 0x36, 0x39, 0xc2, 0x93, + 0x68, 0xd2, 0xb3, 0x99, 0xfc, 0x80, 0x7a, 0x44, 0x59, 0x3a, 0x63, 0xa6, 0x3d, 0x46, 0x81, 0x05, + 0x21, 0x5b, 0x10, 0x7f, 0x30, 0x95, 0xdb, 0x67, 0x2c, 0x88, 0xb3, 0x98, 0x58, 0x92, 0xa2, 0x8d, + 0xcc, 0x1b, 0xf3, 0x1e, 0x67, 0x9d, 0x1f, 0xc3, 0x9a, 0xff, 0xae, 0x1c, 0x6d, 0xc3, 0x24, 0x1f, + 0xb2, 0x12, 0x49, 0x88, 0x59, 0xce, 0xbc, 0xa9, 0x1a, 0x11, 0x43, 0x4c, 0x29, 0x68, 0xc2, 0x4f, + 0x0e, 0xd1, 0x3a, 0xaf, 0x49, 0x1d, 0xfa, 0xdc, 0x26, 0x66, 0x9f, 0x28, 0x2b, 0x67, 0xa8, 0xb3, + 0x16, 0xd3, 0xe1, 0x24, 0x13, 0xda, 0x82, 0x56, 0x62, 0x68, 0x2a, 0x1f, 0x64, 0x5e, 0xdb, 0x8d, + 0x11, 0x62, 0xe2, 0x14, 0x1b, 0xb3, 0x69, 0x57, 0x64, 0xae, 0xca, 0x6a, 0xc6, 0xa6, 0x65, 0x46, + 0x8b, 0x43, 0x02, 0x16, 0x52, 0xdd, 0x30, 0xcb, 0x55, 0x3e, 0xcc, 0x84, 0xd4, 0x28, 0xff, 0xc5, + 0x31, 0x51, 0xfa, 0x36, 0xb8, 0x73, 0xfe, 0xdb, 0xe0, 0xd3, 0x73, 0xdd, 0x06, 0x9f, 0xbd, 0xee, + 0x36, 0xf8, 0xdd, 0xc2, 0xe5, 0xaf, 0x03, 0xf4, 0xa3, 0x64, 0xee, 0x98, 0xa8, 0x0b, 0x8b, 0x67, + 0xd4, 0x85, 0x57, 0x23, 0x8e, 0xc4, 0xeb, 0xc4, 0x8f, 0xa0, 0xcc, 0x1c, 0x0c, 0xdd, 0x86, 0x7a, + 0x54, 0xf7, 0x16, 0xc6, 0xd5, 0xbd, 0x11, 0x89, 0xfa, 0xab, 0x22, 0x54, 0x85, 0x63, 0xa2, 0x2f, + 0x46, 0xde, 0x10, 0xbd, 0x7d, 0x86, 0x1f, 0x8f, 0xbe, 0x20, 0x12, 0x35, 0x00, 0x7f, 0x43, 0xe1, + 0x69, 0xe2, 0x63, 0x99, 0x83, 0xd3, 0x80, 0x88, 0xa6, 0x49, 0x99, 0xd5, 0x00, 0x02, 0xf7, 0x84, + 0xa1, 0xd6, 0x19, 0x46, 0xfd, 0x8f, 0x42, 0xfc, 0x4a, 0x69, 0x1a, 0x2a, 0xa2, 0xcd, 0x2d, 0x72, + 0x5b, 0x31, 0x40, 0x0b, 0xd0, 0x1e, 0x58, 0x8e, 0xe6, 0xd3, 0xa1, 0x67, 0xa4, 0xfb, 0x91, 0x93, + 0x03, 0xcb, 0xd9, 0xe3, 0x60, 0xd1, 0x2d, 0x58, 0x10, 0x7d, 0xd8, 0x14, 0x65, 0x49, 0x52, 0xea, + 0x2f, 0x92, 0x94, 0x4b, 0x80, 0x04, 0x95, 0xa9, 0x99, 0xd4, 0xf0, 0xb5, 0x80, 0x06, 0xba, 0xcd, + 0x2f, 0xb4, 0x32, 0x6e, 0x4b, 0xcc, 0x26, 0x35, 0xfc, 0x7d, 0x06, 0x47, 0x3d, 0xb8, 0x1a, 0x52, + 0xf3, 0xe5, 0x48, 0xf2, 0x0a, 0x27, 0xef, 0x48, 0x14, 0x5f, 0x8e, 0xa0, 0xef, 0xc2, 0x84, 0x4c, + 0xf4, 0x35, 0x93, 0xd8, 0x81, 0xfc, 0xde, 0x0c, 0x37, 0x45, 0x46, 0xbf, 0xc9, 0x40, 0xea, 0xbf, + 0x15, 0xa1, 0xc2, 0xc3, 0xd4, 0x19, 0xb5, 0x4c, 0x61, 0x4c, 0x0f, 0xe5, 0x2b, 0x98, 0x8a, 0x8a, + 0x4d, 0x5e, 0x2d, 0x87, 0xaf, 0x12, 0x16, 0xc6, 0x47, 0xc3, 0x5e, 0xaa, 0x0a, 0xc5, 0x93, 0x07, + 0xc9, 0xa1, 0x8f, 0x7e, 0x02, 0x28, 0xae, 0x5f, 0x65, 0x75, 0x1d, 0x36, 0xb1, 0x16, 0xcf, 0x21, + 0x75, 0x43, 0xf2, 0xe0, 0xce, 0x41, 0x06, 0xe2, 0xab, 0x5f, 0xc0, 0xc4, 0x79, 0x4b, 0xe0, 0x69, + 0xa8, 0x24, 0x0f, 0x58, 0x0c, 0xd4, 0x75, 0x68, 0x67, 0xe7, 0xb9, 0xb0, 0x8c, 0xff, 0x2c, 0xc4, + 0xef, 0x69, 0xcf, 0x7a, 0x11, 0x9a, 0x73, 0x8d, 0xe4, 0xda, 0xf9, 0x05, 0xeb, 0x4f, 0xf5, 0xf4, + 0x75, 0x66, 0x7e, 0x0b, 0x3a, 0xe2, 0x5a, 0x4c, 0x5a, 0xa4, 0xf0, 0x9b, 0x29, 0x81, 0x88, 0x0d, + 0x72, 0x09, 0x90, 0xa4, 0x4d, 0xda, 0x63, 0x49, 0x98, 0xaf, 0xc0, 0xc4, 0xe6, 0xa8, 0xd6, 0xa0, + 0xc2, 0xef, 0x29, 0xf5, 0xaf, 0x0b, 0x50, 0x15, 0x37, 0xd6, 0xb9, 0x3d, 0x5d, 0x90, 0xe7, 0xbc, + 0x0a, 0x3e, 0xcf, 0x7a, 0xc4, 0xad, 0x98, 0xb3, 0x1e, 0x81, 0x48, 0xad, 0x47, 0xd2, 0xe6, 0xac, + 0x47, 0x60, 0x12, 0xeb, 0xf9, 0xbd, 0x42, 0xfa, 0x6b, 0xb2, 0x8b, 0x3b, 0xd0, 0x77, 0x16, 0x72, + 0xd7, 0x60, 0x22, 0x75, 0x01, 0x5f, 0x5c, 0x17, 0xf5, 0x0b, 0x68, 0x26, 0xae, 0xcd, 0x4b, 0x08, + 0xf8, 0x12, 0x5a, 0xc9, 0x7b, 0xf7, 0xe2, 0x12, 0xba, 0x3f, 0x47, 0x50, 0x15, 0x5f, 0xbf, 0xa0, + 0x85, 0x54, 0x2d, 0x32, 0xdd, 0x93, 0xbf, 0x26, 0xc8, 0x29, 0x43, 0xee, 0xe6, 0x97, 0x21, 0x33, + 0x31, 0xcb, 0xf8, 0x0a, 0xe4, 0xce, 0x48, 0x05, 0xa2, 0x64, 0x67, 0xca, 0x29, 0x3e, 0x3e, 0x1e, + 0x2d, 0x3e, 0xd4, 0x91, 0xd9, 0x7e, 0x53, 0x77, 0xe4, 0xd5, 0x1d, 0xe7, 0xa8, 0x12, 0x7a, 0x99, + 0x2a, 0x61, 0x36, 0xf3, 0x61, 0x54, 0xb6, 0x40, 0x58, 0x48, 0x15, 0x08, 0xd3, 0x59, 0xea, 0x44, + 0x6d, 0xd0, 0xcb, 0xd4, 0x06, 0xb3, 0x79, 0xb4, 0x89, 0xb2, 0x60, 0x31, 0x5d, 0x16, 0xcc, 0x64, + 0xc9, 0x53, 0x15, 0xc1, 0x07, 0xd9, 0x8a, 0xe0, 0x5a, 0x2e, 0x79, 0xb2, 0x18, 0x58, 0x4c, 0x17, + 0x03, 0x23, 0xf2, 0x53, 0x75, 0x40, 0x2f, 0x53, 0x07, 0xcc, 0xe6, 0x52, 0xc7, 0x25, 0xc0, 0xe7, + 0xb9, 0x25, 0xc0, 0x8d, 0x51, 0xae, 0x31, 0xd9, 0xff, 0xe6, 0x98, 0xec, 0xff, 0xcd, 0x5c, 0x09, + 0xe3, 0x12, 0xff, 0xdf, 0x64, 0xdb, 0xdf, 0xd7, 0x6c, 0xfb, 0x6f, 0xe2, 0x6c, 0xfb, 0xee, 0xc8, + 0x1d, 0x7c, 0x33, 0xdf, 0x33, 0xbe, 0x93, 0x44, 0xfb, 0xef, 0x7e, 0xfd, 0x12, 0x6d, 0xf5, 0x93, + 0x4b, 0xe7, 0xd0, 0xea, 0xe3, 0x38, 0x1d, 0xbc, 0x78, 0xfe, 0x80, 0xa0, 0x3c, 0x60, 0x01, 0x44, + 0xbc, 0x0b, 0xe6, 0xcf, 0x71, 0x96, 0xf5, 0x3b, 0xa5, 0x28, 0xcb, 0x5a, 0x81, 0xe9, 0xe8, 0xd3, + 0xd3, 0xe4, 0xfa, 0xc5, 0xfb, 0x1a, 0x14, 0xe1, 0xe2, 0x1d, 0x58, 0x85, 0x99, 0x98, 0x23, 0xb9, + 0x07, 0xe2, 0x60, 0xaf, 0x46, 0xc8, 0x44, 0xb9, 0xb1, 0x04, 0xc8, 0xf4, 0x98, 0x75, 0xa7, 0xe6, + 0x90, 0xd9, 0x93, 0xc4, 0xa4, 0xf6, 0x38, 0xa4, 0x4e, 0xca, 0x17, 0x47, 0xd2, 0x91, 0xa8, 0x84, + 0xf4, 0xaf, 0xa0, 0x1d, 0x7f, 0x8b, 0x22, 0x43, 0x52, 0x25, 0xf3, 0x95, 0x7a, 0x2a, 0x14, 0x46, + 0x1f, 0xde, 0x7a, 0x32, 0x36, 0x4d, 0xb9, 0x69, 0x80, 0xaa, 0xc1, 0x54, 0x86, 0x06, 0xa9, 0xfc, + 0x03, 0x2c, 0x73, 0x68, 0x48, 0x2f, 0x6a, 0xe1, 0x68, 0x9c, 0x9f, 0xd0, 0x33, 0x0e, 0xf9, 0x33, + 0x39, 0x51, 0xa7, 0x34, 0x70, 0x34, 0x56, 0x9f, 0xa6, 0x13, 0xc4, 0x71, 0x3e, 0x5f, 0xb8, 0xa8, + 0xcf, 0x4f, 0x65, 0xd2, 0xbd, 0x5b, 0x8b, 0x50, 0xe1, 0x3f, 0x07, 0x44, 0x00, 0xd5, 0xdd, 0x27, + 0xeb, 0x3b, 0xdb, 0x1b, 0xed, 0x2b, 0xa8, 0x09, 0xb5, 0x5d, 0xbc, 0xfd, 0x74, 0x6d, 0x7f, 0xab, + 0x5d, 0x40, 0x0d, 0xa8, 0xec, 0x3c, 0xde, 0x58, 0xdb, 0x69, 0x17, 0x57, 0x1f, 0x40, 0x5d, 0xfe, + 0x5c, 0xcb, 0x43, 0x9f, 0x43, 0x4d, 0x3e, 0xa3, 0xf8, 0xc2, 0x4a, 0xff, 0x90, 0x50, 0x55, 0x46, + 0x11, 0x22, 0xc9, 0x59, 0x29, 0xac, 0xee, 0x40, 0x5d, 0x7e, 0x0a, 0xe8, 0xa1, 0x2f, 0xa1, 0x26, + 0x9f, 0x13, 0xb2, 0xd2, 0x1f, 0x74, 0x26, 0x64, 0x65, 0xbe, 0x20, 0x5c, 0x28, 0xac, 0x14, 0x56, + 0x8f, 0x60, 0x32, 0xfd, 0x91, 0x1d, 0x7a, 0x0a, 0x53, 0xfc, 0x21, 0x02, 0xfb, 0xe8, 0x66, 0x32, + 0xb0, 0x8e, 0x7e, 0xaa, 0xa7, 0xce, 0x8d, 0xc5, 0x27, 0x66, 0x7a, 0x0e, 0xd5, 0x1d, 0xf1, 0xab, + 0xb2, 0x5e, 0x94, 0x73, 0x4e, 0x65, 0xec, 0x48, 0xcd, 0x02, 0x18, 0x27, 0xfa, 0x2c, 0xdd, 0x34, + 0x9f, 0xce, 0x2b, 0x57, 0xd4, 0x5c, 0x28, 0x9f, 0xf8, 0xcf, 0xa2, 0xcf, 0xd4, 0x3e, 0x88, 0xdf, + 0x4c, 0xb5, 0xb3, 0x6f, 0x19, 0xd4, 0x11, 0x08, 0x9f, 0xfb, 0x7f, 0x57, 0xd7, 0xf5, 0xcf, 0xbf, + 0xf9, 0xe7, 0x9b, 0x57, 0xbe, 0xf9, 0xe5, 0xcd, 0xc2, 0xdf, 0xfe, 0xf2, 0x66, 0xe1, 0x4f, 0xfe, + 0xe5, 0x66, 0xe1, 0x27, 0x4b, 0xe7, 0xfa, 0x99, 0x9a, 0x94, 0x77, 0x50, 0xe5, 0xa0, 0x0f, 0xff, + 0x27, 0x00, 0x00, 0xff, 0xff, 0x5d, 0x3e, 0xa7, 0x21, 0xb3, 0x3c, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -6466,6 +6681,22 @@ func (m *Recover) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.ActiveBackfills) > 0 { + for k := range m.ActiveBackfills { + v := m.ActiveBackfills[k] + baseI := i + i -= 8 + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(v)) + i-- + dAtA[i] = 0x11 + i = encodeVarintRuntime(dAtA, i, uint64(k)) + i-- + dAtA[i] = 0x8 + i = encodeVarintRuntime(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0x5a + } + } if len(m.TriggerParamsJson) > 0 { i -= len(m.TriggerParamsJson) copy(dAtA[i:], m.TriggerParamsJson) @@ -6602,6 +6833,15 @@ func (m *Persist) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ActiveBackfillChange != nil { + { + size := m.ActiveBackfillChange.ProtoSize() + i -= size + if _, err := m.ActiveBackfillChange.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + } + } if m.Rescan { i-- if m.Rescan { @@ -6783,6 +7023,81 @@ func (m *Persist) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *Persist_Begin) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Persist_Begin) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Begin != nil { + { + size, err := m.Begin.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRuntime(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x92 + } + return len(dAtA) - i, nil +} +func (m *Persist_CompleteBinding) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Persist_CompleteBinding) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + i = encodeVarintRuntime(dAtA, i, uint64(m.CompleteBinding)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x98 + return len(dAtA) - i, nil +} +func (m *ActiveBackfillBegin) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ActiveBackfillBegin) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ActiveBackfillBegin) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.TruncatedAt != 0 { + i -= 8 + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(m.TruncatedAt)) + i-- + dAtA[i] = 0x11 + } + if m.Binding != 0 { + i = encodeVarintRuntime(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *Persisted) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) @@ -8009,17 +8324,121 @@ func (m *Materialize_Flush) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if len(m.ConnectorPatchesJson) > 0 { - i -= len(m.ConnectorPatchesJson) - copy(dAtA[i:], m.ConnectorPatchesJson) - i = encodeVarintRuntime(dAtA, i, uint64(len(m.ConnectorPatchesJson))) - i-- - dAtA[i] = 0xa + if len(m.BackfillCompletes) > 0 { + for iNdEx := len(m.BackfillCompletes) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.BackfillCompletes[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRuntime(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } } - return len(dAtA) - i, nil -} - -func (m *Materialize_Flushed) Marshal() (dAtA []byte, err error) { + if len(m.BackfillBegins) > 0 { + for iNdEx := len(m.BackfillBegins) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.BackfillBegins[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRuntime(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if len(m.ConnectorPatchesJson) > 0 { + i -= len(m.ConnectorPatchesJson) + copy(dAtA[i:], m.ConnectorPatchesJson) + i = encodeVarintRuntime(dAtA, i, uint64(len(m.ConnectorPatchesJson))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Materialize_Flush_BackfillBegin) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Materialize_Flush_BackfillBegin) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Materialize_Flush_BackfillBegin) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Clock != 0 { + i -= 8 + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(m.Clock)) + i-- + dAtA[i] = 0x11 + } + if m.Binding != 0 { + i = encodeVarintRuntime(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *Materialize_Flush_BackfillComplete) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Materialize_Flush_BackfillComplete) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Materialize_Flush_BackfillComplete) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Clock != 0 { + i -= 8 + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(m.Clock)) + i-- + dAtA[i] = 0x11 + } + if m.Binding != 0 { + i = encodeVarintRuntime(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *Materialize_Flushed) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -10035,6 +10454,14 @@ func (m *Recover) ProtoSize() (n int) { if l > 0 { n += 1 + l + sovRuntime(uint64(l)) } + if len(m.ActiveBackfills) > 0 { + for k, v := range m.ActiveBackfills { + _ = k + _ = v + mapEntrySize := 1 + sovRuntime(uint64(k)) + 1 + 8 + n += mapEntrySize + 1 + sovRuntime(uint64(mapEntrySize)) + } + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -10122,6 +10549,48 @@ func (m *Persist) ProtoSize() (n int) { if m.Rescan { n += 3 } + if m.ActiveBackfillChange != nil { + n += m.ActiveBackfillChange.ProtoSize() + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Persist_Begin) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Begin != nil { + l = m.Begin.ProtoSize() + n += 2 + l + sovRuntime(uint64(l)) + } + return n +} +func (m *Persist_CompleteBinding) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += 2 + sovRuntime(uint64(m.CompleteBinding)) + return n +} +func (m *ActiveBackfillBegin) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Binding != 0 { + n += 1 + sovRuntime(uint64(m.Binding)) + } + if m.TruncatedAt != 0 { + n += 9 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -10599,6 +11068,54 @@ func (m *Materialize_Flush) ProtoSize() (n int) { if l > 0 { n += 1 + l + sovRuntime(uint64(l)) } + if len(m.BackfillBegins) > 0 { + for _, e := range m.BackfillBegins { + l = e.ProtoSize() + n += 1 + l + sovRuntime(uint64(l)) + } + } + if len(m.BackfillCompletes) > 0 { + for _, e := range m.BackfillCompletes { + l = e.ProtoSize() + n += 1 + l + sovRuntime(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Materialize_Flush_BackfillBegin) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Binding != 0 { + n += 1 + sovRuntime(uint64(m.Binding)) + } + if m.Clock != 0 { + n += 9 + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Materialize_Flush_BackfillComplete) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Binding != 0 { + n += 1 + sovRuntime(uint64(m.Binding)) + } + if m.Clock != 0 { + n += 9 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -16294,6 +16811,96 @@ func (m *Recover) Unmarshal(dAtA []byte) error { m.TriggerParamsJson = []byte{} } iNdEx = postIndex + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ActiveBackfills", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRuntime + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRuntime + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ActiveBackfills == nil { + m.ActiveBackfills = make(map[uint32]uint64) + } + var mapkey uint32 + var mapvalue uint64 + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapkey |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + } else if fieldNum == 2 { + if (iNdEx + 8) > l { + return io.ErrUnexpectedEOF + } + mapvalue = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + iNdEx += 8 + } else { + iNdEx = entryPreIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.ActiveBackfills[mapkey] = mapvalue + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRuntime(dAtA[iNdEx:]) @@ -16956,8 +17563,143 @@ func (m *Persist) Unmarshal(dAtA []byte) error { } } m.Rescan = bool(v != 0) - default: - iNdEx = preIndex + case 18: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Begin", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRuntime + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRuntime + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &ActiveBackfillBegin{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.ActiveBackfillChange = &Persist_Begin{v} + iNdEx = postIndex + case 19: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CompleteBinding", wireType) + } + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ActiveBackfillChange = &Persist_CompleteBinding{v} + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ActiveBackfillBegin) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ActiveBackfillBegin: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ActiveBackfillBegin: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 1 { + return fmt.Errorf("proto: wrong wireType = %d for field TruncatedAt", wireType) + } + m.TruncatedAt = 0 + if (iNdEx + 8) > l { + return io.ErrUnexpectedEOF + } + m.TruncatedAt = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + iNdEx += 8 + default: + iNdEx = preIndex skippy, err := skipRuntime(dAtA[iNdEx:]) if err != nil { return err @@ -20201,6 +20943,234 @@ func (m *Materialize_Flush) Unmarshal(dAtA []byte) error { m.ConnectorPatchesJson = []byte{} } iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BackfillBegins", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRuntime + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRuntime + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BackfillBegins = append(m.BackfillBegins, &Materialize_Flush_BackfillBegin{}) + if err := m.BackfillBegins[len(m.BackfillBegins)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BackfillCompletes", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRuntime + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRuntime + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BackfillCompletes = append(m.BackfillCompletes, &Materialize_Flush_BackfillComplete{}) + if err := m.BackfillCompletes[len(m.BackfillCompletes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Materialize_Flush_BackfillBegin) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BackfillBegin: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BackfillBegin: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 1 { + return fmt.Errorf("proto: wrong wireType = %d for field Clock", wireType) + } + m.Clock = 0 + if (iNdEx + 8) > l { + return io.ErrUnexpectedEOF + } + m.Clock = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + iNdEx += 8 + default: + iNdEx = preIndex + skippy, err := skipRuntime(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRuntime + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Materialize_Flush_BackfillComplete) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BackfillComplete: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BackfillComplete: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuntime + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 1 { + return fmt.Errorf("proto: wrong wireType = %d for field Clock", wireType) + } + m.Clock = 0 + if (iNdEx + 8) > l { + return io.ErrUnexpectedEOF + } + m.Clock = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + iNdEx += 8 default: iNdEx = preIndex skippy, err := skipRuntime(dAtA[iNdEx:]) diff --git a/go/protocols/runtime/runtime.proto b/go/protocols/runtime/runtime.proto index 8ca160f73de..081026371a9 100644 --- a/go/protocols/runtime/runtime.proto +++ b/go/protocols/runtime/runtime.proto @@ -546,6 +546,11 @@ message Recover { map max_keys = 9; // Persisted trigger parameters (materialize only), or empty. bytes trigger_params_json = 10 [json_name = "triggerParams"]; + // Active-backfill begin clocks, keyed by binding index. Restored so the + // capture runtime can re-apply truncated-at journal labels on startup and + // resolve a BackfillComplete's truncated_at. Resolved from "AB:{state_key}" + // keys by the scan. + map active_backfills = 11; } // Persist is sent by the leader to shard zero when state must be durably @@ -616,6 +621,23 @@ message Persist { // stopping once no trigger fires against the freshly-scanned state. // Effect: after the WriteBatch commits, scan and reply `Recover` not `Persisted`. bool rescan = 17; + // The active-backfill change this transaction observed, if any. At most one + // per commit — a backfill control signal stands alone in its transaction. + oneof active_backfill_change { + // BackfillBegin: record the binding's begin clock. + // Effect: Put fixed64-LE under "AB:{state_key}" (state_key resolved by the encoder). + ActiveBackfillBegin begin = 18; + // BackfillComplete: clear the binding's active-backfill entry. + // Effect: Delete "AB:{state_key}" (state_key resolved by the encoder). + uint32 complete_binding = 19; + } +} + +// ActiveBackfillBegin records a binding's backfill begin clock — its +// authoritative truncated_at — staged by a committing Persist. +message ActiveBackfillBegin { + uint32 binding = 1; + fixed64 truncated_at = 2; } // Persisted is sent by shard zero to the leader after the state is durable @@ -872,6 +894,32 @@ message Materialize { // Prior transaction's aggregated C:Acknowledged state patches. // State Update Wire Format. bytes connector_patches_json = 1 [json_name = "connectorPatches"]; + + // A backfill-begin marker: a binding index and the begin clock (the + // truncation boundary). + message BackfillBegin { + // Binding index. + uint32 binding = 1; + // Begin clock: the backfill's truncation boundary. + fixed64 clock = 2; + } + // A backfill-complete marker; same shape as BackfillBegin, where `clock` is + // the completed backfill's begin (truncation) boundary. + message BackfillComplete { + // Binding index. + uint32 binding = 1; + // Begin clock the completed backfill reported (its truncation boundary). + fixed64 clock = 2; + } + // Backfill-begin markers observed during this transaction (the leader's + // per-transaction delta). Each shard forwards them to its connector as a + // C:Flush notification. The shuffle reads fold each marker exactly once per + // committed generation, so the set is already a delta — no leader-side + // deduplication. + repeated BackfillBegin backfill_begins = 2 [json_name = "backfillBegins"]; + // Backfill-complete markers observed during this transaction. Forwarded like + // `backfill_begins`. + repeated BackfillComplete backfill_completes = 3 [json_name = "backfillCompletes"]; } Flush flush = 42; diff --git a/go/protocols/shuffle/shuffle.pb.go b/go/protocols/shuffle/shuffle.pb.go index b0c7cbac6f0..89fc366b4f6 100644 --- a/go/protocols/shuffle/shuffle.pb.go +++ b/go/protocols/shuffle/shuffle.pb.go @@ -305,7 +305,7 @@ type JournalFrontier struct { JournalNameSuffix string `protobuf:"bytes,2,opt,name=journal_name_suffix,json=journalNameSuffix,proto3" json:"journal_name_suffix,omitempty"` // Binding index under which the journal is read. // When persisting across sessions, this should be mapped via the task binding's - // `journal_read_suffix` to ensure stability across task versions. + // `state_key` to ensure stability across task versions. Binding uint32 `protobuf:"varint,3,opt,name=binding,proto3" json:"binding,omitempty"` // Delta of journal bytes read since the last checkpoint. // Summed during reduction. @@ -360,10 +360,18 @@ var xxx_messageInfo_JournalFrontier proto.InternalMessageInfo type Frontier struct { Journals []*JournalFrontier `protobuf:"bytes,1,rep,name=journals,proto3" json:"journals,omitempty"` // Per-shard flushed LSN, indexed by shard_index. - FlushedLsn []uint64 `protobuf:"varint,2,rep,packed,name=flushed_lsn,json=flushedLsn,proto3" json:"flushed_lsn,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + FlushedLsn []uint64 `protobuf:"varint,2,rep,packed,name=flushed_lsn,json=flushedLsn,proto3" json:"flushed_lsn,omitempty"` + // Latest backfill-begin clock for each binding in the checkpoint delta. + // Populated only on a terminal (empty-journals) frontier of a Progressed + // or NextCheckpoint sequence. Empty otherwise. + LatestBackfillBegin []*Frontier_BackfillBegin `protobuf:"bytes,3,rep,name=latest_backfill_begin,json=latestBackfillBegin,proto3" json:"latest_backfill_begin,omitempty"` + // Latest backfill-complete clock for each binding in the checkpoint delta. + // Populated only on a terminal (empty-journals) frontier of a Progressed + // or NextCheckpoint sequence. Empty otherwise. + LatestBackfillComplete []*Frontier_BackfillComplete `protobuf:"bytes,4,rep,name=latest_backfill_complete,json=latestBackfillComplete,proto3" json:"latest_backfill_complete,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Frontier) Reset() { *m = Frontier{} } @@ -399,6 +407,96 @@ func (m *Frontier) XXX_DiscardUnknown() { var xxx_messageInfo_Frontier proto.InternalMessageInfo +// BackfillBegin is a binding's latest backfill-begin clock, keyed by binding +// index. +type Frontier_BackfillBegin struct { + // Binding index. + Binding uint32 `protobuf:"varint,1,opt,name=binding,proto3" json:"binding,omitempty"` + // Clock of the binding's most recent backfill-begin. + Clock uint64 `protobuf:"fixed64,2,opt,name=clock,proto3" json:"clock,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Frontier_BackfillBegin) Reset() { *m = Frontier_BackfillBegin{} } +func (m *Frontier_BackfillBegin) String() string { return proto.CompactTextString(m) } +func (*Frontier_BackfillBegin) ProtoMessage() {} +func (*Frontier_BackfillBegin) Descriptor() ([]byte, []int) { + return fileDescriptor_8851eb1ddb7aa19d, []int{5, 0} +} +func (m *Frontier_BackfillBegin) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Frontier_BackfillBegin) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Frontier_BackfillBegin.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Frontier_BackfillBegin) XXX_Merge(src proto.Message) { + xxx_messageInfo_Frontier_BackfillBegin.Merge(m, src) +} +func (m *Frontier_BackfillBegin) XXX_Size() int { + return m.ProtoSize() +} +func (m *Frontier_BackfillBegin) XXX_DiscardUnknown() { + xxx_messageInfo_Frontier_BackfillBegin.DiscardUnknown(m) +} + +var xxx_messageInfo_Frontier_BackfillBegin proto.InternalMessageInfo + +// BackfillComplete reports a binding's most recent backfill-complete event, +// keyed by binding index. +type Frontier_BackfillComplete struct { + // Binding index. + Binding uint32 `protobuf:"varint,1,opt,name=binding,proto3" json:"binding,omitempty"` + // Truncation boundary of the completed backfill: the backfill's begin clock. + Clock uint64 `protobuf:"fixed64,2,opt,name=clock,proto3" json:"clock,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Frontier_BackfillComplete) Reset() { *m = Frontier_BackfillComplete{} } +func (m *Frontier_BackfillComplete) String() string { return proto.CompactTextString(m) } +func (*Frontier_BackfillComplete) ProtoMessage() {} +func (*Frontier_BackfillComplete) Descriptor() ([]byte, []int) { + return fileDescriptor_8851eb1ddb7aa19d, []int{5, 1} +} +func (m *Frontier_BackfillComplete) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Frontier_BackfillComplete) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Frontier_BackfillComplete.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Frontier_BackfillComplete) XXX_Merge(src proto.Message) { + xxx_messageInfo_Frontier_BackfillComplete.Merge(m, src) +} +func (m *Frontier_BackfillComplete) XXX_Size() int { + return m.ProtoSize() +} +func (m *Frontier_BackfillComplete) XXX_DiscardUnknown() { + xxx_messageInfo_Frontier_BackfillComplete.DiscardUnknown(m) +} + +var xxx_messageInfo_Frontier_BackfillComplete proto.InternalMessageInfo + // SessionRequest is sent by the Coordinator to manage the shuffle session. type SessionRequest struct { Open *SessionRequest_Open `protobuf:"bytes,1,opt,name=open,proto3" json:"open,omitempty"` @@ -1375,6 +1473,8 @@ func init() { proto.RegisterType((*ProducerFrontier)(nil), "shuffle.ProducerFrontier") proto.RegisterType((*JournalFrontier)(nil), "shuffle.JournalFrontier") proto.RegisterType((*Frontier)(nil), "shuffle.Frontier") + proto.RegisterType((*Frontier_BackfillBegin)(nil), "shuffle.Frontier.BackfillBegin") + proto.RegisterType((*Frontier_BackfillComplete)(nil), "shuffle.Frontier.BackfillComplete") proto.RegisterType((*SessionRequest)(nil), "shuffle.SessionRequest") proto.RegisterType((*SessionRequest_Open)(nil), "shuffle.SessionRequest.Open") proto.RegisterType((*SessionRequest_NextCheckpoint)(nil), "shuffle.SessionRequest.NextCheckpoint") @@ -1402,105 +1502,111 @@ func init() { } var fileDescriptor_8851eb1ddb7aa19d = []byte{ - // 1564 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x58, 0x3b, 0x73, 0x1b, 0x47, - 0x12, 0xd6, 0xe2, 0x49, 0x34, 0x48, 0x82, 0x1c, 0x91, 0xd2, 0xde, 0x8a, 0x2f, 0x41, 0x75, 0x3a, - 0xde, 0x95, 0x0a, 0x94, 0x28, 0x9d, 0x54, 0xa7, 0xaa, 0x53, 0x89, 0x94, 0x8e, 0x25, 0xdd, 0xf1, - 0x24, 0xdd, 0x80, 0xd1, 0x25, 0x5b, 0x8b, 0xdd, 0xc1, 0x62, 0x8c, 0xc5, 0x0e, 0xbc, 0x33, 0x90, - 0x09, 0xc7, 0x0e, 0x1c, 0x38, 0xb0, 0x13, 0x3b, 0xb5, 0x63, 0x47, 0xce, 0x9c, 0x38, 0x55, 0x29, - 0xf4, 0x5f, 0xb0, 0x14, 0xf9, 0x4f, 0xb8, 0x5c, 0xf3, 0xd8, 0xc5, 0x83, 0x4b, 0xd9, 0x81, 0x03, - 0x3b, 0x21, 0x67, 0xba, 0xbf, 0x9e, 0xee, 0xe9, 0xd7, 0xf4, 0x02, 0x9a, 0x21, 0xdb, 0x1b, 0x26, - 0x4c, 0x30, 0x9f, 0x45, 0x7c, 0x8f, 0xf7, 0x46, 0xdd, 0x6e, 0x44, 0xd2, 0xff, 0x2d, 0xc5, 0x41, - 0x55, 0xb3, 0x75, 0xb6, 0x3a, 0x09, 0xeb, 0x93, 0x24, 0x13, 0xc8, 0x16, 0x1a, 0xe8, 0x6c, 0xcc, - 0x1c, 0xd6, 0x8d, 0xd8, 0x07, 0xea, 0x8f, 0xe1, 0xae, 0x85, 0x2c, 0x64, 0x6a, 0xb9, 0x27, 0x57, - 0x86, 0xba, 0x1d, 0x32, 0x16, 0x46, 0x44, 0xcb, 0x75, 0x46, 0xdd, 0x3d, 0x41, 0x07, 0x84, 0x0b, - 0x6f, 0x30, 0xd4, 0x80, 0xe6, 0x37, 0x16, 0x94, 0xdb, 0x3d, 0x2f, 0x09, 0xd0, 0x32, 0x14, 0x68, - 0x60, 0x5b, 0x3b, 0xd6, 0x6e, 0x0d, 0x17, 0x68, 0x80, 0xfe, 0x0c, 0xe5, 0xc4, 0x8b, 0x43, 0x62, - 0x17, 0x76, 0xac, 0xdd, 0xfa, 0x7e, 0xa3, 0xa5, 0x94, 0x61, 0x49, 0x6a, 0x0f, 0x89, 0x8f, 0x35, - 0x17, 0x39, 0xb0, 0x40, 0xe2, 0x60, 0xc8, 0x68, 0x2c, 0xec, 0xa2, 0x12, 0xce, 0xf6, 0x68, 0x03, - 0x6a, 0x01, 0x4d, 0x88, 0x2f, 0x58, 0x32, 0xb6, 0x4b, 0x8a, 0x39, 0x21, 0xa0, 0x7b, 0x60, 0x9b, - 0xab, 0xbb, 0x01, 0xe5, 0x7d, 0x37, 0xa2, 0x03, 0x2a, 0xdc, 0xce, 0x58, 0x10, 0x6e, 0x97, 0x77, - 0xac, 0xdd, 0x12, 0x5e, 0x37, 0xfc, 0xc7, 0x94, 0xf7, 0x8f, 0x25, 0xf7, 0x50, 0x32, 0x9b, 0x1f, - 0x17, 0x60, 0xed, 0x11, 0x8b, 0x22, 0xe2, 0x0b, 0xca, 0xe2, 0x17, 0x5e, 0x22, 0xa8, 0x5c, 0x70, - 0x74, 0x07, 0xc0, 0xcf, 0xe8, 0xea, 0x2a, 0xf5, 0xfd, 0x35, 0x6d, 0xf7, 0x04, 0xaf, 0x8c, 0x9f, - 0xc2, 0xa1, 0x23, 0x40, 0xc3, 0xf4, 0x0c, 0x97, 0x93, 0x48, 0x99, 0x67, 0x6e, 0x7d, 0xb9, 0x95, - 0x05, 0xe1, 0xd8, 0xeb, 0x90, 0xa8, 0x6d, 0xd8, 0x78, 0x35, 0x13, 0x49, 0x49, 0xe8, 0x1f, 0x00, - 0x31, 0x13, 0x6e, 0x87, 0x74, 0x59, 0x42, 0x94, 0x2f, 0xea, 0xfb, 0x4e, 0x4b, 0x07, 0xa0, 0x95, - 0x06, 0xa0, 0x75, 0x92, 0x06, 0x00, 0xd7, 0x62, 0x26, 0x0e, 0x15, 0x18, 0xdd, 0x03, 0xb9, 0x71, - 0xbd, 0xae, 0x20, 0x89, 0x72, 0xd4, 0xbb, 0x25, 0x17, 0x62, 0x26, 0x0e, 0x24, 0xb6, 0xf9, 0xd6, - 0x82, 0xd2, 0x89, 0xc7, 0xfb, 0xe8, 0x04, 0xd6, 0x27, 0x57, 0x72, 0x33, 0xe3, 0xb8, 0xf1, 0xc2, - 0x66, 0x2b, 0x4d, 0xba, 0x3c, 0xc7, 0x3d, 0xb9, 0x80, 0xd7, 0xfc, 0x3c, 0x87, 0xde, 0x05, 0x08, - 0x48, 0x42, 0x5f, 0x7a, 0xca, 0xa1, 0x85, 0xf3, 0x1d, 0xfa, 0xe4, 0x02, 0x9e, 0x42, 0xa2, 0x7f, - 0x41, 0x63, 0xe0, 0x09, 0x92, 0x50, 0x2f, 0xa2, 0x1f, 0x6a, 0x61, 0xed, 0x8f, 0x3f, 0x69, 0xe1, - 0xff, 0xce, 0x32, 0xcd, 0x09, 0xf3, 0x32, 0x87, 0x15, 0x28, 0x09, 0x8f, 0xf7, 0x9b, 0x9f, 0x58, - 0xb0, 0xf2, 0x22, 0x61, 0xc1, 0xc8, 0x27, 0xc9, 0x51, 0xc2, 0x62, 0x41, 0x49, 0x22, 0x13, 0x6f, - 0x68, 0x68, 0xea, 0x92, 0x45, 0x9c, 0xed, 0xd1, 0x36, 0xd4, 0x23, 0x8f, 0x0b, 0xd7, 0x67, 0x83, - 0x01, 0x15, 0xca, 0xf0, 0x0a, 0x06, 0x49, 0x7a, 0xa4, 0x28, 0xe8, 0x1a, 0x2c, 0xf5, 0x68, 0x2c, - 0x48, 0x90, 0x42, 0x8a, 0x0a, 0xb2, 0xa8, 0x89, 0x06, 0x74, 0x09, 0x2a, 0xac, 0xdb, 0xe5, 0x44, - 0xa8, 0x90, 0x14, 0xb1, 0xd9, 0x35, 0xbf, 0x2e, 0x40, 0xe3, 0xdf, 0x6c, 0x94, 0xc4, 0x5e, 0x94, - 0x59, 0xf3, 0x4f, 0xb8, 0xf2, 0x9e, 0x26, 0xb9, 0xb1, 0x37, 0x20, 0xae, 0x48, 0x46, 0xb1, 0xef, - 0x09, 0xe2, 0x06, 0x24, 0x12, 0x9e, 0x32, 0xb0, 0x8c, 0x6d, 0x03, 0x79, 0xe6, 0x0d, 0xc8, 0x89, - 0x01, 0x3c, 0x96, 0x7c, 0xd4, 0x82, 0x8b, 0x33, 0xe2, 0x7c, 0xd4, 0xed, 0xd2, 0x53, 0x65, 0x78, - 0x0d, 0xaf, 0x4e, 0x89, 0xb5, 0x15, 0x03, 0xd9, 0x50, 0xed, 0xd0, 0x38, 0xa0, 0x71, 0xa8, 0x2c, - 0x5f, 0xc2, 0xe9, 0x16, 0xed, 0xc2, 0x8a, 0x2a, 0x21, 0x37, 0x21, 0x5e, 0x60, 0xb4, 0x6b, 0xf3, - 0x97, 0x15, 0x1d, 0x13, 0x2f, 0xd0, 0x3a, 0x6f, 0x00, 0xd2, 0xc8, 0x0e, 0xe9, 0xd1, 0x38, 0xc5, - 0x96, 0x15, 0x56, 0x9f, 0x71, 0xa8, 0x18, 0x1a, 0x7d, 0x0f, 0x6a, 0xa9, 0x7b, 0xb9, 0x5d, 0xd9, - 0x29, 0xaa, 0x60, 0xa6, 0x49, 0x35, 0x1f, 0x1c, 0x3c, 0xc1, 0x36, 0x3d, 0x58, 0xc8, 0xbc, 0x74, - 0x07, 0x16, 0xcc, 0x5d, 0x64, 0x62, 0xca, 0x33, 0xec, 0xec, 0x8c, 0x39, 0x8f, 0xe2, 0x0c, 0x29, - 0xa3, 0xd9, 0x8d, 0x46, 0xbc, 0x47, 0x02, 0x37, 0xe2, 0x32, 0x0d, 0x8b, 0xbb, 0x25, 0x0c, 0x86, - 0x74, 0xcc, 0xe3, 0xe6, 0xb7, 0x05, 0x58, 0x6e, 0x13, 0xce, 0x29, 0x8b, 0x31, 0x79, 0x7f, 0x44, - 0xb8, 0x40, 0x37, 0xa1, 0xc4, 0x86, 0x24, 0x6d, 0x02, 0x1b, 0x99, 0x96, 0x59, 0x58, 0xeb, 0xf9, - 0x90, 0xc4, 0x58, 0x21, 0xd1, 0x03, 0x58, 0x4d, 0x08, 0x1f, 0x0d, 0x88, 0xeb, 0xf7, 0x88, 0xdf, - 0xd7, 0x1d, 0x4d, 0xa7, 0xfc, 0x6a, 0x26, 0x9e, 0x59, 0xb7, 0xa2, 0xb1, 0x8f, 0x32, 0x28, 0x7a, - 0x0e, 0x8d, 0x98, 0x9c, 0x8a, 0x69, 0x69, 0x9d, 0xf3, 0xd7, 0xcf, 0x53, 0xfe, 0x8c, 0x9c, 0x8a, - 0xc9, 0x01, 0x78, 0x39, 0x9e, 0xd9, 0x3b, 0xff, 0x83, 0x92, 0x34, 0x0f, 0x5d, 0xd5, 0x55, 0x60, - 0x6c, 0x59, 0xca, 0x4e, 0x93, 0x75, 0x8f, 0x15, 0x0b, 0x5d, 0x87, 0x0a, 0x97, 0x4d, 0x9c, 0xdb, - 0x45, 0xe5, 0xd5, 0xe5, 0x89, 0x4a, 0x49, 0xc6, 0x86, 0xeb, 0xac, 0xc0, 0xf2, 0xac, 0xd2, 0xe6, - 0xa7, 0x16, 0x34, 0x32, 0xb3, 0xf8, 0x90, 0xc5, 0x5c, 0x76, 0xa3, 0x8a, 0xf4, 0x08, 0x09, 0x8c, - 0xf7, 0xb6, 0xcf, 0x5e, 0x40, 0x23, 0x95, 0xfb, 0x48, 0x80, 0x0d, 0x1c, 0xdd, 0x3f, 0xeb, 0x82, - 0x73, 0x1d, 0x38, 0x7f, 0xdb, 0x05, 0xa8, 0xe8, 0xd3, 0x9a, 0x5f, 0x94, 0x61, 0xb1, 0x1d, 0x51, - 0x9f, 0xa4, 0xb1, 0x6c, 0xcd, 0xc4, 0xd2, 0x99, 0x58, 0x33, 0x05, 0x9a, 0x8e, 0xe4, 0x2d, 0x28, - 0x73, 0xe1, 0x25, 0xa9, 0xf2, 0x2b, 0xf9, 0x02, 0x6d, 0x09, 0xc1, 0x1a, 0x89, 0x1e, 0x00, 0xa8, - 0x85, 0xaa, 0x1a, 0x13, 0xb7, 0xed, 0x77, 0xc9, 0x11, 0x2f, 0xc0, 0x35, 0x9e, 0x2e, 0xd1, 0x7d, - 0xd5, 0x8c, 0xc2, 0x84, 0x70, 0x6e, 0xfa, 0xf7, 0x56, 0xbe, 0xf4, 0x0b, 0x83, 0xc2, 0x19, 0xde, - 0xf9, 0xcc, 0x32, 0x81, 0xde, 0x04, 0xe0, 0xda, 0xc1, 0xae, 0x79, 0x89, 0xab, 0xb8, 0x66, 0x28, - 0x4f, 0x83, 0xdf, 0x30, 0x0f, 0x64, 0x45, 0xa9, 0x95, 0x4b, 0xe3, 0x80, 0x9c, 0x2a, 0x8b, 0x97, - 0x30, 0x28, 0xd2, 0x53, 0x49, 0x71, 0xaa, 0x50, 0x56, 0xf7, 0x74, 0x7e, 0xb2, 0xa0, 0x96, 0xdd, - 0x78, 0xba, 0xed, 0x58, 0xb3, 0x6d, 0xe7, 0xaf, 0x50, 0xe2, 0x43, 0xe2, 0x1b, 0xe3, 0xd6, 0x27, - 0xcf, 0xa6, 0x29, 0x6b, 0xf5, 0xea, 0x2a, 0x08, 0xfa, 0x0b, 0x34, 0xfc, 0x84, 0xc8, 0xde, 0x98, - 0x90, 0x97, 0x94, 0xa7, 0x8f, 0x43, 0x11, 0x2f, 0x6b, 0x32, 0x36, 0x54, 0x74, 0x15, 0x16, 0x07, - 0x2c, 0x98, 0xa0, 0x74, 0x1b, 0xab, 0x0f, 0x58, 0x90, 0x41, 0xe4, 0x90, 0xc2, 0x46, 0x82, 0xa8, - 0xb6, 0x25, 0x87, 0x94, 0x4c, 0x2f, 0x96, 0x64, 0xac, 0xb9, 0xf2, 0x69, 0x9e, 0xca, 0xc9, 0x5f, - 0xec, 0x5e, 0x53, 0x60, 0x07, 0x60, 0x21, 0x8d, 0x59, 0xf3, 0xf3, 0x22, 0x2c, 0x99, 0x68, 0x9a, - 0x52, 0xf9, 0xfb, 0x5c, 0xa9, 0x6c, 0xce, 0x47, 0x3d, 0xbf, 0x50, 0x9e, 0xc0, 0x52, 0x44, 0xb9, - 0xa0, 0x71, 0xe8, 0x7a, 0x41, 0x40, 0x02, 0xe3, 0xb6, 0x6b, 0xe7, 0x48, 0x1f, 0x6b, 0xec, 0x81, - 0x84, 0xe2, 0xc5, 0x68, 0x6a, 0x87, 0x6e, 0x01, 0xa4, 0x89, 0x44, 0xd2, 0xc4, 0xcd, 0xa9, 0xb6, - 0x29, 0xd0, 0xa4, 0xd2, 0x9c, 0x57, 0x16, 0x2c, 0x4e, 0x9f, 0xfd, 0x47, 0x8d, 0x6f, 0xf3, 0xc7, - 0x32, 0xc0, 0x31, 0x0b, 0xd3, 0x86, 0x71, 0x63, 0xa6, 0x61, 0x4c, 0x9e, 0x98, 0x09, 0x64, 0xba, - 0x5d, 0xec, 0x43, 0xc5, 0x1b, 0x0e, 0x49, 0x9c, 0x46, 0xc1, 0xc9, 0xc3, 0x1f, 0x28, 0x04, 0x36, - 0x48, 0xb4, 0x07, 0x65, 0xf5, 0xfe, 0x64, 0x63, 0x4d, 0x8e, 0xc8, 0x91, 0x04, 0x60, 0x8d, 0x73, - 0xbe, 0xfa, 0x95, 0x45, 0x3e, 0xa9, 0xe0, 0xc2, 0x3b, 0x2b, 0xf8, 0x6f, 0xb0, 0xca, 0x65, 0x8e, - 0xb8, 0xd3, 0x75, 0xac, 0x47, 0x81, 0x86, 0x62, 0xb4, 0xb3, 0x62, 0x46, 0xd7, 0xa1, 0x11, 0xb1, - 0xd0, 0x3d, 0x5b, 0xf1, 0x4b, 0x11, 0x0b, 0x27, 0x38, 0xe7, 0xa3, 0x22, 0x54, 0xf4, 0x3d, 0x7f, - 0x3f, 0xe3, 0x8c, 0x9a, 0xf2, 0x28, 0x4b, 0xa8, 0x18, 0x1b, 0xa3, 0xb3, 0xbd, 0x74, 0x65, 0x3a, - 0xe4, 0x78, 0x63, 0xf3, 0xc9, 0x50, 0x4b, 0xf4, 0x7c, 0xe3, 0x8d, 0x67, 0x06, 0xc4, 0xca, 0xdc, - 0x80, 0xb8, 0x06, 0x65, 0x3f, 0x62, 0x7e, 0xdf, 0xae, 0xaa, 0xb9, 0x4f, 0x6f, 0x24, 0xb5, 0x1b, - 0x79, 0x21, 0xb7, 0x17, 0x94, 0x26, 0xbd, 0x91, 0x6a, 0x86, 0x9e, 0xdf, 0x27, 0x81, 0xdb, 0x27, - 0x63, 0xbb, 0xb6, 0x63, 0xed, 0x2e, 0xe2, 0x9a, 0xa6, 0xfc, 0x87, 0x8c, 0x65, 0x16, 0x07, 0xcc, - 0x77, 0xbd, 0xc4, 0xef, 0xd1, 0x97, 0x24, 0xb0, 0x41, 0x01, 0xea, 0x01, 0xf3, 0x0f, 0x0c, 0x49, - 0x4e, 0x5a, 0x9c, 0x8d, 0x12, 0x9f, 0xa8, 0xaf, 0x1b, 0x37, 0x22, 0x71, 0x28, 0x7a, 0x76, 0x5d, - 0x29, 0x59, 0xd1, 0x1c, 0xf9, 0x65, 0x73, 0xac, 0xe8, 0xce, 0x26, 0x94, 0x55, 0xea, 0x28, 0x23, - 0xc7, 0x7e, 0x44, 0x94, 0xbb, 0x4b, 0x58, 0x6f, 0x9a, 0xaf, 0x2c, 0xa8, 0xab, 0x2c, 0x33, 0x2d, - 0xe8, 0xf6, 0x5c, 0x0b, 0xba, 0x32, 0x9b, 0x8b, 0xf9, 0x0d, 0xe8, 0x2e, 0x54, 0xcd, 0xfc, 0x64, - 0x92, 0x7e, 0x23, 0x57, 0xea, 0x48, 0x63, 0x70, 0x0a, 0x9e, 0xea, 0x1d, 0x0f, 0xa1, 0x6a, 0xb8, - 0xf9, 0x76, 0x9e, 0x9d, 0xda, 0xac, 0xd9, 0xa9, 0x6d, 0xff, 0x3b, 0x0b, 0xaa, 0x6d, 0xad, 0x14, - 0x3d, 0x84, 0xaa, 0x99, 0x2d, 0xd0, 0xe5, 0x73, 0xc6, 0x25, 0xc7, 0x3e, 0x6f, 0x0c, 0xd9, 0xb5, - 0x6e, 0x5a, 0xe8, 0x3e, 0x94, 0x55, 0xd3, 0x44, 0xeb, 0xb9, 0x0f, 0xaf, 0x73, 0x29, 0xbf, 0xb7, - 0x2a, 0xd9, 0x3b, 0x50, 0x3c, 0x66, 0x21, 0xba, 0x98, 0x53, 0xc5, 0xce, 0x5a, 0x9e, 0x63, 0xa4, - 0xd4, 0xe1, 0x83, 0xd7, 0x3f, 0x6c, 0x5d, 0x78, 0xfd, 0x66, 0xcb, 0xfa, 0xfe, 0xcd, 0x96, 0xf5, - 0xe5, 0xdb, 0x2d, 0xeb, 0xff, 0x37, 0x42, 0x2a, 0x7a, 0xa3, 0x4e, 0xcb, 0x67, 0x83, 0x3d, 0xc2, - 0xc5, 0xc8, 0x4b, 0xc6, 0xfa, 0x3b, 0x3d, 0xef, 0x67, 0x80, 0x4e, 0x45, 0x91, 0x6e, 0xff, 0x1c, - 0x00, 0x00, 0xff, 0xff, 0x1e, 0x1f, 0x34, 0x69, 0x25, 0x10, 0x00, 0x00, + // 1652 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x58, 0xcd, 0x73, 0x1b, 0x49, + 0x15, 0xcf, 0x58, 0x5f, 0xd6, 0xf3, 0x87, 0xec, 0x8e, 0x9d, 0x1d, 0x26, 0x89, 0xed, 0xd5, 0x16, + 0xc1, 0x50, 0x29, 0x79, 0xd7, 0x1b, 0x36, 0x45, 0xaa, 0x08, 0x1b, 0x65, 0x49, 0x65, 0xc1, 0xec, + 0x86, 0x56, 0x4e, 0x14, 0x55, 0x53, 0xa3, 0x99, 0xa7, 0x51, 0xa3, 0xd1, 0xb4, 0x98, 0x6e, 0x85, + 0x88, 0x33, 0x07, 0x0e, 0x1c, 0xe0, 0x02, 0xc5, 0x0d, 0xce, 0x9c, 0xb8, 0x71, 0xe1, 0xba, 0xb5, + 0x47, 0xfe, 0x05, 0x36, 0x27, 0xfe, 0x09, 0x8a, 0xea, 0x8f, 0x19, 0x8d, 0xe4, 0xb1, 0x81, 0x2a, + 0x0e, 0xec, 0x25, 0x99, 0x7e, 0xef, 0xf7, 0x3e, 0xfa, 0x7d, 0xf5, 0x93, 0xa1, 0x1b, 0xf3, 0xb3, + 0x59, 0xc6, 0x25, 0x0f, 0x79, 0x22, 0xce, 0xc4, 0x78, 0x3e, 0x1a, 0x25, 0x98, 0xff, 0xdf, 0xd3, + 0x1c, 0xd2, 0xb2, 0x47, 0xef, 0x68, 0x98, 0xf1, 0x09, 0x66, 0x85, 0x40, 0xf1, 0x61, 0x80, 0xde, + 0x9d, 0x15, 0x65, 0xa3, 0x84, 0xff, 0x4c, 0xff, 0x63, 0xb9, 0x07, 0x31, 0x8f, 0xb9, 0xfe, 0x3c, + 0x53, 0x5f, 0x96, 0x7a, 0x1c, 0x73, 0x1e, 0x27, 0x68, 0xe4, 0x86, 0xf3, 0xd1, 0x99, 0x64, 0x53, + 0x14, 0x32, 0x98, 0xce, 0x0c, 0xa0, 0xfb, 0x67, 0x07, 0x1a, 0x83, 0x71, 0x90, 0x45, 0x64, 0x17, + 0x36, 0x58, 0xe4, 0x3a, 0x27, 0xce, 0x69, 0x9b, 0x6e, 0xb0, 0x88, 0x7c, 0x15, 0x1a, 0x59, 0x90, + 0xc6, 0xe8, 0x6e, 0x9c, 0x38, 0xa7, 0x5b, 0xe7, 0x9d, 0x9e, 0x36, 0x46, 0x15, 0x69, 0x30, 0xc3, + 0x90, 0x1a, 0x2e, 0xf1, 0x60, 0x13, 0xd3, 0x68, 0xc6, 0x59, 0x2a, 0xdd, 0x9a, 0x16, 0x2e, 0xce, + 0xe4, 0x0e, 0xb4, 0x23, 0x96, 0x61, 0x28, 0x79, 0xb6, 0x70, 0xeb, 0x9a, 0xb9, 0x24, 0x90, 0x87, + 0xe0, 0xda, 0xab, 0xfb, 0x11, 0x13, 0x13, 0x3f, 0x61, 0x53, 0x26, 0xfd, 0xe1, 0x42, 0xa2, 0x70, + 0x1b, 0x27, 0xce, 0x69, 0x9d, 0x1e, 0x5a, 0xfe, 0x47, 0x4c, 0x4c, 0x2e, 0x14, 0xb7, 0xaf, 0x98, + 0xdd, 0x5f, 0x6e, 0xc0, 0xc1, 0x53, 0x9e, 0x24, 0x18, 0x4a, 0xc6, 0xd3, 0x17, 0x41, 0x26, 0x99, + 0xfa, 0x10, 0xe4, 0x01, 0x40, 0x58, 0xd0, 0xf5, 0x55, 0xb6, 0xce, 0x0f, 0x8c, 0xdf, 0x4b, 0xbc, + 0x76, 0xbe, 0x84, 0x23, 0xcf, 0x80, 0xcc, 0x72, 0x1d, 0xbe, 0xc0, 0x44, 0xbb, 0x67, 0x6f, 0xfd, + 0x56, 0xaf, 0x48, 0xc2, 0x45, 0x30, 0xc4, 0x64, 0x60, 0xd9, 0x74, 0xbf, 0x10, 0xc9, 0x49, 0xe4, + 0x5b, 0x00, 0x29, 0x97, 0xfe, 0x10, 0x47, 0x3c, 0x43, 0x1d, 0x8b, 0xad, 0x73, 0xaf, 0x67, 0x12, + 0xd0, 0xcb, 0x13, 0xd0, 0x7b, 0x99, 0x27, 0x80, 0xb6, 0x53, 0x2e, 0xfb, 0x1a, 0x4c, 0x1e, 0x82, + 0x3a, 0xf8, 0xc1, 0x48, 0x62, 0xa6, 0x03, 0x75, 0xbd, 0xe4, 0x66, 0xca, 0xe5, 0x13, 0x85, 0xed, + 0xbe, 0x71, 0xa0, 0xfe, 0x32, 0x10, 0x13, 0xf2, 0x12, 0x0e, 0x97, 0x57, 0xf2, 0x0b, 0xe7, 0x84, + 0x8d, 0xc2, 0xdd, 0x5e, 0x5e, 0x74, 0x55, 0x81, 0x7b, 0x7e, 0x83, 0x1e, 0x84, 0x55, 0x01, 0xfd, + 0x00, 0x20, 0xc2, 0x8c, 0xbd, 0x0a, 0x74, 0x40, 0x37, 0xae, 0x0e, 0xe8, 0xf3, 0x1b, 0xb4, 0x84, + 0x24, 0xdf, 0x85, 0xce, 0x34, 0x90, 0x98, 0xb1, 0x20, 0x61, 0x3f, 0x37, 0xc2, 0x26, 0x1e, 0x5f, + 0x31, 0xc2, 0x3f, 0x58, 0x65, 0x5a, 0x0d, 0xeb, 0x32, 0xfd, 0x26, 0xd4, 0x65, 0x20, 0x26, 0xdd, + 0x5f, 0x39, 0xb0, 0xf7, 0x22, 0xe3, 0xd1, 0x3c, 0xc4, 0xec, 0x59, 0xc6, 0x53, 0xc9, 0x30, 0x53, + 0x85, 0x37, 0xb3, 0x34, 0x7d, 0xc9, 0x1a, 0x2d, 0xce, 0xe4, 0x18, 0xb6, 0x92, 0x40, 0x48, 0x3f, + 0xe4, 0xd3, 0x29, 0x93, 0xda, 0xf1, 0x26, 0x05, 0x45, 0x7a, 0xaa, 0x29, 0xe4, 0x1d, 0xd8, 0x19, + 0xb3, 0x54, 0x62, 0x94, 0x43, 0x6a, 0x1a, 0xb2, 0x6d, 0x88, 0x16, 0x74, 0x0b, 0x9a, 0x7c, 0x34, + 0x12, 0x28, 0x75, 0x4a, 0x6a, 0xd4, 0x9e, 0xba, 0x7f, 0xda, 0x80, 0xce, 0xf7, 0xf8, 0x3c, 0x4b, + 0x83, 0xa4, 0xf0, 0xe6, 0xdb, 0x70, 0xfb, 0x27, 0x86, 0xe4, 0xa7, 0xc1, 0x14, 0x7d, 0x99, 0xcd, + 0xd3, 0x30, 0x90, 0xe8, 0x47, 0x98, 0xc8, 0x40, 0x3b, 0xd8, 0xa0, 0xae, 0x85, 0x7c, 0x12, 0x4c, + 0xf1, 0xa5, 0x05, 0x7c, 0xa4, 0xf8, 0xa4, 0x07, 0x37, 0x57, 0xc4, 0xc5, 0x7c, 0x34, 0x62, 0xaf, + 0xb5, 0xe3, 0x6d, 0xba, 0x5f, 0x12, 0x1b, 0x68, 0x06, 0x71, 0xa1, 0x35, 0x64, 0x69, 0xc4, 0xd2, + 0x58, 0x7b, 0xbe, 0x43, 0xf3, 0x23, 0x39, 0x85, 0x3d, 0xdd, 0x42, 0x7e, 0x86, 0x41, 0x64, 0xad, + 0x1b, 0xf7, 0x77, 0x35, 0x9d, 0x62, 0x10, 0x19, 0x9b, 0xf7, 0x81, 0x18, 0xe4, 0x10, 0xc7, 0x2c, + 0xcd, 0xb1, 0x0d, 0x8d, 0x35, 0x3a, 0xfa, 0x9a, 0x61, 0xd0, 0x0f, 0xa1, 0x9d, 0x87, 0x57, 0xb8, + 0xcd, 0x93, 0x9a, 0x4e, 0x66, 0x5e, 0x54, 0xeb, 0xc9, 0xa1, 0x4b, 0x6c, 0xf7, 0xf7, 0x35, 0xd8, + 0x2c, 0xc2, 0xf4, 0x00, 0x36, 0xed, 0x65, 0x54, 0x65, 0x2a, 0x25, 0x6e, 0xa1, 0x64, 0x2d, 0xa4, + 0xb4, 0x40, 0xaa, 0x74, 0x8e, 0x92, 0xb9, 0x18, 0x63, 0xe4, 0x27, 0x42, 0xd5, 0x61, 0xed, 0xb4, + 0x4e, 0xc1, 0x92, 0x2e, 0x44, 0x4a, 0x06, 0x70, 0x98, 0x04, 0x12, 0x85, 0xf4, 0x87, 0x41, 0x38, + 0x19, 0xb1, 0x24, 0xf1, 0x87, 0x18, 0x33, 0x55, 0x75, 0xca, 0xc6, 0x71, 0x61, 0x23, 0x57, 0xde, + 0xeb, 0x5b, 0x5c, 0x5f, 0xc1, 0xe8, 0x4d, 0x23, 0xbd, 0x42, 0x24, 0x3f, 0x06, 0x77, 0x5d, 0x69, + 0xc8, 0xa7, 0xb3, 0x04, 0x25, 0xba, 0x75, 0xad, 0xb7, 0x7b, 0xb5, 0xde, 0xa7, 0x16, 0x49, 0x6f, + 0xad, 0xaa, 0xce, 0xe9, 0xde, 0x77, 0x60, 0x67, 0xd5, 0x5c, 0x29, 0xa5, 0xce, 0x6a, 0x4a, 0x0f, + 0xa0, 0x11, 0x26, 0x3c, 0x9c, 0xd8, 0x3a, 0x36, 0x07, 0xaf, 0x0f, 0x7b, 0xeb, 0x4a, 0xff, 0x5b, + 0x1d, 0xdd, 0xbf, 0x6c, 0xc0, 0xee, 0x00, 0x85, 0x60, 0x3c, 0xa5, 0xf8, 0xd3, 0x39, 0x0a, 0x49, + 0xde, 0x85, 0x3a, 0x9f, 0x61, 0x3e, 0x3d, 0xef, 0x14, 0x37, 0x5c, 0x85, 0xf5, 0x3e, 0x9d, 0x61, + 0x4a, 0x35, 0x92, 0x3c, 0x86, 0xfd, 0x0c, 0xc5, 0x7c, 0x8a, 0x7e, 0x38, 0xc6, 0x70, 0x62, 0x9e, + 0x02, 0x33, 0x2b, 0xf6, 0x2f, 0x05, 0x88, 0xee, 0x19, 0xec, 0xd3, 0x02, 0x4a, 0x3e, 0x85, 0x4e, + 0x8a, 0xaf, 0x65, 0x59, 0xda, 0x0c, 0x8b, 0x7b, 0x57, 0x19, 0xff, 0x04, 0x5f, 0xcb, 0xa5, 0x02, + 0xba, 0x9b, 0xae, 0x9c, 0xbd, 0x1f, 0x42, 0x5d, 0xb9, 0x47, 0xde, 0x36, 0xe3, 0xc3, 0xfa, 0xb2, + 0x53, 0x68, 0x53, 0x03, 0x93, 0x6a, 0x16, 0xb9, 0x07, 0x4d, 0xa1, 0x5e, 0x3f, 0x61, 0x2b, 0x65, + 0x77, 0x69, 0x52, 0x91, 0xa9, 0xe5, 0x7a, 0x7b, 0xb0, 0xbb, 0x6a, 0xb4, 0xfb, 0x6b, 0x07, 0x3a, + 0x85, 0x5b, 0x62, 0xc6, 0x53, 0xa1, 0xc6, 0x78, 0x53, 0x45, 0x04, 0x23, 0x1b, 0xbd, 0xe3, 0xcb, + 0x17, 0x30, 0x48, 0x1d, 0x3e, 0x8c, 0xa8, 0x85, 0x93, 0x47, 0x97, 0x43, 0x70, 0x65, 0x00, 0xd7, + 0x6f, 0xbb, 0x09, 0x4d, 0xa3, 0xad, 0xfb, 0xbb, 0x06, 0x6c, 0x0f, 0x12, 0x16, 0x62, 0x9e, 0xcb, + 0xde, 0x4a, 0x2e, 0xbd, 0xa5, 0x37, 0x25, 0x50, 0x39, 0x93, 0xef, 0x41, 0x43, 0xc8, 0x20, 0xcb, + 0x8d, 0xdf, 0xae, 0x16, 0x18, 0x28, 0x08, 0x35, 0x48, 0xf2, 0x18, 0x40, 0x7f, 0xe8, 0x71, 0x63, + 0xf3, 0x76, 0x7c, 0x9d, 0x1c, 0x06, 0x11, 0x6d, 0x8b, 0xfc, 0x93, 0x3c, 0xd2, 0x53, 0x3c, 0xce, + 0x50, 0x08, 0xfb, 0xf0, 0x1d, 0x55, 0x4b, 0xbf, 0xb0, 0x28, 0x5a, 0xe0, 0xbd, 0xdf, 0x38, 0x36, + 0xd1, 0x77, 0x01, 0x84, 0x09, 0xb0, 0x6f, 0x57, 0x98, 0x16, 0x6d, 0x5b, 0xca, 0xc7, 0xd1, 0xff, + 0xb0, 0x0e, 0xd4, 0x24, 0xd2, 0x5f, 0x3e, 0x4b, 0x23, 0x7c, 0xad, 0x3d, 0xde, 0xa1, 0xa0, 0x49, + 0x1f, 0x2b, 0x8a, 0xd7, 0x82, 0x86, 0xbe, 0xa7, 0xf7, 0x4f, 0x07, 0xda, 0xc5, 0x8d, 0xaf, 0x69, + 0xcc, 0xaf, 0x43, 0x5d, 0xcc, 0x30, 0xb4, 0xce, 0x1d, 0x2e, 0xf7, 0x0d, 0x3b, 0x0e, 0xf5, 0xba, + 0xa2, 0x21, 0xe4, 0x6b, 0xd0, 0x09, 0x33, 0x54, 0x8f, 0x4a, 0x86, 0xaf, 0x98, 0xc8, 0x5f, 0xd5, + 0x1a, 0xdd, 0x35, 0x64, 0x6a, 0xa9, 0xe4, 0x6d, 0xd8, 0x9e, 0xf2, 0x68, 0x89, 0x32, 0xf3, 0x7f, + 0x6b, 0xca, 0xa3, 0x02, 0xa2, 0xb6, 0x3b, 0x3e, 0x97, 0xa8, 0xe7, 0xbd, 0xda, 0xee, 0x0a, 0xbb, + 0x54, 0x91, 0xa9, 0xe1, 0xaa, 0x9d, 0xa6, 0x54, 0x93, 0xff, 0x76, 0xec, 0x97, 0xc0, 0x1e, 0xc0, + 0x66, 0x9e, 0xb3, 0xee, 0x6f, 0x6b, 0xb0, 0x63, 0xb3, 0x69, 0x5b, 0xe5, 0x9b, 0x6b, 0xad, 0x72, + 0x77, 0x3d, 0xeb, 0xd5, 0x8d, 0xf2, 0x1c, 0x76, 0x12, 0x26, 0x24, 0x4b, 0x63, 0x3f, 0x88, 0x22, + 0x8c, 0x6c, 0xd8, 0xde, 0xb9, 0x42, 0xfa, 0xc2, 0x60, 0x9f, 0x28, 0x28, 0xdd, 0x4e, 0x4a, 0x27, + 0xf2, 0x1e, 0x40, 0x5e, 0x48, 0x98, 0x17, 0x6e, 0x45, 0xb7, 0x95, 0x40, 0xcb, 0x4e, 0xf3, 0x3e, + 0x73, 0x60, 0xbb, 0xac, 0xfb, 0xcb, 0x9a, 0xdf, 0xee, 0x3f, 0x1a, 0x00, 0x17, 0x3c, 0xce, 0x07, + 0xc6, 0xfd, 0x95, 0x81, 0xb1, 0x7c, 0x9a, 0x97, 0x90, 0xf2, 0xb8, 0x38, 0x87, 0x66, 0x30, 0x9b, + 0x61, 0x9a, 0x67, 0xc1, 0xab, 0xc2, 0x3f, 0xd1, 0x08, 0x6a, 0x91, 0xe4, 0x0c, 0x1a, 0xfa, 0xdd, + 0x2e, 0xf6, 0xc1, 0x0a, 0x91, 0x67, 0x0a, 0x40, 0x0d, 0xce, 0xfb, 0xe3, 0x7f, 0xd8, 0xe4, 0xcb, + 0x0e, 0xde, 0xb8, 0xb6, 0x83, 0xbf, 0x01, 0xfb, 0x42, 0xd5, 0x88, 0x5f, 0xee, 0x63, 0xb3, 0x43, + 0x75, 0x34, 0x63, 0x50, 0x34, 0x33, 0xb9, 0x07, 0x9d, 0x84, 0xc7, 0xfe, 0xe5, 0x8e, 0xdf, 0x49, + 0x78, 0xbc, 0xc4, 0x79, 0xbf, 0xa8, 0x41, 0xd3, 0xdc, 0xf3, 0xff, 0x67, 0x0f, 0xd4, 0xeb, 0x31, + 0xe3, 0x19, 0x93, 0x0b, 0xeb, 0x74, 0x71, 0x56, 0xa1, 0xcc, 0xb7, 0xc3, 0x60, 0x61, 0x7f, 0x6b, + 0xb5, 0x33, 0xb3, 0x18, 0x06, 0x8b, 0x95, 0xcd, 0xba, 0xb9, 0xb6, 0x59, 0x17, 0x7b, 0x44, 0xab, + 0xb4, 0x47, 0x28, 0xea, 0x28, 0x09, 0x62, 0xe1, 0x6e, 0x6a, 0x4b, 0xe6, 0xa0, 0xcc, 0xcc, 0x82, + 0x70, 0x82, 0x91, 0x3f, 0xc1, 0x85, 0xdb, 0x3e, 0x71, 0x4e, 0xb7, 0x69, 0xdb, 0x50, 0xbe, 0x8f, + 0x0b, 0x55, 0xc5, 0x11, 0x0f, 0xfd, 0x20, 0x0b, 0xc7, 0xec, 0x15, 0x46, 0x2e, 0x68, 0xc0, 0x56, + 0xc4, 0xc3, 0x27, 0x96, 0xa4, 0x56, 0x54, 0xc1, 0xe7, 0x59, 0x88, 0xfa, 0x67, 0xa1, 0x9f, 0x60, + 0x1a, 0xcb, 0xb1, 0xbb, 0xa5, 0x8d, 0xec, 0x19, 0x8e, 0xfa, 0x49, 0x78, 0xa1, 0xe9, 0xde, 0x5d, + 0x68, 0xe8, 0xd2, 0xd1, 0x4e, 0x2e, 0xc2, 0x04, 0x75, 0xb8, 0xeb, 0xd4, 0x1c, 0xba, 0x9f, 0x39, + 0xb0, 0xa5, 0xab, 0xcc, 0x8e, 0xa0, 0xf7, 0xd7, 0x46, 0xd0, 0xed, 0xd5, 0x5a, 0xac, 0x1e, 0x40, + 0x1f, 0x40, 0xcb, 0xee, 0x9d, 0xb6, 0xe8, 0xef, 0x54, 0x4a, 0x3d, 0x33, 0x18, 0x9a, 0x83, 0x4b, + 0xb3, 0xe3, 0x43, 0x68, 0x59, 0x6e, 0xb5, 0x9f, 0x97, 0xb7, 0x5d, 0x67, 0x75, 0xdb, 0x3d, 0xff, + 0xab, 0x03, 0xad, 0x81, 0x31, 0x4a, 0x3e, 0x84, 0x96, 0xdd, 0x2d, 0xc8, 0x5b, 0x57, 0xac, 0x4b, + 0x9e, 0x7b, 0xd5, 0x1a, 0x72, 0xea, 0xbc, 0xeb, 0x90, 0x47, 0xd0, 0xd0, 0x43, 0x93, 0x1c, 0x56, + 0x3e, 0xbc, 0xde, 0xad, 0xea, 0xd9, 0xaa, 0x65, 0x1f, 0x40, 0xed, 0x82, 0xc7, 0xe4, 0x66, 0x45, + 0x17, 0x7b, 0x07, 0x55, 0x81, 0x51, 0x52, 0xfd, 0xc7, 0x9f, 0xff, 0xfd, 0xe8, 0xc6, 0xe7, 0x5f, + 0x1c, 0x39, 0x7f, 0xfb, 0xe2, 0xc8, 0xf9, 0xc3, 0x9b, 0x23, 0xe7, 0x47, 0xf7, 0x63, 0x26, 0xc7, + 0xf3, 0x61, 0x2f, 0xe4, 0xd3, 0x33, 0x14, 0x72, 0x1e, 0x64, 0x0b, 0xf3, 0x07, 0x8e, 0xaa, 0xbf, + 0x9f, 0x0c, 0x9b, 0x9a, 0xf4, 0xfe, 0xbf, 0x02, 0x00, 0x00, 0xff, 0xff, 0x19, 0xbb, 0xd3, 0xca, + 0x5e, 0x11, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -2145,6 +2251,34 @@ func (m *Frontier) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.LatestBackfillComplete) > 0 { + for iNdEx := len(m.LatestBackfillComplete) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.LatestBackfillComplete[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShuffle(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } + } + if len(m.LatestBackfillBegin) > 0 { + for iNdEx := len(m.LatestBackfillBegin) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.LatestBackfillBegin[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShuffle(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + } if len(m.FlushedLsn) > 0 { dAtA10 := make([]byte, len(m.FlushedLsn)*10) var j9 int @@ -2180,6 +2314,82 @@ func (m *Frontier) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *Frontier_BackfillBegin) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Frontier_BackfillBegin) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Frontier_BackfillBegin) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Clock != 0 { + i -= 8 + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(m.Clock)) + i-- + dAtA[i] = 0x11 + } + if m.Binding != 0 { + i = encodeVarintShuffle(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *Frontier_BackfillComplete) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Frontier_BackfillComplete) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Frontier_BackfillComplete) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Clock != 0 { + i -= 8 + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(m.Clock)) + i-- + dAtA[i] = 0x11 + } + if m.Binding != 0 { + i = encodeVarintShuffle(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *SessionRequest) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) @@ -3384,6 +3594,54 @@ func (m *Frontier) ProtoSize() (n int) { } n += 1 + sovShuffle(uint64(l)) + l } + if len(m.LatestBackfillBegin) > 0 { + for _, e := range m.LatestBackfillBegin { + l = e.ProtoSize() + n += 1 + l + sovShuffle(uint64(l)) + } + } + if len(m.LatestBackfillComplete) > 0 { + for _, e := range m.LatestBackfillComplete { + l = e.ProtoSize() + n += 1 + l + sovShuffle(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Frontier_BackfillBegin) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Binding != 0 { + n += 1 + sovShuffle(uint64(m.Binding)) + } + if m.Clock != 0 { + n += 9 + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Frontier_BackfillComplete) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Binding != 0 { + n += 1 + sovShuffle(uint64(m.Binding)) + } + if m.Clock != 0 { + n += 9 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4824,6 +5082,234 @@ func (m *Frontier) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field FlushedLsn", wireType) } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field LatestBackfillBegin", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShuffle + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShuffle + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShuffle + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.LatestBackfillBegin = append(m.LatestBackfillBegin, &Frontier_BackfillBegin{}) + if err := m.LatestBackfillBegin[len(m.LatestBackfillBegin)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field LatestBackfillComplete", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShuffle + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShuffle + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShuffle + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.LatestBackfillComplete = append(m.LatestBackfillComplete, &Frontier_BackfillComplete{}) + if err := m.LatestBackfillComplete[len(m.LatestBackfillComplete)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShuffle(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShuffle + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Frontier_BackfillBegin) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShuffle + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BackfillBegin: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BackfillBegin: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShuffle + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 1 { + return fmt.Errorf("proto: wrong wireType = %d for field Clock", wireType) + } + m.Clock = 0 + if (iNdEx + 8) > l { + return io.ErrUnexpectedEOF + } + m.Clock = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + iNdEx += 8 + default: + iNdEx = preIndex + skippy, err := skipShuffle(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShuffle + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Frontier_BackfillComplete) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShuffle + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BackfillComplete: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BackfillComplete: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShuffle + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 1 { + return fmt.Errorf("proto: wrong wireType = %d for field Clock", wireType) + } + m.Clock = 0 + if (iNdEx + 8) > l { + return io.ErrUnexpectedEOF + } + m.Clock = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + iNdEx += 8 default: iNdEx = preIndex skippy, err := skipShuffle(dAtA[iNdEx:]) diff --git a/go/protocols/shuffle/shuffle.proto b/go/protocols/shuffle/shuffle.proto index 7689ac8af53..7af6221411c 100644 --- a/go/protocols/shuffle/shuffle.proto +++ b/go/protocols/shuffle/shuffle.proto @@ -91,7 +91,7 @@ message JournalFrontier { string journal_name_suffix = 2; // Binding index under which the journal is read. // When persisting across sessions, this should be mapped via the task binding's - // `journal_read_suffix` to ensure stability across task versions. + // `state_key` to ensure stability across task versions. uint32 binding = 3; // Delta of journal bytes read since the last checkpoint. // Summed during reduction. @@ -111,6 +111,32 @@ message Frontier { repeated JournalFrontier journals = 1; // Per-shard flushed LSN, indexed by shard_index. repeated uint64 flushed_lsn = 2; + + // BackfillBegin is a binding's latest backfill-begin clock, keyed by binding + // index. + message BackfillBegin { + // Binding index. + uint32 binding = 1; + // Clock of the binding's most recent backfill-begin. + fixed64 clock = 2; + } + // Latest backfill-begin clock for each binding in the checkpoint delta. + // Populated only on a terminal (empty-journals) frontier of a Progressed + // or NextCheckpoint sequence. Empty otherwise. + repeated BackfillBegin latest_backfill_begin = 3; + + // BackfillComplete reports a binding's most recent backfill-complete event, + // keyed by binding index. + message BackfillComplete { + // Binding index. + uint32 binding = 1; + // Truncation boundary of the completed backfill: the backfill's begin clock. + fixed64 clock = 2; + } + // Latest backfill-complete clock for each binding in the checkpoint delta. + // Populated only on a terminal (empty-journals) frontier of a Progressed + // or NextCheckpoint sequence. Empty otherwise. + repeated BackfillComplete latest_backfill_complete = 4; } // SessionRequest is sent by the Coordinator to manage the shuffle session.