From b2200d05efa619c46cdbdba0e343143d81bd0b76 Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:19:42 -0700 Subject: [PATCH 01/17] feat(storage): add segmented write-ahead log --- Cargo.lock | 19 +++ Cargo.toml | 7 +- cloud9-wal/Cargo.toml | 21 +++ cloud9-wal/src/error.rs | 68 ++++++++++ cloud9-wal/src/format.rs | 157 ++++++++++++++++++++++ cloud9-wal/src/lib.rs | 25 ++++ cloud9-wal/src/record.rs | 39 ++++++ cloud9-wal/src/segment.rs | 206 +++++++++++++++++++++++++++++ cloud9-wal/src/tests.rs | 225 ++++++++++++++++++++++++++++++++ cloud9-wal/src/wal.rs | 267 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 1032 insertions(+), 2 deletions(-) create mode 100644 cloud9-wal/Cargo.toml create mode 100644 cloud9-wal/src/error.rs create mode 100644 cloud9-wal/src/format.rs create mode 100644 cloud9-wal/src/lib.rs create mode 100644 cloud9-wal/src/record.rs create mode 100644 cloud9-wal/src/segment.rs create mode 100644 cloud9-wal/src/tests.rs create mode 100644 cloud9-wal/src/wal.rs diff --git a/Cargo.lock b/Cargo.lock index 5caa148..cb675d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -261,12 +261,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "cloud9-wal" +version = "0.0.1" +dependencies = [ + "crc32fast", + "fs-err", + "tempfile", + "thiserror", +] + [[package]] name = "colorchoice" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "deranged" version = "0.5.5" diff --git a/Cargo.toml b/Cargo.toml index dcd0abe..db94ad7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["cloud9", "cloud9-core", "cloud9-node", "cloud9-proto", "cloud9-storage", "consensus/cloud9-raft", "consensus/cloud9-raft-io"] +members = ["cloud9", "cloud9-core", "cloud9-node", "cloud9-proto", "cloud9-storage", "cloud9-wal", "consensus/cloud9-raft", "consensus/cloud9-raft-io"] resolver = "2" [workspace.package] @@ -8,7 +8,7 @@ edition = "2024" rust-version = "1.95.0" license = "MIT" authors = ["Windsor Nguyen "] -repository = "https://github.com/dedalus-labs/cloud9" +repository = "https://github.com/windsornguyen/cloud9" homepage = "https://dedaluslabs.ai" [workspace.lints.rust] @@ -45,6 +45,7 @@ unimplemented = "warn" # Internal crates cloud9-core = { path = "cloud9-core" } cloud9-storage = { path = "cloud9-storage" } +cloud9-wal = { path = "cloud9-wal" } cloud9-raft = { path = "consensus/cloud9-raft" } cloud9-raft-io = { path = "consensus/cloud9-raft-io" } cloud9-proto = { path = "cloud9-proto" } @@ -64,6 +65,7 @@ arcstr = { version = "1.2", features = ["serde"] } fs-err = { version = "3", features = ["tokio"] } miette = { version = "7.2", features = ["fancy-no-backtrace"] } textwrap = "0.16" +crc32fast = "1.5" # Testing - concurrency loom = "0.7.2" @@ -74,3 +76,4 @@ proptest-state-machine = "0.6" # Testing - concurrency (randomized) shuttle = "0.8" +tempfile = "3.20" diff --git a/cloud9-wal/Cargo.toml b/cloud9-wal/Cargo.toml new file mode 100644 index 0000000..444e39d --- /dev/null +++ b/cloud9-wal/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "cloud9-wal" +version = { workspace = true } +edition = { workspace = true } +rust-version = { workspace = true } +license = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +description = "Minimal segmented write-ahead log for Cloud9" + +[lints] +workspace = true + +[dependencies] +crc32fast = { workspace = true } +fs-err = { version = "3", default-features = false } +thiserror = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/cloud9-wal/src/error.rs b/cloud9-wal/src/error.rs new file mode 100644 index 0000000..ad91bd1 --- /dev/null +++ b/cloud9-wal/src/error.rs @@ -0,0 +1,68 @@ +use std::io; +use std::path::PathBuf; + +use thiserror::Error; + +use crate::record::Lsn; + +/// WAL failures. +#[derive(Debug, Error)] +pub enum WalError { + #[error("I/O error at `{path}`")] + Io { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("record kind zero is reserved")] + ReservedRecordKind, + #[error("segment size {segment_size} is smaller than record header {header_len}")] + SegmentTooSmall { segment_size: u64, header_len: usize }, + #[error("record length {len} exceeds segment size {segment_size}")] + RecordTooLarge { len: u64, segment_size: u64 }, + #[error("WAL size {len} exceeds configured maximum {max_size}")] + WalFull { len: u64, max_size: u64 }, + #[error("payload length {len} exceeds u32::MAX")] + PayloadTooLarge { len: usize }, + #[error("segment id exhausted")] + SegmentIdExhausted, + #[error("corrupt WAL record at segment {lsn:?}: {reason}")] + CorruptRecord { lsn: Lsn, reason: Corruption }, + #[error("malformed WAL segment filename `{path}`")] + BadSegmentName { path: PathBuf }, + #[error("missing WAL segment {expected:020}.wal before {found:020}.wal")] + MissingSegment { expected: u64, found: u64 }, + #[error("WAL directory `{path}` already has a writer")] + Locked { path: PathBuf }, + #[error("WAL handle is poisoned; reopen it to recover")] + Poisoned, +} + +impl WalError { + pub(crate) fn io(path: PathBuf, source: io::Error) -> Self { + Self::Io { path, source } + } +} + +/// Specific corruption detected while scanning records. +#[derive(Debug, Error)] +pub enum Corruption { + #[error("bad magic {found:#x}")] + BadMagic { found: u32 }, + #[error("unsupported version {found}")] + UnsupportedVersion { found: u16 }, + #[error("reserved record kind")] + ReservedRecordKind, + #[error("header checksum mismatch")] + HeaderChecksum, + #[error("payload checksum mismatch")] + PayloadChecksum, + #[error("record length {len} exceeds segment size {segment_size}")] + RecordTooLarge { len: u64, segment_size: u64 }, + #[error("segment length {len} exceeds configured size {segment_size}")] + SegmentTooLarge { len: u64, segment_size: u64 }, + #[error("incomplete record")] + IncompleteRecord, +} + +pub type Result = std::result::Result; diff --git a/cloud9-wal/src/format.rs b/cloud9-wal/src/format.rs new file mode 100644 index 0000000..01550a1 --- /dev/null +++ b/cloud9-wal/src/format.rs @@ -0,0 +1,157 @@ +use std::io::{ErrorKind, Read}; +use std::path::Path; + +use crc32fast::Hasher; + +use crate::error::{Corruption, Result, WalError}; +use crate::record::{Lsn, Record, RecordKind}; + +pub(crate) const MAGIC: u32 = 0x4339_574c; // C9WL +pub(crate) const VERSION: u16 = 1; +pub(crate) const HEADER_LEN: usize = 32; +pub(crate) const HEADER_LEN_U64: u64 = 32; +const HEADER_CRC_END: usize = 28; + +pub(crate) enum ReadOne { + Record { record: Record, next_offset: u64 }, + Eof, + Incomplete, +} + +pub(crate) fn encode_record(kind: RecordKind, payload: &[u8]) -> Result> { + let encoded_len = encoded_len(payload.len())?; + let payload_len = u32::try_from(payload.len()) + .map_err(|_| WalError::PayloadTooLarge { len: payload.len() })?; + let mut header = [0_u8; HEADER_LEN]; + header[0..4].copy_from_slice(&MAGIC.to_le_bytes()); + header[4..6].copy_from_slice(&VERSION.to_le_bytes()); + header[6..8].copy_from_slice(&kind.get().to_le_bytes()); + header[8..12].copy_from_slice(&payload_len.to_le_bytes()); + header[12..16].copy_from_slice(&crc32(payload).to_le_bytes()); + let header_crc = crc32(&header[..HEADER_CRC_END]); + header[28..32].copy_from_slice(&header_crc.to_le_bytes()); + + let mut encoded = Vec::with_capacity(encoded_len); + encoded.extend_from_slice(&header); + encoded.extend_from_slice(payload); + Ok(encoded) +} + +pub(crate) fn read_one( + path: &Path, + reader: &mut impl Read, + lsn: Lsn, + segment_size: u64, + physical_len: u64, +) -> Result { + let mut header = [0_u8; HEADER_LEN]; + match read_exact_or_tail(path, reader, &mut header)? { + ReadExact::Eof => return Ok(ReadOne::Eof), + ReadExact::Incomplete => return Ok(ReadOne::Incomplete), + ReadExact::Complete => {} + } + + let kind = decode_header(&header, lsn)?; + let payload_len = u64::from(read_u32(&header, 8)); + let record_len = HEADER_LEN_U64.checked_add(payload_len).ok_or(WalError::CorruptRecord { + lsn, + reason: Corruption::RecordTooLarge { len: u64::MAX, segment_size }, + })?; + if record_len > segment_size { + return Err(WalError::CorruptRecord { + lsn, + reason: Corruption::RecordTooLarge { len: record_len, segment_size }, + }); + } + let next_offset = lsn.offset.checked_add(record_len).ok_or(WalError::CorruptRecord { + lsn, + reason: Corruption::SegmentTooLarge { len: u64::MAX, segment_size }, + })?; + if next_offset > segment_size { + return Err(WalError::CorruptRecord { + lsn, + reason: Corruption::SegmentTooLarge { len: next_offset, segment_size }, + }); + } + if next_offset > physical_len { + return Ok(ReadOne::Incomplete); + } + let payload_len = usize::try_from(payload_len).map_err(|_| WalError::CorruptRecord { + lsn, + reason: Corruption::RecordTooLarge { len: record_len, segment_size }, + })?; + + let mut payload = vec![0; payload_len]; + if matches!( + read_exact_or_tail(path, reader, &mut payload)?, + ReadExact::Incomplete | ReadExact::Eof + ) { + return Ok(ReadOne::Incomplete); + } + if crc32(&payload) != read_u32(&header, 12) { + return Err(WalError::CorruptRecord { lsn, reason: Corruption::PayloadChecksum }); + } + Ok(ReadOne::Record { record: Record { kind, payload }, next_offset }) +} + +pub(crate) fn encoded_len(payload_len: usize) -> Result { + u32::try_from(payload_len).map_err(|_| WalError::PayloadTooLarge { len: payload_len })?; + HEADER_LEN.checked_add(payload_len).ok_or(WalError::PayloadTooLarge { len: payload_len }) +} + +enum ReadExact { + Complete, + Eof, + Incomplete, +} + +fn read_exact_or_tail(path: &Path, reader: &mut impl Read, buf: &mut [u8]) -> Result { + let mut filled = 0; + while filled < buf.len() { + match reader.read(&mut buf[filled..]) { + Ok(0) if filled == 0 => return Ok(ReadExact::Eof), + Ok(0) => return Ok(ReadExact::Incomplete), + Ok(n) => filled += n, + Err(source) if source.kind() == ErrorKind::Interrupted => {} + Err(source) => return Err(WalError::io(path.to_path_buf(), source)), + } + } + Ok(ReadExact::Complete) +} + +fn decode_header(header: &[u8; HEADER_LEN], lsn: Lsn) -> Result { + let magic = read_u32(header, 0); + if magic != MAGIC { + return Err(WalError::CorruptRecord { lsn, reason: Corruption::BadMagic { found: magic } }); + } + let version = read_u16(header, 4); + if version != VERSION { + return Err(WalError::CorruptRecord { + lsn, + reason: Corruption::UnsupportedVersion { found: version }, + }); + } + if crc32(&header[..HEADER_CRC_END]) != read_u32(header, 28) { + return Err(WalError::CorruptRecord { lsn, reason: Corruption::HeaderChecksum }); + } + RecordKind::new(read_u16(header, 6)) + .map_err(|_| WalError::CorruptRecord { lsn, reason: Corruption::ReservedRecordKind }) +} + +fn crc32(bytes: &[u8]) -> u32 { + let mut hasher = Hasher::new(); + hasher.update(bytes); + hasher.finalize() +} + +fn read_u16(bytes: &[u8], start: usize) -> u16 { + let mut out = [0; 2]; + out.copy_from_slice(&bytes[start..start + 2]); + u16::from_le_bytes(out) +} + +fn read_u32(bytes: &[u8], start: usize) -> u32 { + let mut out = [0; 4]; + out.copy_from_slice(&bytes[start..start + 4]); + u32::from_le_bytes(out) +} diff --git a/cloud9-wal/src/lib.rs b/cloud9-wal/src/lib.rs new file mode 100644 index 0000000..53ad063 --- /dev/null +++ b/cloud9-wal/src/lib.rs @@ -0,0 +1,25 @@ +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::panic))] + +//! Tiny segmented write-ahead log. +//! +//! The WAL deliberately owns only the byte-durability problem: +//! 1. append one typed byte record to the active segment, +//! 2. sync the active segment when the caller asks for durability, +//! 3. recover by scanning segment files and truncating only an incomplete tail. +//! +//! Higher layers own meaning. Raft entries, hard state, logical truncation, and +//! snapshots are just payloads encoded by the caller. + +mod error; +mod format; +mod record; +mod segment; +#[cfg(test)] +mod tests; +mod wal; + +pub use error::{Corruption, Result, WalError}; +pub use record::{Lsn, Record, RecordKind, StoredRecord}; +pub use wal::{Records, Wal, WalOptions}; diff --git a/cloud9-wal/src/record.rs b/cloud9-wal/src/record.rs new file mode 100644 index 0000000..0da7fb4 --- /dev/null +++ b/cloud9-wal/src/record.rs @@ -0,0 +1,39 @@ +use crate::{Result, WalError}; + +/// Position of a WAL record. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct Lsn { + pub segment_id: u64, + pub offset: u64, +} + +/// Caller-owned record kind. Kind zero is reserved for invalid headers. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RecordKind(u16); + +impl RecordKind { + /// Create a non-zero record kind. + pub fn new(value: u16) -> Result { + if value == 0 { Err(WalError::ReservedRecordKind) } else { Ok(Self(value)) } + } + + /// Return the on-disk kind value. + #[must_use] + pub const fn get(self) -> u16 { + self.0 + } +} + +/// One logical WAL record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Record { + pub kind: RecordKind, + pub payload: Vec, +} + +/// A record with its log position. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredRecord { + pub lsn: Lsn, + pub record: Record, +} diff --git a/cloud9-wal/src/segment.rs b/cloud9-wal/src/segment.rs new file mode 100644 index 0000000..fcc8fca --- /dev/null +++ b/cloud9-wal/src/segment.rs @@ -0,0 +1,206 @@ +use std::path::{Path, PathBuf}; + +use fs_err::{self as fs, File, OpenOptions}; + +use crate::error::{Corruption, Result, WalError}; +use crate::format::{self, ReadOne}; +use crate::record::{Lsn, StoredRecord}; + +const SEGMENT_SUFFIX: &str = "wal"; +const LOCK_FILE: &str = "LOCK"; + +pub(crate) fn recover(dir: &Path, segment_size: u64) -> Result<(u64, u64)> { + let mut ids = segment_ids(dir)?; + if ids.is_empty() { + let segment_id = 0; + File::create(segment_path(dir, segment_id)) + .map_err(|source| WalError::io(segment_path(dir, segment_id), source))?; + sync_dir(dir)?; + ids.push(segment_id); + } + + for (position, segment_id) in ids.iter().copied().enumerate() { + match valid_len(dir, segment_id, segment_size)? { + SegmentScan::Clean(len) => { + if position + 1 == ids.len() { + return Ok((segment_id, len)); + } + } + SegmentScan::IncompleteTail(len) => { + if position + 1 != ids.len() { + return Err(WalError::CorruptRecord { + lsn: Lsn { segment_id, offset: len }, + reason: Corruption::IncompleteRecord, + }); + } + truncate_segment(dir, segment_id, len)?; + return Ok((segment_id, len)); + } + } + } + let last = ids.last().copied().ok_or(WalError::SegmentIdExhausted)?; + Ok((last, segment_len(dir, last)?)) +} + +pub(crate) struct SegmentReader { + path: PathBuf, + segment_id: u64, + segment_size: u64, + physical_len: u64, + offset: u64, + file: File, +} + +impl SegmentReader { + pub(crate) fn open(dir: &Path, segment_id: u64, segment_size: u64) -> Result { + let path = segment_path(dir, segment_id); + let file = File::open(&path).map_err(|source| WalError::io(path.clone(), source))?; + let physical_len = + file.metadata().map_err(|source| WalError::io(path.clone(), source))?.len(); + Ok(Self { path, segment_id, segment_size, physical_len, offset: 0, file }) + } + + pub(crate) fn next_record(&mut self) -> Result> { + let lsn = Lsn { segment_id: self.segment_id, offset: self.offset }; + match format::read_one( + &self.path, + &mut self.file, + lsn, + self.segment_size, + self.physical_len, + )? { + ReadOne::Record { record, next_offset } => { + self.offset = next_offset; + Ok(Some(StoredRecord { lsn, record })) + } + ReadOne::Eof => Ok(None), + ReadOne::Incomplete => { + Err(WalError::CorruptRecord { lsn, reason: Corruption::IncompleteRecord }) + } + } + } +} + +pub(crate) fn segment_ids(dir: &Path) -> Result> { + let mut ids = Vec::new(); + for entry in fs::read_dir(dir).map_err(|source| WalError::io(dir.to_path_buf(), source))? { + let entry = entry.map_err(|source| WalError::io(dir.to_path_buf(), source))?; + let path = entry.path(); + if path.extension().and_then(std::ffi::OsStr::to_str) == Some(SEGMENT_SUFFIX) { + ids.push(segment_id(&path)?); + } + } + ids.sort_unstable(); + for (expected, found) in ids.iter().copied().enumerate() { + let expected = u64::try_from(expected).map_err(|_| WalError::SegmentIdExhausted)?; + if found != expected { + return Err(WalError::MissingSegment { expected, found }); + } + } + Ok(ids) +} + +pub(crate) fn segment_path(dir: &Path, segment_id: u64) -> PathBuf { + dir.join(format!("{segment_id:020}.wal")) +} + +pub(crate) fn open_segment(dir: &Path, segment_id: u64) -> Result { + let path = segment_path(dir, segment_id); + OpenOptions::new() + .read(true) + .append(true) + .create(true) + .open(&path) + .map_err(|source| WalError::io(path, source)) +} + +pub(crate) fn create_dir_all(path: &Path) -> Result<()> { + let mut missing = Vec::new(); + let mut cursor = path; + while !cursor.try_exists().map_err(|source| WalError::io(cursor.to_path_buf(), source))? { + missing.push(cursor.to_path_buf()); + cursor = cursor.parent().ok_or(WalError::SegmentIdExhausted)?; + } + fs::create_dir_all(path).map_err(|source| WalError::io(path.to_path_buf(), source))?; + for created in missing.iter().rev() { + let parent = created.parent().ok_or(WalError::SegmentIdExhausted)?; + sync_dir(parent)?; + } + Ok(()) +} + +pub(crate) fn lock(dir: &Path) -> Result { + let path = dir.join(LOCK_FILE); + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(&path) + .map_err(|source| WalError::io(path.clone(), source))?; + match file.try_lock() { + Ok(()) => Ok(file), + Err(std::fs::TryLockError::WouldBlock) => Err(WalError::Locked { path }), + Err(std::fs::TryLockError::Error(source)) => Err(WalError::io(path, source)), + } +} + +pub(crate) fn sync_dir(dir: &Path) -> Result<()> { + File::open(dir) + .and_then(|file| file.sync_all()) + .map_err(|source| WalError::io(dir.to_path_buf(), source)) +} + +enum SegmentScan { + Clean(u64), + IncompleteTail(u64), +} + +fn valid_len(dir: &Path, segment_id: u64, segment_size: u64) -> Result { + let path = segment_path(dir, segment_id); + let mut file = File::open(&path).map_err(|source| WalError::io(path.clone(), source))?; + let physical_len = file.metadata().map_err(|source| WalError::io(path.clone(), source))?.len(); + let mut offset = 0; + loop { + match format::read_one( + &path, + &mut file, + Lsn { segment_id, offset }, + segment_size, + physical_len, + )? { + ReadOne::Record { next_offset, .. } => offset = next_offset, + ReadOne::Eof => return Ok(SegmentScan::Clean(offset)), + ReadOne::Incomplete => return Ok(SegmentScan::IncompleteTail(offset)), + } + } +} + +fn segment_id(path: &Path) -> Result { + let Some(stem) = path.file_stem().and_then(std::ffi::OsStr::to_str) else { + return Err(WalError::BadSegmentName { path: path.to_path_buf() }); + }; + stem.parse::().map_err(|_| WalError::BadSegmentName { path: path.to_path_buf() }) +} + +fn segment_len(dir: &Path, segment_id: u64) -> Result { + let path = segment_path(dir, segment_id); + fs::metadata(&path).map_err(|source| WalError::io(path, source)).map(|meta| meta.len()) +} + +pub(crate) fn total_len(dir: &Path) -> Result { + segment_ids(dir)?.into_iter().try_fold(0_u64, |total, segment_id| { + total + .checked_add(segment_len(dir, segment_id)?) + .ok_or(WalError::WalFull { len: u64::MAX, max_size: u64::MAX }) + }) +} + +fn truncate_segment(dir: &Path, segment_id: u64, len: u64) -> Result<()> { + let path = segment_path(dir, segment_id); + let file = OpenOptions::new() + .write(true) + .open(&path) + .map_err(|source| WalError::io(path.clone(), source))?; + file.set_len(len).map_err(|source| WalError::io(path.clone(), source))?; + file.sync_all().map_err(|source| WalError::io(path, source)) +} diff --git a/cloud9-wal/src/tests.rs b/cloud9-wal/src/tests.rs new file mode 100644 index 0000000..80e5ead --- /dev/null +++ b/cloud9-wal/src/tests.rs @@ -0,0 +1,225 @@ +use std::io::{Seek, SeekFrom, Write}; + +use fs_err::{self as fs, File, OpenOptions}; + +use crate::segment::segment_path; +use crate::*; + +fn kind(value: u16) -> RecordKind { + RecordKind::new(value).unwrap() +} + +fn payload(value: u8) -> Vec { + vec![value; 8] +} + +#[test] +fn append_and_recover_records() { + let dir = tempfile::tempdir().unwrap(); + let mut wal = Wal::open(dir.path(), WalOptions::default()).unwrap(); + + let first = wal.append(kind(1), payload(1)).unwrap(); + let second = wal.append(kind(2), payload(2)).unwrap(); + wal.sync().unwrap(); + drop(wal); + + let wal = Wal::open(dir.path(), WalOptions::default()).unwrap(); + let records = wal.records().unwrap().collect::>>().unwrap(); + + assert_eq!(records.len(), 2); + assert_eq!(records[0].lsn, first); + assert_eq!(records[0].record.kind, kind(1)); + assert_eq!(records[1].lsn, second); + assert_eq!(records[1].record.payload, payload(2)); +} + +#[test] +fn rotates_segments() { + let dir = tempfile::tempdir().unwrap(); + let options = WalOptions { segment_size: 96, ..WalOptions::default() }; + let mut wal = Wal::open(dir.path(), options.clone()).unwrap(); + + let first = wal.append(kind(1), payload(1)).unwrap(); + let second = wal.append(kind(1), payload(2)).unwrap(); + let third = wal.append(kind(1), payload(3)).unwrap(); + wal.sync().unwrap(); + drop(wal); + + assert_eq!(first.segment_id, 0); + assert_eq!(second.segment_id, 0); + assert_eq!(third.segment_id, 1); + + let wal = Wal::open(dir.path(), options).unwrap(); + let records = wal.records().unwrap().collect::>>().unwrap(); + assert_eq!(records.iter().map(|r| r.record.payload[0]).collect::>(), [1, 2, 3]); +} + +#[test] +fn recovery_truncates_incomplete_tail() { + let dir = tempfile::tempdir().unwrap(); + let mut wal = Wal::open(dir.path(), WalOptions::default()).unwrap(); + wal.append(kind(1), payload(1)).unwrap(); + wal.sync().unwrap(); + drop(wal); + + let segment = segment_path(dir.path(), 0); + let mut file = OpenOptions::new().append(true).open(&segment).unwrap(); + file.write_all(&[1, 2, 3]).unwrap(); + file.sync_all().unwrap(); + drop(file); + + let wal = Wal::open(dir.path(), WalOptions::default()).unwrap(); + let records = wal.records().unwrap().collect::>>().unwrap(); + + assert_eq!(records.len(), 1); + assert_eq!(records[0].record.payload, payload(1)); + assert_eq!(fs::metadata(segment).unwrap().len(), crate::format::HEADER_LEN_U64 + 8); +} + +#[test] +fn invariant_recovery_never_discards_later_segments() { + let dir = tempfile::tempdir().unwrap(); + let options = WalOptions { segment_size: 40, ..WalOptions::default() }; + let mut wal = Wal::open(dir.path(), options.clone()).unwrap(); + wal.append(kind(1), payload(1)).unwrap(); + wal.append(kind(1), payload(2)).unwrap(); + wal.sync().unwrap(); + drop(wal); + + let first = segment_path(dir.path(), 0); + OpenOptions::new().write(true).open(first).unwrap().set_len(39).unwrap(); + + let err = Wal::open(dir.path(), options).unwrap_err(); + assert!(matches!( + err, + WalError::CorruptRecord { + lsn: Lsn { segment_id: 0, offset: 0 }, + reason: Corruption::IncompleteRecord, + } + )); + assert_eq!(fs::metadata(segment_path(dir.path(), 1)).unwrap().len(), 40); +} + +#[test] +fn recovery_rejects_checksum_corruption() { + let dir = tempfile::tempdir().unwrap(); + let mut wal = Wal::open(dir.path(), WalOptions::default()).unwrap(); + wal.append(kind(1), payload(1)).unwrap(); + wal.sync().unwrap(); + drop(wal); + + let segment = segment_path(dir.path(), 0); + let mut file = OpenOptions::new().read(true).write(true).open(segment).unwrap(); + file.seek(SeekFrom::Start(crate::format::HEADER_LEN_U64)).unwrap(); + file.write_all(&[9]).unwrap(); + file.sync_all().unwrap(); + drop(file); + + let err = Wal::open(dir.path(), WalOptions::default()).unwrap_err(); + assert!(matches!(err, WalError::CorruptRecord { reason: Corruption::PayloadChecksum, .. })); +} + +#[test] +fn recovery_rejects_missing_segment() { + let dir = tempfile::tempdir().unwrap(); + File::create(segment_path(dir.path(), 0)).unwrap(); + File::create(segment_path(dir.path(), 2)).unwrap(); + + let err = Wal::open(dir.path(), WalOptions::default()).unwrap_err(); + assert!(matches!(err, WalError::MissingSegment { expected: 1, found: 2 })); +} + +#[test] +fn recovery_rejects_record_larger_than_segment() { + let dir = tempfile::tempdir().unwrap(); + let mut wal = + Wal::open(dir.path(), WalOptions { segment_size: 40, ..WalOptions::default() }).unwrap(); + wal.append(kind(1), payload(1)).unwrap(); + wal.sync().unwrap(); + drop(wal); + + let err = Wal::open(dir.path(), WalOptions { segment_size: 39, ..WalOptions::default() }) + .unwrap_err(); + assert!(matches!( + err, + WalError::CorruptRecord { + reason: Corruption::RecordTooLarge { len: 40, segment_size: 39 }, + .. + } + )); +} + +#[test] +fn rejects_reserved_kind() { + let err = RecordKind::new(0).unwrap_err(); + assert!(matches!(err, WalError::ReservedRecordKind)); +} + +#[test] +fn rejects_records_larger_than_segment() { + let dir = tempfile::tempdir().unwrap(); + let mut wal = + Wal::open(dir.path(), WalOptions { segment_size: 39, ..WalOptions::default() }).unwrap(); + + let err = wal.append(kind(1), payload(1)).unwrap_err(); + assert!(matches!(err, WalError::RecordTooLarge { .. })); +} + +#[test] +fn invariant_failed_rotation_preserves_active_segment() { + let dir = tempfile::tempdir().unwrap(); + let options = WalOptions { segment_size: 40, ..WalOptions::default() }; + let mut wal = Wal::open(dir.path(), options).unwrap(); + wal.append(kind(1), payload(1)).unwrap(); + + let next = segment_path(dir.path(), 1); + fs::create_dir(&next).unwrap(); + assert!(wal.append(kind(1), payload(2)).is_err()); + fs::remove_dir(next).unwrap(); + + let lsn = wal.append(kind(1), payload(2)).unwrap(); + assert_eq!(lsn.segment_id, 1); +} + +#[test] +fn invariant_wal_has_single_writer() { + let dir = tempfile::tempdir().unwrap(); + let wal = Wal::open(dir.path(), WalOptions::default()).unwrap(); + + assert!(Wal::open(dir.path(), WalOptions::default()).is_err()); + drop(wal); + assert!(Wal::open(dir.path(), WalOptions::default()).is_ok()); +} + +#[test] +fn invariant_recovery_enforces_segment_size() { + let dir = tempfile::tempdir().unwrap(); + let mut wal = + Wal::open(dir.path(), WalOptions { segment_size: 120, ..WalOptions::default() }).unwrap(); + for value in 0..3 { + wal.append(kind(1), payload(value)).unwrap(); + } + wal.sync().unwrap(); + drop(wal); + + let error = Wal::open(dir.path(), WalOptions { segment_size: 80, ..WalOptions::default() }) + .unwrap_err(); + assert!(matches!( + error, + WalError::CorruptRecord { + reason: Corruption::SegmentTooLarge { len: 120, segment_size: 80 }, + .. + } + )); +} + +#[test] +fn invariant_wal_capacity_is_bounded() { + let dir = tempfile::tempdir().unwrap(); + let options = WalOptions { segment_size: 80, max_size: 80, sync_on_append: false }; + let mut wal = Wal::open(dir.path(), options).unwrap(); + wal.append(kind(1), payload(1)).unwrap(); + wal.append(kind(1), payload(2)).unwrap(); + + assert!(matches!(wal.append(kind(1), payload(3)), Err(WalError::WalFull { .. }))); +} diff --git a/cloud9-wal/src/wal.rs b/cloud9-wal/src/wal.rs new file mode 100644 index 0000000..7cdbf19 --- /dev/null +++ b/cloud9-wal/src/wal.rs @@ -0,0 +1,267 @@ +use std::io::{Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use fs_err::File; + +use crate::error::{Result, WalError}; +use crate::format; +use crate::record::{Lsn, RecordKind, StoredRecord}; +use crate::segment::{self, SegmentReader}; + +const DEFAULT_SEGMENT_SIZE: u64 = 64 * 1024 * 1024; +const DEFAULT_MAX_SIZE: u64 = 4 * 1024 * 1024 * 1024; + +/// WAL configuration. +#[derive(Debug, Clone)] +pub struct WalOptions { + pub segment_size: u64, + pub max_size: u64, + pub sync_on_append: bool, +} + +impl Default for WalOptions { + fn default() -> Self { + Self { + segment_size: DEFAULT_SEGMENT_SIZE, + max_size: DEFAULT_MAX_SIZE, + sync_on_append: false, + } + } +} + +/// Append-only segmented WAL. +#[derive(Debug)] +pub struct Wal { + dir: PathBuf, + options: WalOptions, + _lock: File, + active_id: u64, + active_len: u64, + total_len: u64, + active: File, + poisoned: bool, +} + +/// Streaming WAL records in LSN order. +pub struct Records<'a> { + dir: &'a Path, + segment_size: u64, + segment_ids: std::vec::IntoIter, + segment: Option, + done: bool, +} + +impl Iterator for Records<'_> { + type Item = Result; + + fn next(&mut self) -> Option { + if self.done { + return None; + } + loop { + if let Some(segment) = &mut self.segment { + match segment.next_record() { + Ok(Some(record)) => return Some(Ok(record)), + Ok(None) => self.segment = None, + Err(error) => { + self.done = true; + return Some(Err(error)); + } + } + } else if let Some(segment_id) = self.segment_ids.next() { + match SegmentReader::open(self.dir, segment_id, self.segment_size) { + Ok(segment) => self.segment = Some(segment), + Err(error) => { + self.done = true; + return Some(Err(error)); + } + } + } else { + self.done = true; + return None; + } + } + } +} + +impl Wal { + /// Open or create a WAL directory. + pub fn open(dir: impl AsRef, options: WalOptions) -> Result { + validate_options(&options)?; + let dir = absolute_path(dir.as_ref())?; + segment::create_dir_all(&dir)?; + let lock = segment::lock(&dir)?; + let (active_id, active_len) = segment::recover(&dir, options.segment_size)?; + let total_len = segment::total_len(&dir)?; + if total_len > options.max_size { + return Err(WalError::WalFull { len: total_len, max_size: options.max_size }); + } + let active = segment::open_segment(&dir, active_id)?; + let mut wal = Self { + dir, + options, + _lock: lock, + active_id, + active_len, + total_len, + active, + poisoned: false, + }; + wal.active.seek(SeekFrom::Start(active_len)).map_err(|source| { + WalError::io(segment::segment_path(&wal.dir, wal.active_id), source) + })?; + Ok(wal) + } + + /// Append a record and return its LSN. + pub fn append(&mut self, kind: RecordKind, payload: impl AsRef<[u8]>) -> Result { + self.ensure_healthy()?; + let payload = payload.as_ref(); + let record_len = format::encoded_len(payload.len())?; + let next_total = checked_add_len(self.total_len, record_len)?; + if next_total > self.options.max_size { + return Err(WalError::WalFull { len: next_total, max_size: self.options.max_size }); + } + self.rotate_if_needed(record_len)?; + let encoded = format::encode_record(kind, payload)?; + let lsn = Lsn { segment_id: self.active_id, offset: self.active_len }; + let next_len = checked_add_len(self.active_len, encoded.len())?; + write_or_poison(&mut self.active, &encoded, &mut self.poisoned).map_err(|source| { + WalError::io(segment::segment_path(&self.dir, self.active_id), source) + })?; + self.active_len = next_len; + self.total_len = next_total; + if self.options.sync_on_append { + self.sync()?; + } + Ok(lsn) + } + + /// Force pending WAL bytes to stable storage. + pub fn sync(&mut self) -> Result<()> { + self.ensure_healthy()?; + if let Err(source) = self.active.sync_all() { + self.poisoned = true; + return Err(WalError::io(segment::segment_path(&self.dir, self.active_id), source)); + } + Ok(()) + } + + /// Stream every valid record in LSN order. + pub fn records(&self) -> Result> { + self.ensure_healthy()?; + Ok(Records { + dir: &self.dir, + segment_size: self.options.segment_size, + segment_ids: segment::segment_ids(&self.dir)?.into_iter(), + segment: None, + done: false, + }) + } + + fn rotate_if_needed(&mut self, record_len: usize) -> Result<()> { + let record_len = u64::try_from(record_len).map_err(|_| WalError::RecordTooLarge { + len: u64::MAX, + segment_size: self.options.segment_size, + })?; + if record_len > self.options.segment_size { + return Err(WalError::RecordTooLarge { + len: record_len, + segment_size: self.options.segment_size, + }); + } + let next_len = self.active_len.checked_add(record_len).ok_or(WalError::RecordTooLarge { + len: u64::MAX, + segment_size: self.options.segment_size, + })?; + if self.active_len != 0 && next_len > self.options.segment_size { + self.sync()?; + let active_id = self.active_id.checked_add(1).ok_or(WalError::SegmentIdExhausted)?; + let active = segment::open_segment(&self.dir, active_id)?; + segment::sync_dir(&self.dir)?; + self.active_id = active_id; + self.active = active; + self.active_len = 0; + } + Ok(()) + } + + fn ensure_healthy(&self) -> Result<()> { + if self.poisoned { Err(WalError::Poisoned) } else { Ok(()) } + } +} + +fn write_or_poison( + writer: &mut impl Write, + bytes: &[u8], + poisoned: &mut bool, +) -> std::io::Result<()> { + if let Err(error) = writer.write_all(bytes) { + *poisoned = true; + return Err(error); + } + Ok(()) +} + +fn validate_options(options: &WalOptions) -> Result<()> { + if options.segment_size < format::HEADER_LEN_U64 || options.max_size < format::HEADER_LEN_U64 { + Err(WalError::SegmentTooSmall { + segment_size: options.segment_size.min(options.max_size), + header_len: format::HEADER_LEN, + }) + } else { + Ok(()) + } +} + +fn absolute_path(path: &Path) -> Result { + if path.is_absolute() { + Ok(path.to_path_buf()) + } else { + std::env::current_dir() + .map(|current| current.join(path)) + .map_err(|source| WalError::io(PathBuf::from("."), source)) + } +} + +fn checked_add_len(offset: u64, len: usize) -> Result { + offset + .checked_add( + u64::try_from(len) + .map_err(|_| WalError::RecordTooLarge { len: u64::MAX, segment_size: u64::MAX })?, + ) + .ok_or(WalError::RecordTooLarge { len: u64::MAX, segment_size: u64::MAX }) +} + +#[cfg(test)] +mod tests { + use std::io::{self, Write}; + + use super::write_or_poison; + + struct PartialWriter(bool); + + impl Write for PartialWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.0 { + Err(io::Error::other("disk write failed")) + } else { + self.0 = true; + Ok(bytes.len().min(1)) + } + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + #[test] + fn invariant_partial_write_invalidates_handle() { + let mut writer = PartialWriter(false); + let mut poisoned = false; + + assert!(write_or_poison(&mut writer, &[1, 2], &mut poisoned).is_err()); + assert!(poisoned); + } +} From b5666358c845a24a3f7a46ffa698a1634c4938c1 Mon Sep 17 00:00:00 2001 From: Windsor Date: Sat, 9 May 2026 13:17:38 -0700 Subject: [PATCH 02/17] feat(node): add replicated KV service --- .gitignore | 4 + Cargo.lock | 711 +++++++++++++++++++ Cargo.toml | 8 +- cloud9-node/Cargo.toml | 5 + cloud9-node/src/lib.rs | 861 ++++++++++++++++++++++- cloud9-proto/Cargo.toml | 6 + cloud9-proto/build.rs | 8 + cloud9-proto/proto/cloud9/kv/v1/kv.proto | 101 +++ cloud9-proto/src/lib.rs | 5 + cloud9/Cargo.toml | 4 + cloud9/src/main.rs | 128 +++- jepsen/README.md | 53 ++ jepsen/project.clj | 10 + jepsen/scripts/build-target.sh | 9 + jepsen/src/cloud9/jepsen.clj | 341 +++++++++ 15 files changed, 2230 insertions(+), 24 deletions(-) create mode 100644 cloud9-proto/build.rs create mode 100644 cloud9-proto/proto/cloud9/kv/v1/kv.proto create mode 100644 jepsen/README.md create mode 100644 jepsen/project.clj create mode 100755 jepsen/scripts/build-target.sh create mode 100644 jepsen/src/cloud9/jepsen.clj diff --git a/.gitignore b/.gitignore index 42b37e2..85e14fe 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,10 @@ Thumbs.db # Node (commitlint) node_modules/ package-lock.json +jepsen/store/ +jepsen/target/ +jepsen/c9-linux-* +jepsen/.lein-failures # LLM tooling **/.agent/ diff --git a/Cargo.lock b/Cargo.lock index cb675d6..c1b9705 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,6 +61,12 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + [[package]] name = "arcstr" version = "1.2.0" @@ -76,12 +82,76 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfdc70193dadb9d7287fa4b633f15f90c876915b31f6af17da307fc59c9859a8" +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bit-set" version = "0.8.0" @@ -115,11 +185,59 @@ dependencies = [ "wyz", ] +[[package]] +name = "buffa" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a22fed63b429c4928fd5ed18dcacd3a98d0df1f06cd7cf651b2bbf3dbf0500" +dependencies = [ + "base64", + "bytes", + "hashbrown 0.15.5", + "once_cell", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "buffa-codegen" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd9774d97c281d07a61006775c675940371c170ae1b454de0e9c013e0acd19f6" +dependencies = [ + "buffa", + "buffa-descriptor", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "thiserror", +] + +[[package]] +name = "buffa-descriptor" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7b942f9242ea5a79c92486460073a0a36cccc314e69fbc2ff08f55467c47b37" +dependencies = [ + "buffa", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + [[package]] name = "bytes" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +dependencies = [ + "serde", +] [[package]] name = "cc" @@ -184,10 +302,14 @@ version = "0.0.1" dependencies = [ "clap", "cloud9-core", + "cloud9-node", "cloud9-proto", "cloud9-raft", "cloud9-storage", "miette", + "serde", + "tokio", + "toml", "tracing", "tracing-subscriber", ] @@ -210,11 +332,16 @@ dependencies = [ name = "cloud9-node" version = "0.0.1" dependencies = [ + "anyhow", + "axum", "cloud9-core", "cloud9-proto", "cloud9-raft", "cloud9-storage", + "connectrpc", "serde", + "serde_json", + "tokio", "tracing", ] @@ -222,7 +349,11 @@ dependencies = [ name = "cloud9-proto" version = "0.0.1" dependencies = [ + "buffa", "cloud9-core", + "connectrpc", + "connectrpc-build", + "http-body", "serde", ] @@ -286,6 +417,61 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "connectrpc" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cb80f9eb09b593a96880eb1c2588ac6319b129519de8c672f9b8b4ffdeaece" +dependencies = [ + "axum", + "base64", + "buffa", + "bytes", + "futures", + "http", + "http-body", + "http-body-util", + "percent-encoding", + "pin-project", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tower", + "tracing", + "wasm-bindgen-futures", +] + +[[package]] +name = "connectrpc-build" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec3203af15f7075997d16f8ea0a614bcad86879abe9f8ea70c4df69e74f3c14" +dependencies = [ + "anyhow", + "buffa", + "buffa-codegen", + "connectrpc-codegen", + "tempfile", +] + +[[package]] +name = "connectrpc-codegen" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77590b8d66a279f1741bc56bd358f82ce57f26f5a9576fbb8329bce30ea1e96e" +dependencies = [ + "anyhow", + "buffa", + "buffa-codegen", + "heck", + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "deranged" version = "0.5.5" @@ -295,6 +481,12 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -323,6 +515,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "fs-err" version = "3.1.3" @@ -339,6 +546,94 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + [[package]] name = "generator" version = "0.8.7" @@ -376,6 +671,41 @@ dependencies = [ "wasip2", ] +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", + "serde", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -388,6 +718,97 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + [[package]] name = "is_ci" version = "1.2.0" @@ -406,6 +827,18 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "js-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -452,6 +885,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.7.6" @@ -486,6 +925,23 @@ dependencies = [ "syn", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -534,6 +990,32 @@ version = "4.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf0d9e68100b3a7989b4901972f265cd542e560a3a8a724e1e20322f4d06ce9" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -555,6 +1037,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.103" @@ -799,6 +1291,38 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -834,6 +1358,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.1" @@ -846,6 +1386,16 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "strsim" version = "0.11.1" @@ -884,6 +1434,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "tap" version = "1.0.1" @@ -990,8 +1546,14 @@ version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" dependencies = [ + "bytes", + "libc", + "mio", "pin-project-lite", + "signal-hook-registry", + "socket2", "tokio-macros", + "windows-sys 0.61.2", ] [[package]] @@ -1005,12 +1567,94 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow 1.0.2", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -1133,6 +1777,61 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +dependencies = [ + "unicode-ident", +] + [[package]] name = "windows" version = "0.61.3" @@ -1333,6 +2032,18 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + +[[package]] +name = "winnow" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" + [[package]] name = "wit-bindgen" version = "0.46.0" diff --git a/Cargo.toml b/Cargo.toml index db94ad7..8972010 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,8 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "time"] } bytes = "1" serde = { version = "1", features = ["derive"] } -tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +serde_json = "1" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "signal", "sync", "time", "io-util"] } axum = "0.8.6" clap = { version = "4.5", features = ["derive", "env", "string", "wrap_help"] } arcstr = { version = "1.2", features = ["serde"] } @@ -66,6 +67,11 @@ fs-err = { version = "3", features = ["tokio"] } miette = { version = "7.2", features = ["fancy-no-backtrace"] } textwrap = "0.16" crc32fast = "1.5" +toml = "0.9" +buffa = { version = "0.5.2", features = ["json"] } +connectrpc = { version = "0.4.2", default-features = false } +connectrpc-build = "0.4.2" +http-body = "1" # Testing - concurrency loom = "0.7.2" diff --git a/cloud9-node/Cargo.toml b/cloud9-node/Cargo.toml index 7c49808..7a65a0f 100644 --- a/cloud9-node/Cargo.toml +++ b/cloud9-node/Cargo.toml @@ -12,9 +12,14 @@ homepage = { workspace = true } workspace = true [dependencies] +anyhow = { workspace = true } +axum = { workspace = true } cloud9-core = { workspace = true } cloud9-raft = { workspace = true } cloud9-storage = { workspace = true } cloud9-proto = { workspace = true } +connectrpc = { workspace = true, features = ["axum"] } tracing = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } diff --git a/cloud9-node/src/lib.rs b/cloud9-node/src/lib.rs index b402960..e9790e2 100644 --- a/cloud9-node/src/lib.rs +++ b/cloud9-node/src/lib.rs @@ -1,19 +1,868 @@ //! Top-level orchestration for Cloud9 nodes. -use cloud9_raft::ConsensusConfig; +use std::cmp::Ordering; +use std::collections::{BTreeMap, HashMap}; +use std::net::{Ipv4Addr, SocketAddr}; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use axum::Json; +use axum::Router as AxumRouter; +use axum::extract::State; +use axum::http::StatusCode; +use axum::routing::{get, post}; +use cloud9_proto::generated::cloud9::kv::v1::{ + DeleteResponse, GetResponse, HeadResponse, KvService, KvServiceExt, OwnedDeleteRequestView, + OwnedGetRequestView, OwnedHeadRequestView, OwnedPutRequestView, + OwnedRegisterSessionRequestView, OwnedStatusRequestView, PutResponse, RegisterSessionResponse, + StatusResponse, +}; +use cloud9_raft::raft::{Effects, Message}; +use cloud9_raft::{Command, ConsensusConfig, LogIndex, NodeId, ProposeError, RaftNode}; use cloud9_storage::StorageOptions; -use tracing::{info, instrument}; +use connectrpc::{ConnectError, RequestContext, Response, Router as ConnectRouter, ServiceResult}; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::{Mutex, RwLock, oneshot}; +use tokio::time::{Duration, sleep}; +use tracing::{info, instrument, warn}; /// Runtime configuration derived from CLI flags and config files. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct NodeConfig { + pub node_id: NodeId, + pub client_addr: SocketAddr, + pub raft_addr: SocketAddr, + pub peers: BTreeMap, pub storage: StorageOptions, pub consensus: ConsensusConfig, } -/// Launch the storage and consensus subsystems. +impl Default for NodeConfig { + fn default() -> Self { + let node_id = NodeId(0); + let raft_addr = SocketAddr::from((Ipv4Addr::LOCALHOST, 19_091)); + Self { + node_id, + client_addr: SocketAddr::from((Ipv4Addr::LOCALHOST, 19_090)), + raft_addr, + peers: BTreeMap::from([(node_id, raft_addr)]), + storage: StorageOptions::default(), + consensus: raft_config(node_id), + } + } +} + +pub fn raft_config(node_id: NodeId) -> ConsensusConfig { + ConsensusConfig::new(node_id).with_parallel_disk_write(false) +} + +#[derive(Clone)] +struct KvApi { + config: NodeConfig, + state: Arc>, + runtime: Arc, +} + +struct RaftRuntime { + config: NodeConfig, + node: Mutex, + state: Arc>, + waiters: Mutex>>>, +} + +impl RaftRuntime { + fn new(config: NodeConfig, state: Arc>) -> Self { + let voters = config.peers.keys().copied().collect::>(); + Self { + node: Mutex::new(RaftNode::new(config.consensus.clone(), &voters)), + config, + state, + waiters: Mutex::new(HashMap::new()), + } + } + + fn spawn(self: Arc) { + tokio::spawn(async move { + self.tick_loop().await; + }); + } + + async fn tick_loop(&self) { + loop { + sleep(Duration::from_millis(1)).await; + let mut node = self.node.lock().await; + let effects = node.tick(); + self.handle_effects(&mut node, effects).await; + } + } + + async fn step(&self, message: Message) { + let mut node = self.node.lock().await; + let effects = node.step(message); + self.handle_effects(&mut node, effects).await; + } + + async fn propose(&self, command: KvCommand) -> Result { + let bytes = serde_json::to_vec(&command) + .map_err(|_| ConnectError::internal("failed to encode Raft command"))?; + let receiver = { + let mut node = self.node.lock().await; + let (index, effects) = + node.propose(Command(bytes)).map_err(|error| propose_error(&error))?; + let (sender, receiver) = oneshot::channel(); + self.waiters.lock().await.insert(index, sender); + self.handle_effects(&mut node, effects).await; + receiver + }; + receiver.await.map_err(|_| ConnectError::aborted("Raft proposal was dropped"))? + } + + async fn read_barrier(&self) -> Result<(), ConnectError> { + match self.propose(KvCommand::ReadBarrier).await? { + KvApplyResult::ReadBarrier => Ok(()), + KvApplyResult::RegisterSession(_) + | KvApplyResult::Put(_) + | KvApplyResult::Delete(_) => Err(ConnectError::internal("Raft read barrier mismatch")), + } + } + + async fn mode(&self) -> String { + let node = self.node.lock().await; + if node.is_leader() { + "leader" + } else if node.is_candidate() { + "candidate" + } else if node.is_precandidate() { + "precandidate" + } else { + "follower" + } + .to_owned() + } + + async fn handle_effects(&self, node: &mut RaftNode, effects: Effects) { + if !effects.send_snapshots.is_empty() { + warn!( + snapshot_count = effects.send_snapshots.len(), + "Raft snapshot transport is not implemented" + ); + } + + for message in effects.messages { + self.send_message(message); + } + + self.apply_committed(node).await; + } + + async fn apply_committed(&self, node: &mut RaftNode) { + let entries = node.committed().collect::>(); + let mut applied_to = None; + for entry in entries { + let result = self.apply_command(&entry.command).await; + self.complete_waiter(entry.index, result).await; + applied_to = Some(entry.index); + } + if let Some(index) = applied_to { + node.advance(index); + } + } + + async fn apply_command(&self, command: &Command) -> Result { + let command = serde_json::from_slice(&command.0) + .map_err(|_| ConnectError::internal("invalid Raft command payload"))?; + self.state.write().await.apply(command) + } + + async fn complete_waiter(&self, index: LogIndex, result: Result) { + if let Some(sender) = self.waiters.lock().await.remove(&index) { + let _ = sender.send(result); + } + } + + fn send_message(&self, message: Message) { + let Some(addr) = self.config.peers.get(&message.to).copied() else { + warn!(to = message.to.0, "Raft message target is not in cluster config"); + return; + }; + tokio::spawn(async move { + if let Err(error) = post_raft_message(addr, &message).await { + warn!(%error, to = message.to.0, "failed to send Raft message"); + } + }); + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +enum KvCommand { + RegisterSession, + ReadBarrier, + Put { + client_id: u64, + sequence: u64, + namespace: String, + key: String, + body: Vec, + if_match: String, + if_none_match: bool, + }, + Delete { + client_id: u64, + sequence: u64, + namespace: String, + key: String, + if_match: String, + }, +} + +enum KvApplyResult { + RegisterSession(RegisterSessionResponse), + Put(PutResponse), + Delete(DeleteResponse), + ReadBarrier, +} + +#[derive(Default)] +struct KvState { + next_client_id: u64, + next_generation: u64, + entries: HashMap, + sessions: HashMap, +} + +impl KvState { + fn new() -> Self { + Self { next_client_id: 1, next_generation: 1, ..Self::default() } + } + + fn apply(&mut self, command: KvCommand) -> Result { + match command { + KvCommand::RegisterSession => { + let client_id = self.next_client_id()?; + Ok(KvApplyResult::RegisterSession(RegisterSessionResponse { + client_id, + ..Default::default() + })) + } + KvCommand::ReadBarrier => Ok(KvApplyResult::ReadBarrier), + KvCommand::Put { + client_id, + sequence, + namespace, + key, + body, + if_match, + if_none_match, + } => self.apply_put( + client_id, + sequence, + &namespace, + &key, + body, + &if_match, + if_none_match, + ), + KvCommand::Delete { client_id, sequence, namespace, key, if_match } => { + self.apply_delete(client_id, sequence, &namespace, &key, &if_match) + } + } + } + + fn apply_put( + &mut self, + client_id: u64, + sequence: u64, + namespace: &str, + key: &str, + body: Vec, + if_match: &str, + if_none_match: bool, + ) -> Result { + validate_mutation_request(client_id, sequence)?; + validate_put_preconditions(if_match, if_none_match)?; + + let name = KvName::new(namespace, key)?; + if let Some(response) = cached_put(self, client_id, sequence)? { + return Ok(KvApplyResult::Put(response)); + } + + let current = self.entries.get(&name); + check_put_preconditions(current, if_match, if_none_match)?; + + let generation = self.next_generation()?; + let etag = etag_for(generation); + let response = PutResponse { + namespace: name.namespace.clone(), + key: name.key.clone(), + etag: etag.clone(), + generation, + size: body_len(&body)?, + ..Default::default() + }; + self.entries.insert(name, KvRecord { body, etag, generation }); + self.session_mut(client_id)?.record(sequence, MutationResult::Put(response.clone())); + Ok(KvApplyResult::Put(response)) + } + + fn apply_delete( + &mut self, + client_id: u64, + sequence: u64, + namespace: &str, + key: &str, + if_match: &str, + ) -> Result { + validate_mutation_request(client_id, sequence)?; + + let name = KvName::new(namespace, key)?; + if let Some(response) = cached_delete(self, client_id, sequence)? { + return Ok(KvApplyResult::Delete(response)); + } + + let removed = if let Some(record) = self.entries.get(&name) { + if !if_match.is_empty() && if_match != record.etag { + return Err(ConnectError::failed_precondition("ETag precondition failed")); + } + self.entries.remove(&name) + } else { + if !if_match.is_empty() { + return Err(ConnectError::failed_precondition("ETag precondition failed")); + } + None + }; + + let response = if let Some(record) = removed { + DeleteResponse { + namespace: name.namespace, + key: name.key, + etag: record.etag, + generation: record.generation, + deleted: true, + ..Default::default() + } + } else { + DeleteResponse { + namespace: name.namespace, + key: name.key, + etag: String::new(), + generation: 0, + deleted: false, + ..Default::default() + } + }; + self.session_mut(client_id)?.record(sequence, MutationResult::Delete(response.clone())); + Ok(KvApplyResult::Delete(response)) + } + + fn next_client_id(&mut self) -> Result { + let client_id = self.next_client_id; + self.next_client_id = self + .next_client_id + .checked_add(1) + .ok_or_else(|| ConnectError::resource_exhausted("client id space exhausted"))?; + self.sessions.insert(client_id, SessionState::default()); + Ok(client_id) + } + + fn next_generation(&mut self) -> Result { + let generation = self.next_generation; + self.next_generation = self + .next_generation + .checked_add(1) + .ok_or_else(|| ConnectError::resource_exhausted("kv generation space exhausted"))?; + Ok(generation) + } + + fn session(&self, client_id: u64) -> Result<&SessionState, ConnectError> { + self.sessions + .get(&client_id) + .ok_or_else(|| ConnectError::invalid_argument("unknown client session")) + } + + fn session_mut(&mut self, client_id: u64) -> Result<&mut SessionState, ConnectError> { + self.sessions + .get_mut(&client_id) + .ok_or_else(|| ConnectError::invalid_argument("unknown client session")) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct KvName { + namespace: String, + key: String, +} + +impl KvName { + fn new(namespace: &str, key: &str) -> Result { + if namespace.is_empty() { + return Err(ConnectError::invalid_argument("namespace must not be empty")); + } + if key.is_empty() { + return Err(ConnectError::invalid_argument("key must not be empty")); + } + Ok(Self { namespace: namespace.to_owned(), key: key.to_owned() }) + } +} + +#[derive(Clone)] +struct KvRecord { + body: Vec, + etag: String, + generation: u64, +} + +#[derive(Clone, Default)] +struct SessionState { + last_sequence: u64, + last_result: Option, +} + +#[derive(Clone)] +enum MutationResult { + Put(PutResponse), + Delete(DeleteResponse), +} + +/// Launch the node's public KV API and Raft peer API. #[instrument(skip_all)] -pub async fn launch(config: NodeConfig) { +pub async fn launch(config: NodeConfig) -> Result<()> { info!(?config.storage, "initializing storage"); - info!(?config.consensus, "consensus subsystem ready"); + info!(?config.consensus, "consensus subsystem configured"); + + let state = Arc::new(RwLock::new(KvState::new())); + let runtime = Arc::new(RaftRuntime::new(config.clone(), state.clone())); + let api = Arc::new(KvApi { config: config.clone(), state, runtime: runtime.clone() }); + let client_app = kv_app(api); + let raft_app = raft_app(runtime.clone()); + let client_listener = TcpListener::bind(config.client_addr) + .await + .with_context(|| format!("binding Cloud9 KV API to {}", config.client_addr))?; + let raft_listener = TcpListener::bind(config.raft_addr) + .await + .with_context(|| format!("binding Cloud9 Raft API to {}", config.raft_addr))?; + + info!( + node_id = config.node_id.0, + client_addr = %config.client_addr, + raft_addr = %config.raft_addr, + peer_count = config.peers.len(), + "serving Cloud9 KV API" + ); + + runtime.spawn(); + tokio::try_join!( + axum::serve(client_listener, client_app), + axum::serve(raft_listener, raft_app), + ) + .context("serving Cloud9 node")?; + Ok(()) +} + +fn kv_app(api: Arc) -> AxumRouter { + let connect = api.register(ConnectRouter::new()); + AxumRouter::new() + .route("/healthz", get(|| async { "ok" })) + .fallback_service(connect.into_axum_service()) +} + +fn raft_app(runtime: Arc) -> AxumRouter { + AxumRouter::new().route("/raft/message", post(receive_raft)).with_state(runtime) +} + +async fn receive_raft( + State(runtime): State>, + Json(message): Json, +) -> StatusCode { + runtime.step(message).await; + StatusCode::NO_CONTENT +} + +#[allow(refining_impl_trait)] +impl KvService for KvApi { + async fn register_session( + &self, + _ctx: RequestContext, + _request: OwnedRegisterSessionRequestView, + ) -> ServiceResult { + match self.runtime.propose(KvCommand::RegisterSession).await? { + KvApplyResult::RegisterSession(response) => Response::ok(response), + KvApplyResult::Put(_) | KvApplyResult::Delete(_) | KvApplyResult::ReadBarrier => { + Err(ConnectError::internal("Raft session command mismatch")) + } + } + } + + async fn head( + &self, + _ctx: RequestContext, + request: OwnedHeadRequestView, + ) -> ServiceResult { + let name = KvName::new(request.namespace, request.key)?; + self.runtime.read_barrier().await?; + let state = self.state.read().await; + let record = state.entries.get(&name).ok_or_else(key_not_found)?; + Response::ok(HeadResponse { + namespace: name.namespace, + key: name.key, + etag: record.etag.clone(), + generation: record.generation, + size: body_len(&record.body)?, + ..Default::default() + }) + } + + async fn get( + &self, + _ctx: RequestContext, + request: OwnedGetRequestView, + ) -> ServiceResult { + let name = KvName::new(request.namespace, request.key)?; + self.runtime.read_barrier().await?; + let state = self.state.read().await; + let record = state.entries.get(&name).ok_or_else(key_not_found)?; + Response::ok(GetResponse { + namespace: name.namespace, + key: name.key, + etag: record.etag.clone(), + generation: record.generation, + size: body_len(&record.body)?, + body: record.body.clone(), + ..Default::default() + }) + } + + async fn put( + &self, + _ctx: RequestContext, + request: OwnedPutRequestView, + ) -> ServiceResult { + validate_mutation_request(request.client_id, request.sequence)?; + validate_put_preconditions(request.if_match, request.if_none_match)?; + KvName::new(request.namespace, request.key)?; + + let command = KvCommand::Put { + client_id: request.client_id, + sequence: request.sequence, + namespace: request.namespace.to_owned(), + key: request.key.to_owned(), + body: request.body.to_vec(), + if_match: request.if_match.to_owned(), + if_none_match: request.if_none_match, + }; + match self.runtime.propose(command).await? { + KvApplyResult::Put(response) => Response::ok(response), + KvApplyResult::RegisterSession(_) + | KvApplyResult::Delete(_) + | KvApplyResult::ReadBarrier => { + Err(ConnectError::internal("Raft put command mismatch")) + } + } + } + + async fn delete( + &self, + _ctx: RequestContext, + request: OwnedDeleteRequestView, + ) -> ServiceResult { + validate_mutation_request(request.client_id, request.sequence)?; + KvName::new(request.namespace, request.key)?; + + let command = KvCommand::Delete { + client_id: request.client_id, + sequence: request.sequence, + namespace: request.namespace.to_owned(), + key: request.key.to_owned(), + if_match: request.if_match.to_owned(), + }; + match self.runtime.propose(command).await? { + KvApplyResult::Delete(response) => Response::ok(response), + KvApplyResult::RegisterSession(_) + | KvApplyResult::Put(_) + | KvApplyResult::ReadBarrier => { + Err(ConnectError::internal("Raft delete command mismatch")) + } + } + } + + async fn status( + &self, + _ctx: RequestContext, + _request: OwnedStatusRequestView, + ) -> ServiceResult { + let mode = self.runtime.mode().await; + let state = self.state.read().await; + Response::ok(StatusResponse { + node_id: self.config.node_id.0, + mode, + key_count: usize_to_u64(state.entries.len()), + ..Default::default() + }) + } +} + +impl SessionState { + fn record(&mut self, sequence: u64, result: MutationResult) { + self.last_sequence = sequence; + self.last_result = Some(result); + } +} + +fn validate_mutation_request(client_id: u64, sequence: u64) -> Result<(), ConnectError> { + if client_id == 0 { + return Err(ConnectError::invalid_argument("client_id must be registered")); + } + if sequence == 0 { + return Err(ConnectError::invalid_argument("sequence must be positive")); + } + Ok(()) +} + +fn validate_put_preconditions(if_match: &str, if_none_match: bool) -> Result<(), ConnectError> { + if !if_match.is_empty() && if_none_match { + return Err(ConnectError::invalid_argument( + "if_match and if_none_match are mutually exclusive", + )); + } + Ok(()) +} + +fn check_put_preconditions( + current: Option<&KvRecord>, + if_match: &str, + if_none_match: bool, +) -> Result<(), ConnectError> { + if if_none_match && current.is_some() { + return Err(ConnectError::failed_precondition("key already exists")); + } + + if !if_match.is_empty() { + match current { + Some(record) if record.etag == if_match => {} + Some(_) | None => { + return Err(ConnectError::failed_precondition("ETag precondition failed")); + } + } + } + + Ok(()) +} + +fn cached_put( + state: &KvState, + client_id: u64, + sequence: u64, +) -> Result, ConnectError> { + match cached_mutation(state.session(client_id)?, sequence)? { + Some(MutationResult::Put(response)) => Ok(Some(response)), + Some(MutationResult::Delete(_)) => { + Err(ConnectError::aborted("sequence reused for different operation")) + } + None => Ok(None), + } +} + +fn cached_delete( + state: &KvState, + client_id: u64, + sequence: u64, +) -> Result, ConnectError> { + match cached_mutation(state.session(client_id)?, sequence)? { + Some(MutationResult::Delete(response)) => Ok(Some(response)), + Some(MutationResult::Put(_)) => { + Err(ConnectError::aborted("sequence reused for different operation")) + } + None => Ok(None), + } +} + +fn cached_mutation( + session: &SessionState, + sequence: u64, +) -> Result, ConnectError> { + match sequence.cmp(&session.last_sequence) { + Ordering::Less => Err(ConnectError::aborted("stale client sequence")), + Ordering::Equal => Ok(session.last_result.clone()), + Ordering::Greater => Ok(None), + } +} + +fn etag_for(generation: u64) -> String { + format!("\"c9-{generation}\"") +} + +fn body_len(body: &[u8]) -> Result { + u64::try_from(body.len()).map_err(|_| ConnectError::resource_exhausted("value too large")) +} + +fn usize_to_u64(value: usize) -> u64 { + u64::try_from(value).unwrap_or(u64::MAX) +} + +fn key_not_found() -> ConnectError { + ConnectError::not_found("key not found") +} + +fn propose_error(error: &ProposeError) -> ConnectError { + match error { + ProposeError::NotLeader { leader_hint: Some(leader) } => { + ConnectError::failed_precondition(format!("not leader; leader is {}", leader.0)) + } + ProposeError::NotLeader { leader_hint: None } => { + ConnectError::failed_precondition("not leader; leader unknown") + } + ProposeError::Throttled => ConnectError::resource_exhausted("too many Raft proposals"), + } +} + +async fn post_raft_message(addr: SocketAddr, message: &Message) -> Result<()> { + let body = serde_json::to_vec(message).context("encoding Raft message")?; + let mut stream = TcpStream::connect(addr) + .await + .with_context(|| format!("connecting to Raft peer {addr}"))?; + let request = format!( + "POST /raft/message HTTP/1.1\r\n\ + Host: {addr}\r\n\ + Content-Type: application/json\r\n\ + Content-Length: {}\r\n\ + Connection: close\r\n\ + \r\n", + body.len() + ); + stream.write_all(request.as_bytes()).await.context("writing Raft message headers")?; + stream.write_all(&body).await.context("writing Raft message body")?; + + let mut response = Vec::new(); + stream.read_to_end(&mut response).await.context("reading Raft message response")?; + if response.starts_with(b"HTTP/1.1 204") || response.starts_with(b"HTTP/1.1 200") { + return Ok(()); + } + + let response = String::from_utf8_lossy(&response); + anyhow::bail!("Raft peer {addr} rejected message: {response}"); +} + +#[cfg(test)] +mod tests { + use std::io::{Read, Write}; + use std::time::Duration; + + use anyhow::bail; + + use super::*; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn kv_api_enforces_etag_preconditions() -> Result<()> { + let config = NodeConfig::default(); + let state = Arc::new(RwLock::new(KvState::new())); + let runtime = Arc::new(RaftRuntime::new(config.clone(), state.clone())); + let api = Arc::new(KvApi { config: config.clone(), state, runtime: runtime.clone() }); + let listener = TcpListener::bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0))).await?; + let addr = listener.local_addr()?; + let server = tokio::spawn(axum::serve(listener, kv_app(api)).into_future()); + runtime.spawn(); + wait_for_leader(addr).await?; + + let (status, body) = post_json(addr, "RegisterSession", "{}")?; + assert_eq!(200, status); + assert!(body.contains("\"clientId\":\"1\"")); + + let (status, _) = post_json( + addr, + "Put", + r#"{"clientId":"1","sequence":"1","namespace":"jepsen","key":"register","body":"MQ==","ifNoneMatch":true}"#, + )?; + assert_eq!(200, status); + + let (status, _) = post_json( + addr, + "Put", + r#"{"clientId":"1","sequence":"2","namespace":"jepsen","key":"register","body":"Mg==","ifNoneMatch":true}"#, + )?; + assert_eq!(400, status); + + let (status, _) = post_json( + addr, + "Put", + r#"{"clientId":"1","sequence":"3","namespace":"jepsen","key":"register","body":"Mg==","ifMatch":"\"c9-1\""}"#, + )?; + assert_eq!(200, status); + + let (status, body) = post_json(addr, "Get", r#"{"namespace":"jepsen","key":"register"}"#)?; + assert_eq!(200, status); + assert!(body.contains("\"body\":\"Mg==\"")); + + let (status, _) = post_json( + addr, + "Put", + r#"{"clientId":"1","sequence":"4","namespace":"jepsen","key":"register","body":"Mw==","ifMatch":"\"c9-1\""}"#, + )?; + assert_eq!(400, status); + + server.abort(); + Ok(()) + } + + async fn wait_for_leader(addr: SocketAddr) -> Result<()> { + for _ in 0..100 { + let (status, body) = post_json(addr, "Status", "{}")?; + if status == 200 && body.contains("\"mode\":\"leader\"") { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + bail!("single-node Raft runtime did not elect a leader"); + } + + fn post_json(addr: SocketAddr, method: &str, body: &str) -> Result<(u16, String)> { + let mut stream = std::net::TcpStream::connect(addr)?; + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + let request = format!( + "POST /cloud9.kv.v1.KvService/{method} HTTP/1.1\r\n\ + Host: {addr}\r\n\ + Content-Type: application/json\r\n\ + Content-Length: {}\r\n\ + Connection: close\r\n\ + \r\n\ + {body}", + body.len() + ); + stream.write_all(request.as_bytes())?; + + let mut response = String::new(); + stream.read_to_string(&mut response)?; + let Some((head, body)) = response.split_once("\r\n\r\n") else { + bail!("HTTP response missing header separator"); + }; + let Some(status) = head.lines().next().and_then(|line| line.split_whitespace().nth(1)) + else { + bail!("HTTP response missing status"); + }; + let body = if head.to_ascii_lowercase().contains("transfer-encoding: chunked") { + decode_chunked(body)? + } else { + body.to_owned() + }; + + Ok((status.parse()?, body)) + } + + fn decode_chunked(mut body: &str) -> Result { + let mut decoded = String::new(); + loop { + let Some((len, rest)) = body.split_once("\r\n") else { + bail!("chunk missing length"); + }; + let len = usize::from_str_radix(len.trim(), 16)?; + if len == 0 { + return Ok(decoded); + } + if rest.len() < len + 2 { + bail!("chunk shorter than declared length"); + } + decoded.push_str(&rest[..len]); + body = &rest[len + 2..]; + } + } } diff --git a/cloud9-proto/Cargo.toml b/cloud9-proto/Cargo.toml index 4c40219..9178683 100644 --- a/cloud9-proto/Cargo.toml +++ b/cloud9-proto/Cargo.toml @@ -14,3 +14,9 @@ workspace = true [dependencies] cloud9-core = { workspace = true } serde = { workspace = true } +buffa = { workspace = true } +connectrpc = { workspace = true } +http-body = { workspace = true } + +[build-dependencies] +connectrpc-build = { workspace = true } diff --git a/cloud9-proto/build.rs b/cloud9-proto/build.rs new file mode 100644 index 0000000..525e180 --- /dev/null +++ b/cloud9-proto/build.rs @@ -0,0 +1,8 @@ +fn main() -> Result<(), Box> { + connectrpc_build::Config::new() + .files(&["proto/cloud9/kv/v1/kv.proto"]) + .includes(&["proto"]) + .include_file("_cloud9_connect.rs") + .compile()?; + Ok(()) +} diff --git a/cloud9-proto/proto/cloud9/kv/v1/kv.proto b/cloud9-proto/proto/cloud9/kv/v1/kv.proto new file mode 100644 index 0000000..7fd5604 --- /dev/null +++ b/cloud9-proto/proto/cloud9/kv/v1/kv.proto @@ -0,0 +1,101 @@ +syntax = "proto3"; + +package cloud9.kv.v1; + +// KvService is Cloud9's namespace/key API. Cloud9 is a relational database with +// a native KV front door; SQL and KV operations should lower into the same +// MVCC/TxIR/transaction-coordinator path. +service KvService { + // RegisterSession allocates a client id for idempotent mutating requests. + rpc RegisterSession(RegisterSessionRequest) returns (RegisterSessionResponse); + + // Head returns value metadata without the value body. + rpc Head(HeadRequest) returns (HeadResponse); + + // Get returns value metadata and body. + rpc Get(GetRequest) returns (GetResponse); + + // Put creates or replaces a value, subject to optional ETag preconditions. + rpc Put(PutRequest) returns (PutResponse); + + // Delete deletes a value, subject to optional ETag preconditions. + rpc Delete(DeleteRequest) returns (DeleteResponse); + + // Status returns node-local service status. + rpc Status(StatusRequest) returns (StatusResponse); +} + +message RegisterSessionRequest {} + +message RegisterSessionResponse { + uint64 client_id = 1; +} + +message HeadRequest { + string namespace = 1; + string key = 2; +} + +message HeadResponse { + string namespace = 1; + string key = 2; + string etag = 3; + uint64 generation = 4; + uint64 size = 5; +} + +message GetRequest { + string namespace = 1; + string key = 2; +} + +message GetResponse { + string namespace = 1; + string key = 2; + string etag = 3; + uint64 generation = 4; + uint64 size = 5; + bytes body = 6; +} + +message PutRequest { + uint64 client_id = 1; + uint64 sequence = 2; + string namespace = 3; + string key = 4; + bytes body = 5; + string if_match = 6; + bool if_none_match = 7; +} + +message PutResponse { + string namespace = 1; + string key = 2; + string etag = 3; + uint64 generation = 4; + uint64 size = 5; +} + +message DeleteRequest { + uint64 client_id = 1; + uint64 sequence = 2; + string namespace = 3; + string key = 4; + string if_match = 5; +} + +message DeleteResponse { + string namespace = 1; + string key = 2; + string etag = 3; + uint64 generation = 4; + bool deleted = 5; +} + +message StatusRequest {} + +message StatusResponse { + uint64 node_id = 1; + string mode = 2; + uint64 key_count = 3; +} diff --git a/cloud9-proto/src/lib.rs b/cloud9-proto/src/lib.rs index 4f1f27f..3caca69 100644 --- a/cloud9-proto/src/lib.rs +++ b/cloud9-proto/src/lib.rs @@ -3,6 +3,11 @@ use cloud9_core::SharedString; use serde::{Deserialize, Serialize}; +#[allow(warnings)] +pub mod generated { + connectrpc::include_generated!("_cloud9_connect.rs"); +} + /// Identifies a tenant within the global system. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] pub struct TenantId(pub SharedString); diff --git a/cloud9/Cargo.toml b/cloud9/Cargo.toml index be78566..a3921ca 100644 --- a/cloud9/Cargo.toml +++ b/cloud9/Cargo.toml @@ -18,6 +18,7 @@ workspace = true [dependencies] cloud9-core = { workspace = true } +cloud9-node = { workspace = true } cloud9-storage = { workspace = true } cloud9-raft = { workspace = true } cloud9-proto = { workspace = true } @@ -25,3 +26,6 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } clap = { workspace = true } miette = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +toml = { workspace = true } diff --git a/cloud9/src/main.rs b/cloud9/src/main.rs index 81ba0de..e0cb89d 100644 --- a/cloud9/src/main.rs +++ b/cloud9/src/main.rs @@ -1,8 +1,14 @@ -use std::path::PathBuf; +use std::collections::BTreeMap; +use std::net::{Ipv4Addr, SocketAddr, ToSocketAddrs}; +use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; -use cloud9_core::{fs, install_diagnostics}; +use cloud9_core::{SharedString, fs, install_diagnostics}; +use cloud9_node::{NodeConfig, raft_config}; +use cloud9_raft::NodeId; +use cloud9_storage::StorageOptions; use miette::{Context, IntoDiagnostic, Result}; +use serde::Deserialize; use tracing_subscriber::prelude::*; use tracing_subscriber::{EnvFilter, fmt}; @@ -38,37 +44,32 @@ enum Command { }, } -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { install_diagnostics()?; let cli = Cli::parse(); init_tracing(cli.verbose, !cli.no_color)?; + if cli.no_progress { + tracing::debug!("progress disabled for this run"); + } match cli.command { Command::Start { config } => { let config_path = config.unwrap_or_else(|| PathBuf::from("cloud9.toml")); - let maybe_config = load_config(&config_path)?; + let config = load_node_config(&config_path)?; tracing::info!(path = %config_path.display(), "booting node"); - if let Some(config) = maybe_config { - tracing::debug!(contents = %config, "loaded configuration"); - } else { + if !config_path.exists() { tracing::warn!(path = %config_path.display(), "using defaults; config missing"); } + cloud9_node::launch(config).await.map_err(|error| miette::miette!("{error:#}"))?; } Command::CheckConfig { config } => { - load_config(&config) - .and_then(|contents| { - contents.ok_or_else(|| miette::miette!("config `{}` missing", config.display())) - }) - .context("configuration check failed")?; + load_required_node_config(&config).context("configuration check failed")?; tracing::info!(path = %config.display(), "configuration OK"); } } - if cli.no_progress { - tracing::debug!("progress disabled for this run"); - } - Ok(()) } @@ -89,7 +90,50 @@ fn init_tracing(verbosity: u8, color_enabled: bool) -> Result<()> { tracing_subscriber::registry().with(filter).with(fmt_layer).try_init().into_diagnostic() } -fn load_config(path: &PathBuf) -> Result> { +#[derive(Debug, Deserialize)] +struct ConfigFile { + node: NodeSection, + storage: StorageSection, + cluster: ClusterSection, +} + +#[derive(Debug, Deserialize)] +struct NodeSection { + id: u64, + #[serde(rename = "host")] + _host: String, + client_port: u16, + raft_port: u16, +} + +#[derive(Debug, Deserialize)] +struct StorageSection { + data_dir: String, +} + +#[derive(Debug, Deserialize)] +struct ClusterSection { + peers: Vec, +} + +#[derive(Debug, Deserialize)] +struct PeerSection { + id: u64, + host: String, + raft_port: u16, +} + +fn load_node_config(path: &Path) -> Result { + if path.exists() { load_required_node_config(path) } else { Ok(NodeConfig::default()) } +} + +fn load_required_node_config(path: &Path) -> Result { + let contents = + load_config(path)?.ok_or_else(|| miette::miette!("config `{}` missing", path.display()))?; + parse_node_config(&contents).with_context(|| format!("parsing `{}`", path.display())) +} + +fn load_config(path: &Path) -> Result> { if path.exists() { fs::read_to_string(path) .into_diagnostic() @@ -99,3 +143,53 @@ fn load_config(path: &PathBuf) -> Result> { Ok(None) } } + +fn parse_node_config(contents: &str) -> Result { + let config: ConfigFile = toml::from_str(contents).into_diagnostic()?; + let node_id = NodeId(config.node.id); + let client_addr = bind_addr(config.node.client_port); + let raft_addr = bind_addr(config.node.raft_port); + let peers = peer_addrs(&config.cluster.peers)?; + if !peers.contains_key(&node_id) { + return Err(miette::miette!("cluster.peers must include node.id {}", node_id.0)); + } + + Ok(NodeConfig { + node_id, + client_addr, + raft_addr, + peers, + storage: StorageOptions { + name: SharedString::from("default"), + data_dir: SharedString::from(config.storage.data_dir), + }, + consensus: raft_config(node_id), + }) +} + +fn bind_addr(port: u16) -> SocketAddr { + SocketAddr::from((Ipv4Addr::UNSPECIFIED, port)) +} + +fn peer_addrs(peers: &[PeerSection]) -> Result> { + peers + .iter() + .map(|peer| Ok((NodeId(peer.id), resolve_peer_addr(&peer.host, peer.raft_port)?))) + .collect() +} + +fn resolve_peer_addr(host: &str, port: u16) -> Result { + let addrs = (host, port) + .to_socket_addrs() + .into_diagnostic() + .with_context(|| format!("resolving peer address `{host}:{port}`"))? + .collect::>(); + match addrs.as_slice() { + [addr] => Ok(*addr), + [] => Err(miette::miette!("peer address `{host}:{port}` resolved to no addresses")), + _ => Err(miette::miette!( + "peer address `{host}:{port}` resolved ambiguously to {} addresses", + addrs.len() + )), + } +} diff --git a/jepsen/README.md b/jepsen/README.md new file mode 100644 index 0000000..2a19a87 --- /dev/null +++ b/jepsen/README.md @@ -0,0 +1,53 @@ +# Cloud9 Jepsen + +This harness is a Jepsen `db/DB` wrapper for the real Cloud9 `c9` binary. It +uploads `target/release/c9` to every DB node, writes a per-node `cloud9.toml`, +starts `c9 start --config /opt/cloud9/cloud9.toml` with Jepsen's +`start-daemon!`, and drives Cloud9's public KV API with a shared +linearizable register workload. + +Cloud9 is a relational database first: Postgres-compatible SQL and native KV are +peer APIs over one MVCC storage layer, one transactional IR, one timestamp +system, and one transaction coordinator. The KV workload here is the smallest +front door Jepsen can drive today, not a separate product direction. + +The workload maps one shared register to `namespace/key`, writes JSON values as +value bodies, and implements CAS with S3-style ETag preconditions. This KV +surface is only one Cloud9 API front door; SQL and KV are intended to lower into +the same transactional IR. + +## Build + +```bash +./jepsen/scripts/build-target.sh +``` + +## Run + +From `jepsen/`, with SSH-reachable Jepsen DB nodes: + +```bash +lein run test \ + --nodes-file ~/nodes \ + --username root \ + --time-limit 60 \ + --concurrency 5n \ + --stagger 0.01 \ + --binary ../target/release/c9 +``` + +Useful knobs: + +```bash +lein run test --help +lein run serve +``` + +The harness discovers the current Raft leader before opening client sessions. +Followers reject mutating KV RPCs rather than serving node-local state. + +## Current Limit + +`c9 start` now drives KV commands through Raft, but this is still a transient +runtime: Raft persistence, snapshot transfer, read forwarding, failover-aware +clients, and nemesis coverage are intentionally not complete yet. diff --git a/jepsen/project.clj b/jepsen/project.clj new file mode 100644 index 0000000..20f4a2b --- /dev/null +++ b/jepsen/project.clj @@ -0,0 +1,10 @@ +(defproject cloud9-jepsen "0.0.1-SNAPSHOT" + :description "Jepsen tests for Cloud9 Raft" + :license {:name "MIT" + :url "https://opensource.org/licenses/MIT"} + :main cloud9.jepsen + :dependencies [[org.clojure/clojure "1.12.4"] + [jepsen "0.3.11"] + [cheshire "6.1.0"] + [http-kit "2.8.1"] + [org.clj-commons/slingshot "0.13.0"]]) diff --git a/jepsen/scripts/build-target.sh b/jepsen/scripts/build-target.sh new file mode 100755 index 0000000..2a7515e --- /dev/null +++ b/jepsen/scripts/build-target.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repo" + +cargo build --release -p cloud9 --bin c9 --locked + +printf ' ok c9 (%s)\n' "$repo/target/release/c9" diff --git a/jepsen/src/cloud9/jepsen.clj b/jepsen/src/cloud9/jepsen.clj new file mode 100644 index 0000000..78163fe --- /dev/null +++ b/jepsen/src/cloud9/jepsen.clj @@ -0,0 +1,341 @@ +(ns cloud9.jepsen + (:require [cheshire.core :as json] + [clj-commons.slingshot :refer [throw+ try+]] + [clojure.java.io :as io] + [clojure.string :as str] + [clojure.tools.logging :refer [info]] + [jepsen [cli :as cli] + [client :as client] + [checker :as checker] + [control :as c] + [db :as db] + [generator :as gen] + [independent :as independent] + [random :as rand] + [tests :as tests]] + [jepsen.control.util :as cu] + [jepsen.checker.timeline :as timeline] + [jepsen.os.debian :as debian] + [knossos.model :as model] + [org.httpkit.client :as http]) + (:import (java.nio.charset StandardCharsets) + (java.util Base64)) + (:gen-class)) + +(def dir "/opt/cloud9") +(def upload-path "/tmp/c9") +(def binary (str dir "/c9")) +(def config-file (str dir "/cloud9.toml")) +(def data-dir (str dir "/data")) +(def logfile (str dir "/cloud9.log")) +(def pidfile (str dir "/cloud9.pid")) +(def kv-service "cloud9.kv.v1.KvService") +(def kv-namespace "jepsen") + +(defn canonical-path + [path] + (.getCanonicalPath (io/file path))) + +(defn node-id + [test node] + (get (zipmap (:nodes test) (range)) node)) + +(defn peer-config + [test] + (->> (:nodes test) + (map (fn [node] + (str " { id = " (node-id test node) + ", host = \"" node "\"" + ", raft_port = " (:raft-port test) + " }"))) + (str/join ",\n"))) + +(defn node-config + [test node] + (str "# Generated by Cloud9 Jepsen.\n" + "[node]\n" + "id = " (node-id test node) "\n" + "host = \"" node "\"\n" + "client_port = " (:client-port test) "\n" + "raft_port = " (:raft-port test) "\n" + "\n" + "[storage]\n" + "data_dir = \"" data-dir "\"\n" + "\n" + "[cluster]\n" + "peers = [\n" + (peer-config test) + "\n]\n")) + +(defn install-node! + [test node] + (c/upload (:binary test) upload-path) + (c/su + (c/exec :rm :-rf dir) + (c/exec :mkdir :-p dir data-dir) + (c/exec :mv upload-path binary) + (c/exec :chmod :+x binary) + (c/exec :printf "%s" (node-config test node) :> config-file))) + +(defn ensure-running! + [node] + (Thread/sleep 1000) + (when-not (cu/daemon-running? pidfile) + (throw+ {:type ::cloud9-exited + :node node + :pidfile pidfile + :logfile logfile + :message "cloud9 exited after start; c9 start must run a long-lived DB service"}))) + +(defn start-node! + [test node] + (c/su + (cu/start-daemon! + {:logfile logfile + :pidfile pidfile + :chdir dir} + binary + "--no-color" + "--no-progress" + "start" + "--config" + config-file)) + (ensure-running! node)) + +(defn stop-node! + [] + (c/su + (cu/stop-daemon! pidfile) + (c/exec :rm :-rf dir upload-path))) + +(defn cloud9-db + [] + (reify + db/DB + (setup! [_ test node] + (info node "installing cloud9") + (install-node! test node) + (start-node! test node)) + + (teardown! [_ _test _node] + (stop-node!)) + + db/Kill + (kill! [_ _test _node] + (c/su (cu/stop-daemon! pidfile))) + + (start! [_ test node] + (start-node! test node)) + + db/LogFiles + (log-files [_ _test _node] + {logfile "cloud9.log" + config-file "cloud9.toml"}))) + +(defn rpc-url + [test node method] + (str "http://" node ":" (:client-port test) "/" kv-service "/" method)) + +(defn rpc! + [test node method body] + (let [{:keys [status body error]} @(http/post (rpc-url test node method) + {:headers {"content-type" "application/json"} + :body (json/generate-string body) + :connection-timeout 5000 + :socket-timeout 5000}) + decoded (when-not (str/blank? body) + (json/parse-string body true))] + (when error + (throw+ {:type ::rpc-error + :node node + :method method + :error error})) + (if (<= 200 status 299) + decoded + (throw+ {:type ::rpc-error + :node node + :method method + :status status + :body decoded})))) + +(defn encode-value + [value] + (.encodeToString (Base64/getEncoder) + (.getBytes (json/generate-string value) + StandardCharsets/UTF_8))) + +(defn decode-value + [^String body] + (let [bytes (.decode ^java.util.Base64$Decoder (Base64/getDecoder) body)] + (json/parse-string (String. ^bytes bytes StandardCharsets/UTF_8) + true))) + +(defn kv-key + [k] + (str "register/" k)) + +(defn register-write + [_ _] + {:type :invoke + :f :write + :value (independent/tuple 0 (rand/long 5))}) + +(defn register-read + [_ _] + {:type :invoke + :f :read + :value (independent/tuple 0 nil)}) + +(defn register-cas + [_ _] + {:type :invoke + :f :cas + :value (independent/tuple 0 [(rand/long 5) (rand/long 5)])}) + +(defn next-sequence! + [sequence] + (str (swap! sequence inc))) + +(defn register-session! + [test node] + (:clientId (rpc! test node "RegisterSession" {}))) + +(defn status! + [test node] + (rpc! test node "Status" {})) + +(defn current-leader + [test] + (some (fn [node] + (try+ + (when (= "leader" (:mode (status! test node))) + node) + (catch [:type ::rpc-error] _ + nil))) + (:nodes test))) + +(defn await-leader! + [test] + (loop [attempts 100] + (if-let [leader (current-leader test)] + leader + (if (pos? attempts) + (do + (Thread/sleep 100) + (recur (dec attempts))) + (throw+ {:type ::no-leader + :message "no Cloud9 Raft leader elected"}))))) + +(defn get-value! + [test node k] + (rpc! test node "Get" {:namespace kv-namespace + :key (kv-key k)})) + +(defn put-value! + [test node session sequence k value preconditions] + (rpc! test node "Put" (merge {:clientId session + :sequence (next-sequence! sequence) + :namespace kv-namespace + :key (kv-key k) + :body (encode-value value)} + preconditions))) + +(defn read-op + [test node op k] + (try+ + (let [entry (get-value! test node k)] + (assoc op + :type :ok + :value (independent/tuple k (decode-value (:body entry))))) + (catch [:type ::rpc-error] e + (if (= 404 (:status e)) + (assoc op :type :ok :value (independent/tuple k nil)) + (throw+ e))))) + +(defn cas-op + [test node session sequence op k from to] + (try+ + (if (nil? from) + (do (put-value! test node session sequence k to {:ifNoneMatch true}) + (assoc op :type :ok)) + (let [entry (get-value! test node k)] + (if (not= from (decode-value (:body entry))) + (assoc op :type :fail) + (do (put-value! test node session sequence k to {:ifMatch (:etag entry)}) + (assoc op :type :ok))))) + (catch [:type ::rpc-error] e + (if (#{400 404 409 412} (:status e)) + (assoc op :type :fail) + (throw+ e))))) + +(defrecord KvClient [node session sequence] + client/Client + (open! [this test node] + (let [leader (await-leader! test)] + (assoc this + :node leader + :session (register-session! test leader) + :sequence (atom 0)))) + + (setup! [_ _test]) + + (invoke! [_ test op] + (let [[k value] (:value op)] + (case (:f op) + :read (read-op test node op k) + :write (do (put-value! test node session sequence k value {}) + (assoc op :type :ok)) + :cas (let [[from to] value] + (cas-op test node session sequence op k from to))))) + + (teardown! [_ _test]) + + (close! [_ _test]) + + client/Reusable + (reusable? [_ _test] + true)) + +(defn kv-workload + [opts] + {:checker (independent/checker + (checker/compose + {:linearizable (checker/linearizable + {:model (model/cas-register)}) + :timeline (timeline/html)})) + :client (KvClient. nil nil nil) + :generator (cond->> (gen/clients + (gen/mix [register-read register-write register-cas register-cas])) + (pos? (:stagger opts)) (gen/stagger (:stagger opts)) + (:time-limit opts) (gen/time-limit (:time-limit opts)))}) + +(defn cloud9-test + [opts] + (merge tests/noop-test + opts + (kv-workload opts) + {:name "cloud9 db" + :pure-generators true + :os debian/os + :db (cloud9-db)})) + +(def cli-opts + [[nil "--binary PATH" "Local c9 binary to upload" + :default "../target/release/c9" + :parse-fn canonical-path] + [nil "--client-port PORT" "Cloud9 client port on each node" + :default 19090 + :parse-fn parse-long] + [nil "--raft-port PORT" "Cloud9 Raft peer port on each node" + :default 19091 + :parse-fn parse-long] + [nil "--stagger SECONDS" "Average seconds between generated operations" + :default 0.01 + :parse-fn parse-double]]) + +(defn -main + [& args] + (cli/run! (merge (cli/single-test-cmd {:test-fn cloud9-test + :opt-spec cli-opts}) + (cli/serve-cmd)) + args)) From 1941a3f1b23e24e5cc79ee7378a710a0d2d4db8f Mon Sep 17 00:00:00 2001 From: Windsor Date: Sat, 9 May 2026 13:24:30 -0700 Subject: [PATCH 03/17] test(node): exercise leader failover --- jepsen/README.md | 10 +- jepsen/src/cloud9/jepsen.clj | 177 +++++++++++++++++++++++++++-------- 2 files changed, 145 insertions(+), 42 deletions(-) diff --git a/jepsen/README.md b/jepsen/README.md index 2a19a87..d178ba3 100644 --- a/jepsen/README.md +++ b/jepsen/README.md @@ -40,14 +40,16 @@ Useful knobs: ```bash lein run test --help +lein run test --nodes-file ~/nodes --username root --time-limit 60 --concurrency 5n --nemesis-mode kill-leader lein run serve ``` -The harness discovers the current Raft leader before opening client sessions. -Followers reject mutating KV RPCs rather than serving node-local state. +The harness discovers the current Raft leader before opening client sessions and +rediscovers it after failover. Followers reject mutating KV RPCs rather than +serving node-local state. ## Current Limit `c9 start` now drives KV commands through Raft, but this is still a transient -runtime: Raft persistence, snapshot transfer, read forwarding, failover-aware -clients, and nemesis coverage are intentionally not complete yet. +runtime: Raft persistence, snapshot transfer, read forwarding, and richer nemesis +coverage are intentionally not complete yet. diff --git a/jepsen/src/cloud9/jepsen.clj b/jepsen/src/cloud9/jepsen.clj index 78163fe..9f4054a 100644 --- a/jepsen/src/cloud9/jepsen.clj +++ b/jepsen/src/cloud9/jepsen.clj @@ -10,7 +10,9 @@ [control :as c] [db :as db] [generator :as gen] + [history :as h] [independent :as independent] + [nemesis :as nemesis] [random :as rand] [tests :as tests]] [jepsen.control.util :as cu] @@ -196,9 +198,14 @@ [sequence] (str (swap! sequence inc))) +(declare with-leader-retry!) + (defn register-session! - [test node] - (:clientId (rpc! test node "RegisterSession" {}))) + [test leader] + (:clientId + (with-leader-retry! test leader + (fn [node] + (rpc! test node "RegisterSession" {}))))) (defn status! [test node] @@ -226,24 +233,72 @@ (throw+ {:type ::no-leader :message "no Cloud9 Raft leader elected"}))))) -(defn get-value! - [test node k] - (rpc! test node "Get" {:namespace kv-namespace - :key (kv-key k)})) +(defn not-leader? + [e] + (and (= 400 (:status e)) + (str/includes? (str (:body e)) "not leader"))) + +(defn recoverable-rpc-error? + [e] + (or (nil? (:status e)) + (#{408 500 502 503 504} (:status e)) + (not-leader? e))) + +(defrecord ClientOnlyChecker [checker] + checker/Checker + (check [_ test history opts] + (checker/check checker test (h/filter #(not= :nemesis (:process %)) history) opts))) + +(defn client-only + [checker] + (ClientOnlyChecker. checker)) + +(defn with-leader-retry! + [test leader f] + (loop [attempts 20] + (let [node (or @leader (await-leader! test)) + result (try+ + {:type ::ok + :value (f node)} + (catch [:type ::rpc-error] e + {:type ::error + :error e}))] + (if (= ::ok (:type result)) + (:value result) + (let [error (:error result)] + (if (and (pos? attempts) (recoverable-rpc-error? error)) + (do + (reset! leader nil) + (recur (dec attempts))) + (throw+ error))))))) -(defn put-value! - [test node session sequence k value preconditions] +(defn get-value! + [test leader k] + (with-leader-retry! test leader + (fn [node] + (rpc! test node "Get" {:namespace kv-namespace + :key (kv-key k)})))) + +(defn put-value-with-sequence! + [test node session op-sequence k value preconditions] (rpc! test node "Put" (merge {:clientId session - :sequence (next-sequence! sequence) + :sequence op-sequence :namespace kv-namespace :key (kv-key k) :body (encode-value value)} preconditions))) +(defn put-value! + [test leader session sequence k value preconditions] + (let [op-sequence (next-sequence! sequence)] + (with-leader-retry! test leader + (fn [node] + (put-value-with-sequence! test node session op-sequence k value preconditions))))) + (defn read-op - [test node op k] + [test leader op k] (try+ - (let [entry (get-value! test node k)] + (let [entry (get-value! test leader k)] (assoc op :type :ok :value (independent/tuple k (decode-value (:body entry))))) @@ -253,40 +308,47 @@ (throw+ e))))) (defn cas-op - [test node session sequence op k from to] + [test leader session sequence op k from to] (try+ (if (nil? from) - (do (put-value! test node session sequence k to {:ifNoneMatch true}) + (do (put-value! test leader session sequence k to {:ifNoneMatch true}) (assoc op :type :ok)) - (let [entry (get-value! test node k)] + (let [entry (get-value! test leader k)] (if (not= from (decode-value (:body entry))) (assoc op :type :fail) - (do (put-value! test node session sequence k to {:ifMatch (:etag entry)}) + (do (put-value! test leader session sequence k to {:ifMatch (:etag entry)}) (assoc op :type :ok))))) (catch [:type ::rpc-error] e (if (#{400 404 409 412} (:status e)) (assoc op :type :fail) (throw+ e))))) -(defrecord KvClient [node session sequence] +(defrecord KvClient [leader session sequence] client/Client (open! [this test node] - (let [leader (await-leader! test)] + (let [leader (atom (await-leader! test))] (assoc this - :node leader - :session (register-session! test leader) - :sequence (atom 0)))) + :leader leader + :session (register-session! test leader) + :sequence (atom 0)))) (setup! [_ _test]) (invoke! [_ test op] - (let [[k value] (:value op)] - (case (:f op) - :read (read-op test node op k) - :write (do (put-value! test node session sequence k value {}) - (assoc op :type :ok)) - :cas (let [[from to] value] - (cas-op test node session sequence op k from to))))) + (try+ + (let [[k value] (:value op)] + (case (:f op) + :read (read-op test leader op k) + :write (do (put-value! test leader session sequence k value {}) + (assoc op :type :ok)) + :cas (let [[from to] value] + (cas-op test leader session sequence op k from to)))) + (catch [:type ::no-leader] e + (assoc op :type :info :error e)) + (catch [:type ::rpc-error] e + (if (recoverable-rpc-error? e) + (assoc op :type :info :error e) + (throw+ e))))) (teardown! [_ _test]) @@ -296,18 +358,51 @@ (reusable? [_ _test] true)) +(defn kill-leader-nemesis + [] + (nemesis/node-start-stopper + (fn [test _nodes] + (current-leader test)) + (fn [test node] + (db/kill! (:db test) test node) + [:killed node]) + (fn [test node] + (db/start! (:db test) test node) + [:started node]))) + +(defn nemesis-generator + [opts] + (case (:nemesis-mode opts) + "none" nil + "kill-leader" (->> (cycle [{:f :start} {:f :stop}]) + (gen/stagger (:nemesis-interval opts))) + (throw+ {:type ::unknown-nemesis + :nemesis-mode (:nemesis-mode opts)}))) + +(defn test-nemesis + [opts] + (case (:nemesis-mode opts) + "none" (:nemesis tests/noop-test) + "kill-leader" (kill-leader-nemesis) + (throw+ {:type ::unknown-nemesis + :nemesis-mode (:nemesis-mode opts)}))) + (defn kv-workload [opts] - {:checker (independent/checker - (checker/compose - {:linearizable (checker/linearizable - {:model (model/cas-register)}) - :timeline (timeline/html)})) - :client (KvClient. nil nil nil) - :generator (cond->> (gen/clients - (gen/mix [register-read register-write register-cas register-cas])) - (pos? (:stagger opts)) (gen/stagger (:stagger opts)) - (:time-limit opts) (gen/time-limit (:time-limit opts)))}) + (let [client-gen (cond->> (gen/mix [register-read register-write register-cas register-cas]) + (pos? (:stagger opts)) (gen/stagger (:stagger opts))) + nemesis-gen (nemesis-generator opts)] + {:checker (checker/compose + {:linearizable (client-only + (independent/checker + (checker/linearizable + {:model (model/cas-register)}))) + :timeline (timeline/html)}) + :client (KvClient. nil nil nil) + :generator (cond->> (if nemesis-gen + (gen/clients client-gen nemesis-gen) + (gen/clients client-gen)) + (:time-limit opts) (gen/time-limit (:time-limit opts)))})) (defn cloud9-test [opts] @@ -317,7 +412,8 @@ {:name "cloud9 db" :pure-generators true :os debian/os - :db (cloud9-db)})) + :db (cloud9-db) + :nemesis (test-nemesis opts)})) (def cli-opts [[nil "--binary PATH" "Local c9 binary to upload" @@ -331,6 +427,11 @@ :parse-fn parse-long] [nil "--stagger SECONDS" "Average seconds between generated operations" :default 0.01 + :parse-fn parse-double] + [nil "--nemesis-mode NAME" "Nemesis mode: none or kill-leader" + :default "none"] + [nil "--nemesis-interval SECONDS" "Average seconds between nemesis operations" + :default 2 :parse-fn parse-double]]) (defn -main From 9773f47ac6877c2fb2d5e47bbafb4ba0b47b5810 Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 09:50:48 -0700 Subject: [PATCH 04/17] fix(node): persist Raft runtime state --- Cargo.lock | 22 +- cloud9-node/Cargo.toml | 5 + cloud9-node/src/command.rs | 368 +++++++++++++++ cloud9-node/src/config.rs | 46 ++ cloud9-node/src/lib.rs | 877 +---------------------------------- cloud9-node/src/runtime.rs | 249 ++++++++++ cloud9-node/src/service.rs | 207 +++++++++ cloud9-node/src/store.rs | 126 +++++ cloud9-node/src/tests.rs | 186 ++++++++ cloud9-node/src/transport.rs | 76 +++ cloud9/Cargo.toml | 3 + cloud9/src/main.rs | 71 +-- 12 files changed, 1332 insertions(+), 904 deletions(-) create mode 100644 cloud9-node/src/command.rs create mode 100644 cloud9-node/src/config.rs create mode 100644 cloud9-node/src/runtime.rs create mode 100644 cloud9-node/src/service.rs create mode 100644 cloud9-node/src/store.rs create mode 100644 cloud9-node/src/tests.rs create mode 100644 cloud9-node/src/transport.rs diff --git a/Cargo.lock b/Cargo.lock index c1b9705..1e61851 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -308,6 +308,7 @@ dependencies = [ "cloud9-storage", "miette", "serde", + "tempfile", "tokio", "toml", "tracing", @@ -338,9 +339,12 @@ dependencies = [ "cloud9-proto", "cloud9-raft", "cloud9-storage", + "cloud9-wal", "connectrpc", "serde", "serde_json", + "tempfile", + "thiserror", "tokio", "tracing", ] @@ -408,15 +412,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - [[package]] name = "connectrpc" version = "0.4.2" @@ -472,6 +467,15 @@ dependencies = [ "syn", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "deranged" version = "0.5.5" diff --git a/cloud9-node/Cargo.toml b/cloud9-node/Cargo.toml index 7a65a0f..141ef14 100644 --- a/cloud9-node/Cargo.toml +++ b/cloud9-node/Cargo.toml @@ -17,9 +17,14 @@ axum = { workspace = true } cloud9-core = { workspace = true } cloud9-raft = { workspace = true } cloud9-storage = { workspace = true } +cloud9-wal = { workspace = true } cloud9-proto = { workspace = true } connectrpc = { workspace = true, features = ["axum"] } tracing = { workspace = true } +thiserror = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/cloud9-node/src/command.rs b/cloud9-node/src/command.rs new file mode 100644 index 0000000..8d11028 --- /dev/null +++ b/cloud9-node/src/command.rs @@ -0,0 +1,368 @@ +//! Replicated KV commands and deterministic state-machine application. + +use std::cmp::Ordering; +use std::collections::HashMap; + +use cloud9_proto::generated::cloud9::kv::v1::{ + DeleteResponse, PutResponse, RegisterSessionResponse, +}; +use connectrpc::ConnectError; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) enum KvCommand { + RegisterSession, + ReadBarrier, + Put { + client_id: u64, + sequence: u64, + namespace: String, + key: String, + body: Vec, + if_match: String, + if_none_match: bool, + }, + Delete { + client_id: u64, + sequence: u64, + namespace: String, + key: String, + if_match: String, + }, +} + +pub(crate) enum KvApplyResult { + RegisterSession(RegisterSessionResponse), + Put(PutResponse), + Delete(DeleteResponse), + ReadBarrier, +} + +#[derive(Default)] +pub(crate) struct KvState { + next_client_id: u64, + next_generation: u64, + pub(crate) entries: HashMap, + sessions: HashMap, +} + +impl KvState { + pub(crate) fn new() -> Self { + Self { next_client_id: 1, next_generation: 1, ..Self::default() } + } + + pub(crate) fn apply(&mut self, command: KvCommand) -> Result { + match command { + KvCommand::RegisterSession => { + let client_id = self.next_client_id()?; + Ok(KvApplyResult::RegisterSession(RegisterSessionResponse { + client_id, + ..Default::default() + })) + } + KvCommand::ReadBarrier => Ok(KvApplyResult::ReadBarrier), + KvCommand::Put { + client_id, + sequence, + namespace, + key, + body, + if_match, + if_none_match, + } => self.apply_put( + client_id, + sequence, + &namespace, + &key, + body, + &if_match, + if_none_match, + ), + KvCommand::Delete { client_id, sequence, namespace, key, if_match } => { + self.apply_delete(client_id, sequence, &namespace, &key, &if_match) + } + } + } + + fn apply_put( + &mut self, + client_id: u64, + sequence: u64, + namespace: &str, + key: &str, + body: Vec, + if_match: &str, + if_none_match: bool, + ) -> Result { + validate_mutation_request(client_id, sequence)?; + validate_put_preconditions(if_match, if_none_match)?; + + let name = KvName::new(namespace, key)?; + let request = MutationRequest::Put { + name: name.clone(), + body: body.clone(), + if_match: if_match.to_owned(), + if_none_match, + }; + if let Some(response) = cached_put(self, client_id, sequence, &request)? { + return Ok(KvApplyResult::Put(response)); + } + + check_put_preconditions(self.entries.get(&name), if_match, if_none_match)?; + let generation = self.next_generation()?; + let etag = etag_for(generation); + let response = PutResponse { + namespace: name.namespace.clone(), + key: name.key.clone(), + etag: etag.clone(), + generation, + size: body_len(&body)?, + ..Default::default() + }; + self.entries.insert(name, KvRecord { body, etag, generation }); + self.session_mut(client_id)?.record( + sequence, + request, + MutationResult::Put(response.clone()), + ); + Ok(KvApplyResult::Put(response)) + } + + fn apply_delete( + &mut self, + client_id: u64, + sequence: u64, + namespace: &str, + key: &str, + if_match: &str, + ) -> Result { + validate_mutation_request(client_id, sequence)?; + + let name = KvName::new(namespace, key)?; + let request = MutationRequest::Delete { name: name.clone(), if_match: if_match.to_owned() }; + if let Some(response) = cached_delete(self, client_id, sequence, &request)? { + return Ok(KvApplyResult::Delete(response)); + } + + let removed = if let Some(record) = self.entries.get(&name) { + if !if_match.is_empty() && if_match != record.etag { + return Err(ConnectError::failed_precondition("ETag precondition failed")); + } + self.entries.remove(&name) + } else { + if !if_match.is_empty() { + return Err(ConnectError::failed_precondition("ETag precondition failed")); + } + None + }; + + let response = if let Some(record) = removed { + DeleteResponse { + namespace: name.namespace, + key: name.key, + etag: record.etag, + generation: record.generation, + deleted: true, + ..Default::default() + } + } else { + DeleteResponse { + namespace: name.namespace, + key: name.key, + etag: String::new(), + generation: 0, + deleted: false, + ..Default::default() + } + }; + self.session_mut(client_id)?.record( + sequence, + request, + MutationResult::Delete(response.clone()), + ); + Ok(KvApplyResult::Delete(response)) + } + + fn next_client_id(&mut self) -> Result { + let client_id = self.next_client_id; + self.next_client_id = self + .next_client_id + .checked_add(1) + .ok_or_else(|| ConnectError::resource_exhausted("client id space exhausted"))?; + self.sessions.insert(client_id, SessionState::default()); + Ok(client_id) + } + + fn next_generation(&mut self) -> Result { + let generation = self.next_generation; + self.next_generation = self + .next_generation + .checked_add(1) + .ok_or_else(|| ConnectError::resource_exhausted("kv generation space exhausted"))?; + Ok(generation) + } + + fn session(&self, client_id: u64) -> Result<&SessionState, ConnectError> { + self.sessions + .get(&client_id) + .ok_or_else(|| ConnectError::invalid_argument("unknown client session")) + } + + fn session_mut(&mut self, client_id: u64) -> Result<&mut SessionState, ConnectError> { + self.sessions + .get_mut(&client_id) + .ok_or_else(|| ConnectError::invalid_argument("unknown client session")) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(crate) struct KvName { + pub(crate) namespace: String, + pub(crate) key: String, +} + +impl KvName { + pub(crate) fn new(namespace: &str, key: &str) -> Result { + if namespace.is_empty() { + return Err(ConnectError::invalid_argument("namespace must not be empty")); + } + if key.is_empty() { + return Err(ConnectError::invalid_argument("key must not be empty")); + } + Ok(Self { namespace: namespace.to_owned(), key: key.to_owned() }) + } +} + +#[derive(Clone)] +pub(crate) struct KvRecord { + pub(crate) body: Vec, + pub(crate) etag: String, + pub(crate) generation: u64, +} + +#[derive(Clone, Default)] +struct SessionState { + sequence: u64, + request: Option, + result: Option, +} + +#[derive(Clone, PartialEq, Eq)] +enum MutationRequest { + Put { name: KvName, body: Vec, if_match: String, if_none_match: bool }, + Delete { name: KvName, if_match: String }, +} + +#[derive(Clone)] +enum MutationResult { + Put(PutResponse), + Delete(DeleteResponse), +} + +impl SessionState { + fn record(&mut self, sequence: u64, request: MutationRequest, result: MutationResult) { + self.sequence = sequence; + self.request = Some(request); + self.result = Some(result); + } +} + +pub(crate) fn validate_mutation_request(client_id: u64, sequence: u64) -> Result<(), ConnectError> { + if client_id == 0 { + return Err(ConnectError::invalid_argument("client_id must be registered")); + } + if sequence == 0 { + return Err(ConnectError::invalid_argument("sequence must be positive")); + } + Ok(()) +} + +pub(crate) fn validate_put_preconditions( + if_match: &str, + if_none_match: bool, +) -> Result<(), ConnectError> { + if !if_match.is_empty() && if_none_match { + return Err(ConnectError::invalid_argument( + "if_match and if_none_match are mutually exclusive", + )); + } + Ok(()) +} + +fn check_put_preconditions( + current: Option<&KvRecord>, + if_match: &str, + if_none_match: bool, +) -> Result<(), ConnectError> { + if if_none_match && current.is_some() { + return Err(ConnectError::failed_precondition("key already exists")); + } + if !if_match.is_empty() { + match current { + Some(record) if record.etag == if_match => {} + Some(_) | None => { + return Err(ConnectError::failed_precondition("ETag precondition failed")); + } + } + } + Ok(()) +} + +fn cached_put( + state: &KvState, + client_id: u64, + sequence: u64, + request: &MutationRequest, +) -> Result, ConnectError> { + match cached_mutation(state.session(client_id)?, sequence, request)? { + Some(MutationResult::Put(response)) => Ok(Some(response)), + Some(MutationResult::Delete(_)) => Err(ConnectError::internal("session result mismatch")), + None => Ok(None), + } +} + +fn cached_delete( + state: &KvState, + client_id: u64, + sequence: u64, + request: &MutationRequest, +) -> Result, ConnectError> { + match cached_mutation(state.session(client_id)?, sequence, request)? { + Some(MutationResult::Delete(response)) => Ok(Some(response)), + Some(MutationResult::Put(_)) => Err(ConnectError::internal("session result mismatch")), + None => Ok(None), + } +} + +fn cached_mutation( + session: &SessionState, + sequence: u64, + request: &MutationRequest, +) -> Result, ConnectError> { + match sequence.cmp(&session.sequence) { + Ordering::Less => Err(ConnectError::aborted("stale client sequence")), + Ordering::Greater => Ok(None), + Ordering::Equal => match (&session.request, &session.result) { + (Some(cached), Some(result)) if cached == request => Ok(Some(result.clone())), + (Some(_), Some(_)) => { + Err(ConnectError::aborted("sequence reused for different request")) + } + (None, None) => Err(ConnectError::internal("session cache is incomplete")), + (Some(_), None) | (None, Some(_)) => { + Err(ConnectError::internal("session cache is inconsistent")) + } + }, + } +} + +fn etag_for(generation: u64) -> String { + format!("\"c9-{generation}\"") +} + +pub(crate) fn body_len(body: &[u8]) -> Result { + u64::try_from(body.len()).map_err(|_| ConnectError::resource_exhausted("value too large")) +} + +pub(crate) fn key_not_found() -> ConnectError { + ConnectError::not_found("key not found") +} diff --git a/cloud9-node/src/config.rs b/cloud9-node/src/config.rs new file mode 100644 index 0000000..1f3eaed --- /dev/null +++ b/cloud9-node/src/config.rs @@ -0,0 +1,46 @@ +//! Node runtime configuration. + +use std::collections::BTreeMap; +use std::net::{Ipv4Addr, SocketAddr}; +use std::path::{Path, PathBuf}; + +use cloud9_raft::{ConsensusConfig, NodeId}; +use cloud9_storage::StorageOptions; + +/// Runtime configuration derived from CLI flags and config files. +#[derive(Debug, Clone)] +pub struct NodeConfig { + pub node_id: NodeId, + pub client_addr: SocketAddr, + pub raft_addr: SocketAddr, + pub peers: BTreeMap, + pub storage: StorageOptions, + pub consensus: ConsensusConfig, +} + +impl Default for NodeConfig { + fn default() -> Self { + let node_id = NodeId(0); + let raft_addr = SocketAddr::from((Ipv4Addr::LOCALHOST, 19_091)); + Self { + node_id, + client_addr: SocketAddr::from((Ipv4Addr::LOCALHOST, 19_090)), + raft_addr, + peers: BTreeMap::from([(node_id, raft_addr)]), + storage: StorageOptions::default(), + consensus: raft_config(node_id), + } + } +} + +impl NodeConfig { + #[must_use] + pub(crate) fn raft_dir(&self) -> PathBuf { + Path::new(self.storage.data_dir.as_str()).join("raft") + } +} + +#[must_use] +pub fn raft_config(node_id: NodeId) -> ConsensusConfig { + ConsensusConfig::new(node_id).with_parallel_disk_write(false) +} diff --git a/cloud9-node/src/lib.rs b/cloud9-node/src/lib.rs index e9790e2..d9e68d3 100644 --- a/cloud9-node/src/lib.rs +++ b/cloud9-node/src/lib.rs @@ -1,868 +1,21 @@ -//! Top-level orchestration for Cloud9 nodes. - -use std::cmp::Ordering; -use std::collections::{BTreeMap, HashMap}; -use std::net::{Ipv4Addr, SocketAddr}; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use axum::Json; -use axum::Router as AxumRouter; -use axum::extract::State; -use axum::http::StatusCode; -use axum::routing::{get, post}; -use cloud9_proto::generated::cloud9::kv::v1::{ - DeleteResponse, GetResponse, HeadResponse, KvService, KvServiceExt, OwnedDeleteRequestView, - OwnedGetRequestView, OwnedHeadRequestView, OwnedPutRequestView, - OwnedRegisterSessionRequestView, OwnedStatusRequestView, PutResponse, RegisterSessionResponse, - StatusResponse, -}; -use cloud9_raft::raft::{Effects, Message}; -use cloud9_raft::{Command, ConsensusConfig, LogIndex, NodeId, ProposeError, RaftNode}; -use cloud9_storage::StorageOptions; -use connectrpc::{ConnectError, RequestContext, Response, Router as ConnectRouter, ServiceResult}; -use serde::{Deserialize, Serialize}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; -use tokio::sync::{Mutex, RwLock, oneshot}; -use tokio::time::{Duration, sleep}; -use tracing::{info, instrument, warn}; - -/// Runtime configuration derived from CLI flags and config files. -#[derive(Debug, Clone)] -pub struct NodeConfig { - pub node_id: NodeId, - pub client_addr: SocketAddr, - pub raft_addr: SocketAddr, - pub peers: BTreeMap, - pub storage: StorageOptions, - pub consensus: ConsensusConfig, -} - -impl Default for NodeConfig { - fn default() -> Self { - let node_id = NodeId(0); - let raft_addr = SocketAddr::from((Ipv4Addr::LOCALHOST, 19_091)); - Self { - node_id, - client_addr: SocketAddr::from((Ipv4Addr::LOCALHOST, 19_090)), - raft_addr, - peers: BTreeMap::from([(node_id, raft_addr)]), - storage: StorageOptions::default(), - consensus: raft_config(node_id), - } - } -} - -pub fn raft_config(node_id: NodeId) -> ConsensusConfig { - ConsensusConfig::new(node_id).with_parallel_disk_write(false) -} - -#[derive(Clone)] -struct KvApi { - config: NodeConfig, - state: Arc>, - runtime: Arc, -} - -struct RaftRuntime { - config: NodeConfig, - node: Mutex, - state: Arc>, - waiters: Mutex>>>, -} - -impl RaftRuntime { - fn new(config: NodeConfig, state: Arc>) -> Self { - let voters = config.peers.keys().copied().collect::>(); - Self { - node: Mutex::new(RaftNode::new(config.consensus.clone(), &voters)), - config, - state, - waiters: Mutex::new(HashMap::new()), - } - } - - fn spawn(self: Arc) { - tokio::spawn(async move { - self.tick_loop().await; - }); - } - - async fn tick_loop(&self) { - loop { - sleep(Duration::from_millis(1)).await; - let mut node = self.node.lock().await; - let effects = node.tick(); - self.handle_effects(&mut node, effects).await; - } - } - - async fn step(&self, message: Message) { - let mut node = self.node.lock().await; - let effects = node.step(message); - self.handle_effects(&mut node, effects).await; - } - - async fn propose(&self, command: KvCommand) -> Result { - let bytes = serde_json::to_vec(&command) - .map_err(|_| ConnectError::internal("failed to encode Raft command"))?; - let receiver = { - let mut node = self.node.lock().await; - let (index, effects) = - node.propose(Command(bytes)).map_err(|error| propose_error(&error))?; - let (sender, receiver) = oneshot::channel(); - self.waiters.lock().await.insert(index, sender); - self.handle_effects(&mut node, effects).await; - receiver - }; - receiver.await.map_err(|_| ConnectError::aborted("Raft proposal was dropped"))? - } - - async fn read_barrier(&self) -> Result<(), ConnectError> { - match self.propose(KvCommand::ReadBarrier).await? { - KvApplyResult::ReadBarrier => Ok(()), - KvApplyResult::RegisterSession(_) - | KvApplyResult::Put(_) - | KvApplyResult::Delete(_) => Err(ConnectError::internal("Raft read barrier mismatch")), - } - } - - async fn mode(&self) -> String { - let node = self.node.lock().await; - if node.is_leader() { - "leader" - } else if node.is_candidate() { - "candidate" - } else if node.is_precandidate() { - "precandidate" - } else { - "follower" - } - .to_owned() - } - - async fn handle_effects(&self, node: &mut RaftNode, effects: Effects) { - if !effects.send_snapshots.is_empty() { - warn!( - snapshot_count = effects.send_snapshots.len(), - "Raft snapshot transport is not implemented" - ); - } - - for message in effects.messages { - self.send_message(message); - } - - self.apply_committed(node).await; - } - - async fn apply_committed(&self, node: &mut RaftNode) { - let entries = node.committed().collect::>(); - let mut applied_to = None; - for entry in entries { - let result = self.apply_command(&entry.command).await; - self.complete_waiter(entry.index, result).await; - applied_to = Some(entry.index); - } - if let Some(index) = applied_to { - node.advance(index); - } - } - - async fn apply_command(&self, command: &Command) -> Result { - let command = serde_json::from_slice(&command.0) - .map_err(|_| ConnectError::internal("invalid Raft command payload"))?; - self.state.write().await.apply(command) - } - - async fn complete_waiter(&self, index: LogIndex, result: Result) { - if let Some(sender) = self.waiters.lock().await.remove(&index) { - let _ = sender.send(result); - } - } - - fn send_message(&self, message: Message) { - let Some(addr) = self.config.peers.get(&message.to).copied() else { - warn!(to = message.to.0, "Raft message target is not in cluster config"); - return; - }; - tokio::spawn(async move { - if let Err(error) = post_raft_message(addr, &message).await { - warn!(%error, to = message.to.0, "failed to send Raft message"); - } - }); - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -enum KvCommand { - RegisterSession, - ReadBarrier, - Put { - client_id: u64, - sequence: u64, - namespace: String, - key: String, - body: Vec, - if_match: String, - if_none_match: bool, - }, - Delete { - client_id: u64, - sequence: u64, - namespace: String, - key: String, - if_match: String, - }, -} - -enum KvApplyResult { - RegisterSession(RegisterSessionResponse), - Put(PutResponse), - Delete(DeleteResponse), - ReadBarrier, -} - -#[derive(Default)] -struct KvState { - next_client_id: u64, - next_generation: u64, - entries: HashMap, - sessions: HashMap, -} - -impl KvState { - fn new() -> Self { - Self { next_client_id: 1, next_generation: 1, ..Self::default() } - } - - fn apply(&mut self, command: KvCommand) -> Result { - match command { - KvCommand::RegisterSession => { - let client_id = self.next_client_id()?; - Ok(KvApplyResult::RegisterSession(RegisterSessionResponse { - client_id, - ..Default::default() - })) - } - KvCommand::ReadBarrier => Ok(KvApplyResult::ReadBarrier), - KvCommand::Put { - client_id, - sequence, - namespace, - key, - body, - if_match, - if_none_match, - } => self.apply_put( - client_id, - sequence, - &namespace, - &key, - body, - &if_match, - if_none_match, - ), - KvCommand::Delete { client_id, sequence, namespace, key, if_match } => { - self.apply_delete(client_id, sequence, &namespace, &key, &if_match) - } - } - } - - fn apply_put( - &mut self, - client_id: u64, - sequence: u64, - namespace: &str, - key: &str, - body: Vec, - if_match: &str, - if_none_match: bool, - ) -> Result { - validate_mutation_request(client_id, sequence)?; - validate_put_preconditions(if_match, if_none_match)?; - - let name = KvName::new(namespace, key)?; - if let Some(response) = cached_put(self, client_id, sequence)? { - return Ok(KvApplyResult::Put(response)); - } - - let current = self.entries.get(&name); - check_put_preconditions(current, if_match, if_none_match)?; - - let generation = self.next_generation()?; - let etag = etag_for(generation); - let response = PutResponse { - namespace: name.namespace.clone(), - key: name.key.clone(), - etag: etag.clone(), - generation, - size: body_len(&body)?, - ..Default::default() - }; - self.entries.insert(name, KvRecord { body, etag, generation }); - self.session_mut(client_id)?.record(sequence, MutationResult::Put(response.clone())); - Ok(KvApplyResult::Put(response)) - } - - fn apply_delete( - &mut self, - client_id: u64, - sequence: u64, - namespace: &str, - key: &str, - if_match: &str, - ) -> Result { - validate_mutation_request(client_id, sequence)?; - - let name = KvName::new(namespace, key)?; - if let Some(response) = cached_delete(self, client_id, sequence)? { - return Ok(KvApplyResult::Delete(response)); - } - - let removed = if let Some(record) = self.entries.get(&name) { - if !if_match.is_empty() && if_match != record.etag { - return Err(ConnectError::failed_precondition("ETag precondition failed")); - } - self.entries.remove(&name) - } else { - if !if_match.is_empty() { - return Err(ConnectError::failed_precondition("ETag precondition failed")); - } - None - }; - - let response = if let Some(record) = removed { - DeleteResponse { - namespace: name.namespace, - key: name.key, - etag: record.etag, - generation: record.generation, - deleted: true, - ..Default::default() - } - } else { - DeleteResponse { - namespace: name.namespace, - key: name.key, - etag: String::new(), - generation: 0, - deleted: false, - ..Default::default() - } - }; - self.session_mut(client_id)?.record(sequence, MutationResult::Delete(response.clone())); - Ok(KvApplyResult::Delete(response)) - } - - fn next_client_id(&mut self) -> Result { - let client_id = self.next_client_id; - self.next_client_id = self - .next_client_id - .checked_add(1) - .ok_or_else(|| ConnectError::resource_exhausted("client id space exhausted"))?; - self.sessions.insert(client_id, SessionState::default()); - Ok(client_id) - } - - fn next_generation(&mut self) -> Result { - let generation = self.next_generation; - self.next_generation = self - .next_generation - .checked_add(1) - .ok_or_else(|| ConnectError::resource_exhausted("kv generation space exhausted"))?; - Ok(generation) - } - - fn session(&self, client_id: u64) -> Result<&SessionState, ConnectError> { - self.sessions - .get(&client_id) - .ok_or_else(|| ConnectError::invalid_argument("unknown client session")) - } - - fn session_mut(&mut self, client_id: u64) -> Result<&mut SessionState, ConnectError> { - self.sessions - .get_mut(&client_id) - .ok_or_else(|| ConnectError::invalid_argument("unknown client session")) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct KvName { - namespace: String, - key: String, -} - -impl KvName { - fn new(namespace: &str, key: &str) -> Result { - if namespace.is_empty() { - return Err(ConnectError::invalid_argument("namespace must not be empty")); - } - if key.is_empty() { - return Err(ConnectError::invalid_argument("key must not be empty")); - } - Ok(Self { namespace: namespace.to_owned(), key: key.to_owned() }) - } -} - -#[derive(Clone)] -struct KvRecord { - body: Vec, - etag: String, - generation: u64, -} - -#[derive(Clone, Default)] -struct SessionState { - last_sequence: u64, - last_result: Option, -} - -#[derive(Clone)] -enum MutationResult { - Put(PutResponse), - Delete(DeleteResponse), -} - -/// Launch the node's public KV API and Raft peer API. -#[instrument(skip_all)] -pub async fn launch(config: NodeConfig) -> Result<()> { - info!(?config.storage, "initializing storage"); - info!(?config.consensus, "consensus subsystem configured"); - - let state = Arc::new(RwLock::new(KvState::new())); - let runtime = Arc::new(RaftRuntime::new(config.clone(), state.clone())); - let api = Arc::new(KvApi { config: config.clone(), state, runtime: runtime.clone() }); - let client_app = kv_app(api); - let raft_app = raft_app(runtime.clone()); - let client_listener = TcpListener::bind(config.client_addr) - .await - .with_context(|| format!("binding Cloud9 KV API to {}", config.client_addr))?; - let raft_listener = TcpListener::bind(config.raft_addr) - .await - .with_context(|| format!("binding Cloud9 Raft API to {}", config.raft_addr))?; +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::panic))] - info!( - node_id = config.node_id.0, - client_addr = %config.client_addr, - raft_addr = %config.raft_addr, - peer_count = config.peers.len(), - "serving Cloud9 KV API" - ); - - runtime.spawn(); - tokio::try_join!( - axum::serve(client_listener, client_app), - axum::serve(raft_listener, raft_app), - ) - .context("serving Cloud9 node")?; - Ok(()) -} - -fn kv_app(api: Arc) -> AxumRouter { - let connect = api.register(ConnectRouter::new()); - AxumRouter::new() - .route("/healthz", get(|| async { "ok" })) - .fallback_service(connect.into_axum_service()) -} - -fn raft_app(runtime: Arc) -> AxumRouter { - AxumRouter::new().route("/raft/message", post(receive_raft)).with_state(runtime) -} - -async fn receive_raft( - State(runtime): State>, - Json(message): Json, -) -> StatusCode { - runtime.step(message).await; - StatusCode::NO_CONTENT -} - -#[allow(refining_impl_trait)] -impl KvService for KvApi { - async fn register_session( - &self, - _ctx: RequestContext, - _request: OwnedRegisterSessionRequestView, - ) -> ServiceResult { - match self.runtime.propose(KvCommand::RegisterSession).await? { - KvApplyResult::RegisterSession(response) => Response::ok(response), - KvApplyResult::Put(_) | KvApplyResult::Delete(_) | KvApplyResult::ReadBarrier => { - Err(ConnectError::internal("Raft session command mismatch")) - } - } - } - - async fn head( - &self, - _ctx: RequestContext, - request: OwnedHeadRequestView, - ) -> ServiceResult { - let name = KvName::new(request.namespace, request.key)?; - self.runtime.read_barrier().await?; - let state = self.state.read().await; - let record = state.entries.get(&name).ok_or_else(key_not_found)?; - Response::ok(HeadResponse { - namespace: name.namespace, - key: name.key, - etag: record.etag.clone(), - generation: record.generation, - size: body_len(&record.body)?, - ..Default::default() - }) - } - - async fn get( - &self, - _ctx: RequestContext, - request: OwnedGetRequestView, - ) -> ServiceResult { - let name = KvName::new(request.namespace, request.key)?; - self.runtime.read_barrier().await?; - let state = self.state.read().await; - let record = state.entries.get(&name).ok_or_else(key_not_found)?; - Response::ok(GetResponse { - namespace: name.namespace, - key: name.key, - etag: record.etag.clone(), - generation: record.generation, - size: body_len(&record.body)?, - body: record.body.clone(), - ..Default::default() - }) - } - - async fn put( - &self, - _ctx: RequestContext, - request: OwnedPutRequestView, - ) -> ServiceResult { - validate_mutation_request(request.client_id, request.sequence)?; - validate_put_preconditions(request.if_match, request.if_none_match)?; - KvName::new(request.namespace, request.key)?; - - let command = KvCommand::Put { - client_id: request.client_id, - sequence: request.sequence, - namespace: request.namespace.to_owned(), - key: request.key.to_owned(), - body: request.body.to_vec(), - if_match: request.if_match.to_owned(), - if_none_match: request.if_none_match, - }; - match self.runtime.propose(command).await? { - KvApplyResult::Put(response) => Response::ok(response), - KvApplyResult::RegisterSession(_) - | KvApplyResult::Delete(_) - | KvApplyResult::ReadBarrier => { - Err(ConnectError::internal("Raft put command mismatch")) - } - } - } - - async fn delete( - &self, - _ctx: RequestContext, - request: OwnedDeleteRequestView, - ) -> ServiceResult { - validate_mutation_request(request.client_id, request.sequence)?; - KvName::new(request.namespace, request.key)?; - - let command = KvCommand::Delete { - client_id: request.client_id, - sequence: request.sequence, - namespace: request.namespace.to_owned(), - key: request.key.to_owned(), - if_match: request.if_match.to_owned(), - }; - match self.runtime.propose(command).await? { - KvApplyResult::Delete(response) => Response::ok(response), - KvApplyResult::RegisterSession(_) - | KvApplyResult::Put(_) - | KvApplyResult::ReadBarrier => { - Err(ConnectError::internal("Raft delete command mismatch")) - } - } - } - - async fn status( - &self, - _ctx: RequestContext, - _request: OwnedStatusRequestView, - ) -> ServiceResult { - let mode = self.runtime.mode().await; - let state = self.state.read().await; - Response::ok(StatusResponse { - node_id: self.config.node_id.0, - mode, - key_count: usize_to_u64(state.entries.len()), - ..Default::default() - }) - } -} - -impl SessionState { - fn record(&mut self, sequence: u64, result: MutationResult) { - self.last_sequence = sequence; - self.last_result = Some(result); - } -} - -fn validate_mutation_request(client_id: u64, sequence: u64) -> Result<(), ConnectError> { - if client_id == 0 { - return Err(ConnectError::invalid_argument("client_id must be registered")); - } - if sequence == 0 { - return Err(ConnectError::invalid_argument("sequence must be positive")); - } - Ok(()) -} - -fn validate_put_preconditions(if_match: &str, if_none_match: bool) -> Result<(), ConnectError> { - if !if_match.is_empty() && if_none_match { - return Err(ConnectError::invalid_argument( - "if_match and if_none_match are mutually exclusive", - )); - } - Ok(()) -} - -fn check_put_preconditions( - current: Option<&KvRecord>, - if_match: &str, - if_none_match: bool, -) -> Result<(), ConnectError> { - if if_none_match && current.is_some() { - return Err(ConnectError::failed_precondition("key already exists")); - } - - if !if_match.is_empty() { - match current { - Some(record) if record.etag == if_match => {} - Some(_) | None => { - return Err(ConnectError::failed_precondition("ETag precondition failed")); - } - } - } - - Ok(()) -} - -fn cached_put( - state: &KvState, - client_id: u64, - sequence: u64, -) -> Result, ConnectError> { - match cached_mutation(state.session(client_id)?, sequence)? { - Some(MutationResult::Put(response)) => Ok(Some(response)), - Some(MutationResult::Delete(_)) => { - Err(ConnectError::aborted("sequence reused for different operation")) - } - None => Ok(None), - } -} - -fn cached_delete( - state: &KvState, - client_id: u64, - sequence: u64, -) -> Result, ConnectError> { - match cached_mutation(state.session(client_id)?, sequence)? { - Some(MutationResult::Delete(response)) => Ok(Some(response)), - Some(MutationResult::Put(_)) => { - Err(ConnectError::aborted("sequence reused for different operation")) - } - None => Ok(None), - } -} - -fn cached_mutation( - session: &SessionState, - sequence: u64, -) -> Result, ConnectError> { - match sequence.cmp(&session.last_sequence) { - Ordering::Less => Err(ConnectError::aborted("stale client sequence")), - Ordering::Equal => Ok(session.last_result.clone()), - Ordering::Greater => Ok(None), - } -} - -fn etag_for(generation: u64) -> String { - format!("\"c9-{generation}\"") -} - -fn body_len(body: &[u8]) -> Result { - u64::try_from(body.len()).map_err(|_| ConnectError::resource_exhausted("value too large")) -} - -fn usize_to_u64(value: usize) -> u64 { - u64::try_from(value).unwrap_or(u64::MAX) -} - -fn key_not_found() -> ConnectError { - ConnectError::not_found("key not found") -} - -fn propose_error(error: &ProposeError) -> ConnectError { - match error { - ProposeError::NotLeader { leader_hint: Some(leader) } => { - ConnectError::failed_precondition(format!("not leader; leader is {}", leader.0)) - } - ProposeError::NotLeader { leader_hint: None } => { - ConnectError::failed_precondition("not leader; leader unknown") - } - ProposeError::Throttled => ConnectError::resource_exhausted("too many Raft proposals"), - } -} - -async fn post_raft_message(addr: SocketAddr, message: &Message) -> Result<()> { - let body = serde_json::to_vec(message).context("encoding Raft message")?; - let mut stream = TcpStream::connect(addr) - .await - .with_context(|| format!("connecting to Raft peer {addr}"))?; - let request = format!( - "POST /raft/message HTTP/1.1\r\n\ - Host: {addr}\r\n\ - Content-Type: application/json\r\n\ - Content-Length: {}\r\n\ - Connection: close\r\n\ - \r\n", - body.len() - ); - stream.write_all(request.as_bytes()).await.context("writing Raft message headers")?; - stream.write_all(&body).await.context("writing Raft message body")?; - - let mut response = Vec::new(); - stream.read_to_end(&mut response).await.context("reading Raft message response")?; - if response.starts_with(b"HTTP/1.1 204") || response.starts_with(b"HTTP/1.1 200") { - return Ok(()); - } - - let response = String::from_utf8_lossy(&response); - anyhow::bail!("Raft peer {addr} rejected message: {response}"); -} +//! Top-level orchestration for Cloud9 nodes. +mod command; +mod config; +mod runtime; +mod service; +mod store; #[cfg(test)] -mod tests { - use std::io::{Read, Write}; - use std::time::Duration; - - use anyhow::bail; - - use super::*; - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn kv_api_enforces_etag_preconditions() -> Result<()> { - let config = NodeConfig::default(); - let state = Arc::new(RwLock::new(KvState::new())); - let runtime = Arc::new(RaftRuntime::new(config.clone(), state.clone())); - let api = Arc::new(KvApi { config: config.clone(), state, runtime: runtime.clone() }); - let listener = TcpListener::bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0))).await?; - let addr = listener.local_addr()?; - let server = tokio::spawn(axum::serve(listener, kv_app(api)).into_future()); - runtime.spawn(); - wait_for_leader(addr).await?; - - let (status, body) = post_json(addr, "RegisterSession", "{}")?; - assert_eq!(200, status); - assert!(body.contains("\"clientId\":\"1\"")); - - let (status, _) = post_json( - addr, - "Put", - r#"{"clientId":"1","sequence":"1","namespace":"jepsen","key":"register","body":"MQ==","ifNoneMatch":true}"#, - )?; - assert_eq!(200, status); - - let (status, _) = post_json( - addr, - "Put", - r#"{"clientId":"1","sequence":"2","namespace":"jepsen","key":"register","body":"Mg==","ifNoneMatch":true}"#, - )?; - assert_eq!(400, status); - - let (status, _) = post_json( - addr, - "Put", - r#"{"clientId":"1","sequence":"3","namespace":"jepsen","key":"register","body":"Mg==","ifMatch":"\"c9-1\""}"#, - )?; - assert_eq!(200, status); +mod tests; +mod transport; - let (status, body) = post_json(addr, "Get", r#"{"namespace":"jepsen","key":"register"}"#)?; - assert_eq!(200, status); - assert!(body.contains("\"body\":\"Mg==\"")); +pub use config::{NodeConfig, raft_config}; - let (status, _) = post_json( - addr, - "Put", - r#"{"clientId":"1","sequence":"4","namespace":"jepsen","key":"register","body":"Mw==","ifMatch":"\"c9-1\""}"#, - )?; - assert_eq!(400, status); - - server.abort(); - Ok(()) - } - - async fn wait_for_leader(addr: SocketAddr) -> Result<()> { - for _ in 0..100 { - let (status, body) = post_json(addr, "Status", "{}")?; - if status == 200 && body.contains("\"mode\":\"leader\"") { - return Ok(()); - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - bail!("single-node Raft runtime did not elect a leader"); - } - - fn post_json(addr: SocketAddr, method: &str, body: &str) -> Result<(u16, String)> { - let mut stream = std::net::TcpStream::connect(addr)?; - stream.set_read_timeout(Some(Duration::from_secs(2)))?; - let request = format!( - "POST /cloud9.kv.v1.KvService/{method} HTTP/1.1\r\n\ - Host: {addr}\r\n\ - Content-Type: application/json\r\n\ - Content-Length: {}\r\n\ - Connection: close\r\n\ - \r\n\ - {body}", - body.len() - ); - stream.write_all(request.as_bytes())?; - - let mut response = String::new(); - stream.read_to_string(&mut response)?; - let Some((head, body)) = response.split_once("\r\n\r\n") else { - bail!("HTTP response missing header separator"); - }; - let Some(status) = head.lines().next().and_then(|line| line.split_whitespace().nth(1)) - else { - bail!("HTTP response missing status"); - }; - let body = if head.to_ascii_lowercase().contains("transfer-encoding: chunked") { - decode_chunked(body)? - } else { - body.to_owned() - }; - - Ok((status.parse()?, body)) - } - - fn decode_chunked(mut body: &str) -> Result { - let mut decoded = String::new(); - loop { - let Some((len, rest)) = body.split_once("\r\n") else { - bail!("chunk missing length"); - }; - let len = usize::from_str_radix(len.trim(), 16)?; - if len == 0 { - return Ok(decoded); - } - if rest.len() < len + 2 { - bail!("chunk shorter than declared length"); - } - decoded.push_str(&rest[..len]); - body = &rest[len + 2..]; - } - } +/// Launch the node's public KV API and Raft peer API. +pub async fn launch(config: NodeConfig) -> anyhow::Result<()> { + service::launch(config).await } diff --git a/cloud9-node/src/runtime.rs b/cloud9-node/src/runtime.rs new file mode 100644 index 0000000..f8a9885 --- /dev/null +++ b/cloud9-node/src/runtime.rs @@ -0,0 +1,249 @@ +//! Async driver for the pure Raft state machine. +//! +//! Every step is serialized with its WAL. Persistent state is synced before +//! network effects are released, then committed commands are applied in order. + +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use cloud9_raft::raft::{Effects, Message}; +use cloud9_raft::{Command, LogIndex, NodeId, ProposeError, RaftNode}; +use connectrpc::ConnectError; +use thiserror::Error; +use tokio::sync::{Mutex, RwLock, oneshot}; +use tokio::time::{Duration, sleep, timeout}; +use tracing::warn; + +use crate::command::{KvApplyResult, KvCommand, KvState}; +use crate::config::NodeConfig; +use crate::store::{RaftStore, StoreError}; +use crate::transport::post_raft_message; + +const TICK_INTERVAL: Duration = Duration::from_millis(1); +const PROPOSAL_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Debug, Error)] +pub(crate) enum RuntimeError { + #[error(transparent)] + Store(#[from] StoreError), + #[error("Raft snapshot transport is not implemented")] + SnapshotTransportUnsupported, + #[error("Raft emitted a message for unknown peer {peer}")] + UnknownPeer { peer: NodeId }, + #[error("Raft runtime previously failed")] + Failed, +} + +#[derive(Debug, Error)] +pub(crate) enum MessageError { + #[error("Raft message addressed to {actual}, expected {expected}")] + WrongRecipient { expected: NodeId, actual: NodeId }, + #[error("Raft message sender {sender} is not in the cluster")] + UnknownSender { sender: NodeId }, + #[error("Raft message claims this node as its sender")] + SelfMessage, +} + +struct RaftMachine { + node: RaftNode, + store: RaftStore, +} + +pub(crate) struct RaftRuntime { + config: NodeConfig, + machine: Mutex, + state: Arc>, + waiters: Mutex>>>, + failed: AtomicBool, +} + +impl RaftRuntime { + pub(crate) fn open( + config: NodeConfig, + state: Arc>, + ) -> Result { + let voters = config.peers.keys().copied().collect::>(); + let initial = RaftNode::new(config.consensus.clone(), &voters); + let store = RaftStore::open(&config.raft_dir(), initial.persistent().clone())?; + let node = RaftNode::restore(config.consensus.clone(), store.persistent().clone()); + Ok(Self { + config, + machine: Mutex::new(RaftMachine { node, store }), + state, + waiters: Mutex::new(HashMap::new()), + failed: AtomicBool::new(false), + }) + } + + pub(crate) async fn run(self: Arc) -> Result<(), RuntimeError> { + loop { + sleep(TICK_INTERVAL).await; + self.tick_once().await?; + } + } + + pub(crate) async fn step(&self, message: Message) -> Result<(), RuntimeError> { + self.ensure_healthy()?; + let mut machine = self.machine.lock().await; + let effects = machine.node.step(message); + self.handle_effects(&mut machine, effects).await.map_err(|error| self.fail(error)) + } + + pub(crate) fn validate_message(&self, message: &Message) -> Result<(), MessageError> { + if message.to != self.config.node_id { + return Err(MessageError::WrongRecipient { + expected: self.config.node_id, + actual: message.to, + }); + } + if message.from == self.config.node_id { + return Err(MessageError::SelfMessage); + } + if !self.config.peers.contains_key(&message.from) { + return Err(MessageError::UnknownSender { sender: message.from }); + } + Ok(()) + } + + pub(crate) async fn propose(&self, command: KvCommand) -> Result { + self.ensure_healthy().map_err(|error| runtime_connect_error(&error))?; + let bytes = serde_json::to_vec(&command) + .map_err(|_| ConnectError::internal("failed to encode Raft command"))?; + let (index, receiver) = { + let mut machine = self.machine.lock().await; + let (index, effects) = + machine.node.propose(Command(bytes)).map_err(|error| propose_error(&error))?; + let (sender, receiver) = oneshot::channel(); + self.waiters.lock().await.insert(index, sender); + if let Err(error) = self.handle_effects(&mut machine, effects).await { + self.waiters.lock().await.remove(&index); + let error = self.fail(error); + return Err(runtime_connect_error(&error)); + } + (index, receiver) + }; + + match timeout(PROPOSAL_TIMEOUT, receiver).await { + Ok(Ok(result)) => result, + Ok(Err(_)) => Err(ConnectError::aborted("Raft proposal was dropped")), + Err(_) => { + self.waiters.lock().await.remove(&index); + Err(ConnectError::unavailable("Raft proposal timed out")) + } + } + } + + pub(crate) async fn read_barrier(&self) -> Result<(), ConnectError> { + match self.propose(KvCommand::ReadBarrier).await? { + KvApplyResult::ReadBarrier => Ok(()), + KvApplyResult::RegisterSession(_) + | KvApplyResult::Put(_) + | KvApplyResult::Delete(_) => Err(ConnectError::internal("Raft read barrier mismatch")), + } + } + + pub(crate) async fn mode(&self) -> String { + let machine = self.machine.lock().await; + if machine.node.is_leader() { + "leader" + } else if machine.node.is_candidate() { + "candidate" + } else if machine.node.is_precandidate() { + "precandidate" + } else { + "follower" + } + .to_owned() + } + + async fn tick_once(&self) -> Result<(), RuntimeError> { + self.ensure_healthy()?; + let mut machine = self.machine.lock().await; + let effects = machine.node.tick(); + self.handle_effects(&mut machine, effects).await.map_err(|error| self.fail(error)) + } + + async fn handle_effects( + &self, + machine: &mut RaftMachine, + effects: Effects, + ) -> Result<(), RuntimeError> { + if effects.persist { + machine.store.save(machine.node.persistent())?; + } + if !effects.send_snapshots.is_empty() { + return Err(RuntimeError::SnapshotTransportUnsupported); + } + for message in effects.messages { + self.send_message(message)?; + } + self.apply_committed(&mut machine.node).await; + Ok(()) + } + + async fn apply_committed(&self, node: &mut RaftNode) { + let entries = node.committed().collect::>(); + let mut applied_to = None; + for entry in entries { + let result = self.apply_command(&entry.command).await; + self.complete_waiter(entry.index, result).await; + applied_to = Some(entry.index); + } + if let Some(index) = applied_to { + node.advance(index); + } + } + + async fn apply_command(&self, command: &Command) -> Result { + let command = serde_json::from_slice(&command.0) + .map_err(|_| ConnectError::internal("invalid Raft command payload"))?; + self.state.write().await.apply(command) + } + + async fn complete_waiter(&self, index: LogIndex, result: Result) { + if let Some(sender) = self.waiters.lock().await.remove(&index) { + let _ = sender.send(result); + } + } + + fn send_message(&self, message: Message) -> Result<(), RuntimeError> { + let addr = self + .config + .peers + .get(&message.to) + .copied() + .ok_or(RuntimeError::UnknownPeer { peer: message.to })?; + tokio::spawn(async move { + if let Err(error) = post_raft_message(addr, &message).await { + warn!(%error, to = message.to.0, "failed to send Raft message"); + } + }); + Ok(()) + } + + fn ensure_healthy(&self) -> Result<(), RuntimeError> { + if self.failed.load(Ordering::Acquire) { Err(RuntimeError::Failed) } else { Ok(()) } + } + + fn fail(&self, error: RuntimeError) -> RuntimeError { + self.failed.store(true, Ordering::Release); + error + } +} + +fn propose_error(error: &ProposeError) -> ConnectError { + match error { + ProposeError::NotLeader { leader_hint: Some(leader) } => { + ConnectError::failed_precondition(format!("not leader; leader is {}", leader.0)) + } + ProposeError::NotLeader { leader_hint: None } => { + ConnectError::failed_precondition("not leader; leader unknown") + } + ProposeError::Throttled => ConnectError::resource_exhausted("too many Raft proposals"), + } +} + +fn runtime_connect_error(error: &RuntimeError) -> ConnectError { + ConnectError::internal(format!("Raft runtime failed: {error}")) +} diff --git a/cloud9-node/src/service.rs b/cloud9-node/src/service.rs new file mode 100644 index 0000000..1ba51da --- /dev/null +++ b/cloud9-node/src/service.rs @@ -0,0 +1,207 @@ +//! Public KV service and node lifecycle. + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use axum::Router as AxumRouter; +use axum::routing::get; +use cloud9_proto::generated::cloud9::kv::v1::{ + DeleteResponse, GetResponse, HeadResponse, KvService, KvServiceExt, OwnedDeleteRequestView, + OwnedGetRequestView, OwnedHeadRequestView, OwnedPutRequestView, + OwnedRegisterSessionRequestView, OwnedStatusRequestView, PutResponse, RegisterSessionResponse, + StatusResponse, +}; +use connectrpc::{ConnectError, RequestContext, Response, Router as ConnectRouter, ServiceResult}; +use tokio::net::TcpListener; +use tokio::sync::RwLock; +use tracing::{info, instrument}; + +use crate::command::{ + KvApplyResult, KvCommand, KvName, KvState, body_len, key_not_found, validate_mutation_request, + validate_put_preconditions, +}; +use crate::config::NodeConfig; +use crate::runtime::RaftRuntime; +use crate::transport::raft_app; + +#[derive(Clone)] +pub(crate) struct KvApi { + config: NodeConfig, + state: Arc>, + runtime: Arc, +} + +impl KvApi { + pub(crate) fn new( + config: NodeConfig, + state: Arc>, + runtime: Arc, + ) -> Self { + Self { config, state, runtime } + } +} + +#[instrument(skip_all)] +pub(crate) async fn launch(config: NodeConfig) -> Result<()> { + let state = Arc::new(RwLock::new(KvState::new())); + let runtime = Arc::new(RaftRuntime::open(config.clone(), state.clone())?); + let api = Arc::new(KvApi::new(config.clone(), state, runtime.clone())); + let client_app = kv_app(api); + let raft_app = raft_app(runtime.clone()); + let client_listener = TcpListener::bind(config.client_addr) + .await + .with_context(|| format!("binding Cloud9 KV API to {}", config.client_addr))?; + let raft_listener = TcpListener::bind(config.raft_addr) + .await + .with_context(|| format!("binding Cloud9 Raft API to {}", config.raft_addr))?; + + info!( + node_id = config.node_id.0, + client_addr = %config.client_addr, + raft_addr = %config.raft_addr, + peer_count = config.peers.len(), + data_dir = config.storage.data_dir.as_str(), + "serving Cloud9 KV API" + ); + + tokio::try_join!( + async { axum::serve(client_listener, client_app).await.context("serving KV API") }, + async { axum::serve(raft_listener, raft_app).await.context("serving Raft API") }, + async { runtime.run().await.context("driving Raft runtime") }, + )?; + Ok(()) +} + +pub(crate) fn kv_app(api: Arc) -> AxumRouter { + let connect = api.register(ConnectRouter::new()); + AxumRouter::new() + .route("/healthz", get(|| async { "ok" })) + .fallback_service(connect.into_axum_service()) +} + +#[allow(refining_impl_trait)] +impl KvService for KvApi { + async fn register_session( + &self, + _ctx: RequestContext, + _request: OwnedRegisterSessionRequestView, + ) -> ServiceResult { + match self.runtime.propose(KvCommand::RegisterSession).await? { + KvApplyResult::RegisterSession(response) => Response::ok(response), + KvApplyResult::Put(_) | KvApplyResult::Delete(_) | KvApplyResult::ReadBarrier => { + Err(ConnectError::internal("Raft session command mismatch")) + } + } + } + + async fn head( + &self, + _ctx: RequestContext, + request: OwnedHeadRequestView, + ) -> ServiceResult { + let name = KvName::new(request.namespace, request.key)?; + self.runtime.read_barrier().await?; + let state = self.state.read().await; + let record = state.entries.get(&name).ok_or_else(key_not_found)?; + Response::ok(HeadResponse { + namespace: name.namespace, + key: name.key, + etag: record.etag.clone(), + generation: record.generation, + size: body_len(&record.body)?, + ..Default::default() + }) + } + + async fn get( + &self, + _ctx: RequestContext, + request: OwnedGetRequestView, + ) -> ServiceResult { + let name = KvName::new(request.namespace, request.key)?; + self.runtime.read_barrier().await?; + let state = self.state.read().await; + let record = state.entries.get(&name).ok_or_else(key_not_found)?; + Response::ok(GetResponse { + namespace: name.namespace, + key: name.key, + etag: record.etag.clone(), + generation: record.generation, + size: body_len(&record.body)?, + body: record.body.clone(), + ..Default::default() + }) + } + + async fn put( + &self, + _ctx: RequestContext, + request: OwnedPutRequestView, + ) -> ServiceResult { + validate_mutation_request(request.client_id, request.sequence)?; + validate_put_preconditions(request.if_match, request.if_none_match)?; + KvName::new(request.namespace, request.key)?; + + let command = KvCommand::Put { + client_id: request.client_id, + sequence: request.sequence, + namespace: request.namespace.to_owned(), + key: request.key.to_owned(), + body: request.body.to_vec(), + if_match: request.if_match.to_owned(), + if_none_match: request.if_none_match, + }; + match self.runtime.propose(command).await? { + KvApplyResult::Put(response) => Response::ok(response), + KvApplyResult::RegisterSession(_) + | KvApplyResult::Delete(_) + | KvApplyResult::ReadBarrier => { + Err(ConnectError::internal("Raft put command mismatch")) + } + } + } + + async fn delete( + &self, + _ctx: RequestContext, + request: OwnedDeleteRequestView, + ) -> ServiceResult { + validate_mutation_request(request.client_id, request.sequence)?; + KvName::new(request.namespace, request.key)?; + + let command = KvCommand::Delete { + client_id: request.client_id, + sequence: request.sequence, + namespace: request.namespace.to_owned(), + key: request.key.to_owned(), + if_match: request.if_match.to_owned(), + }; + match self.runtime.propose(command).await? { + KvApplyResult::Delete(response) => Response::ok(response), + KvApplyResult::RegisterSession(_) + | KvApplyResult::Put(_) + | KvApplyResult::ReadBarrier => { + Err(ConnectError::internal("Raft delete command mismatch")) + } + } + } + + async fn status( + &self, + _ctx: RequestContext, + _request: OwnedStatusRequestView, + ) -> ServiceResult { + let mode = self.runtime.mode().await; + let state = self.state.read().await; + Response::ok(StatusResponse { + node_id: self.config.node_id.0, + mode, + key_count: usize_to_u64(state.entries.len())?, + ..Default::default() + }) + } +} + +fn usize_to_u64(value: usize) -> Result { + u64::try_from(value).map_err(|_| ConnectError::resource_exhausted("key count overflow")) +} diff --git a/cloud9-node/src/store.rs b/cloud9-node/src/store.rs new file mode 100644 index 0000000..f1bf31a --- /dev/null +++ b/cloud9-node/src/store.rs @@ -0,0 +1,126 @@ +//! Durable Raft state over the Cloud9 WAL. +//! +//! Each record stores the hard state plus only the changed log suffix. Recovery +//! replays those deltas into the exact `Persistent` value consumed by Raft. + +use std::path::Path; + +use cloud9_raft::raft::{Entry, Persistent}; +use cloud9_raft::{LogIndex, NodeId}; +use cloud9_wal::{RecordKind, Wal, WalError, WalOptions}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +const RAFT_STATE_KIND: u16 = 1; + +#[derive(Debug, Error)] +pub(crate) enum StoreError { + #[error(transparent)] + Wal(#[from] WalError), + #[error("failed to encode Raft persistent state")] + Encode(#[source] serde_json::Error), + #[error("failed to decode Raft persistent state")] + Decode(#[source] serde_json::Error), + #[error("unexpected WAL record kind {found}")] + UnexpectedKind { found: u16 }, + #[error("Raft WAL term regressed from {previous} to {next}")] + TermRegression { previous: u64, next: u64 }, + #[error("Raft WAL truncation index {index} exceeds durable log end {last_index}")] + InvalidTruncation { index: LogIndex, last_index: LogIndex }, + #[error("Raft WAL entry index {found} is not the expected index {expected}")] + NoncontiguousEntry { expected: LogIndex, found: LogIndex }, + #[error("snapshot persistence is not implemented")] + SnapshotUnsupported, +} + +#[derive(Debug, Serialize, Deserialize)] +struct PersistentDelta { + term: u64, + voted_for: Option, + bootstrap_config: cloud9_raft::Configuration, + truncate_after: LogIndex, + entries: Vec, +} + +pub(crate) struct RaftStore { + wal: Wal, + kind: RecordKind, + durable: Persistent, +} + +impl RaftStore { + pub(crate) fn open(path: &Path, initial: Persistent) -> Result { + let wal = Wal::open(path, WalOptions::default())?; + let kind = RecordKind::new(RAFT_STATE_KIND)?; + let mut store = Self { wal, kind, durable: initial }; + store.recover()?; + Ok(store) + } + + pub(crate) fn persistent(&self) -> &Persistent { + &self.durable + } + + pub(crate) fn save(&mut self, current: &Persistent) -> Result<(), StoreError> { + if current.log.snapshot_index() != self.durable.log.snapshot_index() { + return Err(StoreError::SnapshotUnsupported); + } + let truncate_after = common_prefix(&self.durable, current); + let entries = current.log.slice(truncate_after + 1, current.log.last_index()).to_vec(); + let delta = PersistentDelta { + term: current.term, + voted_for: current.voted_for, + bootstrap_config: current.bootstrap_config.clone(), + truncate_after, + entries, + }; + let encoded = serde_json::to_vec(&delta).map_err(StoreError::Encode)?; + self.wal.append(self.kind, encoded)?; + self.wal.sync()?; + self.durable = current.clone(); + Ok(()) + } + + fn recover(&mut self) -> Result<(), StoreError> { + for stored in self.wal.records()? { + if stored.record.kind != self.kind { + return Err(StoreError::UnexpectedKind { found: stored.record.kind.get() }); + } + let delta = + serde_json::from_slice(&stored.record.payload).map_err(StoreError::Decode)?; + apply_delta(&mut self.durable, delta)?; + } + Ok(()) + } +} + +fn common_prefix(durable: &Persistent, current: &Persistent) -> LogIndex { + let floor = durable.log.snapshot_index().max(current.log.snapshot_index()); + let mut index = durable.log.last_index().min(current.log.last_index()); + while index > floor && durable.log.term_at(index) != current.log.term_at(index) { + index -= 1; + } + index +} + +fn apply_delta(state: &mut Persistent, delta: PersistentDelta) -> Result<(), StoreError> { + if delta.term < state.term { + return Err(StoreError::TermRegression { previous: state.term, next: delta.term }); + } + let last_index = state.log.last_index(); + if delta.truncate_after > last_index { + return Err(StoreError::InvalidTruncation { index: delta.truncate_after, last_index }); + } + state.log.truncate_after(delta.truncate_after); + for entry in delta.entries { + let expected = state.log.last_index() + 1; + if entry.index != expected { + return Err(StoreError::NoncontiguousEntry { expected, found: entry.index }); + } + state.log.append(entry); + } + state.term = delta.term; + state.voted_for = delta.voted_for; + state.bootstrap_config = delta.bootstrap_config; + Ok(()) +} diff --git a/cloud9-node/src/tests.rs b/cloud9-node/src/tests.rs new file mode 100644 index 0000000..89119ee --- /dev/null +++ b/cloud9-node/src/tests.rs @@ -0,0 +1,186 @@ +use std::io::{Read, Write}; +use std::net::{Ipv4Addr, SocketAddr}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Result, bail}; +use cloud9_core::SharedString; +use cloud9_storage::StorageOptions; +use tokio::net::TcpListener; +use tokio::sync::RwLock; + +use crate::command::{KvApplyResult, KvCommand, KvName, KvState}; +use crate::config::NodeConfig; +use crate::runtime::RaftRuntime; +use crate::service::{KvApi, kv_app}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn invariant_mutation_sequence_identifies_exact_request() -> Result<()> { + let dir = tempfile::tempdir()?; + let config = test_config(dir.path()); + let state = Arc::new(RwLock::new(KvState::new())); + let runtime = Arc::new(RaftRuntime::open(config.clone(), state.clone())?); + let api = Arc::new(KvApi::new(config, state, runtime.clone())); + let listener = TcpListener::bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0))).await?; + let addr = listener.local_addr()?; + let server = tokio::spawn(axum::serve(listener, kv_app(api)).into_future()); + let driver = tokio::spawn(runtime.run()); + wait_for_leader(addr).await?; + + let (status, body) = post_json(addr, "RegisterSession", "{}")?; + assert_eq!(200, status); + assert!(body.contains("\"clientId\":\"1\"")); + + let put_one = r#"{"clientId":"1","sequence":"1","namespace":"jepsen","key":"register","body":"MQ==","ifNoneMatch":true}"#; + assert_eq!(200, post_json(addr, "Put", put_one)?.0); + assert_eq!(200, post_json(addr, "Put", put_one)?.0); + + let changed = r#"{"clientId":"1","sequence":"1","namespace":"jepsen","key":"register","body":"Mg==","ifNoneMatch":true}"#; + assert_eq!(409, post_json(addr, "Put", changed)?.0); + + let create_again = r#"{"clientId":"1","sequence":"2","namespace":"jepsen","key":"register","body":"Mg==","ifNoneMatch":true}"#; + assert_eq!(400, post_json(addr, "Put", create_again)?.0); + + let replace = r#"{"clientId":"1","sequence":"3","namespace":"jepsen","key":"register","body":"Mg==","ifMatch":"\"c9-1\""}"#; + assert_eq!(200, post_json(addr, "Put", replace)?.0); + + let (status, body) = post_json(addr, "Get", r#"{"namespace":"jepsen","key":"register"}"#)?; + assert_eq!(200, status); + assert!(body.contains("\"body\":\"Mg==\"")); + + let stale_etag = r#"{"clientId":"1","sequence":"4","namespace":"jepsen","key":"register","body":"Mw==","ifMatch":"\"c9-1\""}"#; + assert_eq!(400, post_json(addr, "Put", stale_etag)?.0); + + server.abort(); + driver.abort(); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn invariant_restart_recovers_committed_state() -> Result<()> { + let dir = tempfile::tempdir()?; + let config = test_config(dir.path()); + let state = Arc::new(RwLock::new(KvState::new())); + let runtime = Arc::new(RaftRuntime::open(config.clone(), state.clone())?); + let driver = tokio::spawn(runtime.clone().run()); + wait_for_runtime_leader(&runtime).await?; + + let client_id = match runtime.propose(KvCommand::RegisterSession).await? { + KvApplyResult::RegisterSession(response) => response.client_id, + KvApplyResult::Put(_) | KvApplyResult::Delete(_) | KvApplyResult::ReadBarrier => { + bail!("session proposal returned the wrong result") + } + }; + runtime + .propose(KvCommand::Put { + client_id, + sequence: 1, + namespace: "test".to_owned(), + key: "key".to_owned(), + body: b"value".to_vec(), + if_match: String::new(), + if_none_match: false, + }) + .await?; + + driver.abort(); + let _ = driver.await; + drop(runtime); + drop(state); + + let recovered_state = Arc::new(RwLock::new(KvState::new())); + let recovered = Arc::new(RaftRuntime::open(config, recovered_state.clone())?); + let recovered_driver = tokio::spawn(recovered.clone().run()); + wait_for_runtime_leader(&recovered).await?; + recovered.read_barrier().await?; + + let state = recovered_state.read().await; + let record = state + .entries + .get(&KvName::new("test", "key")?) + .ok_or_else(|| anyhow::anyhow!("recovered key is missing"))?; + assert_eq!(b"value", record.body.as_slice()); + + recovered_driver.abort(); + Ok(()) +} + +fn test_config(path: &std::path::Path) -> NodeConfig { + NodeConfig { + storage: StorageOptions { + name: SharedString::literal("test"), + data_dir: SharedString::from(path.to_string_lossy()), + }, + ..NodeConfig::default() + } +} + +async fn wait_for_runtime_leader(runtime: &RaftRuntime) -> Result<()> { + for _ in 0..100 { + if runtime.mode().await == "leader" { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + bail!("single-node Raft runtime did not elect a leader") +} + +async fn wait_for_leader(addr: SocketAddr) -> Result<()> { + for _ in 0..100 { + let (status, body) = post_json(addr, "Status", "{}")?; + if status == 200 && body.contains("\"mode\":\"leader\"") { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + bail!("single-node Raft runtime did not elect a leader") +} + +fn post_json(addr: SocketAddr, method: &str, body: &str) -> Result<(u16, String)> { + let mut stream = std::net::TcpStream::connect(addr)?; + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + let request = format!( + "POST /cloud9.kv.v1.KvService/{method} HTTP/1.1\r\n\ + Host: {addr}\r\n\ + Content-Type: application/json\r\n\ + Content-Length: {}\r\n\ + Connection: close\r\n\ + \r\n\ + {body}", + body.len() + ); + stream.write_all(request.as_bytes())?; + + let mut response = String::new(); + stream.read_to_string(&mut response)?; + let Some((head, body)) = response.split_once("\r\n\r\n") else { + bail!("HTTP response missing header separator"); + }; + let Some(status) = head.lines().next().and_then(|line| line.split_whitespace().nth(1)) else { + bail!("HTTP response missing status"); + }; + let body = if head.to_ascii_lowercase().contains("transfer-encoding: chunked") { + decode_chunked(body)? + } else { + body.to_owned() + }; + Ok((status.parse()?, body)) +} + +fn decode_chunked(mut body: &str) -> Result { + let mut decoded = String::new(); + loop { + let Some((len, rest)) = body.split_once("\r\n") else { + bail!("chunk missing length"); + }; + let len = usize::from_str_radix(len.trim(), 16)?; + if len == 0 { + return Ok(decoded); + } + if rest.len() < len + 2 { + bail!("chunk shorter than declared length"); + } + decoded.push_str(&rest[..len]); + body = &rest[len + 2..]; + } +} diff --git a/cloud9-node/src/transport.rs b/cloud9-node/src/transport.rs new file mode 100644 index 0000000..f17213e --- /dev/null +++ b/cloud9-node/src/transport.rs @@ -0,0 +1,76 @@ +//! Raft peer HTTP transport. + +use std::net::SocketAddr; + +use anyhow::{Context, Result}; +use axum::Json; +use axum::Router as AxumRouter; +use axum::extract::State; +use axum::http::StatusCode; +use axum::routing::post; +use cloud9_raft::raft::Message; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; +use tokio::time::{Duration, timeout}; + +use std::sync::Arc; + +use crate::runtime::RaftRuntime; + +const RAFT_RPC_TIMEOUT: Duration = Duration::from_secs(1); +const MAX_RESPONSE_BYTES: u64 = 8 * 1024; + +pub(crate) fn raft_app(runtime: Arc) -> AxumRouter { + AxumRouter::new().route("/raft/message", post(receive_raft)).with_state(runtime) +} + +async fn receive_raft( + State(runtime): State>, + Json(message): Json, +) -> Result { + runtime + .validate_message(&message) + .map_err(|error| (StatusCode::BAD_REQUEST, error.to_string()))?; + runtime + .step(message) + .await + .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?; + Ok(StatusCode::NO_CONTENT) +} + +pub(crate) async fn post_raft_message(addr: SocketAddr, message: &Message) -> Result<()> { + timeout(RAFT_RPC_TIMEOUT, post_raft_message_inner(addr, message)) + .await + .with_context(|| format!("Raft RPC to {addr} timed out"))? +} + +async fn post_raft_message_inner(addr: SocketAddr, message: &Message) -> Result<()> { + let body = serde_json::to_vec(message).context("encoding Raft message")?; + let mut stream = TcpStream::connect(addr) + .await + .with_context(|| format!("connecting to Raft peer {addr}"))?; + let request = format!( + "POST /raft/message HTTP/1.1\r\n\ + Host: {addr}\r\n\ + Content-Type: application/json\r\n\ + Content-Length: {}\r\n\ + Connection: close\r\n\ + \r\n", + body.len() + ); + stream.write_all(request.as_bytes()).await.context("writing Raft message headers")?; + stream.write_all(&body).await.context("writing Raft message body")?; + + let mut response = Vec::new(); + stream + .take(MAX_RESPONSE_BYTES) + .read_to_end(&mut response) + .await + .context("reading Raft message response")?; + if response.starts_with(b"HTTP/1.1 204") { + return Ok(()); + } + + let response = String::from_utf8_lossy(&response); + anyhow::bail!("Raft peer {addr} rejected message: {response}"); +} diff --git a/cloud9/Cargo.toml b/cloud9/Cargo.toml index a3921ca..f92518e 100644 --- a/cloud9/Cargo.toml +++ b/cloud9/Cargo.toml @@ -29,3 +29,6 @@ miette = { workspace = true } serde = { workspace = true } tokio = { workspace = true } toml = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/cloud9/src/main.rs b/cloud9/src/main.rs index e0cb89d..390f970 100644 --- a/cloud9/src/main.rs +++ b/cloud9/src/main.rs @@ -1,3 +1,7 @@ +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::panic))] + use std::collections::BTreeMap; use std::net::{Ipv4Addr, SocketAddr, ToSocketAddrs}; use std::path::{Path, PathBuf}; @@ -30,11 +34,11 @@ struct Cli { #[derive(Debug, Subcommand)] enum Command { - /// Boot the Cloud9 node using an optional configuration file. + /// Boot the Cloud9 node using a configuration file. Start { - /// Optional path to a configuration file (defaults to `cloud9.toml`). - #[arg(long)] - config: Option, + /// Path to the config file. + #[arg(long, default_value = "cloud9.toml")] + config: PathBuf, }, /// Validate the current configuration and exit. CheckConfig { @@ -56,16 +60,12 @@ async fn main() -> Result<()> { match cli.command { Command::Start { config } => { - let config_path = config.unwrap_or_else(|| PathBuf::from("cloud9.toml")); - let config = load_node_config(&config_path)?; - tracing::info!(path = %config_path.display(), "booting node"); - if !config_path.exists() { - tracing::warn!(path = %config_path.display(), "using defaults; config missing"); - } - cloud9_node::launch(config).await.map_err(|error| miette::miette!("{error:#}"))?; + let node_config = load_node_config(&config)?; + tracing::info!(path = %config.display(), "booting node"); + cloud9_node::launch(node_config).await.map_err(|error| miette::miette!("{error:#}"))?; } Command::CheckConfig { config } => { - load_required_node_config(&config).context("configuration check failed")?; + load_node_config(&config).context("configuration check failed")?; tracing::info!(path = %config.display(), "configuration OK"); } } @@ -74,12 +74,18 @@ async fn main() -> Result<()> { } fn init_tracing(verbosity: u8, color_enabled: bool) -> Result<()> { - let filter = if verbosity == 0 { - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")) + let default_filter = if verbosity == 0 { + "info" } else if verbosity == 1 { - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("cloud9=debug")) + "cloud9=debug" } else { - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("cloud9=trace")) + "cloud9=trace" + }; + let filter = if std::env::var_os("RUST_LOG").is_some() { + let value = std::env::var("RUST_LOG").into_diagnostic()?; + EnvFilter::try_new(value).into_diagnostic()? + } else { + EnvFilter::new(default_filter) }; let fmt_layer = fmt::layer() @@ -124,26 +130,12 @@ struct PeerSection { } fn load_node_config(path: &Path) -> Result { - if path.exists() { load_required_node_config(path) } else { Ok(NodeConfig::default()) } -} - -fn load_required_node_config(path: &Path) -> Result { - let contents = - load_config(path)?.ok_or_else(|| miette::miette!("config `{}` missing", path.display()))?; + let contents = fs::read_to_string(path) + .into_diagnostic() + .with_context(|| format!("reading configuration from `{}`", path.display()))?; parse_node_config(&contents).with_context(|| format!("parsing `{}`", path.display())) } -fn load_config(path: &Path) -> Result> { - if path.exists() { - fs::read_to_string(path) - .into_diagnostic() - .map(Some) - .with_context(|| format!("reading configuration from `{}`", path.display())) - } else { - Ok(None) - } -} - fn parse_node_config(contents: &str) -> Result { let config: ConfigFile = toml::from_str(contents).into_diagnostic()?; let node_id = NodeId(config.node.id); @@ -193,3 +185,16 @@ fn resolve_peer_addr(host: &str, port: u16) -> Result { )), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn invariant_startup_requires_a_configuration_file() { + let dir = tempfile::tempdir().into_diagnostic().unwrap(); + let error = load_node_config(&dir.path().join("missing.toml")).unwrap_err(); + + assert!(error.to_string().contains("missing")); + } +} From 49d76bc8d46aa5271bc871647e704eb4b4b454bf Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:18:01 -0700 Subject: [PATCH 05/17] fix(node): harden replicated request handling --- Cargo.lock | 97 ++++++++++++++++++++ Cargo.toml | 16 ++-- cloud9-node/Cargo.toml | 3 + cloud9-node/src/auth.rs | 102 +++++++++++++++++++++ cloud9-node/src/command.rs | 108 +++++++++++++++++----- cloud9-node/src/config.rs | 24 ++--- cloud9-node/src/lib.rs | 2 + cloud9-node/src/runtime.rs | 77 +++++++++++----- cloud9-node/src/service.rs | 6 +- cloud9-node/src/store.rs | 1 + cloud9-node/src/tests.rs | 172 ++++++++++++++++++++++++++++++++++- cloud9-node/src/transport.rs | 136 +++++++++++++++++++++++++-- cloud9-proto/Cargo.toml | 1 + cloud9.example.toml | 14 +++ cloud9/src/main.rs | 77 ++++++++++++---- 15 files changed, 736 insertions(+), 100 deletions(-) create mode 100644 cloud9-node/src/auth.rs create mode 100644 cloud9.example.toml diff --git a/Cargo.lock b/Cargo.lock index 1e61851..827a42e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -185,6 +185,15 @@ dependencies = [ "wyz", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "buffa" version = "0.5.2" @@ -341,8 +350,10 @@ dependencies = [ "cloud9-storage", "cloud9-wal", "connectrpc", + "hmac", "serde", "serde_json", + "sha2", "tempfile", "thiserror", "tokio", @@ -406,6 +417,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.4" @@ -467,6 +484,21 @@ dependencies = [ "syn", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -476,6 +508,24 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "deranged" version = "0.5.5" @@ -485,6 +535,18 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "ctutils", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -722,6 +784,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "1.4.0" @@ -767,6 +838,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.9.0" @@ -1327,6 +1407,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1715,6 +1806,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unarray" version = "0.1.4" diff --git a/Cargo.toml b/Cargo.toml index 8972010..495dafa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,13 +43,13 @@ unimplemented = "warn" [workspace.dependencies] # Internal crates -cloud9-core = { path = "cloud9-core" } -cloud9-storage = { path = "cloud9-storage" } -cloud9-wal = { path = "cloud9-wal" } -cloud9-raft = { path = "consensus/cloud9-raft" } -cloud9-raft-io = { path = "consensus/cloud9-raft-io" } -cloud9-proto = { path = "cloud9-proto" } -cloud9-node = { path = "cloud9-node" } +cloud9-core = { version = "0.0.1", path = "cloud9-core" } +cloud9-storage = { version = "0.0.1", path = "cloud9-storage" } +cloud9-wal = { version = "0.0.1", path = "cloud9-wal" } +cloud9-raft = { version = "0.0.1", path = "consensus/cloud9-raft" } +cloud9-raft-io = { version = "0.0.1", path = "consensus/cloud9-raft-io" } +cloud9-proto = { version = "0.0.1", path = "cloud9-proto" } +cloud9-node = { version = "0.0.1", path = "cloud9-node" } # External dependencies anyhow = "1" @@ -72,6 +72,8 @@ buffa = { version = "0.5.2", features = ["json"] } connectrpc = { version = "0.4.2", default-features = false } connectrpc-build = "0.4.2" http-body = "1" +hmac = "0.13" +sha2 = "0.11" # Testing - concurrency loom = "0.7.2" diff --git a/cloud9-node/Cargo.toml b/cloud9-node/Cargo.toml index 141ef14..dbe47a5 100644 --- a/cloud9-node/Cargo.toml +++ b/cloud9-node/Cargo.toml @@ -7,6 +7,7 @@ license = { workspace = true } authors = { workspace = true } repository = { workspace = true } homepage = { workspace = true } +description = "Cloud9 replicated database node" [lints] workspace = true @@ -20,10 +21,12 @@ cloud9-storage = { workspace = true } cloud9-wal = { workspace = true } cloud9-proto = { workspace = true } connectrpc = { workspace = true, features = ["axum"] } +hmac = { workspace = true } tracing = { workspace = true } thiserror = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } tokio = { workspace = true } [dev-dependencies] diff --git a/cloud9-node/src/auth.rs b/cloud9-node/src/auth.rs new file mode 100644 index 0000000..4e0a855 --- /dev/null +++ b/cloud9-node/src/auth.rs @@ -0,0 +1,102 @@ +//! Authentication for the Raft peer transport. + +use std::fmt; + +use hmac::{Hmac, KeyInit, Mac}; +use sha2::Sha256; +use thiserror::Error; + +const KEY_BYTES: usize = 32; +const SIGNATURE_BYTES: usize = 32; + +type HmacSha256 = Hmac; + +#[derive(Clone)] +pub struct RaftKey([u8; KEY_BYTES]); + +#[derive(Debug, Error)] +#[error("Raft key must be exactly 64 hexadecimal characters")] +pub struct InvalidRaftKey; + +impl RaftKey { + pub fn from_hex(value: &str) -> Result { + decode_hex(value).map(Self).ok_or(InvalidRaftKey) + } + + pub(crate) fn signature(&self, body: &[u8]) -> Option { + let mut mac = ::new_from_slice(&self.0).ok()?; + mac.update(body); + Some(encode_hex(&mac.finalize().into_bytes())) + } + + pub(crate) fn verify(&self, body: &[u8], signature: &str) -> bool { + let Some(signature) = decode_hex::(signature) else { + return false; + }; + let Ok(mut mac) = ::new_from_slice(&self.0) else { + return false; + }; + mac.update(body); + mac.verify_slice(&signature).is_ok() + } +} + +impl fmt::Debug for RaftKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("RaftKey([REDACTED])") + } +} + +fn decode_hex(value: &str) -> Option<[u8; N]> { + if value.len() != N * 2 { + return None; + } + let mut decoded = [0; N]; + for (output, pair) in decoded.iter_mut().zip(value.as_bytes().chunks_exact(2)) { + *output = nibble(pair[0])?.checked_mul(16)?.checked_add(nibble(pair[1])?)?; + } + Some(decoded) +} + +fn nibble(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +fn encode_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut encoded = String::with_capacity(bytes.len() * 2); + for byte in bytes { + encoded.push(char::from(HEX[usize::from(byte >> 4)])); + encoded.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + encoded +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn invariant_signature_binds_key_and_body() { + let key = RaftKey::from_hex(&"01".repeat(KEY_BYTES)).unwrap(); + let other_key = RaftKey::from_hex(&"02".repeat(KEY_BYTES)).unwrap(); + let signature = key.signature(b"message").unwrap(); + + assert!(key.verify(b"message", &signature)); + assert!(!key.verify(b"tampered", &signature)); + assert!(!other_key.verify(b"message", &signature)); + } + + #[test] + fn malformed_keys_and_signatures_are_rejected() { + assert!(RaftKey::from_hex("short").is_err()); + assert!(RaftKey::from_hex(&"zz".repeat(KEY_BYTES)).is_err()); + let key = RaftKey::from_hex(&"01".repeat(KEY_BYTES)).unwrap(); + assert!(!key.verify(b"message", "invalid")); + } +} diff --git a/cloud9-node/src/command.rs b/cloud9-node/src/command.rs index 8d11028..b5a1471 100644 --- a/cloud9-node/src/command.rs +++ b/cloud9-node/src/command.rs @@ -9,6 +9,11 @@ use cloud9_proto::generated::cloud9::kv::v1::{ use connectrpc::ConnectError; use serde::{Deserialize, Serialize}; +pub(crate) const MAX_VALUE_BYTES: usize = 64 * 1024; +pub(crate) const MAX_NAMESPACE_BYTES: usize = 255; +pub(crate) const MAX_KEY_BYTES: usize = 1024; +pub(crate) const MAX_ETAG_BYTES: usize = 128; + #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) enum KvCommand { RegisterSession, @@ -96,6 +101,7 @@ impl KvState { ) -> Result { validate_mutation_request(client_id, sequence)?; validate_put_preconditions(if_match, if_none_match)?; + validate_value(&body)?; let name = KvName::new(namespace, key)?; let request = MutationRequest::Put { @@ -108,15 +114,23 @@ impl KvState { return Ok(KvApplyResult::Put(response)); } - check_put_preconditions(self.entries.get(&name), if_match, if_none_match)?; - let generation = self.next_generation()?; + if let Err(rejection) = + check_put_preconditions(self.entries.get(&name), if_match, if_none_match) + { + return self.reject(client_id, sequence, request, rejection); + } + let size = body_len(&body)?; + let generation = match self.next_generation() { + Ok(generation) => generation, + Err(rejection) => return self.reject(client_id, sequence, request, rejection), + }; let etag = etag_for(generation); let response = PutResponse { namespace: name.namespace.clone(), key: name.key.clone(), etag: etag.clone(), generation, - size: body_len(&body)?, + size, ..Default::default() }; self.entries.insert(name, KvRecord { body, etag, generation }); @@ -137,6 +151,7 @@ impl KvState { if_match: &str, ) -> Result { validate_mutation_request(client_id, sequence)?; + validate_etag(if_match)?; let name = KvName::new(namespace, key)?; let request = MutationRequest::Delete { name: name.clone(), if_match: if_match.to_owned() }; @@ -144,17 +159,12 @@ impl KvState { return Ok(KvApplyResult::Delete(response)); } - let removed = if let Some(record) = self.entries.get(&name) { - if !if_match.is_empty() && if_match != record.etag { - return Err(ConnectError::failed_precondition("ETag precondition failed")); - } - self.entries.remove(&name) - } else { - if !if_match.is_empty() { - return Err(ConnectError::failed_precondition("ETag precondition failed")); - } - None - }; + if !if_match.is_empty() + && self.entries.get(&name).is_none_or(|record| if_match != record.etag) + { + return self.reject(client_id, sequence, request, MutationRejection::EtagMismatch); + } + let removed = self.entries.remove(&name); let response = if let Some(record) = removed { DeleteResponse { @@ -193,15 +203,24 @@ impl KvState { Ok(client_id) } - fn next_generation(&mut self) -> Result { + fn next_generation(&mut self) -> Result { let generation = self.next_generation; - self.next_generation = self - .next_generation - .checked_add(1) - .ok_or_else(|| ConnectError::resource_exhausted("kv generation space exhausted"))?; + self.next_generation = + self.next_generation.checked_add(1).ok_or(MutationRejection::GenerationExhausted)?; Ok(generation) } + fn reject( + &mut self, + client_id: u64, + sequence: u64, + request: MutationRequest, + rejection: MutationRejection, + ) -> Result { + self.session_mut(client_id)?.record(sequence, request, MutationResult::Rejected(rejection)); + Err(rejection.connect_error()) + } + fn session(&self, client_id: u64) -> Result<&SessionState, ConnectError> { self.sessions .get(&client_id) @@ -226,9 +245,15 @@ impl KvName { if namespace.is_empty() { return Err(ConnectError::invalid_argument("namespace must not be empty")); } + if namespace.len() > MAX_NAMESPACE_BYTES { + return Err(ConnectError::invalid_argument("namespace exceeds 255-byte limit")); + } if key.is_empty() { return Err(ConnectError::invalid_argument("key must not be empty")); } + if key.len() > MAX_KEY_BYTES { + return Err(ConnectError::invalid_argument("key exceeds 1024-byte limit")); + } Ok(Self { namespace: namespace.to_owned(), key: key.to_owned() }) } } @@ -257,6 +282,26 @@ enum MutationRequest { enum MutationResult { Put(PutResponse), Delete(DeleteResponse), + Rejected(MutationRejection), +} + +#[derive(Clone, Copy)] +enum MutationRejection { + KeyExists, + EtagMismatch, + GenerationExhausted, +} + +impl MutationRejection { + fn connect_error(self) -> ConnectError { + match self { + Self::KeyExists => ConnectError::failed_precondition("key already exists"), + Self::EtagMismatch => ConnectError::failed_precondition("ETag precondition failed"), + Self::GenerationExhausted => { + ConnectError::resource_exhausted("kv generation space exhausted") + } + } + } } impl SessionState { @@ -281,6 +326,7 @@ pub(crate) fn validate_put_preconditions( if_match: &str, if_none_match: bool, ) -> Result<(), ConnectError> { + validate_etag(if_match)?; if !if_match.is_empty() && if_none_match { return Err(ConnectError::invalid_argument( "if_match and if_none_match are mutually exclusive", @@ -289,20 +335,32 @@ pub(crate) fn validate_put_preconditions( Ok(()) } +pub(crate) fn validate_etag(etag: &str) -> Result<(), ConnectError> { + if etag.len() > MAX_ETAG_BYTES { + return Err(ConnectError::invalid_argument("ETag exceeds 128-byte limit")); + } + Ok(()) +} + +pub(crate) fn validate_value(body: &[u8]) -> Result<(), ConnectError> { + if body.len() > MAX_VALUE_BYTES { + return Err(ConnectError::resource_exhausted("value exceeds 65536-byte limit")); + } + Ok(()) +} + fn check_put_preconditions( current: Option<&KvRecord>, if_match: &str, if_none_match: bool, -) -> Result<(), ConnectError> { +) -> Result<(), MutationRejection> { if if_none_match && current.is_some() { - return Err(ConnectError::failed_precondition("key already exists")); + return Err(MutationRejection::KeyExists); } if !if_match.is_empty() { match current { Some(record) if record.etag == if_match => {} - Some(_) | None => { - return Err(ConnectError::failed_precondition("ETag precondition failed")); - } + Some(_) | None => return Err(MutationRejection::EtagMismatch), } } Ok(()) @@ -316,6 +374,7 @@ fn cached_put( ) -> Result, ConnectError> { match cached_mutation(state.session(client_id)?, sequence, request)? { Some(MutationResult::Put(response)) => Ok(Some(response)), + Some(MutationResult::Rejected(rejection)) => Err(rejection.connect_error()), Some(MutationResult::Delete(_)) => Err(ConnectError::internal("session result mismatch")), None => Ok(None), } @@ -329,6 +388,7 @@ fn cached_delete( ) -> Result, ConnectError> { match cached_mutation(state.session(client_id)?, sequence, request)? { Some(MutationResult::Delete(response)) => Ok(Some(response)), + Some(MutationResult::Rejected(rejection)) => Err(rejection.connect_error()), Some(MutationResult::Put(_)) => Err(ConnectError::internal("session result mismatch")), None => Ok(None), } diff --git a/cloud9-node/src/config.rs b/cloud9-node/src/config.rs index 1f3eaed..0466df9 100644 --- a/cloud9-node/src/config.rs +++ b/cloud9-node/src/config.rs @@ -1,12 +1,14 @@ //! Node runtime configuration. use std::collections::BTreeMap; -use std::net::{Ipv4Addr, SocketAddr}; +use std::net::SocketAddr; use std::path::{Path, PathBuf}; use cloud9_raft::{ConsensusConfig, NodeId}; use cloud9_storage::StorageOptions; +use crate::RaftKey; + /// Runtime configuration derived from CLI flags and config files. #[derive(Debug, Clone)] pub struct NodeConfig { @@ -14,25 +16,11 @@ pub struct NodeConfig { pub client_addr: SocketAddr, pub raft_addr: SocketAddr, pub peers: BTreeMap, + pub raft_key: RaftKey, pub storage: StorageOptions, pub consensus: ConsensusConfig, } -impl Default for NodeConfig { - fn default() -> Self { - let node_id = NodeId(0); - let raft_addr = SocketAddr::from((Ipv4Addr::LOCALHOST, 19_091)); - Self { - node_id, - client_addr: SocketAddr::from((Ipv4Addr::LOCALHOST, 19_090)), - raft_addr, - peers: BTreeMap::from([(node_id, raft_addr)]), - storage: StorageOptions::default(), - consensus: raft_config(node_id), - } - } -} - impl NodeConfig { #[must_use] pub(crate) fn raft_dir(&self) -> PathBuf { @@ -42,5 +30,7 @@ impl NodeConfig { #[must_use] pub fn raft_config(node_id: NodeId) -> ConsensusConfig { - ConsensusConfig::new(node_id).with_parallel_disk_write(false) + let mut config = ConsensusConfig::new(node_id).with_parallel_disk_write(false); + config.max_entries_per_msg = 1; + config } diff --git a/cloud9-node/src/lib.rs b/cloud9-node/src/lib.rs index d9e68d3..847fb5e 100644 --- a/cloud9-node/src/lib.rs +++ b/cloud9-node/src/lib.rs @@ -4,6 +4,7 @@ //! Top-level orchestration for Cloud9 nodes. +mod auth; mod command; mod config; mod runtime; @@ -13,6 +14,7 @@ mod store; mod tests; mod transport; +pub use auth::RaftKey; pub use config::{NodeConfig, raft_config}; /// Launch the node's public KV API and Raft peer API. diff --git a/cloud9-node/src/runtime.rs b/cloud9-node/src/runtime.rs index f8a9885..cb3fb1f 100644 --- a/cloud9-node/src/runtime.rs +++ b/cloud9-node/src/runtime.rs @@ -13,12 +13,11 @@ use connectrpc::ConnectError; use thiserror::Error; use tokio::sync::{Mutex, RwLock, oneshot}; use tokio::time::{Duration, sleep, timeout}; -use tracing::warn; use crate::command::{KvApplyResult, KvCommand, KvState}; use crate::config::NodeConfig; use crate::store::{RaftStore, StoreError}; -use crate::transport::post_raft_message; +use crate::transport::PeerTransport; const TICK_INTERVAL: Duration = Duration::from_millis(1); const PROPOSAL_TIMEOUT: Duration = Duration::from_secs(5); @@ -50,11 +49,28 @@ struct RaftMachine { store: RaftStore, } +struct PendingProposal { + command: Vec, + sender: oneshot::Sender>, +} + +impl PendingProposal { + fn complete(self, command: &Command, result: Result) { + let result = if self.command == command.0 { + result + } else { + Err(ConnectError::aborted("Raft proposal was superseded")) + }; + let _ = self.sender.send(result); + } +} + pub(crate) struct RaftRuntime { config: NodeConfig, machine: Mutex, state: Arc>, - waiters: Mutex>>>, + waiters: Mutex>, + transport: PeerTransport, failed: AtomicBool, } @@ -67,11 +83,13 @@ impl RaftRuntime { let initial = RaftNode::new(config.consensus.clone(), &voters); let store = RaftStore::open(&config.raft_dir(), initial.persistent().clone())?; let node = RaftNode::restore(config.consensus.clone(), store.persistent().clone()); + let transport = PeerTransport::new(config.node_id, config.raft_key.clone(), &config.peers); Ok(Self { config, machine: Mutex::new(RaftMachine { node, store }), state, waiters: Mutex::new(HashMap::new()), + transport, failed: AtomicBool::new(false), }) } @@ -106,16 +124,22 @@ impl RaftRuntime { Ok(()) } + pub(crate) fn verify_signature(&self, body: &[u8], signature: &str) -> bool { + self.config.raft_key.verify(body, signature) + } + pub(crate) async fn propose(&self, command: KvCommand) -> Result { self.ensure_healthy().map_err(|error| runtime_connect_error(&error))?; let bytes = serde_json::to_vec(&command) .map_err(|_| ConnectError::internal("failed to encode Raft command"))?; let (index, receiver) = { let mut machine = self.machine.lock().await; - let (index, effects) = - machine.node.propose(Command(bytes)).map_err(|error| propose_error(&error))?; + let (index, effects) = machine + .node + .propose(Command(bytes.clone())) + .map_err(|error| propose_error(&error))?; let (sender, receiver) = oneshot::channel(); - self.waiters.lock().await.insert(index, sender); + self.waiters.lock().await.insert(index, PendingProposal { command: bytes, sender }); if let Err(error) = self.handle_effects(&mut machine, effects).await { self.waiters.lock().await.remove(&index); let error = self.fail(error); @@ -187,7 +211,7 @@ impl RaftRuntime { let mut applied_to = None; for entry in entries { let result = self.apply_command(&entry.command).await; - self.complete_waiter(entry.index, result).await; + self.complete_waiter(entry.index, &entry.command, result).await; applied_to = Some(entry.index); } if let Some(index) = applied_to { @@ -201,25 +225,19 @@ impl RaftRuntime { self.state.write().await.apply(command) } - async fn complete_waiter(&self, index: LogIndex, result: Result) { - if let Some(sender) = self.waiters.lock().await.remove(&index) { - let _ = sender.send(result); + async fn complete_waiter( + &self, + index: LogIndex, + command: &Command, + result: Result, + ) { + if let Some(pending) = self.waiters.lock().await.remove(&index) { + pending.complete(command, result); } } fn send_message(&self, message: Message) -> Result<(), RuntimeError> { - let addr = self - .config - .peers - .get(&message.to) - .copied() - .ok_or(RuntimeError::UnknownPeer { peer: message.to })?; - tokio::spawn(async move { - if let Err(error) = post_raft_message(addr, &message).await { - warn!(%error, to = message.to.0, "failed to send Raft message"); - } - }); - Ok(()) + self.transport.send(message).map_err(|peer| RuntimeError::UnknownPeer { peer }) } fn ensure_healthy(&self) -> Result<(), RuntimeError> { @@ -247,3 +265,18 @@ fn propose_error(error: &ProposeError) -> ConnectError { fn runtime_connect_error(error: &RuntimeError) -> ConnectError { ConnectError::internal(format!("Raft runtime failed: {error}")) } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn invariant_log_overwrite_cannot_complete_a_different_proposal() { + let (sender, receiver) = oneshot::channel(); + let pending = PendingProposal { command: b"original".to_vec(), sender }; + + pending.complete(&Command(b"replacement".to_vec()), Ok(KvApplyResult::ReadBarrier)); + + assert!(receiver.await.unwrap().is_err()); + } +} diff --git a/cloud9-node/src/service.rs b/cloud9-node/src/service.rs index 1ba51da..7f9275a 100644 --- a/cloud9-node/src/service.rs +++ b/cloud9-node/src/service.rs @@ -17,8 +17,8 @@ use tokio::sync::RwLock; use tracing::{info, instrument}; use crate::command::{ - KvApplyResult, KvCommand, KvName, KvState, body_len, key_not_found, validate_mutation_request, - validate_put_preconditions, + KvApplyResult, KvCommand, KvName, KvState, body_len, key_not_found, validate_etag, + validate_mutation_request, validate_put_preconditions, validate_value, }; use crate::config::NodeConfig; use crate::runtime::RaftRuntime; @@ -140,6 +140,7 @@ impl KvService for KvApi { ) -> ServiceResult { validate_mutation_request(request.client_id, request.sequence)?; validate_put_preconditions(request.if_match, request.if_none_match)?; + validate_value(request.body)?; KvName::new(request.namespace, request.key)?; let command = KvCommand::Put { @@ -167,6 +168,7 @@ impl KvService for KvApi { request: OwnedDeleteRequestView, ) -> ServiceResult { validate_mutation_request(request.client_id, request.sequence)?; + validate_etag(request.if_match)?; KvName::new(request.namespace, request.key)?; let command = KvCommand::Delete { diff --git a/cloud9-node/src/store.rs b/cloud9-node/src/store.rs index f1bf31a..381b555 100644 --- a/cloud9-node/src/store.rs +++ b/cloud9-node/src/store.rs @@ -83,6 +83,7 @@ impl RaftStore { fn recover(&mut self) -> Result<(), StoreError> { for stored in self.wal.records()? { + let stored = stored?; if stored.record.kind != self.kind { return Err(StoreError::UnexpectedKind { found: stored.record.kind.get() }); } diff --git a/cloud9-node/src/tests.rs b/cloud9-node/src/tests.rs index 89119ee..307f0c0 100644 --- a/cloud9-node/src/tests.rs +++ b/cloud9-node/src/tests.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::io::{Read, Write}; use std::net::{Ipv4Addr, SocketAddr}; use std::sync::Arc; @@ -5,14 +6,52 @@ use std::time::Duration; use anyhow::{Result, bail}; use cloud9_core::SharedString; +use cloud9_raft::NodeId; use cloud9_storage::StorageOptions; use tokio::net::TcpListener; use tokio::sync::RwLock; -use crate::command::{KvApplyResult, KvCommand, KvName, KvState}; +use crate::RaftKey; +use crate::command::{KvApplyResult, KvCommand, KvName, KvState, MAX_VALUE_BYTES}; use crate::config::NodeConfig; use crate::runtime::RaftRuntime; use crate::service::{KvApi, kv_app}; +use crate::transport::raft_app; + +#[test] +fn invariant_committed_failure_is_idempotent() -> Result<()> { + let mut state = KvState::new(); + let client_one = registered_client(&mut state)?; + let client_two = registered_client(&mut state)?; + state.apply(put_command(client_one, 1, false))?; + + assert!(state.apply(put_command(client_one, 2, true)).is_err()); + state.apply(KvCommand::Delete { + client_id: client_two, + sequence: 1, + namespace: "test".to_owned(), + key: "key".to_owned(), + if_match: String::new(), + })?; + + assert!(state.apply(put_command(client_one, 2, true)).is_err()); + assert!(!state.entries.contains_key(&KvName::new("test", "key")?)); + Ok(()) +} + +#[test] +fn invariant_oversized_values_never_enter_the_state_machine() -> Result<()> { + let mut state = KvState::new(); + let client_id = registered_client(&mut state)?; + let mut command = put_command(client_id, 1, false); + if let KvCommand::Put { body, .. } = &mut command { + *body = vec![0; MAX_VALUE_BYTES + 1]; + } + + assert!(state.apply(command).is_err()); + assert!(state.entries.is_empty()); + Ok(()) +} #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn invariant_mutation_sequence_identifies_exact_request() -> Result<()> { @@ -105,13 +144,110 @@ async fn invariant_restart_recovers_committed_state() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn invariant_three_nodes_replicate_and_fail_over() -> Result<()> { + let dir = tempfile::tempdir()?; + let mut listeners = Vec::new(); + for _ in 0..3 { + listeners.push(TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?); + } + let peers = listeners + .iter() + .enumerate() + .map(|(id, listener)| Ok((NodeId(u64::try_from(id)?), listener.local_addr()?))) + .collect::>>()?; + let key = RaftKey::from_hex(&"01".repeat(32))?; + let mut states = Vec::new(); + let mut runtimes = Vec::new(); + let mut servers = Vec::new(); + let mut drivers = Vec::new(); + + for (id, listener) in listeners.into_iter().enumerate() { + let node_id = NodeId(u64::try_from(id)?); + let state = Arc::new(RwLock::new(KvState::new())); + let config = NodeConfig { + node_id, + client_addr: SocketAddr::from((Ipv4Addr::LOCALHOST, 0)), + raft_addr: listener.local_addr()?, + peers: peers.clone(), + raft_key: key.clone(), + storage: StorageOptions { + name: SharedString::literal("test"), + data_dir: SharedString::from(dir.path().join(id.to_string()).to_string_lossy()), + }, + consensus: crate::raft_config(node_id), + }; + let runtime = Arc::new(RaftRuntime::open(config, state.clone())?); + servers.push(tokio::spawn(axum::serve(listener, raft_app(runtime.clone())).into_future())); + drivers.push(tokio::spawn(runtime.clone().run())); + states.push(state); + runtimes.push(runtime); + } + + let leader = wait_for_cluster_leader(&runtimes, &[0, 1, 2]).await?; + let client_id = registered_session(&runtimes[leader]).await?; + runtimes[leader].propose(put_command(client_id, 1, false)).await?; + wait_for_replicated_key(&states).await?; + + drivers[leader].abort(); + servers[leader].abort(); + let survivors = (0..3).filter(|node| *node != leader).collect::>(); + let replacement = wait_for_cluster_leader(&runtimes, &survivors).await?; + runtimes[replacement].read_barrier().await?; + + for task in drivers { + task.abort(); + } + for task in servers { + task.abort(); + } + Ok(()) +} + fn test_config(path: &std::path::Path) -> NodeConfig { + let node_id = cloud9_raft::NodeId(0); + let raft_addr = SocketAddr::from((Ipv4Addr::LOCALHOST, 19_091)); NodeConfig { + node_id, + client_addr: SocketAddr::from((Ipv4Addr::LOCALHOST, 19_090)), + raft_addr, + peers: std::collections::BTreeMap::from([(node_id, raft_addr)]), + raft_key: RaftKey::from_hex(&"01".repeat(32)).unwrap(), storage: StorageOptions { name: SharedString::literal("test"), data_dir: SharedString::from(path.to_string_lossy()), }, - ..NodeConfig::default() + consensus: crate::raft_config(node_id), + } +} + +fn registered_client(state: &mut KvState) -> Result { + match state.apply(KvCommand::RegisterSession)? { + KvApplyResult::RegisterSession(response) => Ok(response.client_id), + KvApplyResult::Put(_) | KvApplyResult::Delete(_) | KvApplyResult::ReadBarrier => { + bail!("session command returned the wrong result") + } + } +} + +async fn registered_session(runtime: &RaftRuntime) -> Result { + match runtime.propose(KvCommand::RegisterSession).await? { + KvApplyResult::RegisterSession(response) => Ok(response.client_id), + KvApplyResult::Put(_) | KvApplyResult::Delete(_) | KvApplyResult::ReadBarrier => { + bail!("session proposal returned the wrong result") + } + } +} + +fn put_command(client_id: u64, sequence: u64, if_none_match: bool) -> KvCommand { + KvCommand::Put { + client_id, + sequence, + namespace: "test".to_owned(), + key: "key".to_owned(), + body: b"value".to_vec(), + if_match: String::new(), + if_none_match, } } @@ -125,6 +261,38 @@ async fn wait_for_runtime_leader(runtime: &RaftRuntime) -> Result<()> { bail!("single-node Raft runtime did not elect a leader") } +async fn wait_for_cluster_leader(runtimes: &[Arc], nodes: &[usize]) -> Result { + for _ in 0..500 { + let mut leader = None; + for node in nodes { + if runtimes[*node].mode().await == "leader" && leader.replace(*node).is_some() { + leader = None; + break; + } + } + if let Some(leader) = leader { + return Ok(leader); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + bail!("Raft cluster did not elect exactly one leader") +} + +async fn wait_for_replicated_key(states: &[Arc>]) -> Result<()> { + let name = KvName::new("test", "key")?; + for _ in 0..200 { + let mut present = true; + for state in states { + present &= state.read().await.entries.contains_key(&name); + } + if present { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + bail!("committed key did not reach every state machine") +} + async fn wait_for_leader(addr: SocketAddr) -> Result<()> { for _ in 0..100 { let (status, body) = post_json(addr, "Status", "{}")?; diff --git a/cloud9-node/src/transport.rs b/cloud9-node/src/transport.rs index f17213e..d9d366b 100644 --- a/cloud9-node/src/transport.rs +++ b/cloud9-node/src/transport.rs @@ -1,33 +1,96 @@ //! Raft peer HTTP transport. +use std::collections::BTreeMap; use std::net::SocketAddr; +use std::sync::Arc; use anyhow::{Context, Result}; -use axum::Json; use axum::Router as AxumRouter; -use axum::extract::State; -use axum::http::StatusCode; +use axum::body::Bytes; +use axum::extract::{DefaultBodyLimit, State}; +use axum::http::{HeaderMap, StatusCode}; use axum::routing::post; +use cloud9_raft::NodeId; use cloud9_raft::raft::Message; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; +use tokio::sync::Semaphore; use tokio::time::{Duration, timeout}; +use tracing::warn; -use std::sync::Arc; - +use crate::RaftKey; use crate::runtime::RaftRuntime; const RAFT_RPC_TIMEOUT: Duration = Duration::from_secs(1); const MAX_RESPONSE_BYTES: u64 = 8 * 1024; +const MAX_RAFT_MESSAGE_BYTES: usize = 2 * 1024 * 1024; +const MAX_IN_FLIGHT_PER_PEER: usize = 16; +const SIGNATURE_HEADER: &str = "x-cloud9-raft-signature"; + +struct Peer { + addr: SocketAddr, + permits: Arc, +} + +pub(crate) struct PeerTransport { + peers: BTreeMap, + key: RaftKey, +} + +impl PeerTransport { + pub(crate) fn new(node_id: NodeId, key: RaftKey, peers: &BTreeMap) -> Self { + let peers = peers + .iter() + .filter(|(peer, _)| **peer != node_id) + .map(|(peer, addr)| { + ( + *peer, + Peer { addr: *addr, permits: Arc::new(Semaphore::new(MAX_IN_FLIGHT_PER_PEER)) }, + ) + }) + .collect(); + Self { peers, key } + } + + pub(crate) fn send(&self, message: Message) -> Result<(), NodeId> { + let peer = self.peers.get(&message.to).ok_or(message.to)?; + let Ok(permit) = peer.permits.clone().try_acquire_owned() else { + warn!(to = message.to.0, "dropping Raft message at peer concurrency limit"); + return Ok(()); + }; + let addr = peer.addr; + let key = self.key.clone(); + tokio::spawn(async move { + let _permit = permit; + if let Err(error) = post_raft_message(addr, &key, &message).await { + warn!(%error, to = message.to.0, "failed to send Raft message"); + } + }); + Ok(()) + } +} pub(crate) fn raft_app(runtime: Arc) -> AxumRouter { - AxumRouter::new().route("/raft/message", post(receive_raft)).with_state(runtime) + AxumRouter::new() + .route("/raft/message", post(receive_raft)) + .layer(DefaultBodyLimit::max(MAX_RAFT_MESSAGE_BYTES)) + .with_state(runtime) } async fn receive_raft( State(runtime): State>, - Json(message): Json, + headers: HeaderMap, + body: Bytes, ) -> Result { + let signature = headers + .get(SIGNATURE_HEADER) + .and_then(|value| value.to_str().ok()) + .ok_or((StatusCode::UNAUTHORIZED, "missing Raft signature".to_owned()))?; + if !runtime.verify_signature(&body, signature) { + return Err((StatusCode::UNAUTHORIZED, "invalid Raft signature".to_owned())); + } + let message = serde_json::from_slice(&body) + .map_err(|error| (StatusCode::BAD_REQUEST, format!("invalid Raft message: {error}")))?; runtime .validate_message(&message) .map_err(|error| (StatusCode::BAD_REQUEST, error.to_string()))?; @@ -38,14 +101,22 @@ async fn receive_raft( Ok(StatusCode::NO_CONTENT) } -pub(crate) async fn post_raft_message(addr: SocketAddr, message: &Message) -> Result<()> { - timeout(RAFT_RPC_TIMEOUT, post_raft_message_inner(addr, message)) +pub(crate) async fn post_raft_message( + addr: SocketAddr, + key: &RaftKey, + message: &Message, +) -> Result<()> { + timeout(RAFT_RPC_TIMEOUT, post_raft_message_inner(addr, key, message)) .await .with_context(|| format!("Raft RPC to {addr} timed out"))? } -async fn post_raft_message_inner(addr: SocketAddr, message: &Message) -> Result<()> { +async fn post_raft_message_inner(addr: SocketAddr, key: &RaftKey, message: &Message) -> Result<()> { let body = serde_json::to_vec(message).context("encoding Raft message")?; + if body.len() > MAX_RAFT_MESSAGE_BYTES { + anyhow::bail!("encoded Raft message exceeds {MAX_RAFT_MESSAGE_BYTES}-byte limit"); + } + let signature = key.signature(&body).context("signing Raft message")?; let mut stream = TcpStream::connect(addr) .await .with_context(|| format!("connecting to Raft peer {addr}"))?; @@ -53,6 +124,7 @@ async fn post_raft_message_inner(addr: SocketAddr, message: &Message) -> Result< "POST /raft/message HTTP/1.1\r\n\ Host: {addr}\r\n\ Content-Type: application/json\r\n\ + {SIGNATURE_HEADER}: {signature}\r\n\ Content-Length: {}\r\n\ Connection: close\r\n\ \r\n", @@ -74,3 +146,47 @@ async fn post_raft_message_inner(addr: SocketAddr, message: &Message) -> Result< let response = String::from_utf8_lossy(&response); anyhow::bail!("Raft peer {addr} rejected message: {response}"); } + +#[cfg(test)] +mod tests { + use cloud9_raft::raft::{AppendRequest, Entry, EntryPayload, Payload}; + use cloud9_raft::{Command, NodeId}; + + use super::*; + use crate::command::{ + KvCommand, MAX_ETAG_BYTES, MAX_KEY_BYTES, MAX_NAMESPACE_BYTES, MAX_VALUE_BYTES, + }; + + #[test] + fn invariant_largest_command_fits_one_raft_message() { + let command = KvCommand::Put { + client_id: u64::MAX, + sequence: u64::MAX, + namespace: "\u{1}".repeat(MAX_NAMESPACE_BYTES), + key: "\u{1}".repeat(MAX_KEY_BYTES), + body: vec![u8::MAX; MAX_VALUE_BYTES], + if_match: "\u{1}".repeat(MAX_ETAG_BYTES), + if_none_match: false, + }; + let command = Command(serde_json::to_vec(&command).unwrap()); + let message = Message { + from: NodeId(u64::MAX), + to: NodeId(u64::MAX), + term: u64::MAX, + payload: Payload::AppendRequest(AppendRequest { + prev_log_index: u64::MAX, + prev_log_term: u64::MAX, + entries: vec![Entry { + term: u64::MAX, + index: u64::MAX, + payload: EntryPayload::Command(command), + }], + leader_commit: u64::MAX, + }), + }; + + let encoded = serde_json::to_vec(&message).unwrap(); + + assert!(encoded.len() <= MAX_RAFT_MESSAGE_BYTES, "encoded {} bytes", encoded.len()); + } +} diff --git a/cloud9-proto/Cargo.toml b/cloud9-proto/Cargo.toml index 9178683..1e45498 100644 --- a/cloud9-proto/Cargo.toml +++ b/cloud9-proto/Cargo.toml @@ -7,6 +7,7 @@ license = { workspace = true } authors = { workspace = true } repository = { workspace = true } homepage = { workspace = true } +description = "Cloud9 Connect RPC protocol types" [lints] workspace = true diff --git a/cloud9.example.toml b/cloud9.example.toml new file mode 100644 index 0000000..2bd7b33 --- /dev/null +++ b/cloud9.example.toml @@ -0,0 +1,14 @@ +[node] +id = 0 +host = "127.0.0.1" +client_port = 19090 +raft_port = 19091 + +[storage] +data_dir = "./cloud9-data" + +[cluster] +raft_key = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" +peers = [ + { id = 0, host = "127.0.0.1", raft_port = 19091 }, +] diff --git a/cloud9/src/main.rs b/cloud9/src/main.rs index 390f970..a6257d6 100644 --- a/cloud9/src/main.rs +++ b/cloud9/src/main.rs @@ -3,12 +3,12 @@ #![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::panic))] use std::collections::BTreeMap; -use std::net::{Ipv4Addr, SocketAddr, ToSocketAddrs}; +use std::net::{SocketAddr, ToSocketAddrs}; use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; use cloud9_core::{SharedString, fs, install_diagnostics}; -use cloud9_node::{NodeConfig, raft_config}; +use cloud9_node::{NodeConfig, RaftKey, raft_config}; use cloud9_raft::NodeId; use cloud9_storage::StorageOptions; use miette::{Context, IntoDiagnostic, Result}; @@ -106,8 +106,7 @@ struct ConfigFile { #[derive(Debug, Deserialize)] struct NodeSection { id: u64, - #[serde(rename = "host")] - _host: String, + host: String, client_port: u16, raft_port: u16, } @@ -119,6 +118,7 @@ struct StorageSection { #[derive(Debug, Deserialize)] struct ClusterSection { + raft_key: String, peers: Vec, } @@ -139,11 +139,21 @@ fn load_node_config(path: &Path) -> Result { fn parse_node_config(contents: &str) -> Result { let config: ConfigFile = toml::from_str(contents).into_diagnostic()?; let node_id = NodeId(config.node.id); - let client_addr = bind_addr(config.node.client_port); - let raft_addr = bind_addr(config.node.raft_port); + let client_addr = resolve_peer_addr(&config.node.host, config.node.client_port)?; + let raft_addr = resolve_peer_addr(&config.node.host, config.node.raft_port)?; + let raft_key = RaftKey::from_hex(&config.cluster.raft_key).into_diagnostic()?; let peers = peer_addrs(&config.cluster.peers)?; - if !peers.contains_key(&node_id) { - return Err(miette::miette!("cluster.peers must include node.id {}", node_id.0)); + match peers.get(&node_id) { + Some(peer_addr) if *peer_addr == raft_addr => {} + Some(peer_addr) => { + return Err(miette::miette!( + "cluster peer {} is {peer_addr}, expected node Raft address {raft_addr}", + node_id.0 + )); + } + None => { + return Err(miette::miette!("cluster.peers must include node.id {}", node_id.0)); + } } Ok(NodeConfig { @@ -151,6 +161,7 @@ fn parse_node_config(contents: &str) -> Result { client_addr, raft_addr, peers, + raft_key, storage: StorageOptions { name: SharedString::from("default"), data_dir: SharedString::from(config.storage.data_dir), @@ -159,15 +170,19 @@ fn parse_node_config(contents: &str) -> Result { }) } -fn bind_addr(port: u16) -> SocketAddr { - SocketAddr::from((Ipv4Addr::UNSPECIFIED, port)) -} - fn peer_addrs(peers: &[PeerSection]) -> Result> { - peers - .iter() - .map(|peer| Ok((NodeId(peer.id), resolve_peer_addr(&peer.host, peer.raft_port)?))) - .collect() + let mut addrs = BTreeMap::new(); + for peer in peers { + let node_id = NodeId(peer.id); + let addr = resolve_peer_addr(&peer.host, peer.raft_port)?; + if addrs.insert(node_id, addr).is_some() { + return Err(miette::miette!("cluster.peers contains duplicate node id {}", peer.id)); + } + } + if addrs.is_empty() { + return Err(miette::miette!("cluster.peers must not be empty")); + } + Ok(addrs) } fn resolve_peer_addr(host: &str, port: u16) -> Result { @@ -197,4 +212,34 @@ mod tests { assert!(error.to_string().contains("missing")); } + + #[test] + fn example_configuration_is_valid() { + let config = parse_node_config(include_str!("../../cloud9.example.toml")).unwrap(); + + assert_eq!(config.node_id, NodeId(0)); + assert_eq!(config.peers.len(), 1); + } + + #[test] + fn invariant_peer_ids_are_unique() { + let contents = include_str!("../../cloud9.example.toml").replace( + "peers = [\n { id = 0, host = \"127.0.0.1\", raft_port = 19091 },\n]", + "peers = [\n { id = 0, host = \"127.0.0.1\", raft_port = 19091 },\n { id = 0, host = \"127.0.0.1\", raft_port = 19092 },\n]", + ); + + let error = parse_node_config(&contents).unwrap_err(); + + assert!(error.to_string().contains("duplicate node id 0")); + } + + #[test] + fn invariant_raft_key_is_256_bits() { + let contents = include_str!("../../cloud9.example.toml") + .replace("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "short"); + + let error = parse_node_config(&contents).unwrap_err(); + + assert!(error.to_string().contains("64 hexadecimal characters")); + } } From 4b0f4e88a992dab1ad49a95641f29bf4310be527 Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:18:08 -0700 Subject: [PATCH 06/17] test(node): harden Jepsen failure detection --- .gitignore | 1 + jepsen/README.md | 20 ++++++++--- jepsen/scripts/build-target.sh | 5 +++ jepsen/src/cloud9/jepsen.clj | 36 ++++++++++++++----- jepsen/test/cloud9/jepsen_test.clj | 56 ++++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 13 deletions(-) create mode 100644 jepsen/test/cloud9/jepsen_test.clj diff --git a/.gitignore b/.gitignore index 85e14fe..c596a52 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ jepsen/store/ jepsen/target/ jepsen/c9-linux-* jepsen/.lein-failures +jepsen/.nrepl-port # LLM tooling **/.agent/ diff --git a/jepsen/README.md b/jepsen/README.md index d178ba3..ff107af 100644 --- a/jepsen/README.md +++ b/jepsen/README.md @@ -6,6 +6,10 @@ starts `c9 start --config /opt/cloud9/cloud9.toml` with Jepsen's `start-daemon!`, and drives Cloud9's public KV API with a shared linearizable register workload. +Each node requires the same 256-bit `cluster.raft_key`; peer RPC bodies are +authenticated with HMAC-SHA256 before deserialization. The checked-in example +key is only for local and Jepsen testing. + Cloud9 is a relational database first: Postgres-compatible SQL and native KV are peer APIs over one MVCC storage layer, one transactional IR, one timestamp system, and one transaction coordinator. The KV workload here is the smallest @@ -18,6 +22,10 @@ the same transactional IR. ## Build +Build on a Linux Jepsen control host with the same CPU architecture as the DB +nodes. The helper rejects host-native macOS builds because Jepsen uploads this +binary directly to Debian. + ```bash ./jepsen/scripts/build-target.sh ``` @@ -44,12 +52,14 @@ lein run test --nodes-file ~/nodes --username root --time-limit 60 --concurrency lein run serve ``` -The harness discovers the current Raft leader before opening client sessions and -rediscovers it after failover. Followers reject mutating KV RPCs rather than +The harness discovers the current Raft leader before opening client sessions, +rediscovers it after failover, then heals the final fault and reads from every +client thread before checking the history. Followers reject KV RPCs rather than serving node-local state. ## Current Limit -`c9 start` now drives KV commands through Raft, but this is still a transient -runtime: Raft persistence, snapshot transfer, read forwarding, and richer nemesis -coverage are intentionally not complete yet. +`c9 start` persists Raft hard state and log entries before sending network +effects, then reconstructs KV state by replaying committed commands. Snapshot +transfer, log compaction, read forwarding, and richer nemesis coverage are not +complete yet. diff --git a/jepsen/scripts/build-target.sh b/jepsen/scripts/build-target.sh index 2a7515e..084be55 100755 --- a/jepsen/scripts/build-target.sh +++ b/jepsen/scripts/build-target.sh @@ -4,6 +4,11 @@ set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$repo" +if [[ "$(uname -s)" != Linux ]]; then + printf 'error: build c9 on a Linux host matching the Jepsen DB nodes\n' >&2 + exit 1 +fi + cargo build --release -p cloud9 --bin c9 --locked printf ' ok c9 (%s)\n' "$repo/target/release/c9" diff --git a/jepsen/src/cloud9/jepsen.clj b/jepsen/src/cloud9/jepsen.clj index 9f4054a..e2d09cb 100644 --- a/jepsen/src/cloud9/jepsen.clj +++ b/jepsen/src/cloud9/jepsen.clj @@ -33,6 +33,7 @@ (def pidfile (str dir "/cloud9.pid")) (def kv-service "cloud9.kv.v1.KvService") (def kv-namespace "jepsen") +(def raft-key "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f") (defn canonical-path [path] @@ -65,6 +66,7 @@ "data_dir = \"" data-dir "\"\n" "\n" "[cluster]\n" + "raft_key = \"" raft-key "\"\n" "peers = [\n" (peer-config test) "\n]\n")) @@ -236,7 +238,12 @@ (defn not-leader? [e] (and (= 400 (:status e)) - (str/includes? (str (:body e)) "not leader"))) + (= "failed_precondition" (:code (:body e))) + (str/includes? (str (:message (:body e))) "not leader"))) + +(defn not-found? + [e] + (= "not_found" (:code (:body e)))) (defn recoverable-rpc-error? [e] @@ -244,6 +251,10 @@ (#{408 500 502 503 504} (:status e)) (not-leader? e))) +(defn expected-cas-failure? + [e] + (#{"failed_precondition" "not_found"} (:code (:body e)))) + (defrecord ClientOnlyChecker [checker] checker/Checker (check [_ test history opts] @@ -303,7 +314,7 @@ :type :ok :value (independent/tuple k (decode-value (:body entry))))) (catch [:type ::rpc-error] e - (if (= 404 (:status e)) + (if (not-found? e) (assoc op :type :ok :value (independent/tuple k nil)) (throw+ e))))) @@ -319,7 +330,7 @@ (do (put-value! test leader session sequence k to {:ifMatch (:etag entry)}) (assoc op :type :ok))))) (catch [:type ::rpc-error] e - (if (#{400 404 409 412} (:status e)) + (if (expected-cas-failure? e) (assoc op :type :fail) (throw+ e))))) @@ -391,18 +402,27 @@ [opts] (let [client-gen (cond->> (gen/mix [register-read register-write register-cas register-cas]) (pos? (:stagger opts)) (gen/stagger (:stagger opts))) - nemesis-gen (nemesis-generator opts)] + nemesis-gen (nemesis-generator opts) + main-gen (cond->> (if nemesis-gen + (gen/clients client-gen nemesis-gen) + (gen/clients client-gen)) + (:time-limit opts) (gen/time-limit (:time-limit opts)))] {:checker (checker/compose {:linearizable (client-only (independent/checker (checker/linearizable {:model (model/cas-register)}))) + :stats (checker/stats) + :exceptions (checker/unhandled-exceptions) :timeline (timeline/html)}) :client (KvClient. nil nil nil) - :generator (cond->> (if nemesis-gen - (gen/clients client-gen nemesis-gen) - (gen/clients client-gen)) - (:time-limit opts) (gen/time-limit (:time-limit opts)))})) + :generator (gen/phases + main-gen + (when nemesis-gen + (gen/nemesis (gen/once {:f :stop}))) + (when nemesis-gen + (gen/sleep 5)) + (gen/clients (gen/each-thread (gen/once register-read))))})) (defn cloud9-test [opts] diff --git a/jepsen/test/cloud9/jepsen_test.clj b/jepsen/test/cloud9/jepsen_test.clj new file mode 100644 index 0000000..a864350 --- /dev/null +++ b/jepsen/test/cloud9/jepsen_test.clj @@ -0,0 +1,56 @@ +(ns cloud9.jepsen-test + (:require [clojure.string :as str] + [clojure.test :refer [deftest is testing]] + [cloud9.jepsen :as cloud9] + [jepsen.checker :as checker] + [jepsen.history :as history])) + +(def test-config + {:nodes ["n1" "n2" "n3"] + :client-port 19090 + :raft-port 19091}) + +(deftest generated-node-config-identifies-the-node-and-full-cluster + (let [config (cloud9/node-config test-config "n2")] + (is (str/includes? config "id = 1\nhost = \"n2\"")) + (is (str/includes? config (str "raft_key = \"" cloud9/raft-key "\""))) + (doseq [[id node] (map-indexed vector (:nodes test-config))] + (is (str/includes? config (str "{ id = " id ", host = \"" node "\"")))))) + +(deftest value-codec-round-trips-jepsen-values + (doseq [value [nil 0 4 {:nested [1 2 3]}]] + (is (= value (cloud9/decode-value (cloud9/encode-value value)))))) + +(deftest rpc-retry-classification-is-fail-closed + (testing "leadership and transient transport failures are retryable" + (is (cloud9/recoverable-rpc-error? + {:status 400 :body {:code "failed_precondition" :message "not leader"}})) + (is (cloud9/recoverable-rpc-error? {:status 503}))) + (testing "application failures are final" + (is (not (cloud9/recoverable-rpc-error? + {:status 400 :body {:code "invalid_argument" :message "not leader"}}))) + (is (not (cloud9/recoverable-rpc-error? {:status 409}))) + (is (not (cloud9/recoverable-rpc-error? {:status 412}))))) + +(deftest cas-failures-require-the-exact-connect-code + (is (cloud9/expected-cas-failure? {:body {:code "failed_precondition"}})) + (is (cloud9/expected-cas-failure? {:body {:code "not_found"}})) + (is (not (cloud9/expected-cas-failure? {:status 400 + :body {:code "invalid_argument"}}))) + (is (not (cloud9/expected-cas-failure? {:status 409})))) + +(deftest workload-builds-with-and-without-the-nemesis + (doseq [mode ["none" "kill-leader"]] + (let [workload (cloud9/kv-workload {:nemesis-mode mode + :nemesis-interval 1 + :stagger 0 + :time-limit 1})] + (is (:checker workload)) + (is (:client workload)) + (is (:generator workload))))) + +(deftest failure-only-history-is-not-green + (let [events (history/history [{:process 0 :type :invoke :f :cas} + {:process 0 :type :fail :f :cas}]) + result (checker/check (checker/stats) {} events {})] + (is (not= true (:valid? result))))) From b7e3adf1eac4a9a67e7b460e3d04d64cb81072c2 Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:18:13 -0700 Subject: [PATCH 07/17] fix(ci): install protobuf compiler --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 081bc8d..0f1a3e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,9 @@ jobs: with: components: rustfmt, clippy + - name: Install protoc + run: sudo apt-get update && sudo apt-get install --yes protobuf-compiler + - name: Cache cargo dependencies uses: Swatinem/rust-cache@v2 @@ -50,6 +53,9 @@ jobs: - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable + - name: Install protoc + run: sudo apt-get update && sudo apt-get install --yes protobuf-compiler + - name: Cache cargo dependencies uses: Swatinem/rust-cache@v2 @@ -69,6 +75,9 @@ jobs: - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable + - name: Install protoc + run: sudo apt-get update && sudo apt-get install --yes protobuf-compiler + - name: Cache cargo dependencies uses: Swatinem/rust-cache@v2 From 0f187e9ac66919ef46a5a2b5c731c7415c245158 Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:18:21 -0700 Subject: [PATCH 08/17] docs(docs): document runnable node setup --- CONTRIBUTING.md | 7 ++++--- README.md | 3 +-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0f0fc11..105a6e9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,8 @@ This project adheres to a code of conduct that all contributors are expected to ### Prerequisites -- **Rust**: 1.75.0 or later (install via [rustup](https://rustup.rs/)) +- **Rust**: 1.95.0 or later (install via [rustup](https://rustup.rs/)) +- **protoc**: Required to generate the Connect RPC types - **Git**: For version control - **Cargo tools**: ```bash @@ -30,10 +31,10 @@ cargo build ```bash # Single-node instance -cargo run --bin c9 +cargo run --bin c9 -- start --config cloud9.example.toml # With debug logging -RUST_LOG=debug cargo run --bin c9 +RUST_LOG=debug cargo run --bin c9 -- start --config cloud9.example.toml ``` ## Testing diff --git a/README.md b/README.md index 5a626a2..f35a559 100644 --- a/README.md +++ b/README.md @@ -295,7 +295,7 @@ cd cloud9 cargo build --release # Run single-node instance -./target/release/c9 +./target/release/c9 start --config cloud9.example.toml # Run tests cargo test --workspace @@ -321,4 +321,3 @@ Cloud9 is released under the [MIT License](LICENSE). - [FoundationDB: A Distributed Unbundled Transactional Key Value Store](https://www.foundationdb.org/files/fdb-paper.pdf) - [Comet: An Active Distributed {Key-Value} Store](https://www.usenix.org/legacy/event/osdi10/tech/full_papers/Geambasu.pdf) - From 047f7b1bf53d25c8647a9397ea52a3ac15ddd556 Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:23:53 -0700 Subject: [PATCH 09/17] fix(node): reject duplicate peer addresses --- cloud9/src/main.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cloud9/src/main.rs b/cloud9/src/main.rs index a6257d6..18cb9ca 100644 --- a/cloud9/src/main.rs +++ b/cloud9/src/main.rs @@ -175,6 +175,9 @@ fn peer_addrs(peers: &[PeerSection]) -> Result> { for peer in peers { let node_id = NodeId(peer.id); let addr = resolve_peer_addr(&peer.host, peer.raft_port)?; + if addrs.values().any(|existing| *existing == addr) { + return Err(miette::miette!("cluster.peers contains duplicate address {addr}")); + } if addrs.insert(node_id, addr).is_some() { return Err(miette::miette!("cluster.peers contains duplicate node id {}", peer.id)); } @@ -233,6 +236,18 @@ mod tests { assert!(error.to_string().contains("duplicate node id 0")); } + #[test] + fn invariant_peer_addresses_are_unique() { + let contents = include_str!("../../cloud9.example.toml").replace( + "peers = [\n { id = 0, host = \"127.0.0.1\", raft_port = 19091 },\n]", + "peers = [\n { id = 0, host = \"127.0.0.1\", raft_port = 19091 },\n { id = 1, host = \"127.0.0.1\", raft_port = 19091 },\n]", + ); + + let error = parse_node_config(&contents).unwrap_err(); + + assert!(error.to_string().contains("duplicate address 127.0.0.1:19091")); + } + #[test] fn invariant_raft_key_is_256_bits() { let contents = include_str!("../../cloud9.example.toml") From e0bc34e9bd27f59efab214b3cfb835252d18e69a Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:23:58 -0700 Subject: [PATCH 10/17] test(node): fail Jepsen on client exceptions --- jepsen/src/cloud9/jepsen.clj | 10 +++++++++- jepsen/test/cloud9/jepsen_test.clj | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/jepsen/src/cloud9/jepsen.clj b/jepsen/src/cloud9/jepsen.clj index e2d09cb..fe8c632 100644 --- a/jepsen/src/cloud9/jepsen.clj +++ b/jepsen/src/cloud9/jepsen.clj @@ -264,6 +264,14 @@ [checker] (ClientOnlyChecker. checker)) +(defrecord NoExceptionsChecker [] + checker/Checker + (check [_ _test history _opts] + (let [exceptions (into [] (h/filter :exception history))] + {:valid? (empty? exceptions) + :count (count exceptions) + :example (first exceptions)}))) + (defn with-leader-retry! [test leader f] (loop [attempts 20] @@ -413,7 +421,7 @@ (checker/linearizable {:model (model/cas-register)}))) :stats (checker/stats) - :exceptions (checker/unhandled-exceptions) + :exceptions (NoExceptionsChecker.) :timeline (timeline/html)}) :client (KvClient. nil nil nil) :generator (gen/phases diff --git a/jepsen/test/cloud9/jepsen_test.clj b/jepsen/test/cloud9/jepsen_test.clj index a864350..06f0088 100644 --- a/jepsen/test/cloud9/jepsen_test.clj +++ b/jepsen/test/cloud9/jepsen_test.clj @@ -54,3 +54,10 @@ {:process 0 :type :fail :f :cas}]) result (checker/check (checker/stats) {} events {})] (is (not= true (:valid? result))))) + +(deftest unhandled-client-exception-fails-the-checker + (let [events (history/history [{:process 0 :type :info :f :read + :exception {:via [{:type "boom"}]}}]) + result (checker/check (cloud9/->NoExceptionsChecker) {} events {})] + (is (false? (:valid? result))) + (is (= 1 (:count result))))) From 6c86dbdc4f2c468590cea4dedba2391082ccd246 Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:27:22 -0700 Subject: [PATCH 11/17] test(node): validate Connect error statuses --- jepsen/src/cloud9/jepsen.clj | 7 +++++-- jepsen/test/cloud9/jepsen_test.clj | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/jepsen/src/cloud9/jepsen.clj b/jepsen/src/cloud9/jepsen.clj index fe8c632..d41a353 100644 --- a/jepsen/src/cloud9/jepsen.clj +++ b/jepsen/src/cloud9/jepsen.clj @@ -243,7 +243,8 @@ (defn not-found? [e] - (= "not_found" (:code (:body e)))) + (and (= 404 (:status e)) + (= "not_found" (:code (:body e))))) (defn recoverable-rpc-error? [e] @@ -253,7 +254,9 @@ (defn expected-cas-failure? [e] - (#{"failed_precondition" "not_found"} (:code (:body e)))) + (or (and (= 400 (:status e)) + (= "failed_precondition" (:code (:body e)))) + (not-found? e))) (defrecord ClientOnlyChecker [checker] checker/Checker diff --git a/jepsen/test/cloud9/jepsen_test.clj b/jepsen/test/cloud9/jepsen_test.clj index 06f0088..78a39a6 100644 --- a/jepsen/test/cloud9/jepsen_test.clj +++ b/jepsen/test/cloud9/jepsen_test.clj @@ -33,11 +33,14 @@ (is (not (cloud9/recoverable-rpc-error? {:status 412}))))) (deftest cas-failures-require-the-exact-connect-code - (is (cloud9/expected-cas-failure? {:body {:code "failed_precondition"}})) - (is (cloud9/expected-cas-failure? {:body {:code "not_found"}})) + (is (cloud9/expected-cas-failure? {:status 400 + :body {:code "failed_precondition"}})) + (is (cloud9/expected-cas-failure? {:status 404 + :body {:code "not_found"}})) (is (not (cloud9/expected-cas-failure? {:status 400 :body {:code "invalid_argument"}}))) - (is (not (cloud9/expected-cas-failure? {:status 409})))) + (is (not (cloud9/expected-cas-failure? {:status 500 + :body {:code "failed_precondition"}})))) (deftest workload-builds-with-and-without-the-nemesis (doseq [mode ["none" "kill-leader"]] From 4eae80f17eeec603a89a73efeb4d35f52d17e74f Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:51:52 -0700 Subject: [PATCH 12/17] fix(consensus): bound uncommitted proposals --- consensus/cloud9-raft/src/raft/core.rs | 32 ++++++++++++- consensus/cloud9-raft/src/raft/mod.rs | 3 ++ consensus/cloud9-raft/tests/admission.rs | 61 ++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 consensus/cloud9-raft/tests/admission.rs diff --git a/consensus/cloud9-raft/src/raft/core.rs b/consensus/cloud9-raft/src/raft/core.rs index 10a79eb..dabb768 100644 --- a/consensus/cloud9-raft/src/raft/core.rs +++ b/consensus/cloud9-raft/src/raft/core.rs @@ -8,9 +8,9 @@ use serde::{Deserialize, Serialize}; -use crate::{LogIndex, NodeId, Term}; +use crate::{Command, LogIndex, NodeId, Term}; -use super::log::Log; +use super::log::{EntryPayload, Log}; use super::membership::{Configuration, MembershipMode}; /// Static configuration for a Raft node. @@ -47,6 +47,10 @@ pub struct Config { pub heartbeat_interval: u64, /// Max entries per `AppendEntries` message. pub max_entries_per_msg: u64, + /// Max uncommitted log entries accepted by a leader. + pub max_uncommitted_entries: u64, + /// Max command bytes across uncommitted log entries. + pub max_uncommitted_bytes: u64, /// How to handle membership changes. pub membership_mode: MembershipMode, /// Enable `PreVote` protocol (§4.2.3). @@ -90,6 +94,8 @@ impl Config { election_timeout: (150, 300), heartbeat_interval: 75, max_entries_per_msg: 100, + max_uncommitted_entries: 1024, + max_uncommitted_bytes: 64 * 1024 * 1024, membership_mode: MembershipMode::default(), prevote: true, parallel_disk_write: true, @@ -224,6 +230,28 @@ impl Core { &mut self.persistent.log } + pub(crate) fn proposal_limit_exceeded(&self, command: &Command) -> bool { + let last_index = self.log().last_index(); + assert!(self.commit_index <= last_index, "commit index cannot exceed the log"); + let entries = self.log().slice(self.commit_index + 1, last_index); + let Ok(uncommitted_entries) = u64::try_from(entries.len()) else { + return true; + }; + if uncommitted_entries >= self.config.max_uncommitted_entries { + return true; + } + + entries + .iter() + .filter_map(|entry| match &entry.payload { + EntryPayload::Command(command) => Some(command.0.len()), + EntryPayload::Config(_) => None, + }) + .chain([command.0.len()]) + .try_fold(0_u64, |total, bytes| total.checked_add(u64::try_from(bytes).ok()?)) + .is_none_or(|bytes| bytes > self.config.max_uncommitted_bytes) + } + /// Update term if the given term is higher. Returns true if term changed. /// /// Invariant: term is monotonically increasing. diff --git a/consensus/cloud9-raft/src/raft/mod.rs b/consensus/cloud9-raft/src/raft/mod.rs index 0ec08f0..77194d9 100644 --- a/consensus/cloud9-raft/src/raft/mod.rs +++ b/consensus/cloud9-raft/src/raft/mod.rs @@ -290,6 +290,9 @@ impl RaftNode { pub fn propose(&mut self, cmd: Command) -> Result<(LogIndex, Effects), ProposeError> { match &mut self.role { RoleState::Leader(leader) => { + if self.core.proposal_limit_exceeded(&cmd) { + return Err(ProposeError::Throttled); + } let (index, effects, should_step_down) = leader.propose(&mut self.core, cmd); // Handle step-down if config change removed us diff --git a/consensus/cloud9-raft/tests/admission.rs b/consensus/cloud9-raft/tests/admission.rs new file mode 100644 index 0000000..edea550 --- /dev/null +++ b/consensus/cloud9-raft/tests/admission.rs @@ -0,0 +1,61 @@ +use cloud9_raft::raft::{Configuration, Message, Payload, Persistent, VoteResponse}; +use cloud9_raft::{Command, ConsensusConfig, NodeId, ProposeError, RaftNode}; + +const VOTERS: &[NodeId] = &[NodeId(0), NodeId(1), NodeId(2)]; + +fn leader(config: ConsensusConfig) -> RaftNode { + elect(RaftNode::new(configure(config), VOTERS)) +} + +fn configure(config: ConsensusConfig) -> ConsensusConfig { + config.with_prevote(false).with_parallel_disk_write(false).with_pipelining(false) +} + +fn elect(mut node: RaftNode) -> RaftNode { + while !node.is_candidate() { + node.tick(); + } + node.step(Message { + from: NodeId(1), + to: NodeId(0), + term: node.term(), + payload: Payload::VoteResponse(VoteResponse { granted: true }), + }); + assert!(node.is_leader()); + node +} + +#[test] +fn leader_throttles_uncommitted_entries() { + let mut config = ConsensusConfig::new(NodeId(0)); + config.max_uncommitted_entries = 2; + let mut node = leader(config); + + assert!(node.propose(Command(vec![1])).is_ok()); + assert!(node.propose(Command(vec![2])).is_ok()); + assert!(matches!(node.propose(Command(vec![3])), Err(ProposeError::Throttled))); +} + +#[test] +fn leader_throttles_uncommitted_command_bytes() { + let mut config = ConsensusConfig::new(NodeId(0)); + config.max_uncommitted_bytes = 2; + let mut node = leader(config); + + assert!(node.propose(Command(vec![1])).is_ok()); + assert!(node.propose(Command(vec![2])).is_ok()); + assert!(matches!(node.propose(Command(vec![3])), Err(ProposeError::Throttled))); +} + +#[test] +fn compacted_prefix_does_not_consume_admission_capacity() { + let mut config = ConsensusConfig::new(NodeId(0)); + config.max_uncommitted_entries = 1; + let cluster = Configuration::simple(VOTERS.iter().copied()); + let mut persistent = + Persistent { term: 1, bootstrap_config: cluster.clone(), ..Persistent::default() }; + persistent.log.install_snapshot(100, 1, cluster); + let mut node = elect(RaftNode::restore(configure(config), persistent)); + + assert!(node.propose(Command(vec![1])).is_ok()); +} From 811177956460ca7cdda91ddd50cb791bb4c5dc7a Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:52:17 -0700 Subject: [PATCH 13/17] fix(node): harden runtime invariants --- cloud9-node/src/runtime.rs | 230 ++++++++++++++++++++++++++++++++----- cloud9-node/src/tests.rs | 9 ++ 2 files changed, 213 insertions(+), 26 deletions(-) diff --git a/cloud9-node/src/runtime.rs b/cloud9-node/src/runtime.rs index cb3fb1f..f3dc13c 100644 --- a/cloud9-node/src/runtime.rs +++ b/cloud9-node/src/runtime.rs @@ -3,9 +3,9 @@ //! Every step is serialized with its WAL. Persistent state is synced before //! network effects are released, then committed commands are applied in order. -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex as StdMutex, MutexGuard as StdMutexGuard}; use cloud9_raft::raft::{Effects, Message}; use cloud9_raft::{Command, LogIndex, NodeId, ProposeError, RaftNode}; @@ -24,6 +24,12 @@ const PROPOSAL_TIMEOUT: Duration = Duration::from_secs(5); #[derive(Debug, Error)] pub(crate) enum RuntimeError { + #[error("node id {node_id} does not match consensus id {consensus_id}")] + IdentityMismatch { node_id: NodeId, consensus_id: NodeId }, + #[error("cluster peers do not map node {node_id} to its Raft address {raft_addr}")] + InvalidSelfPeer { node_id: NodeId, raft_addr: std::net::SocketAddr }, + #[error("cluster peers contain duplicate Raft address {address}")] + DuplicatePeerAddress { address: std::net::SocketAddr }, #[error(transparent)] Store(#[from] StoreError), #[error("Raft snapshot transport is not implemented")] @@ -50,6 +56,7 @@ struct RaftMachine { } struct PendingProposal { + id: u64, command: Vec, sender: oneshot::Sender>, } @@ -65,11 +72,34 @@ impl PendingProposal { } } +struct PendingCleanup { + waiters: Arc>>, + index: LogIndex, + id: u64, + armed: bool, +} + +impl PendingCleanup { + fn finish(mut self) { + remove_pending(&self.waiters, self.index, self.id); + self.armed = false; + } +} + +impl Drop for PendingCleanup { + fn drop(&mut self) { + if self.armed { + remove_pending(&self.waiters, self.index, self.id); + } + } +} + pub(crate) struct RaftRuntime { config: NodeConfig, machine: Mutex, state: Arc>, - waiters: Mutex>, + waiters: Arc>>, + next_proposal_id: AtomicU64, transport: PeerTransport, failed: AtomicBool, } @@ -79,6 +109,7 @@ impl RaftRuntime { config: NodeConfig, state: Arc>, ) -> Result { + validate_config(&config)?; let voters = config.peers.keys().copied().collect::>(); let initial = RaftNode::new(config.consensus.clone(), &voters); let store = RaftStore::open(&config.raft_dir(), initial.persistent().clone())?; @@ -88,7 +119,8 @@ impl RaftRuntime { config, machine: Mutex::new(RaftMachine { node, store }), state, - waiters: Mutex::new(HashMap::new()), + waiters: Arc::new(StdMutex::new(HashMap::new())), + next_proposal_id: AtomicU64::new(1), transport, failed: AtomicBool::new(false), }) @@ -130,32 +162,42 @@ impl RaftRuntime { pub(crate) async fn propose(&self, command: KvCommand) -> Result { self.ensure_healthy().map_err(|error| runtime_connect_error(&error))?; + let proposal_id = self + .next_proposal_id + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |id| id.checked_add(1)) + .map_err(|_| ConnectError::resource_exhausted("proposal id space exhausted"))?; let bytes = serde_json::to_vec(&command) .map_err(|_| ConnectError::internal("failed to encode Raft command"))?; - let (index, receiver) = { + let (receiver, cleanup) = { let mut machine = self.machine.lock().await; let (index, effects) = machine .node .propose(Command(bytes.clone())) .map_err(|error| propose_error(&error))?; let (sender, receiver) = oneshot::channel(); - self.waiters.lock().await.insert(index, PendingProposal { command: bytes, sender }); + lock_waiters(&self.waiters) + .insert(index, PendingProposal { id: proposal_id, command: bytes, sender }); + let cleanup = PendingCleanup { + waiters: self.waiters.clone(), + index, + id: proposal_id, + armed: true, + }; if let Err(error) = self.handle_effects(&mut machine, effects).await { - self.waiters.lock().await.remove(&index); + cleanup.finish(); let error = self.fail(error); return Err(runtime_connect_error(&error)); } - (index, receiver) + (receiver, cleanup) }; - match timeout(PROPOSAL_TIMEOUT, receiver).await { + let result = match timeout(PROPOSAL_TIMEOUT, receiver).await { Ok(Ok(result)) => result, Ok(Err(_)) => Err(ConnectError::aborted("Raft proposal was dropped")), - Err(_) => { - self.waiters.lock().await.remove(&index); - Err(ConnectError::unavailable("Raft proposal timed out")) - } - } + Err(_) => Err(ConnectError::unavailable("Raft proposal timed out")), + }; + cleanup.finish(); + result } pub(crate) async fn read_barrier(&self) -> Result<(), ConnectError> { @@ -207,15 +249,13 @@ impl RaftRuntime { } async fn apply_committed(&self, node: &mut RaftNode) { - let entries = node.committed().collect::>(); - let mut applied_to = None; - for entry in entries { + loop { + let Some(entry) = node.committed().next() else { + return; + }; let result = self.apply_command(&entry.command).await; - self.complete_waiter(entry.index, &entry.command, result).await; - applied_to = Some(entry.index); - } - if let Some(index) = applied_to { - node.advance(index); + node.advance(entry.index); + self.complete_waiter(entry.index, &entry.command, result); } } @@ -225,13 +265,13 @@ impl RaftRuntime { self.state.write().await.apply(command) } - async fn complete_waiter( + fn complete_waiter( &self, index: LogIndex, command: &Command, result: Result, ) { - if let Some(pending) = self.waiters.lock().await.remove(&index) { + if let Some(pending) = lock_waiters(&self.waiters).remove(&index) { pending.complete(command, result); } } @@ -250,6 +290,46 @@ impl RaftRuntime { } } +fn validate_config(config: &NodeConfig) -> Result<(), RuntimeError> { + if config.node_id != config.consensus.id { + return Err(RuntimeError::IdentityMismatch { + node_id: config.node_id, + consensus_id: config.consensus.id, + }); + } + if config.peers.get(&config.node_id) != Some(&config.raft_addr) { + return Err(RuntimeError::InvalidSelfPeer { + node_id: config.node_id, + raft_addr: config.raft_addr, + }); + } + let mut addresses = HashSet::new(); + if let Some(address) = config.peers.values().find(|address| !addresses.insert(**address)) { + return Err(RuntimeError::DuplicatePeerAddress { address: *address }); + } + Ok(()) +} + +fn remove_pending( + waiters: &StdMutex>, + index: LogIndex, + id: u64, +) { + let mut waiters = lock_waiters(waiters); + if waiters.get(&index).is_some_and(|pending| pending.id == id) { + waiters.remove(&index); + } +} + +fn lock_waiters( + waiters: &StdMutex>, +) -> StdMutexGuard<'_, HashMap> { + match waiters.lock() { + Ok(waiters) => waiters, + Err(_poisoned) => std::process::abort(), + } +} + fn propose_error(error: &ProposeError) -> ConnectError { match error { ProposeError::NotLeader { leader_hint: Some(leader) } => { @@ -268,15 +348,113 @@ fn runtime_connect_error(error: &RuntimeError) -> ConnectError { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use std::net::{Ipv4Addr, SocketAddr}; + + use cloud9_core::SharedString; + use cloud9_storage::StorageOptions; + + use crate::RaftKey; + use super::*; #[tokio::test] async fn invariant_log_overwrite_cannot_complete_a_different_proposal() { let (sender, receiver) = oneshot::channel(); - let pending = PendingProposal { command: b"original".to_vec(), sender }; + let pending = PendingProposal { id: 1, command: b"original".to_vec(), sender }; pending.complete(&Command(b"replacement".to_vec()), Ok(KvApplyResult::ReadBarrier)); assert!(receiver.await.unwrap().is_err()); } + + #[test] + fn invariant_runtime_identity_is_unambiguous() { + let dir = tempfile::tempdir().unwrap(); + let mut config = test_config(dir.path()); + config.consensus.id = NodeId(1); + assert!(matches!(validate_config(&config), Err(RuntimeError::IdentityMismatch { .. }))); + + let mut config = test_config(dir.path()); + config.raft_addr.set_port(19_092); + assert!(matches!(validate_config(&config), Err(RuntimeError::InvalidSelfPeer { .. }))); + + let mut config = test_config(dir.path()); + config.peers.insert(NodeId(1), config.raft_addr); + assert!(matches!(validate_config(&config), Err(RuntimeError::DuplicatePeerAddress { .. }))); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn invariant_cancelled_delivery_does_not_reapply_a_command() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(dir.path()); + let state = Arc::new(RwLock::new(KvState::new())); + let runtime = Arc::new(RaftRuntime::open(config, state.clone()).unwrap()); + let driver = tokio::spawn(runtime.clone().run()); + timeout(Duration::from_secs(1), async { + while runtime.mode().await != "leader" { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + let state_guard = state.write().await; + let proposing = tokio::spawn({ + let runtime = runtime.clone(); + async move { runtime.propose(KvCommand::RegisterSession).await } + }); + timeout(Duration::from_secs(1), async { + loop { + if !runtime.waiters.lock().unwrap().is_empty() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let waiters = runtime.waiters.clone(); + let (locked_sender, locked_receiver) = std::sync::mpsc::channel(); + let (release_sender, release_receiver) = std::sync::mpsc::channel(); + let lock_thread = std::thread::spawn(move || { + let _guard = lock_waiters(&waiters); + locked_sender.send(()).unwrap(); + release_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); + }); + locked_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); + drop(state_guard); + let applied_guard = timeout(Duration::from_secs(1), state.read()).await.unwrap(); + + proposing.abort(); + drop(applied_guard); + release_sender.send(()).unwrap(); + lock_thread.join().unwrap(); + let _ = proposing.await; + assert!(runtime.waiters.lock().unwrap().is_empty()); + + let result = runtime.propose(KvCommand::RegisterSession).await.unwrap(); + let KvApplyResult::RegisterSession(response) = result else { + panic!("session proposal returned the wrong result"); + }; + assert_eq!(2, response.client_id); + driver.abort(); + } + + fn test_config(path: &std::path::Path) -> NodeConfig { + let node_id = NodeId(0); + let raft_addr = SocketAddr::from((Ipv4Addr::LOCALHOST, 19_091)); + NodeConfig { + node_id, + client_addr: SocketAddr::from((Ipv4Addr::LOCALHOST, 19_090)), + raft_addr, + peers: BTreeMap::from([(node_id, raft_addr)]), + raft_key: RaftKey::from_hex(&"01".repeat(32)).unwrap(), + storage: StorageOptions { + name: SharedString::literal("test"), + data_dir: SharedString::from(path.to_string_lossy()), + }, + consensus: crate::raft_config(node_id), + } + } } diff --git a/cloud9-node/src/tests.rs b/cloud9-node/src/tests.rs index 307f0c0..9dc8ee8 100644 --- a/cloud9-node/src/tests.rs +++ b/cloud9-node/src/tests.rs @@ -194,6 +194,15 @@ async fn invariant_three_nodes_replicate_and_fail_over() -> Result<()> { let survivors = (0..3).filter(|node| *node != leader).collect::>(); let replacement = wait_for_cluster_leader(&runtimes, &survivors).await?; runtimes[replacement].read_barrier().await?; + let state = states[replacement].read().await; + let record = state + .entries + .get(&KvName::new("test", "key")?) + .ok_or_else(|| anyhow::anyhow!("replacement leader is missing the replicated key"))?; + assert_eq!(b"value", record.body.as_slice()); + assert_eq!("\"c9-1\"", record.etag); + assert_eq!(1, record.generation); + drop(state); for task in drivers { task.abort(); From f4855fef117919f89a20cca6f8c4ea0a22582b02 Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:52:45 -0700 Subject: [PATCH 14/17] fix(ci): reject false-green Jepsen recovery --- .github/workflows/ci.yml | 25 ++++++++++++++++++++++ jepsen/README.md | 7 ++++++- jepsen/src/cloud9/jepsen.clj | 33 ++++++++++++++++++++++++------ jepsen/test/cloud9/jepsen_test.clj | 22 +++++++++++++++++++- 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f1a3e0..8cd0d12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,31 @@ jobs: - name: Run tests run: cargo test --workspace --locked + jepsen: + name: Jepsen harness + runs-on: ubuntu-latest + defaults: + run: + working-directory: jepsen + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + + - name: Install Leiningen + run: sudo apt-get update && sudo apt-get install --yes leiningen + + - name: Check Jepsen harness + run: lein check + + - name: Test Jepsen harness + run: lein test + docs: name: Documentation runs-on: ubuntu-latest diff --git a/jepsen/README.md b/jepsen/README.md index ff107af..bab1ef9 100644 --- a/jepsen/README.md +++ b/jepsen/README.md @@ -62,4 +62,9 @@ serving node-local state. `c9 start` persists Raft hard state and log entries before sending network effects, then reconstructs KV state by replaying committed commands. Snapshot transfer, log compaction, read forwarding, and richer nemesis coverage are not -complete yet. +complete yet. Without snapshot-backed reclamation, the WAL grows to its 4 GiB +limit and the node then fails closed. + +Peer HMAC authenticates message bodies but does not provide confidentiality or +replay protection. Keep this transport on an isolated test network until it is +replaced with mutually authenticated, replay-resistant transport security. diff --git a/jepsen/src/cloud9/jepsen.clj b/jepsen/src/cloud9/jepsen.clj index d41a353..ff01f85 100644 --- a/jepsen/src/cloud9/jepsen.clj +++ b/jepsen/src/cloud9/jepsen.clj @@ -34,6 +34,9 @@ (def kv-service "cloud9.kv.v1.KvService") (def kv-namespace "jepsen") (def raft-key "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f") +(def rpc-timeouts {:timeout 5000 + :connect-timeout 5000 + :idle-timeout 5000}) (defn canonical-path [path] @@ -143,10 +146,9 @@ (defn rpc! [test node method body] (let [{:keys [status body error]} @(http/post (rpc-url test node method) - {:headers {"content-type" "application/json"} - :body (json/generate-string body) - :connection-timeout 5000 - :socket-timeout 5000}) + (merge rpc-timeouts + {:headers {"content-type" "application/json"} + :body (json/generate-string body)})) decoded (when-not (str/blank? body) (json/parse-string body true))] (when error @@ -190,6 +192,10 @@ :f :read :value (independent/tuple 0 nil)}) +(defn final-register-read + [test process] + (assoc (register-read test process) :final? true)) + (defn register-cas [_ _] {:type :invoke @@ -255,7 +261,9 @@ (defn expected-cas-failure? [e] (or (and (= 400 (:status e)) - (= "failed_precondition" (:code (:body e)))) + (= "failed_precondition" (:code (:body e))) + (#{"key already exists" "ETag precondition failed"} + (:message (:body e)))) (not-found? e))) (defrecord ClientOnlyChecker [checker] @@ -275,6 +283,18 @@ :count (count exceptions) :example (first exceptions)}))) +(defrecord RecoveryChecker [] + checker/Checker + (check [_ _test history _opts] + (let [finals (into [] (h/filter #(and (:final? %) + (not= :invoke (:type %))) + history)) + failures (into [] (remove #(= :ok (:type %)) finals))] + {:valid? (and (pos? (count finals)) (empty? failures)) + :count (count finals) + :failures (count failures) + :example (first failures)}))) + (defn with-leader-retry! [test leader f] (loop [attempts 20] @@ -425,6 +445,7 @@ {:model (model/cas-register)}))) :stats (checker/stats) :exceptions (NoExceptionsChecker.) + :recovery (RecoveryChecker.) :timeline (timeline/html)}) :client (KvClient. nil nil nil) :generator (gen/phases @@ -433,7 +454,7 @@ (gen/nemesis (gen/once {:f :stop}))) (when nemesis-gen (gen/sleep 5)) - (gen/clients (gen/each-thread (gen/once register-read))))})) + (gen/clients (gen/each-thread (gen/once final-register-read))))})) (defn cloud9-test [opts] diff --git a/jepsen/test/cloud9/jepsen_test.clj b/jepsen/test/cloud9/jepsen_test.clj index 78a39a6..431f508 100644 --- a/jepsen/test/cloud9/jepsen_test.clj +++ b/jepsen/test/cloud9/jepsen_test.clj @@ -34,14 +34,25 @@ (deftest cas-failures-require-the-exact-connect-code (is (cloud9/expected-cas-failure? {:status 400 - :body {:code "failed_precondition"}})) + :body {:code "failed_precondition" + :message "key already exists"}})) + (is (cloud9/expected-cas-failure? {:status 400 + :body {:code "failed_precondition" + :message "ETag precondition failed"}})) (is (cloud9/expected-cas-failure? {:status 404 :body {:code "not_found"}})) + (is (not (cloud9/expected-cas-failure? {:status 400 + :body {:code "failed_precondition" + :message "not leader"}}))) (is (not (cloud9/expected-cas-failure? {:status 400 :body {:code "invalid_argument"}}))) (is (not (cloud9/expected-cas-failure? {:status 500 :body {:code "failed_precondition"}})))) +(deftest rpc-uses-http-kit-timeout-options + (is (= {:timeout 5000 :connect-timeout 5000 :idle-timeout 5000} + cloud9/rpc-timeouts))) + (deftest workload-builds-with-and-without-the-nemesis (doseq [mode ["none" "kill-leader"]] (let [workload (cloud9/kv-workload {:nemesis-mode mode @@ -64,3 +75,12 @@ result (checker/check (cloud9/->NoExceptionsChecker) {} events {})] (is (false? (:valid? result))) (is (= 1 (:count result))))) + +(deftest final-recovery-reads-must-succeed + (let [checker (cloud9/->RecoveryChecker) + unavailable (history/history [{:process 0 :type :invoke :f :read :final? true} + {:process 0 :type :info :f :read :final? true}]) + recovered (history/history [{:process 0 :type :invoke :f :read :final? true} + {:process 0 :type :ok :f :read :final? true}])] + (is (false? (:valid? (checker/check checker {} unavailable {})))) + (is (true? (:valid? (checker/check checker {} recovered {})))))) From f82fedaf3230419745a025afbf8f75215b1a723e Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:59:59 -0700 Subject: [PATCH 15/17] docs(docs): define product architecture --- CONTRIBUTING.md | 9 +- README.md | 388 ++---- cloud9-proto/proto/cloud9/kv/v1/kv.proto | 5 +- cloud9/Cargo.toml | 2 +- docs/README.md | 21 +- docs/design-notes.md | 6 +- jepsen/README.md | 13 +- spec/00-vision.md | 103 +- spec/01-mvcc.md | 122 +- spec/02-timestamps.md | 303 ++--- spec/03-external-consistency.md | 242 ++-- spec/04-truetime-analysis.md | 243 ++-- spec/05-aws-time-infrastructure.md | 713 ++--------- spec/06-sharding-partitioning.md | 402 ++----- spec/07-sql-kv-unification.md | 912 +++----------- spec/08-transactions.md | 1063 +++------------- spec/09-market-analysis.md | 603 ++-------- spec/10-consensus.md | 416 ++----- spec/11-indexes-schema.md | 971 ++------------- spec/12-implementation-roadmap.md | 1398 +++------------------- spec/README.md | 270 ++--- 21 files changed, 1635 insertions(+), 6570 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 105a6e9..902185d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -255,14 +255,15 @@ Cloud9 follows [Conventional Commits](https://www.conventionalcommits.org/): - `docs`: Documentation changes - `chore`: Build, CI, or tooling changes -**Scopes**: `kv`, `raft`, `txn`, `sql`, `hlc`, `sim`, `ci` +**Scopes**: `core`, `consensus`, `storage`, `node`, `proto`, `ci`, `deps`, +`docs` **Examples**: ``` -feat(txn): implement commit-wait for external consistency +feat(core): implement bounded-time commit-wait -Add HLC-based commit-wait that delays transaction acknowledgment -until all replicas have passed the commit timestamp. +Reject unhealthy time intervals and delay acknowledgment until the commit +timestamp is certainly in the past. Closes #123 ``` diff --git a/README.md b/README.md index f35a559..390de4a 100644 --- a/README.md +++ b/README.md @@ -1,313 +1,144 @@ # Cloud9 -**The only distributed database that unifies SQL and KV under one ACID transaction.** +**An open-source Spanner and MLIR for databases.** -Cloud9 provides Spanner-class external consistency with true Postgres compatibility and a native KV API—all in an MIT-licensed package that runs on your laptop or across the planet with the same binary. +Cloud9 is a database compiler and distributed storage system. It accepts SQL, +key-value, document, object, and analytical workloads. It lowers each workload +through typed intermediate representations into a domain-specific execution +engine. -## Overview +The target is broad: SQLite-like local development and planetary deployment +from one codebase. The semantics stay stable as the topology changes. -Most distributed databases force you to choose: strong consistency with limited scale, or weak consistency with operational complexity. Cloud9 eliminates this trade-off by implementing external consistency—the same guarantee that powers Google Spanner—in an open-source package that runs anywhere. - -The engine compiles both SQL and KV operations into a common transactional IR, ensuring that reads and writes across both APIs observe a single global serialization order. Timestamps are assigned using Hybrid Logical Clocks with bounded uncertainty, and commits wait until all replicas have passed the commit timestamp before acknowledging. This design provides linearizable reads, lock-free snapshots, and deterministic ordering for concurrent writers. - -## Why Cloud9 - -### The Theoretical Guarantee You Deserve - -**External consistency** is the gold standard for distributed databases. It means: if you finish a write and then start a read—anywhere in the world—that read sees your write. Always. No exceptions. No "eventual consistency" footnotes. No "usually works but sometimes doesn't." - -This isn't a feature. It's a **mathematical guarantee**, proven correct with formal methods. The same guarantee Google uses for ads billing, where every cent must be accounted for. - -**You shouldn't need a Google-sized budget to get Google-class correctness.** - -Cloud9 brings this guarantee to everyone: -- **Students** learning distributed systems -- **Startups** building the next platform -- **Enterprises** that need bulletproof data -- **Developers** who refuse to compromise on correctness - -### The Practical Reality You Face - -The market offers you bad choices: - -**Spanner**: Correct but expensive ($1000+/month minimum), vendor lock-in, surprise billing incidents, missing Postgres features (no foreign keys, triggers, or stored procedures). - -**DynamoDB**: Cheap to start but no real transactions, KV-only, eventual consistency, vendor lock-in. - -**Managed Postgres** (CloudSQL/RDS): Familiar but unreliable at scale, performance issues, not truly distributed. - -**Self-hosted Postgres**: Full control but loses strict serializability when sharded, requires expertise to run globally. - -**CockroachDB/YugabyteDB**: Strong technically but either proprietary now (CRDB) or split SQL/KV APIs (YugabyteDB), expensive managed tiers. - -**The gap**: No database gives you theoretical perfection, practical usability, and freedom from vendor lock-in. - -### What Cloud9 Actually Gives You - -**SQL and KV, unified**: -- Write with SQL, read with KV—in the same transaction -- Join SQL tables with KV keyspaces using typed projections -- One snapshot, one timestamp, one consistency guarantee -- No cache coherence problems, no dual writes, no eventual consistency - -**Example**: -```sql -BEGIN; - -- SQL: Complex query - SELECT user_id FROM orders WHERE amount > 1000; - - -- KV: Fast state update - PUT('agent:state:123', state_blob); +## Status - -- Cross-API join - SELECT * FROM users u - JOIN kv_namespace('sessions') s ON s.user_id = u.id; -COMMIT; -- Atomic across both APIs +Cloud9 is under active development. The repository currently contains a pure +Raft state machine, a durable write-ahead log, a replicated key-value service, +and a Jepsen harness. + +Multi-version concurrency control, distributed transactions, SQL, document, +object, analytical dialects, and bounded-time integration remain under +development. The specifications describe the target architecture. They are not +a claim that each feature is complete. + +## One Database, Many Dialects + +Cloud9 treats database APIs as source languages: + +- SQL dialects provide relational queries and transactions. +- DynamoDB-style APIs provide key-value and conditional operations. +- MongoDB-style APIs provide document queries and updates. +- S3-style APIs provide objects, metadata, versions, and byte ranges. +- ClickHouse-style plans provide columnar analytical execution. + +These APIs share identity, transactions, timestamps, placement, and +observability. They do not share one forced physical layout. A columnar scan +and an object read need different data structures. + +## MLIR for Databases + +[MLIR](https://mlir.llvm.org/) preserves domain information through multiple +intermediate representation levels. Cloud9 applies that design to databases. + +```text +SQL | KV | document | object | analytical dialects + | + semantic dialect IRs + | + transactional and time IR + | + placement and physical IRs + | + row | KV | document | object | columnar engines + | + MVCC | Raft | storage ``` -**No other database can do this.** - -**Correctness without compromise**: -- External consistency: Real-time ordering proven with formal methods -- Strict serializability: No anomalies, no "eventually consistent" footnotes -- Lock-free read-only transactions: Backups run concurrently with writes, never block -- True ACID: Full referential integrity with foreign keys, triggers, and constraints (unlike Spanner) - -**Scale without barriers**: -- Local: `cargo run` on your laptop—free, instant, same semantics -- Regional: Multi-AZ replication with single-digit millisecond commits -- Global: Multi-region with cross-shard transactions and external consistency -- Same binary, same guarantees at every scale - -**Freedom without lock-in**: -- MIT licensed—fork it, modify it, run it forever -- Self-host on $12 VPS or bare metal—full Spanner-class guarantees -- Or use Dedalus Cloud managed tier—convenience without lock-in -- Postgres wire protocol—existing tools, ORMs, and drivers just work - -### Who This Is For - -**If you've ever thought**: -- "I wish Postgres could scale globally without losing ACID" -- "I wish Spanner didn't cost $1000/month to try" -- "I wish I could run my production database locally for testing" -- "I wish DynamoDB had SQL and real transactions" -- "I wish I wasn't locked into a vendor who could 10x my bill tomorrow" - -**Cloud9 is for you.** - -Whether you're: -- A student running it on a Raspberry Pi -- A startup prototyping on your laptop -- An enterprise running globally distributed systems -- A researcher building distributed agents - -The same database. The same guarantees. The same code. - -### The Populist Database - -For too long, distributed databases with strong guarantees have been the domain of tech giants. You either pay Google/AWS thousands per month, or you compromise on correctness. - -**Cloud9 says: no more.** - -Theoretical perfection shouldn't require a corporate credit card. The best database architecture should be available to anyone with a computer. You shouldn't have to choose between "correct" and "affordable." - -This is infrastructure that belongs to everyone. Built in the open. Proven correct. Free to use, modify, and run forever. - -**The daily driver database for the distributed era.** - -## Architecture - -### Storage and Replication - -Cloud9 stores data in an MVCC key-value space, partitioned into ranges and replicated via consensus. Each range uses Raft for log replication and leader election. Reads are served by leaseholders for linearizability, or by any replica at a past timestamp. Cross-range transactions use two-phase commit with a coordinator that enforces commit-wait. +Each source dialect keeps its semantics until a legal lowering exists. SQL +nullability, DynamoDB conditions, MongoDB updates, and S3 versioning must not +disappear into a generic key-value operation too early. -### Consensus Driver Interface +Lowering selects physical operators and data placement. Common passes can +enforce transactions, authorization, locality, and cost rules. Specialized +passes can select indexes, columnar scans, object extents, or point reads. -The replication layer is abstracted behind a narrow CDI that any SMR algorithm can implement. The default is Raft; alternative protocols (Multi-Paxos, Flexible Paxos, leaderless variants) can be swapped per range without changing the storage or transaction layers. +## Time and External Consistency -### Dual API Surface +Cloud9 defines a TrueTime-shaped API: -SQL queries compile to range scans and secondary index lookups. KV operations map directly to get/put/scan primitives on the underlying storage. Both share the same snapshot isolation rules, the same timestamp oracle, and the same 2PC coordinator. KV keyspaces can be projected into typed SQL tables using versioned mappings, enabling cross-API joins with predicate pushdown. - -### Timestamp Discipline - -Cloud9 uses HLC to generate monotonic, causally ordered timestamps. Each node tracks observed skew; if the measured uncertainty exceeds a configured bound, writes are refused. Commit-wait is approximately ε, where ε is the current uncertainty. This ensures that any operation starting after a commit observes that commit's effects. - -## Comparison - -| | Cloud9 | Spanner | CockroachDB | YugabyteDB | DynamoDB | PostgreSQL | -|------------------------------------|:------:|:-------:|:-----------:|:----------:|:--------:|:----------:| -| **Consistency** | | | | | | | -| External consistency | ✅ | ✅ | ✅ | ✅ | ❌ | ❌¹ | -| Strict serializability | ✅ | ✅ | ✅ | ✅ | ❌ | ✅² | -| Lock-free snapshot reads | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | -| Lock-free read-only transactions | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | -| **API Surface** | | | | | | | -| SQL (Postgres-compatible) | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | -| Native KV API | ✅ | ❌ | ❌ | ❌³ | ✅ | ❌ | -| Cross-API transactions | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Temporal queries (AS OF) | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | -| **Scale & Deployment** | | | | | | | -| Global distribution | ✅ | ✅ | ✅ | ✅ | ✅ | ❌⁴ | -| Transparent cross-shard transactions| ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | -| Automatic range rebalancing | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | -| Single-binary local mode | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | -| Pluggable consensus | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Online schema changes | ✅ | ✅ | ✅ | ✅ | N/A | ❌⁵ | -| Zero-downtime binary upgrades | ✅ | ✅ | ✅¹⁰ | ❌ | N/A | ❌ | -| **AI & Modern Workloads** | | | | | | | -| Native vector indexing | ✅ | ❌ | ❌⁶ | ❌⁶ | ❌ | ✅⁶ | -| Hybrid dense/sparse search | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Deterministic multi-writer ordering| ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | -| CDC with exactly-once semantics | ✅ | ✅⁷ | ✅ | ✅ | ✅ | ✅⁸ | -| **Implementation** | | | | | | | -| Open source (OSI-approved) | ✅ | ❌ | ❌⁹ | ✅ | ❌ | ✅ | -| Memory-safe core | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Deterministic simulation testing | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | - -¹ Single-instance serializable only; Aurora Global Database offers async replication -² Single node only; distributed Postgres loses strict serializability -³ Separate YCQL API; not schema-compatible with YSQL -⁴ Read replicas available; multi-region writes require application-level coordination -⁵ Most DDL operations require table locks -⁶ Via extensions (pgvector); not transactionally unified with core -⁷ Dataflow/Pub/Sub integration; not built into core storage -⁸ Via logical replication; at-least-once semantics -⁹ Now under CockroachDB Software License (source-available) -¹⁰ Added after years of production pain; Cloud9 designs for it from the start - -## Features - -### Unification - -SQL and KV share one MVCC storage kernel. Cross-model joins allow SQL tables and KV keyspaces to interoperate. All queries lower into a single transactional IR. Both APIs observe the same external-consistency and timestamp semantics. - -### External Consistency - -HLC-based timestamping with bounded uncertainty and commit-wait provides strict real-time order. Every transaction's commit order matches wall-clock order cluster-wide. Snapshot reads are guaranteed safe once closed-timestamp passes. - -### Cloud-Native Scale - -Geo-distributed by default with multi-region quorum replication and latency-based leader placement. Elastic compute/storage split enables independent scale-out and failover. Pluggable consensus supports Raft as baseline with optional Flexible/Multi-Paxos or leaderless modules. - -### Developer Experience - -Postgres-compatible SQL works with existing clients, ORMs, and tools. Low-latency KV API provides millisecond get/put/scan path for agent workloads. Temporal queries with `AS OF` and time-travel are built in. CDC streams all changes with exactly-once semantics. Same binary runs locally or globally—SQLite simplicity with Spanner guarantees. - -### AI & Agentic Workloads - -AI agents need both: KV for hot-path state (millisecond reads/writes), SQL for analytics (complex joins), and vector search for retrieval. Cloud9 is the only database where all three share one MVCC snapshot, one transaction, one consistency model. - -**What this enables**: -- Agent queries vector index, joins with SQL user data, updates KV state—atomic -- Time-travel on vector data: `SELECT * FROM vectors AS OF TIMESTAMP` -- CDC streams vector updates with exactly-once semantics -- No cache coherence, no dual writes, no eventual consistency - -Postgres + pgvector works at single-node scale. Cloud9 works globally with external consistency guarantees. - -### Performance & Reliability - -Commit latency is approximately quorum RTT + ε. Read-anywhere architecture serves region-local snapshot reads at consistent timestamps. Online schema evolution provides non-blocking DDL. Fault containment enables per-range recovery and rebalancing. Tail latency control uses dynamic commit-wait tuning and leader leases. - -### Lock-Free Read-Only Transactions - -Cloud9 supports true lock-free read-only transactions at any timestamp. Read-only transactions never block writes and never acquire locks, enabling high-throughput analytical queries and backups to run concurrently with write traffic. Reads are served directly from any replica that has applied entries up to the requested timestamp, ensuring consistent snapshots without coordination overhead. This makes Cloud9 suitable for mixed workloads where analytical queries, exports, and backups must coexist with latency-sensitive write operations. - -### Zero-Downtime Upgrades - -Cloud9 is designed for rolling upgrades without downtime—a capability that emerges naturally from its architecture rather than being bolted on after the fact. - -**Schema changes are timestamped**: DDL operations create new schema versions at commit timestamps. Old transactions see old schemas, new transactions see new schemas—simultaneously, without locks. This is fundamentally different from traditional databases where schema changes are global, atomic operations that require coordination across all nodes. - -**Online index backfills**: Indexes are built in the background using fence timestamps. Reads and writes continue during index creation. The fence timestamp creates a clean boundary: writes before the fence are handled by the backfill process, writes after the fence are automatically indexed. No gaps, no locks, no downtime. - -**Per-range rebalancing**: Raft membership changes use joint consensus, allowing replicas to be added or removed without quorum loss. This means you can add new nodes running upgraded binaries, wait for them to catch up, promote them to voters, and remove old nodes—all while serving traffic. - -**What this enables**: -- Add or remove nodes without downtime -- Upgrade binary versions by rolling replicas one at a time -- Change schemas while queries run against both old and new versions -- Rebalance ranges under load without impacting availability - -**Why other databases can't do this**: Postgres has MVCC but treats schema as global state—`ALTER TABLE` requires locks that block concurrent access. CockroachDB eventually added zero-downtime upgrades after years of production pain. Spanner has it but is proprietary. Cloud9 designs for it from the start: timestamped schemas, versioned metadata, and Raft-based replication that supports gradual evolution of cluster state. +```text +now() -> [earliest, latest] +``` -No other open-source database combines all four capabilities out of the box. +The interval must contain real time. Cloud9 can use that bound with commit-wait +to provide external consistency, also called strict serializability. -### Global Sharding and Transparent Scaling +TrueTime mode is capability-gated. It starts only when the host provides a +supported bounded-time source. The first production target is +[AWS ClockBound](https://github.com/aws/clock-bound) on supported Linux EC2 +hardware with Amazon Time Sync configured. -Cloud9 partitions data into ranges that are automatically distributed across nodes and regions. Ranges are replicated via consensus groups and rebalanced dynamically based on load and placement policies. Transactions spanning multiple ranges use two-phase commit with external consistency guarantees, ensuring that cross-shard operations observe the same strict serializability as single-range transactions. Applications never manage sharding—queries, joins, and transactions work transparently across the entire keyspace regardless of physical data distribution. +Missing or unhealthy time support is an error. Cloud9 does not silently replace +it with a weaker clock. -### Extensibility & Ecosystem +Local mode does not require bounded-time hardware. It keeps the same data model +and transaction interfaces, but it does not claim hardware-backed TrueTime. -The consensus layer is isolated from storage, enabling operational features like dynamic leader placement, witness replicas, and future consensus innovations without rewriting the database core. All queries—SQL, KV, graph, vector—lower into a common Transactional IR (TxIR), making new APIs straightforward to add. SDKs in TypeScript, Python, Go, and Rust provide unified transaction semantics across languages. Open-core model: MIT-licensed engine, with Dedalus Cloud handling managed orchestration, time coordination, and global operations. +## Storage Architecture -## Use Cases +The shared correctness plane owns: -**When SQL alone isn't enough**: -- Real-time dashboards need SQL analytics + KV session state in one transaction -- API gateways need fast KV reads with SQL for complex authorization rules -- Games need KV for player state, SQL for leaderboards and inventory—atomic updates across both +- bounded time and commit-wait; +- multi-version concurrency control (MVCC); +- transaction coordination; +- schemas, catalogs, and object metadata; +- range placement and replica routing; +- Raft replication and recovery. -**When KV alone isn't enough**: -- AI agents need KV for hot state, but also SQL joins to query relationships -- Caching layers need KV performance, but SQL for cache invalidation logic -- Event sourcing needs KV for writes, SQL for projections and queries +Physical engines own their data structures and execution paths. Data may have +several transactional projections, such as a row layout for writes and a +columnar layout for scans. The catalog records which representation is +authoritative and which projections may lag. -**When Postgres isn't enough**: -- Multi-region SaaS platforms that outgrow single-node but need full ACID -- Financial systems requiring global distribution with audit trails and foreign keys -- E-commerce platforms balancing inventory across continents with strict consistency +## Local to Planetary -**When cloud lock-in isn't acceptable**: -- Enterprises requiring self-hosting option with same guarantees as managed tier -- Startups that want to prototype locally before committing to cloud spend -- Regulated industries needing on-premises deployment with global consistency +Local Cloud9 should feel like SQLite: one binary, one directory, and no control +plane. A local database uses one range and one replica. -**When distributed agents are first-class**: -- Autonomous AI systems writing concurrently across regions with deterministic ordering -- Multi-agent workflows requiring vector search + relational data + state in one transaction -- LLM applications needing consistent snapshots across embeddings, metadata, and user data +Distributed Cloud9 partitions data into replicated ranges. Placement follows +data locality and workload shape. -## Status +Cross-range writes use distributed transactions. The target is to scale the +same logical database from one laptop to clusters that span regions and +continents. -**Cloud9 is under active development.** The architecture is proven (Spanner's model, FoundationDB's layers), but Cloud9's unique combination—SQL+KV unification, true Postgres compatibility, MIT license—is new. We're building in public. +## Performance -**Current focus**: -- Core MVCC and transaction coordinator -- Raft consensus implementation -- SQL+KV unification layer -- Postgres wire protocol compatibility +Cloud9 targets the native performance envelope of specialized systems. This is +a benchmark requirement, not a blanket performance claim. -**Not yet implemented**: -- Global deployment automation -- Production-ready vector indexing -- Full Postgres feature parity -- Managed Dedalus Cloud tier +Every performance claim must name the workload, topology, durability mode, +consistency mode, and comparison system. Domain-specific storage and lowering +exist so the benchmark can improve without weakening the common correctness +contract. -## Getting Started +## Build ```bash -# Clone and build -git clone https://github.com/dedalus-labs/cloud9 +git clone https://github.com/windsornguyen/cloud9 cd cloud9 cargo build --release - -# Run single-node instance -./target/release/c9 start --config cloud9.example.toml - -# Run tests cargo test --workspace ``` -For development setup and contribution guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md). +Run the current replicated key-value node with: -## Community +```bash +./target/release/c9 start --config cloud9.example.toml +``` -- **Issues**: [GitHub Issues](https://github.com/dedalus-labs/cloud9/issues) for bug reports and feature requests -- **Discussions**: [GitHub Discussions](https://github.com/dedalus-labs/cloud9/discussions) for questions and ideas -- **Code of Conduct**: [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) +See [the specifications](spec/README.md) for the target design and +[the Jepsen harness](jepsen/README.md) for current distributed tests. ## License @@ -315,9 +146,8 @@ Cloud9 is released under the [MIT License](LICENSE). ## References -- [Spanner: Google's Globally Distributed Database](https://research.google/pubs/pub39966/) -- [Hybrid Logical Clocks](https://cse.buffalo.edu/tech-reports/2014-04.pdf) -- [Raft Consensus Algorithm](https://raft.github.io/) -- [FoundationDB: A Distributed Unbundled Transactional Key Value Store](https://www.foundationdb.org/files/fdb-paper.pdf) -- [Comet: An Active Distributed {Key-Value} -Store](https://www.usenix.org/legacy/event/osdi10/tech/full_papers/Geambasu.pdf) +- [Spanner](https://research.google/pubs/pub39966/) +- [MLIR dialect conversion](https://mlir.llvm.org/docs/DialectConversion/) +- [AWS ClockBound](https://github.com/aws/clock-bound) +- [Amazon Time Sync on EC2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configure-ec2-ntp.html) +- [Raft](https://raft.github.io/) diff --git a/cloud9-proto/proto/cloud9/kv/v1/kv.proto b/cloud9-proto/proto/cloud9/kv/v1/kv.proto index 7fd5604..2da6519 100644 --- a/cloud9-proto/proto/cloud9/kv/v1/kv.proto +++ b/cloud9-proto/proto/cloud9/kv/v1/kv.proto @@ -2,9 +2,8 @@ syntax = "proto3"; package cloud9.kv.v1; -// KvService is Cloud9's namespace/key API. Cloud9 is a relational database with -// a native KV front door; SQL and KV operations should lower into the same -// MVCC/TxIR/transaction-coordinator path. +// KvService is Cloud9's first implemented source dialect. The target +// architecture lowers it through the shared transaction IR into point storage. service KvService { // RegisterSession allocates a client id for idempotent mutating requests. rpc RegisterSession(RegisterSessionRequest) returns (RegisterSessionResponse); diff --git a/cloud9/Cargo.toml b/cloud9/Cargo.toml index f92518e..2155a1b 100644 --- a/cloud9/Cargo.toml +++ b/cloud9/Cargo.toml @@ -7,7 +7,7 @@ license = { workspace = true } authors = { workspace = true } repository = { workspace = true } homepage = { workspace = true } -description = "A globally distributed database with provable external consistency" +description = "An open-source Spanner and MLIR for databases" [[bin]] name = "c9" diff --git a/docs/README.md b/docs/README.md index 12d9538..b4033cb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,21 +1,16 @@ # Cloud9 Documentation -This directory contains design notes and specifications for Cloud9. +This directory contains historical design notes. The normative specifications +live in [`/spec/`](../spec/). ## Specifications -For detailed technical specifications, see **[`/spec/`](../spec/)** directory: - -- **Foundations**: Vision, MVCC, timestamps, external consistency -- **Implementation**: Sharding, transactions, consensus, indexes -- **Deployment**: AWS time infrastructure, TrueTime analysis -- **Market**: Competitive analysis and user pain points -- **Roadmap**: Implementation milestones and testing strategy - -**Start here**: [`/spec/README.md`](../spec/README.md) +Start with [`/spec/README.md`](../spec/README.md). It covers the product +contract, database IR, MVCC, bounded time, transactions, placement, consensus, +catalogs, and delivery order. ## Design Notes -The original consolidated design notes are available at [`design-notes.md`](design-notes.md). This document contains the full conversation history and context but is now superseded by the organized specifications in `/spec/`. - -For focused reading on specific topics, use the spec documents instead. +The original consolidated notes remain in +[`design-notes.md`](design-notes.md). They preserve design history and contain +superseded decisions. Do not use them as the implementation contract. diff --git a/docs/design-notes.md b/docs/design-notes.md index aeb9207..9ca1b12 100644 --- a/docs/design-notes.md +++ b/docs/design-notes.md @@ -1,4 +1,8 @@ -# Cloud9 Design Notes +# Historical Cloud9 Design Notes + +> This document preserves early design discussion. It is not normative and +> contains superseded decisions, including HLC-based timestamp guidance. Use +> [the current specifications](../spec/README.md) for implementation. This document captures the theoretical foundations, architectural decisions, and market insights that define Cloud9. diff --git a/jepsen/README.md b/jepsen/README.md index bab1ef9..0436d3e 100644 --- a/jepsen/README.md +++ b/jepsen/README.md @@ -10,15 +10,14 @@ Each node requires the same 256-bit `cluster.raft_key`; peer RPC bodies are authenticated with HMAC-SHA256 before deserialization. The checked-in example key is only for local and Jepsen testing. -Cloud9 is a relational database first: Postgres-compatible SQL and native KV are -peer APIs over one MVCC storage layer, one transactional IR, one timestamp -system, and one transaction coordinator. The KV workload here is the smallest -front door Jepsen can drive today, not a separate product direction. +Cloud9 treats SQL, key-value, document, object, and analytical APIs as source +dialects. The KV API is the first implemented dialect and the smallest +interface Jepsen can drive today. It does not define the final storage model. The workload maps one shared register to `namespace/key`, writes JSON values as -value bodies, and implements CAS with S3-style ETag preconditions. This KV -surface is only one Cloud9 API front door; SQL and KV are intended to lower into -the same transactional IR. +value bodies, and implements CAS with S3-style ETag preconditions. The target +architecture lowers this request through the shared transaction IR into a +point-operation physical dialect. ## Build diff --git a/spec/00-vision.md b/spec/00-vision.md index dc0bc84..f4d1b64 100644 --- a/spec/00-vision.md +++ b/spec/00-vision.md @@ -1,49 +1,92 @@ # Cloud9 Vision -**Cloud9 is the distributed database that should have existed from the start.** +Cloud9 is an open-source Spanner and MLIR for databases. -Spanner proved that external consistency is achievable with commit-wait and precise time. FoundationDB proved that SQL and KV can share one transactional core. Postgres proved that full ACID with referential integrity is what developers expect. CockroachDB proved that you can build this in the open. +It combines one distributed correctness plane with several database dialects +and physical engines. The same database can serve relational, key-value, +document, object, and analytical workloads without reducing every workload to +one physical model. -**Nobody combined them all.** +## Product Contract -Cloud9 is the synthesis: Spanner's correctness + Postgres's compatibility + FoundationDB's architecture + open-source transparency. No corporate compromises. No vendor lock-in. No "this feature costs extra." Just the theoretically optimal distributed database, available to everyone. +Cloud9 targets two deployment extremes: -## The Core Guarantee +- Local development should feel like SQLite. +- Distributed deployment should scale across regions and continents. -**If you finish a write and start a read, that read sees the write. Anywhere in the world. Always. Provably.** +The data model and transaction interfaces stay stable between them. Hardware +capabilities may differ. In particular, local mode does not claim +hardware-backed TrueTime. -That's external consistency. It's a mathematical guarantee, proven with formal methods. The same guarantee Google uses for ads billing, where every cent must be accounted for. +## Database Dialects -## What Makes Cloud9 Unique +Cloud9 treats APIs as source dialects: -Every distributed database uses proven components (MVCC, Raft, HLC, commit-wait). None combine all of them with: -- SQL and KV unified under one transaction model -- True Postgres compatibility (foreign keys, triggers, constraints) -- Local-to-global deployment with the same binary -- MIT license with no vendor lock-in +- SQL for relational workloads. +- DynamoDB-style key-value operations. +- MongoDB-style document operations. +- S3-style object operations. +- ClickHouse-style analytical plans. -**This isn't novel research—it's what distributed databases should have been from the start.** +Each dialect preserves its source semantics. Compatibility is not an HTTP skin +over a generic row store. -Spanner proved the foundation (external consistency via commit-wait). FoundationDB proved the layering (SQL+KV over one transactional core). Postgres proved the interface (wire compatibility, full ACID). +## Multi-Level IR -Cloud9 is the **disciplined execution** of combining these proven principles into a coherent whole, without the compromises forced by corporate constraints: -- Spanner compromised: SQL-only, no foreign keys, proprietary, cloud-only -- CockroachDB compromised: SQL-only, then went proprietary (BSL) -- YugabyteDB compromised: SQL and KV exist but aren't unified -- DynamoDB compromised: KV-only, eventual consistency, no transactions +Cloud9 uses several typed intermediate representations (IRs). This follows the +same principle as MLIR: preserve high-level meaning until a lower level can +represent it without loss. -**Cloud9 makes no compromises.** External consistency + SQL + KV + open source + local-to-global. +Surface dialects lower into semantic IRs. Semantic IRs lower into transaction, +time, placement, and physical IRs. Physical IRs select row, key-value, +document, object, or columnar execution. -## The Target +The shared layers own correctness. Specialized layers own performance. -**You shouldn't need a Google-sized budget to get Google-class correctness.** +## Core Guarantees -Cloud9 brings external consistency to: -- Students learning distributed systems -- Startups building the next platform -- Enterprises that need bulletproof data -- Developers who refuse to compromise on correctness +Cloud9 is designed around: -Whether you run it on a Raspberry Pi or across continents, the same database, the same guarantees, the same code. +- atomic transactions across compatible dialects; +- multi-version concurrency control (MVCC); +- Raft-replicated state machines; +- explicit locality and placement; +- external consistency when bounded-time hardware is available; +- fail-closed behavior when a required invariant cannot be proven. -**The daily driver database for the distributed era.** +External consistency means real-time order constrains transaction order. If one +transaction finishes before another starts, the first must appear earlier. + +## Bounded Time + +Cloud9 exposes a TrueTime-shaped interval API: + +```text +now() -> [earliest, latest] +``` + +The interval must contain real time. A valid bound permits commit-wait and safe +ordering across machines. + +TrueTime mode requires a supported bounded-time source. The first target is AWS +ClockBound on supported Linux EC2 hardware. + +Cloud9 refuses TrueTime mode when the provider is absent, unhealthy, or outside +the configured uncertainty bound. There is no silent fallback to Hybrid +Logical Clocks (HLCs). + +## Performance Contract + +Cloud9 targets specialized-database performance by preserving specialization. A +columnar analytical engine should not execute through a row-oriented hot path. +An object read should not become a document query. + +Performance claims require reproducible benchmarks. Each result must state the +workload, topology, hardware, durability, and consistency mode. + +## Current Boundary + +The repository is an implementation in progress. Current code includes Raft, +durable log storage, replicated key-value operations, and Jepsen tests. The +remaining dialects, MVCC, distributed transactions, physical engines, and +bounded-time integration are target architecture. diff --git a/spec/01-mvcc.md b/spec/01-mvcc.md index 2f8b317..6347027 100644 --- a/spec/01-mvcc.md +++ b/spec/01-mvcc.md @@ -1,40 +1,106 @@ -# MVCC (Multi-Version Concurrency Control) +# Multi-Version Concurrency Control -**Question**: How do we support lock-free read-only transactions and backups without blocking writes? +Cloud9 uses multi-version concurrency control (MVCC) for transactional +snapshots. Writes create versions instead of overwriting visible state. -**Answer**: Multi-Version Concurrency Control (MVCC). +## Visibility Rule -## The Core Idea +A committed version has a commit timestamp. A read at timestamp `t_read` sees +the newest committed version whose timestamp is at or before `t_read`. -- Each write transaction T_w gets a commit timestamp t_w -- Values are versioned with their write timestamp (not overwritten) -- Each read-only transaction T_r picks a snapshot timestamp t_r -- T_r observes the most recent committed version with t_w ≤ t_r -- Writes with t_w > t_r are invisible to T_r +```text +visible(key, t_read) = + max(version.commit_time <= t_read) +``` -**Result**: Readers and writers operate on different versions. No locks, no blocking. +Later versions are invisible. Tombstones are versions that hide earlier data. +Uncommitted intents are never returned as committed values. -## Why This Is Natural +## Transaction State -MVCC models how time actually works: the past is immutable, observers can choose which moment to examine. A backup reading at timestamp t_r sees a consistent point-in-time snapshot while new writes (t_w > t_r) continue. +A write transaction may create provisional intents. Each intent names its +transaction and proposed value. -## Alternatives Considered +The durable transaction record is authoritative. Valid transitions are: -**Two-Phase Locking (2PL)**: -- Readers take shared locks, writers take exclusive locks -- Backup would lock the entire database for reads OR block all writes -- No temporal queries ("read as of 5 minutes ago") -- Rejected: contradicts "lock-free read-only transactions" goal +- `Pending -> Preparing` +- `Pending -> Aborted` +- `Preparing -> Committed` +- `Preparing -> Aborted` -**Optimistic Concurrency Control (OCC)**: -- Read without locks, validate at commit -- High abort rate under contention -- Backup could abort if overlapping writes occur -- Rejected: poor fit for long-running analytical queries +`Committed` and `Aborted` are terminal. A durable commit decision cannot +become an abort. Replica recovery and client retries must reach the same +terminal result. -**Timestamp Ordering (TO)**: -- Single version per key, enforce timestamp order -- More aborts, no historical reads -- Rejected: need multi-version for temporal queries +## Snapshots -**Verdict**: MVCC is the only scheme that satisfies Cloud9's requirements (lock-free reads, temporal queries, write concurrency). Every modern OLTP database (Postgres, Spanner, CockroachDB, TiDB) uses MVCC for this reason. +A transaction uses one snapshot across all participating ranges and physical +engines. The snapshot includes compatible catalog and schema versions. + +Read-only transactions do not create intents. They may execute without +blocking writes after Cloud9 proves that each participant can serve the chosen +snapshot. + +Physical engines may encode versions differently. They must implement the same +visibility and transaction-state contract. + +## Conflict Detection + +Serializable read-write transactions declare or derive: + +- point and range reads; +- point and range writes; +- predicates that affect the result; +- observed version timestamps. + +Prepare validates that no conflicting committed version or intent invalidates +the snapshot. Predicate validation must detect phantoms. + +Cloud9 aborts on an unresolvable conflict. It does not return a result from a +weaker isolation level. + +## Garbage Collection + +A version can be removed only when no valid reader, backup, change stream, or +recovery operation can still require it. + +Each range tracks a garbage-collection watermark. Advancing it requires proof +that: + +1. no active snapshot is older; +2. retention policy permits deletion; +3. dependent projections have advanced; +4. backups and recovery points no longer reference the version. + +Compaction preserves the newest visible version before the watermark. Deleting +all older versions without that anchor can resurrect stale data. + +## Long-Lived Reads + +Long transactions and backups hold the watermark back. Cloud9 exposes their +age and storage cost. + +Retention pressure may reject a new long-lived operation. It may not silently +delete versions still covered by the operation's snapshot. + +## Schema and Catalog Versions + +Catalog changes are versioned transactionally. A query resolves data, schema, +indexes, and projection metadata at one compatible snapshot. + +An online schema change creates new metadata and may start a backfill. It does +not make the new representation readable until the catalog records that its +required snapshot is complete. + +## Tests + +MVCC tests cover: + +- version visibility at exact timestamp boundaries; +- tombstones and resurrection prevention; +- intent visibility and terminal transaction states; +- point, range, and predicate conflicts; +- snapshot consistency across physical engines; +- garbage collection with active readers; +- crash recovery during commit and cleanup; +- schema and data snapshot alignment. diff --git a/spec/02-timestamps.md b/spec/02-timestamps.md index 2fd7728..5ae4b64 100644 --- a/spec/02-timestamps.md +++ b/spec/02-timestamps.md @@ -1,262 +1,135 @@ -# Timestamp Strategies +# Timestamp Model -## Why Not Lamport Clocks - -Lamport clocks cannot guarantee external consistency, even within a single cluster. - -### The Problem - -Consider this scenario: +Cloud9 uses bounded physical time for external consistency. The provider +returns an interval that contains real UTC: +```text +now() -> [earliest, latest] ``` -1. Client writes to Replica A → Lamport clock assigns L=50 -2. Write commits, client gets "success" -3. Client immediately sends read to Replica B (in real time, right after) -4. Replica B's Lamport clock is at L=49 (hasn't heard from A yet) -5. Read gets timestamp L=49, doesn't see the write (L=49 < L=50) -``` - -**Violation**: Write finished before read started in real time, but read didn't see write. - -### Why This Happens - -Lamport clocks only advance when: -- A local event happens, OR -- A message from another node arrives - -If Replica B hasn't received any messages from Replica A (or other nodes that know about the write) before the client's read arrives, B's clock can be arbitrarily behind—even though the write finished in real time. - -### What Lamport Clocks Actually Solve - -**Designed for**: Ordering events when all communication goes through the system. - -**Perfect use cases**: -- Distributed tracing (causality in logs) -- CRDTs (eventual consistency with causal order) -- Event sourcing (ordering events in a distributed log) -- Deadlock detection (wait-for graph ordering) -**Key property**: If A → B via system messages, then L(A) < L(B). +The interval is a correctness input. It is not an estimate used only for +observability. -**What they don't capture**: If A finishes before B starts in real time, but no message connects them, Lamport clocks don't guarantee L(A) < L(B). +## Provider Contract -### The Database-Specific Issue +A bounded-time provider returns: -In databases, external communication is constant: -- User sees write succeed in UI, refreshes page (new request to different server) -- Microservice A writes, calls microservice B via HTTP, B reads -- Client writes, tells colleague verbally, colleague reads - -**None of these involve database messages**, so Lamport clocks can't track them. - -## The Three Modes - -Cloud9 needs timestamps that respect real-time order. Three viable approaches exist: - -### 1. Hybrid Logical Clocks (HLC) - -**Design**: -```rust -struct HybridTime { - physical: u64, // Wall-clock microseconds - logical: u32, // Tie-breaker for same physical time -} - -fn next_timestamp(&mut self) -> HybridTime { - let now = wall_clock_micros(); - if now > self.last_physical { - HybridTime { physical: now, logical: 0 } - } else { - HybridTime { physical: self.last_physical, logical: self.last_logical + 1 } - } +```text +TimeInterval { + earliest + latest + status } ``` -**How it works**: -- Physical component tracks wall-clock time -- Logical component breaks ties when physical time doesn't advance -- Even without messages, time moves forward (physical clock) - -**External consistency**: -- Commit-wait: After assigning t_w, wait ~ε (clock uncertainty bound) -- Guarantees that any operation starting after commit has timestamp > t_w -- ε determined by clock synchronization (NTP/PTP) - -**Pros**: -- Decentralized (each node has its own HLC) -- Scales well (no single point of bottleneck) -- Captures real-time order via physical component - -**Cons**: -- Requires clock synchronization (NTP/PTP/chrony) -- Must measure and bound clock uncertainty ε -- Commit-wait adds latency (~ε per write) +The provider must guarantee: -**Used by**: CockroachDB, YugabyteDB +1. Real UTC is within the closed interval. +2. `earliest <= latest`. +3. The reported status is healthy. +4. The interval width is within configured policy. +5. Synchronization status and time-scale behavior are defined. -### 2. TrueTime (Spanner's Approach) +Cloud9 validates every observable condition. The platform and provider remain +responsible for the real-time containment guarantee. -**Design**: Clock API that returns uncertainty interval [earliest, latest]. +## Commit Timestamps -```rust -struct TrueTime { - earliest: u64, - latest: u64, -} +A commit timestamp contains bounded physical time and a deterministic +tie-breaker: -fn now() -> TrueTime { - // GPS + atomic clocks give tight bounds - TrueTime { earliest: ..., latest: ... } +```text +CommitTimestamp { + physical + logical } ``` -**How it works**: -- Google uses GPS receivers + atomic clocks -- Publishes bounded uncertainty (typically ~1-7ms) -- Commit-wait until TT.after(commit_ts) is true - -**External consistency**: Same as HLC but with tighter ε (hardware advantage). - -**Pros**: -- Very tight uncertainty bounds (single-digit milliseconds) -- Proven at Google scale +The physical component is at or after the provider's `latest` bound. Cloud9 +rounds upward when timestamp precision requires it. The logical component +orders transactions that share a physical value. -**Cons**: -- Requires specialized hardware (GPS/atomic clocks) -- Not available on public clouds without custom setup -- Still pays commit-wait latency +The tie-breaker does not replace bounded time. It only completes the order +among concurrent transactions. -**Availability**: Not available on AWS/GCP for customer deployments. +## Commit-Wait -### 3. Timestamp Oracle (TSO) - -**Design**: Single service hands out strictly increasing timestamps. - -```rust -struct TSO { - counter: AtomicU64, -} +Cloud9 may acknowledge a commit only after: -impl TSO { - fn next(&self) -> u64 { - self.counter.fetch_add(1, Ordering::SeqCst) - } -} +```text +now().earliest > commit_timestamp.physical ``` -**How it works**: -- All nodes get timestamps from central oracle -- Oracle ensures strict monotonicity -- No commit-wait needed (ordering is explicit) +Raft replication and transaction decision durability happen before this wait. +The response happens after it. -**External consistency**: Guaranteed by request serialization through oracle. +This order is part of the transaction protocol: -**Pros**: -- Simple reasoning (total order at oracle) -- No clock synchronization needed -- No commit-wait on writes +1. Read a valid time interval. +2. Select the commit timestamp. +3. Make the commit decision durable. +4. Wait until the commit timestamp is certainly in the past. +5. Acknowledge the transaction. -**Cons**: -- Oracle is single point of bottleneck -- Oracle must be highly available (itself replicated) -- Adds network hop to oracle for every transaction +## Provider Modes -**Used by**: FoundationDB (Sequencer), TiDB (PD's TSO) +### TrueTime mode -## Trade-offs +TrueTime mode requires an approved bounded-time provider. The first production +backend is AWS ClockBound on supported Linux EC2 hardware with Amazon Time Sync +and a precision hardware clock. -| Aspect | HLC | TSO | TrueTime | -|--------|-----|-----|----------| -| Scalability | High (decentralized) | Medium (oracle bottleneck) | High (decentralized) | -| Write latency | ~ε commit-wait | No commit-wait, but oracle hop | ~ε commit-wait (tight) | -| Dependency | Clock sync (NTP/PTP) | TSO service (must be HA) | GPS + atomic clocks | -| Operational complexity | Clock monitoring | TSO operations | Specialized hardware | -| Failure mode | Fail-stop on skew > max | Block if TSO unreachable | Hardware-dependent | -| Typical ε | 10-50ms (PTP), 50-100ms (NTP) | N/A (logical ordering) | 1-7ms | +Startup fails when the required capability is absent. Operations that depend +on bounded time fail when the provider becomes unhealthy. -## Cloud9's Choice +### Local mode -**Primary mode**: HLC + commit-wait -- Decentralized, scales well -- Works on commodity cloud hardware (AWS Time Sync Service) -- Matches CockroachDB's approach -- Provides meaningful timestamps (wall-clock time) +Local mode may use a local physical clock plus logical ordering. It supports +single-process development without special hardware. -**Alternative mode** (configurable): TSO -- For deployments where clock synchronization is difficult -- Or when ε is too large (poor NTP sync) -- Trades write latency for simpler time discipline +Local mode is a distinct consistency mode. It does not advertise +hardware-backed TrueTime or cross-machine external consistency. -**Future enhancement**: TrueTime-like with GPS/atomic clocks for Dedalus Cloud premium tier (colocated deployments). +## Hybrid Logical Clocks -### Rationale +A Hybrid Logical Clock (HLC) can carry causal metadata and order events. It +cannot prove a bound around real UTC by itself. -HLC provides the best balance of: -1. **Decentralization** - no single point of bottleneck -2. **Practical deployment** - works on standard cloud infrastructure -3. **Real-time semantics** - timestamps have wall-clock meaning -4. **External consistency** - commit-wait ensures correctness +Cloud9 may use HLC-style metadata inside a subsystem. It cannot use an HLC as a +silent replacement for a failed bounded-time provider. -TSO mode exists for edge cases where clock synchronization is unreliable, but HLC is the default because it scales better and provides more meaningful timestamps. +## Failure Rules -TrueTime represents the theoretical ideal but requires specialized hardware not available on public clouds. Cloud9 can achieve similar guarantees with HLC at slightly higher latency (~10-50ms vs ~1-7ms). +Cloud9 fails closed when: -## Implementation Details +- the provider cannot return an interval; +- provider status is unhealthy; +- interval width exceeds policy; +- timestamps move outside the provider contract; +- the host loses the required clock capability. -### HLC Mode: Commit-Wait Protocol +The node becomes unready for operations that require bounded time. It does not +change consistency mode. -``` -1. Coordinator assigns t_w from HLC -2. Replicate via Raft to quorum -3. Commit-wait until now() > t_w + ε -4. Acknowledge to client -``` +## Separate Clocks -**Clock uncertainty measurement**: -- Query chrony/NTP for current offset and jitter -- Set ε = max_observed_offset + drift_allowance -- Fail-stop if observed skew > max_offset (safety) +Elapsed-time mechanisms use a monotonic clock. This includes timeouts, retries, +election timers, and lease duration measurement. -### TSO Mode: Centralized Sequencing +Transaction timestamps use bounded UTC. Mixing these clock domains is an +error. -``` -1. Coordinator requests timestamp from TSO -2. TSO returns strictly increasing u64 -3. Coordinator uses timestamp for commit -4. No commit-wait needed for time uncertainty -``` - -**TSO availability**: -- TSO itself must be replicated (Raft or similar) -- Failure of TSO blocks all writes -- Can pre-allocate timestamp batches to reduce round-trips - -## Why Commit-Wait Is Unavoidable - -For external consistency with physical timestamps, some barrier is required: -- Wait for time (HLC/TrueTime commit-wait), OR -- Wait for order (TSO safe timestamp propagation), OR -- Wait for batch (deterministic sequencing) - -The PACELC theorem applies: Even without network partitions, we must trade latency for consistency. - -**Cloud9's choice**: Pay the latency. External consistency is non-negotiable for a "daily driver" database where users expect intuitive behavior. - -### What Commit-Wait Fixes - -Even with perfectly synchronized clocks, there's a gap between when a leader commits locally and when followers apply the commit. Commit-wait ensures: - -``` -By the time client gets "success," every replica's clock is past t_w. -Any future operation (even immediately after) will get timestamp > t_w. -``` +## Tests -This covers both clock uncertainty and replication propagation time. +The timestamp layer requires: -## References +- provider contract tests; +- malformed and excessive interval tests; +- provider loss and recovery tests; +- leap-state tests; +- commit-wait boundary tests; +- clock-step and suspend tests; +- Jepsen histories that verify real-time transaction order. -- [Time, Clocks, and the Ordering of Events in a Distributed System](https://lamport.azurewebsites.net/pubs/time-clocks.pdf) — Lamport's original paper -- [Logical Physical Clocks and Consistent Snapshots in Globally Distributed Databases](https://cse.buffalo.edu/tech-reports/2014-04.pdf) — HLC paper -- [Spanner: Google's Globally Distributed Database](https://research.google/pubs/pub39966/) — TrueTime and external consistency -- [CockroachDB: The Resilient Geo-Distributed SQL Database](https://dl.acm.org/doi/10.1145/3318464.3386134) — HLC in production -- [FoundationDB: A Distributed Unbundled Transactional Key Value Store](https://www.foundationdb.org/files/fdb-paper.pdf) — Sequencer architecture +Tests must include the actual production provider. A mock proves protocol +logic, not the host time guarantee. diff --git a/spec/03-external-consistency.md b/spec/03-external-consistency.md index 42cfb1b..7d3c4c4 100644 --- a/spec/03-external-consistency.md +++ b/spec/03-external-consistency.md @@ -1,203 +1,103 @@ # External Consistency -## Formal Definition +Cloud9 provides external consistency only when a valid bounded-time provider +is active. -A system provides **external consistency** (also called **strict serializability**) if: +External consistency means transaction order respects real time. If transaction +`T1` completes before transaction `T2` begins, `T1` must appear before `T2` in +the serial history. This property is also called strict serializability. -1. Transactions execute in some serial order -2. This order respects real-time precedence: if T₁ finishes before T₂ starts, then T₁ appears before T₂ in the serial order +## Required Invariants -## What Users Experience +The protocol depends on four invariants: -``` -if write_ack_received_before(read_started): - read_must_see_write() -``` +1. Every committed transaction has one global commit timestamp. +2. Conflict resolution follows commit timestamp order. +3. The bounded-time interval contains real UTC. +4. A commit is not acknowledged until its timestamp is certainly in the past. -**Concrete example**: -``` -Time → +Raft alone does not establish these invariants across independent ranges. +Multi-version concurrency control (MVCC), distributed transaction +coordination, and bounded time complete the protocol. -Client A: write(x=1) → ✓ success -Client A: tells Client B "I wrote x=1" -Client B: read(x) → expects x=1 -``` +## Write Protocol -If the write finished before the read started **in real time**, the read must see the write. This is the fundamental user expectation Cloud9 guarantees. +For a single-range transaction: -## Why It's Hard +1. Evaluate reads and preconditions at a stable MVCC snapshot. +2. Read `[earliest, latest]` from the bounded-time provider. +3. Choose a commit timestamp at or after `latest`. +4. Replicate the deterministic commit command through Raft. +5. Apply the committed versions at that timestamp. +6. Wait until a new `earliest` is greater than the commit timestamp. +7. Return success. -The database doesn't know about the "Client A tells Client B" step. That communication happened outside the database (Slack, verbal, UI navigation, HTTP call between microservices, etc.). +For a cross-range transaction, two-phase commit adds prepare records and one +durable transaction decision. Every participant commits at the same timestamp. +The coordinator performs commit-wait before returning success. -Without special handling, this violation can occur: -``` -1. Replica A commits write with timestamp t₁=100 -2. Client A gets "success" -3. Client B immediately sends read to Replica B -4. Replica B's clock is at 99 (slightly behind due to skew) -5. Read gets timestamp t₂=99 -6. Read doesn't see write (99 < 100) -``` +Retries use the same transaction identity. A retry cannot create a second +logical commit. -## How Cloud9 Achieves External Consistency +## Why Commit-Wait Works -### Write Path +Assume `T1` returns before `T2` starts. When `T1` returns, real time is later +than `T1`'s commit timestamp because commit-wait has completed. -1. Coordinator assigns commit timestamp `t_w` from HLC -2. Replicate via Raft to quorum -3. **Commit-wait**: wait until `now() > t_w + ε` -4. Acknowledge to client +`T2` then chooses a timestamp at or after its provider's `latest` bound. That +bound is at or after real time. Therefore `T2` receives a later timestamp than +`T1`. -**Guarantee**: By the time client gets "success," every replica's clock is past `t_w`. Any future operation (even immediately after) will get timestamp `> t_w`. +The serialization order now respects the observed real-time order. -**Cost**: ~ε latency per write (where ε is clock uncertainty bound). +## Read Protocol -### Read Path +A read-write transaction reads from its chosen MVCC snapshot and validates +conflicts before commit. -1. Pick snapshot timestamp `t_r` (usually `now()` from HLC) -2. Check: "has this replica applied all entries ≤ `t_r`?" -3. If yes: serve read -4. If no: wait until caught up (or redirect to leader) +A read-only transaction may use an explicit timestamp after Cloud9 proves that +all participating ranges have applied through that timestamp. A current read +must also account for bounded-time uncertainty. -**Guarantee**: Because of commit-wait, any `t_r` picked after a write's acknowledgment will be `> t_w`. +Follower reads require an applied-index or safe-time proof. Replica proximity +alone is insufficient. -## The Commit-Wait Necessity +## Provider Failure -**Question**: Can we get external consistency without commit-wait? +If bounded time is unavailable or outside policy, operations that promise +external consistency fail with a typed time-source error. -**Answer**: No. Here's why: +Cloud9 does not: -### Why Synchronization Alone Isn't Enough +- acknowledge first and wait later; +- use wall-clock point estimates as bounds; +- switch to an HLC consistency mode; +- route the request to a weaker implementation. -Even with perfectly synchronized clocks: +Recovery may restore service after the provider is healthy and the node has +re-established its clock contract. -``` -1. Replica A assigns t_w = 100 (from its clock) -2. Replica A commits, returns "success" immediately -3. Client gets success at real time 100.001 -4. Client immediately sends read to Replica B at real time 100.001 -5. Replica B's clock reads 100.0005 (slightly behind due to network/processing) -6. Read gets t_r = 100.0005 -7. Replica B hasn't yet applied the write (replication lag) -8. Read misses the write -``` +## Local Mode -**Issue**: Even with synchronized clocks, there's a gap between: -- When leader commits locally, and -- When followers apply the commit +A single local process can provide serializable transactions through one +scheduler and MVCC. That property does not depend on bounded-time hardware. -### What Commit-Wait Fixes +Local mode does not claim cross-machine external consistency. Moving a +database into distributed TrueTime mode requires an explicit configuration and +capability check. -**Protocol**: -``` -1. Replica A assigns t_w = 100 -2. Replica A replicates to quorum -3. Replica A waits until its clock > 100 + ε -4. Now: all replicas' clocks are guaranteed > 100 -5. Return "success" to client -``` +## Verification -**Guarantee**: Any future operation (anywhere in the cluster) gets timestamp > 100. +Correctness tests must cover: -**The wait covers**: Clock uncertainty + replication propagation time. +- overlapping and non-overlapping transactions; +- single-range and cross-range commits; +- coordinator failure before and after the durable decision; +- leader changes during commit-wait; +- bounded-time loss and excessive uncertainty; +- retry idempotency; +- follower reads and safe-time advancement; +- Jepsen strict-serializability histories. -### Can We Eliminate ε? - -**No.** Here's why: - -**If you use physical clocks**: Uncertainty is unavoidable due to: -- NTP sync error (milliseconds) -- Clock drift between syncs -- Network jitter -- Relativity (if we're being pedantic) - -**If you use a TSO**: No ε for clock uncertainty, but: -- Still need to wait for "safe timestamp" propagation to followers -- Or accept that "read latest" might wait for TSO fence - -**If you use deterministic ordering** (Calvin-style): No ε, but: -- Batching/sequencing latency instead -- Different programming model - -### PACELC: The Inescapable Trade-Off - -For external consistency with wall-clock-meaningful timestamps: -- Some barrier is unavoidable -- Either wait for time (commit-wait ~ε), OR -- Wait for order (sequencer/TSO propagation), OR -- Wait for batch (deterministic pre-ordering) - -**PACELC theorem**: If Partition, choose Availability or Consistency; Else (no partition), choose Latency or Consistency. - -For external consistency, even without network partitions, we must trade latency (commit-wait) for consistency (real-time order). - -**Alternative**: Drop external consistency, use logical timestamps only (faster writes, but can violate user expectations). - -**Cloud9's choice**: Pay the latency. External consistency is non-negotiable for "daily driver" database where users expect intuitive behavior. - -### Optimizations - -**Read-only transactions**: Don't pay commit-wait (no write to acknowledge). - -**Bounded-staleness reads**: Explicitly tolerate staleness to avoid waiting. - -**Follower reads at closed timestamp**: Serve from any replica at a safe, slightly-stale timestamp without coordination. - -## Comparison to Weaker Models - -### Snapshot Isolation (without external consistency) -- Can have write-skew anomalies -- Timestamps might not respect real-time order -- Cheaper (no commit-wait), but weaker guarantees - -### Eventual Consistency -- No ordering guarantees -- Much cheaper, but unusable for Cloud9's goals - -### Linearizability (single-object) -- Only for single-key operations -- Cloud9 provides this as a subset (single-key reads/writes are linearizable) - -**External consistency = Strict Serializability**: Cloud9's target. - -## Why Server-Side Timestamps - -Cloud9 never allows clients to pass timestamps. All timestamp assignment and coordination happens server-to-server. - -### Why Client Timestamps Don't Work - -**Problem 1: Clients can't be trusted** -- Malicious client sends t = infinity → breaks future operations -- Buggy client sends stale timestamp → violates consistency -- Compromised client manipulates ordering - -**Problem 2: Not all communication involves the client** -``` -Client A → DB: write → t₁ -Client A → tells human → human tells Client B (no t₁ passed) -Client B → DB: read → doesn't know about t₁ -``` - -**Problem 3: API complexity** -- Every client library must track timestamps -- Developers must remember to propagate them -- Easy to get wrong, hard to debug - -**Problem 4: Cross-database scenarios** -If DB1 and DB2 are independent systems, client forwarding t₁ from DB1 to DB2 is meaningless (different timestamp spaces). - -### The Right Approach: Server-Side Timestamps - -**Design**: -- Servers assign all timestamps (from HLC or TSO) -- Servers coordinate among themselves (commit-wait, gossip) -- Clients never see or send timestamps -- External consistency guaranteed by database internals - -**Benefits**: -1. Security: clients can't manipulate time -2. Simplicity: client libraries are trivial -3. Correctness: database controls ordering -4. Works for all scenarios (even when clients never communicate) - -**Cloud9**: Server-side only. Clients are dumb, database is smart. +The history must record invocation and completion times. Serializability alone +cannot verify the real-time ordering requirement. diff --git a/spec/04-truetime-analysis.md b/spec/04-truetime-analysis.md index 7f9595f..ce9acf2 100644 --- a/spec/04-truetime-analysis.md +++ b/spec/04-truetime-analysis.md @@ -1,203 +1,120 @@ -# TrueTime Analysis +# Bounded-Time Analysis -## Overview +Cloud9 needs a bounded interval around real UTC. A synchronized point estimate +is insufficient. -TrueTime is not a heuristic - it is a provably correct approach to bounded clock uncertainty in distributed systems. This document explains the mathematical foundations, correctness guarantees, and implications for Cloud9. +Let a provider return: -## The 30-Second Sync Explained - -TrueTime synchronizes with GPS and atomic clocks every 30 seconds. This interval is not arbitrary - it is an engineered choice based on hardware characteristics. - -**Uncertainty formula**: -``` -ε(t) = sync_error + drift_rate × time_since_sync +```text +TT.now() = [earliest, latest] ``` -**Example calculation**: -- `sync_error` = 1 μs (GPS accuracy) -- `drift_rate` = 200 ppm (200 microseconds per second, typical quartz oscillator) -- `time_since_sync` = 30 seconds +The provider contract is: -``` -ε = 1 μs + (200 μs/s × 30s) = 6001 μs ≈ 6ms +```text +earliest <= real_utc <= latest ``` -**The bound is mathematical, not empirical guesswork.** +Define uncertainty as: -## Mathematical Proof of Correctness - -### TrueTime Invariant - -`TT.now()` returns an interval `[earliest, latest]` where: -``` -earliest ≤ absolute_true_time ≤ latest (always) +```text +epsilon = latest - earliest ``` -### How the Invariant Is Maintained - -1. At sync time t₀: measure offset from GPS/atomic clock → `sync_error` -2. Between syncs: bound grows linearly with known `drift_rate` -3. TrueTime daemon continuously computes: `ε(t) = sync_error + drift_rate × (t - t₀)` -4. Returns interval: `[now - ε, now + ε]` - -### Formal Proof - -**Theorem**: If two TrueTime intervals don't overlap (`l₁ < e₂`), then the events happened in that order in absolute time. - -**Proof sketch**: -- Event 1 occurs at absolute time `t₁`, TrueTime returns `[e₁, l₁]` -- By invariant: `e₁ ≤ t₁ ≤ l₁` -- Event 2 occurs at absolute time `t₂`, TrueTime returns `[e₂, l₂]` -- By invariant: `e₂ ≤ t₂ ≤ l₂` -- If `l₁ < e₂`, then `t₁ ≤ l₁ < e₂ ≤ t₂` -- Therefore `t₁ < t₂` (absolute time ordering) - -**Spanner's external consistency** follows from this property combined with the commit-wait protocol. - -## Why It's Not Heuristic +Cloud9 treats this containment rule as a correctness assumption. The deployment +must use an approved provider that can uphold it. -### The Difference from Heuristics +## Commit Rule -**Heuristic approach** would be: "We think clocks are usually within 10ms, so let's use that." +Cloud9 chooses: -**TrueTime approach** is: "We measure sync error, we know drift rate from hardware specs, we compute ε = f(sync_error, drift_rate, time), and we prove that [now - ε, now + ε] contains true time." - -**The correctness is proven**, assuming: -1. Sync error measurement is accurate (GPS provides this) -2. Drift rate is bounded (quartz spec sheets provide this) -3. No Byzantine faults (time masters don't lie maliciously) - -All three are reasonable assumptions with continuous monitoring. - -### Why 30 Seconds Specifically - -**Trade-offs**: -- **Shorter interval** (e.g., 1 second): Lower ε, higher sync overhead -- **Longer interval** (e.g., 5 minutes): Lower overhead, larger ε - -**Google chose 30s** because: -1. With 200 ppm drift, 30s → ~6ms uncertainty (acceptable for write latency) -2. GPS/atomic clocks are stable enough to trust over 30s -3. Sync overhead is negligible (one request per 30s) -4. Safety margin: can tolerate missed sync without ε explosion - -**It's not arbitrary - it's an engineered choice based on hardware characteristics.** - -## Failure Modes - -### Drift Exceeds Specification -- Next sync detects large offset -- ε grows beyond acceptable threshold -- System can refuse writes (fail-safe) or alert operators -- **Response**: Increase commit-wait proportionally or reject transactions - -### GPS Outage -- Atomic clocks continue providing stable reference -- ε stays small for hours (atomic clock stability) -- Fallback: increase ε bound, continue with higher latency -- **Response**: Graceful degradation with documented impact - -### Both GPS and Atomic Fail -- ε grows unbounded -- System must stop writes or increase commit-wait proportionally -- Spanner paper: "conservatively refuse transactions" in this scenario -- **Response**: Fail-stop to preserve correctness +```text +commit_time >= TT.now().latest +``` -### Key Safety Property +It returns success only after: -**TrueTime never violates the invariant.** If uncertainty cannot be bounded, the system: -1. Increases ε (and thus commit-wait latency) -2. OR refuses to assign timestamps -3. Never silently returns incorrect bounds +```text +TT.now().earliest > commit_time +``` -This is **fail-safe**, not fail-fast: the system prioritizes correctness over availability. +At response time, real UTC is therefore later than `commit_time`. -## Implications for Cloud9 +If another transaction begins after that response, its `latest` bound is later +than real UTC at the first response. Its commit timestamp must be later than +the first timestamp. This establishes real-time order for non-overlapping +transactions. -### Cloud9 Must Implement the Same Rigorous Approach +## What Bounded Time Does Not Prove -```rust -struct TimeSource { - last_sync: Instant, - sync_error: Duration, - drift_rate_ppm: f64, -} +Bounded time does not provide: -impl TimeSource { - fn uncertainty(&self) -> Duration { - let elapsed = self.last_sync.elapsed(); - let drift = Duration::from_micros( - (elapsed.as_micros() as f64 * self.drift_rate_ppm / 1_000_000.0) as u64 - ); - self.sync_error + drift - } +- serializable conflict handling; +- atomic commit across ranges; +- durable replication; +- idempotent retries; +- safe follower reads. - fn now_interval(&self) -> (Timestamp, Timestamp) { - let now = Timestamp::now(); - let ε = self.uncertainty(); - (now - ε, now + ε) - } -} -``` +MVCC, transaction coordination, Raft, and recovery provide those properties. +Bounded time connects their serialization order to real time. -### Continuous Monitoring Required +## ClockBound's Role -- Track actual vs expected sync offsets -- Alert if drift_rate exceeds spec -- Fail-stop if ε > max_offset -- Log all sync errors and clock adjustments +AWS ClockBound exposes an interval and clock status to local clients. Cloud9 +uses it as the first implementation of the bounded-time provider contract. -**Not heuristic - measured, bounded, proven.** +ClockBound is not the transaction protocol. Cloud9 still validates provider +health, enforces uncertainty policy, assigns timestamps, and performs +commit-wait. -### Cloud9 Time Synchronization Options +Cloud9 should describe this mode as TrueTime-shaped. Google TrueTime is a +specific Google service. The shared idea is an API that returns a trustworthy +time interval. -Cloud9 implements the same mathematical rigor as TrueTime, but adapts to available infrastructure: +## Hardware Boundary -#### 1. HLC Mode (Default) -- Use NTP/PTP for clock synchronization -- Measure and track ε using chrony statistics -- Commit-wait duration = ε (typically 10-50ms on cloud) -- **Advantage**: Works on commodity hardware -- **Trade-off**: Larger ε than TrueTime +Cloud9's first TrueTime mode requires supported Linux EC2 hardware, Amazon Time +Sync, a precision hardware clock, and ClockBound. The exact supported instance +families and drivers follow current AWS documentation. -#### 2. GPS + Atomic Clocks (Premium) -- Install GPS receivers and atomic clocks (colo/on-prem) -- Direct PTP feed to Cloud9 nodes -- Achieve ε < 1ms (TrueTime-class performance) -- **Advantage**: Minimal commit-wait latency -- **Trade-off**: Hardware cost and operational complexity +An ordinary NTP-synchronized system clock does not satisfy this mode. An HLC +also does not satisfy it. Either could support a separately named consistency +mode, but neither may appear as an automatic fallback. -#### 3. TSO Mode (Alternative) -- Use centralized timestamp oracle instead of physical time -- No clock synchronization needed -- External consistency guaranteed by serialization -- **Advantage**: Simpler when clock sync is unreliable -- **Trade-off**: Oracle becomes bottleneck +## Uncertainty Cost -### The Key Insight +Commit-wait latency grows with the uncertainty interval. A wider bound is still +correct when it remains within policy, but it delays acknowledgements. -**The protocol (commit-wait + bounded uncertainty) is what matters, not the specific hardware.** +Performance work should reduce measured uncertainty without weakening the +containment guarantee. Benchmarks must report interval width and commit-wait +time. -TrueTime achieves tight bounds (ε < 7ms) because Google has GPS + atomic clocks. Cloud9 can achieve the same correctness with looser bounds (ε = 10-50ms) using NTP/PTP. The latency differs, but the guarantees are identical. +## Failure Model -**External consistency is provable in both cases** - the math doesn't change, only the constant ε. +Cloud9 rejects the provider when: -### What Cloud9 Learns from TrueTime +- status reports unsynchronized or unknown time; +- uncertainty exceeds configured policy; +- the interval is malformed; +- the host loses the required hardware path; +- the provider daemon or client interface is unavailable. -1. **Bounded uncertainty is non-negotiable**: Must measure and enforce ε -2. **Fail-safe is correct**: Refuse transactions rather than violate invariants -3. **Continuous monitoring is essential**: Track clock health in real-time -4. **Hardware determines ε, protocol ensures correctness**: Both matter -5. **Document the math**: Users trust provable systems over heuristics +These failures remove readiness for TrueTime-dependent operations. Existing +data remains durable. The node does not silently change its consistency +contract. -## Summary +## Proof Obligations -TrueTime is not a heuristic. It is a formally proven approach to bounded clock uncertainty: +A production backend needs evidence for: -- **Sync every 30 seconds** is an engineered choice based on drift rate math -- **ε = sync_error + drift_rate × time** is a proven bound on uncertainty -- **[earliest, latest] contains absolute time** is a maintained invariant -- **External consistency follows** from this invariant + commit-wait -- **Failure modes are explicit** and preserve correctness (fail-safe) +1. Real UTC containment under normal operation. +2. Detection of source loss and clock steps. +3. Correct leap-second behavior. +4. Safe behavior across suspend, resume, and migration. +5. Correct interval propagation into commit-wait. +6. A maximum accepted uncertainty policy. +7. End-to-end histories that verify strict serializability. -Cloud9 implements the same rigorous approach, adapting to available time infrastructure while maintaining identical correctness guarantees. +Unit tests can prove Cloud9's interval arithmetic. Hardware integration tests +must prove the provider assumptions. diff --git a/spec/05-aws-time-infrastructure.md b/spec/05-aws-time-infrastructure.md index 15763c8..77a2942 100644 --- a/spec/05-aws-time-infrastructure.md +++ b/spec/05-aws-time-infrastructure.md @@ -1,639 +1,138 @@ -# Time Infrastructure and External Consistency - -This document defines Cloud9's strategy for achieving external consistency through bounded-error time across multiple cloud providers. - -## The Final Position - -**Cloud9 exposes a TrueTime-style interval API with pluggable time backends.** - -Default: **ClockBound on AWS** (microsecond-class ε, zero hardware cost) -Fallback: **HLC or generic PTP** (works anywhere, wider ε) - -This architecture gives Cloud9: -- Best-in-class latency on AWS (competitive with Spanner) -- Multi-cloud portability (works on Azure, GCP, on-prem) -- Transparent ε contracts (published as operational SLO) -- Future extensibility (add GPS/atomic grandmasters without redesign) - -## The Core Insight - -**AWS provides the primitives for a TrueTime-style contract**—no special hardware required. - -On Nitro instances, AWS exposes the **Amazon Time Sync Service as a local PTP hardware clock (PHC)** accessible via `/dev/ptp0`. Combined with **ClockBound** (AWS's open-source daemon that reads chrony error bounds), Cloud9 can expose `now()` as an interval `[earliest, latest]` with a measured error bound ε—exactly the API TrueTime provides, but built from measured telemetry instead of vendor guarantees. - -## The Three-Tier Strategy - -### Tier 1: Nitro + PTP PHC + ClockBound (Default) - -**What it is**: -- Amazon Time Sync Service accessed via PTP hardware clock on Nitro instances -- ClockBound daemon exposes time intervals with measured ε -- No additional AWS fees (included with EC2) -- No hardware to buy - -**Expected ε**: -- **Single instance**: Low double-digit microseconds (10-100 μs within guest OS) — **AWS-documented** -- **Cross-AZ**: Target 0.5-2 ms — **must measure and enforce** (not AWS-guaranteed) -- **Cross-region**: Target 1-5 ms — **must measure and enforce** (varies by region pair, network, drift) - -**AWS added nanosecond-precision hardware packet timestamps** (2025) for improved measurement and telemetry. - -**Key clarification**: -- NTP from Time Sync: **millisecond-class** (not 50-100ms as originally stated) -- PTP/PHC: **microsecond-class** (documented by AWS for single-instance) -- Cross-node ε: **Measured from chrony/ClockBound**, not guaranteed by AWS - -**Important**: AWS publishes microsecond accuracy **per instance**. Cross-node/AZ/region bounds are **your responsibility to measure and enforce**. - -**Important**: NTP from Time Sync is **leap-smeared** (smooths leap seconds over 24 hours). PTP/PHC follows **UTC** (no smear). Don't mix modes within a cluster. - -**Setup**: -```bash -# Ensure ENA driver is current (for PTP device exposure) -sudo yum update ena # Amazon Linux - -# Configure chrony to use PTP PHC (PHC0 / /dev/ptp0) -cat > /etc/chrony/chrony.conf < (Timestamp, Timestamp) { - let bound = self.client.now().expect("ClockBound unavailable"); - ( - Timestamp::from_micros(bound.earliest), - Timestamp::from_micros(bound.latest), - ) - } - - fn uncertainty(&self) -> Duration { - let bound = self.client.now().expect("ClockBound unavailable"); - Duration::from_micros(bound.latest - bound.earliest) - } -} -``` - -**Commit-wait**: -```rust -async fn commit_wait(commit_ts: Timestamp, time: &impl TimeProvider) { - loop { - let (earliest, _) = time.now_interval(); - if earliest > commit_ts { - return; // All clocks definitely past commit_ts - } - tokio::time::sleep(Duration::from_micros(100)).await; - } -} -``` - -**Cost**: $0 incremental (included with Nitro instances) - -**Deployment tier**: **Performance** — Recommended default for production. - -### Tier 2: GNSS + Rubidium PTP Grandmaster (Premium) - -**What it is**: -- GPS-disciplined PTP grandmaster in colocation facility -- Rubidium atomic oscillator for holdover -- Distribute time via PTP to Cloud9 nodes - -**Expected ε**: -- **Intra-rack**: 1-10 μs (hardware timestamping) -- **Intra-colo**: 10-100 μs (Layer-2 PTP) -- **Cross-site** (with low-jitter links): 100-500 μs - -**Hardware costs**: -- GNSS PTP grandmaster with Rb holdover: $9-12k per unit -- Minimum 2 units for redundancy: $18-24k -- PTP-aware switches, cabling, roof antenna: $2-6k -- **Total one-time**: $25-60k per site - -**Recurring costs**: -- Colocation: $1-3k/month per cabinet (region-dependent) -- Maintenance and spares: $500-1k/month - -**When to use**: Cross-node ε must be deterministically <100 μs (rare). - -**Deployment tier**: **Premium** — For customers requiring Spanner-class latency. - -### Tier 3: Outposts (AWS-Managed Premium) - -**What it is**: -- AWS Outposts rack in your facility -- Bring custom PTP grandmaster or use AWS Time Sync over Outposts -- Hybrid cloud/on-prem model - -**Expected ε**: Similar to Tier 2 (1-10 μs intra-rack with custom PTP) - -**Cost**: Outposts capacity commitment (often $100k+ multi-year) - -**When to use**: Already using Outposts for other reasons and need tight time bounds. - -**Deployment tier**: **Premium (Managed)** — When you want colo-class time with AWS operations. - -## ClockBound: AWS's TrueTime Equivalent - -**What ClockBound provides**: -- Time intervals: `[earliest, latest]` with measured error bound -- Based on chrony's tracking (offset, dispersion, drift) -- Continuous monitoring and bound publication -- Fail-safe when uncertainty exceeds threshold - -**From AWS documentation**: -> "ClockBound uses the chronyd process to get an accurate value of the time and the associated error bound. ClockBound gets this information from the chrony tracking report." - -**This is functionally equivalent to TrueTime's API**: Bounded error intervals suitable for commit-wait protocols. - -**Key facts about Amazon Time Sync Service**: -- Backed by **satellite-connected and atomic clocks** in each AWS Region -- GPS + atomic reference (similar infrastructure to TrueTime) -- No vendor-guaranteed ε (you measure it yourself) -- But underlying discipline is tight - -**Important**: ClockBound provides the **mechanism** (interval API like TrueTime), but Cloud9 provides the **guarantee** (by measuring ε and enforcing max-offset). - -**ClockBound ≠ TrueTime**: -- **Shape**: Same (returns `[earliest, latest]` intervals) -- **Source**: Different (ClockBound = your measurements via chrony; TrueTime = Google's GPS+atomic fleet with operational guarantees) -- **Contract**: You own the ε contract with ClockBound; Google owns it with TrueTime - -The Amazon Time Sync Service backend is GPS + atomic (high-quality), but AWS doesn't publish a vendor-guaranteed regional ε. You measure, publish, and enforce your own bounds. - -**Sources**: -- [ClockBound GitHub](https://github.com/aws/clock-bound) -- [AWS: Compare timestamps with ClockBound](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/compare-timestamps-with-clockbound.html) -- [AWS Blog: Microsecond-Accurate Clocks on EC2](https://aws.amazon.com/blogs/compute/its-about-time-microsecond-accurate-clocks-on-amazon-ec2-instances/) -- [AWS: Introducing Amazon Time Sync Service](https://aws.amazon.com/about-aws/whats-new/2017/11/introducing-the-amazon-time-sync-service/) - -## Multi-Cloud Portability - -**The portable design**: - -```rust -pub trait TimeProvider: Send + Sync { - fn now_interval(&self) -> (Timestamp, Timestamp); - fn uncertainty(&self) -> Duration; - fn healthy(&self) -> bool; -} +# AWS ClockBound Backend + +Cloud9's first production bounded-time backend uses AWS ClockBound. It is +available only on hosts that satisfy the declared hardware and software +contract. + +## Deployment Contract + +A TrueTime-enabled node requires: + +- supported Linux on EC2; +- an AWS precision-time placement group; +- a currently supported Nitro instance family; +- a supported Elastic Network Adapter (ENA) driver; +- the Amazon Time Sync precision hardware clock (PHC); +- a healthy ClockBound daemon and client library; +- permission to read the ClockBound shared-memory segment. + +The current instance and driver matrix belongs to AWS documentation. Cloud9 +should test capabilities instead of embedding a stale family list. + +## Data Path + +```text +Amazon Time Sync + | +EC2 precision hardware clock + | +clock synchronization service + | +ClockBound daemon + | +ClockBound client + | +Cloud9 TimeSource + | +timestamp assignment and commit-wait ``` -**Implementations**: - -1. **AwsClockBound** (Tier 1): Read intervals from ClockBound daemon -2. **AzurePtpPhc** (Tier 1): Read chrony tracking on Azure VMs with `/dev/ptp*` -3. **GcpNtp** (Tier 1, wider ε): Compute bound from chrony dispersion on GCP -4. **GenericPtp** (Tier 2): Read chrony tracking with custom PTP grandmaster -5. **HlcFallback** (Degraded): Pure HLC when bounded-error unavailable - -**GCP and Azure support**: - -**Azure**: VMs expose `/dev/ptp*` sourced from Microsoft GPS fleet. Use chrony + read tracking for ε. **Same pattern as AWS.** - -**GCP**: NTP-only (no PHC/PTP exposed to VMs). Compute ε from chrony's dispersion/offset. **Wider ε but works.** - -**On-prem/any cloud**: Install your own PTP grandmaster (GNSS + Rb), use GenericPtp adapter. - -**The interface is cloud-agnostic. The ε varies by infrastructure.** - -## Why This Doesn't Lock Cloud9 to AWS - -**Pluggable time backend**: -- Cloud9 defines `TimeProvider` interface -- Ships with adapters for AWS, Azure, GCP, generic PTP, HLC fallback -- External consistency guaranteed **when ε is bounded** -- System gracefully degrades to wider ε or HLC mode when unavailable - -**Open-source Cloud9**: -- Runs on any cloud (AWS best, Azure good, GCP okay, on-prem with PTP) -- No AWS lock-in (other clouds have PTP or can add it) -- Generic PTP adapter works anywhere -- HLC fallback for environments without bounded time - -**Managed Dedalus Cloud**: -- Optimizes for AWS Nitro (best ε out of box) -- Supports Azure (similar to AWS) -- Supports GCP (wider ε, still works) -- Can deploy in colo with customer PTP (premium tier) - -## The Correct ε Ranges - -**AWS Nitro + PTP PHC + ClockBound**: -- Single instance: 10-100 μs (not 50ms!) -- Cross-AZ: 100 μs - 2 ms (not 10-50ms!) -- Cross-region: 1-5 ms (not 50-100ms!) - -**Our original spec was off by 1-3 orders of magnitude.** - -## The TimeProvider Interface - -**Cloud9's portable abstraction**: - -```rust -/// Pluggable time backend for external consistency. -pub trait TimeProvider: Send + Sync { - /// Returns [earliest, latest] interval containing true time. - fn now_interval(&self) -> (Timestamp, Timestamp); - - /// Current uncertainty bound (latest - earliest). - fn uncertainty(&self) -> Duration; - - /// Whether time source is healthy and within acceptable bounds. - fn healthy(&self) -> bool; - - /// Name of this provider (for metrics/logging). - fn name(&self) -> &'static str; -} -``` - -**Implementation across clouds**: - -```rust -// AWS: ClockBound (TrueTime-shaped, microsecond ε) -pub struct AwsClockBound { - client: ClockBoundClient, - max_uncertainty: Duration, -} - -// Azure: PTP PHC via chrony (similar to AWS) -pub struct AzurePtpPhc { - chrony_tracking: ChronyClient, - max_uncertainty: Duration, -} - -// GCP: NTP with computed ε (millisecond-class) -pub struct GcpNtp { - chrony_tracking: ChronyClient, - max_uncertainty: Duration, -} - -// On-prem/generic: Customer-provided PTP -pub struct GenericPtp { - chrony_tracking: ChronyClient, - max_uncertainty: Duration, -} - -// Fallback: Pure HLC (works anywhere with basic NTP) -pub struct HlcFallback { - max_offset: Duration, // 250-500ms like CockroachDB -} -``` - -**Commit-wait implementation** (same across all backends): - -```rust -async fn commit_wait(commit_ts: Timestamp, time: &impl TimeProvider) -> Result<()> { - if !time.healthy() { - return Err(Error::ClockUnhealthy { - provider: time.name(), - uncertainty: time.uncertainty(), - }); - } - - loop { - let (earliest, _) = time.now_interval(); - if earliest > commit_ts { - return Ok(()); // All clocks definitely past commit_ts - } - tokio::time::sleep(Duration::from_micros(10)).await; - } -} -``` - -## Implementation Priority - -**P0: AwsClockBound adapter** (launch requirement) -- Most users on AWS -- Best ε without custom hardware (10-100 μs) -- Free (included with Nitro) -- Proven (AWS uses ClockBound internally) -- **This is the flagship implementation** - -**P1: HlcFallback adapter** (portability baseline) -- Works anywhere with basic NTP -- Max-offset policy (250-500ms like CockroachDB) -- Ensures "runs anywhere" promise -- Used when ClockBound unavailable - -**P2: AzurePtpPhc adapter** (multi-cloud expansion) -- Azure VMs expose PTP PHC (`/dev/ptp*`) -- Similar ε to AWS (10-100 μs) -- Same chrony-based approach -- Enables Azure-native deployments - -**P3: GcpNtp adapter** (multi-cloud completion) -- Compute ε from chrony tracking/dispersion -- Millisecond-class ε (wider than AWS/Azure) -- Enables GCP deployments -- Still better than pure HLC - -**P4: GenericPtp adapter** (enterprise/on-prem) -- Customer-provided PTP grandmaster -- For regulated/sovereign deployments -- Enables custom time infrastructure - -**Future: GPS/Atomic tier** (premium) -- Colocation with GNSS + Rubidium grandmasters -- Sub-100 μs cross-node ε -- Plugs into same TimeProvider interface -- No database redesign needed - -## Key Insight: ε Drives Latency, Not API Choice - -**Performance depends on ε, not whether you call it "HLC" or "TrueTime".** - -Both approaches do the same thing at commit: **wait until now ≥ commit_ts**. The latency you pay is proportional to the clock uncertainty bound ε. - -**With tight ε** (AWS PTP/PHC, 10-100 μs): -- Commit-wait: ~0 (often overlapped with replication) -- Write latency dominated by quorum RTT, not time - -**With loose ε** (plain NTP, milliseconds): -- Commit-wait: milliseconds -- Noticeable impact on write latency - -**The API (HLC vs TrueTime-style intervals) doesn't change this.** What matters: -1. Clock discipline (PTP/PHC vs NTP) -2. Max-offset policy (how tight you enforce) -3. Measured ε (continuous monitoring) - -## Why CockroachDB Uses HLC + Max-Offset - -**CockroachDB's constraint**: Must run **anywhere** (AWS, GCP, Azure, on-prem, air-gapped). - -**Their choice**: HLC + max-offset (500ms default, tunable to 250ms) works everywhere with basic NTP. - -**Trade-off**: Portability (runs anywhere) vs latency (hundreds of ms safety margin). - -**Why they don't use ClockBound**: -- Would tie them to AWS-specific APIs -- Multi-cloud customers would have different time backends -- On-prem/air-gapped wouldn't have bounded-error source -- One design must work uniformly everywhere - -**Cloud9's advantage**: -- **Use ClockBound/PTP on AWS** (most customers, microsecond ε) -- **Use Azure PTP PHC** (same pattern, similar ε) -- **Use GCP NTP** (compute ε from chrony, wider but works) -- **Fall back to HLC** when bounded-error unavailable (air-gapped, on-prem) - -CockroachDB chose **portability-first** (one design, works everywhere, conservative). -Cloud9 chooses **performance-first** (optimize for each cloud, graceful degradation). - -## Competitive Landscape - -**How others achieve external consistency**: - -| Database | Approach | ε Typical | Clock Dependency | -|----------|----------|-----------|------------------| -| **Spanner** | TrueTime + commit-wait | 1-7 ms | GPS + atomic (proprietary) | -| **CockroachDB** | HLC + max-offset | 250-500 ms | NTP (any cloud) | -| **YugabyteDB** | HLC ("hybrid time") | Not strict external consistency | NTP (any cloud) | -| **TiDB** | TSO (centralized sequencer) | No ε (logical time) | None (sequencer is truth) | -| **FoundationDB** | Sequencer (OCC + MVCC) | No ε (logical versions) | None (sequencer is truth) | -| **Cloud9 (AWS)** | ClockBound + commit-wait | 10-100 μs (single-AZ)
0.5-5 ms (multi-region) | PTP/PHC (AWS Time Sync) | -| **Cloud9 (other)** | HLC + commit-wait or TSO | Varies by infrastructure | PTP (Azure), NTP (GCP), or HLC | - -**Cloud9's positioning**: -- On AWS: Measured ε typically 10-100 μs to low ms (vs CRDB's 250-500ms max-offset threshold) -- Multi-cloud: Pluggable time backend with graceful degradation -- External consistency via measured ε and strict enforcement - -**Note**: CockroachDB's max-offset is a **safety threshold** (triggers shutdown), not per-transaction commit-wait. Their steady-state latency is dominated by quorum RTT and placement, not the 250-500ms number. Cloud9's advantage is **tighter measured ε** for commit-wait, not "100x faster writes." - -## Monitoring and Fail-Safe - -**Continuously track ε**: -```bash -# Check current uncertainty -clockbound-client now - -# Monitor chrony tracking -watch -n 1 'chronyc tracking' -``` - -**Fail-safe when ε exceeds threshold**: -```rust -const MAX_UNCERTAINTY_MS: u64 = 10; - -fn check_clock_health(time: &impl TimeProvider) -> Result<()> { - let uncertainty = time.uncertainty(); - if uncertainty > Duration::from_millis(MAX_UNCERTAINTY_MS) { - return Err(Error::ClockUncertaintyExceeded { - measured: uncertainty, - max_allowed: MAX_UNCERTAINTY_MS, - }); - } - Ok(()) -} - -// Refuse writes when clock is unhealthy -async fn handle_write(req: WriteRequest, time: &impl TimeProvider) -> Result<()> { - check_clock_health(time)?; // Fail-stop if ε too large - // ... proceed with write -} -``` - -**Operational SLO**: Publish ε as metric, alert when > threshold, refuse writes when unsafe. - -## Cost and Feasibility Summary - -### Initial Launch (AWS-Only, No Custom Hardware) - -**Engineering cost**: $30-50k (one-time) -- 0.3 FTE for 3-4 months -- Build TimeProvider interface -- Implement AwsClockBound adapter -- Add ε monitoring and fail-stop logic -- Test commit-wait protocol - -**Annual operations**: $15k/year -- 0.1 FTE for monitoring/alerting -- Dashboard for ε tracking -- Node quarantine automation - -**AWS infrastructure**: $0 incremental -- Amazon Time Sync Service included with Nitro -- ClockBound is open-source -- No additional AWS fees - -**Total first-year cost**: $45-65k (mostly engineering) - -### If Custom Hardware Needed (Future) - -**Full multi-region TrueTime infrastructure**: -- GNSS PTP grandmasters: $20-50k upfront per region -- Atomic clocks (Rubidium): $10-20k per region -- Network (PTP switches, boundary clocks): $10-30k -- Colocation: $1-3k/month per cabinet -- Operations: $30-50k/year - -**Total multi-region (3 regions)**: $100-300k upfront, $30-50k/year ongoing - -### Recommendation - -**Launch with Tier 1** (ClockBound on AWS): -- Sufficient for 99% of deployments -- Competitive with Spanner (both have low-ms commit-wait) -- Better than CockroachDB (tighter measured ε) -- Zero hardware cost - -**Add custom hardware only if**: -- Cross-node ε must be deterministically <100 μs -- Financial/HFT workloads requiring <1ms global commits -- Regulatory requirements for owned time infrastructure - -**For most use cases, ClockBound is sufficient.** - -## Cloud9's Final Time Strategy - -**The decision**: Expose TrueTime-style interval API with pluggable backends. +The provider returns an earliest time, latest time, and status. Cloud9 converts +those values into its internal time interval without discarding the PHC error +bound. -**Why this is correct**: +## Startup Checks -1. **Performance**: ClockBound + PTP/PHC on AWS gives microsecond-class ε (competitive with Spanner) -2. **Portability**: TimeProvider abstraction works on any cloud (graceful degradation) -3. **Cost**: $0 hardware for 99% of deployments (vs $100k+ for custom GPS/atomic) -4. **Transparency**: Publish live ε as SLO (users know exactly what they get) -5. **Future-proof**: Can add GPS/atomic tier without redesigning database +A node configured for TrueTime mode becomes ready only after it verifies: -**What Cloud9 delivers**: -- External consistency with sub-millisecond commit-wait on AWS -- Portable to Azure (similar ε), GCP (wider ε), on-prem (custom PTP or HLC) -- Open-source with no cloud lock-in (same interface, different ε) -- Competitive with Spanner on AWS, better than CockroachDB everywhere +1. The required PHC is present. +2. ClockBound is installed and reachable. +3. The daemon reports a synchronized status. +4. Returned intervals are well formed. +5. Uncertainty is within configured policy. +6. The PHC and synchronization service match deployment policy. +7. Time-scale and leap behavior match cluster policy. -**What Cloud9 does not claim**: -- ❌ "ClockBound is TrueTime" (it's TrueTime-shaped, not TrueTime-guaranteed) -- ❌ "AWS guarantees cross-AZ ε" (AWS documents single-instance; we measure cross-node) -- ❌ "100x faster than CockroachDB" (their max-offset is safety threshold, not latency) +Failure names the missing capability. Startup does not switch to another clock +implementation. -**What Cloud9 can legitimately claim**: -- ✅ "TrueTime-style external consistency on AWS with zero hardware cost" -- ✅ "Measured ε published as operational SLO (transparent uncertainty)" -- ✅ "Sub-millisecond commit-wait typical on AWS Nitro (competitive with Spanner)" -- ✅ "Works on any cloud with pluggable time backend (portable)" +## Runtime Checks -**The architecture balances**: -- Performance (optimize for AWS where most users are) -- Portability (works anywhere with degradation) -- Transparency (publish ε, don't hide uncertainty) -- Cost (free for default tier) +Every time sample carries provider status. Cloud9 rejects a sample before using +it when status is unhealthy or its interval violates policy. -This is the frontier for open-source distributed databases: Spanner-class guarantees on commodity cloud infrastructure. +The node publishes: -## Performance Comparison to Spanner +- interval width; +- provider status; +- sample failures; +- commit-wait duration; +- time since the last healthy sample; +- readiness for TrueTime-dependent operations. -### Spanner's Published Numbers +Alerts should fire before uncertainty reaches the rejection threshold. -From the Spanner OSDI paper and Google Cloud documentation: -- TrueTime uncertainty (ε): "Generally <10 ms" -- Commit-wait: ~5 ms (microbenchmarks) -- Write latency: Quorum RTT + commit-wait +All nodes use one time scale. AWS NTP smears leap seconds while the PHC does +not. Cloud9 rejects a mixed configuration. -### Cloud9 on AWS (Expected) +## Failure Behavior -**Single-region deployments**: -- ε: 10-100 μs (single-instance) to 0.5-2 ms (cross-AZ) -- Commit-wait: Sub-millisecond (often overlapped with replication) -- Write latency: Dominated by Raft quorum, not commit-wait +When ClockBound becomes unavailable, Cloud9 fails operations whose correctness +depends on bounded time. The node reports a typed provider error and becomes +unready for those operations. -**Multi-region deployments**: -- ε: 1-5 ms (measured, varies by region pair) -- Commit-wait: 1-5 ms (same ballpark as Spanner) -- Write latency: Inter-region quorum RTT + commit-wait +Cloud9 does not: -### The Comparison +- read the ordinary system clock as a substitute; +- replace the provider with an HLC; +- reuse an expired interval; +- accept an interval whose status is unknown; +- acknowledge a commit before commit-wait completes. -**Cloud9's commit-wait latency** ≈ **0.5-1× Spanner's** (same ballpark, sometimes better in-region) +Service resumes after ClockBound is healthy and the node re-establishes the +provider contract. -**Why Cloud9 can match**: -- AWS's infrastructure uses GPS + atomic clocks (similar to Google) -- ClockBound provides the interval API (same shape as TrueTime) -- Smaller deployments = lower network RTT (advantage for Cloud9) -- Same commit-wait protocol (wait until now > commit_ts) +## ClockBound and TrueTime -**Why Cloud9 is different**: -- $0 hardware cost (vs Google's GPS/atomic infrastructure) -- Measured ε (you own the contract) vs vendor-guaranteed ε -- Open-source (transparent about ε) vs proprietary -- Multi-cloud portable (works on Azure/GCP with wider ε) vs GCP-only +ClockBound supplies a host-local bounded-time interval. Cloud9 supplies the +database protocol that consumes it. -### The Legitimate Claim +The integration is TrueTime-shaped because both expose an interval around real +time. Cloud9 does not claim to run Google's TrueTime service. -**Cloud9 delivers 90-95% of Spanner's external-consistency performance for <1% of the cost and complexity.** +## Security Boundary -**Breakdown**: -- Commit-wait latency: ✅ Same (both sub-ms to low-ms depending on deployment) -- External consistency: ✅ Same (both proven with commit-wait) -- Hardware cost: ✅ Cloud9 wins ($0 vs Google's GPS/atomic fleet) -- Portability: ✅ Cloud9 wins (multi-cloud vs GCP-only) -- Transparency: ✅ Cloud9 wins (publish ε vs hidden) +The time path is trusted infrastructure. A process that can falsify ClockBound +state or its shared-memory data can violate external consistency. -**The one trade-off**: Google guarantees ε, Cloud9 measures it. But for operational purposes, this doesn't matter—both enforce external consistency via commit-wait on bounded ε. +Deployments must restrict that interface, pin supported versions, and include +time configuration in host attestation and change control. -### Engineering Reality Check +## Local Development -**What this requires**: -- ~0.3 FTE for 3-4 months ($30-50k engineering) -- Nitro instances (already using) -- ClockBound integration (open-source, documented) -- Chrony configuration (standard sysadmin) -- Monitoring infrastructure (standard ops) +Local mode does not emulate ClockBound and does not claim hardware-backed +TrueTime. Tests may inject a deterministic bounded-time provider to exercise +protocol logic. -**Not required**: -- GPS receivers ($10k+ per site) -- Atomic clocks ($10-50k per site) -- Custom time service team -- Multi-year infrastructure buildout +Production certification still requires the supported EC2 path. A mock cannot +prove the host clock guarantee. -**Timeline**: 3-4 months to production-ready TimeProvider with ClockBound backend. +## Validation -**Confidence level**: High. ClockBound is AWS-supported, PTP/PHC is documented, ε measurements are observable. +The AWS test suite must cover: -This is the frontier for open-source distributed databases: Spanner-class guarantees on commodity cloud infrastructure. +- clean startup on supported hardware; +- rejection on unsupported hardware; +- daemon stop and restart; +- PHC loss or synchronization failure; +- excessive uncertainty; +- clock steps and leap state; +- process suspend and resume; +- leader change during commit-wait; +- strict-serializability histories under network and process faults. ## References -- [AWS Blog: Microsecond-Accurate Clocks on EC2](https://aws.amazon.com/blogs/compute/its-about-time-microsecond-accurate-clocks-on-amazon-ec2-instances/) -- [AWS Docs: Set time reference with PTP](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configure-ec2-ntp.html) -- [AWS Docs: Compare timestamps with ClockBound](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/compare-timestamps-with-clockbound.html) -- [ClockBound GitHub](https://github.com/aws/clock-bound) -- [AWS: Introducing Amazon Time Sync Service](https://aws.amazon.com/about-aws/whats-new/2017/11/introducing-the-amazon-time-sync-service/) -- [Azure: Time sync for Linux VMs](https://learn.microsoft.com/en-us/azure/virtual-machines/linux/time-sync) -- [GCP: Configure NTP](https://docs.cloud.google.com/compute/docs/instances/configure-ntp) -- [CockroachDB: Clock Management](https://www.cockroachlabs.com/blog/clock-management-cockroachdb/) -- [Spanner OSDI Paper](https://research.google.com/archive/spanner-osdi2012.pdf) +- [AWS ClockBound](https://github.com/aws/clock-bound) +- [Amazon Time Sync on EC2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configure-ec2-ntp.html) +- [Microsecond-accurate clocks on EC2](https://aws.amazon.com/blogs/compute/its-about-time-microsecond-accurate-clocks-on-amazon-ec2-instances/) diff --git a/spec/06-sharding-partitioning.md b/spec/06-sharding-partitioning.md index 8314a49..1dd7fba 100644 --- a/spec/06-sharding-partitioning.md +++ b/spec/06-sharding-partitioning.md @@ -1,336 +1,150 @@ -# Sharding and Partitioning - -**Question**: How do we scale horizontally while maintaining single-node performance? - -**Answer**: Range-based sharding with the degenerate case: local mode = 1 range, 1 replica. - -## The Core Idea - -Cloud9's entire key-value space is a single MVCC-versioned namespace partitioned into contiguous ranges: - -- **Range**: A contiguous interval of keys [start, end) with all MVCC versions -- **Raft Group**: Each range is replicated via Raft consensus (3+ replicas for distributed, 1 for local) -- **Leaseholder**: One replica per range holds a lease, serves all reads/writes for that range -- **Local Mode**: The entire keyspace = 1 range with 1 replica = zero network overhead - -**Result**: Same binary scales from laptop (1 range, no replication) to global deployment (thousands of ranges, 3+ replicas each). - -## Why This Is Natural - -Physical libraries partition books by call number (contiguous ranges). One shelf = one range. One librarian = one leaseholder. A personal library is still a library—it just happens to have one shelf with one caretaker. Cloud9 uses the same model: local deployment is the degenerate case where all keys fit in one range on one node. - -## Range Sharding Implementation - -### Range Definition - -```rust -struct Range { - range_id: RangeID, // Globally unique - start_key: Key, // Inclusive - end_key: Key, // Exclusive - raft_group: RaftGroupID, // Maps to Raft consensus group - replicas: Vec, // Where this range lives - leaseholder: ReplicaID, // Current lease holder -} -``` - -Ranges partition the keyspace exhaustively: -- Keys are lexicographically ordered -- No gaps: range[i].end_key == range[i+1].start_key -- No overlaps: ranges[i] and ranges[j] (i ≠ j) are disjoint - -### Raft Group per Range - -Each range = one Raft group: -- Raft log contains all writes to keys in [start_key, end_key) -- Replicas store the same MVCC key-value pairs -- Leaseholder serves reads (bypasses Raft quorum for performance) -- Writes go through Raft (majority quorum for durability) - -**Local mode optimization**: 1 replica = 1 Raft group with quorum size 1. Raft becomes a glorified write-ahead log with zero network calls. - -### Leaseholder Architecture - -Leaseholder = the replica with the exclusive right to serve reads/writes for a range: - -```rust -struct Lease { - range_id: RangeID, - replica_id: ReplicaID, - start_time: HLCTimestamp, - expiration: HLCTimestamp, - sequence: u64, // Fencing token -} -``` - -**Guarantees**: -- Only one leaseholder per range at any time (via fencing tokens) -- Reads bypass Raft consensus (leaseholder has the latest committed data) -- Writes use Raft but leaseholder coordinates -- Lease transfers when node fails or rebalancing occurs - -**Why leases?**: Read-heavy workloads (80%+ of traffic) don't pay Raft quorum cost. Writes still go through Raft for durability. This is the CockroachDB/Spanner model. - -## Auto-Split and Auto-Merge - -Ranges dynamically split and merge based on: - -### Split Triggers - -```rust -struct SplitPolicy { - max_size: usize, // 64MB default (CockroachDB uses 512MB) - max_qps: f64, // 1000 QPS default - max_latency_p99: Duration, // 10ms default -} -``` - -**Split algorithm**: -1. Monitor range metrics (size, QPS, latency) -2. When threshold exceeded, propose split at median key -3. Raft quorum approves split point -4. Create two new ranges: [start, split) and [split, end) -5. Update range directory (metadata service) - -**Why split?**: -- **Size**: Large ranges slow down Raft snapshots and rebalancing -- **QPS**: Hot ranges bottleneck on single leaseholder CPU -- **Latency**: Large ranges increase scan time, delaying transactions - -### Merge Triggers - -```rust -struct MergePolicy { - min_size: usize, // 16MB default - min_qps: f64, // 10 QPS default +# Sharding and Placement + +Cloud9 partitions physical dialects into ranges. Each range is one +Raft-replicated state machine with an explicit placement policy. + +Local mode starts with one range and one replica. Distributed mode adds ranges +and replicas without changing transaction semantics. + +## Range Descriptor + +```text +Range { + id + physical_dialect + start + end + raft_group + replicas + generation + placement_policy } ``` -**Merge algorithm**: -1. Detect adjacent ranges both below thresholds -2. Propose merge to both Raft groups -3. Quorum approves on both sides -4. Combine into single range [start_left, end_right) -5. Update range directory - -**Why merge?**: Too many small ranges waste memory (each Raft group has overhead) and increase metadata directory size. - -## Hotspot Handling +`[start, end)` is interpreted by the named physical dialect. A row-key span, +object-extent span, and columnar partition need not share one encoding. -### Workload-Aware Splits +For each dialect namespace, live range descriptors must be exhaustive and +non-overlapping. -Sequential writes (e.g., auto-incrementing IDs) concentrate on one range. Cloud9 detects and mitigates: +## Replication -```rust -struct HotspotDetector { - write_skew_threshold: f64, // 0.8 = 80% of writes to 20% of keys - auto_salt: bool, // Enable subkey salting -} -``` +Each distributed range has an independent Raft group. A committed command is +applied in the same order on every replica. -**Auto-subkey salting**: -- Detect sequential write pattern (e.g., `user:00001`, `user:00002`, ...) -- Inject hash prefix: `hash(key) % N || key` → `3:user:00001`, `7:user:00002`, ... -- Writes distribute across N ranges -- Reads reconstruct via scatter-gather (map phase) + merge (reduce phase) +Local mode uses a one-replica group. It may remove network hops, but it retains +the same log and state-machine invariants. -**Trade-off**: Point lookups become range scans. Only enable for known sequential append workloads (logs, time-series). +## Ownership and Fencing -### Load-Based Splits +One replica coordinates writes for a range. Ownership is fenced by the Raft +term and a monotonically increasing lease sequence. -High QPS on single range triggers split even if size < max_size: - -```rust -fn should_split_on_qps(range: &Range, stats: &RangeStats) -> bool { - stats.qps > range.split_policy.max_qps - && stats.write_skew > 0.5 // Writes not uniformly distributed +```text +Lease { + range_id + replica_id + raft_term + sequence } ``` -Split at the key separating high-write and low-write regions. +Timeout measurement uses a monotonic clock. A bounded UTC interval is used only +when a lease protocol needs a real-time proof. -## Placement Policies and Zone Configs +A stale owner cannot commit after a newer fence is durable. Reads require a +ReadIndex, a valid lease proof, or an applied safe-time proof. -Ranges have configurable replication and placement: +## Splits and Merges -```rust -struct ZoneConfig { - num_replicas: usize, // 3 for distributed, 1 for local - constraints: Vec, -} +Ranges split when size, load, or recovery cost exceeds policy. A split: -enum Constraint { - RequireRegion(String), // "us-west-2" - PreferZone(String), // "us-west-2a" (soft constraint) - ProhibitDatacenter(String), // "dc-deprecated" -} -``` +1. chooses a boundary valid for the physical dialect; +2. records child descriptors and generations transactionally; +3. transfers state through snapshots or shared immutable files; +4. activates routing only after both children can serve; +5. retires the parent after stale requests are fenced. -**Examples**: -- **Local mode**: `ZoneConfig { num_replicas: 1, constraints: [] }` -- **Multi-region**: `ZoneConfig { num_replicas: 5, constraints: [RequireRegion("us-east"), RequireRegion("us-west"), RequireRegion("eu-central")] }` -- **Compliance**: `ZoneConfig { num_replicas: 3, constraints: [ProhibitDatacenter("china"), RequireRegion("eu")] }` +Adjacent ranges may merge when their placement and physical formats are +compatible. The merge has the same atomic routing requirement. -Cloud9 uses CockroachDB's zone config DSL: +Cloud9 does not silently rewrite keys to spread a hotspot. Salting or +repartitioning changes access behavior and requires an explicit schema or +placement decision. -```sql -ALTER RANGE default CONFIGURE ZONE USING num_replicas = 3; -ALTER TABLE sensitive_data CONFIGURE ZONE USING constraints = '[+region=eu]'; -``` - -## Interleaved Tables (Co-location) - -Foreign key relationships benefit from co-location: - -```sql -CREATE TABLE orders ( - order_id UUID PRIMARY KEY, - customer_id UUID, - ... -); +## Range Directory -CREATE TABLE order_items ( - order_id UUID, - item_id UUID, - ... - PRIMARY KEY (order_id, item_id), - INTERLEAVE IN PARENT orders (order_id) -); -``` - -**Physical layout**: -``` -orders: [order:A:..., order:B:..., order:C:...] - └─ order_items: [order:A:item:1, order:A:item:2] - [order:B:item:1, order:B:item:3] - [order:C:item:5] -``` - -Child rows stored adjacent to parent in same range. Joins and FK checks stay local (no cross-range RPC). - -**Why this matters**: `DELETE FROM orders WHERE order_id = X` and cascading deletes to `order_items` happen in one range, one Raft transaction. Distributed FK checks are the #1 performance killer in CockroachDB—interleaving eliminates them. - -## Range Directory (Metadata Service) - -The range directory maps keys to ranges: - -```rust -struct RangeDirectory { - // Meta ranges store range metadata - meta1: Range, // Root range (never splits) - meta2: Vec, // Second-level index - user_ranges: Vec, // Actual data ranges -} -``` - -**Two-level hierarchy** (Bigtable/Spanner model): -1. **Meta1**: Single range storing Meta2 range locations (tiny, fits in memory) -2. **Meta2**: Ranges storing user range locations (sharded, but rarely accessed) -3. **User ranges**: Actual data - -**Lookup algorithm**: -```rust -fn lookup_range(key: &Key) -> Range { - // 1. Meta1 lookup (cached, O(1)) - let meta2_range = meta1_cache.lookup(key); - - // 2. Meta2 lookup (cached, O(1) amortized) - let user_range = meta2_cache.lookup(meta2_range, key); - - // 3. Return user range - user_range -} -``` - -**Cache invalidation**: Range splits/merges broadcast invalidation to all nodes. Stale cache causes misdirected RPC, which returns `RangeNotFound` + correct range hint. - -## Local Mode Implementation - -Local mode = 1 range, 1 replica, no replication: - -```rust -struct LocalModeConfig { - range: Range { - range_id: 1, - start_key: Key::MIN, - end_key: Key::MAX, - raft_group: 1, - replicas: vec![replica_local], - leaseholder: replica_local, - }, - zone_config: ZoneConfig { - num_replicas: 1, - constraints: vec![], - }, -} -``` +The range directory maps a physical-dialect key to its current range +descriptor. Directory records are versioned, replicated metadata. -**Optimizations enabled**: -- Raft quorum size = 1 (no network, just WAL append) -- Leaseholder never transfers (only one replica) -- Range never splits (unless user explicitly configures split policy) -- Metadata directory fits in memory (1 entry) +Clients and nodes may cache records. A stale route returns the current +generation and destination. The caller retries the same idempotent operation. -**Result**: Local mode has zero distributed systems overhead. It's Postgres-level performance with MVCC and versioned storage. +Directory availability must not depend on one unreplicated process. The root +metadata set remains small and strongly replicated. -## Why This Solves Single-Node Performance +## Placement Policy -The insight: **local = distributed with N=1**. +Placement constraints describe: -Traditional databases have separate "embedded" and "clustered" modes (Cassandra, MongoDB). Cloud9 has one code path: -- Local: 1 range, 1 Raft group, 1 replica -- Distributed: N ranges, N Raft groups, 3+ replicas each +- replica count; +- required and prohibited regions; +- failure-domain separation; +- preferred leader locality; +- data residency; +- storage and hardware class; +- physical-engine capability. -Raft with quorum=1 is just a write-ahead log. Range directory with 1 range is just a pointer. Leaseholder with 1 replica is just "this node." +Hard constraints fail when the cluster cannot satisfy them. Preferences may +affect cost without weakening a hard constraint. -**No special cases. No mode switching. The same binary scales from 1 node to 1000 nodes.** +The planner lowers locality requirements from Placement IR into range +descriptors. Operators can inspect the resulting decision. -## Comparison to Alternatives +## Transaction Routing -### Hash-Based Sharding (DynamoDB, Cassandra) +A single-range transaction commits through one Raft group. A cross-range +transaction uses the distributed protocol in +[08-transactions.md](08-transactions.md). -- Keys hashed to partitions (e.g., `hash(key) % N`) -- **Problem**: Range scans impossible (keys scattered across partitions) -- **Problem**: Can't interleave related data (parent/child separated by hash) -- Rejected: SQL requires range scans for `ORDER BY`, `BETWEEN`, secondary indexes +Range movement and splitting preserve transaction identity. A request routed +across a generation change either reaches the authoritative range or returns a +typed retry result. -### Directory-Based Sharding (MongoDB) +## Follower Reads -- Config server stores key → shard mapping -- **Problem**: Config server is SPOF (though replicated, it's still a bottleneck) -- **Problem**: Balancer moves entire chunks (10s of MB), causing thundering herd -- Rejected: Cloud9 rebalances at replica level (Raft snapshots), not chunk level +A follower serves a snapshot only when: -### Consistent Hashing (Riak, Dynamo) +1. it has applied through the required log index; +2. its safe timestamp covers the snapshot; +3. the requested schema and projections are available; +4. placement policy permits serving from that replica. -- Keys map to ring positions, replicas at ring offsets -- **Problem**: No range scans -- **Problem**: Range splits require full rehash -- Rejected: Same as hash-based sharding +Geographic proximity is not a consistency proof. -**Verdict**: Range sharding is the only approach that supports: -- Range scans (required for SQL) -- Co-location (required for FK performance) -- Fine-grained splits (required for hotspot handling) -- Local mode (1 range = entire keyspace) +## Hotspots -Every SQL-compatible distributed database (Spanner, CockroachDB, TiDB, YugabyteDB) uses range sharding. Cloud9 follows this proven path. +Cloud9 can respond to a hotspot by: -## Implementation Checklist +- splitting at a measured boundary; +- moving the leader; +- adding replicas for safe reads; +- selecting a different physical projection; +- applying admission control; +- requesting an explicit repartitioning change. -- [ ] Range struct with Raft group mapping -- [ ] Leaseholder acquisition/transfer protocol -- [ ] Split/merge triggers and policies -- [ ] Range directory (Meta1/Meta2 hierarchy) -- [ ] Cache invalidation protocol -- [ ] Interleaved table support (physical key encoding) -- [ ] Zone config DSL and replication constraints -- [ ] Hotspot detector (write skew, auto-salting) -- [ ] Local mode optimization (quorum=1, no rebalancing) -- [ ] Metrics: range size, QPS, latency, write skew +The system reports which action it selected and why. -## Key Insight +## Tests -**Local mode is not a special case—it's the degenerate case of the general distributed model.** When N=1, Raft becomes a WAL, leaseholder becomes "local replica," range directory becomes a singleton. This means Cloud9 can optimize aggressively for single-node (no network calls, no coordination) while using the exact same code path as distributed mode. +Placement tests cover: -This is the design that should have existed from the start: one binary, one model, scales from 1 to N nodes with zero architectural discontinuity. +- exhaustive, non-overlapping ranges; +- split and merge during reads and writes; +- stale generation fencing; +- leader and replica movement; +- directory loss and recovery; +- hard locality and residency constraints; +- follower safe-time enforcement; +- cross-range transactions during topology changes; +- local one-replica recovery. diff --git a/spec/07-sql-kv-unification.md b/spec/07-sql-kv-unification.md index fc5299d..43f99cf 100644 --- a/spec/07-sql-kv-unification.md +++ b/spec/07-sql-kv-unification.md @@ -1,830 +1,218 @@ -# SQL and KV Unification +# Multi-Model Intermediate Representation -**Question**: How do SQL and KV coexist without being separate systems bolted together? +Cloud9 is an MLIR for databases. Database APIs are source dialects that lower +through several typed intermediate representations (IRs). -**Answer**: They don't coexist—they're the same thing. SQL tables and KV namespaces are both key prefixes in a single MVCC key-value space. +The common layer is not one universal storage model. It is a conversion system +for preserving semantics while selecting transactions, placement, and physical +execution. -## The Core Insight +## Source Dialects -Every distributed database has an MVCC key-value layer at its core. Most databases then build SQL or KV APIs on top as separate, incompatible systems: +Cloud9 targets: -- Spanner: SQL-only, no KV access to the underlying layer -- DynamoDB: KV-only, no SQL -- CockroachDB: SQL-only, abandoned KV API experiments -- YugabyteDB: Both SQL and KV, but they're separate systems (YCQL vs YSQL) +- SQL dialects for relational queries and transactions; +- DynamoDB-style key-value and conditional operations; +- MongoDB-style document queries and updates; +- S3-style objects, metadata, versions, and byte ranges; +- ClickHouse-style analytical plans. -**Cloud9 makes them the same system.** A SQL table is a key prefix. A KV namespace is a key prefix. Both compile to the same transactional IR, execute in the same transaction coordinator, and write to the same MVCC storage. +Compatibility includes behavior, not only request syntax. Each frontend owns +its source types, error model, pagination, consistency options, and mutation +rules. -**Result**: BEGIN a transaction, write to SQL tables, read from KV namespaces, join SQL rows with KV data. One transaction, one commit timestamp, one consistency model. +## Why Several IRs -## Key Encoding: The Foundation +Early lowering to generic key-value operations loses useful information: -All data in Cloud9 lives in a single MVCC key space. Keys are byte strings with structure: +- SQL predicates and nullability; +- key-value conditions and atomic counters; +- document paths and update operators; +- object versions, ranges, and multipart state; +- analytical projections, grouping, and ordering. -``` -[prefix][primary_key][column_or_suffix][version] -``` - -The prefix determines whether something is SQL or KV. The rest is just bytes. - -### SQL Table Encoding - -**Schema**: -```sql -CREATE TABLE users ( - id INT PRIMARY KEY, - name TEXT, - email TEXT -); -``` - -**Key encoding**: -``` -Table prefix: /table/users/ -Row with id=42: - - /table/users/42/name → "Alice" - - /table/users/42/email → "alice@example.com" - -With timestamp: - - /table/users/42/name@t=100 → "Alice" - - /table/users/42/email@t=100 → "alice@example.com" -``` - -**Structure**: -- `/table/{table_name}/{pk}/{column}@{version}` → `value` -- Primary key is part of the key path -- Each column is a separate versioned key -- Row = all keys with same prefix `/table/{table_name}/{pk}/` - -**Benefits**: -- Point reads: single key lookup -- Range scans: iterate keys with common prefix -- Columnar access: read only needed columns -- MVCC: append-only versioned values - -### KV Namespace Encoding - -**API**: -```rust -kv.put("sessions", "sess_abc123", session_data); -``` - -**Key encoding**: -``` -Namespace prefix: /kv/sessions/ -Key "sess_abc123": - - /kv/sessions/sess_abc123@t=100 → session_data -``` - -**Structure**: -- `/kv/{namespace}/{key}@{version}` → `value` -- User-provided key is opaque bytes -- No column decomposition (value is blob) -- Namespace = all keys with prefix `/kv/{namespace}/` - -**Benefits**: -- Simple model: just put/get/delete -- No schema required -- Arbitrary byte keys and values -- Same MVCC versioning as SQL - -### Multi-Column Primary Keys - -**Schema**: -```sql -CREATE TABLE events ( - user_id INT, - timestamp BIGINT, - event_type TEXT, - PRIMARY KEY (user_id, timestamp) -); -``` - -**Key encoding**: -``` -/table/events/{user_id}/{timestamp}/event_type@t → value -``` - -**Example**: -``` -/table/events/42/1609459200/event_type@100 → "login" -/table/events/42/1609459201/event_type@100 → "click" -``` - -**Range scan**: -```sql -SELECT * FROM events WHERE user_id = 42; -``` -→ Scan `/table/events/42/` prefix - -### Secondary Indexes - -**Schema**: -```sql -CREATE INDEX users_email_idx ON users(email); -``` - -**Key encoding**: -``` -Index prefix: /index/users_email_idx/ -Mapping: email → primary key - - /index/users_email_idx/{email}@{version} → primary_key - -Example: - - /index/users_email_idx/alice@example.com@100 → 42 -``` - -**Query**: -```sql -SELECT * FROM users WHERE email = 'alice@example.com'; -``` - -**Plan**: -1. Index lookup: `/index/users_email_idx/alice@example.com@t_r` → `42` -2. Table lookup: `/table/users/42/*@t_r` → full row - -**Uniqueness constraint**: -- Check index key doesn't exist before inserting -- Enforced by transactional write to index key - -## Cross-API Transactions - -Because SQL and KV share the same transactional core, a single transaction can span both: +Cloud9 retains that information until a lower layer can represent it without +loss. This enables domain-specific optimization without duplicating +transactions and replication. -```rust -// Begin transaction (both SQL and KV) -let txn = db.begin().await?; +## IR Levels -// Write to SQL table -txn.execute("INSERT INTO users (id, name) VALUES (42, 'Alice')").await?; +### 1. Surface dialect IR -// Write to KV namespace -txn.kv_put("sessions", "sess_abc", session_data).await?; +Each frontend parses requests into a typed semantic IR: -// Commit both atomically -txn.commit().await?; +```text +SqlIR +KvIR +DocumentIR +ObjectIR +AnalyticalIR ``` -**What happens**: -``` -Writes in transaction: - - /table/users/42/name@t_w → "Alice" - - /kv/sessions/sess_abc@t_w → session_data - -Commit protocol: - 1. Coordinator assigns t_w from HLC - 2. Lock all keys (both SQL and KV) - 3. Check for conflicts - 4. Write all mutations with timestamp t_w - 5. Commit-wait until now() > t_w + ε - 6. Release locks, acknowledge client -``` - -**Guarantee**: Both writes commit or both abort. No partial commits. Both visible at the same timestamp `t_w`. +These dialects describe source behavior. They are independent of network +encoding and physical storage. -### Read-Write Cross-API Transaction +### 2. Transaction IR -```rust -let txn = db.begin().await?; +Transaction IR makes shared correctness explicit: -// Read from KV -let config = txn.kv_get("configs", "app_config").await?; - -// Use config to decide SQL write -if config.feature_enabled { - txn.execute("INSERT INTO features (name) VALUES ('new_feature')").await?; +```text +Transaction { + identity + snapshot + reads + predicates + mutations + consistency + authorization } - -txn.commit().await?; -``` - -**Serialization**: The read from KV establishes a read timestamp. The SQL write must not conflict with any concurrent transaction. Standard MVCC conflict detection applies across both APIs. - -## Cross-API Joins: The Killer Feature - -No other database allows this: join SQL tables with KV namespaces in a single query. - -### KV → SQL Join - -**Scenario**: KV namespace `user_sessions` stores session blobs. SQL table `users` stores structured user data. Join them. - -**API**: -```sql -SELECT - u.name, - u.email, - kv_decode(s.value, 'last_active') AS last_active -FROM - users u - INNER JOIN KV('user_sessions') s ON s.key = u.id::TEXT -WHERE - u.id IN (1, 2, 3); -``` - -**Key insight**: `KV(namespace)` is a virtual table with schema `(key BYTES, value BYTES)`. - -**Execution plan**: -1. Scan `/table/users/*` for id IN (1,2,3) → rows `{id, name, email}` -2. For each row, lookup `/kv/user_sessions/{id}@t_r` → session blob -3. Decode `last_active` field from blob using `kv_decode()` -4. Return joined result - -**Typed mapping**: `kv_decode(value, field)` extracts typed fields from KV blobs: -```sql -kv_decode(value, 'last_active') → BIGINT (Unix timestamp) -kv_decode(value, 'user_agent') → TEXT -kv_decode(value, 'ip_address', 'INET') → INET type -``` - -Cloud9 supports schema-on-read: KV values can be JSON, MessagePack, Protobuf, etc. The decode function interprets bytes at query time. - -### SQL → KV Join - -**Scenario**: SQL table `orders` references KV namespace `product_catalog` (frequently updated, no schema). - -**Query**: -```sql -SELECT - o.order_id, - o.quantity, - kv_decode(p.value, 'name') AS product_name, - kv_decode(p.value, 'price') AS product_price -FROM - orders o - INNER JOIN KV('product_catalog') p ON p.key = o.product_id -WHERE - o.user_id = 42; -``` - -**Execution**: -1. Scan `/table/orders/*` for `user_id = 42` -2. For each order, lookup `/kv/product_catalog/{product_id}@t_r` -3. Decode product name and price -4. Return results - -**Why this matters**: Product catalog can be updated via KV API (fast, no migrations), while orders use SQL (structured, constraints). Best of both worlds. - -### Multi-Way Joins - -**Query**: -```sql -SELECT - u.name, - o.order_id, - kv_decode(p.value, 'name') AS product_name, - kv_decode(s.value, 'status') AS session_status -FROM - users u - INNER JOIN orders o ON o.user_id = u.id - INNER JOIN KV('product_catalog') p ON p.key = o.product_id - LEFT JOIN KV('user_sessions') s ON s.key = u.id::TEXT -WHERE - u.id = 42; -``` - -**Plan**: Standard join optimization. KV namespaces are just another relation. Optimizer can reorder, choose hash joins, nested loops, etc. - -## Schema-on-Read for KV Namespaces - -KV values are opaque bytes. No schema enforced. But SQL queries need types. - -**Solution**: Schema-on-read with typed mappings. - -### Mapping Declarations - -**Define a mapping** (optional, improves query performance): -```sql -CREATE KV MAPPING product_catalog ( - key TEXT, - value JSON ( - name TEXT, - price DECIMAL, - inventory INT, - metadata JSON - ) -); ``` -**Now query with type safety**: -```sql -SELECT - key, - value->>'name' AS name, - CAST(value->>'price' AS DECIMAL) AS price -FROM - KV('product_catalog') -WHERE - CAST(value->>'inventory' AS INT) > 0; -``` - -**Benefits**: -- Planner knows types, can push filters -- No migration required (KV data unchanged) -- Multiple mappings can exist for same namespace (versioned schemas) - -### Versioned Mappings - -**Problem**: KV namespace schema evolves over time. Old and new formats coexist. - -**Solution**: Versioned mappings with discriminator. - -**Example**: -```sql --- Version 1 (old format) -CREATE KV MAPPING product_catalog_v1 ( - key TEXT, - value JSON ( - name TEXT, - price_cents INT - ) -) WHERE value->>'version' = '1'; - --- Version 2 (new format) -CREATE KV MAPPING product_catalog_v2 ( - key TEXT, - value JSON ( - name TEXT, - price DECIMAL, - currency TEXT - ) -) WHERE value->>'version' = '2'; - --- Union view -CREATE VIEW products AS - SELECT key, value->>'name' AS name, value->>'price_cents'::INT / 100.0 AS price - FROM KV('product_catalog') - WHERE value->>'version' = '1' - UNION ALL - SELECT key, value->>'name' AS name, value->>'price'::DECIMAL AS price - FROM KV('product_catalog') - WHERE value->>'version' = '2'; -``` - -**Query**: -```sql -SELECT * FROM products WHERE price > 10.00; -``` +Operations declare read sets, write sets, ranges, predicates, and effects. +Transactions also carry retry identity and required consistency. -**Execution**: Planner knows to scan both mappings, normalize price, filter. +This IR is the boundary for MVCC, conflict detection, atomic commit, and +bounded-time timestamp assignment. -**No data migration needed**: Old and new formats coexist. Query layer unifies them. +### 3. Placement IR -## Transactional IR (TxIR): The Compilation Target +Placement IR maps logical operations to ranges and replicas. It represents: -Both SQL and KV APIs compile to a common intermediate representation: **TxIR** (Transactional IR). +- partition keys and range boundaries; +- replica constraints; +- locality and residency policy; +- leaders and follower-read eligibility; +- data movement and repartitioning; +- cross-range transaction participants. -### TxIR Operations +Placement decisions cannot alter source semantics. -```rust -enum TxIROperation { - /// Read a single key at snapshot timestamp - Get { key: Bytes, snapshot: Timestamp }, +### 4. Physical dialect IR - /// Scan a key range at snapshot timestamp - Scan { start: Bytes, end: Bytes, snapshot: Timestamp }, +Physical dialects select data structures and operators: - /// Write a key-value pair (buffered until commit) - Put { key: Bytes, value: Bytes }, - - /// Delete a key (buffered until commit) - Delete { key: Bytes }, - - /// Check if key exists (for constraints) - Exists { key: Bytes, snapshot: Timestamp }, -} - -struct TxIRPlan { - operations: Vec, - read_set: HashSet, - write_set: HashMap, -} +```text +RowIR +PointIR +DocumentPhysicalIR +ObjectExtentIR +ColumnarIR ``` -### SQL Compilation +A SQL point lookup may lower to `PointIR`. An analytical scan may lower to +`ColumnarIR`. An object read may lower to extent and metadata operations. -**SQL**: -```sql -INSERT INTO users (id, name) VALUES (42, 'Alice'); -``` +Several physical projections may represent one logical dataset. The catalog +marks one representation authoritative and records freshness for derived +projections. -**TxIR**: -```rust -TxIRPlan { - operations: [ - // Check primary key doesn't exist - Exists { key: b"/table/users/42/name", snapshot: t_r }, - - // Write columns - Put { key: b"/table/users/42/name", value: b"Alice" }, - ], - read_set: { b"/table/users/42/name" }, - write_set: { b"/table/users/42/name" => b"Alice" }, -} -``` - -**SQL**: -```sql -SELECT name FROM users WHERE id = 42; -``` +### 5. Replication IR -**TxIR**: -```rust -TxIRPlan { - operations: [ - Get { key: b"/table/users/42/name", snapshot: t_r }, - ], - read_set: { b"/table/users/42/name" }, - write_set: {}, -} -``` +State-changing physical operations lower to deterministic commands before Raft +replication. A command includes all data needed for identical application on +every replica. -### KV Compilation +Replicated state machines do not call wall clocks, random generators, or +external services while applying a command. -**KV**: -```rust -txn.kv_put("sessions", "sess_abc", session_data); -``` +## Legal Lowering -**TxIR**: -```rust -TxIRPlan { - operations: [ - Put { key: b"/kv/sessions/sess_abc", value: session_data }, - ], - read_set: {}, - write_set: { b"/kv/sessions/sess_abc" => session_data }, -} -``` +Every conversion declares which source operations it can preserve. A lowering +fails when the target cannot represent a required semantic. -**KV**: -```rust -txn.kv_get("sessions", "sess_abc"); -``` +Examples include: -**TxIR**: -```rust -TxIRPlan { - operations: [ - Get { key: b"/kv/sessions/sess_abc", snapshot: t_r }, - ], - read_set: { b"/kv/sessions/sess_abc" }, - write_set: {}, -} -``` +- rejecting a document collation unsupported by the selected index; +- rejecting an object consistency mode unsupported by the target topology; +- preserving SQL null semantics through predicate lowering; +- retaining a key-value condition until conflict validation; +- retaining analytical ordering until a physical operator guarantees it. -### Cross-API Transaction Compilation +Cloud9 does not approximate unsupported behavior. -**Mixed transaction**: -```rust -let txn = db.begin().await?; -txn.execute("INSERT INTO users (id, name) VALUES (42, 'Alice')").await?; -txn.kv_put("sessions", "sess_abc", session_data).await?; -txn.commit().await?; -``` +## Cross-Dialect Data -**Combined TxIR**: -```rust -TxIRPlan { - operations: [ - // SQL INSERT - Exists { key: b"/table/users/42/name", snapshot: t_r }, - Put { key: b"/table/users/42/name", value: b"Alice" }, - - // KV PUT - Put { key: b"/kv/sessions/sess_abc", value: session_data }, - ], - read_set: { b"/table/users/42/name" }, - write_set: { - b"/table/users/42/name" => b"Alice", - b"/kv/sessions/sess_abc" => session_data, - }, -} -``` +Dialects may share data through an explicit catalog mapping. The mapping +defines identity, types, nullability, versioning, and ownership. -**Execution**: Transaction coordinator doesn't care whether operations came from SQL or KV. Just executes TxIR, locks keys, checks conflicts, commits. +One physical representation may serve several dialects when their semantics +align. Otherwise Cloud9 maintains a transactional projection or rejects the +mapping. -## Transaction Coordinator: API-Agnostic +Cross-dialect transactions use Transaction IR. Atomicity is available only +when every participating lowering supports the requested semantics. -The coordinator implements standard MVCC + 2PL over TxIR: +## Example Lowerings -```rust -struct TransactionCoordinator { - txn_id: TxnID, - read_timestamp: Timestamp, - write_buffer: HashMap, - read_set: HashSet, -} +A conditional key-value write lowers as: -impl TransactionCoordinator { - /// Execute a TxIR plan (from SQL or KV) - async fn execute(&mut self, plan: TxIRPlan) -> Result<()> { - for op in plan.operations { - match op { - TxIROperation::Get { key, snapshot } => { - let value = self.storage.get(&key, snapshot).await?; - self.read_set.insert(key); - // Return value to caller - } - TxIROperation::Put { key, value } => { - self.write_buffer.insert(key.clone(), value); - } - TxIROperation::Delete { key } => { - self.write_buffer.insert(key, TOMBSTONE); - } - // ... other operations - } - } - Ok(()) - } - - /// Commit: acquire locks, check conflicts, write - async fn commit(&mut self) -> Result { - let commit_ts = self.hlc.now(); - - // 1. Acquire locks for write set - self.lock_manager.acquire_locks(&self.write_buffer.keys()).await?; - - // 2. Validate read set (no writes since read_timestamp) - for key in &self.read_set { - let latest = self.storage.get_timestamp(key).await?; - if latest > self.read_timestamp { - return Err(Error::Conflict); - } - } - - // 3. Write all buffered mutations with commit_ts - for (key, value) in &self.write_buffer { - self.storage.put(key, value, commit_ts).await?; - } - - // 4. Commit-wait - self.commit_wait(commit_ts).await; - - // 5. Release locks - self.lock_manager.release_locks(&self.write_buffer.keys()).await?; - - Ok(commit_ts) - } -} +```text +KvIR conditional put + -> Transaction IR predicate plus mutation + -> PointIR read and versioned write + -> deterministic Raft command ``` -**Key point**: Coordinator has no notion of "SQL" vs "KV". Just byte strings and MVCC semantics. - -## Why No Other Database Has Done This - -### FoundationDB Came Close - -**What FDB got right**: -- SQL (experimental) and KV share one transactional core -- Key-prefix-based namespacing -- ACID transactions span both APIs - -**What FDB didn't do**: -- SQL layer was always experimental, never production-ready -- No cross-API joins (SQL couldn't query KV directly) -- No schema-on-read mappings for KV -- No Postgres wire compatibility - -**Cloud9 completes the vision**: Production SQL (Postgres-compatible) + production KV + cross-API joins + unified transaction model. - -### Why Others Failed - -**Spanner**: -- SQL-only from the start -- No KV API exposed to users -- Google's internal use cases didn't need it - -**CockroachDB**: -- Started SQL-only -- Tried adding KV via "system ranges" but abandoned it -- BSL license killed open experimentation - -**YugabyteDB**: -- Has both YSQL (Postgres fork) and YCQL (Cassandra-like KV) -- But they're **separate systems**: different APIs, different consistency, can't mix in one transaction -- No unification - -**DynamoDB, Cassandra, etc.**: -- KV-only, no SQL -- Adding SQL is bolting a query engine on top (Athena, Spark SQL) -- Not transactional unification - -**Fauna**: -- Tries to unify with GraphQL + FQL -- But no Postgres compatibility, no raw KV API -- Different consistency model (Calvin-style) - -### The Technical Barriers - -**Why this is hard**: - -1. **Key encoding conflicts**: SQL tables need structured keys (row/column). KV needs opaque keys. Most systems can't reconcile this. - -2. **Query optimization**: SQL query planner needs to understand KV as a relation. This requires extending the optimizer. - -3. **Type systems**: SQL is strongly typed. KV is untyped bytes. Bridging them requires schema-on-read with runtime type coercion. - -4. **Transaction semantics**: SQL transactions use read/write locks. KV transactions often use optimistic concurrency. Unifying requires choosing one (Cloud9: MVCC + 2PL). - -5. **Wire protocol**: Postgres wire protocol doesn't understand KV. Extending it without breaking clients is hard. - -**Cloud9's approach**: -- Key encoding with clear prefixes (`/table/` vs `/kv/`) -- TxIR as compilation target (both APIs produce same IR) -- Schema-on-read with explicit mappings -- MVCC + 2PL as universal transaction model -- Extended Postgres protocol with `KV()` virtual table function - -## Concrete Example: End-to-End Transaction - -**Scenario**: E-commerce checkout. SQL for orders, KV for session and inventory cache. - -```rust -let txn = db.begin().await?; +An object upload lowers as: -// 1. Check KV session is valid -let session = txn.kv_get("sessions", user_session_id).await?; -if session.is_expired() { - return Err(Error::SessionExpired); -} - -// 2. Read product from KV cache -let product = txn.kv_get("product_cache", product_id).await?; -let price = product.decode_field("price")?; - -// 3. Insert SQL order -txn.execute( - "INSERT INTO orders (user_id, product_id, price, quantity) VALUES ($1, $2, $3, $4)", - &[&user_id, &product_id, &price, &quantity] -).await?; - -// 4. Update KV inventory -let inventory = txn.kv_get("inventory", product_id).await?; -let new_inventory = inventory - quantity; -txn.kv_put("inventory", product_id, new_inventory.encode()).await?; - -// 5. Commit atomically -txn.commit().await?; +```text +ObjectIR put + -> Transaction IR metadata mutation + -> ObjectExtentIR data placement + -> replicated version and extent metadata ``` -**TxIR generated**: -```rust -TxIRPlan { - operations: [ - // Step 1: Session check - Get { key: b"/kv/sessions/sess_abc", snapshot: t_r }, - - // Step 2: Product lookup - Get { key: b"/kv/product_cache/prod_123", snapshot: t_r }, - - // Step 3: Order insert - Exists { key: b"/table/orders/{order_id}/user_id", snapshot: t_r }, - Put { key: b"/table/orders/{order_id}/user_id", value: encode(user_id) }, - Put { key: b"/table/orders/{order_id}/product_id", value: encode(product_id) }, - Put { key: b"/table/orders/{order_id}/price", value: encode(price) }, - Put { key: b"/table/orders/{order_id}/quantity", value: encode(quantity) }, - - // Step 4: Inventory update - Get { key: b"/kv/inventory/prod_123", snapshot: t_r }, - Put { key: b"/kv/inventory/prod_123", value: encode(new_inventory) }, - ], - read_set: { - b"/kv/sessions/sess_abc", - b"/kv/product_cache/prod_123", - b"/table/orders/{order_id}/user_id", - b"/kv/inventory/prod_123", - }, - write_set: { - b"/table/orders/{order_id}/user_id" => encode(user_id), - b"/table/orders/{order_id}/product_id" => encode(product_id), - b"/table/orders/{order_id}/price" => encode(price), - b"/table/orders/{order_id}/quantity" => encode(quantity), - b"/kv/inventory/prod_123" => encode(new_inventory), - }, -} -``` - -**Commit protocol**: -1. Coordinator assigns `t_w = 1000` from HLC -2. Acquire locks on all write keys (both SQL and KV) -3. Validate read set: no key has version `> t_r` (no concurrent writes) -4. Write all mutations with version `t_w = 1000` -5. Replicate to quorum via Raft -6. Commit-wait until `now() > 1000 + ε` -7. Release locks, acknowledge client - -**Guarantee**: All writes (SQL order + KV inventory) commit at `t_w = 1000` or all abort. No partial commit. Externally consistent. - -## Performance Characteristics - -### SQL Workloads - -**Point reads**: Single key lookup (`/table/{name}/{pk}/{col}`) -- Same as traditional KV: O(1) with index - -**Range scans**: Prefix iteration (`/table/{name}/{pk_start}/` to `/table/{name}/{pk_end}/`) -- Same as traditional SQL: O(log N + K) where K = rows returned - -**Joins**: Standard join algorithms (nested loop, hash join, merge join) -- No overhead vs traditional SQL - -### KV Workloads - -**Point reads/writes**: Single key lookup/insert -- Same as dedicated KV stores - -**Range scans**: Prefix iteration within namespace -- Same as dedicated KV stores - -### Cross-API Joins - -**Overhead**: Minimal if KV mapping is defined -- Planner knows types, can push filters -- Same execution as SQL-SQL joins - -**Without mapping**: Schema-on-read at runtime -- Parse JSON/MessagePack/Protobuf per row -- Slower, but still correct +An analytical query lowers as: -**Optimization**: Create mapping for hot namespaces - -## Limitations and Trade-offs - -### KV Values Are Opaque - -**Implication**: Can't index into KV value fields without a mapping. - -**Example**: Can't do `WHERE kv_decode(value, 'price') > 10` efficiently without a mapping that tells the planner how to extract `price`. - -**Solution**: Create mapping for hot query patterns. - -### No Column-Level Security on KV - -SQL has column-level permissions (`GRANT SELECT (name) ON users TO role`). KV namespaces are key-value; no column concept. - -**Workaround**: Use separate namespaces for sensitive data, control access at namespace level. - -### Schema Evolution Requires Coordination - -**SQL**: ALTER TABLE is a schema change, locks table. - -**KV**: No schema, but mappings are versioned. Adding a new mapping doesn't lock data, but queries must handle multiple versions. - -**Trade-off**: KV is more flexible (no locks), but requires application logic to handle versions. - -## Future Enhancements - -### KV → SQL Promotion - -**Idea**: Start with KV namespace, promote to SQL table when schema stabilizes. - -```sql --- Promote KV namespace to table (inferred schema from mapping) -PROMOTE KV NAMESPACE product_catalog TO TABLE products; +```text +AnalyticalIR scan, filter, aggregate + -> snapshot and placement constraints + -> ColumnarIR operators + -> vectorized execution ``` -**Effect**: Copies data, creates columns, drops KV namespace. Useful for prototyping. - -### Automatic Mapping Inference +These paths share catalogs, snapshots, and durability. They do not share one +forced hot path. -**Idea**: Analyze KV values, infer JSON schema, auto-create mapping. +## Optimization Passes -```sql -ANALYZE KV NAMESPACE product_catalog; --- Cloud9 samples values, infers { name: TEXT, price: DECIMAL, ... } --- Creates mapping automatically -``` +Passes may: -### Foreign Keys Across APIs +- push predicates into compatible physical dialects; +- prune columns and object byte ranges; +- select indexes and projections; +- co-locate transaction participants; +- route safe reads to followers; +- fuse compatible physical operators; +- choose row, point, document, extent, or columnar execution. -**Idea**: SQL foreign key can reference KV namespace. - -```sql -ALTER TABLE orders - ADD CONSTRAINT fk_product - FOREIGN KEY (product_id) - REFERENCES KV('product_catalog')(key); -``` +Each pass must preserve types, effects, consistency, and authorization. The +validator rejects an IR that violates a declared invariant. -**Challenge**: KV values can be deleted without SQL knowing. Requires trigger-like mechanism. +## Versioning and Observability -## Summary +Serialized IR includes a version. Rolling upgrades accept only declared +version pairs. -**SQL and KV are unified in Cloud9 because**: +Traces record the source operation, selected lowerings, placement decision, and +physical plan. Sensitive values may be redacted, but the decision path remains +inspectable. -1. **Single key space**: Both are prefixes in the same MVCC storage -2. **Shared TxIR**: Both APIs compile to same transactional IR -3. **Cross-API transactions**: BEGIN spans both, commit atomically -4. **Cross-API joins**: KV namespaces are queryable as virtual tables -5. **Schema-on-read**: KV values get typed at query time via mappings -6. **One consistency model**: External consistency for both APIs +This is the practical value of an MLIR design: operators can see where meaning +changed and where work was introduced. -**Why this matters**: +## Test Contract -- **Developers get flexibility**: Prototype with KV, harden with SQL -- **Operations get simplicity**: One database, one backup, one transaction log -- **Applications get correctness**: No data synchronization bugs between systems +Each dialect needs: -**The killer feature**: Start a transaction, write to SQL, read from KV, commit atomically. No other database lets you do this. +- parser and type tests; +- source compatibility tests; +- legal and illegal lowering tests; +- differential tests against a reference system; +- optimization equivalence tests; +- deterministic replication tests; +- cross-dialect transaction tests; +- physical-engine benchmarks. -**Cloud9 completes what FoundationDB started**: Production-ready SQL + KV unification with Postgres compatibility and external consistency. +A performance result is valid only when the optimized and reference plans have +the same observable semantics. diff --git a/spec/08-transactions.md b/spec/08-transactions.md index 75cd272..7f45b19 100644 --- a/spec/08-transactions.md +++ b/spec/08-transactions.md @@ -1,984 +1,217 @@ # Transaction Protocol -**Question**: How do we execute multi-shard transactions that maintain external consistency? +Cloud9 uses MVCC, durable transaction records, and two-phase commit for +serializable transactions across ranges. A healthy bounded-time provider adds +external consistency. -**Answer**: Two-phase commit (2PC) with MVCC intents and coordinator-driven commit timestamp assignment. +## Guarantees -## Overview +A committed transaction provides: -Cloud9 transactions must satisfy: -1. **Atomicity**: All writes succeed or all fail (no partial writes visible) -2. **External consistency**: If T₁ finishes before T₂ starts in real time, T₁'s writes are visible to T₂ -3. **Snapshot isolation**: Read-only transactions see a consistent point-in-time snapshot -4. **Lock-free reads**: Read-only transactions never block or wait for locks +- atomicity across all participants; +- serializable conflict ordering; +- one snapshot across compatible dialects; +- idempotent retry by transaction identity; +- external consistency in TrueTime mode. -This document specifies the protocol that achieves these guarantees across sharded data. +Local mode can provide serializability without bounded-time hardware. It does +not claim cross-machine external consistency. -## Transaction Types +## Transaction Record -### Read-Only Transactions +The coordinator stores one durable record: -**Characteristics**: -- No writes, no intents, no locks -- Pick snapshot timestamp `t_r` at start -- Never block, never wait for locks -- No 2PC coordination needed - -**Protocol**: -``` -1. Client starts transaction -2. Coordinator picks t_r = now() (from HLC or TSO) -3. All reads execute at t_r (MVCC snapshot) -4. Transaction completes immediately (no commit phase) -``` - -**Timestamp selection**: -- **HLC mode**: `t_r = coordinator.hlc.now().physical` -- **TSO mode**: `t_r = tso.get_read_timestamp()` - -**Guarantee**: Because of commit-wait on writes, any `t_r` picked after a write's acknowledgment will be `> t_w`. External consistency follows. - -**No commit-wait on reads**: Read-only transactions don't write, so no commit-wait latency. - -### Single-Range Write Transactions - -**Characteristics**: -- All writes fall within a single Raft range (shard) -- Simpler than cross-shard (no 2PC) -- Still use MVCC intents for atomicity - -**Protocol**: -``` -1. Client starts transaction, sends writes to coordinator -2. Coordinator picks provisional timestamp t_p -3. Write intents at t_p (not committed values yet) -4. Replicate intents via Raft to quorum -5. Convert intents to committed values at t_c = max(t_p, participants) -6. Commit-wait until now() > t_c + ε (HLC mode only) -7. Acknowledge to client -``` - -**Intent structure**: -```rust -struct Intent { - key: Key, - value: Value, - txn_id: TxnId, - timestamp: Timestamp, // Provisional, may change at commit +```text +TransactionRecord { + id + state + snapshot + participants + commit_timestamp } ``` -**Why intents**: During steps 3-5, other transactions might read this key. Intent signals "write in progress, not yet committed." - -### Cross-Shard Write Transactions (2PC) - -**Characteristics**: -- Writes span multiple Raft ranges -- Requires two-phase commit for atomicity -- Coordinator drives the protocol - -**Roles**: -- **Coordinator**: Picks commit timestamp, drives 2PC phases -- **Participants**: Raft ranges that hold written keys - -**Protocol** (detailed in next section). +Valid state transitions are: -## Two-Phase Commit Protocol - -### Phase 0: Intent Writing - -``` -For each participant range: -1. Coordinator sends WriteIntent RPC with: - - txn_id: unique transaction identifier - - writes: [(key, value), ...] - - provisional_timestamp: t_p -2. Participant writes intents (not committed values) -3. Participant replicates via Raft to quorum -4. Participant responds: (status, read_timestamp) - - status: OK | ABORT (conflict detected) - - read_timestamp: max timestamp read during execution -``` - -**Intent format on disk**: -``` -key -> Intent { - txn_id: UUID, - value: bytes, - provisional_ts: Timestamp, -} -``` - -**Conflict detection**: If writing intent encounters existing intent or committed value with `t > provisional_ts`, abort immediately (write-write conflict). - -### Phase 1: Prepare - -``` -For each participant: -1. Coordinator sends Prepare RPC with: - - txn_id - - commit_timestamp: t_c = max(participants.read_ts, coordinator.now()) -2. Participant verifies: - - All intents are still present (not rolled back) - - No conflicts at t_c (no committed writes with t ∈ (provisional_ts, t_c]) - - Raft range is still leader -3. Participant writes PreparedRecord to Raft log -4. Participant responds: PREPARED | ABORT -``` - -**PreparedRecord**: -```rust -struct PreparedRecord { - txn_id: TxnId, - commit_timestamp: Timestamp, - intent_keys: Vec, -} -``` +- `Pending -> Preparing` +- `Pending -> Aborted` +- `Preparing -> Committed` +- `Preparing -> Aborted` -**Why write PreparedRecord**: If coordinator crashes between prepare and commit, recovery process needs to know this range voted "yes" and must complete the commit. +`Committed` and `Aborted` are terminal. A durable commit decision cannot +become an abort. A retry reads the record and continues the recorded outcome. -**Abort conditions**: -- Intent missing (already rolled back by timeout) -- Write-write conflict detected at t_c -- Raft leadership lost (can't guarantee replication) +## Read-Only Transactions -### Phase 2: Commit +A read-only transaction chooses one snapshot. Every participant must prove it +has applied all commits through that snapshot. -``` -If all participants vote PREPARED: -1. Coordinator writes CommitRecord to its own Raft log with: - - txn_id - - commit_timestamp: t_c - - participants: [range_ids] - - status: COMMITTED -2. Coordinator sends Commit RPC to all participants -3. Each participant: - - Converts intents to committed values at t_c - - Removes txn_id metadata - - Writes CommitRecord to Raft log - - Responds: COMMITTED -4. Coordinator commit-waits until now() > t_c + ε (HLC mode) -5. Coordinator acknowledges to client -``` +The transaction creates no write intents. Reads may execute on followers when +their applied index, safe timestamp, schema, and projection state cover the +snapshot. -**Committed value format**: -``` -key@t_c -> Value { - data: bytes, - // No txn_id, this is a committed MVCC version -} -``` +A current externally consistent read uses bounded time and safe-time +information. An explicit stale read uses the caller's timestamp and declared +staleness policy. -**If any participant votes ABORT**: -``` -1. Coordinator writes CommitRecord with status: ABORTED -2. Coordinator sends Abort RPC to all participants -3. Each participant: - - Removes intents for txn_id - - Writes AbortRecord to Raft log -4. Coordinator returns error to client (no commit-wait) -``` +## Single-Range Writes -### Commit Timestamp Selection +A single-range transaction uses one replicated command: -**Formula**: -```rust -fn select_commit_timestamp( - coordinator: &Node, - participants: &[ParticipantResponse], -) -> Timestamp { - let max_participant_ts = participants - .iter() - .map(|p| p.read_timestamp) - .max() - .unwrap_or(0); +1. Read and evaluate at a stable snapshot. +2. Validate read versions, ranges, predicates, and conditions. +3. Obtain a valid bounded-time interval when the mode requires it. +4. Select a commit timestamp. +5. Replicate the transaction result through the range's Raft group. +6. Apply all versions atomically. +7. Perform commit-wait in TrueTime mode. +8. Return the durable result. - let coordinator_now = coordinator.hlc.now(); +The command includes every value needed for deterministic application. - Timestamp::max(max_participant_ts, coordinator_now) -} -``` +## Cross-Range Writes -**Why max(participants, coordinator)**: -- **Participants' read_timestamp**: Highest timestamp read during intent phase. Must assign `t_c ≥` this to avoid read-after-write violations. -- **Coordinator's now()**: Ensures `t_c` respects real-time order at coordinator. +Cross-range writes use two-phase commit (2PC). Each participant is a +Raft-replicated range. -**Example**: -``` -1. Participant A reads key@100 during intent phase → read_ts = 100 -2. Participant B reads nothing → read_ts = 0 -3. Coordinator clock = 95 (slightly behind due to skew) -4. Commit timestamp = max(100, 0, 95) = 100 -``` +### Prepare -Must use 100, not 95, because transaction observed data at t=100. +1. Allocate a stable transaction identity. +2. Write provisional intents at each participant. +3. Validate point, range, and predicate reads. +4. Replicate a prepared record in each participant. +5. Return each participant's timestamp constraints. -### Commit-Wait Protocol +Any validation failure aborts the transaction. Prepared participants retain +enough state to recover without the original client. -**HLC mode** (commit-wait required): -```rust -fn commit_wait(commit_timestamp: Timestamp, clock: &HLC, epsilon: Duration) { - let target = commit_timestamp + epsilon; - while clock.now() < target { - sleep(1ms); - } -} -``` +### Decision -**Purpose**: Ensure all replicas' clocks advance past `t_c` before acknowledging. Any future operation gets timestamp `> t_c`, guaranteeing external consistency. +The coordinator: -**Typical duration**: ~10-50ms (PTP/PHC on AWS), ~50-100ms (NTP), ~1-10ms (GPS/atomic). +1. reads a valid bounded-time interval when required; +2. chooses one timestamp that satisfies all participant constraints; +3. stores the participant set and decision durably; +4. sends the same decision and timestamp to every participant. -**TSO mode** (no clock-based commit-wait): -```rust -fn safe_timestamp_fence(commit_timestamp: Timestamp, tso: &TSO) { - // Ensure TSO won't hand out timestamps ≤ commit_timestamp - tso.advance_minimum(commit_timestamp + 1); -} -``` +The durable transaction record is the authority after prepare. A timeout is +not evidence of abort. -**Purpose**: Guarantee future timestamps from TSO are `> t_c`. No time-based waiting, but still coordination overhead. +### Finalize -## MVCC Intent Handling +Each participant replicates the decision. Commit converts intents into versions +at the shared timestamp. Abort removes the provisional intents. -### Write Path: Creating Intents +Cleanup may continue after the decision is durable. Visibility follows the +decision record, not cleanup completion. -``` -Storage layout during transaction: -key -> Intent { - txn_id: UUID, - value: bytes, - provisional_ts: 100, -} +The coordinator performs commit-wait before returning a committed result in +TrueTime mode. -key@50 -> CommittedValue { data: "old" } -``` +## Timestamp Selection -**Intent semantics**: "Transaction `txn_id` intends to write this value at ~t=100, but not yet committed." +In TrueTime mode, the commit timestamp's physical component is at or after: -### Read Path: Encountering Intents +- the bounded-time provider's `latest` value; +- every participant's observed version; +- every causally required predecessor. -When a read at timestamp `t_r` encounters an intent: +Cloud9 acknowledges only after a fresh interval has: -**Case 1: Intent belongs to active transaction with `t_intent ≤ t_r`** -``` -1. Check intent status (query coordinator or transaction record) -2. If COMMITTED: read the intent's value (it's now committed at t_c ≤ t_r) -3. If ABORTED: ignore intent, read older version -4. If ACTIVE: wait or push (see below) +```text +earliest > commit_timestamp.physical ``` -**Case 2: Intent belongs to inactive/expired transaction** -``` -1. If transaction record shows ABORTED: cleanup intent, read older version -2. If transaction expired (timeout): attempt to roll back intent -3. If transaction COMMITTED: resolve intent to committed value -``` +The detailed proof is in +[03-external-consistency.md](03-external-consistency.md). -**Case 3: Intent timestamp > `t_r`** -``` -Ignore intent (it's in the reader's future), read older committed version. -``` +## Serializable Validation -### Intent Resolution: Wait vs Push - -When reading encounters an active intent blocking the read: - -**Wait strategy** (default): -```rust -fn read_with_wait(key: Key, read_ts: Timestamp) -> Result { - loop { - match storage.get(key, read_ts) { - Ok(value) => return Ok(value), - Err(IntentConflict { txn_id, intent_ts }) => { - if intent_ts > read_ts { - // Intent is in our future, should not block us - return storage.get_older_version(key, read_ts); - } - - // Wait for intent to resolve - wait_for_transaction(txn_id, timeout)?; - } - } - } -} -``` +Cloud9 records the effects required to validate the transaction: -**Push strategy** (for high-priority transactions): -```rust -fn read_with_push(key: Key, read_ts: Timestamp, reader_priority: Priority) -> Result { - match storage.get(key, read_ts) { - Err(IntentConflict { txn_id, intent_ts, writer_priority }) => { - if reader_priority > writer_priority { - // Push writer's timestamp forward, forcing it to commit at higher ts - coordinator.push_transaction(txn_id, read_ts + 1)?; - // Re-read after push - storage.get(key, read_ts) - } else { - wait_for_transaction(txn_id, timeout) - } - } - Ok(value) => Ok(value), - } -} -``` +- point reads and observed versions; +- range reads and range generations; +- predicates and selected indexes; +- point and range writes; +- source-dialect conditions. -**Push semantics**: Force writer to commit at `t_c > read_ts`, making its writes invisible to this reader. Prevents deadlocks and priority inversion. +Prepare rejects any intervening commit that changes the transaction's result. +Range and predicate validation must detect phantoms. -**When to push**: -- High-priority reader vs low-priority writer -- Read-only transaction blocked by long-running write -- Deadlock detection (cycle-breaking) +An optimization may reduce validation work only when it preserves this +contract. -## Write-Write Conflict Detection +## Intent Conflicts -Two transactions writing the same key must serialize. Cloud9 uses **first-writer-wins** with intent-based detection. +An intent identifies its transaction. Readers never expose it as committed +data. -### Conflict Scenarios +On conflict, a transaction may wait, push, or abort according to one +deterministic priority policy. The policy must prevent deadlock and preserve +the durable decision. -**Scenario 1: Intent-Intent conflict** -``` -T1: write_intent(key, t=100) → OK -T2: write_intent(key, t=105) → encounters T1's intent → ABORT (T1 got there first) -``` +A participant cannot abort a transaction after discovering a committed +decision. -**Scenario 2: Intent-Committed conflict** -``` -key@90 = "old" -T1: write_intent(key, provisional_ts=100) -T2: commits key@110 = "newer" (different transaction) -T1: prepare at t_c=120 → detect conflict (committed value@110 > provisional@100) → ABORT -``` - -**Scenario 3: Committed-Intent conflict** -``` -key@100 = "committed" -T1: write_intent(key, provisional_ts=95) → must check for committed values@(95, now()] → conflict → ABORT -``` - -### Detection Algorithm - -```rust -fn write_intent(key: Key, txn_id: TxnId, provisional_ts: Timestamp) -> Result<()> { - // Check for existing intent - if let Some(existing_intent) = storage.get_intent(key) { - if existing_intent.txn_id != txn_id { - return Err(WriteConflict::Intent(existing_intent.txn_id)); - } - } - - // Check for committed values after provisional_ts - if let Some(newer_value) = storage.get_next_version(key, provisional_ts) { - return Err(WriteConflict::Committed(newer_value.timestamp)); - } - - // Write intent - storage.put_intent(key, Intent { txn_id, provisional_ts, ... }); - Ok(()) -} - -fn prepare_transaction(txn_id: TxnId, commit_ts: Timestamp) -> Result<()> { - for key in transaction.intent_keys { - // Re-check conflicts at commit_ts (may differ from provisional_ts) - if let Some(newer_value) = storage.get_versions(key, commit_ts) { - if newer_value.timestamp > transaction.provisional_ts { - return Err(WriteConflict::Committed(newer_value.timestamp)); - } - } - } - Ok(()) -} -``` - -**Key insight**: Check conflicts twice: -1. At intent-write time (provisional timestamp) -2. At prepare time (final commit timestamp) - -Between these two checks, another transaction might commit a conflicting write. - -## Transaction Recovery - -### Coordinator Failure - -**Problem**: Coordinator crashes between prepare and commit. Participants are in prepared state, can't proceed without coordinator decision. - -**Solution**: Transaction record recovery. - -```rust -struct TransactionRecord { - txn_id: TxnId, - coordinator: NodeId, - participants: Vec, - commit_timestamp: Timestamp, - status: TxnStatus, // ACTIVE | PREPARED | COMMITTED | ABORTED - heartbeat: Timestamp, -} - -enum TxnStatus { - Active, - Prepared, - Committed, - Aborted, -} -``` - -**Recovery protocol**: -``` -1. New coordinator detects TransactionRecord with status=PREPARED and expired heartbeat -2. Query all participants for their vote: - - If any voted ABORT: abort transaction - - If all voted PREPARED: commit transaction at recorded t_c -3. Complete phase 2 (send Commit/Abort to participants) -4. Update TransactionRecord to COMMITTED/ABORTED -``` - -**Timeout-based cleanup**: -``` -If TransactionRecord heartbeat expires and status=ACTIVE: -1. Coordinator presumed dead -2. Abort transaction (haven't entered prepared state yet) -3. Send Abort to all participants with intents -4. Clean up intents -``` - -### Participant Failure - -**Problem**: Participant crashes during transaction. - -**Solution**: Raft replication handles participant failure. - -``` -1. Intents are replicated via Raft to quorum -2. PreparedRecord is replicated via Raft to quorum -3. If leader crashes, new leader takes over with same state -4. Transaction continues normally on new leader -``` - -**Key property**: Because intents/PreparedRecord are in Raft log, failover doesn't lose transaction state. - -## Read-Only Transactions: Lock-Free Execution - -### Snapshot Selection - -```rust -fn start_read_only_transaction() -> ReadOnlyTxn { - let snapshot_ts = coordinator.hlc.now(); // Or tso.get_read_timestamp() - ReadOnlyTxn { snapshot_ts } -} -``` - -### Execution - -```rust -fn read(txn: &ReadOnlyTxn, key: Key) -> Result { - // Find most recent committed version ≤ snapshot_ts - storage.get_version(key, txn.snapshot_ts) -} -``` - -**No locks taken**: MVCC allows reading old versions while writes proceed on newer versions. - -**Intent handling**: If read encounters intent: -- **Intent timestamp > snapshot_ts**: Ignore intent (it's in the future), read older version -- **Intent timestamp ≤ snapshot_ts**: Intent must have committed by now (due to commit-wait), resolve to committed value - -### Staleness Considerations - -**Potential issue**: Replica might not have applied all commits ≤ `snapshot_ts` yet (replication lag). - -**Solution**: Safe timestamp tracking (see Closed Timestamps section). - -## Closed Timestamps: Follower Reads - -### The Problem - -``` -1. Leader commits write at t_c = 100, performs commit-wait -2. Leader acknowledges to client -3. Client immediately sends read at t_r = 101 to follower -4. Follower hasn't applied commit@100 yet (replication lag) -5. Follower reads stale data -``` - -**Violation**: Write finished before read in real time, but read didn't see write. - -### The Solution: Closed Timestamps - -**Definition**: A **closed timestamp** `t_closed` is a timestamp below which the leader guarantees no new writes will be assigned. - -**Leader protocol**: -```rust -impl RaftLeader { - fn advance_closed_timestamp(&mut self) { - // Pick safe timestamp: below current HLC, all in-flight txns above this - let safe_ts = self.hlc.now() - self.max_clock_skew; - - // Ensure no active transactions have provisional_ts ≤ safe_ts - let min_active_txn_ts = self.active_txns.iter() - .map(|t| t.provisional_ts) - .min() - .unwrap_or(Timestamp::MAX); - - self.closed_timestamp = Timestamp::min(safe_ts, min_active_txn_ts - 1); - - // Replicate via Raft (piggybacked on heartbeats) - self.broadcast_closed_timestamp(self.closed_timestamp); - } -} -``` +## Recovery -**Follower protocol**: -```rust -impl RaftFollower { - fn can_serve_read(&self, read_ts: Timestamp) -> bool { - // Serve read if: - // 1. read_ts ≤ closed_timestamp (no future writes below read_ts) - // 2. We've applied all Raft log entries up to closed_timestamp - read_ts <= self.closed_timestamp && self.applied_index >= self.closed_index - } - - fn read_at_closed_timestamp(&self, key: Key, read_ts: Timestamp) -> Result { - if !self.can_serve_read(read_ts) { - return Err(ReadError::NotSafe); // Redirect to leader or wait - } - self.storage.get_version(key, read_ts) - } -} -``` - -### Closed Timestamp Propagation - -**Mechanism**: Piggyback closed timestamp on Raft heartbeats. - -```rust -struct RaftHeartbeat { - leader_id: NodeId, - term: u64, - commit_index: u64, - closed_timestamp: Timestamp, // <-- New field -} -``` - -**Frequency**: Every heartbeat interval (~100ms typical). - -**Staleness bound**: Follower reads are bounded-stale by heartbeat interval. - -``` -If heartbeat every 100ms: -- Follower reads see data ≤ 100ms old -- Still lock-free, still no coordination -- Acceptable for most workloads -``` - -### Follower Read Guarantees - -**What followers guarantee**: -- **Snapshot isolation**: Read sees consistent snapshot at `t_r` -- **Bounded staleness**: Read is at most `heartbeat_interval` old -- **Lock-free**: No blocking, no waiting - -**What followers don't guarantee**: -- **External consistency without waiting**: If write commits at t=100 and follower hasn't received heartbeat yet, follower might serve read at t=99 (stale) - -**Solution for external consistency**: Client reads from leader (or waits for follower to catch up). - -**Use case**: Follower reads with bounded staleness are perfect for: -- Analytics queries (don't need latest data) -- Geographically distributed reads (low-latency local reads) -- Load balancing read traffic across replicas - -## External Consistency: End-to-End - -### Single-Shard Transaction - -``` -1. Client: Begin transaction -2. Coordinator: Pick t_p = hlc.now() -3. Coordinator: Write intents at t_p -4. Coordinator: Replicate via Raft -5. Coordinator: Commit at t_c = max(t_p, observed_reads) -6. Coordinator: Commit-wait until now() > t_c + ε -7. Coordinator: Return "success" to client - [At this point, all nodes' clocks > t_c] -8. Client: Begin new read transaction (anywhere) -9. Any node: Pick t_r = hlc.now() > t_c (guaranteed by commit-wait) -10. Read sees write (t_r > t_c, so write is visible) -``` - -**Guarantee**: Write finished (step 7) before read started (step 8) in real time. External consistency satisfied. - -### Cross-Shard Transaction - -``` -1. Client: Begin transaction -2. Coordinator: Write intents to participants A, B (provisional t_p) -3. Participants: Replicate intents via Raft -4. Coordinator: Prepare with t_c = max(A.read_ts, B.read_ts, coordinator.now()) -5. Participants: Vote PREPARED -6. Coordinator: Commit transaction at t_c -7. Coordinator: Replicate CommitRecord via Raft -8. Coordinator: Send Commit to participants -9. Participants: Convert intents to committed values@t_c -10. Coordinator: Commit-wait until now() > t_c + ε -11. Coordinator: Return "success" to client - [At this point, all nodes' clocks > t_c] -12. Client: Begin new read (anywhere, any shard) -13. Any node: Pick t_r = hlc.now() > t_c -14. Read sees all writes from committed transaction -``` - -**Guarantee**: Commit-wait at step 10 ensures any future operation (even immediately after) gets `t > t_c`. External consistency satisfied across shards. - -### Why Commit-Wait Is Sufficient - -**Without commit-wait**: -``` -1. Leader commits at t_c = 100 (local clock) -2. Leader returns "success" immediately (no wait) -3. Client gets success at real time 100.001 -4. Client sends read to different node at real time 100.001 -5. Different node's clock = 99.995 (slight skew) -6. Read picks t_r = 99.995 < 100 -7. Read doesn't see write (violation!) -``` - -**With commit-wait (ε = 10ms)**: -``` -1. Leader commits at t_c = 100 -2. Leader waits until now() > 110 (t_c + ε) -3. Leader returns "success" at real time 110 -4. Client sends read at real time 110.001 -5. Different node's clock ≥ 100 (guaranteed: now > 110 - ε = 100) -6. Read picks t_r ≥ 100 -7. Read sees write (correct!) -``` - -**Key insight**: Commit-wait duration ε must cover: -- Maximum clock skew between nodes -- Replication propagation time -- Network jitter - -PTP/chrony provides bounds on clock skew. Raft provides replication guarantees. ε encompasses both. - -## Performance Optimizations - -### Read Timestamp Caching - -**Problem**: Every read queries HLC/TSO for timestamp, adds overhead. - -**Solution**: Cache read timestamp for duration of transaction. - -```rust -struct ReadOnlyTxn { - snapshot_ts: Timestamp, - max_staleness: Duration, - started_at: Instant, -} - -impl ReadOnlyTxn { - fn is_valid(&self) -> bool { - self.started_at.elapsed() < self.max_staleness - } -} -``` - -**Benefit**: Single timestamp acquisition per transaction, not per read. - -### Parallel Commit - -**Problem**: Coordinator waits for all participants to ack commit before commit-wait. - -**Optimization**: Start commit-wait immediately after deciding to commit, while sending commit messages in parallel. - -```rust -async fn commit_transaction(txn: &Transaction) { - // All prepared, decision is COMMIT - let commit_ts = txn.commit_timestamp; - - // Start commit-wait timer immediately - let commit_wait_future = async { - commit_wait(commit_ts, &hlc, epsilon).await; - }; - - // Send commit to participants in parallel - let commit_futures = txn.participants.iter().map(|p| { - async { p.commit(txn.txn_id, commit_ts).await } - }); - - // Wait for both: commit-wait AND participant acks - tokio::join!(commit_wait_future, futures::join_all(commit_futures)); -} -``` - -**Benefit**: Commit-wait overlaps with network/replication latency, reducing total latency. - -### Intent Cleanup: Async Resolution - -**Problem**: Resolving intents to committed values synchronously delays transaction completion. - -**Solution**: Return success to client after commit-wait, resolve intents asynchronously. - -```rust -async fn commit_protocol(txn: &Transaction) { - // Phase 2: Commit decision made - write_commit_record(txn.txn_id, COMMITTED, txn.commit_ts).await; - - // Commit-wait - commit_wait(txn.commit_ts).await; - - // Return success to client (intents still exist, but committed) - client.send_success(); - - // Async: resolve intents to committed values - tokio::spawn(async move { - for participant in txn.participants { - participant.resolve_intents(txn.txn_id, txn.commit_ts).await; - } - }); -} -``` - -**Benefit**: Client latency reduced. Intents marked committed (safe to read), full cleanup happens in background. - -### TSO Timestamp Batching - -**Problem**: TSO mode requires coordinator to contact TSO for every transaction (network hop). - -**Solution**: Pre-fetch timestamp ranges. - -```rust -struct TSOClient { - current_batch: Range, - batch_size: u64, -} - -impl TSOClient { - async fn next_timestamp(&mut self) -> Timestamp { - if self.current_batch.is_empty() { - // Fetch new batch - let start = tso.allocate_range(self.batch_size).await; - self.current_batch = start..(start + self.batch_size); - } - self.current_batch.next().unwrap() - } -} -``` - -**Benefit**: Amortize TSO network cost over multiple transactions. - -## Comparison to Alternatives - -### Percolator (Google Bigtable) - -**Similarities**: -- MVCC with intents -- 2PC for cross-shard transactions -- Lock-free reads - -**Differences**: -- **Percolator**: Primary lock optimization (one primary key per transaction) -- **Cloud9**: No primary lock (all participants symmetric) -- **Percolator**: Client-driven coordination (client acts as coordinator) -- **Cloud9**: Server-driven coordination (database picks coordinator) - -**Why Cloud9's approach**: -- Server-driven coordination simplifies client libraries -- No risk of client failure leaving transaction state ambiguous -- Easier to implement recovery (coordinator is always a database node) - -### Spanner - -**Similarities**: -- MVCC with intents -- External consistency via commit-wait -- Read-only transactions at snapshot timestamp -- Closed timestamps for follower reads - -**Differences**: -- **Spanner**: TrueTime (GPS + atomic clocks) -- **Cloud9**: HLC (PTP/NTP) or TSO -- **Spanner**: Paxos for replication -- **Cloud9**: Raft for replication - -**Why Cloud9's approach**: -- TrueTime not available on public clouds (Cloud9 provides HLC alternative) -- Raft simpler to implement and reason about than Paxos -- Same correctness guarantees, slightly higher latency (acceptable trade-off) - -### CockroachDB - -**Similarities**: -- HLC + commit-wait for external consistency -- MVCC with intents -- Raft replication -- Transaction records for recovery - -**Differences**: -- **CockroachDB**: Transaction record stored as regular key-value pair -- **Cloud9**: Transaction record in dedicated Raft log (TBD: final design choice) -- **CockroachDB**: Complex timestamp cache for push/priority -- **Cloud9**: Simpler wait-based approach initially (push as optimization) - -**Cloud9's design learns from CockroachDB's production experience**: Battle-tested protocol, well-understood failure modes. - -### Calvin (Deterministic Databases) - -**Calvin offers strict serializability without clocks** by pre-ordering transactions through a sequencer and executing them deterministically. This is elegant in theory but imposes constraints that conflict with Cloud9's design goals. - -#### How Calvin Works - -1. Transactions declare read/write sets upfront (or use stored procedures) -2. Sequencer assigns global order -3. All replicas execute transactions in that order deterministically -4. No timestamp-based conflicts, no aborts from clock skew - -#### Why Calvin Doesn't Fit Cloud9's Primary Use Case - -**Problem 1: Requires Known-Upfront Transactions** - -Calvin needs declared read/write sets or stored procedures. This is incompatible with Cloud9's interactive, agent-driven workloads: - -``` -AI Agent workflow (Cloud9 target): -1. GET agent:state:123 -2. if state.mode == "search": - GET vectors:query - PUT agent:state:123 - else: - GET sql:users WHERE condition - PUT agent:cache:xyz -``` - -**With HLC+MVCC**: Works naturally. Agent reads, branches, writes. - -**With Calvin**: Must either: -- Pre-declare all possible paths (impossible for dynamic logic) -- Split into multiple transactions (loses atomicity) -- Use "escape hatch" non-Calvin path (defeats the purpose) - -**Problem 2: Batching Adds Latency** - -Calvin optimizes throughput via batching. At Cloud9's target scale (hundreds to thousands RPS), batching adds pure latency: - -- **Light load**: Wait for batch to fill before sequencing -- **Spiky load**: Queue behind batch boundary -- **Long transactions**: Head-of-line blocking (convoy effect) - -**KV hot-path writes** need <5ms. Calvin's queueing conflicts with this. - -**Problem 3: Deterministic Execution Constraints** - -Calvin requires: -- No wall-clock reads in transactions -- No nondeterministic UDFs (no random(), no system calls) -- Fixed execution order (can't parallelize within transaction) - -**Cloud9 promises**: -- WASM/Python UDFs (user-defined logic) -- Function shipping (arbitrary compute) -- Flexible execution - -**These are incompatible.** Calvin's determinism would cripple extensibility. - -**Problem 4: Read Path Limitations** - -**MVCC+HLC provides**: -- Follower reads at closed-timestamp (no leader coordination) -- Bounded-staleness reads (explicit freshness/latency trade-off) -- Time-travel queries (`AS OF timestamp`) -- Lock-free read-only transactions - -**Calvin's read path**: -- Reads wait for sequencer's stable prefix (coordination overhead) -- Or: Reintroduce snapshot machinery (now you have two systems) -- No natural "read at past timestamp" (conflicts with deterministic order) - -**Cloud9's "lock-free RO transactions" feature requires MVCC**, not Calvin. - -**Problem 5: Sequencer Dependency** - -Calvin's sequencer is: -- The source of truth for ordering -- A hot, critical dependency -- Must be globally available (cross-region Paxos/Raft) - -**Under partition**: Can't proceed without global order. - -**Cloud9's goal**: "Works locally, syncs globally" (agent-friendly). - -**Calvin prevents** partition-tolerant local progress. - -#### When Calvin Does Win - -**Calvin is optimal for**: -- **High write-write contention** (100+ txns/sec touching same keys) -- **Known procedures** (payment processing, inventory reservations) -- **Batch workloads** (analytics writes, bulk updates) -- **Predictable tail latency** (no abort storms) - -**Example use case**: -```sql --- Concert ticket reservation (high contention on same seats) -PROCEDURE reserve_ticket(user_id, seat_id): - READ seats WHERE id = seat_id - IF seat.available: - UPDATE seats SET available = false WHERE id = seat_id - INSERT reservations (user_id, seat_id) -``` +Any node can recover a prepared transaction: -**With standard 2PC**: 1000 concurrent attempts = massive abort storm. -**With Calvin**: Pre-ordered execution serializes cleanly, no aborts. +1. Read the durable transaction record. +2. If committed, finalize every known participant. +3. If aborted, remove every known intent. +4. If still preparing, use the protocol's ownership and timeout rules to elect + one recovery coordinator. +5. Replicate the recovered decision before cleanup. -#### Cloud9's Strategy: Surgical Use +The participant list must be complete before the transaction can commit. +Otherwise recovery could miss a write. -**Default architecture**: HLC + MVCC + Raft -- Interactive SQL queries -- KV hot-path operations (agent state) -- Low-contention workloads (different keys) -- Dynamic workflows (branching logic) -- Rich read modes (follower, time-travel, bounded-staleness) +## Ambiguous Results -**Optional Calvin lane** (future): -- Opt-in per table or procedure type -- Separate sequencer for high-contention procedures -- Used for 5-20% of workload (inventory, payments, rate limits) +A client disconnect after submission may leave an unknown outcome. The API +returns the transaction identity with the ambiguity error. -**This gives you**: -- **Best of both**: Low-latency interactive (HLC) + high-contention optimization (Calvin) -- **No compromise**: Default path fits Cloud9's agent-driven goals -- **Future-proof**: Can add Calvin lane when proven necessary +The client resolves that identity. It does not resubmit the logical mutation +under a new identity. -#### Why Cloud9 Chooses HLC+MVCC as Baseline +## Cross-Dialect Transactions -**Cloud9's primary workloads**: -1. ✅ Concurrent AI agents (dynamic, interactive) → needs MVCC flexibility -2. ✅ SQL analytics (ad-hoc queries) → needs snapshot reads -3. ✅ KV hot-path (low-latency) → needs no batching overhead -4. ✅ Lock-free read-only transactions → requires MVCC -5. ✅ Local development → needs to work without sequencer +Source dialects lower into one Transaction IR. A transaction may span dialects +only when every lowering supports the requested atomicity and consistency. -**Calvin violates**: 1, 3, 4, 5. +Object bytes, columnar projections, and metadata may use different physical +engines. The transaction record states which representations are authoritative +and which updates may complete asynchronously. -**Verdict**: HLC+MVCC+Raft is the correct foundation for Cloud9. Calvin remains an optional optimization for specific high-contention workloads, not the universal substrate. +Derived projections cannot become authoritative by accident. -## Summary +## Performance Rules -Cloud9 transaction protocol provides: +Valid optimizations include: -1. **External consistency**: Commit-wait ensures real-time order is respected -2. **Atomicity**: 2PC guarantees all-or-nothing across shards -3. **Lock-free reads**: MVCC allows readers to access old versions without blocking -4. **Follower reads**: Closed timestamps enable bounded-stale reads from any replica +- one-phase commit for proven single-range transactions; +- parallel prepare; +- batching Raft proposals; +- asynchronous cleanup after a durable decision; +- safe follower reads; +- co-location through explicit placement; +- parallel commit with a proof that the durable record and intents imply one + outcome. -**Key mechanisms**: -- **MVCC intents**: Provisional writes before commit -- **Coordinator-driven 2PC**: Reliable cross-shard atomicity -- **Commit timestamp selection**: `max(participants, coordinator)` ensures consistency -- **Commit-wait**: Time barrier for external consistency (HLC mode) -- **Closed timestamps**: Safe follower reads with bounded staleness +An optimization must state the invariant that removes a protocol step. -**Correctness foundation**: Built on MVCC (01-mvcc.md), timestamp strategies (02-timestamps.md), and external consistency guarantees (03-external-consistency.md). +## Tests -**Deployment flexibility**: Supports HLC (AWS PTP/NTP) and TSO modes, with clock uncertainty bounds determined by infrastructure (05-aws-time-infrastructure.md). +Transaction tests cover: -**Production-tested protocol**: Draws from Spanner, CockroachDB, and Percolator—proven at scale. +- single-range and cross-range atomicity; +- point, range, predicate, and dialect-specific conflicts; +- coordinator and participant crashes at every state transition; +- retries and ambiguous results; +- leader changes during prepare, decision, and commit-wait; +- range splits and movement during 2PC; +- bounded-time failure; +- follower snapshot safety; +- projection authority and freshness; +- Jepsen serializability and strict-serializability histories. diff --git a/spec/09-market-analysis.md b/spec/09-market-analysis.md index 1715dfa..a308dea 100644 --- a/spec/09-market-analysis.md +++ b/spec/09-market-analysis.md @@ -1,558 +1,119 @@ -# Market Analysis: Why Cloud9 Exists +# Product Rationale -Based on extensive user feedback from production deployments of Spanner, DynamoDB, and competing systems, several consistent themes emerge that Cloud9 is designed to address. +Cloud9 addresses fragmentation between database interfaces, execution engines, +and deployment models. -## Spanner Pain Points +The product thesis is that one correctness plane can support several +specialized database domains when a multi-level IR preserves their semantics. -### 1. Cost and Pricing Model +## The Problem -**The Problem**: -- Minimum $65/month (often $1000+/month for production) -- No on-demand pricing — must provision nodes for peak throughput -- Average throughput << peak throughput = wasted spend -- "Half the cost of DynamoDB" marketing ignores provisioning overhead +Teams commonly make coupled choices: -**User quote**: *"We had a huge spanner db with low throughput so had to add idle nodes just for storage which also ballooned costs."* +- SQL selects a relational engine. +- key-value APIs select a managed item store. +- document APIs select a document engine. +- object APIs select separate storage and metadata systems. +- analytical queries select a columnar engine and data pipeline. -**User quote**: *"We were paying tens of thousands of dollars a month for Spanner plus tens of thousands of dollars a month for all the compute sitting in front of it."* +Each system brings a different transaction model, identity model, operational +surface, and locality policy. Moving data between them weakens atomicity and +creates derived state that is difficult to inspect. -**Cloud9 answer**: Serverless on-demand pricing (pay per operation), plus self-hostable open-source option. +Distributed databases add another split. Local development often uses a +different engine from production. The application discovers semantic +differences during deployment. -### 2. GCP Platform Instability +## Cloud9's Position -**The Problem**: -- Constant version churn and breaking changes -- Undocumented features and performance gotchas -- Services deprecated without warning (Google Domains → Squarespace) -- Fear of product cancellation ("Will Spanner be shut down?") +Cloud9 combines: -**User quote**: *"Google runs their tech stack as if it's a startup that builds their CV. Everything is immature, tons of hacks, undocumented features."* +- Spanner-style external consistency; +- SQLite-like local operation; +- SQL, key-value, document, object, and analytical dialects; +- domain-specific physical engines; +- one transaction, catalog, placement, and replication plane; +- an MLIR-style lowering and optimization system. -**User quote**: *"Much of the time GCP feels like a science project, and not a real business."* +The differentiator is the conversion architecture. Cloud9 does not expose five +protocols over one generic row or key-value engine. -**Cloud9 answer**: Open-source MIT license. Code can never be "shut down" by vendor. Community-driven development with stability guarantees. +## Product Requirements -### 3. Support Quality +### Stable semantics -**The Problem**: -- Support ranges from "unhelpful" to "non-existent" -- Escalations go nowhere -- Bug reports closed as stale without resolution -- Sales team doesn't understand enterprise needs +Changing topology must not change the data model. A local database and a +distributed database use the same source interfaces and transaction semantics. -**User quote**: *"Google's support is horrendous. They refer you to idiots that drag you through calls until your will for life dies."* +Capability-dependent guarantees remain explicit. Local mode does not claim +hardware-backed TrueTime. -**User quote**: *"We have a bug reported back in 2020 that got closed recently without any action because it became stale."* +### Inspectable lowering -**User quote**: *"GCP support would suggest to ask in StackOverflow."* +Users can inspect how a source request became a transaction, placement plan, +and physical plan. Unsupported behavior fails at a named conversion boundary. -**Cloud9 answer**: Community-driven support via GitHub Issues/Discussions. No support tax, no gatekeepers. Open development process. +### Domain-specific performance -### 4. Documentation Gaps +Point operations, document updates, object ranges, relational joins, and +columnar scans need different data structures. -**The Problem**: -- Performance characteristics poorly documented -- Sharding/partition behavior not explained clearly -- "Hot shard" problems discovered at scale -- No clear migration guides +Cloud9 keeps one correctness plane while allowing each workload to use an +appropriate physical engine. -**User quote**: *"Google's docs are incomplete; there are lots of performance gotchas that exist throughout the entire service, and they aren't clearly documented."* +### Locality as policy -**Cloud9 answer**: Comprehensive documentation from day one. Open-source allows reading the implementation. Design notes explain trade-offs. +Placement, residency, and replica proximity are part of the plan. They are not +after-the-fact infrastructure hints. -### 5. Operational Complexity +### Open operation -**The Problem**: -- GKE constant version churn forces infrastructure rework -- Network configuration complex (vs AWS/GCP VPC simplicity) -- Hidden costs (discovered $6k database in bill) -- Requires constant vigilance for breaking changes +Cloud9 is open source and self-hostable. File formats, protocols, limits, and +failure modes are documented and testable. -**User quote**: *"50% of time making sure we are prepared for their shit and 50% our ambitious infra plans."* +## Competitive Categories -**Cloud9 answer**: Single binary, minimal operational surface. Works identically local and global. No platform lock-in. +| Category | Strength | Cloud9 requirement | +|----------|----------|--------------------| +| Relational databases | SQL and mature transactions | Preserve relational semantics | +| Key-value stores | Predictable point operations | Match conditional and item behavior | +| Document databases | Flexible nested data | Preserve paths and update operators | +| Object stores | Durable large objects | Preserve versions, ranges, and metadata | +| Analytical databases | Columnar execution | Keep vectorized, locality-aware plans | +| Distributed SQL | Scale and transactions | Add explicit bounded time and placement | +| Embedded databases | Simple local use | Keep one binary and one directory | -## DynamoDB Pain Points +Cloud9 must earn comparison with each category on its native workload. -### 1. Data Model Limitations +## Non-Claims -**The Problem**: -- Key-value only, no SQL -- Must design access patterns upfront -- No ad-hoc queries or joins -- Single-table design patterns are complex +The architecture does not prove: -**User quote**: *"DynamoDB is fantastic for not doing things at scale... an entire RDBMS is way overkill for."* +- complete compatibility with any named service; +- better performance than every specialized system; +- planetary scale in the current implementation; +- TrueTime behavior on unsupported hardware; +- zero-cost transactions across incompatible physical engines. -**Cloud9 answer**: Both SQL and KV. Cross-API joins. Familiar relational model when needed. +Those are measured outcomes, not taglines. -### 2. Capacity Planning Gotchas +## Validation -**The Problem**: -- Hot partition issues (1000 WRU/partition limit) -- Shards get same quota regardless of traffic distribution -- Over-provision or queue requests to handle hot keys -- Not obvious from documentation +Product claims require: -**User quote**: *"Even though you might have paid for 1000rps, that RPS volume is divided across all your shards."* +- compatibility suites against reference systems; +- Jepsen histories for consistency; +- crash and recovery tests; +- local setup and migration tests; +- named benchmarks for each physical domain; +- multi-region fault exercises; +- traces that show selected lowerings and placement. -**Cloud9 answer**: Transparent sharding with automatic rebalancing. Cross-shard transactions at same consistency level. +Every benchmark reports workload, dataset, hardware, topology, durability, +consistency, and software versions. -### 3. No Multi-Item Transactions +## Success -**The Problem**: -- TransactWriteItems limited to 25 items -- No true ACID across arbitrary keys -- Application must handle consistency - -**Cloud9 answer**: Unbounded multi-key transactions with strict serializability. - -## Common Theme: Trust and Lock-In - -**Observation**: Users fear vendor lock-in more than they fear technical limitations. - -**User quote**: *"Doing business with Google is a liability."* - -**User quote**: *"I trust AWS to be a stable, long term foundation to build a product on, I don't trust GCP to be the same."* - -**User quote**: *"Why am I going to sign up for a service that is surely to be canceled on a Google Whim™?"* - -**Cloud9's fundamental answer**: -- Open-source MIT license removes vendor lock-in -- Self-hostable on any infrastructure -- Managed Dedalus Cloud offering for convenience, not lock-in -- Community can fork if Dedalus Labs disappears - -## The Postgres Refuge - -**Observation**: Many threads conclude "just use Postgres" because it's: -- Well-understood and stable -- Not vendor-locked -- Good enough for 99% of use cases - -**User quote**: *"Postgres is a piece of software. Cloud Spanner/Dynamo etc are managed services. It makes no sense to directly compare."* - -**User quote**: *"Golden Rule of data: Use PostgreSQL unless you have an extremely good reason not to."* - -**Cloud9's position**: Be the **Postgres of distributed databases**: -- Open, trusted, boring technology -- Postgres wire compatibility -- Clear documentation and predictable behavior -- Available when you outgrow single-node Postgres - -## Specific Technical Complaints - -**Spanner**: -- DeWitt clause prevents independent benchmarking -- No protobuf column support in Cloud Spanner (only internal Spanner) -- Unclear whether Google services use Cloud Spanner or internal Spanner -- Write-through cache needed for read-heavy workloads (complexity + cost) - -**DynamoDB**: -- Item size limits (400 KB max) -- Read/write unit calculations opaque (1 byte over 1KB = 2 RU charged) -- Connection management nightmare with Lambda/serverless - -**Both**: -- Difficult to meaningfully compare offerings and value -- Lock-in makes switching costs prohibitive -- Enterprise architects push them for imagined scale needs - -## Additional Insights from Developer Communities - -### Spanner Positioning Problem - -**The Problem**: -- "Overkill for prototypes" — minimum cost too high for experimentation -- "Mosquito with a sledgehammer" — power users don't need, small users can't afford -- Recommendation is always "use Cloud SQL instead" — Spanner's own ecosystem recommends against it - -**User quote**: *"Spanner is pricey - do you need that scale/availability? Cloud SQL would be more your speed."* - -**User quote**: *"Spanner is a mosquito with a sledgehammer for most workloads."* - -**User quote**: *"I'd say stick with cloud SQL for prototyping, Spanner is for production."* - -**Implication**: Spanner has no **"grow into it"** story. You can't start small and scale up — the entry point is already enterprise-scale pricing. - -**Cloud9 answer**: Start local (SQLite-level simplicity), scale to regional, scale to global — same binary, same semantics. No cliff between "prototype" and "production." - -### The Postgres Gravitational Pull - -**The Problem**: -- Every Spanner discussion ends with "just use Postgres" -- Postgres-compatible offerings (AlloyDB, Cloud SQL) recommended over Spanner -- Even Google's own advocates suggest Postgres alternatives - -**User quote**: *"Please, do Postgres, not MySQL. Let it die already."* - -**User quote**: *"If you can sling postgres I'd go straight to alloydb."* - -**Cloud9 answer**: Postgres wire compatibility from day one. Be where developers already are, not where they have to migrate to. - -### Developer Experience Friction - -**The Problem**: -- No local development story ("can't install software on desktop") -- Cloud-only development is clunky (Cloud Shell, Cloud Editor) -- No emulator for cost-controlled local dev (unlike AlloyDB) -- Forces developers into specific GCP workflows -- Missing features vs Postgres (no stored procs, no ts_vector, limited data types) - -**User quote**: *"Developing in the cloud is possible. If you go to cloud shell you can open a cloud version of vscode. Haven't used it much so not sure how well it works."* - -**User quote**: *"Spanner [lacks] auto increment counters, ts_vector as type and a bunch more."* - -**User quote**: *"Still no support for user-created stored functions/stored procs."* - -**Cloud9 answer**: Single binary runs locally. Develop on laptop, deploy to cloud without changes. No forced cloud-development workflow. Full Postgres compatibility from day one. - -## What Users Actually Want - -**Synthesis from discussion**: - -1. **Predictable, transparent pricing** — no surprise bills, no forced provisioning -2. **Stability and trust** — won't be deprecated, won't see 10x price increases -3. **Good enough for small scale, grows to large** — DynamoDB's free tier vs Spanner's $65/month floor -4. **Familiar interfaces** — SQL preferred, KV when needed -5. **Open and portable** — can leave vendor without rewrite -6. **Real support** — responsive humans who understand the problem -7. **Clear documentation** — performance characteristics, limits, gotchas all documented upfront -8. **Local development** — prototype locally, deploy globally without workflow changes - -**Cloud9's design targets all eight points.** - -## The Billing Horror Stories - -**The most damaging feedback**: Silent, unexpected charges that destroy user trust. - -### The RAG Engine Incident (September 2024) - -**What happened**: -- Google changed RAG Engine backend to use Spanner (Scaled Tier, 1000 PU) -- **No clear notification** to affected users (some got email, many didn't) -- Users who tried RAG Engine once got charged $30-800/day -- Charges appeared as "Cloud Spanner" even though users never enabled Spanner -- Spanner instances didn't show up in Spanner console (hidden) -- **Auto-provisioned in ALL regions** (US + EU) per project - -**User quote**: *"$30/day for a service I didn't knowingly use seems extremely expensive. I deleted all my projects to make sure no keys were leaked."* - -**User quote**: *"$300 for me 😭"* - -**User quote**: *"Another victim here. $800 gone."* - -**User quote**: *"What Google is doing on this one is, frankly, appalling. It's theft."* - -**User quote**: *"I had bit faith in GCP to convince my company to switch from Azure. Now no way I can/will recommend anyone to use GCP."* - -**The worst case** (£3,000 / $3,800 in one month): -- Dormant account (£0.02/month residual storage) -- Sept 3: charges spike to £60-70/day -- User never created RAG corpus, never uploaded data -- RAG UI shows nothing -- **Billing account frozen** → can't access account to delete resources -- **Catch-22**: Must pay disputed balance to unlock account to stop charges -- Support closes tickets, refuses escalation -- Balance climbing daily with no way to stop - -**User quote**: *"The catch-22: billing suspension prevents me from accessing my account to delete the service/close the account, but Google says I must pay the disputed balance first to unlock it."* - -**User quote**: *"Support has closed my tickets multiple times, refuses to escalate further, and won't deprovision the hidden resources."* - -**Google's response**: *"After final review, charges are valid. These were provisioned as a necessary component of the Vertex AI RAG Engine service you activated... charges are considered legitimate."* - -**User's dilemma**: *"Can I just refuse to pay? What happens if it goes to debt collections?"* - -**The resolution process**: -- 2+ hour wait times for support -- Users had to manually delete RAG Engine (not obvious) -- Must delete **per region** (auto-enabled in multiple regions) -- Some got 90% refund as "one-time courtesy" -- Many charged for weeks before noticing -- Some accounts frozen, unable to delete resources -- Documentation says "free to use" but hides $2k/month Spanner cost - -**User quote**: *"RAG Engine docs say it's 'free to use' but fail to mention the auto-provisioned Spanner instance costs £2k+/month."* - -**User quote**: *"Those RAG Engine Cloud Spanner services were automatically enabled for ALL available regions and for each individual project. I needed to delete them one by one."* - -**User quote**: *"GCP assistant is essentially useless and I couldn't find any live chat support... Wasted half my morning on this crap."* - -**User quote**: *"Very poor customer communication."* - -**User quote**: *"GCP will definitely beat estimates now LOL. Some MBA wearing a vest will get a big bonus in exchange for all the misery."* - -**Separate incident** (2025): Gemini 2.5 Flash billing error generated **$70,000+ bills** for non-existent usage, charges climbing $10,000/day even after API keys deleted. - -### The Pattern - -1. Service defaults change silently -2. Expensive resources provisioned automatically -3. Poor visibility (instances don't show in expected console) -4. Slow/unhelpful support response -5. Users lose trust permanently - -**User response to billing disasters**: Migration away from GCP entirely. - -**User quote**: *"Time to migrate. I can host my rag setup on my vps that I pay $12 a month for."* - -**The exodus**: Users leave GCP for **$12/month VPS** rather than pay surprise Spanner bills. - -**Cloud9's commitment**: -- **No silent defaults** — explicit opt-in for all paid tiers -- **Visible resource usage** — every replica, every shard visible in console -- **Billing transparency** — real-time cost tracking, no surprises -- **Zero-cost local mode** — develop and test for free -- **Community support** — no support tax, no wait queues -- **Self-hosting option** — run on $12/month VPS if desired, same guarantees - -## Pricing Transparency Issues - -**The "Basic vs Standard" confusion**: -- Documentation shows different tiers in different places -- Support agents have access to different pricing calculators -- Processing Units (PU) pricing not obvious -- "Wind down when not using" not possible (always-on charges) - -**User quote**: *"Documentation appears to be inconsistent. Some suggest there is a 'basic' tier, but when you go to the estimate page, it starts with 'Standard'."* - -**User quote**: *"Does anyone know how to lower the costs when you're in Dev mode? Is there a way to wind down the environment when you're not using it?"* - -**Answer from community**: No. Spanner charges for provisioned capacity, not usage. - -**Cloud9 answer**: -- Open-source = free local development -- Managed tier pricing published upfront -- Can "wind down" by stopping the binary (self-hosted) -- Pay-per-operation option (no idle charges) - -## Early Adoption Concerns (2017 Launch) - -**From initial Spanner launch discussions**: - -### Understanding Barrier - -**The Problem**: -- Complex architecture hard to explain -- TrueTime concept not intuitive -- Users struggled to understand when Spanner is needed vs Postgres -- "Shitty article" complaints (marketing-heavy, light on substance) - -**User quote**: *"That was a really shitty article. Can anyone explain the real world benefits of this?"* - -**User quote**: *"All that babel about TrueTime and nowhere a description of the problem it solves."* - -**Cloud9 answer**: Clear design notes (this document) explain trade-offs. No hand-waving about "mastering time." - -### Schema Modeling Constraints - -**The Problem**: -- No explicit foreign keys outside parent/child relationships -- No multi-parent tables (can't model true many-to-many easily) -- Must choose access patterns upfront (parent/child determines sharding) -- No referential integrity constraints across tables -- No triggers -- No reference types for columns - -**User quote**: *"How does a Many-to-Many relationship work? Are they all root level tables? Is there no explicit foreign keys?"* - -**User quote**: *"You cannot put one table as a child of two others."* - -**User quote**: *"There are also no triggers, and no reference types (so you can't define a column in BankAccount as type 'key of Bank')."* - -**Answer from community**: *"No cross-table referential integrity constraints... implementing them would be costly in terms of latency."* - -**The ACID debate**: Users argue Spanner isn't truly ACID because it lacks the "C" (consistency via constraints). - -**User quote**: *"Spanner is not ACID. It's AID. It lacks the C of 'the data in the schema conforms to the business rules.' If you don't have foreign keys, triggers, range limitations, you don't have C."* - -**Google's defense**: *"ACID for Spanner means the consistency rules definable for Spanner's not-really-an-RDBMS model are upheld."* - -**Cloud9 answer**: Standard SQL foreign keys, constraints, and triggers. Postgres compatibility means familiar schema modeling. True ACID with full referential integrity. - -### Trust in Complexity - -**The Problem**: -- Skepticism about needing atomic clocks -- "Why not just use Postgres?" dominates discussion -- Benefits unclear until extreme scale -- GPS/atomic clock dependency seems fragile - -**User quote**: *"If some muppet decides to mess with GPS signals near the datacenter, what happens?"* - -**User quote**: *"Almost no applications have important use cases that make such a solution a requirement for success."* - -**Answer from community**: *"Uses 6 time masters. 3 GPS clocks with individual antennas, 3 atomic clocks. Kalman filter rejects bad GPS, falls back to atomic."* - -**Cloud9 answer**: -- Works without atomic clocks (HLC on commodity hardware) -- Clear failure modes documented -- Benefits obvious from prototype to production (same binary) - -## The "Just Use Postgres" Reflex - -**Consistent theme across all discussions**: Default to Postgres unless you absolutely can't. - -**User quote**: *"Just use PG...until you can't."* - -**User quote**: *"Are you sure your data doesn't fit in PostgreSQL? You should probably try PostgreSQL first."* - -**User quote**: *"The super-power of Postgres is that it supports everything... doesn't suck at anything but horizontal scaling."* - -**The Spanner problem**: No story for "start with Postgres, grow into Spanner." It's a hard cut-over. - -**Cloud9's answer**: -- **Is** Postgres for small scale (wire-compatible, single binary) -- Grows to Spanner-class scale without migration -- No "Postgres vs Spanner" decision — it's both - -## The "Overkill for Real Workloads" Pattern - -**Recurring scenario**: Users evaluate Spanner for moderate scale, realize it's massive overkill. - -**Real case** (9 months ago): -- 500 requests/second (read-heavy) -- 50 GB data -- Public API, no auth -- Looking to replace Firestore (query limitations) - -**User calculations**: "200 processing units handles 15k QPS... I doubt it." - -**Community response**: - -**User quote**: *"Are you stupidly rich? Like the lost son of Sultan of Brunei? No? Then it's too expensive, consider other options."* - -**User quote**: *"Sounds like bringing in a tank into a boxing fight."* - -**User quote**: *"500 requests per second is not that much honestly. Spanner... is designed for much higher throughput."* - -**User quote**: *"A lot of over-engineered tech choices are sometimes to compensate for lack of applying fundamentals with simpler and cheaper alternatives."* - -**Googler's response**: Spanner = $146/month, Cloud SQL = $231/month (Spanner cheaper!) - -**User's conclusion**: *"After more digging into the subject I realize I don't need it."* - -### The Disconnect - -**Google's pitch**: "Spanner is cheaper than Postgres!" - -**Reality**: -- Users still choose Postgres -- Not because of cost -- Because Spanner feels wrong for the scale -- "PostgreSQL enters the chat" (final comment) - -**Why this matters**: -- Even when Spanner is **cheaper**, users reject it -- The "overkill" perception is **psychological**, not economic -- Users want technology that feels appropriate to their scale -- Spanner positioned as "big company tech" → small companies avoid it - -**Cloud9's advantage**: -- Same binary from prototype (500 RPS) to massive scale (500k RPS) -- No psychological barrier -- "Just use Cloud9" → natural default like "just use Postgres" -- Pricing scales with you (free → cheap → expensive as you grow) - -## Cloud SQL Performance Issues - -**Reported problems** with Google's managed Postgres/MySQL: - -### Performance Degradation - -**The Problem**: -- CloudSQL slower than self-hosted on VMs -- Read locking happens frequently -- Replication lag even within same zone -- Trigger execution delays on replicas - -**User quote**: *"My experience with CloudSQL was horrendous. It was slow and read locking happened ridiculously often. Once I spinned up a MySQL instance on a VM, everything worked flawlessly."* - -**User quote**: *"We had significant slowness with cloudsql and moved to managing our own instances on VMs and haven't looked back."* - -**User quote**: *"We've seen significant delay in some replicas (in the same zone) for some more complex triggers."* - -**The irony**: Users migrate to Spanner not because they need global distribution, but because **CloudSQL is unreliable**. - -### The "Fast Reads" Trap - -**User's goal**: "Need the database to be highly available and never waiting on locks... guaranteeing fast reads all the time." - -**Community response**: Spanner doesn't solve this. -- Spanner still uses locks for read-write transactions -- Lock-free read-only transactions exist, but user may not know to use them -- "CloudSpanner would be a way to have GCP manage everything about scaling" (wrong expectation) - -**User quote**: *"CloudSpanner starts to make sense once your DB is larger than 10TB and you need replication across the whole planet."* - -**User's realization**: *"This is not my challenge, but rather guaranteeing fast reads all the time."* - -**Recommendation given**: CloudSQL with read replicas (back to where they started). - -**Cloud9 answer**: -- Lock-free read-only transactions by default (documented clearly) -- MVCC means readers never block writers -- Works locally for testing before cloud deployment -- No CloudSQL performance issues (you control the hardware) - -## The Expectation Mismatch - -**Pattern observed**: -1. User has performance issues with CloudSQL -2. User investigates Spanner as "better managed database" -3. Community asks: "Do you have billions of dollars?" -4. User realizes Spanner solves different problem -5. User sent back to CloudSQL or self-hosting - -**The gap**: No managed database between "CloudSQL (unreliable)" and "Spanner (overkill)". - -**Quote**: *"Do you have billions of dollars? [No] That would be awesome lol - but no."* - -**Cloud9's positioning**: -- Fills the gap between CloudSQL and Spanner -- Self-hostable (control your own performance) -- OR managed tier (Dedalus Cloud) -- Same guarantees at all scales -- No "do you have billions?" barrier - -## The Missing Middle - -**Synthesis of all observations**: - -There is a massive gap in the market between: -- **Postgres/MySQL** (single-node, no horizontal scale) -- **Spanner/DynamoDB** (enterprise-only, high cost floor, vendor lock-in) - -Users in this gap need: -- Multi-region capability (not planet-scale, but 2-3 regions) -- ACID transactions (not just eventual consistency) -- Familiar SQL interface (not KV-only) -- Reasonable cost (not $1000+/month minimum) -- Trust and portability (not vendor lock-in) - -**Cloud9 is built specifically for this missing middle**: -- Start on laptop (zero cost) -- Deploy to 3 regions (reasonable cost) -- Scale to planet-scale (enterprise cost, but optional) -- Open-source MIT (never locked in) -- Postgres-compatible (familiar interface) - -## Summary: The Market Opportunity - -Cloud9 exists because the current landscape forces users into false choices: - -1. **Cost vs Scale**: Pay enterprise prices from day one, or stay single-node forever -2. **Lock-in vs Power**: Accept vendor control, or give up distributed features -3. **Simplicity vs Capability**: Use simple database or learn proprietary API -4. **Local vs Global**: Develop in cloud or deploy single-node - -**Cloud9 eliminates all four false choices**: -- Cost scales with usage (free local → expensive global) -- Open-source removes lock-in without sacrificing power -- Postgres compatibility gives capability without learning curve -- Same binary works local and global (no development gap) - -The market doesn't need another distributed database. It needs a distributed database that acts like Postgres: boring, predictable, trusted, and available when you need it. - -**That's Cloud9.** +Cloud9 succeeds when an application can begin locally, retain its semantics at +distributed scale, and use specialized execution without assembling separate +databases and consistency layers. diff --git a/spec/10-consensus.md b/spec/10-consensus.md index a82b42d..6fa2432 100644 --- a/spec/10-consensus.md +++ b/spec/10-consensus.md @@ -1,357 +1,147 @@ -# Consensus Architecture +# Consensus and Replication -**Question**: How do we replicate data reliably and make it the foundation for operational flexibility? +Cloud9 uses Raft for replicated ordering. Raft remains a small, deterministic +state machine whose transitions can be tested without storage or networking. -**Answer**: Raft consensus with clean abstractions for operational features. +The database node supplies durable storage, transport, timers, and command +application around that state machine. -## Why Raft +## Boundary -Raft is the proven choice for distributed databases: +The Raft layer accepts: -- **Battle-tested**: Used in etcd, CockroachDB, TiKV, Consul -- **Formally verified**: Proven correct in Coq/TLA+ -- **Understandable**: Clear separation between leader election, log replication, and safety -- **Predictable**: Well-documented failure modes and performance characteristics +- messages from peers; +- election and heartbeat ticks; +- client proposals; +- membership changes; +- snapshot completion events. -**Not novel. That's the point.** +It emits: -Cloud9's goal isn't to innovate in consensus algorithms. It's to build a database that operators can trust. Raft is the consensus algorithm you can explain to your team, debug in production, and find papers about when things go wrong. +- messages to send; +- log entries and hard state to persist; +- committed entries to apply; +- snapshots to install; +- leadership and membership changes. -## The Consensus Driver Interface +The caller must persist required state before sending messages or exposing +effects that depend on it. -Cloud9 treats consensus as an isolated component with a narrow interface. This isn't "pluggable consensus" (swapping algorithms at runtime). It's clean architecture: separating the replication mechanism from the state machine it replicates. +## Safety Invariants -### The Interface +Cloud9 preserves: -```rust -trait ConsensusDriver { - // Propose a command for replication - fn propose(&mut self, cmd: Command) -> Result; +1. At most one leader per term. +2. Committed entries remain in every future leader's log. +3. A state machine applies each committed index once and in order. +4. A node never votes twice in one term. +5. A removed replica cannot regain authority without a new configuration. +6. Snapshot installation cannot discard a committed suffix. - // Apply committed entries to state machine - fn poll_committed(&mut self) -> Vec; +Database commands add another invariant: applying one command at one log index +must produce the same state on every replica. - // Read current leader - fn leader(&self) -> Option; +## Durable State - // Transfer leadership (operational control) - fn transfer_leadership(&mut self, target: ReplicaId) -> Result<()>; +The storage boundary includes: - // Add/remove replicas (reconfiguration) - fn reconfigure(&mut self, new_config: RangeConfig) -> Result<()>; -} -``` +- current term and vote; +- log entries; +- commit and applied progress; +- membership configuration; +- snapshots and their metadata. -**That's it.** The rest of Cloud9 (transaction coordinator, MVCC, SQL execution) doesn't care about Raft internals. It sees: -- A log of committed commands (linear order) -- A current leader (for write routing) -- Control knobs for operational needs +Write-ahead ordering is explicit. Recovery either reconstructs one valid Raft +state or fails with a corruption error. -### Why This Boundary Matters +Log truncation follows a durable snapshot. The snapshot records the included +index, term, membership, and database state checksum. -**Testability**: The state machine (MVCC storage engine) can be tested independently with a simulated consensus driver. No need to spin up 5-node Raft clusters to test transaction semantics. +## Range Replication -**Debuggability**: When production debugging consensus issues, engineers only need to understand Raft. When debugging transaction issues, they only need to understand MVCC. Concerns don't bleed. +Each distributed range maps to one Raft group. The group replicates +deterministic physical commands produced by the lowering pipeline. -**Evolution**: If a better consensus algorithm emerges (peer-reviewed, formally verified, widely deployed), the interface provides a migration path. But this is a "someday, maybe" scenario, not a launch requirement. +Raft orders operations within a range. It does not provide atomicity across +ranges. The transaction protocol owns that boundary. -## Operational Advantage: Zero-Downtime Upgrades +Local mode uses a one-replica Raft group. It may commit without network I/O but +still uses the durable log and application order. -Raft's joint consensus protocol, combined with Cloud9's timestamped schema architecture, enables true zero-downtime upgrades—a capability that emerges naturally from the design rather than being retrofitted. +## Linearizable Reads -### The Rolling Upgrade Protocol +A leader may serve a current read only after proving its authority. Valid +mechanisms include Raft ReadIndex or a lease protocol with a proven fence. -**Procedure**: -1. Add new replica running v2.0 binary as learner (receives log, doesn't vote) -2. Wait for learner to catch up on Raft log -3. Promote learner to voter using joint consensus (temporary state where both old and new quorum overlap) -4. Transfer leaseholder to v2.0 replica (leader can now be v2.0 node) -5. Remove old v1.0 replica from configuration -6. Repeat for all ranges in the cluster +A follower requires an applied-index and safe-time proof. Being caught up at +some earlier instant is insufficient. -**Why this works**: +## Membership Changes -**Schema changes are timestamped**: Queries don't ask "what version is this node running?" They ask "what was the schema at timestamp T?" Old nodes query at old timestamps (old schema), new nodes query at new timestamps (new schema). Both interpret the same MVCC key-value data correctly because schema interpretation is decoupled from binary version. +Replica changes use Raft's supported configuration-change protocol. Cloud9 +allows one logical membership change at a time per group unless the +implementation proves a stronger rule. -**Raft log format includes protocol markers**: Each log entry carries version information. Nodes negotiate compatible protocol during handshake. If v2.0 introduces new log entry types, v1.0 nodes can skip unknown entries (forward compatibility) or v2.0 nodes can write v1.0-compatible entries during the joint consensus phase (backward compatibility). +Learners receive state before becoming voters. Removal is complete only after +the new configuration is committed and stale ownership is fenced. -**Quorum never lost during membership changes**: Joint consensus ensures that no single point in time requires agreement from both old and new majorities. Writes continue flowing because either the old quorum or new quorum can commit—never stuck waiting for both. +## Upgrades -**Leaseholder isolation**: Leadership can transfer to v2.0 nodes before v1.0 nodes are removed. The leaseholder (which handles reads) runs the new binary while followers (which only replicate) can still run old binaries. Read path and write path operate on different versions simultaneously. +Raft log commands and snapshots are versioned. A mixed-version group may +commit only encodings understood by every replica required for the active +configuration. -### Why Postgres Can't Do This +An old node never skips an unknown command. Skipping would make replicas apply +different state. -**Postgres treats schema as global state**: `ALTER TABLE users ADD COLUMN` acquires `AccessExclusiveLock` and updates system catalogs (`pg_class`, `pg_attribute`) atomically across all nodes. Even with MVCC for table data, the schema change requires a coordination barrier where all nodes observe the same catalog version. Mixed-version clusters can't agree on schema. +Upgrade gates verify: -**No protocol versioning in replication**: Postgres streaming replication and logical replication protocols lack version negotiation. If v16 changes the replication message format, v15 followers can't decode it. Upgrades require stopping all nodes, upgrading binaries, and restarting—downtime by necessity. +- RPC compatibility; +- log and snapshot compatibility; +- command semantics; +- minimum reader and writer versions; +- downgrade safety. -**Binary format incompatibilities**: Postgres heap tuple format, WAL record structure, and catalog schemas change between major versions. Mixed-version clusters would corrupt data. `pg_upgrade` exists specifically because in-place rolling upgrades are architecturally impossible. +## Failure Handling -### Cloud9's Architectural Advantages +Loss of quorum stops new commits. Cloud9 does not acknowledge an uncommitted +proposal. -**1. Schema is Raft-replicated metadata, not global locks**: DDL operations write new schema versions to the metadata keyspace with commit timestamps. Queries pick schema at transaction start time. No coordination needed—schema evolution is just another replicated write. +Disk corruption, impossible log state, and snapshot checksum failure are fatal +to that replica. Recovery uses another identical replica or a verified backup, +not a different storage implementation. -**2. Protocol versioning from day one**: RPC messages include version headers. Nodes handshake to negotiate compatible protocol. Raft log entries carry format versions. Mixed-version clusters are supported by design, not retrofitted. +## Observability -**3. MVCC extends to schema**: The same mechanism that enables lock-free reads on data enables lock-free schema evolution. Transactions see (data@timestamp, schema@timestamp) pairs. Upgrading binaries doesn't change data layout—only schema interpretation. +Each group reports: -**4. Per-range independence**: Each range can upgrade independently. No cluster-wide "flip the switch" moment. If a range upgrade fails, it doesn't cascade to other ranges. Fault isolation by design. +- term and role; +- leader and membership; +- last log, commit, and applied indexes; +- replication lag; +- snapshot progress; +- proposal latency; +- rejected stale messages; +- storage and checksum failures. -### Production Reality +Logs identify the range, replica, term, and index. -CockroachDB added zero-downtime upgrades after years of production pain (v20.x+, ~2020). Early versions required careful orchestration and failed frequently. Spanner has this capability but is proprietary—can't verify implementation. +## Tests -Cloud9 designs for it from the start: timestamped schemas, versioned protocols, Raft-based replication that supports gradual state evolution. The architecture assumes mixed-version operation is normal, not exceptional. +Consensus tests cover: -This is why Cloud9 can claim "one-click zero-downtime upgrades" as a day-one feature—the primitives are already present in the core design. +- the Raft state machine against a reference model; +- elections, partitions, and message reordering; +- conflicting log repair; +- crash recovery at every persistence boundary; +- snapshot creation and installation; +- membership changes and stale-replica fencing; +- deterministic database command application; +- linearizable reads; +- Jepsen histories under process, network, and disk faults. -## Operational Features: Within Raft +## References -The consensus driver interface enables operational flexibility without algorithmic complexity. These features live within Raft's existing framework: - -### 1. Dynamic Leader Placement - -**Problem**: The leader handles all writes. Placing the leader in the wrong datacenter adds cross-region latency to every write. - -**Solution**: Raft's leadership transfer mechanism. - -``` -// Before: leader in us-east-1, app in eu-west-1 -write latency = RTT(eu-west-1 → us-east-1) = ~80ms - -// After: transfer_leadership(eu-west-1_replica) -write latency = local = ~5ms -``` - -**Cloud9 exposes this** via the range placement API: -```sql -ALTER RANGE users CONFIGURE LEADER PREFERENCE 'eu-west-1'; -``` - -**Not a new algorithm.** Raft already has leadership transfer (§3.10 of the paper). Cloud9 just makes it operationally accessible. - -### 2. Witness Replicas - -**Problem**: You want 5-replica durability (survive 2 failures) but don't want to store 5 full copies of the data. - -**Solution**: Witness replicas participate in quorum but don't store data. - -``` -3 full replicas + 2 witness = 5-node quorum -- Survives 2 failures (any 2 of the 5 can be down) -- Only 3× storage cost (not 5×) -``` - -**Mechanism**: -- Witness receives log entries, votes in elections, participates in quorum -- Witness doesn't apply entries to a state machine (no storage) -- Witness can't serve reads (it doesn't have data) - -**Cloud9 usage**: -```sql -ALTER RANGE orders ADD REPLICA ON 'us-west-2' AS WITNESS; -``` - -**Not novel**: TiKV calls these "learner replicas without data." Raft's joint consensus (§6) handles configuration changes safely. - -### 3. Learner Replicas - -**Problem**: Adding a new full replica requires copying data. If you immediately add it to the quorum, the majority becomes unavailable during the copy (the new replica can't vote yet, but counts toward the quorum size). - -**Solution**: Learner replicas receive log entries but don't vote. - -**Protocol**: -1. Add replica as learner -2. Replica catches up (receives log, applies to state machine) -3. Once caught up, promote to voting member -4. Now safe: it won't block quorum - -**Cloud9 automation**: -```sql -ALTER RANGE products ADD REPLICA ON 'eu-central-1'; --- Internally: --- 1. Add as learner --- 2. Stream snapshot + log --- 3. Auto-promote when caught up -``` - -**Not novel**: Raft's single-server membership changes (§4.1) + learner role (LogCabin implementation, etcd). - -### 4. Per-Range Configuration - -Every range (shard) has independent consensus configuration: - -```rust -struct RangeConfig { - replicas: Vec, - leader_preference: Option, - quorum_size: usize, -} - -struct ReplicaDescriptor { - id: ReplicaId, - region: Region, - role: ReplicaRole, // Voter | Witness | Learner -} -``` - -**Why per-range?** -- Multi-tenant: different tables have different SLAs -- Geo-distributed: hot data in 3 regions, cold data in 1 -- Cost optimization: critical data = 5 replicas, logs = 3 replicas - -**Example**: -```sql --- User data: 5 replicas across 3 regions -ALTER RANGE users CONFIGURE REPLICAS 5 IN REGIONS ('us-east-1', 'eu-west-1', 'ap-southeast-1'); - --- Logs: 3 replicas, same region -ALTER RANGE logs CONFIGURE REPLICAS 3 IN REGIONS ('us-east-1'); -``` - -**Implementation**: Each range runs its own Raft group. Configurations are independent. The SQL layer maps keys to ranges and routes accordingly. - -## What Cloud9 Does NOT Do - -### 1. Ship Multiple Consensus Algorithms - -**Not at launch.** Maybe not ever. - -Shipping multiple algorithms means: -- Testing N² interactions (MVCC × algorithm permutations) -- Documenting N sets of operational behaviors -- Debugging production issues across N state machines -- Maintaining compatibility as algorithms evolve - -**Cost >> benefit** for a young database. - -If Raft proves inadequate (unlikely, given etcd/CockroachDB/TiKV operate at scale), we revisit. But the interface is ready. - -### 2. Multi-Raft by Default - -**Cloud9 uses multi-Raft** (each range = separate Raft group), but this isn't a feature users configure. It's an implementation detail. - -**Why multi-Raft?** -- Scale: Single Raft group limits throughput (leader bottleneck) -- Geo-distribution: Different ranges can have replicas in different regions -- Load balancing: Spread leadership across nodes - -**Users don't care.** They configure ranges (via SQL schema). The system maps ranges to Raft groups internally. - -### 3. Real-Time Reconfiguration Guarantees - -Raft's joint consensus (§6) ensures **safety** during configuration changes: no split-brain, no data loss. - -What Raft doesn't guarantee: **zero downtime** for arbitrary changes. - -**Example**: If you remove 2 replicas from a 3-replica range simultaneously, the range becomes unavailable (no quorum). This is correct behavior (you violated quorum math), not a Raft bug. - -**Cloud9's position**: Provide operator guardrails (warnings, confirmation prompts) but don't prevent valid operations. If an operator says "drain this node NOW," we do it, even if it breaks quorum. Better a deliberate outage than a stuck operator. - -## Addressing the "Pluggable Consensus" Critique - -**Concern**: "Isn't 'extensible consensus' just resume-driven development? Adding complexity to claim buzzword compliance?" - -**Answer**: No. Here's why: - -### What Cloud9 Is NOT Doing - -- Shipping multiple consensus algorithms at launch -- Promising "swap Raft for Paxos in production" -- Building abstractions for hypothetical future algorithms -- Adding configuration options users must understand - -**None of these exist.** - -### What Cloud9 IS Doing - -**Clean separation of concerns:** -- The consensus driver replicates a log -- The state machine applies committed entries -- The interface between them is narrow and testable - -**This isn't about extensibility for extensibility's sake.** It's about not letting Raft internals leak into transaction logic. - -### The Analogy - -Consider a database storage engine: -```rust -trait StorageEngine { - fn put(&mut self, key: Key, value: Value); - fn get(&self, key: Key) -> Option; -} -``` - -This doesn't mean "we ship 5 storage engines." It means: -- The SQL layer doesn't hardcode RocksDB calls -- Testing doesn't require spinning up RocksDB -- If RocksDB has a critical bug, swapping it isn't a SQL-layer rewrite - -**Same principle for consensus.** The interface isn't for users. It's for maintainability. - -## Why This Matters for Cloud9 - -Cloud9's goal: **daily driver database with zero surprises.** - -Raft achieves this because: -- It's proven (safety, liveness) -- It's understandable (operators can reason about it) -- It's flexible (leadership transfer, witnesses, learners) - -The consensus driver interface achieves: -- Testability (mock consensus for unit tests) -- Debuggability (isolate consensus bugs from transaction bugs) -- Evolvability (if needed, but not at launch) - -**Not resume padding. Not pluggability theater. Just clean architecture that happens to enable future evolution if research advances.** - -## Future: If Research Produces Better Algorithms - -**Hypothetical**: A new consensus algorithm emerges with: -- Formal verification (Coq/TLA+ proof) -- Production deployment at scale (5+ years, multiple orgs) -- Clear operational advantages (lower latency / higher throughput / better availability) - -**Then**: The consensus driver interface provides a migration path. - -**Migration strategy**: -1. Implement new algorithm behind ConsensusDriver trait -2. Test exhaustively (months, not weeks) -3. Deploy on non-critical ranges (logs, analytics) -4. Migrate critical ranges only after proving stability -5. Maintain Raft as fallback for years - -**This is a "someday, maybe" scenario.** Raft is good enough for etcd (Kubernetes control plane), CockroachDB (banks), and TiKV (PingCAP's flagship). It's good enough for Cloud9. - -## Comparison to Other Databases - -**CockroachDB**: Uses Raft, but Raft is deeply integrated into the KV layer. No clean interface. - -**TiDB**: Uses Raft (via TiKV), also tightly coupled. Adding operational features (witnesses) requires TiKV changes. - -**Spanner**: Uses Paxos, proprietary, not extensible. - -**YugabyteDB**: Uses Raft, but also has tablet-level coupling. - -**Cloud9**: Raft behind a clean interface. Same operational flexibility, better separation of concerns. - -## Conclusion - -**Cloud9 ships with Raft. Only Raft.** - -But Raft is the foundation for operational control: -- Dynamic leader placement (write latency optimization) -- Witness replicas (storage cost optimization) -- Learner replicas (safe reconfiguration) -- Per-range configuration (multi-tenant flexibility) - -The consensus driver interface isn't about swapping algorithms. It's about: -- Testing transaction logic without Raft -- Debugging consensus issues without understanding MVCC -- Maintaining a codebase where Raft concerns don't leak into SQL - -**This is clean architecture, not resume-driven development.** - -And if a better consensus algorithm emerges in 5 years? The interface is ready. But that's a decision for future Cloud9, after Raft proves insufficient. Which, given etcd/CockroachDB/TiKV, seems unlikely. - -**For now: One consensus algorithm, done right.** +- [In Search of an Understandable Consensus Algorithm](https://raft.github.io/raft.pdf) +- [Ongaro's Raft dissertation](https://github.com/ongardie/dissertation) +- [Raft resources](https://raft.github.io/) diff --git a/spec/11-indexes-schema.md b/spec/11-indexes-schema.md index 6df29fb..bae4a15 100644 --- a/spec/11-indexes-schema.md +++ b/spec/11-indexes-schema.md @@ -1,903 +1,176 @@ -# Indexes and Schema Management +# Catalogs, Schemas, and Projections -**Question**: How do we manage secondary indexes and schema changes in a distributed MVCC system with range sharding? +Cloud9 stores catalog state as versioned transactional metadata. A transaction +resolves data and metadata at one compatible snapshot. -**Answer**: Indexes are sharded key-value ranges with online backfills using fence timestamps. Schema changes are versioned, timestamped metadata that queries evaluate at read time. +Indexes are physical projections. They are not required to use one data +structure across every dialect. -## The Core Principle +## Catalog -In Cloud9, everything is an MVCC key-value range: -- Table data: `/table/{name}/{pk}/{col}@t → value` -- Secondary indexes: `/index/{name}/{indexed_col}/{pk}@t → ∅` -- Vector indexes: `/vector/{name}/{embedding_prefix}/{pk}@t → metadata` +The catalog records: -Indexes aren't separate systems. They're key prefixes that participate in the same sharding, replication, and transaction protocols as tables. +- namespaces and logical identities; +- source-dialect types and constraints; +- physical representations; +- indexes and projections; +- placement and residency policy; +- ownership and authorization; +- schema versions and compatibility; +- backfill and garbage-collection state. -**Result**: Indexes scale horizontally. Creating an index doesn't lock the table. Queries can use old schema during migration. +Catalog mutations use the same transaction protocol as data mutations. -## Why This Matters: Zero-Downtime DDL +## Schema Snapshots -Traditional databases lock tables during schema changes because schema is global state. Cloud9 versions schemas by timestamp, enabling operational capabilities that other databases can't match: +A transaction reads one catalog snapshot. Planning and execution use that +snapshot even when a newer schema commits concurrently. -**1. Rolling upgrades**: Old nodes query at old timestamps (old schema), new nodes query at new timestamps (new schema). Both can coexist in the same cluster, reading the same data with different schema interpretations. This eliminates the upgrade coordination problem that plagues traditional databases. +Metadata changes have explicit validity timestamps. A node cannot plan with +metadata it cannot interpret. -**2. Zero-downtime DDL**: `ALTER TABLE` writes a new schema version with a commit timestamp—no locks acquired, no data rewritten. Queries started before the DDL commit see the old schema. Queries started after see the new schema. Both execute concurrently against the same underlying MVCC key-value ranges. +Historical data is readable only when the required schema and physical decoder +remain available. -**3. Time-travel on schema**: Query data "as of" any timestamp, including historical schemas. `SELECT * FROM users AS OF TIMESTAMP '2024-01-15 10:00:00'` reconstructs not just the data state but the schema state at that moment. This enables reproducible debugging and compliance auditing. +## Physical Projections -**Why Postgres can't do this**: Postgres has MVCC for table data but treats schema metadata as global state. `ALTER TABLE users ADD COLUMN` acquires an `AccessExclusiveLock` on the table and updates system catalogs (`pg_class`, `pg_attribute`) atomically across all nodes. Even with MVCC protecting concurrent reads of user data, the schema change itself requires a barrier where all nodes observe the same catalog version. This is why Postgres major version upgrades require `pg_upgrade` and downtime. +A logical dataset may have several representations: -**Cloud9's advantage**: Schema is versioned metadata replicated via Raft, not a global lock-protected catalog. DDL operations are timestamped writes to the metadata keyspace. Queries pick their schema version based on transaction start timestamp. The same separation that makes lock-free reads possible for data makes lock-free schema evolution possible for DDL. +- row storage for relational writes; +- point storage for key-value access; +- document indexes for nested paths; +- object metadata and extent indexes; +- columnar projections for analytical scans; +- vector indexes for approximate search. -This is the missing piece that prevents existing databases from supporting true zero-downtime upgrades. +The catalog marks each representation as authoritative or derived. Derived +state includes a freshness watermark and rebuild procedure. -## Secondary Indexes +Dropping the authoritative representation requires an explicit migration that +first establishes another authority. -### Index Key Encoding +## Index Semantics -**Local (non-unique) index**: -``` -/index/{index_name}/{indexed_value}/{pk}@version → ∅ -``` - -**Example**: -```sql -CREATE TABLE users ( - id INT PRIMARY KEY, - email TEXT, - created_at TIMESTAMP -); - -CREATE INDEX users_email_idx ON users(email); -``` - -**Physical layout**: -``` -Table: - /table/users/42/email@100 → "alice@example.com" - /table/users/42/created_at@100 → "2024-01-15T10:00:00Z" - -Index: - /index/users_email_idx/alice@example.com/42@100 → ∅ -``` - -**Key insight**: The primary key is part of the index key. This ensures uniqueness (multiple users with same email) and enables direct lookups without storing the full row. - -**Why `→ ∅` (empty value)**: The index entry is a pointer. The actual data lives in the table. During query execution, we: -1. Scan index: `/index/users_email_idx/{email}/` → list of PKs -2. Lookup table: `/table/users/{pk}/` → full rows - -**Global (unique) index**: -``` -/index/{index_name}/{indexed_value}@version → primary_key -``` - -**Example**: -```sql -CREATE UNIQUE INDEX users_email_unique ON users(email); -``` - -**Physical layout**: -``` -/index/users_email_unique/alice@example.com@100 → 42 -``` - -**Uniqueness enforcement**: Before inserting, check if index key exists. If so, abort (duplicate). This check happens within the transaction (MVCC semantics apply). - -### Multi-Column Indexes - -**Schema**: -```sql -CREATE INDEX users_location_idx ON users(country, city); -``` - -**Encoding**: -``` -/index/users_location_idx/{country}/{city}/{pk}@version → ∅ -``` - -**Example**: -``` -/index/users_location_idx/US/Seattle/42@100 → ∅ -/index/users_location_idx/US/Seattle/99@100 → ∅ -/index/users_location_idx/US/Portland/55@100 → ∅ -``` - -**Query optimization**: -```sql -SELECT * FROM users WHERE country = 'US' AND city = 'Seattle'; -``` -→ Scan `/index/users_location_idx/US/Seattle/` (prefix scan) - -```sql -SELECT * FROM users WHERE country = 'US'; -``` -→ Scan `/index/users_location_idx/US/` (partial prefix scan) - -```sql -SELECT * FROM users WHERE city = 'Seattle'; -``` -→ Cannot use index (prefix doesn't match). Full table scan or different index. - -**Ordering matters**: Most selective column should be first (generally). - -### Covering Indexes - -**Problem**: Index scan → table lookup adds latency. If query only needs indexed columns, avoid table lookup. - -**Solution**: Store additional columns in index value. - -**Schema**: -```sql -CREATE INDEX users_email_covering ON users(email) INCLUDE (created_at); -``` - -**Encoding**: -``` -/index/users_email_covering/{email}/{pk}@version → CBOR({created_at: ...}) -``` - -**Query**: -```sql -SELECT email, created_at FROM users WHERE email = 'alice@example.com'; -``` -→ Index scan returns `{email, created_at}`. No table lookup needed. - -**Trade-off**: Index size increases (storing extra data). Worth it for hot query patterns. +An index descriptor states: -## Local vs Global Indexes +- source fields or expressions; +- key encoding and collation; +- uniqueness; +- null and missing-value behavior; +- predicate and partial-index rules; +- physical dialect; +- placement; +- lifecycle state; +- freshness watermark. -**Local index**: Partitioned alongside table data. -- Index entries for range `/table/users/[0, 1000)/` live on same Raft group -- Co-location enables single-range transactions (fast) -- Range scans may require cross-shard scatter-gather +The optimizer may select an index only when its semantics cover the source +operation. -**Global index**: Independently sharded by indexed column. -- Index entries sharded by indexed value (e.g., email), not by PK -- Writes require cross-shard transactions (index range ≠ table range) -- Range scans are local to index shard (fast for queries, slow for writes) +Unique constraints use transactional reservations or equivalent serializable +validation. A local existence check is insufficient for a global unique index. -**Cloud9's default**: Local indexes (CockroachDB model). +## Online Backfill -**When to use global**: -- Heavily skewed access patterns (e.g., `WHERE email = X` queries common, email values scattered) -- Willing to pay cross-shard transaction cost on writes +An index or projection follows a fenced lifecycle: -**Configuration**: -```sql -CREATE INDEX users_email_idx ON users(email) LOCAL; -- Default -CREATE INDEX users_email_idx ON users(email) GLOBAL; -- Explicit +```text +Declared -> Backfilling -> Validating -> Readable -> Retiring ``` -### Local Index Implementation +Creation proceeds as follows: -**Key encoding includes shard hint**: -``` -/table/users/{pk}/... → Table row -/index/users_email_idx/{pk}/{email}/... → Local index entry -``` +1. Commit the descriptor with fence timestamp `t_fence`. +2. Make writes at or after `t_fence` maintain the new projection. +3. Scan the authoritative representation at `t_fence`. +4. Write missing projection entries idempotently. +5. Catch up through a recorded high-water mark. +6. Validate contents against the authoritative representation. +7. Commit the `Readable` state and publication timestamp. -**Why `{pk}` first in index key**: Ensures index entry co-locates with table row. Both hash to same range. +Queries cannot use the projection before publication. A failed backfill stays +unreadable and resumes from durable checkpoints. -**Downside**: Query `WHERE email = 'alice@example.com'` must scan all ranges (scatter-gather). Mitigated by: -- Range caching (hot ranges stay local) -- Parallel scatter-gather (coordinated at SQL layer) +## Concurrent Writes -### Global Index Implementation +Backfill and foreground writes may race on one logical item. Projection writes +therefore carry source version information. -**Key encoding by indexed column**: -``` -/table/users/{pk}/... → Table row (sharded by PK) -/index/users_email_idx/{email}/{pk}/... → Global index entry (sharded by email) -``` +An older backfill result cannot replace an entry produced by a newer +transaction. Deletes create projection tombstones where needed. -**Write path**: -``` -INSERT INTO users (id, email) VALUES (42, 'alice@example.com'); -→ Cross-shard transaction: - 1. Write to table range (PK=42 → range A) - 2. Write to index range (email=alice → range B) - 3. 2PC commit across ranges A and B -``` +## Schema Changes -**Read path**: -```sql -SELECT * FROM users WHERE email = 'alice@example.com'; -→ Single-shard index scan: - 1. Scan /index/users_email_idx/alice@example.com/ (range B, local) - 2. Lookup PKs in table (range A, may be remote) -``` - -**Trade-off**: Writes pay 2PC cost. Reads benefit from index locality. +Schema changes fall into three classes: -## Online Index Backfills +- metadata-only changes; +- changes that require validation; +- changes that require a physical rewrite. -**Problem**: Creating an index on existing table requires scanning all rows. Blocking writes during backfill is unacceptable. +The planner declares the class before execution. -**Solution**: Fence timestamp with incremental backfill. +Adding a nullable field may be metadata-only. Adding a validated constraint +requires a scan. Changing an incompatible physical type requires a new +representation and migration. -### The Protocol +Cloud9 does not label a rewrite as metadata-only to avoid operational cost. -**Phases**: -1. **Schema registration** (instant): Add index metadata to catalog, mark as `BACKFILLING` -2. **Backfill** (background): Scan table, write index entries, track progress -3. **Validation** (fast): Verify no concurrent writes were missed -4. **Activation** (instant): Mark index as `PUBLIC`, queries start using it +## Cross-Dialect Mappings -### Fence Timestamp Mechanism +A mapping between SQL, key-value, document, object, or analytical dialects +defines: -**Fence timestamp `t_fence`**: The moment when index creation begins. +- shared logical identity; +- type conversion; +- null, missing, and default behavior; +- version and conditional-write semantics; +- authoritative representation; +- projection freshness; +- unsupported source operations. -**Invariant**: -- Writes with `t_w < t_fence`: Not automatically indexed (backfill handles them) -- Writes with `t_w ≥ t_fence`: Automatically indexed (transaction includes index write) - -**Implementation**: -```rust -struct IndexMetadata { - index_id: IndexID, - table_id: TableID, - status: IndexStatus, - fence_timestamp: Timestamp, // Set at backfill start -} - -enum IndexStatus { - Backfilling, // Schema registered, backfill in progress - Validating, // Backfill complete, verifying consistency - Public, // Index ready for queries -} -``` - -### Backfill Algorithm - -``` -1. Coordinator starts backfill at t_fence = now() -2. Update catalog: index status = BACKFILLING, fence = t_fence -3. For each range in table: - a. Scan rows at snapshot timestamp t_fence - b. For each row: - - Compute index key from indexed columns - - Write index entry with timestamp t_fence - c. Checkpoint progress -4. Once all ranges scanned: - - Set status = VALIDATING - - Check for concurrent writes in range [t_fence, now()] - - If any writes without index entries: retry -5. Set status = PUBLIC -6. Index is live -``` - -**Key properties**: -- Backfill reads at `t_fence` (consistent snapshot, no locks) -- Concurrent writes at `t > t_fence` automatically include index entries -- No gap: every row is either backfilled or indexed by transaction - -### Concurrent Write Handling - -**Scenario**: -``` -t=100: Start backfill (t_fence = 100) -t=105: Backfill reads row PK=42 (email=alice@example.com), writes index entry -t=110: Concurrent UPDATE: row PK=42 email → bob@example.com -t=120: Backfill completes -``` - -**What happens**: -``` -At t=110, the UPDATE transaction checks catalog: - - Index exists, status = BACKFILLING - - t_w (110) ≥ t_fence (100) - → Write index entries for BOTH old and new values: - - Delete: /index/users_email_idx/alice@example.com/42@110 - - Insert: /index/users_email_idx/bob@example.com/42@110 -``` - -**Result**: Index remains consistent. No missing entries. - -### Validation Phase - -**Why needed**: Ensure backfill didn't miss any writes due to race conditions. - -**Algorithm**: -```rust -fn validate_backfill(index: &Index, t_fence: Timestamp) -> Result<()> { - let now = hlc.now(); - - // Check all writes in [t_fence, now) - for txn in storage.transactions_in_range(t_fence, now) { - for write in txn.writes { - if write.table_id == index.table_id { - // Verify corresponding index entry exists - let index_key = compute_index_key(&index, &write); - let entry = storage.get(index_key, txn.commit_ts); - if entry.is_none() { - return Err(ValidationError::MissingIndexEntry); - } - } - } - } - - Ok(()) -} -``` - -**If validation fails**: Retry backfill for affected rows. - -**Typical duration**: Milliseconds (checking small transaction window). - -### Incremental Backfill Checkpointing - -**Problem**: Backfilling large table takes hours. Coordinator crashes mid-backfill. - -**Solution**: Checkpoint progress, resume from last checkpoint. - -**Metadata**: -```rust -struct BackfillProgress { - index_id: IndexID, - completed_ranges: Vec, - current_range: RangeID, - last_key: Key, // Resume from here -} -``` - -**Resume protocol**: -``` -1. Coordinator recovers, sees index status = BACKFILLING -2. Load BackfillProgress from catalog -3. Skip completed_ranges -4. Resume current_range from last_key -5. Continue backfill -``` - -**Checkpoint frequency**: Every 1M rows or 60 seconds (configurable). +Mappings are versioned catalog objects. A conversion that loses required +semantics is illegal. ## Vector Indexes -Vector similarity search (embeddings, RAG applications) requires specialized indexes. Cloud9 supports HNSW (Hierarchical Navigable Small World) and IVF-PQ (Inverted File with Product Quantization). - -### Key Insight: Vector Indexes Are Sharded Indexes - -Unlike relational indexes (exact match), vector indexes perform approximate nearest neighbor (ANN) search. But they still live in the same MVCC key space. - -**Encoding**: -``` -/vector/{index_name}/hnsw/{layer}/{node_id}@version → neighbors -/vector/{index_name}/ivf/{cluster_id}/{vector_id}@version → quantized_embedding -``` - -### HNSW Index - -**Structure**: Multi-layer graph where each node is a vector. Layers form increasingly sparse skip lists. - -**Schema**: -```sql -CREATE TABLE documents ( - id UUID PRIMARY KEY, - content TEXT, - embedding VECTOR(768) -- Embedding dimension -); - -CREATE INDEX documents_embedding_hnsw ON documents - USING hnsw(embedding) - WITH (m = 16, ef_construction = 200); -``` - -**Parameters**: -- `m`: Max edges per node (connectivity) -- `ef_construction`: Search breadth during construction - -**Physical layout**: -``` -Table: - /table/documents/{uuid}/content@t → "..." - /table/documents/{uuid}/embedding@t → BLOB(float32[768]) - -HNSW index (layer 0, densest): - /vector/documents_embedding_hnsw/hnsw/0/node_A@t → {neighbors: [B, C, D], embedding: ...} - /vector/documents_embedding_hnsw/hnsw/0/node_B@t → {neighbors: [A, E], embedding: ...} - -HNSW index (layer 1, sparser): - /vector/documents_embedding_hnsw/hnsw/1/node_X@t → {neighbors: [Y], embedding: ...} -``` - -**Query**: -```sql -SELECT id, content -FROM documents -ORDER BY embedding <-> '[0.1, 0.2, ...]' -LIMIT 10; -``` - -**Execution**: -1. Parse embedding from query -2. Start at top HNSW layer, find entry node -3. Greedy search through graph layers (navigate to nearest neighbors) -4. Reach layer 0, collect k nearest nodes -5. Lookup table rows for node IDs -6. Return results - -**MVCC semantics**: HNSW nodes are versioned. Query at `t_r` sees graph structure as of `t_r`. Concurrent inserts write new nodes at higher timestamps (invisible to past readers). - -### IVF-PQ Index - -**Structure**: Inverted file index with product quantization (compression). - -**Schema**: -```sql -CREATE INDEX documents_embedding_ivf ON documents - USING ivf_pq(embedding) - WITH (clusters = 1024, subvectors = 8); -``` - -**Parameters**: -- `clusters`: Number of Voronoi cells (IVF buckets) -- `subvectors`: Product quantization splits (compression level) - -**Physical layout**: -``` -Cluster centroids: - /vector/documents_embedding_ivf/ivf/centroids@t → BLOB(float32[1024][768]) - -Inverted lists: - /vector/documents_embedding_ivf/ivf/cluster_0/{uuid}@t → quantized_embedding - /vector/documents_embedding_ivf/ivf/cluster_1/{uuid}@t → quantized_embedding - ... -``` - -**Query**: -1. Find nearest cluster centroids (typically scan top 10-100 clusters) -2. Scan inverted lists for those clusters -3. Compute approximate distances using quantized embeddings -4. Return top k results - -**Sharding**: Clusters shard independently. Query coordinator scans top clusters in parallel. - -### Vector Index Backfills - -**Challenge**: HNSW/IVF construction is computationally expensive (graph building, clustering, quantization). - -**Solution**: Offline construction with fence timestamp. - -**Protocol**: -1. Mark index as `BACKFILLING` at `t_fence` -2. Snapshot all embeddings at `t_fence` -3. Build HNSW graph / cluster IVF offline (hours for large datasets) -4. Write constructed index entries in batches -5. Validate: check writes in `[t_fence, now)` were indexed -6. Activate index - -**Concurrent writes**: New vectors during backfill are indexed incrementally (inserted into partial graph/clusters). Graph may be suboptimal until next rebuild. - -**Optimization**: Periodic rebuild for hot indexes (reclustering, graph optimization). - -## Schema Changes (Online DDL) - -Schema = table definitions, column types, constraints, indexes. Cloud9 treats schema as versioned metadata. - -### Timestamped Schema Evolution - -**Core idea**: Schema changes don't rewrite data. They create new schema versions. Queries evaluate schema based on their read timestamp. - -**Example**: -```sql --- t=100: Initial schema -CREATE TABLE products ( - id INT PRIMARY KEY, - name TEXT -); - --- t=200: Add column -ALTER TABLE products ADD COLUMN price DECIMAL; - --- t=300: Query at t=150 (before ALTER) -SELECT * FROM products; -- Sees schema v1 (no price column) - --- t=300: Query at t=250 (after ALTER) -SELECT * FROM products; -- Sees schema v2 (includes price column) -``` - -### Schema Versioning - -**Catalog structure**: -```rust -struct TableSchema { - table_id: TableID, - version: u32, - valid_from: Timestamp, // When this version became active - valid_to: Option, // When superseded (None = current) - columns: Vec, - indexes: Vec, - constraints: Vec, -} -``` - -**Multiple versions coexist**: -``` -Table "products": - - Schema v1: valid [0, 200), columns: [id, name] - - Schema v2: valid [200, ∞), columns: [id, name, price] -``` - -**Query protocol**: -```rust -fn get_schema_at_timestamp(table_id: TableID, ts: Timestamp) -> TableSchema { - catalog.get_version(table_id) - .filter(|v| v.valid_from <= ts && v.valid_to > ts) - .unwrap() -} -``` - -### ADD COLUMN - -**Operation**: -```sql -ALTER TABLE products ADD COLUMN price DECIMAL DEFAULT 0.0; -``` - -**Protocol**: -``` -1. Coordinator picks t_schema = now() -2. Create schema v2 with new column: - - valid_from = t_schema - - columns += {price: DECIMAL, default: 0.0} -3. Write schema v2 to catalog (replicated via Raft) -4. Mark schema v1: valid_to = t_schema -5. Acknowledge client -``` - -**No data rewrite**: Existing rows don't gain a `price` column. Instead: -- Queries at `t < t_schema`: Use schema v1, don't project `price` -- Queries at `t ≥ t_schema`: Use schema v2, apply default value `0.0` if column missing - -**Physical layout**: -``` -Before: - /table/products/42/name@50 → "Widget" - -After ALTER at t=200: - /table/products/42/name@50 → "Widget" (unchanged) - -Query at t=250: - SELECT id, name, price FROM products WHERE id = 42; - → Read: /table/products/42/*@250 - → Sees: {name: "Widget"} - → Schema v2 fills default: {name: "Widget", price: 0.0} -``` - -**First write with new schema**: -```sql --- t=300 -UPDATE products SET price = 9.99 WHERE id = 42; -→ Writes: /table/products/42/price@300 → 9.99 -``` - -**Subsequent queries**: -```sql --- t=400 -SELECT * FROM products WHERE id = 42; -→ Reads: /table/products/42/name@50 → "Widget" -→ Reads: /table/products/42/price@300 → 9.99 -→ Returns: {id: 42, name: "Widget", price: 9.99} -``` - -### DROP COLUMN - -**Operation**: -```sql -ALTER TABLE products DROP COLUMN description; -``` - -**Protocol**: -``` -1. Coordinator picks t_schema = now() -2. Create schema v3 without column: - - valid_from = t_schema - - columns -= {description} -3. Write schema v3 to catalog -4. Mark schema v2: valid_to = t_schema -5. Acknowledge client -``` - -**Data retention**: Old column values remain in storage (MVCC). Queries at `t ≥ t_schema` don't project them. - -**Garbage collection**: Eventual compaction removes dropped columns for rows with no active snapshots below `t_schema`. - -### RENAME COLUMN - -**Operation**: -```sql -ALTER TABLE products RENAME COLUMN name TO product_name; -``` - -**Protocol**: -``` -1. Create schema v4 with renamed column: - - columns: [id, product_name, price] - - Add mapping: product_name → physical key "name" -2. Write schema v4 to catalog -``` - -**Physical layout unchanged**: Keys still use `/table/products/{id}/name@t`. Schema layer maps `product_name` → `name` at query time. - -**Why**: Avoid rewriting all keys (expensive). Logical rename is sufficient. - -### ALTER COLUMN TYPE - -**Problem**: Changing column type requires rewriting values (e.g., `INT → BIGINT`, `TEXT → JSON`). - -**Protocol**: -```sql -ALTER TABLE products ALTER COLUMN price TYPE NUMERIC(10,2); -``` - -**Two modes**: - -**Mode 1: Compatible type change** (no rewrite): -- `INT → BIGINT`: Just widen reads -- `VARCHAR(50) → VARCHAR(100)`: No storage change -- **Protocol**: Create new schema version with updated type. Queries cast on read. - -**Mode 2: Incompatible type change** (requires rewrite): -- `TEXT → JSON`: Parse required -- `INT → ENUM`: Validation required -- **Protocol**: - 1. Create new column with target type (`price_new NUMERIC`) - 2. Backfill: copy and convert data - 3. Drop old column, rename new column - 4. Validate: check all rows converted - -**Cloud9's approach**: Default to Mode 1 (lazy casting). Mode 2 requires explicit `USING` clause: -```sql -ALTER TABLE products ALTER COLUMN price TYPE JSON USING price::JSON; -``` - -### Schema Change Transactions - -**Question**: Can a transaction span schema changes? - -**Answer**: Yes, but with constraints. - -**Scenario**: -``` -T1 at t=150: Reads products table (schema v1, no price column) -T2 at t=200: Executes ALTER TABLE ... ADD COLUMN price -T1 at t=210: Continues, tries to read price column -``` - -**Conflict**: T1 started before schema change but tries to use new schema. - -**Resolution**: Transaction's schema is locked at start timestamp. - -**Rule**: Transaction T with `t_start` uses schema valid at `t_start`. Schema changes at `t > t_start` are invisible to T. - -**Implementation**: -```rust -struct Transaction { - txn_id: TxnID, - start_ts: Timestamp, - schema_snapshot: HashMap, // Captured at start -} - -fn execute_query(txn: &Transaction, query: &Query) { - let schema = txn.schema_snapshot.get(&query.table_id); - // Use this schema, ignore newer versions -} -``` - -### Catalog Management - -**Catalog = metadata storage**: Tables, columns, indexes, constraints, users, permissions. - -**Storage**: Special key prefix `/catalog/` in same MVCC storage. - -**Encoding**: -``` -/catalog/tables/{table_id}@version → TableSchema -/catalog/indexes/{index_id}@version → IndexMetadata -/catalog/constraints/{constraint_id}@version → Constraint -``` - -**Replication**: Catalog keys replicate via Raft like any other key. Catalog updates are transactional. - -**Caching**: Nodes cache catalog entries. Invalidation on schema change (Raft replication includes invalidation message). - -**Bootstrapping**: Initial catalog (system tables) created at cluster init: -``` -/catalog/tables/0@0 → Schema(pg_tables) -/catalog/tables/1@0 → Schema(pg_indexes) -/catalog/tables/2@0 → Schema(pg_catalog) -``` - -## KV → SQL Projections as Versioned Schema Mappings - -Recall from SQL-KV unification: KV namespaces can be queried via `KV()` virtual table. These projections are schema mappings. - -**Example**: -```sql -CREATE KV MAPPING product_cache ( - key TEXT, - value JSON ( - name TEXT, - price DECIMAL - ) -); -``` - -**Catalog entry**: -```rust -struct KVMapping { - mapping_id: MappingID, - namespace: String, - version: u32, - valid_from: Timestamp, - valid_to: Option, - schema: Vec, -} -``` - -**Versioned evolution**: -```sql --- v1 at t=100 -CREATE KV MAPPING product_cache_v1 ( - key TEXT, - value JSON (name TEXT, price_cents INT) -); - --- v2 at t=200 -CREATE KV MAPPING product_cache_v2 ( - key TEXT, - value JSON (name TEXT, price DECIMAL, currency TEXT) -); -``` - -**Query routing**: -```sql -SELECT * FROM KV('product_cache') WHERE key = 'prod_123'; -→ Executor checks mapping versions valid at query timestamp -→ Applies appropriate schema -``` - -**Union view** (common pattern): -```sql -CREATE VIEW products AS - SELECT key, name, price_cents / 100.0 AS price - FROM KV('product_cache') - WHERE value->>'version' = '1' - UNION ALL - SELECT key, name, price - FROM KV('product_cache') - WHERE value->>'version' = '2'; -``` - -**Benefit**: Multiple schema versions coexist. No data migration. Queries unify at runtime. - -## Performance Considerations - -### Index Maintenance Overhead - -**Write path**: Each INSERT/UPDATE/DELETE on indexed table requires: -- Write to table row: 1 key-value pair -- Write to each index: N key-value pairs (N = number of indexes) - -**Latency impact**: Local indexes co-locate with table (single range, fast). Global indexes require cross-shard transactions (2PC, slower). - -**Optimization**: Batch index writes within transaction. Cloud9 buffers all writes, commits in one Raft round. - -### Backfill Throttling - -**Problem**: Full-speed backfill saturates I/O, degrades query performance. - -**Solution**: Rate limiting. - -**Configuration**: -```sql -ALTER INDEX users_email_idx SET BACKFILL RATE LIMIT 1000 ROWS/SEC; -``` - -**Implementation**: Backfill coordinator sleeps between batches to respect rate limit. - -### Schema Cache Invalidation - -**Problem**: Nodes cache schema. Schema change requires invalidating all caches. Stampede on catalog range. - -**Solution**: Gossip-based invalidation. - -**Protocol**: -1. Coordinator writes new schema version to catalog -2. Coordinator broadcasts invalidation message via Raft heartbeat -3. Each node invalidates local cache on receiving message -4. Nodes lazily refetch schema on next query - -**Typical latency**: ~100ms (one Raft heartbeat interval). - -## Comparison to Other Databases - -### Spanner - -**Similarities**: -- Indexes are sharded tables -- Online schema changes (no blocking) -- MVCC semantics for indexes - -**Differences**: -- **Spanner**: No local indexes (all indexes global, cross-shard writes) -- **Cloud9**: Local indexes default (co-location benefits) -- **Spanner**: Schema changes are synchronous (TrueTime barrier) -- **Cloud9**: Schema changes are timestamped versions (MVCC) - -### CockroachDB - -**Similarities**: -- Local and global indexes -- Online backfills with checkpointing -- Fence timestamp mechanism -- Versioned schema - -**Differences**: -- **CockroachDB**: Interleaved tables (deprecated in v22.1) -- **Cloud9**: Explicit co-location via range configuration -- **CockroachDB**: Backfill runs on leaseholder (single-threaded bottleneck) -- **Cloud9**: Distributed backfill (parallel range scanning) - -### Postgres - -**Similarities**: -- Rich index types (B-tree, GiST, GIN) -- Covering indexes (`INCLUDE`) -- Online index creation (`CONCURRENTLY`) - -**Differences**: -- **Postgres**: Single-node, shared-buffer concurrency -- **Cloud9**: Distributed, MVCC across shards -- **Postgres**: VACUUM required for garbage collection -- **Cloud9**: MVCC compaction automatic (configurable) - -### DynamoDB +Vector indexes are physical projections with declared distance metrics and +recall behavior. Approximate search is valid only when the source operation +permits approximation. -**Similarities**: -- Global secondary indexes (GSI) are independently sharded -- Asynchronous index backfills +The descriptor records algorithm parameters, training data version, and +freshness. Rebuilds publish through the same fenced lifecycle. -**Differences**: -- **DynamoDB**: No local indexes for multi-column queries -- **Cloud9**: Rich local index support -- **DynamoDB**: GSI eventual consistency (separate table) -- **Cloud9**: Indexes transactional (same MVCC semantics) +## Cache Invalidation -## Implementation Checklist +Nodes cache catalog and plan state by version. A transaction pins the version +it planned against. -- [ ] Index key encoding (local and global) -- [ ] Secondary index write path (transaction integration) -- [ ] Secondary index read path (planner integration) -- [ ] Unique constraint enforcement -- [ ] Multi-column and covering indexes -- [ ] Online backfill coordinator -- [ ] Fence timestamp tracking -- [ ] Incremental backfill with checkpointing -- [ ] Validation phase (consistency check) -- [ ] HNSW vector index (graph structure) -- [ ] IVF-PQ vector index (clustering + quantization) -- [ ] Vector index query planner (ANN search) -- [ ] Schema versioning (catalog storage) -- [ ] ADD/DROP/RENAME COLUMN -- [ ] ALTER COLUMN TYPE (compatible and incompatible) -- [ ] Schema snapshot per transaction -- [ ] Catalog caching and invalidation -- [ ] KV mapping versioning -- [ ] Backfill throttling -- [ ] Metrics: index size, backfill progress, query selectivity +New metadata invalidates future planning. It does not mutate an in-flight plan. +Nodes that cannot read the required catalog version become unready for that +operation. -## Key Insights +## Garbage Collection -**Indexes are just sharded MVCC ranges.** They aren't special. They follow the same replication, sharding, and transaction protocols as tables. This uniformity simplifies the system and enables operational flexibility (move indexes independently, replicate differently, etc.). +Retired metadata and physical state remain until: -**Online DDL via timestamped schema.** Schema changes don't rewrite data. They create new metadata versions. Queries at different timestamps see different schemas. This enables zero-downtime migrations. +1. no active transaction can reference them; +2. retention and backup policy permits deletion; +3. all readers support the successor format; +4. rollback is no longer allowed; +5. dependent projections no longer refer to them. -**Fence timestamps enable lock-free backfills.** Concurrent writes during backfill are automatically indexed if they occur after the fence. No read locks, no write locks. Backfill can take hours without blocking traffic. +Deletion progress is durable and observable. -**Vector indexes fit the same model.** HNSW and IVF-PQ are complex structures, but they live in the MVCC key space. Queries see graph/cluster state as of their snapshot timestamp. This is the only way to make vector search transactional. +## Tests -**Schema versioning is schema-on-read.** Multiple schema versions coexist. KV mappings are versioned schemas for unstructured data. This is the bridge between "move fast with KV" and "lock down with SQL." +Catalog and projection tests cover: -Cloud9 doesn't invent new index structures. It applies MVCC and range sharding uniformly, making indexes operational first-class citizens—not bolted-on afterthoughts. +- schema snapshot consistency; +- legal and illegal type changes; +- unique conflicts across ranges; +- backfill races with inserts, updates, and deletes; +- crash recovery in every lifecycle state; +- stale projection rejection; +- cross-dialect mapping semantics; +- vector freshness and declared approximation; +- mixed-version metadata readers; +- garbage collection with active historical snapshots. diff --git a/spec/12-implementation-roadmap.md b/spec/12-implementation-roadmap.md index aef6927..876a80d 100644 --- a/spec/12-implementation-roadmap.md +++ b/spec/12-implementation-roadmap.md @@ -1,1254 +1,234 @@ # Implementation Roadmap -**Question**: How do we build Cloud9 incrementally with provable correctness at each stage? +Cloud9 ships by proving one correctness layer at a time. Later phases depend on +the invariants established earlier. -**Answer**: Six milestones with concrete deliverables and comprehensive testing gates. - -## Philosophy - -Build complexity incrementally. Each milestone must: -1. Add exactly one core capability -2. Prove correctness before proceeding -3. Maintain all previous guarantees -4. Ship production-quality tests - -No milestone is "done" until its test suite catches real bugs and survives adversarial conditions. - -## Testing Strategy Per Milestone - -Each milestone requires multiple testing levels: - -**Unit tests**: Fast, deterministic, cover individual components. - -**Property tests**: QuickCheck-style invariant checking with random inputs. - -**Integration tests**: Multi-component interaction under normal conditions. - -**Loom tests**: Concurrency verification with exhaustive thread interleaving. - -**Simulation tests**: Adversarial conditions (network partitions, crashes, clock skew) with deterministic seeds. - -**Jepsen-style tests**: Distributed system verification with real network partitions, node crashes, and linearizability checking. - -**Performance tests**: Regression detection, not optimization theater. Gate: "don't get slower without justification." - -## Milestone 1: Single-Node MVCC KV - -**Goal**: Prove we can build a correct transactional storage engine before adding distribution. - -**What ships**: -- MVCC key-value storage (versioned by timestamp) -- Write-ahead log (WAL) for durability -- Crash recovery (replay WAL to reconstruct state) -- Single-node transactions (no replication, no Raft) -- Snapshot isolation guarantee - -**What doesn't ship**: -- Replication -- Consensus -- Multi-node anything -- SQL (KV only) -- Distributed transactions - -**API surface**: -```rust -trait MVCCStorage { - fn write(&mut self, key: Key, value: Value, ts: Timestamp) -> Result<()>; - fn read(&self, key: Key, ts: Timestamp) -> Result>; - fn scan(&self, range: Range, ts: Timestamp) -> Result>; - fn begin_txn(&mut self) -> TxnHandle; - fn commit_txn(&mut self, txn: TxnHandle) -> Result; - fn abort_txn(&mut self, txn: TxnHandle); -} -``` - -**Storage layout**: -``` -key@timestamp -> value -key@100 -> "v1" -key@150 -> "v2" // Read at ts=120 returns "v1", ts=200 returns "v2" -``` - -**WAL format**: -``` -[BeginTxn(txn_id)] -[Write(txn_id, key, value, provisional_ts)] -[CommitTxn(txn_id, commit_ts)] -``` - -**Correctness invariants**: -1. Read at timestamp T returns most recent write with ts ≤ T -2. Committed writes survive crashes -3. Aborted transactions leave no visible state -4. Snapshot reads see consistent point-in-time view - -**Test requirements**: - -**Unit tests**: -- Write and read single key at multiple timestamps -- Read returns correct version for given timestamp -- Scan returns keys in order with correct versions -- Abort removes uncommitted writes - -**Property tests** (proptest/quickcheck): -```rust -#[test] -fn mvcc_snapshot_isolation(operations: Vec) { - let storage = MVCCStorage::new(); - let mut committed_state: BTreeMap> = BTreeMap::new(); - - for op in operations { - match op { - Write(key, value, ts) => { - storage.write(key, value, ts)?; - committed_state.entry(ts).or_default().insert(key, value); - } - Read(key, ts) => { - let result = storage.read(key, ts)?; - let expected = committed_state - .range(..=ts) - .rev() - .find_map(|(_, state)| state.get(&key)); - assert_eq!(result, expected); - } - } - } -} - -#[test] -fn mvcc_write_write_conflict() { - // Two concurrent writers to same key should serialize - // First writer wins, second detects conflict -} -``` - -**Crash recovery tests**: -```rust -#[test] -fn recover_from_crash_during_commit() { - let mut storage = MVCCStorage::new(); - storage.write(key, value, ts)?; - - // Simulate crash before commit completes - drop(storage); // Dirty shutdown, no flush - - // Recover and verify - let storage = MVCCStorage::recover()?; - assert_eq!(storage.read(key, ts)?, Some(value)); -} - -#[test] -fn replay_wal_reconstructs_state() { - // Write sequence: Begin, Write(k1), Write(k2), Commit - // Crash after commit but before state machine applies - // Replay WAL should reconstruct exact state -} -``` - -**Loom tests** (concurrency verification): -```rust -#[test] -fn concurrent_reads_and_writes() { - loom::model(|| { - let storage = Arc::new(MVCCStorage::new()); - - let writer = { - let storage = storage.clone(); - loom::thread::spawn(move || { - storage.write(key, value, ts)?; - }) - }; - - let reader = { - let storage = storage.clone(); - loom::thread::spawn(move || { - storage.read(key, ts)?; - }) - }; - - writer.join().unwrap(); - reader.join().unwrap(); - // Loom explores all possible interleavings - }); -} -``` - -**Gate**: Property tests must run 10,000+ random operation sequences without finding invariant violations. Loom tests must explore all thread interleavings without deadlocks or data races. - -## Milestone 2: Raft Replication - -**Goal**: Add consensus-based replication while maintaining single-range (no sharding) semantics. - -**What ships**: -- Raft consensus implementation (leader election, log replication, safety) -- Multi-replica MVCC storage (replicate writes to quorum) -- Leaseholder reads (linearizable reads from leader) -- Follower reads at closed timestamp (bounded-stale reads) -- Raft-driven WAL (consensus log becomes WAL) - -**What doesn't ship**: -- Multi-range sharding -- Cross-shard transactions -- SQL -- Timestamp oracle - -**Consensus Driver Interface**: -```rust -trait ConsensusDriver { - fn propose(&mut self, cmd: Command) -> Result; - fn poll_committed(&mut self) -> Vec; - fn leader(&self) -> Option; - fn transfer_leadership(&mut self, target: ReplicaId) -> Result<()>; -} -``` - -**Replicated state machine**: -``` -Raft log entry = MVCC operation (write/commit/abort) -Apply function = append to MVCC storage -``` - -**Closed timestamp protocol**: -```rust -struct ClosedTimestamp { - timestamp: Timestamp, - raft_index: LogIndex, -} - -impl RaftLeader { - fn advance_closed_timestamp(&mut self) { - let safe_ts = self.hlc.now() - self.max_clock_skew; - let min_active = self.active_txns.min_timestamp(); - self.closed_ts = Timestamp::min(safe_ts, min_active - 1); - self.broadcast_closed_ts(); // Piggyback on heartbeats - } -} - -impl RaftFollower { - fn can_serve_read(&self, read_ts: Timestamp) -> bool { - read_ts <= self.closed_ts && self.applied_index >= self.closed_index - } -} -``` - -**Correctness invariants**: -1. Quorum writes are durable (survive F failures in 2F+1 cluster) -2. Leader reads are linearizable -3. Follower reads see consistent snapshot at closed_ts -4. Log replication preserves order -5. Leadership changes don't lose committed data - -**Test requirements**: - -**Unit tests**: -- Leader election with 3/5/7 nodes -- Log replication to quorum -- Follower catches up after partition heals -- Closed timestamp advances correctly - -**Simulation tests** (deterministic with seeds): -```rust -#[test] -fn partition_minority_nodes() { - let mut sim = Simulation::new(seed); - let cluster = sim.create_cluster(5); - - // Partition 2 nodes away from 3-node majority - sim.partition(&[node1, node2], &[node3, node4, node5]); - - // Majority should elect leader and continue - sim.step_until(leader_elected); - assert!(cluster.majority().has_leader()); - - // Write should succeed (quorum available) - cluster.majority().write(key, value)?; - - // Heal partition - sim.heal(); - - // Minority nodes should catch up - sim.step_until(all_nodes_synced); - assert_eq!(cluster.all_nodes().read(key)?, Some(value)); -} - -#[test] -fn leader_crash_during_replication() { - // Leader proposes write, replicates to 1 follower, crashes - // New leader elected, should see write (was on quorum) -} - -#[test] -fn follower_reads_bounded_stale() { - // Leader writes at t=100, advances closed_ts=100 - // Follower sees closed_ts=100, serves read at ts=100 - // Verify read returns committed value -} -``` - -**Jepsen-style tests** (actual network, real crashes): -```rust -#[test] -fn jepsen_linearizability_check() { - let cluster = deploy_cluster(5); - - // Concurrent clients issuing writes - let mut history = vec![]; - for _ in 0..1000 { - let client_id = random_client(); - let op = random_write(); - let result = cluster.execute(op); - history.push((client_id, op, result)); - } - - // Inject faults - cluster.kill_random_node(); - cluster.partition_random_nodes(); - - // Check linearizability with Knossos/Elle - assert!(knossos::check_linearizable(&history)); -} -``` - -**Performance tests**: -```rust -#[test] -fn write_latency_regression() { - // Measure quorum write latency (RTT + replication) - // Gate: p50 < 10ms, p99 < 50ms (for local 3-node cluster) -} - -#[test] -fn follower_read_latency() { - // Gate: follower reads < 1ms when closed_ts current -} -``` - -**Gate**: Simulation tests must pass with 100 different random seeds. Jepsen tests must survive 10+ crash/partition scenarios without linearizability violations. - -## Milestone 3: Timestamp Oracle + Lock Manager - -**Goal**: Add external consistency primitives (HLC-based timestamping, commit-wait, write-write conflict detection). - -**What ships**: -- Hybrid Logical Clock (HLC) implementation -- Timestamp oracle (assign commit timestamps) -- Lock manager (detect write-write conflicts) -- Commit-wait protocol (ensure real-time order) -- Write-skew prevention (detect read-write conflicts) - -**What doesn't ship**: -- Cross-shard transactions (still single-range) -- SQL -- 2PC - -**HLC implementation**: -```rust -struct HybridLogicalClock { - physical: Timestamp, // Wall clock - logical: u64, // Counter for same physical time -} - -impl HybridLogicalClock { - fn now(&mut self) -> Timestamp { - let wall = system_time(); - if wall > self.physical { - self.physical = wall; - self.logical = 0; - } else { - self.logical += 1; - } - Timestamp::new(self.physical, self.logical) - } - - fn observe(&mut self, remote: Timestamp) { - let wall = system_time(); - self.physical = max(wall, remote.physical); - if self.physical == remote.physical { - self.logical = max(self.logical, remote.logical) + 1; - } else { - self.logical = 0; - } - } -} -``` - -**Commit-wait protocol**: -```rust -fn commit_transaction(txn: &Transaction, hlc: &HLC, epsilon: Duration) -> Result<()> { - let commit_ts = select_commit_timestamp(txn, hlc); - - // Write commit record to Raft log - replicate_commit(txn.txn_id, commit_ts)?; - - // Wait until all nodes' clocks > commit_ts - let target = commit_ts + epsilon; - while hlc.now() < target { - sleep(1ms); - } - - // Now safe: any future operation gets ts > commit_ts - Ok(()) -} -``` - -**Lock manager**: -```rust -struct LockManager { - locks: HashMap, -} - -struct Lock { - txn_id: TxnId, - timestamp: Timestamp, - mode: LockMode, // Shared | Exclusive -} - -impl LockManager { - fn acquire(&mut self, key: Key, txn_id: TxnId, mode: LockMode) -> Result<()> { - if let Some(existing) = self.locks.get(&key) { - if existing.txn_id != txn_id { - return Err(LockConflict); - } - } - self.locks.insert(key, Lock { txn_id, mode, ... }); - Ok(()) - } -} -``` - -**Write-skew detection**: -```rust -#[test] -fn prevent_write_skew() { - // Classic scenario: - // T1: read(x)=0, read(y)=0, write(x=1) - // T2: read(x)=0, read(y)=0, write(y=1) - // Without conflict detection: both commit (invariant x+y>0 violated) - - let storage = MVCCStorage::new(); - - let t1 = storage.begin_txn(); - assert_eq!(storage.read_in_txn(&t1, "x")?, 0); - assert_eq!(storage.read_in_txn(&t1, "y")?, 0); - storage.write_in_txn(&t1, "x", 1)?; - - let t2 = storage.begin_txn(); - assert_eq!(storage.read_in_txn(&t2, "x")?, 0); - assert_eq!(storage.read_in_txn(&t2, "y")?, 0); - storage.write_in_txn(&t2, "y", 1)?; - - // One must abort due to read-write conflict - let r1 = storage.commit_txn(t1); - let r2 = storage.commit_txn(t2); - assert!(r1.is_err() || r2.is_err()); -} -``` - -**Correctness invariants**: -1. HLC never goes backward -2. Commit-wait ensures ts_commit < ts_next_operation -3. Write-write conflicts abort one transaction -4. Serializable isolation (no anomalies: write-skew, dirty read, lost update) - -**Test requirements**: - -**Unit tests**: -- HLC monotonicity under concurrent updates -- Commit-wait duration equals epsilon -- Lock acquisition blocks conflicting transactions - -**Property tests**: -```rust -#[test] -fn hlc_causality(events: Vec<(NodeId, Event)>) { - // Events with happens-before relationship must have ts1 < ts2 - // Property: if event A sends message to B, ts_B > ts_A -} - -#[test] -fn serializable_isolation(txns: Vec) { - // Run concurrent transactions - // Verify committed state is equivalent to some serial order -} -``` - -**Anomaly tests** (based on Adya's formalization): -```rust -#[test] -fn no_g1a_aborted_reads() { - // T1 writes x, aborts - // T2 reads x (should not see T1's write) -} - -#[test] -fn no_g1b_intermediate_reads() { - // T1 writes x=1, then x=2, commits - // T2 should never read x=1 (intermediate value) -} - -#[test] -fn no_g1c_circular_information_flow() { - // T1 writes x, T2 writes y, each reads the other - // One must abort (no cycles in dependency graph) -} - -#[test] -fn no_g2_item_write_skew() { - // The classic write-skew scenario - // Prevented by tracking read-write dependencies -} -``` - -**Performance tests**: -```rust -#[test] -fn commit_wait_latency() { - // Gate: commit-wait adds ≤ epsilon latency (50-100ms on NTP, 10-50ms on PTP) -} - -#[test] -fn lock_manager_overhead() { - // Gate: lock acquisition < 10μs (in-memory hash lookup) -} -``` - -**Gate**: All Adya anomaly tests must pass. Property tests with 10,000+ random transaction schedules find zero serialization violations. - -## Milestone 4: Distributed Transactions (2PC) - -**Goal**: Enable transactions spanning multiple ranges with atomicity and external consistency. - -**What ships**: -- Two-phase commit protocol (prepare, commit/abort) -- Transaction coordinator (drives 2PC) -- Cross-shard intent handling (provisional writes) -- Intent resolution (convert to committed values) -- Transaction recovery (handle coordinator crash) - -**What doesn't ship**: -- Range splits/merges -- SQL -- Dynamic resharding - -**Transaction coordinator**: -```rust -struct TransactionCoordinator { - txn_id: TxnId, - participants: Vec, - state: TxnState, -} - -enum TxnState { - Active, - Preparing, - Prepared, - Committing, - Committed, - Aborted, -} - -impl TransactionCoordinator { - async fn execute_2pc(&mut self) -> Result<()> { - // Phase 0: Write intents - for participant in &self.participants { - participant.write_intent(self.txn_id, &self.writes)?; - } - - // Phase 1: Prepare - let commit_ts = self.select_commit_timestamp(); - let votes = self.send_prepare(commit_ts).await?; - - if votes.all(|v| v == Vote::Prepared) { - // Phase 2: Commit - self.send_commit(commit_ts).await?; - self.commit_wait(commit_ts).await; - Ok(()) - } else { - // Phase 2: Abort - self.send_abort().await?; - Err(TransactionAborted) - } - } -} -``` - -**Intent structure**: -```rust -struct Intent { - txn_id: TxnId, - key: Key, - value: Value, - provisional_ts: Timestamp, -} - -// On-disk layout during transaction: -// key -> Intent { txn_id, value, provisional_ts } -// key@50 -> CommittedValue { "old_value" } - -// After commit at ts=100: -// key@100 -> CommittedValue { "new_value" } -// key@50 -> CommittedValue { "old_value" } -``` - -**Transaction recovery**: -```rust -struct TransactionRecord { - txn_id: TxnId, - coordinator: NodeId, - participants: Vec, - commit_timestamp: Timestamp, - status: TxnStatus, - heartbeat: Timestamp, -} - -async fn recover_transaction(txn_id: TxnId) -> Result<()> { - let record = load_transaction_record(txn_id)?; - - if record.heartbeat_expired() { - if record.status == TxnStatus::Prepared { - // Coordinator crashed during commit - // Query participants to recover decision - let votes = query_participants(&record.participants).await?; - if votes.all(|v| v == Vote::Prepared) { - // All prepared, safe to commit - commit_transaction(txn_id, record.commit_timestamp).await?; - } else { - // At least one aborted, must abort - abort_transaction(txn_id).await?; - } - } else { - // Coordinator crashed before prepare, safe to abort - abort_transaction(txn_id).await?; - } - } - Ok(()) -} -``` - -**Correctness invariants**: -1. Atomicity: All participants commit or all abort -2. External consistency: Cross-shard commits respect real-time order -3. No partial visibility: Readers never see subset of transaction's writes -4. Crash recovery preserves atomicity - -**Test requirements**: - -**Unit tests**: -- 2PC with all participants voting prepared -- 2PC with one participant voting abort -- Intent resolution to committed values -- Transaction record persistence - -**Integration tests**: -```rust -#[test] -fn cross_shard_transaction() { - let cluster = Cluster::new(3); - let range_a = cluster.create_range(key_range("a".."m")); - let range_b = cluster.create_range(key_range("n".."z")); - - let txn = cluster.begin_transaction(); - txn.write("alice", "value1")?; // Range A - txn.write("zoe", "value2")?; // Range B - txn.commit()?; - - // Both writes visible or neither - let read_txn = cluster.begin_read_only_transaction(); - assert_eq!(read_txn.read("alice")?, Some("value1")); - assert_eq!(read_txn.read("zoe")?, Some("value2")); -} - -#[test] -fn partial_abort_rolls_back_all() { - // Write to 3 ranges, range 2 votes abort - // Verify range 1 and 3 don't show writes -} -``` - -**Simulation tests**: -```rust -#[test] -fn coordinator_crash_during_prepare() { - let mut sim = Simulation::new(seed); - let cluster = sim.create_cluster(5); - - let txn = cluster.begin_transaction(); - txn.write("key1", "value1")?; // Range A - txn.write("key2", "value2")?; // Range B - - // Crash coordinator after prepare sent, before commit - sim.inject_crash(cluster.coordinator(), after = "prepare_sent"); - - // New coordinator recovers transaction - sim.step_until(transaction_recovered); - - // Verify atomicity: either both visible or neither - let state = cluster.read_all_ranges(); - assert!( - (state.has("key1") && state.has("key2")) || - (!state.has("key1") && !state.has("key2")) - ); -} - -#[test] -fn participant_crash_during_commit() { - // Participant crashes after voting prepared, before applying commit - // New leader for that range should apply commit after recovery -} - -#[test] -fn intent_cleanup_on_abort() { - // Transaction aborts after writing intents - // Verify intents are cleaned up, not visible to readers -} -``` - -**Jepsen-style tests**: -```rust -#[test] -fn distributed_bank_test() { - // Classic Jepsen test: bank accounts across shards - // Invariant: sum(all_accounts) = constant - - let cluster = deploy_cluster(5); - let accounts = vec!["alice", "bob", "charlie"]; - let initial_sum = 1000; - - for account in &accounts { - cluster.write(account, initial_sum / accounts.len())?; - } - - // Concurrent transfers between accounts - for _ in 0..1000 { - let from = random_account(); - let to = random_account(); - let amount = random(1..100); - - cluster.transfer(from, to, amount)?; // Cross-shard transaction - - // Inject random faults - if random() { cluster.kill_random_node(); } - if random() { cluster.partition_random(); } - } - - // Check invariant - let final_sum = cluster.sum_all_accounts(); - assert_eq!(final_sum, initial_sum); -} -``` - -**Performance tests**: -```rust -#[test] -fn cross_shard_latency() { - // Gate: 2PC adds ≤ 2 RTT vs single-shard (prepare + commit) -} - -#[test] -fn intent_resolution_throughput() { - // Gate: resolve 10k+ intents/sec per node -} -``` - -**Gate**: Jepsen-style tests with 10,000+ cross-shard transactions survive random crashes/partitions without invariant violations. Transaction recovery handles all coordinator/participant crash scenarios. - -## Milestone 5: Range Splits and Merges - -**Goal**: Enable dynamic resharding under live traffic without downtime. - -**What ships**: -- Range split protocol (split one range into two) -- Range merge protocol (merge two ranges into one) -- Online rebalancing (move ranges between nodes) -- Split/merge under live traffic (no downtime) -- Load-based split triggers - -**What doesn't ship**: -- SQL (still KV only) -- Automatic rebalancing (manual only) - -**Range split protocol**: -```rust -struct Range { - id: RangeId, - key_range: (Key, Key), // [start, end) - replicas: Vec, - split_key: Option, // Pending split point -} - -async fn split_range(range_id: RangeId, split_key: Key) -> Result<(RangeId, RangeId)> { - // 1. Find split point (key that divides load roughly in half) - let split_key = find_split_key(&range_id)?; - - // 2. Freeze writes to range (brief quiesce) - freeze_range(range_id).await?; - - // 3. Create two new ranges - let left = create_range(range.start..split_key, range.replicas.clone())?; - let right = create_range(split_key..range.end, range.replicas.clone())?; - - // 4. Copy data to new ranges (scan + replicate) - copy_data(&range, &left, &right).await?; - - // 5. Update range registry (atomic switch) - update_range_registry(range_id, &[left, right]).await?; - - // 6. Resume writes (route to new ranges) - unfreeze_ranges(&[left, right]).await?; - - Ok((left.id, right.id)) -} -``` - -**Online rebalancing**: -```rust -async fn rebalance_range(range_id: RangeId, target_nodes: Vec) -> Result<()> { - // 1. Add new replicas as learners - for node in target_nodes { - add_replica(range_id, node, ReplicaRole::Learner).await?; - } - - // 2. Wait for learners to catch up - wait_for_catchup(range_id).await?; - - // 3. Promote learners to voters (joint consensus) - promote_replicas(range_id, &target_nodes).await?; - - // 4. Remove old replicas - remove_old_replicas(range_id).await?; - - Ok(()) -} -``` - -**Load-based split triggers**: -```rust -struct LoadMonitor { - range_id: RangeId, - qps: u64, // Queries per second - bytes_per_sec: u64, - cpu_percent: f64, -} - -impl LoadMonitor { - fn should_split(&self) -> bool { - self.qps > SPLIT_QPS_THRESHOLD || - self.bytes_per_sec > SPLIT_BYTES_THRESHOLD || - self.cpu_percent > SPLIT_CPU_THRESHOLD - } -} -``` - -**Correctness invariants**: -1. No data loss during split/merge -2. No downtime (writes continue during split) -3. Consistent routing (all clients see new ranges after split) -4. Cross-range transactions continue during split - -**Test requirements**: - -**Unit tests**: -- Split at various key boundaries -- Merge adjacent ranges -- Replica addition/removal -- Range registry updates - -**Integration tests**: -```rust -#[test] -fn split_range_under_load() { - let cluster = Cluster::new(3); - let range = cluster.create_range(key_range("a".."z")); - - // Write load to range - let writer = spawn_writer(&cluster, &range, qps = 1000); - - // Split range while writes continue - let (left, right) = cluster.split_range(range.id, "m").await?; - - // Verify no writes lost - let written_keys = writer.stop(); - for key in written_keys { - assert!(cluster.has_key(key)); - } -} - -#[test] -fn cross_range_transaction_during_split() { - // Start transaction writing to range A and B - // Split range A during transaction - // Verify transaction still commits atomically -} -``` - -**Simulation tests**: -```rust -#[test] -fn concurrent_splits_and_merges() { - let mut sim = Simulation::new(seed); - let cluster = sim.create_cluster(5); - - // Start with 10 ranges - let ranges = (0..10).map(|i| cluster.create_range(..)).collect(); - - // Concurrent operations - for _ in 0..100 { - match random_op() { - Op::Split => { - let range = random(&ranges); - cluster.split_range(range).await?; - } - Op::Merge => { - let (r1, r2) = random_adjacent_ranges(&ranges); - cluster.merge_ranges(r1, r2).await?; - } - Op::Write => { - cluster.write(random_key(), random_value())?; - } - } - } - - // Verify no data loss - sim.check_all_writes_visible(); -} - -#[test] -fn rebalance_during_partition() { - // Start rebalancing range to new nodes - // Partition network mid-rebalance - // Heal partition, verify rebalance completes or safely aborts -} -``` - -**Performance tests**: -```rust -#[test] -fn split_freeze_duration() { - // Gate: writes blocked < 100ms during split -} - -#[test] -fn rebalance_traffic_overhead() { - // Gate: rebalancing adds < 10% latency to foreground writes -} -``` - -**Gate**: Splits complete in < 1 second with < 100ms write freeze. No data loss in 100+ random split/merge/rebalance sequences. - -## Milestone 6: SQL + KV Unified Surface - -**Goal**: Ship the complete Cloud9 vision: SQL and KV under one transaction. - -**What ships**: -- SQL parser (PostgreSQL-compatible dialect) -- SQL planner (query to KV operations) -- SQL executor (scan, filter, join, aggregate) -- KV API (get/put/scan primitives) -- Cross-API transactions (SQL queries + KV operations in one transaction) -- Postgres wire protocol (pg clients connect directly) - -**What doesn't ship** (deferred to future): -- Vector indexing -- Automatic rebalancing -- Full Postgres feature parity (foreign keys, triggers, etc.) - -**Unified transaction API**: -```rust -trait Transaction { - // SQL operations - fn execute_sql(&self, query: &str) -> Result; - - // KV operations - fn get(&self, key: &[u8]) -> Result>>; - fn put(&self, key: &[u8], value: &[u8]) -> Result<()>; - fn scan(&self, range: Range<&[u8]>) -> Result, Vec)>>; - - fn commit(self) -> Result; - fn abort(self); -} -``` - -**Example: SQL + KV in one transaction**: -```sql -BEGIN; - -- SQL: Complex analytics query - SELECT user_id, COUNT(*) - FROM orders - WHERE amount > 1000 - GROUP BY user_id; - - -- KV: Fast state update - PUT('agent:state:123', state_blob); - - -- Cross-API join (this is the magic) - SELECT u.name, kv.session_data - FROM users u - JOIN kv_namespace('sessions') kv ON kv.user_id = u.id - WHERE kv.last_active > NOW() - INTERVAL '1 hour'; -COMMIT; -``` - -**SQL to KV lowering**: -```rust -struct QueryPlan { - operations: Vec, -} - -enum Operation { - Scan { range: Range, filter: Predicate }, - Get { key: Key }, - Put { key: Key, value: Value }, - Filter { predicate: Predicate }, - Join { left: Box, right: Box, condition: JoinCondition }, - Aggregate { group_by: Vec, aggregates: Vec }, -} - -fn plan_sql_query(sql: &str) -> Result { - // Parse SQL - let ast = parse_sql(sql)?; - - // Optimize - let optimized = optimize(ast)?; - - // Lower to KV operations - let plan = lower_to_kv(optimized)?; - - Ok(plan) -} -``` - -**KV namespace projection**: -```rust -// Define schema for KV namespace (typed projection) -CREATE TABLE sessions AS KV_NAMESPACE('sessions') ( - user_id UUID PRIMARY KEY, - session_data JSONB, - last_active TIMESTAMP -); - -// Now can join SQL tables with KV data -SELECT u.name, s.session_data -FROM users u -JOIN sessions s ON s.user_id = u.id; -``` - -**Postgres wire protocol**: -```rust -async fn handle_postgres_client(stream: TcpStream) -> Result<()> { - let mut conn = PostgresConnection::new(stream); - - // Handshake - conn.handshake().await?; - - // Query loop - loop { - match conn.read_message().await? { - PgMessage::Query(sql) => { - let result = execute_sql(&sql).await?; - conn.send_result(result).await?; - } - PgMessage::Parse(stmt) => { - let plan = parse_and_plan(&stmt).await?; - conn.cache_plan(plan); - } - PgMessage::Execute(plan_id) => { - let plan = conn.get_cached_plan(plan_id); - let result = execute_plan(plan).await?; - conn.send_result(result).await?; - } - PgMessage::Terminate => break, - } - } - - Ok(()) -} -``` - -**Correctness invariants**: -1. SQL queries see same snapshot as concurrent KV operations -2. Cross-API joins return consistent results -3. SQL transactions have same ACID guarantees as KV transactions -4. Postgres clients can't tell Cloud9 from real Postgres (compatibility) - -**Test requirements**: - -**Unit tests**: -- Parse SQL into AST -- Plan optimization (predicate pushdown, index selection) -- Lower SQL to KV operations -- Execute KV operations in transaction - -**Integration tests**: -```rust -#[test] -fn sql_kv_unified_transaction() { - let cluster = Cluster::new(3); - - // Create SQL table - cluster.execute_sql("CREATE TABLE users (id UUID, name TEXT)")?; - cluster.execute_sql("INSERT INTO users VALUES ('123', 'alice')")?; - - // Same transaction: SQL + KV - let txn = cluster.begin_transaction(); - - // SQL read - let result = txn.execute_sql("SELECT name FROM users WHERE id = '123'")?; - assert_eq!(result[0]["name"], "alice"); - - // KV write - txn.put(b"session:123", b"active")?; - - txn.commit()?; - - // Verify both visible - let new_txn = cluster.begin_read_only_transaction(); - assert_eq!(new_txn.execute_sql("SELECT name FROM users WHERE id = '123'")?, vec![...]); - assert_eq!(new_txn.get(b"session:123")?, Some(b"active")); -} - -#[test] -fn cross_api_join() { - let cluster = Cluster::new(3); - - // SQL table - cluster.execute_sql("CREATE TABLE users (id UUID, name TEXT)")?; - cluster.execute_sql("INSERT INTO users VALUES ('123', 'alice')")?; - - // KV namespace - cluster.put(b"sessions:123", b"session_data")?; - - // Define KV projection - cluster.execute_sql(r#" - CREATE TABLE sessions AS KV_NAMESPACE('sessions') ( - user_id UUID PRIMARY KEY, - data BYTEA - ) - "#)?; - - // Join SQL + KV - let result = cluster.execute_sql(r#" - SELECT u.name, s.data - FROM users u - JOIN sessions s ON s.user_id = u.id - "#)?; +Dates are not part of this specification. A phase is complete when its exit +tests pass. - assert_eq!(result.len(), 1); - assert_eq!(result[0]["name"], "alice"); -} -``` +## Current Foundation -**Compatibility tests** (PostgreSQL test suite): -```rust -#[test] -fn run_postgres_test_suite() { - // Run subset of PostgreSQL's regression tests - // Gate: 90%+ pass rate on core features (SELECT, JOIN, WHERE, GROUP BY) -} -``` +The repository currently contains: -**Performance tests**: -```rust -#[test] -fn sql_kv_parity() { - // Gate: KV operations accessed via SQL have < 10% overhead vs native KV API -} +- a pure Raft state machine; +- durable write-ahead log storage; +- a replicated key-value service; +- a process-level database interface; +- a Jepsen harness. -#[test] -fn cross_api_join_performance() { - // Gate: Join between SQL table and KV namespace < 2x slower than SQL-only join -} -``` +This foundation is not yet the complete database. MVCC, distributed +transactions, production bounded time, and the additional dialects remain +target work. -**Gate**: Postgres compatibility tests pass 90%+. Cross-API joins return correct results under concurrent writes. Postgres wire protocol compatible with psql, libpq, and popular ORMs (SQLAlchemy, Diesel, pgx). +## Phase 1: Replication Kernel -## Testing Infrastructure Requirements +Finish the smallest durable replicated state machine. -Each milestone requires these testing capabilities: +Deliver: -**Deterministic simulation framework**: -- Control time, network, and node crashes -- Reproducible with seeds -- Fast iteration (seconds, not minutes) +- snapshot installation and compaction; +- crash-safe log recovery; +- membership changes; +- deterministic application; +- explicit durability boundaries; +- metrics for term, commit index, apply index, and storage. -**Jepsen-style verification**: -- Real network, real crashes -- Linearizability checking (Knossos/Elle) -- History analysis (dependency graphs) +Exit tests: -**Property-based testing**: -- QuickCheck/proptest integration -- Invariant checking with random inputs -- Shrinking to minimal failing cases +- Raft model tests; +- crash and torn-write recovery; +- snapshot and log-prefix replacement; +- repeated leader changes; +- Jepsen linearizable register and key-value workloads. -**Loom for concurrency**: -- Exhaustive thread interleaving exploration -- Deadlock detection -- Data race detection +## Phase 2: Bounded Time -**Performance regression detection**: -- Automated benchmarking -- Statistical significance testing -- Alerts on regressions +Implement the `TimeSource` contract before transaction timestamps depend on it. -## Milestone Gates Summary +Deliver: -**Milestone 1**: Property tests find zero MVCC invariant violations in 10,000+ random operation sequences. +- typed time intervals and provider status; +- uncertainty policy; +- commit-wait primitive; +- deterministic mock provider; +- AWS ClockBound backend; +- startup and runtime capability checks. -**Milestone 2**: Simulation tests pass with 100 random seeds. Jepsen tests survive 10+ crash/partition scenarios. +Exit tests: -**Milestone 3**: All Adya anomaly tests pass. Zero serialization violations in 10,000+ transaction schedules. +- interval contract and arithmetic; +- provider loss and recovery; +- excessive uncertainty rejection; +- suspend, clock-step, and leap-state handling; +- hardware integration on supported EC2; +- no silent consistency-mode transition. -**Milestone 4**: Jepsen bank test passes 10,000+ cross-shard transactions with random faults. +## Phase 3: MVCC and Local Transactions -**Milestone 5**: Splits complete in < 1 second with < 100ms write freeze. Zero data loss in 100+ split/merge/rebalance sequences. +Build transaction semantics on one replicated range. -**Milestone 6**: Postgres compatibility 90%+. Cross-API joins correct under concurrent writes. +Deliver: -## Development Timeline Estimate +- versioned keys; +- snapshots and garbage collection; +- read-write conflict detection; +- atomic batches; +- retry identity; +- timestamp assignment and commit-wait; +- serializable local mode. -**Milestone 1**: 2-3 months (MVCC + WAL + crash recovery + property tests) +Exit tests: -**Milestone 2**: 3-4 months (Raft + replication + simulation tests) +- model-based transaction histories; +- write-write and read-write conflicts; +- crash recovery at every commit boundary; +- retry idempotency; +- snapshot retention and garbage collection; +- strict serializability with the bounded-time provider. -**Milestone 3**: 2-3 months (HLC + commit-wait + lock manager + anomaly tests) +## Phase 4: Ranges and Distributed Transactions -**Milestone 4**: 3-4 months (2PC + intent handling + recovery + Jepsen tests) +Partition the database without changing transaction semantics. -**Milestone 5**: 2-3 months (splits/merges + online rebalancing) +Deliver: -**Milestone 6**: 4-6 months (SQL parser + planner + executor + Postgres protocol) +- range metadata and routing; +- split, merge, and replica movement; +- distributed transaction records; +- two-phase commit across ranges; +- durable recovery of coordinator failure; +- safe-time tracking for follower reads. -**Total**: 16-23 months (assuming 1-2 engineers) +Exit tests: -This is aggressive but achievable with: -- Ruthless scope discipline (cut non-essential features) -- Heavy reuse (RocksDB for storage, existing SQL parser) -- Test-first development (bugs caught early, not in production) +- split and merge under load; +- participant and coordinator failure; +- ambiguous client outcomes; +- leader change during prepare and commit-wait; +- cross-range Jepsen transactions; +- locality and residency policy enforcement. -## Success Criteria +## Phase 5: Database IR -Cloud9 is production-ready when: +Establish the multi-level lowering pipeline. -1. **Correctness**: All test gates pass. Zero known correctness bugs. +Deliver: -2. **Performance**: Within 2x of CockroachDB on standard benchmarks (YCSB, TPC-C). +- versioned IR containers; +- types and effect declarations; +- Transaction IR; +- Placement IR; +- physical dialect interfaces; +- lowering legality checks; +- deterministic replication commands; +- plan tracing and validation. -3. **Stability**: Runs for 7+ days under load without crashes or leaks. +Exit tests: -4. **Usability**: Postgres clients connect and execute queries without code changes. +- legal and illegal conversion suites; +- optimizer equivalence tests; +- serialized IR compatibility; +- deterministic command generation; +- invariant validation after every pass. -5. **Documentation**: Every API has examples. Every failure mode has runbook. +## Phase 6: SQL and Key-Value Dialects -**Not** production-ready until all of these are true. No shortcuts. +Prove the IR with relational and key-value workloads. -## Principles +Deliver: -**Build depth, not breadth**: Better to have bulletproof MVCC than half-finished SQL + KV + vector. +- a SQL parser, catalog, planner, and wire protocol; +- DynamoDB-style item and conditional operations; +- row and point physical dialects; +- secondary indexes; +- explicit cross-dialect mappings; +- compatibility error models. -**Test like you'll regret not testing**: Every milestone gate must catch real bugs. If tests don't find issues, the tests are wrong. +Exit tests: -**Cut scope aggressively**: If a milestone is slipping, cut features, don't cut tests. +- SQL logic and transaction suites; +- key-value compatibility tests; +- differential tests against reference systems; +- cross-dialect transaction histories; +- row and point-operation benchmarks. -**Ship when proven, not when scheduled**: Timelines are estimates. Correctness is non-negotiable. +## Phase 7: Document and Object Dialects -**No resume-driven development**: Build what users need, not what looks good on LinkedIn. Complexity is a cost, not a feature. +Add document and object semantics without routing them through relational +compatibility layers. + +Deliver: + +- document paths, queries, updates, and indexes; +- object metadata, versions, ranges, and multipart state; +- document and object-extent physical dialects; +- transactional metadata and projection rules; +- garbage collection for versions and extents. + +Exit tests: + +- reference compatibility suites; +- versioning and conditional request histories; +- multipart recovery; +- range-read correctness; +- transactional cross-dialect mappings; +- document and object workload benchmarks. + +## Phase 8: Analytical Dialect + +Add locality-optimized analytical execution. + +Deliver: + +- analytical logical plans; +- columnar physical storage; +- vectorized operators; +- projection freshness rules; +- distributed scans and exchanges; +- cost-based physical selection. + +Exit tests: + +- query correctness against a reference engine; +- snapshot consistency during ingestion; +- projection recovery and rebuild; +- optimizer equivalence; +- named analytical benchmarks with full configuration. + +## Phase 9: Planetary Operations + +Make placement and failure domains first-class. + +Deliver: + +- multi-region placement policy; +- online rebalancing; +- regional failure handling; +- backup and point-in-time recovery; +- admission control and workload isolation; +- upgrade and downgrade protocols; +- operational compatibility checks. + +Exit tests: + +- region-loss exercises; +- long-running Jepsen campaigns; +- restore and disaster-recovery drills; +- mixed-version operation; +- residency policy audits; +- sustained workload benchmarks. + +## Release Gates + +Every phase must provide: + +1. A written invariant. +2. A test that fails without the implementation. +3. Fault injection at the durability boundary. +4. Metrics that reveal invariant health. +5. A recovery procedure. +6. Reproducible performance results for performance claims. + +Cloud9 does not label target behavior as implemented. It does not trade a +documented consistency guarantee for availability without an explicit mode +change. diff --git a/spec/README.md b/spec/README.md index ea21a18..8145fd3 100644 --- a/spec/README.md +++ b/spec/README.md @@ -1,216 +1,116 @@ # Cloud9 Specifications -This directory contains the complete technical specification for Cloud9, the distributed database that should have existed from the start. These documents explain design decisions, technical foundations, and implementation strategies. +Cloud9 is an open-source Spanner and MLIR for databases. -## Reading Guide +These documents define the target architecture. They distinguish implemented +behavior from planned behavior. The repository currently provides Raft, a +durable write-ahead log, replicated key-value operations, and Jepsen tests. -### For Newcomers: Start Here +## Product Contract -1. **[00-vision.md](00-vision.md)** - Understand why Cloud9 exists and what makes it unique -2. **[01-mvcc.md](01-mvcc.md)** - Core concurrency control mechanism -3. **[02-timestamps.md](02-timestamps.md)** - How Cloud9 handles distributed time -4. **[03-external-consistency.md](03-external-consistency.md)** - The fundamental guarantee Cloud9 provides +Cloud9 combines one correctness plane with several database dialects and +physical engines: -After these four, choose your path based on interest. +- SQL for relational workloads. +- DynamoDB-style key-value operations. +- MongoDB-style document operations. +- S3-style object operations. +- ClickHouse-style analytical plans. -### Quick Reference by Topic +The dialects share transactions, timestamps, identity, placement, and +observability. They keep distinct semantics and physical layouts. -**Understanding timestamps and time synchronization:** -- [02-timestamps.md](02-timestamps.md) - Timestamp strategies (HLC, TrueTime, TSO) -- [03-external-consistency.md](03-external-consistency.md) - Why commit-wait is necessary -- [04-truetime-analysis.md](04-truetime-analysis.md) - Mathematical foundations of bounded clock uncertainty -- [05-aws-time-infrastructure.md](05-aws-time-infrastructure.md) - Practical deployment on AWS infrastructure +Cloud9 targets local development and planetary deployment. Local mode should +feel like SQLite. Distributed mode partitions data into Raft-replicated ranges. -**Understanding transactions and consistency:** -- [01-mvcc.md](01-mvcc.md) - Multi-version concurrency control -- [03-external-consistency.md](03-external-consistency.md) - External consistency guarantees -- [08-transactions.md](08-transactions.md) - Complete transaction protocol (2PC, intents, commit-wait) +## Architectural Decisions -**Understanding distributed architecture:** -- [06-sharding-partitioning.md](06-sharding-partitioning.md) - Range-based sharding and replication -- [10-consensus.md](10-consensus.md) - Raft consensus and operational features -- [08-transactions.md](08-transactions.md) - Cross-shard transaction coordination +### Multi-level database IR -**Understanding data model and APIs:** -- [07-sql-kv-unification.md](07-sql-kv-unification.md) - How SQL and KV share one transactional core +Source APIs lower through typed intermediate representations (IRs). High-level +semantics remain visible until a legal lower level can preserve them. Physical +lowering may select row, key-value, document, object, or columnar execution. -**Understanding the market context:** -- [09-market-analysis.md](09-market-analysis.md) - Why Cloud9 exists, pain points with existing solutions +See [07-sql-kv-unification.md](07-sql-kv-unification.md). -## Specification Index +### Capability-gated TrueTime -### Foundations +Cloud9 exposes a TrueTime-shaped bounded-time API: -#### [00-vision.md](00-vision.md) -Cloud9's core philosophy and goals. Why external consistency matters, what makes Cloud9 unique, and who it's for. Read this first to understand the "why" before diving into the "how." +```text +now() -> [earliest, latest] +``` -**Key concepts:** External consistency guarantee, Spanner+Postgres+FoundationDB synthesis, no compromises philosophy, daily driver database +TrueTime mode starts only with a healthy, approved bounded-time provider. The +first production backend is AWS ClockBound on supported Linux EC2 hardware. +Cloud9 does not substitute a Hybrid Logical Clock when that provider fails. -#### [01-mvcc.md](01-mvcc.md) -Multi-Version Concurrency Control (MVCC) enables lock-free read-only transactions and backups without blocking writes. Every write gets a commit timestamp, readers choose a snapshot timestamp and see a consistent point-in-time view. +Local mode has no bounded-time hardware requirement. It does not claim +hardware-backed TrueTime or cross-machine external consistency. -**Key concepts:** Versioned storage, snapshot isolation, lock-free reads, temporal queries +See [02-timestamps.md](02-timestamps.md), +[03-external-consistency.md](03-external-consistency.md), and +[05-aws-time-infrastructure.md](05-aws-time-infrastructure.md). -**Why alternatives don't work:** Two-phase locking blocks reads, optimistic concurrency causes high abort rates, timestamp ordering without versions can't support historical reads +### Shared correctness, specialized execution -#### [02-timestamps.md](02-timestamps.md) -Distributed timestamp strategies for external consistency. Compares Hybrid Logical Clocks (HLC), TrueTime, and Timestamp Oracle (TSO). Explains why Lamport clocks are insufficient for databases. +The shared plane owns transactions, multi-version concurrency control (MVCC), +replication, recovery, catalogs, and placement. Physical engines own their data +structures and hot paths. -**Key concepts:** HLC with commit-wait (default), TSO mode (alternative), clock uncertainty bounds (epsilon), why commit-wait is unavoidable +Specialization is necessary for performance. It cannot weaken the shared +correctness contract. -**Cloud9's choice:** HLC for most deployments, TSO for unreliable clock sync environments, TrueTime-class performance available with GPS/atomic clocks +## Reading Order -#### [03-external-consistency.md](03-external-consistency.md) -Formal definition and implementation of external consistency (strict serializability). If a write finishes before a read starts in real time, the read must see the write. Explains commit-wait protocol and why server-side timestamps are required. +1. [Vision](00-vision.md) +2. [Multi-version concurrency control](01-mvcc.md) +3. [Timestamp model](02-timestamps.md) +4. [External consistency](03-external-consistency.md) +5. [Bounded-time analysis](04-truetime-analysis.md) +6. [AWS ClockBound backend](05-aws-time-infrastructure.md) +7. [Sharding and placement](06-sharding-partitioning.md) +8. [Multi-model intermediate representation](07-sql-kv-unification.md) +9. [Transaction protocol](08-transactions.md) +10. [Product rationale](09-market-analysis.md) +11. [Consensus and replication](10-consensus.md) +12. [Catalogs, schemas, and projections](11-indexes-schema.md) +13. [Implementation roadmap](12-implementation-roadmap.md) -**Key concepts:** Real-time ordering, commit-wait necessity, PACELC trade-offs, why client timestamps don't work +## Specification Map -**Core protocol:** Assign commit timestamp, replicate via Raft, commit-wait until now() > t_commit + epsilon, acknowledge client +| File | Decision | +|------|----------| +| [00-vision.md](00-vision.md) | Product and consistency contract | +| [01-mvcc.md](01-mvcc.md) | Version visibility and retention | +| [02-timestamps.md](02-timestamps.md) | Bounded-time provider interface | +| [03-external-consistency.md](03-external-consistency.md) | Commit timestamp and commit-wait protocol | +| [04-truetime-analysis.md](04-truetime-analysis.md) | Proof obligations and uncertainty cost | +| [05-aws-time-infrastructure.md](05-aws-time-infrastructure.md) | ClockBound deployment requirements | +| [06-sharding-partitioning.md](06-sharding-partitioning.md) | Ranges, replicas, and locality | +| [07-sql-kv-unification.md](07-sql-kv-unification.md) | Database dialects and lowering pipeline | +| [08-transactions.md](08-transactions.md) | Single-range and distributed transactions | +| [09-market-analysis.md](09-market-analysis.md) | Product thesis and validation | +| [10-consensus.md](10-consensus.md) | Raft persistence and replication | +| [11-indexes-schema.md](11-indexes-schema.md) | Catalog, schema, index, and projection lifecycle | +| [12-implementation-roadmap.md](12-implementation-roadmap.md) | Dependency-ordered delivery plan | -### Time Infrastructure +## Status Language -#### [04-truetime-analysis.md](04-truetime-analysis.md) -Mathematical foundations of TrueTime. Proves why bounded clock uncertainty is not heuristic but formally correct. Explains the 30-second sync interval, drift calculations, and failure modes. +Specifications use three status terms: -**Key concepts:** Uncertainty formula (epsilon = sync_error + drift_rate × time), formal invariant proof, fail-safe behavior, why it's not guesswork +- **Implemented** means code and tests exist in this repository. +- **In progress** means an implementation exists but lacks a required proof. +- **Target** means architecture is specified but not implemented. -**Key insight:** TrueTime is proven mathematics based on hardware specifications, not empirical tuning. Cloud9 implements the same rigorous approach with different infrastructure. +Performance goals are targets until a reproducible benchmark supports them. +Every result must name the workload, hardware, topology, durability mode, and +consistency mode. -#### [05-aws-time-infrastructure.md](05-aws-time-infrastructure.md) -Practical time synchronization options for Cloud9 on AWS. Covers four deployment tiers from standard NTP (50-100ms uncertainty) to GPS/atomic clocks (<1ms uncertainty). +## References -**Deployment tiers:** -- **Standard:** NTP (50-100ms epsilon, zero setup) -- **Performance:** PTP/PHC (10-50ms epsilon, recommended default) -- **Premium:** AWS Outposts with GPS (1-10ms epsilon, hybrid deployment) -- **Premium+:** Colocation with GPS/atomic (< 1ms epsilon, Spanner-class) - -**Key concepts:** Clock uncertainty measurement, chrony monitoring, fail-stop behavior, operational considerations - -### Implementation - -#### [06-sharding-partitioning.md](06-sharding-partitioning.md) -Range-based sharding with the degenerate case: local mode = 1 range, 1 replica. Explains leaseholder architecture, auto-split/merge policies, hotspot handling, and interleaved tables for foreign key performance. - -**Key concepts:** Range = contiguous key interval, Raft group per range, leaseholder serves reads/writes, local mode is distributed with N=1 - -**Why range sharding:** Enables range scans (required for SQL), co-location of related data, fine-grained splits, and seamless local-to-distributed scaling - -**Core insight:** Local deployment is not a special case - it's the degenerate case where the general distributed model has N=1 - -#### [07-sql-kv-unification.md](07-sql-kv-unification.md) -How SQL and KV coexist as the same system, not separate systems bolted together. SQL tables and KV namespaces are both key prefixes in a single MVCC storage layer. Enables cross-API transactions and joins. - -**Key concepts:** Unified key encoding (/table/ vs /kv/), TxIR compilation target, schema-on-read for KV, cross-API joins with KV() virtual table - -**Killer feature:** Begin transaction, write to SQL tables, read from KV namespaces, join SQL and KV data, commit atomically at one timestamp - -**Why others failed:** FoundationDB had experimental SQL, YugabyteDB has separate systems (YSQL vs YCQL), Spanner is SQL-only, DynamoDB is KV-only - -#### [08-transactions.md](08-transactions.md) -Complete transaction protocol: two-phase commit (2PC) with MVCC intents, coordinator-driven commit timestamp assignment, and external consistency via commit-wait. Covers read-only transactions, cross-shard writes, intent resolution, and recovery. - -**Transaction types:** Read-only (lock-free, no 2PC), single-range writes (simplified 2PC), cross-shard writes (full 2PC) - -**Key protocols:** Intent writing (phase 0), prepare phase (conflict detection), commit phase (intent resolution), commit-wait (external consistency) - -**Advanced topics:** Closed timestamps for follower reads, intent cleanup, transaction recovery, timestamp caching, parallel commit optimization - -#### [10-consensus.md](10-consensus.md) -Raft consensus as the replication foundation. Explains why Raft (not Paxos or novel algorithms), the clean consensus driver interface, and operational features: dynamic leader placement, witness replicas, learner replicas, per-range configuration. - -**Core principle:** Ship one consensus algorithm done right. Clean separation between consensus (log replication) and state machine (MVCC storage). - -**Operational features:** Leadership transfer (latency optimization), witnesses (storage cost reduction), learners (safe reconfiguration), per-range config (multi-tenant flexibility) - -**Not resume padding:** The interface exists for testability and maintainability, not to ship multiple algorithms at launch - -### Market Context - -#### [09-market-analysis.md](09-market-analysis.md) -Why Cloud9 exists based on real user feedback from Spanner, DynamoDB, and competing systems. Documents pain points: cost models, vendor lock-in fears, support quality, documentation gaps, and billing disasters. - -**Spanner problems:** High minimum cost ($65-1000+/month), GCP platform instability, poor support, hidden performance gotchas, fear of product cancellation - -**DynamoDB problems:** KV-only limitations, capacity planning complexity, no multi-item transactions, hot partition issues - -**Common theme:** Users fear vendor lock-in more than technical limitations. "Doing business with Google is a liability." - -**The missing middle:** Gap between Postgres (single-node) and Spanner/DynamoDB (enterprise-only, vendor lock-in). Cloud9 fills this gap with open-source, scale-from-laptop-to-global deployment. - -**Billing horror stories:** RAG Engine incident (silent $30-800/day charges), Gemini billing errors ($70k+ bills), CloudSQL performance forcing migration to self-hosted VMs - -## Recommended Reading Paths - -### Path 1: Understand the Core Guarantees -For those who want to understand what Cloud9 promises and how it delivers: - -1. [00-vision.md](00-vision.md) - The promise -2. [03-external-consistency.md](03-external-consistency.md) - The formal guarantee -3. [02-timestamps.md](02-timestamps.md) - How time enables the guarantee -4. [08-transactions.md](08-transactions.md) - The complete protocol - -### Path 2: Deployment and Operations -For those planning to deploy Cloud9: - -1. [00-vision.md](00-vision.md) - What you're deploying -2. [05-aws-time-infrastructure.md](05-aws-time-infrastructure.md) - Time sync options and deployment tiers -3. [06-sharding-partitioning.md](06-sharding-partitioning.md) - Scaling from local to distributed -4. [10-consensus.md](10-consensus.md) - Operational control (leader placement, replicas) - -### Path 3: Implementation Study -For those building Cloud9 or similar systems: - -1. [01-mvcc.md](01-mvcc.md) - Storage foundation -2. [02-timestamps.md](02-timestamps.md) - Timestamp strategies -3. [04-truetime-analysis.md](04-truetime-analysis.md) - Mathematical rigor -4. [08-transactions.md](08-transactions.md) - Transaction machinery -5. [06-sharding-partitioning.md](06-sharding-partitioning.md) - Distributed architecture -6. [10-consensus.md](10-consensus.md) - Replication layer -7. [07-sql-kv-unification.md](07-sql-kv-unification.md) - API unification - -### Path 4: Market Positioning -For those evaluating Cloud9 vs alternatives: - -1. [00-vision.md](00-vision.md) - Cloud9's positioning -2. [09-market-analysis.md](09-market-analysis.md) - Problems with existing solutions -3. [03-external-consistency.md](03-external-consistency.md) - The technical differentiator -4. [07-sql-kv-unification.md](07-sql-kv-unification.md) - Unique capabilities - -## Design Principles - -These specs embody Cloud9's core principles: - -1. **No compromises:** External consistency + SQL + KV + open source + local-to-global -2. **Proven foundations:** MVCC, Raft, HLC, commit-wait - nothing novel, everything battle-tested -3. **Clean architecture:** Consensus driver interface, TxIR compilation target, clear separation of concerns -4. **Operational flexibility:** Leader placement, witness replicas, per-range configuration -5. **Developer experience:** Postgres wire compatibility, start on laptop, deploy to cloud unchanged -6. **No vendor lock-in:** MIT license, self-hostable, community-driven - -## What's Not Here - -These specs intentionally omit: - -- **Implementation details:** Code belongs in source files with inline documentation -- **API references:** Generated from source, not duplicated in specs -- **Benchmarks:** Belong in performance testing suite with reproducible methodology -- **Roadmap timelines:** Tracked in GitHub issues/projects, not static documents - -These specs explain **design decisions, trade-offs, and foundations**. Implementation lives in code. Operations guides live in docs. Roadmap lives in project management. - -## Contributing - -Found an error? Have a clarifying question? Open an issue. - -Want to propose a design change? Write a spec amendment following the existing format: clear problem statement, alternatives considered with rationale, concrete examples, trade-off analysis. - -Specs are living documents. As Cloud9 evolves through production deployment, these specs will be updated to reflect reality, not aspirational design. - -## Status - -**Current:** Specification phase (2025-01). Implementation beginning. - -**Stability:** Design is stable for foundational components (MVCC, timestamps, transactions, consensus). Market analysis reflects 2024-2025 feedback. AWS infrastructure options current as of 2025-01. - -**Updates:** Specs will be versioned when implementation reveals necessary changes. No silent edits - all changes tracked via git history. +- [Spanner](https://research.google/pubs/pub39966/) +- [MLIR dialect conversion](https://mlir.llvm.org/docs/DialectConversion/) +- [AWS ClockBound](https://github.com/aws/clock-bound) +- [Amazon Time Sync on EC2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configure-ec2-ntp.html) +- [Raft](https://raft.github.io/) From 1235b6208ff601235c5c0de2434f6ce0925843ea Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:34:29 -0700 Subject: [PATCH 16/17] docs(docs): document public APIs --- cloud9-node/src/auth.rs | 3 ++ cloud9-node/src/config.rs | 8 ++++ cloud9-node/src/lib.rs | 1 + cloud9-proto/src/lib.rs | 1 + cloud9-wal/src/error.rs | 97 ++++++++++++++++++++++++++++++++++----- cloud9-wal/src/lib.rs | 3 +- cloud9-wal/src/record.rs | 6 +++ cloud9-wal/src/wal.rs | 3 ++ 8 files changed, 109 insertions(+), 13 deletions(-) diff --git a/cloud9-node/src/auth.rs b/cloud9-node/src/auth.rs index 4e0a855..93b54d6 100644 --- a/cloud9-node/src/auth.rs +++ b/cloud9-node/src/auth.rs @@ -11,14 +11,17 @@ const SIGNATURE_BYTES: usize = 32; type HmacSha256 = Hmac; +/// Shared 256-bit key for authenticating Raft peer messages. #[derive(Clone)] pub struct RaftKey([u8; KEY_BYTES]); +/// A configured Raft key is not exactly 32 hexadecimal bytes. #[derive(Debug, Error)] #[error("Raft key must be exactly 64 hexadecimal characters")] pub struct InvalidRaftKey; impl RaftKey { + /// Parse a 64-character hexadecimal key. pub fn from_hex(value: &str) -> Result { decode_hex(value).map(Self).ok_or(InvalidRaftKey) } diff --git a/cloud9-node/src/config.rs b/cloud9-node/src/config.rs index 0466df9..793fcb3 100644 --- a/cloud9-node/src/config.rs +++ b/cloud9-node/src/config.rs @@ -12,12 +12,19 @@ use crate::RaftKey; /// Runtime configuration derived from CLI flags and config files. #[derive(Debug, Clone)] pub struct NodeConfig { + /// Stable identity used by the Raft group. pub node_id: NodeId, + /// Address for the public database API. pub client_addr: SocketAddr, + /// Address for authenticated Raft peer traffic. pub raft_addr: SocketAddr, + /// Complete mapping from Raft node IDs to peer addresses. pub peers: BTreeMap, + /// Shared key used to authenticate peer messages. pub raft_key: RaftKey, + /// Durable storage configuration. pub storage: StorageOptions, + /// Raft state-machine configuration. pub consensus: ConsensusConfig, } @@ -29,6 +36,7 @@ impl NodeConfig { } #[must_use] +/// Build the Raft configuration required by the current node runtime. pub fn raft_config(node_id: NodeId) -> ConsensusConfig { let mut config = ConsensusConfig::new(node_id).with_parallel_disk_write(false); config.max_entries_per_msg = 1; diff --git a/cloud9-node/src/lib.rs b/cloud9-node/src/lib.rs index 847fb5e..644e805 100644 --- a/cloud9-node/src/lib.rs +++ b/cloud9-node/src/lib.rs @@ -1,5 +1,6 @@ #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used)] +#![warn(missing_docs)] #![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::panic))] //! Top-level orchestration for Cloud9 nodes. diff --git a/cloud9-proto/src/lib.rs b/cloud9-proto/src/lib.rs index 3caca69..ecbaf29 100644 --- a/cloud9-proto/src/lib.rs +++ b/cloud9-proto/src/lib.rs @@ -3,6 +3,7 @@ use cloud9_core::SharedString; use serde::{Deserialize, Serialize}; +/// Generated ConnectRPC protocol types and clients. #[allow(warnings)] pub mod generated { connectrpc::include_generated!("_cloud9_connect.rs"); diff --git a/cloud9-wal/src/error.rs b/cloud9-wal/src/error.rs index ad91bd1..ff7f41f 100644 --- a/cloud9-wal/src/error.rs +++ b/cloud9-wal/src/error.rs @@ -8,32 +8,80 @@ use crate::record::Lsn; /// WAL failures. #[derive(Debug, Error)] pub enum WalError { + /// An operation on a WAL path failed. #[error("I/O error at `{path}`")] Io { + /// Path involved in the failed operation. path: PathBuf, + /// Operating-system error returned for the path. #[source] source: io::Error, }, + /// Record kind zero was supplied. #[error("record kind zero is reserved")] ReservedRecordKind, + /// Configured segments cannot hold one record header. #[error("segment size {segment_size} is smaller than record header {header_len}")] - SegmentTooSmall { segment_size: u64, header_len: usize }, + SegmentTooSmall { + /// Configured segment capacity. + segment_size: u64, + /// Bytes required by the fixed header. + header_len: usize, + }, + /// An encoded record cannot fit in one segment. #[error("record length {len} exceeds segment size {segment_size}")] - RecordTooLarge { len: u64, segment_size: u64 }, + RecordTooLarge { + /// Encoded record length. + len: u64, + /// Configured segment capacity. + segment_size: u64, + }, + /// An append would exceed the configured WAL capacity. #[error("WAL size {len} exceeds configured maximum {max_size}")] - WalFull { len: u64, max_size: u64 }, + WalFull { + /// Total bytes after the rejected append. + len: u64, + /// Configured WAL capacity. + max_size: u64, + }, + /// A payload cannot be represented by the on-disk length field. #[error("payload length {len} exceeds u32::MAX")] - PayloadTooLarge { len: usize }, + PayloadTooLarge { + /// Rejected payload length. + len: usize, + }, + /// The next segment ID cannot be represented. #[error("segment id exhausted")] SegmentIdExhausted, + /// Recovery found a malformed durable record. #[error("corrupt WAL record at segment {lsn:?}: {reason}")] - CorruptRecord { lsn: Lsn, reason: Corruption }, + CorruptRecord { + /// Position of the malformed record. + lsn: Lsn, + /// Corruption detected at the position. + reason: Corruption, + }, + /// A segment filename does not use the canonical numeric form. #[error("malformed WAL segment filename `{path}`")] - BadSegmentName { path: PathBuf }, + BadSegmentName { + /// Malformed segment path. + path: PathBuf, + }, + /// The ordered segment sequence contains a gap. #[error("missing WAL segment {expected:020}.wal before {found:020}.wal")] - MissingSegment { expected: u64, found: u64 }, + MissingSegment { + /// Segment ID required next. + expected: u64, + /// Later segment ID found on disk. + found: u64, + }, + /// Another writer holds the directory lock. #[error("WAL directory `{path}` already has a writer")] - Locked { path: PathBuf }, + Locked { + /// Locked WAL directory. + path: PathBuf, + }, + /// A partial write invalidated the open handle. #[error("WAL handle is poisoned; reopen it to recover")] Poisoned, } @@ -47,22 +95,47 @@ impl WalError { /// Specific corruption detected while scanning records. #[derive(Debug, Error)] pub enum Corruption { + /// The record header does not start with the WAL magic number. #[error("bad magic {found:#x}")] - BadMagic { found: u32 }, + BadMagic { + /// Unexpected magic number. + found: u32, + }, + /// The record uses an unsupported format version. #[error("unsupported version {found}")] - UnsupportedVersion { found: u16 }, + UnsupportedVersion { + /// Unsupported on-disk version. + found: u16, + }, + /// The record uses kind zero. #[error("reserved record kind")] ReservedRecordKind, + /// The record header checksum does not match its fields. #[error("header checksum mismatch")] HeaderChecksum, + /// The payload checksum does not match its bytes. #[error("payload checksum mismatch")] PayloadChecksum, + /// A record length exceeds the configured segment capacity. #[error("record length {len} exceeds segment size {segment_size}")] - RecordTooLarge { len: u64, segment_size: u64 }, + RecordTooLarge { + /// Encoded record length from the header. + len: u64, + /// Configured segment capacity. + segment_size: u64, + }, + /// A segment file exceeds its configured capacity. #[error("segment length {len} exceeds configured size {segment_size}")] - SegmentTooLarge { len: u64, segment_size: u64 }, + SegmentTooLarge { + /// Segment file length. + len: u64, + /// Configured segment capacity. + segment_size: u64, + }, + /// The final record ends before its declared length. #[error("incomplete record")] IncompleteRecord, } +/// WAL operation result. pub type Result = std::result::Result; diff --git a/cloud9-wal/src/lib.rs b/cloud9-wal/src/lib.rs index 53ad063..07a1bdd 100644 --- a/cloud9-wal/src/lib.rs +++ b/cloud9-wal/src/lib.rs @@ -1,8 +1,9 @@ #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used)] +#![warn(missing_docs)] #![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::panic))] -//! Tiny segmented write-ahead log. +//! Segmented write-ahead log (WAL). //! //! The WAL deliberately owns only the byte-durability problem: //! 1. append one typed byte record to the active segment, diff --git a/cloud9-wal/src/record.rs b/cloud9-wal/src/record.rs index 0da7fb4..19cabb6 100644 --- a/cloud9-wal/src/record.rs +++ b/cloud9-wal/src/record.rs @@ -3,7 +3,9 @@ use crate::{Result, WalError}; /// Position of a WAL record. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct Lsn { + /// Segment containing the record. pub segment_id: u64, + /// Byte offset of the record header within the segment. pub offset: u64, } @@ -27,13 +29,17 @@ impl RecordKind { /// One logical WAL record. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Record { + /// Caller-defined non-zero record type. pub kind: RecordKind, + /// Uninterpreted record body. pub payload: Vec, } /// A record with its log position. #[derive(Debug, Clone, PartialEq, Eq)] pub struct StoredRecord { + /// Durable position of the record. pub lsn: Lsn, + /// Decoded record contents. pub record: Record, } diff --git a/cloud9-wal/src/wal.rs b/cloud9-wal/src/wal.rs index 7cdbf19..bd5193d 100644 --- a/cloud9-wal/src/wal.rs +++ b/cloud9-wal/src/wal.rs @@ -14,8 +14,11 @@ const DEFAULT_MAX_SIZE: u64 = 4 * 1024 * 1024 * 1024; /// WAL configuration. #[derive(Debug, Clone)] pub struct WalOptions { + /// Maximum byte length of one segment. pub segment_size: u64, + /// Maximum total bytes across all segments. pub max_size: u64, + /// Whether each append reaches stable storage before returning. pub sync_on_append: bool, } From 3edb3e18520a4f3e0ac98c110aef10fe0c6c5037 Mon Sep 17 00:00:00 2001 From: Windsor Nguyen <93564933+windsornguyen@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:39:49 -0700 Subject: [PATCH 17/17] chore(ci): target Node 24 actions --- .github/workflows/ci.yml | 10 +++++----- .github/workflows/commitlint.yml | 2 +- .github/workflows/security.yml | 4 ++-- .github/workflows/typos.yml | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cd0d12..61daf27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -48,7 +48,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -73,10 +73,10 @@ jobs: working-directory: jepsen steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Setup Java - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: temurin java-version: 21 @@ -95,7 +95,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index 3d0f52e..a6f0bbb 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -8,7 +8,7 @@ jobs: commitlint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 93c5d7a..cd1e480 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -26,7 +26,7 @@ jobs: issues: write checks: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: EmbarkStudios/cargo-deny-action@v2 @@ -34,7 +34,7 @@ jobs: name: cargo-audit runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install cargo-audit run: cargo install cargo-audit diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml index 2730a60..8884482 100644 --- a/.github/workflows/typos.yml +++ b/.github/workflows/typos.yml @@ -20,7 +20,7 @@ jobs: name: Spell Check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Check spelling uses: crate-ci/typos@v1