diff --git a/Cargo.lock b/Cargo.lock index 937ead564a..920fee07b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -838,6 +838,7 @@ dependencies = [ "buzz-auth", "buzz-core", "buzz-db", + "buzz-deletion", "buzz-media", "buzz-pubsub", "buzz-search", @@ -1019,6 +1020,28 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-deletion" +version = "0.1.0" +dependencies = [ + "anyhow", + "buzz-core", + "buzz-db", + "buzz-media", + "chrono", + "clap", + "deadpool-redis", + "hex", + "redis", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "uuid", +] + [[package]] name = "buzz-dev-mcp" version = "0.1.0" @@ -1191,6 +1214,7 @@ dependencies = [ "buzz-conformance", "buzz-core", "buzz-db", + "buzz-deletion", "buzz-media", "buzz-pubsub", "buzz-relay-mesh", @@ -1346,6 +1370,7 @@ version = "0.1.0" dependencies = [ "buzz-core", "buzz-db", + "buzz-deletion", "chrono", "cron", "dashmap", diff --git a/Cargo.toml b/Cargo.toml index cc1dd0f9df..f22330c683 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "crates/buzz-test-client", "crates/buzz-ws-client", "crates/buzz-admin", + "crates/buzz-deletion", "crates/buzz-workflow", "crates/buzz-media", "crates/buzz-cli", @@ -134,6 +135,7 @@ schemars = { version = "1", default-features = false } buzz-core = { path = "crates/buzz-core" } buzz-conformance = { path = "crates/buzz-conformance" } buzz-db = { path = "crates/buzz-db" } +buzz-deletion = { path = "crates/buzz-deletion" } buzz-auth = { path = "crates/buzz-auth" } buzz-pubsub = { path = "crates/buzz-pubsub" } buzz-search = { path = "crates/buzz-search" } diff --git a/crates/buzz-admin/Cargo.toml b/crates/buzz-admin/Cargo.toml index 7a69e146bb..00c2804cbc 100644 --- a/crates/buzz-admin/Cargo.toml +++ b/crates/buzz-admin/Cargo.toml @@ -13,6 +13,7 @@ path = "src/main.rs" [dependencies] buzz-db = { workspace = true } +buzz-deletion = { workspace = true } buzz-core = { workspace = true } buzz-auth = { workspace = true } buzz-pubsub = { workspace = true } diff --git a/crates/buzz-admin/src/deletions.rs b/crates/buzz-admin/src/deletions.rs new file mode 100644 index 0000000000..64cb8bd732 --- /dev/null +++ b/crates/buzz-admin/src/deletions.rs @@ -0,0 +1,19 @@ +//! Thin `buzz-admin deletions` adapter. + +pub use buzz_deletion::Command as DeletionsCommand; + +/// Delegate to the shared durable deletion engine. +pub async fn run(command: DeletionsCommand) -> anyhow::Result { + buzz_deletion::run(command).await +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + #[test] + fn continuous_worker_command_is_not_exposed() { + let command = crate::Cli::try_parse_from(["buzz-admin", "deletions", "worker"]); + assert!(command.is_err()); + } +} diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index bb30ddfae4..580d586591 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -20,6 +20,8 @@ //! newest timestamp and collide on the bumped second. run.sh serialization is //! the guard against parallel adds (e.g. `xargs -P`). +mod deletions; + use std::sync::Arc; use anyhow::Result; @@ -81,6 +83,11 @@ enum Command { #[command(subcommand)] command: ProductFeedbackCommand, }, + /// Durable CLI-only whole-community deletion control plane. + Deletions { + #[command(subcommand)] + command: deletions::DeletionsCommand, + }, /// Emit kind:39000/39002 events for channels missing them. /// /// Channels created via direct SQL (seed scripts, pre-migration data) won't @@ -148,6 +155,7 @@ async fn run(cli: Cli) -> Result { Command::ProductFeedback { command: ProductFeedbackCommand::List { limit }, } => cmd_list_product_feedback(limit).await, + Command::Deletions { command } => deletions::run(command).await, Command::ReconcileChannels { relay_key } => { reconcile_channels(relay_key).await?; Ok(0) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs new file mode 100644 index 0000000000..4ba2d98517 --- /dev/null +++ b/crates/buzz-db/src/deletion.rs @@ -0,0 +1,3446 @@ +//! Durable whole-community deletion lifecycle and PostgreSQL adapter. +//! +//! This module owns request inventory, approval, claims, fencing, checkpoints, +//! retries, tombstoning, and logical verification. CLI claim-loop policy and +//! external storage adapters live above it; they never implement state changes. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::str::FromStr; +use std::time::Duration; + +use buzz_core::CommunityId; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use sqlx::{AssertSqlSafe, PgPool, Postgres, Row, Transaction}; +use uuid::Uuid; + +use crate::error::{DbError, Result}; + +/// Default PostgreSQL lease duration for one claimed deletion request. +pub const DEFAULT_LEASE_DURATION: Duration = Duration::from_secs(60); +/// Durable name of the schema manifest's PostgreSQL component. +pub const POSTGRES_STORE_NAME: &str = "postgres"; +/// Durable name of the object-store manifest component. +pub const OBJECT_STORE_NAME: &str = "object_store"; +/// Durable name of the Redis/cache manifest component. +pub const REDIS_STORE_NAME: &str = "redis"; + +/// Control-plane tables that survive the community data purge. +pub const CONTROL_PLANE_TABLES: &[&str] = &[ + "community_deletion_approvals", + "community_deletion_checkpoints", + "community_deletion_executor_heartbeats", + "community_deletion_requests", + "community_serving_write_leases", +]; + +/// Expected community-scoped tables purged by V1. +/// +/// Catalog inventory compares the live database against this exact set before +/// approval and again before PostgreSQL purge. A new tenant table therefore +/// blocks deletion until this manifest is intentionally updated. +pub const EXPECTED_SCOPED_TABLES: &[&str] = &[ + "api_tokens", + "archived_identities", + "audit_log", + "channel_members", + "channels", + "community_bans", + "delivery_log", + "event_mentions", + "events", + "git_repo_names", + "join_policy_acceptances", + "moderation_actions", + "moderation_reports", + "parameterized_event_watermarks", + "product_feedback", + "rate_limit_violations", + "pubkey_allowlist", + "push_leases", + "push_match_queue", + "push_wake_outbox", + "reactions", + "relay_invites", + "relay_members", + "scheduled_workflow_fires", + "subscriptions", + "thread_metadata", + "users", + "workflow_approvals", + "workflow_runs", + "workflows", +]; + +/// Foreign-key-safe child-before-parent order for the PostgreSQL purge. +pub const PURGE_SCOPED_TABLES: &[&str] = &[ + "workflow_approvals", + "scheduled_workflow_fires", + "workflow_runs", + "push_wake_outbox", + "join_policy_acceptances", + "moderation_reports", + "subscriptions", + "api_tokens", + "channel_members", + "thread_metadata", + "moderation_actions", + "workflows", + "event_mentions", + "reactions", + "push_match_queue", + "push_leases", + "relay_invites", + "product_feedback", + "rate_limit_violations", + "delivery_log", + "events", + "parameterized_event_watermarks", + "git_repo_names", + "archived_identities", + "audit_log", + "community_bans", + "pubkey_allowlist", + "relay_members", + "users", + "channels", +]; + +/// Fixed lifecycle order. There are no backwards or skipping transitions. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DeletionStage { + /// Request exists but has not frozen its inventory. + Submitted, + /// PostgreSQL and storage inventory has been frozen. + Inventoried, + /// An operator explicitly approved the frozen inventory digest. + Approved, + /// Universal serving-path write fence is active. + Fenced, + /// In-flight serving writes have drained behind the durable fence. + Drained, + /// Tenant-owned S3/media and Git pointer bindings were removed. + BindingsRemoved, + /// Tenant-scoped PostgreSQL rows were purged. + PostgresPurged, + /// Redis/community process-cache namespace was purged. + CachePurged, + /// Cross-store logical absence was verified. + LogicallyVerified, + /// Logical deletion complete; shared CAS physical expiry is deferred. + RetentionPending, +} + +impl DeletionStage { + /// Next legal stage, if this is not terminal. + pub const fn next(self) -> Option { + match self { + Self::Submitted => Some(Self::Inventoried), + Self::Inventoried => Some(Self::Approved), + Self::Approved => Some(Self::Fenced), + Self::Fenced => Some(Self::Drained), + Self::Drained => Some(Self::BindingsRemoved), + Self::BindingsRemoved => Some(Self::PostgresPurged), + Self::PostgresPurged => Some(Self::CachePurged), + Self::CachePurged => Some(Self::LogicallyVerified), + Self::LogicallyVerified => Some(Self::RetentionPending), + Self::RetentionPending => None, + } + } + + /// Whether execution may claim this stage. + pub const fn runnable(self) -> bool { + matches!( + self, + Self::Approved + | Self::Fenced + | Self::Drained + | Self::BindingsRemoved + | Self::PostgresPurged + | Self::CachePurged + | Self::LogicallyVerified + ) + } +} + +impl fmt::Display for DeletionStage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let value = match self { + Self::Submitted => "submitted", + Self::Inventoried => "inventoried", + Self::Approved => "approved", + Self::Fenced => "fenced", + Self::Drained => "drained", + Self::BindingsRemoved => "bindings_removed", + Self::PostgresPurged => "postgres_purged", + Self::CachePurged => "cache_purged", + Self::LogicallyVerified => "logically_verified", + Self::RetentionPending => "retention_pending", + }; + f.write_str(value) + } +} + +impl FromStr for DeletionStage { + type Err = DbError; + + fn from_str(value: &str) -> std::result::Result { + match value { + "submitted" => Ok(Self::Submitted), + "inventoried" => Ok(Self::Inventoried), + "approved" => Ok(Self::Approved), + "fenced" => Ok(Self::Fenced), + "drained" => Ok(Self::Drained), + "bindings_removed" => Ok(Self::BindingsRemoved), + "postgres_purged" => Ok(Self::PostgresPurged), + "cache_purged" => Ok(Self::CachePurged), + "logically_verified" => Ok(Self::LogicallyVerified), + "retention_pending" => Ok(Self::RetentionPending), + other => Err(DbError::DeletionSafety(format!( + "unknown community deletion stage: {other}" + ))), + } + } +} + +/// Durable community deletion request. +#[derive(Debug, Clone, Serialize)] +pub struct DeletionRequest { + /// Request identifier. + pub id: Uuid, + /// Target community. + #[serde(serialize_with = "serialize_community_id")] + pub community_id: CommunityId, + /// Permanently reserved canonical host. + pub community_host: String, + /// Current lifecycle stage. + pub stage: DeletionStage, + /// Operator identity that submitted the request. + pub requested_by: String, + /// Optional request reason. + pub reason: Option, + /// Frozen catalog manifest. + pub schema_manifest: Option, + /// Frozen community-prefix storage manifest observed at submission. + pub storage_manifest: Option, + /// Destructive storage manifest frozen after the durable fence. + pub destructive_storage_manifest: Option, + /// Frozen inventory aggregate. + pub inventory_manifest: Option, + /// Hex SHA-256 of the frozen inventory. + pub inventory_digest: Option, + /// Durable community fence generation. + pub fence_generation: Option, + /// Current claim owner. + pub lease_owner: Option, + /// Monotonic claim generation. + pub lease_generation: i64, + /// Claim expiry. + pub lease_until: Option>, + /// Number of claims. + pub attempts: i32, + /// Number of failed execution units. + pub retry_count: i32, + /// Last bounded error. + pub last_error: Option, + /// Permanent fail-closed block reason. + pub blocked_reason: Option, + /// Submission time. + pub created_at: DateTime, + /// Last lifecycle update. + pub updated_at: DateTime, + /// Terminal logical-deletion time. + pub completed_at: Option>, +} + +/// Frozen PostgreSQL catalog inventory. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SchemaManifest { + /// Sorted community-scoped table names. + pub scoped_tables: Vec, + /// Per-table row counts for the target. + pub row_counts: BTreeMap, + /// Sorted tables with the universal write-fence trigger. + pub fenced_tables: Vec, +} + +/// Frozen storage inventory supplied by the object-store adapter: slim +/// per-prefix summaries for the target community. The concrete key list never +/// lives on the request row — the destructive freeze persists it as chunked +/// `community_deletion_manifest_keys` rows that must hash to these digests. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StorageManifest { + /// Adapter schema version. + pub version: i32, + /// Per-prefix frozen summaries, strictly sorted by prefix. + pub prefixes: Vec, +} + +/// Frozen summary of one community-scoped key prefix. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PrefixManifest { + /// Exact community-scoped listing prefix. + pub prefix: String, + /// Objects under the prefix at enumeration time. + pub object_count: u64, + /// Total object bytes under the prefix at enumeration time. + pub total_bytes: u64, + /// Hex SHA-256 of the newline-terminated ascending key stream. + pub keys_digest: String, +} + +/// One frozen chunk of the destructive key list. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ManifestKeyChunk { + /// Position in the frozen chunk sequence. + pub chunk_no: i64, + /// The tenant prefix every key in this chunk lives under. + pub prefix: String, + /// Strictly ascending keys. + pub keys: Vec, +} + +/// One durable fleet-wide object-store taxonomy sweep record. +#[derive(Debug, Clone, Serialize)] +pub struct TaxonomySweep { + /// Sweep identity. + pub id: Uuid, + /// Listing start time. + pub started_at: DateTime, + /// Record time. + pub completed_at: DateTime, + /// Total objects listed. + pub listed_objects: i64, + /// Exact count of keys outside the known writer taxonomy. + pub unknown_object_count: i64, + /// Bounded sample of unknown keys. + pub unknown_key_sample: Vec, + /// Fleet object cap the sweep ran under. + pub object_cap: i64, +} + +type TaxonomySweepRow = ( + Uuid, + DateTime, + DateTime, + i64, + i64, + sqlx::types::Json>, + i64, +); + +/// Streaming SHA-256 over a strictly ascending key stream. +/// +/// The executor's prefix enumeration and the destructive freeze's chunk +/// validation both fold keys through this, so "the chunk rows are exactly +/// the frozen enumeration" reduces to digest equality. Each key is hashed +/// with a trailing newline so concatenation cannot alias two streams. +pub struct KeyStreamDigest { + hasher: Sha256, + last: Option, + count: u64, +} + +impl Default for KeyStreamDigest { + fn default() -> Self { + Self::new() + } +} + +impl KeyStreamDigest { + /// Start an empty stream. + pub fn new() -> Self { + Self { + hasher: Sha256::new(), + last: None, + count: 0, + } + } + + /// Fold the next key. Keys must arrive strictly ascending — S3 + /// `ListObjectsV2` order — so one out-of-order or duplicate key fails + /// closed instead of silently producing a different digest. + pub fn fold(&mut self, key: &str) -> Result<()> { + if self.last.as_deref().is_some_and(|last| last >= key) { + return Err(DbError::DeletionSafety(format!( + "storage key stream is not strictly ascending at {key}" + ))); + } + self.hasher.update(key.as_bytes()); + self.hasher.update(b"\n"); + self.last = Some(key.to_owned()); + self.count += 1; + Ok(()) + } + + /// Hex digest and key count of everything folded. + pub fn finish(self) -> (String, u64) { + (hex::encode(self.hasher.finalize()), self.count) + } +} + +/// Full frozen inventory approved at the destructive boundary. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FrozenInventory { + /// PostgreSQL catalog state. + pub schema: SchemaManifest, + /// Object-store state. + pub storage: StorageManifest, +} + +impl FrozenInventory { + /// Canonical JSON bytes and SHA-256 digest used to bind approval. + pub fn digest(&self) -> Result> { + Ok(Sha256::digest(serde_json::to_vec(self)?).to_vec()) + } +} + +/// One durable unit checkpoint. +#[derive(Debug, Clone, Serialize)] +pub struct DeletionCheckpoint { + /// Stage containing the unit. + pub stage: String, + /// Stable unit key. + pub unit_key: String, + /// `started`, `completed`, or `failed`. + pub status: String, + /// Claim generation that last touched it. + pub lease_generation: i64, + /// Attempt count for this unit. + pub attempts: i32, + /// Structured bounded details. + pub detail: serde_json::Value, + /// Last failure. + pub error: Option, + /// Start time. + pub started_at: DateTime, + /// Completion time. + pub completed_at: Option>, +} + +/// Full inspect response. +#[derive(Debug, Clone, Serialize)] +pub struct DeletionInspection { + /// Durable request. + pub request: DeletionRequest, + /// Explicit approval evidence, if present. + pub approval: Option, + /// Unit checkpoints. + pub checkpoints: Vec, +} + +/// Explicit approval evidence. +#[derive(Debug, Clone, Serialize)] +pub struct DeletionApproval { + /// Hex frozen inventory digest. + pub inventory_digest: String, + /// Approving operator identity. + pub approved_by: String, + /// Optional approval note. + pub note: Option, + /// Approval timestamp. + pub approved_at: DateTime, +} + +/// Monotonic lease token required by every execution mutation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LeaseToken { + /// Request id. + pub request_id: Uuid, + /// Executor identity. + pub owner: String, + /// Monotonic lease generation. + pub generation: i64, + /// Target community. + pub community_id: CommunityId, + /// Community fence generation, once fenced. + pub fence_generation: Option, +} + +/// A claimed request with its durable token. +#[derive(Debug, Clone)] +pub struct ClaimedDeletion { + /// Request snapshot. + pub request: DeletionRequest, + /// Required token. + pub lease: LeaseToken, +} + +/// Short-lived durable lease for an external serving side effect. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServingWriteLease { + /// Lease row identifier. + pub id: Uuid, + /// Community protected by this lease. + pub community_id: CommunityId, + /// Operation category for diagnostics. + pub operation: String, + /// Process/executor identity. + pub owner: String, + /// Monotonic lease generation. + pub generation: i64, + /// Community fence generation observed when the lease was acquired. + pub fence_generation: i64, + /// Lease expiry. + pub lease_until: DateTime, +} + +/// Validate the minimum catalog contract used by serving-path fences. +pub const REQUIRED_SERVING_TABLES: &[&str] = &[ + "communities", + "community_serving_write_leases", + "community_deletion_requests", +]; + +/// Bounded-cardinality operational snapshot for the hot serving-lease table. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ServingLeaseStats { + /// Unexpired serving-write leases. + pub active: i64, + /// Expired rows awaiting cleanup. + pub expired: i64, + /// PostgreSQL's estimated dead tuples for the lease table. + pub dead_tuples: i64, +} + +/// PostgreSQL deletion adapter. Clone is cheap. +#[derive(Clone)] +pub struct DeletionStore { + pool: PgPool, +} + +impl DeletionStore { + /// Construct from the writer pool used by [`crate::Db`]. + pub(crate) fn new(pool: PgPool) -> Self { + Self { pool } + } + + /// Check deletion control-plane/schema connectivity. + /// + /// Probe the deployed catalog rather than SQLx's migration ledger. Buzz also + /// supports desired-state schema application through `pgschema`, which creates + /// the same deletion objects without creating `_sqlx_migrations`. + pub async fn ping(&self) -> bool { + sqlx::query_scalar::<_, bool>( + "SELECT to_regclass('community_deletion_requests') IS NOT NULL", + ) + .fetch_one(&self.pool) + .await + .unwrap_or(false) + } + + /// Persist a request. Only active non-tombstone communities may be submitted. + pub async fn submit( + &self, + community_host: &str, + requested_by: &str, + reason: Option<&str>, + ) -> Result { + let row = sqlx::query( + r#" + WITH target AS ( + SELECT id, host + FROM communities + WHERE lower(host) = lower($1) + AND deletion_state = 'active' + AND deleted_at IS NULL + ), inserted AS ( + INSERT INTO community_deletion_requests + (community_id, community_host, requested_by, reason) + SELECT id, host, $2, $3 FROM target + ON CONFLICT (community_id) DO NOTHING + RETURNING * + ) + SELECT * FROM inserted + UNION ALL + SELECT request.* + FROM community_deletion_requests request + JOIN target ON target.id = request.community_id + WHERE request.stage = 'submitted' + AND request.requested_by = $2 + AND NOT EXISTS (SELECT 1 FROM inserted) + LIMIT 1 + "#, + ) + .bind(community_host) + .bind(requested_by) + .bind(reason) + .fetch_optional(&self.pool) + .await?; + match row { + Some(row) => row_to_request(row), + None => Err(DbError::DeletionSafety(format!( + "community {community_host:?} is missing, already requested, fenced, or tombstoned" + ))), + } + } + + /// List requests newest first with a hard bound. + pub async fn list(&self, limit: i64) -> Result> { + let rows = sqlx::query( + "SELECT * FROM community_deletion_requests ORDER BY created_at DESC LIMIT $1", + ) + .bind(limit.clamp(1, 1000)) + .fetch_all(&self.pool) + .await?; + rows.into_iter().map(row_to_request).collect() + } + + /// Read one request. + pub async fn get(&self, request_id: Uuid) -> Result { + let row = sqlx::query("SELECT * FROM community_deletion_requests WHERE id = $1") + .bind(request_id) + .fetch_optional(&self.pool) + .await? + .ok_or_else(|| DbError::NotFound(format!("community deletion {request_id}")))?; + row_to_request(row) + } + + /// Inspect request, approval, checkpoints, and retention holds. + pub async fn inspect(&self, request_id: Uuid) -> Result { + let request = self.get(request_id).await?; + let approval_row = sqlx::query( + "SELECT inventory_digest, approved_by, note, approved_at \ + FROM community_deletion_approvals WHERE request_id = $1", + ) + .bind(request_id) + .fetch_optional(&self.pool) + .await?; + let approval = approval_row + .map(|row| { + Ok::(DeletionApproval { + inventory_digest: hex::encode(row.try_get::, _>("inventory_digest")?), + approved_by: row.try_get("approved_by")?, + note: row.try_get("note")?, + approved_at: row.try_get("approved_at")?, + }) + }) + .transpose()?; + let checkpoints = sqlx::query( + "SELECT stage, unit_key, status, lease_generation, attempts, detail, error, \ + started_at, completed_at \ + FROM community_deletion_checkpoints WHERE request_id = $1 ORDER BY sequence", + ) + .bind(request_id) + .fetch_all(&self.pool) + .await? + .into_iter() + .map(|row| { + Ok(DeletionCheckpoint { + stage: row.try_get("stage")?, + unit_key: row.try_get("unit_key")?, + status: row.try_get("status")?, + lease_generation: row.try_get("lease_generation")?, + attempts: row.try_get("attempts")?, + detail: row.try_get("detail")?, + error: row.try_get("error")?, + started_at: row.try_get("started_at")?, + completed_at: row.try_get("completed_at")?, + }) + }) + .collect::>>()?; + Ok(DeletionInspection { + request, + approval, + checkpoints, + }) + } + + /// Validate the deletion catalog contract required by relay serving. + pub async fn validate_serving_catalog(&self) -> Result<()> { + let runtime_columns = sqlx::query( + "SELECT attname, format_type(atttypid, atttypmod) AS type_name, attnotnull \ + FROM pg_attribute WHERE attrelid = 'communities'::regclass \ + AND attname IN ('deletion_state', 'deletion_fence_generation', 'deleted_at') \ + AND NOT attisdropped ORDER BY attname", + ) + .fetch_all(&self.pool) + .await?; + let column_contract = runtime_columns + .iter() + .map(|row| { + Ok::<_, DbError>(( + row.try_get::("attname")?, + row.try_get::("type_name")?, + row.try_get::("attnotnull")?, + )) + }) + .collect::>>()?; + let expected_columns = BTreeSet::from([ + ( + "deleted_at".to_string(), + "timestamp with time zone".to_string(), + false, + ), + ( + "deletion_fence_generation".to_string(), + "bigint".to_string(), + true, + ), + ("deletion_state".to_string(), "text".to_string(), true), + ]); + if column_contract != expected_columns { + return Err(DbError::DeletionSafety( + "community serving fence columns are missing or incompatible".to_string(), + )); + } + + let required_tables = REQUIRED_SERVING_TABLES + .iter() + .copied() + .map(str::to_owned) + .collect::>(); + let required_table_names = REQUIRED_SERVING_TABLES + .iter() + .map(ToString::to_string) + .collect::>(); + let live_tables: BTreeSet = sqlx::query_scalar( + "SELECT table_name FROM information_schema.tables \ + WHERE table_schema = 'public' AND table_name = ANY($1) \ + ORDER BY table_name", + ) + .bind(&required_table_names) + .fetch_all(&self.pool) + .await? + .into_iter() + .collect(); + if live_tables != required_tables { + return Err(DbError::DeletionSafety(format!( + "community serving fence tables missing: {}", + required_tables + .difference(&live_tables) + .cloned() + .collect::>() + .join(",") + ))); + } + + let required_fences = EXPECTED_SCOPED_TABLES + .iter() + .copied() + .map(str::to_owned) + .collect::>(); + let live_fences = self.live_fenced_tables().await?; + let missing_fences = required_fences + .difference(&live_fences) + .cloned() + .collect::>(); + if !missing_fences.is_empty() { + return Err(DbError::DeletionSafety(format!( + "community serving write fences missing: {}", + missing_fences.join(",") + ))); + } + + let required_objects_present: bool = sqlx::query_scalar( + "SELECT to_regprocedure('community_deletion_lock_key(uuid)') IS NOT NULL \ + AND to_regprocedure('assert_community_write_allowed(uuid)') IS NOT NULL \ + AND to_regprocedure('enforce_community_write_fence()') IS NOT NULL \ + AND EXISTS (SELECT 1 FROM pg_trigger t \ + JOIN pg_class c ON c.oid = t.tgrelid \ + JOIN pg_proc p ON p.oid = t.tgfoid \ + WHERE c.relname = 'communities' \ + AND p.proname = 'enforce_community_tombstone' \ + AND NOT t.tgisinternal AND t.tgenabled = 'O')", + ) + .fetch_one(&self.pool) + .await?; + if !required_objects_present { + return Err(DbError::DeletionSafety( + "community serving fence functions or tombstone trigger are missing".to_string(), + )); + } + Ok(()) + } + + /// Validate the exact live scoped-table and write-fence catalog for destruction. + /// + /// Exact table and fence equality rejects unknown tenant data even + /// while unrelated SQLx migrations continue to advance. + pub async fn validate_catalog(&self) -> Result<()> { + let expected = EXPECTED_SCOPED_TABLES + .iter() + .copied() + .map(str::to_owned) + .collect::>(); + let live_tables = self.live_scoped_tables().await?; + if live_tables != expected { + let missing = expected + .difference(&live_tables) + .cloned() + .collect::>(); + let unknown = live_tables + .difference(&expected) + .cloned() + .collect::>(); + return Err(DbError::DeletionSafety(format!( + "community deletion catalog drift (missing={}, unknown={})", + missing.join(","), + unknown.join(",") + ))); + } + + let fenced_tables = self.live_fenced_tables().await?; + if fenced_tables != expected { + let missing = expected + .difference(&fenced_tables) + .cloned() + .collect::>(); + let unknown = fenced_tables + .difference(&expected) + .cloned() + .collect::>(); + return Err(DbError::DeletionSafety(format!( + "community deletion write-fence drift (missing={}, unknown={})", + missing.join(","), + unknown.join(",") + ))); + } + Ok(()) + } + + /// Build and validate a live PostgreSQL schema inventory. + pub async fn inventory_schema(&self, community: CommunityId) -> Result { + self.validate_catalog().await?; + let live_tables = self.live_scoped_tables().await?; + let fenced_tables = self.live_fenced_tables().await?; + let _ = community; // counts are intentionally not approval-bound for a live tenant. + Ok(SchemaManifest { + scoped_tables: live_tables.into_iter().collect(), + row_counts: BTreeMap::new(), + fenced_tables: fenced_tables.into_iter().collect(), + }) + } + + /// Freeze inventory and move submitted → inventoried atomically. + pub async fn freeze_inventory( + &self, + request_id: Uuid, + inventory: &FrozenInventory, + ) -> Result { + validate_storage_manifest(&inventory.storage)?; + let digest = inventory.digest()?; + let schema = serde_json::to_value(&inventory.schema)?; + let storage = serde_json::to_value(&inventory.storage)?; + let frozen = serde_json::to_value(inventory)?; + let row = sqlx::query( + r#" + UPDATE community_deletion_requests + SET stage = 'inventoried', schema_manifest = $2, storage_manifest = $3, + inventory_manifest = $4, inventory_digest = $5, + inventory_frozen_at = now(), updated_at = now(), + last_error = NULL, last_error_at = NULL + WHERE id = $1 AND stage = 'submitted' AND blocked_at IS NULL + RETURNING * + "#, + ) + .bind(request_id) + .bind(schema) + .bind(storage) + .bind(frozen) + .bind(digest) + .fetch_optional(&self.pool) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion {request_id} is not an unblocked submitted request" + )) + })?; + row_to_request(row) + } + + /// Approve the exact frozen inventory and move inventoried → approved. + pub async fn approve( + &self, + request_id: Uuid, + approved_by: &str, + note: Option<&str>, + ) -> Result { + let mut tx = self.pool.begin().await?; + let (community_id, digest, inventory_manifest): (Uuid, Vec, serde_json::Value) = + sqlx::query_as( + "SELECT community_id, inventory_digest, inventory_manifest \ + FROM community_deletion_requests \ + WHERE id = $1 AND stage = 'inventoried' AND blocked_at IS NULL FOR UPDATE", + ) + .bind(request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion {request_id} is not an unblocked inventoried request" + )) + })?; + let inventory: FrozenInventory = serde_json::from_value(inventory_manifest)?; + let recomputed_digest = inventory.digest()?; + if digest.as_slice() != recomputed_digest { + return Err(DbError::DeletionSafety(format!( + "deletion {request_id} frozen inventory digest does not match its manifest" + ))); + } + sqlx::query( + "INSERT INTO community_deletion_approvals \ + (request_id, community_id, inventory_digest, approved_by, note) \ + VALUES ($1, $2, $3, $4, $5)", + ) + .bind(request_id) + .bind(community_id) + .bind(&digest) + .bind(approved_by) + .bind(note) + .execute(&mut *tx) + .await?; + let row = sqlx::query( + "UPDATE community_deletion_requests \ + SET stage = 'approved', updated_at = now(), next_attempt_at = now() \ + WHERE id = $1 AND stage = 'inventoried' AND blocked_at IS NULL \ + RETURNING *", + ) + .bind(request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion {request_id} changed before approval could be recorded" + )) + })?; + tx.commit().await?; + row_to_request(row) + } + + /// Claim a specific runnable request. Expired claims may be reclaimed. + pub async fn claim_specific( + &self, + request_id: Uuid, + owner: &str, + lease_duration: Duration, + ) -> Result> { + self.claim(Some(request_id), owner, lease_duration).await + } + + /// Claim the oldest runnable request. Expired claims may be reclaimed. + pub async fn claim_next( + &self, + owner: &str, + lease_duration: Duration, + ) -> Result> { + self.claim(None, owner, lease_duration).await + } + + async fn claim( + &self, + request_id: Option, + owner: &str, + lease_duration: Duration, + ) -> Result> { + // Claim is the destructive worker boundary: unlike serving readiness, + // execution refuses any newer/unknown catalog until this engine knows it. + self.validate_catalog().await?; + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let row = sqlx::query( + r#" + WITH candidate AS ( + SELECT request.id + FROM community_deletion_requests request + JOIN community_deletion_approvals approval + ON approval.request_id = request.id + AND approval.community_id = request.community_id + AND approval.inventory_digest = request.inventory_digest + WHERE ($1::uuid IS NULL OR request.id = $1) + AND request.stage IN ('approved', 'fenced', 'drained', 'bindings_removed', + 'postgres_purged', 'cache_purged', 'logically_verified') + AND request.blocked_at IS NULL + AND request.next_attempt_at <= now() + AND (request.lease_until IS NULL OR request.lease_until < now()) + ORDER BY request.created_at, request.id + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + UPDATE community_deletion_requests request + SET lease_owner = $2, + lease_generation = request.lease_generation + 1, + lease_until = now() + make_interval(secs => $3), + attempts = request.attempts + 1, + updated_at = now() + FROM candidate + WHERE request.id = candidate.id + RETURNING request.* + "#, + ) + .bind(request_id) + .bind(owner) + .bind(lease_seconds) + .fetch_optional(&self.pool) + .await?; + row.map(|row| { + let request = row_to_request(row)?; + let lease = LeaseToken { + request_id: request.id, + owner: owner.to_owned(), + generation: request.lease_generation, + community_id: request.community_id, + fence_generation: request.fence_generation, + }; + Ok(ClaimedDeletion { request, lease }) + }) + .transpose() + } + + /// Verify that a deletion lease/fence token is still current for a stage. + pub async fn verify_execution_token( + &self, + token: &LeaseToken, + stage: DeletionStage, + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + if let Some(generation) = token.fence_generation { + verify_lease_and_fence(&mut tx, token, stage, generation).await?; + } else { + verify_lease(&mut tx, token, stage).await?; + } + tx.commit().await?; + Ok(()) + } + + /// Renew an owned claim and persist executor liveness. + pub async fn heartbeat( + &self, + token: &LeaseToken, + executor_mode: &str, + lease_duration: Duration, + draining: bool, + ) -> Result<()> { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + let affected = sqlx::query( + "UPDATE community_deletion_requests request \ + SET lease_until = now() + make_interval(secs => $4), updated_at = now() \ + WHERE request.id = $1 AND request.lease_owner = $2 \ + AND request.lease_generation = $3 AND request.lease_until >= now() \ + AND request.blocked_at IS NULL \ + AND request.stage IN ('approved', 'fenced', 'drained', 'bindings_removed', \ + 'postgres_purged', 'cache_purged', 'logically_verified') \ + AND EXISTS (SELECT 1 FROM community_deletion_approvals approval \ + WHERE approval.request_id = request.id \ + AND approval.community_id = request.community_id \ + AND approval.inventory_digest = request.inventory_digest)", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(lease_seconds) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(stale_lease_error(token)); + } + sqlx::query( + "INSERT INTO community_deletion_executor_heartbeats \ + (executor_id, mode, request_id, draining) VALUES ($1, $2, $3, $4) \ + ON CONFLICT (executor_id) DO UPDATE SET mode = EXCLUDED.mode, \ + request_id = EXCLUDED.request_id, heartbeat_at = now(), \ + draining = EXCLUDED.draining, stopped_at = NULL", + ) + .bind(&token.owner) + .bind(executor_mode) + .bind(token.request_id) + .bind(draining) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Mark an executor stopped and release its current claim if still owned. + pub async fn stop_executor(&self, token: Option<&LeaseToken>, executor_id: &str) -> Result<()> { + let mut tx = self.pool.begin().await?; + if let Some(token) = token { + sqlx::query( + "UPDATE community_deletion_requests \ + SET lease_owner = NULL, lease_until = NULL, updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .execute(&mut *tx) + .await?; + } + sqlx::query( + "UPDATE community_deletion_executor_heartbeats \ + SET request_id = NULL, draining = true, heartbeat_at = now(), stopped_at = now() \ + WHERE executor_id = $1", + ) + .bind(executor_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Persist quiescing intent before waiting for active serving leases. + /// + /// This is the irreversible fail-closed point: a request intentionally has + /// no automatic unquiesce/unblock transition after operator approval. + /// + /// The transition takes the same exclusive advisory lock as serving lease + /// acquisition, so after commit no newer external effect can be admitted. + /// Already-acquired leases remain verifiable/releasable but cannot renew. + pub async fn begin_quiescing(&self, token: &LeaseToken) -> Result<()> { + let mut tx = self.pool.begin().await?; + verify_lease(&mut tx, token, DeletionStage::Approved).await?; + sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await?; + let generation: i64 = sqlx::query_scalar( + "SELECT deletion_fence_generation FROM communities WHERE id = $1 FOR UPDATE", + ) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + set_executor_gucs(&mut tx, token.community_id, generation).await?; + let affected = sqlx::query( + "UPDATE communities SET deletion_state = 'quiescing', \ + archived_at = COALESCE(archived_at, now()) \ + WHERE id = $1 AND deletion_state IN ('active', 'quiescing') \ + AND deleted_at IS NULL", + ) + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(DbError::DeletionSafety(format!( + "community {} cannot enter quiescing", + token.community_id + ))); + } + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Approved, + "quiesce_serving_writes", + serde_json::json!({"community_state": "quiescing"}), + ) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Acquire the universal durable fence after all pre-quiesce serving leases drain. + pub async fn fence(&self, token: &LeaseToken) -> Result { + let mut tx = self.pool.begin().await?; + verify_lease(&mut tx, token, DeletionStage::Approved).await?; + sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await?; + let active_serving_writes = sqlx::query( + "SELECT count(*)::BIGINT AS active_count, \ + COALESCE(array_agg(DISTINCT operation ORDER BY operation), ARRAY[]::TEXT[]) AS operations \ + FROM community_serving_write_leases \ + WHERE community_id = $1 AND lease_until >= now()", + ) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + let active_count: i64 = active_serving_writes.try_get("active_count")?; + if active_count > 0 { + return Err(DbError::ServingWritesNotDrained { + community_id: *token.community_id.as_uuid(), + active_count, + operations: active_serving_writes.try_get("operations")?, + }); + } + let current_generation: i64 = sqlx::query_scalar( + "SELECT deletion_fence_generation FROM communities WHERE id = $1 FOR UPDATE", + ) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + let generation = current_generation.checked_add(1).ok_or_else(|| { + DbError::DeletionSafety("community deletion fence generation overflow".to_string()) + })?; + set_executor_gucs(&mut tx, token.community_id, generation).await?; + let affected = sqlx::query( + "UPDATE communities SET deletion_state = 'fenced', \ + deletion_fence_generation = $2, archived_at = COALESCE(archived_at, now()) \ + WHERE id = $1 AND deletion_state = 'quiescing'", + ) + .bind(token.community_id.as_uuid()) + .bind(generation) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(DbError::DeletionSafety(format!( + "community {} is no longer quiescing while fencing", + token.community_id + ))); + } + advance_request_tx( + &mut tx, + token, + DeletionStage::Approved, + DeletionStage::Fenced, + Some(generation), + ) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Approved, + "activate_fence", + serde_json::json!({"fence_generation": generation}), + ) + .await?; + tx.commit().await?; + Ok(generation) + } + + /// Freeze the exact post-fence storage binding manifest. + pub async fn freeze_destructive_storage_manifest( + &self, + token: &LeaseToken, + manifest: &StorageManifest, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + // Serialize the freeze boundary with chunk INSERTs. The database trigger + // takes the same request-row lock before admitting each new chunk. + sqlx::query("SELECT id FROM community_deletion_requests WHERE id = $1 FOR UPDATE") + .bind(token.request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion request {} disappeared before manifest freeze", + token.request_id + )) + })?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + validate_storage_manifest(manifest)?; + // The chunk rows are the concrete delete list; the freeze commits only + // if they hash to the manifest's frozen per-prefix digests. Loading the + // full chunk stream is a one-time freeze-boundary cost proportional to + // this community's bindings, never the fleet bucket. + let chunks: Vec<(i64, String, sqlx::types::Json>)> = sqlx::query_as( + "SELECT chunk_no, prefix, keys FROM community_deletion_manifest_keys \ + WHERE request_id = $1 ORDER BY chunk_no", + ) + .bind(token.request_id) + .fetch_all(&mut *tx) + .await?; + validate_manifest_key_chunks(manifest, &chunks)?; + let affected = sqlx::query( + "UPDATE community_deletion_requests \ + SET destructive_storage_manifest = COALESCE(destructive_storage_manifest, $4), \ + destructive_storage_frozen_at = COALESCE(destructive_storage_frozen_at, now()), \ + updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3 \ + AND stage = 'fenced' \ + AND (destructive_storage_manifest IS NULL \ + OR destructive_storage_manifest = $4) \ + RETURNING id", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(serde_json::to_value(manifest)?) + .fetch_optional(&mut *tx) + .await?; + if affected.is_none() { + return Err(DbError::DeletionSafety(format!( + "destructive storage manifest changed or deletion lease is stale for request {}", + token.request_id + ))); + } + tx.commit().await?; + Ok(()) + } + + /// Remove key chunks left by an interrupted destructive freeze. + /// + /// The chunk-table guard rejects this once the destructive manifest has + /// frozen, so a retried freeze can only rewrite chunks that were never + /// bound to a committed manifest. + pub async fn clear_manifest_key_chunks(&self, token: &LeaseToken) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + sqlx::query("DELETE FROM community_deletion_manifest_keys WHERE request_id = $1") + .bind(token.request_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Append one immutable chunk of the destructive key list. + pub async fn append_manifest_key_chunk( + &self, + token: &LeaseToken, + chunk_no: i64, + prefix: &str, + keys: &[String], + ) -> Result<()> { + if keys.is_empty() { + return Err(DbError::DeletionSafety( + "refusing to persist an empty manifest key chunk".to_string(), + )); + } + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + sqlx::query( + "INSERT INTO community_deletion_manifest_keys \ + (request_id, chunk_no, prefix, keys) VALUES ($1, $2, $3, $4)", + ) + .bind(token.request_id) + .bind(chunk_no) + .bind(prefix) + .bind(sqlx::types::Json(keys)) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Return the next frozen chunk not yet confirmed deleted, in chunk order. + pub async fn next_pending_manifest_chunk( + &self, + token: &LeaseToken, + ) -> Result> { + let row: Option<(i64, String, sqlx::types::Json>)> = sqlx::query_as( + "SELECT chunk_no, prefix, keys FROM community_deletion_manifest_keys \ + WHERE request_id = $1 AND deleted_at IS NULL ORDER BY chunk_no LIMIT 1", + ) + .bind(token.request_id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|(chunk_no, prefix, keys)| ManifestKeyChunk { + chunk_no, + prefix, + keys: keys.0, + })) + } + + /// Return `(total, deleted)` chunk counts for one request. + pub async fn manifest_chunk_progress(&self, request_id: Uuid) -> Result<(i64, i64)> { + sqlx::query_as( + "SELECT count(*), count(deleted_at) FROM community_deletion_manifest_keys \ + WHERE request_id = $1", + ) + .bind(request_id) + .fetch_one(&self.pool) + .await + .map_err(Into::into) + } + + /// Stamp one chunk's keys durably removed and checkpoint it atomically. + pub async fn mark_manifest_chunk_deleted( + &self, + token: &LeaseToken, + chunk_no: i64, + detail: serde_json::Value, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Drained, generation).await?; + let affected = sqlx::query( + "UPDATE community_deletion_manifest_keys SET deleted_at = now() \ + WHERE request_id = $1 AND chunk_no = $2 AND deleted_at IS NULL", + ) + .bind(token.request_id) + .bind(chunk_no) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(DbError::DeletionSafety(format!( + "manifest key chunk {chunk_no} is missing or already stamped for request {}", + token.request_id + ))); + } + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Drained, + &format!("chunk:{chunk_no}"), + detail, + ) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Record one completed fleet-wide taxonomy sweep. + pub async fn record_taxonomy_sweep( + &self, + started_at: DateTime, + listed_objects: u64, + unknown_object_count: u64, + unknown_key_sample: &[String], + object_cap: u64, + ) -> Result { + let listed = i64::try_from(listed_objects) + .map_err(|_| DbError::DeletionSafety("sweep object count overflow".to_string()))?; + let unknown = i64::try_from(unknown_object_count) + .map_err(|_| DbError::DeletionSafety("sweep unknown count overflow".to_string()))?; + let cap = i64::try_from(object_cap) + .map_err(|_| DbError::DeletionSafety("sweep object cap overflow".to_string()))?; + // Completion is authoritative database time. Small positive sweeper + // skew is clamped at that boundary; materially future starts are rejected. + let row: Option<(Uuid, DateTime, DateTime)> = sqlx::query_as( + "INSERT INTO storage_taxonomy_sweeps \ + (started_at, completed_at, listed_objects, unknown_object_count, \ + unknown_key_sample, object_cap) \ + SELECT LEAST($1, db_now), db_now, $2, $3, $4, $5 \ + FROM (SELECT clock_timestamp() AS db_now) clock \ + WHERE $1 <= db_now + interval '5 minutes' \ + RETURNING id, started_at, completed_at", + ) + .bind(started_at) + .bind(listed) + .bind(unknown) + .bind(sqlx::types::Json(unknown_key_sample)) + .bind(cap) + .fetch_optional(&self.pool) + .await?; + let (id, started_at, completed_at) = row.ok_or_else(|| { + DbError::DeletionSafety( + "taxonomy sweep start time is more than five minutes in the future".to_string(), + ) + })?; + Ok(TaxonomySweep { + id, + started_at, + completed_at, + listed_objects: listed, + unknown_object_count: unknown, + unknown_key_sample: unknown_key_sample.to_vec(), + object_cap: cap, + }) + } + + /// Return the most recently completed taxonomy sweep, if any. + pub async fn latest_taxonomy_sweep(&self) -> Result> { + let row: Option = sqlx::query_as( + "SELECT id, started_at, completed_at, listed_objects, unknown_object_count, \ + unknown_key_sample, object_cap \ + FROM storage_taxonomy_sweeps ORDER BY completed_at DESC LIMIT 1", + ) + .fetch_optional(&self.pool) + .await?; + Ok(row.map( + |(id, started_at, completed_at, listed, unknown, sample, cap)| TaxonomySweep { + id, + started_at, + completed_at, + listed_objects: listed, + unknown_object_count: unknown, + unknown_key_sample: sample.0, + object_cap: cap, + }, + )) + } + + /// Return whether all pre-fence external side-effect leases have expired or released. + pub async fn serving_writes_drained(&self, community: CommunityId) -> Result { + sqlx::query_scalar( + "SELECT NOT EXISTS(SELECT 1 FROM community_serving_write_leases \ + WHERE community_id = $1 AND lease_until >= now())", + ) + .bind(community.as_uuid()) + .fetch_one(&self.pool) + .await + .map_err(Into::into) + } + + /// Verify fence ownership and record that serving writes drained. + pub async fn mark_drained(&self, token: &LeaseToken) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + let active_serving_writes: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases \ + WHERE community_id = $1 AND lease_until >= now())", + ) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + if active_serving_writes { + return Err(DbError::DeletionSafety( + "serving writes have not drained".to_string(), + )); + } + advance_request_tx( + &mut tx, + token, + DeletionStage::Fenced, + DeletionStage::Drained, + Some(generation), + ) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Fenced, + "serving_writes_drained", + serde_json::json!({"fence_generation": generation}), + ) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Mark storage binding removal after adapter verification. + pub async fn mark_bindings_removed( + &self, + token: &LeaseToken, + detail: serde_json::Value, + ) -> Result<()> { + self.advance_with_checkpoint( + token, + DeletionStage::Drained, + DeletionStage::BindingsRemoved, + "remove_storage_bindings", + detail, + ) + .await + } + + /// Purge every scoped PostgreSQL table, preserve the community tombstone, and + /// move bindings_removed → postgres_purged in one transaction. + pub async fn purge_postgres(&self, token: &LeaseToken) -> Result> { + let generation = require_fence_generation(token)?; + // Re-inventory before opening the purge transaction. Any drift blocks; + // the transaction then locks the control and tombstone rows and all + // tenant writes are already fenced. + self.inventory_schema(token.community_id).await?; + + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::BindingsRemoved, generation).await?; + set_executor_gucs(&mut tx, token.community_id, generation).await?; + // Migration 0011 fences hard deletion of NIP-RS rows against legacy + // writers. Whole-community deletion is an intentional hard-delete path, + // and the transaction is already bound to an approved, fenced tenant. + sqlx::query("SELECT set_config('buzz.nip_rs_hard_delete', 'on', true)") + .execute(&mut *tx) + .await?; + + let mut deleted = BTreeMap::new(); + // The order is child-before-parent/FK-safe, not alphabetical. Cascades + // can make later units observe zero rows; each scoped WHERE stays idempotent. + for table in PURGE_SCOPED_TABLES { + let sql = format!("DELETE FROM {table} WHERE community_id = $1"); + let affected = sqlx::query(AssertSqlSafe(sql)) + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await? + .rows_affected(); + deleted.insert((*table).to_owned(), affected); + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::BindingsRemoved, + &format!("purge:{table}"), + serde_json::json!({"rows": affected}), + ) + .await?; + } + + let affected = sqlx::query( + "UPDATE communities SET deletion_state = 'tombstone', \ + deleted_at = COALESCE(deleted_at, now()), \ + archived_at = COALESCE(archived_at, now()), \ + signing_key = NULL, icon = NULL \ + WHERE id = $1 AND deletion_state = 'fenced' \ + AND deletion_fence_generation = $2", + ) + .bind(token.community_id.as_uuid()) + .bind(generation) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(DbError::DeletionSafety(format!( + "community {} tombstone update affected {affected} rows", + token.community_id + ))); + } + advance_request_tx( + &mut tx, + token, + DeletionStage::BindingsRemoved, + DeletionStage::PostgresPurged, + Some(generation), + ) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::BindingsRemoved, + "postgres_tombstone_committed", + serde_json::to_value(&deleted)?, + ) + .await?; + tx.commit().await?; + Ok(deleted) + } + + /// Mark cache purge after Redis adapter verification. + pub async fn mark_cache_purged( + &self, + token: &LeaseToken, + detail: serde_json::Value, + ) -> Result<()> { + self.advance_with_checkpoint( + token, + DeletionStage::PostgresPurged, + DeletionStage::CachePurged, + "purge_cache_namespace", + detail, + ) + .await + } + + /// Verify PostgreSQL logical absence without advancing the cross-store stage. + /// + /// The caller must verify object storage and Redis too, then call + /// [`Self::mark_logically_verified`]. Keeping the transition separate makes + /// a crash after any partial verification safely repeat the whole proof. + pub async fn verify_postgres_logically_deleted(&self, token: &LeaseToken) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::CachePurged, generation).await?; + let tombstone: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 \ + AND deletion_state = 'tombstone' AND deleted_at IS NOT NULL \ + AND deletion_fence_generation = $2)", + ) + .bind(token.community_id.as_uuid()) + .bind(generation) + .fetch_one(&mut *tx) + .await?; + if !tombstone { + return Err(DbError::DeletionSafety(format!( + "community {} tombstone/fence verification failed", + token.community_id + ))); + } + for table in EXPECTED_SCOPED_TABLES { + let sql = + format!("SELECT EXISTS(SELECT 1 FROM {table} WHERE community_id = $1 LIMIT 1)"); + let remains: bool = sqlx::query_scalar(AssertSqlSafe(sql)) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + if remains { + return Err(DbError::DeletionSafety(format!( + "logical verification found tenant rows in {table}" + ))); + } + } + tx.commit().await?; + Ok(()) + } + + /// Commit the cross-store logical verification checkpoint and drop the + /// frozen key chunks in the same transaction. + /// + /// The chunk rows are working data, not audit evidence — per-prefix + /// counts, digests, and checkpoint history stay on the request row, and + /// the raw key list of a deleted community should not be retained. + /// Blocked requests never reach this transition, so their chunks survive + /// for resumption or operator inspection. + pub async fn mark_logically_verified( + &self, + token: &LeaseToken, + detail: serde_json::Value, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::CachePurged, generation).await?; + advance_request_tx( + &mut tx, + token, + DeletionStage::CachePurged, + DeletionStage::LogicallyVerified, + Some(generation), + ) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::CachePurged, + "verify_cross_store_absence", + detail, + ) + .await?; + sqlx::query("DELETE FROM community_deletion_manifest_keys WHERE request_id = $1") + .bind(token.request_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Finish logical deletion and enter the physical-expiry pending state. + pub async fn mark_retention_pending( + &self, + token: &LeaseToken, + detail: serde_json::Value, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::LogicallyVerified, generation) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::LogicallyVerified, + "retention_physical_expiry_pending", + detail, + ) + .await?; + let affected = sqlx::query( + "UPDATE community_deletion_requests \ + SET stage = 'retention_pending', completed_at = now(), updated_at = now(), \ + lease_owner = NULL, lease_until = NULL, last_error = NULL, last_error_at = NULL \ + WHERE id = $1 AND stage = 'logically_verified' \ + AND lease_owner = $2 AND lease_generation = $3 AND lease_until >= now() \ + AND fence_generation = $4", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(generation) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(stale_lease_error(token)); + } + tx.commit().await?; + Ok(()) + } + + /// Persist a retryable unit failure and release the claim. + pub async fn record_retry( + &self, + token: &LeaseToken, + stage: DeletionStage, + unit_key: &str, + error: &str, + retry_after: Duration, + ) -> Result<()> { + let bounded = bound_text(error, 4096); + let retry_seconds = i64::try_from(retry_after.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + verify_lease(&mut tx, token, stage).await?; + checkpoint_failed_tx(&mut tx, token, stage, unit_key, &bounded).await?; + sqlx::query( + "UPDATE community_deletion_requests \ + SET retry_count = retry_count + 1, last_error = $4, last_error_at = now(), \ + next_attempt_at = now() + make_interval(secs => $5), \ + lease_owner = NULL, lease_until = NULL, updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(&bounded) + .bind(retry_seconds) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Persist a fail-closed permanent block and release the claim. + pub async fn block( + &self, + token: &LeaseToken, + stage: DeletionStage, + unit_key: &str, + error: &str, + ) -> Result<()> { + let bounded = bound_text(error, 4096); + let mut tx = self.pool.begin().await?; + verify_lease(&mut tx, token, stage).await?; + checkpoint_failed_tx(&mut tx, token, stage, unit_key, &bounded).await?; + sqlx::query( + "UPDATE community_deletion_requests \ + SET blocked_at = now(), blocked_reason = $4, last_error = $4, \ + last_error_at = now(), lease_owner = NULL, lease_until = NULL, updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(&bounded) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Take the shared community deletion lock inside an existing transaction. + pub async fn guard_transaction( + &self, + tx: &mut Transaction<'_, Postgres>, + community: CommunityId, + ) -> Result<()> { + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(community.as_uuid()) + .execute(&mut **tx) + .await?; + let state: Option = sqlx::query_scalar( + "SELECT deletion_state FROM communities WHERE id = $1 AND deleted_at IS NULL", + ) + .bind(community.as_uuid()) + .fetch_optional(&mut **tx) + .await?; + match state.as_deref() { + Some("active") => Ok(()), + Some(other) => Err(DbError::AccessDenied(format!( + "community {community} is write-fenced ({other})" + ))), + None => Err(DbError::AccessDenied(format!( + "community {community} is missing or tombstoned" + ))), + } + } + + /// Take the shared community deletion lock inside an existing transaction + /// and authorize a final mutation under an already-admitted serving lease. + /// + /// The lease is checked in the same transaction as the mutation. During + /// quiescing, only this exact unexpired lease and fence generation may + /// finish; active communities continue to accept the admitted write too. + pub async fn guard_transaction_with_serving_lease( + &self, + tx: &mut Transaction<'_, Postgres>, + lease: &ServingWriteLease, + ) -> Result<()> { + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(lease.community_id.as_uuid()) + .execute(&mut **tx) + .await?; + let valid: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases lease \ + JOIN communities community ON community.id = lease.community_id \ + WHERE lease.id = $1 AND lease.community_id = $2 AND lease.owner = $3 \ + AND lease.generation = $4 AND lease.fence_generation = $5 \ + AND lease.lease_until >= now() AND community.deleted_at IS NULL \ + AND community.deletion_state IN ('active', 'quiescing') \ + AND community.deletion_fence_generation = lease.fence_generation)", + ) + .bind(lease.id) + .bind(lease.community_id.as_uuid()) + .bind(&lease.owner) + .bind(lease.generation) + .bind(lease.fence_generation) + .fetch_one(&mut **tx) + .await?; + if !valid { + return Err(DbError::AccessDenied(format!( + "stale serving write lease {}", + lease.id + ))); + } + sqlx::query( + "SELECT set_config('buzz.serving_write_community', $1, true), \ + set_config('buzz.serving_write_lease_id', $2, true), \ + set_config('buzz.serving_write_owner', $3, true), \ + set_config('buzz.serving_write_generation', $4, true), \ + set_config('buzz.serving_write_fence_generation', $5, true)", + ) + .bind(lease.community_id.to_string()) + .bind(lease.id.to_string()) + .bind(&lease.owner) + .bind(lease.generation.to_string()) + .bind(lease.fence_generation.to_string()) + .execute(&mut **tx) + .await?; + Ok(()) + } + + /// Acquire a durable, expiring lease for an external serving side effect. + /// + /// The short transaction shares the same advisory lock as the destructive + /// fence. The fence therefore orders after all acquisitions that began + /// first, changes lifecycle state, then refuses every later acquisition. + pub async fn acquire_serving_write_lease( + &self, + community: CommunityId, + operation: &str, + owner: &str, + lease_duration: Duration, + ) -> Result { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(community.as_uuid()) + .execute(&mut *tx) + .await?; + let row = sqlx::query( + "INSERT INTO community_serving_write_leases \ + (community_id, operation, owner, fence_generation, lease_until) \ + SELECT id, $2, $3, deletion_fence_generation, \ + now() + make_interval(secs => $4) \ + FROM communities WHERE id = $1 AND deletion_state = 'active' \ + AND deleted_at IS NULL \ + RETURNING id, generation, fence_generation, lease_until", + ) + .bind(community.as_uuid()) + .bind(operation) + .bind(owner) + .bind(lease_seconds) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::AccessDenied(format!("community {community} is write-fenced or missing")) + })?; + let lease = ServingWriteLease { + id: row.try_get("id")?, + community_id: community, + operation: operation.to_owned(), + owner: owner.to_owned(), + generation: row.try_get("generation")?, + fence_generation: row.try_get("fence_generation")?, + lease_until: row.try_get("lease_until")?, + }; + tx.commit().await?; + Ok(lease) + } + + /// Renew an external side-effect lease only while the community is active. + /// + /// Quiescing rejects new acquisition and renewal. A pre-quiesce caller may + /// still verify/release its unexpired lease, but a long operation is + /// cancelled when its next heartbeat observes quiescing. + pub async fn renew_serving_write_lease( + &self, + lease: &mut ServingWriteLease, + lease_duration: Duration, + ) -> Result<()> { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(lease.community_id.as_uuid()) + .execute(&mut *tx) + .await?; + let lease_until: Option> = sqlx::query_scalar( + "UPDATE community_serving_write_leases lease \ + SET lease_until = now() + make_interval(secs => $6), heartbeat_at = now() \ + FROM communities community \ + WHERE lease.id = $1 AND lease.community_id = $2 AND lease.owner = $3 \ + AND lease.generation = $4 AND lease.fence_generation = $5 \ + AND lease.lease_until >= now() \ + AND community.id = lease.community_id \ + AND community.deletion_state = 'active' \ + AND community.deleted_at IS NULL \ + AND community.deletion_fence_generation = lease.fence_generation \ + RETURNING lease.lease_until", + ) + .bind(lease.id) + .bind(lease.community_id.as_uuid()) + .bind(&lease.owner) + .bind(lease.generation) + .bind(lease.fence_generation) + .bind(lease_seconds) + .fetch_optional(&mut *tx) + .await?; + let lease_until = lease_until.ok_or_else(|| { + DbError::AccessDenied(format!("stale serving write lease {}", lease.id)) + })?; + tx.commit().await?; + lease.lease_until = lease_until; + Ok(()) + } + + /// Release a serving side-effect lease. A stale release is harmless. + pub async fn release_serving_write_lease(&self, lease: &ServingWriteLease) -> Result { + let deleted = sqlx::query( + "DELETE FROM community_serving_write_leases \ + WHERE id = $1 AND community_id = $2 AND owner = $3 AND generation = $4 \ + AND fence_generation = $5", + ) + .bind(lease.id) + .bind(lease.community_id.as_uuid()) + .bind(&lease.owner) + .bind(lease.generation) + .bind(lease.fence_generation) + .execute(&self.pool) + .await? + .rows_affected(); + Ok(deleted == 1) + } + + /// Check that an external side-effect lease remains current for finalization. + /// + /// A lease admitted before quiescing may complete/release, but cannot renew; + /// this preserves an accurate bounded drain without admitting new work. + pub async fn verify_serving_write_lease(&self, lease: &ServingWriteLease) -> Result<()> { + let mut tx = self.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(lease.community_id.as_uuid()) + .execute(&mut *tx) + .await?; + let valid: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases lease \ + JOIN communities community ON community.id = lease.community_id \ + WHERE lease.id = $1 AND lease.community_id = $2 AND lease.owner = $3 \ + AND lease.generation = $4 AND lease.fence_generation = $5 \ + AND lease.lease_until >= now() \ + AND community.deleted_at IS NULL \ + AND community.deletion_state IN ('active', 'quiescing') \ + AND community.deletion_fence_generation = lease.fence_generation)", + ) + .bind(lease.id) + .bind(lease.community_id.as_uuid()) + .bind(&lease.owner) + .bind(lease.generation) + .bind(lease.fence_generation) + .fetch_one(&mut *tx) + .await?; + if valid { + tx.commit().await?; + Ok(()) + } else { + Err(DbError::AccessDenied(format!( + "stale serving write lease {}", + lease.id + ))) + } + } + + /// Delete expired serving leases in a bounded batch. + pub async fn reap_expired_serving_write_leases(&self, limit: i64) -> Result { + let affected = sqlx::query( + "WITH expired AS ( \ + SELECT id FROM community_serving_write_leases \ + WHERE lease_until < now() ORDER BY lease_until LIMIT $1 \ + FOR UPDATE SKIP LOCKED \ + ) DELETE FROM community_serving_write_leases lease \ + USING expired WHERE lease.id = expired.id", + ) + .bind(limit.clamp(1, 10_000)) + .execute(&self.pool) + .await? + .rows_affected(); + Ok(affected) + } + + /// Return serving-lease counts and dead-tuple estimate for observability. + pub async fn serving_lease_stats(&self) -> Result { + let row = sqlx::query( + "SELECT count(*) FILTER (WHERE lease_until >= now())::BIGINT AS active, \ + count(*) FILTER (WHERE lease_until < now())::BIGINT AS expired, \ + COALESCE((SELECT n_dead_tup::BIGINT FROM pg_stat_user_tables \ + WHERE relname = 'community_serving_write_leases'), 0) AS dead_tuples \ + FROM community_serving_write_leases", + ) + .fetch_one(&self.pool) + .await?; + Ok(ServingLeaseStats { + active: row.try_get("active")?, + expired: row.try_get("expired")?, + dead_tuples: row.try_get("dead_tuples")?, + }) + } + + /// Whether a community remains active and serving-write eligible. + pub async fn is_serving_active(&self, community: CommunityId) -> Result { + sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 \ + AND archived_at IS NULL AND deleted_at IS NULL AND deletion_state = 'active')", + ) + .bind(community.as_uuid()) + .fetch_one(&self.pool) + .await + .map_err(Into::into) + } + + async fn advance_with_checkpoint( + &self, + token: &LeaseToken, + from: DeletionStage, + to: DeletionStage, + unit_key: &str, + detail: serde_json::Value, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, from, generation).await?; + advance_request_tx(&mut tx, token, from, to, Some(generation)).await?; + checkpoint_completed_tx(&mut tx, token, from, unit_key, detail).await?; + tx.commit().await?; + Ok(()) + } + + async fn live_scoped_tables(&self) -> Result> { + let rows: Vec = sqlx::query_scalar( + r#" + SELECT c.relname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_attribute a ON a.attrelid = c.oid + WHERE n.nspname = 'public' + AND c.relkind IN ('r', 'p') + AND NOT c.relispartition + AND a.attname = 'community_id' + AND NOT a.attisdropped + AND NOT community_write_fence_excluded_table(c.relname) + ORDER BY c.relname + "#, + ) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().collect()) + } + + async fn live_fenced_tables(&self) -> Result> { + let rows: Vec = sqlx::query_scalar( + r#" + SELECT c.relname + FROM pg_trigger trigger + JOIN pg_class c ON c.oid = trigger.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_proc procedure ON procedure.oid = trigger.tgfoid + WHERE n.nspname = 'public' + AND NOT trigger.tgisinternal + AND NOT c.relispartition + AND procedure.proname = 'enforce_community_write_fence' + AND trigger.tgenabled = 'O' + AND (trigger.tgtype & 1) = 1 + AND (trigger.tgtype & 2) = 2 + AND (trigger.tgtype & 4) = 4 + AND (trigger.tgtype & 8) = 8 + AND (trigger.tgtype & 16) = 16 + ORDER BY c.relname + "#, + ) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().collect()) + } +} + +/// Fail closed when a community-prefix inventory has an unsafe shape. +pub fn validate_storage_manifest(manifest: &StorageManifest) -> Result<()> { + if manifest.version != 4 { + return Err(DbError::DeletionSafety(format!( + "unsupported storage manifest version {}", + manifest.version + ))); + } + if manifest.prefixes.is_empty() { + return Err(DbError::DeletionSafety( + "storage manifest has no tenant prefixes".to_string(), + )); + } + if manifest + .prefixes + .windows(2) + .any(|pair| pair[0].prefix >= pair[1].prefix) + { + return Err(DbError::DeletionSafety( + "storage manifest prefixes are not strictly sorted".to_string(), + )); + } + for prefix in &manifest.prefixes { + // An empty prefix would enumerate — and delete — the whole bucket. + if prefix.prefix.is_empty() { + return Err(DbError::DeletionSafety( + "storage manifest contains an empty prefix".to_string(), + )); + } + if prefix.keys_digest.len() != 64 + || !prefix + .keys_digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(DbError::DeletionSafety(format!( + "storage manifest digest for {} is not lowercase hex sha-256", + prefix.prefix + ))); + } + } + Ok(()) +} + +/// Verify the persisted chunk stream is exactly the frozen enumeration: +/// contiguous chunk numbers, chunks grouped by manifest prefix order, every +/// key under its chunk's prefix, and per-prefix digest/count equality. +fn validate_manifest_key_chunks( + manifest: &StorageManifest, + chunks: &[(i64, String, sqlx::types::Json>)], +) -> Result<()> { + let close = |summary: &PrefixManifest, digest: KeyStreamDigest| -> Result<()> { + let (hex_digest, count) = digest.finish(); + if hex_digest != summary.keys_digest || count != summary.object_count { + return Err(DbError::DeletionSafety(format!( + "frozen key chunks do not match the destructive manifest for prefix {}", + summary.prefix + ))); + } + Ok(()) + }; + let mut remaining = manifest.prefixes.iter(); + let mut current = remaining.next(); + let mut digest = KeyStreamDigest::new(); + for (index, (chunk_no, chunk_prefix, keys)) in chunks.iter().enumerate() { + if *chunk_no != i64::try_from(index).unwrap_or(i64::MAX) { + return Err(DbError::DeletionSafety( + "frozen key chunk sequence has gaps".to_string(), + )); + } + loop { + match current { + Some(summary) if summary.prefix == *chunk_prefix => break, + Some(summary) => { + close(summary, std::mem::take(&mut digest))?; + current = remaining.next(); + } + None => { + return Err(DbError::DeletionSafety(format!( + "frozen key chunk prefix {chunk_prefix} is not in the destructive manifest" + ))); + } + } + } + if keys.0.is_empty() { + return Err(DbError::DeletionSafety( + "frozen key chunk is empty".to_string(), + )); + } + for key in &keys.0 { + if !key.starts_with(chunk_prefix.as_str()) { + return Err(DbError::DeletionSafety(format!( + "frozen key {key} is outside its chunk prefix {chunk_prefix}" + ))); + } + digest.fold(key)?; + } + } + if let Some(summary) = current { + close(summary, digest)?; + } + for summary in remaining { + close(summary, KeyStreamDigest::new())?; + } + Ok(()) +} + +async fn verify_lease( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + stage: DeletionStage, +) -> Result<()> { + let valid: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_deletion_requests request \ + JOIN community_deletion_approvals approval ON approval.request_id = request.id \ + AND approval.community_id = request.community_id \ + AND approval.inventory_digest = request.inventory_digest \ + WHERE request.id = $1 AND request.community_id = $5 AND request.stage = $2 \ + AND request.lease_owner = $3 AND request.lease_generation = $4 \ + AND request.lease_until >= now() AND request.blocked_at IS NULL)", + ) + .bind(token.request_id) + .bind(stage.to_string()) + .bind(&token.owner) + .bind(token.generation) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut **tx) + .await?; + if valid { + Ok(()) + } else { + Err(stale_lease_error(token)) + } +} + +async fn verify_lease_and_fence( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + stage: DeletionStage, + fence_generation: i64, +) -> Result<()> { + let valid: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_deletion_requests request \ + JOIN communities community ON community.id = request.community_id \ + JOIN community_deletion_approvals approval ON approval.request_id = request.id \ + AND approval.community_id = request.community_id \ + AND approval.inventory_digest = request.inventory_digest \ + WHERE request.id = $1 AND request.community_id = $6 \ + AND request.stage = $2 AND request.lease_owner = $3 \ + AND request.lease_generation = $4 AND request.lease_until >= now() \ + AND request.blocked_at IS NULL AND request.fence_generation = $5 \ + AND community.deletion_state IN ('fenced', 'tombstone') \ + AND community.deletion_fence_generation = $5)", + ) + .bind(token.request_id) + .bind(stage.to_string()) + .bind(&token.owner) + .bind(token.generation) + .bind(fence_generation) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut **tx) + .await?; + if valid { + Ok(()) + } else { + Err(DbError::AccessDenied(format!( + "stale lease or fencing generation for deletion {}", + token.request_id + ))) + } +} + +async fn set_executor_gucs( + tx: &mut Transaction<'_, Postgres>, + community: CommunityId, + generation: i64, +) -> Result<()> { + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', $2, true)", + ) + .bind(community.to_string()) + .bind(generation.to_string()) + .execute(&mut **tx) + .await?; + Ok(()) +} + +async fn advance_request_tx( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + from: DeletionStage, + to: DeletionStage, + fence_generation: Option, +) -> Result<()> { + if from.next() != Some(to) { + return Err(DbError::DeletionSafety(format!( + "illegal deletion transition {from} -> {to}" + ))); + } + let affected = sqlx::query( + "UPDATE community_deletion_requests \ + SET stage = $5, fence_generation = COALESCE($6, fence_generation), \ + updated_at = now(), last_error = NULL, last_error_at = NULL \ + WHERE id = $1 AND stage = $4 AND lease_owner = $2 \ + AND lease_generation = $3 AND lease_until >= now() AND blocked_at IS NULL", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(from.to_string()) + .bind(to.to_string()) + .bind(fence_generation) + .execute(&mut **tx) + .await? + .rows_affected(); + if affected == 1 { + Ok(()) + } else { + Err(stale_lease_error(token)) + } +} + +async fn checkpoint_completed_tx( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + stage: DeletionStage, + unit_key: &str, + detail: serde_json::Value, +) -> Result<()> { + sqlx::query( + r#" + INSERT INTO community_deletion_checkpoints + (request_id, stage, unit_key, status, lease_generation, detail, completed_at) + VALUES ($1, $2, $3, 'completed', $4, $5, now()) + ON CONFLICT (request_id, stage, unit_key) DO UPDATE + SET status = 'completed', lease_generation = EXCLUDED.lease_generation, + attempts = community_deletion_checkpoints.attempts + 1, + detail = EXCLUDED.detail, error = NULL, completed_at = now() + "#, + ) + .bind(token.request_id) + .bind(stage.to_string()) + .bind(unit_key) + .bind(token.generation) + .bind(detail) + .execute(&mut **tx) + .await?; + Ok(()) +} + +async fn checkpoint_failed_tx( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + stage: DeletionStage, + unit_key: &str, + error: &str, +) -> Result<()> { + sqlx::query( + r#" + INSERT INTO community_deletion_checkpoints + (request_id, stage, unit_key, status, lease_generation, error) + VALUES ($1, $2, $3, 'failed', $4, $5) + ON CONFLICT (request_id, stage, unit_key) DO UPDATE + SET status = 'failed', lease_generation = EXCLUDED.lease_generation, + attempts = community_deletion_checkpoints.attempts + 1, + error = EXCLUDED.error, completed_at = NULL + "#, + ) + .bind(token.request_id) + .bind(stage.to_string()) + .bind(unit_key) + .bind(token.generation) + .bind(error) + .execute(&mut **tx) + .await?; + Ok(()) +} + +fn serialize_community_id( + community: &CommunityId, + serializer: S, +) -> std::result::Result +where + S: serde::Serializer, +{ + serializer.serialize_str(&community.to_string()) +} + +fn row_to_request(row: sqlx::postgres::PgRow) -> Result { + let community_id: Uuid = row.try_get("community_id")?; + let digest: Option> = row.try_get("inventory_digest")?; + Ok(DeletionRequest { + id: row.try_get("id")?, + community_id: CommunityId::from_uuid(community_id), + community_host: row.try_get("community_host")?, + stage: row.try_get::("stage")?.parse()?, + requested_by: row.try_get("requested_by")?, + reason: row.try_get("reason")?, + schema_manifest: row.try_get("schema_manifest")?, + storage_manifest: row.try_get("storage_manifest")?, + destructive_storage_manifest: row.try_get("destructive_storage_manifest")?, + inventory_manifest: row.try_get("inventory_manifest")?, + inventory_digest: digest.map(hex::encode), + fence_generation: row.try_get("fence_generation")?, + lease_owner: row.try_get("lease_owner")?, + lease_generation: row.try_get("lease_generation")?, + lease_until: row.try_get("lease_until")?, + attempts: row.try_get("attempts")?, + retry_count: row.try_get("retry_count")?, + last_error: row.try_get("last_error")?, + blocked_reason: row.try_get("blocked_reason")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + completed_at: row.try_get("completed_at")?, + }) +} + +/// Return whether an error is the deletion store's typed ownership-loss class. +pub fn is_stale_deletion_lease(error: &DbError) -> bool { + matches!(error, DbError::AccessDenied(message) if message.starts_with("stale deletion lease ") || message.starts_with("stale lease or fencing generation for deletion ")) +} + +fn stale_lease_error(token: &LeaseToken) -> DbError { + DbError::AccessDenied(format!( + "stale deletion lease {} owner {:?} generation {}", + token.request_id, token.owner, token.generation + )) +} + +fn require_fence_generation(token: &LeaseToken) -> Result { + token.fence_generation.ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion {} has no durable fence generation", + token.request_id + )) + }) +} + +fn bound_text(input: &str, max: usize) -> String { + if input.len() <= max { + return input.to_owned(); + } + let mut end = max; + while !input.is_char_boundary(end) { + end -= 1; + } + input[..end].to_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_prefix(prefix: &str) -> PrefixManifest { + PrefixManifest { + prefix: prefix.to_string(), + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + } + } + + fn storage_manifest() -> StorageManifest { + StorageManifest { + version: 4, + prefixes: vec![ + empty_prefix("_meta/c/"), + empty_prefix("_uploads/c/"), + empty_prefix("repos/c/"), + ], + } + } + + #[test] + fn stage_order_is_exact_and_terminal() { + let mut stage = DeletionStage::Submitted; + let mut seen = vec![stage]; + while let Some(next) = stage.next() { + stage = next; + seen.push(stage); + } + assert_eq!( + seen, + vec![ + DeletionStage::Submitted, + DeletionStage::Inventoried, + DeletionStage::Approved, + DeletionStage::Fenced, + DeletionStage::Drained, + DeletionStage::BindingsRemoved, + DeletionStage::PostgresPurged, + DeletionStage::CachePurged, + DeletionStage::LogicallyVerified, + DeletionStage::RetentionPending, + ] + ); + assert!(!DeletionStage::Submitted.runnable()); + assert!(!DeletionStage::Inventoried.runnable()); + assert!(DeletionStage::Approved.runnable()); + assert!(!DeletionStage::RetentionPending.runnable()); + } + + #[test] + fn stale_lease_classifier_does_not_swallow_other_access_denials() { + let stale = stale_lease_error(&LeaseToken { + request_id: Uuid::new_v4(), + owner: "owner".to_string(), + generation: 1, + community_id: CommunityId::from_uuid(Uuid::new_v4()), + fence_generation: None, + }); + assert!(is_stale_deletion_lease(&stale)); + assert!(!is_stale_deletion_lease(&DbError::AccessDenied( + "ordinary authorization failure".to_string() + ))); + } + + #[test] + fn storage_manifest_shape_invariants_fail_closed() { + assert!(validate_storage_manifest(&storage_manifest()).is_ok()); + + let mut unsorted = storage_manifest(); + unsorted.prefixes.swap(0, 1); + assert!(validate_storage_manifest(&unsorted).is_err()); + + let mut whole_bucket = storage_manifest(); + whole_bucket.prefixes[0].prefix = String::new(); + assert!(validate_storage_manifest(&whole_bucket).is_err()); + + let mut malformed_digest = storage_manifest(); + malformed_digest.prefixes[0].keys_digest = "not-hex".to_string(); + assert!(validate_storage_manifest(&malformed_digest).is_err()); + } + + #[test] + fn key_stream_digest_requires_strict_order_and_is_chunking_invariant() { + let keys = ["a/1", "a/2", "a/3"]; + let mut whole = KeyStreamDigest::new(); + for key in keys { + whole.fold(key).expect("ascending fold"); + } + // The digest must not depend on where chunk boundaries fall. + let mut split = KeyStreamDigest::new(); + split.fold(keys[0]).expect("chunk one"); + split.fold(keys[1]).expect("chunk one"); + split.fold(keys[2]).expect("chunk two"); + assert_eq!(whole.finish(), split.finish()); + + let mut out_of_order = KeyStreamDigest::new(); + out_of_order.fold("b").expect("first key"); + assert!(out_of_order.fold("a").is_err()); + let mut duplicate = KeyStreamDigest::new(); + duplicate.fold("a").expect("first key"); + assert!(duplicate.fold("a").is_err()); + } + + #[test] + fn manifest_key_chunks_must_hash_to_the_frozen_summaries() { + let keys = vec!["_meta/c/1".to_string(), "_meta/c/2".to_string()]; + let mut digest = KeyStreamDigest::new(); + for key in &keys { + digest.fold(key).expect("fold"); + } + let (hex_digest, count) = digest.finish(); + let mut manifest = storage_manifest(); + manifest.prefixes[0].object_count = count; + manifest.prefixes[0].keys_digest = hex_digest; + + let chunk = |chunk_no: i64, keys: &[String]| { + ( + chunk_no, + "_meta/c/".to_string(), + sqlx::types::Json(keys.to_vec()), + ) + }; + assert!(validate_manifest_key_chunks( + &manifest, + &[chunk(0, &keys[..1]), chunk(1, &keys[1..])] + ) + .is_ok()); + // Missing, reordered, or extra keys change the digest. + assert!(validate_manifest_key_chunks(&manifest, &[chunk(0, &keys[..1])]).is_err()); + // A gap in the chunk sequence is an interrupted write, not a manifest. + assert!(validate_manifest_key_chunks(&manifest, &[chunk(1, &keys)]).is_err()); + // A key outside its chunk's prefix must never freeze. + let foreign = vec!["_uploads/other/1".to_string()]; + assert!( + validate_manifest_key_chunks(&manifest, &[chunk(0, &keys), chunk(1, &foreign)]) + .is_err() + ); + // No chunks at all only matches an all-empty manifest. + assert!(validate_manifest_key_chunks(&manifest, &[]).is_err()); + assert!(validate_manifest_key_chunks(&storage_manifest(), &[]).is_ok()); + } + + #[test] + fn frozen_inventory_digest_is_stable() { + let inventory = FrozenInventory { + schema: SchemaManifest { + scoped_tables: vec!["events".to_string()], + row_counts: BTreeMap::from([("events".to_string(), 3)]), + fenced_tables: vec!["events".to_string()], + }, + storage: storage_manifest(), + }; + assert_eq!(inventory.digest().unwrap(), inventory.digest().unwrap()); + assert_eq!(inventory.digest().unwrap().len(), 32); + } + + #[test] + fn errors_are_utf8_bounded() { + let input = format!("{}🛸", "x".repeat(4095)); + let bounded = bound_text(&input, 4096); + assert!(bounded.len() <= 4096); + assert!(std::str::from_utf8(bounded.as_bytes()).is_ok()); + } +} + +#[cfg(test)] +mod postgres_tests { + use super::*; + use crate::{CreateCommunityWithOwnerResult, Db, DbConfig}; + + async fn store() -> (Db, DeletionStore) { + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); + let db = Db::new(&DbConfig { + database_url, + max_connections: 5, + min_connections: 0, + ..DbConfig::default() + }) + .await + .expect("connect deletion test DB"); + db.migrate().await.expect("migrate deletion test DB"); + let store = db.deletion_store(); + (db, store) + } + + fn empty_prefix_manifest(prefix: String) -> PrefixManifest { + PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + } + } + + fn empty_storage_manifest(community: CommunityId) -> StorageManifest { + StorageManifest { + version: 4, + prefixes: vec![ + empty_prefix_manifest(format!("_meta/{community}/")), + empty_prefix_manifest(format!("_uploads/{community}/")), + empty_prefix_manifest(format!("repos/{community}/")), + ], + } + } + + async fn inventoried_request( + db: &Db, + store: &DeletionStore, + ) -> (DeletionRequest, FrozenInventory) { + let host = format!("deletion-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("create community"); + let submitted = store + .submit(&host, "test-operator", Some("test deletion")) + .await + .expect("submit"); + assert_eq!(submitted.community_id, community.id); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community.id) + .await + .expect("schema inventory"), + storage: empty_storage_manifest(community.id), + }; + let request = store + .freeze_inventory(submitted.id, &inventory) + .await + .expect("freeze inventory"); + (request, inventory) + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn approval_boundary_blocks_claim_until_exact_inventory_is_approved() { + let (db, store) = store().await; + let (request, inventory) = inventoried_request(&db, &store).await; + assert_eq!(request.stage, DeletionStage::Inventoried); + assert!(store + .claim_specific(request.id, "executor-a", DEFAULT_LEASE_DURATION) + .await + .expect("claim before approval") + .is_none()); + + // Approval binds structural/storage taxonomy, not live row counts. A + // serving write between inventory and approval must not make an active + // community undeletable. + db.add_to_allowlist(request.community_id, &[7_u8; 32], &[8_u8; 32], None) + .await + .expect("post-inventory serving write"); + let current_schema = store + .inventory_schema(request.community_id) + .await + .expect("live schema after row churn"); + assert_eq!(current_schema, inventory.schema); + + let mismatched_insert = sqlx::query( + "INSERT INTO community_deletion_approvals \ + (request_id, community_id, inventory_digest, approved_by) \ + VALUES ($1, $2, $3, 'tampered')", + ) + .bind(request.id) + .bind(*request.community_id.as_uuid()) + .bind(vec![0_u8; 32]) + .execute(&db.pool) + .await; + assert!( + mismatched_insert.is_err(), + "a mismatched approval must be unrepresentable" + ); + let approved = store + .approve(request.id, "approver-a", Some("reviewed")) + .await + .expect("approve"); + assert_eq!(approved.stage, DeletionStage::Approved); + assert_eq!( + approved.inventory_digest, + Some(hex::encode(inventory.digest().unwrap())) + ); + let mismatched_approval = sqlx::query( + "UPDATE community_deletion_approvals SET inventory_digest = $2 WHERE request_id = $1", + ) + .bind(request.id) + .bind(vec![0_u8; 32]) + .execute(&db.pool) + .await; + assert!( + mismatched_approval.is_err(), + "approval digest must remain database-bound to the frozen request digest" + ); + let mismatched_request = sqlx::query( + "UPDATE community_deletion_requests SET inventory_digest = $2 WHERE id = $1", + ) + .bind(request.id) + .bind(vec![1_u8; 32]) + .execute(&db.pool) + .await; + assert!( + mismatched_request.is_err(), + "the frozen request digest must remain bound to its approval" + ); + assert!(store + .claim_specific(request.id, "executor-a", DEFAULT_LEASE_DURATION) + .await + .expect("claim approved") + .is_some()); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn approved_request_cannot_be_retargeted_rewritten_or_claimed_without_approval() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + let other_host = format!("control-{}.example", Uuid::new_v4().simple()); + let control = db + .ensure_configured_community(&other_host) + .await + .expect("create control community"); + + for mutation in [ + sqlx::query("UPDATE community_deletion_requests SET community_id = $2 WHERE id = $1") + .bind(request.id) + .bind(*control.id.as_uuid()) + .execute(&db.pool) + .await, + sqlx::query("UPDATE community_deletion_requests SET community_host = $2 WHERE id = $1") + .bind(request.id) + .bind(&other_host) + .execute(&db.pool) + .await, + sqlx::query( + "UPDATE community_deletion_requests SET inventory_manifest = '{}'::jsonb WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await, + sqlx::query( + "UPDATE community_deletion_requests SET storage_manifest = '{}'::jsonb WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await, + ] { + assert!(mutation.is_err(), "frozen deletion target and inventory must be immutable"); + } + + store + .approve(request.id, "approver", None) + .await + .expect("approve request"); + let claim = store + .claim_specific(request.id, "forged-executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim approved request") + .expect("approved request is claimable"); + let approval_delete = + sqlx::query("DELETE FROM community_deletion_approvals WHERE request_id = $1") + .bind(request.id) + .execute(&db.pool) + .await; + assert!( + approval_delete.is_err(), + "approval evidence must be immutable" + ); + for approval_update in [ + "UPDATE community_deletion_approvals SET approved_by = 'forged' WHERE request_id = $1", + "UPDATE community_deletion_approvals SET approved_at = now() + interval '1 hour' WHERE request_id = $1", + "UPDATE community_deletion_approvals SET note = 'rewritten' WHERE request_id = $1", + ] { + assert!( + sqlx::query(approval_update) + .bind(request.id) + .execute(&db.pool) + .await + .is_err(), + "approval evidence updates must be rejected" + ); + } + store + .verify_execution_token(&claim.lease, DeletionStage::Approved) + .await + .expect("matching approval keeps lease valid"); + sqlx::query( + "UPDATE community_deletion_requests \ + SET blocked_at = now(), blocked_reason = 'operator hold' WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await + .expect("block claimed request"); + assert!( + store + .heartbeat(&claim.lease, "worker", DEFAULT_LEASE_DURATION, false,) + .await + .is_err(), + "blocked requests must not renew destructive leases" + ); + + let (forged, _) = inventoried_request(&db, &store).await; + sqlx::query("UPDATE community_deletion_requests SET stage = 'approved' WHERE id = $1") + .bind(forged.id) + .execute(&db.pool) + .await + .expect("forge runnable stage without approval"); + assert!(store + .claim_specific(forged.id, "forged-executor-2", DEFAULT_LEASE_DURATION) + .await + .expect("claim forged request") + .is_none()); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn stale_claim_and_fence_generation_fail_closed() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + let mut stale = claim.lease.clone(); + stale.generation -= 1; + assert!( + store.fence(&stale).await.is_err(), + "stale lease must reject" + ); + let mut wrong_community = claim.lease.clone(); + wrong_community.community_id = db + .ensure_configured_community(&format!( + "wrong-lease-community-{}.example", + Uuid::new_v4().simple() + )) + .await + .expect("create unrelated community") + .id; + assert!( + store.begin_quiescing(&wrong_community).await.is_err(), + "a lease token must remain bound to its durable request community" + ); + + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + let generation = store.fence(&claim.lease).await.expect("fence"); + let mut wrong_fence = claim.lease.clone(); + wrong_fence.fence_generation = Some(generation + 1); + assert!( + store.mark_drained(&wrong_fence).await.is_err(), + "wrong fence generation must reject" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn fence_waits_for_open_write_and_rejects_it_after_transition() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + + let mut open_write = db + .begin_transaction() + .await + .expect("open write transaction"); + sqlx::query("INSERT INTO pubkey_allowlist (community_id, pubkey) VALUES ($1, $2)") + .bind(request.community_id.as_uuid()) + .bind(vec![7_u8; 32]) + .execute(&mut *open_write) + .await + .expect("write acquires shared deletion lock"); + + let store_for_fence = store.clone(); + let lease = claim.lease.clone(); + let fencing = tokio::spawn(async move { + store_for_fence.begin_quiescing(&lease).await?; + store_for_fence.fence(&lease).await + }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !fencing.is_finished(), + "exclusive fence must wait for open writer" + ); + open_write + .commit() + .await + .expect("pre-fence writer commits first"); + fencing.await.expect("fence task").expect("fence completes"); + + assert!( + db.add_to_allowlist(request.community_id, &[8_u8; 32], &[9_u8; 32], None) + .await + .is_err(), + "post-fence serving write must fail" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn quiescing_rejects_new_and_renewed_leases_before_fence() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + let mut serving = store + .acquire_serving_write_lease( + request.community_id, + "test_external", + "test-owner", + DEFAULT_LEASE_DURATION, + ) + .await + .expect("serving lease"); + + store + .begin_quiescing(&claim.lease) + .await + .expect("persist quiescing"); + assert!(matches!( + store + .acquire_serving_write_lease( + request.community_id, + "late_external", + "late-owner", + DEFAULT_LEASE_DURATION, + ) + .await, + Err(DbError::AccessDenied(_)) + )); + assert!(store.verify_serving_write_lease(&serving).await.is_ok()); + assert!(matches!( + store + .renew_serving_write_lease(&mut serving, DEFAULT_LEASE_DURATION) + .await, + Err(DbError::AccessDenied(_)) + )); + assert!(matches!( + store.fence(&claim.lease).await, + Err(DbError::ServingWritesNotDrained { + active_count: 1, + .. + }) + )); + assert!(store + .release_serving_write_lease(&serving) + .await + .expect("release")); + assert_eq!(store.fence(&claim.lease).await.expect("fence"), 1); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn sustained_admission_cannot_starve_fence_after_quiescing() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + + for attempt in 0..100 { + assert!(matches!( + store + .acquire_serving_write_lease( + request.community_id, + "sustained_admission", + &format!("owner-{attempt}"), + DEFAULT_LEASE_DURATION, + ) + .await, + Err(DbError::AccessDenied(_)) + )); + } + assert_eq!(store.fence(&claim.lease).await.expect("fence"), 1); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn serving_lease_reaper_is_bounded_and_reports_stats() { + let (db, store) = store().await; + let host = format!("lease-reaper-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("community") + .id; + for owner in ["expired-a", "expired-b", "expired-c"] { + let lease = store + .acquire_serving_write_lease( + community, + "reaper_test", + owner, + Duration::from_secs(1), + ) + .await + .expect("lease"); + sqlx::query("UPDATE community_serving_write_leases SET lease_until = now() - interval '1 second' WHERE id = $1") + .bind(lease.id) + .execute(&db.pool) + .await + .expect("expire lease"); + } + let before = store.serving_lease_stats().await.expect("stats before"); + assert!(before.expired >= 3); + assert_eq!(store.reap_expired_serving_write_leases(2).await.unwrap(), 2); + let after = store.serving_lease_stats().await.expect("stats after"); + assert_eq!(after.expired, before.expired - 2); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn checkpointed_resume_is_idempotent_and_tombstone_blocks_name_reuse() { + let (db, store) = store().await; + let (request, inventory) = inventoried_request(&db, &store).await; + let host = request.community_host.clone(); + let read_state_d_tag = format!("read-state:{}", "a".repeat(32)); + sqlx::query( + "INSERT INTO events \ + (community_id, id, pubkey, created_at, kind, tags, content, sig, d_tag) \ + VALUES ($1, $2, $3, now(), 30078, $4, '', $5, $6)", + ) + .bind(request.community_id.as_uuid()) + .bind(vec![1_u8; 32]) + .bind(vec![2_u8; 32]) + .bind(serde_json::json!([ + ["d", &read_state_d_tag], + ["t", "read-state"] + ])) + .bind(vec![3_u8; 64]) + .bind(&read_state_d_tag) + .execute(&db.pool) + .await + .expect("insert guarded NIP-RS row"); + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + let generation = store.fence(&claim.lease).await.expect("fence"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease + }; + store + .freeze_destructive_storage_manifest(&token, &inventory.storage) + .await + .expect("freeze destructive storage"); + store + .freeze_destructive_storage_manifest(&token, &inventory.storage) + .await + .expect("identical destructive manifest retry"); + let mut drifted_storage = inventory.storage.clone(); + let mut drifted_digest = KeyStreamDigest::new(); + drifted_digest + .fold("media/drifted-after-fence") + .expect("fold drifted key"); + let (drifted_hex, drifted_count) = drifted_digest.finish(); + drifted_storage.prefixes[0].object_count = drifted_count; + drifted_storage.prefixes[0].keys_digest = drifted_hex; + assert!(matches!( + store + .freeze_destructive_storage_manifest(&token, &drifted_storage) + .await, + Err(DbError::DeletionSafety(_)) + )); + for mutation in [ + sqlx::query( + "UPDATE community_deletion_requests \ + SET destructive_storage_manifest = '{}'::jsonb WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await, + sqlx::query( + "UPDATE community_deletion_requests \ + SET destructive_storage_frozen_at = destructive_storage_frozen_at + interval '1 second' \ + WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await, + ] { + assert!( + mutation.is_err(), + "frozen destructive storage evidence must be immutable" + ); + } + store.mark_drained(&token).await.expect("drain"); + store + .mark_bindings_removed(&token, serde_json::json!({"keys": 0})) + .await + .expect("bindings"); + let first = store.purge_postgres(&token).await.expect("purge postgres"); + assert_eq!(first.len(), EXPECTED_SCOPED_TABLES.len()); + assert!( + store.purge_postgres(&token).await.is_err(), + "completed stage cannot be replayed under stale checkpoint state" + ); + store + .mark_cache_purged(&token, serde_json::json!({"keys": 0})) + .await + .expect("cache"); + store + .verify_postgres_logically_deleted(&token) + .await + .expect("logical postgres verify"); + store + .mark_logically_verified(&token, serde_json::json!({"all": true})) + .await + .expect("mark verified"); + store + .mark_retention_pending(&token, serde_json::json!({"shared_cas": "retained"})) + .await + .expect("terminal"); + + let terminal = store.get(request.id).await.expect("terminal request"); + assert_eq!(terminal.stage, DeletionStage::RetentionPending); + let recreated = db + .create_community_with_owner( + &host, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ) + .await + .expect("recreate attempt"); + assert_eq!(recreated, CreateCommunityWithOwnerResult::HostExists); + assert!(db + .lookup_community_by_host_for_management(&host) + .await + .expect("tombstone lookup") + .is_some()); + assert!(db + .lookup_community_by_host(&host) + .await + .expect("serving lookup") + .is_none()); + let direct_delete = sqlx::query("DELETE FROM communities WHERE id = $1") + .bind(request.community_id.as_uuid()) + .execute(&db.pool) + .await + .expect_err("tombstone row must be permanent"); + assert!(direct_delete + .to_string() + .contains("tombstones are permanent")); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn taxonomy_sweep_uses_database_completion_order() { + let (_, store) = store().await; + store + .record_taxonomy_sweep(Utc::now() + chrono::Duration::minutes(1), 1, 0, &[], 100) + .await + .expect("record skewed clean sweep"); + let dirty = store + .record_taxonomy_sweep(Utc::now(), 1, 1, &["unknown".to_string()], 100) + .await + .expect("record later dirty sweep"); + + assert_eq!( + store + .latest_taxonomy_sweep() + .await + .expect("latest sweep") + .unwrap() + .id, + dirty.id + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn manifest_key_chunks_bind_freeze_execution_and_cleanup() { + let (db, store) = store().await; + let (request, inventory) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + let generation = store.fence(&claim.lease).await.expect("fence"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease + }; + + let meta_prefix = format!("_meta/{}/", request.community_id); + let keys = vec![ + format!("{meta_prefix}{}.json", "a".repeat(64)), + format!("{meta_prefix}{}.json", "b".repeat(64)), + ]; + store + .append_manifest_key_chunk(&token, 0, &meta_prefix, &keys[..1]) + .await + .expect("append chunk 0"); + store + .append_manifest_key_chunk(&token, 1, &meta_prefix, &keys[1..]) + .await + .expect("append chunk 1"); + + // A manifest whose digests do not cover the chunk stream must not freeze. + assert!(matches!( + store + .freeze_destructive_storage_manifest(&token, &inventory.storage) + .await, + Err(DbError::DeletionSafety(_)) + )); + let mut digest = KeyStreamDigest::new(); + for key in &keys { + digest.fold(key).expect("fold key"); + } + let (hex_digest, count) = digest.finish(); + let mut storage = inventory.storage.clone(); + storage.prefixes[0].object_count = count; + storage.prefixes[0].total_bytes = 2; + storage.prefixes[0].keys_digest = hex_digest; + store + .freeze_destructive_storage_manifest(&token, &storage) + .await + .expect("freeze manifest matching chunks"); + + // Frozen chunks are immutable working data until terminal cleanup. + assert!(sqlx::query( + "UPDATE community_deletion_manifest_keys SET keys = '[]'::jsonb \ + WHERE request_id = $1 AND chunk_no = 0", + ) + .bind(request.id) + .execute(&db.pool) + .await + .is_err()); + assert!( + sqlx::query("DELETE FROM community_deletion_manifest_keys WHERE request_id = $1") + .bind(request.id) + .execute(&db.pool) + .await + .is_err() + ); + assert!(store.clear_manifest_key_chunks(&token).await.is_err()); + assert!( + sqlx::query( + "INSERT INTO community_deletion_manifest_keys \ + (request_id, chunk_no, prefix, keys) VALUES ($1, 2, $2, $3)", + ) + .bind(request.id) + .bind(&meta_prefix) + .bind(sqlx::types::Json(&keys[..1])) + .execute(&db.pool) + .await + .is_err(), + "the database must reject chunks appended after freeze" + ); + + store.mark_drained(&token).await.expect("drained"); + let first = store + .next_pending_manifest_chunk(&token) + .await + .expect("pending chunk") + .expect("chunk 0 pending"); + assert_eq!(first.chunk_no, 0); + assert_eq!(first.keys, keys[..1]); + store + .mark_manifest_chunk_deleted(&token, 0, serde_json::json!({"deleted": 1})) + .await + .expect("stamp chunk 0"); + assert!( + matches!( + store + .mark_manifest_chunk_deleted(&token, 0, serde_json::json!({})) + .await, + Err(DbError::DeletionSafety(_)) + ), + "a chunk stamp is one-way" + ); + let second = store + .next_pending_manifest_chunk(&token) + .await + .expect("pending chunk") + .expect("chunk 1 pending after resume"); + assert_eq!(second.chunk_no, 1); + store + .mark_manifest_chunk_deleted(&token, 1, serde_json::json!({"deleted": 1})) + .await + .expect("stamp chunk 1"); + assert!(store + .next_pending_manifest_chunk(&token) + .await + .expect("pending chunk") + .is_none()); + assert_eq!( + store + .manifest_chunk_progress(request.id) + .await + .expect("progress"), + (2, 2) + ); + + store + .mark_bindings_removed(&token, serde_json::json!({"deleted_keys": 2})) + .await + .expect("bindings removed"); + store.purge_postgres(&token).await.expect("purge postgres"); + store + .mark_cache_purged(&token, serde_json::json!({"keys": 0})) + .await + .expect("cache purged"); + store + .verify_postgres_logically_deleted(&token) + .await + .expect("verify postgres"); + store + .mark_logically_verified(&token, serde_json::json!({"all": true})) + .await + .expect("logically verified"); + assert_eq!( + store + .manifest_chunk_progress(request.id) + .await + .expect("progress after terminal cleanup"), + (0, 0) + ); + } +} diff --git a/crates/buzz-db/src/error.rs b/crates/buzz-db/src/error.rs index f8b8a2eb56..593eea1cca 100644 --- a/crates/buzz-db/src/error.rs +++ b/crates/buzz-db/src/error.rs @@ -45,6 +45,25 @@ pub enum DbError { #[error("invalid data: {0}")] InvalidData(String), + /// A serving write admitted before the lifecycle transition is still live. + /// This is an ordinary retryable drain condition, not a safety violation. + #[error( + "community {community_id} still has {active_count} active serving write lease(s): {operations:?}" + )] + ServingWritesNotDrained { + /// Community whose lifecycle transition must retry. + community_id: uuid::Uuid, + /// Number of currently unexpired serving-write leases. + active_count: i64, + /// Distinct operation categories holding those leases. + operations: Vec, + }, + + /// A deletion safety invariant is structurally violated and requires + /// operator/code remediation rather than blind retry. + #[error("deletion safety error: {0}")] + DeletionSafety(String), + /// A stored timestamp value could not be interpreted. #[error("invalid timestamp: {0}")] InvalidTimestamp(i64), diff --git a/crates/buzz-db/src/event.rs b/crates/buzz-db/src/event.rs index a670a13402..007beb9458 100644 --- a/crates/buzz-db/src/event.rs +++ b/crates/buzz-db/src/event.rs @@ -1111,7 +1111,7 @@ pub struct ThreadMetadataParams<'a> { pub broadcast: bool, } -async fn insert_event_with_thread_metadata_tx( +pub(crate) async fn insert_event_with_thread_metadata_tx( tx: &mut Transaction<'_, Postgres>, community_id: CommunityId, event: &Event, diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 9b26876747..92a2ac942b 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -17,6 +17,8 @@ pub mod api_token; pub mod archived_identities; /// Channel and membership persistence. pub mod channel; +/// Durable whole-community deletion lifecycle and PostgreSQL adapter. +pub mod deletion; /// Direct message channel persistence. pub mod dm; /// Database error types. @@ -1018,6 +1020,16 @@ impl Db { sqlx::query("SELECT 1").execute(&self.pool).await.is_ok() } + /// Validate the minimum deletion fence catalog required by serving paths. + pub async fn validate_deletion_serving_catalog(&self) -> Result<()> { + self.deletion_store().validate_serving_catalog().await + } + + /// Validate the exact live community-deletion tenant catalog for destruction. + pub async fn validate_deletion_catalog(&self) -> Result<()> { + self.deletion_store().validate_catalog().await + } + /// Returns pool utilisation stats for metrics emission. /// /// `size` — total connections (idle + active) @@ -1180,6 +1192,11 @@ impl Db { usage::community_hosts(&self.pool).await } + /// Return the shared durable whole-community deletion adapter. + pub fn deletion_store(&self) -> deletion::DeletionStore { + deletion::DeletionStore::new(self.pool.clone()) + } + /// Begin a database transaction for atomic multi-statement operations. /// /// Returns a `'static` transaction because `PgPool` is `Arc`-backed internally. @@ -1202,6 +1219,8 @@ impl Db { FROM communities WHERE lower(host) = lower($1) AND archived_at IS NULL + AND deleted_at IS NULL + AND deletion_state = 'active' "#, ) .bind(normalized_host) @@ -1223,7 +1242,7 @@ impl Db { /// Returns whether a community id still exists in the active lifecycle state. pub async fn is_community_active(&self, community_id: CommunityId) -> Result { let active = sqlx::query_scalar::<_, bool>( - "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 AND archived_at IS NULL)", + "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 AND archived_at IS NULL AND deleted_at IS NULL AND deletion_state = 'active')", ) .bind(community_id.as_uuid()) .fetch_one(&self.pool) @@ -1305,6 +1324,8 @@ impl Db { FROM communities WHERE id = $1 AND archived_at IS NULL + AND deleted_at IS NULL + AND deletion_state = 'active' "#, ) .bind(community_id.as_uuid()) @@ -1375,12 +1396,19 @@ impl Db { INSERT INTO communities (host) VALUES ($1) ON CONFLICT (lower(host)) DO UPDATE SET host = communities.host + WHERE communities.deletion_state = 'active' + AND communities.deleted_at IS NULL RETURNING id, host, (xmax = 0) AS created "#, ) .bind(normalized_host) - .fetch_one(&self.pool) - .await?; + .fetch_optional(&self.pool) + .await? + .ok_or_else(|| { + DbError::AccessDenied(format!( + "community host {normalized_host:?} is permanently tombstoned" + )) + })?; let id: Uuid = row.try_get("id")?; let host: String = row.try_get("host")?; @@ -1460,6 +1488,8 @@ impl Db { AND lower(rm.pubkey) = lower($2) AND rm.role = 'owner' AND c.archived_at IS NULL + AND c.deletion_state = 'active' + AND c.deleted_at IS NULL "#, ) .bind(normalized_host) @@ -1498,6 +1528,8 @@ impl Db { AND lower(rm.pubkey) = lower($2) AND rm.role = 'owner' AND lower(c.host) <> lower($3) + AND c.deletion_state = 'active' + AND c.deleted_at IS NULL RETURNING c.id, c.host, c.archived_at"#, ) .bind(normalized_host) @@ -1529,6 +1561,8 @@ impl Db { AND rm.community_id = c.id AND lower(rm.pubkey) = lower($2) AND rm.role = 'owner' + AND c.deletion_state = 'active' + AND c.deleted_at IS NULL RETURNING c.id, c.host"#, ) .bind(normalized_host) @@ -1630,6 +1664,49 @@ impl Db { Ok(result) } + /// Insert an event while holding and validating an admitted serving-write + /// lease under the community ordering lock through commit. + /// + /// External side effects use a durable lease rather than one long-lived DB + /// transaction. Their final database mutation presents that exact lease so + /// it may finish during quiescing without admitting any new serving work. + pub async fn insert_event_with_serving_write_guard( + &self, + lease: &deletion::ServingWriteLease, + event: &nostr::Event, + channel_id: Option, + ) -> Result<(StoredEvent, bool)> { + let community_id = lease.community_id; + let kind_u16 = event.kind.as_u16(); + let kind_u32 = u32::from(kind_u16); + if kind_u32 == buzz_core::kind::KIND_AUTH { + return Err(DbError::AuthEventRejected); + } + if buzz_core::kind::is_ephemeral(kind_u32) { + return Err(DbError::EphemeralEventRejected(kind_u16)); + } + + let mut tx = self.pool.begin().await?; + self.deletion_store() + .guard_transaction_with_serving_lease(&mut tx, lease) + .await?; + let result = event::insert_event_with_thread_metadata_tx( + &mut tx, + community_id, + event, + channel_id, + None, + ) + .await?; + tx.commit().await?; + if result.1 { + if let Err(e) = insert_mentions(&self.pool, community_id, event, channel_id).await { + tracing::warn!(event_id = %event.id, "Failed to insert mentions: {e}"); + } + } + Ok(result) + } + /// Queries events matching the given filter parameters. /// /// Always reads from the WRITER pool. If the result influences a write diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 65ca156721..75a26f752a 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -348,6 +348,13 @@ mod tests { "push_gateway_delivery_request_replays", "product_feedback", "replica_heartbeat", + "community_deletion_requests", + "community_deletion_approvals", + "community_deletion_checkpoints", + "community_deletion_manifest_keys", + "storage_taxonomy_sweeps", + "community_serving_write_leases", + "community_deletion_executor_heartbeats", ] { if normalized[insert_pos..].contains(&format!("'{value}'")) { globals.insert(value.to_owned()); @@ -489,6 +496,106 @@ mod tests { .collect() } + fn migrations_missing_community_write_fence_attachment() -> Vec { + let mut migrations: Vec<_> = MIGRATOR.iter().collect(); + migrations.sort_by_key(|migration| migration.version); + migrations + .into_iter() + .filter(|migration| migration.version > 28) + .flat_map(|migration| { + let statements = split_sql_statements(migration.sql.as_str()); + let attachments = statements + .iter() + .filter_map(|statement| { + let normalized = normalize_sql(statement); + if !normalized.contains("attach_community_write_fence(") { + return None; + } + let call = normalized.split_once("attach_community_write_fence(")?.1; + let table = call + .split(')') + .next()? + .trim() + .split("::") + .next()? + .trim_matches(|ch| ch == '\'' || ch == '"') + .rsplit('.') + .next()? + .to_owned(); + (!table.is_empty()).then_some(table) + }) + .collect::>(); + statements.into_iter().filter_map(move |statement| { + let normalized = normalize_sql(&statement); + let table = if normalized.starts_with("create table") + && normalized.contains("community_id") + && !normalized.contains(" partition of ") + { + identifier_after_keyword(&statement, "create table") + } else if normalized.starts_with("alter table") + && normalized.contains("add") + && normalized.contains("community_id") + { + identifier_after_keyword(&statement, "alter table") + } else { + None + }?; + (!attachments.contains(&table)).then(|| { + format!( + "migration {} introduces {table}.community_id without attach_community_write_fence('{table}'::regclass)", + migration.version + ) + }) + }) + }) + .collect() + } + + fn community_write_fence_attachment_violations(sql: &str) -> Vec { + let statements = split_sql_statements(sql); + let attachments = statements + .iter() + .filter_map(|statement| { + let normalized = normalize_sql(statement); + if !normalized.contains("attach_community_write_fence(") { + return None; + } + let call = normalized.split_once("attach_community_write_fence(")?.1; + let table = call + .split(')') + .next()? + .trim() + .split("::") + .next()? + .trim_matches(|ch| ch == '\'' || ch == '"') + .rsplit('.') + .next()? + .to_owned(); + (!table.is_empty()).then_some(table) + }) + .collect::>(); + statements + .into_iter() + .filter_map(|statement| { + let normalized = normalize_sql(&statement); + let table = if normalized.starts_with("create table") + && normalized.contains("community_id") + && !normalized.contains(" partition of ") + { + identifier_after_keyword(&statement, "create table") + } else if normalized.starts_with("alter table") + && normalized.contains("add") + && normalized.contains("community_id") + { + identifier_after_keyword(&statement, "alter table") + } else { + None + }?; + (!attachments.contains(&table)).then_some(table) + }) + .collect() + } + fn scoped_constraint_lints(sql: &str, scoped_tables: &BTreeSet) -> Vec { let mut constraints = table_constraints(sql, scoped_tables); constraints.extend(alter_table_constraints(sql, scoped_tables)); @@ -561,7 +668,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 27); + assert_eq!(migrations.len(), 28); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -920,26 +1027,75 @@ mod tests { assert!(heartbeat.contains("INSERT INTO replica_heartbeat (id) VALUES (1)")); assert!(heartbeat.contains("_operator_global_tables")); - // Channel-id lookup index (0027): serves the tenant-independent - // `channels` lookups that carry no community_id predicate, which no - // community_id-leading index can satisfy. Covering + partial so the - // planner can go index-only; asserted NOT UNIQUE because `id` alone is - // not unique in this table (the same channel id may exist under more - // than one community), so a unique index would encode a false - // constraint and fail to build on such a database. + // Channel-id lookup index (0027): serves tenant-independent channel lookups. assert_eq!(migrations[26].version, 27); let channel_id_index = migrations[26].sql.as_str(); assert!(channel_id_index.contains("idx_channels_id_live")); assert!(channel_id_index.contains("INCLUDE (community_id)")); assert!(channel_id_index.contains("WHERE deleted_at IS NULL")); + assert!(!channel_id_index.contains("CREATE UNIQUE INDEX")); + assert!(desired_schema.contains("idx_channels_id_live")); + + // Durable whole-community deletion control plane and universal DB fence. + assert_eq!(migrations[27].version, 28); + let deletion = migrations[27].sql.as_str(); + assert!(deletion.contains("CREATE TABLE community_deletion_requests")); + assert!(deletion.contains("CREATE TABLE community_deletion_approvals")); + assert!(deletion.contains("CREATE TABLE community_deletion_checkpoints")); + assert!(deletion.contains("CREATE TABLE community_serving_write_leases")); + assert!(deletion.contains("CREATE TABLE community_deletion_executor_heartbeats")); + assert!(deletion.contains("CREATE FUNCTION assert_community_write_allowed")); + assert!(deletion.contains("CREATE FUNCTION enforce_community_write_fence")); + assert!(deletion.contains("CREATE FUNCTION attach_community_write_fence")); + assert!(deletion.contains("community_write_fence_excluded_table")); + assert!(deletion.contains("CREATE FUNCTION enforce_community_tombstone")); + assert!(deletion.contains("community tombstones are permanent")); + assert!(deletion.contains("SET LOCAL lock_timeout = '5s'")); + assert!(deletion.contains("'active', 'quiescing', 'fenced', 'tombstone'")); + assert!(deletion.contains("_operator_global_tables")); + assert!(deletion.contains("'submitted', 'inventoried', 'approved', 'fenced', 'drained'")); + assert!(deletion.contains("UNIQUE (id, community_id, inventory_digest)")); + assert!(deletion.contains("FOREIGN KEY (request_id, community_id, inventory_digest)")); + assert!(deletion.contains("prevent_community_deletion_request_retargeting")); + assert!(deletion.contains("prevent_community_deletion_approval_removal")); + } + + #[test] + fn post_deletion_migrations_attach_every_new_community_write_fence() { + let violations = migrations_missing_community_write_fence_attachment(); assert!( - !channel_id_index.contains("CREATE UNIQUE INDEX"), - "channels.id is not unique across communities — index must not be UNIQUE", + violations.is_empty(), + "migrations after 0028 must explicitly attach every new community write fence:\n{}", + violations.join("\n") ); - assert!( - desired_schema.contains("idx_channels_id_live"), - "desired-state schema must carry the channel-id lookup index", + } + + #[test] + fn community_write_fence_attachment_lint_covers_create_and_alter() { + let missing = r#" + CREATE TABLE created_late ( + community_id UUID NOT NULL, + id UUID NOT NULL + ); + CREATE TABLE altered_late (id UUID NOT NULL); + ALTER TABLE altered_late ADD COLUMN community_id UUID NOT NULL; + "#; + assert_eq!( + community_write_fence_attachment_violations(missing), + vec!["created_late", "altered_late"] ); + + let attached = r#" + CREATE TABLE created_late ( + community_id UUID NOT NULL, + id UUID NOT NULL + ); + SELECT attach_community_write_fence('created_late'::regclass); + CREATE TABLE altered_late (id UUID NOT NULL); + ALTER TABLE altered_late ADD COLUMN community_id UUID NOT NULL; + SELECT attach_community_write_fence('altered_late'::regclass); + "#; + assert!(community_write_fence_attachment_violations(attached).is_empty()); } #[test] @@ -1182,7 +1338,7 @@ mod tests { run_migrations(&pool) .await .expect("retry succeeds after operator repair"); - assert_eq!(applied_versions(&pool).await.last().copied(), Some(27)); + assert_eq!(applied_versions(&pool).await.last().copied(), Some(28)); } #[tokio::test] @@ -1304,5 +1460,163 @@ mod tests { search_expression.contains("ELSE NULL::tsvector"), "fresh installs must default non-allowlisted kinds to NULL: {search_expression}" ); + + let active_a = uuid::Uuid::new_v4(); + let active_b = uuid::Uuid::new_v4(); + let to_fence = uuid::Uuid::new_v4(); + for (community, label) in [ + (active_a, "active-a"), + (active_b, "active-b"), + (to_fence, "to-fence"), + ] { + sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") + .bind(community) + .bind(format!("late-fence-{label}-{}.example", community.simple())) + .execute(&pool) + .await + .expect("insert late-table test community"); + } + sqlx::query( + "CREATE TABLE late_created_scoped (\ + community_id UUID NOT NULL, id BIGINT PRIMARY KEY, value TEXT NOT NULL\ + )", + ) + .execute(&pool) + .await + .expect("create late scoped table"); + sqlx::query("SELECT attach_community_write_fence('late_created_scoped'::regclass)") + .execute(&pool) + .await + .expect("attach late create fence"); + sqlx::query("CREATE TABLE late_altered_scoped (id BIGINT PRIMARY KEY)") + .execute(&pool) + .await + .expect("create table before late alter"); + sqlx::query("ALTER TABLE late_altered_scoped ADD COLUMN community_id UUID NOT NULL") + .execute(&pool) + .await + .expect("add late community id"); + sqlx::query("SELECT attach_community_write_fence('late_altered_scoped'::regclass)") + .execute(&pool) + .await + .expect("attach late alter fence"); + let attached: Vec = sqlx::query_scalar( + "SELECT c.relname FROM pg_trigger trigger \ + JOIN pg_class c ON c.oid = trigger.tgrelid \ + JOIN pg_proc procedure ON procedure.oid = trigger.tgfoid \ + WHERE c.relname IN ('late_created_scoped', 'late_altered_scoped') \ + AND procedure.proname = 'enforce_community_write_fence' \ + AND NOT trigger.tgisinternal ORDER BY c.relname", + ) + .fetch_all(&pool) + .await + .expect("read late trigger catalog"); + assert_eq!(attached, vec!["late_altered_scoped", "late_created_scoped"]); + let malformed_fence_triggers: i64 = sqlx::query_scalar( + "SELECT count(*)::BIGINT FROM pg_trigger trigger \ + JOIN pg_class c ON c.oid = trigger.tgrelid \ + JOIN pg_proc procedure ON procedure.oid = trigger.tgfoid \ + WHERE c.relname IN ('late_created_scoped', 'late_altered_scoped') \ + AND procedure.proname = 'enforce_community_write_fence' \ + AND NOT trigger.tgisinternal \ + AND (trigger.tgenabled <> 'O' OR (trigger.tgtype & 31) <> 31)", + ) + .fetch_one(&pool) + .await + .expect("validate late trigger mode and operations"); + assert_eq!(malformed_fence_triggers, 0); + + sqlx::query( + "INSERT INTO late_created_scoped (community_id, id, value) \ + VALUES ($1, 1, 'same'), ($2, 2, 'source-fenced'), \ + ($1, 3, 'destination-fenced'), ($1, 4, 'opposite-a'), \ + ($3, 5, 'opposite-b')", + ) + .bind(active_a) + .bind(to_fence) + .bind(active_b) + .execute(&pool) + .await + .expect("seed late table while communities active"); + sqlx::query("UPDATE late_created_scoped SET value = 'same-ok' WHERE id = 1") + .execute(&pool) + .await + .expect("same-tenant active update"); + sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 1") + .bind(active_b) + .execute(&pool) + .await + .expect("active-to-active update"); + + let mut fence_connection = pool.acquire().await.expect("fence connection"); + sqlx::query("BEGIN") + .execute(&mut *fence_connection) + .await + .expect("begin direct fence"); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '1', true)", + ) + .bind(to_fence.to_string()) + .execute(&mut *fence_connection) + .await + .expect("authorize direct fence"); + sqlx::query( + "UPDATE communities SET deletion_state = 'fenced', \ + deletion_fence_generation = 1, archived_at = now() WHERE id = $1", + ) + .bind(to_fence) + .execute(&mut *fence_connection) + .await + .expect("fence test destination"); + sqlx::query("COMMIT") + .execute(&mut *fence_connection) + .await + .expect("commit direct fence"); + + let active_to_fenced = + sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 3") + .bind(to_fence) + .execute(&pool) + .await + .expect_err("active to fenced destination must fail"); + assert!(active_to_fenced + .to_string() + .contains("community write fenced")); + let fenced_to_active = + sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 2") + .bind(active_a) + .execute(&pool) + .await + .expect_err("fenced source to active destination must fail"); + assert!(fenced_to_active + .to_string() + .contains("community write fenced")); + let row_locations: Vec<(i64, uuid::Uuid)> = sqlx::query_as( + "SELECT id, community_id FROM late_created_scoped WHERE id IN (2, 3) ORDER BY id", + ) + .fetch_all(&pool) + .await + .expect("failed moves preserve row location"); + assert_eq!(row_locations, vec![(2, to_fence), (3, active_a)]); + + let move_a = sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 4") + .bind(active_b) + .execute(&pool); + let move_b = sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 5") + .bind(active_a) + .execute(&pool); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + let (a, b) = tokio::join!(move_a, move_b); + a.expect("opposite active move A"); + b.expect("opposite active move B"); + }) + .await + .expect("opposite cross-tenant updates must not deadlock"); + + sqlx::query("DROP TABLE late_created_scoped, late_altered_scoped") + .execute(&pool) + .await + .expect("drop late-table fixtures"); } } diff --git a/crates/buzz-deletion/Cargo.toml b/crates/buzz-deletion/Cargo.toml new file mode 100644 index 0000000000..8c308b0a8c --- /dev/null +++ b/crates/buzz-deletion/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "buzz-deletion" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Durable whole-community deletion engine for Buzz" + +[dependencies] +anyhow = { workspace = true } +thiserror = { workspace = true } +buzz-core = { workspace = true } +buzz-db = { workspace = true } +buzz-media = { workspace = true } +chrono = { workspace = true } +clap = { version = "4", features = ["derive"] } +deadpool-redis = { workspace = true } +hex = { workspace = true } +redis = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +sqlx = { workspace = true } diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs new file mode 100644 index 0000000000..49076a3684 --- /dev/null +++ b/crates/buzz-deletion/src/lib.rs @@ -0,0 +1,1844 @@ +#![deny(unsafe_code)] +#![warn(missing_docs)] +//! Shared durable whole-community deletion engine and store adapters. + +#[cfg(test)] +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use buzz_db::deletion::{ + ClaimedDeletion, DeletionRequest, DeletionStage, DeletionStore, FrozenInventory, + KeyStreamDigest, LeaseToken, PrefixManifest, StorageManifest, DEFAULT_LEASE_DURATION, +}; +use buzz_db::{Db, DbConfig}; +use buzz_media::{is_tenant_owned_key, tenant_prefixes, MediaStorage}; +use clap::Subcommand; +use serde::Serialize; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +/// Fleet-wide object cap for one observational taxonomy sweep. +const DEFAULT_SWEEP_OBJECT_CAP: u64 = 10_000_000; +/// Keys per frozen side-table chunk (one `DeleteObjects`-sized unit × 10). +const DEFAULT_MANIFEST_CHUNK_KEYS: usize = 10_000; +/// Unknown keys retained verbatim on a sweep record for diagnosis. +const SWEEP_UNKNOWN_KEY_SAMPLE: usize = 100; +/// One S3 LIST page. +const LIST_PAGE_SIZE: usize = 1000; +const RETRY_DELAY: Duration = Duration::from_secs(30); +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(10); + +#[cfg(test)] +static TEST_HEARTBEAT_INTERVAL_MS: AtomicU64 = AtomicU64::new(0); + +fn heartbeat_interval() -> Duration { + #[cfg(test)] + { + let milliseconds = TEST_HEARTBEAT_INTERVAL_MS.load(Ordering::Relaxed); + if milliseconds > 0 { + return Duration::from_millis(milliseconds); + } + } + HEARTBEAT_INTERVAL +} + +#[derive(Debug, Clone, thiserror::Error)] +#[error("deletion execution lease heartbeat failed")] +struct DeletionLeaseLost; + +#[derive(Debug, Clone, thiserror::Error)] +#[error("{message}")] +struct ServingWriteLeaseLost { + message: String, +} + +/// Return the shared durable deletion store for relay and operator paths. +pub fn store(db: &Db) -> DeletionStore { + db.deletion_store() +} + +/// Durable, heartbeated lease for a serving-path external side effect. +pub struct ServingWriteGuard { + store: DeletionStore, + lease: buzz_db::deletion::ServingWriteLease, + cancel: CancellationToken, + lost: CancellationToken, + finished: bool, +} + +impl ServingWriteGuard { + /// Verify this side-effect lease is still current before an irreversible call. + pub async fn verify(&self) -> Result<()> { + if self.lost.is_cancelled() { + return Err(ServingWriteLeaseLost { + message: "serving write lease heartbeat was lost".to_string(), + } + .into()); + } + self.store + .verify_serving_write_lease(&self.lease) + .await + .map_err(|error| ServingWriteLeaseLost { + message: error.to_string(), + })?; + Ok(()) + } + + /// Run an external side effect while observing lease-heartbeat loss. + /// + /// Dropping the operation future on lease loss prevents a stale caller from + /// continuing network I/O after its durable exclusion proof disappears. + pub async fn protect(&self, operation: F) -> Result + where + F: std::future::Future, + { + self.verify().await?; + let output = tokio::select! { + biased; + output = operation => output, + _ = self.lost.cancelled() => { + return Err(ServingWriteLeaseLost { + message: "serving write lease heartbeat was lost".to_string(), + } + .into()) + } + }; + self.verify().await?; + Ok(output) + } + + /// Whether an error represents loss of a durable serving-write lease. + pub fn is_lease_lost(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() + } + + /// Whether a serving-write acquisition failed because the tenant is fenced. + pub fn acquisition_is_fenced(error: &anyhow::Error) -> bool { + matches!( + error.downcast_ref::(), + Some(buzz_db::DbError::AccessDenied(_)) + ) + } + + /// Signal fired if the background lease heartbeat fails. + pub fn lost(&self) -> CancellationToken { + self.lost.clone() + } + + /// The durable lease token presented to a final database mutation. + pub fn lease(&self) -> &buzz_db::deletion::ServingWriteLease { + &self.lease + } + + /// Release the lease after the side effect completes. + pub async fn finish(mut self) -> Result<()> { + self.cancel.cancel(); + let released = self.store.release_serving_write_lease(&self.lease).await?; + self.finished = true; + if !released { + return Err(ServingWriteLeaseLost { + message: "serving write lease was already stale or released".to_string(), + } + .into()); + } + Ok(()) + } +} + +impl Drop for ServingWriteGuard { + fn drop(&mut self) { + self.cancel.cancel(); + if self.finished { + return; + } + let store = self.store.clone(); + let lease = self.lease.clone(); + tokio::spawn(async move { + let _ = store.release_serving_write_lease(&lease).await; + }); + } +} + +/// Acquire a serving-side external-effect lease without holding a pool connection. +/// +/// A separate short database lease per effect is intentional: it is the only +/// durable proof that deletion can drain S3/Redis/push work across replicas. +/// PostgreSQL lease-table churn is reaped and exported by the relay pool-metrics +/// task; operators should watch the deletion lease gauges documented by Helm. +pub async fn acquire_serving_write( + db: &Db, + community: buzz_core::CommunityId, + operation: &str, +) -> Result { + let store = store(db); + let owner = default_executor_id(); + let lease = store + .acquire_serving_write_lease(community, operation, &owner, DEFAULT_LEASE_DURATION) + .await?; + let heartbeat_store = store.clone(); + let mut heartbeat_lease = lease.clone(); + let cancel = CancellationToken::new(); + let heartbeat_cancel = cancel.clone(); + let lost = CancellationToken::new(); + let heartbeat_lost = lost.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(heartbeat_interval()); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + loop { + tokio::select! { + _ = heartbeat_cancel.cancelled() => return, + _ = interval.tick() => { + if heartbeat_store + .renew_serving_write_lease( + &mut heartbeat_lease, + DEFAULT_LEASE_DURATION, + ) + .await + .is_err() + { + heartbeat_lost.cancel(); + return; + } + } + } + } + }); + Ok(ServingWriteGuard { + store, + lease, + cancel, + lost, + finished: false, + }) +} + +/// CLI-only whole-community deletion commands. +#[derive(Subcommand)] +pub enum Command { + /// Persist a deletion request and freeze its initial cross-store inventory. + Submit { + /// Canonical community host. Defaults to RELAY_URL's authority. + #[arg(long)] + host: Option, + /// Operator identity recorded on the request. + #[arg(long)] + requested_by: String, + /// Optional reason for the request. + #[arg(long)] + reason: Option, + }, + /// List deletion requests as JSON. + List { + /// Maximum records. + #[arg(long, default_value_t = 100, value_parser = clap::value_parser!(u16).range(1..=1000))] + limit: u16, + }, + /// Inspect one request, including approval/checkpoints/errors. + Inspect { + /// Deletion request UUID. + id: Uuid, + }, + /// Explicitly approve the exact frozen inventory digest. + Approve { + /// Deletion request UUID. + id: Uuid, + /// Approving operator identity. + #[arg(long)] + approved_by: String, + /// Optional approval note. + #[arg(long)] + note: Option, + }, + /// Claim and run one request until terminal/blocked. + Run { + /// Deletion request UUID. + id: Uuid, + /// Executor identity (defaults to hostname/pid). + #[arg(long)] + executor_id: Option, + }, + /// Drain the currently runnable deletion queue, then exit. + Drain { + /// Executor identity (defaults to hostname/pid). + #[arg(long)] + executor_id: Option, + }, + /// Sweep the whole bucket's key taxonomy and record observational evidence. + /// + /// This is independent of community deletion. It reports unknown writer + /// shapes but never gates submission or destructive progress. + Sweep, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LoopMode { + Run, + Drain, +} + +impl LoopMode { + const fn as_str(self) -> &'static str { + match self { + Self::Run => "run", + Self::Drain => "drain", + } + } +} + +#[derive(Clone)] +struct Services { + store: DeletionStore, + media: Arc, + redis: deadpool_redis::Pool, +} + +#[derive(Debug, thiserror::Error)] +enum EngineError { + #[error("permanent deletion safety failure: {0}")] + Permanent(String), + #[error("transient deletion dependency failure: {0}")] + Transient(String), +} + +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +struct PermanentSource(#[from] anyhow::Error); + +fn permanent(message: impl Into) -> anyhow::Error { + EngineError::Permanent(message.into()).into() +} + +fn permanent_source(error: impl Into) -> anyhow::Error { + PermanentSource(error.into()).into() +} + +fn transient(message: impl Into) -> anyhow::Error { + EngineError::Transient(message.into()).into() +} + +fn is_permanent_error(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause.is::() + || matches!( + cause.downcast_ref::(), + Some(buzz_db::DbError::DeletionSafety(_)) + ) + || cause + .downcast_ref::() + .is_some_and(|error| matches!(error, EngineError::Permanent(_))) + }) +} + +#[derive(Debug, Serialize)] +struct RunOutput { + request_id: Uuid, + stage: DeletionStage, + blocked_reason: Option, +} + +/// Execute one nested deletion command. +pub async fn run(command: Command) -> Result { + match command { + Command::List { limit } => { + let store = connect_store().await?; + print_json(&store.list(i64::from(limit)).await?)?; + Ok(0) + } + Command::Inspect { id } => { + let store = connect_store().await?; + print_json(&store.inspect(id).await?)?; + Ok(0) + } + Command::Approve { + id, + approved_by, + note, + } => { + let store = connect_store().await?; + print_json(&store.approve(id, &approved_by, note.as_deref()).await?)?; + Ok(0) + } + command => run_with_services(command, connect_services().await?).await, + } +} + +async fn run_with_services(command: Command, services: Services) -> Result { + match command { + Command::Submit { + host, + requested_by, + reason, + } => { + let relay_url = std::env::var("RELAY_URL").ok(); + let host = resolve_submit_host(host.as_deref(), relay_url.as_deref())?; + let request = services + .store + .submit(&host, &requested_by, reason.as_deref()) + .await?; + let inventory = build_inventory(&services, &request).await?; + let request = services + .store + .freeze_inventory(request.id, &inventory) + .await?; + print_json(&request)?; + Ok(0) + } + Command::Run { id, executor_id } => { + run_loop( + services, + LoopMode::Run, + Some(id), + executor_id.unwrap_or_else(default_executor_id), + ) + .await + } + Command::Drain { executor_id } => { + run_loop( + services, + LoopMode::Drain, + None, + executor_id.unwrap_or_else(default_executor_id), + ) + .await + } + Command::Sweep => { + let started_at = chrono::Utc::now(); + let cap = sweep_object_cap(); + let media = Arc::clone(&services.media); + let outcome = + buzz_media::sweep_bucket_taxonomy(cap, SWEEP_UNKNOWN_KEY_SAMPLE, move |token| { + let media = Arc::clone(&media); + async move { media.list_page(token, LIST_PAGE_SIZE).await } + }) + .await?; + let sweep = services + .store + .record_taxonomy_sweep( + started_at, + outcome.listed_objects, + outcome.unknown_object_count, + &outcome.unknown_key_sample, + cap, + ) + .await?; + print_json(&sweep)?; + Ok(i32::from(sweep.unknown_object_count > 0)) + } + Command::List { .. } | Command::Inspect { .. } | Command::Approve { .. } => { + anyhow::bail!("database-only command reached full-service dispatcher") + } + } +} + +fn resolve_submit_host(host: Option<&str>, relay_url: Option<&str>) -> Result { + if let Some(host) = host { + let host = host.trim(); + if host.is_empty() { + anyhow::bail!("--host must not be empty"); + } + return Ok(host.to_owned()); + } + + let relay_url = relay_url + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + anyhow::anyhow!("cannot derive community host; pass --host or set RELAY_URL") + })?; + let host = buzz_core::tenant::relay_url_authority(relay_url); + if host.is_empty() { + anyhow::bail!( + "cannot derive community host from RELAY_URL; pass --host or set a valid RELAY_URL" + ); + } + Ok(host) +} + +async fn connect_store() -> Result { + let database_url = required_env("DATABASE_URL")?; + let db = Db::new(&DbConfig { + database_url, + max_connections: env_parse("BUZZ_DB_POOL_SIZE", 20), + ..DbConfig::default() + }) + .await?; + Ok(store(&db)) +} + +async fn connect_services() -> Result { + let store = connect_store().await?; + let media_config = buzz_media::MediaConfig { + s3_endpoint: required_env("BUZZ_S3_ENDPOINT")?, + s3_access_key: required_env("BUZZ_S3_ACCESS_KEY")?, + s3_secret_key: required_env("BUZZ_S3_SECRET_KEY")?, + s3_bucket: required_env("BUZZ_S3_BUCKET")?, + s3_region: std::env::var("BUZZ_S3_REGION") + .or_else(|_| std::env::var("AWS_REGION")) + .map_err(|_| anyhow::anyhow!("BUZZ_S3_REGION or AWS_REGION is required"))?, + s3_addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE") + .unwrap_or_else(|_| "path".to_string()) + .parse() + .map_err(anyhow::Error::msg)?, + max_image_bytes: 1, + max_gif_bytes: 1, + max_video_bytes: 1, + max_file_bytes: 1, + public_base_url: "http://localhost/media".to_string(), + upload_records_enabled: false, + upload_ip_header: None, + upload_port_header: None, + }; + let media = Arc::new(MediaStorage::new(&media_config)?); + let redis_url = required_env("REDIS_URL")?; + let mut redis_config = deadpool_redis::Config::from_url(&redis_url); + redis_config.pool = Some(deadpool_redis::PoolConfig::new(env_parse( + "BUZZ_REDIS_POOL_SIZE", + 16, + ))); + let redis = redis_config + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .context("create deletion Redis pool")?; + Ok(Services { + store, + media, + redis, + }) +} + +fn required_env(name: &str) -> Result { + std::env::var(name) + .ok() + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) + .ok_or_else(|| anyhow::anyhow!("{name} is required for community deletion")) +} + +fn env_parse(name: &str, default: T) -> T +where + T: std::str::FromStr, +{ + std::env::var(name) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) +} + +fn validate_frozen_inventory(request: &DeletionRequest) -> Result { + let frozen: FrozenInventory = serde_json::from_value( + request + .inventory_manifest + .clone() + .ok_or_else(|| permanent("approved request has no frozen inventory"))?, + ) + .map_err(permanent_source)?; + let expected_digest = request + .inventory_digest + .as_deref() + .ok_or_else(|| permanent("approved request has no frozen inventory digest"))?; + let actual_digest = hex::encode(frozen.digest().map_err(permanent_source)?); + if actual_digest != expected_digest { + return Err(permanent("approved frozen inventory digest mismatch")); + } + validate_storage_ownership(request, &frozen.storage)?; + Ok(frozen) +} + +fn validate_storage_ownership(request: &DeletionRequest, manifest: &StorageManifest) -> Result<()> { + buzz_db::deletion::validate_storage_manifest(manifest)?; + let expected = tenant_prefixes(*request.community_id.as_uuid()); + let actual = manifest + .prefixes + .iter() + .map(|prefix| prefix.prefix.as_str()) + .collect::>(); + if actual != expected.iter().map(String::as_str).collect::>() { + return Err(permanent( + "storage manifest prefixes are not the deletion target's tenant prefixes", + )); + } + Ok(()) +} + +async fn build_inventory( + services: &Services, + request: &DeletionRequest, +) -> Result { + let schema = services + .store + .inventory_schema(request.community_id) + .await?; + let storage = enumerate_tenant_prefixes(services, request, None, None).await?; + Ok(FrozenInventory { schema, storage }) +} + +/// Buffered writer for frozen key chunks during the destructive freeze. +struct ChunkSink<'a> { + token: &'a LeaseToken, + next_chunk_no: i64, + buffered: Vec, +} + +async fn flush_chunk(services: &Services, sink: &mut ChunkSink<'_>, prefix: &str) -> Result<()> { + if sink.buffered.is_empty() { + return Ok(()); + } + services + .store + .append_manifest_key_chunk(sink.token, sink.next_chunk_no, prefix, &sink.buffered) + .await?; + sink.next_chunk_no += 1; + sink.buffered.clear(); + Ok(()) +} + +/// Enumerate the target's three tenant prefixes into per-prefix summaries. +/// +/// Cost is O(tenant objects) regardless of fleet size. Unknown shapes inside +/// one of the owned prefixes fail closed; keys elsewhere in the shared bucket +/// are outside this operation's contract. Memory stays bounded at one listing +/// page plus one buffered chunk — the full key list is never materialized. +/// When `sink` is supplied, keys are also persisted as side-table chunks +/// (never spanning prefixes) for the destructive freeze to bind against these +/// digests. +async fn enumerate_tenant_prefixes( + services: &Services, + request: &DeletionRequest, + heartbeat_lost: Option<&CancellationToken>, + mut sink: Option<&mut ChunkSink<'_>>, +) -> Result { + if services.media.bucket_versioning_detected().await? { + return Err(permanent( + "bucket versioning detected; deletion cannot prove logical absence with delete markers", + )); + } + let community = *request.community_id.as_uuid(); + let chunk_keys = manifest_chunk_keys(); + let mut prefixes = Vec::new(); + for prefix in tenant_prefixes(community) { + let mut digest = KeyStreamDigest::new(); + let mut total_bytes: u64 = 0; + let mut continuation = None; + loop { + if heartbeat_lost.is_some_and(CancellationToken::is_cancelled) { + return Err(DeletionLeaseLost.into()); + } + let page = services + .media + .list_prefix_page(&prefix, continuation.take(), LIST_PAGE_SIZE) + .await?; + for (key, size) in page.objects { + if !is_tenant_owned_key(community, &key) { + return Err(permanent(format!( + "key under a tenant prefix is outside the exact writer taxonomy: {key}" + ))); + } + digest.fold(&key)?; + total_bytes = total_bytes.saturating_add(size); + if let Some(sink) = sink.as_deref_mut() { + sink.buffered.push(key); + if sink.buffered.len() >= chunk_keys { + flush_chunk(services, sink, &prefix).await?; + } + } + } + if !page.is_truncated { + break; + } + continuation = page.next_continuation_token; + if continuation.is_none() { + return Err(transient( + "truncated tenant listing page has no continuation token", + )); + } + } + if let Some(sink) = sink.as_deref_mut() { + flush_chunk(services, sink, &prefix).await?; + } + let (keys_digest, object_count) = digest.finish(); + prefixes.push(PrefixManifest { + prefix, + object_count, + total_bytes, + keys_digest, + }); + } + let manifest = StorageManifest { + version: 4, + prefixes, + }; + buzz_db::deletion::validate_storage_manifest(&manifest)?; + Ok(manifest) +} + +/// Freeze the post-fence, post-drain destructive enumeration: stream the +/// tenant prefixes into side-table chunks, then bind the chunk stream to the +/// request row's digests atomically. +async fn freeze_destructive_manifest( + services: &Services, + request: &DeletionRequest, + token: &LeaseToken, + heartbeat_lost: &CancellationToken, +) -> Result { + validate_frozen_inventory(request)?; + // A prior interrupted freeze may have left partial chunks; they were + // never bound to a committed manifest, so rewrite them from scratch. + services.store.clear_manifest_key_chunks(token).await?; + let mut sink = ChunkSink { + token, + next_chunk_no: 0, + buffered: Vec::new(), + }; + let manifest = + enumerate_tenant_prefixes(services, request, Some(heartbeat_lost), Some(&mut sink)).await?; + services + .store + .freeze_destructive_storage_manifest(token, &manifest) + .await?; + Ok(manifest) +} + +async fn run_loop( + services: Services, + mode: LoopMode, + request_id: Option, + executor_id: String, +) -> Result { + let shutdown = shutdown_token(); + let mut ran = false; + loop { + if shutdown.is_cancelled() { + services + .store + .stop_executor(None, &executor_id) + .await + .context("record executor drain")?; + return Ok(0); + } + let claim = match request_id { + Some(id) => { + services + .store + .claim_specific(id, &executor_id, DEFAULT_LEASE_DURATION) + .await? + } + None => { + services + .store + .claim_next(&executor_id, DEFAULT_LEASE_DURATION) + .await? + } + }; + let Some(claim) = claim else { + if mode == LoopMode::Run && !ran { + anyhow::bail!( + "deletion request is not runnable, is blocked, or is leased by another executor" + ); + } + return Ok(0); + }; + ran = true; + let output = execute_claim(&services, mode, claim, &shutdown).await?; + print_json(&output)?; + if mode == LoopMode::Run || shutdown.is_cancelled() { + return Ok(i32::from(output.blocked_reason.is_some())); + } + } +} + +async fn stop_claim_executor( + services: &Services, + mode: LoopMode, + token: &LeaseToken, +) -> Result<()> { + // A failed draining heartbeat must not prevent the generation-checked release + // attempt. `stop_executor` cannot clear a successor's reclaimed lease. + let _ = services + .store + .heartbeat(token, mode.as_str(), DEFAULT_LEASE_DURATION, true) + .await; + services + .store + .stop_executor(Some(token), &token.owner) + .await?; + Ok(()) +} + +async fn record_stage_failure( + services: &Services, + token: &LeaseToken, + stage: DeletionStage, + error: &anyhow::Error, +) -> Result { + let message = format!("{error:#}"); + let result = if is_permanent_error(error) { + services.store.block(token, stage, "stage", &message).await + } else { + services + .store + .record_retry(token, stage, "stage", &message, RETRY_DELAY) + .await + }; + match result { + Ok(()) => Ok(true), + Err(error) if buzz_db::deletion::is_stale_deletion_lease(&error) => Ok(false), + Err(error) => Err(error.into()), + } +} + +async fn execute_claim( + services: &Services, + mode: LoopMode, + mut claim: ClaimedDeletion, + shutdown: &CancellationToken, +) -> Result { + let token = claim.lease.clone(); + loop { + if shutdown.is_cancelled() { + stop_claim_executor(services, mode, &token).await?; + let request = services.store.get(token.request_id).await?; + return Ok(run_output(request)); + } + services + .store + .heartbeat(&token, mode.as_str(), DEFAULT_LEASE_DURATION, false) + .await?; + let stage_result = run_stage_with_heartbeat(services, mode, &claim, shutdown).await; + match stage_result { + StageOutcome::Completed => {} + StageOutcome::Shutdown => { + stop_claim_executor(services, mode, &token).await?; + let request = services.store.get(token.request_id).await?; + return Ok(run_output(request)); + } + StageOutcome::Failed(error) => { + let request = services.store.get(token.request_id).await?; + if request.lease_owner.as_deref() != Some(&token.owner) + || request.lease_generation != token.generation + { + return Ok(run_output(request)); + } + if !record_stage_failure(services, &token, claim.request.stage, &error).await? { + // Ownership expired between the stage failure and durable + // error recording. A successor now owns retry/block policy. + let request = services.store.get(token.request_id).await?; + return Ok(run_output(request)); + } + let request = services.store.get(token.request_id).await?; + return Ok(run_output(request)); + } + } + let request = services.store.get(token.request_id).await?; + if request.stage == DeletionStage::RetentionPending || request.blocked_reason.is_some() { + return Ok(run_output(request)); + } + claim.request = request.clone(); + claim.lease.fence_generation = request.fence_generation; + } +} + +enum StageOutcome { + Completed, + Shutdown, + Failed(anyhow::Error), +} + +async fn await_stage( + stage: F, + shutdown: &CancellationToken, + heartbeat_error: &CancellationToken, +) -> StageOutcome +where + F: std::future::Future>, +{ + tokio::select! { + biased; + _ = shutdown.cancelled() => StageOutcome::Shutdown, + _ = heartbeat_error.cancelled() => StageOutcome::Failed(DeletionLeaseLost.into()), + result = stage => match result { + Ok(()) => StageOutcome::Completed, + Err(error) => StageOutcome::Failed(error), + }, + } +} + +async fn run_stage_with_heartbeat( + services: &Services, + mode: LoopMode, + claim: &ClaimedDeletion, + shutdown: &CancellationToken, +) -> StageOutcome { + let heartbeat_services = services.clone(); + let heartbeat_token = claim.lease.clone(); + let heartbeat_mode = mode.as_str(); + let heartbeat_shutdown = CancellationToken::new(); + let heartbeat_cancel = heartbeat_shutdown.clone(); + let heartbeat_error = CancellationToken::new(); + let heartbeat_error_signal = heartbeat_error.clone(); + let heartbeat = tokio::spawn(async move { + let mut interval = tokio::time::interval(heartbeat_interval()); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + loop { + tokio::select! { + _ = heartbeat_cancel.cancelled() => return, + _ = interval.tick() => { + if heartbeat_services + .store + .heartbeat( + &heartbeat_token, + heartbeat_mode, + DEFAULT_LEASE_DURATION, + false, + ) + .await + .is_err() + { + heartbeat_error_signal.cancel(); + return; + } + } + } + } + }); + + let stage = await_stage( + execute_stage(services, claim, &heartbeat_error), + shutdown, + &heartbeat_error, + ) + .await; + heartbeat_shutdown.cancel(); + match heartbeat.await { + Ok(()) => stage, + Err(error) => { + StageOutcome::Failed(anyhow::anyhow!("deletion heartbeat task failed: {error}")) + } + } +} + +async fn run_guarded_external_step( + services: &Services, + token: &LeaseToken, + stage: DeletionStage, + heartbeat_lost: &CancellationToken, + operation: F, +) -> Result +where + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + services.store.verify_execution_token(token, stage).await?; + let result = tokio::select! { + biased; + _ = heartbeat_lost.cancelled() => { + return Err(DeletionLeaseLost.into()); + } + result = operation() => result, + }; + let output = result?; + services.store.verify_execution_token(token, stage).await?; + Ok(output) +} + +async fn execute_stage( + services: &Services, + claim: &ClaimedDeletion, + heartbeat_lost: &CancellationToken, +) -> Result<()> { + let request = &claim.request; + let token = token_with_current_fence(&claim.lease, request); + if matches!( + request.stage, + DeletionStage::Approved + | DeletionStage::Fenced + | DeletionStage::Drained + | DeletionStage::BindingsRemoved + | DeletionStage::PostgresPurged + | DeletionStage::CachePurged + | DeletionStage::LogicallyVerified + ) { + validate_frozen_inventory(request)?; + } + match request.stage { + DeletionStage::Approved => { + // Approval binds immutable catalog + community-prefix ownership. + // Live row counts and tenant binding keys are deliberately not + // equality-bound until the durable fence closes all writers. + let live_schema = services + .store + .inventory_schema(request.community_id) + .await?; + let frozen = validate_frozen_inventory(request)?; + if live_schema != frozen.schema { + return Err(permanent( + "approved structural catalog drifted before fencing", + )); + } + services.store.begin_quiescing(&token).await?; + match services.store.fence(&token).await { + Ok(_) => {} + Err(buzz_db::DbError::ServingWritesNotDrained { + active_count, + operations, + .. + }) => { + return Err(transient(format!( + "serving writes not drained before fence: count={active_count}, operations={operations:?}" + ))); + } + Err(error) => return Err(error.into()), + } + } + DeletionStage::Fenced => { + services + .store + .verify_execution_token(&token, DeletionStage::Fenced) + .await?; + let disconnect = tokio::select! { + biased; + _ = heartbeat_lost.cancelled() => Err(DeletionLeaseLost.into()), + result = publish_disconnect_community(&services.redis, request.community_id) => result, + }; + disconnect?; + services + .store + .verify_execution_token(&token, DeletionStage::Fenced) + .await?; + if !services + .store + .serving_writes_drained(request.community_id) + .await? + { + return Err(transient("serving writes have not drained")); + } + // Freeze the destructive enumeration only after the fence closed + // new writers AND every admitted serving write drained: the + // post-drain listing is the final storage state, so no tenant key + // can appear after the freeze. + let destructive = match request.destructive_storage_manifest.clone() { + Some(value) => serde_json::from_value(value)?, + None => { + freeze_destructive_manifest(services, request, &token, heartbeat_lost).await? + } + }; + validate_storage_ownership(request, &destructive)?; + services.store.mark_drained(&token).await?; + } + DeletionStage::Drained => { + let storage: StorageManifest = serde_json::from_value( + request + .destructive_storage_manifest + .clone() + .context("request has no post-fence destructive storage manifest")?, + )?; + validate_storage_ownership(request, &storage)?; + // Resume = first unstamped chunk. Bulk deletes are idempotent + // (missing keys report as deleted), so re-deleting a chunk whose + // stamp was lost to a crash is safe. + let mut removed: u64 = 0; + let mut already_missing: u64 = 0; + while let Some(chunk) = services.store.next_pending_manifest_chunk(&token).await? { + let outcome = run_guarded_external_step( + services, + &token, + DeletionStage::Drained, + heartbeat_lost, + || async { Ok(services.media.delete_objects(&chunk.keys).await?) }, + ) + .await?; + if !outcome.versioned_keys.is_empty() { + return Err(permanent(format!( + "bulk delete produced version artifacts; bucket versioning blocks \ + deletion: {}", + outcome.versioned_keys.join(",") + ))); + } + if !outcome.failed.is_empty() { + let (key, code, message) = &outcome.failed[0]; + return Err(transient(format!( + "bulk delete failed for {} key(s); first: {key}: {code}: {message}", + outcome.failed.len() + ))); + } + let acknowledged = outcome.deleted.saturating_add(outcome.already_missing); + if acknowledged != chunk.keys.len() as u64 { + return Err(transient(format!( + "bulk delete acknowledged {acknowledged} of {} keys in chunk {}", + chunk.keys.len(), + chunk.chunk_no + ))); + } + removed += outcome.deleted; + already_missing += outcome.already_missing; + services + .store + .mark_manifest_chunk_deleted( + &token, + chunk.chunk_no, + serde_json::json!({ + "prefix": chunk.prefix, + "keys": chunk.keys.len(), + "deleted": outcome.deleted, + "already_missing": outcome.already_missing, + }), + ) + .await?; + } + let frozen_keys: u64 = storage + .prefixes + .iter() + .map(|prefix| prefix.object_count) + .sum(); + services + .store + .mark_bindings_removed( + &token, + serde_json::json!({ + "deleted_keys": frozen_keys, + "removed_now": removed, + "already_missing": already_missing, + }), + ) + .await?; + } + DeletionStage::BindingsRemoved => { + services.store.purge_postgres(&token).await?; + } + DeletionStage::PostgresPurged => { + services + .store + .verify_execution_token(&token, DeletionStage::PostgresPurged) + .await?; + let deleted = purge_redis_namespace(&services.redis, request.community_id).await?; + services + .store + .verify_execution_token(&token, DeletionStage::PostgresPurged) + .await?; + services + .store + .mark_cache_purged(&token, serde_json::json!({"deleted_keys": deleted})) + .await?; + } + DeletionStage::CachePurged => { + services + .store + .verify_postgres_logically_deleted(&token) + .await?; + verify_storage_absence(services, request).await?; + verify_redis_absence(&services.redis, request.community_id).await?; + services + .store + .mark_logically_verified( + &token, + serde_json::json!({"postgres": true, "object_store": true, "redis": true}), + ) + .await?; + } + DeletionStage::LogicallyVerified => { + validate_frozen_inventory(request)?; + services + .store + .mark_retention_pending( + &token, + serde_json::json!({ + "policy": "member-erasure and fleet-wide shared-CAS GC are out of V1 scope" + }), + ) + .await?; + } + DeletionStage::Submitted | DeletionStage::Inventoried => { + anyhow::bail!("request has not crossed the explicit approval boundary") + } + DeletionStage::RetentionPending => {} + } + Ok(()) +} + +fn token_with_current_fence(token: &LeaseToken, request: &DeletionRequest) -> LeaseToken { + LeaseToken { + fence_generation: request.fence_generation, + ..token.clone() + } +} + +/// Prove logical absence by listing each tenant prefix and requiring it +/// empty — O(1) requests per prefix, independent of fleet size. +async fn verify_storage_absence(services: &Services, request: &DeletionRequest) -> Result<()> { + for prefix in tenant_prefixes(*request.community_id.as_uuid()) { + let page = services.media.list_prefix_page(&prefix, None, 1).await?; + if let Some((key, _)) = page.objects.first() { + return Err(transient(format!( + "logical verification found a live target object binding: {key}" + ))); + } + } + Ok(()) +} + +async fn publish_disconnect_community( + pool: &deadpool_redis::Pool, + community: buzz_core::CommunityId, +) -> Result<()> { + let mut connection = pool.get().await?; + let channel = format!("buzz:{community}:conn-control"); + let _: u64 = redis::cmd("PUBLISH") + .arg(channel) + .arg(r#"{"op":"DisconnectCommunity"}"#) + .query_async(&mut *connection) + .await?; + Ok(()) +} + +async fn purge_redis_namespace( + pool: &deadpool_redis::Pool, + community: buzz_core::CommunityId, +) -> Result { + let mut connection = pool.get().await?; + let pattern = format!("buzz:{community}:*"); + let mut cursor = 0u64; + let mut deleted = 0u64; + loop { + let (next, keys): (u64, Vec) = redis::cmd("SCAN") + .arg(cursor) + .arg("MATCH") + .arg(&pattern) + .arg("COUNT") + .arg(1000) + .query_async(&mut *connection) + .await?; + if !keys.is_empty() { + let count: u64 = redis::cmd("UNLINK") + .arg(&keys) + .query_async(&mut *connection) + .await?; + deleted = deleted.saturating_add(count); + } + if next == 0 { + break; + } + cursor = next; + } + Ok(deleted) +} + +fn scan_proves_absence(pages: &[(u64, Vec)]) -> bool { + pages.last().is_some_and(|(cursor, _)| *cursor == 0) + && pages.iter().all(|(_, keys)| keys.is_empty()) +} + +async fn scan_redis_namespace( + connection: &mut deadpool_redis::Connection, + pattern: &str, +) -> Result)>> { + let mut cursor = 0u64; + let mut pages = Vec::new(); + loop { + let page: (u64, Vec) = redis::cmd("SCAN") + .arg(cursor) + .arg("MATCH") + .arg(pattern) + .arg("COUNT") + .arg(1000) + .query_async(&mut **connection) + .await?; + cursor = page.0; + pages.push(page); + if cursor == 0 { + return Ok(pages); + } + } +} + +async fn verify_redis_absence( + pool: &deadpool_redis::Pool, + community: buzz_core::CommunityId, +) -> Result<()> { + let mut connection = pool.get().await?; + let pattern = format!("buzz:{community}:*"); + // SCAN is weakly consistent. Two complete empty passes ensure a cursor + // rollover or concurrent expiry cannot make one sparse pass look absent. + let first = scan_redis_namespace(&mut connection, &pattern).await?; + let second = scan_redis_namespace(&mut connection, &pattern).await?; + if scan_proves_absence(&first) && scan_proves_absence(&second) { + Ok(()) + } else { + Err(transient( + "logical verification found a Redis namespace key", + )) + } +} + +fn sweep_object_cap() -> u64 { + std::env::var("BUZZ_DELETION_SWEEP_MAX_OBJECTS") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_SWEEP_OBJECT_CAP) +} + +fn manifest_chunk_keys() -> usize { + std::env::var("BUZZ_DELETION_MANIFEST_CHUNK_KEYS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .map(|value| value.min(100_000)) + .unwrap_or(DEFAULT_MANIFEST_CHUNK_KEYS) +} + +fn default_executor_id() -> String { + let hostname = std::env::var("HOSTNAME").unwrap_or_else(|_| "buzz-admin".to_string()); + format!("{hostname}:{}", std::process::id()) +} + +fn shutdown_token() -> CancellationToken { + let token = CancellationToken::new(); + let signal = token.clone(); + tokio::spawn(async move { + #[cfg(unix)] + { + use tokio::signal::unix::{signal as unix_signal, SignalKind}; + if let Ok(mut terminate) = unix_signal(SignalKind::terminate()) { + tokio::select! { + _ = tokio::signal::ctrl_c() => {}, + _ = terminate.recv() => {}, + } + } else { + let _ = tokio::signal::ctrl_c().await; + } + } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + } + signal.cancel(); + }); + token +} + +fn run_output(request: DeletionRequest) -> RunOutput { + RunOutput { + request_id: request.id, + stage: request.stage, + blocked_reason: request.blocked_reason, + } +} + +fn print_json(value: &impl Serialize) -> Result<()> { + println!("{}", serde_json::to_string_pretty(value)?); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn submit_host_prefers_explicit_host() { + assert_eq!( + resolve_submit_host(Some(" community.example "), Some("wss://ignored.example")) + .expect("explicit host"), + "community.example" + ); + } + + #[test] + fn submit_host_derives_from_relay_url() { + assert_eq!( + resolve_submit_host(None, Some("wss://relay.example:8443/path")) + .expect("relay URL host"), + "relay.example:8443" + ); + } + + #[test] + fn submit_host_requires_an_explicit_source() { + for relay_url in [None, Some(""), Some(" ")] { + let error = resolve_submit_host(None, relay_url).expect_err("missing host must fail"); + assert!(error.to_string().contains("pass --host or set RELAY_URL")); + } + } + + #[test] + fn submit_host_rejects_empty_or_invalid_values() { + assert!(resolve_submit_host(Some(" "), Some("wss://relay.example")).is_err()); + assert!(resolve_submit_host(None, Some("not a URL")).is_err()); + } + + fn empty_storage_manifest(community: buzz_core::CommunityId) -> StorageManifest { + StorageManifest { + version: 4, + prefixes: tenant_prefixes(*community.as_uuid()) + .into_iter() + .map(|prefix| PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + }) + .collect(), + } + } + + async fn claimed_test_deletion(prefix: &str) -> (Services, ClaimedDeletion) { + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .expect("BUZZ_TEST_DATABASE_URL or DATABASE_URL is required"); + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect deletion engine test DB"); + let db = Db::from_pool(pool); + db.migrate().await.expect("migrate deletion engine test DB"); + let store = db.deletion_store(); + let host = format!("{prefix}-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("create deletion engine test community"); + let request = store + .submit(&host, "test", None) + .await + .expect("submit deletion request"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community.id) + .await + .expect("inventory schema"), + storage: empty_storage_manifest(community.id), + }; + store + .freeze_inventory(request.id, &inventory) + .await + .expect("freeze deletion inventory"); + store + .approve(request.id, "test", None) + .await + .expect("approve deletion request"); + let claim = store + .claim_specific(request.id, "test-executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim deletion request") + .expect("runnable deletion request"); + let services = Services { + store, + media: Arc::new( + MediaStorage::new(&buzz_media::MediaConfig { + s3_endpoint: "http://127.0.0.1:1".to_string(), + s3_access_key: "unused".to_string(), + s3_secret_key: "unused".to_string(), + s3_bucket: "unused".to_string(), + s3_region: "us-east-1".to_string(), + s3_addressing_style: buzz_media::S3AddressingStyle::Path, + max_image_bytes: 1, + max_gif_bytes: 1, + max_video_bytes: 1, + max_file_bytes: 1, + public_base_url: "http://localhost/media".to_string(), + upload_records_enabled: false, + upload_ip_header: None, + upload_port_header: None, + }) + .expect("construct unused media service"), + ), + redis: deadpool_redis::Config::from_url("redis://127.0.0.1:1") + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .expect("construct unused Redis pool"), + }; + (services, claim) + } + + fn deletion_test_media_storage() -> Arc { + let endpoint = std::env::var("BUZZ_TEST_S3_ENDPOINT") + .or_else(|_| std::env::var("BUZZ_S3_ENDPOINT")) + .expect("BUZZ_TEST_S3_ENDPOINT or BUZZ_S3_ENDPOINT is required"); + let access_key = std::env::var("BUZZ_TEST_S3_ACCESS_KEY") + .or_else(|_| std::env::var("BUZZ_S3_ACCESS_KEY")) + .expect("BUZZ_TEST_S3_ACCESS_KEY or BUZZ_S3_ACCESS_KEY is required"); + let secret_key = std::env::var("BUZZ_TEST_S3_SECRET_KEY") + .or_else(|_| std::env::var("BUZZ_S3_SECRET_KEY")) + .expect("BUZZ_TEST_S3_SECRET_KEY or BUZZ_S3_SECRET_KEY is required"); + let bucket = std::env::var("BUZZ_TEST_S3_BUCKET") + .or_else(|_| std::env::var("BUZZ_S3_BUCKET")) + .expect("BUZZ_TEST_S3_BUCKET or BUZZ_S3_BUCKET is required"); + Arc::new( + MediaStorage::new(&buzz_media::MediaConfig { + s3_endpoint: endpoint, + s3_access_key: access_key, + s3_secret_key: secret_key, + s3_bucket: bucket, + s3_region: std::env::var("BUZZ_TEST_S3_REGION") + .or_else(|_| std::env::var("BUZZ_S3_REGION")) + .unwrap_or_else(|_| "us-east-1".to_string()), + s3_addressing_style: buzz_media::S3AddressingStyle::Path, + max_image_bytes: 1, + max_gif_bytes: 1, + max_video_bytes: 1, + max_file_bytes: 1, + public_base_url: "http://localhost/media".to_string(), + upload_records_enabled: false, + upload_ip_header: None, + upload_port_header: None, + }) + .expect("construct deletion test media service"), + ) + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn frozen_inventory_digest_and_storage_ownership_fail_closed() { + let (_, claim) = claimed_test_deletion("deletion-integrity").await; + assert!(validate_frozen_inventory(&claim.request).is_ok()); + + let mut digest_tampered = claim.request.clone(); + digest_tampered.inventory_manifest = Some(serde_json::json!({ + "schema": {"scoped_tables": [], "row_counts": {}, "fenced_tables": []}, + "storage": {"version": 4, "prefixes": []} + })); + assert!(validate_frozen_inventory(&digest_tampered).is_err()); + + // A manifest scoped to another community's prefixes is never the + // deletion target's, even when internally valid. + let foreign_manifest = + empty_storage_manifest(buzz_core::CommunityId::from_uuid(Uuid::new_v4())); + assert!(validate_storage_ownership(&claim.request, &foreign_manifest).is_err()); + } + + /// The non-atomic boundary under test: S3 committed a chunk's deletes, + /// then the worker died before the chunk stamp. Resume must re-delete the + /// chunk (missing keys report as deleted — idempotent), stamp it, and + /// finish the stage. + #[tokio::test] + #[ignore = "requires Postgres and S3-compatible storage"] + async fn drained_stage_resumes_chunk_deleted_before_stamp() { + let (mut services, claim) = claimed_test_deletion("deletion-chunk-resume").await; + services.media = deletion_test_media_storage(); + let community = claim.request.community_id; + let meta_prefix = format!("_meta/{community}/"); + let keys = vec![ + format!("{meta_prefix}{}.json", "a".repeat(64)), + format!("{meta_prefix}{}.json", "b".repeat(64)), + ]; + for key in &keys { + services + .media + .put(key, b"chunk-resume", "application/json") + .await + .expect("seed object"); + } + + services + .store + .begin_quiescing(&claim.lease) + .await + .expect("quiesce"); + let generation = services.store.fence(&claim.lease).await.expect("fence"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease.clone() + }; + // Two single-key chunks so resume order is observable. + services + .store + .append_manifest_key_chunk(&token, 0, &meta_prefix, &keys[..1]) + .await + .expect("append chunk 0"); + services + .store + .append_manifest_key_chunk(&token, 1, &meta_prefix, &keys[1..]) + .await + .expect("append chunk 1"); + let mut digest = KeyStreamDigest::new(); + for key in &keys { + digest.fold(key).expect("fold key"); + } + let (keys_digest, object_count) = digest.finish(); + let mut storage = empty_storage_manifest(community); + storage.prefixes[0] = PrefixManifest { + prefix: meta_prefix.clone(), + object_count, + total_bytes: keys.len() as u64 * "chunk-resume".len() as u64, + keys_digest, + }; + services + .store + .freeze_destructive_storage_manifest(&token, &storage) + .await + .expect("freeze chunked manifest"); + services.store.mark_drained(&token).await.expect("drained"); + + // Simulate the crash window: chunk 0's key is already gone from S3 + // but the chunk was never stamped. + services + .media + .delete(&keys[0]) + .await + .expect("simulate committed delete before stamp"); + + let resumed = ClaimedDeletion { + request: services + .store + .get(token.request_id) + .await + .expect("reload drained request"), + lease: claim.lease, + }; + execute_stage(&services, &resumed, &CancellationToken::new()) + .await + .expect("Drained stage resumes at the unstamped chunk"); + + assert_eq!( + services + .store + .manifest_chunk_progress(token.request_id) + .await + .expect("chunk progress"), + (2, 2) + ); + assert_eq!( + services.store.get(token.request_id).await.unwrap().stage, + DeletionStage::BindingsRemoved + ); + for key in &keys { + assert!( + !services.media.head(key).await.expect("verify absence"), + "tenant binding {key} must be gone" + ); + } + } + + #[test] + fn permanent_failures_are_typed_not_string_classified() { + let permanent_error = permanent("catalog drift"); + let transient_error = transient("temporary catalog service reset"); + let nested = permanent_source(anyhow::anyhow!("schema mismatch")).context("outer"); + let db_permanent = anyhow::Error::from(buzz_db::DbError::DeletionSafety( + "typed catalog drift".to_string(), + )); + let db_transient = anyhow::Error::from(buzz_db::DbError::Sqlx(sqlx::Error::PoolTimedOut)); + assert!(is_permanent_error(&permanent_error)); + assert!(is_permanent_error(&nested)); + assert!(is_permanent_error(&db_permanent)); + assert!(!is_permanent_error(&transient_error)); + assert!(!is_permanent_error(&db_transient)); + } + + #[test] + fn deletion_configuration_requires_every_destructive_dependency() { + let variable = format!("BUZZ_DELETION_REQUIRED_TEST_{}", Uuid::new_v4().simple()); + assert!(required_env(&variable).is_err()); + std::env::set_var(&variable, " "); + assert!(required_env(&variable).is_err()); + std::env::set_var(&variable, "configured"); + assert_eq!( + required_env(&variable).expect("configured environment variable"), + "configured" + ); + std::env::remove_var(&variable); + } + + #[test] + fn redis_absence_requires_terminal_cursor_and_all_pages_empty() { + assert!(!scan_proves_absence(&[(9, Vec::new())])); + assert!(!scan_proves_absence(&[ + (9, Vec::new()), + (0, vec!["buzz:tenant:late".to_string()]), + ])); + assert!(scan_proves_absence(&[(9, Vec::new()), (0, Vec::new())])); + } + + #[tokio::test] + #[ignore = "requires Postgres and S3-compatible storage"] + async fn final_storage_verification_rejects_late_target_binding() { + let (mut services, claim) = claimed_test_deletion("deletion-late-binding").await; + services.media = deletion_test_media_storage(); + let community = claim.request.community_id; + let late_key = format!("_meta/{community}/{}.json", "a".repeat(64)); + services + .media + .put(&late_key, b"late", "application/json") + .await + .expect("seed late binding"); + let error = verify_storage_absence(&services, &claim.request) + .await + .expect_err("late target binding must fail verification"); + assert!(format!("{error:#}").contains(&late_key)); + services + .media + .delete(&late_key) + .await + .expect("remove late binding"); + verify_storage_absence(&services, &claim.request) + .await + .expect("empty tenant prefixes verify clean"); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn stale_lease_during_failure_recording_is_lost_ownership() { + let (services, claim) = claimed_test_deletion("deletion-stale-record").await; + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .expect("test database URL"); + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect stale-record test DB"); + sqlx::query( + "UPDATE community_deletion_requests SET lease_until = now() - interval '1 second' WHERE id = $1", + ) + .bind(claim.request.id) + .execute(&pool) + .await + .expect("expire claim"); + let recorded = record_stage_failure( + &services, + &claim.lease, + claim.request.stage, + &transient("test failure"), + ) + .await + .expect("stale ownership is not fatal"); + assert!(!recorded); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn serving_guard_cancels_protected_operation_when_heartbeat_is_lost() { + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .expect("BUZZ_TEST_DATABASE_URL or DATABASE_URL is required"); + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect serving guard test DB"); + let db = Db::from_pool(pool.clone()); + db.migrate().await.expect("migrate serving guard test DB"); + let community = db + .ensure_configured_community(&format!( + "serving-guard-{}.example", + Uuid::new_v4().simple() + )) + .await + .expect("create test community") + .id; + TEST_HEARTBEAT_INTERVAL_MS.store(10, Ordering::Relaxed); + let guard = acquire_serving_write(&db, community, "test_cancel") + .await + .expect("serving guard"); + sqlx::query("DELETE FROM community_serving_write_leases WHERE community_id = $1") + .bind(community.as_uuid()) + .execute(&pool) + .await + .expect("force heartbeat failure"); + let completed = Arc::new(AtomicBool::new(false)); + let operation_completed = Arc::clone(&completed); + let result = guard + .protect(async move { + tokio::time::sleep(Duration::from_secs(1)).await; + operation_completed.store(true, Ordering::Relaxed); + }) + .await; + TEST_HEARTBEAT_INTERVAL_MS.store(0, Ordering::Relaxed); + assert!(result.is_err(), "lease loss must reject the operation"); + assert!( + !completed.load(Ordering::Relaxed), + "lease loss must cancel the protected operation future" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn guarded_external_step_rejects_preexisting_heartbeat_loss_without_polling_operation() { + let (services, claim) = claimed_test_deletion("deletion-heartbeat").await; + let heartbeat_lost = CancellationToken::new(); + heartbeat_lost.cancel(); + let polled = Arc::new(AtomicBool::new(false)); + let operation_polled = Arc::clone(&polled); + let result = run_guarded_external_step( + &services, + &claim.lease, + DeletionStage::Approved, + &heartbeat_lost, + || async move { + operation_polled.store(true, Ordering::Relaxed); + Ok(()) + }, + ) + .await; + assert!(result.is_err(), "heartbeat loss must abort the side effect"); + assert!( + result + .expect_err("heartbeat loss error") + .downcast_ref::() + .is_some(), + "heartbeat loss must stay typed" + ); + assert!( + !polled.load(Ordering::Relaxed), + "a pre-cancelled heartbeat must win before polling the operation" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn shutdown_during_stage_releases_claim_without_recording_retry() { + let (services, claim) = claimed_test_deletion("deletion-shutdown").await; + let request_id = claim.request.id; + let retry_count = claim.request.retry_count; + let shutdown = CancellationToken::new(); + let cancel = shutdown.clone(); + let services_for_run = services.clone(); + let executor = tokio::spawn(async move { + execute_claim(&services_for_run, LoopMode::Drain, claim, &shutdown).await + }); + tokio::time::sleep(Duration::from_millis(10)).await; + cancel.cancel(); + let output = tokio::time::timeout(Duration::from_secs(2), executor) + .await + .expect("shutdown must cancel the active stage") + .expect("deletion executor task") + .expect("graceful deletion executor shutdown"); + let request = services + .store + .get(request_id) + .await + .expect("load deletion request after shutdown"); + assert_eq!(output.stage, DeletionStage::Approved); + assert_eq!(request.stage, DeletionStage::Approved); + assert_eq!(request.retry_count, retry_count); + assert!(request.last_error.is_none()); + assert!(request.lease_owner.is_none()); + assert!(request.lease_until.is_none()); + } + + #[tokio::test] + async fn stage_wait_treats_shutdown_as_control_flow() { + let shutdown = CancellationToken::new(); + shutdown.cancel(); + let heartbeat_lost = CancellationToken::new(); + let outcome = await_stage( + std::future::pending::>(), + &shutdown, + &heartbeat_lost, + ) + .await; + assert!(matches!(outcome, StageOutcome::Shutdown)); + } + + #[tokio::test] + async fn stage_wait_prioritizes_heartbeat_loss_over_a_ready_operation() { + let shutdown = CancellationToken::new(); + let heartbeat_lost = CancellationToken::new(); + heartbeat_lost.cancel(); + let outcome = await_stage(async { Ok(()) }, &shutdown, &heartbeat_lost).await; + match outcome { + StageOutcome::Failed(error) => assert!( + error.downcast_ref::().is_some(), + "heartbeat loss must stay typed" + ), + StageOutcome::Completed | StageOutcome::Shutdown => { + panic!("preexisting heartbeat loss must win") + } + } + } +} diff --git a/crates/buzz-media/src/bucket_index.rs b/crates/buzz-media/src/bucket_index.rs index bb83dc517f..6c78c2e0a1 100644 --- a/crates/buzz-media/src/bucket_index.rs +++ b/crates/buzz-media/src/bucket_index.rs @@ -753,3 +753,258 @@ mod tests { ); } } + +/// The exact community-scoped listing prefixes owned by one tenant, in +/// ascending key order: media sidecars, upload records, and Git repository +/// pointers. Every tenant-owned binding lives under one of these; shared +/// immutable CAS/thumb/probe data is deliberately outside them (fleet-wide +/// physical GC is a separate retention phase). +pub fn tenant_prefixes(community: Uuid) -> [String; 3] { + [ + format!("_meta/{community}/"), + format!("_uploads/{community}/"), + format!("repos/{community}/"), + ] +} + +/// Whether one bucket key is a tenant-owned binding of `community` in the +/// exact writer taxonomy: a media sidecar, an upload record, or a Git +/// repository pointer. A malformed key under a tenant prefix is NOT owned — +/// deletion fails closed on shapes this binary did not write. +pub fn is_tenant_owned_key(community: Uuid, key: &str) -> bool { + match classify_key(key) { + KeyClass::Sidecar { + community: owner, .. + } + | KeyClass::Auxiliary { + community: owner, .. + } => owner == community, + KeyClass::Unknown => git_pointer_community(key) == Some(community), + KeyClass::Blob { .. } | KeyClass::Thumb { .. } => false, + } +} + +/// Whether one bucket key belongs to the fleet's known writer taxonomy: +/// blob/thumb/sidecar/upload shapes, any community's Git pointer, shared Git +/// CAS data, or a `probe/` connectivity key. +pub fn is_known_fleet_key(key: &str) -> bool { + !matches!(classify_key(key), KeyClass::Unknown) + || git_pointer_community(key).is_some() + || is_known_git_shared_key(key) + || key.starts_with("probe/") +} + +/// Durable outcome of one fleet-wide taxonomy sweep. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TaxonomySweepOutcome { + /// Total objects listed. + pub listed_objects: u64, + /// Exact count of keys outside the known writer taxonomy. + pub unknown_object_count: u64, + /// Bounded sample of unknown keys, in listing order. + pub unknown_key_sample: Vec, +} + +/// Fold an entire paginated bucket listing into the fleet taxonomy outcome. +/// +/// Deleting a tenant while the bucket contains a writer shape this binary +/// does not understand is unsafe — but that is a *fleet* invariant, not a +/// per-request one. This sweep records it once; deletion stages then gate on +/// a recent clean sweep instead of re-listing the whole bucket per request. +/// Same pagination/cap contract as [`fold_bucket_listing`]; memory is +/// bounded by `sample_limit`, never the listing size. +pub async fn sweep_bucket_taxonomy( + cap: u64, + sample_limit: usize, + mut fetch_page: F, +) -> Result +where + F: FnMut(Option) -> Fut, + Fut: Future>, +{ + let mut outcome = TaxonomySweepOutcome::default(); + let mut continuation_token = None; + loop { + let page = fetch_page(continuation_token.take()).await?; + outcome.listed_objects += page.objects.len() as u64; + if outcome.listed_objects > cap { + return Err(SweepError::CapExceeded { + seen: outcome.listed_objects, + cap, + }); + } + for (key, _size) in page.objects { + if !is_known_fleet_key(&key) { + outcome.unknown_object_count += 1; + if outcome.unknown_key_sample.len() < sample_limit { + outcome.unknown_key_sample.push(key); + } + } + } + if !page.is_truncated { + break; + } + match page.next_continuation_token { + Some(token) => continuation_token = Some(token), + None => return Err(SweepError::MalformedPage), + } + } + Ok(outcome) +} + +fn git_pointer_community(key: &str) -> Option { + let mut parts = key.split('/'); + if parts.next()? != "repos" { + return None; + } + let community = parse_canonical_uuid(parts.next()?)?; + let owner = parts.next()?; + let repo = parts.next()?; + let pointer = parts.next()?; + if parts.next().is_some() + || owner.len() != 64 + || !owner.bytes().all(|byte| byte.is_ascii_hexdigit()) + || repo.is_empty() + || repo.len() > 64 + || repo.starts_with('.') + || repo.contains("..") + || !repo + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) + || pointer != "pointer" + { + return None; + } + Some(community) +} + +fn is_known_git_shared_key(key: &str) -> bool { + ["packs/", "idx/", "manifests/"].iter().any(|prefix| { + key.strip_prefix(prefix).is_some_and(|digest| { + digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) + }) +} + +#[cfg(test)] +mod deletion_taxonomy_tests { + use super::*; + + #[test] + fn tenant_ownership_is_exact_per_community_and_shape() { + let target = Uuid::from_u128(1); + let other = Uuid::from_u128(2); + let sha = "a".repeat(64); + + assert!(is_tenant_owned_key( + target, + &format!("_meta/{target}/{sha}.json") + )); + assert!(is_tenant_owned_key( + target, + &format!("_uploads/{target}/{sha}/01ARZ3NDEKTSV4RRFFQ69G5FAV.json") + )); + assert!(is_tenant_owned_key( + target, + &format!("repos/{target}/{}/repo/pointer", "b".repeat(64)) + )); + // Another tenant's bindings and shared CAS are never owned. + assert!(!is_tenant_owned_key( + target, + &format!("_meta/{other}/{sha}.json") + )); + assert!(!is_tenant_owned_key(target, &format!("{sha}.png"))); + // A malformed key under the tenant's own prefix fails closed. + assert!(!is_tenant_owned_key( + target, + &format!("_meta/{target}/not-a-sidecar") + )); + assert!(!is_tenant_owned_key( + target, + &format!("repos/{target}/stray-file") + )); + } + + #[test] + fn tenant_prefixes_cover_every_owned_shape_and_sort_ascending() { + let community = Uuid::from_u128(7); + let prefixes = tenant_prefixes(community); + assert!(prefixes.windows(2).all(|pair| pair[0] < pair[1])); + let sha = "c".repeat(64); + for key in [ + format!("_meta/{community}/{sha}.json"), + format!("_uploads/{community}/{sha}/01ARZ3NDEKTSV4RRFFQ69G5FAV.json"), + format!("repos/{community}/{}/repo/pointer", "d".repeat(64)), + ] { + assert!( + prefixes + .iter() + .any(|prefix| key.starts_with(prefix.as_str())), + "owned key {key} must live under a tenant prefix" + ); + assert!(is_tenant_owned_key(community, &key)); + } + } + + #[tokio::test] + async fn taxonomy_sweep_counts_all_unknowns_but_bounds_the_sample() { + let community = Uuid::from_u128(1); + let sha = "a".repeat(64); + let known = vec![ + (format!("{sha}.png"), 1), + (format!("{sha}.thumb.jpg"), 1), + (format!("_meta/{community}/{sha}.json"), 1), + (format!("packs/{sha}"), 1), + ( + format!("repos/{community}/{}/repo/pointer", "b".repeat(64)), + 1, + ), + ("probe/cas-123.txt".to_string(), 1), + ]; + let pages = [ + Page { + objects: known, + next_continuation_token: Some("next".to_string()), + is_truncated: true, + }, + Page { + objects: vec![ + ("future-format/one".to_string(), 1), + ("future-format/two".to_string(), 1), + ("future-format/three".to_string(), 1), + ], + next_continuation_token: None, + is_truncated: false, + }, + ]; + let outcome = sweep_bucket_taxonomy(100, 2, |token| { + let page = match token.as_deref() { + None => pages[0].clone(), + Some("next") => pages[1].clone(), + other => panic!("unexpected continuation token {other:?}"), + }; + async move { Ok(page) } + }) + .await + .expect("sweep synthetic listing"); + assert_eq!(outcome.listed_objects, 9); + assert_eq!(outcome.unknown_object_count, 3); + assert_eq!( + outcome.unknown_key_sample, + vec!["future-format/one", "future-format/two"] + ); + } + + #[tokio::test] + async fn taxonomy_sweep_fails_closed_past_the_fleet_cap() { + let result = sweep_bucket_taxonomy(1, 10, |_token| async { + Ok(Page { + objects: vec![("a".to_string(), 1), ("b".to_string(), 1)], + next_continuation_token: None, + is_truncated: false, + }) + }) + .await; + assert!(matches!(result, Err(SweepError::CapExceeded { .. }))); + } +} diff --git a/crates/buzz-media/src/error.rs b/crates/buzz-media/src/error.rs index c3d180402f..14ce4afe1e 100644 --- a/crates/buzz-media/src/error.rs +++ b/crates/buzz-media/src/error.rs @@ -54,6 +54,10 @@ pub enum MediaError { InsufficientScope, #[error("relay membership required")] RelayMembershipRequired, + #[error("community writes are fenced")] + CommunityWriteFenced, + #[error("media service temporarily unavailable")] + ServiceUnavailable, #[error("token revoked")] TokenRevoked, #[error("pubkey mismatch")] @@ -138,7 +142,10 @@ impl IntoResponse for MediaError { ) } Self::InsufficientScope => (StatusCode::FORBIDDEN, self.to_string()), - Self::RelayMembershipRequired => (StatusCode::FORBIDDEN, self.to_string()), + Self::RelayMembershipRequired | Self::CommunityWriteFenced => { + (StatusCode::FORBIDDEN, self.to_string()) + } + Self::ServiceUnavailable => (StatusCode::SERVICE_UNAVAILABLE, self.to_string()), Self::UploadRateLimitExceeded | Self::UploadConcurrencyLimitReached => { (StatusCode::TOO_MANY_REQUESTS, self.to_string()) } @@ -164,6 +171,21 @@ impl IntoResponse for MediaError { mod tests { use super::*; + #[test] + fn serving_backend_failures_map_to_5xx_but_fences_remain_403() { + for error in [ + MediaError::ServiceUnavailable, + MediaError::Internal, + MediaError::StorageError("backend".to_string()), + ] { + assert!(error.into_response().status().is_server_error()); + } + assert_eq!( + MediaError::CommunityWriteFenced.into_response().status(), + StatusCode::FORBIDDEN + ); + } + #[test] fn unsupported_media_maps_to_415() { for error in [ diff --git a/crates/buzz-media/src/lib.rs b/crates/buzz-media/src/lib.rs index 67896d4ef2..b2ff12c16e 100644 --- a/crates/buzz-media/src/lib.rs +++ b/crates/buzz-media/src/lib.rs @@ -14,12 +14,13 @@ pub mod upload_record; pub mod validation; pub use bucket_index::{ - classify_key, fold_bucket_listing, BucketAggregate, BucketSnapshot, CommunityStorage, KeyClass, - Page, SweepError, + classify_key, fold_bucket_listing, is_tenant_owned_key, sweep_bucket_taxonomy, tenant_prefixes, + BucketAggregate, BucketSnapshot, CommunityStorage, KeyClass, Page, SweepError, + TaxonomySweepOutcome, }; pub use config::{MediaConfig, S3AddressingStyle}; pub use error::MediaError; -pub use storage::{BlobHeadMeta, BlobMeta, ByteStream, MediaStorage}; +pub use storage::{BlobHeadMeta, BlobMeta, BulkDeleteOutcome, ByteStream, MediaStorage}; pub use types::BlobDescriptor; pub use upload::{process_file_upload, process_upload, process_video_upload}; pub use upload_record::{ diff --git a/crates/buzz-media/src/storage.rs b/crates/buzz-media/src/storage.rs index cbf980201f..0f0aa7af62 100644 --- a/crates/buzz-media/src/storage.rs +++ b/crates/buzz-media/src/storage.rs @@ -177,6 +177,47 @@ impl MediaStorage { } } + /// Detect whether the bucket has ever had versioning enabled. + /// + /// rust-s3 exposes no GetBucketVersioning, so this writes and inspects a + /// short-lived fleet probe object instead: versioning-enabled (and + /// versioning-suspended) buckets stamp new writes with a version id. + /// Deletion refuses versioned buckets because bulk deletes without a + /// VersionId would only insert delete markers, not prove logical absence. + pub async fn bucket_versioning_detected(&self) -> Result { + let key = format!("probe/deletion-versioning-{}", uuid::Uuid::new_v4()); + self.put(&key, b"buzz deletion versioning probe", "text/plain") + .await?; + let inspected = self.bucket.head_object(&key).await; + let removed = self.bucket.delete_object(&key).await; + let (head, _) = inspected.map_err(|e| MediaError::StorageError(e.to_string()))?; + removed.map_err(|e| MediaError::StorageError(e.to_string()))?; + Ok(head.version_id.is_some()) + } + + /// Bulk-delete up to one manifest chunk of keys via S3 `DeleteObjects`. + /// + /// Never fails on per-key outcomes: they are folded into + /// [`BulkDeleteOutcome`] so the caller owns retry/fail-closed policy. + /// Historical MinIO releases report already-absent keys as + /// `NoSuchKey`/`NoSuchVersion` errors instead of deleted; both map to + /// `already_missing` to keep checkpointed retry idempotent. + pub async fn delete_objects(&self, keys: &[String]) -> Result { + if keys.is_empty() { + return Ok(BulkDeleteOutcome::default()); + } + let identifiers = keys + .iter() + .map(|key| s3::serde_types::ObjectIdentifier::new(key.clone())) + .collect::>(); + let result = self + .bucket + .delete_objects(identifiers) + .await + .map_err(|e| MediaError::StorageError(e.to_string()))?; + Ok(fold_bulk_delete_result(result)) + } + /// Build the community-scoped sidecar key for a given sha256 (bare hash). /// /// Raw media bytes remain shared content-addressed CAS (`{sha}.{ext}`), but @@ -234,6 +275,11 @@ impl MediaStorage { .map(|m| m.mime_type) } + /// Probe object-store connectivity and bucket access. + pub async fn ping(&self) -> Result<(), MediaError> { + self.list_page(None, 1).await.map(|_| ()) + } + /// One page of a full-bucket listing, for the storage sweep. Wraps /// rust-s3's manual `list_page` (NOT the auto-paginating `list`, which /// has no cap) and converts the result into the storage-agnostic @@ -246,11 +292,28 @@ impl MediaStorage { &self, continuation_token: Option, max_keys: usize, + ) -> Result { + self.list_prefix_page("", continuation_token, max_keys) + .await + } + + /// One page of a prefix-scoped listing. + /// + /// Deletion enumerates the target community's exact key prefixes with + /// this instead of listing the whole fleet bucket: cost stays + /// O(tenant objects) regardless of fleet size. `ListObjectsV2` returns + /// keys in ascending UTF-8 binary order, which callers rely on for + /// streaming key-stream digests. + pub async fn list_prefix_page( + &self, + prefix: &str, + continuation_token: Option, + max_keys: usize, ) -> Result { let (result, _status) = self .bucket .list_page( - String::new(), + prefix.to_string(), None, continuation_token, None, @@ -269,11 +332,93 @@ impl MediaStorage { } } +/// Per-key outcomes of one bulk `DeleteObjects` call. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct BulkDeleteOutcome { + /// Keys the backend reported deleted (S3 reports already-missing keys as + /// deleted too — the API is idempotent by design). + pub deleted: u64, + /// Keys reported absent via legacy MinIO `NoSuchKey`/`NoSuchVersion` + /// per-key errors; equivalent to deleted for retry purposes. + pub already_missing: u64, + /// Keys whose deletion produced a version artifact (delete marker or + /// version id) — evidence of bucket versioning, which deletion must + /// fail closed on. + pub versioned_keys: Vec, + /// Remaining per-key failures as `(key, code, message)`. + pub failed: Vec<(String, String, String)>, +} + +fn fold_bulk_delete_result(result: s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome { + let mut outcome = BulkDeleteOutcome::default(); + for deleted in result.deleted { + if deleted.delete_marker == Some(true) + || deleted.delete_marker_version_id.is_some() + || deleted.version_id.is_some() + { + outcome.versioned_keys.push(deleted.key); + } else { + outcome.deleted += 1; + } + } + for error in result.errors { + if error.code == "NoSuchKey" || error.code == "NoSuchVersion" { + outcome.already_missing += 1; + } else { + outcome.failed.push((error.key, error.code, error.message)); + } + } + outcome +} + #[cfg(test)] mod tests { use super::*; use std::collections::HashMap; + /// The bulk-delete fold is the retry-idempotence contract: legacy MinIO + /// absent-key errors count as success, version artifacts are surfaced for + /// fail-closed handling, and anything else stays a per-key failure. + #[test] + fn bulk_delete_fold_maps_absent_keys_and_version_artifacts() { + use s3::serde_types::{DeleteError, DeleteObjectsResult, DeletedObject}; + let deleted_object = |key: &str, marker: bool| DeletedObject { + key: key.to_string(), + version_id: None, + delete_marker: marker.then_some(true), + delete_marker_version_id: marker.then(|| "v1".to_string()), + }; + let delete_error = |key: &str, code: &str, message: &str| DeleteError { + key: key.to_string(), + code: code.to_string(), + message: message.to_string(), + version_id: None, + }; + let result = DeleteObjectsResult { + deleted: vec![ + deleted_object("plain", false), + deleted_object("marked", true), + ], + errors: vec![ + delete_error("gone", "NoSuchKey", "absent"), + delete_error("gone-version", "NoSuchVersion", "absent"), + delete_error("denied", "AccessDenied", "nope"), + ], + }; + let outcome = fold_bulk_delete_result(result); + assert_eq!(outcome.deleted, 1); + assert_eq!(outcome.already_missing, 2); + assert_eq!(outcome.versioned_keys, vec!["marked".to_string()]); + assert_eq!( + outcome.failed, + vec![( + "denied".to_string(), + "AccessDenied".to_string(), + "nope".to_string() + )] + ); + } + fn tenant(n: u128) -> TenantContext { TenantContext::resolved( CommunityId::from_uuid(uuid::Uuid::from_u128(n)), @@ -351,6 +496,28 @@ mod tests { ); } + #[test] + fn tenant_key_writers_are_covered_by_deletion_taxonomy() { + let ctx = tenant(1); + let community = *ctx.community().as_uuid(); + let sha = "a".repeat(64); + let event_id = "01ARZ3NDEKTSV4RRFFQ69G5FAV"; + let sidecar = MediaStorage::ctx_sidecar_key(&ctx, &sha); + let upload = crate::upload_record::upload_record_key(&ctx, &sha, event_id); + let prefixes = crate::bucket_index::tenant_prefixes(community); + + for key in [sidecar, upload] { + assert!( + prefixes.iter().any(|prefix| key.starts_with(prefix)), + "tenant writer key {key} is outside deletion prefixes" + ); + assert!( + crate::bucket_index::is_tenant_owned_key(community, &key), + "tenant writer key {key} is not recognized by deletion taxonomy" + ); + } + } + #[test] fn sidecar_keys_are_community_scoped() { let a = tenant(1); diff --git a/crates/buzz-relay/Cargo.toml b/crates/buzz-relay/Cargo.toml index 41bdc3b9e9..30eaa6203a 100644 --- a/crates/buzz-relay/Cargo.toml +++ b/crates/buzz-relay/Cargo.toml @@ -19,6 +19,7 @@ path = "src/main.rs" buzz-core = { workspace = true } buzz-conformance = { workspace = true } buzz-db = { workspace = true } +buzz-deletion = { workspace = true } buzz-auth = { workspace = true } buzz-pubsub = { workspace = true } buzz-audit = { workspace = true } diff --git a/crates/buzz-relay/src/api/git/manifest.rs b/crates/buzz-relay/src/api/git/manifest.rs index baf109c1ad..0dfbdb35a4 100644 --- a/crates/buzz-relay/src/api/git/manifest.rs +++ b/crates/buzz-relay/src/api/git/manifest.rs @@ -474,6 +474,17 @@ mod tests { m.validate().expect("no parent is fine (first push)"); } + #[test] + fn pointer_writer_is_covered_by_deletion_taxonomy() { + let community = CommunityId::from_uuid(uuid::Uuid::from_u128(1)); + let owner = "a".repeat(64); + let key = pointer_key(community, &owner, "repo"); + let prefixes = buzz_media::tenant_prefixes(*community.as_uuid()); + + assert!(prefixes.iter().any(|prefix| key.starts_with(prefix))); + assert!(buzz_media::is_tenant_owned_key(*community.as_uuid(), &key)); + } + #[test] fn pointer_key_strips_dot_git() { let c = CommunityId::from_uuid(uuid::Uuid::from_u128(1)); diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index d3118d8a76..1daf151ad4 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -1698,6 +1698,21 @@ pub(crate) struct PushContext { pub repo_handle: HydratedRepo, } +#[derive(Default)] +struct FinalizePushHooks { + #[cfg(test)] + post_cas_gate: Option>, + #[cfg(test)] + fail_ref_state_insert: bool, +} + +#[cfg(test)] +#[derive(Default)] +struct PostCasGate { + reached: tokio::sync::Notify, + resume: tokio::sync::Notify, +} + /// Finalize a push request: CAS-commit the new state into the object /// store, derive kind:30618 from the committed manifest, and only then /// build the success response. @@ -1708,6 +1723,17 @@ pub(crate) struct PushContext { /// constructor of a push 2xx, so the seam is structural (not by /// convention). async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { + finalize_push_inner(state, ctx, &FinalizePushHooks::default()).await +} + +async fn finalize_push_inner( + state: &Arc, + ctx: PushContext, + hooks: &FinalizePushHooks, +) -> Response { + #[cfg(not(test))] + let _ = hooks; + // The push fence, part 0 — **a rejected push publishes nothing.** // // `ctx.pack.ok` is false when git aborted the ref updates: either the @@ -1738,10 +1764,41 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { return response; } + // An already-running receive-pack may cross the durable fence after + // request admission. Revalidate immediately before object-store CAS; DB + // trigger fencing alone cannot roll back an S3 pointer mutation. + let serving_write = match buzz_deletion::acquire_serving_write( + &state.db, + ctx.tenant.community(), + "git_publish", + ) + .await + { + Ok(guard) => guard, + Err(error) => { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "push rejected by community deletion fence"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "community writes are fenced", + ) + .into_response(); + } + }; + + if let Err(error) = serving_write.verify().await { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "push lost community serving lease"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "community write lease lost", + ) + .into_response(); + } + // Step 7 (CAS). The PushContext binds `parent_state` (observed at // hydrate) to the CAS predicate here — no re-reading of the pointer - // between hydrate and CAS. - let success = match cas_publish( + // between hydrate and CAS. Observe serving-lease loss throughout the + // potentially long upload/CAS operation, not only at its boundaries. + let publish = cas_publish( &state.git_store, &ctx.tenant, ctx.repo_handle.path(), @@ -1753,72 +1810,87 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { max_pack_bytes: state.config.git_max_pack_bytes, max_repo_bytes: state.config.git_max_repo_bytes, }, - ) - .await - { - Ok(s) => s, - Err(CasError::Conflict { - winner_manifest_key, - .. - }) => { - warn!( - owner = %ctx.owner, - repo = %ctx.repo, - winner = %winner_manifest_key, - "push lost CAS race; tempdir dropped, returning 409" - ); - return ( - StatusCode::CONFLICT, - "push superseded by a concurrent writer; pull and retry", - ) - .into_response(); - } - Err(CasError::ManifestInvalid(e)) => { - // 4xx-class: the workspace produced refs/HEAD/oids the - // manifest validator rejects (unsafe refname, malformed oid, - // empty head, malformed parent). Pre-CAS — no pointer was - // written. - warn!( - owner = %ctx.owner, - repo = %ctx.repo, - error = %e, - "push rejected: manifest validation failed" - ); - return ( - StatusCode::BAD_REQUEST, - "push produced invalid manifest state", - ) - .into_response(); - } - Err(CasError::ResourceLimit(e)) => { - warn!( - owner = %ctx.owner, - repo = %ctx.repo, - error = %e, - "push rejected: repo exceeds relay resource limits" - ); + ); + let success = match serving_write.protect(publish).await { + Ok(result) => match result { + Ok(s) => s, + Err(CasError::Conflict { + winner_manifest_key, + .. + }) => { + warn!( + owner = %ctx.owner, + repo = %ctx.repo, + winner = %winner_manifest_key, + "push lost CAS race; tempdir dropped, returning 409" + ); + return ( + StatusCode::CONFLICT, + "push superseded by a concurrent writer; pull and retry", + ) + .into_response(); + } + Err(CasError::ManifestInvalid(e)) => { + // 4xx-class: the workspace produced refs/HEAD/oids the + // manifest validator rejects (unsafe refname, malformed oid, + // empty head, malformed parent). Pre-CAS — no pointer was + // written. + warn!( + owner = %ctx.owner, + repo = %ctx.repo, + error = %e, + "push rejected: manifest validation failed" + ); + return ( + StatusCode::BAD_REQUEST, + "push produced invalid manifest state", + ) + .into_response(); + } + Err(CasError::ResourceLimit(e)) => { + warn!( + owner = %ctx.owner, + repo = %ctx.repo, + error = %e, + "push rejected: repo exceeds relay resource limits" + ); + return ( + StatusCode::PAYLOAD_TOO_LARGE, + "repository exceeds relay resource limits", + ) + .into_response(); + } + Err(e) => { + // 5xx-class: ManifestReadFailed (parent corruption), + // Backend, PackCapture. The tempdir drops on scope exit; no + // pointer was written (or, on rare ManifestReadFailed during + // winner-fetch, the winner is already installed and the + // loser's data is unrelated). + error!( + owner = %ctx.owner, + repo = %ctx.repo, + error = %e, + "push failed pre-response" + ); + return (StatusCode::INTERNAL_SERVER_ERROR, "git backend error").into_response(); + } + }, + Err(error) => { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "push lost community serving lease during CAS publish"); return ( - StatusCode::PAYLOAD_TOO_LARGE, - "repository exceeds relay resource limits", + StatusCode::SERVICE_UNAVAILABLE, + "community write lease lost", ) .into_response(); } - Err(e) => { - // 5xx-class: ManifestReadFailed (parent corruption), - // Backend, PackCapture. The tempdir drops on scope exit; no - // pointer was written (or, on rare ManifestReadFailed during - // winner-fetch, the winner is already installed and the - // loser's data is unrelated). - error!( - owner = %ctx.owner, - repo = %ctx.repo, - error = %e, - "push failed pre-response" - ); - return (StatusCode::INTERNAL_SERVER_ERROR, "git backend error").into_response(); - } }; + #[cfg(test)] + if let Some(gate) = &hooks.post_cas_gate { + gate.reached.notify_one(); + gate.resume.notified().await; + } + // Derived after CAS: kind:30618 ref-state event over the *committed* // manifest's refs/head. Spec §Implementation Correspondence: // "kind:30618 is derived after CAS, never the commit." We emit only @@ -1842,7 +1914,7 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { (Some(before), Some(after)) => before != after, _ => true, // first push (parent None) or impossible-shape after key → publish }; - if manifest_changed { + let publication_result: Result<(), String> = if manifest_changed { let inputs = RefStateInputs { repo_id: &ctx.repo_id, head: &success.manifest.head, @@ -1853,11 +1925,23 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { Ok(event) => { // Relay-signed kind:30618 belongs to the same server-resolved // tenant as the git request that committed the pointer. - match state + #[cfg(test)] + let insert_result = if hooks.fail_ref_state_insert { + Err(buzz_db::DbError::InvalidData( + "injected kind:30618 insert failure".to_string(), + )) + } else { + state + .db + .insert_event_with_serving_write_guard(serving_write.lease(), &event, None) + .await + }; + #[cfg(not(test))] + let insert_result = state .db - .insert_event(ctx.tenant.community(), &event, None) - .await - { + .insert_event_with_serving_write_guard(serving_write.lease(), &event, None) + .await; + match insert_result { Ok((stored, true)) => { // Routed through the guarded send path for uniformity; // the access gate no-ops for this globally-scoped @@ -1874,6 +1958,7 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { manifest = %success.manifest_key, "kind:30618 published (derived after CAS)" ); + Ok(()) } Ok((_, false)) => { info!( @@ -1881,26 +1966,41 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { repo = %ctx.repo_id, "kind:30618 deduplicated by relay db" ); + Ok(()) } - Err(e) => { - warn!( - owner = %ctx.owner, - repo = %ctx.repo_id, - error = %e, - "kind:30618 insert failed; push remains durable in object store" - ); - } + Err(error) => Err(format!("kind:30618 insert failed: {error}")), } } - Err(e) => { - warn!( - owner = %ctx.owner, - repo = %ctx.repo_id, - error = %e, - "kind:30618 build failed; push remains durable in object store" - ); - } + Err(error) => Err(format!("kind:30618 build failed: {error}")), } + } else { + Ok(()) + }; + + // The admitted serving write spans the complete publication attempt. Fence + // acquisition cannot overtake the pointer CAS, durable 30618 insert, or + // local fan-out attempt; only now may the lease be released. + if let Err(error) = serving_write.finish().await { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "failed to release community serving lease after push publication"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "community write lease lost during publication", + ) + .into_response(); + } + if let Err(error) = publication_result { + error!( + owner = %ctx.owner, + repo = %ctx.repo_id, + manifest = %success.manifest_key, + %error, + "push pointer committed but kind:30618 publication failed" + ); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "push committed but ref-state publication failed; retry", + ) + .into_response(); } // Only now — after CAS commit and (optional) 30618 emission — build @@ -1929,12 +2029,14 @@ pub fn git_router(state: Arc) -> Router { #[cfg(test)] mod track_c_tests { use super::*; + use crate::api::git::hydrate::{hydrate_for_write, HydrationOptions}; use crate::api::git::manifest::Manifest; use buzz_core::CommunityId; use nostr::{EventBuilder, Keys, Kind, Tag}; use std::collections::BTreeMap; use std::io::Write; use std::process::Output; + use tempfile::TempDir; fn oid_sha1() -> String { "cb09a769da1c01f458fa6959d4e8eded38fac8d3".to_string() @@ -2075,6 +2177,303 @@ mod track_c_tests { assert!(remote.join("refs/heads/master").exists()); } + async fn run_finalize_git(repo: &Path, args: &[&str]) -> std::process::Output { + let mut command = Command::new("git"); + command.current_dir(repo).args(args); + harden_git_env(&mut command); + let output = command.output().await.expect("spawn git"); + assert!( + output.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + output + } + + async fn finalize_test_state() -> (Arc, sqlx::PgPool) { + const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; // sadscan:disable np.postgres.1 + let mut config = crate::config::Config::from_env().expect("default config loads"); + config.require_relay_membership = false; + config.redis_url = "redis://127.0.0.1:1".to_string(); + config.database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| TEST_DB_URL.to_string()); + let pool = sqlx::PgPool::connect(&config.database_url) + .await + .expect("connect test DB"); + let db = buzz_db::Db::from_pool(pool.clone()); + db.migrate().await.expect("migrate test DB"); + let redis_pool = deadpool_redis::Config::from_url(&config.redis_url) + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .expect("redis pool"); + let pubsub = Arc::new( + buzz_pubsub::PubSubManager::new(&config.redis_url, redis_pool.clone()) + .await + .expect("pubsub manager"), + ); + let audit = buzz_audit::AuditService::new(pool.clone()); + let auth = buzz_auth::AuthService::new(config.auth.clone()); + let search = buzz_search::SearchService::new(pool.clone()); + let workflow_engine = Arc::new(buzz_workflow::WorkflowEngine::new( + db.clone(), + buzz_workflow::WorkflowConfig::default(), + )); + let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let (state, _audit_shutdown) = AppState::new( + config, + db, + redis_pool, + audit, + pubsub, + auth, + search, + workflow_engine, + Keys::generate(), + media_storage, + ); + (Arc::new(state), pool) + } + + async fn approved_deletion( + state: &AppState, + host: &str, + ) -> ( + buzz_db::deletion::DeletionRequest, + buzz_db::deletion::ClaimedDeletion, + ) { + use buzz_db::deletion::{ + FrozenInventory, KeyStreamDigest, PrefixManifest, StorageManifest, + DEFAULT_LEASE_DURATION, + }; + + let store = state.db.deletion_store(); + let request = store + .submit(host, "git-finalize-test", Some("post-CAS lease regression")) + .await + .expect("submit deletion"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(request.community_id) + .await + .expect("schema inventory"), + storage: StorageManifest { + version: 4, + prefixes: buzz_media::tenant_prefixes(*request.community_id.as_uuid()) + .into_iter() + .map(|prefix| PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + }) + .collect(), + }, + }; + store + .freeze_inventory(request.id, &inventory) + .await + .expect("freeze inventory"); + store + .approve(request.id, "git-finalize-test", None) + .await + .expect("approve deletion"); + let claim = store + .claim_specific(request.id, "git-finalize-test", DEFAULT_LEASE_DURATION) + .await + .expect("claim deletion") + .expect("won deletion claim"); + (request, claim) + } + + async fn pushed_context( + state: &AppState, + community: CommunityId, + host: &str, + owner: String, + repo: String, + pusher: nostr::PublicKey, + scratch: &Path, + ) -> PushContext { + let tenant = TenantContext::resolved(community, host); + let (hydrated, parent_state) = hydrate_for_write( + &state.git_store, + &tenant, + &owner, + &repo, + HydrationOptions { + pack_cache: &state.git_pack_cache, + scratch_dir: scratch, + max_pack_bytes: 1024 * 1024, + max_repo_bytes: 2 * 1024 * 1024, + }, + ) + .await + .expect("hydrate empty test repo"); + let source = scratch.join("source"); + tokio::fs::create_dir(&source) + .await + .expect("source directory"); + run_finalize_git(&source, &["init", "--quiet", "--initial-branch=main"]).await; + run_finalize_git(&source, &["config", "user.email", "finalize@test"]).await; + run_finalize_git(&source, &["config", "user.name", "finalize"]).await; + tokio::fs::write(source.join("file.txt"), b"committed\n") + .await + .expect("write source file"); + run_finalize_git(&source, &["add", "file.txt"]).await; + run_finalize_git(&source, &["commit", "--quiet", "-m", "committed"]).await; + let remote = hydrated.path().to_str().expect("hydrated path utf8"); + run_finalize_git(&source, &["push", "--quiet", remote, "main"]).await; + + PushContext { + pack: PackOutput { + stdout: b"push-ok".to_vec(), + ok: true, + }, + parent_state, + owner, + repo: repo.clone(), + repo_id: repo, + pusher, + tenant, + repo_handle: hydrated, + } + } + + #[tokio::test] + #[ignore = "requires Postgres and MinIO"] + async fn finalize_push_holds_serving_lease_through_post_cas_publication() { + let (state, pool) = finalize_test_state().await; + let host = format!("git-finalize-{}.example", uuid::Uuid::new_v4().simple()); + let community = state + .db + .ensure_configured_community(&host) + .await + .expect("create test community") + .id; + let (request, claim) = approved_deletion(&state, &host).await; + let scratch = TempDir::new().expect("scratch"); + let owner = format!("owner-{}", uuid::Uuid::new_v4().simple()); + let repo = format!("repo-{}", uuid::Uuid::new_v4().simple()); + let ctx = pushed_context( + &state, + community, + &host, + owner, + repo.clone(), + Keys::generate().public_key(), + scratch.path(), + ) + .await; + let gate = Arc::new(PostCasGate::default()); + let hooks = FinalizePushHooks { + post_cas_gate: Some(Arc::clone(&gate)), + fail_ref_state_insert: false, + }; + let finalize_state = Arc::clone(&state); + let finalize = + tokio::spawn(async move { finalize_push_inner(&finalize_state, ctx, &hooks).await }); + + gate.reached.notified().await; + state + .db + .deletion_store() + .begin_quiescing(&claim.lease) + .await + .expect("quiesce after CAS"); + let error = state + .db + .deletion_store() + .fence(&claim.lease) + .await + .expect_err("post-CAS serving lease must block fence"); + assert!(matches!( + error, + buzz_db::DbError::ServingWritesNotDrained { .. } + )); + assert!(!state + .db + .deletion_store() + .is_serving_active(community) + .await + .expect("quiescing rejects new serving work")); + + gate.resume.notify_one(); + let response = finalize.await.expect("finalize task"); + assert_eq!(response.status(), StatusCode::OK); + let mut query = buzz_db::event::EventQuery::for_community(community); + query.kinds = Some(vec![30_618]); + query.d_tag = Some(repo); + let events = state.db.query_events(&query).await.expect("query 30618"); + assert_eq!(events.len(), 1, "kind:30618 must be durable before release"); + assert!(state + .db + .deletion_store() + .serving_writes_drained(community) + .await + .expect("serving lease released")); + let generation = state + .db + .deletion_store() + .fence(&claim.lease) + .await + .expect("fence after publication"); + assert_eq!(generation, 1); + assert_eq!( + state + .db + .deletion_store() + .get(request.id) + .await + .expect("fenced request") + .stage, + buzz_db::deletion::DeletionStage::Fenced + ); + drop(state); + pool.close().await; + } + + #[tokio::test] + #[ignore = "requires Postgres and MinIO"] + async fn finalize_push_db_failure_after_cas_is_not_success_and_releases_lease() { + let (state, pool) = finalize_test_state().await; + let host = format!( + "git-finalize-fail-{}.example", + uuid::Uuid::new_v4().simple() + ); + let community = state + .db + .ensure_configured_community(&host) + .await + .expect("create test community") + .id; + let scratch = TempDir::new().expect("scratch"); + let ctx = pushed_context( + &state, + community, + &host, + format!("owner-{}", uuid::Uuid::new_v4().simple()), + format!("repo-{}", uuid::Uuid::new_v4().simple()), + Keys::generate().public_key(), + scratch.path(), + ) + .await; + let hooks = FinalizePushHooks { + post_cas_gate: None, + fail_ref_state_insert: true, + }; + + let response = finalize_push_inner(&state, ctx, &hooks).await; + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!(state + .db + .deletion_store() + .serving_writes_drained(community) + .await + .expect("serving lease released on failure")); + drop(state); + pool.close().await; + } + /// A gzip-encoded request body is transparently inflated before it /// reaches the git subprocess. Git's smart-HTTP client gzips the /// upload-pack/receive-pack request body past a size threshold (fires diff --git a/crates/buzz-relay/src/api/invites.rs b/crates/buzz-relay/src/api/invites.rs index 6104171cca..60b7caf2af 100644 --- a/crates/buzz-relay/src/api/invites.rs +++ b/crates/buzz-relay/src/api/invites.rs @@ -305,7 +305,9 @@ pub async fn mint_invite( .mint_relay_invite(tenant.community(), &sender_hex, ttl, max_uses) .await .map_err(|error| match error { - buzz_db::DbError::InvalidData(message) => api_error(StatusCode::BAD_REQUEST, &message), + buzz_db::DbError::InvalidData(message) | buzz_db::DbError::DeletionSafety(message) => { + api_error(StatusCode::BAD_REQUEST, &message) + } error => internal_error(&format!("invite mint: {error}")), })?; diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index fa0401bc26..cfc7750ed4 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -283,6 +283,19 @@ async fn upload_attribution( }) } +fn serving_write_error(error: anyhow::Error) -> MediaError { + if buzz_deletion::ServingWriteGuard::acquisition_is_fenced(&error) { + MediaError::CommunityWriteFenced + } else { + MediaError::ServiceUnavailable + } +} + +fn serving_lease_lost(error: anyhow::Error) -> MediaError { + tracing::warn!(%error, "media serving-write lease lost"); + MediaError::ServiceUnavailable +} + /// PUT `/upload` or the temporary media-only `/media/upload` alias. /// /// Auth is validated via the [`AuthenticatedUpload`] extractor BEFORE the body @@ -310,6 +323,11 @@ pub async fn upload_blob( ) -> Result, MediaError> { let attribution = upload_attribution(&state, &auth, &headers).await; + let serving_write = + buzz_deletion::acquire_serving_write(&state.db, auth.tenant.community(), "media_upload") + .await + .map_err(serving_write_error)?; + if auth.route_mode == UploadRouteMode::LegacyMedia { metrics::counter!("buzz_media_legacy_upload_route_total").increment(1); } @@ -335,69 +353,86 @@ pub async fn upload_blob( } let replay = futures_util::stream::iter(replay_chunks.into_iter().map(Ok)).chain(source); - let mut descriptor = if should_stream_as_video(&sniff) { - // Video path: stream body directly to disk — never fully buffered in RAM. - let content_length = headers - .get("content-length") - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()); - buzz_media::process_video_upload( - &state.media_storage, - &state.config.media, - &auth.tenant, - &auth.auth_event, - replay, - content_length, - attribution, - ) - .await? - } else { - // Non-video path: buffer the body (bounded by the larger of the image - // and generic-file caps), then decide image-vs-generic by sniffed MIME. - // Images go through the thumbnailing pipeline; non-media attachments - // (docs, archives, text, data) take the generic file path and are - // served as downloads. Recognized audio/video cannot fall through it. - let max = state - .config - .media - .max_image_bytes - .max(state.config.media.max_file_bytes); - let bytes = axum::body::to_bytes(axum::body::Body::from_stream(replay), max as usize) - .await - .map_err(|_| MediaError::FileTooLarge { size: 0, max })?; - - let is_image = matches!( - infer::get(&bytes).map(|t| t.mime_type()), - Some("image/jpeg" | "image/png" | "image/gif" | "image/webp") - ); - - if is_image { - buzz_media::process_upload( - &state.media_storage, - &state.config.media, - &auth.tenant, - &auth.auth_event, - bytes, - attribution, - ) - .await? - } else if auth.route_mode == UploadRouteMode::LegacyMedia { - let mime = infer::get(&bytes) - .map(|kind| kind.mime_type().to_string()) - .unwrap_or_else(|| "application/octet-stream".to_string()); - return Err(MediaError::DisallowedContentType(mime)); - } else { - buzz_media::process_file_upload( - &state.media_storage, - &state.config.media, - &auth.tenant, - &auth.auth_event, - bytes, - attribution, - ) - .await? - } - }; + serving_write.verify().await.map_err(serving_lease_lost)?; + + let mut descriptor = serving_write + .protect(async { + Ok(if should_stream_as_video(&sniff) { + // Video path: stream body directly to disk — never fully buffered in RAM. + let content_length = headers + .get("content-length") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + buzz_media::process_video_upload( + &state.media_storage, + &state.config.media, + &auth.tenant, + &auth.auth_event, + replay, + content_length, + attribution, + ) + .await? + } else { + // Non-video path: buffer the body (bounded by the larger of the image + // and generic-file caps), then decide image-vs-generic by sniffed MIME. + // Images go through the thumbnailing pipeline; non-media attachments + // (docs, archives, text, data) take the generic file path and are + // served as downloads. Recognized audio/video cannot fall through it. + let max = state + .config + .media + .max_image_bytes + .max(state.config.media.max_file_bytes); + let bytes = + axum::body::to_bytes(axum::body::Body::from_stream(replay), max as usize) + .await + .map_err(|_| MediaError::FileTooLarge { size: 0, max })?; + + let is_image = matches!( + infer::get(&bytes).map(|t| t.mime_type()), + Some("image/jpeg" | "image/png" | "image/gif" | "image/webp") + ); + + if is_image { + buzz_media::process_upload( + &state.media_storage, + &state.config.media, + &auth.tenant, + &auth.auth_event, + bytes, + attribution, + ) + .await? + } else if auth.route_mode == UploadRouteMode::LegacyMedia { + let mime = infer::get(&bytes) + .map(|kind| kind.mime_type().to_string()) + .unwrap_or_else(|| "application/octet-stream".to_string()); + return Err(MediaError::DisallowedContentType(mime)); + } else { + buzz_media::process_file_upload( + &state.media_storage, + &state.config.media, + &auth.tenant, + &auth.auth_event, + bytes, + attribution, + ) + .await? + } + }) + }) + .await + .map_err(|error| { + if buzz_deletion::ServingWriteGuard::is_lease_lost(&error) { + serving_lease_lost(error) + } else { + match error.downcast::() { + Ok(error) => error, + Err(_) => MediaError::Internal, + } + } + })??; rewrite_descriptor_urls_for_tenant( &mut descriptor, @@ -441,6 +476,7 @@ pub async fn upload_blob( } } + serving_write.finish().await.map_err(serving_lease_lost)?; Ok(Json(descriptor)) } @@ -922,6 +958,20 @@ mod tests { const VALID_HASH: &str = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + #[test] + fn serving_write_error_taxonomy_separates_fence_from_backend_failure() { + let fenced = anyhow::Error::from(buzz_db::DbError::AccessDenied("fenced".to_string())); + assert!(matches!( + serving_write_error(fenced), + MediaError::CommunityWriteFenced + )); + let backend = anyhow::Error::from(buzz_db::DbError::Sqlx(sqlx::Error::PoolTimedOut)); + assert!(matches!( + serving_write_error(backend), + MediaError::ServiceUnavailable + )); + } + #[test] fn upload_routes_distinguish_standard_and_legacy_modes() { assert_eq!( diff --git a/crates/buzz-relay/src/handlers/command_executor.rs b/crates/buzz-relay/src/handlers/command_executor.rs index 2d82736807..85aae9d7b8 100644 --- a/crates/buzz-relay/src/handlers/command_executor.rs +++ b/crates/buzz-relay/src/handlers/command_executor.rs @@ -110,6 +110,12 @@ async fn persist_command_event( .begin_transaction() .await .map_err(|e| IngestError::Internal(format!("error: begin transaction: {e}")))?; + buzz_deletion::store(&state.db) + .guard_transaction(&mut tx, tenant.community()) + .await + .map_err(|error| { + IngestError::Rejected(format!("restricted: community writes are fenced: {error}")) + })?; // INSERT with ON CONFLICT DO NOTHING — idempotency guard. let id_bytes = event.id.as_bytes(); diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index a9cdffcdec..a9a8570307 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -705,16 +705,53 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc guard, + Err(error) => { + reject("restricted"); + conn.send(RelayMessage::ok( + &event_id_hex, + false, + &format!("restricted: community writes are fenced: {error}"), + )); + return; + } + }; + let handled = serving_write + .protect(handle_ephemeral_event( + event, + conn_id, + pubkey_bytes, + auth_pubkey, + Arc::clone(&conn), + state, + )) + .await; + let terminal = match handled { + Ok(Ok(())) => EphemeralTerminal::Accepted, + Ok(Err(message)) => { + reject("invalid"); + EphemeralTerminal::Rejected(message) + } + Err(error) => { + reject("restricted"); + EphemeralTerminal::LeaseLost(format!( + "restricted: community write lease lost: {error}" + )) + } + }; + if matches!(terminal, EphemeralTerminal::Accepted) { + if let Err(error) = serving_write.finish().await { + tracing::warn!(%error, event_id = %event_id_hex, "failed to release ephemeral-event serving lease"); + } + } + conn.send(terminal.response(&event_id_hex)); return; } @@ -758,37 +795,40 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc String { + match self { + Self::Accepted => RelayMessage::ok(event_id, true, ""), + Self::Rejected(message) | Self::LeaseLost(message) => { + RelayMessage::ok(event_id, false, message) + } + } + } +} + /// Handle ephemeral events (kind 20000–29999) — WS-only, never stored. async fn handle_ephemeral_event( event: Event, conn_id: uuid::Uuid, - event_id_hex: &str, pubkey_bytes: Vec, auth_pubkey: nostr::PublicKey, conn: Arc, state: Arc, -) { +) -> Result<(), String> { let event_clone = event.clone(); + let event_id = event.id.to_hex(); let verify_result = tokio::task::spawn_blocking(move || verify_event(&event_clone)).await; match verify_result { Ok(Ok(())) => {} - Ok(Err(e)) => { - conn.send(RelayMessage::ok( - event_id_hex, - false, - &format!("invalid: {e}"), - )); - return; - } - Err(_) => { - conn.send(RelayMessage::ok( - event_id_hex, - false, - "error: internal error", - )); - return; - } + Ok(Err(e)) => return Err(format!("invalid: {e}")), + Err(_) => return Err("error: internal error".to_string()), } // Special handling for presence events (kind:20001). @@ -829,18 +869,8 @@ async fn handle_ephemeral_event( // Check channel membership before publishing other ephemeral events. if let Some(ch_id) = super::ingest::extract_channel_id(&event) { - if let Err(msg) = super::ingest::check_channel_membership( - &conn.tenant, - &state, - ch_id, - &pubkey_bytes, - None, - ) - .await - { - conn.send(RelayMessage::ok(event_id_hex, false, &msg)); - return; - } + super::ingest::check_channel_membership(&conn.tenant, &state, ch_id, &pubkey_bytes, None) + .await?; // Mark as local before Redis publish to prevent double-delivery when // the event comes back through the Redis subscriber loop. @@ -854,7 +884,7 @@ async fn handle_ephemeral_event( state .local_event_ids .invalidate(&(conn.tenant.community(), event.id.to_bytes())); - warn!(conn_id = %conn_id, event_id = %event_id_hex, "Ephemeral publish failed: {e}"); + warn!(conn_id = %conn_id, event_id = %event_id, "Ephemeral publish failed: {e}"); } // Direct fan-out to local WS subscribers, through the guarded send path @@ -882,7 +912,7 @@ async fn handle_ephemeral_event( state .local_event_ids .invalidate(&(conn.tenant.community(), event.id.to_bytes())); - warn!(conn_id = %conn_id, event_id = %event_id_hex, "Ephemeral global publish failed: {e}"); + warn!(conn_id = %conn_id, event_id = %event_id, "Ephemeral global publish failed: {e}"); } // Direct fan-out to local WS subscribers through the guarded send path. @@ -893,7 +923,7 @@ async fn handle_ephemeral_event( fan_out_event_to_local_subscribers(&state, conn.tenant.community(), &stored_event).await; } - conn.send(RelayMessage::ok(event_id_hex, true, "")); + Ok(()) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1173,6 +1203,25 @@ mod tests { use tokio_util::sync::CancellationToken; use uuid::Uuid; + #[test] + fn ephemeral_terminal_builds_one_unambiguous_ok() { + let event_id = "abc"; + assert_eq!( + serde_json::from_str::( + &super::EphemeralTerminal::Accepted.response(event_id) + ) + .unwrap(), + serde_json::json!(["OK", event_id, true, ""]) + ); + assert_eq!( + serde_json::from_str::( + &super::EphemeralTerminal::LeaseLost("restricted".to_string()).response(event_id) + ) + .unwrap(), + serde_json::json!(["OK", event_id, false, "restricted"]) + ); + } + #[test] fn fanout_event_frame_matches_legacy_format_byte_for_byte() { let sub_id = "sub-id"; diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index fcd0d70728..5da29cdff0 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -183,6 +183,24 @@ pub enum IngestError { Internal(String), } +/// Map the durable community write-fence lookup onto the ingest error taxonomy. +/// +/// An inactive community is an authorization decision and keeps the exact +/// `restricted:` wire text the ephemeral path uses. A lookup outage is a +/// server fault and fails closed as `error:`/500 — a Postgres blip can +/// neither admit a write past the fence nor read as a client mistake. +fn map_serving_fence_state(active: Result) -> Result<(), IngestError> { + match active { + Ok(true) => Ok(()), + Ok(false) => Err(IngestError::Rejected( + "restricted: community writes are fenced".into(), + )), + Err(error) => Err(IngestError::Internal(format!( + "error: checking community write fence: {error}" + ))), + } +} + fn map_relay_admin_error(error: super::relay_admin::RelayAdminError) -> IngestError { use super::relay_admin::RelayAdminError; match error { @@ -1814,6 +1832,17 @@ async fn ingest_event_inner( let kind_u32 = event_kind_u32(&event); debug!(event_id = %event_id_hex, kind = kind_u32, "ingest_event"); + // Durable community write fence: persistent ingest is a DB write the + // deletion engine cannot exclude via serving-write leases (those cover + // external side effects only), so the shared WS/HTTP seam must refuse + // writes once the community leaves the active lifecycle state. Row churn + // inside the remaining race window is swept by the destructive DB stage. + map_serving_fence_state( + buzz_deletion::store(&state.db) + .is_serving_active(tenant.community()) + .await, + )?; + if kind_u32 == KIND_AUTH { return Err(IngestError::Rejected( "invalid: AUTH events cannot be submitted".into(), @@ -2972,6 +3001,123 @@ mod tests { } } + /// An active community passes the durable write fence untouched. + #[test] + fn serving_fence_active_community_admits_write() { + assert!(map_serving_fence_state(Ok(true)).is_ok()); + } + + /// A fenced/tombstoned/archived community is an authorization decision: + /// `restricted:` and (via `bridge.rs`) HTTP 400 — with the exact wire text + /// the ephemeral WS path uses, so clients see one refusal vocabulary. + #[test] + fn serving_fence_inactive_community_maps_to_restricted() { + match map_serving_fence_state(Ok(false)) { + Err(IngestError::Rejected(msg)) => { + assert_eq!(msg, "restricted: community writes are fenced"); + } + other => panic!("fenced community must map to Rejected, got {other:?}"), + } + } + + /// A fence-lookup outage is a server fault and must fail closed as + /// `error:`/500 — a Postgres blip can neither admit a write past the + /// fence nor be reported to an innocent client as a bad request. + #[test] + fn serving_fence_lookup_outage_fails_closed_as_internal() { + let outage = buzz_db::DbError::Sqlx(sqlx::Error::PoolTimedOut); + match map_serving_fence_state(Err(outage)) { + Err(IngestError::Internal(msg)) => { + assert!( + msg.starts_with("error: "), + "fence outages need the `error:` NIP-01 prefix, got {msg:?}" + ); + } + other => panic!("fence lookup failure must map to Internal, got {other:?}"), + } + } + + /// Production-path regression: the exact predicate `ingest_event_inner` + /// consults must admit writes while a community is active and refuse them + /// once the community deletion lifecycle fences it. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn ingest_write_fence_follows_community_deletion_lifecycle() { + use buzz_db::deletion::{ + FrozenInventory, KeyStreamDigest, PrefixManifest, StorageManifest, + DEFAULT_LEASE_DURATION, + }; + + let url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); // sadscan:disable np.postgres.1 + let pool = sqlx::PgPool::connect(&url).await.expect("connect test DB"); + let db = buzz_db::Db::from_pool(pool); + db.migrate().await.expect("migrate test DB"); + let store = buzz_deletion::store(&db); + + let host = format!("lane3-fence-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("community") + .id; + + assert!( + map_serving_fence_state(store.is_serving_active(community).await).is_ok(), + "active community must admit persistent ingest" + ); + + let submitted = store + .submit( + &host, + "test-operator", + Some("lane3 ingest fence regression"), + ) + .await + .expect("submit"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community) + .await + .expect("schema inventory"), + storage: StorageManifest { + version: 4, + prefixes: buzz_media::tenant_prefixes(*community.as_uuid()) + .into_iter() + .map(|prefix| PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + }) + .collect(), + }, + }; + let request = store + .freeze_inventory(submitted.id, &inventory) + .await + .expect("freeze inventory"); + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + store.fence(&claim.lease).await.expect("fence"); + + match map_serving_fence_state(store.is_serving_active(community).await) { + Err(IngestError::Rejected(msg)) => { + assert_eq!(msg, "restricted: community writes are fenced"); + } + other => panic!("fenced community must refuse persistent ingest, got {other:?}"), + } + } + #[derive(Debug, Default)] struct VecTracer { steps: Mutex>, diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index 799cf9cf60..b610bb64f0 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -201,6 +201,12 @@ async fn main() -> anyhow::Result<()> { error!("Failed to ensure partitions: {e}"); } + db.validate_deletion_serving_catalog().await.map_err(|e| { + error!("Community deletion serving-fence validation failed: {e}"); + anyhow::anyhow!("Community deletion serving fence is unsafe: {e}") + })?; + info!("Community deletion serving fences verified"); + // Freshness fence probe: cursor pages route to the replica only for // history the probe has verified as fully replayed. Deliberately AFTER // the migration decision: spawn_fence_probe first verifies the @@ -469,6 +475,7 @@ async fn main() -> anyhow::Result<()> { if let Some(handle) = buzz_relay::mesh_boot::boot_mesh( &state.config, state.redis_pool.clone(), + state.db.clone(), &state.relay_keypair, Arc::clone(&state.shutting_down), ) @@ -1018,6 +1025,24 @@ async fn main() -> anyhow::Result<()> { metrics::gauge!("buzz_redis_pool_size").set(rs.size as f64); metrics::gauge!("buzz_redis_pool_max").set(rs.max_size as f64); metrics::gauge!("buzz_redis_pool_waiting").set(rs.waiting as f64); + + let deletion_store = pool_state.db.deletion_store(); + match deletion_store.reap_expired_serving_write_leases(1000).await { + Ok(reaped) => metrics::counter!("buzz_deletion_serving_leases_reaped_total") + .increment(reaped), + Err(error) => tracing::warn!(%error, "serving-lease reaper failed"), + } + match deletion_store.serving_lease_stats().await { + Ok(stats) => { + metrics::gauge!("buzz_deletion_serving_leases_active") + .set(stats.active as f64); + metrics::gauge!("buzz_deletion_serving_leases_expired") + .set(stats.expired as f64); + metrics::gauge!("buzz_deletion_serving_leases_dead_tuples") + .set(stats.dead_tuples as f64); + } + Err(error) => tracing::warn!(%error, "serving-lease metrics failed"), + } } }); } diff --git a/crates/buzz-relay/src/mesh_boot.rs b/crates/buzz-relay/src/mesh_boot.rs index 20e550aa08..cd7c427c72 100644 --- a/crates/buzz-relay/src/mesh_boot.rs +++ b/crates/buzz-relay/src/mesh_boot.rs @@ -411,6 +411,7 @@ fn advertise_addrs(endpoint: &MeshEndpoint) -> Vec { pub async fn boot_mesh( config: &Config, redis_pool: deadpool_redis::Pool, + db: buzz_db::Db, relay_keypair: &nostr::Keys, shutting_down: Arc, ) -> anyhow::Result> { @@ -508,7 +509,7 @@ pub async fn boot_mesh( transport.set_inbound(Box::new(dispatcher.clone())); Ok(Some(MeshHandle { - directory: SessionDirectory::new(redis_pool), + directory: SessionDirectory::with_db(redis_pool, db), transport, membership: membership_arc, local_runtime_id: runtime_id, @@ -535,7 +536,13 @@ mod tests { .create_pool(Some(deadpool_redis::Runtime::Tokio1)) .unwrap(); let keys = nostr::Keys::generate(); - let handle = boot_mesh(&config, pool, &keys, Arc::new(AtomicBool::new(false))) + let db = buzz_db::Db::from_pool( + sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect_lazy("postgres://unused:unused@127.0.0.1:1/unused") + .expect("lazy database pool"), + ); + let handle = boot_mesh(&config, pool, db, &keys, Arc::new(AtomicBool::new(false))) .await .expect("off path is never an error"); assert!(handle.is_none()); diff --git a/crates/buzz-relay/src/push_runtime.rs b/crates/buzz-relay/src/push_runtime.rs index 49845067ea..4946b248c6 100644 --- a/crates/buzz-relay/src/push_runtime.rs +++ b/crates/buzz-relay/src/push_runtime.rs @@ -418,6 +418,23 @@ async fn deliver_one( return; } }; + let serving_write = match buzz_deletion::acquire_serving_write( + &state.db, + outcome.community, + "push_delivery", + ) + .await + { + Ok(guard) => guard, + Err(error) => { + warn!(wake=%outcome.id, %error, "push delivery suppressed by community deletion fence"); + let _ = state + .db + .fail_push_wake(outcome.community, outcome.id, outcome.claim_id) + .await; + return; + } + }; let Some(url) = state.config.push_gateway_delivery_url.as_ref() else { return; }; @@ -429,7 +446,20 @@ async fn deliver_one( return; } }; - let response = send_gateway_request(http, url, body, auth).await; + if let Err(error) = serving_write.verify().await { + warn!(wake=%outcome.id, %error, "push serving lease lost before delivery"); + return; + } + let response = match serving_write + .protect(send_gateway_request(http, url, body, auth)) + .await + { + Ok(response) => response, + Err(error) => { + warn!(wake=%outcome.id, %error, "push serving lease lost during delivery"); + return; + } + }; match response { Ok(r) if r.status().is_success() => match r.json::().await { Ok(DeliveryResponse::Accepted) => { @@ -502,6 +532,9 @@ async fn deliver_one( .await; } } + if let Err(error) = serving_write.finish().await { + warn!(wake=%outcome.id, %error, "failed to release community serving lease after push delivery"); + } } fn delivery_body(endpoint_grant: &str, request_id: uuid::Uuid, expires_at: i64) -> Vec { diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index 400ed1dfe3..82ad9938a2 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -376,22 +376,30 @@ async fn readiness_handler(State(state): State>) -> impl IntoRespo } let check = async { - let (pg_ok, redis_ok) = tokio::join!(state.db.ping(), async { - state.redis_pool.get().await.is_ok() - },); - (pg_ok, redis_ok) + let (pg_ok, redis_ok, deletion_catalog_ok) = tokio::join!( + state.db.ping(), + async { state.redis_pool.get().await.is_ok() }, + async { state.db.validate_deletion_serving_catalog().await.is_ok() }, + ); + (pg_ok, redis_ok, deletion_catalog_ok) }; - let (pg_ok, redis_ok) = tokio::time::timeout(Duration::from_secs(2), check) - .await - .unwrap_or((false, false)); + let (pg_ok, redis_ok, deletion_catalog_ok) = + tokio::time::timeout(Duration::from_secs(2), check) + .await + .unwrap_or((false, false, false)); - if pg_ok && redis_ok { + if pg_ok && redis_ok && deletion_catalog_ok { (StatusCode::OK, Json(json!({"status": "ready"}))).into_response() } else { ( StatusCode::SERVICE_UNAVAILABLE, - Json(json!({"status": "not_ready", "postgres": pg_ok, "redis": redis_ok})), + Json(json!({ + "status": "not_ready", + "postgres": pg_ok, + "redis": redis_ok, + "deletion_catalog": deletion_catalog_ok + })), ) .into_response() } diff --git a/crates/buzz-relay/src/tunnel/directory.rs b/crates/buzz-relay/src/tunnel/directory.rs index 7cd6e22a2d..42b121e431 100644 --- a/crates/buzz-relay/src/tunnel/directory.rs +++ b/crates/buzz-relay/src/tunnel/directory.rs @@ -87,6 +87,7 @@ return {current, known_generation} pub struct SessionDirectory { pool: deadpool_redis::Pool, lease_ttl: Duration, + db: Option, } /// Active session ownership lease read from Redis. @@ -179,6 +180,9 @@ pub enum DirectoryError { /// Lease TTL cannot be represented in Redis milliseconds. #[error("lease ttl must be at least 1ms and fit in i64 milliseconds")] InvalidLeaseTtl, + /// Durable community deletion fence rejected a Redis mutation. + #[error("community write fenced: {0}")] + CommunityWriteFenced(String), } impl SessionDirectory { @@ -187,9 +191,36 @@ impl SessionDirectory { Self::with_lease_ttl(pool, DEFAULT_LEASE_TTL) } + /// Create a serving directory whose Redis mutations use durable, + /// heartbeat-backed community write leases. + pub fn with_db(pool: deadpool_redis::Pool, db: buzz_db::Db) -> Self { + Self { + pool, + lease_ttl: DEFAULT_LEASE_TTL, + db: Some(db), + } + } + /// Create a directory backed by `pool` with an explicit lease TTL. pub fn with_lease_ttl(pool: deadpool_redis::Pool, lease_ttl: Duration) -> Self { - Self { pool, lease_ttl } + Self { + pool, + lease_ttl, + db: None, + } + } + + async fn begin_serving_write( + &self, + community_id: CommunityId, + ) -> Result, DirectoryError> { + match &self.db { + Some(db) => buzz_deletion::acquire_serving_write(db, community_id, "session_directory") + .await + .map(Some) + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string())), + None => Ok(None), + } } /// Attempt to create/take over the session lease. @@ -204,10 +235,11 @@ impl SessionDirectory { owner_runtime_id: RuntimeId, profile: Profile, ) -> Result { + let serving_write = self.begin_serving_write(community_id).await?; let keys = SessionKeys::new(community_id, session_id); let ttl_ms = ttl_ms(self.lease_ttl)?; let mut conn = self.pool.get().await?; - let (status, value, _known_generation): (String, String, String) = + let mutation = async { Script::new(ACQUIRE_SCRIPT) .key(&keys.lease) .key(&keys.generation) @@ -215,8 +247,22 @@ impl SessionDirectory { .arg(profile.as_wire_str()) .arg(ttl_ms) .invoke_async(&mut *conn) - .await?; + .await + }; + let (status, value, _known_generation): (String, String, String) = match &serving_write { + Some(guard) => guard + .protect(mutation) + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))??, + None => mutation.await?, + }; let lease = parse_lease(community_id, session_id, &value)?; + if let Some(guard) = serving_write { + guard + .finish() + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))?; + } match status.as_str() { "acquired" => Ok(AcquireResult::Acquired(lease)), "exists" => Ok(AcquireResult::Exists(lease)), @@ -244,18 +290,34 @@ impl SessionDirectory { /// Renew a lease only if the current Redis value exactly matches the /// caller's owner runtime and generation. pub async fn renew(&self, lease: &SessionLease) -> Result { + let serving_write = self.begin_serving_write(lease.community_id).await?; let keys = SessionKeys::new(lease.community_id, lease.session_id); let ttl_ms = ttl_ms(self.lease_ttl)?; let mut conn = self.pool.get().await?; - let (status, value, known_generation): (String, String, String) = Script::new(RENEW_SCRIPT) - .key(&keys.lease) - .key(&keys.generation) - .arg(lease.owner_runtime_id.to_hex()) - .arg(lease.generation) - .arg(ttl_ms) - .invoke_async(&mut *conn) - .await?; + let mutation = async { + Script::new(RENEW_SCRIPT) + .key(&keys.lease) + .key(&keys.generation) + .arg(lease.owner_runtime_id.to_hex()) + .arg(lease.generation) + .arg(ttl_ms) + .invoke_async(&mut *conn) + .await + }; + let (status, value, known_generation): (String, String, String) = match &serving_write { + Some(guard) => guard + .protect(mutation) + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))??, + None => mutation.await?, + }; let current = parse_optional_lease(lease.community_id, lease.session_id, &value)?; + if let Some(guard) = serving_write { + guard + .finish() + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))?; + } match status.as_str() { "renewed" => Ok(RenewResult::Renewed( current.expect("renewed returns lease"), @@ -275,17 +337,32 @@ impl SessionDirectory { /// Release a lease only if the current Redis value exactly matches the /// caller's owner runtime and generation. pub async fn release(&self, lease: &SessionLease) -> Result { + let serving_write = self.begin_serving_write(lease.community_id).await?; let keys = SessionKeys::new(lease.community_id, lease.session_id); let mut conn = self.pool.get().await?; - let (status, value, known_generation): (String, String, String) = + let mutation = async { Script::new(RELEASE_SCRIPT) .key(&keys.lease) .key(&keys.generation) .arg(lease.owner_runtime_id.to_hex()) .arg(lease.generation) .invoke_async(&mut *conn) - .await?; + .await + }; + let (status, value, known_generation): (String, String, String) = match &serving_write { + Some(guard) => guard + .protect(mutation) + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))??, + None => mutation.await?, + }; let current = parse_optional_lease(lease.community_id, lease.session_id, &value)?; + if let Some(guard) = serving_write { + guard + .finish() + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))?; + } match status.as_str() { "released" => Ok(ReleaseResult::Released( current.expect("released returns lease"), diff --git a/crates/buzz-test-client/tests/e2e_relay.rs b/crates/buzz-test-client/tests/e2e_relay.rs index 6f59299ed2..fa8ceb698a 100644 --- a/crates/buzz-test-client/tests/e2e_relay.rs +++ b/crates/buzz-test-client/tests/e2e_relay.rs @@ -120,7 +120,12 @@ async fn seed_relay_member(host: &str, keys: &Keys, role: &str) { } async fn seed_relay_owner(keys: &Keys) { - seed_relay_member("localhost:3000", keys, "owner").await; + seed_relay_member(&relay_authority(), keys, "owner").await; +} + +fn relay_authority() -> String { + let url = url::Url::parse(&relay_http_url()).expect("relay HTTP URL"); + url[url::Position::BeforeHost..url::Position::AfterPort].to_string() } fn http_origin_for_host(host: &str) -> String { @@ -315,7 +320,7 @@ async fn test_invite_claim_rejects_invalid_code() { #[ignore] async fn test_invite_mint_requires_owner_or_admin() { let member = Keys::generate(); - seed_relay_member("localhost:3000", &member, "member").await; + seed_relay_member(&relay_authority(), &member, "member").await; let response = invite_post(&member, "/api/invites", "{}").await; assert_eq!(response.status(), reqwest::StatusCode::FORBIDDEN); @@ -791,10 +796,10 @@ async fn test_auth_event_kind_rejected() { /// NIP-11 max_subscriptions must be enforced; (limit+1)th REQ gets CLOSED. /// -/// The relay's MAX_SUBSCRIPTIONS is 1024. Opening 1024 subs in a test is slow, -/// so we open a smaller batch and verify the NIP-11 advertised limit matches -/// the actual enforcement constant. The full-limit test is covered by the -/// NIP-11 assertion below (which verifies the advertised value is 1024). +/// This is a protocol-cap test, not an admission-throughput test. Open one REQ +/// at a time and wait out any shared fixed-window quota before retrying a REQ +/// rejected specifically as `rate-limited`, so production admission remains +/// enabled while the test deterministically reaches the independent 1024 cap. #[tokio::test] #[ignore] async fn test_subscription_limit_enforced() { @@ -802,60 +807,75 @@ async fn test_subscription_limit_enforced() { let keys = Keys::generate(); let mut client = BuzzTestClient::connect(&url, &keys).await.expect("connect"); - // Open 1024 subscriptions (the relay's MAX_SUBSCRIPTIONS). for i in 0..1024 { let sid = format!("limit-sub-{i}"); - let filter = Filter::new().kind(Kind::Custom(9)); - client - .subscribe(&sid, vec![filter]) - .await - .expect("subscribe"); - // Drain EOSE to avoid buffer buildup. - client - .collect_until_eose(&sid, Duration::from_secs(5)) - .await - .expect("EOSE"); + let filter = Filter::new().kind(Kind::Custom(49_999)); + subscribe_until_eose(&mut client, &sid, filter).await; } let overflow_sid = sub_id("overflow"); - // Use a kind that no other test writes, so we don't receive stale events. - let filter = Filter::new().kind(Kind::Custom(49999)); - client - .subscribe(&overflow_sid, vec![filter]) - .await - .expect("send REQ"); - - // Drain EOSE and stale events from the 100 earlier subscriptions - // until we receive the CLOSED for the overflow subscription. - let msg = loop { - let m = client - .recv_event(Duration::from_secs(5)) + let filter = Filter::new().kind(Kind::Custom(49_999)); + loop { + client + .subscribe(&overflow_sid, vec![filter.clone()]) .await - .expect("recv CLOSED (or timeout)"); - match &m { - RelayMessage::Eose { .. } => continue, - RelayMessage::Event { .. } => continue, // stale event from earlier subs - _ => break m, - } - }; + .expect("send overflow REQ"); - match msg { - RelayMessage::Closed { - subscription_id, - message, - } => { - assert_eq!(subscription_id, overflow_sid); - assert!( - message.to_lowercase().contains("too many"), - "Expected 'too many' in CLOSED message, got: {message}" - ); + match client + .recv_event(Duration::from_secs(6)) + .await + .expect("recv overflow CLOSED") + { + RelayMessage::Closed { + subscription_id, + message, + } if subscription_id == overflow_sid && message.starts_with("rate-limited:") => { + tokio::time::sleep(Duration::from_secs(5)).await; + } + RelayMessage::Closed { + subscription_id, + message, + } => { + assert_eq!(subscription_id, overflow_sid); + assert!( + message.to_lowercase().contains("too many"), + "Expected 'too many' in CLOSED message, got: {message}" + ); + break; + } + other => panic!("Expected CLOSED for overflow subscription, got {other:?}"), } - other => panic!("Expected CLOSED for overflow subscription, got {other:?}"), } client.disconnect().await.expect("disconnect"); } +async fn subscribe_until_eose(client: &mut BuzzTestClient, sid: &str, filter: Filter) { + loop { + client + .subscribe(sid, vec![filter.clone()]) + .await + .expect("subscribe"); + match client + .recv_event(Duration::from_secs(6)) + .await + .expect("EOSE or rate-limit CLOSED") + { + RelayMessage::Eose { subscription_id } => { + assert_eq!(subscription_id, sid); + return; + } + RelayMessage::Closed { + subscription_id, + message, + } if subscription_id == sid && message.starts_with("rate-limited:") => { + tokio::time::sleep(Duration::from_secs(5)).await; + } + other => panic!("unexpected response while opening {sid}: {other:?}"), + } + } +} + #[tokio::test] #[ignore] async fn test_nip11_relay_info() { diff --git a/crates/buzz-workflow/Cargo.toml b/crates/buzz-workflow/Cargo.toml index d4813e56d4..7d361b1477 100644 --- a/crates/buzz-workflow/Cargo.toml +++ b/crates/buzz-workflow/Cargo.toml @@ -10,6 +10,7 @@ description = "YAML-as-code workflow engine for Buzz" [dependencies] buzz-core = { workspace = true } buzz-db = { workspace = true } +buzz-deletion = { workspace = true } hex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/buzz-workflow/src/executor.rs b/crates/buzz-workflow/src/executor.rs index e30541377e..dffa492716 100644 --- a/crates/buzz-workflow/src/executor.rs +++ b/crates/buzz-workflow/src/executor.rs @@ -526,165 +526,202 @@ pub async fn dispatch_action( ) -> Result { use ActionDef::*; - match action { - SendMessage { text, channel } => { - // Look up workflow metadata for destination validation and - // attribution, scoped to the run's community — the same run/workflow - // UUID may exist in another community, so a bare-id lookup could - // load the wrong row and drive a side effect under it. - let wf_run = engine - .db - .get_workflow_run(community_id, run_id) - .await - .map_err(|e| { - WorkflowError::WebhookError(format!( - "SendMessage: failed to load workflow run {run_id}: {e}" - )) - })?; - let workflow = engine - .db - .get_workflow(community_id, wf_run.workflow_id) - .await - .map_err(|e| { - WorkflowError::WebhookError(format!( - "SendMessage: failed to load workflow {}: {e}", - wf_run.workflow_id - )) - })?; - let channel_id = resolve_send_message_channel( - channel.as_deref(), - &trigger_ctx.channel_id, - workflow.channel_id, - )?; - let owner_pubkey_hex = hex::encode(&workflow.owner_pubkey); - - info!( - run_id = %run_id, - step = step_id, - channel = %channel_id, - "SendMessage → {channel_id}: {text}" - ); - - let event_id = engine - .action_sink()? - .send_message(community_id, &channel_id, text, &owner_pubkey_hex) - .await - .map_err(WorkflowError::from)?; - - Ok(StepResult::Completed(serde_json::json!({ - "sent": true, - "event_id": event_id, - }))) - } - - SendDm { to, text: _ } => { - warn!(run_id = %run_id, step = step_id, "SendDm not yet implemented (to={to})"); - // TODO (WF-07): emit DM event. - Err(WorkflowError::NotImplemented("SendDm".into())) - } - - SetChannelTopic { topic: _ } => { - warn!(run_id = %run_id, step = step_id, "SetChannelTopic not yet implemented"); - // TODO (WF-07): update channel topic via DB. - Err(WorkflowError::NotImplemented("SetChannelTopic".into())) - } - - AddReaction { emoji } => { - info!(run_id = %run_id, step = step_id, "AddReaction → :{emoji}:"); - if trigger_ctx.message_id.is_empty() { - return Err(WorkflowError::InvalidDefinition( - "AddReaction: no trigger.message_id available".into(), - )); - } - - #[cfg(feature = "reqwest")] - { - let result = add_reaction_impl(&trigger_ctx.message_id, emoji).await?; - Ok(StepResult::Completed(result)) - } - - #[cfg(not(feature = "reqwest"))] - { - warn!( - run_id = %run_id, - step = step_id, - "AddReaction: reqwest feature not enabled, skipping HTTP call" - ); - Ok(StepResult::Completed( - serde_json::json!({ "added": false, "skipped": true }), + // The workflow engine can outlive the serving request that spawned it. + // Revalidate the durable community fence immediately before every external + // side effect (message publish, webhook, delay/resume). A storage failure is + // a denial, never permission to continue. + let serving_write = + buzz_deletion::acquire_serving_write(&engine.db, community_id, "workflow_action") + .await + .map_err(|error| { + WorkflowError::WebhookError(format!( + "community write fence rejected workflow side effect: {error}" )) - } - } + })?; - CallWebhook { - url, - method, - headers, - body, - } => { - let method_str = method.as_deref().unwrap_or("POST"); - info!(run_id = %run_id, step = step_id, "CallWebhook → {method_str} {url}"); + serving_write.verify().await.map_err(|error| { + WorkflowError::WebhookError(format!("community write lease lost: {error}")) + })?; - #[cfg(feature = "reqwest")] - { - let result = call_webhook_impl(url, method_str, headers, body).await?; - Ok(StepResult::Completed(result)) - } + let result = serving_write + .protect(async { + match action { + SendMessage { text, channel } => { + // Look up workflow metadata for destination validation and + // attribution, scoped to the run's community — the same run/workflow + // UUID may exist in another community, so a bare-id lookup could + // load the wrong row and drive a side effect under it. + let wf_run = engine + .db + .get_workflow_run(community_id, run_id) + .await + .map_err(|e| { + WorkflowError::WebhookError(format!( + "SendMessage: failed to load workflow run {run_id}: {e}" + )) + })?; + let workflow = engine + .db + .get_workflow(community_id, wf_run.workflow_id) + .await + .map_err(|e| { + WorkflowError::WebhookError(format!( + "SendMessage: failed to load workflow {}: {e}", + wf_run.workflow_id + )) + })?; + let channel_id = resolve_send_message_channel( + channel.as_deref(), + &trigger_ctx.channel_id, + workflow.channel_id, + )?; + let owner_pubkey_hex = hex::encode(&workflow.owner_pubkey); + + info!( + run_id = %run_id, + step = step_id, + channel = %channel_id, + "SendMessage → {channel_id}: {text}" + ); + + let event_id = engine + .action_sink()? + .send_message(community_id, &channel_id, text, &owner_pubkey_hex) + .await + .map_err(WorkflowError::from)?; + + Ok(StepResult::Completed(serde_json::json!({ + "sent": true, + "event_id": event_id, + }))) + } - #[cfg(not(feature = "reqwest"))] - { - // reqwest not enabled — log and return placeholder. - warn!( - run_id = %run_id, step = step_id, - "CallWebhook: reqwest feature not enabled, skipping HTTP call" - ); - let _ = (headers, body); // suppress unused warnings - Ok(StepResult::Completed(serde_json::json!({ - "status": 0, - "body": null, - "skipped": true - }))) - } - } + SendDm { to, text: _ } => { + warn!(run_id = %run_id, step = step_id, "SendDm not yet implemented (to={to})"); + // TODO (WF-07): emit DM event. + Err(WorkflowError::NotImplemented("SendDm".into())) + } - RequestApproval { - from, - message, - timeout, - } => { - let timeout_str = timeout.as_deref().unwrap_or("24h"); - info!( - run_id = %run_id, step = step_id, - "RequestApproval from={from} timeout={timeout_str}: {message}" - ); + SetChannelTopic { topic: _ } => { + warn!(run_id = %run_id, step = step_id, "SetChannelTopic not yet implemented"); + // TODO (WF-07): update channel topic via DB. + Err(WorkflowError::NotImplemented("SetChannelTopic".into())) + } + + AddReaction { emoji } => { + info!(run_id = %run_id, step = step_id, "AddReaction → :{emoji}:"); + if trigger_ctx.message_id.is_empty() { + Err(WorkflowError::InvalidDefinition( + "AddReaction: no trigger.message_id available".into(), + )) + } else { + #[cfg(feature = "reqwest")] + { + let result = add_reaction_impl(&trigger_ctx.message_id, emoji).await?; + Ok(StepResult::Completed(result)) + } + + #[cfg(not(feature = "reqwest"))] + { + warn!( + run_id = %run_id, + step = step_id, + "AddReaction: reqwest feature not enabled, skipping HTTP call" + ); + Ok(StepResult::Completed( + serde_json::json!({ "added": false, "skipped": true }), + )) + } + } + } - let token = generate_approval_token(run_id, step_id); + CallWebhook { + url, + method, + headers, + body, + } => { + let method_str = method.as_deref().unwrap_or("POST"); + info!(run_id = %run_id, step = step_id, "CallWebhook → {method_str} {url}"); + + #[cfg(feature = "reqwest")] + { + let result = call_webhook_impl(url, method_str, headers, body).await?; + Ok(StepResult::Completed(result)) + } - // TODO (WF-08): create approval record in DB, emit kind:46010. - // For now, return Suspended with the token so the caller can persist state. + #[cfg(not(feature = "reqwest"))] + { + // reqwest not enabled — log and return placeholder. + warn!( + run_id = %run_id, step = step_id, + "CallWebhook: reqwest feature not enabled, skipping HTTP call" + ); + let _ = (headers, body); // suppress unused warnings + Ok(StepResult::Completed(serde_json::json!({ + "status": 0, + "body": null, + "skipped": true + }))) + } + } - Ok(StepResult::Suspended { - approval_token: token, - }) - } + RequestApproval { + from, + message, + timeout, + } => { + let timeout_str = timeout.as_deref().unwrap_or("24h"); + info!( + run_id = %run_id, step = step_id, + "RequestApproval from={from} timeout={timeout_str}: {message}" + ); + + let token = generate_approval_token(run_id, step_id); + + // TODO (WF-08): create approval record in DB, emit kind:46010. + // For now, return Suspended with the token so the caller can persist state. + + Ok(StepResult::Suspended { + approval_token: token, + }) + } - Delay { duration } => { - let secs = parse_duration_secs(duration)?; - // Cap delay at 270 seconds (4.5 minutes) — must be less than default_timeout_secs (300s) - // to avoid non-deterministic StepTimeout. Long delays (hours/days) - // should use the scheduled resume pattern (future work: WF-09). - const MAX_DELAY_SECS: u64 = 270; - if secs > MAX_DELAY_SECS { - return Err(WorkflowError::InvalidDefinition(format!( - "delay exceeds maximum of {MAX_DELAY_SECS} seconds (got {secs}s); \ + Delay { duration } => { + let secs = parse_duration_secs(duration)?; + // Cap delay at 270 seconds (4.5 minutes) — must be less than default_timeout_secs (300s) + // to avoid non-deterministic StepTimeout. Long delays (hours/days) + // should use the scheduled resume pattern (future work: WF-09). + const MAX_DELAY_SECS: u64 = 270; + if secs > MAX_DELAY_SECS { + return Err(WorkflowError::InvalidDefinition(format!( + "delay exceeds maximum of {MAX_DELAY_SECS} seconds (got {secs}s); \ use the scheduled resume pattern for long delays" - ))); + ))); + } + info!(run_id = %run_id, step = step_id, "Delay {duration} ({secs}s)"); + tokio::time::sleep(std::time::Duration::from_secs(secs)).await; + Ok(StepResult::Completed( + serde_json::json!({ "slept_secs": secs }), + )) + } } - info!(run_id = %run_id, step = step_id, "Delay {duration} ({secs}s)"); - tokio::time::sleep(std::time::Duration::from_secs(secs)).await; - Ok(StepResult::Completed( - serde_json::json!({ "slept_secs": secs }), - )) + }) + .await + .map_err(|error| { + WorkflowError::WebhookError(format!("community write lease lost: {error}")) + })?; + let release = serving_write.finish().await.map_err(|error| { + WorkflowError::WebhookError(format!("community write lease release failed: {error}")) + }); + match result { + Ok(value) => { + release?; + Ok(value) + } + Err(error) => { + let _ = release; + Err(error) } } } diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql new file mode 100644 index 0000000000..21b71f378b --- /dev/null +++ b/migrations/0028_community_deletion.sql @@ -0,0 +1,536 @@ +-- Durable, CLI-only whole-community deletion control plane. +-- +-- The community row is never removed: it becomes the permanent name tombstone. +-- All destructive progress is lease/fence/checkpoint guarded and every existing +-- community-scoped table receives the same database-enforced write fence. +-- This migration intentionally remains one atomic catalog change so a failed +-- deployment cannot expose only a subset of the universal fences. CREATE +-- TRIGGER takes SHARE ROW EXCLUSIVE on each target; fail quickly rather than +-- queueing behind long transactions. See the chart deletion rollout runbook. +SET LOCAL lock_timeout = '5s'; + +ALTER TABLE communities + ADD COLUMN deletion_state TEXT NOT NULL DEFAULT 'active' + CHECK (deletion_state IN ('active', 'quiescing', 'fenced', 'tombstone')), + ADD COLUMN deletion_fence_generation BIGINT NOT NULL DEFAULT 0 + CHECK (deletion_fence_generation >= 0), + ADD COLUMN deleted_at TIMESTAMPTZ; + +CREATE TABLE community_deletion_requests ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + community_id UUID NOT NULL UNIQUE REFERENCES communities(id), + community_host TEXT NOT NULL, + stage TEXT NOT NULL DEFAULT 'submitted' CHECK (stage IN ( + 'submitted', 'inventoried', 'approved', 'fenced', 'drained', + 'bindings_removed', 'postgres_purged', 'cache_purged', + 'logically_verified', 'retention_pending' + )), + requested_by TEXT NOT NULL, + reason TEXT, + schema_manifest JSONB, + storage_manifest JSONB, + destructive_storage_manifest JSONB, + destructive_storage_frozen_at TIMESTAMPTZ, + inventory_manifest JSONB, + inventory_digest BYTEA CHECK (inventory_digest IS NULL OR length(inventory_digest) = 32), + inventory_frozen_at TIMESTAMPTZ, + fence_generation BIGINT CHECK (fence_generation IS NULL OR fence_generation > 0), + lease_owner TEXT, + lease_generation BIGINT NOT NULL DEFAULT 0 CHECK (lease_generation >= 0), + lease_until TIMESTAMPTZ, + attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), + retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_error TEXT, + last_error_at TIMESTAMPTZ, + blocked_at TIMESTAMPTZ, + blocked_reason TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + CHECK ((blocked_at IS NULL) = (blocked_reason IS NULL)), + CHECK ((inventory_frozen_at IS NULL) = (inventory_digest IS NULL)), + UNIQUE (id, community_id, inventory_digest) +); +CREATE INDEX community_deletion_requests_runnable + ON community_deletion_requests (next_attempt_at, created_at) + WHERE blocked_at IS NULL + AND stage IN ('approved', 'fenced', 'drained', 'bindings_removed', + 'postgres_purged', 'cache_purged', 'logically_verified'); +CREATE INDEX community_deletion_requests_lease + ON community_deletion_requests (lease_until) + WHERE lease_owner IS NOT NULL; + +CREATE TABLE community_deletion_approvals ( + request_id UUID PRIMARY KEY, + community_id UUID NOT NULL, + inventory_digest BYTEA NOT NULL CHECK (length(inventory_digest) = 32), + approved_by TEXT NOT NULL, + note TEXT, + approved_at TIMESTAMPTZ NOT NULL DEFAULT now(), + FOREIGN KEY (request_id, community_id, inventory_digest) + REFERENCES community_deletion_requests(id, community_id, inventory_digest) + ON DELETE RESTRICT +); + +-- The approval identity is only meaningful while its frozen target and +-- inventory remain unchanged. Make those facts irreversible in the database, +-- not merely conventions in the worker. +CREATE FUNCTION prevent_community_deletion_request_retargeting() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.community_id IS DISTINCT FROM OLD.community_id + OR NEW.community_host IS DISTINCT FROM OLD.community_host + THEN + RAISE EXCEPTION 'community deletion target identity is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF OLD.inventory_frozen_at IS NOT NULL AND ( + NEW.schema_manifest IS DISTINCT FROM OLD.schema_manifest + OR NEW.storage_manifest IS DISTINCT FROM OLD.storage_manifest + OR NEW.inventory_manifest IS DISTINCT FROM OLD.inventory_manifest + OR NEW.inventory_digest IS DISTINCT FROM OLD.inventory_digest + OR NEW.inventory_frozen_at IS DISTINCT FROM OLD.inventory_frozen_at + ) THEN + RAISE EXCEPTION 'frozen community deletion inventory is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF OLD.destructive_storage_frozen_at IS NOT NULL AND ( + NEW.destructive_storage_manifest IS DISTINCT FROM OLD.destructive_storage_manifest + OR NEW.destructive_storage_frozen_at IS DISTINCT FROM OLD.destructive_storage_frozen_at + ) THEN + RAISE EXCEPTION 'frozen destructive storage manifest is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER community_deletion_request_retargeting_guard +BEFORE UPDATE ON community_deletion_requests +FOR EACH ROW +EXECUTE FUNCTION prevent_community_deletion_request_retargeting(); + +CREATE FUNCTION prevent_community_deletion_approval_removal() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + RAISE EXCEPTION 'community deletion approval evidence is immutable' + USING ERRCODE = 'integrity_constraint_violation'; +END; +$$; + +CREATE TRIGGER community_deletion_approval_removal_guard +BEFORE UPDATE OR DELETE ON community_deletion_approvals +FOR EACH ROW +EXECUTE FUNCTION prevent_community_deletion_approval_removal(); + +CREATE TABLE community_deletion_checkpoints ( + request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, + sequence BIGINT GENERATED ALWAYS AS IDENTITY, + stage TEXT NOT NULL, + unit_key TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('started', 'completed', 'failed')), + lease_generation BIGINT NOT NULL CHECK (lease_generation > 0), + attempts INTEGER NOT NULL DEFAULT 1 CHECK (attempts > 0), + detail JSONB NOT NULL DEFAULT '{}'::jsonb, + error TEXT, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + PRIMARY KEY (request_id, sequence), + UNIQUE (request_id, stage, unit_key), + CHECK ((status = 'completed') = (completed_at IS NOT NULL)), + CHECK ((status = 'failed') = (error IS NOT NULL)) +); + +-- Frozen destructive key list, chunked out of the request row so a large +-- tenant (100k-1M objects) never materializes as one multi-hundred-MB JSONB +-- value. Rows are written once in the fenced stage, stamped `deleted_at` as +-- the executor confirms each chunk removed, and dropped at logical +-- verification. The request row keeps only per-prefix count/bytes/digest +-- summaries; the chunk stream must hash to those frozen digests. +CREATE TABLE community_deletion_manifest_keys ( + request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE CASCADE, + chunk_no BIGINT NOT NULL CHECK (chunk_no >= 0), + prefix TEXT NOT NULL, + keys JSONB NOT NULL, + deleted_at TIMESTAMPTZ, + PRIMARY KEY (request_id, chunk_no) +); + +-- Chunk content is immutable once written; the only permitted update is the +-- one-way deleted_at stamp. New chunks are permitted only while the request is +-- fenced and its destructive manifest remains unfrozen. Removal is permitted +-- only while the destructive manifest has not yet frozen (a retried partial +-- freeze rewrites its chunks) or once the request has passed logical +-- verification (terminal cleanup). +CREATE FUNCTION protect_community_deletion_manifest_keys() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + frozen_at TIMESTAMPTZ; + request_stage TEXT; +BEGIN + IF TG_OP = 'UPDATE' THEN + IF NEW.request_id IS DISTINCT FROM OLD.request_id + OR NEW.chunk_no IS DISTINCT FROM OLD.chunk_no + OR NEW.prefix IS DISTINCT FROM OLD.prefix + OR NEW.keys IS DISTINCT FROM OLD.keys + OR OLD.deleted_at IS NOT NULL + THEN + RAISE EXCEPTION 'community deletion manifest key chunks are immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + RETURN NEW; + END IF; + SELECT destructive_storage_frozen_at, stage + INTO frozen_at, request_stage + FROM community_deletion_requests + WHERE id = CASE WHEN TG_OP = 'INSERT' THEN NEW.request_id ELSE OLD.request_id END + FOR UPDATE; + IF TG_OP = 'INSERT' THEN + IF FOUND AND frozen_at IS NULL AND request_stage = 'fenced' THEN + RETURN NEW; + END IF; + RAISE EXCEPTION 'community deletion manifest key chunks require an unfrozen fenced request' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF NOT FOUND + OR frozen_at IS NULL + OR request_stage IN ('logically_verified', 'retention_pending') + THEN + RETURN OLD; + END IF; + RAISE EXCEPTION 'community deletion manifest key chunks cannot be removed mid-execution' + USING ERRCODE = 'integrity_constraint_violation'; +END; +$$; + +CREATE TRIGGER community_deletion_manifest_keys_guard +BEFORE INSERT OR UPDATE OR DELETE ON community_deletion_manifest_keys +FOR EACH ROW +EXECUTE FUNCTION protect_community_deletion_manifest_keys(); + +-- Fleet-wide object-store taxonomy sweep evidence. This is an independent +-- observability record: community deletion inventories only the target's owned +-- prefixes and does not gate submission or execution on sweep state. +CREATE TABLE storage_taxonomy_sweeps ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + started_at TIMESTAMPTZ NOT NULL, + completed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + listed_objects BIGINT NOT NULL CHECK (listed_objects >= 0), + unknown_object_count BIGINT NOT NULL CHECK (unknown_object_count >= 0), + unknown_key_sample JSONB NOT NULL DEFAULT '[]'::jsonb, + object_cap BIGINT NOT NULL CHECK (object_cap > 0), + CHECK (completed_at >= started_at) +); +CREATE INDEX storage_taxonomy_sweeps_latest + ON storage_taxonomy_sweeps (completed_at DESC); + +CREATE TABLE community_serving_write_leases ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + community_id UUID NOT NULL REFERENCES communities(id), + operation TEXT NOT NULL, + owner TEXT NOT NULL, + generation BIGINT NOT NULL DEFAULT 1 CHECK (generation > 0), + -- Community fence generation observed when this lease was acquired. + fence_generation BIGINT NOT NULL CHECK (fence_generation >= 0), + lease_until TIMESTAMPTZ NOT NULL, + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX community_serving_write_leases_active + ON community_serving_write_leases (community_id, lease_until); + +CREATE TABLE community_deletion_executor_heartbeats ( + executor_id TEXT PRIMARY KEY, + mode TEXT NOT NULL CHECK (mode IN ('run', 'drain', 'worker')), + request_id UUID REFERENCES community_deletion_requests(id) ON DELETE SET NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now(), + draining BOOLEAN NOT NULL DEFAULT false, + stopped_at TIMESTAMPTZ +); + +INSERT INTO _operator_global_tables (table_name, reason) VALUES + ('community_deletion_requests', 'deployment deletion lifecycle and frozen inventory'), + ('community_deletion_approvals', 'deployment operator destructive approvals'), + ('community_deletion_checkpoints', 'deployment deletion executor checkpoints and failures'), + ('community_deletion_manifest_keys', 'deployment deletion frozen destructive key chunks'), + ('storage_taxonomy_sweeps', 'deployment object-store taxonomy sweep evidence'), + ('community_serving_write_leases', 'deployment serving side-effect leases drained by deletion'), + ('community_deletion_executor_heartbeats', 'deployment deletion worker liveness'); + +-- Shared lock key used by both the trigger and the deletion engine. Every +-- ordinary tenant mutation takes the shared xact lock before checking state; +-- the fence transition takes the exclusive xact lock, so an already-open write +-- transaction cannot commit behind the fence and no new writer can slip ahead. +CREATE FUNCTION community_deletion_lock_key(target UUID) RETURNS BIGINT +LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ + SELECT hashtextextended('buzz-community-deletion:' || target::text, 0) +$$; + +-- Keep the deletion control plane writable while its target tenant is fenced. +-- This predicate is the single SQL source of truth used by attachment and live +-- catalog validation. +CREATE FUNCTION community_write_fence_excluded_table(target NAME) RETURNS BOOLEAN +LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ + SELECT target::TEXT = ANY (ARRAY[ + 'community_deletion_requests', + 'community_deletion_approvals', + 'community_deletion_checkpoints', + 'community_serving_write_leases', + 'community_deletion_executor_heartbeats' + ]::TEXT[]) +$$; + +CREATE FUNCTION assert_community_write_allowed(target UUID) RETURNS VOID +LANGUAGE plpgsql AS $$ +DECLARE + lifecycle TEXT; + generation BIGINT; + executor_community TEXT; + executor_generation TEXT; + serving_community TEXT; + serving_lease_id TEXT; + serving_owner TEXT; + serving_generation TEXT; + serving_fence_generation TEXT; + serving_lease_valid BOOLEAN := false; +BEGIN + -- Nullable operator-attribution rows without a tenant are unrelated. + IF target IS NULL THEN + RETURN; + END IF; + + PERFORM pg_advisory_xact_lock_shared(community_deletion_lock_key(target)); + SELECT deletion_state, deletion_fence_generation + INTO lifecycle, generation + FROM communities + WHERE id = target; + IF NOT FOUND THEN + RAISE EXCEPTION 'community write rejected: community % is missing', target + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + + -- Authorization is evaluated independently for every community checked. + executor_community := current_setting('buzz.deletion_executor_community', true); + executor_generation := current_setting('buzz.deletion_fence_generation', true); + IF executor_community = target::TEXT + AND executor_generation ~ '^[0-9]+$' + AND executor_generation::BIGINT = generation THEN + RETURN; + END IF; + + -- A serving mutation admitted before quiescing may finish only while its + -- exact durable lease remains current and bound to this fence generation. + serving_community := current_setting('buzz.serving_write_community', true); + serving_lease_id := current_setting('buzz.serving_write_lease_id', true); + serving_owner := current_setting('buzz.serving_write_owner', true); + serving_generation := current_setting('buzz.serving_write_generation', true); + serving_fence_generation := current_setting('buzz.serving_write_fence_generation', true); + IF lifecycle IN ('active', 'quiescing') + AND serving_community = target::TEXT + AND serving_lease_id ~ '^[0-9a-fA-F-]{36}$' + AND serving_generation ~ '^[0-9]+$' + AND serving_fence_generation ~ '^[0-9]+$' + AND serving_fence_generation::BIGINT = generation THEN + SELECT EXISTS( + SELECT 1 FROM community_serving_write_leases lease + WHERE lease.id = serving_lease_id::UUID + AND lease.community_id = target + AND lease.owner = serving_owner + AND lease.generation = serving_generation::BIGINT + AND lease.fence_generation = serving_fence_generation::BIGINT + AND lease.lease_until >= now() + ) INTO serving_lease_valid; + IF serving_lease_valid THEN + RETURN; + END IF; + END IF; + + IF lifecycle <> 'active' THEN + RAISE EXCEPTION 'community write fenced: community % generation %', target, generation + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; +END +$$; + +CREATE FUNCTION enforce_community_write_fence() RETURNS TRIGGER +LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + PERFORM assert_community_write_allowed(NEW.community_id); + ELSIF TG_OP = 'DELETE' THEN + PERFORM assert_community_write_allowed(OLD.community_id); + ELSIF OLD.community_id IS NOT DISTINCT FROM NEW.community_id THEN + PERFORM assert_community_write_allowed(OLD.community_id); + ELSIF OLD.community_id IS NULL THEN + PERFORM assert_community_write_allowed(NEW.community_id); + ELSIF NEW.community_id IS NULL THEN + PERFORM assert_community_write_allowed(OLD.community_id); + ELSIF OLD.community_id < NEW.community_id THEN + PERFORM assert_community_write_allowed(OLD.community_id); + PERFORM assert_community_write_allowed(NEW.community_id); + ELSE + PERFORM assert_community_write_allowed(NEW.community_id); + PERFORM assert_community_write_allowed(OLD.community_id); + END IF; + + RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; +END +$$; + +-- Protect the tombstone row itself. Normal updates are permitted only while the +-- row is active and do not change deletion metadata. Deletion executor updates +-- must present the exact durable generation in session-local GUCs. +CREATE FUNCTION enforce_community_tombstone() RETURNS TRIGGER +LANGUAGE plpgsql AS $$ +DECLARE + executor_community TEXT := current_setting('buzz.deletion_executor_community', true); + executor_generation TEXT := current_setting('buzz.deletion_fence_generation', true); + expected_generation BIGINT; +BEGIN + IF TG_OP = 'DELETE' THEN + IF OLD.deletion_state <> 'active' OR OLD.deleted_at IS NOT NULL THEN + RAISE EXCEPTION 'community tombstones are permanent' + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + RETURN OLD; + END IF; + + expected_generation := CASE + WHEN NEW.deletion_fence_generation > OLD.deletion_fence_generation + THEN NEW.deletion_fence_generation + ELSE OLD.deletion_fence_generation + END; + IF executor_community = OLD.id::text + AND executor_generation ~ '^[0-9]+$' + AND executor_generation::BIGINT = expected_generation THEN + RETURN NEW; + END IF; + + IF OLD.deletion_state <> 'active' + OR NEW.deletion_state <> OLD.deletion_state + OR NEW.deletion_fence_generation <> OLD.deletion_fence_generation + OR NEW.deleted_at IS DISTINCT FROM OLD.deleted_at THEN + RAISE EXCEPTION 'community tombstone mutation rejected: community % generation %', + OLD.id, OLD.deletion_fence_generation + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + RETURN NEW; +END +$$; + +CREATE TRIGGER communities_deletion_tombstone +BEFORE UPDATE OR DELETE ON communities +FOR EACH ROW EXECUTE FUNCTION enforce_community_tombstone(); + +-- Attach the universal fence to one community-scoped relation. Future +-- migrations must invoke this helper explicitly after CREATE/ALTER introduces +-- community_id; the migration lint enforces that contract. +CREATE FUNCTION attach_community_write_fence(target REGCLASS) RETURNS VOID +LANGUAGE plpgsql AS $$ +DECLARE + relation_name NAME; +BEGIN + SELECT c.relname + INTO relation_name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = target + AND n.nspname = current_schema() + AND c.relkind IN ('r', 'p') + AND NOT c.relispartition; + IF NOT FOUND THEN + RAISE EXCEPTION 'community write fence target % is not a table in the current schema', target + USING ERRCODE = 'wrong_object_type'; + END IF; + IF community_write_fence_excluded_table(relation_name) THEN + RETURN; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = target AND attname = 'community_id' AND NOT attisdropped + ) THEN + RAISE EXCEPTION 'community write fence target % has no community_id', target + USING ERRCODE = 'undefined_column'; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgrelid = target + AND tgname = 'community_write_fence_' || relation_name + AND NOT tgisinternal + ) THEN + EXECUTE format( + 'CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %s ' + 'FOR EACH ROW EXECUTE FUNCTION enforce_community_write_fence()', + 'community_write_fence_' || relation_name, + target + ); + END IF; +END +$$; + +-- Attach the universal fence to every existing table carrying community_id, +-- including deployment-private sidecars whose community_id is provenance. +DO $$ +DECLARE + target REGCLASS; +BEGIN + FOR target IN + SELECT c.oid::REGCLASS + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_attribute a ON a.attrelid = c.oid + WHERE n.nspname = current_schema() + AND c.relkind IN ('r', 'p') + AND NOT c.relispartition + AND a.attname = 'community_id' + AND NOT a.attisdropped + AND NOT community_write_fence_excluded_table(c.relname) + ORDER BY c.oid::REGCLASS::TEXT + LOOP + PERFORM attach_community_write_fence(target); + END LOOP; +END +$$; + +-- Desired-state schema application does not replay migration history, so keep +-- these explicit calls as first-class catalog declarations. They also make the +-- fence contract visible to migration linting instead of hiding it only in the +-- dynamic bootstrap loop above. +SELECT attach_community_write_fence('api_tokens'); +SELECT attach_community_write_fence('archived_identities'); +SELECT attach_community_write_fence('audit_log'); +SELECT attach_community_write_fence('channel_members'); +SELECT attach_community_write_fence('channels'); +SELECT attach_community_write_fence('community_bans'); +SELECT attach_community_write_fence('delivery_log'); +SELECT attach_community_write_fence('event_mentions'); +SELECT attach_community_write_fence('events'); +SELECT attach_community_write_fence('git_repo_names'); +SELECT attach_community_write_fence('join_policy_acceptances'); +SELECT attach_community_write_fence('moderation_actions'); +SELECT attach_community_write_fence('moderation_reports'); +SELECT attach_community_write_fence('parameterized_event_watermarks'); +SELECT attach_community_write_fence('product_feedback'); +SELECT attach_community_write_fence('pubkey_allowlist'); +SELECT attach_community_write_fence('push_leases'); +SELECT attach_community_write_fence('push_match_queue'); +SELECT attach_community_write_fence('push_wake_outbox'); +SELECT attach_community_write_fence('rate_limit_violations'); +SELECT attach_community_write_fence('reactions'); +SELECT attach_community_write_fence('relay_invites'); +SELECT attach_community_write_fence('relay_members'); +SELECT attach_community_write_fence('scheduled_workflow_fires'); +SELECT attach_community_write_fence('subscriptions'); +SELECT attach_community_write_fence('thread_metadata'); +SELECT attach_community_write_fence('users'); +SELECT attach_community_write_fence('workflow_approvals'); +SELECT attach_community_write_fence('workflow_runs'); +SELECT attach_community_write_fence('workflows'); diff --git a/schema/schema.sql b/schema/schema.sql index 9f3449b066..42068ecfed 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -59,6 +59,9 @@ CREATE TABLE communities ( icon TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), archived_at TIMESTAMPTZ, + deletion_state TEXT NOT NULL DEFAULT 'active' CHECK (deletion_state IN ('active', 'quiescing', 'fenced', 'tombstone')), + deletion_fence_generation BIGINT NOT NULL DEFAULT 0 CHECK (deletion_fence_generation >= 0), + deleted_at TIMESTAMPTZ, CONSTRAINT chk_communities_id_not_nil CHECK (id <> '00000000-0000-0000-0000-000000000000'::uuid) ); @@ -791,6 +794,48 @@ INSERT INTO _operator_global_tables (table_name, reason) VALUES ('communities', 'the tenant registry itself; id IS the community key'), ('rate_limit_violations', 'deployment abuse/health; never tenant-observable; community_id is an attribution label only'), ('_operator_global_tables', 'the registry table itself'); + +-- ── Additive tenant tables represented in migrations 0002/0007/0017 ────────── +-- Keep desired-state schema parity with the embedded SQLx migration path. +CREATE TABLE git_repo_names ( + community_id UUID NOT NULL REFERENCES communities(id), + repo_id TEXT NOT NULL, + owner_pubkey TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, repo_id) +); +CREATE INDEX idx_git_repo_names_owner ON git_repo_names (community_id, owner_pubkey); + +CREATE TABLE parameterized_event_watermarks ( + community_id UUID NOT NULL REFERENCES communities(id), + kind INT NOT NULL, + pubkey BYTEA NOT NULL, + d_tag TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + event_id BYTEA NOT NULL, + PRIMARY KEY (community_id, kind, pubkey, d_tag) +); +CREATE INDEX idx_event_mentions_community_event + ON event_mentions (community_id, event_id); + +CREATE TABLE product_feedback ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + community_id UUID NOT NULL REFERENCES communities(id), + event_id BYTEA NOT NULL CHECK (length(event_id) = 32), + submitter_pubkey BYTEA NOT NULL CHECK (length(submitter_pubkey) = 32), + category TEXT CHECK (category IN ('bug', 'praise', 'needs-work')), + body TEXT NOT NULL CHECK (length(btrim(body)) > 0), + tags JSONB NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(tags) = 'array'), + event_created_at TIMESTAMPTZ NOT NULL, + received_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (event_id) +); +CREATE INDEX idx_product_feedback_received + ON product_feedback (received_at DESC, id); +CREATE INDEX idx_product_feedback_community_received + ON product_feedback (community_id, received_at DESC, id); +INSERT INTO _operator_global_tables (table_name, reason) VALUES + ('product_feedback', 'deployment product inbox; community_id is provenance only'); -- NIP-PL effective lease state and durable wake outbox. Every key is led by -- community_id: client-provided origin is confirmation only, never routing. CREATE TABLE push_leases ( @@ -1080,3 +1125,410 @@ INSERT INTO replica_heartbeat (id) VALUES (1); INSERT INTO _operator_global_tables (table_name, reason) VALUES ('replica_heartbeat', 'single-row replication freshness token; describes deployment topology, never tenant data'); + +-- ── Whole-community deletion control plane (migration 0028) ───────────────── +CREATE TABLE community_deletion_requests ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + community_id UUID NOT NULL UNIQUE REFERENCES communities(id), + community_host TEXT NOT NULL, + stage TEXT NOT NULL DEFAULT 'submitted' CHECK (stage IN ( + 'submitted', 'inventoried', 'approved', 'fenced', 'drained', + 'bindings_removed', 'postgres_purged', 'cache_purged', + 'logically_verified', 'retention_pending' + )), + requested_by TEXT NOT NULL, + reason TEXT, + schema_manifest JSONB, + storage_manifest JSONB, + destructive_storage_manifest JSONB, + destructive_storage_frozen_at TIMESTAMPTZ, + inventory_manifest JSONB, + inventory_digest BYTEA CHECK (inventory_digest IS NULL OR length(inventory_digest) = 32), + inventory_frozen_at TIMESTAMPTZ, + fence_generation BIGINT CHECK (fence_generation IS NULL OR fence_generation > 0), + lease_owner TEXT, + lease_generation BIGINT NOT NULL DEFAULT 0 CHECK (lease_generation >= 0), + lease_until TIMESTAMPTZ, + attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), + retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_error TEXT, + last_error_at TIMESTAMPTZ, + blocked_at TIMESTAMPTZ, + blocked_reason TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + CHECK ((blocked_at IS NULL) = (blocked_reason IS NULL)), + CHECK ((inventory_frozen_at IS NULL) = (inventory_digest IS NULL)), + UNIQUE (id, community_id, inventory_digest) +); +CREATE INDEX community_deletion_requests_runnable + ON community_deletion_requests (next_attempt_at, created_at) + WHERE blocked_at IS NULL + AND stage IN ('approved', 'fenced', 'drained', 'bindings_removed', + 'postgres_purged', 'cache_purged', 'logically_verified'); +CREATE INDEX community_deletion_requests_lease + ON community_deletion_requests (lease_until) WHERE lease_owner IS NOT NULL; + +CREATE TABLE community_deletion_approvals ( + request_id UUID PRIMARY KEY, + community_id UUID NOT NULL, + inventory_digest BYTEA NOT NULL CHECK (length(inventory_digest) = 32), + approved_by TEXT NOT NULL, + note TEXT, + approved_at TIMESTAMPTZ NOT NULL DEFAULT now(), + FOREIGN KEY (request_id, community_id, inventory_digest) + REFERENCES community_deletion_requests(id, community_id, inventory_digest) + ON DELETE RESTRICT +); + +CREATE FUNCTION prevent_community_deletion_request_retargeting() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.community_id IS DISTINCT FROM OLD.community_id + OR NEW.community_host IS DISTINCT FROM OLD.community_host + THEN + RAISE EXCEPTION 'community deletion target identity is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF OLD.inventory_frozen_at IS NOT NULL AND ( + NEW.schema_manifest IS DISTINCT FROM OLD.schema_manifest + OR NEW.storage_manifest IS DISTINCT FROM OLD.storage_manifest + OR NEW.inventory_manifest IS DISTINCT FROM OLD.inventory_manifest + OR NEW.inventory_digest IS DISTINCT FROM OLD.inventory_digest + OR NEW.inventory_frozen_at IS DISTINCT FROM OLD.inventory_frozen_at + ) THEN + RAISE EXCEPTION 'frozen community deletion inventory is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF OLD.destructive_storage_frozen_at IS NOT NULL AND ( + NEW.destructive_storage_manifest IS DISTINCT FROM OLD.destructive_storage_manifest + OR NEW.destructive_storage_frozen_at IS DISTINCT FROM OLD.destructive_storage_frozen_at + ) THEN + RAISE EXCEPTION 'frozen destructive storage manifest is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER community_deletion_request_retargeting_guard +BEFORE UPDATE ON community_deletion_requests +FOR EACH ROW +EXECUTE FUNCTION prevent_community_deletion_request_retargeting(); + +CREATE FUNCTION prevent_community_deletion_approval_removal() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + RAISE EXCEPTION 'community deletion approval evidence is immutable' + USING ERRCODE = 'integrity_constraint_violation'; +END; +$$; + +CREATE TRIGGER community_deletion_approval_removal_guard +BEFORE UPDATE OR DELETE ON community_deletion_approvals +FOR EACH ROW +EXECUTE FUNCTION prevent_community_deletion_approval_removal(); + +CREATE TABLE community_deletion_checkpoints ( + request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, + sequence BIGINT GENERATED ALWAYS AS IDENTITY, + stage TEXT NOT NULL, + unit_key TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('started', 'completed', 'failed')), + lease_generation BIGINT NOT NULL CHECK (lease_generation > 0), + attempts INTEGER NOT NULL DEFAULT 1 CHECK (attempts > 0), + detail JSONB NOT NULL DEFAULT '{}'::jsonb, + error TEXT, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + PRIMARY KEY (request_id, sequence), + UNIQUE (request_id, stage, unit_key), + CHECK ((status = 'completed') = (completed_at IS NOT NULL)), + CHECK ((status = 'failed') = (error IS NOT NULL)) +); +CREATE TABLE community_serving_write_leases ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + community_id UUID NOT NULL REFERENCES communities(id), + operation TEXT NOT NULL, + owner TEXT NOT NULL, + generation BIGINT NOT NULL DEFAULT 1 CHECK (generation > 0), + -- Community fence generation observed when this lease was acquired. + fence_generation BIGINT NOT NULL CHECK (fence_generation >= 0), + lease_until TIMESTAMPTZ NOT NULL, + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX community_serving_write_leases_active + ON community_serving_write_leases (community_id, lease_until); + +CREATE TABLE community_deletion_executor_heartbeats ( + executor_id TEXT PRIMARY KEY, + mode TEXT NOT NULL CHECK (mode IN ('run', 'drain', 'worker')), + request_id UUID REFERENCES community_deletion_requests(id) ON DELETE SET NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now(), + draining BOOLEAN NOT NULL DEFAULT false, + stopped_at TIMESTAMPTZ +); +INSERT INTO _operator_global_tables (table_name, reason) VALUES + ('community_deletion_requests', 'deployment deletion lifecycle and frozen inventory'), + ('community_deletion_approvals', 'deployment operator destructive approvals'), + ('community_deletion_checkpoints', 'deployment deletion executor checkpoints and failures'), + ('community_serving_write_leases', 'deployment serving side-effect leases drained by deletion'), + ('community_deletion_executor_heartbeats', 'deployment deletion worker liveness'); + +CREATE FUNCTION community_deletion_lock_key(target UUID) RETURNS BIGINT +LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ + SELECT hashtextextended('buzz-community-deletion:' || target::text, 0) +$$; +-- Keep the deletion control plane writable while its target tenant is fenced. +-- This predicate is the single SQL source of truth used by attachment and live +-- catalog validation. +CREATE FUNCTION community_write_fence_excluded_table(target NAME) RETURNS BOOLEAN +LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ + SELECT target::TEXT = ANY (ARRAY[ + 'community_deletion_requests', + 'community_deletion_approvals', + 'community_deletion_checkpoints', + 'community_serving_write_leases', + 'community_deletion_executor_heartbeats' + ]::TEXT[]) +$$; + +CREATE FUNCTION assert_community_write_allowed(target UUID) RETURNS VOID +LANGUAGE plpgsql AS $$ +DECLARE + lifecycle TEXT; + generation BIGINT; + executor_community TEXT; + executor_generation TEXT; + serving_community TEXT; + serving_lease_id TEXT; + serving_owner TEXT; + serving_generation TEXT; + serving_fence_generation TEXT; + serving_lease_valid BOOLEAN := false; +BEGIN + -- Nullable operator-attribution rows without a tenant are unrelated. + IF target IS NULL THEN + RETURN; + END IF; + + PERFORM pg_advisory_xact_lock_shared(community_deletion_lock_key(target)); + SELECT deletion_state, deletion_fence_generation + INTO lifecycle, generation + FROM communities + WHERE id = target; + IF NOT FOUND THEN + RAISE EXCEPTION 'community write rejected: community % is missing', target + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + + -- Authorization is evaluated independently for every community checked. + executor_community := current_setting('buzz.deletion_executor_community', true); + executor_generation := current_setting('buzz.deletion_fence_generation', true); + IF executor_community = target::TEXT + AND executor_generation ~ '^[0-9]+$' + AND executor_generation::BIGINT = generation THEN + RETURN; + END IF; + + -- A serving mutation admitted before quiescing may finish only while its + -- exact durable lease remains current and bound to this fence generation. + serving_community := current_setting('buzz.serving_write_community', true); + serving_lease_id := current_setting('buzz.serving_write_lease_id', true); + serving_owner := current_setting('buzz.serving_write_owner', true); + serving_generation := current_setting('buzz.serving_write_generation', true); + serving_fence_generation := current_setting('buzz.serving_write_fence_generation', true); + IF lifecycle IN ('active', 'quiescing') + AND serving_community = target::TEXT + AND serving_lease_id ~ '^[0-9a-fA-F-]{36}$' + AND serving_generation ~ '^[0-9]+$' + AND serving_fence_generation ~ '^[0-9]+$' + AND serving_fence_generation::BIGINT = generation THEN + SELECT EXISTS( + SELECT 1 FROM community_serving_write_leases lease + WHERE lease.id = serving_lease_id::UUID + AND lease.community_id = target + AND lease.owner = serving_owner + AND lease.generation = serving_generation::BIGINT + AND lease.fence_generation = serving_fence_generation::BIGINT + AND lease.lease_until >= now() + ) INTO serving_lease_valid; + IF serving_lease_valid THEN + RETURN; + END IF; + END IF; + + IF lifecycle <> 'active' THEN + RAISE EXCEPTION 'community write fenced: community % generation %', target, generation + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; +END +$$; + +CREATE FUNCTION enforce_community_write_fence() RETURNS TRIGGER +LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + PERFORM assert_community_write_allowed(NEW.community_id); + ELSIF TG_OP = 'DELETE' THEN + PERFORM assert_community_write_allowed(OLD.community_id); + ELSIF OLD.community_id IS NOT DISTINCT FROM NEW.community_id THEN + PERFORM assert_community_write_allowed(OLD.community_id); + ELSIF OLD.community_id IS NULL THEN + PERFORM assert_community_write_allowed(NEW.community_id); + ELSIF NEW.community_id IS NULL THEN + PERFORM assert_community_write_allowed(OLD.community_id); + ELSIF OLD.community_id < NEW.community_id THEN + PERFORM assert_community_write_allowed(OLD.community_id); + PERFORM assert_community_write_allowed(NEW.community_id); + ELSE + PERFORM assert_community_write_allowed(NEW.community_id); + PERFORM assert_community_write_allowed(OLD.community_id); + END IF; + + RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; +END +$$; + +CREATE FUNCTION enforce_community_tombstone() RETURNS TRIGGER +LANGUAGE plpgsql AS $$ +DECLARE + executor_community TEXT := current_setting('buzz.deletion_executor_community', true); + executor_generation TEXT := current_setting('buzz.deletion_fence_generation', true); + expected_generation BIGINT; +BEGIN + IF TG_OP = 'DELETE' THEN + IF OLD.deletion_state <> 'active' OR OLD.deleted_at IS NOT NULL THEN + RAISE EXCEPTION 'community tombstones are permanent' + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + RETURN OLD; + END IF; + expected_generation := CASE WHEN NEW.deletion_fence_generation > OLD.deletion_fence_generation + THEN NEW.deletion_fence_generation ELSE OLD.deletion_fence_generation END; + IF executor_community = OLD.id::text AND executor_generation ~ '^[0-9]+$' + AND executor_generation::BIGINT = expected_generation THEN RETURN NEW; END IF; + IF OLD.deletion_state <> 'active' OR NEW.deletion_state <> OLD.deletion_state + OR NEW.deletion_fence_generation <> OLD.deletion_fence_generation + OR NEW.deleted_at IS DISTINCT FROM OLD.deleted_at THEN + RAISE EXCEPTION 'community tombstone mutation rejected: community % generation %', + OLD.id, OLD.deletion_fence_generation + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + RETURN NEW; +END +$$; +CREATE TRIGGER communities_deletion_tombstone BEFORE UPDATE OR DELETE ON communities +FOR EACH ROW EXECUTE FUNCTION enforce_community_tombstone(); +-- Attach the universal fence to one community-scoped relation. Future +-- migrations must invoke this helper explicitly after CREATE/ALTER introduces +-- community_id; the migration lint enforces that contract. +CREATE FUNCTION attach_community_write_fence(target REGCLASS) RETURNS VOID +LANGUAGE plpgsql AS $$ +DECLARE + relation_name NAME; +BEGIN + SELECT c.relname + INTO relation_name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = target + AND n.nspname = current_schema() + AND c.relkind IN ('r', 'p') + AND NOT c.relispartition; + IF NOT FOUND THEN + RAISE EXCEPTION 'community write fence target % is not a table in the current schema', target + USING ERRCODE = 'wrong_object_type'; + END IF; + IF community_write_fence_excluded_table(relation_name) THEN + RETURN; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = target AND attname = 'community_id' AND NOT attisdropped + ) THEN + RAISE EXCEPTION 'community write fence target % has no community_id', target + USING ERRCODE = 'undefined_column'; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgrelid = target + AND tgname = 'community_write_fence_' || relation_name + AND NOT tgisinternal + ) THEN + EXECUTE format( + 'CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %s ' + 'FOR EACH ROW EXECUTE FUNCTION enforce_community_write_fence()', + 'community_write_fence_' || relation_name, + target + ); + END IF; +END +$$; + +-- Attach the universal fence to every existing table carrying community_id, +-- including deployment-private sidecars whose community_id is provenance. +DO $$ +DECLARE + target REGCLASS; +BEGIN + FOR target IN + SELECT c.oid::REGCLASS + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_attribute a ON a.attrelid = c.oid + WHERE n.nspname = current_schema() + AND c.relkind IN ('r', 'p') + AND NOT c.relispartition + AND a.attname = 'community_id' + AND NOT a.attisdropped + AND NOT community_write_fence_excluded_table(c.relname) + ORDER BY c.oid::REGCLASS::TEXT + LOOP + PERFORM attach_community_write_fence(target); + END LOOP; +END +$$; + +-- Desired-state schema application does not replay migration history, so keep +-- these explicit calls as first-class catalog declarations. They also make the +-- fence contract visible to migration linting instead of hiding it only in the +-- dynamic bootstrap loop above. +SELECT attach_community_write_fence('api_tokens'); +SELECT attach_community_write_fence('archived_identities'); +SELECT attach_community_write_fence('audit_log'); +SELECT attach_community_write_fence('channel_members'); +SELECT attach_community_write_fence('channels'); +SELECT attach_community_write_fence('community_bans'); +SELECT attach_community_write_fence('delivery_log'); +SELECT attach_community_write_fence('event_mentions'); +SELECT attach_community_write_fence('events'); +SELECT attach_community_write_fence('git_repo_names'); +SELECT attach_community_write_fence('join_policy_acceptances'); +SELECT attach_community_write_fence('moderation_actions'); +SELECT attach_community_write_fence('moderation_reports'); +SELECT attach_community_write_fence('parameterized_event_watermarks'); +SELECT attach_community_write_fence('product_feedback'); +SELECT attach_community_write_fence('pubkey_allowlist'); +SELECT attach_community_write_fence('push_leases'); +SELECT attach_community_write_fence('push_match_queue'); +SELECT attach_community_write_fence('push_wake_outbox'); +SELECT attach_community_write_fence('rate_limit_violations'); +SELECT attach_community_write_fence('reactions'); +SELECT attach_community_write_fence('relay_invites'); +SELECT attach_community_write_fence('relay_members'); +SELECT attach_community_write_fence('scheduled_workflow_fires'); +SELECT attach_community_write_fence('subscriptions'); +SELECT attach_community_write_fence('thread_metadata'); +SELECT attach_community_write_fence('users'); +SELECT attach_community_write_fence('workflow_approvals'); +SELECT attach_community_write_fence('workflow_runs'); +SELECT attach_community_write_fence('workflows'); diff --git a/scripts/attach-schema-partitions.sql b/scripts/attach-schema-partitions.sql index d02fc7c82a..5837676f84 100644 --- a/scripts/attach-schema-partitions.sql +++ b/scripts/attach-schema-partitions.sql @@ -21,6 +21,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p_past; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p_past; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p_past; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p_past; ALTER TABLE events ATTACH PARTITION events_p_past FOR VALUES FROM (MINVALUE) TO ('2026-01-01'); END IF; @@ -33,6 +34,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_01; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_01; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_01; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_01; ALTER TABLE events ATTACH PARTITION events_p2026_01 FOR VALUES FROM ('2026-01-01') TO ('2026-02-01'); END IF; @@ -45,6 +47,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_02; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_02; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_02; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_02; ALTER TABLE events ATTACH PARTITION events_p2026_02 FOR VALUES FROM ('2026-02-01') TO ('2026-03-01'); END IF; @@ -57,6 +60,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_03; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_03; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_03; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_03; ALTER TABLE events ATTACH PARTITION events_p2026_03 FOR VALUES FROM ('2026-03-01') TO ('2026-04-01'); END IF; @@ -69,6 +73,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_04; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_04; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_04; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_04; ALTER TABLE events ATTACH PARTITION events_p2026_04 FOR VALUES FROM ('2026-04-01') TO ('2026-05-01'); END IF; @@ -81,6 +86,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_05; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_05; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_05; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_05; ALTER TABLE events ATTACH PARTITION events_p2026_05 FOR VALUES FROM ('2026-05-01') TO ('2026-06-01'); END IF; @@ -93,6 +99,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_06; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_06; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_06; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_06; ALTER TABLE events ATTACH PARTITION events_p2026_06 FOR VALUES FROM ('2026-06-01') TO ('2026-07-01'); END IF; @@ -105,6 +112,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p_future; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p_future; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p_future; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p_future; ALTER TABLE events ATTACH PARTITION events_p_future FOR VALUES FROM ('2026-07-01') TO (MAXVALUE); END IF; @@ -122,6 +130,7 @@ BEGIN AND inhrelid = 'delivery_log_p_past'::regclass ) THEN ALTER TABLE delivery_log_p_past ALTER COLUMN id DROP IDENTITY IF EXISTS; + DROP TRIGGER IF EXISTS community_write_fence_delivery_log ON delivery_log_p_past; ALTER TABLE delivery_log ATTACH PARTITION delivery_log_p_past FOR VALUES FROM (MINVALUE) TO ('2026-03-01'); END IF; @@ -132,6 +141,7 @@ BEGIN AND inhrelid = 'delivery_log_p2026_03'::regclass ) THEN ALTER TABLE delivery_log_p2026_03 ALTER COLUMN id DROP IDENTITY IF EXISTS; + DROP TRIGGER IF EXISTS community_write_fence_delivery_log ON delivery_log_p2026_03; ALTER TABLE delivery_log ATTACH PARTITION delivery_log_p2026_03 FOR VALUES FROM ('2026-03-01') TO ('2026-04-01'); END IF; @@ -142,6 +152,7 @@ BEGIN AND inhrelid = 'delivery_log_p2026_04'::regclass ) THEN ALTER TABLE delivery_log_p2026_04 ALTER COLUMN id DROP IDENTITY IF EXISTS; + DROP TRIGGER IF EXISTS community_write_fence_delivery_log ON delivery_log_p2026_04; ALTER TABLE delivery_log ATTACH PARTITION delivery_log_p2026_04 FOR VALUES FROM ('2026-04-01') TO ('2026-05-01'); END IF; @@ -152,6 +163,7 @@ BEGIN AND inhrelid = 'delivery_log_p2026_05'::regclass ) THEN ALTER TABLE delivery_log_p2026_05 ALTER COLUMN id DROP IDENTITY IF EXISTS; + DROP TRIGGER IF EXISTS community_write_fence_delivery_log ON delivery_log_p2026_05; ALTER TABLE delivery_log ATTACH PARTITION delivery_log_p2026_05 FOR VALUES FROM ('2026-05-01') TO ('2026-06-01'); END IF; @@ -162,6 +174,7 @@ BEGIN AND inhrelid = 'delivery_log_p2026_06'::regclass ) THEN ALTER TABLE delivery_log_p2026_06 ALTER COLUMN id DROP IDENTITY IF EXISTS; + DROP TRIGGER IF EXISTS community_write_fence_delivery_log ON delivery_log_p2026_06; ALTER TABLE delivery_log ATTACH PARTITION delivery_log_p2026_06 FOR VALUES FROM ('2026-06-01') TO ('2026-07-01'); END IF; @@ -172,6 +185,7 @@ BEGIN AND inhrelid = 'delivery_log_p_future'::regclass ) THEN ALTER TABLE delivery_log_p_future ALTER COLUMN id DROP IDENTITY IF EXISTS; + DROP TRIGGER IF EXISTS community_write_fence_delivery_log ON delivery_log_p_future; ALTER TABLE delivery_log ATTACH PARTITION delivery_log_p_future FOR VALUES FROM ('2026-07-01') TO (MAXVALUE); END IF;