diff --git a/.env.example b/.env.example index b9bfcada0e..07202d7a3f 100644 --- a/.env.example +++ b/.env.example @@ -58,6 +58,10 @@ RELAY_URL=ws://localhost:3000 # the web frontend at / for browser requests. Leave unset for local dev # (use `just web` for Vite HMR instead). # BUZZ_WEB_DIR=./web/dist +# Maximum accepted age of an event's created_at in seconds (default 900). +# Future drift stays fixed at +15 min. Raise temporarily to run a history +# import (e.g. `buzz import slack` — see docs/slack-import.md), then restore. +# BUZZ_MAX_PAST_DRIFT_SECS=900 # Shared Redis-backed admission limits. Defaults shown below; each value must # be a positive integer. diff --git a/Cargo.lock b/Cargo.lock index 937ead564a..e580d92de5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1078,6 +1078,34 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-migrate" +version = "0.1.0" +dependencies = [ + "axum", + "base64 0.22.1", + "buzz-core", + "buzz-sdk", + "buzz-ws-client", + "clap", + "hex", + "hmac 0.13.0", + "nostr", + "rand 0.10.1", + "reqwest 0.13.4", + "ring", + "serde", + "serde_json", + "sha2 0.11.0", + "subtle", + "thiserror 2.0.18", + "tokio", + "tower-http", + "tracing", + "tracing-subscriber", + "url", +] + [[package]] name = "buzz-pair-relay" version = "0.1.0" @@ -10418,6 +10446,7 @@ dependencies = [ "getrandom 0.4.3", "js-sys", "serde_core", + "sha1_smol", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index cc1dd0f9df..f13dd7a595 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ members = [ "crates/buzz-workflow", "crates/buzz-media", "crates/buzz-cli", + "crates/buzz-migrate", "crates/buzz-pairing-cli", "crates/buzz-sdk", "crates/buzz-persona", @@ -95,7 +96,7 @@ thiserror = "2" anyhow = "1" # Utilities -uuid = { version = "1", features = ["v4", "serde"] } +uuid = { version = "1", features = ["v4", "v5", "serde"] } chrono = { version = "0.4", features = ["serde"] } # HTTP client (webhook delivery) @@ -106,6 +107,7 @@ sha2 = "0.11" hex = "0.4" hmac = "0.13" base64 = "0.22" +ring = "0.17" # Randomness rand = "0.10" diff --git a/crates/buzz-cli/src/commands/import.rs b/crates/buzz-cli/src/commands/import.rs new file mode 100644 index 0000000000..ac07f05fdd --- /dev/null +++ b/crates/buzz-cli/src/commands/import.rs @@ -0,0 +1,988 @@ +//! `buzz import` — migrate history from external workspaces. +//! +//! v1 supports Slack workspace exports; see `docs/slack-import.md` for the +//! full design (attribution model, security, limitations). +//! +//! ## Attribution model (zero key custody, two-party consent) +//! +//! Every imported event is signed by the CLI identity (bot mode) and carries +//! `import`/`import_author`/`import_ts` provenance tags. Real people are +//! attributed by a **two-party identity binding**, using **public keys only** — +//! no private key is ever generated for or distributed to anyone: +//! +//! 1. An owner/admin **attestation** (kind `KIND_IMPORT_IDENTITY_BINDING`) +//! mapping a Slack user id to a person's Buzz pubkey — `buzz import bind` / +//! `--identity-map`. +//! 2. The subject's own **claim** (kind `KIND_IMPORT_IDENTITY_CLAIM`), +//! self-signed with their key — `buzz import claim`. +//! +//! History renders under the real person only when both exist for the same +//! Slack id and the attestation's pubkey equals the claim's signer. So a member +//! cannot claim another person's history (no admin attestation), and an admin +//! cannot make someone appear to author history they never wrote (no subject +//! claim). See `docs/slack-import.md` for the residual trust in a colluding +//! admin + subject. + +mod channel_state; +mod dm; +mod export; +mod importer; +mod mapping; +mod model; +mod mrkdwn; +mod render; +mod report; +mod state; + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; + +use nostr::PublicKey; + +use crate::client::BuzzClient; +use crate::error::CliError; +use dm::dm_import_blockers; +use export::{SlackChannel, SlackConversationKind, SlackExport}; +use importer::{publish_binding, submit, Importer, ImporterOptions}; +use mapping::load_channel_map; +use report::dry_run_report; +use state::{ChannelState, ImportState}; + +/// Parameters for `buzz import slack`. +pub struct ImportSlackParams { + /// One or more unzipped Slack export directories. Multiple roots support + /// Slackdump's separate public/private export passes. + pub export_dirs: Vec, + /// Slack workspace id (team id) — namespaces identity bindings and channel + /// UUIDs so ids can't collide across workspaces. + pub team_id: String, + /// State file path override. + pub state: Option, + /// Optional Slack conversation id → existing Buzz channel UUID crosswalk. + pub channel_map: Option, + /// Optional comma-separated channel-name filter. + pub channels: Option, + /// Report the plan without writing anything. + pub dry_run: bool, + /// Skip reaction import. + pub skip_reactions: bool, + /// Optional `SLACKID=npub,SLACKID=hex,…` identity bindings to publish + /// (owner/admin-signed) so imported history renders under real people. + pub identity_map: Option, +} + +pub async fn cmd_import_slack(client: &BuzzClient, p: ImportSlackParams) -> Result<(), CliError> { + let team_id = validate_team_id(&p.team_id)?.to_string(); + let export_dirs: Vec = p.export_dirs.iter().map(PathBuf::from).collect(); + let export = SlackExport::load_many(&export_dirs)?; + let channel_map = p + .channel_map + .as_deref() + .map(PathBuf::from) + .map(|path| load_channel_map(&path)) + .transpose()? + .unwrap_or_default(); + + let state_path = p + .state + .as_ref() + .map(PathBuf::from) + .unwrap_or_else(|| export_dirs[0].join("buzz-import-state.json")); + let mut state = ImportState::load_for_workspace(&state_path, &team_id)?; + validate_channel_map(&export, &state, &channel_map)?; + + // Slack user id → display name, for mrkdwn mention rewriting and + // author attribution tags. + let names: HashMap = export + .users + .iter() + .map(|(id, u)| (id.clone(), u.best_name().to_string())) + .collect(); + + // Parse identity bindings (Slack id → Buzz pubkey), public keys only. + let bindings = parse_identity_map(p.identity_map.as_deref())?; + let binding_map: HashMap = bindings.iter().cloned().collect(); + + let selected = select_channels(&export, p.channels.as_deref())?; + let dm_blockers = dm_import_blockers( + &export, + &selected, + &state, + &binding_map, + &client.keys().public_key().to_hex(), + ); + + if p.dry_run { + return dry_run_report( + &export, + &selected, + &state, + &channel_map, + &binding_map, + &dm_blockers, + p.skip_reactions, + ); + } + if !dm_blockers.is_empty() { + let mut blocked: Vec = dm_blockers + .iter() + .map(|(id, reason)| format!("{id}: {reason}")) + .collect(); + blocked.sort(); + return Err(CliError::Usage(format!( + "{} Slack DM/MPIM conversation(s) are not safe to import:\n{}", + blocked.len(), + blocked.join("\n") + ))); + } + + seed_channel_map(&mut state, &channel_map); + let mut importer = Importer::new( + client, + &export, + &names, + ImporterOptions { + team_id: &team_id, + state, + state_path, + skip_reactions: p.skip_reactions, + bindings: &binding_map, + }, + ); + + for channel in &selected { + importer.import_channel(channel).await?; + } + + // Publish owner/admin-signed identity bindings last, so the history they + // attribute is already in place. + importer.publish_bindings(&bindings).await?; + + importer.finish() +} + +/// Publish the owner/admin half of a two-party binding: an attestation that +/// `slack_id` maps to `pubkey`. Inert until the subject also runs +/// `cmd_import_claim` with their own key. +pub async fn cmd_import_bind( + client: &BuzzClient, + team_id: &str, + slack_id: &str, + pubkey: &str, +) -> Result<(), CliError> { + let team_id = validate_team_id(team_id)?; + let slack_id = validate_slack_id(slack_id)?; + let pubkey_hex = parse_pubkey(pubkey)?; + let event_id = publish_binding(client, team_id, slack_id, &pubkey_hex).await?; + print_json(&serde_json::json!({ + "event_id": event_id, + "team_id": team_id, + "slack_id": slack_id, + "pubkey": pubkey_hex, + "accepted": true, + })) +} + +/// Publish the subject half of a two-party binding: the caller's self-signed +/// consent to being attributed `slack_id`. Signed by the CLI identity, so the +/// person whose history it is runs this with their own key. Inert until a +/// community owner/admin has published the matching attestation for this +/// pubkey. +pub async fn cmd_import_claim( + client: &BuzzClient, + team_id: &str, + slack_id: &str, +) -> Result<(), CliError> { + let team_id = validate_team_id(team_id)?; + let slack_id = validate_slack_id(slack_id)?; + let d_tag = buzz_sdk::slack_identity_binding_d_tag(team_id, slack_id); + let builder = buzz_sdk::build_import_identity_claim(&d_tag) + .map_err(|e| CliError::Other(format!("build_import_identity_claim failed: {e}")))?; + let event_id = submit(client, builder).await?; + print_json(&serde_json::json!({ + "event_id": event_id, + "team_id": team_id, + "slack_id": slack_id, + "pubkey": client.keys().public_key().to_hex(), + "accepted": true, + })) +} + +/// Parse a `SLACKID=key,SLACKID=key` list into `(slack_id, pubkey_hex)` pairs. +/// Each key may be an `npub1…` or a 64-char hex pubkey — **public keys only**. +fn parse_identity_map(spec: Option<&str>) -> Result, CliError> { + let Some(spec) = spec else { + return Ok(Vec::new()); + }; + let entries: Vec<(String, String)> = spec + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|entry| { + let (slack_id, key) = entry.split_once('=').ok_or_else(|| { + CliError::Usage(format!( + "--identity-map entry must be SLACKID=npub-or-hex (got {entry:?})" + )) + })?; + Ok(( + validate_slack_id(slack_id)?.to_string(), + parse_pubkey(key.trim())?, + )) + }) + .collect::>()?; + let mut seen = HashSet::new(); + for (slack_id, _) in &entries { + if !seen.insert(slack_id) { + return Err(CliError::Usage(format!( + "--identity-map contains duplicate Slack id {slack_id}" + ))); + } + } + Ok(entries) +} + +/// Validate and normalize a Slack user id supplied on the command line. +fn validate_slack_id(slack_id: &str) -> Result<&str, CliError> { + validate_slack_ident(slack_id, "user id") +} + +/// Validate and normalize a Slack workspace (team) id supplied on the command +/// line. Same character rules as a user id. +fn validate_team_id(team_id: &str) -> Result<&str, CliError> { + validate_slack_ident(team_id, "workspace (team) id") +} + +fn validate_slack_ident<'a>(value: &'a str, label: &str) -> Result<&'a str, CliError> { + let value = value.trim(); + if value.is_empty() + || !value + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') + { + return Err(CliError::Usage(format!("invalid Slack {label} {value:?}"))); + } + Ok(value) +} + +/// Parse an `npub1…` or 64-char hex string into a hex pubkey. Rejects nsec so +/// a private key can never be passed where a public key belongs. +fn parse_pubkey(key: &str) -> Result { + if key.starts_with("nsec1") { + return Err(CliError::Usage( + "identity bindings take a PUBLIC key (npub or hex), not an nsec".into(), + )); + } + PublicKey::parse(key) + .map(|pk| pk.to_hex()) + .map_err(|_| CliError::Usage(format!("invalid pubkey (expected npub or 64-hex): {key}"))) +} + +/// Resolve the `--channels` filter against conversation names or Slack IDs, +/// erroring if any selector is unknown. +fn select_channels<'e>( + export: &'e SlackExport, + filter: Option<&str>, +) -> Result, CliError> { + let filter: Option> = filter.map(|list| { + list.split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect() + }); + if let Some(requested) = &filter { + let available: HashSet<&str> = export + .channels + .iter() + .flat_map(|c| [c.name.as_str(), c.id.as_str()]) + .collect(); + let mut missing: Vec<&str> = requested + .iter() + .map(String::as_str) + .filter(|name| !available.contains(name)) + .collect(); + missing.sort_unstable(); + if !missing.is_empty() { + return Err(CliError::Usage(format!( + "unknown channel(s) in --channels: {}", + missing.join(", ") + ))); + } + } + let selected: Vec<&SlackChannel> = export + .channels + .iter() + .filter(|c| { + filter + .as_ref() + .is_none_or(|f| f.contains(&c.name) || f.contains(&c.id)) + }) + .collect(); + if selected.is_empty() { + return Err(CliError::Usage( + "no conversations selected — check --channels against Slack names or ids".into(), + )); + } + Ok(selected) +} + +fn validate_channel_map( + export: &SlackExport, + state: &ImportState, + channel_map: &HashMap, +) -> Result<(), CliError> { + let conversations: HashMap<&str, &SlackChannel> = export + .channels + .iter() + .map(|conversation| (conversation.id.as_str(), conversation)) + .collect(); + for (slack_id, buzz_id) in channel_map { + let conversation = conversations.get(slack_id.as_str()).ok_or_else(|| { + CliError::Usage(format!( + "channel map references Slack conversation {slack_id}, which is absent from the export" + )) + })?; + if matches!( + conversation.kind, + SlackConversationKind::DirectMessage | SlackConversationKind::GroupDirectMessage + ) { + return Err(CliError::Usage(format!( + "channel map cannot adopt Slack {} {slack_id}; DMs and MPIMs must be opened \ + natively from their complete mapped participant set", + conversation.kind.as_str() + ))); + } + if let Some(existing) = state.channels.get(slack_id) { + if existing.uuid != *buzz_id { + return Err(CliError::Usage(format!( + "channel map assigns Slack conversation {slack_id} to {buzz_id}, but the \ + state file already assigns it to {}", + existing.uuid + ))); + } + } + } + Ok(()) +} + +fn seed_channel_map(state: &mut ImportState, channel_map: &HashMap) { + for (slack_id, buzz_id) in channel_map { + state + .channels + .entry(slack_id.clone()) + .or_insert_with(|| ChannelState { + uuid: buzz_id.clone(), + metadata_done: false, + archived_done: false, + private_visibility_done: false, + prepared_for_import: false, + members_added: HashSet::new(), + }); + } +} + +/// Serialize `value` to compact JSON on stdout. +fn print_json(value: &serde_json::Value) -> Result<(), CliError> { + let rendered = serde_json::to_string(value) + .map_err(|e| CliError::Other(format!("summary serialization failed: {e}")))?; + println!("{rendered}"); + Ok(()) +} + +/// Dispatch for `buzz import`. +pub async fn dispatch(cmd: crate::ImportCmd, client: &BuzzClient) -> Result<(), CliError> { + match cmd { + crate::ImportCmd::Slack { + export_dirs, + team_id, + state, + channel_map, + channels, + dry_run, + skip_reactions, + identity_map, + } => { + cmd_import_slack( + client, + ImportSlackParams { + export_dirs, + team_id, + state, + channel_map, + channels, + dry_run, + skip_reactions, + identity_map, + }, + ) + .await + } + crate::ImportCmd::Bind { + team_id, + slack_id, + pubkey, + } => cmd_import_bind(client, &team_id, &slack_id, &pubkey).await, + crate::ImportCmd::Claim { team_id, slack_id } => { + cmd_import_claim(client, &team_id, &slack_id).await + } + } +} + +#[cfg(test)] +mod tests { + use super::channel_state::metadata_archived_state; + use super::dm::{ + build_import_dm_open, ensure_dm_uuid_unclaimed, mapped_dm_participants, response_payload, + }; + use super::export::SlackMessage; + use super::importer::{ + author_display, author_id, build_imported_message, channel_uuid, provenance_tags, + thread_root_key, + }; + use super::render::{emoji_for_shortcode, render_attachment, render_slack_blocks}; + use super::report::pending_reaction_count; + use super::*; + use nostr::{EventId, Keys}; + use uuid::Uuid; + + fn msg(json: &str) -> SlackMessage { + serde_json::from_str(json).expect("test message parses") + } + + fn channel() -> SlackChannel { + serde_json::from_str(r#"{"id":"C1","name":"general"}"#).expect("channel parses") + } + + #[test] + fn channel_uuid_is_deterministic_and_team_scoped() { + // Same inputs → same UUID, so a crash-resumed run reuses the channel + // instead of minting a duplicate. + assert_eq!(channel_uuid("T1", "C1"), channel_uuid("T1", "C1")); + // Distinct team or channel → distinct UUID (no cross-workspace or + // cross-channel collision). + assert_ne!(channel_uuid("T1", "C1"), channel_uuid("T2", "C1")); + assert_ne!(channel_uuid("T1", "C1"), channel_uuid("T1", "C2")); + } + + #[test] + fn existing_channel_map_is_validated_and_seeded_as_unprepared() { + let dir = + std::env::temp_dir().join(format!("buzz-import-channel-map-export-{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("mkdir"); + std::fs::write( + dir.join("channels.json"), + r#"[{"id":"C1","name":"general"}]"#, + ) + .expect("write channels"); + std::fs::write(dir.join("users.json"), "[]").expect("write users"); + let export = SlackExport::load_many(std::slice::from_ref(&dir)).expect("export"); + let mut state = + ImportState::load_for_workspace(&dir.join("state.json"), "T1").expect("fresh state"); + let buzz_id = Uuid::new_v4().to_string(); + let channel_map = HashMap::from([("C1".to_string(), buzz_id.clone())]); + + validate_channel_map(&export, &state, &channel_map).expect("valid map"); + seed_channel_map(&mut state, &channel_map); + let adopted = &state.channels["C1"]; + assert_eq!(adopted.uuid, buzz_id); + assert!(!adopted.prepared_for_import); + assert!(!adopted.private_visibility_done); + assert!(!adopted.archived_done); + + let unknown = HashMap::from([("C9".to_string(), Uuid::new_v4().to_string())]); + assert!(validate_channel_map(&export, &state, &unknown).is_err()); + let conflict = HashMap::from([("C1".to_string(), Uuid::new_v4().to_string())]); + assert!(validate_channel_map(&export, &state, &conflict).is_err()); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn native_dm_open_requires_the_exact_mapped_participant_set() { + let dir = + std::env::temp_dir().join(format!("buzz-import-dm-participants-{}", Uuid::new_v4())); + std::fs::create_dir_all(dir.join("D1")).expect("mkdir"); + std::fs::write(dir.join("channels.json"), "[]").expect("write channels"); + std::fs::write( + dir.join("users.json"), + r#"[{"id":"U1","name":"alice"},{"id":"U2","name":"bob"}]"#, + ) + .expect("write users"); + std::fs::write( + dir.join("dms.json"), + r#"[{"id":"D1","created":1700000000,"members":["U1","U2"]}]"#, + ) + .expect("write dms"); + let export = SlackExport::load_many(std::slice::from_ref(&dir)).expect("load export"); + let dm = export + .channels + .iter() + .find(|conversation| conversation.id == "D1") + .expect("DM"); + + let importer_keys = Keys::generate(); + let importer_pubkey = importer_keys.public_key().to_hex(); + let other_pubkey = Keys::generate().public_key().to_hex(); + let mut bindings = HashMap::from([("U1".to_string(), importer_pubkey.clone())]); + assert!(mapped_dm_participants(&export, dm, &bindings, &importer_pubkey).is_err()); + + bindings.insert("U2".to_string(), other_pubkey.clone()); + let participants = + mapped_dm_participants(&export, dm, &bindings, &importer_pubkey).expect("complete map"); + assert_eq!( + participants.iter().cloned().collect::>(), + HashSet::from([importer_pubkey.clone(), other_pubkey.clone()]) + ); + assert!( + mapped_dm_participants( + &export, + dm, + &bindings, + &Keys::generate().public_key().to_hex() + ) + .is_err(), + "migration operator cannot become an extra DM participant" + ); + + let event = build_import_dm_open(dm, "T1", &participants, &importer_pubkey) + .expect("build native DM open") + .sign_with_keys(&importer_keys) + .expect("sign"); + assert_eq!(event.kind.as_u16(), 41010); + let tags: Vec> = event + .tags + .iter() + .map(|tag| tag.as_slice().to_vec()) + .collect(); + assert!(tags.contains(&vec!["p".into(), other_pubkey])); + assert!(!tags.contains(&vec!["p".into(), importer_pubkey])); + assert!(tags.contains(&vec!["d".into(), "slack:T1:D1".into()])); + assert!(tags.contains(&vec!["import".into(), "slack".into()])); + assert!(tags.contains(&vec![ + "import_conversation".into(), + "T1:D1".into(), + "direct_message".into() + ])); + + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn native_dm_response_payload_extracts_relay_channel_id() { + let channel_id = Uuid::new_v4(); + let response = serde_json::json!({ + "event_id": "abc", + "accepted": true, + "message": format!( + "response:{}", + serde_json::json!({"channel_id": channel_id, "created": true}) + ), + }) + .to_string(); + let payload = response_payload(&response).expect("response payload"); + let channel_id_string = channel_id.to_string(); + assert_eq!( + payload + .get("channel_id") + .and_then(serde_json::Value::as_str), + Some(channel_id_string.as_str()) + ); + assert_eq!( + payload.get("created").and_then(serde_json::Value::as_bool), + Some(true) + ); + } + + #[test] + fn separate_slack_dms_cannot_collapse_into_one_buzz_participant_set() { + let dir = std::env::temp_dir().join(format!("buzz-import-dm-collision-{}", Uuid::new_v4())); + std::fs::create_dir_all(dir.join("D1")).expect("mkdir D1"); + std::fs::create_dir_all(dir.join("D2")).expect("mkdir D2"); + std::fs::write(dir.join("channels.json"), "[]").expect("channels"); + std::fs::write( + dir.join("users.json"), + r#"[{"id":"U1","name":"alice"},{"id":"U2","name":"bob"}]"#, + ) + .expect("users"); + std::fs::write( + dir.join("dms.json"), + r#"[{"id":"D1","members":["U1","U2"]},{"id":"D2","members":["U1","U2"]}]"#, + ) + .expect("dms"); + let export = SlackExport::load_many(std::slice::from_ref(&dir)).expect("export"); + let selected: Vec<&SlackChannel> = export.channels.iter().collect(); + let importer_pubkey = Keys::generate().public_key().to_hex(); + let bindings = HashMap::from([ + ("U1".to_string(), importer_pubkey.clone()), + ("U2".to_string(), Keys::generate().public_key().to_hex()), + ]); + let state = ImportState::load_for_workspace(&dir.join("state.json"), "T1").expect("state"); + let blockers = dm_import_blockers(&export, &selected, &state, &bindings, &importer_pubkey); + assert_eq!(blockers.len(), 2); + assert!(blockers["D1"].contains("merge separate histories")); + assert!(blockers["D2"].contains("merge separate histories")); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn resumed_dm_cannot_reuse_another_slack_conversations_buzz_uuid() { + let uuid = Uuid::new_v4(); + let mut state = ImportState::default(); + state.channels.insert( + "D1".into(), + ChannelState { + uuid: uuid.to_string(), + metadata_done: true, + archived_done: true, + private_visibility_done: true, + prepared_for_import: true, + members_added: HashSet::new(), + }, + ); + let mut d2 = channel(); + d2.id = "D2".into(); + d2.name = "dm-two".into(); + d2.kind = SlackConversationKind::DirectMessage; + + let error = ensure_dm_uuid_unclaimed(&state, &d2, uuid) + .expect_err("different Slack DM must not claim an existing Buzz DM"); + assert!(error.to_string().contains("already imported for D1")); + assert!(error.to_string().contains("merge separate histories")); + } + + #[test] + fn channel_metadata_readback_reports_archived_state() { + let channel_id = Uuid::new_v4(); + let active = serde_json::json!([{ + "kind": 39000, + "tags": [["d", channel_id], ["name", "general"]], + }]) + .to_string(); + let archived = serde_json::json!([{ + "kind": 39000, + "tags": [["d", channel_id], ["name", "general"], ["archived", "true"]], + }]) + .to_string(); + let unrelated = serde_json::json!([{ + "kind": 39000, + "tags": [["d", Uuid::new_v4()], ["archived", "true"]], + }]) + .to_string(); + + assert_eq!( + metadata_archived_state(&active, channel_id).expect("active metadata"), + Some(false) + ); + assert_eq!( + metadata_archived_state(&archived, channel_id).expect("archived metadata"), + Some(true) + ); + assert_eq!( + metadata_archived_state(&unrelated, channel_id).expect("unrelated metadata"), + None + ); + } + + #[test] + fn author_resolution() { + let user_msg = msg(r#"{"type":"message","user":"U1","text":"x","ts":"1.0"}"#); + assert_eq!(author_id(&user_msg).as_deref(), Some("U1")); + + let bot_msg = msg( + r#"{"type":"message","subtype":"bot_message","bot_id":"B9","username":"CI","text":"x","ts":"1.0"}"#, + ); + assert_eq!(author_id(&bot_msg).as_deref(), Some("B9")); + + let mut names = HashMap::new(); + names.insert("U1".to_string(), "alice".to_string()); + assert_eq!(author_display(&user_msg, &names), "alice"); + assert_eq!(author_display(&bot_msg, &names), "CI"); + } + + #[test] + fn imported_message_escapes_markdown_in_the_author_prefix() { + let message = msg(r#"{"type":"message","user":"U1","text":"hello","ts":"100.0"}"#); + let names = HashMap::from([("U1".to_string(), r"A*B_C\D[bot]`".to_string())]); + let event = + build_imported_message(&channel(), Uuid::new_v4(), &message, &names, "T1", None) + .expect("builder") + .sign_with_keys(&Keys::generate()) + .expect("signs"); + + assert_eq!(event.content, r"**A\*B\_C\\D\[bot\]\`**: hello"); + } + + #[test] + fn thread_root_key_only_for_replies() { + let channel: SlackChannel = + serde_json::from_str(r#"{"id":"C1","name":"general"}"#).expect("channel parses"); + let root = msg(r#"{"type":"message","user":"U1","text":"x","ts":"5.0","thread_ts":"5.0"}"#); + assert_eq!(thread_root_key(&channel, &root), None); + let reply = + msg(r#"{"type":"message","user":"U1","text":"y","ts":"6.0","thread_ts":"5.0"}"#); + assert_eq!( + thread_root_key(&channel, &reply), + Some("C1:5.0".to_string()) + ); + let plain = msg(r#"{"type":"message","user":"U1","text":"z","ts":"7.0"}"#); + assert_eq!(thread_root_key(&channel, &plain), None); + } + + #[test] + fn emoji_mapping() { + assert_eq!(emoji_for_shortcode("+1"), "👍"); + assert_eq!(emoji_for_shortcode("thumbsup::skin-tone-3"), "👍"); + assert_eq!(emoji_for_shortcode("party_parrot"), ":party_parrot:"); + } + + #[test] + fn identity_map_parses_npub_and_hex_and_rejects_nsec() { + let hex = "8f3904246ba9d9cc7e821e7752e123d435234d17c2513d85785f4a0b1ca07e56"; + let parsed = parse_identity_map(Some(&format!("U1={hex}"))).expect("parses hex"); + assert_eq!(parsed, vec![("U1".to_string(), hex.to_string())]); + + assert!( + parse_identity_map(Some("U1=nsec1abc")).is_err(), + "nsec rejected" + ); + assert!( + parse_identity_map(Some("U1")).is_err(), + "missing = rejected" + ); + assert!( + parse_identity_map(Some(&format!("U1={hex},U1={hex}"))).is_err(), + "duplicate Slack ids rejected" + ); + assert!( + parse_identity_map(Some(&format!("={hex}"))).is_err(), + "empty Slack id rejected" + ); + assert!(parse_identity_map(None).expect("none ok").is_empty()); + } + + #[test] + fn pending_reactions_include_resumable_work_and_dedupe_aliases() { + let msg = msg(r#"{"type":"message","user":"U1","text":"x","ts":"1.0", + "reactions":[{"name":"+1"},{"name":"thumbsup"},{"name":"heart"}]}"#); + let mut state = ImportState::default(); + assert_eq!(pending_reaction_count(&state, "C1:1.0", &msg), 2); + state.reactions.insert("C1:1.0:👍".into()); + assert_eq!(pending_reaction_count(&state, "C1:1.0", &msg), 1); + } + + #[test] + fn imported_message_keeps_routing_and_provenance_tags() { + let channel_id = Uuid::new_v4(); + let root = EventId::from_hex(&"11".repeat(32)).expect("event id"); + let thread_ref = buzz_sdk::ThreadRef { + root_event_id: root, + parent_event_id: root, + }; + let message = msg( + r#"{"type":"message","user":"U1","text":"hello","ts":"100.000002", + "thread_ts":"99.000001"}"#, + ); + let mut names = HashMap::new(); + names.insert("U1".to_string(), "Alice".to_string()); + + let event = build_imported_message( + &channel(), + channel_id, + &message, + &names, + "T1", + Some(&thread_ref), + ) + .expect("builder") + .sign_with_keys(&Keys::generate()) + .expect("signs"); + let tags: Vec> = event + .tags + .iter() + .map(|tag| tag.as_slice().to_vec()) + .collect(); + + assert!(tags.contains(&vec!["h".into(), channel_id.to_string()])); + assert!(tags.iter().any(|tag| tag.first().is_some_and(|v| v == "e"))); + assert!(tags.contains(&vec!["import".into(), "slack".into()])); + // The import_author id is workspace-scoped (`:`) so it + // composes to the same `slack:T1:U1` key the identity binding uses. + assert!(tags.contains(&vec![ + "import_author".into(), + "T1:U1".into(), + "Alice".into() + ])); + assert!(tags.contains(&vec!["import_ts".into(), "100.000002".into()])); + assert!(tags.contains(&vec![ + "import_conversation".into(), + "T1:C1".into(), + "public_channel".into() + ])); + assert_eq!(event.created_at.as_secs(), 100); + } + + #[test] + fn imported_thread_broadcast_remains_visible_in_the_channel_timeline() { + let channel_id = Uuid::new_v4(); + let root = EventId::from_hex(&"11".repeat(32)).expect("event id"); + let thread_ref = buzz_sdk::ThreadRef { + root_event_id: root, + parent_event_id: root, + }; + let message = msg( + r#"{"type":"message","subtype":"thread_broadcast","user":"U1", + "text":"shared reply","ts":"100.000002","thread_ts":"99.000001"}"#, + ); + let event = build_imported_message( + &channel(), + channel_id, + &message, + &HashMap::new(), + "T1", + Some(&thread_ref), + ) + .expect("builder") + .sign_with_keys(&Keys::generate()) + .expect("signs"); + + assert!(event.tags.iter().any(|tag| { + let parts = tag.as_slice(); + parts.first().map(String::as_str) == Some("broadcast") + && parts.get(1).map(String::as_str) == Some("1") + })); + } + + #[test] + fn attachment_only_message_retains_visible_card_content() { + let message = msg( + r#"{"type":"message","subtype":"bot_message","bot_id":"B1","username":"CI", + "text":"","ts":"100.000002","attachments":[{ + "author_name":"Buildkite","author_link":"https://ci.example.test", + "title":"main passed","title_link":"https://ci.example.test/build/1", + "text":"*42 tests* passed", + "fields":[{"title":"Commit","value":"`abc123`"}] + }]}"#, + ); + let event = build_imported_message( + &channel(), + Uuid::new_v4(), + &message, + &HashMap::new(), + "T1", + None, + ) + .expect("builder") + .sign_with_keys(&Keys::generate()) + .expect("signs"); + + assert!(event + .content + .contains("[Buildkite](https://ci.example.test)")); + assert!(event + .content + .contains("**[main passed](https://ci.example.test/build/1)**")); + assert!(event.content.contains("**42 tests** passed")); + assert!(event.content.contains("**Commit:** `abc123`")); + } + + #[test] + fn rich_text_blocks_render_lists_mentions_links_and_styles() { + let blocks: Vec = serde_json::from_str( + r#"[{"type":"rich_text","elements":[ + {"type":"rich_text_section","elements":[ + {"type":"text","text":"Hello ","style":{"bold":true}}, + {"type":"user","user_id":"U1"}, + {"type":"text","text":" — "}, + {"type":"link","url":"https://example.test","text":"details"} + ]}, + {"type":"rich_text_list","style":"bullet","elements":[ + {"type":"rich_text_section","elements":[{"type":"text","text":"first"}]}, + {"type":"rich_text_section","elements":[{"type":"text","text":"second"}]} + ]} + ]}]"#, + ) + .expect("blocks"); + let names = HashMap::from([("U1".to_string(), "Alice".to_string())]); + let rendered = render_slack_blocks(&blocks, &names); + assert!(rendered.contains("**Hello **@Alice")); + assert!(rendered.contains("[details](https://example.test)")); + assert!(rendered.contains("- first")); + assert!(rendered.contains("- second")); + + let attachment: export::SlackAttachment = serde_json::from_str( + r#"{"fallback":"fallback only","original_url":"https://example.test/source"}"#, + ) + .expect("attachment"); + assert_eq!( + render_attachment(&attachment, &HashMap::new()), + "fallback only\n[Slack attachment](https://example.test/source)" + ); + } + + #[tokio::test] + async fn dry_run_is_offline_and_reports_counts() { + let dir = std::env::temp_dir().join(format!("buzz-import-dryrun-{}", std::process::id())); + let general = dir.join("general"); + std::fs::create_dir_all(&general).expect("mkdir"); + std::fs::write( + dir.join("channels.json"), + r#"[{"id":"C1","name":"general"}]"#, + ) + .expect("write channels"); + std::fs::write(dir.join("users.json"), r#"[{"id":"U1","name":"alice"}]"#) + .expect("write users"); + std::fs::write( + general.join("2024-01-01.json"), + r#"[{"type":"message","user":"U1","text":"hello","ts":"100.0"}]"#, + ) + .expect("write day"); + + // Points at a port nothing listens on — dry run must never dial it. + let client = BuzzClient::new( + "http://127.0.0.1:1".to_string(), + Keys::generate(), + None, + None, + ) + .expect("client"); + cmd_import_slack( + &client, + ImportSlackParams { + export_dirs: vec![dir.display().to_string()], + team_id: "T1".into(), + state: None, + channel_map: None, + channels: None, + dry_run: true, + skip_reactions: false, + identity_map: None, + }, + ) + .await + .expect("dry run succeeds offline"); + + let export = SlackExport::load_many(std::slice::from_ref(&dir)).expect("export"); + assert!(select_channels(&export, Some("general,missing")).is_err()); + assert!(!dir.join("buzz-import-state.json").exists()); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn provenance_tags_shape() { + let tags = provenance_tags("U1", "alice", "1.000200").expect("tags build"); + let flat: Vec> = tags + .iter() + .map(|t| t.as_slice().iter().map(|s| s.to_string()).collect()) + .collect(); + assert_eq!( + flat, + vec![ + vec!["import".to_string(), "slack".to_string()], + vec![ + "import_author".to_string(), + "U1".to_string(), + "alice".to_string() + ], + vec!["import_ts".to_string(), "1.000200".to_string()], + ] + ); + } +} diff --git a/crates/buzz-cli/src/commands/import/channel_state.rs b/crates/buzz-cli/src/commands/import/channel_state.rs new file mode 100644 index 0000000000..ca179323e3 --- /dev/null +++ b/crates/buzz-cli/src/commands/import/channel_state.rs @@ -0,0 +1,74 @@ +//! Relay-backed channel-state verification for adopted Slack conversations. + +use uuid::Uuid; + +use crate::client::BuzzClient; +use crate::error::CliError; + +/// Read the relay-generated channel metadata after an unarchive request and +/// fail closed unless the channel is visibly active. +pub(super) async fn require_unarchived( + client: &BuzzClient, + channel_id: Uuid, + slack_conversation_id: &str, +) -> Result<(), CliError> { + let filter = serde_json::json!({ + "kinds": [39000], + "#d": [channel_id.to_string()], + "limit": 1, + }); + let response = client.query(&filter).await.map_err(|error| { + CliError::Other(format!( + "could not verify conversation {slack_conversation_id} after its unarchive request: \ + {error}" + )) + })?; + + match metadata_archived_state(&response, channel_id)? { + Some(false) => Ok(()), + Some(true) => Err(CliError::Other(format!( + "could not prepare conversation {slack_conversation_id} for history import: \ + Buzz channel {channel_id} remains archived after the unarchive request" + ))), + None => Err(CliError::Other(format!( + "could not verify conversation {slack_conversation_id} after its unarchive request: \ + relay returned no current metadata for Buzz channel {channel_id}" + ))), + } +} + +/// Return the archived state from the relay's current kind:39000 metadata. +/// +/// Buzz encodes an active channel by omitting the `archived=true` tag. +pub(super) fn metadata_archived_state( + response: &str, + channel_id: Uuid, +) -> Result, CliError> { + let events: Vec = serde_json::from_str(response).map_err(|error| { + CliError::Other(format!( + "could not parse channel metadata after unarchive request: {error}" + )) + })?; + let expected_id = channel_id.to_string(); + + Ok(events.iter().find_map(|event| { + let tags = event.get("tags")?.as_array()?; + let matches_channel = tags.iter().any(|tag| { + tag.as_array().is_some_and(|parts| { + parts.first().and_then(serde_json::Value::as_str) == Some("d") + && parts.get(1).and_then(serde_json::Value::as_str) + == Some(expected_id.as_str()) + }) + }); + if !matches_channel { + return None; + } + let archived = tags.iter().any(|tag| { + tag.as_array().is_some_and(|parts| { + parts.first().and_then(serde_json::Value::as_str) == Some("archived") + && parts.get(1).and_then(serde_json::Value::as_str) == Some("true") + }) + }); + Some(archived) + })) +} diff --git a/crates/buzz-cli/src/commands/import/dm.rs b/crates/buzz-cli/src/commands/import/dm.rs new file mode 100644 index 0000000000..bb82bba6bc --- /dev/null +++ b/crates/buzz-cli/src/commands/import/dm.rs @@ -0,0 +1,181 @@ +//! Native Buzz DM validation and event construction for Slack imports. + +use std::collections::HashMap; + +use nostr::{EventBuilder, Tag}; +use uuid::Uuid; + +use super::export::{SlackChannel, SlackConversationKind, SlackExport}; +use super::state::ImportState; +use crate::error::CliError; + +pub(super) fn dm_import_blockers( + export: &SlackExport, + selected: &[&SlackChannel], + state: &ImportState, + bindings: &HashMap, + importer_pubkey: &str, +) -> HashMap { + let mut blockers = HashMap::new(); + let mut participant_sets: HashMap> = HashMap::new(); + for conversation in selected.iter().copied().filter(|conversation| { + matches!( + conversation.kind, + SlackConversationKind::DirectMessage | SlackConversationKind::GroupDirectMessage + ) + }) { + let is_new = !state.channels.contains_key(&conversation.id); + match mapped_dm_participants(export, conversation, bindings, importer_pubkey) { + Ok(participants) => { + participant_sets + .entry(participants.join(":")) + .or_default() + .push((conversation.id.clone(), is_new)); + } + Err(error) if is_new => { + blockers.insert(conversation.id.clone(), error.to_string()); + } + Err(_) => {} + } + } + + for conversations in participant_sets.values() { + if conversations.len() < 2 || !conversations.iter().any(|(_, is_new)| *is_new) { + continue; + } + let mut ids: Vec<&str> = conversations.iter().map(|(id, _)| id.as_str()).collect(); + ids.sort_unstable(); + let reason = format!( + "Slack conversations {} map to the same Buzz DM participant set; Buzz has one \ + native DM per participant set, so importing them would merge separate histories", + ids.join(", ") + ); + for (id, is_new) in conversations { + if *is_new { + blockers.insert(id.clone(), reason.clone()); + } + } + } + blockers +} + +pub(super) fn response_payload(response: &str) -> Option { + let parsed: serde_json::Value = serde_json::from_str(response).ok()?; + let message = parsed.get("message")?.as_str()?; + serde_json::from_str(message.strip_prefix("response:")?).ok() +} + +pub(super) fn mapped_dm_participants( + export: &SlackExport, + channel: &SlackChannel, + bindings: &HashMap, + importer_pubkey: &str, +) -> Result, CliError> { + let mappable_ids: Vec<&str> = channel + .members + .iter() + .map(String::as_str) + .filter(|id| export.is_mappable_member(id)) + .collect(); + let missing = mappable_ids + .iter() + .filter(|id| !bindings.contains_key(**id)) + .count(); + if missing > 0 { + return Err(CliError::Usage(format!( + "Slack {} {} has {missing} unmapped active participant(s); provide every mapping \ + with --identity-map before importing DM history", + channel.kind.as_str(), + channel.id + ))); + } + + let participant_pubkeys: std::collections::BTreeSet = mappable_ids + .into_iter() + .filter_map(|id| bindings.get(id).cloned()) + .collect(); + if !participant_pubkeys.contains(importer_pubkey) { + return Err(CliError::Usage(format!( + "Slack {} {} does not map any participant to the importer public key; refusing \ + to add the migration operator as an extra DM member", + channel.kind.as_str(), + channel.id + ))); + } + if participant_pubkeys.len() < 2 { + return Err(CliError::Usage(format!( + "Slack {} {} has fewer than two mappable human participants", + channel.kind.as_str(), + channel.id + ))); + } + if participant_pubkeys.len() > 9 { + return Err(CliError::Usage(format!( + "Slack {} {} has {} mapped participants; Buzz DMs support at most 9", + channel.kind.as_str(), + channel.id, + participant_pubkeys.len() + ))); + } + Ok(participant_pubkeys.into_iter().collect()) +} + +pub(super) fn build_import_dm_open( + channel: &SlackChannel, + team_id: &str, + participant_pubkeys: &[String], + importer_pubkey: &str, +) -> Result { + let participant_refs: Vec<&str> = participant_pubkeys + .iter() + .filter(|pubkey| pubkey.as_str() != importer_pubkey) + .map(String::as_str) + .collect(); + let mut tags = Vec::new(); + let import_id = format!("slack:{team_id}:{}", channel.id); + tags.push( + Tag::parse(["d", import_id.as_str()]) + .map_err(|e| CliError::Other(format!("invalid DM import d-tag: {e}")))?, + ); + tags.push( + Tag::parse(["import", "slack"]) + .map_err(|e| CliError::Other(format!("invalid import tag: {e}")))?, + ); + let foreign_id = format!("{team_id}:{}", channel.id); + tags.push( + Tag::parse([ + "import_conversation", + foreign_id.as_str(), + channel.kind.as_str(), + ]) + .map_err(|e| CliError::Other(format!("invalid conversation provenance tag: {e}")))?, + ); + buzz_sdk::build_dm_open(&participant_refs) + .map_err(|e| CliError::Other(format!("build_dm_open failed: {e}"))) + .map(|builder| builder.tags(tags)) +} + +/// Reject a relay-assigned native DM UUID that is already owned by another +/// Slack conversation in the resume ledger. Buzz intentionally reuses a DM +/// for the same participant set, whereas Slack exports may contain multiple +/// distinct D/MPIM ids for that set; merging their histories is never safe. +pub(super) fn ensure_dm_uuid_unclaimed( + state: &ImportState, + channel: &SlackChannel, + uuid: Uuid, +) -> Result<(), CliError> { + let uuid = uuid.to_string(); + if let Some((other, _)) = state + .channels + .iter() + .find(|(id, saved)| id.as_str() != channel.id && saved.uuid == uuid) + { + return Err(CliError::Usage(format!( + "Slack {} {} resolves to the Buzz DM already imported for {other}; \ + refusing to merge separate histories", + channel.kind.as_str(), + channel.id + ))); + } + Ok(()) +} diff --git a/crates/buzz-cli/src/commands/import/export.rs b/crates/buzz-cli/src/commands/import/export.rs new file mode 100644 index 0000000000..015d4d0421 --- /dev/null +++ b/crates/buzz-cli/src/commands/import/export.rs @@ -0,0 +1,947 @@ +//! Slack workspace export parsing. +//! +//! A Slack import accepts one or more directories that collectively contain +//! `users.json` (or the Enterprise `org_users.json`), optional `channels.json`, +//! `groups.json`, `dms.json`, and `mpims.json`, plus one subdirectory per +//! conversation holding `YYYY-MM-DD.json` message arrays. +//! +//! Slackdump deliberately follows that native layout. It may emit public and +//! private conversations into separate export roots, so [`SlackExport`] can +//! merge multiple roots while retaining one workspace-wide conversation index. + +use std::collections::HashMap; +use std::path::{Component, Path, PathBuf}; + +pub(super) use super::model::{ + SlackAttachment, SlackChannel, SlackConversationKind, SlackMessage, SlackUser, +}; +use crate::error::CliError; + +/// A loaded Slack export: user/channel indexes plus the directory root for +/// lazy per-channel message reads. +#[derive(Debug)] +pub struct SlackExport { + /// Users indexed by Slack user ID. + pub users: HashMap, + /// Conversations in source-file order, across all export roots. + pub channels: Vec, + roots: Vec, +} + +/// Per-conversation parsing/audit totals. Dry-run output aggregates these so +/// intentionally skipped Slack records are visible rather than disappearing +/// behind the importable-message count. +#[derive(Debug, Default)] +pub struct SlackMessageScan { + /// Importable messages, deduplicated by Slack timestamp and sorted. + pub messages: Vec, + /// All JSON records encountered in daily files. + pub records_seen: u64, + /// Non-message records. + pub non_message_records: u64, + /// Membership/channel lifecycle events that Buzz materializes itself. + pub system_records: u64, + /// Edit/delete/tombstone records retained by Slack compliance exports. + pub mutation_records: u64, + /// Message-shaped records with no renderable text, attachment, file, or + /// block content. + pub contentless_records: u64, + /// Repeated importable Slack timestamps collapsed deterministically. + pub duplicate_records: u64, +} + +impl SlackExport { + /// Merge one or more Slack/Slackdump export roots. + /// + /// Exact duplicate metadata is tolerated so independently produced public + /// and private exports can share user indexes. A reused Slack conversation + /// ID with conflicting metadata, or two different IDs with the same target + /// name, is rejected before any relay write. + pub fn load_many(dirs: &[PathBuf]) -> Result { + if dirs.is_empty() { + return Err(CliError::Usage( + "at least one --export-dir is required".into(), + )); + } + + let mut users: HashMap = HashMap::new(); + let mut channels: Vec = Vec::new(); + let mut channel_indexes: HashMap = HashMap::new(); + let mut channel_names: HashMap = HashMap::new(); + let mut roots = Vec::new(); + let mut saw_users_file = false; + + // Load the workspace-wide user index before normalizing any DMs. A + // multi-root export may carry users.json in only one root, and the + // root order must not change synthetic DM names or duplicate checks. + for dir in dirs { + if !dir.is_dir() { + return Err(CliError::Usage(format!( + "--export-dir is not a directory: {}", + dir.display() + ))); + } + roots.push(dir.clone()); + + let users_path = if dir.join("users.json").is_file() { + Some(dir.join("users.json")) + } else if dir.join("org_users.json").is_file() { + Some(dir.join("org_users.json")) + } else { + None + }; + if let Some(users_path) = users_path { + saw_users_file = true; + for user in read_json::>(&users_path)? { + users.entry(user.id.clone()).or_insert(user); + } + } + } + + if !saw_users_file { + return Err(CliError::Usage(format!( + "none of the export directories contains users.json or org_users.json: {}", + dirs.iter() + .map(|dir| dir.display().to_string()) + .collect::>() + .join(", ") + ))); + } + + for dir in dirs { + let mut records = Vec::new(); + records.extend( + read_optional_json::>(&dir.join("channels.json"))? + .into_iter() + .map(|channel| normalize_channel(channel, None, &users)) + .collect::, _>>()?, + ); + records.extend( + read_optional_json::>(&dir.join("groups.json"))? + .into_iter() + .map(|channel| { + normalize_channel( + channel, + Some(SlackConversationKind::PrivateChannel), + &users, + ) + }) + .collect::, _>>()?, + ); + records.extend( + read_optional_json::>(&dir.join("dms.json"))? + .into_iter() + .map(|channel| { + normalize_channel( + channel, + Some(SlackConversationKind::DirectMessage), + &users, + ) + }) + .collect::, _>>()?, + ); + records.extend( + read_optional_json::>(&dir.join("mpims.json"))? + .into_iter() + .map(|channel| { + normalize_channel( + channel, + Some(SlackConversationKind::GroupDirectMessage), + &users, + ) + }) + .collect::, _>>()?, + ); + + for record in records { + if let Some(index) = channel_indexes.get(&record.id).copied() { + merge_duplicate_conversation(&mut channels[index], &record)?; + continue; + } + if let Some(existing_id) = channel_names.get(&record.name) { + return Err(CliError::Usage(format!( + "Slack export maps conversation ids {existing_id} and {} to the same \ + Buzz channel name {:?}; rename or filter one before importing", + record.id, record.name + ))); + } + channel_names.insert(record.name.clone(), record.id.clone()); + channel_indexes.insert(record.id.clone(), channels.len()); + channels.push(record); + } + } + + if channels.is_empty() { + return Err(CliError::Usage( + "Slack export contains no conversations".into(), + )); + } + + Ok(Self { + users, + channels, + roots, + }) + } + + /// Read every day file for a channel, keeping only importable messages, + /// deduplicated by `ts` and sorted chronologically. + /// + /// Returns `Ok(vec![])` with no error if the channel directory is + /// missing (an empty channel exports no directory). + pub fn channel_messages(&self, channel: &SlackChannel) -> Result, CliError> { + Ok(self.channel_message_scan(channel)?.messages) + } + + /// Read and classify every daily record for one conversation. + pub fn channel_message_scan( + &self, + channel: &SlackChannel, + ) -> Result { + let mut day_files = Vec::new(); + for root in &self.roots { + let dir = root.join(&channel.export_directory); + if !dir.is_dir() { + continue; + } + let entries = std::fs::read_dir(&dir) + .map_err(|e| CliError::Other(format!("cannot read {}: {e}", dir.display())))?; + for entry in entries { + let path = entry + .map_err(|e| { + CliError::Other(format!("cannot read an entry in {}: {e}", dir.display())) + })? + .path(); + if path.extension().is_some_and(|ext| ext == "json") { + day_files.push(path); + } + } + } + day_files.sort(); + + let mut scan = SlackMessageScan::default(); + let mut by_ts: HashMap = HashMap::new(); + for file in &day_files { + let messages: Vec = read_json(file)?; + for msg in messages { + scan.records_seen += 1; + match message_disposition(&msg) { + SlackMessageDisposition::NonMessage => { + scan.non_message_records += 1; + continue; + } + SlackMessageDisposition::System => { + scan.system_records += 1; + continue; + } + SlackMessageDisposition::Mutation => { + scan.mutation_records += 1; + continue; + } + SlackMessageDisposition::Contentless => { + scan.contentless_records += 1; + continue; + } + SlackMessageDisposition::Import => {} + } + let timestamp = parse_slack_ts(&msg.ts).ok_or_else(|| { + CliError::Usage(format!( + "malformed Slack ts {:?} in {}", + msg.ts, + file.display() + )) + })?; + match by_ts.entry(msg.ts.clone()) { + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert((timestamp, msg)); + } + std::collections::hash_map::Entry::Occupied(entry) => { + if entry.get().1 != msg { + return Err(CliError::Usage(format!( + "Slack timestamp {} has conflicting message records across \ + export roots (latest conflict in {})", + msg.ts, + file.display() + ))); + } + scan.duplicate_records += 1; + } + } + } + } + let mut messages: Vec<(SlackTimestamp, SlackMessage)> = by_ts.into_values().collect(); + messages.sort_by_key(|(timestamp, _)| *timestamp); + scan.messages = messages.into_iter().map(|(_, message)| message).collect(); + Ok(scan) + } + + /// Whether a Slack member can meaningfully map to a Buzz human account. + /// Deactivated users and bot identities still retain message attribution, + /// but are not treated as missing private-channel membership mappings. + pub fn is_mappable_member(&self, slack_id: &str) -> bool { + self.users + .get(slack_id) + .is_none_or(|user| !user.is_bot && !user.deleted) + && slack_id != "USLACKBOT" + } +} + +fn normalize_channel( + mut channel: SlackChannel, + source_kind: Option, + users: &HashMap, +) -> Result { + if channel.id.trim().is_empty() { + return Err(CliError::Usage( + "Slack conversation metadata contains an empty id".into(), + )); + } + channel.members.sort(); + channel.members.dedup(); + channel.kind = source_kind.unwrap_or({ + if channel.is_im { + SlackConversationKind::DirectMessage + } else if channel.is_mpim { + SlackConversationKind::GroupDirectMessage + } else if channel.is_private { + SlackConversationKind::PrivateChannel + } else { + SlackConversationKind::PublicChannel + } + }); + + channel.export_directory = match channel.kind { + SlackConversationKind::DirectMessage => channel.id.clone(), + _ if !channel.name.trim().is_empty() => channel.name.trim().to_string(), + SlackConversationKind::GroupDirectMessage => channel.id.clone(), + SlackConversationKind::PublicChannel | SlackConversationKind::PrivateChannel => { + return Err(CliError::Usage(format!( + "Slack channel {} is missing its name", + channel.id + ))) + } + }; + validate_export_directory(&channel.export_directory)?; + + if channel.kind == SlackConversationKind::DirectMessage { + channel.name = direct_message_channel_name(&channel, users); + if channel.purpose.value.is_empty() { + channel.purpose.value = direct_message_purpose(&channel, users); + } + } else if channel.name.trim().is_empty() { + channel.name = format!("mpdm-{}", channel.id.to_ascii_lowercase().replace('_', "-")); + } else { + channel.name = channel.name.trim().to_string(); + } + Ok(channel) +} + +fn validate_export_directory(value: &str) -> Result<(), CliError> { + let path = Path::new(value); + let mut components = path.components(); + if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() { + return Err(CliError::Usage(format!( + "unsafe Slack export conversation directory {value:?}" + ))); + } + Ok(()) +} + +fn direct_message_channel_name( + channel: &SlackChannel, + users: &HashMap, +) -> String { + let mut labels: Vec = channel + .members + .iter() + .map(|id| { + users + .get(id) + .map(|user| { + if user.name.is_empty() { + user.best_name() + } else { + &user.name + } + }) + .unwrap_or(id) + }) + .map(slug_component) + .filter(|label| !label.is_empty()) + .collect(); + labels.sort(); + labels.dedup(); + let participants = if labels.is_empty() { + "unknown".to_string() + } else { + labels.join("-") + }; + format!( + "dm-{participants}-{}", + channel.id.to_ascii_lowercase().replace('_', "-") + ) +} + +fn direct_message_purpose(channel: &SlackChannel, users: &HashMap) -> String { + let labels: Vec<&str> = channel + .members + .iter() + .map(|id| users.get(id).map(SlackUser::best_name).unwrap_or(id)) + .collect(); + if labels.is_empty() { + "Imported Slack direct message".into() + } else { + format!( + "Imported Slack direct message between {}", + labels.join(", ") + ) + } +} + +fn slug_component(value: &str) -> String { + let mut out = String::new(); + let mut previous_separator = false; + for ch in value.chars() { + if ch.is_alphanumeric() { + out.extend(ch.to_lowercase()); + previous_separator = false; + } else if !previous_separator { + out.push('-'); + previous_separator = true; + } + } + out.trim_matches('-').to_string() +} + +fn merge_duplicate_conversation( + existing: &mut SlackChannel, + duplicate: &SlackChannel, +) -> Result<(), CliError> { + if existing.name != duplicate.name + || existing.kind != duplicate.kind + || existing.export_directory != duplicate.export_directory + { + return Err(CliError::Usage(format!( + "Slack conversation id {} has conflicting metadata across export directories", + existing.id + ))); + } + existing.is_archived |= duplicate.is_archived; + existing.members.extend(duplicate.members.iter().cloned()); + existing.members.sort(); + existing.members.dedup(); + if existing.topic.value.is_empty() { + existing.topic = duplicate.topic.clone(); + } + if existing.purpose.value.is_empty() { + existing.purpose = duplicate.purpose.clone(); + } + Ok(()) +} + +/// Whether a message carries content worth importing. System messages +/// (joins, renames, topic changes, ...) are excluded; the relay materializes +/// its own membership history. +#[cfg(test)] +pub fn is_importable(msg: &SlackMessage) -> bool { + message_disposition(msg) == SlackMessageDisposition::Import +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SlackMessageDisposition { + Import, + NonMessage, + System, + Mutation, + Contentless, +} + +fn message_disposition(msg: &SlackMessage) -> SlackMessageDisposition { + if msg.msg_type != "message" { + return SlackMessageDisposition::NonMessage; + } + if matches!( + msg.subtype.as_deref(), + Some("message_changed" | "message_deleted" | "tombstone") + ) { + return SlackMessageDisposition::Mutation; + } + if matches!( + msg.subtype.as_deref(), + Some( + "channel_join" + | "channel_leave" + | "channel_topic" + | "channel_purpose" + | "channel_name" + | "channel_archive" + | "channel_unarchive" + | "channel_convert_to_private" + | "group_join" + | "group_leave" + | "group_topic" + | "group_purpose" + | "group_name" + | "group_archive" + | "group_unarchive" + | "bot_add" + | "bot_remove" + | "bot_enable" + | "bot_disable" + | "app_conversation_join" + | "pinned_item" + | "unpinned_item" + | "reminder_add" + | "retention_threshold" + | "tabbed_canvas_updated" + | "channel_tab_added" + | "channel_canvas_updated" + | "automatic_ai_huddle_notes_enabled" + ) + ) { + return SlackMessageDisposition::System; + } + if msg.text.trim().is_empty() + && msg.files.is_empty() + && !msg.attachments.iter().any(SlackAttachment::has_content) + && msg.blocks.is_empty() + { + return SlackMessageDisposition::Contentless; + } + SlackMessageDisposition::Import +} + +/// Whole-second part of a Slack `ts` — becomes the Nostr `created_at`. +pub fn ts_seconds(ts: &str) -> Result { + parse_slack_ts(ts) + .map(|timestamp| timestamp.seconds) + .ok_or_else(|| CliError::Other(format!("malformed Slack ts: {ts:?}"))) +} + +/// Exact, microsecond-resolution Slack timestamp used for chronological sort. +/// +/// Slack exports normally use six fractional digits. Shorter fractions are +/// accepted for compatibility with hand-written fixtures and normalized by +/// right-padding; malformed and over-precise values are rejected. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct SlackTimestamp { + seconds: u64, + micros: u32, +} + +fn parse_slack_ts(ts: &str) -> Option { + let (seconds, fraction) = match ts.split_once('.') { + Some((seconds, fraction)) => (seconds, Some(fraction)), + None => (ts, None), + }; + if seconds.is_empty() || !seconds.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let seconds = seconds.parse().ok()?; + let micros = match fraction { + None => 0, + Some(fraction) + if !fraction.is_empty() + && fraction.len() <= 6 + && fraction.bytes().all(|b| b.is_ascii_digit()) => + { + let value: u32 = fraction.parse().ok()?; + value * 10_u32.pow(6 - fraction.len() as u32) + } + Some(_) => return None, + }; + Some(SlackTimestamp { seconds, micros }) +} + +fn read_json(path: &Path) -> Result { + let raw = std::fs::read_to_string(path) + .map_err(|e| CliError::Usage(format!("cannot read {}: {e}", path.display())))?; + serde_json::from_str(&raw) + .map_err(|e| CliError::Usage(format!("cannot parse {}: {e}", path.display()))) +} + +fn read_optional_json(path: &Path) -> Result +where + T: serde::de::DeserializeOwned + Default, +{ + match std::fs::read_to_string(path) { + Ok(raw) => serde_json::from_str(&raw) + .map_err(|e| CliError::Usage(format!("cannot parse {}: {e}", path.display()))), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(T::default()), + Err(e) => Err(CliError::Usage(format!( + "cannot read {}: {e}", + path.display() + ))), + } +} + +#[cfg(test)] +mod tests { + use super::super::model::SlackFile; + use super::*; + + fn msg(json: &str) -> SlackMessage { + serde_json::from_str(json).expect("test message parses") + } + + #[test] + fn importable_filters_system_subtypes() { + assert!(is_importable(&msg( + r#"{"type":"message","user":"U1","text":"hi","ts":"1.000"}"# + ))); + assert!(is_importable(&msg( + r#"{"type":"message","subtype":"thread_broadcast","user":"U1","text":"hi","ts":"1.000"}"# + ))); + assert!(is_importable(&msg( + r#"{"type":"message","subtype":"bot_message","bot_id":"B1","text":"hi","ts":"1.000"}"# + ))); + assert!(!is_importable(&msg( + r#"{"type":"message","subtype":"channel_join","user":"U1","text":"<@U1> joined","ts":"1.000"}"# + ))); + // Empty text with no files carries nothing to import. + assert!(!is_importable(&msg( + r#"{"type":"message","user":"U1","text":"","ts":"1.000"}"# + ))); + // Empty text but a file attachment is importable. + assert!(is_importable(&msg( + r#"{"type":"message","user":"U1","text":"","ts":"1.000","files":[{"name":"a.png"}]}"# + ))); + // Classic attachment-only integration posts retain their visible + // fallback/text instead of disappearing. + assert!(is_importable(&msg( + r#"{"type":"message","subtype":"bot_message","bot_id":"B1","text":"","ts":"1.000", + "attachments":[{"fallback":"build passed","title":"CI"}]}"# + ))); + // Unknown future subtypes are accepted when they carry renderable + // content; known membership/system events remain excluded. + assert!(is_importable(&msg( + r#"{"type":"message","subtype":"future_app_card","text":"useful","ts":"1.000"}"# + ))); + // A contentless bot_message thread root (null user, no text, no files) + // is filtered out — this is the real-export shape whose replies the + // importer must promote to top-level rather than defer forever. + assert!(!is_importable(&msg( + r#"{"type":"message","subtype":"bot_message","bot_id":"B1","user":null,"text":"","ts":"1.000","thread_ts":"1.000"}"# + ))); + assert!(!is_importable(&msg( + r#"{"type":"message","subtype":"channel_join","user":"U1","text":"joined","ts":"1.000", + "attachments":[{"fallback":"joined"}]}"# + ))); + } + + #[test] + fn ts_seconds_parses_whole_part() { + assert_eq!(ts_seconds("1610000000.000200").expect("parses"), 1610000000); + assert_eq!(ts_seconds("1610000000").expect("parses"), 1610000000); + assert!(ts_seconds("not-a-ts").is_err()); + assert!(ts_seconds("1610000000.bad").is_err()); + assert!(ts_seconds("1610000000.").is_err()); + assert!(ts_seconds("1610000000.0000001").is_err()); + } + + #[test] + fn slack_timestamps_sort_exactly() { + let mut timestamps = [ + parse_slack_ts("1700000000.000010").expect("valid"), + parse_slack_ts("1700000000.000002").expect("valid"), + parse_slack_ts("1699999999.999999").expect("valid"), + ]; + timestamps.sort(); + assert_eq!(timestamps[0].seconds, 1_699_999_999); + assert_eq!(timestamps[1].micros, 2); + assert_eq!(timestamps[2].micros, 10); + } + + #[test] + fn user_best_name_falls_back() { + let user: SlackUser = serde_json::from_str( + r#"{"id":"U1","name":"alice","profile":{"display_name":"","real_name":"Alice A"}}"#, + ) + .expect("parses"); + assert_eq!(user.best_name(), "Alice A"); + let bare: SlackUser = serde_json::from_str(r#"{"id":"U2"}"#).expect("parses"); + assert_eq!(bare.best_name(), "U2"); + } + + #[test] + fn loads_fixture_export_directory() { + let dir = std::env::temp_dir().join(format!("buzz-slack-export-{}", std::process::id())); + let general = dir.join("general"); + std::fs::create_dir_all(&general).expect("mkdir"); + std::fs::write( + dir.join("channels.json"), + r#"[{"id":"C1","name":"general","is_archived":false, + "topic":{"value":"the topic"},"purpose":{"value":"the purpose"}}, + {"id":"C2","name":"empty"}]"#, + ) + .expect("write channels"); + std::fs::write( + dir.join("users.json"), + r#"[{"id":"U1","name":"alice","profile":{"display_name":"Alice"}}]"#, + ) + .expect("write users"); + // Two day files, out of order on disk, with a system message, a + // duplicate ts, and a threaded reply. + std::fs::write( + general.join("2024-01-02.json"), + r#"[{"type":"message","user":"U1","text":"reply","ts":"200.000100","thread_ts":"100.000100"}, + {"type":"message","subtype":"channel_join","user":"U1","text":"joined","ts":"150.0"}]"#, + ) + .expect("write day 2"); + std::fs::write( + general.join("2024-01-01.json"), + r#"[{"type":"message","user":"U1","text":"root","ts":"100.000100","thread_ts":"100.000100", + "reactions":[{"name":"+1","users":["U1"]}]}, + {"type":"message","user":"U1","text":"root","ts":"100.000100","thread_ts":"100.000100", + "reactions":[{"name":"+1","users":["U1"]}]}]"#, + ) + .expect("write day 1"); + + let export = SlackExport::load_many(std::slice::from_ref(&dir)).expect("load"); + assert_eq!(export.channels.len(), 2); + assert_eq!(export.users["U1"].best_name(), "Alice"); + + let scan = export + .channel_message_scan(&export.channels[0]) + .expect("messages"); + assert_eq!(scan.records_seen, 4); + assert_eq!(scan.system_records, 1); + assert_eq!(scan.duplicate_records, 1); + assert_eq!(scan.contentless_records, 0); + let messages = scan.messages; + // join filtered, duplicate ts collapsed, sorted oldest-first + assert_eq!(messages.len(), 2); + assert_eq!(messages[0].ts, "100.000100"); + assert_eq!(messages[1].ts, "200.000100"); + assert_eq!(messages[0].reactions.len(), 1); + + // Missing channel directory is an empty channel, not an error. + let empty = export + .channel_messages(&export.channels[1]) + .expect("empty ok"); + assert!(empty.is_empty()); + + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn file_label_and_link() { + let f: SlackFile = + serde_json::from_str(r#"{"name":"a.png","permalink":"https://x/p"}"#).expect("parses"); + assert_eq!(f.label(), "a.png"); + assert_eq!(f.link(), Some("https://x/p")); + let empty: SlackFile = serde_json::from_str(r#"{}"#).expect("parses"); + assert_eq!(empty.label(), "attachment"); + assert_eq!(empty.link(), None); + } + + #[test] + fn rejects_conflicting_duplicate_message_timestamps() { + let dir = std::env::temp_dir().join(format!( + "buzz-slack-message-conflict-{}", + std::process::id() + )); + let general = dir.join("general"); + std::fs::create_dir_all(&general).expect("mkdir"); + std::fs::write(dir.join("users.json"), "[]").expect("users"); + std::fs::write( + dir.join("channels.json"), + r#"[{"id":"C1","name":"general"}]"#, + ) + .expect("channels"); + std::fs::write( + general.join("2024-01-01.json"), + r#"[{"type":"message","user":"U1","text":"first","ts":"100.0"}]"#, + ) + .expect("day one"); + std::fs::write( + general.join("2024-01-02.json"), + r#"[{"type":"message","user":"U1","text":"different","ts":"100.0"}]"#, + ) + .expect("day two"); + + let export = SlackExport::load_many(std::slice::from_ref(&dir)).expect("export"); + let error = export + .channel_messages(&export.channels[0]) + .expect_err("conflicting duplicate must fail"); + assert!(error.to_string().contains("conflicting message records")); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn merges_slackdump_public_private_dm_and_mpim_roots() { + let base = + std::env::temp_dir().join(format!("buzz-slackdump-roots-{}", std::process::id())); + let public = base.join("public"); + let private = base.join("private"); + std::fs::create_dir_all(public.join("general")).expect("public dirs"); + std::fs::create_dir_all(private.join("secret")).expect("private dirs"); + std::fs::create_dir_all(private.join("group-secret")).expect("group dirs"); + std::fs::create_dir_all(private.join("D1")).expect("dm dirs"); + std::fs::create_dir_all(private.join("mpdm-alice--bob--carol-1")).expect("mpim dirs"); + + std::fs::write( + public.join("users.json"), + r#"[ + {"id":"U1","name":"alice","profile":{"display_name":"Alice"}}, + {"id":"U2","name":"bob"}, + {"id":"U3","name":"carol"} + ]"#, + ) + .expect("users"); + std::fs::write( + public.join("channels.json"), + r#"[{"id":"C1","name":"general","members":["U1","U2"]}]"#, + ) + .expect("public channels"); + std::fs::write( + private.join("channels.json"), + r#"[{"id":"C2","name":"secret","is_private":true, + "is_archived":true,"members":["U1","U2"]}]"#, + ) + .expect("private channels"); + std::fs::write( + private.join("groups.json"), + r#"[{"id":"G1","name":"group-secret","members":["U2","U3"]}]"#, + ) + .expect("groups"); + std::fs::write( + private.join("dms.json"), + r#"[{"id":"D1","members":["U1","U2"]}]"#, + ) + .expect("dms"); + std::fs::write( + private.join("mpims.json"), + r#"[{"id":"M1","name":"mpdm-alice--bob--carol-1", + "members":["U1","U2","U3"]}]"#, + ) + .expect("mpims"); + std::fs::write( + private.join("D1").join("2024-01-01.json"), + r#"[{"type":"message","user":"U1","text":"private","ts":"100.0"}]"#, + ) + .expect("dm messages"); + + // The user index exists only in the second root; normalization must be + // root-order independent so the DM still gets stable human labels. + let export = + SlackExport::load_many(&[private.clone(), public.clone()]).expect("merged export"); + assert_eq!(export.channels.len(), 5); + assert_eq!( + export + .channels + .iter() + .filter(|channel| channel.kind == SlackConversationKind::PublicChannel) + .count(), + 1 + ); + assert_eq!( + export + .channels + .iter() + .filter(|channel| channel.kind == SlackConversationKind::PrivateChannel) + .count(), + 2 + ); + assert_eq!( + export + .channels + .iter() + .filter(|channel| channel.kind == SlackConversationKind::DirectMessage) + .count(), + 1 + ); + assert_eq!( + export + .channels + .iter() + .filter(|channel| channel.kind == SlackConversationKind::GroupDirectMessage) + .count(), + 1 + ); + let dm = export + .channels + .iter() + .find(|channel| channel.id == "D1") + .expect("dm"); + assert_eq!(dm.name, "dm-alice-bob-d1"); + assert_eq!(dm.kind.buzz_channel_kind(), buzz_sdk::ChannelKind::Dm); + assert!(dm.kind.is_private()); + assert_eq!(export.channel_messages(dm).expect("dm messages").len(), 1); + let archived = export + .channels + .iter() + .find(|channel| channel.id == "C2") + .expect("archived private"); + assert!(archived.is_archived); + assert!(archived.kind.is_private()); + + std::fs::remove_dir_all(&base).ok(); + } + + #[test] + fn split_private_root_may_omit_channels_json() { + let base = std::env::temp_dir().join(format!( + "buzz-slackdump-optional-channels-{}", + uuid::Uuid::new_v4() + )); + let public = base.join("public"); + let private = base.join("private"); + std::fs::create_dir_all(&public).expect("public dir"); + std::fs::create_dir_all(&private).expect("private dir"); + std::fs::write(public.join("users.json"), "[]").expect("users"); + std::fs::write( + public.join("channels.json"), + r#"[{"id":"C1","name":"general"}]"#, + ) + .expect("public channels"); + std::fs::write( + private.join("groups.json"), + r#"[{"id":"G1","name":"secret"}]"#, + ) + .expect("private groups"); + + let export = + SlackExport::load_many(&[private.clone(), public.clone()]).expect("merged export"); + assert_eq!(export.channels.len(), 2); + assert!(export.channels.iter().any(|channel| channel.id == "C1")); + assert!(export.channels.iter().any(|channel| channel.id == "G1")); + std::fs::remove_dir_all(&base).ok(); + } + + #[test] + fn rejects_conflicting_duplicate_conversation_names() { + let dir = + std::env::temp_dir().join(format!("buzz-slack-name-conflict-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("mkdir"); + std::fs::write(dir.join("users.json"), "[]").expect("users"); + std::fs::write( + dir.join("channels.json"), + r#"[{"id":"C1","name":"same"},{"id":"C2","name":"same"}]"#, + ) + .expect("channels"); + + let error = SlackExport::load_many(std::slice::from_ref(&dir)) + .expect_err("duplicate target names must fail"); + assert!(error.to_string().contains("same Buzz channel name")); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn rejects_unsafe_conversation_directory_names() { + let dir = + std::env::temp_dir().join(format!("buzz-slack-path-traversal-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("mkdir"); + std::fs::write(dir.join("users.json"), "[]").expect("users"); + std::fs::write( + dir.join("channels.json"), + r#"[{"id":"C1","name":"../outside"}]"#, + ) + .expect("channels"); + + let error = SlackExport::load_many(std::slice::from_ref(&dir)) + .expect_err("path traversal must fail"); + assert!(error.to_string().contains("unsafe Slack export")); + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/crates/buzz-cli/src/commands/import/importer.rs b/crates/buzz-cli/src/commands/import/importer.rs new file mode 100644 index 0000000000..94f5f123c2 --- /dev/null +++ b/crates/buzz-cli/src/commands/import/importer.rs @@ -0,0 +1,921 @@ +//! The `buzz import slack` execution engine: the stateful [`Importer`] run loop +//! plus the pure helpers that shape one imported event (message building, +//! provenance tags, emoji mapping, thread resolution) and the relay submit / +//! attestation calls. The command layer in the parent module parses inputs and +//! constructs an [`Importer`]; everything that actually writes to the relay +//! lives here. + +use std::collections::HashMap; +use std::path::PathBuf; + +use nostr::{EventBuilder, EventId, Timestamp}; +use uuid::Uuid; + +use super::channel_state::require_unarchived; +use super::dm::{ + build_import_dm_open, ensure_dm_uuid_unclaimed, mapped_dm_participants, response_payload, +}; +use super::export::{ts_seconds, SlackChannel, SlackExport, SlackMessage}; +use super::mrkdwn; +use super::print_json; +use super::render::{emoji_for_shortcode, render_attachment, render_slack_blocks}; +use super::state::{ChannelState, ImportState}; +use crate::client::BuzzClient; +use crate::error::CliError; + +/// Abort after this many consecutive message-submit failures — a wall of +/// failures means the relay is down or rejecting everything, not a handful +/// of individually bad messages. +const MAX_CONSECUTIVE_FAILURES: usize = 5; + +/// Persist the state ledger at most every N successful writes. +/// +/// The ledger is serialized whole on every save, so saving per message costs +/// O(n²) bytes over a large workspace. Batching is safe because every import +/// write is deterministic and idempotent: a crash loses at most the last N +/// ledger entries, and re-running rebuilds byte-identical events that the relay +/// reports as duplicates (which [`submit`] treats as success), repopulating the +/// ledger. Channel creation, archival, and end-of-channel state are still +/// flushed immediately. +const STATE_CHECKPOINT_WRITES: usize = 50; + +#[derive(Default)] +struct Summary { + channels_created: u64, + messages_imported: u64, + reactions_imported: u64, + bindings_published: u64, + private_members_added: u64, + private_members_unmapped: u64, + skipped: u64, + warnings: Vec, +} + +impl Summary { + fn warn(&mut self, msg: String) { + eprintln!("warning: {msg}"); + self.warnings.push(msg); + } +} + +/// A single `buzz import slack` run: borrowed export/index inputs plus the +/// mutable state ledger and summary threaded through every write. +pub(super) struct Importer<'a> { + client: &'a BuzzClient, + export: &'a SlackExport, + /// Slack user id → display name. + names: &'a HashMap, + /// Slack workspace id — namespaces identity bindings and channel UUIDs. + team_id: &'a str, + state: ImportState, + state_path: PathBuf, + summary: Summary, + skip_reactions: bool, + /// Slack user id → Buzz public key. Besides publishing identity + /// attestations, explicit mappings populate private conversation members. + bindings: &'a HashMap, + /// Writes recorded since the last state flush (see + /// [`STATE_CHECKPOINT_WRITES`]). + unsaved: usize, +} + +/// Owned and borrowed run settings grouped separately from the parsed export +/// indexes supplied to [`Importer::new`]. +pub(super) struct ImporterOptions<'a> { + pub(super) team_id: &'a str, + pub(super) state: ImportState, + pub(super) state_path: PathBuf, + pub(super) skip_reactions: bool, + pub(super) bindings: &'a HashMap, +} + +impl<'a> Importer<'a> { + /// Assemble a run from parsed inputs and a freshly loaded state ledger. + pub(super) fn new( + client: &'a BuzzClient, + export: &'a SlackExport, + names: &'a HashMap, + options: ImporterOptions<'a>, + ) -> Self { + Self { + client, + export, + names, + team_id: options.team_id, + state: options.state, + state_path: options.state_path, + summary: Summary::default(), + skip_reactions: options.skip_reactions, + bindings: options.bindings, + unsaved: 0, + } + } + + /// Persist the running state ledger. + fn save(&self) -> Result<(), CliError> { + self.state.save(&self.state_path) + } + + /// Persist immediately and clear the checkpoint counter. + fn flush(&mut self) -> Result<(), CliError> { + self.save()?; + self.unsaved = 0; + Ok(()) + } + + /// Record one completed write, persisting once a checkpoint's worth has + /// accumulated (see [`STATE_CHECKPOINT_WRITES`]). + fn checkpoint(&mut self) -> Result<(), CliError> { + self.unsaved += 1; + if self.unsaved >= STATE_CHECKPOINT_WRITES { + self.flush()?; + } + Ok(()) + } + + /// Create a channel once and independently resume its topic metadata. + async fn ensure_channel( + &mut self, + channel: &SlackChannel, + needs_writes: bool, + ) -> Result { + if matches!( + channel.kind, + super::export::SlackConversationKind::DirectMessage + | super::export::SlackConversationKind::GroupDirectMessage + ) { + return self.ensure_dm(channel).await; + } + + let (uuid, metadata_done) = match self.state.channels.get(&channel.id) { + Some(state) => ( + Uuid::parse_str(&state.uuid) + .map_err(|e| CliError::Other(format!("state file holds invalid UUID: {e}")))?, + state.metadata_done, + ), + None => { + let uuid = channel_uuid(self.team_id, &channel.id); + let about = if channel.purpose.value.is_empty() { + None + } else { + Some(channel.purpose.value.as_str()) + }; + let builder = buzz_sdk::build_create_channel( + uuid, + &channel.name, + Some(if channel.kind.is_private() { + buzz_sdk::Visibility::Private + } else { + buzz_sdk::Visibility::Open + }), + Some(channel.kind.buzz_channel_kind()), + about, + None, + ) + .map_err(|e| CliError::Other(format!("build_create_channel failed: {e}")))?; + submit(self.client, builder).await.map_err(|e| { + CliError::Other(format!("channel create failed for #{}: {e}", channel.name)) + })?; + + // Persist the UUID before the separate topic write. If that + // write or the process fails, a re-run resumes metadata on the + // same channel instead of creating a duplicate. + self.state.channels.insert( + channel.id.clone(), + ChannelState { + uuid: uuid.to_string(), + metadata_done: false, + archived_done: false, + private_visibility_done: channel.kind.is_private(), + prepared_for_import: true, + members_added: std::collections::HashSet::new(), + }, + ); + self.flush()?; + self.summary.channels_created += 1; + (uuid, false) + } + }; + + let (prepared_for_import, archived_done) = self + .state + .channels + .get(&channel.id) + .map(|state| (state.prepared_for_import, state.archived_done)) + .unwrap_or((true, false)); + if !prepared_for_import || (channel.is_archived && archived_done && needs_writes) { + // The crosswalk says which channel to adopt, but not whether that + // Buzz channel is currently archived. Attempt the transition on + // every first adoption, even when Slack says the source is active. + // Kind:9002 side effects are best-effort at the relay, so its + // accepted response cannot prove the channel is active. Read the + // relay-generated channel metadata back before any history write. + let builder = buzz_sdk::build_unarchive(uuid) + .map_err(|e| CliError::Other(format!("build_unarchive failed: {e}")))?; + submit(self.client, builder).await.map_err(|error| { + CliError::Other(format!( + "could not prepare conversation {} for history import: {error}", + channel.id + )) + })?; + require_unarchived(self.client, uuid, &channel.id).await?; + if let Some(state) = self.state.channels.get_mut(&channel.id) { + state.prepared_for_import = true; + state.archived_done = false; + } + self.flush()?; + } + + // Older importer ledgers could contain a Slackdump private record from + // `channels.json` even though the old create path always emitted + // `visibility=open`. Narrow it before publishing any history. + let private_visibility_done = self + .state + .channels + .get(&channel.id) + .is_some_and(|state| state.private_visibility_done); + if channel.kind.is_private() && !private_visibility_done { + let builder = + buzz_sdk::build_update_channel(uuid, None, None, Some("private"), None) + .map_err(|e| CliError::Other(format!("build_update_channel failed: {e}")))?; + submit(self.client, builder).await.map_err(|e| { + CliError::Other(format!( + "refusing to import private conversation {} because Buzz visibility \ + could not be narrowed: {e}", + channel.id + )) + })?; + if let Some(state) = self.state.channels.get_mut(&channel.id) { + state.private_visibility_done = true; + } + self.flush()?; + } + + if !metadata_done { + let topic_result = if channel.topic.value.is_empty() { + Ok(()) + } else { + match buzz_sdk::build_set_topic(uuid, &channel.topic.value) { + Ok(builder) => submit(self.client, builder).await.map(|_| ()), + Err(e) => Err(CliError::Other(format!("build_set_topic failed: {e}"))), + } + }; + match topic_result { + Ok(()) => { + if let Some(state) = self.state.channels.get_mut(&channel.id) { + state.metadata_done = true; + } + self.flush()?; + } + Err(e) => self + .summary + .warn(format!("topic set failed for #{}: {e}", channel.name)), + } + } + + Ok(uuid) + } + + /// Open a native Buzz DM using the exact mapped Slack participant set. + /// + /// The relay always includes the caller in a DM. Requiring the importer to + /// map to one Slack participant prevents an operator who merely has access + /// to an export from becoming an extra participant in every imported DM. + /// Requiring all active human participants up front avoids Buzz's immutable + /// DM participant-set rule splitting later history across two DMs. + async fn ensure_dm(&mut self, channel: &SlackChannel) -> Result { + if let Some(state) = self.state.channels.get(&channel.id) { + let uuid = Uuid::parse_str(&state.uuid) + .map_err(|e| CliError::Other(format!("state file holds invalid UUID: {e}")))?; + ensure_dm_uuid_unclaimed(&self.state, channel, uuid)?; + return Ok(uuid); + } + + let importer_pubkey = self.client.keys().public_key().to_hex(); + let participant_pubkeys = + mapped_dm_participants(self.export, channel, self.bindings, &importer_pubkey)?; + let builder = build_import_dm_open( + channel, + self.team_id, + &participant_pubkeys, + &importer_pubkey, + )?; + let (_, response) = submit_with_response(self.client, builder) + .await + .map_err(|e| { + CliError::Other(format!( + "DM open failed for Slack conversation {}: {e}", + channel.id + )) + })?; + let payload = response_payload(&response).ok_or_else(|| { + CliError::Other(format!( + "DM open response for Slack conversation {} did not contain a channel id", + channel.id + )) + })?; + let channel_id = payload + .get("channel_id") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + CliError::Other(format!( + "DM open response for Slack conversation {} did not contain a channel id", + channel.id + )) + })?; + let uuid = Uuid::parse_str(channel_id) + .map_err(|e| CliError::Other(format!("relay returned invalid DM UUID: {e}")))?; + ensure_dm_uuid_unclaimed(&self.state, channel, uuid)?; + let created = payload + .get("created") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + + self.state.channels.insert( + channel.id.clone(), + ChannelState { + uuid: uuid.to_string(), + metadata_done: true, + archived_done: true, + private_visibility_done: true, + prepared_for_import: true, + members_added: participant_pubkeys.into_iter().collect(), + }, + ); + self.flush()?; + if created { + self.summary.channels_created += 1; + } + Ok(uuid) + } + + /// Add every explicitly mapped Slack participant to a private Buzz + /// conversation. Unmapped people are counted, never substituted with the + /// importer or another participant, so private history cannot be widened + /// accidentally. + async fn ensure_private_members( + &mut self, + channel: &SlackChannel, + channel_uuid: Uuid, + ) -> Result<(), CliError> { + if !channel.kind.is_private() + || matches!( + channel.kind, + super::export::SlackConversationKind::DirectMessage + | super::export::SlackConversationKind::GroupDirectMessage + ) + { + return Ok(()); + } + + let importer_pubkey = self.client.keys().public_key().to_hex(); + let mut mapped_pubkeys = std::collections::BTreeSet::new(); + let mut unmapped = 0u64; + for slack_id in &channel.members { + if !self.export.is_mappable_member(slack_id) { + continue; + } + match self.bindings.get(slack_id) { + Some(pubkey) => { + mapped_pubkeys.insert(pubkey.clone()); + } + None => unmapped += 1, + } + } + self.summary.private_members_unmapped += unmapped; + + for pubkey in mapped_pubkeys { + let already_added = self + .state + .channels + .get(&channel.id) + .is_some_and(|state| state.members_added.contains(&pubkey)); + if already_added { + continue; + } + + // The channel creator is already its owner. Record an explicit + // mapping to that same key without publishing a redundant + // add-member event. + if pubkey != importer_pubkey { + let builder = buzz_sdk::build_add_member( + channel_uuid, + &pubkey, + Some(buzz_sdk::MemberRole::Member), + ) + .map_err(|e| CliError::Other(format!("build_add_member failed: {e}")))?; + if let Err(e) = submit(self.client, builder).await { + self.summary.warn(format!( + "mapped member add failed for private conversation {}: {e}", + channel.id + )); + continue; + } + } + if let Some(state) = self.state.channels.get_mut(&channel.id) { + state.members_added.insert(pubkey); + } + self.summary.private_members_added += 1; + self.checkpoint()?; + } + Ok(()) + } + + pub(super) async fn import_channel(&mut self, channel: &SlackChannel) -> Result<(), CliError> { + let export = self.export; + let names = self.names; + + let messages = export.channel_messages(channel)?; + eprintln!( + "importing {} {} ({} messages)", + channel.kind.as_str(), + channel.name, + messages.len() + ); + + let needs_writes = self.channel_needs_writes(channel, &messages); + let channel_uuid = self.ensure_channel(channel, needs_writes).await?; + self.ensure_private_members(channel, channel_uuid).await?; + + // Messages, oldest first; thread roots always precede replies. + // + // `channel_messages` already dropped every non-importable message and + // sorted the rest chronologically, so an importable thread root always + // appears before — and is imported before — its replies. Any root that + // is *not* in this set (an empty `bot_message` root, a system subtype) + // can never be imported, so a reply pointing at it must not be deferred + // forever; it is promoted to a top-level message instead. + let importable_ts: std::collections::HashSet<&str> = + messages.iter().map(|m| m.ts.as_str()).collect(); + let mut consecutive_failures = 0usize; + let mut imported_in_channel = 0u64; + for msg in &messages { + let key = ImportState::message_key(&channel.id, &msg.ts); + if self.state.messages.contains_key(&key) { + // Already imported — but a prior run may have stopped between + // the message and its reactions, so reactions still get their + // (state-deduped) pass below. + if !self.skip_reactions { + self.import_reactions(channel, msg, &key).await?; + } + continue; + } + + // Slack threads are flat: thread_ts is the root, every reply is a + // direct reply to it. Roots resolved through the state ledger. + let thread_ref = match thread_root_key(channel, msg) { + Some(root_key) => match self.state.messages.get(&root_key) { + Some(root_hex) => { + let root = EventId::from_hex(root_hex).map_err(|e| { + CliError::Other(format!("state file holds invalid event id: {e}")) + })?; + Some(buzz_sdk::ThreadRef { + root_event_id: root, + parent_event_id: root, + }) + } + None => { + // Root importable but not yet in state → an earlier run + // was interrupted between root and reply; re-running + // resumes. Root absent from the importable set → Slack + // filtered it (empty bot root / system subtype) and it + // will never import, so keep the reply's real content as + // a top-level message rather than deferring it forever. + let root_ts = msg.thread_ts.as_deref().unwrap_or_default(); + if importable_ts.contains(root_ts) { + self.summary.warn(format!( + "thread root {root_key} is not imported yet — deferring reply \ + {key}; re-run to resume" + )); + self.summary.skipped += 1; + continue; + } + self.summary.warn(format!( + "thread root {root_key} was filtered out of the import (empty or \ + system message); importing reply {key} as a top-level message" + )); + None + } + }, + None => None, + }; + + let builder = match build_imported_message( + channel, + channel_uuid, + msg, + names, + self.team_id, + thread_ref.as_ref(), + ) { + Ok(b) => b, + Err(e) => { + self.summary.warn(format!("skipping {key}: {e}")); + self.summary.skipped += 1; + continue; + } + }; + match submit(self.client, builder).await { + Ok(event_id) => { + consecutive_failures = 0; + self.state.messages.insert(key.clone(), event_id); + self.checkpoint()?; + self.summary.messages_imported += 1; + imported_in_channel += 1; + if imported_in_channel.is_multiple_of(50) { + eprintln!(" #{}: {imported_in_channel} imported", channel.name); + } + } + Err(e @ CliError::Auth(_)) => return Err(e), + Err(e) => { + consecutive_failures += 1; + self.summary.warn(format!("message {key} failed: {e}")); + self.summary.skipped += 1; + if consecutive_failures >= MAX_CONSECUTIVE_FAILURES { + self.flush()?; + return Err(CliError::Other(format!( + "{MAX_CONSECUTIVE_FAILURES} consecutive submit failures — aborting; \ + re-run to resume from the state file" + ))); + } + continue; + } + } + + if self.skip_reactions { + continue; + } + self.import_reactions(channel, msg, &key).await?; + } + + // Mirror Slack's archived flag once the channel's history is in. + let archive_done = self + .state + .channels + .get(&channel.id) + .is_some_and(|state| state.archived_done); + if channel.is_archived + && !matches!( + channel.kind, + super::export::SlackConversationKind::DirectMessage + | super::export::SlackConversationKind::GroupDirectMessage + ) + && !archive_done + { + let builder = buzz_sdk::build_archive(channel_uuid) + .map_err(|e| CliError::Other(format!("build_archive failed: {e}")))?; + match submit(self.client, builder).await { + Ok(_) => { + if let Some(state) = self.state.channels.get_mut(&channel.id) { + state.archived_done = true; + } + self.flush()?; + } + Err(e) => self + .summary + .warn(format!("archive failed for #{}: {e}", channel.name)), + } + } + self.flush()?; + Ok(()) + } + + /// Whether an archived stream must be reopened before this run can finish + /// its metadata, membership, message, or reaction writes. + fn channel_needs_writes(&self, channel: &SlackChannel, messages: &[SlackMessage]) -> bool { + let Some(state) = self.state.channels.get(&channel.id) else { + return true; + }; + if !state.prepared_for_import + || !state.metadata_done + || (channel.kind.is_private() && !state.private_visibility_done) + { + return true; + } + + if channel.kind == super::export::SlackConversationKind::PrivateChannel + && channel.members.iter().any(|slack_id| { + self.export.is_mappable_member(slack_id) + && self + .bindings + .get(slack_id) + .is_some_and(|pubkey| !state.members_added.contains(pubkey)) + }) + { + return true; + } + + messages.iter().any(|message| { + let message_key = ImportState::message_key(&channel.id, &message.ts); + !self.state.messages.contains_key(&message_key) + || (!self.skip_reactions + && message.reactions.iter().any(|reaction| { + let emoji = emoji_for_shortcode(&reaction.name); + !self + .state + .reactions + .contains(&format!("{message_key}:{emoji}")) + })) + }) + } + + /// Import reactions for one message. Bot mode signs with a single key, so + /// each distinct emoji becomes one bot-signed reaction (a key can react + /// only once per target); the count of reactors is not preserved. + async fn import_reactions( + &mut self, + channel: &SlackChannel, + msg: &SlackMessage, + message_key: &str, + ) -> Result<(), CliError> { + if msg.reactions.is_empty() { + return Ok(()); + } + let client = self.client; + + let Some(target_hex) = self.state.messages.get(message_key).cloned() else { + return Ok(()); + }; + let target = EventId::from_hex(&target_hex) + .map_err(|e| CliError::Other(format!("state file holds invalid event id: {e}")))?; + // Slack exports don't record reaction times; anchor just after the message. + let created_at = ts_seconds(&msg.ts)?.saturating_add(1); + + for reaction in &msg.reactions { + let emoji = emoji_for_shortcode(&reaction.name); + let dedupe = format!("{message_key}:{emoji}"); + if self.state.reactions.contains(&dedupe) { + continue; + } + let builder = match buzz_sdk::build_reaction(target, &emoji) { + Ok(b) => b, + Err(e) => { + self.summary.warn(format!( + "reaction :{}: on {message_key}: {e}", + reaction.name + )); + continue; + } + }; + let builder = builder + .custom_created_at(Timestamp::from(created_at)) + .tags(provenance_tags("slack", "slack", &msg.ts)?); + match submit(client, builder).await { + Ok(_) => { + self.state.reactions.insert(dedupe); + self.checkpoint()?; + self.summary.reactions_imported += 1; + } + Err(e) => self.summary.warn(format!( + "reaction :{}: on {message_key} in #{} failed: {e}", + reaction.name, channel.name + )), + } + } + Ok(()) + } + + /// Publish owner/admin-signed identity bindings (public keys only). + pub(super) async fn publish_bindings( + &mut self, + bindings: &[(String, String)], + ) -> Result<(), CliError> { + for (slack_id, pubkey_hex) in bindings { + match publish_binding(self.client, self.team_id, slack_id, pubkey_hex).await { + Ok(_) => self.summary.bindings_published += 1, + Err(e) => self + .summary + .warn(format!("identity binding for {slack_id} failed: {e}")), + } + } + Ok(()) + } + + /// Flush the final state and print the run summary as JSON. + pub(super) fn finish(&self) -> Result<(), CliError> { + self.save()?; + print_json(&serde_json::json!({ + "channels_created": self.summary.channels_created, + "messages_imported": self.summary.messages_imported, + "reactions_imported": self.summary.reactions_imported, + "bindings_published": self.summary.bindings_published, + "private_members_added": self.summary.private_members_added, + "private_members_unmapped": self.summary.private_members_unmapped, + "skipped": self.summary.skipped, + "warnings": self.summary.warnings, + "state_file": self.state_path.display().to_string(), + })) + } +} + +/// Build one imported message while preserving the channel/thread tags from +/// the SDK builder and appending Slack provenance. +pub(super) fn build_imported_message( + channel: &SlackChannel, + channel_uuid: Uuid, + msg: &SlackMessage, + names: &HashMap, + team_id: &str, + thread_ref: Option<&buzz_sdk::ThreadRef>, +) -> Result { + let author = author_id(msg); + let author_name = author_display(msg, names); + // The `import_author` id is the workspace-scoped foreign id `:`, + // so it composes to the same `slack::` key the identity binding + // uses — that is how the client joins an imported message to its attributed + // Buzz profile. Without the team prefix the join would miss. + let author_foreign = format!("{team_id}:{}", author.as_deref().unwrap_or("unknown")); + + let mut content_parts = Vec::new(); + if !msg.text.trim().is_empty() { + content_parts.push(mrkdwn::convert(&msg.text, names)); + } else { + let blocks = render_slack_blocks(&msg.blocks, names); + if !blocks.is_empty() { + content_parts.push(blocks); + } + } + for attachment in &msg.attachments { + let rendered = render_attachment(attachment, names); + if !rendered.is_empty() { + content_parts.push(rendered); + } + } + for file in &msg.files { + match file.link() { + Some(link) => content_parts.push(format!("📎 [{}]({link})", file.label())), + None => content_parts.push(format!("📎 {}", file.label())), + } + } + let content = content_parts.join("\n\n"); + // The prefix keeps imported history readable in clients that do not + // understand identity-binding events. Buzz Desktop removes the redundant + // prefix when rendering the provenance-aware message. + let escaped_author_name = escape_markdown_emphasis(&author_name); + let content = format!("**{escaped_author_name}**: {}", content.trim()); + + let broadcast = matches!( + msg.subtype.as_deref(), + Some("thread_broadcast" | "reply_broadcast") + ); + let mut import_tags = provenance_tags(&author_foreign, &author_name, &msg.ts)?; + import_tags.push( + nostr::Tag::parse([ + "import_conversation", + &format!("{team_id}:{}", channel.id), + channel.kind.as_str(), + ]) + .map_err(|e| CliError::Other(format!("invalid conversation provenance tag: {e}")))?, + ); + buzz_sdk::build_message(channel_uuid, &content, thread_ref, &[], broadcast, &[]) + .map_err(|e| CliError::Other(format!("build_message failed: {e}"))) + .and_then(|builder| { + Ok(builder + .custom_created_at(Timestamp::from(ts_seconds(&msg.ts)?)) + .tags(import_tags)) + }) +} + +fn escape_markdown_emphasis(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for character in value.chars() { + if matches!( + character, + '\\' | '*' | '_' | '`' | '[' | ']' | '(' | ')' | '~' + ) { + escaped.push('\\'); + } + escaped.push(character); + } + escaped +} + +/// Sign with `client` and submit, returning the locally computed event id. +/// +/// A 2xx response with `accepted: false` whose message marks a duplicate is +/// success (idempotent re-run after state loss); any other rejection is an +/// error. +/// +/// Bulk imports run head-first into the relay's per-pubkey minute quotas, +/// and the relay's `retry in 0s` hint makes the client's built-in retry +/// spin uselessly — so 429s are absorbed here with a real backoff. The +/// signed event is resubmitted verbatim; a re-send that lands twice is a +/// relay-side duplicate, which the acceptance check below treats as success. +pub(super) async fn submit(client: &BuzzClient, builder: EventBuilder) -> Result { + submit_with_response(client, builder) + .await + .map(|(event_id, _)| event_id) +} + +async fn submit_with_response( + client: &BuzzClient, + builder: EventBuilder, +) -> Result<(String, String), CliError> { + let event = client.sign_event(builder)?; + let event_id = event.id.to_hex(); + let mut backoff_secs = 1u64; + let resp = loop { + match client.submit_event(event.clone()).await { + Ok(resp) => break resp, + Err(CliError::Relay { status: 429, .. }) if backoff_secs <= 64 => { + eprintln!(" rate-limited — retrying in {backoff_secs}s"); + tokio::time::sleep(std::time::Duration::from_secs(backoff_secs)).await; + backoff_secs *= 2; + } + Err(e) => return Err(e), + } + }; + let parsed: serde_json::Value = serde_json::from_str(&resp).unwrap_or_default(); + let accepted = parsed + .get("accepted") + .and_then(|v| v.as_bool()) + .unwrap_or(true); + if !accepted { + let message = parsed + .get("message") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + if !message.contains("duplicate") { + return Err(CliError::Other(format!( + "relay rejected event: {}", + if message.is_empty() { &resp } else { &message } + ))); + } + } + Ok((event_id, resp)) +} + +/// Deterministic Buzz channel UUID for a Slack channel, derived from the +/// workspace + Slack channel id. Making it a pure function of stable Slack ids +/// means a re-run after a crash between the channel-create relay write and the +/// state save reuses the same UUID (an idempotent NIP-33 replace) instead of +/// minting a second channel. The `team_id` prefix keeps channel ids from two +/// workspaces from colliding onto one UUID. +pub(super) fn channel_uuid(team_id: &str, channel_id: &str) -> Uuid { + let name = format!("buzz:slack-import:{team_id}:{channel_id}"); + Uuid::new_v5(&Uuid::NAMESPACE_URL, name.as_bytes()) +} + +/// Build and submit an owner/admin-signed Slack identity binding. +pub(super) async fn publish_binding( + client: &BuzzClient, + team_id: &str, + slack_id: &str, + pubkey_hex: &str, +) -> Result { + let d_tag = buzz_sdk::slack_identity_binding_d_tag(team_id, slack_id); + let builder = buzz_sdk::build_import_identity_binding(&d_tag, pubkey_hex) + .map_err(|e| CliError::Other(format!("build_import_identity_binding failed: {e}")))?; + submit(client, builder).await +} + +/// Provenance tags carried by every imported event. +pub(super) fn provenance_tags( + author_id: &str, + author_name: &str, + slack_ts: &str, +) -> Result, CliError> { + let mk = |parts: &[&str]| { + nostr::Tag::parse(parts.iter().copied()) + .map_err(|e| CliError::Other(format!("invalid provenance tag: {e}"))) + }; + Ok(vec![ + mk(&["import", "slack"])?, + mk(&["import_author", author_id, author_name])?, + mk(&["import_ts", slack_ts])?, + ]) +} + +/// The Slack-side author identifier of a message: user id, else bot id. +pub(super) fn author_id(msg: &SlackMessage) -> Option { + msg.user + .clone() + .filter(|u| !u.is_empty()) + .or_else(|| msg.bot_id.clone().filter(|b| !b.is_empty())) +} + +/// Human-readable author name for prefixes and attribution tags. +pub(super) fn author_display(msg: &SlackMessage, names: &HashMap) -> String { + if let Some(ref user) = msg.user { + if let Some(name) = names.get(user) { + return name.clone(); + } + } + if let Some(ref username) = msg.username { + if !username.is_empty() { + return username.clone(); + } + } + author_id(msg).unwrap_or_else(|| "unknown".to_string()) +} + +/// `Some(state key of the thread root)` when `msg` is a reply (its +/// `thread_ts` differs from its own `ts`). +pub(super) fn thread_root_key(channel: &SlackChannel, msg: &SlackMessage) -> Option { + let root_ts = msg.thread_ts.as_deref()?; + if root_ts == msg.ts { + return None; + } + Some(ImportState::message_key(&channel.id, root_ts)) +} diff --git a/crates/buzz-cli/src/commands/import/mapping.rs b/crates/buzz-cli/src/commands/import/mapping.rs new file mode 100644 index 0000000000..44bd34a74d --- /dev/null +++ b/crates/buzz-cli/src/commands/import/mapping.rs @@ -0,0 +1,293 @@ +//! Explicit Slack-conversation → existing Buzz-channel crosswalk parsing. +//! +//! Importers often run after an operator has already mirrored a workspace's +//! channel shell. The crosswalk lets history adopt those channels instead of +//! creating duplicates. CSV matches the export/audit artifact shape used by +//! migration tooling; JSON is useful for generated pipelines. + +use std::collections::{HashMap, HashSet}; +use std::path::Path; + +use serde::Deserialize; +use uuid::Uuid; + +use crate::error::CliError; + +#[derive(Deserialize)] +struct ChannelMapEntry { + slack_channel_id: String, + buzz_channel_id: String, +} + +/// Load a channel crosswalk from CSV or a JSON array. +/// +/// CSV must contain `slack_channel_id` and `buzz_channel_id` columns (extra +/// columns are ignored). JSON uses an array of objects with those same fields. +pub(super) fn load_channel_map(path: &Path) -> Result, CliError> { + let raw = std::fs::read_to_string(path).map_err(|error| { + CliError::Other(format!( + "cannot read channel map {}: {error}", + path.display() + )) + })?; + let entries = if raw.trim_start().starts_with('[') { + serde_json::from_str::>(&raw).map_err(|error| { + CliError::Usage(format!( + "channel map {} is invalid JSON: {error}", + path.display() + )) + })? + } else { + parse_csv_entries(&raw, path)? + }; + + let mut result = HashMap::new(); + let mut buzz_ids = HashSet::new(); + for entry in entries { + let slack_id = entry.slack_channel_id.trim(); + if slack_id.is_empty() { + return Err(CliError::Usage(format!( + "channel map {} contains an empty Slack channel id", + path.display() + ))); + } + let buzz_id = Uuid::parse_str(entry.buzz_channel_id.trim()) + .map_err(|error| { + CliError::Usage(format!( + "channel map {} has invalid Buzz UUID for {slack_id}: {error}", + path.display() + )) + })? + .to_string(); + if let Some(previous) = result.get(slack_id) { + let detail = if previous == &buzz_id { + "appears more than once" + } else { + "maps to two Buzz UUIDs" + }; + return Err(CliError::Usage(format!( + "channel map {} Slack channel {slack_id} {detail}", + path.display() + ))); + } + if !buzz_ids.insert(buzz_id.clone()) { + return Err(CliError::Usage(format!( + "channel map {} assigns Buzz UUID {buzz_id} more than once", + path.display() + ))); + } + result.insert(slack_id.to_string(), buzz_id); + } + if result.is_empty() { + return Err(CliError::Usage(format!( + "channel map {} contains no mappings", + path.display() + ))); + } + Ok(result) +} + +fn parse_csv_entries(raw: &str, path: &Path) -> Result, CliError> { + let records = parse_csv_records(raw).map_err(|error| { + CliError::Usage(format!( + "channel map {} is invalid CSV: {error}", + path.display() + )) + })?; + let (header, rows) = records + .split_first() + .ok_or_else(|| CliError::Usage(format!("channel map {} is empty", path.display())))?; + let slack_index = header + .iter() + .position(|field| field == "slack_channel_id") + .ok_or_else(|| { + CliError::Usage(format!( + "channel map {} is missing slack_channel_id", + path.display() + )) + })?; + let buzz_index = header + .iter() + .position(|field| field == "buzz_channel_id") + .ok_or_else(|| { + CliError::Usage(format!( + "channel map {} is missing buzz_channel_id", + path.display() + )) + })?; + let expected_columns = header.len(); + rows.iter() + .enumerate() + .map(|(index, row)| { + if row.len() != expected_columns { + return Err(CliError::Usage(format!( + "channel map {} CSV row {} has {} columns; expected {}", + path.display(), + index + 2, + row.len(), + expected_columns + ))); + } + Ok(ChannelMapEntry { + slack_channel_id: row[slack_index].clone(), + buzz_channel_id: row[buzz_index].clone(), + }) + }) + .collect() +} + +/// Small RFC 4180 record parser: handles quoted commas, doubled quotes, CRLF, +/// and embedded newlines without adding another runtime dependency. +fn parse_csv_records(raw: &str) -> Result>, &'static str> { + let mut records = Vec::new(); + let mut record = Vec::new(); + let mut field = String::new(); + let mut chars = raw.chars().peekable(); + let mut quoted = false; + + while let Some(ch) = chars.next() { + if quoted { + if ch == '"' { + if chars.peek() == Some(&'"') { + chars.next(); + field.push('"'); + } else { + quoted = false; + } + } else { + field.push(ch); + } + continue; + } + + match ch { + '"' if field.chars().all(char::is_whitespace) => { + field.clear(); + quoted = true; + } + ',' => record.push(std::mem::take(&mut field)), + '\n' | '\r' => { + if ch == '\r' && chars.peek() == Some(&'\n') { + chars.next(); + } + record.push(std::mem::take(&mut field)); + if !record.iter().all(String::is_empty) { + records.push(std::mem::take(&mut record)); + } else { + record.clear(); + } + } + _ => field.push(ch), + } + } + if quoted { + return Err("unterminated quoted field"); + } + if !field.is_empty() || !record.is_empty() { + record.push(field); + if !record.iter().all(String::is_empty) { + records.push(record); + } + } + Ok(records) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn csv_parser_handles_generated_crosswalk_shape() { + let id_a = Uuid::new_v4(); + let id_b = Uuid::new_v4(); + let dir = std::env::temp_dir().join(format!("buzz-channel-map-{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let path = dir.join("map.csv"); + std::fs::write( + &path, + format!( + "\"slack_channel_id\",\"name\",\"buzz_channel_id\"\r\n\ + \"C1\",\"general\",\"{id_a}\"\r\n\ + \"C2\",\"name, with comma\",\"{id_b}\"\r\n" + ), + ) + .expect("write map"); + let map = load_channel_map(&path).expect("load map"); + assert_eq!(map["C1"], id_a.to_string()); + assert_eq!(map["C2"], id_b.to_string()); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn rejects_duplicate_target_uuid() { + let id = Uuid::new_v4(); + let dir = std::env::temp_dir().join(format!("buzz-channel-map-{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let path = dir.join("map.json"); + std::fs::write( + &path, + serde_json::json!([ + {"slack_channel_id": "C1", "buzz_channel_id": id}, + {"slack_channel_id": "C2", "buzz_channel_id": id}, + ]) + .to_string(), + ) + .expect("write map"); + assert!(load_channel_map(&path).is_err()); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn rejects_duplicate_slack_id_before_target_reuse_diagnostics() { + let id_a = Uuid::new_v4(); + let id_b = Uuid::new_v4(); + let dir = std::env::temp_dir().join(format!("buzz-channel-map-{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let path = dir.join("map.json"); + + std::fs::write( + &path, + serde_json::json!([ + {"slack_channel_id": "C1", "buzz_channel_id": id_a}, + {"slack_channel_id": "C1", "buzz_channel_id": id_a}, + ]) + .to_string(), + ) + .expect("write exact duplicate"); + let exact = load_channel_map(&path).expect_err("exact duplicate must fail"); + assert!(exact.to_string().contains("appears more than once")); + + std::fs::write( + &path, + serde_json::json!([ + {"slack_channel_id": "C1", "buzz_channel_id": id_a}, + {"slack_channel_id": "C1", "buzz_channel_id": id_b}, + ]) + .to_string(), + ) + .expect("write conflicting duplicate"); + let conflicting = load_channel_map(&path).expect_err("conflicting duplicate must fail"); + assert!(conflicting.to_string().contains("maps to two Buzz UUIDs")); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn csv_parser_accepts_leading_space_before_quotes_and_bare_cr_records() { + let id = Uuid::new_v4(); + let dir = std::env::temp_dir().join(format!("buzz-channel-map-{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let path = dir.join("map.csv"); + std::fs::write( + &path, + format!( + " \"slack_channel_id\", \"name\", \"buzz_channel_id\"\r\ + \"C1\", \"name, with comma\", \"{id}\"\r" + ), + ) + .expect("write map"); + + let map = load_channel_map(&path).expect("load map"); + assert_eq!(map["C1"], id.to_string()); + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/crates/buzz-cli/src/commands/import/model.rs b/crates/buzz-cli/src/commands/import/model.rs new file mode 100644 index 0000000000..7c68becca2 --- /dev/null +++ b/crates/buzz-cli/src/commands/import/model.rs @@ -0,0 +1,319 @@ +//! Normalized Slack export records shared by parsing, planning, and import. + +use serde::Deserialize; + +/// A user record from `users.json`. +#[derive(Debug, Clone, Deserialize)] +pub struct SlackUser { + /// Slack user ID (`U...`). + pub id: String, + /// Login-style short name. + #[serde(default)] + pub name: String, + /// Nested profile fields. + #[serde(default)] + pub profile: SlackUserProfile, + /// Whether Slack marks this account as a bot. + #[serde(default)] + pub is_bot: bool, + /// Whether this account is deactivated. + #[serde(default)] + pub deleted: bool, +} + +/// The `profile` object nested in a user record. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct SlackUserProfile { + /// Preferred display name (may be empty). + #[serde(default)] + pub display_name: String, + /// Full real name (may be empty). + #[serde(default)] + pub real_name: String, +} + +impl SlackUser { + /// Best available human-readable name: display name, then real name, + /// then the login name, then the raw ID. + pub fn best_name(&self) -> &str { + if !self.profile.display_name.is_empty() { + &self.profile.display_name + } else if !self.profile.real_name.is_empty() { + &self.profile.real_name + } else if !self.name.is_empty() { + &self.name + } else { + &self.id + } + } +} + +/// Slack conversation class. Private channels map to private Buzz streams; +/// DMs and MPIMs open native Buzz DM conversations. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum SlackConversationKind { + /// Public Slack channel. + #[default] + PublicChannel, + /// Private Slack channel. + PrivateChannel, + /// One-to-one Slack direct message. + DirectMessage, + /// Slack multi-person direct message. + GroupDirectMessage, +} + +impl SlackConversationKind { + /// Stable machine-readable name used by dry-run output and provenance. + pub fn as_str(self) -> &'static str { + match self { + Self::PublicChannel => "public_channel", + Self::PrivateChannel => "private_channel", + Self::DirectMessage => "direct_message", + Self::GroupDirectMessage => "group_direct_message", + } + } + + /// Whether this conversation must remain private in Buzz. + pub fn is_private(self) -> bool { + !matches!(self, Self::PublicChannel) + } + + /// Buzz channel kind used for the imported conversation. + pub fn buzz_channel_kind(self) -> buzz_sdk::ChannelKind { + match self { + Self::DirectMessage | Self::GroupDirectMessage => buzz_sdk::ChannelKind::Dm, + Self::PublicChannel | Self::PrivateChannel => buzz_sdk::ChannelKind::Stream, + } + } +} + +/// A conversation record normalized from `channels.json`, `groups.json`, +/// `dms.json`, or `mpims.json`. +#[derive(Debug, Clone, Deserialize)] +pub struct SlackChannel { + /// Slack conversation ID (`C...`, `G...`, or `D...`). + pub id: String, + /// User-facing channel name. DMs get a deterministic synthetic name. + #[serde(default)] + pub name: String, + /// Whether the channel is archived in Slack. + #[serde(default)] + pub is_archived: bool, + /// Channel topic. + #[serde(default)] + pub topic: SlackTopicLike, + /// Channel purpose (description). + #[serde(default)] + pub purpose: SlackTopicLike, + /// Slack member IDs allowed to see the conversation. + #[serde(default, deserialize_with = "deserialize_null_default")] + pub members: Vec, + /// Slack private-channel flag (used while normalizing `channels.json`). + #[serde(default)] + pub is_private: bool, + /// Slack one-to-one direct-message flag. + #[serde(default)] + pub is_im: bool, + /// Slack multi-person direct-message flag. + #[serde(default)] + pub is_mpim: bool, + /// Normalized conversation class. + #[serde(skip)] + pub kind: SlackConversationKind, + /// Export subdirectory holding daily message JSON. + #[serde(skip)] + pub(super) export_directory: String, +} + +/// Shared shape of Slack `topic` / `purpose` objects. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct SlackTopicLike { + /// The text value. + #[serde(default)] + pub value: String, +} + +/// One message from a per-day export file. +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct SlackMessage { + /// Message type — importable messages have `"message"`. + #[serde(default, rename = "type")] + pub msg_type: String, + /// Slack subtype (`channel_join`, `bot_message`, ...); absent for + /// ordinary user messages. + #[serde(default)] + pub subtype: Option, + /// Author user ID (`U...`); absent for some bot messages. + #[serde(default)] + pub user: Option, + /// Author bot ID (`B...`) for bot messages. + #[serde(default)] + pub bot_id: Option, + /// Display username for bot messages. + #[serde(default)] + pub username: Option, + /// Message text in Slack mrkdwn. + #[serde(default)] + pub text: String, + /// Microsecond-precision timestamp string, e.g. `"1610000000.000200"`. + /// Unique per channel — Slack's message primary key. + pub ts: String, + /// Thread root `ts` when this message is part of a thread. + #[serde(default)] + pub thread_ts: Option, + /// Emoji reactions on this message. + #[serde(default, deserialize_with = "deserialize_null_default")] + pub reactions: Vec, + /// Attached files. + #[serde(default, deserialize_with = "deserialize_null_default")] + pub files: Vec, + /// Classic Slack message attachments (integration cards, unfurls, fields). + #[serde(default, deserialize_with = "deserialize_null_default")] + pub attachments: Vec, + /// Slack Block Kit / rich-text blocks. + #[serde(default, deserialize_with = "deserialize_null_default")] + pub blocks: Vec, +} + +/// One emoji reaction group on a message. Bot mode signs one reaction per +/// distinct emoji, so per-reactor identity cannot be reproduced; the source +/// users and count are still parsed for dry-run fidelity auditing. +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct SlackReaction { + /// Emoji shortcode without colons (may carry `::skin-tone-N`). + pub name: String, + /// Slack user IDs returned for this reaction group. Slack may omit some + /// users even when `count` is larger. + #[serde(default)] + pub users: Vec, + /// Authoritative aggregate count supplied by Slack. + #[serde(default)] + pub count: u64, +} + +/// One file attachment stub. +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct SlackFile { + /// File name. + #[serde(default)] + pub name: Option, + /// Human title. + #[serde(default)] + pub title: Option, + /// Slack-hosted permalink (requires Slack auth to fetch). + #[serde(default)] + pub permalink: Option, + /// Private download URL (requires Slack auth to fetch). + #[serde(default)] + pub url_private: Option, +} + +/// A classic Slack attachment. Slack integrations emit many optional fields; +/// these are the user-visible ones that can be represented faithfully in +/// Markdown without importing interactive application actions. +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +pub struct SlackAttachment { + /// Text shown before the attachment body. + #[serde(default, deserialize_with = "deserialize_null_string")] + pub pretext: String, + /// Main attachment text. + #[serde(default, deserialize_with = "deserialize_null_string")] + pub text: String, + /// Attachment title. + #[serde(default, deserialize_with = "deserialize_null_string")] + pub title: String, + /// Optional link for the title. + #[serde(default, deserialize_with = "deserialize_null_string")] + pub title_link: String, + /// Plain-text fallback supplied by the Slack integration. + #[serde(default, deserialize_with = "deserialize_null_string")] + pub fallback: String, + /// Attachment author/integration label. + #[serde(default, deserialize_with = "deserialize_null_string")] + pub author_name: String, + /// Optional author link. + #[serde(default, deserialize_with = "deserialize_null_string")] + pub author_link: String, + /// Structured label/value fields. + #[serde(default, deserialize_with = "deserialize_null_default")] + pub fields: Vec, + /// Rich-text blocks nested in the attachment. + #[serde(default, deserialize_with = "deserialize_null_default")] + pub blocks: Vec, + /// Unfurl/image target. + #[serde(default, deserialize_with = "deserialize_null_string")] + pub image_url: String, + /// Original unfurled URL. + #[serde(default, deserialize_with = "deserialize_null_string")] + pub original_url: String, + /// Source URL for the unfurl. + #[serde(default, deserialize_with = "deserialize_null_string")] + pub from_url: String, +} + +impl SlackAttachment { + /// Whether the attachment has a non-interactive representation worth + /// importing. + pub fn has_content(&self) -> bool { + [ + &self.pretext, + &self.text, + &self.title, + &self.fallback, + &self.author_name, + &self.image_url, + &self.original_url, + &self.from_url, + ] + .iter() + .any(|value| !value.trim().is_empty()) + || !self.fields.is_empty() + || !self.blocks.is_empty() + } +} + +/// One title/value pair in a classic Slack attachment. +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +pub struct SlackAttachmentField { + /// Field heading. + #[serde(default, deserialize_with = "deserialize_null_string")] + pub title: String, + /// Field value (Slack mrkdwn). + #[serde(default, deserialize_with = "deserialize_null_string")] + pub value: String, +} + +impl SlackFile { + /// Best display label for the attachment. + pub fn label(&self) -> &str { + match (&self.name, &self.title) { + (Some(n), _) if !n.is_empty() => n, + (_, Some(t)) if !t.is_empty() => t, + _ => "attachment", + } + } + + /// Best link target, preferring the permalink. + pub fn link(&self) -> Option<&str> { + self.permalink + .as_deref() + .filter(|s| !s.is_empty()) + .or(self.url_private.as_deref().filter(|s| !s.is_empty())) + } +} + +fn deserialize_null_default<'de, D, T>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, + T: Deserialize<'de> + Default, +{ + Ok(Option::::deserialize(deserializer)?.unwrap_or_default()) +} + +fn deserialize_null_string<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + deserialize_null_default(deserializer) +} diff --git a/crates/buzz-cli/src/commands/import/mrkdwn.rs b/crates/buzz-cli/src/commands/import/mrkdwn.rs new file mode 100644 index 0000000000..da241d78f9 --- /dev/null +++ b/crates/buzz-cli/src/commands/import/mrkdwn.rs @@ -0,0 +1,245 @@ +//! Minimal Slack mrkdwn → markdown conversion. +//! +//! Code blocks (``` fenced) and inline code (single backtick) are preserved +//! verbatim. Outside code, Slack angle-bracket tokens are rewritten to +//! plain-text/markdown equivalents and HTML entities are unescaped. +//! +//! Mentions become plain `@Name` text on purpose — no `p` tags are emitted +//! anywhere in the importer, so backdated history cannot flood mention +//! feeds (see docs/slack-import.md). + +use std::collections::HashMap; + +/// Convert one Slack message body to markdown. +/// +/// `user_names` maps Slack user IDs to display names for `<@U...>` tokens. +pub fn convert(text: &str, user_names: &HashMap) -> String { + map_outside_code_blocks(text, |segment| { + map_outside_inline_code(segment, |plain| { + let replaced = convert_tokens(plain, user_names); + let unescaped = unescape_entities(&replaced); + convert_bold(&unescaped) + }) + }) +} + +/// Split on ``` fences; apply `f` to segments outside fences, keep fenced +/// segments (and the fences themselves) verbatim. +fn map_outside_code_blocks(text: &str, f: impl Fn(&str) -> String) -> String { + let mut out = String::with_capacity(text.len()); + for (i, segment) in text.split("```").enumerate() { + if i > 0 { + out.push_str("```"); + } + if i % 2 == 0 { + out.push_str(&f(segment)); + } else { + out.push_str(segment); + } + } + out +} + +/// Split on single backticks; apply `f` outside inline code spans. +fn map_outside_inline_code(text: &str, f: impl Fn(&str) -> String) -> String { + let mut out = String::with_capacity(text.len()); + for (i, segment) in text.split('`').enumerate() { + if i > 0 { + out.push('`'); + } + if i % 2 == 0 { + out.push_str(&f(segment)); + } else { + out.push_str(segment); + } + } + out +} + +/// Rewrite `<...>` tokens: user/channel mentions, specials, and links. +fn convert_tokens(text: &str, user_names: &HashMap) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(start) = rest.find('<') { + out.push_str(&rest[..start]); + let after = &rest[start + 1..]; + match after.find('>') { + Some(end) => { + out.push_str(&convert_one_token(&after[..end], user_names)); + rest = &after[end + 1..]; + } + None => { + // Unclosed '<' — keep the rest verbatim. + out.push('<'); + rest = after; + } + } + } + out.push_str(rest); + out +} + +fn convert_one_token(inner: &str, user_names: &HashMap) -> String { + // `<@U123>` or `<@U123|fallback>` + if let Some(body) = inner.strip_prefix('@') { + let (id, fallback) = split_pipe(body); + let name = user_names + .get(id) + .map(String::as_str) + .or(fallback) + .unwrap_or(id); + return format!("@{name}"); + } + // `<#C123|name>` or `<#C123>` + if let Some(body) = inner.strip_prefix('#') { + let (id, label) = split_pipe(body); + return format!("#{}", label.unwrap_or(id)); + } + // ``, ``, ``, `` + if let Some(body) = inner.strip_prefix('!') { + let (id, label) = split_pipe(body); + return match id { + "here" | "channel" | "everyone" => format!("@{id}"), + _ => label + .map(str::to_string) + .unwrap_or_else(|| format!("@{id}")), + }; + } + // `` or `` + let (url, label) = split_pipe(inner); + match label { + Some(label) if !label.is_empty() => format!("[{label}]({url})"), + _ => url.to_string(), + } +} + +fn split_pipe(s: &str) -> (&str, Option<&str>) { + match s.split_once('|') { + Some((a, b)) => (a, Some(b)), + None => (s, None), + } +} + +/// Unescape the three entities Slack always escapes in message text. +fn unescape_entities(text: &str) -> String { + text.replace("<", "<") + .replace(">", ">") + .replace("&", "&") +} + +/// Convert Slack `*bold*` to markdown `**bold**`, conservatively: the pair +/// must sit on one line, open must be followed by non-space, close must be +/// preceded by non-space. Anything else is left untouched. +fn convert_bold(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for (i, line) in text.split('\n').enumerate() { + if i > 0 { + out.push('\n'); + } + out.push_str(&convert_bold_line(line)); + } + out +} + +fn convert_bold_line(line: &str) -> String { + let chars: Vec = line.chars().collect(); + let mut doubled: Vec = vec![false; chars.len()]; + let mut open: Option = None; + for (i, &c) in chars.iter().enumerate() { + if c != '*' { + continue; + } + match open { + None => { + let can_open = chars.get(i + 1).is_some_and(|&n| n != ' ' && n != '*') + && (i == 0 || chars[i - 1] != '*'); + if can_open { + open = Some(i); + } + } + Some(start) => { + let can_close = i > 0 && chars[i - 1] != ' ' && chars[i - 1] != '*'; + if can_close { + doubled[start] = true; + doubled[i] = true; + open = None; + } + } + } + } + let mut out = String::with_capacity(line.len() + 8); + for (i, &c) in chars.iter().enumerate() { + out.push(c); + if doubled[i] { + out.push('*'); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn names() -> HashMap { + let mut m = HashMap::new(); + m.insert("U123".to_string(), "alice".to_string()); + m + } + + #[test] + fn converts_user_mentions() { + assert_eq!(convert("hi <@U123>!", &names()), "hi @alice!"); + // Unknown user falls back to the fallback label, then the raw id. + assert_eq!(convert("<@U999|bob>", &names()), "@bob"); + assert_eq!(convert("<@U999>", &names()), "@U999"); + } + + #[test] + fn converts_channels_and_specials() { + assert_eq!(convert("see <#C1|general>", &names()), "see #general"); + assert_eq!(convert(" heads up", &names()), "@here heads up"); + assert_eq!(convert("", &names()), "@channel"); + } + + #[test] + fn converts_links() { + assert_eq!( + convert("see ", &names()), + "see [the docs](https://a.io)" + ); + assert_eq!(convert("", &names()), "https://a.io"); + } + + #[test] + fn unescapes_entities() { + assert_eq!( + convert("a < b && c > d", &names()), + "a < b && c > d" + ); + } + + #[test] + fn converts_bold_conservatively() { + assert_eq!(convert("*bold* text", &names()), "**bold** text"); + assert_eq!(convert("2 * 3 * 4", &names()), "2 * 3 * 4"); + assert_eq!(convert("a *b\nc* d", &names()), "a *b\nc* d"); + } + + #[test] + fn preserves_code() { + assert_eq!( + convert("look ```<@U123> *x*``` done *y*", &names()), + "look ```<@U123> *x*``` done **y**" + ); + assert_eq!( + convert("run `cmd <@U123>` now", &names()), + "run `cmd <@U123>` now" + ); + } + + #[test] + fn keeps_unclosed_angle_verbatim() { + assert_eq!(convert("a < b", &names()), "a < b"); + } +} diff --git a/crates/buzz-cli/src/commands/import/render.rs b/crates/buzz-cli/src/commands/import/render.rs new file mode 100644 index 0000000000..00de717023 --- /dev/null +++ b/crates/buzz-cli/src/commands/import/render.rs @@ -0,0 +1,317 @@ +//! Slack attachment, Block Kit, and reaction rendering helpers. + +use std::collections::HashMap; + +use super::export::SlackAttachment; +use super::mrkdwn; + +/// Render a classic Slack attachment to readable Markdown. Interactive +/// buttons/actions are intentionally omitted; their visible label and context +/// are retained through the attachment fallback or fields. +pub(super) fn render_attachment( + attachment: &SlackAttachment, + names: &HashMap, +) -> String { + let mut parts = Vec::new(); + push_unique( + &mut parts, + mrkdwn::convert(attachment.pretext.trim(), names), + ); + + if !attachment.author_name.trim().is_empty() { + let author = if attachment.author_link.trim().is_empty() { + attachment.author_name.trim().to_string() + } else { + format!( + "[{}]({})", + attachment.author_name.trim(), + attachment.author_link.trim() + ) + }; + push_unique(&mut parts, author); + } + + if !attachment.title.trim().is_empty() { + let title = if attachment.title_link.trim().is_empty() { + format!("**{}**", attachment.title.trim()) + } else { + format!( + "**[{}]({})**", + attachment.title.trim(), + attachment.title_link.trim() + ) + }; + push_unique(&mut parts, title); + } + push_unique(&mut parts, mrkdwn::convert(attachment.text.trim(), names)); + + for field in &attachment.fields { + let title = field.title.trim(); + let value = mrkdwn::convert(field.value.trim(), names); + let rendered = match (title.is_empty(), value.is_empty()) { + (false, false) => format!("**{title}:** {value}"), + (false, true) => format!("**{title}**"), + (true, false) => value, + (true, true) => continue, + }; + push_unique(&mut parts, rendered); + } + + if parts.is_empty() { + push_unique( + &mut parts, + mrkdwn::convert(attachment.fallback.trim(), names), + ); + } + if parts.is_empty() { + push_unique(&mut parts, render_slack_blocks(&attachment.blocks, names)); + } + + let media_url = [ + attachment.image_url.trim(), + attachment.original_url.trim(), + attachment.from_url.trim(), + ] + .into_iter() + .find(|value| !value.is_empty()); + if let Some(url) = media_url { + let label = if attachment.title.trim().is_empty() { + "Slack attachment" + } else { + attachment.title.trim() + }; + let link = format!("[{label}]({url})"); + if !parts.iter().any(|part| part.contains(url)) { + parts.push(link); + } + } + + parts.join("\n") +} + +fn push_unique(parts: &mut Vec, value: String) { + let value = value.trim(); + if !value.is_empty() && !parts.iter().any(|existing| existing == value) { + parts.push(value.to_string()); + } +} + +/// Tolerant renderer for the non-interactive subset of Slack rich-text and +/// Block Kit structures. Unknown container blocks still recurse through their +/// `elements`, so newer Slack block wrappers do not silently erase their text. +pub(super) fn render_slack_blocks( + blocks: &[serde_json::Value], + names: &HashMap, +) -> String { + blocks + .iter() + .map(|block| render_slack_block(block, names)) + .filter(|rendered| !rendered.trim().is_empty()) + .collect::>() + .join("\n") +} + +fn render_slack_block(value: &serde_json::Value, names: &HashMap) -> String { + let Some(object) = value.as_object() else { + return String::new(); + }; + let block_type = object + .get("type") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + + let rendered = match block_type { + "text" | "plain_text" | "mrkdwn" => object + .get("text") + .and_then(serde_json::Value::as_str) + .map(str::to_string) + .unwrap_or_default(), + "link" => { + let url = object + .get("url") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + let label = object + .get("text") + .and_then(serde_json::Value::as_str) + .filter(|text| !text.is_empty()) + .unwrap_or(url); + if url.is_empty() { + label.to_string() + } else { + format!("[{label}]({url})") + } + } + "user" => { + let id = object + .get("user_id") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + format!("@{}", names.get(id).map(String::as_str).unwrap_or(id)) + } + "channel" => object + .get("channel_id") + .and_then(serde_json::Value::as_str) + .map(|id| format!("#{id}")) + .unwrap_or_default(), + "emoji" => object + .get("name") + .and_then(serde_json::Value::as_str) + .map(|name| format!(":{name}:")) + .unwrap_or_default(), + "broadcast" => object + .get("range") + .and_then(serde_json::Value::as_str) + .map(|range| format!("@{range}")) + .unwrap_or_default(), + "rich_text_list" => { + let ordered = + object.get("style").and_then(serde_json::Value::as_str) == Some("ordered"); + child_elements(object) + .iter() + .enumerate() + .map(|(index, child)| { + let body = render_slack_block(child, names); + if ordered { + format!("{}. {body}", index + 1) + } else { + format!("- {body}") + } + }) + .collect::>() + .join("\n") + } + "rich_text_quote" => child_elements(object) + .iter() + .map(|child| render_slack_block(child, names)) + .collect::() + .lines() + .map(|line| format!("> {line}")) + .collect::>() + .join("\n"), + "rich_text_preformatted" => { + let body = child_elements(object) + .iter() + .map(|child| render_slack_block(child, names)) + .collect::(); + format!("```\n{body}\n```") + } + "divider" => "---".to_string(), + _ => { + if let Some(text) = object.get("text") { + text.as_str() + .map(str::to_string) + .unwrap_or_else(|| render_slack_block(text, names)) + } else { + let children = child_elements(object); + let separator = if matches!(block_type, "rich_text" | "section") { + "\n" + } else { + "" + }; + children + .iter() + .map(|child| render_slack_block(child, names)) + .collect::>() + .join(separator) + } + } + }; + + apply_block_style(rendered, object.get("style")) +} + +fn child_elements(object: &serde_json::Map) -> &[serde_json::Value] { + object + .get("elements") + .and_then(serde_json::Value::as_array) + .map(Vec::as_slice) + .unwrap_or(&[]) +} + +fn apply_block_style(mut text: String, style: Option<&serde_json::Value>) -> String { + let Some(style) = style.and_then(serde_json::Value::as_object) else { + return text; + }; + if style + .get("code") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + { + text = format!("`{text}`"); + } + if style + .get("bold") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + { + text = format!("**{text}**"); + } + if style + .get("italic") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + { + text = format!("_{text}_"); + } + if style + .get("strike") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + { + text = format!("~~{text}~~"); + } + text +} + +/// Map common Slack reaction shortcodes to Unicode; anything unknown keeps +/// the `:shortcode:` form (rendered when the custom emoji is registered). +/// Skin-tone suffixes (`::skin-tone-N`) are dropped. +pub(super) fn emoji_for_shortcode(name: &str) -> String { + let base = name.split("::").next().unwrap_or(name); + let mapped = match base { + "+1" | "thumbsup" => "👍", + "-1" | "thumbsdown" => "👎", + "heart" => "❤️", + "joy" => "😂", + "smile" => "😄", + "grin" => "😁", + "laughing" => "😆", + "sweat_smile" => "😅", + "sob" => "😭", + "cry" => "😢", + "tada" => "🎉", + "eyes" => "👀", + "fire" => "🔥", + "rocket" => "🚀", + "pray" => "🙏", + "clap" => "👏", + "wave" => "👋", + "raised_hands" => "🙌", + "ok_hand" => "👌", + "muscle" => "💪", + "100" => "💯", + "thinking_face" => "🤔", + "white_check_mark" => "✅", + "heavy_check_mark" => "✔️", + "x" => "❌", + "heart_eyes" => "😍", + "sunglasses" => "😎", + "sparkles" => "✨", + "star" => "⭐", + "zap" => "⚡", + "warning" => "⚠️", + "question" => "❓", + "exclamation" => "❗", + "bulb" => "💡", + "memo" => "📝", + "bug" => "🐛", + "wink" => "😉", + "point_up" => "☝️", + "point_down" => "👇", + "seedling" => "🌱", + "bee" | "honeybee" => "🐝", + _ => return format!(":{base}:"), + }; + mapped.to_string() +} diff --git a/crates/buzz-cli/src/commands/import/report.rs b/crates/buzz-cli/src/commands/import/report.rs new file mode 100644 index 0000000000..54613e6a6c --- /dev/null +++ b/crates/buzz-cli/src/commands/import/report.rs @@ -0,0 +1,156 @@ +//! Dry-run planning and fidelity totals for Slack imports. + +use std::collections::{HashMap, HashSet}; + +use super::export::{SlackChannel, SlackConversationKind, SlackExport, SlackMessage}; +use super::print_json; +use super::render::emoji_for_shortcode; +use super::state::ImportState; +use crate::error::CliError; + +pub(super) fn dry_run_report( + export: &SlackExport, + selected: &[&SlackChannel], + state: &ImportState, + channel_map: &HashMap, + bindings: &HashMap, + dm_blockers: &HashMap, + skip_reactions: bool, +) -> Result<(), CliError> { + let mut channels_to_create = 0u64; + let mut channels_to_adopt = 0u64; + let mut channels_already_in_state = 0u64; + let mut messages = 0u64; + let mut reaction_groups = 0u64; + let mut reaction_uses = 0u64; + let mut public_channels = 0u64; + let mut private_channels = 0u64; + let mut direct_messages = 0u64; + let mut group_direct_messages = 0u64; + let mut archived = 0u64; + let mut private_members = 0u64; + let mut mappable_private_members = 0u64; + let mut mapped_private_members = 0u64; + let mut message_records_seen = 0u64; + let mut non_message_records = 0u64; + let mut system_records = 0u64; + let mut mutation_records = 0u64; + let mut contentless_records = 0u64; + let mut duplicate_records = 0u64; + let mut dm_conversations_blocked = Vec::new(); + let mut dm_messages_blocked = 0u64; + let mapped: HashSet<&str> = bindings.keys().map(String::as_str).collect(); + for channel in selected { + match channel.kind { + SlackConversationKind::PublicChannel => public_channels += 1, + SlackConversationKind::PrivateChannel => private_channels += 1, + SlackConversationKind::DirectMessage => direct_messages += 1, + SlackConversationKind::GroupDirectMessage => group_direct_messages += 1, + } + if channel.is_archived { + archived += 1; + } + if channel.kind.is_private() { + private_members += channel.members.len() as u64; + let mappable: Vec<&String> = channel + .members + .iter() + .filter(|id| export.is_mappable_member(id)) + .collect(); + mappable_private_members += mappable.len() as u64; + mapped_private_members += mappable + .into_iter() + .filter(|id| mapped.contains(id.as_str())) + .count() as u64; + } + let already_in_state = state.channels.contains_key(&channel.id); + let mapped_existing_channel = channel_map.contains_key(&channel.id); + let dm_blocker = dm_blockers.get(&channel.id); + if let Some(reason) = &dm_blocker { + dm_conversations_blocked.push(serde_json::json!({ + "conversation_id": channel.id, + "conversation_name": channel.name, + "kind": channel.kind.as_str(), + "reason": reason, + })); + } else if already_in_state { + channels_already_in_state += 1; + } else if mapped_existing_channel { + channels_to_adopt += 1; + } else { + channels_to_create += 1; + } + let scan = export.channel_message_scan(channel)?; + message_records_seen += scan.records_seen; + non_message_records += scan.non_message_records; + system_records += scan.system_records; + mutation_records += scan.mutation_records; + contentless_records += scan.contentless_records; + duplicate_records += scan.duplicate_records; + for msg in scan.messages { + let message_key = ImportState::message_key(&channel.id, &msg.ts); + if !state.messages.contains_key(&message_key) { + if dm_blocker.is_some() { + dm_messages_blocked += 1; + } else { + messages += 1; + } + } + if !skip_reactions && dm_blocker.is_none() { + reaction_groups += pending_reaction_count(state, &message_key, &msg) as u64; + reaction_uses += msg + .reactions + .iter() + .map(|reaction| { + reaction + .count + .max(u64::try_from(reaction.users.len()).unwrap_or(u64::MAX)) + }) + .sum::(); + } + } + } + print_json(&serde_json::json!({ + "dry_run": true, + "conversations_selected": selected.len(), + "public_channels_selected": public_channels, + "private_channels_selected": private_channels, + "direct_messages_selected": direct_messages, + "group_direct_messages_selected": group_direct_messages, + "archived_conversations_selected": archived, + "channels_to_create": channels_to_create, + "channels_to_adopt": channels_to_adopt, + "channels_already_in_state": channels_already_in_state, + "dm_conversations_blocked": dm_conversations_blocked, + "dm_messages_blocked": dm_messages_blocked, + "message_records_seen": message_records_seen, + "messages_to_import": messages, + "non_message_records_skipped": non_message_records, + "system_records_skipped": system_records, + "mutation_records_skipped": mutation_records, + "contentless_records_skipped": contentless_records, + "duplicate_records_collapsed": duplicate_records, + "reaction_groups_to_import": reaction_groups, + "reaction_uses_in_export": reaction_uses, + "private_memberships_in_export": private_members, + "private_memberships_mappable": mappable_private_members, + "private_memberships_mapped": mapped_private_members, + "private_memberships_unmapped": mappable_private_members + .saturating_sub(mapped_private_members), + "bindings_to_publish": bindings.len(), + })) +} + +/// Number of distinct bot-signed reactions that still need publishing. +pub(super) fn pending_reaction_count( + state: &ImportState, + message_key: &str, + msg: &SlackMessage, +) -> usize { + msg.reactions + .iter() + .map(|reaction| emoji_for_shortcode(&reaction.name)) + .filter(|emoji| !state.reactions.contains(&format!("{message_key}:{emoji}"))) + .collect::>() + .len() +} diff --git a/crates/buzz-cli/src/commands/import/state.rs b/crates/buzz-cli/src/commands/import/state.rs new file mode 100644 index 0000000000..d19c3372b9 --- /dev/null +++ b/crates/buzz-cli/src/commands/import/state.rs @@ -0,0 +1,256 @@ +//! Import state file — idempotency and resume. +//! +//! The state file records every write the importer has completed, keyed by +//! Slack-side identifiers, so a re-run (after an interruption or on a +//! refreshed export) skips work already done. It also doubles as the +//! Slack-ts → Nostr-event-id ledger that thread replies are resolved from. + +use std::collections::{HashMap, HashSet}; +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +use crate::error::CliError; + +const CURRENT_STATE_VERSION: u32 = 3; + +/// Per-channel state: the Buzz UUID minted for it and whether metadata +/// (create + topic/purpose) has been published. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ChannelState { + /// Buzz channel UUID. + pub uuid: String, + /// Whether the create/topic/purpose events were accepted. + #[serde(default)] + pub metadata_done: bool, + /// Whether an archived Slack channel was archived in Buzz. + #[serde(default)] + pub archived_done: bool, + /// Whether a private Slack conversation has been explicitly narrowed to + /// private visibility in Buzz. This repairs ledgers produced by an older + /// importer that created every `channels.json` record as open. + #[serde(default)] + pub private_visibility_done: bool, + /// Whether an adopted, pre-existing channel has been prepared for history + /// writes. Archived adopted channels are temporarily unarchived first. + #[serde(default)] + pub prepared_for_import: bool, + /// Buzz public keys already added from mapped Slack conversation members. + #[serde(default)] + pub members_added: HashSet, +} + +/// The whole state file. +#[derive(Debug, Serialize, Deserialize)] +pub struct ImportState { + /// State schema version. Missing means the file predates workspace pinning. + #[serde(default)] + version: u32, + /// Slack workspace this ledger belongs to. + #[serde(default)] + team_id: Option, + /// Slack channel ID → channel state. + #[serde(default)] + pub channels: HashMap, + /// `":"` → Nostr event ID (hex). + #[serde(default)] + pub messages: HashMap, + /// Reaction dedupe keys: `"::"`. + #[serde(default)] + pub reactions: HashSet, +} + +impl Default for ImportState { + fn default() -> Self { + Self { + version: CURRENT_STATE_VERSION, + team_id: None, + channels: HashMap::new(), + messages: HashMap::new(), + reactions: HashSet::new(), + } + } +} + +impl ImportState { + /// Load state from `path` and bind it to one Slack workspace. + /// + /// A missing file yields fresh state. A non-empty legacy file is rejected: + /// it may contain unscoped `import_author` values, so silently adopting it + /// could skip messages that can never match workspace-scoped bindings. + pub fn load_for_workspace(path: &Path, team_id: &str) -> Result { + let mut state: Self = match std::fs::read_to_string(path) { + Ok(raw) => serde_json::from_str(&raw).map_err(|e| { + CliError::Usage(format!( + "state file {} is corrupt: {e} — move it aside to restart the import", + path.display() + )) + }), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Self::default()), + Err(e) => Err(CliError::Other(format!( + "cannot read state file {}: {e}", + path.display() + ))), + }?; + + if state.version == 0 && !state.is_empty() { + return Err(CliError::Usage(format!( + "state file {} predates workspace-scoped Slack imports and cannot be resumed \ + safely; use a fresh target community, or add version/team_id only after \ + verifying the existing events already use slack:: identities", + path.display() + ))); + } + if state.version > CURRENT_STATE_VERSION { + return Err(CliError::Usage(format!( + "state file {} uses newer schema version {}; this CLI supports version {}", + path.display(), + state.version, + CURRENT_STATE_VERSION + ))); + } + if let Some(saved_team_id) = state.team_id.as_deref() { + if saved_team_id != team_id { + return Err(CliError::Usage(format!( + "state file {} belongs to Slack workspace {saved_team_id}, not {team_id}", + path.display() + ))); + } + } + if state.version < 3 { + // Versions 1–2 could only contain channels created by this + // importer. They were therefore active when history writes began, + // unlike channels adopted through the v3 crosswalk path. + for channel in state.channels.values_mut() { + channel.prepared_for_import = true; + } + } + state.version = CURRENT_STATE_VERSION; + state.team_id = Some(team_id.to_string()); + Ok(state) + } + + /// Persist state to `path` (write-temp-then-rename so an interrupted + /// save never truncates the previous state). + pub fn save(&self, path: &Path) -> Result<(), CliError> { + let raw = serde_json::to_string(self) + .map_err(|e| CliError::Other(format!("state serialization failed: {e}")))?; + let tmp = path.with_extension("json.tmp"); + std::fs::write(&tmp, raw) + .map_err(|e| CliError::Other(format!("cannot write {}: {e}", tmp.display())))?; + std::fs::rename(&tmp, path) + .map_err(|e| CliError::Other(format!("cannot rename state file into place: {e}")))?; + Ok(()) + } + + /// Ledger key for a message. + pub fn message_key(channel_id: &str, ts: &str) -> String { + format!("{channel_id}:{ts}") + } + + fn is_empty(&self) -> bool { + self.channels.is_empty() && self.messages.is_empty() && self.reactions.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn roundtrips_through_disk() { + let dir = std::env::temp_dir().join(format!("buzz-import-state-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let path = dir.join("state.json"); + + let mut state = + ImportState::load_for_workspace(&path, "T1").expect("missing state is fresh"); + state.channels.insert( + "C1".into(), + ChannelState { + uuid: "u-u-i-d".into(), + metadata_done: true, + archived_done: true, + private_visibility_done: true, + prepared_for_import: true, + members_added: HashSet::from(["aa".repeat(32)]), + }, + ); + state + .messages + .insert(ImportState::message_key("C1", "1.000"), "ff".repeat(32)); + state.reactions.insert("C1:1.000:👍".into()); + state.save(&path).expect("save"); + + let loaded = ImportState::load_for_workspace(&path, "T1").expect("load"); + assert_eq!(loaded.channels["C1"].uuid, "u-u-i-d"); + assert!(loaded.channels["C1"].metadata_done); + assert!(loaded.channels["C1"].archived_done); + assert!(loaded.channels["C1"].private_visibility_done); + assert!(loaded.channels["C1"].prepared_for_import); + assert!(loaded.channels["C1"] + .members_added + .contains(&"aa".repeat(32))); + assert_eq!(loaded.messages["C1:1.000"], "ff".repeat(32)); + assert!(loaded.reactions.contains("C1:1.000:👍")); + + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn missing_file_is_empty_state() { + let path = std::path::PathBuf::from("/nonexistent/dir/state.json"); + let state = ImportState::load_for_workspace(&path, "T1").expect("missing file is fine"); + assert!(state.channels.is_empty()); + assert!(state.messages.is_empty()); + } + + #[test] + fn rejects_state_from_a_different_workspace() { + let dir = + std::env::temp_dir().join(format!("buzz-import-state-team-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let path = dir.join("state.json"); + ImportState::load_for_workspace(&path, "T1") + .expect("fresh") + .save(&path) + .expect("save"); + + let error = ImportState::load_for_workspace(&path, "T2").expect_err("must reject"); + assert!(error.to_string().contains("belongs to Slack workspace T1")); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn rejects_non_empty_legacy_state() { + let dir = + std::env::temp_dir().join(format!("buzz-import-state-legacy-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let path = dir.join("state.json"); + std::fs::write( + &path, + r#"{"channels":{},"messages":{"C1:1.0":"abc"},"reactions":[]}"#, + ) + .expect("write"); + + let error = ImportState::load_for_workspace(&path, "T1").expect_err("must reject"); + assert!(error.to_string().contains("predates workspace-scoped")); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn upgrades_v2_importer_channels_as_already_prepared() { + let dir = std::env::temp_dir().join(format!("buzz-import-state-v2-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let path = dir.join("state.json"); + std::fs::write( + &path, + r#"{"version":2,"team_id":"T1","channels":{"C1":{"uuid":"u-u-i-d"}}}"#, + ) + .expect("write"); + + let state = ImportState::load_for_workspace(&path, "T1").expect("upgrade"); + assert!(state.channels["C1"].prepared_for_import); + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/crates/buzz-cli/src/commands/mod.rs b/crates/buzz-cli/src/commands/mod.rs index 1ccc37a702..16c0300fe2 100644 --- a/crates/buzz-cli/src/commands/mod.rs +++ b/crates/buzz-cli/src/commands/mod.rs @@ -4,6 +4,7 @@ pub mod channels; pub mod dms; pub mod emoji; pub mod feed; +pub mod import; pub mod issues; pub mod mem; pub mod messages; diff --git a/crates/buzz-cli/src/lib.rs b/crates/buzz-cli/src/lib.rs index f745e7b280..486636228b 100644 --- a/crates/buzz-cli/src/lib.rs +++ b/crates/buzz-cli/src/lib.rs @@ -203,6 +203,9 @@ enum Cmd { /// Read the activity feed #[command(subcommand)] Feed(FeedCmd), + /// Import history from an external workspace (Slack export) + #[command(subcommand)] + Import(ImportCmd), /// Publish notes and manage the social graph (NIP-01/02) #[command(subcommand)] Social(SocialCmd), @@ -941,6 +944,94 @@ pub enum WorkflowsCmd { }, } +/// Subcommands for `buzz import`. +#[derive(Subcommand)] +pub enum ImportCmd { + /// Import a Slack workspace export directory (see docs/slack-import.md) + #[command( + after_help = "History is signed by the CLI identity (bot mode) and tagged with the \ +original author (import_author) and timestamp. No private key is ever \ +generated for or distributed to anyone.\n\n\ +Attributing history to a real person takes TWO signatures: an owner/admin \ +attestation (Slack id → that person's PUBLIC key) via --identity-map here or \ +`buzz import bind`, AND the person's own consent via `buzz import claim`, run \ +by them with their key. An attestation alone does not attribute history, so an \ +admin cannot make someone appear to author messages they never wrote.\n\n\ +Re-running resumes from the state file — completed writes are skipped.\n\n\ +Examples:\n \ +buzz import slack --export-dir ./export --team-id T0266FRGM --dry-run\n \ +buzz import slack --export-dir ./export --team-id T0266FRGM\n \ +buzz import slack --export-dir ./export --team-id T0266FRGM --channel-map ./crosswalk.csv --dry-run\n \ +buzz import slack --export-dir ./export --team-id T0266FRGM --identity-map U060=npub1abc,U081=npub1def" + )] + Slack { + /// Path to an unzipped Slack export directory. Repeat for separate + /// Slackdump public/private export roots. + #[arg(long = "export-dir", required = true, action = clap::ArgAction::Append)] + export_dirs: Vec, + /// Slack workspace id (team id, e.g. T0266FRGM). Namespaces identity + /// bindings and channel UUIDs so ids never collide across workspaces. + #[arg(long)] + team_id: String, + /// State file path (default: /buzz-import-state.json) + #[arg(long)] + state: Option, + /// CSV or JSON crosswalk from Slack channel ids to existing Buzz + /// channel UUIDs. Adopted archived channels are reopened for the + /// backfill and restored afterward. + #[arg(long)] + channel_map: Option, + /// Only import these conversation names or Slack IDs (comma-separated) + #[arg(long)] + channels: Option, + /// Parse and report what would be imported without writing + #[arg(long, default_value_t = false)] + dry_run: bool, + /// Skip importing reactions + #[arg(long, default_value_t = false)] + skip_reactions: bool, + /// Owner/admin-signed identity bindings: SLACKID=npub-or-hex, comma-separated. + /// Public keys only — never an nsec. + #[arg(long)] + identity_map: Option, + }, + /// Attest that a Slack id maps to a person's public key (owner/admin half) + #[command( + after_help = "The owner/admin half of a two-party identity binding. The pubkey is \ +PUBLIC (npub or hex) — never an nsec. Requires the CLI identity to be a \ +community owner or admin. History is attributed only once the person also runs \ +`buzz import claim` with their own key.\n\nExample:\n \ +buzz import bind --slack-id U060976D0QN --pubkey npub1abc..." + )] + Bind { + /// Slack workspace id (team id, e.g. T0266FRGM) + #[arg(long)] + team_id: String, + /// Slack user id (e.g. U060976D0QN) + #[arg(long)] + slack_id: String, + /// The person's PUBLIC key: npub1… or 64-char hex + #[arg(long)] + pubkey: String, + }, + /// Consent to being attributed a Slack id — the subject half of a binding + #[command( + after_help = "The subject half of a two-party identity binding. Run this yourself, \ +with your own key: it self-signs your consent that the Slack id is you. It \ +attributes history only once a community owner/admin has published the \ +matching `buzz import bind` attestation for your pubkey.\n\nExample:\n \ +buzz import claim --slack-id U060976D0QN" + )] + Claim { + /// Slack workspace id (team id, e.g. T0266FRGM) + #[arg(long)] + team_id: String, + /// Your Slack user id (e.g. U060976D0QN) + #[arg(long)] + slack_id: String, + }, +} + #[derive(Subcommand)] pub enum FeedCmd { /// Get recent activity feed entries @@ -1981,6 +2072,7 @@ async fn run(cli: Cli) -> Result<(), CliError> { Cmd::Users(sub) => commands::users::dispatch(sub, &client, &cli.format).await, Cmd::Workflows(sub) => commands::workflows::dispatch(sub, &client).await, Cmd::Feed(sub) => commands::feed::dispatch(sub, &client, &cli.format).await, + Cmd::Import(sub) => commands::import::dispatch(sub, &client).await, Cmd::Social(sub) => commands::social::dispatch(sub, &client).await, Cmd::Notes(sub) => commands::notes::dispatch(sub, &client).await, Cmd::Repos(sub) => commands::repos::dispatch(sub, &client).await, @@ -2085,6 +2177,7 @@ mod tests { "dms", "emoji", "feed", + "import", "issues", "media", "mem", @@ -2211,6 +2304,7 @@ mod tests { vec!["approve", "create", "delete", "get", "list", "runs", "trigger", "update"] ); assert_eq!(names(&cmd, "feed"), vec!["get"]); + assert_eq!(names(&cmd, "import"), vec!["bind", "claim", "slack"]); assert_eq!( names(&cmd, "social"), vec![ @@ -2293,6 +2387,7 @@ mod tests { ("dms", 4), ("emoji", 5), ("feed", 1), + ("import", 3), ("issues", 4), ("media", 1), ("messages", 8), diff --git a/crates/buzz-core/src/kind.rs b/crates/buzz-core/src/kind.rs index b1be7c5038..a829424d72 100644 --- a/crates/buzz-core/src/kind.rs +++ b/crates/buzz-core/src/kind.rs @@ -435,6 +435,33 @@ pub const KIND_WORKFLOW_DEF: u32 = 30620; /// `hidden_at` per viewer; this is the only Nostr-visible projection of it. pub const KIND_DM_VISIBILITY: u32 = 30622; +/// Import identity binding: an owner/admin-signed **attestation** that a +/// foreign workspace identity (e.g. a Slack user id) belongs to a given Buzz +/// pubkey. Parameterized-replaceable, `d = :` (e.g. +/// `slack:T0266FRGM:U060976D0QN`), with a single `["p", ]` naming the +/// attested identity. The relay accepts this kind ONLY from a community owner +/// or admin (mirrors the kind:9030 relay-admin authorization). +/// +/// This is one half of a two-party binding: the attestation alone does NOT +/// attribute history. Attribution requires a matching [`KIND_IMPORT_IDENTITY_CLAIM`] +/// self-signed by the attested pubkey, so an admin cannot unilaterally make a +/// member appear to author imported history. It carries public keys only; no +/// secret ever transits. +pub const KIND_IMPORT_IDENTITY_BINDING: u32 = 30623; + +/// Import identity claim: the **subject's** self-signed consent to being +/// attributed a foreign workspace identity. Parameterized-replaceable, +/// `d = :` (same key as the matching +/// [`KIND_IMPORT_IDENTITY_BINDING`] attestation). The signer's own pubkey IS +/// the consent — there is no `p` tag — so the relay's signer==author rule means +/// a pubkey can only ever claim on its own behalf; no special role is required. +/// +/// A binding is *confirmed* (and history rendered under the real person) only +/// when an owner/admin attestation and a subject claim exist for the same +/// `d` key and the attestation's `p` equals the claim's author. Either half +/// alone is inert. +pub const KIND_IMPORT_IDENTITY_CLAIM: u32 = 30624; + /// Lower bound of the NIP-33 parameterized replaceable range (30000–39999). pub const PARAM_REPLACEABLE_KIND_MIN: u32 = 30000; /// Upper bound of the NIP-33 parameterized replaceable range (30000–39999). @@ -748,6 +775,8 @@ pub const ALL_KINDS: &[u32] = &[ KIND_GIT_STATUS_MERGED, KIND_GIT_STATUS_CLOSED, KIND_GIT_STATUS_DRAFT, + KIND_IMPORT_IDENTITY_BINDING, + KIND_IMPORT_IDENTITY_CLAIM, KIND_PROJECT, ]; diff --git a/crates/buzz-db/src/event.rs b/crates/buzz-db/src/event.rs index a670a13402..a7ab814518 100644 --- a/crates/buzz-db/src/event.rs +++ b/crates/buzz-db/src/event.rs @@ -1293,8 +1293,12 @@ pub async fn insert_event_with_thread_metadata( event: &Event, channel_id: Option, thread_meta: Option>, + backfill: bool, ) -> Result<(StoredEvent, bool)> { let mut tx = pool.begin().await?; + if backfill { + disarm_floor_guard_tx(&mut tx).await?; + } let result = insert_event_with_thread_metadata_tx(&mut tx, community_id, event, channel_id, thread_meta) .await?; @@ -1302,6 +1306,21 @@ pub async fn insert_event_with_thread_metadata( Ok(result) } +/// Disarm the commit-time `created_at` floor guard (migration 0021) for the +/// current transaction only. +/// +/// Used by authorized history imports: the guard's GUC is transaction-locally +/// cleared so a backdated row can commit. Callers MUST pair this with +/// [`crate::replica_fence::ReplicaFence::backfill_begin`]/`backfill_end` on +/// replica-enabled deployments — a floor-exempt commit is outside the fence +/// proof until the next fresh handshake. +async fn disarm_floor_guard_tx(tx: &mut sqlx::Transaction<'_, sqlx::Postgres>) -> Result<()> { + sqlx::query("SELECT set_config('buzz.created_at_floor', '', true)") + .execute(&mut **tx) + .await?; + Ok(()) +} + /// Atomically insert a kind:7 reaction event and its reaction row. /// /// Ordering is load-bearing: resolve target, upsert/reactivate the reaction row, @@ -1317,8 +1336,12 @@ pub async fn insert_reaction_event_with_thread_metadata( target_event_id: &[u8], actor_pubkey: &[u8], emoji: &str, + backfill: bool, ) -> Result { let mut tx = pool.begin().await?; + if backfill { + disarm_floor_guard_tx(&mut tx).await?; + } let target_row = sqlx::query( "SELECT created_at FROM events \ @@ -1968,6 +1991,7 @@ mod tests { target.id.as_bytes(), &actor_pubkey, "👍", + false, ) .await .expect("first reaction insert"); @@ -1988,6 +2012,7 @@ mod tests { target.id.as_bytes(), &actor_pubkey, "👍", + false, ) .await .expect("duplicate reaction insert"); @@ -2026,6 +2051,7 @@ mod tests { target.id.as_bytes(), &actor_pubkey, "👍", + false, ) .await .expect("cross-community reaction attempt"); @@ -2085,6 +2111,7 @@ mod tests { target.id.as_bytes(), &actor_pubkey, "👍", + false, ) .await .expect_err("ephemeral event insert must fail after reaction upsert attempt"); @@ -2134,6 +2161,7 @@ mod tests { target.id.as_bytes(), &actor_pubkey, "👍", + false, ) .await .expect("first reaction insert"), @@ -2159,6 +2187,7 @@ mod tests { target.id.as_bytes(), &actor_pubkey, "👍", + false, ) .await .expect("reactivate reaction"); diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 9b26876747..8e8215205d 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -181,6 +181,10 @@ pub struct Db { /// route here (see [`Db::read`]); locks, transactions, and anything /// consistency-critical stays on `pool`. pub(crate) read_pool: Option, + /// The commit-time floor armed on the writer pool (from + /// [`DbConfig::created_at_floor_secs`]); the fence probe must subtract + /// the same value — the two uses must never diverge. + pub(crate) created_at_floor_secs: i64, /// Maximum connections configured for the read-replica pool (from /// [`DbConfig::read_max_connections`], defaulting to the writer's /// sizing). Kept separately from `max_connections` so @@ -520,6 +524,12 @@ pub struct DbConfig { pub max_lifetime_secs: u64, /// Seconds a connection may sit idle before being closed. pub idle_timeout_secs: u64, + /// Seconds of `created_at` history the commit-time floor guard tolerates + /// (migration 0021). Must exceed the relay's accepted past drift by + /// enough slack that a legitimately accepted event still commits within + /// the floor. Defaults to [`replica_fence::CREATED_AT_FLOOR_SECS`]; + /// raised together with `BUZZ_MAX_PAST_DRIFT_SECS` for history imports. + pub created_at_floor_secs: i64, /// Replica read budget `B` in milliseconds (bounded arm, env /// `BUZZ_REPLICA_READ_MAX_AGE_MS`). `0` disables bounded-staleness /// routing — the rollout default. Values above @@ -543,6 +553,7 @@ impl Default for DbConfig { acquire_timeout_secs: 3, max_lifetime_secs: 1800, idle_timeout_secs: 600, + created_at_floor_secs: replica_fence::CREATED_AT_FLOOR_SECS, replica_read_max_age_ms: 0, } } @@ -662,6 +673,7 @@ impl Db { pool, max_connections: config.max_connections, read_pool, + created_at_floor_secs: config.created_at_floor_secs, read_max_connections, fence: std::sync::Arc::new(replica_fence::ReplicaFence::new()), replica_read_max_age, @@ -683,11 +695,12 @@ impl Db { .max_lifetime(Duration::from_secs(config.max_lifetime_secs)) .idle_timeout(Duration::from_secs(config.idle_timeout_secs)); if arm_floor_guard { - options = options.after_connect(|conn, _meta| { + let floor_secs = config.created_at_floor_secs; + options = options.after_connect(move |conn, _meta| { Box::pin(async move { // `SET` cannot take bind parameters; `set_config` can. sqlx::query("SELECT set_config('buzz.created_at_floor', $1, false)") - .bind(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) + .bind(floor_secs.to_string()) .execute(conn) .await?; Ok(()) @@ -778,6 +791,7 @@ impl Db { read_max_connections: pool.options().get_max_connections(), pool, read_pool: None, + created_at_floor_secs: replica_fence::CREATED_AT_FLOOR_SECS, fence: std::sync::Arc::new(replica_fence::ReplicaFence::new()), replica_read_max_age: None, reader_aurora_identity: std::sync::Arc::new(std::sync::OnceLock::new()), @@ -797,6 +811,7 @@ impl Db { read_max_connections: read_pool.options().get_max_connections(), pool, read_pool: Some(read_pool), + created_at_floor_secs: replica_fence::CREATED_AT_FLOOR_SECS, fence: std::sync::Arc::new(replica_fence::ReplicaFence::new()), replica_read_max_age: None, reader_aurora_identity: std::sync::Arc::new(std::sync::OnceLock::new()), @@ -835,10 +850,11 @@ impl Db { return Ok(false); } replica_fence::verify_floor_guard_catalog(&self.pool).await?; - replica_fence::verify_floor_guard_behavior(&self.pool).await?; + replica_fence::verify_floor_guard_behavior(&self.pool, self.created_at_floor_secs).await?; tokio::spawn(replica_fence::run_probe( self.pool.clone(), std::sync::Arc::clone(&self.fence), + self.created_at_floor_secs, )); Ok(true) } @@ -2075,14 +2091,45 @@ impl Db { channel_id: Option, thread_meta: Option>, ) -> Result<(StoredEvent, bool)> { + self.insert_event_with_thread_metadata_opts( + community_id, + event, + channel_id, + thread_meta, + false, + ) + .await + } + + /// [`Self::insert_event_with_thread_metadata`] with a `backfill` switch. + /// + /// `backfill: true` disarms the commit-time `created_at` floor guard for + /// this insert's transaction (authorized history imports) and brackets + /// the write with the replica fence's backfill guard so a floor-exempt + /// commit can never be claimed covered by a stale handshake sample. + pub async fn insert_event_with_thread_metadata_opts( + &self, + community_id: CommunityId, + event: &nostr::Event, + channel_id: Option, + thread_meta: Option>, + backfill: bool, + ) -> Result<(StoredEvent, bool)> { + // RAII: the guard ends the backfill even if this future is dropped + // mid-insert (a cancelled request), which would otherwise leave the + // fence closed forever. + let backfill_guard = backfill.then(|| self.fence.backfill()); let result = event::insert_event_with_thread_metadata( &self.pool, community_id, event, channel_id, thread_meta, + backfill, ) - .await?; + .await; + drop(backfill_guard); + let result = result?; if result.1 { if let Err(e) = insert_mentions(&self.pool, community_id, event, channel_id).await { tracing::warn!(event_id = %event.id, "Failed to insert mentions: {e}"); @@ -2092,6 +2139,10 @@ impl Db { } /// Atomically insert a kind:7 reaction event and its reaction row. + /// + /// `backfill: true` disarms the floor guard for this transaction and + /// brackets the write with the fence's backfill guard (see + /// [`Self::insert_event_with_thread_metadata_opts`]). #[allow(clippy::too_many_arguments)] pub async fn insert_reaction_event_with_thread_metadata( &self, @@ -2102,7 +2153,9 @@ impl Db { target_event_id: &[u8], actor_pubkey: &[u8], emoji: &str, + backfill: bool, ) -> Result { + let backfill_guard = backfill.then(|| self.fence.backfill()); let outcome = event::insert_reaction_event_with_thread_metadata( &self.pool, community_id, @@ -2112,8 +2165,11 @@ impl Db { target_event_id, actor_pubkey, emoji, + backfill, ) - .await?; + .await; + drop(backfill_guard); + let outcome = outcome?; if let event::ReactionEventInsertOutcome::Inserted { was_inserted: true, .. } = &outcome @@ -6566,6 +6622,7 @@ mod tests { depth: 0, broadcast: true, }), + false, ) .await .expect("insert top-level event"); @@ -6598,6 +6655,7 @@ mod tests { depth: 1, broadcast: false, }), + false, ) .await .expect("insert reply"); @@ -8369,6 +8427,7 @@ mod tests { depth: 0, broadcast: true, }), + false, ) .await .expect_err("armed pool must reject below-floor thread-metadata inserts"); diff --git a/crates/buzz-db/src/replica_fence.rs b/crates/buzz-db/src/replica_fence.rs index c840db393e..57799385a8 100644 --- a/crates/buzz-db/src/replica_fence.rs +++ b/crates/buzz-db/src/replica_fence.rs @@ -56,6 +56,7 @@ //! the fence closed for their duration; see `migrations/0021`. use std::collections::VecDeque; +use std::sync::atomic::{AtomicI64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -184,6 +185,17 @@ impl ResolveOutcome { #[derive(Debug, Default)] pub struct ReplicaFence { inner: Mutex, + /// Number of backfill (floor-exempt) inserts currently in flight. + /// + /// A backfill commit writes a row far below the armed floor, which no + /// retained entry's bucket argument can account for — so while any + /// backfill is in flight, and for any probe sample taken before the last + /// backfill finished, recording a fence entry is refused (see + /// [`Self::backfill_begin`]). + backfill_inflight: AtomicI64, + /// Unix micros when the most recent backfill insert finished, or `0` + /// before the first backfill. + backfill_watermark_micros: AtomicI64, } impl ReplicaFence { @@ -198,6 +210,52 @@ impl ReplicaFence { inner.ring.clear(); } + /// Mark a backfill (floor-exempt) insert as starting: closes the fence + /// and blocks new records until [`Self::backfill_end`] moves the + /// watermark. + /// + /// Backfilled rows carry `created_at` far below the armed floor, so no + /// retained entry's bucket argument bounds them: the backfill can commit + /// *after* every retained token while its rows fall under every retained + /// fence wall. Clearing the ring revokes those proofs, and refusing to + /// record from any sample taken while a backfill is in flight — or taken + /// before the last backfill finished — restores the proof: only a sample + /// whose token commit provably follows the backfill commit can reopen + /// the fence, and by then the commit is bucket (a). + pub fn backfill_begin(&self) { + self.backfill_inflight.fetch_add(1, Ordering::SeqCst); + self.close(); + } + + /// Mark a backfill insert as finished (whether it committed or failed). + pub fn backfill_end(&self) { + self.backfill_watermark_micros + .store(Utc::now().timestamp_micros(), Ordering::SeqCst); + self.backfill_inflight.fetch_sub(1, Ordering::SeqCst); + } + + /// Scope a backfill insert: [`Self::backfill_begin`] now, + /// [`Self::backfill_end`] when the returned guard drops. + /// + /// Prefer this over the raw pair: an ingest future can be dropped + /// mid-insert (an HTTP client disconnecting cancels the request future), + /// and a missed `backfill_end` leaks the in-flight count, leaving the + /// fence permanently closed for the rest of the process's life. + pub fn backfill(&self) -> BackfillGuard<'_> { + self.backfill_begin(); + BackfillGuard(self) + } + + /// Whether a probe sample whose `S` was captured at `sampled_at` may + /// record a fence entry. + fn sample_admissible(&self, sampled_at: DateTime) -> bool { + if self.backfill_inflight.load(Ordering::SeqCst) > 0 { + return false; + } + let watermark = self.backfill_watermark_micros.load(Ordering::SeqCst); + watermark == 0 || sampled_at.timestamp_micros() > watermark + } + /// Record one probe sample. Epoch changes (re-seed) clear the ring and /// start a new one under the observed epoch — sound, because an entry /// only proves commits on its own timeline and readers must match the @@ -307,6 +365,17 @@ impl ReplicaFence { } } +/// RAII scope for one backfill (floor-exempt) insert — see +/// [`ReplicaFence::backfill`]. Ends the backfill on drop, including when the +/// enclosing future is cancelled instead of completing. +pub struct BackfillGuard<'a>(&'a ReplicaFence); + +impl Drop for BackfillGuard<'_> { + fn drop(&mut self) { + self.0.backfill_end(); + } +} + /// Catalog-level verification that the commit-time floor guard (migration /// 0021) is present and correctly shaped on the `events` parent AND every /// partition: right function, `DEFERRABLE INITIALLY DEFERRED`, row-level, @@ -359,7 +428,7 @@ pub async fn verify_floor_guard_catalog(pool: &PgPool) -> crate::Result<()> { /// pool; this proves the semantics the fence proof cites, inside one /// rolled-back transaction: /// -/// 1. the pool's session GUC equals [`CREATED_AT_FLOOR_SECS`] (arming); +/// 1. the pool's session GUC equals the configured `floor_secs` (arming); /// 2. an old channel-bearing INSERT raises `check_violation` (23514); /// 3. a fresh channel-bearing INSERT commits; /// 4. rewriting a fresh row's `created_at` below the floor raises; @@ -369,7 +438,7 @@ pub async fn verify_floor_guard_catalog(pool: &PgPool) -> crate::Result<()> { /// `SET CONSTRAINTS ALL IMMEDIATE` makes the deferred trigger fire per /// statement so each adversary is observable under a savepoint; deferral to /// COMMIT is separately pinned by the held-transaction fixture. -pub async fn verify_floor_guard_behavior(pool: &PgPool) -> crate::Result<()> { +pub async fn verify_floor_guard_behavior(pool: &PgPool, floor_secs: i64) -> crate::Result<()> { use crate::error::DbError; let expect_violation = |res: Result, @@ -397,9 +466,9 @@ pub async fn verify_floor_guard_behavior(pool: &PgPool) -> crate::Result<()> { "buzz.created_at_floor GUC not set on this pool: {e}" )) })?; - if armed != CREATED_AT_FLOOR_SECS.to_string() { + if armed != floor_secs.to_string() { return Err(DbError::InvalidData(format!( - "buzz.created_at_floor is '{armed}', expected '{CREATED_AT_FLOOR_SECS}': \ + "buzz.created_at_floor is '{armed}', expected '{floor_secs}': \ pool is not armed" ))); } @@ -431,7 +500,7 @@ pub async fn verify_floor_guard_behavior(pool: &PgPool) -> crate::Result<()> { .bind(vec![0u8; 64]) .bind(ch) }; - let old_age = CREATED_AT_FLOOR_SECS + 60; + let old_age = floor_secs + 60; // 2. Old channel-bearing insert must raise. sqlx::query("SAVEPOINT floor_probe") @@ -626,33 +695,58 @@ async fn sample_writer(writer: &PgPool) -> Result { /// The fence wall proved by one handshake: /// `min(oldest_xact_start, S) - floor - clock_margin`. -fn fence_wall(sample_s: DateTime, oldest_xact_start: Option>) -> DateTime { +/// +/// `floor_secs` is the commit-time floor armed on the writer pool +/// ([`crate::DbConfig::created_at_floor_secs`], default +/// [`CREATED_AT_FLOOR_SECS`]) — the wall must subtract the same value the +/// guard enforces, so it is threaded through rather than read from the +/// constant. +fn fence_wall( + sample_s: DateTime, + oldest_xact_start: Option>, + floor_secs: i64, +) -> DateTime { let lower = match oldest_xact_start { Some(oldest) => oldest.min(sample_s), None => sample_s, }; lower - - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS) + - chrono::Duration::seconds(floor_secs) - chrono::Duration::seconds(FENCE_CLOCK_MARGIN_SECS) } /// Run one full handshake and record the resulting `(token, fence_wall)`. /// +/// `Ok(None)` means the sample was refused because a backfill (floor-exempt) +/// insert was in flight, or finished after the sample's `S` was captured — +/// the handshake's bucket argument cannot bound the backfilled rows, so +/// nothing is recorded; the next interval samples fresh. +/// /// On a same-epoch token regression (the writer was restored from a backup /// that kept its epoch), the retained ring has already been cleared by /// [`ReplicaFence::record`]; this additionally **rotates the epoch** on the /// writer and records the rotated token, so a reader still serving the /// pre-rewind timeline (whose old, higher token would otherwise satisfy /// `token >= M`) fails the epoch check instead of proving stale coverage. -pub async fn probe_once(writer: &PgPool, fence: &ReplicaFence) -> Result { +pub async fn probe_once( + writer: &PgPool, + fence: &ReplicaFence, + floor_secs: i64, +) -> Result, ProbeError> { let sample = sample_writer(writer).await?; - let wall = fence_wall(sample.sampled_at, sample.oldest_xact_start); - match fence.record(sample.token, sample.epoch, sample.committed_at, wall) { - RecordOutcome::Recorded => Ok(TokenEntry { + // A sample taken during (or before the end of) a backfill insert cannot + // prove coverage of the backfilled rows — skip; the next interval + // samples fresh. + if !fence.sample_admissible(sample.sampled_at) { + return Ok(None); + } + let wall = fence_wall(sample.sampled_at, sample.oldest_xact_start, floor_secs); + let entry = match fence.record(sample.token, sample.epoch, sample.committed_at, wall) { + RecordOutcome::Recorded => TokenEntry { token: sample.token, committed_at: sample.committed_at, fence_wall: wall, - }), + }, RecordOutcome::TokenRegression => { tracing::warn!( token = sample.token, @@ -674,13 +768,23 @@ pub async fn probe_once(writer: &PgPool, fence: &ReplicaFence) -> Result) { +pub async fn run_probe(writer: PgPool, fence: Arc, floor_secs: i64) { let mut interval = tokio::time::interval(PROBE_INTERVAL); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { interval.tick().await; - match probe_once(&writer, &fence).await { - Ok(_) => {} + match probe_once(&writer, &fence, floor_secs).await { + Ok(Some(_)) => {} + Ok(None) => { + // Sample refused inside a backfill window: the fence stays + // closed; the next interval samples fresh. + tracing::debug!("replica fence: sample refused during backfill window"); + } Err(e) => { fence.close(); tracing::warn!(error = %e, "replica fence probe failed; fence closed"); @@ -839,6 +948,44 @@ mod tests { assert!(!fence.covers(ts - chrono::Duration::days(365))); } + #[test] + fn backfill_blocks_stale_samples_from_advancing() { + let fence = ReplicaFence::new(); + + // No backfill yet: any sample is admissible. + let before = Utc::now(); + assert!(fence.sample_admissible(before)); + + // In-flight backfill: nothing is admissible and the fence closes. + fence.record(1, Uuid::new_v4(), Instant::now(), before); + assert!(fence.verified_through().is_some(), "entry retained"); + fence.backfill_begin(); + assert!(fence.verified_through().is_none(), "backfill closes fence"); + assert!(!fence.sample_admissible(Utc::now())); + + // Finished: samples taken at/before the watermark stay refused; + // fresh samples are admissible again. + fence.backfill_end(); + assert!(!fence.sample_admissible(before)); + let after = Utc::now() + chrono::Duration::seconds(1); + assert!(fence.sample_admissible(after)); + } + + #[test] + fn dropping_the_backfill_guard_reopens_admissibility() { + // A cancelled insert drops the guard without running any explicit + // end call; the in-flight count must still return to zero, or the + // fence never advances again. + let fence = ReplicaFence::new(); + let before = Utc::now(); + { + let _guard = fence.backfill(); + assert!(!fence.sample_admissible(Utc::now())); + } + assert!(!fence.sample_admissible(before), "watermark still applies"); + assert!(fence.sample_admissible(Utc::now() + chrono::Duration::seconds(1))); + } + #[test] fn stale_fence_reads_as_closed() { let fence = ReplicaFence::new(); @@ -1091,8 +1238,14 @@ mod tests { let (admin, pool, name) = scratch_db().await; let fence = ReplicaFence::new(); - let first = probe_once(&pool, &fence).await.expect("first probe"); - let second = probe_once(&pool, &fence).await.expect("second probe"); + let first = probe_once(&pool, &fence, CREATED_AT_FLOOR_SECS) + .await + .expect("first probe") + .expect("no backfill in flight"); + let second = probe_once(&pool, &fence, CREATED_AT_FLOOR_SECS) + .await + .expect("second probe") + .expect("no backfill in flight"); assert!(second.token > first.token, "tokens strictly increase"); assert!( second.fence_wall >= first.fence_wall @@ -1146,7 +1299,10 @@ mod tests { let (admin, pool, name) = scratch_db().await; let fence = ReplicaFence::new(); - let before = probe_once(&pool, &fence).await.expect("probe"); + let before = probe_once(&pool, &fence, CREATED_AT_FLOOR_SECS) + .await + .expect("probe") + .expect("no backfill in flight"); let mut conn = pool.acquire().await.expect("conn"); let old_epoch = observe_heartbeat(&mut conn, false) .await @@ -1160,7 +1316,10 @@ mod tests { .await .expect("rewind token"); - let after = probe_once(&pool, &fence).await.expect("recovery probe"); + let after = probe_once(&pool, &fence, CREATED_AT_FLOOR_SECS) + .await + .expect("recovery probe") + .expect("no backfill in flight"); // The pre-rewind observation must no longer prove anything. assert_eq!( fence.resolve(before.token, old_epoch), diff --git a/crates/buzz-db/src/thread.rs b/crates/buzz-db/src/thread.rs index 007677e258..acdadde7c7 100644 --- a/crates/buzz-db/src/thread.rs +++ b/crates/buzz-db/src/thread.rs @@ -1008,6 +1008,7 @@ mod tests { depth: 0, broadcast: true, }), + false, ) .await .expect("insert community A metadata"); @@ -1027,6 +1028,7 @@ mod tests { depth: 3, broadcast: false, }), + false, ) .await .expect("insert community B metadata"); @@ -1065,7 +1067,7 @@ mod tests { let root = make_stream_event(&author, "root"); let root_created_at = event_created_at(&root); - insert_event_with_thread_metadata(&pool, community, &root, Some(channel.id), None) + insert_event_with_thread_metadata(&pool, community, &root, Some(channel.id), None, false) .await .expect("insert root event"); @@ -1088,6 +1090,7 @@ mod tests { depth: 1, broadcast: false, }), + false, ) .await .expect("insert reply event and metadata"); @@ -1130,7 +1133,7 @@ mod tests { let root = make_stream_event(&author, "root"); let root_created_at = event_created_at(&root); - insert_event_with_thread_metadata(&pool, community, &root, Some(channel.id), None) + insert_event_with_thread_metadata(&pool, community, &root, Some(channel.id), None, false) .await .expect("insert root event"); @@ -1162,6 +1165,7 @@ mod tests { depth: 1, broadcast: false, }), + false, ) .await .expect("insert tied reply"); @@ -1245,7 +1249,7 @@ mod tests { // Root (no metadata row on first insert — a depth-0 message). let root = make_stream_event(&author, "root"); let root_created_at = event_created_at(&root); - insert_event_with_thread_metadata(&pool, community, &root, Some(channel.id), None) + insert_event_with_thread_metadata(&pool, community, &root, Some(channel.id), None, false) .await .expect("insert root event"); @@ -1268,6 +1272,7 @@ mod tests { depth: 1, broadcast: false, }), + false, ) .await .expect("insert depth-1 child"); @@ -1292,6 +1297,7 @@ mod tests { depth: 2, broadcast: false, }), + false, ) .await .expect("insert depth-2 grandchild"); @@ -1354,7 +1360,7 @@ mod tests { let child = make_stream_event(&author, "child"); let grandchild = make_stream_event(&author, "grandchild"); for ev in [&root, &child, &grandchild] { - insert_event_with_thread_metadata(&pool, community, ev, Some(channel.id), None) + insert_event_with_thread_metadata(&pool, community, ev, Some(channel.id), None, false) .await .expect("insert event"); } @@ -1412,7 +1418,7 @@ mod tests { let root = make_stream_event(&author, "root"); let root_created_at = event_created_at(&root); - insert_event_with_thread_metadata(&pool, community, &root, Some(channel.id), None) + insert_event_with_thread_metadata(&pool, community, &root, Some(channel.id), None, false) .await .expect("insert root event"); @@ -1436,6 +1442,7 @@ mod tests { depth: 1, broadcast: false, }), + false, ) .await .expect("insert good reply"); @@ -1458,6 +1465,7 @@ mod tests { depth: 1, broadcast: false, }), + false, ) .await .expect("insert bad reply"); @@ -1508,6 +1516,7 @@ mod tests { depth: 0, broadcast: true, }), + false, ) .await .expect("insert top-level event"); @@ -1538,6 +1547,7 @@ mod tests { depth: 1, broadcast, }), + false, ) .await .expect("insert reply event"); @@ -1570,7 +1580,7 @@ mod tests { // No thread metadata at all — the legacy-ingest shape. Top-level. let bare = make_stream_event(&author, "bare"); - insert_event_with_thread_metadata(&pool, community, &bare, Some(channel.id), None) + insert_event_with_thread_metadata(&pool, community, &bare, Some(channel.id), None, false) .await .expect("insert bare event"); diff --git a/crates/buzz-migrate/Cargo.toml b/crates/buzz-migrate/Cargo.toml new file mode 100644 index 0000000000..5c2cfc0bdc --- /dev/null +++ b/crates/buzz-migrate/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "buzz-migrate" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Operator claim-service for zero-touch, no-takeover Slack→Buzz identity migration" + +[lib] +name = "buzz_migrate" +path = "src/lib.rs" + +[[bin]] +name = "buzz-migrate" +path = "src/main.rs" + +[dependencies] +buzz-sdk = { workspace = true } +buzz-ws-client = { workspace = true } +buzz-core = { workspace = true } +nostr = { workspace = true } +tokio = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +hmac = { workspace = true } +sha2 = { workspace = true } +ring = { workspace = true } +hex = { workspace = true } +subtle = { workspace = true } +rand = { workspace = true } +axum = { workspace = true } +tower-http = { workspace = true } +reqwest = { workspace = true } +url = { workspace = true } +clap = { version = "4", features = ["derive", "env"] } +base64 = "0.22" diff --git a/crates/buzz-migrate/src/attest.rs b/crates/buzz-migrate/src/attest.rs new file mode 100644 index 0000000000..3df545219e --- /dev/null +++ b/crates/buzz-migrate/src/attest.rs @@ -0,0 +1,82 @@ +//! Publishing the admin-signed membership and identity events produced by a +//! verified migration claim. +//! +//! This is the one place the service uses the operator's admin key. The Slack +//! OAuth join path first publishes an idempotent member-add event, then a +//! `KIND_IMPORT_IDENTITY_BINDING` mapping the proven subject +//! (`slack::`) to the claimant's Buzz pubkey. Email recovery +//! publishes only the binding. +//! +//! The attestation is only *half* of a binding: nothing is attributed until the +//! claimant's own `KIND_IMPORT_IDENTITY_CLAIM` (self-signed) also exists. So a +//! stolen admin key can, at worst, publish attestations that stay inert without +//! each subject's separate consent. + +use buzz_core::kind::RELAY_ADMIN_ADD_MEMBER; +use nostr::{EventBuilder, Keys, Kind, Tag}; + +/// Why publishing an admin-signed event failed. +#[derive(Debug, thiserror::Error)] +pub enum AttestError { + #[error("could not build event: {0}")] + Build(String), + #[error("could not sign event: {0}")] + Sign(String), + #[error("relay rejected the event: {0}")] + Rejected(String), + #[error(transparent)] + Transport(#[from] buzz_ws_client::WsClientError), +} + +/// Add the claimant as a community member (kind 9030 relay-admin add-member). +/// +/// The relay derives the community from the connection, checks the signer is an +/// owner/admin, and — crucially — treats an already-present member as a silent +/// no-op, so re-running a claim never duplicates or downgrades membership. This +/// is the "join" half of a Slack-migration sign-in: the OAuth-verified person +/// is registered before their history is attributed. +pub async fn publish_add_member( + relay_url: &str, + admin: &Keys, + member_pubkey_hex: &str, + auth_tag: Option<&Tag>, +) -> Result { + let p = Tag::parse(["p", member_pubkey_hex]).map_err(|e| AttestError::Build(e.to_string()))?; + let role = Tag::parse(["role", "member"]).map_err(|e| AttestError::Build(e.to_string()))?; + let event = EventBuilder::new(Kind::Custom(RELAY_ADMIN_ADD_MEMBER as u16), "") + .tags([p, role]) + .sign_with_keys(admin) + .map_err(|e| AttestError::Sign(e.to_string()))?; + let event_id = event.id.to_hex(); + let ok = buzz_ws_client::publish_event(relay_url, event, admin, auth_tag, 75).await?; + if !ok.accepted { + return Err(AttestError::Rejected(ok.message)); + } + Ok(event_id) +} + +/// Sign and publish the attestation `subject → bound_pubkey_hex` with the +/// operator's `admin` key. Returns the published event id on success. +/// +/// `subject` is the binding key (`slack::`); `bound_pubkey_hex` is the +/// claimant's 64-char hex pubkey. `auth_tag` carries the relay's community +/// scope when one is required (same tag the CLI injects). +pub async fn publish_attestation( + relay_url: &str, + admin: &Keys, + subject: &str, + bound_pubkey_hex: &str, + auth_tag: Option<&Tag>, +) -> Result { + let builder = buzz_sdk::build_import_identity_binding(subject, bound_pubkey_hex) + .map_err(|e| AttestError::Build(e.to_string()))?; + let event = builder + .sign_with_keys(admin) + .map_err(|e| AttestError::Sign(e.to_string()))?; + let event_id = event.id.to_hex(); + let ok = buzz_ws_client::publish_event(relay_url, event, admin, auth_tag, 75).await?; + if !ok.accepted { + return Err(AttestError::Rejected(ok.message)); + } + Ok(event_id) +} diff --git a/crates/buzz-migrate/src/lib.rs b/crates/buzz-migrate/src/lib.rs new file mode 100644 index 0000000000..df6f983369 --- /dev/null +++ b/crates/buzz-migrate/src/lib.rs @@ -0,0 +1,38 @@ +//! `buzz-migrate` — the operator claim-service for zero-touch, no-takeover +//! Slack→Buzz identity migration. +//! +//! # The problem it solves +//! +//! History imported by `buzz import slack` is bot-signed and attributed only by +//! display name. To render each person's imported history under their real Buzz +//! profile, a **two-party binding** must exist for `slack::`: +//! +//! 1. an owner/admin **attestation** (kind `KIND_IMPORT_IDENTITY_BINDING`), and +//! 2. the subject's self-signed **claim** (kind `KIND_IMPORT_IDENTITY_CLAIM`). +//! +//! Doing (1) by hand (`buzz import bind`) is O(N) manual work for a large team, +//! and manual matching is exactly where account-takeover mistakes creep in. +//! This service automates (1): it proves *which* Slack user a person is, then +//! publishes the attestation for them. The Slack OIDC path also admits that +//! verified public key as a community member, making the dedicated +//! `buzz://join-slack` link the complete migration onboarding entry. +//! +//! # Two proof channels +//! +//! - **Email magic-link** ([`token`], [`roster`]): the operator's export knows +//! each user's email; a single-use, short-TTL token mailed to that address +//! proves control of it, hence of the Slack user. See [`token`] for the exact +//! threat model (why the token carries no pubkey). +//! - **Sign in with Slack (OIDC)**: the person authenticates to Slack live; the +//! service reads back the verified user id. (Built on top of this core in the +//! service layer.) +//! +//! Either way the outcome is one call to [`attest::publish_attestation`], and +//! the person's own client publishes the matching claim. Everything the service +//! signs is public-key-only and NIP-33-revocable. + +pub mod attest; +pub mod oidc; +pub mod roster; +pub mod server; +pub mod token; diff --git a/crates/buzz-migrate/src/main.rs b/crates/buzz-migrate/src/main.rs new file mode 100644 index 0000000000..784e5865fa --- /dev/null +++ b/crates/buzz-migrate/src/main.rs @@ -0,0 +1,349 @@ +//! `buzz-migrate` — the operator claim-service binary. +//! +//! Loads the Slack export roster, holds the operator's admin key, and serves +//! the claim HTTP surface. Its Slack OIDC path admits verified workspace +//! members and automates the owner/admin attestation half of a two-party import +//! identity binding. See the crate docs for the model. + +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use buzz_migrate::oidc::OidcConfig; +use buzz_migrate::roster::Roster; +use buzz_migrate::server::{router, AppState, Inner, Mailer, OidcSessions}; +use buzz_migrate::token::ConsumedNonces; +use clap::Parser; +use nostr::Keys; + +const SLACK_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +// Slack's OIDC endpoints are part of an interactive flow. Bound the complete +// request so an upstream failure returns control promptly and can be retried. +const SLACK_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +/// Operator claim-service for Slack→Buzz identity migration. +#[derive(Parser, Debug)] +#[command(name = "buzz-migrate", version, about)] +struct Args { + /// Relay base URL (http/https/ws/wss). The admin key must be a community + /// owner or admin on this relay. + #[arg(long, env = "BUZZ_RELAY_URL", default_value = "http://localhost:3000")] + relay_url: String, + + /// Operator admin private key (hex or nsec). Used only to sign attestations. + #[arg(long, env = "BUZZ_PRIVATE_KEY")] + admin_key: String, + + /// NIP-OA auth tag JSON (community membership delegation), if the relay + /// requires one. + #[arg(long, env = "BUZZ_AUTH_TAG")] + auth_tag: Option, + + /// Unzipped Slack export directory (must contain users.json). + #[arg(long)] + export_dir: PathBuf, + + /// Address to bind the HTTP service to. + #[arg(long, default_value = "127.0.0.1:8787")] + bind: String, + + /// Public base URL of this service, used in email and OIDC callbacks. + /// Defaults to `http://`. + #[arg(long)] + base_url: Option, + + /// Hex secret (>=32 bytes recommended) that signs magic-link tokens. If + /// omitted, a random one is generated — fine for a single run, but tokens + /// minted before a restart stop verifying. Set it to survive restarts. + #[arg(long, env = "BUZZ_MIGRATE_TOKEN_SECRET")] + token_secret: Option, + + /// Magic-link token lifetime in seconds (default 72h). + #[arg(long, default_value_t = 72 * 3600)] + token_ttl_secs: u64, + + /// Slack OIDC client id (enables the Sign-in-with-Slack channel). + #[arg(long, env = "SLACK_CLIENT_ID")] + slack_client_id: Option, + + /// Slack OIDC client secret. + #[arg(long, env = "SLACK_CLIENT_SECRET")] + slack_client_secret: Option, + + /// Slack workspace id (team id, e.g. T0266FRGM) whose users may claim + /// imported identities. Namespaces every `slack::` subject so + /// ids can't collide across workspaces — required on both channels. + #[arg(long, env = "SLACK_TEAM_ID")] + slack_team_id: String, + + /// OIDC redirect URI registered on the Slack app. Defaults to + /// `/oidc/callback`. + #[arg(long)] + oidc_redirect_uri: Option, + + /// Enable dev-only routes (e.g. /oidc/dev-complete) for local testing + /// without a real Slack app. Never set this in production. + #[arg(long)] + dev: bool, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| "buzz_migrate=info,tower_http=info".into()), + ) + .init(); + + let args = Args::parse(); + + let admin = Keys::parse(&args.admin_key) + .map_err(|e| format!("invalid --admin-key (hex or nsec): {e}"))?; + + let auth_tag = match args.auth_tag.as_deref() { + Some(json) => Some( + buzz_sdk::nip_oa::parse_auth_tag(json) + .map_err(|e| format!("invalid BUZZ_AUTH_TAG: {e}"))?, + ), + None => None, + }; + + let users_path = args.export_dir.join("users.json"); + let users_bytes = std::fs::read(&users_path) + .map_err(|e| format!("could not read {}: {e}", users_path.display()))?; + let roster = Roster::from_users_json(&users_bytes, &args.slack_team_id) + .map_err(|e| format!("could not parse {}: {e}", users_path.display()))?; + tracing::info!( + mailable = roster.mailable_count(), + "loaded Slack export roster" + ); + + let token_secret = match args.token_secret { + Some(hex_secret) => { + hex::decode(hex_secret.trim()).map_err(|_| "--token-secret must be hex")? + } + None => { + let s = rand::random::<[u8; 32]>().to_vec(); + tracing::warn!( + "no --token-secret set: generated an ephemeral one; links minted now will \ + stop verifying after a restart" + ); + s + } + }; + if token_secret.len() < 32 { + return Err("--token-secret must contain at least 32 bytes".into()); + } + if args.token_ttl_secs == 0 { + return Err("--token-ttl-secs must be greater than zero".into()); + } + + let base_url = normalize_service_origin( + &args + .base_url + .unwrap_or_else(|| format!("http://{}", args.bind)), + args.dev, + )?; + + let oidc = match (args.slack_client_id, args.slack_client_secret) { + (Some(client_id), Some(client_secret)) => { + let redirect_uri = args + .oidc_redirect_uri + .unwrap_or_else(|| format!("{base_url}/oidc/callback")); + validate_oidc_redirect_uri(&redirect_uri, args.dev)?; + tracing::info!(%redirect_uri, "OIDC channel enabled (Sign in with Slack)"); + Some(OidcConfig { + client_id, + client_secret, + redirect_uri, + team_id: args.slack_team_id.clone(), + }) + } + (None, None) => { + tracing::info!("OIDC channel disabled (no Slack OIDC credentials)"); + None + } + _ => { + return Err( + "set SLACK_CLIENT_ID and SLACK_CLIENT_SECRET together, or omit both".into(), + ); + } + }; + + if args.dev { + tracing::warn!("--dev enabled: /oidc/dev-complete is active; do NOT use in production"); + } else { + tracing::warn!( + "email channel disabled: no delivery backend is configured, so /email/start \ + answers 503. Use the Slack OIDC channel, or `buzz import bind` for manual \ + attribution" + ); + } + + let relay_url = to_ws_url(&args.relay_url); + if oidc.is_some() { + let join_url = slack_join_url(&relay_url, &base_url); + tracing::info!( + %join_url, + "share this migration link only with members of the imported Slack workspace" + ); + } + + let inner = Inner { + roster, + token_secret, + consumed: Mutex::new(ConsumedNonces::new()), + admin, + team_id: args.slack_team_id, + relay_url, + auth_tag, + base_url, + token_ttl_secs: args.token_ttl_secs, + mailer: if args.dev { + Mailer::Dev + } else { + Mailer::Disabled + }, + http: build_slack_http_client(SLACK_CONNECT_TIMEOUT, SLACK_REQUEST_TIMEOUT) + .map_err(|e| format!("could not build Slack HTTP client: {e}"))?, + oidc, + oidc_sessions: OidcSessions::default(), + dev: args.dev, + }; + let state = AppState(Arc::new(inner)); + + let listener = tokio::net::TcpListener::bind(&args.bind).await?; + tracing::info!(bind = %args.bind, "buzz-migrate claim-service listening"); + axum::serve(listener, router(state)).await?; + Ok(()) +} + +/// Convert an http(s) relay URL to its ws(s) equivalent for event publishing. +/// ws/wss URLs pass through unchanged. +fn to_ws_url(url: &str) -> String { + if let Some(rest) = url.strip_prefix("https://") { + format!("wss://{rest}") + } else if let Some(rest) = url.strip_prefix("http://") { + format!("ws://{rest}") + } else { + url.to_string() + } +} + +fn slack_join_url(relay_url: &str, service_origin: &str) -> String { + let query = url::form_urlencoded::Serializer::new(String::new()) + .append_pair("relay", relay_url) + .append_pair("service", service_origin) + .finish(); + format!("buzz://join-slack?{query}") +} + +fn build_slack_http_client( + connect_timeout: Duration, + request_timeout: Duration, +) -> reqwest::Result { + reqwest::Client::builder() + .connect_timeout(connect_timeout) + .timeout(request_timeout) + .redirect(reqwest::redirect::Policy::none()) + .user_agent(concat!("buzz-migrate/", env!("CARGO_PKG_VERSION"))) + .build() +} + +fn normalize_service_origin(value: &str, allow_insecure: bool) -> Result { + let parsed = url::Url::parse(value).map_err(|error| format!("invalid --base-url: {error}"))?; + if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() { + return Err("--base-url must be an http(s) URL with a host".into()); + } + if parsed.scheme() != "https" && !allow_insecure { + return Err("--base-url must use https unless --dev is enabled".into()); + } + if !parsed.username().is_empty() || parsed.password().is_some() { + return Err("--base-url must not include credentials".into()); + } + if parsed.query().is_some() || parsed.fragment().is_some() { + return Err("--base-url must not include a query or fragment".into()); + } + if parsed.path() != "/" { + return Err("--base-url must be an origin without a path".into()); + } + Ok(parsed.as_str().trim_end_matches('/').to_string()) +} + +fn validate_oidc_redirect_uri(value: &str, allow_insecure: bool) -> Result<(), String> { + let parsed = + url::Url::parse(value).map_err(|error| format!("invalid --oidc-redirect-uri: {error}"))?; + if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() { + return Err("--oidc-redirect-uri must be an http(s) URL with a host".into()); + } + if parsed.scheme() != "https" && !allow_insecure { + return Err("--oidc-redirect-uri must use https unless --dev is enabled".into()); + } + if !parsed.username().is_empty() || parsed.password().is_some() { + return Err("--oidc-redirect-uri must not include credentials".into()); + } + if parsed.query().is_some() || parsed.fragment().is_some() { + return Err("--oidc-redirect-uri must not include a query or fragment".into()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn http_urls_become_ws() { + assert_eq!(to_ws_url("http://localhost:3000"), "ws://localhost:3000"); + assert_eq!(to_ws_url("https://relay.example"), "wss://relay.example"); + assert_eq!(to_ws_url("ws://x:1"), "ws://x:1"); + assert_eq!(to_ws_url("wss://x"), "wss://x"); + } + + #[test] + fn slack_join_url_encodes_both_operator_origins() { + assert_eq!( + slack_join_url( + "wss://buzz.example.com", + "https://migrate.example.com" + ), + "buzz://join-slack?relay=wss%3A%2F%2Fbuzz.example.com&service=https%3A%2F%2Fmigrate.example.com" + ); + } + + #[test] + fn public_base_url_is_validated_and_normalized() { + assert_eq!( + normalize_service_origin("https://migrate.example/", false).unwrap(), + "https://migrate.example" + ); + assert_eq!( + normalize_service_origin("http://localhost:8787/", true).unwrap(), + "http://localhost:8787" + ); + assert!(normalize_service_origin("http://migrate.example", false).is_err()); + assert!(normalize_service_origin("file:///tmp/migrate", true).is_err()); + assert!(normalize_service_origin("https://user@migrate.example", false).is_err()); + assert!(normalize_service_origin("https://migrate.example/path", false).is_err()); + assert!(normalize_service_origin("https://migrate.example?next=evil", false).is_err()); + assert!(normalize_service_origin("https://migrate.example#fragment", false).is_err()); + } + + #[test] + fn oidc_redirect_uri_requires_https_outside_dev() { + assert!(validate_oidc_redirect_uri("https://migrate.example/oidc/callback", false).is_ok()); + assert!(validate_oidc_redirect_uri("http://localhost:8787/oidc/callback", true).is_ok()); + assert!(validate_oidc_redirect_uri("http://migrate.example/oidc/callback", false).is_err()); + assert!( + validate_oidc_redirect_uri("https://user@migrate.example/callback", false).is_err() + ); + } + + #[test] + fn slack_http_client_builds_with_bounded_timeouts() { + assert!(build_slack_http_client(SLACK_CONNECT_TIMEOUT, SLACK_REQUEST_TIMEOUT).is_ok()); + assert_eq!(SLACK_CONNECT_TIMEOUT, Duration::from_secs(5)); + assert_eq!(SLACK_REQUEST_TIMEOUT, Duration::from_secs(30)); + } +} diff --git a/crates/buzz-migrate/src/oidc.rs b/crates/buzz-migrate/src/oidc.rs new file mode 100644 index 0000000000..02d9e1a3d0 --- /dev/null +++ b/crates/buzz-migrate/src/oidc.rs @@ -0,0 +1,428 @@ +//! Sign in with Slack (OIDC). +//! +//! The server verifies the returned ID token (signature, issuer, audience, +//! expiry, nonce, workspace). Identity is then confirmed again through Slack's +//! authenticated userInfo endpoint before the migration service mints an app +//! handoff code. The separate app handoff is bound to a device-held verifier +//! in [`crate::server`]. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; +use ring::signature::{RsaPublicKeyComponents, RSA_PKCS1_2048_8192_SHA256}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +/// Slack OIDC application credentials. Absent means the `/oidc/*` routes are +/// disabled. +#[derive(Clone)] +pub struct OidcConfig { + /// Slack application's OAuth client id. + pub client_id: String, + /// Slack application's OAuth client secret. + pub client_secret: String, + /// Must exactly match a redirect URL registered on the Slack app. + pub redirect_uri: String, + /// The only Slack workspace whose identities may be bound. + pub team_id: String, +} + +const AUTHORIZE_URL: &str = "https://slack.com/openid/connect/authorize"; +const TOKEN_URL: &str = "https://slack.com/api/openid.connect.token"; +const USERINFO_URL: &str = "https://slack.com/api/openid.connect.userInfo"; +const JWKS_URL: &str = "https://slack.com/openid/connect/keys"; +const ISSUER: &str = "https://slack.com"; + +/// Build a Sign in with Slack authorization URL. +/// +/// This follows Slack's OpenID Connect endpoint contract. The migration app's +/// own device-verifier handoff is separate from Slack's authorization-code +/// exchange. +pub fn authorize_url(cfg: &OidcConfig, state: &str, nonce: &str) -> String { + format!( + "{AUTHORIZE_URL}?response_type=code&scope=openid&client_id={}&redirect_uri={}&state={}&nonce={}&team={}", + urlencode(&cfg.client_id), + urlencode(&cfg.redirect_uri), + urlencode(state), + urlencode(nonce), + urlencode(&cfg.team_id), + ) +} + +/// RFC 7636 S256 challenge for a high-entropy verifier. +pub fn pkce_challenge(verifier: &str) -> String { + URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes())) +} + +#[derive(Deserialize)] +struct TokenResp { + #[serde(default)] + ok: bool, + #[serde(default)] + access_token: Option, + #[serde(default)] + id_token: Option, + #[serde(default)] + error: Option, +} + +#[derive(Deserialize)] +struct UserInfoResp { + #[serde(default)] + ok: bool, + #[serde(rename = "https://slack.com/user_id", default)] + user_id: Option, + #[serde(rename = "https://slack.com/team_id", default)] + team_id: Option, + #[serde(default)] + error: Option, +} + +#[derive(Debug, Deserialize)] +struct IdTokenClaims { + iss: String, + aud: Audience, + exp: u64, + nonce: String, + #[serde(rename = "https://slack.com/user_id")] + user_id: String, + #[serde(rename = "https://slack.com/team_id")] + team_id: String, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum Audience { + One(String), + Many(Vec), +} + +impl Audience { + fn contains(&self, expected: &str) -> bool { + match self { + Self::One(value) => value == expected, + Self::Many(values) => values.iter().any(|value| value == expected), + } + } +} + +#[derive(Deserialize)] +struct IdTokenHeader { + alg: String, + kid: String, +} + +#[derive(Deserialize)] +struct JwkSet { + keys: Vec, +} + +#[derive(Deserialize)] +struct Jwk { + kty: String, + kid: String, + #[serde(default)] + alg: Option, + n: String, + e: String, +} + +/// Why an OIDC exchange failed. +#[derive(Debug, thiserror::Error)] +pub enum OidcError { + #[error("slack token exchange failed: {0}")] + Token(String), + #[error("slack userinfo failed: {0}")] + UserInfo(String), + #[error("slack did not return a user id")] + NoUserId, + #[error("slack OIDC nonce did not match")] + NonceMismatch, + #[error("slack OIDC id_token is invalid: {0}")] + InvalidIdToken(String), + #[error("could not verify Slack OIDC signing keys: {0}")] + SigningKeys(String), + #[error("authenticated Slack workspace {actual:?} does not match configured workspace")] + WrongTeam { actual: Option }, + #[error(transparent)] + Http(#[from] reqwest::Error), +} + +/// Exchange an authorization code, verify the OIDC response, and return the +/// workspace-scoped `slack::` subject. +pub async fn exchange_code_for_subject( + http: &reqwest::Client, + cfg: &OidcConfig, + code: &str, + expected_nonce: &str, +) -> Result { + if code.is_empty() { + return Err(OidcError::Token("missing authorization code".into())); + } + let body = token_request_body(cfg, code); + let token: TokenResp = http + .post(TOKEN_URL) + .header("content-type", "application/x-www-form-urlencoded") + .body(body) + .send() + .await? + .error_for_status()? + .json() + .await?; + if !token.ok { + return Err(OidcError::Token( + token.error.unwrap_or_else(|| "unknown".into()), + )); + } + + let access = token + .access_token + .ok_or_else(|| OidcError::Token("no access_token".into()))?; + let claims = verify_id_token( + http, + cfg, + token + .id_token + .as_deref() + .ok_or_else(|| OidcError::InvalidIdToken("missing token".into()))?, + expected_nonce, + unix_now(), + ) + .await?; + + let info: UserInfoResp = http + .get(USERINFO_URL) + .bearer_auth(&access) + .send() + .await? + .error_for_status()? + .json() + .await + .map_err(|error| OidcError::UserInfo(error.to_string()))?; + if !info.ok { + return Err(OidcError::UserInfo( + info.error.unwrap_or_else(|| "unknown".into()), + )); + } + if info.team_id.as_deref() != Some(cfg.team_id.as_str()) { + return Err(OidcError::WrongTeam { + actual: info.team_id, + }); + } + let user_id = info + .user_id + .filter(|value| !value.is_empty()) + .ok_or(OidcError::NoUserId)?; + if claims.team_id != cfg.team_id || claims.user_id != user_id { + return Err(OidcError::InvalidIdToken( + "ID token and userInfo identity did not agree".into(), + )); + } + Ok(format!("slack:{}:{user_id}", cfg.team_id)) +} + +fn token_request_body(cfg: &OidcConfig, code: &str) -> String { + let body = [ + ("client_id", cfg.client_id.as_str()), + ("client_secret", cfg.client_secret.as_str()), + ("code", code), + ("redirect_uri", cfg.redirect_uri.as_str()), + ] + .iter() + .map(|(key, value)| format!("{}={}", urlencode(key), urlencode(value))) + .collect::>() + .join("&"); + body +} + +async fn verify_id_token( + http: &reqwest::Client, + cfg: &OidcConfig, + id_token: &str, + expected_nonce: &str, + now: u64, +) -> Result { + let mut parts = id_token.split('.'); + let (Some(header_wire), Some(claims_wire), Some(signature_wire), None) = + (parts.next(), parts.next(), parts.next(), parts.next()) + else { + return Err(OidcError::InvalidIdToken("malformed compact JWT".into())); + }; + let header: IdTokenHeader = decode_json_part(header_wire)?; + if header.alg != "RS256" || header.kid.is_empty() { + return Err(OidcError::InvalidIdToken( + "unsupported signing algorithm or missing key id".into(), + )); + } + + let jwks: JwkSet = http + .get(JWKS_URL) + .send() + .await? + .error_for_status()? + .json() + .await + .map_err(|error| OidcError::SigningKeys(error.to_string()))?; + let key = jwks + .keys + .iter() + .find(|key| { + key.kid == header.kid + && key.kty == "RSA" + && key + .alg + .as_deref() + .is_none_or(|algorithm| algorithm == "RS256") + }) + .ok_or_else(|| OidcError::SigningKeys("matching RSA key not found".into()))?; + let modulus = decode_base64url(&key.n)?; + let exponent = decode_base64url(&key.e)?; + let signature = decode_base64url(signature_wire)?; + let signed = format!("{header_wire}.{claims_wire}"); + RsaPublicKeyComponents { + n: &modulus, + e: &exponent, + } + .verify(&RSA_PKCS1_2048_8192_SHA256, signed.as_bytes(), &signature) + .map_err(|_| OidcError::InvalidIdToken("signature verification failed".into()))?; + + let claims: IdTokenClaims = decode_json_part(claims_wire)?; + validate_id_token_claims(&claims, cfg, expected_nonce, now)?; + Ok(claims) +} + +fn validate_id_token_claims( + claims: &IdTokenClaims, + cfg: &OidcConfig, + expected_nonce: &str, + now: u64, +) -> Result<(), OidcError> { + if claims.iss != ISSUER { + return Err(OidcError::InvalidIdToken("issuer mismatch".into())); + } + if !claims.aud.contains(&cfg.client_id) { + return Err(OidcError::InvalidIdToken("audience mismatch".into())); + } + if claims.exp <= now { + return Err(OidcError::InvalidIdToken("token expired".into())); + } + if claims.nonce != expected_nonce { + return Err(OidcError::NonceMismatch); + } + if claims.team_id != cfg.team_id || claims.user_id.is_empty() { + return Err(OidcError::WrongTeam { + actual: Some(claims.team_id.clone()), + }); + } + Ok(()) +} + +fn decode_json_part Deserialize<'de>>(part: &str) -> Result { + let decoded = decode_base64url(part)?; + serde_json::from_slice(&decoded) + .map_err(|_| OidcError::InvalidIdToken("malformed JWT JSON".into())) +} + +fn decode_base64url(value: &str) -> Result, OidcError> { + URL_SAFE_NO_PAD + .decode(value) + .map_err(|_| OidcError::InvalidIdToken("malformed base64url".into())) +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or(0) +} + +fn urlencode(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for byte in value.bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(byte as char); + } + _ => out.push_str(&format!("%{byte:02X}")), + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cfg() -> OidcConfig { + OidcConfig { + client_id: "123.456".into(), + client_secret: "shh".into(), + redirect_uri: "https://mig.example/oidc/callback".into(), + team_id: "T060".into(), + } + } + + #[test] + fn authorize_url_encodes_openid_params() { + let url = authorize_url(&cfg(), "st ate", "non/ce"); + assert!(url.starts_with(AUTHORIZE_URL)); + assert!(url.contains("client_id=123.456")); + assert!(url.contains("redirect_uri=https%3A%2F%2Fmig.example%2Foidc%2Fcallback")); + assert!(url.contains("state=st%20ate")); + assert!(url.contains("nonce=non%2Fce")); + assert!(url.contains("team=T060")); + assert!(url.contains("scope=openid")); + assert!(!url.contains("code_challenge")); + } + + #[test] + fn token_request_matches_sign_in_with_slack_contract() { + assert_eq!( + token_request_body(&cfg(), "co/de"), + "client_id=123.456&client_secret=shh&code=co%2Fde&redirect_uri=https%3A%2F%2Fmig.example%2Foidc%2Fcallback" + ); + } + + #[test] + fn pkce_challenge_matches_rfc_7636_example() { + assert_eq!( + pkce_challenge("dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk"), + "E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM" + ); + } + + #[test] + fn userinfo_reads_namespaced_user_id() { + let info: UserInfoResp = serde_json::from_str( + r#"{"ok":true,"sub":"x","https://slack.com/user_id":"U060", + "https://slack.com/team_id":"T060"}"#, + ) + .expect("parse"); + assert_eq!(info.user_id.as_deref(), Some("U060")); + assert_eq!(info.team_id.as_deref(), Some("T060")); + } + + #[test] + fn token_resp_surfaces_error() { + let token: TokenResp = + serde_json::from_str(r#"{"ok":false,"error":"bad_code"}"#).expect("parse"); + assert!(!token.ok); + assert_eq!(token.error.as_deref(), Some("bad_code")); + } + + #[test] + fn id_token_claims_validate_all_security_fields() { + let claims = IdTokenClaims { + iss: ISSUER.into(), + aud: Audience::One(cfg().client_id.clone()), + exp: 2_000, + nonce: "nonce".into(), + user_id: "U060".into(), + team_id: "T060".into(), + }; + assert!(validate_id_token_claims(&claims, &cfg(), "nonce", 1_000).is_ok()); + assert!(validate_id_token_claims(&claims, &cfg(), "wrong", 1_000).is_err()); + assert!(validate_id_token_claims(&claims, &cfg(), "nonce", 2_000).is_err()); + } +} diff --git a/crates/buzz-migrate/src/roster.rs b/crates/buzz-migrate/src/roster.rs new file mode 100644 index 0000000000..e6aff86c92 --- /dev/null +++ b/crates/buzz-migrate/src/roster.rs @@ -0,0 +1,169 @@ +//! The Slack export roster: the trusted mapping from an email address to a +//! Slack user id, loaded from `users.json`. +//! +//! This mapping is what makes the email channel an *identity* proof rather than +//! just an email-ownership proof: the operator's own export says "this email +//! belongs to Slack user U060", so proving control of the email proves control +//! of U060. The roster is loaded once at service start from the same export the +//! history import used. + +use serde::Deserialize; +use std::collections::{HashMap, HashSet}; + +#[derive(Deserialize)] +struct RawUser { + id: String, + #[serde(default)] + deleted: bool, + #[serde(default)] + profile: RawProfile, +} + +#[derive(Default, Deserialize)] +struct RawProfile { + #[serde(default)] + email: String, + #[serde(default)] + display_name: String, + #[serde(default)] + real_name: String, +} + +/// Email→subject and subject→name lookups for the active migration. +#[derive(Debug, Default, Clone)] +pub struct Roster { + /// lowercased, trimmed email → `slack::`. + email_to_subject: HashMap, + /// `slack::` → best human-readable name (for email copy). + subject_to_name: HashMap, + /// Emails shared by more than one active Slack user. These are never + /// eligible for magic-link attribution because ownership is ambiguous. + ambiguous_emails: HashSet, +} + +impl Roster { + /// Build a roster from the bytes of a Slack export `users.json`. + /// + /// Deactivated (`deleted`) users are skipped: their email can no longer + /// receive the magic link, so they can only be attributed by the OIDC + /// channel or a manual `buzz import bind`. + pub fn from_users_json(bytes: &[u8], team_id: &str) -> Result { + let users: Vec = serde_json::from_slice(bytes)?; + let mut email_to_subject = HashMap::new(); + let mut subject_to_name = HashMap::new(); + let mut ambiguous_emails = HashSet::new(); + for u in users { + if u.deleted { + continue; + } + let subject = format!("slack:{team_id}:{}", u.id); + let name = if !u.profile.display_name.is_empty() { + u.profile.display_name + } else if !u.profile.real_name.is_empty() { + u.profile.real_name + } else { + u.id.clone() + }; + subject_to_name.insert(subject.clone(), name); + let email = u.profile.email.trim().to_lowercase(); + if email.is_empty() || ambiguous_emails.contains(&email) { + continue; + } + if email_to_subject.insert(email.clone(), subject).is_some() { + email_to_subject.remove(&email); + ambiguous_emails.insert(email); + } + } + Ok(Self { + email_to_subject, + subject_to_name, + ambiguous_emails, + }) + } + + /// The `slack::` subject for an email, if the export knows it. Matching + /// is case-insensitive and whitespace-trimmed. + pub fn subject_for_email(&self, email: &str) -> Option<&str> { + let email = email.trim().to_lowercase(); + // Defense in depth: shared emails are already absent from the map, but + // reject them explicitly so an ambiguous address can never resolve. + if self.ambiguous_emails.contains(&email) { + return None; + } + self.email_to_subject.get(&email).map(String::as_str) + } + + /// The display name for a subject, for personalizing the email. Held for + /// the pending mail-delivery backend — [`Mailer::Disabled`] and the dev + /// mailer send no copy, so nothing calls it yet. + /// + /// [`Mailer::Disabled`]: crate::server::Mailer::Disabled + pub fn name_for_subject(&self, subject: &str) -> Option<&str> { + self.subject_to_name.get(subject).map(String::as_str) + } + + /// Number of mailable (non-deactivated, has-email) users. + pub fn mailable_count(&self) -> usize { + self.email_to_subject.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const USERS: &str = r#"[ + {"id":"U060","profile":{"email":"Alice@Corp.com","display_name":"Alice"}}, + {"id":"U081","profile":{"email":" bob@corp.com ","real_name":"Bob B"}}, + {"id":"U099","deleted":true,"profile":{"email":"ghost@corp.com","display_name":"Ghost"}}, + {"id":"U100","profile":{"display_name":"NoEmail"}} + ]"#; + + #[test] + fn maps_email_to_subject_case_insensitively() { + let r = Roster::from_users_json(USERS.as_bytes(), "T1").unwrap(); + assert_eq!(r.subject_for_email("alice@corp.com"), Some("slack:T1:U060")); + // Original casing and surrounding whitespace both normalize. + assert_eq!( + r.subject_for_email(" BOB@CORP.COM "), + Some("slack:T1:U081") + ); + } + + #[test] + fn deactivated_users_are_excluded() { + let r = Roster::from_users_json(USERS.as_bytes(), "T1").unwrap(); + assert_eq!(r.subject_for_email("ghost@corp.com"), None); + } + + #[test] + fn users_without_email_are_not_mailable_but_named() { + let r = Roster::from_users_json(USERS.as_bytes(), "T1").unwrap(); + assert_eq!(r.mailable_count(), 2); // U060, U081 + assert_eq!(r.name_for_subject("slack:T1:U100"), Some("NoEmail")); + } + + #[test] + fn best_name_falls_back_display_then_real_then_id() { + let r = Roster::from_users_json(USERS.as_bytes(), "T1").unwrap(); + assert_eq!(r.name_for_subject("slack:T1:U060"), Some("Alice")); + assert_eq!(r.name_for_subject("slack:T1:U081"), Some("Bob B")); + } + + #[test] + fn unknown_email_is_none() { + let r = Roster::from_users_json(USERS.as_bytes(), "T1").unwrap(); + assert_eq!(r.subject_for_email("nobody@corp.com"), None); + } + + #[test] + fn duplicate_email_is_ambiguous_and_not_mailable() { + let users = r#"[ + {"id":"U1","profile":{"email":"shared@corp.com"}}, + {"id":"U2","profile":{"email":"SHARED@corp.com"}} + ]"#; + let r = Roster::from_users_json(users.as_bytes(), "T1").unwrap(); + assert_eq!(r.subject_for_email("shared@corp.com"), None); + assert_eq!(r.mailable_count(), 0); + } +} diff --git a/crates/buzz-migrate/src/server.rs b/crates/buzz-migrate/src/server.rs new file mode 100644 index 0000000000..71bf9e15b0 --- /dev/null +++ b/crates/buzz-migrate/src/server.rs @@ -0,0 +1,1184 @@ +//! The Slack migration claim-service HTTP surface. +//! +//! Email magic links and Slack OIDC share the same operator identity, +//! workspace roster, relay writer, and bounded session ledgers. +//! +//! - `POST /email/start {email}` — mint a magic-link token for the Slack user +//! that email belongs to and mail it there. Deliberately takes **no pubkey** +//! and always answers the same way whether or not the email is known, so it +//! can neither be used to enumerate the workspace nor to smuggle an attacker +//! pubkey (see [`crate::token`]). +//! - `GET /email/verify?token=…` — the link target. Validates the token and +//! hands off to the recipient's Buzz app via a `buzz://import-claim` deep +//! link. Does not consume the token (the app completes the claim). +//! - `POST /email/complete {token, pubkey}` — the app calls this with **its +//! own** key. Re-verifies, atomically consumes (single use), then publishes +//! the owner/admin attestation `subject → pubkey`. The app separately +//! publishes the matching self-claim; only then is history attributed. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use axum::extract::{DefaultBodyLimit, Query, State}; +use axum::http::{header, HeaderName, HeaderValue, StatusCode}; +use axum::response::{Html, IntoResponse, Redirect, Response}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; +use nostr::{Event, JsonUtil, Keys, Tag}; +use serde::{Deserialize, Serialize}; +use subtle::ConstantTimeEq; +use tower_http::cors::CorsLayer; + +use crate::oidc::{self, OidcConfig}; +use crate::roster::Roster; +use crate::token::{self, ConsumedNonces, MagicToken, TokenError, VerifiedToken}; + +/// How long an OIDC `state` is valid between `/oidc/start` and the callback. +const OIDC_STATE_TTL_SECS: u64 = 600; +/// How long a post-callback finalize `code` is valid. The first valid +/// redemption binds it to one key; only that key may retry. +const OIDC_CODE_TTL_SECS: u64 = 300; +/// Bound unauthenticated `/oidc/start` memory use (and the pending-code map). +/// Operators should also rate-limit the public endpoint at their reverse proxy. +const MAX_PENDING_OIDC_STATES: usize = 4096; + +/// A callback code becomes permanently bound to the first valid claimant key +/// that redeems it. Replays by that same key are safe because both relay writes +/// are idempotent; a different key can never take over a partially completed +/// migration. +struct OidcPendingState { + nonce: String, + device_challenge: String, + expires_at: u64, +} + +struct OidcFinalizeCode { + subject: String, + pubkey: Option, + device_challenge: String, + expires_at: u64, +} + +/// Process-local OIDC session ledger. Keeping both stages behind one value +/// prevents callers from constructing mismatched state/code maps. +#[derive(Default)] +pub struct OidcSessions { + states: Mutex>, + codes: Mutex>, +} + +/// How the service delivers magic links. +#[derive(Clone)] +pub enum Mailer { + /// No delivery backend configured — `/email/start` reports the channel as + /// unavailable instead of claiming to have sent a link it cannot send. + /// The answer is identical for every address, so it still cannot be used + /// to enumerate the workspace. + Disabled, + /// Development: don't send anything; the link is logged and returned in the + /// `/email/start` response so a local tester can follow it by hand. + Dev, +} + +/// Immutable service configuration + shared mutable session ledgers. +pub struct Inner { + pub roster: Roster, + pub token_secret: Vec, + pub consumed: Mutex, + pub admin: Keys, + /// Slack workspace id — the `` in every `slack::` subject. + pub team_id: String, + pub relay_url: String, + pub auth_tag: Option, + /// Public base URL of this service, used to build magic links. + pub base_url: String, + pub token_ttl_secs: u64, + pub mailer: Mailer, + /// Shared HTTP client for the OIDC exchange. + pub http: reqwest::Client, + /// Slack OIDC credentials; `None` disables the `/oidc/*` routes. + pub oidc: Option, + /// Process-local Slack authorization and app-handoff sessions. + pub oidc_sessions: OidcSessions, + /// Dev mode: enables `/oidc/dev-complete`, which simulates a verified OIDC + /// result so the publish path can be exercised without a real Slack app. + pub dev: bool, +} + +/// Cloneable handle to the service state (an `Arc` under the hood). +#[derive(Clone)] +pub struct AppState(pub Arc); + +impl AppState { + fn i(&self) -> &Inner { + &self.0 + } +} + +/// Build the router for both claim channels. +pub fn router(state: AppState) -> Router { + Router::new() + .route("/healthz", get(|| async { "ok" })) + // Email channel. + .route("/email/start", post(email_start)) + .route("/email/verify", get(email_verify)) + .route("/email/complete", post(email_complete)) + // OIDC channel (Sign in with Slack). + .route("/oidc/start", get(oidc_start)) + .route("/oidc/callback", get(oidc_callback)) + .route("/oidc/finalize", post(oidc_finalize)) + .route("/oidc/dev-complete", get(oidc_dev_complete)) + // Signed Nostr events are small. Bound JSON bodies before parsing so a + // public migration endpoint cannot be used for oversized allocations. + .layer(DefaultBodyLimit::max(64 * 1024)) + // Buzz Desktop runs in a WebView origin. The public migration + // endpoints are protected by OIDC state, device-verifier-bound + // short-lived codes, and signed claims. CORS is transport + // compatibility, not an authorization boundary. + .layer(CorsLayer::permissive()) + .with_state(state) +} + +fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +fn random_nonce() -> [u8; 32] { + rand::random() +} + +/// Normalize a caller-supplied Buzz pubkey to canonical lowercase hex, or +/// reject it. Only 64-char hex is accepted (the app sends hex, never an nsec). +fn normalize_pubkey(s: &str) -> Option { + let s = s.trim(); + if s.len() == 64 && s.chars().all(|c| c.is_ascii_hexdigit()) { + Some(s.to_lowercase()) + } else { + None + } +} + +// ---- POST /email/start ------------------------------------------------------- + +#[derive(Deserialize)] +struct EmailStartReq { + email: String, +} + +#[derive(Debug, Serialize)] +struct EmailStartResp { + /// Always the same generic message, regardless of whether the email is + /// known — do not leak workspace membership. + message: String, + /// Only populated in [`Mailer::Dev`]: the magic link to follow by hand. + #[serde(skip_serializing_if = "Option::is_none")] + dev_link: Option, +} + +const GENERIC_START_MSG: &str = + "If that address belongs to a workspace member, a migration link has been sent to it."; + +async fn email_start( + State(state): State, + Json(req): Json, +) -> Result, ApiError> { + let inner = state.i(); + if matches!(inner.mailer, Mailer::Disabled) { + // Same answer for every address (no enumeration), but an honest one: + // pretending to have mailed a link the operator cannot send strands + // the claimant waiting for an email that never arrives. + return Err(ApiError { + status: StatusCode::SERVICE_UNAVAILABLE, + message: "email migration links are not configured on this service".into(), + }); + } + let dev_link = match (&inner.mailer, inner.roster.subject_for_email(&req.email)) { + (Mailer::Dev, Some(subject)) => { + let tok = token::mint( + &inner.token_secret, + subject, + now_secs(), + inner.token_ttl_secs, + &random_nonce(), + ) + .map_err(|e| token_err_to_api(&e))?; + let link = format!("{}/email/verify?token={}", inner.base_url, tok.as_str()); + tracing::info!(subject, %link, "dev mailer: magic link (not emailed)"); + Some(link) + } + (Mailer::Dev, None) => { + // Unknown address: do the same amount of visible work, send nothing. + tracing::info!(email = %req.email, "email/start for unknown address (ignored)"); + None + } + (Mailer::Disabled, _) => None, + }; + Ok(Json(EmailStartResp { + message: GENERIC_START_MSG.to_string(), + dev_link, + })) +} + +// ---- GET /email/verify ------------------------------------------------------- + +#[derive(Deserialize)] +struct VerifyQuery { + token: String, +} + +/// The magic-link target. Validates integrity + expiry (not single use) and +/// renders a page that hands the token to the recipient's Buzz app. +async fn email_verify(State(state): State, Query(q): Query) -> Response { + let inner = state.i(); + let tok = MagicToken::from_wire(q.token.clone()); + let response = match token::verify(&inner.token_secret, &tok, now_secs()) { + Ok(v) => { + let deep_link = format!( + "buzz://import-claim?subject={}&token={}&service={}", + urlencode(&v.subject), + urlencode(tok.as_str()), + urlencode(&inner.base_url), + ); + Html(verify_page(&v.subject, &deep_link)).into_response() + } + Err(e) => (StatusCode::BAD_REQUEST, Html(error_page(&e.to_string()))).into_response(), + }; + no_store(response) +} + +// ---- POST /email/complete ---------------------------------------------------- + +#[derive(Deserialize)] +struct CompleteReq { + token: String, + /// The claimant's Buzz pubkey (64-hex) — supplied by their own app. + pubkey: String, +} + +#[derive(Debug, Serialize)] +struct CompleteResp { + subject: String, + /// The published attestation's event id. + attestation_event_id: String, +} + +async fn email_complete( + State(state): State, + Json(req): Json, +) -> Result, ApiError> { + let inner = state.i(); + let pubkey = normalize_pubkey(&req.pubkey) + .ok_or_else(|| ApiError::bad("pubkey must be 64-char hex (the app's own key)"))?; + + // Reserve before the network write to block concurrent redemption. Commit + // only after relay acceptance; a transient relay failure releases the + // reservation so the legitimate claimant can retry the same link. + let verified = reserve_email_token(inner, &req.token, now_secs())?; + let subject = verified.subject.clone(); + let event_id = match publish_attestation_for(inner, &subject, &pubkey, "email").await { + Ok(event_id) => { + consumed_ledger(inner).commit(&verified); + event_id + } + Err(error) => { + consumed_ledger(inner).release(&verified); + return Err(error); + } + }; + Ok(Json(CompleteResp { + subject, + attestation_event_id: event_id, + })) +} + +/// Sign and publish the owner/admin attestation `subject → pubkey`. +/// +/// This does not grant community membership. Email claims are a recovery +/// channel for an already-admitted member; only the dedicated Slack OAuth join +/// path below is allowed to admit a new member. +async fn publish_attestation_for( + inner: &Inner, + subject: &str, + pubkey: &str, + channel: &str, +) -> Result { + let event_id = crate::attest::publish_attestation( + &inner.relay_url, + &inner.admin, + subject, + pubkey, + inner.auth_tag.as_ref(), + ) + .await + .map_err(|e| ApiError::upstream(&e.to_string()))?; + tracing::info!(subject, pubkey, event_id, channel, "published attestation"); + Ok(event_id) +} + +/// Admit an OAuth-verified Slack user, then attest their imported identity. +/// +/// Both relay writes are idempotent: adding an existing member is a no-op and +/// the attestation is parameterized-replaceable. A callback retry therefore +/// cannot duplicate membership or attribution. +async fn publish_join_attestation_for( + inner: &Inner, + subject: &str, + pubkey: &str, + channel: &str, +) -> Result { + let member_event_id = crate::attest::publish_add_member( + &inner.relay_url, + &inner.admin, + pubkey, + inner.auth_tag.as_ref(), + ) + .await + .map_err(|e| ApiError::upstream(&e.to_string()))?; + tracing::info!(pubkey, member_event_id, channel, "ensured community member"); + publish_attestation_for(inner, subject, pubkey, channel).await +} + +// ---- OIDC channel (Sign in with Slack) -------------------------------------- + +/// Begin Sign in with Slack: bind the round-trip to Buzz's device challenge, +/// store Slack state/nonce values, and redirect to Slack. No pubkey is accepted +/// here; key possession is proven only at `/oidc/finalize`. +#[derive(Deserialize)] +struct OidcStartQuery { + /// S256 challenge derived from a verifier generated and retained by Buzz. + challenge: String, +} + +async fn oidc_start( + State(state): State, + Query(query): Query, +) -> Result { + let inner = state.i(); + let cfg = inner.oidc.as_ref().ok_or_else(oidc_unconfigured)?; + validate_device_challenge(&query.challenge)?; + + let st = hex::encode(random_nonce()); + let nonce = hex::encode(random_nonce()); + { + let now = now_secs(); + let mut states = lock_or_recover(&inner.oidc_sessions.states, "oidc states"); + states.retain(|_, pending| pending.expires_at > now); + if states.len() >= MAX_PENDING_OIDC_STATES { + return Err(ApiError::too_many_requests( + "too many pending Slack sign-ins; try again shortly", + )); + } + states.insert( + st.clone(), + OidcPendingState { + nonce: nonce.clone(), + device_challenge: query.challenge, + expires_at: now.saturating_add(OIDC_STATE_TTL_SECS), + }, + ); + } + Ok(no_store(Redirect::to(&oidc::authorize_url( + cfg, &st, &nonce, + )))) +} + +#[derive(Deserialize)] +struct OidcCallbackQuery { + #[serde(default)] + code: String, + #[serde(default)] + state: String, +} + +/// Slack's redirect target: resolve `state`, exchange the code for the verified +/// Slack user id, mint a short-lived finalize `code`, and hand back to the app. +/// +/// It deliberately does NOT publish anything. The attestation is signed only at +/// `/oidc/finalize`, once the app proves control of the pubkey to be attested — +/// so a victim's Slack login can never mint an attestation for someone else's +/// key. +async fn oidc_callback( + State(state): State, + Query(q): Query, +) -> Result { + let inner = state.i(); + let cfg = inner.oidc.as_ref().ok_or_else(oidc_unconfigured)?; + + // Consume the state (CSRF + replay) → the nonce we sent Slack. + let pending = { + let now = now_secs(); + let mut states = lock_or_recover(&inner.oidc_sessions.states, "oidc states"); + states.retain(|_, pending| pending.expires_at > now); + states.remove(&q.state) + } + .ok_or_else(|| ApiError::bad("unknown or expired OIDC state"))?; + + let subject = oidc::exchange_code_for_subject(&inner.http, cfg, &q.code, &pending.nonce) + .await + .map_err(|e| ApiError::upstream(&e.to_string()))?; + + // Mint a short-lived code bound to the verified subject and to the device + // challenge supplied by Buzz before the browser was opened. + let code = insert_oidc_finalize_code(inner, subject.clone(), pending.device_challenge)?; + Ok(no_store(Redirect::to(&oidc_app_return_url( + inner, &subject, &code, + )))) +} + +fn oidc_app_return_url(inner: &Inner, subject: &str, code: &str) -> String { + format!( + "buzz://import-claim?subject={}&via=oidc&code={}&relay={}&service={}", + urlencode(subject), + urlencode(code), + urlencode(&inner.relay_url), + urlencode(&inner.base_url), + ) +} + +#[derive(Deserialize)] +struct OidcFinalizeReq { + /// Short-lived code from the `/oidc/callback` redirect. + code: String, + /// Verifier retained by the Buzz app that initiated `/oidc/start`. + verifier: String, + /// The claimant's signed self-claim (kind `KIND_IMPORT_IDENTITY_CLAIM`). Its + /// valid signature proves control of the pubkey to be attested; its `d` tag + /// must equal the code's OIDC-verified subject. + claim: serde_json::Value, +} + +/// The app's proof-of-possession finalize: redeem a post-OIDC `code` and publish +/// the owner/admin attestation — bound to the pubkey that *signed* the +/// accompanying self-claim, never to an attacker-suppliable parameter. +/// +/// The claim event is fully verified (id hash + Schnorr signature); its `d` tag +/// must equal the Slack-verified subject. The finalize code additionally +/// requires the verifier retained by the Buzz transaction that initiated the +/// browser flow, so intercepting the custom-scheme callback is insufficient. +async fn oidc_finalize( + State(state): State, + Json(req): Json, +) -> Result, ApiError> { + let inner = state.i(); + + // Reject the wrong event kind up front (cheap), then fully verify id+sig + // before trusting any field read off the event. + let kind_ok = req.claim.get("kind").and_then(|k| k.as_u64()) + == Some(u64::from(buzz_core::kind::KIND_IMPORT_IDENTITY_CLAIM)); + if !kind_ok { + return Err(ApiError::bad("claim must be an identity-claim event")); + } + let claim_json = serde_json::to_string(&req.claim) + .map_err(|_| ApiError::bad("claim must be a JSON event"))?; + let event = Event::from_json(&claim_json) + .map_err(|_| ApiError::bad("claim is not a valid Nostr event"))?; + let to_verify = event.clone(); + tokio::task::spawn_blocking(move || buzz_core::verification::verify_event(&to_verify)) + .await + .map_err(|_| ApiError::upstream("claim verification task failed"))? + .map_err(|_| ApiError::bad("claim signature is invalid"))?; + + let claim_subject = event.tags.iter().find_map(|tag| { + let parts = tag.as_slice(); + if parts.first().map(String::as_str) == Some("d") { + parts.get(1).cloned() + } else { + None + } + }); + + // Bind the code to the first key that proves possession. Keeping that + // binding until expiry lets the same app retry if the relay accepted the + // attestation but the subsequent client-side claim publish failed. + let pubkey = event.pubkey.to_hex(); + let subject = bind_oidc_finalize_code( + inner, + &req.code, + &req.verifier, + claim_subject.as_deref(), + &pubkey, + now_secs(), + )?; + let event_id = publish_join_attestation_for(inner, &subject, &pubkey, "oidc").await?; + Ok(Json(CompleteResp { + subject, + attestation_event_id: event_id, + })) +} + +fn bind_oidc_finalize_code( + inner: &Inner, + code: &str, + verifier: &str, + claim_subject: Option<&str>, + pubkey: &str, + now: u64, +) -> Result { + let mut codes = lock_or_recover(&inner.oidc_sessions.codes, "oidc codes"); + codes.retain(|_, pending| pending.expires_at > now); + let pending = codes + .get_mut(code) + .ok_or_else(|| ApiError::bad("unknown or expired finalize code"))?; + validate_device_verifier(verifier, &pending.device_challenge)?; + if claim_subject != Some(pending.subject.as_str()) { + return Err(ApiError::bad( + "self-claim does not match the verified Slack identity", + )); + } + match pending.pubkey.as_deref() { + Some(bound) if bound != pubkey => { + return Err(ApiError::bad( + "finalize code is already bound to a different Buzz account", + )); + } + Some(_) => {} + None => pending.pubkey = Some(pubkey.to_string()), + } + Ok(pending.subject.clone()) +} + +#[derive(Deserialize)] +struct OidcDevCompleteQuery { + /// Simulated Slack user id (e.g. `U060`). + sub: String, + /// S256 challenge generated by Buzz, matching the real `/oidc/start`. + challenge: String, +} + +#[derive(Serialize)] +struct DevCompleteResp { + subject: String, + /// The `buzz://import-claim` URL a real Slack callback would redirect to, + /// carrying the short-lived finalize code — follow it to drive the app. + app_return_url: String, +} + +/// Dev-only: simulate a verified OIDC result. Mints the same finalize code a +/// real callback would and returns the app deep link, so the finalize path can +/// be exercised without a Slack app. Returns 404 unless started with --dev. +async fn oidc_dev_complete( + State(state): State, + Query(q): Query, +) -> Result, ApiError> { + let inner = state.i(); + if !inner.dev { + return Err(ApiError { + status: StatusCode::NOT_FOUND, + message: "not found".into(), + }); + } + let sub = q.sub.trim(); + if sub.is_empty() { + return Err(ApiError::bad("sub must not be empty")); + } + validate_device_challenge(&q.challenge)?; + let subject = format!("slack:{}:{sub}", inner.team_id); + let code = insert_oidc_finalize_code(inner, subject.clone(), q.challenge)?; + let app_return_url = oidc_app_return_url(inner, &subject, &code); + Ok(Json(DevCompleteResp { + subject, + app_return_url, + })) +} + +fn insert_oidc_finalize_code( + inner: &Inner, + subject: String, + device_challenge: String, +) -> Result { + let code = hex::encode(random_nonce()); + let now = now_secs(); + let mut codes = lock_or_recover(&inner.oidc_sessions.codes, "oidc codes"); + codes.retain(|_, pending| pending.expires_at > now); + if codes.len() >= MAX_PENDING_OIDC_STATES { + return Err(ApiError::too_many_requests( + "too many pending Slack sign-ins; try again shortly", + )); + } + codes.insert( + code.clone(), + OidcFinalizeCode { + subject, + pubkey: None, + device_challenge, + expires_at: now.saturating_add(OIDC_CODE_TTL_SECS), + }, + ); + Ok(code) +} + +fn validate_device_challenge(challenge: &str) -> Result<(), ApiError> { + let decoded = URL_SAFE_NO_PAD + .decode(challenge) + .map_err(|_| ApiError::bad("challenge must be an S256 base64url value"))?; + if decoded.len() != 32 { + return Err(ApiError::bad("challenge must be an S256 base64url value")); + } + Ok(()) +} + +fn validate_device_verifier(verifier: &str, expected_challenge: &str) -> Result<(), ApiError> { + if !(43..=128).contains(&verifier.len()) + || !verifier + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~')) + { + return Err(ApiError::bad("invalid device verifier")); + } + let actual = oidc::pkce_challenge(verifier); + if !bool::from(actual.as_bytes().ct_eq(expected_challenge.as_bytes())) { + return Err(ApiError::bad( + "device verifier does not match this Slack sign-in", + )); + } + Ok(()) +} + +fn oidc_unconfigured() -> ApiError { + ApiError { + status: StatusCode::SERVICE_UNAVAILABLE, + message: "OIDC channel is not configured on this service".into(), + } +} + +/// Verify a token and atomically reserve it for one in-flight relay write. +fn reserve_email_token(inner: &Inner, wire: &str, now: u64) -> Result { + let tok = MagicToken::from_wire(wire.to_string()); + let verified = + token::verify(&inner.token_secret, &tok, now).map_err(|e| token_err_to_api(&e))?; + consumed_ledger(inner) + .try_reserve(&verified, now) + .map_err(|e| token_err_to_api(&e))?; + Ok(verified) +} + +fn consumed_ledger(inner: &Inner) -> MutexGuard<'_, ConsumedNonces> { + lock_or_recover(&inner.consumed, "consumed nonce ledger") +} + +fn lock_or_recover<'a, T>(mutex: &'a Mutex, name: &str) -> MutexGuard<'a, T> { + match mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => { + tracing::error!(name, "recovering poisoned migration-service mutex"); + poisoned.into_inner() + } + } +} + +fn no_store(response: impl IntoResponse) -> Response { + let mut response = response.into_response(); + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("no-store, max-age=0"), + ); + response.headers_mut().insert( + HeaderName::from_static("referrer-policy"), + HeaderValue::from_static("no-referrer"), + ); + response +} + +fn token_err_to_api(e: &TokenError) -> ApiError { + match e { + TokenError::Expired | TokenError::AlreadyUsed => ApiError { + status: StatusCode::GONE, + message: e.to_string(), + }, + TokenError::Unavailable => ApiError { + status: StatusCode::SERVICE_UNAVAILABLE, + message: e.to_string(), + }, + _ => ApiError::bad(&e.to_string()), + } +} + +// ---- shared error + small helpers ------------------------------------------- + +#[derive(Debug)] +struct ApiError { + status: StatusCode, + message: String, +} + +impl ApiError { + fn bad(msg: &str) -> Self { + Self { + status: StatusCode::BAD_REQUEST, + message: msg.to_string(), + } + } + fn upstream(msg: &str) -> Self { + Self { + status: StatusCode::BAD_GATEWAY, + message: msg.to_string(), + } + } + fn too_many_requests(msg: &str) -> Self { + Self { + status: StatusCode::TOO_MANY_REQUESTS, + message: msg.to_string(), + } + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> axum::response::Response { + #[derive(Serialize)] + struct Body { + error: String, + } + ( + self.status, + Json(Body { + error: self.message, + }), + ) + .into_response() + } +} + +/// Minimal percent-encoding for the query values we build (no external dep). +fn urlencode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char) + } + _ => out.push_str(&format!("%{b:02X}")), + } + } + out +} + +fn verify_page(subject: &str, deep_link: &str) -> String { + let subject = escape_html(subject); + format!( + "Complete your migration\ + \ +

You're verified

\ +

Open Buzz to finish linking your imported history for {subject}.

\ +

\ + Open in Buzz

\ +

This link is single-use and expires soon. \ + It links your history to the Buzz account on this device — only continue if you \ + started this in Buzz.

" + ) +} + +fn error_page(msg: &str) -> String { + let msg = escape_html(msg); + format!( + "Link problem\ + \ +

This link can't be used

{msg}

\ +

Ask your operator to send a fresh migration link.

" + ) +} + +fn escape_html(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_inner() -> Inner { + let users = + r#"[{"id":"U060","profile":{"email":"alice@corp.com","display_name":"Alice"}}]"#; + Inner { + roster: Roster::from_users_json(users.as_bytes(), "T060").unwrap(), + token_secret: b"secret".to_vec(), + consumed: Mutex::new(ConsumedNonces::new()), + admin: Keys::generate(), + team_id: "T060".into(), + relay_url: "ws://127.0.0.1:1".into(), + auth_tag: None, + base_url: "http://localhost:8787".into(), + token_ttl_secs: 3600, + mailer: Mailer::Dev, + http: reqwest::Client::new(), + oidc: None, + oidc_sessions: OidcSessions::default(), + dev: false, + } + } + + #[test] + fn reserve_happy_path_blocks_concurrent_redemption() { + let inner = test_inner(); + let tok = + token::mint(&inner.token_secret, "slack:U060", 1000, 3600, &[3u8; 16]).expect("mint"); + let verified = reserve_email_token(&inner, tok.as_str(), 1500).expect("reserves"); + assert_eq!(verified.subject, "slack:U060"); + let err = reserve_email_token(&inner, tok.as_str(), 1500).unwrap_err(); + assert_eq!(err.status, StatusCode::GONE); + } + + #[test] + fn reserve_rejects_forged_token() { + let inner = test_inner(); + let forged = + token::mint(b"other-secret", "slack:U060", 1000, 3600, &[3u8; 16]).expect("mint"); + let err = reserve_email_token(&inner, forged.as_str(), 1500).unwrap_err(); + assert_eq!(err.status, StatusCode::BAD_REQUEST); + } + + #[test] + fn reserve_rejects_expired_token() { + let inner = test_inner(); + let tok = + token::mint(&inner.token_secret, "slack:U060", 1000, 10, &[3u8; 16]).expect("mint"); + let err = reserve_email_token(&inner, tok.as_str(), 2000).unwrap_err(); + assert_eq!(err.status, StatusCode::GONE); + } + + #[tokio::test] + async fn disabled_mailer_reports_unavailable_without_exposing_addresses() { + let mut inner = test_inner(); + inner.mailer = Mailer::Disabled; + let state = AppState(Arc::new(inner)); + + let known = email_start( + State(state.clone()), + Json(EmailStartReq { + email: "alice@corp.com".into(), + }), + ) + .await + .expect_err("channel is unavailable"); + let unknown = email_start( + State(state), + Json(EmailStartReq { + email: "unknown@corp.com".into(), + }), + ) + .await + .expect_err("channel is unavailable"); + + // A known and an unknown address are answered identically — no + // enumeration — and neither is told a link was sent. + assert_eq!(known.status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(known.status, unknown.status); + assert_eq!(known.message, unknown.message); + } + + #[tokio::test] + async fn dev_mailer_answers_known_and_unknown_addresses_alike() { + let state = AppState(Arc::new(test_inner())); + let unknown = email_start( + State(state.clone()), + Json(EmailStartReq { + email: "nobody@corp.com".into(), + }), + ) + .await + .expect("generic response"); + let known = email_start( + State(state), + Json(EmailStartReq { + email: "alice@corp.com".into(), + }), + ) + .await + .expect("generic response"); + + assert!(unknown.0.dev_link.is_none()); + assert!(known.0.dev_link.is_some(), "dev mode surfaces the link"); + assert_eq!(known.0.message, unknown.0.message); + } + + #[test] + fn pubkey_normalization() { + assert_eq!(normalize_pubkey(&"AB".repeat(32)).unwrap(), "ab".repeat(32)); + assert!(normalize_pubkey("npub1xyz").is_none()); + assert!(normalize_pubkey("tooshort").is_none()); + assert!(normalize_pubkey(&"g".repeat(64)).is_none()); // non-hex + } + + #[test] + fn urlencode_escapes_reserved() { + assert_eq!(urlencode("slack:U060"), "slack%3AU060"); + assert_eq!(urlencode("a-b_c.d~e"), "a-b_c.d~e"); + } + + #[test] + fn sensitive_responses_disable_caching_and_referrers() { + let response = no_store(Html("secret")); + assert_eq!( + response.headers().get(header::CACHE_CONTROL), + Some(&HeaderValue::from_static("no-store, max-age=0")) + ); + assert_eq!( + response + .headers() + .get(HeaderName::from_static("referrer-policy")), + Some(&HeaderValue::from_static("no-referrer")) + ); + } + + #[test] + fn oidc_return_carries_code_relay_and_service() { + let inner = test_inner(); + assert_eq!( + oidc_app_return_url(&inner, "slack:U060", "c0de"), + "buzz://import-claim?subject=slack%3AU060&via=oidc&code=c0de&relay=ws%3A%2F%2F127.0.0.1%3A1&service=http%3A%2F%2Flocalhost%3A8787" + ); + } + + #[tokio::test] + async fn oidc_start_rejects_when_pending_state_capacity_is_reached() { + let mut inner = test_inner(); + inner.oidc = Some(OidcConfig { + client_id: "client".into(), + client_secret: "secret".into(), + redirect_uri: "https://migrate.example/oidc/callback".into(), + team_id: "T060".into(), + }); + inner.oidc_sessions.states = Mutex::new( + (0..MAX_PENDING_OIDC_STATES) + .map(|index| { + ( + format!("state-{index}"), + OidcPendingState { + nonce: "nonce".into(), + device_challenge: oidc::pkce_challenge(&test_verifier()), + expires_at: u64::MAX, + }, + ) + }) + .collect(), + ); + let state = AppState(Arc::new(inner)); + let error = match oidc_start( + State(state), + Query(OidcStartQuery { + challenge: oidc::pkce_challenge(&test_verifier()), + }), + ) + .await + { + Ok(_) => panic!("capacity must reject"), + Err(error) => error, + }; + assert_eq!(error.status, StatusCode::TOO_MANY_REQUESTS); + } + + /// Build a signed self-claim (kind `KIND_IMPORT_IDENTITY_CLAIM`) with the + /// given `d` subject, as JSON — what the app POSTs to `/oidc/finalize`. + fn signed_claim(subject: &str) -> (serde_json::Value, Keys) { + use nostr::{EventBuilder, Kind}; + let keys = Keys::generate(); + let event = EventBuilder::new( + Kind::Custom(buzz_core::kind::KIND_IMPORT_IDENTITY_CLAIM as u16), + "", + ) + .tags([Tag::parse(["d", subject]).unwrap()]) + .sign_with_keys(&keys) + .expect("sign"); + (serde_json::from_str(&event.as_json()).expect("json"), keys) + } + + fn inner_with_code(code: &str, subject: &str) -> Inner { + let inner = test_inner(); + inner.oidc_sessions.codes.lock().unwrap().insert( + code.to_string(), + OidcFinalizeCode { + subject: subject.to_string(), + pubkey: None, + device_challenge: oidc::pkce_challenge(&test_verifier()), + expires_at: u64::MAX, + }, + ); + inner + } + + fn test_verifier() -> String { + "v".repeat(43) + } + + #[test] + fn finalize_code_allows_same_key_retry_but_rejects_a_different_key() { + let inner = inner_with_code("retry-code", "slack:U060"); + let first = bind_oidc_finalize_code( + &inner, + "retry-code", + &test_verifier(), + Some("slack:U060"), + &"aa".repeat(32), + 100, + ) + .unwrap(); + assert_eq!(first, "slack:U060"); + assert!(bind_oidc_finalize_code( + &inner, + "retry-code", + &test_verifier(), + Some("slack:U060"), + &"aa".repeat(32), + 101, + ) + .is_ok()); + let error = bind_oidc_finalize_code( + &inner, + "retry-code", + &test_verifier(), + Some("slack:U060"), + &"bb".repeat(32), + 102, + ) + .unwrap_err(); + assert_eq!(error.status, StatusCode::BAD_REQUEST); + } + + #[test] + fn mismatched_subject_does_not_consume_or_bind_finalize_code() { + let inner = inner_with_code("subject-code", "slack:U060"); + assert!(bind_oidc_finalize_code( + &inner, + "subject-code", + &test_verifier(), + Some("slack:U999"), + &"aa".repeat(32), + 100, + ) + .is_err()); + assert!(bind_oidc_finalize_code( + &inner, + "subject-code", + &test_verifier(), + Some("slack:U060"), + &"bb".repeat(32), + 101, + ) + .is_ok()); + } + + #[test] + fn wrong_device_verifier_does_not_bind_finalize_code() { + let inner = inner_with_code("device-code", "slack:U060"); + assert!(bind_oidc_finalize_code( + &inner, + "device-code", + &"x".repeat(43), + Some("slack:U060"), + &"aa".repeat(32), + 100, + ) + .is_err()); + assert!(bind_oidc_finalize_code( + &inner, + "device-code", + &test_verifier(), + Some("slack:U060"), + &"bb".repeat(32), + 101, + ) + .is_ok()); + } + + #[tokio::test] + async fn finalize_rejects_claim_for_a_different_subject() { + // Slack verified U060, but the self-claim consents to U999. Even with a + // valid signature and a live code, the mismatch is refused — no attacker + // can redirect a code onto a different identity. + let inner = inner_with_code("code1", "slack:U060"); + let (claim, _keys) = signed_claim("slack:U999"); + let state = AppState(Arc::new(inner)); + let err = oidc_finalize( + State(state), + Json(OidcFinalizeReq { + code: "code1".into(), + verifier: test_verifier(), + claim, + }), + ) + .await + .unwrap_err(); + assert_eq!(err.status, StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn finalize_rejects_tampered_signature() { + // A claim whose signature doesn't verify never reaches the code redemption + // or the relay — the attested pubkey must be genuinely proven. + let inner = inner_with_code("code2", "slack:U060"); + let (mut claim, _keys) = signed_claim("slack:U060"); + claim["sig"] = serde_json::Value::String("0".repeat(128)); + let state = AppState(Arc::new(inner)); + let err = oidc_finalize( + State(state), + Json(OidcFinalizeReq { + code: "code2".into(), + verifier: test_verifier(), + claim, + }), + ) + .await + .unwrap_err(); + assert_eq!(err.status, StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn finalize_rejects_unknown_code() { + let inner = inner_with_code("code3", "slack:U060"); + let (claim, _keys) = signed_claim("slack:U060"); + let state = AppState(Arc::new(inner)); + let err = oidc_finalize( + State(state), + Json(OidcFinalizeReq { + code: "not-the-code".into(), + verifier: test_verifier(), + claim, + }), + ) + .await + .unwrap_err(); + assert_eq!(err.status, StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn finalize_rejects_wrong_kind() { + let inner = inner_with_code("code4", "slack:U060"); + let keys = Keys::generate(); + let event = nostr::EventBuilder::new(nostr::Kind::Custom(1), "") + .tags([Tag::parse(["d", "slack:U060"]).unwrap()]) + .sign_with_keys(&keys) + .expect("sign"); + let claim: serde_json::Value = serde_json::from_str(&event.as_json()).expect("json"); + let state = AppState(Arc::new(inner)); + let err = oidc_finalize( + State(state), + Json(OidcFinalizeReq { + code: "code4".into(), + verifier: test_verifier(), + claim, + }), + ) + .await + .unwrap_err(); + assert_eq!(err.status, StatusCode::BAD_REQUEST); + } + + #[test] + fn verify_page_escapes_export_supplied_subject() { + let page = verify_page(r#"slack:"#, "buzz://safe"); + assert!(!page.contains("