diff --git a/Cargo.lock b/Cargo.lock index b77f967..1b4ebd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3063,7 +3063,7 @@ dependencies = [ [[package]] name = "sparkplane" -version = "0.1.3" +version = "0.1.4" dependencies = [ "anyhow", "arc-swap", @@ -3118,7 +3118,7 @@ dependencies = [ [[package]] name = "sparkplane-core" -version = "0.1.3" +version = "0.1.4" dependencies = [ "anyhow", "sd-notify", @@ -3132,7 +3132,7 @@ dependencies = [ [[package]] name = "sparkplane-ipc" -version = "0.1.3" +version = "0.1.4" dependencies = [ "anyhow", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 6523045..e17dcc9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = [".", "crates/sparkplane-core", "crates/sparkplane-ipc"] resolver = "3" [workspace.package] -version = "0.1.3" +version = "0.1.4" edition = "2024" rust-version = "1.95" license = "MIT" @@ -22,8 +22,8 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["env-filter", "fmt", "json", "registry", "std"] } metrics = "0.24" tempfile = "3" -sparkplane-core = { path = "crates/sparkplane-core", version = "=0.1.3" } -sparkplane-ipc = { path = "crates/sparkplane-ipc", version = "=0.1.3" } +sparkplane-core = { path = "crates/sparkplane-core", version = "=0.1.4" } +sparkplane-ipc = { path = "crates/sparkplane-ipc", version = "=0.1.4" } [package] name = "sparkplane" diff --git a/configs/systemd/system/sparkplane-agent.service b/configs/systemd/system/sparkplane-agent.service index 28c0de8..35389d4 100644 --- a/configs/systemd/system/sparkplane-agent.service +++ b/configs/systemd/system/sparkplane-agent.service @@ -51,6 +51,8 @@ MemoryMax=256M OOMScoreAdjust=-100 AppArmorProfile=sparkplane-agent NotifyAccess=main +# Initial reconciliation may wait for the engine's 1800-second cold start. +TimeoutStartSec=1900s WatchdogSec=30s TimeoutStopSec=15s Restart=on-failure diff --git a/docs/how-to/develop-spark.md b/docs/how-to/develop-spark.md index 7b38ac0..d774604 100644 --- a/docs/how-to/develop-spark.md +++ b/docs/how-to/develop-spark.md @@ -173,6 +173,11 @@ commit, artifact name, and manifest signature in the release notes. The artifact must be the one whose manifest was signed; do not rebuild it after signing. +Installation assigns explicit permissions independently of the operator's +umask. Public release directories are root-owned and traversable by the service +account; executable and catalog payloads remain read-only. Private credentials, +state and rollback evidence retain their restricted modes. + ## Install or upgrade Spark For a new host, use the bootstrap installer. Always inspect the dry run first: @@ -535,6 +540,11 @@ catalogs or the CLI. ## Release checklist +The agent's initial reconciliation can wait for the engine's 1800-second cold +start. Its notify service therefore allows 1900 seconds for startup; do not +restore systemd's shorter default or restart the service while weights load. +Readiness remains gated on semantic engine health, not merely a running process. + Before tagging: - catalog boundary tests pass and every engine/model entry validates; diff --git a/src/migration.rs b/src/migration.rs index c836d75..7bbe030 100644 --- a/src/migration.rs +++ b/src/migration.rs @@ -12,11 +12,13 @@ pub mod commands; pub mod config; pub mod container; pub mod docker; +pub mod emergency; pub mod fence; pub mod host; pub mod journal; pub mod publication; pub mod qualification; +pub mod recovery; pub mod release; pub mod relocation; pub mod runner; diff --git a/src/migration/appliance.rs b/src/migration/appliance.rs index 43c4a1f..61b4c4e 100644 --- a/src/migration/appliance.rs +++ b/src/migration/appliance.rs @@ -136,6 +136,7 @@ pub fn verify_sources(root: &Path, plan: &Plan) -> Result<()> { )?; connection.execute_batch("BEGIN")?; super::verify_database(&connection)?; + super::emergency::validate(&root.join("var/lib/sy-spark"))?; ensure!( inventory_digest(&connection)? == plan.inventory_sha256, "source inventory changed after preflight" @@ -248,6 +249,7 @@ pub fn preflight( )?; connection.execute_batch("BEGIN")?; super::verify_database(&connection)?; + super::emergency::validate(&data)?; let inventory_sha256 = inventory_digest(&connection)?; let mut statement = connection.prepare("SELECT id,repository,commit_sha,metadata_json FROM models")?; diff --git a/src/migration/cli.rs b/src/migration/cli.rs index a6e4b6b..74cbab6 100644 --- a/src/migration/cli.rs +++ b/src/migration/cli.rs @@ -38,6 +38,16 @@ pub struct MigrationArgs { /// Restore an interrupted pre-commit transaction; never restores after commit. #[arg(long, env = "SPARKPLANE_MIGRATION_RECOVER")] pub recover: bool, + /// Installed-authority-signed approval of an exact post-restoration recovery handoff. + #[arg(long, env = "SPARKPLANE_RECOVERY_APPROVAL", requires_all = ["recover", "recovery_signature"])] + pub recovery_approval: Option, + /// Detached signature of --recovery-approval; never authorizes a forward migration. + #[arg( + long, + env = "SPARKPLANE_RECOVERY_SIGNATURE", + requires = "recovery_approval" + )] + pub recovery_signature: Option, /// Finish opening traffic after a committed transaction; never reapplies state. #[arg(long, env = "SPARKPLANE_MIGRATION_RESUME")] pub resume: bool, @@ -96,6 +106,51 @@ fn self_digest() -> Result { )?)) } +fn recovery_approval( + args: &MigrationArgs, + record: &[u8], + host: &str, + executable: &str, +) -> Result, String)>> { + let Some(path) = &args.recovery_approval else { + return Ok(None); + }; + ensure!(args.recover && !args.resume, "handoff is recovery-only"); + let bytes = super::release::read(path, 65536)?; + let signature = String::from_utf8(super::release::read( + args.recovery_signature + .as_ref() + .context("recovery signature required")?, + 4096, + )?)?; + let approval = super::recovery::Approval::verify( + &bytes, + &signature, + &old_authority()?, + [host, &appliance::digest(record), executable], + SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(), + )?; + Ok(Some((approval, bytes, signature))) +} + +fn receipt(work: &Path, bytes: &[u8], signature: &[u8]) -> Result<()> { + for (suffix, value) in [("json", bytes), ("minisig", signature)] { + let path = work.join(format!( + "recovery-approval-{}.{suffix}", + appliance::digest(bytes) + )); + if path.try_exists()? { + ensure!( + super::release::read(&path, 65536)? == value, + "recovery receipt changed" + ); + } else { + write_private(&path, value)?; + } + } + Ok(()) +} + pub fn dispatch(host: &str, args: MigrationArgs) -> Result<()> { ensure!( host == "bootstrap", @@ -119,17 +174,28 @@ pub fn dispatch(host: &str, args: MigrationArgs) -> Result<()> { private_directory(base)?; let _lock = lock(base)?; private_directory(&work)?; - let record: Record = serde_json::from_slice(&super::release::read( - &work.join("record.json"), - 8 * 1024 * 1024, - )?)?; + let record_bytes = super::release::read(&work.join("record.json"), 8 * 1024 * 1024)?; + let mut record: Record = serde_json::from_slice(&record_bytes)?; + let executable = self_digest()?; ensure!( record.plan.schema == "sparkplane.appliance-migration/v1" - && record.plan.host == appliance::host_identity(root)? - && record.plan.executable == self_digest()?, - "recovery requires the original approved executable and host" + && record.plan.host == appliance::host_identity(root)?, + "recovery requires the original approved host and schema" + ); + let approval = recovery_approval(&args, &record_bytes, &record.plan.host, &executable)?; + ensure!( + approval.is_some() || record.plan.executable == executable, + "recovery requires the original executable or signed handoff" ); let mut journal = Journal::open(&work, &record.plan.host, &record.plan.release)?; + if let Some((approval, bytes, signature)) = approval { + ensure!( + journal.restored_checkpoint(), + "recovery handoff requires restored pre-commit state" + ); + approval.apply(&mut record.plan, &platform.inventory(Namespace::Legacy)?)?; + receipt(&work, &bytes, signature.as_bytes())?; + } let mut appliance = Host { root: root.into(), work: work.clone(), diff --git a/src/migration/container.rs b/src/migration/container.rs index 738764d..a6c7ce5 100644 --- a/src/migration/container.rs +++ b/src/migration/container.rs @@ -24,7 +24,7 @@ impl Namespace { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct LegacyContainer { pub container_id: String, diff --git a/src/migration/emergency.rs b/src/migration/emergency.rs new file mode 100644 index 0000000..b761e2c --- /dev/null +++ b/src/migration/emergency.rs @@ -0,0 +1,131 @@ +//! Preserve executor emergency evidence while converting its typed replay journal. +use crate::spark::resources::{EmergencyRecord, read_emergency_records}; +use anyhow::{Result, ensure}; +use rusqlite::{Connection, OpenFlags}; +use std::{fs, path::Path}; + +const JOURNAL: &str = "executor/emergency.jsonl"; + +#[derive(Debug, Default, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Journal { + original: Option, + candidate: Option, +} + +impl Journal { + pub fn prepare(data: &Path, work: &Path) -> Result { + use std::os::unix::fs::MetadataExt; + let Some(bytes) = validate(data)? else { + return Ok(Self::default()); + }; + let source = data.join(JOURNAL); + let original = super::relocation::FileIdentity::read(&source)?; + let candidate = work.join("candidate-emergency.jsonl"); + super::publication::write_private(&candidate, &bytes)?; + let metadata = fs::symlink_metadata(source)?; + rustix::fs::chown( + &candidate, + Some(rustix::fs::Uid::from_raw(metadata.uid())), + Some(rustix::fs::Gid::from_raw(metadata.gid())), + )?; + fs::File::open(&candidate)?.sync_all()?; + Ok(Self { + original: Some(original), + candidate: Some(super::relocation::FileIdentity::read(&candidate)?), + }) + } + + pub fn activate(&self, data: &Path, work: &Path) -> Result<()> { + use super::relocation::{FileIdentity, move_file}; + ensure!( + self.original.is_some() == self.candidate.is_some(), + "invalid emergency snapshot" + ); + if let (Some(original), Some(candidate)) = (&self.original, &self.candidate) { + let backup = work.join("original-emergency.jsonl"); + if backup.try_exists()? { + ensure!( + FileIdentity::read(&backup)? == *original, + "emergency snapshot changed" + ); + } else { + move_file(&data.join(JOURNAL), &backup, original)?; + } + move_file( + &work.join("candidate-emergency.jsonl"), + &data.join(JOURNAL), + candidate, + )?; + } + Ok(()) + } + + pub fn restore(&self, data: &Path, work: &Path) -> Result<()> { + use super::relocation::{FileIdentity, move_file}; + let backup = work.join("original-emergency.jsonl"); + let current = data.join(JOURNAL); + if let Some(original) = &self.original + && !backup.try_exists()? + { + ensure!( + FileIdentity::read(¤t)? == *original, + "original emergency journal is missing" + ); + return Ok(()); + } + if current.try_exists()? || current.is_symlink() { + move_file( + ¤t, + &work.join("failed-emergency.jsonl"), + &FileIdentity::read(¤t)?, + )?; + } + if let Some(original) = &self.original { + move_file(&backup, ¤t, original)?; + } + Ok(()) + } +} + +pub fn validate(data: &Path) -> Result>> { + ensure!( + !data.join("executor").is_symlink(), + "executor journal directory is a symlink" + ); + let metadata = match fs::symlink_metadata(data.join(JOURNAL)) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + result => result?, + }; + ensure!( + metadata.is_file(), + "legacy emergency journal must be a regular file" + ); + let records = read_emergency_records(&data.join(JOURNAL)) + .map_err(|_| anyhow::anyhow!("legacy emergency journal is invalid"))?; + let db = + Connection::open_with_flags(data.join("state.sqlite3"), OpenFlags::SQLITE_OPEN_READ_ONLY)?; + let mut output = Vec::new(); + for mut record in records { + ensure!( + record.schema == "sy.spark.emergency-record/v1" + && record.decision.schema == "sy.spark.emergency-decision/v1" + && record.event_id.parse::().is_ok(), + "unsupported legacy emergency record" + ); + let evidence: String = db.query_row( + "SELECT evidence_json FROM emergency_records WHERE event_id=?1", + [&record.event_id], + |row| row.get(0), + )?; + ensure!( + serde_json::from_str::(&evidence)? == record, + "legacy emergency journal differs from imported evidence" + ); + record.schema = "sparkplane.emergency-record/v1".into(); + record.decision.schema = "sparkplane.emergency-decision/v1".into(); + serde_json::to_writer(&mut output, &record)?; + output.push(b'\n'); + } + Ok(Some(output)) +} diff --git a/src/migration/fence.rs b/src/migration/fence.rs index 179cf4e..3669caf 100644 --- a/src/migration/fence.rs +++ b/src/migration/fence.rs @@ -3,6 +3,7 @@ use anyhow::{Result, ensure}; use std::{fs, path::Path}; const GUARD: &[u8] = b"[Unit]\nConditionPathExists=/run/sparkplane-migration-permit\n"; +const STARTUP: &[u8] = b"[Service]\nTimeoutStartSec=1900s\n"; const UNITS: [&str; 4] = [ "sy-spark-agent", "sy-spark-executor", @@ -10,15 +11,17 @@ const UNITS: [&str; 4] = [ "sparkplane-executor", ]; -fn guards(root: &Path) -> impl Iterator + '_ { - UNITS.into_iter().map(|unit| { - root.join(format!( - "etc/systemd/system/{unit}.service.d/90-sparkplane-migration.conf" - )) +fn guards(root: &Path) -> impl Iterator + '_ { + UNITS.into_iter().flat_map(|unit| { + let dir = root.join(format!("etc/systemd/system/{unit}.service.d")); + std::iter::once((dir.join("90-sparkplane-migration.conf"), GUARD)).chain( + unit.ends_with("-agent") + .then(|| (dir.join("91-sparkplane-migration-startup.conf"), STARTUP)), + ) }) } -fn check(path: &Path) -> Result { +fn check(path: &Path, expected: &[u8]) -> Result { ensure!( !path.parent().expect("fixed guard parent").is_symlink(), "unit drop-in directory must not be a symlink" @@ -29,7 +32,9 @@ fn check(path: &Path) -> Result { } let metadata = fs::symlink_metadata(path)?; ensure!( - metadata.is_file() && metadata.len() == GUARD.len() as u64 && fs::read(path)? == GUARD, + metadata.is_file() + && metadata.len() == expected.len() as u64 + && fs::read(path)? == expected, "migration guard conflicts with an existing file" ); Ok(true) @@ -37,18 +42,18 @@ fn check(path: &Path) -> Result { pub fn install_guards(root: &Path) -> Result<()> { // Check the entire set before touching even the first unit. - for path in guards(root) { - check(&path)?; + for (path, expected) in guards(root) { + check(&path, expected)?; } - for path in guards(root) { + for (path, expected) in guards(root) { let parent = path.parent().expect("fixed guard parent"); ensure!( !parent.is_symlink(), "unit drop-in directory must not be a symlink" ); fs::create_dir_all(parent)?; - if !check(&path)? { - super::publication::write_private(&path, GUARD)?; + if !check(&path, expected)? { + super::publication::write_private(&path, expected)?; } fs::File::open(parent)?.sync_all()?; fs::File::open(parent.parent().expect("unit root"))?.sync_all()?; @@ -57,11 +62,11 @@ pub fn install_guards(root: &Path) -> Result<()> { } pub fn remove_guards(root: &Path) -> Result<()> { - for path in guards(root) { - check(&path)?; + for (path, expected) in guards(root) { + check(&path, expected)?; } - for path in guards(root) { - if check(&path)? { + for (path, expected) in guards(root) { + if check(&path, expected)? { fs::remove_file(&path)?; fs::File::open(path.parent().expect("fixed guard parent"))?.sync_all()?; } diff --git a/src/migration/journal.rs b/src/migration/journal.rs index 181dd24..c0d976a 100644 --- a/src/migration/journal.rs +++ b/src/migration/journal.rs @@ -129,6 +129,15 @@ impl Journal { self.state.pending } + /// A signed replacement may only affect health checks after state restoration. + pub fn restored_checkpoint(&self) -> bool { + !self.poisoned + && !self.state.committed + && self.state.rolling_back + && self.state.pending.is_none() + && self.state.completed.len() <= 3 + } + pub fn committed(&self) -> Result { self.require_durable()?; Ok(self.state.committed) diff --git a/src/migration/recovery.rs b/src/migration/recovery.rs new file mode 100644 index 0000000..ffc7725 --- /dev/null +++ b/src/migration/recovery.rs @@ -0,0 +1,94 @@ +//! Installed-authority approval for a bounded, post-restoration recovery handoff. +use super::{appliance::Plan, container::LegacyContainer}; +use anyhow::{Context, Result, ensure}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Approval { + pub schema: String, + pub host: String, + pub record_sha256: String, + pub executable_sha256: String, + pub expires_at_unix_seconds: u64, + pub replacements: Vec, +} + +impl Approval { + pub fn apply(&self, plan: &mut Plan, inventory: &[serde_json::Value]) -> Result<()> { + ensure!( + self.host == plan.host + && self.replacements.len() == plan.active.len() + && inventory.len() == plan.active.len(), + "recovery instance set changed" + ); + let mut verified = Vec::new(); + for active in &plan.active { + let replacement = self + .replacements + .iter() + .find(|r| r.instance_id == active.before.id) + .context("recovery replacement missing")?; + let mut expected = active.container.clone(); + ensure!( + replacement.generation > expected.generation, + "replacement generation must advance" + ); + expected.generation = replacement.generation; + expected.container_id.clone_from(&replacement.container_id); + ensure!( + replacement == &expected, + "recovery changes immutable engine identity" + ); + let inspect = inventory + .iter() + .find(|v| v["Id"] == replacement.container_id) + .context("approved replacement container missing")?; + replacement.verify(inspect)?; + ensure!( + inspect["State"]["Running"] == true, + "replacement is not running" + ); + super::container::verify_restart_policy(inspect)?; + verified.push(replacement.clone()); + } + for (active, replacement) in plan.active.iter_mut().zip(verified) { + active.before.generation = replacement.generation; + active.container = replacement; + } + Ok(()) + } + pub fn verify( + bytes: &[u8], + signature: &str, + key: &str, + identities: [&str; 3], + now: u64, + ) -> Result { + ensure!(bytes.len() <= 65536, "recovery approval exceeds size limit"); + minisign_verify::PublicKey::from_base64(key)?.verify( + bytes, + &minisign_verify::Signature::decode(signature)?, + false, + )?; + let approval: Self = serde_json::from_slice(bytes)?; + ensure!( + approval.schema == "sparkplane.recovery-approval/v1" + && now < approval.expires_at_unix_seconds, + "unsupported or expired recovery approval" + ); + ensure!( + identities + .iter() + .all(|s| s.len() == 64 && s.bytes().all(|b| b.is_ascii_hexdigit())) + && identities + == [ + &approval.host, + &approval.record_sha256, + &approval.executable_sha256 + ], + "recovery approval identity mismatch" + ); + Ok(approval) + } +} diff --git a/src/migration/storage.rs b/src/migration/storage.rs index 628e511..558d6dc 100644 --- a/src/migration/storage.rs +++ b/src/migration/storage.rs @@ -14,6 +14,8 @@ pub struct Storage { original: Vec<(String, FileIdentity)>, candidate: FileIdentity, caches: Vec<(String, String, Identity)>, + #[serde(default)] + emergency: super::emergency::Journal, } fn cache_key(key: &str) -> bool { @@ -77,6 +79,7 @@ impl Storage { } } Ok(Self { + emergency: super::emergency::Journal::prepare(&data, work)?, data: Identity::read(&data)?, original, candidate: FileIdentity::read(&candidate)?, @@ -133,6 +136,7 @@ impl Storage { let parent = new.join("compile-cache"); relocate(&parent.join(from), &parent.join(to), id)?; } + self.emergency.activate(&new, work)?; Ok(()) } @@ -140,6 +144,8 @@ impl Storage { self.validate()?; let old = root.join("var/lib/sy-spark"); let new = root.join("var/lib/sparkplane"); + self.emergency + .restore(if new.try_exists()? { &new } else { &old }, work)?; if new.try_exists()? { for (from, to, id) in self.caches.iter().rev() { let parent = new.join("compile-cache"); diff --git a/src/spark/install.rs b/src/spark/install.rs index dacc6dd..af559c8 100644 --- a/src/spark/install.rs +++ b/src/spark/install.rs @@ -1085,6 +1085,16 @@ fn write_release_payload( ) -> Result<(), InstallError> { std::fs::create_dir_all(stage.join("configs/sparkplane/engines")) .map_err(|error| configuration_error(format!("create release payload: {error}")))?; + for relative in [ + "", + "configs", + "configs/sparkplane", + "configs/sparkplane/engines", + ] { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(stage.join(relative), std::fs::Permissions::from_mode(0o755)) + .map_err(|error| configuration_error(format!("chmod release directory: {error}")))?; + } for (relative, bytes) in release_payload_files(bundle) { let mode = if relative == "sparkplane" { 0o555 @@ -2362,6 +2372,36 @@ fn restart_control_plane() -> Result<(), InstallError> { ) } +#[cfg(feature = "appliance")] +fn public_code_permissions(root: &Path) -> Result<(), InstallError> { + use std::os::unix::fs::PermissionsExt; + if !std::fs::symlink_metadata(root).is_ok_and(|meta| meta.is_dir()) { + return Err(configuration_error( + "public code root must be a real directory", + )); + } + for entry in walkdir::WalkDir::new(root).follow_links(false) { + let entry = + entry.map_err(|error| configuration_error(format!("inspect public code: {error}")))?; + if entry.file_type().is_dir() || entry.file_type().is_file() { + let metadata = entry + .metadata() + .map_err(|error| configuration_error(format!("stat public code: {error}")))?; + let mode = if entry.file_type().is_dir() || metadata.permissions().mode() & 0o111 != 0 { + 0o755 + } else { + (metadata.permissions().mode() & 0o700) | 0o044 + }; + std::fs::set_permissions(entry.path(), std::fs::Permissions::from_mode(mode)) + .map_err(|error| configuration_error(format!("chmod public code: {error}")))?; + File::open(entry.path()) + .and_then(|file| file.sync_all()) + .map_err(|error| configuration_error(format!("sync public code: {error}")))?; + } + } + Ok(()) +} + #[cfg(feature = "appliance")] pub(crate) fn ensure_http_fallback() -> Result<(), InstallError> { use std::os::unix::fs::symlink; @@ -2399,6 +2439,7 @@ pub(crate) fn ensure_http_fallback() -> Result<(), InstallError> { )); } } + public_code_permissions(&release)?; let current = root.join("current"); if std::fs::read_link(¤t).ok().as_deref() != Some(Path::new(&digest)) { let staged = root.join(format!(".current-{}", uuid::Uuid::new_v4())); @@ -4199,6 +4240,74 @@ mod tests { } } + #[cfg(feature = "appliance")] + #[test] + fn release_payload_is_traversable_after_private_staging() { + use std::os::unix::fs::PermissionsExt; + let stage = tempfile::tempdir().unwrap(); + let bundle = signed_bundle(b"binary", "", "", ""); + for relative in [ + "", + "configs", + "configs/sparkplane", + "configs/sparkplane/engines", + ] { + let path = stage.path().join(relative); + std::fs::create_dir_all(&path).unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + super::write_release_payload(stage.path(), &bundle, &mut vec![]).unwrap(); + for relative in [ + "", + "configs", + "configs/sparkplane", + "configs/sparkplane/engines", + ] { + assert_eq!( + std::fs::metadata(stage.path().join(relative)) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o755 + ); + } + } + + #[cfg(feature = "appliance")] + #[test] + fn fallback_public_code_permissions_do_not_follow_symlinks() { + use std::os::unix::fs::PermissionsExt; + let root = tempfile::tempdir().unwrap(); + let code = root.path().join("code"); + std::fs::create_dir(&code).unwrap(); + std::fs::set_permissions(&code, std::fs::Permissions::from_mode(0o700)).unwrap(); + for (name, mode) in [("library.py", 0o600), ("cli", 0o700)] { + std::fs::write(code.join(name), b"public code").unwrap(); + std::fs::set_permissions(code.join(name), std::fs::Permissions::from_mode(mode)) + .unwrap(); + } + let private = root.path().join("private"); + std::fs::write(&private, b"private").unwrap(); + std::fs::set_permissions(&private, std::fs::Permissions::from_mode(0o600)).unwrap(); + std::os::unix::fs::symlink(&private, code.join("link")).unwrap(); + super::public_code_permissions(&code).unwrap(); + assert_eq!( + std::fs::metadata(&code).unwrap().permissions().mode() & 0o777, + 0o755 + ); + for (path, mode) in [ + (code.join("library.py"), 0o644), + (code.join("cli"), 0o755), + (private, 0o600), + ] { + assert_eq!( + std::fs::metadata(path).unwrap().permissions().mode() & 0o777, + mode + ); + } + } + #[test] fn host_alias_and_paths_are_discrete_argv() { let transfer = ProbeTransfer { diff --git a/tests/migration_emergency.rs b/tests/migration_emergency.rs new file mode 100644 index 0000000..86e26f7 --- /dev/null +++ b/tests/migration_emergency.rs @@ -0,0 +1,141 @@ +#![cfg(feature = "appliance")] +use sparkplane::migration::emergency; +use std::{fs, os::unix::fs::PermissionsExt}; + +fn fixture() -> (tempfile::TempDir, Vec) { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join("executor")).unwrap(); + let record = serde_json::json!({"schema":"sy.spark.emergency-record/v1", + "event_id":"01M303PWWXR02FK2H7XN15NV7B","occurred_at_unix_ms":123, + "decision":{"schema":"sy.spark.emergency-decision/v1","instance_id":"fixture", + "generation":8,"cause":"memory floor","mem_available_bytes":7, + "memory_full_psi_avg10_percent":0.0}}); + let bytes = format!("{record}\n").into_bytes(); + let path = root.path().join("executor/emergency.jsonl"); + fs::write(&path, &bytes).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap(); + let db = rusqlite::Connection::open(root.path().join("state.sqlite3")).unwrap(); + db.execute_batch( + "CREATE TABLE emergency_records(event_id TEXT PRIMARY KEY,evidence_json TEXT NOT NULL)", + ) + .unwrap(); + db.execute( + "INSERT INTO emergency_records VALUES(?1,?2)", + [record["event_id"].as_str().unwrap(), &record.to_string()], + ) + .unwrap(); + (root, bytes) +} + +#[test] +fn emergency_import_changes_only_typed_namespace_and_preserves_source() { + let (root, original) = fixture(); + let candidate = emergency::validate(root.path()).unwrap().unwrap(); + let mut expected: serde_json::Value = serde_json::from_slice(&original).unwrap(); + expected["schema"] = "sparkplane.emergency-record/v1".into(); + expected["decision"]["schema"] = "sparkplane.emergency-decision/v1".into(); + assert_eq!( + serde_json::from_slice::(&candidate).unwrap(), + expected + ); + assert_eq!( + fs::read(root.path().join("executor/emergency.jsonl")).unwrap(), + original + ); +} + +#[test] +fn emergency_cutover_and_recovery_preserve_original_bytes() { + let (root, original) = fixture(); + let work = tempfile::tempdir().unwrap(); + let journal = emergency::Journal::prepare(root.path(), work.path()).unwrap(); + journal.activate(root.path(), work.path()).unwrap(); + journal.activate(root.path(), work.path()).unwrap(); + assert!( + fs::read_to_string(root.path().join("executor/emergency.jsonl")) + .unwrap() + .contains("sparkplane.emergency-record/v1") + ); + journal.restore(root.path(), work.path()).unwrap(); + journal.restore(root.path(), work.path()).unwrap(); + assert_eq!( + fs::read(root.path().join("executor/emergency.jsonl")).unwrap(), + original + ); +} + +#[test] +fn emergency_rejects_unimported_or_changed_evidence_and_unknown_schema() { + for mutation in [ + "DELETE FROM emergency_records", + "UPDATE emergency_records SET evidence_json='{}'", + ] { + let (root, _) = fixture(); + rusqlite::Connection::open(root.path().join("state.sqlite3")) + .unwrap() + .execute_batch(mutation) + .unwrap(); + assert!(emergency::validate(root.path()).is_err()); + } + let (root, original) = fixture(); + fs::write( + root.path().join("executor/emergency.jsonl"), + String::from_utf8(original) + .unwrap() + .replace("sy.spark.emergency-record/v1", "unknown/v1"), + ) + .unwrap(); + assert!(emergency::validate(root.path()).is_err()); +} + +#[test] +fn every_partial_emergency_exchange_recovers_twice_without_discarding_new_records() { + for cutoff in 0..=2 { + let (root, original) = fixture(); + let work = tempfile::tempdir().unwrap(); + let journal = emergency::Journal::prepare(root.path(), work.path()).unwrap(); + let active = root.path().join("executor/emergency.jsonl"); + if cutoff >= 1 { + fs::rename(&active, work.path().join("original-emergency.jsonl")).unwrap(); + } + if cutoff == 2 { + fs::rename(work.path().join("candidate-emergency.jsonl"), &active).unwrap(); + fs::write(&active, b"failed generation evidence").unwrap(); + } + journal.restore(root.path(), work.path()).unwrap(); + journal.restore(root.path(), work.path()).unwrap(); + assert_eq!(fs::read(active).unwrap(), original); + if cutoff == 2 { + assert_eq!( + fs::read(work.path().join("failed-emergency.jsonl")).unwrap(), + b"failed generation evidence" + ); + } + } +} + +#[test] +fn emergency_absence_and_symlink_conflicts_are_explicit() { + let root = tempfile::tempdir().unwrap(); + let work = tempfile::tempdir().unwrap(); + let journal = emergency::Journal::prepare(root.path(), work.path()).unwrap(); + journal.activate(root.path(), work.path()).unwrap(); + fs::create_dir(root.path().join("executor")).unwrap(); + fs::write( + root.path().join("executor/emergency.jsonl"), + b"new evidence", + ) + .unwrap(); + journal.restore(root.path(), work.path()).unwrap(); + journal.restore(root.path(), work.path()).unwrap(); + assert_eq!( + fs::read(work.path().join("failed-emergency.jsonl")).unwrap(), + b"new evidence" + ); + std::os::unix::fs::symlink( + work.path().join("failed-emergency.jsonl"), + root.path().join("executor/emergency.jsonl"), + ) + .unwrap(); + assert!(emergency::validate(root.path()).is_err()); +} diff --git a/tests/migration_fence.rs b/tests/migration_fence.rs index c8725b3..5bf1ec2 100644 --- a/tests/migration_fence.rs +++ b/tests/migration_fence.rs @@ -1,6 +1,27 @@ #![cfg(feature = "appliance")] use sparkplane::migration::fence; +#[test] +fn model_warmup_fits_inside_agent_startup_deadline() { + let unit = include_str!("../configs/systemd/system/sparkplane-agent.service"); + assert!(unit.lines().any(|line| line == "TimeoutStartSec=1900s")); +} + +#[test] +fn recovery_extends_startup_without_rewriting_original_guards() { + let root = tempfile::tempdir().unwrap(); + fence::install_guards(root.path()).unwrap(); + let path = root + .path() + .join("etc/systemd/system/sy-spark-agent.service.d/91-sparkplane-migration-startup.conf"); + assert_eq!( + std::fs::read_to_string(&path).unwrap(), + "[Service]\nTimeoutStartSec=1900s\n" + ); + fence::remove_guards(root.path()).unwrap(); + assert!(!path.exists()); +} + #[test] fn reboot_guard_requires_a_volatile_permit_for_both_namespaces() { let root = tempfile::tempdir().unwrap(); diff --git a/tests/migration_host.rs b/tests/migration_host.rs index 18aba45..cc8d3df 100644 --- a/tests/migration_host.rs +++ b/tests/migration_host.rs @@ -121,6 +121,69 @@ struct Fixture { release: Release, } +#[test] +fn approved_replacement_changes_only_the_in_memory_recovery_generation() { + let mut f = fixture_with_instance(true); + let original = serde_json::to_vec(&f.host.record).unwrap(); + let mut replacement = f.host.record.plan.active[0].container.clone(); + replacement.generation += 1; + replacement.container_id = "e".repeat(64); + let approval: sparkplane::migration::recovery::Approval = serde_json::from_value(serde_json::json!({"schema":"sparkplane.recovery-approval/v1", "host":f.host.record.plan.host, "record_sha256":"b".repeat(64), "executable_sha256":"c".repeat(64), "expires_at_unix_seconds":200, "replacements":[replacement]})).unwrap(); + approval + .apply( + &mut f.host.record.plan, + &[inspect(&replacement, Namespace::Legacy)], + ) + .unwrap(); + assert_eq!( + f.host.record.plan.active[0].before.generation, + replacement.generation + ); + let original: Record = serde_json::from_slice(&original).unwrap(); + assert_eq!( + original.plan.active[0].before.generation + 1, + replacement.generation + ); +} + +#[test] +fn replacement_rejects_identity_drift_without_changing_the_plan() { + let mut f = fixture_with_instance(true); + let original = serde_json::to_vec(&f.host.record.plan).unwrap(); + let mut exact = f.host.record.plan.active[0].container.clone(); + exact.generation += 1; + exact.container_id = "e".repeat(64); + for field in [ + "instance_id", + "generation", + "image_digest", + "engine_id", + "engine_fingerprint", + "artifact_fingerprint", + "model_repository", + "model_commit", + "container_id", + ] { + let mut replacement = serde_json::to_value(&exact).unwrap(); + replacement[field] = if field == "generation" { + 0.into() + } else { + "different".into() + }; + let approval: sparkplane::migration::recovery::Approval = serde_json::from_value(serde_json::json!({"schema":"sparkplane.recovery-approval/v1", "host":f.host.record.plan.host, "record_sha256":"b".repeat(64), "executable_sha256":"c".repeat(64), "expires_at_unix_seconds":200, "replacements":[replacement]})).unwrap(); + assert!( + approval + .apply( + &mut f.host.record.plan, + &[inspect(&exact, Namespace::Legacy)] + ) + .is_err(), + "{field}" + ); + assert_eq!(serde_json::to_vec(&f.host.record.plan).unwrap(), original); + } +} + fn legacy(text: &str) -> String { text.replace("sparkplane.engine/", "sy.spark.engine/") .replace("sparkplane.models/", "sy.spark.models/") diff --git a/tests/migration_journal.rs b/tests/migration_journal.rs index 64e3377..e5f1281 100644 --- a/tests/migration_journal.rs +++ b/tests/migration_journal.rs @@ -1,6 +1,32 @@ #![cfg(feature = "appliance")] use sparkplane::migration::journal::{Journal, Step}; +#[test] +fn replacement_authority_is_only_valid_after_rollback_restores_state() { + let root = tempfile::tempdir().unwrap(); + let mut journal = Journal::open(root.path(), &"a".repeat(64), &"b".repeat(64)).unwrap(); + assert!(!journal.restored_checkpoint()); + for step in [ + Step::FenceTraffic, + Step::Drain, + Step::StopLegacyServices, + Step::Snapshot, + ] { + journal.begin(step).unwrap(); + journal.finish(step).unwrap(); + } + assert!(!journal.restored_checkpoint()); + assert!( + journal + .recover(|step| { + anyhow::ensure!(step != Step::StopLegacyServices, "wait for health"); + Ok(()) + }) + .is_err() + ); + assert!(journal.restored_checkpoint()); +} + #[test] fn interrupted_action_remains_pending_after_reopen() { let root = tempfile::tempdir().unwrap(); diff --git a/tests/migration_storage.rs b/tests/migration_storage.rs index aa18250..00bb95f 100644 --- a/tests/migration_storage.rs +++ b/tests/migration_storage.rs @@ -18,15 +18,51 @@ fn filesystem_cutover_and_recovery_preserve_original_database_and_weights() { ) .unwrap(); let original = std::fs::read(data.join("state.sqlite3")).unwrap(); + let emergency = r#"{"schema":"sy.spark.emergency-record/v1","event_id":"01M303PWWXR02FK2H7XN15NV7B","occurred_at_unix_ms":123,"decision":{"schema":"sy.spark.emergency-decision/v1","instance_id":"fixture","generation":8,"cause":"floor","mem_available_bytes":7,"memory_full_psi_avg10_percent":0.0}}"#; + std::fs::create_dir(data.join("executor")).unwrap(); + let journal_path = data.join("executor/emergency.jsonl"); + std::fs::write(&journal_path, emergency).unwrap(); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&journal_path, std::fs::Permissions::from_mode(0o600)).unwrap(); + rusqlite::Connection::open(data.join("state.sqlite3")).unwrap().execute("INSERT INTO emergency_records VALUES('01M303PWWXR02FK2H7XN15NV7B','fixture',8,'floor',?1,'fixture')", [emergency]).unwrap(); let storage = Storage::prepare(root.path(), &work, &[], &[]).unwrap(); storage.activate(root.path(), &work).unwrap(); storage.activate(root.path(), &work).unwrap(); + assert!( + std::fs::read_to_string( + root.path() + .join("var/lib/sparkplane/executor/emergency.jsonl") + ) + .unwrap() + .contains("sparkplane.emergency-record/v1") + ); + let current = root.path().join("var/lib/sparkplane"); + let replay = sparkplane::spark::resources::read_emergency_records( + ¤t.join("executor/emergency.jsonl"), + ) + .unwrap(); + let actor = sparkplane::spark::state::DbActor::open( + current.join("state.sqlite3"), + current.join("backups"), + 8, + 2, + secrecy::SecretString::from("fixture"), + ) + .unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + assert!( + !runtime + .block_on(actor.import_emergency(replay[0].clone())) + .unwrap() + ); + drop(actor); assert_eq!( std::fs::read(root.path().join("var/lib/sparkplane/huggingface/weights")).unwrap(), b"unchanged" ); storage.restore(root.path(), &work).unwrap(); storage.restore(root.path(), &work).unwrap(); + assert_eq!(std::fs::read_to_string(journal_path).unwrap(), emergency); assert!( std::fs::read(data.join("state.sqlite3")).unwrap() == original, "original database bytes must survive recovery" diff --git a/tests/migration_trust.rs b/tests/migration_trust.rs index c62c868..9b0b972 100644 --- a/tests/migration_trust.rs +++ b/tests/migration_trust.rs @@ -1,6 +1,52 @@ #![cfg(feature = "appliance")] use sparkplane::migration::{Transition, verify_transition}; +#[test] +fn recovery_approval_binds_executable_record_host_expiry_and_installed_key() { + let key = minisign::KeyPair::generate_unencrypted_keypair().unwrap(); + let bytes = serde_json::to_vec(&serde_json::json!({"schema":"sparkplane.recovery-approval/v1", "host":"a".repeat(64), "record_sha256":"b".repeat(64), "executable_sha256":"c".repeat(64), "expires_at_unix_seconds":200, "replacements":[]})).unwrap(); + let signature = minisign::sign(None, &key.sk, std::io::Cursor::new(&bytes), None, None) + .unwrap() + .to_string(); + let wrong = minisign::KeyPair::generate_unencrypted_keypair().unwrap(); + assert!( + sparkplane::migration::recovery::Approval::verify( + &bytes, + &signature, + &wrong.pk.to_base64(), + [&"a".repeat(64), &"b".repeat(64), &"c".repeat(64)], + 100 + ) + .is_err() + ); + let verify = |host: &str, record: &str, exe: &str, now| { + sparkplane::migration::recovery::Approval::verify( + &bytes, + &signature, + &key.pk.to_base64(), + [host, record, exe], + now, + ) + }; + assert!(verify(&"a".repeat(64), &"b".repeat(64), &"c".repeat(64), 100).is_ok()); + for (host, record, exe, now) in [ + ('d', 'b', 'c', 100), + ('a', 'd', 'c', 100), + ('a', 'b', 'd', 100), + ('a', 'b', 'c', 200), + ] { + assert!( + verify( + &host.to_string().repeat(64), + &record.to_string().repeat(64), + &exe.to_string().repeat(64), + now + ) + .is_err() + ); + } +} + #[test] fn transition_requires_the_installed_authority_and_exact_host_and_release() { let old = minisign::KeyPair::generate_unencrypted_keypair().unwrap(); diff --git a/tests/standalone_cli.rs b/tests/standalone_cli.rs index 4db7cfa..1603de4 100644 --- a/tests/standalone_cli.rs +++ b/tests/standalone_cli.rs @@ -33,10 +33,27 @@ fn appliance_migration_is_feature_gated_and_requires_one_explicit_mode() { }; let help = invoke(&["bootstrap", "migrate-appliance", "--help"]); assert_eq!(help.status.success(), cfg!(feature = "appliance")); + if cfg!(feature = "appliance") { + assert!(String::from_utf8_lossy(&help.stdout).contains("--recovery-approval")); + } for args in [ vec!["bootstrap", "migrate-appliance"], vec!["bootstrap", "migrate-appliance", "--yes", "--dry-run"], vec!["dgx-spark", "migrate-appliance", "--yes"], + vec![ + "bootstrap", + "migrate-appliance", + "--yes", + "--recovery-approval", + "approval.json", + ], + vec![ + "bootstrap", + "migrate-appliance", + "--recover", + "--recovery-approval", + "approval.json", + ], ] { assert!(!invoke(&args).status.success()); }