Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ members = [".", "crates/sparkplane-core", "crates/sparkplane-ipc"]
resolver = "3"

[workspace.package]
version = "0.1.3"
version = "0.1.4"
edition = "2024"
rust-version = "1.95"
license = "MIT"
Expand All @@ -22,8 +22,8 @@ tracing = "0.1"
tracing-subscriber = { version = "0.3", default-features = false, features = ["env-filter", "fmt", "json", "registry", "std"] }
metrics = "0.24"
tempfile = "3"
sparkplane-core = { path = "crates/sparkplane-core", version = "=0.1.3" }
sparkplane-ipc = { path = "crates/sparkplane-ipc", version = "=0.1.3" }
sparkplane-core = { path = "crates/sparkplane-core", version = "=0.1.4" }
sparkplane-ipc = { path = "crates/sparkplane-ipc", version = "=0.1.4" }

[package]
name = "sparkplane"
Expand Down
2 changes: 2 additions & 0 deletions configs/systemd/system/sparkplane-agent.service
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ MemoryMax=256M
OOMScoreAdjust=-100
AppArmorProfile=sparkplane-agent
NotifyAccess=main
# Initial reconciliation may wait for the engine's 1800-second cold start.
TimeoutStartSec=1900s
WatchdogSec=30s
TimeoutStopSec=15s
Restart=on-failure
Expand Down
10 changes: 10 additions & 0 deletions docs/how-to/develop-spark.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,11 @@ commit, artifact name, and manifest signature in the release notes. The
artifact must be the one whose manifest was signed; do not rebuild it after
signing.

Installation assigns explicit permissions independently of the operator's
umask. Public release directories are root-owned and traversable by the service
account; executable and catalog payloads remain read-only. Private credentials,
state and rollback evidence retain their restricted modes.

## Install or upgrade Spark

For a new host, use the bootstrap installer. Always inspect the dry run first:
Expand Down Expand Up @@ -535,6 +540,11 @@ catalogs or the CLI.

## Release checklist

The agent's initial reconciliation can wait for the engine's 1800-second cold
start. Its notify service therefore allows 1900 seconds for startup; do not
restore systemd's shorter default or restart the service while weights load.
Readiness remains gated on semantic engine health, not merely a running process.

Before tagging:

- catalog boundary tests pass and every engine/model entry validates;
Expand Down
2 changes: 2 additions & 0 deletions src/migration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ pub mod commands;
pub mod config;
pub mod container;
pub mod docker;
pub mod emergency;
pub mod fence;
pub mod host;
pub mod journal;
pub mod publication;
pub mod qualification;
pub mod recovery;
pub mod release;
pub mod relocation;
pub mod runner;
Expand Down
2 changes: 2 additions & 0 deletions src/migration/appliance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ pub fn verify_sources(root: &Path, plan: &Plan) -> Result<()> {
)?;
connection.execute_batch("BEGIN")?;
super::verify_database(&connection)?;
super::emergency::validate(&root.join("var/lib/sy-spark"))?;
ensure!(
inventory_digest(&connection)? == plan.inventory_sha256,
"source inventory changed after preflight"
Expand Down Expand Up @@ -248,6 +249,7 @@ pub fn preflight(
)?;
connection.execute_batch("BEGIN")?;
super::verify_database(&connection)?;
super::emergency::validate(&data)?;
let inventory_sha256 = inventory_digest(&connection)?;
let mut statement =
connection.prepare("SELECT id,repository,commit_sha,metadata_json FROM models")?;
Expand Down
80 changes: 73 additions & 7 deletions src/migration/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,16 @@ pub struct MigrationArgs {
/// Restore an interrupted pre-commit transaction; never restores after commit.
#[arg(long, env = "SPARKPLANE_MIGRATION_RECOVER")]
pub recover: bool,
/// Installed-authority-signed approval of an exact post-restoration recovery handoff.
#[arg(long, env = "SPARKPLANE_RECOVERY_APPROVAL", requires_all = ["recover", "recovery_signature"])]
pub recovery_approval: Option<PathBuf>,
/// Detached signature of --recovery-approval; never authorizes a forward migration.
#[arg(
long,
env = "SPARKPLANE_RECOVERY_SIGNATURE",
requires = "recovery_approval"
)]
pub recovery_signature: Option<PathBuf>,
/// Finish opening traffic after a committed transaction; never reapplies state.
#[arg(long, env = "SPARKPLANE_MIGRATION_RESUME")]
pub resume: bool,
Expand Down Expand Up @@ -96,6 +106,51 @@ fn self_digest() -> Result<String> {
)?))
}

fn recovery_approval(
args: &MigrationArgs,
record: &[u8],
host: &str,
executable: &str,
) -> Result<Option<(super::recovery::Approval, Vec<u8>, String)>> {
let Some(path) = &args.recovery_approval else {
return Ok(None);
};
ensure!(args.recover && !args.resume, "handoff is recovery-only");
let bytes = super::release::read(path, 65536)?;
let signature = String::from_utf8(super::release::read(
args.recovery_signature
.as_ref()
.context("recovery signature required")?,
4096,
)?)?;
let approval = super::recovery::Approval::verify(
&bytes,
&signature,
&old_authority()?,
[host, &appliance::digest(record), executable],
SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(),
)?;
Ok(Some((approval, bytes, signature)))
}

fn receipt(work: &Path, bytes: &[u8], signature: &[u8]) -> Result<()> {
for (suffix, value) in [("json", bytes), ("minisig", signature)] {
let path = work.join(format!(
"recovery-approval-{}.{suffix}",
appliance::digest(bytes)
));
if path.try_exists()? {
ensure!(
super::release::read(&path, 65536)? == value,
"recovery receipt changed"
);
} else {
write_private(&path, value)?;
}
}
Ok(())
}

pub fn dispatch(host: &str, args: MigrationArgs) -> Result<()> {
ensure!(
host == "bootstrap",
Expand All @@ -119,17 +174,28 @@ pub fn dispatch(host: &str, args: MigrationArgs) -> Result<()> {
private_directory(base)?;
let _lock = lock(base)?;
private_directory(&work)?;
let record: Record = serde_json::from_slice(&super::release::read(
&work.join("record.json"),
8 * 1024 * 1024,
)?)?;
let record_bytes = super::release::read(&work.join("record.json"), 8 * 1024 * 1024)?;
let mut record: Record = serde_json::from_slice(&record_bytes)?;
let executable = self_digest()?;
ensure!(
record.plan.schema == "sparkplane.appliance-migration/v1"
&& record.plan.host == appliance::host_identity(root)?
&& record.plan.executable == self_digest()?,
"recovery requires the original approved executable and host"
&& record.plan.host == appliance::host_identity(root)?,
"recovery requires the original approved host and schema"
);
let approval = recovery_approval(&args, &record_bytes, &record.plan.host, &executable)?;
ensure!(
approval.is_some() || record.plan.executable == executable,
"recovery requires the original executable or signed handoff"
);
let mut journal = Journal::open(&work, &record.plan.host, &record.plan.release)?;
if let Some((approval, bytes, signature)) = approval {
ensure!(
journal.restored_checkpoint(),
"recovery handoff requires restored pre-commit state"
);
approval.apply(&mut record.plan, &platform.inventory(Namespace::Legacy)?)?;
receipt(&work, &bytes, signature.as_bytes())?;
}
let mut appliance = Host {
root: root.into(),
work: work.clone(),
Expand Down
2 changes: 1 addition & 1 deletion src/migration/container.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ impl Namespace {
}
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct LegacyContainer {
pub container_id: String,
Expand Down
131 changes: 131 additions & 0 deletions src/migration/emergency.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
//! Preserve executor emergency evidence while converting its typed replay journal.
use crate::spark::resources::{EmergencyRecord, read_emergency_records};
use anyhow::{Result, ensure};
use rusqlite::{Connection, OpenFlags};
use std::{fs, path::Path};

const JOURNAL: &str = "executor/emergency.jsonl";

#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Journal {
original: Option<super::relocation::FileIdentity>,
candidate: Option<super::relocation::FileIdentity>,
}

impl Journal {
pub fn prepare(data: &Path, work: &Path) -> Result<Self> {
use std::os::unix::fs::MetadataExt;
let Some(bytes) = validate(data)? else {
return Ok(Self::default());
};
let source = data.join(JOURNAL);
let original = super::relocation::FileIdentity::read(&source)?;
let candidate = work.join("candidate-emergency.jsonl");
super::publication::write_private(&candidate, &bytes)?;
let metadata = fs::symlink_metadata(source)?;
rustix::fs::chown(
&candidate,
Some(rustix::fs::Uid::from_raw(metadata.uid())),
Some(rustix::fs::Gid::from_raw(metadata.gid())),
)?;
fs::File::open(&candidate)?.sync_all()?;
Ok(Self {
original: Some(original),
candidate: Some(super::relocation::FileIdentity::read(&candidate)?),
})
}

pub fn activate(&self, data: &Path, work: &Path) -> Result<()> {
use super::relocation::{FileIdentity, move_file};
ensure!(
self.original.is_some() == self.candidate.is_some(),
"invalid emergency snapshot"
);
if let (Some(original), Some(candidate)) = (&self.original, &self.candidate) {
let backup = work.join("original-emergency.jsonl");
if backup.try_exists()? {
ensure!(
FileIdentity::read(&backup)? == *original,
"emergency snapshot changed"
);
} else {
move_file(&data.join(JOURNAL), &backup, original)?;
}
move_file(
&work.join("candidate-emergency.jsonl"),
&data.join(JOURNAL),
candidate,
)?;
}
Ok(())
}

pub fn restore(&self, data: &Path, work: &Path) -> Result<()> {
use super::relocation::{FileIdentity, move_file};
let backup = work.join("original-emergency.jsonl");
let current = data.join(JOURNAL);
if let Some(original) = &self.original
&& !backup.try_exists()?
{
ensure!(
FileIdentity::read(&current)? == *original,
"original emergency journal is missing"
);
return Ok(());
}
if current.try_exists()? || current.is_symlink() {
move_file(
&current,
&work.join("failed-emergency.jsonl"),
&FileIdentity::read(&current)?,
)?;
}
if let Some(original) = &self.original {
move_file(&backup, &current, original)?;
}
Ok(())
}
}

pub fn validate(data: &Path) -> Result<Option<Vec<u8>>> {
ensure!(
!data.join("executor").is_symlink(),
"executor journal directory is a symlink"
);
let metadata = match fs::symlink_metadata(data.join(JOURNAL)) {
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
result => result?,
};
ensure!(
metadata.is_file(),
"legacy emergency journal must be a regular file"
);
let records = read_emergency_records(&data.join(JOURNAL))
.map_err(|_| anyhow::anyhow!("legacy emergency journal is invalid"))?;
let db =
Connection::open_with_flags(data.join("state.sqlite3"), OpenFlags::SQLITE_OPEN_READ_ONLY)?;
let mut output = Vec::new();
for mut record in records {
ensure!(
record.schema == "sy.spark.emergency-record/v1"
&& record.decision.schema == "sy.spark.emergency-decision/v1"
&& record.event_id.parse::<ulid::Ulid>().is_ok(),
"unsupported legacy emergency record"
);
let evidence: String = db.query_row(
"SELECT evidence_json FROM emergency_records WHERE event_id=?1",
[&record.event_id],
|row| row.get(0),
)?;
ensure!(
serde_json::from_str::<EmergencyRecord>(&evidence)? == record,
"legacy emergency journal differs from imported evidence"
);
record.schema = "sparkplane.emergency-record/v1".into();
record.decision.schema = "sparkplane.emergency-decision/v1".into();
serde_json::to_writer(&mut output, &record)?;
output.push(b'\n');
}
Ok(Some(output))
}
Loading
Loading