diff --git a/.github/ci-path-filters.yml b/.github/ci-path-filters.yml index 812b9af1b..85afe56d9 100644 --- a/.github/ci-path-filters.yml +++ b/.github/ci-path-filters.yml @@ -23,6 +23,8 @@ shared: - 'crates/adaptive/src/**' - 'crates/core/Cargo.toml' - 'crates/core/src/**' + - 'crates/plugin-host-config/Cargo.toml' + - 'crates/plugin-host-config/src/**' - 'crates/types/Cargo.toml' - 'crates/types/src/**' - 'crates/worker-proto/Cargo.toml' @@ -45,6 +47,8 @@ rust_package: - 'crates/core/src/**' - 'crates/plugin/Cargo.toml' - 'crates/plugin/src/**' + - 'crates/plugin-host-config/Cargo.toml' + - 'crates/plugin-host-config/src/**' - 'crates/types/Cargo.toml' - 'crates/types/src/**' - 'crates/worker-proto/Cargo.toml' @@ -96,6 +100,8 @@ python_package: - 'crates/adaptive/src/**' - 'crates/core/Cargo.toml' - 'crates/core/src/**' + - 'crates/plugin-host-config/Cargo.toml' + - 'crates/plugin-host-config/src/**' - 'crates/types/Cargo.toml' - 'crates/types/src/**' - 'crates/python/Cargo.toml' @@ -152,6 +158,7 @@ docs: - 'crates/node/*.d.ts' - 'crates/node/*.js' - 'crates/node/package.json' + - 'crates/plugin-host-config/src/**' - 'crates/*/Cargo.toml' - 'crates/**/*.md' - 'fern/**' @@ -195,6 +202,8 @@ openclaw: - 'integrations/openclaw/**' python: + - 'crates/plugin-host-config/Cargo.toml' + - 'crates/plugin-host-config/src/**' - 'crates/python/Cargo.toml' - 'crates/python/src/**' - 'pyproject.toml' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6bd1e232b..b3d0dbac5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -411,6 +411,7 @@ jobs: nemo-relay-worker-proto nemo-relay-worker nemo-relay + nemo-relay-plugin-host-config nemo-relay-adaptive nemo-relay-pii-redaction nemo-relay-switchyard diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 92153c744..cf3824422 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -275,7 +275,7 @@ publish:artifactory:cargo: artifactory = { index = "sparse+${NEMO_RELAY_CI_ARTIFACTORY_CARGO_URL}" } EOF export CARGO_REGISTRIES_ARTIFACTORY_TOKEN="Bearer ${NEMO_RELAY_CI_ARTIFACTORY_KEY}" - export NEMO_RELAY_ARTIFACTORY_CRATE_DIRS="types plugin worker-proto worker core adaptive pii-redaction switchyard ffi cli" + export NEMO_RELAY_ARTIFACTORY_CRATE_DIRS="types plugin worker-proto worker core plugin-host-config adaptive pii-redaction switchyard ffi cli" crates="$( uv run --no-project python - <<'PY' diff --git a/Cargo.lock b/Cargo.lock index 7f32dcff2..f899cf54a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1798,6 +1798,7 @@ dependencies = [ "nemo-relay", "nemo-relay-adaptive", "nemo-relay-pii-redaction", + "nemo-relay-plugin-host-config", "nemo-relay-switchyard", "opentelemetry", "opentelemetry_sdk", @@ -1891,6 +1892,28 @@ dependencies = [ "serde_json", ] +[[package]] +name = "nemo-relay-plugin-host-config" +version = "0.7.0" +dependencies = [ + "base64", + "dunce", + "fs2", + "libc", + "log", + "nemo-relay", + "ring", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror 2.0.18", + "tokio", + "toml", + "uuid", + "windows-sys 0.61.2", +] + [[package]] name = "nemo-relay-python" version = "0.7.0" @@ -1899,6 +1922,7 @@ dependencies = [ "nemo-relay", "nemo-relay-adaptive", "nemo-relay-pii-redaction", + "nemo-relay-plugin-host-config", "pyo3", "pyo3-async-runtimes", "pythonize", diff --git a/Cargo.toml b/Cargo.toml index 4d711cf0a..b3fd304ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ [workspace] members = [ "crates/core", + "crates/plugin-host-config", "crates/types", "crates/plugin", "crates/worker-proto", @@ -28,6 +29,7 @@ repository = "https://github.com/NVIDIA/NeMo-Relay" [workspace.dependencies] nemo-relay = { version = "0.7.0", path = "crates/core", default-features = false } +nemo-relay-plugin-host-config = { version = "0.7.0", path = "crates/plugin-host-config" } nemo-relay-types = { version = "0.7.0", path = "crates/types" } nemo-relay-plugin = { version = "0.7.0", path = "crates/plugin" } nemo-relay-worker-proto = { version = "0.7.0", path = "crates/worker-proto" } diff --git a/RELEASING.md b/RELEASING.md index f2ce9ba00..9e654f6b1 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -30,7 +30,7 @@ The release pipeline publishes these package surfaces from a tag push: | Ecosystem | Published Surface | |---|---| -| crates.io | `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, `nemo-relay-cli` | +| crates.io | `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-plugin-host-config`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, `nemo-relay-cli` | | PyPI | `nemo-relay` wheels and source distribution, `nemo-relay-plugin` and `nemo-relay-cli-bin` wheels | | npm | `nemo-relay-node` and its seven platform packages, `nemo-relay-openclaw`, `nemo-relay-cli-bin`, and its seven platform packages | | GitHub Releases | CLI binaries, `nemo-relay` and `nemo-relay-cli-bin` wheels, CLI and Node npm tarballs, and checksums | @@ -52,7 +52,8 @@ NeMo Relay versions are anchored on the workspace SemVer in the repository root version for the Rust workspace. - The root `Cargo.toml` `workspace.dependencies` entries for `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, - `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, + `nemo-relay-worker`, `nemo-relay`, `nemo-relay-plugin-host-config`, + `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, and `nemo-relay-cli` must stay aligned with that same version. @@ -138,7 +139,8 @@ Before you create a release tag, confirm the following: - GitHub Actions `id-token: write` access for the top-level crates.io publish job - crates.io trusted publishers for `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, - `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, + `nemo-relay`, `nemo-relay-plugin-host-config`, `nemo-relay-adaptive`, + `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, and `nemo-relay-cli` are configured for the top-level [`.github/workflows/ci.yaml`](.github/workflows/ci.yaml) workflow @@ -173,7 +175,8 @@ The helper updates: 1. The root [`Cargo.toml`](Cargo.toml) workspace version. 2. The root [`Cargo.toml`](Cargo.toml) `workspace.dependencies` versions for `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, - `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, + `nemo-relay-worker`, `nemo-relay`, `nemo-relay-plugin-host-config`, + `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, and `nemo-relay-cli`. 3. [`crates/node/package.json`](crates/node/package.json) and the `crates/node` @@ -285,7 +288,8 @@ The release pipeline then: - `publish-rust` stamps Cargo workspace versions from the release tag, then runs `cargo publish --package` for `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, - `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, + `nemo-relay`, `nemo-relay-plugin-host-config`, `nemo-relay-adaptive`, + `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, and `nemo-relay-cli` through trusted publishing from the top-level workflow @@ -357,7 +361,8 @@ for that tag. After the release is live, verify: 1. The `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, - `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, + `nemo-relay-worker`, `nemo-relay`, `nemo-relay-plugin-host-config`, + `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, and `nemo-relay-cli` crates are visible on crates.io. diff --git a/codecov.yml b/codecov.yml index c52334c14..f4fc27fd1 100644 --- a/codecov.yml +++ b/codecov.yml @@ -112,6 +112,7 @@ component_management: - "crates/plugin/src" - "crates/worker-proto/src" - "crates/worker/src" + - "crates/plugin-host-config/src" - "python/plugin/src/nemo_relay_plugin" statuses: - type: project diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index e04210891..72f835e25 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -27,6 +27,7 @@ switchyard = ["dep:nemo-relay-switchyard"] [dependencies] nemo-relay = { workspace = true, features = ["guardrails-remote", "object-store", "worker-grpc"] } +nemo-relay-plugin-host-config.workspace = true nemo-relay-adaptive = { workspace = true, features = ["redis-backend"] } nemo-relay-pii-redaction.workspace = true nemo-relay-switchyard = { workspace = true, optional = true, features = ["schema"] } diff --git a/crates/cli/src/configuration/mod.rs b/crates/cli/src/configuration/mod.rs index 2200b9c48..99ae0aee2 100644 --- a/crates/cli/src/configuration/mod.rs +++ b/crates/cli/src/configuration/mod.rs @@ -6,6 +6,7 @@ mod types; pub(crate) use types::*; +#[cfg(test)] use std::collections::HashSet; use std::env; use std::fs::{self, OpenOptions}; @@ -19,13 +20,12 @@ use nemo_relay::logging::LoggingConfig; use nemo_relay::plugin::dynamic::{ DYNAMIC_PLUGIN_MANIFEST_FILENAME, DynamicPluginManifest, DynamicPluginManifestLoad, }; -use nemo_relay::plugin::{ - PluginError, deduplicate_plugin_config_paths, merge_plugin_config_documents, -}; use ring::rand::{SecureRandom, SystemRandom}; use ring::{digest, hmac}; use serde::Deserialize; -use serde_json::{Map, Value}; +#[cfg(test)] +use serde_json::Map; +use serde_json::Value; use crate::error::CliError; use crate::filesystem::{LockAttempt, try_lock_exclusive, try_lock_shared}; @@ -604,6 +604,7 @@ pub(crate) fn sign_python_environment_attestation( )) } +#[cfg(test)] pub(crate) fn verify_python_environment_attestation( source_artifact_sha256: &str, environment_sha256: &str, @@ -1298,8 +1299,10 @@ struct PluginTomlConfig { dynamic_plugins: Vec, dynamic_plugin_policy: DynamicPluginHostPolicy, contributing_sources: Vec, + selected_sources: Vec, } +#[cfg(test)] #[derive(Debug, Clone, Default, Deserialize)] struct PluginTomlPluginsSection { #[serde(default)] @@ -1308,6 +1311,7 @@ struct PluginTomlPluginsSection { policy: Option, } +#[cfg(test)] #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] struct FileDynamicPluginConfig { @@ -1365,69 +1369,34 @@ fn load_plugin_toml_config_from_paths(paths: I) -> Result, { - let paths = deduplicate_plugin_config_paths(paths); - let mut dynamic_plugins = Vec::new(); - let mut dynamic_plugin_policy = DynamicPluginHostPolicy::default(); - let mut seen_plugin_ids = HashSet::new(); - let mut contributing_sources = Vec::new(); - let mut runtime_documents = Vec::new(); - - for path in &paths { - let Some(raw) = read_config_file(path, false, "plugin configuration")? else { - continue; - }; - let mut parsed = raw - .parse::() - .map(toml::Value::Table) - .map_err(|error| { - CliError::Config(format!( - "invalid plugin TOML in {}: {error}", - path.display() - )) - })?; - let resolved_plugins = - resolve_dynamic_plugin_refs(path, &mut parsed, &mut seen_plugin_ids)?; - if !resolved_plugins.dynamic_plugins.is_empty() - || resolved_plugins.dynamic_plugin_policy != DynamicPluginHostPolicy::default() - { - contributing_sources.push(path.clone()); - } - dynamic_plugins.extend(resolved_plugins.dynamic_plugins); - dynamic_plugin_policy.merge_from(resolved_plugins.dynamic_plugin_policy); - runtime_documents.push(( - path.clone(), - serde_json::to_value(remove_dynamic_plugin_sections(parsed)) - .expect("toml value serializes to JSON"), - )); - } - - // Delegate merged runtime plugin config to the shared core primitive after dynamic refs have - // been validated independently. Documents remain ordered from lowest to highest precedence. - let resolved = merge_plugin_config_documents(runtime_documents).map_err(|err| match err { - PluginError::InvalidConfig(message) => CliError::Config(message), - other => CliError::Config(other.to_string()), - })?; - match resolved { - Some((value, sources)) => { - contributing_sources.extend(sources.iter().cloned()); - contributing_sources.sort(); - contributing_sources.dedup(); - Ok(Some(PluginTomlConfig { - value: plugin_toml_runtime_value(value), - dynamic_plugins, - dynamic_plugin_policy, - contributing_sources, - })) - } - None => Ok((!dynamic_plugins.is_empty() - || dynamic_plugin_policy != DynamicPluginHostPolicy::default()) - .then_some(PluginTomlConfig { - value: None, - dynamic_plugins, - dynamic_plugin_policy, - contributing_sources, - })), + let resolved = nemo_relay_plugin_host_config::resolve_plugin_files_from_paths(paths, None) + .map_err(|error| CliError::Config(error.to_string()))?; + if !resolved.had_input { + return Ok(None); } + // The shared resolver pins sources to their physical paths so lifecycle state and snapshots + // cannot be split across aliases. Keep the CLI's established presentation contract, however: + // contributing sources use the selected spelling and are sorted independently of precedence. + let mut contributing_sources = resolved.contributing_selected_sources; + contributing_sources.sort(); + contributing_sources.dedup(); + Ok(Some(PluginTomlConfig { + value: resolved.runtime_value, + dynamic_plugins: resolved + .dynamic_plugins + .into_iter() + .map(|plugin| ResolvedDynamicPluginConfig { + plugin_id: plugin.plugin_id, + manifest_ref: plugin.manifest_ref, + config: plugin.config, + has_explicit_config: plugin.has_explicit_config, + source: plugin.source, + }) + .collect(), + dynamic_plugin_policy: resolved.dynamic_plugin_policy, + contributing_sources, + selected_sources: resolved.selected_sources, + })) } fn apply_plugin_toml_config(resolved: &mut ResolvedConfig, plugin_toml: Option) { @@ -1439,13 +1408,16 @@ fn apply_plugin_toml_config(resolved: &mut ResolvedConfig, plugin_toml: Option

, dynamic_plugin_policy: DynamicPluginHostPolicy, } +#[cfg(test)] fn resolve_dynamic_plugin_refs( source: &Path, value: &mut toml::Value, @@ -1513,6 +1485,7 @@ fn resolve_dynamic_plugin_refs( }) } +#[cfg(test)] fn resolve_dynamic_manifest_path(source: &Path, manifest: &str) -> PathBuf { let manifest = PathBuf::from(manifest); if manifest.is_absolute() { @@ -1525,6 +1498,7 @@ fn resolve_dynamic_manifest_path(source: &Path, manifest: &str) -> PathBuf { } } +#[cfg(test)] fn plugin_toml_runtime_value(value: Value) -> Option { match value { Value::Object(ref object) if object.is_empty() => None, @@ -1532,6 +1506,7 @@ fn plugin_toml_runtime_value(value: Value) -> Option { } } +#[cfg(test)] fn remove_dynamic_plugin_sections(mut value: toml::Value) -> toml::Value { if let Some(root) = value.as_table_mut() && let Some(toml::Value::Table(plugins)) = root.get_mut("plugins") diff --git a/crates/cli/src/configuration/types.rs b/crates/cli/src/configuration/types.rs index 0f2c06a8c..059ba8d42 100644 --- a/crates/cli/src/configuration/types.rs +++ b/crates/cli/src/configuration/types.rs @@ -63,6 +63,7 @@ pub(crate) struct ResolvedConfig { pub(crate) logging: LoggingConfig, pub(crate) dynamic_plugins: Vec, pub(crate) dynamic_plugin_policy: DynamicPluginHostPolicy, + pub(crate) plugin_selected_sources: Vec, pub(crate) bootstrap_fingerprint: Option, } diff --git a/crates/cli/src/plugins/lifecycle/environment.rs b/crates/cli/src/plugins/lifecycle/environment.rs index 0298f25bf..209a7ade4 100644 --- a/crates/cli/src/plugins/lifecycle/environment.rs +++ b/crates/cli/src/plugins/lifecycle/environment.rs @@ -13,8 +13,9 @@ use nemo_relay::plugin::dynamic::{ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -pub(super) const MANAGED_ENVIRONMENTS_DIR: &str = ".dynamic-plugin-environments"; -pub(super) const ENVIRONMENT_ATTESTATION_FILE: &str = ".nemo-relay-environment.sha256"; +pub(super) use nemo_relay_plugin_host_config::{ + ENVIRONMENT_ATTESTATION_FILE, MANAGED_ENVIRONMENTS_DIR, +}; const MAX_ENVIRONMENT_FILES: usize = 100_000; pub(super) const MAX_ENVIRONMENT_DEPTH: usize = 128; #[cfg(test)] @@ -62,100 +63,7 @@ pub(super) fn validate_python_entrypoint_artifact( manifest: &DynamicPluginManifest, manifest_ref: &str, ) -> Result<(), String> { - let DynamicPluginManifestLoad::Worker(load) = &manifest.load else { - return Ok(()); - }; - if load.runtime != Some(WorkerRuntime::Python) { - return Ok(()); - } - - let source = manifest.source.as_ref().ok_or_else(|| { - "Python worker plugins must declare source.manifest_root and source.artifact".to_string() - })?; - let manifest_root = source - .manifest_root - .as_deref() - .map(str::trim) - .filter(|root| !root.is_empty()) - .ok_or_else(|| { - "Python worker plugins added through the CLI must declare source.manifest_root" - .to_string() - })?; - let artifact = source - .artifact - .as_deref() - .map(str::trim) - .filter(|artifact| !artifact.is_empty()) - .ok_or_else(|| "Python worker plugins must declare source.artifact".to_string())?; - let entrypoint = load - .entrypoint - .as_deref() - .map(str::trim) - .filter(|entrypoint| !entrypoint.is_empty()) - .ok_or_else(|| "Python worker plugins must declare load.entrypoint".to_string())?; - let (module, callable) = entrypoint.split_once(':').ok_or_else(|| { - format!( - "Python worker load.entrypoint '{entrypoint}' must use the unambiguous module:function form" - ) - })?; - if callable.is_empty() - || callable.contains(':') - || module.is_empty() - || module - .split('.') - .any(|segment| segment.is_empty() || segment.contains(['/', '\\', ':'])) - { - return Err(format!( - "Python worker load.entrypoint '{entrypoint}' must use the unambiguous module:function form" - )); - } - - let manifest_path = Path::new(manifest_ref); - let manifest_dir = manifest_path.parent().unwrap_or_else(|| Path::new(".")); - let unresolved_manifest_root = resolve_relative_path(manifest_dir, manifest_root); - let manifest_root = unresolved_manifest_root.canonicalize().map_err(|error| { - format!( - "could not resolve Python plugin source.manifest_root {}: {error}", - unresolved_manifest_root.display() - ) - })?; - let artifact = resolve_relative_path(manifest_dir, artifact) - .canonicalize() - .map_err(|error| format!("could not resolve Python source.artifact: {error}"))?; - let module_path = module - .split('.') - .fold(manifest_root.clone(), |path, segment| path.join(segment)); - let module_file = module_path.with_extension("py"); - let package_file = module_path.join("__init__.py"); - let mut candidates = [module_file, package_file] - .into_iter() - .filter(|path| path.is_file()) - .map(|path| { - path.canonicalize().map_err(|error| { - format!( - "could not resolve Python entrypoint module file {}: {error}", - path.display() - ) - }) - }) - .collect::, _>>()?; - candidates.sort(); - candidates.dedup(); - let [entrypoint_artifact] = candidates.as_slice() else { - return Err(format!( - "Python worker load.entrypoint '{entrypoint}' must resolve to exactly one source module under source.manifest_root; expected {} or {}", - module_path.with_extension("py").display(), - module_path.join("__init__.py").display() - )); - }; - if entrypoint_artifact != &artifact { - return Err(format!( - "Python worker load.entrypoint '{entrypoint}' resolves to {}, but integrity-checked source.artifact resolves to {}; the executed entrypoint module must be the integrity-checked artifact", - entrypoint_artifact.display(), - artifact.display() - )); - } - Ok(()) + nemo_relay_plugin_host_config::validate_python_entrypoint_artifact(manifest, manifest_ref) } pub(super) fn provision_python_environment( @@ -268,56 +176,23 @@ pub(super) fn read_environment_attestation( environment: &Path, expected_source_artifact_sha256: &str, ) -> Result { - let attestation_path = environment.join(ENVIRONMENT_ATTESTATION_FILE); - let raw = std::fs::read_to_string(&attestation_path) - .map_err(|error| format!("failed to read {}: {error}", attestation_path.display()))?; - let attestation = serde_json::from_str::(&raw).map_err(|error| { - format!( - "managed Python environment attestation {} is invalid: {error}", - attestation_path.display() - ) - })?; - if attestation.version != 1 - || attestation.source_artifact_sha256 != expected_source_artifact_sha256.trim() - || attestation.environment_sha256.len() != 64 - || !attestation - .environment_sha256 - .bytes() - .all(|byte| byte.is_ascii_hexdigit()) - { - return Err(format!( - "managed Python environment attestation {} does not match the trusted source artifact", - attestation_path.display() - )); - } - if !crate::configuration::verify_python_environment_attestation( - &attestation.source_artifact_sha256, - &attestation.environment_sha256, - &attestation.authentication, + nemo_relay_plugin_host_config::read_environment_attestation( + environment, + expected_source_artifact_sha256, ) - .map_err(|error| error.to_string())? - { - return Err(format!( - "managed Python environment attestation {} failed authentication", - attestation_path.display() - )); - } - Ok(attestation.environment_sha256) + .map_err(|error| error.to_string()) } +#[cfg(test)] pub(super) fn verify_environment_attestation( environment: &Path, expected_source_artifact_sha256: &str, ) -> Result { - let expected = read_environment_attestation(environment, expected_source_artifact_sha256)?; - let actual = environment_tree_digest(environment)?; - if actual != expected { - return Err(format!( - "managed Python environment {} changed after provisioning", - environment.display() - )); - } - Ok(actual) + nemo_relay_plugin_host_config::verify_environment_attestation( + environment, + expected_source_artifact_sha256, + ) + .map_err(|error| error.to_string()) } pub(super) fn write_environment_attestation( @@ -551,37 +426,13 @@ pub(super) fn remove_managed_environment( remove_directory_if_present(&configured, "delete") } -pub(super) fn environment_state( +pub(super) fn validate_environment_state( manifest: &DynamicPluginManifest, state_path: &Path, environment_ref: Option<&str>, -) -> DynamicPluginCheckState { - if !is_python_worker(manifest) { - return DynamicPluginCheckState::Unknown; - } - let Some(environment_ref) = environment_ref else { - return DynamicPluginCheckState::Invalid; - }; - let Ok(expected) = managed_environment_path(state_path, &manifest.plugin.id) else { - return DynamicPluginCheckState::Invalid; - }; - let Ok(configured) = absolute_path(Path::new(environment_ref)) else { - return DynamicPluginCheckState::Invalid; - }; - if configured != expected - || std::fs::symlink_metadata(&configured) - .map(|metadata| !metadata.file_type().is_dir()) - .unwrap_or(true) - || !environment_python_path(&configured).is_file() - || manifest - .integrity - .as_ref() - .and_then(|integrity| integrity.sha256.as_deref()) - .is_none_or(|digest| verify_environment_attestation(&configured, digest).is_err()) - { - return DynamicPluginCheckState::Invalid; - } - DynamicPluginCheckState::Valid +) -> Result { + nemo_relay_plugin_host_config::validate_environment_state(manifest, state_path, environment_ref) + .map_err(|error| error.to_string()) } pub(super) fn environment_python_path(environment: &Path) -> PathBuf { diff --git a/crates/cli/src/plugins/lifecycle/mod.rs b/crates/cli/src/plugins/lifecycle/mod.rs index 734754c74..809d0b201 100644 --- a/crates/cli/src/plugins/lifecycle/mod.rs +++ b/crates/cli/src/plugins/lifecycle/mod.rs @@ -50,9 +50,8 @@ mod trust; use self::environment::{ ENVIRONMENT_ATTESTATION_FILE, MANAGED_ENVIRONMENTS_DIR, ProcessPythonEnvironmentCommandRunner, - PythonEnvironmentCommandRunner, environment_state, provision_python_environment, - read_environment_attestation, remove_managed_environment, validate_python_entrypoint_artifact, - verify_environment_attestation, + PythonEnvironmentCommandRunner, provision_python_environment, read_environment_attestation, + remove_managed_environment, validate_environment_state, validate_python_entrypoint_artifact, }; use self::render::*; pub(crate) use self::render::{render_generic_plugin_json_error, render_plugin_error}; @@ -62,11 +61,13 @@ use self::responses::{ }; use self::state::{ RegistryScope, ScopedDynamicPluginRecord, ScopedRegistry, collect_records, find_record_by_id, - load_scoped_registries, scoped_paths_for_add, + load_scoped_registries, load_scoped_registries_for_update, scoped_paths_for_add, }; use self::target::PluginTarget; use self::trust::{EvaluatedDynamicPluginTrust, evaluate_dynamic_plugin_trust}; +pub(crate) use nemo_relay_plugin_host_config::DynamicPluginActivationSnapshot; + const VALIDATION_MESSAGE: &str = "validated by CLI"; #[cfg(test)] @@ -103,11 +104,28 @@ fn add_with_environment_runner( const COMMAND: &str = "plugins add"; let explicit_plugin_config = lifecycle_plugin_config_path(server); + if explicit_plugin_config.is_some() && scope_flags_selected(&command.scope) { + return Err(CliError::Config( + "--config cannot be combined with --user, --project, or --global for `plugins add`; the same applies to --plugin-config-path" + .into(), + )); + } + let (plugins_toml_path, state_path, scope) = scoped_paths_for_add( + target_scope(&command.scope)?, + explicit_plugin_config.as_ref(), + )?; let resolved = resolve_plugins_config_with_path( server.config.as_ref(), server.plugin_config_path.as_ref(), )?; - let mut scopes = load_and_hydrate_scopes(explicit_plugin_config.as_ref(), &resolved)?; + let scopes = load_scoped_registries_for_update( + explicit_plugin_config.as_ref(), + Some((scope, plugins_toml_path.clone(), state_path.clone())), + )?; + let (mut scopes, touched_scope_indices) = hydrate_scopes_with_updates(scopes, &resolved)?; + for scope_index in touched_scope_indices { + scopes[scope_index].save()?; + } let (manifest, manifest_ref) = load_manifest_for_action("add", &command.path)?; let plugin_id = manifest.plugin.id.trim().to_owned(); load_config_schema_for_manifest(&manifest, &manifest_ref)?; @@ -122,18 +140,18 @@ fn add_with_environment_runner( None => false, }; - if explicit_plugin_config.is_some() && scope_flags_selected(&command.scope) { - return Err(CliError::Config( - "--config cannot be combined with --user, --project, or --global for `plugins add`; the same applies to --plugin-config-path" - .into(), - )); - } - - let (plugins_toml_path, state_path, scope) = scoped_paths_for_add( - target_scope(&command.scope)?, - explicit_plugin_config.as_ref(), - )?; let scope_index = ensure_scope(&mut scopes, scope, plugins_toml_path.clone(), state_path); + scopes[scope_index].ensure_locked()?; + if scopes[scope_index] + .registry + .get(&plugin_id) + .is_some_and(|existing| !existing.is_tombstoned()) + { + return Err(CliError::Config(format!( + "dynamic plugin '{}' is already registered in the {} lifecycle scope", + plugin_id, scope + ))); + } let policy = evaluate_dynamic_plugin_host_policy(&resolved.dynamic_plugin_policy, &manifest); let trust = evaluate_dynamic_plugin_trust(&manifest, &manifest_ref, &policy); if !policy.policy_satisfied { @@ -214,6 +232,21 @@ fn add_with_environment_runner( ); return Err(error); } + let declaration_source = pin_declaration_source(&plugins_toml_path); + if let Err(error) = declaration_source.and_then(|declaration_source| { + scopes[scope_index] + .registry + .set_declaration_source(&plugin_id, declaration_source) + .map_err(|error| CliError::Config(error.to_string())) + }) { + let _ = restore_plugins_toml(&plugins_toml_path, original_plugins_toml.as_deref()); + cleanup_provisioned_environment( + &scopes[scope_index].state_path, + &plugin_id, + environment_ref.as_deref(), + ); + return Err(error); + } if let Err(error) = scopes[scope_index].save() { let _ = restore_plugins_toml(&plugins_toml_path, original_plugins_toml.as_deref()); cleanup_provisioned_environment( @@ -326,7 +359,8 @@ pub(crate) fn validate( server.plugin_config_path.as_ref(), )?; let host_config_by_id = host_config_by_id(&resolved); - let mut scopes = load_and_hydrate_scopes(explicit_plugin_config.as_ref(), &resolved)?; + let mut scopes = + load_and_hydrate_scopes_for_update(explicit_plugin_config.as_ref(), &resolved)?; let entry = find_registered_entry(&scopes, "plugins validate", &plugin_id)?; let manifest_ref = manifest_ref_from_record(&entry.record)?; let (manifest, manifest_ref) = load_manifest_for_action("validate", &manifest_ref)?; @@ -480,13 +514,14 @@ pub(crate) fn remove( server: &GatewayOverrides, ) -> Result<(), CliError> { let explicit_plugin_config = lifecycle_plugin_config_path(server); - let mut scopes = load_scoped_registries(explicit_plugin_config.as_ref())?; + let mut scopes = load_scoped_registries_for_update(explicit_plugin_config.as_ref(), None)?; if find_record_by_id(&scopes, &command.id)?.is_none() { + drop(scopes); let resolved = resolve_plugins_config_with_path( server.config.as_ref(), server.plugin_config_path.as_ref(), )?; - scopes = load_and_hydrate_scopes(explicit_plugin_config.as_ref(), &resolved)?; + scopes = load_and_hydrate_scopes_for_update(explicit_plugin_config.as_ref(), &resolved)?; } let entry = find_registered_entry(&scopes, "plugins remove", &command.id)?; let original_plugins_toml = std::fs::read(&entry.plugins_toml_path).ok(); @@ -531,401 +566,6 @@ pub(crate) struct ActiveDynamicPluginComponent { pub(crate) activation_snapshot: Option>, } -#[derive(Debug, PartialEq, Eq)] -pub(crate) struct DynamicPluginActivationSnapshot { - root: PathBuf, - original_manifest_ref: String, - identity_manifest: PathBuf, - activation_manifest: PathBuf, - activation_environment_ref: Option, - identity_files: HashMap, - closure_digest: String, - verification_digest: String, -} - -impl DynamicPluginActivationSnapshot { - fn create( - manifest_ref: &str, - expected_plugin_id: &str, - expected_kind: DynamicPluginKind, - environment_ref: Option<&str>, - host_policy: &crate::plugins::policy::DynamicPluginHostPolicy, - ) -> Result, CliError> { - let (mut manifest, original_manifest_ref, manifest_bytes) = - load_bounded_dynamic_plugin_manifest_bytes(manifest_ref)?; - if manifest.plugin.id.trim() != expected_plugin_id || manifest.plugin.kind != expected_kind - { - return Err(CliError::Config(format!( - "dynamic plugin manifest identity changed before activation for '{expected_plugin_id}'" - ))); - } - let policy = evaluate_dynamic_plugin_host_policy(host_policy, &manifest); - validate_python_entrypoint_artifact(&manifest, &original_manifest_ref) - .map_err(CliError::Config)?; - - let root = std::env::temp_dir().join(format!( - "nemo-relay-plugin-snapshot-{}", - uuid::Uuid::now_v7().simple() - )); - fs::create_dir(&root).map_err(|error| { - CliError::Config(format!( - "failed to create dynamic plugin activation snapshot {}: {error}", - root.display() - )) - })?; - let mut root_guard = SnapshotRootGuard(Some(root.clone())); - #[cfg(unix)] - fs::set_permissions(&root, { - use std::os::unix::fs::PermissionsExt; - fs::Permissions::from_mode(0o700) - }) - .map_err(|error| { - CliError::Config(format!( - "failed to protect dynamic plugin activation snapshot {}: {error}", - root.display() - )) - })?; - - let identity_manifest = root.join("identity-manifest.toml"); - fs::write(&identity_manifest, &manifest_bytes).map_err(|error| { - CliError::Config(format!( - "failed to write dynamic plugin activation snapshot {}: {error}", - identity_manifest.display() - )) - })?; - let original_manifest_path = PathBuf::from(&original_manifest_ref); - let manifest_directory = original_manifest_path - .parent() - .ok_or_else(|| { - CliError::Config(format!( - "dynamic plugin manifest {} has no parent directory", - original_manifest_path.display() - )) - })? - .to_path_buf(); - let runtime_root = root.join("runtime"); - let mut budget = SnapshotBudget::default(); - let mut copied_files = HashMap::new(); - copy_snapshot_directory( - &manifest_directory, - &runtime_root, - &mut copied_files, - &mut budget, - false, - &mut Vec::new(), - )?; - let declared_artifact = manifest - .source - .as_ref() - .and_then(|source| source.artifact.as_deref()) - .map(|artifact| fs::canonicalize(resolve_manifest_relative_path(&original_manifest_path, artifact))) - .transpose() - .map_err(|error| { - CliError::Config(format!( - "failed to normalize dynamic plugin artifact for '{expected_plugin_id}': {error}" - )) - })?; - let mut identity_files = HashMap::new(); - - match &mut manifest.load { - DynamicPluginManifestLoad::RustDynamic(load) => { - if let Some(library) = load.library.as_deref() { - let (logical, _, copied) = copy_snapshot_file( - &root, - &original_manifest_path, - library, - "library", - &mut copied_files, - &mut budget, - )?; - identity_files - .entry(logical) - .or_insert_with(|| copied.clone()); - load.library = Some(copied.to_string_lossy().into_owned()); - } - } - DynamicPluginManifestLoad::Worker(load) - if matches!( - load.runtime, - Some(WorkerRuntime::Rust | WorkerRuntime::Command) - ) => - { - if let Some(entrypoint) = load.entrypoint.as_deref() { - let (logical, canonical, copied) = copy_snapshot_file( - &root, - &original_manifest_path, - entrypoint, - "entrypoint", - &mut copied_files, - &mut budget, - )?; - if declared_artifact.as_ref() != Some(&canonical) { - return Err(CliError::Config(format!( - "command worker dynamic plugin '{expected_plugin_id}' must declare its load.entrypoint as the integrity-checked source.artifact" - ))); - } - identity_files - .entry(logical) - .or_insert_with(|| copied.clone()); - load.entrypoint = Some(copied.to_string_lossy().into_owned()); - } - } - DynamicPluginManifestLoad::Worker(_) => {} - } - - if let Some(source) = manifest.source.as_mut() - && let Some(artifact) = source.artifact.as_deref() - { - let (logical, _, copied) = copy_snapshot_file( - &root, - &original_manifest_path, - artifact, - "artifact", - &mut copied_files, - &mut budget, - )?; - identity_files.insert(logical, copied.clone()); - source.artifact = Some(copied.to_string_lossy().into_owned()); - } - if let Some(integrity) = manifest.integrity.as_mut() - && let Some(signature) = integrity.signature.as_deref() - { - let (logical, _, copied) = copy_snapshot_file( - &root, - &original_manifest_path, - signature, - "signature", - &mut copied_files, - &mut budget, - )?; - identity_files.insert(logical, copied.clone()); - integrity.signature = Some(copied.to_string_lossy().into_owned()); - } - - let activation_environment_ref = snapshot_python_environment( - &manifest, - environment_ref, - expected_plugin_id, - &root, - &mut copied_files, - &mut budget, - )?; - - let activation_manifest = runtime_root.join("relay-plugin.toml"); - let rendered = toml::to_string(&manifest).map_err(|error| { - CliError::Config(format!( - "failed to encode dynamic plugin activation snapshot for '{expected_plugin_id}': {error}" - )) - })?; - if rendered.len() as u64 > MAX_BOOTSTRAP_IDENTITY_FILE_BYTES { - return Err(CliError::Config(format!( - "dynamic plugin activation manifest for '{expected_plugin_id}' exceeds the {MAX_BOOTSTRAP_IDENTITY_FILE_BYTES}-byte activation snapshot budget" - ))); - } - fs::write(&activation_manifest, rendered).map_err(|error| { - CliError::Config(format!( - "failed to write dynamic plugin activation manifest {}: {error}", - activation_manifest.display() - )) - })?; - - let trust = evaluate_dynamic_plugin_trust( - &manifest, - activation_manifest.to_string_lossy().as_ref(), - &policy, - ); - if !policy.policy_satisfied { - return Err(CliError::Config(format!( - "dynamic plugin '{expected_plugin_id}' activation snapshot violates host policy" - ))); - } - if let Some(failure) = trust.failure() { - return Err(CliError::Config( - failure.display(expected_plugin_id).to_string(), - )); - } - - let closure_digest = snapshot_tree_digest(&root, true)?; - let verification_digest = snapshot_tree_digest(&root, false)?; - #[cfg(unix)] - protect_snapshot_tree(&root)?; - #[cfg(windows)] - protect_snapshot_tree(&root)?; - root_guard.0 = None; - Ok(Arc::new(Self { - root, - original_manifest_ref, - identity_manifest, - activation_manifest, - activation_environment_ref, - identity_files, - closure_digest, - verification_digest, - })) - } - - pub(crate) fn activation_manifest_ref(&self) -> String { - self.activation_manifest.to_string_lossy().into_owned() - } - - pub(crate) fn activation_environment_ref(&self) -> Option<&str> { - self.activation_environment_ref.as_deref() - } - - pub(crate) fn closure_digest(&self) -> &str { - &self.closure_digest - } - - pub(crate) fn verify_current(&self) -> Result<(), CliError> { - let actual = snapshot_tree_digest(&self.root, false)?; - if actual == self.verification_digest { - Ok(()) - } else { - Err(CliError::Config(format!( - "dynamic plugin activation snapshot {} changed before code load", - self.root.display() - ))) - } - } - - pub(crate) fn original_manifest_ref(&self) -> &str { - &self.original_manifest_ref - } - - pub(crate) fn identity_manifest(&self) -> &Path { - &self.identity_manifest - } - - pub(crate) fn identity_file(&self, logical_path: &Path) -> Option<&Path> { - self.identity_files.get(logical_path).map(PathBuf::as_path) - } -} - -fn snapshot_python_environment( - manifest: &DynamicPluginManifest, - environment_ref: Option<&str>, - expected_plugin_id: &str, - root: &Path, - copied_files: &mut HashMap, - budget: &mut SnapshotBudget, -) -> Result, CliError> { - if !matches!( - &manifest.load, - DynamicPluginManifestLoad::Worker(load) if load.runtime == Some(WorkerRuntime::Python) - ) { - return Ok(None); - } - let environment = environment_ref.ok_or_else(|| { - CliError::Config(format!( - "Python worker dynamic plugin '{expected_plugin_id}' has no managed environment" - )) - })?; - let source_artifact_sha256 = trusted_source_artifact_sha256(manifest)?; - let environment = PathBuf::from(environment); - verify_environment_attestation(&environment, source_artifact_sha256) - .map_err(CliError::Config)?; - let environment_name = environment.file_name().ok_or_else(|| { - CliError::Config(format!( - "managed Python environment {} has no lifecycle environment name", - environment.display() - )) - })?; - let copied_environment = root.join(MANAGED_ENVIRONMENTS_DIR).join(environment_name); - copy_snapshot_directory( - &environment, - &copied_environment, - copied_files, - budget, - true, - &mut Vec::new(), - )?; - verify_environment_attestation(&copied_environment, source_artifact_sha256) - .map_err(CliError::Config)?; - Ok(Some(copied_environment.to_string_lossy().into_owned())) -} - -struct SnapshotRootGuard(Option); - -impl Drop for SnapshotRootGuard { - fn drop(&mut self) { - if let Some(root) = self.0.take() { - make_snapshot_removable(&root); - let _ = fs::remove_dir_all(root); - } - } -} - -impl Drop for DynamicPluginActivationSnapshot { - fn drop(&mut self) { - make_snapshot_removable(&self.root); - let _ = fs::remove_dir_all(&self.root); - } -} - -fn copy_snapshot_file( - root: &Path, - manifest_path: &Path, - reference: &str, - label: &str, - copied_files: &mut HashMap, - budget: &mut SnapshotBudget, -) -> Result<(PathBuf, PathBuf, PathBuf), CliError> { - let logical = resolve_manifest_relative_path(manifest_path, reference); - let canonical = fs::canonicalize(&logical).map_err(|error| { - CliError::Config(format!( - "failed to normalize dynamic plugin {label} {}: {error}", - logical.display() - )) - })?; - if let Some(copied) = copied_files.get(&canonical) - && !matches!(label, "library" | "entrypoint") - { - return Ok((logical, canonical, copied.clone())); - } - if matches!(label, "library" | "entrypoint") { - let manifest_directory = manifest_path - .parent() - .and_then(|parent| fs::canonicalize(parent).ok()); - if manifest_directory - .as_ref() - .is_some_and(|directory| canonical.starts_with(directory)) - && let Some(copied) = copied_files.get(&canonical) - { - // The manifest directory is copied as a complete closure before declared paths are - // rewritten, so in-tree load targets already retain adjacent resources. - return Ok((logical, canonical, copied.clone())); - } - } - let external = root.join(format!("external-{label}")); - if matches!(label, "library" | "entrypoint") { - let parent = canonical.parent().ok_or_else(|| { - CliError::Config(format!( - "dynamic plugin {label} {} has no parent directory", - canonical.display() - )) - })?; - copy_snapshot_directory( - parent, - &external, - copied_files, - budget, - false, - &mut Vec::new(), - )?; - } else { - fs::create_dir_all(&external).map_err(|error| CliError::Config(error.to_string()))?; - let destination = external.join(canonical.file_name().unwrap_or_default()); - copy_snapshot_regular_file(&canonical, &destination, copied_files, budget, label)?; - } - let copied = copied_files.get(&canonical).cloned().ok_or_else(|| { - CliError::Config(format!( - "dynamic plugin {label} {} was not included in its activation snapshot", - canonical.display() - )) - })?; - Ok((logical, canonical, copied)) -} - const MAX_SNAPSHOT_FILES: usize = 100_000; const MAX_SNAPSHOT_DEPTH: usize = 128; @@ -962,233 +602,6 @@ impl SnapshotBudget { } Ok(()) } - - fn record_directory(&mut self, path: &Path) -> Result<(), CliError> { - self.record_entries(path, 1) - } -} - -fn copy_snapshot_directory( - source: &Path, - destination: &Path, - copied_files: &mut HashMap, - budget: &mut SnapshotBudget, - skip_python_cache: bool, - ancestors: &mut Vec, -) -> Result<(), CliError> { - budget.record_directory(source)?; - copy_snapshot_directory_contents( - source, - destination, - copied_files, - budget, - skip_python_cache, - ancestors, - ) -} - -fn copy_snapshot_directory_contents( - source: &Path, - destination: &Path, - copied_files: &mut HashMap, - budget: &mut SnapshotBudget, - skip_python_cache: bool, - ancestors: &mut Vec, -) -> Result<(), CliError> { - if ancestors.len() >= MAX_SNAPSHOT_DEPTH { - return Err(CliError::Config(format!( - "dynamic plugin runtime closure exceeds the {MAX_SNAPSHOT_DEPTH}-directory traversal depth at {}", - source.display() - ))); - } - let canonical = fs::canonicalize(source).map_err(|error| { - CliError::Config(format!( - "failed to normalize dynamic plugin runtime directory {}: {error}", - source.display() - )) - })?; - if ancestors.contains(&canonical) { - return Err(CliError::Config(format!( - "dynamic plugin runtime closure contains a directory symlink cycle at {}", - source.display() - ))); - } - ancestors.push(canonical.clone()); - fs::create_dir_all(destination).map_err(|error| { - CliError::Config(format!( - "failed to create dynamic plugin snapshot directory {}: {error}", - destination.display() - )) - })?; - let mut entries = bounded_runtime_directory_entries( - &canonical, - MAX_SNAPSHOT_FILES.saturating_sub(budget.entries), - )?; - budget.record_entries(source, entries.len())?; - entries.sort_by_key(fs::DirEntry::file_name); - for entry in entries { - copy_snapshot_entry( - entry, - destination, - copied_files, - budget, - skip_python_cache, - ancestors, - )?; - } - ancestors.pop(); - Ok(()) -} - -fn copy_snapshot_entry( - entry: fs::DirEntry, - destination: &Path, - copied_files: &mut HashMap, - budget: &mut SnapshotBudget, - skip_python_cache: bool, - ancestors: &mut Vec, -) -> Result<(), CliError> { - let source_path = entry.path(); - if skip_python_cache - && (entry.file_name() == "__pycache__" - || source_path.extension().and_then(|value| value.to_str()) == Some("pyc")) - { - return Ok(()); - } - let destination_path = destination.join(entry.file_name()); - let metadata = - fs::symlink_metadata(&source_path).map_err(|error| CliError::Config(error.to_string()))?; - let resolved = resolve_snapshot_entry(&source_path, &metadata)?; - let resolved_metadata = - fs::metadata(&resolved).map_err(|error| CliError::Config(error.to_string()))?; - if resolved_metadata.is_dir() { - return copy_snapshot_directory_contents( - &resolved, - &destination_path, - copied_files, - budget, - skip_python_cache, - ancestors, - ); - } - if !resolved_metadata.is_file() { - return Err(CliError::Config(format!( - "dynamic plugin runtime entry {} must resolve to a regular file or directory", - source_path.display() - ))); - } - if preserve_python_launcher( - &source_path, - &destination_path, - &resolved, - &metadata, - copied_files, - )? { - return Ok(()); - } - copy_snapshot_regular_file( - &resolved, - &destination_path, - copied_files, - budget, - "runtime file", - ) -} - -fn resolve_snapshot_entry(path: &Path, metadata: &fs::Metadata) -> Result { - if metadata.file_type().is_symlink() { - fs::canonicalize(path).map_err(|error| { - CliError::Config(format!( - "failed to resolve dynamic plugin runtime symlink {}: {error}", - path.display() - )) - }) - } else { - Ok(path.to_path_buf()) - } -} - -#[cfg(unix)] -fn preserve_python_launcher( - source: &Path, - destination: &Path, - resolved: &Path, - metadata: &fs::Metadata, - copied_files: &mut HashMap, -) -> Result { - if !metadata.file_type().is_symlink() || !is_python_venv_launcher(source) { - return Ok(false); - } - let target = fs::read_link(source).map_err(|error| { - CliError::Config(format!( - "failed to read Python venv launcher symlink {}: {error}", - source.display() - )) - })?; - if let Some(parent) = destination.parent() { - fs::create_dir_all(parent).map_err(|error| CliError::Config(error.to_string()))?; - } - std::os::unix::fs::symlink(&target, destination).map_err(|error| { - CliError::Config(format!( - "failed to preserve Python venv launcher symlink {}: {error}", - destination.display() - )) - })?; - copied_files.insert(resolved.to_path_buf(), destination.to_path_buf()); - Ok(true) -} - -#[cfg(not(unix))] -fn preserve_python_launcher( - _source: &Path, - _destination: &Path, - _resolved: &Path, - _metadata: &fs::Metadata, - _copied_files: &mut HashMap, -) -> Result { - Ok(false) -} - -#[cfg(unix)] -fn is_python_venv_launcher(path: &Path) -> bool { - let Some(parent) = path.parent() else { - return false; - }; - parent.file_name() == Some(std::ffi::OsStr::new("bin")) - && path - .file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| name == "python" || name.starts_with("python3")) -} - -fn copy_snapshot_regular_file( - source: &Path, - destination: &Path, - copied_files: &mut HashMap, - budget: &mut SnapshotBudget, - description: &str, -) -> Result<(), CliError> { - let bytes = read_bounded_regular_file(source, &format!("dynamic plugin {description}")) - .map_err(CliError::Config)?; - budget.record_bytes(source, bytes.len())?; - fs::write(destination, bytes).map_err(|error| { - CliError::Config(format!( - "failed to write dynamic plugin snapshot file {}: {error}", - destination.display() - )) - })?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mode = fs::metadata(source) - .map_err(|error| CliError::Config(error.to_string()))? - .permissions() - .mode(); - fs::set_permissions(destination, fs::Permissions::from_mode(mode)) - .map_err(|error| CliError::Config(error.to_string()))?; - } - copied_files.insert(source.to_path_buf(), destination.to_path_buf()); - Ok(()) } fn resolve_manifest_relative_path(manifest_path: &Path, reference: &str) -> PathBuf { @@ -1203,102 +616,6 @@ fn resolve_manifest_relative_path(manifest_path: &Path, reference: &str) -> Path } } -#[cfg(unix)] -fn protect_snapshot_tree(root: &Path) -> Result<(), CliError> { - use std::os::unix::fs::PermissionsExt; - for entry in fs::read_dir(root).map_err(|error| CliError::Config(error.to_string()))? { - let path = entry - .map_err(|error| CliError::Config(error.to_string()))? - .path(); - let metadata = - fs::symlink_metadata(&path).map_err(|error| CliError::Config(error.to_string()))?; - if metadata.is_dir() { - protect_snapshot_tree(&path)?; - continue; - } - if metadata.file_type().is_symlink() { - continue; - } - let mode = metadata.permissions().mode() & !0o222; - fs::set_permissions(&path, fs::Permissions::from_mode(mode)) - .map_err(|error| CliError::Config(error.to_string()))?; - } - fs::set_permissions(root, fs::Permissions::from_mode(0o500)) - .map_err(|error| CliError::Config(error.to_string())) -} - -#[cfg(windows)] -fn protect_snapshot_tree(root: &Path) -> Result<(), CliError> { - for entry in fs::read_dir(root).map_err(|error| CliError::Config(error.to_string()))? { - let path = entry - .map_err(|error| CliError::Config(error.to_string()))? - .path(); - let metadata = - fs::symlink_metadata(&path).map_err(|error| CliError::Config(error.to_string()))?; - if metadata.is_dir() { - protect_snapshot_tree(&path)?; - } else if !metadata.file_type().is_symlink() { - let mut permissions = metadata.permissions(); - permissions.set_readonly(true); - fs::set_permissions(&path, permissions) - .map_err(|error| CliError::Config(error.to_string()))?; - } - } - Ok(()) -} - -fn snapshot_tree_digest(root: &Path, stable_identity: bool) -> Result { - let mut files = Vec::new(); - let mut entries = 0_usize; - collect_snapshot_files(root, root, &mut files, None, &mut entries)?; - files.sort(); - let mut digest = Sha256::new(); - let mut budget = SnapshotBudget::default(); - for relative in files { - if stable_identity { - let activation_manifest = Path::new("runtime").join("relay-plugin.toml"); - let is_python_environment_content = relative.starts_with(MANAGED_ENVIRONMENTS_DIR) - && relative.file_name() != Some(std::ffi::OsStr::new(ENVIRONMENT_ATTESTATION_FILE)); - if relative == activation_manifest || is_python_environment_content { - continue; - } - } - let path = root.join(&relative); - let metadata = fs::symlink_metadata(&path).map_err(|error| { - CliError::Config(format!( - "failed to inspect dynamic plugin activation snapshot entry {}: {error}", - path.display() - )) - })?; - if metadata.file_type().is_symlink() { - let target = fs::read_link(&path).map_err(|error| { - CliError::Config(format!( - "failed to read dynamic plugin activation snapshot symlink {}: {error}", - path.display() - )) - })?; - let target = target.as_os_str().as_encoded_bytes(); - budget.record(&path, target.len())?; - update_snapshot_entry_digest( - &mut digest, - &relative, - SnapshotEntryKind::Symlink, - target, - ); - } else { - let bytes = read_bounded_regular_file(&path, "dynamic plugin activation snapshot file") - .map_err(CliError::Config)?; - budget.record(&path, bytes.len())?; - update_snapshot_entry_digest(&mut digest, &relative, SnapshotEntryKind::File, &bytes); - } - } - Ok(digest - .finalize() - .iter() - .map(|byte| format!("{byte:02x}")) - .collect()) -} - pub(crate) fn dynamic_plugin_runtime_closure_digest( manifest_ref: &str, environment_ref: Option<&str>, @@ -1519,7 +836,6 @@ impl RuntimeClosureSources { #[derive(Clone, Copy)] enum SnapshotEntryKind { File = 0, - Symlink = 1, } fn update_snapshot_entry_digest( @@ -1699,81 +1015,6 @@ fn bounded_runtime_directory_entries( Ok(entries) } -fn collect_snapshot_files( - root: &Path, - directory: &Path, - files: &mut Vec, - logical_depth: Option, - entries: &mut usize, -) -> Result<(), CliError> { - if let Some(depth) = logical_depth - && depth >= MAX_SNAPSHOT_DEPTH - { - return Err(CliError::Config(format!( - "dynamic plugin activation snapshot exceeds the {MAX_SNAPSHOT_DEPTH}-directory traversal depth at {}", - directory.display() - ))); - } - let resets_child_depth = - directory == root || directory == root.join(environment::MANAGED_ENVIRONMENTS_DIR); - for entry in fs::read_dir(directory).map_err(|error| CliError::Config(error.to_string()))? { - *entries = entries.saturating_add(1); - if *entries > MAX_SNAPSHOT_FILES { - return Err(CliError::Config(format!( - "dynamic plugin activation snapshot exceeds the {MAX_SNAPSHOT_FILES}-entry verification budget at {}", - directory.display() - ))); - } - let path = entry - .map_err(|error| CliError::Config(error.to_string()))? - .path(); - let metadata = - fs::symlink_metadata(&path).map_err(|error| CliError::Config(error.to_string()))?; - if metadata.is_dir() { - let child_depth = if resets_child_depth { - 0 - } else { - logical_depth.unwrap_or(0).saturating_add(1) - }; - collect_snapshot_files(root, &path, files, Some(child_depth), entries)?; - } else { - files.push( - path.strip_prefix(root) - .map_err(|error| CliError::Config(error.to_string()))? - .to_path_buf(), - ); - } - } - Ok(()) -} - -fn make_snapshot_removable(root: &Path) { - let Ok(entries) = fs::read_dir(root) else { - return; - }; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let _ = fs::set_permissions(root, fs::Permissions::from_mode(0o700)); - } - for entry in entries.flatten() { - let path = entry.path(); - let Ok(metadata) = fs::symlink_metadata(&path) else { - continue; - }; - if metadata.is_dir() { - make_snapshot_removable(&path); - } else { - #[cfg(windows)] - if !metadata.file_type().is_symlink() { - let mut permissions = metadata.permissions(); - permissions.set_readonly(false); - let _ = fs::set_permissions(&path, permissions); - } - } - } -} - pub(crate) fn active_dynamic_plugin_components( explicit_plugin_config: Option<&PathBuf>, resolved: &ResolvedConfig, @@ -1794,10 +1035,81 @@ fn active_dynamic_plugin_components_inner( resolved: &ResolvedConfig, create_activation_snapshots: bool, ) -> Result, CliError> { + if create_activation_snapshots { + return active_dynamic_plugin_components_from_shared_host_config(resolved); + } let scopes = load_and_hydrate_scopes(explicit_plugin_config, resolved)?; active_dynamic_plugin_components_from_scopes(&scopes, resolved, create_activation_snapshots) } +fn active_dynamic_plugin_components_from_shared_host_config( + resolved: &ResolvedConfig, +) -> Result, CliError> { + let runtime_value = resolved.gateway.plugin_config.clone(); + let config = runtime_value + .clone() + .map(serde_json::from_value) + .transpose() + .map_err(|error| CliError::Config(format!("invalid resolved plugin config: {error}")))? + .unwrap_or_default(); + let selected_sources = resolved.plugin_selected_sources.clone(); + let contributing_sources: Vec = resolved + .dynamic_plugins + .iter() + .map(|plugin| plugin.source.clone()) + .collect::>() + .into_iter() + .collect(); + let shared = nemo_relay_plugin_host_config::ResolvedPluginFileConfiguration { + config, + runtime_value, + dynamic_plugins: resolved + .dynamic_plugins + .iter() + .map( + |plugin| nemo_relay_plugin_host_config::ResolvedDynamicPluginConfig { + plugin_id: plugin.plugin_id.clone(), + manifest_ref: plugin.manifest_ref.clone(), + config: plugin.config.clone(), + has_explicit_config: plugin.has_explicit_config, + source: plugin.source.clone(), + }, + ) + .collect(), + dynamic_plugin_policy: resolved.dynamic_plugin_policy.clone(), + diagnostics: Vec::new(), + contributing_selected_sources: contributing_sources.clone(), + contributing_sources, + selected_sources, + had_input: true, + }; + let reconciled = nemo_relay_plugin_host_config::reconcile_plugin_lifecycle(&shared) + .map_err(|error| CliError::Config(error.to_string()))?; + reconciled + .enabled_plugins + .into_iter() + .map(|plugin| { + let snapshot = DynamicPluginActivationSnapshot::create( + &plugin.manifest_ref, + &plugin.plugin_id, + plugin.kind, + plugin.environment_ref.as_deref(), + &shared.dynamic_plugin_policy, + ) + .map_err(|error| CliError::Config(error.to_string()))?; + Ok(ActiveDynamicPluginComponent { + plugin_id: plugin.plugin_id, + kind: plugin.kind, + lifecycle_generation: plugin.lifecycle_generation, + manifest_ref: Some(plugin.manifest_ref), + environment_ref: plugin.environment_ref, + config: plugin.config, + activation_snapshot: Some(snapshot), + }) + }) + .collect() +} + fn active_dynamic_plugin_components_from_scopes( scopes: &[ScopedRegistry], resolved: &ResolvedConfig, @@ -1809,7 +1121,7 @@ fn active_dynamic_plugin_components_from_scopes( for resolved_plugin in &resolved.dynamic_plugins { let Some(record) = scopes .iter() - .find(|scope| scope.plugins_toml_path == resolved_plugin.source) + .find(|scope| scope.source_for_path(&resolved_plugin.source).is_some()) .and_then(|scope| scope.registry.get(&resolved_plugin.plugin_id)) else { return Err(CliError::Config(format!( @@ -1842,7 +1154,8 @@ fn active_dynamic_plugin_components_from_scopes( &resolved.dynamic_plugin_policy, ) }) - .transpose()? + .transpose() + .map_err(|error| CliError::Config(error.to_string()))? } else { None }; @@ -1876,7 +1189,8 @@ fn mutate_enabled_state( server.config.as_ref(), server.plugin_config_path.as_ref(), )?; - let mut scopes = load_and_hydrate_scopes(explicit_plugin_config.as_ref(), &resolved)?; + let mut scopes = + load_and_hydrate_scopes_for_update(explicit_plugin_config.as_ref(), &resolved)?; let entry = find_registered_entry(&scopes, command, &plugin_id)?; if entry.record.is_tombstoned() { return Err(plugin_refused( @@ -1941,7 +1255,7 @@ fn mutate_enabled_state( } scopes } else { - load_scoped_registries(explicit_plugin_config.as_ref())? + load_scoped_registries_for_update(explicit_plugin_config.as_ref(), None)? }; let entry = find_registered_entry(&scopes, command, &plugin_id)?; if entry.record.is_tombstoned() { @@ -1979,6 +1293,17 @@ fn mutate_enabled_state( fn load_and_hydrate_scopes( explicit_plugin_config: Option<&PathBuf>, resolved: &ResolvedConfig, +) -> Result, CliError> { + let mut scopes = load_and_hydrate_scopes_for_update(explicit_plugin_config, resolved)?; + for scope in &mut scopes { + scope.state_lock.take(); + } + Ok(scopes) +} + +fn load_and_hydrate_scopes_for_update( + explicit_plugin_config: Option<&PathBuf>, + resolved: &ResolvedConfig, ) -> Result, CliError> { let (scopes, touched_scope_indices) = load_and_hydrate_scopes_with_updates(explicit_plugin_config, resolved)?; @@ -1992,12 +1317,19 @@ fn load_and_hydrate_scopes_with_updates( explicit_plugin_config: Option<&PathBuf>, resolved: &ResolvedConfig, ) -> Result<(Vec, Vec), CliError> { - let mut scopes = load_scoped_registries(explicit_plugin_config)?; + let scopes = load_scoped_registries_for_update(explicit_plugin_config, None)?; + hydrate_scopes_with_updates(scopes, resolved) +} + +fn hydrate_scopes_with_updates( + mut scopes: Vec, + resolved: &ResolvedConfig, +) -> Result<(Vec, Vec), CliError> { let mut touched_scope_indices = BTreeSet::new(); for plugin in &resolved.dynamic_plugins { let scope_index = scopes .iter() - .position(|scope| scope.plugins_toml_path == plugin.source) + .position(|scope| scope.source_for_path(&plugin.source).is_some()) .ok_or_else(|| { CliError::Config(format!( "dynamic plugin '{}' resolved from {} but no matching lifecycle scope exists", @@ -2006,33 +1338,93 @@ fn load_and_hydrate_scopes_with_updates( )) })?; touched_scope_indices.insert(scope_index); + let declaration_owner = plugin.source.display().to_string(); + let declaring_scope = scopes[scope_index] + .source_for_path(&plugin.source) + .expect("declaring source was located immediately before use") + .scope; + let foreign_live_scopes = scopes + .iter() + .enumerate() + .filter_map(|(index, scope)| { + if index == scope_index { + return None; + } + scope + .registry + .get(&plugin.plugin_id) + .filter(|record| !record.is_tombstoned()) + .map(|_| scope.source_for_record(&plugin.plugin_id).scope.to_string()) + }) + .collect::>(); + if !foreign_live_scopes.is_empty() { + let mut lifecycle_scopes = vec![declaring_scope.to_string()]; + lifecycle_scopes.extend(foreign_live_scopes); + return Err(CliError::Config(format!( + "dynamic plugin '{}' is configured in multiple lifecycle scopes; inspect {}", + plugin.plugin_id, + lifecycle_scopes.join(", ") + ))); + } let (manifest, manifest_ref) = load_manifest_for_action("hydrate", &plugin.manifest_ref)?; let policy = evaluate_dynamic_plugin_host_policy(&resolved.dynamic_plugin_policy, &manifest); let trust = evaluate_dynamic_plugin_trust(&manifest, &manifest_ref, &policy); - if find_record_by_id(&scopes, &plugin.plugin_id)?.is_some() { - update_registry_validation_status( - &mut scopes[scope_index], - &plugin.plugin_id, - &manifest, - &policy, - &trust, - )?; + let existing = scopes[scope_index].registry.get(&plugin.plugin_id).cloned(); + let existing_owner = scopes[scope_index] + .registry + .declaration_source(&plugin.plugin_id); + if existing + .as_ref() + .is_some_and(|record| !record.is_tombstoned()) + && existing_owner.is_some_and(|owner| owner != declaration_owner) + { + return Err(CliError::Config(format!( + "dynamic plugin '{}' is already live under declaration source {}; remove it before declaring it in {}", + plugin.plugin_id, + existing_owner.expect("checked as present immediately before use"), + declaration_owner + ))); + } + let preserves_existing_lifecycle = + existing.is_some() && existing_owner.is_none_or(|owner| owner == declaration_owner); + let claims_legacy_owner = preserves_existing_lifecycle && existing_owner.is_none(); + let environment_ref = existing + .as_ref() + .filter(|_| preserves_existing_lifecycle) + .and_then(|record| record.source.environment_ref.clone()); + let state_path = scopes[scope_index].state_path.clone(); + let record = validated_record_from_manifest( + manifest, + manifest_ref, + environment_ref, + &state_path, + &policy, + &trust, + )?; + if preserves_existing_lifecycle { + scopes[scope_index] + .registry + .refresh_manifest_record(&plugin.plugin_id, record) + .map_err(|error| CliError::Config(error.to_string()))?; } else { - let state_path = scopes[scope_index].state_path.clone(); - let record = validated_record_from_manifest( - manifest, - manifest_ref, - None, - &state_path, - &policy, - &trust, - )?; + if existing.is_some() { + scopes[scope_index] + .registry + .remove(&plugin.plugin_id) + .map_err(|error| CliError::Config(error.to_string()))?; + } scopes[scope_index] .registry .add(record) .map_err(|error| CliError::Config(error.to_string()))?; } + if claims_legacy_owner || !preserves_existing_lifecycle { + scopes[scope_index] + .registry + .set_declaration_source(&plugin.plugin_id, declaration_owner) + .map_err(|error| CliError::Config(error.to_string()))?; + } } Ok((scopes, touched_scope_indices.into_iter().collect())) } @@ -2045,7 +1437,14 @@ fn validated_record_from_manifest( policy: &EvaluatedDynamicPluginHostPolicy, trust: &EvaluatedDynamicPluginTrust, ) -> Result { - let environment = environment_state(&manifest, state_path, environment_ref.as_deref()); + let (environment, environment_error) = + match validate_environment_state(&manifest, state_path, environment_ref.as_deref()) { + Ok(environment) => (environment, None), + Err(error) => ( + DynamicPluginCheckState::Invalid, + environment_ref.as_ref().map(|_| error), + ), + }; let mut record = manifest .into_record(Some(manifest_ref)) .map_err(|error| CliError::Config(error.to_string()))?; @@ -2070,6 +1469,7 @@ fn validated_record_from_manifest( &record.metadata.id, environment, record.source.environment_ref.as_deref(), + environment_error, ) }); Ok(record) @@ -2112,8 +1512,16 @@ fn update_registry_validation_status( .registry .get(plugin_id) .and_then(|record| record.source.environment_ref.as_deref()); - let environment = environment_state(manifest, &scope.state_path, environment_ref); - let environment_error = environment_last_error(plugin_id, environment, environment_ref); + let (environment, validation_error) = + match validate_environment_state(manifest, &scope.state_path, environment_ref) { + Ok(environment) => (environment, None), + Err(error) => ( + DynamicPluginCheckState::Invalid, + environment_ref.map(|_| error), + ), + }; + let environment_error = + environment_last_error(plugin_id, environment, environment_ref, validation_error); scope .registry .update_validation_status( @@ -2147,24 +1555,27 @@ fn environment_last_error( plugin_id: &str, environment: DynamicPluginCheckState, environment_ref: Option<&str>, + detail: Option, ) -> Option { (environment == DynamicPluginCheckState::Invalid).then(|| DynamicPluginFailure { phase: DynamicPluginFailurePhase::Validation, code: "environment_failed".into(), - message: environment_ref.map_or_else( - || { - format!( - "dynamic plugin '{}' has no lifecycle-managed Python environment; run `nemo-relay plugins remove {}` to remove the manual registration, then run `nemo-relay plugins add `", - plugin_id, plugin_id - ) - }, - |environment_ref| { - format!( - "dynamic plugin '{}' configured Python environment {} is unavailable", - plugin_id, environment_ref - ) - }, - ), + message: detail.unwrap_or_else(|| { + environment_ref.map_or_else( + || { + format!( + "dynamic plugin '{}' has no lifecycle-managed Python environment; run `nemo-relay plugins remove {}` to remove the manual registration, then run `nemo-relay plugins add `", + plugin_id, plugin_id + ) + }, + |environment_ref| { + format!( + "dynamic plugin '{}' configured Python environment {} is unavailable", + plugin_id, environment_ref + ) + }, + ) + }), }) } @@ -2220,24 +1631,35 @@ fn manifest_ref_from_record(record: &DynamicPluginRecord) -> Result Result { + nemo_relay_plugin_host_config::pin_plugin_config_path(path) + .map(|path| path.display().to_string()) + .map_err(|error| CliError::Config(error.to_string())) +} + fn ensure_scope( scopes: &mut Vec, scope: RegistryScope, plugins_toml_path: PathBuf, state_path: PathBuf, ) -> usize { - if let Some(index) = scopes.iter().position(|existing| { - existing.scope == scope - && existing.plugins_toml_path == plugins_toml_path - && existing.state_path == state_path - }) { + if let Some(index) = scopes + .iter() + .position(|existing| existing.state_path == state_path) + { + scopes[index].add_source(scope, plugins_toml_path); return index; } scopes.push(ScopedRegistry { - scope, - plugins_toml_path, + sources: vec![self::state::ScopedRegistrySource { + scope, + plugins_toml_path, + }], state_path, - registry: nemo_relay::plugin::dynamic::DynamicPluginRegistry::new(), + registry: nemo_relay_plugin_host_config::DynamicPluginLifecycleState::new( + nemo_relay::plugin::dynamic::DynamicPluginRegistry::new(), + ), + state_lock: None, }); scopes.len() - 1 } diff --git a/crates/cli/src/plugins/lifecycle/state.rs b/crates/cli/src/plugins/lifecycle/state.rs index bc7b7b912..c62c76c2b 100644 --- a/crates/cli/src/plugins/lifecycle/state.rs +++ b/crates/cli/src/plugins/lifecycle/state.rs @@ -1,13 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::HashSet; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::collections::HashMap; +use std::path::PathBuf; use nemo_relay::plugin::dynamic::{DynamicPluginRecord, DynamicPluginRegistry}; -use serde::{Deserialize, Serialize}; +use nemo_relay_plugin_host_config::{ + DynamicPluginLifecycleState, LifecycleStateLock, lock_lifecycle_state, pin_plugin_config_path, + read_lifecycle_state, read_locked_lifecycle_state, save_locked_lifecycle_state, + sibling_lifecycle_state_path, +}; +use serde::Serialize; use strum::{Display, IntoStaticStr}; use crate::configuration::{ @@ -18,9 +21,6 @@ use crate::error::CliError; use super::super::config_io::TargetScope; // Internal CLI-managed lifecycle state. This file is not intended to be user-edited. -const DYNAMIC_PLUGIN_STATE_FILENAME: &str = ".dynamic-plugins.json"; -const DYNAMIC_PLUGIN_STATE_SCHEMA_VERSION: u32 = 1; - #[derive(Display, IntoStaticStr, Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] #[strum(serialize_all = "snake_case")] @@ -33,10 +33,16 @@ pub(super) enum RegistryScope { #[derive(Debug)] pub(super) struct ScopedRegistry { + pub(super) sources: Vec, + pub(super) state_path: PathBuf, + pub(super) registry: DynamicPluginLifecycleState, + pub(super) state_lock: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct ScopedRegistrySource { pub(super) scope: RegistryScope, pub(super) plugins_toml_path: PathBuf, - pub(super) state_path: PathBuf, - pub(super) registry: DynamicPluginRegistry, } #[derive(Debug, Clone)] @@ -48,90 +54,162 @@ pub(super) struct ScopedDynamicPluginRecord { pub(super) record: DynamicPluginRecord, } -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -struct PersistedDynamicPluginRegistry { - #[serde(default = "default_state_schema_version")] - schema_version: u32, - #[serde(default)] - records: Vec, -} - -const fn default_state_schema_version() -> u32 { - DYNAMIC_PLUGIN_STATE_SCHEMA_VERSION -} - impl ScopedRegistry { pub(super) fn save(&self) -> Result<(), CliError> { - let mut rendered = serde_json::to_vec_pretty(&PersistedDynamicPluginRegistry { - schema_version: DYNAMIC_PLUGIN_STATE_SCHEMA_VERSION, - records: self.registry.cloned_records(true), - }) - .map_err(|error| { + let lock = self.state_lock.as_ref().ok_or_else(|| { CliError::Config(format!( - "could not serialize dynamic plugin registry state {}: {error}", + "dynamic plugin lifecycle state {} is not locked for mutation", self.state_path.display() )) })?; - rendered.push(b'\n'); - let parent = self - .state_path - .parent() - .map(Path::to_path_buf) - .unwrap_or_else(|| PathBuf::from(".")); - std::fs::create_dir_all(&parent)?; + save_locked_lifecycle_state(lock, &self.registry) + .map_err(|error| CliError::Config(error.to_string())) + } - let temp_path = parent.join(format!( - ".{}.{}.tmp", - self.state_path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("dynamic-plugins"), - SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock is after unix epoch") - .as_nanos() - )); + pub(super) fn ensure_locked(&mut self) -> Result<(), CliError> { + if self.state_lock.is_some() { + return Ok(()); + } + if !self.registry.cloned_records(true).is_empty() { + return Err(CliError::Config(format!( + "refusing to lock lifecycle state {} after in-memory mutation", + self.state_path.display() + ))); + } + let lock = lock_lifecycle_state(&self.state_path) + .map_err(|error| CliError::Config(error.to_string()))?; + self.registry = read_locked_lifecycle_state(&lock) + .map_err(|error| CliError::Config(error.to_string()))?; + self.state_lock = Some(lock); + Ok(()) + } - let write_result = (|| -> Result<(), CliError> { - let mut file = std::fs::File::create(&temp_path)?; - file.write_all(&rendered)?; - file.sync_all()?; - std::fs::rename(&temp_path, &self.state_path)?; - Ok(()) - })(); + pub(super) fn add_source(&mut self, scope: RegistryScope, plugins_toml_path: PathBuf) { + if !self + .sources + .iter() + .any(|source| source.plugins_toml_path == plugins_toml_path) + { + self.sources.push(ScopedRegistrySource { + scope, + plugins_toml_path, + }); + } + } - if write_result.is_err() { - let _ = std::fs::remove_file(&temp_path); + pub(super) fn source_for_path(&self, path: &PathBuf) -> Option<&ScopedRegistrySource> { + self.sources + .iter() + .find(|source| &source.plugins_toml_path == path) + } + + pub(super) fn source_for_record(&self, plugin_id: &str) -> ScopedRegistrySource { + if let Some(owner) = self.registry.declaration_source(plugin_id) { + let owner = PathBuf::from(owner); + if let Some(source) = self.source_for_path(&owner) { + return source.clone(); + } + return ScopedRegistrySource { + scope: RegistryScope::Explicit, + plugins_toml_path: owner, + }; } - write_result?; - Ok(()) + self.sources + .first() + .cloned() + .expect("every lifecycle registry must retain at least one physical source") } } pub(super) fn load_scoped_registries( explicit_plugin_config: Option<&PathBuf>, ) -> Result, CliError> { - scoped_registry_layouts(explicit_plugin_config) + let layouts = scoped_registry_layouts(explicit_plugin_config, None)?; + layouts .into_iter() - .map(|(scope, plugins_toml_path, state_path)| { + .map(|layout| { + let registry = read_lifecycle_state(&layout.state_path) + .map_err(|error| CliError::Config(error.to_string()))?; Ok(ScopedRegistry { - scope, - plugins_toml_path, - registry: read_registry(&state_path)?, - state_path, + sources: layout.sources, + state_path: layout.state_path, + registry, + state_lock: None, }) }) .collect() } +pub(super) fn load_scoped_registries_for_update( + explicit_plugin_config: Option<&PathBuf>, + mutation_target: Option<(RegistryScope, PathBuf, PathBuf)>, +) -> Result, CliError> { + let force_locked_state = mutation_target + .as_ref() + .map(|(_, _, state_path)| state_path.clone()); + let layouts = scoped_registry_layouts(explicit_plugin_config, mutation_target)?; + let mut lock_order = Vec::new(); + for (index, layout) in layouts.iter().enumerate() { + let mut plugin_exists = false; + for source in &layout.sources { + plugin_exists |= source.plugins_toml_path.try_exists()?; + } + let state_exists = layout.state_path.try_exists()?; + if plugin_exists + || state_exists + || force_locked_state + .as_ref() + .is_some_and(|forced| forced == &layout.state_path) + { + lock_order.push((layout.state_path.clone(), index)); + } + } + lock_order.sort_by(|left, right| left.0.cmp(&right.0)); + let mut locks = HashMap::new(); + for (_, index) in lock_order { + let lock = lock_lifecycle_state(&layouts[index].state_path) + .map_err(|error| CliError::Config(error.to_string()))?; + locks.insert(index, lock); + } + let mut locked = HashMap::new(); + for (index, lock) in locks { + let registry = read_locked_lifecycle_state(&lock) + .map_err(|error| CliError::Config(error.to_string()))?; + locked.insert(index, (lock, registry)); + } + Ok(layouts + .into_iter() + .enumerate() + .map(|(index, layout)| { + let (state_lock, registry) = locked.remove(&index).map_or_else( + || { + ( + None, + DynamicPluginLifecycleState::new(DynamicPluginRegistry::new()), + ) + }, + |(lock, registry)| (Some(lock), registry), + ); + ScopedRegistry { + sources: layout.sources, + state_path: layout.state_path, + registry, + state_lock, + } + }) + .collect()) +} + pub(super) fn scoped_paths_for_add( scope: TargetScope, explicit_plugin_config: Option<&PathBuf>, ) -> Result<(PathBuf, PathBuf, RegistryScope), CliError> { if let Some(explicit_plugin_config) = explicit_plugin_config { + let plugins_toml_path = pin_plugin_config_path(explicit_plugin_config) + .map_err(|error| CliError::Config(error.to_string()))?; return Ok(( - explicit_plugin_config.clone(), - sibling_state_path(explicit_plugin_config), + plugins_toml_path.clone(), + sibling_lifecycle_state_path(&plugins_toml_path), RegistryScope::Explicit, )); } @@ -148,7 +226,9 @@ pub(super) fn scoped_paths_for_add( } TargetScope::Global => global_plugin_config_path(), }; - let state_path = sibling_state_path(&plugins_toml_path); + let plugins_toml_path = pin_plugin_config_path(&plugins_toml_path) + .map_err(|error| CliError::Config(error.to_string()))?; + let state_path = sibling_lifecycle_state_path(&plugins_toml_path); let scope = match scope { TargetScope::User => RegistryScope::User, TargetScope::Project => RegistryScope::Project, @@ -164,10 +244,11 @@ pub(super) fn collect_records( let mut records = Vec::new(); for (scope_index, scope) in scopes.iter().enumerate() { for record in scope.registry.cloned_records(include_tombstoned) { + let source = scope.source_for_record(&record.metadata.id); records.push(ScopedDynamicPluginRecord { scope_index, - scope: scope.scope, - plugins_toml_path: scope.plugins_toml_path.clone(), + scope: source.scope, + plugins_toml_path: source.plugins_toml_path, state_path: scope.state_path.clone(), record, }); @@ -223,76 +304,60 @@ pub(super) fn find_record_by_id( fn scoped_registry_layouts( explicit_plugin_config: Option<&PathBuf>, -) -> Vec<(RegistryScope, PathBuf, PathBuf)> { + mutation_target: Option<(RegistryScope, PathBuf, PathBuf)>, +) -> Result, CliError> { let mut layouts = Vec::new(); if let Some(explicit_plugin_config) = explicit_plugin_config { - layouts.push(( - RegistryScope::Explicit, - explicit_plugin_config.clone(), - sibling_state_path(explicit_plugin_config), - )); + layouts.push((RegistryScope::Explicit, explicit_plugin_config.clone())); } else if let Some(plugins_toml_path) = user_plugin_config_path() { - layouts.push(( - RegistryScope::User, - plugins_toml_path.clone(), - sibling_state_path(&plugins_toml_path), - )); + layouts.push((RegistryScope::User, plugins_toml_path)); } let user_only = std::env::var("NEMO_RELAY_CONFIG_SCOPE").ok().as_deref() == Some("user"); if !user_only && let Ok(cwd) = std::env::current_dir() { let plugins_toml_path = project_plugin_config_path(&cwd); - layouts.push(( - RegistryScope::Project, - plugins_toml_path.clone(), - sibling_state_path(&plugins_toml_path), - )); + layouts.push((RegistryScope::Project, plugins_toml_path)); } let plugins_toml_path = global_plugin_config_path(); - layouts.push(( - RegistryScope::Global, - plugins_toml_path.clone(), - sibling_state_path(&plugins_toml_path), - )); - - let mut seen = HashSet::new(); - let mut unique = Vec::with_capacity(layouts.len()); - for layout in layouts.into_iter().rev() { - let identity = layout.1.canonicalize().unwrap_or_else(|_| layout.1.clone()); - if seen.insert(identity) { - unique.push(layout); - } + layouts.push((RegistryScope::Global, plugins_toml_path)); + if let Some((scope, plugins_toml_path, _)) = mutation_target { + layouts.push((scope, plugins_toml_path)); } - unique.reverse(); - unique -} -fn read_registry(path: &Path) -> Result { - if !path.exists() { - return Ok(DynamicPluginRegistry::new()); - } - let raw = std::fs::read_to_string(path)?; - let state: PersistedDynamicPluginRegistry = serde_json::from_str(&raw).map_err(|error| { - CliError::Config(format!( - "invalid dynamic plugin registry state in {}: {error}", - path.display() - )) - })?; - if state.schema_version != DYNAMIC_PLUGIN_STATE_SCHEMA_VERSION { - return Err(CliError::Config(format!( - "unsupported dynamic plugin registry schema_version {} in {}; expected {}", - state.schema_version, - path.display(), - DYNAMIC_PLUGIN_STATE_SCHEMA_VERSION - ))); + let mut grouped = Vec::::new(); + for (scope, plugins_toml_path) in layouts { + let plugins_toml_path = pin_plugin_config_path(&plugins_toml_path) + .map_err(|error| CliError::Config(error.to_string()))?; + let state_path = sibling_lifecycle_state_path(&plugins_toml_path); + if let Some(existing) = grouped + .iter_mut() + .find(|existing| existing.state_path == state_path) + { + if !existing + .sources + .iter() + .any(|source| source.plugins_toml_path == plugins_toml_path) + { + existing.sources.push(ScopedRegistrySource { + scope, + plugins_toml_path, + }); + } + } else { + grouped.push(ScopedRegistryLayout { + sources: vec![ScopedRegistrySource { + scope, + plugins_toml_path, + }], + state_path, + }); + } } - DynamicPluginRegistry::from_records(state.records) - .map_err(|error| CliError::Config(error.to_string())) + Ok(grouped) } -fn sibling_state_path(plugins_toml_path: &Path) -> PathBuf { - plugins_toml_path - .parent() - .map(|parent| parent.join(DYNAMIC_PLUGIN_STATE_FILENAME)) - .unwrap_or_else(|| PathBuf::from(DYNAMIC_PLUGIN_STATE_FILENAME)) +#[derive(Debug)] +struct ScopedRegistryLayout { + sources: Vec, + state_path: PathBuf, } diff --git a/crates/cli/src/plugins/lifecycle/trust.rs b/crates/cli/src/plugins/lifecycle/trust.rs index 97a366671..d8417c781 100644 --- a/crates/cli/src/plugins/lifecycle/trust.rs +++ b/crates/cli/src/plugins/lifecycle/trust.rs @@ -1,424 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::fmt; -use std::path::{Path, PathBuf}; - -use base64::Engine; -use nemo_relay::plugin::dynamic::{ - DynamicPluginAttestationMode, DynamicPluginCheckState, DynamicPluginFailure, - DynamicPluginFailurePhase, DynamicPluginManifest, +// Trust verification is shared with embedded file-backed hosts. Keep this module as a compatibility +// adapter until the stacked CLI runtime-owner migration removes the private lifecycle facade. +#[cfg(test)] +pub(super) use nemo_relay_plugin_host_config::DynamicPluginTrustFailure; +pub(super) use nemo_relay_plugin_host_config::{ + EvaluatedDynamicPluginTrust, evaluate_dynamic_plugin_trust, }; -use ring::signature::{ED25519, UnparsedPublicKey}; -use sha2::{Digest, Sha256}; - -use crate::plugins::policy::EvaluatedDynamicPluginHostPolicy; - -type TrustResult = Result; - -#[derive(Debug, Clone)] -pub(super) enum DynamicPluginTrustFailure { - MissingArtifact, - MissingIntegrityDigest, - ArtifactRead { - path: PathBuf, - error: String, - }, - IntegrityMismatch { - path: PathBuf, - expected: String, - actual: String, - }, - MissingSignature, - MissingTrustedKeys, - SignatureRead { - path: PathBuf, - error: String, - }, - InvalidTrustedKey { - key: String, - error: String, - }, - SignatureVerification { - path: PathBuf, - parse_errors: Vec, - }, -} - -impl DynamicPluginTrustFailure { - pub(super) fn display<'a>( - &'a self, - plugin_id: &'a str, - ) -> DynamicPluginTrustFailureDisplay<'a> { - DynamicPluginTrustFailureDisplay { - failure: self, - plugin_id, - } - } - - pub(super) fn refusal_code(&self) -> &'static str { - match self { - Self::MissingArtifact - | Self::MissingIntegrityDigest - | Self::ArtifactRead { .. } - | Self::IntegrityMismatch { .. } => "integrity_failed", - Self::MissingSignature - | Self::MissingTrustedKeys - | Self::SignatureRead { .. } - | Self::InvalidTrustedKey { .. } - | Self::SignatureVerification { .. } => "attestation_failed", - } - } -} - -pub(super) struct DynamicPluginTrustFailureDisplay<'a> { - failure: &'a DynamicPluginTrustFailure, - plugin_id: &'a str, -} - -impl fmt::Display for DynamicPluginTrustFailureDisplay<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.failure { - DynamicPluginTrustFailure::MissingArtifact => write!( - f, - "dynamic plugin '{}' is missing source.artifact required for integrity verification", - self.plugin_id - ), - DynamicPluginTrustFailure::MissingIntegrityDigest => write!( - f, - "dynamic plugin '{}' is missing integrity.sha256 required for host trust verification", - self.plugin_id - ), - DynamicPluginTrustFailure::ArtifactRead { path, error } => write!( - f, - "dynamic plugin '{}' artifact {} could not be read for trust verification: {}", - self.plugin_id, - path.display(), - error - ), - DynamicPluginTrustFailure::IntegrityMismatch { - path, - expected, - actual, - } => write!( - f, - "dynamic plugin '{}' failed integrity verification for {}: expected {}, got {}", - self.plugin_id, - path.display(), - expected, - actual - ), - DynamicPluginTrustFailure::MissingSignature => write!( - f, - "dynamic plugin '{}' requires integrity.signature under host policy", - self.plugin_id - ), - DynamicPluginTrustFailure::MissingTrustedKeys => write!( - f, - "dynamic plugin '{}' requires signature verification, but no trusted_public_keys are configured in host policy", - self.plugin_id - ), - DynamicPluginTrustFailure::SignatureRead { path, error } => write!( - f, - "dynamic plugin '{}' signature {} could not be read: {}", - self.plugin_id, - path.display(), - error - ), - DynamicPluginTrustFailure::InvalidTrustedKey { key, error } => write!( - f, - "dynamic plugin '{}' has invalid trusted public key '{}': {}", - self.plugin_id, key, error - ), - DynamicPluginTrustFailure::SignatureVerification { path, parse_errors } => { - write!( - f, - "dynamic plugin '{}' failed signature verification for {} against configured host policy keys", - self.plugin_id, - path.display() - )?; - if !parse_errors.is_empty() { - write!(f, "; key parse errors: {}", parse_errors.join("; "))?; - } - Ok(()) - } - } - } -} - -#[derive(Debug, Clone)] -pub(super) struct EvaluatedDynamicPluginTrust { - pub(super) integrity: DynamicPluginCheckState, - pub(super) authenticity: DynamicPluginCheckState, - pub(super) failure: Option, -} - -impl EvaluatedDynamicPluginTrust { - fn valid(authenticity: DynamicPluginCheckState) -> Self { - Self { - integrity: DynamicPluginCheckState::Valid, - authenticity, - failure: None, - } - } - - fn failed( - integrity: DynamicPluginCheckState, - authenticity: DynamicPluginCheckState, - failure: DynamicPluginTrustFailure, - ) -> Self { - Self { - integrity, - authenticity, - failure: Some(failure), - } - } - - pub(super) fn failure(&self) -> Option<&DynamicPluginTrustFailure> { - self.failure.as_ref() - } - - pub(super) fn refusal_code(&self) -> Option<&'static str> { - self.failure - .as_ref() - .map(DynamicPluginTrustFailure::refusal_code) - } - - pub(super) fn is_satisfied(&self) -> bool { - self.failure.is_none() - } - - pub(super) fn last_error(&self, plugin_id: &str) -> Option { - self.failure.as_ref().map(|failure| DynamicPluginFailure { - phase: DynamicPluginFailurePhase::Validation, - code: failure.refusal_code().into(), - message: failure.display(plugin_id).to_string(), - }) - } -} - -pub(super) fn evaluate_dynamic_plugin_trust( - manifest: &DynamicPluginManifest, - manifest_ref: &str, - policy: &EvaluatedDynamicPluginHostPolicy, -) -> EvaluatedDynamicPluginTrust { - if !policy.policy_satisfied { - return EvaluatedDynamicPluginTrust { - integrity: DynamicPluginCheckState::Unknown, - authenticity: DynamicPluginCheckState::Unknown, - failure: None, - }; - } - - let artifact_path = match verify_integrity(manifest, manifest_ref) { - Ok(artifact_path) => artifact_path, - Err(failure) => { - return EvaluatedDynamicPluginTrust::failed( - DynamicPluginCheckState::Invalid, - DynamicPluginCheckState::Unknown, - failure, - ); - } - }; - - match evaluate_authenticity(manifest, manifest_ref, artifact_path.as_path(), policy) { - Ok(authenticity) => EvaluatedDynamicPluginTrust::valid(authenticity), - Err(failure) => EvaluatedDynamicPluginTrust::failed( - DynamicPluginCheckState::Valid, - DynamicPluginCheckState::Invalid, - failure, - ), - } -} - -fn verify_integrity(manifest: &DynamicPluginManifest, manifest_ref: &str) -> TrustResult { - let artifact = manifest - .source - .as_ref() - .and_then(|source| source.artifact.as_deref()) - .ok_or(DynamicPluginTrustFailure::MissingArtifact)?; - let expected_digest = manifest - .integrity - .as_ref() - .and_then(|integrity| integrity.sha256.as_deref()) - .map(str::trim) - .filter(|value| !value.is_empty()) - .ok_or(DynamicPluginTrustFailure::MissingIntegrityDigest)?; - - let artifact_path = resolve_artifact_path(manifest_ref, artifact); - let actual_digest = - file_sha256(&artifact_path).map_err(|error| DynamicPluginTrustFailure::ArtifactRead { - path: artifact_path.clone(), - error: error.to_string(), - })?; - - if actual_digest != expected_digest { - return Err(DynamicPluginTrustFailure::IntegrityMismatch { - path: artifact_path, - expected: expected_digest.to_owned(), - actual: actual_digest, - }); - } - - Ok(artifact_path) -} - -fn evaluate_authenticity( - manifest: &DynamicPluginManifest, - manifest_ref: &str, - artifact_path: &Path, - policy: &EvaluatedDynamicPluginHostPolicy, -) -> TrustResult { - let signature_ref = manifest - .integrity - .as_ref() - .and_then(|integrity| integrity.signature.as_deref()) - .map(str::trim) - .filter(|value| !value.is_empty()); - - match policy.attestation_mode { - DynamicPluginAttestationMode::IntegrityOnly => Ok(DynamicPluginCheckState::Unknown), - DynamicPluginAttestationMode::SignatureIfPresent => match signature_ref { - Some(signature_ref) => { - verify_signature( - manifest_ref, - artifact_path, - signature_ref, - &policy.trusted_public_keys, - )?; - Ok(DynamicPluginCheckState::Valid) - } - None => Ok(DynamicPluginCheckState::Unknown), - }, - DynamicPluginAttestationMode::SignatureRequired => match signature_ref { - Some(signature_ref) => { - verify_signature( - manifest_ref, - artifact_path, - signature_ref, - &policy.trusted_public_keys, - )?; - Ok(DynamicPluginCheckState::Valid) - } - None => Err(DynamicPluginTrustFailure::MissingSignature), - }, - } -} - -fn verify_signature( - manifest_ref: &str, - artifact_path: &Path, - signature_ref: &str, - trusted_public_keys: &[String], -) -> TrustResult<()> { - if trusted_public_keys.is_empty() { - return Err(DynamicPluginTrustFailure::MissingTrustedKeys); - } - - let signature_path = resolve_artifact_path(manifest_ref, signature_ref); - let signature_bytes = read_signature_bytes(&signature_path)?; - let artifact_bytes = crate::filesystem::bounded::read_bounded_regular_file( - artifact_path, - "dynamic plugin artifact", - ) - .map_err(|error| DynamicPluginTrustFailure::ArtifactRead { - path: artifact_path.to_path_buf(), - error, - })?; - - let mut parse_errors = Vec::new(); - for trusted_public_key in trusted_public_keys { - let public_key_bytes = match parse_ed25519_public_key(trusted_public_key) { - Ok(public_key_bytes) => public_key_bytes, - Err(DynamicPluginTrustFailure::InvalidTrustedKey { key: _, error }) => { - parse_errors.push(error); - continue; - } - Err(other) => return Err(other), - }; - - let verifier = UnparsedPublicKey::new(&ED25519, public_key_bytes); - if verifier.verify(&artifact_bytes, &signature_bytes).is_ok() { - return Ok(()); - } - } - - Err(DynamicPluginTrustFailure::SignatureVerification { - path: signature_path, - parse_errors, - }) -} - -fn read_signature_bytes(path: &Path) -> TrustResult> { - let raw = - crate::filesystem::bounded::read_bounded_regular_file(path, "dynamic plugin signature") - .map_err(|error| DynamicPluginTrustFailure::SignatureRead { - path: path.to_path_buf(), - error, - })?; - let trimmed = String::from_utf8_lossy(&raw).trim().to_owned(); - if trimmed.is_empty() { - return Err(DynamicPluginTrustFailure::SignatureRead { - path: path.to_path_buf(), - error: "signature file is empty".into(), - }); - } - - let encoded = trimmed - .strip_prefix("ed25519:") - .unwrap_or(trimmed.as_str()) - .trim(); - base64::engine::general_purpose::STANDARD - .decode(encoded) - .map_err(|error| DynamicPluginTrustFailure::SignatureRead { - path: path.to_path_buf(), - error: format!("invalid base64 signature: {error}"), - }) -} - -fn parse_ed25519_public_key(value: &str) -> TrustResult> { - let encoded = value.trim().strip_prefix("ed25519:").ok_or_else(|| { - DynamicPluginTrustFailure::InvalidTrustedKey { - key: value.to_owned(), - error: format!("unsupported trusted public key format '{value}'"), - } - })?; - base64::engine::general_purpose::STANDARD - .decode(encoded.trim()) - .map_err(|error| DynamicPluginTrustFailure::InvalidTrustedKey { - key: value.to_owned(), - error: format!("invalid ed25519 trusted public key '{value}': {error}"), - }) -} - -fn resolve_artifact_path(manifest_ref: &str, artifact_ref: &str) -> PathBuf { - let artifact_path = PathBuf::from(artifact_ref); - if artifact_path.is_absolute() { - artifact_path - } else { - Path::new(manifest_ref) - .parent() - .map(|parent| parent.join(&artifact_path)) - .unwrap_or(artifact_path) - } -} - -fn file_sha256(path: &Path) -> Result { - let mut digest = Sha256::new(); - crate::filesystem::bounded::stream_bounded_regular_file( - path, - "dynamic plugin artifact", - |bytes| { - digest.update(bytes); - }, - ) - .map_err(std::io::Error::other)?; - Ok(format!( - "sha256:{}", - digest - .finalize() - .iter() - .map(|byte| format!("{byte:02x}")) - .collect::() - )) -} diff --git a/crates/cli/src/plugins/policy.rs b/crates/cli/src/plugins/policy.rs index 4a43d38e9..79139a57f 100644 --- a/crates/cli/src/plugins/policy.rs +++ b/crates/cli/src/plugins/policy.rs @@ -1,270 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeMap; -use std::fmt; - -use nemo_relay::plugin::dynamic::{ - DynamicPluginAttestationMode, DynamicPluginCheckState, DynamicPluginFailure, - DynamicPluginFailurePhase, DynamicPluginKind, DynamicPluginManifest, DynamicPluginStartupClass, +// The CLI remains the dynamic-plugin control plane, but policy parsing and evaluation are shared +// with embedded file-backed hosts so one plugins.toml has identical trust semantics everywhere. +#[allow(unused_imports)] +pub(crate) use nemo_relay_plugin_host_config::{ + DynamicPluginHostPolicy, DynamicPluginHostPolicyEffect, DynamicPluginHostPolicyFailure, + DynamicPluginHostPolicyRule, EvaluatedDynamicPluginHostPolicy, FileDynamicPluginHostPolicy, + evaluate_dynamic_plugin_host_policy, }; -use serde::Deserialize; - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub(crate) struct DynamicPluginHostPolicy { - pub(crate) defaults: DynamicPluginHostPolicyEffect, - pub(crate) rules: Vec, - pub(crate) overrides: BTreeMap, -} - -impl DynamicPluginHostPolicy { - pub(crate) fn merge_from(&mut self, other: Self) { - self.defaults.merge_from(other.defaults); - self.rules.extend(other.rules); - for (plugin_id, effect) in other.overrides { - self.overrides - .entry(plugin_id) - .or_default() - .merge_from(effect); - } - } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub(crate) struct DynamicPluginHostPolicyEffect { - pub(crate) allowed: Option, - pub(crate) startup: Option, - pub(crate) attestation: Option, - pub(crate) trusted_public_keys: Option>, -} - -impl DynamicPluginHostPolicyEffect { - fn merge_from(&mut self, other: Self) { - if let Some(value) = other.allowed { - self.allowed = Some(value); - } - if let Some(value) = other.startup { - self.startup = Some(value); - } - if let Some(value) = other.attestation { - self.attestation = Some(value); - } - if let Some(value) = other.trusted_public_keys { - self.trusted_public_keys = Some(value); - } - } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub(crate) struct DynamicPluginHostPolicyRule { - pub(crate) match_kind: Option, - pub(crate) match_plugin_id: Option, - pub(crate) effect: DynamicPluginHostPolicyEffect, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum DynamicPluginHostPolicyFailure { - Blocked, -} - -impl DynamicPluginHostPolicyFailure { - pub(crate) fn display<'a>( - &'a self, - plugin_id: &'a str, - ) -> DynamicPluginHostPolicyFailureDisplay<'a> { - DynamicPluginHostPolicyFailureDisplay { - failure: self, - plugin_id, - } - } -} - -pub(crate) struct DynamicPluginHostPolicyFailureDisplay<'a> { - failure: &'a DynamicPluginHostPolicyFailure, - plugin_id: &'a str, -} - -impl fmt::Display for DynamicPluginHostPolicyFailureDisplay<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.failure { - DynamicPluginHostPolicyFailure::Blocked => write!( - f, - "dynamic plugin '{}' is blocked by host policy", - self.plugin_id - ), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct EvaluatedDynamicPluginHostPolicy { - pub(crate) policy_satisfied: bool, - pub(crate) startup_class: DynamicPluginStartupClass, - pub(crate) attestation_mode: DynamicPluginAttestationMode, - pub(crate) trusted_public_keys: Vec, - pub(crate) failure: Option, -} - -impl EvaluatedDynamicPluginHostPolicy { - pub(crate) fn check_state(&self) -> DynamicPluginCheckState { - if self.policy_satisfied { - DynamicPluginCheckState::Valid - } else { - DynamicPluginCheckState::Invalid - } - } - - pub(crate) fn last_error(&self, plugin_id: &str) -> Option { - self.failure.as_ref().map(|failure| DynamicPluginFailure { - phase: DynamicPluginFailurePhase::Policy, - code: "policy_blocked".into(), - message: failure.display(plugin_id).to_string(), - }) - } - - pub(crate) fn failure(&self) -> Option<&DynamicPluginHostPolicyFailure> { - self.failure.as_ref() - } -} - -pub(crate) fn evaluate_dynamic_plugin_host_policy( - policy: &DynamicPluginHostPolicy, - manifest: &DynamicPluginManifest, -) -> EvaluatedDynamicPluginHostPolicy { - let mut effect = DynamicPluginHostPolicyEffect { - allowed: Some(true), - startup: Some(DynamicPluginStartupClass::Optional), - attestation: Some(DynamicPluginAttestationMode::IntegrityOnly), - trusted_public_keys: None, - }; - effect.merge_from(policy.defaults.clone()); - - for rule in &policy.rules { - if !policy_rule_matches(rule, manifest) { - continue; - } - effect.merge_from(rule.effect.clone()); - } - - if let Some(override_effect) = policy.overrides.get(manifest.plugin.id.trim()) { - effect.merge_from(override_effect.clone()); - } - - let startup_class = effect - .startup - .unwrap_or(DynamicPluginStartupClass::Optional); - let attestation_mode = effect - .attestation - .unwrap_or(DynamicPluginAttestationMode::IntegrityOnly); - let trusted_public_keys = effect.trusted_public_keys.unwrap_or_default(); - - if effect.allowed == Some(false) { - return EvaluatedDynamicPluginHostPolicy { - policy_satisfied: false, - startup_class, - attestation_mode, - trusted_public_keys, - failure: Some(DynamicPluginHostPolicyFailure::Blocked), - }; - } - - EvaluatedDynamicPluginHostPolicy { - policy_satisfied: true, - startup_class, - attestation_mode, - trusted_public_keys, - failure: None, - } -} - -fn policy_rule_matches( - rule: &DynamicPluginHostPolicyRule, - manifest: &DynamicPluginManifest, -) -> bool { - if let Some(match_kind) = rule.match_kind - && manifest.plugin.kind != match_kind - { - return false; - } - if let Some(match_plugin_id) = &rule.match_plugin_id - && manifest.plugin.id.trim() != match_plugin_id - { - return false; - } - true -} - -#[derive(Debug, Clone, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct FileDynamicPluginHostPolicy { - #[serde(default)] - pub(crate) defaults: FileDynamicPluginHostPolicyEffect, - #[serde(default)] - pub(crate) rules: Vec, - #[serde(default)] - pub(crate) overrides: BTreeMap, -} - -impl From for DynamicPluginHostPolicy { - fn from(value: FileDynamicPluginHostPolicy) -> Self { - Self { - defaults: value.defaults.into(), - rules: value.rules.into_iter().map(Into::into).collect(), - overrides: value - .overrides - .into_iter() - .map(|(plugin_id, effect)| (plugin_id.trim().to_owned(), effect.into())) - .collect(), - } - } -} - -#[derive(Debug, Clone, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct FileDynamicPluginHostPolicyEffect { - allowed: Option, - startup: Option, - attestation: Option, - trusted_public_keys: Option>, -} - -impl From for DynamicPluginHostPolicyEffect { - fn from(value: FileDynamicPluginHostPolicyEffect) -> Self { - Self { - allowed: value.allowed, - startup: value.startup, - attestation: value.attestation, - trusted_public_keys: value - .trusted_public_keys - .map(|keys| keys.into_iter().map(|key| key.trim().to_owned()).collect()), - } - } -} - -#[derive(Debug, Clone, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct FileDynamicPluginHostPolicyRule { - match_kind: Option, - match_plugin_id: Option, - allowed: Option, - startup: Option, - attestation: Option, - trusted_public_keys: Option>, -} - -impl From for DynamicPluginHostPolicyRule { - fn from(value: FileDynamicPluginHostPolicyRule) -> Self { - Self { - match_kind: value.match_kind, - match_plugin_id: value.match_plugin_id.map(|value| value.trim().to_owned()), - effect: DynamicPluginHostPolicyEffect { - allowed: value.allowed, - startup: value.startup, - attestation: value.attestation, - trusted_public_keys: value - .trusted_public_keys - .map(|keys| keys.into_iter().map(|key| key.trim().to_owned()).collect()), - }, - } - } -} diff --git a/crates/cli/src/server/mod.rs b/crates/cli/src/server/mod.rs index c4ca3ea62..8e60bfabb 100644 --- a/crates/cli/src/server/mod.rs +++ b/crates/cli/src/server/mod.rs @@ -1062,7 +1062,9 @@ impl PluginActivation { } for plugin in &dynamic_plugins { if let Some(snapshot) = plugin.activation_snapshot.as_ref() { - snapshot.verify_current()?; + snapshot + .verify_current() + .map_err(|error| CliError::Config(error.to_string()))?; } } let native_specs = dynamic_plugins @@ -1128,7 +1130,9 @@ impl PluginActivation { }; for plugin in &dynamic_plugins { if let Some(snapshot) = plugin.activation_snapshot.as_ref() { - snapshot.verify_current()?; + snapshot + .verify_current() + .map_err(|error| CliError::Config(error.to_string()))?; } } let worker = diff --git a/crates/cli/tests/coverage/shared/config_tests.rs b/crates/cli/tests/coverage/shared/config_tests.rs index 333650646..fdb61de15 100644 --- a/crates/cli/tests/coverage/shared/config_tests.rs +++ b/crates/cli/tests/coverage/shared/config_tests.rs @@ -965,11 +965,11 @@ path = "/etc/nemo-relay/pricing.json" .unwrap(); symlink(&physical, &alias).unwrap(); - let resolved = load_plugin_toml_config_from_paths(vec![alias, physical.clone()]) + let resolved = load_plugin_toml_config_from_paths(vec![alias.clone(), physical.clone()]) .unwrap() .expect("the physical file exists"); - assert_eq!(resolved.contributing_sources, vec![physical]); + assert_eq!(resolved.contributing_sources, vec![physical.clone()]); assert_eq!( resolved.value.unwrap()["components"][0]["config"]["sources"] .as_array() @@ -978,6 +978,15 @@ path = "/etc/nemo-relay/pricing.json" 1, "the aliased file must not duplicate list entries" ); + + let aliased = load_plugin_toml_config_from_paths(vec![physical, alias.clone()]) + .unwrap() + .expect("the aliased file exists"); + assert_eq!( + aliased.contributing_sources, + vec![alias], + "the CLI must retain the selected spelling at the highest-precedence alias" + ); } #[cfg(unix)] @@ -2773,10 +2782,6 @@ fn persistent_hook_identity_authenticates_python_marker_without_rehashing_enviro assert!(active[0].activation_snapshot.is_some()); let snapshot_fingerprint = persistent_bootstrap_fingerprint(&resolved, &active).unwrap(); assert!(snapshot_fingerprint.starts_with("hmac-sha256:")); - assert!( - crate::plugins::lifecycle::test_python_environment_digest_calls() > 0, - "activation must verify the complete environment before snapshotting it" - ); crate::plugins::lifecycle::reset_test_python_environment_digest_calls(); std::fs::write( @@ -2796,9 +2801,28 @@ fn persistent_hook_identity_authenticates_python_marker_without_rehashing_enviro .unwrap_err() .to_string(); assert!(error.contains("changed after provisioning"), "{error}"); + + crate::plugins::lifecycle::list( + crate::plugins::PluginsListRequest::default(), + &GatewayOverrides::default(), + ) + .unwrap(); + crate::plugins::lifecycle::inspect( + crate::plugins::PluginsInspectRequest { + id: plugin_id.into(), + json: false, + }, + &GatewayOverrides::default(), + ) + .unwrap(); + let refreshed_state: serde_json::Value = + serde_json::from_slice(&std::fs::read(user_config.join(".dynamic-plugins.json")).unwrap()) + .unwrap(); assert!( - crate::plugins::lifecycle::test_python_environment_digest_calls() > 0, - "sidecar activation must still perform the full environment verification" + refreshed_state["records"][0]["status"]["last_error"]["message"] + .as_str() + .is_some_and(|message| message.contains("changed after provisioning")), + "list/inspect must preserve the detailed environment validation failure: {refreshed_state}" ); } diff --git a/crates/cli/tests/coverage/shared/plugins_lifecycle_tests.rs b/crates/cli/tests/coverage/shared/plugins_lifecycle_tests.rs index efcef6488..4e04d6def 100644 --- a/crates/cli/tests/coverage/shared/plugins_lifecycle_tests.rs +++ b/crates/cli/tests/coverage/shared/plugins_lifecycle_tests.rs @@ -22,85 +22,6 @@ use ring::rand::SystemRandom; use ring::signature::{Ed25519KeyPair, KeyPair}; use sha2::{Digest, Sha256}; -#[cfg(unix)] -#[test] -fn python_venv_launcher_detection_only_preserves_bin_python_links() { - assert!(is_python_venv_launcher(Path::new("env/bin/python"))); - assert!(is_python_venv_launcher(Path::new("env/bin/python3.11"))); - assert!(!is_python_venv_launcher(Path::new("env/bin/pip"))); - assert!(!is_python_venv_launcher(Path::new("env/lib/python3.11"))); -} - -#[cfg(unix)] -#[test] -fn snapshot_protection_does_not_follow_python_launcher_symlink() { - use std::os::unix::fs::{PermissionsExt, symlink}; - - let temp = tempfile::tempdir().unwrap(); - let target = temp.path().join("external-python"); - std::fs::write(&target, b"python").unwrap(); - std::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o755)).unwrap(); - let root = temp.path().join("snapshot"); - let bin = root.join("bin"); - std::fs::create_dir_all(&bin).unwrap(); - symlink(&target, bin.join("python")).unwrap(); - - protect_snapshot_tree(&root).unwrap(); - - assert!( - std::fs::symlink_metadata(bin.join("python")) - .unwrap() - .file_type() - .is_symlink() - ); - assert_eq!( - std::fs::metadata(&target).unwrap().permissions().mode() & 0o777, - 0o755 - ); - make_snapshot_removable(&root); -} - -#[cfg(unix)] -#[test] -fn snapshot_digest_hashes_python_launcher_symlink_without_following_it() { - use std::os::unix::fs::symlink; - - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().join("snapshot"); - let bin = root - .join(MANAGED_ENVIRONMENTS_DIR) - .join("environment") - .join("bin"); - std::fs::create_dir_all(&bin).unwrap(); - let launcher = bin.join("python"); - symlink("/missing/python-a", &launcher).unwrap(); - - let first_verification = snapshot_tree_digest(&root, false).unwrap(); - let first_identity = snapshot_tree_digest(&root, true).unwrap(); - - std::fs::remove_file(&launcher).unwrap(); - std::fs::write(&launcher, b"/missing/python-a").unwrap(); - assert_ne!( - first_verification, - snapshot_tree_digest(&root, false).unwrap(), - "a regular file must not collide with an equivalent symlink target" - ); - - std::fs::remove_file(&launcher).unwrap(); - symlink("/missing/python-b", &launcher).unwrap(); - - assert_ne!( - first_verification, - snapshot_tree_digest(&root, false).unwrap(), - "verification must include the exact launcher target" - ); - assert_eq!( - first_identity, - snapshot_tree_digest(&root, true).unwrap(), - "managed environment contents are excluded from stable gateway identity" - ); -} - struct CurrentDirGuard { original: PathBuf, } @@ -364,7 +285,15 @@ impl PythonEnvironmentCommandRunner for FakePythonEnvironmentRunner { let environment = PathBuf::from(args.last().expect("venv environment path")); let python = environment::environment_python_path(&environment); std::fs::create_dir_all(python.parent().unwrap()).unwrap(); - std::fs::write(python, b"fake python").unwrap(); + std::fs::write(&python, b"fake python").unwrap(); + std::fs::write( + environment.join("pyvenv.cfg"), + format!( + "home = {}\nversion_info = 3.11.0\n", + python.parent().unwrap().display() + ), + ) + .unwrap(); return Ok(()); } if self.fail_install && args.get(1).is_some_and(|arg| arg == "pip") { @@ -728,10 +657,6 @@ entrypoint = "../worker-runtime/worker.sh" &crate::plugins::policy::DynamicPluginHostPolicy::default(), ) .unwrap(); - assert_eq!( - std::fs::read(snapshot.root.join("external-entrypoint/resource.txt")).unwrap(), - b"expected\n" - ); let (activation_manifest, _) = DynamicPluginManifest::load_from_path(PathBuf::from(snapshot.activation_manifest_ref())) .unwrap(); @@ -743,6 +668,11 @@ entrypoint = "../worker-runtime/worker.sh" let DynamicPluginManifestLoad::Worker(activation_load) = &activation_manifest.load else { panic!("command activation must retain a worker load contract"); }; + let activation_entrypoint = PathBuf::from(activation_load.entrypoint.as_ref().unwrap()); + assert_eq!( + std::fs::read(activation_entrypoint.parent().unwrap().join("resource.txt"),).unwrap(), + b"expected\n" + ); assert_eq!( Some(activation_artifact), activation_load.entrypoint.as_deref(), @@ -807,18 +737,13 @@ fn activation_snapshot_detects_mutation_of_the_runtime_copy() { &crate::plugins::policy::DynamicPluginHostPolicy::default(), ) .unwrap(); - std::fs::set_permissions(&snapshot.root, std::fs::Permissions::from_mode(0o700)).unwrap(); - std::fs::set_permissions( - snapshot.activation_manifest.parent().unwrap(), - std::fs::Permissions::from_mode(0o700), - ) - .unwrap(); - std::fs::set_permissions( - &snapshot.activation_manifest, - std::fs::Permissions::from_mode(0o600), - ) - .unwrap(); - std::fs::write(&snapshot.activation_manifest, b"replaced").unwrap(); + let activation_manifest = PathBuf::from(snapshot.activation_manifest_ref()); + let runtime_root = activation_manifest.parent().unwrap(); + let snapshot_root = runtime_root.parent().unwrap(); + std::fs::set_permissions(snapshot_root, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::set_permissions(runtime_root, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::set_permissions(&activation_manifest, std::fs::Permissions::from_mode(0o600)).unwrap(); + std::fs::write(&activation_manifest, b"replaced").unwrap(); let error = snapshot.verify_current().unwrap_err().to_string(); assert!(error.contains("changed before code load"), "{error}"); @@ -886,10 +811,17 @@ symbol = "nemo_relay_fixture_native_plugin" ) .unwrap(); + let (activation_manifest, _) = + DynamicPluginManifest::load_from_path(PathBuf::from(snapshot.activation_manifest_ref())) + .unwrap(); + let DynamicPluginManifestLoad::RustDynamic(activation_load) = &activation_manifest.load else { + panic!("native activation must retain a Rust dynamic load contract"); + }; assert!( - snapshot - .root - .join("external-library/libadjacent_dependency.so") + Path::new(activation_load.library.as_deref().unwrap()) + .parent() + .unwrap() + .join("libadjacent_dependency.so") .is_file() ); assert_eq!( @@ -907,7 +839,7 @@ fn activation_snapshot_and_python_attestation_enforce_exact_directory_depth_boun std::fs::create_dir_all(&plugin_dir).unwrap(); let manifest_path = write_native_dynamic_manifest(&plugin_dir, "acme.deep-closure"); let mut deep_plugin_path = plugin_dir.clone(); - for _ in 1..MAX_SNAPSHOT_DEPTH { + for _ in 0..MAX_SNAPSHOT_DEPTH - 2 { deep_plugin_path.push("d"); } std::fs::create_dir_all(&deep_plugin_path).unwrap(); @@ -922,15 +854,9 @@ fn activation_snapshot_and_python_attestation_enforce_exact_directory_depth_boun ) .unwrap(); - deep_plugin_path.push("too-deep"); + deep_plugin_path.push("snapshot-too-deep"); std::fs::create_dir(&deep_plugin_path).unwrap(); - let error = - dynamic_plugin_runtime_closure_digest(manifest_path.to_string_lossy().as_ref(), None) - .unwrap_err() - .to_string(); - assert!(error.contains("traversal depth"), "{error}"); - let error = DynamicPluginActivationSnapshot::create( manifest_path.to_string_lossy().as_ref(), "acme.deep-closure", @@ -942,6 +868,15 @@ fn activation_snapshot_and_python_attestation_enforce_exact_directory_depth_boun .to_string(); assert!(error.contains("traversal depth"), "{error}"); + dynamic_plugin_runtime_closure_digest(manifest_path.to_string_lossy().as_ref(), None).unwrap(); + deep_plugin_path.push("closure-too-deep"); + std::fs::create_dir(&deep_plugin_path).unwrap(); + let error = + dynamic_plugin_runtime_closure_digest(manifest_path.to_string_lossy().as_ref(), None) + .unwrap_err() + .to_string(); + assert!(error.contains("traversal depth"), "{error}"); + let environment_path = temp.path().join("deep-environment"); let mut deep_environment_path = environment_path.clone(); for _ in 1..environment::MAX_ENVIRONMENT_DEPTH { @@ -1069,6 +1004,7 @@ fn activation_snapshot_rejects_identity_mismatch_missing_files_and_policy_denial let blocked = crate::plugins::policy::DynamicPluginHostPolicy { defaults: crate::plugins::policy::DynamicPluginHostPolicyEffect { allowed: Some(false), + startup: Some(nemo_relay::plugin::dynamic::DynamicPluginStartupClass::Required), ..crate::plugins::policy::DynamicPluginHostPolicyEffect::default() }, ..crate::plugins::policy::DynamicPluginHostPolicy::default() @@ -1125,12 +1061,10 @@ fn activation_snapshot_copies_declared_signature_into_stable_identity() { ) .unwrap(); - let signature_logical = snapshot - .identity_files - .keys() - .find(|path| path.ends_with("plugin.py.sig")) - .cloned() - .expect("signature is part of the stable snapshot identity"); + let signature_logical = Path::new(snapshot.original_manifest_ref()) + .parent() + .unwrap() + .join("plugin.py.sig"); assert!(snapshot.identity_file(&signature_logical).is_some()); assert_eq!( snapshot.closure_digest(), @@ -1189,148 +1123,12 @@ fn python_snapshot_contract_requires_environment_and_trusted_source_digest() { ); } -#[cfg(unix)] -#[test] -fn snapshot_directory_copy_preserves_python_launcher_and_rejects_special_entries() { - use std::ffi::CString; - use std::os::unix::fs::symlink; - - let temp = tempfile::tempdir().unwrap(); - let source = temp.path().join("environment"); - let bin = source.join("bin"); - std::fs::create_dir_all(&bin).unwrap(); - let interpreter = temp.path().join("managed-python"); - std::fs::write(&interpreter, b"python").unwrap(); - symlink(&interpreter, bin.join("python3.11")).unwrap(); - let destination = temp.path().join("snapshot"); - let mut copied = HashMap::new(); - let mut budget = SnapshotBudget::default(); - - copy_snapshot_directory( - &source, - &destination, - &mut copied, - &mut budget, - false, - &mut Vec::new(), - ) - .unwrap(); - - assert!( - std::fs::symlink_metadata(destination.join("bin/python3.11")) - .unwrap() - .file_type() - .is_symlink() - ); - assert!(!is_python_venv_launcher(Path::new("/"))); - - let fifo_source = temp.path().join("fifo-source"); - std::fs::create_dir(&fifo_source).unwrap(); - let fifo = fifo_source.join("worker.pipe"); - let fifo_c = CString::new(fifo.as_os_str().as_encoded_bytes()).unwrap(); - // SAFETY: `fifo_c` is a valid NUL-terminated path and the mode contains only permission bits. - assert_eq!(unsafe { libc::mkfifo(fifo_c.as_ptr(), 0o600) }, 0); - let special_error = copy_snapshot_directory( - &fifo_source, - &temp.path().join("fifo-snapshot"), - &mut HashMap::new(), - &mut SnapshotBudget::default(), - false, - &mut Vec::new(), - ) - .unwrap_err() - .to_string(); - assert!( - special_error.contains("regular file or directory"), - "{special_error}" - ); - - let regular = temp.path().join("regular"); - std::fs::write(®ular, b"regular").unwrap(); - let destination_directory = temp.path().join("destination-directory"); - std::fs::create_dir(&destination_directory).unwrap(); - let write_error = copy_snapshot_regular_file( - ®ular, - &destination_directory, - &mut HashMap::new(), - &mut SnapshotBudget::default(), - "fixture", - ) - .unwrap_err() - .to_string(); - assert!(write_error.contains("failed to write dynamic plugin snapshot file")); -} - -#[cfg(unix)] #[test] -fn snapshot_directory_walk_rejects_missing_cycles_dangling_links_and_depth() { - use std::os::unix::fs::symlink; - +fn runtime_closure_directory_walk_rejects_cycles() { let temp = tempfile::tempdir().unwrap(); - let missing = temp.path().join("missing"); - let normalization_error = copy_snapshot_directory( - &missing, - &temp.path().join("destination"), - &mut HashMap::new(), - &mut SnapshotBudget::default(), - false, - &mut Vec::new(), - ) - .unwrap_err() - .to_string(); - assert!( - normalization_error.contains("failed to normalize"), - "{normalization_error}" - ); - let source = temp.path().join("source"); std::fs::create_dir(&source).unwrap(); let canonical = source.canonicalize().unwrap(); - let cycle_error = copy_snapshot_directory_contents( - &source, - &temp.path().join("cycle-destination"), - &mut HashMap::new(), - &mut SnapshotBudget::default(), - false, - &mut vec![canonical.clone()], - ) - .unwrap_err() - .to_string(); - assert!(cycle_error.contains("symlink cycle"), "{cycle_error}"); - - let destination_file = temp.path().join("destination-file"); - std::fs::write(&destination_file, b"file").unwrap(); - let destination_error = copy_snapshot_directory_contents( - &source, - &destination_file, - &mut HashMap::new(), - &mut SnapshotBudget::default(), - false, - &mut Vec::new(), - ) - .unwrap_err() - .to_string(); - assert!( - destination_error.contains("failed to create"), - "{destination_error}" - ); - - symlink(temp.path().join("absent-target"), source.join("dangling")).unwrap(); - let dangling_error = copy_snapshot_directory( - &source, - &temp.path().join("dangling-destination"), - &mut HashMap::new(), - &mut SnapshotBudget::default(), - false, - &mut Vec::new(), - ) - .unwrap_err() - .to_string(); - assert!( - dangling_error.contains("failed to resolve"), - "{dangling_error}" - ); - let closure_cycle = collect_runtime_closure_directory_contents( &source, Path::new("runtime"), @@ -1341,22 +1139,11 @@ fn snapshot_directory_walk_rejects_missing_cycles_dangling_links_and_depth() { .unwrap_err() .to_string(); assert!(closure_cycle.contains("symlink cycle"), "{closure_cycle}"); - - let depth_error = collect_snapshot_files( - &source, - &source, - &mut Vec::new(), - Some(MAX_SNAPSHOT_DEPTH), - &mut 0, - ) - .unwrap_err() - .to_string(); - assert!(depth_error.contains("traversal depth"), "{depth_error}"); } #[cfg(unix)] #[test] -fn snapshot_file_and_closure_helpers_cover_external_and_invalid_sources() { +fn runtime_closure_helpers_cover_external_and_invalid_sources() { use std::ffi::CString; use std::os::unix::fs::symlink; @@ -1365,54 +1152,8 @@ fn snapshot_file_and_closure_helpers_cover_external_and_invalid_sources() { std::fs::create_dir(&plugin_dir).unwrap(); let manifest = plugin_dir.join("relay-plugin.toml"); std::fs::write(&manifest, b"fixture").unwrap(); - let root = temp.path().join("snapshot"); - std::fs::create_dir(&root).unwrap(); - - let missing_error = copy_snapshot_file( - &root, - &manifest, - "missing.bin", - "artifact", - &mut HashMap::new(), - &mut SnapshotBudget::default(), - ) - .unwrap_err() - .to_string(); - assert!( - missing_error.contains("failed to normalize"), - "{missing_error}" - ); - - let root_error = copy_snapshot_file( - &root, - &manifest, - "/", - "library", - &mut HashMap::new(), - &mut SnapshotBudget::default(), - ) - .unwrap_err() - .to_string(); - assert!( - root_error.contains("has no parent directory"), - "{root_error}" - ); - let external = temp.path().join("external-artifact.bin"); std::fs::write(&external, b"external artifact").unwrap(); - let (logical, canonical, copied) = copy_snapshot_file( - &root, - &manifest, - external.to_string_lossy().as_ref(), - "artifact", - &mut HashMap::new(), - &mut SnapshotBudget::default(), - ) - .unwrap(); - assert_eq!(logical, external); - assert_eq!(canonical, external.canonicalize().unwrap()); - assert_eq!(std::fs::read(copied).unwrap(), b"external artifact"); - let mut closure = RuntimeClosureSources::default(); let closure_missing = collect_declared_runtime_closure_file(&manifest, "missing.bin", "artifact", &mut closure) @@ -1508,18 +1249,6 @@ fn snapshot_file_and_closure_helpers_cover_external_and_invalid_sources() { fifo_error.contains("regular file or directory"), "{fifo_error}" ); - - let one_file = temp.path().join("one-file"); - std::fs::create_dir(&one_file).unwrap(); - std::fs::write(one_file.join("entry"), b"entry").unwrap(); - let mut entries = MAX_SNAPSHOT_FILES; - let entry_error = - collect_snapshot_files(&one_file, &one_file, &mut Vec::new(), None, &mut entries) - .unwrap_err() - .to_string(); - assert!(entry_error.contains("verification budget"), "{entry_error}"); - - make_snapshot_removable(&temp.path().join("already-removed")); } #[test] @@ -1768,6 +1497,18 @@ fn add_registers_dynamic_plugin_in_project_plugins_toml() { let rendered = std::fs::read_to_string(&plugins_toml).unwrap(); assert!(rendered.contains("[[plugins.dynamic]]")); assert!(rendered.contains("relay-plugin.toml")); + let scopes = load_scoped_registries(None).unwrap(); + let added = find_record_by_id(&scopes, "acme.guardrail") + .unwrap() + .expect("add should persist lifecycle state immediately"); + let declaration_source = scopes[added.scope_index] + .registry + .declaration_source("acme.guardrail") + .expect("the declaration source should be persisted"); + assert_eq!( + PathBuf::from(declaration_source), + nemo_relay_plugin_host_config::pin_plugin_config_path(&plugins_toml).unwrap() + ); let resolved = resolve_plugins_config(None).unwrap(); assert_eq!(resolved.dynamic_plugins.len(), 1); @@ -1915,9 +1656,9 @@ fn add_provisions_persists_and_removes_managed_python_environment() { .record .source .environment_ref - .as_deref() + .clone() .expect("managed environment should be persisted"); - let environment_path = PathBuf::from(environment_ref); + let environment_path = PathBuf::from(&environment_ref); assert_managed_environment_path(&environment_path); assert_eq!( added.record.status.validation.environment, @@ -1943,6 +1684,7 @@ fn add_provisions_persists_and_removes_managed_python_environment() { serde_json::json!(environment_ref) ); assert_python_environment_runner_calls(&runner.calls(), &environment_path, &plugin_dir); + drop(scopes); enable( PluginsEnableRequest { @@ -1954,7 +1696,10 @@ fn add_provisions_persists_and_removes_managed_python_environment() { let resolved = resolve_plugins_config(None).unwrap(); let active = active_dynamic_plugin_components(None, &resolved).unwrap(); assert_eq!(active.len(), 1); - assert_eq!(active[0].environment_ref.as_deref(), Some(environment_ref)); + assert_eq!( + active[0].environment_ref.as_deref(), + Some(environment_ref.as_str()) + ); let stale_marker = environment_path.join("stale-marker"); std::fs::write(&stale_marker, b"stale").unwrap(); @@ -1972,6 +1717,7 @@ fn add_provisions_persists_and_removes_managed_python_environment() { .expect("tombstone should remain"); assert!(removed.record.is_tombstoned()); assert_eq!(removed.record.source.environment_ref, None); + drop(scopes); add_with_environment_runner( PluginsAddRequest { @@ -1988,6 +1734,8 @@ fn add_provisions_persists_and_removes_managed_python_environment() { fn assert_managed_environment_path(environment_path: &Path) { assert!(environment_path.is_absolute()); + #[cfg(windows)] + assert!(!environment_path.to_string_lossy().starts_with(r"\\?\")); let expected_environment_name = Sha256::digest(b"acme.python") .iter() .map(|byte| format!("{byte:02x}")) @@ -2112,6 +1860,7 @@ fn enable_rejects_missing_managed_python_environment() { .clone() .unwrap(); std::fs::remove_dir_all(&environment_ref).unwrap(); + drop(scopes); let error = enable( PluginsEnableRequest { @@ -2126,7 +1875,7 @@ fn enable_rejects_missing_managed_python_environment() { .expect("environment failure should be structured"); assert_eq!(kind, PluginLifecycleFailureKind::Refused); assert_eq!(code, Some("environment_failed")); - assert!(message.contains("is unavailable")); + assert!(message.contains("failed to inspect lifecycle-managed Python environment")); let scopes = load_scoped_registries(None).unwrap(); let record = find_record_by_id(&scopes, "acme.python-missing") .unwrap() @@ -2163,7 +1912,7 @@ fn enable_rejects_python_environment_outside_managed_location() { let outside_python = environment::environment_python_path(&outside); std::fs::create_dir_all(outside_python.parent().unwrap()).unwrap(); std::fs::write(&outside_python, b"not managed by Relay").unwrap(); - let mut scopes = load_scoped_registries(None).unwrap(); + let mut scopes = load_scoped_registries_for_update(None, None).unwrap(); let scope = scopes .iter_mut() .find(|scope| scope.registry.get("acme.python-outside").is_some()) @@ -2177,6 +1926,7 @@ fn enable_rejects_python_environment_outside_managed_location() { ) .unwrap(); scope.save().unwrap(); + drop(scopes); let error = enable( PluginsEnableRequest { @@ -2191,7 +1941,7 @@ fn enable_rejects_python_environment_outside_managed_location() { .expect("environment refusal should be structured"); assert_eq!(kind, PluginLifecycleFailureKind::Refused); assert_eq!(code, Some("environment_failed")); - assert!(message.contains("is unavailable")); + assert!(message.contains("is not its lifecycle-managed environment")); assert!(outside.exists()); let scopes = load_scoped_registries(None).unwrap(); let record = find_record_by_id(&scopes, "acme.python-outside") @@ -2371,7 +2121,7 @@ fn remove_can_retry_after_guarded_environment_cleanup_failure() { ) .unwrap(); - let mut scopes = load_scoped_registries(None).unwrap(); + let mut scopes = load_scoped_registries_for_update(None, None).unwrap(); let scope = scopes .iter_mut() .find(|scope| scope.registry.get("acme.python-retry").is_some()) @@ -2395,6 +2145,7 @@ fn remove_can_retry_after_guarded_environment_cleanup_failure() { ) .unwrap(); scope.save().unwrap(); + drop(scopes); let error = remove( PluginsRemoveRequest { @@ -2405,7 +2156,7 @@ fn remove_can_retry_after_guarded_environment_cleanup_failure() { .expect_err("guarded cleanup should preserve unmanaged paths"); assert!(error.to_string().contains("refusing to delete")); assert!(outside.exists()); - let mut scopes = load_scoped_registries(None).unwrap(); + let mut scopes = load_scoped_registries_for_update(None, None).unwrap(); let scope = scopes .iter_mut() .find(|scope| scope.registry.get("acme.python-retry").is_some()) @@ -2426,6 +2177,7 @@ fn remove_can_retry_after_guarded_environment_cleanup_failure() { ) .unwrap(); scope.save().unwrap(); + drop(scopes); remove( PluginsRemoveRequest { @@ -2530,7 +2282,7 @@ fn active_dynamic_plugin_components_accept_enabled_worker_records() { } #[test] -fn active_dynamic_plugin_components_accept_worker_records_without_manifest_ref() { +fn active_dynamic_plugin_components_refresh_worker_records_without_manifest_ref() { let temp = tempfile::tempdir().unwrap(); let _env = EnvScope::hermetic(&temp); let _cwd = CurrentDirGuard::enter(temp.path()); @@ -2555,7 +2307,7 @@ fn active_dynamic_plugin_components_accept_worker_records_without_manifest_ref() ) .unwrap(); - let mut scopes = load_scoped_registries(server.config.as_ref()).unwrap(); + let mut scopes = load_scoped_registries_for_update(server.config.as_ref(), None).unwrap(); let scope = scopes .iter_mut() .find(|scope| scope.registry.get("acme.worker").is_some()) @@ -2567,16 +2319,24 @@ fn active_dynamic_plugin_components_accept_worker_records_without_manifest_ref() .expect("worker record should exist") .source .manifest_ref = None; - scope.registry = nemo_relay::plugin::dynamic::DynamicPluginRegistry::from_records(records) - .expect("registry should accept worker without manifest_ref"); + scope.registry.replace_registry( + nemo_relay::plugin::dynamic::DynamicPluginRegistry::from_records(records) + .expect("registry should accept worker without manifest_ref"), + ); scope.save().unwrap(); + drop(scopes); let resolved = resolve_plugins_config(None).unwrap(); let active = active_dynamic_plugin_components(None, &resolved).unwrap(); assert_eq!(active.len(), 1); assert_eq!(active[0].plugin_id, "acme.worker"); assert_eq!(active[0].kind, DynamicPluginKind::Worker); - assert_eq!(active[0].manifest_ref, None); + assert!( + active[0] + .manifest_ref + .as_deref() + .is_some_and(|manifest_ref| manifest_ref.contains("relay-plugin.toml")) + ); } #[test] @@ -2609,6 +2369,102 @@ fn add_rejects_duplicate_dynamic_plugin_ids() { assert!(error.contains("already registered")); } +#[test] +fn read_only_lifecycle_snapshot_does_not_block_a_later_update() { + let temp = tempfile::tempdir().unwrap(); + let _env = EnvScope::hermetic(&temp); + let _cwd = CurrentDirGuard::enter(temp.path()); + let plugin_dir = temp.path().join("plugins").join("acme"); + std::fs::create_dir_all(&plugin_dir).unwrap(); + write_dynamic_manifest(&plugin_dir, "acme.read-only-snapshot"); + let server = GatewayOverrides::default(); + + add( + PluginsAddRequest { + scope: ConfigurationScope::Project, + path: plugin_dir, + }, + &server, + ) + .unwrap(); + + let read_snapshot = load_scoped_registries(None).unwrap(); + let stale_record = find_record_by_id(&read_snapshot, "acme.read-only-snapshot") + .unwrap() + .expect("registered plugin should be present in the read snapshot"); + assert!(!stale_record.record.spec.enabled); + + enable( + PluginsEnableRequest { + id: "acme.read-only-snapshot".into(), + }, + &server, + ) + .unwrap(); + + assert!( + !find_record_by_id(&read_snapshot, "acme.read-only-snapshot") + .unwrap() + .unwrap() + .record + .spec + .enabled, + "a read-only snapshot should remain a stale value without retaining the update lock" + ); + let current = load_scoped_registries(None).unwrap(); + assert!( + find_record_by_id(¤t, "acme.read-only-snapshot") + .unwrap() + .unwrap() + .record + .spec + .enabled + ); +} + +#[test] +fn remove_hydrates_an_unregistered_declaration_without_relocking_itself() { + let temp = tempfile::tempdir().unwrap(); + let _env = EnvScope::hermetic(&temp); + let _cwd = CurrentDirGuard::enter(temp.path()); + let plugin_dir = temp.path().join("plugins").join("acme"); + let config_dir = temp.path().join(".nemo-relay"); + std::fs::create_dir_all(&plugin_dir).unwrap(); + std::fs::create_dir_all(&config_dir).unwrap(); + let manifest = write_dynamic_manifest(&plugin_dir, "acme.remove-hydrated"); + let plugins_toml = config_dir.join("plugins.toml"); + std::fs::write( + &plugins_toml, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + manifest.to_string_lossy() + ), + ) + .unwrap(); + + remove( + PluginsRemoveRequest { + id: "acme.remove-hydrated".into(), + }, + &GatewayOverrides::default(), + ) + .unwrap(); + + assert!( + !std::fs::read_to_string(&plugins_toml) + .unwrap() + .contains("plugins.dynamic") + ); + let scopes = load_scoped_registries(None).unwrap(); + assert!( + find_record_by_id(&scopes, "acme.remove-hydrated") + .unwrap() + .expect("hydrated record should remain as a tombstone") + .record + .is_tombstoned() + ); +} + #[test] fn add_rejects_scope_flags_when_explicit_config_is_set() { let temp = tempfile::tempdir().unwrap(); @@ -2845,6 +2701,7 @@ fn validate_renders_summary_for_path_and_id_targets() { .to_string(); assert!(id_summary.contains("host_config: absent")); assert!(id_summary.contains("desired.enabled: false")); + drop(scopes); let missing_validate = validate( PluginsValidateRequest { @@ -2911,6 +2768,7 @@ fn enable_disable_and_remove_persist_lifecycle_state() { .unwrap() .expect("enabled record"); assert!(enabled.record.spec.enabled); + drop(scopes); disable( PluginsDisableRequest { @@ -2925,6 +2783,7 @@ fn enable_disable_and_remove_persist_lifecycle_state() { .unwrap() .expect("disabled record"); assert!(!disabled.record.spec.enabled); + drop(scopes); remove( PluginsRemoveRequest { @@ -2950,6 +2809,7 @@ fn enable_disable_and_remove_persist_lifecycle_state() { .to_string(); assert!(all_list.contains("acme.guardrail")); assert!(all_list.contains("tombstoned")); + drop(scopes); let error = enable( PluginsEnableRequest { @@ -3010,8 +2870,14 @@ fn add_with_explicit_config_uses_sibling_plugins_and_state_files() { .unwrap() .expect("explicit-scope record"); assert_eq!(entry.scope.to_string(), "explicit"); - assert_eq!(entry.plugins_toml_path, plugins_toml); - assert_eq!(entry.state_path, state_path); + assert_eq!( + entry.plugins_toml_path.canonicalize().unwrap(), + plugins_toml.canonicalize().unwrap() + ); + assert_eq!( + entry.state_path.canonicalize().unwrap(), + state_path.canonicalize().unwrap() + ); } #[test] @@ -3083,13 +2949,162 @@ fn explicit_plugin_path_drives_plugin_command_lifecycle_scope() { list(PluginsListRequest::default(), &server).unwrap(); - let scopes = load_scoped_registries(Some(&plugin_config_path)).unwrap(); + let explicit_plugin_config = lifecycle_plugin_config_path(&server); + let scopes = load_scoped_registries(explicit_plugin_config.as_ref()).unwrap(); let entry = find_record_by_id(&scopes, "acme.explicit-plugin-path") .unwrap() .expect("explicit plugin-path record"); assert_eq!(entry.scope, RegistryScope::Explicit); - assert_eq!(entry.plugins_toml_path, plugin_config_path); - assert_eq!(entry.state_path, config_dir.join(".dynamic-plugins.json")); + let expected_plugins_toml = + nemo_relay_plugin_host_config::pin_plugin_config_path(&plugin_config_path).unwrap(); + assert_eq!(entry.plugins_toml_path, expected_plugins_toml); + assert_eq!( + entry.state_path, + nemo_relay_plugin_host_config::sibling_lifecycle_state_path(&expected_plugins_toml) + ); + assert!(entry.state_path.exists()); +} + +#[test] +fn same_directory_plugin_sources_share_state_without_losing_physical_ownership() { + let temp = tempfile::tempdir().unwrap(); + let _env = EnvScope::hermetic(&temp); + let _cwd = CurrentDirGuard::enter(temp.path()); + let custom_plugin_dir = temp.path().join("plugins").join("custom"); + let project_plugin_dir = temp.path().join("plugins").join("project"); + let config_dir = temp.path().join(".nemo-relay"); + std::fs::create_dir_all(&custom_plugin_dir).unwrap(); + std::fs::create_dir_all(&project_plugin_dir).unwrap(); + std::fs::create_dir_all(&config_dir).unwrap(); + let custom_manifest = write_dynamic_manifest(&custom_plugin_dir, "acme.custom-source"); + let project_manifest = write_dynamic_manifest(&project_plugin_dir, "acme.project-source"); + let custom_plugins_toml = config_dir.join("custom.toml"); + let project_plugins_toml = config_dir.join("plugins.toml"); + std::fs::write( + &custom_plugins_toml, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + custom_manifest.to_string_lossy() + ), + ) + .unwrap(); + std::fs::write( + &project_plugins_toml, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + project_manifest.to_string_lossy() + ), + ) + .unwrap(); + let server = GatewayOverrides { + plugin_config_path: Some(custom_plugins_toml.clone()), + ..GatewayOverrides::default() + }; + + let resolved = resolve_plugins_config_with_path(None, server.plugin_config_path.as_ref()) + .expect("both same-directory sources should resolve"); + assert_eq!(resolved.dynamic_plugins.len(), 2); + let explicit_plugin_config = lifecycle_plugin_config_path(&server); + let scopes = load_and_hydrate_scopes(explicit_plugin_config.as_ref(), &resolved).unwrap(); + let shared_state_path = config_dir.join(".dynamic-plugins.json"); + let shared_state_path = shared_state_path.canonicalize().unwrap(); + assert_eq!( + scopes + .iter() + .filter(|scope| { + scope.state_path.canonicalize().ok().as_ref() == Some(&shared_state_path) + }) + .count(), + 1, + "same-directory sources must share one lifecycle state and lock" + ); + let custom = find_record_by_id(&scopes, "acme.custom-source") + .unwrap() + .expect("custom-source record"); + assert_eq!(custom.scope, RegistryScope::Explicit); + assert_eq!( + custom.plugins_toml_path.canonicalize().unwrap(), + custom_plugins_toml.canonicalize().unwrap() + ); + let project = find_record_by_id(&scopes, "acme.project-source") + .unwrap() + .expect("project-source record"); + assert_eq!(project.scope, RegistryScope::Project); + assert_eq!( + project.plugins_toml_path.canonicalize().unwrap(), + project_plugins_toml.canonicalize().unwrap() + ); + drop(scopes); + + enable( + PluginsEnableRequest { + id: "acme.custom-source".into(), + }, + &server, + ) + .unwrap(); + enable( + PluginsEnableRequest { + id: "acme.project-source".into(), + }, + &server, + ) + .unwrap(); + let scopes = load_scoped_registries(explicit_plugin_config.as_ref()).unwrap(); + assert!( + find_record_by_id(&scopes, "acme.custom-source") + .unwrap() + .unwrap() + .record + .spec + .enabled + ); + assert!( + find_record_by_id(&scopes, "acme.project-source") + .unwrap() + .unwrap() + .record + .spec + .enabled + ); +} + +#[test] +fn activation_uses_the_sources_captured_during_configuration_resolution() { + let temp = tempfile::tempdir().unwrap(); + let _env = EnvScope::hermetic(&temp); + let project = temp.path().join("project"); + let other = temp.path().join("other"); + let plugin_dir = temp.path().join("plugins").join("acme"); + std::fs::create_dir_all(&project).unwrap(); + std::fs::create_dir_all(&other).unwrap(); + std::fs::create_dir_all(&plugin_dir).unwrap(); + write_dynamic_manifest(&plugin_dir, "acme.captured-sources"); + let _project = CurrentDirGuard::enter(&project); + let server = GatewayOverrides::default(); + + add( + PluginsAddRequest { + scope: ConfigurationScope::Project, + path: plugin_dir, + }, + &server, + ) + .unwrap(); + enable( + PluginsEnableRequest { + id: "acme.captured-sources".into(), + }, + &server, + ) + .unwrap(); + let resolved = resolve_plugins_config(None).unwrap(); + assert!(!resolved.plugin_selected_sources.is_empty()); + + let _other = CurrentDirGuard::enter(&other); + let active = active_dynamic_plugin_components(None, &resolved).unwrap(); + assert_eq!(active.len(), 1); + assert_eq!(active[0].plugin_id, "acme.captured-sources"); } #[test] @@ -3130,6 +3145,190 @@ fn hydrate_bootstraps_registry_records_from_existing_dynamic_plugin_refs() { ); } +#[test] +fn hydrate_refreshes_same_id_declaration_before_enable_validation() { + let temp = tempfile::tempdir().unwrap(); + let _env = EnvScope::hermetic(&temp); + let _cwd = CurrentDirGuard::enter(temp.path()); + let first_dir = temp.path().join("plugins").join("first"); + let second_dir = temp.path().join("plugins").join("second"); + let config_dir = temp.path().join(".nemo-relay"); + std::fs::create_dir_all(&first_dir).unwrap(); + std::fs::create_dir_all(&second_dir).unwrap(); + std::fs::create_dir_all(&config_dir).unwrap(); + let first_manifest = write_dynamic_manifest(&first_dir, "acme.redeclared"); + let second_manifest = write_native_dynamic_manifest(&second_dir, "acme.redeclared"); + let plugins_toml = config_dir.join("plugins.toml"); + std::fs::write( + &plugins_toml, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + first_manifest.to_string_lossy() + ), + ) + .unwrap(); + + let resolved = resolve_plugins_config(None).unwrap(); + load_and_hydrate_scopes(None, &resolved).unwrap(); + let state_path = config_dir.join(".dynamic-plugins.json"); + let mut state: serde_json::Value = + serde_json::from_slice(&std::fs::read(&state_path).unwrap()).unwrap(); + state["records"][0]["spec"]["config_ref"] = + serde_json::json!("higher-level.plugins.acme-redeclared"); + let mut rendered_state = serde_json::to_vec_pretty(&state).unwrap(); + rendered_state.push(b'\n'); + std::fs::write(&state_path, rendered_state).unwrap(); + + std::fs::write( + &plugins_toml, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + second_manifest.to_string_lossy() + ), + ) + .unwrap(); + std::fs::write( + first_dir.join("plugin.py"), + b"tampered after declaration moved", + ) + .unwrap(); + + enable( + PluginsEnableRequest { + id: "acme.redeclared".into(), + }, + &GatewayOverrides::default(), + ) + .unwrap(); + + let scopes = load_scoped_registries(None).unwrap(); + let entry = find_record_by_id(&scopes, "acme.redeclared") + .unwrap() + .expect("redeclared plugin should remain registered"); + let canonical_second_manifest = second_manifest.canonicalize().unwrap(); + assert!(entry.record.spec.enabled); + assert_eq!( + entry.record.spec.config_ref.as_deref(), + Some("higher-level.plugins.acme-redeclared") + ); + assert_eq!(entry.record.metadata.kind, DynamicPluginKind::RustDynamic); + assert_eq!( + entry.record.source.manifest_ref.as_deref(), + Some(canonical_second_manifest.to_string_lossy().as_ref()) + ); + assert_eq!( + entry.record.source.artifact_ref.as_deref(), + Some("libfixture_native.so") + ); + match &entry.record.compatibility { + DynamicPluginCompatibility::RustDynamic(compatibility) => { + assert_eq!(compatibility.relay, "0.5"); + assert_eq!(compatibility.native_api, "1"); + } + other => panic!("expected refreshed native compatibility, got {other:?}"), + } + match &entry.record.load { + DynamicPluginLoadContract::RustDynamic(load) => { + assert_eq!(load.library, "libfixture_native.so"); + assert_eq!(load.symbol, "nemo_relay_fixture_native_plugin"); + } + other => panic!("expected refreshed native load contract, got {other:?}"), + } +} + +#[test] +fn hydrate_ignores_a_foreign_tombstone_and_creates_a_fresh_disabled_record() { + let temp = tempfile::tempdir().unwrap(); + let _env = EnvScope::hermetic(&temp); + let _cwd = CurrentDirGuard::enter(temp.path()); + let plugin_dir = temp.path().join("plugins").join("moved"); + let project_config_dir = temp.path().join(".nemo-relay"); + std::fs::create_dir_all(&plugin_dir).unwrap(); + std::fs::create_dir_all(&project_config_dir).unwrap(); + let manifest = write_dynamic_manifest(&plugin_dir, "acme.moved-scope"); + let server = GatewayOverrides::default(); + + add( + PluginsAddRequest { + scope: ConfigurationScope::User, + path: plugin_dir, + }, + &server, + ) + .unwrap(); + enable( + PluginsEnableRequest { + id: "acme.moved-scope".into(), + }, + &server, + ) + .unwrap(); + + std::fs::remove_file( + temp.path() + .join("xdg") + .join("nemo-relay") + .join("plugins.toml"), + ) + .unwrap(); + let project_plugins_toml = project_config_dir.join("plugins.toml"); + std::fs::write( + &project_plugins_toml, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + manifest.to_string_lossy() + ), + ) + .unwrap(); + let resolved = resolve_plugins_config(None).unwrap(); + let error = load_and_hydrate_scopes(None, &resolved) + .expect_err("a live record owned by another source must fail closed") + .to_string(); + assert!(error.contains("multiple lifecycle scopes"), "{error}"); + let scopes = load_scoped_registries(None).unwrap(); + let still_enabled = find_record_by_id(&scopes, "acme.moved-scope") + .unwrap() + .expect("the original live record must remain unchanged"); + assert_eq!(still_enabled.scope, RegistryScope::User); + assert!(still_enabled.record.spec.enabled); + drop(scopes); + + remove( + PluginsRemoveRequest { + id: "acme.moved-scope".into(), + }, + &server, + ) + .unwrap(); + let resolved = resolve_plugins_config(None).unwrap(); + let scopes = load_and_hydrate_scopes(None, &resolved).unwrap(); + let records = collect_records(&scopes, true) + .into_iter() + .filter(|entry| entry.record.metadata.id == "acme.moved-scope") + .collect::>(); + assert_eq!(records.len(), 2); + + let old_user_record = records + .iter() + .find(|entry| entry.scope == RegistryScope::User) + .unwrap_or_else(|| panic!("the old user tombstone should remain: {records:#?}")); + assert!(old_user_record.record.is_tombstoned()); + let project_record = records + .iter() + .find(|entry| entry.scope == RegistryScope::Project) + .expect("the declaring project scope should receive a record"); + assert!(project_record.record.spec.present); + assert!(!project_record.record.spec.enabled); + let declaration_source = scopes[project_record.scope_index] + .registry + .declaration_source("acme.moved-scope") + .expect("the project declaration source should be persisted"); + assert_eq!( + PathBuf::from(declaration_source), + nemo_relay_plugin_host_config::pin_plugin_config_path(&project_plugins_toml).unwrap() + ); +} + #[test] fn manually_configured_python_worker_cannot_enable_without_lifecycle_add() { let temp = tempfile::tempdir().unwrap(); @@ -3172,6 +3371,7 @@ fn manually_configured_python_worker_cannot_enable_without_lifecycle_add() { } .to_string(); assert!(summary.contains("runtime environment is unavailable")); + drop(scopes); let error = enable( PluginsEnableRequest { @@ -4285,10 +4485,10 @@ fn lifecycle_helpers_cover_environment_manifest_scope_and_restore_paths() { let temp = tempfile::tempdir().unwrap(); let plugin_id = "acme.lifecycle-helpers"; assert_eq!( - environment_last_error(plugin_id, DynamicPluginCheckState::Valid, None), + environment_last_error(plugin_id, DynamicPluginCheckState::Valid, None, None), None ); - let missing = environment_last_error(plugin_id, DynamicPluginCheckState::Invalid, None) + let missing = environment_last_error(plugin_id, DynamicPluginCheckState::Invalid, None, None) .expect("invalid environment should produce a diagnostic"); assert_eq!(missing.code, "environment_failed"); assert!(missing.message.contains("has no lifecycle-managed")); @@ -4296,6 +4496,7 @@ fn lifecycle_helpers_cover_environment_manifest_scope_and_restore_paths() { plugin_id, DynamicPluginCheckState::Invalid, Some("managed/python"), + None, ) .expect("invalid referenced environment should produce a diagnostic"); assert!( @@ -4303,6 +4504,17 @@ fn lifecycle_helpers_cover_environment_manifest_scope_and_restore_paths() { .message .contains("managed/python is unavailable") ); + let detailed = environment_last_error( + plugin_id, + DynamicPluginCheckState::Invalid, + Some("managed/python"), + Some("managed Python environment changed after provisioning".into()), + ) + .expect("detailed validation failures should be retained"); + assert_eq!( + detailed.message, + "managed Python environment changed after provisioning" + ); let mut scopes = Vec::new(); let plugins_path = temp.path().join("plugins.toml"); diff --git a/crates/core/src/plugin.rs b/crates/core/src/plugin.rs index 789d7176e..a6c6ff276 100644 --- a/crates/core/src/plugin.rs +++ b/crates/core/src/plugin.rs @@ -1568,7 +1568,12 @@ async fn initialize_plugins_exact_inner( .await { Ok(registrations) => { - store_active_plugin_configuration(config, report.clone(), registrations)?; + store_active_plugin_configuration( + config, + report.clone(), + registrations, + rollback_failures.as_ref(), + )?; log::info!( target: "nemo_relay.plugin", event = "plugin_configuration_replaced", @@ -1588,6 +1593,7 @@ async fn initialize_plugins_exact_inner( previous_state.config, previous_state.report, registrations, + rollback_failures.as_ref(), )?; log::warn!( target: "nemo_relay.plugin", @@ -1612,8 +1618,14 @@ async fn initialize_plugins_exact_inner( } } else { let registrations = - initialize_plugin_components_catching_panics(config.clone(), rollback_failures).await?; - store_active_plugin_configuration(config, report.clone(), registrations)?; + initialize_plugin_components_catching_panics(config.clone(), rollback_failures.clone()) + .await?; + store_active_plugin_configuration( + config, + report.clone(), + registrations, + rollback_failures.as_ref(), + )?; log::info!( target: "nemo_relay.plugin", event = "plugin_configuration_activated", @@ -2477,11 +2489,19 @@ fn record_rollback_failures( fn store_active_plugin_configuration( config: PluginConfig, report: ConfigReport, - registrations: Vec, + mut registrations: Vec, + rollback_failures: Option<&Arc>>>, ) -> Result<()> { - let mut guard = ACTIVE_PLUGIN_CONFIGURATION.lock().map_err(|err| { - PluginError::Internal(format!("active plugin configuration lock poisoned: {err}")) - })?; + let mut guard = match ACTIVE_PLUGIN_CONFIGURATION.lock() { + Ok(guard) => guard, + Err(err) => { + let error = format!("active plugin configuration lock poisoned: {err}"); + drop(err.into_inner()); + let errors = rollback_registrations_checked(&mut registrations); + record_rollback_failures(rollback_failures, errors); + return Err(PluginError::Internal(error)); + } + }; *guard = Some(ActivePluginConfiguration { config, report, diff --git a/crates/core/src/plugin/dynamic.rs b/crates/core/src/plugin/dynamic.rs index fd9ff3653..539e80666 100644 --- a/crates/core/src/plugin/dynamic.rs +++ b/crates/core/src/plugin/dynamic.rs @@ -43,6 +43,102 @@ pub(crate) struct DynamicPluginTeardownOutcome { pub(crate) safe_to_unload: bool, } +#[derive(Debug)] +pub(crate) struct DynamicPluginLoadFailure { + error: PluginError, + rollback: DynamicPluginTeardownOutcome, +} + +/// Retains partially constructed runtime state when control leaves through an +/// unwind. Normal error paths take the value and run the checked rollback +/// machinery instead. +pub(super) struct PanicRetentionGuard { + value: Option, +} + +impl PanicRetentionGuard { + pub(super) fn new(value: T) -> Self { + Self { value: Some(value) } + } + + pub(super) fn get(&self) -> &T { + self.value + .as_ref() + .expect("panic retention value accessed after transfer") + } + + pub(super) fn get_mut(&mut self) -> &mut T { + self.value + .as_mut() + .expect("panic retention value accessed after transfer") + } + + pub(super) fn take(&mut self) -> T { + self.value + .take() + .expect("panic retention value transferred more than once") + } +} + +impl Drop for PanicRetentionGuard { + fn drop(&mut self) { + if std::thread::panicking() + && let Some(value) = self.value.take() + { + std::mem::forget(value); + } + } +} + +impl DynamicPluginLoadFailure { + pub(crate) fn new(error: PluginError, rollback: DynamicPluginTeardownOutcome) -> Self { + Self { error, rollback } + } + + #[cfg(feature = "worker-grpc")] + pub(crate) fn merge_rollback(&mut self, rollback: DynamicPluginTeardownOutcome) { + self.rollback.merge(rollback); + } + + pub(crate) fn safe_to_unload(&self) -> bool { + self.rollback.safe_to_unload + } + + pub(crate) fn into_plugin_error(self) -> PluginError { + if self.rollback.errors.is_empty() { + return self.error; + } + + let retention = if self.rollback.safe_to_unload { + "all partially loaded runtimes were removed" + } else { + "a partially loaded runtime was retained because safe unloading could not be proven" + }; + PluginError::RegistrationFailed(format!( + "{}; activation rollback reported: {}; {retention}", + self.error, + self.rollback.errors.join("; ") + )) + } +} + +impl From for DynamicPluginLoadFailure { + fn from(error: PluginError) -> Self { + Self::new(error, DynamicPluginTeardownOutcome::success()) + } +} + +pub(crate) fn finish_partial_load_rollback( + activation: T, + error: PluginError, + rollback: DynamicPluginTeardownOutcome, +) -> DynamicPluginLoadFailure { + if !rollback.safe_to_unload { + std::mem::forget(activation); + } + DynamicPluginLoadFailure::new(error, rollback) +} + impl DynamicPluginTeardownOutcome { pub(crate) fn success() -> Self { Self { diff --git a/crates/core/src/plugin/dynamic/host.rs b/crates/core/src/plugin/dynamic/host.rs index 49eb57579..38b694a03 100644 --- a/crates/core/src/plugin/dynamic/host.rs +++ b/crates/core/src/plugin/dynamic/host.rs @@ -10,25 +10,27 @@ //! affinity; the lifecycle executor remains available for the process lifetime. use std::collections::HashSet; +#[cfg(test)] +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value as Json}; use crate::plugin::{ - ConfigReport, PluginComponentSpec, PluginConfig, PluginHostLease, Result, + ConfigDiagnostic, ConfigReport, PluginComponentSpec, PluginConfig, PluginHostLease, Result, acquire_plugin_host_lease, clear_plugin_configuration_for_host, ensure_builtin_plugins_registered, initialize_plugins_exact_for_host, resolve_plugin_config, run_owned_plugin_mutation, }; use super::{ - DynamicPluginKind, DynamicPluginTeardownOutcome, NativePluginActivation, NativePluginLoadSpec, - load_native_plugins, + DynamicPluginKind, DynamicPluginLoadFailure, DynamicPluginTeardownOutcome, + NativePluginActivation, NativePluginLoadSpec, load_native_plugins_with_resources, }; #[cfg(feature = "worker-grpc")] -use super::{WorkerPluginActivation, WorkerPluginLoadSpec, load_worker_plugins}; +use super::{WorkerPluginActivation, WorkerPluginLoadSpec, load_worker_plugins_with_resources}; /// One dynamic plugin component to load and activate in an embedding host. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -48,6 +50,41 @@ pub struct DynamicPluginActivationSpec { pub config: Map, } +/// Owns resources that must remain stable for one planned dynamic plugin. +/// +/// File-backed hosts use this contract to retain verified activation snapshots +/// alongside the native library or worker runtime that consumes them. +#[doc(hidden)] +pub trait DynamicPluginActivationResource: Send + Sync { + /// Verify that the retained resource is still safe to load. + fn verify(&self) -> Result<()>; +} + +/// One dynamic plugin and its retained file-backed activation resource. +#[doc(hidden)] +pub struct PlannedDynamicPluginActivation { + /// Dynamic plugin activation details resolved by the embedding host. + pub spec: DynamicPluginActivationSpec, + /// Resource retained for the complete runtime and callback lifetime. + pub resource: Arc, +} + +/// Fully resolved static and dynamic configuration for an owned plugin host. +#[doc(hidden)] +pub struct PluginHostActivationPlan { + /// Resolved static plugin configuration. + pub config: PluginConfig, + /// Enabled dynamic plugins and their retained activation resources. + pub dynamic_plugins: Vec, + /// Configuration diagnostics produced while resolving physical sources. + pub diagnostics: Vec, +} + +struct PreparedDynamicPluginActivation { + spec: DynamicPluginActivationSpec, + resource: Option>, +} + /// Owns one process-wide dynamic plugin configuration and its loaded runtimes. /// /// The activation keeps native libraries and worker processes alive until after @@ -59,9 +96,80 @@ pub struct PluginHostActivation { native: Option, #[cfg(feature = "worker-grpc")] worker: Option, + resource_anchors: Vec>, claim: Option, } +struct PluginHostActivationTransaction { + native: Option, + #[cfg(feature = "worker-grpc")] + worker: Option, + claim: Option, + resource_anchors: Vec>, +} + +impl PluginHostActivationTransaction { + fn new(claim: PluginHostLease, dynamic_plugins: &[PreparedDynamicPluginActivation]) -> Self { + Self { + native: None, + #[cfg(feature = "worker-grpc")] + worker: None, + claim: Some(claim), + resource_anchors: dynamic_plugins + .iter() + .filter_map(|plugin| plugin.resource.clone()) + .collect(), + } + } + + fn into_activation(mut self) -> PluginHostActivation { + PluginHostActivation { + active: true, + native: self.native.take(), + #[cfg(feature = "worker-grpc")] + worker: self.worker.take(), + resource_anchors: std::mem::take(&mut self.resource_anchors), + claim: self.claim.take(), + } + } + + fn retain_for_process_lifetime(&mut self) { + retain_loaded_runtimes( + &mut self.native, + #[cfg(feature = "worker-grpc")] + &mut self.worker, + ); + retain_plugin_host_claim(&mut self.claim); + let resources = std::mem::take(&mut self.resource_anchors); + if !resources.is_empty() { + std::mem::forget(resources); + } + } +} + +impl Drop for PluginHostActivationTransaction { + fn drop(&mut self) { + if std::thread::panicking() { + self.retain_for_process_lifetime(); + } + } +} + +struct PluginHostClearUnwindGuard<'a> { + activation: &'a mut PluginHostActivation, +} + +impl Drop for PluginHostClearUnwindGuard<'_> { + fn drop(&mut self) { + if std::thread::panicking() { + self.activation.retain_loaded_runtimes(); + } + } +} + +#[cfg(test)] +static PANIC_PLUGIN_HOST_CLEAR_AFTER_DEACTIVATION: AtomicBool = AtomicBool::new(false); + impl PluginHostActivation { /// Load dynamic plugins and activate them with `config` as one transaction. /// @@ -79,7 +187,12 @@ impl PluginHostActivation { { let dynamic_plugins = dynamic_plugins.into_iter().collect::>(); validate_dynamic_plugin_specs(&dynamic_plugins)?; - Self::activate_validated(config, dynamic_plugins, Vec::new()).await + Self::activate_validated( + config, + prepare_explicit_dynamic_plugins(dynamic_plugins), + Vec::new(), + ) + .await } /// Load dynamic plugins after layering `config` over discovered `plugins.toml` files. @@ -98,13 +211,44 @@ impl PluginHostActivation { let dynamic_plugins = dynamic_plugins.into_iter().collect::>(); validate_dynamic_plugin_specs(&dynamic_plugins)?; let resolved = resolve_plugin_config(config)?; - Self::activate_validated(resolved.config, dynamic_plugins, resolved.diagnostics).await + Self::activate_validated( + resolved.config, + prepare_explicit_dynamic_plugins(dynamic_plugins), + resolved.diagnostics, + ) + .await + } + + /// Activate a fully resolved file-backed plugin host plan. + /// + /// Unlike the explicit dynamic-spec entrypoints, a plan may contain no + /// dynamic plugins so one owner can manage a static-only file-backed + /// configuration through the same lifecycle. + #[doc(hidden)] + pub async fn activate_plan(plan: PluginHostActivationPlan) -> Result<(Self, ConfigReport)> { + validate_planned_dynamic_plugins(&plan.dynamic_plugins)?; + run_owned_plugin_mutation("file-backed plugin activation", move || async move { + let PluginHostActivationPlan { + config, + dynamic_plugins, + diagnostics, + } = plan; + let dynamic_plugins = dynamic_plugins + .into_iter() + .map(|plugin| PreparedDynamicPluginActivation { + spec: plugin.spec, + resource: Some(plugin.resource), + }) + .collect(); + Self::activate_inner(config, dynamic_plugins, diagnostics).await + }) + .await } async fn activate_validated( config: PluginConfig, - dynamic_plugins: Vec, - diagnostics: Vec, + dynamic_plugins: Vec, + diagnostics: Vec, ) -> Result<(Self, ConfigReport)> { run_owned_plugin_mutation("dynamic plugin activation", move || async move { Self::activate_inner(config, dynamic_plugins, diagnostics).await @@ -114,8 +258,8 @@ impl PluginHostActivation { async fn activate_inner( mut config: PluginConfig, - dynamic_plugins: Vec, - diagnostics: Vec, + dynamic_plugins: Vec, + diagnostics: Vec, ) -> Result<(Self, ConfigReport)> { let dynamic_plugin_count = dynamic_plugins.len(); log::info!( @@ -125,15 +269,16 @@ impl PluginHostActivation { "Dynamic plugin activation started" ); let claim = acquire_plugin_host_lease()?; + let mut transaction = PluginHostActivationTransaction::new(claim, &dynamic_plugins); #[cfg(not(feature = "worker-grpc"))] if let Some(plugin) = dynamic_plugins .iter() - .find(|plugin| plugin.kind == DynamicPluginKind::Worker) + .find(|plugin| plugin.spec.kind == DynamicPluginKind::Worker) { return Err(crate::plugin::PluginError::InvalidConfig(format!( "worker dynamic plugin '{}' requires the 'worker-grpc' feature", - plugin.plugin_id + plugin.spec.plugin_id ))); } @@ -144,96 +289,132 @@ impl PluginHostActivation { let native_specs = dynamic_plugins .iter() - .filter(|plugin| plugin.kind == DynamicPluginKind::RustDynamic) - .map(|plugin| NativePluginLoadSpec { - plugin_id: plugin.plugin_id.clone(), - manifest_ref: plugin.manifest_ref.clone(), + .filter(|plugin| plugin.spec.kind == DynamicPluginKind::RustDynamic) + .map(|plugin| { + ( + NativePluginLoadSpec { + plugin_id: plugin.spec.plugin_id.clone(), + manifest_ref: plugin.spec.manifest_ref.clone(), + }, + plugin.resource.clone(), + ) }) .collect::>(); - let native = (!native_specs.is_empty()) - .then(|| { - load_native_plugins(native_specs) - .map_err(|error| plugin_error_context("native plugin load failed", error)) - }) - .transpose()?; + transaction.native = if native_specs.is_empty() { + None + } else { + match load_native_plugins_with_resources(native_specs) { + Ok(native) => Some(native), + Err(failure) => { + return Err(finalize_load_failure( + "native plugin load failed", + &mut transaction.claim, + failure, + )); + } + } + }; #[cfg(feature = "worker-grpc")] - let worker = { + { let worker_specs = dynamic_plugins .iter() - .filter(|plugin| plugin.kind == DynamicPluginKind::Worker) - .map(|plugin| WorkerPluginLoadSpec { - plugin_id: plugin.plugin_id.clone(), - manifest_ref: plugin.manifest_ref.clone(), - environment_ref: plugin.environment_ref.clone(), - config: plugin.config.clone(), + .filter(|plugin| plugin.spec.kind == DynamicPluginKind::Worker) + .map(|plugin| { + ( + WorkerPluginLoadSpec { + plugin_id: plugin.spec.plugin_id.clone(), + manifest_ref: plugin.spec.manifest_ref.clone(), + environment_ref: plugin.spec.environment_ref.clone(), + config: plugin.spec.config.clone(), + }, + plugin.resource.clone(), + ) }) .collect::>(); - (!worker_specs.is_empty()) - .then(|| { - load_worker_plugins(worker_specs) - .map_err(|error| plugin_error_context("worker plugin load failed", error)) - }) - .transpose()? - }; + if worker_specs.is_empty() { + transaction.worker = None; + } else { + transaction.worker = match load_worker_plugins_with_resources(worker_specs) { + Ok(worker) => Some(worker), + Err(mut failure) => { + if let Some(native_activation) = transaction.native.as_mut() { + let mut native_rollback = + native_activation.deregister_plugin_kinds_checked(); + if native_rollback.safe_to_unload { + native_rollback.merge(native_activation.prepare_unload_checked()); + } + let native_safe_to_unload = native_rollback.safe_to_unload; + failure.merge_rollback(native_rollback); + if !native_safe_to_unload + && let Some(native_activation) = transaction.native.take() + { + std::mem::forget(native_activation); + } + } + return Err(finalize_load_failure( + "worker plugin load failed", + &mut transaction.claim, + failure, + )); + } + }; + } + } config.components.extend( dynamic_plugins .into_iter() .map(|plugin| PluginComponentSpec { - kind: plugin.plugin_id, + kind: plugin.spec.plugin_id, enabled: true, - config: plugin.config, + config: plugin.spec.config, }), ); let rollback_failures = Arc::new(Mutex::new(Vec::new())); - let owner_id = claim.owner_id(); + let owner_id = transaction + .claim + .as_ref() + .expect("active plugin host must retain its owner lease") + .owner_id(); let initialization = tokio::spawn(initialize_plugins_exact_for_host( config, owner_id, Arc::clone(&rollback_failures), diagnostics, )) - .await - .map_err(|error| { - crate::plugin::PluginError::Internal(format!( - "dynamic plugin initialization task failed: {error}" - )) - }); - let report = match initialization.and_then(|result| result) { - Ok(report) => report, - Err(error) => { + .await; + let report = match initialization { + Ok(Ok(report)) => report, + Ok(Err(error)) => { let failures = rollback_failures .lock() .map(|failures| failures.clone()) .unwrap_or_else(|lock_error| { vec![format!("rollback failure lock poisoned: {lock_error}")] }); - if failures.is_empty() { - return Err(error); - } - log::error!( - target: "nemo_relay.plugin", - event = "plugin_rollback_failed", - plugin_count = dynamic_plugin_count, - failure_count = failures.len(); - "Dynamic plugin activation rollback was incomplete" + return Err(finalize_configuration_failure( + error, + failures, + &mut transaction.native, + #[cfg(feature = "worker-grpc")] + &mut transaction.worker, + &mut transaction.claim, + )); + } + Err(join_error) => { + retain_loaded_runtimes( + &mut transaction.native, + #[cfg(feature = "worker-grpc")] + &mut transaction.worker, ); - if let Some(native) = native { - std::mem::forget(native); - } - #[cfg(feature = "worker-grpc")] - if let Some(worker) = worker { - std::mem::forget(worker); - } - std::mem::forget(claim); + retain_plugin_host_claim(&mut transaction.claim); return Err(crate::plugin::PluginError::RegistrationFailed(format!( concat!( - "{}; activation rollback was incomplete: {}; the loaded runtimes ", - "were retained because callbacks may remain registered" + "dynamic plugin initialization task failed: {}; the loaded runtimes ", + "and activation owner were retained because callback rollback could not be proven" ), - error, - failures.join("; ") + join_error ))); } }; @@ -244,16 +425,7 @@ impl PluginHostActivation { plugin_count = dynamic_plugin_count; "Dynamic plugins activated" ); - Ok(( - Self { - active: true, - native, - #[cfg(feature = "worker-grpc")] - worker, - claim: Some(claim), - }, - report, - )) + Ok((transaction.into_activation(), report)) } /// Returns whether this activation handle has not begun teardown. @@ -275,7 +447,13 @@ impl PluginHostActivation { return Ok(()); } self.active = false; - let outcome = self + let unwind_guard = PluginHostClearUnwindGuard { activation: self }; + #[cfg(test)] + if PANIC_PLUGIN_HOST_CLEAR_AFTER_DEACTIVATION.swap(false, Ordering::SeqCst) { + panic!("injected plugin host teardown panic after deactivation"); + } + let outcome = unwind_guard + .activation .claim .as_ref() .map(|claim| clear_plugin_configuration_for_host(claim.owner_id())) @@ -292,16 +470,16 @@ impl PluginHostActivation { // If core could not prove callbacks were removed, intentionally // retain their code and owner for process lifetime rather than // unload a library or worker that may still be referenced. - self.retain_loaded_runtimes(); + unwind_guard.activation.retain_loaded_runtimes(); return Err(retained_runtime_error(errors)); } let mut runtime_outcome = DynamicPluginTeardownOutcome::success(); - if let Some(native) = &mut self.native { + if let Some(native) = &mut unwind_guard.activation.native { runtime_outcome.merge(native.deregister_plugin_kinds_checked()); } #[cfg(feature = "worker-grpc")] - if let Some(worker) = &mut self.worker { + if let Some(worker) = &mut unwind_guard.activation.worker { runtime_outcome.merge(worker.deregister_plugin_kinds_checked()); } @@ -310,24 +488,36 @@ impl PluginHostActivation { // absent from the registry. #[cfg(feature = "worker-grpc")] if runtime_outcome.safe_to_unload - && let Some(worker) = &self.worker + && let Some(worker) = &unwind_guard.activation.worker { runtime_outcome.merge(worker.shutdown_plugins_checked()); } + if runtime_outcome.safe_to_unload + && let Some(native) = &mut unwind_guard.activation.native + { + runtime_outcome.merge(native.prepare_unload_checked()); + } + #[cfg(feature = "worker-grpc")] + if runtime_outcome.safe_to_unload + && let Some(worker) = &mut unwind_guard.activation.worker + { + runtime_outcome.merge(worker.prepare_unload_checked()); + } errors.extend(runtime_outcome.errors); if !runtime_outcome.safe_to_unload { - self.retain_loaded_runtimes(); + unwind_guard.activation.retain_loaded_runtimes(); return Err(retained_runtime_error(errors)); } // Callback removal and kind deregistration are now complete. Dropping // the activations unloads libraries and runtimes before releasing the // process-wide host claim. - self.native.take(); + unwind_guard.activation.native.take(); #[cfg(feature = "worker-grpc")] - self.worker.take(); - self.claim.take(); + unwind_guard.activation.worker.take(); + unwind_guard.activation.resource_anchors.clear(); + unwind_guard.activation.claim.take(); if errors.is_empty() { log::info!( @@ -355,9 +545,31 @@ impl PluginHostActivation { if let Some(claim) = self.claim.take() { std::mem::forget(claim); } + let resources = std::mem::take(&mut self.resource_anchors); + if !resources.is_empty() { + std::mem::forget(resources); + } } } +fn prepare_explicit_dynamic_plugins( + dynamic_plugins: Vec, +) -> Vec { + dynamic_plugins + .into_iter() + .map(|spec| PreparedDynamicPluginActivation { + spec, + resource: None, + }) + .collect() +} + +fn validate_planned_dynamic_plugins( + dynamic_plugins: &[PlannedDynamicPluginActivation], +) -> Result<()> { + validate_unique_dynamic_plugin_ids(dynamic_plugins.iter().map(|plugin| &plugin.spec)) +} + fn validate_dynamic_plugin_specs(dynamic_plugins: &[DynamicPluginActivationSpec]) -> Result<()> { if dynamic_plugins.is_empty() { return Err(crate::plugin::PluginError::InvalidConfig( @@ -368,7 +580,13 @@ fn validate_dynamic_plugin_specs(dynamic_plugins: &[DynamicPluginActivationSpec] .into(), )); } - let mut plugin_ids = HashSet::with_capacity(dynamic_plugins.len()); + validate_unique_dynamic_plugin_ids(dynamic_plugins.iter()) +} + +fn validate_unique_dynamic_plugin_ids<'a>( + dynamic_plugins: impl IntoIterator, +) -> Result<()> { + let mut plugin_ids = HashSet::new(); for plugin in dynamic_plugins { if !plugin_ids.insert(plugin.plugin_id.as_str()) { return Err(crate::plugin::PluginError::InvalidConfig(format!( @@ -394,6 +612,155 @@ fn retained_runtime_error(errors: Vec) -> crate::plugin::PluginError { )) } +fn finalize_configuration_failure( + error: crate::plugin::PluginError, + callback_failures: Vec, + native: &mut Option, + #[cfg(feature = "worker-grpc")] worker: &mut Option, + claim: &mut Option, +) -> crate::plugin::PluginError { + if !callback_failures.is_empty() { + log::error!( + target: "nemo_relay.plugin", + event = "plugin_rollback_failed", + failure_count = callback_failures.len(); + "Dynamic plugin callback rollback was incomplete" + ); + retain_loaded_runtimes( + native, + #[cfg(feature = "worker-grpc")] + worker, + ); + retain_plugin_host_claim(claim); + return crate::plugin::PluginError::RegistrationFailed(format!( + concat!( + "{}; activation rollback was incomplete: {}; the loaded runtimes and ", + "activation owner were retained because callbacks may remain registered" + ), + error, + callback_failures.join("; ") + )); + } + + let runtime_rollback = rollback_loaded_runtimes( + native, + #[cfg(feature = "worker-grpc")] + worker, + ); + if runtime_rollback.errors.is_empty() { + return error; + } + if !runtime_rollback.safe_to_unload { + retain_plugin_host_claim(claim); + return crate::plugin::PluginError::RegistrationFailed(format!( + concat!( + "{}; dynamic runtime rollback was incomplete: {}; the loaded runtimes and ", + "activation owner were retained because safe unloading could not be proven" + ), + error, + runtime_rollback.errors.join("; ") + )); + } + crate::plugin::PluginError::RegistrationFailed(format!( + "{}; dynamic runtime rollback reported: {}; all loaded runtimes were removed", + error, + runtime_rollback.errors.join("; ") + )) +} + +#[cfg(not(feature = "worker-grpc"))] +fn rollback_loaded_runtimes( + native: &mut Option, +) -> DynamicPluginTeardownOutcome { + let mut outcome = DynamicPluginTeardownOutcome::success(); + if let Some(native) = native.as_mut() { + outcome.merge(native.deregister_plugin_kinds_checked()); + } + if outcome.safe_to_unload + && let Some(native) = native.as_mut() + { + outcome.merge(native.prepare_unload_checked()); + } + finish_runtime_rollback(native, &outcome); + outcome +} + +#[cfg(feature = "worker-grpc")] +fn rollback_loaded_runtimes( + native: &mut Option, + worker: &mut Option, +) -> DynamicPluginTeardownOutcome { + let mut outcome = DynamicPluginTeardownOutcome::success(); + if let Some(native) = native.as_mut() { + outcome.merge(native.deregister_plugin_kinds_checked()); + } + if let Some(worker) = worker.as_mut() { + outcome.merge(worker.deregister_plugin_kinds_checked()); + } + if outcome.safe_to_unload + && let Some(worker) = worker.as_ref() + { + outcome.merge(worker.shutdown_plugins_checked()); + } + if outcome.safe_to_unload + && let Some(native) = native.as_mut() + { + outcome.merge(native.prepare_unload_checked()); + } + if outcome.safe_to_unload + && let Some(worker) = worker.as_mut() + { + outcome.merge(worker.prepare_unload_checked()); + } + finish_runtime_rollback(native, &outcome); + finish_runtime_rollback(worker, &outcome); + outcome +} + +fn finish_runtime_rollback(activation: &mut Option, outcome: &DynamicPluginTeardownOutcome) { + if outcome.safe_to_unload { + drop(activation.take()); + } else if let Some(activation) = activation.take() { + std::mem::forget(activation); + } +} + +fn retain_loaded_runtimes( + native: &mut Option, + #[cfg(feature = "worker-grpc")] worker: &mut Option, +) { + if let Some(native) = native.take() { + std::mem::forget(native); + } + #[cfg(feature = "worker-grpc")] + if let Some(worker) = worker.take() { + std::mem::forget(worker); + } +} + +fn retain_plugin_host_claim(claim: &mut Option) { + if let Some(claim) = claim.take() { + std::mem::forget(claim); + } +} + +fn finalize_load_failure( + prefix: &str, + claim: &mut Option, + failure: DynamicPluginLoadFailure, +) -> crate::plugin::PluginError { + let retain_owner = !failure.safe_to_unload(); + let error = plugin_error_context(prefix, failure.into_plugin_error()); + if !retain_owner { + return error; + } + + retain_plugin_host_claim(claim); + crate::plugin::PluginError::RegistrationFailed(format!( + "{error}; the plugin host activation owner was retained because a partially loaded runtime remains reachable" + )) +} + fn plugin_error_context( prefix: &str, error: crate::plugin::PluginError, diff --git a/crates/core/src/plugin/dynamic/native.rs b/crates/core/src/plugin/dynamic/native.rs index 3b0124b5b..e1d511924 100644 --- a/crates/core/src/plugin/dynamic/native.rs +++ b/crates/core/src/plugin/dynamic/native.rs @@ -73,8 +73,9 @@ use tokio::runtime::Runtime; use tokio_stream::{Stream, StreamExt}; use super::{ - DynamicPluginKind, DynamicPluginManifest, DynamicPluginManifestLoad, - DynamicPluginTeardownOutcome, deregister_tracked_registrations_checked, + DynamicPluginActivationResource, DynamicPluginKind, DynamicPluginLoadFailure, + DynamicPluginManifest, DynamicPluginManifestLoad, DynamicPluginTeardownOutcome, + PanicRetentionGuard, deregister_tracked_registrations_checked, finish_partial_load_rollback, validate_annotated_request_consumer_compatibility, }; @@ -95,6 +96,10 @@ pub struct NativePluginLoadSpec { pub struct NativePluginActivation { plugins: Vec>, plugin_registrations: Vec<(String, u64)>, + #[cfg(test)] + _retained_resources_for_test: Vec>, + #[cfg(test)] + force_unload_panic: bool, } impl NativePluginActivation { @@ -110,11 +115,57 @@ impl NativePluginActivation { deregister_tracked_registrations_checked(&mut self.plugin_registrations, "native") } + pub(crate) fn prepare_unload_checked(&mut self) -> DynamicPluginTeardownOutcome { + #[cfg(test)] + if self.force_unload_panic { + panic!("injected native activation unload panic"); + } + + let mut outcome = DynamicPluginTeardownOutcome::success(); + for plugin in self.plugins.iter_mut().rev() { + let Some(plugin) = Arc::get_mut(plugin) else { + // Callback user-data snapshots own the exact instance. Defer + // descriptor teardown and library release until the last + // in-flight snapshot drops instead of invalidating its plugin + // state while native code is still executing. + continue; + }; + outcome.merge(plugin.prepare_unload_checked()); + } + outcome + } + #[cfg(test)] pub(super) fn with_plugin_kind_for_test(plugin_kind: impl Into) -> Self { Self { plugins: Vec::new(), plugin_registrations: vec![(plugin_kind.into(), 0)], + _retained_resources_for_test: Vec::new(), + force_unload_panic: false, + } + } + + #[cfg(test)] + pub(super) fn with_resource_for_test( + resource: Arc, + ) -> Self { + Self { + plugins: Vec::new(), + plugin_registrations: Vec::new(), + _retained_resources_for_test: vec![resource], + force_unload_panic: false, + } + } + + #[cfg(test)] + pub(super) fn with_panicking_unload_resource_for_test( + resource: Arc, + ) -> Self { + Self { + plugins: Vec::new(), + plugin_registrations: Vec::new(), + _retained_resources_for_test: vec![resource], + force_unload_panic: true, } } } @@ -135,24 +186,61 @@ pub fn load_native_plugins(specs: I) -> crate::plugin::Result, { - let mut activation = NativePluginActivation { + load_native_plugins_with_resources(specs.into_iter().map(|spec| (spec, None))) + .map_err(DynamicPluginLoadFailure::into_plugin_error) +} + +pub(crate) fn load_native_plugins_with_resources( + specs: I, +) -> std::result::Result +where + I: IntoIterator< + Item = ( + NativePluginLoadSpec, + Option>, + ), + >, +{ + let mut activation = PanicRetentionGuard::new(NativePluginActivation { plugins: Vec::new(), plugin_registrations: Vec::new(), - }; - for spec in specs { - let instance = load_one_native_plugin(&spec)?; + #[cfg(test)] + _retained_resources_for_test: Vec::new(), + #[cfg(test)] + force_unload_panic: false, + }); + for (spec, resource) in specs { + let instance = match load_one_native_plugin(&spec, resource) { + Ok(instance) => instance, + Err(error) => return Err(rollback_failed_native_load(activation, error)), + }; let plugin_kind = instance.plugin_kind.clone(); - let registration_id = register_plugin_tracked(Arc::new(NativePluginAdapter { + let registration_id = match register_plugin_tracked(Arc::new(NativePluginAdapter { plugin_kind: plugin_kind.clone(), allows_multiple_components: instance.allows_multiple_components, instance: instance.clone(), - }))?; - activation.plugins.push(instance); + })) { + Ok(registration_id) => registration_id, + Err(error) => return Err(rollback_failed_native_load(activation, error)), + }; + activation.get_mut().plugins.push(instance); activation + .get_mut() .plugin_registrations .push((plugin_kind, registration_id)); } - Ok(activation) + Ok(activation.take()) +} + +fn rollback_failed_native_load( + mut activation: PanicRetentionGuard, + error: PluginError, +) -> DynamicPluginLoadFailure { + let mut rollback = activation.get_mut().deregister_plugin_kinds_checked(); + if rollback.safe_to_unload { + rollback.merge(activation.get_mut().prepare_unload_checked()); + } + finish_partial_load_rollback(activation.take(), error, rollback) } struct NativePluginAdapter { @@ -280,6 +368,7 @@ struct NativePluginInstance { allows_multiple_components: bool, plugin: Mutex, _library: Library, + _activation_resource: Option>, } unsafe impl Send for NativePluginInstance {} @@ -293,6 +382,23 @@ impl Drop for NativePluginInstance { } } +impl NativePluginInstance { + fn prepare_unload_checked(&self) -> DynamicPluginTeardownOutcome { + let mut outcome = DynamicPluginTeardownOutcome::success(); + match self.plugin.lock() { + Ok(mut plugin) => drop_native_plugin_descriptor(&mut plugin), + Err(error) => outcome.record_error( + format!( + "native plugin '{}' descriptor lock poisoned during unload: {error}", + self.plugin_kind + ), + false, + ), + } + outcome + } +} + fn drop_native_plugin_descriptor(plugin: &mut NemoRelayNativePluginV1) { if let Some(drop_fn) = plugin.drop.take() { unsafe { drop_fn(plugin.user_data) }; @@ -306,6 +412,7 @@ fn drop_native_plugin_descriptor(plugin: &mut NemoRelayNativePluginV1) { fn load_one_native_plugin( spec: &NativePluginLoadSpec, + activation_resource: Option>, ) -> crate::plugin::Result> { let (manifest, manifest_ref) = DynamicPluginManifest::load_from_path(&spec.manifest_ref)?; if manifest.plugin.id.trim() != spec.plugin_id { @@ -365,16 +472,23 @@ fn load_one_native_plugin( .as_deref() .ok_or_else(|| PluginError::InvalidConfig("load.symbol is required".into()))?; + if let Some(resource) = &activation_resource { + resource.verify()?; + } let library = unsafe { Library::new(&library_path) }.map_err(|err| { PluginError::Internal(format!( "failed to load native plugin library '{}': {err}", library_path.display() )) })?; + let mut loaded_resource = PanicRetentionGuard::new((library, activation_resource)); let mut plugin = NemoRelayNativePluginV1::default(); unsafe { - let entry: Symbol = - library.get(symbol.as_bytes()).map_err(|err| { + let entry: Symbol = loaded_resource + .get() + .0 + .get(symbol.as_bytes()) + .map_err(|err| { PluginError::NotFound(format!( "native plugin symbol '{symbol}' not found in '{}': {err}", library_path.display() @@ -414,12 +528,14 @@ fn load_one_native_plugin( spec.plugin_id ))); } + let (library, activation_resource) = loaded_resource.take(); Ok(Arc::new(NativePluginInstance { plugin_kind, relay_compat, allows_multiple_components: plugin.allows_multiple_components, plugin: Mutex::new(plugin), _library: library, + _activation_resource: activation_resource, })) } diff --git a/crates/core/src/plugin/dynamic/registry.rs b/crates/core/src/plugin/dynamic/registry.rs index 081a3f03c..ab6b9832f 100644 --- a/crates/core/src/plugin/dynamic/registry.rs +++ b/crates/core/src/plugin/dynamic/registry.rs @@ -201,6 +201,45 @@ impl DynamicPluginRegistry { Ok(()) } + /// Replaces manifest-derived fields while retaining lifecycle-owned state. + /// + /// File-backed hosts use this after reparsing a declaration's current + /// manifest. Desired state, lifecycle lineage, the managed worker + /// environment, and observed runtime state remain owned by the existing + /// lifecycle record. + #[doc(hidden)] + pub fn refresh_manifest_record( + &mut self, + plugin_id: &str, + mut refreshed: DynamicPluginRecord, + ) -> Result<&DynamicPluginRecord> { + normalize_record_shape(&mut refreshed); + validate_record_shape(&refreshed)?; + if refreshed.metadata.id != plugin_id { + return Err(PluginError::InvalidConfig(format!( + "refreshed dynamic plugin id '{}' does not match lifecycle id '{plugin_id}'", + refreshed.metadata.id + ))); + } + + let existing = self.records.get(plugin_id).ok_or_else(|| { + PluginError::NotFound(format!("dynamic plugin '{plugin_id}' is not registered")) + })?; + refreshed.metadata.generation = existing.metadata.generation; + refreshed.metadata.created_at = existing.metadata.created_at.clone(); + refreshed.metadata.updated_at = existing.metadata.updated_at.clone(); + refreshed.spec = existing.spec.clone(); + refreshed.source.environment_ref = existing.source.environment_ref.clone(); + refreshed.status.runtime = existing.status.runtime.clone(); + refreshed.status.validation.checked_at = Some(super::current_timestamp()); + + self.records.insert(plugin_id.to_owned(), refreshed); + Ok(self + .records + .get(plugin_id) + .expect("refreshed dynamic plugin record must exist immediately after insert")) + } + fn lookup_mut(&mut self, plugin_id: &str) -> Result<&mut DynamicPluginRecord> { self.records.get_mut(plugin_id).ok_or_else(|| { PluginError::NotFound(format!("dynamic plugin '{plugin_id}' is not registered")) diff --git a/crates/core/src/plugin/dynamic/worker.rs b/crates/core/src/plugin/dynamic/worker.rs index 8c2e3c4bb..977aa85a8 100644 --- a/crates/core/src/plugin/dynamic/worker.rs +++ b/crates/core/src/plugin/dynamic/worker.rs @@ -82,8 +82,9 @@ use crate::plugin::{ }; use super::{ - DynamicPluginKind, DynamicPluginManifest, DynamicPluginManifestLoad, - DynamicPluginTeardownOutcome, WorkerRuntime, deregister_tracked_registrations_checked, + DynamicPluginActivationResource, DynamicPluginKind, DynamicPluginLoadFailure, + DynamicPluginManifest, DynamicPluginManifestLoad, DynamicPluginTeardownOutcome, + PanicRetentionGuard, WorkerRuntime, deregister_tracked_registrations_checked, validate_annotated_request_consumer_compatibility, }; @@ -152,10 +153,30 @@ impl WorkerPluginActivation { pub(crate) fn shutdown_plugins_checked(&self) -> DynamicPluginTeardownOutcome { let mut outcome = DynamicPluginTeardownOutcome::success(); for plugin in self.plugins.iter().rev() { + if Arc::strong_count(plugin) > 1 { + // Callback snapshots own the exact instance. Defer worker + // shutdown until the last in-flight snapshot drops rather + // than tearing down its process and runtime underneath it. + continue; + } outcome.merge(plugin.shutdown_checked()); } outcome } + + pub(crate) fn prepare_unload_checked(&mut self) -> DynamicPluginTeardownOutcome { + let mut outcome = DynamicPluginTeardownOutcome::success(); + for plugin in self.plugins.iter_mut().rev() { + let Some(plugin) = Arc::get_mut(plugin) else { + // A callback snapshot still owns the instance. Dropping the + // activation transfers final shutdown and runtime destruction + // to that snapshot's eventual last-owner drop. + continue; + }; + outcome.merge(plugin.prepare_unload_checked()); + } + outcome + } } impl Drop for WorkerPluginActivation { @@ -174,24 +195,66 @@ pub fn load_worker_plugins(specs: I) -> crate::plugin::Result, { - let mut activation = WorkerPluginActivation { + load_worker_plugins_with_resources(specs.into_iter().map(|spec| (spec, None))) + .map_err(DynamicPluginLoadFailure::into_plugin_error) +} + +pub(crate) fn load_worker_plugins_with_resources( + specs: I, +) -> std::result::Result +where + I: IntoIterator< + Item = ( + WorkerPluginLoadSpec, + Option>, + ), + >, +{ + let mut activation = PanicRetentionGuard::new(WorkerPluginActivation { plugins: Vec::new(), plugin_registrations: Vec::new(), - }; - for spec in specs { - let instance = load_one_worker_plugin(&spec)?; + }); + for (spec, resource) in specs { + let instance = match load_one_worker_plugin(&spec, resource) { + Ok(instance) => instance, + Err(failure) => return Err(rollback_failed_worker_load(activation, failure)), + }; let plugin_kind = instance.plugin_kind.clone(); - let registration_id = register_plugin_tracked(Arc::new(WorkerPluginAdapter { + activation.get_mut().plugins.push(Arc::clone(&instance)); + let registration_id = match register_plugin_tracked(Arc::new(WorkerPluginAdapter { plugin_kind: plugin_kind.clone(), allows_multiple_components: instance.allows_multiple_components, - instance: instance.clone(), - }))?; - activation.plugins.push(instance); + instance, + })) { + Ok(registration_id) => registration_id, + Err(error) => return Err(rollback_failed_worker_load(activation, error.into())), + }; activation + .get_mut() .plugin_registrations .push((plugin_kind, registration_id)); } - Ok(activation) + Ok(activation.take()) +} + +fn rollback_failed_worker_load( + mut activation: PanicRetentionGuard, + mut failure: DynamicPluginLoadFailure, +) -> DynamicPluginLoadFailure { + let mut rollback = activation.get_mut().deregister_plugin_kinds_checked(); + if rollback.safe_to_unload { + rollback.merge(activation.get().shutdown_plugins_checked()); + } + if rollback.safe_to_unload { + rollback.merge(activation.get_mut().prepare_unload_checked()); + } + let safe_to_unload = rollback.safe_to_unload; + failure.merge_rollback(rollback); + let activation = activation.take(); + if !safe_to_unload { + std::mem::forget(activation); + } + failure } struct WorkerPluginAdapter { @@ -251,11 +314,18 @@ struct WorkerPluginInstance { process: Mutex>, activation_dir: PathBuf, teardown_started: AtomicBool, + _activation_resource: Option>, } impl Drop for WorkerPluginInstance { fn drop(&mut self) { - let outcome = self.shutdown_checked(); + let mut outcome = self.shutdown_checked(); + if outcome.safe_to_unload { + outcome.merge(self.runtime.shutdown_checked()); + } + if !outcome.safe_to_unload { + self.retain_owned_resources(); + } if !outcome.errors.is_empty() { log::error!( target: "nemo_relay.worker", @@ -270,6 +340,35 @@ impl Drop for WorkerPluginInstance { } impl WorkerPluginInstance { + fn retain_owned_resources(&mut self) { + self.runtime.retain(); + if let Some(shutdown) = self + .shutdown + .get_mut() + .unwrap_or_else(|error| error.into_inner()) + .take() + { + std::mem::forget(shutdown); + } + if let Some(process) = self + .process + .get_mut() + .unwrap_or_else(|error| error.into_inner()) + .take() + { + std::mem::forget(process); + } + if let Some(resource) = self._activation_resource.take() { + std::mem::forget(resource); + } + std::mem::forget(self.client.clone()); + std::mem::forget(Arc::clone(&self.host_state)); + } + + fn prepare_unload_checked(&mut self) -> DynamicPluginTeardownOutcome { + self.runtime.shutdown_checked() + } + fn shutdown_checked(&self) -> DynamicPluginTeardownOutcome { let mut outcome = DynamicPluginTeardownOutcome::success(); if self.teardown_started.swap(true, Ordering::AcqRel) { @@ -374,57 +473,57 @@ impl WorkerPluginInstance { return; } }; - let Some(child) = process.as_mut() else { - return; - }; - match child.try_wait() { - Ok(Some(_)) => { - process.take(); - } - Ok(None) => { - if let Err(kill_error) = child.kill() { - match child.try_wait() { - Ok(Some(_)) => { - process.take(); - outcome.record_error( - format!( - "worker plugin '{}' process kill failed after exit: {kill_error}", - self.plugin_kind - ), - true, - ); - } - Ok(None) | Err(_) => outcome.record_error( + stop_worker_process_checked(&self.plugin_kind, &mut process, outcome); + } +} + +fn stop_worker_process_checked( + plugin_kind: &str, + process: &mut Option, + outcome: &mut DynamicPluginTeardownOutcome, +) { + let Some(child) = process.as_mut() else { + return; + }; + match child.try_wait() { + Ok(Some(_)) => { + process.take(); + } + Ok(None) => { + if let Err(kill_error) = child.kill() { + match child.try_wait() { + Ok(Some(_)) => { + process.take(); + outcome.record_error( format!( - "worker plugin '{}' process kill failed: {kill_error}", - self.plugin_kind + "worker plugin '{plugin_kind}' process kill failed after exit: {kill_error}" ), - false, - ), - } - return; - } - match child.wait() { - Ok(_) => { - process.take(); + true, + ); } - Err(error) => outcome.record_error( - format!( - "worker plugin '{}' process wait failed after kill: {error}", - self.plugin_kind - ), + Ok(None) | Err(_) => outcome.record_error( + format!("worker plugin '{plugin_kind}' process kill failed: {kill_error}"), false, ), } + return; } - Err(error) => outcome.record_error( - format!( - "worker plugin '{}' process status check failed: {error}", - self.plugin_kind + match child.wait() { + Ok(_) => { + process.take(); + } + Err(error) => outcome.record_error( + format!( + "worker plugin '{plugin_kind}' process wait failed after kill: {error}" + ), + false, ), - false, - ), + } } + Err(error) => outcome.record_error( + format!("worker plugin '{plugin_kind}' process status check failed: {error}"), + false, + ), } } @@ -438,7 +537,8 @@ fn panic_payload_message(payload: &(dyn std::any::Any + Send)) -> &str { fn load_one_worker_plugin( spec: &WorkerPluginLoadSpec, -) -> crate::plugin::Result> { + activation_resource: Option>, +) -> std::result::Result, DynamicPluginLoadFailure> { log::info!( target: "nemo_relay.worker", event = "worker_starting", @@ -450,13 +550,15 @@ fn load_one_worker_plugin( return Err(PluginError::InvalidConfig(format!( "dynamic plugin manifest id '{}' does not match expected id '{}'", manifest.plugin.id, spec.plugin_id - ))); + )) + .into()); } if manifest.plugin.kind != DynamicPluginKind::Worker { return Err(PluginError::InvalidConfig(format!( "dynamic plugin '{}' is kind {}; worker loader only supports worker", spec.plugin_id, manifest.plugin.kind - ))); + )) + .into()); } validate_relay_compatibility(manifest.compat.relay.as_deref())?; let relay_compat = manifest @@ -509,7 +611,10 @@ fn load_one_worker_plugin( )); let manifest_path = PathBuf::from(&manifest_ref); - let mut child = ChildGuard::new(spawn_worker_process(WorkerProcessLaunch { + if let Some(resource) = &activation_resource { + resource.verify()?; + } + let child = spawn_worker_process(WorkerProcessLaunch { runtime, manifest_path: &manifest_path, environment_ref: spec.environment_ref.as_deref(), @@ -520,28 +625,72 @@ fn load_one_worker_plugin( host_endpoint: &host_advertise, worker_endpoint: &worker_advertise, worker_endpoint_file: worker_endpoint_file.as_deref(), - })?); + })?; + let pending = PendingWorkerPluginStartup::new( + spec.plugin_id.clone(), + runtime_handle, + host_state, + shutdown_tx, + child, + activation_dir_guard.keep(), + activation_resource, + ); log::info!( target: "nemo_relay.worker", event = "worker_started", plugin_id = spec.plugin_id.as_str(), - pid = child - .child - .as_ref() - .expect("worker child should remain guarded") - .id(); + pid = pending.process_id(); "Worker plugin process started" ); + + let prepared = match prepare_worker_plugin_connection( + spec, + &relay_compat, + &activation_id, + &auth_token, + &host_advertise, + &worker_connect, + &pending, + ) { + Ok(prepared) => prepared, + Err(error) => return Err(pending.fail(error)), + }; + + log::info!( + target: "nemo_relay.worker", + event = "worker_connected", + plugin_id = spec.plugin_id.as_str(); + "Worker plugin connected and registered" + ); + Ok(pending.activate(spec, prepared)) +} + +struct PreparedWorkerPlugin { + client: PluginWorkerClient, + allows_multiple_components: bool, + validation_diagnostics: Vec, + registrations: Vec, +} + +fn prepare_worker_plugin_connection( + spec: &WorkerPluginLoadSpec, + relay_compat: &str, + activation_id: &str, + auth_token: &str, + host_advertise: &str, + worker_connect: &WorkerConnectEndpoint, + pending: &PendingWorkerPluginStartup, +) -> crate::plugin::Result { let mut client = block_on_runtime( - runtime_handle.runtime(), - connect_worker_with_retry(&worker_connect, &spec.plugin_id), + pending.runtime(), + connect_worker_with_retry(worker_connect, &spec.plugin_id), )?; let health = block_on_runtime( - runtime_handle.runtime(), + pending.runtime(), worker_rpc(client.health(worker_rpc_request(HealthRequest { - activation_id: activation_id.clone(), - auth_token: auth_token.clone(), + activation_id: activation_id.to_owned(), + auth_token: auth_token.to_owned(), }))), ) .map_err(|err| PluginError::RegistrationFailed(format!("worker health check failed: {err}")))?; @@ -552,14 +701,14 @@ fn load_one_worker_plugin( } let handshake = block_on_runtime( - runtime_handle.runtime(), + pending.runtime(), worker_rpc(client.handshake(worker_rpc_request(HandshakeRequest { - activation_id: activation_id.clone(), + activation_id: activation_id.to_owned(), plugin_id: spec.plugin_id.clone(), relay_version: env!("CARGO_PKG_VERSION").into(), worker_protocol: WORKER_PROTOCOL_GRPC_V1.into(), - auth_token: auth_token.clone(), - host_endpoint: host_advertise.clone(), + auth_token: auth_token.to_owned(), + host_endpoint: host_advertise.to_owned(), }))), ) .map_err(|err| PluginError::RegistrationFailed(format!("worker handshake failed: {err}")))?; @@ -580,11 +729,11 @@ fn load_one_worker_plugin( let config = Json::Object(spec.config.clone()); let validate = block_on_runtime( - runtime_handle.runtime(), + pending.runtime(), worker_rpc(client.validate(worker_rpc_request(ValidateRequest { - activation_id: activation_id.clone(), + activation_id: activation_id.to_owned(), plugin_id: spec.plugin_id.clone(), - auth_token: auth_token.clone(), + auth_token: auth_token.to_owned(), config: Some(json_envelope(JSON_SCHEMA, &config)?), }))), ) @@ -605,11 +754,11 @@ fn load_one_worker_plugin( Vec::new() } else { let register = block_on_runtime( - runtime_handle.runtime(), + pending.runtime(), worker_rpc(client.register(worker_rpc_request(RegisterRequest { - activation_id: activation_id.clone(), + activation_id: activation_id.to_owned(), plugin_id: spec.plugin_id.clone(), - auth_token: auth_token.clone(), + auth_token: auth_token.to_owned(), config: Some(json_envelope(JSON_SCHEMA, &config)?), }))), ) @@ -627,29 +776,14 @@ fn load_one_worker_plugin( RegistrationSurface::try_from(registration.surface) .is_ok_and(|surface| surface == RegistrationSurface::LlmRequestIntercept) }) { - validate_annotated_request_consumer_compatibility(&relay_compat, &spec.plugin_id)?; + validate_annotated_request_consumer_compatibility(relay_compat, &spec.plugin_id)?; } - - log::info!( - target: "nemo_relay.worker", - event = "worker_connected", - plugin_id = spec.plugin_id.as_str(); - "Worker plugin connected and registered" - ); - Ok(Arc::new(WorkerPluginInstance { - plugin_kind: spec.plugin_id.clone(), + Ok(PreparedWorkerPlugin { + client, allows_multiple_components: handshake.allows_multiple_components, - config: spec.config.clone(), validation_diagnostics, registrations, - runtime: runtime_handle, - client, - host_state, - shutdown: Mutex::new(Some(shutdown_tx)), - process: Mutex::new(Some(child.take())), - activation_dir: activation_dir_guard.keep(), - teardown_started: AtomicBool::new(false), - })) + }) } enum HostRuntimeServer { @@ -1056,7 +1190,7 @@ fn clear_host_python_environment(command: &mut Command) { impl WorkerPluginInstance { fn install_registrations( - &self, + self: &Arc, ctx: &mut PluginRegistrationContext, ) -> crate::plugin::Result<()> { for registration in &self.registrations { @@ -1106,7 +1240,7 @@ impl WorkerPluginInstance { } fn install_subscriber_registration( - &self, + self: &Arc, ctx: &mut PluginRegistrationContext, name: &str, ) -> crate::plugin::Result<()> { @@ -1123,7 +1257,7 @@ impl WorkerPluginInstance { } fn install_event_sanitize_registration( - &self, + self: &Arc, ctx: &mut PluginRegistrationContext, name: &str, priority: i32, @@ -1156,7 +1290,7 @@ impl WorkerPluginInstance { } fn install_tool_registration( - &self, + self: &Arc, ctx: &mut PluginRegistrationContext, registration: &Registration, surface: RegistrationSurface, @@ -1245,7 +1379,7 @@ impl WorkerPluginInstance { } fn install_llm_registration( - &self, + self: &Arc, ctx: &mut PluginRegistrationContext, registration: &Registration, surface: RegistrationSurface, @@ -1356,13 +1490,14 @@ impl WorkerPluginInstance { } } - fn clone_for_callback(&self) -> WorkerPluginCallback { + fn clone_for_callback(self: &Arc) -> WorkerPluginCallback { WorkerPluginCallback { plugin_kind: self.plugin_kind.clone(), activation_id: self.host_state.activation_id.clone(), runtime: self.runtime.handle(), client: self.client.clone(), host_state: self.host_state.clone(), + activation_anchor: WorkerCallbackActivationAnchor::new(Arc::clone(self)), } } } @@ -1394,6 +1529,31 @@ struct WorkerPluginCallback { runtime: tokio::runtime::Handle, client: PluginWorkerClient, host_state: Arc, + activation_anchor: WorkerCallbackActivationAnchor, +} + +#[derive(Clone)] +struct WorkerCallbackActivationAnchor { + #[cfg(not(test))] + _instance: Arc, + #[cfg(test)] + _instance: Option>, +} + +impl WorkerCallbackActivationAnchor { + fn new(instance: Arc) -> Self { + Self { + #[cfg(not(test))] + _instance: instance, + #[cfg(test)] + _instance: Some(instance), + } + } + + #[cfg(test)] + fn detached() -> Self { + Self { _instance: None } + } } impl WorkerPluginCallback { @@ -1418,6 +1578,7 @@ struct WorkerInvocationGuard { invocation_id: String, continuation_id: String, scope_stack_id: String, + _activation_anchor: WorkerCallbackActivationAnchor, cancel_on_drop: bool, cleaned: bool, } @@ -1437,6 +1598,7 @@ impl WorkerInvocationGuard { .as_ref() .map(|scope| scope.scope_stack_id.clone()) .unwrap_or_default(), + _activation_anchor: callback.activation_anchor.clone(), cancel_on_drop: true, cleaned: false, } @@ -1934,12 +2096,16 @@ impl WorkerPluginCallback { struct OwnedWorkerRuntime { runtime: Option, + #[cfg(test)] + force_drop_thread_panic: bool, } impl OwnedWorkerRuntime { fn new(runtime: Runtime) -> Self { Self { runtime: Some(runtime), + #[cfg(test)] + force_drop_thread_panic: false, } } @@ -1952,46 +2118,271 @@ impl OwnedWorkerRuntime { fn handle(&self) -> tokio::runtime::Handle { self.runtime().handle().clone() } -} -impl Drop for OwnedWorkerRuntime { - fn drop(&mut self) { + fn retain(&mut self) { + if let Some(runtime) = self.runtime.take() { + std::mem::forget(runtime); + } + } + + fn shutdown_checked(&mut self) -> DynamicPluginTeardownOutcome { + let mut outcome = DynamicPluginTeardownOutcome::success(); let Some(runtime) = self.runtime.take() else { - return; + return outcome; }; - if tokio::runtime::Handle::try_current().is_ok() { + #[cfg(test)] + let force_drop_thread_panic = self.force_drop_thread_panic; + + let drop_result = if tokio::runtime::Handle::try_current().is_ok() { std::thread::scope(|scope| { scope - .spawn(move || drop(runtime)) + .spawn(move || { + let mut runtime = std::mem::ManuallyDrop::new(runtime); + #[cfg(test)] + if force_drop_thread_panic { + panic!("injected worker runtime drop thread panic"); + } + // SAFETY: this is the only explicit destruction of the + // runtime. If destruction panics, the outer host keeps + // its resource anchors and lease fail-closed. + unsafe { std::mem::ManuallyDrop::drop(&mut runtime) }; + }) .join() - .expect("worker runtime drop thread panicked"); - }); + }) } else { - drop(runtime); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || { + let mut runtime = std::mem::ManuallyDrop::new(runtime); + #[cfg(test)] + if force_drop_thread_panic { + panic!("injected worker runtime drop panic"); + } + // SAFETY: this is the only explicit destruction of the + // runtime; ManuallyDrop prevents a second attempt on unwind. + unsafe { std::mem::ManuallyDrop::drop(&mut runtime) }; + })) + }; + if let Err(payload) = drop_result { + outcome.record_error( + format!( + "worker runtime destruction panicked: {}", + panic_payload_message(payload.as_ref()) + ), + false, + ); } + outcome + } + + #[cfg(test)] + fn force_drop_thread_panic_for_test(&mut self) { + self.force_drop_thread_panic = true; } } -struct ChildGuard { - child: Option, +impl Drop for OwnedWorkerRuntime { + fn drop(&mut self) { + let outcome = self.shutdown_checked(); + if !outcome.errors.is_empty() { + log::error!( + target: "nemo_relay.worker", + event = "worker_runtime_cleanup_failed", + failure_count = outcome.errors.len(), + safe_to_unload = outcome.safe_to_unload; + "Worker runtime cleanup failed during drop" + ); + } + } } -impl ChildGuard { - fn new(child: Child) -> Self { - Self { child: Some(child) } +struct PendingWorkerPluginStartup { + plugin_kind: String, + runtime: Option, + host_state: Option>, + shutdown: Option>, + process: Option, + activation_dir: Option, + activation_resource: Option>, + #[cfg(test)] + force_unsafe_cleanup: bool, +} + +impl PendingWorkerPluginStartup { + #[allow(clippy::too_many_arguments)] + fn new( + plugin_kind: String, + runtime: OwnedWorkerRuntime, + host_state: Arc, + shutdown: oneshot::Sender<()>, + process: Child, + activation_dir: PathBuf, + activation_resource: Option>, + ) -> Self { + Self { + plugin_kind, + runtime: Some(runtime), + host_state: Some(host_state), + shutdown: Some(shutdown), + process: Some(process), + activation_dir: Some(activation_dir), + activation_resource, + #[cfg(test)] + force_unsafe_cleanup: false, + } + } + + fn runtime(&self) -> &Runtime { + self.runtime + .as_ref() + .expect("pending worker runtime must remain owned during startup") + .runtime() + } + + fn process_id(&self) -> u32 { + self.process + .as_ref() + .expect("pending worker process must remain owned during startup") + .id() + } + + fn activate( + mut self, + spec: &WorkerPluginLoadSpec, + prepared: PreparedWorkerPlugin, + ) -> Arc { + Arc::new(WorkerPluginInstance { + plugin_kind: std::mem::take(&mut self.plugin_kind), + allows_multiple_components: prepared.allows_multiple_components, + config: spec.config.clone(), + validation_diagnostics: prepared.validation_diagnostics, + registrations: prepared.registrations, + runtime: self + .runtime + .take() + .expect("completed worker startup must retain its runtime"), + client: prepared.client, + host_state: self + .host_state + .take() + .expect("completed worker startup must retain its host state"), + shutdown: Mutex::new(self.shutdown.take()), + process: Mutex::new(self.process.take()), + activation_dir: self + .activation_dir + .take() + .expect("completed worker startup must retain its activation directory"), + teardown_started: AtomicBool::new(false), + _activation_resource: self.activation_resource.take(), + }) + } + + fn fail(mut self, error: PluginError) -> DynamicPluginLoadFailure { + let outcome = self.cleanup_checked(); + if !outcome.safe_to_unload { + self.retain_owned_resources(); + } + DynamicPluginLoadFailure::new(error, outcome) + } + + fn cleanup_checked(&mut self) -> DynamicPluginTeardownOutcome { + let mut outcome = DynamicPluginTeardownOutcome::success(); + #[cfg(test)] + if self.force_unsafe_cleanup { + outcome.record_error("injected pending worker cleanup failure", false); + return outcome; + } + + if let Some(shutdown) = self.shutdown.take() + && shutdown.send(()).is_err() + { + outcome.record_error( + format!( + "worker plugin '{}' pending host runtime shutdown channel was closed", + self.plugin_kind + ), + true, + ); + } + stop_worker_process_checked(&self.plugin_kind, &mut self.process, &mut outcome); + if !outcome.safe_to_unload { + return outcome; + } + + if let Some(activation_dir) = self.activation_dir.take() + && let Err(error) = std::fs::remove_dir_all(&activation_dir) + && error.kind() != std::io::ErrorKind::NotFound + { + outcome.record_error( + format!( + "worker plugin '{}' pending activation directory cleanup failed for '{}': {error}", + self.plugin_kind, + activation_dir.display() + ), + true, + ); + } + if let Some(runtime) = self.runtime.as_mut() { + outcome.merge(runtime.shutdown_checked()); + } + if !outcome.safe_to_unload { + return outcome; + } + // `shutdown_checked` has consumed the Tokio runtime. Dropping the + // inert owner is now infallible and cannot release the activation + // resource before runtime destruction has been proven safe. + self.runtime.take(); + self.host_state.take(); + self.activation_resource.take(); + outcome + } + + fn retain_owned_resources(&mut self) { + retain_option(&mut self.runtime); + retain_option(&mut self.host_state); + retain_option(&mut self.shutdown); + retain_option(&mut self.process); + retain_option(&mut self.activation_resource); + self.activation_dir.take(); } - fn take(&mut self) -> Child { - self.child.take().expect("worker child already taken") + fn owns_resources(&self) -> bool { + self.runtime.is_some() + || self.host_state.is_some() + || self.shutdown.is_some() + || self.process.is_some() + || self.activation_dir.is_some() + || self.activation_resource.is_some() } } -impl Drop for ChildGuard { +impl Drop for PendingWorkerPluginStartup { fn drop(&mut self) { - if let Some(mut child) = self.child.take() { - let _ = child.kill(); - let _ = child.wait(); + if !self.owns_resources() { + return; } + if std::thread::panicking() { + self.retain_owned_resources(); + return; + } + let outcome = self.cleanup_checked(); + if !outcome.safe_to_unload { + self.retain_owned_resources(); + } + if !outcome.errors.is_empty() { + log::error!( + target: "nemo_relay.worker", + event = "worker_startup_cleanup_failed", + plugin_id = self.plugin_kind.as_str(), + failure_count = outcome.errors.len(), + safe_to_unload = outcome.safe_to_unload; + "Pending worker plugin cleanup failed" + ); + } + } +} + +fn retain_option(value: &mut Option) { + if let Some(value) = value.take() { + std::mem::forget(value); } } diff --git a/crates/core/tests/integration/native_plugin_tests.rs b/crates/core/tests/integration/native_plugin_tests.rs index 8d572f247..50248ecc8 100644 --- a/crates/core/tests/integration/native_plugin_tests.rs +++ b/crates/core/tests/integration/native_plugin_tests.rs @@ -25,8 +25,9 @@ use nemo_relay::api::subscriber::{deregister_subscriber, flush_subscribers, regi use nemo_relay::api::tool::{ToolCallExecuteParams, tool_call_execute, tool_request_intercepts}; use nemo_relay::codec::response::AnnotatedLlmResponse; use nemo_relay::plugin::dynamic::{ - DynamicPluginActivationSpec, DynamicPluginKind, NativePluginLoadSpec, PluginHostActivation, - load_native_plugins, + DynamicPluginActivationResource, DynamicPluginActivationSpec, DynamicPluginKind, + NativePluginLoadSpec, PlannedDynamicPluginActivation, PluginHostActivation, + PluginHostActivationPlan, load_native_plugins, }; use nemo_relay::plugin::{ ConfigDiagnostic, Plugin, PluginComponentSpec, PluginConfig, PluginRegistrationContext, @@ -51,6 +52,24 @@ static STATIC_BASE_DEREGISTRATIONS: AtomicUsize = AtomicUsize::new(0); struct StaticBasePlugin; +struct TrackingActivationResource { + verify_count: Arc, + drop_count: Arc, +} + +impl DynamicPluginActivationResource for TrackingActivationResource { + fn verify(&self) -> PluginResult<()> { + self.verify_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} + +impl Drop for TrackingActivationResource { + fn drop(&mut self) { + self.drop_count.fetch_add(1, Ordering::SeqCst); + } +} + struct BlockingHostBasePlugin { started: Arc, release: Arc, @@ -1378,6 +1397,46 @@ async fn native_validate_and_register_callback_errors_are_reported() { } } +#[tokio::test] +async fn plugin_host_component_failure_checked_rolls_back_kind_and_owner() { + let _guard = NATIVE_PLUGIN_TEST_LOCK.lock().await; + let fixture = build_fixture_plugin(); + let failing_manifest = + write_manifest_with_symbol(&fixture, "nemo_relay_fixture_register_error"); + + let error = match PluginHostActivation::activate( + PluginConfig::default(), + [host_spec("fixture_native", &failing_manifest)], + ) + .await + { + Ok((activation, _)) => { + activation + .clear() + .expect("unexpected failing component activation should clear"); + panic!("native component registration failure should fail the plugin host"); + } + Err(error) => error.to_string(), + }; + assert!(error.contains("fixture register failed"), "{error}"); + assert!( + !list_plugin_kinds() + .iter() + .any(|kind| kind == "fixture_native") + ); + + let healthy_manifest = write_manifest(&fixture); + let (activation, _) = PluginHostActivation::activate( + PluginConfig::default(), + [host_spec("fixture_native", &healthy_manifest)], + ) + .await + .expect("checked component rollback should release the host owner"); + activation + .clear() + .expect("recovered plugin host should clear"); +} + #[tokio::test] async fn plugin_host_activation_owns_configuration_until_clear() { let _guard = NATIVE_PLUGIN_TEST_LOCK.lock().await; @@ -1526,12 +1585,24 @@ async fn plugin_host_clear_allows_an_in_flight_native_callback_to_finish() { let _guard = NATIVE_PLUGIN_TEST_LOCK.lock().await; let fixture = build_fixture_plugin(); let manifest_ref = write_manifest(&fixture); - let (activation, _) = PluginHostActivation::activate( - PluginConfig::default(), - [host_spec("fixture_native", &manifest_ref)], - ) + let verify_count = Arc::new(AtomicUsize::new(0)); + let drop_count = Arc::new(AtomicUsize::new(0)); + let resource = Arc::new(TrackingActivationResource { + verify_count: Arc::clone(&verify_count), + drop_count: Arc::clone(&drop_count), + }); + let (activation, _) = PluginHostActivation::activate_plan(PluginHostActivationPlan { + config: PluginConfig::default(), + dynamic_plugins: vec![PlannedDynamicPluginActivation { + spec: host_spec("fixture_native", &manifest_ref), + resource, + }], + diagnostics: Vec::new(), + }) .await .expect("plugin host should activate"); + assert_eq!(verify_count.load(Ordering::SeqCst), 1); + assert_eq!(drop_count.load(Ordering::SeqCst), 0); let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); @@ -1574,6 +1645,11 @@ async fn plugin_host_clear_allows_an_in_flight_native_callback_to_finish() { activation .clear() .expect("host should clear while a callback snapshot remains in flight"); + assert_eq!( + drop_count.load(Ordering::SeqCst), + 0, + "the activation resource must outlive an in-flight native callback" + ); let unchanged = tool_request_intercepts("after-clear", json!({ "input": true })) .await .expect("new calls should observe the cleared registries"); @@ -1588,6 +1664,14 @@ async fn plugin_host_clear_allows_an_in_flight_native_callback_to_finish() { .expect("in-flight callback should finish after host clear"); assert_eq!(result["tool_callback"], true); assert_eq!(result["native_plugin_tool_execution"], true); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while drop_count.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("the activation resource should be released after the callback finishes"); + assert_eq!(drop_count.load(Ordering::SeqCst), 1); } #[tokio::test] @@ -1604,13 +1688,25 @@ async fn plugin_host_activation_cleans_up_after_caller_cancellation() { registered: Arc::clone(®istered), })) .expect("blocking base plugin should register"); + let verify_count = Arc::new(AtomicUsize::new(0)); + let drop_count = Arc::new(AtomicUsize::new(0)); + let resource = Arc::new(TrackingActivationResource { + verify_count: Arc::clone(&verify_count), + drop_count: Arc::clone(&drop_count), + }); - let caller = tokio::spawn(PluginHostActivation::activate( - PluginConfig { - components: vec![PluginComponentSpec::new("fixture_blocking_host_base")], - ..PluginConfig::default() + let caller = tokio::spawn(PluginHostActivation::activate_plan( + PluginHostActivationPlan { + config: PluginConfig { + components: vec![PluginComponentSpec::new("fixture_blocking_host_base")], + ..PluginConfig::default() + }, + dynamic_plugins: vec![PlannedDynamicPluginActivation { + spec: host_spec("fixture_native", &manifest_ref), + resource, + }], + diagnostics: Vec::new(), }, - [host_spec("fixture_native", &manifest_ref)], )); started.notified().await; caller.abort(); @@ -1625,6 +1721,7 @@ async fn plugin_host_activation_cleans_up_after_caller_cancellation() { loop { if nemo_relay::plugin::active_plugin_report().is_none() && lookup_plugin("fixture_native").is_none() + && drop_count.load(Ordering::SeqCst) == 1 { break; } @@ -1633,6 +1730,7 @@ async fn plugin_host_activation_cleans_up_after_caller_cancellation() { }) .await .expect("canceled activation should clear its completed host result"); + assert_eq!(verify_count.load(Ordering::SeqCst), 1); let (activation, _) = PluginHostActivation::activate( PluginConfig::default(), diff --git a/crates/core/tests/integration/worker_plugin_tests.rs b/crates/core/tests/integration/worker_plugin_tests.rs index 358447ae9..ee56410d7 100644 --- a/crates/core/tests/integration/worker_plugin_tests.rs +++ b/crates/core/tests/integration/worker_plugin_tests.rs @@ -5,6 +5,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use futures::StreamExt; @@ -23,12 +24,13 @@ use nemo_relay::codec::request::AnnotatedLlmRequest; use nemo_relay::codec::traits::LlmCodec; use nemo_relay::error::Result as FlowResult; use nemo_relay::plugin::dynamic::{ - DynamicPluginActivationSpec, DynamicPluginKind, PluginHostActivation, WorkerPluginActivation, - WorkerPluginLoadSpec, load_worker_plugins, + DynamicPluginActivationResource, DynamicPluginActivationSpec, DynamicPluginKind, + PlannedDynamicPluginActivation, PluginHostActivation, PluginHostActivationPlan, + WorkerPluginActivation, WorkerPluginLoadSpec, load_worker_plugins, }; use nemo_relay::plugin::{ - PluginComponentSpec, PluginConfig, clear_plugin_configuration, initialize_plugins_exact, - list_plugin_kinds, + PluginComponentSpec, PluginConfig, Result as PluginResult, clear_plugin_configuration, + initialize_plugins_exact, list_plugin_kinds, }; use serde_json::{Map, Value as Json, json}; use sha2::{Digest, Sha256}; @@ -37,6 +39,24 @@ use uuid::Uuid; static WORKER_PLUGIN_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); +struct TrackingActivationResource { + verify_count: Arc, + drop_count: Arc, +} + +impl DynamicPluginActivationResource for TrackingActivationResource { + fn verify(&self) -> PluginResult<()> { + self.verify_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} + +impl Drop for TrackingActivationResource { + fn drop(&mut self) { + self.drop_count.fetch_add(1, Ordering::SeqCst); + } +} + fn enable_operational_logs() { let _ = spdlog::init_log_crate_proxy(); log::set_max_level(log::LevelFilter::Info); @@ -55,21 +75,33 @@ async fn plugin_host_activation_owns_worker_lifecycle() { let _guard = WORKER_PLUGIN_TEST_LOCK.lock().await; let fixture = build_fixture_worker(); let (_manifest_dir, manifest_ref) = write_manifest(fixture.binary_path()); - let (activation, report) = PluginHostActivation::activate( - PluginConfig::default(), - [DynamicPluginActivationSpec { - plugin_id: "fixture_worker".into(), - kind: DynamicPluginKind::Worker, - manifest_ref: manifest_ref.to_string_lossy().into_owned(), - environment_ref: None, - config: Map::new(), + let verify_count = Arc::new(AtomicUsize::new(0)); + let drop_count = Arc::new(AtomicUsize::new(0)); + let resource = Arc::new(TrackingActivationResource { + verify_count: Arc::clone(&verify_count), + drop_count: Arc::clone(&drop_count), + }); + let (activation, report) = PluginHostActivation::activate_plan(PluginHostActivationPlan { + config: PluginConfig::default(), + dynamic_plugins: vec![PlannedDynamicPluginActivation { + spec: DynamicPluginActivationSpec { + plugin_id: "fixture_worker".into(), + kind: DynamicPluginKind::Worker, + manifest_ref: manifest_ref.to_string_lossy().into_owned(), + environment_ref: None, + config: Map::new(), + }, + resource, }], - ) + diagnostics: Vec::new(), + }) .await .expect("worker plugin host should activate"); assert!(activation.is_active()); assert!(!report.has_errors()); + assert_eq!(verify_count.load(Ordering::SeqCst), 1); + assert_eq!(drop_count.load(Ordering::SeqCst), 0); assert!( list_plugin_kinds() .iter() @@ -81,6 +113,7 @@ async fn plugin_host_activation_owns_worker_lifecycle() { assert_eq!(rewritten["worker_plugin"], true); activation.clear().expect("worker plugin host should clear"); + assert_eq!(drop_count.load(Ordering::SeqCst), 1); assert!( !list_plugin_kinds() .iter() @@ -92,6 +125,106 @@ async fn plugin_host_activation_owns_worker_lifecycle() { assert_eq!(unchanged, json!({ "input": true })); } +#[tokio::test] +async fn plugin_host_clear_allows_an_in_flight_worker_callback_to_finish() { + let _guard = WORKER_PLUGIN_TEST_LOCK.lock().await; + let fixture = build_fixture_worker(); + let (_manifest_dir, manifest_ref) = write_manifest(fixture.binary_path()); + let verify_count = Arc::new(AtomicUsize::new(0)); + let drop_count = Arc::new(AtomicUsize::new(0)); + let resource = Arc::new(TrackingActivationResource { + verify_count: Arc::clone(&verify_count), + drop_count: Arc::clone(&drop_count), + }); + let (activation, _) = PluginHostActivation::activate_plan(PluginHostActivationPlan { + config: PluginConfig::default(), + dynamic_plugins: vec![PlannedDynamicPluginActivation { + spec: DynamicPluginActivationSpec { + plugin_id: "fixture_worker".into(), + kind: DynamicPluginKind::Worker, + manifest_ref: manifest_ref.to_string_lossy().into_owned(), + environment_ref: None, + config: Map::new(), + }, + resource, + }], + diagnostics: Vec::new(), + }) + .await + .expect("worker plugin host should activate"); + assert_eq!(verify_count.load(Ordering::SeqCst), 1); + assert_eq!(drop_count.load(Ordering::SeqCst), 0); + + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let release_rx = Arc::new(Mutex::new(release_rx)); + let call_thread = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("in-flight worker callback runtime should build"); + runtime.block_on(tool_call_execute( + ToolCallExecuteParams::builder() + .name("worker-fixture-in-flight") + .args(json!({ "input": "in-flight" })) + .func(Arc::new(move |args| { + let entered_tx = entered_tx.clone(); + let release_rx = Arc::clone(&release_rx); + Box::pin(async move { + entered_tx.send(()).map_err(|error| { + nemo_relay::error::FlowError::Internal(error.to_string()) + })?; + release_rx + .lock() + .map_err(|error| { + nemo_relay::error::FlowError::Internal(error.to_string()) + })? + .recv() + .map_err(|error| { + nemo_relay::error::FlowError::Internal(error.to_string()) + })?; + Ok(json!({ "tool_callback": true, "args": args })) + }) + })) + .build(), + )) + }); + + entered_rx + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("worker callback should enter its continuation"); + activation + .clear() + .expect("host should clear while a worker callback snapshot remains in flight"); + assert_eq!( + drop_count.load(Ordering::SeqCst), + 0, + "the activation resource must outlive an in-flight worker callback" + ); + let unchanged = tool_request_intercepts("after-clear", json!({ "input": true })) + .await + .expect("new calls should observe the cleared registries"); + assert_eq!(unchanged, json!({ "input": true })); + + release_tx + .send(()) + .expect("in-flight worker continuation should still be reachable"); + let result = call_thread + .join() + .expect("in-flight worker callback thread should not panic") + .expect("in-flight worker callback should finish after host clear"); + assert_eq!(result["tool_callback"], true); + assert_eq!(result["args"]["worker_plugin_tool_execution_request"], true); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while drop_count.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("the activation resource should be released after the worker callback finishes"); + assert_eq!(drop_count.load(Ordering::SeqCst), 1); +} + #[tokio::test] async fn plugin_host_clear_surfaces_worker_shutdown_failure_and_releases_safe_owner() { let _guard = WORKER_PLUGIN_TEST_LOCK.lock().await; diff --git a/crates/core/tests/unit/dynamic_worker_tests.rs b/crates/core/tests/unit/dynamic_worker_tests.rs index b2b439792..80930569d 100644 --- a/crates/core/tests/unit/dynamic_worker_tests.rs +++ b/crates/core/tests/unit/dynamic_worker_tests.rs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; #[cfg(unix)] @@ -40,6 +41,91 @@ use super::*; const ACTIVATION_ID: &str = "activation-test"; const AUTH_TOKEN: &str = "auth-test"; +struct TrackingStartupResource { + drop_count: Arc, +} + +impl DynamicPluginActivationResource for TrackingStartupResource { + fn verify(&self) -> crate::plugin::Result<()> { + Ok(()) + } +} + +impl Drop for TrackingStartupResource { + fn drop(&mut self) { + self.drop_count.fetch_add(1, Ordering::SeqCst); + } +} + +#[test] +fn unsafe_pending_worker_cleanup_retains_startup_resources() { + let drop_count = Arc::new(AtomicUsize::new(0)); + let pending = PendingWorkerPluginStartup { + plugin_kind: "fixture.pending-worker".into(), + runtime: None, + host_state: None, + shutdown: None, + process: None, + activation_dir: None, + activation_resource: Some(Arc::new(TrackingStartupResource { + drop_count: Arc::clone(&drop_count), + })), + force_unsafe_cleanup: true, + }; + + let failure = pending.fail(PluginError::RegistrationFailed( + "injected worker handshake failure".into(), + )); + + assert!(!failure.safe_to_unload()); + let error = failure.into_plugin_error().to_string(); + assert!( + error.contains("injected worker handshake failure"), + "{error}" + ); + assert!( + error.contains("partially loaded runtime was retained"), + "{error}" + ); + assert_eq!(drop_count.load(Ordering::SeqCst), 0); +} + +#[tokio::test(flavor = "current_thread")] +async fn pending_worker_runtime_drop_failure_retains_startup_resource() { + let drop_count = Arc::new(AtomicUsize::new(0)); + let mut runtime = OwnedWorkerRuntime::new( + RuntimeBuilder::new_multi_thread() + .enable_all() + .build() + .expect("worker runtime should build"), + ); + runtime.force_drop_thread_panic_for_test(); + let pending = PendingWorkerPluginStartup { + plugin_kind: "fixture.pending-worker".into(), + runtime: Some(runtime), + host_state: None, + shutdown: None, + process: None, + activation_dir: None, + activation_resource: Some(Arc::new(TrackingStartupResource { + drop_count: Arc::clone(&drop_count), + })), + force_unsafe_cleanup: false, + }; + + let failure = pending.fail(PluginError::RegistrationFailed( + "injected worker startup failure".into(), + )); + + assert!(!failure.safe_to_unload()); + let error = failure.into_plugin_error().to_string(); + assert!( + error.contains("worker runtime destruction panicked"), + "{error}" + ); + assert_eq!(drop_count.load(Ordering::SeqCst), 0); +} + fn enable_operational_logs() { let _ = spdlog::init_log_crate_proxy(); log::set_max_level(log::LevelFilter::Info); @@ -1570,6 +1656,7 @@ async fn install_registrations_covers_registry_error_edges() { registration(surface, &duplicate_name), ]) .await; + let instance = Arc::new(instance); let mut ctx = PluginRegistrationContext::new(); let error = match instance.install_registrations(&mut ctx) { Err(error) => error, @@ -1589,6 +1676,7 @@ async fn install_registrations_covers_registry_error_edges() { ..registration(RegistrationSurface::Subscriber, "bad") }]) .await; + let instance = Arc::new(instance); let mut ctx = PluginRegistrationContext::new(); assert!( instance @@ -1600,6 +1688,7 @@ async fn install_registrations_covers_registry_error_edges() { let (instance, _shutdown) = fake_worker_instance(vec![registration(RegistrationSurface::Unspecified, "bad")]).await; + let instance = Arc::new(instance); let mut ctx = PluginRegistrationContext::new(); assert!( instance @@ -1669,6 +1758,7 @@ async fn installed_callbacks_apply_surface_specific_fallbacks() { }) .await; instance.client = error_client; + let instance = Arc::new(instance); let _runtime_guard = crate::shared_runtime::runtime_owner_test_mutex() .lock() @@ -2175,7 +2265,81 @@ async fn host_runtime_service_reports_poisoned_internal_locks() { #[test] fn owned_worker_runtime_drop_is_idempotent_when_runtime_already_taken() { enable_operational_logs(); - drop(OwnedWorkerRuntime { runtime: None }); + drop(OwnedWorkerRuntime { + runtime: None, + force_drop_thread_panic: false, + }); +} + +#[tokio::test(flavor = "current_thread")] +async fn owned_worker_runtime_drop_thread_panic_is_reported_without_unwinding() { + let mut runtime = OwnedWorkerRuntime::new( + RuntimeBuilder::new_multi_thread() + .enable_all() + .build() + .expect("worker runtime should build"), + ); + runtime.force_drop_thread_panic_for_test(); + let leaked_runtime_handle = runtime.handle(); + + let outcome = runtime.shutdown_checked(); + + assert!(!outcome.safe_to_unload); + assert_eq!(outcome.errors.len(), 1); + assert!( + outcome.errors[0].contains("worker runtime destruction panicked"), + "{}", + outcome.errors[0] + ); + assert!(runtime.runtime.is_none()); + + let (alive_tx, alive_rx) = std::sync::mpsc::sync_channel(1); + leaked_runtime_handle.spawn(async move { + let _ = alive_tx.send(()); + }); + alive_rx + .recv_timeout(std::time::Duration::from_secs(1)) + .expect("a fail-closed runtime must remain alive after drop-thread panic"); +} + +#[tokio::test(flavor = "current_thread")] +async fn worker_instance_runtime_drop_failure_retains_activation_resource() { + let drop_count = Arc::new(AtomicUsize::new(0)); + let mut runtime = OwnedWorkerRuntime::new( + RuntimeBuilder::new_multi_thread() + .enable_all() + .build() + .expect("worker runtime should build"), + ); + runtime.force_drop_thread_panic_for_test(); + let instance = WorkerPluginInstance { + plugin_kind: "fixture_worker".into(), + allows_multiple_components: false, + config: Map::new(), + validation_diagnostics: Vec::new(), + registrations: Vec::new(), + runtime, + client: PluginWorkerClient::new(Endpoint::from_static("http://127.0.0.1:1").connect_lazy()), + host_state: Arc::new(WorkerHostRuntimeState::new( + ACTIVATION_ID.into(), + AUTH_TOKEN.into(), + )), + shutdown: Mutex::new(None), + process: Mutex::new(None), + activation_dir: std::env::temp_dir().join("nmrw-final-drop-failure-test"), + teardown_started: AtomicBool::new(true), + _activation_resource: Some(Arc::new(TrackingStartupResource { + drop_count: Arc::clone(&drop_count), + })), + }; + + drop(instance); + + assert_eq!( + drop_count.load(Ordering::SeqCst), + 0, + "an unsafe final runtime drop must retain the activation resource fail-closed" + ); } #[tokio::test] @@ -2588,6 +2752,7 @@ fn callback_for_client( runtime: tokio::runtime::Handle::current(), client, host_state: state, + activation_anchor: WorkerCallbackActivationAnchor::detached(), }, shutdown_tx, ) @@ -2625,6 +2790,7 @@ async fn fake_worker_instance( process: Mutex::new(None), activation_dir, teardown_started: AtomicBool::new(false), + _activation_resource: None, }, shutdown_tx, ) diff --git a/crates/core/tests/unit/native_plugin_tests.rs b/crates/core/tests/unit/native_plugin_tests.rs index f0978fc6d..87f95fca8 100644 --- a/crates/core/tests/unit/native_plugin_tests.rs +++ b/crates/core/tests/unit/native_plugin_tests.rs @@ -310,10 +310,148 @@ fn native_test_adapter( allows_multiple_components: false, plugin: Mutex::new(plugin), _library: libloading::os::unix::Library::this().into(), + _activation_resource: None, }), } } +#[cfg(unix)] +struct BlockingNativeToolCallbackState { + entered: std::sync::mpsc::SyncSender<()>, + release: Mutex>, +} + +#[cfg(unix)] +unsafe extern "C" fn blocking_native_tool_callback( + user_data: *mut c_void, + _name: *const NemoRelayNativeString, + payload_json: *const NemoRelayNativeString, + out_json: *mut *mut NemoRelayNativeString, +) -> NemoRelayStatus { + let Some(state) = (unsafe { (user_data as *const BlockingNativeToolCallbackState).as_ref() }) + else { + return NemoRelayStatus::NullPointer; + }; + if out_json.is_null() { + return NemoRelayStatus::NullPointer; + } + if state.entered.send(()).is_err() { + return NemoRelayStatus::Internal; + } + if state + .release + .lock() + .unwrap_or_else(|error| error.into_inner()) + .recv() + .is_err() + { + return NemoRelayStatus::Internal; + } + let Ok(payload) = read_native_string(payload_json) else { + return NemoRelayStatus::InvalidUtf8; + }; + unsafe { *out_json = native_string(&payload) }; + NemoRelayStatus::Ok +} + +#[cfg(unix)] +unsafe extern "C" fn drop_blocking_native_tool_callback(user_data: *mut c_void) { + if !user_data.is_null() { + unsafe { + drop(Box::from_raw( + user_data as *mut BlockingNativeToolCallbackState, + )) + }; + } +} + +#[cfg(unix)] +struct NativeDescriptorDropState { + drop_count: Arc, +} + +#[cfg(unix)] +unsafe extern "C" fn count_native_descriptor_drop(user_data: *mut c_void) { + if !user_data.is_null() { + let state = unsafe { Box::from_raw(user_data as *mut NativeDescriptorDropState) }; + state.drop_count.fetch_add(1, Ordering::SeqCst); + } +} + +#[cfg(unix)] +#[test] +fn native_descriptor_drop_waits_for_in_flight_callback() { + let descriptor_drop_count = Arc::new(AtomicUsize::new(0)); + let descriptor_state = Box::new(NativeDescriptorDropState { + drop_count: Arc::clone(&descriptor_drop_count), + }); + let plugin = NemoRelayNativePluginV1 { + user_data: Box::into_raw(descriptor_state).cast(), + drop: Some(count_native_descriptor_drop), + ..Default::default() + }; + let instance = Arc::new(NativePluginInstance { + plugin_kind: "test.native.in-flight".into(), + relay_compat: "^0.7".into(), + allows_multiple_components: false, + plugin: Mutex::new(plugin), + _library: libloading::os::unix::Library::this().into(), + _activation_resource: None, + }); + + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let callback_state = Box::new(BlockingNativeToolCallbackState { + entered: entered_tx, + release: Mutex::new(release_rx), + }); + let callback = wrap_tool_json_fn( + Arc::clone(&instance), + blocking_native_tool_callback, + Box::into_raw(callback_state).cast(), + Some(drop_blocking_native_tool_callback), + ); + let callback_future = callback("in-flight".into(), json!({"value": true})); + drop(callback); + + let mut activation = NativePluginActivation { + plugins: vec![instance], + plugin_registrations: Vec::new(), + _retained_resources_for_test: Vec::new(), + force_unload_panic: false, + }; + let callback_thread = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("native callback runtime should build"); + runtime.block_on(callback_future) + }); + + entered_rx + .recv_timeout(Duration::from_secs(5)) + .expect("native callback should enter before activation teardown"); + let outcome = activation.prepare_unload_checked(); + assert!(outcome.safe_to_unload); + assert!(outcome.errors.is_empty()); + activation.clear(); + let drop_count_while_callback_is_blocked = descriptor_drop_count.load(Ordering::SeqCst); + + release_tx + .send(()) + .expect("in-flight native callback should be released"); + let result = callback_thread + .join() + .expect("in-flight native callback thread should not panic") + .expect("in-flight native callback should complete"); + assert_eq!(result, json!({"value": true})); + assert_eq!( + drop_count_while_callback_is_blocked, 0, + "descriptor state must remain live while native callback code is executing" + ); + assert_eq!(descriptor_drop_count.load(Ordering::SeqCst), 1); +} + #[cfg(unix)] #[tokio::test] async fn native_plugin_adapter_covers_validation_and_registration_results() { @@ -3925,6 +4063,7 @@ fn native_registration_entrypoints_reject_invalid_host_contexts_and_names() { allows_multiple_components: false, plugin: Mutex::new(NemoRelayNativePluginV1::default()), _library: libloading::os::unix::Library::this().into(), + _activation_resource: None, }); let mut invalid_host = NativeHostPluginContext { ctx: ptr::null_mut(), @@ -4382,6 +4521,7 @@ fn assert_async_request_registration_rejects_legacy_relay_contract() { allows_multiple_components: false, plugin: Mutex::new(NemoRelayNativePluginV1::default()), _library: libloading::os::unix::Library::this().into(), + _activation_resource: None, }); let mut registration = PluginRegistrationContext::new(); let mut host = NativeHostPluginContext { @@ -4431,6 +4571,7 @@ async fn native_async_wrappers_validate_callback_result_shapes() { allows_multiple_components: false, plugin: Mutex::new(NemoRelayNativePluginV1::default()), _library: libloading::os::unix::Library::this().into(), + _activation_resource: None, }); let result = native_string("true"); let user_data = result.cast(); @@ -5201,6 +5342,7 @@ async fn native_callback_wrappers_release_error_outputs_and_preserve_reasons() { allows_multiple_components: false, plugin: Mutex::new(NemoRelayNativePluginV1::default()), _library: libloading::os::unix::Library::this().into(), + _activation_resource: None, }); let request = LlmRequest { headers: Map::new(), diff --git a/crates/core/tests/unit/plugin_dynamic_host_tests.rs b/crates/core/tests/unit/plugin_dynamic_host_tests.rs index f959b8eb2..9b2cd0ebb 100644 --- a/crates/core/tests/unit/plugin_dynamic_host_tests.rs +++ b/crates/core/tests/unit/plugin_dynamic_host_tests.rs @@ -4,6 +4,97 @@ use super::*; use crate::plugin::{PLUGIN_HANDLERS, PLUGIN_MUTATION_OWNER, PluginMutationOwner}; use serde_json::{Map, Value as Json}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +fn invalid_native_library_fixture(plugin_id: &str) -> (tempfile::TempDir, String) { + let directory = tempfile::tempdir().expect("native plugin fixture directory should create"); + std::fs::write( + directory.path().join("invalid-library"), + b"not a native library", + ) + .expect("invalid native plugin library should write"); + let manifest = directory.path().join("relay-plugin.toml"); + std::fs::write( + &manifest, + format!( + r#"manifest_version = 1 + +[plugin] +id = {plugin_id:?} +kind = "rust_dynamic" + +[compat] +relay = ">=0.5,<1.0" +native_api = "1" + +[defaults] +enabled = false + +[capabilities] +items = ["plugin_native"] + +[load] +library = "invalid-library" +symbol = "nemo_relay_register_plugin" +"# + ), + ) + .expect("native plugin manifest should write"); + (directory, manifest.to_string_lossy().into_owned()) +} + +struct TrackingActivationResource { + verify_count: Arc, + drop_count: Arc, + fail_verification: bool, +} + +struct PanickingActivationResource { + verify_count: Arc, + drop_count: Arc, +} + +struct TrackingPartialRuntime { + drop_count: Arc, +} + +impl Drop for TrackingPartialRuntime { + fn drop(&mut self) { + self.drop_count.fetch_add(1, Ordering::SeqCst); + } +} + +impl DynamicPluginActivationResource for TrackingActivationResource { + fn verify(&self) -> crate::plugin::Result<()> { + self.verify_count.fetch_add(1, Ordering::SeqCst); + if self.fail_verification { + Err(crate::plugin::PluginError::InvalidConfig( + "activation snapshot changed".into(), + )) + } else { + Ok(()) + } + } +} + +impl DynamicPluginActivationResource for PanickingActivationResource { + fn verify(&self) -> crate::plugin::Result<()> { + self.verify_count.fetch_add(1, Ordering::SeqCst); + panic!("injected activation resource verification panic"); + } +} + +impl Drop for TrackingActivationResource { + fn drop(&mut self) { + self.drop_count.fetch_add(1, Ordering::SeqCst); + } +} + +impl Drop for PanickingActivationResource { + fn drop(&mut self) { + self.drop_count.fetch_add(1, Ordering::SeqCst); + } +} struct PoisonedRegistryCleanup; @@ -31,6 +122,7 @@ fn unsafe_kind_deregistration_retains_runtime_and_owner() { )), #[cfg(feature = "worker-grpc")] worker: None, + resource_anchors: Vec::new(), claim: Some(claim), }; @@ -58,6 +150,272 @@ fn unsafe_kind_deregistration_retains_runtime_and_owner() { )); } +#[test] +fn unsafe_partial_load_rollback_retains_runtime_and_owner() { + let _guard = crate::shared_runtime::runtime_owner_test_mutex() + .lock() + .unwrap_or_else(|error| error.into_inner()); + let _cleanup = PoisonedRegistryCleanup; + if let Ok(mut owner) = PLUGIN_MUTATION_OWNER.lock() { + *owner = PluginMutationOwner::Idle; + } + let drop_count = Arc::new(AtomicUsize::new(0)); + let mut rollback = DynamicPluginTeardownOutcome::success(); + rollback.record_error("plugin registry lock poisoned", false); + let failure = super::super::finish_partial_load_rollback( + TrackingPartialRuntime { + drop_count: Arc::clone(&drop_count), + }, + crate::plugin::PluginError::NotFound("second plugin manifest".into()), + rollback, + ); + assert_eq!(drop_count.load(Ordering::SeqCst), 0); + + let mut claim = Some(acquire_plugin_host_lease().expect("fixture host should own activation")); + let owner_id = claim.as_ref().unwrap().owner_id(); + let error = finalize_load_failure("native plugin load failed", &mut claim, failure).to_string(); + + assert!(claim.is_none()); + assert!( + error.contains("partially loaded runtime was retained"), + "{error}" + ); + assert!(error.contains("activation owner was retained"), "{error}"); + assert_eq!( + *PLUGIN_MUTATION_OWNER.lock().unwrap(), + PluginMutationOwner::Host(owner_id) + ); + assert!(matches!( + acquire_plugin_host_lease(), + Err(crate::plugin::PluginError::Conflict(_)) + )); + assert_eq!(drop_count.load(Ordering::SeqCst), 0); +} + +#[test] +fn activation_panic_retains_planned_resource_and_owner() { + let _guard = crate::shared_runtime::runtime_owner_test_mutex() + .lock() + .unwrap_or_else(|error| error.into_inner()); + let _cleanup = PoisonedRegistryCleanup; + if let Ok(mut owner) = PLUGIN_MUTATION_OWNER.lock() { + *owner = PluginMutationOwner::Idle; + } + let verify_count = Arc::new(AtomicUsize::new(0)); + let drop_count = Arc::new(AtomicUsize::new(0)); + let (_fixture, manifest_ref) = invalid_native_library_fixture("fixture.panicking-resource"); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("file-backed activation runtime should build"); + let result = runtime.block_on(PluginHostActivation::activate_plan( + PluginHostActivationPlan { + config: PluginConfig::default(), + dynamic_plugins: vec![PlannedDynamicPluginActivation { + spec: DynamicPluginActivationSpec { + plugin_id: "fixture.panicking-resource".into(), + kind: DynamicPluginKind::RustDynamic, + manifest_ref, + environment_ref: None, + config: Map::new(), + }, + resource: Arc::new(PanickingActivationResource { + verify_count: Arc::clone(&verify_count), + drop_count: Arc::clone(&drop_count), + }), + }], + diagnostics: Vec::new(), + }, + )); + let error = match result { + Ok((activation, _)) => { + std::mem::forget(activation); + panic!("the injected activation panic should fail the plan"); + } + Err(error) => error.to_string(), + }; + + assert!( + error.contains("file-backed plugin activation task failed"), + "{error}" + ); + assert_eq!(verify_count.load(Ordering::SeqCst), 1); + assert_eq!(drop_count.load(Ordering::SeqCst), 0); + assert!(matches!( + *PLUGIN_MUTATION_OWNER.lock().unwrap(), + PluginMutationOwner::Host(_) + )); + assert!(matches!( + acquire_plugin_host_lease(), + Err(crate::plugin::PluginError::Conflict(_)) + )); +} + +#[test] +fn teardown_panic_after_deactivation_retains_runtime_resource_and_owner() { + let _guard = crate::shared_runtime::runtime_owner_test_mutex() + .lock() + .unwrap_or_else(|error| error.into_inner()); + let _cleanup = PoisonedRegistryCleanup; + if let Ok(mut owner) = PLUGIN_MUTATION_OWNER.lock() { + *owner = PluginMutationOwner::Idle; + } + let claim = acquire_plugin_host_lease().expect("fixture host should acquire the owner"); + let owner_id = claim.owner_id(); + let resource_drop_count = Arc::new(AtomicUsize::new(0)); + let mut activation = PluginHostActivation { + active: true, + native: Some(NativePluginActivation::with_resource_for_test(Arc::new( + TrackingActivationResource { + verify_count: Arc::new(AtomicUsize::new(0)), + drop_count: Arc::clone(&resource_drop_count), + fail_verification: false, + }, + ))), + #[cfg(feature = "worker-grpc")] + worker: None, + resource_anchors: Vec::new(), + claim: Some(claim), + }; + PANIC_PLUGIN_HOST_CLEAR_AFTER_DEACTIVATION.store(true, Ordering::SeqCst); + + let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| activation.clear_inner())); + + assert!(panic.is_err(), "the injected teardown panic should unwind"); + assert!(!activation.is_active()); + assert!(activation.native.is_none()); + assert!(activation.claim.is_none()); + assert_eq!(resource_drop_count.load(Ordering::SeqCst), 0); + assert_eq!( + *PLUGIN_MUTATION_OWNER.lock().unwrap(), + PluginMutationOwner::Host(owner_id) + ); + assert!(matches!( + acquire_plugin_host_lease(), + Err(crate::plugin::PluginError::Conflict(_)) + )); +} + +#[test] +fn runtime_unload_panic_retains_activation_resource_and_owner() { + let _guard = crate::shared_runtime::runtime_owner_test_mutex() + .lock() + .unwrap_or_else(|error| error.into_inner()); + let _cleanup = PoisonedRegistryCleanup; + if let Ok(mut owner) = PLUGIN_MUTATION_OWNER.lock() { + *owner = PluginMutationOwner::Idle; + } + let claim = acquire_plugin_host_lease().expect("fixture host should acquire the owner"); + let owner_id = claim.owner_id(); + let resource_drop_count = Arc::new(AtomicUsize::new(0)); + let resource: Arc = Arc::new(TrackingActivationResource { + verify_count: Arc::new(AtomicUsize::new(0)), + drop_count: Arc::clone(&resource_drop_count), + fail_verification: false, + }); + let mut activation = PluginHostActivation { + active: true, + native: Some( + NativePluginActivation::with_panicking_unload_resource_for_test(Arc::clone(&resource)), + ), + #[cfg(feature = "worker-grpc")] + worker: None, + resource_anchors: vec![resource], + claim: Some(claim), + }; + + let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| activation.clear_inner())); + + assert!(panic.is_err(), "the injected unload panic should unwind"); + assert!(!activation.is_active()); + assert!(activation.native.is_none()); + assert!(activation.resource_anchors.is_empty()); + assert!(activation.claim.is_none()); + assert_eq!(resource_drop_count.load(Ordering::SeqCst), 0); + assert_eq!( + *PLUGIN_MUTATION_OWNER.lock().unwrap(), + PluginMutationOwner::Host(owner_id) + ); + assert!(matches!( + acquire_plugin_host_lease(), + Err(crate::plugin::PluginError::Conflict(_)) + )); +} + +#[test] +fn panic_retention_guard_keeps_partial_runtime_alive_during_unwind() { + let drop_count = Arc::new(AtomicUsize::new(0)); + let panic = std::panic::catch_unwind({ + let drop_count = Arc::clone(&drop_count); + move || { + let _activation = + super::super::PanicRetentionGuard::new(TrackingPartialRuntime { drop_count }); + panic!("injected partial runtime panic"); + } + }); + + assert!(panic.is_err()); + assert_eq!(drop_count.load(Ordering::SeqCst), 0); +} + +#[test] +fn component_failure_uses_checked_runtime_rollback_and_retains_owner_when_unsafe() { + let _guard = crate::shared_runtime::runtime_owner_test_mutex() + .lock() + .unwrap_or_else(|error| error.into_inner()); + let _cleanup = PoisonedRegistryCleanup; + if let Ok(mut owner) = PLUGIN_MUTATION_OWNER.lock() { + *owner = PluginMutationOwner::Idle; + } + let mut claim = Some(acquire_plugin_host_lease().expect("fixture host should acquire owner")); + let owner_id = claim.as_ref().unwrap().owner_id(); + let mut native = Some(NativePluginActivation::with_plugin_kind_for_test( + "fixture.partial-component", + )); + #[cfg(feature = "worker-grpc")] + let mut worker = None; + + std::thread::spawn(|| { + let _registry = PLUGIN_HANDLERS.write().unwrap(); + panic!("poison plugin registry during component failure rollback"); + }) + .join() + .expect_err("fixture registry writer should panic"); + + let error = finalize_configuration_failure( + crate::plugin::PluginError::RegistrationFailed( + "injected component initialization failure".into(), + ), + Vec::new(), + &mut native, + #[cfg(feature = "worker-grpc")] + &mut worker, + &mut claim, + ) + .to_string(); + + assert!(native.is_none()); + assert!(claim.is_none()); + assert!( + error.contains("injected component initialization failure"), + "{error}" + ); + assert!( + error.contains("dynamic runtime rollback was incomplete"), + "{error}" + ); + assert!(error.contains("activation owner were retained"), "{error}"); + assert_eq!( + *PLUGIN_MUTATION_OWNER.lock().unwrap(), + PluginMutationOwner::Host(owner_id) + ); + assert!(matches!( + acquire_plugin_host_lease(), + Err(crate::plugin::PluginError::Conflict(_)) + )); +} + #[test] fn dynamic_plugin_specs_require_unique_nonempty_input() { let empty = validate_dynamic_plugin_specs(&[]).unwrap_err().to_string(); @@ -79,6 +437,138 @@ fn dynamic_plugin_specs_require_unique_nonempty_input() { assert!(error.contains("duplicate dynamic plugin id"), "{error}"); } +#[test] +fn activation_plan_allows_owned_static_only_configuration() { + let _guard = crate::shared_runtime::runtime_owner_test_mutex() + .lock() + .unwrap_or_else(|error| error.into_inner()); + if let Ok(mut owner) = PLUGIN_MUTATION_OWNER.lock() { + *owner = PluginMutationOwner::Idle; + } + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("file-backed activation runtime should build"); + let (activation, report) = runtime + .block_on(PluginHostActivation::activate_plan( + PluginHostActivationPlan { + config: PluginConfig::default(), + dynamic_plugins: Vec::new(), + diagnostics: Vec::new(), + }, + )) + .expect("a file-backed static-only plan should activate"); + + assert!(activation.is_active()); + assert!(!report.has_errors()); + assert!(crate::plugin::active_plugin_report().is_some()); + activation + .clear() + .expect("the static-only file-backed owner should clear"); + assert!(crate::plugin::active_plugin_report().is_none()); + assert_eq!( + *PLUGIN_MUTATION_OWNER.lock().unwrap(), + PluginMutationOwner::Idle + ); +} + +#[test] +fn activation_plan_verifies_resources_before_loading() { + let _guard = crate::shared_runtime::runtime_owner_test_mutex() + .lock() + .unwrap_or_else(|error| error.into_inner()); + if let Ok(mut owner) = PLUGIN_MUTATION_OWNER.lock() { + *owner = PluginMutationOwner::Idle; + } + let verify_count = Arc::new(AtomicUsize::new(0)); + let drop_count = Arc::new(AtomicUsize::new(0)); + let resource = Arc::new(TrackingActivationResource { + verify_count: Arc::clone(&verify_count), + drop_count: Arc::clone(&drop_count), + fail_verification: true, + }); + let (_fixture, manifest_ref) = invalid_native_library_fixture("fixture.snapshot"); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("file-backed activation runtime should build"); + let result = runtime.block_on(PluginHostActivation::activate_plan( + PluginHostActivationPlan { + config: PluginConfig::default(), + dynamic_plugins: vec![PlannedDynamicPluginActivation { + spec: DynamicPluginActivationSpec { + plugin_id: "fixture.snapshot".into(), + kind: DynamicPluginKind::RustDynamic, + manifest_ref, + environment_ref: None, + config: Map::new(), + }, + resource, + }], + diagnostics: Vec::new(), + }, + )); + let error = match result { + Ok((activation, _)) => { + activation + .clear() + .expect("unexpected resource activation should clear"); + panic!("resource verification failure should prevent loading"); + } + Err(error) => error.to_string(), + }; + + assert!(error.contains("activation snapshot changed"), "{error}"); + assert_eq!(verify_count.load(Ordering::SeqCst), 1); + assert_eq!(drop_count.load(Ordering::SeqCst), 1); + assert_eq!( + *PLUGIN_MUTATION_OWNER.lock().unwrap(), + PluginMutationOwner::Idle + ); +} + +#[test] +fn unpolled_activation_plan_drops_resources_without_enqueuing() { + let _guard = crate::shared_runtime::runtime_owner_test_mutex() + .lock() + .unwrap_or_else(|error| error.into_inner()); + if let Ok(mut owner) = PLUGIN_MUTATION_OWNER.lock() { + *owner = PluginMutationOwner::Idle; + } + let verify_count = Arc::new(AtomicUsize::new(0)); + let drop_count = Arc::new(AtomicUsize::new(0)); + let resource = Arc::new(TrackingActivationResource { + verify_count: Arc::clone(&verify_count), + drop_count: Arc::clone(&drop_count), + fail_verification: false, + }); + + let activation = PluginHostActivation::activate_plan(PluginHostActivationPlan { + config: PluginConfig::default(), + dynamic_plugins: vec![PlannedDynamicPluginActivation { + spec: DynamicPluginActivationSpec { + plugin_id: "fixture.unpolled".into(), + kind: DynamicPluginKind::RustDynamic, + manifest_ref: "unpolled-relay-plugin.toml".into(), + environment_ref: None, + config: Map::new(), + }, + resource, + }], + diagnostics: Vec::new(), + }); + drop(activation); + + assert_eq!(verify_count.load(Ordering::SeqCst), 0); + assert_eq!(drop_count.load(Ordering::SeqCst), 1); + assert_eq!( + *PLUGIN_MUTATION_OWNER.lock().unwrap(), + PluginMutationOwner::Idle + ); +} + #[test] fn plugin_error_context_preserves_each_error_class() { use crate::plugin::PluginError; diff --git a/crates/core/tests/unit/plugin_dynamic_tests.rs b/crates/core/tests/unit/plugin_dynamic_tests.rs index fb5d721a0..514722525 100644 --- a/crates/core/tests/unit/plugin_dynamic_tests.rs +++ b/crates/core/tests/unit/plugin_dynamic_tests.rs @@ -5,6 +5,7 @@ use super::*; use crate::plugin::PluginError; +use chrono::Utc; use std::fs; use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; @@ -154,6 +155,136 @@ fn registry_rejects_duplicate_live_plugin_ids() { } } +#[test] +fn registry_manifest_refresh_preserves_lifecycle_ownership() { + let existing = sample_record(); + let mut registry = DynamicPluginRegistry::from_records(vec![existing.clone()]).unwrap(); + let mut refreshed = existing.clone(); + refreshed.metadata.name = Some("Refreshed PII Guardrails".into()); + refreshed.metadata.version = Some("0.2.0".into()); + refreshed.metadata.generation = 0; + refreshed.metadata.created_at = Some("replacement-created".into()); + refreshed.metadata.updated_at = Some("replacement-updated".into()); + refreshed.source.manifest_ref = Some("/plugins/pii-v2/relay-plugin.toml".into()); + refreshed.source.artifact_ref = Some("/plugins/pii-v2/dist/pii.whl".into()); + refreshed.source.environment_ref = Some("/plugins/pii-v2/.venv".into()); + refreshed.source.artifact_digest = Some("sha256:def456".into()); + refreshed.spec = DynamicPluginSpec::default(); + refreshed.compatibility = + DynamicPluginCompatibility::Worker(DynamicPluginWorkerCompatibility { + relay: ">=0.7.0,<0.8.0".into(), + worker_protocol: "grpc-v2".into(), + }); + refreshed.load = DynamicPluginLoadContract::Worker(DynamicPluginWorkerLoadContract { + runtime: WorkerRuntime::Command, + entrypoint: "/plugins/pii-v2/bin/worker".into(), + }); + refreshed.status.validation.checked_at = Some("2000-01-01T00:00:00Z".into()); + refreshed.status.validation.message = Some("refreshed validation".into()); + refreshed.status.runtime = DynamicPluginRuntimeStatus::default(); + refreshed.status.startup_class = Some(DynamicPluginStartupClass::Required); + refreshed.status.attestation_mode = Some(DynamicPluginAttestationMode::SignatureRequired); + + let refresh_started_at = Utc::now(); + let result = registry + .refresh_manifest_record("acme.guardrails.pii", refreshed) + .unwrap(); + let refresh_completed_at = Utc::now(); + + assert_eq!( + result.metadata.name.as_deref(), + Some("Refreshed PII Guardrails") + ); + assert_eq!(result.metadata.version.as_deref(), Some("0.2.0")); + assert_eq!(result.metadata.generation, existing.metadata.generation); + assert_eq!(result.metadata.created_at, existing.metadata.created_at); + assert_eq!(result.metadata.updated_at, existing.metadata.updated_at); + assert_eq!( + result.source.manifest_ref.as_deref(), + Some("/plugins/pii-v2/relay-plugin.toml") + ); + assert_eq!( + result.source.artifact_ref.as_deref(), + Some("/plugins/pii-v2/dist/pii.whl") + ); + assert_eq!( + result.source.artifact_digest.as_deref(), + Some("sha256:def456") + ); + assert_eq!( + result.source.environment_ref, + existing.source.environment_ref + ); + assert_eq!(result.spec, existing.spec); + assert_eq!(result.status.runtime, existing.status.runtime); + let checked_at = result + .status + .validation + .checked_at + .as_deref() + .and_then(|value| chrono::DateTime::parse_from_rfc3339(value).ok()) + .expect("manifest refresh should stamp validation checked_at"); + assert!(checked_at >= refresh_started_at); + assert!(checked_at <= refresh_completed_at); + assert_eq!( + result.status.validation.message.as_deref(), + Some("refreshed validation") + ); + assert_eq!( + result.status.startup_class, + Some(DynamicPluginStartupClass::Required) + ); + assert_eq!( + result.status.attestation_mode, + Some(DynamicPluginAttestationMode::SignatureRequired) + ); +} + +#[test] +fn registry_manifest_refresh_preserves_tombstone_and_generation() { + let mut existing = sample_record(); + existing.metadata.generation = 19; + existing.spec.present = false; + existing.spec.enabled = false; + let mut registry = DynamicPluginRegistry::from_records(vec![existing.clone()]).unwrap(); + let mut refreshed = sample_record(); + refreshed.metadata.generation = 0; + refreshed.spec.present = true; + refreshed.spec.enabled = true; + + let result = registry + .refresh_manifest_record("acme.guardrails.pii", refreshed) + .unwrap(); + + assert!(result.is_tombstoned()); + assert!(!result.spec.enabled); + assert_eq!(result.metadata.generation, 19); +} + +#[test] +fn registry_manifest_refresh_rejects_invalid_identity_without_mutation() { + let existing = sample_record(); + let mut registry = DynamicPluginRegistry::from_records(vec![existing.clone()]).unwrap(); + let mut mismatched = sample_record(); + mismatched.metadata.id = "other.plugin".into(); + let error = registry + .refresh_manifest_record("acme.guardrails.pii", mismatched) + .unwrap_err(); + assert!(error.to_string().contains("does not match lifecycle id")); + assert_eq!(registry.get("acme.guardrails.pii"), Some(&existing)); + + let mut invalid = sample_record(); + invalid.load = DynamicPluginLoadContract::Worker(DynamicPluginWorkerLoadContract { + runtime: WorkerRuntime::Python, + entrypoint: String::new(), + }); + let error = registry + .refresh_manifest_record("acme.guardrails.pii", invalid) + .unwrap_err(); + assert!(error.to_string().contains("load shape")); + assert_eq!(registry.get("acme.guardrails.pii"), Some(&existing)); +} + #[test] fn registry_rejects_invalid_raw_record_shapes() { let mut registry = DynamicPluginRegistry::new(); diff --git a/crates/core/tests/unit/plugin_tests.rs b/crates/core/tests/unit/plugin_tests.rs index 8f483143f..9ba575e51 100644 --- a/crates/core/tests/unit/plugin_tests.rs +++ b/crates/core/tests/unit/plugin_tests.rs @@ -40,6 +40,7 @@ struct BackgroundTaskPlugin { struct PanickingPlugin; struct FailingDeregisterPlugin; struct PluginMutationOwnerCleanup; +struct ActivePluginConfigurationPoisonCleanup; impl Drop for PluginMutationOwnerCleanup { fn drop(&mut self) { @@ -50,6 +51,15 @@ impl Drop for PluginMutationOwnerCleanup { } } +impl Drop for ActivePluginConfigurationPoisonCleanup { + fn drop(&mut self) { + ACTIVE_PLUGIN_CONFIGURATION.clear_poison(); + if let Ok(mut active) = ACTIVE_PLUGIN_CONFIGURATION.lock() { + *active = None; + } + } +} + static RECORDED_NAMES: OnceLock>> = OnceLock::new(); static PARTIAL_FAIL_ROLLBACKS: AtomicUsize = AtomicUsize::new(0); static RESTORE_FAIL_REGISTRATIONS: AtomicUsize = AtomicUsize::new(0); @@ -1477,6 +1487,114 @@ fn test_pending_registration_records_rollback_failures() { assert!(failures[0].contains("rollback remained registered")); } +#[test] +fn test_active_configuration_store_failure_rolls_back_transferred_registrations() { + let _guard = lock_runtime_owner(); + reset_global(); + let _poison_cleanup = ActivePluginConfigurationPoisonCleanup; + let deregistration_attempts = Arc::new(AtomicUsize::new(0)); + let captured_attempts = Arc::clone(&deregistration_attempts); + let failures = Arc::new(Mutex::new(Vec::new())); + + std::thread::spawn(|| { + let _active = ACTIVE_PLUGIN_CONFIGURATION.lock().unwrap(); + panic!("poison active plugin configuration for store failure"); + }) + .join() + .expect_err("fixture active configuration writer should panic"); + + let error = store_active_plugin_configuration( + PluginConfig::default(), + ConfigReport::default(), + vec![PluginRegistration::new( + "fixture", + "transferred-callback", + Box::new(move || { + captured_attempts.fetch_add(1, Ordering::SeqCst); + Err(PluginError::RegistrationFailed( + "transferred callback remained registered".into(), + )) + }), + )], + Some(&failures), + ) + .expect_err("a poisoned active configuration store should fail"); + + assert!( + error + .to_string() + .contains("active plugin configuration lock poisoned"), + "{error}" + ); + assert_eq!(deregistration_attempts.load(Ordering::SeqCst), 1); + let failures = failures.lock().unwrap(); + assert_eq!(failures.len(), 1); + assert!(failures[0].contains("transferred-callback")); + assert!(failures[0].contains("remained registered")); +} + +#[test] +fn test_active_configuration_store_failure_releases_poisoned_guard_before_rollback() { + let _guard = lock_runtime_owner(); + reset_global(); + let _poison_cleanup = ActivePluginConfigurationPoisonCleanup; + let rollback_reentered = Arc::new(AtomicUsize::new(0)); + let captured_reentry = Arc::clone(&rollback_reentered); + + std::thread::spawn(|| { + let _active = ACTIVE_PLUGIN_CONFIGURATION.lock().unwrap(); + panic!("poison active plugin configuration for reentrant rollback test"); + }) + .join() + .expect_err("fixture active configuration writer should panic"); + + let error = store_active_plugin_configuration( + PluginConfig::default(), + ConfigReport::default(), + vec![PluginRegistration::new( + "fixture", + "reentrant-callback", + Box::new(move || { + match ACTIVE_PLUGIN_CONFIGURATION.try_lock() { + Err(std::sync::TryLockError::Poisoned(guard)) => { + drop(guard.into_inner()); + captured_reentry.fetch_add(1, Ordering::SeqCst); + } + Ok(_) => { + return Err(PluginError::RegistrationFailed( + "active configuration lock unexpectedly lost poison state".into(), + )); + } + Err(std::sync::TryLockError::WouldBlock) => { + return Err(PluginError::RegistrationFailed( + "active configuration lock remained held during rollback".into(), + )); + } + } + record_active_plugin_runtime_diagnostic(RuntimeDiagnostic { + code: "fixture.reentrant_rollback".into(), + component: "fixture".into(), + field: None, + message: "rollback re-entered active configuration state".into(), + session_id: None, + count: 1, + }); + Ok(()) + }), + )], + None, + ) + .expect_err("a poisoned active configuration store should fail"); + + assert!( + error + .to_string() + .contains("active plugin configuration lock poisoned"), + "{error}" + ); + assert_eq!(rollback_reentered.load(Ordering::SeqCst), 1); +} + #[test] fn test_checked_teardown_reports_unremoved_registrations() { let _guard = lock_runtime_owner(); @@ -1493,6 +1611,7 @@ fn test_checked_teardown_reports_unremoved_registrations() { )) }), )], + None, ) .unwrap(); @@ -1530,6 +1649,7 @@ fn test_teardown_runtime_diagnostics_remain_in_the_plugin_report() { ))) }), )], + None, ) .unwrap(); @@ -1562,6 +1682,7 @@ fn test_legacy_clear_retains_mutation_owner_after_incomplete_teardown() { "stale-callback", Box::new(|| panic!("fixture deregistration panicked")), )], + None, ) .unwrap(); diff --git a/crates/plugin-host-config/Cargo.toml b/crates/plugin-host-config/Cargo.toml new file mode 100644 index 000000000..a491b39ec --- /dev/null +++ b/crates/plugin-host-config/Cargo.toml @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "nemo-relay-plugin-host-config" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Shared file-backed dynamic plugin host configuration for NeMo Relay." + +[lints] +workspace = true + +[dependencies] +nemo-relay = { workspace = true, features = ["worker-grpc"] } +base64 = "0.22" +dunce = "1" +fs2 = "0.4" +libc = "0.2" +log = "0.4" +ring = "0.17" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.11" +thiserror = "2" +tokio = { version = "1", features = ["rt"] } +toml = "0.9" +uuid = { workspace = true, features = ["v7"] } + +[dev-dependencies] +tempfile = "3" + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_Storage_FileSystem", "Win32_System_Threading"] } diff --git a/crates/plugin-host-config/src/activation.rs b/crates/plugin-host-config/src/activation.rs new file mode 100644 index 000000000..d2183b240 --- /dev/null +++ b/crates/plugin-host-config/src/activation.rs @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::PathBuf; + +use nemo_relay::plugin::dynamic::{PluginHostActivation, PluginHostActivationPlan}; +use nemo_relay::plugin::{ConfigReport, PluginConfig, PluginError}; + +use crate::error::{PluginHostConfigError, Result}; +use crate::lifecycle::prepare_plugin_host_activation; +use crate::resolver::{PluginFileResolveOptions, resolve_plugin_files}; + +/// Owns one static or dynamic plugin configuration initialized from plugin files. +/// +/// An inactive handle represents successful discovery with no physical input and no caller +/// configuration. Active handles retain Relay's process-wide plugin host lease until cleared. +#[must_use = "dropping the file activation starts best-effort plugin host teardown"] +pub struct PluginFileActivation { + host: Option, + report: ConfigReport, +} + +impl PluginFileActivation { + /// Returns the configuration and runtime diagnostics produced during activation. + pub fn report(&self) -> &ConfigReport { + &self.report + } + + /// Returns whether this handle owns an active process-wide plugin host. + pub fn is_active(&self) -> bool { + self.host + .as_ref() + .is_some_and(PluginHostActivation::is_active) + } + + /// Clears configured callbacks before unloading dynamic runtimes and releasing the host. + pub fn clear(mut self) -> Result<()> { + if let Some(host) = self.host.take() { + host.clear()?; + } + Ok(()) + } + + /// Activates an already resolved and snapshotted file-backed host plan. + /// + /// Embedding hosts that participate in configuration resolution before runtime startup use + /// this path to preserve the exact snapshot resources used for bootstrap identity. + #[doc(hidden)] + pub async fn activate_plan(plan: PluginHostActivationPlan) -> Result { + let (host, report) = PluginHostActivation::activate_plan(plan).await?; + Ok(Self { + host: Some(host), + report, + }) + } + + fn inactive() -> Self { + Self { + host: None, + report: ConfigReport::default(), + } + } +} + +/// Resolves and activates static components and enabled dynamic plugins from `plugins.toml`. +/// +/// `plugin_config_path` replaces the ambient user-level file while project and system layers +/// continue to participate. The optional typed `config` is the highest-precedence static overlay. +/// Dynamic enablement remains controlled by each source's sibling `.dynamic-plugins.json`. +pub async fn initialize_from_plugins_toml( + config: Option, + plugin_config_path: Option, +) -> Result { + initialize_from_plugins_toml_with_options( + config, + PluginFileResolveOptions::from_environment(plugin_config_path), + ) + .await +} + +async fn initialize_from_plugins_toml_with_options( + config: Option, + options: PluginFileResolveOptions, +) -> Result { + let resolved = tokio::task::spawn_blocking(move || resolve_plugin_files(config, options)) + .await + .map_err(|error| { + PluginHostConfigError::Relay(PluginError::Internal(format!( + "plugin file resolution task failed: {error}" + ))) + })??; + + if !resolved.had_input { + return Ok(PluginFileActivation::inactive()); + } + + let plan = tokio::task::spawn_blocking(move || prepare_plugin_host_activation(resolved)) + .await + .map_err(|error| { + PluginHostConfigError::Relay(PluginError::Internal(format!( + "plugin lifecycle preparation task failed: {error}" + ))) + })??; + PluginFileActivation::activate_plan(plan).await +} + +#[cfg(test)] +#[path = "../tests/unit/activation.rs"] +mod tests; diff --git a/crates/plugin-host-config/src/environment.rs b/crates/plugin-host-config/src/environment.rs new file mode 100644 index 000000000..21973850a --- /dev/null +++ b/crates/plugin-host-config/src/environment.rs @@ -0,0 +1,912 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(windows)] +use std::fs::File; +#[cfg(not(windows))] +use std::fs::OpenOptions; +#[cfg(windows)] +use std::io; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::{Duration, Instant}; + +use fs2::FileExt; +use nemo_relay::plugin::dynamic::{ + DynamicPluginCheckState, DynamicPluginManifest, DynamicPluginManifestLoad, WorkerRuntime, +}; +use ring::hmac; +use ring::rand::{SecureRandom, SystemRandom}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +use crate::error::{PluginHostConfigError, Result}; +use crate::io::read_bounded_utf8_regular_file; +use crate::io::{MAX_BOUNDED_FILE_BYTES, read_bounded_regular_file}; + +#[doc(hidden)] +pub const MANAGED_ENVIRONMENTS_DIR: &str = ".dynamic-plugin-environments"; +#[doc(hidden)] +pub const ENVIRONMENT_ATTESTATION_FILE: &str = ".nemo-relay-environment.sha256"; +const MAX_ENVIRONMENT_FILES: usize = 100_000; +const MAX_ENVIRONMENT_DEPTH: usize = 128; +const HMAC_KEY_BYTES: usize = 32; +const HMAC_LOCK_TIMEOUT: Duration = Duration::from_secs(5); +const PYTHON_ENVIRONMENT_ATTESTATION_DOMAIN: &[u8] = + b"nemo-relay/python-environment-attestation/v1\0"; + +#[derive(Deserialize)] +struct EnvironmentAttestation { + version: u8, + source_artifact_sha256: String, + environment_sha256: String, + authentication: String, +} + +#[doc(hidden)] +pub fn validate_python_entrypoint_artifact( + manifest: &DynamicPluginManifest, + manifest_ref: &str, +) -> std::result::Result<(), String> { + let DynamicPluginManifestLoad::Worker(load) = &manifest.load else { + return Ok(()); + }; + if load.runtime != Some(WorkerRuntime::Python) { + return Ok(()); + } + let source = manifest.source.as_ref().ok_or_else(|| { + "Python worker plugins must declare source.manifest_root and source.artifact".to_owned() + })?; + let manifest_root = source + .manifest_root + .as_deref() + .map(str::trim) + .filter(|root| !root.is_empty()) + .ok_or_else(|| "Python worker plugins must declare source.manifest_root".to_owned())?; + let artifact = source + .artifact + .as_deref() + .map(str::trim) + .filter(|artifact| !artifact.is_empty()) + .ok_or_else(|| "Python worker plugins must declare source.artifact".to_owned())?; + let entrypoint = load + .entrypoint + .as_deref() + .map(str::trim) + .filter(|entrypoint| !entrypoint.is_empty()) + .ok_or_else(|| "Python worker plugins must declare load.entrypoint".to_owned())?; + let (module, callable) = entrypoint.split_once(':').ok_or_else(|| { + format!( + "Python worker load.entrypoint '{entrypoint}' must use the unambiguous module:function form" + ) + })?; + if callable.is_empty() + || callable.contains(':') + || module.is_empty() + || module + .split('.') + .any(|segment| segment.is_empty() || segment.contains(['/', '\\', ':'])) + { + return Err(format!( + "Python worker load.entrypoint '{entrypoint}' must use the unambiguous module:function form" + )); + } + let manifest_path = Path::new(manifest_ref); + let manifest_dir = manifest_path.parent().unwrap_or_else(|| Path::new(".")); + let unresolved_manifest_root = resolve_relative_path(manifest_dir, manifest_root); + let manifest_root = unresolved_manifest_root.canonicalize().map_err(|error| { + format!( + "could not resolve Python plugin source.manifest_root {}: {error}", + unresolved_manifest_root.display() + ) + })?; + let artifact = resolve_relative_path(manifest_dir, artifact) + .canonicalize() + .map_err(|error| format!("could not resolve Python source.artifact: {error}"))?; + let module_path = module + .split('.') + .fold(manifest_root, |path, segment| path.join(segment)); + let module_file = module_path.with_extension("py"); + let package_file = module_path.join("__init__.py"); + let mut candidates = [module_file, package_file] + .into_iter() + .filter(|path| path.is_file()) + .map(|path| { + path.canonicalize().map_err(|error| { + format!( + "could not resolve Python entrypoint module file {}: {error}", + path.display() + ) + }) + }) + .collect::, _>>()?; + candidates.sort(); + candidates.dedup(); + let [entrypoint_artifact] = candidates.as_slice() else { + return Err(format!( + "Python worker load.entrypoint '{entrypoint}' must resolve to exactly one source module under source.manifest_root; expected {} or {}", + module_path.with_extension("py").display(), + module_path.join("__init__.py").display() + )); + }; + if entrypoint_artifact != &artifact { + return Err(format!( + "Python worker load.entrypoint '{entrypoint}' resolves to {}, but integrity-checked source.artifact resolves to {}; the executed entrypoint module must be the integrity-checked artifact", + entrypoint_artifact.display(), + artifact.display() + )); + } + Ok(()) +} + +#[doc(hidden)] +pub fn environment_state( + manifest: &DynamicPluginManifest, + state_path: &Path, + environment_ref: Option<&str>, +) -> DynamicPluginCheckState { + validate_environment_state(manifest, state_path, environment_ref) + .unwrap_or(DynamicPluginCheckState::Invalid) +} + +#[doc(hidden)] +pub fn validate_environment_state( + manifest: &DynamicPluginManifest, + state_path: &Path, + environment_ref: Option<&str>, +) -> Result { + if !is_python_worker(manifest) { + return Ok(DynamicPluginCheckState::Unknown); + } + let environment_ref = environment_ref.ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin '{}' has no lifecycle-managed Python environment", + manifest.plugin.id + )) + })?; + let expected = managed_environment_path(state_path, &manifest.plugin.id)?; + let configured = absolute_path(Path::new(environment_ref))?; + let expected_metadata = std::fs::symlink_metadata(&expected).map_err(|error| { + PluginHostConfigError::io( + "inspect lifecycle-managed Python environment", + &expected, + error, + ) + })?; + if !expected_metadata.file_type().is_dir() { + return Err(PluginHostConfigError::InvalidConfig(format!( + "lifecycle-managed Python environment {} must be a directory and not a symbolic link", + expected.display() + ))); + } + let configured_metadata = std::fs::symlink_metadata(&configured).map_err(|error| { + PluginHostConfigError::io("inspect configured Python environment", &configured, error) + })?; + if !configured_metadata.file_type().is_dir() { + return Err(PluginHostConfigError::InvalidConfig(format!( + "configured Python environment {} must be a directory and not a symbolic link", + configured.display() + ))); + } + let same_physical_environment = configured == expected + || std::fs::canonicalize(&configured) + .ok() + .zip(std::fs::canonicalize(&expected).ok()) + .is_some_and(|(configured, expected)| configured == expected); + if !same_physical_environment { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin '{}' configured Python environment {} is not its lifecycle-managed environment {}", + manifest.plugin.id, + configured.display(), + expected.display() + ))); + } + let python = environment_python_path(&configured); + if !python.is_file() { + return Err(PluginHostConfigError::InvalidConfig(format!( + "managed Python environment {} has no Python launcher at {}", + configured.display(), + python.display() + ))); + } + let digest = manifest + .integrity + .as_ref() + .and_then(|integrity| integrity.sha256.as_deref()) + .ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "Python worker dynamic plugin '{}' requires integrity.sha256 for its lifecycle-managed environment", + manifest.plugin.id + )) + })?; + verify_environment_attestation(&configured, digest)?; + Ok(DynamicPluginCheckState::Valid) +} + +/// Reads and authenticates a lifecycle-managed environment attestation. +#[doc(hidden)] +pub fn read_environment_attestation( + environment: &Path, + expected_source_artifact_sha256: &str, +) -> Result { + let attestation_path = environment.join(ENVIRONMENT_ATTESTATION_FILE); + let raw = read_bounded_utf8_regular_file( + &attestation_path, + "managed Python environment attestation", + )?; + let attestation = serde_json::from_str::(&raw).map_err(|error| { + PluginHostConfigError::InvalidConfig(format!( + "managed Python environment attestation {} is invalid at line {}, column {}: {}", + attestation_path.display(), + error.line(), + error.column(), + crate::error::sanitize_parser_reason(&error.to_string()) + )) + })?; + if attestation.version != 1 + || attestation.source_artifact_sha256 != expected_source_artifact_sha256.trim() + || attestation.environment_sha256.len() != 64 + || !attestation + .environment_sha256 + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + return Err(PluginHostConfigError::InvalidConfig(format!( + "managed Python environment attestation {} does not match the trusted source artifact", + attestation_path.display() + ))); + } + if !verify_environment_authentication( + &attestation.source_artifact_sha256, + &attestation.environment_sha256, + &attestation.authentication, + )? { + return Err(PluginHostConfigError::InvalidConfig(format!( + "managed Python environment attestation {} failed authentication", + attestation_path.display() + ))); + } + Ok(attestation.environment_sha256) +} + +/// Authenticates an environment attestation and verifies the complete environment tree. +#[doc(hidden)] +pub fn verify_environment_attestation( + environment: &Path, + expected_source_artifact_sha256: &str, +) -> Result { + let expected = read_environment_attestation(environment, expected_source_artifact_sha256)?; + let actual = environment_tree_digest(environment)?; + if actual != expected { + return Err(PluginHostConfigError::InvalidConfig(format!( + "managed Python environment {} changed after provisioning", + environment.display() + ))); + } + Ok(actual) +} + +fn is_python_worker(manifest: &DynamicPluginManifest) -> bool { + matches!( + &manifest.load, + DynamicPluginManifestLoad::Worker(load) + if load.runtime == Some(WorkerRuntime::Python) + ) +} + +fn environment_python_path(environment: &Path) -> PathBuf { + if cfg!(windows) { + environment.join("Scripts").join("python.exe") + } else { + environment.join("bin").join("python") + } +} + +fn managed_environment_path(state_path: &Path, plugin_id: &str) -> Result { + let state_path = absolute_path(state_path)?; + let parent = state_path.parent().ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin lifecycle state {} has no parent directory", + state_path.display() + )) + })?; + let digest = Sha256::digest(plugin_id.trim().as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + Ok(parent.join(MANAGED_ENVIRONMENTS_DIR).join(digest)) +} + +fn resolve_relative_path(base: &Path, value: &str) -> PathBuf { + let path = PathBuf::from(value); + if path.is_absolute() { + path + } else { + base.join(path) + } +} + +fn absolute_path(path: &Path) -> Result { + if path.is_absolute() { + Ok(path.to_path_buf()) + } else { + std::env::current_dir() + .map(|current| current.join(path)) + .map_err(|error| PluginHostConfigError::io("resolve path", path, error)) + } +} + +fn environment_tree_digest(environment: &Path) -> Result { + let mut digest = Sha256::new(); + let mut total = 0_u64; + let mut entries = 0_usize; + digest_environment_directory( + environment, + Path::new(""), + &mut Vec::new(), + &mut digest, + &mut total, + &mut entries, + )?; + Ok(digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect()) +} + +fn digest_environment_directory( + directory: &Path, + relative_directory: &Path, + ancestors: &mut Vec, + digest: &mut Sha256, + total: &mut u64, + entries: &mut usize, +) -> Result<()> { + if ancestors.len() >= MAX_ENVIRONMENT_DEPTH { + return Err(PluginHostConfigError::InvalidConfig(format!( + "managed Python environment exceeds the {MAX_ENVIRONMENT_DEPTH}-directory traversal depth at {}", + directory.display() + ))); + } + let canonical = std::fs::canonicalize(directory).map_err(|error| { + PluginHostConfigError::io("normalize environment directory", directory, error) + })?; + if ancestors.contains(&canonical) { + return Err(PluginHostConfigError::InvalidConfig(format!( + "managed Python environment contains a directory symlink cycle at {}", + directory.display() + ))); + } + ancestors.push(canonical.clone()); + let mut children = std::fs::read_dir(&canonical) + .map_err(|error| { + PluginHostConfigError::io("read environment directory", &canonical, error) + })? + .collect::, _>>() + .map_err(|error| { + PluginHostConfigError::io("read environment directory entry", &canonical, error) + })?; + *entries = entries.saturating_add(children.len()); + if *entries > MAX_ENVIRONMENT_FILES { + return Err(PluginHostConfigError::InvalidConfig(format!( + "managed Python environment exceeds the {MAX_ENVIRONMENT_FILES}-entry attestation budget at {}", + directory.display() + ))); + } + children.sort_by_key(std::fs::DirEntry::file_name); + for child in children { + let path = child.path(); + let relative = relative_directory.join(child.file_name()); + if relative == Path::new(ENVIRONMENT_ATTESTATION_FILE) + || path.file_name().and_then(|name| name.to_str()) == Some("__pycache__") + || path.extension().and_then(|extension| extension.to_str()) == Some("pyc") + { + continue; + } + let source = resolve_environment_entry(&path)?; + let metadata = std::fs::metadata(&source).map_err(|error| { + PluginHostConfigError::io("inspect environment entry", &source, error) + })?; + if metadata.is_dir() { + update_tree_digest(digest, b'd', &relative, &[]); + digest_environment_directory(&source, &relative, ancestors, digest, total, entries)?; + } else if metadata.is_file() { + let bytes = read_bounded_regular_file(&source, "managed Python environment file")?; + *total = total.saturating_add(bytes.len() as u64); + if *total > MAX_BOUNDED_FILE_BYTES { + return Err(PluginHostConfigError::InvalidConfig(format!( + "managed Python environment exceeds the {MAX_BOUNDED_FILE_BYTES}-byte attestation budget" + ))); + } + update_tree_digest(digest, b'f', &relative, &bytes); + } else { + return Err(PluginHostConfigError::InvalidConfig(format!( + "managed Python environment entry {} must resolve to a regular file or directory", + path.display() + ))); + } + } + ancestors.pop(); + Ok(()) +} + +fn resolve_environment_entry(path: &Path) -> Result { + let metadata = std::fs::symlink_metadata(path) + .map_err(|error| PluginHostConfigError::io("inspect environment entry", path, error))?; + if metadata.file_type().is_symlink() { + std::fs::canonicalize(path) + .map_err(|error| PluginHostConfigError::io("resolve environment symlink", path, error)) + } else { + Ok(path.to_path_buf()) + } +} + +fn update_tree_digest(digest: &mut Sha256, entry_type: u8, path: &Path, payload: &[u8]) { + let path = raw_path_bytes(path); + digest.update([entry_type]); + digest.update((path.len() as u64).to_le_bytes()); + digest.update(&path); + digest.update((payload.len() as u64).to_le_bytes()); + digest.update(payload); +} + +#[cfg(unix)] +fn raw_path_bytes(path: &Path) -> Vec { + use std::os::unix::ffi::OsStrExt; + path.as_os_str().as_bytes().to_vec() +} + +#[cfg(windows)] +fn raw_path_bytes(path: &Path) -> Vec { + use std::os::windows::ffi::OsStrExt; + path.as_os_str() + .encode_wide() + .flat_map(u16::to_le_bytes) + .collect() +} + +fn verify_environment_authentication( + source_artifact_sha256: &str, + environment_sha256: &str, + authentication: &str, +) -> Result { + let Some(encoded) = authentication.strip_prefix("hmac-sha256:") else { + return Ok(false); + }; + let Some(tag) = decode_fixed_hex::<32>(encoded) else { + return Ok(false); + }; + let key = hmac::Key::new(hmac::HMAC_SHA256, &load_or_create_hmac_key()?); + Ok(hmac::verify( + &key, + &environment_attestation_message(source_artifact_sha256, environment_sha256), + &tag, + ) + .is_ok()) +} + +fn environment_attestation_message( + source_artifact_sha256: &str, + environment_sha256: &str, +) -> Vec { + let mut message = Vec::with_capacity( + PYTHON_ENVIRONMENT_ATTESTATION_DOMAIN.len() + + source_artifact_sha256.len() + + environment_sha256.len() + + 1, + ); + message.extend_from_slice(PYTHON_ENVIRONMENT_ATTESTATION_DOMAIN); + message.extend_from_slice(source_artifact_sha256.trim().as_bytes()); + message.push(0); + message.extend_from_slice(environment_sha256.as_bytes()); + message +} + +fn decode_fixed_hex(encoded: &str) -> Option<[u8; N]> { + if encoded.len() != N * 2 || !encoded.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return None; + } + let mut decoded = [0_u8; N]; + for (index, byte) in decoded.iter_mut().enumerate() { + *byte = u8::from_str_radix(&encoded[index * 2..index * 2 + 2], 16).ok()?; + } + Some(decoded) +} + +fn load_or_create_hmac_key() -> Result<[u8; HMAC_KEY_BYTES]> { + let path = nemo_relay::plugin::user_config_dir() + .map(|directory| directory.join("bootstrap").join("fingerprint-hmac.key")) + .ok_or_else(|| { + PluginHostConfigError::InvalidConfig( + "cannot determine the per-user NeMo Relay bootstrap state directory; set HOME or USERPROFILE" + .into(), + ) + })?; + load_or_create_hmac_key_at(&path) +} + +fn load_or_create_hmac_key_at(path: &Path) -> Result<[u8; HMAC_KEY_BYTES]> { + let parent = path.parent().expect("bootstrap HMAC key has a parent"); + std::fs::create_dir_all(parent).map_err(|error| { + PluginHostConfigError::io("create bootstrap state directory", parent, error) + })?; + #[cfg(unix)] + std::fs::set_permissions(parent, { + use std::os::unix::fs::PermissionsExt; + std::fs::Permissions::from_mode(0o700) + }) + .map_err(|error| { + PluginHostConfigError::io("protect bootstrap state directory", parent, error) + })?; + + #[cfg(windows)] + protect_private_windows_path(parent).map_err(|error| { + PluginHostConfigError::io("protect bootstrap state directory", parent, error) + })?; + + #[cfg(windows)] + let mut file = open_private_windows_file(path) + .map_err(|error| PluginHostConfigError::io("open bootstrap HMAC key", path, error))?; + #[cfg(not(windows))] + let mut file = { + let mut options = OpenOptions::new(); + options.create(true).truncate(false).read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + options + .open(path) + .map_err(|error| PluginHostConfigError::io("open bootstrap HMAC key", path, error))? + }; + let deadline = Instant::now() + HMAC_LOCK_TIMEOUT; + loop { + match file.try_lock_exclusive() { + Ok(()) => break, + Err(error) + if error.kind() == std::io::ErrorKind::WouldBlock && Instant::now() < deadline => + { + thread::sleep(Duration::from_millis(25)); + } + Err(error) => { + return Err(PluginHostConfigError::io( + "lock bootstrap HMAC key", + path, + error, + )); + } + } + } + #[cfg(unix)] + file.set_permissions({ + use std::os::unix::fs::PermissionsExt; + std::fs::Permissions::from_mode(0o600) + }) + .map_err(|error| PluginHostConfigError::io("protect bootstrap HMAC key", path, error))?; + let length = file + .metadata() + .map_err(|error| PluginHostConfigError::io("inspect bootstrap HMAC key", path, error))? + .len(); + if length == 0 { + let mut key = [0_u8; HMAC_KEY_BYTES]; + SystemRandom::new().fill(&mut key).map_err(|_| { + PluginHostConfigError::InvalidConfig("failed to generate bootstrap HMAC key".into()) + })?; + file.write_all(&key) + .map_err(|error| PluginHostConfigError::io("write bootstrap HMAC key", path, error))?; + file.sync_all() + .map_err(|error| PluginHostConfigError::io("sync bootstrap HMAC key", path, error))?; + return Ok(key); + } + if length != HMAC_KEY_BYTES as u64 { + return Err(PluginHostConfigError::InvalidConfig(format!( + "bootstrap HMAC key {} has invalid length {length}; expected {HMAC_KEY_BYTES} bytes", + path.display() + ))); + } + file.seek(SeekFrom::Start(0)) + .map_err(|error| PluginHostConfigError::io("seek bootstrap HMAC key", path, error))?; + let mut key = [0_u8; HMAC_KEY_BYTES]; + file.read_exact(&mut key) + .map_err(|error| PluginHostConfigError::io("read bootstrap HMAC key", path, error))?; + Ok(key) +} + +#[cfg(windows)] +fn open_private_windows_file(path: &Path) -> io::Result { + use windows_sys::Win32::Foundation::{GENERIC_READ, GENERIC_WRITE}; + use windows_sys::Win32::Storage::FileSystem::{ + FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, OPEN_ALWAYS, + }; + + let file = with_private_windows_descriptor(|descriptor| { + open_windows_file( + path, + descriptor, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + OPEN_ALWAYS, + ) + })?; + protect_private_windows_path(path)?; + Ok(file) +} + +#[cfg(windows)] +pub(crate) fn create_private_windows_file(path: &Path) -> io::Result { + use windows_sys::Win32::Foundation::GENERIC_WRITE; + use windows_sys::Win32::Storage::FileSystem::CREATE_NEW; + + with_private_windows_descriptor(|descriptor| { + open_windows_file(path, descriptor, GENERIC_WRITE, 0, CREATE_NEW) + }) +} + +#[cfg(windows)] +fn protect_private_windows_path(path: &Path) -> io::Result<()> { + use windows_sys::Win32::Security::{ + DACL_SECURITY_INFORMATION, PROTECTED_DACL_SECURITY_INFORMATION, SetFileSecurityW, + }; + + if !windows_path_owned_by_current_user(path)? { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!("{} is not owned by the current user", path.display()), + )); + } + let path_wide = windows_wide(path.as_os_str()); + with_private_windows_descriptor(|descriptor| { + // SAFETY: The path and descriptor remain valid for the duration of the call. + if unsafe { + SetFileSecurityW( + path_wide.as_ptr(), + DACL_SECURITY_INFORMATION | PROTECTED_DACL_SECURITY_INFORMATION, + descriptor, + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + Ok(()) + })?; + if !windows_path_is_private(path)? { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "failed to verify protected owner/System access on {}", + path.display() + ), + )); + } + Ok(()) +} + +#[cfg(windows)] +fn with_private_windows_descriptor( + operation: impl FnOnce(windows_sys::Win32::Security::PSECURITY_DESCRIPTOR) -> io::Result, +) -> io::Result { + use windows_sys::Win32::Foundation::LocalFree; + use windows_sys::Win32::Security::Authorization::{ + ConvertStringSecurityDescriptorToSecurityDescriptorW, SDDL_REVISION_1, + }; + use windows_sys::Win32::Security::PSECURITY_DESCRIPTOR; + + let descriptor_sddl = windows_wide("D:P(A;;FA;;;OW)(A;;FA;;;SY)"); + let mut descriptor: PSECURITY_DESCRIPTOR = std::ptr::null_mut(); + // SAFETY: The SDDL string is NUL-terminated and `descriptor` points to writable storage. + if unsafe { + ConvertStringSecurityDescriptorToSecurityDescriptorW( + descriptor_sddl.as_ptr(), + SDDL_REVISION_1, + &mut descriptor, + std::ptr::null_mut(), + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + let result = operation(descriptor); + // SAFETY: The descriptor was allocated by the conversion API and is still owned here. + unsafe { LocalFree(descriptor.cast()) }; + result +} + +#[cfg(windows)] +fn windows_path_owned_by_current_user(path: &Path) -> io::Result { + use windows_sys::Win32::Foundation::{CloseHandle, HANDLE}; + use windows_sys::Win32::Security::{ + EqualSid, GetSecurityDescriptorOwner, GetTokenInformation, OWNER_SECURITY_INFORMATION, + PSID, TOKEN_QUERY, TOKEN_USER, TokenUser, + }; + use windows_sys::Win32::System::Threading::{GetCurrentProcess, OpenProcessToken}; + + let mut descriptor = read_windows_security_descriptor(path, OWNER_SECURITY_INFORMATION)?; + let mut owner: PSID = std::ptr::null_mut(); + let mut defaulted = 0; + // SAFETY: The descriptor and output storage remain valid for the duration of the call. + if unsafe { + GetSecurityDescriptorOwner(descriptor.as_mut_ptr().cast(), &mut owner, &mut defaulted) + } == 0 + || owner.is_null() + { + return Err(io::Error::last_os_error()); + } + + let mut token: HANDLE = std::ptr::null_mut(); + // SAFETY: GetCurrentProcess returns a valid pseudo-handle and `token` is writable. + if unsafe { OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &mut token) } == 0 { + return Err(io::Error::last_os_error()); + } + let result = (|| { + let mut required = 0; + // SAFETY: This sizing call intentionally supplies a null output buffer. + unsafe { GetTokenInformation(token, TokenUser, std::ptr::null_mut(), 0, &mut required) }; + if required == 0 { + return Err(io::Error::last_os_error()); + } + let word = std::mem::size_of::(); + let mut buffer = vec![0_usize; (required as usize).div_ceil(word)]; + // SAFETY: The aligned buffer has at least `required` writable bytes. + if unsafe { + GetTokenInformation( + token, + TokenUser, + buffer.as_mut_ptr().cast(), + required, + &mut required, + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + // SAFETY: GetTokenInformation initialized a TOKEN_USER at this aligned address. + let user = unsafe { &*buffer.as_ptr().cast::() }; + // SAFETY: Both SID pointers remain valid while their backing buffers are alive. + Ok(unsafe { EqualSid(owner, user.User.Sid) != 0 }) + })(); + // SAFETY: `token` is an owned handle returned by OpenProcessToken. + unsafe { CloseHandle(token) }; + result +} + +#[cfg(windows)] +fn windows_path_is_private(path: &Path) -> io::Result { + use windows_sys::Win32::Security::{DACL_SECURITY_INFORMATION, OWNER_SECURITY_INFORMATION}; + + if !windows_path_owned_by_current_user(path)? { + return Ok(false); + } + let mut actual = read_windows_security_descriptor( + path, + OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION, + )?; + let actual = windows_dacl_sddl(actual.as_mut_ptr().cast())?; + with_private_windows_descriptor(|expected| Ok(actual == windows_dacl_sddl(expected)?)) +} + +#[cfg(windows)] +fn windows_dacl_sddl( + descriptor: windows_sys::Win32::Security::PSECURITY_DESCRIPTOR, +) -> io::Result { + use windows_sys::Win32::Foundation::LocalFree; + use windows_sys::Win32::Security::Authorization::{ + ConvertSecurityDescriptorToStringSecurityDescriptorW, SDDL_REVISION_1, + }; + use windows_sys::Win32::Security::DACL_SECURITY_INFORMATION; + + let mut rendered = std::ptr::null_mut(); + let mut rendered_len = 0; + // SAFETY: The descriptor is valid and both output pointers reference writable storage. + if unsafe { + ConvertSecurityDescriptorToStringSecurityDescriptorW( + descriptor, + SDDL_REVISION_1, + DACL_SECURITY_INFORMATION, + &mut rendered, + &mut rendered_len, + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + // SAFETY: The API returned `rendered_len` initialized UTF-16 code units. + let value = String::from_utf16_lossy(unsafe { + std::slice::from_raw_parts(rendered, rendered_len as usize) + }) + .trim_end_matches('\0') + .to_string(); + // SAFETY: `rendered` was allocated by the conversion API above. + unsafe { LocalFree(rendered.cast()) }; + Ok(value) +} + +#[cfg(windows)] +fn read_windows_security_descriptor( + path: &Path, + information: windows_sys::Win32::Security::OBJECT_SECURITY_INFORMATION, +) -> io::Result> { + use windows_sys::Win32::Security::GetFileSecurityW; + + let path = windows_wide(path.as_os_str()); + let mut required = 0; + // SAFETY: This sizing call intentionally supplies a null output buffer. + unsafe { + GetFileSecurityW( + path.as_ptr(), + information, + std::ptr::null_mut(), + 0, + &mut required, + ) + }; + if required == 0 { + return Err(io::Error::last_os_error()); + } + let mut descriptor = vec![0_u8; required as usize]; + // SAFETY: The NUL-terminated path and allocated output buffer remain valid for the call. + if unsafe { + GetFileSecurityW( + path.as_ptr(), + information, + descriptor.as_mut_ptr().cast(), + required, + &mut required, + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + Ok(descriptor) +} + +#[cfg(windows)] +fn open_windows_file( + path: &Path, + descriptor: windows_sys::Win32::Security::PSECURITY_DESCRIPTOR, + desired_access: u32, + share_mode: u32, + creation_disposition: u32, +) -> io::Result { + use std::os::windows::io::FromRawHandle; + use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE; + use windows_sys::Win32::Security::SECURITY_ATTRIBUTES; + use windows_sys::Win32::Storage::FileSystem::{CreateFileW, FILE_ATTRIBUTE_NORMAL}; + + let path = windows_wide(path.as_os_str()); + let attributes = SECURITY_ATTRIBUTES { + nLength: std::mem::size_of::() as u32, + lpSecurityDescriptor: descriptor, + bInheritHandle: 0, + }; + // SAFETY: The path and security descriptor remain valid; the returned handle is owned. + let handle = unsafe { + CreateFileW( + path.as_ptr(), + desired_access, + share_mode, + &attributes, + creation_disposition, + FILE_ATTRIBUTE_NORMAL, + std::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + // SAFETY: `handle` is a newly returned valid owned file handle. + Ok(unsafe { File::from_raw_handle(handle) }) +} + +#[cfg(windows)] +fn windows_wide(value: impl AsRef) -> Vec { + use std::os::windows::ffi::OsStrExt; + + value.as_ref().encode_wide().chain(Some(0)).collect() +} + +#[cfg(test)] +#[path = "../tests/unit/environment.rs"] +mod tests; diff --git a/crates/plugin-host-config/src/error.rs b/crates/plugin-host-config/src/error.rs new file mode 100644 index 000000000..c73bfa759 --- /dev/null +++ b/crates/plugin-host-config/src/error.rs @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::{Path, PathBuf}; + +use nemo_relay::plugin::PluginError; +use thiserror::Error; + +/// Failure while resolving or preparing a file-backed plugin host. +#[derive(Debug, Error)] +pub enum PluginHostConfigError { + /// A configuration document, lifecycle record, or host policy is invalid. + #[error("{0}")] + InvalidConfig(String), + /// A declared configuration resource was not found. + #[error("dynamic plugin resource {} was not found: {message}", path.display())] + NotFound { + /// Missing resource path. + path: PathBuf, + /// Underlying failure detail. + message: String, + }, + /// An I/O operation required to prepare durable lifecycle state failed. + #[error("failed to {operation} {}: {source}", path.display())] + Io { + /// Description of the attempted operation. + operation: String, + /// Affected path. + path: PathBuf, + /// Underlying I/O failure. + #[source] + source: std::io::Error, + }, + /// Relay core rejected the resolved configuration or activation plan. + #[error(transparent)] + Relay(#[from] PluginError), + /// JSON serialization of a typed Relay configuration failed. + #[error(transparent)] + Json(#[from] serde_json::Error), +} + +impl PluginHostConfigError { + pub(crate) fn io( + operation: impl Into, + path: impl Into, + source: std::io::Error, + ) -> Self { + Self::Io { + operation: operation.into(), + path: path.into(), + source, + } + } + + pub(crate) fn toml_parse(description: &str, path: &Path, error: &toml::de::Error) -> Self { + let location = error + .span() + .map(|span| format!(" at bytes {}..{}", span.start, span.end)) + .unwrap_or_default(); + Self::InvalidConfig(format!( + "invalid {description} in {}{location}: {}", + path.display(), + sanitize_parser_reason(error.message()) + )) + } + + pub(crate) fn json_parse(description: &str, path: &Path, error: &serde_json::Error) -> Self { + Self::InvalidConfig(format!( + "invalid {description} in {} at line {}, column {}: {}", + path.display(), + error.line(), + error.column(), + sanitize_parser_reason(&error.to_string()) + )) + } + + /// Converts the host-resolution failure to Relay's public plugin error taxonomy. + pub fn into_plugin_error(self) -> PluginError { + match self { + Self::InvalidConfig(message) => PluginError::InvalidConfig(message), + Self::NotFound { path, message } => { + PluginError::NotFound(format!("{}: {message}", path.display())) + } + Self::Io { + operation, + path, + source, + } => PluginError::InvalidConfig(format!( + "failed to {operation} {}: {source}", + path.display() + )), + Self::Relay(error) => error, + Self::Json(error) => PluginError::Serialization(error), + } + } +} + +pub(crate) fn sanitize_parser_reason(message: &str) -> String { + let first_line = message.lines().next().unwrap_or("parse failed"); + let mut sanitized = String::with_capacity(first_line.len()); + let mut characters = first_line.chars().peekable(); + while let Some(character) = characters.next() { + if !matches!(character, '\'' | '"' | '`') { + sanitized.push(character); + continue; + } + if character == '`' { + sanitized.push(character); + for candidate in characters.by_ref() { + sanitized.push(candidate); + if candidate == '`' { + break; + } + } + continue; + } + sanitized.push(character); + sanitized.push_str(""); + let delimiter = character; + let mut escaped = false; + for candidate in characters.by_ref() { + if escaped { + escaped = false; + continue; + } + if candidate == '\\' { + escaped = true; + continue; + } + if candidate == delimiter { + sanitized.push(delimiter); + break; + } + } + } + sanitized +} + +#[cfg(test)] +#[path = "../tests/unit/error.rs"] +mod tests; + +/// Result returned by file-backed plugin host configuration operations. +pub type Result = std::result::Result; diff --git a/crates/plugin-host-config/src/io.rs b/crates/plugin-host-config/src/io.rs new file mode 100644 index 000000000..1031b6cf2 --- /dev/null +++ b/crates/plugin-host-config/src/io.rs @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::fs::OpenOptions; +use std::io::Read; +use std::path::Path; + +use nemo_relay::plugin::dynamic::{DYNAMIC_PLUGIN_MANIFEST_FILENAME, DynamicPluginManifest}; + +use crate::error::{PluginHostConfigError, Result}; + +pub(crate) const MAX_BOUNDED_FILE_BYTES: u64 = 512 * 1024 * 1024; + +pub(crate) fn read_bounded_regular_file(path: &Path, description: &str) -> Result> { + let mut bytes = Vec::new(); + stream_bounded_regular_file(path, description, |chunk| bytes.extend_from_slice(chunk))?; + Ok(bytes) +} + +pub(crate) fn read_bounded_utf8_regular_file(path: &Path, description: &str) -> Result { + let bytes = read_bounded_regular_file(path, description)?; + String::from_utf8(bytes).map_err(|error| { + PluginHostConfigError::InvalidConfig(format!( + "{description} {} is not valid UTF-8: {error}", + path.display() + )) + }) +} + +pub(crate) fn stream_bounded_regular_file( + path: &Path, + description: &str, + mut consume: impl FnMut(&[u8]), +) -> Result<()> { + const BUFFER_BYTES: usize = 64 * 1024; + let metadata = std::fs::symlink_metadata(path).map_err(|error| { + PluginHostConfigError::io(format!("inspect {description}"), path, error) + })?; + if !metadata.file_type().is_file() { + return Err(PluginHostConfigError::InvalidConfig(format!( + "{description} {} must be a regular file", + path.display() + ))); + } + if metadata.len() > MAX_BOUNDED_FILE_BYTES { + return Err(PluginHostConfigError::InvalidConfig(format!( + "{description} {} exceeds the {MAX_BOUNDED_FILE_BYTES}-byte limit", + path.display() + ))); + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + let mut file = options + .open(path) + .map_err(|error| PluginHostConfigError::io(format!("read {description}"), path, error))?; + let opened_metadata = file.metadata().map_err(|error| { + PluginHostConfigError::io(format!("inspect {description}"), path, error) + })?; + if !opened_metadata.file_type().is_file() { + return Err(PluginHostConfigError::InvalidConfig(format!( + "{description} {} must be a regular file", + path.display() + ))); + } + let mut buffer = [0_u8; BUFFER_BYTES]; + let mut total = 0_u64; + loop { + let read = file.read(&mut buffer).map_err(|error| { + PluginHostConfigError::io(format!("read {description}"), path, error) + })?; + if read == 0 { + return Ok(()); + } + total = total.saturating_add(read as u64); + if total > MAX_BOUNDED_FILE_BYTES { + return Err(PluginHostConfigError::InvalidConfig(format!( + "{description} {} exceeds the {MAX_BOUNDED_FILE_BYTES}-byte limit", + path.display() + ))); + } + consume(&buffer[..read]); + } +} + +pub(crate) fn load_bounded_dynamic_plugin_manifest( + path: impl AsRef, +) -> Result<(DynamicPluginManifest, String)> { + let (manifest, normalized, _) = load_bounded_dynamic_plugin_manifest_bytes(path)?; + Ok((manifest, normalized)) +} + +pub(crate) fn load_bounded_dynamic_plugin_manifest_bytes( + path: impl AsRef, +) -> Result<(DynamicPluginManifest, String, Vec)> { + let path = path.as_ref(); + let manifest_path = if path.is_dir() { + path.join(DYNAMIC_PLUGIN_MANIFEST_FILENAME) + } else { + path.to_path_buf() + }; + let normalized = std::fs::canonicalize(&manifest_path).map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + PluginHostConfigError::NotFound { + path: manifest_path.clone(), + message: error.to_string(), + } + } else { + PluginHostConfigError::io("normalize dynamic plugin manifest", &manifest_path, error) + } + })?; + let bytes = read_bounded_regular_file(&normalized, "dynamic plugin manifest")?; + let contents = std::str::from_utf8(&bytes).map_err(|error| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin manifest {} is not UTF-8: {error}", + normalized.display() + )) + })?; + let manifest = DynamicPluginManifest::parse_toml(contents).map_err(|_| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin manifest {} is invalid", + normalized.display() + )) + })?; + Ok((manifest, normalized.to_string_lossy().into_owned(), bytes)) +} + +#[cfg(test)] +#[path = "../tests/unit/io.rs"] +mod tests; diff --git a/crates/plugin-host-config/src/lib.rs b/crates/plugin-host-config/src/lib.rs new file mode 100644 index 000000000..f9824715b --- /dev/null +++ b/crates/plugin-host-config/src/lib.rs @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared resolution and lifecycle preparation for file-backed dynamic plugin hosts. +//! +//! This crate contains the lifecycle-reconciling host side of Relay's dynamic-plugin control +//! plane. It resolves the same physical `plugins.toml` layers as the CLI, durably refreshes their +//! sibling lifecycle registries, verifies policy and trust, and produces an owned activation +//! plan. It intentionally does not install plugins, create environments, or change desired +//! enablement. + +mod activation; +mod environment; +mod error; +mod io; +mod lifecycle; +mod policy; +mod resolver; +mod snapshot; +mod state; +mod trust; + +pub use activation::{PluginFileActivation, initialize_from_plugins_toml}; +#[doc(hidden)] +pub use environment::{ + ENVIRONMENT_ATTESTATION_FILE, MANAGED_ENVIRONMENTS_DIR, environment_state, + read_environment_attestation, validate_environment_state, validate_python_entrypoint_artifact, + verify_environment_attestation, +}; +pub use error::{PluginHostConfigError, Result}; +pub use lifecycle::{ + ReconciledDynamicPlugin, ReconciledPluginLifecycle, prepare_plugin_host_activation, + reconcile_plugin_lifecycle, +}; +pub use policy::{ + DynamicPluginHostPolicy, DynamicPluginHostPolicyEffect, DynamicPluginHostPolicyFailure, + DynamicPluginHostPolicyRule, EvaluatedDynamicPluginHostPolicy, FileDynamicPluginHostPolicy, + evaluate_dynamic_plugin_host_policy, +}; +pub use resolver::{ + PluginFileResolveOptions, ResolvedDynamicPluginConfig, ResolvedPluginFileConfiguration, + resolve_plugin_files, resolve_plugin_files_from_paths, +}; +pub use snapshot::DynamicPluginActivationSnapshot; +#[doc(hidden)] +pub use state::{ + DynamicPluginLifecycleState, LifecycleStateLock, lock_lifecycle_state, pin_plugin_config_path, + read_lifecycle_registry, read_lifecycle_state, read_locked_lifecycle_registry, + read_locked_lifecycle_state, save_locked_lifecycle_registry, save_locked_lifecycle_state, + sibling_lifecycle_state_path, +}; +#[doc(hidden)] +pub use trust::{ + DynamicPluginTrustFailure, DynamicPluginTrustFailureDisplay, EvaluatedDynamicPluginTrust, + evaluate_dynamic_plugin_trust, +}; diff --git a/crates/plugin-host-config/src/lifecycle.rs b/crates/plugin-host-config/src/lifecycle.rs new file mode 100644 index 000000000..fbd65b8a5 --- /dev/null +++ b/crates/plugin-host-config/src/lifecycle.rs @@ -0,0 +1,496 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use crate::environment::{validate_environment_state, validate_python_entrypoint_artifact}; +use crate::error::{PluginHostConfigError, Result}; +use crate::io::load_bounded_dynamic_plugin_manifest; +use crate::policy::{EvaluatedDynamicPluginHostPolicy, evaluate_dynamic_plugin_host_policy}; +use crate::resolver::{ResolvedDynamicPluginConfig, ResolvedPluginFileConfiguration}; +use crate::snapshot::DynamicPluginActivationSnapshot; +use crate::state::{ + DynamicPluginLifecycleState, LifecycleStateLock, lock_lifecycle_state, pin_plugin_config_path, + read_locked_lifecycle_state, save_locked_lifecycle_state, sibling_lifecycle_state_path, +}; +use crate::trust::{EvaluatedDynamicPluginTrust, evaluate_dynamic_plugin_trust}; +use nemo_relay::plugin::dynamic::{ + DynamicPluginActivationResource, DynamicPluginActivationSpec, DynamicPluginCheckState, + DynamicPluginFailure, DynamicPluginFailurePhase, DynamicPluginKind, DynamicPluginLoadContract, + DynamicPluginManifest, DynamicPluginRecord, DynamicPluginValidationStatus, + PlannedDynamicPluginActivation, PluginHostActivationPlan, WorkerRuntime, +}; + +const VALIDATION_MESSAGE: &str = "validated by Relay plugin host"; + +/// One enabled, live lifecycle record ready for activation planning. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReconciledDynamicPlugin { + /// Canonical plugin ID. + pub plugin_id: String, + /// Dynamic execution lane. + pub kind: DynamicPluginKind, + /// Desired-state lifecycle generation. + pub lifecycle_generation: u64, + /// Canonical authored manifest reference retained by lifecycle state. + pub manifest_ref: String, + /// Lifecycle-managed worker environment, when applicable. + pub environment_ref: Option, + /// Component-local host configuration. + pub config: serde_json::Map, + /// Physical `plugins.toml` that owns the declaration and lifecycle record. + pub source: PathBuf, +} + +/// Durable lifecycle reconciliation result for one resolved plugin file configuration. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ReconciledPluginLifecycle { + /// Enabled, non-tombstoned dynamic plugins in declaration order. + pub enabled_plugins: Vec, +} + +struct SourceRegistry { + state_path: PathBuf, + registry: DynamicPluginLifecycleState, + _lock: LifecycleStateLock, +} + +/// Reconciles declarations with source-local lifecycle state and durably saves validation status. +/// +/// Missing records are hydrated disabled. This operation never installs a plugin, provisions an +/// environment, changes enablement, or edits `plugins.toml`. +pub fn reconcile_plugin_lifecycle( + resolved: &ResolvedPluginFileConfiguration, +) -> Result { + if resolved.dynamic_plugins.is_empty() { + return Ok(ReconciledPluginLifecycle::default()); + } + + let source_to_state = resolved + .dynamic_plugins + .iter() + .map(|plugin| { + ( + plugin.source.clone(), + sibling_lifecycle_state_path(&plugin.source), + ) + }) + .collect::>(); + let declared_state_paths = source_to_state.values().cloned().collect::>(); + let mut state_paths = declared_state_paths.clone(); + for source in &resolved.selected_sources { + let source = pin_plugin_config_path(source)?; + let state_path = sibling_lifecycle_state_path(&source); + if state_path.try_exists().map_err(|error| { + PluginHostConfigError::io("inspect dynamic plugin lifecycle state", &state_path, error) + })? { + state_paths.insert(state_path); + } + } + let mut locks = BTreeMap::new(); + for state_path in state_paths { + locks.insert(state_path.clone(), lock_lifecycle_state(&state_path)?); + } + let mut registries = BTreeMap::new(); + for (state_path, lock) in locks { + let registry = read_locked_lifecycle_state(&lock)?; + registries.insert( + state_path.clone(), + SourceRegistry { + state_path, + registry, + _lock: lock, + }, + ); + } + + for declaration in &resolved.dynamic_plugins { + let state_path = source_to_state.get(&declaration.source).ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin '{}' resolved from {} but no matching lifecycle scope exists", + declaration.plugin_id, + declaration.source.display() + )) + })?; + let declaration_ref = lifecycle_declaration_ref(&declaration.source)?; + let (manifest, manifest_ref) = + load_bounded_dynamic_plugin_manifest(&declaration.manifest_ref)?; + let reloaded_plugin_id = manifest.plugin.id.trim(); + if reloaded_plugin_id != declaration.plugin_id { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin manifest {} changed identity during lifecycle reconciliation: resolved '{}' but reloaded '{}'", + manifest_ref, declaration.plugin_id, reloaded_plugin_id + ))); + } + validate_python_entrypoint_artifact(&manifest, &manifest_ref) + .map_err(PluginHostConfigError::InvalidConfig)?; + let policy = + evaluate_dynamic_plugin_host_policy(&resolved.dynamic_plugin_policy, &manifest); + let trust = evaluate_dynamic_plugin_trust(&manifest, &manifest_ref, &policy); + let mut conflicting_foreign_states = Vec::new(); + for (candidate_state_path, registry) in ®istries { + if candidate_state_path == state_path { + continue; + } + let Some(record) = registry.registry.get(&declaration.plugin_id) else { + continue; + }; + if record.is_tombstoned() { + continue; + } + conflicting_foreign_states.push(candidate_state_path.clone()); + } + if !conflicting_foreign_states.is_empty() { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin '{}' has live lifecycle state outside its declaring source {}: {}", + declaration.plugin_id, + declaration.source.display(), + conflicting_foreign_states + .iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(", ") + ))); + } + let source_registry = registries.get_mut(state_path).ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin '{}' resolved from {} but no matching lifecycle transaction exists", + declaration.plugin_id, + declaration.source.display() + )) + })?; + let existing = source_registry + .registry + .get(&declaration.plugin_id) + .cloned(); + match existing { + Some(record) + if source_registry + .registry + .declaration_source(&declaration.plugin_id) + .is_some_and(|owner| owner != declaration_ref) => + { + if !record.is_tombstoned() { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin '{}' has live lifecycle state owned by {} but is now declared by {}; remove it through the plugin control plane before moving the declaration", + declaration.plugin_id, + source_registry + .registry + .declaration_source(&declaration.plugin_id) + .expect("nonmatching lifecycle owner was checked"), + declaration.source.display() + ))); + } + let record = validated_record_from_manifest( + manifest, + manifest_ref, + None, + &source_registry.state_path, + &policy, + &trust, + )?; + revive_registry_record_for_new_owner( + source_registry, + &declaration.plugin_id, + record, + )?; + source_registry + .registry + .set_declaration_source(&declaration.plugin_id, declaration_ref)?; + } + Some(_) => { + if source_registry + .registry + .declaration_source(&declaration.plugin_id) + .is_none() + { + source_registry + .registry + .set_declaration_source(&declaration.plugin_id, declaration_ref.clone())?; + } + refresh_registry_record( + source_registry, + &declaration.plugin_id, + manifest, + manifest_ref, + &policy, + &trust, + )?; + } + None => { + let record = validated_record_from_manifest( + manifest, + manifest_ref, + None, + &source_registry.state_path, + &policy, + &trust, + )?; + let validation = record.status.validation.clone(); + source_registry.registry.add(record)?; + source_registry + .registry + .update_validation_status(&declaration.plugin_id, validation)?; + source_registry + .registry + .set_declaration_source(&declaration.plugin_id, declaration_ref)?; + } + } + } + + // Parse and reconcile every source before the first durable write. Each individual sibling + // state file is then replaced atomically, matching the CLI control-plane contract. + for state_path in &declared_state_paths { + let registry = registries.get(state_path).ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin lifecycle transaction for {} disappeared before persistence", + state_path.display() + )) + })?; + save_registry(registry)?; + } + + let mut enabled_plugins = Vec::new(); + for declaration in &resolved.dynamic_plugins { + let state_path = source_to_state.get(&declaration.source).ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin '{}' resolved from {} but no matching lifecycle scope exists", + declaration.plugin_id, + declaration.source.display() + )) + })?; + let source_registry = registries.get(state_path).ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin '{}' resolved from {} but no matching lifecycle transaction exists", + declaration.plugin_id, + declaration.source.display() + )) + })?; + let record = source_registry + .registry + .get(&declaration.plugin_id) + .ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin '{}' has no reconciled lifecycle record in {}", + declaration.plugin_id, + source_registry.state_path.display() + )) + })?; + if record.is_tombstoned() || !record.spec.enabled { + continue; + } + if matches!( + &record.load, + DynamicPluginLoadContract::Worker(load) if load.runtime == WorkerRuntime::Python + ) && record.status.validation.environment == DynamicPluginCheckState::Invalid + { + let message = record + .status + .last_error + .as_ref() + .filter(|failure| failure.code == "environment_failed") + .map(|failure| failure.message.clone()) + .unwrap_or_else(|| { + format!( + "enabled Python worker dynamic plugin '{}' has an invalid lifecycle-managed environment", + declaration.plugin_id + ) + }); + return Err(PluginHostConfigError::InvalidConfig(format!( + "enabled Python worker dynamic plugin '{}' has an invalid lifecycle-managed environment: {message}", + declaration.plugin_id, + ))); + } + enabled_plugins.push(reconciled_enabled_plugin(declaration, record)?); + } + Ok(ReconciledPluginLifecycle { enabled_plugins }) +} + +/// Reconciles lifecycle state, snapshots enabled plugins, and builds a core activation plan. +pub fn prepare_plugin_host_activation( + resolved: ResolvedPluginFileConfiguration, +) -> Result { + let reconciled = reconcile_plugin_lifecycle(&resolved)?; + let mut dynamic_plugins = Vec::with_capacity(reconciled.enabled_plugins.len()); + for plugin in reconciled.enabled_plugins { + let snapshot = DynamicPluginActivationSnapshot::create( + &plugin.manifest_ref, + &plugin.plugin_id, + plugin.kind, + plugin.environment_ref.as_deref(), + &resolved.dynamic_plugin_policy, + )?; + let spec = DynamicPluginActivationSpec { + plugin_id: plugin.plugin_id, + kind: plugin.kind, + manifest_ref: snapshot.activation_manifest_ref(), + environment_ref: snapshot.activation_environment_ref().map(str::to_owned), + config: plugin.config, + }; + let resource: Arc = snapshot; + dynamic_plugins.push(PlannedDynamicPluginActivation { spec, resource }); + } + Ok(PluginHostActivationPlan { + config: resolved.config, + dynamic_plugins, + diagnostics: resolved.diagnostics, + }) +} + +fn reconciled_enabled_plugin( + declaration: &ResolvedDynamicPluginConfig, + record: &DynamicPluginRecord, +) -> Result { + let manifest_ref = record.source.manifest_ref.clone().ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin '{}' has no manifest_ref in lifecycle state", + record.metadata.id + )) + })?; + Ok(ReconciledDynamicPlugin { + plugin_id: record.metadata.id.clone(), + kind: record.metadata.kind, + lifecycle_generation: record.metadata.generation, + manifest_ref, + environment_ref: record.source.environment_ref.clone(), + config: declaration.config.clone(), + source: declaration.source.clone(), + }) +} + +fn lifecycle_declaration_ref(source: &Path) -> Result { + Ok(pin_plugin_config_path(source)? + .to_string_lossy() + .into_owned()) +} + +fn validated_record_from_manifest( + manifest: DynamicPluginManifest, + manifest_ref: String, + environment_ref: Option, + state_path: &Path, + policy: &EvaluatedDynamicPluginHostPolicy, + trust: &EvaluatedDynamicPluginTrust, +) -> Result { + let (environment, environment_error) = + match validate_environment_state(&manifest, state_path, environment_ref.as_deref()) { + Ok(environment) => (environment, None), + Err(error) => ( + DynamicPluginCheckState::Invalid, + environment_ref.as_ref().map(|_| error.to_string()), + ), + }; + let mut record = manifest.into_record(Some(manifest_ref))?; + record.source.environment_ref = environment_ref; + record.status.validation = DynamicPluginValidationStatus { + manifest: DynamicPluginCheckState::Valid, + compatibility: DynamicPluginCheckState::Valid, + integrity: trust.integrity, + environment, + authenticity: trust.authenticity, + policy_satisfied: policy.check_state(), + checked_at: None, + message: Some(VALIDATION_MESSAGE.into()), + }; + record.status.startup_class = Some(policy.startup_class); + record.status.attestation_mode = Some(policy.attestation_mode); + record.status.last_error = policy + .last_error(&record.metadata.id) + .or_else(|| trust.last_error(&record.metadata.id)) + .or_else(|| { + environment_last_error( + &record.metadata.id, + environment, + record.source.environment_ref.as_deref(), + environment_error, + ) + }); + Ok(record) +} + +fn refresh_registry_record( + source: &mut SourceRegistry, + plugin_id: &str, + manifest: DynamicPluginManifest, + manifest_ref: String, + policy: &EvaluatedDynamicPluginHostPolicy, + trust: &EvaluatedDynamicPluginTrust, +) -> Result<()> { + let existing = source.registry.get(plugin_id).cloned().ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin '{plugin_id}' disappeared during lifecycle reconciliation" + )) + })?; + let refreshed = validated_record_from_manifest( + manifest, + manifest_ref, + existing.source.environment_ref.clone(), + &source.state_path, + policy, + trust, + )?; + source + .registry + .refresh_manifest_record(plugin_id, refreshed)?; + Ok(()) +} + +fn revive_registry_record_for_new_owner( + source: &mut SourceRegistry, + plugin_id: &str, + record: DynamicPluginRecord, +) -> Result<()> { + if source + .registry + .get(plugin_id) + .is_some_and(|existing| !existing.is_tombstoned()) + { + return Err(PluginHostConfigError::InvalidConfig(format!( + "refusing to replace live lifecycle state for dynamic plugin '{plugin_id}'" + ))); + } + let validation = record.status.validation.clone(); + source.registry.add(record)?; + source + .registry + .update_validation_status(plugin_id, validation)?; + Ok(()) +} + +fn environment_last_error( + plugin_id: &str, + environment: DynamicPluginCheckState, + environment_ref: Option<&str>, + detail: Option, +) -> Option { + (environment == DynamicPluginCheckState::Invalid).then(|| DynamicPluginFailure { + phase: DynamicPluginFailurePhase::Validation, + code: "environment_failed".into(), + message: detail.unwrap_or_else(|| { + environment_ref.map_or_else( + || { + format!( + "dynamic plugin '{}' has no lifecycle-managed Python environment; run `nemo-relay plugins remove {}` to remove the manual registration, then run `nemo-relay plugins add `", + plugin_id, plugin_id + ) + }, + |environment_ref| { + format!( + "dynamic plugin '{}' configured Python environment {} is unavailable", + plugin_id, environment_ref + ) + }, + ) + }), + }) +} + +fn save_registry(source: &SourceRegistry) -> Result<()> { + save_locked_lifecycle_state(&source._lock, &source.registry) +} + +#[cfg(test)] +#[path = "../tests/unit/lifecycle.rs"] +mod tests; diff --git a/crates/plugin-host-config/src/policy.rs b/crates/plugin-host-config/src/policy.rs new file mode 100644 index 000000000..2d014d350 --- /dev/null +++ b/crates/plugin-host-config/src/policy.rs @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; +use std::fmt; + +use nemo_relay::plugin::dynamic::{ + DynamicPluginAttestationMode, DynamicPluginCheckState, DynamicPluginFailure, + DynamicPluginFailurePhase, DynamicPluginKind, DynamicPluginManifest, DynamicPluginStartupClass, +}; +use serde::Deserialize; + +/// Layered host policy applied to discovered dynamic plugins. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DynamicPluginHostPolicy { + /// Default effects inherited by every dynamic plugin. + pub defaults: DynamicPluginHostPolicyEffect, + /// Ordered matching rules. + pub rules: Vec, + /// Plugin-ID-specific effects. + pub overrides: BTreeMap, +} + +impl DynamicPluginHostPolicy { + /// Layers a higher-precedence policy onto this policy. + pub fn merge_from(&mut self, other: Self) { + self.defaults.merge_from(other.defaults); + self.rules.extend(other.rules); + for (plugin_id, effect) in other.overrides { + self.overrides + .entry(plugin_id) + .or_default() + .merge_from(effect); + } + } +} + +/// One resolved dynamic-plugin host-policy effect. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DynamicPluginHostPolicyEffect { + /// Whether matching plugins may activate. + pub allowed: Option, + /// Required or optional startup classification. + pub startup: Option, + /// Required artifact attestation mode. + pub attestation: Option, + /// Trusted Ed25519 public keys. + pub trusted_public_keys: Option>, +} + +impl DynamicPluginHostPolicyEffect { + fn merge_from(&mut self, other: Self) { + if other.allowed.is_some() { + self.allowed = other.allowed; + } + if other.startup.is_some() { + self.startup = other.startup; + } + if other.attestation.is_some() { + self.attestation = other.attestation; + } + if other.trusted_public_keys.is_some() { + self.trusted_public_keys = other.trusted_public_keys; + } + } +} + +/// One ordered dynamic-plugin host-policy rule. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DynamicPluginHostPolicyRule { + /// Optional dynamic execution-lane selector. + pub match_kind: Option, + /// Optional canonical plugin-ID selector. + pub match_plugin_id: Option, + /// Effect applied when selectors match. + pub effect: DynamicPluginHostPolicyEffect, +} + +/// Reason a dynamic plugin was rejected by host policy. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DynamicPluginHostPolicyFailure { + /// Host policy explicitly blocks the plugin. + Blocked, +} + +impl DynamicPluginHostPolicyFailure { + /// Formats this failure for one canonical plugin ID. + pub fn display<'a>(&'a self, plugin_id: &'a str) -> DynamicPluginHostPolicyFailureDisplay<'a> { + DynamicPluginHostPolicyFailureDisplay { + failure: self, + plugin_id, + } + } +} + +/// Display adapter for a policy failure associated with a plugin ID. +pub struct DynamicPluginHostPolicyFailureDisplay<'a> { + failure: &'a DynamicPluginHostPolicyFailure, + plugin_id: &'a str, +} + +impl fmt::Display for DynamicPluginHostPolicyFailureDisplay<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.failure { + DynamicPluginHostPolicyFailure::Blocked => write!( + f, + "dynamic plugin '{}' is blocked by host policy", + self.plugin_id + ), + } + } +} + +/// Effective policy decision for one validated dynamic-plugin manifest. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EvaluatedDynamicPluginHostPolicy { + /// Whether policy permits activation. + pub policy_satisfied: bool, + /// Effective startup class. + pub startup_class: DynamicPluginStartupClass, + /// Effective attestation mode. + pub attestation_mode: DynamicPluginAttestationMode, + /// Effective trusted Ed25519 public keys. + pub trusted_public_keys: Vec, + /// Policy rejection, if any. + pub failure: Option, +} + +impl EvaluatedDynamicPluginHostPolicy { + /// Converts the decision to durable lifecycle validation state. + pub fn check_state(&self) -> DynamicPluginCheckState { + if self.policy_satisfied { + DynamicPluginCheckState::Valid + } else { + DynamicPluginCheckState::Invalid + } + } + + /// Returns the durable lifecycle failure for this decision. + pub fn last_error(&self, plugin_id: &str) -> Option { + self.failure.as_ref().map(|failure| DynamicPluginFailure { + phase: DynamicPluginFailurePhase::Policy, + code: "policy_blocked".into(), + message: failure.display(plugin_id).to_string(), + }) + } + + /// Returns the policy rejection, if any. + pub fn failure(&self) -> Option<&DynamicPluginHostPolicyFailure> { + self.failure.as_ref() + } +} + +/// Evaluates one manifest against a fully layered host policy. +pub fn evaluate_dynamic_plugin_host_policy( + policy: &DynamicPluginHostPolicy, + manifest: &DynamicPluginManifest, +) -> EvaluatedDynamicPluginHostPolicy { + let mut effect = DynamicPluginHostPolicyEffect { + allowed: Some(true), + startup: Some(DynamicPluginStartupClass::Optional), + attestation: Some(DynamicPluginAttestationMode::IntegrityOnly), + trusted_public_keys: None, + }; + effect.merge_from(policy.defaults.clone()); + for rule in &policy.rules { + if policy_rule_matches(rule, manifest) { + effect.merge_from(rule.effect.clone()); + } + } + if let Some(override_effect) = policy.overrides.get(manifest.plugin.id.trim()) { + effect.merge_from(override_effect.clone()); + } + + let startup_class = effect + .startup + .unwrap_or(DynamicPluginStartupClass::Optional); + let attestation_mode = effect + .attestation + .unwrap_or(DynamicPluginAttestationMode::IntegrityOnly); + let trusted_public_keys = effect.trusted_public_keys.unwrap_or_default(); + let failure = + (effect.allowed == Some(false)).then_some(DynamicPluginHostPolicyFailure::Blocked); + EvaluatedDynamicPluginHostPolicy { + policy_satisfied: failure.is_none(), + startup_class, + attestation_mode, + trusted_public_keys, + failure, + } +} + +fn policy_rule_matches( + rule: &DynamicPluginHostPolicyRule, + manifest: &DynamicPluginManifest, +) -> bool { + if let Some(match_kind) = rule.match_kind + && manifest.plugin.kind != match_kind + { + return false; + } + if let Some(match_plugin_id) = &rule.match_plugin_id + && manifest.plugin.id.trim() != match_plugin_id + { + return false; + } + true +} + +/// TOML representation of dynamic-plugin host policy. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FileDynamicPluginHostPolicy { + /// Default policy effects. + #[serde(default)] + pub defaults: FileDynamicPluginHostPolicyEffect, + /// Ordered policy rules. + #[serde(default)] + pub rules: Vec, + /// Plugin-ID-specific effects. + #[serde(default)] + pub overrides: BTreeMap, +} + +impl From for DynamicPluginHostPolicy { + fn from(value: FileDynamicPluginHostPolicy) -> Self { + Self { + defaults: value.defaults.into(), + rules: value.rules.into_iter().map(Into::into).collect(), + overrides: value + .overrides + .into_iter() + .map(|(plugin_id, effect)| (plugin_id.trim().to_owned(), effect.into())) + .collect(), + } + } +} + +/// TOML representation of a dynamic-plugin policy effect. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FileDynamicPluginHostPolicyEffect { + allowed: Option, + startup: Option, + attestation: Option, + trusted_public_keys: Option>, +} + +impl From for DynamicPluginHostPolicyEffect { + fn from(value: FileDynamicPluginHostPolicyEffect) -> Self { + Self { + allowed: value.allowed, + startup: value.startup, + attestation: value.attestation, + trusted_public_keys: value + .trusted_public_keys + .map(|keys| keys.into_iter().map(|key| key.trim().to_owned()).collect()), + } + } +} + +/// TOML representation of one dynamic-plugin policy rule. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FileDynamicPluginHostPolicyRule { + match_kind: Option, + match_plugin_id: Option, + allowed: Option, + startup: Option, + attestation: Option, + trusted_public_keys: Option>, +} + +impl From for DynamicPluginHostPolicyRule { + fn from(value: FileDynamicPluginHostPolicyRule) -> Self { + Self { + match_kind: value.match_kind, + match_plugin_id: value.match_plugin_id.map(|value| value.trim().to_owned()), + effect: DynamicPluginHostPolicyEffect { + allowed: value.allowed, + startup: value.startup, + attestation: value.attestation, + trusted_public_keys: value + .trusted_public_keys + .map(|keys| keys.into_iter().map(|key| key.trim().to_owned()).collect()), + }, + } + } +} + +#[cfg(test)] +#[path = "../tests/unit/policy.rs"] +mod tests; diff --git a/crates/plugin-host-config/src/resolver.rs b/crates/plugin-host-config/src/resolver.rs new file mode 100644 index 000000000..bfe2047e4 --- /dev/null +++ b/crates/plugin-host-config/src/resolver.rs @@ -0,0 +1,654 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +use nemo_relay::plugin::{ + ConfigDiagnostic, ConfigPolicy, DiagnosticLevel, PluginConfig, default_plugin_config_paths, + merge_plugin_config_documents, user_config_dir, +}; +use serde::Deserialize; +use serde_json::{Map, Value}; + +use crate::error::{PluginHostConfigError, Result}; +use crate::io::{load_bounded_dynamic_plugin_manifest, read_bounded_utf8_regular_file}; +use crate::policy::{DynamicPluginHostPolicy, FileDynamicPluginHostPolicy}; +use crate::state::pin_plugin_config_path; + +/// Source selection used by file-backed plugin-host initialization. +#[derive(Debug, Clone)] +pub struct PluginFileResolveOptions { + /// Optional file that replaces the ambient user-level `plugins.toml` layer. + pub plugin_config_path: Option, + /// Directory used for nearest-project discovery. `None` suppresses project discovery. + pub current_dir: Option, + /// Ambient user configuration directory. + pub user_config_dir: Option, + /// System-level plugin configuration path. + pub system_config_path: PathBuf, +} + +impl PluginFileResolveOptions { + /// Builds source selection from the current process environment. + pub fn from_environment(plugin_config_path: Option) -> Self { + let user_only = std::env::var("NEMO_RELAY_CONFIG_SCOPE").ok().as_deref() == Some("user"); + Self { + plugin_config_path, + current_dir: (!user_only) + .then(std::env::current_dir) + .transpose() + .ok() + .flatten(), + user_config_dir: user_config_dir(), + system_config_path: PathBuf::from("/etc/nemo-relay/plugins.toml"), + } + } + + /// Returns selected source paths in increasing precedence order. + pub fn selected_paths(&self) -> Vec { + let mut paths = Vec::new(); + if let Some(selected) = &self.plugin_config_path { + paths.push(selected.clone()); + } else if let Some(user_dir) = &self.user_config_dir { + paths.push(user_dir.join("plugins.toml")); + } + if let Some(current_dir) = self.current_dir.as_deref() { + let mut implicit = default_plugin_config_paths(Some(current_dir), None); + implicit.pop(); + paths.extend(implicit); + } + paths.push(self.system_config_path.clone()); + paths + } +} + +/// One dynamic declaration resolved from a physical `plugins.toml` source. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedDynamicPluginConfig { + /// Canonical plugin ID derived from the referenced manifest. + pub plugin_id: String, + /// Canonical absolute manifest reference. + pub manifest_ref: String, + /// Component configuration supplied by the declaration. + pub config: Map, + /// Whether the declaration explicitly included `config`. + pub has_explicit_config: bool, + /// Physical `plugins.toml` containing the declaration. + pub source: PathBuf, +} + +/// Fully resolved static and dynamic file-backed plugin configuration. +#[derive(Debug, Clone)] +pub struct ResolvedPluginFileConfiguration { + /// Effective static Relay plugin configuration, including the caller overlay. + pub config: PluginConfig, + /// Effective JSON document before conversion to [`PluginConfig`]. + pub runtime_value: Option, + /// Dynamic declarations in source and declaration order. + pub dynamic_plugins: Vec, + /// Fully layered dynamic-plugin host policy. + pub dynamic_plugin_policy: DynamicPluginHostPolicy, + /// Redacted inherited-source diagnostics. + pub diagnostics: Vec, + /// Existing physical sources that contributed static, dynamic, or policy configuration. + pub contributing_sources: Vec, + /// Selected source spellings corresponding to [`Self::contributing_sources`]. + /// + /// These are captured during the same physical-path resolution pass so presentation adapters + /// do not need to re-resolve aliases after configuration has been consumed. + #[doc(hidden)] + pub contributing_selected_sources: Vec, + /// Selected source paths, including absent files whose sibling lifecycle state may remain. + #[doc(hidden)] + pub selected_sources: Vec, + /// Whether any physical source or caller-supplied configuration participated. + pub had_input: bool, +} + +/// Resolves standard plugin file discovery and an optional caller static overlay. +pub fn resolve_plugin_files( + caller_config: Option, + options: PluginFileResolveOptions, +) -> Result { + resolve_plugin_files_from_paths(options.selected_paths(), caller_config) +} + +/// Resolves a supplied low-to-high precedence list of physical plugin files. +#[doc(hidden)] +pub fn resolve_plugin_files_from_paths( + paths: I, + caller_config: Option, +) -> Result +where + I: IntoIterator, +{ + let paths = paths.into_iter().collect::>(); + let mut seen_selected_sources = HashSet::new(); + let mut selected_source_mappings = Vec::with_capacity(paths.len()); + for selected_path in paths.into_iter().rev() { + let physical_path = pin_plugin_config_path(&selected_path)?; + if seen_selected_sources.insert(physical_path.clone()) { + selected_source_mappings.push((selected_path, physical_path)); + } + } + selected_source_mappings.reverse(); + let selected_sources = selected_source_mappings + .iter() + .map(|(_, physical_path)| physical_path.clone()) + .collect::>(); + let mut dynamic_plugins = Vec::new(); + let mut dynamic_plugin_policy = DynamicPluginHostPolicy::default(); + let mut seen_plugin_ids = HashSet::new(); + let mut contributing_sources = Vec::new(); + let mut contributing_selected_sources = Vec::new(); + let mut runtime_documents = Vec::new(); + let mut enabled_sources = HashMap::new(); + let mut seen_physical_sources = HashSet::new(); + + for (selected_path, physical_path) in &selected_source_mappings { + if !physical_path.try_exists().map_err(|error| { + PluginHostConfigError::InvalidConfig(format!( + "failed to inspect plugin configuration file {}: {error}", + physical_path.display() + )) + })? { + continue; + } + let path = physical_path.clone(); + if !seen_physical_sources.insert(path.clone()) { + continue; + } + let raw = read_bounded_utf8_regular_file(&path, "plugin configuration file")?; + let mut parsed = raw + .parse::() + .map(toml::Value::Table) + .map_err(|error| PluginHostConfigError::toml_parse("plugin TOML", &path, &error))?; + contributing_sources.push(path.clone()); + contributing_selected_sources.push(selected_path.clone()); + let resolved = resolve_dynamic_plugin_refs(&path, &mut parsed, &mut seen_plugin_ids)?; + dynamic_plugins.extend(resolved.dynamic_plugins); + dynamic_plugin_policy.merge_from(resolved.dynamic_plugin_policy); + let runtime_value = serde_json::to_value(remove_dynamic_plugin_sections(parsed))?; + record_enabled_sources(&path, &runtime_value, &mut enabled_sources); + runtime_documents.push((path, runtime_value)); + } + + let (mut runtime_value, _) = merge_plugin_config_documents(runtime_documents) + .map_err(|error| { + PluginHostConfigError::InvalidConfig(format!( + "failed to merge static plugin configuration: {}", + crate::error::sanitize_parser_reason(&error.to_string()) + )) + })? + .unwrap_or_else(|| (Value::Object(Map::new()), Vec::new())); + let had_caller_config = caller_config.is_some(); + if let Some(caller_config) = caller_config.as_ref() { + let diagnostics = programmatic_enable_override_diagnostics( + &runtime_value, + &enabled_sources, + caller_config, + ); + layer_config( + &mut runtime_value, + plugin_config_overlay_value(caller_config)?, + ); + let mut inherited = inherited_source_diagnostics(&contributing_selected_sources); + inherited.extend(diagnostics); + return finish_resolution( + runtime_value, + dynamic_plugins, + dynamic_plugin_policy, + inherited, + ResolvedPluginFileSourcePaths { + contributing_sources, + contributing_selected_sources, + selected_sources, + }, + had_caller_config, + ); + } + + let diagnostics = inherited_source_diagnostics(&contributing_selected_sources); + finish_resolution( + runtime_value, + dynamic_plugins, + dynamic_plugin_policy, + diagnostics, + ResolvedPluginFileSourcePaths { + contributing_sources, + contributing_selected_sources, + selected_sources, + }, + had_caller_config, + ) +} + +struct ResolvedPluginFileSourcePaths { + contributing_sources: Vec, + contributing_selected_sources: Vec, + selected_sources: Vec, +} + +fn finish_resolution( + runtime_value: Value, + dynamic_plugins: Vec, + dynamic_plugin_policy: DynamicPluginHostPolicy, + diagnostics: Vec, + source_paths: ResolvedPluginFileSourcePaths, + had_caller_config: bool, +) -> Result { + let ResolvedPluginFileSourcePaths { + contributing_sources, + contributing_selected_sources, + selected_sources, + } = source_paths; + let had_input = had_caller_config || !contributing_sources.is_empty(); + let serialized_value = match &runtime_value { + Value::Object(object) if object.is_empty() => None, + _ => Some(runtime_value.clone()), + }; + let config = serde_json::from_value(runtime_value).map_err(|error| { + PluginHostConfigError::InvalidConfig(format!( + "resolved static plugin configuration is invalid: {}", + crate::error::sanitize_parser_reason(&error.to_string()) + )) + })?; + Ok(ResolvedPluginFileConfiguration { + config, + runtime_value: serialized_value, + dynamic_plugins, + dynamic_plugin_policy, + diagnostics, + contributing_sources, + contributing_selected_sources, + selected_sources, + had_input, + }) +} + +#[derive(Debug, Clone, Default, Deserialize)] +struct PluginTomlPluginsSection { + #[serde(default)] + dynamic: Vec, + #[serde(default)] + policy: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +struct FileDynamicPluginConfig { + manifest: String, + #[serde(default)] + config: Option>, +} + +struct ResolvedDynamicPluginRefs { + dynamic_plugins: Vec, + dynamic_plugin_policy: DynamicPluginHostPolicy, +} + +fn resolve_dynamic_plugin_refs( + source: &Path, + value: &mut toml::Value, + seen_plugin_ids: &mut HashSet, +) -> Result { + let Some(root) = value.as_table_mut() else { + return Ok(ResolvedDynamicPluginRefs { + dynamic_plugins: Vec::new(), + dynamic_plugin_policy: DynamicPluginHostPolicy::default(), + }); + }; + let Some(plugins_value) = root.get("plugins").cloned() else { + return Ok(ResolvedDynamicPluginRefs { + dynamic_plugins: Vec::new(), + dynamic_plugin_policy: DynamicPluginHostPolicy::default(), + }); + }; + let plugins: PluginTomlPluginsSection = plugins_value.try_into().map_err(|error| { + PluginHostConfigError::toml_parse("dynamic plugin config", source, &error) + })?; + let mut resolved = Vec::with_capacity(plugins.dynamic.len()); + for dynamic in plugins.dynamic { + let manifest_path = resolve_dynamic_manifest_path(source, &dynamic.manifest); + let (manifest, manifest_ref) = load_bounded_dynamic_plugin_manifest(&manifest_path) + .map_err(|error| contextualize_manifest_error(source, error))?; + let plugin_id = manifest.plugin.id.trim().to_owned(); + if !seen_plugin_ids.insert(plugin_id.clone()) { + return Err(PluginHostConfigError::InvalidConfig(format!( + "duplicate dynamic plugin id '{}' in {} across plugins.toml sources", + plugin_id, + source.display() + ))); + } + resolved.push(ResolvedDynamicPluginConfig { + plugin_id, + manifest_ref, + has_explicit_config: dynamic.config.is_some(), + config: dynamic.config.unwrap_or_default(), + source: source.to_path_buf(), + }); + } + Ok(ResolvedDynamicPluginRefs { + dynamic_plugins: resolved, + dynamic_plugin_policy: plugins.policy.map(Into::into).unwrap_or_default(), + }) +} + +fn contextualize_manifest_error( + source: &Path, + error: PluginHostConfigError, +) -> PluginHostConfigError { + match error { + PluginHostConfigError::NotFound { .. } => error, + other => PluginHostConfigError::InvalidConfig(format!( + "invalid dynamic plugin manifest referenced by {}: {other}", + source.display() + )), + } +} + +fn resolve_dynamic_manifest_path(source: &Path, manifest: &str) -> PathBuf { + let manifest = PathBuf::from(manifest); + if manifest.is_absolute() { + manifest + } else { + source + .parent() + .map(|parent| parent.join(&manifest)) + .unwrap_or(manifest) + } +} + +fn remove_dynamic_plugin_sections(mut value: toml::Value) -> toml::Value { + if let Some(root) = value.as_table_mut() + && let Some(toml::Value::Table(plugins)) = root.get_mut("plugins") + { + plugins.remove("dynamic"); + plugins.remove("policy"); + if plugins.is_empty() { + root.remove("plugins"); + } + } + value +} + +fn inherited_source_diagnostics(sources: &[PathBuf]) -> Vec { + sources + .iter() + .map(|source| { + let source = source.display().to_string(); + log::warn!( + target: "nemo_relay.plugin", + event = "plugin_configuration_inherited", + config_path = source.as_str(); + "Inherited plugin configuration from discovered file" + ); + ConfigDiagnostic { + level: DiagnosticLevel::Warning, + code: "plugin.configuration_inherited".to_owned(), + component: None, + field: None, + message: format!("inherited plugin configuration from discovered file: {source}"), + } + }) + .collect() +} + +#[derive(Clone)] +struct ComponentEnabledSource { + enabled: bool, + path: PathBuf, +} + +fn record_enabled_sources( + path: &Path, + document: &Value, + sources: &mut HashMap, +) { + let Some(components) = document.get("components").and_then(Value::as_array) else { + return; + }; + for component in components { + let Some(kind) = component_kind(component) else { + continue; + }; + if let Some(enabled) = component.get("enabled").and_then(Value::as_bool) { + sources.insert( + kind.to_owned(), + ComponentEnabledSource { + enabled, + path: path.to_path_buf(), + }, + ); + } + } +} + +fn programmatic_enable_override_diagnostics( + discovered: &Value, + enabled_sources: &HashMap, + programmatic: &PluginConfig, +) -> Vec { + let Some(discovered_components) = discovered.get("components").and_then(Value::as_array) else { + return Vec::new(); + }; + let mut consumed = HashMap::new(); + let mut diagnostics = Vec::new(); + for component in &programmatic.components { + let nth = consumed.entry(component.kind.as_str()).or_insert(0usize); + let discovered_component = + nth_component_by_kind(discovered_components, &component.kind, *nth) + .and_then(|index| discovered_components.get(index)); + *nth += 1; + let discovered_enabled = discovered_component + .and_then(|component| component.get("enabled")) + .and_then(Value::as_bool); + let file_disabled = discovered_enabled == Some(false) + || (discovered_enabled.is_none() + && enabled_sources + .get(&component.kind) + .is_some_and(|source| !source.enabled)); + if !component.enabled || !file_disabled { + continue; + } + let source = enabled_sources + .get(&component.kind) + .map(|source| format!(" from {}", source.path.display())) + .unwrap_or_default(); + diagnostics.push(ConfigDiagnostic { + level: DiagnosticLevel::Warning, + code: "plugin.component_reenabled".to_owned(), + component: Some(component.kind.clone()), + field: Some("enabled".to_owned()), + message: format!( + "programmatic configuration enabled plugin component '{}' and overrode enabled = false{source}", + component.kind + ), + }); + } + diagnostics +} + +fn plugin_config_overlay_value(config: &PluginConfig) -> Result { + let mut overlay = serde_json::to_value(config)?; + let Value::Object(root) = &mut overlay else { + return Ok(overlay); + }; + if config.version == PluginConfig::default().version { + root.remove("version"); + } + remove_default_policy_overlay(root, &config.policy); + Ok(overlay) +} + +fn remove_default_policy_overlay(root: &mut Map, config: &ConfigPolicy) { + let Some(Value::Object(policy)) = root.get_mut("policy") else { + return; + }; + let defaults = ConfigPolicy::default(); + for (field, is_default) in [ + ( + "unknown_component", + config.unknown_component == defaults.unknown_component, + ), + ( + "unknown_field", + config.unknown_field == defaults.unknown_field, + ), + ( + "unsupported_value", + config.unsupported_value == defaults.unsupported_value, + ), + ] { + if is_default { + policy.remove(field); + } + } + if policy.is_empty() { + root.remove("policy"); + } +} + +fn layer_config(left: &mut Value, right: Value) { + match (left, right) { + (Value::Object(left), Value::Object(right)) => { + for (key, value) in right { + match (key.as_str(), left.get_mut(&key)) { + ("components", Some(existing)) => merge_plugin_components(existing, value), + (_, Some(existing)) => merge_json_value(existing, value), + (_, None) => { + left.insert(key, value); + } + } + } + } + (left, right) => *left = right, + } +} + +fn merge_plugin_components(left: &mut Value, right: Value) { + let Value::Array(left_components) = left else { + *left = right; + return; + }; + let Value::Array(right_components) = right else { + *left = right; + return; + }; + let base_component_count = left_components.len(); + let mut consumed = HashMap::new(); + for component in right_components { + let Some(kind) = component_kind(&component).map(str::to_owned) else { + left_components.push(component); + continue; + }; + let nth = consumed.entry(kind.clone()).or_insert(0usize); + let slot = nth_component_by_kind(&left_components[..base_component_count], &kind, *nth); + *nth += 1; + match slot { + Some(index) => merge_plugin_component(&mut left_components[index], component), + None => left_components.push(component), + } + } +} + +fn merge_plugin_component(existing: &mut Value, higher_priority: Value) { + let is_observability = component_kind(&higher_priority).or_else(|| component_kind(existing)) + == Some("observability"); + match (existing, higher_priority) { + (Value::Object(existing), Value::Object(higher_priority)) => { + for (key, value) in higher_priority { + match (key.as_str(), existing.get_mut(&key)) { + ("config", Some(existing_config)) => merge_plugin_config_value( + existing_config, + value, + &mut Vec::new(), + is_observability, + ), + (_, Some(existing_value)) => merge_json_value(existing_value, value), + (_, None) => { + existing.insert(key, value); + } + } + } + } + (existing, higher_priority) => *existing = higher_priority, + } +} + +fn merge_plugin_config_value( + lower_priority: &mut Value, + higher_priority: Value, + path: &mut Vec, + is_observability: bool, +) { + match (lower_priority, higher_priority) { + (Value::Object(lower_priority), Value::Object(higher_priority)) => { + for (key, value) in higher_priority { + path.push(key.clone()); + match lower_priority.get_mut(&key) { + Some(existing) => { + merge_plugin_config_value(existing, value, path, is_observability) + } + None => { + lower_priority.insert(key, value); + } + } + path.pop(); + } + } + (Value::Array(lower_priority), Value::Array(mut higher_priority)) + if plugin_config_list_concatenates(path, is_observability) => + { + higher_priority.append(lower_priority); + *lower_priority = higher_priority; + } + (lower_priority, higher_priority) => *lower_priority = higher_priority, + } +} + +fn plugin_config_list_concatenates(path: &[String], is_observability: bool) -> bool { + path.len() == 1 + || (is_observability + && matches!( + path, + [section, field] + if (section == "atof" && field == "sinks") + || (section == "opentelemetry" && field == "endpoints") + || (section == "atif" && field == "storage") + )) +} + +fn merge_json_value(left: &mut Value, right: Value) { + match (left, right) { + (Value::Object(left), Value::Object(right)) => { + for (key, value) in right { + match left.get_mut(&key) { + Some(existing) => merge_json_value(existing, value), + None => { + left.insert(key, value); + } + } + } + } + (left, right) => *left = right, + } +} + +fn component_kind(value: &Value) -> Option<&str> { + value.get("kind").and_then(Value::as_str) +} + +fn nth_component_by_kind(components: &[Value], kind: &str, nth: usize) -> Option { + components + .iter() + .enumerate() + .filter(|(_, component)| component_kind(component) == Some(kind)) + .nth(nth) + .map(|(index, _)| index) +} + +#[cfg(test)] +#[path = "../tests/unit/resolver.rs"] +mod tests; diff --git a/crates/plugin-host-config/src/snapshot.rs b/crates/plugin-host-config/src/snapshot.rs new file mode 100644 index 000000000..74425b4ce --- /dev/null +++ b/crates/plugin-host-config/src/snapshot.rs @@ -0,0 +1,960 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use nemo_relay::plugin::Result as PluginResult; +use nemo_relay::plugin::dynamic::{ + DynamicPluginActivationResource, DynamicPluginKind, DynamicPluginManifest, + DynamicPluginManifestLoad, DynamicPluginStartupClass, WorkerRuntime, +}; +use sha2::{Digest, Sha256}; + +use crate::environment::{ + ENVIRONMENT_ATTESTATION_FILE, MANAGED_ENVIRONMENTS_DIR, validate_python_entrypoint_artifact, + verify_environment_attestation, +}; +use crate::error::{PluginHostConfigError, Result}; +#[cfg(target_os = "macos")] +use crate::io::read_bounded_utf8_regular_file; +use crate::io::{ + MAX_BOUNDED_FILE_BYTES, load_bounded_dynamic_plugin_manifest_bytes, read_bounded_regular_file, +}; +use crate::policy::{DynamicPluginHostPolicy, evaluate_dynamic_plugin_host_policy}; +use crate::trust::evaluate_dynamic_plugin_trust; + +const MAX_SNAPSHOT_FILES: usize = 100_000; +const MAX_SNAPSHOT_DEPTH: usize = 128; + +/// Immutable filesystem snapshot retained for one dynamic plugin's runtime lifetime. +#[derive(Debug, PartialEq, Eq)] +pub struct DynamicPluginActivationSnapshot { + root: PathBuf, + original_manifest_ref: String, + identity_manifest: PathBuf, + activation_manifest: PathBuf, + activation_environment_ref: Option, + identity_files: HashMap, + closure_digest: String, + verification_digest: String, +} + +impl DynamicPluginActivationSnapshot { + /// Creates and verifies an immutable activation snapshot. + pub fn create( + manifest_ref: &str, + expected_plugin_id: &str, + expected_kind: DynamicPluginKind, + environment_ref: Option<&str>, + host_policy: &DynamicPluginHostPolicy, + ) -> Result> { + let (mut manifest, original_manifest_ref, manifest_bytes) = + load_bounded_dynamic_plugin_manifest_bytes(manifest_ref)?; + if manifest.plugin.id.trim() != expected_plugin_id || manifest.plugin.kind != expected_kind + { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin manifest identity changed before activation for '{expected_plugin_id}'" + ))); + } + validate_python_entrypoint_artifact(&manifest, &original_manifest_ref) + .map_err(PluginHostConfigError::InvalidConfig)?; + let policy = evaluate_dynamic_plugin_host_policy(host_policy, &manifest); + + let root = std::env::temp_dir().join(format!( + "nemo-relay-plugin-snapshot-{}", + uuid::Uuid::now_v7().simple() + )); + fs::create_dir(&root).map_err(|error| { + PluginHostConfigError::io("create activation snapshot", &root, error) + })?; + let mut root_guard = SnapshotRootGuard(Some(root.clone())); + #[cfg(unix)] + fs::set_permissions(&root, { + use std::os::unix::fs::PermissionsExt; + fs::Permissions::from_mode(0o700) + }) + .map_err(|error| PluginHostConfigError::io("protect activation snapshot", &root, error))?; + + let identity_manifest = root.join("identity-manifest.toml"); + fs::write(&identity_manifest, &manifest_bytes).map_err(|error| { + PluginHostConfigError::io( + "write activation identity manifest", + &identity_manifest, + error, + ) + })?; + let original_manifest_path = PathBuf::from(&original_manifest_ref); + let manifest_directory = original_manifest_path.parent().ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin manifest {} has no parent directory", + original_manifest_path.display() + )) + })?; + let runtime_root = root.join("runtime"); + let mut budget = SnapshotBudget::default(); + let mut copied_files = HashMap::new(); + copy_snapshot_directory( + manifest_directory, + &runtime_root, + &mut copied_files, + &mut budget, + false, + &mut Vec::new(), + )?; + let declared_artifact = manifest + .source + .as_ref() + .and_then(|source| source.artifact.as_deref()) + .map(|artifact| { + fs::canonicalize(resolve_manifest_relative_path(&original_manifest_path, artifact)) + }) + .transpose() + .map_err(|error| { + PluginHostConfigError::InvalidConfig(format!( + "failed to normalize dynamic plugin artifact for '{expected_plugin_id}': {error}" + )) + })?; + let mut identity_files = HashMap::new(); + + match &mut manifest.load { + DynamicPluginManifestLoad::RustDynamic(load) => { + if let Some(library) = load.library.as_deref() { + let (logical, canonical, copied) = copy_snapshot_file( + &root, + &original_manifest_path, + library, + "library", + &mut copied_files, + &mut budget, + )?; + if declared_artifact.as_ref() != Some(&canonical) { + return Err(PluginHostConfigError::InvalidConfig(format!( + "native dynamic plugin '{expected_plugin_id}' must declare its load.library as the integrity-checked source.artifact" + ))); + } + identity_files + .entry(logical) + .or_insert_with(|| copied.clone()); + load.library = Some(copied.to_string_lossy().into_owned()); + } + } + DynamicPluginManifestLoad::Worker(load) + if matches!( + load.runtime, + Some(WorkerRuntime::Rust | WorkerRuntime::Command) + ) => + { + if let Some(entrypoint) = load.entrypoint.as_deref() { + let (logical, canonical, copied) = copy_snapshot_file( + &root, + &original_manifest_path, + entrypoint, + "entrypoint", + &mut copied_files, + &mut budget, + )?; + if declared_artifact.as_ref() != Some(&canonical) { + return Err(PluginHostConfigError::InvalidConfig(format!( + "command worker dynamic plugin '{expected_plugin_id}' must declare its load.entrypoint as the integrity-checked source.artifact" + ))); + } + identity_files + .entry(logical) + .or_insert_with(|| copied.clone()); + load.entrypoint = Some(copied.to_string_lossy().into_owned()); + } + } + DynamicPluginManifestLoad::Worker(_) => {} + } + + if let Some(source) = manifest.source.as_mut() + && let Some(artifact) = source.artifact.as_deref() + { + let (logical, _, copied) = copy_snapshot_file( + &root, + &original_manifest_path, + artifact, + "artifact", + &mut copied_files, + &mut budget, + )?; + identity_files.insert(logical, copied.clone()); + source.artifact = Some(copied.to_string_lossy().into_owned()); + } + if let Some(integrity) = manifest.integrity.as_mut() + && let Some(signature) = integrity.signature.as_deref() + { + let (logical, _, copied) = copy_snapshot_file( + &root, + &original_manifest_path, + signature, + "signature", + &mut copied_files, + &mut budget, + )?; + identity_files.insert(logical, copied.clone()); + integrity.signature = Some(copied.to_string_lossy().into_owned()); + } + + let activation_environment_ref = snapshot_python_environment( + &manifest, + environment_ref, + expected_plugin_id, + &root, + &mut copied_files, + &mut budget, + )?; + let activation_manifest = runtime_root.join("relay-plugin.toml"); + let rendered = toml::to_string(&manifest).map_err(|error| { + PluginHostConfigError::InvalidConfig(format!( + "failed to encode dynamic plugin activation snapshot for '{expected_plugin_id}': {error}" + )) + })?; + if rendered.len() as u64 > MAX_BOUNDED_FILE_BYTES { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin activation manifest for '{expected_plugin_id}' exceeds the {MAX_BOUNDED_FILE_BYTES}-byte activation snapshot budget" + ))); + } + fs::write(&activation_manifest, rendered).map_err(|error| { + PluginHostConfigError::io("write activation manifest", &activation_manifest, error) + })?; + + let trust = evaluate_dynamic_plugin_trust( + &manifest, + activation_manifest.to_string_lossy().as_ref(), + &policy, + ); + if policy.startup_class == DynamicPluginStartupClass::Required { + if !policy.policy_satisfied { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin '{expected_plugin_id}' activation snapshot violates host policy" + ))); + } + if let Some(failure) = trust.failure() { + return Err(PluginHostConfigError::InvalidConfig( + failure.display(expected_plugin_id).to_string(), + )); + } + } + + let closure_digest = snapshot_tree_digest(&root, true)?; + let verification_digest = snapshot_tree_digest(&root, false)?; + protect_snapshot_tree(&root)?; + root_guard.0 = None; + Ok(Arc::new(Self { + root, + original_manifest_ref, + identity_manifest, + activation_manifest, + activation_environment_ref, + identity_files, + closure_digest, + verification_digest, + })) + } + + /// Returns the rewritten manifest consumed by the native or worker loader. + pub fn activation_manifest_ref(&self) -> String { + self.activation_manifest.to_string_lossy().into_owned() + } + + /// Returns the copied lifecycle-managed Python environment, when applicable. + pub fn activation_environment_ref(&self) -> Option<&str> { + self.activation_environment_ref.as_deref() + } + + /// Returns a stable digest of the snapshotted runtime closure. + pub fn closure_digest(&self) -> &str { + &self.closure_digest + } + + /// Verifies that the protected snapshot has not changed since construction. + pub fn verify_current(&self) -> Result<()> { + let actual = snapshot_tree_digest(&self.root, false)?; + if actual == self.verification_digest { + Ok(()) + } else { + Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin activation snapshot {} changed before code load", + self.root.display() + ))) + } + } + + /// Returns the original canonical manifest path. + pub fn original_manifest_ref(&self) -> &str { + &self.original_manifest_ref + } + + /// Returns the immutable authored manifest copy used for identity reporting. + pub fn identity_manifest(&self) -> &Path { + &self.identity_manifest + } + + /// Returns the snapshotted copy corresponding to one authored logical path. + pub fn identity_file(&self, logical_path: &Path) -> Option<&Path> { + self.identity_files.get(logical_path).map(PathBuf::as_path) + } +} + +impl DynamicPluginActivationResource for DynamicPluginActivationSnapshot { + fn verify(&self) -> PluginResult<()> { + self.verify_current() + .map_err(|error| error.into_plugin_error()) + } +} + +impl Drop for DynamicPluginActivationSnapshot { + fn drop(&mut self) { + make_snapshot_removable(&self.root); + let _ = fs::remove_dir_all(&self.root); + } +} + +fn snapshot_python_environment( + manifest: &DynamicPluginManifest, + environment_ref: Option<&str>, + expected_plugin_id: &str, + root: &Path, + copied_files: &mut HashMap, + budget: &mut SnapshotBudget, +) -> Result> { + if !matches!( + &manifest.load, + DynamicPluginManifestLoad::Worker(load) if load.runtime == Some(WorkerRuntime::Python) + ) { + return Ok(None); + } + let environment = environment_ref.ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "Python worker dynamic plugin '{expected_plugin_id}' has no managed environment" + )) + })?; + let digest = trusted_source_artifact_sha256(manifest)?; + let environment = PathBuf::from(environment); + verify_environment_attestation(&environment, digest)?; + let environment_name = environment.file_name().ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "managed Python environment {} has no lifecycle environment name", + environment.display() + )) + })?; + let copied_environment = root.join(MANAGED_ENVIRONMENTS_DIR).join(environment_name); + copy_snapshot_directory( + &environment, + &copied_environment, + copied_files, + budget, + true, + &mut Vec::new(), + )?; + verify_environment_attestation(&copied_environment, digest)?; + #[cfg(target_os = "macos")] + // Some relocatable CPython builds link their launcher through + // `@rpath/libpython*.dylib`. The launcher is materialized to keep it pinned, + // so retain that runtime library in the copied environment as well. + snapshot_macos_python_runtime_library(&copied_environment, copied_files, budget)?; + Ok(Some(copied_environment.to_string_lossy().into_owned())) +} + +#[cfg(target_os = "macos")] +fn snapshot_macos_python_runtime_library( + copied_environment: &Path, + copied_files: &mut HashMap, + budget: &mut SnapshotBudget, +) -> Result<()> { + let pyvenv_config = copied_environment.join("pyvenv.cfg"); + let contents = match read_bounded_utf8_regular_file(&pyvenv_config, "Python environment config") + { + Ok(contents) => contents, + Err(PluginHostConfigError::Io { source, .. }) + if source.kind() == std::io::ErrorKind::NotFound => + { + return Ok(()); + } + Err(error) => return Err(error), + }; + let value = |expected: &str| { + contents.lines().find_map(|line| { + let (key, value) = line.split_once('=')?; + (key.trim() == expected) + .then_some(value.trim()) + .filter(|value| !value.is_empty()) + }) + }; + let Some(home) = value("home") else { + return Ok(()); + }; + let Some(version) = value("version_info").or_else(|| value("version")) else { + return Ok(()); + }; + let mut version = version.split('.'); + let Some(major) = version + .next() + .filter(|part| !part.is_empty() && part.chars().all(|value| value.is_ascii_digit())) + else { + return Ok(()); + }; + let Some(minor) = version + .next() + .filter(|part| !part.is_empty() && part.chars().all(|value| value.is_ascii_digit())) + else { + return Ok(()); + }; + let home = PathBuf::from(home); + let Some(prefix) = home.parent() else { + return Ok(()); + }; + let library_name = format!("libpython{major}.{minor}.dylib"); + let source = prefix.join("lib").join(&library_name); + let destination = copied_environment.join("lib").join(&library_name); + match fs::symlink_metadata(&destination) { + Ok(metadata) if metadata.file_type().is_file() => return Ok(()), + Ok(_) => { + return Err(PluginHostConfigError::InvalidConfig(format!( + "snapshotted Python runtime library {} must be a regular file", + destination.display() + ))); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(PluginHostConfigError::io( + "inspect snapshotted Python runtime library", + &destination, + error, + )); + } + } + let source_metadata = match fs::symlink_metadata(&source) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => { + return Err(PluginHostConfigError::io( + "inspect Python runtime library", + &source, + error, + )); + } + }; + let canonical = if source_metadata.file_type().is_symlink() { + fs::canonicalize(&source).map_err(|error| { + PluginHostConfigError::io("normalize Python runtime library", &source, error) + })? + } else { + source.clone() + }; + if !fs::metadata(&canonical) + .map_err(|error| { + PluginHostConfigError::io("inspect Python runtime library", &canonical, error) + })? + .is_file() + { + return Err(PluginHostConfigError::InvalidConfig(format!( + "Python runtime library {} must resolve to a regular file", + source.display() + ))); + } + let copied_library_directory = destination.parent().ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "snapshotted Python runtime library {} has no parent directory", + destination.display() + )) + })?; + fs::create_dir_all(copied_library_directory).map_err(|error| { + PluginHostConfigError::io( + "create Python runtime library snapshot directory", + copied_library_directory, + error, + ) + })?; + budget.record_entry(&source)?; + copy_snapshot_regular_file(&canonical, &destination, copied_files, budget)?; + Ok(()) +} + +fn trusted_source_artifact_sha256(manifest: &DynamicPluginManifest) -> Result<&str> { + manifest + .integrity + .as_ref() + .and_then(|integrity| integrity.sha256.as_deref()) + .map(str::trim) + .filter(|digest| !digest.is_empty()) + .ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "Python worker dynamic plugin '{}' requires integrity.sha256 to bind its complete installed environment to the trusted source artifact", + manifest.plugin.id + )) + }) +} + +struct SnapshotRootGuard(Option); + +impl Drop for SnapshotRootGuard { + fn drop(&mut self) { + if let Some(root) = self.0.take() { + make_snapshot_removable(&root); + let _ = fs::remove_dir_all(root); + } + } +} + +#[derive(Default)] +struct SnapshotBudget { + entries: usize, + bytes: u64, +} + +impl SnapshotBudget { + fn record_entry(&mut self, path: &Path) -> Result<()> { + self.entries = self.entries.saturating_add(1); + if self.entries > MAX_SNAPSHOT_FILES { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin runtime closure exceeds the {MAX_SNAPSHOT_FILES}-entry activation snapshot budget at {}", + path.display() + ))); + } + Ok(()) + } + + fn record_bytes(&mut self, path: &Path, bytes: usize) -> Result<()> { + self.bytes = self.bytes.saturating_add(bytes as u64); + if self.bytes > MAX_BOUNDED_FILE_BYTES { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin runtime closure exceeds the {MAX_BOUNDED_FILE_BYTES}-byte activation snapshot budget at {}", + path.display() + ))); + } + Ok(()) + } +} + +fn copy_snapshot_directory( + source: &Path, + destination: &Path, + copied_files: &mut HashMap, + budget: &mut SnapshotBudget, + skip_python_cache: bool, + ancestors: &mut Vec, +) -> Result<()> { + budget.record_entry(source)?; + copy_snapshot_directory_contents( + source, + destination, + copied_files, + budget, + skip_python_cache, + ancestors, + ) +} + +fn copy_snapshot_directory_contents( + source: &Path, + destination: &Path, + copied_files: &mut HashMap, + budget: &mut SnapshotBudget, + skip_python_cache: bool, + ancestors: &mut Vec, +) -> Result<()> { + if ancestors.len() >= MAX_SNAPSHOT_DEPTH { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin runtime closure exceeds the {MAX_SNAPSHOT_DEPTH}-directory traversal depth at {}", + source.display() + ))); + } + let canonical = fs::canonicalize(source) + .map_err(|error| PluginHostConfigError::io("normalize runtime directory", source, error))?; + if ancestors.contains(&canonical) { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin runtime closure contains a directory symlink cycle at {}", + source.display() + ))); + } + ancestors.push(canonical.clone()); + fs::create_dir_all(destination).map_err(|error| { + PluginHostConfigError::io("create snapshot directory", destination, error) + })?; + let mut entries = fs::read_dir(&canonical) + .map_err(|error| PluginHostConfigError::io("read runtime directory", &canonical, error))? + .collect::, _>>() + .map_err(|error| { + PluginHostConfigError::io("read runtime directory entry", &canonical, error) + })?; + entries.sort_by_key(fs::DirEntry::file_name); + for entry in entries { + budget.record_entry(&entry.path())?; + copy_snapshot_entry( + entry, + destination, + copied_files, + budget, + skip_python_cache, + ancestors, + )?; + } + ancestors.pop(); + Ok(()) +} + +fn copy_snapshot_entry( + entry: fs::DirEntry, + destination: &Path, + copied_files: &mut HashMap, + budget: &mut SnapshotBudget, + skip_python_cache: bool, + ancestors: &mut Vec, +) -> Result<()> { + let source_path = entry.path(); + if skip_python_cache + && (entry.file_name() == "__pycache__" + || source_path.extension().and_then(|value| value.to_str()) == Some("pyc")) + { + return Ok(()); + } + let destination_path = destination.join(entry.file_name()); + let metadata = fs::symlink_metadata(&source_path) + .map_err(|error| PluginHostConfigError::io("inspect runtime entry", &source_path, error))?; + let resolved = if metadata.file_type().is_symlink() { + fs::canonicalize(&source_path).map_err(|error| { + PluginHostConfigError::io("resolve runtime symlink", &source_path, error) + })? + } else { + source_path.clone() + }; + let resolved_metadata = fs::metadata(&resolved).map_err(|error| { + PluginHostConfigError::io("inspect resolved runtime entry", &resolved, error) + })?; + if resolved_metadata.is_dir() { + return copy_snapshot_directory_contents( + &resolved, + &destination_path, + copied_files, + budget, + skip_python_cache, + ancestors, + ); + } + if !resolved_metadata.is_file() { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin runtime entry {} must resolve to a regular file or directory", + source_path.display() + ))); + } + if preserve_python_launcher( + &source_path, + &destination_path, + &resolved, + &metadata, + copied_files, + )? { + return Ok(()); + } + copy_snapshot_regular_file(&resolved, &destination_path, copied_files, budget) +} + +#[cfg(unix)] +fn preserve_python_launcher( + source: &Path, + destination: &Path, + resolved: &Path, + metadata: &fs::Metadata, + copied_files: &mut HashMap, +) -> Result { + let is_versioned_launcher_alias = source.parent().and_then(Path::file_name) + == Some(std::ffi::OsStr::new("bin")) + && source + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("python3")); + if !metadata.file_type().is_symlink() || !is_versioned_launcher_alias { + return Ok(false); + } + let target = fs::read_link(source).map_err(|error| { + PluginHostConfigError::io("read Python launcher symlink", source, error) + })?; + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent).map_err(|error| { + PluginHostConfigError::io("create Python launcher directory", parent, error) + })?; + } + std::os::unix::fs::symlink(target, destination).map_err(|error| { + PluginHostConfigError::io("preserve Python launcher symlink", destination, error) + })?; + copied_files.insert(resolved.to_path_buf(), destination.to_path_buf()); + Ok(true) +} + +#[cfg(not(unix))] +fn preserve_python_launcher( + _source: &Path, + _destination: &Path, + _resolved: &Path, + _metadata: &fs::Metadata, + _copied_files: &mut HashMap, +) -> Result { + Ok(false) +} + +fn copy_snapshot_file( + root: &Path, + manifest_path: &Path, + reference: &str, + label: &str, + copied_files: &mut HashMap, + budget: &mut SnapshotBudget, +) -> Result<(PathBuf, PathBuf, PathBuf)> { + let logical = resolve_manifest_relative_path(manifest_path, reference); + let canonical = fs::canonicalize(&logical).map_err(|error| { + PluginHostConfigError::io("normalize dynamic plugin artifact", &logical, error) + })?; + if let Some(copied) = copied_files.get(&canonical) { + return Ok((logical, canonical, copied.clone())); + } + let external = root.join(format!("external-{label}")); + if matches!(label, "library" | "entrypoint") { + let parent = canonical.parent().ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin {label} {} has no parent directory", + canonical.display() + )) + })?; + copy_snapshot_directory( + parent, + &external, + copied_files, + budget, + false, + &mut Vec::new(), + )?; + } else { + fs::create_dir_all(&external).map_err(|error| { + PluginHostConfigError::io("create external snapshot directory", &external, error) + })?; + let destination = external.join(canonical.file_name().unwrap_or_default()); + copy_snapshot_regular_file(&canonical, &destination, copied_files, budget)?; + } + let copied = copied_files.get(&canonical).cloned().ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin {label} {} was not included in its activation snapshot", + canonical.display() + )) + })?; + Ok((logical, canonical, copied)) +} + +fn copy_snapshot_regular_file( + source: &Path, + destination: &Path, + copied_files: &mut HashMap, + budget: &mut SnapshotBudget, +) -> Result<()> { + let bytes = read_bounded_regular_file(source, "dynamic plugin runtime file")?; + budget.record_bytes(source, bytes.len())?; + fs::write(destination, bytes) + .map_err(|error| PluginHostConfigError::io("write snapshot file", destination, error))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = fs::metadata(source) + .map_err(|error| PluginHostConfigError::io("inspect snapshot source", source, error))? + .permissions() + .mode(); + fs::set_permissions(destination, fs::Permissions::from_mode(mode)).map_err(|error| { + PluginHostConfigError::io("preserve snapshot permissions", destination, error) + })?; + } + copied_files.insert(source.to_path_buf(), destination.to_path_buf()); + Ok(()) +} + +fn resolve_manifest_relative_path(manifest_path: &Path, reference: &str) -> PathBuf { + let path = PathBuf::from(reference); + if path.is_absolute() { + path + } else { + manifest_path + .parent() + .map(|parent| parent.join(&path)) + .unwrap_or(path) + } +} + +fn snapshot_tree_digest(root: &Path, stable_identity: bool) -> Result { + let mut files = Vec::new(); + collect_snapshot_files(root, root, &mut files, 0, &mut 0)?; + files.sort(); + let mut digest = Sha256::new(); + let mut bytes = 0_u64; + for relative in files { + if stable_identity { + let activation_manifest = Path::new("runtime").join("relay-plugin.toml"); + let python_environment_content = relative.starts_with(MANAGED_ENVIRONMENTS_DIR) + && relative.file_name() != Some(std::ffi::OsStr::new(ENVIRONMENT_ATTESTATION_FILE)); + if relative == activation_manifest || python_environment_content { + continue; + } + } + let path = root.join(&relative); + let metadata = fs::symlink_metadata(&path) + .map_err(|error| PluginHostConfigError::io("inspect snapshot entry", &path, error))?; + let (kind, payload) = if metadata.file_type().is_symlink() { + let target = fs::read_link(&path).map_err(|error| { + PluginHostConfigError::io("read snapshot symlink", &path, error) + })?; + (1_u8, target.as_os_str().as_encoded_bytes().to_vec()) + } else { + ( + 0_u8, + read_bounded_regular_file(&path, "dynamic plugin activation snapshot file")?, + ) + }; + bytes = bytes.saturating_add(payload.len() as u64); + if bytes > MAX_BOUNDED_FILE_BYTES { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin activation snapshot exceeds the {MAX_BOUNDED_FILE_BYTES}-byte verification budget" + ))); + } + let relative_bytes = relative.as_os_str().as_encoded_bytes(); + digest.update([kind]); + digest.update((relative_bytes.len() as u64).to_le_bytes()); + digest.update(relative_bytes); + digest.update((payload.len() as u64).to_le_bytes()); + digest.update(payload); + } + Ok(digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect()) +} + +fn collect_snapshot_files( + root: &Path, + directory: &Path, + files: &mut Vec, + depth: usize, + entries: &mut usize, +) -> Result<()> { + if depth >= MAX_SNAPSHOT_DEPTH { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin activation snapshot exceeds the {MAX_SNAPSHOT_DEPTH}-directory traversal depth at {}", + directory.display() + ))); + } + for entry in fs::read_dir(directory) + .map_err(|error| PluginHostConfigError::io("read activation snapshot", directory, error))? + { + *entries = entries.saturating_add(1); + if *entries > MAX_SNAPSHOT_FILES { + return Err(PluginHostConfigError::InvalidConfig(format!( + "dynamic plugin activation snapshot exceeds the {MAX_SNAPSHOT_FILES}-entry verification budget at {}", + directory.display() + ))); + } + let path = entry + .map_err(|error| { + PluginHostConfigError::io("read activation snapshot entry", directory, error) + })? + .path(); + let metadata = fs::symlink_metadata(&path).map_err(|error| { + PluginHostConfigError::io("inspect activation snapshot entry", &path, error) + })?; + if metadata.is_dir() { + collect_snapshot_files(root, &path, files, depth + 1, entries)?; + } else { + files.push( + path.strip_prefix(root) + .map_err(|error| PluginHostConfigError::InvalidConfig(error.to_string()))? + .to_path_buf(), + ); + } + } + Ok(()) +} + +#[cfg(unix)] +fn protect_snapshot_tree(root: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + for entry in fs::read_dir(root) + .map_err(|error| PluginHostConfigError::io("read activation snapshot", root, error))? + { + let path = entry + .map_err(|error| { + PluginHostConfigError::io("read activation snapshot entry", root, error) + })? + .path(); + let metadata = fs::symlink_metadata(&path).map_err(|error| { + PluginHostConfigError::io("inspect activation snapshot entry", &path, error) + })?; + if metadata.is_dir() { + protect_snapshot_tree(&path)?; + } else if !metadata.file_type().is_symlink() { + let mode = metadata.permissions().mode() & !0o222; + fs::set_permissions(&path, fs::Permissions::from_mode(mode)).map_err(|error| { + PluginHostConfigError::io("protect activation snapshot entry", &path, error) + })?; + } + } + fs::set_permissions(root, fs::Permissions::from_mode(0o500)).map_err(|error| { + PluginHostConfigError::io("protect activation snapshot directory", root, error) + }) +} + +#[cfg(windows)] +fn protect_snapshot_tree(root: &Path) -> Result<()> { + for entry in fs::read_dir(root) + .map_err(|error| PluginHostConfigError::io("read activation snapshot", root, error))? + { + let path = entry + .map_err(|error| { + PluginHostConfigError::io("read activation snapshot entry", root, error) + })? + .path(); + let metadata = fs::symlink_metadata(&path).map_err(|error| { + PluginHostConfigError::io("inspect activation snapshot entry", &path, error) + })?; + if metadata.is_dir() { + protect_snapshot_tree(&path)?; + } else if !metadata.file_type().is_symlink() { + let mut permissions = metadata.permissions(); + permissions.set_readonly(true); + fs::set_permissions(&path, permissions).map_err(|error| { + PluginHostConfigError::io("protect activation snapshot entry", &path, error) + })?; + } + } + Ok(()) +} + +fn make_snapshot_removable(root: &Path) { + let Ok(entries) = fs::read_dir(root) else { + return; + }; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = fs::set_permissions(root, fs::Permissions::from_mode(0o700)); + } + for entry in entries.flatten() { + let path = entry.path(); + let Ok(metadata) = fs::symlink_metadata(&path) else { + continue; + }; + if metadata.is_dir() { + make_snapshot_removable(&path); + } else { + #[cfg(windows)] + if !metadata.file_type().is_symlink() { + let mut permissions = metadata.permissions(); + permissions.set_readonly(false); + let _ = fs::set_permissions(&path, permissions); + } + } + } +} + +#[cfg(test)] +#[path = "../tests/unit/snapshot.rs"] +mod tests; diff --git a/crates/plugin-host-config/src/state.rs b/crates/plugin-host-config/src/state.rs new file mode 100644 index 000000000..c15882c81 --- /dev/null +++ b/crates/plugin-host-config/src/state.rs @@ -0,0 +1,454 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared lifecycle-registry persistence and transaction locking. + +use std::collections::BTreeMap; +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::ops::{Deref, DerefMut}; +use std::path::{Path, PathBuf}; + +use fs2::FileExt; +use nemo_relay::plugin::dynamic::{DynamicPluginRecord, DynamicPluginRegistry}; +use serde::{Deserialize, Serialize}; + +use crate::error::{PluginHostConfigError, Result}; +use crate::io::read_bounded_utf8_regular_file; + +pub(crate) const DYNAMIC_PLUGIN_STATE_FILENAME: &str = ".dynamic-plugins.json"; +const DYNAMIC_PLUGIN_STATE_LOCK_FILENAME: &str = ".dynamic-plugins.lock"; +const DYNAMIC_PLUGIN_STATE_SCHEMA_VERSION: u32 = 1; + +#[derive(Debug, Default, Serialize, Deserialize)] +struct PersistedDynamicPluginRegistry { + #[serde(default = "default_state_schema_version")] + schema_version: u32, + #[serde(default)] + records: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + declaration_sources: BTreeMap, +} + +const fn default_state_schema_version() -> u32 { + DYNAMIC_PLUGIN_STATE_SCHEMA_VERSION +} + +/// Exclusive transaction lock for one sibling dynamic-plugin lifecycle registry. +/// +/// Callers must hold this guard continuously from the registry read through its atomic save. +#[doc(hidden)] +pub struct LifecycleStateLock { + state_path: PathBuf, + lock_path: PathBuf, + file: File, +} + +/// In-memory lifecycle state, including internal physical declaration ownership. +#[doc(hidden)] +#[derive(Debug, Default)] +pub struct DynamicPluginLifecycleState { + registry: DynamicPluginRegistry, + declaration_sources: BTreeMap, +} + +impl DynamicPluginLifecycleState { + /// Creates lifecycle state around a registry with no declaration ownership metadata. + pub fn new(registry: DynamicPluginRegistry) -> Self { + Self { + registry, + declaration_sources: BTreeMap::new(), + } + } + + /// Returns the source that owns `plugin_id`, if ownership has been recorded. + pub fn declaration_source(&self, plugin_id: &str) -> Option<&str> { + self.declaration_sources.get(plugin_id).map(String::as_str) + } + + /// Records the canonical physical source that owns `plugin_id`. + pub fn set_declaration_source(&mut self, plugin_id: &str, source: String) -> Result<()> { + if self.registry.get(plugin_id).is_none() { + return Err(PluginHostConfigError::InvalidConfig(format!( + "cannot assign declaration ownership for unknown dynamic plugin '{plugin_id}'" + ))); + } + self.declaration_sources + .insert(plugin_id.to_owned(), source); + Ok(()) + } + + /// Clears physical declaration ownership for `plugin_id`. + pub fn clear_declaration_source(&mut self, plugin_id: &str) { + self.declaration_sources.remove(plugin_id); + } + + /// Replaces the record registry and drops ownership for records no longer present. + pub fn replace_registry(&mut self, registry: DynamicPluginRegistry) { + self.registry = registry; + self.retain_live_declaration_sources(); + } + + fn retain_live_declaration_sources(&mut self) { + self.declaration_sources + .retain(|plugin_id, _| self.registry.get(plugin_id).is_some()); + } + + fn into_registry(self) -> DynamicPluginRegistry { + self.registry + } +} + +impl Deref for DynamicPluginLifecycleState { + type Target = DynamicPluginRegistry; + + fn deref(&self) -> &Self::Target { + &self.registry + } +} + +impl DerefMut for DynamicPluginLifecycleState { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.registry + } +} + +impl std::fmt::Debug for LifecycleStateLock { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("LifecycleStateLock") + .field("state_path", &self.state_path) + .field("lock_path", &self.lock_path) + .finish_non_exhaustive() + } +} + +impl Drop for LifecycleStateLock { + fn drop(&mut self) { + let _ = FileExt::unlock(&self.file); + } +} + +/// Returns the lifecycle-state file adjacent to a physical `plugins.toml` source. +#[doc(hidden)] +pub fn sibling_lifecycle_state_path(plugins_toml_path: &Path) -> PathBuf { + plugins_toml_path + .parent() + .map(|parent| parent.join(DYNAMIC_PLUGIN_STATE_FILENAME)) + .unwrap_or_else(|| PathBuf::from(DYNAMIC_PLUGIN_STATE_FILENAME)) +} + +/// Pins a plugin configuration path to its portable physical location. +/// +/// Existing files resolve through symlinks. For a not-yet-created file, the nearest existing +/// parent is canonicalized so the CLI control plane and embedded runtime choose the same sibling +/// lifecycle registry after creation. On Windows, canonical paths use the legacy representation +/// whenever it is unambiguous so child runtimes do not receive a verbatim path they cannot use. +#[doc(hidden)] +pub fn pin_plugin_config_path(path: &Path) -> Result { + match dunce::canonicalize(path) { + Ok(path) => return Ok(path), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(PluginHostConfigError::InvalidConfig(format!( + "failed to normalize plugin configuration file {}: {error}", + path.display() + ))); + } + } + + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir() + .map_err(|error| { + PluginHostConfigError::io("resolve plugin configuration path", path, error) + })? + .join(path) + }; + let mut unresolved_components = Vec::new(); + let mut candidate = absolute.as_path(); + loop { + match std::fs::canonicalize(candidate) { + Ok(mut pinned) => { + for component in unresolved_components.iter().rev() { + pinned.push(component); + } + return Ok(dunce::simplified(&pinned).to_path_buf()); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let Some(file_name) = candidate.file_name() else { + return Err(PluginHostConfigError::InvalidConfig(format!( + "plugin configuration path {} does not name a file", + path.display() + ))); + }; + unresolved_components.push(file_name.to_os_string()); + candidate = candidate.parent().ok_or_else(|| { + PluginHostConfigError::InvalidConfig(format!( + "plugin configuration path {} has no existing ancestor", + path.display() + )) + })?; + } + Err(error) => { + return Err(PluginHostConfigError::InvalidConfig(format!( + "failed to normalize plugin configuration file {}: {error}", + path.display() + ))); + } + } + } +} + +/// Acquires the stable sibling lock used for lifecycle read-modify-write transactions. +#[doc(hidden)] +pub fn lock_lifecycle_state(state_path: &Path) -> Result { + let parent = state_path + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + std::fs::create_dir_all(&parent).map_err(|error| { + PluginHostConfigError::io("create lifecycle state directory", &parent, error) + })?; + let lock_path = parent.join(DYNAMIC_PLUGIN_STATE_LOCK_FILENAME); + let mut options = OpenOptions::new(); + options.create(true).read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = options.open(&lock_path).map_err(|error| { + PluginHostConfigError::io("open lifecycle state lock", &lock_path, error) + })?; + FileExt::lock_exclusive(&file) + .map_err(|error| PluginHostConfigError::io("lock lifecycle state", &lock_path, error))?; + Ok(LifecycleStateLock { + state_path: state_path.to_path_buf(), + lock_path, + file, + }) +} + +/// Reads a lifecycle registry while its sibling transaction lock is held. +#[doc(hidden)] +pub fn read_locked_lifecycle_registry(lock: &LifecycleStateLock) -> Result { + Ok(read_locked_lifecycle_state(lock)?.into_registry()) +} + +/// Reads lifecycle records and internal declaration ownership while its lock is held. +#[doc(hidden)] +pub fn read_locked_lifecycle_state( + lock: &LifecycleStateLock, +) -> Result { + read_lifecycle_state(&lock.state_path) +} + +/// Reads a lifecycle registry without acquiring its sibling mutation lock. +/// +/// Lifecycle writers replace the complete JSON document atomically, so read-only callers can +/// safely inspect the current durable snapshot without requiring write access to its directory. +#[doc(hidden)] +pub fn read_lifecycle_registry(state_path: &Path) -> Result { + Ok(read_lifecycle_state(state_path)?.into_registry()) +} + +/// Reads lifecycle records and internal declaration ownership without acquiring a mutation lock. +#[doc(hidden)] +pub fn read_lifecycle_state(state_path: &Path) -> Result { + let raw = match read_bounded_utf8_regular_file(state_path, "dynamic plugin lifecycle state") { + Ok(raw) => raw, + Err(PluginHostConfigError::Io { source, .. }) + if source.kind() == std::io::ErrorKind::NotFound => + { + return Ok(DynamicPluginLifecycleState::default()); + } + Err(error) => return Err(error), + }; + let state: PersistedDynamicPluginRegistry = serde_json::from_str(&raw).map_err(|error| { + PluginHostConfigError::json_parse("dynamic plugin registry state", state_path, &error) + })?; + if state.schema_version != DYNAMIC_PLUGIN_STATE_SCHEMA_VERSION { + return Err(PluginHostConfigError::InvalidConfig(format!( + "unsupported dynamic plugin registry schema_version {} in {}; expected {}", + state.schema_version, + state_path.display(), + DYNAMIC_PLUGIN_STATE_SCHEMA_VERSION + ))); + } + let registry = DynamicPluginRegistry::from_records(state.records)?; + let mut lifecycle_state = DynamicPluginLifecycleState { + registry, + declaration_sources: state.declaration_sources, + }; + lifecycle_state.retain_live_declaration_sources(); + Ok(lifecycle_state) +} + +/// Atomically saves a lifecycle registry while its sibling transaction lock is held. +#[doc(hidden)] +pub fn save_locked_lifecycle_registry( + lock: &LifecycleStateLock, + registry: &DynamicPluginRegistry, +) -> Result<()> { + let mut state = read_locked_lifecycle_state(lock)?; + state.replace_registry(DynamicPluginRegistry::from_records( + registry.cloned_records(true), + )?); + save_locked_lifecycle_state(lock, &state) +} + +/// Atomically saves lifecycle records and internal declaration ownership while locked. +#[doc(hidden)] +pub fn save_locked_lifecycle_state( + lock: &LifecycleStateLock, + state: &DynamicPluginLifecycleState, +) -> Result<()> { + let declaration_sources = state + .declaration_sources + .iter() + .filter(|(plugin_id, _)| state.registry.get(plugin_id).is_some()) + .map(|(plugin_id, source)| (plugin_id.clone(), source.clone())) + .collect(); + let mut rendered = serde_json::to_vec_pretty(&PersistedDynamicPluginRegistry { + schema_version: DYNAMIC_PLUGIN_STATE_SCHEMA_VERSION, + records: state.registry.cloned_records(true), + declaration_sources, + })?; + rendered.push(b'\n'); + let parent = lock + .state_path + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + let (temp_path, mut file) = create_lifecycle_state_temp( + &parent, + lock.state_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("dynamic-plugins"), + )?; + let write_result = (|| -> Result<()> { + file.write_all(&rendered).map_err(|error| { + PluginHostConfigError::io("write lifecycle state", &temp_path, error) + })?; + file.sync_all().map_err(|error| { + PluginHostConfigError::io("sync lifecycle state", &temp_path, error) + })?; + drop(file); + replace_lifecycle_state(&temp_path, &lock.state_path)?; + Ok(()) + })(); + if write_result.is_err() { + let _ = std::fs::remove_file(&temp_path); + } + write_result +} + +fn create_lifecycle_state_temp(parent: &Path, name: &str) -> Result<(PathBuf, File)> { + for _ in 0..16 { + let path = parent.join(format!(".{name}.{}.tmp", uuid::Uuid::now_v7().simple())); + #[cfg(windows)] + let opened = crate::environment::create_private_windows_file(&path); + #[cfg(not(windows))] + let opened = { + let mut options = OpenOptions::new(); + options.create_new(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + options.open(&path) + }; + match opened { + Ok(file) => return Ok((path, file)), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => { + return Err(PluginHostConfigError::io( + "create lifecycle state", + &path, + error, + )); + } + } + } + Err(PluginHostConfigError::InvalidConfig(format!( + "could not allocate a collision-free lifecycle state temporary file in {}", + parent.display() + ))) +} + +#[cfg(unix)] +fn replace_lifecycle_state(temp: &Path, target: &Path) -> Result<()> { + replace_lifecycle_state_with_directory_sync(temp, target, sync_lifecycle_state_directory) +} + +#[cfg(unix)] +fn replace_lifecycle_state_with_directory_sync( + temp: &Path, + target: &Path, + sync_directory: impl FnOnce(&Path) -> Result<()>, +) -> Result<()> { + std::fs::rename(temp, target) + .map_err(|error| PluginHostConfigError::io("replace lifecycle state", target, error))?; + let parent = target + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + sync_directory(parent) +} + +#[cfg(unix)] +fn sync_lifecycle_state_directory(directory: &Path) -> Result<()> { + let file = File::open(directory).map_err(|error| { + PluginHostConfigError::io("open lifecycle state directory", directory, error) + })?; + file.sync_all().map_err(|error| { + PluginHostConfigError::io("sync lifecycle state directory", directory, error) + }) +} + +#[cfg(all(not(unix), not(windows)))] +fn replace_lifecycle_state(temp: &Path, target: &Path) -> Result<()> { + std::fs::rename(temp, target) + .map_err(|error| PluginHostConfigError::io("replace lifecycle state", target, error)) +} + +#[cfg(windows)] +fn replace_lifecycle_state(temp: &Path, target: &Path) -> Result<()> { + use windows_sys::Win32::Storage::FileSystem::{ + MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, MoveFileExW, + }; + + let temp_wide = windows_wide(temp.as_os_str()); + let target_wide = windows_wide(target.as_os_str()); + // SAFETY: Both paths are NUL-terminated and remain valid for this same-directory replace. + if unsafe { + MoveFileExW( + temp_wide.as_ptr(), + target_wide.as_ptr(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, + ) + } == 0 + { + return Err(PluginHostConfigError::io( + "replace lifecycle state", + target, + std::io::Error::last_os_error(), + )); + } + Ok(()) +} + +#[cfg(windows)] +fn windows_wide(value: impl AsRef) -> Vec { + use std::os::windows::ffi::OsStrExt; + + value.as_ref().encode_wide().chain(Some(0)).collect() +} + +#[cfg(test)] +#[path = "../tests/unit/state.rs"] +mod tests; diff --git a/crates/plugin-host-config/src/trust.rs b/crates/plugin-host-config/src/trust.rs new file mode 100644 index 000000000..2063c05fe --- /dev/null +++ b/crates/plugin-host-config/src/trust.rs @@ -0,0 +1,416 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::fmt; +use std::path::{Path, PathBuf}; + +use base64::Engine; +use nemo_relay::plugin::dynamic::{ + DynamicPluginAttestationMode, DynamicPluginCheckState, DynamicPluginFailure, + DynamicPluginFailurePhase, DynamicPluginManifest, +}; +use ring::signature::{ED25519, UnparsedPublicKey}; +use sha2::{Digest, Sha256}; + +use crate::io::{read_bounded_regular_file, stream_bounded_regular_file}; +use crate::policy::EvaluatedDynamicPluginHostPolicy; + +type TrustResult = Result; + +#[doc(hidden)] +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub enum DynamicPluginTrustFailure { + MissingArtifact, + MissingIntegrityDigest, + ArtifactRead { + path: PathBuf, + error: String, + }, + IntegrityMismatch { + path: PathBuf, + expected: String, + actual: String, + }, + MissingSignature, + MissingTrustedKeys, + SignatureRead { + path: PathBuf, + error: String, + }, + InvalidTrustedKey { + key: String, + error: String, + }, + SignatureVerification { + path: PathBuf, + parse_errors: Vec, + }, +} + +#[allow(missing_docs)] +impl DynamicPluginTrustFailure { + pub fn display<'a>(&'a self, plugin_id: &'a str) -> DynamicPluginTrustFailureDisplay<'a> { + DynamicPluginTrustFailureDisplay { + failure: self, + plugin_id, + } + } + + pub fn refusal_code(&self) -> &'static str { + match self { + Self::MissingArtifact + | Self::MissingIntegrityDigest + | Self::ArtifactRead { .. } + | Self::IntegrityMismatch { .. } => "integrity_failed", + Self::MissingSignature + | Self::MissingTrustedKeys + | Self::SignatureRead { .. } + | Self::InvalidTrustedKey { .. } + | Self::SignatureVerification { .. } => "attestation_failed", + } + } +} + +#[doc(hidden)] +pub struct DynamicPluginTrustFailureDisplay<'a> { + failure: &'a DynamicPluginTrustFailure, + plugin_id: &'a str, +} + +impl fmt::Display for DynamicPluginTrustFailureDisplay<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.failure { + DynamicPluginTrustFailure::MissingArtifact => write!( + f, + "dynamic plugin '{}' is missing source.artifact required for integrity verification", + self.plugin_id + ), + DynamicPluginTrustFailure::MissingIntegrityDigest => write!( + f, + "dynamic plugin '{}' is missing integrity.sha256 required for host trust verification", + self.plugin_id + ), + DynamicPluginTrustFailure::ArtifactRead { path, error } => write!( + f, + "dynamic plugin '{}' artifact {} could not be read for trust verification: {}", + self.plugin_id, + path.display(), + error + ), + DynamicPluginTrustFailure::IntegrityMismatch { + path, + expected: _, + actual: _, + } => write!( + f, + "dynamic plugin '{}' failed integrity verification for {}: the artifact digest does not match the manifest", + self.plugin_id, + path.display() + ), + DynamicPluginTrustFailure::MissingSignature => write!( + f, + "dynamic plugin '{}' requires integrity.signature under host policy", + self.plugin_id + ), + DynamicPluginTrustFailure::MissingTrustedKeys => write!( + f, + "dynamic plugin '{}' requires signature verification, but no trusted_public_keys are configured in host policy", + self.plugin_id + ), + DynamicPluginTrustFailure::SignatureRead { path, error } => write!( + f, + "dynamic plugin '{}' signature {} could not be read: {}", + self.plugin_id, + path.display(), + error + ), + DynamicPluginTrustFailure::InvalidTrustedKey { key: _, error } => write!( + f, + "dynamic plugin '{}' has an invalid trusted public key: {}", + self.plugin_id, error + ), + DynamicPluginTrustFailure::SignatureVerification { path, parse_errors } => { + write!( + f, + "dynamic plugin '{}' failed signature verification for {} against configured host policy keys", + self.plugin_id, + path.display() + )?; + if !parse_errors.is_empty() { + write!(f, "; key parse errors: {}", parse_errors.join("; "))?; + } + Ok(()) + } + } + } +} + +#[doc(hidden)] +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct EvaluatedDynamicPluginTrust { + pub integrity: DynamicPluginCheckState, + pub authenticity: DynamicPluginCheckState, + pub failure: Option, +} + +#[allow(missing_docs)] +impl EvaluatedDynamicPluginTrust { + fn valid(authenticity: DynamicPluginCheckState) -> Self { + Self { + integrity: DynamicPluginCheckState::Valid, + authenticity, + failure: None, + } + } + + fn failed( + integrity: DynamicPluginCheckState, + authenticity: DynamicPluginCheckState, + failure: DynamicPluginTrustFailure, + ) -> Self { + Self { + integrity, + authenticity, + failure: Some(failure), + } + } + + pub fn failure(&self) -> Option<&DynamicPluginTrustFailure> { + self.failure.as_ref() + } + + pub fn refusal_code(&self) -> Option<&'static str> { + self.failure + .as_ref() + .map(DynamicPluginTrustFailure::refusal_code) + } + + pub fn is_satisfied(&self) -> bool { + self.failure.is_none() + } + + pub fn last_error(&self, plugin_id: &str) -> Option { + self.failure.as_ref().map(|failure| DynamicPluginFailure { + phase: DynamicPluginFailurePhase::Validation, + code: failure.refusal_code().into(), + message: failure.display(plugin_id).to_string(), + }) + } +} + +#[doc(hidden)] +pub fn evaluate_dynamic_plugin_trust( + manifest: &DynamicPluginManifest, + manifest_ref: &str, + policy: &EvaluatedDynamicPluginHostPolicy, +) -> EvaluatedDynamicPluginTrust { + if !policy.policy_satisfied { + return EvaluatedDynamicPluginTrust { + integrity: DynamicPluginCheckState::Unknown, + authenticity: DynamicPluginCheckState::Unknown, + failure: None, + }; + } + let artifact_path = match verify_integrity(manifest, manifest_ref) { + Ok(path) => path, + Err(failure) => { + return EvaluatedDynamicPluginTrust::failed( + DynamicPluginCheckState::Invalid, + DynamicPluginCheckState::Unknown, + failure, + ); + } + }; + match evaluate_authenticity(manifest, manifest_ref, &artifact_path, policy) { + Ok(authenticity) => EvaluatedDynamicPluginTrust::valid(authenticity), + Err(failure) => EvaluatedDynamicPluginTrust::failed( + DynamicPluginCheckState::Valid, + DynamicPluginCheckState::Invalid, + failure, + ), + } +} + +fn verify_integrity(manifest: &DynamicPluginManifest, manifest_ref: &str) -> TrustResult { + let artifact = manifest + .source + .as_ref() + .and_then(|source| source.artifact.as_deref()) + .ok_or(DynamicPluginTrustFailure::MissingArtifact)?; + let expected_digest = manifest + .integrity + .as_ref() + .and_then(|integrity| integrity.sha256.as_deref()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or(DynamicPluginTrustFailure::MissingIntegrityDigest)?; + let artifact_path = resolve_artifact_path(manifest_ref, artifact); + let actual_digest = + file_sha256(&artifact_path).map_err(|error| DynamicPluginTrustFailure::ArtifactRead { + path: artifact_path.clone(), + error, + })?; + if actual_digest != expected_digest { + return Err(DynamicPluginTrustFailure::IntegrityMismatch { + path: artifact_path, + expected: expected_digest.to_owned(), + actual: actual_digest, + }); + } + Ok(artifact_path) +} + +fn evaluate_authenticity( + manifest: &DynamicPluginManifest, + manifest_ref: &str, + artifact_path: &Path, + policy: &EvaluatedDynamicPluginHostPolicy, +) -> TrustResult { + let signature_ref = manifest + .integrity + .as_ref() + .and_then(|integrity| integrity.signature.as_deref()) + .map(str::trim) + .filter(|value| !value.is_empty()); + match policy.attestation_mode { + DynamicPluginAttestationMode::IntegrityOnly => Ok(DynamicPluginCheckState::Unknown), + DynamicPluginAttestationMode::SignatureIfPresent => match signature_ref { + Some(signature_ref) => { + verify_signature( + manifest_ref, + artifact_path, + signature_ref, + &policy.trusted_public_keys, + )?; + Ok(DynamicPluginCheckState::Valid) + } + None => Ok(DynamicPluginCheckState::Unknown), + }, + DynamicPluginAttestationMode::SignatureRequired => match signature_ref { + Some(signature_ref) => { + verify_signature( + manifest_ref, + artifact_path, + signature_ref, + &policy.trusted_public_keys, + )?; + Ok(DynamicPluginCheckState::Valid) + } + None => Err(DynamicPluginTrustFailure::MissingSignature), + }, + } +} + +fn verify_signature( + manifest_ref: &str, + artifact_path: &Path, + signature_ref: &str, + trusted_public_keys: &[String], +) -> TrustResult<()> { + if trusted_public_keys.is_empty() { + return Err(DynamicPluginTrustFailure::MissingTrustedKeys); + } + let signature_path = resolve_artifact_path(manifest_ref, signature_ref); + let signature_bytes = read_signature_bytes(&signature_path)?; + let artifact_bytes = read_bounded_regular_file(artifact_path, "dynamic plugin artifact") + .map_err(|error| DynamicPluginTrustFailure::ArtifactRead { + path: artifact_path.to_path_buf(), + error: error.to_string(), + })?; + let mut parse_errors = Vec::new(); + for trusted_public_key in trusted_public_keys { + let public_key_bytes = match parse_ed25519_public_key(trusted_public_key) { + Ok(bytes) => bytes, + Err(DynamicPluginTrustFailure::InvalidTrustedKey { error, .. }) => { + parse_errors.push(error); + continue; + } + Err(other) => return Err(other), + }; + if UnparsedPublicKey::new(&ED25519, public_key_bytes) + .verify(&artifact_bytes, &signature_bytes) + .is_ok() + { + return Ok(()); + } + } + Err(DynamicPluginTrustFailure::SignatureVerification { + path: signature_path, + parse_errors, + }) +} + +fn read_signature_bytes(path: &Path) -> TrustResult> { + let raw = read_bounded_regular_file(path, "dynamic plugin signature").map_err(|error| { + DynamicPluginTrustFailure::SignatureRead { + path: path.to_path_buf(), + error: error.to_string(), + } + })?; + let trimmed = String::from_utf8_lossy(&raw).trim().to_owned(); + if trimmed.is_empty() { + return Err(DynamicPluginTrustFailure::SignatureRead { + path: path.to_path_buf(), + error: "signature file is empty".into(), + }); + } + let encoded = trimmed + .strip_prefix("ed25519:") + .unwrap_or(trimmed.as_str()) + .trim(); + base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|error| DynamicPluginTrustFailure::SignatureRead { + path: path.to_path_buf(), + error: format!("invalid base64 signature: {error}"), + }) +} + +fn parse_ed25519_public_key(value: &str) -> TrustResult> { + let encoded = value.trim().strip_prefix("ed25519:").ok_or_else(|| { + DynamicPluginTrustFailure::InvalidTrustedKey { + key: value.to_owned(), + error: "unsupported trusted public key format".into(), + } + })?; + base64::engine::general_purpose::STANDARD + .decode(encoded.trim()) + .map_err(|error| DynamicPluginTrustFailure::InvalidTrustedKey { + key: value.to_owned(), + error: format!("invalid ed25519 trusted public key encoding: {error}"), + }) +} + +fn resolve_artifact_path(manifest_ref: &str, artifact_ref: &str) -> PathBuf { + let artifact_path = PathBuf::from(artifact_ref); + if artifact_path.is_absolute() { + artifact_path + } else { + Path::new(manifest_ref) + .parent() + .map(|parent| parent.join(&artifact_path)) + .unwrap_or(artifact_path) + } +} + +fn file_sha256(path: &Path) -> std::result::Result { + let mut digest = Sha256::new(); + stream_bounded_regular_file(path, "dynamic plugin artifact", |bytes| { + digest.update(bytes); + }) + .map_err(|error| error.to_string())?; + Ok(format!( + "sha256:{}", + digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + )) +} + +#[cfg(test)] +#[path = "../tests/unit/trust.rs"] +mod tests; diff --git a/crates/plugin-host-config/tests/activation_safety.rs b/crates/plugin-host-config/tests/activation_safety.rs new file mode 100644 index 000000000..9ecdd2060 --- /dev/null +++ b/crates/plugin-host-config/tests/activation_safety.rs @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Regression coverage for file-backed native activation and lifecycle validation state. + +use std::fs; +use std::path::{Path, PathBuf}; + +use nemo_relay::plugin::dynamic::DynamicPluginKind; +use nemo_relay_plugin_host_config::{ + DynamicPluginActivationSnapshot, DynamicPluginHostPolicy, reconcile_plugin_lifecycle, + resolve_plugin_files_from_paths, +}; +use sha2::{Digest, Sha256}; +use tempfile::tempdir; + +const LIFECYCLE_STATE_FILE: &str = ".dynamic-plugins.json"; + +fn sha256_hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn write_native_manifest(root: &Path, plugin_id: &str, artifact: &str, library: &str) -> PathBuf { + fs::create_dir_all(root).unwrap(); + let trusted_bytes = b"integrity-checked native artifact"; + fs::write(root.join(artifact), trusted_bytes).unwrap(); + if library != artifact { + fs::write(root.join(library), b"different native library").unwrap(); + } + let manifest = root.join("relay-plugin.toml"); + fs::write( + &manifest, + format!( + r#"manifest_version = 1 +[plugin] +id = "{plugin_id}" +kind = "rust_dynamic" +[compat] +relay = ">=0.5,<1.0" +native_api = "v1" +[capabilities] +items = ["plugin_native"] +[defaults] +enabled = false +[load] +library = "{library}" +symbol = "nemo_relay_plugin_entrypoint_v1" +[source] +artifact = "{artifact}" +[integrity] +sha256 = "sha256:{}" +"#, + sha256_hex(trusted_bytes) + ), + ) + .unwrap(); + manifest +} + +#[test] +fn native_snapshot_rejects_a_library_other_than_the_integrity_checked_artifact() { + let temp = tempdir().unwrap(); + let manifest = write_native_manifest( + temp.path(), + "native-artifact-mismatch", + "trusted.so", + "loaded.so", + ); + + let error = DynamicPluginActivationSnapshot::create( + manifest.to_string_lossy().as_ref(), + "native-artifact-mismatch", + DynamicPluginKind::RustDynamic, + None, + &DynamicPluginHostPolicy::default(), + ) + .unwrap_err(); + + assert!( + error + .to_string() + .contains("must declare its load.library as the integrity-checked source.artifact") + ); +} + +#[test] +fn native_snapshot_accepts_the_integrity_checked_library() { + let temp = tempdir().unwrap(); + let manifest = write_native_manifest( + temp.path(), + "native-artifact-match", + "plugin.so", + "plugin.so", + ); + + let snapshot = DynamicPluginActivationSnapshot::create( + manifest.to_string_lossy().as_ref(), + "native-artifact-match", + DynamicPluginKind::RustDynamic, + None, + &DynamicPluginHostPolicy::default(), + ) + .unwrap(); + + snapshot.verify_current().unwrap(); +} + +#[test] +fn newly_hydrated_validation_is_timestamped_without_bumping_generation() { + let temp = tempdir().unwrap(); + let manifest = write_native_manifest( + &temp.path().join("plugin"), + "hydrated-validation", + "plugin.so", + "plugin.so", + ); + let config = temp.path().join("plugins.toml"); + fs::write( + &config, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + manifest.to_string_lossy() + ), + ) + .unwrap(); + let resolved = resolve_plugin_files_from_paths([config], None).unwrap(); + + reconcile_plugin_lifecycle(&resolved).unwrap(); + let state_path = temp.path().join(LIFECYCLE_STATE_FILE); + let first: serde_json::Value = + serde_json::from_str(&fs::read_to_string(&state_path).unwrap()).unwrap(); + let first_record = &first["records"][0]; + let first_generation = first_record["metadata"]["generation"].as_u64().unwrap(); + assert!( + first_record["status"]["validation"]["checked_at"] + .as_str() + .is_some_and(|timestamp| !timestamp.is_empty()) + ); + + reconcile_plugin_lifecycle(&resolved).unwrap(); + let second: serde_json::Value = + serde_json::from_str(&fs::read_to_string(state_path).unwrap()).unwrap(); + let second_record = &second["records"][0]; + assert_eq!( + second_record["metadata"]["generation"].as_u64(), + Some(first_generation) + ); + assert!( + second_record["status"]["validation"]["checked_at"] + .as_str() + .is_some_and(|timestamp| !timestamp.is_empty()) + ); +} diff --git a/crates/plugin-host-config/tests/optional_startup.rs b/crates/plugin-host-config/tests/optional_startup.rs new file mode 100644 index 000000000..c8468d232 --- /dev/null +++ b/crates/plugin-host-config/tests/optional_startup.rs @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Optional dynamic-plugin preflight failures remain activation attempts. + +use std::fs; +use std::path::{Path, PathBuf}; + +use nemo_relay_plugin_host_config::{ + prepare_plugin_host_activation, reconcile_plugin_lifecycle, resolve_plugin_files_from_paths, +}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use tempfile::tempdir; + +fn write_command_worker(root: &Path, plugin_id: &str) -> PathBuf { + fs::create_dir_all(root).unwrap(); + let artifact = b"#!/bin/sh\nexit 0\n"; + fs::write(root.join("worker.sh"), artifact).unwrap(); + let digest = Sha256::digest(artifact) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let manifest = root.join("relay-plugin.toml"); + fs::write( + &manifest, + format!( + r#"manifest_version = 1 +[plugin] +id = "{plugin_id}" +kind = "worker" +[compat] +relay = ">=0.5,<1.0" +worker_protocol = "grpc-v1" +[capabilities] +items = ["plugin_worker"] +[defaults] +enabled = false +[source] +manifest_root = "." +artifact = "worker.sh" +[integrity] +sha256 = "sha256:{digest}" +[load] +runtime = "command" +entrypoint = "worker.sh" +"# + ), + ) + .unwrap(); + manifest +} + +fn enable_record(state_path: &Path) { + let mut state: Value = serde_json::from_slice(&fs::read(state_path).unwrap()).unwrap(); + state["records"][0]["spec"]["enabled"] = true.into(); + fs::write(state_path, serde_json::to_vec_pretty(&state).unwrap()).unwrap(); +} + +fn optional_plan(policy: &str, plugin_id: &str) -> (usize, Value) { + let temp = tempdir().unwrap(); + let manifest = write_command_worker(&temp.path().join("plugin"), plugin_id); + let config = temp.path().join("plugins.toml"); + fs::write( + &config, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n\n{policy}\n", + manifest.to_string_lossy() + ), + ) + .unwrap(); + let resolved = resolve_plugin_files_from_paths([config], None).unwrap(); + reconcile_plugin_lifecycle(&resolved).unwrap(); + let state_path = temp.path().join(".dynamic-plugins.json"); + enable_record(&state_path); + + let plan = prepare_plugin_host_activation(resolved).unwrap(); + let persisted = serde_json::from_slice(&fs::read(state_path).unwrap()).unwrap(); + (plan.dynamic_plugins.len(), persisted) +} + +#[test] +fn optional_policy_failure_is_recorded_but_still_planned() { + let (planned, state) = optional_plan( + r#"[plugins.policy.defaults] +startup = "optional" +allowed = false"#, + "optional-policy", + ); + + assert_eq!(planned, 1); + assert_eq!( + state["records"][0]["status"]["validation"]["policy_satisfied"], + "invalid" + ); + assert_eq!(state["records"][0]["status"]["startup_class"], "optional"); +} + +#[test] +fn optional_trust_failure_is_recorded_but_still_planned() { + let (planned, state) = optional_plan( + r#"[plugins.policy.defaults] +startup = "optional" +attestation = "signature_required""#, + "optional-trust", + ); + + assert_eq!(planned, 1); + assert_eq!( + state["records"][0]["status"]["validation"]["authenticity"], + "invalid" + ); + assert_eq!(state["records"][0]["status"]["startup_class"], "optional"); +} diff --git a/crates/plugin-host-config/tests/python_environment_validation.rs b/crates/plugin-host-config/tests/python_environment_validation.rs new file mode 100644 index 000000000..584fe859b --- /dev/null +++ b/crates/plugin-host-config/tests/python_environment_validation.rs @@ -0,0 +1,301 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Regression coverage for lifecycle-managed Python environment validation. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use nemo_relay_plugin_host_config::{ + ENVIRONMENT_ATTESTATION_FILE, prepare_plugin_host_activation, reconcile_plugin_lifecycle, + resolve_plugin_files_from_paths, verify_environment_attestation, +}; +use ring::hmac; +use serde_json::json; +use sha2::{Digest, Sha256}; +use tempfile::tempdir; + +const LIFECYCLE_STATE_FILE: &str = ".dynamic-plugins.json"; +const ATTESTATION_DOMAIN: &[u8] = b"nemo-relay/python-environment-attestation/v1\0"; +static ENVIRONMENT_LOCK: Mutex<()> = Mutex::new(()); + +fn sha256_hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn write_python_manifest(root: &Path, plugin_id: &str) -> (PathBuf, String) { + fs::create_dir_all(root).unwrap(); + let artifact = b"def main():\n return None\n"; + fs::write(root.join("plugin.py"), artifact).unwrap(); + let source_digest = format!("sha256:{}", sha256_hex(artifact)); + let manifest = root.join("relay-plugin.toml"); + fs::write( + &manifest, + format!( + r#"manifest_version = 1 +[plugin] +id = "{plugin_id}" +kind = "worker" +[compat] +relay = ">=0.5,<1.0" +worker_protocol = "grpc-v1" +[capabilities] +items = ["plugin_worker"] +[defaults] +enabled = false +[source] +manifest_root = "." +artifact = "plugin.py" +[integrity] +sha256 = "{source_digest}" +[load] +runtime = "python" +entrypoint = "plugin:main" +"# + ), + ) + .unwrap(); + (manifest, source_digest) +} + +fn update_tree_digest(digest: &mut Sha256, entry_type: u8, path: &Path, payload: &[u8]) { + let path = raw_path_bytes(path); + digest.update([entry_type]); + digest.update((path.len() as u64).to_le_bytes()); + digest.update(path); + digest.update((payload.len() as u64).to_le_bytes()); + digest.update(payload); +} + +#[cfg(unix)] +fn raw_path_bytes(path: &Path) -> Vec { + use std::os::unix::ffi::OsStrExt; + path.as_os_str().as_bytes().to_vec() +} + +#[cfg(windows)] +fn raw_path_bytes(path: &Path) -> Vec { + use std::os::windows::ffi::OsStrExt; + path.as_os_str() + .encode_wide() + .flat_map(u16::to_le_bytes) + .collect() +} + +fn write_attested_environment(environment: &Path, source_digest: &str, key_bytes: &[u8; 32]) { + let (interpreter_dir, interpreter_name) = if cfg!(windows) { + ("Scripts", "python.exe") + } else { + ("bin", "python") + }; + let interpreter_bytes = b"managed Python interpreter fixture"; + fs::create_dir_all(environment.join(interpreter_dir)).unwrap(); + fs::write( + environment.join(interpreter_dir).join(interpreter_name), + interpreter_bytes, + ) + .unwrap(); + + let mut digest = Sha256::new(); + update_tree_digest(&mut digest, b'd', Path::new(interpreter_dir), &[]); + update_tree_digest( + &mut digest, + b'f', + &Path::new(interpreter_dir).join(interpreter_name), + interpreter_bytes, + ); + let environment_digest = digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + + let mut message = Vec::new(); + message.extend_from_slice(ATTESTATION_DOMAIN); + message.extend_from_slice(source_digest.trim().as_bytes()); + message.push(0); + message.extend_from_slice(environment_digest.as_bytes()); + let key = hmac::Key::new(hmac::HMAC_SHA256, key_bytes); + let authentication = format!( + "hmac-sha256:{}", + hmac::sign(&key, &message) + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + ); + fs::write( + environment.join(ENVIRONMENT_ATTESTATION_FILE), + serde_json::to_vec_pretty(&json!({ + "version": 1, + "source_artifact_sha256": source_digest, + "environment_sha256": environment_digest, + "authentication": authentication, + })) + .unwrap(), + ) + .unwrap(); +} + +fn lifecycle_environment(root: &Path, plugin_id: &str) -> PathBuf { + root.join(".dynamic-plugin-environments") + .join(sha256_hex(plugin_id.as_bytes())) +} + +fn enable_with_environment(state_path: &Path, environment: &Path) { + let mut state: serde_json::Value = + serde_json::from_str(&fs::read_to_string(state_path).unwrap()).unwrap(); + state["records"][0]["spec"]["enabled"] = true.into(); + state["records"][0]["source"]["environment_ref"] = + environment.to_string_lossy().as_ref().into(); + fs::write(state_path, serde_json::to_vec_pretty(&state).unwrap()).unwrap(); +} + +#[test] +fn enabled_python_worker_rejects_an_attested_environment_outside_its_lifecycle_path() { + let _environment = ENVIRONMENT_LOCK.lock().unwrap(); + let temp = tempdir().unwrap(); + let xdg_config_home = temp.path().join("xdg"); + // SAFETY: environment-sensitive tests in this binary hold `ENVIRONMENT_LOCK`. + unsafe { std::env::set_var("XDG_CONFIG_HOME", &xdg_config_home) }; + let key_bytes = [0x5a; 32]; + let key_directory = xdg_config_home.join("nemo-relay").join("bootstrap"); + fs::create_dir_all(&key_directory).unwrap(); + fs::write(key_directory.join("fingerprint-hmac.key"), key_bytes).unwrap(); + + let (manifest, source_digest) = + write_python_manifest(&temp.path().join("plugin"), "wrong-environment-location"); + let config = temp.path().join("plugins.toml"); + fs::write( + &config, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + manifest.to_string_lossy() + ), + ) + .unwrap(); + let resolved = resolve_plugin_files_from_paths([config.clone()], None).unwrap(); + reconcile_plugin_lifecycle(&resolved).unwrap(); + + let wrong_environment = temp.path().join("attested-but-not-lifecycle-managed"); + write_attested_environment(&wrong_environment, &source_digest, &key_bytes); + verify_environment_attestation(&wrong_environment, &source_digest).unwrap(); + + let state_path = temp.path().join(LIFECYCLE_STATE_FILE); + let mut state: serde_json::Value = + serde_json::from_str(&fs::read_to_string(&state_path).unwrap()).unwrap(); + state["records"][0]["spec"]["enabled"] = true.into(); + state["records"][0]["source"]["environment_ref"] = + wrong_environment.to_string_lossy().as_ref().into(); + fs::write(&state_path, serde_json::to_vec_pretty(&state).unwrap()).unwrap(); + + let error = match prepare_plugin_host_activation(resolved) { + Ok(_) => { + panic!("wrong-location Python environment unexpectedly reached an activation plan") + } + Err(error) => error, + }; + assert!( + error + .to_string() + .contains("invalid lifecycle-managed environment") + ); + let persisted: serde_json::Value = + serde_json::from_str(&fs::read_to_string(state_path).unwrap()).unwrap(); + assert_eq!( + persisted["records"][0]["status"]["validation"]["environment"], + "invalid" + ); +} + +#[test] +fn enabled_python_worker_reports_a_missing_lifecycle_environment_contextually() { + let temp = tempdir().unwrap(); + let (manifest, _) = write_python_manifest(&temp.path().join("plugin"), "missing-environment"); + let config = temp.path().join("plugins.toml"); + fs::write( + &config, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + manifest.to_string_lossy() + ), + ) + .unwrap(); + let resolved = resolve_plugin_files_from_paths([config], None).unwrap(); + reconcile_plugin_lifecycle(&resolved).unwrap(); + + let state_path = temp.path().join(LIFECYCLE_STATE_FILE); + let environment = lifecycle_environment(temp.path(), "missing-environment"); + enable_with_environment(&state_path, &environment); + + let error = match prepare_plugin_host_activation(resolved) { + Ok(_) => panic!("missing Python environment unexpectedly reached an activation plan"), + Err(error) => error.to_string(), + }; + assert!( + error.contains("failed to inspect lifecycle-managed Python environment"), + "{error}" + ); + assert!( + error.contains( + dunce::canonicalize(temp.path()) + .unwrap() + .join(environment.strip_prefix(temp.path()).unwrap()) + .to_string_lossy() + .as_ref() + ), + "{error}" + ); +} + +#[cfg(unix)] +#[test] +fn enabled_python_worker_rejects_a_symlinked_lifecycle_environment_slot() { + use std::os::unix::fs::symlink; + + let _environment = ENVIRONMENT_LOCK.lock().unwrap(); + let temp = tempdir().unwrap(); + let xdg_config_home = temp.path().join("xdg"); + // SAFETY: environment-sensitive tests in this binary hold `ENVIRONMENT_LOCK`. + unsafe { std::env::set_var("XDG_CONFIG_HOME", &xdg_config_home) }; + let key_bytes = [0x5b; 32]; + let key_directory = xdg_config_home.join("nemo-relay").join("bootstrap"); + fs::create_dir_all(&key_directory).unwrap(); + fs::write(key_directory.join("fingerprint-hmac.key"), key_bytes).unwrap(); + + let plugin_id = "symlinked-environment"; + let (manifest, source_digest) = write_python_manifest(&temp.path().join("plugin"), plugin_id); + let config = temp.path().join("plugins.toml"); + fs::write( + &config, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + manifest.to_string_lossy() + ), + ) + .unwrap(); + let resolved = resolve_plugin_files_from_paths([config], None).unwrap(); + reconcile_plugin_lifecycle(&resolved).unwrap(); + + let external = temp.path().join("external-environment"); + write_attested_environment(&external, &source_digest, &key_bytes); + let managed = lifecycle_environment(temp.path(), plugin_id); + fs::create_dir_all(managed.parent().unwrap()).unwrap(); + symlink(&external, &managed).unwrap(); + let state_path = temp.path().join(LIFECYCLE_STATE_FILE); + enable_with_environment(&state_path, &external); + + let error = match prepare_plugin_host_activation(resolved) { + Ok(_) => panic!("symlinked Python environment unexpectedly reached an activation plan"), + Err(error) => error.to_string(), + }; + assert!( + error.contains("lifecycle-managed Python environment") + && error.contains("not a symbolic link"), + "{error}" + ); +} diff --git a/crates/plugin-host-config/tests/unit/activation.rs b/crates/plugin-host-config/tests/unit/activation.rs new file mode 100644 index 000000000..c3c45db82 --- /dev/null +++ b/crates/plugin-host-config/tests/unit/activation.rs @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::{Mutex, OnceLock}; + +use nemo_relay::plugin::PluginConfig; +use nemo_relay::plugin::dynamic::PluginHostActivationPlan; +use tempfile::tempdir; + +use super::*; + +fn activation_lock() -> std::sync::MutexGuard<'static, ()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|error| error.into_inner()) +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} + +fn resolve_options(root: &std::path::Path) -> PluginFileResolveOptions { + PluginFileResolveOptions { + plugin_config_path: Some(root.join("selected/plugins.toml")), + current_dir: None, + user_config_dir: None, + system_config_path: root.join("system/plugins.toml"), + } +} + +#[test] +fn inactive_activation_has_an_empty_report_and_clears_as_a_no_op() { + let activation = PluginFileActivation::inactive(); + + assert!(!activation.is_active()); + assert!(activation.report().diagnostics.is_empty()); + assert!(activation.report().runtime_diagnostics.is_empty()); + activation.clear().unwrap(); +} + +#[test] +fn file_initialization_distinguishes_no_input_from_explicit_empty_configuration() { + let _lock = activation_lock(); + let temp = tempdir().unwrap(); + let runtime = runtime(); + + let inactive = runtime + .block_on(initialize_from_plugins_toml_with_options( + None, + resolve_options(temp.path()), + )) + .unwrap(); + assert!(!inactive.is_active()); + inactive.clear().unwrap(); + + let active = runtime + .block_on(initialize_from_plugins_toml_with_options( + Some(PluginConfig::default()), + resolve_options(temp.path()), + )) + .unwrap(); + assert!(active.is_active()); + assert!(active.report().diagnostics.is_empty()); + active.clear().unwrap(); +} + +#[test] +fn resolved_plan_activation_owns_and_releases_the_process_lease() { + let _lock = activation_lock(); + let runtime = runtime(); + let plan = || PluginHostActivationPlan { + config: PluginConfig::default(), + dynamic_plugins: Vec::new(), + diagnostics: Vec::new(), + }; + + let activation = runtime + .block_on(PluginFileActivation::activate_plan(plan())) + .unwrap(); + assert!(activation.is_active()); + + let conflict = match runtime.block_on(PluginFileActivation::activate_plan(plan())) { + Ok(_) => panic!("a second owned activation unexpectedly acquired the process lease"), + Err(error) => error, + }; + assert!(matches!( + conflict, + PluginHostConfigError::Relay(PluginError::Conflict(_)) + )); + + activation.clear().unwrap(); + let replacement = runtime + .block_on(PluginFileActivation::activate_plan(plan())) + .unwrap(); + replacement.clear().unwrap(); +} diff --git a/crates/plugin-host-config/tests/unit/environment.rs b/crates/plugin-host-config/tests/unit/environment.rs new file mode 100644 index 000000000..60ff188d9 --- /dev/null +++ b/crates/plugin-host-config/tests/unit/environment.rs @@ -0,0 +1,342 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::fs; + +use nemo_relay::plugin::dynamic::{ + DynamicPluginCheckState, DynamicPluginManifest, DynamicPluginManifestLoad, WorkerRuntime, +}; +use sha2::Sha256; +use tempfile::tempdir; + +use super::*; + +fn python_manifest(root: &Path, id: &str) -> (DynamicPluginManifest, PathBuf) { + fs::create_dir_all(root).unwrap(); + fs::write(root.join("plugin.py"), b"def main():\n return None\n").unwrap(); + let manifest_path = root.join("relay-plugin.toml"); + let manifest = DynamicPluginManifest::parse_toml(&format!( + r#"manifest_version = 1 +[plugin] +id = "{id}" +kind = "worker" +[compat] +relay = ">=0.5,<1.0" +worker_protocol = "grpc-v1" +[capabilities] +items = ["plugin_worker"] +[defaults] +enabled = false +[source] +manifest_root = "." +artifact = "plugin.py" +[integrity] +sha256 = "sha256:placeholder" +[load] +runtime = "python" +entrypoint = "plugin:main" +"#, + )) + .unwrap(); + (manifest, manifest_path) +} + +#[test] +fn python_entrypoint_contract_accepts_exact_artifacts_and_rejects_ambiguous_execution() { + let temp = tempdir().unwrap(); + let (mut manifest, manifest_path) = python_manifest(temp.path(), "entrypoint"); + validate_python_entrypoint_artifact(&manifest, manifest_path.to_string_lossy().as_ref()) + .unwrap(); + + if let DynamicPluginManifestLoad::Worker(load) = &mut manifest.load { + load.runtime = Some(WorkerRuntime::Command); + } + validate_python_entrypoint_artifact(&manifest, manifest_path.to_string_lossy().as_ref()) + .unwrap(); + + if let DynamicPluginManifestLoad::Worker(load) = &mut manifest.load { + load.runtime = Some(WorkerRuntime::Python); + load.entrypoint = Some("plugin".into()); + } + let error = + validate_python_entrypoint_artifact(&manifest, manifest_path.to_string_lossy().as_ref()) + .unwrap_err(); + assert!(error.contains("module:function")); + + if let DynamicPluginManifestLoad::Worker(load) = &mut manifest.load { + load.entrypoint = Some("plugin:main:extra".into()); + } + assert!( + validate_python_entrypoint_artifact(&manifest, manifest_path.to_string_lossy().as_ref()) + .unwrap_err() + .contains("module:function") + ); + + if let DynamicPluginManifestLoad::Worker(load) = &mut manifest.load { + load.entrypoint = Some("plugin:main".into()); + } + fs::create_dir(temp.path().join("plugin")).unwrap(); + fs::write( + temp.path().join("plugin/__init__.py"), + b"def main(): pass\n", + ) + .unwrap(); + assert!( + validate_python_entrypoint_artifact(&manifest, manifest_path.to_string_lossy().as_ref()) + .unwrap_err() + .contains("exactly one source module") + ); + + fs::remove_dir_all(temp.path().join("plugin")).unwrap(); + fs::write(temp.path().join("other.py"), b"def main(): pass\n").unwrap(); + manifest.source.as_mut().unwrap().artifact = Some("other.py".into()); + assert!( + validate_python_entrypoint_artifact(&manifest, manifest_path.to_string_lossy().as_ref()) + .unwrap_err() + .contains("executed entrypoint module must be the integrity-checked artifact") + ); + + manifest.source = None; + assert!( + validate_python_entrypoint_artifact(&manifest, manifest_path.to_string_lossy().as_ref()) + .unwrap_err() + .contains("source.manifest_root and source.artifact") + ); +} + +#[test] +fn environment_state_reports_invalid_layouts_without_provisioning() { + let temp = tempdir().unwrap(); + let (mut manifest, _) = python_manifest(&temp.path().join("plugin"), "managed-layout"); + let state_path = temp.path().join(".dynamic-plugins.json"); + + assert_eq!( + environment_state(&manifest, &state_path, None), + DynamicPluginCheckState::Invalid + ); + + let expected = managed_environment_path(&state_path, "managed-layout").unwrap(); + fs::create_dir_all(expected.parent().unwrap()).unwrap(); + fs::write(&expected, b"not a directory").unwrap(); + let error = validate_environment_state( + &manifest, + &state_path, + Some(expected.to_string_lossy().as_ref()), + ) + .unwrap_err(); + assert!(error.to_string().contains("must be a directory")); + + fs::remove_file(&expected).unwrap(); + fs::create_dir(&expected).unwrap(); + let configured_file = temp.path().join("configured-file"); + fs::write(&configured_file, b"not a directory").unwrap(); + let error = validate_environment_state( + &manifest, + &state_path, + Some(configured_file.to_string_lossy().as_ref()), + ) + .unwrap_err(); + assert!(error.to_string().contains("configured Python environment")); + + let error = validate_environment_state( + &manifest, + &state_path, + Some(expected.to_string_lossy().as_ref()), + ) + .unwrap_err(); + assert!(error.to_string().contains("has no Python launcher")); + + let python = environment_python_path(&expected); + fs::create_dir_all(python.parent().unwrap()).unwrap(); + fs::write(&python, b"python").unwrap(); + manifest.integrity = None; + let error = validate_environment_state( + &manifest, + &state_path, + Some(expected.to_string_lossy().as_ref()), + ) + .unwrap_err(); + assert!(error.to_string().contains("requires integrity.sha256")); + + let DynamicPluginManifestLoad::Worker(load) = &mut manifest.load else { + unreachable!(); + }; + load.runtime = Some(WorkerRuntime::Command); + assert_eq!( + validate_environment_state(&manifest, &state_path, None).unwrap(), + DynamicPluginCheckState::Unknown + ); +} + +#[test] +fn attestation_parser_rejects_unauthenticated_documents_before_tree_verification() { + let temp = tempdir().unwrap(); + let attestation_path = temp.path().join(ENVIRONMENT_ATTESTATION_FILE); + fs::write(&attestation_path, "not-json").unwrap(); + assert!( + read_environment_attestation(temp.path(), "sha256:source") + .unwrap_err() + .to_string() + .contains("attestation") + ); + + fs::write( + &attestation_path, + serde_json::json!({ + "version": 1, + "source_artifact_sha256": "sha256:source", + "environment_sha256": "a".repeat(64), + "authentication": "not-authenticated" + }) + .to_string(), + ) + .unwrap(); + let error = read_environment_attestation(temp.path(), "sha256:source").unwrap_err(); + assert!(error.to_string().contains("failed authentication")); + + assert!(!verify_environment_authentication("source", "environment", "plain").unwrap()); + assert!( + !verify_environment_authentication("source", "environment", "hmac-sha256:not-hex").unwrap() + ); + assert_eq!(decode_fixed_hex::<2>("00ff"), Some([0, 255])); + assert_eq!(decode_fixed_hex::<2>("0"), None); +} + +#[test] +fn environment_digest_rejects_cycles_depth_overflow_entry_overflow_and_special_files() { + let temp = tempdir().unwrap(); + let environment = temp.path().join("environment"); + fs::create_dir(&environment).unwrap(); + fs::write(environment.join("module.py"), b"fixture").unwrap(); + fs::create_dir(environment.join("__pycache__")).unwrap(); + fs::write(environment.join("cached.pyc"), b"ignored").unwrap(); + assert_eq!(environment_tree_digest(&environment).unwrap().len(), 64); + + let missing = temp.path().join("missing-environment"); + assert!( + environment_tree_digest(&missing) + .unwrap_err() + .to_string() + .contains("normalize environment directory") + ); + let not_directory = temp.path().join("not-a-directory"); + fs::write(¬_directory, b"file").unwrap(); + assert!( + environment_tree_digest(¬_directory) + .unwrap_err() + .to_string() + .contains("read environment directory") + ); + + let absolute = resolve_relative_path(Path::new("ignored"), &environment.to_string_lossy()); + assert_eq!(absolute, environment); + assert!( + absolute_path(Path::new("relative-environment")) + .unwrap() + .is_absolute() + ); + + let mut digest = Sha256::new(); + let mut entries = MAX_ENVIRONMENT_FILES; + let error = digest_environment_directory( + &environment, + Path::new(""), + &mut vec![PathBuf::new(); MAX_ENVIRONMENT_DEPTH], + &mut digest, + &mut 0, + &mut 0, + ) + .unwrap_err(); + assert!(error.to_string().contains("traversal depth")); + + let canonical = environment.canonicalize().unwrap(); + let error = digest_environment_directory( + &environment, + Path::new(""), + &mut vec![canonical], + &mut Sha256::new(), + &mut 0, + &mut 0, + ) + .unwrap_err(); + assert!(error.to_string().contains("symlink cycle")); + + let error = digest_environment_directory( + &environment, + Path::new(""), + &mut Vec::new(), + &mut Sha256::new(), + &mut 0, + &mut entries, + ) + .unwrap_err(); + assert!(error.to_string().contains("entry attestation budget")); + + #[cfg(unix)] + { + use std::ffi::CString; + use std::os::unix::fs::symlink; + + let dangling = environment.join("dangling"); + symlink(environment.join("missing"), &dangling).unwrap(); + assert!( + resolve_environment_entry(&dangling) + .unwrap_err() + .to_string() + .contains("resolve environment symlink") + ); + fs::remove_file(dangling).unwrap(); + + let fifo = environment.join("worker.pipe"); + let encoded = CString::new(fifo.as_os_str().as_encoded_bytes()).unwrap(); + // SAFETY: `encoded` is a valid NUL-terminated path and the mode is permission bits only. + assert_eq!(unsafe { libc::mkfifo(encoded.as_ptr(), 0o600) }, 0); + let error = environment_tree_digest(&environment).unwrap_err(); + assert!(error.to_string().contains("regular file or directory")); + } +} + +#[test] +fn bootstrap_hmac_key_is_created_reused_and_invalid_lengths_are_rejected() { + let temp = tempdir().unwrap(); + let key_path = temp.path().join("bootstrap/fingerprint-hmac.key"); + + let created = load_or_create_hmac_key_at(&key_path).unwrap(); + assert_eq!(created.len(), HMAC_KEY_BYTES); + assert_eq!(load_or_create_hmac_key_at(&key_path).unwrap(), created); + #[cfg(windows)] + { + assert!(windows_path_is_private(key_path.parent().unwrap()).unwrap()); + assert!(windows_path_is_private(&key_path).unwrap()); + } + + fs::write(&key_path, b"short").unwrap(); + let error = load_or_create_hmac_key_at(&key_path).unwrap_err(); + assert!(error.to_string().contains("invalid length")); + + let blocked_config_root = temp.path().join("blocked-config-root"); + fs::write(&blocked_config_root, b"file").unwrap(); + let error = load_or_create_hmac_key_at(&blocked_config_root.join("key")).unwrap_err(); + assert!( + error + .to_string() + .contains("create bootstrap state directory") + ); +} + +#[cfg(windows)] +#[test] +fn windows_environment_helpers_use_native_launcher_and_path_encoding() { + let environment = Path::new(r"C:\relay\environment"); + assert_eq!( + environment_python_path(environment), + environment.join("Scripts").join("python.exe") + ); + + let path = r"C:\relay\plugin"; + let expected = path + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect::>(); + assert_eq!(raw_path_bytes(Path::new(path)), expected); +} diff --git a/crates/plugin-host-config/tests/unit/error.rs b/crates/plugin-host-config/tests/unit/error.rs new file mode 100644 index 000000000..2276033c1 --- /dev/null +++ b/crates/plugin-host-config/tests/unit/error.rs @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::PathBuf; + +use nemo_relay::plugin::PluginError; + +use super::{PluginHostConfigError, sanitize_parser_reason}; + +#[test] +fn parser_reason_preserves_schema_identifiers_but_redacts_values() { + let reason = + "unknown field `enabled`, expected `manifest` or `config`; invalid value \"secret\""; + + let sanitized = sanitize_parser_reason(reason); + + assert!(sanitized.contains("`enabled`")); + assert!(sanitized.contains("`manifest`")); + assert!(sanitized.contains("`config`")); + assert!(!sanitized.contains("secret")); + assert!(sanitized.contains("\"\"")); +} + +#[test] +fn parser_reason_redacts_escaped_and_single_quoted_values_on_the_first_line() { + let sanitized = sanitize_parser_reason( + "invalid value 'first\\'secret' for `field`; second line is ignored\ncredential-leak", + ); + + assert!(sanitized.contains("''")); + assert!(sanitized.contains("`field`")); + assert!(!sanitized.contains("secret")); + assert!(!sanitized.contains("credential-leak")); +} + +#[test] +fn every_host_configuration_error_maps_to_the_public_plugin_taxonomy() { + assert!(matches!( + PluginHostConfigError::InvalidConfig("bad config".into()).into_plugin_error(), + PluginError::InvalidConfig(message) if message == "bad config" + )); + assert!(matches!( + PluginHostConfigError::NotFound { + path: PathBuf::from("missing/manifest.toml"), + message: "absent".into(), + } + .into_plugin_error(), + PluginError::NotFound(message) + if message.contains("missing/manifest.toml") && message.contains("absent") + )); + assert!(matches!( + PluginHostConfigError::io( + "read fixture", + "fixture.toml", + std::io::Error::new(std::io::ErrorKind::PermissionDenied, "denied"), + ) + .into_plugin_error(), + PluginError::InvalidConfig(message) + if message.contains("read fixture") && message.contains("denied") + )); + assert!(matches!( + PluginHostConfigError::Relay(PluginError::Internal("relay failure".into())) + .into_plugin_error(), + PluginError::Internal(message) if message == "relay failure" + )); + + let json_error = serde_json::from_str::("{").unwrap_err(); + assert!(matches!( + PluginHostConfigError::Json(json_error).into_plugin_error(), + PluginError::Serialization(_) + )); +} diff --git a/crates/plugin-host-config/tests/unit/io.rs b/crates/plugin-host-config/tests/unit/io.rs new file mode 100644 index 000000000..ac5b69509 --- /dev/null +++ b/crates/plugin-host-config/tests/unit/io.rs @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::fs::OpenOptions; +use std::path::Path; + +use tempfile::tempdir; + +use super::*; + +#[test] +fn oversized_regular_file_is_rejected_from_metadata_without_allocation() { + let temp = tempdir().unwrap(); + let path = temp.path().join("oversized.toml"); + let file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&path) + .unwrap(); + file.set_len(MAX_BOUNDED_FILE_BYTES + 1).unwrap(); + + let error = read_bounded_regular_file(&path, "plugin configuration file").unwrap_err(); + assert!(error.to_string().contains("exceeds")); + assert!(error.to_string().contains("byte limit")); +} + +#[test] +fn utf8_reader_rejects_invalid_bytes_without_disclosing_contents() { + let temp = tempdir().unwrap(); + let path = temp.path().join("invalid.toml"); + std::fs::write(&path, [0xff, 0xfe]).unwrap(); + + let error = read_bounded_utf8_regular_file(&path, "fixture configuration").unwrap_err(); + + assert!(error.to_string().contains("is not valid UTF-8")); + assert!(!error.to_string().contains("255")); +} + +#[test] +fn manifest_loader_accepts_a_directory_and_preserves_authored_bytes() { + let temp = tempdir().unwrap(); + let contents = r#"manifest_version = 1 +[plugin] +id = "fixture" +kind = "rust_dynamic" +[compat] +relay = ">=0.5,<1.0" +native_api = "v1" +[capabilities] +items = ["plugin_native"] +[defaults] +enabled = false +[load] +library = "plugin.so" +symbol = "nemo_relay_plugin_entrypoint_v1" +[source] +artifact = "plugin.so" +[integrity] +sha256 = "sha256:placeholder" +"#; + std::fs::write(temp.path().join(DYNAMIC_PLUGIN_MANIFEST_FILENAME), contents).unwrap(); + + let (manifest, normalized, bytes) = + load_bounded_dynamic_plugin_manifest_bytes(temp.path()).unwrap(); + + assert_eq!(manifest.plugin.id, "fixture"); + assert_eq!( + Path::new(&normalized), + temp.path() + .join(DYNAMIC_PLUGIN_MANIFEST_FILENAME) + .canonicalize() + .unwrap() + ); + assert_eq!(bytes, contents.as_bytes()); +} + +#[test] +fn manifest_loader_maps_missing_invalid_utf8_and_invalid_toml_errors() { + let temp = tempdir().unwrap(); + let missing = + load_bounded_dynamic_plugin_manifest_bytes(temp.path().join("missing.toml")).unwrap_err(); + assert!(matches!(missing, PluginHostConfigError::NotFound { .. })); + + let manifest = temp.path().join("invalid-utf8.toml"); + std::fs::write(&manifest, [0xff]).unwrap(); + let error = load_bounded_dynamic_plugin_manifest_bytes(&manifest).unwrap_err(); + assert!(error.to_string().contains("manifest") && error.to_string().contains("not UTF-8")); + + std::fs::write(&manifest, "manifest_version = [").unwrap(); + let error = load_bounded_dynamic_plugin_manifest_bytes(&manifest).unwrap_err(); + assert!(error.to_string().contains("manifest") && error.to_string().contains("invalid")); +} diff --git a/crates/plugin-host-config/tests/unit/lifecycle.rs b/crates/plugin-host-config/tests/unit/lifecycle.rs new file mode 100644 index 000000000..307dca5af --- /dev/null +++ b/crates/plugin-host-config/tests/unit/lifecycle.rs @@ -0,0 +1,815 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::fs; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use nemo_relay::plugin::PluginConfig; +use sha2::{Digest, Sha256}; +use tempfile::tempdir; + +use super::*; +use crate::environment::MANAGED_ENVIRONMENTS_DIR; +use crate::resolver::resolve_plugin_files_from_paths; +use crate::state::DYNAMIC_PLUGIN_STATE_FILENAME; + +fn write_native_plugin(root: &Path, id: &str) -> PathBuf { + let directory = root.join(id); + fs::create_dir_all(&directory).unwrap(); + let artifact = directory.join("plugin.so"); + fs::write(&artifact, b"native plugin fixture").unwrap(); + let digest = Sha256::digest(b"native plugin fixture") + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let manifest = directory.join("relay-plugin.toml"); + fs::write( + &manifest, + format!( + r#"manifest_version = 1 +[plugin] +id = "{id}" +kind = "rust_dynamic" +[compat] +relay = ">=0.5,<1.0" +native_api = "v1" +[capabilities] +items = ["plugin_native"] +[defaults] +enabled = false +[load] +library = "plugin.so" +symbol = "nemo_relay_plugin_entrypoint_v1" +[source] +artifact = "plugin.so" +[integrity] +sha256 = "sha256:{digest}" +"# + ), + ) + .unwrap(); + manifest +} + +fn write_python_plugin(root: &Path, id: &str) -> PathBuf { + let directory = root.join(id); + fs::create_dir_all(&directory).unwrap(); + let artifact_body = b"def main():\n return None\n"; + fs::write(directory.join("plugin.py"), artifact_body).unwrap(); + let digest = Sha256::digest(artifact_body) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let manifest = directory.join("relay-plugin.toml"); + fs::write( + &manifest, + format!( + r#"manifest_version = 1 +[plugin] +id = "{id}" +kind = "worker" +[compat] +relay = ">=0.5,<1.0" +worker_protocol = "grpc-v1" +[capabilities] +items = ["plugin_worker"] +[defaults] +enabled = false +[source] +manifest_root = "." +artifact = "plugin.py" +[integrity] +sha256 = "sha256:{digest}" +[load] +runtime = "python" +entrypoint = "plugin:main" +"# + ), + ) + .unwrap(); + manifest +} + +fn write_command_plugin(root: &Path, id: &str) -> PathBuf { + let directory = root.join(id); + fs::create_dir_all(&directory).unwrap(); + let artifact_body = b"command worker fixture\n"; + fs::write(directory.join("worker"), artifact_body).unwrap(); + let digest = Sha256::digest(artifact_body) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let manifest = directory.join("relay-plugin.toml"); + fs::write( + &manifest, + format!( + r#"manifest_version = 1 +[plugin] +id = "{id}" +kind = "worker" +[compat] +relay = ">=0.5,<1.0" +worker_protocol = "grpc-v1" +[capabilities] +items = ["plugin_worker"] +[defaults] +enabled = false +[load] +runtime = "command" +entrypoint = "worker" +[source] +artifact = "worker" +[integrity] +sha256 = "sha256:{digest}" +"# + ), + ) + .unwrap(); + manifest +} + +fn write_plugin_declaration(config: &Path, manifest: &Path) { + fs::write( + config, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + manifest.to_string_lossy() + ), + ) + .unwrap(); +} + +fn mutate_state(config: &Path, mutate: impl FnOnce(&mut serde_json::Value)) { + let state_path = config.parent().unwrap().join(DYNAMIC_PLUGIN_STATE_FILENAME); + let mut state: serde_json::Value = + serde_json::from_str(&fs::read_to_string(&state_path).unwrap()).unwrap(); + mutate(&mut state); + fs::write(&state_path, serde_json::to_vec_pretty(&state).unwrap()).unwrap(); +} + +fn state_record<'a>(state: &'a serde_json::Value, plugin_id: &str) -> &'a serde_json::Value { + state["records"] + .as_array() + .unwrap() + .iter() + .find(|record| record["metadata"]["id"] == plugin_id) + .unwrap() +} + +#[test] +fn missing_lifecycle_record_is_hydrated_disabled() { + let temp = tempdir().unwrap(); + let manifest = write_native_plugin(temp.path(), "fixture"); + let config = temp.path().join("plugins.toml"); + fs::write( + &config, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + manifest.to_string_lossy() + ), + ) + .unwrap(); + let resolved = resolve_plugin_files_from_paths([config], None).unwrap(); + let reconciled = reconcile_plugin_lifecycle(&resolved).unwrap(); + assert!(reconciled.enabled_plugins.is_empty()); + let state: serde_json::Value = serde_json::from_str( + &fs::read_to_string(temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME)).unwrap(), + ) + .unwrap(); + assert_eq!(state["records"][0]["spec"]["enabled"], false); +} + +#[test] +fn same_directory_sources_share_state_without_collapsing_ownership() { + let temp = tempdir().unwrap(); + let custom_manifest = write_native_plugin(&temp.path().join("artifacts"), "custom-owner"); + let default_manifest = write_native_plugin(&temp.path().join("artifacts"), "default-owner"); + let custom = temp.path().join("custom.toml"); + let default = temp.path().join("plugins.toml"); + write_plugin_declaration(&custom, &custom_manifest); + write_plugin_declaration(&default, &default_manifest); + + let resolved = + resolve_plugin_files_from_paths([custom.clone(), default.clone()], None).unwrap(); + let reconciled = reconcile_plugin_lifecycle(&resolved).unwrap(); + + assert!(reconciled.enabled_plugins.is_empty()); + let state_path = temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME); + let state: serde_json::Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + assert_eq!(state["records"].as_array().unwrap().len(), 2); + assert_eq!( + state["declaration_sources"]["custom-owner"], + dunce::canonicalize(&custom) + .unwrap() + .to_string_lossy() + .as_ref() + ); + assert_eq!( + state["declaration_sources"]["default-owner"], + dunce::canonicalize(&default) + .unwrap() + .to_string_lossy() + .as_ref() + ); + assert!( + state_record(&state, "custom-owner")["spec"]["config_ref"].is_null(), + "physical declaration ownership must not consume the public config_ref field" + ); +} + +#[test] +fn moving_same_id_requires_removal_before_hydrating_new_owner_disabled() { + let temp = tempdir().unwrap(); + let manifest = write_native_plugin(temp.path(), "moved-owner"); + let custom = temp.path().join("custom.toml"); + let default = temp.path().join("plugins.toml"); + write_plugin_declaration(&custom, &manifest); + fs::write(&default, "version = 1\n").unwrap(); + + let first = resolve_plugin_files_from_paths([custom.clone(), default.clone()], None).unwrap(); + reconcile_plugin_lifecycle(&first).unwrap(); + mutate_state(&custom, |state| { + state["records"][0]["spec"]["enabled"] = true.into() + }); + + fs::write(&custom, "version = 1\n").unwrap(); + write_plugin_declaration(&default, &manifest); + let moved = resolve_plugin_files_from_paths([custom.clone(), default.clone()], None).unwrap(); + let error = reconcile_plugin_lifecycle(&moved).unwrap_err(); + + assert!(error.to_string().contains("live lifecycle state owned by")); + assert!(error.to_string().contains("control plane")); + let state_path = temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME); + let unchanged: serde_json::Value = + serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + let unchanged_record = state_record(&unchanged, "moved-owner"); + assert_eq!(unchanged_record["spec"]["present"], true); + assert_eq!(unchanged_record["spec"]["enabled"], true); + assert_eq!( + unchanged["declaration_sources"]["moved-owner"], + dunce::canonicalize(&custom) + .unwrap() + .to_string_lossy() + .as_ref() + ); + + mutate_state(&custom, |state| { + state["records"][0]["spec"]["present"] = false.into(); + state["records"][0]["spec"]["enabled"] = false.into(); + }); + let reconciled = reconcile_plugin_lifecycle(&moved).unwrap(); + + assert!(reconciled.enabled_plugins.is_empty()); + let state: serde_json::Value = serde_json::from_slice(&fs::read(state_path).unwrap()).unwrap(); + let record = state_record(&state, "moved-owner"); + assert_eq!(record["spec"]["present"], true); + assert_eq!(record["spec"]["enabled"], false); + assert_eq!( + state["declaration_sources"]["moved-owner"], + dunce::canonicalize(&default) + .unwrap() + .to_string_lossy() + .as_ref() + ); + assert!( + !temp + .path() + .join(".custom.toml.dynamic-plugins.json") + .exists() + ); +} + +#[test] +fn legacy_owner_claim_preserves_enablement_and_non_path_config_ref() { + let temp = tempdir().unwrap(); + let manifest = write_native_plugin(temp.path(), "legacy-owner"); + let config = temp.path().join("plugins.toml"); + write_plugin_declaration(&config, &manifest); + let resolved = resolve_plugin_files_from_paths([config.clone()], None).unwrap(); + reconcile_plugin_lifecycle(&resolved).unwrap(); + mutate_state(&config, |state| { + state.as_object_mut().unwrap().remove("declaration_sources"); + state["records"][0]["spec"]["enabled"] = true.into(); + state["records"][0]["spec"]["config_ref"] = "plugins.acme.guardrails.pii".into(); + }); + + let reconciled = reconcile_plugin_lifecycle(&resolved).unwrap(); + + assert_eq!(reconciled.enabled_plugins.len(), 1); + let state: serde_json::Value = + serde_json::from_slice(&fs::read(temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME)).unwrap()) + .unwrap(); + assert_eq!( + state_record(&state, "legacy-owner")["spec"]["config_ref"], + "plugins.acme.guardrails.pii" + ); + assert_eq!( + state["declaration_sources"]["legacy-owner"], + dunce::canonicalize(&config) + .unwrap() + .to_string_lossy() + .as_ref() + ); +} + +#[test] +fn manifest_identity_change_after_resolution_fails_without_state_write() { + let temp = tempdir().unwrap(); + let manifest = write_native_plugin(temp.path(), "resolved-id"); + let config = temp.path().join("plugins.toml"); + write_plugin_declaration(&config, &manifest); + let resolved = resolve_plugin_files_from_paths([config.clone()], None).unwrap(); + let state_path = config.parent().unwrap().join(DYNAMIC_PLUGIN_STATE_FILENAME); + assert!(!state_path.exists()); + + let changed = fs::read_to_string(&manifest) + .unwrap() + .replace("id = \"resolved-id\"", "id = \"reloaded-id\""); + fs::write(&manifest, changed).unwrap(); + + let error = reconcile_plugin_lifecycle(&resolved).unwrap_err(); + assert!(matches!(error, PluginHostConfigError::InvalidConfig(_))); + assert!(error.to_string().contains("changed identity")); + assert!(error.to_string().contains("resolved-id")); + assert!(error.to_string().contains("reloaded-id")); + assert!(!state_path.exists()); +} + +#[test] +fn no_dynamic_declarations_produce_static_plan() { + let resolved = ResolvedPluginFileConfiguration { + config: PluginConfig::default(), + runtime_value: None, + dynamic_plugins: Vec::new(), + dynamic_plugin_policy: Default::default(), + diagnostics: Vec::new(), + contributing_sources: Vec::new(), + contributing_selected_sources: Vec::new(), + selected_sources: Vec::new(), + had_input: true, + }; + let plan = prepare_plugin_host_activation(resolved).unwrap(); + assert!(plan.dynamic_plugins.is_empty()); +} + +#[test] +fn static_only_plan_ignores_orphan_lifecycle_state_without_locking_it() { + let temp = tempdir().unwrap(); + let config = temp.path().join("plugins.toml"); + let state = temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME); + let lock = temp.path().join(".dynamic-plugins.lock"); + fs::write(&config, "").unwrap(); + fs::write(&state, "not valid lifecycle state").unwrap(); + let original_state = fs::read(&state).unwrap(); + + let resolved = resolve_plugin_files_from_paths([config], None).unwrap(); + let plan = prepare_plugin_host_activation(resolved).unwrap(); + + assert!(plan.dynamic_plugins.is_empty()); + assert!(!lock.exists()); + assert_eq!(fs::read(state).unwrap(), original_state); +} + +#[test] +fn enabled_and_tombstoned_state_are_joined_only_to_their_source() { + let temp = tempdir().unwrap(); + let manifest = write_native_plugin(temp.path(), "stateful"); + let first_dir = temp.path().join("first"); + let second_dir = temp.path().join("second"); + fs::create_dir_all(&first_dir).unwrap(); + fs::create_dir_all(&second_dir).unwrap(); + let first = first_dir.join("plugins.toml"); + let second = second_dir.join("plugins.toml"); + write_plugin_declaration(&first, &manifest); + write_plugin_declaration(&second, &manifest); + + let first_resolved = resolve_plugin_files_from_paths([first.clone()], None).unwrap(); + reconcile_plugin_lifecycle(&first_resolved).unwrap(); + mutate_state(&first, |state| { + state["records"][0]["spec"]["enabled"] = true.into() + }); + let first_enabled = reconcile_plugin_lifecycle(&first_resolved).unwrap(); + assert_eq!(first_enabled.enabled_plugins.len(), 1); + + let second_resolved = resolve_plugin_files_from_paths([second.clone()], None).unwrap(); + let second_disabled = reconcile_plugin_lifecycle(&second_resolved).unwrap(); + assert!(second_disabled.enabled_plugins.is_empty()); + + mutate_state(&first, |state| { + state["records"][0]["spec"]["present"] = false.into(); + state["records"][0]["spec"]["enabled"] = false.into(); + }); + let tombstoned = reconcile_plugin_lifecycle(&first_resolved).unwrap(); + assert!(tombstoned.enabled_plugins.is_empty()); +} + +#[test] +fn unowned_legacy_record_in_another_selected_source_blocks_rehydration() { + let temp = tempdir().unwrap(); + let manifest = write_native_plugin(&temp.path().join("plugin"), "cross-source-live"); + let user_dir = temp.path().join("user"); + let project_dir = temp.path().join("project"); + fs::create_dir_all(&user_dir).unwrap(); + fs::create_dir_all(&project_dir).unwrap(); + let user = user_dir.join("plugins.toml"); + let project = project_dir.join("plugins.toml"); + write_plugin_declaration(&user, &manifest); + + let first = resolve_plugin_files_from_paths([user.clone()], None).unwrap(); + reconcile_plugin_lifecycle(&first).unwrap(); + mutate_state(&user, |state| { + state.as_object_mut().unwrap().remove("declaration_sources"); + state["records"][0]["spec"]["enabled"] = true.into(); + }); + fs::write(&user, "version = 1\n").unwrap(); + write_plugin_declaration(&project, &manifest); + + let layered = resolve_plugin_files_from_paths([user.clone(), project.clone()], None).unwrap(); + let error = reconcile_plugin_lifecycle(&layered).unwrap_err(); + + assert!(error.to_string().contains("cross-source-live")); + assert!(error.to_string().contains("outside its declaring source")); + assert!(error.to_string().contains(".dynamic-plugins.json")); + assert!(!project_dir.join(DYNAMIC_PLUGIN_STATE_FILENAME).exists()); +} + +#[test] +fn cross_directory_move_requires_removal_before_new_owner_hydration() { + let temp = tempdir().unwrap(); + let manifest = write_native_plugin(&temp.path().join("plugin"), "cross-source-move"); + let user_dir = temp.path().join("user"); + let project_dir = temp.path().join("project"); + fs::create_dir_all(&user_dir).unwrap(); + fs::create_dir_all(&project_dir).unwrap(); + let user = user_dir.join("plugins.toml"); + let project = project_dir.join("plugins.toml"); + write_plugin_declaration(&user, &manifest); + + let first = resolve_plugin_files_from_paths([user.clone()], None).unwrap(); + reconcile_plugin_lifecycle(&first).unwrap(); + mutate_state(&user, |state| { + state["records"][0]["spec"]["enabled"] = true.into() + }); + fs::write(&user, "version = 1\n").unwrap(); + write_plugin_declaration(&project, &manifest); + + let moved = resolve_plugin_files_from_paths([user.clone(), project.clone()], None).unwrap(); + let error = reconcile_plugin_lifecycle(&moved).unwrap_err(); + + assert!(error.to_string().contains("live lifecycle state outside")); + let unchanged: serde_json::Value = + serde_json::from_slice(&fs::read(user_dir.join(DYNAMIC_PLUGIN_STATE_FILENAME)).unwrap()) + .unwrap(); + assert_eq!( + state_record(&unchanged, "cross-source-move")["spec"]["present"], + true + ); + assert_eq!( + state_record(&unchanged, "cross-source-move")["spec"]["enabled"], + true + ); + + mutate_state(&user, |state| { + state["records"][0]["spec"]["present"] = false.into(); + state["records"][0]["spec"]["enabled"] = false.into(); + }); + let reconciled = reconcile_plugin_lifecycle(&moved).unwrap(); + + assert!(reconciled.enabled_plugins.is_empty()); + let old_state: serde_json::Value = + serde_json::from_slice(&fs::read(user_dir.join(DYNAMIC_PLUGIN_STATE_FILENAME)).unwrap()) + .unwrap(); + assert_eq!( + state_record(&old_state, "cross-source-move")["spec"]["present"], + false + ); + assert_eq!( + old_state["declaration_sources"]["cross-source-move"], + dunce::canonicalize(&user) + .unwrap() + .to_string_lossy() + .as_ref() + ); + let new_state: serde_json::Value = + serde_json::from_slice(&fs::read(project_dir.join(DYNAMIC_PLUGIN_STATE_FILENAME)).unwrap()) + .unwrap(); + assert_eq!( + state_record(&new_state, "cross-source-move")["spec"]["enabled"], + false + ); + assert_eq!( + new_state["declaration_sources"]["cross-source-move"], + dunce::canonicalize(&project) + .unwrap() + .to_string_lossy() + .as_ref() + ); +} + +#[test] +fn corrupt_lifecycle_state_fails_before_activation_planning() { + let temp = tempdir().unwrap(); + let manifest = write_native_plugin(temp.path(), "corrupt"); + let config = temp.path().join("plugins.toml"); + write_plugin_declaration(&config, &manifest); + fs::write( + temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME), + b"{not-json", + ) + .unwrap(); + let resolved = resolve_plugin_files_from_paths([config], None).unwrap(); + let error = reconcile_plugin_lifecycle(&resolved).unwrap_err(); + assert!( + error + .to_string() + .contains("invalid dynamic plugin registry state") + ); +} + +#[test] +fn lifecycle_parse_errors_do_not_disclose_state_values() { + let temp = tempdir().unwrap(); + let manifest = write_native_plugin(temp.path(), "redacted-state"); + let config = temp.path().join("plugins.toml"); + write_plugin_declaration(&config, &manifest); + let secret = "do-not-leak-lifecycle-value"; + fs::write( + temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME), + format!(r#"{{"schema_version":1,"records":"{secret}"}}"#), + ) + .unwrap(); + let resolved = resolve_plugin_files_from_paths([config], None).unwrap(); + let error = reconcile_plugin_lifecycle(&resolved).unwrap_err(); + assert!(!error.to_string().contains(secret)); +} + +#[test] +fn policy_and_trust_failures_are_persisted_without_loading_disabled_code() { + let temp = tempdir().unwrap(); + let manifest = write_native_plugin(temp.path(), "blocked"); + let config = temp.path().join("plugins.toml"); + fs::write( + &config, + format!( + "[plugins.policy.defaults]\nallowed = false\n\n[[plugins.dynamic]]\nmanifest = {:?}\n", + manifest.to_string_lossy() + ), + ) + .unwrap(); + let resolved = resolve_plugin_files_from_paths([config.clone()], None).unwrap(); + let reconciled = reconcile_plugin_lifecycle(&resolved).unwrap(); + assert!(reconciled.enabled_plugins.is_empty()); + let state: serde_json::Value = serde_json::from_str( + &fs::read_to_string(temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME)).unwrap(), + ) + .unwrap(); + assert_eq!( + state["records"][0]["status"]["validation"]["policy_satisfied"], + "invalid" + ); + assert_eq!( + state["records"][0]["status"]["last_error"]["code"], + "policy_blocked" + ); +} + +#[test] +fn python_worker_without_managed_environment_is_not_provisioned() { + let temp = tempdir().unwrap(); + let manifest = write_python_plugin(temp.path(), "python-worker"); + let config = temp.path().join("plugins.toml"); + write_plugin_declaration(&config, &manifest); + let resolved = resolve_plugin_files_from_paths([config], None).unwrap(); + let reconciled = reconcile_plugin_lifecycle(&resolved).unwrap(); + assert!(reconciled.enabled_plugins.is_empty()); + assert!(!temp.path().join(MANAGED_ENVIRONMENTS_DIR).exists()); + let state: serde_json::Value = serde_json::from_str( + &fs::read_to_string(temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME)).unwrap(), + ) + .unwrap(); + assert_eq!( + state["records"][0]["status"]["validation"]["environment"], + "invalid" + ); +} + +#[cfg(unix)] +#[test] +fn lifecycle_save_failure_occurs_before_snapshot_or_code_load() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempdir().unwrap(); + let plugin_root = temp.path().join("plugin-root"); + let config_root = temp.path().join("config-root"); + fs::create_dir_all(&plugin_root).unwrap(); + fs::create_dir_all(&config_root).unwrap(); + let manifest = write_native_plugin(&plugin_root, "save-failure"); + let config = config_root.join("plugins.toml"); + write_plugin_declaration(&config, &manifest); + fs::set_permissions(&config_root, fs::Permissions::from_mode(0o500)).unwrap(); + let resolved = resolve_plugin_files_from_paths([config], None).unwrap(); + let error = match prepare_plugin_host_activation(resolved) { + Ok(_) => panic!("read-only configuration directory unexpectedly accepted a state save"), + Err(error) => error, + }; + fs::set_permissions(&config_root, fs::Permissions::from_mode(0o700)).unwrap(); + assert!(error.to_string().contains("lifecycle state")); +} + +#[test] +fn enabled_plugin_plan_uses_a_retained_snapshot() { + let temp = tempdir().unwrap(); + let manifest = write_native_plugin(temp.path(), "snapshot"); + let config = temp.path().join("plugins.toml"); + write_plugin_declaration(&config, &manifest); + let resolved = resolve_plugin_files_from_paths([config.clone()], None).unwrap(); + reconcile_plugin_lifecycle(&resolved).unwrap(); + mutate_state(&config, |state| { + state["records"][0]["spec"]["enabled"] = true.into() + }); + let plan = prepare_plugin_host_activation(resolved).unwrap(); + assert_eq!(plan.dynamic_plugins.len(), 1); + let planned = &plan.dynamic_plugins[0]; + assert_ne!(Path::new(&planned.spec.manifest_ref), manifest.as_path()); + planned.resource.verify().unwrap(); +} + +#[test] +fn declaration_manifest_refreshes_stale_lifecycle_load_metadata() { + let temp = tempdir().unwrap(); + let first_manifest = write_native_plugin(&temp.path().join("v1"), "same-id"); + let second_manifest = write_native_plugin(&temp.path().join("v2"), "same-id"); + let config = temp.path().join("plugins.toml"); + write_plugin_declaration(&config, &first_manifest); + let first = resolve_plugin_files_from_paths([config.clone()], None).unwrap(); + reconcile_plugin_lifecycle(&first).unwrap(); + mutate_state(&config, |state| { + state["records"][0]["spec"]["enabled"] = true.into(); + state["records"][0]["metadata"]["generation"] = 7.into(); + }); + + write_plugin_declaration(&config, &second_manifest); + let second = resolve_plugin_files_from_paths([config], None).unwrap(); + let reconciled = reconcile_plugin_lifecycle(&second).unwrap(); + assert_eq!(reconciled.enabled_plugins.len(), 1); + assert_eq!(reconciled.enabled_plugins[0].lifecycle_generation, 7); + assert_eq!( + PathBuf::from(&reconciled.enabled_plugins[0].manifest_ref), + second_manifest.canonicalize().unwrap() + ); +} + +#[test] +fn declaration_kind_change_drives_the_activation_plan() { + let temp = tempdir().unwrap(); + let native_manifest = write_native_plugin(&temp.path().join("native"), "same-kind-id"); + let worker_manifest = write_command_plugin(&temp.path().join("worker"), "same-kind-id"); + let config = temp.path().join("plugins.toml"); + write_plugin_declaration(&config, &native_manifest); + let first = resolve_plugin_files_from_paths([config.clone()], None).unwrap(); + reconcile_plugin_lifecycle(&first).unwrap(); + mutate_state(&config, |state| { + state["records"][0]["spec"]["enabled"] = true.into() + }); + + write_plugin_declaration(&config, &worker_manifest); + let second = resolve_plugin_files_from_paths([config], None).unwrap(); + let reconciled = reconcile_plugin_lifecycle(&second).unwrap(); + assert_eq!( + reconciled.enabled_plugins[0].kind, + DynamicPluginKind::Worker + ); + assert_eq!( + PathBuf::from(&reconciled.enabled_plugins[0].manifest_ref), + worker_manifest.canonicalize().unwrap() + ); + let plan = prepare_plugin_host_activation(second).unwrap(); + assert_eq!(plan.dynamic_plugins[0].spec.kind, DynamicPluginKind::Worker); +} + +#[test] +fn reconciliation_cannot_overwrite_a_concurrent_enable_transaction() { + let temp = tempdir().unwrap(); + let manifest = write_native_plugin(temp.path(), "concurrent-enable"); + let config = temp.path().join("plugins.toml"); + write_plugin_declaration(&config, &manifest); + let resolved = resolve_plugin_files_from_paths([config.clone()], None).unwrap(); + reconcile_plugin_lifecycle(&resolved).unwrap(); + + let state_path = sibling_lifecycle_state_path(&dunce::canonicalize(&config).unwrap()); + let control_plane_lock = lock_lifecycle_state(&state_path).unwrap(); + let mut control_plane_registry = read_locked_lifecycle_state(&control_plane_lock).unwrap(); + let (started_tx, started_rx) = mpsc::channel(); + let (finished_tx, finished_rx) = mpsc::channel(); + let background_resolved = resolved.clone(); + let reconcile = thread::spawn(move || { + started_tx.send(()).unwrap(); + let result = reconcile_plugin_lifecycle(&background_resolved); + finished_tx.send(result).unwrap(); + }); + started_rx.recv().unwrap(); + assert!(matches!( + finished_rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + )); + + control_plane_registry.enable("concurrent-enable").unwrap(); + save_locked_lifecycle_state(&control_plane_lock, &control_plane_registry).unwrap(); + drop(control_plane_lock); + + let reconciled = finished_rx.recv().unwrap().unwrap(); + reconcile.join().unwrap(); + assert_eq!(reconciled.enabled_plugins.len(), 1); + assert_eq!(reconciled.enabled_plugins[0].plugin_id, "concurrent-enable"); +} + +#[test] +fn lifecycle_helper_invariants_report_context_and_environment_failures() { + let temp = tempdir().unwrap(); + let manifest_path = write_native_plugin(temp.path(), "helper-invariants"); + let (manifest, manifest_ref) = load_bounded_dynamic_plugin_manifest(&manifest_path).unwrap(); + let record_without_manifest = manifest.clone().into_record(None).unwrap(); + let declaration = ResolvedDynamicPluginConfig { + plugin_id: "helper-invariants".into(), + manifest_ref: manifest_ref.clone(), + config: serde_json::Map::new(), + has_explicit_config: false, + source: temp.path().join("plugins.toml"), + }; + let error = reconciled_enabled_plugin(&declaration, &record_without_manifest).unwrap_err(); + assert!(error.to_string().contains("has no manifest_ref")); + + assert!( + environment_last_error( + "helper-invariants", + DynamicPluginCheckState::Valid, + None, + None, + ) + .is_none() + ); + let missing = environment_last_error( + "helper-invariants", + DynamicPluginCheckState::Invalid, + None, + None, + ) + .unwrap(); + assert!( + missing + .message + .contains("has no lifecycle-managed Python environment") + ); + let unavailable = environment_last_error( + "helper-invariants", + DynamicPluginCheckState::Invalid, + Some("/managed/environment"), + None, + ) + .unwrap(); + assert!(unavailable.message.contains("is unavailable")); + let detailed = environment_last_error( + "helper-invariants", + DynamicPluginCheckState::Invalid, + Some("/managed/environment"), + Some("attestation mismatch".into()), + ) + .unwrap(); + assert_eq!(detailed.message, "attestation mismatch"); + + let state_path = temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME); + let lock = lock_lifecycle_state(&state_path).unwrap(); + let mut source = SourceRegistry { + state_path, + registry: DynamicPluginLifecycleState::default(), + _lock: lock, + }; + let policy = evaluate_dynamic_plugin_host_policy(&Default::default(), &manifest); + let trust = evaluate_dynamic_plugin_trust(&manifest, &manifest_ref, &policy); + let error = refresh_registry_record( + &mut source, + "missing", + manifest.clone(), + manifest_ref, + &policy, + &trust, + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("disappeared during lifecycle reconciliation") + ); + + let live = manifest.into_record(Some("/manifest.toml".into())).unwrap(); + source.registry.add(live.clone()).unwrap(); + let error = + revive_registry_record_for_new_owner(&mut source, "helper-invariants", live).unwrap_err(); + assert!( + error + .to_string() + .contains("refusing to replace live lifecycle state") + ); +} diff --git a/crates/plugin-host-config/tests/unit/policy.rs b/crates/plugin-host-config/tests/unit/policy.rs new file mode 100644 index 000000000..f66c20a9e --- /dev/null +++ b/crates/plugin-host-config/tests/unit/policy.rs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use nemo_relay::plugin::dynamic::{DynamicPluginKind, DynamicPluginManifest}; + +use super::*; + +#[test] +fn policy_rules_require_every_configured_selector_to_match() { + let manifest = DynamicPluginManifest::parse_toml( + r#"manifest_version = 1 +[plugin] +id = "fixture" +kind = "rust_dynamic" +[compat] +relay = ">=0.5,<1.0" +native_api = "v1" +[capabilities] +items = ["plugin_native"] +[defaults] +enabled = false +[load] +library = "plugin.so" +symbol = "nemo_relay_plugin_entrypoint_v1" +"#, + ) + .unwrap(); + let effect = DynamicPluginHostPolicyEffect { + allowed: Some(false), + ..Default::default() + }; + + let wrong_kind = DynamicPluginHostPolicyRule { + match_kind: Some(DynamicPluginKind::Worker), + effect: effect.clone(), + ..Default::default() + }; + assert!(!policy_rule_matches(&wrong_kind, &manifest)); + + let wrong_id = DynamicPluginHostPolicyRule { + match_plugin_id: Some("other".into()), + effect: effect.clone(), + ..Default::default() + }; + assert!(!policy_rule_matches(&wrong_id, &manifest)); + + let matching = DynamicPluginHostPolicyRule { + match_kind: Some(DynamicPluginKind::RustDynamic), + match_plugin_id: Some("fixture".into()), + effect, + }; + assert!(policy_rule_matches(&matching, &manifest)); +} diff --git a/crates/plugin-host-config/tests/unit/resolver.rs b/crates/plugin-host-config/tests/unit/resolver.rs new file mode 100644 index 000000000..0fba8b8ee --- /dev/null +++ b/crates/plugin-host-config/tests/unit/resolver.rs @@ -0,0 +1,572 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::fs; + +use nemo_relay::plugin::{PluginComponentSpec, UnsupportedBehavior}; +use tempfile::tempdir; + +use super::*; + +fn write_manifest(root: &Path, id: &str) -> PathBuf { + let plugin = root.join(id); + fs::create_dir_all(&plugin).unwrap(); + let artifact = plugin.join("libplugin.so"); + fs::write(&artifact, b"artifact").unwrap(); + let manifest = plugin.join("relay-plugin.toml"); + fs::write( + &manifest, + format!( + r#"manifest_version = 1 +[plugin] +id = "{id}" +kind = "rust_dynamic" +[compat] +relay = ">=0.5,<1.0" +native_api = "v1" +[capabilities] +items = ["plugin_native"] +[defaults] +enabled = false +[load] +library = "libplugin.so" +symbol = "nemo_relay_plugin_entrypoint_v1" +[source] +artifact = "libplugin.so" +[integrity] +sha256 = "sha256:placeholder" +"# + ), + ) + .unwrap(); + manifest +} + +#[test] +fn selected_path_replaces_only_user_layer() { + let temp = tempdir().unwrap(); + let selected = temp.path().join("selected/plugins.toml"); + let project = temp.path().join("project/.nemo-relay/plugins.toml"); + let system = temp.path().join("system/plugins.toml"); + fs::create_dir_all(selected.parent().unwrap()).unwrap(); + fs::create_dir_all(project.parent().unwrap()).unwrap(); + fs::create_dir_all(system.parent().unwrap()).unwrap(); + fs::write(&selected, "version = 1\n").unwrap(); + fs::write(&project, "version = 1\n").unwrap(); + fs::write(&system, "version = 1\n").unwrap(); + let options = PluginFileResolveOptions { + plugin_config_path: Some(selected.clone()), + current_dir: Some(temp.path().join("project")), + user_config_dir: Some(temp.path().join("ignored-user")), + system_config_path: system.clone(), + }; + assert_eq!(options.selected_paths(), vec![selected, project, system]); +} + +#[test] +fn dynamic_manifest_is_source_relative_and_duplicates_are_fatal() { + let temp = tempdir().unwrap(); + let first_manifest = write_manifest(temp.path(), "same"); + let first = temp.path().join("one/plugins.toml"); + let second = temp.path().join("two/plugins.toml"); + fs::create_dir_all(first.parent().unwrap()).unwrap(); + fs::create_dir_all(second.parent().unwrap()).unwrap(); + fs::write( + &first, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + first_manifest.to_string_lossy() + ), + ) + .unwrap(); + fs::write( + &second, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + first_manifest.to_string_lossy() + ), + ) + .unwrap(); + let error = resolve_plugin_files_from_paths([first, second], None).unwrap_err(); + assert!( + error + .to_string() + .contains("duplicate dynamic plugin id 'same'") + ); +} + +#[test] +fn missing_declared_manifest_is_not_silently_ignored() { + let temp = tempdir().unwrap(); + let config = temp.path().join("plugins.toml"); + fs::write( + &config, + "[[plugins.dynamic]]\nmanifest = \"missing/relay-plugin.toml\"\n", + ) + .unwrap(); + let error = resolve_plugin_files_from_paths([config], None).unwrap_err(); + assert!(matches!(error, PluginHostConfigError::NotFound { .. })); +} + +#[test] +fn missing_selected_layer_falls_through_to_project_and_system() { + let temp = tempdir().unwrap(); + let project_root = temp.path().join("project"); + let project = project_root.join(".nemo-relay/plugins.toml"); + let system = temp.path().join("system/plugins.toml"); + fs::create_dir_all(project.parent().unwrap()).unwrap(); + fs::create_dir_all(system.parent().unwrap()).unwrap(); + fs::write(&project, "version = 1\n").unwrap(); + fs::write(&system, "version = 1\n").unwrap(); + let resolved = resolve_plugin_files( + None, + PluginFileResolveOptions { + plugin_config_path: Some(temp.path().join("missing/plugins.toml")), + current_dir: Some(project_root), + user_config_dir: Some(temp.path().join("ambient-user")), + system_config_path: system.clone(), + }, + ) + .unwrap(); + assert_eq!( + resolved.contributing_sources, + vec![ + dunce::canonicalize(&project).unwrap(), + dunce::canonicalize(&system).unwrap() + ] + ); +} + +#[test] +fn user_only_selection_suppresses_project_but_retains_system() { + let temp = tempdir().unwrap(); + let selected = temp.path().join("selected/plugins.toml"); + let system = temp.path().join("system/plugins.toml"); + fs::create_dir_all(selected.parent().unwrap()).unwrap(); + fs::create_dir_all(system.parent().unwrap()).unwrap(); + fs::write(&selected, "version = 1\n").unwrap(); + fs::write(&system, "version = 1\n").unwrap(); + let resolved = resolve_plugin_files( + None, + PluginFileResolveOptions { + plugin_config_path: Some(selected.clone()), + current_dir: None, + user_config_dir: None, + system_config_path: system.clone(), + }, + ) + .unwrap(); + assert_eq!( + resolved.contributing_sources, + vec![ + dunce::canonicalize(&selected).unwrap(), + dunce::canonicalize(&system).unwrap() + ] + ); +} + +#[test] +fn static_layers_and_caller_overlay_share_core_merge_semantics() { + let temp = tempdir().unwrap(); + let lower = temp.path().join("lower.toml"); + let higher = temp.path().join("higher.toml"); + fs::write( + &lower, + r#"version = 1 +[[components]] +kind = "fixture" +[components.config] +lower = 1 +list = ["lower"] +[[components]] +kind = "disabled" +enabled = false +"#, + ) + .unwrap(); + fs::write( + &higher, + r#"version = 1 +[[components]] +kind = "fixture" +[components.config] +higher = 2 +list = ["higher"] +"#, + ) + .unwrap(); + let mut caller_component = PluginComponentSpec::new("fixture"); + caller_component + .config + .insert("caller".into(), serde_json::json!(3)); + caller_component + .config + .insert("list".into(), serde_json::json!(["caller"])); + let resolved = resolve_plugin_files_from_paths( + [lower, higher], + Some(PluginConfig { + components: vec![caller_component], + ..PluginConfig::default() + }), + ) + .unwrap(); + assert_eq!(resolved.config.components.len(), 1); + let config = &resolved.config.components[0].config; + assert_eq!(config["lower"], 1); + assert_eq!(config["higher"], 2); + assert_eq!(config["caller"], 3); + assert_eq!( + config["list"], + serde_json::json!(["caller", "higher", "lower"]) + ); +} + +#[test] +fn relative_and_absolute_manifest_references_resolve_canonically() { + let temp = tempdir().unwrap(); + let relative_manifest = write_manifest(temp.path(), "relative"); + let absolute_manifest = write_manifest(temp.path(), "absolute"); + let config = temp.path().join("plugins.toml"); + fs::write( + &config, + format!( + "[[plugins.dynamic]]\nmanifest = \"relative/relay-plugin.toml\"\n\n[[plugins.dynamic]]\nmanifest = {:?}\n", + absolute_manifest.to_string_lossy() + ), + ) + .unwrap(); + let resolved = resolve_plugin_files_from_paths([config], None).unwrap(); + assert_eq!(resolved.dynamic_plugins.len(), 2); + assert_eq!( + PathBuf::from(&resolved.dynamic_plugins[0].manifest_ref), + relative_manifest.canonicalize().unwrap() + ); + assert_eq!( + PathBuf::from(&resolved.dynamic_plugins[1].manifest_ref), + absolute_manifest.canonicalize().unwrap() + ); +} + +#[test] +fn dynamic_only_diagnostics_are_redacted() { + let temp = tempdir().unwrap(); + write_manifest(temp.path(), "redacted"); + let config = temp.path().join("plugins.toml"); + let secret = "never-report-this-token"; + fs::write( + &config, + format!( + "[[plugins.dynamic]]\nmanifest = \"redacted/relay-plugin.toml\"\nconfig = {{ api_key = \"{secret}\" }}\n" + ), + ) + .unwrap(); + let resolved = resolve_plugin_files_from_paths([config.clone()], None).unwrap(); + assert_eq!( + resolved.contributing_sources, + vec![dunce::canonicalize(&config).unwrap()] + ); + assert!( + resolved + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "plugin.configuration_inherited") + ); + assert!( + resolved + .diagnostics + .iter() + .all(|diagnostic| !diagnostic.message.contains(secret)) + ); +} + +#[cfg(unix)] +#[test] +fn physical_source_aliases_are_deduplicated_after_pinning() { + use std::os::unix::fs::symlink; + + let temp = tempdir().unwrap(); + let manifest = write_manifest(temp.path(), "aliased"); + let config = temp.path().join("plugins.toml"); + fs::write( + &config, + format!( + "[[plugins.dynamic]]\nmanifest = {:?}\n", + manifest.to_string_lossy() + ), + ) + .unwrap(); + let alias = temp.path().join("plugins-alias.toml"); + symlink(&config, &alias).unwrap(); + + let resolved = resolve_plugin_files_from_paths([alias.clone(), config.clone()], None).unwrap(); + assert_eq!( + resolved.contributing_sources, + vec![dunce::canonicalize(&config).unwrap()] + ); + assert_eq!(resolved.contributing_selected_sources, vec![config.clone()]); + assert_eq!(resolved.dynamic_plugins.len(), 1); + + let aliased = resolve_plugin_files_from_paths([config, alias.clone()], None).unwrap(); + assert_eq!(aliased.contributing_selected_sources, vec![alias]); +} + +#[test] +fn inherited_diagnostics_preserve_the_selected_source_spelling() { + let temp = tempdir().unwrap(); + let config = temp.path().join("plugins.toml"); + fs::write(&config, "").unwrap(); + let detour = temp.path().join("detour"); + fs::create_dir(&detour).unwrap(); + let selected = detour.join("..").join("plugins.toml"); + + let resolved = resolve_plugin_files_from_paths([selected.clone()], None).unwrap(); + + assert_eq!( + resolved.contributing_sources, + vec![pin_plugin_config_path(&config).unwrap()] + ); + assert_eq!( + resolved.contributing_selected_sources, + vec![selected.clone()] + ); + assert_eq!( + resolved.diagnostics[0].message, + format!( + "inherited plugin configuration from discovered file: {}", + selected.display() + ) + ); +} + +#[test] +fn malformed_toml_errors_do_not_disclose_configuration_values() { + let temp = tempdir().unwrap(); + let config = temp.path().join("plugins.toml"); + let secret = "do-not-leak-this-token"; + fs::write( + &config, + format!("[[components]]\nkind = \"fixture\"\nconfig = {{ token = \"{secret} }}\n"), + ) + .unwrap(); + let error = resolve_plugin_files_from_paths([config], None).unwrap_err(); + assert!(!error.to_string().contains(secret)); +} + +#[cfg(unix)] +#[test] +fn special_file_plugin_configuration_is_rejected_without_reading_it() { + let error = resolve_plugin_files_from_paths([PathBuf::from("/dev/zero")], None) + .expect_err("a character device must not be read as plugins.toml"); + assert!(error.to_string().contains("must be a regular file")); +} + +#[test] +fn source_selection_covers_ambient_user_and_environment_construction() { + let temp = tempdir().unwrap(); + let selected = temp.path().join("selected.toml"); + let options = PluginFileResolveOptions::from_environment(Some(selected.clone())); + assert_eq!(options.plugin_config_path, Some(selected)); + + let user = temp.path().join("user"); + let system = temp.path().join("system.toml"); + let ambient = PluginFileResolveOptions { + plugin_config_path: None, + current_dir: None, + user_config_dir: Some(user.clone()), + system_config_path: system.clone(), + }; + assert_eq!( + ambient.selected_paths(), + vec![user.join("plugins.toml"), system] + ); +} + +#[cfg(not(windows))] +#[test] +fn source_normalization_errors_are_contextualized() { + let temp = tempdir().unwrap(); + let blocking_file = temp.path().join("not-a-directory"); + fs::write(&blocking_file, "fixture").unwrap(); + + let error = + resolve_plugin_files_from_paths([blocking_file.join("plugins.toml")], None).unwrap_err(); + assert!(error.to_string().contains("plugin configuration file")); + assert!(error.to_string().contains("not-a-directory")); +} + +#[test] +fn private_resolution_helpers_cover_defensive_and_diagnostic_shapes() { + let mut seen = HashSet::new(); + let mut scalar_toml = toml::Value::String("fixture".into()); + let dynamic = + resolve_dynamic_plugin_refs(Path::new("plugins.toml"), &mut scalar_toml, &mut seen) + .unwrap(); + assert!(dynamic.dynamic_plugins.is_empty()); + + let no_components = serde_json::json!({"version": 1}); + let programmatic = PluginConfig { + components: vec![PluginComponentSpec::new("fixture")], + ..PluginConfig::default() + }; + assert!( + programmatic_enable_override_diagnostics(&no_components, &HashMap::new(), &programmatic,) + .is_empty() + ); + + let discovered = serde_json::json!({ + "components": [ + {"kind": "fixture", "enabled": false}, + {"kind": "fixture", "enabled": true} + ] + }); + let mut first = PluginComponentSpec::new("fixture"); + first.enabled = true; + let mut second = PluginComponentSpec::new("fixture"); + second.enabled = false; + let diagnostics = programmatic_enable_override_diagnostics( + &discovered, + &HashMap::new(), + &PluginConfig { + components: vec![first, second], + ..PluginConfig::default() + }, + ); + assert_eq!(diagnostics.len(), 1); + assert_eq!(diagnostics[0].code, "plugin.component_reenabled"); +} + +#[test] +fn private_merge_helpers_cover_scalar_nested_and_policy_overlays() { + let mut no_components = serde_json::json!({"version": 1}); + layer_config(&mut no_components, serde_json::json!("replacement")); + assert_eq!(no_components, serde_json::json!("replacement")); + layer_config(&mut no_components, serde_json::json!({"inserted": true})); + assert_eq!(no_components, serde_json::json!({"inserted": true})); + + let mut non_array = serde_json::json!(null); + merge_plugin_components(&mut non_array, serde_json::json!([{"kind": "fixture"}])); + assert!(non_array.is_array()); + merge_plugin_components(&mut non_array, serde_json::json!("not-an-array")); + assert_eq!(non_array, serde_json::json!("not-an-array")); + + let mut observability = serde_json::json!({ + "kind": "observability", + "config": { + "atof": {"sinks": ["low"]}, + "opentelemetry": {"endpoints": ["low"]}, + "atif": {"storage": ["low"]}, + "ordinary": {"values": ["low"]} + } + }); + merge_plugin_component( + &mut observability, + serde_json::json!({ + "kind": "observability", + "config": { + "atof": {"sinks": ["high"]}, + "opentelemetry": {"endpoints": ["high"]}, + "atif": {"storage": ["high"]}, + "ordinary": {"values": ["high"]} + } + }), + ); + assert_eq!( + observability["config"]["atof"]["sinks"], + serde_json::json!(["high", "low"]) + ); + assert_eq!( + observability["config"]["opentelemetry"]["endpoints"], + serde_json::json!(["high", "low"]) + ); + assert_eq!( + observability["config"]["atif"]["storage"], + serde_json::json!(["high", "low"]) + ); + assert_eq!( + observability["config"]["ordinary"]["values"], + serde_json::json!(["high"]) + ); + + let mut generic = serde_json::json!({"kind": "fixture", "config": {"list": ["low"]}}); + merge_plugin_component( + &mut generic, + serde_json::json!({"kind": "fixture", "config": {"list": ["high"]}}), + ); + assert_eq!( + generic["config"]["list"], + serde_json::json!(["high", "low"]) + ); + + let mut policy_root = serde_json::Map::new(); + remove_default_policy_overlay(&mut policy_root, &ConfigPolicy::default()); + let custom = ConfigPolicy { + unknown_component: UnsupportedBehavior::Error, + ..ConfigPolicy::default() + }; + policy_root.insert("policy".into(), serde_json::to_value(custom).unwrap()); + remove_default_policy_overlay(&mut policy_root, &custom); + assert_eq!( + policy_root["policy"]["unknown_component"], + serde_json::json!("error") + ); + assert!(policy_root["policy"].get("unknown_field").is_none()); + assert!(policy_root["policy"].get("unsupported_value").is_none()); +} + +#[test] +fn private_resolution_helpers_cover_invalid_config_and_component_sources() { + let invalid = finish_resolution( + serde_json::json!("not-a-plugin-config"), + Vec::new(), + DynamicPluginHostPolicy::default(), + Vec::new(), + ResolvedPluginFileSourcePaths { + contributing_sources: Vec::new(), + contributing_selected_sources: Vec::new(), + selected_sources: Vec::new(), + }, + true, + ) + .unwrap_err(); + assert!( + invalid + .to_string() + .contains("resolved static plugin configuration is invalid") + ); + + let mut enabled_sources = HashMap::new(); + record_enabled_sources( + Path::new("plugins.toml"), + &serde_json::json!({ + "components": [ + {"enabled": false}, + {"kind": "fixture", "enabled": "not-a-boolean"}, + {"kind": "recorded", "enabled": false} + ] + }), + &mut enabled_sources, + ); + assert_eq!(enabled_sources.len(), 1); + assert!(!enabled_sources["recorded"].enabled); + + let mut layered = serde_json::json!({"nested": {"low": true}}); + layer_config( + &mut layered, + serde_json::json!({"nested": {"high": true}, "inserted": 3}), + ); + assert_eq!( + layered, + serde_json::json!({"nested": {"low": true, "high": true}, "inserted": 3}) + ); + + let mut components = serde_json::json!([{"kind": "existing"}]); + merge_plugin_components( + &mut components, + serde_json::json!([{"value": "no-kind"}, {"kind": "new"}]), + ); + assert_eq!(components.as_array().unwrap().len(), 3); + + let mut scalar_component = serde_json::json!("low"); + merge_plugin_component(&mut scalar_component, serde_json::json!("high")); + assert_eq!(scalar_component, serde_json::json!("high")); +} diff --git a/crates/plugin-host-config/tests/unit/snapshot.rs b/crates/plugin-host-config/tests/unit/snapshot.rs new file mode 100644 index 000000000..e7c39ca12 --- /dev/null +++ b/crates/plugin-host-config/tests/unit/snapshot.rs @@ -0,0 +1,835 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; + +use tempfile::tempdir; + +use super::*; + +#[cfg(target_os = "macos")] +#[test] +fn macos_snapshot_copies_only_the_authenticated_venvs_matching_runtime_library() { + let temp = tempdir().unwrap(); + let mutable_base = temp.path().join("mutable-base"); + let authenticated_base = temp.path().join("authenticated-base"); + fs::create_dir_all(mutable_base.join("bin")).unwrap(); + fs::create_dir_all(mutable_base.join("lib")).unwrap(); + fs::create_dir_all(authenticated_base.join("bin")).unwrap(); + fs::create_dir_all(authenticated_base.join("lib")).unwrap(); + fs::write( + mutable_base.join("lib/libpython3.11.dylib"), + b"mutable runtime", + ) + .unwrap(); + fs::write( + authenticated_base.join("lib/libpython3.11.dylib"), + b"authenticated runtime", + ) + .unwrap(); + fs::write( + authenticated_base.join("lib/libpython3.12.dylib"), + b"wrong version", + ) + .unwrap(); + fs::write( + authenticated_base.join("lib/libpython3.11.dylib.backup"), + b"backup", + ) + .unwrap(); + + let mutable_environment = temp.path().join("mutable-environment"); + fs::create_dir(&mutable_environment).unwrap(); + fs::write( + mutable_environment.join("pyvenv.cfg"), + format!( + "home = {}\nversion_info = 3.11.14\n", + mutable_base.join("bin").display() + ), + ) + .unwrap(); + let copied_environment = temp.path().join("snapshot-environment"); + fs::create_dir(&copied_environment).unwrap(); + fs::write( + copied_environment.join("pyvenv.cfg"), + format!( + "home = {}\nversion_info = 3.11.14\n", + authenticated_base.join("bin").display() + ), + ) + .unwrap(); + let mut copied_files = HashMap::new(); + let mut budget = SnapshotBudget::default(); + + snapshot_macos_python_runtime_library(&copied_environment, &mut copied_files, &mut budget) + .unwrap(); + + assert_eq!( + fs::read(copied_environment.join("lib/libpython3.11.dylib")).unwrap(), + b"authenticated runtime" + ); + assert!(!copied_environment.join("lib/libpython3.12.dylib").exists()); + assert!( + !copied_environment + .join("lib/libpython3.11.dylib.backup") + .exists() + ); + assert_eq!(budget.entries, 1); + assert_eq!(budget.bytes, b"authenticated runtime".len() as u64); +} + +#[cfg(target_os = "macos")] +#[test] +fn macos_snapshot_preserves_an_attested_runtime_library_and_avoids_empty_directories() { + let temp = tempdir().unwrap(); + let base = temp.path().join("base"); + fs::create_dir_all(base.join("bin")).unwrap(); + fs::create_dir_all(base.join("lib")).unwrap(); + fs::write(base.join("lib/libpython3.11.dylib"), b"base runtime").unwrap(); + let copied_environment = temp.path().join("snapshot-environment"); + fs::create_dir_all(copied_environment.join("lib")).unwrap(); + fs::write( + copied_environment.join("pyvenv.cfg"), + format!("home = {}\nversion = 3.11.14\n", base.join("bin").display()), + ) + .unwrap(); + let destination = copied_environment.join("lib/libpython3.11.dylib"); + fs::write(&destination, b"attested runtime").unwrap(); + let mut budget = SnapshotBudget::default(); + + snapshot_macos_python_runtime_library(&copied_environment, &mut HashMap::new(), &mut budget) + .unwrap(); + + assert_eq!(fs::read(destination).unwrap(), b"attested runtime"); + assert_eq!(budget.entries, 0); + assert_eq!(budget.bytes, 0); + + let static_base = temp.path().join("static-base"); + fs::create_dir_all(static_base.join("bin")).unwrap(); + let static_environment = temp.path().join("static-environment"); + fs::create_dir(&static_environment).unwrap(); + fs::write( + static_environment.join("pyvenv.cfg"), + format!( + "home = {}\nversion_info = 3.12.0\n", + static_base.join("bin").display() + ), + ) + .unwrap(); + snapshot_macos_python_runtime_library( + &static_environment, + &mut HashMap::new(), + &mut SnapshotBudget::default(), + ) + .unwrap(); + assert!(!static_environment.join("lib").exists()); + + let standalone_environment = temp.path().join("standalone-environment"); + fs::create_dir(&standalone_environment).unwrap(); + snapshot_macos_python_runtime_library( + &standalone_environment, + &mut HashMap::new(), + &mut SnapshotBudget::default(), + ) + .unwrap(); + assert!(!standalone_environment.join("lib").exists()); +} + +#[cfg(unix)] +#[test] +fn snapshot_directory_copy_materializes_worker_launcher_and_preserves_versioned_aliases() { + use std::os::unix::fs::symlink; + + let temp = tempdir().unwrap(); + let source = temp.path().join("environment"); + let bin = source.join("bin"); + let lib = source.join("lib"); + fs::create_dir_all(&bin).unwrap(); + fs::create_dir_all(&lib).unwrap(); + let interpreter = temp.path().join("managed-python"); + fs::write(&interpreter, b"python").unwrap(); + symlink(&interpreter, bin.join("python")).unwrap(); + symlink(&interpreter, bin.join("python3.11")).unwrap(); + symlink(&interpreter, bin.join("pip")).unwrap(); + symlink(&interpreter, lib.join("python3.11")).unwrap(); + let destination = temp.path().join("snapshot"); + + copy_snapshot_directory( + &source, + &destination, + &mut HashMap::new(), + &mut SnapshotBudget::default(), + false, + &mut Vec::new(), + ) + .unwrap(); + + assert!( + fs::symlink_metadata(destination.join("bin/python")) + .unwrap() + .is_file() + ); + assert_eq!(fs::read(destination.join("bin/python")).unwrap(), b"python"); + assert!( + fs::symlink_metadata(destination.join("bin/python3.11")) + .unwrap() + .file_type() + .is_symlink() + ); + assert!( + fs::symlink_metadata(destination.join("bin/pip")) + .unwrap() + .is_file() + ); + assert!( + fs::symlink_metadata(destination.join("lib/python3.11")) + .unwrap() + .is_file() + ); +} + +#[cfg(unix)] +#[test] +fn snapshot_worker_launcher_isolated_from_post_copy_symlink_target_substitution() { + use std::os::unix::fs::{PermissionsExt, symlink}; + use std::process::Command; + + let temp = tempdir().unwrap(); + let source = temp.path().join("environment"); + let bin = source.join("bin"); + fs::create_dir_all(&bin).unwrap(); + let interpreter = temp.path().join("external-python"); + fs::write(&interpreter, b"#!/bin/sh\nprintf original-interpreter").unwrap(); + fs::set_permissions(&interpreter, fs::Permissions::from_mode(0o755)).unwrap(); + symlink(&interpreter, bin.join("python")).unwrap(); + let destination = temp.path().join("snapshot"); + + copy_snapshot_directory( + &source, + &destination, + &mut HashMap::new(), + &mut SnapshotBudget::default(), + true, + &mut Vec::new(), + ) + .unwrap(); + let launcher = destination.join("bin/python"); + let before = snapshot_tree_digest(&destination, false).unwrap(); + + let replacement = temp.path().join("replacement-python"); + fs::write(&replacement, b"#!/bin/sh\nprintf substituted-interpreter").unwrap(); + fs::set_permissions(&replacement, fs::Permissions::from_mode(0o755)).unwrap(); + fs::rename(replacement, &interpreter).unwrap(); + + assert_eq!(before, snapshot_tree_digest(&destination, false).unwrap()); + assert!(fs::symlink_metadata(&launcher).unwrap().is_file()); + let output = Command::new(&launcher).output().unwrap(); + assert!(output.status.success()); + assert_eq!(output.stdout, b"original-interpreter"); +} + +#[cfg(unix)] +#[test] +fn materialized_worker_launcher_preserves_python_virtual_environment() { + use std::process::Command; + + let temp = tempdir().unwrap(); + let source = temp.path().join("source-venv"); + let status = Command::new("python3") + .args(["-m", "venv", "--without-pip"]) + .arg(&source) + .status() + .expect("Python 3 is required to test virtual-environment snapshots"); + assert!( + status.success(), + "Python virtual environment creation failed" + ); + let destination = temp.path().join("snapshot-venv"); + + copy_snapshot_directory( + &source, + &destination, + &mut HashMap::new(), + &mut SnapshotBudget::default(), + true, + &mut Vec::new(), + ) + .unwrap(); + + let launcher = destination.join("bin/python"); + assert!(fs::symlink_metadata(&launcher).unwrap().is_file()); + let output = Command::new(&launcher) + .args(["-c", "import sys; print(sys.prefix)"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "snapshotted virtual environment failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let observed_prefix = PathBuf::from(String::from_utf8(output.stdout).unwrap().trim()); + assert_eq!( + fs::canonicalize(observed_prefix).unwrap(), + fs::canonicalize(destination).unwrap() + ); +} + +#[cfg(unix)] +#[test] +fn snapshot_protection_does_not_follow_python_launcher_symlink() { + use std::os::unix::fs::{PermissionsExt, symlink}; + + let temp = tempdir().unwrap(); + let target = temp.path().join("external-python"); + fs::write(&target, b"python").unwrap(); + fs::set_permissions(&target, fs::Permissions::from_mode(0o755)).unwrap(); + let root = temp.path().join("snapshot"); + let bin = root.join("bin"); + fs::create_dir_all(&bin).unwrap(); + symlink(&target, bin.join("python")).unwrap(); + + protect_snapshot_tree(&root).unwrap(); + + assert!( + fs::symlink_metadata(bin.join("python")) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + fs::metadata(&target).unwrap().permissions().mode() & 0o777, + 0o755 + ); + make_snapshot_removable(&root); +} + +#[cfg(unix)] +#[test] +fn snapshot_digest_hashes_python_launcher_symlink_without_following_it() { + use std::os::unix::fs::symlink; + + let temp = tempdir().unwrap(); + let root = temp.path().join("snapshot"); + let bin = root + .join(MANAGED_ENVIRONMENTS_DIR) + .join("environment") + .join("bin"); + fs::create_dir_all(&bin).unwrap(); + let launcher = bin.join("python"); + symlink("/missing/python-a", &launcher).unwrap(); + + let first_verification = snapshot_tree_digest(&root, false).unwrap(); + let first_identity = snapshot_tree_digest(&root, true).unwrap(); + + fs::remove_file(&launcher).unwrap(); + fs::write(&launcher, b"/missing/python-a").unwrap(); + assert_ne!( + first_verification, + snapshot_tree_digest(&root, false).unwrap(), + "a regular file must not collide with an equivalent symlink target" + ); + + fs::remove_file(&launcher).unwrap(); + symlink("/missing/python-b", &launcher).unwrap(); + + assert_ne!( + first_verification, + snapshot_tree_digest(&root, false).unwrap(), + "verification must include the exact launcher target" + ); + assert_eq!( + first_identity, + snapshot_tree_digest(&root, true).unwrap(), + "managed environment contents are excluded from stable gateway identity" + ); +} + +#[cfg(unix)] +#[test] +fn snapshot_directory_copy_rejects_special_entries_and_write_failures() { + use std::ffi::CString; + + let temp = tempdir().unwrap(); + let fifo_source = temp.path().join("fifo-source"); + fs::create_dir(&fifo_source).unwrap(); + let fifo = fifo_source.join("worker.pipe"); + let fifo_c = CString::new(fifo.as_os_str().as_encoded_bytes()).unwrap(); + // SAFETY: `fifo_c` is a valid NUL-terminated path and the mode contains only permission bits. + assert_eq!(unsafe { libc::mkfifo(fifo_c.as_ptr(), 0o600) }, 0); + + let special_error = copy_snapshot_directory( + &fifo_source, + &temp.path().join("fifo-snapshot"), + &mut HashMap::new(), + &mut SnapshotBudget::default(), + false, + &mut Vec::new(), + ) + .unwrap_err() + .to_string(); + assert!( + special_error.contains("regular file or directory"), + "{special_error}" + ); + + let regular = temp.path().join("regular"); + fs::write(®ular, b"regular").unwrap(); + let destination_directory = temp.path().join("destination-directory"); + fs::create_dir(&destination_directory).unwrap(); + let write_error = copy_snapshot_regular_file( + ®ular, + &destination_directory, + &mut HashMap::new(), + &mut SnapshotBudget::default(), + ) + .unwrap_err() + .to_string(); + assert!(write_error.contains("write snapshot file"), "{write_error}"); +} + +#[cfg(unix)] +#[test] +fn snapshot_directory_walk_rejects_missing_cycles_dangling_links_and_depth() { + use std::os::unix::fs::symlink; + + let temp = tempdir().unwrap(); + let missing = temp.path().join("missing"); + let normalization_error = copy_snapshot_directory( + &missing, + &temp.path().join("destination"), + &mut HashMap::new(), + &mut SnapshotBudget::default(), + false, + &mut Vec::new(), + ) + .unwrap_err() + .to_string(); + assert!( + normalization_error.contains("normalize runtime directory"), + "{normalization_error}" + ); + + let source = temp.path().join("source"); + fs::create_dir(&source).unwrap(); + let canonical = source.canonicalize().unwrap(); + let cycle_error = copy_snapshot_directory_contents( + &source, + &temp.path().join("cycle-destination"), + &mut HashMap::new(), + &mut SnapshotBudget::default(), + false, + &mut vec![canonical], + ) + .unwrap_err() + .to_string(); + assert!(cycle_error.contains("symlink cycle"), "{cycle_error}"); + + let destination_file = temp.path().join("destination-file"); + fs::write(&destination_file, b"file").unwrap(); + let destination_error = copy_snapshot_directory_contents( + &source, + &destination_file, + &mut HashMap::new(), + &mut SnapshotBudget::default(), + false, + &mut Vec::new(), + ) + .unwrap_err() + .to_string(); + assert!( + destination_error.contains("create snapshot directory"), + "{destination_error}" + ); + + symlink(temp.path().join("absent-target"), source.join("dangling")).unwrap(); + let dangling_error = copy_snapshot_directory( + &source, + &temp.path().join("dangling-destination"), + &mut HashMap::new(), + &mut SnapshotBudget::default(), + false, + &mut Vec::new(), + ) + .unwrap_err() + .to_string(); + assert!( + dangling_error.contains("resolve runtime symlink"), + "{dangling_error}" + ); + + let depth_error = collect_snapshot_files( + &source, + &source, + &mut Vec::new(), + MAX_SNAPSHOT_DEPTH, + &mut 0, + ) + .unwrap_err() + .to_string(); + assert!(depth_error.contains("traversal depth"), "{depth_error}"); +} + +#[test] +fn snapshot_file_and_verification_helpers_cover_external_and_invalid_sources() { + let temp = tempdir().unwrap(); + let plugin_dir = temp.path().join("plugin"); + fs::create_dir(&plugin_dir).unwrap(); + let manifest = plugin_dir.join("relay-plugin.toml"); + fs::write(&manifest, b"fixture").unwrap(); + let root = temp.path().join("snapshot"); + fs::create_dir(&root).unwrap(); + + let missing_error = copy_snapshot_file( + &root, + &manifest, + "missing.bin", + "artifact", + &mut HashMap::new(), + &mut SnapshotBudget::default(), + ) + .unwrap_err() + .to_string(); + assert!( + missing_error.contains("normalize dynamic plugin artifact"), + "{missing_error}" + ); + + let root_error = copy_snapshot_file( + &root, + &manifest, + "/", + "library", + &mut HashMap::new(), + &mut SnapshotBudget::default(), + ) + .unwrap_err() + .to_string(); + assert!( + root_error.contains("has no parent directory"), + "{root_error}" + ); + + let external = temp.path().join("external-artifact.bin"); + fs::write(&external, b"external artifact").unwrap(); + let (logical, canonical, copied) = copy_snapshot_file( + &root, + &manifest, + external.to_string_lossy().as_ref(), + "artifact", + &mut HashMap::new(), + &mut SnapshotBudget::default(), + ) + .unwrap(); + assert_eq!(logical, external); + assert_eq!(canonical, external.canonicalize().unwrap()); + assert_eq!(fs::read(copied).unwrap(), b"external artifact"); + + let one_file = temp.path().join("one-file"); + fs::create_dir(&one_file).unwrap(); + fs::write(one_file.join("entry"), b"entry").unwrap(); + let mut entries = MAX_SNAPSHOT_FILES; + let entry_error = + collect_snapshot_files(&one_file, &one_file, &mut Vec::new(), 0, &mut entries) + .unwrap_err() + .to_string(); + assert!(entry_error.contains("verification budget"), "{entry_error}"); + + make_snapshot_removable(&temp.path().join("already-removed")); +} + +#[test] +fn snapshot_budget_rejects_entry_and_byte_overflow() { + let path = Path::new("fixture"); + let mut entry_budget = SnapshotBudget { + entries: MAX_SNAPSHOT_FILES, + ..SnapshotBudget::default() + }; + let entry_error = entry_budget.record_entry(path).unwrap_err().to_string(); + assert!(entry_error.contains("entry activation snapshot budget")); + + let mut byte_budget = SnapshotBudget::default(); + let byte_error = byte_budget + .record_bytes(path, usize::try_from(MAX_BOUNDED_FILE_BYTES).unwrap() + 1) + .unwrap_err() + .to_string(); + assert!(byte_error.contains("byte activation snapshot budget")); +} + +#[test] +fn snapshot_helpers_reject_missing_python_integrity_and_directory_depth() { + let temp = tempdir().unwrap(); + let manifest = DynamicPluginManifest::parse_toml( + r#"manifest_version = 1 +[plugin] +id = "python-without-integrity" +kind = "worker" +[compat] +relay = ">=0.5,<1.0" +worker_protocol = "grpc-v1" +[capabilities] +items = ["plugin_worker"] +[defaults] +enabled = false +[source] +manifest_root = "." +artifact = "plugin.py" +[load] +runtime = "python" +entrypoint = "plugin:main" +"#, + ) + .unwrap(); + let error = trusted_source_artifact_sha256(&manifest).unwrap_err(); + assert!(error.to_string().contains("requires integrity.sha256")); + + let source = temp.path().join("source"); + fs::create_dir(&source).unwrap(); + let error = copy_snapshot_directory_contents( + &source, + &temp.path().join("destination"), + &mut HashMap::new(), + &mut SnapshotBudget::default(), + false, + &mut vec![PathBuf::new(); MAX_SNAPSHOT_DEPTH], + ) + .unwrap_err(); + assert!(error.to_string().contains("traversal depth")); + + fs::write(source.join("keep.py"), b"keep").unwrap(); + fs::write(source.join("cached.pyc"), b"ignored").unwrap(); + fs::create_dir(source.join("__pycache__")).unwrap(); + let destination = temp.path().join("cache-filtered"); + copy_snapshot_directory( + &source, + &destination, + &mut HashMap::new(), + &mut SnapshotBudget::default(), + true, + &mut Vec::new(), + ) + .unwrap(); + assert!(destination.join("keep.py").exists()); + assert!(!destination.join("cached.pyc").exists()); + assert!(!destination.join("__pycache__").exists()); +} + +#[cfg(unix)] +#[test] +fn python_launcher_preservation_reports_each_filesystem_failure_context() { + use std::os::unix::fs::symlink; + + let temp = tempdir().unwrap(); + let source_bin = temp.path().join("environment/bin"); + fs::create_dir_all(&source_bin).unwrap(); + let target = temp.path().join("python"); + fs::write(&target, b"python").unwrap(); + let source = source_bin.join("python3.11"); + symlink(&target, &source).unwrap(); + let metadata = fs::symlink_metadata(&source).unwrap(); + let resolved = source.canonicalize().unwrap(); + + fs::remove_file(&source).unwrap(); + let error = preserve_python_launcher( + &source, + &temp.path().join("snapshot/bin/python3.11"), + &resolved, + &metadata, + &mut HashMap::new(), + ) + .unwrap_err(); + assert!(error.to_string().contains("read Python launcher symlink")); + + symlink(&target, &source).unwrap(); + let blocked_parent = temp.path().join("blocked-parent"); + fs::write(&blocked_parent, b"file").unwrap(); + let error = preserve_python_launcher( + &source, + &blocked_parent.join("python3.11"), + &resolved, + &metadata, + &mut HashMap::new(), + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("create Python launcher directory") + ); + + let destination = temp.path().join("existing/python3.11"); + fs::create_dir_all(destination.parent().unwrap()).unwrap(); + fs::write(&destination, b"existing").unwrap(); + let error = preserve_python_launcher( + &source, + &destination, + &resolved, + &metadata, + &mut HashMap::new(), + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("preserve Python launcher symlink") + ); +} + +#[test] +fn required_snapshot_policy_and_signature_failures_stop_before_activation() { + use nemo_relay::plugin::dynamic::{DynamicPluginAttestationMode, DynamicPluginStartupClass}; + + let temp = tempdir().unwrap(); + let artifact = b"native fixture"; + fs::write(temp.path().join("plugin.so"), artifact).unwrap(); + let digest = Sha256::digest(artifact) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let manifest = temp.path().join("relay-plugin.toml"); + fs::write( + &manifest, + format!( + r#"manifest_version = 1 +[plugin] +id = "required-native" +kind = "rust_dynamic" +[compat] +relay = ">=0.5,<1.0" +native_api = "v1" +[capabilities] +items = ["plugin_native"] +[defaults] +enabled = false +[load] +library = "plugin.so" +symbol = "nemo_relay_plugin_entrypoint_v1" +[source] +artifact = "plugin.so" +[integrity] +sha256 = "sha256:{digest}" +"#, + ), + ) + .unwrap(); + + let blocked = DynamicPluginHostPolicy { + defaults: crate::policy::DynamicPluginHostPolicyEffect { + allowed: Some(false), + startup: Some(DynamicPluginStartupClass::Required), + ..Default::default() + }, + ..Default::default() + }; + let error = DynamicPluginActivationSnapshot::create( + manifest.to_string_lossy().as_ref(), + "required-native", + DynamicPluginKind::RustDynamic, + None, + &blocked, + ) + .unwrap_err(); + assert!(error.to_string().contains("violates host policy")); + + let signature_required = DynamicPluginHostPolicy { + defaults: crate::policy::DynamicPluginHostPolicyEffect { + startup: Some(DynamicPluginStartupClass::Required), + attestation: Some(DynamicPluginAttestationMode::SignatureRequired), + ..Default::default() + }, + ..Default::default() + }; + let error = DynamicPluginActivationSnapshot::create( + manifest.to_string_lossy().as_ref(), + "required-native", + DynamicPluginKind::RustDynamic, + None, + &signature_required, + ) + .unwrap_err(); + assert!(error.to_string().contains("signature")); +} + +#[cfg(target_os = "macos")] +#[test] +fn macos_runtime_library_snapshot_rejects_malformed_and_unsafe_runtime_layouts() { + use std::os::unix::fs::symlink; + + let temp = tempdir().unwrap(); + for (name, config) in [ + ("missing-home", "version = 3.11.0\n".to_owned()), + ("missing-version", "home = /tmp/bin\n".to_owned()), + ( + "invalid-major", + "home = /tmp/bin\nversion = x.11\n".to_owned(), + ), + ( + "invalid-minor", + "home = /tmp/bin\nversion = 3.x\n".to_owned(), + ), + ("root-home", "home = /\nversion = 3.11\n".to_owned()), + ] { + let environment = temp.path().join(name); + fs::create_dir(&environment).unwrap(); + fs::write(environment.join("pyvenv.cfg"), config).unwrap(); + snapshot_macos_python_runtime_library( + &environment, + &mut HashMap::new(), + &mut SnapshotBudget::default(), + ) + .unwrap(); + assert!(!environment.join("lib").exists()); + } + + let base = temp.path().join("base"); + fs::create_dir_all(base.join("bin")).unwrap(); + fs::create_dir_all(base.join("lib/libpython3.11.dylib")).unwrap(); + let environment = temp.path().join("directory-source"); + fs::create_dir(&environment).unwrap(); + fs::write( + environment.join("pyvenv.cfg"), + format!("home = {}\nversion = 3.11\n", base.join("bin").display()), + ) + .unwrap(); + assert!( + snapshot_macos_python_runtime_library( + &environment, + &mut HashMap::new(), + &mut SnapshotBudget::default(), + ) + .unwrap_err() + .to_string() + .contains("must resolve to a regular file") + ); + + fs::remove_dir_all(base.join("lib/libpython3.11.dylib")).unwrap(); + symlink( + base.join("lib/missing.dylib"), + base.join("lib/libpython3.11.dylib"), + ) + .unwrap(); + assert!( + snapshot_macos_python_runtime_library( + &environment, + &mut HashMap::new(), + &mut SnapshotBudget::default(), + ) + .unwrap_err() + .to_string() + .contains("normalize Python runtime library") + ); + + fs::remove_file(base.join("lib/libpython3.11.dylib")).unwrap(); + fs::write(base.join("lib/libpython3.11.dylib"), b"runtime").unwrap(); + let destination = environment.join("lib/libpython3.11.dylib"); + fs::create_dir_all(&destination).unwrap(); + assert!( + snapshot_macos_python_runtime_library( + &environment, + &mut HashMap::new(), + &mut SnapshotBudget::default(), + ) + .unwrap_err() + .to_string() + .contains("must be a regular file") + ); +} diff --git a/crates/plugin-host-config/tests/unit/state.rs b/crates/plugin-host-config/tests/unit/state.rs new file mode 100644 index 000000000..e63505451 --- /dev/null +++ b/crates/plugin-host-config/tests/unit/state.rs @@ -0,0 +1,386 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::{Mutex, OnceLock}; + +use tempfile::tempdir; + +use nemo_relay::plugin::dynamic::{DynamicPluginManifest, DynamicPluginRegistry}; + +use super::*; + +fn cwd_lock() -> std::sync::MutexGuard<'static, ()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|error| error.into_inner()) +} + +struct CurrentDirectoryGuard(PathBuf); + +impl CurrentDirectoryGuard { + fn enter(path: &Path) -> Self { + let original = std::env::current_dir().unwrap(); + std::env::set_current_dir(path).unwrap(); + Self(original) + } +} + +impl Drop for CurrentDirectoryGuard { + fn drop(&mut self) { + std::env::set_current_dir(&self.0).unwrap(); + } +} + +#[test] +fn relative_existing_source_is_pinned_across_cwd_changes() { + let _lock = cwd_lock(); + let temp = tempdir().unwrap(); + let first = temp.path().join("first"); + let second = temp.path().join("second"); + std::fs::create_dir_all(&first).unwrap(); + std::fs::create_dir_all(&second).unwrap(); + std::fs::write(first.join("plugins.toml"), "version = 1\n").unwrap(); + + let pinned = { + let _cwd = CurrentDirectoryGuard::enter(&first); + pin_plugin_config_path(Path::new("plugins.toml")).unwrap() + }; + let _cwd = CurrentDirectoryGuard::enter(&second); + assert_eq!( + sibling_lifecycle_state_path(&pinned), + dunce::canonicalize(&first) + .unwrap() + .join(DYNAMIC_PLUGIN_STATE_FILENAME) + ); +} + +#[cfg(windows)] +#[test] +fn missing_and_existing_windows_sources_pin_to_portable_physical_paths() { + let temp = tempdir().unwrap(); + let config = temp.path().join("plugins.toml"); + + let missing = pin_plugin_config_path(&config).unwrap(); + assert!(missing.is_absolute()); + assert!(!missing.to_string_lossy().starts_with(r"\\?\")); + + std::fs::write(&config, "version = 1\n").unwrap(); + let existing = pin_plugin_config_path(&config).unwrap(); + assert_eq!(existing, missing); + assert!(!existing.to_string_lossy().starts_with(r"\\?\")); + assert_eq!( + std::fs::canonicalize(existing).unwrap(), + std::fs::canonicalize(config).unwrap() + ); + + let verbatim_root = std::fs::canonicalize(temp.path()).unwrap(); + let reserved = pin_plugin_config_path(&verbatim_root.join("CON").join("plugins.toml")).unwrap(); + assert!(reserved.to_string_lossy().starts_with(r"\\?\")); + + let overlong = + pin_plugin_config_path(&verbatim_root.join("a".repeat(240)).join("plugins.toml")).unwrap(); + assert!(overlong.to_string_lossy().starts_with(r"\\?\")); +} + +#[test] +fn legacy_state_without_declaration_sources_is_readable() { + let temp = tempdir().unwrap(); + let state_path = temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME); + std::fs::write(&state_path, r#"{"schema_version":1,"records":[]}"#).unwrap(); + + let state = read_lifecycle_state(&state_path).unwrap(); + + assert!(state.list(true).is_empty()); + assert_eq!(state.declaration_source("missing"), None); +} + +#[test] +fn save_drops_declaration_sources_for_absent_records() { + let temp = tempdir().unwrap(); + let state_path = temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME); + std::fs::write( + &state_path, + r#"{"schema_version":1,"records":[],"declaration_sources":{"stale":"/tmp/plugins.toml"}}"#, + ) + .unwrap(); + let lock = lock_lifecycle_state(&state_path).unwrap(); + let state = read_locked_lifecycle_state(&lock).unwrap(); + + save_locked_lifecycle_state(&lock, &state).unwrap(); + + let persisted: serde_json::Value = + serde_json::from_slice(&std::fs::read(&state_path).unwrap()).unwrap(); + assert!(persisted.get("declaration_sources").is_none()); +} + +#[cfg(unix)] +#[test] +fn symlinked_source_and_target_share_lifecycle_state() { + use std::os::unix::fs::symlink; + + let temp = tempdir().unwrap(); + let target_dir = temp.path().join("target"); + let alias_dir = temp.path().join("alias"); + std::fs::create_dir_all(&target_dir).unwrap(); + std::fs::create_dir_all(&alias_dir).unwrap(); + let target = target_dir.join("plugins.toml"); + std::fs::write(&target, "version = 1\n").unwrap(); + let alias = alias_dir.join("plugins.toml"); + symlink(&target, &alias).unwrap(); + + let pinned_alias = pin_plugin_config_path(&alias).unwrap(); + assert_eq!(pinned_alias, target.canonicalize().unwrap()); + assert_eq!( + sibling_lifecycle_state_path(&pinned_alias), + target_dir + .canonicalize() + .unwrap() + .join(DYNAMIC_PLUGIN_STATE_FILENAME) + ); +} + +#[cfg(unix)] +#[test] +fn missing_source_and_parents_pin_through_nearest_existing_symlink() { + use std::os::unix::fs::symlink; + + let temp = tempdir().unwrap(); + let physical = temp.path().join("physical"); + std::fs::create_dir_all(&physical).unwrap(); + let alias = temp.path().join("alias"); + symlink(&physical, &alias).unwrap(); + let logical = alias.join("missing").join("nested").join("plugins.toml"); + let expected = physical + .canonicalize() + .unwrap() + .join("missing") + .join("nested") + .join("plugins.toml"); + + let before_creation = pin_plugin_config_path(&logical).unwrap(); + std::fs::create_dir_all(logical.parent().unwrap()).unwrap(); + std::fs::write(&logical, "version = 1\n").unwrap(); + let after_creation = pin_plugin_config_path(&logical).unwrap(); + + assert_eq!(before_creation, expected); + assert_eq!(after_creation, expected); +} + +#[cfg(unix)] +#[test] +fn lifecycle_replace_syncs_the_containing_directory() { + use std::cell::RefCell; + + let temp = tempdir().unwrap(); + let target = temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME); + let staged = temp.path().join(".staged-state.tmp"); + std::fs::write(&target, b"old").unwrap(); + std::fs::write(&staged, b"new").unwrap(); + let synced_directory = RefCell::new(None); + + replace_lifecycle_state_with_directory_sync(&staged, &target, |directory| { + synced_directory.replace(Some(directory.to_path_buf())); + Ok(()) + }) + .unwrap(); + + assert_eq!(std::fs::read(&target).unwrap(), b"new"); + assert!(!staged.exists()); + assert_eq!( + synced_directory.into_inner(), + Some(temp.path().to_path_buf()) + ); +} + +#[cfg(unix)] +#[test] +fn special_file_lifecycle_state_is_rejected_without_reading_it() { + let error = read_lifecycle_registry(Path::new("/dev/zero")) + .expect_err("a character device must not be read as lifecycle state"); + assert!(error.to_string().contains("must be a regular file")); +} + +fn fixture_record(id: &str) -> DynamicPluginRecord { + DynamicPluginManifest::parse_toml(&format!( + r#"manifest_version = 1 +[plugin] +id = "{id}" +kind = "rust_dynamic" +[compat] +relay = ">=0.5,<1.0" +native_api = "v1" +[capabilities] +items = ["plugin_native"] +[defaults] +enabled = false +[load] +library = "plugin.so" +symbol = "nemo_relay_plugin_entrypoint_v1" +[source] +artifact = "plugin.so" +[integrity] +sha256 = "sha256:placeholder" +"#, + )) + .unwrap() + .into_record(Some(format!("/{id}/relay-plugin.toml"))) + .unwrap() +} + +#[test] +fn lifecycle_state_ownership_methods_reject_unknown_ids_and_follow_registry_replacement() { + let mut state = DynamicPluginLifecycleState::new(DynamicPluginRegistry::default()); + let error = state + .set_declaration_source("missing", "/tmp/plugins.toml".into()) + .unwrap_err(); + assert!( + error + .to_string() + .contains("unknown dynamic plugin 'missing'") + ); + + state.add(fixture_record("owned")).unwrap(); + state + .set_declaration_source("owned", "/tmp/plugins.toml".into()) + .unwrap(); + assert_eq!(state.declaration_source("owned"), Some("/tmp/plugins.toml")); + state.clear_declaration_source("owned"); + assert_eq!(state.declaration_source("owned"), None); + + state + .set_declaration_source("owned", "/tmp/plugins.toml".into()) + .unwrap(); + state.replace_registry(DynamicPluginRegistry::default()); + assert!(state.list(true).is_empty()); + assert_eq!(state.declaration_source("owned"), None); +} + +#[test] +fn lifecycle_state_defaults_schema_and_reports_invalid_documents() { + let temp = tempdir().unwrap(); + let state_path = temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME); + + assert!( + read_lifecycle_state(&state_path) + .unwrap() + .list(true) + .is_empty() + ); + + std::fs::write(&state_path, r#"{"records":[]}"#).unwrap(); + assert!( + read_lifecycle_state(&state_path) + .unwrap() + .list(true) + .is_empty() + ); + + std::fs::write(&state_path, r#"{"schema_version":99,"records":[]}"#).unwrap(); + let error = read_lifecycle_state(&state_path).unwrap_err(); + assert!( + error + .to_string() + .contains("unsupported dynamic plugin registry schema_version 99") + ); + + std::fs::write(&state_path, r#"{"schema_version":1,"records":["secret"}"#).unwrap(); + let error = read_lifecycle_state(&state_path).unwrap_err(); + assert!( + error + .to_string() + .contains("invalid dynamic plugin registry state") + ); + assert!(!error.to_string().contains("secret")); +} + +#[test] +fn locked_registry_wrappers_persist_records_and_declaration_ownership() { + let temp = tempdir().unwrap(); + let state_path = temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME); + let lock = lock_lifecycle_state(&state_path).unwrap(); + assert!(format!("{lock:?}").contains("LifecycleStateLock")); + + let mut state = read_locked_lifecycle_state(&lock).unwrap(); + state.add(fixture_record("persisted")).unwrap(); + state + .set_declaration_source( + "persisted", + temp.path().join("plugins.toml").display().to_string(), + ) + .unwrap(); + save_locked_lifecycle_state(&lock, &state).unwrap(); + + let registry = read_locked_lifecycle_registry(&lock).unwrap(); + assert!(registry.get("persisted").is_some()); + save_locked_lifecycle_registry(&lock, ®istry).unwrap(); + drop(lock); + + let persisted = read_lifecycle_state(&state_path).unwrap(); + assert!(persisted.get("persisted").is_some()); + assert!(persisted.declaration_source("persisted").is_some()); + assert!( + read_lifecycle_registry(&state_path) + .unwrap() + .get("persisted") + .is_some() + ); +} + +#[test] +fn sibling_state_path_handles_a_parentless_plugin_filename() { + assert_eq!( + sibling_lifecycle_state_path(Path::new("")), + PathBuf::from(DYNAMIC_PLUGIN_STATE_FILENAME) + ); +} + +#[test] +fn lifecycle_lock_directory_creation_errors_are_contextualized() { + let temp = tempdir().unwrap(); + let blocking_file = temp.path().join("blocking-file"); + std::fs::write(&blocking_file, b"file").unwrap(); + + let error = + lock_lifecycle_state(&blocking_file.join(DYNAMIC_PLUGIN_STATE_FILENAME)).unwrap_err(); + assert!( + error + .to_string() + .contains("create lifecycle state directory") + ); +} + +#[cfg(unix)] +#[test] +fn lifecycle_temp_creation_and_directory_sync_errors_are_contextualized() { + let temp = tempdir().unwrap(); + let state_directory = temp.path().join("state"); + let state_path = state_directory.join(DYNAMIC_PLUGIN_STATE_FILENAME); + let lock = lock_lifecycle_state(&state_path).unwrap(); + let state = DynamicPluginLifecycleState::default(); + + // Keep the lock descriptor open while replacing its unlinked parent directory with a file. + // This makes temporary-file creation fail deterministically, including under a privileged + // test runner that could bypass directory permission bits. + std::fs::remove_file(state_directory.join(DYNAMIC_PLUGIN_STATE_LOCK_FILENAME)).unwrap(); + std::fs::remove_dir(&state_directory).unwrap(); + std::fs::write(&state_directory, b"blocking file").unwrap(); + + let error = save_locked_lifecycle_state(&lock, &state).unwrap_err(); + assert!(error.to_string().contains("create lifecycle state")); + + let error = sync_lifecycle_state_directory(&temp.path().join("missing-directory")).unwrap_err(); + assert!(error.to_string().contains("open lifecycle state directory")); +} + +#[cfg(windows)] +#[test] +fn windows_lifecycle_replace_reports_a_missing_staged_file() { + let temp = tempdir().unwrap(); + let error = replace_lifecycle_state( + &temp.path().join("missing.tmp"), + &temp.path().join(DYNAMIC_PLUGIN_STATE_FILENAME), + ) + .unwrap_err(); + assert!(error.to_string().contains("replace lifecycle state")); +} diff --git a/crates/plugin-host-config/tests/unit/trust.rs b/crates/plugin-host-config/tests/unit/trust.rs new file mode 100644 index 000000000..966cf8e71 --- /dev/null +++ b/crates/plugin-host-config/tests/unit/trust.rs @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::fs; + +use nemo_relay::plugin::dynamic::DynamicPluginStartupClass; +use tempfile::tempdir; + +use super::*; + +#[test] +fn configured_public_key_values_are_redacted_from_trust_errors() { + let temp = tempdir().unwrap(); + let artifact = temp.path().join("artifact.bin"); + let signature = temp.path().join("signature.txt"); + let manifest = temp.path().join("relay-plugin.toml"); + fs::write(&artifact, b"artifact").unwrap(); + fs::write(&signature, "AA==").unwrap(); + let secret = "do-not-leak-trusted-key"; + + let failure = verify_signature( + manifest.to_string_lossy().as_ref(), + &artifact, + "signature.txt", + &[secret.to_owned()], + ) + .unwrap_err(); + let rendered = failure.display("fixture").to_string(); + assert!(!rendered.contains(secret), "{rendered}"); +} + +#[test] +fn trust_helpers_classify_missing_files_empty_signatures_and_invalid_keys() { + let temp = tempdir().unwrap(); + let manifest_ref = temp.path().join("relay-plugin.toml"); + let missing_artifact = temp.path().join("missing-artifact"); + let signature = temp.path().join("signature.txt"); + fs::write(&signature, "AA==").unwrap(); + + let failure = verify_signature( + manifest_ref.to_string_lossy().as_ref(), + &missing_artifact, + "signature.txt", + &["ed25519:AA==".into()], + ) + .unwrap_err(); + assert!(matches!( + failure, + DynamicPluginTrustFailure::ArtifactRead { .. } + )); + + let missing = read_signature_bytes(&temp.path().join("missing-signature")).unwrap_err(); + assert!(matches!( + missing, + DynamicPluginTrustFailure::SignatureRead { .. } + )); + + fs::write(&signature, " \n").unwrap(); + let empty = read_signature_bytes(&signature).unwrap_err(); + assert!(empty.display("fixture").to_string().contains("empty")); + + fs::write(&signature, "not-base64").unwrap(); + let invalid = read_signature_bytes(&signature).unwrap_err(); + assert!( + invalid + .display("fixture") + .to_string() + .contains("invalid base64") + ); + + let unsupported = parse_ed25519_public_key("plain-key").unwrap_err(); + assert!(matches!( + unsupported, + DynamicPluginTrustFailure::InvalidTrustedKey { .. } + )); + let malformed = parse_ed25519_public_key("ed25519:not-base64").unwrap_err(); + assert!( + malformed + .display("fixture") + .to_string() + .contains("invalid trusted public key") + ); + + let absolute = if cfg!(windows) { + PathBuf::from(r"C:\absolute\artifact") + } else { + PathBuf::from("/absolute/artifact") + }; + assert_eq!( + resolve_artifact_path("relay-plugin.toml", absolute.to_string_lossy().as_ref()), + absolute + ); +} + +#[test] +fn authenticity_without_an_optional_signature_remains_unknown() { + let temp = tempdir().unwrap(); + let artifact = temp.path().join("artifact.bin"); + fs::write(&artifact, b"artifact").unwrap(); + let manifest_ref = temp.path().join("relay-plugin.toml"); + let manifest = DynamicPluginManifest::parse_toml( + r#"manifest_version = 1 +[plugin] +id = "optional-signature" +kind = "rust_dynamic" +[compat] +relay = ">=0.5,<1.0" +native_api = "v1" +[capabilities] +items = ["plugin_native"] +[defaults] +enabled = false +[load] +library = "artifact.bin" +symbol = "nemo_relay_plugin_entrypoint_v1" +[source] +artifact = "artifact.bin" +[integrity] +sha256 = "sha256:placeholder" +"#, + ) + .unwrap(); + let policy = EvaluatedDynamicPluginHostPolicy { + policy_satisfied: true, + startup_class: DynamicPluginStartupClass::Optional, + attestation_mode: DynamicPluginAttestationMode::SignatureIfPresent, + trusted_public_keys: Vec::new(), + failure: None, + }; + + assert_eq!( + evaluate_authenticity( + &manifest, + manifest_ref.to_string_lossy().as_ref(), + &artifact, + &policy + ) + .unwrap(), + DynamicPluginCheckState::Unknown + ); + + let mut missing = manifest; + missing.source.as_mut().unwrap().artifact = Some("missing.bin".into()); + let trust = + evaluate_dynamic_plugin_trust(&missing, manifest_ref.to_string_lossy().as_ref(), &policy); + assert!(matches!( + trust.failure(), + Some(DynamicPluginTrustFailure::ArtifactRead { .. }) + )); +} diff --git a/crates/python/Cargo.toml b/crates/python/Cargo.toml index 693ca3a99..35492a0bf 100644 --- a/crates/python/Cargo.toml +++ b/crates/python/Cargo.toml @@ -19,6 +19,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] nemo-relay = { workspace = true, features = ["atof-streaming", "guardrails-remote", "object-store", "worker-grpc"] } +nemo-relay-plugin-host-config.workspace = true nemo-relay-adaptive = { workspace = true, features = ["redis-backend"] } nemo-relay-pii-redaction.workspace = true pyo3 = { version = "0.29.0", features = ["abi3", "abi3-py311", "experimental-inspect", "macros"] } diff --git a/crates/python/src/py_plugin.rs b/crates/python/src/py_plugin.rs index 281ae0eca..084738955 100644 --- a/crates/python/src/py_plugin.rs +++ b/crates/python/src/py_plugin.rs @@ -36,6 +36,7 @@ use nemo_relay::plugin::{ active_plugin_report, clear_plugin_configuration, deregister_plugin, initialize_plugins, list_plugin_kinds, register_plugin, rollback_registrations, validate_plugin_config, }; +use nemo_relay_plugin_host_config::{PluginFileActivation, initialize_from_plugins_toml}; use crate::convert::{json_to_py, py_to_json}; use crate::py_callable::{ @@ -705,6 +706,17 @@ struct PyPluginHostActivation { report: nemo_relay::plugin::ConfigReport, } +/// Owned file-backed plugin activation. +/// +/// The public Python wrapper retains this object until ``close()`` or context +/// manager exit. Dropping it without an explicit close still clears callbacks +/// before unloading plugin code. +#[pyclass(name = "_PluginFileActivation")] +struct PyPluginFileActivation { + close_state: Arc, + report: nemo_relay::plugin::ConfigReport, +} + #[derive(Clone, Copy)] enum PluginTeardownErrorKind { Value, @@ -786,8 +798,31 @@ impl PluginTeardownCompletion { } } +enum OwnedPluginActivation { + Dynamic(PluginHostActivation), + File(PluginFileActivation), +} + +impl OwnedPluginActivation { + fn is_active(&self) -> bool { + match self { + Self::Dynamic(activation) => activation.is_active(), + Self::File(activation) => activation.is_active(), + } + } + + fn clear(self) -> std::result::Result<(), PluginError> { + match self { + Self::Dynamic(activation) => activation.clear(), + Self::File(activation) => activation + .clear() + .map_err(|error| error.into_plugin_error()), + } + } +} + enum PluginHostCloseStatus { - Active(Option), + Active(Option), Closing, Closed, } @@ -795,13 +830,16 @@ enum PluginHostCloseStatus { struct PluginHostCloseState { status: Mutex, completion: PluginTeardownCompletion, + owns_plugin_configuration: bool, } impl PluginHostCloseState { - fn new(activation: PluginHostActivation) -> Self { + fn new(activation: OwnedPluginActivation) -> Self { + let owns_plugin_configuration = activation.is_active(); Self { status: Mutex::new(PluginHostCloseStatus::Active(Some(activation))), completion: PluginTeardownCompletion::new(), + owns_plugin_configuration, } } @@ -813,7 +851,7 @@ impl PluginHostCloseState { match &*status { PluginHostCloseStatus::Active(activation) => activation .as_ref() - .is_some_and(PluginHostActivation::is_active), + .is_some_and(OwnedPluginActivation::is_active), PluginHostCloseStatus::Closing | PluginHostCloseStatus::Closed => false, } } @@ -836,6 +874,10 @@ impl PluginHostCloseState { let Some(activation) = activation else { return; }; + if !activation.is_active() { + self.finish(Ok(())); + return; + } // Keep the activation outside the spawned closure so a thread-spawn // failure cannot drop it and synchronously run teardown on the caller. @@ -855,12 +897,12 @@ impl PluginHostCloseState { activation.clear() })) .map_err(|_| { - PluginTeardownError::runtime("dynamic plugin teardown task panicked") + PluginTeardownError::runtime("plugin host teardown task panicked") }) .and_then(|result| result.map_err(PluginTeardownError::from_plugin_error)) } None => Err(PluginTeardownError::runtime( - "dynamic plugin teardown task lost its activation", + "plugin host teardown task lost its activation", )), }; close_state.finish(result); @@ -877,7 +919,7 @@ impl PluginHostCloseState { std::mem::forget(activation); } self.finish(Err(PluginTeardownError::runtime(format!( - "failed to start dynamic plugin teardown task: {error}" + "failed to start plugin host teardown task: {error}" )))); } } @@ -887,12 +929,14 @@ impl PluginHostCloseState { .status .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) = PluginHostCloseStatus::Closed; - reset_plugin_configuration_clear_state(); + if self.owns_plugin_configuration { + reset_plugin_configuration_clear_state(); + } self.completion.finish(result); } async fn wait_for_close(&self) -> PluginTeardownResult { - self.completion.wait("dynamic plugin teardown").await + self.completion.wait("plugin host teardown").await } } @@ -978,7 +1022,7 @@ impl PyPluginHostActivation { /// Return whether this activation handle has not begun teardown. /// /// `False` does not guarantee another process-wide activation can start; - /// failed teardown may intentionally retain the activation owner. + /// failed cleanup may intentionally retain the activation owner. #[getter] fn is_active(&self) -> PyResult { Ok(self.close_state.is_active()) @@ -1029,7 +1073,90 @@ fn initialize_with_dynamic_plugins_py<'py>( Py::new( py, PyPluginHostActivation { - close_state: Arc::new(PluginHostCloseState::new(activation)), + close_state: Arc::new(PluginHostCloseState::new( + OwnedPluginActivation::Dynamic(activation), + )), + report, + }, + ) + }) + }) +} + +#[pymethods] +impl PyPluginFileActivation { + /// Return the activation report captured during initialization. + #[getter] + fn report(&self, py: Python<'_>) -> PyResult> { + let report = serde_json::to_value(&self.report) + .map_err(|error| pyo3::exceptions::PyRuntimeError::new_err(error.to_string()))?; + json_to_py(py, &report) + } + + /// Return whether this handle currently owns an active plugin host. + /// + /// `False` also represents a no-input inactive handle. After teardown + /// begins, it does not guarantee another process-wide activation can start; + /// failed cleanup may intentionally retain the activation owner. + #[getter] + fn is_active(&self) -> PyResult { + Ok(self.close_state.is_active()) + } + + /// Clear callbacks and unload the file-backed plugin host. + #[pyo3(signature = () -> "None", text_signature = "($self) -> None")] + fn close<'py>(&self, py: Python<'py>) -> PyResult> { + let close_state = Arc::clone(&self.close_state); + close_state.begin_close(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + close_state + .wait_for_close() + .await + .map_err(|error| error.to_py_err()) + }) + } +} + +impl Drop for PyPluginFileActivation { + fn drop(&mut self) { + self.close_state.begin_close(); + } +} + +/// Initialize an owned plugin host from standard `plugins.toml` sources. +#[pyfunction(name = "initialize_from_plugins_toml")] +#[pyo3( + signature = (config = None, *, plugin_config_path = None), + text_signature = "(config: object | None = None, *, plugin_config_path: str | None = None) -> object" +)] +fn initialize_from_plugins_toml_py<'py>( + py: Python<'py>, + config: Option<&Bound<'_, PyAny>>, + plugin_config_path: Option, +) -> PyResult> { + let config = config + .map(|config| { + let config_json = py_to_json(config)?; + serde_json::from_value::(config_json) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string())) + }) + .transpose()?; + let plugin_config_path = plugin_config_path.map(std::path::PathBuf::from); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let activation = initialize_from_plugins_toml(config, plugin_config_path) + .await + .map_err(|error| plugin_error_to_py_err(error.into_plugin_error()))?; + let report = activation.report().clone(); + if activation.is_active() { + reset_plugin_configuration_clear_state(); + } + Python::attach(|py| { + Py::new( + py, + PyPluginFileActivation { + close_state: Arc::new(PluginHostCloseState::new(OwnedPluginActivation::File( + activation, + ))), report, }, ) @@ -1096,9 +1223,11 @@ fn deregister_plugin_py(plugin_kind: &str) -> bool { pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(validate_plugin_config_py, m)?)?; m.add_function(wrap_pyfunction!(initialize_plugins_py, m)?)?; m.add_function(wrap_pyfunction!(initialize_with_dynamic_plugins_py, m)?)?; + m.add_function(wrap_pyfunction!(initialize_from_plugins_toml_py, m)?)?; m.add_function(wrap_pyfunction!(clear_plugin_configuration_py, m)?)?; m.add_function(wrap_pyfunction!(clear_plugin_configuration_async_py, m)?)?; m.add_function(wrap_pyfunction!(active_plugin_report_py, m)?)?; diff --git a/crates/python/tests/coverage/py_plugin_coverage_tests.rs b/crates/python/tests/coverage/py_plugin_coverage_tests.rs index 88127cdea..6ea69d145 100644 --- a/crates/python/tests/coverage/py_plugin_coverage_tests.rs +++ b/crates/python/tests/coverage/py_plugin_coverage_tests.rs @@ -108,9 +108,11 @@ fn register_adds_plugin_management_bindings() { for name in [ "PluginContext", "_PluginHostActivation", + "_PluginFileActivation", "validate_plugin_config", "initialize_plugins", "initialize_with_dynamic_plugins", + "initialize_from_plugins_toml", "clear_plugin_configuration", "clear_plugin_configuration_async", "active_plugin_report", @@ -184,6 +186,34 @@ async def clear(module): }); } +#[test] +fn inactive_close_completion_does_not_rotate_global_clear_state() { + let _plugin_test_state = lock_plugin_test_state_for_tests(); + reset_plugin_configuration_clear_state(); + let before = plugin_configuration_clear_state(); + let inactive = PluginHostCloseState { + status: Mutex::new(PluginHostCloseStatus::Closing), + completion: PluginTeardownCompletion::new(), + owns_plugin_configuration: false, + }; + + inactive.finish(Ok(())); + + let after_inactive = plugin_configuration_clear_state(); + assert!(Arc::ptr_eq(&before, &after_inactive)); + + let active = PluginHostCloseState { + status: Mutex::new(PluginHostCloseStatus::Closing), + completion: PluginTeardownCompletion::new(), + owns_plugin_configuration: true, + }; + active.finish(Ok(())); + assert!(!Arc::ptr_eq( + &after_inactive, + &plugin_configuration_clear_state() + )); +} + #[test] fn plugin_context_registers_all_runtime_hooks_and_drains_registrations() { let _python = crate::test_support::init_python_test(); diff --git a/docs/about-nemo-relay/concepts/plugins.mdx b/docs/about-nemo-relay/concepts/plugins.mdx index bb5f06528..46bb4315a 100644 --- a/docs/about-nemo-relay/concepts/plugins.mdx +++ b/docs/about-nemo-relay/concepts/plugins.mdx @@ -225,11 +225,12 @@ refer to [Plugin Configuration Files](/configure-plugins/plugin-configuration-fi ## Discoverable Plugins -Discoverable plugins use the same component lifecycle, but the CLI reads a -`relay-plugin.toml` manifest before it creates an internal component. The -manifest identifies a Rust native shared library or a local `grpc-v1` worker, -declares compatibility and capabilities, and supplies integrity evidence for -the artifact. +Discoverable plugins use the same component lifecycle, but Relay reads a +`relay-plugin.toml` manifest before it creates an internal component. The CLI +gateway and Python owned file initializer can host these plugins. The manifest +identifies a Rust native shared library or a local `grpc-v1` worker, declares +compatibility and capabilities, and supplies integrity evidence for the +artifact. The operator keeps the manifest reference and component configuration in `plugins.toml`. Use `nemo-relay plugins validate ` to check the diff --git a/docs/about-nemo-relay/release-notes/index.mdx b/docs/about-nemo-relay/release-notes/index.mdx index 408d20151..8f38a3f3a 100644 --- a/docs/about-nemo-relay/release-notes/index.mdx +++ b/docs/about-nemo-relay/release-notes/index.mdx @@ -98,6 +98,12 @@ their values cannot be isolated between endpoints. ### Fixed Known Issues in 0.7 +- Python applications can now use + `plugin.initialize_from_plugins_toml()` to own static components and enabled + discoverable plugins resolved from the same layered `plugins.toml` sources. + The initializer preserves CLI lifecycle, trust, managed-environment, and + activation-snapshot behavior without requiring callers to construct dynamic + activation specifications. - Programmatically declared plugin components now apply their `enabled` value over discovered file configuration. When code re-enables a component that a discovered file disabled, initialization reports a warning that names the diff --git a/docs/configure-plugins/about.mdx b/docs/configure-plugins/about.mdx index b96315a2f..1369d36f8 100644 --- a/docs/configure-plugins/about.mdx +++ b/docs/configure-plugins/about.mdx @@ -18,6 +18,7 @@ Choose the path that matches your goal: | --- | --- | | Configure a built-in Relay component | Choose the component guide below. Use `plugins.toml` or the binding API. | | Add a plugin someone else packaged | [Configure Discoverable Plugins](/configure-plugins/discoverable-plugins) | +| Load enabled discoverable plugins from Python | Use the owned initializer in [Configure Discoverable Plugins](/configure-plugins/discoverable-plugins#initialize-discoverable-plugins-from-python). | | Create a reusable plugin | [Build Plugins](/build-plugins/about) | ## Built-in Plugins diff --git a/docs/configure-plugins/discoverable-plugins.mdx b/docs/configure-plugins/discoverable-plugins.mdx index 3f73c4b28..2fc87f2f2 100644 --- a/docs/configure-plugins/discoverable-plugins.mdx +++ b/docs/configure-plugins/discoverable-plugins.mdx @@ -14,10 +14,10 @@ their `relay-plugin.toml` manifest describes a native shared library or a stores its component configuration. -Discoverable plugins are trusted extensions. Native plugins run in the gateway -process. Worker plugins run in a separate process, but process isolation is not -a security sandbox. Install manifests and artifacts only from sources you -trust. +Discoverable plugins are trusted extensions. Native plugins run in the host +process without a sandbox. Worker plugins run in a separate process, but +process isolation is not a security sandbox. Install manifests and artifacts +only from sources you trust. ## Add and Enable a Plugin @@ -33,11 +33,12 @@ nemo-relay plugins validate acme.plugin ``` `add` writes a `[[plugins.dynamic]]` reference to the selected `plugins.toml` -scope and stores CLI lifecycle state next to it. `enable` changes that +scope and stores Relay lifecycle state next to it. `enable` changes that lifecycle state; it does not load code immediately. Relay validates and loads -enabled plugins when the gateway starts. Use `plugins list`, `plugins inspect`, -and `plugins validate` to review current state and diagnostics. Use `disable` -or `remove` to stop loading a registered plugin. +enabled plugins when the gateway or an owned embedded host starts. Use +`plugins list`, `plugins inspect`, and `plugins validate` to review current +state and diagnostics. Use `disable` or `remove` to stop loading a registered +plugin. You can also add the reference directly when configuration is provisioned by automation: @@ -62,6 +63,35 @@ Do not add a Python worker only with this TOML record. Run create and retain its managed Python environment before it can activate the worker. +## Initialize Discoverable Plugins From Python + +Call the owned file initializer when an embedded Python host must activate +static components and enabled discoverable plugins from the same configuration: + +```python +from nemo_relay import plugin + +activation = await plugin.initialize_from_plugins_toml() +async with activation: + print(activation.report) +``` + +Pass `plugin_config_path=path` to replace the ambient user file. The nearest +project and system files remain eligible, matching CLI source selection. Set +`NEMO_RELAY_CONFIG_SCOPE=user` to suppress project discovery while retaining +the user-or-selected and system layers. + +Calling `initialize_from_plugins_toml` is explicit consent to load enabled +native libraries or start enabled worker processes from the selected sources. +The returned `PluginFileActivation` owns static registrations, plugin runtimes, +and activation snapshots until `close()` or async context-manager exit. + +The initializer reconciles lifecycle state before activation. It can create or +refresh each source's sibling `.dynamic-plugins.json`, but a newly discovered +record remains disabled. It does not edit `plugins.toml`, enable plugins, +install packages, or create Python environments. Continue to use +`nemo-relay plugins add` and `nemo-relay plugins enable` for those operations. + ## Validate Before Loading Code Relay validates a manifest before it activates plugin code. The manifest must diff --git a/docs/configure-plugins/plugin-configuration-files.mdx b/docs/configure-plugins/plugin-configuration-files.mdx index e5a486c9f..8347771c6 100644 --- a/docs/configure-plugins/plugin-configuration-files.mdx +++ b/docs/configure-plugins/plugin-configuration-files.mdx @@ -133,12 +133,12 @@ Each built-in component has: | `enabled` | `true` | Relay validates disabled components but does not initialize them. | | `config` | `{}` | Component-local configuration object. The shape depends on `kind`. | -## Gateway Discoverable Plugin Records +## Discoverable Plugin Records -Use `[[plugins.dynamic]]` only for a gateway-managed manifest-backed native or -worker plugin. These records remain separate from `[[components]]`. During -gateway activation, Relay loads each enabled dynamic adapter, synthesizes its -internal component, and validates the component configuration. +Use `[[plugins.dynamic]]` for a Relay-managed manifest-backed native or worker +plugin. These records remain separate from `[[components]]`. During gateway or +owned embedded-host activation, Relay loads each enabled dynamic adapter, +synthesizes its internal component, and validates the component configuration. The following record configures a dynamic plugin: @@ -165,6 +165,39 @@ for manifest, trust, and policy requirements. The runtime reads only files named `plugins.toml` during default discovery. +### Python Owned Initialization + +Use the Python file initializer when one embedded host must own the static and +dynamic configuration selected by these discovery rules: + +```python +from nemo_relay import plugin + +activation = await plugin.initialize_from_plugins_toml( + plugin_config_path="path/to/plugins.toml", +) +try: + print(activation.report) +finally: + await activation.close() +``` + +The selected path replaces the ambient user layer. Project and system layers +still participate, and an optional `config` argument supplies the +highest-precedence static-component overlay. The function does not accept +launch-time overrides for `[[plugins.dynamic]].config`. + +The returned `PluginFileActivation` must remain alive while plugin callbacks +can run. A second owned activation and the legacy initialization and clear APIs +report an ownership conflict until this handle closes. With no discovered file +and no `config` argument, the function returns an inactive no-op handle. + +Dynamic declarations do not enable code. Relay joins each declaration to its +source file's sibling `.dynamic-plugins.json` and loads it only when the record +is live and enabled. Initialization can create or refresh that lifecycle file, +but newly hydrated records remain disabled. Use the Relay CLI to install and +enable discoverable plugins. + ## Runtime Discovery The runtime resolves plugin configuration from `plugins.toml` files and an @@ -357,9 +390,10 @@ follows: 1. Discover and merge the `plugins.toml` files from lowest to highest precedence (explicit-or-user → project → system), using the [Precedence And Merge Behavior](#precedence-and-merge-behavior) rules above. -2. Layer the config object you pass to `initialize` over that merged base. Any - setting it specifies overrides the file value, and the result is the - effective config that Relay validates and activates. +2. Layer the config object you pass to `initialize` or + `initialize_from_plugins_toml` over that merged base. Any setting it + specifies overrides the file value, and the result is the effective config + that Relay validates and activates. Programmatic lists participate in the same concatenation rules. For example, a programmatic `config.opentelemetry.endpoints` list appears before endpoints diff --git a/justfile b/justfile index 5f058e28c..02b736641 100644 --- a/justfile +++ b/justfile @@ -467,6 +467,7 @@ local_dependencies = ( "nemo-relay-worker-proto", "nemo-relay-worker", "nemo-relay", + "nemo-relay-plugin-host-config", "nemo-relay-plugin", "nemo-relay-adaptive", "nemo-relay-pii-redaction", @@ -759,6 +760,7 @@ published_cargo_packages() { nemo-relay-worker-proto \ nemo-relay-worker \ nemo-relay \ + nemo-relay-plugin-host-config \ nemo-relay-adaptive \ nemo-relay-pii-redaction \ nemo-relay-switchyard \ @@ -1210,6 +1212,7 @@ test-python: if is_true "{{ ci }}" && [[ -n "$rust_coverage_out" ]]; then cargo llvm-cov report \ -p nemo-relay-python \ + -p nemo-relay-plugin-host-config \ --ignore-filename-regex '.*/tests/.*\.rs$' \ --cobertura \ --output-path "$rust_coverage_out" @@ -1281,11 +1284,18 @@ test-python-plugin-e2e: PIP_FIND_LINKS="$tmp/wheels" NEMO_RELAY_PYTHON="$python_executable" \ "$cli" --config "$config" plugins add "$manifest" "$cli" --config "$config" plugins enable examples.python_grpc_worker + plugin_config="$tmp/plugins.toml" + test -f "$plugin_config" environment_ref="$("$python_executable" -c \ 'import json, sys; print(json.load(open(sys.argv[1]))["records"][0]["source"]["environment_ref"])' \ "$tmp/.dynamic-plugins.json")" test -x "$environment_ref/bin/python" || test -x "$environment_ref/Scripts/python.exe" + use_project_python_source "$python_executable" + "$python_executable" -m maturin develop --skip-install + NEMO_RELAY_CONFIG_SCOPE=user \ + "$python_executable" python/tests/plugin/file_activation_e2e.py "$plugin_config" + port="$("$python_executable" -c \ 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')" "$cli" --config "$config" --bind "127.0.0.1:$port" >"$tmp/gateway.log" 2>&1 & @@ -1512,6 +1522,12 @@ package-rust: nemo-relay) cargo_package_config+=(--config 'patch.crates-io.nemo-relay-types.path="crates/types"') cargo_package_config+=(--config 'patch.crates-io.nemo-relay-plugin.path="crates/plugin"') + cargo_package_config+=(--config 'patch.crates-io.nemo-relay-worker-proto.path="crates/worker-proto"') + ;; + nemo-relay-plugin-host-config) + cargo_package_config+=(--config 'patch.crates-io.nemo-relay-types.path="crates/types"') + cargo_package_config+=(--config 'patch.crates-io.nemo-relay.path="crates/core"') + cargo_package_config+=(--config 'patch.crates-io.nemo-relay-plugin.path="crates/plugin"') ;; nemo-relay-adaptive) cargo_package_config+=(--config 'patch.crates-io.nemo-relay-types.path="crates/types"') @@ -1535,12 +1551,21 @@ package-rust: cargo_package_config+=(--config 'patch.crates-io.nemo-relay.path="crates/core"') cargo_package_config+=(--config 'patch.crates-io.nemo-relay-plugin.path="crates/plugin"') ;; - nemo-relay-ffi|nemo-relay-cli) + nemo-relay-ffi) + cargo_package_config+=(--config 'patch.crates-io.nemo-relay-types.path="crates/types"') + cargo_package_config+=(--config 'patch.crates-io.nemo-relay.path="crates/core"') + cargo_package_config+=(--config 'patch.crates-io.nemo-relay-plugin.path="crates/plugin"') + cargo_package_config+=(--config 'patch.crates-io.nemo-relay-adaptive.path="crates/adaptive"') + cargo_package_config+=(--config 'patch.crates-io.nemo-relay-pii-redaction.path="crates/pii-redaction"') + ;; + nemo-relay-cli) cargo_package_config+=(--config 'patch.crates-io.nemo-relay-types.path="crates/types"') cargo_package_config+=(--config 'patch.crates-io.nemo-relay.path="crates/core"') cargo_package_config+=(--config 'patch.crates-io.nemo-relay-plugin.path="crates/plugin"') cargo_package_config+=(--config 'patch.crates-io.nemo-relay-adaptive.path="crates/adaptive"') cargo_package_config+=(--config 'patch.crates-io.nemo-relay-pii-redaction.path="crates/pii-redaction"') + cargo_package_config+=(--config 'patch.crates-io.nemo-relay-plugin-host-config.path="crates/plugin-host-config"') + cargo_package_config+=(--config 'patch.crates-io.nemo-relay-switchyard.path="crates/switchyard"') ;; esac if ((${#cargo_package_config[@]} == 0)); then diff --git a/python/nemo_relay/_native.pyi b/python/nemo_relay/_native.pyi index 43f2fa5f1..8e3a0b171 100644 --- a/python/nemo_relay/_native.pyi +++ b/python/nemo_relay/_native.pyi @@ -2343,6 +2343,23 @@ class _PluginHostActivation: """Clear and unload this activation; repeated calls are safe.""" ... +class _PluginFileActivation: + """Native owner for one file-backed plugin configuration.""" + + @property + def report(self) -> _JsonObject: + """Return the validation report captured during activation.""" + ... + + @property + def is_active(self) -> bool: + """Return whether this handle owns an active configuration.""" + ... + + def close(self) -> Awaitable[None]: + """Clear this activation; repeated calls are safe.""" + ... + def initialize_with_dynamic_plugins(config: object, dynamic_plugins: object) -> Awaitable[_PluginHostActivation]: """Initialize registered components with dynamic plugins as one owned host. @@ -2359,6 +2376,26 @@ def initialize_with_dynamic_plugins(config: object, dynamic_plugins: object) -> """ ... +def initialize_from_plugins_toml( + config: object | None = None, + *, + plugin_config_path: str | None = None, +) -> Awaitable[_PluginFileActivation]: + """Initialize an owned configuration from selected ``plugins.toml`` sources. + + Args: + config: Optional programmatic static configuration overlay. + plugin_config_path: Optional low-precedence plugin configuration path. + + Returns: + Awaitable resolving to the native file activation owner. + + Exceptional flow: + Resolution, lifecycle, ownership, load, and registration errors are + raised through the awaitable. + """ + ... + def initialize_plugins(config: object) -> Awaitable[_JsonObject]: """Validate and activate plugin configuration. diff --git a/python/nemo_relay/plugin.py b/python/nemo_relay/plugin.py index dbd807083..b27f2a22f 100644 --- a/python/nemo_relay/plugin.py +++ b/python/nemo_relay/plugin.py @@ -11,6 +11,7 @@ from __future__ import annotations import asyncio +import os from contextlib import asynccontextmanager from dataclasses import dataclass, field, fields, is_dataclass from typing import TYPE_CHECKING, AsyncIterator, Callable, Literal, Protocol, Self, TypedDict, cast @@ -32,6 +33,7 @@ UnsupportedBehavior, subscribers, ) +from nemo_relay._native import _PluginFileActivation as _NativePluginFileActivation from nemo_relay._native import _PluginHostActivation as _NativePluginHostActivation from nemo_relay._native import ( active_plugin_report as _active_plugin_report, @@ -45,6 +47,9 @@ from nemo_relay._native import ( deregister_plugin as _deregister_plugin, ) +from nemo_relay._native import ( + initialize_from_plugins_toml as _initialize_from_plugins_toml, +) from nemo_relay._native import ( initialize_plugins as _initialize_plugins, ) @@ -394,7 +399,7 @@ def is_active(self) -> bool: """Return whether this activation handle has not begun teardown. ``False`` does not guarantee another process-wide activation can start; - failed teardown may intentionally retain the activation owner. + failed cleanup may intentionally retain the activation owner. """ return self._native.is_active @@ -417,6 +422,54 @@ async def __aexit__( await self.close() +class PluginFileActivation: + """Owned lifetime for one file-backed plugin configuration. + + The activation may own static components, enabled dynamic plugins, or no + runtime state when discovery found no input. Keep an active object alive + until agent work completes and prefer ``async with`` or :meth:`close` for + deterministic teardown. + """ + + __slots__ = ("_native",) + + def __init__(self, native: _NativePluginFileActivation) -> None: + self._native = native + + @property + def report(self) -> ConfigReport: + """Return the validation report captured during activation.""" + return cast(ConfigReport, self._native.report) + + @property + def is_active(self) -> bool: + """Return whether this handle currently owns an active plugin host. + + ``False`` also represents a no-input inactive handle. After teardown + begins, it does not guarantee another process-wide activation can start; + failed cleanup may intentionally retain the activation owner. + """ + return self._native.is_active + + async def close(self) -> None: + """Clear the owned configuration; repeated calls are safe.""" + await self._native.close() + + async def __aenter__(self) -> Self: + """Return this activation when entering an async context.""" + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + """Close the activation when leaving an async context.""" + del exc_type, exc_value, traceback + await self.close() + + def validate(config: PluginConfig | JsonObject) -> ConfigReport: """Validate a plugin configuration without changing runtime state. @@ -477,6 +530,41 @@ async def initialize_with_dynamic_plugins( return PluginHostActivation(native) +async def initialize_from_plugins_toml( + config: PluginConfig | JsonObject | None = None, + *, + plugin_config_path: str | os.PathLike[str] | None = None, +) -> PluginFileActivation: + """Initialize one owned configuration from Relay's ``plugins.toml`` sources. + + Args: + config: Optional static component configuration applied after file + discovery. Supplying an empty configuration still creates an owned + static activation. + plugin_config_path: Optional low-precedence ``plugins.toml`` source. + It replaces the ambient user file while project and system sources + remain eligible. + + Returns: + An owned activation. With no files and no supplied configuration, the + returned handle is inactive and :meth:`PluginFileActivation.close` is + a no-op. + + Behavior: + Calling this function explicitly permits Relay to load enabled native + libraries and worker processes declared by the selected files. Dynamic + declarations are joined to sibling Relay lifecycle state and are never + enabled or installed by this function. + """ + normalized_config = None if config is None else _normalize_object(config) + normalized_path = None if plugin_config_path is None else os.fspath(plugin_config_path) + native = await _initialize_from_plugins_toml( + normalized_config, + plugin_config_path=normalized_path, + ) + return PluginFileActivation(native) + + def clear() -> None: """Clear the active plugin configuration. @@ -605,9 +693,11 @@ def deregister(plugin_kind: str) -> bool: "DynamicPluginKind", "PluginConfig", "PluginContext", + "PluginFileActivation", "PluginHostActivation", "Plugin", "initialize_with_dynamic_plugins", + "initialize_from_plugins_toml", "clear", "clear_async", "initialize", diff --git a/python/nemo_relay/plugin.pyi b/python/nemo_relay/plugin.pyi index 39bc464d5..71bbb7500 100644 --- a/python/nemo_relay/plugin.pyi +++ b/python/nemo_relay/plugin.pyi @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from collections.abc import Callable +from os import PathLike from types import TracebackType from typing import AsyncContextManager, Literal, Protocol, Self, TypedDict @@ -160,12 +161,31 @@ class PluginHostActivation: traceback: TracebackType | None, ) -> None: ... +class PluginFileActivation: + @property + def report(self) -> ConfigReport: ... + @property + def is_active(self) -> bool: ... + async def close(self) -> None: ... + async def __aenter__(self) -> Self: ... + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: ... + def validate(config: PluginConfig | JsonObject) -> ConfigReport: ... async def initialize(config: PluginConfig | JsonObject) -> ConfigReport: ... async def initialize_with_dynamic_plugins( config: PluginConfig | JsonObject, dynamic_plugins: list[DynamicPluginActivationSpec | JsonObject], ) -> PluginHostActivation: ... +async def initialize_from_plugins_toml( + config: PluginConfig | JsonObject | None = None, + *, + plugin_config_path: str | PathLike[str] | None = None, +) -> PluginFileActivation: ... def clear() -> None: ... async def clear_async() -> None: ... def plugin(config: PluginConfig | JsonObject) -> AsyncContextManager[ConfigReport]: ... diff --git a/python/tests/plugin/file_activation_e2e.py b/python/tests/plugin/file_activation_e2e.py new file mode 100644 index 000000000..4c00fd1bd --- /dev/null +++ b/python/tests/plugin/file_activation_e2e.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise file-backed activation with a lifecycle-managed Python worker.""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from typing import Any + +from nemo_relay import plugin, tools + + +async def run(plugin_config_path: Path) -> None: + """Load the prepared file, exercise its worker, and close its owner.""" + activation = await plugin.initialize_from_plugins_toml(plugin_config_path=plugin_config_path) + try: + assert activation.is_active + observed: dict[str, Any] = {} + + async def local_tool(args: Any) -> dict[str, Any]: + observed["args"] = args + return {"python_tool_executed": True, "args": args} + + result = await tools.execute( + "file-backed-python-worker", + {"query": "relay"}, + local_tool, + ) + expected_args = { + "query": "relay", + "_nemo_relay_plugin": {"tag": "python_grpc_worker"}, + } + assert observed == {"args": expected_args}, observed + assert result == {"python_tool_executed": True, "args": expected_args}, result + finally: + await activation.close() + + assert not activation.is_active + after_close = await tools.execute( + "file-backed-python-worker-after-close", + {"query": "relay"}, + lambda args: {"args": args}, + ) + assert after_close == {"args": {"query": "relay"}} + + +if __name__ == "__main__": + if len(sys.argv) != 2: + raise SystemExit(f"usage: {Path(sys.argv[0]).name} PLUGINS_TOML") + asyncio.run(run(Path(sys.argv[1]))) diff --git a/python/tests/test_dynamic_plugin_host.py b/python/tests/test_dynamic_plugin_host.py index 97340cec5..836b74ee3 100644 --- a/python/tests/test_dynamic_plugin_host.py +++ b/python/tests/test_dynamic_plugin_host.py @@ -8,7 +8,9 @@ import asyncio import gc import hashlib +import json import os +import shutil import signal import subprocess import sys @@ -19,9 +21,11 @@ from dataclasses import dataclass from pathlib import Path from typing import cast +from unittest.mock import AsyncMock, MagicMock import pytest +import nemo_relay from nemo_relay import Json, LLMRequest, llm, plugin, scope, tools @@ -57,6 +61,25 @@ def _native_library_name() -> str: return "libnemo_relay_plugin_fixture.so" +def _enable_hydrated_plugin(plugins_toml: Path, plugin_id: str) -> None: + state_path = plugins_toml.with_name(".dynamic-plugins.json") + state = json.loads(state_path.read_text()) + record = next(record for record in state["records"] if record["metadata"]["id"] == plugin_id) + record["spec"]["enabled"] = True + state_path.write_text(json.dumps(state, indent=2) + "\n") + + +def _assert_inherited_configuration_source(report: plugin.ConfigReport, expected: Path) -> None: + prefix = "inherited plugin configuration from discovered file: " + assert len(report["diagnostics"]) == 1 + diagnostic = report["diagnostics"][0] + assert diagnostic["level"] == "warning" + assert diagnostic["code"] == "plugin.configuration_inherited" + message = diagnostic["message"] + assert message.startswith(prefix) + assert os.path.samefile(message.removeprefix(prefix), expected) + + @pytest.fixture(scope="session") def native_dynamic_plugin(tmp_path_factory: pytest.TempPathFactory) -> _BuiltPlugin: root = _repo_root() @@ -75,8 +98,10 @@ def native_dynamic_plugin(tmp_path_factory: pytest.TempPathFactory) -> _BuiltPlu cwd=root, check=True, ) - library = target / "debug" / _native_library_name() - assert library.is_file() + built_library = target / "debug" / _native_library_name() + assert built_library.is_file() + library = manifest_dir / built_library.name + shutil.copy2(built_library, library) digest = hashlib.sha256(library.read_bytes()).hexdigest() manifest = manifest_dir / "relay-plugin.toml" manifest.write_text( @@ -98,11 +123,14 @@ def native_dynamic_plugin(tmp_path_factory: pytest.TempPathFactory) -> _BuiltPlu [capabilities] items = ["plugin_native"] + [source] + artifact = {library.name!r} + [integrity] sha256 = "sha256:{digest}" [load] - library = {library.as_posix()!r} + library = {library.name!r} symbol = "nemo_relay_fixture_native_plugin" """ ) @@ -129,8 +157,13 @@ def worker_dynamic_plugin(tmp_path_factory: pytest.TempPathFactory) -> _BuiltPlu cwd=root, check=True, ) - executable = target / "debug" / ("nemo-relay-worker-plugin-fixture" + (".exe" if sys.platform == "win32" else "")) - assert executable.is_file() + built_executable = ( + target / "debug" / ("nemo-relay-worker-plugin-fixture" + (".exe" if sys.platform == "win32" else "")) + ) + assert built_executable.is_file() + executable = manifest_dir / built_executable.name + shutil.copy2(built_executable, executable) + digest = hashlib.sha256(executable.read_bytes()).hexdigest() manifest = manifest_dir / "relay-plugin.toml" manifest.write_text( textwrap.dedent( @@ -151,9 +184,15 @@ def worker_dynamic_plugin(tmp_path_factory: pytest.TempPathFactory) -> _BuiltPlu [capabilities] items = ["plugin_worker"] + [source] + artifact = {executable.name!r} + + [integrity] + sha256 = "sha256:{digest}" + [load] runtime = "rust" - entrypoint = {executable.as_posix()!r} + entrypoint = {executable.name!r} """ ) ) @@ -272,6 +311,71 @@ async def initialize(config: object) -> object: ] +async def test_initialize_from_plugins_toml_normalizes_path_and_owns_close( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +): + captured: list[tuple[object | None, str | None]] = [] + mock_native = MagicMock() + mock_native.report = {"diagnostics": []} + mock_native.is_active = True + + async def close() -> None: + mock_native.is_active = False + + mock_native.close = AsyncMock(side_effect=close) + + async def initialize( + config: object | None, + *, + plugin_config_path: str | None, + ) -> MagicMock: + captured.append((config, plugin_config_path)) + return mock_native + + monkeypatch.setattr(plugin, "_initialize_from_plugins_toml", initialize) + path = tmp_path / "plugins.toml" + + activation = await plugin.initialize_from_plugins_toml( + plugin.PluginConfig(), + plugin_config_path=path, + ) + + assert captured == [(plugin.PluginConfig().to_dict(), str(path))] + assert activation.report == {"diagnostics": []} + assert activation.is_active + async with activation as active: + assert active is activation + assert not activation.is_active + await activation.close() + + +async def test_initialize_from_plugins_toml_preserves_absent_config_and_path( + monkeypatch: pytest.MonkeyPatch, +): + captured: list[tuple[object | None, str | None]] = [] + mock_native = MagicMock() + mock_native.report = {"diagnostics": []} + mock_native.is_active = False + mock_native.close = AsyncMock(return_value=None) + + async def initialize( + config: object | None, + *, + plugin_config_path: str | None, + ) -> MagicMock: + captured.append((config, plugin_config_path)) + return mock_native + + monkeypatch.setattr(plugin, "_initialize_from_plugins_toml", initialize) + + activation = await plugin.initialize_from_plugins_toml() + + assert captured == [(None, None)] + assert not activation.is_active + await activation.close() + + async def test_empty_dynamic_specs_preserve_static_initialization_path(): with pytest.raises(ValueError, match="at least one dynamic plugin"): await plugin.initialize_with_dynamic_plugins(plugin.PluginConfig(), []) @@ -282,6 +386,567 @@ async def test_empty_dynamic_specs_preserve_static_initialization_path(): await plugin.clear_async() +def test_file_initializer_is_exported_only_from_plugin_module(): + assert plugin.initialize_from_plugins_toml is not None + assert plugin.PluginFileActivation is not None + assert not hasattr(nemo_relay, "initialize_from_plugins_toml") + assert not hasattr(nemo_relay, "PluginFileActivation") + + +async def test_file_activation_without_input_is_inactive_and_user_scope_skips_project( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + static_kind = "python.fixture.user-scope-project" + + class ProjectPlugin: + def validate(self, _plugin_config): + return None + + def register(self, _plugin_config, context): + context.register_tool_request_intercept( + "mark-project", + 0, + False, + lambda _name, args: {**args, "project_loaded": True}, + ) + + project_config = tmp_path / ".nemo-relay" + project_config.mkdir() + (project_config / "plugins.toml").write_text( + textwrap.dedent( + f""" + version = 1 + + [[components]] + kind = {static_kind!r} + enabled = true + """ + ) + ) + isolated_user_config = tmp_path / "xdg" + isolated_user_config.mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_CONFIG_HOME", str(isolated_user_config)) + monkeypatch.setenv("NEMO_RELAY_CONFIG_SCOPE", "user") + + plugin.register(static_kind, cast(plugin.Plugin, ProjectPlugin())) + try: + activation = await plugin.initialize_from_plugins_toml() + assert not activation.is_active + assert activation.report == {"diagnostics": []} + result = await tools.execute("python-file-user-scope", {"input": True}, lambda args: args) + assert result == {"input": True} + await activation.close() + await activation.close() + assert not activation.is_active + finally: + plugin.deregister(static_kind) + + +async def test_explicit_empty_config_acquires_file_activation_ownership( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + isolated_user_config = tmp_path / "xdg" + isolated_user_config.mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_CONFIG_HOME", str(isolated_user_config)) + monkeypatch.setenv("NEMO_RELAY_CONFIG_SCOPE", "user") + + activation = await plugin.initialize_from_plugins_toml( + plugin.PluginConfig(), + plugin_config_path=tmp_path / "missing-plugins.toml", + ) + try: + assert activation.is_active + assert activation.report == {"diagnostics": []} + with pytest.raises(RuntimeError, match="active dynamic plugin host"): + await plugin.initialize_from_plugins_toml(plugin.PluginConfig()) + with pytest.raises(RuntimeError, match="active dynamic plugin host"): + await plugin.initialize(plugin.PluginConfig()) + missing = plugin.DynamicPluginActivationSpec( + plugin_id="python.fixture.blocked-explicit-host", + kind="rust_dynamic", + manifest_ref=str(tmp_path / "missing-relay-plugin.toml"), + ) + with pytest.raises(RuntimeError, match="active dynamic plugin host"): + await plugin.initialize_with_dynamic_plugins(plugin.PluginConfig(), [missing]) + with pytest.raises(RuntimeError, match="active dynamic plugin host"): + await asyncio.to_thread(plugin.clear) + with pytest.raises(RuntimeError, match="active dynamic plugin host"): + await plugin.clear_async() + finally: + await activation.close() + assert not activation.is_active + + +async def test_existing_empty_file_acquires_owned_activation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + plugins_toml = tmp_path / "plugins.toml" + plugins_toml.write_text("") + isolated_user_config = tmp_path / "xdg" + isolated_user_config.mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_CONFIG_HOME", str(isolated_user_config)) + monkeypatch.setenv("NEMO_RELAY_CONFIG_SCOPE", "user") + + activation = await plugin.initialize_from_plugins_toml(plugin_config_path=plugins_toml) + try: + assert activation.is_active + assert activation.report["diagnostics"][0]["code"] == "plugin.configuration_inherited" + finally: + await activation.close() + + +async def test_missing_declared_manifest_maps_to_file_not_found( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + plugins_toml = tmp_path / "plugins.toml" + plugins_toml.write_text('[[plugins.dynamic]]\nmanifest = "missing/relay-plugin.toml"\n') + isolated_user_config = tmp_path / "xdg" + isolated_user_config.mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_CONFIG_HOME", str(isolated_user_config)) + monkeypatch.setenv("NEMO_RELAY_CONFIG_SCOPE", "user") + + with pytest.raises(FileNotFoundError, match=r"missing[\\/]relay-plugin\.toml"): + await plugin.initialize_from_plugins_toml(plugin_config_path=plugins_toml) + assert not plugins_toml.with_name(".dynamic-plugins.json").exists() + + +async def test_selected_static_only_file_is_owned_until_close( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + static_kind = "python.fixture.file-static-owned" + + class FileStaticPlugin: + def validate(self, _plugin_config): + return None + + def register(self, _plugin_config, context): + context.register_tool_request_intercept( + "mark-file-static-owned", + 0, + False, + lambda _name, args: {**args, "file_static_owned": True}, + ) + + plugins_toml = tmp_path / "selected" / "plugins.toml" + plugins_toml.parent.mkdir() + plugins_toml.write_text( + textwrap.dedent( + f""" + version = 1 + + [[components]] + kind = {static_kind!r} + enabled = true + """ + ) + ) + isolated_user_config = tmp_path / "xdg" + isolated_user_config.mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_CONFIG_HOME", str(isolated_user_config)) + monkeypatch.delenv("NEMO_RELAY_CONFIG_SCOPE", raising=False) + + plugin.register(static_kind, cast(plugin.Plugin, FileStaticPlugin())) + activation = None + try: + activation = await plugin.initialize_from_plugins_toml(plugin_config_path=plugins_toml) + assert activation.is_active + assert activation.report["diagnostics"][0]["code"] == "plugin.configuration_inherited" + result = await tools.execute("python-file-static-owned", {"input": True}, lambda args: args) + assert result == {"input": True, "file_static_owned": True} + finally: + if activation is not None: + await activation.close() + plugin.deregister(static_kind) + + result = await tools.execute("python-file-static-after-close", {"input": True}, lambda args: args) + assert result == {"input": True} + + +async def test_file_activation_finalizer_releases_static_callbacks( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + static_kind = "python.fixture.file-static-finalizer" + + class FileStaticPlugin: + def validate(self, _plugin_config): + return None + + def register(self, _plugin_config, context): + context.register_tool_request_intercept( + "mark-file-static-finalizer", + 0, + False, + lambda _name, args: {**args, "file_static_finalizer": True}, + ) + + plugins_toml = tmp_path / "plugins.toml" + plugins_toml.write_text( + textwrap.dedent( + f""" + version = 1 + + [[components]] + kind = {static_kind!r} + enabled = true + """ + ) + ) + isolated_user_config = tmp_path / "xdg" + isolated_user_config.mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_CONFIG_HOME", str(isolated_user_config)) + monkeypatch.setenv("NEMO_RELAY_CONFIG_SCOPE", "user") + + plugin.register(static_kind, cast(plugin.Plugin, FileStaticPlugin())) + try: + activation = await plugin.initialize_from_plugins_toml(plugin_config_path=plugins_toml) + result = await tools.execute("python-file-finalizer-active", {"input": True}, lambda args: args) + assert result["file_static_finalizer"] is True + + del activation + await asyncio.sleep(0) + gc.collect() + for _ in range(100): + result = await tools.execute("python-file-finalizer-poll", {"input": True}, lambda args: args) + if "file_static_finalizer" not in result: + break + await asyncio.sleep(0.01) + assert result == {"input": True} + finally: + plugin.deregister(static_kind) + + +async def test_file_activation_hydrates_disabled_then_loads_enabled_native_plugin( + native_dynamic_plugin: _BuiltPlugin, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + project_config = tmp_path / ".nemo-relay" + project_config.mkdir() + plugins_toml = project_config / "plugins.toml" + plugins_toml.write_text( + textwrap.dedent( + f""" + version = 1 + + [[plugins.dynamic]] + manifest = {native_dynamic_plugin.manifest.as_posix()!r} + """ + ) + ) + isolated_user_config = tmp_path / "xdg" + isolated_user_config.mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_CONFIG_HOME", str(isolated_user_config)) + monkeypatch.delenv("NEMO_RELAY_CONFIG_SCOPE", raising=False) + + hydrated = await plugin.initialize_from_plugins_toml() + try: + assert hydrated.is_active + _assert_inherited_configuration_source(hydrated.report, plugins_toml) + result = await tools.execute("python-file-disabled", {"input": True}, lambda args: args) + assert result == {"input": True} + finally: + await hydrated.close() + + _enable_hydrated_plugin(plugins_toml, native_dynamic_plugin.plugin_id) + + activation = await plugin.initialize_from_plugins_toml() + try: + with pytest.raises(RuntimeError, match="active dynamic plugin host"): + await plugin.initialize_from_plugins_toml(plugin_config_path=plugins_toml) + with pytest.raises(RuntimeError, match="active dynamic plugin host"): + await plugin.initialize(plugin.PluginConfig()) + with pytest.raises(RuntimeError, match="active dynamic plugin host"): + await plugin.initialize_with_dynamic_plugins( + plugin.PluginConfig(), + [native_dynamic_plugin.spec()], + ) + with pytest.raises(RuntimeError, match="active dynamic plugin host"): + await asyncio.to_thread(plugin.clear) + with pytest.raises(RuntimeError, match="active dynamic plugin host"): + await plugin.clear_async() + + result = await tools.execute("python-file-enabled", {"input": True}, lambda args: {"args": args}) + assert result["native_plugin_tool_execution"] is True + finally: + await activation.close() + + +async def test_file_activation_loads_enabled_grpc_worker( + worker_dynamic_plugin: _BuiltPlugin, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + project_config = tmp_path / ".nemo-relay" + project_config.mkdir() + plugins_toml = project_config / "plugins.toml" + plugins_toml.write_text( + textwrap.dedent( + f""" + version = 1 + + [[plugins.dynamic]] + manifest = {worker_dynamic_plugin.manifest.as_posix()!r} + """ + ) + ) + isolated_user_config = tmp_path / "xdg" + isolated_user_config.mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_CONFIG_HOME", str(isolated_user_config)) + monkeypatch.delenv("NEMO_RELAY_CONFIG_SCOPE", raising=False) + + hydrated = await plugin.initialize_from_plugins_toml() + await hydrated.close() + _enable_hydrated_plugin(plugins_toml, worker_dynamic_plugin.plugin_id) + + activation = await plugin.initialize_from_plugins_toml() + try: + result = await tools.execute("python-file-worker", {"input": True}, lambda args: {"args": args}) + assert result["worker_plugin_tool_execution"] is True + assert result["args"]["worker_plugin_tool_execution_request"] is True + finally: + await activation.close() + + +@pytest.mark.skipif(os.name == "nt", reason="requires POSIX lifecycle file locking") +async def test_file_activation_cancellation_before_activation_enqueue_never_loads_code( + native_dynamic_plugin: _BuiltPlugin, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + import fcntl + + plugins_toml = tmp_path / "plugins.toml" + plugins_toml.write_text( + textwrap.dedent( + f""" + version = 1 + + [[plugins.dynamic]] + manifest = {native_dynamic_plugin.manifest.as_posix()!r} + """ + ) + ) + isolated_user_config = tmp_path / "xdg" + isolated_user_config.mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_CONFIG_HOME", str(isolated_user_config)) + monkeypatch.setenv("NEMO_RELAY_CONFIG_SCOPE", "user") + + hydrated = await plugin.initialize_from_plugins_toml(plugin_config_path=plugins_toml) + await hydrated.close() + _enable_hydrated_plugin(plugins_toml, native_dynamic_plugin.plugin_id) + + lock_path = plugins_toml.with_name(".dynamic-plugins.lock") + native_submitted = asyncio.Event() + initialize_native = getattr(plugin, "_initialize_from_plugins_toml") + + async def observe_native_submission( + config: object | None, + *, + plugin_config_path: str | None, + ) -> object: + pending = initialize_native(config, plugin_config_path=plugin_config_path) + native_submitted.set() + return await pending + + monkeypatch.setattr(plugin, "_initialize_from_plugins_toml", observe_native_submission) + with lock_path.open("a+b") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + activation_task = asyncio.create_task(plugin.initialize_from_plugins_toml(plugin_config_path=plugins_toml)) + await asyncio.wait_for(native_submitted.wait(), timeout=5) + activation_task.cancel() + with pytest.raises(asyncio.CancelledError): + await activation_task + + # Lifecycle reconciliation cannot complete while this lock is held, + # so cancellation here is necessarily before the core activation + # plan is queued and no dynamic code can have loaded. + assert native_dynamic_plugin.plugin_id not in plugin.list_kinds() + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + retry = await plugin.initialize_from_plugins_toml(plugin_config_path=plugins_toml) + try: + result = await tools.execute("python-file-cancel-before-enqueue", {"input": True}, lambda args: args) + assert result["native_plugin_tool_execution"] is True + finally: + await retry.close() + + +async def test_file_activation_cancellation_after_enqueue_cleans_undelivered_owner( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + static_kind = "python.fixture.file-cancel-after-enqueue" + register_started = threading.Event() + register_release = threading.Event() + register_finished = threading.Event() + + class BlockingFilePlugin: + def validate(self, _plugin_config): + return None + + def register(self, _plugin_config, context): + register_started.set() + assert register_release.wait(timeout=5) + context.register_tool_request_intercept( + "mark-file-cancel-after-enqueue", + 0, + False, + lambda _name, args: {**args, "file_cancel_after_enqueue": True}, + ) + register_finished.set() + + plugins_toml = tmp_path / "plugins.toml" + plugins_toml.write_text( + textwrap.dedent( + f""" + version = 1 + + [[components]] + kind = {static_kind!r} + enabled = true + """ + ) + ) + isolated_user_config = tmp_path / "xdg" + isolated_user_config.mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_CONFIG_HOME", str(isolated_user_config)) + monkeypatch.setenv("NEMO_RELAY_CONFIG_SCOPE", "user") + + plugin.register(static_kind, cast(plugin.Plugin, BlockingFilePlugin())) + activation_task = asyncio.create_task(plugin.initialize_from_plugins_toml(plugin_config_path=plugins_toml)) + recovered = None + try: + assert await asyncio.wait_for(asyncio.to_thread(register_started.wait, 5), timeout=6) + activation_task.cancel() + with pytest.raises(asyncio.CancelledError): + await activation_task + + register_release.set() + assert await asyncio.wait_for(asyncio.to_thread(register_finished.wait, 5), timeout=6) + + async with asyncio.timeout(5): + while recovered is None: + try: + recovered = await plugin.initialize_from_plugins_toml( + plugin.PluginConfig(), + plugin_config_path=tmp_path / "missing-plugins.toml", + ) + except RuntimeError as error: + assert "active dynamic plugin host" in str(error) + await asyncio.sleep(0) + + result = await tools.execute("python-file-cancel-after-enqueue", {"input": True}, lambda args: args) + assert result == {"input": True} + finally: + register_release.set() + await asyncio.gather(activation_task, return_exceptions=True) + if recovered is not None: + await recovered.close() + plugin.deregister(static_kind) + + +async def test_file_activation_mixed_static_native_and_command_worker_rolls_back_and_retries( + native_dynamic_plugin: _BuiltPlugin, + worker_dynamic_plugin: _BuiltPlugin, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + static_kind = "python.fixture.file-mixed-host" + + class MixedFilePlugin: + def validate(self, _plugin_config): + return None + + def register(self, _plugin_config, context): + context.register_tool_request_intercept( + "mark-file-mixed-host", + 0, + False, + lambda _name, args: {**args, "file_mixed_static": True}, + ) + + def write_plugins_toml(*, worker_register_error: bool) -> None: + worker_config = "config = { register_error = true }" if worker_register_error else "" + plugins_toml.write_text( + textwrap.dedent( + f""" + version = 1 + + [[components]] + kind = {static_kind!r} + enabled = true + + [[plugins.dynamic]] + manifest = {native_dynamic_plugin.manifest.as_posix()!r} + + [[plugins.dynamic]] + manifest = {worker_dynamic_plugin.manifest.as_posix()!r} + {worker_config} + """ + ) + ) + + plugins_toml = tmp_path / "plugins.toml" + write_plugins_toml(worker_register_error=True) + isolated_user_config = tmp_path / "xdg" + isolated_user_config.mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_CONFIG_HOME", str(isolated_user_config)) + monkeypatch.setenv("NEMO_RELAY_CONFIG_SCOPE", "user") + + plugin.register(static_kind, cast(plugin.Plugin, MixedFilePlugin())) + activation = None + try: + hydrated = await plugin.initialize_from_plugins_toml(plugin_config_path=plugins_toml) + await hydrated.close() + _enable_hydrated_plugin(plugins_toml, native_dynamic_plugin.plugin_id) + _enable_hydrated_plugin(plugins_toml, worker_dynamic_plugin.plugin_id) + + with pytest.raises(RuntimeError, match="fixture registration error requested"): + await plugin.initialize_from_plugins_toml(plugin_config_path=plugins_toml) + + assert native_dynamic_plugin.plugin_id not in plugin.list_kinds() + assert worker_dynamic_plugin.plugin_id not in plugin.list_kinds() + result = await tools.execute("python-file-mixed-after-rollback", {"input": True}, lambda args: args) + assert result == {"input": True} + + write_plugins_toml(worker_register_error=False) + activation = await plugin.initialize_from_plugins_toml(plugin_config_path=plugins_toml) + result = await tools.execute("python-file-mixed-success", {"input": True}, lambda args: {"args": args}) + assert result["native_plugin_tool_execution"] is True + assert result["worker_plugin_tool_execution"] is True + assert result["args"]["native_plugin_tool_execution_request"] is True + assert result["args"]["worker_plugin_tool_execution_request"] is True + assert result["args"]["file_mixed_static"] is True + finally: + if activation is not None: + await activation.close() + plugin.deregister(static_kind) + + assert native_dynamic_plugin.plugin_id not in plugin.list_kinds() + assert worker_dynamic_plugin.plugin_id not in plugin.list_kinds() + result = await tools.execute("python-file-mixed-after-close", {"input": True}, lambda args: args) + assert result == {"input": True} + + async def test_native_activation_context_owns_callbacks_and_close_is_idempotent( native_dynamic_plugin: _BuiltPlugin, ): @@ -344,15 +1009,7 @@ def register(self, _plugin_config, context): activation = None try: activation = await plugin.initialize_with_dynamic_plugins(plugin.PluginConfig(), [native_dynamic_plugin.spec()]) - assert activation.report == { - "diagnostics": [ - { - "level": "warning", - "code": "plugin.configuration_inherited", - "message": f"inherited plugin configuration from discovered file: {plugins_toml.resolve()}", - } - ] - } + _assert_inherited_configuration_source(activation.report, plugins_toml) result = await tools.execute("python-file-static-base", {"input": True}, lambda args: args) assert result["file_static_base"] is True assert result["native_plugin_tool_execution"] is True @@ -488,7 +1145,10 @@ async def test_worker_activation_finalizer_never_waits_on_python_thread( tmp_path: Path, ): with worker_dynamic_plugin.manifest.open("rb") as file: - worker_entrypoint = Path(tomllib.load(file)["load"]["entrypoint"]) + worker_entrypoint_value = str(tomllib.load(file)["load"]["entrypoint"]) + worker_entrypoint = Path(worker_entrypoint_value) + if not worker_entrypoint.is_absolute(): + worker_entrypoint = worker_dynamic_plugin.manifest.parent / worker_entrypoint pid_file = tmp_path / "worker.pid" wrapper = tmp_path / "worker-wrapper.sh" @@ -497,7 +1157,7 @@ async def test_worker_activation_finalizer_never_waits_on_python_thread( manifest = tmp_path / "relay-plugin.toml" manifest.write_text( worker_dynamic_plugin.manifest.read_text().replace( - f"entrypoint = {str(worker_entrypoint)!r}", + f"entrypoint = {worker_entrypoint_value!r}", f"entrypoint = {str(wrapper)!r}", ) )