From 3c5eff73f9a09a7a5d34d16cef30da6eebc1cdc0 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Mon, 19 Oct 2026 23:53:17 -0700 Subject: [PATCH 1/4] feat(sidecar): expose host-function globals in every guest Node process --- crates/execution/src/javascript.rs | 12 +- .../src/execution/child_process.rs | 6 +- crates/native-sidecar/src/execution/launch.rs | 19 +- crates/native-sidecar/src/execution/mod.rs | 2 +- crates/native-sidecar/src/host_functions.rs | 110 +++++++++ .../native-sidecar/src/language_execution.rs | 132 ----------- docs/content/docs/host-functions.mdx | 10 +- docs/content/docs/javascript.mdx | 4 +- .../embedded/host-functions-javascript.ts | 12 + packages/core/package.json | 2 +- .../host-function-globals.nightly.test.ts | 68 ++++++ .../core/tests/host-function-globals.test.ts | 216 ++++++++++++++++++ .../docs/content/docs/host-functions.mdx | 2 + 13 files changed, 452 insertions(+), 143 deletions(-) create mode 100644 packages/core/tests/host-function-globals.nightly.test.ts create mode 100644 packages/core/tests/host-function-globals.test.ts diff --git a/crates/execution/src/javascript.rs b/crates/execution/src/javascript.rs index d16d075b93..8303f62037 100644 --- a/crates/execution/src/javascript.rs +++ b/crates/execution/src/javascript.rs @@ -515,6 +515,12 @@ pub struct GuestRuntimeConfig { /// keeps the bridge-only snapshot (unchanged behavior). The runtime caches the /// snapshot process-wide keyed by sha256(bridge_code + this bundle). pub snapshot_userland_code: Option, + /// Optional sidecar-owned classic script evaluated in the execution's + /// context before the entrypoint, including before an ES module entrypoint + /// links its static imports. The sidecar uses it to define VM-wide guest + /// globals such as host-function collections. Runner isolates for WASM and + /// Python leave it unset. + pub prelude_script: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -2954,7 +2960,11 @@ impl JavascriptExecutionEngine { mode: u8::from(use_module_mode) | if retain_context { 2 } else { 0 }, file_path: execution_file_path, bridge_code: V8RuntimeHost::bridge_code().to_owned(), - post_restore_script: String::new(), + post_restore_script: request + .guest_runtime + .prelude_script + .clone() + .unwrap_or_default(), userland_code: snapshot_userland_code, high_resolution_time: request.guest_runtime.high_resolution_time, user_code, diff --git a/crates/native-sidecar/src/execution/child_process.rs b/crates/native-sidecar/src/execution/child_process.rs index b69a626ba0..60e7e8b60e 100644 --- a/crates/native-sidecar/src/execution/child_process.rs +++ b/crates/native-sidecar/src/execution/child_process.rs @@ -4893,7 +4893,7 @@ where let execution_result = javascript_engine .start_execution_with_module_reader_and_runtime( StartJavascriptExecutionRequest { - guest_runtime: guest_runtime_identity( + guest_runtime: javascript_guest_runtime( &vm, Some(u64::from(kernel_pid)), Some(u64::from(parent_kernel_pid)), @@ -5575,7 +5575,7 @@ where let replacement_result = javascript_engine .prepare_execution_with_module_reader_and_runtime( StartJavascriptExecutionRequest { - guest_runtime: guest_runtime_identity( + guest_runtime: javascript_guest_runtime( vm, Some(u64::from(kernel_pid)), Some(u64::from(parent_kernel_pid)), @@ -6515,7 +6515,7 @@ where let execution_result = javascript_engine .start_execution_with_module_reader_and_runtime( StartJavascriptExecutionRequest { - guest_runtime: guest_runtime_identity( + guest_runtime: javascript_guest_runtime( &vm, Some(u64::from(kernel_pid)), Some(u64::from(parent_kernel_pid)), diff --git a/crates/native-sidecar/src/execution/launch.rs b/crates/native-sidecar/src/execution/launch.rs index cf087ba904..109ffc9085 100644 --- a/crates/native-sidecar/src/execution/launch.rs +++ b/crates/native-sidecar/src/execution/launch.rs @@ -3023,9 +3023,26 @@ pub(super) fn guest_runtime_identity( .as_ref() .is_some_and(|cfg| cfg.high_resolution_time.unwrap_or(false)), snapshot_userland_code: None, + prelude_script: None, } } +/// Guest-runtime config for a guest Node process. Adds the VM's host-function +/// globals to the shared identity so every guest Node process, however it was +/// launched, can call registered host functions directly. +pub(super) fn javascript_guest_runtime( + vm: &VmState, + virtual_pid: Option, + virtual_ppid: Option, +) -> GuestRuntimeConfig { + let mut config = guest_runtime_identity(vm, virtual_pid, virtual_ppid); + let script = host_function_globals_script(&vm.host_functions); + if !script.is_empty() { + config.prelude_script = Some(script); + } + config +} + /// The guest's virtual home directory, sourced from the VM user profile (the /// same value carried to the guest as `os.homedir()` via `guest_runtime`). Used /// by sidecar-internal `~`-path resolution; falls back to `/root` for a @@ -5076,7 +5093,7 @@ where let execution = javascript_engine .start_execution_with_module_reader_and_runtime( StartJavascriptExecutionRequest { - guest_runtime: guest_runtime_identity(&vm, None, None), + guest_runtime: javascript_guest_runtime(&vm, None, None), vm_id: vm_id.clone(), context_id: context.context_id, argv: std::iter::once(launch_entrypoint.clone()) diff --git a/crates/native-sidecar/src/execution/mod.rs b/crates/native-sidecar/src/execution/mod.rs index 2a2a0034dc..60495fb929 100644 --- a/crates/native-sidecar/src/execution/mod.rs +++ b/crates/native-sidecar/src/execution/mod.rs @@ -102,7 +102,7 @@ use crate::filesystem::{ service_owned_python_filesystem_rpc_request, }; use crate::host_functions::{ - format_host_function_failure_output, is_host_function_command, + format_host_function_failure_output, host_function_globals_script, is_host_function_command, normalized_host_function_command_name, resolve_host_function_command, HostFunctionCommandResolution, }; diff --git a/crates/native-sidecar/src/host_functions.rs b/crates/native-sidecar/src/host_functions.rs index dc26956a13..4b3641f2ba 100644 --- a/crates/native-sidecar/src/host_functions.rs +++ b/crates/native-sidecar/src/host_functions.rs @@ -715,6 +715,59 @@ fn host_function_registration_error(error: HostFunctionRegistrationError) -> Sid } } +/// Convert a kebab-case host function or collection name to the camelCase +/// identifier +/// guest JavaScript uses for it. +fn host_function_identifier(name: &str) -> String { + let mut identifier = String::with_capacity(name.len()); + let mut upper_next = false; + for character in name.chars() { + if character == '-' { + upper_next = true; + } else if upper_next { + identifier.extend(character.to_uppercase()); + upper_next = false; + } else { + identifier.push(character); + } + } + identifier +} + +/// Guest script that defines one frozen global per registered host function +/// collection, so guest JavaScript can `await orders.list({ ... })` instead of +/// spawning the `agentos-orders` command by hand. The sidecar runs it before +/// the entrypoint of every guest Node process in the VM. Each function runs +/// that same command, so the `hostFunction` permission scope, input +/// validation, and timeouts all still apply. A collection whose name is +/// already a global is skipped with a warning on stderr rather than shadowing +/// it. Returns an empty string when the VM has no host functions. +pub(crate) fn host_function_globals_script( + host_functions: &BTreeMap, +) -> String { + const DEFINE_GLOBALS: &str = r#"((groups)=>{const call=(command,name)=>(input={})=>import("node:child_process").then(({execFile})=>new Promise((resolve,reject)=>{execFile(command,[name,"--json",JSON.stringify(input)],{maxBuffer:67108864},(error,stdout,stderr)=>{if(error){reject(new Error(String(stderr||"").trim()||error.message));return;}let reply;try{reply=JSON.parse(String(stdout));}catch(parseError){reject(new Error("host function returned invalid JSON: "+parseError.message));return;}if(!reply||reply.ok!==true){reject(new Error((reply&&reply.error)||"host function failed"));return;}resolve(reply.result);});}));for(const [group,definition] of Object.entries(groups)){if(group in globalThis){console.error("agentos: host function group \""+group+"\" was not defined because globalThis."+group+" already exists");continue;}const functions={};for(const [key,name] of definition.functions)functions[key]=call(definition.command,name);Object.defineProperty(globalThis,group,{value:Object.freeze(functions),enumerable:true});}})"#; + + let groups = host_functions + .values() + .filter_map(|collection| { + let command = collection.command_aliases.first()?; + let functions = collection + .callbacks + .keys() + .map(|name| serde_json::json!([host_function_identifier(name), name])) + .collect::>(); + Some(( + host_function_identifier(&collection.name), + serde_json::json!({ "command": command, "functions": functions }), + )) + }) + .collect::>(); + if groups.is_empty() { + return String::new(); + } + format!("{DEFINE_GLOBALS}({});", serde_json::Value::Object(groups)) +} + enum HostFunctionCommand { Registry(String), Collection { collection_name: String }, @@ -1042,3 +1095,60 @@ mod tests { ); } } + +#[cfg(test)] +mod host_function_globals_tests { + use super::*; + use crate::protocol::RegisteredHostCallbackDefinition; + + fn collection(name: &str, callbacks: &[&str]) -> RegisterHostCallbacksRequest { + RegisterHostCallbacksRequest { + name: name.to_string(), + description: String::new(), + command_aliases: vec![format!("agentos-{name}")], + registry_command_aliases: vec![String::from("agentos")], + callbacks: callbacks + .iter() + .map(|callback| { + ( + callback.to_string(), + RegisteredHostCallbackDefinition { + description: String::new(), + input_schema: String::from("{}"), + timeout_ms: None, + examples: Vec::new(), + }, + ) + }) + .collect(), + } + } + + #[test] + fn converts_kebab_case_names_to_identifiers() { + assert_eq!(host_function_identifier("orders"), "orders"); + assert_eq!(host_function_identifier("order-store"), "orderStore"); + assert_eq!( + host_function_identifier("list-open-orders"), + "listOpenOrders" + ); + } + + #[test] + fn is_empty_without_host_functions() { + assert_eq!(host_function_globals_script(&BTreeMap::new()), ""); + } + + #[test] + fn defines_one_group_per_collection_on_a_single_line() { + let host_functions = BTreeMap::from([( + String::from("order-store"), + collection("order-store", &["list-orders"]), + )]); + let script = host_function_globals_script(&host_functions); + assert!(!script.contains('\n'), "script must stay on one line"); + assert!(script.ends_with( + r#"({"orderStore":{"command":"agentos-order-store","functions":[["listOrders","list-orders"]]}});"# + )); + } +} diff --git a/crates/native-sidecar/src/language_execution.rs b/crates/native-sidecar/src/language_execution.rs index 8a37a2fd6e..3e91972dca 100644 --- a/crates/native-sidecar/src/language_execution.rs +++ b/crates/native-sidecar/src/language_execution.rs @@ -115,59 +115,6 @@ fn inline_inputs_prefix(inputs: Option, python: bool) -> String { } } -/// Convert a kebab-case host function or collection name to the camelCase -/// identifier -/// guest JavaScript uses for it. -fn host_function_identifier(name: &str) -> String { - let mut identifier = String::with_capacity(name.len()); - let mut upper_next = false; - for character in name.chars() { - if character == '-' { - upper_next = true; - } else if upper_next { - identifier.extend(character.to_uppercase()); - upper_next = false; - } else { - identifier.push(character); - } - } - identifier -} - -/// Guest prefix that defines one frozen global per registered host function -/// collection, so inline JavaScript can `await orders.list({ ... })` instead of -/// spawning the `agentos-orders` command by hand. Each function runs that same -/// command, so the `hostFunction` permission scope, input validation, and -/// timeouts -/// all still apply. The prefix stays on one line so guest line numbers do not -/// shift. A collection whose name is already a global is skipped with a warning -/// on stderr rather than shadowing it. -fn inline_host_functions_prefix( - host_functions: &BTreeMap, -) -> String { - const DEFINE_GLOBALS: &str = r#"((groups)=>{const call=(command,name)=>(input={})=>import("node:child_process").then(({execFile})=>new Promise((resolve,reject)=>{execFile(command,[name,"--json",JSON.stringify(input)],{maxBuffer:67108864},(error,stdout,stderr)=>{if(error){reject(new Error(String(stderr||"").trim()||error.message));return;}let reply;try{reply=JSON.parse(String(stdout));}catch(parseError){reject(new Error("host function returned invalid JSON: "+parseError.message));return;}if(!reply||reply.ok!==true){reject(new Error((reply&&reply.error)||"host function failed"));return;}resolve(reply.result);});}));for(const [group,definition] of Object.entries(groups)){if(group in globalThis){console.error("agentos: host function group \""+group+"\" was not defined because globalThis."+group+" already exists");continue;}const functions={};for(const [key,name] of definition.functions)functions[key]=call(definition.command,name);Object.defineProperty(globalThis,group,{value:Object.freeze(functions),enumerable:true});}})"#; - - let groups = host_functions - .values() - .filter_map(|collection| { - let command = collection.command_aliases.first()?; - let functions = collection - .callbacks - .keys() - .map(|name| serde_json::json!([host_function_identifier(name), name])) - .collect::>(); - Some(( - host_function_identifier(&collection.name), - serde_json::json!({ "command": command, "functions": functions }), - )) - }) - .collect::>(); - if groups.is_empty() { - return String::new(); - } - format!("{DEFINE_GLOBALS}({});", serde_json::Value::Object(groups)) -} - fn semantic_result_path() -> String { static NEXT_RESULT_ID: AtomicU64 = AtomicU64::new(1); let nonce = NEXT_RESULT_ID.fetch_add(1, Ordering::Relaxed); @@ -1331,26 +1278,6 @@ where "retainEvents requires a context or spawned language process", )); } - // Inline JavaScript and TypeScript run as `node -e `. Give them the - // VM's registered host functions as globals. - if operation.retained_language == Some(RetainedExecutionLanguage::JavaScript) - && operation.args.first().map(String::as_str) == Some("-e") - { - let prefix = self - .vms - .get(&vm_id) - .map(|vm| inline_host_functions_prefix(&vm.host_functions)) - .unwrap_or_default(); - if !prefix.is_empty() { - if let Some(source) = operation.args.get_mut(1) { - source.insert_str(0, &prefix); - } - if let Some(source) = operation.retained_source.as_mut() { - source.insert_str(0, &prefix); - } - } - } - if !context_execution { // Ephemeral calls must not park a language interpreter after the // operation completes. @@ -3107,62 +3034,3 @@ fn execution_limit_error(limit: ExecutionLimit) -> ExecutionErrorData { details: None, } } - -#[cfg(test)] -mod inline_host_functions_prefix_tests { - use super::*; - - fn collection(name: &str, callbacks: &[&str]) -> RegisterHostCallbacksRequest { - RegisterHostCallbacksRequest { - name: name.to_string(), - description: String::new(), - command_aliases: vec![format!("agentos-{name}")], - registry_command_aliases: vec![String::from("agentos")], - callbacks: callbacks - .iter() - .map(|callback| { - ( - callback.to_string(), - RegisteredHostCallbackDefinition { - description: String::new(), - input_schema: String::from("{}"), - timeout_ms: None, - examples: Vec::new(), - }, - ) - }) - .collect(), - } - } - - #[test] - fn converts_kebab_case_names_to_identifiers() { - assert_eq!(host_function_identifier("orders"), "orders"); - assert_eq!(host_function_identifier("order-store"), "orderStore"); - assert_eq!( - host_function_identifier("list-open-orders"), - "listOpenOrders" - ); - } - - #[test] - fn is_empty_without_host_functions() { - assert_eq!(inline_host_functions_prefix(&BTreeMap::new()), ""); - } - - #[test] - fn defines_one_group_per_collection_on_a_single_line() { - let host_functions = BTreeMap::from([( - String::from("order-store"), - collection("order-store", &["list-orders"]), - )]); - let prefix = inline_host_functions_prefix(&host_functions); - assert!( - !prefix.contains('\n'), - "prefix must not shift guest line numbers" - ); - assert!(prefix.ends_with( - r#"({"orderStore":{"command":"agentos-order-store","functions":[["listOrders","list-orders"]]}});"# - )); - } -} diff --git a/docs/content/docs/host-functions.mdx b/docs/content/docs/host-functions.mdx index 819abe25e7..74546a67cf 100644 --- a/docs/content/docs/host-functions.mdx +++ b/docs/content/docs/host-functions.mdx @@ -4,7 +4,7 @@ description: "Expose trusted host functions to guest programs in agentOS VMs as skill: true --- -Host functions expose JavaScript functions from your process to code running inside the VM. They run as in-process callbacks, so they are available in the library only, not on hosted VMs. Each function is defined with a Zod input schema and becomes an auto-generated CLI command at `/bin/agentos-{name}`, plus an async function that inline JavaScript can call directly. +Host functions expose JavaScript functions from your process to code running inside the VM. They run as in-process callbacks, so they are available in the library only, not on hosted VMs. Each function is defined with a Zod input schema and becomes an auto-generated CLI command at `/bin/agentos-{name}`, plus an async function that guest JavaScript can call directly. ## Getting started @@ -66,9 +66,13 @@ Missing required flag: --city ## Call host functions from JavaScript -Inline JavaScript and TypeScript get each collection as a frozen global, and each +Every guest Node process gets each collection as a frozen global, and each host function as an async function, so generated code calls your tools like any -other API. Keys written in camelCase become kebab-case commands and camelCase guest +other API. This covers inline `vm.javascript` and `vm.typescript` executions, +scripts started with `vm.process.spawn("node", ...)` or `vm.process.exec`, +`node` run from the guest shell, and child processes of guest Node. The globals +are defined before the entry script and its imports run, so a module can use +them at its top level. Keys written in camelCase become kebab-case commands and camelCase guest identifiers, so `orderStore.listOrders` is `agentos-order-store list-orders` on the CLI and `orderStore.listOrders(input)` in guest JavaScript. diff --git a/docs/content/docs/javascript.mdx b/docs/content/docs/javascript.mdx index 4444772178..aac660617f 100644 --- a/docs/content/docs/javascript.mdx +++ b/docs/content/docs/javascript.mdx @@ -192,7 +192,9 @@ See [Filesystem](/agentos/docs/filesystem), [Processes & Shells](/agentos/docs/p ## Host functions Guest JavaScript calls [host functions](/agentos/docs/host-functions) as async globals, so host -credentials stay outside the VM. Host functions are library-only. +credentials stay outside the VM. The globals exist in every guest Node process, +whether it runs inline code, a spawned script, `node` from the shell, or a child +process. Host functions are library-only. diff --git a/examples/embedded/host-functions-javascript.ts b/examples/embedded/host-functions-javascript.ts index 71cbe9e42a..8c38bb64f3 100644 --- a/examples/embedded/host-functions-javascript.ts +++ b/examples/embedded/host-functions-javascript.ts @@ -31,6 +31,18 @@ try { { output: { capture: "all" } }, ); console.log(result.outcome === "succeeded" ? result.stdout : result.error); + + // Every guest Node process gets the same globals, including scripts started + // with process.spawn(), from the shell, or by another guest process. + await vm.writeFile( + "/app/report.mjs", + `const forecast = await weather.forecast({ city: "Tokyo" }); +console.log(forecast);`, + ); + const report = await vm.process.exec("node /app/report.mjs", { + output: { capture: "all" }, + }); + console.log(report.outcome === "succeeded" ? report.stdout : report.error); } finally { await vm.dispose(); } diff --git a/packages/core/package.json b/packages/core/package.json index cd685e92f1..ae0133f96f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -54,7 +54,7 @@ "check-types": "tsc --noEmit && tsc -p tsconfig.type-tests.json", "build": "tsc && node scripts/stage-default-software.mjs", "test": "vitest run --exclude '**/*.nightly.test.ts' --reporter=verbose", - "test:unit": "vitest run tests/agentos-package.test.ts tests/allowed-node-builtins.test.ts tests/host-functions-zod.test.ts tests/host-functions.test.ts tests/host-function-command-exec.test.ts tests/host-function-permissions.test.ts tests/cron-manager.test.ts tests/code-mode-execution.test.ts tests/cron-timer-driver.test.ts tests/exec-timeout.test.ts tests/execution-abort.test.ts tests/generated-protocol.test.ts tests/leak-agent-os-processes.test.ts tests/leak-rpc-client.test.ts tests/mount-descriptors.test.ts tests/mount-reconfigure.test.ts tests/options-schema.test.ts tests/output-replay.test.ts tests/public-api-exports.test.ts tests/root-filesystem-descriptors.test.ts tests/runtime-compat-mount.test.ts tests/sidecar-client.test.ts tests/sidecar-exit.test.ts tests/sidecar-permission-descriptors.test.ts tests/software-install.test.ts tests/wasm-permission-tiers.test.ts --fileParallelism=false", + "test:unit": "vitest run tests/agentos-package.test.ts tests/allowed-node-builtins.test.ts tests/host-functions-zod.test.ts tests/host-functions.test.ts tests/host-function-command-exec.test.ts tests/host-function-permissions.test.ts tests/host-function-globals.test.ts tests/cron-manager.test.ts tests/code-mode-execution.test.ts tests/cron-timer-driver.test.ts tests/exec-timeout.test.ts tests/execution-abort.test.ts tests/generated-protocol.test.ts tests/leak-agent-os-processes.test.ts tests/leak-rpc-client.test.ts tests/mount-descriptors.test.ts tests/mount-reconfigure.test.ts tests/options-schema.test.ts tests/output-replay.test.ts tests/public-api-exports.test.ts tests/root-filesystem-descriptors.test.ts tests/runtime-compat-mount.test.ts tests/sidecar-client.test.ts tests/sidecar-exit.test.ts tests/sidecar-permission-descriptors.test.ts tests/software-install.test.ts tests/wasm-permission-tiers.test.ts --fileParallelism=false", "test:pr": "pnpm test:unit", "test:nightly": "vitest run tests/*.nightly.test.ts --reporter=verbose --passWithNoTests" }, diff --git a/packages/core/tests/host-function-globals.nightly.test.ts b/packages/core/tests/host-function-globals.nightly.test.ts new file mode 100644 index 0000000000..256d500007 --- /dev/null +++ b/packages/core/tests/host-function-globals.nightly.test.ts @@ -0,0 +1,68 @@ +// Nightly: launches Node through the guest shell, which needs the default +// software bundle (`sh` from coreutils). +import { afterEach, describe, expect, test } from "vitest"; +import { z } from "zod"; +import { AgentOs } from "../src/index.js"; +import { ALLOW_ALL_VM_PERMISSIONS } from "./helpers/permissions.js"; + +const orderStore = { + listOrders: { + inputSchema: z.object({ status: z.string() }), + execute: ({ status }: { status: string }) => ({ + orders: [{ id: "order-1", status }], + }), + }, +}; + +const EXPECTED_RESULT = { orders: [{ id: "order-1", status: "open" }] }; + +function lastJsonLine(stdout: string): unknown { + const lines = stdout.trim().split("\n"); + return JSON.parse(lines[lines.length - 1] ?? ""); +} + +describe("host-function globals in shell-launched Node processes", () => { + let vm: AgentOs | undefined; + + afterEach(async () => { + await vm?.dispose(); + vm = undefined; + }); + + test("node run from the guest shell can call host functions", async () => { + vm = await AgentOs.create({ + permissions: ALLOW_ALL_VM_PERMISSIONS, + hostFunctions: { orderStore }, + }); + await vm.writeFile( + "/app/main.mjs", + `const result = await orderStore.listOrders({ status: "open" }); +console.log(JSON.stringify(result));`, + ); + + const decoder = new TextDecoder(); + let stdout = ""; + let stderr = ""; + const { pid } = await vm.process.spawn("sh", ["-c", "node /app/main.mjs"], { + onStdout: (chunk) => { + stdout += decoder.decode(chunk); + }, + onStderr: (chunk) => { + stderr += decoder.decode(chunk); + }, + }); + const exit = await vm.process.wait(pid); + expect({ exitCode: exit.exitCode, stderr }, stdout).toEqual({ + exitCode: 0, + stderr: "", + }); + expect(lastJsonLine(stdout)).toEqual(EXPECTED_RESULT); + + const exec = await vm.process.exec("node /app/main.mjs", { + output: { capture: "all" }, + }); + expect(exec.outcome, JSON.stringify(exec)).toBe("succeeded"); + if (exec.outcome !== "succeeded") return; + expect(lastJsonLine(exec.stdout ?? "")).toEqual(EXPECTED_RESULT); + }, 120_000); +}); diff --git a/packages/core/tests/host-function-globals.test.ts b/packages/core/tests/host-function-globals.test.ts new file mode 100644 index 0000000000..80edefc4c9 --- /dev/null +++ b/packages/core/tests/host-function-globals.test.ts @@ -0,0 +1,216 @@ +import { afterEach, describe, expect, test } from "vitest"; +import { z } from "zod"; +import { AgentOs } from "../src/index.js"; +import { ALLOW_ALL_VM_PERMISSIONS } from "./helpers/permissions.js"; + +// Every guest Node process in a VM with host functions gets one frozen global +// per collection, not only inline `vm.javascript.*` executions. These tests +// launch Node the other ways a guest can: a direct spawn and a child process +// of guest Node. The shell launch paths need the default software bundle, so +// they live in host-function-globals.nightly.test.ts. + +const orderStore = { + listOrders: { + inputSchema: z.object({ status: z.string() }), + execute: ({ status }: { status: string }) => ({ + orders: [{ id: "order-1", status }], + }), + }, +}; + +const EXPECTED_RESULT = { orders: [{ id: "order-1", status: "open" }] }; + +const MAIN_SCRIPT = ` +const result = await orderStore.listOrders({ status: "open" }); +console.log(JSON.stringify(result)); +`; + +async function runNode( + vm: AgentOs, + command: string, + args: string[], +): Promise<{ exitCode?: number; stdout: string; stderr: string }> { + const decoder = new TextDecoder(); + let stdout = ""; + let stderr = ""; + const { pid } = await vm.process.spawn(command, args, { + onStdout: (chunk) => { + stdout += decoder.decode(chunk); + }, + onStderr: (chunk) => { + stderr += decoder.decode(chunk); + }, + }); + const exit = await vm.process.wait(pid); + return { exitCode: exit.exitCode, stdout, stderr }; +} + +function lastJsonLine(stdout: string): unknown { + const lines = stdout.trim().split("\n"); + return JSON.parse(lines[lines.length - 1] ?? ""); +} + +describe("host-function globals in guest Node processes", () => { + let vm: AgentOs | undefined; + + afterEach(async () => { + await vm?.dispose(); + vm = undefined; + }); + + test("spawned and child Node processes can call host functions", async () => { + vm = await AgentOs.create({ + permissions: ALLOW_ALL_VM_PERMISSIONS, + hostFunctions: { orderStore }, + }); + await vm.writeFile("/app/main.mjs", MAIN_SCRIPT); + await vm.writeFile( + "/app/main.cjs", + `orderStore.listOrders({ status: "open" }).then((result) => { + console.log(JSON.stringify(result)); + });`, + ); + // A static import evaluates before the importing module's body, so this + // also covers modules that use the global at their top level. + await vm.writeFile( + "/app/orders.mjs", + `export const result = await orderStore.listOrders({ status: "open" });`, + ); + await vm.writeFile( + "/app/import.mjs", + `import { result } from "./orders.mjs"; +console.log(JSON.stringify(result));`, + ); + await vm.writeFile( + "/app/parent.mjs", + `import { execFile } from "node:child_process"; +execFile("node", ["/app/main.mjs"], (error, stdout, stderr) => { + if (error) { + console.error(stderr || error.message); + process.exit(1); + } + process.stdout.write(stdout); +});`, + ); + + const cases: Array<[string, string, string[]]> = [ + ["process.spawn ESM", "node", ["/app/main.mjs"]], + ["process.spawn CommonJS", "node", ["/app/main.cjs"]], + ["process.spawn static import", "node", ["/app/import.mjs"]], + ["guest child_process", "node", ["/app/parent.mjs"]], + ]; + for (const [label, command, args] of cases) { + const result = await runNode(vm, command, args); + expect( + { label, exitCode: result.exitCode, stderr: result.stderr }, + `stdout: ${result.stdout}`, + ).toEqual({ label, exitCode: 0, stderr: "" }); + expect(lastJsonLine(result.stdout), label).toEqual(EXPECTED_RESULT); + } + + // The generated CLI keeps working alongside the globals. + const cli = await runNode(vm, "agentos-order-store", [ + "list-orders", + "--status", + "open", + ]); + expect(cli.exitCode, cli.stderr).toBe(0); + expect(JSON.parse(cli.stdout)).toEqual({ + ok: true, + result: EXPECTED_RESULT, + }); + }, 120_000); + + test("inline executions keep the globals across a retained context", async () => { + vm = await AgentOs.create({ + permissions: ALLOW_ALL_VM_PERMISSIONS, + hostFunctions: { orderStore }, + }); + await vm.createContext("orders"); + for (let run = 0; run < 2; run++) { + const result = await vm.javascript.evaluate( + `orderStore.listOrders({ status: "open" })`, + { contextId: "orders" }, + ); + expect(result, `run ${run}`).toMatchObject({ + exitCode: 0, + value: EXPECTED_RESULT, + }); + expect(JSON.stringify(result), `run ${run}`).not.toContain( + "was not defined", + ); + } + }, 60_000); + + test("the hostFunction permission still applies to the globals", async () => { + vm = await AgentOs.create({ + permissions: { + ...ALLOW_ALL_VM_PERMISSIONS, + hostFunction: { default: "deny", rules: [] }, + }, + hostFunctions: { orderStore }, + }); + await vm.writeFile( + "/app/denied.mjs", + `try { + await orderStore.listOrders({ status: "open" }); + console.log("ALLOWED"); +} catch (error) { + console.log("DENIED " + error.message); +}`, + ); + + const result = await runNode(vm, "node", ["/app/denied.mjs"]); + expect(result.exitCode, result.stderr).toBe(0); + expect(result.stdout).toContain("DENIED"); + expect(result.stdout).toContain("hostFunction.invoke"); + expect(result.stdout).toContain("order-store:list-orders"); + }, 60_000); + + test("no globals are defined without host functions", async () => { + vm = await AgentOs.create({ permissions: ALLOW_ALL_VM_PERMISSIONS }); + await vm.writeFile( + "/app/types.mjs", + `console.log(JSON.stringify(typeof globalThis.orderStore));`, + ); + + const result = await runNode(vm, "node", ["/app/types.mjs"]); + expect(result.exitCode, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + expect(lastJsonLine(result.stdout)).toBe("undefined"); + }, 60_000); + + test("a collection named like an existing global is skipped with a warning", async () => { + vm = await AgentOs.create({ + permissions: ALLOW_ALL_VM_PERMISSIONS, + hostFunctions: { + process: { + status: { + inputSchema: z.object({}), + execute: () => ({ ok: true }), + }, + }, + orderStore, + }, + }); + await vm.writeFile( + "/app/collision.mjs", + `console.log(JSON.stringify({ + processStatus: typeof process.status, + cwd: typeof process.cwd, + orderStore: typeof orderStore.listOrders, +}));`, + ); + + const result = await runNode(vm, "node", ["/app/collision.mjs"]); + expect(result.exitCode, result.stderr).toBe(0); + expect(result.stderr).toContain( + 'agentos: host function group "process" was not defined because globalThis.process already exists', + ); + expect(lastJsonLine(result.stdout)).toEqual({ + processStatus: "undefined", + cwd: "function", + orderStore: "function", + }); + }, 60_000); +}); diff --git a/secure-exec/docs/content/docs/host-functions.mdx b/secure-exec/docs/content/docs/host-functions.mdx index 8b2585b9ce..de61631b99 100644 --- a/secure-exec/docs/content/docs/host-functions.mdx +++ b/secure-exec/docs/content/docs/host-functions.mdx @@ -24,6 +24,8 @@ Arguments from the guest are validated before your code runs. ## Call them from guest code Inside the VM each collection is a global object and each function is async. +The globals exist in every guest Node process: inline code, spawned scripts, +`node` run from the shell, and child processes of guest Node. From 648b92f83b7a27a703b7bcc26f81ecfb87f674f7 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Tue, 20 Oct 2026 00:03:37 -0700 Subject: [PATCH 2/4] fix(permissions): keep VM-local networking under scalar network deny --- crates/native-sidecar-core/src/permissions.rs | 166 ++++++++++++++---- .../tests/builtin_conformance.rs | 4 +- crates/native-sidecar/tests/python.rs | 9 +- docs/content/docs/networking.mdx | 7 +- docs/content/docs/permissions.mdx | 38 ++++ examples/js-networking/src/index.ts | 39 ++-- packages/core/package.json | 2 +- .../core/tests/network-deny-vm-local.test.ts | 154 ++++++++++++++++ secure-exec/docs/content/docs/permissions.mdx | 5 +- 9 files changed, 373 insertions(+), 51 deletions(-) create mode 100644 packages/core/tests/network-deny-vm-local.test.ts diff --git a/crates/native-sidecar-core/src/permissions.rs b/crates/native-sidecar-core/src/permissions.rs index ee9de7b3ca..b6ca05c5e3 100644 --- a/crates/native-sidecar-core/src/permissions.rs +++ b/crates/native-sidecar-core/src/permissions.rs @@ -54,6 +54,35 @@ pub fn allow_all_policy() -> vm_config::PermissionsPolicy { } } +/// The network scope that keeps the VM self-contained: guest listeners and +/// VM-local loopback traffic work, and everything that would cross the VM +/// boundary is denied. Listeners are never published on the host, and guest +/// loopback reaches a host loopback service only through an explicit +/// loopback exemption. It is the default network scope and the meaning of a +/// scalar `network: "deny"`. +pub fn vm_local_network_scope() -> vm_config::PatternPermissionScope { + vm_config::PatternPermissionScope::Rules(vm_config::PatternPermissionRuleSet { + default: Some(vm_config::PermissionMode::Deny), + rules: vec![ + vm_config::PatternPermissionRule { + mode: vm_config::PermissionMode::Allow, + operations: vec![String::from("listen")], + patterns: vec![String::from("tcp://**"), String::from("unix:**")], + }, + vm_config::PatternPermissionRule { + mode: vm_config::PermissionMode::Allow, + operations: vec![String::from("http")], + patterns: vec![ + String::from("tcp://127.0.0.1:*"), + String::from("tcp://localhost:*"), + String::from("tcp://::1:*"), + String::from("unix:**"), + ], + }, + ], + }) +} + /// The policy for every scope a client leaves out. The guest behaves like a /// sandboxed machine: its virtual filesystem, processes, environment, host /// functions, listeners, and loopback networking work. External network access @@ -69,28 +98,7 @@ pub fn default_permissions_policy() -> vm_config::PermissionsPolicy { fs: Some(vm_config::FsPermissionScope::Mode( vm_config::PermissionMode::Allow, )), - network: Some(vm_config::PatternPermissionScope::Rules( - vm_config::PatternPermissionRuleSet { - default: Some(vm_config::PermissionMode::Deny), - rules: vec![ - vm_config::PatternPermissionRule { - mode: vm_config::PermissionMode::Allow, - operations: vec![String::from("listen")], - patterns: vec![String::from("tcp://**"), String::from("unix:**")], - }, - vm_config::PatternPermissionRule { - mode: vm_config::PermissionMode::Allow, - operations: vec![String::from("http")], - patterns: vec![ - String::from("tcp://127.0.0.1:*"), - String::from("tcp://localhost:*"), - String::from("tcp://::1:*"), - String::from("unix:**"), - ], - }, - ], - }, - )), + network: Some(vm_local_network_scope()), child_process: allow(), process: allow(), env: allow(), @@ -137,8 +145,8 @@ pub fn resolve_permissions_policy( if overrides.fs.is_some() { resolved.fs = overrides.fs; } - if overrides.network.is_some() { - resolved.network = overrides.network; + if let Some(network) = overrides.network { + resolved.network = Some(resolve_network_override(network)); } if overrides.child_process.is_some() { resolved.child_process = overrides.child_process; @@ -156,6 +164,26 @@ pub fn resolve_permissions_policy( resolved } +/// A scalar network denial denies external egress only. Like the default, it +/// keeps VM-local listeners and loopback working, because they never cross +/// the VM boundary. A caller that also wants to deny those lists explicit +/// rules instead. +fn resolve_network_override( + network: vm_config::PatternPermissionScope, +) -> vm_config::PatternPermissionScope { + match network { + vm_config::PatternPermissionScope::Mode(vm_config::PermissionMode::Deny) => { + vm_local_network_scope() + } + vm_config::PatternPermissionScope::Mode( + mode @ (vm_config::PermissionMode::Allow | vm_config::PermissionMode::Ask), + ) => vm_config::PatternPermissionScope::Mode(mode), + vm_config::PatternPermissionScope::Rules(rules) => { + vm_config::PatternPermissionScope::Rules(rules) + } + } +} + pub fn evaluate_permissions_policy( permissions: &vm_config::PermissionsPolicy, domain: &str, @@ -627,7 +655,10 @@ mod tests { "dns://generativelanguage.googleapis.com", "dns://openrouter.ai", ] { - assert!(network.rules[0] + assert!(network + .rules + .last() + .expect("agentOS egress rule") .patterns .iter() .any(|value| value == expected)); @@ -648,12 +679,87 @@ mod tests { ); assert_eq!(overridden.fs, defaults.fs); assert_eq!(overridden.child_process, defaults.child_process); - assert_eq!( - overridden.network, - Some(vm_config::PatternPermissionScope::Mode( - vm_config::PermissionMode::Deny - )) + assert_eq!(overridden.network, Some(vm_local_network_scope())); + } + + fn network_override(scope: vm_config::PatternPermissionScope) -> vm_config::PermissionsPolicy { + vm_config::PermissionsPolicy { + fs: None, + network: Some(scope), + child_process: None, + process: None, + env: None, + host_function: None, + } + } + + #[test] + fn scalar_network_deny_blocks_egress_but_keeps_vm_local_networking() { + for profile in [ + vm_config::VmDefaultsProfile::Secure, + vm_config::VmDefaultsProfile::AgentOs, + ] { + let policy = resolve_permissions_policy( + profile, + Some(network_override(vm_config::PatternPermissionScope::Mode( + vm_config::PermissionMode::Deny, + ))), + ); + + for (capability, resource) in [ + ("network.listen", "tcp://0.0.0.0:3000"), + ("network.listen", "tcp://127.0.0.1:3000"), + ("network.listen", "unix:/tmp/app.sock"), + ("network.http", "tcp://127.0.0.1:3000"), + ("network.http", "tcp://localhost:3000"), + ("network.http", "tcp://::1:3000"), + ("network.http", "unix:/tmp/app.sock"), + ] { + assert!( + allowed(&policy, "network", capability, resource), + "{profile:?}: {capability} {resource} should stay VM-local and allowed", + ); + } + for (capability, resource) in [ + ("network.http", "tcp://example.com:443"), + ("network.http", "tcp://203.0.113.10:80"), + ("network.http", "tcp://api.anthropic.com:443"), + ("network.fetch", "https://example.com/"), + ("network.dns", "dns://example.com"), + ("network.dns", "dns://api.anthropic.com"), + ] { + assert!( + !allowed(&policy, "network", capability, resource), + "{profile:?}: {capability} {resource} crosses the VM boundary", + ); + } + } + } + + #[test] + fn explicit_network_rules_can_still_deny_vm_local_networking() { + let policy = resolve_permissions_policy( + vm_config::VmDefaultsProfile::Secure, + Some(network_override(vm_config::PatternPermissionScope::Rules( + vm_config::PatternPermissionRuleSet { + default: Some(vm_config::PermissionMode::Deny), + rules: Vec::new(), + }, + ))), ); + + assert!(!allowed( + &policy, + "network", + "network.listen", + "tcp://127.0.0.1:3000" + )); + assert!(!allowed( + &policy, + "network", + "network.http", + "tcp://127.0.0.1:3000" + )); } #[test] diff --git a/crates/native-sidecar/tests/builtin_conformance.rs b/crates/native-sidecar/tests/builtin_conformance.rs index f8c3eac190..fcd6eda05e 100644 --- a/crates/native-sidecar/tests/builtin_conformance.rs +++ b/crates/native-sidecar/tests/builtin_conformance.rs @@ -898,7 +898,9 @@ fn http_request_denied_egress_returns_permission_error_impl() { import http from "node:http"; const result = await new Promise((resolve) => { - const req = http.get("http://127.0.0.1:9/denied", (res) => { + // A scalar network denial keeps VM-local loopback, so probe an external + // address that crosses the VM boundary. + const req = http.get("http://203.0.113.10:9/denied", (res) => { res.resume(); resolve({ statusCode: res.statusCode ?? null, diff --git a/crates/native-sidecar/tests/python.rs b/crates/native-sidecar/tests/python.rs index 5a7a34ccdf..ab3295304a 100644 --- a/crates/native-sidecar/tests/python.rs +++ b/crates/native-sidecar/tests/python.rs @@ -2862,7 +2862,14 @@ fn python_runtime_micropip_install_respects_network_permissions() { )]), PermissionsPolicy { fs: wire_permissions_allow_all().fs, - network: Some(PatternPermissionScope::PermissionMode(PermissionMode::Deny)), + // A scalar network denial keeps VM-local loopback, and the exempt port + // would let it reach the host server, so deny every operation. + network: Some(PatternPermissionScope::PatternPermissionRuleSet( + PatternPermissionRuleSet { + default: Some(PermissionMode::Deny), + rules: Vec::new(), + }, + )), child_process: wire_permissions_allow_all().child_process, process: wire_permissions_allow_all().process, env: wire_permissions_allow_all().env, diff --git a/docs/content/docs/networking.mdx b/docs/content/docs/networking.mdx index c8feb02ab2..f462e7adfa 100644 --- a/docs/content/docs/networking.mdx +++ b/docs/content/docs/networking.mdx @@ -77,6 +77,7 @@ browsers, route requests through your application and forward them with Guest listeners and loopback traffic stay inside the VM. Outbound access is limited to the sidecar's default allowlist unless the `network` -[permission](/agentos/docs/permissions) scope allows more. Requests from the -host and previews enter the VM from the host side, so they are subject to -loopback confinement but not to the guest egress policy. +[permission](/agentos/docs/permissions) scope allows more. `network: "deny"` +removes that allowlist and keeps listeners and loopback working. Requests from +the host and previews enter the VM over VM-local loopback, so they are subject +to loopback confinement, not to the guest egress allowlist. diff --git a/docs/content/docs/permissions.mdx b/docs/content/docs/permissions.mdx index 23d22ec30e..8eb0a86de9 100644 --- a/docs/content/docs/permissions.mdx +++ b/docs/content/docs/permissions.mdx @@ -20,6 +20,44 @@ processes, environment, listeners, and loopback networking work, and outbound network access is limited to the sidecar's default allowlist. Library host functions are governed by the `hostFunction` scope. +## Network + +`network: "deny"` denies traffic that crosses the VM boundary and keeps +networking inside the VM working, like a container with no published ports. + +| Traffic | `network: "deny"` | +| --- | --- | +| Guest `listen()` on TCP or Unix sockets | Allowed | +| Guest connections to its own listeners over `127.0.0.1`, `localhost`, `::1`, or Unix sockets | Allowed | +| Host requests to guest servers with `fetch`, `httpRequest`, or previews | Allowed | +| DNS lookups | Denied | +| TCP connections and HTTP requests to any non-loopback address | Denied | +| The default allowlist of model provider endpoints | Denied | + +Guest listeners are never bound on the host, and guest loopback never reaches a +host loopback service unless that host port is listed in `loopbackExemptPorts`. +`network: "deny"` does not change either rule. + +`"deny"` is shorthand for this rule set. A rule set replaces the whole network +scope, so a custom rule set that should keep listeners and loopback working +must include these rules. Omit them to deny VM-local networking too. + +```json +{ + "network": { + "default": "deny", + "rules": [ + { "mode": "allow", "operations": ["listen"], "patterns": ["tcp://**", "unix:**"] }, + { + "mode": "allow", + "operations": ["http"], + "patterns": ["tcp://127.0.0.1:*", "tcp://localhost:*", "tcp://::1:*", "unix:**"] + } + ] + } +} +``` + See the [Security Model](/agentos/docs/security-model) for the trust boundary these policies enforce. diff --git a/examples/js-networking/src/index.ts b/examples/js-networking/src/index.ts index cb4be484ec..b1ea5f1e8e 100644 --- a/examples/js-networking/src/index.ts +++ b/examples/js-networking/src/index.ts @@ -7,7 +7,9 @@ * 1. With network "allow", the guest starts a loopback HTTP server inside the * VM and fetches it - the request and response stay entirely within the * kernel socket table (hermetic, no real host network). - * 2. With an explicit network deny policy, the same fetch is blocked. + * 2. With network "deny", the same VM-local fetch still works, because + * listeners and loopback never leave the VM, but external egress is + * blocked. * 3. Host loopback is separate from VM loopback: even with network "allow", * a guest can reach a host loopback service only when the host port is in * loopbackExemptPorts. @@ -69,25 +71,34 @@ try { } // docs:end vm-network -// 2. Network denied explicitly. Supplying a partial policy denies omitted -// scopes too, so name the runtime capabilities needed to launch the program. +// 2. Network denied. A scalar deny blocks traffic that crosses the VM +// boundary, so the VM-local server still works while external egress fails. +const EGRESS_GUEST = ` +try { + await fetch("http://example.com/"); + console.log("unexpected: external fetch succeeded"); +} catch (error) { + console.log("external fetch:", error.cause?.code || error.code || error.name); +} +`; const denied = await AgentOs.create({ - permissions: { - fs: "allow", - network: "deny", - childProcess: "allow", - process: "allow", - env: "allow", - }, + permissions: { network: "deny" }, }); try { - const result = await denied.javascript.execute(GUEST, { + const local = await denied.javascript.execute(GUEST, { + output: { capture: "all" }, + }); + console.log("[network denied] VM-local exitCode:", local.exitCode); + console.log( + "[network denied] VM-local stdout:", + JSON.stringify(local.stdout?.trim()), + ); + const external = await denied.javascript.execute(EGRESS_GUEST, { output: { capture: "all" }, }); - console.log("[network denied] exitCode:", result.exitCode); console.log( - "[network denied] stderr:", - JSON.stringify(result.stderr?.trim().split("\n")[0]), + "[network denied] egress stdout:", + JSON.stringify(external.stdout?.trim()), ); } finally { await denied.dispose(); diff --git a/packages/core/package.json b/packages/core/package.json index ae0133f96f..b5a4e873d4 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -54,7 +54,7 @@ "check-types": "tsc --noEmit && tsc -p tsconfig.type-tests.json", "build": "tsc && node scripts/stage-default-software.mjs", "test": "vitest run --exclude '**/*.nightly.test.ts' --reporter=verbose", - "test:unit": "vitest run tests/agentos-package.test.ts tests/allowed-node-builtins.test.ts tests/host-functions-zod.test.ts tests/host-functions.test.ts tests/host-function-command-exec.test.ts tests/host-function-permissions.test.ts tests/host-function-globals.test.ts tests/cron-manager.test.ts tests/code-mode-execution.test.ts tests/cron-timer-driver.test.ts tests/exec-timeout.test.ts tests/execution-abort.test.ts tests/generated-protocol.test.ts tests/leak-agent-os-processes.test.ts tests/leak-rpc-client.test.ts tests/mount-descriptors.test.ts tests/mount-reconfigure.test.ts tests/options-schema.test.ts tests/output-replay.test.ts tests/public-api-exports.test.ts tests/root-filesystem-descriptors.test.ts tests/runtime-compat-mount.test.ts tests/sidecar-client.test.ts tests/sidecar-exit.test.ts tests/sidecar-permission-descriptors.test.ts tests/software-install.test.ts tests/wasm-permission-tiers.test.ts --fileParallelism=false", + "test:unit": "vitest run tests/agentos-package.test.ts tests/allowed-node-builtins.test.ts tests/host-functions-zod.test.ts tests/host-functions.test.ts tests/host-function-command-exec.test.ts tests/host-function-permissions.test.ts tests/host-function-globals.test.ts tests/cron-manager.test.ts tests/code-mode-execution.test.ts tests/cron-timer-driver.test.ts tests/exec-timeout.test.ts tests/execution-abort.test.ts tests/generated-protocol.test.ts tests/leak-agent-os-processes.test.ts tests/leak-rpc-client.test.ts tests/mount-descriptors.test.ts tests/mount-reconfigure.test.ts tests/network-deny-vm-local.test.ts tests/options-schema.test.ts tests/output-replay.test.ts tests/public-api-exports.test.ts tests/root-filesystem-descriptors.test.ts tests/runtime-compat-mount.test.ts tests/sidecar-client.test.ts tests/sidecar-exit.test.ts tests/sidecar-permission-descriptors.test.ts tests/software-install.test.ts tests/wasm-permission-tiers.test.ts --fileParallelism=false", "test:pr": "pnpm test:unit", "test:nightly": "vitest run tests/*.nightly.test.ts --reporter=verbose --passWithNoTests" }, diff --git a/packages/core/tests/network-deny-vm-local.test.ts b/packages/core/tests/network-deny-vm-local.test.ts new file mode 100644 index 0000000000..5221f5d85e --- /dev/null +++ b/packages/core/tests/network-deny-vm-local.test.ts @@ -0,0 +1,154 @@ +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterEach, describe, expect, test } from "vitest"; +import { AgentOs } from "../src/index.js"; + +// A scalar `network: "deny"` denies traffic that crosses the VM boundary. Like +// the default policy, it keeps guest listeners and VM-local loopback working, +// so a VM behaves like a container with no published ports. + +const SERVER_SCRIPT = ` +import http from "node:http"; +const server = http.createServer((req, res) => res.end("guest-ok:" + req.url)); +server.on("error", (error) => { + console.log("LISTEN_ERROR " + error.code + " " + error.message); + process.exit(3); +}); +server.listen(3000, "127.0.0.1", () => console.log("LISTENING")); +`; + +const PROBE_SCRIPT = ` +const probe = async (url) => { + try { + const response = await fetch(url); + return { status: response.status, body: await response.text() }; + } catch (error) { + const cause = error.cause ?? error; + return { error: String(cause.code ?? cause.name), message: String(cause.message) }; + } +}; +const [vmLocal, external, hostLoopback] = await Promise.all([ + probe("http://127.0.0.1:3000/self"), + probe("http://example.com/"), + probe("http://127.0.0.1:" + process.argv[2] + "/host"), +]); +console.log(JSON.stringify({ vmLocal, external, hostLoopback })); +`; + +async function startGuestServer(vm: AgentOs): Promise { + const decoder = new TextDecoder(); + let stdout = ""; + let listening!: () => void; + let failed!: (error: Error) => void; + const ready = new Promise((resolve, reject) => { + listening = resolve; + failed = reject; + }); + const { pid } = await vm.process.spawn("node", ["/app/server.mjs"], { + onStdout: (chunk) => { + stdout += decoder.decode(chunk); + if (stdout.includes("LISTENING")) listening(); + if (stdout.includes("LISTEN_ERROR")) failed(new Error(stdout)); + }, + }); + await ready; + return pid; +} + +async function runProbe(vm: AgentOs, hostPort: number) { + const decoder = new TextDecoder(); + let stdout = ""; + let stderr = ""; + const { pid } = await vm.process.spawn( + "node", + ["/app/probe.mjs", String(hostPort)], + { + onStdout: (chunk) => { + stdout += decoder.decode(chunk); + }, + onStderr: (chunk) => { + stderr += decoder.decode(chunk); + }, + }, + ); + const exit = await vm.process.wait(pid); + expect(exit.exitCode, stderr).toBe(0); + return JSON.parse(stdout.trim().split("\n").at(-1) ?? ""); +} + +describe('scalar network "deny"', () => { + let vm: AgentOs | undefined; + let hostServer: Server | undefined; + let hostRequests = 0; + + afterEach(async () => { + await vm?.dispose(); + vm = undefined; + await new Promise((resolve) => { + if (!hostServer) return resolve(); + hostServer.close(() => resolve()); + }); + hostServer = undefined; + hostRequests = 0; + }); + + async function startHostServer(): Promise { + hostServer = createServer((_req, res) => { + hostRequests += 1; + res.end("host-ok"); + }); + await new Promise((resolve) => { + hostServer?.listen(0, "127.0.0.1", resolve); + }); + return (hostServer.address() as AddressInfo).port; + } + + test("keeps VM-local listeners and loopback while denying egress", async () => { + const hostPort = await startHostServer(); + vm = await AgentOs.create({ permissions: { network: "deny" } }); + await vm.writeFile("/app/server.mjs", SERVER_SCRIPT); + await vm.writeFile("/app/probe.mjs", PROBE_SCRIPT); + const serverPid = await startGuestServer(vm); + + // The host reaches the guest server over VM-local loopback. + const response = await vm.network.httpRequest({ + port: 3000, + path: "/from-host", + }); + expect(response.status).toBe(200); + expect(new TextDecoder().decode(response.body)).toBe("guest-ok:/from-host"); + + const result = await runProbe(vm, hostPort); + // Guest loopback reaches the guest's own listener. + expect(result.vmLocal).toEqual({ status: 200, body: "guest-ok:/self" }); + // External egress is denied by the network policy. + expect(result.external.error).toBe("EACCES"); + // Guest loopback is VM-local, so loopback confinement keeps it from + // reaching host loopback services without an explicit exemption. + expect(result.hostLoopback.error).toBe("EACCES"); + expect(result.hostLoopback.message).toContain("not owned by this VM"); + expect(hostRequests).toBe(0); + + await vm.process.kill(serverPid); + await vm.process.wait(serverPid); + }, 60_000); + + test("an explicit rule set can still deny VM-local networking", async () => { + vm = await AgentOs.create({ + permissions: { network: { default: "deny", rules: [] } }, + }); + await vm.writeFile("/app/server.mjs", SERVER_SCRIPT); + + const decoder = new TextDecoder(); + let stdout = ""; + const { pid } = await vm.process.spawn("node", ["/app/server.mjs"], { + onStdout: (chunk) => { + stdout += decoder.decode(chunk); + }, + }); + const exit = await vm.process.wait(pid); + expect(exit.exitCode, stdout).toBe(3); + expect(stdout).toContain("LISTEN_ERROR EACCES"); + expect(stdout).toContain("network.listen"); + }, 60_000); +}); diff --git a/secure-exec/docs/content/docs/permissions.mdx b/secure-exec/docs/content/docs/permissions.mdx index 10337e3159..0f96f22c72 100644 --- a/secure-exec/docs/content/docs/permissions.mdx +++ b/secure-exec/docs/content/docs/permissions.mdx @@ -26,7 +26,10 @@ scope as it was. ## Allow specific hosts A rule set denies by default and allows matching patterns. A host needs both a -`dns://` pattern to resolve it and a `tcp://` pattern to connect to it. +`dns://` pattern to resolve it and a `tcp://` pattern to connect to it. A rule +set replaces the whole network scope, so add the VM-local `listen` and loopback +rules from the [agentOS permissions reference](/agentos/docs/permissions#network) +if guest code also runs a server. `network: "deny"` keeps them. From bd3f6f44a8035076edf25997ad689d0d4258ac1a Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Tue, 20 Oct 2026 00:11:30 -0700 Subject: [PATCH 3/4] docs(software): document software installation for the library and Hosted --- docs/content/docs/software.mdx | 68 ++++++++++++++++--- examples/embedded/package.json | 1 - examples/embedded/software-local.ts | 26 +++++++ examples/embedded/software.ts | 40 ++++++++++- examples/quickstart/bash/package.json | 1 - examples/quickstart/cron/package.json | 1 - examples/quickstart/filesystem/package.json | 1 - examples/quickstart/git/README.md | 2 +- examples/quickstart/git/index.ts | 20 +++++- examples/quickstart/git/package.json | 1 - examples/quickstart/hello-world/package.json | 1 - examples/quickstart/network/package.json | 1 - examples/quickstart/nodejs/package.json | 1 - examples/quickstart/package.json | 1 - examples/quickstart/processes/package.json | 1 - .../quickstart/s3-filesystem/package.json | 1 - examples/quickstart/sandbox/package.json | 1 - pnpm-lock.yaml | 55 --------------- 18 files changed, 140 insertions(+), 83 deletions(-) create mode 100644 examples/embedded/software-local.ts diff --git a/docs/content/docs/software.mdx b/docs/content/docs/software.mdx index 692e864294..44aa12bf63 100644 --- a/docs/content/docs/software.mdx +++ b/docs/content/docs/software.mdx @@ -16,8 +16,34 @@ Browse the available packages in the [registry](/agentos/docs/registry). Registry packages are published with each release under `https://releases.rivet.dev/agent-os//software/`, or `latest` for the newest release. Its `manifest.json` lists each package's name, digest, size, -and path relative to the manifest. Registry software is not installed from npm; -see [Publishing Software](/agentos/docs/custom-software/publishing). +and path relative to the manifest: + +```json +{ + "schemaVersion": 1, + "artifacts": [ + { + "name": "git", + "digest": "sha256:<64 lowercase hex>", + "size": 17961114, + "path": "packages/git/<64 lowercase hex>.aospkg" + } + ] +} +``` + +Resolve `path` against the manifest URL to get the package URL, and pass +`digest` along so the bytes are verified before they are installed. Registry +software is distributed only as `.aospkg` files from this location, not as npm +packages. See [Publishing Software](/agentos/docs/custom-software/publishing). + +## Default software + +Every VM installs a default bundle first: `coreutils` (which provides `sh`), +`sed`, `grep`, `gawk`, `findutils`, `diffutils`, `tar`, and `gzip`. The bundle +ships with the library and the hosted runtime, so it needs no download. Pass +`defaultSoftware: false` to `AgentOs.create()`, or set `"defaultSoftware": false` +in a hosted VM's `config`, for a bare VM that has only the software you list. ## Install while running @@ -82,14 +108,15 @@ import { createAgentOsClient } from "@rivet-dev/agentos"; const client = createAgentOsClient({ endpoint: "https://api.rivet.dev" }); +// Look up the package in a release manifest. Use a release version, or `latest`. +const base = "https://releases.rivet.dev/agent-os/latest/software/"; +const { artifacts } = await (await fetch(`${base}manifest.json`)).json(); +const git = artifacts.find((artifact: { name: string }) => artifact.name === "git"); + const vm = client.agentOS.getOrCreate(["my-vm"], { createWithInput: { config: { - software: [ - { - url: "https://releases.rivet.dev/agent-os/latest/software/packages/git/.aospkg", - }, - ], + software: [{ url: new URL(git.path, base).href, digest: git.digest }], }, }, }); @@ -99,10 +126,29 @@ const vm = client.agentOS.getOrCreate(["my-vm"], { -Hosted VMs take package URLs in their `software` config. The library takes -`{ packagePath }` references to packed `.aospkg` files, which is what the -`@agentos-software/*` packages in the agentOS repository export, and -`defaultSoftware: false` for a bare VM with only the software you list. +Hosted VMs take `{ url, digest }` sources in their `software` config, and +download and verify them when the VM starts. The library's `software` option +takes `{ packagePath }` references to trusted `.aospkg` files on the host, so +the library example downloads the registry package once, verifies its digest, +and passes the cached file. To install from a URL without a local file, create +the VM and call [`software.install`](#install-while-running) before you run +anything that needs the package. + +## Custom and local packages + +A package you build yourself, or any other `.aospkg` file, installs the same way +as a registry package. See +[Software Package Definition](/agentos/docs/custom-software/definition) and +[Building Software](/agentos/docs/custom-software/building-wasm) for how to +build one. + + + +The library reads trusted local paths, both in `software` at creation and with +`software.install({ type: "path", path })` while running. Hosted VMs cannot read +files from your machine, so upload the `.aospkg` to an HTTPS location you +control and pass its URL and `sha256:` digest, at creation or with +`v1.software.install`. ## Verification and caching diff --git a/examples/embedded/package.json b/examples/embedded/package.json index 9d95fd0c4b..13e16bf254 100644 --- a/examples/embedded/package.json +++ b/examples/embedded/package.json @@ -12,7 +12,6 @@ "sandbox-agent": "^0.4.2", "dockerode": "^4.0.9", "get-port": "^7.1.0", - "@agentos-software/git": "workspace:*", "zod": "^4.1.11" }, "devDependencies": { diff --git a/examples/embedded/software-local.ts b/examples/embedded/software-local.ts new file mode 100644 index 0000000000..46cd8a981a --- /dev/null +++ b/examples/embedded/software-local.ts @@ -0,0 +1,26 @@ +import { resolve } from "node:path"; +import { AgentOs } from "@rivet-dev/agentos-core"; + +// A trusted .aospkg file on the host, such as one you built with +// agentos-toolchain or downloaded from the registry. +const packagePath = resolve(process.argv[2] ?? "dist/package.aospkg"); + +// Install it when the VM is created. +const vm = await AgentOs.create({ software: [{ packagePath }] }); +try { + console.log(await vm.software.list()); +} finally { + await vm.dispose(); +} + +// Or install it into a running VM. +const running = await AgentOs.create(); +try { + const installed = await running.software.install({ + type: "path", + path: packagePath, + }); + console.log(installed.packageName, installed.version, installed.commands); +} finally { + await running.dispose(); +} diff --git a/examples/embedded/software.ts b/examples/embedded/software.ts index 39412307c0..9cd43cbf84 100644 --- a/examples/embedded/software.ts +++ b/examples/embedded/software.ts @@ -1,7 +1,43 @@ -import git from "@agentos-software/git"; +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { AgentOs } from "@rivet-dev/agentos-core"; -const vm = await AgentOs.create({ software: [git] }); +// Registry software is published as digest-addressed .aospkg files next to a +// manifest. Use a release version, or `latest`. +const base = "https://releases.rivet.dev/agent-os/latest/software/"; +const { artifacts } = (await (await fetch(`${base}manifest.json`)).json()) as { + artifacts: { name: string; digest: string; path: string }[]; +}; + +// The library installs packages at creation from trusted local paths, so +// download each registry package once, verify its digest, and reuse the file. +async function downloadPackage(name: string): Promise { + const artifact = artifacts.find((entry) => entry.name === name); + if (!artifact) throw new Error(`${name} is not in the manifest`); + const hex = artifact.digest.replace(/^sha256:/, ""); + const cacheDir = join(tmpdir(), "agentos-software"); + const packagePath = join(cacheDir, `${name}-${hex}.aospkg`); + if (existsSync(packagePath)) return packagePath; + + const response = await fetch(new URL(artifact.path, base)); + if (!response.ok) + throw new Error(`download ${name}: HTTP ${response.status}`); + const bytes = new Uint8Array(await response.arrayBuffer()); + const actual = createHash("sha256").update(bytes).digest("hex"); + if (actual !== hex) throw new Error(`${name} digest mismatch: ${actual}`); + await mkdir(cacheDir, { recursive: true }); + await writeFile(packagePath, bytes); + return packagePath; +} + +// The default bundle (sh, coreutils, and the standard CLI tools) is installed +// first, then the packages listed here. +const vm = await AgentOs.create({ + software: [{ packagePath: await downloadPackage("git") }], +}); try { const result = await vm.process.exec("git --version", { diff --git a/examples/quickstart/bash/package.json b/examples/quickstart/bash/package.json index a7c855042f..be70b74910 100644 --- a/examples/quickstart/bash/package.json +++ b/examples/quickstart/bash/package.json @@ -13,7 +13,6 @@ "sandbox-agent": "^0.4.2", "dockerode": "^4.0.9", "get-port": "^7.1.0", - "@agentos-software/git": "workspace:*", "zod": "^4.1.11" }, "devDependencies": { diff --git a/examples/quickstart/cron/package.json b/examples/quickstart/cron/package.json index 2405197038..0d35d805e3 100644 --- a/examples/quickstart/cron/package.json +++ b/examples/quickstart/cron/package.json @@ -13,7 +13,6 @@ "sandbox-agent": "^0.4.2", "dockerode": "^4.0.9", "get-port": "^7.1.0", - "@agentos-software/git": "workspace:*", "zod": "^4.1.11" }, "devDependencies": { diff --git a/examples/quickstart/filesystem/package.json b/examples/quickstart/filesystem/package.json index 98a6aee418..ffb41823f5 100644 --- a/examples/quickstart/filesystem/package.json +++ b/examples/quickstart/filesystem/package.json @@ -13,7 +13,6 @@ "sandbox-agent": "^0.4.2", "dockerode": "^4.0.9", "get-port": "^7.1.0", - "@agentos-software/git": "workspace:*", "zod": "^4.1.11" }, "devDependencies": { diff --git a/examples/quickstart/git/README.md b/examples/quickstart/git/README.md index 20e5e64640..161abe689d 100644 --- a/examples/quickstart/git/README.md +++ b/examples/quickstart/git/README.md @@ -9,7 +9,7 @@ Run real `git` inside a VM: initialize a repo, branch, commit, and clone it — ## How it works -The VM is created with the `@agentos-software/git` package mounted as software and filesystem, child-process, and env permissions enabled. From there it drives ordinary git commands through `vm.process.exec`, mixing in `vm.filesystem.writeFile`/`vm.filesystem.readFile` to stage files and inspect results. It commits on a `feature` branch so that `git clone` resolves HEAD to `feature`, then reads the clone's `.git/HEAD` and the feature file to confirm the branch carried over. +The VM is created with filesystem, child-process, and env permissions enabled, then installs `git` from the agentOS software registry by looking it up in the release `manifest.json` and passing its URL and digest to `vm.software.install`. From there it drives ordinary git commands through `vm.process.exec`, mixing in `vm.filesystem.writeFile`/`vm.filesystem.readFile` to stage files and inspect results. It commits on a `feature` branch so that `git clone` resolves HEAD to `feature`, then reads the clone's `.git/HEAD` and the feature file to confirm the branch carried over. ## Run it diff --git a/examples/quickstart/git/index.ts b/examples/quickstart/git/index.ts index 67e96903ad..a55ef1c179 100644 --- a/examples/quickstart/git/index.ts +++ b/examples/quickstart/git/index.ts @@ -2,7 +2,6 @@ import { createRequire } from "node:module"; import { dirname, resolve } from "node:path"; -import git from "@agentos-software/git"; import { AgentOs } from "@rivet-dev/agentos-core"; type ExecResult = { @@ -47,7 +46,22 @@ function parseHeadRef(content: string): string { const vm = await AgentOs.create({ permissions: GIT_QUICKSTART_PERMISSIONS, - software: [git], +}); + +// Install git from the agentOS software registry. The release manifest lists +// each package's URL and digest, and the bytes are verified before install. +const SOFTWARE_BASE = "https://releases.rivet.dev/agent-os/latest/software/"; +const manifest = (await ( + await fetch(`${SOFTWARE_BASE}manifest.json`) +).json()) as { artifacts: { name: string; digest: string; path: string }[] }; +const gitArtifact = manifest.artifacts.find( + (artifact) => artifact.name === "git", +); +if (!gitArtifact) throw new Error("git is not in the software manifest"); +await vm.software.install({ + type: "url", + url: new URL(gitArtifact.path, SOFTWARE_BASE).href, + expectedDigest: gitArtifact.digest, }); async function run(command: string): Promise { @@ -64,6 +78,8 @@ async function run(command: string): Promise { }; } +await run('git config --global user.name "agentOS"'); +await run('git config --global user.email "agentos@example.com"'); await run("git init /tmp/origin"); await vm.filesystem.writeFile("/tmp/origin/README.md", "# demo repo\n"); await run("git -C /tmp/origin add README.md"); diff --git a/examples/quickstart/git/package.json b/examples/quickstart/git/package.json index 808b44aa93..58a1cbc8bd 100644 --- a/examples/quickstart/git/package.json +++ b/examples/quickstart/git/package.json @@ -13,7 +13,6 @@ "sandbox-agent": "^0.4.2", "dockerode": "^4.0.9", "get-port": "^7.1.0", - "@agentos-software/git": "workspace:*", "zod": "^4.1.11" }, "devDependencies": { diff --git a/examples/quickstart/hello-world/package.json b/examples/quickstart/hello-world/package.json index f1497bb24f..ea9a3fc743 100644 --- a/examples/quickstart/hello-world/package.json +++ b/examples/quickstart/hello-world/package.json @@ -13,7 +13,6 @@ "sandbox-agent": "^0.4.2", "dockerode": "^4.0.9", "get-port": "^7.1.0", - "@agentos-software/git": "workspace:*", "zod": "^4.1.11" }, "devDependencies": { diff --git a/examples/quickstart/network/package.json b/examples/quickstart/network/package.json index d54eb8c575..2f7358dae8 100644 --- a/examples/quickstart/network/package.json +++ b/examples/quickstart/network/package.json @@ -13,7 +13,6 @@ "sandbox-agent": "^0.4.2", "dockerode": "^4.0.9", "get-port": "^7.1.0", - "@agentos-software/git": "workspace:*", "zod": "^4.1.11" }, "devDependencies": { diff --git a/examples/quickstart/nodejs/package.json b/examples/quickstart/nodejs/package.json index f29d71fcd7..7aa90c0b36 100644 --- a/examples/quickstart/nodejs/package.json +++ b/examples/quickstart/nodejs/package.json @@ -13,7 +13,6 @@ "sandbox-agent": "^0.4.2", "dockerode": "^4.0.9", "get-port": "^7.1.0", - "@agentos-software/git": "workspace:*", "zod": "^4.1.11" }, "devDependencies": { diff --git a/examples/quickstart/package.json b/examples/quickstart/package.json index 52df74f3c2..aeb4992a99 100644 --- a/examples/quickstart/package.json +++ b/examples/quickstart/package.json @@ -27,7 +27,6 @@ "sandbox-agent": "^0.4.2", "dockerode": "^4.0.9", "get-port": "^7.1.0", - "@agentos-software/git": "workspace:*", "zod": "^4.1.11" }, "devDependencies": { diff --git a/examples/quickstart/processes/package.json b/examples/quickstart/processes/package.json index ad904a60fe..009099aaae 100644 --- a/examples/quickstart/processes/package.json +++ b/examples/quickstart/processes/package.json @@ -13,7 +13,6 @@ "sandbox-agent": "^0.4.2", "dockerode": "^4.0.9", "get-port": "^7.1.0", - "@agentos-software/git": "workspace:*", "zod": "^4.1.11" }, "devDependencies": { diff --git a/examples/quickstart/s3-filesystem/package.json b/examples/quickstart/s3-filesystem/package.json index fa24bb475c..15ff6945df 100644 --- a/examples/quickstart/s3-filesystem/package.json +++ b/examples/quickstart/s3-filesystem/package.json @@ -13,7 +13,6 @@ "sandbox-agent": "^0.4.2", "dockerode": "^4.0.9", "get-port": "^7.1.0", - "@agentos-software/git": "workspace:*", "zod": "^4.1.11" }, "devDependencies": { diff --git a/examples/quickstart/sandbox/package.json b/examples/quickstart/sandbox/package.json index fd0c599879..c6bf49d05c 100644 --- a/examples/quickstart/sandbox/package.json +++ b/examples/quickstart/sandbox/package.json @@ -13,7 +13,6 @@ "sandbox-agent": "^0.4.2", "dockerode": "^4.0.9", "get-port": "^7.1.0", - "@agentos-software/git": "workspace:*", "zod": "^4.1.11" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6cf6597af0..dea68f65c1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -54,30 +54,8 @@ importers: specifier: ^5.9.2 version: 5.9.3 - examples/bindings: - dependencies: - '@rivet-dev/agentos-core': - specifier: workspace:* - version: link:../../packages/core - zod: - specifier: ^4.1.11 - version: 4.3.6 - devDependencies: - '@types/node': - specifier: ^22.10.2 - version: 22.19.15 - tsx: - specifier: ^4.19.0 - version: 4.21.0 - typescript: - specifier: ^5.7.2 - version: 5.9.3 - examples/embedded: dependencies: - '@agentos-software/git': - specifier: workspace:* - version: link:../../software/git '@rivet-dev/agentos-core': specifier: workspace:* version: link:../../packages/core @@ -432,9 +410,6 @@ importers: examples/quickstart: dependencies: - '@agentos-software/git': - specifier: workspace:* - version: link:../../software/git '@rivet-dev/agentos-core': specifier: workspace:* version: link:../../packages/core @@ -466,9 +441,6 @@ importers: examples/quickstart/bash: dependencies: - '@agentos-software/git': - specifier: workspace:* - version: link:../../../software/git '@rivet-dev/agentos-core': specifier: workspace:* version: link:../../../packages/core @@ -500,9 +472,6 @@ importers: examples/quickstart/cron: dependencies: - '@agentos-software/git': - specifier: workspace:* - version: link:../../../software/git '@rivet-dev/agentos-core': specifier: workspace:* version: link:../../../packages/core @@ -534,9 +503,6 @@ importers: examples/quickstart/filesystem: dependencies: - '@agentos-software/git': - specifier: workspace:* - version: link:../../../software/git '@rivet-dev/agentos-core': specifier: workspace:* version: link:../../../packages/core @@ -568,9 +534,6 @@ importers: examples/quickstart/git: dependencies: - '@agentos-software/git': - specifier: workspace:* - version: link:../../../software/git '@rivet-dev/agentos-core': specifier: workspace:* version: link:../../../packages/core @@ -602,9 +565,6 @@ importers: examples/quickstart/hello-world: dependencies: - '@agentos-software/git': - specifier: workspace:* - version: link:../../../software/git '@rivet-dev/agentos-core': specifier: workspace:* version: link:../../../packages/core @@ -655,9 +615,6 @@ importers: examples/quickstart/network: dependencies: - '@agentos-software/git': - specifier: workspace:* - version: link:../../../software/git '@rivet-dev/agentos-core': specifier: workspace:* version: link:../../../packages/core @@ -689,9 +646,6 @@ importers: examples/quickstart/nodejs: dependencies: - '@agentos-software/git': - specifier: workspace:* - version: link:../../../software/git '@rivet-dev/agentos-core': specifier: workspace:* version: link:../../../packages/core @@ -723,9 +677,6 @@ importers: examples/quickstart/processes: dependencies: - '@agentos-software/git': - specifier: workspace:* - version: link:../../../software/git '@rivet-dev/agentos-core': specifier: workspace:* version: link:../../../packages/core @@ -773,9 +724,6 @@ importers: examples/quickstart/s3-filesystem: dependencies: - '@agentos-software/git': - specifier: workspace:* - version: link:../../../software/git '@rivet-dev/agentos-core': specifier: workspace:* version: link:../../../packages/core @@ -807,9 +755,6 @@ importers: examples/quickstart/sandbox: dependencies: - '@agentos-software/git': - specifier: workspace:* - version: link:../../../software/git '@rivet-dev/agentos-core': specifier: workspace:* version: link:../../../packages/core From 6ca130a21ed3d1922bf65b7cd82c7bdb004a92b0 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Tue, 20 Oct 2026 01:12:57 -0700 Subject: [PATCH 4/4] feat(core): install registry software by name --- crates/client/src/agent_os.rs | 42 +- crates/client/src/config.rs | 23 + crates/client/src/lib.rs | 7 + crates/client/src/software_registry.rs | 452 ++++++++++++++++ crates/client/tests/software_registry.rs | 395 ++++++++++++++ .../docs/custom-software/publishing.mdx | 8 +- docs/content/docs/software.mdx | 69 ++- examples/embedded/config-reference.ts | 8 +- examples/embedded/software-install.ts | 19 +- examples/embedded/software.ts | 38 +- examples/quickstart/git/README.md | 2 +- examples/quickstart/git/index.ts | 19 +- packages/core/package.json | 2 +- packages/core/src/agent-os.ts | 247 ++++++++- packages/core/src/index.ts | 5 + packages/core/src/options-schema.ts | 17 + packages/core/src/packages.ts | 30 +- packages/core/src/software-registry.ts | 402 ++++++++++++++ packages/core/src/types.ts | 14 +- packages/core/tests/software-install.test.ts | 10 + .../tests/software-registry.nightly.test.ts | 21 + packages/core/tests/software-registry.test.ts | 509 ++++++++++++++++++ secure-exec/src/runtime.ts | 2 + software/README.md | 5 +- 24 files changed, 2224 insertions(+), 122 deletions(-) create mode 100644 crates/client/src/software_registry.rs create mode 100644 crates/client/tests/software_registry.rs create mode 100644 packages/core/src/software-registry.ts create mode 100644 packages/core/tests/software-registry.nightly.test.ts create mode 100644 packages/core/tests/software-registry.test.ts diff --git a/crates/client/src/agent_os.rs b/crates/client/src/agent_os.rs index 7c99589420..7e64d934f0 100644 --- a/crates/client/src/agent_os.rs +++ b/crates/client/src/agent_os.rs @@ -250,7 +250,36 @@ impl AgentOs { } } - async fn create_owned(options: AgentOsConfig) -> Result { + async fn create_owned(mut options: AgentOsConfig) -> Result { + // Resolve registry names before allocating a sidecar, so an unknown + // package or release fails without booting a VM. The packages are + // downloaded and verified by the sidecar once the VM exists. + let mut boot_sources = Vec::with_capacity(options.software_sources.len()); + for source in std::mem::take(&mut options.software_sources) { + let source = crate::software_registry::resolve_software_source( + source, + options.software_registry.as_ref(), + ) + .await?; + if !boot_sources.contains(&source) { + boot_sources.push(source); + } + } + let client = Self::create_vm(options).await?; + // A failed install disposes the VM instead of returning one without the + // requested software. + for source in boot_sources { + if let Err(error) = client.install_software(source).await { + if let Err(shutdown_error) = client.shutdown().await { + tracing::error!(%shutdown_error, "failed to dispose VM after create-time software install failed"); + } + return Err(error); + } + } + Ok(client) + } + + async fn create_vm(options: AgentOsConfig) -> Result { let config = Arc::new(options); // 1. Resolve the sidecar handle (shared "default" pool unless configured otherwise) and @@ -764,11 +793,20 @@ impl AgentOs { /// Resolve, verify, pin, and project one exact software artifact inside the /// sidecar. The client sends only the source, never a resolved host path. + /// + /// A [`crate::SoftwareSource::Registry`] source resolves on this host to the + /// release manifest's URL and digest, so the sidecar verifies the bytes + /// exactly as for an explicit URL source. pub async fn install_software( &self, - source: crate::software::PackageSource, + source: impl Into, ) -> Result { let _vm_configuration = try_vm_mutation(&self.inner.vm_configuration_operation)?; + let source = crate::software_registry::resolve_software_source( + source.into(), + self.inner.config.software_registry.as_ref(), + ) + .await?; let source = match source { crate::software::PackageSource::Url { url, diff --git a/crates/client/src/config.rs b/crates/client/src/config.rs index 566e5eb74a..def52810ea 100644 --- a/crates/client/src/config.rs +++ b/crates/client/src/config.rs @@ -57,6 +57,13 @@ pub struct AgentOsConfig { /// Trusted Core package acquisition limits and local-test transport policy. /// Hosted actor callers cannot configure this process-owned policy. pub package_resolver: Option, + /// Registry packages and URL or path sources installed after boot, before + /// [`crate::AgentOs::create`] returns (= TS `software` registry and URL + /// entries). Registry names resolve before the VM is created. Default `[]`. + pub software_sources: Vec, + /// Where registry sources resolve (= TS `softwareRegistry`). Default: + /// `https://releases.rivet.dev/agent-os/` at this client's release. + pub software_registry: Option, /// Loopback ports exempt from the default outbound-to-host block. pub loopback_exempt_ports: Vec, /// Allowed Node.js builtins. Default: the hardened native-bridge set. @@ -123,6 +130,22 @@ impl AgentOsConfigBuilder { self } + pub fn software_sources( + mut self, + sources: Vec, + ) -> Self { + self.config.software_sources = sources; + self + } + + pub fn software_registry( + mut self, + options: crate::software_registry::SoftwareRegistryOptions, + ) -> Self { + self.config.software_registry = Some(options); + self + } + pub fn loopback_exempt_ports(mut self, ports: Vec) -> Self { self.config.loopback_exempt_ports = ports; self diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index e669296efd..eb2712e637 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -26,6 +26,7 @@ mod output_replay; pub mod process; pub mod sidecar; pub mod software; +pub mod software_registry; pub mod stream; pub mod transport; @@ -163,6 +164,12 @@ pub use software::{ DEFAULT_MAX_PACKAGE_BYTES, }; +pub use software_registry::{ + default_registry_version, resolve_registry_software, ResolvedRegistrySoftware, + SoftwareRegistryOptions, SoftwareSource, DEFAULT_SOFTWARE_REGISTRY_URL, + SOFTWARE_REGISTRY_URL_ENV, +}; + pub use fs::{ BatchReadResult, BatchWriteEntry, BatchWriteResult, DirEntry, DirEntryType, DynamicMountDescriptor, FileContent, FilesystemEntry, FilesystemEntryEncoding, diff --git a/crates/client/src/software_registry.rs b/crates/client/src/software_registry.rs new file mode 100644 index 0000000000..d985e73a77 --- /dev/null +++ b/crates/client/src/software_registry.rs @@ -0,0 +1,452 @@ +//! Registry software resolution (= TS `packages/core/src/software-registry.ts`). +//! +//! Each agentOS release publishes its software catalog as digest-addressed +//! `.aospkg` files next to `//software/manifest.json`. A +//! registry source resolves to that release's URL and exact digest, then +//! installs through the ordinary URL source, so the sidecar still verifies the +//! bytes. The manifest is fetched by this host process, not by the guest, so VM +//! network permissions do not apply to it. + +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use once_cell::sync::Lazy; +use reqwest::Url; +use serde::Deserialize; + +use crate::error::ClientError; +use crate::software::PackageSource; + +/// Root that holds `/software/manifest.json` for each release. +pub const DEFAULT_SOFTWARE_REGISTRY_URL: &str = "https://releases.rivet.dev/agent-os/"; +/// Environment variable that overrides the default registry root. +pub const SOFTWARE_REGISTRY_URL_ENV: &str = "AGENTOS_SOFTWARE_REGISTRY_URL"; + +const MANIFEST_FETCH_TIMEOUT: Duration = Duration::from_secs(30); +const MANIFEST_MAX_BYTES: usize = 8 * 1024 * 1024; +const MANIFEST_CACHE_LIMIT: usize = 32; +// `latest` moves with every release; exact versions are immutable. +const LATEST_MANIFEST_TTL: Duration = Duration::from_secs(5 * 60); + +/// Registry location and default release for name-based software. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SoftwareRegistryOptions { + /// Root URL that holds `/software/manifest.json`. Defaults to + /// `AGENTOS_SOFTWARE_REGISTRY_URL`, then `https://releases.rivet.dev/agent-os/`. + pub url: Option, + /// Release to resolve sources that name no version. Defaults to this + /// crate's version, or `latest` for unreleased workspace and preview builds. + pub version: Option, +} + +/// A software source for [`crate::AgentOs::install_software`] and +/// [`crate::AgentOsConfig::software_sources`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SoftwareSource { + /// A URL or trusted local path. + Package(PackageSource), + /// A package in an agentOS release manifest, resolved on this host to a + /// digest-pinned URL source. + Registry { + name: String, + version: Option, + }, +} + +impl From for SoftwareSource { + fn from(source: PackageSource) -> Self { + SoftwareSource::Package(source) + } +} + +/// A registry package resolved to its exact artifact. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedRegistrySoftware { + pub name: String, + /// Release whose manifest listed the package, such as `0.4.2` or `latest`. + pub version: String, + /// Absolute `.aospkg` URL. + pub url: String, + /// Exact `sha256:<64 lowercase hex>` content digest. + pub digest: String, + pub size_bytes: u64, + pub manifest_url: String, +} + +#[derive(Debug, Clone, Deserialize)] +struct ManifestArtifact { + name: String, + digest: String, + size: u64, + path: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Manifest { + schema_version: serde_json::Value, + artifacts: Vec, +} + +struct CachedManifest { + manifest_url: String, + expires_at: Option, + artifacts: Arc>, +} + +// Process-wide like the TS module cache, bounded by MANIFEST_CACHE_LIMIT +// distinct registry/version pairs. The guard is never held across an await. +static MANIFEST_CACHE: Lazy>> = + Lazy::new(|| parking_lot::Mutex::new(VecDeque::new())); + +/// The release a registry source resolves against when it names no version. +/// +/// A published client matches its own release so a given agentOS version gets +/// the software built with it. The committed workspace version `0.0.1` and +/// `0.0.0-*` preview builds never publish software, so they use `latest`. +pub fn default_registry_version(version: &str) -> String { + if version == "0.0.1" || version == "0.0.0" || version.starts_with("0.0.0-") { + return String::from("latest"); + } + version.to_string() +} + +fn registry_root(options: Option<&SoftwareRegistryOptions>) -> Result { + let raw = options + .and_then(|options| options.url.clone()) + .or_else(|| std::env::var(SOFTWARE_REGISTRY_URL_ENV).ok()) + .unwrap_or_else(|| DEFAULT_SOFTWARE_REGISTRY_URL.to_string()); + let normalized = if raw.ends_with('/') { + raw.clone() + } else { + format!("{raw}/") + }; + let root = Url::parse(&normalized).map_err(|_| { + ClientError::InvalidConfig(format!( + "invalid agentOS software registry URL {raw:?}; pass an absolute http(s) URL in software_registry.url or {SOFTWARE_REGISTRY_URL_ENV}" + )) + })?; + ensure_secure_registry_url(&root, "registry URL")?; + Ok(root) +} + +// The manifest supplies the digest every package is verified against, so it +// needs the same transport integrity as the packages: HTTPS, or plain HTTP only +// to a loopback mirror. +fn ensure_secure_registry_url(url: &Url, label: &str) -> Result<(), ClientError> { + let loopback = url.host_str().is_some_and(|host| { + host == "localhost" + || host + .trim_start_matches('[') + .trim_end_matches(']') + .parse::() + .is_ok_and(|ip| ip.is_loopback()) + }); + if url.scheme() == "https" || (url.scheme() == "http" && loopback) { + return Ok(()); + } + Err(ClientError::InvalidConfig(format!( + "agentOS software {label} must use https, or plain http only on a loopback address, got {url}" + ))) +} + +fn validate_name(name: &str) -> Result<(), ClientError> { + let mut chars = name.chars(); + let valid = matches!(chars.next(), Some(c) if c.is_ascii_lowercase() || c.is_ascii_digit()) + && chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-'); + if !valid { + return Err(ClientError::InvalidPackageSource(format!( + "invalid agentOS registry package name {name:?}; names are lowercase letters, digits, and hyphens, such as \"git\"" + ))); + } + Ok(()) +} + +fn validate_version(version: &str) -> Result<(), ClientError> { + let valid = version == "latest" || { + let (core, prerelease) = match version.split_once('-') { + Some((core, prerelease)) => (core, Some(prerelease)), + None => (version, None), + }; + let parts: Vec<&str> = core.split('.').collect(); + parts.len() == 3 + && parts + .iter() + .all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit())) + && prerelease.is_none_or(|prerelease| { + !prerelease.is_empty() + && prerelease + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-') + }) + }; + if !valid { + return Err(ClientError::InvalidPackageSource(format!( + "invalid agentOS registry version {version:?}; pass a release version such as \"0.4.2\", or \"latest\"" + ))); + } + Ok(()) +} + +fn is_digest(value: &str) -> bool { + value.strip_prefix("sha256:").is_some_and(|hex| { + hex.len() == 64 && hex.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) + }) +} + +fn parse_manifest(bytes: &[u8], manifest_url: &str) -> Result, ClientError> { + let manifest: Manifest = serde_json::from_slice(bytes).map_err(|error| { + ClientError::PackageDownload(format!( + "agentOS software manifest {manifest_url} is not a schemaVersion 1 manifest with an artifacts list: {error}" + )) + })?; + if manifest.schema_version != serde_json::json!(1) { + return Err(ClientError::PackageDownload(format!( + "agentOS software manifest {manifest_url} is not a schemaVersion 1 manifest with an artifacts list" + ))); + } + manifest + .artifacts + .into_iter() + .enumerate() + .map(|(index, value)| { + serde_json::from_value::(value) + .ok() + .filter(|artifact| is_digest(&artifact.digest)) + .ok_or_else(|| { + ClientError::PackageDownload(format!( + "agentOS software manifest {manifest_url} has a malformed artifact at index {index}" + )) + }) + }) + .collect() +} + +async fn fetch_manifest( + manifest_url: &str, + version: &str, +) -> Result, ClientError> { + let unreachable = |error: &dyn std::fmt::Display| { + ClientError::PackageDownload(format!( + "failed to fetch agentOS software manifest {manifest_url}: {error}; check host network access or set software_registry.url / {SOFTWARE_REGISTRY_URL_ENV} to a reachable mirror" + )) + }; + let client = reqwest::Client::builder() + .timeout(MANIFEST_FETCH_TIMEOUT) + .build() + .map_err(|error| unreachable(&error))?; + let mut response = client + .get(manifest_url) + .send() + .await + .map_err(|error| unreachable(&error))?; + ensure_secure_registry_url(response.url(), "manifest redirect")?; + let status = response.status(); + if status == reqwest::StatusCode::NOT_FOUND { + return Err(ClientError::PackageDownload(format!( + "agentOS software registry has no release \"{version}\": GET {manifest_url} returned 404; pass a published agentOS version or \"latest\"" + ))); + } + if !status.is_success() { + return Err(ClientError::PackageDownload( + format!( + "failed to fetch agentOS software manifest {manifest_url}: HTTP {} {}", + status.as_u16(), + status.canonical_reason().unwrap_or_default() + ) + .trim_end() + .to_string(), + )); + } + if response + .content_length() + .is_some_and(|length| length > MANIFEST_MAX_BYTES as u64) + { + return Err(ClientError::PackageDownload(format!( + "agentOS software manifest {manifest_url} exceeds the {MANIFEST_MAX_BYTES}-byte limit" + ))); + } + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|error| unreachable(&error))? + { + if body.len() + chunk.len() > MANIFEST_MAX_BYTES { + return Err(ClientError::PackageDownload(format!( + "agentOS software manifest {manifest_url} exceeds the {MANIFEST_MAX_BYTES}-byte limit" + ))); + } + body.extend_from_slice(&chunk); + } + parse_manifest(&body, manifest_url) +} + +async fn load_manifest( + manifest_url: &str, + version: &str, +) -> Result>, ClientError> { + { + let mut cache = MANIFEST_CACHE.lock(); + let now = Instant::now(); + if let Some(index) = cache + .iter() + .position(|entry| entry.manifest_url == manifest_url) + { + if cache[index] + .expires_at + .is_none_or(|expires_at| expires_at > now) + { + return Ok(cache[index].artifacts.clone()); + } + cache.remove(index); + } + } + // Failed fetches are never cached, so the next call retries. + let artifacts = Arc::new(fetch_manifest(manifest_url, version).await?); + let mut cache = MANIFEST_CACHE.lock(); + cache.retain(|entry| entry.manifest_url != manifest_url); + while cache.len() >= MANIFEST_CACHE_LIMIT { + cache.pop_front(); + } + cache.push_back(CachedManifest { + manifest_url: manifest_url.to_string(), + expires_at: if version == "latest" { + Some(Instant::now() + LATEST_MANIFEST_TTL) + } else { + None + }, + artifacts: artifacts.clone(), + }); + Ok(artifacts) +} + +fn edit_distance(left: &str, right: &str) -> usize { + let right: Vec = right.chars().collect(); + let mut previous: Vec = (0..=right.len()).collect(); + for (i, left_char) in left.chars().enumerate() { + let mut current = vec![i + 1]; + for (j, right_char) in right.iter().enumerate() { + current.push( + (previous[j + 1] + 1) + .min(current[j] + 1) + .min(previous[j] + usize::from(left_char != *right_char)), + ); + } + previous = current; + } + previous[right.len()] +} + +fn close_matches<'a>(name: &str, names: impl Iterator) -> Vec<&'a str> { + let mut scored: Vec<(usize, &str)> = names + .map(|candidate| { + let distance = if candidate.contains(name) || name.contains(candidate) { + 0 + } else { + edit_distance(name, candidate) + }; + (distance, candidate) + }) + .filter(|(distance, _)| *distance <= 2) + .collect(); + scored.sort(); + scored + .into_iter() + .take(5) + .map(|(_, candidate)| candidate) + .collect() +} + +/// Resolve a registry package name to the exact `.aospkg` URL and digest listed +/// in a release manifest. `version` overrides `registry.version`. Manifests are +/// cached per registry and version for the life of the process; `latest` is +/// refreshed every five minutes. +pub async fn resolve_registry_software( + name: &str, + version: Option<&str>, + registry: Option<&SoftwareRegistryOptions>, +) -> Result { + validate_name(name)?; + let version = version + .map(str::to_string) + .or_else(|| registry.and_then(|registry| registry.version.clone())) + .unwrap_or_else(|| default_registry_version(env!("CARGO_PKG_VERSION"))); + validate_version(&version)?; + let manifest_url = registry_root(registry)? + .join(&format!("{version}/software/manifest.json")) + .map_err(|error| ClientError::InvalidConfig(format!("invalid registry URL: {error}")))?; + let manifest_url = manifest_url.to_string(); + let artifacts = load_manifest(&manifest_url, &version).await?; + let Some(artifact) = artifacts.iter().find(|artifact| artifact.name == name) else { + let suggestions = close_matches( + name, + artifacts.iter().map(|artifact| artifact.name.as_str()), + ); + let hint = if suggestions.is_empty() { + String::from("; browse https://rivet.dev/agentos/docs/registry for available packages") + } else { + format!( + "; did you mean {}?", + suggestions + .iter() + .map(|suggestion| format!("\"{suggestion}\"")) + .collect::>() + .join(", ") + ) + }; + return Err(ClientError::SoftwareNotFound(format!( + "agentOS registry release \"{version}\" has no package \"{name}\" ({manifest_url}){hint}" + ))); + }; + let url = Url::parse(&manifest_url) + .and_then(|base| base.join(&artifact.path)) + .map_err(|error| { + ClientError::PackageDownload(format!( + "agentOS software manifest {manifest_url} lists {name} at an invalid path: {error}" + )) + })?; + if url.scheme() != "https" && url.scheme() != "http" { + return Err(ClientError::PackageDownload(format!( + "agentOS software manifest {manifest_url} lists {name} at a non-http(s) URL: {url}" + ))); + } + Ok(ResolvedRegistrySoftware { + name: name.to_string(), + version, + url: url.to_string(), + digest: artifact.digest.clone(), + size_bytes: artifact.size, + manifest_url, + }) +} + +/// Resolve a [`SoftwareSource`] to the package source the sidecar acquires. +pub async fn resolve_software_source( + source: SoftwareSource, + registry: Option<&SoftwareRegistryOptions>, +) -> Result { + match source { + SoftwareSource::Package(source) => Ok(source), + SoftwareSource::Registry { name, version } => { + let resolved = resolve_registry_software(&name, version.as_deref(), registry).await?; + Ok(PackageSource::Url { + url: resolved.url, + expected_digest: Some(resolved.digest), + }) + } + } +} + +/// Test hook: forget cached manifests. +#[doc(hidden)] +pub fn clear_software_registry_cache() { + MANIFEST_CACHE.lock().clear(); +} + +/// Test hook: number of cached manifests. +#[doc(hidden)] +pub fn software_registry_cache_size() -> usize { + MANIFEST_CACHE.lock().len() +} diff --git a/crates/client/tests/software_registry.rs b/crates/client/tests/software_registry.rs new file mode 100644 index 0000000000..3e18a040ed --- /dev/null +++ b/crates/client/tests/software_registry.rs @@ -0,0 +1,395 @@ +//! Registry resolution against a local registry with the release layout +//! `/software/manifest.json`. These tests never reach the network. + +use std::collections::HashMap; +use std::sync::Arc; + +use agentos_client::config::AgentOsConfig; +use agentos_client::software_registry::software_registry_cache_size; +use agentos_client::{ + default_registry_version, resolve_registry_software, AgentOs, ClientError, + PackageResolverOptions, SoftwareRegistryOptions, SoftwareSource, +}; +use sha2::{Digest, Sha256}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; + +mod common; + +const DIGEST_HEX: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + +type Routes = HashMap)>; + +#[derive(Clone, Default)] +struct Registry { + url: String, + routes: Arc>, + requests: Arc>>, +} + +impl Registry { + async fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let registry = Registry { + url: format!("http://{}/agent-os/", listener.local_addr().unwrap()), + ..Registry::default() + }; + let served = registry.clone(); + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let served = served.clone(); + tokio::spawn(async move { + let mut request = Vec::new(); + let mut buffer = [0u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let Ok(read) = stream.read(&mut buffer).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&buffer[..read]); + } + let line = String::from_utf8_lossy(&request); + let path = line.split_whitespace().nth(1).unwrap_or("/").to_string(); + served.requests.lock().push(path.clone()); + let (status, body) = served + .routes + .lock() + .get(&path) + .cloned() + .unwrap_or((404, Vec::new())); + let head = format!( + "HTTP/1.1 {status} X\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(head.as_bytes()).await; + let _ = stream.write_all(&body).await; + }); + } + }); + registry + } + + fn options(&self) -> SoftwareRegistryOptions { + SoftwareRegistryOptions { + url: Some(self.url.clone()), + version: None, + } + } + + fn serve_raw(&self, version: &str, status: u16, body: &[u8]) { + self.routes.lock().insert( + format!("/agent-os/{version}/software/manifest.json"), + (status, body.to_vec()), + ); + } + + fn serve(&self, version: &str, names: &[&str]) { + let artifacts: Vec<_> = names + .iter() + .map(|name| { + serde_json::json!({ + "name": name, + "digest": format!("sha256:{DIGEST_HEX}"), + "size": 1, + "path": format!("packages/{name}/{DIGEST_HEX}.aospkg"), + }) + }) + .collect(); + let manifest = serde_json::json!({ "schemaVersion": 1, "artifacts": artifacts }); + self.serve_raw(version, 200, manifest.to_string().as_bytes()); + } + + fn requests(&self) -> Vec { + self.requests.lock().clone() + } +} + +#[test] +fn default_version_matches_the_client_release() { + assert_eq!(default_registry_version("0.4.2"), "0.4.2"); + assert_eq!(default_registry_version("0.4.3-rc.1"), "0.4.3-rc.1"); + assert_eq!(default_registry_version("0.0.1"), "latest"); + assert_eq!( + default_registry_version("0.0.0-my-branch.abc1234"), + "latest" + ); +} + +#[tokio::test] +async fn resolves_a_name_to_a_digest_pinned_url() { + let registry = Registry::start().await; + registry.serve("latest", &["git", "ripgrep"]); + let resolved = resolve_registry_software("git", None, Some(®istry.options())) + .await + .unwrap(); + assert_eq!(resolved.version, "latest"); + assert_eq!(resolved.digest, format!("sha256:{DIGEST_HEX}")); + assert_eq!( + resolved.url, + format!( + "{}latest/software/packages/git/{DIGEST_HEX}.aospkg", + registry.url + ) + ); + assert_eq!( + resolved.manifest_url, + format!("{}latest/software/manifest.json", registry.url) + ); +} + +#[tokio::test] +async fn explicit_versions_win_and_manifests_are_cached() { + let registry = Registry::start().await; + registry.serve("0.4.2", &["git"]); + registry.serve("0.4.1", &["git"]); + let pinned = SoftwareRegistryOptions { + version: Some(String::from("0.4.1")), + ..registry.options() + }; + let explicit = resolve_registry_software("git", Some("0.4.2"), Some(&pinned)) + .await + .unwrap(); + assert_eq!(explicit.version, "0.4.2"); + for _ in 0..2 { + let defaulted = resolve_registry_software("git", None, Some(&pinned)) + .await + .unwrap(); + assert_eq!(defaulted.version, "0.4.1"); + } + assert_eq!( + registry.requests(), + vec![ + "/agent-os/0.4.2/software/manifest.json", + "/agent-os/0.4.1/software/manifest.json", + ] + ); +} + +#[tokio::test] +async fn the_manifest_cache_is_bounded() { + let registry = Registry::start().await; + for patch in 0..40 { + let version = format!("1.0.{patch}"); + registry.serve(&version, &["git"]); + resolve_registry_software("git", Some(&version), Some(®istry.options())) + .await + .unwrap(); + } + assert!(software_registry_cache_size() <= 32); +} + +#[tokio::test] +async fn failed_fetches_are_not_cached() { + let registry = Registry::start().await; + registry.serve_raw("latest", 503, b""); + let error = resolve_registry_software("git", None, Some(®istry.options())) + .await + .unwrap_err(); + assert!( + error.to_string().contains(&format!( + "failed to fetch agentOS software manifest {}latest/software/manifest.json: HTTP 503", + registry.url + )), + "{error}" + ); + registry.serve("latest", &["git"]); + resolve_registry_software("git", None, Some(®istry.options())) + .await + .unwrap(); +} + +#[tokio::test] +async fn unknown_names_list_close_matches() { + let registry = Registry::start().await; + registry.serve("latest", &["git", "gitleaks", "ripgrep"]); + let error = resolve_registry_software("gti", None, Some(®istry.options())) + .await + .unwrap_err(); + assert!(matches!(error, ClientError::SoftwareNotFound(_))); + assert!( + error.to_string().contains("did you mean \"git\"?"), + "{error}" + ); +} + +#[tokio::test] +async fn missing_releases_and_bad_input_fail_explicitly() { + let registry = Registry::start().await; + let missing = resolve_registry_software("git", Some("9.9.9"), Some(®istry.options())) + .await + .unwrap_err(); + assert!( + missing.to_string().contains(&format!( + "has no release \"9.9.9\": GET {}9.9.9/software/manifest.json returned 404", + registry.url + )), + "{missing}" + ); + registry.serve_raw("latest", 200, b"{not json"); + let malformed = resolve_registry_software("git", None, Some(®istry.options())) + .await + .unwrap_err(); + assert!( + malformed.to_string().contains("schemaVersion 1"), + "{malformed}" + ); + for (name, version, message) in [ + ("../git", None, "invalid agentOS registry package name"), + ("git", Some("../latest"), "invalid agentOS registry version"), + ] { + let error = resolve_registry_software(name, version, Some(®istry.options())) + .await + .unwrap_err(); + assert!(error.to_string().contains(message), "{error}"); + } + for url in ["ftp://mirror/", "http://mirror.example.com/"] { + let options = SoftwareRegistryOptions { + url: Some(url.to_string()), + version: None, + }; + let error = resolve_registry_software("git", None, Some(&options)) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("must use https, or plain http only on a loopback address"), + "{error}" + ); + } +} + +fn test_package() -> Vec { + let mut builder = tar::Builder::new(Vec::::new()); + let manifest = br#"{"name":"registry-tool","version":"1.0.0"}"#; + let mut header = tar::Header::new_gnu(); + header.set_size(manifest.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "agentos-package.json", &manifest[..]) + .unwrap(); + let command = b"#!/usr/bin/env node\nprocess.stdout.write('registry-ok\\n');\n"; + let mut header = tar::Header::new_gnu(); + header.set_size(command.len() as u64); + header.set_mode(0o755); + header.set_cksum(); + builder + .append_data(&mut header, "bin/registry-cmd", &command[..]) + .unwrap(); + let tar = builder.into_inner().unwrap(); + vfs::package_format::pack::pack_aospkg_from_tar_bytes(&tar) + .unwrap() + .0 +} + +fn serve_package(registry: &Registry, digest_override: Option<&str>) -> String { + let bytes = test_package(); + let hex = format!("{:x}", Sha256::digest(&bytes)); + let manifest = serde_json::json!({ + "schemaVersion": 1, + "artifacts": [{ + "name": "registry-tool", + "digest": digest_override.map(str::to_string).unwrap_or(format!("sha256:{hex}")), + "size": bytes.len(), + "path": format!("packages/registry-tool/{hex}.aospkg"), + }], + }); + registry.serve_raw("latest", 200, manifest.to_string().as_bytes()); + registry.routes.lock().insert( + format!("/agent-os/latest/software/packages/registry-tool/{hex}.aospkg"), + (200, bytes), + ); + format!("sha256:{hex}") +} + +fn vm_config(registry: &Registry, sources: Vec) -> AgentOsConfig { + AgentOsConfig { + software_sources: sources, + software_registry: Some(registry.options()), + package_resolver: Some(PackageResolverOptions { + allow_insecure_local_http: true, + ..PackageResolverOptions::default() + }), + ..AgentOsConfig::default() + } +} + +fn registry_tool() -> SoftwareSource { + SoftwareSource::Registry { + name: String::from("registry-tool"), + version: None, + } +} + +#[tokio::test] +async fn create_installs_registry_sources_before_returning() { + if !common::require_sidecar("create_installs_registry_sources_before_returning") { + return; + } + let registry = Registry::start().await; + let digest = serve_package(®istry, None); + let os = AgentOs::create(vm_config(®istry, vec![registry_tool()])) + .await + .expect("create VM"); + let installed = os.installed_software(); + assert_eq!(installed.len(), 1); + assert_eq!(installed[0].digest, digest); + assert!(os + .exists("/opt/agentos/bin/registry-cmd") + .await + .expect("check linked command")); + os.shutdown().await.expect("shutdown VM"); +} + +#[tokio::test] +async fn install_software_accepts_a_registry_source() { + if !common::require_sidecar("install_software_accepts_a_registry_source") { + return; + } + let registry = Registry::start().await; + let digest = serve_package(®istry, None); + let os = AgentOs::create(vm_config(®istry, Vec::new())) + .await + .expect("create VM"); + let installed = os.install_software(registry_tool()).await.expect("install"); + assert_eq!(installed.digest, digest); + assert_eq!(installed.package_name, "registry-tool"); + os.shutdown().await.expect("shutdown VM"); +} + +#[tokio::test] +async fn create_fails_on_digest_mismatch_and_unknown_names() { + if !common::require_sidecar("create_fails_on_digest_mismatch_and_unknown_names") { + return; + } + let registry = Registry::start().await; + serve_package(®istry, Some(&format!("sha256:{}", "0".repeat(64)))); + let mismatch = AgentOs::create(vm_config(®istry, vec![registry_tool()])) + .await + .err() + .expect("digest mismatch must fail create"); + assert!(mismatch.to_string().contains("digest"), "{mismatch}"); + + let unknown = AgentOs::create(vm_config( + ®istry, + vec![SoftwareSource::Registry { + name: String::from("registry-tol"), + version: None, + }], + )) + .await + .err() + .expect("unknown name must fail create"); + assert!( + unknown + .to_string() + .contains("did you mean \"registry-tool\"?"), + "{unknown}" + ); +} diff --git a/docs/content/docs/custom-software/publishing.mdx b/docs/content/docs/custom-software/publishing.mdx index a4888d6010..e1867907c2 100644 --- a/docs/content/docs/custom-software/publishing.mdx +++ b/docs/content/docs/custom-software/publishing.mdx @@ -31,6 +31,8 @@ https://releases.rivet.dev/agent-os/latest/software/manifest.json ``` The same directory also holds `default-software.json`, the manifest of the -packages every VM installs by default. Consumers pass an artifact URL and -digest from the manifest directly to `software.install`; see -[Software](/agentos/docs/software#install-while-running). +packages every VM installs by default. The library resolves registry names such +as `{ registry: "git" }` against this manifest. To serve packages from a mirror, +copy the `/software/` layout and point `softwareRegistry.url` or +`AGENTOS_SOFTWARE_REGISTRY_URL` at its root; see +[Software](/agentos/docs/software#registry-packages). diff --git a/docs/content/docs/software.mdx b/docs/content/docs/software.mdx index 44aa12bf63..15510eb3a7 100644 --- a/docs/content/docs/software.mdx +++ b/docs/content/docs/software.mdx @@ -13,6 +13,40 @@ Browse the available packages in the [registry](/agentos/docs/registry). ## Registry packages +Install a registry package by name. In the library, list +`{ registry: "git" }` in `software` when the VM is created, or call +`vm.software.install({ type: "registry", name: "git" })` while it is running. +See the examples under [Install while running](#install-while-running) and +[Install when the VM is created](#install-when-the-vm-is-created). + +A name resolves against the software manifest of one agentOS release, and the +package is installed from its URL with the manifest's digest, so the bytes are +verified before they are projected. Unknown names, missing releases, unreachable +manifests, and digest mismatches fail with an error instead of being skipped. + +- **Version.** By default a name resolves against the release that matches your + `@rivet-dev/agentos-core` version, so a client gets the software built with + it. Unreleased workspace and preview builds use `latest`. Pin one entry with + `{ registry: "git", version: "0.4.2" }` or + `{ type: "registry", name: "git", version: "0.4.2" }`, or every entry with + `softwareRegistry: { version }`. Pass `"latest"` for the newest release. +- **Mirrors.** Set `softwareRegistry: { url }` on `AgentOs.create()`, or the + `AGENTOS_SOFTWARE_REGISTRY_URL` environment variable, to a copy of the release + layout. The default is `https://releases.rivet.dev/agent-os/`. Mirrors must + use HTTPS, because the manifest supplies the digests packages are verified + against. A plain `http://` mirror is allowed only on a loopback address and + also needs `packageResolver: { allowInsecureLocalHttp: true }`, which sets + the download limits for registry and URL packages. +- **Network.** The manifest and package are downloaded by the host process, not + by the VM, so the VM's network permissions do not apply to them. +- **Caching.** Manifests are cached per release for the life of the process, + and `latest` is refreshed every five minutes. + +`resolveRegistrySoftware(name, { version })` returns the resolved `url` and +`digest` without installing anything. + +### Release manifest + Registry packages are published with each release under `https://releases.rivet.dev/agent-os//software/`, or `latest` for the newest release. Its `manifest.json` lists each package's name, digest, size, @@ -32,10 +66,10 @@ and path relative to the manifest: } ``` -Resolve `path` against the manifest URL to get the package URL, and pass -`digest` along so the bytes are verified before they are installed. Registry -software is distributed only as `.aospkg` files from this location, not as npm -packages. See [Publishing Software](/agentos/docs/custom-software/publishing). +Use it when you need raw URLs, such as to mirror packages or to configure a +hosted VM: resolve `path` against the manifest URL and pass `digest` along. +Registry software is distributed only as `.aospkg` files from this location, not +as npm packages. See [Publishing Software](/agentos/docs/custom-software/publishing). ## Default software @@ -48,8 +82,8 @@ in a hosted VM's `config`, for a bare VM that has only the software you list. ## Install while running Install a package into a running VM from an HTTPS URL, with an optional exact -`sha256:<64 lowercase hex>` digest. The library also accepts a trusted local -path. +`sha256:<64 lowercase hex>` digest. The library also accepts a registry package +name and a trusted local path. @@ -59,8 +93,8 @@ import { createAgentOsClient } from "@rivet-dev/agentos"; const client = createAgentOsClient({ endpoint: "https://api.rivet.dev" }); const vm = client.agentOS.getOrCreate(["my-vm"]); -// Registry software is published as digest-addressed .aospkg files next to a -// manifest. Use a release version, or `latest`. +// Hosted install takes a URL and digest. Look the package up in a release +// manifest; use a release version, or `latest`. const base = "https://releases.rivet.dev/agent-os/latest/software/"; const { artifacts } = await (await fetch(`${base}manifest.json`)).json(); const ripgrep = artifacts.find((artifact: { name: string }) => artifact.name === "ripgrep"); @@ -108,7 +142,8 @@ import { createAgentOsClient } from "@rivet-dev/agentos"; const client = createAgentOsClient({ endpoint: "https://api.rivet.dev" }); -// Look up the package in a release manifest. Use a release version, or `latest`. +// Hosted config takes a URL and digest. Look the package up in a release +// manifest; use a release version, or `latest`. const base = "https://releases.rivet.dev/agent-os/latest/software/"; const { artifacts } = await (await fetch(`${base}manifest.json`)).json(); const git = artifacts.find((artifact: { name: string }) => artifact.name === "git"); @@ -126,13 +161,15 @@ const vm = client.agentOS.getOrCreate(["my-vm"], { +The library's `software` option accepts trusted local `{ packagePath }` files, +registry packages by name (`{ registry: "git" }`), and `{ url, digest }` +artifacts. Registry and URL entries are downloaded and verified on the host +before `AgentOs.create()` returns, and they appear in `software.installed()`. + Hosted VMs take `{ url, digest }` sources in their `software` config, and -download and verify them when the VM starts. The library's `software` option -takes `{ packagePath }` references to trusted `.aospkg` files on the host, so -the library example downloads the registry package once, verifies its digest, -and passes the cached file. To install from a URL without a local file, create -the VM and call [`software.install`](#install-while-running) before you run -anything that needs the package. +download and verify them when the VM starts. The hosted client does not resolve +registry names yet, so look the package up in the +[release manifest](#release-manifest) as shown above. ## Custom and local packages @@ -164,5 +201,5 @@ download. Hosted workers also warm popular packages at startup; see ## Embedded API The embedding process is trusted, so `software.install` accepts local package -paths as well as URLs. Read more in the +paths as well as registry names and URLs. Read more in the [embedded API quickstart](/agentos/docs/quickstart-embedded). diff --git a/examples/embedded/config-reference.ts b/examples/embedded/config-reference.ts index 0ee1bbc6dc..89333993c0 100644 --- a/examples/embedded/config-reference.ts +++ b/examples/embedded/config-reference.ts @@ -19,8 +19,12 @@ const vm = await AgentOs.create({ // Kernel permission policy (see /agentos/docs/permissions) and runtime caps. permissions: { network: "allow" }, limits: { jsRuntime: { v8HeapLimitMb: 128 } }, - // Install a packed .aospkg from a trusted local path at boot. - software: [{ packagePath: "/path/to/tool.aospkg" }], + // Install a packed .aospkg from a trusted local path, and a registry package + // by name, at boot. + software: [{ packagePath: "/path/to/tool.aospkg" }, { registry: "git" }], + // Where registry packages resolve. Defaults to releases.rivet.dev at this + // client's release; point `url` at a mirror or pin `version`. + softwareRegistry: { url: "https://releases.rivet.dev/agent-os/" }, // Also install the default software bundle (sh + coreutils). Defaults to true; // set false for a bare VM with only the software you list. defaultSoftware: true, diff --git a/examples/embedded/software-install.ts b/examples/embedded/software-install.ts index d17349b25f..43a4226138 100644 --- a/examples/embedded/software-install.ts +++ b/examples/embedded/software-install.ts @@ -1,23 +1,14 @@ import { AgentOs } from "@rivet-dev/agentos-core"; -// Registry software is published as digest-addressed .aospkg files next to a -// manifest. Use a release version, or `latest`. -const base = "https://releases.rivet.dev/agent-os/latest/software/"; -const manifest = (await (await fetch(`${base}manifest.json`)).json()) as { - artifacts: { name: string; digest: string; path: string }[]; -}; -const ripgrep = manifest.artifacts.find((artifact) => artifact.name === "ripgrep"); -if (!ripgrep) throw new Error("ripgrep is not in the manifest"); - const vm = await AgentOs.create(); try { - // Install a package while the VM is running. The bytes are verified against - // the digest before the package is projected into /opt/agentos. + // Install a registry package while the VM is running. The bytes are verified + // against the published digest before the package is projected into + // /opt/agentos. const installed = await vm.software.install({ - type: "url", - url: new URL(ripgrep.path, base).href, - expectedDigest: ripgrep.digest, + type: "registry", + name: "ripgrep", }); console.log(installed.packageName, installed.version, installed.commands); diff --git a/examples/embedded/software.ts b/examples/embedded/software.ts index 9cd43cbf84..0eb98b3e35 100644 --- a/examples/embedded/software.ts +++ b/examples/embedded/software.ts @@ -1,42 +1,10 @@ -import { createHash } from "node:crypto"; -import { existsSync } from "node:fs"; -import { mkdir, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; import { AgentOs } from "@rivet-dev/agentos-core"; -// Registry software is published as digest-addressed .aospkg files next to a -// manifest. Use a release version, or `latest`. -const base = "https://releases.rivet.dev/agent-os/latest/software/"; -const { artifacts } = (await (await fetch(`${base}manifest.json`)).json()) as { - artifacts: { name: string; digest: string; path: string }[]; -}; - -// The library installs packages at creation from trusted local paths, so -// download each registry package once, verify its digest, and reuse the file. -async function downloadPackage(name: string): Promise { - const artifact = artifacts.find((entry) => entry.name === name); - if (!artifact) throw new Error(`${name} is not in the manifest`); - const hex = artifact.digest.replace(/^sha256:/, ""); - const cacheDir = join(tmpdir(), "agentos-software"); - const packagePath = join(cacheDir, `${name}-${hex}.aospkg`); - if (existsSync(packagePath)) return packagePath; - - const response = await fetch(new URL(artifact.path, base)); - if (!response.ok) - throw new Error(`download ${name}: HTTP ${response.status}`); - const bytes = new Uint8Array(await response.arrayBuffer()); - const actual = createHash("sha256").update(bytes).digest("hex"); - if (actual !== hex) throw new Error(`${name} digest mismatch: ${actual}`); - await mkdir(cacheDir, { recursive: true }); - await writeFile(packagePath, bytes); - return packagePath; -} - // The default bundle (sh, coreutils, and the standard CLI tools) is installed -// first, then the packages listed here. +// first, then the packages listed here. Registry packages resolve from this +// client's release and are verified against their published digest. const vm = await AgentOs.create({ - software: [{ packagePath: await downloadPackage("git") }], + software: [{ registry: "git" }], }); try { diff --git a/examples/quickstart/git/README.md b/examples/quickstart/git/README.md index 161abe689d..6676e2bf55 100644 --- a/examples/quickstart/git/README.md +++ b/examples/quickstart/git/README.md @@ -9,7 +9,7 @@ Run real `git` inside a VM: initialize a repo, branch, commit, and clone it — ## How it works -The VM is created with filesystem, child-process, and env permissions enabled, then installs `git` from the agentOS software registry by looking it up in the release `manifest.json` and passing its URL and digest to `vm.software.install`. From there it drives ordinary git commands through `vm.process.exec`, mixing in `vm.filesystem.writeFile`/`vm.filesystem.readFile` to stage files and inspect results. It commits on a `feature` branch so that `git clone` resolves HEAD to `feature`, then reads the clone's `.git/HEAD` and the feature file to confirm the branch carried over. +The VM is created with filesystem, child-process, and env permissions enabled, and installs `git` from the agentOS software registry with `software: [{ registry: "git" }]`, which resolves the package from the release manifest and verifies its digest. From there it drives ordinary git commands through `vm.process.exec`, mixing in `vm.filesystem.writeFile`/`vm.filesystem.readFile` to stage files and inspect results. It commits on a `feature` branch so that `git clone` resolves HEAD to `feature`, then reads the clone's `.git/HEAD` and the feature file to confirm the branch carried over. ## Run it diff --git a/examples/quickstart/git/index.ts b/examples/quickstart/git/index.ts index a55ef1c179..dfd4e3f177 100644 --- a/examples/quickstart/git/index.ts +++ b/examples/quickstart/git/index.ts @@ -44,24 +44,11 @@ function parseHeadRef(content: string): string { return branch; } +// Install git from the agentOS software registry. The package is resolved from +// this client's release manifest and its bytes are verified before install. const vm = await AgentOs.create({ permissions: GIT_QUICKSTART_PERMISSIONS, -}); - -// Install git from the agentOS software registry. The release manifest lists -// each package's URL and digest, and the bytes are verified before install. -const SOFTWARE_BASE = "https://releases.rivet.dev/agent-os/latest/software/"; -const manifest = (await ( - await fetch(`${SOFTWARE_BASE}manifest.json`) -).json()) as { artifacts: { name: string; digest: string; path: string }[] }; -const gitArtifact = manifest.artifacts.find( - (artifact) => artifact.name === "git", -); -if (!gitArtifact) throw new Error("git is not in the software manifest"); -await vm.software.install({ - type: "url", - url: new URL(gitArtifact.path, SOFTWARE_BASE).href, - expectedDigest: gitArtifact.digest, + software: [{ registry: "git" }], }); async function run(command: string): Promise { diff --git a/packages/core/package.json b/packages/core/package.json index b5a4e873d4..b312f7bae5 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -54,7 +54,7 @@ "check-types": "tsc --noEmit && tsc -p tsconfig.type-tests.json", "build": "tsc && node scripts/stage-default-software.mjs", "test": "vitest run --exclude '**/*.nightly.test.ts' --reporter=verbose", - "test:unit": "vitest run tests/agentos-package.test.ts tests/allowed-node-builtins.test.ts tests/host-functions-zod.test.ts tests/host-functions.test.ts tests/host-function-command-exec.test.ts tests/host-function-permissions.test.ts tests/host-function-globals.test.ts tests/cron-manager.test.ts tests/code-mode-execution.test.ts tests/cron-timer-driver.test.ts tests/exec-timeout.test.ts tests/execution-abort.test.ts tests/generated-protocol.test.ts tests/leak-agent-os-processes.test.ts tests/leak-rpc-client.test.ts tests/mount-descriptors.test.ts tests/mount-reconfigure.test.ts tests/network-deny-vm-local.test.ts tests/options-schema.test.ts tests/output-replay.test.ts tests/public-api-exports.test.ts tests/root-filesystem-descriptors.test.ts tests/runtime-compat-mount.test.ts tests/sidecar-client.test.ts tests/sidecar-exit.test.ts tests/sidecar-permission-descriptors.test.ts tests/software-install.test.ts tests/wasm-permission-tiers.test.ts --fileParallelism=false", + "test:unit": "vitest run tests/agentos-package.test.ts tests/allowed-node-builtins.test.ts tests/host-functions-zod.test.ts tests/host-functions.test.ts tests/host-function-command-exec.test.ts tests/host-function-permissions.test.ts tests/host-function-globals.test.ts tests/cron-manager.test.ts tests/code-mode-execution.test.ts tests/cron-timer-driver.test.ts tests/exec-timeout.test.ts tests/execution-abort.test.ts tests/generated-protocol.test.ts tests/leak-agent-os-processes.test.ts tests/leak-rpc-client.test.ts tests/mount-descriptors.test.ts tests/mount-reconfigure.test.ts tests/network-deny-vm-local.test.ts tests/options-schema.test.ts tests/output-replay.test.ts tests/public-api-exports.test.ts tests/root-filesystem-descriptors.test.ts tests/runtime-compat-mount.test.ts tests/sidecar-client.test.ts tests/sidecar-exit.test.ts tests/sidecar-permission-descriptors.test.ts tests/software-install.test.ts tests/software-registry.test.ts tests/wasm-permission-tiers.test.ts --fileParallelism=false", "test:pr": "pnpm test:unit", "test:nightly": "vitest run tests/*.nightly.test.ts --reporter=verbose --passWithNoTests" }, diff --git a/packages/core/src/agent-os.ts b/packages/core/src/agent-os.ts index 419b5368c1..d541dfb70b 100644 --- a/packages/core/src/agent-os.ts +++ b/packages/core/src/agent-os.ts @@ -223,10 +223,35 @@ export interface BatchReadResult { error?: string; } -/** The hosted actor accepts only URL sources; embedded Core also accepts trusted paths. */ +/** + * The hosted actor accepts only URL sources; embedded Core also accepts trusted + * paths. A `registry` source names a package in an agentOS release manifest and + * is resolved by this host process to a digest-pinned `url` source. + */ export type SoftwarePackageSource = | { type: "url"; url: string; expectedDigest?: string } - | { type: "path"; path: string; expectedDigest?: string }; + | { type: "path"; path: string; expectedDigest?: string } + | { type: "registry"; name: string; version?: string }; + +/** + * Trusted limits for URL package downloads, which the sidecar performs on the + * host. Omitted fields use the sidecar defaults. + */ +export interface PackageResolverOptions { + /** Maximum `.aospkg` size in bytes. */ + maxPackageBytes?: number; + /** Whole-download timeout in milliseconds. */ + downloadTimeoutMs?: number; + /** Connect timeout in milliseconds. */ + connectTimeoutMs?: number; + /** Maximum redirects to follow (at most 16). */ + maxRedirects?: number; + /** + * Allow plain `http://` package URLs on loopback addresses, for local + * mirrors and tests. Defaults to `false`. + */ + allowInsecureLocalHttp?: boolean; +} export interface InstalledSoftware { packageId: string; @@ -296,6 +321,10 @@ import { type SidecarResponsePayload, serializeRootFilesystemForSidecar, } from "./sidecar/rpc-client.js"; +import { + resolveRegistrySoftware, + type SoftwareRegistryOptions, +} from "./software-registry.js"; export interface AgentOsSharedSidecarOptions { pool?: string; @@ -586,10 +615,22 @@ export interface AgentOsOptions< */ environment?: Record; /** - * Software to install in the VM. Each entry is a package-dir ref. Arrays are - * flattened, so meta-packages that export arrays of sub-packages work directly. + * Software to install in the VM: trusted local `{ packagePath }` files, + * registry packages by name (`{ registry: "git" }`), or `{ url, digest }` + * artifacts. Arrays are flattened, so meta-packages that export arrays of + * sub-packages work directly. Registry and URL entries are downloaded and + * verified on the host before `create()` resolves, and appear in + * `software.installed()`. */ software?: SoftwareInput[]; + /** + * Where registry entries resolve. Defaults to + * `https://releases.rivet.dev/agent-os/` (or `AGENTOS_SOFTWARE_REGISTRY_URL`) + * at this client's own release. + */ + softwareRegistry?: SoftwareRegistryOptions; + /** Trusted download limits for registry and URL packages. */ + packageResolver?: PackageResolverOptions; /** * Whether to auto-include Core's vendored default software bundle (`sh`, * coreutils, and the standard CLI tools programs rely on) in addition to @@ -693,6 +734,79 @@ function normalizePackageRef(value: unknown): NormalizedPackageRef | undefined { return undefined; } +type CreateTimeSoftware = + | { kind: "path"; path: string } + | { kind: "registry"; name: string; version?: string } + | { kind: "url"; url: string; digest?: string }; + +// Every create-time entry must be recognized: silently dropping one would boot +// a VM without the package and with no diagnostic. +function classifySoftwareEntry(entry: unknown): CreateTimeSoftware { + const ref = normalizePackageRef(entry); + if (ref) return { kind: "path", path: ref.path }; + const record = toRecord(entry); + if (typeof record.registry === "string") { + if (record.version !== undefined && typeof record.version !== "string") { + throw new Error( + `agentOS \`software\` registry entry "${record.registry}" has a non-string version; pass { registry, version: "0.4.2" }`, + ); + } + return { + kind: "registry", + name: record.registry, + ...(record.version !== undefined + ? { version: record.version as string } + : {}), + }; + } + if (typeof record.url === "string") { + if (record.digest !== undefined && typeof record.digest !== "string") { + throw new Error( + `agentOS \`software\` URL entry ${record.url} has a non-string digest; pass { url, digest: "sha256:<64 lowercase hex>" }`, + ); + } + return { + kind: "url", + url: record.url, + ...(record.digest !== undefined + ? { digest: record.digest as string } + : {}), + }; + } + throw new Error( + `agentOS \`software\` entry is not a package ref (got ${entry === null ? "null" : typeof entry}); pass { packagePath }, { registry: "" }, or { url, digest }`, + ); +} + +function packageResolverWireOptions( + options: PackageResolverOptions | undefined, +): { + max_package_bytes?: bigint; + download_timeout_ms?: bigint; + connect_timeout_ms?: bigint; + max_redirects?: number; + allow_insecure_local_http?: boolean; +} { + if (!options) return {}; + return { + ...(options.maxPackageBytes !== undefined + ? { max_package_bytes: BigInt(options.maxPackageBytes) } + : {}), + ...(options.downloadTimeoutMs !== undefined + ? { download_timeout_ms: BigInt(options.downloadTimeoutMs) } + : {}), + ...(options.connectTimeoutMs !== undefined + ? { connect_timeout_ms: BigInt(options.connectTimeoutMs) } + : {}), + ...(options.maxRedirects !== undefined + ? { max_redirects: options.maxRedirects } + : {}), + ...(options.allowInsecureLocalHttp !== undefined + ? { allow_insecure_local_http: options.allowInsecureLocalHttp } + : {}), + }; +} + const CLOSED_SHELL_ID_RETENTION_LIMIT = 2048; const TERMINAL_LIMIT = 1024; const PROCESS_REGISTRY_LIMIT = 1024; @@ -2644,6 +2758,8 @@ export class AgentOs { private _shellCounter = 0; private _softwareRoots: SoftwareRoot[]; private readonly _installedSoftware = new Map(); + private _softwareRegistry: SoftwareRegistryOptions | undefined; + private _packageResolver: PackageResolverOptions | undefined; private _vmMutationActive = false; private _cronManager!: CronManager; private _hostFunctions: ResolvedHostFunctions[] = []; @@ -2842,14 +2958,44 @@ export class AgentOs { // this the sidecar rejects the second projection with a duplicate-command // error (e.g. coreutils' `[`). const seenPackagePaths = new Set(); - const sidecarPackages = flatSoftware.flatMap((entry) => { - const ref = normalizePackageRef(entry); - if (!ref || seenPackagePaths.has(ref.path)) { - return []; + const sidecarPackages: { path: string }[] = []; + const pendingSoftware: Exclude[] = []; + for (const entry of flatSoftware) { + const classified = classifySoftwareEntry(entry); + if (classified.kind !== "path") { + pendingSoftware.push(classified); + continue; } - seenPackagePaths.add(ref.path); - return [{ path: ref.path }]; - }); + if (seenPackagePaths.has(classified.path)) continue; + seenPackagePaths.add(classified.path); + sidecarPackages.push({ path: classified.path }); + } + // Resolve registry names before allocating a sidecar, so an unknown + // package or release fails without booting a VM. The packages themselves + // are downloaded and verified by the sidecar once the VM exists. + const seenSoftwareUrls = new Set(); + const postBootSoftware: Extract[] = + []; + for (const pending of pendingSoftware) { + const source = + pending.kind === "registry" + ? await resolveRegistrySoftware(pending.name, { + version: pending.version, + registry: options?.softwareRegistry, + }).then((resolved) => ({ + type: "url" as const, + url: resolved.url, + expectedDigest: resolved.digest, + })) + : { + type: "url" as const, + url: pending.url, + expectedDigest: pending.digest, + }; + if (seenSoftwareUrls.has(source.url)) continue; + seenSoftwareUrls.add(source.url); + postBootSoftware.push(source); + } // All package software is projected into `/opt/agentos` by the sidecar. The // client stages nothing host-side and parses no package manifests. const localMounts = await resolveCompatLocalMounts(options?.mounts); @@ -3105,6 +3251,7 @@ export class AgentOs { }; let sidecarLease: AgentOsSidecarVmLease | null = null; + let createdVm!: AgentOs; try { sidecarLease = await leaseAgentOsSidecarVm(sidecar, { @@ -3125,6 +3272,8 @@ export class AgentOs { options?.onLimitWarning, ); vm._sidecarLease = sidecarLease; + vm._softwareRegistry = options?.softwareRegistry; + vm._packageResolver = options?.packageResolver; vm._hostFunctions = vmAdmin.hostFunctions; vm._hostFunctionReference = vmAdmin.hostFunctionReference; vm._disposeHooks.push(...sandboxDisposeHooks); @@ -3133,8 +3282,7 @@ export class AgentOs { vm, options?.scheduleDriver ?? new TimerScheduleDriver(), ); - - return vm; + createdVm = vm; } catch (error) { const cleanupErrors: unknown[] = []; try { @@ -3158,6 +3306,27 @@ export class AgentOs { } throw error; } + + // Registry and URL entries install through the same sidecar path as + // `software.install`, so the bytes are digest-verified before the VM is + // returned. A failed install disposes the VM instead of returning one + // without the requested software. + try { + for (const source of postBootSoftware) { + await createdVm._installSoftware(source); + } + } catch (error) { + try { + await createdVm.dispose(); + } catch (disposeError) { + throw new AggregateError( + [error, disposeError], + "AgentOS VM software install and cleanup failed", + ); + } + throw error; + } + return createdVm; } async createContext(contextId: string): Promise { @@ -5440,22 +5609,48 @@ export class AgentOs { source: SoftwarePackageSource, ): Promise { return this._withVmMutation(async () => { - const wireSource: LivePackageAcquisitionSource = - source.type === "url" - ? { - type: "url", - url: source.url, - expected_digest: source.expectedDigest, - } - : { - type: "path", - path: source.path, - expected_digest: source.expectedDigest, - }; + let wireSource: LivePackageAcquisitionSource; + switch (source.type) { + case "url": + wireSource = { + type: "url", + url: source.url, + expected_digest: source.expectedDigest, + }; + break; + case "path": + wireSource = { + type: "path", + path: source.path, + expected_digest: source.expectedDigest, + }; + break; + case "registry": { + // Resolved on this host; the sidecar verifies the bytes against + // the manifest digest exactly as for an explicit URL source. + const resolved = await resolveRegistrySoftware(source.name, { + version: source.version, + registry: this._softwareRegistry, + }); + wireSource = { + type: "url", + url: resolved.url, + expected_digest: resolved.digest, + }; + break; + } + default: + throw new Error( + `unsupported software source type ${JSON.stringify((source as { type?: unknown }).type)}; use "registry", "url", or "path"`, + ); + } const result = await this._sidecarClient.installPackage( this._sidecarSession, this._sidecarVm, - { source: wireSource }, + { + source: wireSource, + ...packageResolverWireOptions(this._packageResolver), + }, ); const packageInfo = result.package; const installed: InstalledSoftware = { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 41e12f24ad..2738a3c142 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -67,6 +67,11 @@ export type { VirtualStat, } from "./runtime.js"; export { KernelError } from "./runtime-compat.js"; +export { + DEFAULT_SOFTWARE_REGISTRY_URL, + resolveRegistrySoftware, + SOFTWARE_REGISTRY_URL_ENV, +} from "./software-registry.js"; export { createSandboxFs, createSandboxHostFunctions, diff --git a/packages/core/src/options-schema.ts b/packages/core/src/options-schema.ts index 098dffe4cc..91fc4c92ad 100644 --- a/packages/core/src/options-schema.ts +++ b/packages/core/src/options-schema.ts @@ -379,6 +379,23 @@ export const agentOsOptionFieldSchemas = { environment: z.record(z.string(), z.string()).optional(), software: z.array(z.unknown()).optional(), defaultSoftware: z.boolean().optional(), + softwareRegistry: z + .object({ + url: z.string().min(1).optional(), + version: z.string().min(1).optional(), + }) + .strict() + .optional(), + packageResolver: z + .object({ + maxPackageBytes: z.number().int().positive().optional(), + downloadTimeoutMs: z.number().int().positive().optional(), + connectTimeoutMs: z.number().int().positive().optional(), + maxRedirects: z.number().int().min(0).max(16).optional(), + allowInsecureLocalHttp: z.boolean().optional(), + }) + .strict() + .optional(), loopbackExemptPorts: z.array(z.number().int().min(0).max(65535)).optional(), allowedNodeBuiltins: stringArray.optional(), highResolutionTime: z.boolean().optional(), diff --git a/packages/core/src/packages.ts b/packages/core/src/packages.ts index 632083a742..e48011e819 100644 --- a/packages/core/src/packages.ts +++ b/packages/core/src/packages.ts @@ -3,11 +3,33 @@ import type { SoftwarePackageRef } from "./agentos-package.js"; // ── Software Descriptor Types ──────────────────────────────────────── /** - * Input type for the `software` option. Software is a self-contained package - * directory that the agentos sidecar materializes into the `/opt/agentos` - * projection. Accepts a package-dir ref or an array of refs for meta-packages. + * Install a package from the agentOS software registry by name. `version` + * selects the release manifest and defaults to `softwareRegistry.version`, + * then to this client's own release. */ -export type SoftwareEntry = SoftwarePackageRef; +export interface RegistrySoftwareRef { + registry: string; + version?: string; +} + +/** + * Install a `.aospkg` from an http(s) URL. Pass `digest` + * (`sha256:<64 lowercase hex>`) so the bytes are verified before install. + */ +export interface UrlSoftwareRef { + url: string; + digest?: string; +} + +/** + * Input type for the `software` option: a trusted local `{ packagePath }`, a + * registry package `{ registry }`, or a `{ url, digest }` artifact. Accepts an + * entry or an array of entries for meta-packages. + */ +export type SoftwareEntry = + | SoftwarePackageRef + | RegistrySoftwareRef + | UrlSoftwareRef; export type SoftwareInput = SoftwareEntry | SoftwareEntry[]; /** Host-to-VM path mapping for a software package's `/root/node_modules/` mount. */ diff --git a/packages/core/src/software-registry.ts b/packages/core/src/software-registry.ts new file mode 100644 index 0000000000..d8a3d5a50f --- /dev/null +++ b/packages/core/src/software-registry.ts @@ -0,0 +1,402 @@ +import { readFileSync } from "node:fs"; + +/** + * Registry software resolution. + * + * Each agentOS release publishes its software catalog as digest-addressed + * `.aospkg` files next to `//software/manifest.json`. A + * registry entry resolves to that release's URL and exact digest, then installs + * through the ordinary URL source, so the sidecar still verifies the bytes. + * + * The manifest is fetched by this host process, not by the guest, so VM + * network permissions do not apply to it. + */ + +/** Root that holds `/software/manifest.json` for each release. */ +export const DEFAULT_SOFTWARE_REGISTRY_URL = + "https://releases.rivet.dev/agent-os/"; +/** Environment variable that overrides the default registry root. */ +export const SOFTWARE_REGISTRY_URL_ENV = "AGENTOS_SOFTWARE_REGISTRY_URL"; + +const MANIFEST_FETCH_TIMEOUT_MS = 30_000; +const MANIFEST_MAX_BYTES = 8 * 1024 * 1024; +const MANIFEST_CACHE_LIMIT = 32; +// `latest` moves with every release; exact versions are immutable. +const LATEST_MANIFEST_TTL_MS = 5 * 60_000; + +const PACKAGE_NAME_PATTERN = /^[a-z0-9][a-z0-9-]*$/; +const VERSION_PATTERN = /^(?:latest|\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?)$/; +const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; + +/** Registry location and default release for name-based software. */ +export interface SoftwareRegistryOptions { + /** + * Root URL that holds `/software/manifest.json`. Defaults to the + * `AGENTOS_SOFTWARE_REGISTRY_URL` environment variable, then + * `https://releases.rivet.dev/agent-os/`. Point it at a mirror or a + * self-hosted copy of the release layout. + */ + url?: string; + /** + * Release to resolve entries that name no version. Defaults to the version + * of this `@rivet-dev/agentos-core` package, or `latest` for unreleased + * workspace and preview builds. + */ + version?: string; +} + +/** A registry package resolved to its exact artifact. */ +export interface ResolvedRegistrySoftware { + name: string; + /** Release whose manifest listed the package, such as `0.4.2` or `latest`. */ + version: string; + /** Absolute `.aospkg` URL. */ + url: string; + /** Exact `sha256:<64 lowercase hex>` content digest. */ + digest: string; + sizeBytes: number; + manifestUrl: string; +} + +export interface ResolveRegistrySoftwareOptions { + /** Release to read. Overrides `registry.version`. */ + version?: string; + registry?: SoftwareRegistryOptions; +} + +interface ManifestArtifact { + name: string; + digest: string; + size: number; + path: string; +} + +interface CachedManifest { + artifacts: Promise; + expiresAt: number; +} + +// Insertion-ordered so the oldest entry is evicted first. Bounded by +// MANIFEST_CACHE_LIMIT distinct registry/version pairs per process. +const manifestCache = new Map(); + +let cachedCoreVersion: string | undefined; + +/** The version of this `@rivet-dev/agentos-core` package. */ +export function coreVersion(): string { + if (cachedCoreVersion === undefined) { + // `package.json` sits one level above both `src/` and `dist/`, and npm + // always ships it, so this resolves in the workspace and when installed. + const raw = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { version?: unknown }; + if (typeof raw.version !== "string" || raw.version.length === 0) { + throw new Error( + "@rivet-dev/agentos-core package.json has no version; reinstall the package", + ); + } + cachedCoreVersion = raw.version; + } + return cachedCoreVersion; +} + +/** + * The release a registry entry resolves against when it names no version. + * + * A published client matches its own release so a given agentOS version gets + * the software built with it. The committed workspace version `0.0.1` and + * `0.0.0-*` preview builds never publish software, so they use `latest`. + */ +export function defaultRegistryVersion( + version: string = coreVersion(), +): string { + if (version === "0.0.1" || /^0\.0\.0(?:-|$)/.test(version)) { + return "latest"; + } + return version; +} + +function registryRoot(options?: SoftwareRegistryOptions): URL { + const raw = + options?.url ?? + process.env[SOFTWARE_REGISTRY_URL_ENV] ?? + DEFAULT_SOFTWARE_REGISTRY_URL; + let root: URL; + try { + root = new URL(raw.endsWith("/") ? raw : `${raw}/`); + } catch { + throw new Error( + `invalid agentOS software registry URL ${JSON.stringify(raw)}; pass an absolute http(s) URL in softwareRegistry.url or ${SOFTWARE_REGISTRY_URL_ENV}`, + ); + } + assertSecureRegistryUrl(root, "registry URL"); + return root; +} + +// The manifest supplies the digest every package is verified against, so it +// needs the same transport integrity as the packages: HTTPS, or plain HTTP only +// to a loopback mirror. +function isLoopbackHost(hostname: string): boolean { + return ( + hostname === "localhost" || + hostname === "[::1]" || + /^127(?:\.\d{1,3}){3}$/.test(hostname) + ); +} + +function assertSecureRegistryUrl(url: URL, label: string): void { + if (url.protocol === "https:") return; + if (url.protocol === "http:" && isLoopbackHost(url.hostname)) return; + throw new Error( + `agentOS software ${label} must use https, or plain http only on a loopback address, got ${url.href}`, + ); +} + +function validateName(name: unknown): string { + if (typeof name !== "string" || !PACKAGE_NAME_PATTERN.test(name)) { + throw new Error( + `invalid agentOS registry package name ${JSON.stringify(name)}; names are lowercase letters, digits, and hyphens, such as "git"`, + ); + } + return name; +} + +function validateVersion(version: unknown): string { + if (typeof version !== "string" || !VERSION_PATTERN.test(version)) { + throw new Error( + `invalid agentOS registry version ${JSON.stringify(version)}; pass a release version such as "0.4.2", or "latest"`, + ); + } + return version; +} + +async function readBoundedBody( + response: Response, + manifestUrl: string, +): Promise { + const declared = Number(response.headers.get("content-length")); + if (Number.isFinite(declared) && declared > MANIFEST_MAX_BYTES) { + throw new Error( + `agentOS software manifest ${manifestUrl} is ${declared} bytes, over the ${MANIFEST_MAX_BYTES}-byte limit`, + ); + } + if (!response.body) return ""; + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > MANIFEST_MAX_BYTES) { + await reader.cancel(); + throw new Error( + `agentOS software manifest ${manifestUrl} exceeds the ${MANIFEST_MAX_BYTES}-byte limit`, + ); + } + chunks.push(value); + } + return Buffer.concat(chunks).toString("utf8"); +} + +function parseManifest(text: string, manifestUrl: string): ManifestArtifact[] { + let raw: unknown; + try { + raw = JSON.parse(text); + } catch (error) { + throw new Error( + `agentOS software manifest ${manifestUrl} is not valid JSON: ${(error as Error).message}`, + ); + } + const record = raw as { schemaVersion?: unknown; artifacts?: unknown }; + if (record?.schemaVersion !== 1 || !Array.isArray(record.artifacts)) { + throw new Error( + `agentOS software manifest ${manifestUrl} is not a schemaVersion 1 manifest with an artifacts list`, + ); + } + return record.artifacts.map((entry, index) => { + const artifact = entry as Partial; + if ( + typeof artifact?.name !== "string" || + typeof artifact.path !== "string" || + typeof artifact.size !== "number" || + typeof artifact.digest !== "string" || + !DIGEST_PATTERN.test(artifact.digest) + ) { + throw new Error( + `agentOS software manifest ${manifestUrl} has a malformed artifact at index ${index}`, + ); + } + return { + name: artifact.name, + digest: artifact.digest, + size: artifact.size, + path: artifact.path, + }; + }); +} + +async function fetchManifest( + manifestUrl: string, + version: string, +): Promise { + let response: Response; + try { + response = await fetch(manifestUrl, { + signal: AbortSignal.timeout(MANIFEST_FETCH_TIMEOUT_MS), + }); + } catch (error) { + throw new Error( + `failed to fetch agentOS software manifest ${manifestUrl}: ${(error as Error).message}; check host network access or set softwareRegistry.url / ${SOFTWARE_REGISTRY_URL_ENV} to a reachable mirror`, + { cause: error }, + ); + } + if (response.status === 404) { + await response.body?.cancel(); + throw new Error( + `agentOS software registry has no release "${version}": GET ${manifestUrl} returned 404; pass a published agentOS version or "latest"`, + ); + } + if (!response.ok) { + await response.body?.cancel(); + throw new Error( + `failed to fetch agentOS software manifest ${manifestUrl}: HTTP ${response.status} ${response.statusText}`.trimEnd(), + ); + } + if (response.url) { + try { + assertSecureRegistryUrl(new URL(response.url), "manifest redirect"); + } catch (error) { + await response.body?.cancel(); + throw error; + } + } + return parseManifest( + await readBoundedBody(response, manifestUrl), + manifestUrl, + ); +} + +function loadManifest( + manifestUrl: string, + version: string, +): Promise { + const now = Date.now(); + const cached = manifestCache.get(manifestUrl); + if (cached && cached.expiresAt > now) return cached.artifacts; + if (cached) manifestCache.delete(manifestUrl); + + const artifacts = fetchManifest(manifestUrl, version); + const entry: CachedManifest = { + artifacts, + expiresAt: + version === "latest" + ? now + LATEST_MANIFEST_TTL_MS + : Number.POSITIVE_INFINITY, + }; + while (manifestCache.size >= MANIFEST_CACHE_LIMIT) { + const oldest = manifestCache.keys().next().value as string; + manifestCache.delete(oldest); + } + manifestCache.set(manifestUrl, entry); + // A failed fetch must not pin the error; the next call retries. + artifacts.catch(() => { + if (manifestCache.get(manifestUrl) === entry) { + manifestCache.delete(manifestUrl); + } + }); + return artifacts; +} + +function editDistance(left: string, right: string): number { + let previous = Array.from({ length: right.length + 1 }, (_, index) => index); + for (let i = 1; i <= left.length; i++) { + const current = [i]; + for (let j = 1; j <= right.length; j++) { + current[j] = Math.min( + previous[j] + 1, + current[j - 1] + 1, + previous[j - 1] + (left[i - 1] === right[j - 1] ? 0 : 1), + ); + } + previous = current; + } + return previous[right.length]; +} + +function closeMatches(name: string, names: string[]): string[] { + return names + .map((candidate) => ({ + candidate, + distance: + candidate.includes(name) || name.includes(candidate) + ? 0 + : editDistance(name, candidate), + })) + .filter(({ distance }) => distance <= 2) + .sort( + (a, b) => + a.distance - b.distance || a.candidate.localeCompare(b.candidate), + ) + .slice(0, 5) + .map(({ candidate }) => candidate); +} + +/** + * Resolve a registry package name to the exact `.aospkg` URL and digest listed + * in a release manifest. Manifests are cached per registry and version for the + * life of the process; `latest` is refreshed every five minutes. + */ +export async function resolveRegistrySoftware( + name: string, + options: ResolveRegistrySoftwareOptions = {}, +): Promise { + const packageName = validateName(name); + const version = validateVersion( + options.version ?? options.registry?.version ?? defaultRegistryVersion(), + ); + const manifestUrl = new URL( + `${version}/software/manifest.json`, + registryRoot(options.registry), + ).href; + const artifacts = await loadManifest(manifestUrl, version); + const artifact = artifacts.find( + (candidate) => candidate.name === packageName, + ); + if (!artifact) { + const suggestions = closeMatches( + packageName, + artifacts.map((candidate) => candidate.name), + ); + throw new Error( + `agentOS registry release "${version}" has no package "${packageName}" (${manifestUrl})` + + (suggestions.length > 0 + ? `; did you mean ${suggestions.map((s) => `"${s}"`).join(", ")}?` + : "; browse https://rivet.dev/agentos/docs/registry for available packages"), + ); + } + const url = new URL(artifact.path, manifestUrl); + if (url.protocol !== "https:" && url.protocol !== "http:") { + throw new Error( + `agentOS software manifest ${manifestUrl} lists ${packageName} at a non-http(s) URL: ${url.href}`, + ); + } + return { + name: packageName, + version, + url: url.href, + digest: artifact.digest, + sizeBytes: artifact.size, + manifestUrl, + }; +} + +/** @internal Test hook: forget cached manifests. */ +export function clearSoftwareRegistryCache(): void { + manifestCache.clear(); +} + +/** @internal Test hook: number of cached manifests. */ +export function softwareRegistryCacheSize(): number { + return manifestCache.size; +} diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 4ca0ea12d2..a1fdc9365e 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -27,6 +27,7 @@ export type { NativeRootFilesystemConfig, OverlayMountConfig, OverlayRootFilesystemConfig, + PackageResolverOptions, PlainMountConfig, ProcessOutput, ProcessTreeNode, @@ -107,7 +108,18 @@ export type { SnapshotLayerHandle, WritableLayerHandle, } from "./layers.js"; -export type { SoftwareInput, SoftwareRoot } from "./packages.js"; +export type { + RegistrySoftwareRef, + SoftwareEntry, + SoftwareInput, + SoftwareRoot, + UrlSoftwareRef, +} from "./packages.js"; +export type { + ResolvedRegistrySoftware, + ResolveRegistrySoftwareOptions, + SoftwareRegistryOptions, +} from "./software-registry.js"; export type { ChildProcessPermissions, ConnectTerminalOptions, diff --git a/packages/core/tests/software-install.test.ts b/packages/core/tests/software-install.test.ts index 58c74c42e2..2255611f78 100644 --- a/packages/core/tests/software-install.test.ts +++ b/packages/core/tests/software-install.test.ts @@ -114,3 +114,13 @@ describe("sidecar-owned software install", () => { ); }); }); + +describe("create-time software entries", () => { + it("rejects an entry that is not a package ref", async () => { + await expect( + AgentOs.create({ + software: [{ name: "tool" } as unknown as { packagePath: string }], + }), + ).rejects.toThrow("not a package ref"); + }); +}); diff --git a/packages/core/tests/software-registry.nightly.test.ts b/packages/core/tests/software-registry.nightly.test.ts new file mode 100644 index 0000000000..f7f7bf0a76 --- /dev/null +++ b/packages/core/tests/software-registry.nightly.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_SOFTWARE_REGISTRY_URL, + resolveRegistrySoftware, +} from "../src/software-registry.js"; + +// Reaches the published registry, so it runs only in the nightly suite. +describe("published software registry", () => { + it("resolves git from the latest release manifest", async () => { + const resolved = await resolveRegistrySoftware("git", { + version: "latest", + registry: { url: DEFAULT_SOFTWARE_REGISTRY_URL }, + }); + expect(resolved.url).toMatch( + /^https:\/\/releases\.rivet\.dev\/agent-os\/latest\/software\/packages\/git\/[0-9a-f]{64}\.aospkg$/, + ); + expect(resolved.digest).toMatch(/^sha256:[0-9a-f]{64}$/); + const head = await fetch(resolved.url, { method: "HEAD" }); + expect(head.status).toBe(200); + }, 60_000); +}); diff --git a/packages/core/tests/software-registry.test.ts b/packages/core/tests/software-registry.test.ts new file mode 100644 index 0000000000..dd44949318 --- /dev/null +++ b/packages/core/tests/software-registry.test.ts @@ -0,0 +1,509 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from "vitest"; +import { AgentOs } from "../src/agent-os.js"; +import { resolveDefaultSoftware } from "../src/default-software.js"; +import { + clearSoftwareRegistryCache, + coreVersion, + DEFAULT_SOFTWARE_REGISTRY_URL, + defaultRegistryVersion, + resolveRegistrySoftware, + SOFTWARE_REGISTRY_URL_ENV, + softwareRegistryCacheSize, +} from "../src/software-registry.js"; + +// A local registry with the release layout `/software/manifest.json`. +// Unit tests never reach the network. + +const DIGEST = `sha256:${"a".repeat(64)}`; + +function manifest(names: string[], digest = DIGEST) { + return { + schemaVersion: 1, + artifacts: names.map((name) => ({ + name, + digest, + size: 1, + path: `packages/${name}/${digest.slice("sha256:".length)}.aospkg`, + })), + }; +} + +interface Registry { + url: string; + requests: string[]; + releases: Map; + failNext: number; + packages: Map; +} + +async function startRegistry(): Promise<{ + registry: Registry; + server: Server; +}> { + const registry: Registry = { + url: "", + requests: [], + releases: new Map(), + failNext: 0, + packages: new Map(), + }; + const server = createServer((request, response) => { + const path = request.url ?? "/"; + registry.requests.push(path); + const pkg = registry.packages.get(path); + if (pkg) { + response.end(pkg); + return; + } + const match = path.match(/^\/agent-os\/([^/]+)\/software\/manifest\.json$/); + if (!match) { + response.statusCode = 404; + response.end(); + return; + } + if (registry.failNext > 0) { + registry.failNext -= 1; + response.statusCode = 503; + response.statusMessage = "Service Unavailable"; + response.end(); + return; + } + const release = registry.releases.get(match[1]); + if (release === undefined) { + response.statusCode = 404; + response.end(); + return; + } + response.setHeader("content-type", "application/json"); + response.end( + typeof release === "string" ? release : JSON.stringify(release), + ); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + registry.url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/agent-os/`; + return { registry, server }; +} + +let registry: Registry; +let server: Server; + +beforeAll(async () => { + ({ registry, server } = await startRegistry()); +}); + +afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())); +}); + +afterEach(() => { + clearSoftwareRegistryCache(); + registry.requests.length = 0; + registry.releases.clear(); + registry.packages.clear(); + registry.failNext = 0; + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); + +// The local server serves every release under `/agent-os/`. +function serve(version: string, body: unknown) { + registry.releases.set(version, body); +} + +describe("registry version default", () => { + it("uses the client's own release", () => { + expect(defaultRegistryVersion("0.4.2")).toBe("0.4.2"); + expect(defaultRegistryVersion("0.4.3-rc.1")).toBe("0.4.3-rc.1"); + }); + + it("uses latest for unreleased workspace and preview builds", () => { + expect(defaultRegistryVersion("0.0.1")).toBe("latest"); + expect(defaultRegistryVersion("0.0.0-my-branch.abc1234")).toBe("latest"); + // The committed workspace pin is 0.0.1, so this package resolves latest. + expect(coreVersion()).toBe("0.0.1"); + expect(defaultRegistryVersion()).toBe("latest"); + }); + + it("defaults to releases.rivet.dev", () => { + expect(DEFAULT_SOFTWARE_REGISTRY_URL).toBe( + "https://releases.rivet.dev/agent-os/", + ); + }); +}); + +describe("resolveRegistrySoftware", () => { + it("resolves a name to an absolute digest-pinned URL", async () => { + serve("latest", manifest(["git", "ripgrep"])); + const resolved = await resolveRegistrySoftware("git", { + registry: { url: registry.url }, + }); + expect(resolved).toEqual({ + name: "git", + version: "latest", + url: `${registry.url}latest/software/packages/git/${"a".repeat(64)}.aospkg`, + digest: DIGEST, + sizeBytes: 1, + manifestUrl: `${registry.url}latest/software/manifest.json`, + }); + }); + + it("reads an explicit version, which wins over the registry default", async () => { + serve("0.4.2", manifest(["git"])); + serve("0.4.1", manifest(["git"])); + const resolved = await resolveRegistrySoftware("git", { + version: "0.4.2", + registry: { url: registry.url, version: "0.4.1" }, + }); + expect(resolved.version).toBe("0.4.2"); + const pinned = await resolveRegistrySoftware("git", { + registry: { url: registry.url, version: "0.4.1" }, + }); + expect(pinned.version).toBe("0.4.1"); + expect(registry.requests).toEqual([ + "/agent-os/0.4.2/software/manifest.json", + "/agent-os/0.4.1/software/manifest.json", + ]); + }); + + it("honors the registry URL environment variable", async () => { + serve("latest", manifest(["git"])); + vi.stubEnv(SOFTWARE_REGISTRY_URL_ENV, registry.url.replace(/\/$/, "")); + const resolved = await resolveRegistrySoftware("git"); + expect(resolved.manifestUrl).toBe( + `${registry.url}latest/software/manifest.json`, + ); + }); + + it("caches manifests per version and refreshes latest", async () => { + serve("latest", manifest(["git", "ripgrep"])); + serve("0.4.2", manifest(["git"])); + const options = { registry: { url: registry.url } }; + await Promise.all([ + resolveRegistrySoftware("git", options), + resolveRegistrySoftware("ripgrep", options), + ]); + await resolveRegistrySoftware("git", { ...options, version: "0.4.2" }); + await resolveRegistrySoftware("git", { ...options, version: "0.4.2" }); + expect(registry.requests).toHaveLength(2); + + // `latest` moves with releases, so it expires; exact versions do not. + const now = Date.now(); + vi.spyOn(Date, "now").mockReturnValue(now + 6 * 60_000); + await resolveRegistrySoftware("git", options); + await resolveRegistrySoftware("git", { ...options, version: "0.4.2" }); + expect(registry.requests).toEqual([ + "/agent-os/latest/software/manifest.json", + "/agent-os/0.4.2/software/manifest.json", + "/agent-os/latest/software/manifest.json", + ]); + }); + + it("bounds the manifest cache", async () => { + for (let patch = 0; patch < 40; patch++) { + serve(`1.0.${patch}`, manifest(["git"])); + await resolveRegistrySoftware("git", { + version: `1.0.${patch}`, + registry: { url: registry.url }, + }); + } + expect(softwareRegistryCacheSize()).toBe(32); + }); + + it("does not cache a failed fetch", async () => { + serve("latest", manifest(["git"])); + registry.failNext = 1; + const options = { registry: { url: registry.url } }; + await expect(resolveRegistrySoftware("git", options)).rejects.toThrow( + `failed to fetch agentOS software manifest ${registry.url}latest/software/manifest.json: HTTP 503 Service Unavailable`, + ); + await expect(resolveRegistrySoftware("git", options)).resolves.toEqual( + expect.objectContaining({ name: "git" }), + ); + }); + + it("names close matches for an unknown package", async () => { + serve("latest", manifest(["git", "gitleaks", "ripgrep"])); + await expect( + resolveRegistrySoftware("gti", { registry: { url: registry.url } }), + ).rejects.toThrow( + `agentOS registry release "latest" has no package "gti" (${registry.url}latest/software/manifest.json); did you mean "git"?`, + ); + await expect( + resolveRegistrySoftware("zzzzzz", { registry: { url: registry.url } }), + ).rejects.toThrow("browse https://rivet.dev/agentos/docs/registry"); + }); + + it("reports a release that does not exist", async () => { + await expect( + resolveRegistrySoftware("git", { + version: "9.9.9", + registry: { url: registry.url }, + }), + ).rejects.toThrow( + `agentOS software registry has no release "9.9.9": GET ${registry.url}9.9.9/software/manifest.json returned 404`, + ); + }); + + it("reports an unreachable registry with its URL", async () => { + const closed = await startRegistry(); + const url = closed.registry.url; + await new Promise((resolve) => closed.server.close(() => resolve())); + await expect( + resolveRegistrySoftware("git", { registry: { url } }), + ).rejects.toThrow( + `failed to fetch agentOS software manifest ${url}latest/software/manifest.json`, + ); + }); + + it("rejects malformed manifests, names, and versions", async () => { + serve("latest", "{not json"); + serve("0.4.2", { schemaVersion: 2, artifacts: [] }); + serve("0.4.3", { + schemaVersion: 1, + artifacts: [{ name: "git", digest: "md5:x", size: 1, path: "git" }], + }); + const registryOptions = { url: registry.url }; + await expect( + resolveRegistrySoftware("git", { registry: registryOptions }), + ).rejects.toThrow("is not valid JSON"); + await expect( + resolveRegistrySoftware("git", { + version: "0.4.2", + registry: registryOptions, + }), + ).rejects.toThrow("is not a schemaVersion 1 manifest"); + await expect( + resolveRegistrySoftware("git", { + version: "0.4.3", + registry: registryOptions, + }), + ).rejects.toThrow("malformed artifact at index 0"); + await expect( + resolveRegistrySoftware("../git", { registry: registryOptions }), + ).rejects.toThrow("invalid agentOS registry package name"); + await expect( + resolveRegistrySoftware("git", { + version: "../latest", + registry: registryOptions, + }), + ).rejects.toThrow("invalid agentOS registry version"); + for (const url of ["ftp://mirror/", "http://mirror.example.com/"]) { + await expect( + resolveRegistrySoftware("git", { registry: { url } }), + ).rejects.toThrow( + "must use https, or plain http only on a loopback address", + ); + } + expect(registry.requests).toHaveLength(3); + }); +}); + +function createMockAgent() { + const metadata = { + package_id: "pkg-sha256:git", + digest: DIGEST, + size: 1n, + package_name: "git", + version: "2.0.0", + commands: ["git"], + }; + const client = { + onEvent: () => () => {}, + onClose: () => () => {}, + installPackage: vi.fn(async () => ({ + package: metadata, + projected_commands: [], + })), + }; + // Exercise the real install method without booting a native VM. + const agent = Reflect.construct(AgentOs, [ + {}, + {}, + [], + [], + {}, + {}, + client, + { connectionId: "conn", sessionId: "session" }, + { vmId: "vm" }, + ]) as AgentOs; + return { agent, client }; +} + +describe("software.install registry source", () => { + it("forwards the resolved URL and digest to the sidecar", async () => { + serve("0.4.2", manifest(["git"])); + const { agent, client } = createMockAgent(); + Object.assign(agent, { + _softwareRegistry: { url: registry.url }, + _packageResolver: { allowInsecureLocalHttp: true, maxRedirects: 2 }, + }); + const installed = await agent.software.install({ + type: "registry", + name: "git", + version: "0.4.2", + }); + expect(installed.packageName).toBe("git"); + expect(client.installPackage).toHaveBeenCalledWith( + { connectionId: "conn", sessionId: "session" }, + { vmId: "vm" }, + { + source: { + type: "url", + url: `${registry.url}0.4.2/software/packages/git/${"a".repeat(64)}.aospkg`, + expected_digest: DIGEST, + }, + max_redirects: 2, + allow_insecure_local_http: true, + }, + ); + }); + + it("does not reach the sidecar when resolution fails", async () => { + serve("latest", manifest(["git"])); + const { agent, client } = createMockAgent(); + Object.assign(agent, { _softwareRegistry: { url: registry.url } }); + await expect( + agent.software.install({ type: "registry", name: "gitt" }), + ).rejects.toThrow('did you mean "git"?'); + expect(client.installPackage).not.toHaveBeenCalled(); + }); +}); + +describe("create-time registry and URL entries", () => { + it("fails before booting a VM when a registry name is unknown", async () => { + serve("latest", manifest(["git"])); + await expect( + AgentOs.create({ + software: [{ registry: "gti" }], + softwareRegistry: { url: registry.url }, + }), + ).rejects.toThrow('did you mean "git"?'); + }); + + it("rejects malformed entries instead of dropping them", async () => { + await expect( + AgentOs.create({ + software: [ + { registry: "git", version: 1 } as unknown as { registry: string }, + ], + }), + ).rejects.toThrow("non-string version"); + await expect( + AgentOs.create({ + software: [ + { url: "https://example.com/t.aospkg", digest: 1 } as unknown as { + url: string; + }, + ], + }), + ).rejects.toThrow("non-string digest"); + await expect( + AgentOs.create({ + software: [{ name: "tool" } as unknown as { packagePath: string }], + }), + ).rejects.toThrow("not a package ref"); + }); +}); + +// These boot a real VM. The sidecar downloads the package from the local +// registry, so loopback HTTP is allowed explicitly. +// They check installation, not execution: PR CI stages only coreutils with its +// WASM binaries, so the vendored sed package projects no runnable command. +describe("registry install in a VM", () => { + const sedPath = resolveDefaultSoftware().find((ref) => + ref.packagePath.endsWith("/sed.aospkg"), + )?.packagePath; + let vm: AgentOs | undefined; + + afterEach(async () => { + await vm?.dispose(); + vm = undefined; + }); + + function serveSed(digestOverride?: string) { + if (!sedPath) throw new Error("vendored sed.aospkg is missing; build core"); + const bytes = readFileSync(sedPath); + const hex = createHash("sha256").update(bytes).digest("hex"); + serve("latest", { + schemaVersion: 1, + artifacts: [ + { + name: "sed", + digest: digestOverride ?? `sha256:${hex}`, + size: bytes.length, + path: `packages/sed/${hex}.aospkg`, + }, + ], + }); + registry.packages.set( + `/agent-os/latest/software/packages/sed/${hex}.aospkg`, + bytes, + ); + return `sha256:${hex}`; + } + + const vmOptions = () => ({ + defaultSoftware: false, + softwareRegistry: { url: registry.url }, + packageResolver: { allowInsecureLocalHttp: true }, + }); + + it("boots with a registry package installed", async () => { + const digest = serveSed(); + vm = await AgentOs.create({ + ...vmOptions(), + software: [{ registry: "sed" }], + }); + expect(vm.software.installed()).toEqual([ + expect.objectContaining({ packageName: "sed", digest }), + ]); + }, 60_000); + + it("installs a URL entry at create time", async () => { + const digest = serveSed(); + vm = await AgentOs.create({ + ...vmOptions(), + software: [ + { + url: `${registry.url}latest/software/packages/sed/${digest.slice(7)}.aospkg`, + digest, + }, + ], + }); + expect(vm.software.installed()).toEqual([ + expect.objectContaining({ packageName: "sed", digest }), + ]); + }, 60_000); + + it("installs a registry package while running", async () => { + const digest = serveSed(); + vm = await AgentOs.create(vmOptions()); + const installed = await vm.software.install({ + type: "registry", + name: "sed", + }); + expect(installed).toEqual( + expect.objectContaining({ packageName: "sed", digest }), + ); + }, 60_000); + + it("fails create when the downloaded bytes do not match the digest", async () => { + serveSed(`sha256:${"0".repeat(64)}`); + await expect( + AgentOs.create({ ...vmOptions(), software: [{ registry: "sed" }] }), + ).rejects.toThrow(/digest mismatch|expected digest/i); + }, 60_000); +}); diff --git a/secure-exec/src/runtime.ts b/secure-exec/src/runtime.ts index b8eed1bc40..103abd4c69 100644 --- a/secure-exec/src/runtime.ts +++ b/secure-exec/src/runtime.ts @@ -22,6 +22,8 @@ const VM_OPTION_KEYS = Object.keys({ environment: true, software: true, defaultSoftware: true, + softwareRegistry: true, + packageResolver: true, loopbackExemptPorts: true, allowedNodeBuiltins: true, highResolutionTime: true, diff --git a/software/README.md b/software/README.md index b787b81a64..84ca093aa5 100644 --- a/software/README.md +++ b/software/README.md @@ -51,7 +51,10 @@ software/manifest.json It uploads that directory to S3-compatible object storage. The manifest contains the package name, exact digest, byte size, and a path relative to the manifest -URL. It is a build inventory, not a semantic-version resolver. +URL. It is a build inventory, not a semantic-version resolver. Core resolves +registry names such as `{ registry: "git" }` against +`//software/manifest.json`; set +`AGENTOS_SOFTWARE_REGISTRY_URL` to test against a local copy of that layout. Run the credential-free release staging path locally with: