diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 855ff45f..22b917fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,10 @@ name: CI # Trigger model: -# pull_request → lints + release builds + e2e -# push to master → lints + release builds -# workflow_dispatch (release) → push artifacts (TODO) +# pull_request → checks + image builds; publish previews for trusted branches +# push to master / workflow_dispatch → checks + commit-tagged images on: + workflow_dispatch: pull_request: push: branches: [master] @@ -86,7 +86,34 @@ jobs: - if: runner.environment == 'github-hosted' uses: NixOS/nix-installer-action@62c1943b776c509394b550f3f983adc14e9212d6 - name: nix build .#${{ matrix.attr }} - run: nix build --accept-flake-config --print-build-logs --no-link '.#packages.x86_64-linux.${{ matrix.attr }}' + id: build + run: | + output=$(nix build --accept-flake-config --print-build-logs --no-link --print-out-paths '.#packages.x86_64-linux.${{ matrix.attr }}') + echo "output=$output" >> "$GITHUB_OUTPUT" + - name: Package portable binaries and WASM + if: startsWith(matrix.name, 'static-') || matrix.name == 'hellas-rpc-wasm' + env: + BUILD_NAME: ${{ matrix.name }} + BUILD_OUTPUT: ${{ steps.build.outputs.output }} + run: | + set -euo pipefail + if [ "$BUILD_NAME" = hellas-rpc-wasm ]; then + test -s "$BUILD_OUTPUT/lib/libhellas_rpc.rlib" + else + test -x "$BUILD_OUTPUT/bin/hellas-cli" + fi + mkdir artifacts + tar --dereference -C "$BUILD_OUTPUT" -czf "artifacts/hellas-$BUILD_NAME-$GITHUB_SHA.tar.gz" . + cd artifacts + sha256sum ./*.tar.gz > SHA256SUMS + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: startsWith(matrix.name, 'static-') || matrix.name == 'hellas-rpc-wasm' + with: + name: hellas-${{ matrix.name }} + path: artifacts/ + if-no-files-found: error + compression-level: 0 + retention-days: 14 build-smoke-passed: if: always() @@ -160,3 +187,105 @@ jobs: echo '::error::E2E validation did not complete: matrix=${{ needs.e2e-matrix.result }}, tests=${{ needs.e2e.result }}. Inspect the first failed prerequisite, including Lints and Build smoke passed.' exit 1 fi + + # Build archives with read-only permissions. No registry login, Docker daemon, + # or provider credential is needed. Forks exercise the network variants only. + images: + needs: lints + runs-on: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && 'ubuntu-latest' || 'shared' }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + backend: ${{ fromJSON(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && '["network"]' || '["network","cuda","hip"]') }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - if: runner.environment == 'github-hosted' + uses: NixOS/nix-installer-action@62c1943b776c509394b550f3f983adc14e9212d6 + - name: Build ordinary and managed images + env: + BACKEND: ${{ matrix.backend }} + run: | + set -euo pipefail + mkdir images + for variant in node cloud; do + attr=docker + flavor="$BACKEND" + if [ "$variant" = cloud ]; then + attr=docker-cloud + flavor="cloud-$BACKEND" + fi + if [ "$BACKEND" != network ]; then + attr="$attr-$BACKEND" + fi + stream=$(nix build --accept-flake-config --print-build-logs --no-link --print-out-paths ".#packages.x86_64-linux.$attr") + "$stream" | gzip -1 > "images/$flavor.tar.gz" + done + cd images + sha256sum ./*.tar.gz > SHA256SUMS + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: images-${{ matrix.backend }} + path: images/ + if-no-files-found: error + compression-level: 0 + retention-days: 3 + + # Publishing consumes image archives only, on a fresh hosted runner. The job + # never checks out or executes PR scripts with the package-write token. + publish-images: + needs: [images, build-smoke-passed, e2e-passed] + if: >- + always() && + needs.images.result == 'success' && + needs.build-smoke-passed.result == 'success' && + (github.event_name != 'pull_request' || needs.e2e-passed.result == 'success') && + (github.event_name != 'pull_request' || + (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + backend: [network, cuda, hip] + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: images-${{ matrix.backend }} + path: images + - name: Verify and load archives + working-directory: images + run: | + set -euo pipefail + sha256sum --check SHA256SUMS + for archive in ./*.tar.gz; do + docker load --input "$archive" + done + - name: Publish commit-tagged images + env: + GH_TOKEN: ${{ github.token }} + BACKEND: ${{ matrix.backend }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + repository="ghcr.io/${GITHUB_REPOSITORY,,}" + config_dir=$(mktemp -d) + export DOCKER_CONFIG="$config_dir" + trap 'rm -rf "$config_dir"' EXIT + printf '%s' "$GH_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + prefix="sha-$GITHUB_SHA" + if [ -n "$PR_NUMBER" ]; then prefix="pr-$PR_NUMBER-$GITHUB_SHA"; fi + for flavor in "$BACKEND" "cloud-$BACKEND"; do + source="ghcr.io/hellas-ai/hellas:$flavor" + target="$repository:$prefix-$flavor" + docker tag "$source" "$target" + docker push "$target" + digest=$(docker image inspect --format '{{index .RepoDigests 0}}' "$target") + [[ "$digest" == "$repository@sha256:"* ]] + printf '%s: %s\n\nPull by digest: %s\n\n' "$flavor" "$target" "$digest" >> "$GITHUB_STEP_SUMMARY" + done diff --git a/Cargo.lock b/Cargo.lock index c82d4b77..e7bbe7bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2768,6 +2768,7 @@ dependencies = [ "hellas-attestation", "hellas-chain", "hellas-client", + "hellas-cloud", "hellas-executor", "hellas-gateway", "hellas-kernel", @@ -2840,6 +2841,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "hellas-cloud" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "clap", + "hellas-private", + "hex", + "iroh", + "libc", + "reqwest", + "serde", + "serde_json", + "sha2 0.11.0", + "subtle", + "tempfile", + "tokio", +] + [[package]] name = "hellas-executor" version = "0.1.0" @@ -6078,6 +6099,8 @@ dependencies = [ "rustls", "rustls-pki-types", "rustls-platform-verifier", + "serde", + "serde_json", "sync_wrapper", "tokio", "tokio-rustls", diff --git a/Cargo.toml b/Cargo.toml index c995fc4f..bf1e2e53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/chain", "crates/cli", "crates/client", + "crates/cloud", "crates/executor", "crates/gateway", "crates/genesis", @@ -59,6 +60,7 @@ hellas-adaptors = { path = "crates/adaptors", default-features = false } hellas-attestation = { path = "crates/attestation", default-features = false } hellas-chain = { path = "crates/chain", default-features = false } hellas-client = { path = "crates/client", default-features = false } +hellas-cloud = { path = "crates/cloud", default-features = false } hellas-executor = { path = "crates/executor", default-features = false } hellas-gateway = { path = "crates/gateway", default-features = false } hellas-genesis = { path = "crates/genesis", default-features = false } diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index d5b3a88c..3dd7209e 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -10,6 +10,9 @@ documentation.workspace = true [features] default = [] +# Remote machine management; credentials stay with the operator identity. +cloud = ["dep:hellas-cloud"] + # Enables Apple App Attest verification for clients. Native enrollment and # proof production belong to the signed host application (Gate), not the CLI. apple-app-attest = [] @@ -93,6 +96,7 @@ hellas-adaptors.workspace = true hellas-attestation = { workspace = true, features = ["apple-app-attest"] } hellas-chain = { workspace = true, default-features = false, optional = true } hellas-client = { workspace = true, features = ["iroh"] } +hellas-cloud = { workspace = true, optional = true } hellas-executor = { workspace = true, default-features = false, optional = true } hellas-gateway = { workspace = true, optional = true } hellas-kernel = { workspace = true, default-features = false, optional = true } diff --git a/crates/cli/src/cloud.rs b/crates/cli/src/cloud.rs new file mode 100644 index 00000000..967af372 --- /dev/null +++ b/crates/cli/src/cloud.rs @@ -0,0 +1,44 @@ +//! Thin adapter between the main CLI identity and the machine management library. +use super::{Commands, identity}; +use anyhow::{Context, Result}; +use hellas_cloud::management::Service; +use std::path::Path; + +pub(super) async fn run(command: Commands, identity_path: Option<&Path>) -> Result<()> { + let needs_identity = match &command { + Commands::Cloud(args) => args.needs_identity() || identity_path.is_some(), + _ => true, + }; + let service = if needs_identity { + Some(Service::open( + identity::load_existing(identity_path)?.transport_key, + )?) + } else { + None + }; + match command { + Commands::Cloud(args) => args.run_owned(service.as_ref()).await, + Commands::Machines(args) => args.run(service.context("owner identity required")?).await, + Commands::Control(args) => args.run(service.context("owner identity required")?).await, + _ => unreachable!("only management commands reach this adapter"), + } +} + +pub(super) async fn machine_route( + machine: Option<&str>, + key: &iroh::SecretKey, + node_id: Option, + mut trust: super::RemoteTrustArgs, +) -> Result<(Option, super::RemoteTrustArgs)> { + let Some(machine) = machine else { + return Ok((node_id, trust)); + }; + anyhow::ensure!( + trust.assurance == hellas_rpc::Assurance::ProducerSigned, + "owned machine currently supports producer-signed assurance only" + ); + let enrollment = Service::open(key.clone())?.resolve(machine).await?; + trust.provider_genesis = + Some(super::parse_content_id_hex(&enrollment.enrollment_id).map_err(anyhow::Error::msg)?); + Ok((Some(enrollment.node_id.parse()?), trust)) +} diff --git a/crates/cli/src/cloud_tests.rs b/crates/cli/src/cloud_tests.rs new file mode 100644 index 00000000..3a8d1ce7 --- /dev/null +++ b/crates/cli/src/cloud_tests.rs @@ -0,0 +1,453 @@ +use super::*; + +#[test] +fn cloud_commands_accept_an_existing_owner_identity() { + let cli = + Cli::try_parse_from(["hellas", "cloud", "runpod", "--account", "work", "list"]).unwrap(); + assert!(validate_identity_options(&cli.command, None, false).is_ok()); + assert!(validate_identity_options(&cli.command, Some(Path::new("identity")), false).is_ok()); + assert!(validate_identity_options(&cli.command, None, true).is_err()); + let Commands::Cloud(hellas_cloud::cloud::CloudArgs { + provider: hellas_cloud::cloud::CloudCommand::Runpod(args), + .. + }) = cli.command + else { + panic!("expected cloud runpod"); + }; + assert_eq!(args.account.as_deref(), Some("work")); + assert!(matches!( + args.command, + hellas_cloud::cloud::RunpodCommand::List + )); +} + +#[test] +fn management_commands_require_an_existing_identity_and_do_not_generate_one() { + let temp = tempfile::tempdir().unwrap(); + let missing = temp.path().join("identity"); + for args in [ + vec!["hellas", "machines", "list"], + vec!["hellas", "control", "serve"], + ] { + let cli = Cli::try_parse_from(args).unwrap(); + assert!(validate_identity_options(&cli.command, Some(&missing), false).is_ok()); + assert!(validate_identity_options(&cli.command, Some(&missing), true).is_err()); + } + assert!(identity::load_existing(Some(&missing)).is_err()); + assert!(!missing.exists()); +} + +#[test] +fn fetch_machine_uses_an_existing_identity_and_rejects_conflicting_pins() { + let args = [ + "hellas", + "fetch", + "--machine", + "metal", + "--service", + "http", + "--method", + "request", + "--execution-environment", + TEST_PROVIDER, + "--payload", + "{}", + ]; + assert!(Cli::try_parse_from(args.into_iter().chain(["--node-addr", "127.0.0.1:3000"])).is_ok()); + let cli = Cli::try_parse_from(args).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("identity"); + assert!(load_command_identity(&cli.command, Some(&missing)).is_err()); + assert!(!missing.exists()); + assert!(validate_identity_options(&cli.command, None, true).is_err()); + for extra in [vec!["--provider", TEST_PROVIDER], vec![TEST_PROVIDER]] { + assert!(Cli::try_parse_from(args.into_iter().chain(extra)).is_err()); + } +} + +#[cfg(feature = "gateway")] +#[test] +fn gateway_machine_accepts_the_fetch_backend_without_a_model() { + let cli = Cli::try_parse_from([ + "hellas", + "gateway", + "--machine", + "metal", + "--responses-backend", + "fetch", + "--responses-fetch-execution-environment", + "openai-responses", + ]) + .unwrap(); + assert_eq!(cli.command.owned_machine(), Some("metal")); +} + +#[cfg(feature = "gateway")] +#[test] +fn gateway_can_select_owned_machines_without_conflicting_route_pins() { + let cli = parse_gateway(&["--machine", "gpu"]).unwrap(); + assert!(validate_identity_options(&cli.command, None, true).is_err()); + let temp = tempfile::tempdir().unwrap(); + let missing = temp.path().join("identity"); + assert!(load_command_identity(&cli.command, Some(&missing)).is_err()); + assert!(!missing.exists()); + assert!(parse_gateway(&["--machine", "gpu", "--provider", TEST_PROVIDER]).is_err()); + assert!(parse_gateway(&["--machine", "gpu", "--node-id", TEST_PROVIDER]).is_err()); + #[cfg(feature = "node")] + assert!(parse_gateway(&["--machine", "gpu", "--paid-work-config", "work.json"]).is_err()); +} + +#[cfg(feature = "gateway")] +#[test] +fn http_gateway_requires_a_paid_pool_instead_of_an_owned_machine() { + let error = Cli::try_parse_from([ + "hellas", + "gateway", + "--machine", + "metal", + "--http-fetch-config", + "gateway.json", + ]) + .err() + .expect("HTTP gateway cannot select a machine without a paid pool"); + assert_eq!(error.kind(), clap::error::ErrorKind::ArgumentConflict); +} + +/// Actual local companion + Hellas server, with two real stored Hellas identities. +#[cfg(feature = "node")] +#[tokio::test] +#[ignore = "set HELLAS_CLI and HELLAS_AGENT to freshly built executables"] +async fn owner_controls_admin_and_hellas_rpc_even_when_receipt_is_stolen() { + use hellas_cloud::{ + config::Credentials, + management::{Request, Service}, + wire::{self, Operation, Response}, + }; + use std::time::Duration; + let cli = PathBuf::from(std::env::var("HELLAS_CLI").unwrap()); + let agent = PathBuf::from(std::env::var("HELLAS_AGENT").unwrap()); + let dir = tempfile::tempdir().unwrap(); + let owner_path = dir.path().join("owner"); + let owner = identity::load_or_create(Some(&owner_path)).unwrap(); + let stranger_path = dir.path().join("stranger"); + let stranger = identity::load_or_create(Some(&stranger_path)).unwrap(); + let service = Service::new(owner.transport_key.clone(), &dir.path().join("inventory")).unwrap(); + let admin_socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + let admin_addr = admin_socket.local_addr().unwrap(); + drop(admin_socket); + let node_socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + let node_addr = node_socket.local_addr().unwrap(); + drop(node_socket); + let bootstrap = dir.path().join("bootstrap.json"); + service + .execute(Request::Prepare { + name: "metal".into(), + bootstrap_file: bootstrap.clone(), + admin_addr: Some(admin_addr), + serve_args: vec![ + "--port".into(), + node_addr.port().to_string(), + "--execute-policy".into(), + "none".into(), + "--output-cache".into(), + "record".into(), + "--store-dir".into(), + dir.path().join("store").to_string_lossy().into_owned(), + ], + }) + .await + .unwrap(); + let mut child = tokio::process::Command::new(&agent) + .args([ + "--no-relay", + "--bind", + &admin_addr.to_string(), + "--bootstrap", + ]) + .arg(&bootstrap) + .arg("--cli") + .arg(&cli) + .arg("--data") + .arg(dir.path().join("worker")) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::from( + std::fs::File::create(dir.path().join("agent.log")).unwrap(), + )) + .kill_on_drop(true) + .spawn() + .unwrap(); + tokio::time::sleep(Duration::from_secs(2)).await; + assert!( + child.try_wait().unwrap().is_none(), + "{}", + std::fs::read_to_string(dir.path().join("agent.log")).unwrap() + ); + let enrollment = service.resolve("metal").await.unwrap(); + assert_ne!(enrollment.node_id, owner.transport_key.public().to_string()); + let env: serde_json::Value = hellas_cloud::config::read_json(&bootstrap).unwrap(); + // Simulate theft of the entire receipt: valid token and server key, wrong caller key. + let stolen = Credentials { + admin_secret: env["HELLAS_REMOTE_KEY"].as_str().unwrap().into(), + token: env["HELLAS_REMOTE_TOKEN"].as_str().unwrap().into(), + owner: None, + }; + let denied = wire::call_as( + &stolen, + Some(admin_addr), + Operation::Restart, + Some(&stranger.transport_key), + ) + .await + .unwrap(); + assert!(matches!(denied, Response::Error { .. })); + let configuration = hellas_cloud::configuration::Configuration { + fetch_config: serde_json::json!({ + "routes": [{"service": "openai", "method": "responses", + "destination": {"type": "openai-responses", "api_key_env": "TEST_UPSTREAM_KEY"}}, + {"service": "codex", "method": "responses", + "destination": {"type": "codex-responses", "auth_path": "@files/auth.json"}}], + "callers": [] + }), + env: [( + "TEST_UPSTREAM_KEY".into(), + "integration-fixture-value".into(), + )] + .into(), + files: [( + "auth.json".into(), + serde_json::json!({ + "version": 1, "tokens": {"access_token": "integration-file-value", + "refresh_token": "integration-refresh-value", "account_id": "test-account"}, + "last_refresh": null, "refresh_token_blocked": null + }), + )] + .into(), + }; + let denied = wire::call_as( + &stolen, + Some(admin_addr), + Operation::Configure { + configuration: configuration.clone(), + }, + Some(&stranger.transport_key), + ) + .await + .unwrap(); + assert!(matches!(denied, Response::Error { .. })); + let configuration_path = dir.path().join("worker/configuration.json"); + assert!(!configuration_path.exists()); + let configured = service + .execute(Request::Configure { + name: "metal".into(), + configuration: configuration.clone(), + }) + .await + .unwrap(); + assert_eq!(configured["running"], true); + assert_eq!(service.resolve("metal").await.unwrap(), enrollment); + assert!( + !service + .list() + .unwrap() + .to_string() + .contains("integration-fixture-value") + ); + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(&configuration_path) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + let saved = std::fs::read(&configuration_path).unwrap(); + let installed: serde_json::Value = serde_json::from_slice(&saved).unwrap(); + let private_fetch = PathBuf::from(installed["fetch_config"].as_str().unwrap()); + let private_file = private_fetch.parent().unwrap().join("files/auth.json"); + assert_eq!( + std::fs::metadata(&private_file) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + let mut refreshed: serde_json::Value = + serde_json::from_slice(&std::fs::read(&private_file).unwrap()).unwrap(); + refreshed["tokens"]["access_token"] = serde_json::json!("rotated-fixture-value"); + std::fs::write(&private_file, serde_json::to_vec(&refreshed).unwrap()).unwrap(); + let mut invalid = configuration; + invalid.fetch_config["unknown_field"] = serde_json::json!(true); + assert!( + service + .execute(Request::Configure { + name: "metal".into(), + configuration: invalid, + }) + .await + .is_err() + ); + assert_eq!(std::fs::read(&configuration_path).unwrap(), saved); + assert_eq!(service.resolve("metal").await.unwrap(), enrollment); + // The legitimate owner can restart without changing the worker's enrollment. + service + .execute(Request::Restart { + name: "metal".into(), + }) + .await + .unwrap(); + assert_eq!(service.resolve("metal").await.unwrap(), enrollment); + tokio::time::sleep(Duration::from_secs(1)).await; + let after_restart: serde_json::Value = + serde_json::from_slice(&std::fs::read(&private_file).unwrap()).unwrap(); + assert_eq!( + after_restart, refreshed, + "restart must preserve refreshed provider credentials" + ); + for (caller, allowed) in [(&owner_path, true), (&stranger_path, false)] { + let address = node_addr.to_string(); + for args in [ + vec!["rpc", &enrollment.node_id, "--node-addr", &address], + vec![ + "output-cache", + "--node-id", + &enrollment.node_id, + "--node-addr", + &address, + "stats", + ], + ] { + let output = tokio::time::timeout( + Duration::from_secs(30), + tokio::process::Command::new(&cli) + .arg("--identity") + .arg(caller) + .args(&args) + .kill_on_drop(true) + .output(), + ) + .await + .unwrap() + .unwrap(); + assert_eq!( + output.status.success(), + allowed, + "{args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + } + // A bare-metal inventory entry selects the same authenticated Fetch route. + // No caller grant is installed, so this must reach the remote policy denial + // without making an upstream API call. + let output = tokio::time::timeout(Duration::from_secs(30), + tokio::process::Command::new(&cli) + .env("HELLAS_MACHINES_DIR", dir.path().join("inventory")) + .arg("--identity").arg(&owner_path) + .args(["fetch", "--machine", "metal", "--node-addr", &node_addr.to_string(), + "--service", "openai", "--method", "responses", "--execution-environment", "openai-responses", + "--payload", r#"{"model":"fixture","input":"hello","stream":true,"store":false,"max_output_tokens":1}"#]) + .kill_on_drop(true).output() + ).await.unwrap().unwrap(); + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr).contains("fetch caller key is not authorized"), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + + #[cfg(feature = "gateway")] + { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let bearer = dir.path().join("gateway.bearer"); + let mut gateway = tokio::process::Command::new(&cli) + .env("HELLAS_MACHINES_DIR", dir.path().join("inventory")) + .arg("--identity") + .arg(&owner_path) + .args([ + "gateway", + "--machine", + "metal", + "--node-addr", + &node_addr.to_string(), + "--responses-backend", + "fetch", + "--responses-fetch-route-service", + "openai", + "--responses-fetch-execution-environment", + "openai-responses", + "--port", + &port.to_string(), + "--bearer-token-file", + ]) + .arg(&bearer) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::from( + std::fs::File::create(dir.path().join("gateway.log")).unwrap(), + )) + .kill_on_drop(true) + .spawn() + .unwrap(); + let client = reqwest::Client::new(); + let mut response = None; + for _ in 0..100 { + assert!( + gateway.try_wait().unwrap().is_none(), + "{}", + std::fs::read_to_string(dir.path().join("gateway.log")).unwrap() + ); + if let Ok(token) = std::fs::read_to_string(&bearer) + && let Ok(result) = client + .post(format!("http://127.0.0.1:{port}/v1/responses")) + .bearer_auth(token.trim()) + .json( + &serde_json::json!({"model":"fixture", "input":"hello", "stream":false, + "store":false, "max_output_tokens":1}), + ) + .timeout(Duration::from_secs(10)) + .send() + .await + { + response = Some(result); + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + let response = response.expect("fetch gateway became ready"); + assert!(!response.status().is_success()); + let body = response.text().await.unwrap(); + assert!( + body.contains("fetch caller key is not authorized"), + "{body}" + ); + gateway.kill().await.unwrap(); + gateway.wait().await.unwrap(); + } + unsafe { + libc::kill(child.id().unwrap() as i32, libc::SIGTERM); + } + tokio::time::timeout(Duration::from_secs(15), child.wait()) + .await + .unwrap() + .unwrap(); + // An existing worker volume cannot silently change owners on restart. + let mut changed = env; + changed["HELLAS_REMOTE_OWNER"] = serde_json::json!(stranger.transport_key.public().to_string()); + let changed_path = dir.path().join("changed.json"); + std::fs::write(&changed_path, serde_json::to_vec(&changed).unwrap()).unwrap(); + let output = tokio::process::Command::new(&agent) + .args(["--bootstrap"]) + .arg(changed_path) + .arg("--cli") + .arg(cli) + .arg("--data") + .arg(dir.path().join("worker")) + .output() + .await + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("machine owner changed")); +} diff --git a/crates/cli/src/commands/serve/mod.rs b/crates/cli/src/commands/serve/mod.rs index b42787f4..9ba0f9df 100644 --- a/crates/cli/src/commands/serve/mod.rs +++ b/crates/cli/src/commands/serve/mod.rs @@ -34,6 +34,7 @@ pub use work_config::{WorkConfig, load_work_config}; pub struct ServeOptions { pub admin_peers: Vec, pub output_cache: hellas_rpc::cache::CacheOptions, + pub owner: Option, pub port: Option, pub execute_policy: ExecutePolicy, pub queue_size: usize, @@ -199,6 +200,7 @@ async fn run_with_store( let node = node::spawn_node(node::NodeConfig { admin_peers: options.admin_peers, output_cache: options.output_cache, + owner: options.owner, port: options.port, execute_policy: options.execute_policy.clone(), queue_size: options.queue_size, @@ -277,6 +279,10 @@ async fn wait_for_shutdown_signal() -> std::io::Result<()> { /// credentials, capabilities) and the caller access policy, cross-validated so /// a caller grant naming an undefined route is a load error rather than a /// silent dead entry. +pub(crate) fn validate_fetch_config(path: &std::path::Path) -> CliResult<()> { + load_fetch_config(path).map(|_| ()) +} + fn load_fetch_config(path: &std::path::Path) -> CliResult<(FetchRouteRegistry, FetchAccessPolicy)> { let bytes = fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; let file: FetchConfigFile = serde_json::from_slice(&bytes) diff --git a/crates/cli/src/commands/serve/node.rs b/crates/cli/src/commands/serve/node.rs index a2b38ab6..fa2efbf7 100644 --- a/crates/cli/src/commands/serve/node.rs +++ b/crates/cli/src/commands/serve/node.rs @@ -145,6 +145,7 @@ impl NodeHandle { pub(super) struct NodeConfig { pub(super) admin_peers: Vec, pub(super) output_cache: hellas_rpc::cache::CacheOptions, + pub(super) owner: Option, pub(super) port: Option, pub(super) execute_policy: ExecutePolicy, pub(super) queue_size: usize, @@ -190,6 +191,7 @@ pub(super) async fn spawn_node(config: NodeConfig) -> anyhow::Result peers: config .admin_peers .iter() + .chain(config.owner.iter()) .map(|peer| hellas_wire::PeerIdentity(*peer.as_bytes())) .collect(), }; @@ -308,6 +310,7 @@ pub(super) async fn spawn_node(config: NodeConfig) -> anyhow::Result // -- Accept loop: one task per inbound Connection; per-Connection // dispatch routed by ALPN to the matching service handler. + let owner = config.owner; let accept_endpoint = endpoint.clone(); let accept_task = tokio::spawn(async move { let connection_slots = Arc::new(Semaphore::new(MAX_ACTIVE_RPC_CONNECTIONS)); @@ -348,6 +351,10 @@ pub(super) async fn spawn_node(config: NodeConfig) -> anyhow::Result return; } }; + if owner.is_some_and(|owner| owner != conn.remote_id()) { + conn.close(0u32.into(), b"unauthorized"); + return; + } let alpn = conn.alpn().to_vec(); debug!( alpn = %String::from_utf8_lossy(&alpn), diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index e88cdd1c..e8e37122 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -10,6 +10,8 @@ use std::path::{Path, PathBuf}; #[cfg(feature = "evaluate")] use std::time::Duration; +#[cfg(all(feature = "cloud", unix))] +mod cloud; mod commands; mod identity; #[cfg(feature = "node")] @@ -78,7 +80,12 @@ fn load_command_identity( .. } ); - let read_only = settles_paid_work + #[cfg(all(feature = "cloud", unix))] + let owned_machine = command.owned_machine().is_some(); + #[cfg(not(all(feature = "cloud", unix)))] + let owned_machine = false; + let read_only = owned_machine + || settles_paid_work || matches!( command, Commands::Identity { @@ -313,6 +320,15 @@ enum CodexAuthCommand { #[allow(clippy::large_enum_variant)] #[derive(Subcommand)] enum Commands { + #[cfg(all(feature = "cloud", unix))] + /// Provision and inspect remote Hellas workers. + Cloud(hellas_cloud::cloud::CloudArgs), + #[cfg(all(feature = "cloud", unix))] + /// Discover and administer machines owned by the selected Hellas identity. + Machines(hellas_cloud::machines::MachinesArgs), + #[cfg(all(feature = "cloud", unix))] + /// Internal management RPC for local applications. + Control(hellas_cloud::machines::ControlArgs), #[cfg(feature = "node")] /// Run the RPC server Serve { @@ -320,6 +336,9 @@ enum Commands { /// all cached transcripts and clearing them. Omitted means no remote admin. #[arg(long = "admin-peer")] admin_peers: Vec, + /// Restrict every inbound Hellas RPC connection to this owner identity. + #[arg(long)] + owner: Option, /// Assurance offered by this provider. #[arg(long, default_value = "producer-signed", value_parser = parse_assurance)] assurance: hellas_rpc::Assurance, @@ -407,6 +426,9 @@ enum Commands { /// means this node serves no Fetch routes. #[arg(long = "fetch-config")] fetch_config_file: Option, + /// Validate fetch routes and credentials without starting a node. + #[arg(long, requires = "fetch_config_file")] + check_config: bool, /// Maximum number of Fetch provider streams running at once. #[arg( long = "fetch-max-in-flight", @@ -468,9 +490,13 @@ enum Commands { .requires("tokenizer")), mut_arg("tokenizer", |arg| arg.required(false).requires("environment")) )] + #[cfg_attr(all(feature = "cloud", unix), command(group( + clap::ArgGroup::new("remote_target").args(["node_id", "machine"]) + )))] Gateway { /// Serve exact HTTP routes through paid HTTPS Fetch. #[arg(long, value_name = "FILE", conflicts_with_all = ["responses_backend", "environment"])] + #[cfg_attr(all(feature = "cloud", unix), arg(conflicts_with = "machine"))] #[cfg_attr(feature = "node", arg(requires = "paid_work_config"))] http_fetch_config: Option, /// Request/response archive directory (default: ~/.hellas/gateway-archive). @@ -492,6 +518,12 @@ enum Commands { /// Explicit local text-chat template. #[arg(long = "chat-template", value_name = "TEMPLATE")] chat_template: Option, + /// Select an owned machine from this identity's inventory. + #[cfg(all(feature = "cloud", unix))] + #[arg(long, conflicts_with_all = ["node_id", "provider_genesis", "apple_app_attest_app_id", "apple_app_attest_cdhashes"])] + #[cfg_attr(feature = "evaluate", arg(conflicts_with = "local"))] + #[cfg_attr(feature = "node", arg(conflicts_with = "paid_work_config"))] + machine: Option, #[command(flatten)] remote_trust: RemoteTrustArgs, #[command(flatten)] @@ -506,7 +538,9 @@ enum Commands { #[arg(long)] node_id: Option, /// Direct UDP address hint for the target node. Repeat or use commas. - #[arg(long = "node-addr", value_delimiter = ',', requires = "node_id")] + #[arg(long = "node-addr", value_delimiter = ',')] + #[cfg_attr(all(feature = "cloud", unix), arg(requires = "remote_target"))] + #[cfg_attr(not(all(feature = "cloud", unix)), arg(requires = "node_id"))] node_addrs: Vec, /// Run locally with Catena instead of the Hellas network #[cfg(feature = "evaluate")] @@ -648,7 +682,15 @@ enum Commands { .multiple(true) )) )] + #[cfg_attr(all(feature = "cloud", unix), command(group( + clap::ArgGroup::new("remote_target").args(["node_id", "machine"]) + )))] Llm { + /// Select any owned cloud or bare-metal machine from this identity's inventory. + #[cfg(all(feature = "cloud", unix))] + #[arg(long, conflicts_with_all = ["node_id", "provider_genesis", "apple_app_attest_app_id", "apple_app_attest_cdhashes"])] + #[cfg_attr(feature = "evaluate", arg(conflicts_with_all = ["local", "verify_local"]))] + machine: Option, #[command(flatten)] remote_trust: RemoteTrustArgs, #[command(flatten)] @@ -656,7 +698,9 @@ enum Commands { /// Node ID to run on remotely (omit to auto-discover) node_id: Option, /// Direct UDP address hint for the target node. Repeat or use commas. - #[arg(long = "node-addr", value_delimiter = ',', requires = "node_id")] + #[arg(long = "node-addr", value_delimiter = ',')] + #[cfg_attr(all(feature = "cloud", unix), arg(requires = "remote_target"))] + #[cfg_attr(not(all(feature = "cloud", unix)), arg(requires = "node_id"))] node_addrs: Vec, /// Prompt to send (required) #[arg(short = 'p', long = "prompt")] @@ -698,13 +742,22 @@ enum Commands { /// The selected Fetch contract strictly structures the upstream request /// and destructures its adversarial response into signed output. A /// platform-backed assurance authenticates the app; producer-signed does not. + #[cfg_attr(all(feature = "cloud", unix), command(group( + clap::ArgGroup::new("remote_target").args(["node_id", "machine"]) + )))] Fetch { + /// Select any owned cloud or bare-metal machine from this identity's inventory. + #[cfg(all(feature = "cloud", unix))] + #[arg(long, conflicts_with_all = ["node_id", "provider_genesis", "apple_app_attest_app_id", "apple_app_attest_cdhashes"])] + machine: Option, #[command(flatten)] remote_trust: RemoteTrustArgs, /// Node ID to run on remotely (omit to auto-discover) node_id: Option, /// Direct UDP address hint for the target node. Repeat or use commas. - #[arg(long = "node-addr", value_delimiter = ',', requires = "node_id")] + #[arg(long = "node-addr", value_delimiter = ',')] + #[cfg_attr(all(feature = "cloud", unix), arg(requires = "remote_target"))] + #[cfg_attr(not(all(feature = "cloud", unix)), arg(requires = "node_id"))] node_addrs: Vec, /// Fetch service label. The protocol records it but does not interpret it. #[arg(long)] @@ -799,6 +852,20 @@ enum Commands { }, } +#[cfg(all(feature = "cloud", unix))] +impl Commands { + fn owned_machine(&self) -> Option<&str> { + match self { + Self::Fetch { machine, .. } => machine.as_deref(), + #[cfg(feature = "llm")] + Self::Llm { machine, .. } => machine.as_deref(), + #[cfg(feature = "gateway")] + Self::Gateway { machine, .. } => machine.as_deref(), + _ => None, + } + } +} + fn validate_identity_options( command: &Commands, identity: Option<&Path>, @@ -808,6 +875,10 @@ fn validate_identity_options( Commands::OutputCache(args) if args.node_id.is_none() => Some("output-cache"), Commands::Store { .. } => Some("store"), Commands::Environment { .. } => Some("environment"), + #[cfg(feature = "node")] + Commands::Serve { + check_config: true, .. + } => Some("serve --check-config"), #[cfg(feature = "chain")] Commands::Chain { .. } => Some("chain"), Commands::CodexAuth { .. } => Some("codex-auth"), @@ -829,6 +900,10 @@ fn validate_identity_options( let reads_existing_identity = match command { Commands::OutputCache(args) => args.node_id.is_some(), + #[cfg(all(feature = "cloud", unix))] + command if command.owned_machine().is_some() => true, + #[cfg(all(feature = "cloud", unix))] + Commands::Cloud(_) | Commands::Machines(_) | Commands::Control(_) => true, Commands::Identity { command: IdentityCommand::ShowNodeId | IdentityCommand::ShowEnrollmentId, } @@ -913,6 +988,22 @@ async fn async_main() { std::process::exit(1); } let tracer_provider = tracing_config::init_tracing(cli.log_file.as_deref()); + #[cfg(feature = "node")] + if let Commands::Serve { + check_config: true, + fetch_config_file: Some(path), + .. + } = &cli.command + { + let result = commands::serve::validate_fetch_config(path); + tracer_provider.shutdown(); + if let Err(error) = result { + eprintln!("error: {error:#}"); + std::process::exit(1); + } + return; + } + if let Commands::ProducerKey { command: ProducerKeyCommand::Show, } = &cli.command @@ -965,6 +1056,16 @@ async fn async_main() { } return; } + #[cfg(all(feature = "cloud", unix))] + command @ (Commands::Cloud(_) | Commands::Machines(_) | Commands::Control(_)) => { + let result = cloud::run(command, cli.identity.as_deref()).await; + tracer_provider.shutdown(); + if let Err(err) = result { + eprintln!("error: {err:#}"); + std::process::exit(1); + } + return; + } Commands::Store { command } => { let result = commands::store::run(command, cli.store_dir).await; tracer_provider.shutdown(); @@ -1025,6 +1126,7 @@ async fn async_main() { let result = match command { #[cfg(feature = "node")] Commands::Serve { + owner, assurance, admin_peers, port, @@ -1057,6 +1159,7 @@ async fn async_main() { metrics_port, graffiti, fetch_config_file, + check_config: _, fetch_max_in_flight, fetch_queue_size, fetch_retained_transcript_capacity, @@ -1100,6 +1203,7 @@ async fn async_main() { commands::serve::run(commands::serve::ServeOptions { admin_peers, output_cache: cache_options, + owner, port, execute_policy, queue_size, @@ -1179,6 +1283,8 @@ async fn async_main() { bearer_token_file, allow_remote, chat_template, + #[cfg(all(feature = "cloud", unix))] + machine, remote_trust, causal_lm, host, @@ -1272,6 +1378,19 @@ async fn async_main() { }; #[cfg(not(feature = "evaluate"))] let () = local_content_store; + #[cfg(all(feature = "cloud", unix))] + anyhow::ensure!( + machine.is_none() || responses_backend != GatewayResponsesBackend::Proxy, + "--machine cannot be used with the external proxy backend" + ); + #[cfg(all(feature = "cloud", unix))] + let (node_id, remote_trust) = cloud::machine_route( + machine.as_deref(), + &secret_key, + node_id, + remote_trust, + ) + .await?; let assurance = remote_trust.assurance; #[cfg(not(feature = "evaluate"))] let local = false; @@ -1400,11 +1519,17 @@ async fn async_main() { #[cfg(feature = "chain")] Commands::Chain { .. } => unreachable!("chain commands handled before identity load"), Commands::Store { .. } => unreachable!("store commands handled before identity load"), + #[cfg(all(feature = "cloud", unix))] + Commands::Cloud(_) | Commands::Machines(_) | Commands::Control(_) => { + unreachable!("management commands handled before identity load") + } Commands::Environment { .. } => { unreachable!("environment commands handled before identity load") } #[cfg(feature = "llm")] Commands::Llm { + #[cfg(all(feature = "cloud", unix))] + machine, remote_trust, causal_lm, node_id, @@ -1454,6 +1579,10 @@ async fn async_main() { }), )? }; + #[cfg(all(feature = "cloud", unix))] + let (node_id, remote_trust) = cloud::machine_route( + machine.as_deref(), &secret_key, node_id, remote_trust, + ).await?; commands::llm::run( commands::llm::ExecuteOptions { output_cache: commands::output_cache::options( @@ -1491,6 +1620,8 @@ async fn async_main() { .await } Commands::Fetch { + #[cfg(all(feature = "cloud", unix))] + machine, remote_trust, node_id, node_addrs, @@ -1511,6 +1642,10 @@ async fn async_main() { match payload { Ok(payload) => { async { + #[cfg(all(feature = "cloud", unix))] + let (node_id, remote_trust) = cloud::machine_route( + machine.as_deref(), &secret_key, node_id, remote_trust, + ).await?; commands::fetch::run( commands::fetch::ExecuteOptions { output_cache: commands::output_cache::options( diff --git a/crates/cli/src/tests.rs b/crates/cli/src/tests.rs index 36075d1e..6b22cc8f 100644 --- a/crates/cli/src/tests.rs +++ b/crates/cli/src/tests.rs @@ -1,5 +1,9 @@ use super::*; +#[cfg(all(feature = "cloud", unix))] +#[path = "cloud_tests.rs"] +mod cloud; + #[cfg(feature = "llm")] const TEST_ENVIRONMENT: &str = "/path/to/model.environment"; #[cfg(feature = "llm")] @@ -887,7 +891,7 @@ fn fetch_retention_defaults_off_and_can_be_enabled() { } #[test] -fn fetch_rejects_node_addr_without_node_id() { +fn fetch_rejects_node_addr_without_a_target() { let result = Cli::try_parse_from([ "hellas", "fetch", @@ -907,7 +911,7 @@ fn fetch_rejects_node_addr_without_node_id() { .err() .expect("node address must be rejected") .to_string(); - assert!(error.contains(""), "{error}"); + assert!(error.contains("NODE_ID"), "{error}"); } #[test] diff --git a/crates/cloud/Cargo.toml b/crates/cloud/Cargo.toml new file mode 100644 index 00000000..62067b0f --- /dev/null +++ b/crates/cloud/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "hellas-cloud" +description = "Identity-bound remote machine management for Hellas" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +documentation.workspace = true + +[[bin]] +name = "hellas-agent" +path = "src/bin/agent.rs" + +[dependencies] +anyhow.workspace = true +async-trait = "0.1" +clap = { version = "4", features = ["derive"] } +hellas-private.workspace = true +hex.workspace = true +iroh = { workspace = true, features = ["tls-ring"] } +libc = "0.2" +reqwest = { workspace = true, features = ["json", "stream"] } +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +subtle = "2" +tempfile.workspace = true +tokio.workspace = true diff --git a/crates/cloud/src/accounts.rs b/crates/cloud/src/accounts.rs new file mode 100644 index 00000000..d53342d5 --- /dev/null +++ b/crates/cloud/src/accounts.rs @@ -0,0 +1,168 @@ +use std::{collections::BTreeMap, path::PathBuf, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use serde::Deserialize; + +/// Public signup attribution for the account owning the Foundation templates. +pub const RUNPOD_REFERRAL_URL: &str = "https://runpod.io?ref=u887dgii"; + +#[derive(Debug)] +pub(crate) struct RunpodAccountRequired; + +impl std::fmt::Display for RunpodAccountRequired { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Runpod account is not configured. Set RUNPOD_API_KEY or configure a named account.\n\ + Use Hellas' referral link to support the foundation: {RUNPOD_REFERRAL_URL}" + ) + } +} + +#[derive(Debug)] +pub(crate) struct MissingCredential; + +impl std::fmt::Display for MissingCredential { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("credential environment variable is unset or empty") + } +} + +impl std::error::Error for MissingCredential {} + +/// Profiles contain references to credentials, never API keys. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Accounts { + pub runpod: BTreeMap, +} + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub enum CredentialSource { + Env(String), + Command(Vec), +} + +impl CredentialSource { + pub async fn token(&self) -> Result { + let token = match self { + Self::Env(name) => { + let token = std::env::var(name) + .map_err(|error| match error { + std::env::VarError::NotPresent => anyhow::Error::new(MissingCredential), + error => error.into(), + }) + .with_context(|| format!("set {name}"))?; + ensure!(!token.trim().is_empty(), MissingCredential); + token + } + Self::Command(args) => { + let (program, args) = args.split_first().context("empty credential command")?; + let output = tokio::time::timeout( + Duration::from_secs(30), + tokio::process::Command::new(program) + .args(args) + .stdin(std::process::Stdio::null()) + .kill_on_drop(true) + .output(), + ) + .await + .context("credential command timed out")? + .context("could not start credential command")?; + ensure!( + output.status.success(), + "credential command failed; output withheld" + ); + ensure!( + output.stdout.len() <= 65536, + "credential command output too large" + ); + String::from_utf8(output.stdout) + .context("credential command returned invalid UTF-8")? + .lines() + .next() + .unwrap_or_default() + .to_owned() + } + }; + let token = token.trim().to_owned(); + ensure!( + !token.is_empty() && token.len() <= 4096 && token.bytes().all(|b| b.is_ascii_graphic()), + "credential must be a nonempty API token" + ); + Ok(token) + } +} + +pub fn runpod(account: Option<&str>) -> Result { + let Some(account) = account else { + return Ok(CredentialSource::Env("RUNPOD_API_KEY".into())); + }; + validate_name(account)?; + let path = match std::env::var_os("HELLAS_CLOUD_ACCOUNTS") { + Some(path) => PathBuf::from(path), + None => { + let root = match std::env::var_os("XDG_CONFIG_HOME") { + Some(root) => PathBuf::from(root), + None => PathBuf::from( + std::env::var_os("HOME").context("set HOME or HELLAS_CLOUD_ACCOUNTS")?, + ) + .join(".config"), + }; + root.join("hellas/cloud-accounts.json") + } + }; + let bytes = std::fs::read(&path).map_err(|error| { + let missing = error.kind() == std::io::ErrorKind::NotFound; + let error = anyhow::Error::new(error).context(format!("read {}", path.display())); + if missing { + error.context(RunpodAccountRequired) + } else { + error + } + })?; + let accounts: Accounts = serde_json::from_slice(&bytes) + .map_err(|_| anyhow::anyhow!("invalid account configuration in {}", path.display()))?; + accounts + .runpod + .get(account) + .cloned() + .with_context(|| format!("unknown Runpod account {account:?} in {}", path.display())) + .context(RunpodAccountRequired) +} + +pub fn validate_name(name: &str) -> Result<()> { + ensure!( + !name.is_empty() + && name.len() <= 48 + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_".contains(&b)), + "account name must contain 1..48 ASCII letters, digits, hyphens, or underscores" + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn credential_commands_use_first_line_and_withhold_failure_output() { + let source = CredentialSource::Command(vec![ + "sh".into(), + "-c".into(), + "printf 'test-token\\nextra pass metadata\\n'".into(), + ]); + assert_eq!(source.token().await.unwrap(), "test-token"); + let failed = CredentialSource::Command(vec![ + "sh".into(), + "-c".into(), + "printf do-not-leak; printf do-not-leak >&2; exit 1".into(), + ]); + let error = format!("{:#}", failed.token().await.unwrap_err()); + assert!(error.contains("output withheld")); + assert!(!error.contains("do-not-leak")); + } +} diff --git a/crates/cloud/src/agent.rs b/crates/cloud/src/agent.rs new file mode 100644 index 00000000..39561808 --- /dev/null +++ b/crates/cloud/src/agent.rs @@ -0,0 +1,554 @@ +use std::{ + net::SocketAddr, + path::{Path, PathBuf}, + process::Stdio, + sync::Arc, + time::Duration, +}; + +use anyhow::{Context, Result, bail, ensure}; +use iroh::{Endpoint, endpoint::presets}; +use sha2::{Digest, Sha256}; +use subtle::ConstantTimeEq; +use tokio::{ + io::AsyncWriteExt, + process::{Child, Command}, + sync::{Mutex, Semaphore}, +}; + +use crate::{ + config::{Credentials, Enrollment, validate_hex}, + wire::{ALPN, MAX_MESSAGE, Operation, Request, Response}, +}; + +pub struct AgentOptions { + pub credentials: Credentials, + pub data: PathBuf, + /// Separate private filesystem for credentials when the data volume lacks Unix modes. + pub configuration_dir: Option, + pub cli: PathBuf, + /// Original OCI entrypoint, including `serve` and GPU backend defaults. + pub launcher: Vec, + pub serve_args: Vec, + pub bind: Option, + pub no_relay: bool, +} + +struct Process { + child: Option, + launcher: Vec, + args: Vec, + identity: PathBuf, + owner: Option, + cli: PathBuf, + configuration: Option, + configuration_dir: PathBuf, +} + +impl Process { + fn start(&mut self) -> Result<()> { + let (program, args) = self + .launcher + .split_first() + .context("empty Hellas launcher")?; + let mut command = Command::new(program); + command + .args(args) + .args(&self.args) + .arg("--identity") + .arg(&self.identity) + .arg("--artifact-store-path") + .arg( + self.identity + .parent() + .context("identity needs a data directory")? + .join("artifacts"), + ) + .args(["--assurance", "producer-signed"]) + .env_remove("HELLAS_REMOTE_KEY") + .env_remove("HELLAS_REMOTE_TOKEN") + .env_remove("HELLAS_REMOTE_ARGS") + .env_remove("HELLAS_REMOTE_OWNER") + .stdin(Stdio::null()) + .stdout(Stdio::from(std::io::stderr())) + .stderr(Stdio::inherit()) + .kill_on_drop(true); + if let Some(owner) = &self.owner { + command.args(["--owner", owner]); + } + if let Some(configuration) = &self.configuration { + command + .arg("--fetch-config") + .arg(&configuration.fetch_config) + .envs(&configuration.env); + } + #[cfg(unix)] + command.process_group(0); + self.child = Some(command.spawn().context("start Hellas")?); + Ok(()) + } + + fn running(&mut self) -> Result { + Ok(match self.child.as_mut() { + Some(child) => child.try_wait()?.is_none(), + None => false, + }) + } + + async fn stop(&mut self) -> Result<()> { + if let Some(mut child) = self.child.take() + && let Some(id) = child.id() + { + #[cfg(unix)] + unsafe { + libc::kill(-(id as i32), libc::SIGTERM); + } + #[cfg(not(unix))] + child.start_kill()?; + if tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .is_err() + { + #[cfg(unix)] + unsafe { + libc::kill(-(id as i32), libc::SIGKILL); + } + child.kill().await?; + } + } + Ok(()) + } + + async fn configure( + &mut self, + configuration: crate::configuration::Configuration, + ) -> Result<(), &'static str> { + configuration + .validate() + .map_err(|_| "invalid worker configuration")?; + if self + .args + .iter() + .any(|arg| arg.split('=').next() == Some("--fetch-config")) + { + return Err("managed configuration conflicts with a launch-time fetch config"); + } + let (candidate, installed) = configuration.stage(&self.configuration_dir)?; + // The worker's own CLI validates the exact config and credentials before + // disrupting the running process. Validation output can contain secrets. + let checked = tokio::time::timeout( + Duration::from_secs(30), + Command::new(&self.cli) + .args(["serve", "--check-config", "--fetch-config"]) + .arg(&installed.fetch_config) + .envs(&configuration.env) + .env_remove("HELLAS_REMOTE_KEY") + .env_remove("HELLAS_REMOTE_TOKEN") + .env_remove("HELLAS_REMOTE_ARGS") + .env_remove("HELLAS_REMOTE_OWNER") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .status(), + ) + .await + .map_err(|_| "worker configuration validator timed out")? + .map_err(|_| "could not run worker configuration validator")?; + if !checked.success() { + return Err("worker rejected fetch configuration"); + } + let path = self.configuration_dir.join("configuration.json"); + self.stop() + .await + .map_err(|_| "could not stop worker for configuration")?; + let previous = self.configuration.replace(installed); + let applied = self + .start() + .map_err(|_| "could not start configured worker") + .and_then(|()| { + crate::config::save_private(&path, self.configuration.as_ref().unwrap(), false) + .map_err(|_| "could not persist worker configuration") + }); + if let Err(error) = applied { + self.stop() + .await + .map_err(|_| "could not stop worker during configuration rollback")?; + self.configuration = previous; + if let Some(previous) = &self.configuration { + crate::config::save_private(&path, previous, false) + .map_err(|_| "could not restore previous worker configuration")?; + } else { + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Err("could not remove failed worker configuration"), + } + } + self.start() + .map_err(|_| "could not restart worker after configuration rollback")?; + return Err(error); + } + let _ = candidate.keep(); + if let Some(previous) = previous + && let Some(directory) = previous.fetch_config.parent() + && directory.parent() == Some(self.configuration_dir.as_path()) + && directory + .file_name() + .is_some_and(|name| name.to_string_lossy().starts_with(".worker-config-")) + { + let _ = std::fs::remove_dir_all(directory); + } + Ok(()) + } +} + +async fn identity_command(cli: &Path, identity: &Path, operation: &str) -> Result { + let mut command = Command::new(cli); + command.arg("--identity").arg(identity); + if operation == "init" { + command.arg("--software-root"); + } + let output = tokio::time::timeout( + Duration::from_secs(30), + command + .args(["identity", operation]) + .env_remove("HELLAS_REMOTE_KEY") + .env_remove("HELLAS_REMOTE_TOKEN") + .env_remove("HELLAS_REMOTE_ARGS") + .env_remove("HELLAS_REMOTE_OWNER") + .kill_on_drop(true) + .output(), + ) + .await??; + ensure!( + output.status.success(), + "Hellas identity {operation} failed" + ); + Ok(String::from_utf8(output.stdout)?.trim().to_owned()) +} + +struct State { + token: String, + owner: Option, + enrollment: Enrollment, + process: Mutex, + downloads: Semaphore, + content: PathBuf, +} + +impl State { + async fn dispatch(&self, peer: iroh::EndpointId, request: Request) -> Response { + if self.owner.is_some_and(|owner| owner != peer) + || !bool::from(self.token.as_bytes().ct_eq(request.token.as_bytes())) + { + return Response::Error { + message: "unauthorized".into(), + }; + } + let result: Result = async { + match request.operation { + Operation::Status => Ok(Response::Status { + enrollment: self.enrollment.clone(), + owner: self.owner.map(|owner| owner.to_string()), + running: self.process.lock().await.running()?, + }), + Operation::Restart => { + let mut process = self.process.lock().await; + process.stop().await?; + process.start()?; + Ok(Response::Status { + enrollment: self.enrollment.clone(), + owner: self.owner.map(|owner| owner.to_string()), + running: process.running()?, + }) + } + Operation::Configure { configuration } => { + let mut process = self.process.lock().await; + // Only static stage labels cross this boundary, never the + // validator's output or errors containing credential paths. + if let Err(message) = process.configure(configuration).await { + return Ok(Response::Error { + message: message.into(), + }); + } + Ok(Response::Status { + enrollment: self.enrollment.clone(), + owner: self.owner.map(|owner| owner.to_string()), + running: process.running()?, + }) + } + Operation::Fetch { url, sha256, bytes } => { + let _permit = self + .downloads + .try_acquire() + .context("download already in progress")?; + fetch_content(&self.content, &url, &sha256, bytes).await?; + Ok(Response::Fetched { sha256, bytes }) + } + } + } + .await; + match result { + Ok(response) => response, + // reqwest errors can include signed URLs. Do not reflect them. + Err(_) => Response::Error { + message: + "operation failed; check arguments, child state, and content digest/length" + .into(), + }, + } + } +} + +pub async fn run(options: AgentOptions) -> Result<()> { + let secret = options.credentials.secret_key()?; + tokio::fs::create_dir_all(&options.data).await?; + // Refuse silent reassignment, including a restart that omits the owner. + let binding = options.data.join("owner.json"); + let _lease = crate::config::lock_state(&binding)?; + if binding.exists() { + let saved: Option = crate::config::read_json(&binding)?; + ensure!( + saved == options.credentials.owner, + "machine owner changed; refusing startup" + ); + } else { + crate::config::save_private(&binding, &options.credentials.owner, true)?; + } + let content = options.data.join("content"); + tokio::fs::create_dir_all(&content).await?; + let identity = options.data.join("identity"); + let configuration_dir = options + .configuration_dir + .unwrap_or_else(|| options.data.clone()); + if configuration_dir != options.data { + crate::management::private_directory(&configuration_dir)?; + } + let configuration_path = configuration_dir.join("configuration.json"); + let configuration: Option = configuration_path + .exists() + .then(|| crate::config::read_json(&configuration_path)) + .transpose()?; + identity_command(&options.cli, &identity, "init").await?; + let enrollment = Enrollment { + node_id: identity_command(&options.cli, &identity, "show-node-id").await?, + enrollment_id: identity_command(&options.cli, &identity, "show-enrollment-id").await?, + }; + enrollment.validate()?; + let mut builder = Endpoint::builder(presets::N0) + .secret_key(secret) + .alpns(vec![ALPN.to_vec()]); + if let Some(bind) = options.bind { + builder = builder.bind_addr(bind)?; + } + if options.no_relay { + builder = builder.relay_mode(iroh::RelayMode::Disabled); + } + let endpoint = builder.bind().await?; + let mut process = Process { + child: None, + launcher: options.launcher, + args: options.serve_args, + identity, + owner: options.credentials.owner.clone(), + cli: options.cli, + configuration, + configuration_dir, + }; + process.start()?; + let state = Arc::new(State { + token: options.credentials.token, + owner: options + .credentials + .owner + .map(|owner| owner.parse()) + .transpose()?, + enrollment, + process: Mutex::new(process), + content, + downloads: Semaphore::new(1), + }); + eprintln!("admin node: {}", endpoint.id()); + let slots = Arc::new(Semaphore::new(16)); + let mut tasks = tokio::task::JoinSet::new(); + let shutdown = shutdown_signal(); + tokio::pin!(shutdown); + loop { + tokio::select! { + _ = &mut shutdown => break, + Some(_) = tasks.join_next(), if !tasks.is_empty() => {}, + incoming = endpoint.accept() => { + let Some(incoming) = incoming else { break; }; + let Ok(permit) = slots.clone().try_acquire_owned() else { incoming.refuse(); continue; }; + let state = state.clone(); + tasks.spawn(async move { + let _permit = permit; + let result: Result<()> = async { + let connection = tokio::time::timeout(Duration::from_secs(10), incoming).await??; + let (mut send, mut recv) = tokio::time::timeout(Duration::from_secs(10), connection.accept_bi()).await??; + let bytes = tokio::time::timeout(Duration::from_secs(10), recv.read_to_end(MAX_MESSAGE)).await??; + let request: Request = serde_json::from_slice(&bytes)?; + let response = state.dispatch(connection.remote_id(), request).await; + send.write_all(&serde_json::to_vec(&response)?).await?; + send.finish()?; + // Keep the connection alive until the reply was consumed. + let _ = tokio::time::timeout(Duration::from_secs(10), connection.closed()).await; + Ok(()) + }.await; + // Invalid unauthenticated traffic is intentionally not logged. + let _ = result; + }); + } + } + } + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + let stopped = state.process.lock().await.stop().await; + endpoint.close().await; + stopped +} + +async fn shutdown_signal() { + #[cfg(unix)] + { + let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("install SIGTERM handler"); + tokio::select! { _ = tokio::signal::ctrl_c() => {}, _ = term.recv() => {} } + } + #[cfg(not(unix))] + let _ = tokio::signal::ctrl_c().await; +} + +pub fn validate_download(url: &str, sha256: &str, bytes: u64) -> Result { + validate_hex(sha256)?; + ensure!( + bytes > 0 && bytes <= 1024 * 1024 * 1024 * 1024, + "expected size must be 1 byte..1 TiB" + ); + let url = reqwest::Url::parse(url)?; + ensure!( + url.scheme() == "https" + && url.host_str().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.fragment().is_none(), + "content URL must be HTTPS without userinfo or fragment" + ); + Ok(url) +} + +pub async fn fetch_content(root: &Path, url: &str, sha256: &str, bytes: u64) -> Result<()> { + let url = validate_download(url, sha256, bytes)?; + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(20)) + .timeout(Duration::from_secs(3500)) + .build()?; + let response = client.get(url).send().await?; + receive_content(root, response, sha256, bytes).await +} + +async fn receive_content( + root: &Path, + mut response: reqwest::Response, + sha256: &str, + bytes: u64, +) -> Result<()> { + ensure!( + response.status() == reqwest::StatusCode::OK, + "download requires HTTP 200; redirects are refused" + ); + if let Some(length) = response.content_length() { + ensure!(length == bytes, "content length mismatch"); + } + // Stage outside the indexed content tree. Cancellation/errors remove the temp + // file; only an exact length+digest match becomes visible to Hellas. + let temp = + tempfile::NamedTempFile::new_in(root.parent().context("content root needs a parent")?)?; + let mut file = tokio::fs::File::from_std(temp.reopen()?); + let mut hash = Sha256::new(); + let mut received = 0u64; + while let Some(chunk) = response.chunk().await? { + received = received + .checked_add(chunk.len() as u64) + .context("size overflow")?; + ensure!(received <= bytes, "download exceeds declared size"); + hash.update(&chunk); + file.write_all(&chunk).await?; + } + ensure!( + received == bytes && hex::encode(hash.finalize()) == sha256, + "download length or digest mismatch" + ); + file.sync_all().await?; + drop(file); + let destination = root.join(sha256); + match temp.persist_noclobber(&destination) { + Ok(_) => { + std::fs::File::open(root)?.sync_all()?; + Ok(()) + } + Err(error) if error.error.kind() == std::io::ErrorKind::AlreadyExists => { + bail!("content already exists; no existing content was replaced") + } + Err(error) => Err(error.error.into()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::AsyncReadExt; + + async fn response(body: &'static [u8]) -> reqwest::Response { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut buffer = [0; 4096]; + assert!(stream.read(&mut buffer).await.unwrap() > 0); + stream + .write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .as_bytes(), + ) + .await + .unwrap(); + stream.write_all(body).await.unwrap(); + }); + reqwest::get(format!("http://{addr}")).await.unwrap() + } + + #[tokio::test] + async fn only_verified_content_is_published_and_existing_content_is_immutable() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("content"); + std::fs::create_dir(&root).unwrap(); + let hash = hex::encode(Sha256::digest(b"hello")); + assert!( + receive_content(&root, response(b"hello").await, &"0".repeat(64), 5) + .await + .is_err() + ); + assert!( + receive_content(&root, response(b"hello").await, &hash, 4) + .await + .is_err() + ); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); + receive_content(&root, response(b"hello").await, &hash, 5) + .await + .unwrap(); + assert_eq!(std::fs::read(root.join(&hash)).unwrap(), b"hello"); + assert!( + receive_content(&root, response(b"hello").await, &hash, 5) + .await + .is_err() + ); + } +} diff --git a/crates/cloud/src/bin/agent.rs b/crates/cloud/src/bin/agent.rs new file mode 100644 index 00000000..7f3495c5 --- /dev/null +++ b/crates/cloud/src/bin/agent.rs @@ -0,0 +1,98 @@ +//! Image companion. Allocation and operator commands live in the main Hellas CLI. +#[cfg(unix)] +use anyhow::{Context, Result, ensure}; +#[cfg(unix)] +use clap::Parser; +#[cfg(unix)] +use hellas_cloud::{ + agent, + config::{Credentials, read_json, validate_serve_args}, +}; +#[cfg(unix)] +use std::{collections::BTreeMap, net::SocketAddr, path::PathBuf}; + +#[cfg(unix)] +#[derive(Parser)] +#[command(version, about = "Run an owner-bound Hellas worker")] +struct Args { + #[arg(long, default_value = "/var/lib/hellas")] + data: PathBuf, + /// Private settings/credentials directory; defaults to --data. + /// Use a filesystem supporting owner-only permissions. + #[arg(long)] + configuration_dir: Option, + /// Private environment map prepared by `hellas machines prepare`. + #[arg(long)] + bootstrap: Option, + #[arg(long, default_value = "/bin/hellas-cli")] + cli: PathBuf, + /// Original OCI entrypoint encoded as a JSON argv array. + #[arg(long)] + launcher: Option, + #[arg(long)] + bind: Option, + #[arg(long)] + no_relay: bool, + #[arg(last = true)] + serve_args: Vec, +} + +#[cfg(unix)] +#[tokio::main] +async fn main() -> Result<()> { + let Args { + data, + configuration_dir, + bootstrap, + cli, + launcher, + bind, + no_relay, + mut serve_args, + } = Args::parse(); + let bootstrap: BTreeMap = bootstrap + .as_deref() + .map(read_json) + .transpose()? + .unwrap_or_default(); + let setting = |name: &str| { + bootstrap + .get(name) + .cloned() + .or_else(|| std::env::var(name).ok()) + }; + let credentials = Credentials { + admin_secret: setting("HELLAS_REMOTE_KEY").context("missing HELLAS_REMOTE_KEY")?, + token: setting("HELLAS_REMOTE_TOKEN").context("missing HELLAS_REMOTE_TOKEN")?, + owner: Some(setting("HELLAS_REMOTE_OWNER").context("missing HELLAS_REMOTE_OWNER")?), + }; + if let Some(args) = setting("HELLAS_REMOTE_ARGS") { + ensure!( + serve_args.is_empty(), + "serve args supplied both in env and argv" + ); + serve_args = serde_json::from_slice(&hex::decode(args)?)?; + } + validate_serve_args(&serve_args)?; + let launcher = launcher + .as_deref() + .map(read_json) + .transpose()? + .unwrap_or_else(|| vec![cli.to_string_lossy().into_owned(), "serve".into()]); + agent::run(agent::AgentOptions { + credentials, + data, + configuration_dir, + cli, + launcher, + serve_args, + bind, + no_relay, + }) + .await +} + +#[cfg(not(unix))] +fn main() -> anyhow::Result<()> { + anyhow::bail!("hellas-agent currently requires Unix") +} diff --git a/crates/cloud/src/cloud.rs b/crates/cloud/src/cloud.rs new file mode 100644 index 00000000..72f910e6 --- /dev/null +++ b/crates/cloud/src/cloud.rs @@ -0,0 +1,219 @@ +//! The command subtree embedded directly in the main Hellas CLI. +use std::path::PathBuf; + +use anyhow::{Context, Result, bail, ensure}; +use clap::{Args, Subcommand}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::{ + config::{Deployment, ProviderConfig, Spec, Trust, read_json}, + deployment, + provider::{Cloud, Provider, validate_id}, +}; + +#[derive(Args)] +pub struct CloudArgs { + /// Use a running private management service for these operations. + #[arg(long, global = true)] + pub socket: Option, + #[command(subcommand)] + pub provider: CloudCommand, +} + +#[derive(Subcommand)] +pub enum CloudCommand { + /// Manage Runpod GPU pods. + Runpod(RunpodArgs), +} + +#[derive(Args, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RunpodArgs { + /// Named profile in cloud-accounts.json. Omit to use RUNPOD_API_KEY. + #[arg(long, global = true)] + #[serde(default)] + pub account: Option, + #[command(subcommand)] + pub command: RunpodCommand, +} + +#[derive(Subcommand, Deserialize, Serialize)] +#[serde(tag = "op", rename_all = "kebab-case", deny_unknown_fields)] +pub enum RunpodCommand { + /// Show one pod's provider metadata (not Hellas readiness). + Info { + #[arg(value_parser = parse_id)] + pod_id: String, + }, + /// List pods in the selected account, without credentials or environment. + List, + /// Create one Hellas pod and save its private administration receipt. + Create { + #[arg(long)] + name: String, + /// Existing Hellas Runpod template; its image must be digest-pinned. + #[arg(long, value_parser = parse_id)] + template: String, + #[arg(long = "gpu")] + gpu_type: String, + /// Request a spot pod that Runpod may interrupt at any time. + #[arg(long)] + #[serde(default)] + interruptible: bool, + /// Optional receipt path; defaults to the owner machine inventory. + #[arg(long)] + state: Option, + /// Validate and print the request without reading credentials or allocating. + #[arg(long)] + #[serde(default)] + dry_run: bool, + /// Arguments for Hellas serve. Execution defaults to disabled. + #[arg(last = true)] + #[serde(default)] + serve_args: Vec, + }, + /// Delete a pod. With --state, also mark its receipt as destroyed. + Destroy { + /// Required unless --state supplies the pod ID and account. + #[arg(required_unless_present = "state", value_parser = parse_id)] + pod_id: Option, + /// Use the saved account; reject conflicting --account or pod ID. + #[arg(long)] + state: Option, + }, +} + +fn parse_id(value: &str) -> std::result::Result { + validate_id(value).map_err(|error| error.to_string())?; + Ok(value.to_owned()) +} + +impl CloudArgs { + pub fn needs_identity(&self) -> bool { + self.socket.is_some() + || matches!( + &self.provider, + CloudCommand::Runpod(RunpodArgs { + command: RunpodCommand::Create { dry_run: false, .. } + | RunpodCommand::Destroy { state: Some(_), .. }, + .. + }) + ) + } + + pub async fn run_owned(self, service: Option<&crate::management::Service>) -> Result<()> { + ensure!( + !self.needs_identity() || service.is_some(), + "an existing Hellas owner identity is required" + ); + if let Some(socket) = &self.socket { + let service = service.context("owner identity required")?; + let inventory = + crate::internal_rpc::call(socket, crate::management::Request::List).await?; + ensure!( + inventory["owner"] == service.owner(), + "control socket serves another identity" + ); + let CloudCommand::Runpod(args) = self.provider; + let result = + crate::internal_rpc::call(socket, crate::management::Request::Runpod(args)).await?; + println!("{}", serde_json::to_string_pretty(&result)?); + return Ok(()); + } + let value = match self.provider { + CloudCommand::Runpod(args) => args.run_managed(service).await?, + }; + println!("{}", serde_json::to_string_pretty(&value)?); + Ok(()) + } +} + +impl RunpodArgs { + pub async fn run(self) -> Result { + self.run_managed(None).await + } + + pub async fn run_managed(self, service: Option<&crate::management::Service>) -> Result { + let mut account = self.account; + let result = match self.command { + RunpodCommand::List => Cloud::runpod(account.as_deref())?.list().await?, + RunpodCommand::Info { pod_id } => { + Cloud::runpod(account.as_deref())?.inspect(&pod_id).await? + } + RunpodCommand::Create { + name, + template, + gpu_type, + interruptible, + state, + dry_run, + mut serve_args, + } => { + if serve_args.is_empty() { + serve_args = vec!["--execute-policy".into(), "none".into()]; + } + let spec = Spec { + name, + image: String::new(), + trust: Trust::Token, + serve_args, + provider: ProviderConfig::Runpod { + account: account.clone(), + template_id: Some(template), + gpu_type, + interruptible, + disk_gb: 0, + volume_gb: 0, + container_registry_auth_id: None, + }, + }; + spec.validate()?; + if dry_run { + // The payload does not depend on an API credential or profile file. + Cloud::new(crate::provider::CloudKind::Runpod)?.plan(&spec)? + } else if let Some(service) = service { + service.create(spec, state).await? + } else { + let state = state.context("create requires --state")?; + let id = deployment::create(spec, &state).await?; + json!({"id":id, "state":state}) + } + } + RunpodCommand::Destroy { + pod_id, + state: Some(path), + } => { + let state: Deployment = read_json(&path)?; + if let Some(owner) = &state.credentials.owner { + ensure!( + service.is_some_and(|service| service.owner() == *owner), + "receipt belongs to another identity" + ); + } + let ProviderConfig::Runpod { account: saved, .. } = &state.spec.provider else { + bail!("receipt belongs to another cloud provider; refusing termination"); + }; + if let Some(selected) = &account { + ensure!( + saved.as_ref() == Some(selected), + "--account does not match receipt; refusing termination" + ); + } + account = saved.clone(); + let id = deployment::destroy(&path, Some(&state.spec.provider), pod_id.as_deref()) + .await?; + json!({"id":id, "destroyed":true, "state":path}) + } + RunpodCommand::Destroy { + pod_id, + state: None, + } => { + let id = pod_id.context("destroy requires a pod ID or --state")?; + Cloud::runpod(account.as_deref())?.destroy(&id).await?; + json!({"id":id, "destroyed":true}) + } + }; + Ok(json!({"provider":"runpod", "account":account, "result":result})) + } +} diff --git a/crates/cloud/src/config.rs b/crates/cloud/src/config.rs new file mode 100644 index 00000000..59c59f04 --- /dev/null +++ b/crates/cloud/src/config.rs @@ -0,0 +1,299 @@ +use std::{collections::BTreeMap, fs, path::Path}; + +use anyhow::{Context, Result, ensure}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Spec { + pub name: String, + /// Digest-pinned image; resolved from the template before Runpod allocation. + #[serde(default)] + pub image: String, + pub provider: ProviderConfig, + #[serde(default)] + pub trust: Trust, + /// Arguments to the image's original `hellas-cli serve` entrypoint. + #[serde(default)] + pub serve_args: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum ProviderConfig { + Docker { + #[serde(default)] + gpus: bool, + }, + Runpod { + /// Named credential profile; omitted for the legacy RUNPOD_API_KEY account. + #[serde(default, skip_serializing_if = "Option::is_none")] + account: Option, + /// None only for receipts from the earlier image-based prototype. + #[serde(default, skip_serializing_if = "Option::is_none")] + template_id: Option, + gpu_type: String, + #[serde(default)] + interruptible: bool, + #[serde(default)] + disk_gb: u32, + #[serde(default)] + volume_gb: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + container_registry_auth_id: Option, + }, + Vast { + offer_id: u64, + disk_gb: u32, + }, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum Trust { + #[default] + Token, + MeasuredBoot, +} + +impl Spec { + pub fn validate(&self) -> Result<()> { + ensure!( + self.trust == Trust::Token, + "measured-boot requires a platform verifier and channel binding; no adapter implements it yet" + ); + ensure!( + !self.name.is_empty() + && self.name.len() <= 48 + && self + .name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-'), + "name must contain 1..48 ASCII letters, digits, or hyphens" + ); + let unresolved_template = self.image.is_empty() + && matches!( + &self.provider, + ProviderConfig::Runpod { + template_id: Some(_), + .. + } + ); + if unresolved_template { + // A credential-free dry run cannot resolve account template metadata. + } else if let (ProviderConfig::Docker { .. }, Some(digest)) = + (&self.provider, self.image.strip_prefix("sha256:")) + { + validate_hex(digest)?; + } else { + let (repo, digest) = self + .image + .rsplit_once("@sha256:") + .context("image must be pinned as repository@sha256:<64 lowercase hex digits>")?; + ensure!( + !repo.is_empty() + && !repo.starts_with('-') + && repo + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"/._:-".contains(&b)), + "invalid image repository" + ); + validate_hex(digest)?; + } + validate_serve_args(&self.serve_args)?; + match &self.provider { + ProviderConfig::Runpod { + account, + template_id, + gpu_type, + disk_gb, + volume_gb, + .. + } => { + if let Some(account) = account { + crate::accounts::validate_name(account)?; + } + if let Some(id) = template_id { + crate::provider::validate_id(id)?; + } + ensure!( + !gpu_type.trim().is_empty() + && (unresolved_template || (*disk_gb > 0 && *volume_gb > 0)), + "Runpod requires a GPU type and positive disk/volume sizes" + ); + } + ProviderConfig::Vast { offer_id, disk_gb } => ensure!( + *offer_id > 0 && *disk_gb > 0, + "Vast requires a positive offer ID and disk size" + ), + _ => {} + } + Ok(()) + } +} + +pub fn validate_serve_args(args: &[String]) -> Result<()> { + ensure!( + args.iter().all(|v| !v.contains('\0')), + "NUL in serve argument" + ); + for arg in args { + let key = arg.split('=').next().unwrap_or(arg); + ensure!( + ![ + "--identity", + "--owner", + "--software-root", + "--assurance", + "--artifact-store-path", + "--help", + "-h", + "--version", + "--check-config", + "-V" + ] + .contains(&key), + "reserved serve argument: {key}" + ); + } + Ok(()) +} + +pub fn validate_hex(value: &str) -> Result<()> { + ensure!( + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)), + "expected 64 lowercase hex digits" + ); + Ok(()) +} + +// Deliberately no Debug implementation for credentials or deployment state. +#[derive(Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Credentials { + pub admin_secret: String, + pub token: String, + /// Public transport identity allowed to administer and use this machine. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner: Option, +} + +impl Credentials { + pub fn generate() -> Self { + Self { + admin_secret: hex::encode(iroh::SecretKey::generate().to_bytes()), + token: hex::encode(iroh::SecretKey::generate().to_bytes()), + owner: None, + } + } + + pub fn secret_key(&self) -> Result { + if let Some(owner) = &self.owner { + validate_hex(owner)?; + owner.parse::()?; + } + validate_hex(&self.admin_secret)?; + validate_hex(&self.token)?; + Ok(iroh::SecretKey::from_bytes( + &hex::decode(&self.admin_secret)?.try_into().unwrap(), + )) + } + + pub fn env(&self, spec: &Spec) -> Result> { + self.env_for_args(&spec.serve_args) + } + + pub fn env_for_args(&self, args: &[String]) -> Result> { + self.secret_key()?; + let mut env = BTreeMap::from([ + ("HELLAS_REMOTE_KEY".into(), self.admin_secret.clone()), + ("HELLAS_REMOTE_TOKEN".into(), self.token.clone()), + // Hex makes the Vast Docker-flag representation unambiguous. + ( + "HELLAS_REMOTE_ARGS".into(), + hex::encode(serde_json::to_vec(args)?), + ), + ]); + if let Some(owner) = &self.owner { + env.insert("HELLAS_REMOTE_OWNER".into(), owner.clone()); + } + Ok(env) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Enrollment { + pub node_id: String, + pub enrollment_id: String, +} + +impl Enrollment { + pub fn validate(&self) -> Result<()> { + self.node_id.parse::()?; + validate_hex(&self.enrollment_id) + } +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Deployment { + pub spec: Spec, + pub credentials: Credentials, + pub resource_id: Option, + pub enrollment: Option, + #[serde(default)] + pub destroyed: bool, +} + +pub fn read_json(path: &Path) -> Result { + serde_json::from_slice(&fs::read(path).with_context(|| format!("read {}", path.display()))?) + .context("invalid JSON file") +} + +/// Serialize operations on a receipt, including the API call between reads and +/// writes. Advisory lock files contain no credentials and survive crashes safely. +pub fn lock_state(path: &Path) -> Result { + let parent = path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + fs::create_dir_all(parent)?; + let mut name = path.as_os_str().to_owned(); + name.push(".lock"); + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(name)?; + file.try_lock() + .context("another command is using this deployment receipt")?; + Ok(file) +} + +/// Atomic, owner-only state; a failed create leaves the original pending record. +pub fn save_state(path: &Path, value: &Deployment, new: bool) -> Result<()> { + save_private(path, value, new) +} + +pub fn save_private(path: &Path, value: &T, new: bool) -> Result<()> { + let parent = path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + fs::create_dir_all(parent)?; + let mut file = hellas_private::private_tempfile(parent, ".cloud-", ".tmp")?; + serde_json::to_writer_pretty(&mut file, value)?; + file.as_file().sync_all()?; + if new { + file.persist_noclobber(path).map_err(|e| e.error)?; + } else { + file.persist(path).map_err(|e| e.error)?; + } + hellas_private::sync_directory(parent)?; + Ok(()) +} diff --git a/crates/cloud/src/configuration.rs b/crates/cloud/src/configuration.rs new file mode 100644 index 00000000..d9c6a5ef --- /dev/null +++ b/crates/cloud/src/configuration.rs @@ -0,0 +1,225 @@ +//! Private worker settings delivered over the owner's authenticated admin channel. +use std::{ + collections::BTreeMap, + fmt, + path::{Path, PathBuf}, +}; + +use anyhow::{Result, ensure}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Configuration { + /// The CLI's fetch route file, including explicit caller grants. + pub fetch_config: Value, + /// Provider-local credential environment, never returned by status or inventory. + #[serde(default)] + pub env: BTreeMap, + /// Private JSON files referenced from fetch_config as @files/NAME. + #[serde(default)] + pub files: BTreeMap, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::os::unix::fs::PermissionsExt; + + #[test] + fn staging_restricts_permissions_but_refuses_symlinks() { + let parent = tempfile::tempdir().unwrap(); + let directory = parent.path().join("fresh"); + std::fs::create_dir(&directory).unwrap(); + std::fs::set_permissions(&directory, std::fs::Permissions::from_mode(0o777)).unwrap(); + restrict_staging_directory(&directory).unwrap(); + assert_eq!( + std::fs::metadata(&directory).unwrap().permissions().mode() & 0o777, + 0o700 + ); + let link = parent.path().join("link"); + std::os::unix::fs::symlink(&directory, &link).unwrap(); + assert!(restrict_staging_directory(&link).is_err()); + } + + #[test] + fn staged_credentials_are_private_and_missing_references_leave_no_files() { + let parent = tempfile::tempdir().unwrap(); + let mut configuration = Configuration { + fetch_config: json!({"routes":[{"auth_path":"@files/auth.json"}],"callers":[]}), + env: BTreeMap::new(), + files: [("auth.json".into(), json!({"token":"fixture"}))].into(), + }; + let (staged, installed) = configuration.stage(parent.path()).unwrap(); + let config: Value = crate::config::read_json(&installed.fetch_config).unwrap(); + let auth = Path::new(config["routes"][0]["auth_path"].as_str().unwrap()); + assert!(auth.starts_with(staged.path())); + assert_eq!( + crate::config::read_json::(auth).unwrap(), + configuration.files["auth.json"] + ); + for path in [auth, installed.fetch_config.as_path()] { + assert_eq!( + std::fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + let serialized = serde_json::to_string(&installed).unwrap(); + assert!(!serialized.contains("fixture")); + drop(staged); + configuration.files.clear(); + assert!(configuration.stage(parent.path()).is_err()); + assert_eq!(std::fs::read_dir(parent.path()).unwrap().count(), 0); + } +} + +// Keep a pointer to the installed files, not a second copy of their credentials. +// The provider may refresh its auth file; ordinary restarts must preserve that. +#[derive(Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct InstalledConfiguration { + pub fetch_config: PathBuf, + pub env: BTreeMap, +} + +impl fmt::Debug for Configuration { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Configuration { contents withheld }") + } +} + +impl Configuration { + pub fn validate(&self) -> Result<()> { + ensure!( + self.fetch_config.is_object() + && self.fetch_config.get("routes").is_some_and(Value::is_array) + && self + .fetch_config + .get("callers") + .is_some_and(Value::is_array), + "fetch configuration requires routes and callers arrays" + ); + for (name, value) in &self.env { + ensure!( + !name.is_empty() + && name.len() <= 128 + && name + .bytes() + .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_') + && !name.as_bytes()[0].is_ascii_digit() + && !["HOME", "PATH", "TMPDIR", "TMP", "TEMP"].contains(&name.as_str()) + && !["HELLAS_", "LD_", "DYLD_", "RUST_", "SSL_", "NIX_"] + .iter() + .any(|p| name.starts_with(p)) + && !value.contains('\0'), + "invalid or reserved credential environment variable" + ); + } + for name in self.files.keys() { + ensure!( + !name.is_empty() + && name.len() <= 128 + && !name.starts_with('.') + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b)), + "credential file name must be a simple filename" + ); + } + ensure!( + serde_json::to_vec(self)?.len() <= 48 * 1024, + "worker configuration exceeds 48 KiB" + ); + Ok(()) + } + + pub(crate) fn stage( + &self, + parent: &Path, + ) -> Result<(tempfile::TempDir, InstalledConfiguration), &'static str> { + self.validate() + .map_err(|_| "invalid worker configuration")?; + let stage = tempfile::Builder::new() + .prefix(".worker-config-") + .tempdir_in(parent) + .map_err(|_| "could not create worker configuration directory")?; + restrict_staging_directory(stage.path())?; + let files = stage.path().join("files"); + std::fs::create_dir(&files).map_err(|_| "could not create worker credential directory")?; + restrict_staging_directory(&files)?; + for (name, value) in &self.files { + crate::config::save_private(&files.join(name), value, true) + .map_err(|_| "could not write private worker credential file")?; + } + fn resolve(value: &mut Value, files: &Path, names: &BTreeMap) -> Result<()> { + match value { + Value::String(text) if text.starts_with("@files/") => { + let name = text.strip_prefix("@files/").unwrap(); + ensure!( + names.contains_key(name), + "fetch config references a missing credential file" + ); + *text = files.join(name).to_string_lossy().into_owned(); + } + Value::Array(values) => { + for value in values { + resolve(value, files, names)?; + } + } + Value::Object(values) => { + for value in values.values_mut() { + resolve(value, files, names)?; + } + } + _ => {} + } + Ok(()) + } + let mut fetch_config = self.fetch_config.clone(); + resolve(&mut fetch_config, &files, &self.files) + .map_err(|_| "fetch config references a missing credential file")?; + let path = stage.path().join("fetch.json"); + crate::config::save_private(&path, &fetch_config, true) + .map_err(|_| "could not write private worker fetch configuration")?; + Ok(( + stage, + InstalledConfiguration { + fetch_config: path, + env: self.env.clone(), + }, + )) + } +} + +// Only for fresh, empty staging directories. Some provider filesystems ignore +// mkdir's mode but support chmod. Verify the result before writing any secrets. +fn restrict_staging_directory(path: &Path) -> Result<(), &'static str> { + use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt}; + let directory = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path) + .map_err(|_| "could not open worker credential directory")?; + let metadata = directory + .metadata() + .map_err(|_| "could not inspect worker credential directory")?; + if metadata.uid() != unsafe { libc::geteuid() } { + return Err("worker filesystem does not preserve credential directory ownership"); + } + directory + .set_permissions(std::fs::Permissions::from_mode(0o700)) + .map_err(|_| "could not restrict worker credential directory permissions")?; + if directory + .metadata() + .map_err(|_| "could not inspect worker credential directory")? + .permissions() + .mode() + & 0o777 + != 0o700 + { + return Err("worker filesystem does not preserve private credential directory permissions"); + } + Ok(()) +} diff --git a/crates/cloud/src/deployment.rs b/crates/cloud/src/deployment.rs new file mode 100644 index 00000000..cf8343ce --- /dev/null +++ b/crates/cloud/src/deployment.rs @@ -0,0 +1,81 @@ +use std::path::Path; + +use anyhow::{Context, Result, ensure}; + +use crate::{ + config::{Credentials, Deployment, ProviderConfig, Spec, lock_state, read_json, save_state}, + provider, +}; + +pub async fn create(spec: Spec, state: &Path) -> Result { + create_with_credentials(spec, state, Credentials::generate()).await +} + +pub async fn create_with_credentials( + mut spec: Spec, + state: &Path, + credentials: Credentials, +) -> Result { + credentials.secret_key()?; + let _lock = lock_state(state)?; + spec.validate()?; + ensure!( + !state.exists(), + "state already exists; allocation was not attempted" + ); + let provider = provider::adapter(&spec.provider)?; + provider.prepare(&mut spec).await?; + let mut deployment = Deployment { + spec, + credentials, + resource_id: None, + enrollment: None, + destroyed: false, + }; + save_state(state, &deployment, true) + .context("state already exists or cannot be saved; allocation was not attempted")?; + let id = provider + .create(&deployment.spec, &deployment.credentials) + .await + .context("pending receipt retained; reconcile by name before retrying allocation")?; + eprintln!("allocated resource: {id}"); + deployment.resource_id = Some(id.clone()); + save_state(state, &deployment, false)?; + provider + .verify(&deployment.spec, &id) + .await + .context("allocated pod retained in receipt; inspect or destroy it before retrying")?; + Ok(id) +} + +pub async fn destroy( + path: &Path, + expected_provider: Option<&ProviderConfig>, + expected_id: Option<&str>, +) -> Result { + let _lock = lock_state(path)?; + let mut state: Deployment = read_json(path)?; + state.spec.validate()?; + ensure!(!state.destroyed, "deployment was destroyed"); + if let Some(expected) = expected_provider { + ensure!( + &state.spec.provider == expected, + "receipt provider/account changed; refusing termination" + ); + } + let id = state + .resource_id + .as_deref() + .context("pending allocation; reconcile by name and adopt before destroying")?; + if let Some(expected) = expected_id { + ensure!( + id == expected, + "pod ID does not match receipt; refusing termination" + ); + } + provider::adapter(&state.spec.provider)?.destroy(id).await?; + let id = id.to_owned(); + state.destroyed = true; + save_state(path, &state, false)?; + Ok(id) +} diff --git a/crates/cloud/src/internal_rpc.rs b/crates/cloud/src/internal_rpc.rs new file mode 100644 index 00000000..fef011fc --- /dev/null +++ b/crates/cloud/src/internal_rpc.rs @@ -0,0 +1,230 @@ +//! Private Unix-socket JSON-RPC 2.0: one JSON object per line, bounded frames. +//! The socket is an owner-authorized local control surface, never a public service. +use std::{ + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use serde_json::{Value, json}; +use tokio::{ + io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}, + net::{UnixListener, UnixStream}, + sync::Semaphore, +}; + +use crate::management::{Request, Service, private_directory}; + +const MAX_FRAME: u64 = 1024 * 1024; + +pub fn default_socket(owner: &str) -> Result { + crate::config::validate_hex(owner)?; + let base = std::env::var_os("XDG_RUNTIME_DIR") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir); + Ok(base + .join(format!( + "hellas-{}-{}", + unsafe { libc::geteuid() }, + &owner[..16] + )) + .join("control.sock")) +} + +fn error(id: Value, code: i32, message: &str) -> Value { + json!({"jsonrpc":"2.0", "id":id, "error":{"code":code, "message":message}}) +} + +async fn dispatch(service: &Service, input: &[u8]) -> Option { + let value: Value = match serde_json::from_slice(input) { + Ok(value) => value, + Err(_) => return Some(error(Value::Null, -32700, "Parse error")), + }; + let id = value.get("id").cloned().unwrap_or(Value::Null); + if !value.is_object() + || value["jsonrpc"] != "2.0" + || !value["method"].is_string() + || !(id.is_null() || id.is_string() || id.is_number()) + { + return Some(error(Value::Null, -32600, "Invalid Request")); + } + let notification = value.get("id").is_none(); + let method = value["method"].as_str().unwrap(); + let known = matches!( + method, + "machines.list" + | "machines.status" + | "machines.resolve" + | "machines.restart" + | "machines.configure" + | "machines.fetch" + | "machines.prepare" + | "machines.destroy" + | "cloud.runpod" + ); + let result = if !known { + error(id.clone(), -32601, "Method not found") + } else { + let mut request = json!({"method":method}); + if method != "machines.list" { + request["params"] = value.get("params").cloned().unwrap_or(json!({})); + } + match serde_json::from_value::(request) { + Err(_) => error(id.clone(), -32602, "Invalid params"), + Ok(request) => match service.execute(request).await { + Ok(result) => json!({"jsonrpc":"2.0", "id":id, "result":result}), + // This typed setup hint contains only public, static text. + Err(cause) if cause.is::() => error( + id, + -32000, + &crate::accounts::RunpodAccountRequired.to_string(), + ), + // Provider errors and malformed credential commands must never leak secrets. + Err(_) => error( + id, + -32000, + "Management operation failed; inspect the machine with the CLI", + ), + }, + } + }; + (!notification).then_some(result) +} + +async fn connection(service: Arc, stream: UnixStream) -> Result<()> { + ensure!( + stream.peer_cred()?.uid() == unsafe { libc::geteuid() }, + "unauthorized local user" + ); + let (read, mut write) = stream.into_split(); + let mut read = BufReader::new(read); + loop { + let mut frame = Vec::new(); + let size = tokio::time::timeout( + Duration::from_secs(30), + (&mut read) + .take(MAX_FRAME + 1) + .read_until(b'\n', &mut frame), + ) + .await??; + if size == 0 { + break; + } + ensure!( + size as u64 <= MAX_FRAME && frame.ends_with(b"\n"), + "invalid RPC frame" + ); + if let Some(response) = dispatch(&service, &frame).await { + let mut bytes = serde_json::to_vec(&response)?; + ensure!(bytes.len() as u64 <= MAX_FRAME, "RPC response too large"); + bytes.push(b'\n'); + tokio::time::timeout(Duration::from_secs(30), write.write_all(&bytes)).await??; + } + } + Ok(()) +} + +pub async fn serve(service: Service, socket: &Path) -> Result<()> { + serve_until(service, socket, async { + let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("install management SIGTERM handler"); + tokio::select! { _ = tokio::signal::ctrl_c() => {}, _ = term.recv() => {} } + }) + .await +} + +pub async fn serve_until( + service: Service, + socket: &Path, + shutdown: impl std::future::Future, +) -> Result<()> { + use std::os::unix::fs::{FileTypeExt, PermissionsExt}; + private_directory( + socket + .parent() + .context("socket needs a private directory")?, + )?; + let _lock = crate::config::lock_state(socket)?; + if socket.try_exists()? { + ensure!( + std::fs::symlink_metadata(socket)?.file_type().is_socket(), + "refusing to replace a non-socket path" + ); + std::fs::remove_file(socket)?; + } + let listener = UnixListener::bind(socket)?; + std::fs::set_permissions(socket, std::fs::Permissions::from_mode(0o600))?; + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } + } + let _cleanup = Cleanup(socket.to_owned()); + let service = Arc::new(service); + let slots = Arc::new(Semaphore::new(16)); + let mut tasks = tokio::task::JoinSet::new(); + tokio::pin!(shutdown); + loop { + tokio::select! { + _ = &mut shutdown => break, + Some(_) = tasks.join_next(), if !tasks.is_empty() => {}, + incoming = listener.accept() => { + let (stream, _) = incoming?; + let Ok(permit) = slots.clone().try_acquire_owned() else { continue; }; + let service = service.clone(); + tasks.spawn(async move { let _permit = permit; let _ = connection(service, stream).await; }); + } + } + } + // Finish in-flight allocations so disconnect/shutdown doesn't abandon a create response. + // Idle readers time out after 30 seconds; receipts precede all cloud API calls. + while tasks.join_next().await.is_some() {} + Ok(()) +} + +pub async fn call(socket: &Path, request: Request) -> Result { + let stream = UnixStream::connect(socket).await?; + ensure!( + stream.peer_cred()?.uid() == unsafe { libc::geteuid() }, + "unexpected local server user" + ); + let (read, mut write) = stream.into_split(); + let mut value = serde_json::to_value(request)?; + value["jsonrpc"] = json!("2.0"); + value["id"] = json!(1); + let mut bytes = serde_json::to_vec(&value)?; + ensure!(bytes.len() as u64 <= MAX_FRAME, "RPC request too large"); + bytes.push(b'\n'); + write.write_all(&bytes).await?; + let mut response = Vec::new(); + tokio::time::timeout( + Duration::from_secs(3600), + BufReader::new(read) + .take(MAX_FRAME + 1) + .read_until(b'\n', &mut response), + ) + .await??; + ensure!( + response.len() as u64 <= MAX_FRAME && response.ends_with(b"\n"), + "invalid RPC response frame" + ); + let response: Value = serde_json::from_slice(&response)?; + ensure!( + response["jsonrpc"] == "2.0" && response["id"] == 1, + "invalid RPC response" + ); + if response.get("error").is_some() { + anyhow::bail!( + "internal management RPC failed: {}", + response["error"]["message"] + .as_str() + .unwrap_or("Management operation failed") + ); + } + response + .get("result") + .cloned() + .context("RPC result missing") +} diff --git a/crates/cloud/src/lib.rs b/crates/cloud/src/lib.rs new file mode 100644 index 00000000..b41d96df --- /dev/null +++ b/crates/cloud/src/lib.rs @@ -0,0 +1,14 @@ +//! Remote worker management for Unix operators and workers. +#![cfg(unix)] + +pub mod accounts; +pub mod agent; +pub mod cloud; +pub mod config; +pub mod configuration; +pub mod deployment; +pub mod internal_rpc; +pub mod machines; +pub mod management; +pub mod provider; +pub mod wire; diff --git a/crates/cloud/src/machines.rs b/crates/cloud/src/machines.rs new file mode 100644 index 00000000..11294273 --- /dev/null +++ b/crates/cloud/src/machines.rs @@ -0,0 +1,192 @@ +use crate::management::{Request, Service}; +use anyhow::Result; +use clap::{Args, Subcommand}; +use std::{net::SocketAddr, path::PathBuf}; + +#[derive(Args)] +pub struct MachinesArgs { + /// Call a running internal management service instead of running in-process. + #[arg(long, global = true)] + pub socket: Option, + #[command(subcommand)] + pub command: MachineCommand, +} + +#[derive(Subcommand)] +pub enum MachineCommand { + /// Show this identity's machines. Last observations are not liveness guarantees. + List, + Status { + name: String, + }, + /// Authenticate the live machine and return its pinned execution route. + Resolve { + name: String, + }, + Restart { + name: String, + }, + /// Install fetch routes and provider credentials over iroh, then restart Hellas. + Configure { + name: String, + /// Local JSON route configuration, including its caller grants. + #[arg(long)] + fetch_config: PathBuf, + /// Copy a credential from this local environment variable. Repeat as needed. + #[arg(long = "env", value_name = "NAME")] + env: Vec, + /// Upload a private JSON credential file, referenced in config as @files/NAME. + #[arg(long = "file", value_name = "NAME=PATH")] + files: Vec, + }, + Fetch { + name: String, + #[arg(long)] + url: String, + #[arg(long)] + sha256: String, + #[arg(long)] + bytes: u64, + }, + /// Prepare an owner-bound bare-metal agent bootstrap file (mode 0600). + Prepare { + name: String, + #[arg(long)] + bootstrap_file: PathBuf, + #[arg(long)] + admin_addr: Option, + #[arg(last = true)] + serve_args: Vec, + }, + Destroy { + name: String, + }, +} + +impl MachinesArgs { + pub async fn run(self, service: Service) -> Result<()> { + let request = match self.command { + MachineCommand::List => Request::List, + MachineCommand::Status { name } => Request::Status { name }, + MachineCommand::Resolve { name } => Request::Resolve { name }, + MachineCommand::Restart { name } => Request::Restart { name }, + MachineCommand::Configure { + name, + fetch_config, + env, + files, + } => { + let metadata = std::fs::metadata(&fetch_config)?; + anyhow::ensure!( + metadata.is_file() && metadata.len() <= 48 * 1024, + "fetch configuration must be a regular file of at most 48 KiB" + ); + let fetch_config = serde_json::from_slice(&std::fs::read(fetch_config)?) + .map_err(|_| anyhow::anyhow!("invalid fetch configuration JSON"))?; + let env = env + .into_iter() + .map(|name| { + let value = std::env::var(&name).map_err(|_| { + anyhow::anyhow!("credential environment variable is unset or invalid") + })?; + Ok((name, value)) + }) + .collect::>()?; + let files = files + .into_iter() + .map(|argument| { + let (name, path) = argument + .split_once('=') + .ok_or_else(|| anyhow::anyhow!("--file requires NAME=PATH"))?; + let metadata = std::fs::metadata(path)?; + anyhow::ensure!( + metadata.is_file() && metadata.len() <= 48 * 1024, + "credential file must be a regular file of at most 48 KiB" + ); + let value = + serde_json::from_slice(&std::fs::read(path)?).map_err(|_| { + anyhow::anyhow!("invalid credential JSON; contents withheld") + })?; + Ok((name.to_owned(), value)) + }) + .collect::>()?; + let configuration = crate::configuration::Configuration { + fetch_config, + env, + files, + }; + configuration.validate()?; + Request::Configure { + name, + configuration, + } + } + MachineCommand::Fetch { + name, + url, + sha256, + bytes, + } => Request::Fetch { + name, + url, + sha256, + bytes, + }, + MachineCommand::Prepare { + name, + bootstrap_file, + admin_addr, + serve_args, + } => Request::Prepare { + name, + bootstrap_file, + admin_addr, + serve_args, + }, + MachineCommand::Destroy { name } => Request::Destroy { name }, + }; + let result = if let Some(socket) = self.socket { + // A GUI uses the same methods. Reject a socket serving another loaded identity. + let inventory = crate::internal_rpc::call(&socket, Request::List).await?; + anyhow::ensure!( + inventory["owner"] == service.owner(), + "control socket serves another identity" + ); + crate::internal_rpc::call(&socket, request).await? + } else { + service.execute(request).await? + }; + println!("{}", serde_json::to_string_pretty(&result)?); + Ok(()) + } +} + +#[derive(Args)] +pub struct ControlArgs { + #[command(subcommand)] + pub command: ControlCommand, +} + +#[derive(Subcommand)] +pub enum ControlCommand { + /// Serve private JSON-RPC for local applications, using the selected identity. + Serve { + #[arg(long)] + socket: Option, + }, +} + +impl ControlArgs { + pub async fn run(self, service: Service) -> Result<()> { + let ControlCommand::Serve { socket } = self.command; + let socket = socket + .map(Ok) + .unwrap_or_else(|| crate::internal_rpc::default_socket(&service.owner()))?; + eprintln!( + "management owner: {}\nmanagement socket: {}", + service.owner(), + socket.display() + ); + crate::internal_rpc::serve(service, &socket).await + } +} diff --git a/crates/cloud/src/management.rs b/crates/cloud/src/management.rs new file mode 100644 index 00000000..c8e7b843 --- /dev/null +++ b/crates/cloud/src/management.rs @@ -0,0 +1,384 @@ +//! Identity-scoped management shared by the CLI and internal RPC. +use std::{ + net::SocketAddr, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result, bail, ensure}; +use iroh::SecretKey; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::{ + config::{ + Credentials, Deployment, Enrollment, Spec, lock_state, read_json, save_private, + validate_serve_args, + }, + wire::{self, Operation, Response}, +}; + +#[derive(Deserialize, Serialize)] +#[serde(tag = "method", content = "params", deny_unknown_fields)] +pub enum Request { + #[serde(rename = "machines.list")] + List, + #[serde(rename = "machines.status")] + Status { name: String }, + #[serde(rename = "machines.resolve")] + Resolve { name: String }, + #[serde(rename = "machines.restart")] + Restart { name: String }, + #[serde(rename = "machines.configure")] + Configure { + name: String, + configuration: crate::configuration::Configuration, + }, + #[serde(rename = "machines.fetch")] + Fetch { + name: String, + url: String, + sha256: String, + bytes: u64, + }, + #[serde(rename = "machines.prepare")] + Prepare { + name: String, + bootstrap_file: PathBuf, + #[serde(default)] + admin_addr: Option, + #[serde(default)] + serve_args: Vec, + }, + #[serde(rename = "machines.destroy")] + Destroy { name: String }, + #[serde(rename = "cloud.runpod")] + Runpod(crate::cloud::RunpodArgs), +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Machine { + name: String, + owner: String, + source: Source, + enrollment: Option, + last_seen_unix: Option, + running: Option, +} + +#[derive(Deserialize, Serialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +enum Source { + Cloud { + receipt: PathBuf, + }, + BareMetal { + credentials: Credentials, + admin_addr: Option, + }, +} + +pub struct Service { + key: SecretKey, + root: PathBuf, +} + +impl Service { + /// `base` is a trusted local state directory; each identity has its own inventory. + pub fn new(key: SecretKey, base: &Path) -> Result { + let root = base.join(key.public().to_string()); + private_directory(&root)?; + Ok(Self { key, root }) + } + + pub fn open(key: SecretKey) -> Result { + let base = match std::env::var_os("HELLAS_MACHINES_DIR") { + Some(path) => PathBuf::from(path), + None => PathBuf::from(std::env::var_os("HOME").context("HOME is unset")?) + .join(".hellas/machines"), + }; + Self::new(key, &base) + } + + pub fn owner(&self) -> String { + self.key.public().to_string() + } + + fn path(&self, name: &str) -> Result { + crate::accounts::validate_name(name)?; + Ok(self.root.join(format!("{name}.json"))) + } + + fn load(&self, name: &str) -> Result { + let machine: Machine = read_json(&self.path(name)?)?; + ensure!( + machine.name == name && machine.owner == self.owner(), + "machine belongs to another identity" + ); + Ok(machine) + } + + fn credentials(&self, machine: &Machine) -> Result<(Credentials, Option)> { + let (credentials, address) = match &machine.source { + Source::Cloud { receipt } => { + let state: Deployment = read_json(receipt)?; + ensure!(!state.destroyed, "machine was destroyed"); + (state.credentials, None) + } + Source::BareMetal { + credentials, + admin_addr, + } => (credentials.clone(), *admin_addr), + }; + ensure!( + credentials.owner.as_deref() == Some(&self.owner()), + "machine is not bound to this identity" + ); + credentials.secret_key()?; + Ok((credentials, address)) + } + + fn view(&self, machine: &Machine) -> Result { + let (location, lifecycle) = match &machine.source { + Source::Cloud { receipt } if receipt.exists() => { + let state: Deployment = read_json(receipt)?; + ensure!( + state.credentials.owner.as_deref() == Some(&self.owner()), + "receipt owner mismatch" + ); + ( + json!({"provider":state.spec.provider, "resource_id":state.resource_id}), + if state.destroyed { + "destroyed" + } else if state.resource_id.is_some() { + "allocated" + } else { + "pending" + }, + ) + } + Source::Cloud { .. } => (json!({}), "pending"), + Source::BareMetal { .. } => (json!({"provider":{"kind":"bare-metal"}}), "registered"), + }; + Ok( + json!({"name":machine.name, "owner":machine.owner, "location":location, + "lifecycle":lifecycle, "enrollment":machine.enrollment, + "last_seen_unix":machine.last_seen_unix, "last_observed_running":machine.running}), + ) + } + + pub fn list(&self) -> Result { + let mut names = Vec::new(); + for entry in std::fs::read_dir(&self.root)? { + let path = entry?.path(); + if path.extension().is_some_and(|ext| ext == "json") { + names.push( + path.file_stem() + .context("invalid machine filename")? + .to_string_lossy() + .into_owned(), + ); + } + } + names.sort(); + let machines = names + .iter() + .map(|name| self.view(&self.load(name)?)) + .collect::>>()?; + Ok(json!({"owner":self.owner(), "machines":machines})) + } + + pub async fn create(&self, spec: Spec, receipt: Option) -> Result { + spec.validate()?; + let _lock = lock_state(&self.root.join("inventory"))?; + let path = self.path(&spec.name)?; + ensure!(!path.exists(), "machine name already exists"); + let receipt = receipt.unwrap_or_else(|| { + self.root + .join("receipts") + .join(format!("{}.json", spec.name)) + }); + let receipt = std::path::absolute(receipt)?; + ensure!( + !receipt.exists(), + "receipt already exists; allocation was not attempted" + ); + let mut credentials = Credentials::generate(); + credentials.owner = Some(self.owner()); + let machine = Machine { + name: spec.name.clone(), + owner: self.owner(), + source: Source::Cloud { + receipt: receipt.clone(), + }, + enrollment: None, + last_seen_unix: None, + running: None, + }; + // Keep the machine visible even if allocation or the final receipt write fails. + save_private(&path, &machine, true)?; + let result = crate::deployment::create_with_credentials(spec, &receipt, credentials).await; + if result.is_err() && !receipt.exists() { + // No allocation can precede the receipt. A local preflight failure is retryable. + std::fs::remove_file(&path)?; + } + let id = result?; + Ok(json!({"id":id, "state":receipt, "machine":machine.name, "owner":self.owner()})) + } + + fn prepare( + &self, + name: String, + bootstrap_file: PathBuf, + admin_addr: Option, + mut serve_args: Vec, + ) -> Result { + let _lock = lock_state(&self.root.join("inventory"))?; + let path = self.path(&name)?; + ensure!(!path.exists(), "machine name already exists"); + if serve_args.is_empty() { + serve_args = vec!["--execute-policy".into(), "none".into()]; + } + validate_serve_args(&serve_args)?; + let mut credentials = Credentials::generate(); + credentials.owner = Some(self.owner()); + // JSON avoids executable shell snippets; install this private file on the host. + let bootstrap = credentials.env_for_args(&serve_args)?; + save_private(&bootstrap_file, &bootstrap, true)?; + let machine = Machine { + name, + owner: self.owner(), + source: Source::BareMetal { + credentials, + admin_addr, + }, + enrollment: None, + last_seen_unix: None, + running: None, + }; + save_private(&path, &machine, true)?; + self.view(&machine) + } + + async fn admin(&self, name: &str, operation: Operation) -> Result { + let path = self.path(name)?; + let _lock = lock_state(&path)?; + let mut machine = self.load(name)?; + let (credentials, address) = self.credentials(&machine)?; + let response = wire::call_as(&credentials, address, operation, Some(&self.key)).await?; + match &response { + Response::Error { message } => bail!("{message}"), + Response::Status { + enrollment, + owner, + running, + } => { + ensure!( + *owner == credentials.owner, + "agent did not confirm the owner binding; upgrade the companion image" + ); + enrollment.validate()?; + if let Some(expected) = &machine.enrollment { + ensure!( + expected == enrollment, + "machine enrollment changed; refusing to replace its trust anchor" + ); + } else { + ensure!( + *running, + "Hellas is not running; check the image supports serve --owner" + ); + machine.enrollment = Some(enrollment.clone()); + } + machine.running = Some(*running); + machine.last_seen_unix = Some( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_secs(), + ); + save_private(&path, &machine, false)?; + } + _ => {} + } + Ok(serde_json::to_value(response)?) + } + + /// Live owner authentication plus a stable, enrolled execution route. + pub async fn resolve(&self, name: &str) -> Result { + let response = self.admin(name, Operation::Status).await?; + ensure!(response["running"] == true, "machine is not running"); + self.load(name)? + .enrollment + .context("machine has not enrolled") + } + + pub async fn execute(&self, request: Request) -> Result { + match request { + Request::List => self.list(), + Request::Status { name } => self.admin(&name, Operation::Status).await, + Request::Resolve { name } => Ok(serde_json::to_value(self.resolve(&name).await?)?), + Request::Restart { name } => { + self.resolve(&name).await?; + self.admin(&name, Operation::Restart).await + } + Request::Configure { + name, + configuration, + } => { + configuration.validate()?; + // Authenticate and check the pinned enrollment even when a previous + // configuration stopped the child. Configuration must remain repairable. + self.admin(&name, Operation::Status).await?; + self.admin(&name, Operation::Configure { configuration }) + .await + } + Request::Fetch { + name, + url, + sha256, + bytes, + } => { + crate::agent::validate_download(&url, &sha256, bytes)?; + // Confirm the owner-capable protocol before any mutation. + self.resolve(&name).await?; + self.admin(&name, Operation::Fetch { url, sha256, bytes }) + .await + } + Request::Prepare { + name, + bootstrap_file, + admin_addr, + serve_args, + } => self.prepare(name, bootstrap_file, admin_addr, serve_args), + Request::Destroy { name } => { + let path = self.path(&name)?; + let _lock = lock_state(&path)?; + let machine = self.load(&name)?; + self.credentials(&machine)?; + let Source::Cloud { receipt } = machine.source else { + bail!("bare-metal hosts cannot be destroyed through a cloud adapter") + }; + let id = crate::deployment::destroy(&receipt, None, None).await?; + Ok(json!({"id":id,"destroyed":true})) + } + Request::Runpod(args) => args.run_managed(Some(self)).await, + } + } +} + +/// Refuse an existing shared or foreign directory instead of silently changing it. +pub(crate) fn private_directory(path: &Path) -> Result<()> { + use std::os::unix::fs::{DirBuilderExt, MetadataExt, PermissionsExt}; + let mut builder = std::fs::DirBuilder::new(); + builder.recursive(true).mode(0o700); + builder.create(path)?; + let meta = std::fs::symlink_metadata(path)?; + ensure!( + meta.is_dir() + && !meta.file_type().is_symlink() + && meta.uid() == unsafe { libc::geteuid() } + && meta.permissions().mode() & 0o077 == 0, + "management directory must be owned by this user with mode 0700" + ); + Ok(()) +} diff --git a/crates/cloud/src/provider.rs b/crates/cloud/src/provider.rs new file mode 100644 index 00000000..af1ed6b2 --- /dev/null +++ b/crates/cloud/src/provider.rs @@ -0,0 +1,587 @@ +use std::{collections::BTreeMap, time::Duration}; + +use anyhow::{Context, Result, bail, ensure}; +use async_trait::async_trait; +use serde_json::{Value, json}; +use tokio::{io::AsyncWriteExt, process::Command}; + +use crate::config::{Credentials, ProviderConfig, Spec}; + +/// Providers own allocation only; administration and Hellas routing are shared. +/// An additional backend implements these methods and adds one config variant. +#[async_trait] +pub trait Provider: Send + Sync { + /// Pure, credential-free description for review before allocation. + fn plan(&self, spec: &Spec) -> Result; + /// Resolve provider metadata before saving the pending allocation receipt. + async fn prepare(&self, spec: &mut Spec) -> Result<()> { + spec.validate() + } + async fn create(&self, spec: &Spec, credentials: &Credentials) -> Result; + /// Check the allocated resource after its ID is safely recorded. + async fn verify(&self, _spec: &Spec, _id: &str) -> Result<()> { + Ok(()) + } + async fn inspect(&self, id: &str) -> Result; + async fn destroy(&self, id: &str) -> Result<()>; +} + +pub fn adapter(config: &ProviderConfig) -> Result> { + match config { + ProviderConfig::Docker { gpus } => Ok(Box::new(Docker { gpus: *gpus })), + ProviderConfig::Runpod { account, .. } => Ok(Box::new(Cloud::runpod(account.as_deref())?)), + ProviderConfig::Vast { .. } => Ok(Box::new(Cloud::new(CloudKind::Vast)?)), + } +} + +pub struct Docker { + pub gpus: bool, +} + +async fn docker(args: &[&str], input: Option<&[u8]>) -> Result { + let mut command = Command::new("docker"); + command + .args(args) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true); + if input.is_some() { + command.stdin(std::process::Stdio::piped()); + } + let mut child = command.spawn().context("start docker")?; + if let Some(bytes) = input { + child.stdin.take().unwrap().write_all(bytes).await?; + } + let output = tokio::time::timeout(Duration::from_secs(600), child.wait_with_output()).await??; + ensure!( + output.status.success(), + "docker operation failed (details suppressed to avoid leaking bootstrap credentials)" + ); + Ok(String::from_utf8(output.stdout)?.trim().to_owned()) +} + +#[async_trait] +impl Provider for Docker { + fn plan(&self, spec: &Spec) -> Result { + spec.validate()?; + Ok( + json!({"provider":"docker", "name":spec.name, "image":spec.image, + "gpus":self.gpus, "data_volume":format!("{}-data",spec.name), "publish_ports":[], "trust":"token"}), + ) + } + + async fn create(&self, spec: &Spec, credentials: &Credentials) -> Result { + self.plan(spec)?; + let env = credentials + .env(spec)? + .into_iter() + .map(|(k, v)| format!("{k}={v}\n")) + .collect::(); + // Credentials travel over stdin, never in argv or a printed command. + let mut args = vec![ + "run", + "--detach", + "--name", + &spec.name, + "--env-file", + "/dev/stdin", + "--mount", + ]; + let mount = format!( + "type=volume,source={}-data,target=/var/lib/hellas", + spec.name + ); + args.push(&mount); + if self.gpus { + args.extend(["--gpus", "all"]); + } + args.push(&spec.image); + docker(&args, Some(env.as_bytes())).await + } + + async fn inspect(&self, id: &str) -> Result { + validate_id(id)?; + let output = docker(&["inspect", "--format", "{{json .State}}", id], None).await?; + let state: Value = serde_json::from_str(&output)?; + Ok(json!({"id":id, "status":state["Status"], "exit_code":state["ExitCode"]})) + } + + async fn destroy(&self, id: &str) -> Result<()> { + validate_id(id)?; + docker(&["rm", "--force", id], None).await?; + Ok(()) // Deliberately retain the data volume and enrollment identity. + } +} + +#[derive(Clone, Copy)] +pub enum CloudKind { + Runpod, + Vast, +} + +pub struct Cloud { + kind: CloudKind, + client: reqwest::Client, + base: String, + credential: crate::accounts::CredentialSource, +} + +impl Cloud { + pub fn new(kind: CloudKind) -> Result { + Ok(Self { + kind, + client: reqwest::Client::builder() + .user_agent(concat!("hellas-cloud/", env!("CARGO_PKG_VERSION"))) + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(60)) + .build()?, + base: match kind { + CloudKind::Runpod => "https://rest.runpod.io/v1", + CloudKind::Vast => "https://console.vast.ai/api/v0", + } + .into(), + credential: crate::accounts::CredentialSource::Env( + match kind { + CloudKind::Runpod => "RUNPOD_API_KEY", + CloudKind::Vast => "VAST_API_KEY", + } + .into(), + ), + }) + } + + pub fn runpod(account: Option<&str>) -> Result { + let mut client = Self::new(CloudKind::Runpod)?; + client.credential = crate::accounts::runpod(account)?; + Ok(client) + } + + pub async fn list(&self) -> Result { + ensure!( + matches!(self.kind, CloudKind::Runpod), + "listing is not implemented for this provider" + ); + let value = self.request(reqwest::Method::GET, "/pods", None).await?; + let pods = value.as_array().context("invalid pod list")?; + Ok(Value::Array(pods.iter().map(runpod_summary).collect())) + } + + async fn request( + &self, + method: reqwest::Method, + path: &str, + body: Option, + ) -> Result { + let key = self.credential.token().await.map_err(|error| { + if matches!(self.kind, CloudKind::Runpod) + && error.is::() + { + error.context(crate::accounts::RunpodAccountRequired) + } else { + error + } + })?; + let mut request = self + .client + .request(method, format!("{}{path}", self.base)) + .bearer_auth(key); + if let Some(body) = body { + request = request.json(&body); + } + // Do not retry allocation: a lost response can hide a billable resource. + let mut response = request.send().await.context( + "provider request failed; reconcile resource by name before retrying allocation", + )?; + let status = response.status(); + ensure!( + status.is_success(), + "provider returned HTTP {status}; response body withheld (may contain credentials)" + ); + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await? { + ensure!( + bytes.len() + chunk.len() <= 1024 * 1024, + "provider response too large" + ); + bytes.extend_from_slice(&chunk); + } + if bytes.is_empty() { + return Ok(Value::Null); + } + let value: Value = serde_json::from_slice(&bytes).context("invalid provider JSON")?; + ensure!( + value.get("success") != Some(&Value::Bool(false)), + "provider refused operation; inspect provider console" + ); + Ok(value) + } + + pub fn create_body(&self, spec: &Spec, env: BTreeMap) -> Result { + spec.validate()?; + match (&self.kind, &spec.provider) { + ( + CloudKind::Runpod, + ProviderConfig::Runpod { + template_id, + gpu_type, + interruptible, + .. + }, + ) => { + let template_id = template_id + .as_deref() + .context("Runpod creation requires a template")?; + validate_id(template_id)?; + // Inherit image, disks, registry auth, ports and startup settings. + // Cloning a template into an ad-hoc image request loses attribution. + Ok(json!({ + "name":spec.name, "templateId":template_id, + "computeType":"GPU", "cloudType":"SECURE", + "gpuTypeIds":[gpu_type], "gpuCount":1, + "interruptible":interruptible, "env":env + })) + } + (CloudKind::Vast, ProviderConfig::Vast { disk_gb, .. }) => { + // Values are generated hex only, never arbitrary shell fragments. + ensure!( + env.iter().all( + |(k, v)| k.bytes().all(|b| b.is_ascii_uppercase() || b == b'_') + && v.bytes().all(|b| b.is_ascii_hexdigit()) + ), + "invalid bootstrap environment" + ); + Ok( + json!({"label":spec.name, "image":spec.image, "disk":disk_gb, + "runtype":"args", "args":[], "target_state":"running", "cancel_unavail":true, + "env":env.into_iter().map(|(k,v)|format!("-e {k}={v}")).collect::>().join(" ")}), + ) + } + _ => bail!("provider/spec mismatch"), + } + } +} + +#[async_trait] +impl Provider for Cloud { + fn plan(&self, spec: &Spec) -> Result { + self.create_body(spec, BTreeMap::new()) + } + + async fn prepare(&self, spec: &mut Spec) -> Result<()> { + if let ProviderConfig::Runpod { template_id, .. } = &spec.provider { + let id = template_id + .as_deref() + .context("Runpod creation requires a template")?; + validate_id(id)?; + let template = self + .request(reqwest::Method::GET, &format!("/templates/{id}"), None) + .await?; + resolve_runpod_template(spec, &template)?; + } + spec.validate() + } + + async fn verify(&self, spec: &Spec, id: &str) -> Result<()> { + if let ProviderConfig::Runpod { + template_id: Some(template_id), + .. + } = &spec.provider + { + let pod = self.inspect(id).await?; + ensure!( + pod["template_id"].as_str() == Some(template_id), + "Runpod did not retain the requested template attribution" + ); + ensure!( + pod["image"].as_str() == Some(&spec.image), + "Runpod image differs from the resolved template; template may have changed" + ); + } + Ok(()) + } + + async fn create(&self, spec: &Spec, credentials: &Credentials) -> Result { + let body = self.create_body(spec, credentials.env(spec)?)?; + let (method, path) = match spec.provider { + ProviderConfig::Runpod { .. } => (reqwest::Method::POST, "/pods".to_owned()), + ProviderConfig::Vast { offer_id, .. } => { + (reqwest::Method::PUT, format!("/asks/{offer_id}/")) + } + _ => bail!("provider/spec mismatch"), + }; + let response = self.request(method, &path, Some(body)).await?; + let id = match self.kind { + CloudKind::Runpod => response["id"] + .as_str() + .context("missing pod ID; reconcile by name in provider console")? + .to_owned(), + CloudKind::Vast => response["new_contract"] + .as_u64() + .context("missing instance ID; reconcile by name in provider console")? + .to_string(), + }; + validate_id(&id)?; + Ok(id) + } + + async fn inspect(&self, id: &str) -> Result { + validate_id(id)?; + let path = match self.kind { + CloudKind::Runpod => format!("/pods/{id}"), + CloudKind::Vast => format!("/instances/{id}/"), + }; + let value = self.request(reqwest::Method::GET, &path, None).await?; + // Provider responses echo env; return only the fields we intend to show. + Ok(match self.kind { + CloudKind::Runpod => runpod_summary(&value), + CloudKind::Vast => { + json!({"id":id, "status":value["instances"]["actual_status"], "image":value["instances"]["image_uuid"]}) + } + }) + } + + async fn destroy(&self, id: &str) -> Result<()> { + validate_id(id)?; + let path = match self.kind { + CloudKind::Runpod => format!("/pods/{id}"), + CloudKind::Vast => format!("/instances/{id}/"), + }; + self.request(reqwest::Method::DELETE, &path, None).await?; + Ok(()) + } +} + +fn resolve_runpod_template(spec: &mut Spec, template: &Value) -> Result<()> { + let ProviderConfig::Runpod { + template_id, + disk_gb, + volume_gb, + .. + } = &mut spec.provider + else { + bail!("template resolution requires Runpod"); + }; + ensure!( + template["id"].as_str() == template_id.as_deref(), + "unexpected Runpod template" + ); + ensure!( + template["isServerless"] != true, + "Hellas requires a Pod template" + ); + ensure!( + template["volumeMountPath"] == "/var/lib/hellas", + "template must mount persistent worker data at /var/lib/hellas" + ); + let image = template["imageName"] + .as_str() + .context("template has no image")?; + ensure!( + spec.image.is_empty() || spec.image == image, + "template image changed from the requested digest" + ); + spec.image = image.to_owned(); + *disk_gb = template["containerDiskInGb"] + .as_u64() + .and_then(|v| u32::try_from(v).ok()) + .context("invalid template container disk size")?; + *volume_gb = template["volumeInGb"] + .as_u64() + .and_then(|v| u32::try_from(v).ok()) + .context("invalid template volume size")?; + // Only non-secret deployment metadata is copied into the receipt. + spec.validate() +} + +/// Provider responses also contain environment secrets: project an allowlist. +fn runpod_summary(value: &Value) -> Value { + json!({"id":value["id"], "name":value["name"], + "desired_status":value["desiredStatus"], "image":value.get("imageName").filter(|v| v.is_string()).unwrap_or(&value["image"]), + "template_id":value["templateId"], + "gpu_count":value["gpuCount"], "hourly_rate":value["costPerHr"], + "interruptible":value["interruptible"], + "disk_gb":value["containerDiskInGb"], "volume_gb":value["volumeInGb"]}) +} + +pub fn validate_id(id: &str) -> Result<()> { + ensure!( + !id.is_empty() + && id.len() <= 128 + && id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-'), + "invalid provider resource ID" + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read, Write}; + + fn template_spec() -> Spec { + Spec { + name: "test-worker".into(), + image: String::new(), + provider: ProviderConfig::Runpod { + account: None, + template_id: Some("foundation-template".into()), + gpu_type: "NVIDIA L4".into(), + interruptible: true, + disk_gb: 0, + volume_gb: 0, + container_registry_auth_id: None, + }, + trust: crate::config::Trust::Token, + serve_args: vec![], + } + } + + fn template() -> Value { + json!({"id":"foundation-template", "imageName":format!("registry/image@sha256:{}", "a".repeat(64)), + "containerDiskInGb":20, "volumeInGb":4, "volumeMountPath":"/var/lib/hellas", + "env":{"UPSTREAM_KEY":"must-not-enter-receipt"}}) + } + + #[test] + fn template_resolution_requires_pinned_image_and_preserves_only_metadata() { + let mut spec = template_spec(); + resolve_runpod_template(&mut spec, &template()).unwrap(); + assert_eq!(spec.image, template()["imageName"]); + assert!( + !serde_json::to_string(&spec) + .unwrap() + .contains("must-not-enter-receipt") + ); + for (field, value) in [ + ("id", json!("another-template")), + ("imageName", json!("registry/image:latest")), + ("volumeMountPath", json!("/workspace")), + ("volumeInGb", json!(0)), + ("isServerless", json!(true)), + ] { + let mut invalid = template(); + invalid[field] = value; + assert!(resolve_runpod_template(&mut template_spec(), &invalid).is_err()); + } + } + + #[tokio::test] + async fn template_api_preserves_attribution_and_detects_missing_attribution() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + for index in 0..4 { + let (mut socket, _) = listener.accept().unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + let mut bytes = Vec::new(); + while !bytes.ends_with(b"\r\n\r\n") { + let mut byte = [0]; + socket.read_exact(&mut byte).unwrap(); + bytes.push(byte[0]); + } + let headers = String::from_utf8(bytes).unwrap(); + let response = match index { + 0 => { + assert!(headers.starts_with("GET /templates/foundation-template ")); + template() + } + 1 => { + assert!(headers.starts_with("POST /pods ")); + let length: usize = headers.lines().find_map(|line| line.to_lowercase() + .strip_prefix("content-length: ").map(str::parse)).unwrap().unwrap(); + let mut body = vec![0; length]; + socket.read_exact(&mut body).unwrap(); + let body: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(body["templateId"], "foundation-template"); + assert_eq!(body["interruptible"], true); + assert!(body["env"]["HELLAS_REMOTE_TOKEN"].is_string()); + assert!(body.get("imageName").is_none()); + assert!(body.get("volumeMountPath").is_none()); + json!({"id":"test-pod"}) + } + _ => { + assert!(headers.starts_with("GET /pods/test-pod ")); + // Live REST responses use `image`, not always `imageName`. + json!({"id":"test-pod", "templateId":if index == 2 { json!("foundation-template") } else { Value::Null }, + "image":template()["imageName"], "env":{"secret":"do-not-leak"}}) + } + }.to_string(); + write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", response.len(), response).unwrap(); + } + }); + let mut cloud = Cloud::new(CloudKind::Runpod).unwrap(); + cloud.base = format!("http://{address}"); + cloud.credential = + crate::accounts::CredentialSource::Command(vec!["printf".into(), "fixture".into()]); + let mut spec = template_spec(); + cloud.prepare(&mut spec).await.unwrap(); + let id = cloud.create(&spec, &Credentials::generate()).await.unwrap(); + cloud.verify(&spec, &id).await.unwrap(); + assert!( + cloud + .verify(&spec, &id) + .await + .unwrap_err() + .to_string() + .contains("attribution") + ); + server.join().unwrap(); + } + + #[tokio::test] + async fn concurrent_accounts_keep_their_own_tokens_and_redact_pod_environment() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + for _ in 0..2 { + let (mut socket, _) = listener.accept().unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + let mut bytes = Vec::new(); + while !bytes.ends_with(b"\r\n\r\n") { + let mut byte = [0]; + socket.read_exact(&mut byte).unwrap(); + bytes.push(byte[0]); + } + let request = String::from_utf8(bytes).unwrap(); + let (id, list) = if request.starts_with("GET /pods ") { + assert!( + request + .to_lowercase() + .contains("authorization: bearer account-a") + ); + ("pod-a", true) + } else { + assert!(request.starts_with("GET /pods/pod-b ")); + assert!( + request + .to_lowercase() + .contains("authorization: bearer account-b") + ); + ("pod-b", false) + }; + let pod = json!({"id":id, "desiredStatus":"RUNNING", "imageName":"pinned-image", + "env":{"HELLAS_REMOTE_TOKEN":"do-not-leak"}, "registryPassword":"do-not-leak"}); + let body = if list { json!([pod]) } else { pod }.to_string(); + write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body).unwrap(); + } + }); + let mut a = Cloud::new(CloudKind::Runpod).unwrap(); + let mut b = Cloud::new(CloudKind::Runpod).unwrap(); + for (client, token) in [(&mut a, "account-a"), (&mut b, "account-b")] { + client.base = format!("http://{address}"); + client.credential = + crate::accounts::CredentialSource::Command(vec!["printf".into(), token.into()]); + } + let (a, b) = tokio::join!(a.list(), b.inspect("pod-b")); + let (a, b) = (a.unwrap(), b.unwrap()); + assert_eq!(a[0]["id"], "pod-a"); + assert_eq!(b["id"], "pod-b"); + assert_eq!(b["image"], "pinned-image"); + assert!(!format!("{a}{b}").contains("do-not-leak")); + server.join().unwrap(); + } +} diff --git a/crates/cloud/src/wire.rs b/crates/cloud/src/wire.rs new file mode 100644 index 00000000..bd550ba7 --- /dev/null +++ b/crates/cloud/src/wire.rs @@ -0,0 +1,99 @@ +use std::{net::SocketAddr, time::Duration}; + +use anyhow::{Result, ensure}; +use iroh::{Endpoint, EndpointAddr, endpoint::presets}; +use serde::{Deserialize, Serialize}; + +use crate::config::{Credentials, Enrollment}; + +pub const ALPN: &[u8] = b"hellas-extras/admin/1"; +pub const MAX_MESSAGE: usize = 64 * 1024; + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Request { + pub token: String, + pub operation: Operation, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(tag = "op", rename_all = "kebab-case", deny_unknown_fields)] +pub enum Operation { + Status, + Restart, + Configure { + configuration: crate::configuration::Configuration, + }, + Fetch { + url: String, + sha256: String, + bytes: u64, + }, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] +pub enum Response { + Status { + enrollment: Enrollment, + #[serde(default)] + owner: Option, + running: bool, + }, + Fetched { + sha256: String, + bytes: u64, + }, + Error { + message: String, + }, +} + +pub async fn call( + credentials: &Credentials, + address: Option, + operation: Operation, +) -> Result { + call_as(credentials, address, operation, None).await +} + +pub async fn call_as( + credentials: &Credentials, + address: Option, + operation: Operation, + owner_key: Option<&iroh::SecretKey>, +) -> Result { + if let Some(owner) = &credentials.owner { + ensure!( + owner_key.is_some_and(|key| key.public().to_string() == *owner), + "machine belongs to another identity; owner key required" + ); + } + let mut builder = Endpoint::builder(presets::N0); + if let Some(key) = owner_key { + builder = builder.secret_key(key.clone()); + } + let endpoint = builder.bind().await?; + let mut addr = EndpointAddr::from(credentials.secret_key()?.public()); + if let Some(address) = address { + addr = addr.with_ip_addr(address); + } + let result = tokio::time::timeout(Duration::from_secs(3600), async { + let connection = + tokio::time::timeout(Duration::from_secs(30), endpoint.connect(addr, ALPN)).await??; + let (mut send, mut recv) = connection.open_bi().await?; + let request = serde_json::to_vec(&Request { + token: credentials.token.clone(), + operation, + })?; + ensure!(request.len() <= MAX_MESSAGE, "request too large"); + send.write_all(&request).await?; + send.finish()?; + let response = serde_json::from_slice(&recv.read_to_end(MAX_MESSAGE).await?)?; + connection.close(0u32.into(), b"done"); + Ok::<_, anyhow::Error>(response) + }) + .await; + endpoint.close().await; + result? +} diff --git a/crates/cloud/tests/cloud.rs b/crates/cloud/tests/cloud.rs new file mode 100644 index 00000000..50700e8b --- /dev/null +++ b/crates/cloud/tests/cloud.rs @@ -0,0 +1,180 @@ +#![cfg(unix)] + +use clap::{Parser, Subcommand}; +use hellas_cloud::{ + cloud::{CloudArgs, CloudCommand, RunpodArgs, RunpodCommand}, + config::{Credentials, Deployment, ProviderConfig, Spec, Trust, read_json, save_state}, + deployment, +}; + +#[derive(Parser)] +struct Cli { + #[command(subcommand)] + command: Command, +} +#[derive(Subcommand)] +enum Command { + Cloud(CloudArgs), +} + +fn receipt() -> Deployment { + Deployment { + spec: Spec { + name: "test".into(), + image: format!("registry/image@sha256:{}", "a".repeat(64)), + provider: ProviderConfig::Runpod { + account: Some("work".into()), + template_id: Some("foundation-template".into()), + gpu_type: "NVIDIA L4".into(), + interruptible: false, + disk_gb: 20, + volume_gb: 20, + container_registry_auth_id: None, + }, + trust: Trust::Token, + serve_args: vec![], + }, + credentials: Credentials::generate(), + resource_id: Some("test-pod".into()), + enrollment: None, + destroyed: false, + } +} + +#[test] +fn command_contract_requires_info_target_and_explicit_destroy_target() { + assert!(Cli::try_parse_from(["hellas", "cloud", "runpod", "info"]).is_err()); + assert!(Cli::try_parse_from(["hellas", "cloud", "runpod", "destroy"]).is_err()); + let cli = Cli::try_parse_from([ + "hellas", + "cloud", + "runpod", + "info", + "pod-123", + "--account", + "work", + ]) + .unwrap(); + let Command::Cloud(CloudArgs { + provider: CloudCommand::Runpod(args), + .. + }) = cli.command; + assert_eq!(args.account.as_deref(), Some("work")); + assert!(matches!(args.command, RunpodCommand::Info { pod_id } if pod_id == "pod-123")); +} + +#[tokio::test] +async fn dry_run_needs_no_profile_credentials_or_receipt() { + for interruptible in [false, true] { + let mut argv = vec![ + "hellas", + "cloud", + "runpod", + "--account", + "unconfigured", + "create", + "--name", + "trial", + "--template", + "foundation-template", + "--gpu", + "NVIDIA L4", + "--dry-run", + ]; + if interruptible { + argv.push("--interruptible"); + } + let cli = Cli::try_parse_from(argv).unwrap(); + let Command::Cloud(CloudArgs { + provider: CloudCommand::Runpod(args), + .. + }) = cli.command; + // The control socket serializes the same command before dispatching it. + let mut rpc = serde_json::to_value(args).unwrap(); + assert_eq!(rpc["command"]["interruptible"], interruptible); + if !interruptible { + // Clients predating the flag still request on-demand pods. + rpc["command"] + .as_object_mut() + .unwrap() + .remove("interruptible"); + } + let value = serde_json::from_value::(rpc) + .unwrap() + .run() + .await + .unwrap(); + assert_eq!(value["account"], "unconfigured"); + assert_eq!(value["result"]["env"], serde_json::json!({})); + assert_eq!(value["result"]["interruptible"], interruptible); + assert_eq!(value["result"]["templateId"], "foundation-template"); + assert!(value["result"].get("imageName").is_none()); + } +} + +#[test] +fn receipts_preserve_spot_choice_and_default_old_receipts_to_on_demand() { + let mut state = receipt(); + let mut old = serde_json::to_value(&state).unwrap(); + old["spec"]["provider"] + .as_object_mut() + .unwrap() + .remove("interruptible"); + let old: Deployment = serde_json::from_value(old).unwrap(); + assert_eq!(old.spec.provider, state.spec.provider); + + if let ProviderConfig::Runpod { interruptible, .. } = &mut state.spec.provider { + *interruptible = true; + } + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("receipt.json"); + save_state(&path, &state, true).unwrap(); + let saved: Deployment = read_json(&path).unwrap(); + assert_eq!(saved.spec.provider, state.spec.provider); +} + +#[tokio::test] +async fn conflicting_account_or_pod_cannot_destroy_or_modify_a_receipt() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("receipt.json"); + let state = receipt(); + save_state(&path, &state, true).unwrap(); + let before = std::fs::read(&path).unwrap(); + let args = RunpodArgs { + account: Some("personal".into()), + command: RunpodCommand::Destroy { + pod_id: None, + state: Some(path.clone()), + }, + }; + assert!( + args.run() + .await + .unwrap_err() + .to_string() + .contains("--account does not match") + ); + let error = deployment::destroy(&path, None, Some("different-pod")) + .await + .unwrap_err(); + assert!(error.to_string().contains("pod ID does not match")); + assert_eq!(before, std::fs::read(&path).unwrap()); + let reread: Deployment = read_json(&path).unwrap(); + assert!(!reread.destroyed); + assert_eq!(reread.spec.provider, state.spec.provider); +} + +#[tokio::test] +async fn duplicate_create_keeps_the_original_receipt_before_contacting_provider() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("receipt.json"); + let mut state = receipt(); + if let ProviderConfig::Runpod { account, .. } = &mut state.spec.provider { + *account = None; + } + save_state(&path, &state, true).unwrap(); + let before = std::fs::read(&path).unwrap(); + let error = deployment::create(state.spec, &path).await.unwrap_err(); + assert!(error.to_string().contains("allocation was not attempted")); + assert_eq!(before, std::fs::read(&path).unwrap()); +} diff --git a/crates/cloud/tests/configuration.rs b/crates/cloud/tests/configuration.rs new file mode 100644 index 00000000..1c3e7728 --- /dev/null +++ b/crates/cloud/tests/configuration.rs @@ -0,0 +1,45 @@ +#![cfg(unix)] +use hellas_cloud::configuration::Configuration; +use serde_json::json; + +#[test] +fn configuration_bounds_credentials_and_withholds_debug_contents() { + let mut config = Configuration { + fetch_config: json!({"routes": [], "callers": []}), + env: [("UPSTREAM_API_KEY".into(), "test-private-value".into())].into(), + files: Default::default(), + }; + config.validate().unwrap(); + assert!(!format!("{config:?}").contains("test-private-value")); + for name in [ + "", + "1KEY", + "BAD-NAME", + "HOME", + "PATH", + "LD_PRELOAD", + "HELLAS_REMOTE_OWNER", + ] { + config.env = [(name.into(), "test-private-value".into())].into(); + assert!(config.validate().is_err()); + } + config.env = [("API_KEY".into(), "x".repeat(48 * 1024))].into(); + assert!(config.validate().is_err()); + config.env.clear(); + config.env.insert("API_KEY".into(), "invalid\0value".into()); + assert!(config.validate().is_err()); + config.env.clear(); + for name in [ + "../auth.json", + "/auth.json", + ".hidden", + "files/auth.json", + "", + ] { + config.files = [(name.into(), json!({"token":"fixture"}))].into(); + assert!(config.validate().is_err()); + } + config.files.clear(); + config.fetch_config = json!({"routes": []}); + assert!(config.validate().is_err()); +} diff --git a/crates/cloud/tests/contracts.rs b/crates/cloud/tests/contracts.rs new file mode 100644 index 00000000..78db7fb7 --- /dev/null +++ b/crates/cloud/tests/contracts.rs @@ -0,0 +1,152 @@ +#![cfg(unix)] + +use hellas_cloud::{ + agent::validate_download, + config::*, + provider::{Cloud, CloudKind, Docker, Provider}, +}; + +fn spec(provider: ProviderConfig) -> Spec { + Spec { + name: "test-worker".into(), + image: format!("ghcr.io/hellas-ai/hellas@sha256:{}", "a".repeat(64)), + provider, + trust: Trust::Token, + serve_args: vec![], + } +} + +#[test] +fn reject_mutable_images_unsupported_trust_and_identity_overrides() { + let mut value = spec(ProviderConfig::Docker { gpus: false }); + value.validate().unwrap(); + value.image = "ghcr.io/hellas-ai/hellas:cuda".into(); + assert!(value.validate().is_err()); + value = spec(ProviderConfig::Docker { gpus: false }); + value.trust = Trust::MeasuredBoot; + assert!( + Docker { gpus: false } + .plan(&value) + .unwrap_err() + .to_string() + .contains("verifier") + ); + value.trust = Trust::Token; + for arg in [ + "--identity=/tmp/stolen", + "--owner=other", + "--assurance=apple-app-attest", + "--software-root", + "--help", + ] { + value.serve_args = vec![arg.into()]; + assert!(value.validate().is_err()); + } +} + +#[test] +fn runpod_preserves_entrypoint_and_mounts_persistent_identity() { + let spec = spec(ProviderConfig::Runpod { + account: None, + template_id: Some("foundation-template".into()), + gpu_type: "NVIDIA A100 80GB PCIe".into(), + interruptible: false, + disk_gb: 20, + volume_gb: 80, + container_registry_auth_id: Some("registry-credential-id".into()), + }); + let provider = Cloud::new(CloudKind::Runpod).unwrap(); + let mut credentials = Credentials::generate(); + credentials.owner = Some(iroh::SecretKey::generate().public().to_string()); + let body = provider + .create_body(&spec, credentials.env(&spec).unwrap()) + .unwrap(); + assert_eq!(body["templateId"], "foundation-template"); + for key in [ + "imageName", + "volumeMountPath", + "containerDiskInGb", + "volumeInGb", + "containerRegistryAuthId", + "ports", + ] { + assert!( + body.get(key).is_none(), + "template setting overridden: {key}" + ); + } + assert_eq!(body["env"]["HELLAS_REMOTE_TOKEN"], credentials.token); + assert_eq!( + body["env"]["HELLAS_REMOTE_OWNER"], + credentials.owner.unwrap() + ); + assert!(body.get("dockerEntrypoint").is_none()); + assert!(body.get("dockerStartCmd").is_none()); + let plan = provider.plan(&spec).unwrap().to_string(); + assert!(!plan.contains(&credentials.token)); + assert!(!plan.contains(&credentials.admin_secret)); +} + +#[test] +fn vast_uses_args_mode_and_encodes_bootstrap_without_shell_quoting() { + let mut spec = spec(ProviderConfig::Vast { + offer_id: 123, + disk_gb: 50, + }); + spec.serve_args = vec!["--execute-policy".into(), "only(ab*)".into()]; + let provider = Cloud::new(CloudKind::Vast).unwrap(); + let env = Credentials::generate().env(&spec).unwrap(); + let body = provider.create_body(&spec, env.clone()).unwrap(); + assert_eq!(body["runtype"], "args"); + assert_eq!(body["args"], serde_json::json!([])); + assert!(body.get("onstart").is_none()); + assert!(!body["env"].as_str().unwrap().contains('(')); + let decoded: Vec = + serde_json::from_slice(&hex::decode(&env["HELLAS_REMOTE_ARGS"]).unwrap()).unwrap(); + assert_eq!(decoded, spec.serve_args); +} + +#[test] +fn state_is_private_and_cannot_accidentally_reallocate() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("receipt.json"); + let state = Deployment { + spec: spec(ProviderConfig::Docker { gpus: false }), + credentials: Credentials::generate(), + resource_id: None, + enrollment: None, + destroyed: false, + }; + save_state(&path, &state, true).unwrap(); + let lock = lock_state(&path).unwrap(); + assert!(lock_state(&path).is_err()); + drop(lock); + assert!(lock_state(&path).is_ok()); + assert!(save_state(&path, &state, true).is_err()); + let read: Deployment = read_json(&path).unwrap(); + assert_eq!(read.credentials.token, state.credentials.token); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } +} + +#[test] +fn download_urls_and_sizes_are_explicit() { + let hash = "a".repeat(64); + assert!(validate_download("https://example.com/object", &hash, 1).is_ok()); + for url in [ + "http://example.com/a", + "file:///etc/passwd", + "https://u:p@example.com/a", + "https://example.com/a#b", + ] { + assert!(validate_download(url, &hash, 1).is_err()); + } + assert!(validate_download("https://example.com/a", "../escape", 1).is_err()); + assert!(validate_download("https://example.com/a", &hash, 0).is_err()); +} diff --git a/crates/cloud/tests/management.rs b/crates/cloud/tests/management.rs new file mode 100644 index 00000000..38182644 --- /dev/null +++ b/crates/cloud/tests/management.rs @@ -0,0 +1,292 @@ +#![cfg(unix)] + +use hellas_cloud::{ + config::read_json, + internal_rpc, + management::{Request, Service}, +}; +use serde_json::json; +use std::{ + os::unix::fs::{MetadataExt, PermissionsExt}, + time::Duration, +}; +use tokio::{ + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, + net::UnixStream, +}; + +#[tokio::test] +async fn inventory_is_scoped_to_identity_and_never_returns_bootstrap_secrets() { + let dir = tempfile::tempdir().unwrap(); + let key = iroh::SecretKey::generate(); + let a = Service::new(key.clone(), dir.path()).unwrap(); + let b = Service::new(iroh::SecretKey::generate(), dir.path()).unwrap(); + let bootstrap = dir.path().join("bootstrap.json"); + a.execute(Request::Prepare { + name: "metal".into(), + bootstrap_file: bootstrap.clone(), + admin_addr: None, + serve_args: vec![], + }) + .await + .unwrap(); + let env: serde_json::Value = read_json(&bootstrap).unwrap(); + assert_eq!(env["HELLAS_REMOTE_OWNER"], key.public().to_string()); + assert_eq!(std::fs::metadata(&bootstrap).unwrap().mode() & 0o777, 0o600); + let visible = a.list().unwrap(); + assert_eq!(visible["machines"][0]["name"], "metal"); + assert_eq!( + visible["machines"][0]["enrollment"], + serde_json::Value::Null + ); + assert!( + !visible + .to_string() + .contains(env["HELLAS_REMOTE_TOKEN"].as_str().unwrap()) + ); + assert!( + !visible + .to_string() + .contains(env["HELLAS_REMOTE_KEY"].as_str().unwrap()) + ); + assert!(b.list().unwrap()["machines"].as_array().unwrap().is_empty()); + assert!(b.resolve("metal").await.is_err()); + assert!( + a.execute(Request::Destroy { + name: "metal".into() + }) + .await + .is_err() + ); + assert!( + a.execute(Request::Prepare { + name: "../escape".into(), + bootstrap_file: dir.path().join("bad"), + admin_addr: None, + serve_args: vec![] + }) + .await + .is_err() + ); + // Restarting the local service keeps inventory scoped to the same identity. + assert_eq!( + visible, + Service::new(key, dir.path()).unwrap().list().unwrap() + ); +} + +#[tokio::test] +async fn rpc_handles_real_frames_and_shares_the_cli_service() { + let dir = tempfile::tempdir().unwrap(); + let key = iroh::SecretKey::generate(); + let service = Service::new(key.clone(), &dir.path().join("inventory")).unwrap(); + let socket = dir.path().join("rpc/control.sock"); + let server_socket = socket.clone(); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + internal_rpc::serve_until(service, &server_socket, async { + let _ = stopped.await; + }) + .await + }); + for _ in 0..100 { + if socket.exists() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(std::fs::metadata(&socket).unwrap().mode() & 0o777, 0o600); + let bootstrap = dir.path().join("bootstrap.json"); + internal_rpc::call( + &socket, + Request::Prepare { + name: "metal".into(), + bootstrap_file: bootstrap.clone(), + admin_addr: None, + serve_args: vec![], + }, + ) + .await + .unwrap(); + let inventory = internal_rpc::call(&socket, Request::List).await.unwrap(); + assert_eq!(inventory["owner"], key.public().to_string()); + assert_eq!(inventory["machines"][0]["name"], "metal"); + let mut stream = BufReader::new(UnixStream::connect(&socket).await.unwrap()); + for (request, code) in [ + ("not-json\n".to_owned(), -32700), + ( + format!( + "{}\n", + json!({"jsonrpc":"2.0","id":"test","method":"unknown"}) + ), + -32601, + ), + ( + format!( + "{}\n", + json!({"jsonrpc":"2.0","id":"test","method":"machines.status","params":{"name":3}}) + ), + -32602, + ), + ( + format!( + "{}\n", + json!({"jsonrpc":"2.0","id":"test","method":"machines.prepare","params":{"name":"metal","bootstrap_file":bootstrap}}) + ), + -32000, + ), + ] { + stream + .get_mut() + .write_all(request.as_bytes()) + .await + .unwrap(); + let mut response = String::new(); + stream.read_line(&mut response).await.unwrap(); + let response: serde_json::Value = serde_json::from_str(&response).unwrap(); + assert_eq!(response["error"]["code"], code); + assert!(!response.to_string().contains("HELLAS_REMOTE_TOKEN")); + } + drop(stream); + // A second server cannot steal an active socket. + let duplicate = Service::new(key, &dir.path().join("inventory")).unwrap(); + assert!( + internal_rpc::serve_until(duplicate, &socket, std::future::ready(())) + .await + .is_err() + ); + assert!(internal_rpc::call(&socket, Request::List).await.is_ok()); + stop.send(()).unwrap(); + task.await.unwrap().unwrap(); + assert!(!socket.exists()); +} + +#[tokio::test] +async fn rpc_refuses_a_shared_socket_directory_or_existing_regular_file() { + let dir = tempfile::tempdir().unwrap(); + let socket_dir = dir.path().join("shared"); + std::fs::create_dir(&socket_dir).unwrap(); + std::fs::set_permissions(&socket_dir, std::fs::Permissions::from_mode(0o755)).unwrap(); + let service = Service::new(iroh::SecretKey::generate(), &dir.path().join("inventory")).unwrap(); + assert!( + internal_rpc::serve_until( + service, + &socket_dir.join("control.sock"), + std::future::ready(()) + ) + .await + .is_err() + ); + std::fs::set_permissions(&socket_dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + let socket = socket_dir.join("control.sock"); + std::fs::write(&socket, "keep me").unwrap(); + let service = Service::new(iroh::SecretKey::generate(), &dir.path().join("inventory")).unwrap(); + assert!( + internal_rpc::serve_until(service, &socket, std::future::ready(())) + .await + .is_err() + ); + assert_eq!(std::fs::read_to_string(&socket).unwrap(), "keep me"); +} + +#[tokio::test] +async fn enrollment_requires_owner_confirmation_and_cannot_be_silently_replaced() { + use hellas_cloud::{ + config::{Credentials, Enrollment}, + wire::{self, Operation, Response}, + }; + use iroh::{Endpoint, endpoint::presets}; + let dir = tempfile::tempdir().unwrap(); + let key = iroh::SecretKey::generate(); + let owner = key.public().to_string(); + let service = Service::new(key, dir.path()).unwrap(); + let credentials = Credentials::generate(); + let endpoint = Endpoint::builder(presets::N0) + .secret_key(credentials.secret_key().unwrap()) + .relay_mode(iroh::RelayMode::Disabled) + .alpns(vec![wire::ALPN.to_vec()]) + .bind_addr("127.0.0.1:0".parse::().unwrap()) + .unwrap() + .bind() + .await + .unwrap(); + let address = endpoint + .bound_sockets() + .into_iter() + .find(|addr| addr.is_ipv4()) + .unwrap(); + let bootstrap = dir.path().join("bootstrap.json"); + service + .execute(Request::Prepare { + name: "metal".into(), + bootstrap_file: bootstrap, + admin_addr: Some(address), + serve_args: vec![], + }) + .await + .unwrap(); + // Replace only the test server bootstrap key so this local mock is the pinned agent. + let record = dir.path().join(&owner).join("metal.json"); + let mut state: serde_json::Value = read_json(&record).unwrap(); + state["source"]["credentials"]["admin_secret"] = json!(credentials.admin_secret); + std::fs::write(&record, serde_json::to_vec(&state).unwrap()).unwrap(); + let first = Enrollment { + node_id: iroh::SecretKey::generate().public().to_string(), + enrollment_id: "a".repeat(64), + }; + let changed = Enrollment { + node_id: iroh::SecretKey::generate().public().to_string(), + enrollment_id: "b".repeat(64), + }; + let expected = first.clone(); + let task = tokio::spawn(async move { + for (owner, enrollment) in [ + (None, first.clone()), + (Some(owner.clone()), first), + (Some(owner), changed), + ] { + let connection = endpoint.accept().await.unwrap().await.unwrap(); + let (mut send, mut recv) = connection.accept_bi().await.unwrap(); + let request: wire::Request = + serde_json::from_slice(&recv.read_to_end(wire::MAX_MESSAGE).await.unwrap()) + .unwrap(); + assert!(matches!(request.operation, Operation::Status)); + let response = Response::Status { + owner, + enrollment, + running: true, + }; + send.write_all(&serde_json::to_vec(&response).unwrap()) + .await + .unwrap(); + send.finish().unwrap(); + connection.closed().await; + } + endpoint.close().await; + }); + assert!( + service + .execute(Request::Restart { + name: "metal".into() + }) + .await + .unwrap_err() + .to_string() + .contains("owner binding") + ); + assert!(service.list().unwrap()["machines"][0]["enrollment"].is_null()); + assert_eq!(service.resolve("metal").await.unwrap(), expected); + assert!( + service + .resolve("metal") + .await + .unwrap_err() + .to_string() + .contains("enrollment changed") + ); + assert_eq!( + service.list().unwrap()["machines"][0]["enrollment"]["node_id"], + expected.node_id + ); + task.await.unwrap(); +} diff --git a/nix/ci.nix b/nix/ci.nix index 09ba37e3..b2f3060f 100644 --- a/nix/ci.nix +++ b/nix/ci.nix @@ -91,6 +91,8 @@ let "node" "llm" "gateway" + "cloud" + "cloud,node,gateway" "otel" ] ++ lib.optionals isValidatorHost [ "validator" ] @@ -100,6 +102,17 @@ let cli = mkCargo "check-cli" "cargo test -p hellas-cli --no-default-features --features node,gateway" ( cargoEnv rustToolchain ); + cloud = mkCargo "check-cloud" '' + cargo test -p hellas-cloud + cargo test -p hellas-cli --features cloud,node,gateway + cargo build -p hellas-cli --features cloud,node,gateway + cargo build -p hellas-cloud --bin hellas-agent + target_dir="''${CARGO_TARGET_DIR:-target}" + HELLAS_CLI="$(realpath "$target_dir/debug/hellas-cli")" \ + HELLAS_AGENT="$(realpath "$target_dir/debug/hellas-agent")" \ + cargo test -p hellas-cli --features cloud,node,gateway \ + owner_controls_admin_and_hellas_rpc_even_when_receipt_is_stolen -- --ignored + '' (cargoEnv rustToolchain); # The kernel's whole suite, including `tests/itf.rs` — the Quint↔Rust # replay that the entire abstract-correspondence story rests on — and # the exact-error pins in `tests/channel/`. `--all-features` is load @@ -293,9 +306,10 @@ let } // lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { docker = "docker"; + docker-cloud = "docker-cloud"; } - # CUDA and HIP images are intentionally omitted from the hosted matrix. - # Build them on the self-hosted release runner once it is registered again. + # CUDA/HIP images run in the dedicated trusted-branch image job, which + # streams archives for publishing without requiring a Docker daemon. // lib.optionalAttrs isValidatorHost { cli-validator = "cli-validator"; cli-catena = "cli-catena"; diff --git a/nix/default.nix b/nix/default.nix index dc1e3b5d..03d783af 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -189,6 +189,7 @@ let "chain" "gateway" ] + ++ lib.optional pkgSpec.pkgs.stdenv.hostPlatform.isUnix "cloud" ++ lib.optionals (crossSystem == null) [ "node" ] ++ lib.optional otel "otel"; } @@ -203,6 +204,22 @@ let } ) { }; } + // lib.optionalAttrs (crossSystem == null) { + agent = pkgSpec.mkHellasPackage { + pname = "hellas-agent"; + cargoBuildFlags = [ + "-p" + "hellas-cloud" + "--bin" + "hellas-agent" + ]; + cargoTestFlags = [ + "-p" + "hellas-cloud" + ]; + meta.mainProgram = "hellas-agent"; + }; + } # Do not advertise the safe local GPU runtime on platforms where the # packaged provider toolchain is not yet supported. // @@ -214,7 +231,11 @@ let }: pkgSpec.mkHellasPackage { buildNoDefaultFeatures = true; - buildFeatures = [ "evaluate" ] ++ lib.optional otel "otel"; + buildFeatures = [ + "evaluate" + "cloud" + ] + ++ lib.optional otel "otel"; } ) { }; }; @@ -266,15 +287,25 @@ let nativePkg.commonArgs // { pname = "hellas-rpc-wasm"; - cargoBuildFlags = [ - "-p" - "hellas-rpc" - ]; - CARGO_BUILD_TARGET = "wasm32-unknown-unknown"; - # wasm tests need wasm-bindgen-test infra (deferred). buildRustPackage's - # canExecute heuristic doesn't see our CARGO_BUILD_TARGET override, so - # without this it'd try to invoke `cargo test` against wasm and fail. + # cargoBuildHook passes the native --target explicitly, which overrides + # CARGO_BUILD_TARGET. Select the wasm target in the actual command. + buildPhase = '' + runHook preBuild + cargo build --offline --release --target wasm32-unknown-unknown -p hellas-rpc --jobs "$NIX_BUILD_CORES" + runHook postBuild + ''; + # Wasm tests need a wasm test runtime. doCheck = false; + dontStrip = true; + separateDebugInfo = false; + # The default Cargo install hook installs executables, not this rlib. + installPhase = '' + runHook preInstall + install -Dm644 target/wasm32-unknown-unknown/release/libhellas_rpc.rlib "$out/lib/libhellas_rpc.rlib" + mkdir -p "$out/lib/deps" + cp target/wasm32-unknown-unknown/release/deps/*.rlib "$out/lib/deps/" + runHook postInstall + ''; } ); @@ -286,16 +317,19 @@ let rustToolchain ; inherit (nativePackages) cli; + revision = self.rev or self.dirtyRev or "unknown"; }; dockerCuda = import ./docker.nix { inherit pkgs rustToolchain; cli = nativePackages.cli-catena; backend = "cuda"; + revision = self.rev or self.dirtyRev or "unknown"; }; dockerHip = import ./docker.nix { inherit pkgs rustToolchain; cli = nativePackages.cli-catena; backend = "hip"; + revision = self.rev or self.dirtyRev or "unknown"; }; nixosTests = lib.optionalAttrs isX86_64Linux ( @@ -310,10 +344,32 @@ let { packages = { docker = docker.image; + docker-cloud = + (import ./docker.nix { + inherit pkgs rustToolchain; + inherit (nativePackages) cli agent; + revision = self.rev or self.dirtyRev or "unknown"; + }).image; } // lib.optionalAttrs isX86_64Linux { docker-cuda = dockerCuda.image; docker-hip = dockerHip.image; + docker-cloud-cuda = + (import ./docker.nix { + inherit pkgs rustToolchain; + cli = nativePackages.cli-catena; + inherit (nativePackages) agent; + backend = "cuda"; + revision = self.rev or self.dirtyRev or "unknown"; + }).image; + docker-cloud-hip = + (import ./docker.nix { + inherit pkgs rustToolchain; + cli = nativePackages.cli-catena; + inherit (nativePackages) agent; + backend = "hip"; + revision = self.rev or self.dirtyRev or "unknown"; + }).image; }; apps = { diff --git a/nix/docker.nix b/nix/docker.nix index 7b1e79c9..9b43fe64 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -3,6 +3,8 @@ rustToolchain, cli, backend ? "network", + agent ? null, + revision ? "unknown", }: let imageRepository = "ghcr.io/hellas-ai/hellas"; @@ -20,6 +22,29 @@ let chmod 0555 "$out/bin/hellas-cli" ''; + agentRuntime = + pkgs.runCommand "hellas-agent-runtime" + { + nativeBuildInputs = [ pkgs.removeReferencesTo ]; + } + '' + mkdir -p "$out/bin" + cp "${agent}/bin/hellas-agent" "$out/bin/hellas-agent" + chmod u+w "$out/bin/hellas-agent" + remove-references-to -t ${rustToolchain} "$out/bin/hellas-agent" + chmod 0555 "$out/bin/hellas-agent" + ''; + originalEntrypoint = [ + (if cuda then "${entrypoint}" else "${runtime}/bin/hellas-cli") + "serve" + ] + ++ pkgs.lib.optionals gpu [ + "--gpu-backend" + backend + ]; + launcher = pkgs.writeText "hellas-${backend}-launcher.json" (builtins.toJSON originalEntrypoint); + imageTag = if agent == null then backend else "cloud-${backend}"; + gpu = backend != "network"; cuda = backend == "cuda"; toolkit = if cuda then pkgs.hellasLib.cudaToolkit else pkgs.hellasLib.rocmToolkit; @@ -39,7 +64,7 @@ let ); image = pkgs.dockerTools.streamLayeredImage { name = imageRepository; - tag = backend; + tag = imageTag; extraCommands = pkgs.lib.optionalString gpu '' mkdir -m 1777 -p tmp ''; @@ -56,16 +81,28 @@ let pkgs.binutils pkgs.bash ] - ++ pkgs.lib.optional cuda compiler; + ++ pkgs.lib.optional cuda compiler + ++ pkgs.lib.optional (agent != null) agentRuntime; config = { - Entrypoint = [ - (if cuda then "${entrypoint}" else "${runtime}/bin/hellas-cli") - "serve" - ] - ++ pkgs.lib.optionals gpu [ - "--gpu-backend" - backend - ]; + Entrypoint = + if agent == null then + originalEntrypoint + else + [ + "${agentRuntime}/bin/hellas-agent" + "--cli" + "${runtime}/bin/hellas-cli" + "--launcher" + "${launcher}" + # Provider volumes may ignore Unix permissions. Keep credentials on + # the container disk; identity/content still use the mounted volume. + "--configuration-dir" + "/var/lib/hellas-private" + ]; + Labels = { + "org.opencontainers.image.source" = "https://github.com/hellas-ai/hellas"; + "org.opencontainers.image.revision" = revision; + }; WorkingDir = "/var/lib/hellas"; Volumes."/var/lib/hellas" = { }; ExposedPorts."31145/udp" = { }; @@ -112,7 +149,7 @@ let name = "docker-push"; runtimeInputs = [ pkgs.skopeo ]; text = '' - ${image} | skopeo copy docker-archive:/dev/stdin "docker://${imageRepository}:${backend}" "$@" + ${image} | skopeo copy docker-archive:/dev/stdin "docker://${imageRepository}:${imageTag}" "$@" ''; }; in diff --git a/nix/noq-udp-musl.patch b/nix/noq-udp-musl.patch new file mode 100644 index 00000000..b7fbfeee --- /dev/null +++ b/nix/noq-udp-musl.patch @@ -0,0 +1,20 @@ +# Backport https://github.com/n0-computer/noq/pull/775 +# Reviewed at c239d5fbcb8647e721c538480b891f9d5f4e094c. +# musl cmsghdr alignment is weaker than its timestamp payload. +--- a/src/cmsg/mod.rs ++++ b/src/cmsg/mod.rs +@@ -44 +43,0 @@ +- /// - If `T` has stricter alignment requirements than `M::ControlMessage` +@@ -46 +44,0 @@ +- assert!(align_of::() <= align_of::()); +@@ -57 +55,2 @@ +- ptr::write(cmsg.cmsg_data() as *const T as *mut T, value); ++ // Unaligned: `align_of::()` is 4 on musl. ++ ptr::write_unaligned(cmsg.cmsg_data() as *const T as *mut T, value); +@@ -81 +79,0 @@ +- assert!(align_of::() <= align_of::()); +@@ -83 +81,3 @@ +- unsafe { ptr::read(cmsg.cmsg_data() as *const T) } ++ // Unaligned: `align_of::()` is 4 on musl, but payloads such as ++ // `libc::timespec` (SCM_TIMESTAMPNS) need 8. ++ unsafe { ptr::read_unaligned(cmsg.cmsg_data() as *const T) } diff --git a/nix/package.nix b/nix/package.nix index 6c239db4..35c4c2fc 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -82,6 +82,22 @@ let NIX_SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; buildInputs = workspaceBuildInputs; nativeBuildInputs = workspaceNativeBuildInputs; + # noq-udp 1.1.0 assumes cmsghdr has its timestamp payload's alignment. + # On musl that panics on the first received UDP packet. Keep the backport + # limited to musl packages until the upstream fix is released. + postPatch = lib.optionalString pkgs.stdenv.hostPlatform.isMusl '' + patch -d "$cargoDepsCopy/noq-udp-1.1.0" -p1 < ${./noq-udp-musl.patch} + ''; + # CLI unit tests do not exercise UDP receive. Run the cloud crate's real + # iroh enrollment test wherever this builder can execute the musl target. + postBuild = + lib.optionalString + (pkgs.stdenv.hostPlatform.isMusl && pkgs.stdenv.buildPlatform.canExecute pkgs.stdenv.hostPlatform) + '' + timeout 120 cargo test --offline --release --target ${targetTriple} \ + -p hellas-cloud --test management \ + enrollment_requires_owner_confirmation_and_cannot_be_silently_replaced -- --exact + ''; checkInputs = with pkgs; [ cargo-outdated ]; separateDebugInfo = true; # stdenv's default stripDebugList only does --strip-debug on bin/;