diff --git a/Cargo.lock b/Cargo.lock index b22a34b3..c82d4b77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aead" version = "0.5.2" @@ -207,6 +213,45 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -411,6 +456,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -429,7 +480,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", ] [[package]] @@ -438,6 +489,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitcoin-consensus-encoding" version = "1.1.0" @@ -1381,6 +1441,15 @@ dependencies = [ "spin 0.10.1", ] +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if 1.0.4", +] + [[package]] name = "criterion" version = "0.8.2" @@ -1730,6 +1799,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "der_derive" version = "0.7.3" @@ -2147,6 +2230,17 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "fluent-uri" version = "0.4.1" @@ -2681,6 +2775,7 @@ dependencies = [ "hellas-private", "hellas-providers", "hellas-rpc", + "hellas-sdk", "hellas-store", "hellas-wire", "hellas-work", @@ -2786,6 +2881,8 @@ dependencies = [ "anyhow", "async-stream", "axum", + "base64 0.22.1", + "flate2", "futures", "hellas-adaptors", "hellas-client", @@ -2795,9 +2892,11 @@ dependencies = [ "hellas-rpc", "hellas-store", "hellas-wire", + "httpdate", "iroh", "libc", "opentelemetry", + "opentelemetry_sdk", "prometheus-client", "rand 0.10.2", "reqwest", @@ -2809,6 +2908,7 @@ dependencies = [ "tower", "tracing", "tracing-opentelemetry", + "zstd", ] [[package]] @@ -2871,20 +2971,31 @@ dependencies = [ "anyhow", "async-stream", "axum", + "base64 0.22.1", "futures", "hellas-adaptors", "hellas-executor", + "hellas-private", "hellas-rpc", "hellas-wire", "opentelemetry", "opentelemetry_sdk", + "rcgen", "reqwest", + "rustls", "serde", "serde_json", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.18", "tokio", + "tokio-rustls", "tracing", "tracing-opentelemetry", "tracing-subscriber", + "url", + "webpki-roots 1.0.8", + "x509-cert", ] [[package]] @@ -2892,6 +3003,7 @@ name = "hellas-rpc" version = "0.1.0" dependencies = [ "axum", + "base64 0.22.1", "base64ct", "bytes", "ed25519-dalek", @@ -2928,6 +3040,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "url", "web-time", ] @@ -2935,19 +3048,31 @@ dependencies = [ name = "hellas-sdk" version = "0.1.0" dependencies = [ - "anyhow", + "async-stream", + "bytes", + "futures", "futures-core", "hellas-attestation", + "hellas-chain", "hellas-client", "hellas-executor", "hellas-gateway", + "hellas-kernel", + "hellas-private", "hellas-providers", "hellas-rpc", "hellas-wire", + "hellas-work", + "hex", "iroh", + "rand 0.10.2", + "serde", + "serde_json", "tempfile", + "thiserror 2.0.18", "tokio", "tracing", + "url", ] [[package]] @@ -2995,6 +3120,7 @@ dependencies = [ "tokio-stream", "tokio-tungstenite", "tracing", + "tracing-subscriber", "wasm-bindgen-futures", "web-time", "windows-sys 0.61.2", @@ -4235,6 +4361,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.1" @@ -4830,6 +4966,15 @@ dependencies = [ "objc2-security", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -5058,6 +5203,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -5433,7 +5588,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", - "bit-vec", + "bit-vec 0.8.0", "bitflags", "num-traits", "rand 0.9.4", @@ -5791,6 +5946,20 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "redb" version = "3.1.3" @@ -5979,6 +6148,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "1.1.4" @@ -6473,6 +6651,12 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "simd_cesu8" version = "1.1.1" @@ -8027,6 +8211,24 @@ dependencies = [ "spki 0.7.3", ] +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + [[package]] name = "xml-rs" version = "0.8.28" @@ -8042,6 +8244,16 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" version = "0.8.3" @@ -8159,6 +8371,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.21" diff --git a/README.md b/README.md index e37cf27d..b8476da0 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,11 @@ trusted app itself requires a platform-backed assurance. ## HTTP gateway -The gateway requires the same canonical causal-LM environment and an explicit +For upstream APIs, [HTTP Fetch routes](docs/http-gateway.md) preserve the vendor's +request, response and streaming formats. CLI gateways archive payloads by default; +`--zdr` or `x-hellas-zdr: true` disables application payload persistence. + +The causal-LM gateway requires the same canonical environment and an explicit presentation tokenizer. `--model` is only an API response label; when omitted, the manifest ID is used. diff --git a/crates/chain/src/server/tests.rs b/crates/chain/src/server/tests.rs index 93df2714..70bea065 100644 --- a/crates/chain/src/server/tests.rs +++ b/crates/chain/src/server/tests.rs @@ -384,7 +384,7 @@ async fn bound_and_relay_transports_answer_from_one_node_state() { .await .expect("the second transport receives a prompt saturation result") .expect_err("all sixteen node response permits are held by the first transport"); - assert_eq!(overflow.code(), WireCode::ResourceExhausted); + assert_eq!(overflow.code(), WireCode::ResourceExhausted, "{overflow:?}"); blocker.release.add_permits(16); for call in held { diff --git a/crates/chain/src/work_blocks.rs b/crates/chain/src/work_blocks.rs index 7c706907..193d94a0 100644 --- a/crates/chain/src/work_blocks.rs +++ b/crates/chain/src/work_blocks.rs @@ -63,6 +63,8 @@ pub struct WorkBlocks(C); /// What one production clock step did for a mounted paid channel. #[derive(Debug)] pub struct PaidWorkClockAdvance { + /// The verified snapshot already read for settlement, reusable by admission. + pub snapshot: Option, /// The journaled start-or-response drive that ran first. pub close: CloseProgress, /// Submission outcome for a due adjudicated payment close. @@ -158,6 +160,7 @@ where .map_err(PaidWorkClockError::Snapshot)? else { return Ok(PaidWorkClockAdvance { + snapshot: None, close, adjudication: None, bond_timeout: None, @@ -194,6 +197,7 @@ where }; Ok(PaidWorkClockAdvance { + snapshot: Some(snapshot), close, adjudication, bond_timeout, @@ -476,7 +480,7 @@ mod tests { network: TEST_NETWORK, policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: expected_values(), min_omit_response_blocks: hellas_kernel::MIN_OMIT_RESPONSE_BLOCKS, } @@ -893,7 +897,7 @@ mod tests { payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: expected_values(), }) { Ok(descriptor) => descriptor, diff --git a/crates/chain/src/work_e2e.rs b/crates/chain/src/work_e2e.rs index 681daf2e..f8d6b674 100644 --- a/crates/chain/src/work_e2e.rs +++ b/crates/chain/src/work_e2e.rs @@ -128,7 +128,7 @@ use hellas_rpc::{ use hellas_wire::mux::{MessagePipe, MuxConfig, MuxTransport, Role as MuxRole}; use hellas_wire::{DefaultClock, Dispatcher, StreamTransport as _, TransportContext}; use hellas_work::work::{ - BackendFault, ClientEndpoint, CloseEndpoint, JobProposal, PaidEvaluateBackend, PaymentError, + BackendFault, ClientEndpoint, CloseEndpoint, JobProposal, PaidWorkBackend, PaymentError, PreparedEvaluateInput, RunOutcome, WorkService, propose_work, run_accepted_work, }; use hellas_work::work_close::{CloseProgress, TxSink, close_start}; @@ -396,7 +396,7 @@ fn provider_policy() -> ProviderChannelPolicy { network: TEST_NETWORK, policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: expected_values(), min_omit_response_blocks: hellas_kernel::MIN_OMIT_RESPONSE_BLOCKS, } @@ -409,7 +409,7 @@ fn descriptor(allocations: &[(SettlementKey, u64)]) -> WorkChannelDescriptor { payment_terms: payment_terms(allocations), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: expected_values(), }) { Ok(descriptor) => descriptor, @@ -901,7 +901,7 @@ struct ProviderBackend { prompt: Vec, } -impl PaidEvaluateBackend for ProviderBackend { +impl PaidWorkBackend for ProviderBackend { fn evaluate( &self, input: PreparedEvaluateInput, @@ -1276,7 +1276,7 @@ async fn run_one_paid_job(devnet: &Devnet, opened: &mut Opened) -> u64 { calls: Arc::clone(&client_calls), }; let proposal = JobProposal { - prepared_input: prepared_input(), + prepared_input: prepared_input().into(), deadlines: deadlines(), }; diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 7fc11240..d5b3a88c 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -23,6 +23,9 @@ apple-app-attest = [] # is driven from; without it the runner has a chain and no way to ask it # anything. node = [ + "dep:hellas-sdk", + "hellas-sdk/paid-client", + "hellas-sdk/paid-provider", "chain", "dep:async-stream", "dep:axum", @@ -37,7 +40,7 @@ node = [ # This deliberately does not pull Catena into a client that only dials a node. llm = ["hellas-client/evaluate", "dep:hellas-presentation"] # HTTP gateway front-end (OpenAI/Anthropic/plain APIs). -gateway = ["llm", "dep:hellas-gateway"] +gateway = ["llm", "dep:hellas-gateway", "hellas-sdk?/paid-gateway"] evaluate = [ "node", "gateway", @@ -101,6 +104,7 @@ hellas-rpc = { workspace = true, default-features = false, features = [ "work", "host-control", ] } +hellas-sdk = { workspace = true, default-features = false, optional = true } hellas-store = { workspace = true } hellas-wire = { workspace = true, features = [ "unix", diff --git a/crates/cli/src/commands/environment/tests.rs b/crates/cli/src/commands/environment/tests.rs index 1fdffdc4..3ad46c56 100644 --- a/crates/cli/src/commands/environment/tests.rs +++ b/crates/cli/src/commands/environment/tests.rs @@ -86,7 +86,10 @@ fn build_rejects_oversized_settings_before_indexing_artifacts() { .expect_err("oversized settings must be refused before artifact indexing"); let message = error.to_string(); assert!(message.contains("environment settings"), "{message}"); - assert!(message.contains("byte limit"), "{message}"); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::InvalidData + ); assert!(!output.exists()); } @@ -103,7 +106,10 @@ fn inspect_rejects_oversized_environment_metadata() { let error = inspect(&environment).expect_err("oversized environment must be refused"); let message = error.to_string(); assert!(message.contains("environment"), "{message}"); - assert!(message.contains("byte limit"), "{message}"); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::InvalidData + ); } #[cfg(unix)] @@ -111,8 +117,11 @@ fn inspect_rejects_oversized_environment_metadata() { fn inspect_rejects_a_device_before_reading_from_it() { let error = inspect(Path::new("/dev/zero")).expect_err("environment metadata must be an ordinary file"); - assert!(error.to_string().contains("failed to open environment")); - assert!(format!("{error:#}").contains("not a regular file")); + assert!(error.to_string().contains("environment")); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::InvalidInput + ); } #[cfg(unix)] diff --git a/crates/cli/src/commands/fetch.rs b/crates/cli/src/commands/fetch.rs index 6e53cc6e..cebcdca3 100644 --- a/crates/cli/src/commands/fetch.rs +++ b/crates/cli/src/commands/fetch.rs @@ -147,7 +147,10 @@ mod tests { std::fs::write(&path, vec![b' '; MAX_FETCH_REQUEST_BODY_BYTES + 1]).unwrap(); let error = load_payload_file(&path).expect_err("oversized payload must be refused"); - assert!(error.to_string().contains("over the 1048576-byte limit")); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::InvalidData + ); } #[cfg(unix)] diff --git a/crates/cli/src/commands/llm.rs b/crates/cli/src/commands/llm.rs index b1a4cbe3..02e29235 100644 --- a/crates/cli/src/commands/llm.rs +++ b/crates/cli/src/commands/llm.rs @@ -411,7 +411,10 @@ mod tests { let error = load_environment(&path, None) .err() .expect("oversize is refused"); - assert!(error.to_string().contains("over the")); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::InvalidData + ); } #[cfg(unix)] diff --git a/crates/cli/src/commands/mod.rs b/crates/cli/src/commands/mod.rs index 6309f832..b9715d52 100644 --- a/crates/cli/src/commands/mod.rs +++ b/crates/cli/src/commands/mod.rs @@ -1,7 +1,6 @@ pub type CliResult = anyhow::Result; use anyhow::Context as _; -use std::io::Read as _; use std::path::Path; use std::time::Duration; @@ -38,20 +37,12 @@ pub(crate) fn read_bounded_regular_file( label: &str, maximum: usize, ) -> CliResult> { - let file = hellas_store::open_regular_file(path) - .with_context(|| format!("failed to open {label} {}", path.display()))?; - let limit = u64::try_from(maximum).unwrap_or(u64::MAX).saturating_add(1); - let mut bytes = Vec::new(); - file.take(limit) - .read_to_end(&mut bytes) - .with_context(|| format!("failed to read {label} {}", path.display()))?; - anyhow::ensure!( - bytes.len() <= maximum, - "{label} {} is {} bytes, over the {maximum}-byte limit", - path.display(), - bytes.len() - ); - Ok(bytes) + hellas_private::read_bounded_regular_file(path, maximum).with_context(|| { + format!( + "failed to read {label} {} (limit {maximum} bytes)", + path.display() + ) + }) } pub(crate) fn http_client(request_timeout: Duration) -> reqwest::Client { diff --git a/crates/cli/src/commands/paid_work.rs b/crates/cli/src/commands/paid_work.rs index 79226ae1..19d6d8e2 100644 --- a/crates/cli/src/commands/paid_work.rs +++ b/crates/cli/src/commands/paid_work.rs @@ -5,55 +5,35 @@ //! production setup and channel journals; this command does not keep a //! parallel receipt or invent a second protocol. -use std::net::SocketAddr; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::{Duration, Instant}; - use anyhow::{Context as _, bail}; use clap::{Args, Subcommand}; -use hellas_chain::client::{RemoteLightClient, VerifiedRemoteLightClient}; +use hellas_chain::client::RemoteLightClient; use hellas_chain::{ - ConsensusInfo, ConsensusVerifier, FinalizedBlockQuery, FinalizedBlockView, - FinalizedWorkView as _, LightClient as _, WorkBlocks, WorkChannelQuery, -}; -use hellas_client::work::payment::pay_for_result; -use hellas_client::work::{CollectResultOutcome, collect_result}; -use hellas_kernel::{ - CoinId, EdgeId, Funding, List, MAX_PARTY_INPUTS, MAX_START_VALIDITY_BLOCKS, Secp256k1Signer, - Secp256k1Verifier, WorkPaymentTerms, + ConsensusInfo, ConsensusVerifier, FinalizedBlockQuery, FinalizedBlockView, LightClient as _, }; -use hellas_rpc::protocol::artifacts::{Canonical as _, PreparedPaidInputV1}; -use hellas_rpc::protocol::work::{ - JobDeadlines, generation_policy_digest, identity_source_digest, private_policy_commitment, -}; -use hellas_rpc::protocol::work_setup::{ProviderChannelPolicy, WorkChannelDescriptor}; -use hellas_wire::ServiceMarker; -use hellas_wire::iroh::IrohTransport; -use hellas_work::work::{ClientEndpoint, JobProposal, propose_work, resume_work_proposal}; -use hellas_work::work_close::CloseProgress; -use hellas_work::work_close::FinalizedBlocks as _; -use hellas_work::work_handshake::{ - PaymentAdmission, SetupEndpoint, SetupService, apply_setup_exchange, prepare_setup_exchange, - send_setup_exchange, -}; -use hellas_work::work_open::{SetupAdvance, SetupProgress}; +use hellas_kernel::{CoinId, EdgeId, Funding, List, MAX_PARTY_INPUTS, Secp256k1Signer}; +use hellas_rpc::protocol::artifacts::PreparedPaidInputV1; +#[cfg(test)] +use hellas_rpc::protocol::work::{JobDeadlines, private_policy_commitment}; +use hellas_rpc::protocol::work_fetch::PreparedPaidFetchInputV1; +use hellas_rpc::protocol::work_profile::PreparedPaidWorkInput; +#[cfg(test)] +use hellas_sdk::paid_client::check_genesis_payload; +use hellas_sdk::paid_client::{InputIdentities, PaidWorkSession, bind_paid_endpoint}; use hellas_work::work_store::journal::MAX_RECORD_BYTES; -use hellas_work::work_store::{Role, SetupScan, SetupStore}; -use iroh::endpoint::presets; -use iroh::{Endpoint, EndpointAddr, EndpointId, SecretKey, TransportAddr}; +use iroh::{EndpointId, SecretKey}; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::time::Duration; use super::CliResult; -use super::serve::work_config::{WorkConfig, load_work_config}; - -#[cfg(feature = "gateway")] -mod gateway; -#[cfg(feature = "gateway")] -pub use gateway::load_gateway_backend; +use super::serve::work_config::load_work_config; /// Paid-work commands intended for deployment bring-up and smoke tests. #[derive(Debug, Subcommand)] pub enum PaidWorkCommand { + /// Build a signed ephemeral Fetch input for a paid channel. + PrepareFetch(PrepareFetchArgs), /// Build canonical paid input from a causal-LM environment and prompt. #[cfg(feature = "llm")] PrepareInput(PrepareInputArgs), @@ -65,6 +45,26 @@ pub enum PaidWorkCommand { Run(Box), } +#[derive(Debug, Args)] +pub struct PrepareFetchArgs { + #[arg(long)] + service: String, + #[arg(long)] + method: String, + /// The trusted transformation to execute. + #[arg(long, value_parser = ["openai-responses", "codex-responses", "http"])] + execution_environment: String, + /// Assurance authenticated before the paid request is disclosed. + #[arg(long, default_value = "producer-signed", value_parser = ["producer-signed", "apple-app-attest"])] + assurance: String, + /// Provider-shaped UTF-8 JSON request, read from an ordinary file. + #[arg(long, value_name = "FILE")] + payload_file: PathBuf, + /// Client-owned output file containing the signed request. + #[arg(long, value_name = "FILE")] + out: PathBuf, +} + #[cfg(feature = "llm")] #[derive(Debug, Args)] pub struct PrepareInputArgs { @@ -109,6 +109,13 @@ pub struct InspectChainArgs { #[derive(Clone, Debug, Args)] pub struct RunArgs { + /// Out-of-band provider enrollment pin (required for App Attest). + #[arg(long)] + provider_genesis: Option, + #[arg(long)] + apple_app_id: Option, + #[arg(long, value_delimiter = ',', value_parser = crate::parse_hex_array::<32>)] + apple_cd_hashes: Vec<[u8; 32]>, /// Provider work configuration, including chain identity and policy. #[arg(long = "work-config", value_name = "FILE")] work_config: PathBuf, @@ -137,7 +144,7 @@ pub struct RunArgs { #[arg(long = "omission-bond")] omission_bond: u64, - /// Canonical PreparedPaidInputV1 bytes to execute. + /// Canonical prepared Evaluate or Fetch input to execute. #[arg(long = "prepared-input", value_name = "FILE")] prepared_input: PathBuf, @@ -171,8 +178,10 @@ pub async fn run( command: PaidWorkCommand, transport_key: SecretKey, settlement_key: Secp256k1Signer, + producer_key: hellas_rpc::ProducerSigningKey, ) -> CliResult<()> { match command { + PaidWorkCommand::PrepareFetch(args) => prepare_fetch(args, &producer_key), #[cfg(feature = "llm")] PaidWorkCommand::PrepareInput(args) => prepare_input(args, &transport_key, &settlement_key), PaidWorkCommand::InspectInput(args) => { @@ -192,6 +201,39 @@ pub async fn run( } } +fn prepare_fetch(args: PrepareFetchArgs, key: &hellas_rpc::ProducerSigningKey) -> CliResult<()> { + let environment = match args.execution_environment.as_str() { + "openai-responses" => hellas_rpc::FetchEnvironment::OpenAiResponses, + "codex-responses" => hellas_rpc::FetchEnvironment::CodexResponses, + "http" => hellas_rpc::FetchEnvironment::Http, + _ => bail!("unsupported fetch environment"), + }; + let payload = super::fetch::load_payload_file(&args.payload_file)?; + if environment == hellas_rpc::FetchEnvironment::Http { + hellas_rpc::http_fetch::HttpFetchRequest::decode(&payload)?; + } + let assurance = match args.assurance.as_str() { + "producer-signed" => hellas_rpc::Assurance::ProducerSigned, + "apple-app-attest" => hellas_rpc::Assurance::AppleAppAttest, + _ => bail!("unsupported assurance"), + }; + let events = hellas_rpc::fetch::build_input_events_with_retention( + &args.service, + &args.method, + &payload, + environment.manifest_id(), + assurance, + key, + hellas_rpc::Retention::Ephemeral, + )?; + let prepared = PreparedPaidFetchInputV1::new(&events, &environment.manifest())?; + write_private(&args.out, &prepared.encode()?)?; + println!("prepared_input: {}", args.out.display()); + println!("allowed_environment: {}", environment.manifest_id()); + println!("provider_payload_retention: memory-only"); + Ok(()) +} + #[cfg(feature = "llm")] fn prepare_input( args: PrepareInputArgs, @@ -257,8 +299,27 @@ fn inspect_input( transport_key: &SecretKey, settlement_key: &Secp256k1Signer, ) -> CliResult<()> { - let prepared = read_prepared_input(path)?; - inspect_prepared(&prepared, transport_key, settlement_key) + match read_prepared_work_input(path)? { + PreparedPaidWorkInput::Evaluate(prepared) => { + inspect_prepared(&prepared, transport_key, settlement_key) + } + PreparedPaidWorkInput::Fetch(prepared) => { + let parts = prepared.parts()?; + let request = hellas_rpc::fetch::verify_input_events(&parts.fetch_input_transcript)?; + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "profile": "fetch", + "service": request.service, + "method": request.method, + "allowed_environment": parts.manifest.content_id().to_string(), + "caller_key": hex::encode(request.caller_key.bytes()), + "provider_payload_retention": "memory-only", + }))? + ); + Ok(()) + } + } } fn inspect_prepared( @@ -281,37 +342,6 @@ fn inspect_prepared( Ok(()) } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct InputIdentities { - allowed_environment: hellas_rpc::ContentId, - generation_policy_digest: hellas_rpc::Digest, - identity_source_digest: hellas_rpc::Digest, -} - -impl InputIdentities { - fn from_prepared(prepared: &PreparedPaidInputV1) -> CliResult { - let parts = prepared - .parts() - .context("prepared input contains a non-canonical body")?; - let allowed_environment = parts.manifest.content_id(); - anyhow::ensure!( - parts.evaluate_request.execution_environment == allowed_environment, - "prepared input request names environment {}, but its manifest derives {}", - parts.evaluate_request.execution_environment, - allowed_environment, - ); - Ok(Self { - allowed_environment, - generation_policy_digest: generation_policy_digest( - &parts.text_policy.canonical_bytes(), - )?, - identity_source_digest: identity_source_digest( - &parts.identity_artifact.canonical_bytes(), - )?, - }) - } -} - async fn inspect_chain(validators: &[String]) -> CliResult<()> { anyhow::ensure!( validators.len() == 6, @@ -377,25 +407,80 @@ async fn inspect_chain(validators: &[String]) -> CliResult<()> { Ok(()) } +fn paid_provider_trust( + args: &RunArgs, + assurance: hellas_rpc::Assurance, +) -> CliResult> { + if args.provider_genesis.is_some() || assurance != hellas_rpc::Assurance::ProducerSigned { + Ok(Some(crate::identity::provider_trust( + args.provider_genesis, + assurance, + args.apple_app_id.clone(), + args.apple_cd_hashes.clone(), + )?)) + } else { + Ok(None) + } +} + +async fn open_paid_channel( + args: &RunArgs, + endpoint: iroh::Endpoint, + settlement_key: Secp256k1Signer, + assurance: hellas_rpc::Assurance, +) -> CliResult { + anyhow::ensure!( + !args.payment_coins.is_empty(), + "at least one --payment-coin is required" + ); + let provider_trust = paid_provider_trust(args, assurance)?; + PaidWorkSession::open( + hellas_sdk::paid_client::PaidWorkOptions { + config: load_work_config(&args.work_config)?, + journal_root: args.journal_root.clone(), + provider: args.provider, + provider_addrs: args.provider_addrs.clone(), + provider_trust, + bond: edge_id("--bond", &args.bond)?, + payment_funding: Funding::new(coins(&args.payment_coins)?, empty_coins()), + omission_bond: args.omission_bond, + acceptance_blocks: args.acceptance_blocks, + terminal_blocks: args.terminal_blocks, + payment_blocks: args.payment_blocks, + timeout: Duration::from_secs(args.timeout_secs), + }, + endpoint, + settlement_key, + ) + .await + .map_err(Into::into) +} + async fn run_one( args: RunArgs, transport_key: SecretKey, settlement_key: Secp256k1Signer, ) -> CliResult<()> { - let prepared = read_prepared_input(&args.prepared_input)?; + let prepared = read_prepared_work_input(&args.prepared_input)?; let endpoint = bind_paid_endpoint(transport_key).await?; - let mut channel = OpenPaidChannel::open(args, endpoint, settlement_key).await?; + let mut channel = open_paid_channel( + &args, + endpoint.clone(), + settlement_key, + prepared.assurance()?, + ) + .await?; println!( "bond_edge: {}", - hex::encode(channel.descriptor.bond_edge().to_bytes()) + hex::encode(channel.descriptor().bond_edge().to_bytes()) ); println!( "payment_edge: {}", - hex::encode(channel.descriptor.channel().payment_edge().to_bytes()) + hex::encode(channel.descriptor().channel().payment_edge().to_bytes()) ); println!( "channel_id: {}", - hex::encode(channel.descriptor.channel().id().as_bytes()) + hex::encode(channel.descriptor().channel().id().as_bytes()) ); let result = channel .run(Some(prepared), false, None) @@ -405,819 +490,36 @@ async fn run_one( println!("job_price: {}", result.job_price); println!("credited_cumulative: {}", result.credited_cumulative); println!("authenticated_result: true"); - if let Some(output) = channel.args.output.as_ref() { - std::fs::write(output, &result.transcript).with_context(|| { - format!("failed to write result transcript to {}", output.display()) - })?; - println!( - "result: {} ({} bytes)", - output.display(), - result.transcript.len() - ); - } else { - println!("result_bytes: {}", result.transcript.len()); + if let Some(output) = &args.output { + write_private(output, &result.transcript)?; } - if channel.args.settle { - channel - .client - .prepare_close() - .context("failed to prepare the client payment close")?; - loop { - match channel - .client - .advance_close(&channel.chain, &channel.chain) - .await - .context("failed to advance the client payment close")? - { - CloseProgress::Settled { provider_payout } => { - println!("settled: true"); - println!("settled_provider_payout: {provider_payout}"); - println!( - "settled_finalized_height: {}", - channel.client.state().cursor().0 - ); - break; - } - CloseProgress::Submitted { outcome, .. } => { - tracing::info!(?outcome, "client payment close submitted"); - } - CloseProgress::Opened { .. } | CloseProgress::Nothing => {} - } - tokio::time::sleep(channel.config.poll).await; - } - } else { - println!("settled: false"); + println!("result_bytes: {}", result.transcript.len()); + if args.settle { + println!("settled_provider_payout: {}", channel.settle().await?); } - println!("client_journals: {}", channel.args.journal_root.display()); + println!("settled: {}", args.settle); + println!("client_journals: {}", args.journal_root.display()); + endpoint.close().await; Ok(()) } -struct PaidOutput { - work_id: hellas_rpc::Digest, - job_price: u64, - credited_cumulative: u64, - transcript: Vec, - #[cfg(feature = "gateway")] - provider_key: hellas_rpc::PublicKey, - #[cfg(feature = "gateway")] - input: hellas_rpc::InputCommitment, -} - -struct OpenPaidChannel { - args: RunArgs, - config: WorkConfig, - descriptor: WorkChannelDescriptor, - dialer: ProviderDialer, - chain: WorkBlocks, - next_validator: usize, - client: ClientEndpoint, - needs_recovery: bool, -} - -impl OpenPaidChannel { - async fn open( - args: RunArgs, - endpoint: Endpoint, - settlement_key: Secp256k1Signer, - ) -> CliResult { - anyhow::ensure!( - args.acceptance_blocks > 0 && args.terminal_blocks > 0 && args.payment_blocks > 0, - "all three deadline spans must be greater than zero", - ); - anyhow::ensure!( - !args.payment_coins.is_empty(), - "at least one --payment-coin is required", - ); - - let config = load_work_config(&args.work_config)?; - let policy = config.provider_policy(); - let bond = edge_id("--bond", &args.bond)?; - let payment_funding = Funding::new(coins(&args.payment_coins)?, empty_coins()); - let mut next_validator = 0; - let chain = connect_chain(&config, &mut next_validator).await?; - check_genesis(&config, &chain).await?; - - std::fs::create_dir_all(&args.journal_root).with_context(|| { - format!( - "failed to create client journal root {}", - args.journal_root.display(), - ) - })?; - let store = SetupStore::open( - &args.journal_root, - config.chain.network, - bond, - Role::Client, - &Secp256k1Verifier::new(), - ) - .with_context(|| { - format!( - "failed to open client setup journal under {}", - args.journal_root.display(), - ) - })?; - let mut setup = SetupEndpoint::new( - store, - settlement_key.clone(), - PaymentAdmission::Proposes(Box::new(policy.clone())), - ); - let dialer = ProviderDialer::new(args.provider, args.provider_addrs.clone(), endpoint); - - if setup.state().revision().is_none() { - exchange_setup(&dialer, &mut setup).await?; - } - let bundle = setup - .state() - .bundle() - .cloned() - .context("provider returned no bond proposal")?; - anyhow::ensure!( - bundle.bond_edge() == bond, - "provider proposed a different bond edge" - ); - anyhow::ensure!( - bundle.bond_terms().parties.taker() == settlement_key.party_key(), - "provider bond names client settlement key {}, not this identity's {}", - hex::encode(bundle.bond_terms().parties.taker().to_bytes()), - hex::encode(settlement_key.party_key().to_bytes()), - ); - if setup.state().scan_armed().is_none() { - setup.arm_scan(finalized_floor(&chain).await?)?; - } - if setup.state().revision() == Some(1) { - let terms = payment_terms(&config, &policy, &bundle, args.omission_bond); - setup.propose_payment(payment_funding, terms)?; - } - if setup.state().revision() == Some(2) { - exchange_setup(&dialer, &mut setup).await?; - } - anyhow::ensure!( - setup.state().revision() == Some(3), - "setup did not reach its countersigned revision", - ); - - let setup_service = SetupService::new(setup); - let (mounted, descriptor) = - drive_setup(&setup_service, &policy, &chain, config.poll).await?; - let ready = ready_channel(&descriptor, &chain).await?; - let client = ClientEndpoint::new(ready.clone(), mounted, settlement_key)?; - - Ok(Self { - args, - config, - descriptor, - dialer, - chain, - next_validator, - client, - needs_recovery: true, - }) - } - - async fn follow_chain(&mut self) -> CliResult<()> { - for attempt in 0..self.config.validators.len() { - match self.client.catch_up(&self.chain).await { - Ok(_) => return Ok(()), - Err(error) if attempt + 1 == self.config.validators.len() => { - return Err(error.into()); - } - Err(error) => { - tracing::debug!(%error, "paid channel will continue catch-up through another validator"); - self.chain = connect_chain(&self.config, &mut self.next_validator).await?; - check_genesis(&self.config, &self.chain).await?; - } - } - } - bail!("no configured validator supplied finalized history") - } - - async fn run( - &mut self, - prepared: Option, - recover: bool, - progress: Option, - ) -> CliResult> { - self.run_with_admission(prepared, recover, progress, None) - .await - } - - async fn run_with_admission( - &mut self, - prepared: Option, - recover: bool, - progress: Option, - proposed: Option<&AtomicBool>, - ) -> CliResult> { - self.follow_chain().await?; - let Self { - args, - config, - descriptor, - dialer, - chain, - client, - needs_recovery, - .. - } = self; - if let Some(prepared) = prepared.as_ref() { - check_policy_input(&config.provider_policy(), prepared)?; - } - let ready = caught_up_channel(descriptor, client, &*chain).await?; - if recover && *needs_recovery { - if let Some(payment) = client.state().last_payment() { - // The provider may have committed payment while its acknowledgement - // was lost. Re-send the retained certificate before accepting work. - pay_for_result(dialer.work().await?, client, payment.work_id).await?; - } - let pending = client - .state() - .jobs() - .filter(|job| { - // The journal forbids signing payment after this height. - // Keep the evidence, but do not let an unpayable old job - // prevent this channel from serving a new request. Retained - // certificates are re-sent separately above. - let payable = job.authorization().payment_deadline >= client.state().cursor().0; - if !payable { - tracing::info!( - work_id = %hex::encode(job.work_id().as_bytes()), - payment_deadline = job.authorization().payment_deadline, - "retaining expired unpaid job without retrying execution", - ); - } - payable - }) - .map(|job| { - PreparedPaidInputV1::decode(job.prepared_input(), MAX_RECORD_BYTES) - .map(|input| (job.work_id(), job.phase(), input)) - }) - .collect::, _>>()?; - for (work_id, phase, pending) in pending { - let result = execute_paid_job( - args, - pending, - dialer, - client, - &ready, - &*chain, - config.poll, - None, - JobLookup::Retained(work_id), - None, - ) - .await; - if let Err(error) = &result - && ((phase == hellas_work::work_store::JobPhase::HalfSigned - && matches!( - error.downcast_ref::(), - Some(hellas_work::work::ProposeError::Refused { - refusal, - .. - }) if !refusal.is_retryable() - )) - || permanently_refused_delivery(error)) - { - // Keep the signed evidence without deciding that an unpaid - // job was paid or cancelled. A permanent provider refusal - // cannot be repaired by blocking every later request here. - tracing::info!(%work_id, %error, "retaining an unpaid job refused by the provider"); - continue; - } - result?; - } - } - // Keep recovery armed across any error or cancellation after acceptance. - *needs_recovery = prepared.is_some(); - let result = match prepared { - Some(prepared) => { - let ready = caught_up_channel(descriptor, client, &*chain).await?; - Some( - execute_paid_job( - args, - prepared, - dialer, - client, - &ready, - &*chain, - config.poll, - progress.as_ref(), - if recover { - JobLookup::New - } else { - JobLookup::PreparedInput - }, - proposed, - ) - .await?, - ) - } - None => None, - }; - *needs_recovery = false; - Ok(result) - } -} - -fn permanently_refused_delivery(error: &anyhow::Error) -> bool { - use hellas_client::work::CollectResultError; - use hellas_work::work::DeliverError; - let delivery = error.downcast_ref::().or_else(|| { - match error.downcast_ref::() { - Some(CollectResultError::Deliver(delivery)) => Some(delivery), - _ => None, - } - }); - matches!(delivery, Some(DeliverError::Refused { refusal, .. }) if !refusal.is_retryable()) -} - -/// Proposes until a provider accepts or the caller's execution window closes. -/// A provider catching its chain cursor up replies `NotReady`; that is not an -/// answer to the job. The retained proposal makes each retry the same request, -/// while bounded exponential backoff avoids turning recovery into a request -/// flood. -async fn propose_when_ready( - dialer: &ProviderDialer, - client: &mut ClientEndpoint, - proposal: &JobProposal, - retained: Option, - poll: Duration, - timeout: Duration, - proposed: Option<&AtomicBool>, -) -> CliResult { - let deadline = Instant::now() + timeout; - let mut delay = poll.max(Duration::from_secs(1)); - loop { - if Instant::now() >= deadline { - bail!("provider remained not ready for {timeout:?}"); - } - let transport = dialer.work().await?; - // Once a proposal can leave this process, a lost acknowledgement must - // be treated as accepted work. HTTP cancellation may no longer stop it. - if let Some(proposed) = proposed { - proposed.store(true, Ordering::Release); - } - let result = match retained { - Some(work_id) => resume_work_proposal(transport, client, work_id).await, - None => propose_work(transport, client, proposal).await, - }; - match result { - Ok(work_id) => return Ok(work_id), - Err(hellas_work::work::ProposeError::Refused { refusal, .. }) - if refusal.is_retryable() => - { - let remaining = deadline.saturating_duration_since(Instant::now()); - tokio::time::sleep(delay.min(remaining)).await; - delay = delay.saturating_mul(2).min(Duration::from_secs(10)); - } - Err(error) => return Err(error.into()), - } - } -} - -enum JobLookup { - Retained(hellas_rpc::Digest), - PreparedInput, - New, -} - -#[allow(clippy::too_many_arguments)] -async fn execute_paid_job( - args: &RunArgs, - prepared: PreparedPaidInputV1, - dialer: &ProviderDialer, - client: &mut ClientEndpoint, - ready: &hellas_rpc::protocol::work_setup::ReadyChannel, - chain: &WorkBlocks, - poll: Duration, - progress: Option<&hellas_work::work::PaidProgress>, - lookup: JobLookup, - proposed: Option<&AtomicBool>, -) -> CliResult { - let readiness_timeout = Duration::from_secs(args.timeout_secs); - let prepared_bytes = prepared.encode()?; - let current = client.state().cursor().0; - let deadlines = relative_deadlines(current, args)?; - let proposal = JobProposal { - prepared_input: prepared, - deadlines, - }; - let existing = client - .state() - .jobs() - .filter(|job| match &lookup { - JobLookup::Retained(work_id) => job.work_id() == *work_id, - JobLookup::New => false, - JobLookup::PreparedInput => { - job.prepared_input() == prepared_bytes.as_slice() - && job.authorization().payment_deadline >= current - && (job.phase() != hellas_work::work_store::JobPhase::HalfSigned - || job.authorization().acceptance_deadline >= current) - } - }) - .map(|job| (job.work_id(), job.phase(), *job.authorization())) - .collect::>(); - anyhow::ensure!( - existing.len() <= 1, - "more than one active job matches this prepared input; inspect the retained channel journal", - ); - if !existing.is_empty() - && let Some(proposed) = proposed - { - proposed.store(true, Ordering::Release); - } - let (work_id, already_collected) = match existing.first().copied() { - Some((work_id, hellas_work::work_store::JobPhase::HalfSigned, _)) => ( - propose_when_ready( - dialer, - client, - &proposal, - Some(work_id), - poll, - readiness_timeout, - proposed, - ) - .await?, - false, - ), - Some((work_id, hellas_work::work_store::JobPhase::Ready, _)) - | Some((work_id, hellas_work::work_store::JobPhase::Matched, _)) => (work_id, true), - Some((work_id, _, _)) => (work_id, false), - None => ( - propose_when_ready( - dialer, - client, - &proposal, - None, - poll, - readiness_timeout, - proposed, - ) - .await?, - false, - ), - }; - let transcript = if already_collected { - client - .state() - .job_by_id(work_id) - .map(|job| job.transcript().to_vec()) - .context("collected job disappeared from its journal")? - } else if let Some(progress) = progress { - let mut emitted = false; - let delivery = loop { - let result = hellas_work::work::fetch_result_stream( - dialer.work().await?, - client, - ready, - work_id, - |event| { - emitted = true; - progress(event.clone()).map_err(|error| { - hellas_rpc::protocol::work::PaidWorkError::Transcript(error.to_string()) - }) - }, - ) - .await; - let error = match result { - Ok(delivery) => break delivery, - Err(error) => error, - }; - let retryable = match &error { - hellas_work::work::DeliverError::Transport(status) => { - status.code == hellas_wire::WireCode::Unavailable - } - hellas_work::work::DeliverError::Refused { refusal, .. } => refusal.is_retryable(), - _ => false, - }; - // Retry delivery of this accepted job only before exposing output. - // Reopening after a prefix would replay it into the user's stream. - if emitted || !retryable { - return Err(error.into()); - } - client.catch_up(chain).await?; - let job = client - .state() - .job_by_id(work_id) - .context("accepted job disappeared")?; - anyhow::ensure!( - client.state().cursor().0 <= job.authorization().payment_deadline, - "payment deadline elapsed while waiting for result stream" - ); - tracing::debug!(%error, %work_id, "waiting for paid result stream readiness"); - tokio::time::sleep(poll.max(Duration::from_secs(1))).await; - }; - client.catch_up(chain).await?; - anyhow::ensure!( - client.state().cursor().0 <= proposal.deadlines.payment, - "payment deadline elapsed during delivery" - ); - delivery.transcript - } else { - collect_until_ready(dialer, client, ready, chain, work_id, poll).await? - }; - let credited = pay_for_result(dialer.work().await?, client, work_id).await?; - Ok(PaidOutput { - work_id, - job_price: ready.execution_policy().fixed_price, - credited_cumulative: credited, - transcript, - #[cfg(feature = "gateway")] - provider_key: hellas_rpc::PublicKey::Secp256k1(ready.channel().provider_key().to_bytes()), - #[cfg(feature = "gateway")] - input: hellas_rpc::evaluate::input_commitment( - &proposal.prepared_input.parts()?.evaluate_request, - ), - }) -} - -fn check_policy_input( - policy: &ProviderChannelPolicy, - prepared: &PreparedPaidInputV1, -) -> CliResult<()> { - let input = InputIdentities::from_prepared(prepared)?; - let parts = prepared.parts()?; - let expected = policy.execution_policy; - anyhow::ensure!( - expected.allowed_environment == input.allowed_environment, - "work config allows environment {}, but prepared input uses {}", - expected.allowed_environment, - input.allowed_environment, - ); - anyhow::ensure!( - hellas_rpc::protocol::work::matches_generation_policy(&expected, &parts.text_policy)?, - "work config generation_policy_digest does not match prepared input", - ); - anyhow::ensure!( - expected.identity_source_digest == input.identity_source_digest, - "work config identity_source_digest does not match prepared input", - ); - Ok(()) -} - -fn payment_terms( - config: &WorkConfig, - policy: &ProviderChannelPolicy, - bundle: &hellas_rpc::protocol::work_bundle::WorkChannelSetupBundleV1, - omission_bond: u64, -) -> WorkPaymentTerms { - WorkPaymentTerms { - bond_edge: bundle.bond_edge(), - bond_terms: bundle.bond_terms().clone(), - private_policy_commitment: private_policy_commitment( - config.chain.network, - &policy.policy_salt, - &policy.channel_policy, - ), - omit_response_blocks: policy.min_omit_response_blocks, - start_validity_blocks: MAX_START_VALIDITY_BLOCKS, - omission_bond, - } -} - -async fn drive_setup( - setup: &SetupService, - policy: &ProviderChannelPolicy, - chain: &WorkBlocks, - poll: Duration, -) -> CliResult<(hellas_work::work_store::ChannelStore, WorkChannelDescriptor)> { - loop { - let SetupAdvance { progress, mounted } = setup - .advance_setup(chain, chain, chain) - .await - .context("failed to advance paid-work setup")?; - if let Some(store) = mounted { - let channel = store.state().channel(); - let descriptor = policy - .admit(channel.payment_edge(), channel.payment_terms().clone()) - .context("the funded channel no longer satisfies the configured policy")?; - return Ok((store, descriptor)); - } - match progress { - SetupProgress::Aborted(reason) => bail!("paid-work setup aborted: {reason:?}"), - SetupProgress::Faulted(reason) => bail!("paid-work setup faulted: {reason:?}"), - SetupProgress::TimeoutBond => bail!("provider bond timed out before setup completed"), - _ => tokio::time::sleep(poll).await, - } - } -} - -async fn ready_channel( - descriptor: &WorkChannelDescriptor, - chain: &WorkBlocks, -) -> CliResult { - let query = WorkChannelQuery { - bond_edge: descriptor.bond_edge(), - payment_edge: descriptor.channel().payment_edge(), - funding: Default::default(), - }; - let snapshot = chain - .work_channel_snapshot(query) - .await? - .context("no finalized channel snapshot is available")?; - descriptor - .check_ready(&snapshot.observed_channel()) - .context("the finalized channel is not ready") -} - -/// Reads the ready snapshot once the client has processed it. -/// -/// The snapshot and the blocks the cursor follows are answered by -/// validators independently, so the snapshot can name a height the -/// light client has not finalized yet, and a snapshot read after the -/// catch-up on a moving chain always lands a few blocks ahead of it. -/// The snapshot is sampled first and then the cursor is brought to it: -/// a fixed height is a target the catch-up reaches, where a fresh -/// snapshot every round was not. A state at or behind the cursor is -/// the direction `check_caught_up` accepts. -async fn caught_up_channel( - descriptor: &WorkChannelDescriptor, - client: &mut ClientEndpoint, - chain: &WorkBlocks, -) -> CliResult { - let ready = ready_channel(descriptor, chain).await?; - for _ in 0..16 { - let cursor = client.catch_up(chain).await?; - if ready.check_caught_up(cursor).is_ok() { - return Ok(ready); - } - tokio::time::sleep(Duration::from_millis(500)).await; - } - let cursor = client.catch_up(chain).await?; - ready.check_caught_up(cursor)?; - Ok(ready) -} - -async fn collect_until_ready( - dialer: &ProviderDialer, - client: &mut ClientEndpoint, - ready: &hellas_rpc::protocol::work_setup::ReadyChannel, - chain: &WorkBlocks, - work_id: hellas_rpc::Digest, - poll: Duration, -) -> CliResult> { - loop { - match collect_result(dialer.work().await?, client, ready, chain, work_id).await? { - CollectResultOutcome::Collected(result) => return Ok(result.transcript), - CollectResultOutcome::NotReady { reason } => { - tracing::debug!(%reason, "waiting for paid result"); - tokio::time::sleep(poll.max(Duration::from_secs(1))).await; - } - } - } -} - +#[cfg(test)] fn relative_deadlines(current: u64, args: &RunArgs) -> CliResult { - let acceptance = current - .checked_add(args.acceptance_blocks) - .context("acceptance deadline overflow")?; - let terminal = acceptance - .checked_add(args.terminal_blocks) - .context("terminal deadline overflow")?; - let payment = terminal - .checked_add(args.payment_blocks) - .context("payment deadline overflow")?; - Ok(JobDeadlines { - acceptance, - terminal, - payment, - }) -} - -async fn exchange_setup(dialer: &ProviderDialer, setup: &mut SetupEndpoint) -> CliResult<()> { - let request = prepare_setup_exchange(setup); - let response = send_setup_exchange(dialer.setup().await?, request).await?; - apply_setup_exchange(setup, response)?; - Ok(()) -} - -struct ProviderDialer { - endpoint: Endpoint, - provider: EndpointAddr, -} - -async fn bind_paid_endpoint(secret_key: SecretKey) -> CliResult { - Endpoint::builder(presets::N0) - .secret_key(secret_key) - .alpns(vec![ - hellas_rpc::services::work_setup::WorkSetup::ALPN - .as_bytes() - .to_vec(), - hellas_rpc::services::work::Work::ALPN.as_bytes().to_vec(), - ]) - .bind() - .await - .context("failed to bind paid-work Iroh endpoint") -} - -impl ProviderDialer { - fn new(provider: EndpointId, addresses: Vec, endpoint: Endpoint) -> Self { - Self { - endpoint, - provider: EndpointAddr::from_parts( - provider, - addresses.into_iter().map(TransportAddr::Ip), - ), - } - } - - async fn setup(&self) -> CliResult { - self.connect(hellas_rpc::services::work_setup::WorkSetup::ALPN.as_bytes()) - .await - } - - async fn work(&self) -> CliResult { - self.connect(hellas_rpc::services::work::Work::ALPN.as_bytes()) - .await - } - - async fn connect(&self, alpn: &[u8]) -> CliResult { - let connection = self - .endpoint - .connect(self.provider.clone(), alpn) - .await - .with_context(|| format!("failed to connect to provider {}", self.provider.id))?; - Ok(IrohTransport::new(connection)) - } -} - -async fn connect_chain( - config: &WorkConfig, - next_validator: &mut usize, -) -> CliResult> { - let verifier = ConsensusVerifier::new(&ConsensusInfo { - validators: config.validators.clone(), - threshold_identity: config.chain.threshold_identity.clone(), - network_id: config.chain.network.as_str().to_owned(), - }) - .context("configured threshold identity is unusable")?; - // A peer can accept connections while lacking a historical certificate. - // Reconnects must make progress through the configured alternatives. - let start = *next_validator; - let mut failures = Vec::new(); - for url in config - .validators - .iter() - .cycle() - .skip(start) - .take(config.validators.len()) - { - *next_validator = (*next_validator + 1) % config.validators.len(); - match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { - Ok(client) => return Ok(WorkBlocks::new(client)), - Err(error) => failures.push(format!("{url}: {error}")), - } - } - bail!("no configured validator answered: {}", failures.join("; ")) -} - -async fn check_genesis( - config: &WorkConfig, - chain: &WorkBlocks, -) -> CliResult<()> { - let first = chain - .block_at(1) - .await? - .context( - "configured validator has no finalized block 1; genesis cannot be authenticated until block 1 is finalized", - )?; - check_genesis_payload( - config.chain.genesis_payload_digest.as_bytes(), - &first.parent, + hellas_sdk::paid_client::deadlines( + current, + args.acceptance_blocks, + args.terminal_blocks, + args.payment_blocks, ) + .map_err(Into::into) } -fn check_genesis_payload(expected: &[u8; 32], actual: &[u8; 32]) -> CliResult<()> { - anyhow::ensure!( - actual == expected, - "validator genesis payload {} does not match configured {}", - hex::encode(actual), - hex::encode(expected), - ); - Ok(()) -} - -async fn finalized_floor(chain: &WorkBlocks) -> CliResult { - let height = chain - .latest_height() - .await? - .context("configured validator has finalized no blocks")?; - let block = chain - .block_at(height) - .await? - .context("configured validator did not return its finalized tip")?; - Ok(SetupScan { - height, - payload: block.payload, - }) -} - -fn read_prepared_input(path: &Path) -> CliResult { +fn read_prepared_work_input(path: &Path) -> CliResult { let bytes = super::read_bounded_regular_file(path, "prepared paid input", MAX_RECORD_BYTES)?; - PreparedPaidInputV1::decode(&bytes, MAX_RECORD_BYTES) + PreparedPaidWorkInput::decode(&bytes, MAX_RECORD_BYTES) .map_err(|error| anyhow::anyhow!("invalid prepared paid input {}: {error}", path.display())) } -#[cfg(feature = "llm")] fn write_private(path: &Path, bytes: &[u8]) -> CliResult<()> { if let Some(parent) = path .parent() @@ -1254,7 +556,7 @@ fn empty_coins() -> List { ) } -fn fixed_hex(flag: &str, value: &str) -> CliResult<[u8; N]> { +pub(crate) fn fixed_hex(flag: &str, value: &str) -> CliResult<[u8; N]> { let bytes = hex::decode(value).with_context(|| format!("{flag} is not hex"))?; bytes .try_into() diff --git a/crates/cli/src/commands/paid_work/tests.rs b/crates/cli/src/commands/paid_work/tests.rs index f21cbdf5..42cae70b 100644 --- a/crates/cli/src/commands/paid_work/tests.rs +++ b/crates/cli/src/commands/paid_work/tests.rs @@ -2,38 +2,12 @@ use super::*; use hellas_kernel::NetworkId; use hellas_rpc::peers::PeerId; -#[test] -fn recovery_skips_only_permanent_delivery_refusals() { - use hellas_client::work::CollectResultError; - use hellas_work::work::{DeliverError, WorkRefusal}; - - for (refusal, permanent) in [ - (WorkRefusal::Declined, true), - (WorkRefusal::Expired, true), - (WorkRefusal::NotReady, false), - (WorkRefusal::Unavailable, false), - ] { - let delivery = || DeliverError::Refused { - refusal, - reason: "provider diagnostic".to_owned(), - }; - assert_eq!(permanently_refused_delivery(&delivery().into()), permanent); - assert_eq!( - permanently_refused_delivery(&CollectResultError::Deliver(delivery()).into()), - permanent, - ); - } - assert!(!permanently_refused_delivery( - &DeliverError::Malformed("result").into() - )); - assert!(!permanently_refused_delivery(&anyhow::anyhow!( - "connection lost" - ))); -} - #[test] fn relative_deadlines_are_ordered_from_the_current_cursor() { let args = RunArgs { + provider_genesis: None, + apple_app_id: None, + apple_cd_hashes: Vec::new(), work_config: "work.json".into(), journal_root: "journal".into(), provider: SecretKey::generate().public(), @@ -79,11 +53,10 @@ fn genesis_check_compares_the_configured_digest_with_block_ones_parent() { assert!(check_genesis_payload(&configured, &configured).is_ok()); let observed_parent = [0x32; 32]; - let error = check_genesis_payload(&configured, &observed_parent) - .unwrap_err() - .to_string(); - assert!(error.contains(&hex::encode(observed_parent)), "{error}"); - assert!(error.contains(&hex::encode(configured)), "{error}"); + let error = check_genesis_payload(&configured, &observed_parent).unwrap_err(); + assert!(matches!(error, + hellas_sdk::paid_client::PaidClientError::GenesisMismatch { expected, actual } + if expected == configured && actual == observed_parent)); } #[test] @@ -158,7 +131,10 @@ fn prepare_input_builds_a_bundle_from_an_environment_and_prompt() { ) .unwrap(); - let prepared = read_prepared_input(&output).unwrap(); + let PreparedPaidWorkInput::Evaluate(prepared) = read_prepared_work_input(&output).unwrap() + else { + panic!("an Evaluate bundle"); + }; let parts = prepared.parts().unwrap(); assert_eq!( parts @@ -180,3 +156,39 @@ fn prepare_input_builds_a_bundle_from_an_environment_and_prompt() { environment.manifest().content_id(), ); } +#[test] +fn prepare_fetch_signs_an_ephemeral_client_request() { + let root = tempfile::tempdir().unwrap(); + let payload_file = root.path().join("request.json"); + let out = root.path().join("input.bin"); + std::fs::write(&payload_file, br#"{"input":"private prompt"}"#).unwrap(); + let key = hellas_rpc::ProducerSigningKey::from_secret_bytes([7; 32]).unwrap(); + prepare_fetch( + PrepareFetchArgs { + assurance: "producer-signed".into(), + service: "openai".into(), + method: "responses".into(), + execution_environment: "openai-responses".into(), + payload_file, + out: out.clone(), + }, + &key, + ) + .unwrap(); + let PreparedPaidWorkInput::Fetch(input) = read_prepared_work_input(&out).unwrap() else { + panic!("a Fetch bundle"); + }; + let parts = input.parts().unwrap(); + let request = hellas_rpc::fetch::verify_input_events(&parts.fetch_input_transcript).unwrap(); + assert_eq!(request.retention, hellas_rpc::Retention::Ephemeral); + assert_eq!(request.caller_key, key.public_key()); + assert_eq!(request.execution_environment, parts.manifest.content_id()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + assert_eq!( + std::fs::metadata(out).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } +} diff --git a/crates/cli/src/commands/serve/mod.rs b/crates/cli/src/commands/serve/mod.rs index 94b6525f..b42787f4 100644 --- a/crates/cli/src/commands/serve/mod.rs +++ b/crates/cli/src/commands/serve/mod.rs @@ -165,11 +165,13 @@ async fn run_with_store( ); work_runner = Some(node::WorkRunnerConfig { network: work.chain.network, + genesis_payload_digest: work.chain.genesis_payload_digest, threshold_identity: work.chain.threshold_identity.clone(), journal_root: work.journal_root.clone(), routes: work.routes.clone(), validators: work.validators.clone(), poll: work.poll, + max_observation_age: work.max_observation_age, settlement_key: options.settlement_key.clone(), policy: work.provider_policy(), }); @@ -350,6 +352,10 @@ struct FetchConfigRoute { #[derive(Debug, Deserialize)] #[serde(tag = "type", rename_all = "kebab-case", deny_unknown_fields)] enum FetchDestination { + /// Caller-signed HTTPS URL and TLS settings with operator-owned account aliases. + Http { + config: hellas_providers::HttpProviderConfig, + }, /// Official Codex Responses, authenticated by the local Codex OAuth store. CodexResponses { #[serde(default)] @@ -371,6 +377,7 @@ impl FetchDestination { fn into_entry(self, capabilities: FetchRoutePolicy) -> CliResult { let (environment, provider): (FetchEnvironment, Arc) = match self { + Self::Http { config } => return Ok(config.into_entry(capabilities)?), Self::CodexResponses { auth_path } => ( FetchEnvironment::CodexResponses, Arc::new(codex_provider::CodexResponsesFetchProvider::new( diff --git a/crates/cli/src/commands/serve/node.rs b/crates/cli/src/commands/serve/node.rs index db002236..a2b38ab6 100644 --- a/crates/cli/src/commands/serve/node.rs +++ b/crates/cli/src/commands/serve/node.rs @@ -8,18 +8,15 @@ //! owned by the service-discovery path and is not started from this //! bootstrap. -use std::collections::BTreeMap; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; +#[cfg(test)] +use std::sync::Mutex; use std::time::Duration; use anyhow::Context; -use futures::future::BoxFuture; -use hellas_chain::client::VerifiedRemoteLightClient; -use hellas_chain::work_blocks::{PaidWorkClockError, advance_paid_work_clock}; -use hellas_chain::{ - ConsensusInfo, ConsensusVerifier, FinalizedWorkView, WorkBlocks, WorkChannelQuery, -}; +#[cfg(test)] +use hellas_chain::FinalizedWorkView; #[cfg(feature = "evaluate")] use hellas_executor::ArtifactStoreConfig; #[cfg(feature = "evaluate")] @@ -29,53 +26,59 @@ use hellas_executor::{ FetchAccessPolicy, FetchQuotaStoreBackend, FetchRouteRegistry, FetchServer, FetchTranscriptStoreBackend, }; +#[cfg(test)] use hellas_kernel::{EdgeId, NetworkId, Secp256k1Signer, Secp256k1Verifier}; use hellas_rpc::cache::control::CacheController; use hellas_rpc::open::OpenDispatcher; +#[cfg(test)] use hellas_rpc::pb::work::{ AcceptWorkRequest, AcceptWorkResponse, AdmitCertificateRequest, AdmitCertificateResponse, DeliverResultRequest, DeliverResultResponse, ExchangeSetupRequest, ExchangeSetupResponse, WorkRefusalCode, WorkRefused, accept_work_response, admit_certificate_response, deliver_result_response, exchange_setup_response, }; -use hellas_rpc::peers::{PeerDirectory, PeerId, PeerManager}; +use hellas_rpc::peers::PeerId; +use hellas_rpc::peers::{PeerDirectory, PeerManager}; use hellas_rpc::policy::ExecutePolicy; -use hellas_rpc::protocol::Digest; -use hellas_rpc::protocol::work::{ - PaidJobAuthorizationV1, PrivateRecord as _, work_id as accepted_work_id, -}; -use hellas_rpc::protocol::work_setup::{ - ProviderChannelPolicy, ReadyChannel, WorkChannelDescriptor, -}; use hellas_rpc::serve::{AccountingDispatcher, AdminPolicy, Authorized, MethodDispatcher}; use hellas_rpc::services::cache_control::{CacheControl, CacheControlServer}; use hellas_rpc::services::courtesy::{Courtesy, Open as CourtesyOpen}; use hellas_rpc::services::execute::RunTicket; use hellas_rpc::services::fetch::{Fetch, Open as FetchOpen}; use hellas_rpc::services::node::{Node, NodeServer}; -use hellas_rpc::services::work::{Work, WorkHandler, WorkServer}; -use hellas_rpc::services::work_setup::{WorkSetup, WorkSetupHandler, WorkSetupServer}; +use hellas_rpc::services::work::{Work, WorkServer}; +use hellas_rpc::services::work_setup::{WorkSetup, WorkSetupServer}; +#[cfg(test)] +use hellas_rpc::services::{work::WorkHandler, work_setup::WorkSetupHandler}; use hellas_rpc::{Assurance, ProducerSigningKey}; use hellas_wire::iroh::{IrohTransport, IrohTransportError}; -use hellas_wire::{Dispatcher, ServiceMarker, StreamTransport, TransportContext, WireStatus}; -use hellas_work::work::{ - CloseEndpoint, PaidEvaluateBackend, RunError, RunOutcome, WorkService, run_accepted_work, -}; -use hellas_work::work_close::{CatchUpError, FinalizedBlocks, TxSink}; -use hellas_work::work_handshake::{PaymentAdmission, SetupEndpoint, SetupService}; -use hellas_work::work_open::{SetupAdvance, SetupDriveError, SetupProgress, SetupView}; -use hellas_work::work_store::{ChannelStore, JobPhase, Role, SetupStore, discover_setups}; +use hellas_wire::{Dispatcher, ServiceMarker, StreamTransport}; +#[cfg(test)] +use hellas_wire::{TransportContext, WireStatus}; +#[cfg(test)] +use hellas_work::work::PaidWorkBackend; +#[cfg(test)] +use hellas_work::work_close::FinalizedBlocks; +#[cfg(test)] +use hellas_work::work_close::TxSink; +#[cfg(test)] +use hellas_work::work_handshake::{PaymentAdmission, SetupEndpoint}; +#[cfg(test)] +use hellas_work::work_open::SetupView; +#[cfg(test)] +use hellas_work::work_store::{ChannelStore, JobPhase, Role, SetupStore}; use iroh::{Endpoint, EndpointId, SecretKey, endpoint::Connection, endpoint::presets}; -use tokio::sync::{Mutex as AsyncMutex, Semaphore, mpsc, oneshot}; +use tokio::sync::{Semaphore, mpsc, oneshot}; use tokio::task::{JoinHandle, JoinSet}; use tracing::{debug, info, warn}; use super::node_handler::NodeHandlerImpl; -use super::work_config::WorkRoutes; use crate::commands::discovery::{DiscoveryAdvertiser, served_alpns, start_server_advertising}; use crate::identity::OpenIdentity; -type ProductionWorkSource = WorkBlocks; +pub(super) use hellas_sdk::paid_provider::{ + MountedSetup, MountedWork, UnmountedWork, WorkRunner, WorkRunnerConfig, +}; /// Keep peer-controlled transport state finite. A connection can multiplex /// several RPCs, so these are deliberately transport limits rather than job @@ -199,29 +202,30 @@ pub(super) async fn spawn_node(config: NodeConfig) -> anyhow::Result .with_store(FetchQuotaStoreBackend::fs( config.artifact_store_path.join("fetch-quota"), )); - let handle = Executor::spawn_configured(ExecutorSpawnConfig { - output_cache: config.output_cache, - execute_policy: config.execute_policy, - queue_capacity: config.queue_size, - metrics: config.metrics.clone(), - producer_key: Arc::new(config.producer_key), - provider_genesis: Arc::new(config.provider_genesis), - assurance: config.assurance, - fetch_access_policy, - fetch_routes: config.fetch_routes, - fetch_max_in_flight: config.fetch_max_in_flight, - fetch_queue_capacity: config.fetch_queue_size, - fetch_replay_max_in_flight: config.fetch_replay_max_in_flight, - fetch_store, - #[cfg(feature = "evaluate")] - artifact_store: config.artifact_store, - #[cfg(feature = "evaluate")] - content_store: config.content_store, - #[cfg(feature = "evaluate")] - gpu_config: config.gpu_config, - }) - .await - .context("failed to spawn executor")?; + let mut executor = ExecutorSpawnConfig::fetch_only( + Arc::new(config.producer_key), + Arc::new(config.provider_genesis), + config.assurance, + config.fetch_routes, + ); + executor.output_cache = config.output_cache; + executor.execute_policy = config.execute_policy; + executor.queue_capacity = config.queue_size; + executor.metrics = config.metrics.clone(); + executor.fetch_access_policy = fetch_access_policy; + executor.fetch_max_in_flight = config.fetch_max_in_flight; + executor.fetch_queue_capacity = config.fetch_queue_size; + executor.fetch_replay_max_in_flight = config.fetch_replay_max_in_flight; + executor.fetch_store = fetch_store; + #[cfg(feature = "evaluate")] + { + executor.artifact_store = config.artifact_store; + executor.content_store = config.content_store; + executor.gpu_config = config.gpu_config; + } + let handle = Executor::spawn_configured(executor) + .await + .context("failed to spawn executor")?; let advertised_alpns = served_alpns(config.work.is_some()); let mut alpns = advertised_alpns.clone(); if !admin_policy.peers.is_empty() { @@ -280,7 +284,7 @@ pub(super) async fn spawn_node(config: NodeConfig) -> anyhow::Result executor: handle.clone(), open_identity: config.open_identity, }; - let work_mount: MountedWork = MountedWork::with_backend(handle); + let work_mount: MountedWork = MountedWork::with_backend(handle); let setup_mount = MountedSetup::default(); let work = config.work.map(|work| { let poll = work.poll; @@ -377,18 +381,15 @@ pub(super) async fn spawn_node(config: NodeConfig) -> anyhow::Result /// Per-connection serve: each inbound substream is dispatched to the /// service selected by the connection's negotiated ALPN. -async fn serve_connection( +async fn serve_connection( alpn: Vec, conn: Connection, remote_execution: RemoteExecutionServices, node_handler: NodeHandlerImpl, manager: PeerManager, setup: Option, - work: Option>, -) -> anyhow::Result<()> -where - S: FinalizedBlocks + FinalizedWorkView + Sync, -{ + work: Option, +) -> anyhow::Result<()> { let transport = Arc::new(IrohTransport::new(conn)); let context = transport.context(); @@ -434,11 +435,23 @@ where { match setup.service(&context) { Some(mounted) => { - let server = AccountingDispatcher::new(WorkSetupServer(mounted), manager); + let server = AccountingDispatcher::new( + OpenDispatcher::<_, _, hellas_rpc::services::work_setup::Open>::new( + WorkSetupServer(mounted), + remote_execution.open_identity.clone(), + ), + manager, + ); serve_loop(transport, server).await } None => { - let server = AccountingDispatcher::new(WorkSetupServer(UnmountedWork), manager); + let server = AccountingDispatcher::new( + OpenDispatcher::<_, _, hellas_rpc::services::work_setup::Open>::new( + WorkSetupServer(UnmountedWork), + remote_execution.open_identity.clone(), + ), + manager, + ); serve_loop(transport, server).await } } @@ -448,11 +461,23 @@ where // is still the bounded retryable `NotReady` §3 left here. match work.handler(&context) { Some(mounted) => { - let server = AccountingDispatcher::new(WorkServer(mounted), manager); + let server = AccountingDispatcher::new( + OpenDispatcher::<_, _, hellas_rpc::services::work::Open>::new( + WorkServer(mounted), + remote_execution.open_identity.clone(), + ), + manager, + ); serve_loop(transport, server).await } None => { - let server = AccountingDispatcher::new(WorkServer(UnmountedWork), manager); + let server = AccountingDispatcher::new( + OpenDispatcher::<_, _, hellas_rpc::services::work::Open>::new( + WorkServer(UnmountedWork), + remote_execution.open_identity.clone(), + ), + manager, + ); serve_loop(transport, server).await } } @@ -462,1130 +487,6 @@ where } } -#[derive(Clone, Copy, Debug)] -struct UnmountedWork; - -fn not_ready() -> WorkRefused { - WorkRefused { - code: WorkRefusalCode::NotReady as i32, - reason: "work state is not mounted".to_string(), - } -} - -impl WorkSetupHandler for UnmountedWork { - fn exchange_setup( - &self, - _request: ExchangeSetupRequest, - _context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - core::future::ready(Ok(ExchangeSetupResponse { - outcome: Some(exchange_setup_response::Outcome::Refused(not_ready())), - })) - } -} - -impl WorkHandler for UnmountedWork { - fn accept_work( - &self, - _request: AcceptWorkRequest, - _context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - core::future::ready(Ok(AcceptWorkResponse { - outcome: Some(accept_work_response::Outcome::Refused(not_ready())), - })) - } - - fn deliver_result( - &self, - _request: DeliverResultRequest, - _context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - core::future::ready(Ok(DeliverResultResponse { - outcome: Some(deliver_result_response::Outcome::Refused(not_ready())), - })) - } - - async fn stream_result( - &self, - _request: DeliverResultRequest, - _context: TransportContext, - ) -> Result { - Err(WireStatus::new( - hellas_wire::WireCode::Unavailable, - "paid work channel is not mounted", - )) - } - - fn admit_certificate( - &self, - _request: AdmitCertificateRequest, - _context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - core::future::ready(Ok(AdmitCertificateResponse { - outcome: Some(admit_certificate_response::Outcome::Refused(not_ready())), - })) - } -} - -// ── The clock ───────────────────────────────────────────────────────── -// -// Everything below is a runner and nothing below is a decision. It -// builds no transaction, fixes no deadline, chooses no settlement, -// judges no duty due, and does not decide whether admission is on: each -// of those is a library edge it calls on a cadence, and the cadence is -// the whole of what this file adds. What it owns is *when* — and the -// journals, which is why it hands them to nobody. - -/// What the clock over one node's paid-work journals is built from. -/// -/// Every field is something the serve path has already loaded and -/// checked. The policy most of all: it is the loaded work -/// configuration's, carried here rather than derived again, so there is -/// no second place a node could decide what it countersigns over. -pub(super) struct WorkRunnerConfig { - /// The network the journals are keyed and the signatures bound to. - pub(super) network: NetworkId, - /// The threshold identity finalized blocks must authenticate under. - pub(super) threshold_identity: Vec, - /// The configured root the setup journals live under. - pub(super) journal_root: PathBuf, - /// Bilateral routes from authenticated peers to owned journals. - pub(super) routes: WorkRoutes, - /// The validator RPCs a read and a submission go to. - pub(super) validators: Vec, - /// How often the clock ticks. - pub(super) poll: Duration, - /// The key every settlement this node signs is signed with. - pub(super) settlement_key: Secp256k1Signer, - /// What every setup endpoint this node builds countersigns over. - pub(super) policy: ProviderChannelPolicy, -} - -/// The channel this node answers `Work` from, once the runner has been -/// handed one. -/// -/// Written by the runner and read by the accept loop. What crosses is a -/// clone of a handler whose mutable pieces are themselves behind `Arc`s, -/// so the lock is held for a clone and never across a request: the -/// dispatch path never waits while holding the mount, and the clock never -/// waits on a request. -#[derive(Clone)] -pub(super) struct MountedWork { - mounted: Arc>>>>, - driver: Option, -} - -impl Default for MountedWork { - fn default() -> Self { - Self { - mounted: Arc::new(Mutex::new(BTreeMap::new())), - driver: None, - } - } -} - -/// A cloneable, type-erased owner of the backend that runs accepted work. -/// -/// The production value owns an [`hellas_executor::ExecutorHandle`]. -/// Keeping the backend behind this narrow local seam means the clock and -/// ALPN dispatcher stay parameterized only over their finalized source; -/// neither has a second opinion about paid admission or execution failure. -#[derive(Clone)] -struct AcceptedWorkDriver(Arc); - -trait DriveAcceptedWork: Send + Sync { - fn run( - &self, - service: WorkService, - ready: ReadyChannel, - work_id: Digest, - ) -> BoxFuture<'static, Result>; -} - -struct BackendWorkDriver { - backend: Arc, -} - -impl DriveAcceptedWork for BackendWorkDriver -where - B: PaidEvaluateBackend + Send + Sync + 'static, -{ - fn run( - &self, - service: WorkService, - ready: ReadyChannel, - work_id: Digest, - ) -> BoxFuture<'static, Result> { - let backend = Arc::clone(&self.backend); - Box::pin( - async move { run_accepted_work(&service, &ready, backend.as_ref(), work_id).await }, - ) - } -} - -impl AcceptedWorkDriver { - fn new(backend: B) -> Self - where - B: PaidEvaluateBackend + Send + Sync + 'static, - { - Self(Arc::new(BackendWorkDriver { - backend: Arc::new(backend), - })) - } - - /// Starts one accepted job without lending its lifetime to either the - /// request path or the close clock. - fn spawn(&self, service: WorkService, ready: ReadyChannel, work_id: Digest) { - let running = self.0.run(service, ready, work_id); - let span = hellas_rpc::request_span!(target: "hellas_request", "paid.provider.execute", hellas.work.id = ?work_id, otel.status_code = tracing::field::Empty); - tokio::spawn(tracing::Instrument::instrument( - async move { - match running.await { - Ok(RunOutcome::Completed { .. }) => { - debug!(?work_id, "the accepted paid job completed") - } - Ok(RunOutcome::Ready { .. }) => { - debug!(?work_id, "the accepted paid job was already complete") - } - Ok(RunOutcome::Running) => { - debug!(?work_id, "the accepted paid job was already running") - } - Ok(RunOutcome::Indeterminate) => { - warn!( - ?work_id, - "the accepted paid job is indeterminate after restart" - ) - } - // `run_accepted_work` has already made backend and - // transcript faults terminal before returning them. The - // remaining errors have no node-local terminal policy; - // keep the exact failure visible to the operator. - Err(error) => { - tracing::Span::current().record("otel.status_code", "ERROR"); - warn!(?work_id, %error, "the accepted paid job did not complete"); - } - } - }, - span, - )); - } -} - -/// One mounted channel's served handler. -/// -/// `source` is replaceable because the runner redials a failed validator. -/// The request path copies the current source under the plain mutex and -/// drops that guard before its coherent read awaits. `accepting` spans the -/// complete fresh-read-to-signature sequence, so two acceptance attempts -/// cannot each refresh and then race to consume the same channel credit. -#[derive(Clone)] -struct MountedWorkService { - bond_edge: EdgeId, - service: WorkService, - descriptor: Option, - source: Arc>, - accepting: Arc>, - driver: Option, -} - -impl MountedWorkService -where - S: FinalizedBlocks + FinalizedWorkView + Sync, -{ - /// Re-establishes admission from one fresh coherent read. - /// - /// The service is the exact clone the runner drives. Its cursor is - /// checked after readiness, and that same service receives the fresh - /// decision before the raw handler is reached. A missing policy, - /// failed read, failed predicate, lagging cursor, or endpoint failure - /// therefore leaves the request on the retryable `NotReady` side. - async fn refresh_admission(&self) -> anyhow::Result { - // A `std::sync::MutexGuard` is deliberately confined to this - // block. Holding the source-slot guard across the read would make - // this handler's future non-`Send` and is not a valid dispatch. - let source = { - let held = self - .source - .lock() - .map_err(|_| anyhow::anyhow!("the finalized source lock is poisoned"))?; - held.clone() - }; - refresh_work_admission(&self.service, self.descriptor.as_ref(), &source).await - } - - async fn refresh_delivery(&self, request: &DeliverResultRequest) -> anyhow::Result<()> { - let Ok(bytes) = request.work_id.as_slice().try_into() else { - return Ok(()); - }; - let work_id = Digest::from_bytes(bytes); - let active = self - .service - .with_state(|state| state.job_by_id(work_id).is_some()); - if !matches!(active, Ok(true)) { - return Ok(()); - } - // A restarted mount has no readiness cached. A retained result must - // be collectable without first accepting another job. Terminal replies - // need no fresh admission and are authenticated by the core service. - let _accepting = self.accepting.lock().await; - let result = self.refresh_admission().await; - if let Err(error) = &result { - debug!(%error, "a delivery attempt found no fresh channel readiness"); - } - result.map(|_| ()) - } -} - -/// Re-establishes admission for the exact driven channel from one coherent -/// finalized read. -/// -/// Both the wire handler and restart recovery call this function. A recovered -/// job therefore gets no weaker interpretation of readiness than a new job, -/// and neither path can accidentally trust the readiness cached at mount. -async fn refresh_work_admission( - service: &WorkService, - descriptor: Option<&WorkChannelDescriptor>, - source: &S, -) -> anyhow::Result -where - S: FinalizedBlocks + FinalizedWorkView + Sync, -{ - let Some(descriptor) = descriptor else { - anyhow::bail!("this channel has no admission descriptor"); - }; - let query = WorkChannelQuery { - bond_edge: descriptor.bond_edge(), - payment_edge: descriptor.channel().payment_edge(), - funding: Default::default(), - }; - let Some(snapshot) = source - .work_channel_snapshot(query.clone()) - .await - .context("the fresh coherent channel read failed")? - else { - anyhow::bail!("no finalized channel snapshot is available"); - }; - if snapshot.query() != &query { - anyhow::bail!("the finalized source answered for another channel"); - } - let ready = descriptor - .check_ready(&snapshot.observed_channel()) - .context("the fresh channel snapshot is not ready")?; - // Keep one snapshot as the target. Recovery runs on the clock itself, - // so waiting for another tick here would prevent the cursor advancing. - // Take the service's existing driver when available; a concurrent clock - // drive keeps that authority until it finishes its own catch-up. - for _ in 0..16 { - let cursor = service - .with_state(|state| state.cursor().0) - .context("the mounted channel cursor is unavailable")?; - if ready.check_caught_up(cursor).is_ok() { - break; - } - if let Ok(mut driver) = service.drive() { - driver - .catch_up(source) - .await - .context("the mounted channel could not catch up to the fresh snapshot")?; - } - tokio::time::sleep(Duration::from_millis(250)).await; - } - let cursor = service - .with_state(|state| state.cursor().0) - .context("the mounted channel cursor is unavailable")?; - ready - .check_caught_up(cursor) - .context("the mounted channel has not caught up to the fresh snapshot")?; - service - .admit_new_work(ready.clone()) - .context("the driven work service refused its fresh readiness")?; - Ok(ready) -} - -impl WorkHandler for MountedWorkService -where - S: FinalizedBlocks + FinalizedWorkView + Sync, -{ - async fn accept_work( - &self, - request: AcceptWorkRequest, - _context: TransportContext, - ) -> Result> + Send, WireStatus> - { - if let Some(response) = self.service.precheck_acceptance(&request) { - return Ok(response); - } - let _accepting = self.accepting.lock().await; - // A preceding request or the clock may have resolved this proposal - // while admission was serialized. Retained replies need no fresh read. - if let Some(response) = self.service.precheck_acceptance(&request) { - return Ok(response); - } - let ready = match self.refresh_admission().await { - Ok(ready) => ready, - Err(error) => { - debug!(%error, "an acceptance attempt found no fresh channel readiness"); - return Ok(AcceptWorkResponse { - outcome: Some(accept_work_response::Outcome::Refused(WorkRefused { - code: WorkRefusalCode::NotReady as i32, - reason: "fresh channel readiness is unavailable".to_string(), - })), - }); - } - }; - // Derive the id from the request while the accepted response - // is still only a possibility. The response carries only the - // provider signature, and consulting `state.jobs().next()` after it - // leaves would race the clock terminalizing that same job. - let work_id = self - .service - .with_state(|state| { - PaidJobAuthorizationV1::decode(&request.authorization) - .ok() - .map(|authorization| accepted_work_id(state.channel(), &authorization)) - }) - .ok() - .flatten(); - let response = self.service.accept(&request); - if matches!( - response.outcome.as_ref(), - Some(accept_work_response::Outcome::Accepted(_)) - ) { - match (self.driver.as_ref(), work_id) { - (Some(driver), Some(work_id)) => { - driver.spawn(self.service.clone(), ready, work_id); - } - (None, Some(work_id)) => { - warn!(?work_id, "accepted paid work has no execution backend") - } - (_, None) => warn!("accepted paid work has no mounted job to execute"), - } - } - Ok(response) - } - - async fn deliver_result( - &self, - request: DeliverResultRequest, - context: TransportContext, - ) -> Result> + Send, WireStatus> - { - let response: hellas_rpc::call::WithTrailer = - if self.refresh_delivery(&request).await.is_ok() { - self.service.deliver_result(request, context).await?.into() - } else { - DeliverResultResponse { - outcome: Some(deliver_result_response::Outcome::Refused(WorkRefused { - code: WorkRefusalCode::NotReady as i32, - reason: "fresh channel readiness is unavailable".to_string(), - })), - } - .into() - }; - Ok(response) - } - - async fn stream_result( - &self, - request: DeliverResultRequest, - context: TransportContext, - ) -> Result { - self.refresh_delivery(&request).await.map_err(|_| { - WireStatus::new( - hellas_wire::WireCode::Unavailable, - "fresh channel readiness is unavailable", - ) - })?; - let mounted = self.clone(); - let mut stream = self.service.stream_result(request.clone(), context).await?; - Ok(Box::pin(async_stream::try_stream! { - let mut refresh = tokio::time::interval(Duration::from_secs(1)); - loop { - let event = tokio::select! { - event = futures::StreamExt::next(&mut stream) => Some(event), - _ = refresh.tick() => None, - }; - // Even buffered prefixes need a fresh finalized decision before - // leaving the node; a contest can start after this stream opens. - mounted.refresh_delivery(&request).await.map_err(|_| WireStatus::new( - hellas_wire::WireCode::Unavailable, - "fresh channel readiness is unavailable", - ))?; - if let Some(event) = event { - match event { - Some(event) => yield event?, - None => break, - } - } - } - })) - } - - fn admit_certificate( - &self, - request: AdmitCertificateRequest, - context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - self.service.admit_certificate(request, context) - } -} - -impl MountedWork { - fn with_backend(backend: B) -> Self - where - B: PaidEvaluateBackend + Send + Sync + 'static, - { - Self { - mounted: Arc::new(Mutex::new(BTreeMap::new())), - driver: Some(AcceptedWorkDriver::new(backend)), - } - } - - /// Adds one owned channel under its authenticated peer. - /// - /// A peer is served only while exactly one channel is mounted under - /// it. Retaining a second candidate rather than overwriting either one - /// makes an ambiguity fail closed instead of turning insertion order - /// into routing policy. - fn mount( - &self, - peer: PeerId, - bond_edge: EdgeId, - service: &WorkService, - descriptor: Option, - accepting: Arc>, - source: &S, - ) -> bool { - match self.mounted.lock() { - Ok(mut held) => { - let mounted = held.entry(peer).or_default(); - mounted.push(MountedWorkService { - bond_edge, - service: service.clone(), - descriptor, - source: Arc::new(Mutex::new(source.clone())), - accepting, - driver: self.driver.clone(), - }); - mounted.len() == 1 - } - Err(_) => false, - } - } - - /// The one handler mounted for the transport-vouched peer. - fn handler(&self, context: &TransportContext) -> Option> { - let peer = context - .vouched_peer() - .map(|peer| PeerId::from_bytes(peer.0))?; - self.mounted.lock().ok().and_then(|held| { - let [mounted] = held.get(&peer)?.as_slice() else { - return None; - }; - Some(mounted.clone()) - }) - } - - /// The mounted channel's service, reached through the served - /// handler. - /// - /// Only the tests below ask for it. Serving takes [`Self::handler`], - /// and the clock drives the very same [`WorkService`] out of - /// [`Driven::Channel`] without asking the mount for it — so this - /// exists to let a test read the state of what the node is actually - /// answering from, and scoping it to `cfg(test)` is what keeps the - /// served slot to the one accessor that serves. - #[cfg(test)] - fn service(&self, context: &TransportContext) -> Option { - self.handler(context).map(|mounted| mounted.service) - } - - /// Replaces the finalized source for the matching driven channel. - /// - /// A reconnect reaches handlers already cloned by live connections, - /// because they share this inner source slot. Neither mount lock is - /// held across a source request. - fn refresh_source(&self, peer: PeerId, bond_edge: EdgeId, source: &S) { - let source_slot = self.mounted.lock().ok().and_then(|held| { - held.get(&peer)? - .iter() - .find(|mounted| mounted.bond_edge == bond_edge) - .map(|mounted| Arc::clone(&mounted.source)) - }); - if let Some(source_slot) = source_slot - && let Ok(mut held) = source_slot.lock() - { - *held = source.clone(); - } - } - - /// Stops serving `Work` from every channel. - /// - /// The clock's last act. A channel nobody is advancing is not a - /// channel to answer from — its journal is closed the moment the - /// runner drops it, and a handler still holding it open would be the - /// one thing keeping the files this process no longer owns. - fn clear_all(&self) { - if let Ok(mut held) = self.mounted.lock() { - held.clear(); - } - } -} - -/// Provider setups this node answers `WorkSetup` from by authenticated peer. -/// -/// Written by discovery and read by the accept loop, beside -/// [`MountedWork`]. The clone in this slot is the exact [`SetupService`] -/// stored in [`Driven::Setup`], so serving and driving share one exclusive -/// journal rather than attempting to reopen it. -#[derive(Clone, Debug, Default)] -pub(super) struct MountedSetup(Arc>>>); - -#[derive(Clone, Debug)] -struct MountedSetupService { - bond_edge: EdgeId, - service: SetupService, -} - -impl MountedSetup { - /// Adds one owned setup under its authenticated peer. - fn mount(&self, peer: PeerId, bond_edge: EdgeId, service: &SetupService) -> bool { - match self.0.lock() { - Ok(mut held) => { - let mounted = held.entry(peer).or_default(); - mounted.push(MountedSetupService { - bond_edge, - service: service.clone(), - }); - mounted.len() == 1 - } - Err(_) => false, - } - } - - /// The exact setup service mounted for the transport-vouched peer, - /// cloned without holding the map across dispatch. - fn service(&self, context: &TransportContext) -> Option { - let peer = context - .vouched_peer() - .map(|peer| PeerId::from_bytes(peer.0))?; - self.0.lock().ok().and_then(|held| { - let [mounted] = held.get(&peer)?.as_slice() else { - return None; - }; - Some(mounted.service.clone()) - }) - } - - /// Stops serving only the setup that made this transition. - fn clear(&self, peer: PeerId, bond_edge: EdgeId) { - if let Ok(mut held) = self.0.lock() - && let Some(mounted) = held.get_mut(&peer) - { - mounted.retain(|mounted| mounted.bond_edge != bond_edge); - if mounted.is_empty() { - held.remove(&peer); - } - } - } - - /// Stops serving every setup during runner shutdown. - fn clear_all(&self) { - if let Ok(mut held) = self.0.lock() { - held.clear(); - } - } -} - -/// One journal, and what the clock drives it as. -/// -/// Two live states and one transition between them: a setup is driven -/// until it hands back the channel it mounted, and from then on the -/// channel is what is driven. Nothing here re-derives a mount — -/// [`SetupAdvance::mounted`] is the only way a [`ChannelStore`] reaches -/// this file, and the setup is not driven again afterwards, because a -/// second step would open a second journal on the same file. -enum Driven { - /// The journal is driven behind the setup service that answers for - /// it. The policy is retained beside the service: it is the provider - /// authority from which the full channel descriptor is rebuilt after - /// the setup reveals its actual terms. - Setup { - /// The endpoint this journal is both driven and served behind. - service: SetupService, - /// The retained provider authority, behind a pointer. - /// - /// Boxed because it is the widest thing this enum carries by a - /// long way — every other payload here is a handle or a store - /// pointer, one or two words each — and a journal is one value - /// with three shapes, so the two that hold no policy would - /// otherwise each be as large as the one that does. - /// [`PaymentAdmission`] already holds it behind the same - /// indirection, and this is built from that one, once per - /// journal at startup. - policy: Box, - }, - /// The channel this setup mounted, including the recovery authority - /// needed to finish a job accepted before a process restart. - Channel(Box), - /// The setup ended, or its mount was refused. Nothing left to - /// drive. - Done, -} - -/// One mounted channel as driven by the paid-work clock. -/// -/// Recovery lives here rather than in the served route: an accepted job is an -/// obligation recorded by this journal even if peer routing changes while the -/// process is down. `accepting` is also lent to the route when one is mounted, -/// so live acceptance and restart recovery serialize their readiness checks. -struct DrivenChannel { - service: WorkService, - descriptor: Option, - accepting: Arc>, - driver: Option, -} - -impl DrivenChannel { - fn accepted_work_id(&self) -> anyhow::Result> { - self.service - .with_state(|state| { - let mut jobs = state.jobs(); - let job = jobs.next()?; - (jobs.next().is_none() && job.phase() == JobPhase::Accepted).then(|| job.work_id()) - }) - .context("the driven channel state is unavailable") - } - - /// Starts a journaled Accepted job after proving current readiness. - /// - /// No in-memory `attempted` marker is needed. A racing live request or - /// clock tick reaches the same endpoint; its durable `JobRunning` record - /// lets exactly one caller receive `Invoke` and every other caller receive - /// `Running`. - async fn resume_accepted(&self, source: &S) -> anyhow::Result - where - S: FinalizedBlocks + FinalizedWorkView + Sync, - { - if self.accepted_work_id()?.is_none() { - return Ok(false); - } - let _accepting = self.accepting.lock().await; - let Some(work_id) = self.accepted_work_id()? else { - return Ok(false); - }; - let driver = self - .driver - .as_ref() - .context("the accepted paid job has no execution backend")?; - let ready = refresh_work_admission(&self.service, self.descriptor.as_ref(), source).await?; - driver.spawn(self.service.clone(), ready, work_id); - Ok(true) - } -} - -/// One setup journal on a clock. -struct SetupClock { - /// The bond this journal stakes, so a log line names which one. - bond_edge: EdgeId, - /// The authenticated peer whose configured route names this bond. - /// Together with `bond_edge`, this is the journal's route identity; - /// `None` keeps an unconfigured owned journal on its close clock - /// without making it a fallback service. - route_peer: Option, - /// What is being driven for it. - driven: Driven, -} - -impl SetupClock { - /// Takes this journal's one step, and says whether the chain - /// answered. - /// - /// A source failure is the only outcome the caller acts on: a - /// validator that stopped answering is dialled again rather than - /// asked forever. Everything else is this journal's own business and - /// is logged where it happens. - async fn tick( - &mut self, - source: &S, - signer: &Secp256k1Signer, - work_mount: &MountedWork, - setup_mount: &MountedSetup, - ) -> bool - where - S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, - { - let bond = hex::encode(self.bond_edge.to_bytes()); - let mut answered = true; - // One step of the setup this journal holds, with the policy the - // mount it may hand back is rebuilt from; nothing, once the - // journal is past its setup. - let step = match &mut self.driven { - Driven::Setup { service, policy } => Some(( - service.advance_setup(source, source, source).await, - policy.clone(), - )), - Driven::Channel(_) | Driven::Done => None, - }; - if let Some((step, policy)) = step { - match step { - Ok(SetupAdvance { progress, mounted }) => { - if let Some(store) = mounted { - if let Some(peer) = self.route_peer { - setup_mount.clear(peer, self.bond_edge); - } - self.take_mount(store, signer, &policy, source, work_mount); - } else if matches!( - progress, - SetupProgress::Aborted(_) | SetupProgress::Faulted(_) - ) { - warn!(bond, ?progress, "this setup ended with no channel to drive"); - self.driven = Driven::Done; - } else { - debug!(bond, ?progress, "the setup advanced"); - } - } - Err(error) => { - answered = !matches!(error, SetupDriveError::Source(_)); - warn!(bond, %error, "this setup did not advance"); - } - } - } - if let Driven::Channel(channel) = &self.driven { - if let Some(peer) = self.route_peer { - work_mount.refresh_source(peer, self.bond_edge, source); - } - if let Err(error) = channel.resume_accepted(source).await { - warn!(bond, %error, "an accepted paid job did not resume"); - } - match advance_paid_work_clock(&channel.service, source).await { - Ok(progress) => debug!(bond, ?progress, "the channel advanced"), - // `resume_accepted` owns the channel while it starts the - // durable execution. The clock's concurrent close pass has - // no work to do until that owner returns the cursor. - Err(PaidWorkClockError::CloseDrive(CatchUpError::Busy)) => { - debug!(bond, "the channel is already being driven") - } - Err(error) => { - answered &= !error.source_failed(); - warn!(bond, %error, "this channel's close did not advance"); - } - } - } - answered - } - - /// Mounts the store the driver handed back. - /// - /// Handed back, never reopened: the journal is exclusive, so a - /// second `ChannelStore::open` on the same file is a refusal rather - /// than a second view, and the settlement and origin this one - /// carries are the ones the completing read established. - fn take_mount( - &mut self, - store: ChannelStore, - signer: &Secp256k1Signer, - policy: &ProviderChannelPolicy, - source: &S, - mount: &MountedWork, - ) { - let bond = hex::encode(self.bond_edge.to_bytes()); - // The setup's retained policy supplies the provider-controlled - // fields, while the mounted channel supplies the payment edge and - // complete terms the two parties actually signed. This is a full - // descriptor reconstruction, not a mount-time readiness cache. - let descriptor = { - let channel = store.state().channel(); - match policy.admit(channel.payment_edge(), channel.payment_terms().clone()) { - Ok(descriptor) => Some(descriptor), - Err(error) => { - warn!(bond, %error, "the mounted channel no longer satisfies its admission policy"); - None - } - } - }; - match CloseEndpoint::new(store, signer.clone()) { - Ok(close) => { - let service = WorkService::close_only(close); - let accepting = Arc::new(AsyncMutex::new(())); - if self.route_peer.is_some_and(|peer| { - mount.mount( - peer, - self.bond_edge, - &service, - descriptor.clone(), - Arc::clone(&accepting), - source, - ) - }) { - info!( - bond, - "this node now answers Work from the channel it mounted" - ); - } else { - warn!( - bond, - "this channel has no unique peer route; it is driven and not served", - ); - } - self.driven = Driven::Channel(Box::new(DrivenChannel { - service, - descriptor, - accepting, - driver: mount.driver.clone(), - })); - } - // The journal and the key are not both the provider's view - // of one channel. Nothing this runner can do about it, and - // dropping the mount is what stops it being reopened every - // tick. - Err(error) => { - warn!(bond, %error, "the mounted channel is not this node's to close"); - self.driven = Driven::Done; - } - } - } -} - -/// The clock, over every paid-work journal this node owns. -pub(super) struct WorkRunner { - clocks: Vec, - signer: Secp256k1Signer, - work_mount: MountedWork, - setup_mount: MountedSetup, - poll: Duration, - validators: Vec, - consensus_verifier: ConsensusVerifier, -} - -impl WorkRunner -where - S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, -{ - /// Opens every setup journal under the configured root. - /// - /// The root and the network are the whole of what a restarting node - /// is told; the bond each journal is about and the role it was - /// written at come out of the files, which is what `discover_setups` - /// is for. A journal that cannot be named is reported and not - /// skipped silently: a file this node cannot open may be a channel - /// it still owes a close. - /// - /// # Errors - /// - /// When the root itself cannot be enumerated. - /// Every owned journal is driven. A configured route additionally - /// mounts its exact setup service under the authenticated peer that - /// names it; an unconfigured journal remains a close duty, not a - /// fallback answer. - pub(super) fn discover( - config: WorkRunnerConfig, - work_mount: MountedWork, - setup_mount: MountedSetup, - ) -> anyhow::Result { - let consensus_verifier = ConsensusVerifier::new(&ConsensusInfo { - validators: config.validators.clone(), - threshold_identity: config.threshold_identity, - network_id: config.network.as_str().to_owned(), - }) - .context("the configured threshold identity is not usable")?; - let settlement_verifier = Secp256k1Verifier::new(); - let found = discover_setups(&config.journal_root, config.network).with_context(|| { - format!( - "failed to enumerate the work journals under {}", - config.journal_root.display(), - ) - })?; - for unnamed in &found.unidentified { - warn!( - path = %unnamed.path.display(), - reason = %unnamed.reason, - "a setup journal under the work root could not be named", - ); - } - let mut clocks = Vec::with_capacity(found.setups.len()); - for setup in found.setups { - let bond = hex::encode(setup.bond_edge.to_bytes()); - let route_peer = config - .routes - .iter() - .find(|route| route.bond == setup.bond_edge) - .map(|route| route.peer); - // A close capability binds the provider half, and this - // process holds the provider's key. A client journal beside - // this node's own is another party's, and this runner has - // nothing to sign for it. - if setup.role != Role::Provider { - warn!( - bond, - "a setup journal under the work root is not this node's half" - ); - continue; - } - let store = match SetupStore::open( - &config.journal_root, - config.network, - setup.bond_edge, - setup.role, - &settlement_verifier, - ) { - Ok(store) => store, - Err(error) => { - warn!(bond, %error, "a discovered setup journal did not open"); - continue; - } - }; - let policy = Box::new(config.policy.clone()); - let service = SetupService::new(SetupEndpoint::new( - store, - config.settlement_key.clone(), - PaymentAdmission::Admits(policy.clone()), - )); - if let Some(peer) = route_peer { - if setup_mount.mount(peer, setup.bond_edge, &service) { - info!( - bond, - "this node now answers WorkSetup from its driven setup" - ); - } else { - warn!(bond, "this provider setup has an ambiguous peer route"); - } - } else { - warn!(bond, "this provider setup has no configured peer route"); - } - let driven = Driven::Setup { service, policy }; - clocks.push(SetupClock { - bond_edge: setup.bond_edge, - route_peer, - driven, - }); - } - Ok(Self { - clocks, - signer: config.settlement_key, - work_mount, - setup_mount, - poll: config.poll, - validators: config.validators, - consensus_verifier, - }) - } - - /// Takes one step of every journal, and says whether the chain - /// answered all of them. - async fn tick(&mut self, source: &S) -> bool { - let mut answered = true; - for clock in &mut self.clocks { - answered &= clock - .tick(source, &self.signer, &self.work_mount, &self.setup_mount) - .await; - } - answered - } - - /// The loop, over whatever chain `dial` produces. - /// - /// One tick of every journal per period, and a chain that stopped - /// answering is dialled again rather than asked forever. The whole - /// of the cadence is here, and none of the decisions are. - async fn run_over(mut self, mut stop: oneshot::Receiver<()>, dial: D) - where - D: Fn() -> F, - F: core::future::Future>, - { - if self.clocks.is_empty() { - info!("no provider setup journal under the work root; the clock has nothing to drive"); - self.work_mount.clear_all(); - self.setup_mount.clear_all(); - return; - } - let mut chain = None; - loop { - tokio::select! { - _ = &mut stop => break, - () = tokio::time::sleep(self.poll) => {} - } - let Some(source) = chain.take() else { - chain = dial().await; - continue; - }; - if self.tick(&source).await { - chain = Some(source); - } - } - self.work_mount.clear_all(); - self.setup_mount.clear_all(); - info!("the paid-work clock stopped, and its journals are closed"); - } -} - -impl WorkRunner { - /// Ticks until told to stop, over the validators the configuration - /// names. - async fn run(self, stop: oneshot::Receiver<()>) { - let validators = self.validators.clone(); - let verifier = self.consensus_verifier.clone(); - self.run_over(stop, move || { - let validators = validators.clone(); - let verifier = verifier.clone(); - async move { connect_chain(&validators, verifier).await } - }) - .await; - } -} - -/// Rotate the first candidate on reconnect, including when a connected peer -/// cannot supply historical finalized blocks. Reads and submissions use the -/// selected verified connection. -/// -/// One endpoint for both directions. §1's concurrent fan-out to all six -/// is a submission strategy with an outcome rule, and neither exists in -/// this tree yet; inventing one here would be the runner deciding what -/// a submission means. -async fn connect_chain( - validators: &[String], - verifier: ConsensusVerifier, -) -> Option { - static NEXT_VALIDATOR: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); - let start = - NEXT_VALIDATOR.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % validators.len(); - for url in validators.iter().cycle().skip(start).take(validators.len()) { - match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { - Ok(client) => { - info!(validator = %url, "the paid-work clock reads and submits here"); - return Some(WorkBlocks::new(client)); - } - Err(error) => warn!(validator = %url, %error, "a configured validator did not answer"), - } - } - None -} - async fn serve_loop(transport: Arc, server: S) -> anyhow::Result<()> where S: Dispatcher + Send + Sync + 'static, diff --git a/crates/cli/src/commands/serve/node/tests.rs b/crates/cli/src/commands/serve/node/tests.rs index 6428495b..aa56be71 100644 --- a/crates/cli/src/commands/serve/node/tests.rs +++ b/crates/cli/src/commands/serve/node/tests.rs @@ -363,7 +363,7 @@ async fn remote_cache_cli_uses_the_nodes_existing_dispatch_and_admin_grant() { let accepting = server.clone(); let task = tokio::spawn(async move { let connection = accepting.accept().await.unwrap().await.unwrap(); - serve_connection::( + serve_connection( connection.alpn().to_vec(), connection, execution, @@ -466,7 +466,7 @@ async fn production_fetch_alpn_dispatches_run_ticket_on_its_connection() { .expect("the server accepts the Fetch dial") .await .expect("the Fetch handshake completes"); - serve_connection::( + serve_connection( connection.alpn().to_vec(), connection, test_remote_execution(), @@ -541,7 +541,7 @@ async fn exchange_routed_setup( .expect("the routed setup connection starts") .await .expect("the routed setup handshake completes"); - serve_connection::( + serve_connection( connection.alpn().to_vec(), connection, test_remote_execution(), @@ -619,22 +619,24 @@ async fn two_vouched_peers_receive_their_distinct_configured_offers() { let runner = WorkRunner::discover( WorkRunnerConfig { network: network(), + genesis_payload_digest: [0; 32].into(), threshold_identity: threshold_identity(), journal_root: dir.path().to_path_buf(), routes: configured_routes(&[ (first_peer, first.bond_edge(), first.client().party_key()), (second_peer, second.bond_edge(), second.client().party_key()), ]), - validators: Vec::new(), + validators: vec!["ws://unused.invalid".to_owned()], poll: Duration::from_millis(1), + max_observation_age: Duration::from_secs(5), settlement_key: provider(), policy: provider_policy(), }, - MountedWork::::default(), + MountedWork::default(), setup_mount.clone(), ) .expect("both owned provider journals are discovered"); - assert_eq!(runner.clocks.len(), 2, "both journals keep a clock"); + assert_eq!(runner.journal_count(), 2, "both journals keep a clock"); let alpn = ::ALPN.as_bytes(); let server = Endpoint::builder(presets::Minimal) @@ -714,14 +716,15 @@ async fn two_vouched_peers_receive_their_distinct_configured_offers() { fn discover_two_route_runner( root: &Path, - work_mount: &MountedWork, + work_mount: &MountedWork, setup_mount: &MountedSetup, -) -> WorkRunner { +) -> WorkRunner { let first = OfferFixture::first(); let second = OfferFixture::second(); match WorkRunner::discover( WorkRunnerConfig { network: network(), + genesis_payload_digest: [0; 32].into(), threshold_identity: threshold_identity(), journal_root: root.to_path_buf(), routes: configured_routes(&[ @@ -736,8 +739,9 @@ fn discover_two_route_runner( second.client().party_key(), ), ]), - validators: Vec::new(), + validators: vec!["ws://unused.invalid".to_owned()], poll: Duration::from_millis(1), + max_observation_age: Duration::from_secs(5), settlement_key: provider(), policy: provider_policy(), }, @@ -750,7 +754,7 @@ fn discover_two_route_runner( } async fn accept_mounted_route( - mount: &MountedWork, + mount: &MountedWork, peer: PeerId, request: AcceptWorkRequest, ) -> AcceptWorkResponse { @@ -850,7 +854,7 @@ async fn completion_clears_and_mounts_only_the_completing_route() { /// B's independent snapshot and service remain ready. Retained answers and /// already-expired proposals do not wait for another admission decision. #[tokio::test] -async fn fresh_readiness_is_per_request_and_per_routed_channel() { +async fn requests_use_local_observation_and_one_contested_route_does_not_disable_another() { let dir = temp(); let first = OfferFixture::first(); let second = OfferFixture::second(); @@ -869,6 +873,7 @@ async fn fresh_readiness_is_per_request_and_per_routed_channel() { assert!(runner.tick(&source).await, "both completed routes mount"); let first_request = first.signed_accept_request(); + let reads_before_request = source.0.lock().unwrap().snapshot_reads; let accepted = accept_mounted_route(&work_mount, first_route_peer(), first_request.clone()).await; assert!( @@ -879,16 +884,25 @@ async fn fresh_readiness_is_per_request_and_per_routed_channel() { "A's first fresh snapshot permits its signature", ); + assert_eq!( + source.0.lock().unwrap().snapshot_reads, + reads_before_request, + "acceptance performs no validator reads" + ); + source.set_snapshot(first.ready_snapshot(ORIGIN, Some(pending_contest(false)))); - let handler = work_mount - .handler(&vouched_context(first_route_peer())) - .unwrap(); - let checkpoint = handler - .service - .with_state(ChannelState::checkpoint) + let service = work_mount + .service(&vouched_context(first_route_peer())) .unwrap(); - // Simulate a different admission request holding the slow chain read. - let held = handler.accepting.lock().await; + let checkpoint = service.with_state(ChannelState::checkpoint).unwrap(); + // Block the observer's read, not a request. Existing local evidence still + // permits replies, and those replies cannot initiate validator I/O. + let entered = Arc::new(Semaphore::new(0)); + let release = Arc::new(Semaphore::new(0)); + source.0.lock().unwrap().next_read = Some((entered.clone(), release.clone())); + let observer_source = source.clone(); + let observing = tokio::spawn(async move { runner.tick(&observer_source).await }); + entered.acquire().await.unwrap().forget(); let repeated = tokio::time::timeout( Duration::from_secs(1), accept_mounted_route(&work_mount, first_route_peer(), first_request.clone()), @@ -923,13 +937,11 @@ async fn fresh_readiness_is_per_request_and_per_routed_channel() { Some(accept_work_response::Outcome::Refused(refusal)) if refusal.code == WorkRefusalCode::Expired as i32)); assert_eq!( - handler - .service - .with_state(ChannelState::checkpoint) - .unwrap(), + service.with_state(ChannelState::checkpoint).unwrap(), checkpoint ); - drop(held); + release.add_permits(1); + assert!(observing.await.unwrap()); let mut fresh_authorization = first.authorization(); fresh_authorization.proposal_nonce = 2; @@ -1041,7 +1053,7 @@ async fn one_tick_drives_every_owned_journal_without_connected_clients() { "one tick submits all three exact duties" ); assert_eq!( - runner.clocks.len(), + runner.journal_count(), 2, "both journals remain on the clock without a client", ); @@ -1354,7 +1366,7 @@ fn provider_policy() -> ProviderChannelPolicy { network: network(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: EdgeValues::new(PAYMENT_VALUE, PAYMENT_RESERVE, Fees::ZERO), min_omit_response_blocks: MIN_OMIT_RESPONSE_BLOCKS, } @@ -1634,7 +1646,7 @@ struct AnsweringPaidBackend { calls: Arc, } -impl PaidEvaluateBackend for AnsweringPaidBackend { +impl PaidWorkBackend for AnsweringPaidBackend { async fn evaluate( &self, input: PreparedEvaluateInput, @@ -1676,7 +1688,7 @@ impl BlockingPaidBackend { } } -impl PaidEvaluateBackend for BlockingPaidBackend { +impl PaidWorkBackend for BlockingPaidBackend { async fn evaluate_stream( &self, input: PreparedEvaluateInput, @@ -2118,6 +2130,8 @@ impl TxSink for TestChain { struct RoutedChain(Arc>); struct RoutedChainState { + snapshot_reads: usize, + next_read: Option<(Arc, Arc)>, completed_setups: Vec, snapshots: Vec, submitted: Vec, @@ -2129,6 +2143,8 @@ impl RoutedChain { snapshots: impl IntoIterator, ) -> Self { Self(Arc::new(Mutex::new(RoutedChainState { + snapshot_reads: 0, + next_read: None, completed_setups: completed_setups.into_iter().collect(), snapshots: snapshots.into_iter().collect(), submitted: Vec::new(), @@ -2192,6 +2208,15 @@ impl FinalizedWorkView for RoutedChain { &self, query: WorkChannelQuery, ) -> Result, QueryError> { + let gate = { + let mut state = self.0.lock().unwrap(); + state.snapshot_reads += 1; + state.next_read.take() + }; + if let Some((entered, release)) = gate { + entered.add_permits(1); + release.acquire().await.unwrap().forget(); + } match self.0.lock() { Ok(held) => held .snapshots @@ -2450,19 +2475,17 @@ impl TxSink for NodeChain { /// The runner a node starts with: the configured root, the stored /// identity, and the policy it was configured with. -fn runner( - root: &Path, - policy: ProviderChannelPolicy, - mount: &MountedWork, -) -> WorkRunner { +fn runner(root: &Path, policy: ProviderChannelPolicy, mount: &MountedWork) -> WorkRunner { match WorkRunner::discover( WorkRunnerConfig { network: network(), + genesis_payload_digest: [0; 32].into(), threshold_identity: threshold_identity(), journal_root: root.to_path_buf(), routes: configured_routes(&[(default_route_peer(), bond_edge(), client().party_key())]), - validators: Vec::new(), + validators: vec!["ws://unused.invalid".to_owned()], poll: Duration::from_millis(1), + max_observation_age: Duration::from_secs(5), settlement_key: provider(), policy, }, @@ -2926,7 +2949,7 @@ struct RunningPaidNode { runner_task: JoinHandle<()>, stop: Option>, setup_mount: MountedSetup, - work_mount: MountedWork, + work_mount: MountedWork, execution_calls: Arc, peer: PeerId, } @@ -2942,11 +2965,13 @@ impl RunningPaidNode { let runner = match WorkRunner::discover( WorkRunnerConfig { network: network(), + genesis_payload_digest: [0; 32].into(), threshold_identity: threshold_identity(), journal_root: root.to_path_buf(), routes: configured_routes(&[(peer, bond_edge(), client().party_key())]), - validators: Vec::new(), + validators: vec!["ws://unused.invalid".to_owned()], poll: Duration::from_millis(1), + max_observation_age: Duration::from_secs(5), settlement_key: provider(), policy, }, @@ -3291,17 +3316,22 @@ async fn run_advertised_paid_exchange( "the production clock submits and finalizes both setup Opens", ); - // Deliberately leave the raw driven service holding the readiness - // that was true when the channel mounted. The advertised Work - // wrapper must not trust that cached value: the contested arm below - // changes the coherent source before its first network request. - let mounted = node - .work_mount - .handler(&node.context()) - .expect("the production clock mounted a request handler"); - if let Err(error) = mounted.refresh_admission().await { - panic!("mount-time readiness primes the exact driven service: {error}"); - } + // The observer must publish readiness before the first request. Request + // handling itself has no validator connection and cannot refresh it. + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if node + .work_mount + .service(&node.context()) + .is_some_and(|service| service.readiness().is_ok()) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the channel observer publishes readiness"); if contested { let contest_height = source.open_contest(); @@ -3561,10 +3591,7 @@ async fn the_clock_serves_work_from_the_channel_it_was_handed() { // And the setup is not driven again, so no second journal is // opened on the file the first mount holds. - let [clock] = runner.clocks.as_slice() else { - panic!("one setup journal was written and one is driven") - }; - assert!(matches!(clock.driven, Driven::Channel(_))); + assert_eq!(runner.channel_count(), 1); assert!( runner.tick(&chain).await, "a second tick drives the channel" @@ -3695,6 +3722,7 @@ fn the_clock_resumes_an_accepted_job_after_restart() { ); chain.set_snapshot(ready_channel_snapshot(ORIGIN, Some(pending_contest(false)))); + assert!(restarted.tick(&chain).await); let response: WithTrailer = handler .deliver_result(request, context) .await @@ -3748,12 +3776,21 @@ async fn open_result_stream_stops_when_a_contest_finalizes() { )); chain.set_snapshot(ready_channel_snapshot(ORIGIN, Some(pending_contest(false)))); + assert!(runner.tick(&chain).await); backend.finish(); let next = tokio::time::timeout(Duration::from_secs(5), stream.next()) .await .unwrap() .expect("the open stream terminates with a refusal"); - assert_eq!(next.unwrap_err().code, hellas_wire::WireCode::Unavailable); + assert!( + match next { + Err(status) => status.code == hellas_wire::WireCode::Unavailable, + Ok(event) => matches!(event.outcome, + Some(hellas_rpc::pb::work::work_stream_event::Outcome::Refused(refusal)) + if refusal.code == WorkRefusalCode::NotReady as i32 || refusal.code == WorkRefusalCode::Unavailable as i32), + }, + "no payload is released after the observer revokes readiness" + ); assert!(stream.next().await.is_none()); } @@ -3770,6 +3807,9 @@ async fn the_clock_does_not_starve_the_request_path() { let backend = BlockingPaidBackend::new(); let mount = MountedWork::with_backend(backend.clone()); let mut runner = runner(dir.path(), provider_policy(), &mount); + let initial = TestChain::new(); + initial.set_snapshot(ready_channel_snapshot(ORIGIN, None)); + assert!(runner.tick(&initial).await); let chain = TestChain::slow(); chain.set_snapshot(ready_channel_snapshot(ORIGIN, None)); @@ -3893,3 +3933,75 @@ async fn a_clean_shutdown_leaves_a_replayable_journal() { "the answer the clock fixed replays", ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_blocked_channel_observer_does_not_stop_other_channels_or_shutdown() { + let dir = temp(); + let first = OfferFixture::first(); + let second = OfferFixture::second(); + first.write_completed_setup(dir.path()); + second.write_completed_setup(dir.path()); + let source = RoutedChain::new( + [first.bond_edge(), second.bond_edge()], + [ + first.ready_snapshot(ORIGIN, None), + second.ready_snapshot(ORIGIN, None), + ], + ); + let entered = Arc::new(Semaphore::new(0)); + let release = Arc::new(Semaphore::new(0)); + source.0.lock().unwrap().next_read = Some((entered.clone(), release)); + let mount = MountedWork::default(); + let setups = MountedSetup::default(); + let runner = discover_two_route_runner(dir.path(), &mount, &setups); + let (stop, stopped) = oneshot::channel(); + let observing = tokio::spawn(async move { + runner + .run_over(stopped, move || { + let source = source.clone(); + async move { Some(source) } + }) + .await; + }); + tokio::time::timeout(Duration::from_secs(2), entered.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if mount + .service(&vouched_context(second_route_peer())) + .is_some_and(|service| service.readiness().is_ok()) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the second channel advances while the first source is blocked"); + let response = + accept_mounted_route(&mount, second_route_peer(), second.signed_accept_request()).await; + assert!(matches!( + response.outcome, + Some(accept_work_response::Outcome::Accepted(_)) + )); + let held = mount + .service(&vouched_context(second_route_peer())) + .unwrap(); + stop.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(2), observing) + .await + .expect("shutdown cancels the blocked read") + .unwrap(); + assert!( + held.readiness().is_err(), + "already-held handlers cannot admit after shutdown" + ); + assert!( + mount + .service(&vouched_context(second_route_peer())) + .is_none() + ); +} diff --git a/crates/cli/src/commands/serve/provision.rs b/crates/cli/src/commands/serve/provision.rs index b48d131a..31c9345a 100644 --- a/crates/cli/src/commands/serve/provision.rs +++ b/crates/cli/src/commands/serve/provision.rs @@ -1,566 +1,27 @@ -//! Making the offers a fresh provider has nothing to serve without. -//! -//! `WorkRunner::discover` answers `WorkSetup` from the setup journals it -//! finds under the configured work root, and finding is the whole of what -//! it does. A correctly configured provider with no journal therefore -//! refuses every client that dials it, and the paid path is unreachable -//! from a clean install. This is the operator's step that writes them. -//! -//! # The order is the journal's, and none of its rules are here -//! -//! Three library calls. [`SetupStore`] is opened as the provider's half -//! of one bond, the immutable history floor is armed, and -//! [`SetupEndpoint::propose_bond`] signs the stake and journals it before -//! there is anything to export. -//! -//! Arming is first because it has to be: the setup state refuses -//! "recording revision 1 before arming its scan floor", so the floor is -//! not a preparation this command chose to do early but the step every -//! later one is refused before. The floor is a finalized height and the -//! payload digest at it, and the setup's own history must name that -//! digest as the parent of its first block. So it is read from a -//! validator rather than written down by an operator: a floor naming a -//! block this chain does not have is a setup whose history can never be -//! contiguous, and nothing later would say so out loud. -//! -//! # Exit means durable -//! -//! [`SetupStore::commit`] fsyncs a revision before it returns, and this -//! command still reopens the journal and replays it before printing -//! anything. That reopen is the one `WorkRunner::discover` will do, run -//! early: an operator told the offer exists has been told about the disk, -//! and about a file whose exclusive lock is already free for the runner -//! to take. -//! -//! # One recourse backs one route -//! -//! A provider offer reserves a route, a bond, and every coin funding that -//! bond. A second offer is safe only when all three are disjoint from every -//! provider offer already under the root. Existing peers come from the -//! durable route table, while existing coins come from the bond funding in -//! each retained setup bundle. Revision one is enough: it holds the funding -//! before a client has answered, while [`SetupState::funding_coins`] is still -//! empty because there is no executable Open yet. -//! -//! Discovery, route agreement, and funding comparison all happen while the -//! candidate is only a value. The candidate journal is not opened until -//! afterwards, so every collision is refused before a floor is written or a -//! bond signature is made. -//! -//! # What the operator chooses, and what is built -//! -//! Every number in the bond is the operator's and this command invents -//! none of them. Two parts of the shape are not choices: a stake bond is -//! funded by its maker alone, so the taker's side of the funding is -//! empty, and its timeout pays the staking party and nobody else, so -//! there is one payout and it names the provider's own key. A second -//! payout to that same party would only raise the close cost the payout -//! has to clear. Before signing, the timeout is checked against the -//! authenticated finalized height and the chain's maximum edge lifetime. -//! The kernel checks the remaining terms when the Open reaches it — -//! that the payout total is the edge's close value, that the price cap -//! covers a job, that the timeout is ahead of the block including it — -//! and re-spelling any of that here would be a second answer to a -//! question consensus already answers. -//! -//! [`SetupState::funding_coins`]: hellas_work::work_store::SetupState::funding_coins +//! CLI presentation for SDK bond provisioning. +use super::super::CliResult; +pub use hellas_sdk::work_provision::ProvisionOptions; +use hellas_sdk::work_provision::{preview_bond, provision_offer}; -use std::collections::BTreeSet; -use std::path::{Path, PathBuf}; - -use anyhow::{Context as _, bail}; -use hellas_chain::client::VerifiedRemoteLightClient; -use hellas_chain::domain::MAX_EDGE_LIFETIME_BLOCKS; -use hellas_chain::{ConsensusInfo, ConsensusVerifier, WorkBlocks}; -use hellas_kernel::{ - BlockHeight, CoinId, EdgeId, Funding, Key, List, MAX_EDGE_OUTPUTS, MAX_PARTY_INPUTS, NetworkId, - Parties, Payout, Secp256k1Signer, Secp256k1Verifier, Terms, Tx, WorkStakeBondTerms, -}; -use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; -use hellas_work::work_close::FinalizedBlocks; -use hellas_work::work_handshake::{PaymentAdmission, SetupEndpoint}; -use hellas_work::work_store::{Role, SetupScan, SetupStore, discover_setups}; -use tracing::{info, warn}; - -use super::work_config::{WorkConfig, WorkRoute}; -use crate::commands::CliResult; - -/// What an operator asks for when they make one offer. -pub struct ProvisionOptions { - /// The loaded paid-work configuration, not the path it came from. It - /// carries the network the bond is bound to, the root the journal is - /// written under, the validators the floor is read from, and the - /// artifact the provider's policy rests on. - pub work_config: WorkConfig, - /// The key this provider stakes and signs the bond with, read from - /// the identity the operator already has and never made here. - pub settlement_key: Secp256k1Signer, - /// The client this bond names as taker, hex-encoded. - pub client: String, - /// The coins this provider stakes, hex-encoded. - pub stake_coins: Vec, - /// Height the bond expires at, which is also the admission horizon of - /// the channel it insures. - pub bond_timeout: u64, - /// What the bond's timeout returns to the staking provider. - pub timeout_payout: u64, - /// The largest job price this bond covers. - pub max_job_price: u64, - /// Print the deterministic bond edge and stop before any external read or write. - pub print_bond_only: bool, -} - -/// Makes one offer, and says where it is. -/// -/// # Errors -/// -/// A configuration with no matching bilateral route, a route, bond, or funding coin already reserved by another offer, -/// a key or coin id that is not one, no configured validator with a finalized -/// block to read a floor from, and whatever the setup journal says about the -/// revision it refused or could not make durable. -pub async fn run_provision(options: ProvisionOptions) -> CliResult<()> { - // The candidate is the one source of the bond edge for both preview and - // provisioning. Keep this before evidence, routing, validators and the - // journal: the preview exists so an operator can put this value into the - // route table those later steps require. - let candidate = BondCandidate::plan(&options)?; - if options.print_bond_only { - println!("bond_edge: {}", hex::encode(candidate.bond_edge.to_bytes())); +pub async fn run_provision(options: ProvisionOptions, print_bond_only: bool) -> CliResult<()> { + if print_bond_only { + println!( + "bond_edge: {}", + hex::encode(preview_bond(&options)?.to_bytes()) + ); return Ok(()); } - let offer = Offer::plan(&options, options.work_config.provider_policy(), candidate)?; - // Dialled after every refusal that can be made without a chain, and - // before the journal exists: a floor is the first thing written into - // it, so a run that cannot read one leaves no half-made offer behind. - let made = offer.journal(finalized_floor(&options.work_config).await?)?; - + let root = options.work_config.journal_root.clone(); + let made = provision_offer(options).await?; println!( "offer journaled: bond {} under {}", hex::encode(made.bond_edge.to_bytes()), - options.work_config.journal_root.display(), + root.display() ); - // The floor read back out of the journal rather than the one just - // dialled, because those differ on a retry and the durable one is the - // one this setup's history will be measured against. println!( "history floor: finalized height {} with payload {}", made.floor.height, - hex::encode(made.floor.payload), + hex::encode(made.floor.payload) ); Ok(()) } - -/// One offer as the disk holds it, read back after it was written. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct Provisioned { - /// The bond this journal is keyed to, which is what discovery names - /// it by. - bond_edge: EdgeId, - /// The floor its history starts above, as retained. - floor: SetupScan, -} - -/// The deterministic bond inputs, built without evidence, routing, a chain, -/// or a journal. -/// -/// Preview and real provisioning both pass through this value. In particular, -/// the real path does not recompute the edge after printing it, so a preview -/// cannot drift from the offer later signed. -struct BondCandidate { - network: NetworkId, - journal_root: PathBuf, - bond_edge: EdgeId, - bond_funding: Funding, - bond_terms: WorkStakeBondTerms, - settlement_key: Secp256k1Signer, -} - -impl BondCandidate { - fn plan(options: &ProvisionOptions) -> CliResult { - let network = options.work_config.chain.network; - let journal_root = options.work_config.journal_root.clone(); - // Maker is the provider and taker is the client, which is what - // makes this signature the maker's: `propose_bond` refuses a bond - // whose staking party this key is not. - let provider = options.settlement_key.party_key(); - let bond_terms = WorkStakeBondTerms { - parties: Parties::new( - provider, - Key::from_bytes(fixed::<{ Key::LENGTH }>("--client", &options.client)?), - ), - timeout: BlockHeight::new(options.bond_timeout), - timeout_outputs: List::take( - [Payout::new(provider, options.timeout_payout); MAX_EDGE_OUTPUTS], - 1, - ), - max_job_price: options.max_job_price, - }; - let bond_funding = Funding::new( - staked(&options.stake_coins)?, - List::empty(CoinId::from_bytes([0; CoinId::LENGTH])), - ); - let bond_edge = Tx::edge_id_of(&bond_funding, &Terms::work_stake_bond(bond_terms.clone())); - Ok(Self { - network, - journal_root, - bond_edge, - bond_funding, - bond_terms, - settlement_key: options.settlement_key.clone(), - }) - } -} - -/// One offer, decided before anything is dialled or written. -struct Offer { - network: NetworkId, - journal_root: PathBuf, - bond_edge: EdgeId, - bond_funding: Funding, - bond_terms: WorkStakeBondTerms, - admission: PaymentAdmission, - settlement_key: Secp256k1Signer, -} - -impl Offer { - /// Reads the operator's answers, and refuses everything refusable - /// without a chain. - fn plan( - options: &ProvisionOptions, - policy: ProviderChannelPolicy, - candidate: BondCandidate, - ) -> CliResult { - let admission = PaymentAdmission::Admits(Box::new(policy)); - let BondCandidate { - network, - journal_root, - bond_edge, - bond_funding, - bond_terms, - settlement_key, - } = candidate; - let route = route_for_candidate(&options.work_config, bond_edge, &bond_terms)?; - refuse_offer_collisions(&options.work_config, route, &bond_funding)?; - Ok(Self { - network, - journal_root, - bond_edge, - bond_funding, - bond_terms, - admission, - settlement_key, - }) - } - - /// Journals revision 1, and returns only once a fresh open of the - /// journal replays it. - fn journal(self, floor: SetupScan) -> CliResult { - let timeout = self.bond_terms.timeout.get(); - anyhow::ensure!( - timeout > floor.height, - "--bond-timeout {timeout} must be after finalized height {}", - floor.height, - ); - anyhow::ensure!( - timeout - floor.height <= MAX_EDGE_LIFETIME_BLOCKS, - "--bond-timeout {timeout} exceeds the chain's maximum edge lifetime of \ - {MAX_EDGE_LIFETIME_BLOCKS} blocks from finalized height {}", - floor.height, - ); - let Self { - network, - journal_root, - bond_edge, - bond_funding, - bond_terms, - admission, - settlement_key, - } = self; - { - let store = open_provider_journal(&journal_root, network, bond_edge)?; - let mut endpoint = SetupEndpoint::new(store, settlement_key, admission); - // The floor is immutable and the store writes exactly one arm - // of it, so a run that arms and then fails keeps the height - // its successor starts from rather than moving it. - if let Some(held) = endpoint.state().scan_armed() { - info!( - height = held.height, - "this journal already holds its history floor, and a floor does not move", - ); - } else { - endpoint - .arm_scan(floor) - .context("failed to make this setup's immutable history floor durable")?; - } - endpoint - .propose_bond(network, bond_funding, bond_terms) - .context("failed to sign and journal the bond proposal")?; - } - - // The journal is closed above, so this is a second process's view - // of it: the same replay and the same signature checks the runner - // runs, before an operator is told there is anything to run them - // on. - let reopened = open_provider_journal(&journal_root, network, bond_edge)?; - let state = reopened.state(); - let (Some(1), Some(floor)) = (state.revision(), state.scan_armed()) else { - bail!( - "the journal under {} replays as revision {:?} over floor {:?}, not the armed \ - proposal that was just written", - journal_root.display(), - state.revision(), - state.scan_armed().map(|scan| scan.height), - ); - }; - Ok(Provisioned { bond_edge, floor }) - } -} - -fn open_provider_journal( - root: &Path, - network: NetworkId, - bond_edge: EdgeId, -) -> CliResult { - SetupStore::open( - root, - network, - bond_edge, - Role::Provider, - &Secp256k1Verifier::new(), - ) - .with_context(|| { - format!( - "failed to open the provider setup journal for bond {} under {}", - hex::encode(bond_edge.to_bytes()), - root.display(), - ) - }) -} - -/// Returns the configured bilateral route the candidate would occupy. -/// -/// The bond is derived from the exact funding and terms first. Matching by -/// that canonical value means a route cannot be selected by insertion order, -/// and checking the client here refuses a journal the next startup would -/// reject before the provider signs it. -fn route_for_candidate<'config>( - config: &'config WorkConfig, - bond_edge: EdgeId, - bond_terms: &WorkStakeBondTerms, -) -> CliResult<&'config WorkRoute> { - let Some(route) = config.routes.iter().find(|route| route.bond == bond_edge) else { - bail!( - "bond {} has no bilateral route in this work configuration; an offer is signed only \ - after its peer, bond, and client are named together", - hex::encode(bond_edge.to_bytes()), - ); - }; - let client = bond_terms.parties.taker(); - if route.client != client { - bail!( - "route for peer {:#} expects client {}, but candidate bond {} names {} as its taker", - route.peer, - hex::encode(route.client.to_bytes()), - hex::encode(bond_edge.to_bytes()), - hex::encode(client.to_bytes()), - ); - } - Ok(route) -} - -/// Refuses every collision before the candidate journal is opened. -/// -/// An existing bond is named by discovery, its peer is named by the durable -/// route table, and its funding is named by the retained bundle. Failure to -/// recover any one of those facts is a refusal: absence of evidence is not -/// evidence that the candidate is disjoint. -fn refuse_offer_collisions( - config: &WorkConfig, - candidate: &WorkRoute, - candidate_funding: &Funding, -) -> CliResult<()> { - let root = &config.journal_root; - let network = config.chain.network; - let found = discover_setups(root, network).with_context(|| { - format!( - "failed to enumerate the work journals under {}", - root.display(), - ) - })?; - for unnamed in &found.unidentified { - warn!( - path = %unnamed.path.display(), - reason = %unnamed.reason, - "a setup journal under the work root could not be named", - ); - } - if let Some(unnamed) = found.unidentified.first() { - bail!( - "setup journal {} cannot be identified, so a new offer cannot be proved disjoint: {}", - unnamed.path.display(), - unnamed.reason, - ); - } - - let candidate_coins = funding_coins(candidate_funding); - for held in found - .setups - .iter() - .filter(|setup| setup.role == Role::Provider) - { - if held.bond_edge == candidate.bond { - bail!( - "candidate bond {} collides with a provider offer already under {}", - hex::encode(candidate.bond.to_bytes()), - root.display(), - ); - } - let Some(route) = config - .routes - .iter() - .find(|route| route.bond == held.bond_edge) - else { - bail!( - "provider offer over bond {} under {} has no configured route, so the candidate \ - route cannot be proved disjoint", - hex::encode(held.bond_edge.to_bytes()), - root.display(), - ); - }; - let store = open_provider_journal(root, network, held.bond_edge)?; - let Some(bundle) = store.state().bundle() else { - bail!( - "provider offer over bond {} was discovered without a retained revision", - hex::encode(held.bond_edge.to_bytes()), - ); - }; - let held_client = bundle.bond_terms().parties.taker(); - if route.client != held_client { - bail!( - "route for peer {:#} expects client {}, but provider offer over bond {} names {} \ - as its taker", - route.peer, - hex::encode(route.client.to_bytes()), - hex::encode(held.bond_edge.to_bytes()), - hex::encode(held_client.to_bytes()), - ); - } - if route.peer == candidate.peer { - bail!( - "candidate route peer {:#} collides with the provider offer over bond {}", - candidate.peer, - hex::encode(held.bond_edge.to_bytes()), - ); - } - // The retained revision's own staked funding, not the executable - // Opens: the provider signed these coins when it made the offer, so - // they are promised from that moment, while `funding_coins` answers - // from Opens that do not exist until the client countersigns. Read - // from there, every offer no client has answered would look like it - // reserved nothing. - let reserved = funding_coins(bundle.bond_funding()); - if let Some(coin) = candidate_coins.intersection(&reserved).next() { - bail!( - "candidate stake coin {} is already reserved by provider offer over bond {}", - hex::encode(coin.to_bytes()), - hex::encode(held.bond_edge.to_bytes()), - ); - } - } - Ok(()) -} - -/// Every input one bond funding consumes, irrespective of party position. -fn funding_coins(funding: &Funding) -> BTreeSet { - funding - .maker() - .iter() - .chain(funding.taker().iter()) - .copied() - .collect() -} - -/// Reads one finalized block from the first configured validator that -/// answers, as the floor this setup's history starts above. -async fn finalized_floor(config: &WorkConfig) -> CliResult { - let verifier = ConsensusVerifier::new(&ConsensusInfo { - validators: config.validators.clone(), - threshold_identity: config.chain.threshold_identity.clone(), - network_id: config.chain.network.as_str().to_owned(), - }) - .context("the configured threshold identity is not usable")?; - for url in &config.validators { - let client = match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { - Ok(client) => client, - Err(error) => { - warn!(validator = %url, %error, "a configured validator did not answer"); - continue; - } - }; - match floor_of(&WorkBlocks::new(client)).await { - Ok(Some(floor)) => { - info!(validator = %url, height = floor.height, "the history floor was read here"); - return Ok(floor); - } - Ok(None) => warn!(validator = %url, "a configured validator has finalized nothing"), - Err(error) => warn!(validator = %url, %error, "a configured validator did not answer"), - } - } - bail!("no configured validator answered with a finalized block to floor this offer at") -} - -/// Returns the finalized tip as a scan floor, or `None` before anything -/// is finalized. -/// -/// The height and the payload come from one block rather than from two -/// reads, because the setup's first history block must name that exact -/// payload as its parent. -async fn floor_of(blocks: &B) -> CliResult> -where - B: FinalizedBlocks + ?Sized, -{ - let Some(height) = blocks.latest_height().await? else { - return Ok(None); - }; - let Some(block) = blocks.block_at(height).await? else { - return Ok(None); - }; - Ok(Some(SetupScan { - height: block.height, - payload: block.payload, - })) -} - -/// Reads the coins one provider stakes. -fn staked(ids: &[String]) -> CliResult> { - let mut slots = [CoinId::from_bytes([0; CoinId::LENGTH]); MAX_PARTY_INPUTS]; - for (slot, id) in slots.iter_mut().zip(ids) { - *slot = CoinId::from_bytes(fixed::<{ CoinId::LENGTH }>("--stake-coin", id)?); - } - // The zip above stops at the shorter side, so a list the array cannot - // hold is refused here rather than silently staking the first four of - // it. - List::new(slots, ids.len()).with_context(|| { - format!( - "--stake-coin names {} coins, and one party funds an open with at most \ - {MAX_PARTY_INPUTS}", - ids.len(), - ) - }) -} - -/// Reads exactly `N` bytes of hex, or says which flag was not that. -fn fixed(flag: &str, value: &str) -> CliResult<[u8; N]> { - let bytes = - hex::decode(value).with_context(|| format!("{flag} {value:?} is not hex-encoded bytes"))?; - let Ok(fixed) = <[u8; N]>::try_from(bytes.as_slice()) else { - bail!( - "{flag} {value:?} is {} bytes, and {N} are wanted", - bytes.len() - ); - }; - Ok(fixed) -} - -#[cfg(test)] -mod tests; diff --git a/crates/cli/src/commands/serve/work_config.rs b/crates/cli/src/commands/serve/work_config.rs index 0ce1c113..275fdf1d 100644 --- a/crates/cli/src/commands/serve/work_config.rs +++ b/crates/cli/src/commands/serve/work_config.rs @@ -1,628 +1,4 @@ -//! The paid-work configuration file: what an operator writes down, and -//! what a node refuses to start without. -//! -//! `--work-config` was a path whose *presence* advertised two ALPNs and -//! whose contents were never opened. A node cannot mount a channel from -//! a path, so this is the schema and the loader for what is in it: the -//! three-part chain cross-check, the six validator URLs a write is -//! fanned to, the journal root, the bilateral route table, the two -//! policies this provider works under, the watcher's poll cadence, the -//! funding it expects a payment edge to carry, and the shortest response -//! window it will sign terms over. -//! -//! # The cross-check is not an anchor -//! -//! `(network_id, genesis_payload_digest, threshold_identity)` is a -//! fail-fast configuration cross-check. Only the threshold identity ever -//! authenticates a finalized block; the other two are how a node started -//! against the wrong chain says so at startup instead of at the first -//! settlement. The identity is decoded here, by the same constructor -//! consensus verification uses, so a typo is a startup error and not a -//! block that never verifies. -//! -//! # What is deliberately not here -//! -//! There is no Start-span field, no mutual-margin field, and no journal -//! cap. All were deleted: the Start span is fixed at 64, a work-payment -//! edge has no Mutual route, and the journal's active and checkpoint -//! ceilings are constants it enforces on itself -//! ([`MAX_ACTIVE_JOURNAL_BYTES`]), so any of them appearing in a file is -//! an operator configuring something that does not exist. Every struct -//! below denies unknown fields, which is what turns that into an error -//! naming the field. -//! -//! Nor is there a measured artifact. An earlier design derived the -//! response window and an alarm margin from latencies a bootstrap probe -//! recorded, pinned to the digest of the measuring binary; no deployed -//! node ever carried one, and every rebuild would have invalidated it. -//! The two numbers that design would have produced are written down -//! here instead, by the operator, and [`WorkConfig::provider_policy`] is -//! the whole of what is made from them. -//! -//! [`MAX_ACTIVE_JOURNAL_BYTES`]: hellas_work::work_store::journal::MAX_ACTIVE_JOURNAL_BYTES - -use std::collections::{BTreeMap, BTreeSet}; -use std::fs; -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use anyhow::{Context as _, bail}; -use hellas_kernel::{ - EdgeId, EdgeValues, Fees, Key, MIN_OMIT_RESPONSE_BLOCKS, NetworkId, Secp256k1Verifier, -}; -use hellas_rpc::ContentId; -use hellas_rpc::peers::PeerId; -use hellas_rpc::protocol::Digest; -use hellas_rpc::protocol::work::{ - PaidChannelPolicyV1, PaidExecutionPolicyV1, check_execution_policy, -}; -use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; -use hellas_work::work_store::{Role, SetupStore, discover_setups}; -use serde::Deserialize; - -use crate::commands::CliResult; - -/// How many validator RPCs a write names. -/// -/// Reads come from a follower and writes are fanned to all six; a -/// configuration naming five has one validator whose acceptance this -/// node can never win, and one naming seven names something this -/// deployment does not have. -pub const VALIDATOR_COUNT: usize = 6; - -/// One operator's complete paid-work configuration, loaded and structurally -/// checked. -/// -/// A plain record with public fields, for [`WorkChannelConfig`]'s -/// reason: this is the shape a file fills in. Every file-local gate has -/// already run in [`load_work_config`]; the serve path then runs -/// [`validate_work_routes`] against the journals that must exist when it -/// starts. Provisioning shares the file loader before it creates one, which is -/// why disk agreement is not pretended to be a parse-time fact. -/// -/// [`WorkChannelConfig`]: hellas_rpc::protocol::work_setup::WorkChannelConfig -#[derive(Clone, Debug)] -#[allow( - dead_code, - reason = "the fields a mount consumes are read by the node runner; loading and checking them is this half" -)] -pub struct WorkConfig { - /// The chain this node believes it is configured against. - pub chain: ChainCrossCheck, - /// The six validator RPC URLs every write is fanned to. - pub validators: Vec, - /// Directory holding the setup and channel journals. - pub journal_root: PathBuf, - /// Bilateral setup routes, keyed by the authenticated transport peer. - pub routes: WorkRoutes, - /// Salt of the private credit-policy commitment. - pub policy_salt: [u8; 32], - /// The credit policy this provider will work under. - pub channel_policy: PaidChannelPolicyV1, - /// The execution policy this provider will run jobs under. - pub execution_policy: PaidExecutionPolicyV1, - /// How often the watcher asks the chain for the next block. - pub poll: Duration, - /// The payment edge's value, reserve, and close fees as this provider - /// requires a client to fund them. - pub expected_payment_values: EdgeValues, - /// The shortest response window this provider signs terms over. - pub min_omit_response_blocks: u64, -} - -/// One bilateral setup route written in the paid-work configuration. -/// -/// The bond names the provider setup journal under [`WorkConfig::journal_root`]. -/// The client key is repeated here deliberately: startup compares it with the -/// taker committed inside that journal, turning a stale or mistyped route into -/// a refusal before the node binds. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct WorkRoute { - /// The transport-authenticated peer allowed to reach this bond. - pub peer: PeerId, - /// The bond whose provider setup journal this route names. - pub bond: EdgeId, - /// The settlement key the bond terms must name as taker. - pub client: Key, -} - -/// Paid-work routes keyed by their authenticated peer. -/// -/// Construction is private to the checked file loader. In particular, there -/// is no insertion API through which a caller could recreate last-one-wins -/// handling after duplicate peers and bonds have been refused. -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct WorkRoutes { - by_peer: BTreeMap, -} - -impl WorkRoutes { - /// Returns every configured route in peer order. - pub fn iter(&self) -> impl Iterator { - self.by_peer.values() - } - - /// Returns how many bilateral routes were configured. - #[must_use] - pub fn len(&self) -> usize { - self.by_peer.len() - } - - /// Returns whether no bilateral route was configured. - #[must_use] - pub fn is_empty(&self) -> bool { - self.by_peer.is_empty() - } - - fn from_files(files: Vec) -> CliResult { - let mut by_peer = BTreeMap::new(); - let mut bonds = BTreeSet::new(); - for file in files { - let peer = PeerId::from_bytes(parse_fixed_hex("routes[].peer", &file.peer)?); - let bond = EdgeId::from_bytes(parse_fixed_hex("routes[].bond", &file.bond)?); - let client = Key::from_bytes(parse_fixed_hex("routes[].client", &file.client)?); - let route = WorkRoute { peer, bond, client }; - if by_peer.insert(peer, route).is_some() { - bail!("routes names peer {peer:#} twice; one authenticated peer has one route"); - } - if !bonds.insert(bond) { - bail!( - "routes names bond {} twice; one provider journal has one route", - hex::encode(bond.to_bytes()), - ); - } - } - Ok(Self { by_peer }) - } -} - -impl WorkConfig { - /// The provider policy this configuration makes. - /// - /// Every field is the operator's: the four the policy commits to, - /// the funding it expects, and the window it insists on. Nothing is - /// measured and nothing is inferred, so a node with a configuration - /// has a policy and countersigns over it. - #[must_use] - pub fn provider_policy(&self) -> ProviderChannelPolicy { - ProviderChannelPolicy { - network: self.chain.network, - policy_salt: self.policy_salt, - channel_policy: self.channel_policy, - execution_policy: self.execution_policy, - expected_payment_values: self.expected_payment_values, - min_omit_response_blocks: self.min_omit_response_blocks, - } - } -} - -/// The three fields that say which chain this is. -#[derive(Clone, Debug, PartialEq, Eq)] -#[allow( - dead_code, - reason = "the fields a mount consumes are read by the node runner; loading and checking them is this half" -)] -pub struct ChainCrossCheck { - /// The network every signature on this node's channels is bound to. - pub network: NetworkId, - /// Payload digest of the genesis block this deployment began at. - pub genesis_payload_digest: Digest, - /// The threshold identity finalized blocks are verified under. - pub threshold_identity: Vec, -} - -/// Loads and checks one paid-work configuration file. -/// -/// Every failure is a startup failure naming the field that failed, for -/// the reason §4 gives: a node that started with an unreadable -/// configuration would be one whose first symptom is an unsettleable -/// channel. -/// -/// # Errors -/// -/// The read and the parse, and then: a network id that is not one, a -/// digest that is not thirty-two bytes, a threshold identity consensus -/// cannot decode, a validator list that is not exactly -/// [`VALIDATOR_COUNT`] URLs with distinct normalised forms, an execution -/// policy the protocol's own [`check_execution_policy`] rejects, an -/// empty journal root, a route field of the wrong width, duplicate peers or -/// bonds in the route table, a zero poll cadence, and a response window -/// under the kernel's own minimum. -pub fn load_work_config(path: &Path) -> CliResult { - let bytes = fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; - let file: WorkConfigFile = serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse {}", path.display()))?; - file.into_config() - .with_context(|| format!("invalid work config {}", path.display())) -} - -/// Verifies that every configured route names this root's provider journal -/// and the client settlement key committed by its bond terms. -/// -/// This is a serve-startup check rather than part of [`load_work_config`]: -/// provisioning uses the same configuration loader before it creates a -/// journal, while a serving node must already have every journal it promises. -/// Discovery comes first so [`SetupStore::open`] is never allowed to create a -/// missing journal merely because a route named its bond. -/// -/// # Errors -/// -/// The root cannot be enumerated, a route's provider journal is absent from -/// that root or cannot be opened, the journal holds no bond proposal, or its -/// bond names a taker other than the route's configured client. -pub(super) fn validate_work_routes(config: &WorkConfig) -> CliResult<()> { - if config.routes.is_empty() { - return Ok(()); - } - let found = discover_setups(&config.journal_root, config.chain.network).with_context(|| { - format!( - "failed to enumerate configured work routes under journal.root {}", - config.journal_root.display(), - ) - })?; - for route in config.routes.iter() { - if !found - .setups - .iter() - .any(|setup| setup.role == Role::Provider && setup.bond_edge == route.bond) - { - bail!( - "route for peer {:#} names bond {}, but its provider setup journal is not under \ - journal.root {}", - route.peer, - hex::encode(route.bond.to_bytes()), - config.journal_root.display(), - ); - } - let store = SetupStore::open( - &config.journal_root, - config.chain.network, - route.bond, - Role::Provider, - &Secp256k1Verifier::new(), - ) - .with_context(|| { - format!( - "route for peer {:#} could not open provider setup journal for bond {} under {}", - route.peer, - hex::encode(route.bond.to_bytes()), - config.journal_root.display(), - ) - })?; - let Some(bundle) = store.state().bundle() else { - bail!( - "route for peer {:#} names provider setup journal for bond {}, but it holds no \ - bond proposal", - route.peer, - hex::encode(route.bond.to_bytes()), - ); - }; - let journal_client = bundle.bond_terms().parties.taker(); - if journal_client != route.client { - bail!( - "route for peer {:#} expects client settlement key {}, but provider setup journal \ - for bond {} names {} as its taker", - route.peer, - hex::encode(route.client.to_bytes()), - hex::encode(route.bond.to_bytes()), - hex::encode(journal_client.to_bytes()), - ); - } - } - Ok(()) -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct WorkConfigFile { - chain: ChainFile, - validators: Vec, - journal: JournalFile, - routes: Vec, - policies: PoliciesFile, - /// How often the watcher asks the chain for the next block. - poll_ms: u64, - expected_payment_values: PaymentValuesFile, - min_omit_response_blocks: u64, -} - -impl WorkConfigFile { - fn into_config(self) -> CliResult { - let Some(network) = NetworkId::new(self.chain.network_id.trim()) else { - bail!( - "chain.network_id {:?} is not a network id", - self.chain.network_id - ); - }; - let threshold_identity = - parse_hex("chain.threshold_identity", &self.chain.threshold_identity)?; - // Parsed before the verifier is built, so the list consensus is - // handed is the normalised one this node will actually dial. - let validators = parse_validators(self.validators)?; - // The same constructor consensus verification uses. A threshold - // identity that cannot be decoded here is one no finalized block - // would ever verify under, and the node says so before it serves. - hellas_chain::ConsensusVerifier::new(&hellas_chain::light_client::ConsensusInfo { - validators: validators.clone(), - threshold_identity: threshold_identity.clone(), - network_id: self.chain.network_id.clone(), - }) - .map_err(|error| anyhow::anyhow!("chain.threshold_identity is not usable: {error}"))?; - - let journal_root = self.journal.into_root()?; - let routes = WorkRoutes::from_files(self.routes)?; - let policies = self.policies.into_policies()?; - if self.poll_ms == 0 { - bail!("poll_ms must be greater than zero"); - } - // The kernel refuses a shorter window at every payment open, so a - // configuration under it would sign terms consensus then throws - // away. - if self.min_omit_response_blocks < MIN_OMIT_RESPONSE_BLOCKS { - bail!( - "min_omit_response_blocks {} is under the kernel's minimum {MIN_OMIT_RESPONSE_BLOCKS}", - self.min_omit_response_blocks, - ); - } - - Ok(WorkConfig { - chain: ChainCrossCheck { - network, - genesis_payload_digest: parse_digest( - "chain.genesis_payload_digest", - &self.chain.genesis_payload_digest, - )?, - threshold_identity, - }, - validators, - journal_root, - routes, - policy_salt: policies.0, - channel_policy: policies.1, - execution_policy: policies.2, - poll: Duration::from_millis(self.poll_ms), - expected_payment_values: self.expected_payment_values.into_values(), - min_omit_response_blocks: self.min_omit_response_blocks, - }) - } -} - -/// One bilateral route exactly as the operator writes it. -/// -/// All three values are fixed-width lowercase-or-uppercase hexadecimal on -/// input and canonical byte values after loading. A peer or bond written in a -/// second spelling is therefore still the same key for duplicate detection. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct WorkRouteFile { - peer: String, - bond: String, - client: String, -} - -/// Parses the six validator RPC URLs, and refuses anything that is not -/// one. -/// -/// Both halves matter. A string that is not a URL is not an address this -/// node can ever fan a write to, and "trimmed and non-empty" admits `not -/// a URL` verbatim — a configuration whose first symptom would be five -/// validators answering and one that never does. And uniqueness is a -/// question about *addresses*, not about spellings: `HTTP://Host:443/` -/// and `http://host:443/` are one validator written twice, and a fan-out -/// to five validators is not six however it is spelled. So the -/// comparison is between parsed, normalised URLs, and the normalised -/// forms are what is kept. -/// -/// A host is required, because these are dialled: a `mailto:` or a -/// `data:` parses perfectly well and is not a validator. -fn parse_validators(entries: Vec) -> CliResult> { - let mut validators: Vec = Vec::with_capacity(VALIDATOR_COUNT); - for entry in entries { - let entry = entry.trim(); - if entry.is_empty() { - bail!("validators entries must be non-empty"); - } - let url = reqwest::Url::parse(entry) - .with_context(|| format!("validators entry {entry:?} is not a URL"))?; - if url.host_str().is_none() { - bail!("validators entry {entry:?} names no host to dial"); - } - let normalised = url.as_str().to_string(); - if validators.contains(&normalised) { - bail!("validators names {normalised} twice; a fan-out to five validators is not six"); - } - validators.push(normalised); - } - if validators.len() != VALIDATOR_COUNT { - bail!( - "validators must name exactly {VALIDATOR_COUNT} validator URLs, found {}", - validators.len(), - ); - } - Ok(validators) -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct ChainFile { - network_id: String, - genesis_payload_digest: String, - threshold_identity: String, -} - -/// Where the work journals live. -/// -/// A root and nothing else. How large a journal may grow is not an -/// operator's to say: the active and checkpoint ceilings are constants -/// the journal enforces on itself, so a cap here would be a number -/// written down and ignored. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct JournalFile { - root: PathBuf, -} - -impl JournalFile { - fn into_root(self) -> CliResult { - if self.root.as_os_str().is_empty() { - bail!("journal.root must be a path"); - } - Ok(self.root) - } -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct PoliciesFile { - policy_salt: String, - channel: ChannelPolicyFile, - execution: ExecutionPolicyFile, -} - -impl PoliciesFile { - fn into_policies(self) -> CliResult<([u8; 32], PaidChannelPolicyV1, PaidExecutionPolicyV1)> { - let salt = parse_fixed_hex("policies.policy_salt", &self.policy_salt)?; - Ok(( - salt, - PaidChannelPolicyV1 { - compute_credit_limit: self.channel.compute_credit_limit, - delivery_credit_limit: self.channel.delivery_credit_limit, - }, - self.execution.into_policy()?, - )) - } -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct ChannelPolicyFile { - compute_credit_limit: u64, - delivery_credit_limit: u64, -} - -/// The execution policy, field for field. -/// -/// Spelled out rather than flattened from some smaller shape because -/// every one of these is a value both parties sign: a default here would -/// be this node quietly proposing a policy its operator never wrote. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct ExecutionPolicyFile { - allowed_environment: String, - generation_policy_digest: String, - identity_source_digest: String, - max_prompt_tokens: u32, - max_new_tokens: u32, - max_stop_token_ids: u16, - max_spool_bytes: u64, - max_encoded_result_frame: u32, - max_encoded_quote_response: u32, - dispatch_margin_blocks: u64, - delivery_margin_blocks: u64, - oracle_grace_blocks: u64, - fixed_price: u64, -} - -impl ExecutionPolicyFile { - fn into_policy(self) -> CliResult { - let allowed_environment: ContentId = - self.allowed_environment.parse().with_context(|| { - format!( - "policies.execution.allowed_environment {:?} is not a ContentId", - self.allowed_environment - ) - })?; - let policy = PaidExecutionPolicyV1 { - allowed_environment, - generation_policy_digest: parse_digest( - "policies.execution.generation_policy_digest", - &self.generation_policy_digest, - )?, - identity_source_digest: parse_digest( - "policies.execution.identity_source_digest", - &self.identity_source_digest, - )?, - max_prompt_tokens: self.max_prompt_tokens, - max_new_tokens: self.max_new_tokens, - max_stop_token_ids: self.max_stop_token_ids, - max_spool_bytes: self.max_spool_bytes, - max_encoded_result_frame: self.max_encoded_result_frame, - max_encoded_quote_response: self.max_encoded_quote_response, - dispatch_margin_blocks: self.dispatch_margin_blocks, - delivery_margin_blocks: self.delivery_margin_blocks, - oracle_grace_blocks: self.oracle_grace_blocks, - fixed_price: self.fixed_price, - }; - // The protocol's own gate, run here rather than at the first - // admission. A zero here is not a small bound, it is an absent - // one — a zero margin gives a deadline no time to be met in, and - // a zero price is a job nobody is paid for. Copying the fields - // through unchecked moves that discovery to the moment a - // counterparty is already waiting on a proposal. - check_execution_policy(&policy) - .map_err(|error| anyhow::anyhow!("policies.execution is not usable: {error}"))?; - Ok(policy) - } -} - -/// The payment edge's funding, field for field, as this provider requires -/// a client to fund it. -/// -/// Spelled out for [`ExecutionPolicyFile`]'s reason: every one of these -/// bounds what a certificate on the channel may name, so a default here -/// would be this node quietly accepting funding its operator never -/// priced. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct PaymentValuesFile { - value: u64, - reserve: u64, - close_fees: CloseFeesFile, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct CloseFeesFile { - base: u64, - slot: u64, - proof: u64, - lifetime: u64, -} - -impl PaymentValuesFile { - fn into_values(self) -> EdgeValues { - EdgeValues::new( - self.value, - self.reserve, - Fees::new( - self.close_fees.base, - self.close_fees.slot, - self.close_fees.proof, - self.close_fees.lifetime, - ), - ) - } -} - -fn parse_hex(field: &str, raw: &str) -> CliResult> { - let bytes = hex::decode(raw.trim()).with_context(|| format!("{field} is not hexadecimal"))?; - if bytes.is_empty() { - bail!("{field} must not be empty"); - } - Ok(bytes) -} - -fn parse_fixed_hex(field: &str, raw: &str) -> CliResult<[u8; N]> { - let bytes = parse_hex(field, raw)?; - let Ok(bytes) = <[u8; N]>::try_from(bytes.as_slice()) else { - bail!("{field} must be {N} bytes, found {}", bytes.len()); - }; - Ok(bytes) -} - -fn parse_digest(field: &str, raw: &str) -> CliResult { - Ok(Digest::from_bytes(parse_fixed_hex(field, raw)?)) -} +pub use hellas_sdk::work_config::*; #[cfg(test)] mod tests; diff --git a/crates/cli/src/commands/serve/work_config/tests.rs b/crates/cli/src/commands/serve/work_config/tests.rs index ec9d98a0..498d07c0 100644 --- a/crates/cli/src/commands/serve/work_config/tests.rs +++ b/crates/cli/src/commands/serve/work_config/tests.rs @@ -1,4 +1,46 @@ use super::*; +use hellas_kernel::{EdgeValues, Fees, Key, NetworkId}; +use hellas_rpc::peers::PeerId; +use hellas_rpc::protocol::Digest; +use hellas_rpc::protocol::work::*; +use hellas_rpc::protocol::work_profile::PaidWorkPolicy; +use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; +use std::time::Duration; +use std::{ + fs, + path::{Path, PathBuf}, +}; +type CliResult = anyhow::Result; + +#[test] +fn fetch_policy_loads_and_cannot_be_combined_with_evaluate() { + let mut value = config(); + value["policies"]["fetch"] = serde_json::json!({ + "allowed_environment": hex32(0x11), + "service": "openai", "method": "responses", + "max_request_body_bytes": 4096, "max_output_events": 64, + "max_output_bytes": 16384, "max_spool_bytes": 65536, + "max_encoded_result_frame": 65536, "max_encoded_prepared_input": 65536, + "dispatch_margin_blocks": 4, "delivery_margin_blocks": 2, + "oracle_grace_blocks": 6, "fixed_price": 10, + }); + assert!(load(value.clone()).is_err()); + value["policies"] + .as_object_mut() + .unwrap() + .remove("execution"); + let loaded = load(value.clone()).unwrap(); + assert!(matches!( + loaded.execution_policy, + PaidWorkPolicy::Fetch { .. } + )); + assert_eq!( + PaidWorkPolicy::decode(&loaded.execution_policy.encode()).unwrap(), + loaded.execution_policy + ); + value["policies"].as_object_mut().unwrap().remove("fetch"); + assert!(load(value).is_err()); +} fn hex32(byte: u8) -> String { hex::encode([byte; 32]) @@ -86,7 +128,7 @@ fn write(dir: &tempfile::TempDir, value: &serde_json::Value) -> PathBuf { fn load(value: serde_json::Value) -> CliResult { let dir = tempfile::tempdir().unwrap(); - load_work_config(&write(&dir, &value)) + Ok(load_work_config(&write(&dir, &value))?) } /// Delete `field` from the object at `path`. @@ -129,8 +171,14 @@ fn a_work_config_round_trips_from_a_file() { assert_eq!(route.client, Key::from_bytes([0x02; Key::LENGTH])); assert_eq!(loaded.policy_salt, [0x5a; 32]); assert_eq!(loaded.channel_policy.compute_credit_limit, 40); - assert_eq!(loaded.execution_policy.fixed_price, 10); - assert_eq!(loaded.execution_policy.max_stop_token_ids, 4); + assert_eq!(loaded.execution_policy.fixed_price(), 10); + assert_eq!( + match loaded.execution_policy { + PaidWorkPolicy::Evaluate(policy) => policy.max_stop_token_ids, + _ => panic!("expected Evaluate policy"), + }, + 4 + ); assert_eq!(loaded.poll, Duration::from_millis(250)); assert_eq!( loaded.expected_payment_values, @@ -354,7 +402,13 @@ fn a_zero_execution_policy_field_is_refused() { let mut value = config(); value["policies"]["execution"]["max_stop_token_ids"] = serde_json::json!(0); let loaded = load(value).expect("no stop tokens is a usable channel"); - assert_eq!(loaded.execution_policy.max_stop_token_ids, 0); + assert_eq!( + match loaded.execution_policy { + PaidWorkPolicy::Evaluate(policy) => policy.max_stop_token_ids, + _ => panic!("expected Evaluate policy"), + }, + 0 + ); } #[test] @@ -617,7 +671,7 @@ fn a_configuration_makes_its_policy_field_for_field() { assert_eq!(policy.network, network()); assert_eq!(policy.policy_salt, [0x5a; 32]); assert_eq!(policy.channel_policy.compute_credit_limit, 40); - assert_eq!(policy.execution_policy.fixed_price, 10); + assert_eq!(policy.execution_policy.fixed_price(), 10); assert_eq!( policy.expected_payment_values, EdgeValues::new(PAYMENT_VALUE, PAYMENT_RESERVE, Fees::new(0, 0, 0, 0)), @@ -776,3 +830,27 @@ fn a_restarted_node_rebuilds_its_endpoint_from_the_root_and_the_identity() { assert_eq!(state.revision(), Some(1)); } + +#[test] +fn observer_freshness_must_leave_time_for_a_poll() { + let poll = config()["poll_ms"].as_u64().unwrap(); + for age in [0, poll] { + let error = load(with( + config(), + "max_observation_age_ms", + serde_json::json!(age), + )) + .unwrap_err(); + assert!(format!("{error:#}").contains("max_observation_age_ms")); + } + let configured = load(with( + config(), + "max_observation_age_ms", + serde_json::json!(poll + 1), + )) + .unwrap(); + assert_eq!( + configured.max_observation_age, + std::time::Duration::from_millis(poll + 1) + ); +} diff --git a/crates/cli/src/identity.rs b/crates/cli/src/identity.rs index e472a010..c7463022 100644 --- a/crates/cli/src/identity.rs +++ b/crates/cli/src/identity.rs @@ -290,6 +290,11 @@ fn default_path() -> anyhow::Result { default_hellas_path(IDENTITY_FILE, "--identity") } +#[cfg(feature = "gateway")] +pub(crate) fn default_gateway_archive_path() -> anyhow::Result { + default_hellas_path("gateway-archive", "--archive-dir") +} + #[cfg(feature = "node")] pub(crate) fn default_artifact_store_path() -> anyhow::Result { default_hellas_path(ARTIFACT_STORE_DIR, "--artifact-store-path") diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index df2b7115..e88cdd1c 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -463,12 +463,22 @@ enum Commands { #[command( mut_arg("environment", |arg| arg .required(false) - .required_unless_present("responses_backend") + .required_unless_present_any(["responses_backend", "http_fetch_config"]) .required_if_eq("responses_backend", "hellas") .requires("tokenizer")), mut_arg("tokenizer", |arg| arg.required(false).requires("environment")) )] Gateway { + /// Serve exact HTTP routes through paid HTTPS Fetch. + #[arg(long, value_name = "FILE", conflicts_with_all = ["responses_backend", "environment"])] + #[cfg_attr(feature = "node", arg(requires = "paid_work_config"))] + http_fetch_config: Option, + /// Request/response archive directory (default: ~/.hellas/gateway-archive). + #[arg(long, value_name = "DIRECTORY")] + archive_dir: Option, + /// Disable payload archives and require ZDR for every HTTP request. + #[arg(long)] + zdr: bool, /// Pay a pool of providers using durable on-chain funded work channels. #[cfg(feature = "node")] #[arg(long = "paid-work-config", value_name = "FILE")] @@ -1060,7 +1070,7 @@ async fn async_main() { .map(commands::serve::load_work_config) .transpose() { - Err(error) => Err(error), + Err(error) => Err(error.into()), Ok(work_config) => { async { // The key every settlement this node signs is signed @@ -1139,27 +1149,31 @@ async fn async_main() { max_job_price, print_bond_only, } => match commands::serve::load_work_config(&work_config) { - Err(error) => Err(error), - Ok(work_config) => { + Err(error) => Err(error.into()), + Ok(work_config) => async { commands::serve::run_provision(commands::serve::ProvisionOptions { work_config, // The bond is staked by the party this node already // settles as, taken from the identity loaded above // and never made here. settlement_key: identity::settlement_signer(&local_identity), - client, - stake_coins: stake_coin, + client: hellas_kernel::Key::from_bytes(commands::paid_work::fixed_hex("--client", &client)?), + stake_coins: stake_coin.iter().map(|coin| { + commands::paid_work::fixed_hex("--stake-coin", coin).map(hellas_kernel::CoinId::from_bytes) + }).collect::>()?, bond_timeout, timeout_payout, max_job_price, - print_bond_only, - }) + }, print_bond_only) .await - } + }.await }, Commands::OutputCache(..) => unreachable!("cache commands handled before identity load"), #[cfg(feature = "gateway")] Commands::Gateway { + http_fetch_config, + archive_dir, + zdr, #[cfg(feature = "node")] paid_work_config, bearer_token_file, @@ -1272,20 +1286,19 @@ async fn async_main() { ); #[cfg(feature = "evaluate")] anyhow::ensure!(!verify_local, "--paid-work-config cannot use --verify-local"); + anyhow::ensure!(http_fetch_config.is_some() || assurance == hellas_rpc::Assurance::ProducerSigned, + "token-native paid work uses producer-signed assurance"); anyhow::ensure!( - assurance == hellas_rpc::Assurance::ProducerSigned, - "paid-work execution uses producer-signed assurance", - ); - anyhow::ensure!( - remote_trust.provider_genesis.is_none(), - "--paid-work-config uses the pool's bond and endpoint identities; omit the courtesy --provider pin", + remote_trust.provider_genesis.is_none() + && remote_trust.apple_app_attest_app_id.is_none() + && remote_trust.apple_app_attest_cdhashes.is_empty(), + "set provider enrollment and Apple trust pins in --paid-work-config", ); Some( - commands::paid_work::load_gateway_backend( - path, - secret_key.clone(), - identity::settlement_signer(&local_identity), - ).await? + hellas_sdk::paid_gateway::PaidGateway::open( + hellas_sdk::paid_gateway::load_pool_options(path, assurance)?, + hellas_sdk::ClientIdentity::from_secret_bytes(secret_key.to_bytes(), local_identity.producer_key.to_secret_bytes())?, + ).await? as std::sync::Arc ) } else { None @@ -1306,7 +1319,23 @@ async fn async_main() { remote_trust.apple_app_attest_cdhashes, )? }; + let archive = hellas_gateway::ArchiveOptions { + directory: archive_dir.map(Ok).unwrap_or_else(identity::default_gateway_archive_path)?, + zdr, + }; + if let Some(path) = http_fetch_config { + anyhow::ensure!(output_cache == hellas_rpc::cache::CachePolicy::Off, + "HTTP routes archive exchanges; inference replay must be off"); + anyhow::ensure!(metrics_port.is_none(), "HTTP Fetch exports OpenTelemetry metrics; --metrics-port is unsupported"); + let bytes = commands::read_bounded_regular_file(&path, "HTTP gateway config", 4 << 20)?; + return hellas_gateway::run_http(hellas_gateway::HttpGatewayOptions { + config: serde_json::from_slice(&bytes)?, + paid: paid_work.ok_or_else(|| anyhow::anyhow!("HTTP proxy requires --paid-work-config"))?, + archive, bearer_token_file, allow_remote, host, port, wrap, wrap_args, + }).await; + } hellas_gateway::run(hellas_gateway::GatewayOptions { + archive, output_cache: cache_options, paid_work, bearer_token_file, @@ -1364,6 +1393,7 @@ async fn async_main() { command, secret_key, identity::settlement_signer(&local_identity), + local_identity.producer_key.clone(), ) .await } diff --git a/crates/cli/src/tests.rs b/crates/cli/src/tests.rs index 87db0177..36075d1e 100644 --- a/crates/cli/src/tests.rs +++ b/crates/cli/src/tests.rs @@ -445,6 +445,25 @@ fn gateway_requires_explicit_environment_and_tokenizer() { assert!(Cli::try_parse_from(["hellas", "gateway", "--tokenizer", TEST_TOKENIZER]).is_err()); } +#[cfg(all(feature = "gateway", feature = "node"))] +#[test] +fn http_gateway_requires_a_paid_pool() { + assert!( + Cli::try_parse_from(["hellas", "gateway", "--http-fetch-config", "/http.json"]).is_err() + ); + assert!( + Cli::try_parse_from([ + "hellas", + "gateway", + "--http-fetch-config", + "/http.json", + "--paid-work-config", + "/pool.json" + ]) + .is_ok() + ); +} + #[cfg(feature = "llm")] #[test] fn package_flags_are_not_accepted_as_compatibility_aliases() { diff --git a/crates/cli/src/tracing_config/otel.rs b/crates/cli/src/tracing_config/otel.rs index 6ac1fccb..4b620f2e 100644 --- a/crates/cli/src/tracing_config/otel.rs +++ b/crates/cli/src/tracing_config/otel.rs @@ -89,6 +89,8 @@ where .with_tracer(tracer) .with_filter(tracing_subscriber::filter::filter_fn(|metadata| { metadata.is_span() + && (metadata.target() == "hellas_request" + || metadata.fields().field("otel.kind").is_some()) })); (Some(layer), Some(provider)) } diff --git a/crates/client/src/fetch/tests.rs b/crates/client/src/fetch/tests.rs index 0fe0e8c3..fe3e778e 100644 --- a/crates/client/src/fetch/tests.rs +++ b/crates/client/src/fetch/tests.rs @@ -250,7 +250,11 @@ fn verifier_bounds_retained_streamed_event_count() { .verify_chunk(builder.push_event(Vec::new()).unwrap()) .unwrap_err(); - assert!(error.to_string().contains("4095-chunk limit")); + assert!( + error + .to_string() + .contains(&format!("{}-chunk limit", MAX_FETCH_OUTPUT_EVENTS - 1)) + ); assert_eq!(verifier.events.len(), MAX_FETCH_OUTPUT_EVENTS - 1); } @@ -274,11 +278,9 @@ fn verifier_bounds_retained_streamed_payload_bytes() { .verify_chunk(builder.push_event(vec![1]).unwrap()) .unwrap_err(); - assert!( - error - .to_string() - .contains("2097152-byte signed payload limit") - ); + assert!(error.to_string().contains(&format!( + "{MAX_FETCH_OUTPUT_PAYLOAD_BYTES}-byte signed payload limit" + ))); assert_eq!(verifier.events.len(), 1); assert_eq!( verifier.next_position, diff --git a/crates/client/src/iroh.rs b/crates/client/src/iroh.rs index cc8eea33..654c0f7a 100644 --- a/crates/client/src/iroh.rs +++ b/crates/client/src/iroh.rs @@ -333,7 +333,8 @@ pub fn build_client_registry(endpoint: &::iroh::Endpoint) -> ClientResult( +/// Authenticates a provider on this live connection before disclosing a request. +pub async fn confidential_open( transport: &IrohTransport, trust: &ProviderTrustAnchor, ) -> ClientResult diff --git a/crates/client/src/iroh/tests.rs b/crates/client/src/iroh/tests.rs index 27b4519c..6a2f84a3 100644 --- a/crates/client/src/iroh/tests.rs +++ b/crates/client/src/iroh/tests.rs @@ -430,6 +430,16 @@ fn apple_open_response( exporter: &[u8; 32], nonce: &[u8; 32], counter: u32, + counters: Arc, +) -> (ProviderTrustAnchor, OpenResponse, [u8; 33]) { + apple_open_response_for_alpn(exporter, nonce, ALPN, counter, counters) +} + +fn apple_open_response_for_alpn( + exporter: &[u8; 32], + nonce: &[u8; 32], + alpn: &[u8], + counter: u32, counter_store: Arc, ) -> (ProviderTrustAnchor, OpenResponse, [u8; 33]) { let signing_key = P256SigningKey::from_bytes((&[7; 32]).into()).unwrap(); @@ -474,7 +484,7 @@ fn apple_open_response( nonce, &bundle.genesis.statement.producer_public_key, expected_genesis, - ALPN, + alpn, ); let apple = AppleAppAttestTrust::new("TESTTEAM.example.app", vec![cd_hash], counter_store); *apple.credential.lock().unwrap() = Some(RegisteredAppleCredential { @@ -906,3 +916,71 @@ async fn fetch_open_ticket_and_signed_output_share_one_verified_connection() { ); server.close().await; } + +#[cfg(feature = "work")] +#[test] +fn paid_services_app_attest_binds_the_exact_connection_and_service() { + for alpn in [ + hellas_rpc::services::work::Work::ALPN, + hellas_rpc::services::work_setup::WorkSetup::ALPN, + ] { + let counters = Arc::new(TestCounterStore::default()); + let (trust, response, _) = + apple_open_response_for_alpn(&[1; 32], &[2; 32], alpn.as_bytes(), 1, counters); + assert!( + verify_open_response( + &trust, + &[1; 32], + &[2; 32], + ALPN, + ENROLLED_PEER, + response.clone() + ) + .is_err() + ); + assert!( + verify_open_response( + &trust, + &[3; 32], + &[2; 32], + alpn.as_bytes(), + ENROLLED_PEER, + response.clone() + ) + .is_err() + ); + assert!( + verify_open_response( + &trust, + &[1; 32], + &[4; 32], + alpn.as_bytes(), + ENROLLED_PEER, + response.clone() + ) + .is_err() + ); + assert!( + verify_open_response( + &trust, + &[1; 32], + &[2; 32], + alpn.as_bytes(), + ENROLLED_PEER, + response.clone() + ) + .is_ok() + ); + assert!( + verify_open_response( + &trust, + &[1; 32], + &[2; 32], + alpn.as_bytes(), + ENROLLED_PEER, + response + ) + .is_err() + ); + } +} diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index 5e55cd80..5d418881 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -26,7 +26,9 @@ pub use fetch::{ verified_fetch_input, verify_fetch_work_event, }; #[cfg(feature = "iroh")] -pub use iroh::{AppleAppAttestTrust, ExecutionRoute, ProviderTrustAnchor, RemoteNodeTarget}; +pub use iroh::{ + AppleAppAttestTrust, ExecutionRoute, ProviderTrustAnchor, RemoteNodeTarget, confidential_open, +}; pub use run_ticket::{runner_public_key, signed_run_ticket_request}; /// Client capabilities shared by local and transport-specific orchestration. diff --git a/crates/client/src/work/payment.rs b/crates/client/src/work/payment.rs index ff20fd8e..01e4d142 100644 --- a/crates/client/src/work/payment.rs +++ b/crates/client/src/work/payment.rs @@ -7,7 +7,7 @@ use hellas_rpc::protocol::Digest; use hellas_rpc::services::work::WorkClientImpl; use hellas_wire::StreamTransport; -use hellas_work::work::{ClientEndpoint, PaymentError, admit_payment}; +use hellas_work::work::{ClientChannel, PaymentError, admit_payment}; /// Signs and admits payment for one authenticated result. /// @@ -22,7 +22,7 @@ use hellas_work::work::{ClientEndpoint, PaymentError, admit_payment}; /// journal rejects the transition, or the acknowledgement names another amount. pub async fn pay_for_result( transport: T, - endpoint: &mut ClientEndpoint, + endpoint: &mut impl ClientChannel, work_id: Digest, ) -> Result where diff --git a/crates/client/tests/work.rs b/crates/client/tests/work.rs index 97d9cfe0..d7ca327c 100644 --- a/crates/client/tests/work.rs +++ b/crates/client/tests/work.rs @@ -53,7 +53,7 @@ use hellas_rpc::{ use hellas_wire::mux::{MessagePipe, MuxConfig, MuxTransport, Role as MuxRole}; use hellas_wire::{DefaultClock, Dispatcher, StreamTransport}; use hellas_work::work::{ - BackendFault, ClientEndpoint, PaidEvaluateBackend, PreparedEvaluateInput, ProviderEndpoint, + BackendFault, ClientEndpoint, PaidWorkBackend, PreparedEvaluateInput, ProviderEndpoint, RunOutcome, WorkService, run_accepted_work, }; use hellas_work::work_close::{BlockSourceError, FinalizedBlocks, FinalizedWork, observe}; @@ -190,7 +190,7 @@ fn descriptor_with(policy: PaidExecutionPolicyV1) -> WorkChannelDescriptor { payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: policy, + execution_policy: policy.into(), expected_payment_values: payment_values(), }; match WorkChannelDescriptor::open(config) { @@ -462,7 +462,7 @@ impl AnsweringBackend { } } -impl PaidEvaluateBackend for AnsweringBackend { +impl PaidWorkBackend for AnsweringBackend { fn evaluate( &self, input: PreparedEvaluateInput, diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml index eba5751b..3dbf533e 100644 --- a/crates/executor/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -50,4 +50,5 @@ zeroize = "1" [dev-dependencies] opentelemetry_sdk = { workspace = true, features = ["testing"] } +tokio = { workspace = true, features = ["test-util"] } tracing-subscriber = { workspace = true } diff --git a/crates/executor/src/executor/actor/execution.rs b/crates/executor/src/executor/actor/execution.rs index a36f5b37..57176a1a 100644 --- a/crates/executor/src/executor/actor/execution.rs +++ b/crates/executor/src/executor/actor/execution.rs @@ -60,9 +60,10 @@ use super::{ }; /// Backpressure buffer for the per-execution event channel. The worker keeps -/// one slot reserved for the terminal frame and cancels a consumer that does -/// not drain the rest; it never blocks the sole execution thread. -const PER_EXECUTION_CHANNEL_CAPACITY: usize = 64; +/// one slot reserved for the terminal frame. Backpressure waits in the spawned +/// provider task, leaving the execution actor free to admit and finish work. +pub(super) const PER_EXECUTION_CHANNEL_CAPACITY: usize = 64; +const FETCH_STREAM_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(90); const FETCH_STREAM_STALLED_ERROR: &str = "fetch stream consumer did not drain its bounded event channel"; @@ -594,7 +595,7 @@ impl Executor { result.map_err(|error| error.to_string()) } - fn retry_deferred_fetch_quota_cancellations(&mut self) { + pub(super) fn retry_deferred_fetch_quota_cancellations(&mut self) { let attempts = self .pending_fetch_quota_cancellations .len() @@ -667,7 +668,7 @@ impl Executor { } } - fn retry_deferred_fetch_quota_settlements(&mut self) { + pub(super) fn retry_deferred_fetch_quota_settlements(&mut self) { let attempts = self .pending_fetch_quota_settlements .len() @@ -807,7 +808,8 @@ impl Executor { self.finish_fetch_slot(); } - fn dispatch_next_fetch(&mut self) { + pub(super) fn dispatch_next_fetch(&mut self) { + self.dispatch_paid_fetches(); while self.active_fetches < self.fetch_max_in_flight { let Some(pending) = self.pending_fetches.pop_front() else { return; @@ -984,7 +986,7 @@ fn spawn_fetch_provider( )); } -async fn run_fetch_provider( +pub(super) async fn run_fetch_provider( provider: Arc, request: PreparedFetchRequest, mut projector: Box, @@ -997,9 +999,10 @@ async fn run_fetch_provider( let mut position = 0_u64; let mut terminal = None; let mut projection_budget = FetchProjectionBudget::default(); - let response = provider - .run(request) - .await + let response = tokio::select! { + response = provider.run(request) => response, + _ = sender.closed() => Err(FetchProviderError::failed("fetch stream consumer disconnected")), + } .map_err(|error| FetchProviderFailure { position, error })?; let projected = projector .begin(response.head) @@ -1018,7 +1021,16 @@ async fn run_fetch_provider( .await?; let mut stream = response.stream; - while let Some(next) = stream.next().await { + loop { + let next = tokio::select! { + next = stream.next() => next, + _ = sender.closed() => return Err(FetchProviderFailure { + position, error: FetchProviderError::failed("fetch stream consumer disconnected"), + }), + }; + let Some(next) = next else { + break; + }; let chunk = next.map_err(|error| FetchProviderFailure { position, error })?; let projected = projector .project(&chunk) @@ -1096,15 +1108,19 @@ async fn process_projected_fetch( ), }); } - // The actor needs one guaranteed permit for WorkFinished or - // WorkFailed. Fail before signing another chunk when only that - // permit remains; never await a slow consumer here. - if sender.capacity() <= 1 { - return Err(FetchProviderFailure { - position: *position, - error: FetchProviderError::failed(FETCH_STREAM_STALLED_ERROR), - }); - } + // Reserve the event and terminal slots together. Temporary + // backpressure must not truncate a valid upstream response. + let mut permits = + tokio::time::timeout(FETCH_STREAM_DRAIN_TIMEOUT, sender.reserve_many(2)) + .await + .map_err(|_| FetchProviderFailure { + position: *position, + error: FetchProviderError::failed(FETCH_STREAM_STALLED_ERROR), + })? + .map_err(|_| FetchProviderFailure { + position: *position, + error: FetchProviderError::failed("fetch stream consumer disconnected"), + })?; let payload_len = projection_budget .record_event(payload.len()) @@ -1128,31 +1144,16 @@ async fn process_projected_fetch( "fetch output event transcript failed: {err}" )), })?; - match sender.try_send(Ok(WorkEvent { - kind: Some(work_event::Kind::Chunk(WorkChunk { - output_event: Some(output_event_to_pb(&output_event)), - })), - })) { - Ok(()) => { - *position = next_position; - // Give the just-returned RPC receiver a scheduling - // opportunity before classifying a full burst as a - // stalled consumer. - tokio::task::yield_now().await; - } - Err(mpsc::error::TrySendError::Full(_)) => { - return Err(FetchProviderFailure { - position: *position, - error: FetchProviderError::failed(FETCH_STREAM_STALLED_ERROR), - }); - } - Err(mpsc::error::TrySendError::Closed(_)) => { - return Err(FetchProviderFailure { - position: *position, - error: FetchProviderError::failed("fetch stream consumer disconnected"), - }); - } - } + permits + .next() + .expect("two reserved permits") + .send(Ok(WorkEvent { + kind: Some(work_event::Kind::Chunk(WorkChunk { + output_event: Some(output_event_to_pb(&output_event)), + })), + })); + drop(permits); + *position = next_position; } ProjectedFetch::Terminal(payload) => { if terminal.is_some() { diff --git a/crates/executor/src/executor/actor/execution/tests.rs b/crates/executor/src/executor/actor/execution/tests.rs index 7bd1733e..7a41e8f5 100644 --- a/crates/executor/src/executor/actor/execution/tests.rs +++ b/crates/executor/src/executor/actor/execution/tests.rs @@ -1220,7 +1220,7 @@ async fn fetch_projection_rejects_event_buffered_after_terminal() { assert_eq!(projection_budget.events, 1); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn fetch_projection_preserves_one_permit_for_failure_terminal() { let signing_key = key(); let input_commitment = InputCommitment::from_digest(Digest::from_bytes([8; 32])); @@ -1268,6 +1268,48 @@ async fn fetch_projection_preserves_one_permit_for_failure_terminal() { )); } +#[tokio::test(start_paused = true)] +async fn fetch_projection_waits_for_temporary_backpressure_then_delivers_in_order() { + let signing_key = key(); + let input = InputCommitment::from_digest(Digest::from_bytes([8; 32])); + let mut builder = FetchOutputTranscriptBuilder::new(input, test_assurance(), &signing_key); + let mut terminal = None; + let mut budget = FetchProjectionBudget::default(); + let mut position = 0; + let (sender, mut receiver) = mpsc::channel(2); + sender + .try_send(Ok(WorkEvent { + kind: Some(work_event::Kind::Chunk(WorkChunk { output_event: None })), + })) + .unwrap(); + let delayed_reader = async { + tokio::time::sleep(Duration::from_secs(2)).await; + assert!(receiver.recv().await.unwrap().unwrap().kind.is_some()); + let event = receiver.recv().await.unwrap().unwrap(); + assert!(matches!( + event.kind, + Some(work_event::Kind::Chunk(WorkChunk { + output_event: Some(_) + })) + )); + }; + let deliver = process_projected_fetch( + vec![ProjectedFetch::Event(b"delayed".to_vec())], + &mut builder, + &mut terminal, + &mut budget, + &mut position, + &sender, + ); + let (result, ()) = tokio::join!(deliver, delayed_reader); + let Ok(reached_terminal) = result else { + panic!("temporary backpressure truncated the response"); + }; + assert!(!reached_terminal); + assert_eq!(position, 7); + assert_eq!(budget.events, 1); +} + #[tokio::test] async fn retained_replay_waits_for_a_consumer_without_losing_events() { let signing_key = key(); @@ -1369,7 +1411,11 @@ fn fetch_projection_budget_caps_event_count_and_payload_bytes() { }; event_budget.record_event(0).unwrap(); let event_error = event_budget.record_event(0).unwrap_err(); - assert!(event_error.to_string().contains("4095-event limit")); + assert!( + event_error + .to_string() + .contains(&format!("{}-event limit", MAX_FETCH_OUTPUT_EVENTS - 1)) + ); assert_eq!(event_budget.events, MAX_FETCH_OUTPUT_EVENTS - 1); event_budget.record_terminal(0).unwrap(); assert_eq!(event_budget.events, MAX_FETCH_OUTPUT_EVENTS); @@ -1380,11 +1426,9 @@ fn fetch_projection_budget_caps_event_count_and_payload_bytes() { }; payload_budget.record_event(1).unwrap(); let payload_error = payload_budget.record_event(1).unwrap_err(); - assert!( - payload_error - .to_string() - .contains("2097152-byte signed payload limit") - ); + assert!(payload_error.to_string().contains(&format!( + "{MAX_FETCH_OUTPUT_PAYLOAD_BYTES}-byte signed payload limit" + ))); assert_eq!( payload_budget.signed_payload_bytes, MAX_FETCH_OUTPUT_PAYLOAD_BYTES @@ -1425,7 +1469,9 @@ async fn projected_payload_limit_is_reported_as_work_failed() { let failed = run_failed(&handle, ticket, &signing_key).await; assert_eq!(failed.position, 0); - assert!(failed.error.contains("2097152-byte signed payload limit")); + assert!(failed.error.contains(&format!( + "{MAX_FETCH_OUTPUT_PAYLOAD_BYTES}-byte signed payload limit" + ))); } #[tokio::test] diff --git a/crates/executor/src/executor/actor/mod.rs b/crates/executor/src/executor/actor/mod.rs index 823bb709..8b2087ea 100644 --- a/crates/executor/src/executor/actor/mod.rs +++ b/crates/executor/src/executor/actor/mod.rs @@ -1,4 +1,5 @@ pub(super) mod execution; +mod paid_fetch; mod quote; use crate::ExecutorError; @@ -65,6 +66,7 @@ struct InboxArbiter { preference: TrustedPreference, request_open: bool, owed_open: bool, + owed_paused: bool, } impl Default for InboxArbiter { @@ -73,6 +75,7 @@ impl Default for InboxArbiter { preference: TrustedPreference::Owed, request_open: true, owed_open: true, + owed_paused: false, } } } @@ -122,6 +125,7 @@ pub struct Executor { pub(super) fetch_routes: FetchRouteRegistry, pub(super) fetch_cache: Option>, pub(super) pending_fetches: VecDeque, + pending_paid_fetches: VecDeque, pending_fetch_quota_cancellations: VecDeque, pending_fetch_quota_settlements: VecDeque, pub(super) fetch_max_in_flight: usize, @@ -152,6 +156,39 @@ pub struct ExecutorSpawnConfig { pub gpu_config: crate::GpuConfig, } +impl ExecutorSpawnConfig { + /// Construct a Fetch-only runtime independently of whether another Cargo + /// consumer enables Evaluate. Hosts can then override limits and stores. + pub fn fetch_only( + producer_key: Arc, + provider_genesis: Arc>, + assurance: Assurance, + fetch_routes: FetchRouteRegistry, + ) -> Self { + Self { + output_cache: hellas_rpc::cache::CacheOptions::default(), + execute_policy: ExecutePolicy::Deny, + queue_capacity: 1, + metrics: Arc::new(ExecutorMetrics::default()), + producer_key, + provider_genesis, + assurance, + fetch_access_policy: FetchAccessPolicy::trusted_callers([]), + fetch_routes, + fetch_max_in_flight: hellas_rpc::DEFAULT_FETCH_MAX_IN_FLIGHT, + fetch_queue_capacity: hellas_rpc::DEFAULT_FETCH_QUEUE_CAPACITY, + fetch_replay_max_in_flight: hellas_rpc::DEFAULT_FETCH_REPLAY_MAX_IN_FLIGHT, + fetch_store: FetchTranscriptStoreBackend::memory(), + #[cfg(feature = "evaluate")] + artifact_store: ArtifactStoreConfig::memory(), + #[cfg(feature = "evaluate")] + content_store: ContentStore::new(), + #[cfg(feature = "evaluate")] + gpu_config: crate::GpuConfig::default(), + } + } +} + struct ExecutorRuntimeConfig { output_cache: hellas_rpc::cache::CacheOptions, #[cfg_attr(not(feature = "evaluate"), allow(dead_code))] @@ -368,6 +405,7 @@ impl Executor { fetch_access_policy: config.fetch_access_policy, fetch_routes: config.fetch_routes, pending_fetches: VecDeque::new(), + pending_paid_fetches: VecDeque::new(), fetch_cache: execution::cache::FetchCache::open(config.output_cache)?, pending_fetch_quota_cancellations: VecDeque::new(), pending_fetch_quota_settlements: VecDeque::new(), @@ -385,14 +423,18 @@ impl Executor { async fn run(mut self) { let mut arbiter = InboxArbiter::default(); - while let Some(message) = recv_next( - &mut self.request_rx, - &mut self.owed_rx, - &mut self.completion_rx, - &mut arbiter, - ) - .await - { + loop { + arbiter.owed_paused = self.pending_paid_fetches.len() >= EXECUTOR_OWED_MAILBOX_CAPACITY; + let Some(message) = recv_next( + &mut self.request_rx, + &mut self.owed_rx, + &mut self.completion_rx, + &mut arbiter, + ) + .await + else { + break; + }; match message { ExecutorInbox::Completion(completion) => { let evaluate_finished = self.handle_completion(completion).await; @@ -419,6 +461,14 @@ impl Executor { self.evaluate.on_completion(*completion).await; true } + ExecutorCompletion::PaidFetch { reply, result } => { + self.active_fetches = self.active_fetches.saturating_sub(1); + self.dispatch_next_fetch(); + self.retry_deferred_fetch_quota_settlements(); + self.retry_deferred_fetch_quota_cancellations(); + let _ = reply.send(result); + false + } ExecutorCompletion::FetchFinished(completion) => { self.handle_fetch_finished(*completion); false @@ -433,6 +483,9 @@ impl Executor { let ready = self.owed_rx.len(); let mut admitted = 0; for _ in 0..ready { + if self.pending_paid_fetches.len() >= EXECUTOR_OWED_MAILBOX_CAPACITY { + break; + } let Ok(request) = self.owed_rx.try_recv() else { break; }; @@ -450,6 +503,14 @@ impl Executor { async fn handle_owed_request(&mut self, request: ExecutorOwedRequest) { match request { + ExecutorOwedRequest::RunPaidFetch { + span, + input, + progress, + reply, + } => { + self.start_paid_fetch(*input, progress, reply, span); + } ExecutorOwedRequest::RunPaidEvaluate { input, reply, span } => { #[cfg(feature = "evaluate")] let result = tracing::Instrument::instrument( @@ -566,7 +627,7 @@ async fn recv_next( TrustedPreference::Owed => { tokio::select! { biased; - owed = owed_rx.recv(), if arbiter.owed_open => { + owed = owed_rx.recv(), if arbiter.owed_open && !arbiter.owed_paused => { match owed { Some(request) => Some(ExecutorInbox::Owed(request)), None => { @@ -595,7 +656,7 @@ async fn recv_next( Some(completion) = completion_rx.recv() => { Some(ExecutorInbox::Completion(completion)) } - owed = owed_rx.recv(), if arbiter.owed_open => { + owed = owed_rx.recv(), if arbiter.owed_open && !arbiter.owed_paused => { match owed { Some(request) => Some(ExecutorInbox::Owed(request)), None => { @@ -735,6 +796,7 @@ mod mailbox_tests { fetch_access_policy: FetchAccessPolicy::trusted_callers([caller]), fetch_routes: FetchRouteRegistry::default(), pending_fetches: VecDeque::new(), + pending_paid_fetches: VecDeque::new(), pending_fetch_quota_cancellations: VecDeque::new(), pending_fetch_quota_settlements: VecDeque::new(), fetch_max_in_flight: 1, diff --git a/crates/executor/src/executor/actor/paid_fetch.rs b/crates/executor/src/executor/actor/paid_fetch.rs new file mode 100644 index 00000000..cce5654b --- /dev/null +++ b/crates/executor/src/executor/actor/paid_fetch.rs @@ -0,0 +1,781 @@ +//! Paid Fetch runs have their own durable admission in hellas-work. Bodies +//! remain in memory; neither Courtesy replay nor its transcript store is used. + +use std::sync::Arc; + +use hellas_rpc::OutputEventEnvelope; +use hellas_work::work::PreparedFetchInput; +use tokio::sync::{mpsc, oneshot}; +use tracing::Instrument as _; + +use super::Executor; +use crate::ExecutorError; +use crate::executor::ExecutorCompletion; +use crate::fetch_policy::FetchRoute; +use crate::fetch_provider::FetchCall; + +pub(super) struct PendingPaidFetch { + input: PreparedFetchInput, + progress: Option, + reply: oneshot::Sender, ExecutorError>>, + span: tracing::Span, +} + +impl Executor { + /// Admits one journaled paid Fetch, or refuses it before any provider + /// work begins. + /// + /// Already accepted work waits in a bounded FIFO when dispatch is full. + /// Backpressure then reaches the owed mailbox; peer requests and completion + /// handling remain live while the upstream is busy. + pub(super) fn start_paid_fetch( + &mut self, + input: PreparedFetchInput, + progress: Option, + reply: oneshot::Sender, ExecutorError>>, + span: tracing::Span, + ) { + if self.active_fetches >= self.fetch_max_in_flight { + self.pending_paid_fetches.push_back(PendingPaidFetch { + input, + progress, + reply, + span, + }); + return; + } + let prepared = self.prepare_paid_fetch(input); + let (entry, session, request, policy) = match prepared { + Ok(prepared) => prepared, + Err(error) => { + let _ = reply.send(Err(error)); + return; + } + }; + self.active_fetches += 1; + let completion = self.completion_tx.clone(); + let key = Arc::clone(&self.provider.producer_key); + let task = async move { + let (sender, mut receiver) = + mpsc::channel(super::execution::PER_EXECUTION_CHANNEL_CAPACITY); + let run = super::execution::run_fetch_provider( + entry.provider, + session.provider_request, + session.projector, + request.input_commitment, + request.assurance, + &key, + sender, + ); + let drain = async move { + while let Some(event) = receiver.recv().await { + let event = + event.map_err(|error| ExecutorError::Execution(error.to_string()))?; + if let Some(hellas_rpc::pb::execute::work_event::Kind::Chunk(chunk)) = + event.kind + && let Some(progress) = &progress + { + let event = chunk.output_event.ok_or_else(|| { + ExecutorError::Execution( + "paid Fetch chunk omitted its signature".into(), + ) + })?; + let event = hellas_rpc::stream::output_event_from_pb(event) + .map_err(|error| ExecutorError::Execution(error.to_string()))?; + progress(event) + .map_err(|error| ExecutorError::Execution(error.to_string()))?; + } + } + Ok::<_, ExecutorError>(()) + }; + let (result, drained) = tokio::join!(run, drain); + let result = drained.and_then(|()| { + result + .map_err(|failure| { + // The peer receives the sanitized message below; the + // operator needs the position and cause to tell + // upstream, transport and projection faults apart. + tracing::warn!( + position = failure.position, + error = %failure.error, + "paid fetch upstream or projection failed" + ); + ExecutorError::Execution("paid fetch upstream or projection failed".into()) + }) + .and_then(|run| { + hellas_rpc::protocol::work_fetch::check_fetch_output_limits( + &policy, + &run.output_events, + ) + .map_err(|_| { + ExecutorError::Execution( + "paid fetch output exceeds its signed limits".into(), + ) + })?; + Ok(run.output_events) + }) + }); + let _ = completion + .send(ExecutorCompletion::PaidFetch { reply, result }) + .await; + }; + tokio::spawn(task.instrument(span)); + } + + fn prepare_paid_fetch( + &self, + input: PreparedFetchInput, + ) -> Result< + ( + crate::FetchRouteEntry, + crate::FetchAdaptorSession, + hellas_rpc::fetch::FetchInput, + hellas_rpc::protocol::work_fetch::PaidFetchPolicyV1, + ), + ExecutorError, + > { + let policy = *input.policy(); + let parts = input.into_parts(); + let request = hellas_rpc::fetch::verify_input_events(&parts.fetch_input_transcript) + .map_err(|_| ExecutorError::InvalidQuoteRequest("invalid paid fetch input".into()))?; + if request.retention != hellas_rpc::Retention::Ephemeral + || request.assurance != self.provider.assurance + || request.execution_environment != parts.manifest.content_id() + { + return Err(ExecutorError::InvalidQuoteRequest( + "paid fetch contract mismatch".into(), + )); + } + let route = FetchRoute::new(&request.service, &request.method); + let entry = self.fetch_routes.entry(&route).cloned().ok_or_else(|| { + ExecutorError::InvalidQuoteRequest("paid fetch route is unavailable".into()) + })?; + if entry.execution_environment() != request.execution_environment { + return Err(ExecutorError::InvalidQuoteRequest( + "paid fetch route manifest mismatch".into(), + )); + } + let call = FetchCall::new( + &request.service, + &request.method, + request.body.clone(), + request.input_commitment, + ); + let session = entry.adaptor_factory.create(&call).map_err(|_| { + ExecutorError::InvalidQuoteRequest("paid fetch adaptor rejected request".into()) + })?; + entry + .capabilities + .validate(&session.request_view) + .map_err(|_| { + ExecutorError::PolicyDenied("paid fetch exceeds route capabilities".into()) + })?; + Ok((entry, session, request, policy)) + } + + pub(super) fn dispatch_paid_fetches(&mut self) { + while self.active_fetches < self.fetch_max_in_flight { + let Some(pending) = self.pending_paid_fetches.pop_front() else { + break; + }; + // The durable invocation is owed even if its original waiter left. + self.start_paid_fetch(pending.input, pending.progress, pending.reply, pending.span); + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::time::Duration; + + use futures_util::stream; + use hellas_rpc::fetch::{ + FetchTerminalPayload, build_input_events_with_retention, decode_fetch_event_payload, + decode_fetch_terminal_payload, encode_fetch_event_payload, encode_fetch_terminal_payload, + verify_input_events, verify_output_events, + }; + use hellas_rpc::output::{OutputEvent, StopReason, TextChannel}; + use hellas_rpc::protocol::work_fetch::{PaidFetchPolicyV1, PreparedPaidFetchInputParts}; + use hellas_rpc::{ + Assurance, ContentId, Digest, FetchEnvironment, InputCommitment, InputEventEnvelope, + ProducerSigningKey, Retention, + }; + use hellas_work::work::PaidProgress; + use tokio::sync::Notify; + use tokio::time::timeout; + + use super::*; + use crate::ExecutorSpawnConfig; + use crate::executor::{ExecutorHandle, ExecutorOwedRequest}; + use crate::fetch_policy::FetchRoutePolicy; + use crate::fetch_projection::{ + FetchAdaptorError, FetchAdaptorFactory, FetchAdaptorSession, FetchProjector, + FetchRequestView, ProjectedFetch, + }; + use crate::fetch_provider::{ + FetchProvider, FetchProviderFuture, FetchProviderResponse, FetchProviderResponseHead, + FetchProviderStream, MockFetchProvider, PreparedFetchRequest, + }; + use crate::fetch_registry::{FetchRouteEntry, FetchRouteRegistry}; + + const SERVICE: &str = "openai"; + const METHOD: &str = "responses"; + const BODY: &[u8] = br#"{"input":"paid"}"#; + + fn environment() -> FetchEnvironment { + FetchEnvironment::OpenAiResponses + } + + fn signing_key() -> ProducerSigningKey { + ProducerSigningKey::from_secret_bytes([7; 32]).expect("valid test key") + } + + fn input_events(retention: Retention, assurance: Assurance) -> Vec { + build_input_events_with_retention( + SERVICE, + METHOD, + BODY, + environment().manifest_id(), + assurance, + &signing_key(), + retention, + ) + .expect("test input transcript builds") + } + + fn policy() -> PaidFetchPolicyV1 { + PaidFetchPolicyV1 { + allowed_environment: environment().manifest_id(), + route_commitment: Digest::from_bytes([3; 32]), + max_request_body_bytes: 4096, + max_output_events: 64, + max_output_bytes: 16384, + max_spool_bytes: 65536, + max_encoded_result_frame: 65536, + max_encoded_prepared_input: 65536, + dispatch_margin_blocks: 4, + delivery_margin_blocks: 2, + oracle_grace_blocks: 6, + fixed_price: 7, + } + } + + fn prepared_input(events: Vec) -> PreparedFetchInput { + PreparedFetchInput::new( + PreparedPaidFetchInputParts { + fetch_input_transcript: events, + manifest: environment().manifest(), + }, + policy(), + ) + } + + struct StubFetchAdaptorFactory { + environment: ContentId, + reject: bool, + } + + impl FetchAdaptorFactory for StubFetchAdaptorFactory { + fn execution_environment(&self) -> ContentId { + self.environment + } + + fn create(&self, request: &FetchCall) -> Result { + if self.reject { + return Err(FetchAdaptorError::failed( + "stub adaptor rejects the request", + )); + } + Ok(FetchAdaptorSession { + request_view: FetchRequestView::from_call(request), + provider_request: PreparedFetchRequest::new(request, request.body.clone()), + projector: Box::new(StubFetchProjector { projected: false }), + }) + } + } + + struct StubFetchProjector { + projected: bool, + } + + impl FetchProjector for StubFetchProjector { + fn project(&mut self, _bytes: &[u8]) -> Result, FetchAdaptorError> { + if std::mem::replace(&mut self.projected, true) { + return Ok(Vec::new()); + } + let event = encode_fetch_event_payload(&OutputEvent::TextDelta { + index: 0, + delta: "paid-output".to_string(), + channel: TextChannel::Output, + }) + .map_err(|error| FetchAdaptorError::failed(error.to_string()))?; + let terminal = encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .map_err(|error| FetchAdaptorError::failed(error.to_string()))?; + Ok(vec![ + ProjectedFetch::Event(event), + ProjectedFetch::Terminal(terminal), + ]) + } + + fn finish(&mut self) -> Result, FetchAdaptorError> { + Ok(Vec::new()) + } + } + + #[derive(Clone)] + struct BlockingFetchProvider { + environment: ContentId, + started: Arc, + released: Arc, + notify: Arc, + } + + impl BlockingFetchProvider { + fn new(environment: ContentId) -> Self { + Self { + environment, + started: Arc::new(AtomicUsize::new(0)), + released: Arc::new(AtomicBool::new(false)), + notify: Arc::new(Notify::new()), + } + } + + fn started(&self) -> usize { + self.started.load(Ordering::SeqCst) + } + + fn release(&self) { + self.released.store(true, Ordering::SeqCst); + self.notify.notify_waiters(); + } + } + + impl FetchProvider for BlockingFetchProvider { + fn execution_environment(&self) -> ContentId { + self.environment + } + + fn run(&self, _request: PreparedFetchRequest) -> FetchProviderFuture<'_> { + let started = Arc::clone(&self.started); + let released = Arc::clone(&self.released); + let notify = Arc::clone(&self.notify); + Box::pin(async move { + started.fetch_add(1, Ordering::SeqCst); + while !released.load(Ordering::SeqCst) { + notify.notified().await; + } + Ok(FetchProviderResponse { + head: FetchProviderResponseHead::default(), + stream: Box::pin(stream::iter([Ok(b"chunk".to_vec())])) as FetchProviderStream, + }) + }) + } + } + + fn routes( + provider: Arc, + adaptor_factory: Arc, + capabilities: FetchRoutePolicy, + ) -> FetchRouteRegistry { + let mut registry = FetchRouteRegistry::new(); + registry + .register( + FetchRoute::new(SERVICE, METHOD), + FetchRouteEntry::new(provider, adaptor_factory, capabilities) + .expect("test provider and adaptor identities match"), + ) + .expect("test route registers"); + registry + } + + fn stub_adaptor(environment: ContentId) -> Arc { + Arc::new(StubFetchAdaptorFactory { + environment, + reject: false, + }) + } + + async fn spawn_executor( + routes: FetchRouteRegistry, + fetch_max_in_flight: usize, + ) -> ExecutorHandle { + let mut config = ExecutorSpawnConfig::fetch_only( + Arc::new(signing_key()), + Arc::new(b"paid-fetch-tests".to_vec()), + Assurance::ProducerSigned, + routes, + ); + config.fetch_max_in_flight = fetch_max_in_flight; + Executor::spawn_configured(config) + .await + .expect("test executor spawns") + } + + async fn spawn_stub_executor(provider: MockFetchProvider) -> ExecutorHandle { + spawn_executor( + routes( + Arc::new(provider), + stub_adaptor(environment().manifest_id()), + FetchRoutePolicy::default(), + ), + 1, + ) + .await + } + + async fn run_paid_fetch( + handle: &ExecutorHandle, + input: PreparedFetchInput, + progress: Option, + ) -> Result, ExecutorError> { + let (reply, receive) = oneshot::channel(); + handle + .owed_tx + .send(ExecutorOwedRequest::RunPaidFetch { + span: tracing::Span::none(), + input: Box::new(input), + progress, + reply, + }) + .await + .expect("owed ingress remains open"); + timeout(Duration::from_secs(5), receive) + .await + .expect("paid fetch answers within the test budget") + .expect("actor answers the paid fetch") + } + + fn commitment_of(events: &[InputEventEnvelope]) -> InputCommitment { + verify_input_events(events) + .expect("test input verifies") + .input_commitment + } + + #[tokio::test] + async fn paid_fetch_rejects_a_tampered_input_transcript() { + let handle = spawn_stub_executor(MockFetchProvider::new(environment().manifest_id())).await; + let mut events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + let foreign = build_input_events_with_retention( + SERVICE, + METHOD, + BODY, + environment().manifest_id(), + Assurance::ProducerSigned, + &ProducerSigningKey::from_secret_bytes([8; 32]).expect("valid foreign key"), + Retention::Ephemeral, + ) + .expect("foreign transcript builds"); + // Both halves are well formed; the spliced terminal event fails + // signature verification under the transcript's signer. + *events.last_mut().expect("input transcript has events") = foreign + .last() + .expect("foreign transcript has events") + .clone(); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "invalid paid fetch input") + ); + } + + #[tokio::test] + async fn paid_fetch_rejects_retained_input() { + let handle = spawn_stub_executor(MockFetchProvider::new(environment().manifest_id())).await; + let events = input_events(Retention::Retain, Assurance::ProducerSigned); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "paid fetch contract mismatch") + ); + } + + #[tokio::test] + async fn paid_fetch_rejects_mismatched_assurance() { + let handle = spawn_stub_executor(MockFetchProvider::new(environment().manifest_id())).await; + let events = input_events(Retention::Ephemeral, Assurance::AppleAppAttest); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "paid fetch contract mismatch") + ); + } + + #[tokio::test] + async fn paid_fetch_rejects_a_manifest_other_than_the_signed_environment() { + let handle = spawn_stub_executor(MockFetchProvider::new(environment().manifest_id())).await; + // The transcript is valid but signs another environment than the + // bundled manifest commits to. + let events = build_input_events_with_retention( + SERVICE, + METHOD, + BODY, + ContentId::from_bytes([8; 32]), + Assurance::ProducerSigned, + &signing_key(), + Retention::Ephemeral, + ) + .expect("foreign environment transcript builds"); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "paid fetch contract mismatch") + ); + } + + #[tokio::test] + async fn paid_fetch_rejects_an_unregistered_route() { + let handle = spawn_executor(FetchRouteRegistry::new(), 1).await; + let events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "paid fetch route is unavailable") + ); + } + + #[tokio::test] + async fn paid_fetch_rejects_a_route_for_another_environment() { + let other = ContentId::from_bytes([8; 32]); + let handle = spawn_executor( + routes( + Arc::new(MockFetchProvider::new(other)), + stub_adaptor(other), + FetchRoutePolicy::default(), + ), + 1, + ) + .await; + let events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "paid fetch route manifest mismatch") + ); + } + + #[tokio::test] + async fn paid_fetch_rejects_an_adaptor_refusal_without_reaching_the_provider() { + let provider = MockFetchProvider::new(environment().manifest_id()); + let handle = spawn_executor( + routes( + Arc::new(provider.clone()), + Arc::new(StubFetchAdaptorFactory { + environment: environment().manifest_id(), + reject: true, + }), + FetchRoutePolicy::default(), + ), + 1, + ) + .await; + let events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "paid fetch adaptor rejected request") + ); + assert_eq!(provider.calls(SERVICE, METHOD, BODY), 0); + } + + #[tokio::test] + async fn paid_fetch_rejects_a_request_beyond_route_capabilities() { + let capabilities = FetchRoutePolicy { + allowed_models: Some(BTreeSet::from(["gpt-5".to_string()])), + max_output_units: None, + }; + let handle = spawn_executor( + routes( + Arc::new(MockFetchProvider::new(environment().manifest_id())), + stub_adaptor(environment().manifest_id()), + capabilities, + ), + 1, + ) + .await; + let events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::PolicyDenied(error)) if error == "paid fetch exceeds route capabilities") + ); + } + + #[tokio::test] + async fn paid_fetch_waits_for_capacity_without_blocking_the_actor() { + let provider = BlockingFetchProvider::new(environment().manifest_id()); + let handle = spawn_executor( + routes( + Arc::new(provider.clone()), + stub_adaptor(environment().manifest_id()), + FetchRoutePolicy::default(), + ), + 1, + ) + .await; + let first_events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + let first_commitment = commitment_of(&first_events); + let first = tokio::spawn({ + let handle = handle.clone(); + async move { run_paid_fetch(&handle, prepared_input(first_events), None).await } + }); + timeout(Duration::from_secs(5), async { + while provider.started() == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("first paid fetch reaches the provider"); + + let second = tokio::spawn({ + let handle = handle.clone(); + async move { + run_paid_fetch( + &handle, + prepared_input(input_events( + Retention::Ephemeral, + Assurance::ProducerSigned, + )), + None, + ) + .await + } + }); + // Peer-facing requests remain serviceable while both paid jobs wait. + timeout(Duration::from_secs(1), handle.get_stats_handle()) + .await + .unwrap() + .unwrap(); + assert!(!second.is_finished()); + assert_eq!(provider.started(), 1); + + provider.release(); + let output = first + .await + .expect("first paid fetch task completes") + .expect("first paid fetch succeeds"); + second.await.unwrap().expect("queued paid fetch succeeds"); + assert_eq!(provider.started(), 2); + verify_output_events(first_commitment, Assurance::ProducerSigned, &output) + .expect("first paid fetch output verifies"); + } + + #[tokio::test] + async fn full_paid_queue_backpressures_ingress_but_keeps_completions_live() { + timeout(Duration::from_secs(10), async { + let provider = BlockingFetchProvider::new(environment().manifest_id()); + let handle = spawn_executor( + routes( + Arc::new(provider.clone()), + stub_adaptor(environment().manifest_id()), + FetchRoutePolicy::default(), + ), + 1, + ) + .await; + let input = prepared_input(input_events( + Retention::Ephemeral, + Assurance::ProducerSigned, + )); + let request = |reply| crate::executor::ExecutorOwedRequest::RunPaidFetch { + span: tracing::Span::none(), + input: Box::new(input.clone()), + progress: None, + reply, + }; + let (reply, first) = oneshot::channel(); + handle.owed_tx.send(request(reply)).await.unwrap(); + while provider.started() == 0 { + tokio::task::yield_now().await; + } + let mut pending = Vec::new(); + // One bounded actor FIFO plus its bounded ingress mailbox. + for _ in 0..2 * super::super::EXECUTOR_OWED_MAILBOX_CAPACITY { + let (reply, result) = oneshot::channel(); + handle.owed_tx.send(request(reply)).await.unwrap(); + pending.push(result); + } + let (reply, last) = oneshot::channel(); + let blocked = handle.owed_tx.send(request(reply)); + tokio::pin!(blocked); + assert!( + timeout(Duration::from_millis(50), &mut blocked) + .await + .is_err() + ); + handle.get_stats_handle().await.unwrap(); + assert_eq!(handle.owed_tx.capacity(), 0); + assert_eq!(provider.started(), 1); + // An abandoned waiter cannot discard an already owed invocation. + drop(pending.pop().unwrap()); + provider.release(); + blocked.await.unwrap(); + first.await.unwrap().unwrap(); + for result in pending { + result.await.unwrap().unwrap(); + } + last.await.unwrap().unwrap(); + assert_eq!( + provider.started(), + 2 + 2 * super::super::EXECUTOR_OWED_MAILBOX_CAPACITY + ); + }) + .await + .expect("queue drains once the upstream is available"); + } + + #[tokio::test] + async fn paid_fetch_streams_signed_progress_and_returns_signed_output() { + let provider = MockFetchProvider::new(environment().manifest_id()); + provider.insert(SERVICE, METHOD, BODY, [b"chunk".to_vec()]); + let handle = spawn_stub_executor(provider.clone()).await; + let events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + let input_commitment = commitment_of(&events); + let progressed = Arc::new(AtomicUsize::new(0)); + let progress: PaidProgress = { + let progressed = Arc::clone(&progressed); + Arc::new(move |event| { + assert_eq!( + event.event().body().kind(), + hellas_rpc::fetch::OUTPUT_EVENT_KIND + ); + progressed.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + }; + + let output = run_paid_fetch(&handle, prepared_input(events), Some(progress)) + .await + .expect("paid fetch succeeds"); + + assert_eq!(progressed.load(Ordering::SeqCst), 1); + assert_eq!(provider.calls(SERVICE, METHOD, BODY), 1); + let verified = verify_output_events(input_commitment, Assurance::ProducerSigned, &output) + .expect("paid fetch output verifies"); + assert_eq!(verified.producer_key, signing_key().public_key()); + let (payloads, terminal) = verified.output_event_payloads(); + let [payload] = payloads else { + panic!("one streamed event before the terminal") + }; + assert!(matches!( + decode_fetch_event_payload(payload), + Ok(OutputEvent::TextDelta { .. }) + )); + assert!(matches!( + decode_fetch_terminal_payload(terminal), + Ok(FetchTerminalPayload::Finished { .. }) + )); + } +} diff --git a/crates/executor/src/executor/mod.rs b/crates/executor/src/executor/mod.rs index fb39fcb1..c415858f 100644 --- a/crates/executor/src/executor/mod.rs +++ b/crates/executor/src/executor/mod.rs @@ -107,6 +107,12 @@ pub(crate) enum ExecutorRequest { /// This has a distinct bounded ingress so peer RPC traffic cannot delay its /// admission past queued best-effort execution. pub(crate) enum ExecutorOwedRequest { + RunPaidFetch { + span: tracing::Span, + input: Box, + progress: Option, + reply: oneshot::Sender, ExecutorError>>, + }, /// Start one already-authorized paid job. /// /// No ticket, no quote, and no admission of its own: the paid endpoint @@ -128,6 +134,10 @@ pub(crate) enum ExecutorOwedRequest { /// Trusted notifications from the bounded set of active execution producers. pub(crate) enum ExecutorCompletion { + PaidFetch { + reply: oneshot::Sender, ExecutorError>>, + result: Result, ExecutorError>, + }, #[cfg(feature = "evaluate")] EvaluateFinished(Box), FetchFinished(Box), diff --git a/crates/executor/src/fetch.rs b/crates/executor/src/fetch.rs index e1e0b59e..4ee27534 100644 --- a/crates/executor/src/fetch.rs +++ b/crates/executor/src/fetch.rs @@ -25,9 +25,7 @@ pub(crate) const MAX_FETCH_IN_MEMORY_TICKETS: usize = MAX_OUTSTANDING_QUOTES; pub(crate) const MAX_FETCH_IN_MEMORY_INPUT_BYTES: usize = 32 * 1024 * 1024; const FETCH_CAPACITY_LOCK_WAIT: Duration = Duration::from_millis(100); const MAX_FETCH_CAPACITY_METADATA_BYTES: usize = 32; -/// Persisted DAG-CBOR contains at most eight input and 4,096 output envelopes. -/// One KiB of structural/signature headroom per envelope is deliberately -/// generous beside the protocol's bounded three MiB of signed payload. +/// Payload budgets plus one KiB of encoding/signature headroom per envelope. const MAX_FETCH_TRANSCRIPT_BYTES: usize = MAX_FETCH_REQUEST_BODY_BYTES + MAX_FETCH_OUTPUT_PAYLOAD_BYTES + (MAX_FETCH_OUTPUT_EVENTS + 8) * 1024; diff --git a/crates/executor/src/fetch_policy.rs b/crates/executor/src/fetch_policy.rs index eedbef3b..6c313697 100644 --- a/crates/executor/src/fetch_policy.rs +++ b/crates/executor/src/fetch_policy.rs @@ -624,7 +624,7 @@ impl FetchRoutePolicy { } } - fn validate(&self, request: &FetchRequestView) -> Result<(), FetchAccessError> { + pub(crate) fn validate(&self, request: &FetchRequestView) -> Result<(), FetchAccessError> { if let Some(models) = &self.allowed_models { let Some(model) = &request.model else { return Err(FetchAccessError::Denied( diff --git a/crates/executor/src/fetch_provider.rs b/crates/executor/src/fetch_provider.rs index 7ca50652..9cf4bed3 100644 --- a/crates/executor/src/fetch_provider.rs +++ b/crates/executor/src/fetch_provider.rs @@ -16,11 +16,18 @@ pub type FetchProviderStream = #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct FetchProviderResponseHead { pub effective_model: Option, + pub http: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HttpResponseHead { + pub status: u16, + pub headers: Vec<(String, String)>, } impl FetchProviderResponseHead { pub const fn is_empty(&self) -> bool { - self.effective_model.is_none() + self.effective_model.is_none() && self.http.is_none() } } diff --git a/crates/executor/src/fetch_registry.rs b/crates/executor/src/fetch_registry.rs index e04238fb..0cadf8a8 100644 --- a/crates/executor/src/fetch_registry.rs +++ b/crates/executor/src/fetch_registry.rs @@ -100,6 +100,13 @@ impl FetchRouteRegistry { } } + /// Whether at least one configured route implements this trusted manifest. + pub fn has_environment(&self, environment: ContentId) -> bool { + self.routes + .values() + .any(|entry| entry.execution_environment() == environment) + } + pub fn entry(&self, route: &FetchRoute) -> Option<&FetchRouteEntry> { self.routes.get(route) } diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs index 7a5de75d..e17aa111 100644 --- a/crates/executor/src/lib.rs +++ b/crates/executor/src/lib.rs @@ -47,7 +47,8 @@ pub use fetch_projection::{ }; pub use fetch_provider::{ FetchCall, FetchProvider, FetchProviderError, FetchProviderFuture, FetchProviderResponse, - FetchProviderResponseHead, FetchProviderStream, MockFetchProvider, PreparedFetchRequest, + FetchProviderResponseHead, FetchProviderStream, HttpResponseHead, MockFetchProvider, + PreparedFetchRequest, }; pub use fetch_registry::{ DuplicateFetchRoute, FetchRouteBindingError, FetchRouteEntry, FetchRouteRegistry, diff --git a/crates/executor/src/work.rs b/crates/executor/src/work.rs index cddad3f5..9158027e 100644 --- a/crates/executor/src/work.rs +++ b/crates/executor/src/work.rs @@ -2,7 +2,7 @@ //! //! # What this is //! -//! One implementation of [`PaidEvaluateBackend`], over this crate's own +//! One implementation of [`PaidWorkBackend`], over this crate's own //! Evaluate engine. The gate that decides whether a backend may be //! called at all is [`hellas_work::work::run_accepted_work`]'s, and it is //! not here: it belongs beside the journal that records the decision, @@ -44,7 +44,7 @@ use crate::ExecutorError; use crate::executor::{ExecutorHandle, ExecutorOwedRequest}; use hellas_rpc::OutputEventEnvelope; -use hellas_work::work::{BackendFault, PaidEvaluateBackend, PaidProgress, PreparedEvaluateInput}; +use hellas_work::work::{BackendFault, PaidProgress, PaidWorkBackend, PreparedEvaluateInput}; impl ExecutorHandle { /// Runs one already-authorized paid job to its terminal. @@ -169,7 +169,36 @@ async fn drain_transcript_with_progress( } } -impl PaidEvaluateBackend for ExecutorHandle { +impl PaidWorkBackend for ExecutorHandle { + async fn fetch( + &self, + input: hellas_work::work::PreparedFetchInput, + ) -> Result, BackendFault> { + self.send_owed(|reply| ExecutorOwedRequest::RunPaidFetch { + span: hellas_rpc::request_span!(target: "hellas_request", "paid.executor.fetch"), + input: Box::new(input), + progress: None, + reply, + }) + .await + .map_err(|error| BackendFault::new(error.to_string())) + } + + async fn fetch_stream( + &self, + input: hellas_work::work::PreparedFetchInput, + progress: PaidProgress, + ) -> Result, BackendFault> { + self.send_owed(|reply| ExecutorOwedRequest::RunPaidFetch { + span: hellas_rpc::request_span!(target: "hellas_request", "paid.executor.fetch"), + input: Box::new(input), + progress: Some(progress), + reply, + }) + .await + .map_err(|error| BackendFault::new(error.to_string())) + } + async fn evaluate( &self, input: PreparedEvaluateInput, diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index 38bb51c4..3df26aec 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -26,6 +26,8 @@ evaluate = [ anyhow = { workspace = true } async-stream = { workspace = true } axum = { workspace = true } +base64.workspace = true +flate2 = "1" futures = { workspace = true } hellas-adaptors.workspace = true hellas-client = { workspace = true, features = ["iroh", "evaluate"] } @@ -35,6 +37,7 @@ hellas-private.workspace = true hellas-rpc = { workspace = true, features = ["courtesy", "fetch"] } hellas-store = { workspace = true, optional = true } hellas-wire = { workspace = true, features = ["iroh"] } +httpdate = "1" iroh.workspace = true libc = "0.2" opentelemetry = { workspace = true, optional = true } @@ -49,8 +52,10 @@ tokio.workspace = true tower = { version = "0.5", default-features = false, features = ["util"] } tracing.workspace = true tracing-opentelemetry = { workspace = true, optional = true } +zstd = "0.13" [dev-dependencies] hellas-rpc = { workspace = true, features = ["host-control"] } hellas-wire = { workspace = true, features = ["unix"] } +opentelemetry_sdk = { workspace = true, features = ["testing"] } tempfile = "3" diff --git a/crates/gateway/src/archive.rs b/crates/gateway/src/archive.rs new file mode 100644 index 00000000..0db90a11 --- /dev/null +++ b/crates/gateway/src/archive.rs @@ -0,0 +1,301 @@ +use axum::{ + body::{Bytes, HttpBody}, + http::HeaderMap, +}; +use futures::StreamExt; +use serde_json::json; +use std::{ + io, + path::{Path, PathBuf}, + time::{Instant, SystemTime, UNIX_EPOCH}, +}; +use tokio::io::AsyncWriteExt; +#[cfg(test)] +mod tests; + +#[derive(Clone)] +pub struct ArchiveOptions { + pub directory: PathBuf, + pub zdr: bool, +} + +#[derive(Clone)] +pub(crate) struct Policy { + pub options: ArchiveOptions, + pub cache_enabled: bool, + #[cfg(feature = "otel")] + failures: opentelemetry::metrics::Counter, +} + +impl Policy { + pub(crate) fn new(options: ArchiveOptions, cache_enabled: bool) -> Self { + Self { + options, + cache_enabled, + #[cfg(feature = "otel")] + failures: opentelemetry::global::meter("hellas.gateway.archive") + .u64_counter("hellas.gateway.archive.failures") + .build(), + } + } + + pub(crate) fn prepare(&self) { + if !self.options.zdr + && let Err(error) = prepare(&self.options.directory) + { + self.failed("prepare", &error); + } + } + + fn failed(&self, stage: &'static str, error: &io::Error) { + // Filesystem error strings can contain paths. Report only bounded + // classifications, never request bodies, credentials or raw errors. + warn!( + target: "hellas_archive", + archive_stage = stage, + error_kind = ?error.kind(), + error_os_code = error.raw_os_error(), + "archive failed; continuing response delivery" + ); + #[cfg(feature = "otel")] + self.failures.add( + 1, + &[ + opentelemetry::KeyValue::new("archive.stage", stage), + opentelemetry::KeyValue::new("error.type", format!("{:?}", error.kind())), + ], + ); + } +} + +pub(crate) fn zdr(headers: &HeaderMap, required: bool) -> Result { + let values: Vec<_> = headers.get_all("x-hellas-zdr").iter().collect(); + match values.as_slice() { + [] => Ok(required), + [value] if *value == "true" => Ok(true), + [value] if *value == "false" && !required => Ok(false), + _ => Err("invalid x-hellas-zdr header"), + } +} + +pub(crate) async fn record( + axum::extract::State(policy): axum::extract::State, + request: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + use axum::{body::Body, http::StatusCode, response::IntoResponse}; + let ephemeral = match zdr(request.headers(), policy.options.zdr) { + Ok(value) => value, + Err(message) => return (StatusCode::BAD_REQUEST, message).into_response(), + }; + if ephemeral && policy.cache_enabled { + return (StatusCode::BAD_REQUEST, "ZDR requires output-cache off").into_response(); + } + let (parts, body) = request.into_parts(); + let body = match axum::body::to_bytes(body, 2 * 1024 * 1024).await { + Ok(body) => body, + Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(), + }; + if ephemeral { + // Decide retention on the decoded body: a compressed store=true must + // not slip past this check onto a retaining upstream. + let decoded = match super::http_fetch::affinity::decoded_body(&parts.headers, &body) { + Ok(decoded) => decoded, + Err(message) => return (StatusCode::BAD_REQUEST, message).into_response(), + }; + if serde_json::from_slice::(&decoded) + .ok() + .and_then(|value| value.get("store").cloned()) + == Some(json!(true)) + { + return (StatusCode::BAD_REQUEST, "ZDR forbids store=true").into_response(); + } + return next + .run(axum::extract::Request::from_parts(parts, Body::from(body))) + .await; + } + let archive = match Exchange::new( + &policy.options.directory, + parts.uri.path(), + parts.method.as_str(), + &body, + &parts.headers, + ) + .await + { + Ok(archive) => Some(archive), + Err(error) => { + policy.failed("request", &error); + None + } + }; + let response = next + .run(axum::extract::Request::from_parts(parts, Body::from(body))) + .await; + match archive { + Some(archive) => archive_response(policy, archive, response).await, + None => response, + } +} + +async fn archive_response( + policy: Policy, + mut archive: Exchange, + mut response: axum::response::Response, +) -> axum::response::Response { + use axum::body::Body; + if let Some(id) = archive.directory.file_name().and_then(|id| id.to_str()) { + response + .headers_mut() + .insert("x-hellas-request-id", id.parse().unwrap()); + } + if let Some(backend) = response + .extensions() + .get::() + { + archive.metadata["backend"] = json!(backend.0); + } + if let Err(error) = archive + .head(response.status().as_u16(), response.headers()) + .await + { + policy.failed("response_head", &error); + return response; + } + let (parts, body) = response.into_parts(); + if body.is_end_stream() { + if let Err(error) = archive.finish().await { + policy.failed("finish", &error); + } + return axum::response::Response::from_parts(parts, body); + } + let mut source = body.into_data_stream(); + let mut archive = Some(archive); + let stream: futures::stream::BoxStream<'static, Result> = + Box::pin(async_stream::try_stream! { + while let Some(bytes) = source.next().await { + let bytes = bytes.map_err(|_| io::Error::other("response stream failed"))?; + if let Some(exchange) = archive.as_mut() + && let Err(error) = exchange.chunk(&bytes).await + { + policy.failed("response_body", &error); + archive = None; + } + yield bytes; + } + if let Some(mut exchange) = archive + && let Err(error) = exchange.finish().await + { + policy.failed("finish", &error); + } + }); + axum::response::Response::from_parts(parts, Body::from_stream(stream)) +} + +pub(super) fn prepare(path: &Path) -> io::Result<()> { + hellas_private::create_dir_all_durable(path)?; + hellas_private::restrict_directory(path) +} + +pub(super) struct Exchange { + directory: PathBuf, + metadata: serde_json::Value, + response: tokio::fs::File, + started: Instant, + bytes: usize, +} + +impl Exchange { + pub(super) async fn new( + root: &Path, + path: &str, + method: &str, + body: &Bytes, + headers: &HeaderMap, + ) -> io::Result { + let root = root.to_owned(); + let body = body.clone(); + let mut metadata = json!({ + "version": 1, "path": path, "method": method, "complete": false, + "started_unix_ms": SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64, + "request_bytes": body.len(), + "request_content_type": headers.get("content-type").and_then(|value| value.to_str().ok()), + "request_content_encoding": headers.get("content-encoding").and_then(|value| value.to_str().ok()), + }); + #[cfg(feature = "otel")] + { + let mut context = hellas_wire::metadata::Metadata::new(); + hellas_rpc::telemetry::inject(&tracing::Span::current(), &mut context); + metadata["traceparent"] = + json!(context.get("traceparent").and_then(|value| value.as_text())); + } + let initial = metadata.clone(); + let (directory, response) = tokio::task::spawn_blocking(move || { + let directory = root.join(format!("{:032x}", rand::random::())); + prepare(&directory)?; + hellas_private::write_atomically(&directory.join("request.bin"), ".tmp", &body)?; + write_metadata(&directory, &initial)?; + let temporary = hellas_private::private_tempfile(&directory, ".response-", ".tmp")?; + let file = temporary + .persist(directory.join("response.bin")) + .map_err(|error| error.error)?; + Ok::<_, io::Error>((directory, file)) + }) + .await + .map_err(io::Error::other)??; + metadata["response_bytes"] = json!(0); + Ok(Self { + directory, + metadata, + response: tokio::fs::File::from_std(response), + started: Instant::now(), + bytes: 0, + }) + } + + pub(super) async fn head(&mut self, status: u16, headers: &HeaderMap) -> io::Result<()> { + self.metadata["status"] = json!(status); + // Protocol metadata only. Arbitrary response headers may carry secrets. + self.metadata["content_type"] = json!( + headers + .get("content-type") + .and_then(|value| value.to_str().ok()) + ); + self.metadata["content_encoding"] = json!( + headers + .get("content-encoding") + .and_then(|value| value.to_str().ok()) + ); + self.save().await + } + + pub(super) async fn chunk(&mut self, bytes: &[u8]) -> io::Result<()> { + self.response.write_all(bytes).await?; + self.bytes += bytes.len(); + Ok(()) + } + + pub(super) async fn finish(&mut self) -> io::Result<()> { + self.response.sync_all().await?; + self.metadata["response_bytes"] = json!(self.bytes); + self.metadata["duration_ms"] = json!(self.started.elapsed().as_millis() as u64); + self.metadata["complete"] = json!(true); + self.save().await + } + + async fn save(&self) -> io::Result<()> { + let directory = self.directory.clone(); + let metadata = self.metadata.clone(); + tokio::task::spawn_blocking(move || write_metadata(&directory, &metadata)) + .await + .map_err(io::Error::other)? + } +} + +fn write_metadata(directory: &Path, metadata: &serde_json::Value) -> io::Result<()> { + hellas_private::write_atomically( + &directory.join("metadata.json"), + ".tmp", + &serde_json::to_vec(metadata)?, + ) +} diff --git a/crates/gateway/src/archive/tests.rs b/crates/gateway/src/archive/tests.rs new file mode 100644 index 00000000..46cd8990 --- /dev/null +++ b/crates/gateway/src/archive/tests.rs @@ -0,0 +1,436 @@ +use super::*; +use axum::{ + Router, + body::Body, + http::{Request, StatusCode}, + routing::post, +}; +use tower::ServiceExt; + +fn policy(directory: &Path) -> Policy { + Policy::new( + ArchiveOptions { + directory: directory.into(), + zdr: false, + }, + false, + ) +} + +async fn exchange(directory: &Path) -> Exchange { + Exchange::new( + directory, + "/v1/test", + "POST", + &Bytes::from_static(b"synthetic request"), + &HeaderMap::new(), + ) + .await + .unwrap() +} + +fn metadata(directory: &Path) -> serde_json::Value { + serde_json::from_slice(&std::fs::read(directory.join("metadata.json")).unwrap()).unwrap() +} + +fn router(directory: &Path, required: bool, cache_enabled: bool) -> Router { + Router::new() + .route( + "/v1/test", + post(|body: Bytes| async move { (StatusCode::TOO_MANY_REQUESTS, body) }), + ) + .layer(axum::middleware::from_fn_with_state( + Policy::new( + ArchiveOptions { + directory: directory.into(), + zdr: required, + }, + cache_enabled, + ), + record, + )) +} + +#[tokio::test] +async fn archives_errors_by_default_without_auth_headers_and_zdr_leaves_no_payload() { + let root = tempfile::tempdir().unwrap(); + let archive = root.path().join("archive"); + prepare(&archive).unwrap(); + let app = router(&archive, false, false); + let response = app + .clone() + .oneshot( + Request::post("/v1/test") + .header("authorization", "Bearer SECRET-HEADER") + .body(Body::from("ordinary payload")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); + let id = response.headers()["x-hellas-request-id"] + .to_str() + .unwrap() + .to_owned(); + let bytes = axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(); + assert_eq!(bytes, "ordinary payload"); + let directory = archive.join(id); + assert_eq!( + std::fs::read(directory.join("request.bin")).unwrap(), + b"ordinary payload" + ); + assert_eq!( + std::fs::read(directory.join("response.bin")).unwrap(), + b"ordinary payload" + ); + let metadata: serde_json::Value = + serde_json::from_slice(&std::fs::read(directory.join("metadata.json")).unwrap()).unwrap(); + assert_eq!(metadata["complete"], true); + assert_eq!(metadata["status"], 429); + assert!(!metadata.to_string().contains("SECRET-HEADER")); + for entry in std::fs::read_dir(&directory).unwrap() { + let file = std::fs::File::open(entry.unwrap().path()).unwrap(); + assert!(hellas_private::is_private(&file).unwrap()); + } + let response = app + .oneshot( + Request::post("/v1/test") + .header("x-hellas-zdr", "true") + .body(Body::from("ZDR-SECRET")) + .unwrap(), + ) + .await + .unwrap(); + assert!(response.headers().get("x-hellas-request-id").is_none()); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "ZDR-SECRET" + ); + assert_eq!(std::fs::read_dir(&archive).unwrap().count(), 1); +} + +#[tokio::test] +async fn cancellation_leaves_an_incomplete_archive() { + let root = tempfile::tempdir().unwrap(); + let response = router(root.path(), false, false) + .oneshot( + Request::post("/v1/test") + .body(Body::from("cancelled")) + .unwrap(), + ) + .await + .unwrap(); + let id = response.headers()["x-hellas-request-id"] + .to_str() + .unwrap() + .to_owned(); + drop(response); + let metadata: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.path().join(id).join("metadata.json")).unwrap()) + .unwrap(); + assert_eq!(metadata["complete"], false); +} + +#[tokio::test] +async fn zdr_rejects_ambiguous_flags_retain_and_enabled_replay_before_disk_writes() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("must-not-exist"); + for (required, cache, header, body) in [ + (false, false, "yes", "{}"), + (true, false, "false", "{}"), + (false, true, "true", "{}"), + (false, false, "true", "{\"store\":true}"), + ] { + let response = router(&path, required, cache) + .oneshot( + Request::post("/v1/test") + .header("x-hellas-zdr", header) + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(!path.exists()); + } + let response = router(&path, true, false) + .oneshot( + Request::post("/v1/test") + .body(Body::from("private")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "private" + ); + assert!(!path.exists()); +} + +#[tokio::test] +async fn zdr_rejects_compressed_store_true_before_forwarding() { + use std::io::Write as _; + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("must-not-exist"); + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(b"{\"store\":true}").unwrap(); + let body = encoder.finish().unwrap(); + let response = router(&path, true, false) + .oneshot( + Request::post("/v1/test") + .header("content-encoding", "gzip") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "ZDR forbids store=true" + ); + assert!(!path.exists()); +} + +#[tokio::test] +async fn unavailable_archive_preserves_errors_and_recovers_on_the_next_request() { + let root = tempfile::tempdir().unwrap(); + let archive = root.path().join("unavailable"); + std::fs::write(&archive, b"not a directory").unwrap(); + policy(&archive).prepare(); // Startup must also remain available. + let app = router(&archive, false, false); + for recovered in [false, true] { + if recovered { + std::fs::remove_file(&archive).unwrap(); + } + let response = app + .clone() + .oneshot( + Request::post("/v1/test") + .body(Body::from("unchanged error")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); + let id = response.headers().get("x-hellas-request-id").cloned(); + assert_eq!(id.is_some(), recovered); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "unchanged error" + ); + if let Some(id) = id { + assert_eq!( + metadata(&archive.join(id.to_str().unwrap()))["complete"], + true + ); + } + } +} + +#[tokio::test] +async fn response_head_archive_failure_preserves_status_headers_and_body() { + let root = tempfile::tempdir().unwrap(); + let archive = exchange(root.path()).await; + let id = archive + .directory + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned(); + let moved = root.path().join("moved"); + std::fs::rename(&archive.directory, &moved).unwrap(); + std::fs::write(&archive.directory, b"unavailable").unwrap(); + let response = axum::response::Response::builder() + .status(429) + .header("retry-after", "3") + .body(Body::from("upstream error")) + .unwrap(); + let response = archive_response(policy(root.path()), archive, response).await; + assert_eq!(response.status(), 429); + assert_eq!(response.headers()["x-hellas-request-id"], id); + assert_eq!(response.headers()["retry-after"], "3"); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "upstream error" + ); + assert_eq!(metadata(&moved)["complete"], false); +} + +#[cfg(target_os = "linux")] +#[tokio::test] +async fn disk_full_does_not_truncate_a_stream_or_replace_an_empty_response() { + let root = tempfile::tempdir().unwrap(); + for empty in [false, true] { + let mut archive = exchange(root.path()).await; + let directory = archive.directory.clone(); + // /dev/full injects ENOSPC on writes without filling the host filesystem. + archive.response = tokio::fs::OpenOptions::new() + .write(true) + .open("/dev/full") + .await + .unwrap(); + let body = if empty { + Body::empty() + } else { + Body::from_stream(futures::stream::iter( + ["first", "second", "third"] + .map(|s| Ok::<_, io::Error>(Bytes::from_static(s.as_bytes()))), + )) + }; + let response = axum::response::Response::builder() + .status(if empty { 204 } else { 200 }) + .body(body) + .unwrap(); + let response = archive_response(policy(root.path()), archive, response).await; + assert_eq!(response.status(), if empty { 204 } else { 200 }); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + if empty { "" } else { "firstsecondthird" } + ); + assert_eq!(metadata(&directory)["complete"], false); + } +} + +#[tokio::test] +async fn final_metadata_failure_keeps_all_delivered_bytes() { + let root = tempfile::tempdir().unwrap(); + let archive = exchange(root.path()).await; + let directory = archive.directory.clone(); + let moved = root.path().join("moved"); + let moved_in_stream = moved.clone(); + let body: futures::stream::BoxStream<'static, Result> = + Box::pin(async_stream::try_stream! { + yield Bytes::from_static(b"first"); + std::fs::rename(&directory, &moved_in_stream)?; + std::fs::write(&directory, b"unavailable")?; + yield Bytes::from_static(b"second"); + }); + let response = axum::response::Response::new(Body::from_stream(body)); + let response = archive_response(policy(root.path()), archive, response).await; + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "firstsecond" + ); + assert_eq!( + std::fs::read(moved.join("response.bin")).unwrap(), + b"firstsecond" + ); + assert_eq!(metadata(&moved)["complete"], false); +} + +#[tokio::test] +async fn genuine_upstream_stream_errors_still_fail() { + let root = tempfile::tempdir().unwrap(); + let archive = exchange(root.path()).await; + let directory = archive.directory.clone(); + let body = Body::from_stream(futures::stream::iter([ + Ok(Bytes::from_static(b"partial")), + Err(io::Error::other("synthetic upstream failure")), + ])); + let response = archive_response( + policy(root.path()), + archive, + axum::response::Response::new(body), + ) + .await; + let mut stream = response.into_body().into_data_stream(); + assert_eq!(stream.next().await.unwrap().unwrap(), "partial"); + assert!(stream.next().await.unwrap().is_err()); + assert_eq!(metadata(&directory)["complete"], false); +} + +#[cfg(feature = "otel")] +#[tokio::test] +async fn archive_failures_are_counted_without_changing_http_outcomes_or_counting_zdr() { + use opentelemetry::metrics::MeterProvider; + use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; + use opentelemetry_sdk::metrics::{InMemoryMetricExporter, SdkMeterProvider}; + + let exporter = InMemoryMetricExporter::default(); + let provider = SdkMeterProvider::builder() + .with_periodic_exporter(exporter.clone()) + .build(); + let root = tempfile::tempdir().unwrap(); + let unavailable = root.path().join("unavailable"); + std::fs::write(&unavailable, b"not a directory").unwrap(); + let mut policy = policy(&unavailable); + policy.failures = provider + .meter("archive-test") + .u64_counter("hellas.gateway.archive.failures") + .build(); + policy.prepare(); + let app = Router::new() + .route("/", post(|| async { "success" })) + .layer(axum::middleware::from_fn_with_state(policy, record)); + for zdr in [false, true] { + let response = app + .clone() + .oneshot( + Request::post("/") + .header("x-hellas-zdr", zdr.to_string()) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), 200); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "success" + ); + } + provider.force_flush().unwrap(); + let mut stages = Vec::new(); + for resource in exporter.get_finished_metrics().unwrap() { + for scope in resource.scope_metrics() { + for metric in scope.metrics() { + assert_eq!(metric.name(), "hellas.gateway.archive.failures"); + let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else { + panic!("failure counter must be a sum"); + }; + for point in sum.data_points() { + assert_eq!(point.value(), 1); + let attributes: Vec<_> = point.attributes().collect(); + assert_eq!(attributes.len(), 2); + stages.push( + attributes + .iter() + .find(|a| a.key.as_str() == "archive.stage") + .unwrap() + .value + .to_string(), + ); + assert!( + attributes + .iter() + .all(|a| !a.value.to_string().contains("unavailable")) + ); + } + } + } + } + stages.sort(); + assert_eq!(stages, ["prepare", "request"]); +} diff --git a/crates/gateway/src/http_fetch.rs b/crates/gateway/src/http_fetch.rs new file mode 100644 index 00000000..1089655a --- /dev/null +++ b/crates/gateway/src/http_fetch.rs @@ -0,0 +1,383 @@ +//! HTTP bytes over authenticated Fetch, without translating vendor schemas. +pub(crate) mod affinity; +mod config; +mod observation; +mod routing; + +pub use config::HttpGatewayConfig; +#[cfg(test)] +mod tests; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::{Request, State}, + http::{HeaderMap, HeaderValue, StatusCode}, + response::{IntoResponse, Response}, +}; +use futures::StreamExt; +use hellas_rpc::output::{AdaptorEvent, HttpResponseEvent, OutputEvent, StopReason}; +use std::sync::Arc; +use tracing::Instrument; + +use super::{GatewayHandle, PaidExecutionBackend, PaidFetchRequest, access}; + +#[derive(Clone)] +pub(crate) struct BackendName(pub String); + +fn attributed(mut response: Response, name: &str) -> Response { + response.extensions_mut().insert(BackendName(name.into())); + response +} + +struct HttpState { + service: String, + method: String, + paid: Arc, + routing: Arc, + metrics: observation::Metrics, +} + +#[derive(Debug, thiserror::Error)] +enum HttpOpenError { + #[error("missing authenticated HTTP response head")] + MissingHead, + #[error(transparent)] + Paid(#[from] super::PaidGatewayError), + #[error(transparent)] + Headers(#[from] hellas_rpc::http_fetch::HttpRequestError), +} + +/// Configuration for transparent HTTP over paid Fetch. Every route uses the +/// supplied paid pool; tokenizers and inference replay do not apply here. +pub struct HttpGatewayOptions { + pub config: HttpGatewayConfig, + pub paid: Arc, + pub archive: super::ArchiveOptions, + pub host: String, + pub port: Option, + pub bearer_token_file: Option, + pub allow_remote: bool, + pub wrap: Option, + pub wrap_args: Vec, +} + +/// Start without signal handlers. Failed startup drains any recovering paid jobs. +pub async fn start_http(options: HttpGatewayOptions) -> anyhow::Result { + let paid = options.paid.clone(); + let result = start(options).await; + if result.is_err() { + paid.drain().await; + } + result +} + +async fn start(options: HttpGatewayOptions) -> anyhow::Result { + let config = options.config; + config.validate()?; + let paid = options.paid; + let archive_policy = super::archive::Policy::new(options.archive.clone(), false); + archive_policy.prepare(); + let routing = Arc::new(routing::Routing::new(&config, &paid.fetch_providers())?); + let state = Arc::new(HttpState { + service: config.service, + method: config.method, + paid: paid.clone(), + routing, + metrics: observation::Metrics::new(), + }); + let bearer = Arc::new(match &options.bearer_token_file { + Some(path) => access::Bearer::load_or_create(path)?, + None => access::Bearer::generate(), + }); + let mut app = Router::new(); + let paths: std::collections::BTreeSet<_> = state + .routing + .backends + .iter() + .flat_map(|backend| backend.routes.iter().map(|route| route.path.as_str())) + .collect(); + for path in paths { + app = app.route(path, axum::routing::any(handle)); + } + let app = app + .with_state(state) + .layer(axum::middleware::from_fn_with_state( + archive_policy, + super::archive::record, + )); + #[cfg(feature = "otel")] + let app = app.layer(axum::middleware::from_fn( + hellas_rpc::telemetry::http::trace_request, + )); + let app = app.layer(access::BearerLayer::new(bearer.clone())); + let listener = super::bind_gateway( + &options.host, + options.port, + options.allow_remote && options.bearer_token_file.is_some(), + ) + .await?; + super::launch_gateway( + app, + listener, + bearer, + options.wrap.as_deref(), + &options.wrap_args, + Some(paid), + ) + .await +} + +fn error(status: StatusCode, message: &'static str) -> Response { + ( + status, + axum::Json(serde_json::json!({"error": {"message": message}})), + ) + .into_response() +} + +async fn handle(State(state): State>, request: Request) -> Response { + let mut observed = observation::Observation::new(&state.metrics, request.uri().path()); + let (parts, body) = request.into_parts(); + // Reserve space for URL, headers and JSON around the base64 body. + let body_limit = (hellas_rpc::fetch::MAX_FETCH_REQUEST_BODY_BYTES - 64 * 1024) / 4 * 3; + let body = match axum::body::to_bytes(body, body_limit).await { + Ok(body) => body, + Err(_) => { + observed.status(413); + observed.complete(); + return error( + StatusCode::PAYLOAD_TOO_LARGE, + "HTTP request exceeds Fetch limit", + ); + } + }; + let connection = parts + .extensions + .get::>() + .map(|c| &c.0); + let selected = match state.routing.select( + parts.uri.path(), + parts.method.as_str(), + &parts.headers, + &body, + connection, + ) { + Ok(selected) => selected, + Err(failure) => { + observed.status(failure.status.as_u16()); + observed.complete(); + let mut response = error(failure.status, failure.message); + if let Some(seconds) = failure.retry { + response + .headers_mut() + .insert("retry-after", seconds.to_string().parse().unwrap()); + } + if let Some(backend) = failure.backend { + let name = &state.routing.backends[backend].name; + observed.backend(name, failure.affinity.unwrap_or("session")); + return attributed(response, name); + } + return response; + } + }; + let backend = &state.routing.backends[selected.backend]; + let route = &backend.routes[selected.endpoint]; + let permit = selected.permit; + observed.backend(&backend.name, selected.affinity); + observed.model(selected.model.as_deref()); + observed.bind_response( + state + .routing + .response_binding(selected.backend, &route.path), + ); + let mut upstream = match route.request(body, &parts.headers) { + Ok(upstream) => upstream, + Err(_) => { + observed.status(400); + observed.complete(); + return error(StatusCode::BAD_REQUEST, "unsupported HTTP header encoding"); + } + }; + if let Some(query) = parts.uri.query() { + upstream + .url + .push(if upstream.url.contains('?') { '&' } else { '?' }); + upstream.url.push_str(query); + } + let payload = match serde_json::to_vec(&upstream) { + Ok(payload) if upstream.validate().is_ok() => payload, + _ => { + observed.status(400); + observed.complete(); + return error(StatusCode::BAD_REQUEST, "invalid HTTP request"); + } + }; + let result = open(&state, backend.provider, payload) + .instrument(observed.span.clone()) + .await; + let (status, headers, mut events) = match result { + Ok(value) => value, + Err(HttpOpenError::Paid(super::PaidGatewayError::Busy(_))) => { + observed.status(503); + return attributed( + error( + StatusCode::SERVICE_UNAVAILABLE, + "paid Fetch is busy; retry later", + ), + &backend.name, + ); + } + Err(_) => { + observed.status(502); + state.routing.transport_failed(selected.backend); + return attributed( + error(StatusCode::BAD_GATEWAY, "authenticated Fetch failed"), + &backend.name, + ); + } + }; + observed.status(status); + let representation_length = if parts.method == axum::http::Method::HEAD || status == 304 { + headers + .iter() + .find(|(name, _)| name == "content-length") + .and_then(|(_, value)| HeaderValue::from_str(value).ok()) + } else { + None + }; + let mut headers = response_headers(headers); + if let Some(length) = representation_length { + headers.insert("content-length", length); + } + observed.content( + headers + .get("content-type") + .and_then(|value| value.to_str().ok()), + headers + .get("content-encoding") + .and_then(|value| value.to_str().ok()), + ); + state.routing.observe(selected.backend, status, &headers); + if parts.method == axum::http::Method::HEAD || status == 204 || status == 304 { + let end = events.next().instrument(observed.span.clone()).await; + if !matches!( + end, + Some(Ok(OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None + })) + ) { + observed.status(502); + return error(StatusCode::BAD_GATEWAY, "invalid bodyless HTTP completion"); + } + observed.complete(); + let mut response = Response::new(Body::empty()); + *response.status_mut() = StatusCode::from_u16(status).unwrap(); + *response.headers_mut() = std::mem::take(&mut headers); + return attributed(response, &backend.name); + } + let stream = async_stream::try_stream! { + let _permit = permit; + let mut size = 0usize; + while let Some(event) = events.next().instrument(observed.span.clone()).await { + let event = event.map_err(|_| std::io::Error::other("Fetch stream verification failed"))?; + match event { + OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Body { base64 })) => { + let bytes = hellas_rpc::http_fetch::decode_base64(&base64) + .map_err(|_| std::io::Error::other("invalid HTTP response encoding"))?; + size = size.saturating_add(bytes.len()); + if size > upstream.max_response_bytes as usize { + Err(std::io::Error::other("HTTP response exceeds Fetch limit"))?; + } + if bytes.is_empty() { + continue; + } + observed.chunk(&bytes); + yield Bytes::from(bytes); + } + OutputEvent::Finished { stop_reason: StopReason::EndOfText, usage: None } => { + observed.complete(); + return; + } + _ => Err(std::io::Error::other("unexpected HTTP Fetch event"))?, + } + } + Err(std::io::Error::other("HTTP Fetch ended without verified completion"))?; + }; + let stream: futures::stream::BoxStream<'static, Result> = + Box::pin(stream); + let mut response = Response::new(Body::from_stream(stream)); + *response.status_mut() = StatusCode::from_u16(status).unwrap(); + *response.headers_mut() = headers; + attributed(response, &backend.name) +} + +async fn open( + state: &HttpState, + provider: iroh::EndpointId, + payload: Vec, +) -> Result<(u16, Vec<(String, String)>, super::PaidFetchStream), HttpOpenError> { + let mut stream = state.paid.fetch(PaidFetchRequest { + provider, + service: state.service.clone(), + method: state.method.clone(), + body: payload, + })?; + match stream.next().await.transpose()? { + Some(OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Head { + status, + headers, + }))) if (200..=599).contains(&status) => { + hellas_rpc::http_fetch::check_headers(&headers, false)?; + Ok((status, headers, stream)) + } + _ => Err(HttpOpenError::MissingHead), + } +} + +fn response_headers(headers: Vec<(String, String)>) -> HeaderMap { + let mut result = HeaderMap::new(); + let connection = connection_headers( + headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_str())), + ); + for (name, value) in headers { + if !hop_header(&name) + && !connection.contains(&name) + && !matches!(name.as_str(), "content-length" | "set-cookie") + && !name.starts_with("x-hellas-") + && let (Ok(name), Ok(value)) = ( + name.parse::(), + value.parse::(), + ) + { + result.append(name, value); + } + } + result +} + +fn hop_header(name: &str) -> bool { + matches!( + name, + "connection" + | "keep-alive" + | "proxy-authenticate" + | "proxy-authorization" + | "proxy-connection" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + ) +} + +fn connection_headers<'a>(headers: impl Iterator) -> Vec { + headers + .filter(|(name, _)| *name == "connection") + .flat_map(|(_, value)| value.split(',')) + .map(|name| name.trim().to_ascii_lowercase()) + .collect() +} diff --git a/crates/gateway/src/http_fetch/affinity.rs b/crates/gateway/src/http_fetch/affinity.rs new file mode 100644 index 00000000..7630424a --- /dev/null +++ b/crates/gateway/src/http_fetch/affinity.rs @@ -0,0 +1,179 @@ +//! Read routing hints without changing the bytes forwarded upstream. +use axum::http::HeaderMap; +use serde_json::Value; +use std::borrow::Cow; +use std::io::Read; + +/// Decodes a gzip/zstd request body the same way routing does, bounding the +/// decoded size. Callers that make decisions on body contents must use this +/// rather than the wire bytes, or a compressed body bypasses their check. +pub(crate) fn decoded_body<'a>( + headers: &HeaderMap, + body: &'a [u8], +) -> Result, &'static str> { + let decode = |reader: &mut dyn Read| -> Result, &'static str> { + let mut output = Vec::new(); + reader + .take(8 * 1024 * 1024 + 1) + .read_to_end(&mut output) + .map_err(|_| "invalid compressed request body")?; + if output.len() > 8 * 1024 * 1024 { + return Err("decoded request body exceeds limit"); + } + Ok(output) + }; + let encoding = headers + .get("content-encoding") + .map(|v| v.to_str().map(|s| s.trim().to_ascii_lowercase())) + .transpose() + .map_err(|_| "invalid request encoding")?; + Ok(match encoding.as_deref() { + None | Some("") | Some("identity") => Cow::Borrowed(body), + Some("gzip") => Cow::Owned(decode(&mut flate2::read::MultiGzDecoder::new(body))?), + Some("zstd") => { + let mut decoder = zstd::stream::read::Decoder::new(body) + .map_err(|_| "invalid compressed request body")?; + decoder + .window_log_max(23) + .map_err(|_| "invalid compressed request body")?; + Cow::Owned(decode(&mut decoder)?) + } + _ => return Err("unsupported request encoding for model routing"), + }) +} + +#[derive(Default)] +pub(super) struct Hints { + pub model: Option, + pub session: Option<(String, String)>, + pub previous: Option, + pub conversation: Option, +} + +impl Hints { + pub fn read(headers: &HeaderMap, body: &[u8]) -> Result { + let bytes = decoded_body(headers, body)?; + let value = if bytes.is_empty() { + Value::Null + } else { + serde_json::from_slice::(&bytes) + .map_err(|_| "model routing requires a JSON request")? + }; + fn field(value: Option<&Value>) -> Result, &'static str> { + match value { + None | Some(Value::Null) => Ok(None), + Some(Value::String(s)) if !s.is_empty() && s.len() <= 1024 => Ok(Some(s.clone())), + _ => Err("invalid model or session identifier"), + } + } + let mut session = None; + for name in [ + "x-hellas-session-id", + "x-claude-code-session-id", + "session-id", + "thread-id", + ] { + if let Some(value) = headers.get(name) { + if headers.get_all(name).iter().count() != 1 { + return Err("ambiguous session header"); + } + let value = value.to_str().map_err(|_| "invalid session header")?; + if value.is_empty() || value.len() > 1024 { + return Err("invalid session header"); + } + if session.is_none() { + session = Some((name.into(), value.into())); + } + } + } + if session.is_none() + && let Some(user) = value.pointer("/metadata/user_id").and_then(Value::as_str) + { + let id = serde_json::from_str::(user) + .ok() + .and_then(|v| { + v.get("session_id") + .and_then(Value::as_str) + .map(str::to_owned) + }) + .or_else(|| user.rsplit_once("_session_").map(|(_, id)| id.to_owned())); + if let Some(id) = id.filter(|s| !s.is_empty() && s.len() <= 1024) { + session = Some(("claude-session".into(), id)); + } + } + if session.is_none() { + session = field(value.get("prompt_cache_key"))?.map(|v| ("prompt-cache".into(), v)); + } + let conversation = value.get("conversation"); + Ok(Self { + model: field(value.get("model"))?, + session, + previous: field(value.get("previous_response_id"))?, + conversation: field( + conversation.and_then(|v| if v.is_object() { v.get("id") } else { Some(v) }), + )?, + }) + } +} + +pub(super) fn family(path: &str) -> &str { + path.strip_suffix("/compact") + .or_else(|| path.strip_suffix("/count_tokens")) + .unwrap_or(path) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::HeaderValue; + use std::io::Write; + + #[test] + fn native_client_hints_and_compressed_json_are_read_without_rewriting() { + let body = br#"{"model":"k3","prompt_cache_key":"kimi-session"}"#; + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(body).unwrap(); + let wire = encoder.finish().unwrap(); + let headers = HeaderMap::from_iter([( + "content-encoding".parse().unwrap(), + HeaderValue::from_static("gzip"), + )]); + let hints = Hints::read(&headers, &wire).unwrap(); + assert_eq!(hints.model.as_deref(), Some("k3")); + assert_eq!(hints.session.unwrap().1, "kimi-session"); + let compressed = zstd::stream::encode_all(body.as_slice(), 1).unwrap(); + let headers = HeaderMap::from_iter([( + "content-encoding".parse().unwrap(), + HeaderValue::from_static("zstd"), + )]); + assert_eq!( + Hints::read(&headers, &compressed).unwrap().model.as_deref(), + Some("k3") + ); + let mut headers = HeaderMap::new(); + headers.insert("session-id", HeaderValue::from_static("codex-session")); + assert_eq!( + Hints::read(&headers, body).unwrap().session.unwrap().1, + "codex-session" + ); + let body=serde_json::to_vec(&serde_json::json!({"model":"claude", "metadata":{"user_id":"{\"device_id\":\"not-a-session\",\"session_id\":\"claude-session\"}"}})).unwrap(); + assert_eq!( + Hints::read(&HeaderMap::new(), &body) + .unwrap() + .session + .unwrap() + .1, + "claude-session" + ); + headers.append("session-id", HeaderValue::from_static("ambiguous")); + assert!(Hints::read(&headers, &body).is_err()); + assert!(Hints::read(&HeaderMap::new(), br#"{"model":4}"#).is_err()); + assert!( + Hints::read( + &HeaderMap::new(), + br#"{"model":"k3","previous_response_id":{}}"# + ) + .is_err() + ); + } +} diff --git a/crates/gateway/src/http_fetch/config.rs b/crates/gateway/src/http_fetch/config.rs new file mode 100644 index 00000000..73b5dcf2 --- /dev/null +++ b/crates/gateway/src/http_fetch/config.rs @@ -0,0 +1,189 @@ +use super::{connection_headers, hop_header}; +use anyhow::ensure; +use axum::{body::Bytes, http::HeaderMap}; +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use hellas_rpc::http_fetch::{HttpFetchRequest, HttpTls, HttpTrustRoots}; +use serde::Deserialize; +use std::collections::BTreeMap; + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpGatewayConfig { + pub service: String, + pub method: String, + #[serde(default)] + pub routes: Vec, + #[serde(default)] + pub backends: BTreeMap, + #[serde(default = "default_concurrency")] + pub max_in_flight: usize, +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpBackend { + pub models: Vec, + pub credential: Option, + pub routes: Vec, + pub max_in_flight: Option, + /// Endpoint in the paid pool. May be omitted when there is exactly one. + pub provider: Option, +} + +fn default_concurrency() -> usize { + 4 +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpRoute { + pub path: String, + pub method: String, + pub url: String, + pub credential: Option, + #[serde(default = "public_tls")] + pub tls: HttpTls, + #[serde(default)] + pub headers: Vec<(String, String)>, +} + +pub(super) fn public_tls() -> HttpTls { + HttpTls { + roots: HttpTrustRoots::WebPki, + spki_sha256: vec![], + } +} + +impl HttpGatewayConfig { + pub(super) fn validate(&self) -> anyhow::Result<()> { + ensure!( + self.routes.is_empty() != self.backends.is_empty(), + "configure either HTTP backends or opaque routes" + ); + ensure!( + self.max_in_flight > 0 && self.max_in_flight <= 1024, + "invalid HTTP concurrency limit" + ); + if !self.routes.is_empty() { + return validate_routes(&self.routes); + } + ensure!(self.backends.len() <= 256, "too many HTTP backends"); + for (name, backend) in &self.backends { + ensure!(!name.is_empty() && name.len() <= 64, "invalid backend name"); + ensure!( + !backend.models.is_empty() + && backend.models.len() <= 256 + && backend + .models + .iter() + .all(|m| !m.is_empty() && m.len() <= 256), + "backend needs explicit model names" + ); + ensure!( + backend + .max_in_flight + .is_none_or(|n| (1..=1024).contains(&n)), + "invalid backend concurrency limit" + ); + ensure!( + backend.routes.iter().all(|r| r.credential.is_none()), + "set the credential on the backend, not its routes" + ); + validate_routes(&backend.routes)?; + } + Ok(()) + } +} + +fn validate_routes(routes: &[HttpRoute]) -> anyhow::Result<()> { + ensure!(!routes.is_empty(), "HTTP backend needs at least one route"); + let mut paths = std::collections::BTreeSet::new(); + for route in routes { + ensure!( + route.path.starts_with('/') && !route.path.contains(['?', '#', '{', '}']), + "HTTP routes must be exact paths" + ); + ensure!( + paths.insert((&route.path, &route.method)), + "duplicate HTTP route" + ); + ensure!( + route.headers.iter().all(|(name, _)| !matches!( + name.as_str(), + "authorization" | "x-api-key" | "cookie" + )), + "use a provider credential alias for authentication" + ); + route.request(Bytes::new(), &HeaderMap::new())?.validate()?; + } + Ok(()) +} + +impl HttpRoute { + pub(super) fn account(&self) -> String { + match &self.credential { + Some(alias) => format!("credential:{alias}"), + None => format!( + "origin:{}", + self.url + .parse::() + .expect("validated URL") + .origin() + .ascii_serialization() + ), + } + } + + pub(super) fn request( + &self, + body: Bytes, + incoming: &HeaderMap, + ) -> anyhow::Result { + let mut headers = self.headers.clone(); + // Connection values are ASCII by contract; a non-UTF-8 value still + // names its tokens, so parse lossily rather than silently keeping + // every header it named. + let values: Vec = incoming + .get_all("connection") + .iter() + .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned()) + .collect(); + let connection = connection_headers( + std::iter::repeat("connection").zip(values.iter().map(String::as_str)), + ); + for (name, value) in incoming { + let name = name.as_str(); + if !hop_header(name) + && !connection.iter().any(|token| token == name) + && !matches!( + name, + "host" + | "content-length" + | "authorization" + | "x-api-key" + | "api-key" + | "x-goog-api-key" + | "cookie" + | "forwarded" + ) + && !name.starts_with("x-hellas-") + && !name.starts_with("x-forwarded-") + && !self + .headers + .iter() + .any(|(configured, _)| configured == name) + { + headers.push((name.into(), value.to_str()?.into())); + } + } + Ok(HttpFetchRequest { + url: self.url.clone(), + method: self.method.clone(), + headers, + body_base64: STANDARD.encode(body), + tls: self.tls.clone(), + credential: self.credential.clone(), + max_response_bytes: hellas_rpc::http_fetch::MAX_HTTP_RESPONSE_BYTES, + }) + } +} diff --git a/crates/gateway/src/http_fetch/observation.rs b/crates/gateway/src/http_fetch/observation.rs new file mode 100644 index 00000000..333508c0 --- /dev/null +++ b/crates/gateway/src/http_fetch/observation.rs @@ -0,0 +1,476 @@ +use serde_json::Value; +use std::{io::Write, time::Instant}; + +#[derive(Clone)] +pub(super) struct Metrics { + #[cfg(feature = "otel")] + requests: opentelemetry::metrics::Counter, + #[cfg(feature = "otel")] + duration: opentelemetry::metrics::Histogram, + #[cfg(feature = "otel")] + first_byte: opentelemetry::metrics::Histogram, + #[cfg(feature = "otel")] + tokens: opentelemetry::metrics::Counter, +} + +impl Metrics { + pub(super) fn new() -> Self { + #[cfg(feature = "otel")] + let meter = opentelemetry::global::meter("hellas.gateway.http_fetch"); + Self { + #[cfg(feature = "otel")] + requests: meter.u64_counter("hellas.gateway.http.requests").build(), + #[cfg(feature = "otel")] + duration: meter + .f64_histogram("hellas.gateway.http.duration") + .with_unit("s") + .with_boundaries(vec![ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1., 2., 4., 8., 16., 32., 64., + 128., 300., + ]) + .build(), + #[cfg(feature = "otel")] + first_byte: meter + .f64_histogram("hellas.gateway.http.time_to_first_byte") + .with_unit("s") + .with_boundaries(vec![ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1., 2., 4., 8., 16., 32., 64., + ]) + .build(), + #[cfg(feature = "otel")] + tokens: meter + .u64_counter("hellas.gateway.http.tokens") + .with_unit("{token}") + .build(), + } + } +} + +pub(super) struct Observation { + pub(super) span: tracing::Span, + started: Instant, + complete: bool, + status: u16, + bytes: u64, + first_byte: Option, + usage: Usage, + compressed_usage: Option>, + #[cfg(feature = "otel")] + backend: Option, + #[cfg(feature = "otel")] + model: Option, + #[cfg(feature = "otel")] + route: String, + #[cfg(feature = "otel")] + metrics: Metrics, +} + +impl Observation { + pub(super) fn new(metrics: &Metrics, route: &str) -> Self { + #[cfg(not(feature = "otel"))] + let _ = (metrics, route); + Self { + span: hellas_rpc::request_span!(target: "hellas_request", "http.fetch", + otel.kind = "client", http.route = route, + hellas.backend = tracing::field::Empty, + gen_ai.request.model = tracing::field::Empty, + hellas.routing.affinity = tracing::field::Empty, + http.response.status_code = tracing::field::Empty, + http.response.body.size = tracing::field::Empty, + gen_ai.usage.input_tokens = tracing::field::Empty, + gen_ai.usage.output_tokens = tracing::field::Empty, + gen_ai.usage.cache_read.input_tokens = tracing::field::Empty, + gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty, + hellas.response.time_to_first_byte = tracing::field::Empty, + hellas.response.complete = tracing::field::Empty, + error.type = tracing::field::Empty, otel.status_code = tracing::field::Empty), + started: Instant::now(), + complete: false, + status: 0, + bytes: 0, + first_byte: None, + usage: Usage::default(), + compressed_usage: None, + #[cfg(feature = "otel")] + backend: None, + #[cfg(feature = "otel")] + model: None, + #[cfg(feature = "otel")] + route: route.into(), + #[cfg(feature = "otel")] + metrics: metrics.clone(), + } + } + pub(super) fn backend(&mut self, name: &str, affinity: &'static str) { + self.span.record("hellas.backend", name); + self.span.record("hellas.routing.affinity", affinity); + #[cfg(feature = "otel")] + { + self.backend = Some(name.into()); + } + } + pub(super) fn bind_response(&mut self, binding: Option) { + self.usage.binding = binding; + } + pub(super) fn model(&mut self, model: Option<&str>) { + if let Some(model) = model { + self.span.record("gen_ai.request.model", model); + #[cfg(feature = "otel")] + { + self.model = Some(model.into()); + } + } + } + pub(super) fn status(&mut self, status: u16) { + self.status = status; + self.span.record("http.response.status_code", status); + } + pub(super) fn content(&mut self, value: Option<&str>, encoding: Option<&str>) { + self.usage.sse = + value.is_some_and(|value| value.split(';').next() == Some("text/event-stream")); + match encoding.map(str::trim) { + None | Some("") => {} + Some(value) if value.eq_ignore_ascii_case("identity") => {} + Some(value) if value.eq_ignore_ascii_case("gzip") => { + self.compressed_usage = Some(flate2::write::GzDecoder::new(std::mem::take( + &mut self.usage, + ))); + } + Some(_) => self.usage.overflow = true, + } + } + pub(super) fn chunk(&mut self, bytes: &[u8]) { + if self.first_byte.is_none() { + self.first_byte = Some(self.started.elapsed().as_secs_f64()); + } + self.bytes += bytes.len() as u64; + if let Some(decoder) = self.compressed_usage.as_mut() { + if decoder.write_all(bytes).is_err() { + self.compressed_usage = None; + self.usage.overflow = true; + } + } else { + self.usage.push(bytes); + } + } + pub(super) fn complete(&mut self) { + if let Some(decoder) = self.compressed_usage.take() { + self.usage = decoder.finish().unwrap_or_else(|_| Usage { + overflow: true, + ..Default::default() + }); + } + self.usage.finish(); + self.complete = true; + } +} + +impl Drop for Observation { + fn drop(&mut self) { + let usage = self + .compressed_usage + .as_ref() + .map(|decoder| decoder.get_ref()) + .unwrap_or(&self.usage); + self.span.record("http.response.body.size", self.bytes); + self.span.record("hellas.response.complete", self.complete); + if let Some(ttfb) = self.first_byte { + self.span.record("hellas.response.time_to_first_byte", ttfb); + } + for (field, value) in [ + ("gen_ai.usage.input_tokens", usage.input), + ("gen_ai.usage.output_tokens", usage.output), + ("gen_ai.usage.cache_read.input_tokens", usage.cached), + ( + "gen_ai.usage.cache_creation.input_tokens", + usage.cache_write, + ), + ] { + if let Some(value) = value { + self.span.record(field, value); + } + } + if !self.complete || self.status >= 400 { + self.span.record("otel.status_code", "ERROR"); + self.span.record( + "error.type", + if self.status == 429 { + "rate_limited" + } else if !self.complete { + "incomplete" + } else { + "http_error" + }, + ); + } + #[cfg(feature = "otel")] + { + use opentelemetry::KeyValue; + let mut labels = vec![ + KeyValue::new("http.route", self.route.clone()), + KeyValue::new("http.response.status_code", i64::from(self.status)), + KeyValue::new("hellas.response.complete", self.complete), + ]; + if let Some(backend) = &self.backend { + labels.push(KeyValue::new("hellas.backend", backend.clone())); + } + if let Some(model) = &self.model { + labels.push(KeyValue::new("gen_ai.request.model", model.clone())); + } + self.metrics.requests.add(1, &labels); + self.metrics + .duration + .record(self.started.elapsed().as_secs_f64(), &labels); + if let Some(ttfb) = self.first_byte { + self.metrics.first_byte.record(ttfb, &labels); + } + for (kind, value) in [ + ("input", usage.input), + ("output", usage.output), + ("cache_read", usage.cached), + ("cache_write", usage.cache_write), + ] { + if let Some(value) = value { + let mut labels = labels.to_vec(); + labels.push(KeyValue::new("gen_ai.token.type", kind)); + self.metrics.tokens.add(value, &labels); + } + } + } + } +} + +/// Observe standard usage fields without reserializing or delaying the wire body. +#[derive(Default)] +struct Usage { + binding: Option, + sse: bool, + pending: Vec, + input: Option, + output: Option, + cached: Option, + cache_write: Option, + overflow: bool, + decoded_bytes: usize, +} + +impl std::io::Write for Usage { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + // Observation must not allow compressed data to consume unbounded CPU + // or memory. Stopping this sink never changes the response sent onward. + self.decoded_bytes = self.decoded_bytes.saturating_add(bytes.len()); + if self.overflow || self.decoded_bytes > 32 * 1024 * 1024 { + return Err(std::io::Error::other("usage observation limit")); + } + self.push(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl Usage { + fn push(&mut self, bytes: &[u8]) { + if self.overflow { + return; + } + self.pending.extend_from_slice(bytes); + // Some Responses endpoints omit Content-Type. Detect their SSE prelude + // after enough bytes have arrived, without changing the forwarded body. + self.sse |= self.pending.starts_with(b"event:") + || self.pending.starts_with(b"data:") + || self.pending.starts_with(b":"); + while self.sse + && let Some(end) = self.pending.iter().position(|byte| *byte == b'\n') + { + let line: Vec<_> = self.pending.drain(..=end).collect(); + if let Some(data) = line.strip_prefix(b"data:") { + self.parse(data); + } + } + if self.pending.len() > 512 * 1024 { + *self = Self { + overflow: true, + ..Default::default() + }; + } + } + fn finish(&mut self) { + let pending = std::mem::take(&mut self.pending); + self.parse(pending.strip_prefix(b"data:").unwrap_or(&pending)); + } + fn parse(&mut self, bytes: &[u8]) { + let Ok(value) = serde_json::from_slice::(bytes) else { + return; + }; + if let Some(binding) = &self.binding { + binding.observe(&value); + } + let Some(usage) = value + .get("usage") + .or_else(|| value.pointer("/message/usage")) + .or_else(|| value.pointer("/response/usage")) + else { + return; + }; + fn update(target: &mut Option, value: Option) { + if let Some(value) = value { + *target = Some(target.unwrap_or_default().max(value)); + } + } + update( + &mut self.input, + usage + .get("prompt_tokens") + .or_else(|| usage.get("input_tokens")) + .and_then(Value::as_u64), + ); + update( + &mut self.output, + usage + .get("completion_tokens") + .or_else(|| usage.get("output_tokens")) + .and_then(Value::as_u64), + ); + update( + &mut self.cached, + usage + .get("cache_read_input_tokens") + .or_else(|| usage.pointer("/prompt_tokens_details/cached_tokens")) + .or_else(|| usage.pointer("/input_tokens_details/cached_tokens")) + .and_then(Value::as_u64), + ); + update( + &mut self.cache_write, + usage + .get("cache_creation_input_tokens") + .or_else(|| usage.pointer("/input_tokens_details/cache_write_tokens")) + .or_else(|| usage.pointer("/prompt_tokens_details/cache_write_tokens")) + .and_then(Value::as_u64), + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn gzip(bytes: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(bytes).unwrap(); + encoder.finish().unwrap() + } + + #[test] + fn compressed_usage_survives_fragmentation_without_changing_wire_accounting() { + let wire = gzip(b"event: message_start\ndata: {\"message\":{\"usage\":{\"input_tokens\":2,\"cache_creation_input_tokens\":400,\"cache_read_input_tokens\":1244}}}\n\ndata: {\"usage\":{\"output_tokens\":99}}\n\n"); + for size in 1..=wire.len() { + let mut observed = Observation::new(&Metrics::new(), "/v1/messages"); + observed.content(Some("text/event-stream; charset=utf-8"), Some("gzip")); + for chunk in wire.chunks(size) { + observed.chunk(chunk); + } + observed.complete(); + assert!(observed.complete); + assert_eq!(observed.bytes, wire.len() as u64); + assert_eq!( + ( + observed.usage.input, + observed.usage.output, + observed.usage.cached, + observed.usage.cache_write + ), + (Some(2), Some(99), Some(1244), Some(400)) + ); + } + } + + #[test] + fn bad_or_unsupported_compression_leaves_usage_unknown() { + let wire = gzip(b"{\"usage\":{\"input_tokens\":10,\"output_tokens\":5}}"); + let mut corrupt = wire.clone(); + let crc = corrupt.len() - 8; + corrupt[crc] ^= 1; + for (bytes, encoding) in [ + (&wire[..wire.len() - 4], "gzip"), + (corrupt.as_slice(), "gzip"), + (wire.as_slice(), "br"), + ] { + let mut observed = Observation::new(&Metrics::new(), "/v1/responses"); + observed.content(Some("application/json"), Some(encoding)); + observed.chunk(bytes); + observed.complete(); + assert!(observed.complete); + assert_eq!(observed.bytes, bytes.len() as u64); + assert_eq!((observed.usage.input, observed.usage.output), (None, None)); + } + } + + #[test] + fn compressed_observation_stops_at_its_budget() { + let data = b": ping\n\n".repeat(32 * 1024 * 1024 / 8 + 1); + let wire = gzip(&data); + let mut observed = Observation::new(&Metrics::new(), "/v1/messages"); + observed.content(Some("text/event-stream"), Some("gzip")); + observed.chunk(&wire); + observed.complete(); + assert!(observed.usage.overflow); + assert_eq!(observed.bytes, wire.len() as u64); + assert_eq!(observed.usage.input, None); + } + + #[test] + fn usage_survives_arbitrary_sse_boundaries_and_cumulative_updates() { + let wire = b"event: message_start\r\ndata: {\"message\":{\"usage\":{\"input_tokens\":11,\"output_tokens\":1,\"cache_read_input_tokens\":7,\"cache_creation_input_tokens\":13}}}\r\n\r\ndata: {\"usage\":{\"output_tokens\":5}}\n\ndata: {\"usage\":{\"output_tokens\":5}}\n\ndata: [DONE]\n\n"; + for size in 1..=wire.len() { + let mut usage = Usage { + sse: true, + ..Default::default() + }; + for chunk in wire.chunks(size) { + usage.push(chunk); + } + usage.finish(); + assert_eq!( + (usage.input, usage.output, usage.cached), + (Some(11), Some(5), Some(7)) + ); + assert_eq!(usage.cache_write, Some(13)); + } + } + + #[test] + fn nonstreaming_usage_and_absent_usage_remain_distinct() { + let mut usage = Usage::default(); + usage.push(b"{\n\"usage\":{\"prompt_tokens\":12,\"completion_tokens\":3,\"prompt_tokens_details\":{\"cached_tokens\":8}}\n}"); + usage.finish(); + assert_eq!( + (usage.input, usage.output, usage.cached), + (Some(12), Some(3), Some(8)) + ); + let mut absent = Usage::default(); + absent.push(b"{\"error\":\"private message\"}"); + absent.finish(); + assert_eq!(absent.input, None); + } + + #[test] + fn responses_completed_usage_survives_fragmented_delivery() { + let wire = b"event: response.completed\ndata: {\"response\":{\"usage\":{\"input_tokens\":12,\"output_tokens\":3,\"input_tokens_details\":{\"cached_tokens\":8,\"cache_write_tokens\":2}}}}\n\n"; + for size in 1..=wire.len() { + // Live subscription Responses can omit Content-Type entirely. + let mut usage = Usage::default(); + for chunk in wire.chunks(size) { + usage.push(chunk); + } + usage.finish(); + assert_eq!( + (usage.input, usage.output, usage.cached), + (Some(12), Some(3), Some(8)) + ); + assert_eq!(usage.cache_write, Some(2)); + } + } +} diff --git a/crates/gateway/src/http_fetch/routing.rs b/crates/gateway/src/http_fetch/routing.rs new file mode 100644 index 00000000..d6778192 --- /dev/null +++ b/crates/gateway/src/http_fetch/routing.rs @@ -0,0 +1,512 @@ +use super::{ + affinity::{Hints, family}, + config::{HttpBackend, HttpGatewayConfig, HttpRoute}, +}; +use axum::http::{HeaderMap, StatusCode}; +use hellas_rpc::ContentId; +use std::{ + collections::HashMap, + sync::{Arc, Mutex, Weak}, + time::{Duration, Instant, SystemTime}, +}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +struct Account { + capacity: usize, + slots: Arc, + backoff: Mutex>, +} + +impl Account { + fn new(capacity: usize) -> Self { + Self { + capacity, + slots: Arc::new(Semaphore::new(capacity)), + backoff: Mutex::new(None), + } + } + + fn load(&self) -> usize { + (self.capacity - self.slots.available_permits()) * 1024 / self.capacity + } + + fn acquire(&self) -> Result { + if let Some((status, delay)) = self.cooldown() { + return Err((status, delay)); + } + self.slots + .clone() + .try_acquire_owned() + .map_err(|_| (503, Duration::ZERO)) + } + + fn cooldown(&self) -> Option<(u16, Duration)> { + let (until, status) = (*self.backoff.lock().unwrap())?; + let delay = until.saturating_duration_since(Instant::now()); + (!delay.is_zero()).then_some((status, delay)) + } + + fn observe(&self, status: u16, headers: &HeaderMap) { + if status == 429 || (status >= 500 && headers.contains_key("retry-after")) { + self.back_off(status, retry_delay(headers)); + } + } + + fn back_off(&self, status: u16, delay: Duration) { + let until = Instant::now() + delay.min(Duration::from_secs(u32::MAX as u64)); + let mut backoff = self.backoff.lock().unwrap(); + if backoff.is_none_or(|(previous, _)| until > previous) { + *backoff = Some((until, status)); + } + } +} + +const MAX_BINDINGS: usize = 16_384; +const SESSION_IDLE: Duration = Duration::from_secs(24 * 60 * 60); + +#[derive(Debug, thiserror::Error)] +pub(super) enum RoutingError { + #[error("HTTP backend {backend} must name a provider from the paid pool")] + AmbiguousProvider { backend: String }, + #[error( + "HTTP backend {backend} names provider {provider} without a paid HTTP Fetch configuration" + )] + UnfundedProvider { + backend: String, + provider: iroh::EndpointId, + }, + #[error("HTTP backend {backend} must use one account or origin")] + MixedAccounts { backend: String }, + #[error("HTTP backend {backend} shares an account with a different concurrency limit")] + InconsistentCapacity { backend: String }, +} + +pub(super) struct Backend { + pub name: String, + pub models: Vec, + pub routes: Vec, + pub provider: iroh::EndpointId, + account: Arc, +} + +struct Binding { + // None means different backends reported the same server-side identifier. + backend: Option, + touched: Instant, + connection: Option>, +} + +#[derive(Default)] +struct State { + bindings: HashMap, + next: usize, +} + +impl State { + fn resolve( + &mut self, + session: Option, + continuations: &[ContentId], + ) -> Result, Unavailable> { + let now = Instant::now(); + self.bindings.retain(|_, binding| { + now.duration_since(binding.touched) < SESSION_IDLE + && binding + .connection + .as_ref() + .is_none_or(|c| c.strong_count() > 0) + }); + let mut pinned = None; + for (key, required) in continuations + .iter() + .map(|key| (*key, true)) + .chain(session.map(|key| (key, false))) + { + let Some(binding) = self.bindings.get_mut(&key) else { + if required { + return Err(Unavailable::conflict( + "unknown or expired server-side continuation; resend full context", + )); + } + continue; + }; + binding.touched = now; + let backend = binding + .backend + .ok_or_else(|| Unavailable::conflict("ambiguous server-side continuation"))?; + if pinned.is_some_and(|p| p != backend) { + return Err(Unavailable::conflict( + "conflicting session or continuation backends", + )); + } + pinned = Some(backend); + } + Ok(pinned) + } + + fn check_capacity(&self, session: Option) -> Result<(), Unavailable> { + if session.is_some_and(|key| !self.bindings.contains_key(&key)) + && self.bindings.len() >= MAX_BINDINGS + { + return Err(Unavailable::busy(503, Duration::ZERO)); + } + Ok(()) + } + + fn bind(&mut self, key: ContentId, backend: usize, connection: Option>) { + // Never overwrite a conflicting identifier or evict a live session. + // An unrecorded or ambiguous continuation will fail closed later. + if let Some(binding) = self.bindings.get_mut(&key) { + binding.backend = binding.backend.filter(|existing| *existing == backend); + binding.touched = Instant::now(); + } else if self.bindings.len() < MAX_BINDINGS { + self.bindings.insert( + key, + Binding { + backend: Some(backend), + touched: Instant::now(), + connection, + }, + ); + } + } +} + +pub(super) struct Routing { + pub backends: Vec, + pooled: bool, + salt: [u8; 32], + state: Mutex, +} + +pub(super) struct Selected { + pub backend: usize, + pub endpoint: usize, + pub permit: OwnedSemaphorePermit, + pub affinity: &'static str, + pub model: Option, +} + +#[derive(Debug)] +pub(super) struct Unavailable { + pub status: StatusCode, + pub message: &'static str, + pub retry: Option, + pub backend: Option, + /// Affinity that produced `backend`, matching `Selected::affinity`'s + /// vocabulary, for failure attribution. None when nothing was pinned. + pub affinity: Option<&'static str>, +} + +impl Unavailable { + fn new(status: StatusCode, message: &'static str) -> Self { + Self { + status, + message, + retry: None, + backend: None, + affinity: None, + } + } + + fn conflict(message: &'static str) -> Self { + Self::new(StatusCode::CONFLICT, message) + } + + fn busy(status: u16, delay: Duration) -> Self { + Self { + retry: Some(delay.as_secs() + 1), + ..Self::new( + StatusCode::from_u16(status).unwrap(), + "selected backend is temporarily unavailable", + ) + } + } +} + +impl Routing { + pub fn new( + config: &HttpGatewayConfig, + providers: &[iroh::EndpointId], + ) -> Result { + let routes = config.routes.iter().enumerate().map(|(index, route)| { + ( + format!("route-{index}"), + HttpBackend { + models: vec![], + credential: route.credential.clone(), + routes: vec![route.clone()], + max_in_flight: None, + provider: None, + }, + ) + }); + let mut backends = Vec::new(); + let mut accounts = HashMap::new(); + for (name, mut backend) in routes.chain(config.backends.clone()) { + let provider = match backend.provider { + Some(provider) => provider, + None if providers.len() == 1 => providers[0], + None => return Err(RoutingError::AmbiguousProvider { backend: name }), + }; + if !providers.contains(&provider) { + return Err(RoutingError::UnfundedProvider { + backend: name, + provider, + }); + } + for route in &mut backend.routes { + route.credential = backend.credential.clone(); + } + let account = backend.routes[0].account(); + if backend.routes.iter().any(|r| r.account() != account) { + return Err(RoutingError::MixedAccounts { backend: name }); + } + let capacity = backend.max_in_flight.unwrap_or(config.max_in_flight); + let account = accounts + .entry((provider, account)) + .or_insert_with(|| Arc::new(Account::new(capacity))); + if account.capacity != capacity { + return Err(RoutingError::InconsistentCapacity { backend: name }); + } + backends.push(Backend { + name, + models: backend.models, + routes: backend.routes, + provider, + account: account.clone(), + }); + } + Ok(Self { + backends, + pooled: !config.backends.is_empty(), + salt: rand::random(), + state: Mutex::new(State::default()), + }) + } + + fn key(&self, domain: &str, parts: &[&str]) -> ContentId { + let mut fields: Vec<&[u8]> = vec![&self.salt]; + fields.extend(parts.iter().map(|s| s.as_bytes())); + ContentId::from_slice(hellas_rpc::hash_tuple(domain, &fields).as_bytes()).unwrap() + } + + pub fn select( + &self, + path: &str, + method: &str, + headers: &HeaderMap, + body: &[u8], + connection: Option<&crate::ConnectionId>, + ) -> Result { + let hints = if self.pooled { + Hints::read(headers, body) + .map_err(|message| Unavailable::new(StatusCode::BAD_REQUEST, message))? + } else { + Hints::default() + }; + let model = hints.model.as_deref(); + let mut candidates: Vec<_> = self + .backends + .iter() + .enumerate() + .filter(|(_, backend)| { + model.is_none_or(|model| { + backend.models.is_empty() || backend.models.iter().any(|m| m == model) + }) + }) + .filter_map(|(index, backend)| { + backend + .routes + .iter() + .position(|r| r.path == path && r.method == method) + .map(|endpoint| (index, endpoint)) + }) + .collect(); + if candidates.is_empty() { + return Err(Unavailable::new( + StatusCode::NOT_FOUND, + "no backend serves this route and model", + )); + } + let family = family(path); + let continuations: Vec<_> = [ + ("response", &hints.previous), + ("conversation", &hints.conversation), + ] + .into_iter() + .filter_map(|(kind, id)| id.as_ref().map(|id| self.key(kind, &[family, id]))) + .collect(); + if self.pooled && model.is_none() && !body.is_empty() && continuations.is_empty() { + return Err(Unavailable::new( + StatusCode::BAD_REQUEST, + "model is required", + )); + } + let connection = connection.filter(|_| self.pooled && hints.session.is_none()); + let session = hints + .session + .as_ref() + .map(|(kind, id)| self.key("session", &[family, model.unwrap_or(""), kind, id])) + .or_else(|| { + connection.map(|c| { + self.key( + "connection", + &[family, model.unwrap_or(""), &c.id.to_string()], + ) + }) + }); + // Keep resolution, admission and first binding atomic across requests. + let mut state = self.state.lock().unwrap(); + let pinned = state.resolve(session, &continuations)?; + if let Some(index) = pinned { + candidates.retain(|(backend, _)| *backend == index); + if candidates.is_empty() { + return Err(Unavailable::conflict( + "session backend cannot serve this model and route", + )); + } + } + state.check_capacity(session)?; + let count = self.backends.len(); + // Snapshot loads before sorting: other responses may release permits. + // Least occupied account first, rotating ties; aliases share its permits. + candidates.sort_by_cached_key(|(index, _)| { + ( + self.backends[*index].account.load(), + (*index + count - state.next) % count, + ) + }); + let mut unavailable: Option<(u16, Duration)> = None; + for (backend, endpoint) in candidates { + let permit = match self.backends[backend].account.acquire() { + Ok(permit) => permit, + Err(failure) => { + if unavailable.as_ref().is_none_or(|old| failure.1 < old.1) { + unavailable = Some(failure); + } + continue; + } + }; + if let Some(key) = session { + state.bind(key, backend, connection.map(|c| Arc::downgrade(&c.alive))); + } + state.next = (backend + 1) % count; + return Ok(Selected { + backend, + endpoint, + permit, + model: hints.model, + affinity: if !continuations.is_empty() { + "continuation" + } else if pinned.is_none() { + "new" + } else if connection.is_some() { + "connection" + } else { + "session" + }, + }); + } + let (status, delay) = unavailable.expect("nonempty backend candidates"); + let mut failure = Unavailable::busy(status, delay); + failure.backend = pinned; + failure.affinity = if !continuations.is_empty() { + Some("continuation") + } else if connection.is_some() { + Some("connection") + } else { + pinned.map(|_| "session") + }; + Err(failure) + } + + pub fn observe(&self, backend: usize, status: u16, headers: &HeaderMap) { + self.backends[backend].account.observe(status, headers); + } + + pub fn transport_failed(&self, backend: usize) { + if self.pooled { + self.backends[backend] + .account + .back_off(503, Duration::from_secs(1)); + } + } + + pub fn response_binding( + self: &Arc, + backend: usize, + path: &str, + ) -> Option { + (self.pooled && family(path).ends_with("/responses")).then(|| ResponseBinding { + routing: self.clone(), + backend, + family: family(path).into(), + }) + } +} + +#[derive(Clone)] +pub(super) struct ResponseBinding { + routing: Arc, + backend: usize, + family: String, +} + +impl ResponseBinding { + pub fn observe(&self, value: &serde_json::Value) { + let response = value.get("response").unwrap_or(value); + for (kind, id) in [ + ( + "response", + response + .get("id") + .and_then(serde_json::Value::as_str) + .filter(|_| { + response.get("object").and_then(serde_json::Value::as_str) + == Some("response") + }), + ), + ( + "conversation", + response.get("conversation").and_then(|v| { + v.as_str() + .or_else(|| v.get("id").and_then(serde_json::Value::as_str)) + }), + ), + ] { + let Some(id) = id.filter(|id| !id.is_empty() && id.len() <= 1024) else { + continue; + }; + let key = self.routing.key(kind, &[&self.family, id]); + self.routing + .state + .lock() + .unwrap() + .bind(key, self.backend, None); + } + } +} + +pub(super) fn retry_delay(headers: &HeaderMap) -> Duration { + let value = headers + .get("retry-after") + .and_then(|value| value.to_str().ok()); + value + .and_then(|value| { + value + .parse::() + .ok() + .map(Duration::from_secs) + .or_else(|| { + httpdate::parse_http_date(value) + .ok() + .map(|date| date.duration_since(SystemTime::now()).unwrap_or_default()) + }) + }) + .unwrap_or(Duration::from_secs(1)) + .max(Duration::from_secs(1)) +} + +#[cfg(test)] +mod tests; diff --git a/crates/gateway/src/http_fetch/routing/tests.rs b/crates/gateway/src/http_fetch/routing/tests.rs new file mode 100644 index 00000000..be63395d --- /dev/null +++ b/crates/gateway/src/http_fetch/routing/tests.rs @@ -0,0 +1,392 @@ +use super::*; +use axum::http::HeaderValue; +use serde_json::json; + +fn config() -> HttpGatewayConfig { + let backend = |credential: &str, models: &[&str]| { + json!({ + "credential":credential, "models":models, + "routes":[{"path":"/v1/chat/completions","method":"POST","url":"https://example.com/v1/chat/completions"}, + {"path":"/v1/responses","method":"POST","url":"https://example.com/v1/responses"}] + }) + }; + serde_json::from_value(json!({"service":"http","method":"request","max_in_flight":1, + "backends":{"a":backend("a", &["k3"]),"b":backend("b", &["k3"]),"c":backend("c", &["other"])}})).unwrap() +} + +fn routing(mut config: HttpGatewayConfig) -> Arc { + config.validate().unwrap(); + let default = iroh::SecretKey::from_bytes(&[1; 32]).public(); + let mut providers = vec![default]; + for backend in config.backends.values_mut() { + let provider = *backend.provider.get_or_insert(default); + if !providers.contains(&provider) { + providers.push(provider); + } + } + Arc::new(Routing::new(&config, &providers).unwrap()) +} + +#[test] +fn every_backend_must_resolve_to_a_configured_paid_provider() { + let first = iroh::SecretKey::from_bytes(&[1; 32]).public(); + let second = iroh::SecretKey::from_bytes(&[2; 32]).public(); + let mut config = config(); + assert!(matches!( + Routing::new(&config, &[]), + Err(RoutingError::AmbiguousProvider { .. }) + )); + assert!(matches!( + Routing::new(&config, &[first, second]), + Err(RoutingError::AmbiguousProvider { .. }) + )); + config.backends.get_mut("a").unwrap().provider = Some(second); + assert!(matches!( + Routing::new(&config, &[first]), + Err(RoutingError::UnfundedProvider { .. }) + )); +} + +fn select(routing: &Routing, model: &str, session: &str) -> Result { + routing.select( + "/v1/chat/completions", + "POST", + &HeaderMap::new(), + &serde_json::to_vec(&json!({"model":model,"prompt_cache_key":session})).unwrap(), + None, + ) +} + +#[test] +fn models_admission_and_cooldowns_do_not_move_existing_sessions() { + let routing = routing(config()); + let first = select(&routing, "k3", "first").unwrap(); + assert_eq!(first.backend, 0); + assert_eq!( + select(&routing, "k3", "first").err().unwrap().status, + StatusCode::SERVICE_UNAVAILABLE + ); + let second = select(&routing, "k3", "second").unwrap(); + assert_eq!(second.backend, 1); + assert_eq!( + select(&routing, "k3", "third").err().unwrap().status, + StatusCode::SERVICE_UNAVAILABLE + ); + assert_eq!(select(&routing, "other", "first").unwrap().backend, 2); + assert_eq!( + select(&routing, "unconfigured", "first") + .err() + .unwrap() + .status, + StatusCode::NOT_FOUND + ); + drop((first, second)); + let rate_limit = HeaderMap::from_iter([( + "retry-after".parse().unwrap(), + HeaderValue::from_static("60"), + )]); + routing.observe(0, 429, &rate_limit); + for binding in routing.state.lock().unwrap().bindings.values_mut() { + binding.touched = Instant::now() - SESSION_IDLE / 2; + } + let pinned = select(&routing, "k3", "first").err().unwrap(); + assert!( + routing + .state + .lock() + .unwrap() + .bindings + .values() + .any(|b| b.backend == Some(0) && b.touched.elapsed() < Duration::from_secs(1)), + "rate-limited retries still keep a session active" + ); + assert_eq!(pinned.status, StatusCode::TOO_MANY_REQUESTS); + assert!((59..=61).contains(&pinned.retry.unwrap())); + assert_eq!(select(&routing, "k3", "third").unwrap().backend, 1); + *routing.backends[0].account.backoff.lock().unwrap() = None; + assert_eq!(select(&routing, "k3", "first").unwrap().backend, 0); +} + +#[test] +fn connection_is_a_fallback_and_explicit_sessions_survive_reconnects() { + let routing = routing(config()); + let body = br#"{"model":"k3"}"#; + let connections: Vec<_> = (0..4) + .map(|id| crate::ConnectionId { + id, + alive: Arc::new(()), + }) + .collect(); + let request = |connection: usize, headers: &HeaderMap| { + routing + .select( + "/v1/chat/completions", + "POST", + headers, + body, + Some(&connections[connection]), + ) + .unwrap() + }; + let empty = HeaderMap::new(); + assert_eq!(request(1, &empty).backend, 0); + assert_eq!(request(1, &empty).backend, 0); + assert_eq!(request(2, &empty).backend, 1); + let headers = HeaderMap::from_iter([( + "session-id".parse().unwrap(), + HeaderValue::from_static("new-session"), + )]); + assert_eq!(request(2, &headers).backend, 0); + assert_eq!(request(3, &headers).backend, 0); +} + +#[test] +fn concurrent_first_requests_bind_atomically_and_release_capacity() { + let mut config = config(); + config.max_in_flight = 32; + let routing = routing(config); + let threads: Vec<_> = (0..16) + .map(|_| { + let routing = routing.clone(); + std::thread::spawn(move || select(&routing, "k3", "same-session").unwrap()) + }) + .collect(); + let requests: Vec<_> = threads.into_iter().map(|t| t.join().unwrap()).collect(); + assert!(requests.iter().all(|r| r.backend == requests[0].backend)); + let backend = requests[0].backend; + assert_eq!( + routing.backends[backend].account.slots.available_permits(), + 16 + ); + drop(requests); + assert_eq!( + routing.backends[backend].account.slots.available_permits(), + 32 + ); +} + +#[test] +fn server_state_uses_its_original_backend_and_unknown_or_conflicting_state_fails_closed() { + let routing = routing(config()); + let request = |value| { + routing.select( + "/v1/responses", + "POST", + &HeaderMap::new(), + &serde_json::to_vec(&value).unwrap(), + None, + ) + }; + let first = request(json!({"model":"k3","prompt_cache_key":"first"})).unwrap(); + let binding = routing + .response_binding(first.backend, "/v1/responses") + .unwrap(); + binding.observe( + &json!({"response":{"object":"response","id":"resp_a","conversation":{"id":"conv_a"}}}), + ); + drop(first); + assert_eq!( + request(json!({"previous_response_id":"resp_a"})) + .unwrap() + .backend, + 0 + ); + assert_eq!( + request(json!({"model":"k3","previous_response_id":"resp_a"})) + .unwrap() + .backend, + 0 + ); + assert_eq!( + request(json!({"model":"k3","conversation":"conv_a"})) + .unwrap() + .backend, + 0 + ); + assert_eq!( + request(json!({"model":"k3","previous_response_id":"unknown"})) + .err() + .unwrap() + .status, + StatusCode::CONFLICT + ); + assert_eq!( + request(json!({"model":"k3","prompt_cache_key":"second"})) + .unwrap() + .backend, + 1 + ); + assert_eq!( + request(json!({"model":"k3","prompt_cache_key":"second","previous_response_id":"resp_a"})) + .err() + .unwrap() + .status, + StatusCode::CONFLICT + ); + // Neither account may repair a collision by reporting the ID again. + for backend in [1, 0, 1] { + routing + .response_binding(backend, "/v1/responses") + .unwrap() + .observe(&json!({"object":"response","id":"resp_a"})); + assert_eq!( + request(json!({"model":"k3","previous_response_id":"resp_a"})) + .err() + .unwrap() + .status, + StatusCode::CONFLICT + ); + } +} + +#[test] +fn full_affinity_table_does_not_evict_live_sessions_and_expired_server_state_is_rejected() { + let routing = routing(config()); + assert_eq!(select(&routing, "k3", "keep").unwrap().backend, 0); + { + let mut state = routing.state.lock().unwrap(); + for i in 1..MAX_BINDINGS { + state.bindings.insert( + ContentId::hash(&i.to_le_bytes()), + Binding { + backend: Some(1), + touched: Instant::now(), + connection: None, + }, + ); + } + } + assert_eq!(select(&routing, "k3", "keep").unwrap().backend, 0); + assert_eq!( + select(&routing, "k3", "new").err().unwrap().status, + StatusCode::SERVICE_UNAVAILABLE + ); + { + let mut state = routing.state.lock().unwrap(); + for binding in state.bindings.values_mut() { + binding.touched = Instant::now() - SESSION_IDLE - Duration::from_secs(1); + } + } + assert!(select(&routing, "k3", "new").is_ok()); + routing + .response_binding(0, "/v1/responses") + .unwrap() + .observe(&json!({"object":"response","id":"expired"})); + let key = routing.key("response", &["/v1/responses", "expired"]); + routing + .state + .lock() + .unwrap() + .bindings + .get_mut(&key) + .unwrap() + .touched = Instant::now() - SESSION_IDLE - Duration::from_secs(1); + assert_eq!( + routing + .select( + "/v1/responses", + "POST", + &HeaderMap::new(), + br#"{"model":"k3","previous_response_id":"expired"}"#, + None + ) + .err() + .unwrap() + .status, + StatusCode::CONFLICT + ); +} + +#[test] +fn repeated_aliases_share_limits_but_distinct_provider_nodes_keep_them_separate() { + let mut config = config(); + config.backends.get_mut("b").unwrap().credential = Some("a".into()); + let shared = routing(config.clone()); + let _request = select(&shared, "k3", "first").unwrap(); + assert_eq!( + select(&shared, "k3", "second").err().unwrap().status, + StatusCode::SERVICE_UNAVAILABLE + ); + let node = iroh::SecretKey::generate().public(); + config.backends.get_mut("b").unwrap().provider = Some(node); + let mut same_provider = config.clone(); + same_provider.backends.get_mut("b").unwrap().provider = + Some(iroh::SecretKey::from_bytes(&[1; 32]).public()); + let same_provider = routing(same_provider); + let _first = select(&same_provider, "k3", "first").unwrap(); + assert_eq!( + select(&same_provider, "k3", "second").err().unwrap().status, + StatusCode::SERVICE_UNAVAILABLE + ); + let split = routing(config); + let _first = select(&split, "k3", "first").unwrap(); + assert_eq!(select(&split, "k3", "second").unwrap().backend, 1); + assert_eq!(split.backends[1].provider, node); +} + +#[test] +fn closed_client_connections_release_their_affinity_entries() { + let routing = routing(config()); + for id in 0..32 { + let connection = crate::ConnectionId { + id, + alive: Arc::new(()), + }; + routing + .select( + "/v1/chat/completions", + "POST", + &HeaderMap::new(), + br#"{"model":"k3"}"#, + Some(&connection), + ) + .unwrap(); + assert_eq!(routing.state.lock().unwrap().bindings.len(), 1); + } + select(&routing, "k3", "explicit-session").unwrap(); + assert_eq!(routing.state.lock().unwrap().bindings.len(), 1); +} + +#[test] +fn account_backoff_is_shared_and_never_shortened_by_another_response() { + let account = Arc::new(Account::new(1)); + let other_route = account.clone(); + let header = |seconds: &'static str| { + HeaderMap::from_iter([( + "retry-after".parse().unwrap(), + HeaderValue::from_static(seconds), + )]) + }; + assert!(account.cooldown().is_none()); + account.observe(429, &header("60")); + assert!(other_route.cooldown().unwrap().1 > Duration::from_secs(59)); + other_route.observe(429, &header("1")); + assert!(account.cooldown().unwrap().1 > Duration::from_secs(59)); + other_route.observe(503, &header("120")); + assert_eq!(account.cooldown().unwrap().0, 503); + assert!(account.cooldown().unwrap().1 > Duration::from_secs(119)); + // A short rate limit must not turn a retriable overload into a quota error. + other_route.observe(429, &header("1")); + assert_eq!(account.cooldown().unwrap().0, 503); + account.observe(200, &header("600")); + assert!(account.cooldown().unwrap().1 < Duration::from_secs(121)); + *account.backoff.lock().unwrap() = Some((Instant::now(), 503)); + assert!(account.cooldown().is_none()); + let permit = account.slots.clone().try_acquire_owned().unwrap(); + assert!(other_route.slots.clone().try_acquire_owned().is_err()); + drop(permit); + assert!(other_route.slots.clone().try_acquire_owned().is_ok()); +} + +#[test] +fn new_sessions_use_relative_account_load() { + let mut config = config(); + config.backends.get_mut("b").unwrap().max_in_flight = Some(2); + let routing = routing(config); + let a = select(&routing, "k3", "a").unwrap(); + let b = select(&routing, "k3", "b").unwrap(); + let c = select(&routing, "k3", "c").unwrap(); + assert_eq!((a.backend, b.backend, c.backend), (0, 1, 1)); + drop(a); + assert_eq!(select(&routing, "k3", "d").unwrap().backend, 0); +} diff --git a/crates/gateway/src/http_fetch/tests.rs b/crates/gateway/src/http_fetch/tests.rs new file mode 100644 index 00000000..c789dfbf --- /dev/null +++ b/crates/gateway/src/http_fetch/tests.rs @@ -0,0 +1,259 @@ +use super::config::{HttpRoute, public_tls}; +use super::routing::retry_delay; +use super::*; +use std::{ + collections::BTreeMap, + time::{Duration, SystemTime}, +}; + +#[tokio::test] +async fn http_uses_the_paid_backend_and_waits_for_its_payment_completion() { + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + struct Paid { + requests: AtomicUsize, + drained: AtomicBool, + ack: Arc, + provider: iroh::EndpointId, + busy: bool, + } + impl PaidExecutionBackend for Paid { + fn fetch_providers(&self) -> Vec { + vec![self.provider] + } + fn execute( + &self, + _: crate::PaidExecutionRequest, + ) -> Result, crate::PaidGatewayError> + { + unreachable!("HTTP must use Fetch") + } + fn fetch( + &self, + request: PaidFetchRequest, + ) -> Result { + self.requests.fetch_add(1, Ordering::Relaxed); + assert_eq!(request.provider, self.provider); + assert_eq!((&*request.service, &*request.method), ("http", "request")); + let http = hellas_rpc::http_fetch::HttpFetchRequest::decode(&request.body).unwrap(); + assert_eq!(http.url, "https://example.com/v1/responses?x=%2F&x=y"); + assert_eq!(http.body().unwrap(), b"opaque request"); + assert_eq!(http.credential.as_deref(), Some("account")); + if self.busy { + return Err(crate::PaidGatewayBusy.into()); + } + let ack = self.ack.clone(); + Ok(Box::pin(async_stream::try_stream! { + yield OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Head { + status: 200, headers: vec![("content-type".into(), "application/octet-stream".into())], + })); + yield OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Body { base64: "AP8K".into() })); + ack.notified().await; + yield OutputEvent::Finished { stop_reason: StopReason::EndOfText, usage: None }; + })) + } + fn drain(&self) -> futures::future::BoxFuture<'_, ()> { + Box::pin(async { + self.drained.store(true, Ordering::Relaxed); + }) + } + } + for busy in [false, true] { + let provider = iroh::SecretKey::from_bytes(&[12; 32]).public(); + let paid = Arc::new(Paid { + requests: AtomicUsize::new(0), + drained: AtomicBool::new(false), + ack: Arc::default(), + provider, + busy, + }); + let config: HttpGatewayConfig = serde_json::from_value(serde_json::json!({ + "service":"http", "method":"request", "routes":[{ + "path":"/v1/responses", "method":"POST", "url":"https://example.com/v1/responses", "credential":"account" + }] + })).unwrap(); + let archive = tempfile::tempdir().unwrap(); + let handle = start_http(HttpGatewayOptions { + config, + paid: paid.clone(), + archive: crate::ArchiveOptions { + directory: archive.path().into(), + zdr: true, + }, + host: "127.0.0.1".into(), + port: Some(0), + bearer_token_file: None, + allow_remote: false, + wrap: None, + wrap_args: Vec::new(), + }) + .await + .unwrap(); + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/responses?x=%2F&x=y", handle.address()); + assert_eq!( + client + .post(&url) + .body("opaque request") + .send() + .await + .unwrap() + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!(paid.requests.load(Ordering::Relaxed), 0); + let response = client + .post(&url) + .bearer_auth(handle.bearer()) + .body("opaque request") + .send() + .await + .unwrap(); + assert_eq!(paid.requests.load(Ordering::Relaxed), 1); + if busy { + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + } else { + assert_eq!(response.status(), StatusCode::OK); + let mut body = response.bytes_stream(); + assert_eq!(body.next().await.unwrap().unwrap(), &b"\x00\xff\n"[..]); + assert!( + tokio::time::timeout(Duration::from_millis(20), body.next()) + .await + .is_err() + ); + paid.ack.notify_one(); + assert!(body.next().await.is_none()); + } + handle.shutdown().await.unwrap(); + assert!(paid.drained.load(Ordering::Relaxed)); + assert_eq!(std::fs::read_dir(archive.path()).unwrap().count(), 0); + } +} + +#[test] +fn forward_only_protocol_headers_and_keep_retry_and_quota_metadata() { + let route = HttpRoute { + path: "/v1/messages".into(), + method: "POST".into(), + url: "https://api.example.com/v1/messages".into(), + credential: Some("account".into()), + tls: public_tls(), + headers: vec![("anthropic-version".into(), "2023-06-01".into())], + }; + let incoming = HeaderMap::from_iter([ + ( + "authorization".parse().unwrap(), + HeaderValue::from_static("Bearer BUYER"), + ), + ( + "x-api-key".parse().unwrap(), + HeaderValue::from_static("BUYER"), + ), + ( + "content-type".parse().unwrap(), + HeaderValue::from_static("application/json"), + ), + ( + "anthropic-version".parse().unwrap(), + HeaderValue::from_static("override"), + ), + ]); + let request = route + .request(Bytes::from_static(b"{\"private\":1}"), &incoming) + .unwrap(); + assert_eq!(request.body().unwrap(), b"{\"private\":1}"); + assert_eq!( + request.headers, + vec![ + ("anthropic-version".into(), "2023-06-01".into()), + ("content-type".into(), "application/json".into()) + ] + ); + let response = response_headers(vec![ + ("retry-after".into(), "3".into()), + ("x-ratelimit-remaining-tokens".into(), "0".into()), + ("content-length".into(), "1234".into()), + ("set-cookie".into(), "secret".into()), + ("connection".into(), "keep-alive".into()), + ("content-type".into(), "text/event-stream".into()), + ]); + assert_eq!(response.len(), 3); + assert_eq!(response["retry-after"], "3"); + assert_eq!(response["x-ratelimit-remaining-tokens"], "0"); + assert_eq!(retry_delay(&response), Duration::from_secs(3)); +} + +#[test] +fn retry_after_supports_dates_and_has_a_nonzero_fallback() { + assert_eq!(retry_delay(&HeaderMap::new()), Duration::from_secs(1)); + for value in ["invalid", "0", "-2"] { + let headers = + HeaderMap::from_iter([("retry-after".parse().unwrap(), value.parse().unwrap())]); + assert_eq!(retry_delay(&headers), Duration::from_secs(1)); + } + let date = httpdate::fmt_http_date(SystemTime::now() + Duration::from_secs(60)); + let headers = HeaderMap::from_iter([("retry-after".parse().unwrap(), date.parse().unwrap())]); + assert!((59..=60).contains(&retry_delay(&headers).as_secs())); +} + +#[test] +fn credentials_and_hop_headers_cannot_be_configured_as_static_headers() { + for name in ["authorization", "x-api-key", "cookie", "host", "connection"] { + let config = HttpGatewayConfig { + service: "http".into(), + method: "request".into(), + max_in_flight: 2, + backends: BTreeMap::new(), + routes: vec![HttpRoute { + path: "/v1/messages".into(), + method: "POST".into(), + url: "https://example.com/v1/messages".into(), + credential: Some("account".into()), + tls: public_tls(), + headers: vec![(name.into(), "secret".into())], + }], + }; + assert!(config.validate().is_err()); + } +} + +#[test] +fn extension_headers_and_duplicates_survive_but_connection_tokens_do_not() { + let route: HttpRoute = serde_json::from_value(serde_json::json!({ + "path":"/v1/chat/completions", "method":"POST", "url":"https://example.com/v1/chat/completions" + })).unwrap(); + let mut headers = HeaderMap::new(); + for (name, value) in [ + ("idempotency-key", "key"), + ("x-stainless-retry-count", "0"), + ("content-encoding", "gzip"), + ("connection", "X-Private-Hop"), + ("x-private-hop", "secret"), + ("x-beta", "one"), + ("x-beta", "two"), + ("x-hellas-zdr", "true"), + ("cookie", "secret"), + ] { + headers.append(name, value.parse().unwrap()); + } + let request = route.request(Bytes::new(), &headers).unwrap(); + assert_eq!(request.headers.len(), 5); + assert_eq!( + request + .headers + .iter() + .filter(|(n, _)| n == "x-beta") + .count(), + 2 + ); + assert!(!request.headers.iter().any(|(name, _)| { + ["connection", "x-private-hop", "cookie", "x-hellas-zdr"].contains(&name.as_str()) + })); + let response = response_headers(vec![ + ("connection".into(), "X-Private-Hop".into()), + ("x-private-hop".into(), "secret".into()), + ("location".into(), "/next".into()), + ("etag".into(), "v1".into()), + ]); + assert_eq!(response.len(), 2); + assert_eq!(response["location"], "/next"); +} diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index 29235573..a9d7e39b 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -3,9 +3,11 @@ extern crate tracing; mod access; mod anthropic; +mod archive; mod backend; mod dispatch; mod fetch_backend; +mod http_fetch; mod metrics; mod openai; mod plain; @@ -36,10 +38,12 @@ use std::time::{SystemTime, UNIX_EPOCH}; use self::state::GatewayState; +pub use archive::ArchiveOptions; pub use execution::{ CausalLmExecutionEnvironment, CliRuntime, ExecutionEvent, ExecutionRequest, ExecutionRequestOptions, ExecutionStrategy, Outcome, PreparedExecution, StopReason, }; +pub use http_fetch::{HttpGatewayConfig, HttpGatewayOptions, start_http}; const DEFAULT_HTTP_PORT: u16 = 8080; @@ -51,19 +55,46 @@ pub struct PaidExecutionRequest { pub stop_token_ids: Vec, } +/// A Fetch pinned to one funded provider by the HTTP account router. +pub struct PaidFetchRequest { + pub provider: EndpointId, + pub service: String, + pub method: String, + pub body: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum PaidGatewayError { + #[error("paid backend does not support HTTP Fetch")] + Unsupported, + #[error("provider {0} has no paid HTTP Fetch configuration")] + Provider(EndpointId), + #[error(transparent)] + Busy(#[from] PaidGatewayBusy), + #[error("paid work failed: {0}")] + Payment(#[source] Box), +} + +pub type PaidOutputStream = futures::stream::BoxStream<'static, Result>; +pub type PaidFetchStream = PaidOutputStream; + /// Paid admission capacity is exhausted or the backend is shutting down. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] +#[error("paid gateway is busy; retry later")] pub struct PaidGatewayBusy; -impl std::fmt::Display for PaidGatewayBusy { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("paid gateway is busy; retry later") +pub trait PaidExecutionBackend: Send + Sync { + /// Providers with a funded-pool configuration for the HTTP Fetch manifest. + fn fetch_providers(&self) -> Vec { + Vec::new() } -} -impl std::error::Error for PaidGatewayBusy {} + /// Authenticated prefixes, followed by completion only after payment ACK. + /// After proposal the backend owns collection and payment across disconnects. + fn fetch(&self, _request: PaidFetchRequest) -> Result { + Err(PaidGatewayError::Unsupported) + } -pub trait PaidExecutionBackend: Send + Sync { /// End-to-end budget, including queued time, advertised to HTTP consumers. fn timeout(&self) -> std::time::Duration { std::time::Duration::from_secs(300) @@ -77,7 +108,7 @@ pub trait PaidExecutionBackend: Send + Sync { fn execute( &self, request: PaidExecutionRequest, - ) -> anyhow::Result>>; + ) -> Result, PaidGatewayError>; /// Finish outstanding payment operations during graceful shutdown. fn drain(&self) -> futures::future::BoxFuture<'_, ()>; @@ -85,7 +116,27 @@ pub trait PaidExecutionBackend: Send + Sync { static NEXT_ID: AtomicU64 = AtomicU64::new(1); +#[derive(Clone)] +struct ConnectionId { + id: u64, + alive: Arc<()>, +} + +impl + axum::extract::connect_info::Connected> + for ConnectionId +{ + fn connect_info(_: axum::serve::IncomingStream<'_, tokio::net::TcpListener>) -> Self { + static NEXT_CONNECTION: AtomicU64 = AtomicU64::new(1); + Self { + id: NEXT_CONNECTION.fetch_add(1, Ordering::Relaxed), + alive: Arc::new(()), + } + } +} + pub struct GatewayOptions { + pub archive: ArchiveOptions, pub output_cache: cache::CacheOptions, pub paid_work: Option>, /// Load or create a stable bearer credential in a private file. @@ -236,6 +287,11 @@ async fn start_gateway(options: GatewayOptions) -> anyhow::Result ) .await?; let state = Arc::new(GatewayState::from_options(&options).await?); + let archive_policy = archive::Policy::new( + options.archive.clone(), + options.output_cache.policy != cache::CachePolicy::Off, + ); + archive_policy.prepare(); // Every route below reaches an executor, so every route below is // behind this run's credential. The layer goes on last, which in axum @@ -251,7 +307,11 @@ async fn start_gateway(options: GatewayOptions) -> anyhow::Result .route("/v1/messages", post(anthropic::handle)) .route("/v1/completions", post(plain::handle)) .with_state(state.clone()) - .layer(provenance_layer::ProvenanceLayer); + .layer(provenance_layer::ProvenanceLayer) + .layer(axum::middleware::from_fn_with_state( + archive_policy, + archive::record, + )); #[cfg(feature = "otel")] let app = app.layer(axum::middleware::from_fn( hellas_rpc::telemetry::http::trace_request, @@ -352,7 +412,11 @@ async fn launch_gateway( let shutdown = Arc::new(tokio::sync::Notify::new()); let server_shutdown = shutdown.clone(); let server = std::future::IntoFuture::into_future( - axum::serve(listener, app).with_graceful_shutdown(async move { + axum::serve( + listener, + app.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { server_shutdown.notified().await; }), ); @@ -411,7 +475,15 @@ async fn finish_paid_work( /// CLI lifecycle wrapper around [`start`]. pub async fn run(options: GatewayOptions) -> anyhow::Result<()> { - let mut handle = start(options).await?; + wait_for_shutdown(start(options).await?).await +} + +/// Run a paid HTTP gateway with process signal handling. +pub async fn run_http(options: HttpGatewayOptions) -> anyhow::Result<()> { + wait_for_shutdown(start_http(options).await?).await +} + +async fn wait_for_shutdown(mut handle: GatewayHandle) -> anyhow::Result<()> { tokio::select! { signal = shutdown_signal() => { signal?; @@ -533,8 +605,7 @@ mod paid_shutdown_tests { fn execute( &self, _: PaidExecutionRequest, - ) -> anyhow::Result>> - { + ) -> Result, PaidGatewayError> { unreachable!("shutdown does not submit new work") } fn drain(&self) -> futures::future::BoxFuture<'_, ()> { diff --git a/crates/gateway/src/state.rs b/crates/gateway/src/state.rs index f859b0e5..557a0da4 100644 --- a/crates/gateway/src/state.rs +++ b/crates/gateway/src/state.rs @@ -395,22 +395,13 @@ impl GatewayState { max_new_tokens: max_tokens, stop_token_ids: self.stop_token_ids.clone(), }) - .map_err(|error| { - // anyhow's allocation-preserving conversion hides the - // concrete error type from std::error::Error downcasts. - let error: Box = - match error.downcast::() { - Ok(busy) => Box::new(busy), - Err(error) => error.into_boxed_dyn_error(), - }; - hellas_client::ClientError::External(error) - })?; + .map_err(|error| hellas_client::ClientError::External(Box::new(error)))?; Ok(( None, payment .map(|result| { result.map_err(|error| { - hellas_client::ClientError::External(error.into_boxed_dyn_error()) + hellas_client::ClientError::External(Box::new(error)) }) }) .boxed(), @@ -419,7 +410,10 @@ impl GatewayState { .await .map_err(|error| match error { hellas_client::ClientError::External(error) => HttpError { - status: if error.is::() { + status: if matches!( + error.downcast_ref::(), + Some(super::PaidGatewayError::Busy(_)) + ) { StatusCode::SERVICE_UNAVAILABLE } else { StatusCode::BAD_REQUEST diff --git a/crates/gateway/src/state/tests.rs b/crates/gateway/src/state/tests.rs index eac685c0..d96439ca 100644 --- a/crates/gateway/src/state/tests.rs +++ b/crates/gateway/src/state/tests.rs @@ -51,6 +51,10 @@ fn test_environment() -> CausalLmExecutionEnvironment { /// A gateway pointed at one node, which callers then vary. fn options(provider_trust: Option) -> GatewayOptions { GatewayOptions { + archive: super::super::ArchiveOptions { + directory: "unused-test-archive".into(), + zdr: true, + }, output_cache: Default::default(), paid_work: None, bearer_token_file: None, @@ -143,6 +147,10 @@ data: {"type":"response.completed","response":{"id":"resp_1","object":"response" store: Some(Arc::new(MemoryCacheStore::default())), }; let directory = tempfile::tempdir().unwrap(); + options.archive = crate::ArchiveOptions { + directory: directory.path().join("archive"), + zdr: false, + }; std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); let socket = directory.path().join("control.sock"); let _control = LocalControlServer::bind( @@ -346,8 +354,7 @@ async fn paid_generation_records_after_payment_and_replays_without_a_backend() { fn execute( &self, request: PaidExecutionRequest, - ) -> anyhow::Result>> - { + ) -> Result, crate::PaidGatewayError> { self.calls.fetch_add(1, Ordering::SeqCst); assert_eq!(request.input_ids, vec![0]); assert_eq!(request.max_new_tokens, 1); @@ -470,9 +477,8 @@ async fn paid_capacity_is_a_retryable_error_and_uses_the_pool_deadline() { fn execute( &self, _: crate::PaidExecutionRequest, - ) -> anyhow::Result< - futures::stream::BoxStream<'static, anyhow::Result>, - > { + ) -> Result, crate::PaidGatewayError> + { Err(crate::PaidGatewayBusy.into()) } fn drain(&self) -> futures::future::BoxFuture<'_, ()> { diff --git a/crates/gateway/src/wrap.rs b/crates/gateway/src/wrap.rs index 8c68b5e0..1595765b 100644 --- a/crates/gateway/src/wrap.rs +++ b/crates/gateway/src/wrap.rs @@ -22,13 +22,9 @@ mod linux { /// is configured with `kill_on_drop(true)` so dropping the returned `Child` /// (e.g. on gateway shutdown) tears it down too. /// -/// The gateway's routes require this run's credential, so the child is -/// handed it in the two environment variables whose clients put it in -/// `Authorization: Bearer …` — `OPENAI_API_KEY` and, for Anthropic -/// clients, `ANTHROPIC_AUTH_TOKEN` rather than `ANTHROPIC_API_KEY`, which -/// would send `x-api-key` and be refused. Both overwrite whatever the -/// operator had: the base URL already points at us, so an upstream key -/// would be the wrong secret to send anyway. +/// OpenAI, Anthropic and Kimi clients receive the gateway URL and its bearer +/// credential through their environment overrides. Anthropic uses +/// `ANTHROPIC_AUTH_TOKEN` so it sends the required Authorization header. pub fn spawn( cmd: &str, args: &[String], @@ -42,6 +38,8 @@ pub fn spawn( .env("ANTHROPIC_BASE_URL", base_url) .env("OPENAI_API_KEY", credential) .env("ANTHROPIC_AUTH_TOKEN", credential) + .env("KIMI_MODEL_BASE_URL", format!("{base_url}/v1")) + .env("KIMI_MODEL_API_KEY", credential) .stdin(Stdio::inherit()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) diff --git a/crates/private/src/lib.rs b/crates/private/src/lib.rs index e0f1fb56..35ae217a 100644 --- a/crates/private/src/lib.rs +++ b/crates/private/src/lib.rs @@ -8,6 +8,9 @@ //! in a Windows DACL for the same reason root is on Unix: they can reach the //! object regardless of what it says. +mod regular; +pub use regular::{open_regular_file, read_bounded_regular_file}; + use std::fs::File; use std::io::{self, Write as _}; use std::path::Path; diff --git a/crates/private/src/regular.rs b/crates/private/src/regular.rs new file mode 100644 index 00000000..3b6e99df --- /dev/null +++ b/crates/private/src/regular.rs @@ -0,0 +1,184 @@ +use std::{fs::OpenOptions, io, path::Path}; + +/// Opens a local regular file without first opening a FIFO or device for I/O. +/// +/// Explicit content paths may be symlinks: the target descriptor, rather than +/// the symlink name, is the authority. Linux first acquires an `O_PATH` +/// descriptor, which does not open the underlying object, checks its type, and +/// then reopens that exact inode through `/proc/self/fd`. Consequently a path +/// replacement between the type check and the readable open cannot substitute +/// a FIFO or device. The returned descriptor is read-only, seekable, and +/// close-on-exec. +/// +/// Other platforms do not expose an equivalent through `std`: they preflight +/// the followed path, use nonblocking open where Unix provides it, and verify +/// the resulting descriptor. That rejects stable special files but cannot +/// close an adversarial replacement race as Linux does. +pub fn open_regular_file(path: &Path) -> io::Result { + open_regular_file_impl(path) +} + +fn not_a_regular_file(path: &Path) -> io::Error { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{} is not a regular file", path.display()), + ) +} + +#[cfg(target_os = "linux")] +fn open_regular_file_impl(path: &Path) -> io::Result { + let path_handle = open_path_handle(path)?; + reopen_regular_path_handle(path, &path_handle) +} + +/// Acquires an inode reference without invoking the target's file operations. +#[cfg(target_os = "linux")] +fn open_path_handle(path: &Path) -> io::Result { + use std::ffi::CString; + use std::os::fd::FromRawFd as _; + use std::os::unix::ffi::OsStrExt as _; + + let path = CString::new(path.as_os_str().as_bytes()) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + // OpenOptions masks custom flags with !O_ACCMODE. On musl that mask + // includes O_PATH, turning a descriptor-only open into a blocking read. + loop { + // SAFETY: path is NUL-terminated and live for the call. These flags + // do not create a file, so open needs no variadic mode argument. + let fd = unsafe { libc::open(path.as_ptr(), libc::O_PATH | libc::O_CLOEXEC) }; + if fd >= 0 { + // SAFETY: open returned a new descriptor, owned only here. + return Ok(unsafe { std::fs::File::from_raw_fd(fd) }); + } + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } + } +} + +/// Converts an `O_PATH` reference to a readable descriptor for the same inode. +#[cfg(target_os = "linux")] +fn reopen_regular_path_handle( + path: &Path, + path_handle: &std::fs::File, +) -> io::Result { + use std::os::fd::AsRawFd as _; + use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; + + let expected = path_handle.metadata()?; + if !expected.file_type().is_file() { + return Err(not_a_regular_file(path)); + } + + let descriptor_path = Path::new("/proc/self/fd").join(path_handle.as_raw_fd().to_string()); + let mut options = OpenOptions::new(); + options.read(true).custom_flags(libc::O_CLOEXEC); + let file = options.open(&descriptor_path).map_err(|source| { + // Once the held descriptor has been fstat-ed successfully, ENOENT can + // only mean procfs cannot provide the safe reopen. Do not report the + // caller's existing content as a cache miss. + let kind = if source.kind() == io::ErrorKind::NotFound { + io::ErrorKind::Unsupported + } else { + source.kind() + }; + io::Error::new( + kind, + format!( + "cannot safely reopen {} through {}: {source}", + path.display(), + descriptor_path.display() + ), + ) + })?; + let actual = file.metadata()?; + if !actual.file_type().is_file() + || (expected.dev(), expected.ino()) != (actual.dev(), actual.ino()) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{} did not reopen the inode held by its path descriptor", + path.display() + ), + )); + } + Ok(file) +} + +/// Best available fallback where `O_PATH` plus descriptor reopen is absent. +#[cfg(all(unix, not(target_os = "linux")))] +fn open_regular_file_impl(path: &Path) -> io::Result { + use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; + + let expected = std::fs::metadata(path)?; + if !expected.file_type().is_file() { + return Err(not_a_regular_file(path)); + } + + let mut options = OpenOptions::new(); + options + .read(true) + .custom_flags(libc::O_NONBLOCK | libc::O_CLOEXEC); + let file = options.open(path)?; + let actual = file.metadata()?; + if !actual.file_type().is_file() + || (expected.dev(), expected.ino()) != (actual.dev(), actual.ino()) + { + return Err(not_a_regular_file(path)); + } + Ok(file) +} + +/// Best available fallback where Unix descriptors are absent. Windows adds a +/// read-only share mode, so no writer can open the file while it is being +/// hashed -- the guarantee the Unix paths approximate with identity checks +/// (fastresume's Windows identity is deliberately weak; see there). +#[cfg(not(unix))] +fn open_regular_file_impl(path: &Path) -> io::Result { + let expected = std::fs::metadata(path)?; + if !expected.file_type().is_file() { + return Err(not_a_regular_file(path)); + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt as _; + // FILE_SHARE_READ: other readers may share the file; writers and + // deleters are refused until it is closed. + options.share_mode(0x0000_0001); + } + let file = options.open(path)?; + if !file.metadata()?.file_type().is_file() { + return Err(not_a_regular_file(path)); + } + Ok(file) +} + +/// Read a regular file with a one-byte sentinel so concurrent growth cannot +/// turn a bounded configuration read into an unbounded allocation. +pub fn read_bounded_regular_file(path: &Path, maximum: usize) -> io::Result> { + use io::Read as _; + let file = open_regular_file(path)?; + if file.metadata()?.len() > maximum as u64 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "file exceeds byte limit", + )); + } + let mut bytes = Vec::new(); + file.take((maximum as u64).saturating_add(1)) + .read_to_end(&mut bytes)?; + if bytes.len() > maximum { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "file exceeds byte limit", + )); + } + Ok(bytes) +} + +#[cfg(test)] +mod tests; diff --git a/crates/private/src/regular/tests.rs b/crates/private/src/regular/tests.rs new file mode 100644 index 00000000..9983c6dc --- /dev/null +++ b/crates/private/src/regular/tests.rs @@ -0,0 +1,101 @@ +use super::*; +#[cfg(target_os = "linux")] +use std::path::PathBuf; + +/// The Linux primitive is the contract, not merely an implementation +/// detail: a harmless path descriptor is acquired before type inspection, +/// and the readable descriptor remains bound to that inode even if the +/// name is replaced with a device. +#[cfg(target_os = "linux")] +#[test] +fn regular_open_is_safe_readable_seekable_and_close_on_exec() { + use std::io::{Read as _, Seek as _, SeekFrom}; + use std::os::fd::AsRawFd as _; + + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let target = dir.join("blob"); + let content = b"ordinary bytes"; + std::fs::write(&target, content).expect("write target"); + let link = dir.join("snapshot"); + std::os::unix::fs::symlink(&target, &link).expect("symlink"); + + let mut file = open_regular_file(&link).expect("open regular symlink"); + let mut read = Vec::new(); + file.read_to_end(&mut read).expect("read"); + assert_eq!(read, content); + file.seek(SeekFrom::Start(0)).expect("seek"); + read.clear(); + file.read_to_end(&mut read).expect("read again"); + assert_eq!(read, content); + // SAFETY: F_GETFD only observes the live descriptor owned by `file`. + let descriptor_flags = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFD) }; + assert!( + descriptor_flags >= 0, + "F_GETFD: {}", + io::Error::last_os_error() + ); + assert_ne!(descriptor_flags & libc::FD_CLOEXEC, 0); + + // Hold the ordinary inode without opening it for I/O, replace its + // public name with a device, then finish the reopen. The bytes must + // still come from the held inode. + let path_handle = open_path_handle(&link).expect("path handle"); + // O_PATH must survive libc-specific access-mode flags (notably musl). + // SAFETY: F_GETFL only observes the live descriptor owned by path_handle. + let path_flags = unsafe { libc::fcntl(path_handle.as_raw_fd(), libc::F_GETFL) }; + assert!(path_flags >= 0, "F_GETFL: {}", io::Error::last_os_error()); + assert_ne!(path_flags & libc::O_PATH, 0); + let replacement = dir.join("replacement"); + std::os::unix::fs::symlink("/dev/null", &replacement).expect("device symlink"); + std::fs::rename(&replacement, &link).expect("replace link"); + let mut held = reopen_regular_path_handle(&link, &path_handle).expect("reopen held inode"); + read.clear(); + held.read_to_end(&mut read).expect("read held inode"); + assert_eq!(read, content); + + let fifo = dir.join("fifo"); + let status = std::process::Command::new("mkfifo") + .arg(&fifo) + .status() + .expect("mkfifo"); + assert!(status.success(), "the fixture needs a fifo"); + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = sender.send(open_regular_file(&fifo)); + }); + let fifo_error = receiver + .recv_timeout(std::time::Duration::from_secs(2)) + .expect("opening the FIFO blocked before its type check") + .expect_err("a FIFO is not regular content"); + assert_eq!(fifo_error.kind(), io::ErrorKind::InvalidInput); + + if Path::new("/dev/null").exists() { + let device_error = open_regular_file(Path::new("/dev/null")) + .expect_err("a character device is not regular content"); + assert_eq!(device_error.kind(), io::ErrorKind::InvalidInput); + } + + use std::os::unix::ffi::{OsStrExt as _, OsStringExt as _}; + let mut nul_path = target.as_os_str().as_bytes().to_vec(); + nul_path.extend_from_slice(b"\0ignored-suffix"); + let nul_path = PathBuf::from(std::ffi::OsString::from_vec(nul_path)); + assert_eq!( + open_regular_file(&nul_path) + .expect_err("NUL must not truncate the path") + .kind(), + io::ErrorKind::InvalidInput, + ); +} + +#[test] +fn bounded_read_accepts_exact_limit_and_rejects_larger_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config"); + std::fs::write(&path, b"1234").unwrap(); + assert_eq!(read_bounded_regular_file(&path, 4).unwrap(), b"1234"); + assert_eq!( + read_bounded_regular_file(&path, 3).unwrap_err().kind(), + io::ErrorKind::InvalidData + ); +} diff --git a/crates/providers/Cargo.toml b/crates/providers/Cargo.toml index 9b505d66..3146b9a7 100644 --- a/crates/providers/Cargo.toml +++ b/crates/providers/Cargo.toml @@ -14,21 +14,39 @@ otel = ["hellas-rpc/otel", "dep:hellas-wire"] [dependencies] anyhow = { workspace = true } async-stream = { workspace = true } +base64.workspace = true futures.workspace = true hellas-adaptors.workspace = true hellas-executor = { workspace = true, default-features = false } +hellas-private.workspace = true hellas-rpc = { workspace = true, features = ["fetch"] } hellas-wire = { workspace = true, optional = true } reqwest.workspace = true +rustls = { version = "0.23", default-features = false, features = [ + "ring", + "std", + "tls12", +] } serde.workspace = true serde_json.workspace = true +sha2.workspace = true +thiserror.workspace = true tokio.workspace = true tracing.workspace = true +url = "2" +webpki-roots = "1" +x509-cert.workspace = true [dev-dependencies] axum.workspace = true opentelemetry.workspace = true opentelemetry_sdk = { workspace = true, features = ["testing"] } +rcgen = "0.14" +tempfile.workspace = true tokio = { workspace = true, features = ["test-util"] } +tokio-rustls = { version = "0.26", default-features = false, features = [ + "ring", + "tls12", +] } tracing-opentelemetry.workspace = true tracing-subscriber.workspace = true diff --git a/crates/providers/HTTPS.md b/crates/providers/HTTPS.md new file mode 100644 index 00000000..2465e613 --- /dev/null +++ b/crates/providers/HTTPS.md @@ -0,0 +1,168 @@ +# Caller-signed HTTPS Fetch + +The `http` environment signs raw HTTP metadata and bytes, allowing OpenAI, +Anthropic, Google, Kimi or another API without adding a vendor to the app. +It does not translate request schemas: supply the upstream's own JSON or binary +body, encoded as standard padded base64. + +```json +{ + "url": "https://api.example.com/v1/messages", + "method": "POST", + "headers": [["content-type", "application/json"]], + "body_base64": "e30=", + "tls": { + "roots": {"mode": "web_pki"}, + "spki_sha256": [] + }, + "credential": "account-1", + "max_response_bytes": 262144 +} +``` + +`body_base64` above is `{}`; replace it with your actual API request. Header +names are lowercase. `credential` may be omitted for unauthenticated requests +or caller-owned authorization headers. The response ceiling is at most 8 MiB. +All final HTTP statuses, including errors and redirects, are returned to the +client as signed responses; they are never logged with their bodies. Redirects +are not followed. A transport failure or an oversized response is not a +successful paid result, and is never retried automatically. + +`roots.mode` is either `web_pki` (bundled public trust roots) or `certificates` +with `der_base64`, a nonempty array of exact DER trust anchors. System roots +are not silently added. Hostname, certificate validity, chain signatures and +TLS handshake signatures are always checked. `spki_sha256` is an optional +array of lowercase 64-digit SHA-256 hashes of the leaf certificate's DER SPKI; +at least one must match when supplied. Pins add constraints to chain validation. +A caller-supplied trust anchor does not qualify for provider-owned credentials. + +The operator configures account aliases separately: + +```json +{ + "allowed_hosts": [], + "allow_private_addresses": false, + "credentials": { + "account-1": { + "allowed_origins": ["https://api.example.com"], + "allowed_paths": ["/v1/messages"], + "allowed_methods": ["POST"], + "header_name": "authorization", + "secret_env": "ACCOUNT_ONE_API_KEY", + "prefix": "Bearer " + }, + "account-2": { + "allowed_origins": ["https://api.example.com"], + "allowed_paths": ["/v1/messages"], + "allowed_methods": ["POST"], + "header_name": "x-api-key", + "secret_env": "ACCOUNT_TWO_API_KEY", + "prefix": "" + } + } +} +``` + +Aliases select independent accounts, including several accounts at the same +origin. Secrets are read from the provider process's environment and held in +memory. Alternatively, replace `secret_env` with `secret_file` and `secret_field` +to read a private JSON login file on each request. The loader rejects symlinks, +non-regular files, files that are not owner-only, and files over 64 KiB; atomic +token rotation is picked up without restarting the provider. +`secret_field` and `expires_field` accept a top-level key or a JSON Pointer +starting with `/`, for example `/tokens/access_token` or `/claudeAiOauth/accessToken`. +This lets the provider read native login files directly. The native login tool +still owns token refresh; do not independently rotate a CLI's refresh token. + +For short-lived tokens, a file credential can include: + +```json +{ + "secret_file": "/private/account.json", + "secret_field": "access_token", + "refresh": { + "expires_field": "expires_at", + "command": ["/absolute/path/to/account-tool", "refresh"] + } +} +``` + +`expires_at` is Unix time in seconds. The trusted operator command runs when +expiry is within 30 seconds, with no customer data or captured output. Refreshes +are serialized per account, have a 30 second timeout, and must replace the file +with a renewed token. Failures impose a cooldown. The account's login tool owns +OAuth and refresh-token persistence; Hellas does not interpret vendor logins. + +Resolved secrets are injected only for an exact authorized origin, path and method, +with public WebPKI roots. Callers cannot override that account's header or +change trust roots to impersonate its origin. Restrict paths to the inference +endpoints the account is intended to expose. Path restrictions do not cover +query strings: if an API selects models or pricing through query parameters, +a caller can still steer cost within an allowed path. + +DNS answers are resolved and checked on every request. Connections are reused +only for the same origin, DNS address set, TLS roots/pins and credential alias. +Each provider retains at most 32 client pools, with at most four idle connections +per origin and a 90 second idle expiry. Credentials are still resolved per request; +the pools contain no default authorization headers. Private, +loopback, link-local, multicast and other special-purpose addresses are denied +by default. An operator can explicitly enable private addresses only with a +nonempty exact host allowlist, for controlled private services. Environment +proxies, redirects, transparent decompression and request retries are disabled. + +In Gate, select **HTTPS (caller selects URL)** and paste this operator config +into **HTTPS accounts and egress**. `{}` enables public HTTPS without provider +accounts. In the CLI's Fetch route file, use: + +```json +{ + "service": "http", + "method": "request", + "destination": {"type": "http", "config": {"credentials": {}}} +} +``` + +The normal route file still needs its `routes` and `callers` envelope. Paid +channels use their payment policy for admission; Courtesy callers use explicit +route grants. On Gate's Run page, `http` is accepted as the execution environment. +For CLI paid requests, `paid-work prepare-fetch --execution-environment http` +prints the manifest ID to put in the work config. + +Generic HTTP responses carry no token usage, so a successful courtesy `http` +request settles zero billable units: courtesy spend quotas (`max_units` per +window) never accumulate on `http` routes, only concurrency is bounded. Paid +channels are unaffected — they charge the payment policy's fixed price per job. +If courtesy callers reach provider-funded accounts, bound their exposure with +`fetch_max_in_flight`/queue limits, or require a paid channel. + +Results contain one `Adaptor.Http.Head` event followed by base64 body events +and a signed terminal. Hop-by-hop and framing headers (`connection`, +`transfer-encoding`, `content-length` and friends) describe the provider's own +connection, so they are not part of the signed head; the body bytes are the +only length truth. `HttpFetchResponse::from_output` reconstructs the body +and checks ordering and size after transcript signature verification. + +## Provider-owned credential demonstration + +Generic HTTPS preserves provider-side API-key injection. The buyer signs only +`credential: "account-1"`; the provider resolves that alias and inserts its +secret header immediately before sending HTTPS. The key is absent from the +buyer input transcript and the normal response transcript. Credentialed +requests require public trust roots, an exact permitted origin/path/method, +and no caller override of the credential header. Redirects are not followed. +Scope aliases to upstreams you trust: an upstream receives the header and could +include it in its response; a transparent proxy cannot promise to hide a secret +that the upstream itself returns. + +Run the offline TLS demonstration from the Hellas checkout: + +```sh +nix develop --command cargo test -p hellas-providers provider_injects_secret -- --nocapture +``` + +It runs the production HTTP provider and adaptor against a local TLS server, +checks that the server receives the key, verifies the signed input and output, +and confirms neither transcript contains the key. It also checks that warmed +connection pools cannot bypass credential scope or header-override checks. The +fixture installs its own CA in the test client pool; production credentialed +requests still refuse buyer-supplied roots. diff --git a/crates/providers/src/codex_responses/tests.rs b/crates/providers/src/codex_responses/tests.rs index fae28284..4a2ec71b 100644 --- a/crates/providers/src/codex_responses/tests.rs +++ b/crates/providers/src/codex_responses/tests.rs @@ -645,6 +645,7 @@ fn model_head_and_lifecycle_claims_are_correlated_before_signing() { projector .begin(FetchProviderResponseHead { effective_model: Some("routed-model".to_string()), + http: None, }) .unwrap(); let events = vec![ @@ -663,6 +664,7 @@ fn model_head_and_lifecycle_claims_are_correlated_before_signing() { projector .begin(FetchProviderResponseHead { effective_model: Some("header-model".to_string()), + http: None, }) .unwrap(); let conflicting = diff --git a/crates/providers/src/http.rs b/crates/providers/src/http.rs new file mode 100644 index 00000000..91950bab --- /dev/null +++ b/crates/providers/src/http.rs @@ -0,0 +1,434 @@ +//! A generic HTTPS driver. The URL and TLS contract are in the signed input; +//! provider credentials are origin-scoped and held in memory. + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use futures::StreamExt as _; +use hellas_executor::{ + FetchAdaptorError, FetchAdaptorFactory, FetchAdaptorSession, FetchCall, FetchProjector, + FetchProvider, FetchProviderError, FetchProviderFuture, FetchProviderResponse, + FetchProviderResponseHead, FetchRequestView, HttpResponseHead, PreparedFetchRequest, + ProjectedFetch, +}; +use hellas_rpc::http_fetch::{HttpFetchRequest, check_headers}; +use hellas_rpc::output::{AdaptorEvent, HttpResponseEvent, OutputEvent, StopReason}; +use hellas_rpc::{ContentId, FetchEnvironment}; +use reqwest::{ + Url, + header::{HeaderName, HeaderValue}, +}; +use std::{ + collections::BTreeMap, + net::{IpAddr, SocketAddr}, + sync::Arc, + time::Duration, +}; +use tracing::Instrument; +mod clients; +mod config; +mod tls; +pub use config::{CredentialRefresh, HttpCredentialConfig, HttpProviderConfig, HttpSecret}; + +const CHUNK_BYTES: usize = 16 * 1024; +const IDLE: Duration = Duration::from_secs(90); + +/// Exact DNS names (not suffixes/wildcards). Empty allows any public host. +/// Private addresses require an explicit operator opt-in and a nonempty host +/// allowlist. Resolved addresses are checked and pinned for the connection. +#[derive(Clone, Debug, Default)] +pub struct HttpEgressPolicy { + pub allowed_hosts: Vec, + pub allow_private_addresses: bool, +} + +/// In-memory account credential. The exact HTTPS origins are operator-owned; +/// neither a request URL nor its trust anchors can widen this scope. +#[derive(Clone)] +pub struct HttpCredential { + pub allowed_origins: Vec, + pub allowed_paths: Vec, + pub allowed_methods: Vec, + pub header_name: String, + pub header_value: HttpSecret, +} + +impl std::fmt::Debug for HttpCredential { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HttpCredential") + .field("allowed_origins", &self.allowed_origins) + .field("header_name", &self.header_name) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug)] +pub struct HttpFetchProvider { + policy: HttpEgressPolicy, + credentials: Arc>, + clients: Arc, +} + +fn fault(message: &'static str) -> FetchProviderError { + FetchProviderError::failed(message) +} + +impl HttpFetchProvider { + pub fn new( + policy: HttpEgressPolicy, + credentials: BTreeMap, + ) -> Result { + if policy.allow_private_addresses && policy.allowed_hosts.is_empty() { + return Err(fault("private egress requires an explicit host allowlist")); + } + for host in &policy.allowed_hosts { + let parsed = Url::parse(&format!("https://{host}/")) + .map_err(|_| fault("invalid egress host"))?; + if parsed.host_str() != Some(host) + || parsed.port().is_some() + || !parsed.username().is_empty() + { + return Err(fault("egress hosts must be canonical names without ports")); + } + } + for (alias, credential) in &credentials { + if alias.is_empty() + || credential.allowed_origins.is_empty() + || credential.allowed_paths.is_empty() + || credential.allowed_methods.is_empty() + { + return Err(fault( + "credential requires an alias, exact origins, paths and methods", + )); + } + check_headers(&[(credential.header_name.clone(), String::new())], true) + .map_err(|_| fault("invalid credential header"))?; + if let HttpSecret::Value(value) = &credential.header_value { + check_headers(&[(credential.header_name.clone(), value.clone())], true) + .map_err(|_| fault("invalid credential header"))?; + } + for path in &credential.allowed_paths { + let parsed = Url::parse(&format!("https://scope.invalid{path}")) + .map_err(|_| fault("invalid credential path"))?; + if !path.starts_with('/') + || parsed.path() != path + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(fault("credential paths must be exact canonical URL paths")); + } + } + if credential.allowed_methods.iter().any(|method| { + !matches!( + method.as_str(), + "GET" | "HEAD" | "POST" | "PUT" | "PATCH" | "DELETE" | "OPTIONS" + ) + }) { + return Err(fault("invalid credential method")); + } + for origin in &credential.allowed_origins { + let parsed = Url::parse(origin).map_err(|_| fault("invalid credential origin"))?; + if parsed.scheme() != "https" || parsed.origin().ascii_serialization() != *origin { + return Err(fault( + "credential origins must be exact canonical HTTPS origins", + )); + } + } + } + Ok(Self { + policy, + credentials: Arc::new(credentials), + clients: Arc::new(clients::Clients::default()), + }) + } + + fn credential( + &self, + request: &HttpFetchRequest, + url: &Url, + ) -> Result, FetchProviderError> { + let Some(alias) = &request.credential else { + return Ok(None); + }; + let credential = self + .credentials + .get(alias) + .ok_or_else(|| fault("unknown credential alias"))?; + if !credential + .allowed_origins + .contains(&url.origin().ascii_serialization()) + || !credential + .allowed_paths + .iter() + .any(|path| path == url.path()) + || !credential.allowed_methods.contains(&request.method) + { + return Err(fault( + "credential is not authorized for this URL and method", + )); + } + // A caller-controlled CA could impersonate the allowed origin and + // steal the provider's credential. Account requests must keep the + // public WebPKI trust boundary; additional pins can only narrow it. + if !matches!( + request.tls.roots, + hellas_rpc::http_fetch::HttpTrustRoots::WebPki + ) { + return Err(fault("provider credentials require WebPKI roots")); + } + if request + .headers + .iter() + .any(|(name, _)| name == &credential.header_name) + { + return Err(fault("request overrides a provider credential header")); + } + Ok(Some(credential)) + } + + async fn execute( + &self, + prepared: PreparedFetchRequest, + ) -> Result { + let request = HttpFetchRequest::decode(prepared.body.as_bytes()) + .map_err(|_| fault("invalid HTTPS request"))?; + let url = request + .parsed_url() + .map_err(|_| fault("invalid HTTPS URL"))?; + let host = url.host_str().ok_or_else(|| fault("missing HTTPS host"))?; + if !self.policy.allowed_hosts.is_empty() + && !self + .policy + .allowed_hosts + .iter() + .any(|allowed| allowed == host) + { + return Err(fault("HTTPS host is outside the egress policy")); + } + let credential = self.credential(&request, &url)?; + let port = url + .port_or_known_default() + .ok_or_else(|| fault("missing HTTPS port"))?; + let addresses: Vec = + match url.host().ok_or_else(|| fault("missing HTTPS host"))? { + url::Host::Ipv4(ip) => vec![SocketAddr::new(ip.into(), port)], + url::Host::Ipv6(ip) => vec![SocketAddr::new(ip.into(), port)], + url::Host::Domain(host) => tokio::time::timeout( + Duration::from_secs(10), + tokio::net::lookup_host((host, port)), + ) + .await + .map_err(|_| fault("HTTPS DNS lookup timed out"))? + .map_err(|_| fault("HTTPS DNS lookup failed"))? + .take(65) + .collect(), + }; + if addresses.is_empty() + || addresses.len() > 64 + || addresses.iter().any(|address| { + !self.policy.allow_private_addresses && !public_address(address.ip()) + }) + { + return Err(fault("HTTPS DNS addresses are outside the egress policy")); + } + let client = self.clients.get(&request, &url, addresses)?; + let method = request + .method + .parse() + .map_err(|_| fault("invalid HTTP method"))?; + let mut outbound = client.request(method, url.clone()); + for (name, value) in &request.headers { + outbound = outbound.header( + HeaderName::from_bytes(name.as_bytes()) + .map_err(|_| fault("invalid HTTP header"))?, + HeaderValue::from_str(value).map_err(|_| fault("invalid HTTP header"))?, + ); + } + if let Some(credential) = credential { + let secret = credential.header_value.resolve().await?; + let mut value = + HeaderValue::from_str(&secret).map_err(|_| fault("invalid credential header"))?; + value.set_sensitive(true); + outbound = outbound.header(&credential.header_name, value); + } + let mut trace = + crate::responses_fetch::telemetry::Request::for_method(&url, &request.method); + let response = trace + .send(outbound.body(request.body().map_err(|_| fault("invalid HTTP body"))?)) + .instrument(trace.span.clone()) + .await + .map_err(|_| { + trace.fail("transport_error"); + fault("HTTPS transport or certificate verification failed") + })?; + trace.status(response.status().as_u16()); + // A non-2xx status is still a completed HTTP exchange. Return it, with + // its exact body, to the authenticated client; do not log it. + // The signed head carries end-to-end headers only: hop-by-hop and + // framing headers describe this connection, not the response, and a + // content-length could contradict the body the events actually carry. + let headers = response + .headers() + .iter() + .filter(|(k, _)| { + !matches!( + k.as_str(), + "connection" + | "keep-alive" + | "proxy-authenticate" + | "proxy-authorization" + | "proxy-connection" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + | "content-length" + ) + }) + .map(|(k, v)| { + Ok(( + k.to_string(), + v.to_str() + .map_err(|_| fault("unsupported HTTP response header encoding"))? + .to_string(), + )) + }) + .collect::, FetchProviderError>>()?; + check_headers(&headers, false).map_err(|_| fault("HTTP response header limit"))?; + let head = HttpResponseHead { + status: response.status().as_u16(), + headers, + }; + let limit = request.max_response_bytes as usize; + let stream = async_stream::try_stream! { + let mut upstream = response.bytes_stream(); + let mut received = 0usize; + loop { + let next = tokio::time::timeout(IDLE, upstream.next()).await + .map_err(|_| fault("HTTPS response idle timeout"))?; + let Some(chunk) = next else { break; }; + let chunk = chunk.map_err(|_| fault("HTTPS response stream failed"))?; + received = received.checked_add(chunk.len()).ok_or_else(|| fault("HTTPS response size overflow"))?; + if received > limit { Err(fault("HTTPS response exceeds signed byte limit"))?; } + // Flush each upstream delivery. Waiting for a full 16 KiB + // record otherwise holds small SSE responses until EOF. + for part in chunk.chunks(CHUNK_BYTES) { + yield part.to_vec(); + } + } + }; + Ok(FetchProviderResponse { + head: FetchProviderResponseHead { + effective_model: None, + http: Some(head), + }, + stream: Box::pin(trace.stream(stream)), + }) + } +} + +impl FetchProvider for HttpFetchProvider { + fn execution_environment(&self) -> ContentId { + FetchEnvironment::Http.manifest_id() + } + fn run(&self, request: PreparedFetchRequest) -> FetchProviderFuture<'_> { + Box::pin(self.execute(request)) + } +} + +fn public_address(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(ip) => { + let [a, b, _, _] = ip.octets(); + !(ip.is_private() + || ip.is_loopback() + || ip.is_link_local() + || ip.is_broadcast() + || ip.is_documentation() + || ip.is_unspecified() + || ip.is_multicast() + || a == 0 + || a >= 240 + || (a == 100 && (64..=127).contains(&b)) + || (a == 198 && (b == 18 || b == 19)) + || (a == 192 && b == 0)) + } + IpAddr::V6(ip) => { + // Restrict to global unicast, excluding documentation, special + // transition ranges, and mapped addresses. + let s = ip.segments(); + (s[0] & 0xe000 == 0x2000) + && s[0] != 0x2002 + && !(s[0] == 0x2001 && (s[1] < 0x0200 || s[1] == 0x0db8)) + && !(s[0] == 0x3fff && s[1] < 0x1000) + } + } +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct HttpFetchAdaptorFactory; +impl FetchAdaptorFactory for HttpFetchAdaptorFactory { + fn execution_environment(&self) -> ContentId { + FetchEnvironment::Http.manifest_id() + } + fn create(&self, call: &FetchCall) -> Result { + HttpFetchRequest::decode(call.body.as_bytes()) + .map_err(|_| FetchAdaptorError::failed("invalid HTTPS request"))?; + Ok(FetchAdaptorSession { + request_view: FetchRequestView::from_call(call), + provider_request: PreparedFetchRequest::new(call, call.body.clone()), + projector: Box::new(HttpProjector { started: false }), + }) + } +} +struct HttpProjector { + started: bool, +} +fn event(event: HttpResponseEvent) -> Result { + hellas_rpc::fetch::encode_fetch_event_payload(&OutputEvent::Adaptor(AdaptorEvent::Http(event))) + .map(ProjectedFetch::Event) + .map_err(|_| FetchAdaptorError::failed("HTTP event encoding failed")) +} +impl FetchProjector for HttpProjector { + fn begin( + &mut self, + head: FetchProviderResponseHead, + ) -> Result, FetchAdaptorError> { + if self.started || head.effective_model.is_some() { + return Err(FetchAdaptorError::failed("invalid HTTP response head")); + } + let head = head + .http + .ok_or_else(|| FetchAdaptorError::failed("missing HTTP response head"))?; + if !(100..=599).contains(&head.status) { + return Err(FetchAdaptorError::failed("invalid HTTP status")); + } + check_headers(&head.headers, false) + .map_err(|_| FetchAdaptorError::failed("invalid HTTP response headers"))?; + self.started = true; + Ok(vec![event(HttpResponseEvent::Head { + status: head.status, + headers: head.headers, + })?]) + } + fn project(&mut self, bytes: &[u8]) -> Result, FetchAdaptorError> { + if !self.started { + return Err(FetchAdaptorError::failed("HTTP body before headers")); + } + Ok(vec![event(HttpResponseEvent::Body { + base64: STANDARD.encode(bytes), + })?]) + } + fn finish(&mut self) -> Result, FetchAdaptorError> { + if !self.started { + return Err(FetchAdaptorError::failed("HTTP response missing")); + } + hellas_rpc::fetch::encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .map(|payload| vec![ProjectedFetch::Terminal(payload)]) + .map_err(|_| FetchAdaptorError::failed("HTTP terminal encoding failed")) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/providers/src/http/clients.rs b/crates/providers/src/http/clients.rs new file mode 100644 index 00000000..bec48cf3 --- /dev/null +++ b/crates/providers/src/http/clients.rs @@ -0,0 +1,102 @@ +//! Bounded connection reuse after per-request credential and DNS validation. +use super::{fault, tls}; +use hellas_executor::FetchProviderError; +use hellas_rpc::http_fetch::{HttpFetchRequest, HttpTls}; +use reqwest::{Client, Url}; +use std::{collections::VecDeque, net::SocketAddr, sync::Mutex, time::Duration}; + +const MAX_CLIENTS: usize = 32; + +#[derive(Debug, PartialEq, Eq)] +struct Key { + origin: String, + addresses: Vec, + tls: HttpTls, + credential: Option, +} + +#[derive(Debug, Default)] +pub(super) struct Clients(Mutex>); + +impl Clients { + /// `addresses` must have passed the provider's current egress policy. + /// No request bodies, authorization headers or credential values are cached. + pub(super) fn get( + &self, + request: &HttpFetchRequest, + url: &Url, + mut addresses: Vec, + ) -> Result { + // DNS answer order may rotate without changing the permitted set. + addresses.sort_unstable(); + addresses.dedup(); + let key = Key { + origin: url.origin().ascii_serialization(), + addresses, + tls: request.tls.clone(), + credential: request.credential.clone(), + }; + if let Some(client) = Self::cached(&mut self.0.lock().unwrap(), &key) { + return Ok(client); + } + // Build outside the lock. A concurrent miss may build a second client, + // but the insertion check below shares one pool before either sends. + let client = Client::builder() + .no_proxy() + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() + .redirect(reqwest::redirect::Policy::none()) + .retry(reqwest::retry::never()) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(20 * 60)) + .pool_idle_timeout(Duration::from_secs(90)) + .pool_max_idle_per_host(4) + .resolve_to_addrs(url.host_str().expect("validated HTTPS URL"), &key.addresses) + .tls_backend_preconfigured(tls::config(&key.tls).map_err(fault)?) + .build() + .map_err(|_| fault("HTTPS client initialization failed"))?; + let mut entries = self.0.lock().unwrap(); + if let Some(client) = Self::cached(&mut entries, &key) { + return Ok(client); + } + if entries.len() == MAX_CLIENTS { + entries.pop_front(); + } + entries.push_back((key, client.clone())); + Ok(client) + } + + // The fixture's CA acts as a provider-installed public root. Production + // callers still cannot combine their own trust anchors with credentials. + #[cfg(test)] + pub(super) fn trust_fixture( + &self, + request: &HttpFetchRequest, + url: &Url, + mut addresses: Vec, + ) { + let client = self.get(request, url, addresses.clone()).unwrap(); + addresses.sort_unstable(); + addresses.dedup(); + let key = Key { + origin: url.origin().ascii_serialization(), + addresses, + tls: HttpTls { + roots: hellas_rpc::http_fetch::HttpTrustRoots::WebPki, + spki_sha256: request.tls.spki_sha256.clone(), + }, + credential: request.credential.clone(), + }; + self.0.lock().unwrap().push_back((key, client)); + } + + fn cached(entries: &mut VecDeque<(Key, Client)>, key: &Key) -> Option { + let index = entries.iter().position(|(existing, _)| existing == key)?; + let entry = entries.remove(index).unwrap(); + let client = entry.1.clone(); + entries.push_back(entry); + Some(client) + } +} diff --git a/crates/providers/src/http/config.rs b/crates/providers/src/http/config.rs new file mode 100644 index 00000000..25a5a076 --- /dev/null +++ b/crates/providers/src/http/config.rs @@ -0,0 +1,390 @@ +//! Operator-owned account aliases reference environment variables or private +//! credential files; route configuration contains no secret values. +use super::{HttpCredential, HttpEgressPolicy, HttpFetchProvider}; +use hellas_executor::{FetchProviderError, FetchRouteEntry, FetchRoutePolicy}; +use serde::Deserialize; +use std::collections::BTreeMap; +use std::{ + io::Read, + path::PathBuf, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpProviderConfig { + #[serde(default)] + pub allowed_hosts: Vec, + #[serde(default)] + pub allow_private_addresses: bool, + #[serde(default)] + pub credentials: BTreeMap, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpCredentialConfig { + pub allowed_origins: Vec, + pub allowed_paths: Vec, + pub allowed_methods: Vec, + pub header_name: String, + pub secret_env: Option, + /// A private JSON credential file maintained by the account's login tool. + pub secret_file: Option, + pub secret_field: Option, + pub refresh: Option, + #[serde(default)] + pub prefix: String, +} + +/// An operator-owned login tool updates the same private JSON file. It receives +/// no customer data, and its output is never forwarded to clients or logs. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CredentialRefresh { + pub command: Vec, + pub expires_field: String, + #[serde(skip)] + lock: Arc>>, +} + +impl CredentialRefresh { + fn due(&self, object: &serde_json::Value) -> Result { + let expiry = credential_field(object, &self.expires_field) + .and_then(serde_json::Value::as_f64) + .filter(|value| value.is_finite()) + .ok_or_else(|| super::fault("credential expiry unavailable"))?; + Ok(expiry + <= SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() + + 30.) + } + async fn run(&self) -> Result<(), FetchProviderError> { + use tracing::Instrument; + let span = hellas_rpc::request_span!(target: "hellas_request", "credential.refresh", + otel.kind = "internal", otel.status_code = tracing::field::Empty, error.type = tracing::field::Empty); + let Some((program, args)) = self.command.split_first() else { + return Err(super::fault("empty credential refresh command")); + }; + let mut command = tokio::process::Command::new(program); + command + .args(args) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true); + let result = tokio::time::timeout(Duration::from_secs(30), command.status()) + .instrument(span.clone()) + .await; + if !matches!(result, Ok(Ok(status)) if status.success()) { + span.record("otel.status_code", "ERROR"); + span.record("error.type", "refresh_failed"); + return Err(super::fault("credential refresh failed")); + } + Ok(()) + } +} + +#[derive(Clone)] +pub enum HttpSecret { + Value(String), + JsonFile { + path: PathBuf, + field: String, + prefix: String, + refresh: Option, + }, +} + +impl From for HttpSecret { + fn from(value: String) -> Self { + Self::Value(value) + } +} +impl From<&str> for HttpSecret { + fn from(value: &str) -> Self { + Self::Value(value.into()) + } +} + +impl HttpSecret { + pub(super) async fn resolve(&self) -> Result { + match self { + Self::Value(value) => Ok(value.clone()), + Self::JsonFile { + path, + field, + prefix, + refresh, + } => { + let mut guard = match refresh { + Some(refresh) => Some(refresh.lock.lock().await), + None => None, + }; + let mut object = read_credential(path.clone()).await?; + if let Some(refresh) = refresh + && refresh.due(&object)? + { + let retry_at = guard.as_mut().expect("refresh holds its account lock"); + if retry_at.is_some_and(|deadline| deadline > std::time::Instant::now()) { + return Err(super::fault("credential refresh is cooling down")); + } + **retry_at = Some(std::time::Instant::now() + Duration::from_secs(30)); + refresh.run().await?; + object = read_credential(path.clone()).await?; + if refresh.due(&object)? { + return Err(super::fault("credential refresh did not renew the token")); + } + **retry_at = None; + } + let secret = credential_field(&object, field) + .and_then(serde_json::Value::as_str) + .filter(|secret| !secret.is_empty()) + .ok_or_else(|| super::fault("credential field unavailable"))?; + Ok(format!("{prefix}{secret}")) + } + } + } +} + +fn credential_field<'a>( + object: &'a serde_json::Value, + field: &str, +) -> Option<&'a serde_json::Value> { + if field.starts_with('/') { + object.pointer(field) + } else { + object.get(field) + } +} + +async fn read_credential(path: PathBuf) -> Result { + tokio::task::spawn_blocking(move || { + let file = hellas_private::open_nofollow(&path) + .map_err(|_| super::fault("credential file unavailable"))?; + if !hellas_private::is_private(&file).unwrap_or(false) { + return Err(super::fault("credential file must be owner-only")); + } + let mut bytes = Vec::new(); + file.take(65537) + .read_to_end(&mut bytes) + .map_err(|_| super::fault("credential file read failed"))?; + if bytes.len() > 65536 { + return Err(super::fault("credential file exceeds limit")); + } + let object: serde_json::Value = serde_json::from_slice(&bytes) + .map_err(|_| super::fault("credential file is not JSON"))?; + Ok(object) + }) + .await + .map_err(|_| super::fault("credential loader failed"))? +} + +impl HttpProviderConfig { + pub fn into_entry( + self, + capabilities: FetchRoutePolicy, + ) -> Result { + let mut credentials = BTreeMap::new(); + for (alias, config) in self.credentials { + if config.refresh.as_ref().is_some_and(|refresh| { + refresh.command.is_empty() || refresh.expires_field.is_empty() + }) { + return Err(super::fault( + "credential refresh requires a command and expiry field", + )); + } + let header_value = match (config.secret_env, config.secret_file, config.secret_field) { + (Some(name), None, None) if config.refresh.is_none() => { + let secret = std::env::var(name).map_err(|_| { + super::fault("account secret environment variable is unavailable") + })?; + if secret.is_empty() { + return Err(super::fault("account secret is empty")); + } + HttpSecret::Value(format!("{}{secret}", config.prefix)) + } + (None, Some(path), Some(field)) if !field.is_empty() => HttpSecret::JsonFile { + path, + field, + prefix: config.prefix, + refresh: config.refresh, + }, + _ => { + return Err(super::fault( + "configure either secret_env or secret_file with secret_field", + )); + } + }; + credentials.insert( + alias, + HttpCredential { + allowed_origins: config.allowed_origins, + allowed_paths: config.allowed_paths, + allowed_methods: config.allowed_methods, + header_name: config.header_name, + header_value, + }, + ); + } + let provider = HttpFetchProvider::new( + HttpEgressPolicy { + allowed_hosts: self.allowed_hosts, + allow_private_addresses: self.allow_private_addresses, + }, + credentials, + )?; + FetchRouteEntry::new( + std::sync::Arc::new(provider), + std::sync::Arc::new(super::HttpFetchAdaptorFactory), + capabilities, + ) + .map_err(|_| super::fault("invalid HTTP Fetch route")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn nested_file_credentials_follow_rotation_and_reject_missing_fields() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("account.json"); + let source = HttpSecret::JsonFile { + path: path.clone(), + field: "/tokens/access_token".into(), + prefix: "Bearer ".into(), + refresh: None, + }; + for token in ["first", "rotated"] { + hellas_private::write_atomically( + &path, + ".tmp", + &serde_json::to_vec(&serde_json::json!({"tokens":{"access_token":token}})).unwrap(), + ) + .unwrap(); + assert_eq!(source.resolve().await.unwrap(), format!("Bearer {token}")); + } + hellas_private::write_atomically( + &path, + ".tmp", + b"{\"tokens\":{\"refresh_token\":\"private\"}}", + ) + .unwrap(); + assert!(source.resolve().await.is_err()); + let refresh = CredentialRefresh { + command: vec!["unused".into()], + expires_field: "/tokens/expires_at".into(), + lock: Default::default(), + }; + assert!( + refresh + .due(&serde_json::json!({"tokens":{"expires_at":0}})) + .unwrap() + ); + assert!( + !refresh + .due(&serde_json::json!({"tokens":{"expires_at":4102444800_u64}})) + .unwrap() + ); + assert!(refresh.due(&serde_json::json!({"tokens":{}})).is_err()); + } + + #[cfg(unix)] + #[tokio::test] + async fn failed_refresh_is_not_repeated_by_waiting_requests() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("account.json"); + let counter = directory.path().join("count"); + hellas_private::write_atomically( + &path, + ".tmp", + b"{\"access_token\":\"old\",\"expires_at\":0}", + ) + .unwrap(); + let source = HttpSecret::JsonFile { + path, + field: "access_token".into(), + prefix: String::new(), + refresh: Some(CredentialRefresh { + command: vec![ + "/bin/sh".into(), + "-c".into(), + "printf 'attempt\\n' >> \"$1\"; exit 1".into(), + "refresh-test".into(), + counter.to_string_lossy().into(), + ], + expires_field: "expires_at".into(), + lock: Default::default(), + }), + }; + let (a, b) = tokio::join!(source.resolve(), source.resolve()); + assert!(a.is_err() && b.is_err()); + assert_eq!(std::fs::read_to_string(counter).unwrap(), "attempt\n"); + } + + #[cfg(unix)] + #[tokio::test] + async fn concurrent_reads_refresh_once_and_verify_the_replacement() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("account.json"); + let counter = directory.path().join("count"); + hellas_private::write_atomically( + &path, + ".tmp", + b"{\"access_token\":\"old\",\"expires_at\":0}", + ) + .unwrap(); + let refresh = CredentialRefresh { + command: vec!["/bin/sh".into(), "-c".into(), + "printf '%s' '{\"access_token\":\"new\",\"expires_at\":4102444800}' > \"$1\"; printf 'one\\n' >> \"$2\"".into(), + "refresh-test".into(), path.to_string_lossy().into(), counter.to_string_lossy().into()], + expires_field: "expires_at".into(), lock: Default::default(), + }; + let source = HttpSecret::JsonFile { + path, + field: "access_token".into(), + prefix: "Bearer ".into(), + refresh: Some(refresh), + }; + let (a, b) = tokio::join!(source.resolve(), source.resolve()); + assert_eq!(a.unwrap(), "Bearer new"); + assert_eq!(b.unwrap(), "Bearer new"); + assert_eq!(std::fs::read_to_string(counter).unwrap(), "one\n"); + } + #[tokio::test] + async fn file_credentials_follow_atomic_rotation_and_reject_missing_or_public_secrets() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("account.json"); + let source = HttpSecret::JsonFile { + path: path.clone(), + field: "access_token".into(), + prefix: "Bearer ".into(), + refresh: None, + }; + assert!(source.resolve().await.is_err()); + for value in ["first", "rotated"] { + hellas_private::write_atomically( + &path, + ".tmp", + &serde_json::to_vec(&serde_json::json!({"access_token":value})).unwrap(), + ) + .unwrap(); + assert_eq!(source.resolve().await.unwrap(), format!("Bearer {value}")); + } + hellas_private::write_atomically(&path, ".tmp", b"{\"refresh_token\":\"DO-NOT-LEAK\"}") + .unwrap(); + let error = source.resolve().await.unwrap_err().to_string(); + assert!(!error.contains("DO-NOT-LEAK")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(source.resolve().await.is_err()); + } + } +} diff --git a/crates/providers/src/http/tests.rs b/crates/providers/src/http/tests.rs new file mode 100644 index 00000000..cc454630 --- /dev/null +++ b/crates/providers/src/http/tests.rs @@ -0,0 +1,615 @@ +use super::*; +use hellas_rpc::http_fetch::{HttpTls, HttpTrustRoots}; +use hellas_rpc::{Digest, InputCommitment, JsonBytes}; +use rcgen::generate_simple_self_signed; +use sha2::{Digest as _, Sha256}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use x509_cert::der::{Decode as _, Encode as _}; + +async fn server( + status: u16, + bytes: Vec, + location: Option, +) -> ( + HttpFetchRequest, + Arc, + Arc, + tokio::task::JoinHandle<()>, +) { + server_with_pause(status, bytes, location, None).await +} + +async fn server_with_pause( + status: u16, + bytes: Vec, + location: Option, + pause: Option>, +) -> ( + HttpFetchRequest, + Arc, + Arc, + tokio::task::JoinHandle<()>, +) { + server_fixture(status, bytes, location, pause, None).await +} + +async fn server_fixture( + status: u16, + bytes: Vec, + location: Option, + pause: Option>, + capture: Option>>, +) -> ( + HttpFetchRequest, + Arc, + Arc, + tokio::task::JoinHandle<()>, +) { + let key = generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let cert = key.cert.der().clone(); + let parsed = x509_cert::Certificate::from_der(cert.as_ref()).unwrap(); + let pin = Sha256::digest( + parsed + .tbs_certificate + .subject_public_key_info + .to_der() + .unwrap(), + ) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + let private = rustls::pki_types::PrivatePkcs8KeyDer::from(key.signing_key.serialize_der()); + let config = rustls::ServerConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert(vec![cert.clone()], private.into()) + .unwrap(); + let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!( + "https://localhost:{}/resource", + listener.local_addr().unwrap().port() + ); + let calls = Arc::new(AtomicUsize::new(0)); + let seen = calls.clone(); + let connections = Arc::new(AtomicUsize::new(0)); + let accepted = connections.clone(); + let task = tokio::spawn(async move { + while let Ok((socket, _)) = listener.accept().await { + accepted.fetch_add(1, Ordering::SeqCst); + let acceptor = acceptor.clone(); + let seen = seen.clone(); + let bytes = bytes.clone(); + let location = location.clone(); + let pause = pause.clone(); + let capture = capture.clone(); + tokio::spawn(async move { + let Ok(mut socket) = acceptor.accept(socket).await else { + return; + }; + loop { + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + while !request.ends_with(b"\r\n\r\n") && request.len() < 32768 { + if socket.read_exact(&mut byte).await.is_err() { + return; + } + request.push(byte[0]); + } + if let Some(capture) = &capture { + let _ = capture.send(request); + } + seen.fetch_add(1, Ordering::SeqCst); + let location = location + .as_ref() + .map(|v| format!("Location: {v}\r\n")) + .unwrap_or_default(); + let retry_after = if status == 429 { + "Retry-After: 7\r\n" + } else { + "" + }; + let header = format!( + "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: keep-alive\r\n{location}{retry_after}\r\n", + bytes.len() + ); + let _ = socket.write_all(header.as_bytes()).await; + if let Some(pause) = pause.as_ref() { + let _ = socket.write_all(&bytes[..1]).await; + let _ = socket.flush().await; + pause.notified().await; + let _ = socket.write_all(&bytes[1..]).await; + let _ = socket.shutdown().await; + return; + } + if socket.write_all(&bytes).await.is_err() || socket.flush().await.is_err() { + return; + } + } + }); + } + }); + ( + HttpFetchRequest { + url, + method: "GET".into(), + headers: vec![], + body_base64: String::new(), + tls: HttpTls { + roots: HttpTrustRoots::Certificates { + der_base64: vec![STANDARD.encode(&cert)], + }, + spki_sha256: vec![pin], + }, + credential: None, + max_response_bytes: 4096, + }, + calls, + connections, + task, + ) +} + +#[tokio::test] +async fn rate_limit_is_returned_with_its_body_and_delay_without_retrying() { + let body = br#"{"error":{"type":"rate_limit"}}"#.to_vec(); + let (request, calls, _, task) = server(429, body.clone(), None).await; + let mut response = provider().run(prepared(&request)).await.unwrap(); + let head = response.head.http.unwrap(); + assert_eq!(head.status, 429); + assert!(head.headers.contains(&("retry-after".into(), "7".into()))); + let mut received = Vec::new(); + while let Some(chunk) = response.stream.next().await { + received.extend(chunk.unwrap()); + } + assert_eq!(received, body); + assert_eq!(calls.load(Ordering::SeqCst), 1); + task.abort(); +} + +#[tokio::test] +async fn coding_response_can_exceed_the_old_half_megabyte_ceiling() { + let body = vec![b'x'; 1024 * 1024]; + let (mut request, calls, _, task) = server(200, body.clone(), None).await; + request.max_response_bytes = hellas_rpc::http_fetch::MAX_HTTP_RESPONSE_BYTES; + let mut response = provider().run(prepared(&request)).await.unwrap(); + let mut received = Vec::new(); + while let Some(chunk) = response.stream.next().await { + let chunk = chunk.unwrap(); + assert!(chunk.len() <= CHUNK_BYTES); + received.extend(chunk); + } + assert_eq!(received, body); + assert_eq!(calls.load(Ordering::SeqCst), 1); + task.abort(); +} + +#[tokio::test] +async fn small_stream_delivery_does_not_wait_for_record_capacity_or_eof() { + let resume = Arc::new(tokio::sync::Notify::new()); + let (request, _, _, server) = + server_with_pause(200, b"ab".to_vec(), None, Some(resume.clone())).await; + let provider = provider(); + let mut response = provider.run(prepared(&request)).await.unwrap(); + let first = tokio::time::timeout(Duration::from_secs(1), response.stream.next()) + .await + .expect("first byte was held until EOF") + .unwrap() + .unwrap(); + assert_eq!(first, b"a"); + resume.notify_one(); + assert_eq!(response.stream.next().await.unwrap().unwrap(), b"b"); + assert!(response.stream.next().await.is_none()); + server.abort(); +} + +fn provider() -> HttpFetchProvider { + HttpFetchProvider::new( + HttpEgressPolicy { + allowed_hosts: vec!["localhost".into()], + allow_private_addresses: true, + }, + BTreeMap::new(), + ) + .unwrap() +} +fn prepared(request: &HttpFetchRequest) -> PreparedFetchRequest { + let call = FetchCall::new( + "http", + "request", + JsonBytes::new(serde_json::to_vec(request).unwrap()), + InputCommitment::from_digest(Digest::from_bytes([1; 32])), + ); + HttpFetchAdaptorFactory + .create(&call) + .unwrap() + .provider_request +} + +#[tokio::test] +async fn custom_roots_and_spki_deliver_exact_binary_bytes() { + let bytes = vec![0, 255, 1, 13, 10, 128]; + let (request, calls, _, task) = server(200, bytes.clone(), None).await; + let mut response = provider().run(prepared(&request)).await.unwrap(); + assert_eq!(response.head.http.as_ref().unwrap().status, 200); + let mut body = Vec::new(); + while let Some(chunk) = response.stream.next().await { + body.extend(chunk.unwrap()); + } + assert_eq!(body, bytes); + assert_eq!(calls.load(Ordering::SeqCst), 1); + task.abort(); +} + +#[tokio::test] +async fn wrong_pin_wrong_roots_and_wrong_hostname_send_no_http_request() { + let (valid, calls, _, task) = server(200, b"ok".to_vec(), None).await; + let mut wrong_pin = valid.clone(); + wrong_pin.tls.spki_sha256 = vec!["00".repeat(32)]; + let mut wrong_root = valid.clone(); + wrong_root.tls.roots = HttpTrustRoots::WebPki; + let mut wrong_name = valid.clone(); + wrong_name.url = wrong_name.url.replace("localhost", "127.0.0.1"); + let configured = HttpFetchProvider::new( + HttpEgressPolicy { + allowed_hosts: vec!["localhost".into(), "127.0.0.1".into()], + allow_private_addresses: true, + }, + BTreeMap::new(), + ) + .unwrap(); + let response = configured.run(prepared(&valid)).await.unwrap(); + assert_eq!(collect(response).await, b"ok"); + for request in [wrong_pin, wrong_root, wrong_name] { + assert!(configured.run(prepared(&request)).await.is_err()); + } + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "a warm connection must not bypass TLS validation" + ); + task.abort(); +} + +async fn collect(mut response: FetchProviderResponse) -> Vec { + let mut body = Vec::new(); + while let Some(chunk) = response.stream.next().await { + body.extend(chunk.unwrap()); + } + body +} + +#[tokio::test] +async fn repeated_requests_reuse_https_but_still_validate_each_request() { + let (request, calls, connections, task) = server(200, b"ok".to_vec(), None).await; + let provider = provider(); + for _ in 0..3 { + assert_eq!( + collect(provider.run(prepared(&request)).await.unwrap()).await, + b"ok" + ); + } + assert_eq!(calls.load(Ordering::SeqCst), 3); + assert_eq!(connections.load(Ordering::SeqCst), 1); + let mut forbidden = request.clone(); + forbidden.credential = Some("unknown-account".into()); + assert!(provider.run(prepared(&forbidden)).await.is_err()); + forbidden.credential = None; + forbidden.max_response_bytes = 1; + let mut response = provider.run(prepared(&forbidden)).await.unwrap(); + assert!(response.stream.next().await.unwrap().is_err()); + assert_eq!(connections.load(Ordering::SeqCst), 1); + task.abort(); +} + +#[tokio::test] +async fn client_pools_separate_dns_answers_aliases_and_tls_settings_and_evict() { + let (mut request, calls, connections, task) = server(200, b"ok".to_vec(), None).await; + let valid_tls = request.tls.clone(); + let url = request.parsed_url().unwrap(); + let address = SocketAddr::from(([127, 0, 0, 1], url.port().unwrap())); + let other = SocketAddr::from(([127, 0, 0, 2], url.port().unwrap())); + let clients = clients::Clients::default(); + for addresses in [vec![address, other], vec![other, address, address]] { + let client = clients.get(&request, &url, addresses).unwrap(); + assert_eq!( + client + .get(url.clone()) + .send() + .await + .unwrap() + .bytes() + .await + .unwrap(), + "ok" + ); + } + assert_eq!( + connections.load(Ordering::SeqCst), + 1, + "DNS order must not discard a usable pool" + ); + let denied = clients.get(&request, &url, vec![other]).unwrap(); + assert!( + denied.get(url.clone()).send().await.is_err(), + "a new DNS answer must not reuse the old address" + ); + assert_eq!(calls.load(Ordering::SeqCst), 2); + // Exercise the cache boundary directly: production credential authorization + // runs before this method and requires public roots, tested separately. + request.credential = Some("second-account".into()); + let client = clients.get(&request, &url, vec![address, other]).unwrap(); + assert_eq!( + client + .get(url.clone()) + .send() + .await + .unwrap() + .bytes() + .await + .unwrap(), + "ok" + ); + drop(client); + assert_eq!(connections.load(Ordering::SeqCst), 2); + request.tls.spki_sha256 = vec!["00".repeat(32)]; + let client = clients.get(&request, &url, vec![address, other]).unwrap(); + assert!(client.get(url.clone()).send().await.is_err()); + assert_eq!( + calls.load(Ordering::SeqCst), + 3, + "wrong pins must not reuse a verified socket" + ); + drop(client); + request.tls.spki_sha256.clear(); + // More distinct policies than the cache admits must evict its old pools. + for index in 0..40 { + request.credential = Some(format!("account-{index}")); + clients.get(&request, &url, vec![address]).unwrap(); + } + let before = connections.load(Ordering::SeqCst); + request.credential = None; + request.tls = valid_tls; + let client = clients.get(&request, &url, vec![address, other]).unwrap(); + assert_eq!( + client.get(url).send().await.unwrap().bytes().await.unwrap(), + "ok" + ); + assert_eq!(connections.load(Ordering::SeqCst), before + 1); + task.abort(); +} + +#[tokio::test] +async fn redirects_are_returned_without_following_them() { + let (target, target_calls, _, target_task) = server(200, b"private".to_vec(), None).await; + let (origin, origin_calls, _, origin_task) = server(302, vec![], Some(target.url)).await; + let response = provider().run(prepared(&origin)).await.unwrap(); + assert_eq!(response.head.http.unwrap().status, 302); + assert_eq!(origin_calls.load(Ordering::SeqCst), 1); + assert_eq!(target_calls.load(Ordering::SeqCst), 0); + origin_task.abort(); + target_task.abort(); +} + +#[tokio::test] +async fn signed_response_size_and_default_private_address_denial_are_enforced() { + let (mut request, calls, _, task) = server(200, vec![42; 100], None).await; + let public = HttpFetchProvider::new(HttpEgressPolicy::default(), BTreeMap::new()).unwrap(); + assert!(public.run(prepared(&request)).await.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 0); + request.max_response_bytes = 16; + let mut response = provider().run(prepared(&request)).await.unwrap(); + assert!(response.stream.next().await.unwrap().is_err()); + task.abort(); +} + +#[test] +fn credentials_cannot_be_redirected_or_used_with_caller_trust_anchors() { + let credential = HttpCredential { + allowed_origins: vec!["https://api.example.com".into()], + allowed_paths: vec!["/path".into()], + allowed_methods: vec!["POST".into()], + header_name: "authorization".into(), + header_value: "Bearer PRIVATE".into(), + }; + let provider = HttpFetchProvider::new( + HttpEgressPolicy::default(), + BTreeMap::from([("account-1".into(), credential)]), + ) + .unwrap(); + let mut request = HttpFetchRequest { + url: "https://api.example.com/path".into(), + method: "POST".into(), + headers: vec![], + body_base64: String::new(), + tls: HttpTls { + roots: HttpTrustRoots::WebPki, + spki_sha256: vec![], + }, + credential: Some("account-1".into()), + max_response_bytes: 1024, + }; + assert!( + provider + .credential(&request, &request.parsed_url().unwrap()) + .unwrap() + .is_some() + ); + request.method = "DELETE".into(); + assert!( + provider + .credential(&request, &request.parsed_url().unwrap()) + .is_err() + ); + request.method = "POST".into(); + request.url = "https://api.example.com/other".into(); + assert!( + provider + .credential(&request, &request.parsed_url().unwrap()) + .is_err() + ); + request.url = "https://attacker.example/".into(); + assert!( + provider + .credential(&request, &request.parsed_url().unwrap()) + .is_err() + ); + request.url = "https://api.example.com/path".into(); + request.tls.roots = HttpTrustRoots::Certificates { der_base64: vec![] }; + assert!( + provider + .credential(&request, &request.parsed_url().unwrap()) + .is_err() + ); + assert!(!format!("{provider:?}").contains("PRIVATE")); +} + +#[test] +fn special_addresses_never_qualify_as_public_egress() { + for ip in [ + "127.0.0.1", + "10.1.2.3", + "169.254.169.254", + "100.64.0.1", + "0.0.0.0", + "192.0.2.1", + "198.18.0.1", + "224.0.0.1", + "::1", + "::ffff:8.8.8.8", + "64:ff9b::808:808", + "2001:db8::1", + "2002:7f00:1::", + ] { + assert!(!public_address(ip.parse().unwrap()), "{ip}"); + } + assert!(public_address("8.8.8.8".parse().unwrap())); + assert!(public_address("2606:4700:4700::1111".parse().unwrap())); +} + +/// Offline demonstration: the TLS upstream sees the provider's credential; +/// the signed buyer request and authenticated response contain only the alias. +#[tokio::test] +async fn provider_injects_secret_without_disclosing_it_to_the_buyer() { + use hellas_rpc::fetch::{ + FetchOutputTranscriptBuilder, build_input_events_with_retention, verify_input_events, + verify_output_events, + }; + use hellas_rpc::{Assurance, ProducerSigningKey, Retention}; + const SECRET: &str = "Bearer provider-only-demo-key"; + let (capture, mut captured) = tokio::sync::mpsc::unbounded_channel(); + let (mut request, calls, _, server) = server_fixture( + 200, + b"upstream response".to_vec(), + None, + None, + Some(capture), + ) + .await; + let url = request.parsed_url().unwrap(); + request.credential = Some("provider-account".into()); + let provider = HttpFetchProvider::new( + HttpEgressPolicy { + allowed_hosts: vec!["localhost".into()], + allow_private_addresses: true, + }, + BTreeMap::from([( + "provider-account".into(), + HttpCredential { + allowed_origins: vec![url.origin().ascii_serialization()], + allowed_paths: vec!["/resource".into()], + allowed_methods: vec!["GET".into()], + header_name: "authorization".into(), + header_value: HttpSecret::Value(SECRET.into()), + }, + )]), + ) + .unwrap(); + let addresses = tokio::net::lookup_host(("localhost", url.port().unwrap())) + .await + .unwrap() + .take(65) + .collect(); + provider.clients.trust_fixture(&request, &url, addresses); + request.tls.roots = HttpTrustRoots::WebPki; + let buyer = ProducerSigningKey::from_secret_bytes([71; 32]).unwrap(); + let seller = ProducerSigningKey::from_secret_bytes([72; 32]).unwrap(); + let input_events = build_input_events_with_retention( + "http", + "request", + &serde_json::to_vec(&request).unwrap(), + FetchEnvironment::Http.manifest_id(), + Assurance::ProducerSigned, + &buyer, + Retention::Ephemeral, + ) + .unwrap(); + let input = verify_input_events(&input_events).unwrap(); + let call = FetchCall::new( + &input.service, + &input.method, + input.body.clone(), + input.input_commitment, + ); + let session = HttpFetchAdaptorFactory.create(&call).unwrap(); + let mut projector = session.projector; + let response = provider.run(session.provider_request).await.unwrap(); + let mut projected = projector.begin(response.head).unwrap(); + let mut stream = response.stream; + while let Some(chunk) = stream.next().await { + projected.extend(projector.project(&chunk.unwrap()).unwrap()); + } + projected.extend(projector.finish().unwrap()); + let terminal = match projected.pop().unwrap() { + ProjectedFetch::Terminal(bytes) => bytes, + _ => panic!("terminal"), + }; + let mut output = + FetchOutputTranscriptBuilder::new(input.input_commitment, input.assurance, &seller); + for event in projected { + let ProjectedFetch::Event(bytes) = event else { + panic!("event") + }; + output.push_event(bytes).unwrap(); + } + let output_events = output.finish(terminal).unwrap(); + let verified = + verify_output_events(input.input_commitment, input.assurance, &output_events).unwrap(); + let delivered = + hellas_rpc::http_fetch::HttpFetchResponse::from_output(&request, &verified).unwrap(); + assert_eq!(delivered.body, b"upstream response"); + let upstream = String::from_utf8(captured.recv().await.unwrap()).unwrap(); + assert!(upstream.contains(&format!("authorization: {SECRET}\r\n"))); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert!(input_events.iter().all(|event| { + !event + .payload() + .windows(SECRET.len()) + .any(|w| w == SECRET.as_bytes()) + })); + assert!(output_events.iter().all(|event| { + !event + .payload() + .windows(SECRET.len()) + .any(|w| w == SECRET.as_bytes()) + })); + // Reusing the TLS pool cannot bypass credential scope or override checks. + for (path, headers) in [ + ("/elsewhere", vec![]), + ( + "/resource", + vec![("authorization".into(), "buyer override".into())], + ), + ] { + let mut forbidden = request.clone(); + forbidden.url = format!("{}{path}", url.origin().ascii_serialization()); + forbidden.headers = headers; + assert!(provider.run(prepared(&forbidden)).await.is_err()); + } + assert_eq!(calls.load(Ordering::SeqCst), 1); + server.abort(); +} diff --git a/crates/providers/src/http/tls.rs b/crates/providers/src/http/tls.rs new file mode 100644 index 00000000..1d2a95bd --- /dev/null +++ b/crates/providers/src/http/tls.rs @@ -0,0 +1,100 @@ +use hellas_rpc::http_fetch::{HttpTls, HttpTrustRoots, decode_base64, decode_pin}; +use rustls::client::WebPkiServerVerifier; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{DigitallySignedStruct, SignatureScheme}; +use sha2::{Digest as _, Sha256}; +use std::sync::Arc; +use x509_cert::der::{Decode as _, Encode as _}; + +pub(super) fn config(settings: &HttpTls) -> Result { + settings.validate().map_err(|_| "invalid TLS settings")?; + let mut roots = rustls::RootCertStore::empty(); + match &settings.roots { + HttpTrustRoots::WebPki => roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()), + HttpTrustRoots::Certificates { der_base64 } => { + for der in der_base64 { + let bytes = decode_base64(der).map_err(|_| "invalid trust anchor")?; + roots + .add(CertificateDer::from(bytes)) + .map_err(|_| "invalid trust anchor")?; + } + } + } + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let chain = WebPkiServerVerifier::builder_with_provider(Arc::new(roots), provider.clone()) + .build() + .map_err(|_| "invalid trust anchors")?; + let pins = settings + .spki_sha256 + .iter() + .map(|pin| decode_pin(pin)) + .collect::, _>>() + .map_err(|_| "invalid SPKI pins")?; + Ok(rustls::ClientConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .map_err(|_| "TLS versions unavailable")? + .dangerous() + .with_custom_certificate_verifier(Arc::new(PinnedVerifier { chain, pins })) + .with_no_client_auth()) +} + +#[derive(Debug)] +struct PinnedVerifier { + chain: Arc, + pins: Vec<[u8; 32]>, +} + +impl ServerCertVerifier for PinnedVerifier { + fn verify_server_cert( + &self, + leaf: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + name: &ServerName<'_>, + ocsp: &[u8], + now: UnixTime, + ) -> Result { + // Pins are additional constraints: they never bypass chain, validity, + // hostname, or the handshake proof of possession. + let verified = self + .chain + .verify_server_cert(leaf, intermediates, name, ocsp, now)?; + if !self.pins.is_empty() { + let cert = x509_cert::Certificate::from_der(leaf.as_ref()).map_err(|_| { + rustls::Error::InvalidCertificate(rustls::CertificateError::BadEncoding) + })?; + let spki = cert + .tbs_certificate + .subject_public_key_info + .to_der() + .map_err(|_| { + rustls::Error::InvalidCertificate(rustls::CertificateError::BadEncoding) + })?; + let digest: [u8; 32] = Sha256::digest(spki).into(); + if !self.pins.contains(&digest) { + return Err(rustls::Error::General("SPKI pin mismatch".into())); + } + } + Ok(verified) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + signature: &DigitallySignedStruct, + ) -> Result { + self.chain.verify_tls12_signature(message, cert, signature) + } + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + signature: &DigitallySignedStruct, + ) -> Result { + self.chain.verify_tls13_signature(message, cert, signature) + } + fn supported_verify_schemes(&self) -> Vec { + self.chain.supported_verify_schemes() + } +} diff --git a/crates/providers/src/lib.rs b/crates/providers/src/lib.rs index 9fe164fd..843f8582 100644 --- a/crates/providers/src/lib.rs +++ b/crates/providers/src/lib.rs @@ -2,11 +2,16 @@ //! hosts and the Hellas CLI. mod codex_responses; +mod http; mod openai; mod responses_fetch; mod responses_projector; mod stream; +pub use http::{ + CredentialRefresh, HttpCredential, HttpCredentialConfig, HttpEgressPolicy, + HttpFetchAdaptorFactory, HttpFetchProvider, HttpProviderConfig, HttpSecret, +}; -pub use openai::OpenAiResponsesFetchProvider; +pub use openai::{EmptyOpenAiKey, OpenAiResponsesFetchProvider}; pub use responses_fetch::{execute_responses_request, responses_http_client}; pub use responses_projector::ResponsesFetchAdaptorFactory; diff --git a/crates/providers/src/openai.rs b/crates/providers/src/openai.rs index 1a2c5b6c..30e40697 100644 --- a/crates/providers/src/openai.rs +++ b/crates/providers/src/openai.rs @@ -8,6 +8,10 @@ use reqwest::Url; use super::responses_fetch::{execute_responses_request, responses_http_client}; +#[derive(Debug, thiserror::Error)] +#[error("OpenAI API key is empty")] +pub struct EmptyOpenAiKey; + #[derive(Clone)] pub struct OpenAiResponsesFetchProvider { client: reqwest::Client, @@ -23,16 +27,16 @@ impl OpenAiResponsesFetchProvider { if bearer_token.is_empty() { bail!("environment variable {api_key_env} is empty"); } - Self::with_bearer(bearer_token) + Ok(Self::with_bearer(bearer_token)?) } /// Construct from a secret supplied by an embedding host without routing /// it through process-wide environment state. - pub fn with_bearer(bearer_token: impl Into) -> anyhow::Result { + pub fn with_bearer(bearer_token: impl Into) -> Result { let bearer_token = bearer_token.into(); let bearer_token = bearer_token.trim().to_owned(); if bearer_token.is_empty() { - bail!("OpenAI API key is empty"); + return Err(EmptyOpenAiKey); } let endpoint = Url::parse(OPENAI_RESPONSES_ENDPOINT) .expect("built-in OpenAI Responses endpoint is valid"); diff --git a/crates/providers/src/responses_fetch.rs b/crates/providers/src/responses_fetch.rs index 840745aa..98299fa7 100644 --- a/crates/providers/src/responses_fetch.rs +++ b/crates/providers/src/responses_fetch.rs @@ -8,15 +8,12 @@ use tracing::Instrument; #[cfg_attr(feature = "otel", path = "responses_fetch/telemetry/otel.rs")] #[cfg_attr(not(feature = "otel"), path = "responses_fetch/telemetry/noop.rs")] -mod telemetry; +pub(crate) mod telemetry; -/// Maximum diagnostic prefix retained from an unsuccessful HTTP response. -const MAX_FETCH_ERROR_BODY_BYTES: usize = 2 * 1024; /// A total request deadline bounds the whole call; this independent idle /// deadline prevents a peer that stops producing SSE bytes from occupying a /// Fetch execution slot for that entire window. Ordinary SSE keepalives count -/// as activity and reset it. Error bodies are diagnostic only, so the same -/// duration bounds their entire prefix read without renewal. +/// as activity and reset it. Unsuccessful response bodies are never read or logged. const FETCH_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(90); /// HTTP client for attested Fetch egress. Redirects are disabled because the @@ -42,14 +39,14 @@ pub async fn execute_responses_request( label: &str, ) -> Result { let mut telemetry = telemetry::Request::new(&endpoint); - let request = telemetry - .propagate(client.post(endpoint)) + let request = client + .post(endpoint) .header(CONTENT_TYPE, "application/json") .header(AUTHORIZATION, format!("Bearer {bearer_token}")) .header("Idempotency-Key", idempotency_key) .body(body); - let upstream = request - .send() + let upstream = telemetry + .send(request) .instrument(telemetry.span.clone()) .await .map_err(|source| { @@ -61,12 +58,13 @@ pub async fn execute_responses_request( telemetry.status(status.as_u16()); if !status.is_success() { telemetry.fail("http_error"); - let diagnostic = error_body_prefix(upstream).await; + // Error bodies can echo customer input. Drop them without reading or + // logging a prefix, even when the request itself is ephemeral. + drop(upstream); tracing::warn!( provider = label, upstream_status = status.as_u16(), - upstream_diagnostic = %diagnostic, - "upstream Fetch request failed; response details are omitted from the caller error" + "upstream Fetch request failed" ); return Err(FetchProviderError::failed(format!( "{label} upstream rejected the request (HTTP {})", @@ -90,6 +88,7 @@ pub async fn execute_responses_request( let head = FetchProviderResponseHead { effective_model: effective_model_from_headers(upstream.headers()) .inspect_err(|_| telemetry.fail("invalid_response_headers"))?, + http: None, }; Ok(FetchProviderResponse { head, @@ -133,82 +132,6 @@ fn unique_model_header( Ok(value) } -async fn error_body_prefix(upstream: reqwest::Response) -> String { - error_body_prefix_with_deadline(upstream, FETCH_STREAM_IDLE_TIMEOUT).await -} - -async fn error_body_prefix_with_deadline( - upstream: reqwest::Response, - read_timeout: Duration, -) -> String { - let mut body = Vec::new(); - let mut chunks = upstream.bytes_stream(); - let deadline = tokio::time::Instant::now() + read_timeout; - - while body.len() < MAX_FETCH_ERROR_BODY_BYTES { - let next = match tokio::time::timeout_at(deadline, chunks.next()).await { - Ok(next) => next, - Err(_) => { - let excerpt = diagnostic_excerpt(&body); - let note = format!( - "[error body read timed out after {} seconds]", - read_timeout.as_secs_f64() - ); - return if excerpt.is_empty() { - note - } else { - format!("{excerpt} {note}") - }; - } - }; - let Some(chunk) = next else { - break; - }; - let chunk = match chunk { - Ok(chunk) => chunk, - Err(source) => { - let excerpt = diagnostic_excerpt(&body); - let note = format!("[error body read failed: {source}]"); - return if excerpt.is_empty() { - note - } else { - format!("{excerpt} {note}") - }; - } - }; - let remaining = MAX_FETCH_ERROR_BODY_BYTES - body.len(); - let retained = remaining.min(chunk.len()); - body.extend_from_slice(&chunk[..retained]); - if retained < chunk.len() { - break; - } - } - - let limited = body.len() == MAX_FETCH_ERROR_BODY_BYTES; - let body = diagnostic_excerpt(&body); - if limited { - format!("{body} [body prefix limited to {MAX_FETCH_ERROR_BODY_BYTES} bytes]") - } else { - body - } -} - -fn diagnostic_excerpt(body: &[u8]) -> String { - String::from_utf8_lossy(body) - .chars() - .map(|character| { - if character.is_control() { - ' ' - } else { - character - } - }) - .collect::() - .split_whitespace() - .collect::>() - .join(" ") -} - fn stream_response( upstream: reqwest::Response, label: String, diff --git a/crates/providers/src/responses_fetch/telemetry/noop.rs b/crates/providers/src/responses_fetch/telemetry/noop.rs index 190e23a7..27f4dae8 100644 --- a/crates/providers/src/responses_fetch/telemetry/noop.rs +++ b/crates/providers/src/responses_fetch/telemetry/noop.rs @@ -2,15 +2,21 @@ use futures::Stream; use hellas_executor::FetchProviderError; use tracing::Span; -pub(super) struct Request { +pub(crate) struct Request { pub span: Span, } impl Request { pub fn new(_: &reqwest::Url) -> Self { Self { span: Span::none() } } - pub fn propagate(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder { - request + pub fn for_method(_: &reqwest::Url, _: &str) -> Self { + Self { span: Span::none() } + } + pub async fn send( + &self, + request: reqwest::RequestBuilder, + ) -> reqwest::Result { + request.send().await } pub fn status(&self, _: u16) {} pub fn fail(&mut self, _: &'static str) {} diff --git a/crates/providers/src/responses_fetch/telemetry/otel.rs b/crates/providers/src/responses_fetch/telemetry/otel.rs index 70f54f6b..177855c4 100644 --- a/crates/providers/src/responses_fetch/telemetry/otel.rs +++ b/crates/providers/src/responses_fetch/telemetry/otel.rs @@ -4,16 +4,19 @@ use hellas_executor::FetchProviderError; use hellas_wire::metadata::Metadata; use tracing::{Instrument, Span}; -pub(super) struct Request { +pub(crate) struct Request { pub span: Span, complete: bool, } impl Request { pub fn new(endpoint: &reqwest::Url) -> Self { + Self::for_method(endpoint, "POST") + } + pub fn for_method(endpoint: &reqwest::Url, method: &str) -> Self { Self { complete: false, - span: hellas_rpc::request_span!(target: "hellas_request", "POST", - otel.kind = "client", http.request.method = "POST", + span: hellas_rpc::request_span!(target: "hellas_request", "http.upstream", + otel.kind = "client", otel.name = method, http.request.method = method, server.address = endpoint.host_str().unwrap_or(""), server.port = endpoint.port_or_known_default().map(i64::from), url.scheme = endpoint.scheme(), @@ -22,15 +25,31 @@ impl Request { otel.status_code = tracing::field::Empty), } } - pub fn propagate(&self, mut request: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + pub async fn send( + &self, + builder: reqwest::RequestBuilder, + ) -> reqwest::Result { + let (client, request) = builder.build_split(); + let mut request = request?; let mut metadata = Metadata::new(); hellas_rpc::telemetry::inject(&self.span, &mut metadata); - for name in ["traceparent", "tracestate"] { - if let Some(value) = metadata.get(name).and_then(|v| v.as_text()) { - request = request.header(name, value); + // Without an active tracing context, preserve the caller's headers. + // Otherwise replace both fields, including stale tracestate when the + // new context has none. RequestBuilder::header would append duplicates. + if metadata.get("traceparent").is_some() { + for name in ["traceparent", "tracestate"] { + request.headers_mut().remove(name); + if let Some(value) = metadata + .get(name) + .and_then(|v| v.as_text()) + .filter(|value| !value.is_empty()) + && let Ok(value) = value.parse() + { + request.headers_mut().insert(name, value); + } } } - request + client.execute(request).await } pub fn status(&self, code: u16) { self.span @@ -70,3 +89,76 @@ impl Drop for Request { } } } + +#[cfg(test)] +mod tests { + use super::*; + use opentelemetry::trace::TracerProvider; + use tracing_subscriber::prelude::*; + + #[tokio::test] + async fn propagation_replaces_duplicates_and_removes_stale_tracestate() { + opentelemetry::global::set_text_map_propagator( + opentelemetry_sdk::propagation::TraceContextPropagator::new(), + ); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build(); + let dispatch = tracing::Dispatch::new( + tracing_subscriber::registry() + .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("headers"))), + ); + let (tx, mut rx) = tokio::sync::mpsc::channel(2); + let app = axum::Router::new().route( + "/", + axum::routing::get(move |headers: axum::http::HeaderMap| { + let tx = tx.clone(); + async move { + tx.send(headers).await.unwrap(); + "ok" + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url: reqwest::Url = format!("http://{}/", listener.local_addr().unwrap()) + .parse() + .unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let client = reqwest::Client::new(); + for active in [true, false] { + let trace = if active { + tracing::dispatcher::with_default(&dispatch, || Request::new(&url)) + } else { + Request { + span: Span::none(), + complete: false, + } + }; + let mut context = Metadata::new(); + hellas_rpc::telemetry::inject(&trace.span, &mut context); + let builder = client + .get(url.clone()) + .header("traceparent", "old-one") + .header("traceparent", "old-two") + .header("tracestate", "vendor=stale") + .header("x-extension", "one") + .header("x-extension", "two"); + assert_eq!( + trace.send(builder).await.unwrap().text().await.unwrap(), + "ok" + ); + let headers = rx.recv().await.unwrap(); + assert_eq!(headers.get_all("x-extension").iter().count(), 2); + if active { + assert_eq!(headers.get_all("traceparent").iter().count(), 1); + assert_eq!( + headers["traceparent"], + context.get("traceparent").unwrap().as_text().unwrap() + ); + assert!(!headers.contains_key("tracestate")); + } else { + assert_eq!(headers.get_all("traceparent").iter().count(), 2); + assert_eq!(headers["tracestate"], "vendor=stale"); + } + } + server.abort(); + } +} diff --git a/crates/providers/src/responses_fetch/tests.rs b/crates/providers/src/responses_fetch/tests.rs index acdf3a0e..73345faf 100644 --- a/crates/providers/src/responses_fetch/tests.rs +++ b/crates/providers/src/responses_fetch/tests.rs @@ -9,8 +9,37 @@ use futures::stream; use reqwest::header::{HeaderMap, HeaderValue}; use std::convert::Infallible; use std::sync::Arc; +use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; +#[derive(Clone, Default)] +struct CapturedLogs(Arc>); + +impl tracing::field::Visit for CapturedLogs { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + use std::fmt::Write as _; + write!(self.0.lock().unwrap(), "{field}={value:?} ").unwrap(); + } +} + +impl tracing::Subscriber for CapturedLogs { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, values: &tracing::span::Record<'_>) { + values.record(&mut self.clone()); + } + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + event.record(&mut self.clone()); + } + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} +} + async fn redirect() -> Redirect { Redirect::temporary("/sink") } @@ -26,15 +55,6 @@ async fn json_success() -> Response { .unwrap() } -async fn oversized_error() -> Response { - let mut body = vec![b'x'; MAX_FETCH_ERROR_BODY_BYTES]; - body.extend_from_slice(b"SECRET_AFTER_LIMIT"); - Response::builder() - .status(StatusCode::BAD_REQUEST) - .body(Body::from(body)) - .unwrap() -} - async fn sensitive_error() -> Response { Response::builder() .status(StatusCode::UNPROCESSABLE_ENTITY) @@ -59,43 +79,6 @@ async fn oversized_event_stream() -> Response { .unwrap() } -fn stalled_error_before_first_byte() -> reqwest::Response { - axum::http::Response::builder() - .status(StatusCode::BAD_GATEWAY) - .body(reqwest::Body::wrap_stream(stream::pending::< - Result, - >())) - .unwrap() - .into() -} - -fn stalled_error_after_partial_body() -> reqwest::Response { - let chunks = - stream::once(async { Ok::<_, Infallible>(Bytes::from_static(b"useful diagnostic")) }) - .chain(stream::pending()); - axum::http::Response::builder() - .status(StatusCode::BAD_GATEWAY) - .body(reqwest::Body::wrap_stream(chunks)) - .unwrap() - .into() -} - -fn promptly_streamed_error() -> reqwest::Response { - let chunks = stream::iter([ - Bytes::from_static(b"useful "), - Bytes::from_static(b"diagnostic"), - ]) - .then(|chunk| async move { - tokio::time::sleep(Duration::from_secs(1)).await; - Ok::<_, Infallible>(chunk) - }); - axum::http::Response::builder() - .status(StatusCode::BAD_GATEWAY) - .body(reqwest::Body::wrap_stream(chunks)) - .unwrap() - .into() -} - async fn test_endpoint(app: Router, path: &str) -> Url { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let endpoint = Url::parse(&format!( @@ -156,31 +139,17 @@ async fn successful_fetch_requires_event_stream_content_type() { } #[tokio::test] -async fn provider_local_error_diagnostic_retains_only_a_bounded_body_prefix() { - let endpoint = test_endpoint( - Router::new().route("/responses", post(oversized_error)), - "/responses", - ) - .await; - - let response = responses_http_client().post(endpoint).send().await.unwrap(); - let diagnostic = error_body_prefix(response).await; - - assert!(diagnostic.contains(&format!( - "body prefix limited to {MAX_FETCH_ERROR_BODY_BYTES} bytes" - ))); - assert!(!diagnostic.contains("SECRET_AFTER_LIMIT")); - assert!(diagnostic.len() < MAX_FETCH_ERROR_BODY_BYTES + 256); -} - -#[tokio::test] -async fn unsuccessful_fetch_never_forwards_the_upstream_body_to_the_caller() { +async fn unsuccessful_fetch_never_logs_or_returns_the_upstream_body() { let endpoint = test_endpoint( Router::new().route("/responses", post(sensitive_error)), "/responses", ) .await; + let logs = CapturedLogs::default(); + // Keep this subscriber installed for the test binary: other HTTP tests + // also register the shared warning callsite on their runtime threads. + tracing::subscriber::set_global_default(logs.clone()).unwrap(); let Err(error) = execute_test_request(endpoint).await else { panic!("HTTP error unexpectedly passed"); }; @@ -191,45 +160,33 @@ async fn unsuccessful_fetch_never_forwards_the_upstream_body_to_the_caller() { "fetch provider failed: test upstream rejected the request (HTTP 422)" ); assert!(!message.contains("UPSTREAM_PRIVATE_SENTINEL")); + let logged = logs.0.lock().unwrap(); + assert!(logged.contains("upstream_status=422"), "{logged}"); + assert!(!logged.contains("UPSTREAM_PRIVATE_SENTINEL"), "{logged}"); } -#[tokio::test(start_paused = true)] -async fn error_body_stalled_before_first_byte_hits_the_read_deadline() { - let upstream = stalled_error_before_first_byte(); - let read_timeout = Duration::from_secs(5); - let started = tokio::time::Instant::now(); - - let diagnostic = error_body_prefix_with_deadline(upstream, read_timeout).await; - - assert_eq!(started.elapsed(), read_timeout); - assert_eq!(diagnostic, "[error body read timed out after 5 seconds]"); -} - -#[tokio::test(start_paused = true)] -async fn partial_error_body_is_retained_when_the_read_deadline_expires() { - let upstream = stalled_error_after_partial_body(); - let read_timeout = Duration::from_secs(5); - let started = tokio::time::Instant::now(); - - let diagnostic = error_body_prefix_with_deadline(upstream, read_timeout).await; - - assert_eq!(started.elapsed(), read_timeout); - assert_eq!( - diagnostic, - "useful diagnostic [error body read timed out after 5 seconds]" - ); -} - -#[tokio::test(start_paused = true)] -async fn promptly_streamed_error_body_completes_within_the_read_deadline() { - let upstream = promptly_streamed_error(); - let read_timeout = Duration::from_secs(5); - let started = tokio::time::Instant::now(); - - let diagnostic = error_body_prefix_with_deadline(upstream, read_timeout).await; - - assert_eq!(started.elapsed(), Duration::from_secs(2)); - assert_eq!(diagnostic, "useful diagnostic"); +#[tokio::test] +async fn an_unsuccessful_fetch_does_not_wait_for_an_error_body() { + async fn stalled() -> Response { + Response::builder() + .status(StatusCode::BAD_GATEWAY) + .body(Body::from_stream(stream::pending::< + Result, + >())) + .unwrap() + } + let endpoint = test_endpoint( + Router::new().route("/responses", post(stalled)), + "/responses", + ) + .await; + let result = tokio::time::timeout(Duration::from_secs(2), execute_test_request(endpoint)) + .await + .expect("the response body must not be polled"); + let Err(error) = result else { + panic!("HTTP error unexpectedly passed"); + }; + assert!(error.to_string().contains("HTTP 502")); } #[tokio::test] diff --git a/crates/providers/src/responses_projector.rs b/crates/providers/src/responses_projector.rs index 2c2b53ca..e531c558 100644 --- a/crates/providers/src/responses_projector.rs +++ b/crates/providers/src/responses_projector.rs @@ -35,6 +35,9 @@ impl FetchAdaptorFactory for ResponsesFetchAdaptorFactory { } fn create(&self, request: &FetchCall) -> Result { + if self.environment == FetchEnvironment::Http { + return Err(fetch_failed("HTTP requires the HTTP adaptor")); + } if self.environment == FetchEnvironment::CodexResponses { return super::codex_responses::create_session(request); } diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index ab04a824..0902959b 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -45,11 +45,15 @@ all-protocols = ["swarm", "courtesy", "execute", "evaluate", "fetch", "chain"] # needs the consensus kernel's types — the certificate, the network id, # and the edge/terms commitments — and a wasm or fetch-only client has no # use for them. The alternative was restating those kernel types here, -# which is how two definitions of a signed digest start. -work = ["dep:hellas-kernel", "evaluate", "hellas-kernel/secp256k1"] +# which is how two definitions of a signed digest start. Both paid +# profiles are pulled forward: `evaluate` for the causal-LM profile and +# `fetch` for the sealed-fetch profile (`protocol::work_fetch`), whose +# records commit to signed fetch transcripts. +work = ["dep:hellas-kernel", "evaluate", "fetch", "hellas-kernel/secp256k1"] [dependencies] axum = { workspace = true, optional = true } +base64.workspace = true base64ct = { version = "1", optional = true, features = ["alloc"] } bytes = { workspace = true } ed25519-dalek = { workspace = true } @@ -74,6 +78,7 @@ thiserror = { workspace = true } # sample, and a node with no subscriber pays one atomic load per seam. tracing = { workspace = true } tracing-opentelemetry = { workspace = true, optional = true } +url = "2" web-time = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/crates/rpc/build.rs b/crates/rpc/build.rs index 63f7f6ec..831da70a 100644 --- a/crates/rpc/build.rs +++ b/crates/rpc/build.rs @@ -776,6 +776,27 @@ fn handler_signature(m: &MethodPlan) -> TokenStream { Shape::Unary => quote! { impl Into> + Send }, _ => boxed_stream(response), }; + // Every service carries the confidential-Open handshake method, named + // `open` in each package. A dispatcher that never mounted an OpenHandler + // must still answer it: refusing with Unavailable keeps a missing mount + // a runtime refusal for the peer instead of a compile error here. The + // match is on the bare method name, so any future `open` rpc in any + // package gets this same default. + if fn_name == "open" { + let allow_open_context_unused = if m.connection_bound { + quote! { let _ = context; } + } else { + quote! {} + }; + return quote! { + fn #fn_name(&self, _request: #request_ty #context) + -> impl ::core::future::Future> + Send { + #allow_open_context_unused + ::core::future::ready(Err(::hellas_wire::WireStatus::new( + ::hellas_wire::WireCode::Unavailable, "confidential Open is not mounted"))) + } + }; + } quote! { fn #fn_name( &self, diff --git a/crates/rpc/src/call.rs b/crates/rpc/src/call.rs index 751746b2..21b8be6e 100644 --- a/crates/rpc/src/call.rs +++ b/crates/rpc/src/call.rs @@ -67,19 +67,8 @@ where .open(M::METHOD_ID, headers) .await .map_err(transport_to_status)?; - let (mut send, recv) = WireStream::split(stream); - let mut recv = Box::pin(recv); - - let mut buf = BytesMut::with_capacity(request.encoded_len()); - request - .encode(&mut buf) - .map_err(|e| WireStatus::internal(format!("prost encode: {e}")))?; - send.send_body(buf.freeze()) - .await - .map_err(|e| WireStatus::internal(format!("send: {e}")))?; - send.close_send(None) - .await - .map_err(|e| WireStatus::internal(format!("close_send: {e}")))?; + let (mut send, mut recv) = WireStream::split(stream); + send_request(&mut send, &mut recv, request).await?; // Unary protocol shape: exactly one body chunk, then EOF, then a // terminal trailer. Anything else is a server-side bug and must @@ -147,6 +136,42 @@ fn trailer_to_status(t: &Trailer) -> WireStatus { } } +/// A server may reject at Open, before accepting the request body. Preserve +/// its refusal when stopping the upload also makes a local write fail. +async fn send_request( + send: &mut S, + recv: &mut R, + request: Q, +) -> Result<(), WireStatus> { + let mut buf = BytesMut::with_capacity(request.encoded_len()); + request + .encode(&mut buf) + .map_err(|e| WireStatus::internal(format!("prost encode: {e}")))?; + let written = async { + send.send_body(buf.freeze()) + .await + .map_err(|e| WireStatus::internal(format!("send: {e}")))?; + send.close_send(None) + .await + .map_err(|e| WireStatus::internal(format!("close_send: {e}"))) + } + .await; + if let Err(error) = written { + // Only a terminal error can explain an early rejection. Do not drain + // arbitrary bodies or wait indefinitely for a peer that stopped reading. + if matches!( + n0_future::time::timeout(UNARY_RECEIVE_TIMEOUT, recv.next()).await, + Ok(None) + ) && let Some(trailer) = recv.trailer() + && trailer.status != WireCode::Ok + { + return Err(trailer_to_status(trailer)); + } + return Err(error); + } + Ok(()) +} + /// Server-streaming call: send one request, receive a stream of responses /// + a terminal trailer. /// @@ -181,18 +206,8 @@ where .open(M::METHOD_ID, headers) .await .map_err(transport_to_status)?; - let (mut send, recv) = WireStream::split(stream); - - let mut buf = BytesMut::with_capacity(request.encoded_len()); - request - .encode(&mut buf) - .map_err(|e| WireStatus::internal(format!("prost encode: {e}")))?; - send.send_body(buf.freeze()) - .await - .map_err(|e| WireStatus::internal(format!("send: {e}")))?; - send.close_send(None) - .await - .map_err(|e| WireStatus::internal(format!("close_send: {e}")))?; + let (mut send, mut recv) = WireStream::split(stream); + send_request(&mut send, &mut recv, request).await?; Ok(StreamingCall::new(recv, operation)) }, @@ -988,7 +1003,22 @@ async fn write_streaming_response( mut responses: impl futures_util::Stream> + Unpin, call: &CallSpan, ) -> Result<(), TransportError> { - while let Some(response) = responses.next().await { + loop { + let next = match futures_util::future::select( + Box::pin(responses.next()), + Box::pin(send.stopped()), + ) + .await + { + futures_util::future::Either::Left((next, _)) => next, + futures_util::future::Either::Right(_) => { + call.finish(WireCode::Cancelled); + return Err(TransportError::Io("response consumer disconnected".into())); + } + }; + let Some(response) = next else { + break; + }; let response = match response { Ok(response) => response, Err(status) => return write_trailer(send, status.into(), call).await, @@ -1281,6 +1311,35 @@ mod streaming_call_tests { x: u32, } + #[tokio::test] + async fn stopped_consumer_cancels_an_idle_response_stream() { + struct StoppedSend; + impl SendHalf for StoppedSend { + type Error = std::io::Error; + async fn send_body(&mut self, _: Bytes) -> Result<(), Self::Error> { + panic!("no response available"); + } + async fn close_send(&mut self, _: Option) -> Result<(), Self::Error> { + panic!("cancelled transport"); + } + fn reset(&mut self, _: WireCode) {} + async fn stopped(&mut self) {} + } + let mut send = StoppedSend; + let call = CallSpan::new(tracing::Span::none()); + let result = tokio::time::timeout( + std::time::Duration::from_secs(1), + write_streaming_response( + &mut send, + futures_util::stream::pending::>(), + &call, + ), + ) + .await + .expect("idle cancellation must not wait for another response"); + assert!(result.is_err()); + } + #[derive(Clone, PartialEq, ::prost::Message)] struct BytesMsg { #[prost(bytes = "vec", tag = "1")] @@ -1448,6 +1507,21 @@ mod streaming_call_tests { struct MockTransport; + struct ReplyTransport(Mutex>); + + impl StreamTransport for ReplyTransport { + type Stream = MockWireStream; + type Error = std::io::Error; + + async fn open(&self, _: u32, _: Metadata) -> Result { + Ok(self.0.lock().unwrap().take().unwrap()) + } + + async fn accept(&self) -> Result>, Self::Error> { + Ok(None) + } + } + impl StreamTransport for MockTransport { type Stream = MockWireStream; type Error = std::io::Error; @@ -1494,6 +1568,74 @@ mod streaming_call_tests { ) } + #[tokio::test] + async fn early_refusal_survives_request_body_and_close_write_failures() { + for streaming in [false, true] { + for fail_close in [false, true] { + let (mut inbound, _) = route_inbound(None, false, None); + inbound.stream.send.state.fail_send = !fail_close; + inbound.stream.send.state.fail_close = fail_close; + let mut trailer = Trailer::from_status(WireCode::ResourceExhausted, "route full"); + trailer.metadata.insert_text("retry-after", "1"); + inbound.stream.recv.trailer = trailer; + let transport = ReplyTransport(Mutex::new(Some(inbound.stream))); + let request = BytesMsg::default(); + let result = if streaming { + server_streaming::<_, MockMethod>(&transport, request, Metadata::new()) + .await + .map(|_| ()) + } else { + unary::<_, MockMethod>(&transport, request, Metadata::new()) + .await + .map(|_| ()) + }; + let error = result.expect_err("early rejection remains a wire refusal"); + assert_eq!(error.code, WireCode::ResourceExhausted); + assert_eq!(error.message, "route full"); + assert_eq!( + error.metadata.get("retry-after").unwrap().as_text(), + Some("1") + ); + } + } + } + + #[tokio::test(start_paused = true)] + async fn failed_upload_does_not_accept_success_or_drain_a_stalled_response() { + for streaming in [false, true] { + for (body, pending) in [ + (None, false), + (Some(Bytes::from_static(b"unsolicited body")), false), + (None, true), + ] { + let (mut inbound, _) = route_inbound(body, pending, None); + inbound.stream.send.state.fail_send = true; + let transport = ReplyTransport(Mutex::new(Some(inbound.stream))); + let request = BytesMsg::default(); + let result = async { + if streaming { + server_streaming::<_, MockMethod>(&transport, request, Metadata::new()) + .await + .map(|_| ()) + } else { + unary::<_, MockMethod>(&transport, request, Metadata::new()) + .await + .map(|_| ()) + } + }; + let error = tokio::time::timeout( + UNARY_RECEIVE_TIMEOUT + std::time::Duration::from_secs(1), + result, + ) + .await + .expect("recovering an early refusal has a deadline") + .expect_err("a failed upload cannot become a successful call"); + assert_eq!(error.code, WireCode::Internal); + assert_eq!(error.message, "send: send failed"); + } + } + } + #[cfg(feature = "otel")] #[tokio::test] async fn telemetry_records_transport_protocol_and_cancellation_failures() { @@ -1502,19 +1644,6 @@ mod streaming_call_tests { use tracing::instrument::WithSubscriber; use tracing_subscriber::prelude::*; - struct ReplyTransport(Mutex>); - impl StreamTransport for ReplyTransport { - type Stream = MockWireStream; - type Error = std::io::Error; - async fn open(&self, _: u32, _: Metadata) -> Result { - Ok(self.0.lock().unwrap().take().unwrap()) - } - async fn accept( - &self, - ) -> Result>, Self::Error> { - Ok(None) - } - } let exporter = InMemorySpanExporter::default(); let provider = SdkTracerProvider::builder() .with_simple_exporter(exporter.clone()) diff --git a/crates/rpc/src/fetch.rs b/crates/rpc/src/fetch.rs index 93a67c40..24f55e41 100644 --- a/crates/rpc/src/fetch.rs +++ b/crates/rpc/src/fetch.rs @@ -11,8 +11,8 @@ use k256::elliptic_curve::rand_core::{OsRng, RngCore}; const INPUT_CANONICALIZATION: &[u8] = b"hellas.fetch.input.v3"; const OUTPUT_CANONICALIZATION: &[u8] = b"hellas.fetch.output.v2"; -const OUTPUT_EVENT_KIND: &str = "response.event"; -const OUTPUT_TERMINAL_KIND: &str = "response.terminal"; +pub const OUTPUT_EVENT_KIND: &str = "response.event"; +pub const OUTPUT_TERMINAL_KIND: &str = "response.terminal"; const INPUT_EVENT_KINDS: [&str; 8] = [ "assurance", "execution.environment", @@ -29,18 +29,18 @@ const INPUT_EVENT_KINDS: [&str; 8] = [ /// /// Each envelope has its own wire frame; this bound limits retained state and /// transcript verification work rather than the size of a terminal frame. -pub const MAX_FETCH_OUTPUT_EVENTS: usize = 4_096; +pub const MAX_FETCH_OUTPUT_EVENTS: usize = 32_768; /// Maximum cumulative payload bytes across all signed Fetch output envelopes, /// including the terminal payload. -pub const MAX_FETCH_OUTPUT_PAYLOAD_BYTES: usize = 2 * 1024 * 1024; +pub const MAX_FETCH_OUTPUT_PAYLOAD_BYTES: usize = 16 * 1024 * 1024; /// Maximum UTF-8 JSON bytes in the signed Fetch `request.body` event. /// /// This is checked before callers allocate/sign a transcript and again after /// verifiers authenticate it, so every Fetch implementation shares the same /// v0.0.1 admission bound. -pub const MAX_FETCH_REQUEST_BODY_BYTES: usize = 1024 * 1024; +pub const MAX_FETCH_REQUEST_BODY_BYTES: usize = 2 * 1024 * 1024; /// Maximum UTF-8 bytes in each signed Fetch route component (`service` and /// `method`). diff --git a/crates/rpc/src/fetch/tests.rs b/crates/rpc/src/fetch/tests.rs index d159b448..9132a8a4 100644 --- a/crates/rpc/src/fetch/tests.rs +++ b/crates/rpc/src/fetch/tests.rs @@ -148,6 +148,13 @@ fn request_body_limit_accepts_the_exact_boundary() { .unwrap(); assert_eq!(verify_input_events(&events).unwrap().body.as_bytes(), body); + let request = crate::pb::fetch::FetchRequest { + input: events + .iter() + .map(crate::stream::input_event_to_pb) + .collect(), + }; + assert!(prost::Message::encoded_len(&request) < hellas_wire::frame::MAX_FRAME_BYTES); } #[test] diff --git a/crates/rpc/src/http_fetch.rs b/crates/rpc/src/http_fetch.rs new file mode 100644 index 00000000..0ac65911 --- /dev/null +++ b/crates/rpc/src/http_fetch.rs @@ -0,0 +1,386 @@ +//! Caller-signed HTTPS request vocabulary. Provider-owned credentials are selected +//! by alias; their secret values stay with the provider. + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use serde::{Deserialize, Serialize}; + +pub const MAX_HTTP_RESPONSE_BYTES: u32 = 8 * 1024 * 1024; +pub const MAX_HTTP_HEADERS_BYTES: usize = 16 * 1024; +pub const MAX_HTTP_HEADERS: usize = 64; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpFetchRequest { + pub url: String, + pub method: String, + #[serde(default)] + pub headers: Vec<(String, String)>, + #[serde(default)] + pub body_base64: String, + pub tls: HttpTls, + #[serde(default)] + pub credential: Option, + pub max_response_bytes: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpTls { + pub roots: HttpTrustRoots, + /// Additional SHA-256 pins over the leaf certificate's DER SPKI. + #[serde(default)] + pub spki_sha256: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)] +pub enum HttpTrustRoots { + WebPki, + /// Exact base64 DER trust anchors, without implicit system roots. + Certificates { + der_base64: Vec, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid HTTPS Fetch request: {0}")] +pub struct HttpRequestError(pub &'static str); + +impl HttpFetchRequest { + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > crate::fetch::MAX_FETCH_REQUEST_BODY_BYTES { + return Err(HttpRequestError("request exceeds byte limit")); + } + let request: Self = + serde_json::from_slice(bytes).map_err(|_| HttpRequestError("request schema"))?; + request.validate()?; + Ok(request) + } + + pub fn validate(&self) -> Result<(), HttpRequestError> { + self.parsed_url()?; + if !matches!( + self.method.as_str(), + "GET" | "HEAD" | "POST" | "PUT" | "PATCH" | "DELETE" | "OPTIONS" + ) { + return Err(HttpRequestError("HTTP method")); + } + check_headers(&self.headers, true)?; + if self.max_response_bytes == 0 || self.max_response_bytes > MAX_HTTP_RESPONSE_BYTES { + return Err(HttpRequestError("response byte limit")); + } + self.body()?; + if self.credential.as_ref().is_some_and(|alias| { + alias.is_empty() + || alias.len() > 128 + || !alias + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b)) + }) { + return Err(HttpRequestError("credential alias")); + } + self.tls.validate() + } + + pub fn parsed_url(&self) -> Result { + if self.url.len() > 8192 { + return Err(HttpRequestError("URL length")); + } + let url = url::Url::parse(&self.url).map_err(|_| HttpRequestError("URL"))?; + if url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.fragment().is_some() + { + return Err(HttpRequestError( + "URL must be HTTPS without userinfo or fragment", + )); + } + Ok(url) + } + + pub fn body(&self) -> Result, HttpRequestError> { + if self.body_base64.len() > crate::fetch::MAX_FETCH_REQUEST_BODY_BYTES { + return Err(HttpRequestError("body length")); + } + decode_base64(&self.body_base64) + } +} + +impl HttpTls { + pub fn validate(&self) -> Result<(), HttpRequestError> { + if let HttpTrustRoots::Certificates { der_base64 } = &self.roots { + if der_base64.is_empty() || der_base64.len() > 16 { + return Err(HttpRequestError("trust anchor count")); + } + for der in der_base64 { + if der.is_empty() || der.len() > 32768 { + return Err(HttpRequestError("trust anchor size")); + } + decode_base64(der)?; + } + } + if self.spki_sha256.len() > 16 { + return Err(HttpRequestError("SPKI pin count")); + } + for pin in &self.spki_sha256 { + decode_pin(pin)?; + } + Ok(()) + } +} + +pub fn decode_pin(pin: &str) -> Result<[u8; 32], HttpRequestError> { + if pin.len() != 64 { + return Err(HttpRequestError("SPKI pin must be 64 lowercase hex digits")); + } + let mut out = [0; 32]; + for (i, pair) in pin.as_bytes().chunks_exact(2).enumerate() { + let digit = |b: u8| match b { + b'0'..=b'9' => Ok(b - b'0'), + b'a'..=b'f' => Ok(b - b'a' + 10), + _ => Err(HttpRequestError("SPKI pin encoding")), + }; + out[i] = digit(pair[0])? * 16 + digit(pair[1])?; + } + Ok(out) +} + +pub fn decode_base64(encoded: &str) -> Result, HttpRequestError> { + let bytes = STANDARD + .decode(encoded) + .map_err(|_| HttpRequestError("base64 encoding"))?; + if STANDARD.encode(&bytes) != encoded { + return Err(HttpRequestError("noncanonical base64")); + } + Ok(bytes) +} + +pub fn check_headers(headers: &[(String, String)], request: bool) -> Result<(), HttpRequestError> { + if headers.len() > MAX_HTTP_HEADERS + || headers + .iter() + .map(|(k, v)| k.len().saturating_add(v.len())) + .sum::() + > MAX_HTTP_HEADERS_BYTES + { + return Err(HttpRequestError("header limit")); + } + for (name, value) in headers { + if name.is_empty() + || !name.bytes().all(|b| { + b.is_ascii_lowercase() + || b.is_ascii_digit() + || b == 0x60 + || b"!#$%&'*+-.^_|~".contains(&b) + }) + || !value + .bytes() + .all(|b| b == b'\t' || (0x20..=0x7e).contains(&b)) + { + return Err(HttpRequestError("header encoding")); + } + if request + && matches!( + name.as_str(), + "host" + | "content-length" + | "transfer-encoding" + | "connection" + | "keep-alive" + | "upgrade" + | "proxy-authenticate" + | "proxy-authorization" + | "proxy-connection" + | "te" + | "trailer" + ) + { + return Err(HttpRequestError("reserved request header")); + } + } + Ok(()) +} + +/// A complete, structurally checked response reconstructed from authenticated +/// Fetch output. Callers verify signatures and the expected producer first. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HttpFetchResponse { + pub status: u16, + pub headers: Vec<(String, String)>, + pub body: Vec, +} + +impl HttpFetchResponse { + pub fn from_output( + request: &HttpFetchRequest, + output: &crate::fetch::FetchOutput, + ) -> Result { + use crate::output::{AdaptorEvent, HttpResponseEvent, OutputEvent, StopReason}; + let (events, terminal) = output.output_event_payloads(); + let mut response = None; + for payload in events { + let event = crate::fetch::decode_fetch_event_payload(payload) + .map_err(|_| HttpRequestError("response event codec"))?; + match event { + OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Head { + status, + headers, + })) if response.is_none() => { + // Only a final status can answer the job: an interim 1xx + // is a hop artifact, not a completed paid response. + if !(200..=599).contains(&status) { + return Err(HttpRequestError("response status")); + } + check_headers(&headers, false)?; + response = Some(Self { + status, + headers, + body: Vec::new(), + }); + } + OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Body { base64 })) => { + let response = response + .as_mut() + .ok_or(HttpRequestError("response body before head"))?; + let bytes = decode_base64(&base64)?; + if bytes.is_empty() + || response.body.len().saturating_add(bytes.len()) + > request.max_response_bytes as usize + { + return Err(HttpRequestError("response body size")); + } + response.body.extend_from_slice(&bytes); + } + _ => return Err(HttpRequestError("unexpected or duplicate response event")), + } + } + if !matches!( + crate::fetch::decode_fetch_terminal_payload(terminal), + Ok(crate::fetch::FetchTerminalPayload::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + .. + }) + ) { + return Err(HttpRequestError("HTTP terminal")); + } + response.ok_or(HttpRequestError("missing response head")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fetch::{ + FetchOutputTranscriptBuilder, encode_fetch_event_payload, encode_fetch_terminal_payload, + verify_output_events, + }; + use crate::output::{AdaptorEvent, HttpResponseEvent, OutputEvent, StopReason}; + use crate::{Assurance, Digest, InputCommitment, ProducerSigningKey}; + + fn request() -> HttpFetchRequest { + HttpFetchRequest { + url: "https://example.com/v1/messages".into(), + method: "POST".into(), + headers: vec![], + body_base64: "e30=".into(), + tls: HttpTls { + roots: HttpTrustRoots::WebPki, + spki_sha256: vec![], + }, + credential: None, + max_response_bytes: 3, + } + } + fn output(events: Vec) -> crate::fetch::FetchOutput { + let key = ProducerSigningKey::from_secret_bytes([1; 32]).unwrap(); + let input = InputCommitment::from_digest(Digest::from_bytes([2; 32])); + let mut builder = FetchOutputTranscriptBuilder::new(input, Assurance::ProducerSigned, &key); + for event in events { + builder + .push_event( + encode_fetch_event_payload(&OutputEvent::Adaptor(AdaptorEvent::Http(event))) + .unwrap(), + ) + .unwrap(); + } + let events = builder + .finish( + encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .unwrap(), + ) + .unwrap(); + verify_output_events(input, Assurance::ProducerSigned, &events).unwrap() + } + fn head() -> HttpResponseEvent { + HttpResponseEvent::Head { + status: 200, + headers: vec![], + } + } + fn body(bytes: &[u8]) -> HttpResponseEvent { + HttpResponseEvent::Body { + base64: STANDARD.encode(bytes), + } + } + #[test] + fn even_signed_http_responses_must_obey_structure_and_request_limit() { + let good = output(vec![head(), body(&[0, 255, 3])]); + assert_eq!( + HttpFetchResponse::from_output(&request(), &good) + .unwrap() + .body, + vec![0, 255, 3] + ); + for events in [ + vec![], + vec![body(b"a"), head()], + vec![head(), head()], + vec![head(), body(b"abcd")], + vec![ + head(), + HttpResponseEvent::Body { + base64: "!!".into(), + }, + ], + ] { + assert!(HttpFetchResponse::from_output(&request(), &output(events)).is_err()); + } + } + #[test] + fn requests_reject_ambiguous_or_unbounded_transport_inputs() { + for url in [ + "http://example.com/", + "https://user:secret@example.com/", + "https://example.com/#fragment", + ] { + let mut input = request(); + input.url = url.into(); + assert!(input.validate().is_err()); + } + for name in [ + "host", + "content-length", + "transfer-encoding", + "proxy-authorization", + ] { + let mut input = request(); + input.headers = vec![(name.into(), "x".into())]; + assert!(input.validate().is_err()); + } + let mut input = request(); + input.headers = vec![("authorization".into(), "x\r\ny".into())]; + assert!(input.validate().is_err()); + input = request(); + input.max_response_bytes = MAX_HTTP_RESPONSE_BYTES + 1; + assert!(input.validate().is_err()); + input = request(); + input.body_base64 = "e30".into(); + assert!(input.validate().is_err()); + } +} diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index 1a10184b..76638aed 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -47,6 +47,8 @@ pub mod edge_index; pub mod evaluate; #[cfg(feature = "fetch")] pub mod fetch; +#[cfg(feature = "fetch")] +pub mod http_fetch; pub mod observe; #[cfg(feature = "execute")] pub mod open; diff --git a/crates/rpc/src/output.rs b/crates/rpc/src/output.rs index 3e5610a9..0f024ed4 100644 --- a/crates/rpc/src/output.rs +++ b/crates/rpc/src/output.rs @@ -52,6 +52,18 @@ impl OutputEvent { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum AdaptorEvent { CodexResponses(CodexResponsesEvent), + /// Exact HTTP metadata and bytes, without LLM-specific projection. + Http(HttpResponseEvent), +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum HttpResponseEvent { + Head { + status: u16, + headers: Vec<(String, String)>, + }, + /// Standard padded base64, preserving arbitrary binary bodies. + Body { base64: String }, } /// The subset of Codex Responses streaming semantics committed by the sealed diff --git a/crates/rpc/src/pb/mod.rs b/crates/rpc/src/pb/mod.rs index 4bf86163..083cb722 100644 --- a/crates/rpc/src/pb/mod.rs +++ b/crates/rpc/src/pb/mod.rs @@ -188,12 +188,16 @@ mod id_pins { #[test] fn work_ids_are_stable() { use super::services::work::{ - AcceptWork, AdmitCertificate, DeliverResult, StreamResult, Work, + AcceptWork, AdmitCertificate, DeliverResult, Open, StreamResult, Work, }; - assert_eq!(::SERVICE_ID, 0x1a6d_285a); + // Fetch's separate terminal frame extends WorkStreamEvent, deliberately + // changing StreamResult and its enclosing service. Other method IDs stay + // pinned: gateways and providers must upgrade together for this schema. + assert_eq!(::SERVICE_ID, 0x22b5_f1f8); + assert_eq!(::METHOD_ID, 0x93cb0b39); assert_eq!(::METHOD_ID, 0xe6a7_13c2); assert_eq!(::METHOD_ID, 0xf15a_a80e); - assert_eq!(::METHOD_ID, 0xf9e8_388a); + assert_eq!(::METHOD_ID, 0x76bf_afcf); assert_eq!(::METHOD_ID, 0x0ffb_b4f9); } @@ -207,8 +211,9 @@ mod id_pins { #[cfg(feature = "work")] #[test] fn work_setup_ids_are_stable() { - use super::services::work_setup::{ExchangeSetup, WorkSetup}; - assert_eq!(::SERVICE_ID, 0x0235_ddf0); + use super::services::work_setup::{ExchangeSetup, Open, WorkSetup}; + assert_eq!(::SERVICE_ID, 0x3ed2cd2f); + assert_eq!(::METHOD_ID, 0xcbe4ebd5); assert_eq!(::METHOD_ID, 0x1cde_46e8); } diff --git a/crates/rpc/src/protocol/artifacts.rs b/crates/rpc/src/protocol/artifacts.rs index d249e4e4..6115df58 100644 --- a/crates/rpc/src/protocol/artifacts.rs +++ b/crates/rpc/src/protocol/artifacts.rs @@ -786,11 +786,7 @@ impl PreparedPaidInputV1 { bytes.len() ))); } - let mut reader = BundleReader { - bytes, - offset: 0, - budget, - }; + let mut reader = BundleReader::new(bytes, budget); let bundle = Self { evaluate_request: reader.body("evaluate_request")?, manifest: reader.body("manifest")?, @@ -834,14 +830,39 @@ impl PreparedPaidInputV1 { } } -struct BundleReader<'a> { +/// Cursor over a length-prefixed bundle's bodies. +/// +/// Shared with the fetch profile's two-body bundle +/// ([`super::work_fetch::PreparedPaidFetchInputV1`]) so the two rules that +/// keep a declared length honest — never past the input, never past the +/// budget — exist once. The error messages name "prepared input"; the +/// fetch bundle is one. +pub(crate) struct BundleReader<'a> { bytes: &'a [u8], offset: usize, budget: usize, } -impl BundleReader<'_> { - fn body(&mut self, field: &'static str) -> Result, CanonicalDecodeError> { +impl<'a> BundleReader<'a> { + pub(crate) const fn new(bytes: &'a [u8], budget: usize) -> Self { + Self { + bytes, + offset: 0, + budget, + } + } + + /// Bytes consumed so far; the caller compares it against the input + /// length to refuse trailing bytes. + /// + /// Used by the fetch profile's bundle; the evaluate bundle reads the + /// field directly, one module over. + #[cfg(feature = "work")] + pub(crate) const fn offset(&self) -> usize { + self.offset + } + + pub(crate) fn body(&mut self, field: &'static str) -> Result, CanonicalDecodeError> { let start = self .offset .checked_add(LENGTH_PREFIX) diff --git a/crates/rpc/src/protocol/fetch_environment.rs b/crates/rpc/src/protocol/fetch_environment.rs index d3f1494f..be4e213a 100644 --- a/crates/rpc/src/protocol/fetch_environment.rs +++ b/crates/rpc/src/protocol/fetch_environment.rs @@ -11,8 +11,9 @@ //! upstream credentials, and access policy remain provider-local and are //! deliberately absent from this root. //! Selecting the upstream destination is trusted execution semantics, so every -//! built-in root commits an exact HTTPS endpoint and driver contract. The -//! service at that endpoint and every response claim remain adversarial input. +//! built-in root commits a driver contract. Sealed roots fix the endpoint; the +//! HTTP root interprets the URL and TLS policy in the caller-signed request. +//! The upstream service and every response claim remain adversarial input. use crate::{Application, ContentId, DagCborEncoder, ProgramManifest}; @@ -41,11 +42,13 @@ const FETCH_ENVIRONMENT_DOMAIN: &str = "hellas.fetch.environment.v1"; /// An exact built-in request/response transformation and its trusted config. /// -/// The first version has no configurable trusted inputs: its complete -/// behaviour is compiled into the attested application. Adding trusted config -/// later requires a new canonical variant (and therefore a new manifest ID). +/// Each variant fixes the interpreter compiled into the attested application. +/// The HTTP interpreter binds request-selected destinations and TLS settings +/// through the signed input; it does not enumerate upstream vendors. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FetchEnvironment { + /// HTTPS to the signed URL with selected trust roots and optional SPKI pins. + Http, /// Build an official Codex Responses request, send it only to the fixed /// ChatGPT Codex endpoint, and project its SSE response. CodexResponses, @@ -61,6 +64,7 @@ impl FetchEnvironment { #[must_use] pub const fn adaptor(self) -> &'static str { match self { + Self::Http => "http-0.0.1", Self::CodexResponses => CODEX_RESPONSES_ADAPTOR, Self::OpenAiResponses => OPENAI_RESPONSES_ADAPTOR, } @@ -70,6 +74,7 @@ impl FetchEnvironment { #[must_use] pub const fn driver(self) -> &'static str { match self { + Self::Http => "https-request-roots-and-spki-no-redirect-0.0.1", Self::CodexResponses => CODEX_RESPONSES_DRIVER, Self::OpenAiResponses => OPENAI_RESPONSES_DRIVER, } @@ -79,6 +84,7 @@ impl FetchEnvironment { #[must_use] pub const fn endpoint(self) -> &'static str { match self { + Self::Http => "caller-signed:https-url", Self::CodexResponses => CODEX_RESPONSES_ENDPOINT, Self::OpenAiResponses => OPENAI_RESPONSES_ENDPOINT, } @@ -111,6 +117,11 @@ impl FetchEnvironment { decoder.finish()?; let environment = match (adaptor, driver, endpoint) { + ( + "http-0.0.1", + "https-request-roots-and-spki-no-redirect-0.0.1", + "caller-signed:https-url", + ) => Self::Http, (CODEX_RESPONSES_ADAPTOR, CODEX_RESPONSES_DRIVER, CODEX_RESPONSES_ENDPOINT) => { Self::CodexResponses } diff --git a/crates/rpc/src/protocol/mod.rs b/crates/rpc/src/protocol/mod.rs index 6b3b0d9b..51adeea7 100644 --- a/crates/rpc/src/protocol/mod.rs +++ b/crates/rpc/src/protocol/mod.rs @@ -20,6 +20,10 @@ pub mod work; #[cfg(feature = "work")] pub mod work_bundle; #[cfg(feature = "work")] +pub mod work_fetch; +#[cfg(feature = "work")] +pub mod work_profile; +#[cfg(feature = "work")] pub mod work_setup; pub use causal_lm::{ diff --git a/crates/rpc/src/protocol/work.rs b/crates/rpc/src/protocol/work.rs index e0a254e1..1a72faf0 100644 --- a/crates/rpc/src/protocol/work.rs +++ b/crates/rpc/src/protocol/work.rs @@ -72,13 +72,16 @@ const FORMAT_VERSION: u8 = 1; /// /// These are RPC-protocol tags. They are not kernel canonical tags and /// they are never accepted L1 bytes; the numbers are local to this -/// module and shared only between the two endpoints. -mod tag { - pub(super) const PAID_CHANNEL_POLICY: u8 = 0; - pub(super) const PAID_EXECUTION_POLICY: u8 = 1; - pub(super) const PAID_JOB_AUTHORIZATION: u8 = 2; - pub(super) const PAID_JOB_RESULT: u8 = 3; - pub(super) const PAYMENT_BINDING: u8 = 4; +/// module and shared only between the two endpoints. The fetch profile's +/// policy record ([`super::work_fetch`]) takes its tag from this same +/// registry, so two profiles can never assign one number. +pub(crate) mod tag { + pub(crate) const PAID_CHANNEL_POLICY: u8 = 0; + pub(crate) const PAID_EXECUTION_POLICY: u8 = 1; + pub(crate) const PAID_JOB_AUTHORIZATION: u8 = 2; + pub(crate) const PAID_JOB_RESULT: u8 = 3; + pub(crate) const PAYMENT_BINDING: u8 = 4; + pub(crate) const PAID_FETCH_POLICY: u8 = 5; } /// Bytes the envelope occupies: `format_version:u8 || record_tag:u8`. @@ -564,11 +567,11 @@ impl PrivateRecord for PaymentBindingV1 { // ── Body reading and writing ────────────────────────────────────────── -fn put_u32(out: &mut Vec, value: u32) { +pub(crate) fn put_u32(out: &mut Vec, value: u32) { out.extend_from_slice(&value.to_be_bytes()); } -fn put_u64(out: &mut Vec, value: u64) { +pub(crate) fn put_u64(out: &mut Vec, value: u64) { out.extend_from_slice(&value.to_be_bytes()); } @@ -597,7 +600,7 @@ impl BodyReader<'_> { Ok(out) } - fn bytes32(&mut self) -> Result<[u8; 32], PaidWorkError> { + pub(crate) fn bytes32(&mut self) -> Result<[u8; 32], PaidWorkError> { self.take::<32>() } @@ -605,11 +608,11 @@ impl BodyReader<'_> { self.take::<2>().map(u16::from_be_bytes) } - fn u32(&mut self) -> Result { + pub(crate) fn u32(&mut self) -> Result { self.take::<4>().map(u32::from_be_bytes) } - fn u64(&mut self) -> Result { + pub(crate) fn u64(&mut self) -> Result { self.take::<8>().map(u64::from_be_bytes) } } @@ -623,13 +626,13 @@ impl BodyReader<'_> { /// digests are the same bytes inside a kernel authorization hash. The /// length prefix is why a two-character id followed by a channel cannot /// hash like a one-character id followed by a different one. -struct EncodedNetwork { +pub(crate) struct EncodedNetwork { bytes: [u8; ::MAX_ENCODED_SIZE], len: usize, } impl EncodedNetwork { - fn new(network: NetworkId) -> Self { + pub(crate) fn new(network: NetworkId) -> Self { let mut bytes = [0_u8; ::MAX_ENCODED_SIZE]; let mut writer = BufferWriter::new(&mut bytes); network.encode_to(&mut writer); @@ -646,7 +649,7 @@ impl EncodedNetwork { /// zero-padded buffer — and every digest in this module would move /// silently under it. A bug that cannot happen should not have a /// second answer ready. - fn as_slice(&self) -> &[u8] { + pub(crate) fn as_slice(&self) -> &[u8] { &self.bytes[..self.len] } } @@ -657,7 +660,7 @@ impl EncodedNetwork { /// Every caller's complete preimage is bounded at compile time by an /// assertion below, because [`SingleChunkHasher::update`] panics rather /// than erroring once a preimage reaches [`MIN_CHUNK_SIZE`]. -fn xh(domain: &[u8], fields: &[&[u8]]) -> Digest { +pub(crate) fn xh(domain: &[u8], fields: &[&[u8]]) -> Digest { let mut hasher = SingleChunkHasher::new(); hasher.update(domain); for field in fields { @@ -672,7 +675,7 @@ fn xh(domain: &[u8], fields: &[&[u8]]) -> Digest { /// one-shot [`Digest::hash`] of the same concatenation under every write /// segmentation, which is what lets a caller stream a large body without /// holding it. -fn xfh(domain: &[u8], fields: &[&[u8]]) -> Digest { +pub(crate) fn xfh(domain: &[u8], fields: &[&[u8]]) -> Digest { let mut hasher = XetFileHasher::new(); hasher.update(domain); for field in fields { @@ -844,7 +847,7 @@ impl PaidChannel { self.payment_terms.parties().taker() } - fn network_bytes(&self) -> EncodedNetwork { + pub(crate) fn network_bytes(&self) -> EncodedNetwork { EncodedNetwork::new(self.network) } } @@ -941,7 +944,7 @@ pub fn identity_source_digest( /// separates nothing. A body that large is unreachable through any /// bounded decoder here; it is refused rather than truncated because a /// truncated prefix is a second legal spelling of the same bytes. -fn length_prefix(bytes: &[u8], field: &'static str) -> Result<[u8; 4], PaidWorkError> { +pub(crate) fn length_prefix(bytes: &[u8], field: &'static str) -> Result<[u8; 4], PaidWorkError> { let len = u32::try_from(bytes.len()).map_err(|_| PaidWorkError::Overflow { field })?; Ok(len.to_be_bytes()) } @@ -1252,7 +1255,34 @@ pub fn check_authorization( finalized_height: u64, ) -> Result { check_execution_policy(policy)?; + check_authorization_core( + channel, + authorization, + execution_policy_digest(channel, policy), + policy.allowed_environment, + policy.fixed_price, + finalized_height, + ) +} +/// The authorization arithmetic every paid-work profile shares. +/// +/// The profiles differ in what a policy *is* — the evaluate profile's is +/// [`PaidExecutionPolicyV1`], the fetch profile's is in +/// [`super::work_fetch`] — but not in what signing one *means*: the same +/// channel fields, the same bond cover, the same credit-limit cover, the +/// same deadline window. That part is written once here so the two +/// profiles cannot drift into two different definitions of acceptance. +/// What each caller supplies is its own policy digest, environment, and +/// price; what this function supplies is everything else. +pub(crate) fn check_authorization_core( + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + policy_digest: Digest, + allowed_environment: ContentId, + fixed_price: u64, + finalized_height: u64, +) -> Result { let terms = channel.payment_terms(); let expected = [ ( @@ -1277,13 +1307,11 @@ pub fn check_authorization( ), ( "execution_policy_digest", - authorization.execution_policy_digest.as_bytes() - == execution_policy_digest(channel, policy).as_bytes(), + authorization.execution_policy_digest.as_bytes() == policy_digest.as_bytes(), ), ( "environment_commitment", - authorization.environment_commitment.as_bytes() - == policy.allowed_environment.as_bytes(), + authorization.environment_commitment.as_bytes() == allowed_environment.as_bytes(), ), ]; for (field, holds) in expected { @@ -1299,11 +1327,11 @@ pub fn check_authorization( max_job_price, }); } - if authorization.price != policy.fixed_price { + if authorization.price != fixed_price { return Err(PaidWorkError::Mismatch { field: "price" }); } - // The execution policy is per-authorization and only its digest is + // The policy is per-authorization and only its digest is // signed, so a channel may see many of them. The credit limits are // per-channel and are opened once, at construction. That is why the // two are compared here, against this job's price, rather than once diff --git a/crates/rpc/src/protocol/work_fetch.rs b/crates/rpc/src/protocol/work_fetch.rs new file mode 100644 index 00000000..b88c245e --- /dev/null +++ b/crates/rpc/src/protocol/work_fetch.rs @@ -0,0 +1,766 @@ +//! Paid Fetch binds a caller-signed request and manifest to a channel authorization. +//! Results must verify under the requested assurance and the channel's provider key. +//! Fetch policies use record tag 5 and separate digest domains from Evaluate. + +use hellas_kernel::{Encode, NetworkId}; +use hellas_xet::{MIN_CHUNK_SIZE, XetFileHasher}; + +use crate::fetch::{FetchProtocolError, MAX_FETCH_ROUTE_COMPONENT_BYTES}; +use crate::protocol::artifacts::BundleReader; +use crate::protocol::value::{ + CanonicalDecodeError, CanonicalDecoder, canonical_dag_cbor, decode_canonical_dag_cbor, +}; +use crate::protocol::work::{ + BodyReader, EncodedNetwork, JobDeadlines, PaidChannel, PaidJobAuthorizationV1, PaidJobResultV1, + PaidWorkError, PrivateRecord, check_authorization_core, length_prefix, put_u32, put_u64, tag, + work_id, xfh, xh, +}; +use crate::{ + Assurance, ContentId, Digest, InputCommitment, InputEventEnvelope, OutputEventEnvelope, + ProgramManifest, PublicKey, RequestCommitment, +}; + +// Digest domains are part of the wire contract. + +/// Commitment to the canonical route-policy body. +const FETCH_ROUTE: &[u8] = b"hellas.work.fetch-route.v1"; +/// Commitment to the per-channel fetch execution policy. +const FETCH_POLICY: &[u8] = b"hellas.work.fetch-policy.v1"; +/// Commitment to the prepared fetch input bundle. +const PREPARED_FETCH_INPUT: &[u8] = b"hellas.work.prepared-fetch-input.v1"; +/// The normalized fetch answer authenticated by the signed transcript. +const FETCH_OUTPUT: &[u8] = b"hellas.work.fetch-output.v1"; + +/// RAM bound for a complete Fetch transcript, including signed envelopes. +/// Wire frames remain independently bounded; Fetch journals omit this payload. +pub const MAX_FETCH_TRANSCRIPT_BYTES: usize = 32 * 1024 * 1024; + +// ── The route policy ────────────────────────────────────────────────── + +/// Schema tag of the sealed-route body. +const SEALED_ROUTE_SCHEMA: &str = "hellas.work.fetch-route.sealed.v1"; +/// Schema tag of the open-fetch body. +const OPEN_FETCH_SCHEMA: &str = "hellas.work.fetch-route.open.v1"; + +/// Canonical route policy. Its variable-length body is committed by digest in +/// the fixed-width `PaidFetchPolicyV1` record and opened during input validation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FetchRoutePolicy { + /// Requires the signed request to use this service and method. + SealedRoute { + /// The signed `service` component, e.g. `openai`. + service: String, + /// The signed `method` component, e.g. `responses`. + method: String, + }, + /// Caller-signed HTTPS URLs under the generic HTTP manifest. Host and + /// required-pin constraints are checked before paid acceptance; the driver + /// enforces normal certificate validation, DNS/address checks and no redirects. + /// An empty host list permits any public host under the operator's egress policy. + OpenFetch { + /// Whether the driver must pin the upstream's SPKI. + require_spki_pin: bool, + /// Hosts the driver may egress to; empty means any public host. + allowed_hosts: Vec, + }, +} + +impl FetchRoutePolicy { + /// Builds a route with nonempty service and method components bounded by + /// `MAX_FETCH_ROUTE_COMPONENT_BYTES`. + pub fn sealed_route( + service: impl Into, + method: impl Into, + ) -> Result { + let service = service.into(); + let method = method.into(); + if service.is_empty() { + return Err(FetchProtocolError::EmptyService); + } + if method.is_empty() { + return Err(FetchProtocolError::EmptyMethod); + } + if service.len() > MAX_FETCH_ROUTE_COMPONENT_BYTES { + return Err(FetchProtocolError::RouteComponentLimit { + field: "service", + actual: service.len(), + }); + } + if method.len() > MAX_FETCH_ROUTE_COMPONENT_BYTES { + return Err(FetchProtocolError::RouteComponentLimit { + field: "method", + actual: method.len(), + }); + } + Ok(Self::SealedRoute { service, method }) + } + + /// Sorts and deduplicates the host allowlist before commitment. Admission checks + /// exact hosts and required pins; the HTTPS driver checks certificates and addresses. + #[must_use] + pub fn open_fetch(require_spki_pin: bool, allowed_hosts: impl Into>) -> Self { + // Admission compares against Url::host_str, which is lowercase + // punycode; normalize here so a host spelled with case or padding + // can still match, and drop entries that never could. + let mut allowed_hosts: Vec = allowed_hosts + .into() + .into_iter() + .map(|host| host.trim().to_ascii_lowercase()) + .filter(|host| !host.is_empty()) + .collect(); + allowed_hosts.sort_unstable(); + allowed_hosts.dedup(); + Self::OpenFetch { + require_spki_pin, + allowed_hosts, + } + } + + /// Canonical route encoding. The pin flag is encoded as integer 0 or 1. + #[must_use] + pub fn canonical_body_bytes(&self) -> Vec { + let mut encoder = crate::DagCborEncoder::new(); + match self { + Self::SealedRoute { service, method } => { + encoder.array(3); + encoder.str(SEALED_ROUTE_SCHEMA); + encoder.str(service); + encoder.str(method); + } + Self::OpenFetch { + require_spki_pin, + allowed_hosts, + } => { + encoder.array(3); + encoder.str(OPEN_FETCH_SCHEMA); + encoder.u64(u64::from(*require_spki_pin)); + encoder.array(allowed_hosts.len() as u64); + for host in allowed_hosts { + encoder.str(host); + } + } + } + encoder.into_bytes() + } + + /// Rejects unknown schemas, invalid components, noncanonical encodings and + /// trailing bytes. Re-encoding must reproduce the input. + pub fn from_canonical_body_bytes(bytes: &[u8]) -> Result { + let mut decoder = CanonicalDecoder::new(bytes); + let len = decoder.array_len()?; + let policy = match decoder.str()? { + SEALED_ROUTE_SCHEMA => { + if len != 3 { + return Err(CanonicalDecodeError::new(format!( + "{SEALED_ROUTE_SCHEMA} expected array length 3, got {len}" + ))); + } + let service = decoder.str()?.to_string(); + let method = decoder.str()?.to_string(); + Self::sealed_route(service, method) + .map_err(|error| CanonicalDecodeError::new(error.to_string()))? + } + OPEN_FETCH_SCHEMA => { + if len != 3 { + return Err(CanonicalDecodeError::new(format!( + "{OPEN_FETCH_SCHEMA} expected array length 3, got {len}" + ))); + } + let require_spki_pin = match decoder.u64()? { + 0 => false, + 1 => true, + other => { + return Err(CanonicalDecodeError::new(format!( + "require_spki_pin must be 0 or 1, got {other}" + ))); + } + }; + let hosts = decoder.array_len()?; + let mut allowed_hosts = Vec::with_capacity(hosts); + for _ in 0..hosts { + allowed_hosts.push(decoder.str()?.to_string()); + } + Self::open_fetch(require_spki_pin, allowed_hosts) + } + other => { + return Err(CanonicalDecodeError::new(format!( + "unexpected fetch route schema tag {other:?}" + ))); + } + }; + decoder.finish()?; + if policy.canonical_body_bytes() != bytes { + return Err(CanonicalDecodeError::new( + "fetch route policy is not in canonical DAG-CBOR form", + )); + } + Ok(policy) + } +} + +/// Hashes the length-prefixed route body with the streaming hasher. +pub fn fetch_route_commitment(canonical_body_bytes: &[u8]) -> Result { + Ok(xfh( + FETCH_ROUTE, + &[ + &length_prefix(canonical_body_bytes, "fetch route policy length")?, + canonical_body_bytes, + ], + )) +} + +// ── The fetch execution policy ──────────────────────────────────────── + +/// Fixed-width Fetch policy: manifest, route digest, resource limits, timing +/// margins and fixed price. Encoded under record tag 5. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PaidFetchPolicyV1 { + /// The one fetch environment manifest this channel will run. + pub allowed_environment: ContentId, + /// Commitment to the canonical route-policy body. + pub route_commitment: Digest, + /// Largest signed `request.body` this channel accepts, in bytes. + pub max_request_body_bytes: u32, + /// Largest output transcript this channel accepts, in events. + pub max_output_events: u32, + /// Largest cumulative output payload this channel accepts, in bytes. + pub max_output_bytes: u32, + /// Largest spool the provider may retain for one job. + pub max_spool_bytes: u64, + /// Largest complete encoded result frame, transport framing + /// included. + pub max_encoded_result_frame: u32, + /// Maximum size of the complete encoded prepared-input bundle. + pub max_encoded_prepared_input: u32, + /// Blocks allowed from acceptance to durable terminal readiness. + pub dispatch_margin_blocks: u64, + /// Blocks allowed to transfer the largest legal result. + pub delivery_margin_blocks: u64, + /// Blocks allowed for reexecution, invoicing, and admission. + pub oracle_grace_blocks: u64, + /// Price of one accepted terminal result. + pub fixed_price: u64, +} + +impl PrivateRecord for PaidFetchPolicyV1 { + const TAG: u8 = tag::PAID_FETCH_POLICY; + const BODY_SIZE: usize = 2 * 32 + 3 * 4 + 8 + 2 * 4 + 4 * 8; + + fn encode_body(&self, out: &mut Vec) { + out.extend_from_slice(self.allowed_environment.as_bytes()); + out.extend_from_slice(self.route_commitment.as_bytes()); + put_u32(out, self.max_request_body_bytes); + put_u32(out, self.max_output_events); + put_u32(out, self.max_output_bytes); + put_u64(out, self.max_spool_bytes); + put_u32(out, self.max_encoded_result_frame); + put_u32(out, self.max_encoded_prepared_input); + put_u64(out, self.dispatch_margin_blocks); + put_u64(out, self.delivery_margin_blocks); + put_u64(out, self.oracle_grace_blocks); + put_u64(out, self.fixed_price); + } + + fn decode_body(reader: &mut BodyReader<'_>) -> Result { + Ok(Self { + allowed_environment: ContentId::from_bytes(reader.bytes32()?), + route_commitment: Digest::from_bytes(reader.bytes32()?), + max_request_body_bytes: reader.u32()?, + max_output_events: reader.u32()?, + max_output_bytes: reader.u32()?, + max_spool_bytes: reader.u64()?, + max_encoded_result_frame: reader.u32()?, + max_encoded_prepared_input: reader.u32()?, + dispatch_margin_blocks: reader.u64()?, + delivery_margin_blocks: reader.u64()?, + oracle_grace_blocks: reader.u64()?, + fixed_price: reader.u64()?, + }) + } +} + +/// Requires positive resource limits, timing margins and price. +pub fn check_fetch_policy(policy: &PaidFetchPolicyV1) -> Result<(), PaidWorkError> { + for (field, value) in [ + ("fixed_price", policy.fixed_price), + ( + "max_request_body_bytes", + u64::from(policy.max_request_body_bytes), + ), + ("max_output_events", u64::from(policy.max_output_events)), + ("max_output_bytes", u64::from(policy.max_output_bytes)), + ("max_spool_bytes", policy.max_spool_bytes), + ( + "max_encoded_result_frame", + u64::from(policy.max_encoded_result_frame), + ), + ( + "max_encoded_prepared_input", + u64::from(policy.max_encoded_prepared_input), + ), + ("dispatch_margin_blocks", policy.dispatch_margin_blocks), + ("delivery_margin_blocks", policy.delivery_margin_blocks), + ("oracle_grace_blocks", policy.oracle_grace_blocks), + ] { + if value == 0 { + return Err(PaidWorkError::PolicyZero { field }); + } + } + Ok(()) +} + +/// Fetch policy digest stored in the authorization's `execution_policy_digest`. +pub fn fetch_policy_digest(channel: &PaidChannel, policy: &PaidFetchPolicyV1) -> Digest { + let network_bytes = channel.network_bytes(); + xh( + FETCH_POLICY, + &[ + network_bytes.as_slice(), + channel.id().as_bytes(), + &policy.encode(), + ], + ) +} + +// ── The prepared fetch input bundle ─────────────────────────────────── + +/// Canonical signed input transcript and manifest, each prefixed by a big-endian +/// `u32` length. The original bytes are retained because the authorization commits +/// to them; `parts` verifies canonical encoding when parsing. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PreparedPaidFetchInputV1 { + fetch_input_transcript: Vec, + environment_manifest: Vec, +} + +/// Parsed canonical request transcript and manifest. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PreparedPaidFetchInputParts { + /// The signed fetch input events, strictly decoded. + pub fetch_input_transcript: Vec, + /// The strictly decoded environment manifest carried by the bundle. + pub manifest: ProgramManifest, +} + +impl PreparedPaidFetchInputV1 { + /// Encodes the signed input events and manifest into a prepared bundle. + pub fn new( + fetch_input_transcript: &[InputEventEnvelope], + environment_manifest: &ProgramManifest, + ) -> Result { + Ok(Self { + fetch_input_transcript: encode_input_transcript(fetch_input_transcript)?, + environment_manifest: environment_manifest.canonical_bytes(), + }) + } + + /// Encodes each body with a big-endian `u32` length, rejecting overflow. + pub fn encode(&self) -> Result, CanonicalDecodeError> { + let mut bytes = Vec::new(); + for body in self.bodies() { + let len = u32::try_from(body.len()).map_err(|_| { + CanonicalDecodeError::new(format!( + "prepared fetch input body is {} bytes, over the u32 length prefix", + body.len() + )) + })?; + bytes.extend_from_slice(&len.to_be_bytes()); + bytes.extend_from_slice(body); + } + Ok(bytes) + } + + /// Decodes two length-prefixed bodies within the total `budget`, rejecting + /// truncation, overflow and trailing bytes. + pub fn decode(bytes: &[u8], budget: usize) -> Result { + if bytes.len() > budget { + return Err(CanonicalDecodeError::new(format!( + "prepared fetch input is {} bytes, over the {budget}-byte budget", + bytes.len() + ))); + } + let mut reader = BundleReader::new(bytes, budget); + let bundle = Self { + fetch_input_transcript: reader.body("fetch_input_transcript")?, + environment_manifest: reader.body("environment_manifest")?, + }; + if reader.offset() != bytes.len() { + return Err(CanonicalDecodeError::new(format!( + "trailing bytes after prepared fetch input: {}", + bytes.len() - reader.offset() + ))); + } + Ok(bundle) + } + + /// Parses both bodies and requires their canonical encoding to match the + /// committed bytes. + pub fn parts(&self) -> Result { + Ok(PreparedPaidFetchInputParts { + fetch_input_transcript: decode_input_transcript(&self.fetch_input_transcript)?, + manifest: ProgramManifest::from_canonical_bytes(&self.environment_manifest)?, + }) + } + + fn bodies(&self) -> [&[u8]; 2] { + [&self.fetch_input_transcript, &self.environment_manifest] + } +} + +/// Canonical DAG-CBOR encoding of the signed input envelopes. +fn encode_input_transcript(transcript: &[InputEventEnvelope]) -> Result, PaidWorkError> { + canonical_dag_cbor(&transcript.to_vec()) + .map_err(|error| PaidWorkError::Transcript(error.to_string())) +} + +/// Reads the signed fetch input events back, accepting only the canonical +/// spelling. +fn decode_input_transcript(bytes: &[u8]) -> Result, CanonicalDecodeError> { + decode_canonical_dag_cbor(bytes) +} + +/// Hashes the complete prepared bundle, bound to the channel and network. +pub fn prepared_fetch_input_digest( + channel: &PaidChannel, + bundle: &PreparedPaidFetchInputV1, +) -> Result { + let network_bytes = channel.network_bytes(); + Ok(xfh( + PREPARED_FETCH_INPUT, + &[ + network_bytes.as_slice(), + channel.id().as_bytes(), + &bundle.encode()?, + ], + )) +} + +// ── Authorization ───────────────────────────────────────────────────── + +/// Derives authorization commitments from the verified request, policy and channel. +/// Only the nonce and deadlines are caller-selected. Admission rules are enforced +/// by `check_fetch_authorization` and `check_prepared_fetch_input`. +pub fn propose_fetch_authorization( + channel: &PaidChannel, + policy: &PaidFetchPolicyV1, + bundle: &PreparedPaidFetchInputV1, + proposal_nonce: u64, + deadlines: JobDeadlines, +) -> Result { + let terms = channel.payment_terms(); + let parts = bundle.parts()?; + let input = crate::fetch::verify_input_events(&parts.fetch_input_transcript) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + Ok(PaidJobAuthorizationV1 { + channel_id: channel.id(), + bond_edge: terms.bond_edge, + bond_terms_hash: terms.bond_terms_hash(), + payment_edge: channel.payment_edge(), + payment_terms_hash: channel.payment_terms_hash(), + execution_policy_digest: fetch_policy_digest(channel, policy), + prepared_input_digest: prepared_fetch_input_digest(channel, bundle)?, + proposal_nonce, + acceptance_deadline: deadlines.acceptance, + request_commitment: RequestCommitment::from_digest(input.input_commitment.digest()), + environment_commitment: parts.manifest.content_id(), + price: policy.fixed_price, + terminal_deadline: deadlines.terminal, + payment_deadline: deadlines.payment, + }) +} + +/// Checks channel, policy, price and deadline bounds at the finalized height; +/// returns the work ID. +pub fn check_fetch_authorization( + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + policy: &PaidFetchPolicyV1, + finalized_height: u64, +) -> Result { + check_fetch_policy(policy)?; + check_authorization_core( + channel, + authorization, + fetch_policy_digest(channel, policy), + policy.allowed_environment, + policy.fixed_price, + finalized_height, + ) +} + +/// Checks bundle bounds, commitments, caller signature, route and environment +/// against the channel policy and authorization. Open Fetch also checks its HTTP +/// schema, host allowlist and required pins. The HTTPS driver checks certificates +/// and addresses when connecting. +pub fn check_prepared_fetch_input( + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + policy: &PaidFetchPolicyV1, + route: &FetchRoutePolicy, + bundle: &PreparedPaidFetchInputV1, +) -> Result<(), PaidWorkError> { + let encoded = bundle.encode()?; + let limit = u64::from(policy.max_encoded_prepared_input); + let actual = u64::try_from(encoded.len()).map_err(|_| PaidWorkError::Overflow { + field: "prepared input length", + })?; + if actual > limit { + return Err(PaidWorkError::OverEnvelope { + field: "prepared input length", + actual, + limit, + }); + } + if prepared_fetch_input_digest(channel, bundle)?.as_bytes() + != authorization.prepared_input_digest.as_bytes() + { + return Err(PaidWorkError::Mismatch { + field: "prepared_input_digest", + }); + } + + let parts = bundle.parts()?; + let input = crate::fetch::verify_input_events(&parts.fetch_input_transcript) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + + // The signing key must also be the channel's client key. + if input.caller_key != PublicKey::Secp256k1(channel.client_key().to_bytes()) { + return Err(PaidWorkError::Mismatch { + field: "caller_key", + }); + } + + // The supplied route body is accounted to the signed policy before it + // is allowed to constrain anything: a route whose commitment differs + // is a route this channel never sold, whatever it claims. + if fetch_route_commitment(&route.canonical_body_bytes())?.as_bytes() + != policy.route_commitment.as_bytes() + { + return Err(PaidWorkError::Mismatch { + field: "route_commitment", + }); + } + if let FetchRoutePolicy::SealedRoute { service, method } = route { + for (field, holds) in [ + ("service", input.service == *service), + ("method", input.method == *method), + ] { + if !holds { + return Err(PaidWorkError::Mismatch { field }); + } + } + } + + if let FetchRoutePolicy::OpenFetch { + require_spki_pin, + allowed_hosts, + } = route + { + if parts.manifest.content_id() != crate::FetchEnvironment::Http.manifest_id() { + return Err(PaidWorkError::Mismatch { + field: "open-fetch HTTPS manifest", + }); + } + let request = crate::http_fetch::HttpFetchRequest::decode(input.body.as_bytes()) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; + let url = request + .parsed_url() + .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; + if (*require_spki_pin && request.tls.spki_sha256.is_empty()) + || (!allowed_hosts.is_empty() + && !allowed_hosts + .iter() + .any(|h| Some(h.as_str()) == url.host_str())) + { + return Err(PaidWorkError::Mismatch { + field: "open-fetch host or pin policy", + }); + } + // A response cap the channel can never deliver would run the + // provider's upstream fetch and refuse the terminal at payment. + // Base64 body events carry at most three raw bytes per four + // payload bytes, so that bound is decidable before admission. + let deliverable = u64::from(policy.max_output_bytes) / 4 * 3; + if u64::from(request.max_response_bytes) > deliverable { + return Err(PaidWorkError::OverEnvelope { + field: "max_response_bytes", + actual: u64::from(request.max_response_bytes), + limit: deliverable, + }); + } + } + + let graph = [ + ( + "manifest content id", + parts.manifest.content_id().as_bytes() == input.execution_environment.as_bytes(), + ), + ( + "environment_commitment", + input.execution_environment.as_bytes() + == authorization.environment_commitment.as_bytes(), + ), + ( + "allowed_environment", + parts.manifest.content_id().as_bytes() == policy.allowed_environment.as_bytes(), + ), + ( + "request_commitment", + RequestCommitment::from_digest(input.input_commitment.digest()).as_bytes() + == authorization.request_commitment.as_bytes(), + ), + ]; + for (field, holds) in graph { + if !holds { + return Err(PaidWorkError::Mismatch { field }); + } + } + + let body = u64::try_from(input.body.as_bytes().len()).map_err(|_| PaidWorkError::Overflow { + field: "request body length", + })?; + let limit = u64::from(policy.max_request_body_bytes); + if body > limit { + return Err(PaidWorkError::OverEnvelope { + field: "request body", + actual: body, + limit, + }); + } + + Ok(()) +} + +/// Enforces the channel's output limits before a result is recorded or paid. +/// Both counts include the terminal envelope, matching the Fetch wire limits. +pub fn check_fetch_output_limits( + policy: &PaidFetchPolicyV1, + transcript: &[OutputEventEnvelope], +) -> Result<(), PaidWorkError> { + let bytes = transcript.iter().try_fold(0_u64, |total, event| { + total + .checked_add(event.payload().len() as u64) + .ok_or(PaidWorkError::Overflow { + field: "fetch output bytes", + }) + })?; + for (field, actual, limit) in [ + ( + "fetch output events", + transcript.len() as u64, + u64::from(policy.max_output_events), + ), + ( + "fetch output bytes", + bytes, + u64::from(policy.max_output_bytes), + ), + ] { + if actual > limit { + return Err(PaidWorkError::OverEnvelope { + field, + actual, + limit, + }); + } + } + Ok(()) +} + +// ── The terminal result ─────────────────────────────────────────────── + +/// Hashes the concatenated canonical event and terminal payload bytes. +/// Envelope framing is bound separately by the terminal transcript commitment. +/// Changing payload encodings, including how content is split into events, can +/// change this digest. +#[must_use] +pub fn fetch_canonical_output_digest( + network: NetworkId, + work_id: Digest, + event_payloads: &[Vec], + terminal_payload: &[u8], +) -> Digest { + let network_bytes = EncodedNetwork::new(network); + let mut hasher = XetFileHasher::new(); + hasher.update(FETCH_OUTPUT); + hasher.update(network_bytes.as_slice()); + hasher.update(work_id.as_bytes()); + for payload in event_payloads { + hasher.update(payload); + } + hasher.update(terminal_payload); + hasher.finalize() +} + +/// Verifies the result using the assurance from the authenticated request and +/// requires the channel's provider key. Returns commitments to the signed +/// transcript and its payload bytes. +pub fn terminal_fetch_result( + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + transcript: &[OutputEventEnvelope], + assurance: Assurance, +) -> Result { + Ok(verify_terminal_fetch_result(channel, authorization, transcript, assurance)?.0) +} + +/// Keep the authenticated output for profile-specific checks at the same boundary. +pub(super) fn verify_terminal_fetch_result( + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + transcript: &[OutputEventEnvelope], + assurance: Assurance, +) -> Result<(PaidJobResultV1, crate::fetch::FetchOutput), PaidWorkError> { + let input = InputCommitment::from_digest(authorization.request_commitment.digest()); + let output = crate::fetch::verify_output_events(input, assurance, transcript) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + + // Bind the transcript signer to the channel's provider. + if output.producer_key != PublicKey::Secp256k1(channel.provider_key().to_bytes()) { + return Err(PaidWorkError::Mismatch { + field: "transcript producer key", + }); + } + + let Some(terminal_event) = transcript.last() else { + // Verification already requires a terminal; keep this path fallible. + return Err(PaidWorkError::Transcript( + "the terminal transcript is empty".to_string(), + )); + }; + + let (event_payloads, terminal_payload) = output.output_event_payloads(); + for payload in event_payloads { + crate::fetch::decode_fetch_event_payload(payload) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + } + crate::fetch::decode_fetch_terminal_payload(terminal_payload) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + let work_id = work_id(channel, authorization); + Ok(( + PaidJobResultV1 { + work_id, + terminal_transcript_commitment: terminal_event.event_commitment(), + canonical_output_digest: fetch_canonical_output_digest( + channel.network(), + work_id, + event_payloads, + terminal_payload, + ), + }, + output, + )) +} + +// ── Bounds ──────────────────────────────────────────────────────────── + +// The fixed policy digest must fit the single-chunk hasher. Variable bodies +// use the streaming hasher. +const _: () = assert!( + FETCH_POLICY.len() + + ::MAX_ENCODED_SIZE + + 32 + + PaidFetchPolicyV1::ENCODED_SIZE + < MIN_CHUNK_SIZE, + "a single-chunk preimage that reaches MIN_CHUNK_SIZE panics the hasher" +); diff --git a/crates/rpc/src/protocol/work_profile.rs b/crates/rpc/src/protocol/work_profile.rs new file mode 100644 index 00000000..a9eb24f8 --- /dev/null +++ b/crates/rpc/src/protocol/work_profile.rs @@ -0,0 +1,336 @@ +//! Dispatch for the private paid-work profiles. The channel economics are +//! shared; each profile verifies its own input and terminal transcript. + +use super::artifacts::PreparedPaidInputV1; +use super::value::CanonicalDecodeError; +use super::work::{ + self, JobDeadlines, PaidChannel, PaidExecutionPolicyV1, PaidJobAuthorizationV1, + PaidJobResultV1, PaidWorkError, PrivateRecord, +}; +use super::work_fetch::{self, FetchRoutePolicy, PaidFetchPolicyV1, PreparedPaidFetchInputV1}; +use crate::{ContentId, Digest, OutputEventEnvelope}; + +/// The execution contract fixed when a channel is mounted. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PaidWorkPolicy { + /// Reproducible local evaluation. + Evaluate(PaidExecutionPolicyV1), + /// An authenticated fetch transcript under the committed route policy. + Fetch { + /// Fixed-width resource and price envelope. + policy: PaidFetchPolicyV1, + /// Canonical body opening the policy's route commitment. + route: FetchRoutePolicy, + }, +} + +impl From for PaidWorkPolicy { + fn from(policy: PaidExecutionPolicyV1) -> Self { + Self::Evaluate(policy) + } +} + +impl PaidWorkPolicy { + /// Checks the envelope and opens every variable policy commitment. + pub fn check(&self) -> Result<(), PaidWorkError> { + match self { + Self::Evaluate(policy) => work::check_execution_policy(policy), + Self::Fetch { policy, route } => { + work_fetch::check_fetch_policy(policy)?; + let body = route.canonical_body_bytes(); + FetchRoutePolicy::from_canonical_body_bytes(&body)?; + if work_fetch::fetch_route_commitment(&body)? != policy.route_commitment { + return Err(PaidWorkError::Mismatch { + field: "route_commitment", + }); + } + if matches!(route, FetchRoutePolicy::OpenFetch { .. }) + && policy.allowed_environment != crate::FetchEnvironment::Http.manifest_id() + { + return Err(PaidWorkError::Mismatch { + field: "open-fetch HTTPS manifest", + }); + } + Ok(()) + } + } + } + + /// The manifest that every proposal must name. + pub const fn allowed_environment(&self) -> ContentId { + match self { + Self::Evaluate(p) => p.allowed_environment, + Self::Fetch { policy, .. } => policy.allowed_environment, + } + } + + /// Fixed price both endpoints derive before admission. + pub const fn fixed_price(&self) -> u64 { + match self { + Self::Evaluate(p) => p.fixed_price, + Self::Fetch { policy, .. } => policy.fixed_price, + } + } + + /// Maximum retained encoded transcript. + pub const fn max_spool_bytes(&self) -> u64 { + match self { + Self::Evaluate(p) => p.max_spool_bytes, + Self::Fetch { policy, .. } => policy.max_spool_bytes, + } + } + + /// Maximum encoded delivery frame. + pub const fn max_encoded_result_frame(&self) -> u32 { + match self { + Self::Evaluate(p) => p.max_encoded_result_frame, + Self::Fetch { policy, .. } => policy.max_encoded_result_frame, + } + } + + /// Dispatch, delivery, and oracle grace margins, in finalized blocks. + pub const fn margins(&self) -> (u64, u64, u64) { + match self { + Self::Evaluate(p) => ( + p.dispatch_margin_blocks, + p.delivery_margin_blocks, + p.oracle_grace_blocks, + ), + Self::Fetch { policy: p, .. } => ( + p.dispatch_margin_blocks, + p.delivery_margin_blocks, + p.oracle_grace_blocks, + ), + } + } + + /// Checks a proposal against this profile's commitment and envelope. + pub fn check_authorization( + &self, + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + height: u64, + ) -> Result { + match self { + Self::Evaluate(policy) => { + work::check_authorization(channel, authorization, policy, height) + } + Self::Fetch { policy, .. } => { + work_fetch::check_fetch_authorization(channel, authorization, policy, height) + } + } + } + + /// Builds the authorization using the matching profile's digest domains. + pub fn propose( + &self, + channel: &PaidChannel, + input: &PreparedPaidWorkInput, + nonce: u64, + deadlines: JobDeadlines, + ) -> Result { + match (self, input) { + (Self::Evaluate(policy), PreparedPaidWorkInput::Evaluate(input)) => { + work::propose_authorization(channel, policy, input, nonce, deadlines) + } + (Self::Fetch { policy, .. }, PreparedPaidWorkInput::Fetch(input)) => { + work_fetch::propose_fetch_authorization(channel, policy, input, nonce, deadlines) + } + _ => Err(PaidWorkError::Mismatch { + field: "paid work profile", + }), + } + } + + /// Opens the prepared input and checks its profile-specific constraints. + pub fn check_input( + &self, + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + input: &PreparedPaidWorkInput, + ) -> Result<(), PaidWorkError> { + match (self, input) { + (Self::Evaluate(policy), PreparedPaidWorkInput::Evaluate(input)) => { + work::check_prepared_input(channel, authorization, policy, input) + } + (Self::Fetch { policy, route }, PreparedPaidWorkInput::Fetch(input)) => { + work_fetch::check_prepared_fetch_input( + channel, + authorization, + policy, + route, + input, + )?; + let parts = input.parts()?; + let request = crate::fetch::verify_input_events(&parts.fetch_input_transcript) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + if request.retention != crate::Retention::Ephemeral { + return Err(PaidWorkError::Mismatch { + field: "paid fetch requires ephemeral retention", + }); + } + Ok(()) + } + _ => Err(PaidWorkError::Mismatch { + field: "paid work profile", + }), + } + } + + /// Verifies bounds and the result against the authenticated prepared input. + pub fn terminal_result( + &self, + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + input: &PreparedPaidWorkInput, + transcript: &[OutputEventEnvelope], + ) -> Result { + self.check_input(channel, authorization, input)?; + if let Self::Fetch { policy, .. } = self { + work_fetch::check_fetch_output_limits(policy, transcript)?; + } + input.terminal_result(channel, authorization, transcript) + } + + /// Canonical bytes retained in a close descriptor. Evaluate bytes are unchanged. + pub fn encode(&self) -> Vec { + match self { + Self::Evaluate(policy) => policy.encode(), + Self::Fetch { policy, route } => { + let mut bytes = policy.encode(); + bytes.extend_from_slice(&route.canonical_body_bytes()); + bytes + } + } + } + + /// Strictly decodes one complete profile policy. + pub fn decode(bytes: &[u8]) -> Result { + let profile = if bytes.get(1) == Some(&5) { + let (record, body) = bytes + .split_at_checked(PaidFetchPolicyV1::ENCODED_SIZE) + .ok_or_else(|| CanonicalDecodeError::new("truncated fetch policy"))?; + Self::Fetch { + policy: PaidFetchPolicyV1::decode(record)?, + route: FetchRoutePolicy::from_canonical_body_bytes(body)?, + } + } else { + Self::Evaluate(PaidExecutionPolicyV1::decode(bytes)?) + }; + profile.check()?; + Ok(profile) + } +} + +/// Canonical prepared input retained by a paid-work journal. +/// +/// The formats are disjoint: Evaluate has exactly six length-prefixed bodies, +/// Fetch exactly two. Both decoders reject trailing bytes. Dispatch therefore +/// preserves existing Evaluate bytes without guessing from unsigned metadata. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PreparedPaidWorkInput { + /// The committed local artifact graph. + Evaluate(PreparedPaidInputV1), + /// The signed fetch request and its manifest. + Fetch(PreparedPaidFetchInputV1), +} + +impl From for PreparedPaidWorkInput { + fn from(input: PreparedPaidInputV1) -> Self { + Self::Evaluate(input) + } +} + +impl From for PreparedPaidWorkInput { + fn from(input: PreparedPaidFetchInputV1) -> Self { + Self::Fetch(input) + } +} + +impl PreparedPaidWorkInput { + /// Decodes a complete bounded bundle under exactly one profile. + pub fn decode(bytes: &[u8], budget: usize) -> Result { + if let Ok(input) = PreparedPaidInputV1::decode(bytes, budget) { + return Ok(Self::Evaluate(input)); + } + PreparedPaidFetchInputV1::decode(bytes, budget).map(Self::Fetch) + } + + /// Returns the original profile's canonical bytes. + pub fn encode(&self) -> Result, CanonicalDecodeError> { + match self { + Self::Evaluate(input) => input.encode(), + Self::Fetch(input) => input.encode(), + } + } + + /// The caller's authenticated request identity, independent of payment. + pub fn input_commitment(&self) -> Result { + match self { + Self::Evaluate(input) => Ok(crate::evaluate::input_commitment( + &input.parts()?.evaluate_request, + )), + Self::Fetch(input) => Ok(crate::fetch::verify_input_events( + &input.parts()?.fetch_input_transcript, + ) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))? + .input_commitment), + } + } + + /// Assurance requested by the canonical input, checked before disclosing it. + pub fn assurance(&self) -> Result { + match self { + Self::Evaluate(input) => Ok(input.parts()?.evaluate_request.assurance), + Self::Fetch(input) => Ok(crate::fetch::verify_input_events( + &input.parts()?.fetch_input_transcript, + ) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))? + .assurance), + } + } + + /// Recomputes the prepared-input commitment during journal replay. + pub fn digest(&self, channel: &PaidChannel) -> Result { + match self { + Self::Evaluate(input) => work::prepared_input_digest(channel, input), + Self::Fetch(input) => work_fetch::prepared_fetch_input_digest(channel, input), + } + } + + /// Rebuilds a result in the profile selected by the journaled input. + pub fn terminal_result( + &self, + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + transcript: &[OutputEventEnvelope], + ) -> Result { + match self { + Self::Evaluate(_) => work::terminal_result(channel, authorization, transcript), + Self::Fetch(bundle) => { + let parts = bundle.parts()?; + let input = crate::fetch::verify_input_events(&parts.fetch_input_transcript) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; + let (result, output) = work_fetch::verify_terminal_fetch_result( + channel, + authorization, + transcript, + input.assurance, + )?; + if input.input_commitment.digest() != authorization.request_commitment.digest() { + return Err(PaidWorkError::Mismatch { + field: "Fetch input commitment", + }); + } + if input.execution_environment == crate::FetchEnvironment::Http.manifest_id() { + let request = + crate::http_fetch::HttpFetchRequest::decode(input.body.as_bytes()) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; + crate::http_fetch::HttpFetchResponse::from_output(&request, &output) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; + } + Ok(result) + } + } + } +} diff --git a/crates/rpc/src/protocol/work_setup.rs b/crates/rpc/src/protocol/work_setup.rs index 89061997..75b395bc 100644 --- a/crates/rpc/src/protocol/work_setup.rs +++ b/crates/rpc/src/protocol/work_setup.rs @@ -18,8 +18,8 @@ use hellas_kernel::{ use crate::protocol::work::{ PaidChannel, PaidChannelPolicyV1, PaidExecutionPolicyV1, PaidWorkError, PrivateRecord, - check_execution_policy, }; +use crate::protocol::work_profile::PaidWorkPolicy; /// Why a configured channel is not one this endpoint may work over. #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] @@ -226,7 +226,7 @@ pub struct WorkChannelConfig { /// The credit policy those terms commit to. pub channel_policy: PaidChannelPolicyV1, /// The execution policy in force on this channel. - pub execution_policy: PaidExecutionPolicyV1, + pub execution_policy: PaidWorkPolicy, /// The payment edge's value, reserve, and close fees as the /// operator expects them to be funded. pub expected_payment_values: EdgeValues, @@ -252,7 +252,7 @@ pub struct ProviderChannelPolicy { /// The credit policy this provider will work under. pub channel_policy: PaidChannelPolicyV1, /// The execution policy this provider will run jobs under. - pub execution_policy: PaidExecutionPolicyV1, + pub execution_policy: PaidWorkPolicy, /// The payment edge's value, reserve, and close fees as the provider /// requires them to be funded. pub expected_payment_values: EdgeValues, @@ -293,7 +293,7 @@ impl ProviderChannelPolicy { &self.policy_salt, self.channel_policy, )?; - check_execution_policy(&self.execution_policy)?; + self.execution_policy.check()?; if work_payment_settlement( self.expected_payment_values, channel.payment_terms().omission_bond, @@ -306,7 +306,7 @@ impl ProviderChannelPolicy { channel, bond_edge, policy_salt: self.policy_salt, - execution_policy: self.execution_policy, + execution_policy: self.execution_policy.clone(), expected_payment_values: self.expected_payment_values, }) } @@ -361,7 +361,7 @@ impl ProviderChannelPolicy { payment_terms, policy_salt: self.policy_salt, channel_policy: self.channel_policy, - execution_policy: self.execution_policy, + execution_policy: self.execution_policy.clone(), expected_payment_values: self.expected_payment_values, }) } @@ -380,7 +380,7 @@ pub struct WorkChannelDescriptor { bond_edge: EdgeId, bond_terms_hash: TermsHash, policy_salt: [u8; 32], - execution_policy: PaidExecutionPolicyV1, + execution_policy: PaidWorkPolicy, expected_payment_values: EdgeValues, } @@ -418,7 +418,7 @@ impl WorkChannelDescriptor { &config.policy_salt, config.channel_policy, )?; - check_execution_policy(&config.execution_policy)?; + config.execution_policy.check()?; let settlement = work_payment_settlement(config.expected_payment_values, omission_bond) .ok_or(WorkSetupError::Unsettleable)?; @@ -445,7 +445,7 @@ impl WorkChannelDescriptor { } /// Returns the execution policy in force on this channel. - pub const fn execution_policy(&self) -> &PaidExecutionPolicyV1 { + pub const fn execution_policy(&self) -> &PaidWorkPolicy { &self.execution_policy } @@ -462,7 +462,7 @@ impl WorkChannelDescriptor { channel: self.channel.clone(), bond_edge: self.bond_edge, policy_salt: self.policy_salt, - execution_policy: self.execution_policy, + execution_policy: self.execution_policy.clone(), expected_payment_values: self.expected_payment_values, } } @@ -578,7 +578,7 @@ impl WorkChannelDescriptor { Ok(ReadyChannel { channel: self.channel.clone(), - execution_policy: self.execution_policy, + execution_policy: self.execution_policy.clone(), settlement, finalized_height: observed.height, admission_horizon: horizon, @@ -607,12 +607,15 @@ pub struct CloseDescriptor { channel: PaidChannel, bond_edge: EdgeId, policy_salt: [u8; 32], - execution_policy: PaidExecutionPolicyV1, + execution_policy: PaidWorkPolicy, expected_payment_values: EdgeValues, } /// First byte of the close descriptor stored in an armed setup record. const CLOSE_DESCRIPTOR_VERSION: u8 = 1; +/// Version 2 length-prefixes the execution policy so the Fetch profile can +/// extend it; v1 is the fixed-size Evaluate policy exactly. +const CLOSE_DESCRIPTOR_VERSION_V2: u8 = 2; impl CloseDescriptor { /// Returns the payment channel, including both complete terms bodies and @@ -630,7 +633,7 @@ impl CloseDescriptor { /// Returns the execution policy the armed endpoint accepted. #[must_use] - pub const fn execution_policy(&self) -> &PaidExecutionPolicyV1 { + pub const fn execution_policy(&self) -> &PaidWorkPolicy { &self.execution_policy } @@ -673,7 +676,12 @@ impl CloseDescriptor { /// Returns the descriptor's canonical journal bytes. #[must_use] pub fn encode(&self) -> Vec { - let mut out = vec![CLOSE_DESCRIPTOR_VERSION]; + let version = if matches!(self.execution_policy, PaidWorkPolicy::Evaluate(_)) { + CLOSE_DESCRIPTOR_VERSION + } else { + CLOSE_DESCRIPTOR_VERSION_V2 + }; + let mut out = vec![version]; push_kernel(&mut out, &self.channel.network()); push_kernel(&mut out, &self.channel.payment_edge()); push_kernel( @@ -682,7 +690,11 @@ impl CloseDescriptor { ); out.extend_from_slice(&self.policy_salt); out.extend_from_slice(&self.channel.channel_policy().encode()); - out.extend_from_slice(&self.execution_policy.encode()); + let policy_bytes = self.execution_policy.encode(); + if version == CLOSE_DESCRIPTOR_VERSION_V2 { + out.extend_from_slice(&(policy_bytes.len() as u64).to_be_bytes()); + } + out.extend_from_slice(&policy_bytes); out.extend_from_slice(&self.expected_payment_values.value().to_be_bytes()); out.extend_from_slice(&self.expected_payment_values.reserve().to_be_bytes()); push_kernel(&mut out, &self.expected_payment_values.close_fees()); @@ -697,7 +709,8 @@ impl CloseDescriptor { /// rerun them. pub fn decode(bytes: &[u8]) -> Result { let mut cursor = CloseCursor { bytes }; - if cursor.byte()? != CLOSE_DESCRIPTOR_VERSION { + let version = cursor.byte()?; + if version != CLOSE_DESCRIPTOR_VERSION && version != CLOSE_DESCRIPTOR_VERSION_V2 { return Err(WorkSetupError::DescriptorMalformed); } let network = cursor.network()?; @@ -710,9 +723,18 @@ impl CloseDescriptor { let channel_policy = PaidChannelPolicyV1::decode(cursor.take(PaidChannelPolicyV1::ENCODED_SIZE)?) .map_err(|_| WorkSetupError::DescriptorMalformed)?; - let execution_policy = - PaidExecutionPolicyV1::decode(cursor.take(PaidExecutionPolicyV1::ENCODED_SIZE)?) - .map_err(|_| WorkSetupError::DescriptorMalformed)?; + let policy_len = if version == CLOSE_DESCRIPTOR_VERSION { + PaidExecutionPolicyV1::ENCODED_SIZE + } else { + usize::try_from(cursor.u64()?).map_err(|_| WorkSetupError::DescriptorMalformed)? + }; + let execution_policy = PaidWorkPolicy::decode(cursor.take(policy_len)?) + .map_err(|_| WorkSetupError::DescriptorMalformed)?; + if (version == CLOSE_DESCRIPTOR_VERSION) + != matches!(execution_policy, PaidWorkPolicy::Evaluate(_)) + { + return Err(WorkSetupError::DescriptorMalformed); + } let expected_payment_values = EdgeValues::new(cursor.u64()?, cursor.u64()?, cursor.field::()?); if !cursor.bytes.is_empty() { @@ -726,7 +748,7 @@ impl CloseDescriptor { &policy_salt, channel_policy, )?; - check_execution_policy(&execution_policy)?; + execution_policy.check()?; if work_payment_settlement( expected_payment_values, channel.payment_terms().omission_bond, @@ -840,18 +862,16 @@ pub struct ObservedChannel<'a> { /// method here, because the objects it would have to re-read are not /// carried. /// -/// So the obligation is the holder's, in the same way [`ObservedChannel`] -/// owes coherence: run [`WorkChannelDescriptor::check_ready`] again -/// against a fresh snapshot before each signature, and sign against the -/// `ReadyChannel` that read produced. [`Self::check_signable`] is the -/// per-signature *arithmetic* — the horizon and the deadline margins, -/// against the height the endpoint has actually reached. It is not a -/// substitute for the refresh, and it does not claim to be one: no -/// arithmetic over a stale read can see a contest that opened after it. +/// A live channel observer must apply finalized history and refresh this +/// decision independently of requests. Admission, delivery and new certificates +/// check local observer freshness and the journal's close cutoff under the same +/// state lock. An expired observer must suspend those operations; cached +/// arithmetic alone cannot establish that the channel remains open. +/// [`Self::check_signable`] checks deadline margins against the applied cursor. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ReadyChannel { channel: PaidChannel, - execution_policy: PaidExecutionPolicyV1, + execution_policy: PaidWorkPolicy, settlement: WorkPaymentSettlement, finalized_height: u64, admission_horizon: u64, @@ -875,7 +895,7 @@ impl ReadyChannel { /// whose margins [`Self::check_signable`] measures against and whose /// digest the authorization names. Carrying it here is what stops /// those three from being three copies. - pub const fn execution_policy(&self) -> &PaidExecutionPolicyV1 { + pub const fn execution_policy(&self) -> &PaidWorkPolicy { &self.execution_policy } @@ -936,18 +956,18 @@ impl ReadyChannel { }); } - let policy = &self.execution_policy; + let (dispatch, delivery, oracle_grace) = self.execution_policy.margins(); let reachable = cursor_height - .checked_add(policy.dispatch_margin_blocks) - .and_then(|sum| sum.checked_add(policy.delivery_margin_blocks)) + .checked_add(dispatch) + .and_then(|sum| sum.checked_add(delivery)) .ok_or(PaidWorkError::Overflow { field: "dispatch and delivery margins", })?; if reachable > terminal_deadline { return Err(WorkSetupError::TerminalUnreachable { height: cursor_height, - dispatch: policy.dispatch_margin_blocks, - delivery: policy.delivery_margin_blocks, + dispatch, + delivery, terminal: terminal_deadline, }); } @@ -958,12 +978,12 @@ impl ReadyChannel { .ok_or(PaidWorkError::Overflow { field: "oracle grace interval", })?; - if grace < policy.oracle_grace_blocks { + if grace < oracle_grace { return Err(WorkSetupError::OracleGraceTooShort { terminal: terminal_deadline, payment: payment_deadline, actual: grace, - grace: policy.oracle_grace_blocks, + grace: oracle_grace, }); } Ok(()) @@ -1018,7 +1038,7 @@ impl ReadyChannel { terminal_deadline: u64, ) -> Result<(), WorkSetupError> { self.check_caught_up(cursor_height)?; - let delivery = self.execution_policy.delivery_margin_blocks; + let delivery = self.execution_policy.margins().1; let arrives = cursor_height .checked_add(delivery) .ok_or(PaidWorkError::Overflow { diff --git a/crates/rpc/src/telemetry/http.rs b/crates/rpc/src/telemetry/http.rs index 8ba43228..3cfdf4cb 100644 --- a/crates/rpc/src/telemetry/http.rs +++ b/crates/rpc/src/telemetry/http.rs @@ -62,7 +62,10 @@ pub async fn trace_request(request: Request, next: Next) -> Response { super::inject(&trace.span, &mut context); if let Some(value) = context.get("traceparent").and_then(|value| value.as_text()) { if let Ok(header) = HeaderValue::from_str(value) { - response.headers_mut().insert("traceparent", header); + response + .headers_mut() + .entry("traceparent") + .or_insert(header); } if let Some(trace_id) = value.split('-').nth(1) && let Ok(header) = HeaderValue::from_str(trace_id) @@ -165,6 +168,69 @@ mod tests { use tracing::instrument::WithSubscriber; use tracing_subscriber::prelude::*; + #[tokio::test] + async fn upstream_trace_headers_survive_with_separate_gateway_correlation() { + opentelemetry::global::set_text_map_propagator( + opentelemetry_sdk::propagation::TraceContextPropagator::new(), + ); + let provider = SdkTracerProvider::builder().build(); + let dispatch = tracing::Dispatch::new( + tracing_subscriber::registry() + .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("headers"))), + ); + const UPSTREAM: &str = "00-33333333333333333333333333333333-4444444444444444-01"; + const CALLER: &str = "00-11111111111111111111111111111111-2222222222222222-01"; + let app = axum::Router::new() + .route("/plain", get(|| async { "ok" })) + .route( + "/upstream", + get(|| async { + Response::builder() + .header("traceparent", UPSTREAM) + .header("tracestate", "vendor=upstream") + .body(Body::from("ok")) + .unwrap() + }), + ) + .layer(axum::middleware::from_fn(trace_request)); + for path in ["/upstream", "/plain"] { + let response = app + .clone() + .oneshot( + Request::builder() + .uri(path) + .header("traceparent", CALLER) + .body(Body::empty()) + .unwrap(), + ) + .with_subscriber(dispatch.clone()) + .await + .unwrap(); + assert_eq!( + response.headers()["x-hellas-trace-id"], + "11111111111111111111111111111111" + ); + assert_eq!(response.headers().get_all("traceparent").iter().count(), 1); + if path == "/upstream" { + assert_eq!(response.headers()["traceparent"], UPSTREAM); + assert_eq!(response.headers()["tracestate"], "vendor=upstream"); + } else { + assert!( + response.headers()["traceparent"] + .to_str() + .unwrap() + .starts_with("00-11111111111111111111111111111111-") + ); + } + assert_eq!( + axum::body::to_bytes(response.into_body(), 16) + .await + .unwrap(), + "ok" + ); + } + } + #[tokio::test] async fn http_spans_cover_handler_and_body_completion_errors_and_cancellation() { let exporter = InMemorySpanExporter::default(); diff --git a/crates/rpc/tests/paid_fetch_vectors.rs b/crates/rpc/tests/paid_fetch_vectors.rs new file mode 100644 index 00000000..61e871a6 --- /dev/null +++ b/crates/rpc/tests/paid_fetch_vectors.rs @@ -0,0 +1,1726 @@ +//! Independent vectors and mutation proofs for the paid-fetch records. +//! +//! The fetch-profile companion to `paid_work_vectors.rs`, in the same +//! style: canonical encodings pinned as hex decoded field by field, +//! digests pinned as hex, and every refusal covered by a mutation that +//! names what it breaks. +//! +//! The signed input transcript is built with the nonce pinned rather than +//! through `build_input_events_with_retention`: that constructor draws the +//! nonce from `OsRng`, and a golden digest over a random nonce is a +//! different vector every run. The transcript here is otherwise the exact +//! eight events that constructor signs, and the randomized constructor is +//! exercised end-to-end in +//! `the_real_input_constructor_passes_the_whole_pipeline`. + +#![cfg(feature = "work")] + +use hellas_kernel::{ + BlockHeight, EdgeId, EdgeValues, Fees, List, NetworkId, Parties, Payout, Secp256k1Signer, + Secp256k1Verifier, SigVerifier, WorkPaymentSettlement, WorkPaymentTerms, WorkStakeBondTerms, + work_payment_settlement, +}; +use hellas_rpc::fetch::{ + FetchOutputTranscriptBuilder, build_input_events_with_retention, encode_fetch_event_payload, + encode_fetch_terminal_payload, input_canonicalization, output_canonicalization, +}; +use hellas_rpc::output::{OutputEvent, StopReason, TextChannel, Usage}; +use hellas_rpc::protocol::work::{ + CreditLedger, JobDeadlines, PaidChannel, PaidChannelPolicyV1, PaidJobAuthorizationV1, + PaidWorkError, PrivateRecord, check_authorization, check_result, execution_policy_digest, + next_payment, result_digest, signing_hash, work_id, +}; +use hellas_rpc::protocol::work_fetch::{ + FetchRoutePolicy, PaidFetchPolicyV1, PreparedPaidFetchInputV1, check_fetch_authorization, + check_fetch_policy, check_prepared_fetch_input, fetch_canonical_output_digest, + fetch_policy_digest, fetch_route_commitment, prepared_fetch_input_digest, + propose_fetch_authorization, terminal_fetch_result, +}; +use hellas_rpc::{ + Assurance, ContentId, Digest, FetchEnvironment, InputCommitment, InputEventEnvelope, + InputTranscriptBuilder, Operation, OutputEventEnvelope, ProducerSigningKey, ProgramManifest, + PublicKey, RequestCommitment, Retention, scheme_id, +}; + +// ── Fixtures ────────────────────────────────────────────────────────── + +const NETWORK: &str = "hellas-devnet-1"; +const OTHER_NETWORK: &str = "hellas-devnet-22"; +const SALT: [u8; 32] = [0x5a; 32]; + +/// The pinned `request.nonce`. `build_input_events_with_retention` draws +/// this from `OsRng`; the golden digests below need it fixed. +const NONCE: [u8; 32] = [7; 32]; +const PROPOSAL_NONCE: u64 = 0x0102_0304_0506_0708; +const SERVICE: &str = "openai"; +const METHOD: &str = "responses"; +const REQUEST_BODY: &[u8] = br#"{"model":"gpt-5.2","input":"paid fetch"}"#; + +/// The channel's certificate capacity, from the kernel's own settlement +/// arithmetic over an edge that locks a million and reserves nothing. +fn capacity() -> WorkPaymentSettlement { + let Some(settlement) = work_payment_settlement( + EdgeValues::new(1_000_000 + payment_terms().omission_bond, 0, Fees::ZERO), + payment_terms().omission_bond, + ) else { + panic!("a funded edge prices both exits"); + }; + assert_eq!(settlement.capacity(), 1_000_000); + settlement +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn network() -> NetworkId { + NetworkId::new(NETWORK).expect("legal network id") +} + +/// The settlement signer the channel terms name as the client. +fn client() -> Secp256k1Signer { + Secp256k1Signer::from_secret_scalar([3; 32]).expect("legal scalar") +} + +/// The settlement signer the channel terms name as the provider. +fn provider() -> Secp256k1Signer { + Secp256k1Signer::from_secret_scalar([4; 32]).expect("legal scalar") +} + +/// The transcript signing key over the client's secret: a signed fetch +/// input is the client's request, so its key and the settlement key are +/// the one scalar in two wrappers. +fn caller_key() -> ProducerSigningKey { + ProducerSigningKey::from_secret_bytes([3; 32]).expect("legal scalar") +} + +/// The transcript signing key over the provider's secret. +fn producer_key() -> ProducerSigningKey { + ProducerSigningKey::from_secret_bytes([4; 32]).expect("legal scalar") +} + +fn channel_policy() -> PaidChannelPolicyV1 { + PaidChannelPolicyV1 { + compute_credit_limit: 700, + delivery_credit_limit: 800, + } +} + +fn payment_terms() -> WorkPaymentTerms { + let bond = WorkStakeBondTerms { + // Bond maker is the provider, taker is the client; the payment + // edge mirrors that, which is where `PaidChannel` reads its keys. + parties: Parties::new(provider().party_key(), client().party_key()), + timeout: BlockHeight::new(5_000), + timeout_outputs: List::take( + [ + Payout::new(provider().party_key(), 900), + Payout::default(), + Payout::default(), + Payout::default(), + ], + 1, + ), + max_job_price: 500, + }; + WorkPaymentTerms { + bond_edge: EdgeId::from_bytes([0xb0; 32]), + bond_terms: bond, + private_policy_commitment: hellas_rpc::protocol::work::private_policy_commitment( + network(), + &SALT, + &channel_policy(), + ), + omit_response_blocks: 32, + start_validity_blocks: 16, + omission_bond: 100, + } +} + +fn channel() -> PaidChannel { + PaidChannel::new( + network(), + EdgeId::from_bytes([0xe1; 32]), + payment_terms(), + &SALT, + channel_policy(), + ) + .expect("terms that commit to this credit policy") +} + +/// A channel on another network, or another payment edge. +fn channel_on(network: NetworkId, payment_edge: EdgeId) -> PaidChannel { + let terms = WorkPaymentTerms { + private_policy_commitment: hellas_rpc::protocol::work::private_policy_commitment( + network, + &SALT, + &channel_policy(), + ), + ..payment_terms() + }; + PaidChannel::new(network, payment_edge, terms, &SALT, channel_policy()) + .expect("terms that commit to this credit policy") +} + +fn environment() -> FetchEnvironment { + FetchEnvironment::OpenAiResponses +} + +fn manifest() -> ProgramManifest { + environment().manifest() +} + +fn route() -> FetchRoutePolicy { + FetchRoutePolicy::sealed_route(SERVICE, METHOD).expect("a legal route") +} + +fn route_commitment() -> Digest { + fetch_route_commitment(&route().canonical_body_bytes()).expect("a representable route body") +} + +fn fetch_policy() -> PaidFetchPolicyV1 { + PaidFetchPolicyV1 { + allowed_environment: environment().manifest_id(), + route_commitment: route_commitment(), + max_request_body_bytes: 4_096, + max_output_events: 64, + max_output_bytes: 1_048_576, + max_spool_bytes: 65_536, + max_encoded_result_frame: 262_144, + max_encoded_prepared_input: 1_048_576, + dispatch_margin_blocks: 20, + delivery_margin_blocks: 10, + oracle_grace_blocks: 30, + fixed_price: 250, + } +} + +fn deadlines() -> JobDeadlines { + JobDeadlines { + acceptance: 1_000, + terminal: 1_050, + payment: 1_100, + } +} + +/// Signs the eight fetch input events in the order +/// `build_input_events_with_retention` pushes them, with the nonce pinned. +fn signed_input( + key: &ProducerSigningKey, + assurance: Assurance, + environment: ContentId, + service: &str, + method: &str, + body: &[u8], +) -> Vec { + let mut builder = InputTranscriptBuilder::new( + scheme_id(Operation::Fetch, assurance), + key, + input_canonicalization(), + ); + builder + .push("assurance", vec![assurance.to_byte()]) + .unwrap(); + builder + .push("execution.environment", environment.as_bytes().to_vec()) + .unwrap(); + builder.push("request.nonce", NONCE.to_vec()).unwrap(); + builder + .push("service", service.as_bytes().to_vec()) + .unwrap(); + builder.push("method", method.as_bytes().to_vec()).unwrap(); + builder + .push( + "request.retain", + vec![u8::from(Retention::Retain.should_retain())], + ) + .unwrap(); + builder.push("request.body", body.to_vec()).unwrap(); + builder.push("input.end", Vec::new()).unwrap(); + builder.finish().unwrap().0 +} + +fn input_events() -> Vec { + signed_input( + &caller_key(), + Assurance::ProducerSigned, + environment().manifest_id(), + SERVICE, + METHOD, + REQUEST_BODY, + ) +} + +/// The request commitment exactly as the fetch ticket flow reads it: the +/// verified input transcript's commitment, re-wrapped. +fn input_request_commitment() -> RequestCommitment { + let input = hellas_rpc::fetch::verify_input_events(&input_events()) + .expect("the fixture transcript verifies"); + RequestCommitment::from_digest(input.input_commitment.digest()) +} + +fn bundle() -> PreparedPaidFetchInputV1 { + PreparedPaidFetchInputV1::new(&input_events(), &manifest()).expect("a legal transcript") +} + +fn propose( + channel: &PaidChannel, + policy: &PaidFetchPolicyV1, + bundle: &PreparedPaidFetchInputV1, +) -> PaidJobAuthorizationV1 { + propose_fetch_authorization(channel, policy, bundle, PROPOSAL_NONCE, deadlines()) + .expect("a legal proposal") +} + +fn authorization() -> PaidJobAuthorizationV1 { + propose(&channel(), &fetch_policy(), &bundle()) +} + +/// Assembles bundle bytes from two arbitrary bodies, so a test can build +/// a spelling the constructor would never produce. +fn assemble(bodies: &[&[u8]]) -> Vec { + let mut bytes = Vec::new(); + for body in bodies { + bytes.extend_from_slice(&(body.len() as u32).to_be_bytes()); + bytes.extend_from_slice(body); + } + bytes +} + +fn input_digest(channel: &PaidChannel, bundle: &PreparedPaidFetchInputV1) -> Digest { + prepared_fetch_input_digest(channel, bundle).expect("a representable bundle") +} + +/// One semantic output event payload, encoded with the fetch profile's +/// own payload codec: the paid layer treats payloads as opaque, but the +/// fixture answers are real ones. +fn event_payload(delta: &str) -> Vec { + encode_fetch_event_payload(&OutputEvent::TextDelta { + index: 0, + delta: delta.to_string(), + channel: TextChannel::Output, + }) + .expect("a legal event payload") +} + +fn terminal_payload() -> Vec { + encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: Some(Usage { + input_tokens: Some(12), + output_tokens: Some(2), + total_tokens: Some(14), + }), + }) + .expect("a legal terminal payload") +} + +/// The provider-signed output transcript answering the authorization's +/// request: two semantic events and a terminal. +fn output_transcript(authorization: &PaidJobAuthorizationV1) -> Vec { + let input = InputCommitment::from_digest(authorization.request_commitment.digest()); + let key = producer_key(); + let mut builder = FetchOutputTranscriptBuilder::new(input, Assurance::ProducerSigned, &key); + builder.push_event(event_payload("paid ")).unwrap(); + builder.push_event(event_payload("fetch")).unwrap(); + builder.finish(terminal_payload()).unwrap() +} + +// ── Golden encodings ────────────────────────────────────────────────── + +/// The fetch policy record's exact bytes, decoded field by field. +#[test] +fn golden_fetch_policy_encoding_is_pinned() { + assert_eq!(PaidFetchPolicyV1::BODY_SIZE, 124); + assert_eq!(PaidFetchPolicyV1::ENCODED_SIZE, 126); + + let policy = PaidFetchPolicyV1 { + allowed_environment: ContentId::from_bytes([0x40; 32]), + route_commitment: Digest::from_bytes([0x41; 32]), + max_request_body_bytes: 1, + max_output_events: 2, + max_output_bytes: 3, + max_spool_bytes: 4, + max_encoded_result_frame: 5, + max_encoded_prepared_input: 6, + dispatch_margin_blocks: 7, + delivery_margin_blocks: 8, + oracle_grace_blocks: 9, + fixed_price: 10, + }; + assert_eq!( + hex(&policy.encode()), + concat!( + "01", + "05", // format version 1, tag 5 = PAID_FETCH_POLICY + "4040404040404040404040404040404040404040404040404040404040404040", // environment + "4141414141414141414141414141414141414141414141414141414141414141", // route + "00000001", // max_request_body_bytes + "00000002", // max_output_events + "00000003", // max_output_bytes + "0000000000000004", // max_spool_bytes + "00000005", // max_encoded_result_frame + "00000006", // max_encoded_prepared_input + "0000000000000007", // dispatch_margin_blocks + "0000000000000008", // delivery_margin_blocks + "0000000000000009", // oracle_grace_blocks + "000000000000000a", // fixed_price + ) + ); + assert_eq!(PaidFetchPolicyV1::decode(&policy.encode()), Ok(policy)); +} + +/// Wrong tag, wrong version, truncation, and a trailing byte all reject, +/// on the new record. +#[test] +fn fetch_policy_envelope_mutations_reject() { + let bytes = fetch_policy().encode(); + assert!(PaidFetchPolicyV1::decode(&bytes).is_ok()); + + // MUTATION: the fetch policy presented under the evaluate policy tag. + // Both are policies; only the envelope says which profile they are. + let mut evaluate_tag = bytes.clone(); + evaluate_tag[1] = 1; + assert_eq!( + PaidFetchPolicyV1::decode(&evaluate_tag), + Err(PaidWorkError::WrongRecordTag { + expected: 5, + actual: 1 + }) + ); + + // MUTATION: a future format version. + let mut wrong_version = bytes.clone(); + wrong_version[0] = 2; + assert_eq!( + PaidFetchPolicyV1::decode(&wrong_version), + Err(PaidWorkError::UnknownFormatVersion { actual: 2 }) + ); + + // MUTATION: one byte short. + let truncated = &bytes[..bytes.len() - 1]; + assert_eq!( + PaidFetchPolicyV1::decode(truncated), + Err(PaidWorkError::RecordLength { + expected: 126, + actual: 125 + }) + ); + + // MUTATION: one byte long. + let mut trailing = bytes.clone(); + trailing.push(0); + assert_eq!( + PaidFetchPolicyV1::decode(&trailing), + Err(PaidWorkError::RecordLength { + expected: 126, + actual: 127 + }) + ); +} + +/// The two route-policy bodies, hand-decoded. +#[test] +fn golden_route_body_encodings_are_pinned() { + // ["hellas.work.fetch-route.sealed.v1", "openai", "responses"]: a + // three-element array, the 33-byte schema, the two route components. + let sealed = route().canonical_body_bytes(); + assert_eq!( + hex(&sealed), + concat!( + "83", // array(3) + "7821", // str(33) + "68656c6c61732e776f726b2e66657463682d726f7574652e7365616c65642e7631", + "66", // str(6) + "6f70656e6169", // "openai" + "69", // str(9) + "726573706f6e736573", // "responses" + ) + ); + assert_eq!( + FetchRoutePolicy::from_canonical_body_bytes(&sealed), + Ok(route()) + ); + + // ["hellas.work.fetch-route.open.v1", 1, ["api.openai.com"]]: the pin + // as the integer 1, and one host. + let open = FetchRoutePolicy::open_fetch(true, vec!["api.openai.com".to_string()]); + let open_bytes = open.canonical_body_bytes(); + assert_eq!( + hex(&open_bytes), + concat!( + "83", // array(3) + "781f", // str(31) + "68656c6c61732e776f726b2e66657463682d726f7574652e6f70656e2e7631", + "01", // require_spki_pin = true + "81", // array(1) + "6e", // str(14) + "6170692e6f70656e61692e636f6d", // "api.openai.com" + ) + ); + assert_eq!( + FetchRoutePolicy::from_canonical_body_bytes(&open_bytes), + Ok(open) + ); +} + +/// The route codec's refusals: unknown schema, wrong shape, a pin that is +/// not 0 or 1, a trailing byte, a noncanonical spelling, and route +/// components the transcript scheme would not admit. +#[test] +fn route_policy_refusals() { + let sealed = route().canonical_body_bytes(); + + // MUTATION: an unknown schema tag. + let unknown_schema = { + let mut body = Vec::new(); + body.push(0x83); + body.extend_from_slice(&[0x78, 0x21]); + body.extend_from_slice(b"hellas.work.fetch-route.sealed.v9"); + body.extend_from_slice(&sealed[sealed.len() - 17..]); + body + }; + assert!(unknown_schema.len() == sealed.len()); + assert!( + FetchRoutePolicy::from_canonical_body_bytes(&unknown_schema) + .expect_err("an unknown schema must not parse") + .to_string() + .contains("unexpected fetch route schema tag") + ); + + // MUTATION: the array shortened to two elements. + let mut short = sealed.clone(); + short[0] = 0x82; + assert!( + FetchRoutePolicy::from_canonical_body_bytes(&short) + .expect_err("a shortened array must not parse") + .to_string() + .contains("expected array length 3, got 2") + ); + + // MUTATION: a trailing byte. + let mut trailing = sealed.clone(); + trailing.push(0); + assert!( + FetchRoutePolicy::from_canonical_body_bytes(&trailing) + .expect_err("a trailing byte must not parse") + .to_string() + .contains("trailing bytes") + ); + + // MUTATION: the pin spelled as the integer 2. + let mut bad_pin = + FetchRoutePolicy::open_fetch(true, Vec::::new()).canonical_body_bytes(); + let pin = bad_pin.len() - 2; // [schema, pin, empty array]: pin is third from last + assert_eq!(bad_pin[pin], 1); + bad_pin[pin] = 2; + assert!( + FetchRoutePolicy::from_canonical_body_bytes(&bad_pin) + .expect_err("a pin of 2 must not parse") + .to_string() + .contains("require_spki_pin must be 0 or 1") + ); + + // MUTATION: an empty service in the constructor and in a body. + assert!(matches!( + FetchRoutePolicy::sealed_route("", METHOD), + Err(hellas_rpc::fetch::FetchProtocolError::EmptyService) + )); + assert!(matches!( + FetchRoutePolicy::sealed_route(SERVICE, ""), + Err(hellas_rpc::fetch::FetchProtocolError::EmptyMethod) + )); + let overlong = "x".repeat(hellas_rpc::fetch::MAX_FETCH_ROUTE_COMPONENT_BYTES + 1); + assert!(matches!( + FetchRoutePolicy::sealed_route(&overlong, METHOD), + Err(hellas_rpc::fetch::FetchProtocolError::RouteComponentLimit { + field: "service", + .. + }) + )); + let mut empty_service = sealed.clone(); + let service_at = empty_service.len() - "openai".len() - "responses".len() - 2; + empty_service[service_at] = 0x60; // str(0) + let mut shortened = empty_service[..service_at + 1].to_vec(); + shortened.extend_from_slice(&empty_service[service_at + 1 + "openai".len()..]); + assert!( + FetchRoutePolicy::from_canonical_body_bytes(&shortened) + .expect_err("an empty service must not parse") + .to_string() + .contains("fetch service must not be empty") + ); + + // The allowlist is a set: the constructor sorts and deduplicates, so + // the canonical body of a shuffled, doubled list is the same bytes. + let shuffled = FetchRoutePolicy::open_fetch( + false, + vec![ + "b.example".to_string(), + "a.example".to_string(), + "b.example".to_string(), + ], + ); + let sorted = FetchRoutePolicy::open_fetch( + false, + vec!["a.example".to_string(), "b.example".to_string()], + ); + assert_eq!(shuffled, sorted); +} + +// ── Golden digests ──────────────────────────────────────────────────── + +/// The profile's pinned digests, over the deterministic fixture. +#[test] +fn golden_fetch_digests_are_pinned() { + // XFH("hellas.work.fetch-route.v1" || u32be(len) || route body), both + // variants. + assert_eq!( + hex(route_commitment().as_bytes()), + "a2e44135e61c07424562ab53dfe8b74d18ddd6872d39a59d9206485d4ac41530" + ); + let open = FetchRoutePolicy::open_fetch(true, vec!["api.openai.com".to_string()]); + assert_eq!( + hex(fetch_route_commitment(&open.canonical_body_bytes()) + .expect("a representable route body") + .as_bytes()), + "55e7ccf1928e21128357726422c637ddc5cf0437dde10568f0f2f83e0d69c299" + ); + let channel = channel(); + // XH("hellas.work.fetch-policy.v1" || network || channel || record). + assert_eq!( + hex(fetch_policy_digest(&channel, &fetch_policy()).as_bytes()), + "6a675a75bd5b40e8fad1e35272f1c971a1a9cc34a8b756ab5298cc747ff55ede" + ); + // XFH("hellas.work.prepared-fetch-input.v1" || network || channel || + // bundle). + assert_eq!( + hex(input_digest(&channel, &bundle()).as_bytes()), + "8bb0796694ed05a0e32fe2f40d32b16256bc802fbe073a8a5a06e6c59e58c3a9" + ); + // XH("hellas.work.paid-job-authorize.v1" || network || channel || + // authorization): the shared record, signed under this profile. + assert_eq!( + hex(work_id(&channel, &authorization()).as_bytes()), + "0258caba196b7a4c13c1ad8ea8f575ea9b943f58e80d73c17a3d41d534de0f30" + ); +} + +/// The same bodies in another channel are other digests, and the check +/// that names the channel refuses them. +#[test] +fn fetch_records_do_not_cross_channels() { + let here = channel(); + let sibling = channel_on(network(), EdgeId::from_bytes([0xe2; 32])); + let elsewhere = channel_on( + NetworkId::new(OTHER_NETWORK).expect("legal network id"), + here.payment_edge(), + ); + let authorization = authorization(); + + assert_ne!(here.id(), sibling.id()); + for (label, other) in [("sibling", &sibling), ("elsewhere", &elsewhere)] { + assert_ne!( + fetch_policy_digest(&here, &fetch_policy()), + fetch_policy_digest(other, &fetch_policy()), + "{label} shares this channel's fetch policy digest" + ); + assert_ne!( + input_digest(&here, &bundle()), + input_digest(other, &bundle()), + "{label} shares this channel's prepared fetch input digest" + ); + // MUTATION: replay this channel's fetch authorization on another + // one. + assert_eq!( + check_fetch_authorization(other, &authorization, &fetch_policy(), 900), + Err(PaidWorkError::Mismatch { + field: "channel_id" + }), + "{label} accepted a foreign authorization" + ); + } +} + +// ── Prepared fetch input bundle ─────────────────────────────────────── + +/// The bundle encodes as two big-endian lengths and two bodies, and its +/// digest is reproducible from the same two component bodies. +#[test] +fn prepared_fetch_input_is_reproducible_from_its_components() { + let bundle = bundle(); + let encoded = bundle.encode().expect("a representable bundle"); + + // The bundle body is the canonical DAG-CBOR of the signed events; + // rebuilding it from the decoded events must give the same bytes. + let transcript = bundle.parts().expect("the canonical bodies parse"); + let reencoded = hellas_rpc::canonical_dag_cbor(&transcript.fetch_input_transcript) + .expect("the decoded events re-encode"); + let manifest_bytes = manifest().canonical_bytes(); + let expected = assemble(&[&reencoded, &manifest_bytes]); + assert_eq!(encoded, expected); + + let decoded = PreparedPaidFetchInputV1::decode(&encoded, 1_048_576).expect("legal bundle"); + assert_eq!(decoded, bundle); + assert_eq!( + decoded + .parts() + .expect("the canonical bodies parse") + .manifest + .canonical_bytes(), + manifest_bytes + ); + assert_eq!( + input_digest(&channel(), &decoded), + input_digest(&channel(), &bundle) + ); +} + +/// Padding, shortening, or re-labelling the bundle rejects rather than +/// producing a second acceptable spelling of the same job. +#[test] +fn prepared_fetch_input_mutations_reject() { + let encoded = bundle().encode().expect("a representable bundle"); + let budget = 1_048_576; + assert!(PreparedPaidFetchInputV1::decode(&encoded, budget).is_ok()); + + // MUTATION: one appended byte. + let mut trailing = encoded.clone(); + trailing.push(0); + let err = PreparedPaidFetchInputV1::decode(&trailing, budget) + .expect_err("a trailing byte must not decode"); + assert!( + err.to_string() + .contains("trailing bytes after prepared fetch input"), + "{err}" + ); + + // MUTATION: the first declared length one byte short. The manifest's + // length is then read out of the transcript's tail and refused before + // anything is allocated for it. + let mut short_length = encoded.clone(); + let first_len = u32::from_be_bytes([ + short_length[0], + short_length[1], + short_length[2], + short_length[3], + ]); + short_length[..4].copy_from_slice(&(first_len - 1).to_be_bytes()); + let err = PreparedPaidFetchInputV1::decode(&short_length, budget) + .expect_err("a shortened length must not decode"); + assert!(err.to_string().contains("budget"), "{err}"); + + // MUTATION: u32::MAX in the first length. + let mut huge = encoded.clone(); + huge[..4].copy_from_slice(&u32::MAX.to_be_bytes()); + let err = PreparedPaidFetchInputV1::decode(&huge, budget) + .expect_err("a u32::MAX length must not decode"); + assert!(err.to_string().contains("budget"), "{err}"); + + // MUTATION: a bundle one byte over its budget; the exact budget is + // legal. + let err = PreparedPaidFetchInputV1::decode(&encoded, encoded.len() - 1) + .expect_err("a bundle over its budget must not decode"); + assert!(err.to_string().contains("budget"), "{err}"); + assert!(PreparedPaidFetchInputV1::decode(&encoded, encoded.len()).is_ok()); + + // MUTATION: the two bodies swapped. The transcript is not a manifest. + let transcript_bytes = &encoded[4..4 + (first_len as usize)]; + let swapped = assemble(&[&manifest().canonical_bytes(), transcript_bytes]); + let err = PreparedPaidFetchInputV1::decode(&swapped, budget) + .expect("lengths are still well formed") + .parts() + .expect_err("swapped components must not parse"); + assert!(!err.to_string().is_empty(), "{err}"); + + // MUTATION: a noncanonical transcript body — the DAG-CBOR array + // header re-spelled with a wider integer width. + let mut widened = transcript_bytes.to_vec(); + assert_eq!(widened[0], 0x88, "the transcript is an eight-element array"); + widened.splice(0..1, [0x98, 0x08]); + let noncanonical = assemble(&[&widened, &manifest().canonical_bytes()]); + let err = PreparedPaidFetchInputV1::decode(&noncanonical, budget) + .expect("lengths are still well formed") + .parts() + .expect_err("a noncanonical transcript must not parse"); + assert!(err.to_string().contains("canonical"), "{err}"); + + // MUTATION: a noncanonical manifest body — trailing garbage inside + // the second segment. + let mut noncanonical_manifest = manifest().canonical_bytes(); + noncanonical_manifest.push(0); + let noncanonical = assemble(&[transcript_bytes, &noncanonical_manifest]); + let err = PreparedPaidFetchInputV1::decode(&noncanonical, budget) + .expect("lengths are still well formed") + .parts() + .expect_err("a noncanonical manifest body must not parse"); + assert!(err.to_string().contains("trailing bytes"), "{err}"); +} + +/// Rebuilds the fetch policy preimage by hand and hashes it through the +/// crate's *other* hashing entry point. +/// +/// Same discipline as `digest_preimages_are_reproducible_by_hand` in +/// `paid_work_vectors.rs`: if the domain, the network's length prefix, +/// the channel argument, or the envelope-plus-body order disagreed with +/// the module's, the two would not meet here. +#[test] +fn fetch_policy_preimage_is_reproducible_by_hand() { + let channel = channel(); + + let mut preimage = b"hellas.work.fetch-policy.v1".to_vec(); + preimage.push(NETWORK.len() as u8); + preimage.extend_from_slice(NETWORK.as_bytes()); + preimage.extend_from_slice(channel.id().as_bytes()); + preimage.extend_from_slice(&fetch_policy().encode()); + assert_eq!( + Digest::hash(&preimage), + fetch_policy_digest(&channel, &fetch_policy()) + ); + + // The route commitment: domain, u32 length prefix, body. + let route_body = route().canonical_body_bytes(); + let mut preimage = b"hellas.work.fetch-route.v1".to_vec(); + preimage.extend_from_slice(&(route_body.len() as u32).to_be_bytes()); + preimage.extend_from_slice(&route_body); + assert_eq!(Digest::hash(&preimage), route_commitment()); + + // The prepared input: domain, network, channel, bundle. + let mut preimage = b"hellas.work.prepared-fetch-input.v1".to_vec(); + preimage.push(NETWORK.len() as u8); + preimage.extend_from_slice(NETWORK.as_bytes()); + preimage.extend_from_slice(channel.id().as_bytes()); + preimage.extend_from_slice(&bundle().encode().expect("a representable bundle")); + assert_eq!(Digest::hash(&preimage), input_digest(&channel, &bundle())); +} + +// ── Authorization ───────────────────────────────────────────────────── + +/// The proposal derives every field the channel, policy, and bundle fix, +/// and the request commitment is the fetch ticket flow's: the verified +/// input transcript's own commitment. +#[test] +fn propose_fetch_authorization_derives_every_field() { + let channel = channel(); + let terms = payment_terms(); + let authorization = authorization(); + + assert_eq!(authorization.channel_id, channel.id()); + assert_eq!(authorization.bond_edge, terms.bond_edge); + assert_eq!(authorization.bond_terms_hash, terms.bond_terms_hash()); + assert_eq!(authorization.payment_edge, channel.payment_edge()); + assert_eq!( + authorization.payment_terms_hash, + channel.payment_terms_hash() + ); + assert_eq!( + authorization.execution_policy_digest, + fetch_policy_digest(&channel, &fetch_policy()) + ); + assert_eq!( + authorization.prepared_input_digest, + input_digest(&channel, &bundle()) + ); + assert_eq!(authorization.proposal_nonce, PROPOSAL_NONCE); + assert_eq!(authorization.acceptance_deadline, 1_000); + assert_eq!(authorization.request_commitment, input_request_commitment()); + assert_eq!( + authorization.environment_commitment, + environment().manifest_id() + ); + assert_eq!(authorization.price, 250); + assert_eq!(authorization.terminal_deadline, 1_050); + assert_eq!(authorization.payment_deadline, 1_100); + + // The signing bytes both parties produce are the authorization's + // `work_id`, and both settlement keys verify over them. + let id = check_fetch_authorization(&channel, &authorization, &fetch_policy(), 900) + .expect("a legal authorization"); + assert_eq!(id, work_id(&channel, &authorization)); + let payload = signing_hash(id); + let verifier = Secp256k1Verifier; + assert!(verifier.verify_sig(client().sign(payload), channel.client_key(), payload)); + assert!(verifier.verify_sig(provider().sign(payload), channel.provider_key(), payload)); +} + +/// The fetch authorization refusals: the wiring of the shared core to +/// this profile's policy, plus the profile's own zero-bound rule. +#[test] +fn check_fetch_authorization_refusals() { + let channel = channel(); + let policy = fetch_policy(); + let base = authorization(); + assert!(check_fetch_authorization(&channel, &base, &policy, 900).is_ok()); + + // MUTATION: a policy whose digest is not the one the authorization + // names. + let mut other_policy = policy; + other_policy.max_output_events += 1; + assert_eq!( + check_fetch_authorization(&channel, &base, &other_policy, 900), + Err(PaidWorkError::Mismatch { + field: "execution_policy_digest" + }) + ); + + // MUTATION: a zero bound. The policy check fires before any + // comparison. + let mut zeroed_policy = policy; + zeroed_policy.max_output_bytes = 0; + let zeroed = propose(&channel, &zeroed_policy, &bundle()); + assert_eq!( + check_fetch_authorization(&channel, &zeroed, &zeroed_policy, 900), + Err(PaidWorkError::PolicyZero { + field: "max_output_bytes" + }) + ); + + // MUTATION: a price that disagrees with the signed policy. + let mut mispriced = base; + mispriced.price = 249; + assert_eq!( + check_fetch_authorization(&channel, &mispriced, &policy, 900), + Err(PaidWorkError::Mismatch { field: "price" }) + ); + + // MUTATION: a price the bond does not cover, honestly proposed under + // a policy that names it. + let mut dear_policy = policy; + dear_policy.fixed_price = 501; + let dear = propose(&channel, &dear_policy, &bundle()); + assert_eq!( + check_fetch_authorization(&channel, &dear, &dear_policy, 900), + Err(PaidWorkError::PriceOutOfRange { + price: 501, + max_job_price: 500 + }) + ); + + // MUTATION: signing after the acceptance window closed. + assert_eq!( + check_fetch_authorization(&channel, &base, &policy, 1_001), + Err(PaidWorkError::AcceptanceExpired { + height: 1_001, + deadline: 1_000 + }) + ); + + // MUTATION: a payment deadline past the channel's admission horizon. + let mut late = base; + late.payment_deadline = 5_000; + assert_eq!( + check_fetch_authorization(&channel, &late, &policy, 900), + Err(PaidWorkError::DeadlineOrder { + acceptance: 1_000, + terminal: 1_050, + payment: 5_000, + horizon: 5_000 + }) + ); +} + +/// Every bound the profile requires to be positive is refused at zero, +/// one field at a time. +#[test] +fn an_absent_fetch_policy_bound_is_refused() { + let base = fetch_policy(); + assert!(check_fetch_policy(&base).is_ok()); + + let zeroed: Vec<(&str, PaidFetchPolicyV1)> = vec![ + ( + "fixed_price", + PaidFetchPolicyV1 { + fixed_price: 0, + ..base + }, + ), + ( + "max_request_body_bytes", + PaidFetchPolicyV1 { + max_request_body_bytes: 0, + ..base + }, + ), + ( + "max_output_events", + PaidFetchPolicyV1 { + max_output_events: 0, + ..base + }, + ), + ( + "max_output_bytes", + PaidFetchPolicyV1 { + max_output_bytes: 0, + ..base + }, + ), + ( + "max_spool_bytes", + PaidFetchPolicyV1 { + max_spool_bytes: 0, + ..base + }, + ), + ( + "max_encoded_result_frame", + PaidFetchPolicyV1 { + max_encoded_result_frame: 0, + ..base + }, + ), + ( + "max_encoded_prepared_input", + PaidFetchPolicyV1 { + max_encoded_prepared_input: 0, + ..base + }, + ), + ( + "dispatch_margin_blocks", + PaidFetchPolicyV1 { + dispatch_margin_blocks: 0, + ..base + }, + ), + ( + "delivery_margin_blocks", + PaidFetchPolicyV1 { + delivery_margin_blocks: 0, + ..base + }, + ), + ( + "oracle_grace_blocks", + PaidFetchPolicyV1 { + oracle_grace_blocks: 0, + ..base + }, + ), + ]; + assert_eq!(zeroed.len(), 10, "every required bound must be zeroed"); + for (field, policy) in zeroed { + assert_eq!( + check_fetch_policy(&policy), + Err(PaidWorkError::PolicyZero { field }), + "{field} was accepted at zero" + ); + } +} + +/// Every fetch-policy field is inside the digest the authorization pins. +#[test] +fn every_fetch_policy_field_moves_its_digest() { + let channel = channel(); + let base = fetch_policy(); + let pinned = fetch_policy_digest(&channel, &base); + + let mutations: Vec<(&str, PaidFetchPolicyV1)> = vec![ + ("allowed_environment", { + let mut m = base; + m.allowed_environment = ContentId::from_bytes([0; 32]); + m + }), + ("route_commitment", { + let mut m = base; + m.route_commitment = Digest::from_bytes([0; 32]); + m + }), + ("max_request_body_bytes", { + let mut m = base; + m.max_request_body_bytes += 1; + m + }), + ("max_output_events", { + let mut m = base; + m.max_output_events += 1; + m + }), + ("max_output_bytes", { + let mut m = base; + m.max_output_bytes += 1; + m + }), + ("max_spool_bytes", { + let mut m = base; + m.max_spool_bytes += 1; + m + }), + ("max_encoded_result_frame", { + let mut m = base; + m.max_encoded_result_frame += 1; + m + }), + ("max_encoded_prepared_input", { + let mut m = base; + m.max_encoded_prepared_input += 1; + m + }), + ("dispatch_margin_blocks", { + let mut m = base; + m.dispatch_margin_blocks += 1; + m + }), + ("delivery_margin_blocks", { + let mut m = base; + m.delivery_margin_blocks += 1; + m + }), + ("oracle_grace_blocks", { + let mut m = base; + m.oracle_grace_blocks += 1; + m + }), + ("fixed_price", { + let mut m = base; + m.fixed_price += 1; + m + }), + ]; + assert_eq!(mutations.len(), 12, "every field must be mutated"); + for (field, mutated) in mutations { + assert_ne!( + fetch_policy_digest(&channel, &mutated), + pinned, + "{field} left the policy digest alone" + ); + } +} + +// ── Prepared input checks ───────────────────────────────────────────── + +/// The happy path, and the key correspondence it stands on: the +/// transcript's caller key and the channel's client settlement key are +/// the one scalar. +#[test] +fn check_prepared_fetch_input_happy_path() { + let channel = channel(); + assert_eq!( + caller_key().public_key(), + PublicKey::Secp256k1(channel.client_key().to_bytes()), + "the fixture caller must be the channel's client" + ); + assert_eq!( + producer_key().public_key(), + PublicKey::Secp256k1(channel.provider_key().to_bytes()), + "the fixture producer must be the channel's provider" + ); + check_prepared_fetch_input( + &channel, + &authorization(), + &fetch_policy(), + &route(), + &bundle(), + ) + .expect("a legal prepared fetch input"); +} + +/// Each binding the check makes is broken on its own, with the bundle +/// digest and the authorization recomputed so the named check is the +/// only thing that can refuse it. +#[test] +fn each_fetch_input_binding_is_checked_on_its_own() { + let channel = channel(); + let policy = fetch_policy(); + let base = authorization(); + assert!(check_prepared_fetch_input(&channel, &base, &policy, &route(), &bundle()).is_ok()); + + // A bundle whose digest is not the one the authorization named. + let mut unbound = base; + unbound.prepared_input_digest = Digest::from_bytes([0x84; 32]); + assert_eq!( + check_prepared_fetch_input(&channel, &unbound, &policy, &route(), &bundle()), + Err(PaidWorkError::Mismatch { + field: "prepared_input_digest" + }) + ); + + // A bundle larger than the policy's prepared-input bound. Refused + // before its digest is even computed. + let encoded = bundle().encode().expect("a representable bundle"); + let cramped = PaidFetchPolicyV1 { + max_encoded_prepared_input: 100, + ..policy + }; + assert_eq!( + check_prepared_fetch_input(&channel, &base, &cramped, &route(), &bundle()), + Err(PaidWorkError::OverEnvelope { + field: "prepared input length", + actual: encoded.len() as u64, + limit: 100 + }) + ); + // The exact length is legal; one byte less is not. + let exact = PaidFetchPolicyV1 { + max_encoded_prepared_input: encoded.len() as u32, + ..policy + }; + assert!(check_prepared_fetch_input(&channel, &base, &exact, &route(), &bundle()).is_ok()); + + // Input events signed by the provider's key rather than the client's: + // a self-consistent chain whose caller is not this channel's client. + let delegated_events = signed_input( + &producer_key(), + Assurance::ProducerSigned, + environment().manifest_id(), + SERVICE, + METHOD, + REQUEST_BODY, + ); + let delegated_bundle = + PreparedPaidFetchInputV1::new(&delegated_events, &manifest()).expect("a legal transcript"); + let delegated = propose(&channel, &policy, &delegated_bundle); + assert_eq!( + check_prepared_fetch_input(&channel, &delegated, &policy, &route(), &delegated_bundle), + Err(PaidWorkError::Mismatch { + field: "caller_key" + }) + ); + + // Another assurance mode under an otherwise legal bundle. + let attested_events = signed_input( + &caller_key(), + Assurance::AppleAppAttest, + environment().manifest_id(), + SERVICE, + METHOD, + REQUEST_BODY, + ); + let attested_bundle = + PreparedPaidFetchInputV1::new(&attested_events, &manifest()).expect("a legal transcript"); + let attested = propose(&channel, &policy, &attested_bundle); + assert_eq!( + check_prepared_fetch_input(&channel, &attested, &policy, &route(), &attested_bundle), + Ok(()) + ); + + // A route body the policy does not commit to: the commitment check + // fires before the route may constrain anything. + let mut rerouted_policy = policy; + rerouted_policy.route_commitment = Digest::from_bytes([0x86; 32]); + assert_eq!( + check_prepared_fetch_input(&channel, &base, &rerouted_policy, &route(), &bundle()), + Err(PaidWorkError::Mismatch { + field: "route_commitment" + }) + ); + + // A sealed route whose service the signed events do not name, with + // the policy's commitment honestly recomputed over it. + let other_service = FetchRoutePolicy::sealed_route("anthropic", METHOD).expect("a legal route"); + let mut resold_policy = policy; + resold_policy.route_commitment = + fetch_route_commitment(&other_service.canonical_body_bytes()).expect("representable"); + assert_eq!( + check_prepared_fetch_input(&channel, &base, &resold_policy, &other_service, &bundle()), + Err(PaidWorkError::Mismatch { field: "service" }) + ); + + // The method, on its own. + let other_method = FetchRoutePolicy::sealed_route(SERVICE, "chat").expect("a legal route"); + let mut resold_policy = policy; + resold_policy.route_commitment = + fetch_route_commitment(&other_method.canonical_body_bytes()).expect("representable"); + assert_eq!( + check_prepared_fetch_input(&channel, &base, &resold_policy, &other_method, &bundle()), + Err(PaidWorkError::Mismatch { field: "method" }) + ); + + // Events that name another environment than the bundle's manifest. + let elsewhere_events = signed_input( + &caller_key(), + Assurance::ProducerSigned, + ContentId::from_bytes([0x99; 32]), + SERVICE, + METHOD, + REQUEST_BODY, + ); + let elsewhere_bundle = + PreparedPaidFetchInputV1::new(&elsewhere_events, &manifest()).expect("a legal transcript"); + let elsewhere = propose(&channel, &policy, &elsewhere_bundle); + assert_eq!( + check_prepared_fetch_input(&channel, &elsewhere, &policy, &route(), &elsewhere_bundle), + Err(PaidWorkError::Mismatch { + field: "manifest content id" + }) + ); + + // An authorization whose environment commitment is not the events'. + let mut reenvironed = base; + reenvironed.environment_commitment = ContentId::from_bytes([0x87; 32]); + assert_eq!( + check_prepared_fetch_input(&channel, &reenvironed, &policy, &route(), &bundle()), + Err(PaidWorkError::Mismatch { + field: "environment_commitment" + }) + ); + + // A policy whose allowed environment is not the manifest's. + let mut rehomed_policy = policy; + rehomed_policy.allowed_environment = ContentId::from_bytes([0x85; 32]); + assert_eq!( + check_prepared_fetch_input(&channel, &base, &rehomed_policy, &route(), &bundle()), + Err(PaidWorkError::Mismatch { + field: "allowed_environment" + }) + ); + + // A request commitment the authorization does not carry. + let mut restamped = base; + restamped.request_commitment = RequestCommitment::from_digest(Digest::from_bytes([0x88; 32])); + assert_eq!( + check_prepared_fetch_input(&channel, &restamped, &policy, &route(), &bundle()), + Err(PaidWorkError::Mismatch { + field: "request_commitment" + }) + ); + + // A request body over the policy's bound. The fetch protocol's own + // 1 MiB bound still holds; the policy's tighter one is what refuses + // here. + let tight = PaidFetchPolicyV1 { + max_request_body_bytes: 2, + ..policy + }; + assert_eq!( + check_prepared_fetch_input(&channel, &base, &tight, &route(), &bundle()), + Err(PaidWorkError::OverEnvelope { + field: "request body", + actual: REQUEST_BODY.len() as u64, + limit: 2 + }) + ); +} + +/// Open Fetch requires its HTTPS interpreter and enforces the signed host/pin contract. +#[test] +fn open_fetch_checks_manifest_host_and_tls_pins() { + let channel = channel(); + let route = FetchRoutePolicy::open_fetch(true, vec!["api.example.com".into()]); + let environment = hellas_rpc::FetchEnvironment::Http; + let mut policy = fetch_policy(); + policy.allowed_environment = environment.manifest_id(); + policy.route_commitment = fetch_route_commitment(&route.canonical_body_bytes()).unwrap(); + let make = |host: &str, pins: Vec| { + let body = serde_json::to_vec(&serde_json::json!({ + "url": format!("https://{host}/v1/messages"), "method":"POST", "body_base64":"e30=", + "tls":{"roots":{"mode":"web_pki"},"spki_sha256":pins}, "max_response_bytes":4096 + })) + .unwrap(); + let events = signed_input( + &caller_key(), + Assurance::AppleAppAttest, + environment.manifest_id(), + SERVICE, + METHOD, + &body, + ); + PreparedPaidFetchInputV1::new(&events, &environment.manifest()).unwrap() + }; + for (host, pins, valid) in [ + ("api.example.com", vec!["01".repeat(32)], true), + ("other.example", vec!["01".repeat(32)], false), + ("api.example.com", vec![], false), + ] { + let bundle = make(host, pins); + let auth = propose(&channel, &policy, &bundle); + assert_eq!( + check_prepared_fetch_input(&channel, &auth, &policy, &route, &bundle).is_ok(), + valid + ); + } + let sealed = bundle(); + let mut old = fetch_policy(); + old.route_commitment = policy.route_commitment; + assert!( + check_prepared_fetch_input( + &channel, + &propose(&channel, &old, &sealed), + &old, + &route, + &sealed + ) + .is_err() + ); +} + +#[test] +fn paid_fetch_result_cannot_downgrade_the_signed_assurance() { + use hellas_rpc::protocol::work_profile::PreparedPaidWorkInput; + let channel = channel(); + let input = signed_input( + &caller_key(), + Assurance::AppleAppAttest, + environment().manifest_id(), + SERVICE, + METHOD, + REQUEST_BODY, + ); + let bundle = PreparedPaidFetchInputV1::new(&input, &manifest()).unwrap(); + let auth = propose(&channel, &fetch_policy(), &bundle); + let prepared = PreparedPaidWorkInput::Fetch(bundle); + let weaker = output_transcript(&auth); + assert!(prepared.terminal_result(&channel, &auth, &weaker).is_err()); + let key = producer_key(); + let builder = FetchOutputTranscriptBuilder::new( + InputCommitment::from_digest(auth.request_commitment.digest()), + Assurance::AppleAppAttest, + &key, + ); + let output = builder + .finish(weaker.last().unwrap().payload().to_vec()) + .unwrap(); + assert!(prepared.terminal_result(&channel, &auth, &output).is_ok()); +} + +// ── The terminal result ─────────────────────────────────────────────── + +/// The golden result: one legal job carried from proposal to a signed +/// result, every digest pinned. +#[test] +fn terminal_fetch_result_is_pinned() { + let channel = channel(); + let authorization = authorization(); + let transcript = output_transcript(&authorization); + let result = terminal_fetch_result( + &channel, + &authorization, + &transcript, + Assurance::ProducerSigned, + ) + .expect("a legal terminal transcript"); + + assert_eq!(result.work_id, work_id(&channel, &authorization)); + assert_eq!( + hex(result.terminal_transcript_commitment.as_bytes()), + "986663facdd944d47bd210bc3379ac5aba2c48e31b7a303ccdee128a455172af" + ); + assert_eq!( + hex(result.canonical_output_digest.as_bytes()), + "d7749786cb4d09c0949760f9cad1c6289a32a3fbfa62217f14c51eba40567975" + ); + assert_eq!( + hex(result_digest(&channel, &result).as_bytes()), + "2d8d52936484533b65b37ed0e6d983eff30a7e8cf0d82c81821879d58888265b" + ); + + // The result record round-trips through the wire codec and the + // profile-agnostic result check accepts it. + let encoded = result.encode(); + assert_eq!( + hellas_rpc::protocol::work::PaidJobResultV1::decode(&encoded), + Ok(result) + ); + assert_eq!( + check_result(&channel, result.work_id, &result).expect("a legal result"), + result_digest(&channel, &result) + ); +} + +/// The result builder's refusals: an empty transcript, a transcript for +/// another request, a transcript the provider did not sign, and a +/// transcript whose last event is not a terminal. +#[test] +fn terminal_fetch_result_refusals() { + let channel = channel(); + let authorization = authorization(); + + // MUTATION: an empty transcript. + assert!(matches!( + terminal_fetch_result(&channel, &authorization, &[], Assurance::ProducerSigned), + Err(PaidWorkError::Transcript(_)) + )); + + // MUTATION: a well-formed fetch transcript answering a different + // request. + let wrong_input = InputCommitment::from_digest(Digest::from_bytes([0x99; 32])); + let key = producer_key(); + let builder = FetchOutputTranscriptBuilder::new(wrong_input, Assurance::ProducerSigned, &key); + let wrong_request = builder.finish(terminal_payload()).unwrap(); + assert!(matches!( + terminal_fetch_result( + &channel, + &authorization, + &wrong_request, + Assurance::ProducerSigned + ), + Err(PaidWorkError::Transcript(_)) + )); + + // MUTATION: the client signs the output. The chain verifies; the + // producer is not the channel's provider. + let input = InputCommitment::from_digest(authorization.request_commitment.digest()); + let key = caller_key(); + let mut builder = FetchOutputTranscriptBuilder::new(input, Assurance::ProducerSigned, &key); + builder.push_event(event_payload("paid ")).unwrap(); + let client_signed = builder.finish(terminal_payload()).unwrap(); + assert_eq!( + terminal_fetch_result( + &channel, + &authorization, + &client_signed, + Assurance::ProducerSigned + ), + Err(PaidWorkError::Mismatch { + field: "transcript producer key" + }) + ); + + // MUTATION: the chain ends in a semantic event rather than a + // terminal. + let key = producer_key(); + let mut builder = hellas_rpc::OutputTranscriptBuilder::new( + scheme_id(Operation::Fetch, Assurance::ProducerSigned), + input, + &key, + output_canonicalization(), + ); + builder + .push("response.event", event_payload("paid ")) + .unwrap(); + let no_terminal = builder.finish().unwrap().0; + assert!(matches!( + terminal_fetch_result( + &channel, + &authorization, + &no_terminal, + Assurance::ProducerSigned + ), + Err(PaidWorkError::Transcript(_)) + )); +} + +/// The normalized fetch answer ignores event boundaries and binds the +/// network and the job. +#[test] +fn fetch_output_digest_normalizes_chunking() { + let network = network(); + let id = work_id(&channel(), &authorization()); + let first = event_payload("paid "); + let second = event_payload("fetch"); + let terminal = terminal_payload(); + let digest = + fetch_canonical_output_digest(network, id, &[first.clone(), second.clone()], &terminal); + + // The same bytes signed as one event rather than two: the boundary + // is not part of the answer. + let mut joined = first.clone(); + joined.extend_from_slice(&second); + assert_eq!( + fetch_canonical_output_digest(network, id, &[joined], &terminal), + digest + ); + + // MUTATION: a different answer. + assert_ne!( + fetch_canonical_output_digest(network, id, &[second, first], &terminal), + digest + ); + + // MUTATION: a different terminal. + let other_terminal = encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .expect("a legal terminal payload"); + assert_ne!( + fetch_canonical_output_digest( + network, + id, + &[event_payload("paid "), event_payload("fetch")], + &other_terminal + ), + digest + ); + + // The same answer for another job, or on another network, is another + // digest. + assert_ne!( + fetch_canonical_output_digest( + network, + Digest::from_bytes([0; 32]), + &[event_payload("paid "), event_payload("fetch")], + &terminal + ), + digest + ); + let other_network = NetworkId::new(OTHER_NETWORK).expect("legal network id"); + assert_ne!( + fetch_canonical_output_digest( + other_network, + id, + &[event_payload("paid "), event_payload("fetch")], + &terminal + ), + digest + ); + + // And the pinned value: the answer bytes are the flattened payload + // stream and nothing else. + assert_eq!( + hex(digest.as_bytes()), + "d7749786cb4d09c0949760f9cad1c6289a32a3fbfa62217f14c51eba40567975" + ); +} + +// ── The profile-agnostic tail ───────────────────────────────────────── + +/// A fetch result pays through the shared payment machinery unchanged: +/// the profile ends at `terminal_fetch_result`, and what consensus +/// settles is built from the same records as an evaluate job's. +#[test] +fn fetch_result_pays_through_the_shared_ledger() { + let channel = channel(); + let authorization = authorization(); + let transcript = output_transcript(&authorization); + let result = terminal_fetch_result( + &channel, + &authorization, + &transcript, + Assurance::ProducerSigned, + ) + .expect("a legal terminal transcript"); + + let (certificate, binding) = + next_payment(&channel, &authorization, &result, 0, capacity()).expect("a legal payment"); + assert_eq!(certificate.earned_cumulative(), 250); + assert_eq!(binding.work_id, work_id(&channel, &authorization)); + assert_eq!(binding.result_digest, result_digest(&channel, &result)); + assert_eq!(binding.certificate_digest, certificate.digest(network())); + + CreditLedger::new() + .credit_payment( + &channel, + &authorization, + &result, + &binding, + &certificate, + capacity(), + ) + .expect("the ledger credits the fetch job's payment"); +} + +// ── The real constructor, end to end ────────────────────────────────── + +/// The randomized fetch constructor's transcript passes the whole paid +/// pipeline: propose, both checks, and the terminal result. +#[test] +fn the_real_input_constructor_passes_the_whole_pipeline() { + let channel = channel(); + let events = build_input_events_with_retention( + SERVICE, + METHOD, + REQUEST_BODY, + environment().manifest_id(), + Assurance::ProducerSigned, + &caller_key(), + Retention::Retain, + ) + .expect("the real constructor builds a legal transcript"); + let bundle = PreparedPaidFetchInputV1::new(&events, &manifest()).expect("a legal transcript"); + let authorization = propose(&channel, &fetch_policy(), &bundle); + + check_fetch_authorization(&channel, &authorization, &fetch_policy(), 900) + .expect("a legal authorization"); + check_prepared_fetch_input(&channel, &authorization, &fetch_policy(), &route(), &bundle) + .expect("a legal prepared input"); + + let transcript = output_transcript(&authorization); + let result = terminal_fetch_result( + &channel, + &authorization, + &transcript, + Assurance::ProducerSigned, + ) + .expect("a legal terminal transcript"); + assert_eq!(result.work_id, work_id(&channel, &authorization)); +} + +// ── Bounds ──────────────────────────────────────────────────────────── + +/// The fetch module's one fixed-record preimage is measured, not assumed, +/// to be under the single-chunk limit — and is actually hashed, on the +/// longest legal network id. +#[test] +fn widest_fetch_preimage_is_measured() { + let longest_network = + NetworkId::new(&"n".repeat(hellas_kernel::MAX_NETWORK_ID_LENGTH)).expect("legal id"); + let encoded_network = 1 + hellas_kernel::MAX_NETWORK_ID_LENGTH; + + let widest = "hellas.work.fetch-policy.v1".len() + + encoded_network + + 32 + + PaidFetchPolicyV1::ENCODED_SIZE; + assert_eq!(widest, 27 + 64 + 32 + 126); + assert_eq!(widest, 249); + assert!(widest < hellas_xet::MIN_CHUNK_SIZE); + + // The variable-body domains are streamed; asserting their digests + // still compute over a multi-chunk body is what makes the choice of + // hasher load-bearing. + let mut big_body = route().canonical_body_bytes(); + big_body.extend_from_slice(&[0xab; 20_000]); + let mut preimage = b"hellas.work.fetch-route.v1".to_vec(); + preimage.extend_from_slice(&(big_body.len() as u32).to_be_bytes()); + preimage.extend_from_slice(&big_body); + assert_eq!( + fetch_route_commitment(&big_body).expect("a representable body"), + Digest::hash(&preimage) + ); + + // The widest fixed preimage must hash rather than panic, so it is + // actually hashed here. + let channel = channel_on(longest_network, EdgeId::from_bytes([0xe1; 32])); + let _ = fetch_policy_digest(&channel, &fetch_policy()); +} + +// ── Profile separation ──────────────────────────────────────────────── + +/// The two profiles share the channel, the authorization record, and the +/// payment tail, and nothing else: each profile's check refuses the other +/// profile's policy, and the digests differ under every body. +#[test] +fn the_two_profiles_do_not_mix() { + let channel = channel(); + let fetch_policy = fetch_policy(); + + // An evaluate execution policy over the same environment is not a + // fetch policy: different domains, different digests. + let evaluate_policy = hellas_rpc::protocol::work::PaidExecutionPolicyV1 { + allowed_environment: fetch_policy.allowed_environment, + generation_policy_digest: Digest::from_bytes([0x41; 32]), + identity_source_digest: Digest::from_bytes([0x42; 32]), + max_prompt_tokens: 8, + max_new_tokens: 64, + max_stop_token_ids: 4, + max_spool_bytes: fetch_policy.max_spool_bytes, + max_encoded_result_frame: fetch_policy.max_encoded_result_frame, + max_encoded_quote_response: fetch_policy.max_encoded_prepared_input, + dispatch_margin_blocks: fetch_policy.dispatch_margin_blocks, + delivery_margin_blocks: fetch_policy.delivery_margin_blocks, + oracle_grace_blocks: fetch_policy.oracle_grace_blocks, + fixed_price: fetch_policy.fixed_price, + }; + assert_ne!( + execution_policy_digest(&channel, &evaluate_policy), + fetch_policy_digest(&channel, &fetch_policy) + ); + + // The fetch authorization names the fetch policy's digest, so the + // evaluate check refuses it on exactly that field. + assert_eq!( + check_authorization(&channel, &authorization(), &evaluate_policy, 900), + Err(PaidWorkError::Mismatch { + field: "execution_policy_digest" + }) + ); + // And the fetch check refuses an authorization that names the + // evaluate policy. + let mut evaluate_signed = authorization(); + evaluate_signed.execution_policy_digest = execution_policy_digest(&channel, &evaluate_policy); + assert_eq!( + check_fetch_authorization(&channel, &evaluate_signed, &fetch_policy, 900), + Err(PaidWorkError::Mismatch { + field: "execution_policy_digest" + }) + ); +} diff --git a/crates/rpc/tests/paid_work_vectors.rs b/crates/rpc/tests/paid_work_vectors.rs index a405f588..5f2e5e7d 100644 --- a/crates/rpc/tests/paid_work_vectors.rs +++ b/crates/rpc/tests/paid_work_vectors.rs @@ -431,7 +431,8 @@ fn envelope_and_length_mutations_reject() { }) ); - // MUTATION: an unassigned tag. + // MUTATION: a tag this record does not own (5 is the fetch profile's + // policy tag). let mut unknown_tag = bytes.clone(); unknown_tag[1] = 5; assert_eq!( diff --git a/crates/rpc/tests/work_setup.rs b/crates/rpc/tests/work_setup.rs index 197653c0..4dc4fc98 100644 --- a/crates/rpc/tests/work_setup.rs +++ b/crates/rpc/tests/work_setup.rs @@ -116,7 +116,7 @@ fn config() -> WorkChannelConfig { payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: payment_values(), } } @@ -375,7 +375,12 @@ fn a_descriptor_opens_only_against_its_own_committed_policy() { ); let mut zero_price = config(); - zero_price.execution_policy.fixed_price = 0; + let hellas_rpc::protocol::work_profile::PaidWorkPolicy::Evaluate(policy) = + &mut zero_price.execution_policy + else { + panic!("fixture uses Evaluate"); + }; + policy.fixed_price = 0; assert_eq!( WorkChannelDescriptor::open(zero_price), Err(WorkSetupError::Record(PaidWorkError::PolicyZero { diff --git a/crates/sdk/Cargo.toml b/crates/sdk/Cargo.toml index 997fc1c6..bd954b32 100644 --- a/crates/sdk/Cargo.toml +++ b/crates/sdk/Cargo.toml @@ -20,7 +20,6 @@ apple-verifier = ["hellas-attestation/apple-app-attest"] gateway = ["dep:hellas-gateway"] provider = [ "client", - "dep:anyhow", "dep:hellas-executor", "dep:hellas-providers", "dep:iroh", @@ -30,20 +29,56 @@ provider = [ ] local-control = ["dep:tokio", "hellas-rpc/host-control", "hellas-wire/unix"] +paid-client = ["work", "client", "dep:async-stream", "hellas-client/work"] +paid-provider = ["work", "provider"] +paid-gateway = ["paid-client", "gateway", "dep:rand"] +# Shared channel configuration and provisioning, independent of execution servers. +work = [ + "dep:iroh", + "hellas-wire/iroh", + "dep:tokio", + "dep:tracing", + "dep:hellas-chain", + "hellas-chain/client", + "hellas-chain/work-watcher", + "dep:hellas-kernel", + "hellas-kernel/secp256k1", + "dep:hellas-work", + "dep:hellas-private", + "hellas-rpc/work", + "dep:futures", + "dep:serde", + "dep:serde_json", + "dep:hex", + "dep:url", +] + [dependencies] -anyhow = { workspace = true, optional = true } +async-stream = { workspace = true, optional = true } +futures = { workspace = true, optional = true } futures-core = { workspace = true, optional = true } hellas-attestation.workspace = true +hellas-chain = { workspace = true, default-features = false, optional = true } hellas-client = { workspace = true, optional = true } hellas-executor = { workspace = true, default-features = false, optional = true } hellas-gateway = { workspace = true, optional = true } +hellas-kernel = { workspace = true, default-features = false, optional = true } +hellas-private = { workspace = true, optional = true } hellas-providers = { workspace = true, optional = true } hellas-rpc = { workspace = true, features = ["execute", "fetch", "courtesy"] } hellas-wire = { workspace = true, features = ["mux"] } +hellas-work = { workspace = true, optional = true } +hex = { workspace = true, optional = true } iroh = { workspace = true, features = ["tls-ring"], optional = true } +rand = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } tempfile = { workspace = true } +thiserror.workspace = true tokio = { workspace = true, features = ["net", "sync"], optional = true } tracing = { workspace = true, optional = true } +url = { version = "2", optional = true } [dev-dependencies] -tokio.workspace = true +bytes.workspace = true +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/sdk/src/lib.rs b/crates/sdk/src/lib.rs index b94aa36f..2bf8d168 100644 --- a/crates/sdk/src/lib.rs +++ b/crates/sdk/src/lib.rs @@ -24,7 +24,18 @@ pub use hellas_gateway as gateway; #[cfg(feature = "provider")] mod provider; #[cfg(feature = "provider")] -pub use provider::{OpenAiProviderOptions, ProviderHandle, start_openai_provider}; +pub use hellas_executor::{FetchRoute, FetchRouteEntry, FetchRoutePolicy, FetchRouteRegistry}; +#[cfg(feature = "work")] +pub use hellas_kernel as kernel; +#[cfg(feature = "provider")] +pub use hellas_providers::{ + HttpProviderConfig, OpenAiResponsesFetchProvider, ResponsesFetchAdaptorFactory, +}; +#[cfg(feature = "provider")] +pub use provider::{ + FetchProviderOptions, OpenAiProviderOptions, ProviderError, ProviderHandle, + start_fetch_provider, start_openai_provider, +}; #[cfg(feature = "client")] pub use remote::{ClientIdentity, HellasClient, RemoteFetchRequest}; @@ -36,3 +47,20 @@ pub mod local { pub use hellas_rpc::cache::control::CacheController; pub use hellas_wire::local::{LOCAL_MUX_SLOTS, LocalControlServer, connect, transport}; } + +#[cfg(feature = "paid-provider")] +pub mod paid_provider; +#[cfg(feature = "work")] +pub mod work_config; + +#[cfg(feature = "paid-client")] +pub mod paid_client; + +#[cfg(feature = "work")] +pub mod work_provision; + +#[cfg(all(test, any(feature = "paid-client", feature = "paid-provider")))] +mod test_support; + +#[cfg(feature = "paid-gateway")] +pub mod paid_gateway; diff --git a/crates/sdk/src/paid_client.rs b/crates/sdk/src/paid_client.rs new file mode 100644 index 00000000..1cf2db27 --- /dev/null +++ b/crates/sdk/src/paid_client.rs @@ -0,0 +1,1262 @@ +//! Reusable paid-work sessions: provider authentication, admission, verified delivery, +//! journal recovery, payment, and finalized settlement. Journals are client-owned. +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; +use tracing::Instrument as _; + +mod error; +pub use error::PaidClientError; +type Result = std::result::Result; +use hellas_chain::client::VerifiedRemoteLightClient; +use hellas_chain::{ + ConsensusInfo, ConsensusVerifier, FinalizedWorkView as _, WorkBlocks, WorkChannelQuery, +}; +use hellas_client::work::payment::pay_for_result; +use hellas_kernel::{ + EdgeId, Funding, MAX_START_VALIDITY_BLOCKS, Secp256k1Signer, Secp256k1Verifier, + WorkPaymentTerms, +}; +use hellas_rpc::protocol::artifacts::{Canonical as _, PreparedPaidInputV1}; +use hellas_rpc::protocol::work::{ + JobDeadlines, generation_policy_digest, identity_source_digest, private_policy_commitment, +}; +use hellas_rpc::protocol::work_profile::{PaidWorkPolicy, PreparedPaidWorkInput}; +use hellas_rpc::protocol::work_setup::{ProviderChannelPolicy, WorkChannelDescriptor}; +use hellas_wire::ServiceMarker; +use hellas_wire::iroh::IrohTransport; +use hellas_work::work::{ + ClientChannel as _, ClientEndpoint, ClientObserver, ClientService, JobProposal, fetch_result, + propose_work, resume_work_proposal, +}; +use hellas_work::work_close::FinalizedBlocks as _; +use hellas_work::work_handshake::{ + PaymentAdmission, SetupEndpoint, SetupService, apply_setup_exchange, prepare_setup_exchange, + send_setup_exchange, +}; +use hellas_work::work_open::{SetupAdvance, SetupProgress}; +use hellas_work::work_store::journal::MAX_RECORD_BYTES; +use hellas_work::work_store::{Role, SetupScan, SetupStore}; +use iroh::endpoint::presets; +use iroh::{Endpoint, EndpointAddr, EndpointId, SecretKey, TransportAddr}; + +use crate::work_config::WorkConfig; + +/// Options shared by all jobs on a paid channel. +#[derive(Clone)] +pub struct PaidWorkOptions { + pub config: WorkConfig, + pub journal_root: PathBuf, + pub provider: EndpointId, + pub provider_addrs: Vec, + pub provider_trust: Option, + pub bond: EdgeId, + pub payment_funding: Funding, + pub omission_bond: u64, + pub acceptance_blocks: u64, + pub terminal_blocks: u64, + pub payment_blocks: u64, + /// Bounds retries while a provider is not ready. A session's overall lifetime + /// is controlled by its caller; `run_paid_work` also enforces an overall limit. + pub timeout: Duration, +} + +/// Convenience options for one complete paid job. +pub struct PaidWorkRun { + pub config: WorkConfig, + pub journal_root: PathBuf, + pub provider: EndpointId, + pub provider_addrs: Vec, + pub provider_trust: Option, + pub bond: EdgeId, + pub payment_funding: Funding, + pub omission_bond: u64, + pub prepared_input: PreparedPaidWorkInput, + pub acceptance_blocks: u64, + pub terminal_blocks: u64, + pub payment_blocks: u64, + pub timeout: Duration, + pub settle: bool, +} + +pub struct PaidWorkResult { + pub transcript: Vec, + pub work_id: hellas_rpc::Digest, + pub credited_cumulative: u64, + pub job_price: u64, + pub provider_key: hellas_rpc::PublicKey, + pub input: hellas_rpc::InputCommitment, + pub settled_provider_payout: Option, +} + +/// Executes one job using a client-owned journal, with an overall time limit. +pub async fn run_paid_work( + args: PaidWorkRun, + transport_key: SecretKey, + settlement_key: Secp256k1Signer, +) -> Result { + if args.timeout.is_zero() { + return Err(PaidClientError::InvalidOptions("timeout must be positive")); + } + tokio::time::timeout(args.timeout, async move { + let PaidWorkRun { + config, + journal_root, + provider, + provider_addrs, + provider_trust, + bond, + payment_funding, + omission_bond, + prepared_input, + acceptance_blocks, + terminal_blocks, + payment_blocks, + timeout, + settle, + } = args; + let options = PaidWorkOptions { + config, + journal_root, + provider, + provider_addrs, + provider_trust, + bond, + payment_funding, + omission_bond, + acceptance_blocks, + terminal_blocks, + payment_blocks, + timeout, + }; + check_request( + &options.config.provider_policy(), + &prepared_input, + options.provider_trust.as_ref(), + hellas_rpc::PublicKey::Secp256k1(settlement_key.party_key().to_bytes()), + )?; + let endpoint = bind_paid_endpoint(transport_key).await?; + let mut session = PaidWorkSession::open(options, endpoint.clone(), settlement_key).await?; + let mut result = session + .run(Some(prepared_input), false, None) + .await? + .ok_or(PaidClientError::MissingState( + "paid execution returned no result", + ))?; + if settle { + result.settled_provider_payout = Some(session.settle().await?); + } + session.shutdown().await; + endpoint.close().await; + Ok(result) + }) + .await + .map_err(|_| PaidClientError::Timeout { stage: "paid job" })? +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct InputIdentities { + pub allowed_environment: hellas_rpc::ContentId, + pub generation_policy_digest: hellas_rpc::Digest, + pub identity_source_digest: hellas_rpc::Digest, +} + +impl InputIdentities { + pub fn from_prepared(prepared: &PreparedPaidInputV1) -> Result { + let parts = prepared.parts()?; + Self::from_parts(&parts) + } + + fn from_parts(parts: &hellas_rpc::protocol::artifacts::PreparedPaidInputParts) -> Result { + let allowed_environment = parts.manifest.content_id(); + if parts.evaluate_request.execution_environment != allowed_environment { + return Err(PaidClientError::InputMismatch("environment manifest")); + } + Ok(Self { + allowed_environment, + generation_policy_digest: generation_policy_digest( + &parts.text_policy.canonical_bytes(), + )?, + identity_source_digest: identity_source_digest( + &parts.identity_artifact.canonical_bytes(), + )?, + }) + } +} + +/// A funded client channel. Keep one session per journal and serialize its jobs. +/// Cancellation retains recovery state; call `run(None, true, None)` to resume it. +pub struct PaidWorkSession { + args: PaidWorkOptions, + descriptor: WorkChannelDescriptor, + dialer: ProviderDialer, + client: ClientService, + observer: Option>, + needs_recovery: bool, +} + +impl Drop for PaidWorkSession { + fn drop(&mut self) { + let _ = self.client.observer().suspend(); + if let Some(observer) = &self.observer { + observer.abort(); + } + } +} + +fn spawn_observer( + client: ClientObserver, + descriptor: WorkChannelDescriptor, + config: WorkConfig, + mut chain: WorkBlocks, + mut next_validator: usize, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + loop { + if client + .with_state(|state| state.close_settled().is_some()) + .unwrap_or(false) + { + break; + } + let started = hellas_work::work::ObservationTime::now(); + let span = hellas_rpc::request_span!(target: "hellas_request", parent: None, "paid.channel.observe", hellas.channel.role = "client"); + let result = tokio::time::timeout( + config.max_observation_age, + async { + let mut driver = client.drive()?; + driver.advance_close(&chain, &chain).await?; + let ready = ready_channel(&descriptor, &chain).await?; + if ready + .check_caught_up(client.with_state(|state| state.cursor().0)?) + .is_err() + { + driver.catch_up_to(&chain, ready.finalized_height()).await?; + } + client.observe_ready(ready, started, config.max_observation_age)?; + Ok::<_, PaidClientError>(()) + } + .instrument(span), + ) + .await + .unwrap_or(Err(PaidClientError::Timeout { + stage: "channel observer", + })); + if let Err(error) = result { + let _ = client.suspend(); + tracing::debug!(%error, "paid client observation unavailable"); + // Redial a bounded connection attempt; close duties remain armed. + if !matches!( + error, + PaidClientError::WorkSetup(_) | PaidClientError::Endpoint(_) + ) && let Ok(Ok(replacement)) = + tokio::time::timeout(config.max_observation_age, async { + let replacement = connect_chain(&config, &mut next_validator).await?; + check_genesis(&config, &replacement).await?; + Ok::<_, PaidClientError>(replacement) + }) + .await + { + chain = replacement; + } + } + tokio::time::sleep(config.poll).await; + } + }) +} + +fn check_payment_window(client: &ClientService, work_id: hellas_rpc::Digest) -> Result<()> { + client.with_state(|state| { + let job = state + .job_by_id(work_id) + .ok_or(PaidClientError::MissingState("accepted job disappeared"))?; + if state.cursor().0 > job.authorization().payment_deadline { + Err(PaidClientError::PaymentExpired) + } else { + Ok(()) + } + })? +} + +impl PaidWorkSession { + pub async fn open( + args: PaidWorkOptions, + endpoint: Endpoint, + settlement_key: Secp256k1Signer, + ) -> Result { + if !(args.acceptance_blocks > 0 && args.terminal_blocks > 0 && args.payment_blocks > 0) { + return Err(PaidClientError::InvalidOptions( + "deadline spans must be positive", + )); + } + if args.timeout.is_zero() { + return Err(PaidClientError::InvalidOptions("timeout must be positive")); + } + + let config = &args.config; + // Reject impossible funding before signing either setup transaction. + let settlement = hellas_kernel::work_payment_settlement( + config.expected_payment_values, + args.omission_bond, + ) + .ok_or(hellas_rpc::protocol::work_setup::WorkSetupError::Unsettleable)?; + hellas_rpc::protocol::work_setup::check_collateral( + args.omission_bond, + settlement.capacity(), + )?; + let policy = config.provider_policy(); + let bond = args.bond; + let payment_funding = args.payment_funding.clone(); + let mut next_validator = 0; + let chain = connect_chain(config, &mut next_validator).await?; + check_genesis(config, &chain).await?; + + std::fs::create_dir_all(&args.journal_root).map_err(|source| { + PaidClientError::JournalDirectory { + path: args.journal_root.clone(), + source, + } + })?; + let store = SetupStore::open( + &args.journal_root, + config.chain.network, + bond, + Role::Client, + &Secp256k1Verifier::new(), + )?; + let mut setup = SetupEndpoint::new( + store, + settlement_key.clone(), + PaymentAdmission::Proposes(Box::new(policy.clone())), + ); + let dialer = ProviderDialer::new( + args.provider, + args.provider_addrs.clone(), + endpoint, + args.provider_trust.clone(), + ); + + if setup.state().revision().is_none() { + exchange_setup(&dialer, &mut setup).await?; + } + let bundle = setup + .state() + .bundle() + .cloned() + .ok_or(PaidClientError::MissingState( + "provider returned no bond proposal", + ))?; + if bundle.bond_edge() != bond { + return Err(PaidClientError::InputMismatch("bond edge")); + } + if bundle.bond_terms().parties.taker() != settlement_key.party_key() { + return Err(PaidClientError::InputMismatch("client settlement identity")); + } + dialer.require_producer(hellas_rpc::PublicKey::Secp256k1( + bundle.bond_terms().parties.maker().to_bytes(), + ))?; + if setup.state().scan_armed().is_none() { + setup.arm_scan(finalized_floor(&chain).await?)?; + } + if setup.state().revision() == Some(1) { + let terms = payment_terms(config, &policy, &bundle, args.omission_bond); + setup.propose_payment(payment_funding, terms)?; + } + if setup.state().revision() == Some(2) { + exchange_setup(&dialer, &mut setup).await?; + } + if setup.state().revision() != Some(3) { + return Err(PaidClientError::MissingState( + "countersigned setup revision", + )); + } + + let setup_service = SetupService::new(setup); + let (mounted, descriptor) = + drive_setup(&setup_service, &policy, &chain, config.poll).await?; + let started = hellas_work::work::ObservationTime::now(); + let ready = match ready_channel(&descriptor, &chain).await { + Ok(ready) => Some(ready), + Err(PaidClientError::WorkSetup(_)) => None, + Err(error) => return Err(error), + }; + let initially_ready = ready.is_some(); + let mut endpoint = ClientEndpoint::recover(mounted, settlement_key)?; + endpoint.catch_up(&chain).await?; + let client = ClientService::new(endpoint); + if let Some(ready) = ready { + client + .observer() + .observe_ready(ready, started, config.max_observation_age)?; + } + let observer = spawn_observer( + client.observer(), + descriptor.clone(), + config.clone(), + chain, + next_validator, + ); + + let session = Self { + args, + descriptor, + dialer, + client, + observer: Some(observer), + needs_recovery: true, + }; + if initially_ready { + // A long restart catch-up may outlive the first observation. Wait + // during channel opening; requests never perform this catch-up. + // The session already owns the task, so cancellation aborts it. + tokio::time::timeout(session.args.timeout, async { + while session.client.readiness().is_err() + && !session.client.with_state(|state| state.is_closing())? + { + tokio::time::sleep(session.args.config.poll).await; + } + Ok::<_, PaidClientError>(()) + }) + .await + .map_err(|_| PaidClientError::Timeout { + stage: "initial channel observation", + })??; + } + Ok(session) + } + + /// Stops observation and joins the task before releasing the journal. + /// Call after draining jobs; retained accounting is recovered on restart. + pub async fn shutdown(&mut self) { + let _ = self.client.observer().suspend(); + if let Some(observer) = self.observer.take() { + observer.abort(); + let _ = observer.await; + } + } + + /// Identifies the funded channel and its admitted execution policy. + pub fn descriptor(&self) -> &WorkChannelDescriptor { + &self.descriptor + } + + /// Reads local journal state under a short lock. + pub fn with_state( + &self, + read: impl FnOnce(&hellas_work::work_store::ChannelState) -> R, + ) -> Result { + Ok(self.client.with_state(read)?) + } + + /// True after reopening a journal or an interrupted request. + pub fn needs_recovery(&self) -> bool { + self.needs_recovery + } + + /// Opens a close; the independent observer drives its finalized payout. + /// There is no deadline here: with the observer stopped or validators + /// unreachable this waits as long as the process runs. Callers that need + /// a bound should wrap it in one. + pub async fn settle(&mut self) -> Result { + if !self.client.with_state(|state| state.is_closing())? { + self.client.with_client(|client| client.prepare_close())??; + } + loop { + if let Some(settlement) = self.client.with_state(|state| state.close_settled())? { + return Ok(settlement.provider_payout); + } + tokio::time::sleep(self.args.config.poll).await; + } + } + + /// Runs a request and pays only after verifying its complete result. + /// With `recover`, resume payable journaled work first and admit this as a new + /// job; otherwise reuse a still-active job matching the input. `None` only + /// performs recovery. Prefixes are authenticated before incremental delivery. + pub async fn run( + &mut self, + prepared: Option, + recover: bool, + progress: Option, + ) -> Result> { + self.run_with_admission(prepared, recover, progress, None) + .await + } + + /// Like `run`, also marking `proposed` before a proposal can leave the process. + /// A caller using cancellation must continue payment/recovery once it is set. + pub async fn run_with_admission( + &mut self, + prepared: Option, + recover: bool, + progress: Option, + proposed: Option<&AtomicBool>, + ) -> Result> { + let Self { + args, + descriptor, + dialer, + client, + needs_recovery, + .. + } = self; + let config = &args.config; + if let Some(prepared) = prepared.as_ref() { + check_request( + &config.provider_policy(), + prepared, + dialer.trust.as_ref(), + hellas_rpc::PublicKey::Secp256k1(descriptor.channel().client_key().to_bytes()), + )?; + } + if recover + && *needs_recovery + && let Some(work_id) = + client.with_state(|state| state.last_payment().map(|payment| payment.work_id))? + { + // A lost acknowledgement never requires fresh admission: these + // exact certificate bytes are already durable on the client. + pay_for_result(dialer.work().await?, client, work_id).await?; + } + if client.with_state(|state| state.is_closing())? { + return if prepared.is_none() { + Ok(None) + } else { + Err(hellas_work::work::EndpointError::NotAdmitting.into()) + }; + } + let ready = client.readiness()?; + if recover && *needs_recovery { + enum RecoveredJob { + /// The result was delivered, verified and committed before the + /// restart; only its payment is outstanding, and settling it + /// needs none of the payload a Fetch journal strips. + Delivered(hellas_rpc::protocol::Digest), + /// No payload and no result: the job can neither execute nor + /// settle. Its evidence is retained until the payment deadline + /// filters it out, without blocking every later request. + Unrecoverable(hellas_rpc::protocol::Digest), + Replay( + hellas_rpc::protocol::Digest, + hellas_work::work_store::JobPhase, + PreparedPaidWorkInput, + ), + } + let pending = client.with_state(|state| { + state + .jobs() + .filter(|job| { + // The journal forbids signing payment after this height. + // Keep the evidence, but do not let an unpayable old job + // prevent this channel from serving a new request. Retained + // certificates are re-sent separately above. + let payable = job.authorization().payment_deadline >= state.cursor().0; + if !payable { + tracing::info!( + work_id = %hex::encode(job.work_id().as_bytes()), + payment_deadline = job.authorization().payment_deadline, + "retaining expired unpaid job without retrying execution", + ); + } + payable + }) + .map(|job| { + // Fetch journals retain accounting only. A restart cannot + // reconstruct a lost request or authorize another + // execution, but an already delivered result still + // settles: signing payment needs no payload. + if job.prepared_input().is_empty() { + return Ok(if job.result().is_some() { + RecoveredJob::Delivered(job.work_id()) + } else { + RecoveredJob::Unrecoverable(job.work_id()) + }); + } + PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) + .map(|input| RecoveredJob::Replay(job.work_id(), job.phase(), input)) + .map_err(PaidClientError::from) + }) + .collect::, _>>() + })??; + for recovered in pending { + let (work_id, phase, pending) = match recovered { + RecoveredJob::Delivered(work_id) => { + pay_for_result(dialer.work().await?, client, work_id).await?; + continue; + } + RecoveredJob::Unrecoverable(work_id) => { + tracing::warn!( + %work_id, + "retaining an unpaid job whose request payload is lost; \ + it can neither execute nor settle before its payment deadline" + ); + continue; + } + RecoveredJob::Replay(work_id, phase, input) => (work_id, phase, input), + }; + check_request( + &config.provider_policy(), + &pending, + dialer.trust.as_ref(), + hellas_rpc::PublicKey::Secp256k1(descriptor.channel().client_key().to_bytes()), + )?; + let result = execute_paid_job( + args, + pending, + dialer, + client, + &ready, + config.poll, + None, + JobLookup::Retained(work_id), + None, + ) + .await; + if let Err(error) = &result + && ((phase == hellas_work::work_store::JobPhase::HalfSigned + && matches!( + error, + PaidClientError::Propose(hellas_work::work::ProposeError::Refused { + refusal, + .. + }) if !refusal.is_retryable() + )) + || permanently_refused_delivery(error)) + { + // Keep the signed evidence without deciding that an unpaid + // job was paid or cancelled. A permanent provider refusal + // cannot be repaired by blocking every later request here. + tracing::info!(%work_id, %error, "retaining an unpaid job refused by the provider"); + continue; + } + result?; + } + } + // Keep recovery armed across any error or cancellation after acceptance. + // A caller that declined recovery keeps the flag for a later run. + if recover { + *needs_recovery = prepared.is_some(); + } + let result = match prepared { + Some(prepared) => { + let ready = client.readiness()?; + Some( + execute_paid_job( + args, + prepared, + dialer, + client, + &ready, + config.poll, + progress.as_ref(), + if recover { + JobLookup::New + } else { + JobLookup::PreparedInput + }, + proposed, + ) + .await?, + ) + } + None => None, + }; + if recover { + *needs_recovery = false; + } + Ok(result) + } +} + +fn permanently_refused_delivery(error: &PaidClientError) -> bool { + use hellas_client::work::CollectResultError; + use hellas_work::work::DeliverError; + let delivery = match error { + PaidClientError::Deliver(error) + | PaidClientError::Collect(CollectResultError::Deliver(error)) => Some(error), + _ => None, + }; + matches!(delivery, Some(DeliverError::Refused { refusal, .. }) if !refusal.is_retryable()) +} + +/// Proposes until a provider accepts or the caller's execution window closes. +/// A provider catching its chain cursor up replies `NotReady`; that is not an +/// answer to the job. The retained proposal makes each retry the same request, +/// while bounded exponential backoff avoids turning recovery into a request +/// flood. +async fn propose_when_ready( + dialer: &ProviderDialer, + client: &mut ClientService, + proposal: &JobProposal, + retained: Option, + poll: Duration, + timeout: Duration, + proposed: Option<&AtomicBool>, +) -> Result { + let deadline = Instant::now() + timeout; + let mut delay = poll.max(Duration::from_secs(1)); + loop { + if Instant::now() >= deadline { + return Err(PaidClientError::Timeout { + stage: "provider admission", + }); + } + let transport = dialer.work().await?; + // Once a proposal can leave this process, a lost acknowledgement must + // be treated as accepted work. HTTP cancellation may no longer stop it. + if let Some(proposed) = proposed { + proposed.store(true, Ordering::Release); + } + let result = match retained { + Some(work_id) => resume_work_proposal(transport, client, work_id).await, + None => propose_work(transport, client, proposal).await, + }; + match result { + Ok(work_id) => return Ok(work_id), + Err(hellas_work::work::ProposeError::Refused { refusal, .. }) + if refusal.is_retryable() => + { + let remaining = deadline.saturating_duration_since(Instant::now()); + tokio::time::sleep(delay.min(remaining)).await; + delay = delay.saturating_mul(2).min(Duration::from_secs(10)); + } + Err(error) => return Err(error.into()), + } + } +} + +enum JobLookup { + Retained(hellas_rpc::Digest), + PreparedInput, + New, +} + +#[allow(clippy::too_many_arguments)] +async fn execute_paid_job( + args: &PaidWorkOptions, + prepared: PreparedPaidWorkInput, + dialer: &ProviderDialer, + client: &mut ClientService, + ready: &hellas_rpc::protocol::work_setup::ReadyChannel, + poll: Duration, + progress: Option<&hellas_work::work::PaidProgress>, + lookup: JobLookup, + proposed: Option<&AtomicBool>, +) -> Result { + let readiness_timeout = args.timeout; + let prepared_bytes = prepared.encode()?; + let current = client.with_state(|state| state.cursor().0)?; + let deadlines = deadlines( + current, + args.acceptance_blocks, + args.terminal_blocks, + args.payment_blocks, + )?; + let proposal = JobProposal { + prepared_input: prepared, + deadlines, + }; + let existing = client.with_state(|state| { + state + .jobs() + .filter(|job| match &lookup { + JobLookup::Retained(work_id) => job.work_id() == *work_id, + JobLookup::New => false, + JobLookup::PreparedInput => { + job.prepared_input() == prepared_bytes.as_slice() + && job.authorization().payment_deadline >= current + && (job.phase() != hellas_work::work_store::JobPhase::HalfSigned + || job.authorization().acceptance_deadline >= current) + } + }) + .map(|job| (job.work_id(), job.phase(), *job.authorization())) + .collect::>() + })?; + if existing.len() > 1 { + return Err(PaidClientError::AmbiguousRecovery); + } + if !existing.is_empty() + && let Some(proposed) = proposed + { + proposed.store(true, Ordering::Release); + } + let (work_id, already_collected) = match existing.first().copied() { + Some((work_id, hellas_work::work_store::JobPhase::HalfSigned, _)) => ( + propose_when_ready( + dialer, + client, + &proposal, + Some(work_id), + poll, + readiness_timeout, + proposed, + ) + .await?, + false, + ), + Some((work_id, hellas_work::work_store::JobPhase::Ready, _)) + | Some((work_id, hellas_work::work_store::JobPhase::Matched, _)) => (work_id, true), + Some((work_id, _, _)) => (work_id, false), + None => ( + propose_when_ready( + dialer, + client, + &proposal, + None, + poll, + readiness_timeout, + proposed, + ) + .await?, + false, + ), + }; + let transcript = if already_collected { + client + .with_state(|state| { + state + .job_by_id(work_id) + .map(|job| job.transcript().to_vec()) + })? + .ok_or(PaidClientError::MissingState( + "collected job disappeared from its journal", + ))? + } else if progress.is_some() + || matches!(proposal.prepared_input, PreparedPaidWorkInput::Fetch(_)) + { + let mut emitted = false; + let delivery = loop { + let result = hellas_work::work::fetch_result_stream( + dialer.work().await?, + client, + ready, + work_id, + |event| { + // Output is only exposed when a user stream observes it; + // recovery without one may safely retry after any event. + if let Some(progress) = progress { + emitted = true; + progress(event.clone()).map_err(|error| { + hellas_rpc::protocol::work::PaidWorkError::Transcript(error.to_string()) + })?; + } + Ok(()) + }, + ) + .await; + let error = match result { + Ok(delivery) => break delivery, + Err(error) => error, + }; + let retryable = match &error { + hellas_work::work::DeliverError::Transport(status) => { + status.code == hellas_wire::WireCode::Unavailable + } + hellas_work::work::DeliverError::Refused { refusal, .. } => refusal.is_retryable(), + _ => false, + }; + // Retry delivery of this accepted job only before exposing output. + // Reopening after a prefix would replay it into the user's stream. + if emitted || !retryable { + return Err(error.into()); + } + check_payment_window(client, work_id)?; + tracing::debug!(%error, %work_id, "waiting for paid result stream readiness"); + tokio::time::sleep(poll.max(Duration::from_secs(1))).await; + }; + check_payment_window(client, work_id)?; + delivery.transcript + } else { + collect_until_ready(dialer, client, ready, work_id, poll).await? + }; + let credited = pay_for_result(dialer.work().await?, client, work_id).await?; + Ok(PaidWorkResult { + work_id, + job_price: ready.execution_policy().fixed_price(), + credited_cumulative: credited, + transcript, + provider_key: hellas_rpc::PublicKey::Secp256k1(ready.channel().provider_key().to_bytes()), + input: proposal.prepared_input.input_commitment()?, + settled_provider_payout: None, + }) +} + +fn check_request( + policy: &ProviderChannelPolicy, + prepared: &PreparedPaidWorkInput, + trust: Option<&hellas_client::ProviderTrustAnchor>, + caller: hellas_rpc::PublicKey, +) -> Result<()> { + let assurance = prepared.assurance()?; + if trust + .as_ref() + .is_some_and(|trust| trust.required_assurance != assurance) + { + return Err(PaidClientError::InputMismatch( + "assurance differs from provider trust", + )); + } + if assurance != hellas_rpc::Assurance::ProducerSigned && trust.is_none() { + return Err(PaidClientError::InvalidOptions( + "attested work requires a provider trust anchor", + )); + } + match (prepared, &policy.execution_policy) { + (PreparedPaidWorkInput::Evaluate(input), PaidWorkPolicy::Evaluate(_)) => { + check_evaluate_input(policy, input)?; + } + (PreparedPaidWorkInput::Fetch(input), PaidWorkPolicy::Fetch { policy, route }) => { + let parts = input.parts()?; + let request = hellas_rpc::fetch::verify_input_events(&parts.fetch_input_transcript)?; + if request.caller_key != caller { + return Err(PaidClientError::InputMismatch("Fetch caller")); + } + if !(request.execution_environment == policy.allowed_environment + && parts.manifest.content_id() == policy.allowed_environment) + { + return Err(PaidClientError::InputMismatch("Fetch environment")); + } + if request.retention != hellas_rpc::Retention::Ephemeral { + return Err(PaidClientError::InputMismatch( + "Fetch retention must be ephemeral", + )); + } + if let hellas_rpc::protocol::work_fetch::FetchRoutePolicy::SealedRoute { + service, + method, + } = route + && (&request.service != service || &request.method != method) + { + return Err(PaidClientError::InputMismatch("Fetch route")); + } + } + _ => return Err(PaidClientError::InputMismatch("work profile")), + } + Ok(()) +} +/// Checks an Evaluate request against the configured execution bounds. +pub fn check_evaluate_input( + policy: &ProviderChannelPolicy, + prepared: &PreparedPaidInputV1, +) -> Result<()> { + let parts = prepared.parts()?; + let input = InputIdentities::from_parts(&parts)?; + let PaidWorkPolicy::Evaluate(expected) = &policy.execution_policy else { + return Err(PaidClientError::InputMismatch("expected Evaluate profile")); + }; + if expected.allowed_environment != input.allowed_environment { + return Err(PaidClientError::InputMismatch("Evaluate environment")); + } + if !(hellas_rpc::protocol::work::matches_generation_policy(expected, &parts.text_policy)?) { + return Err(PaidClientError::InputMismatch("generation policy")); + } + if expected.identity_source_digest != input.identity_source_digest { + return Err(PaidClientError::InputMismatch("identity source")); + } + Ok(()) +} + +fn payment_terms( + config: &WorkConfig, + policy: &ProviderChannelPolicy, + bundle: &hellas_rpc::protocol::work_bundle::WorkChannelSetupBundleV1, + omission_bond: u64, +) -> WorkPaymentTerms { + WorkPaymentTerms { + bond_edge: bundle.bond_edge(), + bond_terms: bundle.bond_terms().clone(), + private_policy_commitment: private_policy_commitment( + config.chain.network, + &policy.policy_salt, + &policy.channel_policy, + ), + omit_response_blocks: policy.min_omit_response_blocks, + start_validity_blocks: MAX_START_VALIDITY_BLOCKS, + omission_bond, + } +} + +async fn drive_setup( + setup: &SetupService, + policy: &ProviderChannelPolicy, + chain: &WorkBlocks, + poll: Duration, +) -> Result<(hellas_work::work_store::ChannelStore, WorkChannelDescriptor)> { + loop { + let SetupAdvance { progress, mounted } = setup.advance_setup(chain, chain, chain).await?; + if let Some(store) = mounted { + let channel = store.state().channel(); + let descriptor = + policy.admit(channel.payment_edge(), channel.payment_terms().clone())?; + return Ok((store, descriptor)); + } + match progress { + end @ (SetupProgress::Aborted(_) + | SetupProgress::Faulted(_) + | SetupProgress::TimeoutBond) => return Err(PaidClientError::SetupEnded(end)), + _ => tokio::time::sleep(poll).await, + } + } +} + +async fn ready_channel( + descriptor: &WorkChannelDescriptor, + chain: &WorkBlocks, +) -> Result { + let query = WorkChannelQuery { + bond_edge: descriptor.bond_edge(), + payment_edge: descriptor.channel().payment_edge(), + funding: Default::default(), + }; + let snapshot = + chain + .work_channel_snapshot(query) + .await? + .ok_or(PaidClientError::MissingState( + "no finalized channel snapshot is available", + ))?; + descriptor + .check_ready(&snapshot.observed_channel()) + .map_err(PaidClientError::from) +} + +async fn collect_until_ready( + dialer: &ProviderDialer, + client: &mut ClientService, + ready: &hellas_rpc::protocol::work_setup::ReadyChannel, + work_id: hellas_rpc::Digest, + poll: Duration, +) -> Result> { + loop { + match fetch_result(dialer.work().await?, client, ready, work_id).await { + Ok(delivery) => { + check_payment_window(client, work_id)?; + return Ok(delivery.transcript); + } + Err(hellas_work::work::DeliverError::Refused { refusal, reason }) + if refusal.is_retryable() => + { + check_payment_window(client, work_id)?; + tracing::debug!(%reason, "waiting for paid result"); + tokio::time::sleep(poll.max(Duration::from_secs(1))).await; + } + Err(error) => return Err(error.into()), + } + } +} + +pub fn deadlines( + current: u64, + acceptance_blocks: u64, + terminal_blocks: u64, + payment_blocks: u64, +) -> Result { + let acceptance = + current + .checked_add(acceptance_blocks) + .ok_or(PaidClientError::DeadlineOverflow { + stage: "acceptance", + })?; + let terminal = acceptance + .checked_add(terminal_blocks) + .ok_or(PaidClientError::DeadlineOverflow { stage: "terminal" })?; + let payment = terminal + .checked_add(payment_blocks) + .ok_or(PaidClientError::DeadlineOverflow { stage: "payment" })?; + Ok(JobDeadlines { + acceptance, + terminal, + payment, + }) +} + +pub async fn bind_paid_endpoint(secret_key: SecretKey) -> Result { + Endpoint::builder(presets::N0) + .secret_key(secret_key) + .alpns(vec![ + hellas_rpc::services::work_setup::WorkSetup::ALPN + .as_bytes() + .to_vec(), + hellas_rpc::services::work::Work::ALPN.as_bytes().to_vec(), + ]) + .bind() + .await + .map_err(PaidClientError::from) +} + +async fn exchange_setup(dialer: &ProviderDialer, setup: &mut SetupEndpoint) -> Result<()> { + let request = prepare_setup_exchange(setup); + let response = send_setup_exchange(dialer.setup().await?, request).await?; + apply_setup_exchange(setup, response)?; + Ok(()) +} + +struct ProviderDialer { + endpoint: Endpoint, + provider: EndpointAddr, + trust: Option, + producer: std::sync::Mutex>, + // Open is authenticated once per exact connection and ALPN. Serialize dials + // so concurrent callers cannot reuse a connection before its proof passes. + setup_connection: tokio::sync::Mutex>, + work_connection: tokio::sync::Mutex>, +} + +impl ProviderDialer { + fn new( + provider: EndpointId, + addresses: Vec, + endpoint: Endpoint, + trust: Option, + ) -> Self { + Self { + trust, + producer: std::sync::Mutex::new(None), + setup_connection: tokio::sync::Mutex::new(None), + work_connection: tokio::sync::Mutex::new(None), + endpoint, + provider: EndpointAddr::from_parts( + provider, + addresses.into_iter().map(TransportAddr::Ip), + ), + } + } + + fn require_producer(&self, key: hellas_rpc::PublicKey) -> Result<()> { + let mut expected = self + .producer + .lock() + .map_err(|_| PaidClientError::ProviderIdentityPoisoned)?; + if expected.as_ref().is_some_and(|old| *old != key) { + return Err(PaidClientError::ProviderIdentityChanged); + } + *expected = Some(key); + Ok(()) + } + + async fn setup(&self) -> Result { + self.connect(hellas_rpc::services::work_setup::WorkSetup::ALPN.as_bytes()) + .await + } + + async fn work(&self) -> Result { + self.connect(hellas_rpc::services::work::Work::ALPN.as_bytes()) + .await + } + + async fn connect(&self, alpn: &[u8]) -> Result { + let cache = if alpn == hellas_rpc::services::work::Work::ALPN.as_bytes() { + &self.work_connection + } else { + &self.setup_connection + }; + let mut cached = cache.lock().await; + if let Some(connection) = cached.as_ref().filter(|c| c.close_reason().is_none()) { + return Ok(IrohTransport::new(connection.clone())); + } + // A failed authentication never enters the cache. Reconnects repeat Open + // and still have to match the producer pinned by the payment channel. + *cached = None; + let connection = self + .endpoint + .connect(self.provider.clone(), alpn) + .await + .map_err(|source| PaidClientError::Connect { + provider: self.provider.id, + source, + })?; + let transport = IrohTransport::new(connection); + if let Some(trust) = &self.trust { + let producer = if alpn == hellas_rpc::services::work::Work::ALPN.as_bytes() { + hellas_client::confidential_open::( + &transport, trust, + ) + .await? + } else { + hellas_client::confidential_open::( + &transport, trust, + ) + .await? + }; + self.require_producer(producer)?; + } + *cached = Some(transport.connection().clone()); + Ok(transport) + } +} + +async fn connect_chain( + config: &WorkConfig, + next_validator: &mut usize, +) -> Result> { + let verifier = ConsensusVerifier::new(&ConsensusInfo { + validators: config.validators.clone(), + threshold_identity: config.chain.threshold_identity.clone(), + network_id: config.chain.network.as_str().to_owned(), + })?; + // A peer can accept connections while lacking a historical certificate. + // Reconnects must make progress through the configured alternatives. + let start = *next_validator; + let mut failures = Vec::new(); + for url in config + .validators + .iter() + .cycle() + .skip(start) + .take(config.validators.len()) + { + *next_validator = (*next_validator + 1) % config.validators.len(); + match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { + Ok(client) => return Ok(WorkBlocks::new(client)), + Err(error) => failures.push((url.clone(), error)), + } + } + Err(PaidClientError::ValidatorsUnavailable(failures)) +} + +async fn check_genesis( + config: &WorkConfig, + chain: &WorkBlocks, +) -> Result<()> { + let first = chain + .block_at(1) + .await? + .ok_or(PaidClientError::MissingState( + "finalized block 1 for genesis authentication", + ))?; + check_genesis_payload( + config.chain.genesis_payload_digest.as_bytes(), + &first.parent, + ) +} +pub fn check_genesis_payload(expected: &[u8; 32], actual: &[u8; 32]) -> Result<()> { + if actual != expected { + return Err(PaidClientError::GenesisMismatch { + expected: *expected, + actual: *actual, + }); + } + Ok(()) +} + +async fn finalized_floor(chain: &WorkBlocks) -> Result { + let height = chain + .latest_height() + .await? + .ok_or(PaidClientError::MissingState( + "configured validator has finalized no blocks", + ))?; + let block = chain + .block_at(height) + .await? + .ok_or(PaidClientError::MissingState( + "configured validator did not return its finalized tip", + ))?; + Ok(SetupScan { + height, + payload: block.payload, + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/sdk/src/paid_client/error.rs b/crates/sdk/src/paid_client/error.rs new file mode 100644 index 00000000..a96917bf --- /dev/null +++ b/crates/sdk/src/paid_client/error.rs @@ -0,0 +1,81 @@ +use std::path::PathBuf; + +/// Errors retain the admission/payment stage and the original typed cause. +#[derive(Debug, thiserror::Error)] +pub enum PaidClientError { + #[error("invalid paid-work options: {0}")] + InvalidOptions(&'static str), + #[error("paid input does not match {0}")] + InputMismatch(&'static str), + #[error("paid-work state is missing: {0}")] + MissingState(&'static str), + #[error("multiple active jobs match this prepared input")] + AmbiguousRecovery, + #[error("payment deadline elapsed during result delivery")] + PaymentExpired, + #[error("{stage} deadline overflow")] + DeadlineOverflow { stage: &'static str }, + #[error("{stage} timed out; journals retain payment state")] + Timeout { stage: &'static str }, + #[error("no configured validator answered ({0:?})")] + ValidatorsUnavailable(Vec<(String, hellas_chain::QueryError)>), + #[error("authenticated provider key differs from the payment channel")] + ProviderIdentityChanged, + #[error("provider identity lock poisoned")] + ProviderIdentityPoisoned, + #[error("validator genesis differs from the configured genesis")] + GenesisMismatch { + expected: [u8; 32], + actual: [u8; 32], + }, + #[error("paid-work setup ended: {0:?}")] + SetupEnded(hellas_work::work_open::SetupProgress), + #[error("cannot create journal directory {}: {source}", path.display())] + JournalDirectory { + path: PathBuf, + source: std::io::Error, + }, + #[error("cannot connect to provider {provider}: {source}")] + Connect { + provider: iroh::EndpointId, + source: iroh::endpoint::ConnectError, + }, + #[error(transparent)] + Bind(#[from] iroh::endpoint::BindError), + #[error(transparent)] + Client(#[from] hellas_client::ClientError), + #[error(transparent)] + Canonical(#[from] hellas_rpc::protocol::value::CanonicalDecodeError), + #[error(transparent)] + Work(#[from] hellas_rpc::protocol::work::PaidWorkError), + #[error(transparent)] + WorkSetup(#[from] hellas_rpc::protocol::work_setup::WorkSetupError), + #[error(transparent)] + Store(#[from] hellas_work::work_store::WorkStoreError), + #[error(transparent)] + Endpoint(#[from] hellas_work::work::EndpointError), + #[error(transparent)] + Propose(#[from] hellas_work::work::ProposeError), + #[error(transparent)] + Deliver(#[from] hellas_work::work::DeliverError), + #[error(transparent)] + Payment(#[from] hellas_work::work::PaymentError), + #[error(transparent)] + Collect(#[from] hellas_client::work::CollectResultError), + #[error(transparent)] + Close(#[from] hellas_work::work_close::CloseError), + #[error(transparent)] + CatchUp(#[from] hellas_work::work_close::CatchUpError), + #[error(transparent)] + Setup(#[from] hellas_work::work_handshake::SetupExchangeError), + #[error(transparent)] + SetupDrive(#[from] hellas_work::work_open::SetupDriveError), + #[error(transparent)] + BlockSource(#[from] hellas_work::work_close::BlockSourceError), + #[error(transparent)] + Consensus(#[from] hellas_chain::ConsensusVerificationError), + #[error(transparent)] + Query(#[from] hellas_chain::QueryError), + #[error(transparent)] + Fetch(#[from] hellas_rpc::fetch::FetchProtocolError), +} diff --git a/crates/sdk/src/paid_client/tests.rs b/crates/sdk/src/paid_client/tests.rs new file mode 100644 index 00000000..38eabf0e --- /dev/null +++ b/crates/sdk/src/paid_client/tests.rs @@ -0,0 +1,342 @@ +use super::*; +use crate::test_support::enrollment; +mod recovery; +use hellas_rpc::pb::execute::{OpenRequest, OpenResponse, open_response}; +use hellas_rpc::{Assurance, ProducerSigningKey, ProviderEnrollmentBundle, PublicKey}; +use hellas_wire::{MethodMarker, StreamTransport, WireStatus}; + +#[tokio::test] +async fn insufficient_collateral_is_rejected_before_network_or_journal_creation() { + let (policy, _, _) = fetch_request(Assurance::ProducerSigned, hellas_rpc::Retention::Ephemeral); + let root = tempfile::tempdir().unwrap(); + let journal_root = root.path().join("journal"); + let endpoint = Endpoint::builder(presets::Minimal) + .bind_addr("127.0.0.1:0".parse::().unwrap()) + .unwrap() + .bind() + .await + .unwrap(); + let args = PaidWorkOptions { + config: WorkConfig { + chain: crate::work_config::ChainCrossCheck { + network: policy.network, + genesis_payload_digest: [0; 32].into(), + threshold_identity: Vec::new(), + }, + // No chain is needed to reject these terms. + validators: Vec::new(), + journal_root: root.path().join("provider"), + routes: Default::default(), + policy_salt: policy.policy_salt, + channel_policy: policy.channel_policy, + execution_policy: policy.execution_policy, + poll: Duration::from_millis(200), + max_observation_age: Duration::from_secs(5), + expected_payment_values: hellas_kernel::EdgeValues::new( + 1000, + 0, + hellas_kernel::Fees::ZERO, + ), + min_omit_response_blocks: policy.min_omit_response_blocks, + }, + journal_root: journal_root.clone(), + provider: endpoint.id(), + provider_addrs: Vec::new(), + provider_trust: None, + bond: EdgeId::from_bytes([0; 32]), + payment_funding: Funding::new( + hellas_kernel::List::empty(hellas_kernel::CoinId::from_bytes([0; 32])), + hellas_kernel::List::empty(hellas_kernel::CoinId::from_bytes([0; 32])), + ), + omission_bond: 51, + acceptance_blocks: 300, + terminal_blocks: 3000, + payment_blocks: 600, + timeout: Duration::from_secs(30), + }; + let result = PaidWorkSession::open( + args, + endpoint.clone(), + Secp256k1Signer::from_secret_scalar([1; 32]).unwrap(), + ) + .await; + assert!(matches!( + result, + Err(PaidClientError::WorkSetup( + hellas_rpc::protocol::work_setup::WorkSetupError::Undercollateralised { + bond: 51, + capacity: 949, + } + )) + )); + assert!(!journal_root.exists()); + endpoint.close().await; +} + +async fn serve_open( + transport: &IrohTransport, + bundle: ProviderEnrollmentBundle, + producer: ProducerSigningKey, +) where + M: MethodMarker, +{ + let inbound = transport.accept().await.unwrap().unwrap(); + // A paid request must never be the first RPC on a trusted connection. + assert_eq!(inbound.method_id, M::METHOD_ID); + hellas_rpc::call::dispatch_unary_with_context::( + inbound, + move |request, context| async move { + let nonce: [u8; 32] = request.nonce.try_into().unwrap(); + let binding = hellas_rpc::open_proof_binding( + &context.open_exporter.unwrap(), + &nonce, + &producer.public_key(), + bundle.content_id(), + M::Service::ALPN.as_bytes(), + ); + Ok::<_, WireStatus>(OpenResponse { + provider_genesis: bundle.canonical_bytes(), + proof: Some(open_response::Proof::ProducerSignature( + hellas_rpc::run_ticket::signature_to_pb( + &producer.sign_digest(binding).unwrap(), + ), + )), + }) + }, + ) + .await + .unwrap(); +} + +#[tokio::test] +async fn both_paid_connections_open_before_disclosure_and_refuse_wrong_assurance_or_party() { + tokio::time::timeout(Duration::from_secs(15), async { + let server = Endpoint::builder(presets::Minimal) + .secret_key(SecretKey::from_bytes(&[4; 32])) + .alpns(vec![ + hellas_rpc::services::work::Work::ALPN.as_bytes().to_vec(), + hellas_rpc::services::work_setup::WorkSetup::ALPN + .as_bytes() + .to_vec(), + ]) + .bind_addr("127.0.0.1:0".parse::().unwrap()) + .unwrap() + .bind() + .await + .unwrap(); + let (bundle, key) = enrollment(server.id()); + let trust = hellas_client::ProviderTrustAnchor { + expected_genesis: bundle.content_id(), + required_assurance: Assurance::ProducerSigned, + apple_app_attest: None, + }; + let addresses = server.bound_sockets(); + let serving = server.clone(); + let (done, completed) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let mut connections = Vec::new(); + for _ in 0..4 { + let connection = serving.accept().await.unwrap().await.unwrap(); + let setup = connection.alpn() + == hellas_rpc::services::work_setup::WorkSetup::ALPN.as_bytes(); + let transport = IrohTransport::new(connection); + if setup { + serve_open::( + &transport, + bundle.clone(), + key.clone(), + ) + .await; + } else { + serve_open::( + &transport, + bundle.clone(), + key.clone(), + ) + .await; + } + connections.push(transport); + } + let _ = completed.await; + }); + let mut dialer = ProviderDialer::new( + server.id(), + addresses, + bind_paid_endpoint(SecretKey::from_bytes(&[5; 32])) + .await + .unwrap(), + Some(trust), + ); + let setup = dialer.setup().await.expect("setup authenticates"); + let work = dialer.work().await.expect("work authenticates"); + for _ in 0..3 { + assert_eq!( + setup.open_exporter().unwrap(), + dialer.setup().await.unwrap().open_exporter().unwrap() + ); + assert_eq!( + work.open_exporter().unwrap(), + dialer.work().await.unwrap().open_exporter().unwrap() + ); + } + work.connection().close(0u32.into(), b"test reconnect"); + *dialer.producer.lock().unwrap() = Some( + ProducerSigningKey::from_secret_bytes([6; 32]) + .unwrap() + .public_key(), + ); + let error = dialer + .work() + .await + .err() + .expect("different payment party is refused"); + assert!( + error + .to_string() + .contains("differs from the payment channel") + ); + *dialer.producer.lock().unwrap() = None; + setup.connection().close(0u32.into(), b"test reconnect"); + dialer.trust.as_mut().unwrap().required_assurance = Assurance::AppleAppAttest; + let error = dialer + .setup() + .await + .err() + .expect("software proof cannot satisfy App Attest"); + assert!(error.to_string().contains("Secure Enclave provider root")); + done.send(()).unwrap(); + task.await.unwrap(); + dialer.endpoint.close().await; + server.close().await; + }) + .await + .unwrap(); +} + +#[test] +fn recovery_skips_only_permanent_delivery_refusals() { + use hellas_client::work::CollectResultError; + use hellas_work::work::{DeliverError, WorkRefusal}; + + for (refusal, permanent) in [ + (WorkRefusal::Declined, true), + (WorkRefusal::Expired, true), + (WorkRefusal::NotReady, false), + (WorkRefusal::Unavailable, false), + ] { + let delivery = || DeliverError::Refused { + refusal, + reason: "provider diagnostic".to_owned(), + }; + assert_eq!(permanently_refused_delivery(&delivery().into()), permanent); + assert_eq!( + permanently_refused_delivery(&CollectResultError::Deliver(delivery()).into()), + permanent, + ); + } + assert!(!permanently_refused_delivery( + &DeliverError::Malformed("result").into() + )); + assert!(!permanently_refused_delivery(&PaidClientError::Timeout { + stage: "connection" + })); +} + +fn fetch_request( + assurance: Assurance, + retention: hellas_rpc::Retention, +) -> (ProviderChannelPolicy, PreparedPaidWorkInput, PublicKey) { + use hellas_rpc::protocol::work::PaidChannelPolicyV1; + use hellas_rpc::protocol::work_fetch::{ + FetchRoutePolicy, PaidFetchPolicyV1, PreparedPaidFetchInputV1, fetch_route_commitment, + }; + let environment = hellas_rpc::FetchEnvironment::OpenAiResponses; + let caller = ProducerSigningKey::from_secret_bytes([7; 32]).unwrap(); + let events = hellas_rpc::fetch::build_input_events_with_retention( + "openai", + "responses", + br#"{"input":"private request"}"#, + environment.manifest_id(), + assurance, + &caller, + retention, + ) + .unwrap(); + let prepared = PreparedPaidFetchInputV1::new(&events, &environment.manifest()) + .unwrap() + .into(); + let route = FetchRoutePolicy::sealed_route("openai", "responses").unwrap(); + let policy = ProviderChannelPolicy { + network: hellas_kernel::NetworkId::new("paid-client-test").unwrap(), + policy_salt: [8; 32], + channel_policy: PaidChannelPolicyV1 { + compute_credit_limit: 40, + delivery_credit_limit: 40, + }, + execution_policy: PaidWorkPolicy::Fetch { + policy: PaidFetchPolicyV1 { + allowed_environment: environment.manifest_id(), + route_commitment: fetch_route_commitment(&route.canonical_body_bytes()).unwrap(), + max_request_body_bytes: 4096, + max_output_events: 64, + max_output_bytes: 16384, + max_spool_bytes: 65536, + max_encoded_result_frame: 65536, + max_encoded_prepared_input: 65536, + dispatch_margin_blocks: 4, + delivery_margin_blocks: 2, + oracle_grace_blocks: 6, + fixed_price: 10, + }, + route, + }, + expected_payment_values: hellas_kernel::EdgeValues::new( + 1000, + 200, + hellas_kernel::Fees::ZERO, + ), + min_omit_response_blocks: hellas_kernel::MIN_OMIT_RESPONSE_BLOCKS, + }; + (policy, prepared, caller.public_key()) +} + +#[test] +fn fetch_preflight_requires_matching_trust_caller_and_ephemeral_retention() { + use hellas_rpc::Retention; + let (policy, prepared, caller) = fetch_request(Assurance::AppleAppAttest, Retention::Ephemeral); + let error = check_request(&policy, &prepared, None, caller).unwrap_err(); + assert!(matches!(error, PaidClientError::InvalidOptions(_))); + let mut trust = hellas_client::ProviderTrustAnchor { + expected_genesis: hellas_rpc::ContentId::from_bytes([8; 32]), + required_assurance: Assurance::ProducerSigned, + apple_app_attest: None, + }; + assert!( + check_request(&policy, &prepared, Some(&trust), caller) + .unwrap_err() + .to_string() + .contains("assurance differs") + ); + trust.required_assurance = Assurance::AppleAppAttest; + // Preflight checks the anchor selection; the live Open validates its proof. + check_request(&policy, &prepared, Some(&trust), caller).unwrap(); + let other = ProducerSigningKey::from_secret_bytes([9; 32]) + .unwrap() + .public_key(); + assert!( + check_request(&policy, &prepared, Some(&trust), other) + .unwrap_err() + .to_string() + .contains("Fetch caller") + ); + let (policy, prepared, caller) = fetch_request(Assurance::ProducerSigned, Retention::Ephemeral); + check_request(&policy, &prepared, None, caller).unwrap(); + assert_eq!(prepared.assurance().unwrap(), Assurance::ProducerSigned); + let (policy, retained, caller) = fetch_request(Assurance::ProducerSigned, Retention::Retain); + assert!( + check_request(&policy, &retained, None, caller) + .unwrap_err() + .to_string() + .contains("ephemeral") + ); +} diff --git a/crates/sdk/src/paid_client/tests/recovery.rs b/crates/sdk/src/paid_client/tests/recovery.rs new file mode 100644 index 00000000..68c544de --- /dev/null +++ b/crates/sdk/src/paid_client/tests/recovery.rs @@ -0,0 +1,222 @@ +use super::*; +use crate::test_support::{BODY, PRICE, PaidFixture, signer}; +use hellas_rpc::fetch::{ + FetchOutputTranscriptBuilder, encode_fetch_event_payload, encode_fetch_terminal_payload, + verify_input_events, +}; +use hellas_rpc::output::{OutputEvent, StopReason, TextChannel}; +use hellas_rpc::pb::work::WorkDelivered; +use hellas_rpc::protocol::work::PrivateRecord as _; +use hellas_rpc::services::work::{Work, WorkServer}; +use hellas_wire::Dispatcher; +use hellas_work::work::{ + BackendFault, ObservationTime, PaidWorkBackend, PreparedFetchInput, run_accepted_work, +}; +use std::sync::atomic::AtomicUsize; + +const RESPONSE: &str = "sdk-private-response"; + +#[derive(Default)] +struct Backend(AtomicUsize); + +impl PaidWorkBackend for Backend { + async fn fetch( + &self, + input: PreparedFetchInput, + ) -> Result, BackendFault> { + self.0.fetch_add(1, Ordering::SeqCst); + let request = verify_input_events(&input.into_parts().fetch_input_transcript).unwrap(); + let key = ProducerSigningKey::from_secret_bytes([2; 32]).unwrap(); + let mut output = + FetchOutputTranscriptBuilder::new(request.input_commitment, request.assurance, &key); + output + .push_event( + encode_fetch_event_payload(&OutputEvent::TextDelta { + index: 0, + delta: RESPONSE.into(), + channel: TextChannel::Output, + }) + .unwrap(), + ) + .unwrap(); + Ok(output + .finish( + encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .unwrap(), + ) + .unwrap()) + } +} + +fn assert_no_payloads(root: &std::path::Path) { + for entry in std::fs::read_dir(root).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + assert_no_payloads(&path); + } else { + let bytes = std::fs::read(&path).unwrap(); + for payload in [BODY, RESPONSE.as_bytes()] { + assert!( + !bytes.windows(payload.len()).any(|window| window == payload), + "payload in {}", + path.display() + ); + } + } + } +} + +#[tokio::test] +async fn restart_pays_delivered_fetch_once_and_skips_lost_payload_jobs() { + tokio::time::timeout(Duration::from_secs(15), async { + let fixture = PaidFixture::new(); + let provider = fixture.provider(); + let mut client = fixture.client(); + let request = client.propose(&fixture.proposal()).unwrap(); + let lost = client.accepted(&provider.accept(&request)).unwrap(); + let request = client.propose(&fixture.proposal()).unwrap(); + let delivered = client.accepted(&provider.accept(&request)).unwrap(); + assert_ne!(lost, delivered); + let backend = Backend::default(); + run_accepted_work(&provider, &fixture.ready, &backend, delivered) + .await + .unwrap(); + let request = client.request_delivery(delivered, &[8; 32]).unwrap(); + let result = provider.deliver(&request, &[8; 32]).unwrap(); + client + .receive( + delivered, + &fixture.ready, + &WorkDelivered { + result: result.result.encode(), + provider_signature: result.signature.as_bytes().to_vec(), + transcript: result.transcript, + }, + ) + .unwrap(); + // Crash after verified delivery, before creating any payment certificate. + assert!(client.state().last_payment().is_none()); + drop(client); + assert_no_payloads(fixture.root.path()); + + let server = Endpoint::builder(presets::Minimal) + .alpns(vec![Work::ALPN.as_bytes().to_vec()]) + .bind_addr("127.0.0.1:0".parse::().unwrap()) + .unwrap() + .bind() + .await + .unwrap(); + let (bundle, key) = enrollment(server.id()); + let trust = hellas_client::ProviderTrustAnchor { + expected_genesis: bundle.content_id(), + required_assurance: Assurance::ProducerSigned, + apple_app_attest: None, + }; + let serving = tokio::spawn({ + let server = server.clone(); + let provider = provider.clone(); + async move { + let mut tasks = tokio::task::JoinSet::new(); + while let Some(incoming) = server.accept().await { + let bundle = bundle.clone(); + let key = key.clone(); + let provider = provider.clone(); + tasks.spawn(async move { + let transport = IrohTransport::new(incoming.await.unwrap()); + serve_open::(&transport, bundle, key) + .await; + let handler = WorkServer(provider); + while let Ok(Some(inbound)) = transport.accept().await { + Dispatcher::::dispatch(&handler, inbound) + .await + .unwrap(); + } + }); + } + while let Some(result) = tasks.join_next().await { + result.unwrap(); + } + } + }); + for restart in 0..2 { + let store = fixture.store(Role::Client); + assert!(store.state().job_by_id(lost).is_some()); + assert_eq!(store.state().last_payment().is_some(), restart == 1); + for job in store.state().jobs() { + assert!(job.prepared_input().is_empty()); + assert!(job.transcript().is_empty()); + assert_eq!(job.result().is_some(), job.work_id() == delivered); + } + if restart == 0 { + assert!( + store + .state() + .job_by_id(delivered) + .unwrap() + .result() + .is_some() + ); + } + let client = ClientService::new(ClientEndpoint::recover(store, signer(1)).unwrap()); + client + .observer() + .observe_ready( + fixture.ready.clone(), + ObservationTime::now(), + Duration::from_secs(60), + ) + .unwrap(); + let endpoint = bind_paid_endpoint(SecretKey::from_bytes(&[5; 32])) + .await + .unwrap(); + let mut session = PaidWorkSession { + args: PaidWorkOptions { + config: fixture.config.clone(), + journal_root: fixture.root.path().join("client"), + provider: server.id(), + provider_addrs: server.bound_sockets(), + provider_trust: Some(trust.clone()), + bond: fixture.descriptor.bond_edge(), + payment_funding: Funding::new( + hellas_kernel::List::empty(hellas_kernel::CoinId::from_bytes([0; 32])), + hellas_kernel::List::empty(hellas_kernel::CoinId::from_bytes([0; 32])), + ), + omission_bond: 601, + acceptance_blocks: 50, + terminal_blocks: 100, + payment_blocks: 200, + timeout: Duration::from_secs(5), + }, + descriptor: fixture.descriptor.clone(), + dialer: ProviderDialer::new( + server.id(), + server.bound_sockets(), + endpoint.clone(), + Some(trust.clone()), + ), + client, + observer: None, + needs_recovery: true, + }; + assert!(session.run(None, true, None).await.unwrap().is_none()); + assert!(!session.needs_recovery); + assert_eq!( + provider + .with_state(|state| state.ledger().credited_cumulative()) + .unwrap(), + PRICE + ); + drop(session); + endpoint.close().await; + assert_no_payloads(fixture.root.path()); + } + assert_eq!(backend.0.load(Ordering::SeqCst), 1); + server.close().await; + serving.await.unwrap(); + }) + .await + .expect("recovery completes without a validator or payload replay"); +} diff --git a/crates/cli/src/commands/paid_work/gateway.rs b/crates/sdk/src/paid_gateway.rs similarity index 63% rename from crates/cli/src/commands/paid_work/gateway.rs rename to crates/sdk/src/paid_gateway.rs index dc94e838..ba032c1c 100644 --- a/crates/cli/src/commands/paid_work/gateway.rs +++ b/crates/sdk/src/paid_gateway.rs @@ -1,16 +1,35 @@ -//! HTTP gateway adapter over the same durable paid-work client as the CLI. +//! Reusable paid gateway pool over durable client-owned state channels. -use super::*; +use crate::paid_client::{ + PaidClientError, PaidWorkOptions, PaidWorkResult, PaidWorkSession, bind_paid_endpoint, + check_evaluate_input, +}; +use hellas_kernel::Secp256k1Signer; +use hellas_rpc::protocol::artifacts::PreparedPaidInputV1; +use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; +use hellas_work::work_store::journal::MAX_RECORD_BYTES; +use iroh::EndpointId; +use std::sync::atomic::AtomicBool; +use std::time::Duration; + +mod config; +mod error; +pub use config::{PaidGatewayOptions, load_pool_options}; +pub use error::PoolError; +type Result = std::result::Result; use futures::future::BoxFuture; use futures::stream::BoxStream; use hellas_gateway::{ - ExecutionEvent, Outcome, PaidExecutionBackend, PaidExecutionRequest, StopReason, + ExecutionEvent, Outcome, PaidExecutionBackend, PaidExecutionRequest, PaidFetchRequest, + StopReason, }; +use hellas_rpc::output::OutputEvent as FetchEvent; use hellas_rpc::protocol::artifacts::{ BoundTermId, InputAddressed as _, OutputAddressed as _, SourceRef, TextArtifact, TextExecution, TextPolicy, TokenIds, }; -use serde::Deserialize; +use hellas_rpc::protocol::work_profile::{PaidWorkPolicy, PreparedPaidWorkInput}; +use iroh::Endpoint; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Instant; @@ -18,39 +37,6 @@ use tokio::sync::{Mutex as AsyncMutex, OwnedSemaphorePermit, Semaphore, mpsc, wa use tokio::task::JoinHandle; use tracing::Instrument; -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct PoolFile { - providers: Vec, - #[serde(default = "acceptance_blocks")] - acceptance_blocks: u64, - #[serde(default = "terminal_blocks")] - terminal_blocks: u64, - #[serde(default = "payment_blocks")] - payment_blocks: u64, - #[serde(default = "timeout_secs")] - timeout_secs: u64, - #[serde(default = "max_pending_requests")] - max_pending_requests: usize, -} - -const fn max_pending_requests() -> usize { - 64 -} - -const fn acceptance_blocks() -> u64 { - 16 -} -const fn terminal_blocks() -> u64 { - 64 -} -const fn payment_blocks() -> u64 { - 32 -} -const fn timeout_secs() -> u64 { - 300 -} - // An OpenCode conversation has a substantial shared chat prefix. Smaller // checkpoints are not worth routing work around. const MIN_CACHE_AFFINITY_TOKENS: usize = 128; @@ -61,32 +47,31 @@ const PROVIDER_CONNECTION_TIMEOUT: Duration = Duration::from_secs(10); // A retained job is durable, but it must not monopolize the channel that // serves interactive requests after a restart or a provider interruption. const RECOVERY_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30); -// A background catch-up pass yields within the request's lock-wait budget. -const CHANNEL_FOLLOW_BUDGET: Duration = Duration::from_secs(1); +// Bound queueing for Evaluate routes that can try another funded provider. +const CHANNEL_QUEUE_BUDGET: Duration = Duration::from_secs(1); // Bound HTTP delivery independently of the authenticated transcript spool. const OUTPUT_BUFFER_BYTES: usize = 2 * MAX_RECORD_BYTES; const OUTPUT_EVENT_OVERHEAD: usize = 1024; const OUTPUT_BUFFER_EVENTS: usize = OUTPUT_BUFFER_BYTES / OUTPUT_EVENT_OVERHEAD; -type BufferedEvent = (CliResult, OwnedSemaphorePermit); +type BufferedEvent = (Result, OwnedSemaphorePermit); -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct ProviderFile { - work_config: PathBuf, - journal_root: PathBuf, - provider: EndpointId, - #[serde(default)] - provider_addrs: Vec, - bond: String, - payment_coins: Vec, - omission_bond: u64, +trait GatewayEvent: Send + 'static { + fn prefix(event: hellas_rpc::OutputEventEnvelope) -> Result + where + Self: Sized; + fn completed(output: PaidWorkResult) -> Result> + where + Self: Sized; + fn is_terminal(&self) -> bool; + fn bytes(&self) -> usize; } struct Provider { - args: RunArgs, + args: PaidWorkOptions, policy: ProviderChannelPolicy, + assurance: hellas_rpc::Assurance, /// A setup/channel journal has a single owner even with concurrent HTTP calls. - serial: AsyncMutex>, + serial: AsyncMutex>, pending: AtomicUsize, cache: Mutex, unavailable_until: Mutex>, @@ -204,170 +189,132 @@ impl Drop for ProviderUse { } } -struct PaidGateway { +pub struct PaidGateway { providers: Vec>, next: AtomicUsize, endpoint: Endpoint, settlement_key: Secp256k1Signer, + producer_key: hellas_rpc::ProducerSigningKey, admission: Arc, tasks: Mutex>>, - followers: Mutex>>, } -pub async fn load_gateway_backend( - path: &Path, - transport_key: SecretKey, - settlement_key: Secp256k1Signer, -) -> CliResult> { - let bytes = - crate::commands::read_bounded_regular_file(path, "paid gateway config", MAX_RECORD_BYTES)?; - let file: PoolFile = serde_json::from_slice(&bytes) - .with_context(|| format!("invalid paid gateway config {}", path.display()))?; - anyhow::ensure!( - !file.providers.is_empty(), - "paid gateway requires at least one provider" - ); - anyhow::ensure!( - file.timeout_secs > 0, - "paid gateway timeout_secs must be greater than zero" - ); - anyhow::ensure!( - file.acceptance_blocks > 0 && file.terminal_blocks > 0 && file.payment_blocks > 0, - "paid gateway deadline spans must be greater than zero" - ); - anyhow::ensure!( - file.max_pending_requests > 0 && file.max_pending_requests <= Semaphore::MAX_PERMITS, - "paid gateway max_pending_requests must be a positive supported semaphore capacity" - ); - let mut providers = Vec::new(); - let mut journals = std::collections::BTreeSet::new(); - let mut endpoints = std::collections::BTreeSet::new(); - let mut funding = std::collections::BTreeSet::new(); - for provider in file.providers { - anyhow::ensure!( - provider.work_config.is_absolute() && provider.journal_root.is_absolute(), - "paid gateway work_config and journal_root must be absolute runtime paths" - ); - anyhow::ensure!( - journals.insert(provider.journal_root.clone()), - "paid providers must have distinct journal roots" - ); - anyhow::ensure!( - endpoints.insert(provider.provider), - "paid gateway repeats provider {}", - provider.provider - ); - anyhow::ensure!( - !provider.payment_coins.is_empty(), - "paid provider needs payment_coins" - ); - coins(&provider.payment_coins)?; - edge_id("bond", &provider.bond)?; - for coin in &provider.payment_coins { - anyhow::ensure!( - funding.insert(fixed_hex::<32>("payment_coins", coin)?), - "a payment coin cannot fund two provider channels" - ); +impl PaidGateway { + /// Open a reusable pool. Funding, recovery, queueing and payment are shared + /// by CLI and embedding applications. Call `drain` during host shutdown. + pub async fn open( + options: PaidGatewayOptions, + identity: crate::ClientIdentity, + ) -> Result> { + options.validate()?; + let settlement_key = Secp256k1Signer::from_secret_scalar(identity.caller_secret_bytes()) + .map_err(|_| PoolError::Invalid("settlement identity is not secp256k1"))?; + let providers = options + .providers + .into_iter() + .map(|args| { + Arc::new(Provider { + policy: args.config.provider_policy(), + assurance: args + .provider_trust + .as_ref() + .map_or(hellas_rpc::Assurance::ProducerSigned, |trust| { + trust.required_assurance + }), + args, + serial: AsyncMutex::new(None), + pending: AtomicUsize::new(0), + cache: Mutex::new(PrefixCache::default()), + unavailable_until: Mutex::new(None), + }) + }) + .collect(); + let gateway = Arc::new(PaidGateway { + admission: Arc::new(Semaphore::new(options.max_pending_requests)), + providers, + next: AtomicUsize::new(0), + // One transport identity has one relay registration, shared by every + // provider and request for the lifetime of this gateway. + endpoint: bind_paid_endpoint(identity.transport_key()).await?, + settlement_key, + producer_key: identity.caller_key().clone(), + tasks: Mutex::new(Vec::new()), + }); + // Restart recovery uses the retained input and certificate, never a new job. + // Empty journal roots do not fund a channel until an HTTP request arrives. + for provider in &gateway.providers { + let _recovery = gateway.submit::( + vec![( + provider.clone(), + Route { + available: true, + cache_affinity_tokens: 0, + pending: 0, + }, + )], + None, + None, + Some(RECOVERY_ATTEMPT_TIMEOUT), + None, + )?; } - let config = load_work_config(&provider.work_config)?; - providers.push(Arc::new(Provider { - policy: config.provider_policy(), - args: RunArgs { - work_config: provider.work_config, - journal_root: provider.journal_root, - provider: provider.provider, - provider_addrs: provider.provider_addrs, - bond: provider.bond, - payment_coins: provider.payment_coins, - omission_bond: provider.omission_bond, - prepared_input: PathBuf::new(), - output: None, - acceptance_blocks: file.acceptance_blocks, - terminal_blocks: file.terminal_blocks, - payment_blocks: file.payment_blocks, - timeout_secs: file.timeout_secs, - settle: false, - }, - serial: AsyncMutex::new(None), - pending: AtomicUsize::new(0), - cache: Mutex::new(PrefixCache::default()), - unavailable_until: Mutex::new(None), - })); - } - let gateway = Arc::new(PaidGateway { - admission: Arc::new(Semaphore::new(file.max_pending_requests)), - providers, - next: AtomicUsize::new(0), - // One transport identity has one relay registration, shared by every - // provider and request for the lifetime of this gateway. - endpoint: bind_paid_endpoint(transport_key).await?, - settlement_key, - tasks: Mutex::new(Vec::new()), - followers: Mutex::new(Vec::new()), - }); - // Restart recovery uses the retained input and certificate, never a new job. - // Empty journal roots do not fund a channel until an HTTP request arrives. - for provider in &gateway.providers { - let _recovery = gateway.submit( - vec![( - provider.clone(), - Route { - available: true, - cache_affinity_tokens: 0, - pending: 0, - }, - )], - None, - None, - Some(RECOVERY_ATTEMPT_TIMEOUT), - None, - )?; - let provider = provider.clone(); - gateway - .followers - .lock() - .expect("paid followers poisoned") - .push(tokio::spawn(async move { - let mut reported_failure = false; - loop { - // Leave a gap after each pass, including a slow one, and - // let startup recovery and waiting requests own the channel. - tokio::time::sleep(CHANNEL_FOLLOW_BUDGET).await; - let result = { - let Ok(mut session) = provider.serial.try_lock() else { - continue; - }; - let Some(session) = session.as_mut() else { - continue; - }; - // Catch-up journals complete blocks as it advances; - // cancellation resumes from that durable cursor. - tokio::time::timeout(CHANNEL_FOLLOW_BUDGET, session.follow_chain()).await - }; - match result { - Ok(Ok(())) => reported_failure = false, - Ok(Err(error)) if !reported_failure => { - reported_failure = true; - tracing::warn!(provider = %provider.args.provider, %error, - "paid channel chain follower will retry"); - } - Ok(Err(_)) | Err(_) => {} - } - } - })); + Ok(gateway) } - Ok(gateway) } impl PaidGateway { - fn submit( + fn execute_tokens( + &self, + request: PaidExecutionRequest, + ) -> Result>> { + let permit = self + .admission + .clone() + .try_acquire_owned() + .map_err(|_| hellas_gateway::PaidGatewayBusy)?; + let input_ids = request.input_ids.clone(); + let cache_update = CacheUpdate::from_request(&request.environment, &input_ids); + let prepared = prepare_request(request, &self.settlement_key)?; + let eligible = self + .providers + .iter() + .filter(|provider| check_evaluate_input(&provider.policy, &prepared).is_ok()) + .collect::>(); + if eligible.is_empty() { + return Err(PoolError::NoMatchingPolicy); + } + let start = self.next.fetch_add(1, Ordering::Relaxed) % eligible.len(); + let mut candidates = (0..eligible.len()) + .map(|offset| { + let provider = eligible[(start + offset) % eligible.len()]; + let cache = provider.cache.lock().expect("provider cache poisoned"); + let route = Route { + available: provider.available(), + cache_affinity_tokens: cache.affinity(&input_ids), + pending: provider.pending.load(Ordering::Relaxed), + }; + (provider.clone(), route) + }) + .collect::>(); + // Stable sorting preserves the rotating order for equally ranked routes. + candidates.sort_by_key(|(_, route)| std::cmp::Reverse(route.score())); + self.submit( + candidates, + Some(prepared.into()), + Some(cache_update), + None, + Some(permit), + ) + } + + fn submit( &self, candidates: Vec<(Arc, Route)>, - prepared: Option, + prepared: Option, cache_update: Option, recovery_timeout: Option, permit: Option, - ) -> CliResult>> { + ) -> Result>> { // Admission can precede local request preparation. Serialize the final // gate check and registration with drain's close-and-snapshot boundary. let mut tasks = self.tasks.lock().expect("paid task list poisoned"); @@ -376,10 +323,12 @@ impl PaidGateway { } let endpoint = self.endpoint.clone(); let settlement_key = self.settlement_key.clone(); - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::>(OUTPUT_BUFFER_EVENTS); let (overflow, overflow_receiver) = watch::channel(false); let output_budget = Arc::new(Semaphore::new(OUTPUT_BUFFER_BYTES)); - let (initial_provider, _initial_route) = candidates.first().expect("paid provider exists"); + let Some((initial_provider, _initial_route)) = candidates.first() else { + return Err(PoolError::NoMatchingPolicy); + }; let span = hellas_rpc::request_span!( target: "hellas_request", "paid.gateway", hellas.provider.id = %initial_provider.args.provider, @@ -392,8 +341,7 @@ impl PaidGateway { let mut occupied = prepared .as_ref() .map(|_| ProviderUse::new(initial_provider.clone())); - let timeout = recovery_timeout - .unwrap_or_else(|| Duration::from_secs(initial_provider.args.timeout_secs)); + let timeout = recovery_timeout.unwrap_or_else(|| initial_provider.args.timeout); // Queueing, recovery and fallback all consume the same request budget. let deadline = tokio::time::Instant::now() + timeout; let task_span = span.clone(); @@ -405,14 +353,12 @@ impl PaidGateway { let streamed = prepared.is_some(); let recovery = !streamed; let progress: hellas_work::work::PaidProgress = Arc::new(move |event| { - let delta = hellas_rpc::evaluate::decode_token_delta_payload(event.payload()) - .map_err(|error| hellas_work::work::BackendFault::new(error.to_string()))?; - let position = delta.end_position().map_err(|error| hellas_work::work::BackendFault::new(error.to_string()))?; - emit(&token_sender, &token_overflow, &token_budget, Ok(ExecutionEvent::Chunk { position, tokens: delta.token_bytes() })); + let event = E::prefix(event).map_err(|error| hellas_work::work::BackendFault::new(error.to_string()))?; + emit(&token_sender, &token_overflow, &token_budget, Ok(event)); Ok(()) }); let result = async { - let prepared_bytes = prepared.as_ref().map(PreparedPaidInputV1::encode).transpose()?; + let prepared_bytes = prepared.as_ref().map(PreparedPaidWorkInput::encode).transpose()?; let mut provider_errors = Vec::new(); for (provider, route) in candidates { let _occupied = occupied.take().or_else(|| { @@ -421,16 +367,15 @@ impl PaidGateway { task_span.record("hellas.provider.id", tracing::field::display(provider.args.provider)); task_span.record("hellas.route.cache_affinity_tokens", route.cache_affinity_tokens); task_span.record("hellas.route.pending", route.pending); - // Wait for a brief follower pass to yield, but route around - // channels still occupied by recovery or other requests. - let mut session = if recovery { + // Serialize jobs on one funded channel; its observer runs independently. + let mut session = if recovery || matches!(prepared, Some(PreparedPaidWorkInput::Fetch(_))) { before_proposal( - &sender, false, deadline, + &sender, streamed, deadline, |_| provider.serial.lock().instrument(hellas_rpc::request_span!(target: "hellas_request", "paid.queue")), ).await? } else { match tokio::time::timeout( - CHANNEL_FOLLOW_BUDGET, + CHANNEL_QUEUE_BUDGET, provider.serial.lock(), ).await { Ok(session) => session, @@ -443,7 +388,7 @@ impl PaidGateway { } } }; - if prepared.is_none() && (!provider.args.journal_root.try_exists()? || std::fs::read_dir(&provider.args.journal_root)?.next().is_none()) { + if prepared.is_none() && !has_retained_setup(&provider.args)? { continue; } // Recheck after queueing, including restored channels whose @@ -455,14 +400,14 @@ impl PaidGateway { if session.is_none() { match connect_before_deadline( &sender, streamed, deadline, PROVIDER_CONNECTION_TIMEOUT, - OpenPaidChannel::open(provider.args.clone(), endpoint.clone(), settlement_key.clone()), + async { PaidWorkSession::open(provider.args.clone(), endpoint.clone(), settlement_key.clone()).await.map_err(PoolError::from) }, ).await { Ok(opened) => { *session = Some(opened); } Err(error) => { - if !recovery && error.is::() { return Err(error); } + if !recovery && matches!(error, PoolError::Stopped(_)) { return Err(error); } provider.connection_failed(); if recovery { tracing::debug!(provider = %provider.args.provider, error = %format!("{error:#}"), @@ -477,7 +422,7 @@ impl PaidGateway { } } let session = session.as_mut().expect("channel was opened"); - if prepared.is_some() && session.needs_recovery { + if prepared.is_some() && session.needs_recovery() { let recovery_deadline = deadline.min( tokio::time::Instant::now() + RECOVERY_ATTEMPT_TIMEOUT, ); @@ -486,10 +431,8 @@ impl PaidGateway { session.run(None, true, None), ) .await - .map_err(|_| anyhow::anyhow!( - "retained paid work did not recover within {RECOVERY_ATTEMPT_TIMEOUT:?}" - )) - .and_then(|result| result); + .map_err(|_| PoolError::RecoveryTimeout) + .and_then(|result| result.map_err(PoolError::from)); if let Err(error) = recovery { // Nothing in this path has proposed the fresh request. // Keep the journal for a later recovery and route this @@ -508,10 +451,8 @@ impl PaidGateway { // ClientEndpoint journals the proposal nonce before releasing // its signature. Recovery and a failed dial need not propose // this request; a lost acceptance response does advance it. - let proposal_nonce = session.client.state().proposal_nonce_high_water(); - let already_proposed = prepared_bytes.as_ref().is_some_and(|input| { - session.client.state().jobs().any(|job| job.prepared_input() == input) - }); + let proposal_nonce = session.with_state(|state| state.proposal_nonce_high_water())?; + let already_proposed = session.with_state(|state| prepared_bytes.as_ref().is_some_and(|input| state.jobs().any(|job| job.prepared_input() == input)))?; let request_session = &mut *session; let input = prepared.clone(); let on_progress = streamed.then(|| progress.clone()); @@ -521,31 +462,29 @@ impl PaidGateway { request_session.run_with_admission(input, true, on_progress, Some(&proposed)).await }, ).await - .and_then(|result| result) - .and_then(|output| output.map(output_events).transpose()) + .and_then(|result| result.map_err(PoolError::from)) + .and_then(|output| output.map(E::completed).transpose()) .map(Option::unwrap_or_default); if let Err(error) = &result { - if !recovery && error.is::() - && session.client.state().proposal_nonce_high_water() == proposal_nonce + if !recovery && matches!(error, PoolError::Stopped(_)) + && session.with_state(|state| state.proposal_nonce_high_water())? == proposal_nonce { return result; } provider.cache.lock().expect("provider cache poisoned").replace(None); if recovery { provider.connection_failed(); - return result.with_context(|| { - format!("retained paid work at provider {}", provider.args.provider) - }); + return result.map_err(|source| PoolError::Provider { provider: provider.args.provider, source: Box::new(source) }); } // A failed journal append can leave a durable proposal that // is not reflected in memory yet. Keep that failure with // this provider, just like a proposal with no response. let uncertain_append = matches!( - error.downcast_ref::(), - Some(hellas_work::work::ProposeError::Store(_)), + error, + PoolError::Client(PaidClientError::Propose(hellas_work::work::ProposeError::Store(_))), ); if !already_proposed && !uncertain_append - && session.client.state().proposal_nonce_high_water() == proposal_nonce + && session.with_state(|state| state.proposal_nonce_high_water())? == proposal_nonce { provider.connection_failed(); tracing::debug!(provider = %provider.args.provider, error = %format!("{error:#}"), @@ -559,12 +498,9 @@ impl PaidGateway { cache_update.apply(&mut provider.cache.lock().expect("provider cache poisoned")); } } - return result.with_context(|| format!("paid provider {}", provider.args.provider)); + return result.map_err(|source| PoolError::Provider { provider: provider.args.provider, source: Box::new(source) }); } - Err(anyhow::anyhow!( - "no eligible paid provider could start this request: {}", - provider_errors.join("; ") - )) + Err(PoolError::ProvidersUnavailable(provider_errors)) }.await; if let Err(error) = &result { if recovery { @@ -580,7 +516,7 @@ impl PaidGateway { // unfinished operation for recovery. match result { Ok(events) => for event in events { - if !streamed || matches!(event, ExecutionEvent::Done(_)) { + if !streamed || event.is_terminal() { emit(&sender, &overflow, &output_budget, Ok(event)); } }, @@ -593,54 +529,88 @@ impl PaidGateway { } } +fn has_retained_setup(options: &PaidWorkOptions) -> Result { + let found = hellas_work::work_store::discover_setups( + &options.journal_root, + options.config.chain.network, + )?; + // Counter files and other metadata alone must never trigger channel funding. + // Unidentified setup journals still go through ordinary recovery and fail + // closed; discovery is only a prefilter, not a signature check. + Ok(!found.unidentified.is_empty() + || found.setups.iter().any(|setup| { + setup.role == hellas_work::work_store::Role::Client && setup.bond_edge == options.bond + })) +} + impl PaidExecutionBackend for PaidGateway { - fn timeout(&self) -> Duration { - Duration::from_secs(self.providers[0].args.timeout_secs) + fn fetch_providers(&self) -> Vec { + self.providers.iter().filter(|provider| matches!(provider.policy.execution_policy, + PaidWorkPolicy::Fetch { policy, .. } if policy.allowed_environment == hellas_rpc::FetchEnvironment::Http.manifest_id())) + .map(|provider| provider.args.provider).collect() } - fn execute( + fn fetch( &self, - request: PaidExecutionRequest, - ) -> CliResult>> { + request: PaidFetchRequest, + ) -> Result { + use hellas_gateway::PaidGatewayError; + let provider = self.providers.iter().find(|provider| provider.args.provider == request.provider + && matches!(provider.policy.execution_policy, PaidWorkPolicy::Fetch { policy, .. } + if policy.allowed_environment == hellas_rpc::FetchEnvironment::Http.manifest_id())) + .ok_or(PaidGatewayError::Provider(request.provider))?; let permit = self .admission .clone() .try_acquire_owned() .map_err(|_| hellas_gateway::PaidGatewayBusy)?; - let input_ids = request.input_ids.clone(); - let cache_update = CacheUpdate::from_request(&request.environment, &input_ids); - let prepared = prepare_request(request, &self.settlement_key)?; - let eligible = self - .providers + let submit = || -> Result<_> { + let environment = hellas_rpc::FetchEnvironment::Http; + let input = hellas_rpc::fetch::build_input_events_with_retention( + &request.service, + &request.method, + &request.body, + environment.manifest_id(), + provider.assurance, + &self.producer_key, + hellas_rpc::Retention::Ephemeral, + )?; + let prepared = hellas_rpc::protocol::work_fetch::PreparedPaidFetchInputV1::new( + &input, + &environment.manifest(), + )?; + let route = Route { + available: provider.available(), + cache_affinity_tokens: 0, + pending: provider.pending.load(Ordering::Relaxed), + }; + // Account/session selection is already final. Never substitute another + // provider, including on failure before acceptance. + self.submit::( + vec![(provider.clone(), route)], + Some(prepared.into()), + None, + None, + Some(permit), + ) + }; + paid_stream(submit()) + } + + fn timeout(&self) -> Duration { + self.providers .iter() - .filter(|provider| check_policy_input(&provider.policy, &prepared).is_ok()) - .collect::>(); - anyhow::ensure!( - !eligible.is_empty(), - "no provider policy matches this environment, token limit, and stop token list" - ); - let start = self.next.fetch_add(1, Ordering::Relaxed) % eligible.len(); - let mut candidates = (0..eligible.len()) - .map(|offset| { - let provider = eligible[(start + offset) % eligible.len()]; - let cache = provider.cache.lock().expect("provider cache poisoned"); - let route = Route { - available: provider.available(), - cache_affinity_tokens: cache.affinity(&input_ids), - pending: provider.pending.load(Ordering::Relaxed), - }; - (provider.clone(), route) - }) - .collect::>(); - // Stable sorting preserves the rotating order for equally ranked routes. - candidates.sort_by_key(|(_, route)| std::cmp::Reverse(route.score())); - self.submit( - candidates, - Some(prepared), - Some(cache_update), - None, - Some(permit), - ) + .map(|provider| provider.args.timeout) + .max() + .unwrap_or_default() + } + + fn execute( + &self, + request: PaidExecutionRequest, + ) -> Result, hellas_gateway::PaidGatewayError> + { + paid_stream(self.execute_tokens(request)) } fn drain(&self) -> BoxFuture<'_, ()> { @@ -649,14 +619,6 @@ impl PaidExecutionBackend for PaidGateway { self.admission.close(); std::mem::take(&mut *tasks) }; - for follower in self - .followers - .lock() - .expect("paid followers poisoned") - .drain(..) - { - follower.abort(); - } Box::pin(async move { let deadline = tokio::time::Instant::now() + Duration::from_secs(60); let mut interrupted = 0; @@ -668,6 +630,7 @@ impl PaidExecutionBackend for PaidGateway { } Err(_) => { task.abort(); + let _ = task.await; interrupted += 1; } } @@ -678,38 +641,42 @@ impl PaidExecutionBackend for PaidGateway { "paid gateway shutdown deadline reached; retained work will recover on startup" ); } + for provider in &self.providers { + if let Some(mut session) = provider.serial.lock().await.take() { + session.shutdown().await; + } + } self.endpoint.close().await; }) } } -fn emit( - sender: &mpsc::Sender, +fn paid_stream( + stream: Result>>, +) -> Result, hellas_gateway::PaidGatewayError> { + use futures::StreamExt as _; + fn convert(error: PoolError) -> hellas_gateway::PaidGatewayError { + match error { + PoolError::Busy(busy) => busy.into(), + error => hellas_gateway::PaidGatewayError::Payment(Box::new(error)), + } + } + Ok(Box::pin( + stream.map_err(convert)?.map(|event| event.map_err(convert)), + )) +} + +fn emit( + sender: &mpsc::Sender>, overflow: &watch::Sender, budget: &Arc, - event: CliResult, + event: Result, ) { if *overflow.borrow() || sender.is_closed() { return; } let bytes = match &event { - Ok(ExecutionEvent::Chunk { tokens, .. }) => tokens.len(), - // Payload bytes only. `OUTPUT_EVENT_OVERHEAD` is charged once below, - // for the one channel message this becomes -- it is the per-message - // constant that makes a full channel (`OUTPUT_BUFFER_EVENTS`) exactly - // exhaust `OUTPUT_BUFFER_BYTES`. Charging it per *contained* event - // instead conflated the transcript with the queue: a completion - // carrying ~8000 signed events, well inside the `MAX_RECORD_BYTES` - // transcript cap and ordinary for `max_new_tokens` in the thousands, - // asked for more permits than the semaphore can ever hold. The - // acquire then failed, `overflow` latched, and the client was told - // "paid output consumer is too slow" for work it had already paid - // for and was reading promptly. - Ok(ExecutionEvent::Done(Outcome::Completed { output_events, .. })) => output_events - .iter() - .map(|event| event.payload().len()) - .fold(0usize, usize::saturating_add), - Ok(ExecutionEvent::Done(_)) => MAX_RECORD_BYTES, + Ok(event) => event.bytes(), Err(error) => error.to_string().len(), } .saturating_add(OUTPUT_EVENT_OVERHEAD); @@ -728,16 +695,16 @@ fn emit( } } -fn response_stream( - mut receiver: mpsc::Receiver, +fn response_stream( + mut receiver: mpsc::Receiver>, mut overflow: watch::Receiver, -) -> BoxStream<'static, CliResult> { +) -> BoxStream<'static, Result> { Box::pin(async_stream::try_stream! { loop { let full = *overflow.borrow(); if full { receiver.close(); - Err(anyhow::anyhow!("paid output consumer is too slow; accepted work continues settlement"))?; + Err(PoolError::SlowConsumer)?; } let event = tokio::select! { biased; @@ -756,7 +723,7 @@ fn response_stream( } #[derive(Debug, thiserror::Error)] -enum RequestStopped { +pub enum RequestStopped { #[error("paid request disconnected before proposal")] Disconnected, #[error("paid request deadline elapsed, including queue wait")] @@ -765,34 +732,30 @@ enum RequestStopped { // A route's short connection allowance is recoverable by trying another // provider; only the enclosing request deadline or disconnect is terminal. -async fn connect_before_deadline( - sender: &mpsc::Sender, +async fn connect_before_deadline( + sender: &mpsc::Sender>, cancel_on_disconnect: bool, deadline: tokio::time::Instant, connection_timeout: Duration, - connection: impl std::future::Future>, -) -> CliResult { + connection: impl std::future::Future>, +) -> Result { let connection_deadline = deadline.min(tokio::time::Instant::now() + connection_timeout); before_proposal(sender, cancel_on_disconnect, deadline, |_| async { tokio::time::timeout_at(connection_deadline, connection) .await - .map_err(|_| { - anyhow::anyhow!( - "paid provider connection exceeded its {connection_timeout:?} limit" - ) - })? + .map_err(|_| PoolError::ConnectionTimeout(connection_timeout))? }) .await? } /// Cancel an unproposed HTTP operation without interrupting work whose /// proposal signature may already have reached a provider. -async fn before_proposal>( - sender: &mpsc::Sender, +async fn before_proposal, E>( + sender: &mpsc::Sender>, cancel_on_disconnect: bool, deadline: tokio::time::Instant, operation: impl FnOnce(Arc) -> F, -) -> CliResult { +) -> Result { if tokio::time::Instant::now() >= deadline { return Err(RequestStopped::Deadline.into()); } @@ -818,11 +781,12 @@ async fn before_proposal>( fn prepare_request( request: PaidExecutionRequest, signer: &Secp256k1Signer, -) -> CliResult { - anyhow::ensure!( - request.max_new_tokens > 0, - "max_new_tokens must be greater than zero" - ); +) -> Result { + if request.max_new_tokens == 0 { + return Err(PoolError::Invalid( + "max_new_tokens must be greater than zero", + )); + } let manifest = request.environment.manifest(); hellas_client::iroh::validate_causal_lm_quote_request( &hellas_rpc::pb::courtesy::QuoteTokensRequest { @@ -856,7 +820,7 @@ fn prepare_request( )) } -fn output_events(output: PaidOutput) -> CliResult> { +fn output_events(output: PaidWorkResult) -> Result> { let events = hellas_rpc::protocol::work::decode_transcript(&output.transcript, MAX_RECORD_BYTES)?; let verified = hellas_rpc::evaluate::verify_output_events_for_producer( @@ -875,11 +839,9 @@ fn output_events(output: PaidOutput) -> CliResult> { let terminal = verified.terminal; let stop_reason = if terminal.stop_reason == hellas_rpc::evaluate::EvaluateStopReason::STOP_TOKEN { - StopReason::StopToken( - terminal - .matched_stop_token_id - .context("signed stop-token result omitted its token ID")?, - ) + StopReason::StopToken(terminal.matched_stop_token_id.ok_or( + PoolError::MissingOutput("signed stop-token result omitted its token ID"), + )?) } else { StopReason::MaxNewTokens }; @@ -899,13 +861,141 @@ fn output_events(output: PaidOutput) -> CliResult> { Ok(result) } +impl GatewayEvent for ExecutionEvent { + fn prefix(event: hellas_rpc::OutputEventEnvelope) -> Result { + let delta = hellas_rpc::evaluate::decode_token_delta_payload(event.payload())?; + Ok(Self::Chunk { + position: delta.end_position()?, + tokens: delta.token_bytes(), + }) + } + fn completed(output: PaidWorkResult) -> Result> { + output_events(output) + } + fn is_terminal(&self) -> bool { + matches!(self, Self::Done(_)) + } + fn bytes(&self) -> usize { + match self { + Self::Chunk { tokens, .. } => tokens.len(), + Self::Done(Outcome::Completed { output_events, .. }) => output_events + .iter() + .map(|event| event.payload().len()) + .fold(0usize, usize::saturating_add), + Self::Done(_) => MAX_RECORD_BYTES, + } + } +} + +impl GatewayEvent for FetchEvent { + fn prefix(event: hellas_rpc::OutputEventEnvelope) -> Result { + Ok(hellas_rpc::fetch::decode_fetch_event_payload( + event.payload(), + )?) + } + fn completed(output: PaidWorkResult) -> Result> { + let events = hellas_rpc::protocol::work::decode_transcript( + &output.transcript, + hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES, + )?; + let terminal = events + .last() + .ok_or(PoolError::MissingOutput("paid Fetch omitted terminal"))?; + if terminal.event().body().kind() != hellas_rpc::fetch::OUTPUT_TERMINAL_KIND { + return Err(PoolError::MissingOutput("paid Fetch terminal kind")); + } + let terminal = + hellas_rpc::fetch::decode_fetch_terminal_payload(terminal.payload())?.to_output_event(); + tracing::info!(work_id = %hex::encode(output.work_id.as_bytes()), job_price = output.job_price, + credited_cumulative = output.credited_cumulative, result_bytes = output.transcript.len(), "paid Fetch result acknowledged"); + Ok(vec![terminal]) + } + fn is_terminal(&self) -> bool { + self.terminal().is_some() + } + fn bytes(&self) -> usize { + serde_json::to_vec(self).map_or(MAX_RECORD_BYTES, |bytes| bytes.len()) + } +} + #[cfg(test)] mod tests { use super::*; + use iroh::Endpoint; + + fn pool_options(fixture: &crate::test_support::PaidFixture) -> PaidGatewayOptions { + use hellas_kernel::{CoinId, Funding, List}; + PaidGatewayOptions { + max_pending_requests: 4, + providers: vec![PaidWorkOptions { + config: fixture.config.clone(), + journal_root: fixture.root.path().join("client"), + provider: iroh::SecretKey::from_bytes(&[3; 32]).public(), + provider_addrs: Vec::new(), + provider_trust: None, + bond: fixture.descriptor.bond_edge(), + payment_funding: Funding::new( + List::take([CoinId::from_bytes([1; 32]); 4], 1), + List::empty(CoinId::from_bytes([0; 32])), + ), + omission_bond: 601, + acceptance_blocks: 16, + terminal_blocks: 64, + payment_blocks: 32, + timeout: Duration::from_secs(30), + }], + } + } + + #[test] + fn counter_files_do_not_trigger_funding_but_incomplete_setup_recovers() { + let fixture = crate::test_support::PaidFixture::new(); + let options = pool_options(&fixture); + let provider = &options.providers[0]; + assert!(!has_retained_setup(provider).unwrap()); + std::fs::create_dir_all(provider.journal_root.join("apple-counters")).unwrap(); + std::fs::write( + provider + .journal_root + .join("apple-counters/producer.counter"), + [1], + ) + .unwrap(); + assert!(!has_retained_setup(provider).unwrap()); + let store = hellas_work::work_store::SetupStore::open( + &provider.journal_root, + provider.config.chain.network, + provider.bond, + hellas_work::work_store::Role::Client, + &hellas_kernel::Secp256k1Verifier::new(), + ) + .unwrap(); + drop(store); + assert!(has_retained_setup(provider).unwrap()); + } + + #[tokio::test] + async fn shared_pool_rejects_reused_funding_before_opening_or_writing() { + let fixture = crate::test_support::PaidFixture::new(); + let mut options = pool_options(&fixture); + let mut second = options.providers[0].clone(); + second.provider = iroh::SecretKey::from_bytes(&[4; 32]).public(); + second.journal_root = fixture.root.path().join("other"); + options.providers.push(second); + let result = PaidGateway::open(options, crate::ClientIdentity::generate()).await; + assert!(matches!( + result, + Err(PoolError::Invalid( + "a payment coin cannot fund two provider channels or appear twice" + )) + )); + assert!(!fixture.root.path().join("client").exists()); + assert!(!fixture.root.path().join("other").exists()); + } #[tokio::test] async fn disconnected_queued_request_never_starts_work() { - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); drop(receiver); let started = AtomicBool::new(false); let result = before_proposal( @@ -923,7 +1013,7 @@ mod tests { #[tokio::test] async fn disconnect_after_proposal_still_finishes_payment() { - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let (proposal_sent, proposal_seen) = tokio::sync::oneshot::channel(); let (payment_ready, payment_wait) = tokio::sync::oneshot::channel(); let work = before_proposal( @@ -949,7 +1039,7 @@ mod tests { #[tokio::test] async fn disconnect_during_dial_cancels_before_signature_release() { - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let signed = AtomicBool::new(false); let signed_ref = &signed; let (dial_started, dial_seen) = tokio::sync::oneshot::channel(); @@ -969,24 +1059,24 @@ mod tests { drop(receiver); }; let (result, ()) = tokio::join!(operation, disconnect); - assert!(result.unwrap_err().is::()); + assert!(matches!(result.unwrap_err(), PoolError::Stopped(_))); assert!(!signed.load(Ordering::Acquire)); } #[tokio::test] async fn connection_timeout_allows_fallback_within_the_request_deadline() { - let (sender, _receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, _receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let deadline = tokio::time::Instant::now() + Duration::from_secs(1); let failed = connect_before_deadline( &sender, true, deadline, Duration::from_millis(10), - std::future::pending::>(), + std::future::pending::>(), ) .await .unwrap_err(); - assert!(!failed.is::()); + assert!(!matches!(failed, PoolError::Stopped(_))); assert_eq!( connect_before_deadline(&sender, true, deadline, Duration::from_millis(10), async { Ok("second provider") @@ -1004,12 +1094,12 @@ mod tests { ) .await .unwrap_err(); - assert!(expired.is::()); + assert!(matches!(expired, PoolError::Stopped(_))); } #[tokio::test] async fn previous_proposal_does_not_authorize_disconnected_next_operation() { - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let deadline = tokio::time::Instant::now() + Duration::from_secs(1); before_proposal(&sender, true, deadline, |proposed| async move { proposed.store(true, Ordering::Release); @@ -1024,7 +1114,7 @@ mod tests { started.store(true, Ordering::Release); }) .await; - assert!(result.unwrap_err().is::()); + assert!(matches!(result.unwrap_err(), PoolError::Stopped(_))); assert!(!started.load(Ordering::Acquire)); } @@ -1033,23 +1123,24 @@ mod tests { let gateway = PaidGateway { providers: Vec::new(), next: AtomicUsize::new(0), - endpoint: Endpoint::builder(presets::Minimal).bind().await.unwrap(), + endpoint: Endpoint::builder(iroh::endpoint::presets::Minimal) + .bind() + .await + .unwrap(), settlement_key: Secp256k1Signer::from_secret_scalar([7; 32]).unwrap(), + producer_key: hellas_rpc::ProducerSigningKey::from_secret_bytes([7; 32]).unwrap(), admission: Arc::new(Semaphore::new(1)), tasks: Mutex::new(Vec::new()), - followers: Mutex::new(Vec::new()), }; // Deterministically pause execute at the point after admission but // before preparation/routing has reached task registration. let permit = gateway.admission.clone().try_acquire_owned().unwrap(); gateway.drain().await; - let result = gateway.submit(Vec::new(), None, None, None, Some(permit)); - assert!( - result - .err() - .expect("submission after drain") - .is::() - ); + let result = gateway.submit::(Vec::new(), None, None, None, Some(permit)); + assert!(matches!( + result.err().expect("submission after drain"), + PoolError::Busy(_) + )); assert!(gateway.tasks.lock().unwrap().is_empty()); assert_eq!(gateway.admission.available_permits(), 1); } @@ -1079,7 +1170,7 @@ mod tests { #[tokio::test] async fn buffered_reconnect_burst_is_delivered_and_releases_byte_budget() { use futures::StreamExt; - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let (overflow, overflow_receiver) = watch::channel(false); let budget = Arc::new(Semaphore::new(OUTPUT_BUFFER_BYTES)); // Retained output can arrive in one burst before HTTP gets a poll. @@ -1109,7 +1200,7 @@ mod tests { #[tokio::test] async fn slow_reader_gets_an_error_without_blocking_payment() { use futures::StreamExt; - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let (overflow, overflow_receiver) = watch::channel(false); let budget = Arc::new(Semaphore::new(OUTPUT_BUFFER_BYTES)); // A stalled reader cannot retain more than the byte budget, and the @@ -1142,7 +1233,7 @@ mod tests { #[tokio::test] async fn queue_wait_uses_the_execution_deadline() { - let (sender, _receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, _receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let serial = AsyncMutex::new(()); let _busy = serial.lock().await; let result = before_proposal( @@ -1162,7 +1253,7 @@ mod tests { #[tokio::test] async fn expired_request_does_not_start_a_fallback_provider() { - let (sender, _receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, _receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let started = AtomicBool::new(false); let result = before_proposal(&sender, true, tokio::time::Instant::now(), |_| async { started.store(true, Ordering::Relaxed); diff --git a/crates/sdk/src/paid_gateway/config.rs b/crates/sdk/src/paid_gateway/config.rs new file mode 100644 index 00000000..3a55cdee --- /dev/null +++ b/crates/sdk/src/paid_gateway/config.rs @@ -0,0 +1,249 @@ +use super::{PaidWorkOptions, PoolError, Result}; +use crate::work_config::load_work_config; +use hellas_kernel::{CoinId, EdgeId, Funding, List, MAX_PARTY_INPUTS}; +use hellas_rpc::{Assurance, ContentId}; +use iroh::EndpointId; +use serde::Deserialize; +use std::{ + collections::BTreeSet, + net::SocketAddr, + path::{Path, PathBuf}, + time::Duration, +}; + +/// Typed funding and admission options for an application-owned paid pool. +pub struct PaidGatewayOptions { + pub providers: Vec, + pub max_pending_requests: usize, +} + +impl PaidGatewayOptions { + pub(super) fn validate(&self) -> Result<()> { + if self.providers.is_empty() { + return Err(PoolError::Invalid( + "paid gateway requires at least one provider", + )); + } + if self.max_pending_requests == 0 + || self.max_pending_requests > tokio::sync::Semaphore::MAX_PERMITS + { + return Err(PoolError::Invalid( + "max_pending_requests must be a positive supported semaphore capacity", + )); + } + let mut journals = BTreeSet::new(); + let mut endpoints = BTreeSet::new(); + let mut funding = BTreeSet::new(); + for provider in &self.providers { + if provider.timeout.is_zero() + || provider.acceptance_blocks == 0 + || provider.terminal_blocks == 0 + || provider.payment_blocks == 0 + { + return Err(PoolError::Invalid( + "timeouts and deadline spans must be positive", + )); + } + if !provider.journal_root.is_absolute() { + return Err(PoolError::Invalid("journal_root must be absolute")); + } + if !journals.insert(&provider.journal_root) { + return Err(PoolError::Invalid( + "paid providers must have distinct journal roots", + )); + } + if !endpoints.insert(provider.provider) { + return Err(PoolError::DuplicateProvider(provider.provider)); + } + let coins = provider + .payment_funding + .maker() + .iter() + .chain(provider.payment_funding.taker().iter()) + .collect::>(); + if coins.is_empty() { + return Err(PoolError::Invalid("paid provider needs payment coins")); + } + for coin in coins { + if !funding.insert(*coin) { + return Err(PoolError::Invalid( + "a payment coin cannot fund two provider channels or appear twice", + )); + } + } + } + Ok(()) + } +} + +/// Load the operator JSON format used by both CLI and native hosts. File paths +/// must be absolute; Apple counter state lives beside the channel journals. +pub fn load_pool_options(path: &Path, assurance: Assurance) -> Result { + let bytes = hellas_private::read_bounded_regular_file( + path, + hellas_work::work_store::journal::MAX_RECORD_BYTES, + ) + .map_err(|source| PoolError::Read { + path: path.into(), + source, + })?; + let file: PoolFile = serde_json::from_slice(&bytes).map_err(|source| PoolError::Parse { + path: path.into(), + source, + })?; + let providers = file + .providers + .into_iter() + .map(|p| { + if !p.work_config.is_absolute() { + return Err(PoolError::Invalid("work_config must be absolute")); + } + if p.payment_coins.is_empty() || p.payment_coins.len() > MAX_PARTY_INPUTS { + return Err(PoolError::Invalid("payment_coins must name 1 to 4 coins")); + } + let coins = p + .payment_coins + .iter() + .map(|value| fixed("payment_coins", value).map(CoinId::from_bytes)) + .collect::>>()?; + let slots = std::array::from_fn(|i| { + coins + .get(i) + .copied() + .unwrap_or(CoinId::from_bytes([0; CoinId::LENGTH])) + }); + let provider_trust = trust(&p, assurance)?; + Ok(PaidWorkOptions { + config: load_work_config(&p.work_config)?, + journal_root: p.journal_root, + provider: p.provider, + provider_addrs: p.provider_addrs, + provider_trust, + bond: EdgeId::from_bytes(fixed("bond", &p.bond)?), + payment_funding: Funding::new( + List::take(slots, coins.len()), + List::empty(CoinId::from_bytes([0; CoinId::LENGTH])), + ), + omission_bond: p.omission_bond, + acceptance_blocks: file.acceptance_blocks, + terminal_blocks: file.terminal_blocks, + payment_blocks: file.payment_blocks, + timeout: Duration::from_secs(file.timeout_secs), + }) + }) + .collect::>()?; + let options = PaidGatewayOptions { + providers, + max_pending_requests: file.max_pending_requests, + }; + options.validate()?; + Ok(options) +} + +fn trust( + p: &ProviderFile, + assurance: Assurance, +) -> Result> { + if p.provider_genesis.is_none() && assurance == Assurance::ProducerSigned { + if p.apple_app_id.is_some() || !p.apple_cd_hashes.is_empty() { + return Err(PoolError::Invalid("Apple trust requires provider_genesis")); + } + return Ok(None); + } + let expected_genesis = p.provider_genesis.ok_or(PoolError::Invalid( + "attested paid providers require provider_genesis", + ))?; + let apple_app_attest = if p.apple_app_id.is_none() + && p.apple_cd_hashes.is_empty() + && assurance == Assurance::ProducerSigned + { + None + } else { + #[cfg(feature = "apple-verifier")] + { + let app_id = p + .apple_app_id + .clone() + .filter(|v| !v.is_empty()) + .ok_or(PoolError::Invalid("Apple trust requires apple_app_id"))?; + if p.apple_cd_hashes.is_empty() { + return Err(PoolError::Invalid("Apple trust requires apple_cd_hashes")); + } + Some(hellas_client::AppleAppAttestTrust::new( + app_id, + p.apple_cd_hashes + .iter() + .map(|v| fixed("apple_cd_hashes", v)) + .collect::>()?, + std::sync::Arc::new(crate::FilesystemAssertionCounterStore::new( + p.journal_root.join("apple-counters"), + )), + )) + } + #[cfg(not(feature = "apple-verifier"))] + { + return Err(PoolError::Invalid( + "Apple trust requires the SDK apple-verifier feature", + )); + } + }; + Ok(Some(hellas_client::ProviderTrustAnchor { + expected_genesis, + required_assurance: assurance, + apple_app_attest, + })) +} + +fn fixed(field: &'static str, value: &str) -> Result<[u8; N]> { + let mut bytes = [0; N]; + hex::decode_to_slice(value, &mut bytes).map_err(|source| PoolError::Hex { field, source })?; + Ok(bytes) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PoolFile { + providers: Vec, + #[serde(default = "acceptance_blocks")] + acceptance_blocks: u64, + #[serde(default = "terminal_blocks")] + terminal_blocks: u64, + #[serde(default = "payment_blocks")] + payment_blocks: u64, + #[serde(default = "timeout_secs")] + timeout_secs: u64, + #[serde(default = "max_pending_requests")] + max_pending_requests: usize, +} +const fn acceptance_blocks() -> u64 { + 16 +} +const fn terminal_blocks() -> u64 { + 64 +} +const fn payment_blocks() -> u64 { + 32 +} +const fn timeout_secs() -> u64 { + 300 +} +const fn max_pending_requests() -> usize { + 64 +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ProviderFile { + work_config: PathBuf, + journal_root: PathBuf, + provider: EndpointId, + #[serde(default)] + provider_addrs: Vec, + bond: String, + payment_coins: Vec, + omission_bond: u64, + provider_genesis: Option, + apple_app_id: Option, + #[serde(default)] + apple_cd_hashes: Vec, +} diff --git a/crates/sdk/src/paid_gateway/error.rs b/crates/sdk/src/paid_gateway/error.rs new file mode 100644 index 00000000..a631bc08 --- /dev/null +++ b/crates/sdk/src/paid_gateway/error.rs @@ -0,0 +1,65 @@ +use std::{path::PathBuf, time::Duration}; + +#[derive(Debug, thiserror::Error)] +pub enum PoolError { + #[error("invalid paid pool: {0}")] + Invalid(&'static str), + #[error("paid gateway repeats provider {0}")] + DuplicateProvider(iroh::EndpointId), + #[error("no provider policy matches this environment, token limit, and stop token list")] + NoMatchingPolicy, + #[error("no eligible paid provider could start this request: {0:?}")] + ProvidersUnavailable(Vec), + #[error("retained paid work did not recover within its deadline")] + RecoveryTimeout, + #[error("paid provider connection exceeded its {0:?} limit")] + ConnectionTimeout(Duration), + #[error("paid output consumer is too slow; accepted work continues settlement")] + SlowConsumer, + #[error("{0}")] + MissingOutput(&'static str), + #[error("paid provider {provider}: {source}")] + Provider { + provider: iroh::EndpointId, + source: Box, + }, + #[error("cannot read {}: {source}", path.display())] + Read { + path: PathBuf, + source: std::io::Error, + }, + #[error("invalid paid gateway config {}: {source}", path.display())] + Parse { + path: PathBuf, + source: serde_json::Error, + }, + #[error("invalid {field}: {source}")] + Hex { + field: &'static str, + source: hex::FromHexError, + }, + #[error(transparent)] + Stopped(#[from] super::RequestStopped), + #[error(transparent)] + Busy(#[from] hellas_gateway::PaidGatewayBusy), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Config(#[from] crate::work_config::WorkConfigError), + #[error(transparent)] + Client(#[from] crate::paid_client::PaidClientError), + #[error(transparent)] + Canonical(#[from] hellas_rpc::protocol::value::CanonicalDecodeError), + #[error(transparent)] + Work(#[from] hellas_rpc::protocol::work::PaidWorkError), + #[error(transparent)] + Store(#[from] hellas_work::work_store::WorkStoreError), + #[error(transparent)] + Fetch(#[from] hellas_rpc::fetch::FetchProtocolError), + #[error(transparent)] + FetchPayload(#[from] hellas_rpc::fetch::FetchPayloadError), + #[error(transparent)] + Evaluate(#[from] hellas_rpc::evaluate::EvaluateProtocolError), + #[error(transparent)] + Remote(#[from] hellas_client::ClientError), +} diff --git a/crates/sdk/src/paid_provider.rs b/crates/sdk/src/paid_provider.rs new file mode 100644 index 00000000..f65644c3 --- /dev/null +++ b/crates/sdk/src/paid_provider.rs @@ -0,0 +1,1027 @@ +//! Shared provider routing and finalized-chain clock for paid work. +use crate::work_config::WorkRoutes; +use futures::future::BoxFuture; +use hellas_chain::client::VerifiedRemoteLightClient; +use hellas_chain::work_blocks::advance_paid_work_clock; +use hellas_chain::{ + ConsensusInfo, ConsensusVerifier, FinalizedWorkView, WorkBlocks, WorkChannelQuery, +}; +use hellas_kernel::{EdgeId, NetworkId, Secp256k1Signer, Secp256k1Verifier}; +use hellas_rpc::call::WithTrailer; +use hellas_rpc::pb::work::*; +use hellas_rpc::peers::PeerId; +use hellas_rpc::protocol::Digest; +use hellas_rpc::protocol::work::{ + PaidJobAuthorizationV1, PrivateRecord as _, work_id as accepted_work_id, +}; +use hellas_rpc::protocol::work_setup::{ + ProviderChannelPolicy, ReadyChannel, WorkChannelDescriptor, +}; +use hellas_rpc::services::work::WorkHandler; +use hellas_rpc::services::work_setup::WorkSetupHandler; +use hellas_wire::{TransportContext, WireStatus}; +use hellas_work::work::{ + CloseEndpoint, PaidWorkBackend, RunError, RunOutcome, WorkService, run_accepted_work, +}; +use hellas_work::work_close::{FinalizedBlocks, TxSink}; +use hellas_work::work_handshake::{PaymentAdmission, SetupEndpoint, SetupService}; +use hellas_work::work_open::{SetupAdvance, SetupDriveError, SetupProgress, SetupView}; +use hellas_work::work_store::{ChannelStore, JobPhase, Role, SetupStore, discover_setups}; +use std::{ + collections::BTreeMap, + path::PathBuf, + sync::{Arc, Mutex}, + time::Duration, +}; +use tokio::sync::oneshot; +use tracing::{Instrument as _, debug, info, warn}; +/// Provider observation and recovery failures preserve their typed causes. +#[derive(Debug, thiserror::Error)] +pub enum PaidProviderError { + #[error("channel has no admission descriptor")] + NoDescriptor, + #[error("no finalized channel snapshot is available")] + NoSnapshot, + #[error("finalized snapshot names another channel")] + WrongSnapshot, + #[error("accepted work has no execution backend")] + NoBackend, + #[error("observer age must exceed a positive polling interval")] + InvalidObservationPolicy, + #[error("the paid-work clock requires at least one validator")] + NoValidators, + #[error("validator has no finalized block 1 for genesis authentication")] + MissingGenesis, + #[error("validator genesis does not match the configured payload digest")] + GenesisMismatch, + #[error(transparent)] + BlockSource(#[from] hellas_work::work_close::BlockSourceError), + #[error(transparent)] + Endpoint(#[from] hellas_work::work::EndpointError), + #[error(transparent)] + Setup(#[from] hellas_rpc::protocol::work_setup::WorkSetupError), + #[error(transparent)] + Query(#[from] hellas_chain::QueryError), + #[error(transparent)] + CatchUp(#[from] hellas_work::work_close::CatchUpError), + #[error(transparent)] + Consensus(#[from] hellas_chain::ConsensusVerificationError), + #[error("cannot discover work journals under {}: {source}", path.display())] + Discover { + path: PathBuf, + source: hellas_work::work_store::WorkStoreError, + }, +} + +pub type ProductionWorkSource = WorkBlocks; +#[derive(Clone, Copy, Debug)] +pub struct UnmountedWork; + +fn not_ready() -> WorkRefused { + WorkRefused { + code: WorkRefusalCode::NotReady as i32, + reason: "work state is not mounted".to_string(), + } +} + +impl WorkSetupHandler for UnmountedWork { + async fn exchange_setup( + &self, + _request: ExchangeSetupRequest, + _context: TransportContext, + ) -> Result> + Send, WireStatus> { + Ok(ExchangeSetupResponse { + outcome: Some(exchange_setup_response::Outcome::Refused(not_ready())), + }) + } +} + +impl WorkHandler for UnmountedWork { + async fn accept_work( + &self, + _request: AcceptWorkRequest, + _context: TransportContext, + ) -> Result> + Send, WireStatus> { + Ok(AcceptWorkResponse { + outcome: Some(accept_work_response::Outcome::Refused(not_ready())), + }) + } + + async fn deliver_result( + &self, + _request: DeliverResultRequest, + _context: TransportContext, + ) -> Result> + Send, WireStatus> { + Ok(DeliverResultResponse { + outcome: Some(deliver_result_response::Outcome::Refused(not_ready())), + }) + } + + async fn stream_result( + &self, + _request: DeliverResultRequest, + _context: TransportContext, + ) -> Result { + Err(WireStatus::new( + hellas_wire::WireCode::Unavailable, + "paid work channel is not mounted", + )) + } + + async fn admit_certificate( + &self, + _request: AdmitCertificateRequest, + _context: TransportContext, + ) -> Result> + Send, WireStatus> { + Ok(AdmitCertificateResponse { + outcome: Some(admit_certificate_response::Outcome::Refused(not_ready())), + }) + } +} + +/// Validated configuration for driving provider journals. +pub struct WorkRunnerConfig { + /// The network the journals are keyed and the signatures bound to. + pub network: NetworkId, + /// The parent payload digest authenticated by finalized block 1. + pub genesis_payload_digest: Digest, + /// The threshold identity finalized blocks must authenticate under. + pub threshold_identity: Vec, + /// The configured root the setup journals live under. + pub journal_root: PathBuf, + /// Bilateral routes from authenticated peers to owned journals. + pub routes: WorkRoutes, + /// The validator RPCs a read and a submission go to. + pub validators: Vec, + /// How often the clock ticks. + pub poll: Duration, + /// Bounds observer stalls and the age of local admission evidence. + pub max_observation_age: Duration, + /// The key every settlement this node signs is signed with. + pub settlement_key: Secp256k1Signer, + /// What every setup endpoint this node builds countersigns over. + pub policy: ProviderChannelPolicy, +} + +/// Channels indexed by authenticated peer. Handlers share the runner's state; +/// mount locks are released before processing requests. +#[derive(Clone)] +pub struct MountedWork { + mounted: Arc>>>, + driver: Option, +} + +impl Default for MountedWork { + fn default() -> Self { + Self { + mounted: Arc::new(Mutex::new(BTreeMap::new())), + driver: None, + } + } +} + +/// Type-erased runner for an accepted job. +type RunAcceptedWork = dyn Fn(WorkService, ReadyChannel, Digest) -> BoxFuture<'static, Result> + + Send + + Sync; + +#[derive(Clone)] +struct AcceptedWorkDriver(Arc); + +impl AcceptedWorkDriver { + fn new(backend: B) -> Self + where + B: PaidWorkBackend + Send + Sync + 'static, + { + let backend = Arc::new(backend); + Self(Arc::new(move |service, ready, work_id| { + let backend = Arc::clone(&backend); + Box::pin( + async move { run_accepted_work(&service, &ready, backend.as_ref(), work_id).await }, + ) + })) + } + + /// Starts one accepted job without lending its lifetime to either the + /// request path or the close clock. + fn spawn(&self, service: WorkService, ready: ReadyChannel, work_id: Digest) { + let running = (self.0)(service, ready, work_id); + let span = hellas_rpc::request_span!(target: "hellas_request", "paid.provider.execute", hellas.work.id = ?work_id, otel.status_code = tracing::field::Empty); + tokio::spawn(tracing::Instrument::instrument( + async move { + match running.await { + Ok(RunOutcome::Completed { .. }) => { + debug!(?work_id, "the accepted paid job completed") + } + Ok(RunOutcome::Ready { .. }) => { + debug!(?work_id, "the accepted paid job was already complete") + } + Ok(RunOutcome::Running) => { + debug!(?work_id, "the accepted paid job was already running") + } + Ok(RunOutcome::Indeterminate) => { + warn!( + ?work_id, + "the accepted paid job is indeterminate after restart" + ) + } + // `run_accepted_work` has already made backend and + // transcript faults terminal before returning them. The + // remaining errors have no node-local terminal policy; + // keep the exact failure visible to the operator. + Err(error) => { + tracing::Span::current().record("otel.status_code", "ERROR"); + warn!(?work_id, %error, "the accepted paid job did not complete"); + } + } + }, + span, + )); + } +} + +/// A peer's local paid channel. Validator connections belong to the observer. +#[derive(Clone)] +pub struct MountedWorkService { + service: WorkService, + driver: Option, +} + +impl WorkHandler for MountedWorkService { + async fn accept_work( + &self, + request: AcceptWorkRequest, + _context: TransportContext, + ) -> Result> + Send, WireStatus> { + if let Some(response) = self.service.precheck_acceptance(&request) { + return Ok(response); + } + let ready = match self.service.readiness() { + Ok(ready) => ready, + Err(error) => { + debug!(%error, "channel observer is not ready for acceptance"); + return Ok(AcceptWorkResponse { + outcome: Some(accept_work_response::Outcome::Refused(WorkRefused { + code: WorkRefusalCode::NotReady as i32, + reason: error.to_string(), + })), + }); + } + }; + // Derive the id from the request while the accepted response + // is still only a possibility. The response carries only the + // provider signature, and consulting `state.jobs().next()` after it + // leaves would race the clock terminalizing that same job. + let work_id = self + .service + .with_state(|state| { + PaidJobAuthorizationV1::decode(&request.authorization) + .ok() + .map(|authorization| accepted_work_id(state.channel(), &authorization)) + }) + .ok() + .flatten(); + let response = self.service.accept(&request); + if matches!( + response.outcome.as_ref(), + Some(accept_work_response::Outcome::Accepted(_)) + ) { + match (self.driver.as_ref(), work_id) { + (Some(driver), Some(work_id)) => { + driver.spawn(self.service.clone(), ready, work_id); + } + (None, Some(work_id)) => { + warn!(?work_id, "accepted paid work has no execution backend") + } + (_, None) => warn!("accepted paid work has no mounted job to execute"), + } + } + Ok(response) + } + + async fn deliver_result( + &self, + request: DeliverResultRequest, + context: TransportContext, + ) -> Result> + Send, WireStatus> { + self.service.deliver_result(request, context).await + } + + async fn stream_result( + &self, + request: DeliverResultRequest, + context: TransportContext, + ) -> Result { + self.service.stream_result(request, context).await + } + + async fn admit_certificate( + &self, + request: AdmitCertificateRequest, + context: TransportContext, + ) -> Result> + Send, WireStatus> { + self.service.admit_certificate(request, context).await + } +} + +impl MountedWork { + pub fn with_backend(backend: B) -> Self + where + B: PaidWorkBackend + Send + Sync + 'static, + { + Self { + mounted: Arc::new(Mutex::new(BTreeMap::new())), + driver: Some(AcceptedWorkDriver::new(backend)), + } + } + + /// Mounts a channel for a peer. Multiple candidates disable routing for that peer. + pub fn mount(&self, peer: PeerId, service: &WorkService) -> bool { + match self.mounted.lock() { + Ok(mut held) => { + let mounted = held.entry(peer).or_default(); + mounted.push(MountedWorkService { + service: service.clone(), + driver: self.driver.clone(), + }); + mounted.len() == 1 + } + Err(_) => false, + } + } + + /// The one handler mounted for the transport-vouched peer. + pub fn handler(&self, context: &TransportContext) -> Option { + let peer = context + .vouched_peer() + .map(|peer| PeerId::from_bytes(peer.0))?; + self.mounted.lock().ok().and_then(|held| { + let [mounted] = held.get(&peer)?.as_slice() else { + return None; + }; + Some(mounted.clone()) + }) + } + + /// Returns the local service for journal inspection and recovery. + /// Remote requests must use `handler`, which enforces local observer readiness. + pub fn service(&self, context: &TransportContext) -> Option { + self.handler(context).map(|mounted| mounted.service) + } + + /// Unmounts all channels when the clock stops, releasing its journal handles. + pub fn clear_all(&self) { + if let Ok(mut held) = self.mounted.lock() { + for channel in held.values().flatten() { + let _ = channel.service.suspend(); + } + held.clear(); + } + } +} + +/// Setups indexed by authenticated peer. Serving and driving share each journal. +#[derive(Clone, Debug, Default)] +pub struct MountedSetup(Arc>>>); + +#[derive(Clone, Debug)] +struct MountedSetupService { + bond_edge: EdgeId, + service: SetupService, +} + +impl MountedSetup { + /// Adds one owned setup under its authenticated peer. + pub fn mount(&self, peer: PeerId, bond_edge: EdgeId, service: &SetupService) -> bool { + match self.0.lock() { + Ok(mut held) => { + let mounted = held.entry(peer).or_default(); + mounted.push(MountedSetupService { + bond_edge, + service: service.clone(), + }); + mounted.len() == 1 + } + Err(_) => false, + } + } + + /// The exact setup service mounted for the transport-vouched peer, + /// cloned without holding the map across dispatch. + pub fn service(&self, context: &TransportContext) -> Option { + let peer = context + .vouched_peer() + .map(|peer| PeerId::from_bytes(peer.0))?; + self.0.lock().ok().and_then(|held| { + let [mounted] = held.get(&peer)?.as_slice() else { + return None; + }; + Some(mounted.service.clone()) + }) + } + + /// Stops serving only the setup that made this transition. + fn clear(&self, peer: PeerId, bond_edge: EdgeId) { + if let Ok(mut held) = self.0.lock() + && let Some(mounted) = held.get_mut(&peer) + { + mounted.retain(|mounted| mounted.bond_edge != bond_edge); + if mounted.is_empty() { + held.remove(&peer); + } + } + } + + /// Stops serving every setup during runner shutdown. + pub fn clear_all(&self) { + if let Ok(mut held) = self.0.lock() { + held.clear(); + } + } +} + +/// A journal transitions from setup to channel when `advance_setup` returns its +/// mounted store. The store is transferred without reopening its exclusive file. +enum Driven { + /// The journal is driven behind the setup service that answers for + /// it. The policy is retained beside the service: it is the provider + /// authority from which the full channel descriptor is rebuilt after + /// the setup reveals its actual terms. + Setup { + /// The endpoint this journal is both driven and served behind. + service: SetupService, + /// Boxed to keep the other enum variants small. + policy: Box, + }, + /// The channel this setup mounted, including the recovery authority + /// needed to finish a job accepted before a process restart. + Channel(Box), + /// The setup ended, or its mount was refused. Nothing left to + /// drive. + Done, +} + +/// A driven channel recovers accepted jobs even without a peer route. +/// Live acceptance and recovery use the same journaled running marker. +struct DrivenChannel { + service: WorkService, + descriptor: Option, + max_observation_age: Duration, + driver: Option, +} + +impl DrivenChannel { + async fn refresh( + &self, + source: &S, + started: hellas_work::work::ObservationTime, + snapshot: Option, + ) -> Result<(), PaidProviderError> + where + S: FinalizedBlocks + FinalizedWorkView + Sync, + { + let descriptor = self + .descriptor + .as_ref() + .ok_or(PaidProviderError::NoDescriptor)?; + let query = WorkChannelQuery { + bond_edge: descriptor.bond_edge(), + payment_edge: descriptor.channel().payment_edge(), + funding: Default::default(), + }; + let snapshot = snapshot.ok_or(PaidProviderError::NoSnapshot)?; + if snapshot.query() != &query { + return Err(PaidProviderError::WrongSnapshot); + } + let ready = descriptor.check_ready(&snapshot.observed_channel())?; + let cursor = self.service.with_state(|state| state.cursor().0)?; + if ready.check_caught_up(cursor).is_err() { + self.service + .drive()? + .catch_up_to(source, ready.finalized_height()) + .await?; + } + self.service + .observe_ready(ready, started, self.max_observation_age)?; + Ok(()) + } + + fn accepted_work_id(&self) -> Result, PaidProviderError> { + self.service + .with_state(|state| { + let mut jobs = state.jobs(); + let job = jobs.next()?; + (jobs.next().is_none() && job.phase() == JobPhase::Accepted).then(|| job.work_id()) + }) + .map_err(PaidProviderError::from) + } + + /// Resumes accepted work after a fresh readiness check. The durable `JobRunning` + /// record admits one invocation even when a live request races this clock tick. + fn resume_accepted(&self) -> Result { + let Some(work_id) = self.accepted_work_id()? else { + return Ok(false); + }; + let driver = self.driver.as_ref().ok_or(PaidProviderError::NoBackend)?; + let ready = self.service.readiness()?; + driver.spawn(self.service.clone(), ready, work_id); + Ok(true) + } +} + +/// One setup journal on a clock. +struct SetupClock { + max_observation_age: Duration, + /// The bond this journal stakes, so a log line names which one. + bond_edge: EdgeId, + /// The authenticated peer whose configured route names this bond. + /// Together with `bond_edge`, this is the journal's route identity; + /// `None` keeps an unconfigured owned journal on its close clock + /// without making it a fallback service. + route_peer: Option, + /// What is being driven for it. + driven: Driven, +} + +impl SetupClock { + /// Advances the journal; returns false on a source failure so the caller redials. + async fn tick( + &mut self, + source: &S, + signer: &Secp256k1Signer, + work_mount: &MountedWork, + setup_mount: &MountedSetup, + ) -> bool + where + S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, + { + let bond = hex::encode(self.bond_edge.to_bytes()); + let mut answered = true; + // One step of the setup this journal holds, with the policy the + // mount it may hand back is rebuilt from; nothing, once the + // journal is past its setup. + let step = match &mut self.driven { + Driven::Setup { service, policy } => Some(( + service.advance_setup(source, source, source).await, + policy.clone(), + )), + Driven::Channel(_) | Driven::Done => None, + }; + if let Some((step, policy)) = step { + match step { + Ok(SetupAdvance { progress, mounted }) => { + if let Some(store) = mounted { + if let Some(peer) = self.route_peer { + setup_mount.clear(peer, self.bond_edge); + } + self.take_mount(store, signer, &policy, work_mount); + } else if matches!( + progress, + SetupProgress::Aborted(_) | SetupProgress::Faulted(_) + ) { + warn!(bond, ?progress, "this setup ended with no channel to drive"); + self.driven = Driven::Done; + } else { + debug!(bond, ?progress, "the setup advanced"); + } + } + Err(error) => { + answered = !matches!(error, SetupDriveError::Source(_)); + warn!(bond, %error, "this setup did not advance"); + } + } + } + if let Driven::Channel(channel) = &self.driven { + let started = hellas_work::work::ObservationTime::now(); + let snapshot = match advance_paid_work_clock(&channel.service, source).await { + Ok(progress) => { + debug!(bond, close = ?progress.close, "the channel advanced"); + progress.snapshot + } + Err(error) => { + let _ = channel.service.suspend(); + warn!(bond, %error, "this channel's close did not advance"); + return !error.source_failed(); + } + }; + if let Err(error) = channel.refresh(source, started, snapshot).await { + let _ = channel.service.suspend(); + debug!(bond, %error, "channel observation did not renew admission"); + answered &= !matches!( + error, + PaidProviderError::Query(_) + | PaidProviderError::CatchUp( + hellas_work::work_close::CatchUpError::Source(_) + ) + ); + } else if let Err(error) = channel.resume_accepted() { + warn!(bond, %error, "an accepted paid job did not resume"); + } + } + answered + } + + /// Transfers the mounted store returned by setup, preserving its exclusive lock. + fn take_mount( + &mut self, + store: ChannelStore, + signer: &Secp256k1Signer, + policy: &ProviderChannelPolicy, + mount: &MountedWork, + ) { + let bond = hex::encode(self.bond_edge.to_bytes()); + // Rebuild the descriptor from the retained policy and signed payment terms. + let descriptor = { + let channel = store.state().channel(); + match policy.admit(channel.payment_edge(), channel.payment_terms().clone()) { + Ok(descriptor) => Some(descriptor), + Err(error) => { + warn!(bond, %error, "the mounted channel no longer satisfies its admission policy"); + None + } + } + }; + match CloseEndpoint::new(store, signer.clone()) { + Ok(close) => { + let service = WorkService::close_only(close); + if let Err(error) = service.require_observer() { + warn!(bond, %error, "channel observer could not be installed"); + self.driven = Driven::Done; + return; + } + if self + .route_peer + .is_some_and(|peer| mount.mount(peer, &service)) + { + info!( + bond, + "this node now answers Work from the channel it mounted" + ); + } else { + warn!( + bond, + "this channel has no unique peer route; it is driven and not served", + ); + } + self.driven = Driven::Channel(Box::new(DrivenChannel { + service, + descriptor, + max_observation_age: self.max_observation_age, + driver: mount.driver.clone(), + })); + } + // The journal and the key are not both the provider's view + // of one channel. Nothing this runner can do about it, and + // dropping the mount is what stops it being reopened every + // tick. + Err(error) => { + warn!(bond, %error, "the mounted channel is not this node's to close"); + self.driven = Driven::Done; + } + } + } +} + +/// The clock, over every paid-work journal this node owns. +pub struct WorkRunner { + clocks: Vec, + signer: Secp256k1Signer, + work_mount: MountedWork, + setup_mount: MountedSetup, + poll: Duration, + validators: Vec, + consensus_verifier: ConsensusVerifier, + genesis_payload_digest: Digest, +} + +impl WorkRunner { + pub fn journal_count(&self) -> usize { + self.clocks.len() + } + pub fn channel_count(&self) -> usize { + self.clocks + .iter() + .filter(|c| matches!(c.driven, Driven::Channel(_))) + .count() + } +} + +impl WorkRunner { + /// Discovers provider journals and mounts configured routes. Unrouted journals + /// are still driven through close. Unreadable journals are logged; failure to + /// enumerate the root is returned to the caller. + pub fn discover( + config: WorkRunnerConfig, + work_mount: MountedWork, + setup_mount: MountedSetup, + ) -> Result { + if config.poll.is_zero() || config.max_observation_age <= config.poll { + return Err(PaidProviderError::InvalidObservationPolicy); + } + if config.validators.is_empty() { + return Err(PaidProviderError::NoValidators); + } + let consensus_verifier = ConsensusVerifier::new(&ConsensusInfo { + validators: config.validators.clone(), + threshold_identity: config.threshold_identity, + network_id: config.network.as_str().to_owned(), + })?; + let settlement_verifier = Secp256k1Verifier::new(); + let found = discover_setups(&config.journal_root, config.network).map_err(|source| { + PaidProviderError::Discover { + path: config.journal_root.clone(), + source, + } + })?; + for unnamed in &found.unidentified { + warn!( + path = %unnamed.path.display(), + reason = %unnamed.reason, + "a setup journal under the work root could not be named", + ); + } + let mut clocks = Vec::with_capacity(found.setups.len()); + for setup in found.setups { + let bond = hex::encode(setup.bond_edge.to_bytes()); + let route_peer = config + .routes + .iter() + .find(|route| route.bond == setup.bond_edge) + .map(|route| route.peer); + // A close capability binds the provider half, and this + // process holds the provider's key. A client journal beside + // this node's own is another party's, and this runner has + // nothing to sign for it. + if setup.role != Role::Provider { + warn!( + bond, + "a setup journal under the work root is not this node's half" + ); + continue; + } + let store = match SetupStore::open( + &config.journal_root, + config.network, + setup.bond_edge, + setup.role, + &settlement_verifier, + ) { + Ok(store) => store, + Err(error) => { + warn!(bond, %error, "a discovered setup journal did not open"); + continue; + } + }; + let policy = Box::new(config.policy.clone()); + let service = SetupService::new(SetupEndpoint::new( + store, + config.settlement_key.clone(), + PaymentAdmission::Admits(policy.clone()), + )); + if let Some(peer) = route_peer { + if setup_mount.mount(peer, setup.bond_edge, &service) { + info!( + bond, + "this node now answers WorkSetup from its driven setup" + ); + } else { + warn!(bond, "this provider setup has an ambiguous peer route"); + } + } else { + warn!(bond, "this provider setup has no configured peer route"); + } + let driven = Driven::Setup { service, policy }; + clocks.push(SetupClock { + max_observation_age: config.max_observation_age, + bond_edge: setup.bond_edge, + route_peer, + driven, + }); + } + Ok(Self { + clocks, + signer: config.settlement_key, + work_mount, + setup_mount, + poll: config.poll, + validators: config.validators, + consensus_verifier, + genesis_payload_digest: config.genesis_payload_digest, + }) + } + + /// Takes one step of every journal, and says whether the chain + /// answered all of them. + pub async fn tick(&mut self, source: &S) -> bool + where + S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, + { + let mut answered = true; + for clock in &mut self.clocks { + answered &= clock + .tick(source, &self.signer, &self.work_mount, &self.setup_mount) + .await; + } + answered + } + + /// Ticks each journal once per period and redials after a source failure. + pub async fn run_over(mut self, mut stop: oneshot::Receiver<()>, dial: D) + where + S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, + D: Fn() -> F, + F: core::future::Future>, + { + if self.clocks.is_empty() { + info!("no provider setup journal under the work root; the clock has nothing to drive"); + self.work_mount.clear_all(); + self.setup_mount.clear_all(); + return; + } + // Each channel owns its observer loop. A slow source or a large restart + // backlog on one channel cannot stop another channel's close response. + { + use futures::{StreamExt as _, stream::FuturesUnordered}; + let signer = &self.signer; + let work_mount = &self.work_mount; + let setup_mount = &self.setup_mount; + let dial = &dial; + let poll = self.poll; + let mut observers = self + .clocks + .iter_mut() + .map(|clock| async move { + let mut chain = None; + loop { + let budget = clock.max_observation_age; + if chain.is_none() { + chain = tokio::time::timeout(budget, dial()).await.ok().flatten(); + } + if let Some(source) = chain.as_ref() { + match tokio::time::timeout( + budget, + clock.tick(source, signer, work_mount, setup_mount).instrument(hellas_rpc::request_span!(target: "hellas_request", parent: None, "paid.channel.observe", hellas.channel.role = "provider")), + ) + .await + { + Ok(true) => {} + Ok(false) | Err(_) => { + if let Driven::Channel(channel) = &clock.driven { + let _ = channel.service.suspend(); + } + chain = None; + } + } + } + if matches!(clock.driven, Driven::Done) { + break; + } + tokio::time::sleep(poll).await; + } + }) + .collect::>(); + loop { + tokio::select! { + _ = &mut stop => break, + next = observers.next() => if next.is_none() { break; }, + } + } + } + self.work_mount.clear_all(); + self.setup_mount.clear_all(); + info!("the paid-work clock stopped, and its journals are closed"); + } +} + +impl WorkRunner { + /// Ticks until told to stop, over the validators the configuration + /// names. + pub async fn run(self, stop: oneshot::Receiver<()>) { + let validators = self.validators.clone(); + let verifier = self.consensus_verifier.clone(); + let genesis = self.genesis_payload_digest; + let next = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + self.run_over(stop, move || { + let validators = validators.clone(); + let verifier = verifier.clone(); + let next = next.clone(); + async move { connect_chain(&validators, verifier, genesis, &next).await } + }) + .await; + } +} + +/// Rotates the first candidate on reconnect, per runner. Chain reads and +/// submissions use the selected verified connection. +async fn connect_chain( + validators: &[String], + verifier: ConsensusVerifier, + genesis: Digest, + next: &std::sync::atomic::AtomicUsize, +) -> Option { + if validators.is_empty() { + return None; + } + let start = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % validators.len(); + for url in validators.iter().cycle().skip(start).take(validators.len()) { + match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { + Ok(client) => { + let source = WorkBlocks::new(client); + if let Err(error) = authenticate_genesis(&source, genesis).await { + warn!(validator = %url, %error, "validator failed genesis authentication"); + continue; + } + info!(validator = %url, "the paid-work clock reads and submits here"); + return Some(source); + } + Err(error) => warn!(validator = %url, %error, "a configured validator did not answer"), + } + } + None +} + +async fn authenticate_genesis( + source: &impl FinalizedBlocks, + expected: Digest, +) -> Result<(), PaidProviderError> { + let first = source + .block_at(1) + .await? + .ok_or(PaidProviderError::MissingGenesis)?; + if &first.parent != expected.as_bytes() { + return Err(PaidProviderError::GenesisMismatch); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use hellas_work::work_close::{BlockSourceError, FinalizedWork}; + + struct Source(Result, BlockSourceError>); + + impl FinalizedBlocks for Source { + async fn latest_height(&self) -> Result, BlockSourceError> { + panic!("genesis authentication must request block 1 directly") + } + + async fn block_at(&self, height: u64) -> Result, BlockSourceError> { + assert_eq!(height, 1); + self.0.clone() + } + } + + #[tokio::test] + async fn validator_connection_requires_the_configured_genesis() { + let block = FinalizedWork { + height: 1, + parent: [7; 32], + payload: [8; 32], + txs: Vec::new(), + }; + authenticate_genesis(&Source(Ok(Some(block.clone()))), [7; 32].into()) + .await + .unwrap(); + assert!(matches!( + authenticate_genesis(&Source(Ok(Some(block))), [9; 32].into()).await, + Err(PaidProviderError::GenesisMismatch) + )); + assert!(matches!( + authenticate_genesis(&Source(Ok(None)), [7; 32].into()).await, + Err(PaidProviderError::MissingGenesis) + )); + assert!(matches!( + authenticate_genesis( + &Source(Err(BlockSourceError::new("offline"))), + [7; 32].into() + ) + .await, + Err(PaidProviderError::BlockSource(_)) + )); + } + + #[test] + fn missing_validators_are_rejected_before_discovering_journals() { + let fixture = crate::test_support::PaidFixture::new(); + let config = &fixture.config; + let runner = WorkRunner::discover( + WorkRunnerConfig { + network: config.chain.network, + genesis_payload_digest: config.chain.genesis_payload_digest, + threshold_identity: config.chain.threshold_identity.clone(), + journal_root: config.journal_root.clone(), + routes: config.routes.clone(), + validators: Vec::new(), + poll: config.poll, + max_observation_age: config.max_observation_age, + settlement_key: crate::test_support::signer(2), + policy: config.provider_policy(), + }, + MountedWork::default(), + MountedSetup::default(), + ); + assert!(matches!(runner, Err(PaidProviderError::NoValidators))); + assert!(!config.journal_root.exists()); + } +} diff --git a/crates/sdk/src/provider.rs b/crates/sdk/src/provider.rs index 8dd6d757..b9338df7 100644 --- a/crates/sdk/src/provider.rs +++ b/crates/sdk/src/provider.rs @@ -3,13 +3,12 @@ use std::sync::Arc; use hellas_attestation::RootProver; use hellas_executor::{ - ExecuteServer, Executor, ExecutorMetrics, ExecutorSpawnConfig, FetchAccessPolicy, - FetchQuotaStoreBackend, FetchRoute, FetchRouteEntry, FetchRoutePolicy, FetchRouteRegistry, - FetchServer, FetchTranscriptStoreBackend, + ExecuteServer, Executor, ExecutorSpawnConfig, FetchAccessPolicy, FetchQuotaStoreBackend, + FetchRoute, FetchRouteEntry, FetchRoutePolicy, FetchRouteRegistry, FetchServer, + FetchTranscriptStoreBackend, }; use hellas_rpc::open::OpenDispatcher; use hellas_rpc::pb::execute::{OpenRequest, OpenResponse, open_response}; -use hellas_rpc::policy::ExecutePolicy; use hellas_rpc::run_ticket::signature_to_pb; use hellas_rpc::serve::MethodDispatcher; use hellas_rpc::services::execute::RunTicket; @@ -17,7 +16,7 @@ use hellas_rpc::services::fetch::{Fetch, Open as FetchOpen}; use hellas_rpc::{ Assurance, OPEN_NONCE_LEN, ProviderEnrollmentBundle, PublicKey, RootProof, open_proof_binding, }; -use hellas_wire::iroh::{IrohTransport, IrohTransportError}; +use hellas_wire::iroh::IrohTransport; use hellas_wire::{ Dispatcher, ServiceMarker, StreamTransport, TransportContext, WireCode, WireStatus, }; @@ -28,6 +27,41 @@ use crate::ClientIdentity; const MAX_ACTIVE_CONNECTIONS: usize = 64; const HANDSHAKE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); +/// Startup errors, before a provider accepts requests. +#[derive(Debug, thiserror::Error)] +pub enum ProviderError { + #[error("provider requires allowed callers or paid-work configuration")] + MissingAccessPolicy, + #[error("paid channel names an unavailable Fetch route or manifest")] + MissingPaidRoute, + #[error("Fetch provider requires a paid Fetch policy")] + WrongPaidPolicy, + #[error("paid Fetch provider requires zero retained transcript capacity")] + PaidRetention, + #[error("invalid provider settlement key")] + SettlementKey, + #[error(transparent)] + OpenAiKey(#[from] hellas_providers::EmptyOpenAiKey), + #[error(transparent)] + RouteBinding(#[from] hellas_executor::FetchRouteBindingError), + #[error(transparent)] + DuplicateRoute(#[from] hellas_executor::DuplicateFetchRoute), + #[error(transparent)] + Executor(#[from] hellas_executor::ExecutorError), + #[error(transparent)] + Bind(#[from] iroh::endpoint::BindError), + #[error(transparent)] + Address(#[from] iroh::endpoint::InvalidSocketAddr), + #[error(transparent)] + Io(#[from] std::io::Error), + #[cfg(feature = "paid-provider")] + #[error(transparent)] + Config(#[from] crate::work_config::WorkConfigError), + #[cfg(feature = "paid-provider")] + #[error(transparent)] + Paid(#[from] crate::paid_provider::PaidProviderError), +} + pub struct OpenAiProviderOptions { pub port: Option, pub identity: ClientIdentity, @@ -44,9 +78,41 @@ pub struct OpenAiProviderOptions { pub fetch_replay_max_in_flight: usize, } +/// An attested Fetch provider with an operator-supplied route registry. +pub struct FetchProviderOptions { + pub port: Option, + pub identity: ClientIdentity, + pub enrollment: ProviderEnrollmentBundle, + pub root: Arc, + pub state_directory: PathBuf, + pub routes: FetchRouteRegistry, + pub allowed_callers: Vec, + pub fetch_max_in_flight: usize, + pub fetch_queue_capacity: usize, + pub retained_transcript_capacity: usize, + pub fetch_replay_max_in_flight: usize, + #[cfg(feature = "paid-provider")] + pub paid_work: Option, +} + +#[cfg(feature = "paid-provider")] +struct WorkWatcher { + stop: Option>, + task: tokio::task::JoinHandle<()>, +} + +#[cfg(feature = "paid-provider")] +impl Drop for WorkWatcher { + fn drop(&mut self) { + self.stop.take(); + } +} + pub struct ProviderHandle { endpoint: Endpoint, accept_task: tokio::task::JoinHandle<()>, + #[cfg(feature = "paid-provider")] + work: Option, } impl ProviderHandle { @@ -61,6 +127,13 @@ impl ProviderHandle { pub async fn shutdown(mut self) { self.accept_task.abort(); let _ = (&mut self.accept_task).await; + #[cfg(feature = "paid-provider")] + if let Some(mut work) = self.work.take() { + if let Some(stop) = work.stop.take() { + let _ = stop.send(()); + } + let _ = (&mut work.task).await; + } self.endpoint.close().await; } } @@ -68,19 +141,19 @@ impl ProviderHandle { impl Drop for ProviderHandle { fn drop(&mut self) { self.accept_task.abort(); + #[cfg(feature = "paid-provider")] + if let Some(work) = &mut self.work { + work.stop.take(); + } } } pub async fn start_openai_provider( options: OpenAiProviderOptions, -) -> anyhow::Result +) -> Result where R: RootProver + Send + Sync + 'static, { - anyhow::ensure!( - !options.allowed_callers.is_empty(), - "provider requires at least one allowed caller" - ); let upstream = Arc::new(hellas_providers::OpenAiResponsesFetchProvider::with_bearer( options.bearer_token, )?); @@ -92,39 +165,156 @@ where FetchRoute::new(options.service, options.method), FetchRouteEntry::new(upstream, adaptor, FetchRoutePolicy::default())?, )?; - let producer_key = Arc::new(options.identity.caller_key().clone()); - let access = FetchAccessPolicy::trusted_callers(options.allowed_callers).with_store( - FetchQuotaStoreBackend::fs(options.state_directory.join("quota")), - ); - let executor = Executor::spawn_configured(ExecutorSpawnConfig { - output_cache: Default::default(), - execute_policy: ExecutePolicy::Deny, - queue_capacity: 1, - metrics: Arc::new(ExecutorMetrics::default()), - producer_key: producer_key.clone(), - provider_genesis: Arc::new(options.enrollment.canonical_bytes()), - assurance: Assurance::AppleAppAttest, - fetch_access_policy: access, - fetch_routes: routes, + start_fetch_provider(FetchProviderOptions { + port: options.port, + identity: options.identity, + enrollment: options.enrollment, + root: options.root, + state_directory: options.state_directory, + routes, + allowed_callers: options.allowed_callers, fetch_max_in_flight: options.fetch_max_in_flight, fetch_queue_capacity: options.fetch_queue_capacity, + retained_transcript_capacity: options.retained_transcript_capacity, fetch_replay_max_in_flight: options.fetch_replay_max_in_flight, - fetch_store: FetchTranscriptStoreBackend::fs_with_capacity( - options.state_directory.join("transcripts"), - options.retained_transcript_capacity, - ), + #[cfg(feature = "paid-provider")] + paid_work: None, }) - .await?; + .await +} +pub async fn start_fetch_provider( + options: FetchProviderOptions, +) -> Result +where + R: RootProver + Send + Sync + 'static, +{ + #[cfg(feature = "paid-provider")] + let has_paid_work = options.paid_work.is_some(); + #[cfg(not(feature = "paid-provider"))] + let has_paid_work = false; + if options.allowed_callers.is_empty() && !has_paid_work { + return Err(ProviderError::MissingAccessPolicy); + } + #[cfg(feature = "paid-provider")] + if let Some(config) = &options.paid_work { + use hellas_rpc::protocol::{ + work_fetch::FetchRoutePolicy as PaidRoute, work_profile::PaidWorkPolicy, + }; + match &config.execution_policy { + PaidWorkPolicy::Fetch { + policy, + route: PaidRoute::SealedRoute { service, method }, + } => { + if !options + .routes + .entry(&FetchRoute::new(service, method)) + .is_some_and(|entry| { + entry.execution_environment() == policy.allowed_environment + }) + { + return Err(ProviderError::MissingPaidRoute); + } + } + PaidWorkPolicy::Fetch { + policy, + route: PaidRoute::OpenFetch { .. }, + } => { + if !options.routes.has_environment(policy.allowed_environment) { + return Err(ProviderError::MissingPaidRoute); + } + } + _ => return Err(ProviderError::WrongPaidPolicy), + } + if options.retained_transcript_capacity != 0 { + return Err(ProviderError::PaidRetention); + } + crate::work_config::validate_work_routes(config)?; + } + let producer_key = Arc::new(options.identity.caller_key().clone()); + let access = FetchAccessPolicy::trusted_callers(options.allowed_callers).with_store( + FetchQuotaStoreBackend::fs(options.state_directory.join("quota")), + ); + let mut executor_config = ExecutorSpawnConfig::fetch_only( + producer_key.clone(), + Arc::new(options.enrollment.canonical_bytes()), + Assurance::AppleAppAttest, + options.routes, + ); + executor_config.fetch_access_policy = access; + executor_config.fetch_max_in_flight = options.fetch_max_in_flight; + executor_config.fetch_queue_capacity = options.fetch_queue_capacity; + executor_config.fetch_replay_max_in_flight = options.fetch_replay_max_in_flight; + executor_config.fetch_store = FetchTranscriptStoreBackend::fs_with_capacity( + options.state_directory.join("transcripts"), + options.retained_transcript_capacity, + ); + let executor = Executor::spawn_configured(executor_config).await?; + + #[cfg(feature = "paid-provider")] + let work_mount = crate::paid_provider::MountedWork::with_backend(executor.clone()); + #[cfg(feature = "paid-provider")] + let setup_mount = crate::paid_provider::MountedSetup::default(); + #[cfg(feature = "paid-provider")] + let work = if let Some(config) = options.paid_work { + let policy = config.provider_policy(); + let settlement_key = hellas_kernel::Secp256k1Signer::from_secret_scalar( + options.identity.caller_secret_bytes(), + ) + .map_err(|_| ProviderError::SettlementKey)?; + let runner = crate::paid_provider::WorkRunner::discover( + crate::paid_provider::WorkRunnerConfig { + network: config.chain.network, + genesis_payload_digest: config.chain.genesis_payload_digest, + threshold_identity: config.chain.threshold_identity, + journal_root: config.journal_root, + routes: config.routes, + validators: config.validators, + poll: config.poll, + max_observation_age: config.max_observation_age, + settlement_key, + policy, + }, + work_mount.clone(), + setup_mount.clone(), + )?; + let (stop, stopped) = tokio::sync::oneshot::channel(); + Some(WorkWatcher { + stop: Some(stop), + task: tokio::spawn(runner.run(stopped)), + }) + } else { + None + }; let open = ProviderOpen { root: options.root, enrollment: options.enrollment, }; + // A paid provider must not expose a second, unpaid route to the same + // upstream credentials, even when legacy Courtesy callers are configured. + let alpns = if has_paid_work { + Vec::new() + } else { + vec![::ALPN.as_bytes().to_vec()] + }; + #[cfg(feature = "paid-provider")] + let alpns = if has_paid_work { + let mut alpns = alpns; + alpns.extend([ + hellas_rpc::services::work::Work::ALPN.as_bytes().to_vec(), + hellas_rpc::services::work_setup::WorkSetup::ALPN + .as_bytes() + .to_vec(), + ]); + alpns + } else { + alpns + }; let mut builder = Endpoint::builder(presets::N0) .secret_key(options.identity.transport_key()) - .alpns(vec![::ALPN.as_bytes().to_vec()]); + .alpns(alpns); if let Some(port) = options.port { - builder = builder.bind_addr(format!("0.0.0.0:{port}").parse::()?)?; + builder = builder.bind_addr(std::net::SocketAddr::from(([0, 0, 0, 0], port)))?; } let endpoint = builder.bind().await?; let accept_endpoint = endpoint.clone(); @@ -156,6 +346,8 @@ where } }; let executor = executor.clone(); + #[cfg(feature = "paid-provider")] + let (work_mount, setup_mount) = (work_mount.clone(), setup_mount.clone()); let open = open.clone(); connections.spawn(async move { let _slot = slot; @@ -170,10 +362,55 @@ where return; } }; - if connection.alpn() != ::ALPN.as_bytes() { + let alpn = connection.alpn().to_vec(); + let transport = Arc::new(IrohTransport::new(connection)); + #[cfg(feature = "paid-provider")] + if has_paid_work && alpn == hellas_rpc::services::work::Work::ALPN.as_bytes() { + let context = transport.context(); + if let Some(handler) = work_mount.handler(&context) { + let server = OpenDispatcher::<_, _, hellas_rpc::services::work::Open>::new( + hellas_rpc::services::work::WorkServer(handler), + open, + ); + serve(transport, server).await; + } else { + let server = OpenDispatcher::<_, _, hellas_rpc::services::work::Open>::new( + hellas_rpc::services::work::WorkServer( + crate::paid_provider::UnmountedWork, + ), + open, + ); + serve(transport, server).await; + } + return; + } + #[cfg(feature = "paid-provider")] + if has_paid_work + && alpn == hellas_rpc::services::work_setup::WorkSetup::ALPN.as_bytes() + { + let context = transport.context(); + if let Some(handler) = setup_mount.service(&context) { + let server = + OpenDispatcher::<_, _, hellas_rpc::services::work_setup::Open>::new( + hellas_rpc::services::work_setup::WorkSetupServer(handler), + open, + ); + serve(transport, server).await; + } else { + let server = + OpenDispatcher::<_, _, hellas_rpc::services::work_setup::Open>::new( + hellas_rpc::services::work_setup::WorkSetupServer( + crate::paid_provider::UnmountedWork, + ), + open, + ); + serve(transport, server).await; + } + return; + } + if has_paid_work || alpn != Fetch::ALPN.as_bytes() { return; } - let transport = Arc::new(IrohTransport::new(connection)); let server = OpenDispatcher::<_, _, FetchOpen>::new( MethodDispatcher::<_, _, RunTicket>::new( ExecuteServer(executor.clone()), @@ -181,32 +418,50 @@ where ), open, ); - loop { - match transport.accept().await { - Ok(Some(inbound)) => { - if let Err(error) = - Dispatcher::::dispatch(&server, inbound).await - { - tracing::warn!(%error, "provider RPC failed"); - break; - } - } - Ok(None) | Err(IrohTransportError::Connection(_)) => break, - Err(error) => { - tracing::warn!(%error, "provider transport failed"); - break; - } - } - } + serve(transport, server).await; }); } }); Ok(ProviderHandle { endpoint, accept_task, + #[cfg(feature = "paid-provider")] + work, }) } +/// Serves one connection until the peer leaves or a dispatch fails. Dispatch +/// is deliberately unbounded here: streaming methods such as `StreamResult` +/// stay open for the lifetime of a job and enforce their own application-level +/// deadlines, so a wall-clock timeout at this layer would cancel paid work +/// mid-delivery. +async fn serve(transport: Arc, server: S) +where + T: StreamTransport + Send + Sync + 'static, + T::Error: std::fmt::Display, + S: Dispatcher + Send + Sync + 'static, + S::Error: std::fmt::Display + Send + Sync + 'static, +{ + loop { + match transport.accept().await { + Ok(Some(inbound)) => { + if let Err(error) = Dispatcher::::dispatch(&server, inbound).await { + tracing::warn!(%error, "provider RPC failed"); + break; + } + } + Ok(None) => break, + Err(error) => { + tracing::debug!(%error, "provider transport ended"); + break; + } + } + } +} + +#[cfg(all(test, feature = "paid-provider", feature = "paid-client"))] +mod tests; + struct ProviderOpen { root: Arc, enrollment: ProviderEnrollmentBundle, diff --git a/crates/sdk/src/provider/tests.rs b/crates/sdk/src/provider/tests.rs new file mode 100644 index 00000000..512df1ae --- /dev/null +++ b/crates/sdk/src/provider/tests.rs @@ -0,0 +1,186 @@ +use super::*; +use bytes::Bytes; +use futures::StreamExt as _; +use hellas_rpc::pb::work::{DeliverResultRequest, WorkStreamEvent}; +use hellas_rpc::services::work::StreamResult; +use hellas_wire::mux::{MessagePipe, MuxConfig, MuxTransport, Role}; +use hellas_wire::{DefaultClock, Metadata}; +use tokio::sync::mpsc; +use tokio::time::{Duration, Instant}; + +struct UnusedRoot; + +impl RootProver for UnusedRoot { + async fn prove_statement( + &self, + _: &[u8], + ) -> Result { + panic!("ALPN negotiation must not invoke attestation") + } + + async fn prove_open_binding( + &self, + _: hellas_rpc::Digest, + ) -> Result { + panic!("ALPN negotiation must not invoke attestation") + } +} + +#[tokio::test] +async fn paid_provider_refuses_unpaid_fetch_even_with_legacy_allowed_callers() { + tokio::time::timeout(Duration::from_secs(15), async { + let fixture = crate::test_support::PaidFixture::new(); + let identity = ClientIdentity::from_secret_bytes([4; 32], [2; 32]).unwrap(); + let (enrollment, _) = crate::test_support::enrollment(identity.transport_key().public()); + let mut routes = FetchRouteRegistry::new(); + routes + .register( + FetchRoute::new("openai", "responses"), + FetchRouteEntry::new( + Arc::new( + hellas_providers::OpenAiResponsesFetchProvider::with_bearer("fixture") + .unwrap(), + ), + Arc::new(hellas_providers::ResponsesFetchAdaptorFactory::new( + hellas_rpc::FetchEnvironment::OpenAiResponses, + )), + FetchRoutePolicy::default(), + ) + .unwrap(), + ) + .unwrap(); + let provider = start_fetch_provider(FetchProviderOptions { + port: Some(0), + identity, + enrollment, + root: Arc::new(UnusedRoot), + state_directory: fixture.root.path().join("state"), + routes, + allowed_callers: vec![ + hellas_rpc::ProducerSigningKey::from_secret_bytes([1; 32]) + .unwrap() + .public_key(), + ], + fetch_max_in_flight: 1, + fetch_queue_capacity: 1, + retained_transcript_capacity: 0, + fetch_replay_max_in_flight: 1, + paid_work: Some(fixture.config), + }) + .await + .unwrap(); + let client = Endpoint::builder(presets::Minimal) + .bind_addr("127.0.0.1:0".parse::().unwrap()) + .unwrap() + .bind() + .await + .unwrap(); + let addr = iroh::EndpointAddr::new(provider.node_id()).with_addrs( + provider + .bound_sockets() + .into_iter() + .filter(|addr| addr.is_ipv4()) + .map(|mut addr| { + addr.set_ip(std::net::Ipv4Addr::LOCALHOST.into()); + iroh::TransportAddr::Ip(addr) + }), + ); + assert!( + client + .connect(addr.clone(), Fetch::ALPN.as_bytes()) + .await + .is_err() + ); + for alpn in [ + hellas_rpc::services::work::Work::ALPN, + hellas_rpc::services::work_setup::WorkSetup::ALPN, + ] { + let connection = client.connect(addr.clone(), alpn.as_bytes()).await.unwrap(); + connection.close(0u32.into(), b"tested"); + } + client.close().await; + provider.shutdown().await; + }) + .await + .expect("paid provider protocol negotiation completes"); +} + +struct Pipe(mpsc::UnboundedSender, mpsc::UnboundedReceiver); + +impl MessagePipe for Pipe { + type SendError = std::io::Error; + type RecvError = std::io::Error; + + async fn send_message(&mut self, bytes: Bytes) -> Result<(), Self::SendError> { + self.0 + .send(bytes) + .map_err(|_| std::io::ErrorKind::BrokenPipe.into()) + } + + async fn recv_message(&mut self) -> Result, Self::RecvError> { + Ok(self.1.recv().await) + } +} + +struct SlowStream; + +impl Dispatcher for SlowStream { + type Error = hellas_wire::TransportError; + + async fn dispatch( + &self, + inbound: hellas_wire::Inbound, + ) -> Result<(), Self::Error> { + hellas_rpc::call::dispatch_server_streaming::( + inbound, + |_: DeliverResultRequest| async { + Ok(Box::pin(async_stream::stream! { + yield Ok(WorkStreamEvent::default()); + tokio::time::sleep(Duration::from_secs(61)).await; + yield Ok(WorkStreamEvent::default()); + })) + }, + ) + .await + } +} + +#[tokio::test(start_paused = true)] +async fn paid_stream_survives_a_slow_upstream_and_an_idle_connection() { + let (to_server, server_inbox) = mpsc::unbounded_channel(); + let (to_client, client_inbox) = mpsc::unbounded_channel(); + let client = MuxTransport::spawn::<8, _, _>( + Role::Client, + DefaultClock, + MuxConfig::default(), + Pipe(to_server, client_inbox), + TransportContext::default(), + ); + let server = MuxTransport::spawn::<8, _, _>( + Role::Server, + DefaultClock, + MuxConfig::default(), + Pipe(to_client, server_inbox), + TransportContext::default(), + ); + let serving = tokio::spawn(serve(Arc::new(server), SlowStream)); + for _ in 0..2 { + let started = Instant::now(); + let mut response = hellas_rpc::call::server_streaming::<_, StreamResult>( + &client, + DeliverResultRequest::default(), + Metadata::new(), + ) + .await + .unwrap(); + assert!(response.next().await.unwrap().is_ok()); + assert!(response.next().await.unwrap().is_ok()); + assert!(response.next().await.is_none()); + response.finish().unwrap(); + assert!(started.elapsed() >= Duration::from_secs(61)); + tokio::time::sleep(Duration::from_secs(121)).await; + assert!(!serving.is_finished()); + } + serving.abort(); + let _ = serving.await; +} diff --git a/crates/sdk/src/test_support.rs b/crates/sdk/src/test_support.rs new file mode 100644 index 00000000..88245c00 --- /dev/null +++ b/crates/sdk/src/test_support.rs @@ -0,0 +1,264 @@ +//! A funded, metadata-only Fetch channel for SDK boundary tests. +// Client and provider feature suites use different parts of this fixture. +#![allow(dead_code)] +use crate::work_config::{ChainCrossCheck, WorkConfig}; +use hellas_kernel::{ + BlockHeight, Decode as _, Edge, EdgeId, EdgeValues, Fees, Key, LeaseSlots, List, + MAX_EDGE_OUTPUTS, NetworkId, Parties, Payout, PendingSlot, RegistryChunk, RegistryNamespace, + RegistryRecordTag, Secp256k1Signer, Secp256k1Verifier, Terms, TermsHash, WorkPaymentTerms, + WorkStakeBondTerms, work_payment_settlement, +}; +use hellas_rpc::protocol::work::{JobDeadlines, PaidChannelPolicyV1, private_policy_commitment}; +use hellas_rpc::protocol::work_fetch::{ + FetchRoutePolicy, PaidFetchPolicyV1, PreparedPaidFetchInputV1, fetch_route_commitment, +}; +use hellas_rpc::protocol::work_profile::PaidWorkPolicy; +use hellas_rpc::protocol::work_setup::{ + ObservedChannel, ReadyChannel, WorkChannelConfig, WorkChannelDescriptor, payment_terms_hash, +}; +use hellas_rpc::{Assurance, FetchEnvironment, ProducerSigningKey, Retention}; +use hellas_work::work::{ClientEndpoint, JobProposal, ProviderEndpoint, WorkService}; +use hellas_work::work_store::{ChannelStore, Role, SetupOrigin}; + +use hellas_rpc::{ + Digest, PlatformCredential, PlatformEnrollment, ProviderEnrollmentBundle, + ProviderGenesisStatement, PublicKey, RootKind, RootProof, SignedProviderGenesis, +}; +use iroh::EndpointId; + +pub(crate) const PRICE: u64 = 10; +pub(crate) const BODY: &[u8] = br#"{"input":"sdk-private-request"}"#; +const HORIZON: u64 = 500; +const OMISSION_BOND: u64 = 601; + +pub(crate) struct PaidFixture { + pub root: tempfile::TempDir, + pub ready: ReadyChannel, + pub descriptor: WorkChannelDescriptor, + pub config: WorkConfig, +} + +pub(crate) fn signer(byte: u8) -> Secp256k1Signer { + Secp256k1Signer::from_secret_scalar([byte; 32]).unwrap() +} + +impl PaidFixture { + pub fn new() -> Self { + let root = tempfile::tempdir().unwrap(); + let network = NetworkId::new("sdk-paid-test").unwrap(); + let bond_id = EdgeId::from_bytes([11; 32]); + let payment_id = EdgeId::from_bytes([12; 32]); + let values = EdgeValues::new(1000, 200, Fees::ZERO); + let channel_policy = PaidChannelPolicyV1 { + compute_credit_limit: 40, + delivery_credit_limit: 40, + }; + let route = FetchRoutePolicy::sealed_route("openai", "responses").unwrap(); + let execution_policy = PaidWorkPolicy::Fetch { + policy: PaidFetchPolicyV1 { + allowed_environment: FetchEnvironment::OpenAiResponses.manifest_id(), + route_commitment: fetch_route_commitment(&route.canonical_body_bytes()).unwrap(), + max_request_body_bytes: 4096, + max_output_events: 64, + max_output_bytes: 16384, + max_spool_bytes: 65536, + max_encoded_result_frame: 65536, + max_encoded_prepared_input: 65536, + dispatch_margin_blocks: 4, + delivery_margin_blocks: 2, + oracle_grace_blocks: 6, + fixed_price: PRICE, + }, + route, + }; + let bond_terms = WorkStakeBondTerms { + parties: Parties::new(signer(2).party_key(), signer(1).party_key()), + timeout: BlockHeight::new(HORIZON), + timeout_outputs: List::take( + [Payout::new(signer(2).party_key(), 64); MAX_EDGE_OUTPUTS], + 1, + ), + max_job_price: 40, + }; + let terms = WorkPaymentTerms { + bond_edge: bond_id, + bond_terms: bond_terms.clone(), + private_policy_commitment: private_policy_commitment( + network, + &[9; 32], + &channel_policy, + ), + omit_response_blocks: hellas_kernel::MIN_OMIT_RESPONSE_BLOCKS, + start_validity_blocks: 8, + omission_bond: OMISSION_BOND, + }; + let descriptor = WorkChannelDescriptor::open(WorkChannelConfig { + network, + payment_edge: payment_id, + payment_terms: terms.clone(), + policy_salt: [9; 32], + channel_policy, + execution_policy: execution_policy.clone(), + expected_payment_values: values, + }) + .unwrap(); + let bond = edge( + 64, + 0, + signer(2).party_key(), + signer(1).party_key(), + Terms::work_stake_bond(bond_terms).hash(), + 0b10, + ); + let payment = edge( + 1000, + 200, + signer(1).party_key(), + signer(2).party_key(), + payment_terms_hash(terms.clone()), + 0b11000, + ); + let mut lease = vec![1, 31, 2]; + lease.extend_from_slice(&bond_id.to_bytes()); + lease.extend_from_slice(&payment_id.to_bytes()); + lease.extend_from_slice(payment_terms_hash(terms.clone()).as_bytes()); + lease.extend_from_slice(&terms.private_policy_commitment); + lease.extend_from_slice(&HORIZON.to_be_bytes()); + let lease = hellas_kernel::parse_bond_lease( + [0, 1].map(|index| { + RegistryChunk::split( + RegistryNamespace::BondLease, + RegistryRecordTag::BondLease, + &lease, + index, + ) + }), + bond_id, + ); + assert!(matches!(lease, LeaseSlots::Present(_))); + let ready = descriptor + .check_ready(&ObservedChannel { + height: 1, + bond: Some(&bond), + payment: Some(&payment), + lease, + pending: PendingSlot::Absent, + }) + .unwrap(); + let config = WorkConfig { + chain: ChainCrossCheck { network, genesis_payload_digest: [0; 32].into(), threshold_identity: hex::decode("97f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb").unwrap() }, + validators: vec!["ws://unused.invalid".to_owned()], journal_root: root.path().join("provider"), routes: Default::default(), + policy_salt: [9; 32], channel_policy, execution_policy, + poll: std::time::Duration::from_millis(10), max_observation_age: std::time::Duration::from_secs(60), + expected_payment_values: values, min_omit_response_blocks: hellas_kernel::MIN_OMIT_RESPONSE_BLOCKS, + }; + Self { + root, + ready, + descriptor, + config, + } + } + + pub fn store(&self, role: Role) -> ChannelStore { + let name = if role == Role::Client { + "client" + } else { + "provider" + }; + ChannelStore::open_metadata_only( + &self.root.path().join(name), + self.ready.channel().clone(), + work_payment_settlement(self.config.expected_payment_values, OMISSION_BOND).unwrap(), + role, + SetupOrigin { + payment_edge: self.ready.channel().payment_edge(), + height: 1, + payload: [1; 32], + parent: [0; 32], + }, + &Secp256k1Verifier::new(), + ) + .unwrap() + } + + pub fn client(&self) -> ClientEndpoint { + ClientEndpoint::new(self.ready.clone(), self.store(Role::Client), signer(1)).unwrap() + } + + pub fn provider(&self) -> WorkService { + WorkService::new( + ProviderEndpoint::new(self.ready.clone(), self.store(Role::Provider), signer(2)) + .unwrap(), + ) + } + + pub fn proposal(&self) -> JobProposal { + let events = hellas_rpc::fetch::build_input_events_with_retention( + "openai", + "responses", + BODY, + FetchEnvironment::OpenAiResponses.manifest_id(), + Assurance::ProducerSigned, + &ProducerSigningKey::from_secret_bytes([1; 32]).unwrap(), + Retention::Ephemeral, + ) + .unwrap(); + JobProposal { + prepared_input: PreparedPaidFetchInputV1::new( + &events, + &FetchEnvironment::OpenAiResponses.manifest(), + ) + .unwrap() + .into(), + deadlines: JobDeadlines { + acceptance: 50, + terminal: 100, + payment: 200, + }, + } + } +} + +fn edge(value: u64, reserve: u64, maker: Key, taker: Key, terms: TermsHash, allowed: u8) -> Edge { + // Canonical kernel edges, including their close permissions. + let mut bytes = vec![1, 5]; + bytes.extend_from_slice(&value.to_be_bytes()); + bytes.extend_from_slice(&reserve.to_be_bytes()); + bytes.extend_from_slice(&[1, 2]); + bytes.extend_from_slice(&[0; 32]); + bytes.extend_from_slice(&[1, 1]); + bytes.extend_from_slice(&HORIZON.to_be_bytes()); + bytes.extend_from_slice(&[1, 3]); + bytes.extend_from_slice(&maker.to_bytes()); + bytes.extend_from_slice(&taker.to_bytes()); + bytes.extend_from_slice(terms.as_bytes()); + bytes.push(allowed); + Edge::decode_exact(&bytes).unwrap() +} + +pub(crate) fn enrollment(peer: EndpointId) -> (ProviderEnrollmentBundle, ProducerSigningKey) { + let root = ProducerSigningKey::from_secret_bytes([1; 32]).unwrap(); + let producer = ProducerSigningKey::from_secret_bytes([2; 32]).unwrap(); + let statement = ProviderGenesisStatement { + root_kind: RootKind::Software, + root_public_key: root.public_key(), + producer_public_key: producer.public_key(), + transport_public_key: PublicKey::Ed25519(*peer.as_bytes()), + platform_credential: PlatformCredential::Absent, + installation_nonce: [3; 32], + }; + let proof = root + .sign_digest(Digest::hash(&statement.canonical_bytes())) + .unwrap(); + ( + ProviderEnrollmentBundle { + genesis: SignedProviderGenesis { + statement, + root_proof: RootProof::Software(proof), + }, + platform: PlatformEnrollment::Absent, + }, + producer, + ) +} diff --git a/crates/sdk/src/work_config.rs b/crates/sdk/src/work_config.rs new file mode 100644 index 00000000..d79880d0 --- /dev/null +++ b/crates/sdk/src/work_config.rs @@ -0,0 +1,596 @@ +//! Paid-work configuration: chain identity, routes, execution policy and funding. +//! +//! The threshold identity authenticates finalized blocks; the network and genesis +//! digest detect configuration mismatches. Unknown fields are rejected. Call +//! `validate_work_routes` before serving to check the configuration against journals. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +mod error; +pub use error::WorkConfigError; +type Result = std::result::Result; +use hellas_kernel::{ + EdgeId, EdgeValues, Fees, Key, MIN_OMIT_RESPONSE_BLOCKS, NetworkId, Secp256k1Verifier, +}; +use hellas_rpc::ContentId; +use hellas_rpc::peers::PeerId; +use hellas_rpc::protocol::Digest; +use hellas_rpc::protocol::work::{ + PaidChannelPolicyV1, PaidExecutionPolicyV1, check_execution_policy, +}; +use hellas_rpc::protocol::work_fetch::{ + FetchRoutePolicy as PaidFetchRoutePolicy, PaidFetchPolicyV1, fetch_route_commitment, +}; +use hellas_rpc::protocol::work_profile::PaidWorkPolicy; +use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; +use hellas_work::work_store::{Role, SetupStore, discover_setups}; +use serde::Deserialize; + +/// Number of distinct validator RPC URLs required by this deployment. +pub const VALIDATOR_COUNT: usize = 6; + +/// Parsed paid-work configuration. `load_work_config` checks its fields; +/// `validate_work_routes` checks agreement with provider journals at startup. +#[derive(Clone, Debug)] +pub struct WorkConfig { + /// The chain this node believes it is configured against. + pub chain: ChainCrossCheck, + /// Validator RPC URLs used for chain reads and transaction submission. + pub validators: Vec, + /// Directory holding the setup and channel journals. + pub journal_root: PathBuf, + /// Bilateral setup routes, keyed by the authenticated transport peer. + pub routes: WorkRoutes, + /// Salt of the private credit-policy commitment. + pub policy_salt: [u8; 32], + /// The credit policy this provider will work under. + pub channel_policy: PaidChannelPolicyV1, + /// The execution policy this provider will run jobs under. + pub execution_policy: PaidWorkPolicy, + /// How often the watcher asks the chain for the next block. + pub poll: Duration, + /// Maximum time without new verified finalized progress before admission stops. + /// Renewal requires an advancing finalized height, so this must also fit + /// the deployment's block interval, not only the polling interval. + pub max_observation_age: Duration, + /// The payment edge's value, reserve, and close fees as this provider + /// requires a client to fund them. + pub expected_payment_values: EdgeValues, + /// The shortest response window this provider signs terms over. + pub min_omit_response_blocks: u64, +} + +/// Maps an authenticated peer to a bond and its client settlement key. +/// Startup checks the client key against the journal before mounting the route. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct WorkRoute { + /// The transport-authenticated peer allowed to reach this bond. + pub peer: PeerId, + /// The bond whose provider setup journal this route names. + pub bond: EdgeId, + /// The settlement key the bond terms must name as taker. + pub client: Key, +} + +/// Routes indexed by authenticated peer, with duplicate peers and bonds rejected. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct WorkRoutes { + by_peer: BTreeMap, +} + +impl WorkRoutes { + /// Returns every configured route in peer order. + pub fn iter(&self) -> impl Iterator { + self.by_peer.values() + } + + /// Returns how many bilateral routes were configured. + #[must_use] + pub fn len(&self) -> usize { + self.by_peer.len() + } + + /// Returns whether no bilateral route was configured. + #[must_use] + pub fn is_empty(&self) -> bool { + self.by_peer.is_empty() + } + + fn from_files(files: Vec) -> Result { + let mut by_peer = BTreeMap::new(); + let mut bonds = BTreeSet::new(); + for file in files { + let peer = PeerId::from_bytes(parse_fixed_hex("routes[].peer", &file.peer)?); + let bond = EdgeId::from_bytes(parse_fixed_hex("routes[].bond", &file.bond)?); + let client = Key::from_bytes(parse_fixed_hex("routes[].client", &file.client)?); + let route = WorkRoute { peer, bond, client }; + if by_peer.insert(peer, route).is_some() { + return Err(WorkConfigError::DuplicatePeer(peer)); + } + if !bonds.insert(bond) { + return Err(WorkConfigError::DuplicateBond(bond)); + } + } + Ok(Self { by_peer }) + } +} + +impl WorkConfig { + /// Builds the policy used by both provisioning and channel admission. + #[must_use] + pub fn provider_policy(&self) -> ProviderChannelPolicy { + ProviderChannelPolicy { + network: self.chain.network, + policy_salt: self.policy_salt, + channel_policy: self.channel_policy, + execution_policy: self.execution_policy.clone(), + expected_payment_values: self.expected_payment_values, + min_omit_response_blocks: self.min_omit_response_blocks, + } + } +} + +/// The three fields that say which chain this is. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ChainCrossCheck { + /// The network every signature on this node's channels is bound to. + pub network: NetworkId, + /// Payload digest of the genesis block this deployment began at. + pub genesis_payload_digest: Digest, + /// The threshold identity finalized blocks are verified under. + pub threshold_identity: Vec, +} + +/// Loads configuration, checks chain identity and policy bounds, and normalizes +/// validator URLs. Route-to-journal validation is deferred until serve startup. +pub fn load_work_config(path: &Path) -> Result { + let bytes = hellas_private::read_bounded_regular_file( + path, + hellas_work::work_store::journal::MAX_RECORD_BYTES, + ) + .map_err(|source| WorkConfigError::Read { + path: path.to_path_buf(), + source, + })?; + let file: WorkConfigFile = + serde_json::from_slice(&bytes).map_err(|source| WorkConfigError::Parse { + path: path.to_path_buf(), + source, + })?; + file.into_config().map_err(|source| WorkConfigError::File { + path: path.to_path_buf(), + source: Box::new(source), + }) +} + +/// Checks each route against a provider journal under the configured root, +/// including its bond and client key. Run after provisioning and before serving. +pub fn validate_work_routes(config: &WorkConfig) -> Result<()> { + if config.routes.is_empty() { + return Ok(()); + } + let found = discover_setups(&config.journal_root, config.chain.network).map_err(|source| { + WorkConfigError::Journal { + root: config.journal_root.clone(), + source, + } + })?; + for route in config.routes.iter() { + if !found + .setups + .iter() + .any(|setup| setup.role == Role::Provider && setup.bond_edge == route.bond) + { + return Err(WorkConfigError::MissingRoute { + bond: route.bond, + root: config.journal_root.clone(), + }); + } + let store = SetupStore::open( + &config.journal_root, + config.chain.network, + route.bond, + Role::Provider, + &Secp256k1Verifier::new(), + ) + .map_err(|source| WorkConfigError::Journal { + root: config.journal_root.clone(), + source, + })?; + let Some(bundle) = store.state().bundle() else { + return Err(WorkConfigError::MissingProposal(route.bond)); + }; + let journal_client = bundle.bond_terms().parties.taker(); + if journal_client != route.client { + return Err(WorkConfigError::WrongClient { + bond: route.bond, + expected: route.client, + actual: journal_client, + }); + } + } + Ok(()) +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkConfigFile { + chain: ChainFile, + validators: Vec, + journal: JournalFile, + routes: Vec, + policies: PoliciesFile, + /// How often the watcher asks the chain for the next block. + poll_ms: u64, + #[serde(default = "default_observation_age_ms")] + max_observation_age_ms: u64, + expected_payment_values: PaymentValuesFile, + min_omit_response_blocks: u64, +} + +fn default_observation_age_ms() -> u64 { + 5_000 +} + +impl WorkConfigFile { + fn into_config(self) -> Result { + let Some(network) = NetworkId::new(self.chain.network_id.trim()) else { + return Err(WorkConfigError::Invalid { + field: "chain.network_id", + reason: "not a network id", + }); + }; + let threshold_identity = + parse_hex("chain.threshold_identity", &self.chain.threshold_identity)?; + // Parsed before the verifier is built, so the list consensus is + // handed is the normalised one this node will actually dial. + let validators = parse_validators(self.validators)?; + // The same constructor consensus verification uses. A threshold + // identity that cannot be decoded here is one no finalized block + // would ever verify under, and the node says so before it serves. + hellas_chain::ConsensusVerifier::new(&hellas_chain::light_client::ConsensusInfo { + validators: validators.clone(), + threshold_identity: threshold_identity.clone(), + network_id: network.as_str().to_owned(), + })?; + + let journal_root = self.journal.into_root()?; + let routes = WorkRoutes::from_files(self.routes)?; + let policies = self.policies.into_policies()?; + if self.poll_ms == 0 { + return Err(WorkConfigError::Invalid { + field: "poll_ms", + reason: "must be greater than zero", + }); + } + if self.max_observation_age_ms <= self.poll_ms { + return Err(WorkConfigError::Invalid { + field: "max_observation_age_ms", + reason: "must exceed poll_ms", + }); + } + // The kernel refuses a shorter window at every payment open, so a + // configuration under it would sign terms consensus then throws + // away. + if self.min_omit_response_blocks < MIN_OMIT_RESPONSE_BLOCKS { + return Err(WorkConfigError::ResponseWindow { + actual: self.min_omit_response_blocks, + minimum: MIN_OMIT_RESPONSE_BLOCKS, + }); + } + + Ok(WorkConfig { + chain: ChainCrossCheck { + network, + genesis_payload_digest: parse_digest( + "chain.genesis_payload_digest", + &self.chain.genesis_payload_digest, + )?, + threshold_identity, + }, + validators, + journal_root, + routes, + policy_salt: policies.0, + channel_policy: policies.1, + execution_policy: policies.2, + poll: Duration::from_millis(self.poll_ms), + max_observation_age: Duration::from_millis(self.max_observation_age_ms), + expected_payment_values: self.expected_payment_values.into_values(), + min_omit_response_blocks: self.min_omit_response_blocks, + }) + } +} + +/// Hex-encoded route fields, parsed before duplicate detection. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkRouteFile { + peer: String, + bond: String, + client: String, +} + +/// Normalizes validator URLs and requires distinct addresses with hosts. +fn parse_validators(entries: Vec) -> Result> { + let mut validators: Vec = Vec::with_capacity(VALIDATOR_COUNT); + for entry in entries { + let entry = entry.trim(); + if entry.is_empty() { + return Err(WorkConfigError::Invalid { + field: "validators", + reason: "entries must be non-empty", + }); + } + let url = url::Url::parse(entry).map_err(|source| WorkConfigError::ValidatorUrl { + entry: entry.to_owned(), + source, + })?; + if url.host_str().is_none() { + return Err(WorkConfigError::ValidatorHost(entry.to_owned())); + } + let normalised = url.as_str().to_string(); + if validators.contains(&normalised) { + return Err(WorkConfigError::DuplicateValidator(normalised)); + } + validators.push(normalised); + } + if validators.len() != VALIDATOR_COUNT { + return Err(WorkConfigError::ValidatorCount { + expected: VALIDATOR_COUNT, + actual: validators.len(), + }); + } + Ok(validators) +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ChainFile { + network_id: String, + genesis_payload_digest: String, + threshold_identity: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct JournalFile { + root: PathBuf, +} + +impl JournalFile { + fn into_root(self) -> Result { + if self.root.as_os_str().is_empty() { + return Err(WorkConfigError::Invalid { + field: "journal.root", + reason: "must be a path", + }); + } + Ok(self.root) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct PoliciesFile { + policy_salt: String, + channel: ChannelPolicyFile, + execution: Option, + fetch: Option, +} + +impl PoliciesFile { + fn into_policies(self) -> Result<([u8; 32], PaidChannelPolicyV1, PaidWorkPolicy)> { + let salt = parse_fixed_hex("policies.policy_salt", &self.policy_salt)?; + Ok(( + salt, + PaidChannelPolicyV1 { + compute_credit_limit: self.channel.compute_credit_limit, + delivery_credit_limit: self.channel.delivery_credit_limit, + }, + match (self.execution, self.fetch) { + (Some(execution), None) => execution.into_policy()?.into(), + (None, Some(fetch)) => fetch.into_policy()?, + _ => { + return Err(WorkConfigError::Invalid { + field: "policies", + reason: "must select exactly one of execution or fetch", + }); + } + }, + )) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ChannelPolicyFile { + compute_credit_limit: u64, + delivery_credit_limit: u64, +} + +/// Required execution-policy fields. Defaults could change the terms being signed. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ExecutionPolicyFile { + allowed_environment: String, + generation_policy_digest: String, + identity_source_digest: String, + max_prompt_tokens: u32, + max_new_tokens: u32, + max_stop_token_ids: u16, + max_spool_bytes: u64, + max_encoded_result_frame: u32, + max_encoded_quote_response: u32, + dispatch_margin_blocks: u64, + delivery_margin_blocks: u64, + oracle_grace_blocks: u64, + fixed_price: u64, +} + +impl ExecutionPolicyFile { + fn into_policy(self) -> Result { + let allowed_environment: ContentId = + self.allowed_environment + .parse() + .map_err(|source| WorkConfigError::ContentId { + field: "policies.execution.allowed_environment", + source, + })?; + let policy = PaidExecutionPolicyV1 { + allowed_environment, + generation_policy_digest: parse_digest( + "policies.execution.generation_policy_digest", + &self.generation_policy_digest, + )?, + identity_source_digest: parse_digest( + "policies.execution.identity_source_digest", + &self.identity_source_digest, + )?, + max_prompt_tokens: self.max_prompt_tokens, + max_new_tokens: self.max_new_tokens, + max_stop_token_ids: self.max_stop_token_ids, + max_spool_bytes: self.max_spool_bytes, + max_encoded_result_frame: self.max_encoded_result_frame, + max_encoded_quote_response: self.max_encoded_quote_response, + dispatch_margin_blocks: self.dispatch_margin_blocks, + delivery_margin_blocks: self.delivery_margin_blocks, + oracle_grace_blocks: self.oracle_grace_blocks, + fixed_price: self.fixed_price, + }; + // Validate with the protocol rules before any channel is proposed. + check_execution_policy(&policy)?; + Ok(policy) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct FetchPolicyFile { + allowed_environment: String, + service: Option, + method: Option, + open_fetch: Option, + max_request_body_bytes: u32, + max_output_events: u32, + max_output_bytes: u32, + max_spool_bytes: u64, + max_encoded_result_frame: u32, + max_encoded_prepared_input: u32, + dispatch_margin_blocks: u64, + delivery_margin_blocks: u64, + oracle_grace_blocks: u64, + fixed_price: u64, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct OpenFetchPolicyFile { + #[serde(default)] + require_spki_pin: bool, + #[serde(default)] + allowed_hosts: Vec, +} + +impl FetchPolicyFile { + fn into_policy(self) -> Result { + let route = match (self.service, self.method, self.open_fetch) { + (Some(service), Some(method), None) => { + PaidFetchRoutePolicy::sealed_route(service, method)? + } + (None, None, Some(open)) => { + PaidFetchRoutePolicy::open_fetch(open.require_spki_pin, open.allowed_hosts) + } + _ => { + return Err(WorkConfigError::Invalid { + field: "policies.fetch", + reason: "requires service+method or open_fetch", + }); + } + }; + let policy = PaidFetchPolicyV1 { + allowed_environment: self.allowed_environment.parse().map_err(|source| { + WorkConfigError::ContentId { + field: "policies.fetch.allowed_environment", + source, + } + })?, + route_commitment: fetch_route_commitment(&route.canonical_body_bytes())?, + max_request_body_bytes: self.max_request_body_bytes, + max_output_events: self.max_output_events, + max_output_bytes: self.max_output_bytes, + max_spool_bytes: self.max_spool_bytes, + max_encoded_result_frame: self.max_encoded_result_frame, + max_encoded_prepared_input: self.max_encoded_prepared_input, + dispatch_margin_blocks: self.dispatch_margin_blocks, + delivery_margin_blocks: self.delivery_margin_blocks, + oracle_grace_blocks: self.oracle_grace_blocks, + fixed_price: self.fixed_price, + }; + let profile = PaidWorkPolicy::Fetch { policy, route }; + profile.check()?; + Ok(profile) + } +} + +/// Required funding values and close fees. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct PaymentValuesFile { + value: u64, + reserve: u64, + close_fees: CloseFeesFile, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct CloseFeesFile { + base: u64, + slot: u64, + proof: u64, + lifetime: u64, +} + +impl PaymentValuesFile { + fn into_values(self) -> EdgeValues { + EdgeValues::new( + self.value, + self.reserve, + Fees::new( + self.close_fees.base, + self.close_fees.slot, + self.close_fees.proof, + self.close_fees.lifetime, + ), + ) + } +} + +fn parse_hex(field: &'static str, raw: &str) -> Result> { + let bytes = hex::decode(raw.trim()).map_err(|source| WorkConfigError::Hex { field, source })?; + if bytes.is_empty() { + return Err(WorkConfigError::Invalid { + field, + reason: "must not be empty", + }); + } + Ok(bytes) +} + +fn parse_fixed_hex(field: &'static str, raw: &str) -> Result<[u8; N]> { + let bytes = parse_hex(field, raw)?; + let Ok(bytes) = <[u8; N]>::try_from(bytes.as_slice()) else { + return Err(WorkConfigError::Length { + field, + expected: N, + actual: bytes.len(), + }); + }; + Ok(bytes) +} + +fn parse_digest(field: &'static str, raw: &str) -> Result { + Ok(Digest::from_bytes(parse_fixed_hex(field, raw)?)) +} diff --git a/crates/sdk/src/work_config/error.rs b/crates/sdk/src/work_config/error.rs new file mode 100644 index 00000000..616f89c7 --- /dev/null +++ b/crates/sdk/src/work_config/error.rs @@ -0,0 +1,78 @@ +use super::{ContentId, EdgeId, Key, PeerId}; +use std::path::PathBuf; + +/// Configuration errors retain the field, route or file that failed. +#[derive(Debug, thiserror::Error)] +pub enum WorkConfigError { + #[error("failed to read {}: {source}", path.display())] + Read { + path: PathBuf, + source: std::io::Error, + }, + #[error("failed to parse {}: {source}", path.display())] + Parse { + path: PathBuf, + source: serde_json::Error, + }, + #[error("invalid work config {}: {source}", path.display())] + File { path: PathBuf, source: Box }, + #[error("{field}: {reason}")] + Invalid { + field: &'static str, + reason: &'static str, + }, + #[error("routes names peer {0:#} twice; one authenticated peer has one route")] + DuplicatePeer(PeerId), + #[error("routes names bond {} twice; one provider journal has one route", hex::encode(.0.to_bytes()))] + DuplicateBond(EdgeId), + #[error("provider setup journal for bond {} is not under journal.root {}", hex::encode(bond.to_bytes()), root.display())] + MissingRoute { bond: EdgeId, root: PathBuf }, + #[error("work journal under {}: {source}", root.display())] + Journal { + root: PathBuf, + source: hellas_work::work_store::WorkStoreError, + }, + #[error("provider setup journal for bond {} holds no bond proposal", hex::encode(.0.to_bytes()))] + MissingProposal(EdgeId), + #[error("route expects client settlement key {}, but provider setup journal for bond {} names {} as its taker", hex::encode(expected.to_bytes()), hex::encode(bond.to_bytes()), hex::encode(actual.to_bytes()))] + WrongClient { + bond: EdgeId, + expected: Key, + actual: Key, + }, + #[error("chain.threshold_identity is not usable: {0}")] + Consensus(#[from] hellas_chain::ConsensusVerificationError), + #[error("min_omit_response_blocks {actual} is under the kernel's minimum {minimum}")] + ResponseWindow { actual: u64, minimum: u64 }, + #[error("validators entry {entry:?} is not a URL: {source}")] + ValidatorUrl { + entry: String, + source: url::ParseError, + }, + #[error("validators entry {0:?} names no host to dial")] + ValidatorHost(String), + #[error("validators names {0} twice")] + DuplicateValidator(String), + #[error("validators must name exactly {expected} validator URLs, found {actual}")] + ValidatorCount { expected: usize, actual: usize }, + #[error("{field} is not a ContentId: {source}")] + ContentId { + field: &'static str, + source: ::Err, + }, + #[error(transparent)] + Fetch(#[from] hellas_rpc::fetch::FetchProtocolError), + #[error("paid execution policy is not usable: {0}")] + Policy(#[from] hellas_rpc::protocol::work::PaidWorkError), + #[error("{field} is not hexadecimal: {source}")] + Hex { + field: &'static str, + source: hex::FromHexError, + }, + #[error("{field} must be {expected} bytes, found {actual}")] + Length { + field: &'static str, + expected: usize, + actual: usize, + }, +} diff --git a/crates/sdk/src/work_provision.rs b/crates/sdk/src/work_provision.rs new file mode 100644 index 00000000..b0e8db7a --- /dev/null +++ b/crates/sdk/src/work_provision.rs @@ -0,0 +1,369 @@ +//! Provider bond provisioning. +//! +//! Preview derives the bond without reading the chain or writing a journal. +//! Provisioning checks that the route, bond and staked coins are unreserved, then +//! arms a finalized history floor and journals the signed offer. It reopens the +//! journal to verify durability before returning. Unanswered offers reserve their +//! staked coins as soon as revision one is signed. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +mod error; +pub use error::ProvisionError; +type Result = std::result::Result; +use hellas_chain::client::VerifiedRemoteLightClient; +use hellas_chain::domain::MAX_EDGE_LIFETIME_BLOCKS; +use hellas_chain::{ConsensusInfo, ConsensusVerifier, WorkBlocks}; +use hellas_kernel::{ + BlockHeight, CoinId, EdgeId, Funding, Key, List, MAX_EDGE_OUTPUTS, MAX_PARTY_INPUTS, NetworkId, + Parties, Payout, Secp256k1Signer, Secp256k1Verifier, Terms, Tx, WorkStakeBondTerms, +}; +use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; +use hellas_work::work_close::FinalizedBlocks; +use hellas_work::work_handshake::{PaymentAdmission, SetupEndpoint}; +use hellas_work::work_store::{Role, SetupScan, SetupStore, discover_setups}; +use tracing::{info, warn}; + +use crate::work_config::{WorkConfig, WorkRoute}; + +/// What an operator asks for when they make one offer. +pub struct ProvisionOptions { + /// Chain, journal, route and execution-policy configuration. + pub work_config: WorkConfig, + /// The key this provider stakes and signs the bond with, read from + /// the identity the operator already has and never made here. + pub settlement_key: Secp256k1Signer, + /// The client this bond names as taker. + pub client: Key, + /// The coins this provider stakes. + pub stake_coins: Vec, + /// Height the bond expires at, which is also the admission horizon of + /// the channel it insures. + pub bond_timeout: u64, + /// What the bond's timeout returns to the staking provider. + pub timeout_payout: u64, + /// The largest job price this bond covers. + pub max_job_price: u64, +} + +/// Compute the bond before the operator adds its bilateral route. +pub fn preview_bond(options: &ProvisionOptions) -> Result { + Ok(BondCandidate::plan(options)?.bond_edge) +} + +/// Sign and journal an offer under an existing provider identity. +pub async fn provision_offer(options: ProvisionOptions) -> Result { + let candidate = BondCandidate::plan(&options)?; + let offer = Offer::plan(&options, options.work_config.provider_policy(), candidate)?; + offer.journal(finalized_floor(&options.work_config).await?) +} + +/// One offer as the disk holds it, read back after it was written. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Provisioned { + /// The bond this journal is keyed to, which is what discovery names + /// it by. + pub bond_edge: EdgeId, + /// The floor its history starts above, as retained. + pub floor: SetupScan, +} + +/// Deterministic inputs shared by bond preview and provisioning. +struct BondCandidate { + network: NetworkId, + journal_root: PathBuf, + bond_edge: EdgeId, + bond_funding: Funding, + bond_terms: WorkStakeBondTerms, + settlement_key: Secp256k1Signer, +} + +impl BondCandidate { + fn plan(options: &ProvisionOptions) -> Result { + let network = options.work_config.chain.network; + let journal_root = options.work_config.journal_root.clone(); + // Maker is the provider and taker is the client, which is what + // makes this signature the maker's: `propose_bond` refuses a bond + // whose staking party this key is not. + let provider = options.settlement_key.party_key(); + let bond_terms = WorkStakeBondTerms { + parties: Parties::new(provider, options.client), + timeout: BlockHeight::new(options.bond_timeout), + timeout_outputs: List::take( + [Payout::new(provider, options.timeout_payout); MAX_EDGE_OUTPUTS], + 1, + ), + max_job_price: options.max_job_price, + }; + let bond_funding = Funding::new( + staked(&options.stake_coins)?, + List::empty(CoinId::from_bytes([0; CoinId::LENGTH])), + ); + let bond_edge = Tx::edge_id_of(&bond_funding, &Terms::work_stake_bond(bond_terms.clone())); + Ok(Self { + network, + journal_root, + bond_edge, + bond_funding, + bond_terms, + settlement_key: options.settlement_key.clone(), + }) + } +} + +/// One offer, decided before anything is dialled or written. +struct Offer { + candidate: BondCandidate, + admission: PaymentAdmission, +} + +impl Offer { + /// Reads the operator's answers, and refuses everything refusable + /// without a chain. + fn plan( + options: &ProvisionOptions, + policy: ProviderChannelPolicy, + candidate: BondCandidate, + ) -> Result { + let route = route_for_candidate( + &options.work_config, + candidate.bond_edge, + &candidate.bond_terms, + )?; + refuse_offer_collisions(&options.work_config, route, &candidate.bond_funding)?; + Ok(Self { + candidate, + admission: PaymentAdmission::Admits(Box::new(policy)), + }) + } + + /// Journals revision 1, and returns only once a fresh open of the + /// journal replays it. + fn journal(self, floor: SetupScan) -> Result { + let timeout = self.candidate.bond_terms.timeout.get(); + if timeout <= floor.height { + return Err(ProvisionError::ExpiredBond { + timeout, + height: floor.height, + }); + } + if timeout - floor.height > MAX_EDGE_LIFETIME_BLOCKS { + return Err(ProvisionError::BondLifetime); + } + let Self { + candidate, + admission, + } = self; + let BondCandidate { + network, + journal_root, + bond_edge, + bond_funding, + bond_terms, + settlement_key, + } = candidate; + { + let store = open_provider_journal(&journal_root, network, bond_edge)?; + let mut endpoint = SetupEndpoint::new(store, settlement_key, admission); + // Preserve the journal's immutable history floor on retry. + if let Some(held) = endpoint.state().scan_armed() { + info!( + height = held.height, + "this journal already holds its history floor, and a floor does not move", + ); + } else { + endpoint.arm_scan(floor)?; + } + endpoint.propose_bond(network, bond_funding, bond_terms)?; + } + + // Reopen with the same replay and signature checks used at startup. + let reopened = open_provider_journal(&journal_root, network, bond_edge)?; + let state = reopened.state(); + let (Some(1), Some(floor)) = (state.revision(), state.scan_armed()) else { + return Err(ProvisionError::JournalReplay { + root: journal_root, + revision: state.revision(), + floor: state.scan_armed().map(|scan| scan.height), + }); + }; + Ok(Provisioned { bond_edge, floor }) + } +} + +fn open_provider_journal(root: &Path, network: NetworkId, bond_edge: EdgeId) -> Result { + SetupStore::open( + root, + network, + bond_edge, + Role::Provider, + &Secp256k1Verifier::new(), + ) + .map_err(|source| ProvisionError::Journal { + root: root.into(), + source, + }) +} + +/// Finds the candidate bond's route and checks its client before signing. +fn route_for_candidate<'config>( + config: &'config WorkConfig, + bond_edge: EdgeId, + bond_terms: &WorkStakeBondTerms, +) -> Result<&'config WorkRoute> { + let route = config + .routes + .iter() + .find(|route| route.bond == bond_edge) + .ok_or(ProvisionError::MissingRoute(bond_edge))?; + check_client(route, bond_edge, bond_terms.parties.taker())?; + Ok(route) +} + +/// Checks route, bond and coin reservations before opening the candidate journal. +/// Unidentified or unreadable existing journals prevent provisioning. +fn refuse_offer_collisions( + config: &WorkConfig, + candidate: &WorkRoute, + candidate_funding: &Funding, +) -> Result<()> { + let root = &config.journal_root; + let network = config.chain.network; + let found = discover_setups(root, network).map_err(|source| ProvisionError::Journal { + root: root.clone(), + source, + })?; + if let Some(unnamed) = found.unidentified.into_iter().next() { + return Err(ProvisionError::Unidentified { + path: unnamed.path, + source: unnamed.reason, + }); + } + + let candidate_coins = funding_coins(candidate_funding); + for held in found + .setups + .iter() + .filter(|setup| setup.role == Role::Provider) + { + if held.bond_edge == candidate.bond { + return Err(ProvisionError::BondCollision(candidate.bond)); + } + let route = config + .routes + .iter() + .find(|route| route.bond == held.bond_edge) + .ok_or(ProvisionError::MissingRoute(held.bond_edge))?; + let store = open_provider_journal(root, network, held.bond_edge)?; + let bundle = store + .state() + .bundle() + .ok_or(ProvisionError::MissingProposal(held.bond_edge))?; + check_client(route, held.bond_edge, bundle.bond_terms().parties.taker())?; + if route.peer == candidate.peer { + return Err(ProvisionError::PeerCollision { + peer: candidate.peer, + bond: held.bond_edge, + }); + } + // Revision-one funding is reserved even before an executable Open exists. + let reserved = funding_coins(bundle.bond_funding()); + if let Some(coin) = candidate_coins.intersection(&reserved).next() { + return Err(ProvisionError::ReservedCoin { + coin: *coin, + bond: held.bond_edge, + }); + } + } + Ok(()) +} + +fn check_client(route: &WorkRoute, bond: EdgeId, actual: Key) -> Result<()> { + if route.client != actual { + return Err(ProvisionError::WrongClient { + bond, + expected: route.client, + actual, + }); + } + Ok(()) +} + +/// Every input one bond funding consumes, irrespective of party position. +fn funding_coins(funding: &Funding) -> BTreeSet { + funding + .maker() + .iter() + .chain(funding.taker().iter()) + .copied() + .collect() +} + +/// Reads one finalized block from the first configured validator that +/// answers, as the floor this setup's history starts above. +async fn finalized_floor(config: &WorkConfig) -> Result { + let verifier = ConsensusVerifier::new(&ConsensusInfo { + validators: config.validators.clone(), + threshold_identity: config.chain.threshold_identity.clone(), + network_id: config.chain.network.as_str().to_owned(), + })?; + for url in &config.validators { + let client = match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { + Ok(client) => client, + Err(error) => { + warn!(validator = %url, %error, "a configured validator did not answer"); + continue; + } + }; + match floor_of(&WorkBlocks::new(client)).await { + Ok(Some(floor)) => { + info!(validator = %url, height = floor.height, "the history floor was read here"); + return Ok(floor); + } + Ok(None) => warn!(validator = %url, "a configured validator has finalized nothing"), + Err(error) => warn!(validator = %url, %error, "a configured validator did not answer"), + } + } + Err(ProvisionError::NoFinalizedBlock) +} + +/// Returns the finalized tip and its payload, or `None` before the first block. +/// Both values must come from the same block to start a contiguous history. +async fn floor_of(blocks: &B) -> Result> +where + B: FinalizedBlocks + ?Sized, +{ + let Some(height) = blocks.latest_height().await? else { + return Ok(None); + }; + let Some(block) = blocks.block_at(height).await? else { + return Ok(None); + }; + Ok(Some(SetupScan { + height: block.height, + payload: block.payload, + })) +} + +/// Reads the coins one provider stakes. +fn staked(ids: &[CoinId]) -> Result> { + if ids.is_empty() || ids.len() > MAX_PARTY_INPUTS { + return Err(ProvisionError::StakeCount(ids.len())); + } + let mut seen = BTreeSet::new(); + for coin in ids { + if !seen.insert(*coin) { + return Err(ProvisionError::DuplicateCoin(*coin)); + } + } + let slots = std::array::from_fn(|i| { + ids.get(i) + .copied() + .unwrap_or(CoinId::from_bytes([0; CoinId::LENGTH])) + }); + Ok(List::take(slots, ids.len())) +} + +#[cfg(test)] +mod tests; diff --git a/crates/sdk/src/work_provision/error.rs b/crates/sdk/src/work_provision/error.rs new file mode 100644 index 00000000..6206c866 --- /dev/null +++ b/crates/sdk/src/work_provision/error.rs @@ -0,0 +1,55 @@ +use hellas_kernel::{CoinId, EdgeId, Key}; +use hellas_rpc::peers::PeerId; +use std::path::PathBuf; + +#[derive(Debug, thiserror::Error)] +pub enum ProvisionError { + #[error("bond timeout {timeout} must be after finalized height {height}")] + ExpiredBond { timeout: u64, height: u64 }, + #[error("bond timeout exceeds the chain maximum lifetime")] + BondLifetime, + #[error("stake requires between 1 and at most 4 coins, found {0}")] + StakeCount(usize), + #[error("duplicate stake coin {}", hex::encode(.0.to_bytes()))] + DuplicateCoin(CoinId), + #[error("bond {} has no bilateral route in this work configuration", hex::encode(.0.to_bytes()))] + MissingRoute(EdgeId), + #[error("provider offer over bond {} has no retained revision", hex::encode(.0.to_bytes()))] + MissingProposal(EdgeId), + #[error("candidate bond {} collides with an existing provider offer", hex::encode(.0.to_bytes()))] + BondCollision(EdgeId), + #[error("candidate peer {peer:#} collides with bond {}", hex::encode(bond.to_bytes()))] + PeerCollision { peer: PeerId, bond: EdgeId }, + #[error("candidate stake coin {} is reserved by bond {}", hex::encode(coin.to_bytes()), hex::encode(bond.to_bytes()))] + ReservedCoin { coin: CoinId, bond: EdgeId }, + #[error("route for bond {} expects client {}, but the bond names {}", hex::encode(bond.to_bytes()), hex::encode(expected.to_bytes()), hex::encode(actual.to_bytes()))] + WrongClient { + bond: EdgeId, + expected: Key, + actual: Key, + }, + #[error("journal under {} replays as revision {revision:?} over floor {floor:?}, not the armed proposal", root.display())] + JournalReplay { + root: PathBuf, + revision: Option, + floor: Option, + }, + #[error("work journal under {}: {source}", root.display())] + Journal { + root: PathBuf, + source: hellas_work::work_store::WorkStoreError, + }, + #[error("setup journal {} cannot be identified: {source}", path.display())] + Unidentified { + path: PathBuf, + source: hellas_work::work_store::SetupDiscoveryError, + }, + #[error("no configured validator answered with a finalized block to floor this offer at")] + NoFinalizedBlock, + #[error(transparent)] + Setup(#[from] hellas_work::work_handshake::SetupExchangeError), + #[error(transparent)] + Consensus(#[from] hellas_chain::ConsensusVerificationError), + #[error(transparent)] + BlockSource(#[from] hellas_work::work_close::BlockSourceError), +} diff --git a/crates/cli/src/commands/serve/provision/tests.rs b/crates/sdk/src/work_provision/tests.rs similarity index 96% rename from crates/cli/src/commands/serve/provision/tests.rs rename to crates/sdk/src/work_provision/tests.rs index 78818ae7..8c6fa8b3 100644 --- a/crates/cli/src/commands/serve/provision/tests.rs +++ b/crates/sdk/src/work_provision/tests.rs @@ -6,8 +6,8 @@ use hellas_rpc::protocol::work::{PaidChannelPolicyV1, PaidExecutionPolicyV1}; use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; use hellas_work::work_close::{BlockSourceError, FinalizedWork}; -use super::super::work_config::load_work_config; use super::*; +use crate::work_config::load_work_config; fn network() -> NetworkId { let Some(network) = NetworkId::new("hellas-devnet") else { @@ -66,7 +66,8 @@ fn policy() -> ProviderChannelPolicy { delivery_margin_blocks: 2, oracle_grace_blocks: 6, fixed_price: 10, - }, + } + .into(), expected_payment_values: EdgeValues::new(1_000, 200, Fees::new(0, 0, 0, 0)), min_omit_response_blocks: MIN_OMIT_RESPONSE_BLOCKS, } @@ -90,7 +91,10 @@ fn route(peer: u8, bond: EdgeId, client: Key) -> serde_json::Value { /// Loads routes through the production parser, so their duplicate-peer /// and duplicate-bond invariants are facts these provisioning tests use, /// not a test-only constructor that can make impossible route tables. -fn routed_work_config(root: &Path, routes: Vec) -> CliResult { +fn routed_work_config( + root: &Path, + routes: Vec, +) -> std::result::Result> { let validators: Vec = (1..=6) .map(|index| format!("http://127.0.0.1:900{index}")) .collect(); @@ -134,9 +138,8 @@ fn routed_work_config(root: &Path, routes: Vec) -> CliResult< "min_omit_response_blocks": MIN_OMIT_RESPONSE_BLOCKS, }); let path = root.join("work-config.json"); - fs::write(&path, file.to_string()) - .with_context(|| format!("the route fixture writes {}", path.display()))?; - load_work_config(&path) + fs::write(&path, file.to_string())?; + Ok(load_work_config(&path)?) } fn options(root: &Path, max_job_price: u64) -> ProvisionOptions { @@ -156,15 +159,14 @@ fn options_for( ProvisionOptions { work_config, settlement_key: provider(), - client: hex::encode(client.to_bytes()), + client, stake_coins: stake_coins .iter() - .map(|coin| hex::encode([*coin; 32])) + .map(|coin| CoinId::from_bytes([*coin; 32])) .collect(), bond_timeout: 500, timeout_payout: 64, max_job_price, - print_bond_only: false, } } @@ -173,14 +175,14 @@ fn provision( root: &Path, policy: ProviderChannelPolicy, max_job_price: u64, -) -> CliResult { +) -> Result { provision_options(&options(root, max_job_price), policy) } fn provision_options( options: &ProvisionOptions, policy: ProviderChannelPolicy, -) -> CliResult { +) -> Result { let candidate = BondCandidate::plan(options)?; Offer::plan(options, policy, candidate)?.journal(floor()) } @@ -235,19 +237,17 @@ fn preview_and_real_offer_use_the_identical_bond_candidate() { let offer = Offer::plan(&options, policy(), candidate) .unwrap_or_else(|error| panic!("the routed offer plans: {error:#}")); assert_eq!(expected, expected_bond(40)); - assert_eq!(offer.bond_edge, expected); + assert_eq!(offer.candidate.bond_edge, expected); } -#[tokio::test] -async fn preview_needs_neither_a_route_nor_a_chain_nor_a_journal() { +#[test] +fn preview_needs_neither_a_route_nor_a_chain_nor_a_journal() { let root = tempfile::tempdir().unwrap(); let config = routed_work_config(root.path(), Vec::new()) .unwrap_or_else(|error| panic!("a route-free config loads: {error:#}")); - let mut options = options_for(config, client().party_key(), &[0xa1], 40); - options.print_bond_only = true; + let options = options_for(config, client().party_key(), &[0xa1], 40); - run_provision(options) - .await + preview_bond(&options) .unwrap_or_else(|error| panic!("the isolated preview succeeds: {error:#}")); assert_eq!( provider_setups(root.path()), @@ -584,7 +584,7 @@ fn a_stake_wider_than_an_open_is_refused_rather_than_truncated() { let dir = tempfile::tempdir().unwrap(); let mut options = options(dir.path(), 40); options.stake_coins = (0..=u8::try_from(MAX_PARTY_INPUTS).unwrap()) - .map(|byte| hex::encode([byte; 32])) + .map(|byte| CoinId::from_bytes([byte; 32])) .collect(); let Err(error) = BondCandidate::plan(&options) else { diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 965d0007..a8c72131 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -49,7 +49,6 @@ pub mod state; pub mod xorb; use std::collections::HashMap; -use std::fs::OpenOptions; use std::io; use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock}; @@ -60,162 +59,7 @@ use hellas_xet::{Chunk, XetFileHasher, XetHash}; /// not the bottleneck, small enough to be irrelevant beside a model. const STREAM_BUFFER: usize = 1024 * 1024; -/// Opens a local regular file without first opening a FIFO or device for I/O. -/// -/// Explicit content paths may be symlinks: the target descriptor, rather than -/// the symlink name, is the authority. Linux first acquires an `O_PATH` -/// descriptor, which does not open the underlying object, checks its type, and -/// then reopens that exact inode through `/proc/self/fd`. Consequently a path -/// replacement between the type check and the readable open cannot substitute -/// a FIFO or device. The returned descriptor is read-only, seekable, and -/// close-on-exec. -/// -/// Other platforms do not expose an equivalent through `std`: they preflight -/// the followed path, use nonblocking open where Unix provides it, and verify -/// the resulting descriptor. That rejects stable special files but cannot -/// close an adversarial replacement race as Linux does. -pub fn open_regular_file(path: &Path) -> io::Result { - open_regular_file_impl(path) -} - -fn not_a_regular_file(path: &Path) -> io::Error { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("{} is not a regular file", path.display()), - ) -} - -#[cfg(target_os = "linux")] -fn open_regular_file_impl(path: &Path) -> io::Result { - let path_handle = open_path_handle(path)?; - reopen_regular_path_handle(path, &path_handle) -} - -/// Acquires an inode reference without invoking the target's file operations. -#[cfg(target_os = "linux")] -fn open_path_handle(path: &Path) -> io::Result { - use std::ffi::CString; - use std::os::fd::FromRawFd as _; - use std::os::unix::ffi::OsStrExt as _; - - let path = CString::new(path.as_os_str().as_bytes()) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; - // OpenOptions masks custom flags with !O_ACCMODE. On musl that mask - // includes O_PATH, turning a descriptor-only open into a blocking read. - loop { - // SAFETY: path is NUL-terminated and live for the call. These flags - // do not create a file, so open needs no variadic mode argument. - let fd = unsafe { libc::open(path.as_ptr(), libc::O_PATH | libc::O_CLOEXEC) }; - if fd >= 0 { - // SAFETY: open returned a new descriptor, owned only here. - return Ok(unsafe { std::fs::File::from_raw_fd(fd) }); - } - let error = io::Error::last_os_error(); - if error.kind() != io::ErrorKind::Interrupted { - return Err(error); - } - } -} - -/// Converts an `O_PATH` reference to a readable descriptor for the same inode. -#[cfg(target_os = "linux")] -fn reopen_regular_path_handle( - path: &Path, - path_handle: &std::fs::File, -) -> io::Result { - use std::os::fd::AsRawFd as _; - use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; - - let expected = path_handle.metadata()?; - if !expected.file_type().is_file() { - return Err(not_a_regular_file(path)); - } - - let descriptor_path = Path::new("/proc/self/fd").join(path_handle.as_raw_fd().to_string()); - let mut options = OpenOptions::new(); - options.read(true).custom_flags(libc::O_CLOEXEC); - let file = options.open(&descriptor_path).map_err(|source| { - // Once the held descriptor has been fstat-ed successfully, ENOENT can - // only mean procfs cannot provide the safe reopen. Do not report the - // caller's existing content as a cache miss. - let kind = if source.kind() == io::ErrorKind::NotFound { - io::ErrorKind::Unsupported - } else { - source.kind() - }; - io::Error::new( - kind, - format!( - "cannot safely reopen {} through {}: {source}", - path.display(), - descriptor_path.display() - ), - ) - })?; - let actual = file.metadata()?; - if !actual.file_type().is_file() - || (expected.dev(), expected.ino()) != (actual.dev(), actual.ino()) - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "{} did not reopen the inode held by its path descriptor", - path.display() - ), - )); - } - Ok(file) -} - -/// Best available fallback where `O_PATH` plus descriptor reopen is absent. -#[cfg(all(unix, not(target_os = "linux")))] -fn open_regular_file_impl(path: &Path) -> io::Result { - use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; - - let expected = std::fs::metadata(path)?; - if !expected.file_type().is_file() { - return Err(not_a_regular_file(path)); - } - - let mut options = OpenOptions::new(); - options - .read(true) - .custom_flags(libc::O_NONBLOCK | libc::O_CLOEXEC); - let file = options.open(path)?; - let actual = file.metadata()?; - if !actual.file_type().is_file() - || (expected.dev(), expected.ino()) != (actual.dev(), actual.ino()) - { - return Err(not_a_regular_file(path)); - } - Ok(file) -} - -/// Best available fallback where Unix descriptors are absent. Windows adds a -/// read-only share mode, so no writer can open the file while it is being -/// hashed -- the guarantee the Unix paths approximate with identity checks -/// (fastresume's Windows identity is deliberately weak; see there). -#[cfg(not(unix))] -fn open_regular_file_impl(path: &Path) -> io::Result { - let expected = std::fs::metadata(path)?; - if !expected.file_type().is_file() { - return Err(not_a_regular_file(path)); - } - let mut options = OpenOptions::new(); - options.read(true); - #[cfg(windows)] - { - use std::os::windows::fs::OpenOptionsExt as _; - // FILE_SHARE_READ: other readers may share the file; writers and - // deleters are refused until it is closed. - options.share_mode(0x0000_0001); - } - let file = options.open(path)?; - if !file.metadata()?.file_type().is_file() { - return Err(not_a_regular_file(path)); - } - Ok(file) -} +pub use hellas_private::open_regular_file; /// What indexing one file learned about it. #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/crates/store/src/tests.rs b/crates/store/src/tests.rs index ea9517f6..a54d7298 100644 --- a/crates/store/src/tests.rs +++ b/crates/store/src/tests.rs @@ -1,4 +1,5 @@ use super::*; +use std::fs::OpenOptions; fn scratch(name: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!( @@ -16,93 +17,6 @@ fn opened(path: &Path) -> (std::fs::File, fastresume::FileIdentity) { (file, identity) } -/// The Linux primitive is the contract, not merely an implementation -/// detail: a harmless path descriptor is acquired before type inspection, -/// and the readable descriptor remains bound to that inode even if the -/// name is replaced with a device. -#[cfg(target_os = "linux")] -#[test] -fn regular_open_is_safe_readable_seekable_and_close_on_exec() { - use std::io::{Read as _, Seek as _, SeekFrom}; - use std::os::fd::AsRawFd as _; - - let dir = scratch("regular-open"); - let target = dir.join("blob"); - let content = b"ordinary bytes"; - std::fs::write(&target, content).expect("write target"); - let link = dir.join("snapshot"); - std::os::unix::fs::symlink(&target, &link).expect("symlink"); - - let mut file = open_regular_file(&link).expect("open regular symlink"); - let mut read = Vec::new(); - file.read_to_end(&mut read).expect("read"); - assert_eq!(read, content); - file.seek(SeekFrom::Start(0)).expect("seek"); - read.clear(); - file.read_to_end(&mut read).expect("read again"); - assert_eq!(read, content); - // SAFETY: F_GETFD only observes the live descriptor owned by `file`. - let descriptor_flags = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFD) }; - assert!( - descriptor_flags >= 0, - "F_GETFD: {}", - io::Error::last_os_error() - ); - assert_ne!(descriptor_flags & libc::FD_CLOEXEC, 0); - - // Hold the ordinary inode without opening it for I/O, replace its - // public name with a device, then finish the reopen. The bytes must - // still come from the held inode. - let path_handle = open_path_handle(&link).expect("path handle"); - // O_PATH must survive libc-specific access-mode flags (notably musl). - // SAFETY: F_GETFL only observes the live descriptor owned by path_handle. - let path_flags = unsafe { libc::fcntl(path_handle.as_raw_fd(), libc::F_GETFL) }; - assert!(path_flags >= 0, "F_GETFL: {}", io::Error::last_os_error()); - assert_ne!(path_flags & libc::O_PATH, 0); - let replacement = dir.join("replacement"); - std::os::unix::fs::symlink("/dev/null", &replacement).expect("device symlink"); - std::fs::rename(&replacement, &link).expect("replace link"); - let mut held = reopen_regular_path_handle(&link, &path_handle).expect("reopen held inode"); - read.clear(); - held.read_to_end(&mut read).expect("read held inode"); - assert_eq!(read, content); - - let fifo = dir.join("fifo"); - let status = std::process::Command::new("mkfifo") - .arg(&fifo) - .status() - .expect("mkfifo"); - assert!(status.success(), "the fixture needs a fifo"); - let (sender, receiver) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let _ = sender.send(open_regular_file(&fifo)); - }); - let fifo_error = receiver - .recv_timeout(std::time::Duration::from_secs(2)) - .expect("opening the FIFO blocked before its type check") - .expect_err("a FIFO is not regular content"); - assert_eq!(fifo_error.kind(), io::ErrorKind::InvalidInput); - - if Path::new("/dev/null").exists() { - let device_error = open_regular_file(Path::new("/dev/null")) - .expect_err("a character device is not regular content"); - assert_eq!(device_error.kind(), io::ErrorKind::InvalidInput); - } - - use std::os::unix::ffi::{OsStrExt as _, OsStringExt as _}; - let mut nul_path = target.as_os_str().as_bytes().to_vec(); - nul_path.extend_from_slice(b"\0ignored-suffix"); - let nul_path = PathBuf::from(std::ffi::OsString::from_vec(nul_path)); - assert_eq!( - open_regular_file(&nul_path) - .expect_err("NUL must not truncate the path") - .kind(), - io::ErrorKind::InvalidInput, - ); - - let _ = std::fs::remove_dir_all(&dir); -} - /// Hashing is bounded by the descriptor length captured before the read. /// Mutations are injected between real file reads, so growth is found by /// the one-byte sentinel and truncation by an early EOF. diff --git a/crates/wire/Cargo.toml b/crates/wire/Cargo.toml index d2c91349..94040572 100644 --- a/crates/wire/Cargo.toml +++ b/crates/wire/Cargo.toml @@ -115,3 +115,4 @@ windows-sys = { version = "0.61", optional = true, features = [ [dev-dependencies] iroh = { workspace = true, features = ["tls-ring"] } tokio = { workspace = true, features = ["full"] } +tracing-subscriber = { workspace = true } diff --git a/crates/wire/src/iroh/pool.rs b/crates/wire/src/iroh/pool.rs index c22a5134..4de1db45 100644 --- a/crates/wire/src/iroh/pool.rs +++ b/crates/wire/src/iroh/pool.rs @@ -39,6 +39,7 @@ use ::iroh::{Endpoint, EndpointId}; // Xtensa (esp32-s3) lacks native 64-bit atomics; portable-atomic provides a // mutex-fallback so embedded targets still compile. use portable_atomic::{AtomicU64, Ordering}; +use tracing::Instrument; use web_time::Instant; use crate::iroh::transport::IrohTransport; @@ -235,10 +236,15 @@ impl Pool { // loser's connection is dropped immediately when we re-insert. // Pass the full EndpointAddr to iroh::Endpoint::connect so any // CLI-supplied direct addresses become dial hints. + // The QUIC driver retains the current span for the connection's entire + // lifetime. Give it a root span so it cannot keep the dialing request + // and its ancestors open while subsequent requests reuse the connection. + let span = tracing::info_span!(target: "hellas_request", parent: None, "quic.connection"); let connect = self .inner .endpoint - .connect(target, self.inner.alpn.as_slice()); + .connect(target, self.inner.alpn.as_slice()) + .instrument(span); let conn = n0_future::time::timeout(self.inner.options.connect_timeout, connect) .await .map_err(|_| PoolError::Timeout)? @@ -362,3 +368,59 @@ impl Drop for Pool { } } } + +#[cfg(test)] +mod tests { + use super::*; + use tracing::instrument::{Instrument, WithSubscriber}; + use tracing_subscriber::{Layer, layer::Context, prelude::*, registry::LookupSpan}; + + #[derive(Clone, Default)] + struct ClosedSpans(Arc>>); + + impl LookupSpan<'a>> Layer for ClosedSpans { + fn on_close(&self, id: tracing::span::Id, ctx: Context<'_, S>) { + self.0.lock().unwrap().push(ctx.span(&id).unwrap().name()); + } + } + + #[tokio::test] + async fn pooled_connection_does_not_hold_request_span_open() { + let closed = ClosedSpans::default(); + let dispatch = tracing::Dispatch::new( + tracing_subscriber::registry() + .with(tracing_subscriber::filter::LevelFilter::INFO) + .with(closed.clone()), + ); + async { + let alpn = b"/hellas-test/pool/1"; + let server = Endpoint::builder(::iroh::endpoint::presets::Minimal) + .alpns(vec![alpn.to_vec()]) + .bind() + .await + .unwrap(); + let client = Endpoint::builder(::iroh::endpoint::presets::Minimal) + .bind() + .await + .unwrap(); + let pool = Pool::new(client.clone(), alpn, PoolOptions::default()); + let request = tracing::info_span!("request"); + let (connected, accepted) = + tokio::join!(pool.connection(server.addr()).instrument(request), async { + server.accept().await.unwrap().await.unwrap() + },); + let connected = connected.unwrap(); + assert!(connected.close_reason().is_none()); + assert!(accepted.close_reason().is_none()); + assert!( + closed.0.lock().unwrap().contains(&"request"), + "request span must close while the pooled connection is still live" + ); + pool.shutdown(); + client.close().await; + server.close().await; + } + .with_subscriber(dispatch) + .await; + } +} diff --git a/crates/wire/src/iroh/stream.rs b/crates/wire/src/iroh/stream.rs index 866f86c4..c8b955f1 100644 --- a/crates/wire/src/iroh/stream.rs +++ b/crates/wire/src/iroh/stream.rs @@ -208,6 +208,12 @@ impl Drop for IrohSendHalf { impl crate::transport::SendHalf for IrohSendHalf { type Error = std::io::Error; + async fn stopped(&mut self) { + if let Some(send) = self.send.as_mut() { + let _ = send.stopped().await; + } + } + async fn send_body(&mut self, payload: Bytes) -> Result<(), Self::Error> { self.write_framed(Frame::Body(payload)).await } diff --git a/crates/wire/src/mux/state.rs b/crates/wire/src/mux/state.rs index 83f3c6e0..85cc86b9 100644 --- a/crates/wire/src/mux/state.rs +++ b/crates/wire/src/mux/state.rs @@ -514,8 +514,12 @@ impl Multiplexer { if slot.generation != keyed.key.generation { return Ok(events); // stale-generation drop } - if slot.peer_terminal { - return Ok(events); // peer already terminal; drop late frames + // End closes the peer's send half. It can still consume our response + // (returning credit) or cancel it until our own send half ends. + if slot.peer_terminal + && (slot.local_terminal || !matches!(keyed.frame, Frame::Credit(_) | Frame::Reset(_))) + { + return Ok(events); } match keyed.frame { diff --git a/crates/wire/src/mux/state/tests.rs b/crates/wire/src/mux/state/tests.rs index deb433a7..d97c96e1 100644 --- a/crates/wire/src/mux/state/tests.rs +++ b/crates/wire/src/mux/state/tests.rs @@ -168,6 +168,43 @@ fn credit_returns_only_after_application_consumption() { )); } +#[test] +fn request_end_still_allows_response_credit_and_cancellation() { + let cfg = MuxConfig { stream_window: 8 }; + let mut client: Multiplexer<8, _> = Multiplexer::new(Role::Client, DefaultClock, cfg); + let mut server: Multiplexer<8, _> = Multiplexer::new(Role::Server, DefaultClock, cfg); + let slot = client.open(7, Metadata::new()).unwrap(); + drain(&mut client, &mut server); + client.close_send(slot, None).unwrap(); + drain(&mut client, &mut server); + queue_body(&mut server, slot, Bytes::from_static(b"response")); + drain(&mut server, &mut client); + assert!(matches!( + server.try_send_body(slot, Bytes::from_static(b"next")), + Ok(SendBodyOutcome::Blocked(_)) + )); + client.consume(slot, 8).unwrap(); + assert!(matches!( + drain(&mut client, &mut server).as_slice(), + [Event::PeerCredit { .. }] + )); + queue_body(&mut server, slot, Bytes::from_static(b"next")); + drain(&mut server, &mut client); + client.reset(slot, WireCode::Cancelled); + assert!(matches!( + drain(&mut client, &mut server).as_slice(), + [Event::ResetStream { + code: WireCode::Cancelled, + .. + }] + )); + assert!( + server + .try_send_body(slot, Bytes::from_static(b"late")) + .is_err() + ); +} + #[test] fn stale_gen_discarded() { let (mut client, mut server) = pair::<32>(); diff --git a/crates/wire/src/transport.rs b/crates/wire/src/transport.rs index 6ffbef1e..5bb60394 100644 --- a/crates/wire/src/transport.rs +++ b/crates/wire/src/transport.rs @@ -163,6 +163,12 @@ pub trait SendHalf: Send { /// Cancel both directions of the underlying stream. Idempotent. fn reset(&mut self, code: WireCode); + + /// Wait until the peer stops receiving. Carriers without a cancellation + /// notification retain the default pending future and detect write errors. + fn stopped(&mut self) -> impl Future + Send { + std::future::pending() + } } /// Recv half. Yields body chunks; trailer is available after the diff --git a/crates/work/README.md b/crates/work/README.md new file mode 100644 index 00000000..e697e246 --- /dev/null +++ b/crates/work/README.md @@ -0,0 +1,164 @@ +# Paid Fetch and provider payload retention + +Paid Fetch shares the existing channel authorization, execution gate, delivery, +payment certificate, and settlement flow with Evaluate. A Fetch channel selects +`PaidWorkPolicy::Fetch`; the policy commits to its route and manifest as well +as its fixed price and input/output limits. Evaluate's canonical records and +version-1 close descriptors remain unchanged. Fetch close descriptors use +version 2. + +## Provider storage + +The Fetch profile requires `Retention::Ephemeral` and a metadata-only provider +journal. Channel mounting selects that journal automatically; binding a Fetch +provider to an ordinary payload journal fails. Live requests and responses are +fully verified before their accounting records are committed. + +The journal persists authorizations, signatures, request/result digests, running +and delivery markers, payment certificates, ledger state, and close state. +It omits prepared requests and output transcripts from both append records and +rotation checkpoints. Those bodies exist in memory while the job is active. +The paid executor invokes the configured Fetch route directly, without passing +through Courtesy's transcript store or replay cache. Upstream and projection +faults are reduced to fixed error messages before they reach the paid-work +driver's logs. + +Fetch channels open metadata-only journals on both endpoints: a client +restart cannot reconstruct a lost request or replay a received response, +though a delivered result's retained evidence still settles payment. +This policy covers application-managed storage; it does not establish the +upstream API's retention policy or replace host memory/swap/crash-dump +controls. + +## Restart behavior + +| Provider state before restart | Recovery | +| --- | --- | +| Accepted, not dispatched | Input is gone; do not dispatch automatically | +| Running | Outcome is indeterminate; do not invoke again | +| Result recorded | Signed result metadata survives; response body cannot be delivered or replayed | +| Already delivered | Client can still submit its signed payment; duplicate submissions are idempotent | +| Paid | Payment and settlement recovery continue from durable metadata | + +Existing deadline and close rules still apply to jobs whose bodies were lost. +Provider failure is not turned into a paid result. The client must not assume +that retrying delivery after a provider restart can recover its response. + +The journal header distinguishes payload journals from metadata-only journals. +Opening one as the other fails. This change does not migrate or erase old files. + +## CLI + +Build the CLI with `node,llm`. Use `paid-work prepare-fetch` to create a +client-owned signed request: + +```sh +hellas-cli paid-work prepare-fetch \ + --service openai --method responses \ + --execution-environment openai-responses \ + --payload-file request.json --out paid-fetch.bin +``` + +Use the same local identity when preparing the input and running +`paid-work run --prepared-input paid-fetch.bin`. The existing run command +handles setup, acceptance, delivery, authenticated payment, and optional +settlement; it still needs its existing provider/funding/config arguments. + +In the work config, replace `policies.execution` with `policies.fetch` (exactly +one is allowed): + +```json +{ + "allowed_environment": "<64 hex digits of the Fetch manifest ID>", + "service": "openai", + "method": "responses", + "max_request_body_bytes": 4096, + "max_output_events": 64, + "max_output_bytes": 16384, + "max_spool_bytes": 65536, + "max_encoded_result_frame": 65536, + "max_encoded_prepared_input": 65536, + "dispatch_margin_blocks": 4, + "delivery_margin_blocks": 2, + "oracle_grace_blocks": 6, + "fixed_price": 10 +} +``` + +The route must also exist in the provider's Fetch route configuration. +Route-wide capabilities and shared Fetch concurrency limits remain enforced. +Courtesy's caller quota policy is not the paid admission policy; a paid channel +authorizes its own work. + +## Generic HTTPS and App Attest + +`FetchEnvironment::Http` interprets the caller-signed HTTPS request described +in [the HTTPS guide](../providers/HTTPS.md). The body binds the full URL, +method, headers, binary request body, TLS roots, additional SPKI pins, account +alias, and response-size ceiling. The result signs HTTP status, headers and +exact response bytes. No upstream vendor needs to be compiled into Gate. + +To sell this interpreter, use its manifest ID as `allowed_environment` and +replace `service`/`method` in `policies.fetch` with: + +```json +"open_fetch": { + "require_spki_pin": false, + "allowed_hosts": [] +} +``` + +An empty list permits any public host, subject to the operator's independently +configured egress restrictions. A nonempty list matches exact host names. +The host and required-pin conditions are checked before paid acceptance. +The HTTPS driver enforces certificate validation and address restrictions at +connection time. A registered route such as `http/request` must implement the +HTTP manifest. The paid policy's byte/frame limits must cover the encoded HTTP +response as well as its envelopes (base64 expands binary bodies). + +Use `paid-work prepare-fetch --execution-environment http` for this request. +Add `--assurance apple-app-attest` when targeting Gate. On `paid-work run`, +provide `--provider-genesis`, `--apple-app-id` and `--apple-cd-hashes`. Each paid +Work and WorkSetup connection verifies a fresh Open proof before sending its +request. The proof binds the TLS exporter, nonce, service ALPN, enrollment and +producer key; the producer must also be the payment channel's provider. +Result verification takes the assurance from the client's signed input and +rejects a weaker output scheme. + +## Application interface + +`hellas-sdk` is the application boundary for both CLI and signed hosts: + +- `paid_client::PaidWorkSession` owns one funded channel. It authenticates the + provider, resumes journaled jobs, verifies results, pays, and closes. The CLI + and paid gateway use this same session. `run_paid_work` wraps one complete job. +- `PreparedPaidWorkInput` and `PaidWorkPolicy` select Evaluate or Fetch. Each + profile validates its own canonical input, bounds and terminal result; the + payment lifecycle does not interpret HTTP or model output. +- `FetchProviderOptions` and `start_fetch_provider` take an operator's route + registry, enrollment and root prover. The host owns its identity and credentials; + the SDK supplies the shared provider routing and finalized-chain clock. +- `HttpFetchRequest` and `HttpFetchResponse` express HTTPS semantics. The Fetch + backend executes an admitted request; canonical transcript verification checks + its result under the assurance requested by the caller. + +A session serializes its jobs. After cancellation, recover the journal before +admitting another job. Evaluate supports authenticated incremental token delivery; +Fetch currently returns a complete bounded response. Fetch journals are +metadata-only on both endpoints; Evaluate client journals may retain payloads. + +Apple App Attest requires a provisioned, signed macOS host. `ProducerSigned` +verifies the key and transcript but does not attest the binary. These are the +implemented assurance choices, checked explicitly before request disclosure and +again during result verification. + +Adding Open changed the Work and WorkSetup service schema IDs. Existing paid +method IDs and canonical payment encodings are unchanged; use matching updated +client and provider builds. + +Tests inspect provider journal files after each commit and rotation, reopen +them after simulated process loss, verify payment recovery without bodies, +and cover both signed result schemes. Local TLS tests exercise roots, pins, +wrong hostnames, redirects, address restrictions and response bounds. Portable +App Attest tests cover connection/service binding and replay counters. They do +not contact a paid upstream or enroll a real Apple device. diff --git a/crates/work/src/work.rs b/crates/work/src/work.rs index 2cbd812e..1e0d009f 100644 --- a/crates/work/src/work.rs +++ b/crates/work/src/work.rs @@ -5,7 +5,7 @@ //! //! Canonical private records, and nothing else. `AcceptWorkRequest` //! carries the exact bytes of a [`PaidJobAuthorizationV1`] and a -//! [`PreparedPaidInputV1`], not a protobuf transcription of their +//! [`hellas_rpc::protocol::artifacts::PreparedPaidInputV1`], not a protobuf transcription of their //! fields, because the two signatures on this exchange are over digests //! of those bytes. A protobuf spelling beside them would be a second //! definition of what both parties signed, agreeing with the first until @@ -117,6 +117,13 @@ //! What neither endpoint does here is wait. `advance_close` is one //! step, and the caller that owns a clock is the one that repeats it. +mod observation; +use observation::Observation; +pub use observation::ObservationTime; + +mod client_channel; +pub use client_channel::{ClientChannel, ClientDriver, ClientObserver, ClientService}; + mod stream; pub use stream::{PaidProgress, PaidResultStream, fetch_result_stream}; @@ -151,13 +158,14 @@ use hellas_rpc::pb::work::{ deliver_result_response::Outcome as DeliverOutcome, }; use hellas_rpc::protocol::Digest; -use hellas_rpc::protocol::artifacts::{PreparedPaidInputParts, PreparedPaidInputV1}; +use hellas_rpc::protocol::artifacts::PreparedPaidInputParts; use hellas_rpc::protocol::work::{ JobDeadlines, PaidJobAuthorizationV1, PaidJobResultV1, PaidWorkError, PaymentBindingV1, - PrivateRecord as _, check_authorization, check_prepared_input, delivery_request_digest, - encode_transcript, next_payment, payment_binding_digest, propose_authorization, result_digest, - signing_hash, terminal_result, work_id, + PrivateRecord as _, delivery_request_digest, encode_transcript, next_payment, + payment_binding_digest, result_digest, signing_hash, work_id, }; +use hellas_rpc::protocol::work_fetch::{PaidFetchPolicyV1, PreparedPaidFetchInputParts}; +use hellas_rpc::protocol::work_profile::{PaidWorkPolicy, PreparedPaidWorkInput}; use hellas_rpc::protocol::work_setup::{ObservedChannel, ReadyChannel, WorkSetupError}; use hellas_rpc::services::work::{WorkClientImpl, WorkHandler}; use hellas_rpc::{EvaluateRequest, OutputEventEnvelope, SubmitTxOutcome}; @@ -378,6 +386,9 @@ const fn channel_refusal(error: &ChannelStateError) -> WorkRefusal { /// new work. #[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] pub enum EndpointError { + /// The background observer has not confirmed recent finalized progress. + #[error("channel observation is stale; awaiting finalized progress")] + ObservationStale, /// The journal is another channel's. #[error("the store's channel is not the one this readiness decided")] WrongChannel, @@ -385,6 +396,9 @@ pub enum EndpointError { /// found, so the two bound payments differently. #[error("the store's settlement is not the one this readiness read")] WrongSettlement, + /// Paid Fetch providers must never persist customer payloads. + #[error("paid fetch requires a provider journal containing metadata only")] + PayloadRetention, /// The journal is the other role's. #[error("the store is the {found} journal, and this is the {expected} endpoint")] WrongRole { @@ -433,6 +447,12 @@ fn bind( signer: &Secp256k1Signer, role: Role, ) -> Result<(), EndpointError> { + if role == Role::Provider + && matches!(ready.execution_policy(), PaidWorkPolicy::Fetch { .. }) + && !store.metadata_only() + { + return Err(EndpointError::PayloadRetention); + } let state = store.state(); if state.channel() != ready.channel() { return Err(EndpointError::WrongChannel); @@ -509,7 +529,7 @@ pub struct CloseEndpoint { /// /// It owns one channel's journal and answers proposals on that channel /// alone. A proposal naming another channel is refused by -/// [`check_authorization`], never routed: routing many channels through +/// [`hellas_rpc::protocol::work::check_authorization`], never routed: routing many channels through /// one endpoint is a later concern, and pretending to do it here would /// mean an authorization whose `channel_id` selects its own validator. /// @@ -521,6 +541,7 @@ pub struct CloseEndpoint { pub struct ProviderEndpoint { close: CloseEndpoint, ready: Option, + observation: Option, } /// How far this process has got with the answer to a live contest. @@ -596,6 +617,7 @@ impl ProviderEndpoint { close_handoff: None, }, ready: Some(ready), + observation: None, }) } @@ -606,7 +628,11 @@ impl ProviderEndpoint { /// [`WorkService::close_only`] serves it behind, where one type has /// to carry both halves because one handler answers the wire. const fn close_only(close: CloseEndpoint) -> Self { - Self { close, ready: None } + Self { + close, + ready: None, + observation: None, + } } /// The readiness this endpoint admits new work under, or the refusal @@ -617,6 +643,9 @@ impl ProviderEndpoint { /// [`EndpointError::NotAdmitting`] when no readiness decision is /// held. Nothing about a close reaches this. fn admitting(&self) -> Result<&ReadyChannel, EndpointError> { + if let Some(observation) = &self.observation { + observation.check()?; + } self.ready.as_ref().ok_or(EndpointError::NotAdmitting) } @@ -730,11 +759,11 @@ impl ProviderEndpoint { .map_err(|error| Refusal::new(endpoint_refusal(error), error.to_string()))? .clone(); let (cursor_height, _) = self.state().cursor(); - let policy = *ready.execution_policy(); - check_authorization(ready.channel(), &authorization, &policy, cursor_height)?; - let bundle = PreparedPaidInputV1::decode(&request.prepared_input, MAX_RECORD_BYTES) + let policy = ready.execution_policy(); + policy.check_authorization(ready.channel(), &authorization, cursor_height)?; + let bundle = PreparedPaidWorkInput::decode(&request.prepared_input, MAX_RECORD_BYTES) .map_err(|error| Refusal::invalid(error.to_string()))?; - check_prepared_input(ready.channel(), &authorization, &policy, &bundle)?; + policy.check_input(ready.channel(), &authorization, &bundle)?; ready.check_signable( cursor_height, authorization.terminal_deadline, @@ -774,14 +803,17 @@ impl ProviderEndpoint { /// Decides whether the backend may be invoked for `work_id`, and /// makes that decision durable before it is returned. /// - /// [`RunAdmission::Invoke`] is returned only when the job was + /// [`RunAdmission::Invoke`] or [`RunAdmission::InvokeFetch`] is + /// returned only when the job was /// accepted and not yet running, and only after the running marker /// is on the disk — so a crash between the marker and the answer /// costs the invocation, never a second one. Every other phase past /// acceptance answers with what it already is: a job this process is /// running, a job whose result is already signed, or a job whose /// marker was found by a process that did not write it. A job that - /// was never co-signed is refused rather than answered. + /// was never co-signed is refused rather than answered. A job whose + /// input a metadata-only journal no longer holds is + /// [`RunAdmission::Indeterminate`]: it must not be executed again. /// /// `ready` is a *fresh* readiness decision, and the freshness is the /// caller's to owe in exactly the sense [`ReadyChannel`] already @@ -853,22 +885,40 @@ impl ProviderEndpoint { authorization.payment_deadline, )?; - let bundle = PreparedPaidInputV1::decode(job.prepared_input(), MAX_RECORD_BYTES) + if job.prepared_input().is_empty() { + return Ok(RunAdmission::Indeterminate); + } + let bundle = PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) .map_err(PaidWorkError::from)?; - let parts = bundle.parts().map_err(PaidWorkError::from)?; - let input = PreparedEvaluateInput { parts }; + ready + .execution_policy() + .check_input(ready.channel(), &authorization, &bundle)?; + let admission = match (bundle, ready.execution_policy()) { + (PreparedPaidWorkInput::Evaluate(bundle), PaidWorkPolicy::Evaluate(_)) => { + RunAdmission::Invoke(Box::new(PreparedEvaluateInput { + parts: bundle.parts().map_err(PaidWorkError::from)?, + })) + } + (PreparedPaidWorkInput::Fetch(bundle), PaidWorkPolicy::Fetch { policy, .. }) => { + RunAdmission::InvokeFetch(Box::new(PreparedFetchInput { + parts: bundle.parts().map_err(PaidWorkError::from)?, + policy: *policy, + })) + } + _ => return Err(RunError::Policy), + }; self.close.store.commit( ChannelRecord::JobRunning { work_id }, &Secp256k1Verifier::new(), )?; - Ok(RunAdmission::Invoke(Box::new(input))) + Ok(admission) } /// Signs the result of the transcript this job's invocation /// produced, and returns it only once it is on the disk. /// - /// The transcript is the provider's own: [`terminal_result`] refuses + /// The transcript is the provider's own: [`hellas_rpc::protocol::work::terminal_result`] refuses /// events that are not one verified chain for this authorization's /// request under the channel's provider key, and the journal refuses /// the record unless the job is running, is not indeterminate, and @@ -891,13 +941,26 @@ impl ProviderEndpoint { ) -> Result<(PaidJobResultV1, Sig), RunError> { let job = self.state().job_by_id(work_id).ok_or(RunError::NoSuchJob)?; let authorization = *job.authorization(); - let ready = self.admitting()?.clone(); + let input = PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) + .map_err(|e| RunError::Transcript(e.into()))?; + let ready = self + .ready + .as_ref() + .ok_or(EndpointError::NotAdmitting)? + .clone(); + // The last fresh readiness, deliberately not `admitting()`: a result + // signature commits nothing the payment path does not gate on its + // own freshness check, and refusing here would discard completed + // upstream work that the client may still pay for. Release and + // payment still require a fresh observation. let channel = ready.channel(); - let result = - terminal_result(channel, &authorization, transcript).map_err(RunError::Transcript)?; + let result = ready + .execution_policy() + .terminal_result(channel, &authorization, &input, transcript) + .map_err(RunError::Transcript)?; let spool = encode_transcript(transcript).map_err(RunError::Transcript)?; let spooled = u64::try_from(spool.len()).unwrap_or(u64::MAX); - let limit = ready.execution_policy().max_spool_bytes; + let limit = ready.execution_policy().max_spool_bytes(); if spooled > limit { return Err(RunError::Record(PaidWorkError::OverEnvelope { field: "spooled transcript length", @@ -917,16 +980,32 @@ impl ProviderEndpoint { // ledger charges this client for over an answer it was never // able to take. The client's own check is the other end of the // same bound, against a provider that does not apply this one. - let frame = u64::try_from( - WorkDelivered { - result: result.encode(), - provider_signature: signature.as_bytes().to_vec(), - transcript: spool.clone(), + let delivered = WorkDelivered { + result: result.encode(), + provider_signature: signature.as_bytes().to_vec(), + transcript: spool.clone(), + }; + let frame = if matches!(ready.execution_policy(), PaidWorkPolicy::Fetch { .. }) { + if spool.len() > hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES { + return Err(RunError::Record(PaidWorkError::OverEnvelope { + field: "Fetch transcript", + actual: spool.len() as u64, + limit: hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES as u64, + })); } - .encoded_len(), - ) - .unwrap_or(u64::MAX); - let frame_limit = u64::from(ready.execution_policy().max_encoded_result_frame); + stream::fetch_frames( + &delivered, + transcript, + ready.execution_policy().max_encoded_result_frame(), + ) + .try_fold(0, |size, frame| { + Ok::<_, RunError>(size.max(frame?.encoded_len() as u64)) + })? + } else { + delivered.encoded_len() as u64 + }; + let frame_limit = u64::from(ready.execution_policy().max_encoded_result_frame()) + .min(hellas_wire::frame::MAX_FRAME_BYTES as u64); if frame > frame_limit { return Err(RunError::Record(PaidWorkError::OverEnvelope { field: "encoded result frame", @@ -1033,6 +1112,9 @@ impl ProviderEndpoint { }; let (result, signature) = (*result, *signature); let transcript = job.transcript().to_vec(); + if transcript.is_empty() { + return Err(DeliverError::NoResult { phase: job.phase() }); + } let terminal_deadline = job.authorization().terminal_deadline; bind(ready, &self.close.store, &self.close.signer, Role::Provider)?; @@ -1094,6 +1176,12 @@ impl ProviderEndpoint { let certificate_signature = signature(&request.certificate_signature) .ok_or(PaymentError::Malformed("certificate signature"))?; + if self.state().payment(work_id).is_none() + && let Some(observation) = &self.observation + { + observation.check()?; + } + let state = self.close.store.commit( ChannelRecord::JobTerminated { work_id, @@ -1634,7 +1722,7 @@ impl BackendFault { /// The complete journaled input handed to a paid Evaluate backend. /// -/// All six bodies come from the same strictly decoded [`PreparedPaidInputV1`] +/// All six bodies come from the same strictly decoded [`hellas_rpc::protocol::artifacts::PreparedPaidInputV1`] /// whose digest the parties signed. Keeping the execution, tokens, policy, and /// identity here is what lets a backend run after restart without depending on /// transient Courtesy state. Environment bytes remain content-store data below @@ -1664,10 +1752,38 @@ impl PreparedEvaluateInput { } } +/// A paid Fetch input admitted under a channel policy. It is held only in +/// memory on the provider, even while the job's accounting state is durable. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PreparedFetchInput { + parts: PreparedPaidFetchInputParts, + policy: PaidFetchPolicyV1, +} + +impl PreparedFetchInput { + /// Builds an admitted input from its verified parts and channel policy. + #[must_use] + pub const fn new(parts: PreparedPaidFetchInputParts, policy: PaidFetchPolicyV1) -> Self { + Self { parts, policy } + } + + /// Returns the fixed output bounds the provider must enforce while running. + pub const fn policy(&self) -> &PaidFetchPolicyV1 { + &self.policy + } + + /// Transfers the signed request and canonical manifest to the backend. + pub fn into_parts(self) -> PreparedPaidFetchInputParts { + self.parts + } +} + /// The one seam a paid job crosses on its way to real execution. /// -/// One method, and it takes the complete prepared graph this endpoint rebuilt -/// from its own journal rather than anything transient or supplied at dispatch. +/// The endpoint selects Evaluate or Fetch from the admitted profile and passes +/// the verified prepared input. Evaluate inputs survive journal recovery; +/// Fetch bodies exist only in this process's memory. Implement only the profiles +/// the backend serves; unsupported profiles fail without running work. /// What comes back is the complete signed transcript of that invocation — not /// a digest of one, because a digest is exactly what a backend that ran nothing /// could also return. @@ -1675,12 +1791,41 @@ impl PreparedEvaluateInput { /// Implementors must invoke once per call. That is not a property this /// trait can check, and it is not the one the gate rests on: the gate /// calls this at most once per `work_id` whatever the implementor does. -pub trait PaidEvaluateBackend: Sync { +pub trait PaidWorkBackend: Sync { + /// Runs an authorized Fetch without persisting request or response bodies. + fn fetch( + &self, + _input: PreparedFetchInput, + ) -> impl core::future::Future, BackendFault>> + Send + { + async { Err(BackendFault::new("paid fetch backend is unavailable")) } + } + + /// Runs once, exposing signed Fetch prefixes before terminal delivery. + fn fetch_stream( + &self, + input: PreparedFetchInput, + progress: PaidProgress, + ) -> impl core::future::Future, BackendFault>> + Send + { + async move { + let events = self.fetch(input).await?; + for event in &events { + if event.event().body().kind() == hellas_rpc::fetch::OUTPUT_EVENT_KIND { + progress(event.clone())?; + } + } + Ok(events) + } + } /// Runs one journaled Evaluate input to its terminal. fn evaluate( &self, - input: PreparedEvaluateInput, - ) -> impl core::future::Future, BackendFault>> + Send; + _input: PreparedEvaluateInput, + ) -> impl core::future::Future, BackendFault>> + Send + { + async { Err(BackendFault::new("paid evaluate backend is unavailable")) } + } /// Run once, exposing authenticated token prefixes while retaining the /// complete transcript for durable terminal delivery. @@ -1708,6 +1853,8 @@ pub enum RunAdmission { /// The marker is durable and the backend has not been called. /// Invoke exactly once, with this journaled input. Invoke(Box), + /// The fetch running marker is durable; invoke once with the in-memory input. + InvokeFetch(Box), /// This process marked the job and has not recorded its result. Running, /// A signed result already exists. @@ -1717,9 +1864,8 @@ pub enum RunAdmission { /// The provider's signature over its digest. signature: Sig, }, - /// A marker was found by a process that did not write it. Whether - /// the backend ran is not knowable here, and nothing resolves it - /// automatically. + /// A prior process left a running marker, or an accepted Fetch lost its + /// in-memory input. Neither case is automatically re-executed. Indeterminate, } @@ -1830,11 +1976,15 @@ pub async fn run_accepted_work( work_id: Digest, ) -> Result where - B: PaidEvaluateBackend + Sync, + B: PaidWorkBackend + Sync, { let admission = service.begin_run(work_id, ready)?; - let input = match admission { - RunAdmission::Invoke(input) => *input, + let progress_service = service.clone(); + let progress: PaidProgress = + Arc::new(move |event| progress_service.publish_progress(work_id, event)); + let invoked = match admission { + RunAdmission::Invoke(input) => backend.evaluate_stream(*input, progress).await, + RunAdmission::InvokeFetch(input) => backend.fetch_stream(*input, progress).await, RunAdmission::Running => return Ok(RunOutcome::Running), RunAdmission::Indeterminate => return Ok(RunOutcome::Indeterminate), RunAdmission::Ready { result, signature } => { @@ -1842,10 +1992,7 @@ where } }; - let progress_service = service.clone(); - let progress: PaidProgress = - Arc::new(move |event| progress_service.publish_progress(work_id, event)); - let transcript = match backend.evaluate_stream(input, progress).await { + let transcript = match invoked { Ok(transcript) => transcript, Err(fault) => return Err(end_failed(service, work_id, RunError::Backend(fault))), }; @@ -1877,7 +2024,7 @@ pub async fn run_accepted_work_after_catch_up( ) -> Result where S: FinalizedBlocks + ?Sized, - B: PaidEvaluateBackend + Sync, + B: PaidWorkBackend + Sync, { service .catch_up_job(source, work_id) @@ -1990,7 +2137,9 @@ impl From for Refusal { let code = match error { DeliverError::NoSuchJob | DeliverError::Terminated { .. } => WorkRefusal::Declined, DeliverError::NoResult { .. } => WorkRefusal::NotReady, - DeliverError::Endpoint(EndpointError::CatchingUp) => WorkRefusal::NotReady, + DeliverError::Endpoint(EndpointError::CatchingUp | EndpointError::ObservationStale) => { + WorkRefusal::NotReady + } DeliverError::Endpoint(EndpointError::NotAdmitting) => WorkRefusal::Unavailable, DeliverError::Unbound => WorkRefusal::Invalid, DeliverError::Setup(setup) => return Refusal::from(setup), @@ -2010,6 +2159,9 @@ impl From for Refusal { /// only one of them can raise are documented where they are mapped. #[derive(Debug, thiserror::Error)] pub enum PaymentError { + /// The shared client endpoint is unavailable. + #[error(transparent)] + Endpoint(#[from] EndpointError), /// No open job on this channel carries this `work_id`. #[error("no open job on this channel carries this work id")] NoSuchJob, @@ -2069,6 +2221,7 @@ impl From for Refusal { let code = match error { PaymentError::NoSuchJob => WorkRefusal::Declined, PaymentError::NotPayable { .. } => WorkRefusal::NotReady, + PaymentError::Endpoint(error) => endpoint_refusal(error), PaymentError::Store(store) => return Refusal::from(store), _ => WorkRefusal::Invalid, }; @@ -2123,7 +2276,7 @@ pub struct WorkService { endpoint: Arc>, driving: Arc, changed: Arc, - progress: Arc>>>, + progress: Arc>>, } /// The authority to advance this channel's cursor, and the only thing @@ -2247,10 +2400,20 @@ impl ChannelDriver<'_> { &mut self, source: &S, ) -> Result { - let mut cursor = self.cursor()?; + let cursor = self.cursor()?; let Some(latest) = source.latest_height().await? else { return Ok(cursor); }; + self.catch_up_to(source, latest).await + } + + /// Applies history through a fixed snapshot height, respecting close duties. + pub async fn catch_up_to( + &mut self, + source: &S, + latest: u64, + ) -> Result { + let mut cursor = self.cursor()?; while cursor < latest { let next = cursor.saturating_add(1); let block = source @@ -2378,6 +2541,54 @@ impl WorkService { self.endpoint()?.admit_new_work(ready) } + /// Requires a live background observer before accepting or releasing work. + pub fn require_observer(&self) -> Result<(), EndpointError> { + self.endpoint()?.observation = Some(Observation::default()); + Ok(()) + } + + /// Publishes readiness after applying all blocks through its snapshot. + pub fn observe_ready( + &self, + ready: ReadyChannel, + started: ObservationTime, + max_age: std::time::Duration, + ) -> Result<(), EndpointError> { + let mut endpoint = self.endpoint()?; + if ready.check_caught_up(endpoint.state().cursor().0).is_err() { + return Err(EndpointError::CatchingUp); + } + let height = ready.finalized_height(); + endpoint.admit_new_work(ready)?; + endpoint + .observation + .get_or_insert_with(Observation::default) + .confirm(height, started, max_age); + drop(endpoint); + self.changed.notify_waiters(); + Ok(()) + } + + /// Reads current local admission; no chain I/O is performed. + pub fn readiness(&self) -> Result { + let endpoint = self.endpoint()?; + if endpoint.state().is_closing() { + return Err(EndpointError::NotAdmitting); + } + endpoint.admitting().cloned() + } + + /// Revokes observer freshness without discarding results or close duties. + pub fn suspend(&self) -> Result<(), EndpointError> { + // An endpoint without an observer has no freshness to revoke; + // installing a defaulted one here would only close admission. + if let Some(observation) = self.endpoint()?.observation.as_mut() { + observation.suspend(); + self.changed.notify_waiters(); + } + Ok(()) + } + /// Borrows the endpoint, privately. /// /// Private, and that is only half of the discipline. The other half @@ -2726,7 +2937,9 @@ impl WorkService { /// proposal is wrong. const fn endpoint_refusal(error: EndpointError) -> WorkRefusal { match error { - EndpointError::CatchingUp | EndpointError::NotAdmitting => WorkRefusal::NotReady, + EndpointError::CatchingUp + | EndpointError::NotAdmitting + | EndpointError::ObservationStale => WorkRefusal::NotReady, _ => WorkRefusal::Unavailable, } } @@ -2789,11 +3002,11 @@ fn work_id_bytes(bytes: &[u8]) -> Option { /// /// The two things a proposal actually chooses. Everything else in the /// authorization is derived from the channel, the policy, and this -/// bundle by [`propose_authorization`]. +/// bundle by [`hellas_rpc::protocol::work::propose_authorization`]. #[derive(Clone, Debug, PartialEq, Eq)] pub struct JobProposal { /// The inputs the job runs on. - pub prepared_input: PreparedPaidInputV1, + pub prepared_input: PreparedPaidWorkInput, /// The three heights the job is bound by. pub deadlines: JobDeadlines, } @@ -2801,6 +3014,9 @@ pub struct JobProposal { /// Why a client could not complete one acceptance exchange. #[derive(Debug, thiserror::Error)] pub enum ProposeError { + /// The shared client endpoint is unavailable. + #[error(transparent)] + Endpoint(#[from] EndpointError), /// The provider refused. #[error("the provider refused as {refusal}: {reason}")] Refused { @@ -2845,7 +3061,8 @@ pub enum ProposeError { /// It owns one channel's journal and proposes on that channel alone. #[derive(Debug)] pub struct ClientEndpoint { - ready: ReadyChannel, + ready: Option, + observation: Option, store: ChannelStore, signer: Secp256k1Signer, } @@ -2864,7 +3081,19 @@ impl ClientEndpoint { ) -> Result { bind(&ready, &store, &signer, Role::Client)?; Ok(Self { - ready, + ready: Some(ready), + observation: None, + store, + signer, + }) + } + + /// Reopens accounting and close duties without requiring an open channel. + pub fn recover(store: ChannelStore, signer: Secp256k1Signer) -> Result { + bind_store(&store, &signer, Role::Client)?; + Ok(Self { + ready: None, + observation: None, store, signer, }) @@ -2897,8 +3126,12 @@ impl ClientEndpoint { /// outstanding, and the record, readiness, and journal errors the /// proposal itself raises. pub fn propose(&mut self, proposal: &JobProposal) -> Result { + if let Some(observation) = &self.observation { + observation.check()?; + } let (cursor_height, _) = self.state().cursor(); - let policy = *self.ready.execution_policy(); + let ready = self.ready.as_ref().ok_or(EndpointError::NotAdmitting)?; + let policy = ready.execution_policy(); for job in self .state() @@ -2906,9 +3139,8 @@ impl ClientEndpoint { .filter(|job| job.phase() == JobPhase::HalfSigned) { let retained = *job.authorization(); - let rebuilt = propose_authorization( - self.ready.channel(), - &policy, + let rebuilt = policy.propose( + self.state().channel(), &proposal.prepared_input, retained.proposal_nonce, proposal.deadlines, @@ -2927,21 +3159,19 @@ impl ClientEndpoint { // wait for the first, but no crash or reordered response can make // the sequence move backwards or reuse a number. let proposal_nonce = self.state().proposal_nonce_high_water().saturating_add(1); - let authorization = propose_authorization( - self.ready.channel(), - &policy, + let authorization = policy.propose( + self.state().channel(), &proposal.prepared_input, proposal_nonce, proposal.deadlines, )?; - check_authorization(self.ready.channel(), &authorization, &policy, cursor_height)?; - check_prepared_input( - self.ready.channel(), + policy.check_authorization(self.state().channel(), &authorization, cursor_height)?; + policy.check_input( + self.state().channel(), &authorization, - &policy, &proposal.prepared_input, )?; - self.ready.check_signable( + ready.check_signable( cursor_height, authorization.terminal_deadline, authorization.payment_deadline, @@ -2950,7 +3180,7 @@ impl ClientEndpoint { .prepared_input .encode() .map_err(PaidWorkError::from)?; - let work_id = work_id(self.ready.channel(), &authorization); + let work_id = work_id(self.state().channel(), &authorization); let signature = self.signer.sign(signing_hash(work_id)); self.store.commit( @@ -3059,7 +3289,7 @@ impl ClientEndpoint { return Ok(retained.clone()); } let start = close_start( - self.ready.channel(), + self.state().channel(), Party::Maker, height, self.state().executable_certificate(), @@ -3124,13 +3354,29 @@ impl ClientEndpoint { work_id: Digest, ready: &ReadyChannel, delivered: &WorkDelivered, + ) -> Result { + self.receive_inner(work_id, ready, delivered, false) + } + + fn receive_inner( + &mut self, + work_id: Digest, + ready: &ReadyChannel, + delivered: &WorkDelivered, + streamed: bool, ) -> Result { self.state() .job_by_id(work_id) .ok_or(DeliverError::NoSuchJob)?; bind(ready, &self.store, &self.signer, Role::Client)?; - if ready.execution_policy() != self.ready.execution_policy() { + if ready.execution_policy() + != self + .ready + .as_ref() + .ok_or(EndpointError::NotAdmitting)? + .execution_policy() + { return Err(DeliverError::Policy); } let (cursor_height, _) = self.state().cursor(); @@ -3139,13 +3385,54 @@ impl ClientEndpoint { // The bound is on the encoded message, which is what this // endpoint agreed to hold; the transport's own framing around // it is the transport's and is not measured here. - let limit = u64::from(ready.execution_policy().max_encoded_result_frame); + let limit = u64::from(ready.execution_policy().max_encoded_result_frame()); let actual = u64::try_from(delivered.encoded_len()).unwrap_or(u64::MAX); - if actual > limit { + if !streamed && actual > limit { return Err(DeliverError::OverFrame { actual, limit }); } let result = PaidJobResultV1::decode(&delivered.result)?; + if matches!(ready.execution_policy(), PaidWorkPolicy::Fetch { .. }) { + let actual = delivered.transcript.len() as u64; + let limit = ready.execution_policy().max_spool_bytes(); + if actual > limit { + return Err(PaidWorkError::OverEnvelope { + field: "spooled transcript length", + actual, + limit, + } + .into()); + } + let authorization = self + .state() + .job_by_id(work_id) + .ok_or(DeliverError::NoSuchJob)? + .authorization(); + let transcript = hellas_rpc::protocol::work::decode_transcript( + &delivered.transcript, + hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES, + )?; + let input = PreparedPaidWorkInput::decode( + self.state() + .job_by_id(work_id) + .ok_or(DeliverError::NoSuchJob)? + .prepared_input(), + MAX_RECORD_BYTES, + ) + .map_err(PaidWorkError::from)?; + let expected = ready.execution_policy().terminal_result( + ready.channel(), + authorization, + &input, + &transcript, + )?; + if expected != result { + return Err(PaidWorkError::Mismatch { + field: "result against its transcript", + } + .into()); + } + } let signature = signature(&delivered.provider_signature) .ok_or(DeliverError::Malformed("provider signature"))?; self.store.commit( @@ -3186,7 +3473,7 @@ impl ClientEndpoint { .job_by_id(work_id) .ok_or(DeliverError::NoSuchJob)?; let signature = self.signer.sign(signing_hash(delivery_request_digest( - self.ready.channel(), + self.state().channel(), work_id, exporter, ))); @@ -3244,7 +3531,7 @@ impl ClientEndpoint { .job_by_id(work_id) .ok_or(DeliverError::NoSuchJob)?; let (result, _) = job.result().ok_or(DeliverError::NoSuchJob)?; - let result_digest = result_digest(self.ready.channel(), result); + let result_digest = result_digest(self.state().channel(), result); self.store.commit( ChannelRecord::JobTerminated { work_id, @@ -3291,6 +3578,9 @@ impl ClientEndpoint { if let Some(retained) = self.state().payment(work_id) { return Ok(admit_request(&retained)); } + if let Some(observation) = &self.observation { + observation.check()?; + } let job = self .state() .job_by_id(work_id) @@ -3301,19 +3591,19 @@ impl ClientEndpoint { let (authorization, result) = (*job.authorization(), *result); let (certificate, binding) = next_payment( - self.ready.channel(), + self.state().channel(), &authorization, &result, self.state().ledger().credited_cumulative(), self.state().settlement(), )?; let binding_signature = self.signer.sign(signing_hash(payment_binding_digest( - self.ready.channel(), + self.state().channel(), &binding, ))); let certificate_signature = self .signer - .sign(certificate.digest(self.ready.channel().network())); + .sign(certificate.digest(self.state().channel().network())); self.store.commit( ChannelRecord::JobTerminated { @@ -3398,7 +3688,7 @@ impl ClientEndpoint { /// raises. pub async fn admit_payment( client: &WorkClientImpl, - endpoint: &mut ClientEndpoint, + endpoint: &mut impl ClientChannel, work_id: Digest, ) -> Result where @@ -3406,9 +3696,9 @@ where T::Error: std::error::Error + Send + Sync + 'static, T::Stream: 'static, { - let request = endpoint.pay(work_id)?; + let request = endpoint.with_client(|endpoint| endpoint.pay(work_id))??; let response = client.admit_certificate(request).await?; - endpoint.acknowledged(work_id, &response) + endpoint.with_client(|endpoint| endpoint.acknowledged(work_id, &response))? } /// Asks for one accepted job's answer over a live transport and makes it @@ -3430,7 +3720,7 @@ where /// raise. pub async fn fetch_result( transport: T, - endpoint: &mut ClientEndpoint, + endpoint: &mut impl ClientChannel, ready: &ReadyChannel, work_id: Digest, ) -> Result @@ -3442,12 +3732,15 @@ where let Some(exporter) = transport.context().open_exporter else { return Err(DeliverError::Unbindable); }; - let request = endpoint.request_delivery(work_id, &exporter)?; + let request = + endpoint.with_client(|endpoint| endpoint.request_delivery(work_id, &exporter))??; let response = WorkClientImpl::new(transport) .deliver_result(request) .await?; match response.outcome { - Some(DeliverOutcome::Delivered(delivered)) => endpoint.receive(work_id, ready, &delivered), + Some(DeliverOutcome::Delivered(delivered)) => { + endpoint.with_client(|endpoint| endpoint.receive(work_id, ready, &delivered))? + } Some(DeliverOutcome::Refused(refused)) => Err(DeliverError::Refused { refusal: WorkRefusal::from_code(refused.code) .ok_or(DeliverError::Malformed("refusal code"))?, @@ -3471,7 +3764,7 @@ where /// raises. pub async fn propose_work( transport: T, - endpoint: &mut ClientEndpoint, + endpoint: &mut impl ClientChannel, proposal: &JobProposal, ) -> Result where @@ -3479,9 +3772,9 @@ where T::Error: std::error::Error + Send + Sync + 'static, T::Stream: 'static, { - let request = endpoint.propose(proposal)?; + let request = endpoint.with_client(|endpoint| endpoint.propose(proposal))??; let response = WorkClientImpl::new(transport).accept_work(request).await?; - endpoint.accepted(&response) + endpoint.with_client(|endpoint| endpoint.accepted(&response))? } /// Resend a retained proposal over a live transport and journal its answer. @@ -3491,7 +3784,7 @@ where /// or [`ClientEndpoint::accepted`]. pub async fn resume_work_proposal( transport: T, - endpoint: &mut ClientEndpoint, + endpoint: &mut impl ClientChannel, work_id: Digest, ) -> Result where @@ -3499,9 +3792,9 @@ where T::Error: std::error::Error + Send + Sync + 'static, T::Stream: 'static, { - let request = endpoint.resume_proposal(work_id)?; + let request = endpoint.with_client(|endpoint| endpoint.resume_proposal(work_id))??; let response = WorkClientImpl::new(transport).accept_work(request).await?; - endpoint.accepted(&response) + endpoint.with_client(|endpoint| endpoint.accepted(&response))? } // ── Wire shapes ─────────────────────────────────────────────────────── diff --git a/crates/work/src/work/client_channel.rs b/crates/work/src/work/client_channel.rs new file mode 100644 index 00000000..a26e8ead --- /dev/null +++ b/crates/work/src/work/client_channel.rs @@ -0,0 +1,212 @@ +//! Shared client state. Network waits never hold the journal lock. + +use super::*; + +/// Synchronous access to a client journal, used between transport awaits. +pub trait ClientChannel: Send { + /// Runs one journal operation without allowing the borrow to escape. + fn with_client( + &mut self, + step: impl FnOnce(&mut ClientEndpoint) -> R, + ) -> Result; +} + +impl ClientChannel for ClientEndpoint { + fn with_client( + &mut self, + step: impl FnOnce(&mut ClientEndpoint) -> R, + ) -> Result { + Ok(step(self)) + } +} + +/// The single request owner for a client channel. Requests retain exclusive +/// exchange ordering while the observer independently borrows the journal. +#[derive(Debug)] +pub struct ClientService { + observer: ClientObserver, +} + +impl ClientService { + /// Starts sharing an already opened journal with its observer. + #[must_use] + pub fn new(mut endpoint: ClientEndpoint) -> Self { + endpoint.observation = Some(Observation::default()); + Self { + observer: ClientObserver { + endpoint: Arc::new(Mutex::new(endpoint)), + driving: Arc::new(AtomicBool::new(false)), + }, + } + } + + /// Gives the chain task observation authority, never payment authority. + #[must_use] + pub fn observer(&self) -> ClientObserver { + self.observer.clone() + } + + /// Reads local accounting without holding a borrow across an await. + pub fn with_state(&self, read: impl FnOnce(&ChannelState) -> R) -> Result { + self.observer.with_state(read) + } + + /// Checks the observer's current admission decision locally. + pub fn readiness(&self) -> Result { + self.observer.readiness() + } +} + +/// Shares finalized-history and readiness updates with the request owner. +/// Cloning this handle cannot create another payer. Driving remains exclusive. +#[derive(Clone, Debug)] +pub struct ClientObserver { + endpoint: Arc>, + driving: Arc, +} + +impl ClientObserver { + fn endpoint(&self) -> Result, EndpointError> { + self.endpoint.lock().map_err(|_| EndpointError::Poisoned) + } + + /// Reads local state without holding it across an await. + pub fn with_state(&self, read: impl FnOnce(&ChannelState) -> R) -> Result { + Ok(read(self.endpoint()?.state())) + } + + /// Publishes a coherent snapshot only after its history has been applied. + pub fn observe_ready( + &self, + ready: ReadyChannel, + started: ObservationTime, + max_age: std::time::Duration, + ) -> Result<(), EndpointError> { + let mut endpoint = self.endpoint()?; + bind(&ready, &endpoint.store, &endpoint.signer, Role::Client)?; + if ready.check_caught_up(endpoint.state().cursor().0).is_err() { + return Err(EndpointError::CatchingUp); + } + let height = ready.finalized_height(); + endpoint.ready = Some(ready); + endpoint + .observation + .as_mut() + .expect("shared client observation") + .confirm(height, started, max_age); + Ok(()) + } + + /// Returns local readiness; never contacts a validator. + pub fn readiness(&self) -> Result { + let endpoint = self.endpoint()?; + endpoint + .observation + .as_ref() + .expect("shared client observation") + .check()?; + if endpoint.state().is_closing() { + return Err(EndpointError::NotAdmitting); + } + endpoint.ready.clone().ok_or(EndpointError::NotAdmitting) + } + + /// Stops new signatures while the observer reconnects or shuts down. + pub fn suspend(&self) -> Result<(), EndpointError> { + self.endpoint()? + .observation + .as_mut() + .expect("shared client observation") + .suspend(); + Ok(()) + } + + /// Takes exclusive authority to drive finalized history, not requests. + pub fn drive(&self) -> Result, EndpointError> { + self.driving + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| EndpointError::CatchingUp)?; + Ok(ClientDriver { service: self }) + } +} + +impl ClientChannel for ClientService { + fn with_client( + &mut self, + step: impl FnOnce(&mut ClientEndpoint) -> R, + ) -> Result { + Ok(step(&mut *self.observer.endpoint()?)) + } +} + +/// The client's one chain driver. Dropping it releases driving authority. +pub struct ClientDriver<'a> { + service: &'a ClientObserver, +} + +impl Drop for ClientDriver<'_> { + fn drop(&mut self) { + self.service.driving.store(false, Ordering::Release); + } +} + +impl CloseChannel for ClientDriver<'_> { + fn with_store( + &mut self, + step: impl FnOnce(&mut ChannelStore) -> R, + ) -> Result { + let mut endpoint = self.service.endpoint().map_err(|_| CatchUpError::Busy)?; + Ok(step(&mut endpoint.store)) + } +} + +impl ClientDriver<'_> { + /// Applies a verified block under the same lock used by payments. + pub fn observe_finalized(&mut self, block: &FinalizedWork) -> Result<(), CatchUpError> { + self.with_store(|store| observe(store, block, &Secp256k1Verifier::new()))??; + Ok(()) + } + + /// Fetches missing blocks without borrowing the journal across network I/O. + pub async fn catch_up( + &mut self, + source: &S, + ) -> Result { + let Some(latest) = source.latest_height().await? else { + return self.with_store(|store| store.state().cursor().0); + }; + self.catch_up_to(source, latest).await + } + + /// Applies a fixed snapshot's history without querying a moving chain tip. + pub async fn catch_up_to( + &mut self, + source: &S, + latest: u64, + ) -> Result { + let next = self + .with_store(|store| store.state().cursor().0)? + .saturating_add(1); + for height in next..=latest { + let block = source + .block_at(height) + .await? + .ok_or(CatchUpError::Missing { height })?; + self.observe_finalized(&block)?; + } + self.with_store(|store| store.state().cursor().0) + } + + /// Drives a retained close without blocking ordinary state access. + pub async fn advance_close( + &mut self, + source: &S, + sink: &T, + ) -> Result + where + S: FinalizedBlocks + ?Sized, + T: TxSink + ?Sized, + { + advance_close(source, sink, self, &Secp256k1Verifier::new()).await + } +} diff --git a/crates/work/src/work/observation.rs b/crates/work/src/work/observation.rs new file mode 100644 index 00000000..6dab3863 --- /dev/null +++ b/crates/work/src/work/observation.rs @@ -0,0 +1,92 @@ +//! Process-local freshness of a continuously observed channel. + +use super::EndpointError; +use std::time::{Duration, SystemTime}; +use tokio::time::Instant; + +/// Capture before reading the chain. Using both clocks counts host sleep even +/// where monotonic time excludes it, and fails closed on wall-clock rollback. +#[derive(Clone, Copy, Debug)] +pub struct ObservationTime { + monotonic: Instant, + wall: SystemTime, +} + +impl ObservationTime { + /// Marks the beginning of a chain observation. + #[must_use] + pub fn now() -> Self { + Self { + monotonic: Instant::now(), + wall: SystemTime::now(), + } + } + + fn elapsed(self) -> Option { + self.wall + .elapsed() + .ok() + .map(|wall| wall.max(self.monotonic.elapsed())) + } +} + +/// A successful read only renews freshness when finalized history advances. +/// Reading an old tip repeatedly cannot keep a disconnected channel admitting. +#[derive(Debug, Default)] +pub(super) struct Observation { + height: Option, + confirmed: Option<(ObservationTime, Duration)>, +} + +impl Observation { + pub(super) fn confirm(&mut self, height: u64, started: ObservationTime, max_age: Duration) { + if self.height.is_none_or(|previous| height > previous) { + self.height = Some(height); + self.confirmed = Some((started, max_age)); + } + } + + pub(super) fn check(&self) -> Result<(), EndpointError> { + if self + .confirmed + .is_some_and(|(started, max_age)| started.elapsed().is_some_and(|age| age < max_age)) + { + Ok(()) + } else { + Err(EndpointError::ObservationStale) + } + } + + pub(super) fn suspend(&mut self) { + self.confirmed = None; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn host_sleep_and_clock_rollback_cannot_extend_readiness() { + for wall in [ + SystemTime::now() - Duration::from_secs(60), + SystemTime::now() + Duration::from_secs(60), + ] { + let mut observation = Observation::default(); + observation.confirm( + 1, + ObservationTime { + monotonic: Instant::now(), + wall, + }, + Duration::from_secs(5), + ); + assert_eq!(observation.check(), Err(EndpointError::ObservationStale)); + observation.confirm(2, ObservationTime::now(), Duration::from_secs(5)); + assert!( + observation.check().is_ok(), + "new finalized progress restores readiness" + ); + } + } +} diff --git a/crates/work/src/work/stream.rs b/crates/work/src/work/stream.rs index 148a310f..95d44a35 100644 --- a/crates/work/src/work/stream.rs +++ b/crates/work/src/work/stream.rs @@ -3,6 +3,7 @@ use super::*; use futures::{StreamExt, stream::BoxStream}; use hellas_rpc::pb::work::{WorkStreamEvent, work_stream_event}; use hellas_rpc::protocol::work::decode_transcript; +use hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES; use hellas_rpc::{ Operation, StreamId, output_genesis, scheme_id, verify_output_event_continuation, }; @@ -10,6 +11,84 @@ use hellas_rpc::{ pub type PaidProgress = Arc Result<(), BackendFault> + Send + Sync>; pub type PaidResultStream = BoxStream<'static, Result>; +#[derive(Debug, Default)] +pub(super) struct Progress { + events: Vec, + bytes: usize, +} + +/// Fetch's terminal authenticates the assembled transcript without repeating it +/// in one wire frame. Signatures, result digests and payment records are unchanged. +pub(super) fn fetch_frames<'a>( + delivered: &'a WorkDelivered, + events: &'a [OutputEventEnvelope], + frame_limit: u32, +) -> impl Iterator> + 'a { + let prefixes = &events[..events.len().saturating_sub(1)]; + prefix_batches(prefixes, frame_limit, true) + .map(|batch| { + Ok(WorkStreamEvent { + outcome: Some(work_stream_event::Outcome::Prefix(encode_transcript( + batch, + )?)), + }) + }) + .chain(events.last().into_iter().map(|event| { + let transcript = encode_transcript(std::slice::from_ref(event))?; + let outcome = + work_stream_event::Outcome::Terminal(hellas_rpc::pb::work::WorkStreamTerminal { + result: delivered.result.clone(), + provider_signature: delivered.provider_signature.clone(), + terminal_transcript: transcript, + }); + Ok(WorkStreamEvent { + outcome: Some(outcome), + }) + })) +} + +// Batch only events already available. Each bounded frame still passes the +// provider's local readiness check; the client verifies each envelope separately. +fn prefix_batches( + mut events: &[OutputEventEnvelope], + frame_limit: u32, + fetch: bool, +) -> impl Iterator { + let budget = if fetch { + (frame_limit as usize).min(64 * 1024).saturating_sub(32) + } else { + 0 + }; + std::iter::from_fn(move || { + if events.is_empty() { + return None; + } + let mut bytes = 0usize; + let mut count = 0; + for event in events { + let next = bytes.saturating_add(spool_charge(event, fetch)); + if count > 0 && next > budget { + break; + } + bytes = next; + count += 1; + } + let (batch, rest) = events.split_at(count); + events = rest; + Some(batch) + }) +} + +fn spool_charge(event: &OutputEventEnvelope, fetch: bool) -> usize { + // The canonical envelope represents payload bytes as CBOR integers, up to + // two encoded bytes each. Include signature/structure overhead before decode. + event + .payload() + .len() + .saturating_mul(if fetch { 2 } else { 1 }) + .saturating_add(1024) +} + impl ProviderEndpoint { fn reserve_stream( &mut self, @@ -47,33 +126,50 @@ impl ProviderEndpoint { } impl WorkService { + fn check_stream_release( + &self, + request: &DeliverResultRequest, + exporter: &[u8; 32], + ) -> Result<(), WireStatus> { + self.endpoint() + .map_err(DeliverError::from) + .and_then(|mut endpoint| endpoint.reserve_stream(request, exporter)) + .map_err(|error| WireStatus::new(hellas_wire::WireCode::Unavailable, error.to_string())) + } + pub(super) fn publish_progress( &self, work_id: Digest, event: OutputEventEnvelope, ) -> Result<(), BackendFault> { - let limit = self + let policy = self .endpoint() .map_err(|error| BackendFault::new(error.to_string()))? - .admitting() - .map_err(|error| BackendFault::new(error.to_string()))? + .ready + .as_ref() + .ok_or_else(|| BackendFault::new("channel has no execution policy"))? .execution_policy() - .max_spool_bytes; - let mut progress = self.progress.lock().expect("paid progress poisoned"); - let events = progress.entry(work_id).or_default(); - // Signed token events are bounded by the same spool as final delivery. - let bytes = event.payload().len() + 1024; - let retained = events - .iter() - .map(|event| event.payload().len() + 1024) - .sum::(); - if retained.saturating_add(bytes) as u64 > limit { + .clone(); + let limit = match &policy { + PaidWorkPolicy::Fetch { .. } => policy + .max_spool_bytes() + .min(MAX_FETCH_TRANSCRIPT_BYTES as u64), + PaidWorkPolicy::Evaluate(_) => policy.max_spool_bytes(), + }; + let mut jobs = self.progress.lock().expect("paid progress poisoned"); + let progress = jobs.entry(work_id).or_default(); + // Signed prefixes are bounded by the same spool as final delivery. + let fetch = matches!(policy, PaidWorkPolicy::Fetch { .. }); + let bytes = spool_charge(&event, fetch); + let retained = progress.bytes.saturating_add(bytes); + if retained as u64 > limit { return Err(BackendFault::new( "live result exceeds its authorized spool", )); } - events.push(event); - drop(progress); + progress.bytes = retained; + progress.events.push(event); + drop(jobs); self.changed.notify_waiters(); Ok(()) } @@ -102,10 +198,42 @@ impl WorkService { .and_then(|mut endpoint| endpoint.reserve_stream(&request, &exporter)); match reserved { Ok(()) => { + let limits = service.endpoint().and_then(|endpoint| { + let policy = endpoint.admitting()?.execution_policy(); + Ok((policy.max_encoded_result_frame(), matches!(policy, PaidWorkPolicy::Fetch { .. }))) + }); + let (frame_limit, fetch) = match limits { + Ok(limits) => limits, + Err(error) => { + yield Err(WireStatus::new(hellas_wire::WireCode::Unavailable, error.to_string())); + return; + } + }; let response = service.release(&request, &context); match response.outcome { Some(DeliverOutcome::Delivered(delivered)) => { - yield Ok(WorkStreamEvent { outcome: Some(work_stream_event::Outcome::Delivered(delivered)) }); + let events = match decode_transcript(&delivered.transcript, MAX_FETCH_TRANSCRIPT_BYTES) { + Ok(events) => events, + Err(error) => { + yield Err(WireStatus::new(hellas_wire::WireCode::Internal, error.to_string())); + return; + } + }; + if events.last().is_some_and(|event| event.event().body().kind() == hellas_rpc::fetch::OUTPUT_TERMINAL_KIND) { + let Some(tail) = events.get(position..) else { + yield Err(WireStatus::new(hellas_wire::WireCode::Internal, "terminal result is shorter than streamed output")); + return; + }; + for frame in fetch_frames(&delivered, tail, frame_limit) { + if let Err(error) = service.check_stream_release(&request, &exporter) { + yield Err(error); + return; + } + yield frame.map_err(|error| WireStatus::new(hellas_wire::WireCode::Internal, error.to_string())); + } + } else { + yield Ok(WorkStreamEvent { outcome: Some(work_stream_event::Outcome::Delivered(delivered)) }); + } return; } Some(DeliverOutcome::Refused(refused)) if refused.code != WorkRefusal::NotReady.code() as i32 => { @@ -116,12 +244,16 @@ impl WorkService { } let pending = { let progress = service.progress.lock().expect("paid progress poisoned"); - progress.get(&work_id).map(|events| events[position.min(events.len())..].to_vec()).unwrap_or_default() + progress.get(&work_id).map(|progress| progress.events[position.min(progress.events.len())..].to_vec()).unwrap_or_default() }; - for event in pending { - match encode_transcript(&[event]) { + for batch in prefix_batches(&pending, frame_limit, fetch) { + if let Err(error) = service.check_stream_release(&request, &exporter) { + yield Err(error); + return; + } + match encode_transcript(batch) { Ok(prefix) => { - position += 1; + position += batch.len(); yield Ok(WorkStreamEvent { outcome: Some(work_stream_event::Outcome::Prefix(prefix)) }); } Err(error) => { @@ -148,11 +280,11 @@ impl WorkService { } } -/// Deliver verified token prefixes and finally journal the complete result. +/// Deliver verified prefixes and finally journal the complete result. /// The caller pays only after this returns; it must keep running if its UI drops. pub async fn fetch_result_stream( transport: T, - endpoint: &mut ClientEndpoint, + endpoint: &mut impl ClientChannel, ready: &ReadyChannel, work_id: Digest, mut progress: impl FnMut(&OutputEventEnvelope) -> Result<(), PaidWorkError>, @@ -166,26 +298,75 @@ where .context() .open_exporter .ok_or(DeliverError::Unbindable)?; - let request = endpoint.request_delivery(work_id, &exporter)?; - let job = endpoint - .state() - .job_by_id(work_id) - .ok_or(DeliverError::NoSuchJob)?; - let prepared = PreparedPaidInputV1::decode(job.prepared_input(), MAX_RECORD_BYTES) - .map_err(PaidWorkError::from)?; - let parts = prepared.parts().map_err(PaidWorkError::from)?; - let input = hellas_rpc::evaluate::input_commitment(&parts.evaluate_request); + let (request, prepared) = endpoint.with_client(|endpoint| { + let request = endpoint.request_delivery(work_id, &exporter)?; + let job = endpoint + .state() + .job_by_id(work_id) + .ok_or(DeliverError::NoSuchJob)?; + let prepared = PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) + .map_err(PaidWorkError::from)?; + Ok::<_, DeliverError>((request, prepared)) + })??; + let input = prepared.input_commitment()?; + let (operation, kind, max_tokens, max_events, max_bytes) = + match (&prepared, ready.execution_policy()) { + (PreparedPaidWorkInput::Evaluate(prepared), PaidWorkPolicy::Evaluate(_)) => ( + Operation::Evaluate, + hellas_rpc::evaluate::TOKEN_DELTA_EVENT_KIND, + u64::from( + prepared + .parts() + .map_err(PaidWorkError::from)? + .text_policy + .max_new_tokens(), + ), + u64::MAX, + u64::MAX, + ), + (PreparedPaidWorkInput::Fetch(_), PaidWorkPolicy::Fetch { policy, .. }) => ( + Operation::Fetch, + hellas_rpc::fetch::OUTPUT_EVENT_KIND, + 0, + u64::from(policy.max_output_events) + .min(hellas_rpc::fetch::MAX_FETCH_OUTPUT_EVENTS as u64), + u64::from(policy.max_output_bytes) + .min(hellas_rpc::fetch::MAX_FETCH_OUTPUT_PAYLOAD_BYTES as u64), + ), + _ => { + return Err(DeliverError::Malformed( + "stream profile differs from channel", + )); + } + }; let mut token_count = 0u64; + let mut payload_bytes = 0u64; let mut retained_bytes = 0usize; + let spool_limit = if operation == Operation::Fetch { + ready + .execution_policy() + .max_spool_bytes() + .min(MAX_FETCH_TRANSCRIPT_BYTES as u64) + } else { + ready.execution_policy().max_spool_bytes() + }; let key = hellas_rpc::PublicKey::Secp256k1(ready.channel().provider_key().to_bytes()); - let scheme = scheme_id(Operation::Evaluate, hellas_rpc::Assurance::ProducerSigned); + let scheme = scheme_id(operation, prepared.assurance()?); let mut previous = output_genesis(input, StreamId::from_input_commitment(input)); let mut streamed = Vec::new(); let mut stream = WorkClientImpl::new(transport) .stream_result(request) .await?; while let Some(event) = stream.next().await { - match event?.outcome { + let event = event?; + let frame_limit = ready.execution_policy().max_encoded_result_frame() as u64; + if operation == Operation::Fetch && event.encoded_len() as u64 > frame_limit { + return Err(DeliverError::OverFrame { + actual: event.encoded_len() as u64, + limit: frame_limit, + }); + } + match event.outcome { Some(work_stream_event::Outcome::Prefix(prefix)) => { let events = decode_transcript(&prefix, MAX_RECORD_BYTES)?; for event in events { @@ -198,17 +379,28 @@ where &event, ) .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; - if event.event().body().kind() != hellas_rpc::evaluate::TOKEN_DELTA_EVENT_KIND { + if event.event().body().kind() != kind { return Err(DeliverError::Malformed( - "live prefix must contain token deltas", + "live prefix has the wrong event kind", )); } - let delta = hellas_rpc::evaluate::decode_token_delta_payload(event.payload()) - .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; - token_count = token_count.saturating_add(delta.token_ids.len() as u64); - retained_bytes = retained_bytes.saturating_add(event.payload().len() + 1024); - if token_count > u64::from(parts.text_policy.max_new_tokens()) - || retained_bytes as u64 > ready.execution_policy().max_spool_bytes + if operation == Operation::Evaluate { + let delta = + hellas_rpc::evaluate::decode_token_delta_payload(event.payload()) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + token_count = token_count.saturating_add(delta.token_ids.len() as u64); + } else { + hellas_rpc::fetch::decode_fetch_event_payload(event.payload()) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + } + payload_bytes = payload_bytes.saturating_add(event.payload().len() as u64); + retained_bytes = retained_bytes + .saturating_add(spool_charge(&event, operation == Operation::Fetch)); + // Reserve one envelope for the required terminal. + if token_count > max_tokens + || streamed.len() as u64 + 1 >= max_events + || payload_bytes > max_bytes + || retained_bytes as u64 > spool_limit { return Err(DeliverError::Malformed( "live result exceeds authorized output", @@ -229,14 +421,56 @@ where return Err(DeliverError::Malformed("event after terminal result")); } stream.finish()?; - let result = endpoint.receive(work_id, ready, &delivered)?; + let result = endpoint + .with_client(|endpoint| endpoint.receive(work_id, ready, &delivered))??; for event in &events[streamed.len()..] { - if event.event().body().kind() == hellas_rpc::evaluate::TOKEN_DELTA_EVENT_KIND { + if event.event().body().kind() == kind { progress(event)?; } } return Ok(result); } + Some(work_stream_event::Outcome::Terminal(terminal)) => { + if operation != Operation::Fetch { + return Err(DeliverError::Malformed("stream terminal requires Fetch")); + } + let mut tail = decode_transcript(&terminal.terminal_transcript, MAX_RECORD_BYTES)?; + if tail.len() != 1 + || tail[0].event().body().kind() != hellas_rpc::fetch::OUTPUT_TERMINAL_KIND + { + return Err(DeliverError::Malformed( + "stream terminal must contain one terminal envelope", + )); + } + if payload_bytes.saturating_add(tail[0].payload().len() as u64) > max_bytes + || retained_bytes.saturating_add(spool_charge(&tail[0], true)) as u64 + > spool_limit + { + return Err(DeliverError::Malformed( + "live result exceeds authorized output", + )); + } + // No delivery or payment is recorded before the stream's final + // status and the complete signed transcript have both verified. + if stream.next().await.transpose()?.is_some() { + return Err(DeliverError::Malformed("event after terminal result")); + } + stream.finish()?; + streamed.append(&mut tail); + let transcript = encode_transcript(&streamed)?; + return endpoint.with_client(|endpoint| { + endpoint.receive_inner( + work_id, + ready, + &WorkDelivered { + result: terminal.result, + provider_signature: terminal.provider_signature, + transcript, + }, + true, + ) + })?; + } Some(work_stream_event::Outcome::Refused(refused)) => { return Err(DeliverError::Refused { refusal: WorkRefusal::from_code(refused.code) @@ -252,3 +486,53 @@ where "result stream ended without terminal result", )) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn batched_fetch_preserves_every_envelope_and_bounds_each_frame() { + let key = hellas_rpc::ProducerSigningKey::from_secret_bytes([1; 32]).unwrap(); + let mut builder = hellas_rpc::fetch::FetchOutputTranscriptBuilder::new( + hellas_rpc::InputCommitment::from_digest(hellas_rpc::Digest::from_bytes([2; 32])), + hellas_rpc::Assurance::ProducerSigned, + &key, + ); + for _ in 0..100 { + builder.push_event(vec![7; 256]).unwrap(); + } + let events = builder.finish(vec![8; 16]).unwrap(); + let delivered = WorkDelivered { + result: vec![3; 128], + provider_signature: vec![4; 64], + transcript: encode_transcript(&events).unwrap(), + }; + for limit in [4096, 65536] { + // Resuming after an arbitrary prefix must skip envelopes, not batches. + for position in [0, 1, 47, 100] { + let frames = fetch_frames(&delivered, &events[position..], limit) + .collect::, _>>() + .unwrap(); + let mut decoded = Vec::new(); + for frame in &frames { + assert!(frame.encoded_len() <= limit as usize); + let bytes = match frame.outcome.as_ref().unwrap() { + work_stream_event::Outcome::Prefix(bytes) => bytes, + work_stream_event::Outcome::Terminal(end) => &end.terminal_transcript, + _ => panic!("unexpected frame"), + }; + decoded.extend(decode_transcript(bytes, MAX_RECORD_BYTES).unwrap()); + } + assert_eq!(decoded, events[position..]); + assert!(matches!( + frames.last().unwrap().outcome, + Some(work_stream_event::Outcome::Terminal(_)) + )); + if position == 0 { + assert!(frames.len() < events.len()); + } + } + } + } +} diff --git a/crates/work/src/work_open.rs b/crates/work/src/work_open.rs index 3434fa3d..062d0c70 100644 --- a/crates/work/src/work_open.rs +++ b/crates/work/src/work_open.rs @@ -755,14 +755,28 @@ fn mount( None => descriptor.expected_settlement().map_err(to_store)?, Some(payment) => descriptor.funded_settlement(payment).map_err(to_store)?, }; - let mut channel = ChannelStore::open( - store.root(), - descriptor.channel().clone(), - settlement, - store.role(), - origin, - verifier, - )?; + let mut channel = if matches!( + descriptor.execution_policy(), + hellas_rpc::protocol::work_profile::PaidWorkPolicy::Fetch { .. } + ) { + ChannelStore::open_metadata_only( + store.root(), + descriptor.channel().clone(), + settlement, + store.role(), + origin, + verifier, + )? + } else { + ChannelStore::open( + store.root(), + descriptor.channel().clone(), + settlement, + store.role(), + origin, + verifier, + )? + }; // The origin block carries the payment Open that established this // channel, and the store opened with its cursor already on that block. // A `StartPaymentClose` ordered after the Open in the very same block diff --git a/crates/work/src/work_store/channel.rs b/crates/work/src/work_store/channel.rs index 4789ef02..0bbf0ad1 100644 --- a/crates/work/src/work_store/channel.rs +++ b/crates/work/src/work_store/channel.rs @@ -22,12 +22,12 @@ use crate::work_store::{ take_option, }; use hellas_rpc::protocol::Digest; -use hellas_rpc::protocol::artifacts::PreparedPaidInputV1; use hellas_rpc::protocol::work::{ CreditLedger, PaidChannel, PaidJobAuthorizationV1, PaidJobResultV1, PaidWorkError, - PaymentBindingV1, PrivateRecord as _, decode_transcript, payment_binding_digest, - prepared_input_digest, result_digest, signing_hash, terminal_result, work_id, + PaymentBindingV1, PrivateRecord as _, decode_transcript, payment_binding_digest, result_digest, + signing_hash, work_id, }; +use hellas_rpc::protocol::work_profile::PreparedPaidWorkInput; mod codec; mod state; diff --git a/crates/work/src/work_store/channel/codec.rs b/crates/work/src/work_store/channel/codec.rs index 6b6cdb7d..95dd3d53 100644 --- a/crates/work/src/work_store/channel/codec.rs +++ b/crates/work/src/work_store/channel/codec.rs @@ -41,6 +41,10 @@ impl ChannelRecord { /// bytes whose digests the signatures beside them cover. #[must_use] pub fn encode(&self) -> Vec { + self.encode_for_storage(false) + } + + pub(super) fn encode_for_storage(&self, metadata_only: bool) -> Vec { let mut out = Vec::new(); match self { Self::CursorAdvanced { @@ -64,7 +68,9 @@ impl ChannelRecord { // Last field, and the whole of the rest: the journal // frame already carries this record's length, and a // second length here could disagree with it. - out.extend_from_slice(prepared_input); + if !metadata_only { + out.extend_from_slice(prepared_input); + } } Self::JobAccepted { work_id, @@ -91,7 +97,9 @@ impl ChannelRecord { // Last field, and the whole of the rest, for the reason // `JobProposed`'s bundle is: the journal frame already // carries this record's length. - out.extend_from_slice(transcript); + if !metadata_only { + out.extend_from_slice(transcript); + } } Self::PlaintextReleased { work_id } => { out.push(tag::PLAINTEXT); @@ -406,6 +414,10 @@ impl ChannelState { /// is one channel and not two that happen to agree. #[must_use] pub fn checkpoint(&self) -> Vec { + self.checkpoint_for_storage(false) + } + + pub(super) fn checkpoint_for_storage(&self, metadata_only: bool) -> Vec { let Self { channel, settlement, @@ -453,8 +465,8 @@ impl ChannelState { out.extend_from_slice(&result.encode()); out.extend_from_slice(signature.as_bytes()); }); - put_bytes(&mut out, prepared_input); - put_bytes(&mut out, transcript); + put_bytes(&mut out, if metadata_only { &[] } else { prepared_input }); + put_bytes(&mut out, if metadata_only { &[] } else { transcript }); } put_u64(&mut out, terminals.len() as u64); for terminal in terminals.values() { diff --git a/crates/work/src/work_store/channel/state.rs b/crates/work/src/work_store/channel/state.rs index 6d340643..240ebfe4 100644 --- a/crates/work/src/work_store/channel/state.rs +++ b/crates/work/src/work_store/channel/state.rs @@ -1,6 +1,42 @@ use super::*; impl ChannelState { + fn check_result_payload( + &self, + job: &JobState, + result: &PaidJobResultV1, + transcript: &[u8], + metadata_only: bool, + ) -> Result<(), ChannelStateError> { + if result.work_id != job.work_id { + return Err(ChannelStateError::WrongChannel { + field: "result work_id", + }); + } + if metadata_only { + return if transcript.is_empty() { + Ok(()) + } else { + Err(ChannelStateError::Malformed) + }; + } + let input = PreparedPaidWorkInput::decode(&job.prepared_input, MAX_RECORD_BYTES) + .map_err(PaidWorkError::from)?; + let budget = match &input { + PreparedPaidWorkInput::Fetch(_) => { + hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES + } + PreparedPaidWorkInput::Evaluate(_) => MAX_RECORD_BYTES, + }; + let events = decode_transcript(transcript, budget)?; + if input.terminal_result(&self.channel, &job.authorization, &events)? != *result { + return Err(ChannelStateError::WrongChannel { + field: "result against its transcript", + }); + } + Ok(()) + } + pub(super) fn new( channel: PaidChannel, settlement: WorkPaymentSettlement, @@ -37,9 +73,10 @@ impl ChannelState { settlement: WorkPaymentSettlement, role: Role, verifier: &V, + metadata_only: bool, ) -> Result { let state = Self::decode_checkpoint(bytes, channel, settlement, role)?; - state.revalidate(verifier)?; + state.revalidate(verifier, metadata_only)?; Ok(state) } @@ -51,7 +88,7 @@ impl ChannelState { /// the client's authorization, the provider's co-signature, the /// provider's result, and the client's binding and certificate. The /// result is rebuilt from the transcript beside it by - /// [`terminal_result`], exactly as a record replay rebuilds it. The + /// [`hellas_rpc::protocol::work::terminal_result`], exactly as a record replay rebuilds it. The /// inputs are hashed against the digest the authorization commits /// to. The retained close start is checked against this channel and /// this role, and the fixed answer is re-derived from the contest @@ -64,7 +101,11 @@ impl ChannelState { /// is the one thing this does not claim, and it is the reason the /// deadline rules stay where they are — on the records, at the /// heights they were taken. - fn revalidate(&self, verifier: &V) -> Result<(), ChannelStateError> { + fn revalidate( + &self, + verifier: &V, + metadata_only: bool, + ) -> Result<(), ChannelStateError> { // The one number consensus sees, against the one terminal that // could have moved it. A ledger the terminal does not produce is // a channel that would credit a second payment. @@ -77,7 +118,7 @@ impl ChannelState { if job.work_id != work_id(&self.channel, &job.authorization) { return Err(ChannelStateError::WrongChannel { field: "work_id" }); } - self.check_authorization(&job.authorization, &job.prepared_input)?; + self.check_authorization(&job.authorization, &job.prepared_input, metadata_only)?; if job.phase.only_role().is_some_and(|role| role != self.role) || job.provider_signature.is_some() != (job.phase != JobPhase::HalfSigned) || job.result.is_some() != job.phase.has_result() @@ -110,12 +151,7 @@ impl ChannelState { }); } if let Some((result, provider_signature)) = &job.result { - let events = decode_transcript(&job.transcript, MAX_RECORD_BYTES)?; - if terminal_result(&self.channel, &job.authorization, &events)? != *result { - return Err(ChannelStateError::WrongChannel { - field: "result against its transcript", - }); - } + self.check_result_payload(job, result, &job.transcript, metadata_only)?; if !verifier.verify_sig( *provider_signature, self.provider_key(), @@ -552,6 +588,28 @@ impl ChannelState { } } + /// Whether this record would apply as [`Applied::Redundant`], answered + /// without cloning the state. Only checks that stay cheap at the largest + /// legal state belong here; anything else falls through to the full + /// [`Self::apply`], which decides. + pub(super) fn is_redundant(&self, record: &ChannelRecord) -> bool { + match record { + // The stream replay path commits a release per emitted frame, + // and every one after the first is redundant. Detecting that + // here keeps a full-state clone — transcript included — off + // the per-frame hot path. The arms mirror apply_plaintext: + // any doubt falls through to the full apply and its refusal. + ChannelRecord::PlaintextReleased { work_id } => { + self.role == Role::Provider + && self + .jobs + .get(work_id) + .is_some_and(|job| job.phase.delivered()) + } + _ => false, + } + } + /// Applies one record, or says why it may not be applied. /// /// Every rule this endpoint has is here, and replay runs it too, so @@ -561,6 +619,7 @@ impl ChannelState { &mut self, record: &ChannelRecord, verifier: &V, + metadata_only: bool, ) -> Result { match record { ChannelRecord::CursorAdvanced { @@ -572,7 +631,13 @@ impl ChannelState { authorization, client_signature, prepared_input, - } => self.apply_proposed(authorization, *client_signature, prepared_input, verifier), + } => self.apply_proposed( + authorization, + *client_signature, + prepared_input, + verifier, + metadata_only, + ), ChannelRecord::JobAccepted { work_id, provider_signature, @@ -583,7 +648,14 @@ impl ChannelState { result, provider_signature, transcript, - } => self.apply_result(*work_id, result, *provider_signature, transcript, verifier), + } => self.apply_result( + *work_id, + result, + *provider_signature, + transcript, + verifier, + metadata_only, + ), ChannelRecord::PlaintextReleased { work_id } => self.apply_plaintext(*work_id), ChannelRecord::ResultMatched { work_id } => self.apply_matched(*work_id), ChannelRecord::JobTerminated { work_id, outcome } => { @@ -873,6 +945,7 @@ impl ChannelState { &self, authorization: &PaidJobAuthorizationV1, prepared_input: &[u8], + metadata_only: bool, ) -> Result<(), ChannelStateError> { let terms = self.channel.payment_terms(); for (field, holds) in [ @@ -902,9 +975,16 @@ impl ChannelState { // The inputs this job will be executed from, against the digest // the authorization both parties sign commits to. A bundle that // does not hash to it is a job neither party agreed to run. - let bundle = PreparedPaidInputV1::decode(prepared_input, MAX_RECORD_BYTES) + if metadata_only { + return if prepared_input.is_empty() { + Ok(()) + } else { + Err(ChannelStateError::Malformed) + }; + } + let bundle = PreparedPaidWorkInput::decode(prepared_input, MAX_RECORD_BYTES) .map_err(PaidWorkError::from)?; - if prepared_input_digest(&self.channel, &bundle)?.as_bytes() + if bundle.digest(&self.channel)?.as_bytes() != authorization.prepared_input_digest.as_bytes() { return Err(ChannelStateError::Record(PaidWorkError::Mismatch { @@ -958,6 +1038,7 @@ impl ChannelState { client_signature: Sig, prepared_input: &[u8], verifier: &V, + metadata_only: bool, ) -> Result { let work_id = work_id(&self.channel, authorization); if let Some(job) = self.jobs.get(&work_id) { @@ -978,7 +1059,7 @@ impl ChannelState { // whose payment no close could carry. self.refuse_if_closing("proposing a job")?; - self.check_authorization(authorization, prepared_input)?; + self.check_authorization(authorization, prepared_input, metadata_only)?; if authorization.proposal_nonce <= self.proposal_nonce_high_water { return Err(ChannelStateError::WrongChannel { field: "proposal_nonce", @@ -1088,7 +1169,7 @@ impl ChannelState { /// summarises. /// /// The rule that makes this more than a signature check is the - /// reproduction below: [`terminal_result`] is handed the stored + /// reproduction below: [`hellas_rpc::protocol::work::terminal_result`] is handed the stored /// events and this job's own authorization, and what it builds must /// be the result byte for byte. That establishes, on commit and on /// every replay, that the events are one verified signed chain for @@ -1106,6 +1187,7 @@ impl ChannelState { provider_signature: Sig, transcript: &[u8], verifier: &V, + metadata_only: bool, ) -> Result { let mut job = self.open_job(work_id, "recording a result")?; if let Some((held, signature)) = &job.result { @@ -1159,12 +1241,7 @@ impl ChannelState { }); } - let events = decode_transcript(transcript, MAX_RECORD_BYTES)?; - if terminal_result(&self.channel, &job.authorization, &events)? != *result { - return Err(ChannelStateError::WrongChannel { - field: "result against its transcript", - }); - } + self.check_result_payload(&job, result, transcript, metadata_only)?; if !verifier.verify_sig( provider_signature, self.provider_key(), diff --git a/crates/work/src/work_store/channel/store.rs b/crates/work/src/work_store/channel/store.rs index 7acb2d3b..e871d525 100644 --- a/crates/work/src/work_store/channel/store.rs +++ b/crates/work/src/work_store/channel/store.rs @@ -7,6 +7,7 @@ pub struct ChannelStore { journal: Journal, state: ChannelState, torn_tail: bool, + metadata_only: bool, } impl ChannelStore { @@ -45,6 +46,36 @@ impl ChannelStore { role: Role, origin: SetupOrigin, verifier: &V, + ) -> Result { + Self::open_inner(root, channel, settlement, role, origin, verifier, false) + } + + /// Opens a journal that stores accounting evidence only. + /// + /// Inputs and output transcripts are verified before commit and retained + /// in memory. Append and checkpoint encodings omit both bodies. After a + /// restart an accepted request cannot run and a completed response cannot + /// be replayed; a signed payment for an already delivered result can still + /// be credited. Existing payload-bearing journals are refused. + pub fn open_metadata_only( + root: &Path, + channel: PaidChannel, + settlement: WorkPaymentSettlement, + role: Role, + origin: SetupOrigin, + verifier: &V, + ) -> Result { + Self::open_inner(root, channel, settlement, role, origin, verifier, true) + } + + fn open_inner( + root: &Path, + channel: PaidChannel, + settlement: WorkPaymentSettlement, + role: Role, + origin: SetupOrigin, + verifier: &V, + metadata_only: bool, ) -> Result { if origin.payment_edge != channel.payment_edge() { return Err(ChannelStateError::WrongChannel { @@ -58,21 +89,30 @@ impl ChannelStore { root, &format!("channel-{}", hex(&key)), JournalId { - kind: JournalKind::Channel, + kind: if metadata_only { + JournalKind::MetadataChannel + } else { + JournalKind::Channel + }, role, key, generation: 0, }, )?; let mut state = match &replay.checkpoint { - Some(bytes) => { - ChannelState::from_checkpoint(bytes, channel, settlement, role, verifier)? - } + Some(bytes) => ChannelState::from_checkpoint( + bytes, + channel, + settlement, + role, + verifier, + metadata_only, + )?, None => ChannelState::new(channel, settlement, role, origin), }; for bytes in &replay.records { let record = ChannelRecord::decode(bytes)?; - state.apply(&record, verifier)?; + state.apply(&record, verifier, metadata_only)?; } state.indeterminate = state .jobs @@ -85,6 +125,7 @@ impl ChannelStore { journal, state, torn_tail: replay.truncated_tail, + metadata_only, }; Ok(store) } @@ -107,6 +148,12 @@ impl ChannelStore { &self.state } + /// Whether this journal excludes request and response bodies from disk. + #[must_use] + pub const fn metadata_only(&self) -> bool { + self.metadata_only + } + /// Journals one step, and returns only once it is on the disk. /// /// The rule this exists to enforce: call it *before* the bytes it @@ -126,9 +173,19 @@ impl ChannelStore { verifier: &V, ) -> Result<&ChannelState, WorkStoreError> { // Applied to a copy first: a record the rules refuse must leave - // neither the file nor the state touched. + // neither the file nor the state touched. A cheap exact check + // first: the stream replay path offers a redundant release per + // emitted frame, and the copy would clone the whole transcript. + if self.state.is_redundant(&record) { + return Ok(&self.state); + } let mut next = self.state.clone(); - if next.apply(&record, verifier)? == Applied::Changed { + // The `false` keeps commit-time validation at full strength even + // for a metadata-only journal: bodies are verified — digest and + // transcript-to-result reproduction — before they are stripped + // from the bytes written. Only replay and checkpoint validation + // downgrade for body-less records. + if next.apply(&record, verifier, false)? == Applied::Changed { // The signature this record carries leaves after this // returns, so the state that authorises it has to be one a // rotation can still carry. The proposed job is charged the @@ -136,13 +193,17 @@ impl ChannelStore { // answered with and the terminal that pays for it — because // by then there is no refusal left that costs nothing. if let Some(tail) = record.checkpoint_tail() { - let len = next.checkpoint().len().saturating_add(tail); + let len = next + .checkpoint_for_storage(self.metadata_only) + .len() + .saturating_add(tail); if len > MAX_CHECKPOINT_BYTES { return Err(JournalError::CheckpointTooLarge { len }.into()); } } self.rotate_if_full(record.is_new_work())?; - self.journal.append(&record.encode())?; + self.journal + .append(&record.encode_for_storage(self.metadata_only))?; self.state = next; } Ok(&self.state) @@ -162,7 +223,8 @@ impl ChannelStore { /// [`WorkStoreError::Journal`] when the checkpoint does not fit one /// frame or an install step fails. pub fn rotate(&mut self) -> Result<(), WorkStoreError> { - self.journal.rotate(&self.state.checkpoint())?; + self.journal + .rotate(&self.state.checkpoint_for_storage(self.metadata_only))?; Ok(()) } @@ -178,7 +240,10 @@ impl ChannelStore { if !self.journal.at_soft_limit() { return Ok(()); } - match self.journal.rotate(&self.state.checkpoint()) { + match self + .journal + .rotate(&self.state.checkpoint_for_storage(self.metadata_only)) + { Ok(()) => Ok(()), Err(error) if new_work => Err(error.into()), Err(_) => Ok(()), diff --git a/crates/work/src/work_store/journal.rs b/crates/work/src/work_store/journal.rs index e6012001..17a57096 100644 --- a/crates/work/src/work_store/journal.rs +++ b/crates/work/src/work_store/journal.rs @@ -85,6 +85,9 @@ pub enum JournalKind { Setup, /// One channel's job, credit, and certificate. Channel, + /// Provider accounting whose request and response bodies never reach disk. + /// A different kind prevents reopening a payload-bearing journal as ZDR. + MetadataChannel, } impl JournalKind { @@ -92,6 +95,7 @@ impl JournalKind { match self { Self::Setup => 1, Self::Channel => 2, + Self::MetadataChannel => 3, } } @@ -99,6 +103,7 @@ impl JournalKind { match code { 1 => Some(Self::Setup), 2 => Some(Self::Channel), + 3 => Some(Self::MetadataChannel), _ => None, } } diff --git a/crates/work/tests/work_accept.rs b/crates/work/tests/work_accept.rs index 586601f9..d4a4ffd2 100644 --- a/crates/work/tests/work_accept.rs +++ b/crates/work/tests/work_accept.rs @@ -144,7 +144,7 @@ fn descriptor_with_policy(policy: PaidExecutionPolicyV1) -> WorkChannelDescripto payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: policy, + execution_policy: policy.into(), expected_payment_values: payment_values(), }; match WorkChannelDescriptor::open(config) { @@ -312,7 +312,7 @@ const fn deadlines() -> JobDeadlines { fn proposal(nonce: u8) -> JobProposal { JobProposal { - prepared_input: bundle(nonce), + prepared_input: bundle(nonce).into(), deadlines: deadlines(), } } @@ -1087,7 +1087,7 @@ fn a_proposal_refused_before_signing_leaves_no_job() { // Terminal deadline inside the measured dispatch and delivery // margins: legal, ordered, and unreachable. let unreachable = JobProposal { - prepared_input: bundle(1), + prepared_input: bundle(1).into(), deadlines: JobDeadlines { acceptance: 11, terminal: 12, @@ -1987,3 +1987,144 @@ fn the_wire_carries_the_authorization_the_signature_covers() { "the signature on the wire verifies over the work id", ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn client_observer_does_not_hold_the_journal_during_chain_io() { + use hellas_work::work::{ClientChannel as _, ClientService}; + let root = temp(); + let mut service = ClientService::new(client_endpoint(root.path())); + service + .observer() + .observe_ready( + ready(), + hellas_work::work::ObservationTime::now(), + std::time::Duration::from_secs(30), + ) + .unwrap(); + let released = Arc::new(AtomicBool::new(false)); + let source = HeldChain::over(CURSOR + 1..=CURSOR + 1, &released); + let observer = service.observer(); + let held_source = source.clone(); + let driving = + tokio::spawn(async move { observer.drive().unwrap().catch_up(&*held_source).await }); + reaches(&source.fetches, 1).await; + assert!(matches!( + service.observer().drive(), + Err(EndpointError::CatchingUp) + )); + // The provider exchange can sign and journal while the observer waits. + let request = service + .with_client(|endpoint| endpoint.propose(&proposal(1))) + .unwrap() + .unwrap(); + assert!(!request.client_signature.is_empty()); + assert_eq!( + service + .with_state(|state| state.proposal_nonce_high_water()) + .unwrap(), + 1 + ); + released.store(true, Ordering::SeqCst); + assert_eq!(driving.await.unwrap().unwrap(), CURSOR + 1); + assert!(service.observer().drive().is_ok()); +} + +#[tokio::test] +async fn shared_client_can_follow_blocks_while_waiting_for_provider_acceptance() { + use hellas_work::work::ClientService; + let client_root = temp(); + let provider_root = temp(); + let mut client = ClientService::new(client_endpoint(client_root.path())); + client + .observer() + .observe_ready( + ready(), + hellas_work::work::ObservationTime::now(), + std::time::Duration::from_secs(30), + ) + .unwrap(); + let observer = client.observer(); + let (client_transport, server_transport) = transport_pair(); + let request = + tokio::spawn( + async move { propose_work(client_transport, &mut client, &proposal(1)).await }, + ); + // No server is running yet, so the request must wait for its response. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while observer + .with_state(|state| state.proposal_nonce_high_water()) + .unwrap() + == 0 + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + observer + .drive() + .unwrap() + .observe_finalized(&FinalizedWork { + height: CURSOR + 1, + parent: payload_at(CURSOR), + payload: payload_at(CURSOR + 1), + txs: Vec::new(), + }) + .unwrap(); + let serving = serve( + server_transport, + WorkService::new(provider_endpoint(provider_root.path())), + ); + request.await.unwrap().unwrap(); + assert_eq!( + observer.with_state(|state| state.cursor().0).unwrap(), + CURSOR + 1 + ); + serving.abort(); +} + +#[test] +fn managed_observers_start_closed_and_old_tips_cannot_renew_expired_readiness() { + use hellas_work::work::{ClientChannel as _, ClientService}; + let client_root = temp(); + let provider_root = temp(); + let mut client = ClientService::new(client_endpoint(client_root.path())); + let provider = WorkService::new(provider_endpoint(provider_root.path())); + provider.require_observer().unwrap(); + assert!(matches!( + client.readiness(), + Err(EndpointError::ObservationStale) + )); + assert_eq!( + refusal_code(&provider.accept(&signed_request(1, 1))), + WorkRefusalCode::NotReady + ); + for max_age in [ + std::time::Duration::ZERO, + std::time::Duration::from_secs(30), + ] { + let started = hellas_work::work::ObservationTime::now(); + client + .observer() + .observe_ready(ready(), started, max_age) + .unwrap(); + provider.observe_ready(ready(), started, max_age).unwrap(); + assert!(matches!( + client + .with_client(|endpoint| endpoint.propose(&proposal(1))) + .unwrap(), + Err(ProposeError::Endpoint(EndpointError::ObservationStale)) + )); + assert_eq!( + refusal_code(&provider.accept(&signed_request(1, 1))), + WorkRefusalCode::NotReady + ); + } + assert_eq!( + client + .with_state(|state| state.proposal_nonce_high_water()) + .unwrap(), + 0 + ); + assert_eq!(provider.with_state(|state| state.jobs().len()).unwrap(), 0); +} diff --git a/crates/work/tests/work_close.rs b/crates/work/tests/work_close.rs index 60fed43d..4a5cd7d7 100644 --- a/crates/work/tests/work_close.rs +++ b/crates/work/tests/work_close.rs @@ -53,7 +53,7 @@ use hellas_rpc::{ use hellas_wire::mux::MuxTransport; use hellas_wire::{Dispatcher, StreamTransport}; use hellas_work::work::{ - BackendFault, ClientEndpoint, CloseEndpoint, PaidEvaluateBackend, PaymentError, + BackendFault, ClientEndpoint, CloseEndpoint, PaidWorkBackend, PaymentError, PreparedEvaluateInput, ProviderEndpoint, RunOutcome, WorkService, admit_payment, fetch_result, run_accepted_work, }; @@ -175,7 +175,7 @@ fn descriptor() -> WorkChannelDescriptor { payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: payment_values(), }; match WorkChannelDescriptor::open(config) { @@ -386,7 +386,7 @@ impl AnsweringBackend { } } -impl PaidEvaluateBackend for AnsweringBackend { +impl PaidWorkBackend for AnsweringBackend { fn evaluate( &self, input: PreparedEvaluateInput, diff --git a/crates/work/tests/work_delivery.rs b/crates/work/tests/work_delivery.rs index 1508f10e..7a47d5f5 100644 --- a/crates/work/tests/work_delivery.rs +++ b/crates/work/tests/work_delivery.rs @@ -47,7 +47,7 @@ use hellas_rpc::{ use hellas_wire::mux::{MessagePipe, MuxConfig, MuxTransport, Role as MuxRole}; use hellas_wire::{DefaultClock, Dispatcher, StreamTransport}; use hellas_work::work::{ - BackendFault, ClientEndpoint, CloseEndpoint, DeliverError, PaidEvaluateBackend, + BackendFault, ClientEndpoint, CloseEndpoint, DeliverError, PaidWorkBackend, PreparedEvaluateInput, ProviderEndpoint, RunError, RunOutcome, WorkService, fetch_result, run_accepted_work, }; @@ -171,7 +171,7 @@ fn descriptor_with(policy: PaidExecutionPolicyV1) -> WorkChannelDescriptor { payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: policy, + execution_policy: policy.into(), expected_payment_values: payment_values(), }; match WorkChannelDescriptor::open(config) { @@ -392,7 +392,7 @@ impl AnsweringBackend { } } -impl PaidEvaluateBackend for AnsweringBackend { +impl PaidWorkBackend for AnsweringBackend { fn evaluate( &self, input: PreparedEvaluateInput, @@ -719,7 +719,7 @@ async fn a_job_with_no_result_releases_nothing_yet() { #[tokio::test] async fn a_failed_job_returns_its_terminal_after_authentication() { struct FailingBackend; - impl PaidEvaluateBackend for FailingBackend { + impl PaidWorkBackend for FailingBackend { async fn evaluate( &self, _input: PreparedEvaluateInput, @@ -1463,7 +1463,7 @@ async fn live_prefix_precedes_terminal_and_reserves_delivery_credit() { struct PausedBackend { release: Arc, } - impl PaidEvaluateBackend for PausedBackend { + impl PaidWorkBackend for PausedBackend { async fn evaluate( &self, input: PreparedEvaluateInput, diff --git a/crates/work/tests/work_handshake.rs b/crates/work/tests/work_handshake.rs index 908e7071..46d88547 100644 --- a/crates/work/tests/work_handshake.rs +++ b/crates/work/tests/work_handshake.rs @@ -185,7 +185,7 @@ fn provider_policy() -> ProviderChannelPolicy { network: network(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: EdgeValues::new( PAYMENT_VALUE, PAYMENT_RESERVE, diff --git a/crates/work/tests/work_payment.rs b/crates/work/tests/work_payment.rs index f58fee64..13007859 100644 --- a/crates/work/tests/work_payment.rs +++ b/crates/work/tests/work_payment.rs @@ -53,7 +53,7 @@ use hellas_rpc::{ use hellas_wire::mux::MuxTransport; use hellas_wire::{Dispatcher, StreamTransport}; use hellas_work::work::{ - BackendFault, ClientEndpoint, PaidEvaluateBackend, PaymentError, PreparedEvaluateInput, + BackendFault, ClientEndpoint, PaidWorkBackend, PaymentError, PreparedEvaluateInput, ProviderEndpoint, RunError, RunOutcome, WorkRefusal, WorkService, admit_payment, fetch_result, run_accepted_work, }; @@ -167,7 +167,7 @@ fn descriptor() -> WorkChannelDescriptor { payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: payment_values(), }; match WorkChannelDescriptor::open(config) { @@ -378,7 +378,7 @@ impl AnsweringBackend { } } -impl PaidEvaluateBackend for AnsweringBackend { +impl PaidWorkBackend for AnsweringBackend { fn evaluate( &self, input: PreparedEvaluateInput, @@ -1160,3 +1160,157 @@ async fn a_defaulted_job_is_not_paid_for_as_well() { PRICE, ); } + +#[tokio::test] +async fn stale_observation_blocks_new_money_but_preserves_retransmission() { + use hellas_work::work::{ClientChannel as _, ClientService, EndpointError}; + let fixture = checked_job().await; + let mut client = ClientService::new(fixture.client); + let id = fixture.id; + assert!(matches!( + client.with_client(|endpoint| endpoint.pay(id)).unwrap(), + Err(PaymentError::Endpoint(EndpointError::ObservationStale)) + )); + assert_eq!( + client + .with_state(|state| state.ledger().credited_cumulative()) + .unwrap(), + 0 + ); + client + .observer() + .observe_ready( + fixture.ready.clone(), + hellas_work::work::ObservationTime::now(), + std::time::Duration::from_secs(30), + ) + .unwrap(); + let request = client + .with_client(|endpoint| endpoint.pay(id)) + .unwrap() + .unwrap(); + client.observer().suspend().unwrap(); + assert_eq!( + client + .with_client(|endpoint| endpoint.pay(id)) + .unwrap() + .unwrap(), + request + ); + + fixture.service.require_observer().unwrap(); + let refused = admit_response(&fixture.service, request.clone()).await; + assert_eq!(refusal_of(&refused), WorkRefusalCode::NotReady); + assert_nothing_credited(&fixture.service, "observer has not initialized").await; + fixture + .service + .observe_ready( + fixture.ready, + hellas_work::work::ObservationTime::now(), + std::time::Duration::from_secs(30), + ) + .unwrap(); + let accepted = admit_response(&fixture.service, request.clone()).await; + assert!(matches!(accepted.outcome, Some(AdmitOutcome::Paid(_)))); + fixture.service.suspend().unwrap(); + assert_eq!(admit_response(&fixture.service, request).await, accepted); + assert_eq!( + fixture + .service + .with_state(|state| state.ledger().credited_cumulative()) + .unwrap(), + PRICE + ); +} + +#[tokio::test] +async fn client_close_recovery_does_not_require_new_work_readiness() { + let mut fixture = checked_job().await; + fixture.client.pay(fixture.id).unwrap(); + let start = fixture.client.prepare_close().unwrap(); + drop(fixture.client); + let recovered_store = store_at( + fixture.client_root.path(), + &fixture.ready, + Role::Client, + CURSOR, + ); + let mut recovered = ClientEndpoint::recover(recovered_store, client()).unwrap(); + assert!(recovered.state().is_closing()); + assert_eq!(recovered.prepare_close().unwrap(), start); + assert_eq!(recovered.state().max_executable_certificate(), PRICE); +} + +#[tokio::test] +async fn observed_close_and_payment_have_one_journal_order() { + use hellas_work::work::{ClientChannel as _, ClientService, ObservationTime}; + use hellas_work::work_close::close_start; + for payment_first in [false, true] { + let fixture = checked_job().await; + let mut client = ClientService::new(fixture.client); + let observer = client.observer(); + observer + .observe_ready( + fixture.ready.clone(), + ObservationTime::now(), + std::time::Duration::from_secs(30), + ) + .unwrap(); + let start = close_start( + fixture.ready.channel(), + hellas_kernel::Party::Maker, + CURSOR, + None, + &self::client(), + ) + .unwrap(); + let block = FinalizedWork { + height: CURSOR + 1, + parent: payload_at(CURSOR), + payload: payload_at(CURSOR + 1), + txs: vec![hellas_kernel::Tx::move_action( + hellas_kernel::Move::StartPaymentClose(start), + )], + }; + if payment_first { + let request = client + .with_client(|endpoint| endpoint.pay(fixture.id)) + .unwrap() + .unwrap(); + let credited = admit_response(&fixture.service, request).await; + assert!(matches!(credited.outcome, Some(AdmitOutcome::Paid(_)))); + } + observer.drive().unwrap().observe_finalized(&block).unwrap(); + fixture + .service + .drive() + .unwrap() + .observe_finalized(&block) + .unwrap(); + assert!(observer.with_state(|state| state.is_closing()).unwrap()); + let retried = client + .with_client(|endpoint| endpoint.pay(fixture.id)) + .unwrap(); + if payment_first { + // Retrying the durable certificate is allowed after close observation. + let response = admit_response(&fixture.service, retried.unwrap()).await; + assert!(matches!(response.outcome, Some(AdmitOutcome::Paid(_)))); + } else { + assert!(matches!(retried, Err(PaymentError::NoSuchJob))); + } + let expected = if payment_first { PRICE } else { 0 }; + assert_eq!( + client + .with_state(|state| state.max_executable_certificate()) + .unwrap(), + expected + ); + assert_eq!( + fixture + .service + .with_state(|state| state.max_executable_certificate()) + .unwrap(), + expected + ); + } +} diff --git a/crates/work/tests/work_run.rs b/crates/work/tests/work_run.rs index e7fcd3a4..08efe7f2 100644 --- a/crates/work/tests/work_run.rs +++ b/crates/work/tests/work_run.rs @@ -39,14 +39,16 @@ use hellas_rpc::{ OutputEventEnvelope, ProducerSigningKey, ProgramManifest, PublicKey, }; use hellas_work::work::{ - BackendFault, PaidEvaluateBackend, PreparedEvaluateInput, ProviderEndpoint, RunAdmission, - RunError, RunOutcome, WorkService, run_accepted_work, + BackendFault, PaidWorkBackend, PreparedEvaluateInput, ProviderEndpoint, RunAdmission, RunError, + RunOutcome, WorkService, run_accepted_work, }; use hellas_work::work_store::{ ChannelRecord, ChannelStateError, ChannelStore, JobPhase, JobState, Role, SetupOrigin, TerminalOutcome, WorkStoreError, }; +#[path = "work_run/fetch.rs"] +mod fetch; mod support; use support::{advance, bond_edge, client, network, payload_at, payment_edge, provider, temp}; @@ -163,14 +165,16 @@ fn payment_values() -> EdgeValues { EdgeValues::new(PAYMENT_VALUE, PAYMENT_RESERVE, Fees::new(0, 0, 0, 0)) } -fn descriptor_with(policy: PaidExecutionPolicyV1) -> WorkChannelDescriptor { +fn descriptor_with( + policy: impl Into, +) -> WorkChannelDescriptor { let config = WorkChannelConfig { network: network(), payment_edge: payment_edge(), payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: policy, + execution_policy: policy.into(), expected_payment_values: payment_values(), }; match WorkChannelDescriptor::open(config) { @@ -452,7 +456,7 @@ impl CountingBackend { } } -impl PaidEvaluateBackend for CountingBackend { +impl PaidWorkBackend for CountingBackend { fn evaluate( &self, input: PreparedEvaluateInput, diff --git a/crates/work/tests/work_run/fetch.rs b/crates/work/tests/work_run/fetch.rs new file mode 100644 index 00000000..0d1b573f --- /dev/null +++ b/crates/work/tests/work_run/fetch.rs @@ -0,0 +1,678 @@ +use super::*; +use hellas_rpc::fetch::{ + FetchOutputTranscriptBuilder, build_input_events_with_retention, encode_fetch_event_payload, + encode_fetch_terminal_payload, verify_input_events, +}; +use hellas_rpc::output::{OutputEvent, StopReason, TextChannel}; +use hellas_rpc::pb::work::WorkDelivered; +use hellas_rpc::protocol::work::PrivateRecord as _; +use hellas_rpc::protocol::work_fetch::{ + FetchRoutePolicy, PaidFetchPolicyV1, PreparedPaidFetchInputV1, fetch_route_commitment, +}; +use hellas_rpc::protocol::work_profile::PaidWorkPolicy; +use hellas_rpc::{FetchEnvironment, Retention}; +use hellas_work::work::{ + ClientEndpoint, DeliverError, EndpointError, JobProposal, PreparedFetchInput, +}; + +const PROMPT: &[u8] = br#"{"input":"private-customer-prompt-never-on-provider-disk"}"#; +const RESPONSE: &str = "private-provider-response-never-on-provider-disk"; + +#[path = "../support/transport.rs"] +mod transport; + +#[tokio::test] +async fn fetch_streams_before_terminal_then_pays_once_with_metadata_only_journals() { + use hellas_rpc::services::work::{WorkClientImpl, WorkServer}; + use hellas_wire::{Dispatcher, StreamTransport, mux::MuxTransport}; + use hellas_work::work::{PaidProgress, admit_payment, fetch_result_stream}; + + struct PausedFetch(Arc); + impl PaidWorkBackend for PausedFetch { + async fn fetch_stream( + &self, + input: PreparedFetchInput, + progress: PaidProgress, + ) -> Result, BackendFault> { + let events = FetchBackend::default().fetch(input).await?; + progress(events[0].clone())?; + self.0.notified().await; + Ok(events) + } + } + for assurance in [Assurance::ProducerSigned, Assurance::AppleAppAttest] { + let provider_dir = temp(); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let service = service(provider_dir.path()); + let proposal = JobProposal { + prepared_input: input_with_assurance(Retention::Ephemeral, assurance).into(), + deadlines: deadlines(), + }; + let request = client.propose(&proposal).unwrap(); + let id = client.accepted(&service.accept(&request)).unwrap(); + let release = Arc::new(tokio::sync::Notify::new()); + let worker = { + let service = service.clone(); + let backend = PausedFetch(release.clone()); + tokio::spawn( + async move { run_accepted_work(&service, &fetch_ready(), &backend, id).await }, + ) + }; + let (transport, server) = transport::transport_pair(); + let serving = { + let service = service.clone(); + tokio::spawn(async move { + let handler = WorkServer(service); + while let Ok(Some(inbound)) = server.accept().await { + let _ = Dispatcher::::dispatch(&handler, inbound).await; + } + }) + }; + let mut prefixes = 0; + tokio::time::timeout( + std::time::Duration::from_secs(5), + fetch_result_stream( + transport.clone(), + &mut client, + &fetch_ready(), + id, + |event| { + assert!(!worker.is_finished()); + assert_eq!( + service + .with_state(|s| s.job_by_id(id).unwrap().phase()) + .unwrap(), + JobPhase::Streaming + ); + assert!(hellas_rpc::fetch::decode_fetch_event_payload(event.payload()).is_ok()); + prefixes += 1; + release.notify_one(); + Ok(()) + }, + ), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(prefixes, 1); + assert!(matches!( + worker.await.unwrap().unwrap(), + RunOutcome::Completed { .. } + )); + let rpc = WorkClientImpl::new(transport); + assert_eq!(admit_payment(&rpc, &mut client, id).await.unwrap(), PRICE); + assert_eq!(admit_payment(&rpc, &mut client, id).await.unwrap(), PRICE); + assert_no_bodies(client_dir.path()); + assert_no_bodies(provider_dir.path()); + serving.abort(); + let _ = serving.await; + } +} + +fn policy() -> PaidWorkPolicy { + let route = FetchRoutePolicy::sealed_route("openai", "responses").unwrap(); + PaidWorkPolicy::Fetch { + policy: PaidFetchPolicyV1 { + allowed_environment: FetchEnvironment::OpenAiResponses.manifest_id(), + route_commitment: fetch_route_commitment(&route.canonical_body_bytes()).unwrap(), + max_request_body_bytes: 4096, + max_output_events: 64, + max_output_bytes: 16384, + max_spool_bytes: 65536, + max_encoded_result_frame: 65536, + max_encoded_prepared_input: 65536, + dispatch_margin_blocks: 4, + delivery_margin_blocks: 2, + oracle_grace_blocks: 6, + fixed_price: PRICE, + }, + route, + } +} + +fn fetch_ready() -> ReadyChannel { + ready_of(descriptor_with(policy()), CURSOR) +} + +#[tokio::test] +async fn invalid_paid_authorizations_never_invoke_fetch() { + for invalid in ["signature", "price", "deadline"] { + let provider_dir = temp(); + let client_dir = temp(); + let service = service(provider_dir.path()); + let mut client = client_endpoint(client_dir.path()); + let mut request = client.propose(&proposal()).unwrap(); + let mut authorization = PaidJobAuthorizationV1::decode(&request.authorization).unwrap(); + match invalid { + "price" => authorization.price += 1, + "deadline" => authorization.acceptance_deadline = CURSOR - 1, + _ => {} + } + let id = work_id(fetch_ready().channel(), &authorization); + request.authorization = authorization.encode(); + request.client_signature = super::client().sign(signing_hash(id)).as_bytes().to_vec(); + if invalid == "signature" { + request.client_signature[0] ^= 1; + } + assert!(matches!( + service.accept(&request).outcome, + Some(hellas_rpc::pb::work::accept_work_response::Outcome::Refused(_)) + )); + let backend = FetchBackend::default(); + assert!(matches!( + run_accepted_work(&service, &fetch_ready(), &backend, id).await, + Err(RunError::NoSuchJob) + )); + assert_eq!(backend.0.load(Ordering::SeqCst), 0); + assert_no_bodies(provider_dir.path()); + assert_no_bodies(client_dir.path()); + } +} + +fn input(retention: Retention) -> PreparedPaidFetchInputV1 { + input_with_assurance(retention, Assurance::ProducerSigned) +} +fn input_with_assurance(retention: Retention, assurance: Assurance) -> PreparedPaidFetchInputV1 { + let environment = FetchEnvironment::OpenAiResponses; + let events = build_input_events_with_retention( + "openai", + "responses", + PROMPT, + environment.manifest_id(), + assurance, + &producer(0x21), + retention, + ) + .unwrap(); + PreparedPaidFetchInputV1::new(&events, &environment.manifest()).unwrap() +} + +fn proposal() -> JobProposal { + JobProposal { + prepared_input: input(Retention::Ephemeral).into(), + deadlines: deadlines(), + } +} + +fn metadata_store(root: &std::path::Path) -> ChannelStore { + store_for(root, &fetch_ready(), Role::Provider) +} + +fn store_for(root: &std::path::Path, ready: &ReadyChannel, role: Role) -> ChannelStore { + let mut store = ChannelStore::open_metadata_only( + root, + ready.channel().clone(), + settlement(), + role, + origin(), + &Secp256k1Verifier::new(), + ) + .unwrap(); + advance(&mut store, CURSOR); + store +} + +fn service(root: &std::path::Path) -> WorkService { + WorkService::new( + ProviderEndpoint::new(fetch_ready(), metadata_store(root), provider()).unwrap(), + ) +} + +fn client_endpoint(root: &std::path::Path) -> ClientEndpoint { + let store = store_for(root, &fetch_ready(), Role::Client); + ClientEndpoint::new(fetch_ready(), store, client()).unwrap() +} + +#[tokio::test] +async fn paid_fetch_larger_than_a_wire_frame_streams_and_pays_without_disk_payloads() { + use hellas_rpc::services::work::{WorkClientImpl, WorkServer}; + use hellas_wire::{Dispatcher, StreamTransport, mux::MuxTransport}; + use hellas_work::work::{admit_payment, fetch_result_stream}; + struct LargeFetch; + impl PaidWorkBackend for LargeFetch { + async fn fetch( + &self, + input: PreparedFetchInput, + ) -> Result, BackendFault> { + let request = verify_input_events(&input.into_parts().fetch_input_transcript).unwrap(); + let key = provider_producer(); + let mut transcript = FetchOutputTranscriptBuilder::new( + request.input_commitment, + request.assurance, + &key, + ); + for index in 0..256 { + transcript + .push_event( + encode_fetch_event_payload(&OutputEvent::TextDelta { + index, + delta: RESPONSE.repeat(400), + channel: TextChannel::Output, + }) + .unwrap(), + ) + .unwrap(); + } + Ok(transcript + .finish( + encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .unwrap(), + ) + .unwrap()) + } + } + let mut profile = policy(); + let PaidWorkPolicy::Fetch { policy, .. } = &mut profile else { + unreachable!() + }; + policy.max_output_events = 512; + policy.max_output_bytes = 8 << 20; + policy.max_spool_bytes = 16 << 20; + policy.max_encoded_result_frame = 64 << 10; + let ready = ready_of(descriptor_with(profile), CURSOR); + let client_dir = temp(); + let provider_dir = temp(); + let mut client = ClientEndpoint::new( + ready.clone(), + store_for(client_dir.path(), &ready, Role::Client), + super::client(), + ) + .unwrap(); + let service = WorkService::new( + ProviderEndpoint::new( + ready.clone(), + store_for(provider_dir.path(), &ready, Role::Provider), + provider(), + ) + .unwrap(), + ); + let request = client.propose(&proposal()).unwrap(); + let id = client.accepted(&service.accept(&request)).unwrap(); + run_accepted_work(&service, &ready, &LargeFetch, id) + .await + .unwrap(); + let (transport, server) = transport::transport_pair(); + let serving = tokio::spawn(async move { + let handler = WorkServer(service); + while let Ok(Some(inbound)) = server.accept().await { + let _ = Dispatcher::::dispatch(&handler, inbound).await; + } + }); + let mut received = 0; + let delivered = tokio::time::timeout( + std::time::Duration::from_secs(45), + fetch_result_stream(transport.clone(), &mut client, &ready, id, |_| { + received += 1; + Ok(()) + }), + ) + .await + .unwrap_or_else(|error| { + panic!( + "{error}: received {received} prefixes, server finished={}", + serving.is_finished() + ) + }) + .unwrap(); + assert_eq!(received, 256); + assert!(delivered.transcript.len() > hellas_wire::frame::MAX_FRAME_BYTES); + assert_eq!( + admit_payment(&WorkClientImpl::new(transport), &mut client, id) + .await + .unwrap(), + PRICE + ); + assert_no_bodies(client_dir.path()); + assert_no_bodies(provider_dir.path()); + serving.abort(); + let _ = serving.await; +} + +#[tokio::test] +async fn streamed_fetch_cannot_pay_without_a_complete_authenticated_terminal() { + use hellas_rpc::pb::work::{WorkStreamEvent, WorkStreamTerminal, work_stream_event::Outcome}; + use hellas_rpc::protocol::work::{decode_transcript, encode_transcript}; + use hellas_rpc::services::work::StreamResult; + use hellas_wire::{StreamTransport, WireCode, WireStatus, mux::MuxTransport}; + use hellas_work::work::fetch_result_stream; + for fault in ["missing-prefix", "signature", "duplicate", "status", "none"] { + let provider_dir = temp(); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let service = service(provider_dir.path()); + let request = client.propose(&proposal()).unwrap(); + let id = client.accepted(&service.accept(&request)).unwrap(); + run_accepted_work(&service, &fetch_ready(), &FetchBackend::default(), id) + .await + .unwrap(); + let exporter = [0x5e; 32]; + let delivered = service + .deliver(&client.request_delivery(id, &exporter).unwrap(), &exporter) + .unwrap(); + let events = decode_transcript(&delivered.transcript, 65536).unwrap(); + let mut terminal = WorkStreamTerminal { + result: delivered.result.encode(), + provider_signature: delivered.signature.as_bytes().to_vec(), + terminal_transcript: encode_transcript(&events[1..]).unwrap(), + }; + if fault == "signature" { + terminal.provider_signature[0] ^= 1; + } + let end = WorkStreamEvent { + outcome: Some(Outcome::Terminal(terminal)), + }; + let mut frames = Vec::new(); + if fault != "missing-prefix" { + frames.push(Ok(WorkStreamEvent { + outcome: Some(Outcome::Prefix(encode_transcript(&events[..1]).unwrap())), + })); + } + frames.push(Ok(end.clone())); + if fault == "duplicate" { + frames.push(Ok(end)); + } + if fault == "status" { + frames.push(Err(WireStatus::new(WireCode::Unavailable, "interrupted"))); + } + let (transport, server) = transport::transport_pair(); + let serving = tokio::spawn(async move { + let inbound = server.accept().await.unwrap().unwrap(); + hellas_rpc::call::dispatch_server_streaming::( + inbound, + move |_| async move { Ok(futures::stream::iter(frames)) }, + ) + .await + .unwrap(); + }); + let result = + fetch_result_stream(transport, &mut client, &fetch_ready(), id, |_| Ok(())).await; + serving.await.unwrap(); + if fault == "none" { + result.unwrap(); + assert!(client.pay(id).is_ok()); + } else { + assert!(result.is_err(), "{fault}"); + assert!( + client.pay(id).is_err(), + "{fault} must not create payable evidence" + ); + assert_eq!( + client.state().job_by_id(id).unwrap().phase(), + JobPhase::Accepted + ); + } + assert_no_bodies(client_dir.path()); + } +} + +#[derive(Default)] +struct FetchBackend(AtomicUsize); + +impl PaidWorkBackend for FetchBackend { + async fn fetch( + &self, + input: PreparedFetchInput, + ) -> Result, BackendFault> { + self.0.fetch_add(1, Ordering::SeqCst); + let parts = input.into_parts(); + let request = verify_input_events(&parts.fetch_input_transcript).unwrap(); + assert_eq!(request.body.as_bytes(), PROMPT); + assert_eq!(request.retention, Retention::Ephemeral); + let key = provider_producer(); + let mut builder = + FetchOutputTranscriptBuilder::new(request.input_commitment, request.assurance, &key); + builder + .push_event( + encode_fetch_event_payload(&OutputEvent::TextDelta { + index: 0, + delta: RESPONSE.into(), + channel: TextChannel::Output, + }) + .unwrap(), + ) + .unwrap(); + Ok(builder + .finish( + encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .unwrap(), + ) + .unwrap()) + } +} + +fn assert_no_bodies(root: &std::path::Path) { + for entry in std::fs::read_dir(root).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + assert_no_bodies(&path); + } else { + let bytes = std::fs::read(&path).unwrap(); + for body in [PROMPT, RESPONSE.as_bytes()] { + assert!( + !bytes.windows(body.len()).any(|window| window == body), + "payload in {}", + path.display() + ); + } + } + } +} + +#[tokio::test] +async fn fetch_is_run_delivered_and_paid_without_provider_disk_bodies() { + let provider_dir = temp(); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let service = service(provider_dir.path()); + let backend = FetchBackend::default(); + let request = client.propose(&proposal()).unwrap(); + let id = client.accepted(&service.accept(&request)).unwrap(); + assert_no_bodies(provider_dir.path()); + assert!(matches!( + run_accepted_work(&service, &fetch_ready(), &backend, id) + .await + .unwrap(), + RunOutcome::Completed { .. } + )); + assert_no_bodies(provider_dir.path()); + let request = client.request_delivery(id, &[8; 32]).unwrap(); + let delivered = service.deliver(&request, &[8; 32]).unwrap(); + client + .receive( + id, + &fetch_ready(), + &WorkDelivered { + result: delivered.result.encode(), + provider_signature: delivered.signature.as_bytes().to_vec(), + transcript: delivered.transcript, + }, + ) + .unwrap(); + assert_no_bodies(provider_dir.path()); + assert_no_bodies(client_dir.path()); + let payment = client.pay(id).unwrap(); + drop(client); + let mut client = client_endpoint(client_dir.path()); + assert_eq!(client.pay(id).unwrap(), payment); + assert_no_bodies(client_dir.path()); + + // A delivery already happened: restart loses bodies but can still bank the signed payment. + drop(service); + let store = metadata_store(provider_dir.path()); + assert!( + store + .state() + .job_by_id(id) + .unwrap() + .prepared_input() + .is_empty() + ); + assert!(store.state().job_by_id(id).unwrap().transcript().is_empty()); + let mut recovered = ProviderEndpoint::new(fetch_ready(), store, provider()).unwrap(); + assert!(matches!( + recovered.deliver(&request, &fetch_ready(), &[8; 32]), + Err(DeliverError::NoResult { .. }) + )); + assert_eq!(recovered.admit(&payment).unwrap(), PRICE); + assert_eq!(recovered.admit(&payment).unwrap(), PRICE); + assert_eq!(backend.0.load(Ordering::SeqCst), 1); + assert_no_bodies(provider_dir.path()); +} + +#[tokio::test] +async fn restarting_an_accepted_fetch_never_reinvokes_from_a_missing_payload() { + let provider_dir = temp(); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let service_before = service(provider_dir.path()); + let request = client.propose(&proposal()).unwrap(); + let id = client.accepted(&service_before.accept(&request)).unwrap(); + drop(service_before); + let backend = FetchBackend::default(); + let outcome = run_accepted_work(&service(provider_dir.path()), &fetch_ready(), &backend, id) + .await + .unwrap(); + assert_eq!(outcome, RunOutcome::Indeterminate); + assert_eq!(backend.0.load(Ordering::SeqCst), 0); + assert_no_bodies(provider_dir.path()); +} + +#[test] +fn fetch_refuses_payload_journals_and_retention_requests() { + let provider_dir = temp(); + assert!(matches!( + ProviderEndpoint::new( + fetch_ready(), + store_at(provider_dir.path(), CURSOR), + provider() + ), + Err(EndpointError::PayloadRetention) + )); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let retained = JobProposal { + prepared_input: input(Retention::Retain).into(), + deadlines: deadlines(), + }; + assert!(client.propose(&retained).is_err()); + + // A peer bypassing the client API cannot obtain a provider acceptance either. + let provider_dir = temp(); + let channel = fetch_ready(); + let authorization = policy() + .propose(channel.channel(), &retained.prepared_input, 1, deadlines()) + .unwrap(); + let id = work_id(channel.channel(), &authorization); + let request = hellas_rpc::pb::work::AcceptWorkRequest { + authorization: authorization.encode(), + client_signature: super::client().sign(signing_hash(id)).as_bytes().to_vec(), + prepared_input: retained.prepared_input.encode().unwrap(), + }; + assert!(matches!( + service(provider_dir.path()).accept(&request).outcome, + Some(hellas_rpc::pb::work::accept_work_response::Outcome::Refused(_)) + )); + assert_no_bodies(provider_dir.path()); +} + +#[test] +fn fetch_close_descriptor_round_trips_and_rejects_profile_version_confusion() { + use hellas_rpc::protocol::work_setup::CloseDescriptor; + let descriptor = descriptor_with(policy()).close_descriptor(); + let bytes = descriptor.encode(); + assert_eq!(CloseDescriptor::decode(&bytes).unwrap(), descriptor); + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(CloseDescriptor::decode(&trailing).is_err()); + let mut wrong_version = bytes; + wrong_version[0] = 1; + assert!(CloseDescriptor::decode(&wrong_version).is_err()); +} + +#[tokio::test] +async fn fetch_output_limits_include_terminal_and_bad_results_are_never_payable() { + let provider_dir = temp(); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let service = service(provider_dir.path()); + let request = client.propose(&proposal()).unwrap(); + let id = client.accepted(&service.accept(&request)).unwrap(); + let RunAdmission::InvokeFetch(input) = service.begin_run(id, &fetch_ready()).unwrap() else { + panic!("first Fetch dispatch"); + }; + let transcript = FetchBackend::default().fetch(*input).await.unwrap(); + let PaidWorkPolicy::Fetch { mut policy, .. } = policy() else { + unreachable!() + }; + policy.max_output_events = u32::try_from(transcript.len() - 1).unwrap(); + assert!( + hellas_rpc::protocol::work_fetch::check_fetch_output_limits(&policy, &transcript).is_err() + ); + policy.max_output_events += 1; + policy.max_output_bytes = 1; + assert!( + hellas_rpc::protocol::work_fetch::check_fetch_output_limits(&policy, &transcript).is_err() + ); + assert!( + service + .record_result(id, &transcript[..transcript.len() - 1]) + .is_err() + ); + assert!(client.pay(id).is_err()); + assert_no_bodies(provider_dir.path()); +} + +#[tokio::test] +async fn app_attest_paid_fetch_uses_the_signed_scheme_without_disk_payloads() { + let provider_dir = temp(); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let service = service(provider_dir.path()); + let proposal = JobProposal { + prepared_input: input_with_assurance(Retention::Ephemeral, Assurance::AppleAppAttest) + .into(), + deadlines: deadlines(), + }; + let request = client.propose(&proposal).unwrap(); + let id = client.accepted(&service.accept(&request)).unwrap(); + let backend = FetchBackend::default(); + assert!(matches!( + run_accepted_work(&service, &fetch_ready(), &backend, id) + .await + .unwrap(), + RunOutcome::Completed { .. } + )); + let request = client.request_delivery(id, &[8; 32]).unwrap(); + let delivered = service.deliver(&request, &[8; 32]).unwrap(); + client + .receive( + id, + &fetch_ready(), + &WorkDelivered { + result: delivered.result.encode(), + provider_signature: delivered.signature.as_bytes().to_vec(), + transcript: delivered.transcript, + }, + ) + .unwrap(); + let payment = client.pay(id).unwrap(); + use hellas_rpc::services::work::WorkHandler; + let response: hellas_rpc::call::WithTrailer = + service + .admit_certificate(payment, hellas_wire::TransportContext::default()) + .await + .unwrap() + .into(); + assert!(matches!( + response.response.outcome, + Some(hellas_rpc::pb::work::admit_certificate_response::Outcome::Paid(_)) + )); + assert_no_bodies(provider_dir.path()); +} diff --git a/crates/work/tests/work_store_channel.rs b/crates/work/tests/work_store_channel.rs index 9c4f6187..8dbe9247 100644 --- a/crates/work/tests/work_store_channel.rs +++ b/crates/work/tests/work_store_channel.rs @@ -443,6 +443,156 @@ fn commit_all(store: &mut ChannelStore, records: &[ChannelRecord]) { } } +fn open_metadata(root: &std::path::Path) -> ChannelStore { + ChannelStore::open_metadata_only( + root, + channel(), + settlement(), + Role::Provider, + origin_of(&channel()), + &Secp256k1Verifier::new(), + ) + .expect("metadata provider journal opens") +} + +fn assert_payloads_absent(root: &std::path::Path, needles: &[&[u8]]) { + for entry in std::fs::read_dir(root).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + assert_payloads_absent(&path, needles); + } else { + let bytes = std::fs::read(&path).unwrap(); + for needle in needles { + assert!(!needle.is_empty()); + assert!( + !bytes.windows(needle.len()).any(|window| window == *needle), + "customer payload reached {}", + path.display() + ); + } + } + } +} + +#[test] +fn metadata_journal_never_writes_payloads_at_any_commit_or_rotation() { + let channel = channel(); + let job = job_at(&channel, 1, 0); + let sequence = provider_sequence(&channel, &job); + let input = bundle_bytes(1); + let output = spool(&job.transcript); + let payloads = [&input[..], &output[..], job.transcript[0].payload()]; + for boundary in 1..=sequence.len() { + let root = temp(); + let mut store = open_metadata(root.path()); + for record in &sequence[..boundary] { + store + .commit(record.clone(), &Secp256k1Verifier::new()) + .unwrap(); + assert_payloads_absent(root.path(), &payloads); + } + if let Some(active) = store.state().job_by_id(job.work_id) { + assert_eq!(active.prepared_input(), input); + if active.result().is_some() { + assert_eq!(active.transcript(), output); + } + } + store.rotate().unwrap(); + assert_payloads_absent(root.path(), &payloads); + drop(store); + let reopened = open_metadata(root.path()); + if let Some(active) = reopened.state().job_by_id(job.work_id) { + assert!(active.prepared_input().is_empty()); + assert!(active.transcript().is_empty()); + assert_eq!(active.authorization(), &job.authorization); + } else { + assert_eq!(reopened.state().ledger().credited_cumulative(), PRICE); + } + } +} + +#[test] +fn metadata_journal_can_credit_an_already_delivered_result_after_restart() { + let root = temp(); + let channel = channel(); + let job = job_at(&channel, 1, 0); + let mut store = open_metadata(root.path()); + commit_all(&mut store, &provider_sequence(&channel, &job)[..5]); + drop(store); + let mut recovered = open_metadata(root.path()); + recovered + .commit(job.paid(&channel), &Secp256k1Verifier::new()) + .unwrap(); + assert_eq!(recovered.state().ledger().credited_cumulative(), PRICE); + recovered.rotate().unwrap(); + drop(recovered); + assert_eq!( + open_metadata(root.path()) + .state() + .ledger() + .credited_cumulative(), + PRICE + ); +} + +#[test] +fn metadata_replay_does_not_weaken_live_payload_validation() { + let root = temp(); + let channel = channel(); + let job = job_at(&channel, 1, 0); + let mut store = open_metadata(root.path()); + let mut proposal = job.proposed(); + if let ChannelRecord::JobProposed { prepared_input, .. } = &mut proposal { + prepared_input.clear(); + } + assert!(store.commit(proposal, &Secp256k1Verifier::new()).is_err()); + assert!(store.is_empty()); + commit_all(&mut store, &provider_sequence(&channel, &job)[..3]); + let mut result = job.result_record(&channel); + if let ChannelRecord::JobResult { transcript, .. } = &mut result { + transcript.clear(); + } + let records = store.len(); + assert!(store.commit(result, &Secp256k1Verifier::new()).is_err()); + assert_eq!(store.len(), records); +} + +#[test] +fn journal_retention_mode_cannot_change_when_reopened() { + for metadata in [false, true] { + let root = temp(); + let store = if metadata { + open_metadata(root.path()) + } else { + open(root.path(), Role::Provider) + }; + drop(store); + let reopened = if metadata { + ChannelStore::open( + root.path(), + channel(), + settlement(), + Role::Provider, + origin_of(&channel()), + &Secp256k1Verifier::new(), + ) + } else { + ChannelStore::open_metadata_only( + root.path(), + channel(), + settlement(), + Role::Provider, + origin_of(&channel()), + &Secp256k1Verifier::new(), + ) + }; + assert!( + reopened.is_err(), + "retention mode is part of journal identity" + ); + } +} + // ── The happy path ──────────────────────────────────────────────────── /// One paid job credits its certificate and rests at a certified diff --git a/crates/work/tests/work_store_setup.rs b/crates/work/tests/work_store_setup.rs index 9a4a00fe..71b6f0d4 100644 --- a/crates/work/tests/work_store_setup.rs +++ b/crates/work/tests/work_store_setup.rs @@ -157,7 +157,7 @@ fn provider_policy() -> ProviderChannelPolicy { network: network(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: EdgeValues::new(3_000, 200, Fees::ZERO), min_omit_response_blocks: hellas_kernel::MIN_OMIT_RESPONSE_BLOCKS, } diff --git a/docs/http-gateway.md b/docs/http-gateway.md new file mode 100644 index 00000000..3f8643a1 --- /dev/null +++ b/docs/http-gateway.md @@ -0,0 +1,264 @@ +# HTTP APIs through Fetch + +`gateway --http-fetch-config FILE --paid-work-config POOL` exposes exact HTTP +routes over paid HTTPS Fetch. Request and response bodies keep the upstream +format, including SSE, tool calls and non-2xx errors. Every request uses a funded +channel from the [paid pool](paid-gateway.md); missing payment configuration is a +startup error. See [provider account and egress configuration](../crates/providers/HTTPS.md). + +The gateway's standard API paths can serve several accounts or provider nodes. +It reads the requested model, finds backends configured to serve that model and +route, and keeps an existing session on its assigned backend. Model names and +request/response bodies are forwarded unchanged. + +Example with two Kimi accounts behind one endpoint: + +```json +{ + "service": "https", + "method": "request", + "max_in_flight": 2, + "backends": { + "kimi-a": { + "models": ["k3"], + "credential": "kimi-a", + "routes": [{ + "path": "/v1/chat/completions", + "method": "POST", + "url": "https://api.kimi.com/coding/v1/chat/completions" + }] + }, + "kimi-b": { + "models": ["k3"], + "credential": "kimi-b", + "routes": [{ + "path": "/v1/chat/completions", + "method": "POST", + "url": "https://api.kimi.com/coding/v1/chat/completions" + }] + } + } +} +``` + +Both accounts use the same client base URL, such as `http://127.0.0.1:8080/v1`. +Add backends with their exact supported models and API routes for other accounts. +The configured service and method must match the provider's Fetch route. An +optional backend `max_in_flight` overrides the global per-account limit. Routes +sharing a credential alias on the same provider share capacity and cooldown; +duplicating that alias does not multiply its allowance. + +A backend's optional `provider` is an endpoint ID from the paid pool. It may be +omitted only when the pool contains exactly one provider for the HTTP Fetch +manifest. Addresses, funding, journals and enrollment/Apple trust pins belong in +the pool. The provider must mount a paid Fetch channel for the gateway's transport +and settlement identities and expose the configured service/method. + +The gateway bearer authorizes access to the gateway's payment identity. The +router selects an account, then the paid client signs the exact Fetch request +and proposes it on that provider's channel. The provider checks the funded edge, +credit, deadlines, signature and Fetch policy before durably accepting work; +only that accepted job can invoke the upstream. The credential's origin, path +and method restrictions remain enforced by the provider. Courtesy caller grants +do not authorize this HTTP path. + +The SDK Fetch provider advertises only Work and WorkSetup when configured for +paid work; legacy allowed callers cannot reach its credentials through unpaid +Fetch. Accepted paid Fetch jobs wait in a bounded provider queue when upstream +concurrency is full. Further submissions wait at the executor mailbox while +completion handling remains available. Provider validator connections check the +configured genesis digest before advancing payment state; this check runs on +connection, not per HTTP request. + +Signed response prefixes stream as they arrive. Successful HTTP completion +follows verification of the complete result and the provider's durable payment +acknowledgement. Each accepted valid terminal costs the channel's agreed fixed +price, including HTTP error statuses. Already available prefixes share bounded +wire frames, with a local observer-readiness check before each frame and individual +signature checks for every event. Batching does not wait for more upstream data. +Requests are not retried on another +provider after account selection. The pool serializes work within each provider +channel; concurrent requests assigned to that provider queue within its deadline, including requests for different +accounts on that provider. Separate providers have independent channels. + +Paths and methods match exactly; query parameters retain their order, repeats +and percent encoding. Ordinary client headers, including idempotency keys and +vendor extensions, pass through. Connection-specific headers, credentials, +cookies and gateway control headers are removed; the caller's gateway bearer +never becomes an upstream credential. Optional route `headers` holds operator +`[name, value]` pairs, with lowercase names. Configured values override the +corresponding client header. Account credentials belong on the backend. + +For opaque HTTP passthrough without model selection, configure top-level `routes` +instead of `backends`. + +Omit `credential` for an unauthenticated upstream. Such routes share admission +by origin. Optional `tls` uses the [Fetch TLS vocabulary](../crates/providers/HTTPS.md) +for exact trust anchors and pins, defaulting to WebPKI. Custom roots cannot be +combined with a provider credential; that restriction remains enforced by the +provider. Private destinations also require its explicit egress opt-in. + +```sh +hellas-cli --identity ./gateway.identity gateway \ + --http-fetch-config ./http-gateway.json \ + --paid-work-config /srv/hellas/paid-pool.json \ + --port 8080 --bearer-token-file ./gateway.bearer +``` + +The client signs the request and verifies each response chunk before forwarding +it. Provider-owned credentials require public WebPKI roots. The gateway asks the +provider for ephemeral retention and disables inference replay. Its own archive +policy is independent of provider retention and upstream `store` fields. + +For Kimi Code, set `KIMI_MODEL_NAME` to an available model and use `--wrap kimi`; +the wrapper supplies `KIMI_MODEL_BASE_URL` and `KIMI_MODEL_API_KEY`. A separately +launched client can use the same variables and the private gateway bearer file. +Choose a context size that fits the byte limits below. + +## Model selection and session affinity + +New sessions use an eligible account outside cooldown with available capacity. +The gateway chooses the least occupied account, rotating ties. Existing sessions +stay pinned even when that account becomes busy or rate-limited: they receive a +retry response; new sessions can use another account. A request is sent once. +Neither a transport failure nor an upstream error causes automatic replay on a +different backend. A transport failure briefly excludes that backend from new +sessions. Quota polling remains monitoring; selection uses configured model +capabilities, current capacity and observed response cooldowns, not estimates +from token usage or external quota scrapes. + +Affinity recognizes `x-hellas-session-id`, Claude Code's +`x-claude-code-session-id`, Codex's `session-id`/`thread-id`, Claude's session ID +in `metadata.user_id`, and `prompt_cache_key` (used by Kimi Code), in that order. +An explicit session survives changes of client connection. Without one, requests +for the same model/API on the same accepted HTTP connection stay together. +Connection-only bindings are reclaimed once that connection closes. Different +explicit sessions may share that connection. Affinity is scoped by +model and API family; Responses compaction and Messages token counting share +their generation API's family. + +Responses API IDs observed in JSON or SSE bind `previous_response_id` and +`conversation` continuations to the original backend, including continuations +that omit the model. Unknown, expired or +conflicting state returns 409 before contacting an upstream. Session identifiers +are held as salted hashes in a bounded memory table (16,384 entries, 24 hours +idle). Unexpired entries are not evicted to admit new sessions; a full table +returns 503. Restarting the gateway clears affinity. Server-side continuations +whose mapping was lost require full context rather than being sent to a guessed +account. Full-history requests can establish new affinity after a restart. + +Model routing reads JSON, gzip or zstd requests with an 8 MiB decoded limit and +an 8 MiB zstd window limit. It forwards the original encoded bytes. An unsupported +model receives 404; malformed or unsupported routing input receives 400. These +checks precede upstream execution. Model-less, empty-body auxiliary requests use +an eligible backend; account-specific quota endpoints are not aggregated. + +## Archives and ZDR + +Fetch payment journals on both endpoints retain hashes, signatures and accounting +records, with payloads held in memory. After a restart they can re-send an existing +payment certificate, but cannot reconstruct or replay a lost request/response. +Unpaid work with a lost payload keeps its channel reserved until its payment +deadline; it is never re-executed from a new request. Existing payload-bearing +Fetch journals are refused instead of silently changing their retention policy. +The HTTP archive below is the gateway's separate payload persistence policy. + +CLI gateways archive authenticated requests and responses by default under +`~/.hellas/gateway-archive`, or `--archive-dir DIRECTORY`. This applies to the +existing inference routes as well as HTTP Fetch. Each exchange has owner-only +`request.bin`, `response.bin` and `metadata.json` files. Metadata records status, +size, content type/encoding, elapsed time, completion, selected backend and trace context; it excludes authentication +headers. `x-hellas-request-id` identifies the exchange. Failed or cancelled +streams retain an incomplete archive. Archiving is best-effort: failures during +setup, request/response writes or finalization are reported without replacing +the upstream status or interrupting its response. After a write fails, archiving +stops for that exchange; the next ordinary request attempts a fresh archive. +Once an archive has been created, `x-hellas-request-id` remains present even if +saving its response metadata fails. It identifies an archive attempt, not a +guarantee of a complete durable record. There is no automatic archive pruning. + +Archive failures emit a `hellas_archive` warning with the operation, I/O error +kind and OS error code, without payloads, credentials, paths or raw error text. +With `otel`, `hellas.gateway.archive.failures` counts failures by `archive.stage` +(`prepare`, `request`, `response_head`, `response_body`, `finish`) and `error.type`. +The Prometheus name is `hellas_gateway_archive_failures_total`. Successful HTTP +responses remain successful in request telemetry even when their archive fails; +monitor the archive counter separately. ZDR requests do not attempt archival or +increment this counter. + +`x-hellas-zdr: true` disables application payload persistence for that request. +`--zdr` enforces this for all requests. Ambiguous flags, `store: true`, and ZDR +with inference caching enabled are rejected before archive writes. The +`store: true` check reads the decoded body, so a gzip or zstd request cannot +smuggle the flag past it. A non-ZDR +request may be archived even when its upstream `store` is false. Unauthorized, +invalid ZDR requests and bodies exceeding the 2 MiB ingress limit are rejected +without payload archives. Non-ZDR requests that pass ingress but exceed the +smaller Fetch envelope limit are archived with their 413 response. + +These switches govern this gateway's application writes. They do not establish +OS swap/dump protection, a remote provider's host policy, upstream retention, +or the coding client's own session-log policy. Software identities do not +attest those properties. The embedded Responses-only `start_fetch` API retains +its host-managed storage policy. + +## Limits and telemetry + +HTTP input is limited to 1,523,712 bytes, reserving envelope space inside the +2 MiB signed Fetch request. Responses are limited to 8 MiB and the Fetch stream +to 32,768 events and 16 MiB of signed payload. The paid policy can impose tighter +limits; its transcript spool is capped at 32 MiB. Configure the spool and output +budgets for the encoded HTTP response (including base64 and signed envelopes). +Each wire frame remains bounded independently: Fetch completion authenticates +all preceding prefixes without packing the full response back into one frame. + +The provider has a 20 minute total HTTP deadline and a 90 second response idle +timeout. The pool's `timeout_secs` and on-chain block deadlines must also fit the +expected execution time. Exceeding a stream limit closes it as incomplete. +Individual upstream deliveries are forwarded promptly, split at 16 KiB. +Before proposal, a disconnect cancels the request. After proposal, collection and +payment continue independently of the HTTP reader. The bounded HTTP output queue +reports an error to a stalled reader without stopping settlement; graceful +shutdown drains accepted operations. Unfinished operations retain their accounting +evidence for recovery, subject to Fetch's in-memory payload lifetime. + +No generation is automatically retried upstream. Upstream status and end-to-end +response headers reach the client, including Location, ETag, `Retry-After`, +request IDs and rate limits. Hop-by-hop headers and cookies are removed. Body +framing is regenerated; representation Content-Length is retained for HEAD/304. +A 429, or a 5xx carrying `Retry-After`, starts a shared account cooldown; +delta seconds and HTTP dates are accepted. Requests during cooldown receive +the status that established the longest remaining delay: a 503 remains a 503, +so clients can keep retrying overloads. Missing or invalid delays on a 429 use one second. Excess +concurrency returns 503 with `Retry-After: 1`. + +With `otel`, traces connect HTTP ingress, Fetch RPCs, credential refresh and +upstream HTTP. Span attributes exclude request/response bodies and credentials. +`hellas.backend` and `gen_ai.request.model` identify configured backend/model +choices in request metrics and traces; `hellas.routing.affinity` records the +selection reason in traces. Raw session IDs are not exported. +Outbound trace context replaces the caller's propagation headers instead of +appending duplicates. Upstream response `traceparent` and `tracestate` are +preserved; `x-hellas-trace-id` identifies the gateway's trace independently. +Pooled QUIC connections have separate root spans so their lifetime cannot delay +exporting the first request's trace. +`hellas.gateway.http.requests`, `.duration`, `.time_to_first_byte` and `.tokens` +record status, completion, timings and standard OpenAI/Anthropic usage fields. +SSE usage is recognized even when the upstream omits Content-Type. Cache reads +and writes are reported separately as `cache_read` and `cache_write` token kinds; +the input/output counts retain the upstream's meaning. Usage is unknown when +upstream omits it. +Bodies remain encoded on the wire and in archives. Usage observation accepts +plain JSON/SSE and gzip, with a 32 MiB decoded-byte budget and a 512 KiB pending +JSON/line budget. Invalid compression, unsupported encodings or exceeded budgets +leave usage unknown without affecting delivery. +Account quota windows still come from the upstream quota exporter. Request +token totals alone cannot determine subscription quota or billing. + +This is an HTTP API bridge with explicit routes, authentication and resource +limits. It does not implement CONNECT, WebSocket upgrades, streaming uploads, +HTTP trailers or automatic rewriting of redirect URLs. Clients using Responses +WebSockets must select their HTTP/SSE transport. Archive I/O remains on the +response path, although its errors no longer fail requests. These are contract +differences from a general transparent HTTP proxy, even when inference and +tool-call payloads are preserved exactly. diff --git a/docs/paid-gateway.md b/docs/paid-gateway.md index 2c230f02..d4efa113 100644 --- a/docs/paid-gateway.md +++ b/docs/paid-gateway.md @@ -1,4 +1,4 @@ -# Paid inference gateway +# Paid gateway `hellas-cli gateway --paid-work-config /srv/hellas/pool.json` sends token-native inference to the configured providers and pays through their real funded work @@ -6,6 +6,15 @@ channels. Each provider has its own endpoint identity, bond, payment funding, and client journals. The gateway prefers idle matching providers and matching prompt prefixes, then serializes requests within each channel. +With `--http-fetch-config`, the same pool carries HTTP Fetch requests. Its work +config must select the HTTP Fetch manifest and a matching Fetch route policy; +see [HTTP routing](http-gateway.md). The HTTP router chooses the provider and +account, so this path never falls back to a different provider or Courtesy. + +Paid Fetch extends the Work/WorkSetup service descriptors and the StreamResult +response schema. Upgrade gateways and providers together: older peers reject +the changed wire IDs. The on-chain payment certificate format is unchanged. + The pool file uses the provider's existing `--work-config` policy and chain configuration: @@ -30,9 +39,18 @@ configuration: } ``` +For `--assurance apple-app-attest`, each pool entry also needs +`provider_genesis` (hex enrollment ContentId), `apple_app_id`, and +`apple_cd_hashes` (an array of hex 32-byte hashes). Setup and work connections +verify this anchor before disclosing requests. Producer-signed entries may also +pin `provider_genesis`; their funded bond fixes the settlement identity in all cases. + Provision provider bonds with `hellas-cli provision`; coin values and policy terms must agree with the provider work config. Payment coins must be owned by the gateway settlement identity and cannot fund two channels. +The omission bond must exceed the remaining payment capacity: with 1,000 units +and zero fees/reserve, a bond of 501 leaves 499 spendable units. The client checks +this before signing setup terms or creating a journal. The provider's route table must authorize the gateway transport and settlement identities. All participants must use the same compatible chain revision. @@ -49,9 +67,39 @@ The provider reserves delivery credit before streaming signed token prefixes. The gateway verifies each prefix as it arrives, then authenticates and journals the complete result and obtains the provider's durable payment acknowledgement before reporting completion. One open subscription replaces repeated result -polls. Payment is an accumulating channel certificate. Channels stay open and -follow finalized blocks while idle; they settle through the existing close -protocol. The pool does not automatically refill exhausted channels. +polls. Payment is an accumulating channel certificate. An established, healthy +channel performs no validator RPCs on its request or streaming paths. Requests +use local credit checks, signatures and journal commits. Channels settle through +the existing close protocol; the pool does not automatically refill them. + +Each channel observes finalized history independently of its requests. The client +has one request owner and a separate observer handle, so a slow response cannot +block close monitoring or reorder payment exchanges. The provider also drives +channels independently: one stalled channel cannot block another's recovery. +Applying a close and crediting a certificate use the same short journal lock. +Jobs within a channel still finish in order, including collection and payment +after an HTTP client disconnects. A server that keeps an HTTP response open after +an application completion event therefore holds that channel until EOF. The +generic HTTP route preserves those trailing bytes. Separate funded channels +execute independently; `paid.queue` measures time waiting for a channel. + +The work configuration accepts `max_observation_age_ms` (default 5000), which +must exceed `poll_ms`. Admission, delivery and new certificates stop when the +observer fails or no new verified finalized height arrives within that age. +Repeated reads of an old tip do not renew it. The bound starts before the chain +read, not after it finishes, and readiness is published only after all intervening +blocks have been applied. Results may still be retained during an observation +outage; existing payment certificates remain safe to retransmit. Because renewal +requires an *advancing* finalized height, this age should also exceed the +deployment's block interval by a comfortable margin: a chain finalizing slower +than the age spends every gap stale, and admission flaps on each block. + +This is an online channel: its observer must detect unilateral closes and submit +any response within `omit_response_blocks`. Configure observation age, network +recovery and submission latency to fit that window at the deployment's block +rate. The local age limit cannot guarantee recovery through a partition longer +than the on-chain response window. Setup and restart catch-up may wait on +validators; ordinary requests do not perform that work. The shared `--output-cache record` mode also covers paid inference. Repeating an identical recorded request reuses its output without another paid job. A @@ -61,13 +109,17 @@ providers or validators. Keep gateway and provider identities and journals across service restarts, including when the operating system's store is ephemeral. Startup recovers -unfinished jobs and re-sends retained payments idempotently. Disconnecting an +retained Evaluate jobs and re-sends payment certificates idempotently. Fetch +journals omit bodies. A verified delivery recorded before a crash is paid from +its retained evidence, even when the payload is gone. Jobs without either payload +or verified delivery retain their credit reservation until the payment deadline; +recovery skips them and never submits them anew. Disconnecting an HTTP client cancels work that has not yet been proposed. Once a signed proposal may have reached a provider, collection and payment continue despite disconnects. The pool admits at most `max_pending_requests` queued or running requests (default 64); additional requests receive HTTP 503 and may be retried. `timeout_secs` bounds queueing, recovery, provider fallback, execution and payment together, -rather than restarting for each provider. Interactive requests skip busy provider +rather than restarting for each provider. Token-native requests skip busy provider channels; each connection attempt gets at most 10 seconds before trying another route within that shared budget. The HTTP paid route uses this same configured budget; non-paid routes retain their existing 3600-second default. HTTP delivery has an @@ -110,7 +162,7 @@ Hellas RPC metadata carries the context over both Iroh and WebSocket mux connections, including validator queries and submissions. Transparent frame relays preserve this metadata. The explorer relay continues the context through its forwarding span, and proof-origin HTTP queries continue the caller trace. -Background indexer polling is independent of the paid request trace. +Background chain observation uses separate `paid.channel.observe` traces. Useful spans include `http.server`, `paid.gateway`, `paid.queue`, `paid.executor.stream`, GenAI `chat MODEL` and `text_completion` operations, @@ -163,3 +215,31 @@ prefill chunks when its memory budget permits. The gateway conservatively clears its affinity hint after short requests; the provider decides actual reuse. Worker spans report that observed reuse as `gen_ai.usage.cache_read.input_tokens` without recording token values. + +## Embedding + +The CLI and native hosts use `hellas_sdk::paid_gateway::PaidGateway`. Enable +`paid-gateway`, load the same pool file with `load_pool_options(path, assurance)`, +and pass the resulting typed `PaidGatewayOptions` and a `ClientIdentity` to +`PaidGateway::open`. Applications can also construct the options directly from +`PaidWorkOptions`; the pool validates duplicate providers, journals and funding +before opening any connections. It owns recovery, admission, per-channel +serialization and payment tasks. No CLI argument structures are part of its API. + +For transparent HTTP, pass this pool to `hellas_gateway::start_http` using +`HttpGatewayOptions`. Its `paid` field is required. The options contain HTTP +routes, archive policy, listener authentication and optional process wrapping; +they have no model, tokenizer, inference cache or Evaluate settings. Shutting +down the returned handle drains accepted paid work. Failed startup also drains +recovery tasks. A host using the pool without a gateway must call its +`PaidExecutionBackend::drain` method when shutting down. + +`paid-client` enables sessions without the executor or upstream-provider +crates. `paid-provider` enables the provider runner and provisioning. Both +include `work`, the shared typed configuration and provisioning API. Apple +verification additionally needs `apple-verifier`; the file loader stores +assertion counters under each provider's journal root in `apple-counters`. +Counter files alone do not trigger channel funding during startup recovery. +Work and WorkSetup each reuse an authenticated QUIC connection. Reconnection +repeats Open with that connection's exporter and the channel's producer pin; +no new consensus query is added to the per-request path. diff --git a/nix/ci.nix b/nix/ci.nix index a2256b1f..09ba37e3 100644 --- a/nix/ci.nix +++ b/nix/ci.nix @@ -96,6 +96,10 @@ let ++ lib.optionals isValidatorHost [ "validator" ] ) )) (cargoEnv rustToolchain); + # Run command tests locally as well as in the release package builds. + cli = mkCargo "check-cli" "cargo test -p hellas-cli --no-default-features --features node,gateway" ( + cargoEnv rustToolchain + ); # The kernel's whole suite, including `tests/itf.rs` — the Quint↔Rust # replay that the entire abstract-correspondence story rests on — and # the exact-error pins in `tests/channel/`. `--all-features` is load @@ -129,6 +133,17 @@ let mkCargo "check-client-work" "cargo test -p hellas-client --features work && cargo clippy -p hellas-client --features work --all-targets -- -D warnings" (cargoEnv rustToolchain); + # HTTP boundaries and the bounded file primitives shared by hosts. + http-fetch = + mkCargo "check-http-fetch" + "cargo test -p hellas-gateway -p hellas-providers -p hellas-private -p hellas-store" + (cargoEnv rustToolchain); + # Exercise the production provider loop and paid session recovery; their + # feature is absent from the SDK's default tests. + sdk-paid-work = + mkCargo "check-sdk-paid-work" + "cargo test -p hellas-sdk --features paid-client,paid-provider,paid-gateway && cargo clippy -p hellas-sdk --features paid-client,paid-provider,paid-gateway --all-targets -- -D warnings && cargo clippy -p hellas-sdk --no-default-features --features paid-client --all-targets -- -D warnings && cargo clippy -p hellas-sdk --no-default-features --features paid-provider --all-targets -- -D warnings && cargo clippy -p hellas-sdk --no-default-features --features paid-gateway --all-targets -- -D warnings" + (cargoEnv rustToolchain); # The chain service's wire-id pins compile only under `chain`, which # `work` does not pull in. `check-validator` links hellas-rpc with # that feature but runs hellas-chain's tests, not hellas-rpc's, so diff --git a/proto/hellas/work/v1/work.proto b/proto/hellas/work/v1/work.proto index 6fbfeea4..c922dd8e 100644 --- a/proto/hellas/work/v1/work.proto +++ b/proto/hellas/work/v1/work.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package hellas.work.v1; +import "hellas/v1/hellas.proto"; + // The paid-work service: the wire by which a client proposes one job on // one payment channel and a provider accepts or refuses it. // @@ -11,6 +13,8 @@ package hellas.work.v1; // exact encodings. A protobuf re-spelling of an authorization would be a // second definition of what both parties signed. service Work { + rpc Open(.hellas.v1.OpenRequest) returns (.hellas.v1.OpenResponse); + // One bilateral acceptance. The client sends the authorization it has // already signed; the provider answers with its co-signature or with // one refusal code. @@ -24,7 +28,8 @@ service Work { // This unary form waits for the durable terminal result. The job's // price is reserved as delivery credit before release; replay returns // the same bytes without a second debit. StreamResult offers signed - // token prefixes under the same authorization and delivery credit. + // prefixes under the same authorization and delivery credit. Use StreamResult + // for Fetch responses that do not fit in one wire frame. rpc DeliverResult(DeliverResultRequest) returns (DeliverResultResponse); // Authenticated live output. Delivery credit is durably reserved before @@ -58,6 +63,8 @@ service Work { // would make every paid method answer for a channel that does not exist // yet, and every setup method answer for one that already does. service WorkSetup { + rpc Open(.hellas.v1.OpenRequest) returns (.hellas.v1.OpenResponse); + // One step of the handshake: the caller offers the revision it holds, // and is answered with the revision that extends it. // @@ -251,5 +258,14 @@ message WorkStreamEvent { bytes prefix = 1; // canonical transcript containing signed output events WorkDelivered delivered = 2; WorkRefused refused = 3; + // Fetch completion after all nonterminal envelopes in prefix frames. + // The receiver verifies the assembled transcript against this result. + WorkStreamTerminal terminal = 4; } } + +message WorkStreamTerminal { + bytes result = 1; + bytes provider_signature = 2; + bytes terminal_transcript = 3; // exactly one signed terminal envelope +}