From 6983538fb9ab1696f717db67284d852cc9c74088 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Wed, 23 Sep 2026 02:37:58 +0200 Subject: [PATCH 01/32] fetch: keep upstream error response bodies out of provider logs --- crates/providers/src/responses_fetch.rs | 88 +--------- crates/providers/src/responses_fetch/tests.rs | 161 +++++++----------- 2 files changed, 64 insertions(+), 185 deletions(-) diff --git a/crates/providers/src/responses_fetch.rs b/crates/providers/src/responses_fetch.rs index 840745aa..a0d08523 100644 --- a/crates/providers/src/responses_fetch.rs +++ b/crates/providers/src/responses_fetch.rs @@ -10,13 +10,10 @@ use tracing::Instrument; #[cfg_attr(not(feature = "otel"), path = "responses_fetch/telemetry/noop.rs")] mod telemetry; -/// Maximum diagnostic prefix retained from an unsuccessful HTTP response. -const MAX_FETCH_ERROR_BODY_BYTES: usize = 2 * 1024; /// A total request deadline bounds the whole call; this independent idle /// deadline prevents a peer that stops producing SSE bytes from occupying a /// Fetch execution slot for that entire window. Ordinary SSE keepalives count -/// as activity and reset it. Error bodies are diagnostic only, so the same -/// duration bounds their entire prefix read without renewal. +/// as activity and reset it. Unsuccessful response bodies are never read or logged. const FETCH_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(90); /// HTTP client for attested Fetch egress. Redirects are disabled because the @@ -61,12 +58,13 @@ pub async fn execute_responses_request( telemetry.status(status.as_u16()); if !status.is_success() { telemetry.fail("http_error"); - let diagnostic = error_body_prefix(upstream).await; + // Error bodies can echo customer input. Drop them without reading or + // logging a prefix, even when the request itself is ephemeral. + drop(upstream); tracing::warn!( provider = label, upstream_status = status.as_u16(), - upstream_diagnostic = %diagnostic, - "upstream Fetch request failed; response details are omitted from the caller error" + "upstream Fetch request failed" ); return Err(FetchProviderError::failed(format!( "{label} upstream rejected the request (HTTP {})", @@ -133,82 +131,6 @@ fn unique_model_header( Ok(value) } -async fn error_body_prefix(upstream: reqwest::Response) -> String { - error_body_prefix_with_deadline(upstream, FETCH_STREAM_IDLE_TIMEOUT).await -} - -async fn error_body_prefix_with_deadline( - upstream: reqwest::Response, - read_timeout: Duration, -) -> String { - let mut body = Vec::new(); - let mut chunks = upstream.bytes_stream(); - let deadline = tokio::time::Instant::now() + read_timeout; - - while body.len() < MAX_FETCH_ERROR_BODY_BYTES { - let next = match tokio::time::timeout_at(deadline, chunks.next()).await { - Ok(next) => next, - Err(_) => { - let excerpt = diagnostic_excerpt(&body); - let note = format!( - "[error body read timed out after {} seconds]", - read_timeout.as_secs_f64() - ); - return if excerpt.is_empty() { - note - } else { - format!("{excerpt} {note}") - }; - } - }; - let Some(chunk) = next else { - break; - }; - let chunk = match chunk { - Ok(chunk) => chunk, - Err(source) => { - let excerpt = diagnostic_excerpt(&body); - let note = format!("[error body read failed: {source}]"); - return if excerpt.is_empty() { - note - } else { - format!("{excerpt} {note}") - }; - } - }; - let remaining = MAX_FETCH_ERROR_BODY_BYTES - body.len(); - let retained = remaining.min(chunk.len()); - body.extend_from_slice(&chunk[..retained]); - if retained < chunk.len() { - break; - } - } - - let limited = body.len() == MAX_FETCH_ERROR_BODY_BYTES; - let body = diagnostic_excerpt(&body); - if limited { - format!("{body} [body prefix limited to {MAX_FETCH_ERROR_BODY_BYTES} bytes]") - } else { - body - } -} - -fn diagnostic_excerpt(body: &[u8]) -> String { - String::from_utf8_lossy(body) - .chars() - .map(|character| { - if character.is_control() { - ' ' - } else { - character - } - }) - .collect::() - .split_whitespace() - .collect::>() - .join(" ") -} - fn stream_response( upstream: reqwest::Response, label: String, diff --git a/crates/providers/src/responses_fetch/tests.rs b/crates/providers/src/responses_fetch/tests.rs index acdf3a0e..73345faf 100644 --- a/crates/providers/src/responses_fetch/tests.rs +++ b/crates/providers/src/responses_fetch/tests.rs @@ -9,8 +9,37 @@ use futures::stream; use reqwest::header::{HeaderMap, HeaderValue}; use std::convert::Infallible; use std::sync::Arc; +use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; +#[derive(Clone, Default)] +struct CapturedLogs(Arc>); + +impl tracing::field::Visit for CapturedLogs { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + use std::fmt::Write as _; + write!(self.0.lock().unwrap(), "{field}={value:?} ").unwrap(); + } +} + +impl tracing::Subscriber for CapturedLogs { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, values: &tracing::span::Record<'_>) { + values.record(&mut self.clone()); + } + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + event.record(&mut self.clone()); + } + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} +} + async fn redirect() -> Redirect { Redirect::temporary("/sink") } @@ -26,15 +55,6 @@ async fn json_success() -> Response { .unwrap() } -async fn oversized_error() -> Response { - let mut body = vec![b'x'; MAX_FETCH_ERROR_BODY_BYTES]; - body.extend_from_slice(b"SECRET_AFTER_LIMIT"); - Response::builder() - .status(StatusCode::BAD_REQUEST) - .body(Body::from(body)) - .unwrap() -} - async fn sensitive_error() -> Response { Response::builder() .status(StatusCode::UNPROCESSABLE_ENTITY) @@ -59,43 +79,6 @@ async fn oversized_event_stream() -> Response { .unwrap() } -fn stalled_error_before_first_byte() -> reqwest::Response { - axum::http::Response::builder() - .status(StatusCode::BAD_GATEWAY) - .body(reqwest::Body::wrap_stream(stream::pending::< - Result, - >())) - .unwrap() - .into() -} - -fn stalled_error_after_partial_body() -> reqwest::Response { - let chunks = - stream::once(async { Ok::<_, Infallible>(Bytes::from_static(b"useful diagnostic")) }) - .chain(stream::pending()); - axum::http::Response::builder() - .status(StatusCode::BAD_GATEWAY) - .body(reqwest::Body::wrap_stream(chunks)) - .unwrap() - .into() -} - -fn promptly_streamed_error() -> reqwest::Response { - let chunks = stream::iter([ - Bytes::from_static(b"useful "), - Bytes::from_static(b"diagnostic"), - ]) - .then(|chunk| async move { - tokio::time::sleep(Duration::from_secs(1)).await; - Ok::<_, Infallible>(chunk) - }); - axum::http::Response::builder() - .status(StatusCode::BAD_GATEWAY) - .body(reqwest::Body::wrap_stream(chunks)) - .unwrap() - .into() -} - async fn test_endpoint(app: Router, path: &str) -> Url { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let endpoint = Url::parse(&format!( @@ -156,31 +139,17 @@ async fn successful_fetch_requires_event_stream_content_type() { } #[tokio::test] -async fn provider_local_error_diagnostic_retains_only_a_bounded_body_prefix() { - let endpoint = test_endpoint( - Router::new().route("/responses", post(oversized_error)), - "/responses", - ) - .await; - - let response = responses_http_client().post(endpoint).send().await.unwrap(); - let diagnostic = error_body_prefix(response).await; - - assert!(diagnostic.contains(&format!( - "body prefix limited to {MAX_FETCH_ERROR_BODY_BYTES} bytes" - ))); - assert!(!diagnostic.contains("SECRET_AFTER_LIMIT")); - assert!(diagnostic.len() < MAX_FETCH_ERROR_BODY_BYTES + 256); -} - -#[tokio::test] -async fn unsuccessful_fetch_never_forwards_the_upstream_body_to_the_caller() { +async fn unsuccessful_fetch_never_logs_or_returns_the_upstream_body() { let endpoint = test_endpoint( Router::new().route("/responses", post(sensitive_error)), "/responses", ) .await; + let logs = CapturedLogs::default(); + // Keep this subscriber installed for the test binary: other HTTP tests + // also register the shared warning callsite on their runtime threads. + tracing::subscriber::set_global_default(logs.clone()).unwrap(); let Err(error) = execute_test_request(endpoint).await else { panic!("HTTP error unexpectedly passed"); }; @@ -191,45 +160,33 @@ async fn unsuccessful_fetch_never_forwards_the_upstream_body_to_the_caller() { "fetch provider failed: test upstream rejected the request (HTTP 422)" ); assert!(!message.contains("UPSTREAM_PRIVATE_SENTINEL")); + let logged = logs.0.lock().unwrap(); + assert!(logged.contains("upstream_status=422"), "{logged}"); + assert!(!logged.contains("UPSTREAM_PRIVATE_SENTINEL"), "{logged}"); } -#[tokio::test(start_paused = true)] -async fn error_body_stalled_before_first_byte_hits_the_read_deadline() { - let upstream = stalled_error_before_first_byte(); - let read_timeout = Duration::from_secs(5); - let started = tokio::time::Instant::now(); - - let diagnostic = error_body_prefix_with_deadline(upstream, read_timeout).await; - - assert_eq!(started.elapsed(), read_timeout); - assert_eq!(diagnostic, "[error body read timed out after 5 seconds]"); -} - -#[tokio::test(start_paused = true)] -async fn partial_error_body_is_retained_when_the_read_deadline_expires() { - let upstream = stalled_error_after_partial_body(); - let read_timeout = Duration::from_secs(5); - let started = tokio::time::Instant::now(); - - let diagnostic = error_body_prefix_with_deadline(upstream, read_timeout).await; - - assert_eq!(started.elapsed(), read_timeout); - assert_eq!( - diagnostic, - "useful diagnostic [error body read timed out after 5 seconds]" - ); -} - -#[tokio::test(start_paused = true)] -async fn promptly_streamed_error_body_completes_within_the_read_deadline() { - let upstream = promptly_streamed_error(); - let read_timeout = Duration::from_secs(5); - let started = tokio::time::Instant::now(); - - let diagnostic = error_body_prefix_with_deadline(upstream, read_timeout).await; - - assert_eq!(started.elapsed(), Duration::from_secs(2)); - assert_eq!(diagnostic, "useful diagnostic"); +#[tokio::test] +async fn an_unsuccessful_fetch_does_not_wait_for_an_error_body() { + async fn stalled() -> Response { + Response::builder() + .status(StatusCode::BAD_GATEWAY) + .body(Body::from_stream(stream::pending::< + Result, + >())) + .unwrap() + } + let endpoint = test_endpoint( + Router::new().route("/responses", post(stalled)), + "/responses", + ) + .await; + let result = tokio::time::timeout(Duration::from_secs(2), execute_test_request(endpoint)) + .await + .expect("the response body must not be polled"); + let Err(error) = result else { + panic!("HTTP error unexpectedly passed"); + }; + assert!(error.to_string().contains("HTTP 502")); } #[tokio::test] From 7acb2a2bd54978d2d3625ce7a65d284b8d7af0f3 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Wed, 23 Sep 2026 03:52:26 +0200 Subject: [PATCH 02/32] fetch: add paid HTTPS and shared attested work sessions Share channel setup, recovery, verified delivery and payment between SDK hosts, the CLI and the paid gateway. Keep Evaluate streaming and chain recovery while adding bounded caller-signed HTTPS Fetch with per-connection App Attest checks. Fetch providers journal accounting metadata without persisting request or result bodies, and never repeat an invocation after losing its in-memory payload. Validation: 1,029 passing tests across CLI, client, executor, providers, RPC, SDK, work journals and chain settlement; Fetch-only SDK check and formatting pass. Real Apple enrollment requires a provisioned signed host and was not exercised. --- Cargo.lock | 162 +- crates/chain/src/work_blocks.rs | 4 +- crates/chain/src/work_e2e.rs | 10 +- crates/cli/Cargo.toml | 3 + crates/cli/src/commands/paid_work.rs | 1048 ++-------- crates/cli/src/commands/paid_work/gateway.rs | 24 +- crates/cli/src/commands/paid_work/tests.rs | 68 +- crates/cli/src/commands/serve/mod.rs | 5 + crates/cli/src/commands/serve/node.rs | 1262 +----------- crates/cli/src/commands/serve/node/tests.rs | 62 +- crates/cli/src/commands/serve/provision.rs | 567 +----- crates/cli/src/commands/serve/work_config.rs | 626 +----- .../src/commands/serve/work_config/tests.rs | 62 +- crates/cli/src/main.rs | 1 + crates/client/src/iroh.rs | 3 +- crates/client/src/iroh/tests.rs | 80 +- crates/client/src/lib.rs | 4 +- crates/client/tests/work.rs | 6 +- .../executor/src/executor/actor/execution.rs | 4 +- crates/executor/src/executor/actor/mod.rs | 43 + .../executor/src/executor/actor/paid_fetch.rs | 126 ++ crates/executor/src/executor/mod.rs | 8 + crates/executor/src/fetch_policy.rs | 2 +- crates/executor/src/fetch_provider.rs | 9 +- crates/executor/src/fetch_registry.rs | 7 + crates/executor/src/lib.rs | 3 +- crates/executor/src/work.rs | 18 +- crates/providers/Cargo.toml | 8 + crates/providers/HTTPS.md | 99 + crates/providers/src/codex_responses/tests.rs | 2 + crates/providers/src/http.rs | 427 +++++ crates/providers/src/http/config.rs | 68 + crates/providers/src/http/tests.rs | 269 +++ crates/providers/src/http/tls.rs | 100 + crates/providers/src/lib.rs | 5 + crates/providers/src/responses_fetch.rs | 1 + crates/providers/src/responses_projector.rs | 3 + crates/rpc/Cargo.toml | 9 +- crates/rpc/build.rs | 15 + crates/rpc/src/http_fetch.rs | 382 ++++ crates/rpc/src/lib.rs | 2 + crates/rpc/src/output.rs | 12 + crates/rpc/src/pb/mod.rs | 10 +- crates/rpc/src/protocol/artifacts.rs | 37 +- crates/rpc/src/protocol/fetch_environment.rs | 21 +- crates/rpc/src/protocol/mod.rs | 4 + crates/rpc/src/protocol/work.rs | 78 +- crates/rpc/src/protocol/work_fetch.rs | 993 ++++++++++ crates/rpc/src/protocol/work_profile.rs | 337 ++++ crates/rpc/src/protocol/work_setup.rs | 79 +- crates/rpc/tests/paid_fetch_vectors.rs | 1696 +++++++++++++++++ crates/rpc/tests/paid_work_vectors.rs | 3 +- crates/rpc/tests/work_setup.rs | 9 +- crates/sdk/Cargo.toml | 27 + crates/sdk/src/lib.rs | 24 +- crates/sdk/src/paid_client.rs | 1105 +++++++++++ crates/sdk/src/paid_client/tests.rs | 290 +++ crates/sdk/src/paid_provider.rs | 1165 +++++++++++ crates/sdk/src/provider.rs | 312 ++- crates/sdk/src/work_config.rs | 696 +++++++ crates/sdk/src/work_provision.rs | 574 ++++++ .../src/work_provision}/tests.rs | 5 +- crates/work/README.md | 161 ++ crates/work/src/work.rs | 205 +- crates/work/src/work/stream.rs | 14 +- crates/work/src/work_open.rs | 30 +- crates/work/src/work_store/channel.rs | 6 +- crates/work/src/work_store/channel/codec.rs | 20 +- crates/work/src/work_store/channel/state.rs | 93 +- crates/work/src/work_store/channel/store.rs | 82 +- crates/work/src/work_store/journal.rs | 5 + crates/work/tests/work_accept.rs | 6 +- crates/work/tests/work_close.rs | 6 +- crates/work/tests/work_delivery.rs | 10 +- crates/work/tests/work_handshake.rs | 2 +- crates/work/tests/work_payment.rs | 6 +- crates/work/tests/work_run.rs | 14 +- crates/work/tests/work_run/fetch.rs | 365 ++++ crates/work/tests/work_store_channel.rs | 148 ++ crates/work/tests/work_store_setup.rs | 2 +- proto/hellas/work/v1/work.proto | 6 + 81 files changed, 10695 insertions(+), 3570 deletions(-) create mode 100644 crates/executor/src/executor/actor/paid_fetch.rs create mode 100644 crates/providers/HTTPS.md create mode 100644 crates/providers/src/http.rs create mode 100644 crates/providers/src/http/config.rs create mode 100644 crates/providers/src/http/tests.rs create mode 100644 crates/providers/src/http/tls.rs create mode 100644 crates/rpc/src/http_fetch.rs create mode 100644 crates/rpc/src/protocol/work_fetch.rs create mode 100644 crates/rpc/src/protocol/work_profile.rs create mode 100644 crates/rpc/tests/paid_fetch_vectors.rs create mode 100644 crates/sdk/src/paid_client.rs create mode 100644 crates/sdk/src/paid_client/tests.rs create mode 100644 crates/sdk/src/paid_provider.rs create mode 100644 crates/sdk/src/work_config.rs create mode 100644 crates/sdk/src/work_provision.rs rename crates/{cli/src/commands/serve/provision => sdk/src/work_provision}/tests.rs (99%) create mode 100644 crates/work/README.md create mode 100644 crates/work/tests/work_run/fetch.rs diff --git a/Cargo.lock b/Cargo.lock index b22a34b3..511f18b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -207,6 +207,45 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -411,6 +450,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -429,7 +474,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", ] [[package]] @@ -438,6 +483,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitcoin-consensus-encoding" version = "1.1.0" @@ -1730,6 +1784,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "der_derive" version = "0.7.3" @@ -2681,6 +2749,7 @@ dependencies = [ "hellas-private", "hellas-providers", "hellas-rpc", + "hellas-sdk", "hellas-store", "hellas-wire", "hellas-work", @@ -2871,6 +2940,7 @@ dependencies = [ "anyhow", "async-stream", "axum", + "base64 0.22.1", "futures", "hellas-adaptors", "hellas-executor", @@ -2878,13 +2948,20 @@ dependencies = [ "hellas-wire", "opentelemetry", "opentelemetry_sdk", + "rcgen", "reqwest", + "rustls", "serde", "serde_json", + "sha2 0.11.0", "tokio", + "tokio-rustls", "tracing", "tracing-opentelemetry", "tracing-subscriber", + "url", + "webpki-roots 1.0.8", + "x509-cert", ] [[package]] @@ -2892,6 +2969,7 @@ name = "hellas-rpc" version = "0.1.0" dependencies = [ "axum", + "base64 0.22.1", "base64ct", "bytes", "ed25519-dalek", @@ -2928,6 +3006,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "url", "web-time", ] @@ -2936,18 +3015,27 @@ name = "hellas-sdk" version = "0.1.0" dependencies = [ "anyhow", + "async-stream", + "futures", "futures-core", "hellas-attestation", + "hellas-chain", "hellas-client", "hellas-executor", "hellas-gateway", + "hellas-kernel", "hellas-providers", "hellas-rpc", "hellas-wire", + "hellas-work", + "hex", "iroh", + "serde", + "serde_json", "tempfile", "tokio", "tracing", + "url", ] [[package]] @@ -4830,6 +4918,15 @@ dependencies = [ "objc2-security", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -5058,6 +5155,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -5433,7 +5540,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", - "bit-vec", + "bit-vec 0.8.0", "bitflags", "num-traits", "rand 0.9.4", @@ -5791,6 +5898,20 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "redb" version = "3.1.3" @@ -5979,6 +6100,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "1.1.4" @@ -8027,6 +8157,24 @@ dependencies = [ "spki 0.7.3", ] +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + [[package]] name = "xml-rs" version = "0.8.28" @@ -8042,6 +8190,16 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" version = "0.8.3" diff --git a/crates/chain/src/work_blocks.rs b/crates/chain/src/work_blocks.rs index 7c706907..1a52b33d 100644 --- a/crates/chain/src/work_blocks.rs +++ b/crates/chain/src/work_blocks.rs @@ -476,7 +476,7 @@ mod tests { network: TEST_NETWORK, policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: expected_values(), min_omit_response_blocks: hellas_kernel::MIN_OMIT_RESPONSE_BLOCKS, } @@ -893,7 +893,7 @@ mod tests { payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: expected_values(), }) { Ok(descriptor) => descriptor, diff --git a/crates/chain/src/work_e2e.rs b/crates/chain/src/work_e2e.rs index 681daf2e..f8d6b674 100644 --- a/crates/chain/src/work_e2e.rs +++ b/crates/chain/src/work_e2e.rs @@ -128,7 +128,7 @@ use hellas_rpc::{ use hellas_wire::mux::{MessagePipe, MuxConfig, MuxTransport, Role as MuxRole}; use hellas_wire::{DefaultClock, Dispatcher, StreamTransport as _, TransportContext}; use hellas_work::work::{ - BackendFault, ClientEndpoint, CloseEndpoint, JobProposal, PaidEvaluateBackend, PaymentError, + BackendFault, ClientEndpoint, CloseEndpoint, JobProposal, PaidWorkBackend, PaymentError, PreparedEvaluateInput, RunOutcome, WorkService, propose_work, run_accepted_work, }; use hellas_work::work_close::{CloseProgress, TxSink, close_start}; @@ -396,7 +396,7 @@ fn provider_policy() -> ProviderChannelPolicy { network: TEST_NETWORK, policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: expected_values(), min_omit_response_blocks: hellas_kernel::MIN_OMIT_RESPONSE_BLOCKS, } @@ -409,7 +409,7 @@ fn descriptor(allocations: &[(SettlementKey, u64)]) -> WorkChannelDescriptor { payment_terms: payment_terms(allocations), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: expected_values(), }) { Ok(descriptor) => descriptor, @@ -901,7 +901,7 @@ struct ProviderBackend { prompt: Vec, } -impl PaidEvaluateBackend for ProviderBackend { +impl PaidWorkBackend for ProviderBackend { fn evaluate( &self, input: PreparedEvaluateInput, @@ -1276,7 +1276,7 @@ async fn run_one_paid_job(devnet: &Devnet, opened: &mut Opened) -> u64 { calls: Arc::clone(&client_calls), }; let proposal = JobProposal { - prepared_input: prepared_input(), + prepared_input: prepared_input().into(), deadlines: deadlines(), }; diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 7fc11240..d049bf07 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -23,6 +23,8 @@ apple-app-attest = [] # is driven from; without it the runner has a chain and no way to ask it # anything. node = [ + "dep:hellas-sdk", + "hellas-sdk/paid-work", "chain", "dep:async-stream", "dep:axum", @@ -96,6 +98,7 @@ hellas-kernel = { workspace = true, default-features = false, optional = true } hellas-presentation = { workspace = true, optional = true } hellas-private.workspace = true hellas-providers.workspace = true +hellas-sdk = { workspace = true, default-features = false, optional = true } hellas-rpc = { workspace = true, default-features = false, features = [ "all-protocols", "work", diff --git a/crates/cli/src/commands/paid_work.rs b/crates/cli/src/commands/paid_work.rs index 79226ae1..053c3a2a 100644 --- a/crates/cli/src/commands/paid_work.rs +++ b/crates/cli/src/commands/paid_work.rs @@ -5,46 +5,35 @@ //! production setup and channel journals; this command does not keep a //! parallel receipt or invent a second protocol. -use std::net::SocketAddr; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::{Duration, Instant}; - use anyhow::{Context as _, bail}; use clap::{Args, Subcommand}; -use hellas_chain::client::{RemoteLightClient, VerifiedRemoteLightClient}; +use hellas_chain::client::RemoteLightClient; use hellas_chain::{ - ConsensusInfo, ConsensusVerifier, FinalizedBlockQuery, FinalizedBlockView, - FinalizedWorkView as _, LightClient as _, WorkBlocks, WorkChannelQuery, -}; -use hellas_client::work::payment::pay_for_result; -use hellas_client::work::{CollectResultOutcome, collect_result}; -use hellas_kernel::{ - CoinId, EdgeId, Funding, List, MAX_PARTY_INPUTS, MAX_START_VALIDITY_BLOCKS, Secp256k1Signer, - Secp256k1Verifier, WorkPaymentTerms, + ConsensusInfo, ConsensusVerifier, FinalizedBlockQuery, FinalizedBlockView, LightClient as _, }; -use hellas_rpc::protocol::artifacts::{Canonical as _, PreparedPaidInputV1}; -use hellas_rpc::protocol::work::{ - JobDeadlines, generation_policy_digest, identity_source_digest, private_policy_commitment, -}; -use hellas_rpc::protocol::work_setup::{ProviderChannelPolicy, WorkChannelDescriptor}; -use hellas_wire::ServiceMarker; -use hellas_wire::iroh::IrohTransport; -use hellas_work::work::{ClientEndpoint, JobProposal, propose_work, resume_work_proposal}; -use hellas_work::work_close::CloseProgress; -use hellas_work::work_close::FinalizedBlocks as _; -use hellas_work::work_handshake::{ - PaymentAdmission, SetupEndpoint, SetupService, apply_setup_exchange, prepare_setup_exchange, - send_setup_exchange, +use hellas_kernel::{CoinId, EdgeId, Funding, List, MAX_PARTY_INPUTS, Secp256k1Signer}; +use hellas_rpc::protocol::artifacts::PreparedPaidInputV1; +#[cfg(test)] +use hellas_rpc::protocol::work::{JobDeadlines, private_policy_commitment}; +use hellas_rpc::protocol::work_fetch::PreparedPaidFetchInputV1; +use hellas_rpc::protocol::work_profile::PreparedPaidWorkInput; +#[cfg(feature = "gateway")] +use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; +#[cfg(feature = "gateway")] +use hellas_sdk::paid_client::{ + PaidWorkResult as PaidOutput, check_evaluate_input as check_policy_input, }; -use hellas_work::work_open::{SetupAdvance, SetupProgress}; +use hellas_sdk::paid_client::{PaidWorkSession as OpenPaidChannel, bind_paid_endpoint}; use hellas_work::work_store::journal::MAX_RECORD_BYTES; -use hellas_work::work_store::{Role, SetupScan, SetupStore}; -use iroh::endpoint::presets; -use iroh::{Endpoint, EndpointAddr, EndpointId, SecretKey, TransportAddr}; +use iroh::{EndpointId, SecretKey}; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +#[cfg(feature = "gateway")] +use std::sync::atomic::AtomicBool; +use std::time::Duration; use super::CliResult; -use super::serve::work_config::{WorkConfig, load_work_config}; +use super::serve::work_config::load_work_config; #[cfg(feature = "gateway")] mod gateway; @@ -54,6 +43,8 @@ pub use gateway::load_gateway_backend; /// Paid-work commands intended for deployment bring-up and smoke tests. #[derive(Debug, Subcommand)] pub enum PaidWorkCommand { + /// Build a signed ephemeral Fetch input for a paid channel. + PrepareFetch(PrepareFetchArgs), /// Build canonical paid input from a causal-LM environment and prompt. #[cfg(feature = "llm")] PrepareInput(PrepareInputArgs), @@ -65,6 +56,26 @@ pub enum PaidWorkCommand { Run(Box), } +#[derive(Debug, Args)] +pub struct PrepareFetchArgs { + #[arg(long)] + service: String, + #[arg(long)] + method: String, + /// The trusted transformation to execute. + #[arg(long, value_parser = ["openai-responses", "codex-responses", "http"])] + execution_environment: String, + /// Assurance authenticated before the paid request is disclosed. + #[arg(long, default_value = "producer-signed", value_parser = ["producer-signed", "apple-app-attest"])] + assurance: String, + /// Provider-shaped UTF-8 JSON request, read from an ordinary file. + #[arg(long, value_name = "FILE")] + payload_file: PathBuf, + /// Client-owned output file containing the signed request. + #[arg(long, value_name = "FILE")] + out: PathBuf, +} + #[cfg(feature = "llm")] #[derive(Debug, Args)] pub struct PrepareInputArgs { @@ -109,6 +120,13 @@ pub struct InspectChainArgs { #[derive(Clone, Debug, Args)] pub struct RunArgs { + /// Out-of-band provider enrollment pin (required for App Attest). + #[arg(long)] + provider_genesis: Option, + #[arg(long)] + apple_app_id: Option, + #[arg(long, value_delimiter = ',', value_parser = crate::parse_hex_array::<32>)] + apple_cd_hashes: Vec<[u8; 32]>, /// Provider work configuration, including chain identity and policy. #[arg(long = "work-config", value_name = "FILE")] work_config: PathBuf, @@ -137,7 +155,7 @@ pub struct RunArgs { #[arg(long = "omission-bond")] omission_bond: u64, - /// Canonical PreparedPaidInputV1 bytes to execute. + /// Canonical prepared Evaluate or Fetch input to execute. #[arg(long = "prepared-input", value_name = "FILE")] prepared_input: PathBuf, @@ -171,8 +189,10 @@ pub async fn run( command: PaidWorkCommand, transport_key: SecretKey, settlement_key: Secp256k1Signer, + producer_key: hellas_rpc::ProducerSigningKey, ) -> CliResult<()> { match command { + PaidWorkCommand::PrepareFetch(args) => prepare_fetch(args, &producer_key), #[cfg(feature = "llm")] PaidWorkCommand::PrepareInput(args) => prepare_input(args, &transport_key, &settlement_key), PaidWorkCommand::InspectInput(args) => { @@ -192,6 +212,39 @@ pub async fn run( } } +fn prepare_fetch(args: PrepareFetchArgs, key: &hellas_rpc::ProducerSigningKey) -> CliResult<()> { + let environment = match args.execution_environment.as_str() { + "openai-responses" => hellas_rpc::FetchEnvironment::OpenAiResponses, + "codex-responses" => hellas_rpc::FetchEnvironment::CodexResponses, + "http" => hellas_rpc::FetchEnvironment::Http, + _ => bail!("unsupported fetch environment"), + }; + let payload = super::fetch::load_payload_file(&args.payload_file)?; + if environment == hellas_rpc::FetchEnvironment::Http { + hellas_rpc::http_fetch::HttpFetchRequest::decode(&payload)?; + } + let assurance = match args.assurance.as_str() { + "producer-signed" => hellas_rpc::Assurance::ProducerSigned, + "apple-app-attest" => hellas_rpc::Assurance::AppleAppAttest, + _ => bail!("unsupported assurance"), + }; + let events = hellas_rpc::fetch::build_input_events_with_retention( + &args.service, + &args.method, + &payload, + environment.manifest_id(), + assurance, + key, + hellas_rpc::Retention::Ephemeral, + )?; + let prepared = PreparedPaidFetchInputV1::new(&events, &environment.manifest())?; + write_private(&args.out, &prepared.encode()?)?; + println!("prepared_input: {}", args.out.display()); + println!("allowed_environment: {}", environment.manifest_id()); + println!("provider_payload_retention: memory-only"); + Ok(()) +} + #[cfg(feature = "llm")] fn prepare_input( args: PrepareInputArgs, @@ -257,8 +310,27 @@ fn inspect_input( transport_key: &SecretKey, settlement_key: &Secp256k1Signer, ) -> CliResult<()> { - let prepared = read_prepared_input(path)?; - inspect_prepared(&prepared, transport_key, settlement_key) + match read_prepared_work_input(path)? { + PreparedPaidWorkInput::Evaluate(prepared) => { + inspect_prepared(&prepared, transport_key, settlement_key) + } + PreparedPaidWorkInput::Fetch(prepared) => { + let parts = prepared.parts()?; + let request = hellas_rpc::fetch::verify_input_events(&parts.fetch_input_transcript)?; + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "profile": "fetch", + "service": request.service, + "method": request.method, + "allowed_environment": parts.manifest.content_id().to_string(), + "caller_key": hex::encode(request.caller_key.bytes()), + "provider_payload_retention": "memory-only", + }))? + ); + Ok(()) + } + } } fn inspect_prepared( @@ -281,36 +353,7 @@ fn inspect_prepared( Ok(()) } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct InputIdentities { - allowed_environment: hellas_rpc::ContentId, - generation_policy_digest: hellas_rpc::Digest, - identity_source_digest: hellas_rpc::Digest, -} - -impl InputIdentities { - fn from_prepared(prepared: &PreparedPaidInputV1) -> CliResult { - let parts = prepared - .parts() - .context("prepared input contains a non-canonical body")?; - let allowed_environment = parts.manifest.content_id(); - anyhow::ensure!( - parts.evaluate_request.execution_environment == allowed_environment, - "prepared input request names environment {}, but its manifest derives {}", - parts.evaluate_request.execution_environment, - allowed_environment, - ); - Ok(Self { - allowed_environment, - generation_policy_digest: generation_policy_digest( - &parts.text_policy.canonical_bytes(), - )?, - identity_source_digest: identity_source_digest( - &parts.identity_artifact.canonical_bytes(), - )?, - }) - } -} +use hellas_sdk::paid_client::InputIdentities; async fn inspect_chain(validators: &[String]) -> CliResult<()> { anyhow::ensure!( @@ -377,25 +420,73 @@ async fn inspect_chain(validators: &[String]) -> CliResult<()> { Ok(()) } +async fn open_paid_channel( + args: &RunArgs, + endpoint: iroh::Endpoint, + settlement_key: Secp256k1Signer, + assurance: hellas_rpc::Assurance, +) -> CliResult { + anyhow::ensure!( + !args.payment_coins.is_empty(), + "at least one --payment-coin is required" + ); + let provider_trust = + if args.provider_genesis.is_some() || assurance != hellas_rpc::Assurance::ProducerSigned { + Some(crate::identity::provider_trust( + args.provider_genesis, + assurance, + args.apple_app_id.clone(), + args.apple_cd_hashes.clone(), + )?) + } else { + None + }; + OpenPaidChannel::open( + hellas_sdk::paid_client::PaidWorkOptions { + config: load_work_config(&args.work_config)?, + journal_root: args.journal_root.clone(), + provider: args.provider, + provider_addrs: args.provider_addrs.clone(), + provider_trust, + bond: edge_id("--bond", &args.bond)?, + payment_funding: Funding::new(coins(&args.payment_coins)?, empty_coins()), + omission_bond: args.omission_bond, + acceptance_blocks: args.acceptance_blocks, + terminal_blocks: args.terminal_blocks, + payment_blocks: args.payment_blocks, + timeout: Duration::from_secs(args.timeout_secs), + }, + endpoint, + settlement_key, + ) + .await +} + async fn run_one( args: RunArgs, transport_key: SecretKey, settlement_key: Secp256k1Signer, ) -> CliResult<()> { - let prepared = read_prepared_input(&args.prepared_input)?; + let prepared = read_prepared_work_input(&args.prepared_input)?; let endpoint = bind_paid_endpoint(transport_key).await?; - let mut channel = OpenPaidChannel::open(args, endpoint, settlement_key).await?; + let mut channel = open_paid_channel( + &args, + endpoint.clone(), + settlement_key, + prepared.assurance()?, + ) + .await?; println!( "bond_edge: {}", - hex::encode(channel.descriptor.bond_edge().to_bytes()) + hex::encode(channel.descriptor().bond_edge().to_bytes()) ); println!( "payment_edge: {}", - hex::encode(channel.descriptor.channel().payment_edge().to_bytes()) + hex::encode(channel.descriptor().channel().payment_edge().to_bytes()) ); println!( "channel_id: {}", - hex::encode(channel.descriptor.channel().id().as_bytes()) + hex::encode(channel.descriptor().channel().id().as_bytes()) ); let result = channel .run(Some(prepared), false, None) @@ -405,819 +496,44 @@ async fn run_one( println!("job_price: {}", result.job_price); println!("credited_cumulative: {}", result.credited_cumulative); println!("authenticated_result: true"); - if let Some(output) = channel.args.output.as_ref() { - std::fs::write(output, &result.transcript).with_context(|| { - format!("failed to write result transcript to {}", output.display()) - })?; - println!( - "result: {} ({} bytes)", - output.display(), - result.transcript.len() - ); - } else { - println!("result_bytes: {}", result.transcript.len()); - } - if channel.args.settle { - channel - .client - .prepare_close() - .context("failed to prepare the client payment close")?; - loop { - match channel - .client - .advance_close(&channel.chain, &channel.chain) - .await - .context("failed to advance the client payment close")? - { - CloseProgress::Settled { provider_payout } => { - println!("settled: true"); - println!("settled_provider_payout: {provider_payout}"); - println!( - "settled_finalized_height: {}", - channel.client.state().cursor().0 - ); - break; - } - CloseProgress::Submitted { outcome, .. } => { - tracing::info!(?outcome, "client payment close submitted"); - } - CloseProgress::Opened { .. } | CloseProgress::Nothing => {} - } - tokio::time::sleep(channel.config.poll).await; - } - } else { - println!("settled: false"); - } - println!("client_journals: {}", channel.args.journal_root.display()); - Ok(()) -} - -struct PaidOutput { - work_id: hellas_rpc::Digest, - job_price: u64, - credited_cumulative: u64, - transcript: Vec, - #[cfg(feature = "gateway")] - provider_key: hellas_rpc::PublicKey, - #[cfg(feature = "gateway")] - input: hellas_rpc::InputCommitment, -} - -struct OpenPaidChannel { - args: RunArgs, - config: WorkConfig, - descriptor: WorkChannelDescriptor, - dialer: ProviderDialer, - chain: WorkBlocks, - next_validator: usize, - client: ClientEndpoint, - needs_recovery: bool, -} - -impl OpenPaidChannel { - async fn open( - args: RunArgs, - endpoint: Endpoint, - settlement_key: Secp256k1Signer, - ) -> CliResult { - anyhow::ensure!( - args.acceptance_blocks > 0 && args.terminal_blocks > 0 && args.payment_blocks > 0, - "all three deadline spans must be greater than zero", - ); - anyhow::ensure!( - !args.payment_coins.is_empty(), - "at least one --payment-coin is required", - ); - - let config = load_work_config(&args.work_config)?; - let policy = config.provider_policy(); - let bond = edge_id("--bond", &args.bond)?; - let payment_funding = Funding::new(coins(&args.payment_coins)?, empty_coins()); - let mut next_validator = 0; - let chain = connect_chain(&config, &mut next_validator).await?; - check_genesis(&config, &chain).await?; - - std::fs::create_dir_all(&args.journal_root).with_context(|| { - format!( - "failed to create client journal root {}", - args.journal_root.display(), - ) - })?; - let store = SetupStore::open( - &args.journal_root, - config.chain.network, - bond, - Role::Client, - &Secp256k1Verifier::new(), - ) - .with_context(|| { - format!( - "failed to open client setup journal under {}", - args.journal_root.display(), - ) - })?; - let mut setup = SetupEndpoint::new( - store, - settlement_key.clone(), - PaymentAdmission::Proposes(Box::new(policy.clone())), - ); - let dialer = ProviderDialer::new(args.provider, args.provider_addrs.clone(), endpoint); - - if setup.state().revision().is_none() { - exchange_setup(&dialer, &mut setup).await?; - } - let bundle = setup - .state() - .bundle() - .cloned() - .context("provider returned no bond proposal")?; - anyhow::ensure!( - bundle.bond_edge() == bond, - "provider proposed a different bond edge" - ); - anyhow::ensure!( - bundle.bond_terms().parties.taker() == settlement_key.party_key(), - "provider bond names client settlement key {}, not this identity's {}", - hex::encode(bundle.bond_terms().parties.taker().to_bytes()), - hex::encode(settlement_key.party_key().to_bytes()), - ); - if setup.state().scan_armed().is_none() { - setup.arm_scan(finalized_floor(&chain).await?)?; - } - if setup.state().revision() == Some(1) { - let terms = payment_terms(&config, &policy, &bundle, args.omission_bond); - setup.propose_payment(payment_funding, terms)?; - } - if setup.state().revision() == Some(2) { - exchange_setup(&dialer, &mut setup).await?; - } - anyhow::ensure!( - setup.state().revision() == Some(3), - "setup did not reach its countersigned revision", - ); - - let setup_service = SetupService::new(setup); - let (mounted, descriptor) = - drive_setup(&setup_service, &policy, &chain, config.poll).await?; - let ready = ready_channel(&descriptor, &chain).await?; - let client = ClientEndpoint::new(ready.clone(), mounted, settlement_key)?; - - Ok(Self { - args, - config, - descriptor, - dialer, - chain, - next_validator, - client, - needs_recovery: true, - }) - } - - async fn follow_chain(&mut self) -> CliResult<()> { - for attempt in 0..self.config.validators.len() { - match self.client.catch_up(&self.chain).await { - Ok(_) => return Ok(()), - Err(error) if attempt + 1 == self.config.validators.len() => { - return Err(error.into()); - } - Err(error) => { - tracing::debug!(%error, "paid channel will continue catch-up through another validator"); - self.chain = connect_chain(&self.config, &mut self.next_validator).await?; - check_genesis(&self.config, &self.chain).await?; - } - } - } - bail!("no configured validator supplied finalized history") - } - - async fn run( - &mut self, - prepared: Option, - recover: bool, - progress: Option, - ) -> CliResult> { - self.run_with_admission(prepared, recover, progress, None) - .await - } - - async fn run_with_admission( - &mut self, - prepared: Option, - recover: bool, - progress: Option, - proposed: Option<&AtomicBool>, - ) -> CliResult> { - self.follow_chain().await?; - let Self { - args, - config, - descriptor, - dialer, - chain, - client, - needs_recovery, - .. - } = self; - if let Some(prepared) = prepared.as_ref() { - check_policy_input(&config.provider_policy(), prepared)?; - } - let ready = caught_up_channel(descriptor, client, &*chain).await?; - if recover && *needs_recovery { - if let Some(payment) = client.state().last_payment() { - // The provider may have committed payment while its acknowledgement - // was lost. Re-send the retained certificate before accepting work. - pay_for_result(dialer.work().await?, client, payment.work_id).await?; - } - let pending = client - .state() - .jobs() - .filter(|job| { - // The journal forbids signing payment after this height. - // Keep the evidence, but do not let an unpayable old job - // prevent this channel from serving a new request. Retained - // certificates are re-sent separately above. - let payable = job.authorization().payment_deadline >= client.state().cursor().0; - if !payable { - tracing::info!( - work_id = %hex::encode(job.work_id().as_bytes()), - payment_deadline = job.authorization().payment_deadline, - "retaining expired unpaid job without retrying execution", - ); - } - payable - }) - .map(|job| { - PreparedPaidInputV1::decode(job.prepared_input(), MAX_RECORD_BYTES) - .map(|input| (job.work_id(), job.phase(), input)) - }) - .collect::, _>>()?; - for (work_id, phase, pending) in pending { - let result = execute_paid_job( - args, - pending, - dialer, - client, - &ready, - &*chain, - config.poll, - None, - JobLookup::Retained(work_id), - None, - ) - .await; - if let Err(error) = &result - && ((phase == hellas_work::work_store::JobPhase::HalfSigned - && matches!( - error.downcast_ref::(), - Some(hellas_work::work::ProposeError::Refused { - refusal, - .. - }) if !refusal.is_retryable() - )) - || permanently_refused_delivery(error)) - { - // Keep the signed evidence without deciding that an unpaid - // job was paid or cancelled. A permanent provider refusal - // cannot be repaired by blocking every later request here. - tracing::info!(%work_id, %error, "retaining an unpaid job refused by the provider"); - continue; - } - result?; - } - } - // Keep recovery armed across any error or cancellation after acceptance. - *needs_recovery = prepared.is_some(); - let result = match prepared { - Some(prepared) => { - let ready = caught_up_channel(descriptor, client, &*chain).await?; - Some( - execute_paid_job( - args, - prepared, - dialer, - client, - &ready, - &*chain, - config.poll, - progress.as_ref(), - if recover { - JobLookup::New - } else { - JobLookup::PreparedInput - }, - proposed, - ) - .await?, - ) - } - None => None, - }; - *needs_recovery = false; - Ok(result) - } -} - -fn permanently_refused_delivery(error: &anyhow::Error) -> bool { - use hellas_client::work::CollectResultError; - use hellas_work::work::DeliverError; - let delivery = error.downcast_ref::().or_else(|| { - match error.downcast_ref::() { - Some(CollectResultError::Deliver(delivery)) => Some(delivery), - _ => None, - } - }); - matches!(delivery, Some(DeliverError::Refused { refusal, .. }) if !refusal.is_retryable()) -} - -/// Proposes until a provider accepts or the caller's execution window closes. -/// A provider catching its chain cursor up replies `NotReady`; that is not an -/// answer to the job. The retained proposal makes each retry the same request, -/// while bounded exponential backoff avoids turning recovery into a request -/// flood. -async fn propose_when_ready( - dialer: &ProviderDialer, - client: &mut ClientEndpoint, - proposal: &JobProposal, - retained: Option, - poll: Duration, - timeout: Duration, - proposed: Option<&AtomicBool>, -) -> CliResult { - let deadline = Instant::now() + timeout; - let mut delay = poll.max(Duration::from_secs(1)); - loop { - if Instant::now() >= deadline { - bail!("provider remained not ready for {timeout:?}"); - } - let transport = dialer.work().await?; - // Once a proposal can leave this process, a lost acknowledgement must - // be treated as accepted work. HTTP cancellation may no longer stop it. - if let Some(proposed) = proposed { - proposed.store(true, Ordering::Release); - } - let result = match retained { - Some(work_id) => resume_work_proposal(transport, client, work_id).await, - None => propose_work(transport, client, proposal).await, - }; - match result { - Ok(work_id) => return Ok(work_id), - Err(hellas_work::work::ProposeError::Refused { refusal, .. }) - if refusal.is_retryable() => - { - let remaining = deadline.saturating_duration_since(Instant::now()); - tokio::time::sleep(delay.min(remaining)).await; - delay = delay.saturating_mul(2).min(Duration::from_secs(10)); - } - Err(error) => return Err(error.into()), - } + if let Some(output) = &args.output { + write_private(output, &result.transcript)?; } -} - -enum JobLookup { - Retained(hellas_rpc::Digest), - PreparedInput, - New, -} - -#[allow(clippy::too_many_arguments)] -async fn execute_paid_job( - args: &RunArgs, - prepared: PreparedPaidInputV1, - dialer: &ProviderDialer, - client: &mut ClientEndpoint, - ready: &hellas_rpc::protocol::work_setup::ReadyChannel, - chain: &WorkBlocks, - poll: Duration, - progress: Option<&hellas_work::work::PaidProgress>, - lookup: JobLookup, - proposed: Option<&AtomicBool>, -) -> CliResult { - let readiness_timeout = Duration::from_secs(args.timeout_secs); - let prepared_bytes = prepared.encode()?; - let current = client.state().cursor().0; - let deadlines = relative_deadlines(current, args)?; - let proposal = JobProposal { - prepared_input: prepared, - deadlines, - }; - let existing = client - .state() - .jobs() - .filter(|job| match &lookup { - JobLookup::Retained(work_id) => job.work_id() == *work_id, - JobLookup::New => false, - JobLookup::PreparedInput => { - job.prepared_input() == prepared_bytes.as_slice() - && job.authorization().payment_deadline >= current - && (job.phase() != hellas_work::work_store::JobPhase::HalfSigned - || job.authorization().acceptance_deadline >= current) - } - }) - .map(|job| (job.work_id(), job.phase(), *job.authorization())) - .collect::>(); - anyhow::ensure!( - existing.len() <= 1, - "more than one active job matches this prepared input; inspect the retained channel journal", - ); - if !existing.is_empty() - && let Some(proposed) = proposed - { - proposed.store(true, Ordering::Release); + println!("result_bytes: {}", result.transcript.len()); + if args.settle { + println!("settled_provider_payout: {}", channel.settle().await?); } - let (work_id, already_collected) = match existing.first().copied() { - Some((work_id, hellas_work::work_store::JobPhase::HalfSigned, _)) => ( - propose_when_ready( - dialer, - client, - &proposal, - Some(work_id), - poll, - readiness_timeout, - proposed, - ) - .await?, - false, - ), - Some((work_id, hellas_work::work_store::JobPhase::Ready, _)) - | Some((work_id, hellas_work::work_store::JobPhase::Matched, _)) => (work_id, true), - Some((work_id, _, _)) => (work_id, false), - None => ( - propose_when_ready( - dialer, - client, - &proposal, - None, - poll, - readiness_timeout, - proposed, - ) - .await?, - false, - ), - }; - let transcript = if already_collected { - client - .state() - .job_by_id(work_id) - .map(|job| job.transcript().to_vec()) - .context("collected job disappeared from its journal")? - } else if let Some(progress) = progress { - let mut emitted = false; - let delivery = loop { - let result = hellas_work::work::fetch_result_stream( - dialer.work().await?, - client, - ready, - work_id, - |event| { - emitted = true; - progress(event.clone()).map_err(|error| { - hellas_rpc::protocol::work::PaidWorkError::Transcript(error.to_string()) - }) - }, - ) - .await; - let error = match result { - Ok(delivery) => break delivery, - Err(error) => error, - }; - let retryable = match &error { - hellas_work::work::DeliverError::Transport(status) => { - status.code == hellas_wire::WireCode::Unavailable - } - hellas_work::work::DeliverError::Refused { refusal, .. } => refusal.is_retryable(), - _ => false, - }; - // Retry delivery of this accepted job only before exposing output. - // Reopening after a prefix would replay it into the user's stream. - if emitted || !retryable { - return Err(error.into()); - } - client.catch_up(chain).await?; - let job = client - .state() - .job_by_id(work_id) - .context("accepted job disappeared")?; - anyhow::ensure!( - client.state().cursor().0 <= job.authorization().payment_deadline, - "payment deadline elapsed while waiting for result stream" - ); - tracing::debug!(%error, %work_id, "waiting for paid result stream readiness"); - tokio::time::sleep(poll.max(Duration::from_secs(1))).await; - }; - client.catch_up(chain).await?; - anyhow::ensure!( - client.state().cursor().0 <= proposal.deadlines.payment, - "payment deadline elapsed during delivery" - ); - delivery.transcript - } else { - collect_until_ready(dialer, client, ready, chain, work_id, poll).await? - }; - let credited = pay_for_result(dialer.work().await?, client, work_id).await?; - Ok(PaidOutput { - work_id, - job_price: ready.execution_policy().fixed_price, - credited_cumulative: credited, - transcript, - #[cfg(feature = "gateway")] - provider_key: hellas_rpc::PublicKey::Secp256k1(ready.channel().provider_key().to_bytes()), - #[cfg(feature = "gateway")] - input: hellas_rpc::evaluate::input_commitment( - &proposal.prepared_input.parts()?.evaluate_request, - ), - }) -} - -fn check_policy_input( - policy: &ProviderChannelPolicy, - prepared: &PreparedPaidInputV1, -) -> CliResult<()> { - let input = InputIdentities::from_prepared(prepared)?; - let parts = prepared.parts()?; - let expected = policy.execution_policy; - anyhow::ensure!( - expected.allowed_environment == input.allowed_environment, - "work config allows environment {}, but prepared input uses {}", - expected.allowed_environment, - input.allowed_environment, - ); - anyhow::ensure!( - hellas_rpc::protocol::work::matches_generation_policy(&expected, &parts.text_policy)?, - "work config generation_policy_digest does not match prepared input", - ); - anyhow::ensure!( - expected.identity_source_digest == input.identity_source_digest, - "work config identity_source_digest does not match prepared input", - ); + println!("settled: {}", args.settle); + println!("client_journals: {}", args.journal_root.display()); + endpoint.close().await; Ok(()) } -fn payment_terms( - config: &WorkConfig, - policy: &ProviderChannelPolicy, - bundle: &hellas_rpc::protocol::work_bundle::WorkChannelSetupBundleV1, - omission_bond: u64, -) -> WorkPaymentTerms { - WorkPaymentTerms { - bond_edge: bundle.bond_edge(), - bond_terms: bundle.bond_terms().clone(), - private_policy_commitment: private_policy_commitment( - config.chain.network, - &policy.policy_salt, - &policy.channel_policy, - ), - omit_response_blocks: policy.min_omit_response_blocks, - start_validity_blocks: MAX_START_VALIDITY_BLOCKS, - omission_bond, - } -} - -async fn drive_setup( - setup: &SetupService, - policy: &ProviderChannelPolicy, - chain: &WorkBlocks, - poll: Duration, -) -> CliResult<(hellas_work::work_store::ChannelStore, WorkChannelDescriptor)> { - loop { - let SetupAdvance { progress, mounted } = setup - .advance_setup(chain, chain, chain) - .await - .context("failed to advance paid-work setup")?; - if let Some(store) = mounted { - let channel = store.state().channel(); - let descriptor = policy - .admit(channel.payment_edge(), channel.payment_terms().clone()) - .context("the funded channel no longer satisfies the configured policy")?; - return Ok((store, descriptor)); - } - match progress { - SetupProgress::Aborted(reason) => bail!("paid-work setup aborted: {reason:?}"), - SetupProgress::Faulted(reason) => bail!("paid-work setup faulted: {reason:?}"), - SetupProgress::TimeoutBond => bail!("provider bond timed out before setup completed"), - _ => tokio::time::sleep(poll).await, - } - } -} - -async fn ready_channel( - descriptor: &WorkChannelDescriptor, - chain: &WorkBlocks, -) -> CliResult { - let query = WorkChannelQuery { - bond_edge: descriptor.bond_edge(), - payment_edge: descriptor.channel().payment_edge(), - funding: Default::default(), - }; - let snapshot = chain - .work_channel_snapshot(query) - .await? - .context("no finalized channel snapshot is available")?; - descriptor - .check_ready(&snapshot.observed_channel()) - .context("the finalized channel is not ready") -} - -/// Reads the ready snapshot once the client has processed it. -/// -/// The snapshot and the blocks the cursor follows are answered by -/// validators independently, so the snapshot can name a height the -/// light client has not finalized yet, and a snapshot read after the -/// catch-up on a moving chain always lands a few blocks ahead of it. -/// The snapshot is sampled first and then the cursor is brought to it: -/// a fixed height is a target the catch-up reaches, where a fresh -/// snapshot every round was not. A state at or behind the cursor is -/// the direction `check_caught_up` accepts. -async fn caught_up_channel( - descriptor: &WorkChannelDescriptor, - client: &mut ClientEndpoint, - chain: &WorkBlocks, -) -> CliResult { - let ready = ready_channel(descriptor, chain).await?; - for _ in 0..16 { - let cursor = client.catch_up(chain).await?; - if ready.check_caught_up(cursor).is_ok() { - return Ok(ready); - } - tokio::time::sleep(Duration::from_millis(500)).await; - } - let cursor = client.catch_up(chain).await?; - ready.check_caught_up(cursor)?; - Ok(ready) -} - -async fn collect_until_ready( - dialer: &ProviderDialer, - client: &mut ClientEndpoint, - ready: &hellas_rpc::protocol::work_setup::ReadyChannel, - chain: &WorkBlocks, - work_id: hellas_rpc::Digest, - poll: Duration, -) -> CliResult> { - loop { - match collect_result(dialer.work().await?, client, ready, chain, work_id).await? { - CollectResultOutcome::Collected(result) => return Ok(result.transcript), - CollectResultOutcome::NotReady { reason } => { - tracing::debug!(%reason, "waiting for paid result"); - tokio::time::sleep(poll.max(Duration::from_secs(1))).await; - } - } - } -} - +#[cfg(test)] fn relative_deadlines(current: u64, args: &RunArgs) -> CliResult { - let acceptance = current - .checked_add(args.acceptance_blocks) - .context("acceptance deadline overflow")?; - let terminal = acceptance - .checked_add(args.terminal_blocks) - .context("terminal deadline overflow")?; - let payment = terminal - .checked_add(args.payment_blocks) - .context("payment deadline overflow")?; - Ok(JobDeadlines { - acceptance, - terminal, - payment, - }) -} - -async fn exchange_setup(dialer: &ProviderDialer, setup: &mut SetupEndpoint) -> CliResult<()> { - let request = prepare_setup_exchange(setup); - let response = send_setup_exchange(dialer.setup().await?, request).await?; - apply_setup_exchange(setup, response)?; - Ok(()) -} - -struct ProviderDialer { - endpoint: Endpoint, - provider: EndpointAddr, -} - -async fn bind_paid_endpoint(secret_key: SecretKey) -> CliResult { - Endpoint::builder(presets::N0) - .secret_key(secret_key) - .alpns(vec![ - hellas_rpc::services::work_setup::WorkSetup::ALPN - .as_bytes() - .to_vec(), - hellas_rpc::services::work::Work::ALPN.as_bytes().to_vec(), - ]) - .bind() - .await - .context("failed to bind paid-work Iroh endpoint") -} - -impl ProviderDialer { - fn new(provider: EndpointId, addresses: Vec, endpoint: Endpoint) -> Self { - Self { - endpoint, - provider: EndpointAddr::from_parts( - provider, - addresses.into_iter().map(TransportAddr::Ip), - ), - } - } - - async fn setup(&self) -> CliResult { - self.connect(hellas_rpc::services::work_setup::WorkSetup::ALPN.as_bytes()) - .await - } - - async fn work(&self) -> CliResult { - self.connect(hellas_rpc::services::work::Work::ALPN.as_bytes()) - .await - } - - async fn connect(&self, alpn: &[u8]) -> CliResult { - let connection = self - .endpoint - .connect(self.provider.clone(), alpn) - .await - .with_context(|| format!("failed to connect to provider {}", self.provider.id))?; - Ok(IrohTransport::new(connection)) - } -} - -async fn connect_chain( - config: &WorkConfig, - next_validator: &mut usize, -) -> CliResult> { - let verifier = ConsensusVerifier::new(&ConsensusInfo { - validators: config.validators.clone(), - threshold_identity: config.chain.threshold_identity.clone(), - network_id: config.chain.network.as_str().to_owned(), - }) - .context("configured threshold identity is unusable")?; - // A peer can accept connections while lacking a historical certificate. - // Reconnects must make progress through the configured alternatives. - let start = *next_validator; - let mut failures = Vec::new(); - for url in config - .validators - .iter() - .cycle() - .skip(start) - .take(config.validators.len()) - { - *next_validator = (*next_validator + 1) % config.validators.len(); - match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { - Ok(client) => return Ok(WorkBlocks::new(client)), - Err(error) => failures.push(format!("{url}: {error}")), - } - } - bail!("no configured validator answered: {}", failures.join("; ")) -} - -async fn check_genesis( - config: &WorkConfig, - chain: &WorkBlocks, -) -> CliResult<()> { - let first = chain - .block_at(1) - .await? - .context( - "configured validator has no finalized block 1; genesis cannot be authenticated until block 1 is finalized", - )?; - check_genesis_payload( - config.chain.genesis_payload_digest.as_bytes(), - &first.parent, + hellas_sdk::paid_client::deadlines( + current, + args.acceptance_blocks, + args.terminal_blocks, + args.payment_blocks, ) } +#[cfg(test)] +use hellas_sdk::paid_client::check_genesis_payload; -fn check_genesis_payload(expected: &[u8; 32], actual: &[u8; 32]) -> CliResult<()> { - anyhow::ensure!( - actual == expected, - "validator genesis payload {} does not match configured {}", - hex::encode(actual), - hex::encode(expected), - ); - Ok(()) -} - -async fn finalized_floor(chain: &WorkBlocks) -> CliResult { - let height = chain - .latest_height() - .await? - .context("configured validator has finalized no blocks")?; - let block = chain - .block_at(height) - .await? - .context("configured validator did not return its finalized tip")?; - Ok(SetupScan { - height, - payload: block.payload, - }) +fn read_prepared_work_input(path: &Path) -> CliResult { + let bytes = super::read_bounded_regular_file(path, "prepared paid input", MAX_RECORD_BYTES)?; + PreparedPaidWorkInput::decode(&bytes, MAX_RECORD_BYTES) + .map_err(|error| anyhow::anyhow!("invalid prepared paid input {}: {error}", path.display())) } +#[cfg(test)] fn read_prepared_input(path: &Path) -> CliResult { let bytes = super::read_bounded_regular_file(path, "prepared paid input", MAX_RECORD_BYTES)?; PreparedPaidInputV1::decode(&bytes, MAX_RECORD_BYTES) .map_err(|error| anyhow::anyhow!("invalid prepared paid input {}: {error}", path.display())) } -#[cfg(feature = "llm")] fn write_private(path: &Path, bytes: &[u8]) -> CliResult<()> { if let Some(parent) = path .parent() diff --git a/crates/cli/src/commands/paid_work/gateway.rs b/crates/cli/src/commands/paid_work/gateway.rs index dc94e838..cb30fbd6 100644 --- a/crates/cli/src/commands/paid_work/gateway.rs +++ b/crates/cli/src/commands/paid_work/gateway.rs @@ -10,6 +10,7 @@ use hellas_rpc::protocol::artifacts::{ BoundTermId, InputAddressed as _, OutputAddressed as _, SourceRef, TextArtifact, TextExecution, TextPolicy, TokenIds, }; +use iroh::Endpoint; use serde::Deserialize; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -273,6 +274,9 @@ pub async fn load_gateway_backend( providers.push(Arc::new(Provider { policy: config.provider_policy(), args: RunArgs { + provider_genesis: None, + apple_app_id: None, + apple_cd_hashes: Vec::new(), work_config: provider.work_config, journal_root: provider.journal_root, provider: provider.provider, @@ -455,7 +459,7 @@ impl PaidGateway { if session.is_none() { match connect_before_deadline( &sender, streamed, deadline, PROVIDER_CONNECTION_TIMEOUT, - OpenPaidChannel::open(provider.args.clone(), endpoint.clone(), settlement_key.clone()), + open_paid_channel(&provider.args, endpoint.clone(), settlement_key.clone(), hellas_rpc::Assurance::ProducerSigned), ).await { Ok(opened) => { @@ -477,7 +481,7 @@ impl PaidGateway { } } let session = session.as_mut().expect("channel was opened"); - if prepared.is_some() && session.needs_recovery { + if prepared.is_some() && session.needs_recovery() { let recovery_deadline = deadline.min( tokio::time::Instant::now() + RECOVERY_ATTEMPT_TIMEOUT, ); @@ -508,9 +512,9 @@ impl PaidGateway { // ClientEndpoint journals the proposal nonce before releasing // its signature. Recovery and a failed dial need not propose // this request; a lost acceptance response does advance it. - let proposal_nonce = session.client.state().proposal_nonce_high_water(); + let proposal_nonce = session.state().proposal_nonce_high_water(); let already_proposed = prepared_bytes.as_ref().is_some_and(|input| { - session.client.state().jobs().any(|job| job.prepared_input() == input) + session.state().jobs().any(|job| job.prepared_input() == input) }); let request_session = &mut *session; let input = prepared.clone(); @@ -518,7 +522,7 @@ impl PaidGateway { let result = before_proposal( &sender, streamed, deadline, |proposed| async move { - request_session.run_with_admission(input, true, on_progress, Some(&proposed)).await + request_session.run_with_admission(input.map(Into::into), true, on_progress, Some(&proposed)).await }, ).await .and_then(|result| result) @@ -526,7 +530,7 @@ impl PaidGateway { .map(Option::unwrap_or_default); if let Err(error) = &result { if !recovery && error.is::() - && session.client.state().proposal_nonce_high_water() == proposal_nonce + && session.state().proposal_nonce_high_water() == proposal_nonce { return result; } @@ -545,7 +549,7 @@ impl PaidGateway { Some(hellas_work::work::ProposeError::Store(_)), ); if !already_proposed && !uncertain_append - && session.client.state().proposal_nonce_high_water() == proposal_nonce + && session.state().proposal_nonce_high_water() == proposal_nonce { provider.connection_failed(); tracing::debug!(provider = %provider.args.provider, error = %format!("{error:#}"), @@ -902,6 +906,7 @@ fn output_events(output: PaidOutput) -> CliResult> { #[cfg(test)] mod tests { use super::*; + use iroh::Endpoint; #[tokio::test] async fn disconnected_queued_request_never_starts_work() { @@ -1033,7 +1038,10 @@ mod tests { let gateway = PaidGateway { providers: Vec::new(), next: AtomicUsize::new(0), - endpoint: Endpoint::builder(presets::Minimal).bind().await.unwrap(), + endpoint: Endpoint::builder(iroh::endpoint::presets::Minimal) + .bind() + .await + .unwrap(), settlement_key: Secp256k1Signer::from_secret_scalar([7; 32]).unwrap(), admission: Arc::new(Semaphore::new(1)), tasks: Mutex::new(Vec::new()), diff --git a/crates/cli/src/commands/paid_work/tests.rs b/crates/cli/src/commands/paid_work/tests.rs index f21cbdf5..32c60768 100644 --- a/crates/cli/src/commands/paid_work/tests.rs +++ b/crates/cli/src/commands/paid_work/tests.rs @@ -2,38 +2,12 @@ use super::*; use hellas_kernel::NetworkId; use hellas_rpc::peers::PeerId; -#[test] -fn recovery_skips_only_permanent_delivery_refusals() { - use hellas_client::work::CollectResultError; - use hellas_work::work::{DeliverError, WorkRefusal}; - - for (refusal, permanent) in [ - (WorkRefusal::Declined, true), - (WorkRefusal::Expired, true), - (WorkRefusal::NotReady, false), - (WorkRefusal::Unavailable, false), - ] { - let delivery = || DeliverError::Refused { - refusal, - reason: "provider diagnostic".to_owned(), - }; - assert_eq!(permanently_refused_delivery(&delivery().into()), permanent); - assert_eq!( - permanently_refused_delivery(&CollectResultError::Deliver(delivery()).into()), - permanent, - ); - } - assert!(!permanently_refused_delivery( - &DeliverError::Malformed("result").into() - )); - assert!(!permanently_refused_delivery(&anyhow::anyhow!( - "connection lost" - ))); -} - #[test] fn relative_deadlines_are_ordered_from_the_current_cursor() { let args = RunArgs { + provider_genesis: None, + apple_app_id: None, + apple_cd_hashes: Vec::new(), work_config: "work.json".into(), journal_root: "journal".into(), provider: SecretKey::generate().public(), @@ -180,3 +154,39 @@ fn prepare_input_builds_a_bundle_from_an_environment_and_prompt() { environment.manifest().content_id(), ); } +#[test] +fn prepare_fetch_signs_an_ephemeral_client_request() { + let root = tempfile::tempdir().unwrap(); + let payload_file = root.path().join("request.json"); + let out = root.path().join("input.bin"); + std::fs::write(&payload_file, br#"{"input":"private prompt"}"#).unwrap(); + let key = hellas_rpc::ProducerSigningKey::from_secret_bytes([7; 32]).unwrap(); + prepare_fetch( + PrepareFetchArgs { + assurance: "producer-signed".into(), + service: "openai".into(), + method: "responses".into(), + execution_environment: "openai-responses".into(), + payload_file, + out: out.clone(), + }, + &key, + ) + .unwrap(); + let PreparedPaidWorkInput::Fetch(input) = read_prepared_work_input(&out).unwrap() else { + panic!("a Fetch bundle"); + }; + let parts = input.parts().unwrap(); + let request = hellas_rpc::fetch::verify_input_events(&parts.fetch_input_transcript).unwrap(); + assert_eq!(request.retention, hellas_rpc::Retention::Ephemeral); + assert_eq!(request.caller_key, key.public_key()); + assert_eq!(request.execution_environment, parts.manifest.content_id()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + assert_eq!( + std::fs::metadata(out).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } +} diff --git a/crates/cli/src/commands/serve/mod.rs b/crates/cli/src/commands/serve/mod.rs index 94b6525f..7f65371a 100644 --- a/crates/cli/src/commands/serve/mod.rs +++ b/crates/cli/src/commands/serve/mod.rs @@ -350,6 +350,10 @@ struct FetchConfigRoute { #[derive(Debug, Deserialize)] #[serde(tag = "type", rename_all = "kebab-case", deny_unknown_fields)] enum FetchDestination { + /// Caller-signed HTTPS URL and TLS settings with operator-owned account aliases. + Http { + config: hellas_providers::HttpProviderConfig, + }, /// Official Codex Responses, authenticated by the local Codex OAuth store. CodexResponses { #[serde(default)] @@ -371,6 +375,7 @@ impl FetchDestination { fn into_entry(self, capabilities: FetchRoutePolicy) -> CliResult { let (environment, provider): (FetchEnvironment, Arc) = match self { + Self::Http { config } => return Ok(config.into_entry(capabilities)?), Self::CodexResponses { auth_path } => ( FetchEnvironment::CodexResponses, Arc::new(codex_provider::CodexResponsesFetchProvider::new( diff --git a/crates/cli/src/commands/serve/node.rs b/crates/cli/src/commands/serve/node.rs index db002236..bd7eb017 100644 --- a/crates/cli/src/commands/serve/node.rs +++ b/crates/cli/src/commands/serve/node.rs @@ -8,18 +8,14 @@ //! owned by the service-discovery path and is not started from this //! bootstrap. -use std::collections::BTreeMap; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; +#[cfg(test)] +use std::sync::Mutex; use std::time::Duration; use anyhow::Context; -use futures::future::BoxFuture; -use hellas_chain::client::VerifiedRemoteLightClient; -use hellas_chain::work_blocks::{PaidWorkClockError, advance_paid_work_clock}; -use hellas_chain::{ - ConsensusInfo, ConsensusVerifier, FinalizedWorkView, WorkBlocks, WorkChannelQuery, -}; +use hellas_chain::FinalizedWorkView; #[cfg(feature = "evaluate")] use hellas_executor::ArtifactStoreConfig; #[cfg(feature = "evaluate")] @@ -29,53 +25,58 @@ use hellas_executor::{ FetchAccessPolicy, FetchQuotaStoreBackend, FetchRouteRegistry, FetchServer, FetchTranscriptStoreBackend, }; +#[cfg(test)] use hellas_kernel::{EdgeId, NetworkId, Secp256k1Signer, Secp256k1Verifier}; use hellas_rpc::cache::control::CacheController; use hellas_rpc::open::OpenDispatcher; +#[cfg(test)] use hellas_rpc::pb::work::{ AcceptWorkRequest, AcceptWorkResponse, AdmitCertificateRequest, AdmitCertificateResponse, DeliverResultRequest, DeliverResultResponse, ExchangeSetupRequest, ExchangeSetupResponse, WorkRefusalCode, WorkRefused, accept_work_response, admit_certificate_response, deliver_result_response, exchange_setup_response, }; -use hellas_rpc::peers::{PeerDirectory, PeerId, PeerManager}; +use hellas_rpc::peers::PeerId; +use hellas_rpc::peers::{PeerDirectory, PeerManager}; use hellas_rpc::policy::ExecutePolicy; -use hellas_rpc::protocol::Digest; -use hellas_rpc::protocol::work::{ - PaidJobAuthorizationV1, PrivateRecord as _, work_id as accepted_work_id, -}; -use hellas_rpc::protocol::work_setup::{ - ProviderChannelPolicy, ReadyChannel, WorkChannelDescriptor, -}; use hellas_rpc::serve::{AccountingDispatcher, AdminPolicy, Authorized, MethodDispatcher}; use hellas_rpc::services::cache_control::{CacheControl, CacheControlServer}; use hellas_rpc::services::courtesy::{Courtesy, Open as CourtesyOpen}; use hellas_rpc::services::execute::RunTicket; use hellas_rpc::services::fetch::{Fetch, Open as FetchOpen}; use hellas_rpc::services::node::{Node, NodeServer}; -use hellas_rpc::services::work::{Work, WorkHandler, WorkServer}; -use hellas_rpc::services::work_setup::{WorkSetup, WorkSetupHandler, WorkSetupServer}; +use hellas_rpc::services::work::{Work, WorkServer}; +use hellas_rpc::services::work_setup::{WorkSetup, WorkSetupServer}; +#[cfg(test)] +use hellas_rpc::services::{work::WorkHandler, work_setup::WorkSetupHandler}; use hellas_rpc::{Assurance, ProducerSigningKey}; use hellas_wire::iroh::{IrohTransport, IrohTransportError}; -use hellas_wire::{Dispatcher, ServiceMarker, StreamTransport, TransportContext, WireStatus}; -use hellas_work::work::{ - CloseEndpoint, PaidEvaluateBackend, RunError, RunOutcome, WorkService, run_accepted_work, -}; -use hellas_work::work_close::{CatchUpError, FinalizedBlocks, TxSink}; -use hellas_work::work_handshake::{PaymentAdmission, SetupEndpoint, SetupService}; -use hellas_work::work_open::{SetupAdvance, SetupDriveError, SetupProgress, SetupView}; -use hellas_work::work_store::{ChannelStore, JobPhase, Role, SetupStore, discover_setups}; +use hellas_wire::{Dispatcher, ServiceMarker, StreamTransport}; +#[cfg(test)] +use hellas_wire::{TransportContext, WireStatus}; +#[cfg(test)] +use hellas_work::work::PaidWorkBackend; +use hellas_work::work_close::FinalizedBlocks; +#[cfg(test)] +use hellas_work::work_close::TxSink; +#[cfg(test)] +use hellas_work::work_handshake::{PaymentAdmission, SetupEndpoint}; +#[cfg(test)] +use hellas_work::work_open::SetupView; +#[cfg(test)] +use hellas_work::work_store::{ChannelStore, JobPhase, Role, SetupStore}; use iroh::{Endpoint, EndpointId, SecretKey, endpoint::Connection, endpoint::presets}; -use tokio::sync::{Mutex as AsyncMutex, Semaphore, mpsc, oneshot}; +use tokio::sync::{Semaphore, mpsc, oneshot}; use tokio::task::{JoinHandle, JoinSet}; use tracing::{debug, info, warn}; use super::node_handler::NodeHandlerImpl; -use super::work_config::WorkRoutes; use crate::commands::discovery::{DiscoveryAdvertiser, served_alpns, start_server_advertising}; use crate::identity::OpenIdentity; -type ProductionWorkSource = WorkBlocks; +pub(super) use hellas_sdk::paid_provider::{ + MountedSetup, MountedWork, ProductionWorkSource, UnmountedWork, WorkRunner, WorkRunnerConfig, +}; /// Keep peer-controlled transport state finite. A connection can multiplex /// several RPCs, so these are deliberately transport limits rather than job @@ -199,29 +200,30 @@ pub(super) async fn spawn_node(config: NodeConfig) -> anyhow::Result .with_store(FetchQuotaStoreBackend::fs( config.artifact_store_path.join("fetch-quota"), )); - let handle = Executor::spawn_configured(ExecutorSpawnConfig { - output_cache: config.output_cache, - execute_policy: config.execute_policy, - queue_capacity: config.queue_size, - metrics: config.metrics.clone(), - producer_key: Arc::new(config.producer_key), - provider_genesis: Arc::new(config.provider_genesis), - assurance: config.assurance, - fetch_access_policy, - fetch_routes: config.fetch_routes, - fetch_max_in_flight: config.fetch_max_in_flight, - fetch_queue_capacity: config.fetch_queue_size, - fetch_replay_max_in_flight: config.fetch_replay_max_in_flight, - fetch_store, - #[cfg(feature = "evaluate")] - artifact_store: config.artifact_store, - #[cfg(feature = "evaluate")] - content_store: config.content_store, - #[cfg(feature = "evaluate")] - gpu_config: config.gpu_config, - }) - .await - .context("failed to spawn executor")?; + let mut executor = ExecutorSpawnConfig::fetch_only( + Arc::new(config.producer_key), + Arc::new(config.provider_genesis), + config.assurance, + config.fetch_routes, + ); + executor.output_cache = config.output_cache; + executor.execute_policy = config.execute_policy; + executor.queue_capacity = config.queue_size; + executor.metrics = config.metrics.clone(); + executor.fetch_access_policy = fetch_access_policy; + executor.fetch_max_in_flight = config.fetch_max_in_flight; + executor.fetch_queue_capacity = config.fetch_queue_size; + executor.fetch_replay_max_in_flight = config.fetch_replay_max_in_flight; + executor.fetch_store = fetch_store; + #[cfg(feature = "evaluate")] + { + executor.artifact_store = config.artifact_store; + executor.content_store = config.content_store; + executor.gpu_config = config.gpu_config; + } + let handle = Executor::spawn_configured(executor) + .await + .context("failed to spawn executor")?; let advertised_alpns = served_alpns(config.work.is_some()); let mut alpns = advertised_alpns.clone(); if !admin_policy.peers.is_empty() { @@ -434,11 +436,23 @@ where { match setup.service(&context) { Some(mounted) => { - let server = AccountingDispatcher::new(WorkSetupServer(mounted), manager); + let server = AccountingDispatcher::new( + OpenDispatcher::<_, _, hellas_rpc::services::work_setup::Open>::new( + WorkSetupServer(mounted), + remote_execution.open_identity.clone(), + ), + manager, + ); serve_loop(transport, server).await } None => { - let server = AccountingDispatcher::new(WorkSetupServer(UnmountedWork), manager); + let server = AccountingDispatcher::new( + OpenDispatcher::<_, _, hellas_rpc::services::work_setup::Open>::new( + WorkSetupServer(UnmountedWork), + remote_execution.open_identity.clone(), + ), + manager, + ); serve_loop(transport, server).await } } @@ -448,11 +462,23 @@ where // is still the bounded retryable `NotReady` §3 left here. match work.handler(&context) { Some(mounted) => { - let server = AccountingDispatcher::new(WorkServer(mounted), manager); + let server = AccountingDispatcher::new( + OpenDispatcher::<_, _, hellas_rpc::services::work::Open>::new( + WorkServer(mounted), + remote_execution.open_identity.clone(), + ), + manager, + ); serve_loop(transport, server).await } None => { - let server = AccountingDispatcher::new(WorkServer(UnmountedWork), manager); + let server = AccountingDispatcher::new( + OpenDispatcher::<_, _, hellas_rpc::services::work::Open>::new( + WorkServer(UnmountedWork), + remote_execution.open_identity.clone(), + ), + manager, + ); serve_loop(transport, server).await } } @@ -462,1130 +488,6 @@ where } } -#[derive(Clone, Copy, Debug)] -struct UnmountedWork; - -fn not_ready() -> WorkRefused { - WorkRefused { - code: WorkRefusalCode::NotReady as i32, - reason: "work state is not mounted".to_string(), - } -} - -impl WorkSetupHandler for UnmountedWork { - fn exchange_setup( - &self, - _request: ExchangeSetupRequest, - _context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - core::future::ready(Ok(ExchangeSetupResponse { - outcome: Some(exchange_setup_response::Outcome::Refused(not_ready())), - })) - } -} - -impl WorkHandler for UnmountedWork { - fn accept_work( - &self, - _request: AcceptWorkRequest, - _context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - core::future::ready(Ok(AcceptWorkResponse { - outcome: Some(accept_work_response::Outcome::Refused(not_ready())), - })) - } - - fn deliver_result( - &self, - _request: DeliverResultRequest, - _context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - core::future::ready(Ok(DeliverResultResponse { - outcome: Some(deliver_result_response::Outcome::Refused(not_ready())), - })) - } - - async fn stream_result( - &self, - _request: DeliverResultRequest, - _context: TransportContext, - ) -> Result { - Err(WireStatus::new( - hellas_wire::WireCode::Unavailable, - "paid work channel is not mounted", - )) - } - - fn admit_certificate( - &self, - _request: AdmitCertificateRequest, - _context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - core::future::ready(Ok(AdmitCertificateResponse { - outcome: Some(admit_certificate_response::Outcome::Refused(not_ready())), - })) - } -} - -// ── The clock ───────────────────────────────────────────────────────── -// -// Everything below is a runner and nothing below is a decision. It -// builds no transaction, fixes no deadline, chooses no settlement, -// judges no duty due, and does not decide whether admission is on: each -// of those is a library edge it calls on a cadence, and the cadence is -// the whole of what this file adds. What it owns is *when* — and the -// journals, which is why it hands them to nobody. - -/// What the clock over one node's paid-work journals is built from. -/// -/// Every field is something the serve path has already loaded and -/// checked. The policy most of all: it is the loaded work -/// configuration's, carried here rather than derived again, so there is -/// no second place a node could decide what it countersigns over. -pub(super) struct WorkRunnerConfig { - /// The network the journals are keyed and the signatures bound to. - pub(super) network: NetworkId, - /// The threshold identity finalized blocks must authenticate under. - pub(super) threshold_identity: Vec, - /// The configured root the setup journals live under. - pub(super) journal_root: PathBuf, - /// Bilateral routes from authenticated peers to owned journals. - pub(super) routes: WorkRoutes, - /// The validator RPCs a read and a submission go to. - pub(super) validators: Vec, - /// How often the clock ticks. - pub(super) poll: Duration, - /// The key every settlement this node signs is signed with. - pub(super) settlement_key: Secp256k1Signer, - /// What every setup endpoint this node builds countersigns over. - pub(super) policy: ProviderChannelPolicy, -} - -/// The channel this node answers `Work` from, once the runner has been -/// handed one. -/// -/// Written by the runner and read by the accept loop. What crosses is a -/// clone of a handler whose mutable pieces are themselves behind `Arc`s, -/// so the lock is held for a clone and never across a request: the -/// dispatch path never waits while holding the mount, and the clock never -/// waits on a request. -#[derive(Clone)] -pub(super) struct MountedWork { - mounted: Arc>>>>, - driver: Option, -} - -impl Default for MountedWork { - fn default() -> Self { - Self { - mounted: Arc::new(Mutex::new(BTreeMap::new())), - driver: None, - } - } -} - -/// A cloneable, type-erased owner of the backend that runs accepted work. -/// -/// The production value owns an [`hellas_executor::ExecutorHandle`]. -/// Keeping the backend behind this narrow local seam means the clock and -/// ALPN dispatcher stay parameterized only over their finalized source; -/// neither has a second opinion about paid admission or execution failure. -#[derive(Clone)] -struct AcceptedWorkDriver(Arc); - -trait DriveAcceptedWork: Send + Sync { - fn run( - &self, - service: WorkService, - ready: ReadyChannel, - work_id: Digest, - ) -> BoxFuture<'static, Result>; -} - -struct BackendWorkDriver { - backend: Arc, -} - -impl DriveAcceptedWork for BackendWorkDriver -where - B: PaidEvaluateBackend + Send + Sync + 'static, -{ - fn run( - &self, - service: WorkService, - ready: ReadyChannel, - work_id: Digest, - ) -> BoxFuture<'static, Result> { - let backend = Arc::clone(&self.backend); - Box::pin( - async move { run_accepted_work(&service, &ready, backend.as_ref(), work_id).await }, - ) - } -} - -impl AcceptedWorkDriver { - fn new(backend: B) -> Self - where - B: PaidEvaluateBackend + Send + Sync + 'static, - { - Self(Arc::new(BackendWorkDriver { - backend: Arc::new(backend), - })) - } - - /// Starts one accepted job without lending its lifetime to either the - /// request path or the close clock. - fn spawn(&self, service: WorkService, ready: ReadyChannel, work_id: Digest) { - let running = self.0.run(service, ready, work_id); - let span = hellas_rpc::request_span!(target: "hellas_request", "paid.provider.execute", hellas.work.id = ?work_id, otel.status_code = tracing::field::Empty); - tokio::spawn(tracing::Instrument::instrument( - async move { - match running.await { - Ok(RunOutcome::Completed { .. }) => { - debug!(?work_id, "the accepted paid job completed") - } - Ok(RunOutcome::Ready { .. }) => { - debug!(?work_id, "the accepted paid job was already complete") - } - Ok(RunOutcome::Running) => { - debug!(?work_id, "the accepted paid job was already running") - } - Ok(RunOutcome::Indeterminate) => { - warn!( - ?work_id, - "the accepted paid job is indeterminate after restart" - ) - } - // `run_accepted_work` has already made backend and - // transcript faults terminal before returning them. The - // remaining errors have no node-local terminal policy; - // keep the exact failure visible to the operator. - Err(error) => { - tracing::Span::current().record("otel.status_code", "ERROR"); - warn!(?work_id, %error, "the accepted paid job did not complete"); - } - } - }, - span, - )); - } -} - -/// One mounted channel's served handler. -/// -/// `source` is replaceable because the runner redials a failed validator. -/// The request path copies the current source under the plain mutex and -/// drops that guard before its coherent read awaits. `accepting` spans the -/// complete fresh-read-to-signature sequence, so two acceptance attempts -/// cannot each refresh and then race to consume the same channel credit. -#[derive(Clone)] -struct MountedWorkService { - bond_edge: EdgeId, - service: WorkService, - descriptor: Option, - source: Arc>, - accepting: Arc>, - driver: Option, -} - -impl MountedWorkService -where - S: FinalizedBlocks + FinalizedWorkView + Sync, -{ - /// Re-establishes admission from one fresh coherent read. - /// - /// The service is the exact clone the runner drives. Its cursor is - /// checked after readiness, and that same service receives the fresh - /// decision before the raw handler is reached. A missing policy, - /// failed read, failed predicate, lagging cursor, or endpoint failure - /// therefore leaves the request on the retryable `NotReady` side. - async fn refresh_admission(&self) -> anyhow::Result { - // A `std::sync::MutexGuard` is deliberately confined to this - // block. Holding the source-slot guard across the read would make - // this handler's future non-`Send` and is not a valid dispatch. - let source = { - let held = self - .source - .lock() - .map_err(|_| anyhow::anyhow!("the finalized source lock is poisoned"))?; - held.clone() - }; - refresh_work_admission(&self.service, self.descriptor.as_ref(), &source).await - } - - async fn refresh_delivery(&self, request: &DeliverResultRequest) -> anyhow::Result<()> { - let Ok(bytes) = request.work_id.as_slice().try_into() else { - return Ok(()); - }; - let work_id = Digest::from_bytes(bytes); - let active = self - .service - .with_state(|state| state.job_by_id(work_id).is_some()); - if !matches!(active, Ok(true)) { - return Ok(()); - } - // A restarted mount has no readiness cached. A retained result must - // be collectable without first accepting another job. Terminal replies - // need no fresh admission and are authenticated by the core service. - let _accepting = self.accepting.lock().await; - let result = self.refresh_admission().await; - if let Err(error) = &result { - debug!(%error, "a delivery attempt found no fresh channel readiness"); - } - result.map(|_| ()) - } -} - -/// Re-establishes admission for the exact driven channel from one coherent -/// finalized read. -/// -/// Both the wire handler and restart recovery call this function. A recovered -/// job therefore gets no weaker interpretation of readiness than a new job, -/// and neither path can accidentally trust the readiness cached at mount. -async fn refresh_work_admission( - service: &WorkService, - descriptor: Option<&WorkChannelDescriptor>, - source: &S, -) -> anyhow::Result -where - S: FinalizedBlocks + FinalizedWorkView + Sync, -{ - let Some(descriptor) = descriptor else { - anyhow::bail!("this channel has no admission descriptor"); - }; - let query = WorkChannelQuery { - bond_edge: descriptor.bond_edge(), - payment_edge: descriptor.channel().payment_edge(), - funding: Default::default(), - }; - let Some(snapshot) = source - .work_channel_snapshot(query.clone()) - .await - .context("the fresh coherent channel read failed")? - else { - anyhow::bail!("no finalized channel snapshot is available"); - }; - if snapshot.query() != &query { - anyhow::bail!("the finalized source answered for another channel"); - } - let ready = descriptor - .check_ready(&snapshot.observed_channel()) - .context("the fresh channel snapshot is not ready")?; - // Keep one snapshot as the target. Recovery runs on the clock itself, - // so waiting for another tick here would prevent the cursor advancing. - // Take the service's existing driver when available; a concurrent clock - // drive keeps that authority until it finishes its own catch-up. - for _ in 0..16 { - let cursor = service - .with_state(|state| state.cursor().0) - .context("the mounted channel cursor is unavailable")?; - if ready.check_caught_up(cursor).is_ok() { - break; - } - if let Ok(mut driver) = service.drive() { - driver - .catch_up(source) - .await - .context("the mounted channel could not catch up to the fresh snapshot")?; - } - tokio::time::sleep(Duration::from_millis(250)).await; - } - let cursor = service - .with_state(|state| state.cursor().0) - .context("the mounted channel cursor is unavailable")?; - ready - .check_caught_up(cursor) - .context("the mounted channel has not caught up to the fresh snapshot")?; - service - .admit_new_work(ready.clone()) - .context("the driven work service refused its fresh readiness")?; - Ok(ready) -} - -impl WorkHandler for MountedWorkService -where - S: FinalizedBlocks + FinalizedWorkView + Sync, -{ - async fn accept_work( - &self, - request: AcceptWorkRequest, - _context: TransportContext, - ) -> Result> + Send, WireStatus> - { - if let Some(response) = self.service.precheck_acceptance(&request) { - return Ok(response); - } - let _accepting = self.accepting.lock().await; - // A preceding request or the clock may have resolved this proposal - // while admission was serialized. Retained replies need no fresh read. - if let Some(response) = self.service.precheck_acceptance(&request) { - return Ok(response); - } - let ready = match self.refresh_admission().await { - Ok(ready) => ready, - Err(error) => { - debug!(%error, "an acceptance attempt found no fresh channel readiness"); - return Ok(AcceptWorkResponse { - outcome: Some(accept_work_response::Outcome::Refused(WorkRefused { - code: WorkRefusalCode::NotReady as i32, - reason: "fresh channel readiness is unavailable".to_string(), - })), - }); - } - }; - // Derive the id from the request while the accepted response - // is still only a possibility. The response carries only the - // provider signature, and consulting `state.jobs().next()` after it - // leaves would race the clock terminalizing that same job. - let work_id = self - .service - .with_state(|state| { - PaidJobAuthorizationV1::decode(&request.authorization) - .ok() - .map(|authorization| accepted_work_id(state.channel(), &authorization)) - }) - .ok() - .flatten(); - let response = self.service.accept(&request); - if matches!( - response.outcome.as_ref(), - Some(accept_work_response::Outcome::Accepted(_)) - ) { - match (self.driver.as_ref(), work_id) { - (Some(driver), Some(work_id)) => { - driver.spawn(self.service.clone(), ready, work_id); - } - (None, Some(work_id)) => { - warn!(?work_id, "accepted paid work has no execution backend") - } - (_, None) => warn!("accepted paid work has no mounted job to execute"), - } - } - Ok(response) - } - - async fn deliver_result( - &self, - request: DeliverResultRequest, - context: TransportContext, - ) -> Result> + Send, WireStatus> - { - let response: hellas_rpc::call::WithTrailer = - if self.refresh_delivery(&request).await.is_ok() { - self.service.deliver_result(request, context).await?.into() - } else { - DeliverResultResponse { - outcome: Some(deliver_result_response::Outcome::Refused(WorkRefused { - code: WorkRefusalCode::NotReady as i32, - reason: "fresh channel readiness is unavailable".to_string(), - })), - } - .into() - }; - Ok(response) - } - - async fn stream_result( - &self, - request: DeliverResultRequest, - context: TransportContext, - ) -> Result { - self.refresh_delivery(&request).await.map_err(|_| { - WireStatus::new( - hellas_wire::WireCode::Unavailable, - "fresh channel readiness is unavailable", - ) - })?; - let mounted = self.clone(); - let mut stream = self.service.stream_result(request.clone(), context).await?; - Ok(Box::pin(async_stream::try_stream! { - let mut refresh = tokio::time::interval(Duration::from_secs(1)); - loop { - let event = tokio::select! { - event = futures::StreamExt::next(&mut stream) => Some(event), - _ = refresh.tick() => None, - }; - // Even buffered prefixes need a fresh finalized decision before - // leaving the node; a contest can start after this stream opens. - mounted.refresh_delivery(&request).await.map_err(|_| WireStatus::new( - hellas_wire::WireCode::Unavailable, - "fresh channel readiness is unavailable", - ))?; - if let Some(event) = event { - match event { - Some(event) => yield event?, - None => break, - } - } - } - })) - } - - fn admit_certificate( - &self, - request: AdmitCertificateRequest, - context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - self.service.admit_certificate(request, context) - } -} - -impl MountedWork { - fn with_backend(backend: B) -> Self - where - B: PaidEvaluateBackend + Send + Sync + 'static, - { - Self { - mounted: Arc::new(Mutex::new(BTreeMap::new())), - driver: Some(AcceptedWorkDriver::new(backend)), - } - } - - /// Adds one owned channel under its authenticated peer. - /// - /// A peer is served only while exactly one channel is mounted under - /// it. Retaining a second candidate rather than overwriting either one - /// makes an ambiguity fail closed instead of turning insertion order - /// into routing policy. - fn mount( - &self, - peer: PeerId, - bond_edge: EdgeId, - service: &WorkService, - descriptor: Option, - accepting: Arc>, - source: &S, - ) -> bool { - match self.mounted.lock() { - Ok(mut held) => { - let mounted = held.entry(peer).or_default(); - mounted.push(MountedWorkService { - bond_edge, - service: service.clone(), - descriptor, - source: Arc::new(Mutex::new(source.clone())), - accepting, - driver: self.driver.clone(), - }); - mounted.len() == 1 - } - Err(_) => false, - } - } - - /// The one handler mounted for the transport-vouched peer. - fn handler(&self, context: &TransportContext) -> Option> { - let peer = context - .vouched_peer() - .map(|peer| PeerId::from_bytes(peer.0))?; - self.mounted.lock().ok().and_then(|held| { - let [mounted] = held.get(&peer)?.as_slice() else { - return None; - }; - Some(mounted.clone()) - }) - } - - /// The mounted channel's service, reached through the served - /// handler. - /// - /// Only the tests below ask for it. Serving takes [`Self::handler`], - /// and the clock drives the very same [`WorkService`] out of - /// [`Driven::Channel`] without asking the mount for it — so this - /// exists to let a test read the state of what the node is actually - /// answering from, and scoping it to `cfg(test)` is what keeps the - /// served slot to the one accessor that serves. - #[cfg(test)] - fn service(&self, context: &TransportContext) -> Option { - self.handler(context).map(|mounted| mounted.service) - } - - /// Replaces the finalized source for the matching driven channel. - /// - /// A reconnect reaches handlers already cloned by live connections, - /// because they share this inner source slot. Neither mount lock is - /// held across a source request. - fn refresh_source(&self, peer: PeerId, bond_edge: EdgeId, source: &S) { - let source_slot = self.mounted.lock().ok().and_then(|held| { - held.get(&peer)? - .iter() - .find(|mounted| mounted.bond_edge == bond_edge) - .map(|mounted| Arc::clone(&mounted.source)) - }); - if let Some(source_slot) = source_slot - && let Ok(mut held) = source_slot.lock() - { - *held = source.clone(); - } - } - - /// Stops serving `Work` from every channel. - /// - /// The clock's last act. A channel nobody is advancing is not a - /// channel to answer from — its journal is closed the moment the - /// runner drops it, and a handler still holding it open would be the - /// one thing keeping the files this process no longer owns. - fn clear_all(&self) { - if let Ok(mut held) = self.mounted.lock() { - held.clear(); - } - } -} - -/// Provider setups this node answers `WorkSetup` from by authenticated peer. -/// -/// Written by discovery and read by the accept loop, beside -/// [`MountedWork`]. The clone in this slot is the exact [`SetupService`] -/// stored in [`Driven::Setup`], so serving and driving share one exclusive -/// journal rather than attempting to reopen it. -#[derive(Clone, Debug, Default)] -pub(super) struct MountedSetup(Arc>>>); - -#[derive(Clone, Debug)] -struct MountedSetupService { - bond_edge: EdgeId, - service: SetupService, -} - -impl MountedSetup { - /// Adds one owned setup under its authenticated peer. - fn mount(&self, peer: PeerId, bond_edge: EdgeId, service: &SetupService) -> bool { - match self.0.lock() { - Ok(mut held) => { - let mounted = held.entry(peer).or_default(); - mounted.push(MountedSetupService { - bond_edge, - service: service.clone(), - }); - mounted.len() == 1 - } - Err(_) => false, - } - } - - /// The exact setup service mounted for the transport-vouched peer, - /// cloned without holding the map across dispatch. - fn service(&self, context: &TransportContext) -> Option { - let peer = context - .vouched_peer() - .map(|peer| PeerId::from_bytes(peer.0))?; - self.0.lock().ok().and_then(|held| { - let [mounted] = held.get(&peer)?.as_slice() else { - return None; - }; - Some(mounted.service.clone()) - }) - } - - /// Stops serving only the setup that made this transition. - fn clear(&self, peer: PeerId, bond_edge: EdgeId) { - if let Ok(mut held) = self.0.lock() - && let Some(mounted) = held.get_mut(&peer) - { - mounted.retain(|mounted| mounted.bond_edge != bond_edge); - if mounted.is_empty() { - held.remove(&peer); - } - } - } - - /// Stops serving every setup during runner shutdown. - fn clear_all(&self) { - if let Ok(mut held) = self.0.lock() { - held.clear(); - } - } -} - -/// One journal, and what the clock drives it as. -/// -/// Two live states and one transition between them: a setup is driven -/// until it hands back the channel it mounted, and from then on the -/// channel is what is driven. Nothing here re-derives a mount — -/// [`SetupAdvance::mounted`] is the only way a [`ChannelStore`] reaches -/// this file, and the setup is not driven again afterwards, because a -/// second step would open a second journal on the same file. -enum Driven { - /// The journal is driven behind the setup service that answers for - /// it. The policy is retained beside the service: it is the provider - /// authority from which the full channel descriptor is rebuilt after - /// the setup reveals its actual terms. - Setup { - /// The endpoint this journal is both driven and served behind. - service: SetupService, - /// The retained provider authority, behind a pointer. - /// - /// Boxed because it is the widest thing this enum carries by a - /// long way — every other payload here is a handle or a store - /// pointer, one or two words each — and a journal is one value - /// with three shapes, so the two that hold no policy would - /// otherwise each be as large as the one that does. - /// [`PaymentAdmission`] already holds it behind the same - /// indirection, and this is built from that one, once per - /// journal at startup. - policy: Box, - }, - /// The channel this setup mounted, including the recovery authority - /// needed to finish a job accepted before a process restart. - Channel(Box), - /// The setup ended, or its mount was refused. Nothing left to - /// drive. - Done, -} - -/// One mounted channel as driven by the paid-work clock. -/// -/// Recovery lives here rather than in the served route: an accepted job is an -/// obligation recorded by this journal even if peer routing changes while the -/// process is down. `accepting` is also lent to the route when one is mounted, -/// so live acceptance and restart recovery serialize their readiness checks. -struct DrivenChannel { - service: WorkService, - descriptor: Option, - accepting: Arc>, - driver: Option, -} - -impl DrivenChannel { - fn accepted_work_id(&self) -> anyhow::Result> { - self.service - .with_state(|state| { - let mut jobs = state.jobs(); - let job = jobs.next()?; - (jobs.next().is_none() && job.phase() == JobPhase::Accepted).then(|| job.work_id()) - }) - .context("the driven channel state is unavailable") - } - - /// Starts a journaled Accepted job after proving current readiness. - /// - /// No in-memory `attempted` marker is needed. A racing live request or - /// clock tick reaches the same endpoint; its durable `JobRunning` record - /// lets exactly one caller receive `Invoke` and every other caller receive - /// `Running`. - async fn resume_accepted(&self, source: &S) -> anyhow::Result - where - S: FinalizedBlocks + FinalizedWorkView + Sync, - { - if self.accepted_work_id()?.is_none() { - return Ok(false); - } - let _accepting = self.accepting.lock().await; - let Some(work_id) = self.accepted_work_id()? else { - return Ok(false); - }; - let driver = self - .driver - .as_ref() - .context("the accepted paid job has no execution backend")?; - let ready = refresh_work_admission(&self.service, self.descriptor.as_ref(), source).await?; - driver.spawn(self.service.clone(), ready, work_id); - Ok(true) - } -} - -/// One setup journal on a clock. -struct SetupClock { - /// The bond this journal stakes, so a log line names which one. - bond_edge: EdgeId, - /// The authenticated peer whose configured route names this bond. - /// Together with `bond_edge`, this is the journal's route identity; - /// `None` keeps an unconfigured owned journal on its close clock - /// without making it a fallback service. - route_peer: Option, - /// What is being driven for it. - driven: Driven, -} - -impl SetupClock { - /// Takes this journal's one step, and says whether the chain - /// answered. - /// - /// A source failure is the only outcome the caller acts on: a - /// validator that stopped answering is dialled again rather than - /// asked forever. Everything else is this journal's own business and - /// is logged where it happens. - async fn tick( - &mut self, - source: &S, - signer: &Secp256k1Signer, - work_mount: &MountedWork, - setup_mount: &MountedSetup, - ) -> bool - where - S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, - { - let bond = hex::encode(self.bond_edge.to_bytes()); - let mut answered = true; - // One step of the setup this journal holds, with the policy the - // mount it may hand back is rebuilt from; nothing, once the - // journal is past its setup. - let step = match &mut self.driven { - Driven::Setup { service, policy } => Some(( - service.advance_setup(source, source, source).await, - policy.clone(), - )), - Driven::Channel(_) | Driven::Done => None, - }; - if let Some((step, policy)) = step { - match step { - Ok(SetupAdvance { progress, mounted }) => { - if let Some(store) = mounted { - if let Some(peer) = self.route_peer { - setup_mount.clear(peer, self.bond_edge); - } - self.take_mount(store, signer, &policy, source, work_mount); - } else if matches!( - progress, - SetupProgress::Aborted(_) | SetupProgress::Faulted(_) - ) { - warn!(bond, ?progress, "this setup ended with no channel to drive"); - self.driven = Driven::Done; - } else { - debug!(bond, ?progress, "the setup advanced"); - } - } - Err(error) => { - answered = !matches!(error, SetupDriveError::Source(_)); - warn!(bond, %error, "this setup did not advance"); - } - } - } - if let Driven::Channel(channel) = &self.driven { - if let Some(peer) = self.route_peer { - work_mount.refresh_source(peer, self.bond_edge, source); - } - if let Err(error) = channel.resume_accepted(source).await { - warn!(bond, %error, "an accepted paid job did not resume"); - } - match advance_paid_work_clock(&channel.service, source).await { - Ok(progress) => debug!(bond, ?progress, "the channel advanced"), - // `resume_accepted` owns the channel while it starts the - // durable execution. The clock's concurrent close pass has - // no work to do until that owner returns the cursor. - Err(PaidWorkClockError::CloseDrive(CatchUpError::Busy)) => { - debug!(bond, "the channel is already being driven") - } - Err(error) => { - answered &= !error.source_failed(); - warn!(bond, %error, "this channel's close did not advance"); - } - } - } - answered - } - - /// Mounts the store the driver handed back. - /// - /// Handed back, never reopened: the journal is exclusive, so a - /// second `ChannelStore::open` on the same file is a refusal rather - /// than a second view, and the settlement and origin this one - /// carries are the ones the completing read established. - fn take_mount( - &mut self, - store: ChannelStore, - signer: &Secp256k1Signer, - policy: &ProviderChannelPolicy, - source: &S, - mount: &MountedWork, - ) { - let bond = hex::encode(self.bond_edge.to_bytes()); - // The setup's retained policy supplies the provider-controlled - // fields, while the mounted channel supplies the payment edge and - // complete terms the two parties actually signed. This is a full - // descriptor reconstruction, not a mount-time readiness cache. - let descriptor = { - let channel = store.state().channel(); - match policy.admit(channel.payment_edge(), channel.payment_terms().clone()) { - Ok(descriptor) => Some(descriptor), - Err(error) => { - warn!(bond, %error, "the mounted channel no longer satisfies its admission policy"); - None - } - } - }; - match CloseEndpoint::new(store, signer.clone()) { - Ok(close) => { - let service = WorkService::close_only(close); - let accepting = Arc::new(AsyncMutex::new(())); - if self.route_peer.is_some_and(|peer| { - mount.mount( - peer, - self.bond_edge, - &service, - descriptor.clone(), - Arc::clone(&accepting), - source, - ) - }) { - info!( - bond, - "this node now answers Work from the channel it mounted" - ); - } else { - warn!( - bond, - "this channel has no unique peer route; it is driven and not served", - ); - } - self.driven = Driven::Channel(Box::new(DrivenChannel { - service, - descriptor, - accepting, - driver: mount.driver.clone(), - })); - } - // The journal and the key are not both the provider's view - // of one channel. Nothing this runner can do about it, and - // dropping the mount is what stops it being reopened every - // tick. - Err(error) => { - warn!(bond, %error, "the mounted channel is not this node's to close"); - self.driven = Driven::Done; - } - } - } -} - -/// The clock, over every paid-work journal this node owns. -pub(super) struct WorkRunner { - clocks: Vec, - signer: Secp256k1Signer, - work_mount: MountedWork, - setup_mount: MountedSetup, - poll: Duration, - validators: Vec, - consensus_verifier: ConsensusVerifier, -} - -impl WorkRunner -where - S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, -{ - /// Opens every setup journal under the configured root. - /// - /// The root and the network are the whole of what a restarting node - /// is told; the bond each journal is about and the role it was - /// written at come out of the files, which is what `discover_setups` - /// is for. A journal that cannot be named is reported and not - /// skipped silently: a file this node cannot open may be a channel - /// it still owes a close. - /// - /// # Errors - /// - /// When the root itself cannot be enumerated. - /// Every owned journal is driven. A configured route additionally - /// mounts its exact setup service under the authenticated peer that - /// names it; an unconfigured journal remains a close duty, not a - /// fallback answer. - pub(super) fn discover( - config: WorkRunnerConfig, - work_mount: MountedWork, - setup_mount: MountedSetup, - ) -> anyhow::Result { - let consensus_verifier = ConsensusVerifier::new(&ConsensusInfo { - validators: config.validators.clone(), - threshold_identity: config.threshold_identity, - network_id: config.network.as_str().to_owned(), - }) - .context("the configured threshold identity is not usable")?; - let settlement_verifier = Secp256k1Verifier::new(); - let found = discover_setups(&config.journal_root, config.network).with_context(|| { - format!( - "failed to enumerate the work journals under {}", - config.journal_root.display(), - ) - })?; - for unnamed in &found.unidentified { - warn!( - path = %unnamed.path.display(), - reason = %unnamed.reason, - "a setup journal under the work root could not be named", - ); - } - let mut clocks = Vec::with_capacity(found.setups.len()); - for setup in found.setups { - let bond = hex::encode(setup.bond_edge.to_bytes()); - let route_peer = config - .routes - .iter() - .find(|route| route.bond == setup.bond_edge) - .map(|route| route.peer); - // A close capability binds the provider half, and this - // process holds the provider's key. A client journal beside - // this node's own is another party's, and this runner has - // nothing to sign for it. - if setup.role != Role::Provider { - warn!( - bond, - "a setup journal under the work root is not this node's half" - ); - continue; - } - let store = match SetupStore::open( - &config.journal_root, - config.network, - setup.bond_edge, - setup.role, - &settlement_verifier, - ) { - Ok(store) => store, - Err(error) => { - warn!(bond, %error, "a discovered setup journal did not open"); - continue; - } - }; - let policy = Box::new(config.policy.clone()); - let service = SetupService::new(SetupEndpoint::new( - store, - config.settlement_key.clone(), - PaymentAdmission::Admits(policy.clone()), - )); - if let Some(peer) = route_peer { - if setup_mount.mount(peer, setup.bond_edge, &service) { - info!( - bond, - "this node now answers WorkSetup from its driven setup" - ); - } else { - warn!(bond, "this provider setup has an ambiguous peer route"); - } - } else { - warn!(bond, "this provider setup has no configured peer route"); - } - let driven = Driven::Setup { service, policy }; - clocks.push(SetupClock { - bond_edge: setup.bond_edge, - route_peer, - driven, - }); - } - Ok(Self { - clocks, - signer: config.settlement_key, - work_mount, - setup_mount, - poll: config.poll, - validators: config.validators, - consensus_verifier, - }) - } - - /// Takes one step of every journal, and says whether the chain - /// answered all of them. - async fn tick(&mut self, source: &S) -> bool { - let mut answered = true; - for clock in &mut self.clocks { - answered &= clock - .tick(source, &self.signer, &self.work_mount, &self.setup_mount) - .await; - } - answered - } - - /// The loop, over whatever chain `dial` produces. - /// - /// One tick of every journal per period, and a chain that stopped - /// answering is dialled again rather than asked forever. The whole - /// of the cadence is here, and none of the decisions are. - async fn run_over(mut self, mut stop: oneshot::Receiver<()>, dial: D) - where - D: Fn() -> F, - F: core::future::Future>, - { - if self.clocks.is_empty() { - info!("no provider setup journal under the work root; the clock has nothing to drive"); - self.work_mount.clear_all(); - self.setup_mount.clear_all(); - return; - } - let mut chain = None; - loop { - tokio::select! { - _ = &mut stop => break, - () = tokio::time::sleep(self.poll) => {} - } - let Some(source) = chain.take() else { - chain = dial().await; - continue; - }; - if self.tick(&source).await { - chain = Some(source); - } - } - self.work_mount.clear_all(); - self.setup_mount.clear_all(); - info!("the paid-work clock stopped, and its journals are closed"); - } -} - -impl WorkRunner { - /// Ticks until told to stop, over the validators the configuration - /// names. - async fn run(self, stop: oneshot::Receiver<()>) { - let validators = self.validators.clone(); - let verifier = self.consensus_verifier.clone(); - self.run_over(stop, move || { - let validators = validators.clone(); - let verifier = verifier.clone(); - async move { connect_chain(&validators, verifier).await } - }) - .await; - } -} - -/// Rotate the first candidate on reconnect, including when a connected peer -/// cannot supply historical finalized blocks. Reads and submissions use the -/// selected verified connection. -/// -/// One endpoint for both directions. §1's concurrent fan-out to all six -/// is a submission strategy with an outcome rule, and neither exists in -/// this tree yet; inventing one here would be the runner deciding what -/// a submission means. -async fn connect_chain( - validators: &[String], - verifier: ConsensusVerifier, -) -> Option { - static NEXT_VALIDATOR: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); - let start = - NEXT_VALIDATOR.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % validators.len(); - for url in validators.iter().cycle().skip(start).take(validators.len()) { - match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { - Ok(client) => { - info!(validator = %url, "the paid-work clock reads and submits here"); - return Some(WorkBlocks::new(client)); - } - Err(error) => warn!(validator = %url, %error, "a configured validator did not answer"), - } - } - None -} - async fn serve_loop(transport: Arc, server: S) -> anyhow::Result<()> where S: Dispatcher + Send + Sync + 'static, diff --git a/crates/cli/src/commands/serve/node/tests.rs b/crates/cli/src/commands/serve/node/tests.rs index 6428495b..2459bbde 100644 --- a/crates/cli/src/commands/serve/node/tests.rs +++ b/crates/cli/src/commands/serve/node/tests.rs @@ -634,7 +634,7 @@ async fn two_vouched_peers_receive_their_distinct_configured_offers() { setup_mount.clone(), ) .expect("both owned provider journals are discovered"); - assert_eq!(runner.clocks.len(), 2, "both journals keep a clock"); + assert_eq!(runner.journal_count(), 2, "both journals keep a clock"); let alpn = ::ALPN.as_bytes(); let server = Endpoint::builder(presets::Minimal) @@ -880,15 +880,31 @@ async fn fresh_readiness_is_per_request_and_per_routed_channel() { ); source.set_snapshot(first.ready_snapshot(ORIGIN, Some(pending_contest(false)))); - let handler = work_mount - .handler(&vouched_context(first_route_peer())) + let service = work_mount + .service(&vouched_context(first_route_peer())) .unwrap(); - let checkpoint = handler - .service - .with_state(ChannelState::checkpoint) - .unwrap(); - // Simulate a different admission request holding the slow chain read. - let held = handler.accepting.lock().await; + let checkpoint = service.with_state(ChannelState::checkpoint).unwrap(); + // Hold a real fresh admission in its chain read through the public handler. + let entered = Arc::new(Semaphore::new(0)); + let release = Arc::new(Semaphore::new(0)); + source.0.lock().unwrap().next_read = Some((entered.clone(), release.clone())); + let mut waiting_authorization = first.authorization(); + waiting_authorization.proposal_nonce = 3; + let waiting_id = work_id(first.descriptor().channel(), &waiting_authorization); + let waiting_request = AcceptWorkRequest { + authorization: waiting_authorization.encode(), + client_signature: first + .client() + .sign(signing_hash(waiting_id)) + .as_bytes() + .to_vec(), + prepared_input: first_request.prepared_input.clone(), + }; + let waiting_mount = work_mount.clone(); + let waiting = tokio::spawn(async move { + accept_mounted_route(&waiting_mount, first_route_peer(), waiting_request).await + }); + entered.acquire().await.unwrap().forget(); let repeated = tokio::time::timeout( Duration::from_secs(1), accept_mounted_route(&work_mount, first_route_peer(), first_request.clone()), @@ -923,13 +939,11 @@ async fn fresh_readiness_is_per_request_and_per_routed_channel() { Some(accept_work_response::Outcome::Refused(refusal)) if refusal.code == WorkRefusalCode::Expired as i32)); assert_eq!( - handler - .service - .with_state(ChannelState::checkpoint) - .unwrap(), + service.with_state(ChannelState::checkpoint).unwrap(), checkpoint ); - drop(held); + release.add_permits(1); + waiting.await.unwrap(); let mut fresh_authorization = first.authorization(); fresh_authorization.proposal_nonce = 2; @@ -1041,7 +1055,7 @@ async fn one_tick_drives_every_owned_journal_without_connected_clients() { "one tick submits all three exact duties" ); assert_eq!( - runner.clocks.len(), + runner.journal_count(), 2, "both journals remain on the clock without a client", ); @@ -1354,7 +1368,7 @@ fn provider_policy() -> ProviderChannelPolicy { network: network(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: EdgeValues::new(PAYMENT_VALUE, PAYMENT_RESERVE, Fees::ZERO), min_omit_response_blocks: MIN_OMIT_RESPONSE_BLOCKS, } @@ -1634,7 +1648,7 @@ struct AnsweringPaidBackend { calls: Arc, } -impl PaidEvaluateBackend for AnsweringPaidBackend { +impl PaidWorkBackend for AnsweringPaidBackend { async fn evaluate( &self, input: PreparedEvaluateInput, @@ -1676,7 +1690,7 @@ impl BlockingPaidBackend { } } -impl PaidEvaluateBackend for BlockingPaidBackend { +impl PaidWorkBackend for BlockingPaidBackend { async fn evaluate_stream( &self, input: PreparedEvaluateInput, @@ -2118,6 +2132,7 @@ impl TxSink for TestChain { struct RoutedChain(Arc>); struct RoutedChainState { + next_read: Option<(Arc, Arc)>, completed_setups: Vec, snapshots: Vec, submitted: Vec, @@ -2129,6 +2144,7 @@ impl RoutedChain { snapshots: impl IntoIterator, ) -> Self { Self(Arc::new(Mutex::new(RoutedChainState { + next_read: None, completed_setups: completed_setups.into_iter().collect(), snapshots: snapshots.into_iter().collect(), submitted: Vec::new(), @@ -2192,6 +2208,11 @@ impl FinalizedWorkView for RoutedChain { &self, query: WorkChannelQuery, ) -> Result, QueryError> { + let gate = self.0.lock().unwrap().next_read.take(); + if let Some((entered, release)) = gate { + entered.add_permits(1); + release.acquire().await.unwrap().forget(); + } match self.0.lock() { Ok(held) => held .snapshots @@ -3561,10 +3582,7 @@ async fn the_clock_serves_work_from_the_channel_it_was_handed() { // And the setup is not driven again, so no second journal is // opened on the file the first mount holds. - let [clock] = runner.clocks.as_slice() else { - panic!("one setup journal was written and one is driven") - }; - assert!(matches!(clock.driven, Driven::Channel(_))); + assert_eq!(runner.channel_count(), 1); assert!( runner.tick(&chain).await, "a second tick drives the channel" diff --git a/crates/cli/src/commands/serve/provision.rs b/crates/cli/src/commands/serve/provision.rs index b48d131a..de84e95e 100644 --- a/crates/cli/src/commands/serve/provision.rs +++ b/crates/cli/src/commands/serve/provision.rs @@ -1,566 +1 @@ -//! Making the offers a fresh provider has nothing to serve without. -//! -//! `WorkRunner::discover` answers `WorkSetup` from the setup journals it -//! finds under the configured work root, and finding is the whole of what -//! it does. A correctly configured provider with no journal therefore -//! refuses every client that dials it, and the paid path is unreachable -//! from a clean install. This is the operator's step that writes them. -//! -//! # The order is the journal's, and none of its rules are here -//! -//! Three library calls. [`SetupStore`] is opened as the provider's half -//! of one bond, the immutable history floor is armed, and -//! [`SetupEndpoint::propose_bond`] signs the stake and journals it before -//! there is anything to export. -//! -//! Arming is first because it has to be: the setup state refuses -//! "recording revision 1 before arming its scan floor", so the floor is -//! not a preparation this command chose to do early but the step every -//! later one is refused before. The floor is a finalized height and the -//! payload digest at it, and the setup's own history must name that -//! digest as the parent of its first block. So it is read from a -//! validator rather than written down by an operator: a floor naming a -//! block this chain does not have is a setup whose history can never be -//! contiguous, and nothing later would say so out loud. -//! -//! # Exit means durable -//! -//! [`SetupStore::commit`] fsyncs a revision before it returns, and this -//! command still reopens the journal and replays it before printing -//! anything. That reopen is the one `WorkRunner::discover` will do, run -//! early: an operator told the offer exists has been told about the disk, -//! and about a file whose exclusive lock is already free for the runner -//! to take. -//! -//! # One recourse backs one route -//! -//! A provider offer reserves a route, a bond, and every coin funding that -//! bond. A second offer is safe only when all three are disjoint from every -//! provider offer already under the root. Existing peers come from the -//! durable route table, while existing coins come from the bond funding in -//! each retained setup bundle. Revision one is enough: it holds the funding -//! before a client has answered, while [`SetupState::funding_coins`] is still -//! empty because there is no executable Open yet. -//! -//! Discovery, route agreement, and funding comparison all happen while the -//! candidate is only a value. The candidate journal is not opened until -//! afterwards, so every collision is refused before a floor is written or a -//! bond signature is made. -//! -//! # What the operator chooses, and what is built -//! -//! Every number in the bond is the operator's and this command invents -//! none of them. Two parts of the shape are not choices: a stake bond is -//! funded by its maker alone, so the taker's side of the funding is -//! empty, and its timeout pays the staking party and nobody else, so -//! there is one payout and it names the provider's own key. A second -//! payout to that same party would only raise the close cost the payout -//! has to clear. Before signing, the timeout is checked against the -//! authenticated finalized height and the chain's maximum edge lifetime. -//! The kernel checks the remaining terms when the Open reaches it — -//! that the payout total is the edge's close value, that the price cap -//! covers a job, that the timeout is ahead of the block including it — -//! and re-spelling any of that here would be a second answer to a -//! question consensus already answers. -//! -//! [`SetupState::funding_coins`]: hellas_work::work_store::SetupState::funding_coins - -use std::collections::BTreeSet; -use std::path::{Path, PathBuf}; - -use anyhow::{Context as _, bail}; -use hellas_chain::client::VerifiedRemoteLightClient; -use hellas_chain::domain::MAX_EDGE_LIFETIME_BLOCKS; -use hellas_chain::{ConsensusInfo, ConsensusVerifier, WorkBlocks}; -use hellas_kernel::{ - BlockHeight, CoinId, EdgeId, Funding, Key, List, MAX_EDGE_OUTPUTS, MAX_PARTY_INPUTS, NetworkId, - Parties, Payout, Secp256k1Signer, Secp256k1Verifier, Terms, Tx, WorkStakeBondTerms, -}; -use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; -use hellas_work::work_close::FinalizedBlocks; -use hellas_work::work_handshake::{PaymentAdmission, SetupEndpoint}; -use hellas_work::work_store::{Role, SetupScan, SetupStore, discover_setups}; -use tracing::{info, warn}; - -use super::work_config::{WorkConfig, WorkRoute}; -use crate::commands::CliResult; - -/// What an operator asks for when they make one offer. -pub struct ProvisionOptions { - /// The loaded paid-work configuration, not the path it came from. It - /// carries the network the bond is bound to, the root the journal is - /// written under, the validators the floor is read from, and the - /// artifact the provider's policy rests on. - pub work_config: WorkConfig, - /// The key this provider stakes and signs the bond with, read from - /// the identity the operator already has and never made here. - pub settlement_key: Secp256k1Signer, - /// The client this bond names as taker, hex-encoded. - pub client: String, - /// The coins this provider stakes, hex-encoded. - pub stake_coins: Vec, - /// Height the bond expires at, which is also the admission horizon of - /// the channel it insures. - pub bond_timeout: u64, - /// What the bond's timeout returns to the staking provider. - pub timeout_payout: u64, - /// The largest job price this bond covers. - pub max_job_price: u64, - /// Print the deterministic bond edge and stop before any external read or write. - pub print_bond_only: bool, -} - -/// Makes one offer, and says where it is. -/// -/// # Errors -/// -/// A configuration with no matching bilateral route, a route, bond, or funding coin already reserved by another offer, -/// a key or coin id that is not one, no configured validator with a finalized -/// block to read a floor from, and whatever the setup journal says about the -/// revision it refused or could not make durable. -pub async fn run_provision(options: ProvisionOptions) -> CliResult<()> { - // The candidate is the one source of the bond edge for both preview and - // provisioning. Keep this before evidence, routing, validators and the - // journal: the preview exists so an operator can put this value into the - // route table those later steps require. - let candidate = BondCandidate::plan(&options)?; - if options.print_bond_only { - println!("bond_edge: {}", hex::encode(candidate.bond_edge.to_bytes())); - return Ok(()); - } - let offer = Offer::plan(&options, options.work_config.provider_policy(), candidate)?; - // Dialled after every refusal that can be made without a chain, and - // before the journal exists: a floor is the first thing written into - // it, so a run that cannot read one leaves no half-made offer behind. - let made = offer.journal(finalized_floor(&options.work_config).await?)?; - - println!( - "offer journaled: bond {} under {}", - hex::encode(made.bond_edge.to_bytes()), - options.work_config.journal_root.display(), - ); - // The floor read back out of the journal rather than the one just - // dialled, because those differ on a retry and the durable one is the - // one this setup's history will be measured against. - println!( - "history floor: finalized height {} with payload {}", - made.floor.height, - hex::encode(made.floor.payload), - ); - Ok(()) -} - -/// One offer as the disk holds it, read back after it was written. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct Provisioned { - /// The bond this journal is keyed to, which is what discovery names - /// it by. - bond_edge: EdgeId, - /// The floor its history starts above, as retained. - floor: SetupScan, -} - -/// The deterministic bond inputs, built without evidence, routing, a chain, -/// or a journal. -/// -/// Preview and real provisioning both pass through this value. In particular, -/// the real path does not recompute the edge after printing it, so a preview -/// cannot drift from the offer later signed. -struct BondCandidate { - network: NetworkId, - journal_root: PathBuf, - bond_edge: EdgeId, - bond_funding: Funding, - bond_terms: WorkStakeBondTerms, - settlement_key: Secp256k1Signer, -} - -impl BondCandidate { - fn plan(options: &ProvisionOptions) -> CliResult { - let network = options.work_config.chain.network; - let journal_root = options.work_config.journal_root.clone(); - // Maker is the provider and taker is the client, which is what - // makes this signature the maker's: `propose_bond` refuses a bond - // whose staking party this key is not. - let provider = options.settlement_key.party_key(); - let bond_terms = WorkStakeBondTerms { - parties: Parties::new( - provider, - Key::from_bytes(fixed::<{ Key::LENGTH }>("--client", &options.client)?), - ), - timeout: BlockHeight::new(options.bond_timeout), - timeout_outputs: List::take( - [Payout::new(provider, options.timeout_payout); MAX_EDGE_OUTPUTS], - 1, - ), - max_job_price: options.max_job_price, - }; - let bond_funding = Funding::new( - staked(&options.stake_coins)?, - List::empty(CoinId::from_bytes([0; CoinId::LENGTH])), - ); - let bond_edge = Tx::edge_id_of(&bond_funding, &Terms::work_stake_bond(bond_terms.clone())); - Ok(Self { - network, - journal_root, - bond_edge, - bond_funding, - bond_terms, - settlement_key: options.settlement_key.clone(), - }) - } -} - -/// One offer, decided before anything is dialled or written. -struct Offer { - network: NetworkId, - journal_root: PathBuf, - bond_edge: EdgeId, - bond_funding: Funding, - bond_terms: WorkStakeBondTerms, - admission: PaymentAdmission, - settlement_key: Secp256k1Signer, -} - -impl Offer { - /// Reads the operator's answers, and refuses everything refusable - /// without a chain. - fn plan( - options: &ProvisionOptions, - policy: ProviderChannelPolicy, - candidate: BondCandidate, - ) -> CliResult { - let admission = PaymentAdmission::Admits(Box::new(policy)); - let BondCandidate { - network, - journal_root, - bond_edge, - bond_funding, - bond_terms, - settlement_key, - } = candidate; - let route = route_for_candidate(&options.work_config, bond_edge, &bond_terms)?; - refuse_offer_collisions(&options.work_config, route, &bond_funding)?; - Ok(Self { - network, - journal_root, - bond_edge, - bond_funding, - bond_terms, - admission, - settlement_key, - }) - } - - /// Journals revision 1, and returns only once a fresh open of the - /// journal replays it. - fn journal(self, floor: SetupScan) -> CliResult { - let timeout = self.bond_terms.timeout.get(); - anyhow::ensure!( - timeout > floor.height, - "--bond-timeout {timeout} must be after finalized height {}", - floor.height, - ); - anyhow::ensure!( - timeout - floor.height <= MAX_EDGE_LIFETIME_BLOCKS, - "--bond-timeout {timeout} exceeds the chain's maximum edge lifetime of \ - {MAX_EDGE_LIFETIME_BLOCKS} blocks from finalized height {}", - floor.height, - ); - let Self { - network, - journal_root, - bond_edge, - bond_funding, - bond_terms, - admission, - settlement_key, - } = self; - { - let store = open_provider_journal(&journal_root, network, bond_edge)?; - let mut endpoint = SetupEndpoint::new(store, settlement_key, admission); - // The floor is immutable and the store writes exactly one arm - // of it, so a run that arms and then fails keeps the height - // its successor starts from rather than moving it. - if let Some(held) = endpoint.state().scan_armed() { - info!( - height = held.height, - "this journal already holds its history floor, and a floor does not move", - ); - } else { - endpoint - .arm_scan(floor) - .context("failed to make this setup's immutable history floor durable")?; - } - endpoint - .propose_bond(network, bond_funding, bond_terms) - .context("failed to sign and journal the bond proposal")?; - } - - // The journal is closed above, so this is a second process's view - // of it: the same replay and the same signature checks the runner - // runs, before an operator is told there is anything to run them - // on. - let reopened = open_provider_journal(&journal_root, network, bond_edge)?; - let state = reopened.state(); - let (Some(1), Some(floor)) = (state.revision(), state.scan_armed()) else { - bail!( - "the journal under {} replays as revision {:?} over floor {:?}, not the armed \ - proposal that was just written", - journal_root.display(), - state.revision(), - state.scan_armed().map(|scan| scan.height), - ); - }; - Ok(Provisioned { bond_edge, floor }) - } -} - -fn open_provider_journal( - root: &Path, - network: NetworkId, - bond_edge: EdgeId, -) -> CliResult { - SetupStore::open( - root, - network, - bond_edge, - Role::Provider, - &Secp256k1Verifier::new(), - ) - .with_context(|| { - format!( - "failed to open the provider setup journal for bond {} under {}", - hex::encode(bond_edge.to_bytes()), - root.display(), - ) - }) -} - -/// Returns the configured bilateral route the candidate would occupy. -/// -/// The bond is derived from the exact funding and terms first. Matching by -/// that canonical value means a route cannot be selected by insertion order, -/// and checking the client here refuses a journal the next startup would -/// reject before the provider signs it. -fn route_for_candidate<'config>( - config: &'config WorkConfig, - bond_edge: EdgeId, - bond_terms: &WorkStakeBondTerms, -) -> CliResult<&'config WorkRoute> { - let Some(route) = config.routes.iter().find(|route| route.bond == bond_edge) else { - bail!( - "bond {} has no bilateral route in this work configuration; an offer is signed only \ - after its peer, bond, and client are named together", - hex::encode(bond_edge.to_bytes()), - ); - }; - let client = bond_terms.parties.taker(); - if route.client != client { - bail!( - "route for peer {:#} expects client {}, but candidate bond {} names {} as its taker", - route.peer, - hex::encode(route.client.to_bytes()), - hex::encode(bond_edge.to_bytes()), - hex::encode(client.to_bytes()), - ); - } - Ok(route) -} - -/// Refuses every collision before the candidate journal is opened. -/// -/// An existing bond is named by discovery, its peer is named by the durable -/// route table, and its funding is named by the retained bundle. Failure to -/// recover any one of those facts is a refusal: absence of evidence is not -/// evidence that the candidate is disjoint. -fn refuse_offer_collisions( - config: &WorkConfig, - candidate: &WorkRoute, - candidate_funding: &Funding, -) -> CliResult<()> { - let root = &config.journal_root; - let network = config.chain.network; - let found = discover_setups(root, network).with_context(|| { - format!( - "failed to enumerate the work journals under {}", - root.display(), - ) - })?; - for unnamed in &found.unidentified { - warn!( - path = %unnamed.path.display(), - reason = %unnamed.reason, - "a setup journal under the work root could not be named", - ); - } - if let Some(unnamed) = found.unidentified.first() { - bail!( - "setup journal {} cannot be identified, so a new offer cannot be proved disjoint: {}", - unnamed.path.display(), - unnamed.reason, - ); - } - - let candidate_coins = funding_coins(candidate_funding); - for held in found - .setups - .iter() - .filter(|setup| setup.role == Role::Provider) - { - if held.bond_edge == candidate.bond { - bail!( - "candidate bond {} collides with a provider offer already under {}", - hex::encode(candidate.bond.to_bytes()), - root.display(), - ); - } - let Some(route) = config - .routes - .iter() - .find(|route| route.bond == held.bond_edge) - else { - bail!( - "provider offer over bond {} under {} has no configured route, so the candidate \ - route cannot be proved disjoint", - hex::encode(held.bond_edge.to_bytes()), - root.display(), - ); - }; - let store = open_provider_journal(root, network, held.bond_edge)?; - let Some(bundle) = store.state().bundle() else { - bail!( - "provider offer over bond {} was discovered without a retained revision", - hex::encode(held.bond_edge.to_bytes()), - ); - }; - let held_client = bundle.bond_terms().parties.taker(); - if route.client != held_client { - bail!( - "route for peer {:#} expects client {}, but provider offer over bond {} names {} \ - as its taker", - route.peer, - hex::encode(route.client.to_bytes()), - hex::encode(held.bond_edge.to_bytes()), - hex::encode(held_client.to_bytes()), - ); - } - if route.peer == candidate.peer { - bail!( - "candidate route peer {:#} collides with the provider offer over bond {}", - candidate.peer, - hex::encode(held.bond_edge.to_bytes()), - ); - } - // The retained revision's own staked funding, not the executable - // Opens: the provider signed these coins when it made the offer, so - // they are promised from that moment, while `funding_coins` answers - // from Opens that do not exist until the client countersigns. Read - // from there, every offer no client has answered would look like it - // reserved nothing. - let reserved = funding_coins(bundle.bond_funding()); - if let Some(coin) = candidate_coins.intersection(&reserved).next() { - bail!( - "candidate stake coin {} is already reserved by provider offer over bond {}", - hex::encode(coin.to_bytes()), - hex::encode(held.bond_edge.to_bytes()), - ); - } - } - Ok(()) -} - -/// Every input one bond funding consumes, irrespective of party position. -fn funding_coins(funding: &Funding) -> BTreeSet { - funding - .maker() - .iter() - .chain(funding.taker().iter()) - .copied() - .collect() -} - -/// Reads one finalized block from the first configured validator that -/// answers, as the floor this setup's history starts above. -async fn finalized_floor(config: &WorkConfig) -> CliResult { - let verifier = ConsensusVerifier::new(&ConsensusInfo { - validators: config.validators.clone(), - threshold_identity: config.chain.threshold_identity.clone(), - network_id: config.chain.network.as_str().to_owned(), - }) - .context("the configured threshold identity is not usable")?; - for url in &config.validators { - let client = match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { - Ok(client) => client, - Err(error) => { - warn!(validator = %url, %error, "a configured validator did not answer"); - continue; - } - }; - match floor_of(&WorkBlocks::new(client)).await { - Ok(Some(floor)) => { - info!(validator = %url, height = floor.height, "the history floor was read here"); - return Ok(floor); - } - Ok(None) => warn!(validator = %url, "a configured validator has finalized nothing"), - Err(error) => warn!(validator = %url, %error, "a configured validator did not answer"), - } - } - bail!("no configured validator answered with a finalized block to floor this offer at") -} - -/// Returns the finalized tip as a scan floor, or `None` before anything -/// is finalized. -/// -/// The height and the payload come from one block rather than from two -/// reads, because the setup's first history block must name that exact -/// payload as its parent. -async fn floor_of(blocks: &B) -> CliResult> -where - B: FinalizedBlocks + ?Sized, -{ - let Some(height) = blocks.latest_height().await? else { - return Ok(None); - }; - let Some(block) = blocks.block_at(height).await? else { - return Ok(None); - }; - Ok(Some(SetupScan { - height: block.height, - payload: block.payload, - })) -} - -/// Reads the coins one provider stakes. -fn staked(ids: &[String]) -> CliResult> { - let mut slots = [CoinId::from_bytes([0; CoinId::LENGTH]); MAX_PARTY_INPUTS]; - for (slot, id) in slots.iter_mut().zip(ids) { - *slot = CoinId::from_bytes(fixed::<{ CoinId::LENGTH }>("--stake-coin", id)?); - } - // The zip above stops at the shorter side, so a list the array cannot - // hold is refused here rather than silently staking the first four of - // it. - List::new(slots, ids.len()).with_context(|| { - format!( - "--stake-coin names {} coins, and one party funds an open with at most \ - {MAX_PARTY_INPUTS}", - ids.len(), - ) - }) -} - -/// Reads exactly `N` bytes of hex, or says which flag was not that. -fn fixed(flag: &str, value: &str) -> CliResult<[u8; N]> { - let bytes = - hex::decode(value).with_context(|| format!("{flag} {value:?} is not hex-encoded bytes"))?; - let Ok(fixed) = <[u8; N]>::try_from(bytes.as_slice()) else { - bail!( - "{flag} {value:?} is {} bytes, and {N} are wanted", - bytes.len() - ); - }; - Ok(fixed) -} - -#[cfg(test)] -mod tests; +pub use hellas_sdk::work_provision::{ProvisionOptions, run_provision}; diff --git a/crates/cli/src/commands/serve/work_config.rs b/crates/cli/src/commands/serve/work_config.rs index 0ce1c113..275fdf1d 100644 --- a/crates/cli/src/commands/serve/work_config.rs +++ b/crates/cli/src/commands/serve/work_config.rs @@ -1,628 +1,4 @@ -//! The paid-work configuration file: what an operator writes down, and -//! what a node refuses to start without. -//! -//! `--work-config` was a path whose *presence* advertised two ALPNs and -//! whose contents were never opened. A node cannot mount a channel from -//! a path, so this is the schema and the loader for what is in it: the -//! three-part chain cross-check, the six validator URLs a write is -//! fanned to, the journal root, the bilateral route table, the two -//! policies this provider works under, the watcher's poll cadence, the -//! funding it expects a payment edge to carry, and the shortest response -//! window it will sign terms over. -//! -//! # The cross-check is not an anchor -//! -//! `(network_id, genesis_payload_digest, threshold_identity)` is a -//! fail-fast configuration cross-check. Only the threshold identity ever -//! authenticates a finalized block; the other two are how a node started -//! against the wrong chain says so at startup instead of at the first -//! settlement. The identity is decoded here, by the same constructor -//! consensus verification uses, so a typo is a startup error and not a -//! block that never verifies. -//! -//! # What is deliberately not here -//! -//! There is no Start-span field, no mutual-margin field, and no journal -//! cap. All were deleted: the Start span is fixed at 64, a work-payment -//! edge has no Mutual route, and the journal's active and checkpoint -//! ceilings are constants it enforces on itself -//! ([`MAX_ACTIVE_JOURNAL_BYTES`]), so any of them appearing in a file is -//! an operator configuring something that does not exist. Every struct -//! below denies unknown fields, which is what turns that into an error -//! naming the field. -//! -//! Nor is there a measured artifact. An earlier design derived the -//! response window and an alarm margin from latencies a bootstrap probe -//! recorded, pinned to the digest of the measuring binary; no deployed -//! node ever carried one, and every rebuild would have invalidated it. -//! The two numbers that design would have produced are written down -//! here instead, by the operator, and [`WorkConfig::provider_policy`] is -//! the whole of what is made from them. -//! -//! [`MAX_ACTIVE_JOURNAL_BYTES`]: hellas_work::work_store::journal::MAX_ACTIVE_JOURNAL_BYTES - -use std::collections::{BTreeMap, BTreeSet}; -use std::fs; -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use anyhow::{Context as _, bail}; -use hellas_kernel::{ - EdgeId, EdgeValues, Fees, Key, MIN_OMIT_RESPONSE_BLOCKS, NetworkId, Secp256k1Verifier, -}; -use hellas_rpc::ContentId; -use hellas_rpc::peers::PeerId; -use hellas_rpc::protocol::Digest; -use hellas_rpc::protocol::work::{ - PaidChannelPolicyV1, PaidExecutionPolicyV1, check_execution_policy, -}; -use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; -use hellas_work::work_store::{Role, SetupStore, discover_setups}; -use serde::Deserialize; - -use crate::commands::CliResult; - -/// How many validator RPCs a write names. -/// -/// Reads come from a follower and writes are fanned to all six; a -/// configuration naming five has one validator whose acceptance this -/// node can never win, and one naming seven names something this -/// deployment does not have. -pub const VALIDATOR_COUNT: usize = 6; - -/// One operator's complete paid-work configuration, loaded and structurally -/// checked. -/// -/// A plain record with public fields, for [`WorkChannelConfig`]'s -/// reason: this is the shape a file fills in. Every file-local gate has -/// already run in [`load_work_config`]; the serve path then runs -/// [`validate_work_routes`] against the journals that must exist when it -/// starts. Provisioning shares the file loader before it creates one, which is -/// why disk agreement is not pretended to be a parse-time fact. -/// -/// [`WorkChannelConfig`]: hellas_rpc::protocol::work_setup::WorkChannelConfig -#[derive(Clone, Debug)] -#[allow( - dead_code, - reason = "the fields a mount consumes are read by the node runner; loading and checking them is this half" -)] -pub struct WorkConfig { - /// The chain this node believes it is configured against. - pub chain: ChainCrossCheck, - /// The six validator RPC URLs every write is fanned to. - pub validators: Vec, - /// Directory holding the setup and channel journals. - pub journal_root: PathBuf, - /// Bilateral setup routes, keyed by the authenticated transport peer. - pub routes: WorkRoutes, - /// Salt of the private credit-policy commitment. - pub policy_salt: [u8; 32], - /// The credit policy this provider will work under. - pub channel_policy: PaidChannelPolicyV1, - /// The execution policy this provider will run jobs under. - pub execution_policy: PaidExecutionPolicyV1, - /// How often the watcher asks the chain for the next block. - pub poll: Duration, - /// The payment edge's value, reserve, and close fees as this provider - /// requires a client to fund them. - pub expected_payment_values: EdgeValues, - /// The shortest response window this provider signs terms over. - pub min_omit_response_blocks: u64, -} - -/// One bilateral setup route written in the paid-work configuration. -/// -/// The bond names the provider setup journal under [`WorkConfig::journal_root`]. -/// The client key is repeated here deliberately: startup compares it with the -/// taker committed inside that journal, turning a stale or mistyped route into -/// a refusal before the node binds. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct WorkRoute { - /// The transport-authenticated peer allowed to reach this bond. - pub peer: PeerId, - /// The bond whose provider setup journal this route names. - pub bond: EdgeId, - /// The settlement key the bond terms must name as taker. - pub client: Key, -} - -/// Paid-work routes keyed by their authenticated peer. -/// -/// Construction is private to the checked file loader. In particular, there -/// is no insertion API through which a caller could recreate last-one-wins -/// handling after duplicate peers and bonds have been refused. -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct WorkRoutes { - by_peer: BTreeMap, -} - -impl WorkRoutes { - /// Returns every configured route in peer order. - pub fn iter(&self) -> impl Iterator { - self.by_peer.values() - } - - /// Returns how many bilateral routes were configured. - #[must_use] - pub fn len(&self) -> usize { - self.by_peer.len() - } - - /// Returns whether no bilateral route was configured. - #[must_use] - pub fn is_empty(&self) -> bool { - self.by_peer.is_empty() - } - - fn from_files(files: Vec) -> CliResult { - let mut by_peer = BTreeMap::new(); - let mut bonds = BTreeSet::new(); - for file in files { - let peer = PeerId::from_bytes(parse_fixed_hex("routes[].peer", &file.peer)?); - let bond = EdgeId::from_bytes(parse_fixed_hex("routes[].bond", &file.bond)?); - let client = Key::from_bytes(parse_fixed_hex("routes[].client", &file.client)?); - let route = WorkRoute { peer, bond, client }; - if by_peer.insert(peer, route).is_some() { - bail!("routes names peer {peer:#} twice; one authenticated peer has one route"); - } - if !bonds.insert(bond) { - bail!( - "routes names bond {} twice; one provider journal has one route", - hex::encode(bond.to_bytes()), - ); - } - } - Ok(Self { by_peer }) - } -} - -impl WorkConfig { - /// The provider policy this configuration makes. - /// - /// Every field is the operator's: the four the policy commits to, - /// the funding it expects, and the window it insists on. Nothing is - /// measured and nothing is inferred, so a node with a configuration - /// has a policy and countersigns over it. - #[must_use] - pub fn provider_policy(&self) -> ProviderChannelPolicy { - ProviderChannelPolicy { - network: self.chain.network, - policy_salt: self.policy_salt, - channel_policy: self.channel_policy, - execution_policy: self.execution_policy, - expected_payment_values: self.expected_payment_values, - min_omit_response_blocks: self.min_omit_response_blocks, - } - } -} - -/// The three fields that say which chain this is. -#[derive(Clone, Debug, PartialEq, Eq)] -#[allow( - dead_code, - reason = "the fields a mount consumes are read by the node runner; loading and checking them is this half" -)] -pub struct ChainCrossCheck { - /// The network every signature on this node's channels is bound to. - pub network: NetworkId, - /// Payload digest of the genesis block this deployment began at. - pub genesis_payload_digest: Digest, - /// The threshold identity finalized blocks are verified under. - pub threshold_identity: Vec, -} - -/// Loads and checks one paid-work configuration file. -/// -/// Every failure is a startup failure naming the field that failed, for -/// the reason §4 gives: a node that started with an unreadable -/// configuration would be one whose first symptom is an unsettleable -/// channel. -/// -/// # Errors -/// -/// The read and the parse, and then: a network id that is not one, a -/// digest that is not thirty-two bytes, a threshold identity consensus -/// cannot decode, a validator list that is not exactly -/// [`VALIDATOR_COUNT`] URLs with distinct normalised forms, an execution -/// policy the protocol's own [`check_execution_policy`] rejects, an -/// empty journal root, a route field of the wrong width, duplicate peers or -/// bonds in the route table, a zero poll cadence, and a response window -/// under the kernel's own minimum. -pub fn load_work_config(path: &Path) -> CliResult { - let bytes = fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; - let file: WorkConfigFile = serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse {}", path.display()))?; - file.into_config() - .with_context(|| format!("invalid work config {}", path.display())) -} - -/// Verifies that every configured route names this root's provider journal -/// and the client settlement key committed by its bond terms. -/// -/// This is a serve-startup check rather than part of [`load_work_config`]: -/// provisioning uses the same configuration loader before it creates a -/// journal, while a serving node must already have every journal it promises. -/// Discovery comes first so [`SetupStore::open`] is never allowed to create a -/// missing journal merely because a route named its bond. -/// -/// # Errors -/// -/// The root cannot be enumerated, a route's provider journal is absent from -/// that root or cannot be opened, the journal holds no bond proposal, or its -/// bond names a taker other than the route's configured client. -pub(super) fn validate_work_routes(config: &WorkConfig) -> CliResult<()> { - if config.routes.is_empty() { - return Ok(()); - } - let found = discover_setups(&config.journal_root, config.chain.network).with_context(|| { - format!( - "failed to enumerate configured work routes under journal.root {}", - config.journal_root.display(), - ) - })?; - for route in config.routes.iter() { - if !found - .setups - .iter() - .any(|setup| setup.role == Role::Provider && setup.bond_edge == route.bond) - { - bail!( - "route for peer {:#} names bond {}, but its provider setup journal is not under \ - journal.root {}", - route.peer, - hex::encode(route.bond.to_bytes()), - config.journal_root.display(), - ); - } - let store = SetupStore::open( - &config.journal_root, - config.chain.network, - route.bond, - Role::Provider, - &Secp256k1Verifier::new(), - ) - .with_context(|| { - format!( - "route for peer {:#} could not open provider setup journal for bond {} under {}", - route.peer, - hex::encode(route.bond.to_bytes()), - config.journal_root.display(), - ) - })?; - let Some(bundle) = store.state().bundle() else { - bail!( - "route for peer {:#} names provider setup journal for bond {}, but it holds no \ - bond proposal", - route.peer, - hex::encode(route.bond.to_bytes()), - ); - }; - let journal_client = bundle.bond_terms().parties.taker(); - if journal_client != route.client { - bail!( - "route for peer {:#} expects client settlement key {}, but provider setup journal \ - for bond {} names {} as its taker", - route.peer, - hex::encode(route.client.to_bytes()), - hex::encode(route.bond.to_bytes()), - hex::encode(journal_client.to_bytes()), - ); - } - } - Ok(()) -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct WorkConfigFile { - chain: ChainFile, - validators: Vec, - journal: JournalFile, - routes: Vec, - policies: PoliciesFile, - /// How often the watcher asks the chain for the next block. - poll_ms: u64, - expected_payment_values: PaymentValuesFile, - min_omit_response_blocks: u64, -} - -impl WorkConfigFile { - fn into_config(self) -> CliResult { - let Some(network) = NetworkId::new(self.chain.network_id.trim()) else { - bail!( - "chain.network_id {:?} is not a network id", - self.chain.network_id - ); - }; - let threshold_identity = - parse_hex("chain.threshold_identity", &self.chain.threshold_identity)?; - // Parsed before the verifier is built, so the list consensus is - // handed is the normalised one this node will actually dial. - let validators = parse_validators(self.validators)?; - // The same constructor consensus verification uses. A threshold - // identity that cannot be decoded here is one no finalized block - // would ever verify under, and the node says so before it serves. - hellas_chain::ConsensusVerifier::new(&hellas_chain::light_client::ConsensusInfo { - validators: validators.clone(), - threshold_identity: threshold_identity.clone(), - network_id: self.chain.network_id.clone(), - }) - .map_err(|error| anyhow::anyhow!("chain.threshold_identity is not usable: {error}"))?; - - let journal_root = self.journal.into_root()?; - let routes = WorkRoutes::from_files(self.routes)?; - let policies = self.policies.into_policies()?; - if self.poll_ms == 0 { - bail!("poll_ms must be greater than zero"); - } - // The kernel refuses a shorter window at every payment open, so a - // configuration under it would sign terms consensus then throws - // away. - if self.min_omit_response_blocks < MIN_OMIT_RESPONSE_BLOCKS { - bail!( - "min_omit_response_blocks {} is under the kernel's minimum {MIN_OMIT_RESPONSE_BLOCKS}", - self.min_omit_response_blocks, - ); - } - - Ok(WorkConfig { - chain: ChainCrossCheck { - network, - genesis_payload_digest: parse_digest( - "chain.genesis_payload_digest", - &self.chain.genesis_payload_digest, - )?, - threshold_identity, - }, - validators, - journal_root, - routes, - policy_salt: policies.0, - channel_policy: policies.1, - execution_policy: policies.2, - poll: Duration::from_millis(self.poll_ms), - expected_payment_values: self.expected_payment_values.into_values(), - min_omit_response_blocks: self.min_omit_response_blocks, - }) - } -} - -/// One bilateral route exactly as the operator writes it. -/// -/// All three values are fixed-width lowercase-or-uppercase hexadecimal on -/// input and canonical byte values after loading. A peer or bond written in a -/// second spelling is therefore still the same key for duplicate detection. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct WorkRouteFile { - peer: String, - bond: String, - client: String, -} - -/// Parses the six validator RPC URLs, and refuses anything that is not -/// one. -/// -/// Both halves matter. A string that is not a URL is not an address this -/// node can ever fan a write to, and "trimmed and non-empty" admits `not -/// a URL` verbatim — a configuration whose first symptom would be five -/// validators answering and one that never does. And uniqueness is a -/// question about *addresses*, not about spellings: `HTTP://Host:443/` -/// and `http://host:443/` are one validator written twice, and a fan-out -/// to five validators is not six however it is spelled. So the -/// comparison is between parsed, normalised URLs, and the normalised -/// forms are what is kept. -/// -/// A host is required, because these are dialled: a `mailto:` or a -/// `data:` parses perfectly well and is not a validator. -fn parse_validators(entries: Vec) -> CliResult> { - let mut validators: Vec = Vec::with_capacity(VALIDATOR_COUNT); - for entry in entries { - let entry = entry.trim(); - if entry.is_empty() { - bail!("validators entries must be non-empty"); - } - let url = reqwest::Url::parse(entry) - .with_context(|| format!("validators entry {entry:?} is not a URL"))?; - if url.host_str().is_none() { - bail!("validators entry {entry:?} names no host to dial"); - } - let normalised = url.as_str().to_string(); - if validators.contains(&normalised) { - bail!("validators names {normalised} twice; a fan-out to five validators is not six"); - } - validators.push(normalised); - } - if validators.len() != VALIDATOR_COUNT { - bail!( - "validators must name exactly {VALIDATOR_COUNT} validator URLs, found {}", - validators.len(), - ); - } - Ok(validators) -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct ChainFile { - network_id: String, - genesis_payload_digest: String, - threshold_identity: String, -} - -/// Where the work journals live. -/// -/// A root and nothing else. How large a journal may grow is not an -/// operator's to say: the active and checkpoint ceilings are constants -/// the journal enforces on itself, so a cap here would be a number -/// written down and ignored. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct JournalFile { - root: PathBuf, -} - -impl JournalFile { - fn into_root(self) -> CliResult { - if self.root.as_os_str().is_empty() { - bail!("journal.root must be a path"); - } - Ok(self.root) - } -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct PoliciesFile { - policy_salt: String, - channel: ChannelPolicyFile, - execution: ExecutionPolicyFile, -} - -impl PoliciesFile { - fn into_policies(self) -> CliResult<([u8; 32], PaidChannelPolicyV1, PaidExecutionPolicyV1)> { - let salt = parse_fixed_hex("policies.policy_salt", &self.policy_salt)?; - Ok(( - salt, - PaidChannelPolicyV1 { - compute_credit_limit: self.channel.compute_credit_limit, - delivery_credit_limit: self.channel.delivery_credit_limit, - }, - self.execution.into_policy()?, - )) - } -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct ChannelPolicyFile { - compute_credit_limit: u64, - delivery_credit_limit: u64, -} - -/// The execution policy, field for field. -/// -/// Spelled out rather than flattened from some smaller shape because -/// every one of these is a value both parties sign: a default here would -/// be this node quietly proposing a policy its operator never wrote. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct ExecutionPolicyFile { - allowed_environment: String, - generation_policy_digest: String, - identity_source_digest: String, - max_prompt_tokens: u32, - max_new_tokens: u32, - max_stop_token_ids: u16, - max_spool_bytes: u64, - max_encoded_result_frame: u32, - max_encoded_quote_response: u32, - dispatch_margin_blocks: u64, - delivery_margin_blocks: u64, - oracle_grace_blocks: u64, - fixed_price: u64, -} - -impl ExecutionPolicyFile { - fn into_policy(self) -> CliResult { - let allowed_environment: ContentId = - self.allowed_environment.parse().with_context(|| { - format!( - "policies.execution.allowed_environment {:?} is not a ContentId", - self.allowed_environment - ) - })?; - let policy = PaidExecutionPolicyV1 { - allowed_environment, - generation_policy_digest: parse_digest( - "policies.execution.generation_policy_digest", - &self.generation_policy_digest, - )?, - identity_source_digest: parse_digest( - "policies.execution.identity_source_digest", - &self.identity_source_digest, - )?, - max_prompt_tokens: self.max_prompt_tokens, - max_new_tokens: self.max_new_tokens, - max_stop_token_ids: self.max_stop_token_ids, - max_spool_bytes: self.max_spool_bytes, - max_encoded_result_frame: self.max_encoded_result_frame, - max_encoded_quote_response: self.max_encoded_quote_response, - dispatch_margin_blocks: self.dispatch_margin_blocks, - delivery_margin_blocks: self.delivery_margin_blocks, - oracle_grace_blocks: self.oracle_grace_blocks, - fixed_price: self.fixed_price, - }; - // The protocol's own gate, run here rather than at the first - // admission. A zero here is not a small bound, it is an absent - // one — a zero margin gives a deadline no time to be met in, and - // a zero price is a job nobody is paid for. Copying the fields - // through unchecked moves that discovery to the moment a - // counterparty is already waiting on a proposal. - check_execution_policy(&policy) - .map_err(|error| anyhow::anyhow!("policies.execution is not usable: {error}"))?; - Ok(policy) - } -} - -/// The payment edge's funding, field for field, as this provider requires -/// a client to fund it. -/// -/// Spelled out for [`ExecutionPolicyFile`]'s reason: every one of these -/// bounds what a certificate on the channel may name, so a default here -/// would be this node quietly accepting funding its operator never -/// priced. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct PaymentValuesFile { - value: u64, - reserve: u64, - close_fees: CloseFeesFile, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct CloseFeesFile { - base: u64, - slot: u64, - proof: u64, - lifetime: u64, -} - -impl PaymentValuesFile { - fn into_values(self) -> EdgeValues { - EdgeValues::new( - self.value, - self.reserve, - Fees::new( - self.close_fees.base, - self.close_fees.slot, - self.close_fees.proof, - self.close_fees.lifetime, - ), - ) - } -} - -fn parse_hex(field: &str, raw: &str) -> CliResult> { - let bytes = hex::decode(raw.trim()).with_context(|| format!("{field} is not hexadecimal"))?; - if bytes.is_empty() { - bail!("{field} must not be empty"); - } - Ok(bytes) -} - -fn parse_fixed_hex(field: &str, raw: &str) -> CliResult<[u8; N]> { - let bytes = parse_hex(field, raw)?; - let Ok(bytes) = <[u8; N]>::try_from(bytes.as_slice()) else { - bail!("{field} must be {N} bytes, found {}", bytes.len()); - }; - Ok(bytes) -} - -fn parse_digest(field: &str, raw: &str) -> CliResult { - Ok(Digest::from_bytes(parse_fixed_hex(field, raw)?)) -} +pub use hellas_sdk::work_config::*; #[cfg(test)] mod tests; diff --git a/crates/cli/src/commands/serve/work_config/tests.rs b/crates/cli/src/commands/serve/work_config/tests.rs index ec9d98a0..b3f25ccd 100644 --- a/crates/cli/src/commands/serve/work_config/tests.rs +++ b/crates/cli/src/commands/serve/work_config/tests.rs @@ -1,4 +1,46 @@ use super::*; +use hellas_kernel::{EdgeValues, Fees, Key, NetworkId}; +use hellas_rpc::peers::PeerId; +use hellas_rpc::protocol::Digest; +use hellas_rpc::protocol::work::*; +use hellas_rpc::protocol::work_profile::PaidWorkPolicy; +use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; +use std::time::Duration; +use std::{ + fs, + path::{Path, PathBuf}, +}; +type CliResult = anyhow::Result; + +#[test] +fn fetch_policy_loads_and_cannot_be_combined_with_evaluate() { + let mut value = config(); + value["policies"]["fetch"] = serde_json::json!({ + "allowed_environment": hex32(0x11), + "service": "openai", "method": "responses", + "max_request_body_bytes": 4096, "max_output_events": 64, + "max_output_bytes": 16384, "max_spool_bytes": 65536, + "max_encoded_result_frame": 65536, "max_encoded_prepared_input": 65536, + "dispatch_margin_blocks": 4, "delivery_margin_blocks": 2, + "oracle_grace_blocks": 6, "fixed_price": 10, + }); + assert!(load(value.clone()).is_err()); + value["policies"] + .as_object_mut() + .unwrap() + .remove("execution"); + let loaded = load(value.clone()).unwrap(); + assert!(matches!( + loaded.execution_policy, + PaidWorkPolicy::Fetch { .. } + )); + assert_eq!( + PaidWorkPolicy::decode(&loaded.execution_policy.encode()).unwrap(), + loaded.execution_policy + ); + value["policies"].as_object_mut().unwrap().remove("fetch"); + assert!(load(value).is_err()); +} fn hex32(byte: u8) -> String { hex::encode([byte; 32]) @@ -129,8 +171,14 @@ fn a_work_config_round_trips_from_a_file() { assert_eq!(route.client, Key::from_bytes([0x02; Key::LENGTH])); assert_eq!(loaded.policy_salt, [0x5a; 32]); assert_eq!(loaded.channel_policy.compute_credit_limit, 40); - assert_eq!(loaded.execution_policy.fixed_price, 10); - assert_eq!(loaded.execution_policy.max_stop_token_ids, 4); + assert_eq!(loaded.execution_policy.fixed_price(), 10); + assert_eq!( + match loaded.execution_policy { + PaidWorkPolicy::Evaluate(policy) => policy.max_stop_token_ids, + _ => panic!("expected Evaluate policy"), + }, + 4 + ); assert_eq!(loaded.poll, Duration::from_millis(250)); assert_eq!( loaded.expected_payment_values, @@ -354,7 +402,13 @@ fn a_zero_execution_policy_field_is_refused() { let mut value = config(); value["policies"]["execution"]["max_stop_token_ids"] = serde_json::json!(0); let loaded = load(value).expect("no stop tokens is a usable channel"); - assert_eq!(loaded.execution_policy.max_stop_token_ids, 0); + assert_eq!( + match loaded.execution_policy { + PaidWorkPolicy::Evaluate(policy) => policy.max_stop_token_ids, + _ => panic!("expected Evaluate policy"), + }, + 0 + ); } #[test] @@ -617,7 +671,7 @@ fn a_configuration_makes_its_policy_field_for_field() { assert_eq!(policy.network, network()); assert_eq!(policy.policy_salt, [0x5a; 32]); assert_eq!(policy.channel_policy.compute_credit_limit, 40); - assert_eq!(policy.execution_policy.fixed_price, 10); + assert_eq!(policy.execution_policy.fixed_price(), 10); assert_eq!( policy.expected_payment_values, EdgeValues::new(PAYMENT_VALUE, PAYMENT_RESERVE, Fees::new(0, 0, 0, 0)), diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index df2b7115..68071132 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1364,6 +1364,7 @@ async fn async_main() { command, secret_key, identity::settlement_signer(&local_identity), + local_identity.producer_key.clone(), ) .await } diff --git a/crates/client/src/iroh.rs b/crates/client/src/iroh.rs index cc8eea33..654c0f7a 100644 --- a/crates/client/src/iroh.rs +++ b/crates/client/src/iroh.rs @@ -333,7 +333,8 @@ pub fn build_client_registry(endpoint: &::iroh::Endpoint) -> ClientResult( +/// Authenticates a provider on this live connection before disclosing a request. +pub async fn confidential_open( transport: &IrohTransport, trust: &ProviderTrustAnchor, ) -> ClientResult diff --git a/crates/client/src/iroh/tests.rs b/crates/client/src/iroh/tests.rs index 27b4519c..6a2f84a3 100644 --- a/crates/client/src/iroh/tests.rs +++ b/crates/client/src/iroh/tests.rs @@ -430,6 +430,16 @@ fn apple_open_response( exporter: &[u8; 32], nonce: &[u8; 32], counter: u32, + counters: Arc, +) -> (ProviderTrustAnchor, OpenResponse, [u8; 33]) { + apple_open_response_for_alpn(exporter, nonce, ALPN, counter, counters) +} + +fn apple_open_response_for_alpn( + exporter: &[u8; 32], + nonce: &[u8; 32], + alpn: &[u8], + counter: u32, counter_store: Arc, ) -> (ProviderTrustAnchor, OpenResponse, [u8; 33]) { let signing_key = P256SigningKey::from_bytes((&[7; 32]).into()).unwrap(); @@ -474,7 +484,7 @@ fn apple_open_response( nonce, &bundle.genesis.statement.producer_public_key, expected_genesis, - ALPN, + alpn, ); let apple = AppleAppAttestTrust::new("TESTTEAM.example.app", vec![cd_hash], counter_store); *apple.credential.lock().unwrap() = Some(RegisteredAppleCredential { @@ -906,3 +916,71 @@ async fn fetch_open_ticket_and_signed_output_share_one_verified_connection() { ); server.close().await; } + +#[cfg(feature = "work")] +#[test] +fn paid_services_app_attest_binds_the_exact_connection_and_service() { + for alpn in [ + hellas_rpc::services::work::Work::ALPN, + hellas_rpc::services::work_setup::WorkSetup::ALPN, + ] { + let counters = Arc::new(TestCounterStore::default()); + let (trust, response, _) = + apple_open_response_for_alpn(&[1; 32], &[2; 32], alpn.as_bytes(), 1, counters); + assert!( + verify_open_response( + &trust, + &[1; 32], + &[2; 32], + ALPN, + ENROLLED_PEER, + response.clone() + ) + .is_err() + ); + assert!( + verify_open_response( + &trust, + &[3; 32], + &[2; 32], + alpn.as_bytes(), + ENROLLED_PEER, + response.clone() + ) + .is_err() + ); + assert!( + verify_open_response( + &trust, + &[1; 32], + &[4; 32], + alpn.as_bytes(), + ENROLLED_PEER, + response.clone() + ) + .is_err() + ); + assert!( + verify_open_response( + &trust, + &[1; 32], + &[2; 32], + alpn.as_bytes(), + ENROLLED_PEER, + response.clone() + ) + .is_ok() + ); + assert!( + verify_open_response( + &trust, + &[1; 32], + &[2; 32], + alpn.as_bytes(), + ENROLLED_PEER, + response + ) + .is_err() + ); + } +} diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index 5e55cd80..5d418881 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -26,7 +26,9 @@ pub use fetch::{ verified_fetch_input, verify_fetch_work_event, }; #[cfg(feature = "iroh")] -pub use iroh::{AppleAppAttestTrust, ExecutionRoute, ProviderTrustAnchor, RemoteNodeTarget}; +pub use iroh::{ + AppleAppAttestTrust, ExecutionRoute, ProviderTrustAnchor, RemoteNodeTarget, confidential_open, +}; pub use run_ticket::{runner_public_key, signed_run_ticket_request}; /// Client capabilities shared by local and transport-specific orchestration. diff --git a/crates/client/tests/work.rs b/crates/client/tests/work.rs index 97d9cfe0..d7ca327c 100644 --- a/crates/client/tests/work.rs +++ b/crates/client/tests/work.rs @@ -53,7 +53,7 @@ use hellas_rpc::{ use hellas_wire::mux::{MessagePipe, MuxConfig, MuxTransport, Role as MuxRole}; use hellas_wire::{DefaultClock, Dispatcher, StreamTransport}; use hellas_work::work::{ - BackendFault, ClientEndpoint, PaidEvaluateBackend, PreparedEvaluateInput, ProviderEndpoint, + BackendFault, ClientEndpoint, PaidWorkBackend, PreparedEvaluateInput, ProviderEndpoint, RunOutcome, WorkService, run_accepted_work, }; use hellas_work::work_close::{BlockSourceError, FinalizedBlocks, FinalizedWork, observe}; @@ -190,7 +190,7 @@ fn descriptor_with(policy: PaidExecutionPolicyV1) -> WorkChannelDescriptor { payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: policy, + execution_policy: policy.into(), expected_payment_values: payment_values(), }; match WorkChannelDescriptor::open(config) { @@ -462,7 +462,7 @@ impl AnsweringBackend { } } -impl PaidEvaluateBackend for AnsweringBackend { +impl PaidWorkBackend for AnsweringBackend { fn evaluate( &self, input: PreparedEvaluateInput, diff --git a/crates/executor/src/executor/actor/execution.rs b/crates/executor/src/executor/actor/execution.rs index a36f5b37..c45a0927 100644 --- a/crates/executor/src/executor/actor/execution.rs +++ b/crates/executor/src/executor/actor/execution.rs @@ -807,7 +807,7 @@ impl Executor { self.finish_fetch_slot(); } - fn dispatch_next_fetch(&mut self) { + pub(super) fn dispatch_next_fetch(&mut self) { while self.active_fetches < self.fetch_max_in_flight { let Some(pending) = self.pending_fetches.pop_front() else { return; @@ -984,7 +984,7 @@ fn spawn_fetch_provider( )); } -async fn run_fetch_provider( +pub(super) async fn run_fetch_provider( provider: Arc, request: PreparedFetchRequest, mut projector: Box, diff --git a/crates/executor/src/executor/actor/mod.rs b/crates/executor/src/executor/actor/mod.rs index 823bb709..62e11f8f 100644 --- a/crates/executor/src/executor/actor/mod.rs +++ b/crates/executor/src/executor/actor/mod.rs @@ -1,4 +1,5 @@ pub(super) mod execution; +mod paid_fetch; mod quote; use crate::ExecutorError; @@ -152,6 +153,39 @@ pub struct ExecutorSpawnConfig { pub gpu_config: crate::GpuConfig, } +impl ExecutorSpawnConfig { + /// Construct a Fetch-only runtime independently of whether another Cargo + /// consumer enables Evaluate. Hosts can then override limits and stores. + pub fn fetch_only( + producer_key: Arc, + provider_genesis: Arc>, + assurance: Assurance, + fetch_routes: FetchRouteRegistry, + ) -> Self { + Self { + output_cache: hellas_rpc::cache::CacheOptions::default(), + execute_policy: ExecutePolicy::Deny, + queue_capacity: 1, + metrics: Arc::new(ExecutorMetrics::default()), + producer_key, + provider_genesis, + assurance, + fetch_access_policy: FetchAccessPolicy::trusted_callers([]), + fetch_routes, + fetch_max_in_flight: hellas_rpc::DEFAULT_FETCH_MAX_IN_FLIGHT, + fetch_queue_capacity: hellas_rpc::DEFAULT_FETCH_QUEUE_CAPACITY, + fetch_replay_max_in_flight: hellas_rpc::DEFAULT_FETCH_REPLAY_MAX_IN_FLIGHT, + fetch_store: FetchTranscriptStoreBackend::memory(), + #[cfg(feature = "evaluate")] + artifact_store: ArtifactStoreConfig::memory(), + #[cfg(feature = "evaluate")] + content_store: ContentStore::new(), + #[cfg(feature = "evaluate")] + gpu_config: crate::GpuConfig::default(), + } + } +} + struct ExecutorRuntimeConfig { output_cache: hellas_rpc::cache::CacheOptions, #[cfg_attr(not(feature = "evaluate"), allow(dead_code))] @@ -419,6 +453,12 @@ impl Executor { self.evaluate.on_completion(*completion).await; true } + ExecutorCompletion::PaidFetch { reply, result } => { + self.active_fetches = self.active_fetches.saturating_sub(1); + self.dispatch_next_fetch(); + let _ = reply.send(result); + false + } ExecutorCompletion::FetchFinished(completion) => { self.handle_fetch_finished(*completion); false @@ -450,6 +490,9 @@ impl Executor { async fn handle_owed_request(&mut self, request: ExecutorOwedRequest) { match request { + ExecutorOwedRequest::RunPaidFetch { input, reply } => { + self.start_paid_fetch(*input, reply); + } ExecutorOwedRequest::RunPaidEvaluate { input, reply, span } => { #[cfg(feature = "evaluate")] let result = tracing::Instrument::instrument( diff --git a/crates/executor/src/executor/actor/paid_fetch.rs b/crates/executor/src/executor/actor/paid_fetch.rs new file mode 100644 index 00000000..618fbf45 --- /dev/null +++ b/crates/executor/src/executor/actor/paid_fetch.rs @@ -0,0 +1,126 @@ +//! Paid Fetch runs have their own durable admission in hellas-work. Bodies +//! remain in memory; neither Courtesy replay nor its transcript store is used. + +use std::sync::Arc; + +use hellas_rpc::OutputEventEnvelope; +use hellas_work::work::PreparedFetchInput; +use tokio::sync::{mpsc, oneshot}; + +use super::Executor; +use crate::ExecutorError; +use crate::executor::ExecutorCompletion; +use crate::fetch_policy::FetchRoute; +use crate::fetch_provider::FetchCall; + +impl Executor { + pub(super) fn start_paid_fetch( + &mut self, + input: PreparedFetchInput, + reply: oneshot::Sender, ExecutorError>>, + ) { + let prepared = self.prepare_paid_fetch(input); + let (entry, session, request, policy) = match prepared { + Ok(prepared) => prepared, + Err(error) => { + let _ = reply.send(Err(error)); + return; + } + }; + self.active_fetches += 1; + let completion = self.completion_tx.clone(); + let key = Arc::clone(&self.provider.producer_key); + tokio::spawn(async move { + let (sender, mut receiver) = mpsc::channel(64); + let run = super::execution::run_fetch_provider( + entry.provider, + session.provider_request, + session.projector, + request.input_commitment, + request.assurance, + &key, + sender, + ); + // The shared runner emits wire events as well as its final signed + // transcript. Drain the bounded channel without publishing or + // retaining a second copy of the response. + let drain = async { while receiver.recv().await.is_some() {} }; + let (result, ()) = tokio::join!(run, drain); + let result = result + .map_err(|_| { + ExecutorError::Execution("paid fetch upstream or projection failed".into()) + }) + .and_then(|run| { + hellas_rpc::protocol::work_fetch::check_fetch_output_limits( + &policy, + &run.output_events, + ) + .map_err(|_| { + ExecutorError::Execution( + "paid fetch output exceeds its signed limits".into(), + ) + })?; + Ok(run.output_events) + }); + let _ = completion + .send(ExecutorCompletion::PaidFetch { reply, result }) + .await; + }); + } + + fn prepare_paid_fetch( + &self, + input: PreparedFetchInput, + ) -> Result< + ( + crate::FetchRouteEntry, + crate::FetchAdaptorSession, + hellas_rpc::fetch::FetchInput, + hellas_rpc::protocol::work_fetch::PaidFetchPolicyV1, + ), + ExecutorError, + > { + if self.active_fetches >= self.fetch_max_in_flight { + return Err(ExecutorError::ResourceExhausted( + "fetch concurrency limit reached".into(), + )); + } + let policy = *input.policy(); + let parts = input.into_parts(); + let request = hellas_rpc::fetch::verify_input_events(&parts.fetch_input_transcript) + .map_err(|_| ExecutorError::InvalidQuoteRequest("invalid paid fetch input".into()))?; + if request.retention != hellas_rpc::Retention::Ephemeral + || request.assurance != self.provider.assurance + || request.execution_environment != parts.manifest.content_id() + { + return Err(ExecutorError::InvalidQuoteRequest( + "paid fetch contract mismatch".into(), + )); + } + let route = FetchRoute::new(&request.service, &request.method); + let entry = self.fetch_routes.entry(&route).cloned().ok_or_else(|| { + ExecutorError::InvalidQuoteRequest("paid fetch route is unavailable".into()) + })?; + if entry.execution_environment() != request.execution_environment { + return Err(ExecutorError::InvalidQuoteRequest( + "paid fetch route manifest mismatch".into(), + )); + } + let call = FetchCall::new( + &request.service, + &request.method, + request.body.clone(), + request.input_commitment, + ); + let session = entry.adaptor_factory.create(&call).map_err(|_| { + ExecutorError::InvalidQuoteRequest("paid fetch adaptor rejected request".into()) + })?; + entry + .capabilities + .validate(&session.request_view) + .map_err(|_| { + ExecutorError::PolicyDenied("paid fetch exceeds route capabilities".into()) + })?; + Ok((entry, session, request, policy)) + } +} diff --git a/crates/executor/src/executor/mod.rs b/crates/executor/src/executor/mod.rs index fb39fcb1..f6a99752 100644 --- a/crates/executor/src/executor/mod.rs +++ b/crates/executor/src/executor/mod.rs @@ -107,6 +107,10 @@ pub(crate) enum ExecutorRequest { /// This has a distinct bounded ingress so peer RPC traffic cannot delay its /// admission past queued best-effort execution. pub(crate) enum ExecutorOwedRequest { + RunPaidFetch { + input: Box, + reply: oneshot::Sender, ExecutorError>>, + }, /// Start one already-authorized paid job. /// /// No ticket, no quote, and no admission of its own: the paid endpoint @@ -128,6 +132,10 @@ pub(crate) enum ExecutorOwedRequest { /// Trusted notifications from the bounded set of active execution producers. pub(crate) enum ExecutorCompletion { + PaidFetch { + reply: oneshot::Sender, ExecutorError>>, + result: Result, ExecutorError>, + }, #[cfg(feature = "evaluate")] EvaluateFinished(Box), FetchFinished(Box), diff --git a/crates/executor/src/fetch_policy.rs b/crates/executor/src/fetch_policy.rs index eedbef3b..6c313697 100644 --- a/crates/executor/src/fetch_policy.rs +++ b/crates/executor/src/fetch_policy.rs @@ -624,7 +624,7 @@ impl FetchRoutePolicy { } } - fn validate(&self, request: &FetchRequestView) -> Result<(), FetchAccessError> { + pub(crate) fn validate(&self, request: &FetchRequestView) -> Result<(), FetchAccessError> { if let Some(models) = &self.allowed_models { let Some(model) = &request.model else { return Err(FetchAccessError::Denied( diff --git a/crates/executor/src/fetch_provider.rs b/crates/executor/src/fetch_provider.rs index 7ca50652..9cf4bed3 100644 --- a/crates/executor/src/fetch_provider.rs +++ b/crates/executor/src/fetch_provider.rs @@ -16,11 +16,18 @@ pub type FetchProviderStream = #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct FetchProviderResponseHead { pub effective_model: Option, + pub http: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HttpResponseHead { + pub status: u16, + pub headers: Vec<(String, String)>, } impl FetchProviderResponseHead { pub const fn is_empty(&self) -> bool { - self.effective_model.is_none() + self.effective_model.is_none() && self.http.is_none() } } diff --git a/crates/executor/src/fetch_registry.rs b/crates/executor/src/fetch_registry.rs index e04238fb..0cadf8a8 100644 --- a/crates/executor/src/fetch_registry.rs +++ b/crates/executor/src/fetch_registry.rs @@ -100,6 +100,13 @@ impl FetchRouteRegistry { } } + /// Whether at least one configured route implements this trusted manifest. + pub fn has_environment(&self, environment: ContentId) -> bool { + self.routes + .values() + .any(|entry| entry.execution_environment() == environment) + } + pub fn entry(&self, route: &FetchRoute) -> Option<&FetchRouteEntry> { self.routes.get(route) } diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs index 7a5de75d..e17aa111 100644 --- a/crates/executor/src/lib.rs +++ b/crates/executor/src/lib.rs @@ -47,7 +47,8 @@ pub use fetch_projection::{ }; pub use fetch_provider::{ FetchCall, FetchProvider, FetchProviderError, FetchProviderFuture, FetchProviderResponse, - FetchProviderResponseHead, FetchProviderStream, MockFetchProvider, PreparedFetchRequest, + FetchProviderResponseHead, FetchProviderStream, HttpResponseHead, MockFetchProvider, + PreparedFetchRequest, }; pub use fetch_registry::{ DuplicateFetchRoute, FetchRouteBindingError, FetchRouteEntry, FetchRouteRegistry, diff --git a/crates/executor/src/work.rs b/crates/executor/src/work.rs index cddad3f5..73b6965a 100644 --- a/crates/executor/src/work.rs +++ b/crates/executor/src/work.rs @@ -2,7 +2,7 @@ //! //! # What this is //! -//! One implementation of [`PaidEvaluateBackend`], over this crate's own +//! One implementation of [`PaidWorkBackend`], over this crate's own //! Evaluate engine. The gate that decides whether a backend may be //! called at all is [`hellas_work::work::run_accepted_work`]'s, and it is //! not here: it belongs beside the journal that records the decision, @@ -44,7 +44,7 @@ use crate::ExecutorError; use crate::executor::{ExecutorHandle, ExecutorOwedRequest}; use hellas_rpc::OutputEventEnvelope; -use hellas_work::work::{BackendFault, PaidEvaluateBackend, PaidProgress, PreparedEvaluateInput}; +use hellas_work::work::{BackendFault, PaidProgress, PaidWorkBackend, PreparedEvaluateInput}; impl ExecutorHandle { /// Runs one already-authorized paid job to its terminal. @@ -169,7 +169,19 @@ async fn drain_transcript_with_progress( } } -impl PaidEvaluateBackend for ExecutorHandle { +impl PaidWorkBackend for ExecutorHandle { + async fn fetch( + &self, + input: hellas_work::work::PreparedFetchInput, + ) -> Result, BackendFault> { + self.send_owed(|reply| ExecutorOwedRequest::RunPaidFetch { + input: Box::new(input), + reply, + }) + .await + .map_err(|error| BackendFault::new(error.to_string())) + } + async fn evaluate( &self, input: PreparedEvaluateInput, diff --git a/crates/providers/Cargo.toml b/crates/providers/Cargo.toml index 9b505d66..c4241db4 100644 --- a/crates/providers/Cargo.toml +++ b/crates/providers/Cargo.toml @@ -12,6 +12,12 @@ default = [] otel = ["hellas-rpc/otel", "dep:hellas-wire"] [dependencies] +base64.workspace = true +url = "2" +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } +webpki-roots = "1" +sha2.workspace = true +x509-cert.workspace = true anyhow = { workspace = true } async-stream = { workspace = true } futures.workspace = true @@ -26,6 +32,8 @@ tokio.workspace = true tracing.workspace = true [dev-dependencies] +rcgen = "0.14" +tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } axum.workspace = true opentelemetry.workspace = true opentelemetry_sdk = { workspace = true, features = ["testing"] } diff --git a/crates/providers/HTTPS.md b/crates/providers/HTTPS.md new file mode 100644 index 00000000..0b6f8435 --- /dev/null +++ b/crates/providers/HTTPS.md @@ -0,0 +1,99 @@ +# Caller-signed HTTPS Fetch + +The `http` environment signs raw HTTP metadata and bytes, allowing OpenAI, +Anthropic, Google, Kimi or another API without adding a vendor to the app. +It does not translate request schemas: supply the upstream's own JSON or binary +body, encoded as standard padded base64. + +```json +{ + "url": "https://api.example.com/v1/messages", + "method": "POST", + "headers": [["content-type", "application/json"]], + "body_base64": "e30=", + "tls": { + "roots": {"mode": "web_pki"}, + "spki_sha256": [] + }, + "credential": "account-1", + "max_response_bytes": 262144 +} +``` + +`body_base64` above is `{}`; replace it with your actual API request. Header +names are lowercase. `credential` may be omitted for unauthenticated requests +or caller-owned authorization headers. The response ceiling is at most 512 KiB. +All final HTTP statuses, including errors and redirects, are returned to the +client as signed responses; they are never logged with their bodies. Redirects +are not followed. A transport failure or an oversized response is not a +successful paid result, and is never retried automatically. + +`roots.mode` is either `web_pki` (bundled public trust roots) or `certificates` +with `der_base64`, a nonempty array of exact DER trust anchors. System roots +are not silently added. Hostname, certificate validity, chain signatures and +TLS handshake signatures are always checked. `spki_sha256` is an optional +array of lowercase 64-digit SHA-256 hashes of the leaf certificate's DER SPKI; +at least one must match when supplied. Pins add constraints to chain validation. +A caller-supplied trust anchor does not qualify for provider-owned credentials. + +The operator configures account aliases separately: + +```json +{ + "allowed_hosts": [], + "allow_private_addresses": false, + "credentials": { + "account-1": { + "allowed_origins": ["https://api.example.com"], + "allowed_paths": ["/v1/messages"], + "allowed_methods": ["POST"], + "header_name": "authorization", + "secret_env": "ACCOUNT_ONE_API_KEY", + "prefix": "Bearer " + }, + "account-2": { + "allowed_origins": ["https://api.example.com"], + "allowed_paths": ["/v1/messages"], + "allowed_methods": ["POST"], + "header_name": "x-api-key", + "secret_env": "ACCOUNT_TWO_API_KEY", + "prefix": "" + } + } +} +``` + +Aliases select independent accounts, including several accounts at the same +origin. Secrets are read from the provider process's environment and held in +memory. They are injected only for an exact authorized origin, path and method, +with public WebPKI roots. Callers cannot override that account's header or +change trust roots to impersonate its origin. Restrict paths to the inference +endpoints the account is intended to expose. + +DNS answers are checked and pinned to the request's connection. Private, +loopback, link-local, multicast and other special-purpose addresses are denied +by default. An operator can explicitly enable private addresses only with a +nonempty exact host allowlist, for controlled private services. Environment +proxies, redirects, transparent decompression and request retries are disabled. + +In Gate, select **HTTPS (caller selects URL)** and paste this operator config +into **HTTPS accounts and egress**. `{}` enables public HTTPS without provider +accounts. In the CLI's Fetch route file, use: + +```json +{ + "service": "http", + "method": "request", + "destination": {"type": "http", "config": {"credentials": {}}} +} +``` + +The normal route file still needs its `routes` and `callers` envelope. Paid +channels use their payment policy for admission; Courtesy callers use explicit +route grants. On Gate's Run page, `http` is accepted as the execution environment. +For CLI paid requests, `paid-work prepare-fetch --execution-environment http` +prints the manifest ID to put in the work config. + +Results contain one `Adaptor.Http.Head` event followed by base64 body events +and a signed terminal. `HttpFetchResponse::from_output` reconstructs the body +and checks ordering and size after transcript signature verification. diff --git a/crates/providers/src/codex_responses/tests.rs b/crates/providers/src/codex_responses/tests.rs index fae28284..4a2ec71b 100644 --- a/crates/providers/src/codex_responses/tests.rs +++ b/crates/providers/src/codex_responses/tests.rs @@ -645,6 +645,7 @@ fn model_head_and_lifecycle_claims_are_correlated_before_signing() { projector .begin(FetchProviderResponseHead { effective_model: Some("routed-model".to_string()), + http: None, }) .unwrap(); let events = vec![ @@ -663,6 +664,7 @@ fn model_head_and_lifecycle_claims_are_correlated_before_signing() { projector .begin(FetchProviderResponseHead { effective_model: Some("header-model".to_string()), + http: None, }) .unwrap(); let conflicting = diff --git a/crates/providers/src/http.rs b/crates/providers/src/http.rs new file mode 100644 index 00000000..7d821686 --- /dev/null +++ b/crates/providers/src/http.rs @@ -0,0 +1,427 @@ +//! A generic HTTPS driver. The URL and TLS contract are in the signed input; +//! provider credentials are origin-scoped and held in memory. + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use futures::StreamExt as _; +use hellas_executor::{ + FetchAdaptorError, FetchAdaptorFactory, FetchAdaptorSession, FetchCall, FetchProjector, + FetchProvider, FetchProviderError, FetchProviderFuture, FetchProviderResponse, + FetchProviderResponseHead, FetchRequestView, HttpResponseHead, PreparedFetchRequest, + ProjectedFetch, +}; +use hellas_rpc::http_fetch::{HttpFetchRequest, check_headers}; +use hellas_rpc::output::{AdaptorEvent, HttpResponseEvent, OutputEvent, StopReason}; +use hellas_rpc::{ContentId, FetchEnvironment}; +use reqwest::{ + Url, + header::{HeaderName, HeaderValue}, +}; +use std::{ + collections::BTreeMap, + net::{IpAddr, SocketAddr}, + sync::Arc, + time::Duration, +}; +mod config; +mod tls; +pub use config::{HttpCredentialConfig, HttpProviderConfig}; + +const CHUNK_BYTES: usize = 16 * 1024; +const IDLE: Duration = Duration::from_secs(90); + +/// Exact DNS names (not suffixes/wildcards). Empty allows any public host. +/// Private addresses require an explicit operator opt-in and a nonempty host +/// allowlist. Resolved addresses are checked and pinned for the connection. +#[derive(Clone, Debug, Default)] +pub struct HttpEgressPolicy { + pub allowed_hosts: Vec, + pub allow_private_addresses: bool, +} + +/// In-memory account credential. The exact HTTPS origins are operator-owned; +/// neither a request URL nor its trust anchors can widen this scope. +#[derive(Clone)] +pub struct HttpCredential { + pub allowed_origins: Vec, + pub allowed_paths: Vec, + pub allowed_methods: Vec, + pub header_name: String, + pub header_value: String, +} + +impl std::fmt::Debug for HttpCredential { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HttpCredential") + .field("allowed_origins", &self.allowed_origins) + .field("header_name", &self.header_name) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug)] +pub struct HttpFetchProvider { + policy: HttpEgressPolicy, + credentials: Arc>, +} + +fn fault(message: &'static str) -> FetchProviderError { + FetchProviderError::failed(message) +} + +impl HttpFetchProvider { + pub fn new( + policy: HttpEgressPolicy, + credentials: BTreeMap, + ) -> Result { + if policy.allow_private_addresses && policy.allowed_hosts.is_empty() { + return Err(fault("private egress requires an explicit host allowlist")); + } + for host in &policy.allowed_hosts { + let parsed = Url::parse(&format!("https://{host}/")) + .map_err(|_| fault("invalid egress host"))?; + if parsed.host_str() != Some(host) + || parsed.port().is_some() + || !parsed.username().is_empty() + { + return Err(fault("egress hosts must be canonical names without ports")); + } + } + for (alias, credential) in &credentials { + if alias.is_empty() + || credential.allowed_origins.is_empty() + || credential.allowed_paths.is_empty() + || credential.allowed_methods.is_empty() + { + return Err(fault( + "credential requires an alias, exact origins, paths and methods", + )); + } + check_headers( + &[( + credential.header_name.clone(), + credential.header_value.clone(), + )], + true, + ) + .map_err(|_| fault("invalid credential header"))?; + for path in &credential.allowed_paths { + let parsed = Url::parse(&format!("https://scope.invalid{path}")) + .map_err(|_| fault("invalid credential path"))?; + if !path.starts_with('/') + || parsed.path() != path + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(fault("credential paths must be exact canonical URL paths")); + } + } + if credential.allowed_methods.iter().any(|method| { + !matches!( + method.as_str(), + "GET" | "HEAD" | "POST" | "PUT" | "PATCH" | "DELETE" | "OPTIONS" + ) + }) { + return Err(fault("invalid credential method")); + } + for origin in &credential.allowed_origins { + let parsed = Url::parse(origin).map_err(|_| fault("invalid credential origin"))?; + if parsed.scheme() != "https" || parsed.origin().ascii_serialization() != *origin { + return Err(fault( + "credential origins must be exact canonical HTTPS origins", + )); + } + } + } + Ok(Self { + policy, + credentials: Arc::new(credentials), + }) + } + + fn credential( + &self, + request: &HttpFetchRequest, + url: &Url, + ) -> Result, FetchProviderError> { + let Some(alias) = &request.credential else { + return Ok(None); + }; + let credential = self + .credentials + .get(alias) + .ok_or_else(|| fault("unknown credential alias"))?; + if !credential + .allowed_origins + .contains(&url.origin().ascii_serialization()) + || !credential + .allowed_paths + .iter() + .any(|path| path == url.path()) + || !credential.allowed_methods.contains(&request.method) + { + return Err(fault( + "credential is not authorized for this URL and method", + )); + } + // A caller-controlled CA could impersonate the allowed origin and + // steal the provider's credential. Account requests must keep the + // public WebPKI trust boundary; additional pins can only narrow it. + if !matches!( + request.tls.roots, + hellas_rpc::http_fetch::HttpTrustRoots::WebPki + ) { + return Err(fault("provider credentials require WebPKI roots")); + } + if request + .headers + .iter() + .any(|(name, _)| name == &credential.header_name) + { + return Err(fault("request overrides a provider credential header")); + } + Ok(Some(credential)) + } + + async fn execute( + &self, + prepared: PreparedFetchRequest, + ) -> Result { + let request = HttpFetchRequest::decode(prepared.body.as_bytes()) + .map_err(|_| fault("invalid HTTPS request"))?; + let url = request + .parsed_url() + .map_err(|_| fault("invalid HTTPS URL"))?; + let host = url.host_str().ok_or_else(|| fault("missing HTTPS host"))?; + if !self.policy.allowed_hosts.is_empty() + && !self + .policy + .allowed_hosts + .iter() + .any(|allowed| allowed == host) + { + return Err(fault("HTTPS host is outside the egress policy")); + } + let credential = self.credential(&request, &url)?; + let port = url + .port_or_known_default() + .ok_or_else(|| fault("missing HTTPS port"))?; + let addresses: Vec = + match url.host().ok_or_else(|| fault("missing HTTPS host"))? { + url::Host::Ipv4(ip) => vec![SocketAddr::new(ip.into(), port)], + url::Host::Ipv6(ip) => vec![SocketAddr::new(ip.into(), port)], + url::Host::Domain(host) => tokio::time::timeout( + Duration::from_secs(10), + tokio::net::lookup_host((host, port)), + ) + .await + .map_err(|_| fault("HTTPS DNS lookup timed out"))? + .map_err(|_| fault("HTTPS DNS lookup failed"))? + .take(65) + .collect(), + }; + if addresses.is_empty() + || addresses.len() > 64 + || addresses.iter().any(|address| { + !self.policy.allow_private_addresses && !public_address(address.ip()) + }) + { + return Err(fault("HTTPS DNS addresses are outside the egress policy")); + } + let tls = tls::config(&request.tls).map_err(fault)?; + let client = reqwest::Client::builder() + .no_proxy() + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() + .redirect(reqwest::redirect::Policy::none()) + .retry(reqwest::retry::never()) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(300)) + .resolve_to_addrs(host, &addresses) + .tls_backend_preconfigured(tls) + .build() + .map_err(|_| fault("HTTPS client initialization failed"))?; + let method = request + .method + .parse() + .map_err(|_| fault("invalid HTTP method"))?; + let mut outbound = client.request(method, url.clone()); + for (name, value) in &request.headers { + outbound = outbound.header( + HeaderName::from_bytes(name.as_bytes()) + .map_err(|_| fault("invalid HTTP header"))?, + HeaderValue::from_str(value).map_err(|_| fault("invalid HTTP header"))?, + ); + } + if let Some(credential) = credential { + let mut value = HeaderValue::from_str(&credential.header_value) + .map_err(|_| fault("invalid credential header"))?; + value.set_sensitive(true); + outbound = outbound.header(&credential.header_name, value); + } + let response = outbound + .body(request.body().map_err(|_| fault("invalid HTTP body"))?) + .send() + .await + .map_err(|_| fault("HTTPS transport or certificate verification failed"))?; + // A non-2xx status is still a completed HTTP exchange. Return it, with + // its exact body, to the authenticated client; do not log it. + let headers = response + .headers() + .iter() + .map(|(k, v)| { + Ok(( + k.to_string(), + v.to_str() + .map_err(|_| fault("unsupported HTTP response header encoding"))? + .to_string(), + )) + }) + .collect::, FetchProviderError>>()?; + check_headers(&headers, false).map_err(|_| fault("HTTP response header limit"))?; + let head = HttpResponseHead { + status: response.status().as_u16(), + headers, + }; + let limit = request.max_response_bytes as usize; + let stream = async_stream::try_stream! { + let mut upstream = response.bytes_stream(); + let mut buffered = Vec::with_capacity(CHUNK_BYTES); + let mut received = 0usize; + loop { + let next = tokio::time::timeout(IDLE, upstream.next()).await + .map_err(|_| fault("HTTPS response idle timeout"))?; + let Some(chunk) = next else { break; }; + let chunk = chunk.map_err(|_| fault("HTTPS response stream failed"))?; + received = received.checked_add(chunk.len()).ok_or_else(|| fault("HTTPS response size overflow"))?; + if received > limit { Err(fault("HTTPS response exceeds signed byte limit"))?; } + let mut remaining = chunk.as_ref(); + while !remaining.is_empty() { + let count = (CHUNK_BYTES - buffered.len()).min(remaining.len()); + buffered.extend_from_slice(&remaining[..count]); + remaining = &remaining[count..]; + if buffered.len() == CHUNK_BYTES { + yield std::mem::replace(&mut buffered, Vec::with_capacity(CHUNK_BYTES)); + } + } + } + if !buffered.is_empty() { yield buffered; } + }; + Ok(FetchProviderResponse { + head: FetchProviderResponseHead { + effective_model: None, + http: Some(head), + }, + stream: Box::pin(stream), + }) + } +} + +impl FetchProvider for HttpFetchProvider { + fn execution_environment(&self) -> ContentId { + FetchEnvironment::Http.manifest_id() + } + fn run(&self, request: PreparedFetchRequest) -> FetchProviderFuture<'_> { + Box::pin(self.execute(request)) + } +} + +fn public_address(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(ip) => { + let [a, b, _, _] = ip.octets(); + !(ip.is_private() + || ip.is_loopback() + || ip.is_link_local() + || ip.is_broadcast() + || ip.is_documentation() + || ip.is_unspecified() + || ip.is_multicast() + || a == 0 + || a >= 240 + || (a == 100 && (64..=127).contains(&b)) + || (a == 198 && (b == 18 || b == 19)) + || (a == 192 && b == 0)) + } + IpAddr::V6(ip) => { + // Restrict to global unicast, excluding documentation, special + // transition ranges, and mapped addresses. + let s = ip.segments(); + (s[0] & 0xe000 == 0x2000) + && s[0] != 0x2002 + && !(s[0] == 0x2001 && (s[1] < 0x0200 || s[1] == 0x0db8)) + && !(s[0] == 0x3fff && s[1] < 0x1000) + } + } +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct HttpFetchAdaptorFactory; +impl FetchAdaptorFactory for HttpFetchAdaptorFactory { + fn execution_environment(&self) -> ContentId { + FetchEnvironment::Http.manifest_id() + } + fn create(&self, call: &FetchCall) -> Result { + HttpFetchRequest::decode(call.body.as_bytes()) + .map_err(|_| FetchAdaptorError::failed("invalid HTTPS request"))?; + Ok(FetchAdaptorSession { + request_view: FetchRequestView::from_call(call), + provider_request: PreparedFetchRequest::new(call, call.body.clone()), + projector: Box::new(HttpProjector { started: false }), + }) + } +} +struct HttpProjector { + started: bool, +} +fn event(event: HttpResponseEvent) -> Result { + hellas_rpc::fetch::encode_fetch_event_payload(&OutputEvent::Adaptor(AdaptorEvent::Http(event))) + .map(ProjectedFetch::Event) + .map_err(|_| FetchAdaptorError::failed("HTTP event encoding failed")) +} +impl FetchProjector for HttpProjector { + fn begin( + &mut self, + head: FetchProviderResponseHead, + ) -> Result, FetchAdaptorError> { + if self.started || head.effective_model.is_some() { + return Err(FetchAdaptorError::failed("invalid HTTP response head")); + } + let head = head + .http + .ok_or_else(|| FetchAdaptorError::failed("missing HTTP response head"))?; + if !(100..=599).contains(&head.status) { + return Err(FetchAdaptorError::failed("invalid HTTP status")); + } + check_headers(&head.headers, false) + .map_err(|_| FetchAdaptorError::failed("invalid HTTP response headers"))?; + self.started = true; + Ok(vec![event(HttpResponseEvent::Head { + status: head.status, + headers: head.headers, + })?]) + } + fn project(&mut self, bytes: &[u8]) -> Result, FetchAdaptorError> { + if !self.started { + return Err(FetchAdaptorError::failed("HTTP body before headers")); + } + Ok(vec![event(HttpResponseEvent::Body { + base64: STANDARD.encode(bytes), + })?]) + } + fn finish(&mut self) -> Result, FetchAdaptorError> { + if !self.started { + return Err(FetchAdaptorError::failed("HTTP response missing")); + } + hellas_rpc::fetch::encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .map(|payload| vec![ProjectedFetch::Terminal(payload)]) + .map_err(|_| FetchAdaptorError::failed("HTTP terminal encoding failed")) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/providers/src/http/config.rs b/crates/providers/src/http/config.rs new file mode 100644 index 00000000..6b1beca6 --- /dev/null +++ b/crates/providers/src/http/config.rs @@ -0,0 +1,68 @@ +//! Operator-owned account aliases. Files contain environment variable names, +//! never customer payloads or API keys. +use super::{HttpCredential, HttpEgressPolicy, HttpFetchProvider}; +use hellas_executor::{FetchProviderError, FetchRouteEntry, FetchRoutePolicy}; +use serde::Deserialize; +use std::collections::BTreeMap; + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpProviderConfig { + #[serde(default)] + pub allowed_hosts: Vec, + #[serde(default)] + pub allow_private_addresses: bool, + #[serde(default)] + pub credentials: BTreeMap, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpCredentialConfig { + pub allowed_origins: Vec, + pub allowed_paths: Vec, + pub allowed_methods: Vec, + pub header_name: String, + pub secret_env: String, + #[serde(default)] + pub prefix: String, +} + +impl HttpProviderConfig { + pub fn into_entry( + self, + capabilities: FetchRoutePolicy, + ) -> Result { + let mut credentials = BTreeMap::new(); + for (alias, config) in self.credentials { + let secret = std::env::var(&config.secret_env) + .map_err(|_| super::fault("account secret environment variable is unavailable"))?; + if secret.is_empty() { + return Err(super::fault("account secret is empty")); + } + credentials.insert( + alias, + HttpCredential { + allowed_origins: config.allowed_origins, + allowed_paths: config.allowed_paths, + allowed_methods: config.allowed_methods, + header_name: config.header_name, + header_value: format!("{}{secret}", config.prefix), + }, + ); + } + let provider = HttpFetchProvider::new( + HttpEgressPolicy { + allowed_hosts: self.allowed_hosts, + allow_private_addresses: self.allow_private_addresses, + }, + credentials, + )?; + FetchRouteEntry::new( + std::sync::Arc::new(provider), + std::sync::Arc::new(super::HttpFetchAdaptorFactory), + capabilities, + ) + .map_err(|_| super::fault("invalid HTTP Fetch route")) + } +} diff --git a/crates/providers/src/http/tests.rs b/crates/providers/src/http/tests.rs new file mode 100644 index 00000000..e5874729 --- /dev/null +++ b/crates/providers/src/http/tests.rs @@ -0,0 +1,269 @@ +use super::*; +use hellas_rpc::http_fetch::{HttpTls, HttpTrustRoots}; +use hellas_rpc::{Digest, InputCommitment, JsonBytes}; +use rcgen::generate_simple_self_signed; +use sha2::{Digest as _, Sha256}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use x509_cert::der::{Decode as _, Encode as _}; + +async fn server( + status: u16, + bytes: Vec, + location: Option, +) -> ( + HttpFetchRequest, + Arc, + tokio::task::JoinHandle<()>, +) { + let key = generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let cert = key.cert.der().clone(); + let parsed = x509_cert::Certificate::from_der(cert.as_ref()).unwrap(); + let pin = Sha256::digest( + parsed + .tbs_certificate + .subject_public_key_info + .to_der() + .unwrap(), + ) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + let private = rustls::pki_types::PrivatePkcs8KeyDer::from(key.signing_key.serialize_der()); + let config = rustls::ServerConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert(vec![cert.clone()], private.into()) + .unwrap(); + let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!( + "https://localhost:{}/resource", + listener.local_addr().unwrap().port() + ); + let calls = Arc::new(AtomicUsize::new(0)); + let seen = calls.clone(); + let task = tokio::spawn(async move { + while let Ok((socket, _)) = listener.accept().await { + let acceptor = acceptor.clone(); + let seen = seen.clone(); + let bytes = bytes.clone(); + let location = location.clone(); + tokio::spawn(async move { + let Ok(mut socket) = acceptor.accept(socket).await else { + return; + }; + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + while !request.ends_with(b"\r\n\r\n") && request.len() < 32768 { + if socket.read_exact(&mut byte).await.is_err() { + return; + } + request.push(byte[0]); + } + seen.fetch_add(1, Ordering::SeqCst); + let location = location + .map(|v| format!("Location: {v}\r\n")) + .unwrap_or_default(); + let header = format!( + "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: close\r\n{location}\r\n", + bytes.len() + ); + let _ = socket.write_all(header.as_bytes()).await; + let _ = socket.write_all(&bytes).await; + let _ = socket.shutdown().await; + }); + } + }); + ( + HttpFetchRequest { + url, + method: "GET".into(), + headers: vec![], + body_base64: String::new(), + tls: HttpTls { + roots: HttpTrustRoots::Certificates { + der_base64: vec![STANDARD.encode(&cert)], + }, + spki_sha256: vec![pin], + }, + credential: None, + max_response_bytes: 4096, + }, + calls, + task, + ) +} + +fn provider() -> HttpFetchProvider { + HttpFetchProvider::new( + HttpEgressPolicy { + allowed_hosts: vec!["localhost".into()], + allow_private_addresses: true, + }, + BTreeMap::new(), + ) + .unwrap() +} +fn prepared(request: &HttpFetchRequest) -> PreparedFetchRequest { + let call = FetchCall::new( + "http", + "request", + JsonBytes::new(serde_json::to_vec(request).unwrap()), + InputCommitment::from_digest(Digest::from_bytes([1; 32])), + ); + HttpFetchAdaptorFactory + .create(&call) + .unwrap() + .provider_request +} + +#[tokio::test] +async fn custom_roots_and_spki_deliver_exact_binary_bytes() { + let bytes = vec![0, 255, 1, 13, 10, 128]; + let (request, calls, task) = server(200, bytes.clone(), None).await; + let mut response = provider().run(prepared(&request)).await.unwrap(); + assert_eq!(response.head.http.as_ref().unwrap().status, 200); + let mut body = Vec::new(); + while let Some(chunk) = response.stream.next().await { + body.extend(chunk.unwrap()); + } + assert_eq!(body, bytes); + assert_eq!(calls.load(Ordering::SeqCst), 1); + task.abort(); +} + +#[tokio::test] +async fn wrong_pin_wrong_roots_and_wrong_hostname_send_no_http_request() { + let (valid, calls, task) = server(200, b"ok".to_vec(), None).await; + let mut wrong_pin = valid.clone(); + wrong_pin.tls.spki_sha256 = vec!["00".repeat(32)]; + let mut wrong_root = valid.clone(); + wrong_root.tls.roots = HttpTrustRoots::WebPki; + let mut wrong_name = valid.clone(); + wrong_name.url = wrong_name.url.replace("localhost", "127.0.0.1"); + let configured = HttpFetchProvider::new( + HttpEgressPolicy { + allowed_hosts: vec!["localhost".into(), "127.0.0.1".into()], + allow_private_addresses: true, + }, + BTreeMap::new(), + ) + .unwrap(); + for request in [wrong_pin, wrong_root, wrong_name] { + assert!(configured.run(prepared(&request)).await.is_err()); + } + assert_eq!(calls.load(Ordering::SeqCst), 0); + task.abort(); +} + +#[tokio::test] +async fn redirects_are_returned_without_following_them() { + let (target, target_calls, target_task) = server(200, b"private".to_vec(), None).await; + let (origin, origin_calls, origin_task) = server(302, vec![], Some(target.url)).await; + let response = provider().run(prepared(&origin)).await.unwrap(); + assert_eq!(response.head.http.unwrap().status, 302); + assert_eq!(origin_calls.load(Ordering::SeqCst), 1); + assert_eq!(target_calls.load(Ordering::SeqCst), 0); + origin_task.abort(); + target_task.abort(); +} + +#[tokio::test] +async fn signed_response_size_and_default_private_address_denial_are_enforced() { + let (mut request, calls, task) = server(200, vec![42; 100], None).await; + let public = HttpFetchProvider::new(HttpEgressPolicy::default(), BTreeMap::new()).unwrap(); + assert!(public.run(prepared(&request)).await.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 0); + request.max_response_bytes = 16; + let mut response = provider().run(prepared(&request)).await.unwrap(); + assert!(response.stream.next().await.unwrap().is_err()); + task.abort(); +} + +#[test] +fn credentials_cannot_be_redirected_or_used_with_caller_trust_anchors() { + let credential = HttpCredential { + allowed_origins: vec!["https://api.example.com".into()], + allowed_paths: vec!["/path".into()], + allowed_methods: vec!["POST".into()], + header_name: "authorization".into(), + header_value: "Bearer PRIVATE".into(), + }; + let provider = HttpFetchProvider::new( + HttpEgressPolicy::default(), + BTreeMap::from([("account-1".into(), credential)]), + ) + .unwrap(); + let mut request = HttpFetchRequest { + url: "https://api.example.com/path".into(), + method: "POST".into(), + headers: vec![], + body_base64: String::new(), + tls: HttpTls { + roots: HttpTrustRoots::WebPki, + spki_sha256: vec![], + }, + credential: Some("account-1".into()), + max_response_bytes: 1024, + }; + assert!( + provider + .credential(&request, &request.parsed_url().unwrap()) + .unwrap() + .is_some() + ); + request.method = "DELETE".into(); + assert!( + provider + .credential(&request, &request.parsed_url().unwrap()) + .is_err() + ); + request.method = "POST".into(); + request.url = "https://api.example.com/other".into(); + assert!( + provider + .credential(&request, &request.parsed_url().unwrap()) + .is_err() + ); + request.url = "https://attacker.example/".into(); + assert!( + provider + .credential(&request, &request.parsed_url().unwrap()) + .is_err() + ); + request.url = "https://api.example.com/path".into(); + request.tls.roots = HttpTrustRoots::Certificates { der_base64: vec![] }; + assert!( + provider + .credential(&request, &request.parsed_url().unwrap()) + .is_err() + ); + assert!(!format!("{provider:?}").contains("PRIVATE")); +} + +#[test] +fn special_addresses_never_qualify_as_public_egress() { + for ip in [ + "127.0.0.1", + "10.1.2.3", + "169.254.169.254", + "100.64.0.1", + "0.0.0.0", + "192.0.2.1", + "198.18.0.1", + "224.0.0.1", + "::1", + "::ffff:8.8.8.8", + "64:ff9b::808:808", + "2001:db8::1", + "2002:7f00:1::", + ] { + assert!(!public_address(ip.parse().unwrap()), "{ip}"); + } + assert!(public_address("8.8.8.8".parse().unwrap())); + assert!(public_address("2606:4700:4700::1111".parse().unwrap())); +} diff --git a/crates/providers/src/http/tls.rs b/crates/providers/src/http/tls.rs new file mode 100644 index 00000000..1d2a95bd --- /dev/null +++ b/crates/providers/src/http/tls.rs @@ -0,0 +1,100 @@ +use hellas_rpc::http_fetch::{HttpTls, HttpTrustRoots, decode_base64, decode_pin}; +use rustls::client::WebPkiServerVerifier; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{DigitallySignedStruct, SignatureScheme}; +use sha2::{Digest as _, Sha256}; +use std::sync::Arc; +use x509_cert::der::{Decode as _, Encode as _}; + +pub(super) fn config(settings: &HttpTls) -> Result { + settings.validate().map_err(|_| "invalid TLS settings")?; + let mut roots = rustls::RootCertStore::empty(); + match &settings.roots { + HttpTrustRoots::WebPki => roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()), + HttpTrustRoots::Certificates { der_base64 } => { + for der in der_base64 { + let bytes = decode_base64(der).map_err(|_| "invalid trust anchor")?; + roots + .add(CertificateDer::from(bytes)) + .map_err(|_| "invalid trust anchor")?; + } + } + } + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let chain = WebPkiServerVerifier::builder_with_provider(Arc::new(roots), provider.clone()) + .build() + .map_err(|_| "invalid trust anchors")?; + let pins = settings + .spki_sha256 + .iter() + .map(|pin| decode_pin(pin)) + .collect::, _>>() + .map_err(|_| "invalid SPKI pins")?; + Ok(rustls::ClientConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .map_err(|_| "TLS versions unavailable")? + .dangerous() + .with_custom_certificate_verifier(Arc::new(PinnedVerifier { chain, pins })) + .with_no_client_auth()) +} + +#[derive(Debug)] +struct PinnedVerifier { + chain: Arc, + pins: Vec<[u8; 32]>, +} + +impl ServerCertVerifier for PinnedVerifier { + fn verify_server_cert( + &self, + leaf: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + name: &ServerName<'_>, + ocsp: &[u8], + now: UnixTime, + ) -> Result { + // Pins are additional constraints: they never bypass chain, validity, + // hostname, or the handshake proof of possession. + let verified = self + .chain + .verify_server_cert(leaf, intermediates, name, ocsp, now)?; + if !self.pins.is_empty() { + let cert = x509_cert::Certificate::from_der(leaf.as_ref()).map_err(|_| { + rustls::Error::InvalidCertificate(rustls::CertificateError::BadEncoding) + })?; + let spki = cert + .tbs_certificate + .subject_public_key_info + .to_der() + .map_err(|_| { + rustls::Error::InvalidCertificate(rustls::CertificateError::BadEncoding) + })?; + let digest: [u8; 32] = Sha256::digest(spki).into(); + if !self.pins.contains(&digest) { + return Err(rustls::Error::General("SPKI pin mismatch".into())); + } + } + Ok(verified) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + signature: &DigitallySignedStruct, + ) -> Result { + self.chain.verify_tls12_signature(message, cert, signature) + } + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + signature: &DigitallySignedStruct, + ) -> Result { + self.chain.verify_tls13_signature(message, cert, signature) + } + fn supported_verify_schemes(&self) -> Vec { + self.chain.supported_verify_schemes() + } +} diff --git a/crates/providers/src/lib.rs b/crates/providers/src/lib.rs index 9fe164fd..9d32049f 100644 --- a/crates/providers/src/lib.rs +++ b/crates/providers/src/lib.rs @@ -2,10 +2,15 @@ //! hosts and the Hellas CLI. mod codex_responses; +mod http; mod openai; mod responses_fetch; mod responses_projector; mod stream; +pub use http::{ + HttpCredential, HttpCredentialConfig, HttpEgressPolicy, HttpFetchAdaptorFactory, + HttpFetchProvider, HttpProviderConfig, +}; pub use openai::OpenAiResponsesFetchProvider; pub use responses_fetch::{execute_responses_request, responses_http_client}; diff --git a/crates/providers/src/responses_fetch.rs b/crates/providers/src/responses_fetch.rs index a0d08523..fb9fe569 100644 --- a/crates/providers/src/responses_fetch.rs +++ b/crates/providers/src/responses_fetch.rs @@ -88,6 +88,7 @@ pub async fn execute_responses_request( let head = FetchProviderResponseHead { effective_model: effective_model_from_headers(upstream.headers()) .inspect_err(|_| telemetry.fail("invalid_response_headers"))?, + http: None, }; Ok(FetchProviderResponse { head, diff --git a/crates/providers/src/responses_projector.rs b/crates/providers/src/responses_projector.rs index 2c2b53ca..e531c558 100644 --- a/crates/providers/src/responses_projector.rs +++ b/crates/providers/src/responses_projector.rs @@ -35,6 +35,9 @@ impl FetchAdaptorFactory for ResponsesFetchAdaptorFactory { } fn create(&self, request: &FetchCall) -> Result { + if self.environment == FetchEnvironment::Http { + return Err(fetch_failed("HTTP requires the HTTP adaptor")); + } if self.environment == FetchEnvironment::CodexResponses { return super::codex_responses::create_session(request); } diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index ab04a824..29f3f470 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -45,12 +45,17 @@ all-protocols = ["swarm", "courtesy", "execute", "evaluate", "fetch", "chain"] # needs the consensus kernel's types — the certificate, the network id, # and the edge/terms commitments — and a wasm or fetch-only client has no # use for them. The alternative was restating those kernel types here, -# which is how two definitions of a signed digest start. -work = ["dep:hellas-kernel", "evaluate", "hellas-kernel/secp256k1"] +# which is how two definitions of a signed digest start. Both paid +# profiles are pulled forward: `evaluate` for the causal-LM profile and +# `fetch` for the sealed-fetch profile (`protocol::work_fetch`), whose +# records commit to signed fetch transcripts. +work = ["dep:hellas-kernel", "evaluate", "fetch", "hellas-kernel/secp256k1"] [dependencies] axum = { workspace = true, optional = true } +base64.workspace = true base64ct = { version = "1", optional = true, features = ["alloc"] } +url = "2" bytes = { workspace = true } ed25519-dalek = { workspace = true } futures-core = { workspace = true } diff --git a/crates/rpc/build.rs b/crates/rpc/build.rs index 63f7f6ec..f06fd336 100644 --- a/crates/rpc/build.rs +++ b/crates/rpc/build.rs @@ -776,6 +776,21 @@ fn handler_signature(m: &MethodPlan) -> TokenStream { Shape::Unary => quote! { impl Into> + Send }, _ => boxed_stream(response), }; + if fn_name == "open" { + let allow_open_context_unused = if m.connection_bound { + quote! { let _ = context; } + } else { + quote! {} + }; + return quote! { + fn #fn_name(&self, _request: #request_ty #context) + -> impl ::core::future::Future> + Send { + #allow_open_context_unused + ::core::future::ready(Err(::hellas_wire::WireStatus::new( + ::hellas_wire::WireCode::Unavailable, "confidential Open is not mounted"))) + } + }; + } quote! { fn #fn_name( &self, diff --git a/crates/rpc/src/http_fetch.rs b/crates/rpc/src/http_fetch.rs new file mode 100644 index 00000000..938f5e35 --- /dev/null +++ b/crates/rpc/src/http_fetch.rs @@ -0,0 +1,382 @@ +//! Caller-signed HTTPS request vocabulary. Account secrets are selected by +//! alias and never carried in a client request. + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use serde::{Deserialize, Serialize}; + +pub const MAX_HTTP_RESPONSE_BYTES: u32 = 512 * 1024; +pub const MAX_HTTP_HEADERS_BYTES: usize = 16 * 1024; +pub const MAX_HTTP_HEADERS: usize = 64; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpFetchRequest { + pub url: String, + pub method: String, + #[serde(default)] + pub headers: Vec<(String, String)>, + #[serde(default)] + pub body_base64: String, + pub tls: HttpTls, + #[serde(default)] + pub credential: Option, + pub max_response_bytes: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpTls { + pub roots: HttpTrustRoots, + /// Additional SHA-256 pins over the leaf certificate's DER SPKI. + #[serde(default)] + pub spki_sha256: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)] +pub enum HttpTrustRoots { + WebPki, + /// Exact base64 DER trust anchors, without implicit system roots. + Certificates { + der_base64: Vec, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid HTTPS Fetch request: {0}")] +pub struct HttpRequestError(pub &'static str); + +impl HttpFetchRequest { + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > crate::fetch::MAX_FETCH_REQUEST_BODY_BYTES { + return Err(HttpRequestError("request exceeds byte limit")); + } + let request: Self = + serde_json::from_slice(bytes).map_err(|_| HttpRequestError("request schema"))?; + request.validate()?; + Ok(request) + } + + pub fn validate(&self) -> Result<(), HttpRequestError> { + self.parsed_url()?; + if !matches!( + self.method.as_str(), + "GET" | "HEAD" | "POST" | "PUT" | "PATCH" | "DELETE" | "OPTIONS" + ) { + return Err(HttpRequestError("HTTP method")); + } + check_headers(&self.headers, true)?; + if self.max_response_bytes == 0 || self.max_response_bytes > MAX_HTTP_RESPONSE_BYTES { + return Err(HttpRequestError("response byte limit")); + } + self.body()?; + if self.credential.as_ref().is_some_and(|alias| { + alias.is_empty() + || alias.len() > 128 + || !alias + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b)) + }) { + return Err(HttpRequestError("credential alias")); + } + self.tls.validate() + } + + pub fn parsed_url(&self) -> Result { + if self.url.len() > 8192 { + return Err(HttpRequestError("URL length")); + } + let url = url::Url::parse(&self.url).map_err(|_| HttpRequestError("URL"))?; + if url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.fragment().is_some() + { + return Err(HttpRequestError( + "URL must be HTTPS without userinfo or fragment", + )); + } + Ok(url) + } + + pub fn body(&self) -> Result, HttpRequestError> { + if self.body_base64.len() > crate::fetch::MAX_FETCH_REQUEST_BODY_BYTES { + return Err(HttpRequestError("body length")); + } + decode_base64(&self.body_base64) + } +} + +impl HttpTls { + pub fn validate(&self) -> Result<(), HttpRequestError> { + if let HttpTrustRoots::Certificates { der_base64 } = &self.roots { + if der_base64.is_empty() || der_base64.len() > 16 { + return Err(HttpRequestError("trust anchor count")); + } + for der in der_base64 { + if der.is_empty() || der.len() > 32768 { + return Err(HttpRequestError("trust anchor size")); + } + decode_base64(der)?; + } + } + if self.spki_sha256.len() > 16 { + return Err(HttpRequestError("SPKI pin count")); + } + for pin in &self.spki_sha256 { + decode_pin(pin)?; + } + Ok(()) + } +} + +pub fn decode_pin(pin: &str) -> Result<[u8; 32], HttpRequestError> { + if pin.len() != 64 { + return Err(HttpRequestError("SPKI pin must be 64 lowercase hex digits")); + } + let mut out = [0; 32]; + for (i, pair) in pin.as_bytes().chunks_exact(2).enumerate() { + let digit = |b: u8| match b { + b'0'..=b'9' => Ok(b - b'0'), + b'a'..=b'f' => Ok(b - b'a' + 10), + _ => Err(HttpRequestError("SPKI pin encoding")), + }; + out[i] = digit(pair[0])? * 16 + digit(pair[1])?; + } + Ok(out) +} + +pub fn decode_base64(encoded: &str) -> Result, HttpRequestError> { + let bytes = STANDARD + .decode(encoded) + .map_err(|_| HttpRequestError("base64 encoding"))?; + if STANDARD.encode(&bytes) != encoded { + return Err(HttpRequestError("noncanonical base64")); + } + Ok(bytes) +} + +pub fn check_headers(headers: &[(String, String)], request: bool) -> Result<(), HttpRequestError> { + if headers.len() > MAX_HTTP_HEADERS + || headers + .iter() + .map(|(k, v)| k.len().saturating_add(v.len())) + .sum::() + > MAX_HTTP_HEADERS_BYTES + { + return Err(HttpRequestError("header limit")); + } + for (name, value) in headers { + if name.is_empty() + || !name.bytes().all(|b| { + b.is_ascii_lowercase() + || b.is_ascii_digit() + || b == 0x60 + || b"!#$%&'*+-.^_|~".contains(&b) + }) + || !value + .bytes() + .all(|b| b == b'\t' || (0x20..=0x7e).contains(&b)) + { + return Err(HttpRequestError("header encoding")); + } + if request + && matches!( + name.as_str(), + "host" + | "content-length" + | "transfer-encoding" + | "connection" + | "upgrade" + | "proxy-authorization" + | "proxy-connection" + | "te" + | "trailer" + ) + { + return Err(HttpRequestError("reserved request header")); + } + } + Ok(()) +} + +/// A complete, structurally checked response reconstructed from authenticated +/// Fetch output. Callers verify signatures and the expected producer first. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HttpFetchResponse { + pub status: u16, + pub headers: Vec<(String, String)>, + pub body: Vec, +} + +impl HttpFetchResponse { + pub fn from_output( + request: &HttpFetchRequest, + output: &crate::fetch::FetchOutput, + ) -> Result { + use crate::output::{AdaptorEvent, HttpResponseEvent, OutputEvent, StopReason}; + let (events, terminal) = output.output_event_payloads(); + let mut response = None; + for payload in events { + let event = crate::fetch::decode_fetch_event_payload(payload) + .map_err(|_| HttpRequestError("response event codec"))?; + match event { + OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Head { + status, + headers, + })) if response.is_none() => { + if !(100..=599).contains(&status) { + return Err(HttpRequestError("response status")); + } + check_headers(&headers, false)?; + response = Some(Self { + status, + headers, + body: Vec::new(), + }); + } + OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Body { base64 })) => { + let response = response + .as_mut() + .ok_or(HttpRequestError("response body before head"))?; + let bytes = decode_base64(&base64)?; + if bytes.is_empty() + || response.body.len().saturating_add(bytes.len()) + > request.max_response_bytes as usize + { + return Err(HttpRequestError("response body size")); + } + response.body.extend_from_slice(&bytes); + } + _ => return Err(HttpRequestError("unexpected or duplicate response event")), + } + } + if !matches!( + crate::fetch::decode_fetch_terminal_payload(terminal), + Ok(crate::fetch::FetchTerminalPayload::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + .. + }) + ) { + return Err(HttpRequestError("HTTP terminal")); + } + response.ok_or(HttpRequestError("missing response head")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fetch::{ + FetchOutputTranscriptBuilder, encode_fetch_event_payload, encode_fetch_terminal_payload, + verify_output_events, + }; + use crate::output::{AdaptorEvent, HttpResponseEvent, OutputEvent, StopReason}; + use crate::{Assurance, Digest, InputCommitment, ProducerSigningKey}; + + fn request() -> HttpFetchRequest { + HttpFetchRequest { + url: "https://example.com/v1/messages".into(), + method: "POST".into(), + headers: vec![], + body_base64: "e30=".into(), + tls: HttpTls { + roots: HttpTrustRoots::WebPki, + spki_sha256: vec![], + }, + credential: None, + max_response_bytes: 3, + } + } + fn output(events: Vec) -> crate::fetch::FetchOutput { + let key = ProducerSigningKey::from_secret_bytes([1; 32]).unwrap(); + let input = InputCommitment::from_digest(Digest::from_bytes([2; 32])); + let mut builder = FetchOutputTranscriptBuilder::new(input, Assurance::ProducerSigned, &key); + for event in events { + builder + .push_event( + encode_fetch_event_payload(&OutputEvent::Adaptor(AdaptorEvent::Http(event))) + .unwrap(), + ) + .unwrap(); + } + let events = builder + .finish( + encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .unwrap(), + ) + .unwrap(); + verify_output_events(input, Assurance::ProducerSigned, &events).unwrap() + } + fn head() -> HttpResponseEvent { + HttpResponseEvent::Head { + status: 200, + headers: vec![], + } + } + fn body(bytes: &[u8]) -> HttpResponseEvent { + HttpResponseEvent::Body { + base64: STANDARD.encode(bytes), + } + } + #[test] + fn even_signed_http_responses_must_obey_structure_and_request_limit() { + let good = output(vec![head(), body(&[0, 255, 3])]); + assert_eq!( + HttpFetchResponse::from_output(&request(), &good) + .unwrap() + .body, + vec![0, 255, 3] + ); + for events in [ + vec![], + vec![body(b"a"), head()], + vec![head(), head()], + vec![head(), body(b"abcd")], + vec![ + head(), + HttpResponseEvent::Body { + base64: "!!".into(), + }, + ], + ] { + assert!(HttpFetchResponse::from_output(&request(), &output(events)).is_err()); + } + } + #[test] + fn requests_reject_ambiguous_or_unbounded_transport_inputs() { + for url in [ + "http://example.com/", + "https://user:secret@example.com/", + "https://example.com/#fragment", + ] { + let mut input = request(); + input.url = url.into(); + assert!(input.validate().is_err()); + } + for name in [ + "host", + "content-length", + "transfer-encoding", + "proxy-authorization", + ] { + let mut input = request(); + input.headers = vec![(name.into(), "x".into())]; + assert!(input.validate().is_err()); + } + let mut input = request(); + input.headers = vec![("authorization".into(), "x\r\ny".into())]; + assert!(input.validate().is_err()); + input = request(); + input.max_response_bytes = MAX_HTTP_RESPONSE_BYTES + 1; + assert!(input.validate().is_err()); + input = request(); + input.body_base64 = "e30".into(); + assert!(input.validate().is_err()); + } +} diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index 1a10184b..76638aed 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -47,6 +47,8 @@ pub mod edge_index; pub mod evaluate; #[cfg(feature = "fetch")] pub mod fetch; +#[cfg(feature = "fetch")] +pub mod http_fetch; pub mod observe; #[cfg(feature = "execute")] pub mod open; diff --git a/crates/rpc/src/output.rs b/crates/rpc/src/output.rs index 3e5610a9..0f024ed4 100644 --- a/crates/rpc/src/output.rs +++ b/crates/rpc/src/output.rs @@ -52,6 +52,18 @@ impl OutputEvent { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum AdaptorEvent { CodexResponses(CodexResponsesEvent), + /// Exact HTTP metadata and bytes, without LLM-specific projection. + Http(HttpResponseEvent), +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum HttpResponseEvent { + Head { + status: u16, + headers: Vec<(String, String)>, + }, + /// Standard padded base64, preserving arbitrary binary bodies. + Body { base64: String }, } /// The subset of Codex Responses streaming semantics committed by the sealed diff --git a/crates/rpc/src/pb/mod.rs b/crates/rpc/src/pb/mod.rs index 4bf86163..6f692c77 100644 --- a/crates/rpc/src/pb/mod.rs +++ b/crates/rpc/src/pb/mod.rs @@ -188,9 +188,10 @@ mod id_pins { #[test] fn work_ids_are_stable() { use super::services::work::{ - AcceptWork, AdmitCertificate, DeliverResult, StreamResult, Work, + AcceptWork, AdmitCertificate, DeliverResult, Open, StreamResult, Work, }; - assert_eq!(::SERVICE_ID, 0x1a6d_285a); + assert_eq!(::SERVICE_ID, 0x3b672fa8); + assert_eq!(::METHOD_ID, 0x93cb0b39); assert_eq!(::METHOD_ID, 0xe6a7_13c2); assert_eq!(::METHOD_ID, 0xf15a_a80e); assert_eq!(::METHOD_ID, 0xf9e8_388a); @@ -207,8 +208,9 @@ mod id_pins { #[cfg(feature = "work")] #[test] fn work_setup_ids_are_stable() { - use super::services::work_setup::{ExchangeSetup, WorkSetup}; - assert_eq!(::SERVICE_ID, 0x0235_ddf0); + use super::services::work_setup::{ExchangeSetup, Open, WorkSetup}; + assert_eq!(::SERVICE_ID, 0x3ed2cd2f); + assert_eq!(::METHOD_ID, 0xcbe4ebd5); assert_eq!(::METHOD_ID, 0x1cde_46e8); } diff --git a/crates/rpc/src/protocol/artifacts.rs b/crates/rpc/src/protocol/artifacts.rs index d249e4e4..6115df58 100644 --- a/crates/rpc/src/protocol/artifacts.rs +++ b/crates/rpc/src/protocol/artifacts.rs @@ -786,11 +786,7 @@ impl PreparedPaidInputV1 { bytes.len() ))); } - let mut reader = BundleReader { - bytes, - offset: 0, - budget, - }; + let mut reader = BundleReader::new(bytes, budget); let bundle = Self { evaluate_request: reader.body("evaluate_request")?, manifest: reader.body("manifest")?, @@ -834,14 +830,39 @@ impl PreparedPaidInputV1 { } } -struct BundleReader<'a> { +/// Cursor over a length-prefixed bundle's bodies. +/// +/// Shared with the fetch profile's two-body bundle +/// ([`super::work_fetch::PreparedPaidFetchInputV1`]) so the two rules that +/// keep a declared length honest — never past the input, never past the +/// budget — exist once. The error messages name "prepared input"; the +/// fetch bundle is one. +pub(crate) struct BundleReader<'a> { bytes: &'a [u8], offset: usize, budget: usize, } -impl BundleReader<'_> { - fn body(&mut self, field: &'static str) -> Result, CanonicalDecodeError> { +impl<'a> BundleReader<'a> { + pub(crate) const fn new(bytes: &'a [u8], budget: usize) -> Self { + Self { + bytes, + offset: 0, + budget, + } + } + + /// Bytes consumed so far; the caller compares it against the input + /// length to refuse trailing bytes. + /// + /// Used by the fetch profile's bundle; the evaluate bundle reads the + /// field directly, one module over. + #[cfg(feature = "work")] + pub(crate) const fn offset(&self) -> usize { + self.offset + } + + pub(crate) fn body(&mut self, field: &'static str) -> Result, CanonicalDecodeError> { let start = self .offset .checked_add(LENGTH_PREFIX) diff --git a/crates/rpc/src/protocol/fetch_environment.rs b/crates/rpc/src/protocol/fetch_environment.rs index d3f1494f..be4e213a 100644 --- a/crates/rpc/src/protocol/fetch_environment.rs +++ b/crates/rpc/src/protocol/fetch_environment.rs @@ -11,8 +11,9 @@ //! upstream credentials, and access policy remain provider-local and are //! deliberately absent from this root. //! Selecting the upstream destination is trusted execution semantics, so every -//! built-in root commits an exact HTTPS endpoint and driver contract. The -//! service at that endpoint and every response claim remain adversarial input. +//! built-in root commits a driver contract. Sealed roots fix the endpoint; the +//! HTTP root interprets the URL and TLS policy in the caller-signed request. +//! The upstream service and every response claim remain adversarial input. use crate::{Application, ContentId, DagCborEncoder, ProgramManifest}; @@ -41,11 +42,13 @@ const FETCH_ENVIRONMENT_DOMAIN: &str = "hellas.fetch.environment.v1"; /// An exact built-in request/response transformation and its trusted config. /// -/// The first version has no configurable trusted inputs: its complete -/// behaviour is compiled into the attested application. Adding trusted config -/// later requires a new canonical variant (and therefore a new manifest ID). +/// Each variant fixes the interpreter compiled into the attested application. +/// The HTTP interpreter binds request-selected destinations and TLS settings +/// through the signed input; it does not enumerate upstream vendors. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FetchEnvironment { + /// HTTPS to the signed URL with selected trust roots and optional SPKI pins. + Http, /// Build an official Codex Responses request, send it only to the fixed /// ChatGPT Codex endpoint, and project its SSE response. CodexResponses, @@ -61,6 +64,7 @@ impl FetchEnvironment { #[must_use] pub const fn adaptor(self) -> &'static str { match self { + Self::Http => "http-0.0.1", Self::CodexResponses => CODEX_RESPONSES_ADAPTOR, Self::OpenAiResponses => OPENAI_RESPONSES_ADAPTOR, } @@ -70,6 +74,7 @@ impl FetchEnvironment { #[must_use] pub const fn driver(self) -> &'static str { match self { + Self::Http => "https-request-roots-and-spki-no-redirect-0.0.1", Self::CodexResponses => CODEX_RESPONSES_DRIVER, Self::OpenAiResponses => OPENAI_RESPONSES_DRIVER, } @@ -79,6 +84,7 @@ impl FetchEnvironment { #[must_use] pub const fn endpoint(self) -> &'static str { match self { + Self::Http => "caller-signed:https-url", Self::CodexResponses => CODEX_RESPONSES_ENDPOINT, Self::OpenAiResponses => OPENAI_RESPONSES_ENDPOINT, } @@ -111,6 +117,11 @@ impl FetchEnvironment { decoder.finish()?; let environment = match (adaptor, driver, endpoint) { + ( + "http-0.0.1", + "https-request-roots-and-spki-no-redirect-0.0.1", + "caller-signed:https-url", + ) => Self::Http, (CODEX_RESPONSES_ADAPTOR, CODEX_RESPONSES_DRIVER, CODEX_RESPONSES_ENDPOINT) => { Self::CodexResponses } diff --git a/crates/rpc/src/protocol/mod.rs b/crates/rpc/src/protocol/mod.rs index 6b3b0d9b..51adeea7 100644 --- a/crates/rpc/src/protocol/mod.rs +++ b/crates/rpc/src/protocol/mod.rs @@ -20,6 +20,10 @@ pub mod work; #[cfg(feature = "work")] pub mod work_bundle; #[cfg(feature = "work")] +pub mod work_fetch; +#[cfg(feature = "work")] +pub mod work_profile; +#[cfg(feature = "work")] pub mod work_setup; pub use causal_lm::{ diff --git a/crates/rpc/src/protocol/work.rs b/crates/rpc/src/protocol/work.rs index e0a254e1..1a72faf0 100644 --- a/crates/rpc/src/protocol/work.rs +++ b/crates/rpc/src/protocol/work.rs @@ -72,13 +72,16 @@ const FORMAT_VERSION: u8 = 1; /// /// These are RPC-protocol tags. They are not kernel canonical tags and /// they are never accepted L1 bytes; the numbers are local to this -/// module and shared only between the two endpoints. -mod tag { - pub(super) const PAID_CHANNEL_POLICY: u8 = 0; - pub(super) const PAID_EXECUTION_POLICY: u8 = 1; - pub(super) const PAID_JOB_AUTHORIZATION: u8 = 2; - pub(super) const PAID_JOB_RESULT: u8 = 3; - pub(super) const PAYMENT_BINDING: u8 = 4; +/// module and shared only between the two endpoints. The fetch profile's +/// policy record ([`super::work_fetch`]) takes its tag from this same +/// registry, so two profiles can never assign one number. +pub(crate) mod tag { + pub(crate) const PAID_CHANNEL_POLICY: u8 = 0; + pub(crate) const PAID_EXECUTION_POLICY: u8 = 1; + pub(crate) const PAID_JOB_AUTHORIZATION: u8 = 2; + pub(crate) const PAID_JOB_RESULT: u8 = 3; + pub(crate) const PAYMENT_BINDING: u8 = 4; + pub(crate) const PAID_FETCH_POLICY: u8 = 5; } /// Bytes the envelope occupies: `format_version:u8 || record_tag:u8`. @@ -564,11 +567,11 @@ impl PrivateRecord for PaymentBindingV1 { // ── Body reading and writing ────────────────────────────────────────── -fn put_u32(out: &mut Vec, value: u32) { +pub(crate) fn put_u32(out: &mut Vec, value: u32) { out.extend_from_slice(&value.to_be_bytes()); } -fn put_u64(out: &mut Vec, value: u64) { +pub(crate) fn put_u64(out: &mut Vec, value: u64) { out.extend_from_slice(&value.to_be_bytes()); } @@ -597,7 +600,7 @@ impl BodyReader<'_> { Ok(out) } - fn bytes32(&mut self) -> Result<[u8; 32], PaidWorkError> { + pub(crate) fn bytes32(&mut self) -> Result<[u8; 32], PaidWorkError> { self.take::<32>() } @@ -605,11 +608,11 @@ impl BodyReader<'_> { self.take::<2>().map(u16::from_be_bytes) } - fn u32(&mut self) -> Result { + pub(crate) fn u32(&mut self) -> Result { self.take::<4>().map(u32::from_be_bytes) } - fn u64(&mut self) -> Result { + pub(crate) fn u64(&mut self) -> Result { self.take::<8>().map(u64::from_be_bytes) } } @@ -623,13 +626,13 @@ impl BodyReader<'_> { /// digests are the same bytes inside a kernel authorization hash. The /// length prefix is why a two-character id followed by a channel cannot /// hash like a one-character id followed by a different one. -struct EncodedNetwork { +pub(crate) struct EncodedNetwork { bytes: [u8; ::MAX_ENCODED_SIZE], len: usize, } impl EncodedNetwork { - fn new(network: NetworkId) -> Self { + pub(crate) fn new(network: NetworkId) -> Self { let mut bytes = [0_u8; ::MAX_ENCODED_SIZE]; let mut writer = BufferWriter::new(&mut bytes); network.encode_to(&mut writer); @@ -646,7 +649,7 @@ impl EncodedNetwork { /// zero-padded buffer — and every digest in this module would move /// silently under it. A bug that cannot happen should not have a /// second answer ready. - fn as_slice(&self) -> &[u8] { + pub(crate) fn as_slice(&self) -> &[u8] { &self.bytes[..self.len] } } @@ -657,7 +660,7 @@ impl EncodedNetwork { /// Every caller's complete preimage is bounded at compile time by an /// assertion below, because [`SingleChunkHasher::update`] panics rather /// than erroring once a preimage reaches [`MIN_CHUNK_SIZE`]. -fn xh(domain: &[u8], fields: &[&[u8]]) -> Digest { +pub(crate) fn xh(domain: &[u8], fields: &[&[u8]]) -> Digest { let mut hasher = SingleChunkHasher::new(); hasher.update(domain); for field in fields { @@ -672,7 +675,7 @@ fn xh(domain: &[u8], fields: &[&[u8]]) -> Digest { /// one-shot [`Digest::hash`] of the same concatenation under every write /// segmentation, which is what lets a caller stream a large body without /// holding it. -fn xfh(domain: &[u8], fields: &[&[u8]]) -> Digest { +pub(crate) fn xfh(domain: &[u8], fields: &[&[u8]]) -> Digest { let mut hasher = XetFileHasher::new(); hasher.update(domain); for field in fields { @@ -844,7 +847,7 @@ impl PaidChannel { self.payment_terms.parties().taker() } - fn network_bytes(&self) -> EncodedNetwork { + pub(crate) fn network_bytes(&self) -> EncodedNetwork { EncodedNetwork::new(self.network) } } @@ -941,7 +944,7 @@ pub fn identity_source_digest( /// separates nothing. A body that large is unreachable through any /// bounded decoder here; it is refused rather than truncated because a /// truncated prefix is a second legal spelling of the same bytes. -fn length_prefix(bytes: &[u8], field: &'static str) -> Result<[u8; 4], PaidWorkError> { +pub(crate) fn length_prefix(bytes: &[u8], field: &'static str) -> Result<[u8; 4], PaidWorkError> { let len = u32::try_from(bytes.len()).map_err(|_| PaidWorkError::Overflow { field })?; Ok(len.to_be_bytes()) } @@ -1252,7 +1255,34 @@ pub fn check_authorization( finalized_height: u64, ) -> Result { check_execution_policy(policy)?; + check_authorization_core( + channel, + authorization, + execution_policy_digest(channel, policy), + policy.allowed_environment, + policy.fixed_price, + finalized_height, + ) +} +/// The authorization arithmetic every paid-work profile shares. +/// +/// The profiles differ in what a policy *is* — the evaluate profile's is +/// [`PaidExecutionPolicyV1`], the fetch profile's is in +/// [`super::work_fetch`] — but not in what signing one *means*: the same +/// channel fields, the same bond cover, the same credit-limit cover, the +/// same deadline window. That part is written once here so the two +/// profiles cannot drift into two different definitions of acceptance. +/// What each caller supplies is its own policy digest, environment, and +/// price; what this function supplies is everything else. +pub(crate) fn check_authorization_core( + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + policy_digest: Digest, + allowed_environment: ContentId, + fixed_price: u64, + finalized_height: u64, +) -> Result { let terms = channel.payment_terms(); let expected = [ ( @@ -1277,13 +1307,11 @@ pub fn check_authorization( ), ( "execution_policy_digest", - authorization.execution_policy_digest.as_bytes() - == execution_policy_digest(channel, policy).as_bytes(), + authorization.execution_policy_digest.as_bytes() == policy_digest.as_bytes(), ), ( "environment_commitment", - authorization.environment_commitment.as_bytes() - == policy.allowed_environment.as_bytes(), + authorization.environment_commitment.as_bytes() == allowed_environment.as_bytes(), ), ]; for (field, holds) in expected { @@ -1299,11 +1327,11 @@ pub fn check_authorization( max_job_price, }); } - if authorization.price != policy.fixed_price { + if authorization.price != fixed_price { return Err(PaidWorkError::Mismatch { field: "price" }); } - // The execution policy is per-authorization and only its digest is + // The policy is per-authorization and only its digest is // signed, so a channel may see many of them. The credit limits are // per-channel and are opened once, at construction. That is why the // two are compared here, against this job's price, rather than once diff --git a/crates/rpc/src/protocol/work_fetch.rs b/crates/rpc/src/protocol/work_fetch.rs new file mode 100644 index 00000000..34d47fcc --- /dev/null +++ b/crates/rpc/src/protocol/work_fetch.rs @@ -0,0 +1,993 @@ +//! The paid-fetch profile: private records for paid jobs whose execution +//! is a Fetch request rather than a local evaluation. +//! +//! The shared machinery in [`super::work`] is profile-agnostic: the +//! authorization binds a policy digest, a prepared-input digest, a request +//! commitment, and an environment commitment, and the result and payment +//! records derive from it. What a profile owns is the meaning of those +//! four commitments. For the evaluate profile the policy is +//! [`super::work::PaidExecutionPolicyV1`] and the prepared input is a +//! six-body artifact bundle; for this profile the policy is +//! [`PaidFetchPolicyV1`] and the prepared input is the client's signed +//! fetch input transcript together with the environment manifest it runs +//! in. +//! +//! A paid fetch job buys one provider-signed HTTP transformation — one +//! [`crate::fetch`] call — rather than local model evaluation. The client +//! signs the request as a fetch input transcript; the provider answers +//! with a signed fetch output transcript; this module is where those +//! transcripts meet the channel's money. The record tag and the digest +//! domains below are what separate the two profiles on the wire: a tag-5 +//! policy is a fetch policy under every decoder, and a digest computed +//! here cannot be reproduced by the evaluate profile's domains. + +use hellas_kernel::{Encode, NetworkId}; +use hellas_xet::{MIN_CHUNK_SIZE, XetFileHasher}; + +use crate::fetch::{FetchProtocolError, MAX_FETCH_ROUTE_COMPONENT_BYTES}; +use crate::protocol::artifacts::BundleReader; +use crate::protocol::value::{ + CanonicalDecodeError, CanonicalDecoder, canonical_dag_cbor, decode_canonical_dag_cbor, +}; +use crate::protocol::work::{ + BodyReader, EncodedNetwork, JobDeadlines, PaidChannel, PaidJobAuthorizationV1, PaidJobResultV1, + PaidWorkError, PrivateRecord, check_authorization_core, length_prefix, put_u32, put_u64, tag, + work_id, xfh, xh, +}; +use crate::{ + Assurance, ContentId, Digest, InputCommitment, InputEventEnvelope, OutputEventEnvelope, + ProgramManifest, PublicKey, RequestCommitment, +}; + +// ── Domains ─────────────────────────────────────────────────────────── +// +// Same discipline as `super::work`: every digest here is the Xet hash of +// one of these byte strings followed by canonical fields, the strings are +// written once, and changing one changes every digest computed under it. + +/// Commitment to the canonical route-policy body. +const FETCH_ROUTE: &[u8] = b"hellas.work.fetch-route.v1"; +/// Commitment to the per-channel fetch execution policy. +const FETCH_POLICY: &[u8] = b"hellas.work.fetch-policy.v1"; +/// Commitment to the prepared fetch input bundle. +const PREPARED_FETCH_INPUT: &[u8] = b"hellas.work.prepared-fetch-input.v1"; +/// The normalized fetch answer authenticated by the signed transcript. +const FETCH_OUTPUT: &[u8] = b"hellas.work.fetch-output.v1"; + +// ── The route policy ────────────────────────────────────────────────── + +/// Schema tag of the sealed-route body. +const SEALED_ROUTE_SCHEMA: &str = "hellas.work.fetch-route.sealed.v1"; +/// Schema tag of the open-fetch body. +const OPEN_FETCH_SCHEMA: &str = "hellas.work.fetch-route.open.v1"; + +/// The route a fetch channel sells, committed to by digest. +/// +/// A [`PaidFetchPolicyV1`] cannot carry this body — it is variable-length, +/// and the policy record is fixed-width — so the policy carries +/// [`fetch_route_commitment`] of its canonical bytes instead, and +/// [`check_prepared_fetch_input`] opens the commitment before comparing +/// anything against it. That is the same arrangement the evaluate profile +/// uses for its generation policy: a variable-length canonical body, a +/// fixed digest in the record. +/// +/// The body is canonical DAG-CBOR with one schema tag per variant, so a +/// route has exactly one byte spelling and the commitment cannot be +/// opened to a different route than the one that was hashed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FetchRoutePolicy { + /// The one `(service, method)` pair this channel sells. + /// + /// These are the route labels a sealed fetch input transcript signs; + /// the paid profile refuses a job whose signed labels differ, so the + /// pair here is the whole of the channel's routing vocabulary. + SealedRoute { + /// The signed `service` component, e.g. `openai`. + service: String, + /// The signed `method` component, e.g. `responses`. + method: String, + }, + /// Caller-signed HTTPS URLs under the generic HTTP manifest. Host and + /// required-pin constraints are checked before paid acceptance; the driver + /// enforces normal certificate validation, DNS/address checks and no redirects. + /// An empty host list permits any public host under the operator's egress policy. + OpenFetch { + /// Whether the driver must pin the upstream's SPKI. + require_spki_pin: bool, + /// Hosts the driver may egress to; empty means any public host. + allowed_hosts: Vec, + }, +} + +impl FetchRoutePolicy { + /// Builds the sealed route, applying the same component rules the + /// fetch transcript scheme applies to the signed `service` and + /// `method` events: neither empty, neither over + /// [`MAX_FETCH_ROUTE_COMPONENT_BYTES`]. The two rules are shared with + /// `crate::fetch` rather than restated, so a route this profile + /// accepts is a route the transcript scheme accepts. + /// + /// # Errors + /// + /// [`FetchProtocolError::EmptyService`], [`FetchProtocolError::EmptyMethod`], + /// or [`FetchProtocolError::RouteComponentLimit`] — the fetch + /// protocol's own route errors, because they are the fetch protocol's + /// own rules. + pub fn sealed_route( + service: impl Into, + method: impl Into, + ) -> Result { + let service = service.into(); + let method = method.into(); + if service.is_empty() { + return Err(FetchProtocolError::EmptyService); + } + if method.is_empty() { + return Err(FetchProtocolError::EmptyMethod); + } + if service.len() > MAX_FETCH_ROUTE_COMPONENT_BYTES { + return Err(FetchProtocolError::RouteComponentLimit { + field: "service", + actual: service.len(), + }); + } + if method.len() > MAX_FETCH_ROUTE_COMPONENT_BYTES { + return Err(FetchProtocolError::RouteComponentLimit { + field: "method", + actual: method.len(), + }); + } + Ok(Self::SealedRoute { service, method }) + } + + /// Builds the open-fetch vocabulary. + /// + /// The host list is sorted and deduplicated: an allowlist is a set, + /// and a set with two spellings would be two commitments to one + /// policy. Admission matches exact canonical URL host names and checks + /// required pins; the HTTPS interpreter performs certificate and address + /// checks when connecting. + #[must_use] + pub fn open_fetch(require_spki_pin: bool, allowed_hosts: impl Into>) -> Self { + let mut allowed_hosts = allowed_hosts.into(); + allowed_hosts.sort_unstable(); + allowed_hosts.dedup(); + Self::OpenFetch { + require_spki_pin, + allowed_hosts, + } + } + + /// Returns the canonical body bytes the policy commits to. + /// + /// The one boolean encodes as the integer `0` or `1`: the canonical + /// encoder has no boolean primitive, and a definite two-value integer + /// is the smallest encoding that cannot drift. + #[must_use] + pub fn canonical_body_bytes(&self) -> Vec { + let mut encoder = crate::DagCborEncoder::new(); + match self { + Self::SealedRoute { service, method } => { + encoder.array(3); + encoder.str(SEALED_ROUTE_SCHEMA); + encoder.str(service); + encoder.str(method); + } + Self::OpenFetch { + require_spki_pin, + allowed_hosts, + } => { + encoder.array(3); + encoder.str(OPEN_FETCH_SCHEMA); + encoder.u64(u64::from(*require_spki_pin)); + encoder.array(allowed_hosts.len() as u64); + for host in allowed_hosts { + encoder.str(host); + } + } + } + encoder.into_bytes() + } + + /// Decodes one canonical route-policy body, strictly. + /// + /// Strict the way a committed body has to be: an unknown schema, a + /// wrong array length, a pin that is not `0` or `1`, a trailing byte, + /// and a noncanonical integer are all refused, and the value is + /// re-encoded and compared so the bytes accepted are the bytes + /// [`Self::canonical_body_bytes`] would have produced. The sealed + /// variant is rebuilt through [`Self::sealed_route`], so a body that + /// spells a route the transcript scheme would refuse is refused here + /// too. + pub fn from_canonical_body_bytes(bytes: &[u8]) -> Result { + let mut decoder = CanonicalDecoder::new(bytes); + let len = decoder.array_len()?; + let policy = match decoder.str()? { + SEALED_ROUTE_SCHEMA => { + if len != 3 { + return Err(CanonicalDecodeError::new(format!( + "{SEALED_ROUTE_SCHEMA} expected array length 3, got {len}" + ))); + } + let service = decoder.str()?.to_string(); + let method = decoder.str()?.to_string(); + Self::sealed_route(service, method) + .map_err(|error| CanonicalDecodeError::new(error.to_string()))? + } + OPEN_FETCH_SCHEMA => { + if len != 3 { + return Err(CanonicalDecodeError::new(format!( + "{OPEN_FETCH_SCHEMA} expected array length 3, got {len}" + ))); + } + let require_spki_pin = match decoder.u64()? { + 0 => false, + 1 => true, + other => { + return Err(CanonicalDecodeError::new(format!( + "require_spki_pin must be 0 or 1, got {other}" + ))); + } + }; + let hosts = decoder.array_len()?; + let mut allowed_hosts = Vec::with_capacity(hosts); + for _ in 0..hosts { + allowed_hosts.push(decoder.str()?.to_string()); + } + Self::open_fetch(require_spki_pin, allowed_hosts) + } + other => { + return Err(CanonicalDecodeError::new(format!( + "unexpected fetch route schema tag {other:?}" + ))); + } + }; + decoder.finish()?; + if policy.canonical_body_bytes() != bytes { + return Err(CanonicalDecodeError::new( + "fetch route policy is not in canonical DAG-CBOR form", + )); + } + Ok(policy) + } +} + +/// Returns the commitment a [`PaidFetchPolicyV1`] names as its route. +/// +/// Length-prefixed and streamed, mirroring +/// [`super::work::generation_policy_digest`]: the route body is +/// variable-length, so this digest must not use the single-chunk hasher. +pub fn fetch_route_commitment(canonical_body_bytes: &[u8]) -> Result { + Ok(xfh( + FETCH_ROUTE, + &[ + &length_prefix(canonical_body_bytes, "fetch route policy length")?, + canonical_body_bytes, + ], + )) +} + +// ── The fetch execution policy ──────────────────────────────────────── + +/// Everything about *how* a paid fetch job may execute, fixed before it +/// does: the fetch profile's tag-5 analogue of +/// [`super::work::PaidExecutionPolicyV1`]. +/// +/// One signed body carries the environment, the route commitment, the +/// resource envelope, the timing margins, and the price. Where the +/// evaluate policy commits to a generation-policy body and an identity +/// artifact, this one commits to a [`FetchRoutePolicy`] body: what the +/// channel may *reach* replaces what the channel may *generate*, because +/// a fetch job's variable choices are its upstream route, not its +/// sampling parameters. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PaidFetchPolicyV1 { + /// The one fetch environment manifest this channel will run. + pub allowed_environment: ContentId, + /// Commitment to the canonical route-policy body. + pub route_commitment: Digest, + /// Largest signed `request.body` this channel accepts, in bytes. + pub max_request_body_bytes: u32, + /// Largest output transcript this channel accepts, in events. + pub max_output_events: u32, + /// Largest cumulative output payload this channel accepts, in bytes. + pub max_output_bytes: u32, + /// Largest spool the provider may retain for one job. + pub max_spool_bytes: u64, + /// Largest complete encoded result frame, transport framing + /// included. + pub max_encoded_result_frame: u32, + /// Largest complete encoded prepared-input bundle. + /// + /// The evaluate profile bounds its bundle with the quote-response + /// limit because the bundle is the quote's payload; this profile + /// names the bundle's own limit instead, so the two numbers a fetch + /// channel wants cannot force each other. + pub max_encoded_prepared_input: u32, + /// Blocks allowed from acceptance to durable terminal readiness. + pub dispatch_margin_blocks: u64, + /// Blocks allowed to transfer the largest legal result. + pub delivery_margin_blocks: u64, + /// Blocks allowed for reexecution, invoicing, and admission. + pub oracle_grace_blocks: u64, + /// Price of one accepted terminal result. + pub fixed_price: u64, +} + +impl PrivateRecord for PaidFetchPolicyV1 { + const TAG: u8 = tag::PAID_FETCH_POLICY; + const BODY_SIZE: usize = 2 * 32 + 3 * 4 + 8 + 2 * 4 + 4 * 8; + + fn encode_body(&self, out: &mut Vec) { + out.extend_from_slice(self.allowed_environment.as_bytes()); + out.extend_from_slice(self.route_commitment.as_bytes()); + put_u32(out, self.max_request_body_bytes); + put_u32(out, self.max_output_events); + put_u32(out, self.max_output_bytes); + put_u64(out, self.max_spool_bytes); + put_u32(out, self.max_encoded_result_frame); + put_u32(out, self.max_encoded_prepared_input); + put_u64(out, self.dispatch_margin_blocks); + put_u64(out, self.delivery_margin_blocks); + put_u64(out, self.oracle_grace_blocks); + put_u64(out, self.fixed_price); + } + + fn decode_body(reader: &mut BodyReader<'_>) -> Result { + Ok(Self { + allowed_environment: ContentId::from_bytes(reader.bytes32()?), + route_commitment: Digest::from_bytes(reader.bytes32()?), + max_request_body_bytes: reader.u32()?, + max_output_events: reader.u32()?, + max_output_bytes: reader.u32()?, + max_spool_bytes: reader.u64()?, + max_encoded_result_frame: reader.u32()?, + max_encoded_prepared_input: reader.u32()?, + dispatch_margin_blocks: reader.u64()?, + delivery_margin_blocks: reader.u64()?, + oracle_grace_blocks: reader.u64()?, + fixed_price: reader.u64()?, + }) + } +} + +/// Checks that a fetch policy is a usable profile at all. +/// +/// A zero here is not a small bound, it is an absent one — the same rule +/// [`super::work::check_execution_policy`] states for the evaluate +/// profile. Unlike that policy there is no field whose zero reads as a +/// usable limit: a zero request-body bound admits no valid JSON body, a +/// zero output bound admits no terminal event, and the margins and price +/// are zero for no lawful channel. All ten are therefore required. +pub fn check_fetch_policy(policy: &PaidFetchPolicyV1) -> Result<(), PaidWorkError> { + for (field, value) in [ + ("fixed_price", policy.fixed_price), + ( + "max_request_body_bytes", + u64::from(policy.max_request_body_bytes), + ), + ("max_output_events", u64::from(policy.max_output_events)), + ("max_output_bytes", u64::from(policy.max_output_bytes)), + ("max_spool_bytes", policy.max_spool_bytes), + ( + "max_encoded_result_frame", + u64::from(policy.max_encoded_result_frame), + ), + ( + "max_encoded_prepared_input", + u64::from(policy.max_encoded_prepared_input), + ), + ("dispatch_margin_blocks", policy.dispatch_margin_blocks), + ("delivery_margin_blocks", policy.delivery_margin_blocks), + ("oracle_grace_blocks", policy.oracle_grace_blocks), + ] { + if value == 0 { + return Err(PaidWorkError::PolicyZero { field }); + } + } + Ok(()) +} + +/// Returns the digest an authorization names as its execution policy. +/// +/// The shared [`PaidJobAuthorizationV1`] field is called +/// `execution_policy_digest` because the record predates the second +/// profile; under this profile it carries this digest. +pub fn fetch_policy_digest(channel: &PaidChannel, policy: &PaidFetchPolicyV1) -> Digest { + let network_bytes = channel.network_bytes(); + xh( + FETCH_POLICY, + &[ + network_bytes.as_slice(), + channel.id().as_bytes(), + &policy.encode(), + ], + ) +} + +// ── The prepared fetch input bundle ─────────────────────────────────── + +/// The two canonical bodies a paid fetch job is prepared from, in one +/// length-delimited byte string. +/// +/// The fetch analogue of +/// [`crate::protocol::artifacts::PreparedPaidInputV1`]: +/// same `u32` big-endian length prefixes, same budget-checked +/// [`Self::decode`], same strict [`Self::parts`]. Where the evaluate +/// bundle carries an artifact graph, this one carries exactly the two +/// things the profile's commitments are opened from: the client's signed +/// fetch input transcript, and the manifest of the environment it runs +/// in. +/// +/// It carries bytes rather than parsed values for the same reason the +/// evaluate bundle does — the bytes are what the digest commits to — with +/// one sharpened consequence: the signed transcript's canonical bytes are +/// hashed here, so [`Self::parts`] decodes it strictly rather than with +/// the permissive DAG-CBOR decoder a transport boundary might use. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PreparedPaidFetchInputV1 { + fetch_input_transcript: Vec, + environment_manifest: Vec, +} + +/// The two bodies of a [`PreparedPaidFetchInputV1`], parsed. +/// +/// The manifest is retained as its parsed value for the same reason the +/// evaluate bundle retains its own: parsing proves the carried bytes were +/// canonical, and re-encoding recovers them exactly. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PreparedPaidFetchInputParts { + /// The signed fetch input events, strictly decoded. + pub fetch_input_transcript: Vec, + /// The strictly decoded environment manifest carried by the bundle. + pub manifest: ProgramManifest, +} + +impl PreparedPaidFetchInputV1 { + /// Builds a bundle from the signed input events and the manifest, + /// encoding each body once. + /// + /// # Errors + /// + /// [`PaidWorkError::Transcript`] when the transcript's canonical + /// encoding cannot be allocated. + pub fn new( + fetch_input_transcript: &[InputEventEnvelope], + environment_manifest: &ProgramManifest, + ) -> Result { + Ok(Self { + fetch_input_transcript: encode_input_transcript(fetch_input_transcript)?, + environment_manifest: environment_manifest.canonical_bytes(), + }) + } + + /// Returns the canonical encoding: two unsigned big-endian `u32` + /// lengths, each immediately followed by that many body bytes. + /// + /// Fallible for the one reason the evaluate bundle's is: a body whose + /// length does not fit its `u32` prefix has no encoding here, because + /// truncating the prefix would be a second spelling of the same bytes. + pub fn encode(&self) -> Result, CanonicalDecodeError> { + let mut bytes = Vec::new(); + for body in self.bodies() { + let len = u32::try_from(body.len()).map_err(|_| { + CanonicalDecodeError::new(format!( + "prepared fetch input body is {} bytes, over the u32 length prefix", + body.len() + )) + })?; + bytes.extend_from_slice(&len.to_be_bytes()); + bytes.extend_from_slice(body); + } + Ok(bytes) + } + + /// Decodes a bundle, refusing anything that does not fit `budget`. + /// + /// `budget` is the profile's complete-bundle limit + /// ([`PaidFetchPolicyV1::max_encoded_prepared_input`]), checked against + /// the input before the first length is read and against the running + /// total after each one, so the two individually representable lengths + /// cannot add up to a bundle this endpoint never agreed to hold. + pub fn decode(bytes: &[u8], budget: usize) -> Result { + if bytes.len() > budget { + return Err(CanonicalDecodeError::new(format!( + "prepared fetch input is {} bytes, over the {budget}-byte budget", + bytes.len() + ))); + } + let mut reader = BundleReader::new(bytes, budget); + let bundle = Self { + fetch_input_transcript: reader.body("fetch_input_transcript")?, + environment_manifest: reader.body("environment_manifest")?, + }; + if reader.offset() != bytes.len() { + return Err(CanonicalDecodeError::new(format!( + "trailing bytes after prepared fetch input: {}", + bytes.len() - reader.offset() + ))); + } + Ok(bundle) + } + + /// Parses both bodies, rejecting either that is not canonical. + /// + /// Both bodies arrive already bounded: by [`Self::decode`]'s budget + /// for a received bundle, or by [`Self::new`]'s construction for a + /// local one. The transcript is decoded strictly — decoded, + /// re-encoded, and compared — because its bytes are what + /// [`prepared_fetch_input_digest`] commits to, so a noncanonical + /// spelling is a different bundle and must not parse as this one. + pub fn parts(&self) -> Result { + Ok(PreparedPaidFetchInputParts { + fetch_input_transcript: decode_input_transcript(&self.fetch_input_transcript)?, + manifest: ProgramManifest::from_canonical_bytes(&self.environment_manifest)?, + }) + } + + fn bodies(&self) -> [&[u8]; 2] { + [&self.fetch_input_transcript, &self.environment_manifest] + } +} + +/// Encodes the signed fetch input events for the bundle. +/// +/// The input-side analogue of [`super::work::encode_transcript`]: DAG-CBOR +/// over the signed envelopes, through the same derived `Serialize`. The +/// one difference is that this encoding *is* hashed — it is a bundle body +/// — which is why [`decode_input_transcript`] below is strict where +/// [`super::work::decode_transcript`] is not. +fn encode_input_transcript(transcript: &[InputEventEnvelope]) -> Result, PaidWorkError> { + canonical_dag_cbor(&transcript.to_vec()) + .map_err(|error| PaidWorkError::Transcript(error.to_string())) +} + +/// Reads the signed fetch input events back, accepting only the canonical +/// spelling. +fn decode_input_transcript(bytes: &[u8]) -> Result, CanonicalDecodeError> { + decode_canonical_dag_cbor(bytes) +} + +/// Returns the digest an authorization names as its prepared input. +/// +/// Streamed, mirroring [`super::work::prepared_input_digest`]: the bundle +/// carries a signed transcript whose size is the client's to choose +/// within the policy bound, so it is one of the preimages with no fixed +/// width. +pub fn prepared_fetch_input_digest( + channel: &PaidChannel, + bundle: &PreparedPaidFetchInputV1, +) -> Result { + let network_bytes = channel.network_bytes(); + Ok(xfh( + PREPARED_FETCH_INPUT, + &[ + network_bytes.as_slice(), + channel.id().as_bytes(), + &bundle.encode()?, + ], + )) +} + +// ── Authorization ───────────────────────────────────────────────────── + +/// Builds one fetch job's authorization, deriving every field the +/// channel, the policy, and the prepared inputs already fix. +/// +/// The fetch analogue of [`super::work::propose_authorization`], with the +/// same narrow guarantee: the nonce and the three deadlines are the only +/// choices left to the caller, and every other field is read out of +/// something that already exists. The request commitment is computed the +/// fetch ticket flow's way — the signed input transcript is verified by +/// [`crate::fetch::verify_input_events`], and its input commitment is the +/// request commitment — so a proposal cannot name a commitment its own +/// transcript does not produce. +/// +/// It checks nothing beyond what deriving those fields requires. +/// [`check_fetch_authorization`] and [`check_prepared_fetch_input`] are +/// where the refusals are written, and they are what the *other* party +/// runs. +/// +/// # Errors +/// +/// [`PaidWorkError::Body`] when the bundle's own bodies are not canonical, +/// [`PaidWorkError::Transcript`] when the carried events are not one +/// well-formed signed fetch input, and [`PaidWorkError::Overflow`] when +/// the bundle is too large to length-prefix. +pub fn propose_fetch_authorization( + channel: &PaidChannel, + policy: &PaidFetchPolicyV1, + bundle: &PreparedPaidFetchInputV1, + proposal_nonce: u64, + deadlines: JobDeadlines, +) -> Result { + let terms = channel.payment_terms(); + let parts = bundle.parts()?; + let input = crate::fetch::verify_input_events(&parts.fetch_input_transcript) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + Ok(PaidJobAuthorizationV1 { + channel_id: channel.id(), + bond_edge: terms.bond_edge, + bond_terms_hash: terms.bond_terms_hash(), + payment_edge: channel.payment_edge(), + payment_terms_hash: channel.payment_terms_hash(), + execution_policy_digest: fetch_policy_digest(channel, policy), + prepared_input_digest: prepared_fetch_input_digest(channel, bundle)?, + proposal_nonce, + acceptance_deadline: deadlines.acceptance, + request_commitment: RequestCommitment::from_digest(input.input_commitment.digest()), + environment_commitment: parts.manifest.content_id(), + price: policy.fixed_price, + terminal_deadline: deadlines.terminal, + payment_deadline: deadlines.payment, + }) +} + +/// Checks one authorization against the channel, the fetch policy it +/// names, and the height it is being signed at, and returns its +/// `work_id`. +/// +/// The fetch analogue of [`super::work::check_authorization`]. Everything +/// the two profiles mean by *accepting* a job — channel fields, bond +/// cover, price cover, deadline window — is shared +/// [`check_authorization_core`]; what differs is which policy digest, +/// environment, and price those rules are applied to, and that is what +/// this wrapper supplies. +pub fn check_fetch_authorization( + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + policy: &PaidFetchPolicyV1, + finalized_height: u64, +) -> Result { + check_fetch_policy(policy)?; + check_authorization_core( + channel, + authorization, + fetch_policy_digest(channel, policy), + policy.allowed_environment, + policy.fixed_price, + finalized_height, + ) +} + +/// Checks the prepared fetch bundle against the authorization, the +/// policy, and the route policy that commit to it. +/// +/// The fetch analogue of [`super::work::check_prepared_input`], and the +/// same statement in this profile's vocabulary: holding a bundle whose +/// digest matches is not knowing what is in it. The checks, in order: +/// the bundle fits the policy's encoding bound; its digest is the +/// authorization's; the signed events verify as one caller-signed fetch +/// input chain under [`crate::fetch::verify_input_events`] — which also +/// applies the fetch protocol's own hard bounds, including +/// [`crate::fetch::MAX_FETCH_REQUEST_BODY_BYTES`]; the caller is the +/// channel's client; the signed assurance selects the result scheme; the route body +/// opens the policy's route commitment and, for a sealed route, names the +/// service and method the events sign; the manifest, the events' +/// environment, the policy's allowed environment, and the +/// authorization's environment commitment are one; and the request +/// commitment is the verified transcript's. +/// +/// The route policy is an argument rather than a bundle body because the +/// bundle is the client's statement of the job and the route is the +/// channel's statement of what it sells: the two meet here, against the +/// commitment both signed into the policy. +/// +/// Open Fetch additionally verifies the generic HTTPS manifest, request schema, +/// host allowlist and any required SPKI pin. Network-dependent certificate and +/// address checks happen inside that interpreter at dispatch. +pub fn check_prepared_fetch_input( + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + policy: &PaidFetchPolicyV1, + route: &FetchRoutePolicy, + bundle: &PreparedPaidFetchInputV1, +) -> Result<(), PaidWorkError> { + let encoded = bundle.encode()?; + let limit = u64::from(policy.max_encoded_prepared_input); + let actual = u64::try_from(encoded.len()).map_err(|_| PaidWorkError::Overflow { + field: "prepared input length", + })?; + if actual > limit { + return Err(PaidWorkError::OverEnvelope { + field: "prepared input length", + actual, + limit, + }); + } + if prepared_fetch_input_digest(channel, bundle)?.as_bytes() + != authorization.prepared_input_digest.as_bytes() + { + return Err(PaidWorkError::Mismatch { + field: "prepared_input_digest", + }); + } + + let parts = bundle.parts()?; + let input = crate::fetch::verify_input_events(&parts.fetch_input_transcript) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + + // The fetch analogue of the evaluate profile's runner-key rule: the + // request must be the client's own, and one verified signature chain + // is how a fetch request says whose it is. Verification takes the key + // from the first event, so it cannot say whose key it is; this is + // what says it is the client's. + if input.caller_key != PublicKey::Secp256k1(channel.client_key().to_bytes()) { + return Err(PaidWorkError::Mismatch { + field: "caller_key", + }); + } + + // The supplied route body is accounted to the signed policy before it + // is allowed to constrain anything: a route whose commitment differs + // is a route this channel never sold, whatever it claims. + if fetch_route_commitment(&route.canonical_body_bytes())?.as_bytes() + != policy.route_commitment.as_bytes() + { + return Err(PaidWorkError::Mismatch { + field: "route_commitment", + }); + } + if let FetchRoutePolicy::SealedRoute { service, method } = route { + for (field, holds) in [ + ("service", input.service == *service), + ("method", input.method == *method), + ] { + if !holds { + return Err(PaidWorkError::Mismatch { field }); + } + } + } + + if let FetchRoutePolicy::OpenFetch { + require_spki_pin, + allowed_hosts, + } = route + { + if parts.manifest.content_id() != crate::FetchEnvironment::Http.manifest_id() { + return Err(PaidWorkError::Mismatch { + field: "open-fetch HTTPS manifest", + }); + } + let request = crate::http_fetch::HttpFetchRequest::decode(input.body.as_bytes()) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; + let url = request + .parsed_url() + .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; + if (*require_spki_pin && request.tls.spki_sha256.is_empty()) + || (!allowed_hosts.is_empty() + && !allowed_hosts + .iter() + .any(|h| Some(h.as_str()) == url.host_str())) + { + return Err(PaidWorkError::Mismatch { + field: "open-fetch host or pin policy", + }); + } + } + + let graph = [ + ( + "manifest content id", + parts.manifest.content_id().as_bytes() == input.execution_environment.as_bytes(), + ), + ( + "environment_commitment", + input.execution_environment.as_bytes() + == authorization.environment_commitment.as_bytes(), + ), + ( + "allowed_environment", + parts.manifest.content_id().as_bytes() == policy.allowed_environment.as_bytes(), + ), + ( + "request_commitment", + RequestCommitment::from_digest(input.input_commitment.digest()).as_bytes() + == authorization.request_commitment.as_bytes(), + ), + ]; + for (field, holds) in graph { + if !holds { + return Err(PaidWorkError::Mismatch { field }); + } + } + + let body = u64::try_from(input.body.as_bytes().len()).map_err(|_| PaidWorkError::Overflow { + field: "request body length", + })?; + let limit = u64::from(policy.max_request_body_bytes); + if body > limit { + return Err(PaidWorkError::OverEnvelope { + field: "request body", + actual: body, + limit, + }); + } + + Ok(()) +} + +/// Enforces the channel's output limits before a result is recorded or paid. +/// Both counts include the terminal envelope, matching the Fetch wire limits. +pub fn check_fetch_output_limits( + policy: &PaidFetchPolicyV1, + transcript: &[OutputEventEnvelope], +) -> Result<(), PaidWorkError> { + let bytes = transcript.iter().try_fold(0_u64, |total, event| { + total + .checked_add(event.payload().len() as u64) + .ok_or(PaidWorkError::Overflow { + field: "fetch output bytes", + }) + })?; + for (field, actual, limit) in [ + ( + "fetch output events", + transcript.len() as u64, + u64::from(policy.max_output_events), + ), + ( + "fetch output bytes", + bytes, + u64::from(policy.max_output_bytes), + ), + ] { + if actual > limit { + return Err(PaidWorkError::OverEnvelope { + field, + actual, + limit, + }); + } + } + Ok(()) +} + +// ── The terminal result ─────────────────────────────────────────────── + +/// Returns the digest of the normalized fetch answer. +/// +/// Chunk boundaries are not part of the answer, exactly as in +/// [`super::work::canonical_output_digest`]: the semantic event payloads +/// and the terminal payload are flattened into one byte stream, so two +/// providers that split the same answer across different signed events +/// produce the same digest here while producing different transcript +/// commitments. The fetch answer has no token ids and no counts to +/// cross-check — the payload bytes are the answer, which is why this +/// digest, unlike the evaluate one, has nothing to refuse. The payloads +/// are canonical DAG-CBOR with distinct event and terminal codec strings, +/// so a reader that parses the stream can still tell where the terminal +/// begins even though this digest does not bind that boundary; the +/// transcript commitment beside it binds the framing exactly. +#[must_use] +pub fn fetch_canonical_output_digest( + network: NetworkId, + work_id: Digest, + event_payloads: &[Vec], + terminal_payload: &[u8], +) -> Digest { + let network_bytes = EncodedNetwork::new(network); + let mut hasher = XetFileHasher::new(); + hasher.update(FETCH_OUTPUT); + hasher.update(network_bytes.as_slice()); + hasher.update(work_id.as_bytes()); + for payload in event_payloads { + hasher.update(payload); + } + hasher.update(terminal_payload); + hasher.finalize() +} + +/// Builds the result record for the fetch transcript one invocation of +/// this job produced. +/// +/// The fetch analogue of [`super::work::terminal_result`], with the same +/// shape and the same reason for it: what makes a result the provider's +/// own is that the events it summarises verify as one signed chain — the +/// fetch scheme, this authorization's request commitment, contiguous +/// sequence from the output genesis, and a terminal event at the end, +/// which is exactly what [`crate::fetch::verify_output_events`] checks. +/// None of that can be supplied by a caller holding a commitment, so no +/// path here accepts one. +/// +/// The two digests it produces say different things about the same +/// invocation, as they do for evaluate: +/// `terminal_transcript_commitment` binds the provider's exact signed +/// framing; [`fetch_canonical_output_digest`] binds the flattened answer, +/// so two transcripts that split the same payloads differently agree on +/// it. +/// +/// # Errors +/// +/// [`PaidWorkError::Transcript`] when the events are not one verified +/// fetch output transcript for this authorization's request. +/// [`PaidWorkError::Mismatch`] when they were produced under a key this +/// channel does not call the provider. +pub fn terminal_fetch_result( + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + transcript: &[OutputEventEnvelope], +) -> Result { + terminal_fetch_result_with_assurance( + channel, + authorization, + transcript, + Assurance::ProducerSigned, + ) +} + +/// Verifies the assurance committed by the caller's authenticated input. +pub fn terminal_fetch_result_with_assurance( + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + transcript: &[OutputEventEnvelope], + assurance: Assurance, +) -> Result { + let input = InputCommitment::from_digest(authorization.request_commitment.digest()); + let output = crate::fetch::verify_output_events(input, assurance, transcript) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + + // Verification above establishes that one key signed every event; it + // takes that key from the first event, so it cannot say whose key it + // is. This is what says it is the provider's — the same compressed + // secp256k1 point the payment terms name as a party. + if output.producer_key != PublicKey::Secp256k1(channel.provider_key().to_bytes()) { + return Err(PaidWorkError::Mismatch { + field: "transcript producer key", + }); + } + + let Some(terminal_event) = transcript.last() else { + // Unreachable for the same reason as in + // `super::work::terminal_result`: verification refuses an empty + // transcript, and a non-empty slice has a last element. Written as + // a refusal because nothing in this module panics. + return Err(PaidWorkError::Transcript( + "the terminal transcript is empty".to_string(), + )); + }; + + let (event_payloads, terminal_payload) = output.output_event_payloads(); + for payload in event_payloads { + crate::fetch::decode_fetch_event_payload(payload) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + } + crate::fetch::decode_fetch_terminal_payload(terminal_payload) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + let work_id = work_id(channel, authorization); + Ok(PaidJobResultV1 { + work_id, + terminal_transcript_commitment: terminal_event.event_commitment(), + canonical_output_digest: fetch_canonical_output_digest( + channel.network(), + work_id, + event_payloads, + terminal_payload, + ), + }) +} + +// ── Bounds ──────────────────────────────────────────────────────────── + +const ENCODED_NETWORK: usize = ::MAX_ENCODED_SIZE; + +/// Widest of this module's fixed records: the one record, today. Written +/// as a named bound rather than inline so a second fetch record must +/// widen it deliberately, as `super::work::WIDEST_RECORD` does for its +/// five. +const WIDEST_RECORD: usize = PaidFetchPolicyV1::ENCODED_SIZE; + +/// Longest of this module's single-chunk domains. The three +/// variable-body digests are streamed and need no bound; the one +/// fixed-record digest — [`fetch_policy_digest`] — does. +const LONGEST_XH_DOMAIN: usize = FETCH_POLICY.len(); + +/// Largest complete `XH` preimage this module can produce: +/// `domain || network || channel_id || record`, the one record-shaped +/// preimage here. +const WIDEST_XH_PREIMAGE: usize = LONGEST_XH_DOMAIN + ENCODED_NETWORK + 32 + WIDEST_RECORD; + +const _: () = assert!( + WIDEST_XH_PREIMAGE < MIN_CHUNK_SIZE, + "a single-chunk preimage that reaches MIN_CHUNK_SIZE panics the hasher" +); diff --git a/crates/rpc/src/protocol/work_profile.rs b/crates/rpc/src/protocol/work_profile.rs new file mode 100644 index 00000000..f7a2141e --- /dev/null +++ b/crates/rpc/src/protocol/work_profile.rs @@ -0,0 +1,337 @@ +//! Dispatch for the private paid-work profiles. The channel economics are +//! shared; each profile verifies its own input and terminal transcript. + +use super::artifacts::PreparedPaidInputV1; +use super::value::CanonicalDecodeError; +use super::work::{ + self, JobDeadlines, PaidChannel, PaidExecutionPolicyV1, PaidJobAuthorizationV1, + PaidJobResultV1, PaidWorkError, PrivateRecord, +}; +use super::work_fetch::{self, FetchRoutePolicy, PaidFetchPolicyV1, PreparedPaidFetchInputV1}; +use crate::{ContentId, Digest, OutputEventEnvelope}; + +/// The execution contract fixed when a channel is mounted. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PaidWorkPolicy { + /// Reproducible local evaluation. + Evaluate(PaidExecutionPolicyV1), + /// An authenticated fetch transcript under the committed route policy. + Fetch { + /// Fixed-width resource and price envelope. + policy: PaidFetchPolicyV1, + /// Canonical body opening the policy's route commitment. + route: FetchRoutePolicy, + }, +} + +impl From for PaidWorkPolicy { + fn from(policy: PaidExecutionPolicyV1) -> Self { + Self::Evaluate(policy) + } +} + +impl PaidWorkPolicy { + /// Checks the envelope and opens every variable policy commitment. + pub fn check(&self) -> Result<(), PaidWorkError> { + match self { + Self::Evaluate(policy) => work::check_execution_policy(policy), + Self::Fetch { policy, route } => { + work_fetch::check_fetch_policy(policy)?; + let body = route.canonical_body_bytes(); + FetchRoutePolicy::from_canonical_body_bytes(&body)?; + if work_fetch::fetch_route_commitment(&body)? != policy.route_commitment { + return Err(PaidWorkError::Mismatch { + field: "route_commitment", + }); + } + if matches!(route, FetchRoutePolicy::OpenFetch { .. }) + && policy.allowed_environment != crate::FetchEnvironment::Http.manifest_id() + { + return Err(PaidWorkError::Mismatch { + field: "open-fetch HTTPS manifest", + }); + } + Ok(()) + } + } + } + + /// The manifest that every proposal must name. + pub const fn allowed_environment(&self) -> ContentId { + match self { + Self::Evaluate(p) => p.allowed_environment, + Self::Fetch { policy, .. } => policy.allowed_environment, + } + } + + /// Fixed price both endpoints derive before admission. + pub const fn fixed_price(&self) -> u64 { + match self { + Self::Evaluate(p) => p.fixed_price, + Self::Fetch { policy, .. } => policy.fixed_price, + } + } + + /// Maximum retained encoded transcript. + pub const fn max_spool_bytes(&self) -> u64 { + match self { + Self::Evaluate(p) => p.max_spool_bytes, + Self::Fetch { policy, .. } => policy.max_spool_bytes, + } + } + + /// Maximum encoded delivery frame. + pub const fn max_encoded_result_frame(&self) -> u32 { + match self { + Self::Evaluate(p) => p.max_encoded_result_frame, + Self::Fetch { policy, .. } => policy.max_encoded_result_frame, + } + } + + /// Dispatch, delivery, and oracle grace margins, in finalized blocks. + pub const fn margins(&self) -> (u64, u64, u64) { + match self { + Self::Evaluate(p) => ( + p.dispatch_margin_blocks, + p.delivery_margin_blocks, + p.oracle_grace_blocks, + ), + Self::Fetch { policy: p, .. } => ( + p.dispatch_margin_blocks, + p.delivery_margin_blocks, + p.oracle_grace_blocks, + ), + } + } + + /// Checks a proposal against this profile's commitment and envelope. + pub fn check_authorization( + &self, + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + height: u64, + ) -> Result { + match self { + Self::Evaluate(policy) => { + work::check_authorization(channel, authorization, policy, height) + } + Self::Fetch { policy, .. } => { + work_fetch::check_fetch_authorization(channel, authorization, policy, height) + } + } + } + + /// Builds the authorization using the matching profile's digest domains. + pub fn propose( + &self, + channel: &PaidChannel, + input: &PreparedPaidWorkInput, + nonce: u64, + deadlines: JobDeadlines, + ) -> Result { + match (self, input) { + (Self::Evaluate(policy), PreparedPaidWorkInput::Evaluate(input)) => { + work::propose_authorization(channel, policy, input, nonce, deadlines) + } + (Self::Fetch { policy, .. }, PreparedPaidWorkInput::Fetch(input)) => { + work_fetch::propose_fetch_authorization(channel, policy, input, nonce, deadlines) + } + _ => Err(PaidWorkError::Mismatch { + field: "paid work profile", + }), + } + } + + /// Opens the prepared input and checks its profile-specific constraints. + pub fn check_input( + &self, + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + input: &PreparedPaidWorkInput, + ) -> Result<(), PaidWorkError> { + match (self, input) { + (Self::Evaluate(policy), PreparedPaidWorkInput::Evaluate(input)) => { + work::check_prepared_input(channel, authorization, policy, input) + } + (Self::Fetch { policy, route }, PreparedPaidWorkInput::Fetch(input)) => { + work_fetch::check_prepared_fetch_input( + channel, + authorization, + policy, + route, + input, + )?; + let parts = input.parts()?; + let request = crate::fetch::verify_input_events(&parts.fetch_input_transcript) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + if request.retention != crate::Retention::Ephemeral { + return Err(PaidWorkError::Mismatch { + field: "paid fetch requires ephemeral retention", + }); + } + Ok(()) + } + _ => Err(PaidWorkError::Mismatch { + field: "paid work profile", + }), + } + } + + /// Verifies bounds and the result against the authenticated prepared input. + pub fn terminal_result( + &self, + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + input: &PreparedPaidWorkInput, + transcript: &[OutputEventEnvelope], + ) -> Result { + self.check_input(channel, authorization, input)?; + if let Self::Fetch { policy, .. } = self { + work_fetch::check_fetch_output_limits(policy, transcript)?; + } + input.terminal_result(channel, authorization, transcript) + } + + /// Canonical bytes retained in a close descriptor. Evaluate bytes are unchanged. + pub fn encode(&self) -> Vec { + match self { + Self::Evaluate(policy) => policy.encode(), + Self::Fetch { policy, route } => { + let mut bytes = policy.encode(); + bytes.extend_from_slice(&route.canonical_body_bytes()); + bytes + } + } + } + + /// Strictly decodes one complete profile policy. + pub fn decode(bytes: &[u8]) -> Result { + let profile = if bytes.get(1) == Some(&5) { + let (record, body) = bytes + .split_at_checked(PaidFetchPolicyV1::ENCODED_SIZE) + .ok_or_else(|| CanonicalDecodeError::new("truncated fetch policy"))?; + Self::Fetch { + policy: PaidFetchPolicyV1::decode(record)?, + route: FetchRoutePolicy::from_canonical_body_bytes(body)?, + } + } else { + Self::Evaluate(PaidExecutionPolicyV1::decode(bytes)?) + }; + profile.check()?; + Ok(profile) + } +} + +/// Canonical prepared input retained by a paid-work journal. +/// +/// The formats are disjoint: Evaluate has exactly six length-prefixed bodies, +/// Fetch exactly two. Both decoders reject trailing bytes. Dispatch therefore +/// preserves existing Evaluate bytes without guessing from unsigned metadata. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PreparedPaidWorkInput { + /// The committed local artifact graph. + Evaluate(PreparedPaidInputV1), + /// The signed fetch request and its manifest. + Fetch(PreparedPaidFetchInputV1), +} + +impl From for PreparedPaidWorkInput { + fn from(input: PreparedPaidInputV1) -> Self { + Self::Evaluate(input) + } +} + +impl From for PreparedPaidWorkInput { + fn from(input: PreparedPaidFetchInputV1) -> Self { + Self::Fetch(input) + } +} + +impl PreparedPaidWorkInput { + /// Decodes a complete bounded bundle under exactly one profile. + pub fn decode(bytes: &[u8], budget: usize) -> Result { + if let Ok(input) = PreparedPaidInputV1::decode(bytes, budget) { + return Ok(Self::Evaluate(input)); + } + PreparedPaidFetchInputV1::decode(bytes, budget).map(Self::Fetch) + } + + /// Returns the original profile's canonical bytes. + pub fn encode(&self) -> Result, CanonicalDecodeError> { + match self { + Self::Evaluate(input) => input.encode(), + Self::Fetch(input) => input.encode(), + } + } + + /// The caller's authenticated request identity, independent of payment. + pub fn input_commitment(&self) -> Result { + match self { + Self::Evaluate(input) => Ok(crate::evaluate::input_commitment( + &input.parts()?.evaluate_request, + )), + Self::Fetch(input) => Ok(crate::fetch::verify_input_events( + &input.parts()?.fetch_input_transcript, + ) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))? + .input_commitment), + } + } + + /// Assurance requested by the canonical input, checked before disclosing it. + pub fn assurance(&self) -> Result { + match self { + Self::Evaluate(input) => Ok(input.parts()?.evaluate_request.assurance), + Self::Fetch(input) => Ok(crate::fetch::verify_input_events( + &input.parts()?.fetch_input_transcript, + ) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))? + .assurance), + } + } + + /// Recomputes the prepared-input commitment during journal replay. + pub fn digest(&self, channel: &PaidChannel) -> Result { + match self { + Self::Evaluate(input) => work::prepared_input_digest(channel, input), + Self::Fetch(input) => work_fetch::prepared_fetch_input_digest(channel, input), + } + } + + /// Rebuilds a result in the profile selected by the journaled input. + pub fn terminal_result( + &self, + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + transcript: &[OutputEventEnvelope], + ) -> Result { + match self { + Self::Evaluate(_) => work::terminal_result(channel, authorization, transcript), + Self::Fetch(bundle) => { + let parts = bundle.parts()?; + let input = crate::fetch::verify_input_events(&parts.fetch_input_transcript) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; + let result = work_fetch::terminal_fetch_result_with_assurance( + channel, + authorization, + transcript, + input.assurance, + )?; + if input.execution_environment == crate::FetchEnvironment::Http.manifest_id() { + let request = + crate::http_fetch::HttpFetchRequest::decode(input.body.as_bytes()) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; + let output = crate::fetch::verify_output_events( + input.input_commitment, + input.assurance, + transcript, + ) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; + crate::http_fetch::HttpFetchResponse::from_output(&request, &output) + .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; + } + Ok(result) + } + } + } +} diff --git a/crates/rpc/src/protocol/work_setup.rs b/crates/rpc/src/protocol/work_setup.rs index 89061997..3b6b6447 100644 --- a/crates/rpc/src/protocol/work_setup.rs +++ b/crates/rpc/src/protocol/work_setup.rs @@ -18,8 +18,8 @@ use hellas_kernel::{ use crate::protocol::work::{ PaidChannel, PaidChannelPolicyV1, PaidExecutionPolicyV1, PaidWorkError, PrivateRecord, - check_execution_policy, }; +use crate::protocol::work_profile::PaidWorkPolicy; /// Why a configured channel is not one this endpoint may work over. #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] @@ -226,7 +226,7 @@ pub struct WorkChannelConfig { /// The credit policy those terms commit to. pub channel_policy: PaidChannelPolicyV1, /// The execution policy in force on this channel. - pub execution_policy: PaidExecutionPolicyV1, + pub execution_policy: PaidWorkPolicy, /// The payment edge's value, reserve, and close fees as the /// operator expects them to be funded. pub expected_payment_values: EdgeValues, @@ -252,7 +252,7 @@ pub struct ProviderChannelPolicy { /// The credit policy this provider will work under. pub channel_policy: PaidChannelPolicyV1, /// The execution policy this provider will run jobs under. - pub execution_policy: PaidExecutionPolicyV1, + pub execution_policy: PaidWorkPolicy, /// The payment edge's value, reserve, and close fees as the provider /// requires them to be funded. pub expected_payment_values: EdgeValues, @@ -293,7 +293,7 @@ impl ProviderChannelPolicy { &self.policy_salt, self.channel_policy, )?; - check_execution_policy(&self.execution_policy)?; + self.execution_policy.check()?; if work_payment_settlement( self.expected_payment_values, channel.payment_terms().omission_bond, @@ -306,7 +306,7 @@ impl ProviderChannelPolicy { channel, bond_edge, policy_salt: self.policy_salt, - execution_policy: self.execution_policy, + execution_policy: self.execution_policy.clone(), expected_payment_values: self.expected_payment_values, }) } @@ -361,7 +361,7 @@ impl ProviderChannelPolicy { payment_terms, policy_salt: self.policy_salt, channel_policy: self.channel_policy, - execution_policy: self.execution_policy, + execution_policy: self.execution_policy.clone(), expected_payment_values: self.expected_payment_values, }) } @@ -380,7 +380,7 @@ pub struct WorkChannelDescriptor { bond_edge: EdgeId, bond_terms_hash: TermsHash, policy_salt: [u8; 32], - execution_policy: PaidExecutionPolicyV1, + execution_policy: PaidWorkPolicy, expected_payment_values: EdgeValues, } @@ -418,7 +418,7 @@ impl WorkChannelDescriptor { &config.policy_salt, config.channel_policy, )?; - check_execution_policy(&config.execution_policy)?; + config.execution_policy.check()?; let settlement = work_payment_settlement(config.expected_payment_values, omission_bond) .ok_or(WorkSetupError::Unsettleable)?; @@ -445,7 +445,7 @@ impl WorkChannelDescriptor { } /// Returns the execution policy in force on this channel. - pub const fn execution_policy(&self) -> &PaidExecutionPolicyV1 { + pub const fn execution_policy(&self) -> &PaidWorkPolicy { &self.execution_policy } @@ -462,7 +462,7 @@ impl WorkChannelDescriptor { channel: self.channel.clone(), bond_edge: self.bond_edge, policy_salt: self.policy_salt, - execution_policy: self.execution_policy, + execution_policy: self.execution_policy.clone(), expected_payment_values: self.expected_payment_values, } } @@ -578,7 +578,7 @@ impl WorkChannelDescriptor { Ok(ReadyChannel { channel: self.channel.clone(), - execution_policy: self.execution_policy, + execution_policy: self.execution_policy.clone(), settlement, finalized_height: observed.height, admission_horizon: horizon, @@ -607,7 +607,7 @@ pub struct CloseDescriptor { channel: PaidChannel, bond_edge: EdgeId, policy_salt: [u8; 32], - execution_policy: PaidExecutionPolicyV1, + execution_policy: PaidWorkPolicy, expected_payment_values: EdgeValues, } @@ -630,7 +630,7 @@ impl CloseDescriptor { /// Returns the execution policy the armed endpoint accepted. #[must_use] - pub const fn execution_policy(&self) -> &PaidExecutionPolicyV1 { + pub const fn execution_policy(&self) -> &PaidWorkPolicy { &self.execution_policy } @@ -673,7 +673,12 @@ impl CloseDescriptor { /// Returns the descriptor's canonical journal bytes. #[must_use] pub fn encode(&self) -> Vec { - let mut out = vec![CLOSE_DESCRIPTOR_VERSION]; + let version = if matches!(self.execution_policy, PaidWorkPolicy::Evaluate(_)) { + CLOSE_DESCRIPTOR_VERSION + } else { + 2 + }; + let mut out = vec![version]; push_kernel(&mut out, &self.channel.network()); push_kernel(&mut out, &self.channel.payment_edge()); push_kernel( @@ -682,7 +687,11 @@ impl CloseDescriptor { ); out.extend_from_slice(&self.policy_salt); out.extend_from_slice(&self.channel.channel_policy().encode()); - out.extend_from_slice(&self.execution_policy.encode()); + let policy_bytes = self.execution_policy.encode(); + if version == 2 { + out.extend_from_slice(&(policy_bytes.len() as u64).to_be_bytes()); + } + out.extend_from_slice(&policy_bytes); out.extend_from_slice(&self.expected_payment_values.value().to_be_bytes()); out.extend_from_slice(&self.expected_payment_values.reserve().to_be_bytes()); push_kernel(&mut out, &self.expected_payment_values.close_fees()); @@ -697,7 +706,8 @@ impl CloseDescriptor { /// rerun them. pub fn decode(bytes: &[u8]) -> Result { let mut cursor = CloseCursor { bytes }; - if cursor.byte()? != CLOSE_DESCRIPTOR_VERSION { + let version = cursor.byte()?; + if version != CLOSE_DESCRIPTOR_VERSION && version != 2 { return Err(WorkSetupError::DescriptorMalformed); } let network = cursor.network()?; @@ -710,9 +720,18 @@ impl CloseDescriptor { let channel_policy = PaidChannelPolicyV1::decode(cursor.take(PaidChannelPolicyV1::ENCODED_SIZE)?) .map_err(|_| WorkSetupError::DescriptorMalformed)?; - let execution_policy = - PaidExecutionPolicyV1::decode(cursor.take(PaidExecutionPolicyV1::ENCODED_SIZE)?) - .map_err(|_| WorkSetupError::DescriptorMalformed)?; + let policy_len = if version == CLOSE_DESCRIPTOR_VERSION { + PaidExecutionPolicyV1::ENCODED_SIZE + } else { + usize::try_from(cursor.u64()?).map_err(|_| WorkSetupError::DescriptorMalformed)? + }; + let execution_policy = PaidWorkPolicy::decode(cursor.take(policy_len)?) + .map_err(|_| WorkSetupError::DescriptorMalformed)?; + if (version == CLOSE_DESCRIPTOR_VERSION) + != matches!(execution_policy, PaidWorkPolicy::Evaluate(_)) + { + return Err(WorkSetupError::DescriptorMalformed); + } let expected_payment_values = EdgeValues::new(cursor.u64()?, cursor.u64()?, cursor.field::()?); if !cursor.bytes.is_empty() { @@ -726,7 +745,7 @@ impl CloseDescriptor { &policy_salt, channel_policy, )?; - check_execution_policy(&execution_policy)?; + execution_policy.check()?; if work_payment_settlement( expected_payment_values, channel.payment_terms().omission_bond, @@ -851,7 +870,7 @@ pub struct ObservedChannel<'a> { #[derive(Clone, Debug, PartialEq, Eq)] pub struct ReadyChannel { channel: PaidChannel, - execution_policy: PaidExecutionPolicyV1, + execution_policy: PaidWorkPolicy, settlement: WorkPaymentSettlement, finalized_height: u64, admission_horizon: u64, @@ -875,7 +894,7 @@ impl ReadyChannel { /// whose margins [`Self::check_signable`] measures against and whose /// digest the authorization names. Carrying it here is what stops /// those three from being three copies. - pub const fn execution_policy(&self) -> &PaidExecutionPolicyV1 { + pub const fn execution_policy(&self) -> &PaidWorkPolicy { &self.execution_policy } @@ -936,18 +955,18 @@ impl ReadyChannel { }); } - let policy = &self.execution_policy; + let (dispatch, delivery, oracle_grace) = self.execution_policy.margins(); let reachable = cursor_height - .checked_add(policy.dispatch_margin_blocks) - .and_then(|sum| sum.checked_add(policy.delivery_margin_blocks)) + .checked_add(dispatch) + .and_then(|sum| sum.checked_add(delivery)) .ok_or(PaidWorkError::Overflow { field: "dispatch and delivery margins", })?; if reachable > terminal_deadline { return Err(WorkSetupError::TerminalUnreachable { height: cursor_height, - dispatch: policy.dispatch_margin_blocks, - delivery: policy.delivery_margin_blocks, + dispatch, + delivery, terminal: terminal_deadline, }); } @@ -958,12 +977,12 @@ impl ReadyChannel { .ok_or(PaidWorkError::Overflow { field: "oracle grace interval", })?; - if grace < policy.oracle_grace_blocks { + if grace < oracle_grace { return Err(WorkSetupError::OracleGraceTooShort { terminal: terminal_deadline, payment: payment_deadline, actual: grace, - grace: policy.oracle_grace_blocks, + grace: oracle_grace, }); } Ok(()) @@ -1018,7 +1037,7 @@ impl ReadyChannel { terminal_deadline: u64, ) -> Result<(), WorkSetupError> { self.check_caught_up(cursor_height)?; - let delivery = self.execution_policy.delivery_margin_blocks; + let delivery = self.execution_policy.margins().1; let arrives = cursor_height .checked_add(delivery) .ok_or(PaidWorkError::Overflow { diff --git a/crates/rpc/tests/paid_fetch_vectors.rs b/crates/rpc/tests/paid_fetch_vectors.rs new file mode 100644 index 00000000..e71f7b06 --- /dev/null +++ b/crates/rpc/tests/paid_fetch_vectors.rs @@ -0,0 +1,1696 @@ +//! Independent vectors and mutation proofs for the paid-fetch records. +//! +//! The fetch-profile companion to `paid_work_vectors.rs`, in the same +//! style: canonical encodings pinned as hex decoded field by field, +//! digests pinned as hex, and every refusal covered by a mutation that +//! names what it breaks. +//! +//! The signed input transcript is built with the nonce pinned rather than +//! through `build_input_events_with_retention`: that constructor draws the +//! nonce from `OsRng`, and a golden digest over a random nonce is a +//! different vector every run. The transcript here is otherwise the exact +//! eight events that constructor signs, and the randomized constructor is +//! exercised end-to-end in +//! `the_real_input_constructor_passes_the_whole_pipeline`. + +#![cfg(feature = "work")] + +use hellas_kernel::{ + BlockHeight, EdgeId, EdgeValues, Fees, List, NetworkId, Parties, Payout, Secp256k1Signer, + Secp256k1Verifier, SigVerifier, WorkPaymentSettlement, WorkPaymentTerms, WorkStakeBondTerms, + work_payment_settlement, +}; +use hellas_rpc::fetch::{ + FetchOutputTranscriptBuilder, build_input_events_with_retention, encode_fetch_event_payload, + encode_fetch_terminal_payload, input_canonicalization, output_canonicalization, +}; +use hellas_rpc::output::{OutputEvent, StopReason, TextChannel, Usage}; +use hellas_rpc::protocol::work::{ + CreditLedger, JobDeadlines, PaidChannel, PaidChannelPolicyV1, PaidJobAuthorizationV1, + PaidWorkError, PrivateRecord, check_authorization, check_result, execution_policy_digest, + next_payment, result_digest, signing_hash, work_id, +}; +use hellas_rpc::protocol::work_fetch::{ + FetchRoutePolicy, PaidFetchPolicyV1, PreparedPaidFetchInputV1, check_fetch_authorization, + check_fetch_policy, check_prepared_fetch_input, fetch_canonical_output_digest, + fetch_policy_digest, fetch_route_commitment, prepared_fetch_input_digest, + propose_fetch_authorization, terminal_fetch_result, +}; +use hellas_rpc::{ + Assurance, ContentId, Digest, FetchEnvironment, InputCommitment, InputEventEnvelope, + InputTranscriptBuilder, Operation, OutputEventEnvelope, ProducerSigningKey, ProgramManifest, + PublicKey, RequestCommitment, Retention, scheme_id, +}; + +// ── Fixtures ────────────────────────────────────────────────────────── + +const NETWORK: &str = "hellas-devnet-1"; +const OTHER_NETWORK: &str = "hellas-devnet-22"; +const SALT: [u8; 32] = [0x5a; 32]; + +/// The pinned `request.nonce`. `build_input_events_with_retention` draws +/// this from `OsRng`; the golden digests below need it fixed. +const NONCE: [u8; 32] = [7; 32]; +const PROPOSAL_NONCE: u64 = 0x0102_0304_0506_0708; +const SERVICE: &str = "openai"; +const METHOD: &str = "responses"; +const REQUEST_BODY: &[u8] = br#"{"model":"gpt-5.2","input":"paid fetch"}"#; + +/// The channel's certificate capacity, from the kernel's own settlement +/// arithmetic over an edge that locks a million and reserves nothing. +fn capacity() -> WorkPaymentSettlement { + let Some(settlement) = work_payment_settlement( + EdgeValues::new(1_000_000 + payment_terms().omission_bond, 0, Fees::ZERO), + payment_terms().omission_bond, + ) else { + panic!("a funded edge prices both exits"); + }; + assert_eq!(settlement.capacity(), 1_000_000); + settlement +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn network() -> NetworkId { + NetworkId::new(NETWORK).expect("legal network id") +} + +/// The settlement signer the channel terms name as the client. +fn client() -> Secp256k1Signer { + Secp256k1Signer::from_secret_scalar([3; 32]).expect("legal scalar") +} + +/// The settlement signer the channel terms name as the provider. +fn provider() -> Secp256k1Signer { + Secp256k1Signer::from_secret_scalar([4; 32]).expect("legal scalar") +} + +/// The transcript signing key over the client's secret: a signed fetch +/// input is the client's request, so its key and the settlement key are +/// the one scalar in two wrappers. +fn caller_key() -> ProducerSigningKey { + ProducerSigningKey::from_secret_bytes([3; 32]).expect("legal scalar") +} + +/// The transcript signing key over the provider's secret. +fn producer_key() -> ProducerSigningKey { + ProducerSigningKey::from_secret_bytes([4; 32]).expect("legal scalar") +} + +fn channel_policy() -> PaidChannelPolicyV1 { + PaidChannelPolicyV1 { + compute_credit_limit: 700, + delivery_credit_limit: 800, + } +} + +fn payment_terms() -> WorkPaymentTerms { + let bond = WorkStakeBondTerms { + // Bond maker is the provider, taker is the client; the payment + // edge mirrors that, which is where `PaidChannel` reads its keys. + parties: Parties::new(provider().party_key(), client().party_key()), + timeout: BlockHeight::new(5_000), + timeout_outputs: List::take( + [ + Payout::new(provider().party_key(), 900), + Payout::default(), + Payout::default(), + Payout::default(), + ], + 1, + ), + max_job_price: 500, + }; + WorkPaymentTerms { + bond_edge: EdgeId::from_bytes([0xb0; 32]), + bond_terms: bond, + private_policy_commitment: hellas_rpc::protocol::work::private_policy_commitment( + network(), + &SALT, + &channel_policy(), + ), + omit_response_blocks: 32, + start_validity_blocks: 16, + omission_bond: 100, + } +} + +fn channel() -> PaidChannel { + PaidChannel::new( + network(), + EdgeId::from_bytes([0xe1; 32]), + payment_terms(), + &SALT, + channel_policy(), + ) + .expect("terms that commit to this credit policy") +} + +/// A channel on another network, or another payment edge. +fn channel_on(network: NetworkId, payment_edge: EdgeId) -> PaidChannel { + let terms = WorkPaymentTerms { + private_policy_commitment: hellas_rpc::protocol::work::private_policy_commitment( + network, + &SALT, + &channel_policy(), + ), + ..payment_terms() + }; + PaidChannel::new(network, payment_edge, terms, &SALT, channel_policy()) + .expect("terms that commit to this credit policy") +} + +fn environment() -> FetchEnvironment { + FetchEnvironment::OpenAiResponses +} + +fn manifest() -> ProgramManifest { + environment().manifest() +} + +fn route() -> FetchRoutePolicy { + FetchRoutePolicy::sealed_route(SERVICE, METHOD).expect("a legal route") +} + +fn route_commitment() -> Digest { + fetch_route_commitment(&route().canonical_body_bytes()).expect("a representable route body") +} + +fn fetch_policy() -> PaidFetchPolicyV1 { + PaidFetchPolicyV1 { + allowed_environment: environment().manifest_id(), + route_commitment: route_commitment(), + max_request_body_bytes: 4_096, + max_output_events: 64, + max_output_bytes: 1_048_576, + max_spool_bytes: 65_536, + max_encoded_result_frame: 262_144, + max_encoded_prepared_input: 1_048_576, + dispatch_margin_blocks: 20, + delivery_margin_blocks: 10, + oracle_grace_blocks: 30, + fixed_price: 250, + } +} + +fn deadlines() -> JobDeadlines { + JobDeadlines { + acceptance: 1_000, + terminal: 1_050, + payment: 1_100, + } +} + +/// Signs the eight fetch input events in the order +/// `build_input_events_with_retention` pushes them, with the nonce pinned. +fn signed_input( + key: &ProducerSigningKey, + assurance: Assurance, + environment: ContentId, + service: &str, + method: &str, + body: &[u8], +) -> Vec { + let mut builder = InputTranscriptBuilder::new( + scheme_id(Operation::Fetch, assurance), + key, + input_canonicalization(), + ); + builder + .push("assurance", vec![assurance.to_byte()]) + .unwrap(); + builder + .push("execution.environment", environment.as_bytes().to_vec()) + .unwrap(); + builder.push("request.nonce", NONCE.to_vec()).unwrap(); + builder + .push("service", service.as_bytes().to_vec()) + .unwrap(); + builder.push("method", method.as_bytes().to_vec()).unwrap(); + builder + .push( + "request.retain", + vec![u8::from(Retention::Retain.should_retain())], + ) + .unwrap(); + builder.push("request.body", body.to_vec()).unwrap(); + builder.push("input.end", Vec::new()).unwrap(); + builder.finish().unwrap().0 +} + +fn input_events() -> Vec { + signed_input( + &caller_key(), + Assurance::ProducerSigned, + environment().manifest_id(), + SERVICE, + METHOD, + REQUEST_BODY, + ) +} + +/// The request commitment exactly as the fetch ticket flow reads it: the +/// verified input transcript's commitment, re-wrapped. +fn input_request_commitment() -> RequestCommitment { + let input = hellas_rpc::fetch::verify_input_events(&input_events()) + .expect("the fixture transcript verifies"); + RequestCommitment::from_digest(input.input_commitment.digest()) +} + +fn bundle() -> PreparedPaidFetchInputV1 { + PreparedPaidFetchInputV1::new(&input_events(), &manifest()).expect("a legal transcript") +} + +fn propose( + channel: &PaidChannel, + policy: &PaidFetchPolicyV1, + bundle: &PreparedPaidFetchInputV1, +) -> PaidJobAuthorizationV1 { + propose_fetch_authorization(channel, policy, bundle, PROPOSAL_NONCE, deadlines()) + .expect("a legal proposal") +} + +fn authorization() -> PaidJobAuthorizationV1 { + propose(&channel(), &fetch_policy(), &bundle()) +} + +/// Assembles bundle bytes from two arbitrary bodies, so a test can build +/// a spelling the constructor would never produce. +fn assemble(bodies: &[&[u8]]) -> Vec { + let mut bytes = Vec::new(); + for body in bodies { + bytes.extend_from_slice(&(body.len() as u32).to_be_bytes()); + bytes.extend_from_slice(body); + } + bytes +} + +fn input_digest(channel: &PaidChannel, bundle: &PreparedPaidFetchInputV1) -> Digest { + prepared_fetch_input_digest(channel, bundle).expect("a representable bundle") +} + +/// One semantic output event payload, encoded with the fetch profile's +/// own payload codec: the paid layer treats payloads as opaque, but the +/// fixture answers are real ones. +fn event_payload(delta: &str) -> Vec { + encode_fetch_event_payload(&OutputEvent::TextDelta { + index: 0, + delta: delta.to_string(), + channel: TextChannel::Output, + }) + .expect("a legal event payload") +} + +fn terminal_payload() -> Vec { + encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: Some(Usage { + input_tokens: Some(12), + output_tokens: Some(2), + total_tokens: Some(14), + }), + }) + .expect("a legal terminal payload") +} + +/// The provider-signed output transcript answering the authorization's +/// request: two semantic events and a terminal. +fn output_transcript(authorization: &PaidJobAuthorizationV1) -> Vec { + let input = InputCommitment::from_digest(authorization.request_commitment.digest()); + let key = producer_key(); + let mut builder = FetchOutputTranscriptBuilder::new(input, Assurance::ProducerSigned, &key); + builder.push_event(event_payload("paid ")).unwrap(); + builder.push_event(event_payload("fetch")).unwrap(); + builder.finish(terminal_payload()).unwrap() +} + +// ── Golden encodings ────────────────────────────────────────────────── + +/// The fetch policy record's exact bytes, decoded field by field. +#[test] +fn golden_fetch_policy_encoding_is_pinned() { + assert_eq!(PaidFetchPolicyV1::BODY_SIZE, 124); + assert_eq!(PaidFetchPolicyV1::ENCODED_SIZE, 126); + + let policy = PaidFetchPolicyV1 { + allowed_environment: ContentId::from_bytes([0x40; 32]), + route_commitment: Digest::from_bytes([0x41; 32]), + max_request_body_bytes: 1, + max_output_events: 2, + max_output_bytes: 3, + max_spool_bytes: 4, + max_encoded_result_frame: 5, + max_encoded_prepared_input: 6, + dispatch_margin_blocks: 7, + delivery_margin_blocks: 8, + oracle_grace_blocks: 9, + fixed_price: 10, + }; + assert_eq!( + hex(&policy.encode()), + concat!( + "01", + "05", // format version 1, tag 5 = PAID_FETCH_POLICY + "4040404040404040404040404040404040404040404040404040404040404040", // environment + "4141414141414141414141414141414141414141414141414141414141414141", // route + "00000001", // max_request_body_bytes + "00000002", // max_output_events + "00000003", // max_output_bytes + "0000000000000004", // max_spool_bytes + "00000005", // max_encoded_result_frame + "00000006", // max_encoded_prepared_input + "0000000000000007", // dispatch_margin_blocks + "0000000000000008", // delivery_margin_blocks + "0000000000000009", // oracle_grace_blocks + "000000000000000a", // fixed_price + ) + ); + assert_eq!(PaidFetchPolicyV1::decode(&policy.encode()), Ok(policy)); +} + +/// Wrong tag, wrong version, truncation, and a trailing byte all reject, +/// on the new record. +#[test] +fn fetch_policy_envelope_mutations_reject() { + let bytes = fetch_policy().encode(); + assert!(PaidFetchPolicyV1::decode(&bytes).is_ok()); + + // MUTATION: the fetch policy presented under the evaluate policy tag. + // Both are policies; only the envelope says which profile they are. + let mut evaluate_tag = bytes.clone(); + evaluate_tag[1] = 1; + assert_eq!( + PaidFetchPolicyV1::decode(&evaluate_tag), + Err(PaidWorkError::WrongRecordTag { + expected: 5, + actual: 1 + }) + ); + + // MUTATION: a future format version. + let mut wrong_version = bytes.clone(); + wrong_version[0] = 2; + assert_eq!( + PaidFetchPolicyV1::decode(&wrong_version), + Err(PaidWorkError::UnknownFormatVersion { actual: 2 }) + ); + + // MUTATION: one byte short. + let truncated = &bytes[..bytes.len() - 1]; + assert_eq!( + PaidFetchPolicyV1::decode(truncated), + Err(PaidWorkError::RecordLength { + expected: 126, + actual: 125 + }) + ); + + // MUTATION: one byte long. + let mut trailing = bytes.clone(); + trailing.push(0); + assert_eq!( + PaidFetchPolicyV1::decode(&trailing), + Err(PaidWorkError::RecordLength { + expected: 126, + actual: 127 + }) + ); +} + +/// The two route-policy bodies, hand-decoded. +#[test] +fn golden_route_body_encodings_are_pinned() { + // ["hellas.work.fetch-route.sealed.v1", "openai", "responses"]: a + // three-element array, the 33-byte schema, the two route components. + let sealed = route().canonical_body_bytes(); + assert_eq!( + hex(&sealed), + concat!( + "83", // array(3) + "7821", // str(33) + "68656c6c61732e776f726b2e66657463682d726f7574652e7365616c65642e7631", + "66", // str(6) + "6f70656e6169", // "openai" + "69", // str(9) + "726573706f6e736573", // "responses" + ) + ); + assert_eq!( + FetchRoutePolicy::from_canonical_body_bytes(&sealed), + Ok(route()) + ); + + // ["hellas.work.fetch-route.open.v1", 1, ["api.openai.com"]]: the pin + // as the integer 1, and one host. + let open = FetchRoutePolicy::open_fetch(true, vec!["api.openai.com".to_string()]); + let open_bytes = open.canonical_body_bytes(); + assert_eq!( + hex(&open_bytes), + concat!( + "83", // array(3) + "781f", // str(31) + "68656c6c61732e776f726b2e66657463682d726f7574652e6f70656e2e7631", + "01", // require_spki_pin = true + "81", // array(1) + "6e", // str(14) + "6170692e6f70656e61692e636f6d", // "api.openai.com" + ) + ); + assert_eq!( + FetchRoutePolicy::from_canonical_body_bytes(&open_bytes), + Ok(open) + ); +} + +/// The route codec's refusals: unknown schema, wrong shape, a pin that is +/// not 0 or 1, a trailing byte, a noncanonical spelling, and route +/// components the transcript scheme would not admit. +#[test] +fn route_policy_refusals() { + let sealed = route().canonical_body_bytes(); + + // MUTATION: an unknown schema tag. + let unknown_schema = { + let mut body = Vec::new(); + body.push(0x83); + body.extend_from_slice(&[0x78, 0x21]); + body.extend_from_slice(b"hellas.work.fetch-route.sealed.v9"); + body.extend_from_slice(&sealed[sealed.len() - 17..]); + body + }; + assert!(unknown_schema.len() == sealed.len()); + assert!( + FetchRoutePolicy::from_canonical_body_bytes(&unknown_schema) + .expect_err("an unknown schema must not parse") + .to_string() + .contains("unexpected fetch route schema tag") + ); + + // MUTATION: the array shortened to two elements. + let mut short = sealed.clone(); + short[0] = 0x82; + assert!( + FetchRoutePolicy::from_canonical_body_bytes(&short) + .expect_err("a shortened array must not parse") + .to_string() + .contains("expected array length 3, got 2") + ); + + // MUTATION: a trailing byte. + let mut trailing = sealed.clone(); + trailing.push(0); + assert!( + FetchRoutePolicy::from_canonical_body_bytes(&trailing) + .expect_err("a trailing byte must not parse") + .to_string() + .contains("trailing bytes") + ); + + // MUTATION: the pin spelled as the integer 2. + let mut bad_pin = + FetchRoutePolicy::open_fetch(true, Vec::::new()).canonical_body_bytes(); + let pin = bad_pin.len() - 2; // [schema, pin, empty array]: pin is third from last + assert_eq!(bad_pin[pin], 1); + bad_pin[pin] = 2; + assert!( + FetchRoutePolicy::from_canonical_body_bytes(&bad_pin) + .expect_err("a pin of 2 must not parse") + .to_string() + .contains("require_spki_pin must be 0 or 1") + ); + + // MUTATION: an empty service in the constructor and in a body. + assert!(matches!( + FetchRoutePolicy::sealed_route("", METHOD), + Err(hellas_rpc::fetch::FetchProtocolError::EmptyService) + )); + assert!(matches!( + FetchRoutePolicy::sealed_route(SERVICE, ""), + Err(hellas_rpc::fetch::FetchProtocolError::EmptyMethod) + )); + let overlong = "x".repeat(hellas_rpc::fetch::MAX_FETCH_ROUTE_COMPONENT_BYTES + 1); + assert!(matches!( + FetchRoutePolicy::sealed_route(&overlong, METHOD), + Err(hellas_rpc::fetch::FetchProtocolError::RouteComponentLimit { + field: "service", + .. + }) + )); + let mut empty_service = sealed.clone(); + let service_at = empty_service.len() - "openai".len() - "responses".len() - 2; + empty_service[service_at] = 0x60; // str(0) + let mut shortened = empty_service[..service_at + 1].to_vec(); + shortened.extend_from_slice(&empty_service[service_at + 1 + "openai".len()..]); + assert!( + FetchRoutePolicy::from_canonical_body_bytes(&shortened) + .expect_err("an empty service must not parse") + .to_string() + .contains("fetch service must not be empty") + ); + + // The allowlist is a set: the constructor sorts and deduplicates, so + // the canonical body of a shuffled, doubled list is the same bytes. + let shuffled = FetchRoutePolicy::open_fetch( + false, + vec![ + "b.example".to_string(), + "a.example".to_string(), + "b.example".to_string(), + ], + ); + let sorted = FetchRoutePolicy::open_fetch( + false, + vec!["a.example".to_string(), "b.example".to_string()], + ); + assert_eq!(shuffled, sorted); +} + +// ── Golden digests ──────────────────────────────────────────────────── + +/// The profile's pinned digests, over the deterministic fixture. +#[test] +fn golden_fetch_digests_are_pinned() { + // XFH("hellas.work.fetch-route.v1" || u32be(len) || route body), both + // variants. + assert_eq!( + hex(route_commitment().as_bytes()), + "a2e44135e61c07424562ab53dfe8b74d18ddd6872d39a59d9206485d4ac41530" + ); + let open = FetchRoutePolicy::open_fetch(true, vec!["api.openai.com".to_string()]); + assert_eq!( + hex(fetch_route_commitment(&open.canonical_body_bytes()) + .expect("a representable route body") + .as_bytes()), + "55e7ccf1928e21128357726422c637ddc5cf0437dde10568f0f2f83e0d69c299" + ); + let channel = channel(); + // XH("hellas.work.fetch-policy.v1" || network || channel || record). + assert_eq!( + hex(fetch_policy_digest(&channel, &fetch_policy()).as_bytes()), + "6a675a75bd5b40e8fad1e35272f1c971a1a9cc34a8b756ab5298cc747ff55ede" + ); + // XFH("hellas.work.prepared-fetch-input.v1" || network || channel || + // bundle). + assert_eq!( + hex(input_digest(&channel, &bundle()).as_bytes()), + "8bb0796694ed05a0e32fe2f40d32b16256bc802fbe073a8a5a06e6c59e58c3a9" + ); + // XH("hellas.work.paid-job-authorize.v1" || network || channel || + // authorization): the shared record, signed under this profile. + assert_eq!( + hex(work_id(&channel, &authorization()).as_bytes()), + "0258caba196b7a4c13c1ad8ea8f575ea9b943f58e80d73c17a3d41d534de0f30" + ); +} + +/// The same bodies in another channel are other digests, and the check +/// that names the channel refuses them. +#[test] +fn fetch_records_do_not_cross_channels() { + let here = channel(); + let sibling = channel_on(network(), EdgeId::from_bytes([0xe2; 32])); + let elsewhere = channel_on( + NetworkId::new(OTHER_NETWORK).expect("legal network id"), + here.payment_edge(), + ); + let authorization = authorization(); + + assert_ne!(here.id(), sibling.id()); + for (label, other) in [("sibling", &sibling), ("elsewhere", &elsewhere)] { + assert_ne!( + fetch_policy_digest(&here, &fetch_policy()), + fetch_policy_digest(other, &fetch_policy()), + "{label} shares this channel's fetch policy digest" + ); + assert_ne!( + input_digest(&here, &bundle()), + input_digest(other, &bundle()), + "{label} shares this channel's prepared fetch input digest" + ); + // MUTATION: replay this channel's fetch authorization on another + // one. + assert_eq!( + check_fetch_authorization(other, &authorization, &fetch_policy(), 900), + Err(PaidWorkError::Mismatch { + field: "channel_id" + }), + "{label} accepted a foreign authorization" + ); + } +} + +// ── Prepared fetch input bundle ─────────────────────────────────────── + +/// The bundle encodes as two big-endian lengths and two bodies, and its +/// digest is reproducible from the same two component bodies. +#[test] +fn prepared_fetch_input_is_reproducible_from_its_components() { + let bundle = bundle(); + let encoded = bundle.encode().expect("a representable bundle"); + + // The bundle body is the canonical DAG-CBOR of the signed events; + // rebuilding it from the decoded events must give the same bytes. + let transcript = bundle.parts().expect("the canonical bodies parse"); + let reencoded = hellas_rpc::canonical_dag_cbor(&transcript.fetch_input_transcript) + .expect("the decoded events re-encode"); + let manifest_bytes = manifest().canonical_bytes(); + let expected = assemble(&[&reencoded, &manifest_bytes]); + assert_eq!(encoded, expected); + + let decoded = PreparedPaidFetchInputV1::decode(&encoded, 1_048_576).expect("legal bundle"); + assert_eq!(decoded, bundle); + assert_eq!( + decoded + .parts() + .expect("the canonical bodies parse") + .manifest + .canonical_bytes(), + manifest_bytes + ); + assert_eq!( + input_digest(&channel(), &decoded), + input_digest(&channel(), &bundle) + ); +} + +/// Padding, shortening, or re-labelling the bundle rejects rather than +/// producing a second acceptable spelling of the same job. +#[test] +fn prepared_fetch_input_mutations_reject() { + let encoded = bundle().encode().expect("a representable bundle"); + let budget = 1_048_576; + assert!(PreparedPaidFetchInputV1::decode(&encoded, budget).is_ok()); + + // MUTATION: one appended byte. + let mut trailing = encoded.clone(); + trailing.push(0); + let err = PreparedPaidFetchInputV1::decode(&trailing, budget) + .expect_err("a trailing byte must not decode"); + assert!( + err.to_string() + .contains("trailing bytes after prepared fetch input"), + "{err}" + ); + + // MUTATION: the first declared length one byte short. The manifest's + // length is then read out of the transcript's tail and refused before + // anything is allocated for it. + let mut short_length = encoded.clone(); + let first_len = u32::from_be_bytes([ + short_length[0], + short_length[1], + short_length[2], + short_length[3], + ]); + short_length[..4].copy_from_slice(&(first_len - 1).to_be_bytes()); + let err = PreparedPaidFetchInputV1::decode(&short_length, budget) + .expect_err("a shortened length must not decode"); + assert!(err.to_string().contains("budget"), "{err}"); + + // MUTATION: u32::MAX in the first length. + let mut huge = encoded.clone(); + huge[..4].copy_from_slice(&u32::MAX.to_be_bytes()); + let err = PreparedPaidFetchInputV1::decode(&huge, budget) + .expect_err("a u32::MAX length must not decode"); + assert!(err.to_string().contains("budget"), "{err}"); + + // MUTATION: a bundle one byte over its budget; the exact budget is + // legal. + let err = PreparedPaidFetchInputV1::decode(&encoded, encoded.len() - 1) + .expect_err("a bundle over its budget must not decode"); + assert!(err.to_string().contains("budget"), "{err}"); + assert!(PreparedPaidFetchInputV1::decode(&encoded, encoded.len()).is_ok()); + + // MUTATION: the two bodies swapped. The transcript is not a manifest. + let transcript_bytes = &encoded[4..4 + (first_len as usize)]; + let swapped = assemble(&[&manifest().canonical_bytes(), transcript_bytes]); + let err = PreparedPaidFetchInputV1::decode(&swapped, budget) + .expect("lengths are still well formed") + .parts() + .expect_err("swapped components must not parse"); + assert!(!err.to_string().is_empty(), "{err}"); + + // MUTATION: a noncanonical transcript body — the DAG-CBOR array + // header re-spelled with a wider integer width. + let mut widened = transcript_bytes.to_vec(); + assert_eq!(widened[0], 0x88, "the transcript is an eight-element array"); + widened.splice(0..1, [0x98, 0x08]); + let noncanonical = assemble(&[&widened, &manifest().canonical_bytes()]); + let err = PreparedPaidFetchInputV1::decode(&noncanonical, budget) + .expect("lengths are still well formed") + .parts() + .expect_err("a noncanonical transcript must not parse"); + assert!(err.to_string().contains("canonical"), "{err}"); + + // MUTATION: a noncanonical manifest body — trailing garbage inside + // the second segment. + let mut noncanonical_manifest = manifest().canonical_bytes(); + noncanonical_manifest.push(0); + let noncanonical = assemble(&[transcript_bytes, &noncanonical_manifest]); + let err = PreparedPaidFetchInputV1::decode(&noncanonical, budget) + .expect("lengths are still well formed") + .parts() + .expect_err("a noncanonical manifest body must not parse"); + assert!(err.to_string().contains("trailing bytes"), "{err}"); +} + +/// Rebuilds the fetch policy preimage by hand and hashes it through the +/// crate's *other* hashing entry point. +/// +/// Same discipline as `digest_preimages_are_reproducible_by_hand` in +/// `paid_work_vectors.rs`: if the domain, the network's length prefix, +/// the channel argument, or the envelope-plus-body order disagreed with +/// the module's, the two would not meet here. +#[test] +fn fetch_policy_preimage_is_reproducible_by_hand() { + let channel = channel(); + + let mut preimage = b"hellas.work.fetch-policy.v1".to_vec(); + preimage.push(NETWORK.len() as u8); + preimage.extend_from_slice(NETWORK.as_bytes()); + preimage.extend_from_slice(channel.id().as_bytes()); + preimage.extend_from_slice(&fetch_policy().encode()); + assert_eq!( + Digest::hash(&preimage), + fetch_policy_digest(&channel, &fetch_policy()) + ); + + // The route commitment: domain, u32 length prefix, body. + let route_body = route().canonical_body_bytes(); + let mut preimage = b"hellas.work.fetch-route.v1".to_vec(); + preimage.extend_from_slice(&(route_body.len() as u32).to_be_bytes()); + preimage.extend_from_slice(&route_body); + assert_eq!(Digest::hash(&preimage), route_commitment()); + + // The prepared input: domain, network, channel, bundle. + let mut preimage = b"hellas.work.prepared-fetch-input.v1".to_vec(); + preimage.push(NETWORK.len() as u8); + preimage.extend_from_slice(NETWORK.as_bytes()); + preimage.extend_from_slice(channel.id().as_bytes()); + preimage.extend_from_slice(&bundle().encode().expect("a representable bundle")); + assert_eq!(Digest::hash(&preimage), input_digest(&channel, &bundle())); +} + +// ── Authorization ───────────────────────────────────────────────────── + +/// The proposal derives every field the channel, policy, and bundle fix, +/// and the request commitment is the fetch ticket flow's: the verified +/// input transcript's own commitment. +#[test] +fn propose_fetch_authorization_derives_every_field() { + let channel = channel(); + let terms = payment_terms(); + let authorization = authorization(); + + assert_eq!(authorization.channel_id, channel.id()); + assert_eq!(authorization.bond_edge, terms.bond_edge); + assert_eq!(authorization.bond_terms_hash, terms.bond_terms_hash()); + assert_eq!(authorization.payment_edge, channel.payment_edge()); + assert_eq!( + authorization.payment_terms_hash, + channel.payment_terms_hash() + ); + assert_eq!( + authorization.execution_policy_digest, + fetch_policy_digest(&channel, &fetch_policy()) + ); + assert_eq!( + authorization.prepared_input_digest, + input_digest(&channel, &bundle()) + ); + assert_eq!(authorization.proposal_nonce, PROPOSAL_NONCE); + assert_eq!(authorization.acceptance_deadline, 1_000); + assert_eq!(authorization.request_commitment, input_request_commitment()); + assert_eq!( + authorization.environment_commitment, + environment().manifest_id() + ); + assert_eq!(authorization.price, 250); + assert_eq!(authorization.terminal_deadline, 1_050); + assert_eq!(authorization.payment_deadline, 1_100); + + // The signing bytes both parties produce are the authorization's + // `work_id`, and both settlement keys verify over them. + let id = check_fetch_authorization(&channel, &authorization, &fetch_policy(), 900) + .expect("a legal authorization"); + assert_eq!(id, work_id(&channel, &authorization)); + let payload = signing_hash(id); + let verifier = Secp256k1Verifier; + assert!(verifier.verify_sig(client().sign(payload), channel.client_key(), payload)); + assert!(verifier.verify_sig(provider().sign(payload), channel.provider_key(), payload)); +} + +/// The fetch authorization refusals: the wiring of the shared core to +/// this profile's policy, plus the profile's own zero-bound rule. +#[test] +fn check_fetch_authorization_refusals() { + let channel = channel(); + let policy = fetch_policy(); + let base = authorization(); + assert!(check_fetch_authorization(&channel, &base, &policy, 900).is_ok()); + + // MUTATION: a policy whose digest is not the one the authorization + // names. + let mut other_policy = policy; + other_policy.max_output_events += 1; + assert_eq!( + check_fetch_authorization(&channel, &base, &other_policy, 900), + Err(PaidWorkError::Mismatch { + field: "execution_policy_digest" + }) + ); + + // MUTATION: a zero bound. The policy check fires before any + // comparison. + let mut zeroed_policy = policy; + zeroed_policy.max_output_bytes = 0; + let zeroed = propose(&channel, &zeroed_policy, &bundle()); + assert_eq!( + check_fetch_authorization(&channel, &zeroed, &zeroed_policy, 900), + Err(PaidWorkError::PolicyZero { + field: "max_output_bytes" + }) + ); + + // MUTATION: a price that disagrees with the signed policy. + let mut mispriced = base; + mispriced.price = 249; + assert_eq!( + check_fetch_authorization(&channel, &mispriced, &policy, 900), + Err(PaidWorkError::Mismatch { field: "price" }) + ); + + // MUTATION: a price the bond does not cover, honestly proposed under + // a policy that names it. + let mut dear_policy = policy; + dear_policy.fixed_price = 501; + let dear = propose(&channel, &dear_policy, &bundle()); + assert_eq!( + check_fetch_authorization(&channel, &dear, &dear_policy, 900), + Err(PaidWorkError::PriceOutOfRange { + price: 501, + max_job_price: 500 + }) + ); + + // MUTATION: signing after the acceptance window closed. + assert_eq!( + check_fetch_authorization(&channel, &base, &policy, 1_001), + Err(PaidWorkError::AcceptanceExpired { + height: 1_001, + deadline: 1_000 + }) + ); + + // MUTATION: a payment deadline past the channel's admission horizon. + let mut late = base; + late.payment_deadline = 5_000; + assert_eq!( + check_fetch_authorization(&channel, &late, &policy, 900), + Err(PaidWorkError::DeadlineOrder { + acceptance: 1_000, + terminal: 1_050, + payment: 5_000, + horizon: 5_000 + }) + ); +} + +/// Every bound the profile requires to be positive is refused at zero, +/// one field at a time. +#[test] +fn an_absent_fetch_policy_bound_is_refused() { + let base = fetch_policy(); + assert!(check_fetch_policy(&base).is_ok()); + + let zeroed: Vec<(&str, PaidFetchPolicyV1)> = vec![ + ( + "fixed_price", + PaidFetchPolicyV1 { + fixed_price: 0, + ..base + }, + ), + ( + "max_request_body_bytes", + PaidFetchPolicyV1 { + max_request_body_bytes: 0, + ..base + }, + ), + ( + "max_output_events", + PaidFetchPolicyV1 { + max_output_events: 0, + ..base + }, + ), + ( + "max_output_bytes", + PaidFetchPolicyV1 { + max_output_bytes: 0, + ..base + }, + ), + ( + "max_spool_bytes", + PaidFetchPolicyV1 { + max_spool_bytes: 0, + ..base + }, + ), + ( + "max_encoded_result_frame", + PaidFetchPolicyV1 { + max_encoded_result_frame: 0, + ..base + }, + ), + ( + "max_encoded_prepared_input", + PaidFetchPolicyV1 { + max_encoded_prepared_input: 0, + ..base + }, + ), + ( + "dispatch_margin_blocks", + PaidFetchPolicyV1 { + dispatch_margin_blocks: 0, + ..base + }, + ), + ( + "delivery_margin_blocks", + PaidFetchPolicyV1 { + delivery_margin_blocks: 0, + ..base + }, + ), + ( + "oracle_grace_blocks", + PaidFetchPolicyV1 { + oracle_grace_blocks: 0, + ..base + }, + ), + ]; + assert_eq!(zeroed.len(), 10, "every required bound must be zeroed"); + for (field, policy) in zeroed { + assert_eq!( + check_fetch_policy(&policy), + Err(PaidWorkError::PolicyZero { field }), + "{field} was accepted at zero" + ); + } +} + +/// Every fetch-policy field is inside the digest the authorization pins. +#[test] +fn every_fetch_policy_field_moves_its_digest() { + let channel = channel(); + let base = fetch_policy(); + let pinned = fetch_policy_digest(&channel, &base); + + let mutations: Vec<(&str, PaidFetchPolicyV1)> = vec![ + ("allowed_environment", { + let mut m = base; + m.allowed_environment = ContentId::from_bytes([0; 32]); + m + }), + ("route_commitment", { + let mut m = base; + m.route_commitment = Digest::from_bytes([0; 32]); + m + }), + ("max_request_body_bytes", { + let mut m = base; + m.max_request_body_bytes += 1; + m + }), + ("max_output_events", { + let mut m = base; + m.max_output_events += 1; + m + }), + ("max_output_bytes", { + let mut m = base; + m.max_output_bytes += 1; + m + }), + ("max_spool_bytes", { + let mut m = base; + m.max_spool_bytes += 1; + m + }), + ("max_encoded_result_frame", { + let mut m = base; + m.max_encoded_result_frame += 1; + m + }), + ("max_encoded_prepared_input", { + let mut m = base; + m.max_encoded_prepared_input += 1; + m + }), + ("dispatch_margin_blocks", { + let mut m = base; + m.dispatch_margin_blocks += 1; + m + }), + ("delivery_margin_blocks", { + let mut m = base; + m.delivery_margin_blocks += 1; + m + }), + ("oracle_grace_blocks", { + let mut m = base; + m.oracle_grace_blocks += 1; + m + }), + ("fixed_price", { + let mut m = base; + m.fixed_price += 1; + m + }), + ]; + assert_eq!(mutations.len(), 12, "every field must be mutated"); + for (field, mutated) in mutations { + assert_ne!( + fetch_policy_digest(&channel, &mutated), + pinned, + "{field} left the policy digest alone" + ); + } +} + +// ── Prepared input checks ───────────────────────────────────────────── + +/// The happy path, and the key correspondence it stands on: the +/// transcript's caller key and the channel's client settlement key are +/// the one scalar. +#[test] +fn check_prepared_fetch_input_happy_path() { + let channel = channel(); + assert_eq!( + caller_key().public_key(), + PublicKey::Secp256k1(channel.client_key().to_bytes()), + "the fixture caller must be the channel's client" + ); + assert_eq!( + producer_key().public_key(), + PublicKey::Secp256k1(channel.provider_key().to_bytes()), + "the fixture producer must be the channel's provider" + ); + check_prepared_fetch_input( + &channel, + &authorization(), + &fetch_policy(), + &route(), + &bundle(), + ) + .expect("a legal prepared fetch input"); +} + +/// Each binding the check makes is broken on its own, with the bundle +/// digest and the authorization recomputed so the named check is the +/// only thing that can refuse it. +#[test] +fn each_fetch_input_binding_is_checked_on_its_own() { + let channel = channel(); + let policy = fetch_policy(); + let base = authorization(); + assert!(check_prepared_fetch_input(&channel, &base, &policy, &route(), &bundle()).is_ok()); + + // A bundle whose digest is not the one the authorization named. + let mut unbound = base; + unbound.prepared_input_digest = Digest::from_bytes([0x84; 32]); + assert_eq!( + check_prepared_fetch_input(&channel, &unbound, &policy, &route(), &bundle()), + Err(PaidWorkError::Mismatch { + field: "prepared_input_digest" + }) + ); + + // A bundle larger than the policy's prepared-input bound. Refused + // before its digest is even computed. + let encoded = bundle().encode().expect("a representable bundle"); + let cramped = PaidFetchPolicyV1 { + max_encoded_prepared_input: 100, + ..policy + }; + assert_eq!( + check_prepared_fetch_input(&channel, &base, &cramped, &route(), &bundle()), + Err(PaidWorkError::OverEnvelope { + field: "prepared input length", + actual: encoded.len() as u64, + limit: 100 + }) + ); + // The exact length is legal; one byte less is not. + let exact = PaidFetchPolicyV1 { + max_encoded_prepared_input: encoded.len() as u32, + ..policy + }; + assert!(check_prepared_fetch_input(&channel, &base, &exact, &route(), &bundle()).is_ok()); + + // Input events signed by the provider's key rather than the client's: + // a self-consistent chain whose caller is not this channel's client. + let delegated_events = signed_input( + &producer_key(), + Assurance::ProducerSigned, + environment().manifest_id(), + SERVICE, + METHOD, + REQUEST_BODY, + ); + let delegated_bundle = + PreparedPaidFetchInputV1::new(&delegated_events, &manifest()).expect("a legal transcript"); + let delegated = propose(&channel, &policy, &delegated_bundle); + assert_eq!( + check_prepared_fetch_input(&channel, &delegated, &policy, &route(), &delegated_bundle), + Err(PaidWorkError::Mismatch { + field: "caller_key" + }) + ); + + // Another assurance mode under an otherwise legal bundle. + let attested_events = signed_input( + &caller_key(), + Assurance::AppleAppAttest, + environment().manifest_id(), + SERVICE, + METHOD, + REQUEST_BODY, + ); + let attested_bundle = + PreparedPaidFetchInputV1::new(&attested_events, &manifest()).expect("a legal transcript"); + let attested = propose(&channel, &policy, &attested_bundle); + assert_eq!( + check_prepared_fetch_input(&channel, &attested, &policy, &route(), &attested_bundle), + Ok(()) + ); + + // A route body the policy does not commit to: the commitment check + // fires before the route may constrain anything. + let mut rerouted_policy = policy; + rerouted_policy.route_commitment = Digest::from_bytes([0x86; 32]); + assert_eq!( + check_prepared_fetch_input(&channel, &base, &rerouted_policy, &route(), &bundle()), + Err(PaidWorkError::Mismatch { + field: "route_commitment" + }) + ); + + // A sealed route whose service the signed events do not name, with + // the policy's commitment honestly recomputed over it. + let other_service = FetchRoutePolicy::sealed_route("anthropic", METHOD).expect("a legal route"); + let mut resold_policy = policy; + resold_policy.route_commitment = + fetch_route_commitment(&other_service.canonical_body_bytes()).expect("representable"); + assert_eq!( + check_prepared_fetch_input(&channel, &base, &resold_policy, &other_service, &bundle()), + Err(PaidWorkError::Mismatch { field: "service" }) + ); + + // The method, on its own. + let other_method = FetchRoutePolicy::sealed_route(SERVICE, "chat").expect("a legal route"); + let mut resold_policy = policy; + resold_policy.route_commitment = + fetch_route_commitment(&other_method.canonical_body_bytes()).expect("representable"); + assert_eq!( + check_prepared_fetch_input(&channel, &base, &resold_policy, &other_method, &bundle()), + Err(PaidWorkError::Mismatch { field: "method" }) + ); + + // Events that name another environment than the bundle's manifest. + let elsewhere_events = signed_input( + &caller_key(), + Assurance::ProducerSigned, + ContentId::from_bytes([0x99; 32]), + SERVICE, + METHOD, + REQUEST_BODY, + ); + let elsewhere_bundle = + PreparedPaidFetchInputV1::new(&elsewhere_events, &manifest()).expect("a legal transcript"); + let elsewhere = propose(&channel, &policy, &elsewhere_bundle); + assert_eq!( + check_prepared_fetch_input(&channel, &elsewhere, &policy, &route(), &elsewhere_bundle), + Err(PaidWorkError::Mismatch { + field: "manifest content id" + }) + ); + + // An authorization whose environment commitment is not the events'. + let mut reenvironed = base; + reenvironed.environment_commitment = ContentId::from_bytes([0x87; 32]); + assert_eq!( + check_prepared_fetch_input(&channel, &reenvironed, &policy, &route(), &bundle()), + Err(PaidWorkError::Mismatch { + field: "environment_commitment" + }) + ); + + // A policy whose allowed environment is not the manifest's. + let mut rehomed_policy = policy; + rehomed_policy.allowed_environment = ContentId::from_bytes([0x85; 32]); + assert_eq!( + check_prepared_fetch_input(&channel, &base, &rehomed_policy, &route(), &bundle()), + Err(PaidWorkError::Mismatch { + field: "allowed_environment" + }) + ); + + // A request commitment the authorization does not carry. + let mut restamped = base; + restamped.request_commitment = RequestCommitment::from_digest(Digest::from_bytes([0x88; 32])); + assert_eq!( + check_prepared_fetch_input(&channel, &restamped, &policy, &route(), &bundle()), + Err(PaidWorkError::Mismatch { + field: "request_commitment" + }) + ); + + // A request body over the policy's bound. The fetch protocol's own + // 1 MiB bound still holds; the policy's tighter one is what refuses + // here. + let tight = PaidFetchPolicyV1 { + max_request_body_bytes: 2, + ..policy + }; + assert_eq!( + check_prepared_fetch_input(&channel, &base, &tight, &route(), &bundle()), + Err(PaidWorkError::OverEnvelope { + field: "request body", + actual: REQUEST_BODY.len() as u64, + limit: 2 + }) + ); +} + +/// Open Fetch requires its HTTPS interpreter and enforces the signed host/pin contract. +#[test] +fn open_fetch_checks_manifest_host_and_tls_pins() { + let channel = channel(); + let route = FetchRoutePolicy::open_fetch(true, vec!["api.example.com".into()]); + let environment = hellas_rpc::FetchEnvironment::Http; + let mut policy = fetch_policy(); + policy.allowed_environment = environment.manifest_id(); + policy.route_commitment = fetch_route_commitment(&route.canonical_body_bytes()).unwrap(); + let make = |host: &str, pins: Vec| { + let body = serde_json::to_vec(&serde_json::json!({ + "url": format!("https://{host}/v1/messages"), "method":"POST", "body_base64":"e30=", + "tls":{"roots":{"mode":"web_pki"},"spki_sha256":pins}, "max_response_bytes":4096 + })) + .unwrap(); + let events = signed_input( + &caller_key(), + Assurance::AppleAppAttest, + environment.manifest_id(), + SERVICE, + METHOD, + &body, + ); + PreparedPaidFetchInputV1::new(&events, &environment.manifest()).unwrap() + }; + for (host, pins, valid) in [ + ("api.example.com", vec!["01".repeat(32)], true), + ("other.example", vec!["01".repeat(32)], false), + ("api.example.com", vec![], false), + ] { + let bundle = make(host, pins); + let auth = propose(&channel, &policy, &bundle); + assert_eq!( + check_prepared_fetch_input(&channel, &auth, &policy, &route, &bundle).is_ok(), + valid + ); + } + let sealed = bundle(); + let mut old = fetch_policy(); + old.route_commitment = policy.route_commitment; + assert!( + check_prepared_fetch_input( + &channel, + &propose(&channel, &old, &sealed), + &old, + &route, + &sealed + ) + .is_err() + ); +} + +#[test] +fn paid_fetch_result_cannot_downgrade_the_signed_assurance() { + use hellas_rpc::protocol::work_profile::PreparedPaidWorkInput; + let channel = channel(); + let input = signed_input( + &caller_key(), + Assurance::AppleAppAttest, + environment().manifest_id(), + SERVICE, + METHOD, + REQUEST_BODY, + ); + let bundle = PreparedPaidFetchInputV1::new(&input, &manifest()).unwrap(); + let auth = propose(&channel, &fetch_policy(), &bundle); + let prepared = PreparedPaidWorkInput::Fetch(bundle); + let weaker = output_transcript(&auth); + assert!(prepared.terminal_result(&channel, &auth, &weaker).is_err()); + let key = producer_key(); + let builder = FetchOutputTranscriptBuilder::new( + InputCommitment::from_digest(auth.request_commitment.digest()), + Assurance::AppleAppAttest, + &key, + ); + let output = builder + .finish(weaker.last().unwrap().payload().to_vec()) + .unwrap(); + assert!(prepared.terminal_result(&channel, &auth, &output).is_ok()); +} + +// ── The terminal result ─────────────────────────────────────────────── + +/// The golden result: one legal job carried from proposal to a signed +/// result, every digest pinned. +#[test] +fn terminal_fetch_result_is_pinned() { + let channel = channel(); + let authorization = authorization(); + let transcript = output_transcript(&authorization); + let result = terminal_fetch_result(&channel, &authorization, &transcript) + .expect("a legal terminal transcript"); + + assert_eq!(result.work_id, work_id(&channel, &authorization)); + assert_eq!( + hex(result.terminal_transcript_commitment.as_bytes()), + "986663facdd944d47bd210bc3379ac5aba2c48e31b7a303ccdee128a455172af" + ); + assert_eq!( + hex(result.canonical_output_digest.as_bytes()), + "d7749786cb4d09c0949760f9cad1c6289a32a3fbfa62217f14c51eba40567975" + ); + assert_eq!( + hex(result_digest(&channel, &result).as_bytes()), + "2d8d52936484533b65b37ed0e6d983eff30a7e8cf0d82c81821879d58888265b" + ); + + // The result record round-trips through the wire codec and the + // profile-agnostic result check accepts it. + let encoded = result.encode(); + assert_eq!( + hellas_rpc::protocol::work::PaidJobResultV1::decode(&encoded), + Ok(result) + ); + assert_eq!( + check_result(&channel, result.work_id, &result).expect("a legal result"), + result_digest(&channel, &result) + ); +} + +/// The result builder's refusals: an empty transcript, a transcript for +/// another request, a transcript the provider did not sign, and a +/// transcript whose last event is not a terminal. +#[test] +fn terminal_fetch_result_refusals() { + let channel = channel(); + let authorization = authorization(); + + // MUTATION: an empty transcript. + assert!(matches!( + terminal_fetch_result(&channel, &authorization, &[]), + Err(PaidWorkError::Transcript(_)) + )); + + // MUTATION: a well-formed fetch transcript answering a different + // request. + let wrong_input = InputCommitment::from_digest(Digest::from_bytes([0x99; 32])); + let key = producer_key(); + let builder = FetchOutputTranscriptBuilder::new(wrong_input, Assurance::ProducerSigned, &key); + let wrong_request = builder.finish(terminal_payload()).unwrap(); + assert!(matches!( + terminal_fetch_result(&channel, &authorization, &wrong_request), + Err(PaidWorkError::Transcript(_)) + )); + + // MUTATION: the client signs the output. The chain verifies; the + // producer is not the channel's provider. + let input = InputCommitment::from_digest(authorization.request_commitment.digest()); + let key = caller_key(); + let mut builder = FetchOutputTranscriptBuilder::new(input, Assurance::ProducerSigned, &key); + builder.push_event(event_payload("paid ")).unwrap(); + let client_signed = builder.finish(terminal_payload()).unwrap(); + assert_eq!( + terminal_fetch_result(&channel, &authorization, &client_signed), + Err(PaidWorkError::Mismatch { + field: "transcript producer key" + }) + ); + + // MUTATION: the chain ends in a semantic event rather than a + // terminal. + let key = producer_key(); + let mut builder = hellas_rpc::OutputTranscriptBuilder::new( + scheme_id(Operation::Fetch, Assurance::ProducerSigned), + input, + &key, + output_canonicalization(), + ); + builder + .push("response.event", event_payload("paid ")) + .unwrap(); + let no_terminal = builder.finish().unwrap().0; + assert!(matches!( + terminal_fetch_result(&channel, &authorization, &no_terminal), + Err(PaidWorkError::Transcript(_)) + )); +} + +/// The normalized fetch answer ignores event boundaries and binds the +/// network and the job. +#[test] +fn fetch_output_digest_normalizes_chunking() { + let network = network(); + let id = work_id(&channel(), &authorization()); + let first = event_payload("paid "); + let second = event_payload("fetch"); + let terminal = terminal_payload(); + let digest = + fetch_canonical_output_digest(network, id, &[first.clone(), second.clone()], &terminal); + + // The same bytes signed as one event rather than two: the boundary + // is not part of the answer. + let mut joined = first.clone(); + joined.extend_from_slice(&second); + assert_eq!( + fetch_canonical_output_digest(network, id, &[joined], &terminal), + digest + ); + + // MUTATION: a different answer. + assert_ne!( + fetch_canonical_output_digest(network, id, &[second, first], &terminal), + digest + ); + + // MUTATION: a different terminal. + let other_terminal = encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .expect("a legal terminal payload"); + assert_ne!( + fetch_canonical_output_digest( + network, + id, + &[event_payload("paid "), event_payload("fetch")], + &other_terminal + ), + digest + ); + + // The same answer for another job, or on another network, is another + // digest. + assert_ne!( + fetch_canonical_output_digest( + network, + Digest::from_bytes([0; 32]), + &[event_payload("paid "), event_payload("fetch")], + &terminal + ), + digest + ); + let other_network = NetworkId::new(OTHER_NETWORK).expect("legal network id"); + assert_ne!( + fetch_canonical_output_digest( + other_network, + id, + &[event_payload("paid "), event_payload("fetch")], + &terminal + ), + digest + ); + + // And the pinned value: the answer bytes are the flattened payload + // stream and nothing else. + assert_eq!( + hex(digest.as_bytes()), + "d7749786cb4d09c0949760f9cad1c6289a32a3fbfa62217f14c51eba40567975" + ); +} + +// ── The profile-agnostic tail ───────────────────────────────────────── + +/// A fetch result pays through the shared payment machinery unchanged: +/// the profile ends at `terminal_fetch_result`, and what consensus +/// settles is built from the same records as an evaluate job's. +#[test] +fn fetch_result_pays_through_the_shared_ledger() { + let channel = channel(); + let authorization = authorization(); + let transcript = output_transcript(&authorization); + let result = terminal_fetch_result(&channel, &authorization, &transcript) + .expect("a legal terminal transcript"); + + let (certificate, binding) = + next_payment(&channel, &authorization, &result, 0, capacity()).expect("a legal payment"); + assert_eq!(certificate.earned_cumulative(), 250); + assert_eq!(binding.work_id, work_id(&channel, &authorization)); + assert_eq!(binding.result_digest, result_digest(&channel, &result)); + assert_eq!(binding.certificate_digest, certificate.digest(network())); + + CreditLedger::new() + .credit_payment( + &channel, + &authorization, + &result, + &binding, + &certificate, + capacity(), + ) + .expect("the ledger credits the fetch job's payment"); +} + +// ── The real constructor, end to end ────────────────────────────────── + +/// The randomized fetch constructor's transcript passes the whole paid +/// pipeline: propose, both checks, and the terminal result. +#[test] +fn the_real_input_constructor_passes_the_whole_pipeline() { + let channel = channel(); + let events = build_input_events_with_retention( + SERVICE, + METHOD, + REQUEST_BODY, + environment().manifest_id(), + Assurance::ProducerSigned, + &caller_key(), + Retention::Retain, + ) + .expect("the real constructor builds a legal transcript"); + let bundle = PreparedPaidFetchInputV1::new(&events, &manifest()).expect("a legal transcript"); + let authorization = propose(&channel, &fetch_policy(), &bundle); + + check_fetch_authorization(&channel, &authorization, &fetch_policy(), 900) + .expect("a legal authorization"); + check_prepared_fetch_input(&channel, &authorization, &fetch_policy(), &route(), &bundle) + .expect("a legal prepared input"); + + let transcript = output_transcript(&authorization); + let result = terminal_fetch_result(&channel, &authorization, &transcript) + .expect("a legal terminal transcript"); + assert_eq!(result.work_id, work_id(&channel, &authorization)); +} + +// ── Bounds ──────────────────────────────────────────────────────────── + +/// The fetch module's one fixed-record preimage is measured, not assumed, +/// to be under the single-chunk limit — and is actually hashed, on the +/// longest legal network id. +#[test] +fn widest_fetch_preimage_is_measured() { + let longest_network = + NetworkId::new(&"n".repeat(hellas_kernel::MAX_NETWORK_ID_LENGTH)).expect("legal id"); + let encoded_network = 1 + hellas_kernel::MAX_NETWORK_ID_LENGTH; + + let widest = "hellas.work.fetch-policy.v1".len() + + encoded_network + + 32 + + PaidFetchPolicyV1::ENCODED_SIZE; + assert_eq!(widest, 27 + 64 + 32 + 126); + assert_eq!(widest, 249); + assert!(widest < hellas_xet::MIN_CHUNK_SIZE); + + // The variable-body domains are streamed; asserting their digests + // still compute over a multi-chunk body is what makes the choice of + // hasher load-bearing. + let mut big_body = route().canonical_body_bytes(); + big_body.extend_from_slice(&[0xab; 20_000]); + let mut preimage = b"hellas.work.fetch-route.v1".to_vec(); + preimage.extend_from_slice(&(big_body.len() as u32).to_be_bytes()); + preimage.extend_from_slice(&big_body); + assert_eq!( + fetch_route_commitment(&big_body).expect("a representable body"), + Digest::hash(&preimage) + ); + + // The widest fixed preimage must hash rather than panic, so it is + // actually hashed here. + let channel = channel_on(longest_network, EdgeId::from_bytes([0xe1; 32])); + let _ = fetch_policy_digest(&channel, &fetch_policy()); +} + +// ── Profile separation ──────────────────────────────────────────────── + +/// The two profiles share the channel, the authorization record, and the +/// payment tail, and nothing else: each profile's check refuses the other +/// profile's policy, and the digests differ under every body. +#[test] +fn the_two_profiles_do_not_mix() { + let channel = channel(); + let fetch_policy = fetch_policy(); + + // An evaluate execution policy over the same environment is not a + // fetch policy: different domains, different digests. + let evaluate_policy = hellas_rpc::protocol::work::PaidExecutionPolicyV1 { + allowed_environment: fetch_policy.allowed_environment, + generation_policy_digest: Digest::from_bytes([0x41; 32]), + identity_source_digest: Digest::from_bytes([0x42; 32]), + max_prompt_tokens: 8, + max_new_tokens: 64, + max_stop_token_ids: 4, + max_spool_bytes: fetch_policy.max_spool_bytes, + max_encoded_result_frame: fetch_policy.max_encoded_result_frame, + max_encoded_quote_response: fetch_policy.max_encoded_prepared_input, + dispatch_margin_blocks: fetch_policy.dispatch_margin_blocks, + delivery_margin_blocks: fetch_policy.delivery_margin_blocks, + oracle_grace_blocks: fetch_policy.oracle_grace_blocks, + fixed_price: fetch_policy.fixed_price, + }; + assert_ne!( + execution_policy_digest(&channel, &evaluate_policy), + fetch_policy_digest(&channel, &fetch_policy) + ); + + // The fetch authorization names the fetch policy's digest, so the + // evaluate check refuses it on exactly that field. + assert_eq!( + check_authorization(&channel, &authorization(), &evaluate_policy, 900), + Err(PaidWorkError::Mismatch { + field: "execution_policy_digest" + }) + ); + // And the fetch check refuses an authorization that names the + // evaluate policy. + let mut evaluate_signed = authorization(); + evaluate_signed.execution_policy_digest = execution_policy_digest(&channel, &evaluate_policy); + assert_eq!( + check_fetch_authorization(&channel, &evaluate_signed, &fetch_policy, 900), + Err(PaidWorkError::Mismatch { + field: "execution_policy_digest" + }) + ); +} diff --git a/crates/rpc/tests/paid_work_vectors.rs b/crates/rpc/tests/paid_work_vectors.rs index a405f588..5f2e5e7d 100644 --- a/crates/rpc/tests/paid_work_vectors.rs +++ b/crates/rpc/tests/paid_work_vectors.rs @@ -431,7 +431,8 @@ fn envelope_and_length_mutations_reject() { }) ); - // MUTATION: an unassigned tag. + // MUTATION: a tag this record does not own (5 is the fetch profile's + // policy tag). let mut unknown_tag = bytes.clone(); unknown_tag[1] = 5; assert_eq!( diff --git a/crates/rpc/tests/work_setup.rs b/crates/rpc/tests/work_setup.rs index 197653c0..4dc4fc98 100644 --- a/crates/rpc/tests/work_setup.rs +++ b/crates/rpc/tests/work_setup.rs @@ -116,7 +116,7 @@ fn config() -> WorkChannelConfig { payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: payment_values(), } } @@ -375,7 +375,12 @@ fn a_descriptor_opens_only_against_its_own_committed_policy() { ); let mut zero_price = config(); - zero_price.execution_policy.fixed_price = 0; + let hellas_rpc::protocol::work_profile::PaidWorkPolicy::Evaluate(policy) = + &mut zero_price.execution_policy + else { + panic!("fixture uses Evaluate"); + }; + policy.fixed_price = 0; assert_eq!( WorkChannelDescriptor::open(zero_price), Err(WorkSetupError::Record(PaidWorkError::PolicyZero { diff --git a/crates/sdk/Cargo.toml b/crates/sdk/Cargo.toml index 997fc1c6..7c789e53 100644 --- a/crates/sdk/Cargo.toml +++ b/crates/sdk/Cargo.toml @@ -30,7 +30,34 @@ provider = [ ] local-control = ["dep:tokio", "hellas-rpc/host-control", "hellas-wire/unix"] +paid-work = [ + "dep:async-stream", + "provider", + "dep:hellas-chain", + "hellas-chain/client", + "hellas-chain/work-watcher", + "dep:hellas-kernel", + "hellas-kernel/secp256k1", + "dep:hellas-work", + "hellas-client/work", + "hellas-rpc/work", + "dep:futures", + "dep:serde", + "dep:serde_json", + "dep:hex", + "dep:url", +] + [dependencies] +async-stream = { workspace = true, optional = true } +hellas-chain = { workspace = true, default-features = false, optional = true } +hellas-kernel = { workspace = true, default-features = false, optional = true } +hellas-work = { workspace = true, optional = true } +futures = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } +hex = { workspace = true, optional = true } +url = { version = "2", optional = true } anyhow = { workspace = true, optional = true } futures-core = { workspace = true, optional = true } hellas-attestation.workspace = true diff --git a/crates/sdk/src/lib.rs b/crates/sdk/src/lib.rs index b94aa36f..92abd115 100644 --- a/crates/sdk/src/lib.rs +++ b/crates/sdk/src/lib.rs @@ -24,7 +24,18 @@ pub use hellas_gateway as gateway; #[cfg(feature = "provider")] mod provider; #[cfg(feature = "provider")] -pub use provider::{OpenAiProviderOptions, ProviderHandle, start_openai_provider}; +pub use hellas_executor::{FetchRoute, FetchRouteEntry, FetchRoutePolicy, FetchRouteRegistry}; +#[cfg(feature = "paid-work")] +pub use hellas_kernel as kernel; +#[cfg(feature = "provider")] +pub use hellas_providers::{ + HttpProviderConfig, OpenAiResponsesFetchProvider, ResponsesFetchAdaptorFactory, +}; +#[cfg(feature = "provider")] +pub use provider::{ + FetchProviderOptions, OpenAiProviderOptions, ProviderHandle, start_fetch_provider, + start_openai_provider, +}; #[cfg(feature = "client")] pub use remote::{ClientIdentity, HellasClient, RemoteFetchRequest}; @@ -36,3 +47,14 @@ pub mod local { pub use hellas_rpc::cache::control::CacheController; pub use hellas_wire::local::{LOCAL_MUX_SLOTS, LocalControlServer, connect, transport}; } + +#[cfg(feature = "paid-work")] +pub mod paid_provider; +#[cfg(feature = "paid-work")] +pub mod work_config; + +#[cfg(feature = "paid-work")] +pub mod paid_client; + +#[cfg(feature = "paid-work")] +pub mod work_provision; diff --git a/crates/sdk/src/paid_client.rs b/crates/sdk/src/paid_client.rs new file mode 100644 index 00000000..10e6bbcb --- /dev/null +++ b/crates/sdk/src/paid_client.rs @@ -0,0 +1,1105 @@ +//! Reusable paid-work sessions: provider authentication, admission, verified delivery, +//! journal recovery, payment, and finalized settlement. Journals are client-owned. +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use anyhow::{Context as _, Result, bail}; +use hellas_chain::client::VerifiedRemoteLightClient; +use hellas_chain::{ + ConsensusInfo, ConsensusVerifier, FinalizedWorkView as _, WorkBlocks, WorkChannelQuery, +}; +use hellas_client::work::payment::pay_for_result; +use hellas_client::work::{CollectResultOutcome, collect_result}; +use hellas_kernel::{ + EdgeId, Funding, MAX_START_VALIDITY_BLOCKS, Secp256k1Signer, Secp256k1Verifier, + WorkPaymentTerms, +}; +use hellas_rpc::protocol::artifacts::{Canonical as _, PreparedPaidInputV1}; +use hellas_rpc::protocol::work::{ + JobDeadlines, generation_policy_digest, identity_source_digest, private_policy_commitment, +}; +use hellas_rpc::protocol::work_profile::{PaidWorkPolicy, PreparedPaidWorkInput}; +use hellas_rpc::protocol::work_setup::{ProviderChannelPolicy, WorkChannelDescriptor}; +use hellas_wire::ServiceMarker; +use hellas_wire::iroh::IrohTransport; +use hellas_work::work::{ClientEndpoint, JobProposal, propose_work, resume_work_proposal}; +use hellas_work::work_close::CloseProgress; +use hellas_work::work_close::FinalizedBlocks as _; +use hellas_work::work_handshake::{ + PaymentAdmission, SetupEndpoint, SetupService, apply_setup_exchange, prepare_setup_exchange, + send_setup_exchange, +}; +use hellas_work::work_open::{SetupAdvance, SetupProgress}; +use hellas_work::work_store::journal::MAX_RECORD_BYTES; +use hellas_work::work_store::{Role, SetupScan, SetupStore}; +use iroh::endpoint::presets; +use iroh::{Endpoint, EndpointAddr, EndpointId, SecretKey, TransportAddr}; + +use crate::work_config::WorkConfig; + +/// Options shared by all jobs on a paid channel. +pub struct PaidWorkOptions { + pub config: WorkConfig, + pub journal_root: PathBuf, + pub provider: EndpointId, + pub provider_addrs: Vec, + pub provider_trust: Option, + pub bond: EdgeId, + pub payment_funding: Funding, + pub omission_bond: u64, + pub acceptance_blocks: u64, + pub terminal_blocks: u64, + pub payment_blocks: u64, + /// Bounds retries while a provider is not ready. A session's overall lifetime + /// is controlled by its caller; `run_paid_work` also enforces an overall limit. + pub timeout: Duration, +} + +/// Convenience options for one complete paid job. +pub struct PaidWorkRun { + pub config: WorkConfig, + pub journal_root: PathBuf, + pub provider: EndpointId, + pub provider_addrs: Vec, + pub provider_trust: Option, + pub bond: EdgeId, + pub payment_funding: Funding, + pub omission_bond: u64, + pub prepared_input: PreparedPaidWorkInput, + pub acceptance_blocks: u64, + pub terminal_blocks: u64, + pub payment_blocks: u64, + pub timeout: Duration, + pub settle: bool, +} + +pub struct PaidWorkResult { + pub transcript: Vec, + pub work_id: hellas_rpc::Digest, + pub credited_cumulative: u64, + pub job_price: u64, + pub provider_key: hellas_rpc::PublicKey, + pub input: hellas_rpc::InputCommitment, + pub settled_provider_payout: Option, +} + +/// Executes one job using a client-owned journal, with an overall time limit. +pub async fn run_paid_work( + args: PaidWorkRun, + transport_key: SecretKey, + settlement_key: Secp256k1Signer, +) -> Result { + anyhow::ensure!( + !args.timeout.is_zero(), + "paid-work timeout must be positive" + ); + tokio::time::timeout(args.timeout, async move { + let PaidWorkRun { + config, + journal_root, + provider, + provider_addrs, + provider_trust, + bond, + payment_funding, + omission_bond, + prepared_input, + acceptance_blocks, + terminal_blocks, + payment_blocks, + timeout, + settle, + } = args; + let options = PaidWorkOptions { + config, + journal_root, + provider, + provider_addrs, + provider_trust, + bond, + payment_funding, + omission_bond, + acceptance_blocks, + terminal_blocks, + payment_blocks, + timeout, + }; + check_request( + &options.config.provider_policy(), + &prepared_input, + options.provider_trust.as_ref(), + hellas_rpc::PublicKey::Secp256k1(settlement_key.party_key().to_bytes()), + )?; + let endpoint = bind_paid_endpoint(transport_key).await?; + let mut session = PaidWorkSession::open(options, endpoint.clone(), settlement_key).await?; + let mut result = session + .run(Some(prepared_input), false, None) + .await? + .context("paid execution returned no result")?; + if settle { + result.settled_provider_payout = Some(session.settle().await?); + } + endpoint.close().await; + Ok(result) + }) + .await + .map_err(|_| { + anyhow::anyhow!("paid-work run timed out; client journals retain its payment state") + })? +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct InputIdentities { + pub allowed_environment: hellas_rpc::ContentId, + pub generation_policy_digest: hellas_rpc::Digest, + pub identity_source_digest: hellas_rpc::Digest, +} + +impl InputIdentities { + pub fn from_prepared(prepared: &PreparedPaidInputV1) -> Result { + let parts = prepared + .parts() + .context("prepared input contains a non-canonical body")?; + let allowed_environment = parts.manifest.content_id(); + anyhow::ensure!( + parts.evaluate_request.execution_environment == allowed_environment, + "prepared input request names environment {}, but its manifest derives {}", + parts.evaluate_request.execution_environment, + allowed_environment, + ); + Ok(Self { + allowed_environment, + generation_policy_digest: generation_policy_digest( + &parts.text_policy.canonical_bytes(), + )?, + identity_source_digest: identity_source_digest( + &parts.identity_artifact.canonical_bytes(), + )?, + }) + } +} + +/// A funded client channel. Keep one session per journal and serialize its jobs. +/// Cancellation retains recovery state; call `run(None, true, None)` to resume it. +pub struct PaidWorkSession { + args: PaidWorkOptions, + config: WorkConfig, + descriptor: WorkChannelDescriptor, + dialer: ProviderDialer, + chain: WorkBlocks, + next_validator: usize, + client: ClientEndpoint, + needs_recovery: bool, +} + +impl PaidWorkSession { + pub async fn open( + args: PaidWorkOptions, + endpoint: Endpoint, + settlement_key: Secp256k1Signer, + ) -> Result { + anyhow::ensure!( + args.acceptance_blocks > 0 && args.terminal_blocks > 0 && args.payment_blocks > 0, + "all three deadline spans must be greater than zero", + ); + anyhow::ensure!( + !args.timeout.is_zero(), + "paid-work timeout must be positive" + ); + + let config = args.config.clone(); + let policy = config.provider_policy(); + let bond = args.bond; + let payment_funding = args.payment_funding.clone(); + let mut next_validator = 0; + let chain = connect_chain(&config, &mut next_validator).await?; + check_genesis(&config, &chain).await?; + + std::fs::create_dir_all(&args.journal_root).with_context(|| { + format!( + "failed to create client journal root {}", + args.journal_root.display(), + ) + })?; + let store = SetupStore::open( + &args.journal_root, + config.chain.network, + bond, + Role::Client, + &Secp256k1Verifier::new(), + ) + .with_context(|| { + format!( + "failed to open client setup journal under {}", + args.journal_root.display(), + ) + })?; + let mut setup = SetupEndpoint::new( + store, + settlement_key.clone(), + PaymentAdmission::Proposes(Box::new(policy.clone())), + ); + let dialer = ProviderDialer::new( + args.provider, + args.provider_addrs.clone(), + endpoint, + args.provider_trust.clone(), + ); + + if setup.state().revision().is_none() { + exchange_setup(&dialer, &mut setup).await?; + } + let bundle = setup + .state() + .bundle() + .cloned() + .context("provider returned no bond proposal")?; + anyhow::ensure!( + bundle.bond_edge() == bond, + "provider proposed a different bond edge" + ); + anyhow::ensure!( + bundle.bond_terms().parties.taker() == settlement_key.party_key(), + "provider bond names client settlement key {}, not this identity's {}", + hex::encode(bundle.bond_terms().parties.taker().to_bytes()), + hex::encode(settlement_key.party_key().to_bytes()), + ); + dialer.require_producer(hellas_rpc::PublicKey::Secp256k1( + bundle.bond_terms().parties.maker().to_bytes(), + ))?; + if setup.state().scan_armed().is_none() { + setup.arm_scan(finalized_floor(&chain).await?)?; + } + if setup.state().revision() == Some(1) { + let terms = payment_terms(&config, &policy, &bundle, args.omission_bond); + setup.propose_payment(payment_funding, terms)?; + } + if setup.state().revision() == Some(2) { + exchange_setup(&dialer, &mut setup).await?; + } + anyhow::ensure!( + setup.state().revision() == Some(3), + "setup did not reach its countersigned revision", + ); + + let setup_service = SetupService::new(setup); + let (mounted, descriptor) = + drive_setup(&setup_service, &policy, &chain, config.poll).await?; + let ready = ready_channel(&descriptor, &chain).await?; + let client = ClientEndpoint::new(ready.clone(), mounted, settlement_key)?; + + Ok(Self { + args, + config, + descriptor, + dialer, + chain, + next_validator, + client, + needs_recovery: true, + }) + } + + /// Identifies the funded channel and its admitted execution policy. + pub fn descriptor(&self) -> &WorkChannelDescriptor { + &self.descriptor + } + + /// Read-only journal state for routing, recovery, and admission decisions. + pub fn state(&self) -> &hellas_work::work_store::ChannelState { + self.client.state() + } + + /// True after reopening a journal or an interrupted request. + pub fn needs_recovery(&self) -> bool { + self.needs_recovery + } + + /// Opens the client close and waits for its finalized provider payout. + pub async fn settle(&mut self) -> Result { + self.client + .prepare_close() + .context("failed to prepare the client payment close")?; + loop { + match self + .client + .advance_close(&self.chain, &self.chain) + .await + .context("failed to advance the client payment close")? + { + CloseProgress::Settled { provider_payout } => return Ok(provider_payout), + CloseProgress::Submitted { outcome, .. } => { + tracing::info!(?outcome, "client payment close submitted") + } + CloseProgress::Opened { .. } | CloseProgress::Nothing => {} + } + tokio::time::sleep(self.config.poll).await; + } + } + + /// Refreshes finalized state, rotating through configured validators on failure. + pub async fn follow_chain(&mut self) -> Result<()> { + for attempt in 0..self.config.validators.len() { + match self.client.catch_up(&self.chain).await { + Ok(_) => return Ok(()), + Err(error) if attempt + 1 == self.config.validators.len() => { + return Err(error.into()); + } + Err(error) => { + tracing::debug!(%error, "paid channel will continue catch-up through another validator"); + self.chain = connect_chain(&self.config, &mut self.next_validator).await?; + check_genesis(&self.config, &self.chain).await?; + } + } + } + bail!("no configured validator supplied finalized history") + } + + /// Runs a request and pays only after verifying its complete result. + /// With `recover`, resume payable journaled work first and admit this as a new + /// job; otherwise reuse a still-active job matching the input. `None` only + /// performs recovery. Incremental progress is currently supported for Evaluate. + pub async fn run( + &mut self, + prepared: Option, + recover: bool, + progress: Option, + ) -> Result> { + self.run_with_admission(prepared, recover, progress, None) + .await + } + + /// Like `run`, also marking `proposed` before a proposal can leave the process. + /// A caller using cancellation must continue payment/recovery once it is set. + pub async fn run_with_admission( + &mut self, + prepared: Option, + recover: bool, + progress: Option, + proposed: Option<&AtomicBool>, + ) -> Result> { + self.follow_chain().await?; + let Self { + args, + config, + descriptor, + dialer, + chain, + client, + needs_recovery, + .. + } = self; + if let Some(prepared) = prepared.as_ref() { + check_request( + &config.provider_policy(), + prepared, + dialer.trust.as_ref(), + hellas_rpc::PublicKey::Secp256k1(descriptor.channel().client_key().to_bytes()), + )?; + anyhow::ensure!( + progress.is_none() || matches!(prepared, PreparedPaidWorkInput::Evaluate(_)), + "incremental result delivery is only supported for Evaluate" + ); + } + let ready = caught_up_channel(descriptor, client, &*chain).await?; + if recover && *needs_recovery { + if let Some(payment) = client.state().last_payment() { + // The provider may have committed payment while its acknowledgement + // was lost. Re-send the retained certificate before accepting work. + pay_for_result(dialer.work().await?, client, payment.work_id).await?; + } + let pending = client + .state() + .jobs() + .filter(|job| { + // The journal forbids signing payment after this height. + // Keep the evidence, but do not let an unpayable old job + // prevent this channel from serving a new request. Retained + // certificates are re-sent separately above. + let payable = job.authorization().payment_deadline >= client.state().cursor().0; + if !payable { + tracing::info!( + work_id = %hex::encode(job.work_id().as_bytes()), + payment_deadline = job.authorization().payment_deadline, + "retaining expired unpaid job without retrying execution", + ); + } + payable + }) + .map(|job| { + PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) + .map(|input| (job.work_id(), job.phase(), input)) + }) + .collect::, _>>()?; + for (work_id, phase, pending) in pending { + check_request( + &config.provider_policy(), + &pending, + dialer.trust.as_ref(), + hellas_rpc::PublicKey::Secp256k1(descriptor.channel().client_key().to_bytes()), + )?; + let result = execute_paid_job( + args, + pending, + dialer, + client, + &ready, + &*chain, + config.poll, + None, + JobLookup::Retained(work_id), + None, + ) + .await; + if let Err(error) = &result + && ((phase == hellas_work::work_store::JobPhase::HalfSigned + && matches!( + error.downcast_ref::(), + Some(hellas_work::work::ProposeError::Refused { + refusal, + .. + }) if !refusal.is_retryable() + )) + || permanently_refused_delivery(error)) + { + // Keep the signed evidence without deciding that an unpaid + // job was paid or cancelled. A permanent provider refusal + // cannot be repaired by blocking every later request here. + tracing::info!(%work_id, %error, "retaining an unpaid job refused by the provider"); + continue; + } + result?; + } + } + // Keep recovery armed across any error or cancellation after acceptance. + *needs_recovery = prepared.is_some(); + let result = match prepared { + Some(prepared) => { + let ready = caught_up_channel(descriptor, client, &*chain).await?; + Some( + execute_paid_job( + args, + prepared, + dialer, + client, + &ready, + &*chain, + config.poll, + progress.as_ref(), + if recover { + JobLookup::New + } else { + JobLookup::PreparedInput + }, + proposed, + ) + .await?, + ) + } + None => None, + }; + *needs_recovery = false; + Ok(result) + } +} + +fn permanently_refused_delivery(error: &anyhow::Error) -> bool { + use hellas_client::work::CollectResultError; + use hellas_work::work::DeliverError; + let delivery = error.downcast_ref::().or_else(|| { + match error.downcast_ref::() { + Some(CollectResultError::Deliver(delivery)) => Some(delivery), + _ => None, + } + }); + matches!(delivery, Some(DeliverError::Refused { refusal, .. }) if !refusal.is_retryable()) +} + +/// Proposes until a provider accepts or the caller's execution window closes. +/// A provider catching its chain cursor up replies `NotReady`; that is not an +/// answer to the job. The retained proposal makes each retry the same request, +/// while bounded exponential backoff avoids turning recovery into a request +/// flood. +async fn propose_when_ready( + dialer: &ProviderDialer, + client: &mut ClientEndpoint, + proposal: &JobProposal, + retained: Option, + poll: Duration, + timeout: Duration, + proposed: Option<&AtomicBool>, +) -> Result { + let deadline = Instant::now() + timeout; + let mut delay = poll.max(Duration::from_secs(1)); + loop { + if Instant::now() >= deadline { + bail!("provider remained not ready for {timeout:?}"); + } + let transport = dialer.work().await?; + // Once a proposal can leave this process, a lost acknowledgement must + // be treated as accepted work. HTTP cancellation may no longer stop it. + if let Some(proposed) = proposed { + proposed.store(true, Ordering::Release); + } + let result = match retained { + Some(work_id) => resume_work_proposal(transport, client, work_id).await, + None => propose_work(transport, client, proposal).await, + }; + match result { + Ok(work_id) => return Ok(work_id), + Err(hellas_work::work::ProposeError::Refused { refusal, .. }) + if refusal.is_retryable() => + { + let remaining = deadline.saturating_duration_since(Instant::now()); + tokio::time::sleep(delay.min(remaining)).await; + delay = delay.saturating_mul(2).min(Duration::from_secs(10)); + } + Err(error) => return Err(error.into()), + } + } +} + +enum JobLookup { + Retained(hellas_rpc::Digest), + PreparedInput, + New, +} + +#[allow(clippy::too_many_arguments)] +async fn execute_paid_job( + args: &PaidWorkOptions, + prepared: PreparedPaidWorkInput, + dialer: &ProviderDialer, + client: &mut ClientEndpoint, + ready: &hellas_rpc::protocol::work_setup::ReadyChannel, + chain: &WorkBlocks, + poll: Duration, + progress: Option<&hellas_work::work::PaidProgress>, + lookup: JobLookup, + proposed: Option<&AtomicBool>, +) -> Result { + let readiness_timeout = args.timeout; + let prepared_bytes = prepared.encode()?; + let current = client.state().cursor().0; + let deadlines = deadlines( + current, + args.acceptance_blocks, + args.terminal_blocks, + args.payment_blocks, + )?; + let proposal = JobProposal { + prepared_input: prepared, + deadlines, + }; + let existing = client + .state() + .jobs() + .filter(|job| match &lookup { + JobLookup::Retained(work_id) => job.work_id() == *work_id, + JobLookup::New => false, + JobLookup::PreparedInput => { + job.prepared_input() == prepared_bytes.as_slice() + && job.authorization().payment_deadline >= current + && (job.phase() != hellas_work::work_store::JobPhase::HalfSigned + || job.authorization().acceptance_deadline >= current) + } + }) + .map(|job| (job.work_id(), job.phase(), *job.authorization())) + .collect::>(); + anyhow::ensure!( + existing.len() <= 1, + "more than one active job matches this prepared input; inspect the retained channel journal", + ); + if !existing.is_empty() + && let Some(proposed) = proposed + { + proposed.store(true, Ordering::Release); + } + let (work_id, already_collected) = match existing.first().copied() { + Some((work_id, hellas_work::work_store::JobPhase::HalfSigned, _)) => ( + propose_when_ready( + dialer, + client, + &proposal, + Some(work_id), + poll, + readiness_timeout, + proposed, + ) + .await?, + false, + ), + Some((work_id, hellas_work::work_store::JobPhase::Ready, _)) + | Some((work_id, hellas_work::work_store::JobPhase::Matched, _)) => (work_id, true), + Some((work_id, _, _)) => (work_id, false), + None => ( + propose_when_ready( + dialer, + client, + &proposal, + None, + poll, + readiness_timeout, + proposed, + ) + .await?, + false, + ), + }; + let transcript = if already_collected { + client + .state() + .job_by_id(work_id) + .map(|job| job.transcript().to_vec()) + .context("collected job disappeared from its journal")? + } else if let Some(progress) = progress { + let mut emitted = false; + let delivery = loop { + let result = hellas_work::work::fetch_result_stream( + dialer.work().await?, + client, + ready, + work_id, + |event| { + emitted = true; + progress(event.clone()).map_err(|error| { + hellas_rpc::protocol::work::PaidWorkError::Transcript(error.to_string()) + }) + }, + ) + .await; + let error = match result { + Ok(delivery) => break delivery, + Err(error) => error, + }; + let retryable = match &error { + hellas_work::work::DeliverError::Transport(status) => { + status.code == hellas_wire::WireCode::Unavailable + } + hellas_work::work::DeliverError::Refused { refusal, .. } => refusal.is_retryable(), + _ => false, + }; + // Retry delivery of this accepted job only before exposing output. + // Reopening after a prefix would replay it into the user's stream. + if emitted || !retryable { + return Err(error.into()); + } + client.catch_up(chain).await?; + let job = client + .state() + .job_by_id(work_id) + .context("accepted job disappeared")?; + anyhow::ensure!( + client.state().cursor().0 <= job.authorization().payment_deadline, + "payment deadline elapsed while waiting for result stream" + ); + tracing::debug!(%error, %work_id, "waiting for paid result stream readiness"); + tokio::time::sleep(poll.max(Duration::from_secs(1))).await; + }; + client.catch_up(chain).await?; + anyhow::ensure!( + client.state().cursor().0 + <= client + .state() + .job_by_id(work_id) + .context("accepted job disappeared")? + .authorization() + .payment_deadline, + "payment deadline elapsed during delivery" + ); + delivery.transcript + } else { + collect_until_ready(dialer, client, ready, chain, work_id, poll).await? + }; + let credited = pay_for_result(dialer.work().await?, client, work_id).await?; + Ok(PaidWorkResult { + work_id, + job_price: ready.execution_policy().fixed_price(), + credited_cumulative: credited, + transcript, + provider_key: hellas_rpc::PublicKey::Secp256k1(ready.channel().provider_key().to_bytes()), + input: proposal.prepared_input.input_commitment()?, + settled_provider_payout: None, + }) +} + +fn check_request( + policy: &ProviderChannelPolicy, + prepared: &PreparedPaidWorkInput, + trust: Option<&hellas_client::ProviderTrustAnchor>, + caller: hellas_rpc::PublicKey, +) -> Result<()> { + let assurance = prepared.assurance()?; + anyhow::ensure!( + trust + .as_ref() + .is_none_or(|trust| trust.required_assurance == assurance), + "paid request assurance differs from provider trust" + ); + anyhow::ensure!( + assurance == hellas_rpc::Assurance::ProducerSigned || trust.is_some(), + "attested paid work requires a provider trust anchor before disclosure" + ); + match (prepared, &policy.execution_policy) { + (PreparedPaidWorkInput::Evaluate(input), PaidWorkPolicy::Evaluate(_)) => { + check_evaluate_input(policy, input)?; + } + (PreparedPaidWorkInput::Fetch(input), PaidWorkPolicy::Fetch { policy, route }) => { + let parts = input.parts()?; + let request = hellas_rpc::fetch::verify_input_events(&parts.fetch_input_transcript)?; + anyhow::ensure!( + request.caller_key == caller, + "prepared fetch caller does not match the client identity" + ); + anyhow::ensure!( + request.execution_environment == policy.allowed_environment + && parts.manifest.content_id() == policy.allowed_environment, + "prepared fetch environment does not match work config" + ); + anyhow::ensure!( + request.retention == hellas_rpc::Retention::Ephemeral, + "paid fetch requires ephemeral retention" + ); + if let hellas_rpc::protocol::work_fetch::FetchRoutePolicy::SealedRoute { + service, + method, + } = route + { + anyhow::ensure!( + &request.service == service && &request.method == method, + "prepared fetch route does not match work config" + ); + } + } + _ => bail!("prepared input and work config select different profiles"), + } + Ok(()) +} +/// Checks an Evaluate request against the configured execution bounds. +pub fn check_evaluate_input( + policy: &ProviderChannelPolicy, + prepared: &PreparedPaidInputV1, +) -> Result<()> { + let input = InputIdentities::from_prepared(prepared)?; + let parts = prepared.parts()?; + let PaidWorkPolicy::Evaluate(expected) = &policy.execution_policy else { + bail!("work config does not select the Evaluate profile"); + }; + anyhow::ensure!( + expected.allowed_environment == input.allowed_environment, + "work config allows environment {}, but prepared input uses {}", + expected.allowed_environment, + input.allowed_environment, + ); + anyhow::ensure!( + hellas_rpc::protocol::work::matches_generation_policy(expected, &parts.text_policy)?, + "work config generation_policy_digest does not match prepared input", + ); + anyhow::ensure!( + expected.identity_source_digest == input.identity_source_digest, + "work config identity_source_digest does not match prepared input", + ); + Ok(()) +} + +fn payment_terms( + config: &WorkConfig, + policy: &ProviderChannelPolicy, + bundle: &hellas_rpc::protocol::work_bundle::WorkChannelSetupBundleV1, + omission_bond: u64, +) -> WorkPaymentTerms { + WorkPaymentTerms { + bond_edge: bundle.bond_edge(), + bond_terms: bundle.bond_terms().clone(), + private_policy_commitment: private_policy_commitment( + config.chain.network, + &policy.policy_salt, + &policy.channel_policy, + ), + omit_response_blocks: policy.min_omit_response_blocks, + start_validity_blocks: MAX_START_VALIDITY_BLOCKS, + omission_bond, + } +} + +async fn drive_setup( + setup: &SetupService, + policy: &ProviderChannelPolicy, + chain: &WorkBlocks, + poll: Duration, +) -> Result<(hellas_work::work_store::ChannelStore, WorkChannelDescriptor)> { + loop { + let SetupAdvance { progress, mounted } = setup + .advance_setup(chain, chain, chain) + .await + .context("failed to advance paid-work setup")?; + if let Some(store) = mounted { + let channel = store.state().channel(); + let descriptor = policy + .admit(channel.payment_edge(), channel.payment_terms().clone()) + .context("the funded channel no longer satisfies the configured policy")?; + return Ok((store, descriptor)); + } + match progress { + SetupProgress::Aborted(reason) => bail!("paid-work setup aborted: {reason:?}"), + SetupProgress::Faulted(reason) => bail!("paid-work setup faulted: {reason:?}"), + SetupProgress::TimeoutBond => bail!("provider bond timed out before setup completed"), + _ => tokio::time::sleep(poll).await, + } + } +} + +async fn ready_channel( + descriptor: &WorkChannelDescriptor, + chain: &WorkBlocks, +) -> Result { + let query = WorkChannelQuery { + bond_edge: descriptor.bond_edge(), + payment_edge: descriptor.channel().payment_edge(), + funding: Default::default(), + }; + let snapshot = chain + .work_channel_snapshot(query) + .await? + .context("no finalized channel snapshot is available")?; + descriptor + .check_ready(&snapshot.observed_channel()) + .context("the finalized channel is not ready") +} + +/// Reads the ready snapshot once the client has processed it. +/// +/// The snapshot and the blocks the cursor follows are answered by +/// validators independently, so the snapshot can name a height the +/// light client has not finalized yet, and a snapshot read after the +/// catch-up on a moving chain always lands a few blocks ahead of it. +/// The snapshot is sampled first and then the cursor is brought to it: +/// a fixed height is a target the catch-up reaches, where a fresh +/// snapshot every round was not. A state at or behind the cursor is +/// the direction `check_caught_up` accepts. +async fn caught_up_channel( + descriptor: &WorkChannelDescriptor, + client: &mut ClientEndpoint, + chain: &WorkBlocks, +) -> Result { + let ready = ready_channel(descriptor, chain).await?; + for _ in 0..16 { + let cursor = client.catch_up(chain).await?; + if ready.check_caught_up(cursor).is_ok() { + return Ok(ready); + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + let cursor = client.catch_up(chain).await?; + ready.check_caught_up(cursor)?; + Ok(ready) +} + +async fn collect_until_ready( + dialer: &ProviderDialer, + client: &mut ClientEndpoint, + ready: &hellas_rpc::protocol::work_setup::ReadyChannel, + chain: &WorkBlocks, + work_id: hellas_rpc::Digest, + poll: Duration, +) -> Result> { + loop { + match collect_result(dialer.work().await?, client, ready, chain, work_id).await? { + CollectResultOutcome::Collected(result) => return Ok(result.transcript), + CollectResultOutcome::NotReady { reason } => { + tracing::debug!(%reason, "waiting for paid result"); + tokio::time::sleep(poll.max(Duration::from_secs(1))).await; + } + } + } +} + +pub fn deadlines( + current: u64, + acceptance_blocks: u64, + terminal_blocks: u64, + payment_blocks: u64, +) -> Result { + let acceptance = current + .checked_add(acceptance_blocks) + .context("acceptance deadline overflow")?; + let terminal = acceptance + .checked_add(terminal_blocks) + .context("terminal deadline overflow")?; + let payment = terminal + .checked_add(payment_blocks) + .context("payment deadline overflow")?; + Ok(JobDeadlines { + acceptance, + terminal, + payment, + }) +} + +pub async fn bind_paid_endpoint(secret_key: SecretKey) -> Result { + Endpoint::builder(presets::N0) + .secret_key(secret_key) + .alpns(vec![ + hellas_rpc::services::work_setup::WorkSetup::ALPN + .as_bytes() + .to_vec(), + hellas_rpc::services::work::Work::ALPN.as_bytes().to_vec(), + ]) + .bind() + .await + .context("failed to bind paid-work Iroh endpoint") +} + +async fn exchange_setup(dialer: &ProviderDialer, setup: &mut SetupEndpoint) -> Result<()> { + let request = prepare_setup_exchange(setup); + let response = send_setup_exchange(dialer.setup().await?, request).await?; + apply_setup_exchange(setup, response)?; + Ok(()) +} + +struct ProviderDialer { + endpoint: Endpoint, + provider: EndpointAddr, + trust: Option, + producer: std::sync::Mutex>, +} + +impl ProviderDialer { + fn new( + provider: EndpointId, + addresses: Vec, + endpoint: Endpoint, + trust: Option, + ) -> Self { + Self { + trust, + producer: std::sync::Mutex::new(None), + endpoint, + provider: EndpointAddr::from_parts( + provider, + addresses.into_iter().map(TransportAddr::Ip), + ), + } + } + + fn require_producer(&self, key: hellas_rpc::PublicKey) -> Result<()> { + let mut expected = self + .producer + .lock() + .map_err(|_| anyhow::anyhow!("provider key lock poisoned"))?; + anyhow::ensure!( + expected.as_ref().is_none_or(|old| *old == key), + "authenticated provider key differs from the payment channel" + ); + *expected = Some(key); + Ok(()) + } + + async fn setup(&self) -> Result { + self.connect(hellas_rpc::services::work_setup::WorkSetup::ALPN.as_bytes()) + .await + } + + async fn work(&self) -> Result { + self.connect(hellas_rpc::services::work::Work::ALPN.as_bytes()) + .await + } + + async fn connect(&self, alpn: &[u8]) -> Result { + let connection = self + .endpoint + .connect(self.provider.clone(), alpn) + .await + .with_context(|| format!("failed to connect to provider {}", self.provider.id))?; + let transport = IrohTransport::new(connection); + if let Some(trust) = &self.trust { + let producer = if alpn == hellas_rpc::services::work::Work::ALPN.as_bytes() { + hellas_client::confidential_open::( + &transport, trust, + ) + .await? + } else { + hellas_client::confidential_open::( + &transport, trust, + ) + .await? + }; + self.require_producer(producer)?; + } + Ok(transport) + } +} + +async fn connect_chain( + config: &WorkConfig, + next_validator: &mut usize, +) -> Result> { + let verifier = ConsensusVerifier::new(&ConsensusInfo { + validators: config.validators.clone(), + threshold_identity: config.chain.threshold_identity.clone(), + network_id: config.chain.network.as_str().to_owned(), + }) + .context("configured threshold identity is unusable")?; + // A peer can accept connections while lacking a historical certificate. + // Reconnects must make progress through the configured alternatives. + let start = *next_validator; + let mut failures = Vec::new(); + for url in config + .validators + .iter() + .cycle() + .skip(start) + .take(config.validators.len()) + { + *next_validator = (*next_validator + 1) % config.validators.len(); + match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { + Ok(client) => return Ok(WorkBlocks::new(client)), + Err(error) => failures.push(format!("{url}: {error}")), + } + } + bail!("no configured validator answered: {}", failures.join("; ")) +} + +async fn check_genesis( + config: &WorkConfig, + chain: &WorkBlocks, +) -> Result<()> { + let first = chain + .block_at(1) + .await? + .context( + "configured validator has no finalized block 1; genesis cannot be authenticated until block 1 is finalized", + )?; + check_genesis_payload( + config.chain.genesis_payload_digest.as_bytes(), + &first.parent, + ) +} +pub fn check_genesis_payload(expected: &[u8; 32], actual: &[u8; 32]) -> Result<()> { + anyhow::ensure!( + actual == expected, + "validator genesis payload {} does not match configured {}", + hex::encode(actual), + hex::encode(expected), + ); + Ok(()) +} + +async fn finalized_floor(chain: &WorkBlocks) -> Result { + let height = chain + .latest_height() + .await? + .context("configured validator has finalized no blocks")?; + let block = chain + .block_at(height) + .await? + .context("configured validator did not return its finalized tip")?; + Ok(SetupScan { + height, + payload: block.payload, + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/sdk/src/paid_client/tests.rs b/crates/sdk/src/paid_client/tests.rs new file mode 100644 index 00000000..0f69397f --- /dev/null +++ b/crates/sdk/src/paid_client/tests.rs @@ -0,0 +1,290 @@ +use super::*; +use hellas_rpc::pb::execute::{OpenRequest, OpenResponse, open_response}; +use hellas_rpc::{ + Assurance, Digest, PlatformCredential, PlatformEnrollment, ProducerSigningKey, + ProviderEnrollmentBundle, ProviderGenesisStatement, PublicKey, RootKind, RootProof, + SignedProviderGenesis, +}; +use hellas_wire::{MethodMarker, StreamTransport, WireStatus}; + +fn enrollment(peer: EndpointId) -> (ProviderEnrollmentBundle, ProducerSigningKey) { + let root = ProducerSigningKey::from_secret_bytes([1; 32]).unwrap(); + let producer = ProducerSigningKey::from_secret_bytes([2; 32]).unwrap(); + let statement = ProviderGenesisStatement { + root_kind: RootKind::Software, + root_public_key: root.public_key(), + producer_public_key: producer.public_key(), + transport_public_key: PublicKey::Ed25519(*peer.as_bytes()), + platform_credential: PlatformCredential::Absent, + installation_nonce: [3; 32], + }; + let proof = root + .sign_digest(Digest::hash(&statement.canonical_bytes())) + .unwrap(); + ( + ProviderEnrollmentBundle { + genesis: SignedProviderGenesis { + statement, + root_proof: RootProof::Software(proof), + }, + platform: PlatformEnrollment::Absent, + }, + producer, + ) +} + +async fn serve_open( + transport: &IrohTransport, + bundle: ProviderEnrollmentBundle, + producer: ProducerSigningKey, +) where + M: MethodMarker, +{ + let inbound = transport.accept().await.unwrap().unwrap(); + // A paid request must never be the first RPC on a trusted connection. + assert_eq!(inbound.method_id, M::METHOD_ID); + hellas_rpc::call::dispatch_unary_with_context::( + inbound, + move |request, context| async move { + let nonce: [u8; 32] = request.nonce.try_into().unwrap(); + let binding = hellas_rpc::open_proof_binding( + &context.open_exporter.unwrap(), + &nonce, + &producer.public_key(), + bundle.content_id(), + M::Service::ALPN.as_bytes(), + ); + Ok::<_, WireStatus>(OpenResponse { + provider_genesis: bundle.canonical_bytes(), + proof: Some(open_response::Proof::ProducerSignature( + hellas_rpc::run_ticket::signature_to_pb( + &producer.sign_digest(binding).unwrap(), + ), + )), + }) + }, + ) + .await + .unwrap(); +} + +#[tokio::test] +async fn both_paid_connections_open_before_disclosure_and_refuse_wrong_assurance_or_party() { + tokio::time::timeout(Duration::from_secs(15), async { + let server = Endpoint::builder(presets::Minimal) + .secret_key(SecretKey::from_bytes(&[4; 32])) + .alpns(vec![ + hellas_rpc::services::work::Work::ALPN.as_bytes().to_vec(), + hellas_rpc::services::work_setup::WorkSetup::ALPN + .as_bytes() + .to_vec(), + ]) + .bind_addr("127.0.0.1:0".parse::().unwrap()) + .unwrap() + .bind() + .await + .unwrap(); + let (bundle, key) = enrollment(server.id()); + let trust = hellas_client::ProviderTrustAnchor { + expected_genesis: bundle.content_id(), + required_assurance: Assurance::ProducerSigned, + apple_app_attest: None, + }; + let addresses = server.bound_sockets(); + let serving = server.clone(); + let (done, completed) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let mut connections = Vec::new(); + for _ in 0..4 { + let connection = serving.accept().await.unwrap().await.unwrap(); + let setup = connection.alpn() + == hellas_rpc::services::work_setup::WorkSetup::ALPN.as_bytes(); + let transport = IrohTransport::new(connection); + if setup { + serve_open::( + &transport, + bundle.clone(), + key.clone(), + ) + .await; + } else { + serve_open::( + &transport, + bundle.clone(), + key.clone(), + ) + .await; + } + connections.push(transport); + } + let _ = completed.await; + }); + let mut dialer = ProviderDialer::new( + server.id(), + addresses, + bind_paid_endpoint(SecretKey::from_bytes(&[5; 32])) + .await + .unwrap(), + Some(trust), + ); + dialer.setup().await.expect("setup authenticates"); + dialer.work().await.expect("work authenticates"); + *dialer.producer.lock().unwrap() = Some( + ProducerSigningKey::from_secret_bytes([6; 32]) + .unwrap() + .public_key(), + ); + let error = dialer + .work() + .await + .err() + .expect("different payment party is refused"); + assert!( + error + .to_string() + .contains("differs from the payment channel") + ); + *dialer.producer.lock().unwrap() = None; + dialer.trust.as_mut().unwrap().required_assurance = Assurance::AppleAppAttest; + let error = dialer + .setup() + .await + .err() + .expect("software proof cannot satisfy App Attest"); + assert!(error.to_string().contains("Secure Enclave provider root")); + done.send(()).unwrap(); + task.await.unwrap(); + dialer.endpoint.close().await; + server.close().await; + }) + .await + .unwrap(); +} + +#[test] +fn recovery_skips_only_permanent_delivery_refusals() { + use hellas_client::work::CollectResultError; + use hellas_work::work::{DeliverError, WorkRefusal}; + + for (refusal, permanent) in [ + (WorkRefusal::Declined, true), + (WorkRefusal::Expired, true), + (WorkRefusal::NotReady, false), + (WorkRefusal::Unavailable, false), + ] { + let delivery = || DeliverError::Refused { + refusal, + reason: "provider diagnostic".to_owned(), + }; + assert_eq!(permanently_refused_delivery(&delivery().into()), permanent); + assert_eq!( + permanently_refused_delivery(&CollectResultError::Deliver(delivery()).into()), + permanent, + ); + } + assert!(!permanently_refused_delivery( + &DeliverError::Malformed("result").into() + )); + assert!(!permanently_refused_delivery(&anyhow::anyhow!( + "connection lost" + ))); +} + +fn fetch_request( + assurance: Assurance, + retention: hellas_rpc::Retention, +) -> (ProviderChannelPolicy, PreparedPaidWorkInput, PublicKey) { + use hellas_rpc::protocol::work::PaidChannelPolicyV1; + use hellas_rpc::protocol::work_fetch::{ + FetchRoutePolicy, PaidFetchPolicyV1, PreparedPaidFetchInputV1, fetch_route_commitment, + }; + let environment = hellas_rpc::FetchEnvironment::OpenAiResponses; + let caller = ProducerSigningKey::from_secret_bytes([7; 32]).unwrap(); + let events = hellas_rpc::fetch::build_input_events_with_retention( + "openai", + "responses", + br#"{"input":"private request"}"#, + environment.manifest_id(), + assurance, + &caller, + retention, + ) + .unwrap(); + let prepared = PreparedPaidFetchInputV1::new(&events, &environment.manifest()) + .unwrap() + .into(); + let route = FetchRoutePolicy::sealed_route("openai", "responses").unwrap(); + let policy = ProviderChannelPolicy { + network: hellas_kernel::NetworkId::new("paid-client-test").unwrap(), + policy_salt: [8; 32], + channel_policy: PaidChannelPolicyV1 { + compute_credit_limit: 40, + delivery_credit_limit: 40, + }, + execution_policy: PaidWorkPolicy::Fetch { + policy: PaidFetchPolicyV1 { + allowed_environment: environment.manifest_id(), + route_commitment: fetch_route_commitment(&route.canonical_body_bytes()).unwrap(), + max_request_body_bytes: 4096, + max_output_events: 64, + max_output_bytes: 16384, + max_spool_bytes: 65536, + max_encoded_result_frame: 65536, + max_encoded_prepared_input: 65536, + dispatch_margin_blocks: 4, + delivery_margin_blocks: 2, + oracle_grace_blocks: 6, + fixed_price: 10, + }, + route, + }, + expected_payment_values: hellas_kernel::EdgeValues::new( + 1000, + 200, + hellas_kernel::Fees::ZERO, + ), + min_omit_response_blocks: hellas_kernel::MIN_OMIT_RESPONSE_BLOCKS, + }; + (policy, prepared, caller.public_key()) +} + +#[test] +fn fetch_preflight_requires_matching_trust_caller_and_ephemeral_retention() { + use hellas_rpc::Retention; + let (policy, prepared, caller) = fetch_request(Assurance::AppleAppAttest, Retention::Ephemeral); + let error = check_request(&policy, &prepared, None, caller).unwrap_err(); + assert!(error.to_string().contains("trust anchor before disclosure")); + let mut trust = hellas_client::ProviderTrustAnchor { + expected_genesis: hellas_rpc::ContentId::from_bytes([8; 32]), + required_assurance: Assurance::ProducerSigned, + apple_app_attest: None, + }; + assert!( + check_request(&policy, &prepared, Some(&trust), caller) + .unwrap_err() + .to_string() + .contains("assurance differs") + ); + trust.required_assurance = Assurance::AppleAppAttest; + // Preflight checks the anchor selection; the live Open validates its proof. + check_request(&policy, &prepared, Some(&trust), caller).unwrap(); + let other = ProducerSigningKey::from_secret_bytes([9; 32]) + .unwrap() + .public_key(); + assert!( + check_request(&policy, &prepared, Some(&trust), other) + .unwrap_err() + .to_string() + .contains("caller does not match") + ); + let (policy, prepared, caller) = fetch_request(Assurance::ProducerSigned, Retention::Ephemeral); + check_request(&policy, &prepared, None, caller).unwrap(); + assert_eq!(prepared.assurance().unwrap(), Assurance::ProducerSigned); + let (policy, retained, caller) = fetch_request(Assurance::ProducerSigned, Retention::Retain); + assert!( + check_request(&policy, &retained, None, caller) + .unwrap_err() + .to_string() + .contains("ephemeral retention") + ); +} diff --git a/crates/sdk/src/paid_provider.rs b/crates/sdk/src/paid_provider.rs new file mode 100644 index 00000000..95233e38 --- /dev/null +++ b/crates/sdk/src/paid_provider.rs @@ -0,0 +1,1165 @@ +//! Shared provider routing and finalized-chain clock for paid work. +use crate::work_config::WorkRoutes; +use anyhow::Context; +use futures::future::BoxFuture; +use hellas_chain::client::VerifiedRemoteLightClient; +use hellas_chain::work_blocks::{PaidWorkClockError, advance_paid_work_clock}; +use hellas_chain::{ + ConsensusInfo, ConsensusVerifier, FinalizedWorkView, WorkBlocks, WorkChannelQuery, +}; +use hellas_kernel::{EdgeId, NetworkId, Secp256k1Signer, Secp256k1Verifier}; +use hellas_rpc::pb::work::*; +use hellas_rpc::peers::PeerId; +use hellas_rpc::protocol::Digest; +use hellas_rpc::protocol::work::{ + PaidJobAuthorizationV1, PrivateRecord as _, work_id as accepted_work_id, +}; +use hellas_rpc::protocol::work_setup::{ + ProviderChannelPolicy, ReadyChannel, WorkChannelDescriptor, +}; +use hellas_rpc::services::work::WorkHandler; +use hellas_rpc::services::work_setup::WorkSetupHandler; +use hellas_wire::{TransportContext, WireStatus}; +use hellas_work::work::{ + CloseEndpoint, PaidWorkBackend, RunError, RunOutcome, WorkService, run_accepted_work, +}; +use hellas_work::work_close::{CatchUpError, FinalizedBlocks, TxSink}; +use hellas_work::work_handshake::{PaymentAdmission, SetupEndpoint, SetupService}; +use hellas_work::work_open::{SetupAdvance, SetupDriveError, SetupProgress, SetupView}; +use hellas_work::work_store::{ChannelStore, JobPhase, Role, SetupStore, discover_setups}; +use std::{ + collections::BTreeMap, + path::PathBuf, + sync::{Arc, Mutex}, + time::Duration, +}; +use tokio::sync::{Mutex as AsyncMutex, oneshot}; +use tracing::{debug, info, warn}; +pub type ProductionWorkSource = WorkBlocks; +#[derive(Clone, Copy, Debug)] +pub struct UnmountedWork; + +fn not_ready() -> WorkRefused { + WorkRefused { + code: WorkRefusalCode::NotReady as i32, + reason: "work state is not mounted".to_string(), + } +} + +impl WorkSetupHandler for UnmountedWork { + fn exchange_setup( + &self, + _request: ExchangeSetupRequest, + _context: TransportContext, + ) -> impl core::future::Future< + Output = Result< + impl Into> + Send, + WireStatus, + >, + > + Send { + core::future::ready(Ok(ExchangeSetupResponse { + outcome: Some(exchange_setup_response::Outcome::Refused(not_ready())), + })) + } +} + +impl WorkHandler for UnmountedWork { + fn accept_work( + &self, + _request: AcceptWorkRequest, + _context: TransportContext, + ) -> impl core::future::Future< + Output = Result< + impl Into> + Send, + WireStatus, + >, + > + Send { + core::future::ready(Ok(AcceptWorkResponse { + outcome: Some(accept_work_response::Outcome::Refused(not_ready())), + })) + } + + fn deliver_result( + &self, + _request: DeliverResultRequest, + _context: TransportContext, + ) -> impl core::future::Future< + Output = Result< + impl Into> + Send, + WireStatus, + >, + > + Send { + core::future::ready(Ok(DeliverResultResponse { + outcome: Some(deliver_result_response::Outcome::Refused(not_ready())), + })) + } + + async fn stream_result( + &self, + _request: DeliverResultRequest, + _context: TransportContext, + ) -> Result { + Err(WireStatus::new( + hellas_wire::WireCode::Unavailable, + "paid work channel is not mounted", + )) + } + + fn admit_certificate( + &self, + _request: AdmitCertificateRequest, + _context: TransportContext, + ) -> impl core::future::Future< + Output = Result< + impl Into> + Send, + WireStatus, + >, + > + Send { + core::future::ready(Ok(AdmitCertificateResponse { + outcome: Some(admit_certificate_response::Outcome::Refused(not_ready())), + })) + } +} + +// ── The clock ───────────────────────────────────────────────────────── +// +// Everything below is a runner and nothing below is a decision. It +// builds no transaction, fixes no deadline, chooses no settlement, +// judges no duty due, and does not decide whether admission is on: each +// of those is a library edge it calls on a cadence, and the cadence is +// the whole of what this file adds. What it owns is *when* — and the +// journals, which is why it hands them to nobody. + +/// What the clock over one node's paid-work journals is built from. +/// +/// Every field is something the serve path has already loaded and +/// checked. The policy most of all: it is the loaded work +/// configuration's, carried here rather than derived again, so there is +/// no second place a node could decide what it countersigns over. +pub struct WorkRunnerConfig { + /// The network the journals are keyed and the signatures bound to. + pub network: NetworkId, + /// The threshold identity finalized blocks must authenticate under. + pub threshold_identity: Vec, + /// The configured root the setup journals live under. + pub journal_root: PathBuf, + /// Bilateral routes from authenticated peers to owned journals. + pub routes: WorkRoutes, + /// The validator RPCs a read and a submission go to. + pub validators: Vec, + /// How often the clock ticks. + pub poll: Duration, + /// The key every settlement this node signs is signed with. + pub settlement_key: Secp256k1Signer, + /// What every setup endpoint this node builds countersigns over. + pub policy: ProviderChannelPolicy, +} + +/// The channel this node answers `Work` from, once the runner has been +/// handed one. +/// +/// Written by the runner and read by the accept loop. What crosses is a +/// clone of a handler whose mutable pieces are themselves behind `Arc`s, +/// so the lock is held for a clone and never across a request: the +/// dispatch path never waits while holding the mount, and the clock never +/// waits on a request. +#[derive(Clone)] +pub struct MountedWork { + mounted: Arc>>>>, + driver: Option, +} + +impl Default for MountedWork { + fn default() -> Self { + Self { + mounted: Arc::new(Mutex::new(BTreeMap::new())), + driver: None, + } + } +} + +/// A cloneable, type-erased owner of the backend that runs accepted work. +/// +/// The production value owns an [`hellas_executor::ExecutorHandle`]. +/// Keeping the backend behind this narrow local seam means the clock and +/// ALPN dispatcher stay parameterized only over their finalized source; +/// neither has a second opinion about paid admission or execution failure. +#[derive(Clone)] +struct AcceptedWorkDriver(Arc); + +trait DriveAcceptedWork: Send + Sync { + fn run( + &self, + service: WorkService, + ready: ReadyChannel, + work_id: Digest, + ) -> BoxFuture<'static, Result>; +} + +struct BackendWorkDriver { + backend: Arc, +} + +impl DriveAcceptedWork for BackendWorkDriver +where + B: PaidWorkBackend + Send + Sync + 'static, +{ + fn run( + &self, + service: WorkService, + ready: ReadyChannel, + work_id: Digest, + ) -> BoxFuture<'static, Result> { + let backend = Arc::clone(&self.backend); + Box::pin( + async move { run_accepted_work(&service, &ready, backend.as_ref(), work_id).await }, + ) + } +} + +impl AcceptedWorkDriver { + fn new(backend: B) -> Self + where + B: PaidWorkBackend + Send + Sync + 'static, + { + Self(Arc::new(BackendWorkDriver { + backend: Arc::new(backend), + })) + } + + /// Starts one accepted job without lending its lifetime to either the + /// request path or the close clock. + fn spawn(&self, service: WorkService, ready: ReadyChannel, work_id: Digest) { + let running = self.0.run(service, ready, work_id); + let span = hellas_rpc::request_span!(target: "hellas_request", "paid.provider.execute", hellas.work.id = ?work_id, otel.status_code = tracing::field::Empty); + tokio::spawn(tracing::Instrument::instrument( + async move { + match running.await { + Ok(RunOutcome::Completed { .. }) => { + debug!(?work_id, "the accepted paid job completed") + } + Ok(RunOutcome::Ready { .. }) => { + debug!(?work_id, "the accepted paid job was already complete") + } + Ok(RunOutcome::Running) => { + debug!(?work_id, "the accepted paid job was already running") + } + Ok(RunOutcome::Indeterminate) => { + warn!( + ?work_id, + "the accepted paid job is indeterminate after restart" + ) + } + // `run_accepted_work` has already made backend and + // transcript faults terminal before returning them. The + // remaining errors have no node-local terminal policy; + // keep the exact failure visible to the operator. + Err(error) => { + tracing::Span::current().record("otel.status_code", "ERROR"); + warn!(?work_id, %error, "the accepted paid job did not complete"); + } + } + }, + span, + )); + } +} + +/// One mounted channel's served handler. +/// +/// `source` is replaceable because the runner redials a failed validator. +/// The request path copies the current source under the plain mutex and +/// drops that guard before its coherent read awaits. `accepting` spans the +/// complete fresh-read-to-signature sequence, so two acceptance attempts +/// cannot each refresh and then race to consume the same channel credit. +#[derive(Clone)] +pub struct MountedWorkService { + bond_edge: EdgeId, + service: WorkService, + descriptor: Option, + source: Arc>, + accepting: Arc>, + driver: Option, +} + +impl MountedWorkService +where + S: FinalizedBlocks + FinalizedWorkView + Sync, +{ + /// Re-establishes admission from one fresh coherent read. + /// + /// The service is the exact clone the runner drives. Its cursor is + /// checked after readiness, and that same service receives the fresh + /// decision before the raw handler is reached. A missing policy, + /// failed read, failed predicate, lagging cursor, or endpoint failure + /// therefore leaves the request on the retryable `NotReady` side. + pub async fn refresh_admission(&self) -> anyhow::Result { + // A `std::sync::MutexGuard` is deliberately confined to this + // block. Holding the source-slot guard across the read would make + // this handler's future non-`Send` and is not a valid dispatch. + let source = { + let held = self + .source + .lock() + .map_err(|_| anyhow::anyhow!("the finalized source lock is poisoned"))?; + held.clone() + }; + refresh_work_admission(&self.service, self.descriptor.as_ref(), &source).await + } + + async fn refresh_delivery(&self, request: &DeliverResultRequest) -> anyhow::Result<()> { + let Ok(bytes) = request.work_id.as_slice().try_into() else { + return Ok(()); + }; + let work_id = Digest::from_bytes(bytes); + let active = self + .service + .with_state(|state| state.job_by_id(work_id).is_some()); + if !matches!(active, Ok(true)) { + return Ok(()); + } + // A restarted mount has no readiness cached. A retained result must + // be collectable without first accepting another job. Terminal replies + // need no fresh admission and are authenticated by the core service. + let _accepting = self.accepting.lock().await; + let result = self.refresh_admission().await; + if let Err(error) = &result { + debug!(%error, "a delivery attempt found no fresh channel readiness"); + } + result.map(|_| ()) + } +} + +/// Re-establishes admission for the exact driven channel from one coherent +/// finalized read. +/// +/// Both the wire handler and restart recovery call this function. A recovered +/// job therefore gets no weaker interpretation of readiness than a new job, +/// and neither path can accidentally trust the readiness cached at mount. +async fn refresh_work_admission( + service: &WorkService, + descriptor: Option<&WorkChannelDescriptor>, + source: &S, +) -> anyhow::Result +where + S: FinalizedBlocks + FinalizedWorkView + Sync, +{ + let Some(descriptor) = descriptor else { + anyhow::bail!("this channel has no admission descriptor"); + }; + let query = WorkChannelQuery { + bond_edge: descriptor.bond_edge(), + payment_edge: descriptor.channel().payment_edge(), + funding: Default::default(), + }; + let Some(snapshot) = source + .work_channel_snapshot(query.clone()) + .await + .context("the fresh coherent channel read failed")? + else { + anyhow::bail!("no finalized channel snapshot is available"); + }; + if snapshot.query() != &query { + anyhow::bail!("the finalized source answered for another channel"); + } + let ready = descriptor + .check_ready(&snapshot.observed_channel()) + .context("the fresh channel snapshot is not ready")?; + // Keep one snapshot as the target. Recovery runs on the clock itself, + // so waiting for another tick here would prevent the cursor advancing. + // Take the service's existing driver when available; a concurrent clock + // drive keeps that authority until it finishes its own catch-up. + for _ in 0..16 { + let cursor = service + .with_state(|state| state.cursor().0) + .context("the mounted channel cursor is unavailable")?; + if ready.check_caught_up(cursor).is_ok() { + break; + } + if let Ok(mut driver) = service.drive() { + driver + .catch_up(source) + .await + .context("the mounted channel could not catch up to the fresh snapshot")?; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + let cursor = service + .with_state(|state| state.cursor().0) + .context("the mounted channel cursor is unavailable")?; + ready + .check_caught_up(cursor) + .context("the mounted channel has not caught up to the fresh snapshot")?; + service + .admit_new_work(ready.clone()) + .context("the driven work service refused its fresh readiness")?; + Ok(ready) +} + +impl WorkHandler for MountedWorkService +where + S: FinalizedBlocks + FinalizedWorkView + Sync, +{ + async fn accept_work( + &self, + request: AcceptWorkRequest, + _context: TransportContext, + ) -> Result> + Send, WireStatus> + { + if let Some(response) = self.service.precheck_acceptance(&request) { + return Ok(response); + } + let _accepting = self.accepting.lock().await; + // A preceding request or the clock may have resolved this proposal + // while admission was serialized. Retained replies need no fresh read. + if let Some(response) = self.service.precheck_acceptance(&request) { + return Ok(response); + } + let ready = match self.refresh_admission().await { + Ok(ready) => ready, + Err(error) => { + debug!(%error, "an acceptance attempt found no fresh channel readiness"); + return Ok(AcceptWorkResponse { + outcome: Some(accept_work_response::Outcome::Refused(WorkRefused { + code: WorkRefusalCode::NotReady as i32, + reason: "fresh channel readiness is unavailable".to_string(), + })), + }); + } + }; + // Derive the id from the request while the accepted response + // is still only a possibility. The response carries only the + // provider signature, and consulting `state.jobs().next()` after it + // leaves would race the clock terminalizing that same job. + let work_id = self + .service + .with_state(|state| { + PaidJobAuthorizationV1::decode(&request.authorization) + .ok() + .map(|authorization| accepted_work_id(state.channel(), &authorization)) + }) + .ok() + .flatten(); + let response = self.service.accept(&request); + if matches!( + response.outcome.as_ref(), + Some(accept_work_response::Outcome::Accepted(_)) + ) { + match (self.driver.as_ref(), work_id) { + (Some(driver), Some(work_id)) => { + driver.spawn(self.service.clone(), ready, work_id); + } + (None, Some(work_id)) => { + warn!(?work_id, "accepted paid work has no execution backend") + } + (_, None) => warn!("accepted paid work has no mounted job to execute"), + } + } + Ok(response) + } + + async fn deliver_result( + &self, + request: DeliverResultRequest, + context: TransportContext, + ) -> Result> + Send, WireStatus> + { + let response: hellas_rpc::call::WithTrailer = + if self.refresh_delivery(&request).await.is_ok() { + self.service.deliver_result(request, context).await?.into() + } else { + DeliverResultResponse { + outcome: Some(deliver_result_response::Outcome::Refused(WorkRefused { + code: WorkRefusalCode::NotReady as i32, + reason: "fresh channel readiness is unavailable".to_string(), + })), + } + .into() + }; + Ok(response) + } + + async fn stream_result( + &self, + request: DeliverResultRequest, + context: TransportContext, + ) -> Result { + self.refresh_delivery(&request).await.map_err(|_| { + WireStatus::new( + hellas_wire::WireCode::Unavailable, + "fresh channel readiness is unavailable", + ) + })?; + let mounted = self.clone(); + let mut stream = self.service.stream_result(request.clone(), context).await?; + Ok(Box::pin(async_stream::try_stream! { + let mut refresh = tokio::time::interval(Duration::from_secs(1)); + loop { + let event = tokio::select! { + event = futures::StreamExt::next(&mut stream) => Some(event), + _ = refresh.tick() => None, + }; + // Even buffered prefixes need a fresh finalized decision before + // leaving the node; a contest can start after this stream opens. + mounted.refresh_delivery(&request).await.map_err(|_| WireStatus::new( + hellas_wire::WireCode::Unavailable, + "fresh channel readiness is unavailable", + ))?; + if let Some(event) = event { + match event { + Some(event) => yield event?, + None => break, + } + } + } + })) + } + + fn admit_certificate( + &self, + request: AdmitCertificateRequest, + context: TransportContext, + ) -> impl core::future::Future< + Output = Result< + impl Into> + Send, + WireStatus, + >, + > + Send { + self.service.admit_certificate(request, context) + } +} + +impl MountedWork { + pub fn with_backend(backend: B) -> Self + where + B: PaidWorkBackend + Send + Sync + 'static, + { + Self { + mounted: Arc::new(Mutex::new(BTreeMap::new())), + driver: Some(AcceptedWorkDriver::new(backend)), + } + } + + /// Adds one owned channel under its authenticated peer. + /// + /// A peer is served only while exactly one channel is mounted under + /// it. Retaining a second candidate rather than overwriting either one + /// makes an ambiguity fail closed instead of turning insertion order + /// into routing policy. + pub fn mount( + &self, + peer: PeerId, + bond_edge: EdgeId, + service: &WorkService, + descriptor: Option, + accepting: Arc>, + source: &S, + ) -> bool { + match self.mounted.lock() { + Ok(mut held) => { + let mounted = held.entry(peer).or_default(); + mounted.push(MountedWorkService { + bond_edge, + service: service.clone(), + descriptor, + source: Arc::new(Mutex::new(source.clone())), + accepting, + driver: self.driver.clone(), + }); + mounted.len() == 1 + } + Err(_) => false, + } + } + + /// The one handler mounted for the transport-vouched peer. + pub fn handler(&self, context: &TransportContext) -> Option> { + let peer = context + .vouched_peer() + .map(|peer| PeerId::from_bytes(peer.0))?; + self.mounted.lock().ok().and_then(|held| { + let [mounted] = held.get(&peer)?.as_slice() else { + return None; + }; + Some(mounted.clone()) + }) + } + + /// Returns the local service for journal inspection and recovery. + /// Remote requests must use `handler`, which refreshes finalized readiness. + pub fn service(&self, context: &TransportContext) -> Option { + self.handler(context).map(|mounted| mounted.service) + } + + /// Replaces the finalized source for the matching driven channel. + /// + /// A reconnect reaches handlers already cloned by live connections, + /// because they share this inner source slot. Neither mount lock is + /// held across a source request. + fn refresh_source(&self, peer: PeerId, bond_edge: EdgeId, source: &S) { + let source_slot = self.mounted.lock().ok().and_then(|held| { + held.get(&peer)? + .iter() + .find(|mounted| mounted.bond_edge == bond_edge) + .map(|mounted| Arc::clone(&mounted.source)) + }); + if let Some(source_slot) = source_slot + && let Ok(mut held) = source_slot.lock() + { + *held = source.clone(); + } + } + + /// Stops serving `Work` from every channel. + /// + /// The clock's last act. A channel nobody is advancing is not a + /// channel to answer from — its journal is closed the moment the + /// runner drops it, and a handler still holding it open would be the + /// one thing keeping the files this process no longer owns. + pub fn clear_all(&self) { + if let Ok(mut held) = self.mounted.lock() { + held.clear(); + } + } +} + +/// Provider setups this node answers `WorkSetup` from by authenticated peer. +/// +/// Written by discovery and read by the accept loop, beside +/// [`MountedWork`]. The clone in this slot is the exact [`SetupService`] +/// stored in [`Driven::Setup`], so serving and driving share one exclusive +/// journal rather than attempting to reopen it. +#[derive(Clone, Debug, Default)] +pub struct MountedSetup(Arc>>>); + +#[derive(Clone, Debug)] +struct MountedSetupService { + bond_edge: EdgeId, + service: SetupService, +} + +impl MountedSetup { + /// Adds one owned setup under its authenticated peer. + pub fn mount(&self, peer: PeerId, bond_edge: EdgeId, service: &SetupService) -> bool { + match self.0.lock() { + Ok(mut held) => { + let mounted = held.entry(peer).or_default(); + mounted.push(MountedSetupService { + bond_edge, + service: service.clone(), + }); + mounted.len() == 1 + } + Err(_) => false, + } + } + + /// The exact setup service mounted for the transport-vouched peer, + /// cloned without holding the map across dispatch. + pub fn service(&self, context: &TransportContext) -> Option { + let peer = context + .vouched_peer() + .map(|peer| PeerId::from_bytes(peer.0))?; + self.0.lock().ok().and_then(|held| { + let [mounted] = held.get(&peer)?.as_slice() else { + return None; + }; + Some(mounted.service.clone()) + }) + } + + /// Stops serving only the setup that made this transition. + fn clear(&self, peer: PeerId, bond_edge: EdgeId) { + if let Ok(mut held) = self.0.lock() + && let Some(mounted) = held.get_mut(&peer) + { + mounted.retain(|mounted| mounted.bond_edge != bond_edge); + if mounted.is_empty() { + held.remove(&peer); + } + } + } + + /// Stops serving every setup during runner shutdown. + pub fn clear_all(&self) { + if let Ok(mut held) = self.0.lock() { + held.clear(); + } + } +} + +/// One journal, and what the clock drives it as. +/// +/// Two live states and one transition between them: a setup is driven +/// until it hands back the channel it mounted, and from then on the +/// channel is what is driven. Nothing here re-derives a mount — +/// [`SetupAdvance::mounted`] is the only way a [`ChannelStore`] reaches +/// this file, and the setup is not driven again afterwards, because a +/// second step would open a second journal on the same file. +enum Driven { + /// The journal is driven behind the setup service that answers for + /// it. The policy is retained beside the service: it is the provider + /// authority from which the full channel descriptor is rebuilt after + /// the setup reveals its actual terms. + Setup { + /// The endpoint this journal is both driven and served behind. + service: SetupService, + /// The retained provider authority, behind a pointer. + /// + /// Boxed because it is the widest thing this enum carries by a + /// long way — every other payload here is a handle or a store + /// pointer, one or two words each — and a journal is one value + /// with three shapes, so the two that hold no policy would + /// otherwise each be as large as the one that does. + /// [`PaymentAdmission`] already holds it behind the same + /// indirection, and this is built from that one, once per + /// journal at startup. + policy: Box, + }, + /// The channel this setup mounted, including the recovery authority + /// needed to finish a job accepted before a process restart. + Channel(Box), + /// The setup ended, or its mount was refused. Nothing left to + /// drive. + Done, +} + +/// One mounted channel as driven by the paid-work clock. +/// +/// Recovery lives here rather than in the served route: an accepted job is an +/// obligation recorded by this journal even if peer routing changes while the +/// process is down. `accepting` is also lent to the route when one is mounted, +/// so live acceptance and restart recovery serialize their readiness checks. +struct DrivenChannel { + service: WorkService, + descriptor: Option, + accepting: Arc>, + driver: Option, +} + +impl DrivenChannel { + fn accepted_work_id(&self) -> anyhow::Result> { + self.service + .with_state(|state| { + let mut jobs = state.jobs(); + let job = jobs.next()?; + (jobs.next().is_none() && job.phase() == JobPhase::Accepted).then(|| job.work_id()) + }) + .context("the driven channel state is unavailable") + } + + /// Starts a journaled Accepted job after proving current readiness. + /// + /// No in-memory `attempted` marker is needed. A racing live request or + /// clock tick reaches the same endpoint; its durable `JobRunning` record + /// lets exactly one caller receive `Invoke` and every other caller receive + /// `Running`. + async fn resume_accepted(&self, source: &S) -> anyhow::Result + where + S: FinalizedBlocks + FinalizedWorkView + Sync, + { + if self.accepted_work_id()?.is_none() { + return Ok(false); + } + let _accepting = self.accepting.lock().await; + let Some(work_id) = self.accepted_work_id()? else { + return Ok(false); + }; + let driver = self + .driver + .as_ref() + .context("the accepted paid job has no execution backend")?; + let ready = refresh_work_admission(&self.service, self.descriptor.as_ref(), source).await?; + driver.spawn(self.service.clone(), ready, work_id); + Ok(true) + } +} + +/// One setup journal on a clock. +struct SetupClock { + /// The bond this journal stakes, so a log line names which one. + bond_edge: EdgeId, + /// The authenticated peer whose configured route names this bond. + /// Together with `bond_edge`, this is the journal's route identity; + /// `None` keeps an unconfigured owned journal on its close clock + /// without making it a fallback service. + route_peer: Option, + /// What is being driven for it. + driven: Driven, +} + +impl SetupClock { + /// Takes this journal's one step, and says whether the chain + /// answered. + /// + /// A source failure is the only outcome the caller acts on: a + /// validator that stopped answering is dialled again rather than + /// asked forever. Everything else is this journal's own business and + /// is logged where it happens. + async fn tick( + &mut self, + source: &S, + signer: &Secp256k1Signer, + work_mount: &MountedWork, + setup_mount: &MountedSetup, + ) -> bool + where + S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, + { + let bond = hex::encode(self.bond_edge.to_bytes()); + let mut answered = true; + // One step of the setup this journal holds, with the policy the + // mount it may hand back is rebuilt from; nothing, once the + // journal is past its setup. + let step = match &mut self.driven { + Driven::Setup { service, policy } => Some(( + service.advance_setup(source, source, source).await, + policy.clone(), + )), + Driven::Channel(_) | Driven::Done => None, + }; + if let Some((step, policy)) = step { + match step { + Ok(SetupAdvance { progress, mounted }) => { + if let Some(store) = mounted { + if let Some(peer) = self.route_peer { + setup_mount.clear(peer, self.bond_edge); + } + self.take_mount(store, signer, &policy, source, work_mount); + } else if matches!( + progress, + SetupProgress::Aborted(_) | SetupProgress::Faulted(_) + ) { + warn!(bond, ?progress, "this setup ended with no channel to drive"); + self.driven = Driven::Done; + } else { + debug!(bond, ?progress, "the setup advanced"); + } + } + Err(error) => { + answered = !matches!(error, SetupDriveError::Source(_)); + warn!(bond, %error, "this setup did not advance"); + } + } + } + if let Driven::Channel(channel) = &self.driven { + if let Some(peer) = self.route_peer { + work_mount.refresh_source(peer, self.bond_edge, source); + } + if let Err(error) = channel.resume_accepted(source).await { + warn!(bond, %error, "an accepted paid job did not resume"); + } + match advance_paid_work_clock(&channel.service, source).await { + Ok(progress) => debug!(bond, ?progress, "the channel advanced"), + // `resume_accepted` owns the channel while it starts the + // durable execution. The clock's concurrent close pass has + // no work to do until that owner returns the cursor. + Err(PaidWorkClockError::CloseDrive(CatchUpError::Busy)) => { + debug!(bond, "the channel is already being driven") + } + Err(error) => { + answered &= !error.source_failed(); + warn!(bond, %error, "this channel's close did not advance"); + } + } + } + answered + } + + /// Mounts the store the driver handed back. + /// + /// Handed back, never reopened: the journal is exclusive, so a + /// second `ChannelStore::open` on the same file is a refusal rather + /// than a second view, and the settlement and origin this one + /// carries are the ones the completing read established. + fn take_mount( + &mut self, + store: ChannelStore, + signer: &Secp256k1Signer, + policy: &ProviderChannelPolicy, + source: &S, + mount: &MountedWork, + ) { + let bond = hex::encode(self.bond_edge.to_bytes()); + // The setup's retained policy supplies the provider-controlled + // fields, while the mounted channel supplies the payment edge and + // complete terms the two parties actually signed. This is a full + // descriptor reconstruction, not a mount-time readiness cache. + let descriptor = { + let channel = store.state().channel(); + match policy.admit(channel.payment_edge(), channel.payment_terms().clone()) { + Ok(descriptor) => Some(descriptor), + Err(error) => { + warn!(bond, %error, "the mounted channel no longer satisfies its admission policy"); + None + } + } + }; + match CloseEndpoint::new(store, signer.clone()) { + Ok(close) => { + let service = WorkService::close_only(close); + let accepting = Arc::new(AsyncMutex::new(())); + if self.route_peer.is_some_and(|peer| { + mount.mount( + peer, + self.bond_edge, + &service, + descriptor.clone(), + Arc::clone(&accepting), + source, + ) + }) { + info!( + bond, + "this node now answers Work from the channel it mounted" + ); + } else { + warn!( + bond, + "this channel has no unique peer route; it is driven and not served", + ); + } + self.driven = Driven::Channel(Box::new(DrivenChannel { + service, + descriptor, + accepting, + driver: mount.driver.clone(), + })); + } + // The journal and the key are not both the provider's view + // of one channel. Nothing this runner can do about it, and + // dropping the mount is what stops it being reopened every + // tick. + Err(error) => { + warn!(bond, %error, "the mounted channel is not this node's to close"); + self.driven = Driven::Done; + } + } + } +} + +/// The clock, over every paid-work journal this node owns. +pub struct WorkRunner { + clocks: Vec, + signer: Secp256k1Signer, + work_mount: MountedWork, + setup_mount: MountedSetup, + poll: Duration, + validators: Vec, + consensus_verifier: ConsensusVerifier, +} + +impl WorkRunner { + pub fn journal_count(&self) -> usize { + self.clocks.len() + } + pub fn channel_count(&self) -> usize { + self.clocks + .iter() + .filter(|c| matches!(c.driven, Driven::Channel(_))) + .count() + } +} + +impl WorkRunner +where + S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, +{ + /// Opens every setup journal under the configured root. + /// + /// The root and the network are the whole of what a restarting node + /// is told; the bond each journal is about and the role it was + /// written at come out of the files, which is what `discover_setups` + /// is for. A journal that cannot be named is reported and not + /// skipped silently: a file this node cannot open may be a channel + /// it still owes a close. + /// + /// # Errors + /// + /// When the root itself cannot be enumerated. + /// Every owned journal is driven. A configured route additionally + /// mounts its exact setup service under the authenticated peer that + /// names it; an unconfigured journal remains a close duty, not a + /// fallback answer. + pub fn discover( + config: WorkRunnerConfig, + work_mount: MountedWork, + setup_mount: MountedSetup, + ) -> anyhow::Result { + let consensus_verifier = ConsensusVerifier::new(&ConsensusInfo { + validators: config.validators.clone(), + threshold_identity: config.threshold_identity, + network_id: config.network.as_str().to_owned(), + }) + .context("the configured threshold identity is not usable")?; + let settlement_verifier = Secp256k1Verifier::new(); + let found = discover_setups(&config.journal_root, config.network).with_context(|| { + format!( + "failed to enumerate the work journals under {}", + config.journal_root.display(), + ) + })?; + for unnamed in &found.unidentified { + warn!( + path = %unnamed.path.display(), + reason = %unnamed.reason, + "a setup journal under the work root could not be named", + ); + } + let mut clocks = Vec::with_capacity(found.setups.len()); + for setup in found.setups { + let bond = hex::encode(setup.bond_edge.to_bytes()); + let route_peer = config + .routes + .iter() + .find(|route| route.bond == setup.bond_edge) + .map(|route| route.peer); + // A close capability binds the provider half, and this + // process holds the provider's key. A client journal beside + // this node's own is another party's, and this runner has + // nothing to sign for it. + if setup.role != Role::Provider { + warn!( + bond, + "a setup journal under the work root is not this node's half" + ); + continue; + } + let store = match SetupStore::open( + &config.journal_root, + config.network, + setup.bond_edge, + setup.role, + &settlement_verifier, + ) { + Ok(store) => store, + Err(error) => { + warn!(bond, %error, "a discovered setup journal did not open"); + continue; + } + }; + let policy = Box::new(config.policy.clone()); + let service = SetupService::new(SetupEndpoint::new( + store, + config.settlement_key.clone(), + PaymentAdmission::Admits(policy.clone()), + )); + if let Some(peer) = route_peer { + if setup_mount.mount(peer, setup.bond_edge, &service) { + info!( + bond, + "this node now answers WorkSetup from its driven setup" + ); + } else { + warn!(bond, "this provider setup has an ambiguous peer route"); + } + } else { + warn!(bond, "this provider setup has no configured peer route"); + } + let driven = Driven::Setup { service, policy }; + clocks.push(SetupClock { + bond_edge: setup.bond_edge, + route_peer, + driven, + }); + } + Ok(Self { + clocks, + signer: config.settlement_key, + work_mount, + setup_mount, + poll: config.poll, + validators: config.validators, + consensus_verifier, + }) + } + + /// Takes one step of every journal, and says whether the chain + /// answered all of them. + pub async fn tick(&mut self, source: &S) -> bool { + let mut answered = true; + for clock in &mut self.clocks { + answered &= clock + .tick(source, &self.signer, &self.work_mount, &self.setup_mount) + .await; + } + answered + } + + /// The loop, over whatever chain `dial` produces. + /// + /// One tick of every journal per period, and a chain that stopped + /// answering is dialled again rather than asked forever. The whole + /// of the cadence is here, and none of the decisions are. + pub async fn run_over(mut self, mut stop: oneshot::Receiver<()>, dial: D) + where + D: Fn() -> F, + F: core::future::Future>, + { + if self.clocks.is_empty() { + info!("no provider setup journal under the work root; the clock has nothing to drive"); + self.work_mount.clear_all(); + self.setup_mount.clear_all(); + return; + } + let mut chain = None; + loop { + tokio::select! { + _ = &mut stop => break, + () = tokio::time::sleep(self.poll) => {} + } + let Some(source) = chain.take() else { + chain = dial().await; + continue; + }; + if self.tick(&source).await { + chain = Some(source); + } + } + self.work_mount.clear_all(); + self.setup_mount.clear_all(); + info!("the paid-work clock stopped, and its journals are closed"); + } +} + +impl WorkRunner { + /// Ticks until told to stop, over the validators the configuration + /// names. + pub async fn run(self, stop: oneshot::Receiver<()>) { + let validators = self.validators.clone(); + let verifier = self.consensus_verifier.clone(); + self.run_over(stop, move || { + let validators = validators.clone(); + let verifier = verifier.clone(); + async move { connect_chain(&validators, verifier).await } + }) + .await; + } +} + +/// Rotate the first candidate on reconnect, including when a connected peer +/// cannot supply historical finalized blocks. Reads and submissions use the +/// selected verified connection. +/// +/// One endpoint for both directions. §1's concurrent fan-out to all six +/// is a submission strategy with an outcome rule, and neither exists in +/// this tree yet; inventing one here would be the runner deciding what +/// a submission means. +async fn connect_chain( + validators: &[String], + verifier: ConsensusVerifier, +) -> Option { + static NEXT_VALIDATOR: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let start = + NEXT_VALIDATOR.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % validators.len(); + for url in validators.iter().cycle().skip(start).take(validators.len()) { + match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { + Ok(client) => { + info!(validator = %url, "the paid-work clock reads and submits here"); + return Some(WorkBlocks::new(client)); + } + Err(error) => warn!(validator = %url, %error, "a configured validator did not answer"), + } + } + None +} diff --git a/crates/sdk/src/provider.rs b/crates/sdk/src/provider.rs index 8dd6d757..e29c73f8 100644 --- a/crates/sdk/src/provider.rs +++ b/crates/sdk/src/provider.rs @@ -3,13 +3,12 @@ use std::sync::Arc; use hellas_attestation::RootProver; use hellas_executor::{ - ExecuteServer, Executor, ExecutorMetrics, ExecutorSpawnConfig, FetchAccessPolicy, - FetchQuotaStoreBackend, FetchRoute, FetchRouteEntry, FetchRoutePolicy, FetchRouteRegistry, - FetchServer, FetchTranscriptStoreBackend, + ExecuteServer, Executor, ExecutorSpawnConfig, FetchAccessPolicy, FetchQuotaStoreBackend, + FetchRoute, FetchRouteEntry, FetchRoutePolicy, FetchRouteRegistry, FetchServer, + FetchTranscriptStoreBackend, }; use hellas_rpc::open::OpenDispatcher; use hellas_rpc::pb::execute::{OpenRequest, OpenResponse, open_response}; -use hellas_rpc::policy::ExecutePolicy; use hellas_rpc::run_ticket::signature_to_pb; use hellas_rpc::serve::MethodDispatcher; use hellas_rpc::services::execute::RunTicket; @@ -17,7 +16,7 @@ use hellas_rpc::services::fetch::{Fetch, Open as FetchOpen}; use hellas_rpc::{ Assurance, OPEN_NONCE_LEN, ProviderEnrollmentBundle, PublicKey, RootProof, open_proof_binding, }; -use hellas_wire::iroh::{IrohTransport, IrohTransportError}; +use hellas_wire::iroh::IrohTransport; use hellas_wire::{ Dispatcher, ServiceMarker, StreamTransport, TransportContext, WireCode, WireStatus, }; @@ -44,9 +43,41 @@ pub struct OpenAiProviderOptions { pub fetch_replay_max_in_flight: usize, } +/// An attested Fetch provider with an operator-supplied route registry. +pub struct FetchProviderOptions { + pub port: Option, + pub identity: ClientIdentity, + pub enrollment: ProviderEnrollmentBundle, + pub root: Arc, + pub state_directory: PathBuf, + pub routes: FetchRouteRegistry, + pub allowed_callers: Vec, + pub fetch_max_in_flight: usize, + pub fetch_queue_capacity: usize, + pub retained_transcript_capacity: usize, + pub fetch_replay_max_in_flight: usize, + #[cfg(feature = "paid-work")] + pub paid_work: Option, +} + +#[cfg(feature = "paid-work")] +struct WorkWatcher { + stop: Option>, + task: tokio::task::JoinHandle<()>, +} + +#[cfg(feature = "paid-work")] +impl Drop for WorkWatcher { + fn drop(&mut self) { + self.stop.take(); + } +} + pub struct ProviderHandle { endpoint: Endpoint, accept_task: tokio::task::JoinHandle<()>, + #[cfg(feature = "paid-work")] + work: Option, } impl ProviderHandle { @@ -61,6 +92,13 @@ impl ProviderHandle { pub async fn shutdown(mut self) { self.accept_task.abort(); let _ = (&mut self.accept_task).await; + #[cfg(feature = "paid-work")] + if let Some(mut work) = self.work.take() { + if let Some(stop) = work.stop.take() { + let _ = stop.send(()); + } + let _ = (&mut work.task).await; + } self.endpoint.close().await; } } @@ -68,6 +106,10 @@ impl ProviderHandle { impl Drop for ProviderHandle { fn drop(&mut self) { self.accept_task.abort(); + #[cfg(feature = "paid-work")] + if let Some(work) = &mut self.work { + work.stop.take(); + } } } @@ -77,10 +119,6 @@ pub async fn start_openai_provider( where R: RootProver + Send + Sync + 'static, { - anyhow::ensure!( - !options.allowed_callers.is_empty(), - "provider requires at least one allowed caller" - ); let upstream = Arc::new(hellas_providers::OpenAiResponsesFetchProvider::with_bearer( options.bearer_token, )?); @@ -92,37 +130,156 @@ where FetchRoute::new(options.service, options.method), FetchRouteEntry::new(upstream, adaptor, FetchRoutePolicy::default())?, )?; - let producer_key = Arc::new(options.identity.caller_key().clone()); - let access = FetchAccessPolicy::trusted_callers(options.allowed_callers).with_store( - FetchQuotaStoreBackend::fs(options.state_directory.join("quota")), - ); - let executor = Executor::spawn_configured(ExecutorSpawnConfig { - output_cache: Default::default(), - execute_policy: ExecutePolicy::Deny, - queue_capacity: 1, - metrics: Arc::new(ExecutorMetrics::default()), - producer_key: producer_key.clone(), - provider_genesis: Arc::new(options.enrollment.canonical_bytes()), - assurance: Assurance::AppleAppAttest, - fetch_access_policy: access, - fetch_routes: routes, + start_fetch_provider(FetchProviderOptions { + port: options.port, + identity: options.identity, + enrollment: options.enrollment, + root: options.root, + state_directory: options.state_directory, + routes, + allowed_callers: options.allowed_callers, fetch_max_in_flight: options.fetch_max_in_flight, fetch_queue_capacity: options.fetch_queue_capacity, + retained_transcript_capacity: options.retained_transcript_capacity, fetch_replay_max_in_flight: options.fetch_replay_max_in_flight, - fetch_store: FetchTranscriptStoreBackend::fs_with_capacity( - options.state_directory.join("transcripts"), - options.retained_transcript_capacity, - ), + #[cfg(feature = "paid-work")] + paid_work: None, }) - .await?; + .await +} +pub async fn start_fetch_provider( + options: FetchProviderOptions, +) -> anyhow::Result +where + R: RootProver + Send + Sync + 'static, +{ + #[cfg(feature = "paid-work")] + let has_paid_work = options.paid_work.is_some(); + #[cfg(not(feature = "paid-work"))] + let has_paid_work = false; + anyhow::ensure!( + !options.allowed_callers.is_empty() || has_paid_work, + "provider requires allowed callers or paid-work configuration" + ); + #[cfg(feature = "paid-work")] + if let Some(config) = &options.paid_work { + use hellas_rpc::protocol::{ + work_fetch::FetchRoutePolicy as PaidRoute, work_profile::PaidWorkPolicy, + }; + match &config.execution_policy { + PaidWorkPolicy::Fetch { + policy, + route: PaidRoute::SealedRoute { service, method }, + } => { + anyhow::ensure!( + options + .routes + .entry(&FetchRoute::new(service, method)) + .is_some_and( + |entry| entry.execution_environment() == policy.allowed_environment + ), + "paid channel names an unavailable Fetch route or manifest" + ); + } + PaidWorkPolicy::Fetch { + policy, + route: PaidRoute::OpenFetch { .. }, + } => { + anyhow::ensure!( + options.routes.has_environment(policy.allowed_environment), + "paid channel names an unavailable HTTPS manifest" + ); + } + _ => anyhow::bail!("Fetch provider requires a paid Fetch policy"), + } + } + let producer_key = Arc::new(options.identity.caller_key().clone()); + let access = FetchAccessPolicy::trusted_callers(options.allowed_callers).with_store( + FetchQuotaStoreBackend::fs(options.state_directory.join("quota")), + ); + let mut executor_config = ExecutorSpawnConfig::fetch_only( + producer_key.clone(), + Arc::new(options.enrollment.canonical_bytes()), + Assurance::AppleAppAttest, + options.routes, + ); + executor_config.fetch_access_policy = access; + executor_config.fetch_max_in_flight = options.fetch_max_in_flight; + executor_config.fetch_queue_capacity = options.fetch_queue_capacity; + executor_config.fetch_replay_max_in_flight = options.fetch_replay_max_in_flight; + executor_config.fetch_store = FetchTranscriptStoreBackend::fs_with_capacity( + options.state_directory.join("transcripts"), + options.retained_transcript_capacity, + ); + let executor = Executor::spawn_configured(executor_config).await?; + + #[cfg(feature = "paid-work")] + let work_mount = crate::paid_provider::MountedWork::::with_backend(executor.clone()); + #[cfg(feature = "paid-work")] + let setup_mount = crate::paid_provider::MountedSetup::default(); + #[cfg(feature = "paid-work")] + let work = if let Some(config) = options.paid_work { + anyhow::ensure!( + matches!( + config.execution_policy, + hellas_rpc::protocol::work_profile::PaidWorkPolicy::Fetch { .. } + ), + "Fetch provider requires a paid Fetch policy" + ); + anyhow::ensure!( + options.retained_transcript_capacity == 0, + "paid Fetch provider requires zero retained transcript capacity" + ); + crate::work_config::validate_work_routes(&config)?; + let policy = config.provider_policy(); + let settlement_key = hellas_kernel::Secp256k1Signer::from_secret_scalar( + options.identity.caller_secret_bytes(), + ) + .map_err(|_| anyhow::anyhow!("invalid provider settlement key"))?; + let runner = crate::paid_provider::WorkRunner::discover( + crate::paid_provider::WorkRunnerConfig { + network: config.chain.network, + threshold_identity: config.chain.threshold_identity, + journal_root: config.journal_root, + routes: config.routes, + validators: config.validators, + poll: config.poll, + settlement_key, + policy, + }, + work_mount.clone(), + setup_mount.clone(), + )?; + let (stop, stopped) = tokio::sync::oneshot::channel(); + Some(WorkWatcher { + stop: Some(stop), + task: tokio::spawn(runner.run(stopped)), + }) + } else { + None + }; let open = ProviderOpen { root: options.root, enrollment: options.enrollment, }; + let alpns = vec![::ALPN.as_bytes().to_vec()]; + #[cfg(feature = "paid-work")] + let alpns = if has_paid_work { + let mut alpns = alpns; + alpns.extend([ + hellas_rpc::services::work::Work::ALPN.as_bytes().to_vec(), + hellas_rpc::services::work_setup::WorkSetup::ALPN + .as_bytes() + .to_vec(), + ]); + alpns + } else { + alpns + }; let mut builder = Endpoint::builder(presets::N0) .secret_key(options.identity.transport_key()) - .alpns(vec![::ALPN.as_bytes().to_vec()]); + .alpns(alpns); if let Some(port) = options.port { builder = builder.bind_addr(format!("0.0.0.0:{port}").parse::()?)?; } @@ -156,6 +313,8 @@ where } }; let executor = executor.clone(); + #[cfg(feature = "paid-work")] + let (work_mount, setup_mount) = (work_mount.clone(), setup_mount.clone()); let open = open.clone(); connections.spawn(async move { let _slot = slot; @@ -170,10 +329,55 @@ where return; } }; - if connection.alpn() != ::ALPN.as_bytes() { + let alpn = connection.alpn().to_vec(); + let transport = Arc::new(IrohTransport::new(connection)); + #[cfg(feature = "paid-work")] + if has_paid_work && alpn == hellas_rpc::services::work::Work::ALPN.as_bytes() { + let context = transport.context(); + if let Some(handler) = work_mount.handler(&context) { + let server = OpenDispatcher::<_, _, hellas_rpc::services::work::Open>::new( + hellas_rpc::services::work::WorkServer(handler), + open, + ); + serve(transport, server).await; + } else { + let server = OpenDispatcher::<_, _, hellas_rpc::services::work::Open>::new( + hellas_rpc::services::work::WorkServer( + crate::paid_provider::UnmountedWork, + ), + open, + ); + serve(transport, server).await; + } + return; + } + #[cfg(feature = "paid-work")] + if has_paid_work + && alpn == hellas_rpc::services::work_setup::WorkSetup::ALPN.as_bytes() + { + let context = transport.context(); + if let Some(handler) = setup_mount.service(&context) { + let server = + OpenDispatcher::<_, _, hellas_rpc::services::work_setup::Open>::new( + hellas_rpc::services::work_setup::WorkSetupServer(handler), + open, + ); + serve(transport, server).await; + } else { + let server = + OpenDispatcher::<_, _, hellas_rpc::services::work_setup::Open>::new( + hellas_rpc::services::work_setup::WorkSetupServer( + crate::paid_provider::UnmountedWork, + ), + open, + ); + serve(transport, server).await; + } + return; + } + if alpn != Fetch::ALPN.as_bytes() { return; } - let transport = Arc::new(IrohTransport::new(connection)); let server = OpenDispatcher::<_, _, FetchOpen>::new( MethodDispatcher::<_, _, RunTicket>::new( ExecuteServer(executor.clone()), @@ -181,32 +385,44 @@ where ), open, ); - loop { - match transport.accept().await { - Ok(Some(inbound)) => { - if let Err(error) = - Dispatcher::::dispatch(&server, inbound).await - { - tracing::warn!(%error, "provider RPC failed"); - break; - } - } - Ok(None) | Err(IrohTransportError::Connection(_)) => break, - Err(error) => { - tracing::warn!(%error, "provider transport failed"); - break; - } - } - } + serve(transport, server).await; }); } }); Ok(ProviderHandle { endpoint, accept_task, + #[cfg(feature = "paid-work")] + work, }) } +async fn serve(transport: Arc, server: S) +where + S: Dispatcher + Send + Sync + 'static, + S::Error: Send + Sync + 'static, +{ + loop { + let incoming = + tokio::time::timeout(std::time::Duration::from_secs(120), transport.accept()).await; + match incoming { + Ok(Ok(Some(inbound))) => { + if !matches!( + tokio::time::timeout( + std::time::Duration::from_secs(30), + server.dispatch(inbound) + ) + .await, + Ok(Ok(())) + ) { + break; + } + } + _ => break, + } + } +} + struct ProviderOpen { root: Arc, enrollment: ProviderEnrollmentBundle, diff --git a/crates/sdk/src/work_config.rs b/crates/sdk/src/work_config.rs new file mode 100644 index 00000000..2f3318a4 --- /dev/null +++ b/crates/sdk/src/work_config.rs @@ -0,0 +1,696 @@ +//! The paid-work configuration file: what an operator writes down, and +//! what a node refuses to start without. +//! +//! `--work-config` was a path whose *presence* advertised two ALPNs and +//! whose contents were never opened. A node cannot mount a channel from +//! a path, so this is the schema and the loader for what is in it: the +//! three-part chain cross-check, the six validator URLs a write is +//! fanned to, the journal root, the bilateral route table, the two +//! policies this provider works under, the watcher's poll cadence, the +//! funding it expects a payment edge to carry, and the shortest response +//! window it will sign terms over. +//! +//! # The cross-check is not an anchor +//! +//! `(network_id, genesis_payload_digest, threshold_identity)` is a +//! fail-fast configuration cross-check. Only the threshold identity ever +//! authenticates a finalized block; the other two are how a node started +//! against the wrong chain says so at startup instead of at the first +//! settlement. The identity is decoded here, by the same constructor +//! consensus verification uses, so a typo is a startup error and not a +//! block that never verifies. +//! +//! # What is deliberately not here +//! +//! There is no Start-span field, no mutual-margin field, and no journal +//! cap. All were deleted: the Start span is fixed at 64, a work-payment +//! edge has no Mutual route, and the journal's active and checkpoint +//! ceilings are constants it enforces on itself +//! ([`MAX_ACTIVE_JOURNAL_BYTES`]), so any of them appearing in a file is +//! an operator configuring something that does not exist. Every struct +//! below denies unknown fields, which is what turns that into an error +//! naming the field. +//! +//! Nor is there a measured artifact. An earlier design derived the +//! response window and an alarm margin from latencies a bootstrap probe +//! recorded, pinned to the digest of the measuring binary; no deployed +//! node ever carried one, and every rebuild would have invalidated it. +//! The two numbers that design would have produced are written down +//! here instead, by the operator, and [`WorkConfig::provider_policy`] is +//! the whole of what is made from them. +//! +//! [`MAX_ACTIVE_JOURNAL_BYTES`]: hellas_work::work_store::journal::MAX_ACTIVE_JOURNAL_BYTES + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context as _, bail}; +use hellas_kernel::{ + EdgeId, EdgeValues, Fees, Key, MIN_OMIT_RESPONSE_BLOCKS, NetworkId, Secp256k1Verifier, +}; +use hellas_rpc::ContentId; +use hellas_rpc::peers::PeerId; +use hellas_rpc::protocol::Digest; +use hellas_rpc::protocol::work::{ + PaidChannelPolicyV1, PaidExecutionPolicyV1, check_execution_policy, +}; +use hellas_rpc::protocol::work_fetch::{ + FetchRoutePolicy as PaidFetchRoutePolicy, PaidFetchPolicyV1, fetch_route_commitment, +}; +use hellas_rpc::protocol::work_profile::PaidWorkPolicy; +use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; +use hellas_work::work_store::{Role, SetupStore, discover_setups}; +use serde::Deserialize; + +type CliResult = anyhow::Result; + +/// How many validator RPCs a write names. +/// +/// Reads come from a follower and writes are fanned to all six; a +/// configuration naming five has one validator whose acceptance this +/// node can never win, and one naming seven names something this +/// deployment does not have. +pub const VALIDATOR_COUNT: usize = 6; + +/// One operator's complete paid-work configuration, loaded and structurally +/// checked. +/// +/// A plain record with public fields, for [`WorkChannelConfig`]'s +/// reason: this is the shape a file fills in. Every file-local gate has +/// already run in [`load_work_config`]; the serve path then runs +/// [`validate_work_routes`] against the journals that must exist when it +/// starts. Provisioning shares the file loader before it creates one, which is +/// why disk agreement is not pretended to be a parse-time fact. +/// +/// [`WorkChannelConfig`]: hellas_rpc::protocol::work_setup::WorkChannelConfig +#[derive(Clone, Debug)] +#[allow( + dead_code, + reason = "the fields a mount consumes are read by the node runner; loading and checking them is this half" +)] +pub struct WorkConfig { + /// The chain this node believes it is configured against. + pub chain: ChainCrossCheck, + /// The six validator RPC URLs every write is fanned to. + pub validators: Vec, + /// Directory holding the setup and channel journals. + pub journal_root: PathBuf, + /// Bilateral setup routes, keyed by the authenticated transport peer. + pub routes: WorkRoutes, + /// Salt of the private credit-policy commitment. + pub policy_salt: [u8; 32], + /// The credit policy this provider will work under. + pub channel_policy: PaidChannelPolicyV1, + /// The execution policy this provider will run jobs under. + pub execution_policy: PaidWorkPolicy, + /// How often the watcher asks the chain for the next block. + pub poll: Duration, + /// The payment edge's value, reserve, and close fees as this provider + /// requires a client to fund them. + pub expected_payment_values: EdgeValues, + /// The shortest response window this provider signs terms over. + pub min_omit_response_blocks: u64, +} + +/// One bilateral setup route written in the paid-work configuration. +/// +/// The bond names the provider setup journal under [`WorkConfig::journal_root`]. +/// The client key is repeated here deliberately: startup compares it with the +/// taker committed inside that journal, turning a stale or mistyped route into +/// a refusal before the node binds. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct WorkRoute { + /// The transport-authenticated peer allowed to reach this bond. + pub peer: PeerId, + /// The bond whose provider setup journal this route names. + pub bond: EdgeId, + /// The settlement key the bond terms must name as taker. + pub client: Key, +} + +/// Paid-work routes keyed by their authenticated peer. +/// +/// Construction is private to the checked file loader. In particular, there +/// is no insertion API through which a caller could recreate last-one-wins +/// handling after duplicate peers and bonds have been refused. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct WorkRoutes { + by_peer: BTreeMap, +} + +impl WorkRoutes { + /// Returns every configured route in peer order. + pub fn iter(&self) -> impl Iterator { + self.by_peer.values() + } + + /// Returns how many bilateral routes were configured. + #[must_use] + pub fn len(&self) -> usize { + self.by_peer.len() + } + + /// Returns whether no bilateral route was configured. + #[must_use] + pub fn is_empty(&self) -> bool { + self.by_peer.is_empty() + } + + fn from_files(files: Vec) -> CliResult { + let mut by_peer = BTreeMap::new(); + let mut bonds = BTreeSet::new(); + for file in files { + let peer = PeerId::from_bytes(parse_fixed_hex("routes[].peer", &file.peer)?); + let bond = EdgeId::from_bytes(parse_fixed_hex("routes[].bond", &file.bond)?); + let client = Key::from_bytes(parse_fixed_hex("routes[].client", &file.client)?); + let route = WorkRoute { peer, bond, client }; + if by_peer.insert(peer, route).is_some() { + bail!("routes names peer {peer:#} twice; one authenticated peer has one route"); + } + if !bonds.insert(bond) { + bail!( + "routes names bond {} twice; one provider journal has one route", + hex::encode(bond.to_bytes()), + ); + } + } + Ok(Self { by_peer }) + } +} + +impl WorkConfig { + /// The provider policy this configuration makes. + /// + /// Every field is the operator's: the four the policy commits to, + /// the funding it expects, and the window it insists on. Nothing is + /// measured and nothing is inferred, so a node with a configuration + /// has a policy and countersigns over it. + #[must_use] + pub fn provider_policy(&self) -> ProviderChannelPolicy { + ProviderChannelPolicy { + network: self.chain.network, + policy_salt: self.policy_salt, + channel_policy: self.channel_policy, + execution_policy: self.execution_policy.clone(), + expected_payment_values: self.expected_payment_values, + min_omit_response_blocks: self.min_omit_response_blocks, + } + } +} + +/// The three fields that say which chain this is. +#[derive(Clone, Debug, PartialEq, Eq)] +#[allow( + dead_code, + reason = "the fields a mount consumes are read by the node runner; loading and checking them is this half" +)] +pub struct ChainCrossCheck { + /// The network every signature on this node's channels is bound to. + pub network: NetworkId, + /// Payload digest of the genesis block this deployment began at. + pub genesis_payload_digest: Digest, + /// The threshold identity finalized blocks are verified under. + pub threshold_identity: Vec, +} + +/// Loads and checks one paid-work configuration file. +/// +/// Every failure is a startup failure naming the field that failed, for +/// the reason §4 gives: a node that started with an unreadable +/// configuration would be one whose first symptom is an unsettleable +/// channel. +/// +/// # Errors +/// +/// The read and the parse, and then: a network id that is not one, a +/// digest that is not thirty-two bytes, a threshold identity consensus +/// cannot decode, a validator list that is not exactly +/// [`VALIDATOR_COUNT`] URLs with distinct normalised forms, an execution +/// policy the protocol's own [`check_execution_policy`] rejects, an +/// empty journal root, a route field of the wrong width, duplicate peers or +/// bonds in the route table, a zero poll cadence, and a response window +/// under the kernel's own minimum. +pub fn load_work_config(path: &Path) -> CliResult { + let bytes = fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; + let file: WorkConfigFile = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse {}", path.display()))?; + file.into_config() + .with_context(|| format!("invalid work config {}", path.display())) +} + +/// Verifies that every configured route names this root's provider journal +/// and the client settlement key committed by its bond terms. +/// +/// This is a serve-startup check rather than part of [`load_work_config`]: +/// provisioning uses the same configuration loader before it creates a +/// journal, while a serving node must already have every journal it promises. +/// Discovery comes first so [`SetupStore::open`] is never allowed to create a +/// missing journal merely because a route named its bond. +/// +/// # Errors +/// +/// The root cannot be enumerated, a route's provider journal is absent from +/// that root or cannot be opened, the journal holds no bond proposal, or its +/// bond names a taker other than the route's configured client. +pub fn validate_work_routes(config: &WorkConfig) -> CliResult<()> { + if config.routes.is_empty() { + return Ok(()); + } + let found = discover_setups(&config.journal_root, config.chain.network).with_context(|| { + format!( + "failed to enumerate configured work routes under journal.root {}", + config.journal_root.display(), + ) + })?; + for route in config.routes.iter() { + if !found + .setups + .iter() + .any(|setup| setup.role == Role::Provider && setup.bond_edge == route.bond) + { + bail!( + "route for peer {:#} names bond {}, but its provider setup journal is not under \ + journal.root {}", + route.peer, + hex::encode(route.bond.to_bytes()), + config.journal_root.display(), + ); + } + let store = SetupStore::open( + &config.journal_root, + config.chain.network, + route.bond, + Role::Provider, + &Secp256k1Verifier::new(), + ) + .with_context(|| { + format!( + "route for peer {:#} could not open provider setup journal for bond {} under {}", + route.peer, + hex::encode(route.bond.to_bytes()), + config.journal_root.display(), + ) + })?; + let Some(bundle) = store.state().bundle() else { + bail!( + "route for peer {:#} names provider setup journal for bond {}, but it holds no \ + bond proposal", + route.peer, + hex::encode(route.bond.to_bytes()), + ); + }; + let journal_client = bundle.bond_terms().parties.taker(); + if journal_client != route.client { + bail!( + "route for peer {:#} expects client settlement key {}, but provider setup journal \ + for bond {} names {} as its taker", + route.peer, + hex::encode(route.client.to_bytes()), + hex::encode(route.bond.to_bytes()), + hex::encode(journal_client.to_bytes()), + ); + } + } + Ok(()) +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkConfigFile { + chain: ChainFile, + validators: Vec, + journal: JournalFile, + routes: Vec, + policies: PoliciesFile, + /// How often the watcher asks the chain for the next block. + poll_ms: u64, + expected_payment_values: PaymentValuesFile, + min_omit_response_blocks: u64, +} + +impl WorkConfigFile { + fn into_config(self) -> CliResult { + let Some(network) = NetworkId::new(self.chain.network_id.trim()) else { + bail!( + "chain.network_id {:?} is not a network id", + self.chain.network_id + ); + }; + let threshold_identity = + parse_hex("chain.threshold_identity", &self.chain.threshold_identity)?; + // Parsed before the verifier is built, so the list consensus is + // handed is the normalised one this node will actually dial. + let validators = parse_validators(self.validators)?; + // The same constructor consensus verification uses. A threshold + // identity that cannot be decoded here is one no finalized block + // would ever verify under, and the node says so before it serves. + hellas_chain::ConsensusVerifier::new(&hellas_chain::light_client::ConsensusInfo { + validators: validators.clone(), + threshold_identity: threshold_identity.clone(), + network_id: self.chain.network_id.clone(), + }) + .map_err(|error| anyhow::anyhow!("chain.threshold_identity is not usable: {error}"))?; + + let journal_root = self.journal.into_root()?; + let routes = WorkRoutes::from_files(self.routes)?; + let policies = self.policies.into_policies()?; + if self.poll_ms == 0 { + bail!("poll_ms must be greater than zero"); + } + // The kernel refuses a shorter window at every payment open, so a + // configuration under it would sign terms consensus then throws + // away. + if self.min_omit_response_blocks < MIN_OMIT_RESPONSE_BLOCKS { + bail!( + "min_omit_response_blocks {} is under the kernel's minimum {MIN_OMIT_RESPONSE_BLOCKS}", + self.min_omit_response_blocks, + ); + } + + Ok(WorkConfig { + chain: ChainCrossCheck { + network, + genesis_payload_digest: parse_digest( + "chain.genesis_payload_digest", + &self.chain.genesis_payload_digest, + )?, + threshold_identity, + }, + validators, + journal_root, + routes, + policy_salt: policies.0, + channel_policy: policies.1, + execution_policy: policies.2, + poll: Duration::from_millis(self.poll_ms), + expected_payment_values: self.expected_payment_values.into_values(), + min_omit_response_blocks: self.min_omit_response_blocks, + }) + } +} + +/// One bilateral route exactly as the operator writes it. +/// +/// All three values are fixed-width lowercase-or-uppercase hexadecimal on +/// input and canonical byte values after loading. A peer or bond written in a +/// second spelling is therefore still the same key for duplicate detection. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkRouteFile { + peer: String, + bond: String, + client: String, +} + +/// Parses the six validator RPC URLs, and refuses anything that is not +/// one. +/// +/// Both halves matter. A string that is not a URL is not an address this +/// node can ever fan a write to, and "trimmed and non-empty" admits `not +/// a URL` verbatim — a configuration whose first symptom would be five +/// validators answering and one that never does. And uniqueness is a +/// question about *addresses*, not about spellings: `HTTP://Host:443/` +/// and `http://host:443/` are one validator written twice, and a fan-out +/// to five validators is not six however it is spelled. So the +/// comparison is between parsed, normalised URLs, and the normalised +/// forms are what is kept. +/// +/// A host is required, because these are dialled: a `mailto:` or a +/// `data:` parses perfectly well and is not a validator. +fn parse_validators(entries: Vec) -> CliResult> { + let mut validators: Vec = Vec::with_capacity(VALIDATOR_COUNT); + for entry in entries { + let entry = entry.trim(); + if entry.is_empty() { + bail!("validators entries must be non-empty"); + } + let url = url::Url::parse(entry) + .with_context(|| format!("validators entry {entry:?} is not a URL"))?; + if url.host_str().is_none() { + bail!("validators entry {entry:?} names no host to dial"); + } + let normalised = url.as_str().to_string(); + if validators.contains(&normalised) { + bail!("validators names {normalised} twice; a fan-out to five validators is not six"); + } + validators.push(normalised); + } + if validators.len() != VALIDATOR_COUNT { + bail!( + "validators must name exactly {VALIDATOR_COUNT} validator URLs, found {}", + validators.len(), + ); + } + Ok(validators) +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ChainFile { + network_id: String, + genesis_payload_digest: String, + threshold_identity: String, +} + +/// Where the work journals live. +/// +/// A root and nothing else. How large a journal may grow is not an +/// operator's to say: the active and checkpoint ceilings are constants +/// the journal enforces on itself, so a cap here would be a number +/// written down and ignored. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct JournalFile { + root: PathBuf, +} + +impl JournalFile { + fn into_root(self) -> CliResult { + if self.root.as_os_str().is_empty() { + bail!("journal.root must be a path"); + } + Ok(self.root) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct PoliciesFile { + policy_salt: String, + channel: ChannelPolicyFile, + execution: Option, + fetch: Option, +} + +impl PoliciesFile { + fn into_policies(self) -> CliResult<([u8; 32], PaidChannelPolicyV1, PaidWorkPolicy)> { + let salt = parse_fixed_hex("policies.policy_salt", &self.policy_salt)?; + Ok(( + salt, + PaidChannelPolicyV1 { + compute_credit_limit: self.channel.compute_credit_limit, + delivery_credit_limit: self.channel.delivery_credit_limit, + }, + match (self.execution, self.fetch) { + (Some(execution), None) => execution.into_policy()?.into(), + (None, Some(fetch)) => fetch.into_policy()?, + _ => bail!("policies must select exactly one of execution or fetch"), + }, + )) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ChannelPolicyFile { + compute_credit_limit: u64, + delivery_credit_limit: u64, +} + +/// The execution policy, field for field. +/// +/// Spelled out rather than flattened from some smaller shape because +/// every one of these is a value both parties sign: a default here would +/// be this node quietly proposing a policy its operator never wrote. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ExecutionPolicyFile { + allowed_environment: String, + generation_policy_digest: String, + identity_source_digest: String, + max_prompt_tokens: u32, + max_new_tokens: u32, + max_stop_token_ids: u16, + max_spool_bytes: u64, + max_encoded_result_frame: u32, + max_encoded_quote_response: u32, + dispatch_margin_blocks: u64, + delivery_margin_blocks: u64, + oracle_grace_blocks: u64, + fixed_price: u64, +} + +impl ExecutionPolicyFile { + fn into_policy(self) -> CliResult { + let allowed_environment: ContentId = + self.allowed_environment.parse().with_context(|| { + format!( + "policies.execution.allowed_environment {:?} is not a ContentId", + self.allowed_environment + ) + })?; + let policy = PaidExecutionPolicyV1 { + allowed_environment, + generation_policy_digest: parse_digest( + "policies.execution.generation_policy_digest", + &self.generation_policy_digest, + )?, + identity_source_digest: parse_digest( + "policies.execution.identity_source_digest", + &self.identity_source_digest, + )?, + max_prompt_tokens: self.max_prompt_tokens, + max_new_tokens: self.max_new_tokens, + max_stop_token_ids: self.max_stop_token_ids, + max_spool_bytes: self.max_spool_bytes, + max_encoded_result_frame: self.max_encoded_result_frame, + max_encoded_quote_response: self.max_encoded_quote_response, + dispatch_margin_blocks: self.dispatch_margin_blocks, + delivery_margin_blocks: self.delivery_margin_blocks, + oracle_grace_blocks: self.oracle_grace_blocks, + fixed_price: self.fixed_price, + }; + // The protocol's own gate, run here rather than at the first + // admission. A zero here is not a small bound, it is an absent + // one — a zero margin gives a deadline no time to be met in, and + // a zero price is a job nobody is paid for. Copying the fields + // through unchecked moves that discovery to the moment a + // counterparty is already waiting on a proposal. + check_execution_policy(&policy) + .map_err(|error| anyhow::anyhow!("policies.execution is not usable: {error}"))?; + Ok(policy) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct FetchPolicyFile { + allowed_environment: String, + service: Option, + method: Option, + open_fetch: Option, + max_request_body_bytes: u32, + max_output_events: u32, + max_output_bytes: u32, + max_spool_bytes: u64, + max_encoded_result_frame: u32, + max_encoded_prepared_input: u32, + dispatch_margin_blocks: u64, + delivery_margin_blocks: u64, + oracle_grace_blocks: u64, + fixed_price: u64, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct OpenFetchPolicyFile { + #[serde(default)] + require_spki_pin: bool, + #[serde(default)] + allowed_hosts: Vec, +} + +impl FetchPolicyFile { + fn into_policy(self) -> CliResult { + let route = match (self.service, self.method, self.open_fetch) { + (Some(service), Some(method), None) => { + PaidFetchRoutePolicy::sealed_route(service, method)? + } + (None, None, Some(open)) => { + PaidFetchRoutePolicy::open_fetch(open.require_spki_pin, open.allowed_hosts) + } + _ => bail!("policies.fetch requires service+method or open_fetch"), + }; + let policy = PaidFetchPolicyV1 { + allowed_environment: self + .allowed_environment + .parse() + .context("policies.fetch.allowed_environment is not a ContentId")?, + route_commitment: fetch_route_commitment(&route.canonical_body_bytes())?, + max_request_body_bytes: self.max_request_body_bytes, + max_output_events: self.max_output_events, + max_output_bytes: self.max_output_bytes, + max_spool_bytes: self.max_spool_bytes, + max_encoded_result_frame: self.max_encoded_result_frame, + max_encoded_prepared_input: self.max_encoded_prepared_input, + dispatch_margin_blocks: self.dispatch_margin_blocks, + delivery_margin_blocks: self.delivery_margin_blocks, + oracle_grace_blocks: self.oracle_grace_blocks, + fixed_price: self.fixed_price, + }; + let profile = PaidWorkPolicy::Fetch { policy, route }; + profile.check().context("policies.fetch is not usable")?; + Ok(profile) + } +} + +/// The payment edge's funding, field for field, as this provider requires +/// a client to fund it. +/// +/// Spelled out for [`ExecutionPolicyFile`]'s reason: every one of these +/// bounds what a certificate on the channel may name, so a default here +/// would be this node quietly accepting funding its operator never +/// priced. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct PaymentValuesFile { + value: u64, + reserve: u64, + close_fees: CloseFeesFile, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct CloseFeesFile { + base: u64, + slot: u64, + proof: u64, + lifetime: u64, +} + +impl PaymentValuesFile { + fn into_values(self) -> EdgeValues { + EdgeValues::new( + self.value, + self.reserve, + Fees::new( + self.close_fees.base, + self.close_fees.slot, + self.close_fees.proof, + self.close_fees.lifetime, + ), + ) + } +} + +fn parse_hex(field: &str, raw: &str) -> CliResult> { + let bytes = hex::decode(raw.trim()).with_context(|| format!("{field} is not hexadecimal"))?; + if bytes.is_empty() { + bail!("{field} must not be empty"); + } + Ok(bytes) +} + +fn parse_fixed_hex(field: &str, raw: &str) -> CliResult<[u8; N]> { + let bytes = parse_hex(field, raw)?; + let Ok(bytes) = <[u8; N]>::try_from(bytes.as_slice()) else { + bail!("{field} must be {N} bytes, found {}", bytes.len()); + }; + Ok(bytes) +} + +fn parse_digest(field: &str, raw: &str) -> CliResult { + Ok(Digest::from_bytes(parse_fixed_hex(field, raw)?)) +} diff --git a/crates/sdk/src/work_provision.rs b/crates/sdk/src/work_provision.rs new file mode 100644 index 00000000..dc1d22ee --- /dev/null +++ b/crates/sdk/src/work_provision.rs @@ -0,0 +1,574 @@ +//! Making the offers a fresh provider has nothing to serve without. +//! +//! `WorkRunner::discover` answers `WorkSetup` from the setup journals it +//! finds under the configured work root, and finding is the whole of what +//! it does. A correctly configured provider with no journal therefore +//! refuses every client that dials it, and the paid path is unreachable +//! from a clean install. This is the operator's step that writes them. +//! +//! # The order is the journal's, and none of its rules are here +//! +//! Three library calls. [`SetupStore`] is opened as the provider's half +//! of one bond, the immutable history floor is armed, and +//! [`SetupEndpoint::propose_bond`] signs the stake and journals it before +//! there is anything to export. +//! +//! Arming is first because it has to be: the setup state refuses +//! "recording revision 1 before arming its scan floor", so the floor is +//! not a preparation this command chose to do early but the step every +//! later one is refused before. The floor is a finalized height and the +//! payload digest at it, and the setup's own history must name that +//! digest as the parent of its first block. So it is read from a +//! validator rather than written down by an operator: a floor naming a +//! block this chain does not have is a setup whose history can never be +//! contiguous, and nothing later would say so out loud. +//! +//! # Exit means durable +//! +//! [`SetupStore::commit`] fsyncs a revision before it returns, and this +//! command still reopens the journal and replays it before printing +//! anything. That reopen is the one `WorkRunner::discover` will do, run +//! early: an operator told the offer exists has been told about the disk, +//! and about a file whose exclusive lock is already free for the runner +//! to take. +//! +//! # One recourse backs one route +//! +//! A provider offer reserves a route, a bond, and every coin funding that +//! bond. A second offer is safe only when all three are disjoint from every +//! provider offer already under the root. Existing peers come from the +//! durable route table, while existing coins come from the bond funding in +//! each retained setup bundle. Revision one is enough: it holds the funding +//! before a client has answered, while [`SetupState::funding_coins`] is still +//! empty because there is no executable Open yet. +//! +//! Discovery, route agreement, and funding comparison all happen while the +//! candidate is only a value. The candidate journal is not opened until +//! afterwards, so every collision is refused before a floor is written or a +//! bond signature is made. +//! +//! # What the operator chooses, and what is built +//! +//! Every number in the bond is the operator's and this command invents +//! none of them. Two parts of the shape are not choices: a stake bond is +//! funded by its maker alone, so the taker's side of the funding is +//! empty, and its timeout pays the staking party and nobody else, so +//! there is one payout and it names the provider's own key. A second +//! payout to that same party would only raise the close cost the payout +//! has to clear. The kernel checks the rest when the Open reaches it — +//! that the payout total is the edge's close value, that the price cap +//! covers a job, that the timeout is ahead of the block including it — +//! and re-spelling any of that here would be a second answer to a +//! question consensus already answers. +//! +//! [`SetupState::funding_coins`]: hellas_work::work_store::SetupState::funding_coins + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use anyhow::{Context as _, bail}; +use hellas_chain::client::VerifiedRemoteLightClient; +use hellas_chain::domain::MAX_EDGE_LIFETIME_BLOCKS; +use hellas_chain::{ConsensusInfo, ConsensusVerifier, WorkBlocks}; +use hellas_kernel::{ + BlockHeight, CoinId, EdgeId, Funding, Key, List, MAX_EDGE_OUTPUTS, MAX_PARTY_INPUTS, NetworkId, + Parties, Payout, Secp256k1Signer, Secp256k1Verifier, Terms, Tx, WorkStakeBondTerms, +}; +use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; +use hellas_work::work_close::FinalizedBlocks; +use hellas_work::work_handshake::{PaymentAdmission, SetupEndpoint}; +use hellas_work::work_store::{Role, SetupScan, SetupStore, discover_setups}; +use tracing::{info, warn}; + +use crate::work_config::{WorkConfig, WorkRoute}; +type CliResult = anyhow::Result; + +/// What an operator asks for when they make one offer. +pub struct ProvisionOptions { + /// The loaded paid-work configuration, not the path it came from. It + /// carries the network the bond is bound to, the root the journal is + /// written under, the validators the floor is read from, and the + /// artifact the provider's policy rests on. + pub work_config: WorkConfig, + /// The key this provider stakes and signs the bond with, read from + /// the identity the operator already has and never made here. + pub settlement_key: Secp256k1Signer, + /// The client this bond names as taker, hex-encoded. + pub client: String, + /// The coins this provider stakes, hex-encoded. + pub stake_coins: Vec, + /// Height the bond expires at, which is also the admission horizon of + /// the channel it insures. + pub bond_timeout: u64, + /// What the bond's timeout returns to the staking provider. + pub timeout_payout: u64, + /// The largest job price this bond covers. + pub max_job_price: u64, + /// Print the deterministic bond edge and stop before any external read or write. + pub print_bond_only: bool, +} + +/// Makes one offer, and says where it is. +/// +/// # Errors +/// +/// A configuration with no matching bilateral route, a route, bond, or funding coin already reserved by another offer, +/// a key or coin id that is not one, no configured validator with a finalized +/// block to read a floor from, and whatever the setup journal says about the +/// revision it refused or could not make durable. +pub async fn run_provision(options: ProvisionOptions) -> CliResult<()> { + // The candidate is the one source of the bond edge for both preview and + // provisioning. Keep this before evidence, routing, validators and the + // journal: the preview exists so an operator can put this value into the + // route table those later steps require. + let candidate = BondCandidate::plan(&options)?; + if options.print_bond_only { + println!("bond_edge: {}", hex::encode(candidate.bond_edge.to_bytes())); + return Ok(()); + } + let offer = Offer::plan(&options, options.work_config.provider_policy(), candidate)?; + // Dialled after every refusal that can be made without a chain, and + // before the journal exists: a floor is the first thing written into + // it, so a run that cannot read one leaves no half-made offer behind. + let made = offer.journal(finalized_floor(&options.work_config).await?)?; + + println!( + "offer journaled: bond {} under {}", + hex::encode(made.bond_edge.to_bytes()), + options.work_config.journal_root.display(), + ); + // The floor read back out of the journal rather than the one just + // dialled, because those differ on a retry and the durable one is the + // one this setup's history will be measured against. + println!( + "history floor: finalized height {} with payload {}", + made.floor.height, + hex::encode(made.floor.payload), + ); + Ok(()) +} + +/// Compute the bond before the operator adds its bilateral route. +pub fn preview_bond(options: &ProvisionOptions) -> CliResult { + Ok(BondCandidate::plan(options)?.bond_edge) +} + +/// Sign and journal an offer under an existing provider identity. +pub async fn provision_offer(options: ProvisionOptions) -> CliResult { + let candidate = BondCandidate::plan(&options)?; + let offer = Offer::plan(&options, options.work_config.provider_policy(), candidate)?; + offer.journal(finalized_floor(&options.work_config).await?) +} + +/// One offer as the disk holds it, read back after it was written. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Provisioned { + /// The bond this journal is keyed to, which is what discovery names + /// it by. + pub bond_edge: EdgeId, + /// The floor its history starts above, as retained. + pub floor: SetupScan, +} + +/// The deterministic bond inputs, built without evidence, routing, a chain, +/// or a journal. +/// +/// Preview and real provisioning both pass through this value. In particular, +/// the real path does not recompute the edge after printing it, so a preview +/// cannot drift from the offer later signed. +struct BondCandidate { + network: NetworkId, + journal_root: PathBuf, + bond_edge: EdgeId, + bond_funding: Funding, + bond_terms: WorkStakeBondTerms, + settlement_key: Secp256k1Signer, +} + +impl BondCandidate { + fn plan(options: &ProvisionOptions) -> CliResult { + let network = options.work_config.chain.network; + let journal_root = options.work_config.journal_root.clone(); + // Maker is the provider and taker is the client, which is what + // makes this signature the maker's: `propose_bond` refuses a bond + // whose staking party this key is not. + let provider = options.settlement_key.party_key(); + let bond_terms = WorkStakeBondTerms { + parties: Parties::new( + provider, + Key::from_bytes(fixed::<{ Key::LENGTH }>("--client", &options.client)?), + ), + timeout: BlockHeight::new(options.bond_timeout), + timeout_outputs: List::take( + [Payout::new(provider, options.timeout_payout); MAX_EDGE_OUTPUTS], + 1, + ), + max_job_price: options.max_job_price, + }; + let bond_funding = Funding::new( + staked(&options.stake_coins)?, + List::empty(CoinId::from_bytes([0; CoinId::LENGTH])), + ); + let bond_edge = Tx::edge_id_of(&bond_funding, &Terms::work_stake_bond(bond_terms.clone())); + Ok(Self { + network, + journal_root, + bond_edge, + bond_funding, + bond_terms, + settlement_key: options.settlement_key.clone(), + }) + } +} + +/// One offer, decided before anything is dialled or written. +struct Offer { + network: NetworkId, + journal_root: PathBuf, + bond_edge: EdgeId, + bond_funding: Funding, + bond_terms: WorkStakeBondTerms, + admission: PaymentAdmission, + settlement_key: Secp256k1Signer, +} + +impl Offer { + /// Reads the operator's answers, and refuses everything refusable + /// without a chain. + fn plan( + options: &ProvisionOptions, + policy: ProviderChannelPolicy, + candidate: BondCandidate, + ) -> CliResult { + let admission = PaymentAdmission::Admits(Box::new(policy)); + let BondCandidate { + network, + journal_root, + bond_edge, + bond_funding, + bond_terms, + settlement_key, + } = candidate; + let route = route_for_candidate(&options.work_config, bond_edge, &bond_terms)?; + refuse_offer_collisions(&options.work_config, route, &bond_funding)?; + Ok(Self { + network, + journal_root, + bond_edge, + bond_funding, + bond_terms, + admission, + settlement_key, + }) + } + + /// Journals revision 1, and returns only once a fresh open of the + /// journal replays it. + fn journal(self, floor: SetupScan) -> CliResult { + let timeout = self.bond_terms.timeout.get(); + anyhow::ensure!( + timeout > floor.height, + "bond timeout must be after finalized height {}", + floor.height + ); + anyhow::ensure!( + timeout - floor.height <= MAX_EDGE_LIFETIME_BLOCKS, + "bond timeout exceeds the chain maximum lifetime" + ); + let Self { + network, + journal_root, + bond_edge, + bond_funding, + bond_terms, + admission, + settlement_key, + } = self; + { + let store = open_provider_journal(&journal_root, network, bond_edge)?; + let mut endpoint = SetupEndpoint::new(store, settlement_key, admission); + // The floor is immutable and the store writes exactly one arm + // of it, so a run that arms and then fails keeps the height + // its successor starts from rather than moving it. + if let Some(held) = endpoint.state().scan_armed() { + info!( + height = held.height, + "this journal already holds its history floor, and a floor does not move", + ); + } else { + endpoint + .arm_scan(floor) + .context("failed to make this setup's immutable history floor durable")?; + } + endpoint + .propose_bond(network, bond_funding, bond_terms) + .context("failed to sign and journal the bond proposal")?; + } + + // The journal is closed above, so this is a second process's view + // of it: the same replay and the same signature checks the runner + // runs, before an operator is told there is anything to run them + // on. + let reopened = open_provider_journal(&journal_root, network, bond_edge)?; + let state = reopened.state(); + let (Some(1), Some(floor)) = (state.revision(), state.scan_armed()) else { + bail!( + "the journal under {} replays as revision {:?} over floor {:?}, not the armed \ + proposal that was just written", + journal_root.display(), + state.revision(), + state.scan_armed().map(|scan| scan.height), + ); + }; + Ok(Provisioned { bond_edge, floor }) + } +} + +fn open_provider_journal( + root: &Path, + network: NetworkId, + bond_edge: EdgeId, +) -> CliResult { + SetupStore::open( + root, + network, + bond_edge, + Role::Provider, + &Secp256k1Verifier::new(), + ) + .with_context(|| { + format!( + "failed to open the provider setup journal for bond {} under {}", + hex::encode(bond_edge.to_bytes()), + root.display(), + ) + }) +} + +/// Returns the configured bilateral route the candidate would occupy. +/// +/// The bond is derived from the exact funding and terms first. Matching by +/// that canonical value means a route cannot be selected by insertion order, +/// and checking the client here refuses a journal the next startup would +/// reject before the provider signs it. +fn route_for_candidate<'config>( + config: &'config WorkConfig, + bond_edge: EdgeId, + bond_terms: &WorkStakeBondTerms, +) -> CliResult<&'config WorkRoute> { + let Some(route) = config.routes.iter().find(|route| route.bond == bond_edge) else { + bail!( + "bond {} has no bilateral route in this work configuration; an offer is signed only \ + after its peer, bond, and client are named together", + hex::encode(bond_edge.to_bytes()), + ); + }; + let client = bond_terms.parties.taker(); + if route.client != client { + bail!( + "route for peer {:#} expects client {}, but candidate bond {} names {} as its taker", + route.peer, + hex::encode(route.client.to_bytes()), + hex::encode(bond_edge.to_bytes()), + hex::encode(client.to_bytes()), + ); + } + Ok(route) +} + +/// Refuses every collision before the candidate journal is opened. +/// +/// An existing bond is named by discovery, its peer is named by the durable +/// route table, and its funding is named by the retained bundle. Failure to +/// recover any one of those facts is a refusal: absence of evidence is not +/// evidence that the candidate is disjoint. +fn refuse_offer_collisions( + config: &WorkConfig, + candidate: &WorkRoute, + candidate_funding: &Funding, +) -> CliResult<()> { + let root = &config.journal_root; + let network = config.chain.network; + let found = discover_setups(root, network).with_context(|| { + format!( + "failed to enumerate the work journals under {}", + root.display(), + ) + })?; + for unnamed in &found.unidentified { + warn!( + path = %unnamed.path.display(), + reason = %unnamed.reason, + "a setup journal under the work root could not be named", + ); + } + if let Some(unnamed) = found.unidentified.first() { + bail!( + "setup journal {} cannot be identified, so a new offer cannot be proved disjoint: {}", + unnamed.path.display(), + unnamed.reason, + ); + } + + let candidate_coins = funding_coins(candidate_funding); + for held in found + .setups + .iter() + .filter(|setup| setup.role == Role::Provider) + { + if held.bond_edge == candidate.bond { + bail!( + "candidate bond {} collides with a provider offer already under {}", + hex::encode(candidate.bond.to_bytes()), + root.display(), + ); + } + let Some(route) = config + .routes + .iter() + .find(|route| route.bond == held.bond_edge) + else { + bail!( + "provider offer over bond {} under {} has no configured route, so the candidate \ + route cannot be proved disjoint", + hex::encode(held.bond_edge.to_bytes()), + root.display(), + ); + }; + let store = open_provider_journal(root, network, held.bond_edge)?; + let Some(bundle) = store.state().bundle() else { + bail!( + "provider offer over bond {} was discovered without a retained revision", + hex::encode(held.bond_edge.to_bytes()), + ); + }; + let held_client = bundle.bond_terms().parties.taker(); + if route.client != held_client { + bail!( + "route for peer {:#} expects client {}, but provider offer over bond {} names {} \ + as its taker", + route.peer, + hex::encode(route.client.to_bytes()), + hex::encode(held.bond_edge.to_bytes()), + hex::encode(held_client.to_bytes()), + ); + } + if route.peer == candidate.peer { + bail!( + "candidate route peer {:#} collides with the provider offer over bond {}", + candidate.peer, + hex::encode(held.bond_edge.to_bytes()), + ); + } + // The retained revision's own staked funding, not the executable + // Opens: the provider signed these coins when it made the offer, so + // they are promised from that moment, while `funding_coins` answers + // from Opens that do not exist until the client countersigns. Read + // from there, every offer no client has answered would look like it + // reserved nothing. + let reserved = funding_coins(bundle.bond_funding()); + if let Some(coin) = candidate_coins.intersection(&reserved).next() { + bail!( + "candidate stake coin {} is already reserved by provider offer over bond {}", + hex::encode(coin.to_bytes()), + hex::encode(held.bond_edge.to_bytes()), + ); + } + } + Ok(()) +} + +/// Every input one bond funding consumes, irrespective of party position. +fn funding_coins(funding: &Funding) -> BTreeSet { + funding + .maker() + .iter() + .chain(funding.taker().iter()) + .copied() + .collect() +} + +/// Reads one finalized block from the first configured validator that +/// answers, as the floor this setup's history starts above. +async fn finalized_floor(config: &WorkConfig) -> CliResult { + let verifier = ConsensusVerifier::new(&ConsensusInfo { + validators: config.validators.clone(), + threshold_identity: config.chain.threshold_identity.clone(), + network_id: config.chain.network.as_str().to_owned(), + }) + .context("the configured threshold identity is not usable")?; + for url in &config.validators { + let client = match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { + Ok(client) => client, + Err(error) => { + warn!(validator = %url, %error, "a configured validator did not answer"); + continue; + } + }; + match floor_of(&WorkBlocks::new(client)).await { + Ok(Some(floor)) => { + info!(validator = %url, height = floor.height, "the history floor was read here"); + return Ok(floor); + } + Ok(None) => warn!(validator = %url, "a configured validator has finalized nothing"), + Err(error) => warn!(validator = %url, %error, "a configured validator did not answer"), + } + } + bail!("no configured validator answered with a finalized block to floor this offer at") +} + +/// Returns the finalized tip as a scan floor, or `None` before anything +/// is finalized. +/// +/// The height and the payload come from one block rather than from two +/// reads, because the setup's first history block must name that exact +/// payload as its parent. +async fn floor_of(blocks: &B) -> CliResult> +where + B: FinalizedBlocks + ?Sized, +{ + let Some(height) = blocks.latest_height().await? else { + return Ok(None); + }; + let Some(block) = blocks.block_at(height).await? else { + return Ok(None); + }; + Ok(Some(SetupScan { + height: block.height, + payload: block.payload, + })) +} + +/// Reads the coins one provider stakes. +fn staked(ids: &[String]) -> CliResult> { + let mut slots = [CoinId::from_bytes([0; CoinId::LENGTH]); MAX_PARTY_INPUTS]; + for (slot, id) in slots.iter_mut().zip(ids) { + *slot = CoinId::from_bytes(fixed::<{ CoinId::LENGTH }>("--stake-coin", id)?); + } + // The zip above stops at the shorter side, so a list the array cannot + // hold is refused here rather than silently staking the first four of + // it. + List::new(slots, ids.len()).with_context(|| { + format!( + "--stake-coin names {} coins, and one party funds an open with at most \ + {MAX_PARTY_INPUTS}", + ids.len(), + ) + }) +} + +/// Reads exactly `N` bytes of hex, or says which flag was not that. +fn fixed(flag: &str, value: &str) -> CliResult<[u8; N]> { + let bytes = + hex::decode(value).with_context(|| format!("{flag} {value:?} is not hex-encoded bytes"))?; + let Ok(fixed) = <[u8; N]>::try_from(bytes.as_slice()) else { + bail!( + "{flag} {value:?} is {} bytes, and {N} are wanted", + bytes.len() + ); + }; + Ok(fixed) +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/commands/serve/provision/tests.rs b/crates/sdk/src/work_provision/tests.rs similarity index 99% rename from crates/cli/src/commands/serve/provision/tests.rs rename to crates/sdk/src/work_provision/tests.rs index 78818ae7..c521f533 100644 --- a/crates/cli/src/commands/serve/provision/tests.rs +++ b/crates/sdk/src/work_provision/tests.rs @@ -6,8 +6,8 @@ use hellas_rpc::protocol::work::{PaidChannelPolicyV1, PaidExecutionPolicyV1}; use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; use hellas_work::work_close::{BlockSourceError, FinalizedWork}; -use super::super::work_config::load_work_config; use super::*; +use crate::work_config::load_work_config; fn network() -> NetworkId { let Some(network) = NetworkId::new("hellas-devnet") else { @@ -66,7 +66,8 @@ fn policy() -> ProviderChannelPolicy { delivery_margin_blocks: 2, oracle_grace_blocks: 6, fixed_price: 10, - }, + } + .into(), expected_payment_values: EdgeValues::new(1_000, 200, Fees::new(0, 0, 0, 0)), min_omit_response_blocks: MIN_OMIT_RESPONSE_BLOCKS, } diff --git a/crates/work/README.md b/crates/work/README.md new file mode 100644 index 00000000..4efe558d --- /dev/null +++ b/crates/work/README.md @@ -0,0 +1,161 @@ +# Paid Fetch and provider payload retention + +Paid Fetch shares the existing channel authorization, execution gate, delivery, +payment certificate, and settlement flow with Evaluate. A Fetch channel selects +`PaidWorkPolicy::Fetch`; the policy commits to its route and manifest as well +as its fixed price and input/output limits. Evaluate's canonical records and +version-1 close descriptors remain unchanged. Fetch close descriptors use +version 2. + +## Provider storage + +The Fetch profile requires `Retention::Ephemeral` and a metadata-only provider +journal. Channel mounting selects that journal automatically; binding a Fetch +provider to an ordinary payload journal fails. Live requests and responses are +fully verified before their accounting records are committed. + +The journal persists authorizations, signatures, request/result digests, running +and delivery markers, payment certificates, ledger state, and close state. +It omits prepared requests and output transcripts from both append records and +rotation checkpoints. Those bodies exist in memory while the job is active. +The paid executor invokes the configured Fetch route directly, without passing +through Courtesy's transcript store or replay cache. Upstream and projection +faults are reduced to fixed error messages before they reach the paid-work +driver's logs. + +The client may store its own inputs and received responses. This policy covers +application-managed provider storage; it does not establish the upstream API's +retention policy or replace host memory/swap/crash-dump controls. + +## Restart behavior + +| Provider state before restart | Recovery | +| --- | --- | +| Accepted, not dispatched | Input is gone; do not dispatch automatically | +| Running | Outcome is indeterminate; do not invoke again | +| Result recorded | Signed result metadata survives; response body cannot be delivered or replayed | +| Already delivered | Client can still submit its signed payment; duplicate submissions are idempotent | +| Paid | Payment and settlement recovery continue from durable metadata | + +Existing deadline and close rules still apply to jobs whose bodies were lost. +Provider failure is not turned into a paid result. The client must not assume +that retrying delivery after a provider restart can recover its response. + +The journal header distinguishes payload journals from metadata-only journals. +Opening one as the other fails. This change does not migrate or erase old files. + +## CLI + +Build the CLI with `node,llm`. Use `paid-work prepare-fetch` to create a +client-owned signed request: + +```sh +hellas-cli paid-work prepare-fetch \ + --service openai --method responses \ + --execution-environment openai-responses \ + --payload-file request.json --out paid-fetch.bin +``` + +Use the same local identity when preparing the input and running +`paid-work run --prepared-input paid-fetch.bin`. The existing run command +handles setup, acceptance, delivery, authenticated payment, and optional +settlement; it still needs its existing provider/funding/config arguments. + +In the work config, replace `policies.execution` with `policies.fetch` (exactly +one is allowed): + +```json +{ + "allowed_environment": "<64 hex digits of the Fetch manifest ID>", + "service": "openai", + "method": "responses", + "max_request_body_bytes": 4096, + "max_output_events": 64, + "max_output_bytes": 16384, + "max_spool_bytes": 65536, + "max_encoded_result_frame": 65536, + "max_encoded_prepared_input": 65536, + "dispatch_margin_blocks": 4, + "delivery_margin_blocks": 2, + "oracle_grace_blocks": 6, + "fixed_price": 10 +} +``` + +The route must also exist in the provider's Fetch route configuration. +Route-wide capabilities and shared Fetch concurrency limits remain enforced. +Courtesy's caller quota policy is not the paid admission policy; a paid channel +authorizes its own work. + +## Generic HTTPS and App Attest + +`FetchEnvironment::Http` interprets the caller-signed HTTPS request described +in [the HTTPS guide](../providers/HTTPS.md). The body binds the full URL, +method, headers, binary request body, TLS roots, additional SPKI pins, account +alias, and response-size ceiling. The result signs HTTP status, headers and +exact response bytes. No upstream vendor needs to be compiled into Gate. + +To sell this interpreter, use its manifest ID as `allowed_environment` and +replace `service`/`method` in `policies.fetch` with: + +```json +"open_fetch": { + "require_spki_pin": false, + "allowed_hosts": [] +} +``` + +An empty list permits any public host, subject to the operator's independently +configured egress restrictions. A nonempty list matches exact host names. +The host and required-pin conditions are checked before paid acceptance. +The HTTPS driver enforces certificate validation and address restrictions at +connection time. A registered route such as `http/request` must implement the +HTTP manifest. The paid policy's byte/frame limits must cover the encoded HTTP +response as well as its envelopes (base64 expands binary bodies). + +Use `paid-work prepare-fetch --execution-environment http` for this request. +Add `--assurance apple-app-attest` when targeting Gate. On `paid-work run`, +provide `--provider-genesis`, `--apple-app-id` and `--apple-cd-hashes`. Each paid +Work and WorkSetup connection verifies a fresh Open proof before sending its +request. The proof binds the TLS exporter, nonce, service ALPN, enrollment and +producer key; the producer must also be the payment channel's provider. +Result verification takes the assurance from the client's signed input and +rejects a weaker output scheme. + +## Application interface + +`hellas-sdk` is the application boundary for both CLI and signed hosts: + +- `paid_client::PaidWorkSession` owns one funded channel. It authenticates the + provider, resumes journaled jobs, verifies results, pays, and closes. The CLI + and paid gateway use this same session. `run_paid_work` wraps one complete job. +- `PreparedPaidWorkInput` and `PaidWorkPolicy` select Evaluate or Fetch. Each + profile validates its own canonical input, bounds and terminal result; the + payment lifecycle does not interpret HTTP or model output. +- `FetchProviderOptions` and `start_fetch_provider` take an operator's route + registry, enrollment and root prover. The host owns its identity and credentials; + the SDK supplies the shared provider routing and finalized-chain clock. +- `HttpFetchRequest` and `HttpFetchResponse` express HTTPS semantics. The Fetch + backend executes an admitted request; canonical transcript verification checks + its result under the assurance requested by the caller. + +A session serializes its jobs. After cancellation, recover the journal before +admitting another job. Evaluate supports authenticated incremental token delivery; +Fetch currently returns a complete bounded response. Client journals may retain +payloads; the Fetch provider's journal remains metadata-only. + +Apple App Attest requires a provisioned, signed macOS host. `ProducerSigned` +verifies the key and transcript but does not attest the binary. These are the +implemented assurance choices, checked explicitly before request disclosure and +again during result verification. + +Adding Open changed the Work and WorkSetup service schema IDs. Existing paid +method IDs and canonical payment encodings are unchanged; use matching updated +client and provider builds. + +Tests inspect provider journal files after each commit and rotation, reopen +them after simulated process loss, verify payment recovery without bodies, +and cover both signed result schemes. Local TLS tests exercise roots, pins, +wrong hostnames, redirects, address restrictions and response bounds. Portable +App Attest tests cover connection/service binding and replay counters. They do +not contact a paid upstream or enroll a real Apple device. diff --git a/crates/work/src/work.rs b/crates/work/src/work.rs index 2cbd812e..a376ab30 100644 --- a/crates/work/src/work.rs +++ b/crates/work/src/work.rs @@ -5,7 +5,7 @@ //! //! Canonical private records, and nothing else. `AcceptWorkRequest` //! carries the exact bytes of a [`PaidJobAuthorizationV1`] and a -//! [`PreparedPaidInputV1`], not a protobuf transcription of their +//! [`hellas_rpc::protocol::artifacts::PreparedPaidInputV1`], not a protobuf transcription of their //! fields, because the two signatures on this exchange are over digests //! of those bytes. A protobuf spelling beside them would be a second //! definition of what both parties signed, agreeing with the first until @@ -151,13 +151,14 @@ use hellas_rpc::pb::work::{ deliver_result_response::Outcome as DeliverOutcome, }; use hellas_rpc::protocol::Digest; -use hellas_rpc::protocol::artifacts::{PreparedPaidInputParts, PreparedPaidInputV1}; +use hellas_rpc::protocol::artifacts::PreparedPaidInputParts; use hellas_rpc::protocol::work::{ JobDeadlines, PaidJobAuthorizationV1, PaidJobResultV1, PaidWorkError, PaymentBindingV1, - PrivateRecord as _, check_authorization, check_prepared_input, delivery_request_digest, - encode_transcript, next_payment, payment_binding_digest, propose_authorization, result_digest, - signing_hash, terminal_result, work_id, + PrivateRecord as _, delivery_request_digest, encode_transcript, next_payment, + payment_binding_digest, result_digest, signing_hash, work_id, }; +use hellas_rpc::protocol::work_fetch::{PaidFetchPolicyV1, PreparedPaidFetchInputParts}; +use hellas_rpc::protocol::work_profile::{PaidWorkPolicy, PreparedPaidWorkInput}; use hellas_rpc::protocol::work_setup::{ObservedChannel, ReadyChannel, WorkSetupError}; use hellas_rpc::services::work::{WorkClientImpl, WorkHandler}; use hellas_rpc::{EvaluateRequest, OutputEventEnvelope, SubmitTxOutcome}; @@ -385,6 +386,9 @@ pub enum EndpointError { /// found, so the two bound payments differently. #[error("the store's settlement is not the one this readiness read")] WrongSettlement, + /// Paid Fetch providers must never persist customer payloads. + #[error("paid fetch requires a provider journal containing metadata only")] + PayloadRetention, /// The journal is the other role's. #[error("the store is the {found} journal, and this is the {expected} endpoint")] WrongRole { @@ -433,6 +437,12 @@ fn bind( signer: &Secp256k1Signer, role: Role, ) -> Result<(), EndpointError> { + if role == Role::Provider + && matches!(ready.execution_policy(), PaidWorkPolicy::Fetch { .. }) + && !store.metadata_only() + { + return Err(EndpointError::PayloadRetention); + } let state = store.state(); if state.channel() != ready.channel() { return Err(EndpointError::WrongChannel); @@ -509,7 +519,7 @@ pub struct CloseEndpoint { /// /// It owns one channel's journal and answers proposals on that channel /// alone. A proposal naming another channel is refused by -/// [`check_authorization`], never routed: routing many channels through +/// [`hellas_rpc::protocol::work::check_authorization`], never routed: routing many channels through /// one endpoint is a later concern, and pretending to do it here would /// mean an authorization whose `channel_id` selects its own validator. /// @@ -730,11 +740,11 @@ impl ProviderEndpoint { .map_err(|error| Refusal::new(endpoint_refusal(error), error.to_string()))? .clone(); let (cursor_height, _) = self.state().cursor(); - let policy = *ready.execution_policy(); - check_authorization(ready.channel(), &authorization, &policy, cursor_height)?; - let bundle = PreparedPaidInputV1::decode(&request.prepared_input, MAX_RECORD_BYTES) + let policy = ready.execution_policy(); + policy.check_authorization(ready.channel(), &authorization, cursor_height)?; + let bundle = PreparedPaidWorkInput::decode(&request.prepared_input, MAX_RECORD_BYTES) .map_err(|error| Refusal::invalid(error.to_string()))?; - check_prepared_input(ready.channel(), &authorization, &policy, &bundle)?; + policy.check_input(ready.channel(), &authorization, &bundle)?; ready.check_signable( cursor_height, authorization.terminal_deadline, @@ -853,22 +863,40 @@ impl ProviderEndpoint { authorization.payment_deadline, )?; - let bundle = PreparedPaidInputV1::decode(job.prepared_input(), MAX_RECORD_BYTES) + if job.prepared_input().is_empty() { + return Ok(RunAdmission::Indeterminate); + } + let bundle = PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) .map_err(PaidWorkError::from)?; - let parts = bundle.parts().map_err(PaidWorkError::from)?; - let input = PreparedEvaluateInput { parts }; + ready + .execution_policy() + .check_input(ready.channel(), &authorization, &bundle)?; + let admission = match (bundle, ready.execution_policy()) { + (PreparedPaidWorkInput::Evaluate(bundle), PaidWorkPolicy::Evaluate(_)) => { + RunAdmission::Invoke(Box::new(PreparedEvaluateInput { + parts: bundle.parts().map_err(PaidWorkError::from)?, + })) + } + (PreparedPaidWorkInput::Fetch(bundle), PaidWorkPolicy::Fetch { policy, .. }) => { + RunAdmission::InvokeFetch(Box::new(PreparedFetchInput { + parts: bundle.parts().map_err(PaidWorkError::from)?, + policy: *policy, + })) + } + _ => return Err(RunError::Policy), + }; self.close.store.commit( ChannelRecord::JobRunning { work_id }, &Secp256k1Verifier::new(), )?; - Ok(RunAdmission::Invoke(Box::new(input))) + Ok(admission) } /// Signs the result of the transcript this job's invocation /// produced, and returns it only once it is on the disk. /// - /// The transcript is the provider's own: [`terminal_result`] refuses + /// The transcript is the provider's own: [`hellas_rpc::protocol::work::terminal_result`] refuses /// events that are not one verified chain for this authorization's /// request under the channel's provider key, and the journal refuses /// the record unless the job is running, is not indeterminate, and @@ -891,13 +919,17 @@ impl ProviderEndpoint { ) -> Result<(PaidJobResultV1, Sig), RunError> { let job = self.state().job_by_id(work_id).ok_or(RunError::NoSuchJob)?; let authorization = *job.authorization(); + let input = PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) + .map_err(|e| RunError::Transcript(e.into()))?; let ready = self.admitting()?.clone(); let channel = ready.channel(); - let result = - terminal_result(channel, &authorization, transcript).map_err(RunError::Transcript)?; + let result = ready + .execution_policy() + .terminal_result(channel, &authorization, &input, transcript) + .map_err(RunError::Transcript)?; let spool = encode_transcript(transcript).map_err(RunError::Transcript)?; let spooled = u64::try_from(spool.len()).unwrap_or(u64::MAX); - let limit = ready.execution_policy().max_spool_bytes; + let limit = ready.execution_policy().max_spool_bytes(); if spooled > limit { return Err(RunError::Record(PaidWorkError::OverEnvelope { field: "spooled transcript length", @@ -926,7 +958,7 @@ impl ProviderEndpoint { .encoded_len(), ) .unwrap_or(u64::MAX); - let frame_limit = u64::from(ready.execution_policy().max_encoded_result_frame); + let frame_limit = u64::from(ready.execution_policy().max_encoded_result_frame()); if frame > frame_limit { return Err(RunError::Record(PaidWorkError::OverEnvelope { field: "encoded result frame", @@ -1033,6 +1065,9 @@ impl ProviderEndpoint { }; let (result, signature) = (*result, *signature); let transcript = job.transcript().to_vec(); + if transcript.is_empty() { + return Err(DeliverError::NoResult { phase: job.phase() }); + } let terminal_deadline = job.authorization().terminal_deadline; bind(ready, &self.close.store, &self.close.signer, Role::Provider)?; @@ -1634,7 +1669,7 @@ impl BackendFault { /// The complete journaled input handed to a paid Evaluate backend. /// -/// All six bodies come from the same strictly decoded [`PreparedPaidInputV1`] +/// All six bodies come from the same strictly decoded [`hellas_rpc::protocol::artifacts::PreparedPaidInputV1`] /// whose digest the parties signed. Keeping the execution, tokens, policy, and /// identity here is what lets a backend run after restart without depending on /// transient Courtesy state. Environment bytes remain content-store data below @@ -1664,10 +1699,32 @@ impl PreparedEvaluateInput { } } +/// A paid Fetch input admitted under a channel policy. It is held only in +/// memory on the provider, even while the job's accounting state is durable. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PreparedFetchInput { + parts: PreparedPaidFetchInputParts, + policy: PaidFetchPolicyV1, +} + +impl PreparedFetchInput { + /// Returns the fixed output bounds the provider must enforce while running. + pub const fn policy(&self) -> &PaidFetchPolicyV1 { + &self.policy + } + + /// Transfers the signed request and canonical manifest to the backend. + pub fn into_parts(self) -> PreparedPaidFetchInputParts { + self.parts + } +} + /// The one seam a paid job crosses on its way to real execution. /// -/// One method, and it takes the complete prepared graph this endpoint rebuilt -/// from its own journal rather than anything transient or supplied at dispatch. +/// The endpoint selects Evaluate or Fetch from the admitted profile and passes +/// the verified prepared input. Evaluate inputs survive journal recovery; +/// Fetch bodies exist only in this process's memory. Implement only the profiles +/// the backend serves; unsupported profiles fail without running work. /// What comes back is the complete signed transcript of that invocation — not /// a digest of one, because a digest is exactly what a backend that ran nothing /// could also return. @@ -1675,12 +1732,23 @@ impl PreparedEvaluateInput { /// Implementors must invoke once per call. That is not a property this /// trait can check, and it is not the one the gate rests on: the gate /// calls this at most once per `work_id` whatever the implementor does. -pub trait PaidEvaluateBackend: Sync { +pub trait PaidWorkBackend: Sync { + /// Runs an authorized Fetch without persisting request or response bodies. + fn fetch( + &self, + _input: PreparedFetchInput, + ) -> impl core::future::Future, BackendFault>> + Send + { + async { Err(BackendFault::new("paid fetch backend is unavailable")) } + } /// Runs one journaled Evaluate input to its terminal. fn evaluate( &self, - input: PreparedEvaluateInput, - ) -> impl core::future::Future, BackendFault>> + Send; + _input: PreparedEvaluateInput, + ) -> impl core::future::Future, BackendFault>> + Send + { + async { Err(BackendFault::new("paid evaluate backend is unavailable")) } + } /// Run once, exposing authenticated token prefixes while retaining the /// complete transcript for durable terminal delivery. @@ -1702,12 +1770,17 @@ pub trait PaidEvaluateBackend: Sync { } } +/// Compatibility name for implementations that provide only Evaluate. +pub use PaidWorkBackend as PaidEvaluateBackend; + /// What [`ProviderEndpoint::begin_run`] found, and what may be done next. #[derive(Clone, Debug, PartialEq, Eq)] pub enum RunAdmission { /// The marker is durable and the backend has not been called. /// Invoke exactly once, with this journaled input. Invoke(Box), + /// The fetch running marker is durable; invoke once with the in-memory input. + InvokeFetch(Box), /// This process marked the job and has not recorded its result. Running, /// A signed result already exists. @@ -1717,9 +1790,8 @@ pub enum RunAdmission { /// The provider's signature over its digest. signature: Sig, }, - /// A marker was found by a process that did not write it. Whether - /// the backend ran is not knowable here, and nothing resolves it - /// automatically. + /// A prior process left a running marker, or an accepted Fetch lost its + /// in-memory input. Neither case is automatically re-executed. Indeterminate, } @@ -1830,11 +1902,15 @@ pub async fn run_accepted_work( work_id: Digest, ) -> Result where - B: PaidEvaluateBackend + Sync, + B: PaidWorkBackend + Sync, { let admission = service.begin_run(work_id, ready)?; - let input = match admission { - RunAdmission::Invoke(input) => *input, + let progress_service = service.clone(); + let progress: PaidProgress = + Arc::new(move |event| progress_service.publish_progress(work_id, event)); + let invoked = match admission { + RunAdmission::Invoke(input) => backend.evaluate_stream(*input, progress).await, + RunAdmission::InvokeFetch(input) => backend.fetch(*input).await, RunAdmission::Running => return Ok(RunOutcome::Running), RunAdmission::Indeterminate => return Ok(RunOutcome::Indeterminate), RunAdmission::Ready { result, signature } => { @@ -1842,10 +1918,7 @@ where } }; - let progress_service = service.clone(); - let progress: PaidProgress = - Arc::new(move |event| progress_service.publish_progress(work_id, event)); - let transcript = match backend.evaluate_stream(input, progress).await { + let transcript = match invoked { Ok(transcript) => transcript, Err(fault) => return Err(end_failed(service, work_id, RunError::Backend(fault))), }; @@ -1877,7 +1950,7 @@ pub async fn run_accepted_work_after_catch_up( ) -> Result where S: FinalizedBlocks + ?Sized, - B: PaidEvaluateBackend + Sync, + B: PaidWorkBackend + Sync, { service .catch_up_job(source, work_id) @@ -2789,11 +2862,11 @@ fn work_id_bytes(bytes: &[u8]) -> Option { /// /// The two things a proposal actually chooses. Everything else in the /// authorization is derived from the channel, the policy, and this -/// bundle by [`propose_authorization`]. +/// bundle by [`hellas_rpc::protocol::work::propose_authorization`]. #[derive(Clone, Debug, PartialEq, Eq)] pub struct JobProposal { /// The inputs the job runs on. - pub prepared_input: PreparedPaidInputV1, + pub prepared_input: PreparedPaidWorkInput, /// The three heights the job is bound by. pub deadlines: JobDeadlines, } @@ -2898,7 +2971,7 @@ impl ClientEndpoint { /// proposal itself raises. pub fn propose(&mut self, proposal: &JobProposal) -> Result { let (cursor_height, _) = self.state().cursor(); - let policy = *self.ready.execution_policy(); + let policy = self.ready.execution_policy(); for job in self .state() @@ -2906,9 +2979,8 @@ impl ClientEndpoint { .filter(|job| job.phase() == JobPhase::HalfSigned) { let retained = *job.authorization(); - let rebuilt = propose_authorization( + let rebuilt = policy.propose( self.ready.channel(), - &policy, &proposal.prepared_input, retained.proposal_nonce, proposal.deadlines, @@ -2927,18 +2999,16 @@ impl ClientEndpoint { // wait for the first, but no crash or reordered response can make // the sequence move backwards or reuse a number. let proposal_nonce = self.state().proposal_nonce_high_water().saturating_add(1); - let authorization = propose_authorization( + let authorization = policy.propose( self.ready.channel(), - &policy, &proposal.prepared_input, proposal_nonce, proposal.deadlines, )?; - check_authorization(self.ready.channel(), &authorization, &policy, cursor_height)?; - check_prepared_input( + policy.check_authorization(self.ready.channel(), &authorization, cursor_height)?; + policy.check_input( self.ready.channel(), &authorization, - &policy, &proposal.prepared_input, )?; self.ready.check_signable( @@ -3139,13 +3209,54 @@ impl ClientEndpoint { // The bound is on the encoded message, which is what this // endpoint agreed to hold; the transport's own framing around // it is the transport's and is not measured here. - let limit = u64::from(ready.execution_policy().max_encoded_result_frame); + let limit = u64::from(ready.execution_policy().max_encoded_result_frame()); let actual = u64::try_from(delivered.encoded_len()).unwrap_or(u64::MAX); if actual > limit { return Err(DeliverError::OverFrame { actual, limit }); } let result = PaidJobResultV1::decode(&delivered.result)?; + if matches!(ready.execution_policy(), PaidWorkPolicy::Fetch { .. }) { + let actual = delivered.transcript.len() as u64; + let limit = ready.execution_policy().max_spool_bytes(); + if actual > limit { + return Err(PaidWorkError::OverEnvelope { + field: "spooled transcript length", + actual, + limit, + } + .into()); + } + let authorization = self + .state() + .job_by_id(work_id) + .ok_or(DeliverError::NoSuchJob)? + .authorization(); + let transcript = hellas_rpc::protocol::work::decode_transcript( + &delivered.transcript, + MAX_RECORD_BYTES, + )?; + let input = PreparedPaidWorkInput::decode( + self.state() + .job_by_id(work_id) + .ok_or(DeliverError::NoSuchJob)? + .prepared_input(), + MAX_RECORD_BYTES, + ) + .map_err(PaidWorkError::from)?; + let expected = ready.execution_policy().terminal_result( + ready.channel(), + authorization, + &input, + &transcript, + )?; + if expected != result { + return Err(PaidWorkError::Mismatch { + field: "result against its transcript", + } + .into()); + } + } let signature = signature(&delivered.provider_signature) .ok_or(DeliverError::Malformed("provider signature"))?; self.store.commit( diff --git a/crates/work/src/work/stream.rs b/crates/work/src/work/stream.rs index 148a310f..58a93efa 100644 --- a/crates/work/src/work/stream.rs +++ b/crates/work/src/work/stream.rs @@ -58,7 +58,7 @@ impl WorkService { .admitting() .map_err(|error| BackendFault::new(error.to_string()))? .execution_policy() - .max_spool_bytes; + .max_spool_bytes(); let mut progress = self.progress.lock().expect("paid progress poisoned"); let events = progress.entry(work_id).or_default(); // Signed token events are bounded by the same spool as final delivery. @@ -171,8 +171,14 @@ where .state() .job_by_id(work_id) .ok_or(DeliverError::NoSuchJob)?; - let prepared = PreparedPaidInputV1::decode(job.prepared_input(), MAX_RECORD_BYTES) - .map_err(PaidWorkError::from)?; + let PreparedPaidWorkInput::Evaluate(prepared) = + PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) + .map_err(PaidWorkError::from)? + else { + return Err(DeliverError::Malformed( + "live token delivery requires Evaluate", + )); + }; let parts = prepared.parts().map_err(PaidWorkError::from)?; let input = hellas_rpc::evaluate::input_commitment(&parts.evaluate_request); let mut token_count = 0u64; @@ -208,7 +214,7 @@ where token_count = token_count.saturating_add(delta.token_ids.len() as u64); retained_bytes = retained_bytes.saturating_add(event.payload().len() + 1024); if token_count > u64::from(parts.text_policy.max_new_tokens()) - || retained_bytes as u64 > ready.execution_policy().max_spool_bytes + || retained_bytes as u64 > ready.execution_policy().max_spool_bytes() { return Err(DeliverError::Malformed( "live result exceeds authorized output", diff --git a/crates/work/src/work_open.rs b/crates/work/src/work_open.rs index 3434fa3d..ad14f739 100644 --- a/crates/work/src/work_open.rs +++ b/crates/work/src/work_open.rs @@ -755,14 +755,28 @@ fn mount( None => descriptor.expected_settlement().map_err(to_store)?, Some(payment) => descriptor.funded_settlement(payment).map_err(to_store)?, }; - let mut channel = ChannelStore::open( - store.root(), - descriptor.channel().clone(), - settlement, - store.role(), - origin, - verifier, - )?; + let mut channel = if store.role() == crate::work_store::Role::Provider + && matches!( + descriptor.execution_policy(), + hellas_rpc::protocol::work_profile::PaidWorkPolicy::Fetch { .. } + ) { + ChannelStore::open_metadata_only( + store.root(), + descriptor.channel().clone(), + settlement, + origin, + verifier, + )? + } else { + ChannelStore::open( + store.root(), + descriptor.channel().clone(), + settlement, + store.role(), + origin, + verifier, + )? + }; // The origin block carries the payment Open that established this // channel, and the store opened with its cursor already on that block. // A `StartPaymentClose` ordered after the Open in the very same block diff --git a/crates/work/src/work_store/channel.rs b/crates/work/src/work_store/channel.rs index 4789ef02..0bbf0ad1 100644 --- a/crates/work/src/work_store/channel.rs +++ b/crates/work/src/work_store/channel.rs @@ -22,12 +22,12 @@ use crate::work_store::{ take_option, }; use hellas_rpc::protocol::Digest; -use hellas_rpc::protocol::artifacts::PreparedPaidInputV1; use hellas_rpc::protocol::work::{ CreditLedger, PaidChannel, PaidJobAuthorizationV1, PaidJobResultV1, PaidWorkError, - PaymentBindingV1, PrivateRecord as _, decode_transcript, payment_binding_digest, - prepared_input_digest, result_digest, signing_hash, terminal_result, work_id, + PaymentBindingV1, PrivateRecord as _, decode_transcript, payment_binding_digest, result_digest, + signing_hash, work_id, }; +use hellas_rpc::protocol::work_profile::PreparedPaidWorkInput; mod codec; mod state; diff --git a/crates/work/src/work_store/channel/codec.rs b/crates/work/src/work_store/channel/codec.rs index 6b6cdb7d..95dd3d53 100644 --- a/crates/work/src/work_store/channel/codec.rs +++ b/crates/work/src/work_store/channel/codec.rs @@ -41,6 +41,10 @@ impl ChannelRecord { /// bytes whose digests the signatures beside them cover. #[must_use] pub fn encode(&self) -> Vec { + self.encode_for_storage(false) + } + + pub(super) fn encode_for_storage(&self, metadata_only: bool) -> Vec { let mut out = Vec::new(); match self { Self::CursorAdvanced { @@ -64,7 +68,9 @@ impl ChannelRecord { // Last field, and the whole of the rest: the journal // frame already carries this record's length, and a // second length here could disagree with it. - out.extend_from_slice(prepared_input); + if !metadata_only { + out.extend_from_slice(prepared_input); + } } Self::JobAccepted { work_id, @@ -91,7 +97,9 @@ impl ChannelRecord { // Last field, and the whole of the rest, for the reason // `JobProposed`'s bundle is: the journal frame already // carries this record's length. - out.extend_from_slice(transcript); + if !metadata_only { + out.extend_from_slice(transcript); + } } Self::PlaintextReleased { work_id } => { out.push(tag::PLAINTEXT); @@ -406,6 +414,10 @@ impl ChannelState { /// is one channel and not two that happen to agree. #[must_use] pub fn checkpoint(&self) -> Vec { + self.checkpoint_for_storage(false) + } + + pub(super) fn checkpoint_for_storage(&self, metadata_only: bool) -> Vec { let Self { channel, settlement, @@ -453,8 +465,8 @@ impl ChannelState { out.extend_from_slice(&result.encode()); out.extend_from_slice(signature.as_bytes()); }); - put_bytes(&mut out, prepared_input); - put_bytes(&mut out, transcript); + put_bytes(&mut out, if metadata_only { &[] } else { prepared_input }); + put_bytes(&mut out, if metadata_only { &[] } else { transcript }); } put_u64(&mut out, terminals.len() as u64); for terminal in terminals.values() { diff --git a/crates/work/src/work_store/channel/state.rs b/crates/work/src/work_store/channel/state.rs index 6d340643..d24b552e 100644 --- a/crates/work/src/work_store/channel/state.rs +++ b/crates/work/src/work_store/channel/state.rs @@ -1,6 +1,36 @@ use super::*; impl ChannelState { + fn check_result_payload( + &self, + job: &JobState, + result: &PaidJobResultV1, + transcript: &[u8], + metadata_only: bool, + ) -> Result<(), ChannelStateError> { + if result.work_id != job.work_id { + return Err(ChannelStateError::WrongChannel { + field: "result work_id", + }); + } + if metadata_only { + return if transcript.is_empty() { + Ok(()) + } else { + Err(ChannelStateError::Malformed) + }; + } + let events = decode_transcript(transcript, MAX_RECORD_BYTES)?; + let input = PreparedPaidWorkInput::decode(&job.prepared_input, MAX_RECORD_BYTES) + .map_err(PaidWorkError::from)?; + if input.terminal_result(&self.channel, &job.authorization, &events)? != *result { + return Err(ChannelStateError::WrongChannel { + field: "result against its transcript", + }); + } + Ok(()) + } + pub(super) fn new( channel: PaidChannel, settlement: WorkPaymentSettlement, @@ -37,9 +67,10 @@ impl ChannelState { settlement: WorkPaymentSettlement, role: Role, verifier: &V, + metadata_only: bool, ) -> Result { let state = Self::decode_checkpoint(bytes, channel, settlement, role)?; - state.revalidate(verifier)?; + state.revalidate(verifier, metadata_only)?; Ok(state) } @@ -51,7 +82,7 @@ impl ChannelState { /// the client's authorization, the provider's co-signature, the /// provider's result, and the client's binding and certificate. The /// result is rebuilt from the transcript beside it by - /// [`terminal_result`], exactly as a record replay rebuilds it. The + /// [`hellas_rpc::protocol::work::terminal_result`], exactly as a record replay rebuilds it. The /// inputs are hashed against the digest the authorization commits /// to. The retained close start is checked against this channel and /// this role, and the fixed answer is re-derived from the contest @@ -64,7 +95,11 @@ impl ChannelState { /// is the one thing this does not claim, and it is the reason the /// deadline rules stay where they are — on the records, at the /// heights they were taken. - fn revalidate(&self, verifier: &V) -> Result<(), ChannelStateError> { + fn revalidate( + &self, + verifier: &V, + metadata_only: bool, + ) -> Result<(), ChannelStateError> { // The one number consensus sees, against the one terminal that // could have moved it. A ledger the terminal does not produce is // a channel that would credit a second payment. @@ -77,7 +112,7 @@ impl ChannelState { if job.work_id != work_id(&self.channel, &job.authorization) { return Err(ChannelStateError::WrongChannel { field: "work_id" }); } - self.check_authorization(&job.authorization, &job.prepared_input)?; + self.check_authorization(&job.authorization, &job.prepared_input, metadata_only)?; if job.phase.only_role().is_some_and(|role| role != self.role) || job.provider_signature.is_some() != (job.phase != JobPhase::HalfSigned) || job.result.is_some() != job.phase.has_result() @@ -110,12 +145,7 @@ impl ChannelState { }); } if let Some((result, provider_signature)) = &job.result { - let events = decode_transcript(&job.transcript, MAX_RECORD_BYTES)?; - if terminal_result(&self.channel, &job.authorization, &events)? != *result { - return Err(ChannelStateError::WrongChannel { - field: "result against its transcript", - }); - } + self.check_result_payload(job, result, &job.transcript, metadata_only)?; if !verifier.verify_sig( *provider_signature, self.provider_key(), @@ -561,6 +591,7 @@ impl ChannelState { &mut self, record: &ChannelRecord, verifier: &V, + metadata_only: bool, ) -> Result { match record { ChannelRecord::CursorAdvanced { @@ -572,7 +603,13 @@ impl ChannelState { authorization, client_signature, prepared_input, - } => self.apply_proposed(authorization, *client_signature, prepared_input, verifier), + } => self.apply_proposed( + authorization, + *client_signature, + prepared_input, + verifier, + metadata_only, + ), ChannelRecord::JobAccepted { work_id, provider_signature, @@ -583,7 +620,14 @@ impl ChannelState { result, provider_signature, transcript, - } => self.apply_result(*work_id, result, *provider_signature, transcript, verifier), + } => self.apply_result( + *work_id, + result, + *provider_signature, + transcript, + verifier, + metadata_only, + ), ChannelRecord::PlaintextReleased { work_id } => self.apply_plaintext(*work_id), ChannelRecord::ResultMatched { work_id } => self.apply_matched(*work_id), ChannelRecord::JobTerminated { work_id, outcome } => { @@ -873,6 +917,7 @@ impl ChannelState { &self, authorization: &PaidJobAuthorizationV1, prepared_input: &[u8], + metadata_only: bool, ) -> Result<(), ChannelStateError> { let terms = self.channel.payment_terms(); for (field, holds) in [ @@ -902,9 +947,16 @@ impl ChannelState { // The inputs this job will be executed from, against the digest // the authorization both parties sign commits to. A bundle that // does not hash to it is a job neither party agreed to run. - let bundle = PreparedPaidInputV1::decode(prepared_input, MAX_RECORD_BYTES) + if metadata_only { + return if prepared_input.is_empty() { + Ok(()) + } else { + Err(ChannelStateError::Malformed) + }; + } + let bundle = PreparedPaidWorkInput::decode(prepared_input, MAX_RECORD_BYTES) .map_err(PaidWorkError::from)?; - if prepared_input_digest(&self.channel, &bundle)?.as_bytes() + if bundle.digest(&self.channel)?.as_bytes() != authorization.prepared_input_digest.as_bytes() { return Err(ChannelStateError::Record(PaidWorkError::Mismatch { @@ -958,6 +1010,7 @@ impl ChannelState { client_signature: Sig, prepared_input: &[u8], verifier: &V, + metadata_only: bool, ) -> Result { let work_id = work_id(&self.channel, authorization); if let Some(job) = self.jobs.get(&work_id) { @@ -978,7 +1031,7 @@ impl ChannelState { // whose payment no close could carry. self.refuse_if_closing("proposing a job")?; - self.check_authorization(authorization, prepared_input)?; + self.check_authorization(authorization, prepared_input, metadata_only)?; if authorization.proposal_nonce <= self.proposal_nonce_high_water { return Err(ChannelStateError::WrongChannel { field: "proposal_nonce", @@ -1088,7 +1141,7 @@ impl ChannelState { /// summarises. /// /// The rule that makes this more than a signature check is the - /// reproduction below: [`terminal_result`] is handed the stored + /// reproduction below: [`hellas_rpc::protocol::work::terminal_result`] is handed the stored /// events and this job's own authorization, and what it builds must /// be the result byte for byte. That establishes, on commit and on /// every replay, that the events are one verified signed chain for @@ -1106,6 +1159,7 @@ impl ChannelState { provider_signature: Sig, transcript: &[u8], verifier: &V, + metadata_only: bool, ) -> Result { let mut job = self.open_job(work_id, "recording a result")?; if let Some((held, signature)) = &job.result { @@ -1159,12 +1213,7 @@ impl ChannelState { }); } - let events = decode_transcript(transcript, MAX_RECORD_BYTES)?; - if terminal_result(&self.channel, &job.authorization, &events)? != *result { - return Err(ChannelStateError::WrongChannel { - field: "result against its transcript", - }); - } + self.check_result_payload(&job, result, transcript, metadata_only)?; if !verifier.verify_sig( provider_signature, self.provider_key(), diff --git a/crates/work/src/work_store/channel/store.rs b/crates/work/src/work_store/channel/store.rs index 7acb2d3b..26176e97 100644 --- a/crates/work/src/work_store/channel/store.rs +++ b/crates/work/src/work_store/channel/store.rs @@ -7,6 +7,7 @@ pub struct ChannelStore { journal: Journal, state: ChannelState, torn_tail: bool, + metadata_only: bool, } impl ChannelStore { @@ -45,6 +46,43 @@ impl ChannelStore { role: Role, origin: SetupOrigin, verifier: &V, + ) -> Result { + Self::open_inner(root, channel, settlement, role, origin, verifier, false) + } + + /// Opens a provider journal that stores accounting evidence only. + /// + /// Inputs and output transcripts are verified before commit and retained + /// in memory. Append and checkpoint encodings omit both bodies. After a + /// restart an accepted request cannot run and a completed response cannot + /// be replayed; a signed payment for an already delivered result can still + /// be credited. Existing payload-bearing journals are refused. + pub fn open_metadata_only( + root: &Path, + channel: PaidChannel, + settlement: WorkPaymentSettlement, + origin: SetupOrigin, + verifier: &V, + ) -> Result { + Self::open_inner( + root, + channel, + settlement, + Role::Provider, + origin, + verifier, + true, + ) + } + + fn open_inner( + root: &Path, + channel: PaidChannel, + settlement: WorkPaymentSettlement, + role: Role, + origin: SetupOrigin, + verifier: &V, + metadata_only: bool, ) -> Result { if origin.payment_edge != channel.payment_edge() { return Err(ChannelStateError::WrongChannel { @@ -58,21 +96,30 @@ impl ChannelStore { root, &format!("channel-{}", hex(&key)), JournalId { - kind: JournalKind::Channel, + kind: if metadata_only { + JournalKind::MetadataChannel + } else { + JournalKind::Channel + }, role, key, generation: 0, }, )?; let mut state = match &replay.checkpoint { - Some(bytes) => { - ChannelState::from_checkpoint(bytes, channel, settlement, role, verifier)? - } + Some(bytes) => ChannelState::from_checkpoint( + bytes, + channel, + settlement, + role, + verifier, + metadata_only, + )?, None => ChannelState::new(channel, settlement, role, origin), }; for bytes in &replay.records { let record = ChannelRecord::decode(bytes)?; - state.apply(&record, verifier)?; + state.apply(&record, verifier, metadata_only)?; } state.indeterminate = state .jobs @@ -85,6 +132,7 @@ impl ChannelStore { journal, state, torn_tail: replay.truncated_tail, + metadata_only, }; Ok(store) } @@ -107,6 +155,12 @@ impl ChannelStore { &self.state } + /// Whether this journal excludes request and response bodies from disk. + #[must_use] + pub const fn metadata_only(&self) -> bool { + self.metadata_only + } + /// Journals one step, and returns only once it is on the disk. /// /// The rule this exists to enforce: call it *before* the bytes it @@ -128,7 +182,7 @@ impl ChannelStore { // Applied to a copy first: a record the rules refuse must leave // neither the file nor the state touched. let mut next = self.state.clone(); - if next.apply(&record, verifier)? == Applied::Changed { + if next.apply(&record, verifier, false)? == Applied::Changed { // The signature this record carries leaves after this // returns, so the state that authorises it has to be one a // rotation can still carry. The proposed job is charged the @@ -136,13 +190,17 @@ impl ChannelStore { // answered with and the terminal that pays for it — because // by then there is no refusal left that costs nothing. if let Some(tail) = record.checkpoint_tail() { - let len = next.checkpoint().len().saturating_add(tail); + let len = next + .checkpoint_for_storage(self.metadata_only) + .len() + .saturating_add(tail); if len > MAX_CHECKPOINT_BYTES { return Err(JournalError::CheckpointTooLarge { len }.into()); } } self.rotate_if_full(record.is_new_work())?; - self.journal.append(&record.encode())?; + self.journal + .append(&record.encode_for_storage(self.metadata_only))?; self.state = next; } Ok(&self.state) @@ -162,7 +220,8 @@ impl ChannelStore { /// [`WorkStoreError::Journal`] when the checkpoint does not fit one /// frame or an install step fails. pub fn rotate(&mut self) -> Result<(), WorkStoreError> { - self.journal.rotate(&self.state.checkpoint())?; + self.journal + .rotate(&self.state.checkpoint_for_storage(self.metadata_only))?; Ok(()) } @@ -178,7 +237,10 @@ impl ChannelStore { if !self.journal.at_soft_limit() { return Ok(()); } - match self.journal.rotate(&self.state.checkpoint()) { + match self + .journal + .rotate(&self.state.checkpoint_for_storage(self.metadata_only)) + { Ok(()) => Ok(()), Err(error) if new_work => Err(error.into()), Err(_) => Ok(()), diff --git a/crates/work/src/work_store/journal.rs b/crates/work/src/work_store/journal.rs index e6012001..17a57096 100644 --- a/crates/work/src/work_store/journal.rs +++ b/crates/work/src/work_store/journal.rs @@ -85,6 +85,9 @@ pub enum JournalKind { Setup, /// One channel's job, credit, and certificate. Channel, + /// Provider accounting whose request and response bodies never reach disk. + /// A different kind prevents reopening a payload-bearing journal as ZDR. + MetadataChannel, } impl JournalKind { @@ -92,6 +95,7 @@ impl JournalKind { match self { Self::Setup => 1, Self::Channel => 2, + Self::MetadataChannel => 3, } } @@ -99,6 +103,7 @@ impl JournalKind { match code { 1 => Some(Self::Setup), 2 => Some(Self::Channel), + 3 => Some(Self::MetadataChannel), _ => None, } } diff --git a/crates/work/tests/work_accept.rs b/crates/work/tests/work_accept.rs index 586601f9..64c2679a 100644 --- a/crates/work/tests/work_accept.rs +++ b/crates/work/tests/work_accept.rs @@ -144,7 +144,7 @@ fn descriptor_with_policy(policy: PaidExecutionPolicyV1) -> WorkChannelDescripto payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: policy, + execution_policy: policy.into(), expected_payment_values: payment_values(), }; match WorkChannelDescriptor::open(config) { @@ -312,7 +312,7 @@ const fn deadlines() -> JobDeadlines { fn proposal(nonce: u8) -> JobProposal { JobProposal { - prepared_input: bundle(nonce), + prepared_input: bundle(nonce).into(), deadlines: deadlines(), } } @@ -1087,7 +1087,7 @@ fn a_proposal_refused_before_signing_leaves_no_job() { // Terminal deadline inside the measured dispatch and delivery // margins: legal, ordered, and unreachable. let unreachable = JobProposal { - prepared_input: bundle(1), + prepared_input: bundle(1).into(), deadlines: JobDeadlines { acceptance: 11, terminal: 12, diff --git a/crates/work/tests/work_close.rs b/crates/work/tests/work_close.rs index 60fed43d..4a5cd7d7 100644 --- a/crates/work/tests/work_close.rs +++ b/crates/work/tests/work_close.rs @@ -53,7 +53,7 @@ use hellas_rpc::{ use hellas_wire::mux::MuxTransport; use hellas_wire::{Dispatcher, StreamTransport}; use hellas_work::work::{ - BackendFault, ClientEndpoint, CloseEndpoint, PaidEvaluateBackend, PaymentError, + BackendFault, ClientEndpoint, CloseEndpoint, PaidWorkBackend, PaymentError, PreparedEvaluateInput, ProviderEndpoint, RunOutcome, WorkService, admit_payment, fetch_result, run_accepted_work, }; @@ -175,7 +175,7 @@ fn descriptor() -> WorkChannelDescriptor { payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: payment_values(), }; match WorkChannelDescriptor::open(config) { @@ -386,7 +386,7 @@ impl AnsweringBackend { } } -impl PaidEvaluateBackend for AnsweringBackend { +impl PaidWorkBackend for AnsweringBackend { fn evaluate( &self, input: PreparedEvaluateInput, diff --git a/crates/work/tests/work_delivery.rs b/crates/work/tests/work_delivery.rs index 1508f10e..7a47d5f5 100644 --- a/crates/work/tests/work_delivery.rs +++ b/crates/work/tests/work_delivery.rs @@ -47,7 +47,7 @@ use hellas_rpc::{ use hellas_wire::mux::{MessagePipe, MuxConfig, MuxTransport, Role as MuxRole}; use hellas_wire::{DefaultClock, Dispatcher, StreamTransport}; use hellas_work::work::{ - BackendFault, ClientEndpoint, CloseEndpoint, DeliverError, PaidEvaluateBackend, + BackendFault, ClientEndpoint, CloseEndpoint, DeliverError, PaidWorkBackend, PreparedEvaluateInput, ProviderEndpoint, RunError, RunOutcome, WorkService, fetch_result, run_accepted_work, }; @@ -171,7 +171,7 @@ fn descriptor_with(policy: PaidExecutionPolicyV1) -> WorkChannelDescriptor { payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: policy, + execution_policy: policy.into(), expected_payment_values: payment_values(), }; match WorkChannelDescriptor::open(config) { @@ -392,7 +392,7 @@ impl AnsweringBackend { } } -impl PaidEvaluateBackend for AnsweringBackend { +impl PaidWorkBackend for AnsweringBackend { fn evaluate( &self, input: PreparedEvaluateInput, @@ -719,7 +719,7 @@ async fn a_job_with_no_result_releases_nothing_yet() { #[tokio::test] async fn a_failed_job_returns_its_terminal_after_authentication() { struct FailingBackend; - impl PaidEvaluateBackend for FailingBackend { + impl PaidWorkBackend for FailingBackend { async fn evaluate( &self, _input: PreparedEvaluateInput, @@ -1463,7 +1463,7 @@ async fn live_prefix_precedes_terminal_and_reserves_delivery_credit() { struct PausedBackend { release: Arc, } - impl PaidEvaluateBackend for PausedBackend { + impl PaidWorkBackend for PausedBackend { async fn evaluate( &self, input: PreparedEvaluateInput, diff --git a/crates/work/tests/work_handshake.rs b/crates/work/tests/work_handshake.rs index 908e7071..46d88547 100644 --- a/crates/work/tests/work_handshake.rs +++ b/crates/work/tests/work_handshake.rs @@ -185,7 +185,7 @@ fn provider_policy() -> ProviderChannelPolicy { network: network(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: EdgeValues::new( PAYMENT_VALUE, PAYMENT_RESERVE, diff --git a/crates/work/tests/work_payment.rs b/crates/work/tests/work_payment.rs index f58fee64..2fd49fe3 100644 --- a/crates/work/tests/work_payment.rs +++ b/crates/work/tests/work_payment.rs @@ -53,7 +53,7 @@ use hellas_rpc::{ use hellas_wire::mux::MuxTransport; use hellas_wire::{Dispatcher, StreamTransport}; use hellas_work::work::{ - BackendFault, ClientEndpoint, PaidEvaluateBackend, PaymentError, PreparedEvaluateInput, + BackendFault, ClientEndpoint, PaidWorkBackend, PaymentError, PreparedEvaluateInput, ProviderEndpoint, RunError, RunOutcome, WorkRefusal, WorkService, admit_payment, fetch_result, run_accepted_work, }; @@ -167,7 +167,7 @@ fn descriptor() -> WorkChannelDescriptor { payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: payment_values(), }; match WorkChannelDescriptor::open(config) { @@ -378,7 +378,7 @@ impl AnsweringBackend { } } -impl PaidEvaluateBackend for AnsweringBackend { +impl PaidWorkBackend for AnsweringBackend { fn evaluate( &self, input: PreparedEvaluateInput, diff --git a/crates/work/tests/work_run.rs b/crates/work/tests/work_run.rs index e7fcd3a4..08efe7f2 100644 --- a/crates/work/tests/work_run.rs +++ b/crates/work/tests/work_run.rs @@ -39,14 +39,16 @@ use hellas_rpc::{ OutputEventEnvelope, ProducerSigningKey, ProgramManifest, PublicKey, }; use hellas_work::work::{ - BackendFault, PaidEvaluateBackend, PreparedEvaluateInput, ProviderEndpoint, RunAdmission, - RunError, RunOutcome, WorkService, run_accepted_work, + BackendFault, PaidWorkBackend, PreparedEvaluateInput, ProviderEndpoint, RunAdmission, RunError, + RunOutcome, WorkService, run_accepted_work, }; use hellas_work::work_store::{ ChannelRecord, ChannelStateError, ChannelStore, JobPhase, JobState, Role, SetupOrigin, TerminalOutcome, WorkStoreError, }; +#[path = "work_run/fetch.rs"] +mod fetch; mod support; use support::{advance, bond_edge, client, network, payload_at, payment_edge, provider, temp}; @@ -163,14 +165,16 @@ fn payment_values() -> EdgeValues { EdgeValues::new(PAYMENT_VALUE, PAYMENT_RESERVE, Fees::new(0, 0, 0, 0)) } -fn descriptor_with(policy: PaidExecutionPolicyV1) -> WorkChannelDescriptor { +fn descriptor_with( + policy: impl Into, +) -> WorkChannelDescriptor { let config = WorkChannelConfig { network: network(), payment_edge: payment_edge(), payment_terms: payment_terms(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: policy, + execution_policy: policy.into(), expected_payment_values: payment_values(), }; match WorkChannelDescriptor::open(config) { @@ -452,7 +456,7 @@ impl CountingBackend { } } -impl PaidEvaluateBackend for CountingBackend { +impl PaidWorkBackend for CountingBackend { fn evaluate( &self, input: PreparedEvaluateInput, diff --git a/crates/work/tests/work_run/fetch.rs b/crates/work/tests/work_run/fetch.rs new file mode 100644 index 00000000..36a29ea0 --- /dev/null +++ b/crates/work/tests/work_run/fetch.rs @@ -0,0 +1,365 @@ +use super::*; +use hellas_rpc::fetch::{ + FetchOutputTranscriptBuilder, build_input_events_with_retention, encode_fetch_event_payload, + encode_fetch_terminal_payload, verify_input_events, +}; +use hellas_rpc::output::{OutputEvent, StopReason, TextChannel}; +use hellas_rpc::pb::work::WorkDelivered; +use hellas_rpc::protocol::work::PrivateRecord as _; +use hellas_rpc::protocol::work_fetch::{ + FetchRoutePolicy, PaidFetchPolicyV1, PreparedPaidFetchInputV1, fetch_route_commitment, +}; +use hellas_rpc::protocol::work_profile::PaidWorkPolicy; +use hellas_rpc::{FetchEnvironment, Retention}; +use hellas_work::work::{ + ClientEndpoint, DeliverError, EndpointError, JobProposal, PreparedFetchInput, +}; + +const PROMPT: &[u8] = br#"{"input":"private-customer-prompt-never-on-provider-disk"}"#; +const RESPONSE: &str = "private-provider-response-never-on-provider-disk"; + +fn policy() -> PaidWorkPolicy { + let route = FetchRoutePolicy::sealed_route("openai", "responses").unwrap(); + PaidWorkPolicy::Fetch { + policy: PaidFetchPolicyV1 { + allowed_environment: FetchEnvironment::OpenAiResponses.manifest_id(), + route_commitment: fetch_route_commitment(&route.canonical_body_bytes()).unwrap(), + max_request_body_bytes: 4096, + max_output_events: 64, + max_output_bytes: 16384, + max_spool_bytes: 65536, + max_encoded_result_frame: 65536, + max_encoded_prepared_input: 65536, + dispatch_margin_blocks: 4, + delivery_margin_blocks: 2, + oracle_grace_blocks: 6, + fixed_price: PRICE, + }, + route, + } +} + +fn fetch_ready() -> ReadyChannel { + ready_of(descriptor_with(policy()), CURSOR) +} + +fn input(retention: Retention) -> PreparedPaidFetchInputV1 { + input_with_assurance(retention, Assurance::ProducerSigned) +} +fn input_with_assurance(retention: Retention, assurance: Assurance) -> PreparedPaidFetchInputV1 { + let environment = FetchEnvironment::OpenAiResponses; + let events = build_input_events_with_retention( + "openai", + "responses", + PROMPT, + environment.manifest_id(), + assurance, + &producer(0x21), + retention, + ) + .unwrap(); + PreparedPaidFetchInputV1::new(&events, &environment.manifest()).unwrap() +} + +fn proposal() -> JobProposal { + JobProposal { + prepared_input: input(Retention::Ephemeral).into(), + deadlines: deadlines(), + } +} + +fn metadata_store(root: &std::path::Path) -> ChannelStore { + let mut store = ChannelStore::open_metadata_only( + root, + fetch_ready().channel().clone(), + settlement(), + origin(), + &Secp256k1Verifier::new(), + ) + .unwrap(); + advance(&mut store, CURSOR); + store +} + +fn service(root: &std::path::Path) -> WorkService { + WorkService::new( + ProviderEndpoint::new(fetch_ready(), metadata_store(root), provider()).unwrap(), + ) +} + +fn client_endpoint(root: &std::path::Path) -> ClientEndpoint { + let mut store = ChannelStore::open( + root, + fetch_ready().channel().clone(), + settlement(), + Role::Client, + origin(), + &Secp256k1Verifier::new(), + ) + .unwrap(); + advance(&mut store, CURSOR); + ClientEndpoint::new(fetch_ready(), store, client()).unwrap() +} + +#[derive(Default)] +struct FetchBackend(AtomicUsize); + +impl PaidWorkBackend for FetchBackend { + async fn fetch( + &self, + input: PreparedFetchInput, + ) -> Result, BackendFault> { + self.0.fetch_add(1, Ordering::SeqCst); + let parts = input.into_parts(); + let request = verify_input_events(&parts.fetch_input_transcript).unwrap(); + assert_eq!(request.body.as_bytes(), PROMPT); + assert_eq!(request.retention, Retention::Ephemeral); + let key = provider_producer(); + let mut builder = + FetchOutputTranscriptBuilder::new(request.input_commitment, request.assurance, &key); + builder + .push_event( + encode_fetch_event_payload(&OutputEvent::TextDelta { + index: 0, + delta: RESPONSE.into(), + channel: TextChannel::Output, + }) + .unwrap(), + ) + .unwrap(); + Ok(builder + .finish( + encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .unwrap(), + ) + .unwrap()) + } +} + +fn assert_no_bodies(root: &std::path::Path) { + for entry in std::fs::read_dir(root).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + assert_no_bodies(&path); + } else { + let bytes = std::fs::read(&path).unwrap(); + for body in [PROMPT, RESPONSE.as_bytes()] { + assert!( + !bytes.windows(body.len()).any(|window| window == body), + "payload in {}", + path.display() + ); + } + } + } +} + +#[tokio::test] +async fn fetch_is_run_delivered_and_paid_without_provider_disk_bodies() { + let provider_dir = temp(); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let service = service(provider_dir.path()); + let backend = FetchBackend::default(); + let request = client.propose(&proposal()).unwrap(); + let id = client.accepted(&service.accept(&request)).unwrap(); + assert_no_bodies(provider_dir.path()); + assert!(matches!( + run_accepted_work(&service, &fetch_ready(), &backend, id) + .await + .unwrap(), + RunOutcome::Completed { .. } + )); + assert_no_bodies(provider_dir.path()); + let request = client.request_delivery(id, &[8; 32]).unwrap(); + let delivered = service.deliver(&request, &[8; 32]).unwrap(); + client + .receive( + id, + &fetch_ready(), + &WorkDelivered { + result: delivered.result.encode(), + provider_signature: delivered.signature.as_bytes().to_vec(), + transcript: delivered.transcript, + }, + ) + .unwrap(); + assert_no_bodies(provider_dir.path()); + let payment = client.pay(id).unwrap(); + + // A delivery already happened: restart loses bodies but can still bank the signed payment. + drop(service); + let store = metadata_store(provider_dir.path()); + assert!( + store + .state() + .job_by_id(id) + .unwrap() + .prepared_input() + .is_empty() + ); + assert!(store.state().job_by_id(id).unwrap().transcript().is_empty()); + let mut recovered = ProviderEndpoint::new(fetch_ready(), store, provider()).unwrap(); + assert!(matches!( + recovered.deliver(&request, &fetch_ready(), &[8; 32]), + Err(DeliverError::NoResult { .. }) + )); + assert_eq!(recovered.admit(&payment).unwrap(), PRICE); + assert_eq!(recovered.admit(&payment).unwrap(), PRICE); + assert_eq!(backend.0.load(Ordering::SeqCst), 1); + assert_no_bodies(provider_dir.path()); +} + +#[tokio::test] +async fn restarting_an_accepted_fetch_never_reinvokes_from_a_missing_payload() { + let provider_dir = temp(); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let service_before = service(provider_dir.path()); + let request = client.propose(&proposal()).unwrap(); + let id = client.accepted(&service_before.accept(&request)).unwrap(); + drop(service_before); + let backend = FetchBackend::default(); + let outcome = run_accepted_work(&service(provider_dir.path()), &fetch_ready(), &backend, id) + .await + .unwrap(); + assert_eq!(outcome, RunOutcome::Indeterminate); + assert_eq!(backend.0.load(Ordering::SeqCst), 0); + assert_no_bodies(provider_dir.path()); +} + +#[test] +fn fetch_refuses_payload_journals_and_retention_requests() { + let provider_dir = temp(); + assert!(matches!( + ProviderEndpoint::new( + fetch_ready(), + store_at(provider_dir.path(), CURSOR), + provider() + ), + Err(EndpointError::PayloadRetention) + )); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let retained = JobProposal { + prepared_input: input(Retention::Retain).into(), + deadlines: deadlines(), + }; + assert!(client.propose(&retained).is_err()); + + // A peer bypassing the client API cannot obtain a provider acceptance either. + let provider_dir = temp(); + let channel = fetch_ready(); + let authorization = policy() + .propose(channel.channel(), &retained.prepared_input, 1, deadlines()) + .unwrap(); + let id = work_id(channel.channel(), &authorization); + let request = hellas_rpc::pb::work::AcceptWorkRequest { + authorization: authorization.encode(), + client_signature: super::client().sign(signing_hash(id)).as_bytes().to_vec(), + prepared_input: retained.prepared_input.encode().unwrap(), + }; + assert!(matches!( + service(provider_dir.path()).accept(&request).outcome, + Some(hellas_rpc::pb::work::accept_work_response::Outcome::Refused(_)) + )); + assert_no_bodies(provider_dir.path()); +} + +#[test] +fn fetch_close_descriptor_round_trips_and_rejects_profile_version_confusion() { + use hellas_rpc::protocol::work_setup::CloseDescriptor; + let descriptor = descriptor_with(policy()).close_descriptor(); + let bytes = descriptor.encode(); + assert_eq!(CloseDescriptor::decode(&bytes).unwrap(), descriptor); + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(CloseDescriptor::decode(&trailing).is_err()); + let mut wrong_version = bytes; + wrong_version[0] = 1; + assert!(CloseDescriptor::decode(&wrong_version).is_err()); +} + +#[tokio::test] +async fn fetch_output_limits_include_terminal_and_bad_results_are_never_payable() { + let provider_dir = temp(); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let service = service(provider_dir.path()); + let request = client.propose(&proposal()).unwrap(); + let id = client.accepted(&service.accept(&request)).unwrap(); + let RunAdmission::InvokeFetch(input) = service.begin_run(id, &fetch_ready()).unwrap() else { + panic!("first Fetch dispatch"); + }; + let transcript = FetchBackend::default().fetch(*input).await.unwrap(); + let PaidWorkPolicy::Fetch { mut policy, .. } = policy() else { + unreachable!() + }; + policy.max_output_events = u32::try_from(transcript.len() - 1).unwrap(); + assert!( + hellas_rpc::protocol::work_fetch::check_fetch_output_limits(&policy, &transcript).is_err() + ); + policy.max_output_events += 1; + policy.max_output_bytes = 1; + assert!( + hellas_rpc::protocol::work_fetch::check_fetch_output_limits(&policy, &transcript).is_err() + ); + assert!( + service + .record_result(id, &transcript[..transcript.len() - 1]) + .is_err() + ); + assert!(client.pay(id).is_err()); + assert_no_bodies(provider_dir.path()); +} + +#[tokio::test] +async fn app_attest_paid_fetch_uses_the_signed_scheme_without_disk_payloads() { + let provider_dir = temp(); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let service = service(provider_dir.path()); + let proposal = JobProposal { + prepared_input: input_with_assurance(Retention::Ephemeral, Assurance::AppleAppAttest) + .into(), + deadlines: deadlines(), + }; + let request = client.propose(&proposal).unwrap(); + let id = client.accepted(&service.accept(&request)).unwrap(); + let backend = FetchBackend::default(); + assert!(matches!( + run_accepted_work(&service, &fetch_ready(), &backend, id) + .await + .unwrap(), + RunOutcome::Completed { .. } + )); + let request = client.request_delivery(id, &[8; 32]).unwrap(); + let delivered = service.deliver(&request, &[8; 32]).unwrap(); + client + .receive( + id, + &fetch_ready(), + &WorkDelivered { + result: delivered.result.encode(), + provider_signature: delivered.signature.as_bytes().to_vec(), + transcript: delivered.transcript, + }, + ) + .unwrap(); + let payment = client.pay(id).unwrap(); + use hellas_rpc::services::work::WorkHandler; + let response: hellas_rpc::call::WithTrailer = + service + .admit_certificate(payment, hellas_wire::TransportContext::default()) + .await + .unwrap() + .into(); + assert!(matches!( + response.response.outcome, + Some(hellas_rpc::pb::work::admit_certificate_response::Outcome::Paid(_)) + )); + assert_no_bodies(provider_dir.path()); +} diff --git a/crates/work/tests/work_store_channel.rs b/crates/work/tests/work_store_channel.rs index 9c4f6187..45a1834c 100644 --- a/crates/work/tests/work_store_channel.rs +++ b/crates/work/tests/work_store_channel.rs @@ -443,6 +443,154 @@ fn commit_all(store: &mut ChannelStore, records: &[ChannelRecord]) { } } +fn open_metadata(root: &std::path::Path) -> ChannelStore { + ChannelStore::open_metadata_only( + root, + channel(), + settlement(), + origin_of(&channel()), + &Secp256k1Verifier::new(), + ) + .expect("metadata provider journal opens") +} + +fn assert_payloads_absent(root: &std::path::Path, needles: &[&[u8]]) { + for entry in std::fs::read_dir(root).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + assert_payloads_absent(&path, needles); + } else { + let bytes = std::fs::read(&path).unwrap(); + for needle in needles { + assert!(!needle.is_empty()); + assert!( + !bytes.windows(needle.len()).any(|window| window == *needle), + "customer payload reached {}", + path.display() + ); + } + } + } +} + +#[test] +fn metadata_journal_never_writes_payloads_at_any_commit_or_rotation() { + let channel = channel(); + let job = job_at(&channel, 1, 0); + let sequence = provider_sequence(&channel, &job); + let input = bundle_bytes(1); + let output = spool(&job.transcript); + let payloads = [&input[..], &output[..], job.transcript[0].payload()]; + for boundary in 1..=sequence.len() { + let root = temp(); + let mut store = open_metadata(root.path()); + for record in &sequence[..boundary] { + store + .commit(record.clone(), &Secp256k1Verifier::new()) + .unwrap(); + assert_payloads_absent(root.path(), &payloads); + } + if let Some(active) = store.state().job_by_id(job.work_id) { + assert_eq!(active.prepared_input(), input); + if active.result().is_some() { + assert_eq!(active.transcript(), output); + } + } + store.rotate().unwrap(); + assert_payloads_absent(root.path(), &payloads); + drop(store); + let reopened = open_metadata(root.path()); + if let Some(active) = reopened.state().job_by_id(job.work_id) { + assert!(active.prepared_input().is_empty()); + assert!(active.transcript().is_empty()); + assert_eq!(active.authorization(), &job.authorization); + } else { + assert_eq!(reopened.state().ledger().credited_cumulative(), PRICE); + } + } +} + +#[test] +fn metadata_journal_can_credit_an_already_delivered_result_after_restart() { + let root = temp(); + let channel = channel(); + let job = job_at(&channel, 1, 0); + let mut store = open_metadata(root.path()); + commit_all(&mut store, &provider_sequence(&channel, &job)[..5]); + drop(store); + let mut recovered = open_metadata(root.path()); + recovered + .commit(job.paid(&channel), &Secp256k1Verifier::new()) + .unwrap(); + assert_eq!(recovered.state().ledger().credited_cumulative(), PRICE); + recovered.rotate().unwrap(); + drop(recovered); + assert_eq!( + open_metadata(root.path()) + .state() + .ledger() + .credited_cumulative(), + PRICE + ); +} + +#[test] +fn metadata_replay_does_not_weaken_live_payload_validation() { + let root = temp(); + let channel = channel(); + let job = job_at(&channel, 1, 0); + let mut store = open_metadata(root.path()); + let mut proposal = job.proposed(); + if let ChannelRecord::JobProposed { prepared_input, .. } = &mut proposal { + prepared_input.clear(); + } + assert!(store.commit(proposal, &Secp256k1Verifier::new()).is_err()); + assert!(store.is_empty()); + commit_all(&mut store, &provider_sequence(&channel, &job)[..3]); + let mut result = job.result_record(&channel); + if let ChannelRecord::JobResult { transcript, .. } = &mut result { + transcript.clear(); + } + let records = store.len(); + assert!(store.commit(result, &Secp256k1Verifier::new()).is_err()); + assert_eq!(store.len(), records); +} + +#[test] +fn journal_retention_mode_cannot_change_when_reopened() { + for metadata in [false, true] { + let root = temp(); + let store = if metadata { + open_metadata(root.path()) + } else { + open(root.path(), Role::Provider) + }; + drop(store); + let reopened = if metadata { + ChannelStore::open( + root.path(), + channel(), + settlement(), + Role::Provider, + origin_of(&channel()), + &Secp256k1Verifier::new(), + ) + } else { + ChannelStore::open_metadata_only( + root.path(), + channel(), + settlement(), + origin_of(&channel()), + &Secp256k1Verifier::new(), + ) + }; + assert!( + reopened.is_err(), + "retention mode is part of journal identity" + ); + } +} + // ── The happy path ──────────────────────────────────────────────────── /// One paid job credits its certificate and rests at a certified diff --git a/crates/work/tests/work_store_setup.rs b/crates/work/tests/work_store_setup.rs index 9a4a00fe..71b6f0d4 100644 --- a/crates/work/tests/work_store_setup.rs +++ b/crates/work/tests/work_store_setup.rs @@ -157,7 +157,7 @@ fn provider_policy() -> ProviderChannelPolicy { network: network(), policy_salt: SALT, channel_policy: channel_policy(), - execution_policy: execution_policy(), + execution_policy: execution_policy().into(), expected_payment_values: EdgeValues::new(3_000, 200, Fees::ZERO), min_omit_response_blocks: hellas_kernel::MIN_OMIT_RESPONSE_BLOCKS, } diff --git a/proto/hellas/work/v1/work.proto b/proto/hellas/work/v1/work.proto index 6fbfeea4..73edfd97 100644 --- a/proto/hellas/work/v1/work.proto +++ b/proto/hellas/work/v1/work.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package hellas.work.v1; +import "hellas/v1/hellas.proto"; + // The paid-work service: the wire by which a client proposes one job on // one payment channel and a provider accepts or refuses it. // @@ -11,6 +13,8 @@ package hellas.work.v1; // exact encodings. A protobuf re-spelling of an authorization would be a // second definition of what both parties signed. service Work { + rpc Open(.hellas.v1.OpenRequest) returns (.hellas.v1.OpenResponse); + // One bilateral acceptance. The client sends the authorization it has // already signed; the provider answers with its co-signature or with // one refusal code. @@ -58,6 +62,8 @@ service Work { // would make every paid method answer for a channel that does not exist // yet, and every setup method answer for one that already does. service WorkSetup { + rpc Open(.hellas.v1.OpenRequest) returns (.hellas.v1.OpenResponse); + // One step of the handshake: the caller offers the revision it holds, // and is answered with the revision that extends it. // From 994e1eaceb67bf90876c00c8a6030b4d8e2d08c2 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Thu, 24 Sep 2026 06:23:39 +0200 Subject: [PATCH 03/32] refactor: simplify paid Fetch interfaces and documentation Share provisioning logic between frontends, remove redundant session and offer state, and replace the one-method backend adapter with a callback. Remove the unused Evaluate compatibility alias and require explicit assurance when verifying Fetch results. Keep Gate's one-job client entry point. Condense historical commentary and document payload-digest behavior precisely. Canonical records and wire encodings are unchanged. Validation: 691 tests passed across CLI, SDK, RPC and work; formatting and diff checks pass. The cleanup removes 637 net lines. --- crates/cli/src/commands/serve/provision.rs | 28 +- crates/rpc/src/http_fetch.rs | 4 +- crates/rpc/src/protocol/work_fetch.rs | 380 ++++----------------- crates/rpc/src/protocol/work_profile.rs | 2 +- crates/rpc/tests/paid_fetch_vectors.rs | 50 ++- crates/sdk/src/paid_client.rs | 28 +- crates/sdk/src/paid_provider.rs | 229 +++---------- crates/sdk/src/work_config.rs | 208 ++--------- crates/sdk/src/work_provision.rs | 235 +++---------- crates/sdk/src/work_provision/tests.rs | 18 +- crates/work/src/work.rs | 3 - 11 files changed, 274 insertions(+), 911 deletions(-) diff --git a/crates/cli/src/commands/serve/provision.rs b/crates/cli/src/commands/serve/provision.rs index de84e95e..62911fbf 100644 --- a/crates/cli/src/commands/serve/provision.rs +++ b/crates/cli/src/commands/serve/provision.rs @@ -1 +1,27 @@ -pub use hellas_sdk::work_provision::{ProvisionOptions, run_provision}; +//! CLI presentation for SDK bond provisioning. +use super::super::CliResult; +pub use hellas_sdk::work_provision::ProvisionOptions; +use hellas_sdk::work_provision::{preview_bond, provision_offer}; + +pub async fn run_provision(options: ProvisionOptions) -> CliResult<()> { + if options.print_bond_only { + println!( + "bond_edge: {}", + hex::encode(preview_bond(&options)?.to_bytes()) + ); + return Ok(()); + } + let root = options.work_config.journal_root.clone(); + let made = provision_offer(options).await?; + println!( + "offer journaled: bond {} under {}", + hex::encode(made.bond_edge.to_bytes()), + root.display() + ); + println!( + "history floor: finalized height {} with payload {}", + made.floor.height, + hex::encode(made.floor.payload) + ); + Ok(()) +} diff --git a/crates/rpc/src/http_fetch.rs b/crates/rpc/src/http_fetch.rs index 938f5e35..d645d756 100644 --- a/crates/rpc/src/http_fetch.rs +++ b/crates/rpc/src/http_fetch.rs @@ -1,5 +1,5 @@ -//! Caller-signed HTTPS request vocabulary. Account secrets are selected by -//! alias and never carried in a client request. +//! Caller-signed HTTPS request vocabulary. Provider-owned credentials are selected +//! by alias; their secret values stay with the provider. use base64::{Engine as _, engine::general_purpose::STANDARD}; use serde::{Deserialize, Serialize}; diff --git a/crates/rpc/src/protocol/work_fetch.rs b/crates/rpc/src/protocol/work_fetch.rs index 34d47fcc..9b33bb45 100644 --- a/crates/rpc/src/protocol/work_fetch.rs +++ b/crates/rpc/src/protocol/work_fetch.rs @@ -1,25 +1,6 @@ -//! The paid-fetch profile: private records for paid jobs whose execution -//! is a Fetch request rather than a local evaluation. -//! -//! The shared machinery in [`super::work`] is profile-agnostic: the -//! authorization binds a policy digest, a prepared-input digest, a request -//! commitment, and an environment commitment, and the result and payment -//! records derive from it. What a profile owns is the meaning of those -//! four commitments. For the evaluate profile the policy is -//! [`super::work::PaidExecutionPolicyV1`] and the prepared input is a -//! six-body artifact bundle; for this profile the policy is -//! [`PaidFetchPolicyV1`] and the prepared input is the client's signed -//! fetch input transcript together with the environment manifest it runs -//! in. -//! -//! A paid fetch job buys one provider-signed HTTP transformation — one -//! [`crate::fetch`] call — rather than local model evaluation. The client -//! signs the request as a fetch input transcript; the provider answers -//! with a signed fetch output transcript; this module is where those -//! transcripts meet the channel's money. The record tag and the digest -//! domains below are what separate the two profiles on the wire: a tag-5 -//! policy is a fetch policy under every decoder, and a digest computed -//! here cannot be reproduced by the evaluate profile's domains. +//! Paid Fetch binds a caller-signed request and manifest to a channel authorization. +//! Results must verify under the requested assurance and the channel's provider key. +//! Fetch policies use record tag 5 and separate digest domains from Evaluate. use hellas_kernel::{Encode, NetworkId}; use hellas_xet::{MIN_CHUNK_SIZE, XetFileHasher}; @@ -39,11 +20,7 @@ use crate::{ ProgramManifest, PublicKey, RequestCommitment, }; -// ── Domains ─────────────────────────────────────────────────────────── -// -// Same discipline as `super::work`: every digest here is the Xet hash of -// one of these byte strings followed by canonical fields, the strings are -// written once, and changing one changes every digest computed under it. +// Digest domains are part of the wire contract. /// Commitment to the canonical route-policy body. const FETCH_ROUTE: &[u8] = b"hellas.work.fetch-route.v1"; @@ -61,26 +38,11 @@ const SEALED_ROUTE_SCHEMA: &str = "hellas.work.fetch-route.sealed.v1"; /// Schema tag of the open-fetch body. const OPEN_FETCH_SCHEMA: &str = "hellas.work.fetch-route.open.v1"; -/// The route a fetch channel sells, committed to by digest. -/// -/// A [`PaidFetchPolicyV1`] cannot carry this body — it is variable-length, -/// and the policy record is fixed-width — so the policy carries -/// [`fetch_route_commitment`] of its canonical bytes instead, and -/// [`check_prepared_fetch_input`] opens the commitment before comparing -/// anything against it. That is the same arrangement the evaluate profile -/// uses for its generation policy: a variable-length canonical body, a -/// fixed digest in the record. -/// -/// The body is canonical DAG-CBOR with one schema tag per variant, so a -/// route has exactly one byte spelling and the commitment cannot be -/// opened to a different route than the one that was hashed. +/// Canonical route policy. Its variable-length body is committed by digest in +/// the fixed-width `PaidFetchPolicyV1` record and opened during input validation. #[derive(Clone, Debug, PartialEq, Eq)] pub enum FetchRoutePolicy { - /// The one `(service, method)` pair this channel sells. - /// - /// These are the route labels a sealed fetch input transcript signs; - /// the paid profile refuses a job whose signed labels differ, so the - /// pair here is the whole of the channel's routing vocabulary. + /// Requires the signed request to use this service and method. SealedRoute { /// The signed `service` component, e.g. `openai`. service: String, @@ -100,19 +62,8 @@ pub enum FetchRoutePolicy { } impl FetchRoutePolicy { - /// Builds the sealed route, applying the same component rules the - /// fetch transcript scheme applies to the signed `service` and - /// `method` events: neither empty, neither over - /// [`MAX_FETCH_ROUTE_COMPONENT_BYTES`]. The two rules are shared with - /// `crate::fetch` rather than restated, so a route this profile - /// accepts is a route the transcript scheme accepts. - /// - /// # Errors - /// - /// [`FetchProtocolError::EmptyService`], [`FetchProtocolError::EmptyMethod`], - /// or [`FetchProtocolError::RouteComponentLimit`] — the fetch - /// protocol's own route errors, because they are the fetch protocol's - /// own rules. + /// Builds a route with nonempty service and method components bounded by + /// `MAX_FETCH_ROUTE_COMPONENT_BYTES`. pub fn sealed_route( service: impl Into, method: impl Into, @@ -140,13 +91,8 @@ impl FetchRoutePolicy { Ok(Self::SealedRoute { service, method }) } - /// Builds the open-fetch vocabulary. - /// - /// The host list is sorted and deduplicated: an allowlist is a set, - /// and a set with two spellings would be two commitments to one - /// policy. Admission matches exact canonical URL host names and checks - /// required pins; the HTTPS interpreter performs certificate and address - /// checks when connecting. + /// Sorts and deduplicates the host allowlist before commitment. Admission checks + /// exact hosts and required pins; the HTTPS driver checks certificates and addresses. #[must_use] pub fn open_fetch(require_spki_pin: bool, allowed_hosts: impl Into>) -> Self { let mut allowed_hosts = allowed_hosts.into(); @@ -158,11 +104,7 @@ impl FetchRoutePolicy { } } - /// Returns the canonical body bytes the policy commits to. - /// - /// The one boolean encodes as the integer `0` or `1`: the canonical - /// encoder has no boolean primitive, and a definite two-value integer - /// is the smallest encoding that cannot drift. + /// Canonical route encoding. The pin flag is encoded as integer 0 or 1. #[must_use] pub fn canonical_body_bytes(&self) -> Vec { let mut encoder = crate::DagCborEncoder::new(); @@ -189,16 +131,8 @@ impl FetchRoutePolicy { encoder.into_bytes() } - /// Decodes one canonical route-policy body, strictly. - /// - /// Strict the way a committed body has to be: an unknown schema, a - /// wrong array length, a pin that is not `0` or `1`, a trailing byte, - /// and a noncanonical integer are all refused, and the value is - /// re-encoded and compared so the bytes accepted are the bytes - /// [`Self::canonical_body_bytes`] would have produced. The sealed - /// variant is rebuilt through [`Self::sealed_route`], so a body that - /// spells a route the transcript scheme would refuse is refused here - /// too. + /// Rejects unknown schemas, invalid components, noncanonical encodings and + /// trailing bytes. Re-encoding must reproduce the input. pub fn from_canonical_body_bytes(bytes: &[u8]) -> Result { let mut decoder = CanonicalDecoder::new(bytes); let len = decoder.array_len()?; @@ -252,11 +186,7 @@ impl FetchRoutePolicy { } } -/// Returns the commitment a [`PaidFetchPolicyV1`] names as its route. -/// -/// Length-prefixed and streamed, mirroring -/// [`super::work::generation_policy_digest`]: the route body is -/// variable-length, so this digest must not use the single-chunk hasher. +/// Hashes the length-prefixed route body with the streaming hasher. pub fn fetch_route_commitment(canonical_body_bytes: &[u8]) -> Result { Ok(xfh( FETCH_ROUTE, @@ -269,17 +199,8 @@ pub fn fetch_route_commitment(canonical_body_bytes: &[u8]) -> Result Result<(), PaidWorkError> { for (field, value) in [ ("fixed_price", policy.fixed_price), @@ -388,11 +297,7 @@ pub fn check_fetch_policy(policy: &PaidFetchPolicyV1) -> Result<(), PaidWorkErro Ok(()) } -/// Returns the digest an authorization names as its execution policy. -/// -/// The shared [`PaidJobAuthorizationV1`] field is called -/// `execution_policy_digest` because the record predates the second -/// profile; under this profile it carries this digest. +/// Fetch policy digest stored in the authorization's `execution_policy_digest`. pub fn fetch_policy_digest(channel: &PaidChannel, policy: &PaidFetchPolicyV1) -> Digest { let network_bytes = channel.network_bytes(); xh( @@ -407,34 +312,16 @@ pub fn fetch_policy_digest(channel: &PaidChannel, policy: &PaidFetchPolicyV1) -> // ── The prepared fetch input bundle ─────────────────────────────────── -/// The two canonical bodies a paid fetch job is prepared from, in one -/// length-delimited byte string. -/// -/// The fetch analogue of -/// [`crate::protocol::artifacts::PreparedPaidInputV1`]: -/// same `u32` big-endian length prefixes, same budget-checked -/// [`Self::decode`], same strict [`Self::parts`]. Where the evaluate -/// bundle carries an artifact graph, this one carries exactly the two -/// things the profile's commitments are opened from: the client's signed -/// fetch input transcript, and the manifest of the environment it runs -/// in. -/// -/// It carries bytes rather than parsed values for the same reason the -/// evaluate bundle does — the bytes are what the digest commits to — with -/// one sharpened consequence: the signed transcript's canonical bytes are -/// hashed here, so [`Self::parts`] decodes it strictly rather than with -/// the permissive DAG-CBOR decoder a transport boundary might use. +/// Canonical signed input transcript and manifest, each prefixed by a big-endian +/// `u32` length. The original bytes are retained because the authorization commits +/// to them; `parts` verifies canonical encoding when parsing. #[derive(Clone, Debug, PartialEq, Eq)] pub struct PreparedPaidFetchInputV1 { fetch_input_transcript: Vec, environment_manifest: Vec, } -/// The two bodies of a [`PreparedPaidFetchInputV1`], parsed. -/// -/// The manifest is retained as its parsed value for the same reason the -/// evaluate bundle retains its own: parsing proves the carried bytes were -/// canonical, and re-encoding recovers them exactly. +/// Parsed canonical request transcript and manifest. #[derive(Clone, Debug, PartialEq, Eq)] pub struct PreparedPaidFetchInputParts { /// The signed fetch input events, strictly decoded. @@ -444,13 +331,7 @@ pub struct PreparedPaidFetchInputParts { } impl PreparedPaidFetchInputV1 { - /// Builds a bundle from the signed input events and the manifest, - /// encoding each body once. - /// - /// # Errors - /// - /// [`PaidWorkError::Transcript`] when the transcript's canonical - /// encoding cannot be allocated. + /// Encodes the signed input events and manifest into a prepared bundle. pub fn new( fetch_input_transcript: &[InputEventEnvelope], environment_manifest: &ProgramManifest, @@ -461,12 +342,7 @@ impl PreparedPaidFetchInputV1 { }) } - /// Returns the canonical encoding: two unsigned big-endian `u32` - /// lengths, each immediately followed by that many body bytes. - /// - /// Fallible for the one reason the evaluate bundle's is: a body whose - /// length does not fit its `u32` prefix has no encoding here, because - /// truncating the prefix would be a second spelling of the same bytes. + /// Encodes each body with a big-endian `u32` length, rejecting overflow. pub fn encode(&self) -> Result, CanonicalDecodeError> { let mut bytes = Vec::new(); for body in self.bodies() { @@ -482,13 +358,8 @@ impl PreparedPaidFetchInputV1 { Ok(bytes) } - /// Decodes a bundle, refusing anything that does not fit `budget`. - /// - /// `budget` is the profile's complete-bundle limit - /// ([`PaidFetchPolicyV1::max_encoded_prepared_input`]), checked against - /// the input before the first length is read and against the running - /// total after each one, so the two individually representable lengths - /// cannot add up to a bundle this endpoint never agreed to hold. + /// Decodes two length-prefixed bodies within the total `budget`, rejecting + /// truncation, overflow and trailing bytes. pub fn decode(bytes: &[u8], budget: usize) -> Result { if bytes.len() > budget { return Err(CanonicalDecodeError::new(format!( @@ -510,14 +381,8 @@ impl PreparedPaidFetchInputV1 { Ok(bundle) } - /// Parses both bodies, rejecting either that is not canonical. - /// - /// Both bodies arrive already bounded: by [`Self::decode`]'s budget - /// for a received bundle, or by [`Self::new`]'s construction for a - /// local one. The transcript is decoded strictly — decoded, - /// re-encoded, and compared — because its bytes are what - /// [`prepared_fetch_input_digest`] commits to, so a noncanonical - /// spelling is a different bundle and must not parse as this one. + /// Parses both bodies and requires their canonical encoding to match the + /// committed bytes. pub fn parts(&self) -> Result { Ok(PreparedPaidFetchInputParts { fetch_input_transcript: decode_input_transcript(&self.fetch_input_transcript)?, @@ -530,13 +395,7 @@ impl PreparedPaidFetchInputV1 { } } -/// Encodes the signed fetch input events for the bundle. -/// -/// The input-side analogue of [`super::work::encode_transcript`]: DAG-CBOR -/// over the signed envelopes, through the same derived `Serialize`. The -/// one difference is that this encoding *is* hashed — it is a bundle body -/// — which is why [`decode_input_transcript`] below is strict where -/// [`super::work::decode_transcript`] is not. +/// Canonical DAG-CBOR encoding of the signed input envelopes. fn encode_input_transcript(transcript: &[InputEventEnvelope]) -> Result, PaidWorkError> { canonical_dag_cbor(&transcript.to_vec()) .map_err(|error| PaidWorkError::Transcript(error.to_string())) @@ -548,12 +407,7 @@ fn decode_input_transcript(bytes: &[u8]) -> Result, Cano decode_canonical_dag_cbor(bytes) } -/// Returns the digest an authorization names as its prepared input. -/// -/// Streamed, mirroring [`super::work::prepared_input_digest`]: the bundle -/// carries a signed transcript whose size is the client's to choose -/// within the policy bound, so it is one of the preimages with no fixed -/// width. +/// Hashes the complete prepared bundle, bound to the channel and network. pub fn prepared_fetch_input_digest( channel: &PaidChannel, bundle: &PreparedPaidFetchInputV1, @@ -571,29 +425,9 @@ pub fn prepared_fetch_input_digest( // ── Authorization ───────────────────────────────────────────────────── -/// Builds one fetch job's authorization, deriving every field the -/// channel, the policy, and the prepared inputs already fix. -/// -/// The fetch analogue of [`super::work::propose_authorization`], with the -/// same narrow guarantee: the nonce and the three deadlines are the only -/// choices left to the caller, and every other field is read out of -/// something that already exists. The request commitment is computed the -/// fetch ticket flow's way — the signed input transcript is verified by -/// [`crate::fetch::verify_input_events`], and its input commitment is the -/// request commitment — so a proposal cannot name a commitment its own -/// transcript does not produce. -/// -/// It checks nothing beyond what deriving those fields requires. -/// [`check_fetch_authorization`] and [`check_prepared_fetch_input`] are -/// where the refusals are written, and they are what the *other* party -/// runs. -/// -/// # Errors -/// -/// [`PaidWorkError::Body`] when the bundle's own bodies are not canonical, -/// [`PaidWorkError::Transcript`] when the carried events are not one -/// well-formed signed fetch input, and [`PaidWorkError::Overflow`] when -/// the bundle is too large to length-prefix. +/// Derives authorization commitments from the verified request, policy and channel. +/// Only the nonce and deadlines are caller-selected. Admission rules are enforced +/// by `check_fetch_authorization` and `check_prepared_fetch_input`. pub fn propose_fetch_authorization( channel: &PaidChannel, policy: &PaidFetchPolicyV1, @@ -623,16 +457,8 @@ pub fn propose_fetch_authorization( }) } -/// Checks one authorization against the channel, the fetch policy it -/// names, and the height it is being signed at, and returns its -/// `work_id`. -/// -/// The fetch analogue of [`super::work::check_authorization`]. Everything -/// the two profiles mean by *accepting* a job — channel fields, bond -/// cover, price cover, deadline window — is shared -/// [`check_authorization_core`]; what differs is which policy digest, -/// environment, and price those rules are applied to, and that is what -/// this wrapper supplies. +/// Checks channel, policy, price and deadline bounds at the finalized height; +/// returns the work ID. pub fn check_fetch_authorization( channel: &PaidChannel, authorization: &PaidJobAuthorizationV1, @@ -650,32 +476,10 @@ pub fn check_fetch_authorization( ) } -/// Checks the prepared fetch bundle against the authorization, the -/// policy, and the route policy that commit to it. -/// -/// The fetch analogue of [`super::work::check_prepared_input`], and the -/// same statement in this profile's vocabulary: holding a bundle whose -/// digest matches is not knowing what is in it. The checks, in order: -/// the bundle fits the policy's encoding bound; its digest is the -/// authorization's; the signed events verify as one caller-signed fetch -/// input chain under [`crate::fetch::verify_input_events`] — which also -/// applies the fetch protocol's own hard bounds, including -/// [`crate::fetch::MAX_FETCH_REQUEST_BODY_BYTES`]; the caller is the -/// channel's client; the signed assurance selects the result scheme; the route body -/// opens the policy's route commitment and, for a sealed route, names the -/// service and method the events sign; the manifest, the events' -/// environment, the policy's allowed environment, and the -/// authorization's environment commitment are one; and the request -/// commitment is the verified transcript's. -/// -/// The route policy is an argument rather than a bundle body because the -/// bundle is the client's statement of the job and the route is the -/// channel's statement of what it sells: the two meet here, against the -/// commitment both signed into the policy. -/// -/// Open Fetch additionally verifies the generic HTTPS manifest, request schema, -/// host allowlist and any required SPKI pin. Network-dependent certificate and -/// address checks happen inside that interpreter at dispatch. +/// Checks bundle bounds, commitments, caller signature, route and environment +/// against the channel policy and authorization. Open Fetch also checks its HTTP +/// schema, host allowlist and required pins. The HTTPS driver checks certificates +/// and addresses when connecting. pub fn check_prepared_fetch_input( channel: &PaidChannel, authorization: &PaidJobAuthorizationV1, @@ -707,11 +511,7 @@ pub fn check_prepared_fetch_input( let input = crate::fetch::verify_input_events(&parts.fetch_input_transcript) .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; - // The fetch analogue of the evaluate profile's runner-key rule: the - // request must be the client's own, and one verified signature chain - // is how a fetch request says whose it is. Verification takes the key - // from the first event, so it cannot say whose key it is; this is - // what says it is the client's. + // The signing key must also be the channel's client key. if input.caller_key != PublicKey::Secp256k1(channel.client_key().to_bytes()) { return Err(PaidWorkError::Mismatch { field: "caller_key", @@ -845,20 +645,10 @@ pub fn check_fetch_output_limits( // ── The terminal result ─────────────────────────────────────────────── -/// Returns the digest of the normalized fetch answer. -/// -/// Chunk boundaries are not part of the answer, exactly as in -/// [`super::work::canonical_output_digest`]: the semantic event payloads -/// and the terminal payload are flattened into one byte stream, so two -/// providers that split the same answer across different signed events -/// produce the same digest here while producing different transcript -/// commitments. The fetch answer has no token ids and no counts to -/// cross-check — the payload bytes are the answer, which is why this -/// digest, unlike the evaluate one, has nothing to refuse. The payloads -/// are canonical DAG-CBOR with distinct event and terminal codec strings, -/// so a reader that parses the stream can still tell where the terminal -/// begins even though this digest does not bind that boundary; the -/// transcript commitment beside it binds the framing exactly. +/// Hashes the concatenated canonical event and terminal payload bytes. +/// Envelope framing is bound separately by the terminal transcript commitment. +/// Changing payload encodings, including how content is split into events, can +/// change this digest. #[must_use] pub fn fetch_canonical_output_digest( network: NetworkId, @@ -878,59 +668,20 @@ pub fn fetch_canonical_output_digest( hasher.finalize() } -/// Builds the result record for the fetch transcript one invocation of -/// this job produced. -/// -/// The fetch analogue of [`super::work::terminal_result`], with the same -/// shape and the same reason for it: what makes a result the provider's -/// own is that the events it summarises verify as one signed chain — the -/// fetch scheme, this authorization's request commitment, contiguous -/// sequence from the output genesis, and a terminal event at the end, -/// which is exactly what [`crate::fetch::verify_output_events`] checks. -/// None of that can be supplied by a caller holding a commitment, so no -/// path here accepts one. -/// -/// The two digests it produces say different things about the same -/// invocation, as they do for evaluate: -/// `terminal_transcript_commitment` binds the provider's exact signed -/// framing; [`fetch_canonical_output_digest`] binds the flattened answer, -/// so two transcripts that split the same payloads differently agree on -/// it. -/// -/// # Errors -/// -/// [`PaidWorkError::Transcript`] when the events are not one verified -/// fetch output transcript for this authorization's request. -/// [`PaidWorkError::Mismatch`] when they were produced under a key this -/// channel does not call the provider. +/// Verifies the result using the assurance from the authenticated request and +/// requires the channel's provider key. Returns commitments to the signed +/// transcript and its payload bytes. pub fn terminal_fetch_result( channel: &PaidChannel, authorization: &PaidJobAuthorizationV1, transcript: &[OutputEventEnvelope], -) -> Result { - terminal_fetch_result_with_assurance( - channel, - authorization, - transcript, - Assurance::ProducerSigned, - ) -} - -/// Verifies the assurance committed by the caller's authenticated input. -pub fn terminal_fetch_result_with_assurance( - channel: &PaidChannel, - authorization: &PaidJobAuthorizationV1, - transcript: &[OutputEventEnvelope], assurance: Assurance, ) -> Result { let input = InputCommitment::from_digest(authorization.request_commitment.digest()); let output = crate::fetch::verify_output_events(input, assurance, transcript) .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; - // Verification above establishes that one key signed every event; it - // takes that key from the first event, so it cannot say whose key it - // is. This is what says it is the provider's — the same compressed - // secp256k1 point the payment terms name as a party. + // Bind the transcript signer to the channel's provider. if output.producer_key != PublicKey::Secp256k1(channel.provider_key().to_bytes()) { return Err(PaidWorkError::Mismatch { field: "transcript producer key", @@ -938,10 +689,7 @@ pub fn terminal_fetch_result_with_assurance( } let Some(terminal_event) = transcript.last() else { - // Unreachable for the same reason as in - // `super::work::terminal_result`: verification refuses an empty - // transcript, and a non-empty slice has a last element. Written as - // a refusal because nothing in this module panics. + // Verification already requires a terminal; keep this path fallible. return Err(PaidWorkError::Transcript( "the terminal transcript is empty".to_string(), )); @@ -969,25 +717,13 @@ pub fn terminal_fetch_result_with_assurance( // ── Bounds ──────────────────────────────────────────────────────────── -const ENCODED_NETWORK: usize = ::MAX_ENCODED_SIZE; - -/// Widest of this module's fixed records: the one record, today. Written -/// as a named bound rather than inline so a second fetch record must -/// widen it deliberately, as `super::work::WIDEST_RECORD` does for its -/// five. -const WIDEST_RECORD: usize = PaidFetchPolicyV1::ENCODED_SIZE; - -/// Longest of this module's single-chunk domains. The three -/// variable-body digests are streamed and need no bound; the one -/// fixed-record digest — [`fetch_policy_digest`] — does. -const LONGEST_XH_DOMAIN: usize = FETCH_POLICY.len(); - -/// Largest complete `XH` preimage this module can produce: -/// `domain || network || channel_id || record`, the one record-shaped -/// preimage here. -const WIDEST_XH_PREIMAGE: usize = LONGEST_XH_DOMAIN + ENCODED_NETWORK + 32 + WIDEST_RECORD; - +// The fixed policy digest must fit the single-chunk hasher. Variable bodies +// use the streaming hasher. const _: () = assert!( - WIDEST_XH_PREIMAGE < MIN_CHUNK_SIZE, + FETCH_POLICY.len() + + ::MAX_ENCODED_SIZE + + 32 + + PaidFetchPolicyV1::ENCODED_SIZE + < MIN_CHUNK_SIZE, "a single-chunk preimage that reaches MIN_CHUNK_SIZE panics the hasher" ); diff --git a/crates/rpc/src/protocol/work_profile.rs b/crates/rpc/src/protocol/work_profile.rs index f7a2141e..071a50ec 100644 --- a/crates/rpc/src/protocol/work_profile.rs +++ b/crates/rpc/src/protocol/work_profile.rs @@ -311,7 +311,7 @@ impl PreparedPaidWorkInput { let parts = bundle.parts()?; let input = crate::fetch::verify_input_events(&parts.fetch_input_transcript) .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; - let result = work_fetch::terminal_fetch_result_with_assurance( + let result = work_fetch::terminal_fetch_result( channel, authorization, transcript, diff --git a/crates/rpc/tests/paid_fetch_vectors.rs b/crates/rpc/tests/paid_fetch_vectors.rs index e71f7b06..61e871a6 100644 --- a/crates/rpc/tests/paid_fetch_vectors.rs +++ b/crates/rpc/tests/paid_fetch_vectors.rs @@ -1381,8 +1381,13 @@ fn terminal_fetch_result_is_pinned() { let channel = channel(); let authorization = authorization(); let transcript = output_transcript(&authorization); - let result = terminal_fetch_result(&channel, &authorization, &transcript) - .expect("a legal terminal transcript"); + let result = terminal_fetch_result( + &channel, + &authorization, + &transcript, + Assurance::ProducerSigned, + ) + .expect("a legal terminal transcript"); assert_eq!(result.work_id, work_id(&channel, &authorization)); assert_eq!( @@ -1421,7 +1426,7 @@ fn terminal_fetch_result_refusals() { // MUTATION: an empty transcript. assert!(matches!( - terminal_fetch_result(&channel, &authorization, &[]), + terminal_fetch_result(&channel, &authorization, &[], Assurance::ProducerSigned), Err(PaidWorkError::Transcript(_)) )); @@ -1432,7 +1437,12 @@ fn terminal_fetch_result_refusals() { let builder = FetchOutputTranscriptBuilder::new(wrong_input, Assurance::ProducerSigned, &key); let wrong_request = builder.finish(terminal_payload()).unwrap(); assert!(matches!( - terminal_fetch_result(&channel, &authorization, &wrong_request), + terminal_fetch_result( + &channel, + &authorization, + &wrong_request, + Assurance::ProducerSigned + ), Err(PaidWorkError::Transcript(_)) )); @@ -1444,7 +1454,12 @@ fn terminal_fetch_result_refusals() { builder.push_event(event_payload("paid ")).unwrap(); let client_signed = builder.finish(terminal_payload()).unwrap(); assert_eq!( - terminal_fetch_result(&channel, &authorization, &client_signed), + terminal_fetch_result( + &channel, + &authorization, + &client_signed, + Assurance::ProducerSigned + ), Err(PaidWorkError::Mismatch { field: "transcript producer key" }) @@ -1464,7 +1479,12 @@ fn terminal_fetch_result_refusals() { .unwrap(); let no_terminal = builder.finish().unwrap().0; assert!(matches!( - terminal_fetch_result(&channel, &authorization, &no_terminal), + terminal_fetch_result( + &channel, + &authorization, + &no_terminal, + Assurance::ProducerSigned + ), Err(PaidWorkError::Transcript(_)) )); } @@ -1552,8 +1572,13 @@ fn fetch_result_pays_through_the_shared_ledger() { let channel = channel(); let authorization = authorization(); let transcript = output_transcript(&authorization); - let result = terminal_fetch_result(&channel, &authorization, &transcript) - .expect("a legal terminal transcript"); + let result = terminal_fetch_result( + &channel, + &authorization, + &transcript, + Assurance::ProducerSigned, + ) + .expect("a legal terminal transcript"); let (certificate, binding) = next_payment(&channel, &authorization, &result, 0, capacity()).expect("a legal payment"); @@ -1600,8 +1625,13 @@ fn the_real_input_constructor_passes_the_whole_pipeline() { .expect("a legal prepared input"); let transcript = output_transcript(&authorization); - let result = terminal_fetch_result(&channel, &authorization, &transcript) - .expect("a legal terminal transcript"); + let result = terminal_fetch_result( + &channel, + &authorization, + &transcript, + Assurance::ProducerSigned, + ) + .expect("a legal terminal transcript"); assert_eq!(result.work_id, work_id(&channel, &authorization)); } diff --git a/crates/sdk/src/paid_client.rs b/crates/sdk/src/paid_client.rs index 10e6bbcb..e1ebb954 100644 --- a/crates/sdk/src/paid_client.rs +++ b/crates/sdk/src/paid_client.rs @@ -161,6 +161,10 @@ impl InputIdentities { let parts = prepared .parts() .context("prepared input contains a non-canonical body")?; + Self::from_parts(&parts) + } + + fn from_parts(parts: &hellas_rpc::protocol::artifacts::PreparedPaidInputParts) -> Result { let allowed_environment = parts.manifest.content_id(); anyhow::ensure!( parts.evaluate_request.execution_environment == allowed_environment, @@ -184,7 +188,6 @@ impl InputIdentities { /// Cancellation retains recovery state; call `run(None, true, None)` to resume it. pub struct PaidWorkSession { args: PaidWorkOptions, - config: WorkConfig, descriptor: WorkChannelDescriptor, dialer: ProviderDialer, chain: WorkBlocks, @@ -208,13 +211,13 @@ impl PaidWorkSession { "paid-work timeout must be positive" ); - let config = args.config.clone(); + let config = &args.config; let policy = config.provider_policy(); let bond = args.bond; let payment_funding = args.payment_funding.clone(); let mut next_validator = 0; - let chain = connect_chain(&config, &mut next_validator).await?; - check_genesis(&config, &chain).await?; + let chain = connect_chain(config, &mut next_validator).await?; + check_genesis(config, &chain).await?; std::fs::create_dir_all(&args.journal_root).with_context(|| { format!( @@ -272,7 +275,7 @@ impl PaidWorkSession { setup.arm_scan(finalized_floor(&chain).await?)?; } if setup.state().revision() == Some(1) { - let terms = payment_terms(&config, &policy, &bundle, args.omission_bond); + let terms = payment_terms(config, &policy, &bundle, args.omission_bond); setup.propose_payment(payment_funding, terms)?; } if setup.state().revision() == Some(2) { @@ -291,7 +294,6 @@ impl PaidWorkSession { Ok(Self { args, - config, descriptor, dialer, chain, @@ -334,22 +336,22 @@ impl PaidWorkSession { } CloseProgress::Opened { .. } | CloseProgress::Nothing => {} } - tokio::time::sleep(self.config.poll).await; + tokio::time::sleep(self.args.config.poll).await; } } /// Refreshes finalized state, rotating through configured validators on failure. pub async fn follow_chain(&mut self) -> Result<()> { - for attempt in 0..self.config.validators.len() { + for attempt in 0..self.args.config.validators.len() { match self.client.catch_up(&self.chain).await { Ok(_) => return Ok(()), - Err(error) if attempt + 1 == self.config.validators.len() => { + Err(error) if attempt + 1 == self.args.config.validators.len() => { return Err(error.into()); } Err(error) => { tracing::debug!(%error, "paid channel will continue catch-up through another validator"); - self.chain = connect_chain(&self.config, &mut self.next_validator).await?; - check_genesis(&self.config, &self.chain).await?; + self.chain = connect_chain(&self.args.config, &mut self.next_validator).await?; + check_genesis(&self.args.config, &self.chain).await?; } } } @@ -382,7 +384,6 @@ impl PaidWorkSession { self.follow_chain().await?; let Self { args, - config, descriptor, dialer, chain, @@ -390,6 +391,7 @@ impl PaidWorkSession { needs_recovery, .. } = self; + let config = &args.config; if let Some(prepared) = prepared.as_ref() { check_request( &config.provider_policy(), @@ -781,8 +783,8 @@ pub fn check_evaluate_input( policy: &ProviderChannelPolicy, prepared: &PreparedPaidInputV1, ) -> Result<()> { - let input = InputIdentities::from_prepared(prepared)?; let parts = prepared.parts()?; + let input = InputIdentities::from_parts(&parts)?; let PaidWorkPolicy::Evaluate(expected) = &policy.execution_policy else { bail!("work config does not select the Evaluate profile"); }; diff --git a/crates/sdk/src/paid_provider.rs b/crates/sdk/src/paid_provider.rs index 95233e38..862d3720 100644 --- a/crates/sdk/src/paid_provider.rs +++ b/crates/sdk/src/paid_provider.rs @@ -121,21 +121,7 @@ impl WorkHandler for UnmountedWork { } } -// ── The clock ───────────────────────────────────────────────────────── -// -// Everything below is a runner and nothing below is a decision. It -// builds no transaction, fixes no deadline, chooses no settlement, -// judges no duty due, and does not decide whether admission is on: each -// of those is a library edge it calls on a cadence, and the cadence is -// the whole of what this file adds. What it owns is *when* — and the -// journals, which is why it hands them to nobody. - -/// What the clock over one node's paid-work journals is built from. -/// -/// Every field is something the serve path has already loaded and -/// checked. The policy most of all: it is the loaded work -/// configuration's, carried here rather than derived again, so there is -/// no second place a node could decide what it countersigns over. +/// Validated configuration for driving provider journals. pub struct WorkRunnerConfig { /// The network the journals are keyed and the signatures bound to. pub network: NetworkId, @@ -155,14 +141,8 @@ pub struct WorkRunnerConfig { pub policy: ProviderChannelPolicy, } -/// The channel this node answers `Work` from, once the runner has been -/// handed one. -/// -/// Written by the runner and read by the accept loop. What crosses is a -/// clone of a handler whose mutable pieces are themselves behind `Arc`s, -/// so the lock is held for a clone and never across a request: the -/// dispatch path never waits while holding the mount, and the clock never -/// waits on a request. +/// Channels indexed by authenticated peer. Handlers share the runner's state; +/// mount locks are released before processing requests. #[derive(Clone)] pub struct MountedWork { mounted: Arc>>>>, @@ -178,59 +158,32 @@ impl Default for MountedWork { } } -/// A cloneable, type-erased owner of the backend that runs accepted work. -/// -/// The production value owns an [`hellas_executor::ExecutorHandle`]. -/// Keeping the backend behind this narrow local seam means the clock and -/// ALPN dispatcher stay parameterized only over their finalized source; -/// neither has a second opinion about paid admission or execution failure. -#[derive(Clone)] -struct AcceptedWorkDriver(Arc); - -trait DriveAcceptedWork: Send + Sync { - fn run( - &self, - service: WorkService, - ready: ReadyChannel, - work_id: Digest, - ) -> BoxFuture<'static, Result>; -} - -struct BackendWorkDriver { - backend: Arc, -} +/// Type-erased runner for an accepted job. +type RunAcceptedWork = dyn Fn(WorkService, ReadyChannel, Digest) -> BoxFuture<'static, Result> + + Send + + Sync; -impl DriveAcceptedWork for BackendWorkDriver -where - B: PaidWorkBackend + Send + Sync + 'static, -{ - fn run( - &self, - service: WorkService, - ready: ReadyChannel, - work_id: Digest, - ) -> BoxFuture<'static, Result> { - let backend = Arc::clone(&self.backend); - Box::pin( - async move { run_accepted_work(&service, &ready, backend.as_ref(), work_id).await }, - ) - } -} +#[derive(Clone)] +struct AcceptedWorkDriver(Arc); impl AcceptedWorkDriver { fn new(backend: B) -> Self where B: PaidWorkBackend + Send + Sync + 'static, { - Self(Arc::new(BackendWorkDriver { - backend: Arc::new(backend), + let backend = Arc::new(backend); + Self(Arc::new(move |service, ready, work_id| { + let backend = Arc::clone(&backend); + Box::pin( + async move { run_accepted_work(&service, &ready, backend.as_ref(), work_id).await }, + ) })) } /// Starts one accepted job without lending its lifetime to either the /// request path or the close clock. fn spawn(&self, service: WorkService, ready: ReadyChannel, work_id: Digest) { - let running = self.0.run(service, ready, work_id); + let running = (self.0)(service, ready, work_id); let span = hellas_rpc::request_span!(target: "hellas_request", "paid.provider.execute", hellas.work.id = ?work_id, otel.status_code = tracing::field::Empty); tokio::spawn(tracing::Instrument::instrument( async move { @@ -265,13 +218,9 @@ impl AcceptedWorkDriver { } } -/// One mounted channel's served handler. -/// -/// `source` is replaceable because the runner redials a failed validator. -/// The request path copies the current source under the plain mutex and -/// drops that guard before its coherent read awaits. `accepting` spans the -/// complete fresh-read-to-signature sequence, so two acceptance attempts -/// cannot each refresh and then race to consume the same channel credit. +/// One channel's handler. The runner replaces `source` when it redials a validator. +/// `accepting` serializes each fresh readiness read through signing to prevent +/// concurrent requests from consuming the same credit. #[derive(Clone)] pub struct MountedWorkService { bond_edge: EdgeId, @@ -286,17 +235,10 @@ impl MountedWorkService where S: FinalizedBlocks + FinalizedWorkView + Sync, { - /// Re-establishes admission from one fresh coherent read. - /// - /// The service is the exact clone the runner drives. Its cursor is - /// checked after readiness, and that same service receives the fresh - /// decision before the raw handler is reached. A missing policy, - /// failed read, failed predicate, lagging cursor, or endpoint failure - /// therefore leaves the request on the retryable `NotReady` side. + /// Refreshes admission and checks the cursor using one coherent finalized read. + /// Any failure leaves admission disabled until a later successful refresh. pub async fn refresh_admission(&self) -> anyhow::Result { - // A `std::sync::MutexGuard` is deliberately confined to this - // block. Holding the source-slot guard across the read would make - // this handler's future non-`Send` and is not a valid dispatch. + // Drop the source lock before awaiting the chain read. let source = { let held = self .source @@ -330,12 +272,8 @@ where } } -/// Re-establishes admission for the exact driven channel from one coherent -/// finalized read. -/// -/// Both the wire handler and restart recovery call this function. A recovered -/// job therefore gets no weaker interpretation of readiness than a new job, -/// and neither path can accidentally trust the readiness cached at mount. +/// Shared readiness check for live requests and restart recovery. +/// Both paths use a fresh finalized snapshot. async fn refresh_work_admission( service: &WorkService, descriptor: Option<&WorkChannelDescriptor>, @@ -540,12 +478,7 @@ impl MountedWork { } } - /// Adds one owned channel under its authenticated peer. - /// - /// A peer is served only while exactly one channel is mounted under - /// it. Retaining a second candidate rather than overwriting either one - /// makes an ambiguity fail closed instead of turning insertion order - /// into routing policy. + /// Mounts a channel for a peer. Multiple candidates disable routing for that peer. pub fn mount( &self, peer: PeerId, @@ -591,11 +524,7 @@ impl MountedWork { self.handler(context).map(|mounted| mounted.service) } - /// Replaces the finalized source for the matching driven channel. - /// - /// A reconnect reaches handlers already cloned by live connections, - /// because they share this inner source slot. Neither mount lock is - /// held across a source request. + /// Updates the shared source, including handlers held by live connections. fn refresh_source(&self, peer: PeerId, bond_edge: EdgeId, source: &S) { let source_slot = self.mounted.lock().ok().and_then(|held| { held.get(&peer)? @@ -610,12 +539,7 @@ impl MountedWork { } } - /// Stops serving `Work` from every channel. - /// - /// The clock's last act. A channel nobody is advancing is not a - /// channel to answer from — its journal is closed the moment the - /// runner drops it, and a handler still holding it open would be the - /// one thing keeping the files this process no longer owns. + /// Unmounts all channels when the clock stops, releasing its journal handles. pub fn clear_all(&self) { if let Ok(mut held) = self.mounted.lock() { held.clear(); @@ -623,12 +547,7 @@ impl MountedWork { } } -/// Provider setups this node answers `WorkSetup` from by authenticated peer. -/// -/// Written by discovery and read by the accept loop, beside -/// [`MountedWork`]. The clone in this slot is the exact [`SetupService`] -/// stored in [`Driven::Setup`], so serving and driving share one exclusive -/// journal rather than attempting to reopen it. +/// Setups indexed by authenticated peer. Serving and driving share each journal. #[derive(Clone, Debug, Default)] pub struct MountedSetup(Arc>>>); @@ -688,14 +607,8 @@ impl MountedSetup { } } -/// One journal, and what the clock drives it as. -/// -/// Two live states and one transition between them: a setup is driven -/// until it hands back the channel it mounted, and from then on the -/// channel is what is driven. Nothing here re-derives a mount — -/// [`SetupAdvance::mounted`] is the only way a [`ChannelStore`] reaches -/// this file, and the setup is not driven again afterwards, because a -/// second step would open a second journal on the same file. +/// A journal transitions from setup to channel when `advance_setup` returns its +/// mounted store. The store is transferred without reopening its exclusive file. enum Driven { /// The journal is driven behind the setup service that answers for /// it. The policy is retained beside the service: it is the provider @@ -704,16 +617,7 @@ enum Driven { Setup { /// The endpoint this journal is both driven and served behind. service: SetupService, - /// The retained provider authority, behind a pointer. - /// - /// Boxed because it is the widest thing this enum carries by a - /// long way — every other payload here is a handle or a store - /// pointer, one or two words each — and a journal is one value - /// with three shapes, so the two that hold no policy would - /// otherwise each be as large as the one that does. - /// [`PaymentAdmission`] already holds it behind the same - /// indirection, and this is built from that one, once per - /// journal at startup. + /// Boxed to keep the other enum variants small. policy: Box, }, /// The channel this setup mounted, including the recovery authority @@ -724,12 +628,8 @@ enum Driven { Done, } -/// One mounted channel as driven by the paid-work clock. -/// -/// Recovery lives here rather than in the served route: an accepted job is an -/// obligation recorded by this journal even if peer routing changes while the -/// process is down. `accepting` is also lent to the route when one is mounted, -/// so live acceptance and restart recovery serialize their readiness checks. +/// A driven channel recovers accepted jobs even without a peer route. +/// Live acceptance and recovery share the same admission lock. struct DrivenChannel { service: WorkService, descriptor: Option, @@ -748,12 +648,8 @@ impl DrivenChannel { .context("the driven channel state is unavailable") } - /// Starts a journaled Accepted job after proving current readiness. - /// - /// No in-memory `attempted` marker is needed. A racing live request or - /// clock tick reaches the same endpoint; its durable `JobRunning` record - /// lets exactly one caller receive `Invoke` and every other caller receive - /// `Running`. + /// Resumes accepted work after a fresh readiness check. The durable `JobRunning` + /// record admits one invocation even when a live request races this clock tick. async fn resume_accepted(&self, source: &S) -> anyhow::Result where S: FinalizedBlocks + FinalizedWorkView + Sync, @@ -789,13 +685,7 @@ struct SetupClock { } impl SetupClock { - /// Takes this journal's one step, and says whether the chain - /// answered. - /// - /// A source failure is the only outcome the caller acts on: a - /// validator that stopped answering is dialled again rather than - /// asked forever. Everything else is this journal's own business and - /// is logged where it happens. + /// Advances the journal; returns false on a source failure so the caller redials. async fn tick( &mut self, source: &S, @@ -866,12 +756,7 @@ impl SetupClock { answered } - /// Mounts the store the driver handed back. - /// - /// Handed back, never reopened: the journal is exclusive, so a - /// second `ChannelStore::open` on the same file is a refusal rather - /// than a second view, and the settlement and origin this one - /// carries are the ones the completing read established. + /// Transfers the mounted store returned by setup, preserving its exclusive lock. fn take_mount( &mut self, store: ChannelStore, @@ -881,10 +766,7 @@ impl SetupClock { mount: &MountedWork, ) { let bond = hex::encode(self.bond_edge.to_bytes()); - // The setup's retained policy supplies the provider-controlled - // fields, while the mounted channel supplies the payment edge and - // complete terms the two parties actually signed. This is a full - // descriptor reconstruction, not a mount-time readiness cache. + // Rebuild the descriptor from the retained policy and signed payment terms. let descriptor = { let channel = store.state().channel(); match policy.admit(channel.payment_edge(), channel.payment_terms().clone()) { @@ -965,22 +847,9 @@ impl WorkRunner where S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, { - /// Opens every setup journal under the configured root. - /// - /// The root and the network are the whole of what a restarting node - /// is told; the bond each journal is about and the role it was - /// written at come out of the files, which is what `discover_setups` - /// is for. A journal that cannot be named is reported and not - /// skipped silently: a file this node cannot open may be a channel - /// it still owes a close. - /// - /// # Errors - /// - /// When the root itself cannot be enumerated. - /// Every owned journal is driven. A configured route additionally - /// mounts its exact setup service under the authenticated peer that - /// names it; an unconfigured journal remains a close duty, not a - /// fallback answer. + /// Discovers provider journals and mounts configured routes. Unrouted journals + /// are still driven through close. Unreadable journals are logged; failure to + /// enumerate the root is returned to the caller. pub fn discover( config: WorkRunnerConfig, work_mount: MountedWork, @@ -1086,11 +955,7 @@ where answered } - /// The loop, over whatever chain `dial` produces. - /// - /// One tick of every journal per period, and a chain that stopped - /// answering is dialled again rather than asked forever. The whole - /// of the cadence is here, and none of the decisions are. + /// Ticks each journal once per period and redials after a source failure. pub async fn run_over(mut self, mut stop: oneshot::Receiver<()>, dial: D) where D: Fn() -> F, @@ -1137,14 +1002,8 @@ impl WorkRunner { } } -/// Rotate the first candidate on reconnect, including when a connected peer -/// cannot supply historical finalized blocks. Reads and submissions use the -/// selected verified connection. -/// -/// One endpoint for both directions. §1's concurrent fan-out to all six -/// is a submission strategy with an outcome rule, and neither exists in -/// this tree yet; inventing one here would be the runner deciding what -/// a submission means. +/// Rotates the first candidate on reconnect. Chain reads and submissions use +/// the selected verified connection. async fn connect_chain( validators: &[String], verifier: ConsensusVerifier, diff --git a/crates/sdk/src/work_config.rs b/crates/sdk/src/work_config.rs index 2f3318a4..0a78e04e 100644 --- a/crates/sdk/src/work_config.rs +++ b/crates/sdk/src/work_config.rs @@ -1,52 +1,15 @@ -//! The paid-work configuration file: what an operator writes down, and -//! what a node refuses to start without. +//! Paid-work configuration: chain identity, routes, execution policy and funding. //! -//! `--work-config` was a path whose *presence* advertised two ALPNs and -//! whose contents were never opened. A node cannot mount a channel from -//! a path, so this is the schema and the loader for what is in it: the -//! three-part chain cross-check, the six validator URLs a write is -//! fanned to, the journal root, the bilateral route table, the two -//! policies this provider works under, the watcher's poll cadence, the -//! funding it expects a payment edge to carry, and the shortest response -//! window it will sign terms over. -//! -//! # The cross-check is not an anchor -//! -//! `(network_id, genesis_payload_digest, threshold_identity)` is a -//! fail-fast configuration cross-check. Only the threshold identity ever -//! authenticates a finalized block; the other two are how a node started -//! against the wrong chain says so at startup instead of at the first -//! settlement. The identity is decoded here, by the same constructor -//! consensus verification uses, so a typo is a startup error and not a -//! block that never verifies. -//! -//! # What is deliberately not here -//! -//! There is no Start-span field, no mutual-margin field, and no journal -//! cap. All were deleted: the Start span is fixed at 64, a work-payment -//! edge has no Mutual route, and the journal's active and checkpoint -//! ceilings are constants it enforces on itself -//! ([`MAX_ACTIVE_JOURNAL_BYTES`]), so any of them appearing in a file is -//! an operator configuring something that does not exist. Every struct -//! below denies unknown fields, which is what turns that into an error -//! naming the field. -//! -//! Nor is there a measured artifact. An earlier design derived the -//! response window and an alarm margin from latencies a bootstrap probe -//! recorded, pinned to the digest of the measuring binary; no deployed -//! node ever carried one, and every rebuild would have invalidated it. -//! The two numbers that design would have produced are written down -//! here instead, by the operator, and [`WorkConfig::provider_policy`] is -//! the whole of what is made from them. -//! -//! [`MAX_ACTIVE_JOURNAL_BYTES`]: hellas_work::work_store::journal::MAX_ACTIVE_JOURNAL_BYTES +//! The threshold identity authenticates finalized blocks; the network and genesis +//! digest detect configuration mismatches. Unknown fields are rejected. Call +//! `validate_work_routes` before serving to check the configuration against journals. use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::path::{Path, PathBuf}; use std::time::Duration; -use anyhow::{Context as _, bail}; +use anyhow::{Context as _, Result, bail}; use hellas_kernel::{ EdgeId, EdgeValues, Fees, Key, MIN_OMIT_RESPONSE_BLOCKS, NetworkId, Secp256k1Verifier, }; @@ -64,36 +27,16 @@ use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; use hellas_work::work_store::{Role, SetupStore, discover_setups}; use serde::Deserialize; -type CliResult = anyhow::Result; - -/// How many validator RPCs a write names. -/// -/// Reads come from a follower and writes are fanned to all six; a -/// configuration naming five has one validator whose acceptance this -/// node can never win, and one naming seven names something this -/// deployment does not have. +/// Number of distinct validator RPC URLs required by this deployment. pub const VALIDATOR_COUNT: usize = 6; -/// One operator's complete paid-work configuration, loaded and structurally -/// checked. -/// -/// A plain record with public fields, for [`WorkChannelConfig`]'s -/// reason: this is the shape a file fills in. Every file-local gate has -/// already run in [`load_work_config`]; the serve path then runs -/// [`validate_work_routes`] against the journals that must exist when it -/// starts. Provisioning shares the file loader before it creates one, which is -/// why disk agreement is not pretended to be a parse-time fact. -/// -/// [`WorkChannelConfig`]: hellas_rpc::protocol::work_setup::WorkChannelConfig +/// Parsed paid-work configuration. `load_work_config` checks its fields; +/// `validate_work_routes` checks agreement with provider journals at startup. #[derive(Clone, Debug)] -#[allow( - dead_code, - reason = "the fields a mount consumes are read by the node runner; loading and checking them is this half" -)] pub struct WorkConfig { /// The chain this node believes it is configured against. pub chain: ChainCrossCheck, - /// The six validator RPC URLs every write is fanned to. + /// Validator RPC URLs used for chain reads and transaction submission. pub validators: Vec, /// Directory holding the setup and channel journals. pub journal_root: PathBuf, @@ -114,12 +57,8 @@ pub struct WorkConfig { pub min_omit_response_blocks: u64, } -/// One bilateral setup route written in the paid-work configuration. -/// -/// The bond names the provider setup journal under [`WorkConfig::journal_root`]. -/// The client key is repeated here deliberately: startup compares it with the -/// taker committed inside that journal, turning a stale or mistyped route into -/// a refusal before the node binds. +/// Maps an authenticated peer to a bond and its client settlement key. +/// Startup checks the client key against the journal before mounting the route. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct WorkRoute { /// The transport-authenticated peer allowed to reach this bond. @@ -130,11 +69,7 @@ pub struct WorkRoute { pub client: Key, } -/// Paid-work routes keyed by their authenticated peer. -/// -/// Construction is private to the checked file loader. In particular, there -/// is no insertion API through which a caller could recreate last-one-wins -/// handling after duplicate peers and bonds have been refused. +/// Routes indexed by authenticated peer, with duplicate peers and bonds rejected. #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct WorkRoutes { by_peer: BTreeMap, @@ -158,7 +93,7 @@ impl WorkRoutes { self.by_peer.is_empty() } - fn from_files(files: Vec) -> CliResult { + fn from_files(files: Vec) -> Result { let mut by_peer = BTreeMap::new(); let mut bonds = BTreeSet::new(); for file in files { @@ -181,12 +116,7 @@ impl WorkRoutes { } impl WorkConfig { - /// The provider policy this configuration makes. - /// - /// Every field is the operator's: the four the policy commits to, - /// the funding it expects, and the window it insists on. Nothing is - /// measured and nothing is inferred, so a node with a configuration - /// has a policy and countersigns over it. + /// Builds the policy used by both provisioning and channel admission. #[must_use] pub fn provider_policy(&self) -> ProviderChannelPolicy { ProviderChannelPolicy { @@ -202,10 +132,6 @@ impl WorkConfig { /// The three fields that say which chain this is. #[derive(Clone, Debug, PartialEq, Eq)] -#[allow( - dead_code, - reason = "the fields a mount consumes are read by the node runner; loading and checking them is this half" -)] pub struct ChainCrossCheck { /// The network every signature on this node's channels is bound to. pub network: NetworkId, @@ -215,24 +141,9 @@ pub struct ChainCrossCheck { pub threshold_identity: Vec, } -/// Loads and checks one paid-work configuration file. -/// -/// Every failure is a startup failure naming the field that failed, for -/// the reason §4 gives: a node that started with an unreadable -/// configuration would be one whose first symptom is an unsettleable -/// channel. -/// -/// # Errors -/// -/// The read and the parse, and then: a network id that is not one, a -/// digest that is not thirty-two bytes, a threshold identity consensus -/// cannot decode, a validator list that is not exactly -/// [`VALIDATOR_COUNT`] URLs with distinct normalised forms, an execution -/// policy the protocol's own [`check_execution_policy`] rejects, an -/// empty journal root, a route field of the wrong width, duplicate peers or -/// bonds in the route table, a zero poll cadence, and a response window -/// under the kernel's own minimum. -pub fn load_work_config(path: &Path) -> CliResult { +/// Loads configuration, checks chain identity and policy bounds, and normalizes +/// validator URLs. Route-to-journal validation is deferred until serve startup. +pub fn load_work_config(path: &Path) -> Result { let bytes = fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; let file: WorkConfigFile = serde_json::from_slice(&bytes) .with_context(|| format!("failed to parse {}", path.display()))?; @@ -240,21 +151,9 @@ pub fn load_work_config(path: &Path) -> CliResult { .with_context(|| format!("invalid work config {}", path.display())) } -/// Verifies that every configured route names this root's provider journal -/// and the client settlement key committed by its bond terms. -/// -/// This is a serve-startup check rather than part of [`load_work_config`]: -/// provisioning uses the same configuration loader before it creates a -/// journal, while a serving node must already have every journal it promises. -/// Discovery comes first so [`SetupStore::open`] is never allowed to create a -/// missing journal merely because a route named its bond. -/// -/// # Errors -/// -/// The root cannot be enumerated, a route's provider journal is absent from -/// that root or cannot be opened, the journal holds no bond proposal, or its -/// bond names a taker other than the route's configured client. -pub fn validate_work_routes(config: &WorkConfig) -> CliResult<()> { +/// Checks each route against a provider journal under the configured root, +/// including its bond and client key. Run after provisioning and before serving. +pub fn validate_work_routes(config: &WorkConfig) -> Result<()> { if config.routes.is_empty() { return Ok(()); } @@ -331,7 +230,7 @@ struct WorkConfigFile { } impl WorkConfigFile { - fn into_config(self) -> CliResult { + fn into_config(self) -> Result { let Some(network) = NetworkId::new(self.chain.network_id.trim()) else { bail!( "chain.network_id {:?} is not a network id", @@ -391,11 +290,7 @@ impl WorkConfigFile { } } -/// One bilateral route exactly as the operator writes it. -/// -/// All three values are fixed-width lowercase-or-uppercase hexadecimal on -/// input and canonical byte values after loading. A peer or bond written in a -/// second spelling is therefore still the same key for duplicate detection. +/// Hex-encoded route fields, parsed before duplicate detection. #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct WorkRouteFile { @@ -404,22 +299,8 @@ struct WorkRouteFile { client: String, } -/// Parses the six validator RPC URLs, and refuses anything that is not -/// one. -/// -/// Both halves matter. A string that is not a URL is not an address this -/// node can ever fan a write to, and "trimmed and non-empty" admits `not -/// a URL` verbatim — a configuration whose first symptom would be five -/// validators answering and one that never does. And uniqueness is a -/// question about *addresses*, not about spellings: `HTTP://Host:443/` -/// and `http://host:443/` are one validator written twice, and a fan-out -/// to five validators is not six however it is spelled. So the -/// comparison is between parsed, normalised URLs, and the normalised -/// forms are what is kept. -/// -/// A host is required, because these are dialled: a `mailto:` or a -/// `data:` parses perfectly well and is not a validator. -fn parse_validators(entries: Vec) -> CliResult> { +/// Normalizes validator URLs and requires distinct addresses with hosts. +fn parse_validators(entries: Vec) -> Result> { let mut validators: Vec = Vec::with_capacity(VALIDATOR_COUNT); for entry in entries { let entry = entry.trim(); @@ -454,12 +335,6 @@ struct ChainFile { threshold_identity: String, } -/// Where the work journals live. -/// -/// A root and nothing else. How large a journal may grow is not an -/// operator's to say: the active and checkpoint ceilings are constants -/// the journal enforces on itself, so a cap here would be a number -/// written down and ignored. #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct JournalFile { @@ -467,7 +342,7 @@ struct JournalFile { } impl JournalFile { - fn into_root(self) -> CliResult { + fn into_root(self) -> Result { if self.root.as_os_str().is_empty() { bail!("journal.root must be a path"); } @@ -485,7 +360,7 @@ struct PoliciesFile { } impl PoliciesFile { - fn into_policies(self) -> CliResult<([u8; 32], PaidChannelPolicyV1, PaidWorkPolicy)> { + fn into_policies(self) -> Result<([u8; 32], PaidChannelPolicyV1, PaidWorkPolicy)> { let salt = parse_fixed_hex("policies.policy_salt", &self.policy_salt)?; Ok(( salt, @@ -509,11 +384,7 @@ struct ChannelPolicyFile { delivery_credit_limit: u64, } -/// The execution policy, field for field. -/// -/// Spelled out rather than flattened from some smaller shape because -/// every one of these is a value both parties sign: a default here would -/// be this node quietly proposing a policy its operator never wrote. +/// Required execution-policy fields. Defaults could change the terms being signed. #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct ExecutionPolicyFile { @@ -533,7 +404,7 @@ struct ExecutionPolicyFile { } impl ExecutionPolicyFile { - fn into_policy(self) -> CliResult { + fn into_policy(self) -> Result { let allowed_environment: ContentId = self.allowed_environment.parse().with_context(|| { format!( @@ -562,12 +433,7 @@ impl ExecutionPolicyFile { oracle_grace_blocks: self.oracle_grace_blocks, fixed_price: self.fixed_price, }; - // The protocol's own gate, run here rather than at the first - // admission. A zero here is not a small bound, it is an absent - // one — a zero margin gives a deadline no time to be met in, and - // a zero price is a job nobody is paid for. Copying the fields - // through unchecked moves that discovery to the moment a - // counterparty is already waiting on a proposal. + // Validate with the protocol rules before any channel is proposed. check_execution_policy(&policy) .map_err(|error| anyhow::anyhow!("policies.execution is not usable: {error}"))?; Ok(policy) @@ -603,7 +469,7 @@ struct OpenFetchPolicyFile { } impl FetchPolicyFile { - fn into_policy(self) -> CliResult { + fn into_policy(self) -> Result { let route = match (self.service, self.method, self.open_fetch) { (Some(service), Some(method), None) => { PaidFetchRoutePolicy::sealed_route(service, method)? @@ -636,13 +502,7 @@ impl FetchPolicyFile { } } -/// The payment edge's funding, field for field, as this provider requires -/// a client to fund it. -/// -/// Spelled out for [`ExecutionPolicyFile`]'s reason: every one of these -/// bounds what a certificate on the channel may name, so a default here -/// would be this node quietly accepting funding its operator never -/// priced. +/// Required funding values and close fees. #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct PaymentValuesFile { @@ -675,7 +535,7 @@ impl PaymentValuesFile { } } -fn parse_hex(field: &str, raw: &str) -> CliResult> { +fn parse_hex(field: &str, raw: &str) -> Result> { let bytes = hex::decode(raw.trim()).with_context(|| format!("{field} is not hexadecimal"))?; if bytes.is_empty() { bail!("{field} must not be empty"); @@ -683,7 +543,7 @@ fn parse_hex(field: &str, raw: &str) -> CliResult> { Ok(bytes) } -fn parse_fixed_hex(field: &str, raw: &str) -> CliResult<[u8; N]> { +fn parse_fixed_hex(field: &str, raw: &str) -> Result<[u8; N]> { let bytes = parse_hex(field, raw)?; let Ok(bytes) = <[u8; N]>::try_from(bytes.as_slice()) else { bail!("{field} must be {N} bytes, found {}", bytes.len()); @@ -691,6 +551,6 @@ fn parse_fixed_hex(field: &str, raw: &str) -> CliResult<[u8; N]> Ok(bytes) } -fn parse_digest(field: &str, raw: &str) -> CliResult { +fn parse_digest(field: &str, raw: &str) -> Result { Ok(Digest::from_bytes(parse_fixed_hex(field, raw)?)) } diff --git a/crates/sdk/src/work_provision.rs b/crates/sdk/src/work_provision.rs index dc1d22ee..a845f745 100644 --- a/crates/sdk/src/work_provision.rs +++ b/crates/sdk/src/work_provision.rs @@ -1,72 +1,15 @@ -//! Making the offers a fresh provider has nothing to serve without. +//! Provider bond provisioning. //! -//! `WorkRunner::discover` answers `WorkSetup` from the setup journals it -//! finds under the configured work root, and finding is the whole of what -//! it does. A correctly configured provider with no journal therefore -//! refuses every client that dials it, and the paid path is unreachable -//! from a clean install. This is the operator's step that writes them. -//! -//! # The order is the journal's, and none of its rules are here -//! -//! Three library calls. [`SetupStore`] is opened as the provider's half -//! of one bond, the immutable history floor is armed, and -//! [`SetupEndpoint::propose_bond`] signs the stake and journals it before -//! there is anything to export. -//! -//! Arming is first because it has to be: the setup state refuses -//! "recording revision 1 before arming its scan floor", so the floor is -//! not a preparation this command chose to do early but the step every -//! later one is refused before. The floor is a finalized height and the -//! payload digest at it, and the setup's own history must name that -//! digest as the parent of its first block. So it is read from a -//! validator rather than written down by an operator: a floor naming a -//! block this chain does not have is a setup whose history can never be -//! contiguous, and nothing later would say so out loud. -//! -//! # Exit means durable -//! -//! [`SetupStore::commit`] fsyncs a revision before it returns, and this -//! command still reopens the journal and replays it before printing -//! anything. That reopen is the one `WorkRunner::discover` will do, run -//! early: an operator told the offer exists has been told about the disk, -//! and about a file whose exclusive lock is already free for the runner -//! to take. -//! -//! # One recourse backs one route -//! -//! A provider offer reserves a route, a bond, and every coin funding that -//! bond. A second offer is safe only when all three are disjoint from every -//! provider offer already under the root. Existing peers come from the -//! durable route table, while existing coins come from the bond funding in -//! each retained setup bundle. Revision one is enough: it holds the funding -//! before a client has answered, while [`SetupState::funding_coins`] is still -//! empty because there is no executable Open yet. -//! -//! Discovery, route agreement, and funding comparison all happen while the -//! candidate is only a value. The candidate journal is not opened until -//! afterwards, so every collision is refused before a floor is written or a -//! bond signature is made. -//! -//! # What the operator chooses, and what is built -//! -//! Every number in the bond is the operator's and this command invents -//! none of them. Two parts of the shape are not choices: a stake bond is -//! funded by its maker alone, so the taker's side of the funding is -//! empty, and its timeout pays the staking party and nobody else, so -//! there is one payout and it names the provider's own key. A second -//! payout to that same party would only raise the close cost the payout -//! has to clear. The kernel checks the rest when the Open reaches it — -//! that the payout total is the edge's close value, that the price cap -//! covers a job, that the timeout is ahead of the block including it — -//! and re-spelling any of that here would be a second answer to a -//! question consensus already answers. -//! -//! [`SetupState::funding_coins`]: hellas_work::work_store::SetupState::funding_coins +//! Preview derives the bond without reading the chain or writing a journal. +//! Provisioning checks that the route, bond and staked coins are unreserved, then +//! arms a finalized history floor and journals the signed offer. It reopens the +//! journal to verify durability before returning. Unanswered offers reserve their +//! staked coins as soon as revision one is signed. use std::collections::BTreeSet; use std::path::{Path, PathBuf}; -use anyhow::{Context as _, bail}; +use anyhow::{Context as _, Result, bail}; use hellas_chain::client::VerifiedRemoteLightClient; use hellas_chain::domain::MAX_EDGE_LIFETIME_BLOCKS; use hellas_chain::{ConsensusInfo, ConsensusVerifier, WorkBlocks}; @@ -81,14 +24,10 @@ use hellas_work::work_store::{Role, SetupScan, SetupStore, discover_setups}; use tracing::{info, warn}; use crate::work_config::{WorkConfig, WorkRoute}; -type CliResult = anyhow::Result; /// What an operator asks for when they make one offer. pub struct ProvisionOptions { - /// The loaded paid-work configuration, not the path it came from. It - /// carries the network the bond is bound to, the root the journal is - /// written under, the validators the floor is read from, and the - /// artifact the provider's policy rests on. + /// Chain, journal, route and execution-policy configuration. pub work_config: WorkConfig, /// The key this provider stakes and signs the bond with, read from /// the identity the operator already has and never made here. @@ -104,57 +43,17 @@ pub struct ProvisionOptions { pub timeout_payout: u64, /// The largest job price this bond covers. pub max_job_price: u64, - /// Print the deterministic bond edge and stop before any external read or write. + /// Select bond preview in operator frontends. pub print_bond_only: bool, } -/// Makes one offer, and says where it is. -/// -/// # Errors -/// -/// A configuration with no matching bilateral route, a route, bond, or funding coin already reserved by another offer, -/// a key or coin id that is not one, no configured validator with a finalized -/// block to read a floor from, and whatever the setup journal says about the -/// revision it refused or could not make durable. -pub async fn run_provision(options: ProvisionOptions) -> CliResult<()> { - // The candidate is the one source of the bond edge for both preview and - // provisioning. Keep this before evidence, routing, validators and the - // journal: the preview exists so an operator can put this value into the - // route table those later steps require. - let candidate = BondCandidate::plan(&options)?; - if options.print_bond_only { - println!("bond_edge: {}", hex::encode(candidate.bond_edge.to_bytes())); - return Ok(()); - } - let offer = Offer::plan(&options, options.work_config.provider_policy(), candidate)?; - // Dialled after every refusal that can be made without a chain, and - // before the journal exists: a floor is the first thing written into - // it, so a run that cannot read one leaves no half-made offer behind. - let made = offer.journal(finalized_floor(&options.work_config).await?)?; - - println!( - "offer journaled: bond {} under {}", - hex::encode(made.bond_edge.to_bytes()), - options.work_config.journal_root.display(), - ); - // The floor read back out of the journal rather than the one just - // dialled, because those differ on a retry and the durable one is the - // one this setup's history will be measured against. - println!( - "history floor: finalized height {} with payload {}", - made.floor.height, - hex::encode(made.floor.payload), - ); - Ok(()) -} - /// Compute the bond before the operator adds its bilateral route. -pub fn preview_bond(options: &ProvisionOptions) -> CliResult { +pub fn preview_bond(options: &ProvisionOptions) -> Result { Ok(BondCandidate::plan(options)?.bond_edge) } /// Sign and journal an offer under an existing provider identity. -pub async fn provision_offer(options: ProvisionOptions) -> CliResult { +pub async fn provision_offer(options: ProvisionOptions) -> Result { let candidate = BondCandidate::plan(&options)?; let offer = Offer::plan(&options, options.work_config.provider_policy(), candidate)?; offer.journal(finalized_floor(&options.work_config).await?) @@ -170,12 +69,7 @@ pub struct Provisioned { pub floor: SetupScan, } -/// The deterministic bond inputs, built without evidence, routing, a chain, -/// or a journal. -/// -/// Preview and real provisioning both pass through this value. In particular, -/// the real path does not recompute the edge after printing it, so a preview -/// cannot drift from the offer later signed. +/// Deterministic inputs shared by bond preview and provisioning. struct BondCandidate { network: NetworkId, journal_root: PathBuf, @@ -186,7 +80,7 @@ struct BondCandidate { } impl BondCandidate { - fn plan(options: &ProvisionOptions) -> CliResult { + fn plan(options: &ProvisionOptions) -> Result { let network = options.work_config.chain.network; let journal_root = options.work_config.journal_root.clone(); // Maker is the provider and taker is the client, which is what @@ -223,13 +117,8 @@ impl BondCandidate { /// One offer, decided before anything is dialled or written. struct Offer { - network: NetworkId, - journal_root: PathBuf, - bond_edge: EdgeId, - bond_funding: Funding, - bond_terms: WorkStakeBondTerms, + candidate: BondCandidate, admission: PaymentAdmission, - settlement_key: Secp256k1Signer, } impl Offer { @@ -239,33 +128,23 @@ impl Offer { options: &ProvisionOptions, policy: ProviderChannelPolicy, candidate: BondCandidate, - ) -> CliResult { - let admission = PaymentAdmission::Admits(Box::new(policy)); - let BondCandidate { - network, - journal_root, - bond_edge, - bond_funding, - bond_terms, - settlement_key, - } = candidate; - let route = route_for_candidate(&options.work_config, bond_edge, &bond_terms)?; - refuse_offer_collisions(&options.work_config, route, &bond_funding)?; + ) -> Result { + let route = route_for_candidate( + &options.work_config, + candidate.bond_edge, + &candidate.bond_terms, + )?; + refuse_offer_collisions(&options.work_config, route, &candidate.bond_funding)?; Ok(Self { - network, - journal_root, - bond_edge, - bond_funding, - bond_terms, - admission, - settlement_key, + candidate, + admission: PaymentAdmission::Admits(Box::new(policy)), }) } /// Journals revision 1, and returns only once a fresh open of the /// journal replays it. - fn journal(self, floor: SetupScan) -> CliResult { - let timeout = self.bond_terms.timeout.get(); + fn journal(self, floor: SetupScan) -> Result { + let timeout = self.candidate.bond_terms.timeout.get(); anyhow::ensure!( timeout > floor.height, "bond timeout must be after finalized height {}", @@ -276,20 +155,21 @@ impl Offer { "bond timeout exceeds the chain maximum lifetime" ); let Self { + candidate, + admission, + } = self; + let BondCandidate { network, journal_root, bond_edge, bond_funding, bond_terms, - admission, settlement_key, - } = self; + } = candidate; { let store = open_provider_journal(&journal_root, network, bond_edge)?; let mut endpoint = SetupEndpoint::new(store, settlement_key, admission); - // The floor is immutable and the store writes exactly one arm - // of it, so a run that arms and then fails keeps the height - // its successor starts from rather than moving it. + // Preserve the journal's immutable history floor on retry. if let Some(held) = endpoint.state().scan_armed() { info!( height = held.height, @@ -305,10 +185,7 @@ impl Offer { .context("failed to sign and journal the bond proposal")?; } - // The journal is closed above, so this is a second process's view - // of it: the same replay and the same signature checks the runner - // runs, before an operator is told there is anything to run them - // on. + // Reopen with the same replay and signature checks used at startup. let reopened = open_provider_journal(&journal_root, network, bond_edge)?; let state = reopened.state(); let (Some(1), Some(floor)) = (state.revision(), state.scan_armed()) else { @@ -324,11 +201,7 @@ impl Offer { } } -fn open_provider_journal( - root: &Path, - network: NetworkId, - bond_edge: EdgeId, -) -> CliResult { +fn open_provider_journal(root: &Path, network: NetworkId, bond_edge: EdgeId) -> Result { SetupStore::open( root, network, @@ -345,17 +218,12 @@ fn open_provider_journal( }) } -/// Returns the configured bilateral route the candidate would occupy. -/// -/// The bond is derived from the exact funding and terms first. Matching by -/// that canonical value means a route cannot be selected by insertion order, -/// and checking the client here refuses a journal the next startup would -/// reject before the provider signs it. +/// Finds the candidate bond's route and checks its client before signing. fn route_for_candidate<'config>( config: &'config WorkConfig, bond_edge: EdgeId, bond_terms: &WorkStakeBondTerms, -) -> CliResult<&'config WorkRoute> { +) -> Result<&'config WorkRoute> { let Some(route) = config.routes.iter().find(|route| route.bond == bond_edge) else { bail!( "bond {} has no bilateral route in this work configuration; an offer is signed only \ @@ -376,17 +244,13 @@ fn route_for_candidate<'config>( Ok(route) } -/// Refuses every collision before the candidate journal is opened. -/// -/// An existing bond is named by discovery, its peer is named by the durable -/// route table, and its funding is named by the retained bundle. Failure to -/// recover any one of those facts is a refusal: absence of evidence is not -/// evidence that the candidate is disjoint. +/// Checks route, bond and coin reservations before opening the candidate journal. +/// Unidentified or unreadable existing journals prevent provisioning. fn refuse_offer_collisions( config: &WorkConfig, candidate: &WorkRoute, candidate_funding: &Funding, -) -> CliResult<()> { +) -> Result<()> { let root = &config.journal_root; let network = config.chain.network; let found = discover_setups(root, network).with_context(|| { @@ -460,12 +324,7 @@ fn refuse_offer_collisions( hex::encode(held.bond_edge.to_bytes()), ); } - // The retained revision's own staked funding, not the executable - // Opens: the provider signed these coins when it made the offer, so - // they are promised from that moment, while `funding_coins` answers - // from Opens that do not exist until the client countersigns. Read - // from there, every offer no client has answered would look like it - // reserved nothing. + // Revision-one funding is already reserved, even before an executable Open exists. let reserved = funding_coins(bundle.bond_funding()); if let Some(coin) = candidate_coins.intersection(&reserved).next() { bail!( @@ -490,7 +349,7 @@ fn funding_coins(funding: &Funding) -> BTreeSet { /// Reads one finalized block from the first configured validator that /// answers, as the floor this setup's history starts above. -async fn finalized_floor(config: &WorkConfig) -> CliResult { +async fn finalized_floor(config: &WorkConfig) -> Result { let verifier = ConsensusVerifier::new(&ConsensusInfo { validators: config.validators.clone(), threshold_identity: config.chain.threshold_identity.clone(), @@ -517,13 +376,9 @@ async fn finalized_floor(config: &WorkConfig) -> CliResult { bail!("no configured validator answered with a finalized block to floor this offer at") } -/// Returns the finalized tip as a scan floor, or `None` before anything -/// is finalized. -/// -/// The height and the payload come from one block rather than from two -/// reads, because the setup's first history block must name that exact -/// payload as its parent. -async fn floor_of(blocks: &B) -> CliResult> +/// Returns the finalized tip and its payload, or `None` before the first block. +/// Both values must come from the same block to start a contiguous history. +async fn floor_of(blocks: &B) -> Result> where B: FinalizedBlocks + ?Sized, { @@ -540,7 +395,7 @@ where } /// Reads the coins one provider stakes. -fn staked(ids: &[String]) -> CliResult> { +fn staked(ids: &[String]) -> Result> { let mut slots = [CoinId::from_bytes([0; CoinId::LENGTH]); MAX_PARTY_INPUTS]; for (slot, id) in slots.iter_mut().zip(ids) { *slot = CoinId::from_bytes(fixed::<{ CoinId::LENGTH }>("--stake-coin", id)?); @@ -558,7 +413,7 @@ fn staked(ids: &[String]) -> CliResult> { } /// Reads exactly `N` bytes of hex, or says which flag was not that. -fn fixed(flag: &str, value: &str) -> CliResult<[u8; N]> { +fn fixed(flag: &str, value: &str) -> Result<[u8; N]> { let bytes = hex::decode(value).with_context(|| format!("{flag} {value:?} is not hex-encoded bytes"))?; let Ok(fixed) = <[u8; N]>::try_from(bytes.as_slice()) else { diff --git a/crates/sdk/src/work_provision/tests.rs b/crates/sdk/src/work_provision/tests.rs index c521f533..7d50a475 100644 --- a/crates/sdk/src/work_provision/tests.rs +++ b/crates/sdk/src/work_provision/tests.rs @@ -91,7 +91,7 @@ fn route(peer: u8, bond: EdgeId, client: Key) -> serde_json::Value { /// Loads routes through the production parser, so their duplicate-peer /// and duplicate-bond invariants are facts these provisioning tests use, /// not a test-only constructor that can make impossible route tables. -fn routed_work_config(root: &Path, routes: Vec) -> CliResult { +fn routed_work_config(root: &Path, routes: Vec) -> Result { let validators: Vec = (1..=6) .map(|index| format!("http://127.0.0.1:900{index}")) .collect(); @@ -174,14 +174,14 @@ fn provision( root: &Path, policy: ProviderChannelPolicy, max_job_price: u64, -) -> CliResult { +) -> Result { provision_options(&options(root, max_job_price), policy) } fn provision_options( options: &ProvisionOptions, policy: ProviderChannelPolicy, -) -> CliResult { +) -> Result { let candidate = BondCandidate::plan(options)?; Offer::plan(options, policy, candidate)?.journal(floor()) } @@ -236,19 +236,17 @@ fn preview_and_real_offer_use_the_identical_bond_candidate() { let offer = Offer::plan(&options, policy(), candidate) .unwrap_or_else(|error| panic!("the routed offer plans: {error:#}")); assert_eq!(expected, expected_bond(40)); - assert_eq!(offer.bond_edge, expected); + assert_eq!(offer.candidate.bond_edge, expected); } -#[tokio::test] -async fn preview_needs_neither_a_route_nor_a_chain_nor_a_journal() { +#[test] +fn preview_needs_neither_a_route_nor_a_chain_nor_a_journal() { let root = tempfile::tempdir().unwrap(); let config = routed_work_config(root.path(), Vec::new()) .unwrap_or_else(|error| panic!("a route-free config loads: {error:#}")); - let mut options = options_for(config, client().party_key(), &[0xa1], 40); - options.print_bond_only = true; + let options = options_for(config, client().party_key(), &[0xa1], 40); - run_provision(options) - .await + preview_bond(&options) .unwrap_or_else(|error| panic!("the isolated preview succeeds: {error:#}")); assert_eq!( provider_setups(root.path()), diff --git a/crates/work/src/work.rs b/crates/work/src/work.rs index a376ab30..01db3405 100644 --- a/crates/work/src/work.rs +++ b/crates/work/src/work.rs @@ -1770,9 +1770,6 @@ pub trait PaidWorkBackend: Sync { } } -/// Compatibility name for implementations that provide only Evaluate. -pub use PaidWorkBackend as PaidEvaluateBackend; - /// What [`ProviderEndpoint::begin_run`] found, and what may be done next. #[derive(Clone, Debug, PartialEq, Eq)] pub enum RunAdmission { From 30ff4d789501fb1182eccb06efbdc154ef99d92f Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Thu, 24 Sep 2026 07:38:42 +0200 Subject: [PATCH 04/32] Route HTTP API clients through Fetch with archives and telemetry --- Cargo.lock | 4 + README.md | 6 +- crates/cli/src/commands/fetch.rs | 4 +- crates/cli/src/identity.rs | 5 + crates/cli/src/main.rs | 23 +- crates/cli/src/tracing_config/otel.rs | 2 + crates/client/src/fetch/tests.rs | 14 +- .../src/executor/actor/execution/tests.rs | 18 +- crates/executor/src/fetch.rs | 4 +- crates/gateway/Cargo.toml | 2 + crates/gateway/src/archive.rs | 213 ++++++++ crates/gateway/src/archive/tests.rs | 150 ++++++ crates/gateway/src/http_fetch.rs | 460 ++++++++++++++++++ crates/gateway/src/http_fetch/observation.rs | 286 +++++++++++ crates/gateway/src/http_fetch/tests.rs | 112 +++++ crates/gateway/src/lib.rs | 21 +- crates/gateway/src/state/tests.rs | 9 + crates/gateway/src/wrap.rs | 12 +- crates/providers/Cargo.toml | 2 + crates/providers/HTTPS.md | 28 +- crates/providers/src/http.rs | 54 +- crates/providers/src/http/config.rs | 284 ++++++++++- crates/providers/src/http/tests.rs | 82 +++- crates/providers/src/lib.rs | 4 +- crates/providers/src/responses_fetch.rs | 2 +- .../src/responses_fetch/telemetry/noop.rs | 5 +- .../src/responses_fetch/telemetry/otel.rs | 9 +- crates/rpc/src/fetch.rs | 6 +- crates/rpc/src/fetch/tests.rs | 7 + crates/rpc/src/http_fetch.rs | 2 +- docs/http-gateway.md | 96 ++++ 31 files changed, 1851 insertions(+), 75 deletions(-) create mode 100644 crates/gateway/src/archive.rs create mode 100644 crates/gateway/src/archive/tests.rs create mode 100644 crates/gateway/src/http_fetch.rs create mode 100644 crates/gateway/src/http_fetch/observation.rs create mode 100644 crates/gateway/src/http_fetch/tests.rs create mode 100644 docs/http-gateway.md diff --git a/Cargo.lock b/Cargo.lock index 511f18b9..c115dabd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2855,6 +2855,7 @@ dependencies = [ "anyhow", "async-stream", "axum", + "base64 0.22.1", "futures", "hellas-adaptors", "hellas-client", @@ -2864,6 +2865,7 @@ dependencies = [ "hellas-rpc", "hellas-store", "hellas-wire", + "httpdate", "iroh", "libc", "opentelemetry", @@ -2944,6 +2946,7 @@ dependencies = [ "futures", "hellas-adaptors", "hellas-executor", + "hellas-private", "hellas-rpc", "hellas-wire", "opentelemetry", @@ -2954,6 +2957,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.0", + "tempfile", "tokio", "tokio-rustls", "tracing", diff --git a/README.md b/README.md index e37cf27d..b8476da0 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,11 @@ trusted app itself requires a platform-backed assurance. ## HTTP gateway -The gateway requires the same canonical causal-LM environment and an explicit +For upstream APIs, [HTTP Fetch routes](docs/http-gateway.md) preserve the vendor's +request, response and streaming formats. CLI gateways archive payloads by default; +`--zdr` or `x-hellas-zdr: true` disables application payload persistence. + +The causal-LM gateway requires the same canonical environment and an explicit presentation tokenizer. `--model` is only an API response label; when omitted, the manifest ID is used. diff --git a/crates/cli/src/commands/fetch.rs b/crates/cli/src/commands/fetch.rs index 6e53cc6e..dfa0dcb4 100644 --- a/crates/cli/src/commands/fetch.rs +++ b/crates/cli/src/commands/fetch.rs @@ -147,7 +147,9 @@ mod tests { std::fs::write(&path, vec![b' '; MAX_FETCH_REQUEST_BODY_BYTES + 1]).unwrap(); let error = load_payload_file(&path).expect_err("oversized payload must be refused"); - assert!(error.to_string().contains("over the 1048576-byte limit")); + assert!(error.to_string().contains(&format!( + "over the {MAX_FETCH_REQUEST_BODY_BYTES}-byte limit" + ))); } #[cfg(unix)] diff --git a/crates/cli/src/identity.rs b/crates/cli/src/identity.rs index e472a010..c7463022 100644 --- a/crates/cli/src/identity.rs +++ b/crates/cli/src/identity.rs @@ -290,6 +290,11 @@ fn default_path() -> anyhow::Result { default_hellas_path(IDENTITY_FILE, "--identity") } +#[cfg(feature = "gateway")] +pub(crate) fn default_gateway_archive_path() -> anyhow::Result { + default_hellas_path("gateway-archive", "--archive-dir") +} + #[cfg(feature = "node")] pub(crate) fn default_artifact_store_path() -> anyhow::Result { default_hellas_path(ARTIFACT_STORE_DIR, "--artifact-store-path") diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 68071132..56be9c4a 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -463,12 +463,21 @@ enum Commands { #[command( mut_arg("environment", |arg| arg .required(false) - .required_unless_present("responses_backend") + .required_unless_present_any(["responses_backend", "http_fetch_config"]) .required_if_eq("responses_backend", "hellas") .requires("tokenizer")), mut_arg("tokenizer", |arg| arg.required(false).requires("environment")) )] Gateway { + /// Serve exact HTTP routes through the generic HTTPS Fetch environment. + #[arg(long, value_name = "FILE", conflicts_with_all = ["responses_backend", "environment"])] + http_fetch_config: Option, + /// Request/response archive directory (default: ~/.hellas/gateway-archive). + #[arg(long, value_name = "DIRECTORY")] + archive_dir: Option, + /// Disable payload archives and require ZDR for every HTTP request. + #[arg(long)] + zdr: bool, /// Pay a pool of providers using durable on-chain funded work channels. #[cfg(feature = "node")] #[arg(long = "paid-work-config", value_name = "FILE")] @@ -1160,6 +1169,9 @@ async fn async_main() { Commands::OutputCache(..) => unreachable!("cache commands handled before identity load"), #[cfg(feature = "gateway")] Commands::Gateway { + http_fetch_config, + archive_dir, + zdr, #[cfg(feature = "node")] paid_work_config, bearer_token_file, @@ -1307,6 +1319,15 @@ async fn async_main() { )? }; hellas_gateway::run(hellas_gateway::GatewayOptions { + archive: hellas_gateway::ArchiveOptions { + directory: archive_dir.map(Ok).unwrap_or_else(identity::default_gateway_archive_path)?, + zdr, + }, + http_fetch: http_fetch_config + .map(|path| -> anyhow::Result<_> { + Ok(serde_json::from_slice(&std::fs::read(path)?)?) + }) + .transpose()?, output_cache: cache_options, paid_work, bearer_token_file, diff --git a/crates/cli/src/tracing_config/otel.rs b/crates/cli/src/tracing_config/otel.rs index 6ac1fccb..4b620f2e 100644 --- a/crates/cli/src/tracing_config/otel.rs +++ b/crates/cli/src/tracing_config/otel.rs @@ -89,6 +89,8 @@ where .with_tracer(tracer) .with_filter(tracing_subscriber::filter::filter_fn(|metadata| { metadata.is_span() + && (metadata.target() == "hellas_request" + || metadata.fields().field("otel.kind").is_some()) })); (Some(layer), Some(provider)) } diff --git a/crates/client/src/fetch/tests.rs b/crates/client/src/fetch/tests.rs index 0fe0e8c3..fe3e778e 100644 --- a/crates/client/src/fetch/tests.rs +++ b/crates/client/src/fetch/tests.rs @@ -250,7 +250,11 @@ fn verifier_bounds_retained_streamed_event_count() { .verify_chunk(builder.push_event(Vec::new()).unwrap()) .unwrap_err(); - assert!(error.to_string().contains("4095-chunk limit")); + assert!( + error + .to_string() + .contains(&format!("{}-chunk limit", MAX_FETCH_OUTPUT_EVENTS - 1)) + ); assert_eq!(verifier.events.len(), MAX_FETCH_OUTPUT_EVENTS - 1); } @@ -274,11 +278,9 @@ fn verifier_bounds_retained_streamed_payload_bytes() { .verify_chunk(builder.push_event(vec![1]).unwrap()) .unwrap_err(); - assert!( - error - .to_string() - .contains("2097152-byte signed payload limit") - ); + assert!(error.to_string().contains(&format!( + "{MAX_FETCH_OUTPUT_PAYLOAD_BYTES}-byte signed payload limit" + ))); assert_eq!(verifier.events.len(), 1); assert_eq!( verifier.next_position, diff --git a/crates/executor/src/executor/actor/execution/tests.rs b/crates/executor/src/executor/actor/execution/tests.rs index 7bd1733e..b70eb2e9 100644 --- a/crates/executor/src/executor/actor/execution/tests.rs +++ b/crates/executor/src/executor/actor/execution/tests.rs @@ -1369,7 +1369,11 @@ fn fetch_projection_budget_caps_event_count_and_payload_bytes() { }; event_budget.record_event(0).unwrap(); let event_error = event_budget.record_event(0).unwrap_err(); - assert!(event_error.to_string().contains("4095-event limit")); + assert!( + event_error + .to_string() + .contains(&format!("{}-event limit", MAX_FETCH_OUTPUT_EVENTS - 1)) + ); assert_eq!(event_budget.events, MAX_FETCH_OUTPUT_EVENTS - 1); event_budget.record_terminal(0).unwrap(); assert_eq!(event_budget.events, MAX_FETCH_OUTPUT_EVENTS); @@ -1380,11 +1384,9 @@ fn fetch_projection_budget_caps_event_count_and_payload_bytes() { }; payload_budget.record_event(1).unwrap(); let payload_error = payload_budget.record_event(1).unwrap_err(); - assert!( - payload_error - .to_string() - .contains("2097152-byte signed payload limit") - ); + assert!(payload_error.to_string().contains(&format!( + "{MAX_FETCH_OUTPUT_PAYLOAD_BYTES}-byte signed payload limit" + ))); assert_eq!( payload_budget.signed_payload_bytes, MAX_FETCH_OUTPUT_PAYLOAD_BYTES @@ -1425,7 +1427,9 @@ async fn projected_payload_limit_is_reported_as_work_failed() { let failed = run_failed(&handle, ticket, &signing_key).await; assert_eq!(failed.position, 0); - assert!(failed.error.contains("2097152-byte signed payload limit")); + assert!(failed.error.contains(&format!( + "{MAX_FETCH_OUTPUT_PAYLOAD_BYTES}-byte signed payload limit" + ))); } #[tokio::test] diff --git a/crates/executor/src/fetch.rs b/crates/executor/src/fetch.rs index e1e0b59e..4ee27534 100644 --- a/crates/executor/src/fetch.rs +++ b/crates/executor/src/fetch.rs @@ -25,9 +25,7 @@ pub(crate) const MAX_FETCH_IN_MEMORY_TICKETS: usize = MAX_OUTSTANDING_QUOTES; pub(crate) const MAX_FETCH_IN_MEMORY_INPUT_BYTES: usize = 32 * 1024 * 1024; const FETCH_CAPACITY_LOCK_WAIT: Duration = Duration::from_millis(100); const MAX_FETCH_CAPACITY_METADATA_BYTES: usize = 32; -/// Persisted DAG-CBOR contains at most eight input and 4,096 output envelopes. -/// One KiB of structural/signature headroom per envelope is deliberately -/// generous beside the protocol's bounded three MiB of signed payload. +/// Payload budgets plus one KiB of encoding/signature headroom per envelope. const MAX_FETCH_TRANSCRIPT_BYTES: usize = MAX_FETCH_REQUEST_BODY_BYTES + MAX_FETCH_OUTPUT_PAYLOAD_BYTES + (MAX_FETCH_OUTPUT_EVENTS + 8) * 1024; diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index 38bb51c4..c7769936 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -26,6 +26,8 @@ evaluate = [ anyhow = { workspace = true } async-stream = { workspace = true } axum = { workspace = true } +base64.workspace = true +httpdate = "1" futures = { workspace = true } hellas-adaptors.workspace = true hellas-client = { workspace = true, features = ["iroh", "evaluate"] } diff --git a/crates/gateway/src/archive.rs b/crates/gateway/src/archive.rs new file mode 100644 index 00000000..7c0c0e63 --- /dev/null +++ b/crates/gateway/src/archive.rs @@ -0,0 +1,213 @@ +use axum::{body::Bytes, http::HeaderMap}; +use futures::StreamExt; +use serde_json::json; +use std::{ + io, + path::{Path, PathBuf}, + time::{Instant, SystemTime, UNIX_EPOCH}, +}; +use tokio::io::AsyncWriteExt; +#[cfg(test)] +mod tests; + +#[derive(Clone)] +pub struct ArchiveOptions { + pub directory: PathBuf, + pub zdr: bool, +} + +#[derive(Clone)] +pub(crate) struct Policy { + pub options: ArchiveOptions, + pub cache_enabled: bool, +} + +pub(crate) fn zdr(headers: &HeaderMap, required: bool) -> Result { + let values: Vec<_> = headers.get_all("x-hellas-zdr").iter().collect(); + match values.as_slice() { + [] => Ok(required), + [value] if *value == "true" => Ok(true), + [value] if *value == "false" && !required => Ok(false), + _ => Err("invalid x-hellas-zdr header"), + } +} + +pub(crate) async fn record( + axum::extract::State(policy): axum::extract::State, + request: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + use axum::{body::Body, http::StatusCode, response::IntoResponse}; + let ephemeral = match zdr(request.headers(), policy.options.zdr) { + Ok(value) => value, + Err(message) => return (StatusCode::BAD_REQUEST, message).into_response(), + }; + if ephemeral && policy.cache_enabled { + return (StatusCode::BAD_REQUEST, "ZDR requires output-cache off").into_response(); + } + let (parts, body) = request.into_parts(); + let body = match axum::body::to_bytes(body, 2 * 1024 * 1024).await { + Ok(body) => body, + Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(), + }; + if ephemeral { + if serde_json::from_slice::(&body) + .ok() + .and_then(|value| value.get("store").cloned()) + == Some(json!(true)) + { + return (StatusCode::BAD_REQUEST, "ZDR forbids store=true").into_response(); + } + return next + .run(axum::extract::Request::from_parts(parts, Body::from(body))) + .await; + } + let mut archive = match Exchange::new( + &policy.options.directory, + parts.uri.path(), + parts.method.as_str(), + &body, + ) + .await + { + Ok(archive) => archive, + Err(_) => { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "request archive unavailable", + ) + .into_response(); + } + }; + let response = next + .run(axum::extract::Request::from_parts(parts, Body::from(body))) + .await; + if archive + .head(response.status().as_u16(), response.headers()) + .await + .is_err() + { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "response archive unavailable", + ) + .into_response(); + } + let (mut parts, body) = response.into_parts(); + if let Some(id) = archive.directory.file_name().and_then(|id| id.to_str()) { + parts + .headers + .insert("x-hellas-request-id", id.parse().unwrap()); + } + let mut source = body.into_data_stream(); + let stream: futures::stream::BoxStream<'static, Result> = + Box::pin(async_stream::try_stream! { + while let Some(bytes) = source.next().await { + let bytes = bytes.map_err(|_| io::Error::other("response stream failed"))?; + archive.chunk(&bytes).await?; + yield bytes; + } + archive.finish().await?; + }); + axum::response::Response::from_parts(parts, Body::from_stream(stream)) +} + +pub(super) fn prepare(path: &Path) -> io::Result<()> { + hellas_private::create_dir_all_durable(path)?; + hellas_private::restrict_directory(path) +} + +pub(super) struct Exchange { + directory: PathBuf, + metadata: serde_json::Value, + response: tokio::fs::File, + started: Instant, + bytes: usize, +} + +impl Exchange { + pub(super) async fn new( + root: &Path, + path: &str, + method: &str, + body: &Bytes, + ) -> io::Result { + let root = root.to_owned(); + let body = body.clone(); + let mut metadata = json!({ + "version": 1, "path": path, "method": method, "complete": false, + "started_unix_ms": SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64, + "request_bytes": body.len(), + }); + #[cfg(feature = "otel")] + { + let mut context = hellas_wire::metadata::Metadata::new(); + hellas_rpc::telemetry::inject(&tracing::Span::current(), &mut context); + metadata["traceparent"] = + json!(context.get("traceparent").and_then(|value| value.as_text())); + } + let initial = metadata.clone(); + let (directory, response) = tokio::task::spawn_blocking(move || { + let directory = root.join(format!("{:032x}", rand::random::())); + prepare(&directory)?; + hellas_private::write_atomically(&directory.join("request.bin"), ".tmp", &body)?; + write_metadata(&directory, &initial)?; + let temporary = hellas_private::private_tempfile(&directory, ".response-", ".tmp")?; + let file = temporary + .persist(directory.join("response.bin")) + .map_err(|error| error.error)?; + Ok::<_, io::Error>((directory, file)) + }) + .await + .map_err(io::Error::other)??; + metadata["response_bytes"] = json!(0); + Ok(Self { + directory, + metadata, + response: tokio::fs::File::from_std(response), + started: Instant::now(), + bytes: 0, + }) + } + + pub(super) async fn head(&mut self, status: u16, headers: &HeaderMap) -> io::Result<()> { + self.metadata["status"] = json!(status); + // Protocol metadata only. Arbitrary response headers may carry secrets. + self.metadata["content_type"] = json!( + headers + .get("content-type") + .and_then(|value| value.to_str().ok()) + ); + self.save().await + } + + pub(super) async fn chunk(&mut self, bytes: &[u8]) -> io::Result<()> { + self.response.write_all(bytes).await?; + self.bytes += bytes.len(); + Ok(()) + } + + pub(super) async fn finish(&mut self) -> io::Result<()> { + self.response.sync_all().await?; + self.metadata["response_bytes"] = json!(self.bytes); + self.metadata["duration_ms"] = json!(self.started.elapsed().as_millis() as u64); + self.metadata["complete"] = json!(true); + self.save().await + } + + async fn save(&self) -> io::Result<()> { + let directory = self.directory.clone(); + let metadata = self.metadata.clone(); + tokio::task::spawn_blocking(move || write_metadata(&directory, &metadata)) + .await + .map_err(io::Error::other)? + } +} + +fn write_metadata(directory: &Path, metadata: &serde_json::Value) -> io::Result<()> { + hellas_private::write_atomically( + &directory.join("metadata.json"), + ".tmp", + &serde_json::to_vec(metadata)?, + ) +} diff --git a/crates/gateway/src/archive/tests.rs b/crates/gateway/src/archive/tests.rs new file mode 100644 index 00000000..e3976ca2 --- /dev/null +++ b/crates/gateway/src/archive/tests.rs @@ -0,0 +1,150 @@ +use super::*; +use axum::{ + Router, + body::Body, + http::{Request, StatusCode}, + routing::post, +}; +use tower::ServiceExt; + +fn router(directory: &Path, required: bool, cache_enabled: bool) -> Router { + Router::new() + .route( + "/v1/test", + post(|body: Bytes| async move { (StatusCode::TOO_MANY_REQUESTS, body) }), + ) + .layer(axum::middleware::from_fn_with_state( + Policy { + options: ArchiveOptions { + directory: directory.into(), + zdr: required, + }, + cache_enabled, + }, + record, + )) +} + +#[tokio::test] +async fn archives_errors_by_default_without_auth_headers_and_zdr_leaves_no_payload() { + let root = tempfile::tempdir().unwrap(); + let archive = root.path().join("archive"); + prepare(&archive).unwrap(); + let app = router(&archive, false, false); + let response = app + .clone() + .oneshot( + Request::post("/v1/test") + .header("authorization", "Bearer SECRET-HEADER") + .body(Body::from("ordinary payload")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); + let id = response.headers()["x-hellas-request-id"] + .to_str() + .unwrap() + .to_owned(); + let bytes = axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(); + assert_eq!(bytes, "ordinary payload"); + let directory = archive.join(id); + assert_eq!( + std::fs::read(directory.join("request.bin")).unwrap(), + b"ordinary payload" + ); + assert_eq!( + std::fs::read(directory.join("response.bin")).unwrap(), + b"ordinary payload" + ); + let metadata: serde_json::Value = + serde_json::from_slice(&std::fs::read(directory.join("metadata.json")).unwrap()).unwrap(); + assert_eq!(metadata["complete"], true); + assert_eq!(metadata["status"], 429); + assert!(!metadata.to_string().contains("SECRET-HEADER")); + for entry in std::fs::read_dir(&directory).unwrap() { + let file = std::fs::File::open(entry.unwrap().path()).unwrap(); + assert!(hellas_private::is_private(&file).unwrap()); + } + let response = app + .oneshot( + Request::post("/v1/test") + .header("x-hellas-zdr", "true") + .body(Body::from("ZDR-SECRET")) + .unwrap(), + ) + .await + .unwrap(); + assert!(response.headers().get("x-hellas-request-id").is_none()); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "ZDR-SECRET" + ); + assert_eq!(std::fs::read_dir(&archive).unwrap().count(), 1); +} + +#[tokio::test] +async fn cancellation_leaves_an_incomplete_archive() { + let root = tempfile::tempdir().unwrap(); + let response = router(root.path(), false, false) + .oneshot( + Request::post("/v1/test") + .body(Body::from("cancelled")) + .unwrap(), + ) + .await + .unwrap(); + let id = response.headers()["x-hellas-request-id"] + .to_str() + .unwrap() + .to_owned(); + drop(response); + let metadata: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.path().join(id).join("metadata.json")).unwrap()) + .unwrap(); + assert_eq!(metadata["complete"], false); +} + +#[tokio::test] +async fn zdr_rejects_ambiguous_flags_retain_and_enabled_replay_before_disk_writes() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("must-not-exist"); + for (required, cache, header, body) in [ + (false, false, "yes", "{}"), + (true, false, "false", "{}"), + (false, true, "true", "{}"), + (false, false, "true", "{\"store\":true}"), + ] { + let response = router(&path, required, cache) + .oneshot( + Request::post("/v1/test") + .header("x-hellas-zdr", header) + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(!path.exists()); + } + let response = router(&path, true, false) + .oneshot( + Request::post("/v1/test") + .body(Body::from("private")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "private" + ); + assert!(!path.exists()); +} diff --git a/crates/gateway/src/http_fetch.rs b/crates/gateway/src/http_fetch.rs new file mode 100644 index 00000000..f083003e --- /dev/null +++ b/crates/gateway/src/http_fetch.rs @@ -0,0 +1,460 @@ +//! HTTP bytes over authenticated Fetch, without translating vendor schemas. +mod observation; +#[cfg(test)] +mod tests; + +use anyhow::{Context, bail, ensure}; +use axum::{ + Router, + body::{Body, Bytes}, + extract::{Request, State}, + http::{HeaderMap, HeaderValue, StatusCode}, + response::{IntoResponse, Response}, +}; +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use futures::StreamExt; +use hellas_client::{ExecutionRoute, cache::fetch_output_stream}; +use hellas_rpc::{ + Assurance, FetchEnvironment, ProducerSigningKey, Retention, + http_fetch::{HttpFetchRequest, HttpTls, HttpTrustRoots}, + output::{AdaptorEvent, HttpResponseEvent, OutputEvent, StopReason}, +}; +use serde::Deserialize; +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, + time::{Duration, Instant, SystemTime}, +}; +use tokio::sync::Semaphore; +use tracing::Instrument; + +use super::{GatewayHandle, GatewayOptions, access, execution::CliRuntime}; + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpGatewayConfig { + pub service: String, + pub method: String, + pub routes: Vec, + #[serde(default = "default_concurrency")] + pub max_in_flight: usize, +} + +fn default_concurrency() -> usize { + 4 +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpRoute { + pub path: String, + pub method: String, + pub url: String, + pub credential: String, + #[serde(default)] + pub headers: Vec<(String, String)>, + /// Only these caller headers cross the gateway; credentials never do. + #[serde(default)] + pub forward_headers: Vec, +} + +struct Account { + slots: Arc, + retry_at: Mutex, +} + +impl Account { + fn delay(&self) -> Duration { + self.retry_at + .lock() + .unwrap() + .saturating_duration_since(Instant::now()) + } + + fn observe(&self, status: u16, headers: &HeaderMap) { + if status == 429 || (status >= 500 && headers.contains_key("retry-after")) { + let delay = retry_delay(headers).min(Duration::from_secs(u32::MAX as u64)); + let mut retry_at = self.retry_at.lock().unwrap(); + *retry_at = (*retry_at).max(Instant::now() + delay); + } + } +} + +struct HttpState { + config: HttpGatewayConfig, + runtime: CliRuntime, + route: ExecutionRoute, + signer: Arc, + assurance: Assurance, + accounts: BTreeMap>, + metrics: observation::Metrics, +} + +impl HttpGatewayConfig { + fn validate(&self) -> anyhow::Result<()> { + ensure!( + !self.routes.is_empty(), + "HTTP gateway needs at least one route" + ); + ensure!( + self.max_in_flight > 0 && self.max_in_flight <= 1024, + "invalid HTTP concurrency limit" + ); + let mut paths = std::collections::BTreeSet::new(); + for route in &self.routes { + ensure!( + route.path.starts_with('/') && !route.path.contains(['?', '#', '{', '}']), + "HTTP routes must be exact paths" + ); + ensure!( + paths.insert((&route.path, &route.method)), + "duplicate HTTP route" + ); + ensure!( + route.forward_headers.iter().all(|name| matches!( + name.as_str(), + "content-type" + | "accept" + | "anthropic-version" + | "anthropic-beta" + | "user-agent" + )), + "unsupported forwarded header" + ); + ensure!( + route.headers.iter().all(|(name, _)| !matches!( + name.as_str(), + "authorization" | "x-api-key" | "cookie" + )), + "use a provider credential alias for authentication" + ); + route.request(Bytes::new(), &HeaderMap::new()).validate()?; + } + Ok(()) + } +} + +impl HttpRoute { + fn request(&self, body: Bytes, incoming: &HeaderMap) -> HttpFetchRequest { + let mut headers = self.headers.clone(); + for name in &self.forward_headers { + if !headers.iter().any(|(configured, _)| configured == name) { + for value in incoming.get_all(name) { + if let Ok(value) = value.to_str() { + headers.push((name.clone(), value.into())); + } + } + } + } + HttpFetchRequest { + url: self.url.clone(), + method: self.method.clone(), + headers, + body_base64: STANDARD.encode(body), + tls: HttpTls { + roots: HttpTrustRoots::WebPki, + spki_sha256: vec![], + }, + credential: Some(self.credential.clone()), + max_response_bytes: hellas_rpc::http_fetch::MAX_HTTP_RESPONSE_BYTES, + } + } +} + +pub(super) async fn start(options: GatewayOptions) -> anyhow::Result { + let config = options + .http_fetch + .as_ref() + .context("missing HTTP configuration")? + .clone(); + config.validate()?; + ensure!( + options.paid_work.is_none(), + "HTTP routes cannot use a token-native paid pool" + ); + ensure!( + options.output_cache.policy == hellas_rpc::cache::CachePolicy::Off, + "HTTP routes archive exchanges; inference replay must be off" + ); + if !options.archive.zdr { + super::archive::prepare(&options.archive.directory)?; + } + let route = ExecutionRoute::remote( + options.node_id, + options.node_addrs.clone(), + options.retries, + options + .provider_trust + .clone() + .context("HTTP Fetch requires a provider trust anchor")?, + ); + let accounts = config + .routes + .iter() + .map(|route| { + ( + route.credential.clone(), + Arc::new(Account { + slots: Arc::new(Semaphore::new(config.max_in_flight)), + retry_at: Mutex::new(Instant::now()), + }), + ) + }) + .collect(); + let state = Arc::new(HttpState { + config, + runtime: CliRuntime::remote(options.secret_key.clone()).await?, + route, + signer: Arc::new(options.producer_key.clone()), + assurance: options.assurance, + accounts, + metrics: observation::Metrics::new(), + }); + let bearer = Arc::new(match &options.bearer_token_file { + Some(path) => access::Bearer::load_or_create(path)?, + None => access::Bearer::generate(), + }); + let mut app = Router::new(); + let paths: std::collections::BTreeSet<_> = state + .config + .routes + .iter() + .map(|route| route.path.as_str()) + .collect(); + for path in paths { + app = app.route(path, axum::routing::any(handle)); + } + let app = app + .with_state(state) + .layer(axum::middleware::from_fn_with_state( + super::archive::Policy { + options: options.archive.clone(), + cache_enabled: false, + }, + super::archive::record, + )); + #[cfg(feature = "otel")] + let app = app.layer(axum::middleware::from_fn( + hellas_rpc::telemetry::http::trace_request, + )); + let app = app.layer(access::BearerLayer::new(bearer.clone())); + let listener = super::bind_gateway( + &options.host, + options.port, + options.allow_remote && options.bearer_token_file.is_some(), + ) + .await?; + super::launch_gateway( + app, + listener, + bearer, + options.wrap.as_deref(), + &options.wrap_args, + None, + ) + .await +} + +fn error(status: StatusCode, message: &'static str) -> Response { + ( + status, + axum::Json(serde_json::json!({"error": {"message": message}})), + ) + .into_response() +} + +fn limited(status: StatusCode, seconds: u64) -> Response { + let mut response = error( + status, + "provider temporarily unavailable; retry after the indicated delay", + ); + response.headers_mut().insert( + "retry-after", + HeaderValue::from_str(&seconds.max(1).to_string()).unwrap(), + ); + response +} + +async fn handle(State(state): State>, request: Request) -> Response { + let Some(route) = state.config.routes.iter().find(|route| { + route.path == request.uri().path() && route.method == request.method().as_str() + }) else { + return error(StatusCode::NOT_FOUND, "no configured HTTP route"); + }; + if request.uri().query().is_some() { + return error( + StatusCode::BAD_REQUEST, + "query parameters are not supported on this route", + ); + } + let mut observed = observation::Observation::new(&state.metrics, &route.path); + let account = state.accounts[&route.credential].clone(); + let delay = account.delay(); + if !delay.is_zero() { + observed.status(429); + observed.complete(); + return limited(StatusCode::TOO_MANY_REQUESTS, delay.as_secs() + 1); + } + let Ok(permit) = account.slots.clone().try_acquire_owned() else { + observed.status(503); + observed.complete(); + return limited(StatusCode::SERVICE_UNAVAILABLE, 1); + }; + let (parts, body) = request.into_parts(); + // Reserve space for URL, headers and JSON around the base64 body. + let body_limit = (hellas_rpc::fetch::MAX_FETCH_REQUEST_BODY_BYTES - 64 * 1024) / 4 * 3; + let body = match axum::body::to_bytes(body, body_limit).await { + Ok(body) => body, + Err(_) => { + observed.status(413); + observed.complete(); + return error( + StatusCode::PAYLOAD_TOO_LARGE, + "HTTP request exceeds Fetch limit", + ); + } + }; + let upstream = route.request(body, &parts.headers); + let payload = match serde_json::to_vec(&upstream) { + Ok(payload) if upstream.validate().is_ok() => payload, + _ => { + observed.status(400); + observed.complete(); + return error(StatusCode::BAD_REQUEST, "invalid HTTP request"); + } + }; + let result = open(&state, &payload) + .instrument(observed.span.clone()) + .await; + let (status, headers, mut events) = match result { + Ok(value) => value, + Err(_) => { + observed.status(502); + return error(StatusCode::BAD_GATEWAY, "authenticated Fetch failed"); + } + }; + observed.status(status); + let headers = response_headers(headers); + observed.content_type( + headers + .get("content-type") + .and_then(|value| value.to_str().ok()), + ); + account.observe(status, &headers); + let stream = async_stream::try_stream! { + let _permit = permit; + let mut size = 0usize; + while let Some(event) = events.next().instrument(observed.span.clone()).await { + let event = event.map_err(|_| std::io::Error::other("Fetch stream verification failed"))?; + match event { + OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Body { base64 })) => { + let bytes = hellas_rpc::http_fetch::decode_base64(&base64) + .map_err(|_| std::io::Error::other("invalid HTTP response encoding"))?; + size = size.saturating_add(bytes.len()); + if bytes.is_empty() || size > upstream.max_response_bytes as usize { + Err(std::io::Error::other("HTTP response exceeds Fetch limit"))?; + } + observed.chunk(&bytes); + yield Bytes::from(bytes); + } + OutputEvent::Finished { stop_reason: StopReason::EndOfText, usage: None } => { + observed.complete(); + return; + } + _ => Err(std::io::Error::other("unexpected HTTP Fetch event"))?, + } + } + Err(std::io::Error::other("HTTP Fetch ended without verified completion"))?; + }; + let stream: futures::stream::BoxStream<'static, Result> = + Box::pin(stream); + let mut response = Response::new(Body::from_stream(stream)); + *response.status_mut() = StatusCode::from_u16(status).unwrap(); + *response.headers_mut() = headers; + response +} + +async fn open( + state: &HttpState, + payload: &[u8], +) -> anyhow::Result<( + u16, + Vec<(String, String)>, + hellas_adaptors::OutputEventStream, +)> { + let events = hellas_rpc::fetch::build_input_events_with_retention( + &state.config.service, + &state.config.method, + payload, + FetchEnvironment::Http.manifest_id(), + state.assurance, + state.signer.as_ref(), + Retention::Ephemeral, + )?; + let request = hellas_rpc::pb::fetch::FetchRequest { + input: events + .iter() + .map(hellas_rpc::stream::input_event_to_pb) + .collect(), + }; + let mut stream = fetch_output_stream( + state.runtime.clone(), + request, + Some(state.route.clone()), + state.signer.clone(), + None, + ) + .await? + .events; + match stream.next().await.transpose()? { + Some(OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Head { + status, + headers, + }))) if (200..=599).contains(&status) => { + hellas_rpc::http_fetch::check_headers(&headers, false)?; + Ok((status, headers, stream)) + } + _ => bail!("missing authenticated HTTP response head"), + } +} + +fn response_headers(headers: Vec<(String, String)>) -> HeaderMap { + let mut result = HeaderMap::new(); + for (name, value) in headers { + if matches!( + name.as_str(), + "content-type" | "content-encoding" | "retry-after" | "request-id" | "x-request-id" + ) || name.starts_with("x-ratelimit-") + || name.starts_with("ratelimit-") + || name.starts_with("anthropic-ratelimit-") + { + if let (Ok(name), Ok(value)) = ( + name.parse::(), + value.parse::(), + ) { + result.append(name, value); + } + } + } + result +} + +fn retry_delay(headers: &HeaderMap) -> Duration { + let value = headers + .get("retry-after") + .and_then(|value| value.to_str().ok()); + value + .and_then(|value| { + value + .parse::() + .ok() + .map(Duration::from_secs) + .or_else(|| { + httpdate::parse_http_date(value) + .ok() + .map(|date| date.duration_since(SystemTime::now()).unwrap_or_default()) + }) + }) + .unwrap_or(Duration::from_secs(1)) + .max(Duration::from_secs(1)) +} diff --git a/crates/gateway/src/http_fetch/observation.rs b/crates/gateway/src/http_fetch/observation.rs new file mode 100644 index 00000000..33fe50ce --- /dev/null +++ b/crates/gateway/src/http_fetch/observation.rs @@ -0,0 +1,286 @@ +use serde_json::Value; +use std::time::Instant; + +pub(super) struct Metrics { + #[cfg(feature = "otel")] + requests: opentelemetry::metrics::Counter, + #[cfg(feature = "otel")] + duration: opentelemetry::metrics::Histogram, + #[cfg(feature = "otel")] + first_byte: opentelemetry::metrics::Histogram, + #[cfg(feature = "otel")] + tokens: opentelemetry::metrics::Counter, +} + +impl Metrics { + pub(super) fn new() -> Self { + #[cfg(feature = "otel")] + let meter = opentelemetry::global::meter("hellas.gateway.http_fetch"); + Self { + #[cfg(feature = "otel")] + requests: meter.u64_counter("hellas.gateway.http.requests").build(), + #[cfg(feature = "otel")] + duration: meter + .f64_histogram("hellas.gateway.http.duration") + .with_unit("s") + .with_boundaries(vec![ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1., 2., 4., 8., 16., 32., 64., + 128., 300., + ]) + .build(), + #[cfg(feature = "otel")] + first_byte: meter + .f64_histogram("hellas.gateway.http.time_to_first_byte") + .with_unit("s") + .with_boundaries(vec![ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1., 2., 4., 8., 16., 32., 64., + ]) + .build(), + #[cfg(feature = "otel")] + tokens: meter + .u64_counter("hellas.gateway.http.tokens") + .with_unit("{token}") + .build(), + } + } +} + +pub(super) struct Observation { + pub(super) span: tracing::Span, + started: Instant, + complete: bool, + status: u16, + bytes: u64, + first_byte: Option, + usage: Usage, + #[cfg(feature = "otel")] + route: String, + #[cfg(feature = "otel")] + metrics: Metrics, +} + +impl Observation { + pub(super) fn new(metrics: &Metrics, route: &str) -> Self { + #[cfg(not(feature = "otel"))] + let _ = (metrics, route); + Self { + span: hellas_rpc::request_span!(target: "hellas_request", "http.fetch", + otel.kind = "client", http.route = route, + http.response.status_code = tracing::field::Empty, + http.response.body.size = tracing::field::Empty, + gen_ai.usage.input_tokens = tracing::field::Empty, + gen_ai.usage.output_tokens = tracing::field::Empty, + gen_ai.usage.cache_read.input_tokens = tracing::field::Empty, + hellas.response.time_to_first_byte = tracing::field::Empty, + hellas.response.complete = tracing::field::Empty, + error.type = tracing::field::Empty, otel.status_code = tracing::field::Empty), + started: Instant::now(), + complete: false, + status: 0, + bytes: 0, + first_byte: None, + usage: Usage::default(), + #[cfg(feature = "otel")] + route: route.into(), + #[cfg(feature = "otel")] + metrics: Metrics { + requests: metrics.requests.clone(), + duration: metrics.duration.clone(), + first_byte: metrics.first_byte.clone(), + tokens: metrics.tokens.clone(), + }, + } + } + pub(super) fn status(&mut self, status: u16) { + self.status = status; + self.span.record("http.response.status_code", status); + } + pub(super) fn content_type(&mut self, value: Option<&str>) { + self.usage.sse = + value.is_some_and(|value| value.split(';').next() == Some("text/event-stream")); + } + pub(super) fn chunk(&mut self, bytes: &[u8]) { + if self.first_byte.is_none() { + self.first_byte = Some(self.started.elapsed().as_secs_f64()); + } + self.bytes += bytes.len() as u64; + self.usage.push(bytes); + } + pub(super) fn complete(&mut self) { + self.usage.finish(); + self.complete = true; + } +} + +impl Drop for Observation { + fn drop(&mut self) { + self.span.record("http.response.body.size", self.bytes); + self.span.record("hellas.response.complete", self.complete); + if let Some(ttfb) = self.first_byte { + self.span.record("hellas.response.time_to_first_byte", ttfb); + } + for (field, value) in [ + ("gen_ai.usage.input_tokens", self.usage.input), + ("gen_ai.usage.output_tokens", self.usage.output), + ("gen_ai.usage.cache_read.input_tokens", self.usage.cached), + ] { + if let Some(value) = value { + self.span.record(field, value); + } + } + if !self.complete || self.status >= 400 { + self.span.record("otel.status_code", "ERROR"); + self.span.record( + "error.type", + if self.status == 429 { + "rate_limited" + } else if !self.complete { + "incomplete" + } else { + "http_error" + }, + ); + } + #[cfg(feature = "otel")] + { + use opentelemetry::KeyValue; + let labels = [ + KeyValue::new("http.route", self.route.clone()), + KeyValue::new("http.response.status_code", i64::from(self.status)), + KeyValue::new("hellas.response.complete", self.complete), + ]; + self.metrics.requests.add(1, &labels); + self.metrics + .duration + .record(self.started.elapsed().as_secs_f64(), &labels); + if let Some(ttfb) = self.first_byte { + self.metrics.first_byte.record(ttfb, &labels); + } + for (kind, value) in [ + ("input", self.usage.input), + ("output", self.usage.output), + ("cache_read", self.usage.cached), + ] { + if let Some(value) = value { + let mut labels = labels.to_vec(); + labels.push(KeyValue::new("gen_ai.token.type", kind)); + self.metrics.tokens.add(value, &labels); + } + } + } + } +} + +/// Observe standard usage fields without reserializing or delaying the wire body. +#[derive(Default)] +struct Usage { + sse: bool, + pending: Vec, + input: Option, + output: Option, + cached: Option, + overflow: bool, +} + +impl Usage { + fn push(&mut self, bytes: &[u8]) { + if self.overflow { + return; + } + self.pending.extend_from_slice(bytes); + if !self.sse { + return; + } + while let Some(end) = self.pending.iter().position(|byte| *byte == b'\n') { + let line: Vec<_> = self.pending.drain(..=end).collect(); + if let Some(data) = line.strip_prefix(b"data:") { + self.parse(data); + } + } + if self.pending.len() > 512 * 1024 { + self.pending.clear(); + self.overflow = true; + } + } + fn finish(&mut self) { + let pending = std::mem::take(&mut self.pending); + self.parse(pending.strip_prefix(b"data:").unwrap_or(&pending)); + } + fn parse(&mut self, bytes: &[u8]) { + let Ok(value) = serde_json::from_slice::(bytes) else { + return; + }; + let Some(usage) = value.get("usage").or_else(|| { + value + .get("message") + .and_then(|message| message.get("usage")) + }) else { + return; + }; + fn update(target: &mut Option, value: Option) { + if let Some(value) = value { + *target = Some(target.unwrap_or_default().max(value)); + } + } + update( + &mut self.input, + usage + .get("prompt_tokens") + .or_else(|| usage.get("input_tokens")) + .and_then(Value::as_u64), + ); + update( + &mut self.output, + usage + .get("completion_tokens") + .or_else(|| usage.get("output_tokens")) + .and_then(Value::as_u64), + ); + update( + &mut self.cached, + usage + .get("cache_read_input_tokens") + .or_else(|| usage.pointer("/prompt_tokens_details/cached_tokens")) + .and_then(Value::as_u64), + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn usage_survives_arbitrary_sse_boundaries_and_cumulative_updates() { + let wire = b"event: message_start\r\ndata: {\"message\":{\"usage\":{\"input_tokens\":11,\"output_tokens\":1,\"cache_read_input_tokens\":7}}}\r\n\r\ndata: {\"usage\":{\"output_tokens\":5}}\n\ndata: {\"usage\":{\"output_tokens\":5}}\n\ndata: [DONE]\n\n"; + for size in 1..=wire.len() { + let mut usage = Usage { + sse: true, + ..Default::default() + }; + for chunk in wire.chunks(size) { + usage.push(chunk); + } + usage.finish(); + assert_eq!( + (usage.input, usage.output, usage.cached), + (Some(11), Some(5), Some(7)) + ); + } + } + + #[test] + fn nonstreaming_usage_and_absent_usage_remain_distinct() { + let mut usage = Usage::default(); + usage.push(b"{\n\"usage\":{\"prompt_tokens\":12,\"completion_tokens\":3,\"prompt_tokens_details\":{\"cached_tokens\":8}}\n}"); + usage.finish(); + assert_eq!( + (usage.input, usage.output, usage.cached), + (Some(12), Some(3), Some(8)) + ); + let mut absent = Usage::default(); + absent.push(b"{\"error\":\"private message\"}"); + absent.finish(); + assert_eq!(absent.input, None); + } +} diff --git a/crates/gateway/src/http_fetch/tests.rs b/crates/gateway/src/http_fetch/tests.rs new file mode 100644 index 00000000..225bbe3d --- /dev/null +++ b/crates/gateway/src/http_fetch/tests.rs @@ -0,0 +1,112 @@ +use super::*; + +#[test] +fn account_backoff_is_shared_and_never_shortened_by_another_response() { + let account = Arc::new(Account { + slots: Arc::new(Semaphore::new(1)), + retry_at: Mutex::new(Instant::now()), + }); + let other_route = account.clone(); + let header = |seconds: &'static str| { + HeaderMap::from_iter([( + "retry-after".parse().unwrap(), + HeaderValue::from_static(seconds), + )]) + }; + account.observe(429, &header("60")); + assert!(other_route.delay() > Duration::from_secs(59)); + other_route.observe(429, &header("1")); + assert!(account.delay() > Duration::from_secs(59)); + other_route.observe(503, &header("120")); + assert!(account.delay() > Duration::from_secs(119)); + account.observe(200, &header("600")); + assert!(account.delay() < Duration::from_secs(121)); + let permit = account.slots.clone().try_acquire_owned().unwrap(); + assert!(other_route.slots.clone().try_acquire_owned().is_err()); + drop(permit); + assert!(other_route.slots.clone().try_acquire_owned().is_ok()); +} + +#[test] +fn forward_only_protocol_headers_and_keep_retry_and_quota_metadata() { + let route = HttpRoute { + path: "/v1/messages".into(), + method: "POST".into(), + url: "https://api.example.com/v1/messages".into(), + credential: "account".into(), + headers: vec![("anthropic-version".into(), "2023-06-01".into())], + forward_headers: vec!["content-type".into(), "anthropic-version".into()], + }; + let incoming = HeaderMap::from_iter([ + ( + "authorization".parse().unwrap(), + HeaderValue::from_static("Bearer BUYER"), + ), + ( + "x-api-key".parse().unwrap(), + HeaderValue::from_static("BUYER"), + ), + ( + "content-type".parse().unwrap(), + HeaderValue::from_static("application/json"), + ), + ( + "anthropic-version".parse().unwrap(), + HeaderValue::from_static("override"), + ), + ]); + let request = route.request(Bytes::from_static(b"{\"private\":1}"), &incoming); + assert_eq!(request.body().unwrap(), b"{\"private\":1}"); + assert_eq!( + request.headers, + vec![ + ("anthropic-version".into(), "2023-06-01".into()), + ("content-type".into(), "application/json".into()) + ] + ); + let response = response_headers(vec![ + ("retry-after".into(), "3".into()), + ("x-ratelimit-remaining-tokens".into(), "0".into()), + ("content-length".into(), "1234".into()), + ("set-cookie".into(), "secret".into()), + ("connection".into(), "keep-alive".into()), + ("content-type".into(), "text/event-stream".into()), + ]); + assert_eq!(response.len(), 3); + assert_eq!(response["retry-after"], "3"); + assert_eq!(response["x-ratelimit-remaining-tokens"], "0"); + assert_eq!(retry_delay(&response), Duration::from_secs(3)); +} + +#[test] +fn retry_after_supports_dates_and_has_a_nonzero_fallback() { + assert_eq!(retry_delay(&HeaderMap::new()), Duration::from_secs(1)); + for value in ["invalid", "0", "-2"] { + let headers = + HeaderMap::from_iter([("retry-after".parse().unwrap(), value.parse().unwrap())]); + assert_eq!(retry_delay(&headers), Duration::from_secs(1)); + } + let date = httpdate::fmt_http_date(SystemTime::now() + Duration::from_secs(60)); + let headers = HeaderMap::from_iter([("retry-after".parse().unwrap(), date.parse().unwrap())]); + assert!((59..=60).contains(&retry_delay(&headers).as_secs())); +} + +#[test] +fn credential_and_hop_headers_cannot_be_added_to_the_forward_list() { + for name in ["authorization", "x-api-key", "cookie", "host", "connection"] { + let config = HttpGatewayConfig { + service: "http".into(), + method: "request".into(), + max_in_flight: 2, + routes: vec![HttpRoute { + path: "/v1/messages".into(), + method: "POST".into(), + url: "https://example.com/v1/messages".into(), + credential: "account".into(), + headers: vec![], + forward_headers: vec![name.into()], + }], + }; + assert!(config.validate().is_err()); + } +} diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index 29235573..4c313488 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -3,9 +3,11 @@ extern crate tracing; mod access; mod anthropic; +mod archive; mod backend; mod dispatch; mod fetch_backend; +mod http_fetch; mod metrics; mod openai; mod plain; @@ -36,10 +38,12 @@ use std::time::{SystemTime, UNIX_EPOCH}; use self::state::GatewayState; +pub use archive::ArchiveOptions; pub use execution::{ CausalLmExecutionEnvironment, CliRuntime, ExecutionEvent, ExecutionRequest, ExecutionRequestOptions, ExecutionStrategy, Outcome, PreparedExecution, StopReason, }; +pub use http_fetch::HttpGatewayConfig; const DEFAULT_HTTP_PORT: u16 = 8080; @@ -86,6 +90,8 @@ pub trait PaidExecutionBackend: Send + Sync { static NEXT_ID: AtomicU64 = AtomicU64::new(1); pub struct GatewayOptions { + pub archive: ArchiveOptions, + pub http_fetch: Option, pub output_cache: cache::CacheOptions, pub paid_work: Option>, /// Load or create a stable bearer credential in a private file. @@ -229,6 +235,9 @@ pub async fn start(options: GatewayOptions) -> anyhow::Result { } async fn start_gateway(options: GatewayOptions) -> anyhow::Result { + if options.http_fetch.is_some() { + return http_fetch::start(options).await; + } let listener = bind_gateway( &options.host, options.port, @@ -236,6 +245,9 @@ async fn start_gateway(options: GatewayOptions) -> anyhow::Result ) .await?; let state = Arc::new(GatewayState::from_options(&options).await?); + if !options.archive.zdr { + archive::prepare(&options.archive.directory)?; + } // Every route below reaches an executor, so every route below is // behind this run's credential. The layer goes on last, which in axum @@ -251,7 +263,14 @@ async fn start_gateway(options: GatewayOptions) -> anyhow::Result .route("/v1/messages", post(anthropic::handle)) .route("/v1/completions", post(plain::handle)) .with_state(state.clone()) - .layer(provenance_layer::ProvenanceLayer); + .layer(provenance_layer::ProvenanceLayer) + .layer(axum::middleware::from_fn_with_state( + archive::Policy { + options: options.archive.clone(), + cache_enabled: options.output_cache.policy != cache::CachePolicy::Off, + }, + archive::record, + )); #[cfg(feature = "otel")] let app = app.layer(axum::middleware::from_fn( hellas_rpc::telemetry::http::trace_request, diff --git a/crates/gateway/src/state/tests.rs b/crates/gateway/src/state/tests.rs index eac685c0..8491c813 100644 --- a/crates/gateway/src/state/tests.rs +++ b/crates/gateway/src/state/tests.rs @@ -51,6 +51,11 @@ fn test_environment() -> CausalLmExecutionEnvironment { /// A gateway pointed at one node, which callers then vary. fn options(provider_trust: Option) -> GatewayOptions { GatewayOptions { + archive: super::super::ArchiveOptions { + directory: "unused-test-archive".into(), + zdr: true, + }, + http_fetch: None, output_cache: Default::default(), paid_work: None, bearer_token_file: None, @@ -143,6 +148,10 @@ data: {"type":"response.completed","response":{"id":"resp_1","object":"response" store: Some(Arc::new(MemoryCacheStore::default())), }; let directory = tempfile::tempdir().unwrap(); + options.archive = crate::ArchiveOptions { + directory: directory.path().join("archive"), + zdr: false, + }; std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); let socket = directory.path().join("control.sock"); let _control = LocalControlServer::bind( diff --git a/crates/gateway/src/wrap.rs b/crates/gateway/src/wrap.rs index 8c68b5e0..1595765b 100644 --- a/crates/gateway/src/wrap.rs +++ b/crates/gateway/src/wrap.rs @@ -22,13 +22,9 @@ mod linux { /// is configured with `kill_on_drop(true)` so dropping the returned `Child` /// (e.g. on gateway shutdown) tears it down too. /// -/// The gateway's routes require this run's credential, so the child is -/// handed it in the two environment variables whose clients put it in -/// `Authorization: Bearer …` — `OPENAI_API_KEY` and, for Anthropic -/// clients, `ANTHROPIC_AUTH_TOKEN` rather than `ANTHROPIC_API_KEY`, which -/// would send `x-api-key` and be refused. Both overwrite whatever the -/// operator had: the base URL already points at us, so an upstream key -/// would be the wrong secret to send anyway. +/// OpenAI, Anthropic and Kimi clients receive the gateway URL and its bearer +/// credential through their environment overrides. Anthropic uses +/// `ANTHROPIC_AUTH_TOKEN` so it sends the required Authorization header. pub fn spawn( cmd: &str, args: &[String], @@ -42,6 +38,8 @@ pub fn spawn( .env("ANTHROPIC_BASE_URL", base_url) .env("OPENAI_API_KEY", credential) .env("ANTHROPIC_AUTH_TOKEN", credential) + .env("KIMI_MODEL_BASE_URL", format!("{base_url}/v1")) + .env("KIMI_MODEL_API_KEY", credential) .stdin(Stdio::inherit()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) diff --git a/crates/providers/Cargo.toml b/crates/providers/Cargo.toml index c4241db4..d2da68d2 100644 --- a/crates/providers/Cargo.toml +++ b/crates/providers/Cargo.toml @@ -24,6 +24,7 @@ futures.workspace = true hellas-adaptors.workspace = true hellas-executor = { workspace = true, default-features = false } hellas-rpc = { workspace = true, features = ["fetch"] } +hellas-private.workspace = true hellas-wire = { workspace = true, optional = true } reqwest.workspace = true serde.workspace = true @@ -32,6 +33,7 @@ tokio.workspace = true tracing.workspace = true [dev-dependencies] +tempfile.workspace = true rcgen = "0.14" tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } axum.workspace = true diff --git a/crates/providers/HTTPS.md b/crates/providers/HTTPS.md index 0b6f8435..e4555044 100644 --- a/crates/providers/HTTPS.md +++ b/crates/providers/HTTPS.md @@ -22,7 +22,7 @@ body, encoded as standard padded base64. `body_base64` above is `{}`; replace it with your actual API request. Header names are lowercase. `credential` may be omitted for unauthenticated requests -or caller-owned authorization headers. The response ceiling is at most 512 KiB. +or caller-owned authorization headers. The response ceiling is at most 8 MiB. All final HTTP statuses, including errors and redirects, are returned to the client as signed responses; they are never logged with their bodies. Redirects are not followed. A transport failure or an oversized response is not a @@ -65,7 +65,31 @@ The operator configures account aliases separately: Aliases select independent accounts, including several accounts at the same origin. Secrets are read from the provider process's environment and held in -memory. They are injected only for an exact authorized origin, path and method, +memory. Alternatively, replace `secret_env` with `secret_file` and `secret_field` +to read a private JSON login file on each request. The loader rejects symlinks, +non-regular files, files that are not owner-only, and files over 64 KiB; atomic +token rotation is picked up without restarting the provider. + +For short-lived tokens, a file credential can include: + +```json +{ + "secret_file": "/private/account.json", + "secret_field": "access_token", + "refresh": { + "expires_field": "expires_at", + "command": ["/absolute/path/to/account-tool", "refresh"] + } +} +``` + +`expires_at` is Unix time in seconds. The trusted operator command runs when +expiry is within 30 seconds, with no customer data or captured output. Refreshes +are serialized per account, have a 30 second timeout, and must replace the file +with a renewed token. Failures impose a cooldown. The account's login tool owns +OAuth and refresh-token persistence; Hellas does not interpret vendor logins. + +Resolved secrets are injected only for an exact authorized origin, path and method, with public WebPKI roots. Callers cannot override that account's header or change trust roots to impersonate its origin. Restrict paths to the inference endpoints the account is intended to expose. diff --git a/crates/providers/src/http.rs b/crates/providers/src/http.rs index 7d821686..ccd7444d 100644 --- a/crates/providers/src/http.rs +++ b/crates/providers/src/http.rs @@ -22,9 +22,10 @@ use std::{ sync::Arc, time::Duration, }; +use tracing::Instrument; mod config; mod tls; -pub use config::{HttpCredentialConfig, HttpProviderConfig}; +pub use config::{CredentialRefresh, HttpCredentialConfig, HttpProviderConfig, HttpSecret}; const CHUNK_BYTES: usize = 16 * 1024; const IDLE: Duration = Duration::from_secs(90); @@ -46,7 +47,7 @@ pub struct HttpCredential { pub allowed_paths: Vec, pub allowed_methods: Vec, pub header_name: String, - pub header_value: String, + pub header_value: HttpSecret, } impl std::fmt::Debug for HttpCredential { @@ -96,14 +97,12 @@ impl HttpFetchProvider { "credential requires an alias, exact origins, paths and methods", )); } - check_headers( - &[( - credential.header_name.clone(), - credential.header_value.clone(), - )], - true, - ) - .map_err(|_| fault("invalid credential header"))?; + check_headers(&[(credential.header_name.clone(), String::new())], true) + .map_err(|_| fault("invalid credential header"))?; + if let HttpSecret::Value(value) = &credential.header_value { + check_headers(&[(credential.header_name.clone(), value.clone())], true) + .map_err(|_| fault("invalid credential header"))?; + } for path in &credential.allowed_paths { let parsed = Url::parse(&format!("https://scope.invalid{path}")) .map_err(|_| fault("invalid credential path"))?; @@ -237,7 +236,7 @@ impl HttpFetchProvider { .redirect(reqwest::redirect::Policy::none()) .retry(reqwest::retry::never()) .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(300)) + .timeout(Duration::from_secs(20 * 60)) .resolve_to_addrs(host, &addresses) .tls_backend_preconfigured(tls) .build() @@ -255,16 +254,25 @@ impl HttpFetchProvider { ); } if let Some(credential) = credential { - let mut value = HeaderValue::from_str(&credential.header_value) - .map_err(|_| fault("invalid credential header"))?; + let secret = credential.header_value.resolve().await?; + let mut value = + HeaderValue::from_str(&secret).map_err(|_| fault("invalid credential header"))?; value.set_sensitive(true); outbound = outbound.header(&credential.header_name, value); } - let response = outbound + let mut trace = + crate::responses_fetch::telemetry::Request::for_method(&url, &request.method); + let response = trace + .propagate(outbound) .body(request.body().map_err(|_| fault("invalid HTTP body"))?) .send() + .instrument(trace.span.clone()) .await - .map_err(|_| fault("HTTPS transport or certificate verification failed"))?; + .map_err(|_| { + trace.fail("transport_error"); + fault("HTTPS transport or certificate verification failed") + })?; + trace.status(response.status().as_u16()); // A non-2xx status is still a completed HTTP exchange. Return it, with // its exact body, to the authenticated client; do not log it. let headers = response @@ -287,7 +295,6 @@ impl HttpFetchProvider { let limit = request.max_response_bytes as usize; let stream = async_stream::try_stream! { let mut upstream = response.bytes_stream(); - let mut buffered = Vec::with_capacity(CHUNK_BYTES); let mut received = 0usize; loop { let next = tokio::time::timeout(IDLE, upstream.next()).await @@ -296,24 +303,19 @@ impl HttpFetchProvider { let chunk = chunk.map_err(|_| fault("HTTPS response stream failed"))?; received = received.checked_add(chunk.len()).ok_or_else(|| fault("HTTPS response size overflow"))?; if received > limit { Err(fault("HTTPS response exceeds signed byte limit"))?; } - let mut remaining = chunk.as_ref(); - while !remaining.is_empty() { - let count = (CHUNK_BYTES - buffered.len()).min(remaining.len()); - buffered.extend_from_slice(&remaining[..count]); - remaining = &remaining[count..]; - if buffered.len() == CHUNK_BYTES { - yield std::mem::replace(&mut buffered, Vec::with_capacity(CHUNK_BYTES)); - } + // Flush each upstream delivery. Waiting for a full 16 KiB + // record otherwise holds small SSE responses until EOF. + for part in chunk.chunks(CHUNK_BYTES) { + yield part.to_vec(); } } - if !buffered.is_empty() { yield buffered; } }; Ok(FetchProviderResponse { head: FetchProviderResponseHead { effective_model: None, http: Some(head), }, - stream: Box::pin(stream), + stream: Box::pin(trace.stream(stream)), }) } } diff --git a/crates/providers/src/http/config.rs b/crates/providers/src/http/config.rs index 6b1beca6..46ecc587 100644 --- a/crates/providers/src/http/config.rs +++ b/crates/providers/src/http/config.rs @@ -1,9 +1,15 @@ -//! Operator-owned account aliases. Files contain environment variable names, -//! never customer payloads or API keys. +//! Operator-owned account aliases reference environment variables or private +//! credential files; route configuration contains no secret values. use super::{HttpCredential, HttpEgressPolicy, HttpFetchProvider}; use hellas_executor::{FetchProviderError, FetchRouteEntry, FetchRoutePolicy}; use serde::Deserialize; use std::collections::BTreeMap; +use std::{ + io::Read, + path::PathBuf, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; #[derive(Clone, Debug, Default, Deserialize)] #[serde(deny_unknown_fields)] @@ -23,11 +29,151 @@ pub struct HttpCredentialConfig { pub allowed_paths: Vec, pub allowed_methods: Vec, pub header_name: String, - pub secret_env: String, + pub secret_env: Option, + /// A private JSON credential file maintained by the account's login tool. + pub secret_file: Option, + pub secret_field: Option, + pub refresh: Option, #[serde(default)] pub prefix: String, } +/// An operator-owned login tool updates the same private JSON file. It receives +/// no customer data, and its output is never forwarded to clients or logs. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CredentialRefresh { + pub command: Vec, + pub expires_field: String, + #[serde(skip)] + lock: Arc>>, +} + +impl CredentialRefresh { + fn due(&self, object: &serde_json::Value) -> Result { + let expiry = object + .get(&self.expires_field) + .and_then(serde_json::Value::as_f64) + .filter(|value| value.is_finite()) + .ok_or_else(|| super::fault("credential expiry unavailable"))?; + Ok(expiry + <= SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() + + 30.) + } + async fn run(&self) -> Result<(), FetchProviderError> { + use tracing::Instrument; + let span = hellas_rpc::request_span!(target: "hellas_request", "credential.refresh", + otel.kind = "internal", otel.status_code = tracing::field::Empty, error.type = tracing::field::Empty); + let Some((program, args)) = self.command.split_first() else { + return Err(super::fault("empty credential refresh command")); + }; + let mut command = tokio::process::Command::new(program); + command + .args(args) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true); + let result = tokio::time::timeout(Duration::from_secs(30), command.status()) + .instrument(span.clone()) + .await; + if !matches!(result, Ok(Ok(status)) if status.success()) { + span.record("otel.status_code", "ERROR"); + span.record("error.type", "refresh_failed"); + return Err(super::fault("credential refresh failed")); + } + Ok(()) + } +} + +#[derive(Clone)] +pub enum HttpSecret { + Value(String), + JsonFile { + path: PathBuf, + field: String, + prefix: String, + refresh: Option, + }, +} + +impl From for HttpSecret { + fn from(value: String) -> Self { + Self::Value(value) + } +} +impl From<&str> for HttpSecret { + fn from(value: &str) -> Self { + Self::Value(value.into()) + } +} + +impl HttpSecret { + pub(super) async fn resolve(&self) -> Result { + match self { + Self::Value(value) => Ok(value.clone()), + Self::JsonFile { + path, + field, + prefix, + refresh, + } => { + let mut guard = match refresh { + Some(refresh) => Some(refresh.lock.lock().await), + None => None, + }; + let mut object = read_credential(path.clone()).await?; + if let Some(refresh) = refresh { + if refresh.due(&object)? { + let retry_at = guard.as_mut().expect("refresh holds its account lock"); + if retry_at.is_some_and(|deadline| deadline > std::time::Instant::now()) { + return Err(super::fault("credential refresh is cooling down")); + } + **retry_at = Some(std::time::Instant::now() + Duration::from_secs(30)); + refresh.run().await?; + object = read_credential(path.clone()).await?; + if refresh.due(&object)? { + return Err(super::fault("credential refresh did not renew the token")); + } + **retry_at = None; + } + } + let secret = object + .get(field) + .and_then(serde_json::Value::as_str) + .filter(|secret| !secret.is_empty()) + .ok_or_else(|| super::fault("credential field unavailable"))?; + Ok(format!("{prefix}{secret}")) + } + } + } +} + +async fn read_credential(path: PathBuf) -> Result { + tokio::task::spawn_blocking(move || { + let file = hellas_private::open_nofollow(&path) + .map_err(|_| super::fault("credential file unavailable"))?; + if !hellas_private::is_private(&file).unwrap_or(false) { + return Err(super::fault("credential file must be owner-only")); + } + let mut bytes = Vec::new(); + file.take(65537) + .read_to_end(&mut bytes) + .map_err(|_| super::fault("credential file read failed"))?; + if bytes.len() > 65536 { + return Err(super::fault("credential file exceeds limit")); + } + let object: serde_json::Value = serde_json::from_slice(&bytes) + .map_err(|_| super::fault("credential file is not JSON"))?; + Ok(object) + }) + .await + .map_err(|_| super::fault("credential loader failed"))? +} + impl HttpProviderConfig { pub fn into_entry( self, @@ -35,11 +181,35 @@ impl HttpProviderConfig { ) -> Result { let mut credentials = BTreeMap::new(); for (alias, config) in self.credentials { - let secret = std::env::var(&config.secret_env) - .map_err(|_| super::fault("account secret environment variable is unavailable"))?; - if secret.is_empty() { - return Err(super::fault("account secret is empty")); + if config.refresh.as_ref().is_some_and(|refresh| { + refresh.command.is_empty() || refresh.expires_field.is_empty() + }) { + return Err(super::fault( + "credential refresh requires a command and expiry field", + )); } + let header_value = match (config.secret_env, config.secret_file, config.secret_field) { + (Some(name), None, None) if config.refresh.is_none() => { + let secret = std::env::var(name).map_err(|_| { + super::fault("account secret environment variable is unavailable") + })?; + if secret.is_empty() { + return Err(super::fault("account secret is empty")); + } + HttpSecret::Value(format!("{}{secret}", config.prefix)) + } + (None, Some(path), Some(field)) if !field.is_empty() => HttpSecret::JsonFile { + path, + field, + prefix: config.prefix, + refresh: config.refresh, + }, + _ => { + return Err(super::fault( + "configure either secret_env or secret_file with secret_field", + )); + } + }; credentials.insert( alias, HttpCredential { @@ -47,7 +217,7 @@ impl HttpProviderConfig { allowed_paths: config.allowed_paths, allowed_methods: config.allowed_methods, header_name: config.header_name, - header_value: format!("{}{secret}", config.prefix), + header_value, }, ); } @@ -66,3 +236,101 @@ impl HttpProviderConfig { .map_err(|_| super::fault("invalid HTTP Fetch route")) } } + +#[cfg(test)] +mod tests { + use super::*; + #[cfg(unix)] + #[tokio::test] + async fn failed_refresh_is_not_repeated_by_waiting_requests() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("account.json"); + let counter = directory.path().join("count"); + hellas_private::write_atomically( + &path, + ".tmp", + b"{\"access_token\":\"old\",\"expires_at\":0}", + ) + .unwrap(); + let source = HttpSecret::JsonFile { + path, + field: "access_token".into(), + prefix: String::new(), + refresh: Some(CredentialRefresh { + command: vec![ + "/bin/sh".into(), + "-c".into(), + "printf 'attempt\\n' >> \"$1\"; exit 1".into(), + "refresh-test".into(), + counter.to_string_lossy().into(), + ], + expires_field: "expires_at".into(), + lock: Default::default(), + }), + }; + let (a, b) = tokio::join!(source.resolve(), source.resolve()); + assert!(a.is_err() && b.is_err()); + assert_eq!(std::fs::read_to_string(counter).unwrap(), "attempt\n"); + } + + #[cfg(unix)] + #[tokio::test] + async fn concurrent_reads_refresh_once_and_verify_the_replacement() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("account.json"); + let counter = directory.path().join("count"); + hellas_private::write_atomically( + &path, + ".tmp", + b"{\"access_token\":\"old\",\"expires_at\":0}", + ) + .unwrap(); + let refresh = CredentialRefresh { + command: vec!["/bin/sh".into(), "-c".into(), + "printf '%s' '{\"access_token\":\"new\",\"expires_at\":4102444800}' > \"$1\"; printf 'one\\n' >> \"$2\"".into(), + "refresh-test".into(), path.to_string_lossy().into(), counter.to_string_lossy().into()], + expires_field: "expires_at".into(), lock: Default::default(), + }; + let source = HttpSecret::JsonFile { + path, + field: "access_token".into(), + prefix: "Bearer ".into(), + refresh: Some(refresh), + }; + let (a, b) = tokio::join!(source.resolve(), source.resolve()); + assert_eq!(a.unwrap(), "Bearer new"); + assert_eq!(b.unwrap(), "Bearer new"); + assert_eq!(std::fs::read_to_string(counter).unwrap(), "one\n"); + } + #[tokio::test] + async fn file_credentials_follow_atomic_rotation_and_reject_missing_or_public_secrets() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("account.json"); + let source = HttpSecret::JsonFile { + path: path.clone(), + field: "access_token".into(), + prefix: "Bearer ".into(), + refresh: None, + }; + assert!(source.resolve().await.is_err()); + for value in ["first", "rotated"] { + hellas_private::write_atomically( + &path, + ".tmp", + &serde_json::to_vec(&serde_json::json!({"access_token":value})).unwrap(), + ) + .unwrap(); + assert_eq!(source.resolve().await.unwrap(), format!("Bearer {value}")); + } + hellas_private::write_atomically(&path, ".tmp", b"{\"refresh_token\":\"DO-NOT-LEAK\"}") + .unwrap(); + let error = source.resolve().await.unwrap_err().to_string(); + assert!(!error.contains("DO-NOT-LEAK")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(source.resolve().await.is_err()); + } + } +} diff --git a/crates/providers/src/http/tests.rs b/crates/providers/src/http/tests.rs index e5874729..e50c9478 100644 --- a/crates/providers/src/http/tests.rs +++ b/crates/providers/src/http/tests.rs @@ -15,6 +15,19 @@ async fn server( HttpFetchRequest, Arc, tokio::task::JoinHandle<()>, +) { + server_with_pause(status, bytes, location, None).await +} + +async fn server_with_pause( + status: u16, + bytes: Vec, + location: Option, + pause: Option>, +) -> ( + HttpFetchRequest, + Arc, + tokio::task::JoinHandle<()>, ) { let key = generate_simple_self_signed(vec!["localhost".into()]).unwrap(); let cert = key.cert.der().clone(); @@ -52,6 +65,7 @@ async fn server( let seen = seen.clone(); let bytes = bytes.clone(); let location = location.clone(); + let pause = pause.clone(); tokio::spawn(async move { let Ok(mut socket) = acceptor.accept(socket).await else { return; @@ -68,11 +82,24 @@ async fn server( let location = location .map(|v| format!("Location: {v}\r\n")) .unwrap_or_default(); + let retry_after = if status == 429 { + "Retry-After: 7\r\n" + } else { + "" + }; let header = format!( - "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: close\r\n{location}\r\n", + "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: close\r\n{location}{retry_after}\r\n", bytes.len() ); let _ = socket.write_all(header.as_bytes()).await; + if let Some(pause) = pause { + let _ = socket.write_all(&bytes[..1]).await; + let _ = socket.flush().await; + pause.notified().await; + let _ = socket.write_all(&bytes[1..]).await; + let _ = socket.shutdown().await; + return; + } let _ = socket.write_all(&bytes).await; let _ = socket.shutdown().await; }); @@ -98,6 +125,59 @@ async fn server( ) } +#[tokio::test] +async fn rate_limit_is_returned_with_its_body_and_delay_without_retrying() { + let body = br#"{"error":{"type":"rate_limit"}}"#.to_vec(); + let (request, calls, task) = server(429, body.clone(), None).await; + let mut response = provider().run(prepared(&request)).await.unwrap(); + let head = response.head.http.unwrap(); + assert_eq!(head.status, 429); + assert!(head.headers.contains(&("retry-after".into(), "7".into()))); + let mut received = Vec::new(); + while let Some(chunk) = response.stream.next().await { + received.extend(chunk.unwrap()); + } + assert_eq!(received, body); + assert_eq!(calls.load(Ordering::SeqCst), 1); + task.abort(); +} + +#[tokio::test] +async fn coding_response_can_exceed_the_old_half_megabyte_ceiling() { + let body = vec![b'x'; 1024 * 1024]; + let (mut request, calls, task) = server(200, body.clone(), None).await; + request.max_response_bytes = hellas_rpc::http_fetch::MAX_HTTP_RESPONSE_BYTES; + let mut response = provider().run(prepared(&request)).await.unwrap(); + let mut received = Vec::new(); + while let Some(chunk) = response.stream.next().await { + let chunk = chunk.unwrap(); + assert!(chunk.len() <= CHUNK_BYTES); + received.extend(chunk); + } + assert_eq!(received, body); + assert_eq!(calls.load(Ordering::SeqCst), 1); + task.abort(); +} + +#[tokio::test] +async fn small_stream_delivery_does_not_wait_for_record_capacity_or_eof() { + let resume = Arc::new(tokio::sync::Notify::new()); + let (request, _, server) = + server_with_pause(200, b"ab".to_vec(), None, Some(resume.clone())).await; + let provider = provider(); + let mut response = provider.run(prepared(&request)).await.unwrap(); + let first = tokio::time::timeout(Duration::from_secs(1), response.stream.next()) + .await + .expect("first byte was held until EOF") + .unwrap() + .unwrap(); + assert_eq!(first, b"a"); + resume.notify_one(); + assert_eq!(response.stream.next().await.unwrap().unwrap(), b"b"); + assert!(response.stream.next().await.is_none()); + server.abort(); +} + fn provider() -> HttpFetchProvider { HttpFetchProvider::new( HttpEgressPolicy { diff --git a/crates/providers/src/lib.rs b/crates/providers/src/lib.rs index 9d32049f..39a34dbf 100644 --- a/crates/providers/src/lib.rs +++ b/crates/providers/src/lib.rs @@ -8,8 +8,8 @@ mod responses_fetch; mod responses_projector; mod stream; pub use http::{ - HttpCredential, HttpCredentialConfig, HttpEgressPolicy, HttpFetchAdaptorFactory, - HttpFetchProvider, HttpProviderConfig, + CredentialRefresh, HttpCredential, HttpCredentialConfig, HttpEgressPolicy, + HttpFetchAdaptorFactory, HttpFetchProvider, HttpProviderConfig, HttpSecret, }; pub use openai::OpenAiResponsesFetchProvider; diff --git a/crates/providers/src/responses_fetch.rs b/crates/providers/src/responses_fetch.rs index fb9fe569..5dc03432 100644 --- a/crates/providers/src/responses_fetch.rs +++ b/crates/providers/src/responses_fetch.rs @@ -8,7 +8,7 @@ use tracing::Instrument; #[cfg_attr(feature = "otel", path = "responses_fetch/telemetry/otel.rs")] #[cfg_attr(not(feature = "otel"), path = "responses_fetch/telemetry/noop.rs")] -mod telemetry; +pub(crate) mod telemetry; /// A total request deadline bounds the whole call; this independent idle /// deadline prevents a peer that stops producing SSE bytes from occupying a diff --git a/crates/providers/src/responses_fetch/telemetry/noop.rs b/crates/providers/src/responses_fetch/telemetry/noop.rs index 190e23a7..8948ce84 100644 --- a/crates/providers/src/responses_fetch/telemetry/noop.rs +++ b/crates/providers/src/responses_fetch/telemetry/noop.rs @@ -2,13 +2,16 @@ use futures::Stream; use hellas_executor::FetchProviderError; use tracing::Span; -pub(super) struct Request { +pub(crate) struct Request { pub span: Span, } impl Request { pub fn new(_: &reqwest::Url) -> Self { Self { span: Span::none() } } + pub fn for_method(_: &reqwest::Url, _: &str) -> Self { + Self { span: Span::none() } + } pub fn propagate(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder { request } diff --git a/crates/providers/src/responses_fetch/telemetry/otel.rs b/crates/providers/src/responses_fetch/telemetry/otel.rs index 70f54f6b..162ede5c 100644 --- a/crates/providers/src/responses_fetch/telemetry/otel.rs +++ b/crates/providers/src/responses_fetch/telemetry/otel.rs @@ -4,16 +4,19 @@ use hellas_executor::FetchProviderError; use hellas_wire::metadata::Metadata; use tracing::{Instrument, Span}; -pub(super) struct Request { +pub(crate) struct Request { pub span: Span, complete: bool, } impl Request { pub fn new(endpoint: &reqwest::Url) -> Self { + Self::for_method(endpoint, "POST") + } + pub fn for_method(endpoint: &reqwest::Url, method: &str) -> Self { Self { complete: false, - span: hellas_rpc::request_span!(target: "hellas_request", "POST", - otel.kind = "client", http.request.method = "POST", + span: hellas_rpc::request_span!(target: "hellas_request", "http.upstream", + otel.kind = "client", otel.name = method, http.request.method = method, server.address = endpoint.host_str().unwrap_or(""), server.port = endpoint.port_or_known_default().map(i64::from), url.scheme = endpoint.scheme(), diff --git a/crates/rpc/src/fetch.rs b/crates/rpc/src/fetch.rs index 93a67c40..314066f2 100644 --- a/crates/rpc/src/fetch.rs +++ b/crates/rpc/src/fetch.rs @@ -29,18 +29,18 @@ const INPUT_EVENT_KINDS: [&str; 8] = [ /// /// Each envelope has its own wire frame; this bound limits retained state and /// transcript verification work rather than the size of a terminal frame. -pub const MAX_FETCH_OUTPUT_EVENTS: usize = 4_096; +pub const MAX_FETCH_OUTPUT_EVENTS: usize = 32_768; /// Maximum cumulative payload bytes across all signed Fetch output envelopes, /// including the terminal payload. -pub const MAX_FETCH_OUTPUT_PAYLOAD_BYTES: usize = 2 * 1024 * 1024; +pub const MAX_FETCH_OUTPUT_PAYLOAD_BYTES: usize = 16 * 1024 * 1024; /// Maximum UTF-8 JSON bytes in the signed Fetch `request.body` event. /// /// This is checked before callers allocate/sign a transcript and again after /// verifiers authenticate it, so every Fetch implementation shares the same /// v0.0.1 admission bound. -pub const MAX_FETCH_REQUEST_BODY_BYTES: usize = 1024 * 1024; +pub const MAX_FETCH_REQUEST_BODY_BYTES: usize = 2 * 1024 * 1024; /// Maximum UTF-8 bytes in each signed Fetch route component (`service` and /// `method`). diff --git a/crates/rpc/src/fetch/tests.rs b/crates/rpc/src/fetch/tests.rs index d159b448..9132a8a4 100644 --- a/crates/rpc/src/fetch/tests.rs +++ b/crates/rpc/src/fetch/tests.rs @@ -148,6 +148,13 @@ fn request_body_limit_accepts_the_exact_boundary() { .unwrap(); assert_eq!(verify_input_events(&events).unwrap().body.as_bytes(), body); + let request = crate::pb::fetch::FetchRequest { + input: events + .iter() + .map(crate::stream::input_event_to_pb) + .collect(), + }; + assert!(prost::Message::encoded_len(&request) < hellas_wire::frame::MAX_FRAME_BYTES); } #[test] diff --git a/crates/rpc/src/http_fetch.rs b/crates/rpc/src/http_fetch.rs index d645d756..fc25c12e 100644 --- a/crates/rpc/src/http_fetch.rs +++ b/crates/rpc/src/http_fetch.rs @@ -4,7 +4,7 @@ use base64::{Engine as _, engine::general_purpose::STANDARD}; use serde::{Deserialize, Serialize}; -pub const MAX_HTTP_RESPONSE_BYTES: u32 = 512 * 1024; +pub const MAX_HTTP_RESPONSE_BYTES: u32 = 8 * 1024 * 1024; pub const MAX_HTTP_HEADERS_BYTES: usize = 16 * 1024; pub const MAX_HTTP_HEADERS: usize = 64; diff --git a/docs/http-gateway.md b/docs/http-gateway.md new file mode 100644 index 00000000..1c1f3d05 --- /dev/null +++ b/docs/http-gateway.md @@ -0,0 +1,96 @@ +# HTTP APIs through Fetch + +`gateway --http-fetch-config FILE` exposes exact HTTP routes over the generic +HTTPS Fetch environment. Request and response bodies keep the upstream format, +including SSE, tool calls and non-2xx errors. It requires a provider trust anchor +and a provider route with a caller grant; it does not use the token-native paid +pool. See [provider account and egress configuration](../crates/providers/HTTPS.md). + +Example gateway configuration for Kimi Code: + +```json +{ + "service": "https", + "method": "request", + "max_in_flight": 2, + "routes": [{ + "path": "/v1/chat/completions", + "method": "POST", + "url": "https://api.kimi.com/coding/v1/chat/completions", + "credential": "kimi", + "forward_headers": ["content-type", "accept", "user-agent"] + }] +} +``` + +The service and method must match the provider's configured Fetch route. The +credential alias belongs to that provider. Each alias shares a concurrency +limit and cooldown across its HTTP routes. Paths and methods match exactly; +query parameters are rejected. The forward list only accepts protocol headers; +the caller's gateway bearer never becomes an upstream credential. Optional +`headers` holds operator-supplied `[name, value]` pairs, with lowercase names. + +```sh +hellas-cli --identity ./gateway.identity gateway \ + --http-fetch-config ./http-gateway.json \ + --provider "$PROVIDER_ENROLLMENT" \ + --node-id "$PROVIDER_NODE_ID" --node-addr "$PROVIDER_ADDRESS" \ + --port 8080 --bearer-token-file ./gateway.bearer +``` + +The client signs the request and verifies each response chunk before forwarding +it. Provider-owned credentials require public WebPKI roots. The gateway asks the +provider for ephemeral retention and disables inference replay. Its own archive +policy is independent of provider retention and upstream `store` fields. + +For Kimi Code, set `KIMI_MODEL_NAME` to an available model and use `--wrap kimi`; +the wrapper supplies `KIMI_MODEL_BASE_URL` and `KIMI_MODEL_API_KEY`. A separately +launched client can use the same variables and the private gateway bearer file. +Choose a context size that fits the byte limits below. + +## Archives and ZDR + +CLI gateways archive authenticated requests and responses by default under +`~/.hellas/gateway-archive`, or `--archive-dir DIRECTORY`. This applies to the +existing inference routes as well as HTTP Fetch. Each exchange has owner-only +`request.bin`, `response.bin` and `metadata.json` files. Metadata records status, +size, elapsed time, completion and trace context; it excludes authentication +headers. `x-hellas-request-id` identifies the exchange. Failed or cancelled +streams retain an incomplete archive. The gateway refuses requests when it +cannot open their archive, and stops streaming on an archive write failure. +There is no automatic archive pruning. + +`x-hellas-zdr: true` disables application payload persistence for that request. +`--zdr` enforces this for all requests. Ambiguous flags, `store: true`, and ZDR +with inference caching enabled are rejected before archive writes. A non-ZDR +request may be archived even when its upstream `store` is false. Unauthorized, +oversized and invalid ZDR requests are rejected without payload archives. + +These switches govern this gateway's application writes. They do not establish +OS swap/dump protection, a remote provider's host policy, upstream retention, +or the coding client's own session-log policy. Software identities do not +attest those properties. The embedded Responses-only `start_fetch` API retains +its host-managed storage policy. + +## Limits and telemetry + +HTTP input is limited to 1,523,712 bytes, reserving envelope space inside the +2 MiB signed Fetch request. Responses are limited to 8 MiB and the Fetch stream +to 32,768 events and 16 MiB of signed payload. The provider has a 20 minute total +HTTP deadline and a 90 second response idle timeout. Exceeding a stream limit +closes it as incomplete. Individual upstream deliveries are forwarded promptly, +split at 16 KiB; the provider does not wait for a full buffer. + +No generation is automatically retried upstream. Upstream status, `Retry-After`, +request IDs and rate-limit headers reach the client. A 429, or a 5xx carrying +`Retry-After`, starts a shared account cooldown; delta seconds and HTTP dates +are accepted. Missing or invalid delays on a 429 use one second. Excess +concurrency returns 503 with `Retry-After: 1`. + +With `otel`, traces connect HTTP ingress, Fetch RPCs, credential refresh and +upstream HTTP. Span attributes exclude request/response bodies and credentials. +`hellas.gateway.http.requests`, `.duration`, `.time_to_first_byte` and `.tokens` +record status, completion, timings and standard OpenAI/Anthropic usage fields. +Usage is unknown when upstream omits it; cached tokens are reported separately. +Account quota windows still come from the upstream quota exporter. Request +token totals alone cannot determine subscription quota or billing. From 79e3161bb1d08809f6804ee8fd5b21b6c2a6e9b3 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Thu, 24 Sep 2026 08:28:10 +0200 Subject: [PATCH 05/32] Preserve HTTP semantics and streaming backpressure through Fetch --- crates/executor/Cargo.toml | 1 + .../executor/src/executor/actor/execution.rs | 80 +++---- .../src/executor/actor/execution/tests.rs | 44 +++- crates/gateway/src/archive.rs | 24 +- crates/gateway/src/http_fetch.rs | 210 +++++++++++++----- crates/gateway/src/http_fetch/observation.rs | 30 ++- crates/gateway/src/http_fetch/tests.rs | 75 ++++++- crates/rpc/src/call.rs | 46 +++- crates/wire/src/iroh/stream.rs | 6 + crates/wire/src/transport.rs | 6 + docs/http-gateway.md | 48 +++- 11 files changed, 439 insertions(+), 131 deletions(-) diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml index eba5751b..8c7a2b0b 100644 --- a/crates/executor/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -49,5 +49,6 @@ uuid = { version = "1", features = ["v4"] } zeroize = "1" [dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } opentelemetry_sdk = { workspace = true, features = ["testing"] } tracing-subscriber = { workspace = true } diff --git a/crates/executor/src/executor/actor/execution.rs b/crates/executor/src/executor/actor/execution.rs index c45a0927..9ec85e4e 100644 --- a/crates/executor/src/executor/actor/execution.rs +++ b/crates/executor/src/executor/actor/execution.rs @@ -60,9 +60,10 @@ use super::{ }; /// Backpressure buffer for the per-execution event channel. The worker keeps -/// one slot reserved for the terminal frame and cancels a consumer that does -/// not drain the rest; it never blocks the sole execution thread. +/// one slot reserved for the terminal frame. Backpressure waits in the spawned +/// provider task, leaving the execution actor free to admit and finish work. const PER_EXECUTION_CHANNEL_CAPACITY: usize = 64; +const FETCH_STREAM_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(90); const FETCH_STREAM_STALLED_ERROR: &str = "fetch stream consumer did not drain its bounded event channel"; @@ -997,9 +998,10 @@ pub(super) async fn run_fetch_provider( let mut position = 0_u64; let mut terminal = None; let mut projection_budget = FetchProjectionBudget::default(); - let response = provider - .run(request) - .await + let response = tokio::select! { + response = provider.run(request) => response, + _ = sender.closed() => Err(FetchProviderError::failed("fetch stream consumer disconnected")), + } .map_err(|error| FetchProviderFailure { position, error })?; let projected = projector .begin(response.head) @@ -1018,7 +1020,16 @@ pub(super) async fn run_fetch_provider( .await?; let mut stream = response.stream; - while let Some(next) = stream.next().await { + loop { + let next = tokio::select! { + next = stream.next() => next, + _ = sender.closed() => return Err(FetchProviderFailure { + position, error: FetchProviderError::failed("fetch stream consumer disconnected"), + }), + }; + let Some(next) = next else { + break; + }; let chunk = next.map_err(|error| FetchProviderFailure { position, error })?; let projected = projector .project(&chunk) @@ -1096,15 +1107,19 @@ async fn process_projected_fetch( ), }); } - // The actor needs one guaranteed permit for WorkFinished or - // WorkFailed. Fail before signing another chunk when only that - // permit remains; never await a slow consumer here. - if sender.capacity() <= 1 { - return Err(FetchProviderFailure { - position: *position, - error: FetchProviderError::failed(FETCH_STREAM_STALLED_ERROR), - }); - } + // Reserve the event and terminal slots together. Temporary + // backpressure must not truncate a valid upstream response. + let mut permits = + tokio::time::timeout(FETCH_STREAM_DRAIN_TIMEOUT, sender.reserve_many(2)) + .await + .map_err(|_| FetchProviderFailure { + position: *position, + error: FetchProviderError::failed(FETCH_STREAM_STALLED_ERROR), + })? + .map_err(|_| FetchProviderFailure { + position: *position, + error: FetchProviderError::failed("fetch stream consumer disconnected"), + })?; let payload_len = projection_budget .record_event(payload.len()) @@ -1128,31 +1143,16 @@ async fn process_projected_fetch( "fetch output event transcript failed: {err}" )), })?; - match sender.try_send(Ok(WorkEvent { - kind: Some(work_event::Kind::Chunk(WorkChunk { - output_event: Some(output_event_to_pb(&output_event)), - })), - })) { - Ok(()) => { - *position = next_position; - // Give the just-returned RPC receiver a scheduling - // opportunity before classifying a full burst as a - // stalled consumer. - tokio::task::yield_now().await; - } - Err(mpsc::error::TrySendError::Full(_)) => { - return Err(FetchProviderFailure { - position: *position, - error: FetchProviderError::failed(FETCH_STREAM_STALLED_ERROR), - }); - } - Err(mpsc::error::TrySendError::Closed(_)) => { - return Err(FetchProviderFailure { - position: *position, - error: FetchProviderError::failed("fetch stream consumer disconnected"), - }); - } - } + permits + .next() + .expect("two reserved permits") + .send(Ok(WorkEvent { + kind: Some(work_event::Kind::Chunk(WorkChunk { + output_event: Some(output_event_to_pb(&output_event)), + })), + })); + drop(permits); + *position = next_position; } ProjectedFetch::Terminal(payload) => { if terminal.is_some() { diff --git a/crates/executor/src/executor/actor/execution/tests.rs b/crates/executor/src/executor/actor/execution/tests.rs index b70eb2e9..7a41e8f5 100644 --- a/crates/executor/src/executor/actor/execution/tests.rs +++ b/crates/executor/src/executor/actor/execution/tests.rs @@ -1220,7 +1220,7 @@ async fn fetch_projection_rejects_event_buffered_after_terminal() { assert_eq!(projection_budget.events, 1); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn fetch_projection_preserves_one_permit_for_failure_terminal() { let signing_key = key(); let input_commitment = InputCommitment::from_digest(Digest::from_bytes([8; 32])); @@ -1268,6 +1268,48 @@ async fn fetch_projection_preserves_one_permit_for_failure_terminal() { )); } +#[tokio::test(start_paused = true)] +async fn fetch_projection_waits_for_temporary_backpressure_then_delivers_in_order() { + let signing_key = key(); + let input = InputCommitment::from_digest(Digest::from_bytes([8; 32])); + let mut builder = FetchOutputTranscriptBuilder::new(input, test_assurance(), &signing_key); + let mut terminal = None; + let mut budget = FetchProjectionBudget::default(); + let mut position = 0; + let (sender, mut receiver) = mpsc::channel(2); + sender + .try_send(Ok(WorkEvent { + kind: Some(work_event::Kind::Chunk(WorkChunk { output_event: None })), + })) + .unwrap(); + let delayed_reader = async { + tokio::time::sleep(Duration::from_secs(2)).await; + assert!(receiver.recv().await.unwrap().unwrap().kind.is_some()); + let event = receiver.recv().await.unwrap().unwrap(); + assert!(matches!( + event.kind, + Some(work_event::Kind::Chunk(WorkChunk { + output_event: Some(_) + })) + )); + }; + let deliver = process_projected_fetch( + vec![ProjectedFetch::Event(b"delayed".to_vec())], + &mut builder, + &mut terminal, + &mut budget, + &mut position, + &sender, + ); + let (result, ()) = tokio::join!(deliver, delayed_reader); + let Ok(reached_terminal) = result else { + panic!("temporary backpressure truncated the response"); + }; + assert!(!reached_terminal); + assert_eq!(position, 7); + assert_eq!(budget.events, 1); +} + #[tokio::test] async fn retained_replay_waits_for_a_consumer_without_losing_events() { let signing_key = key(); diff --git a/crates/gateway/src/archive.rs b/crates/gateway/src/archive.rs index 7c0c0e63..2d000271 100644 --- a/crates/gateway/src/archive.rs +++ b/crates/gateway/src/archive.rs @@ -1,4 +1,7 @@ -use axum::{body::Bytes, http::HeaderMap}; +use axum::{ + body::{Bytes, HttpBody}, + http::HeaderMap, +}; use futures::StreamExt; use serde_json::json; use std::{ @@ -67,6 +70,7 @@ pub(crate) async fn record( parts.uri.path(), parts.method.as_str(), &body, + &parts.headers, ) .await { @@ -99,6 +103,16 @@ pub(crate) async fn record( .headers .insert("x-hellas-request-id", id.parse().unwrap()); } + if body.is_end_stream() { + if archive.finish().await.is_err() { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "response archive unavailable", + ) + .into_response(); + } + return axum::response::Response::from_parts(parts, body); + } let mut source = body.into_data_stream(); let stream: futures::stream::BoxStream<'static, Result> = Box::pin(async_stream::try_stream! { @@ -131,6 +145,7 @@ impl Exchange { path: &str, method: &str, body: &Bytes, + headers: &HeaderMap, ) -> io::Result { let root = root.to_owned(); let body = body.clone(); @@ -138,6 +153,8 @@ impl Exchange { "version": 1, "path": path, "method": method, "complete": false, "started_unix_ms": SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64, "request_bytes": body.len(), + "request_content_type": headers.get("content-type").and_then(|value| value.to_str().ok()), + "request_content_encoding": headers.get("content-encoding").and_then(|value| value.to_str().ok()), }); #[cfg(feature = "otel")] { @@ -178,6 +195,11 @@ impl Exchange { .get("content-type") .and_then(|value| value.to_str().ok()) ); + self.metadata["content_encoding"] = json!( + headers + .get("content-encoding") + .and_then(|value| value.to_str().ok()) + ); self.save().await } diff --git a/crates/gateway/src/http_fetch.rs b/crates/gateway/src/http_fetch.rs index f083003e..9024a77c 100644 --- a/crates/gateway/src/http_fetch.rs +++ b/crates/gateway/src/http_fetch.rs @@ -50,32 +50,40 @@ pub struct HttpRoute { pub path: String, pub method: String, pub url: String, - pub credential: String, + pub credential: Option, + #[serde(default = "public_tls")] + pub tls: HttpTls, #[serde(default)] pub headers: Vec<(String, String)>, - /// Only these caller headers cross the gateway; credentials never do. - #[serde(default)] - pub forward_headers: Vec, +} + +fn public_tls() -> HttpTls { + HttpTls { + roots: HttpTrustRoots::WebPki, + spki_sha256: vec![], + } } struct Account { slots: Arc, - retry_at: Mutex, + backoff: Mutex>, } impl Account { - fn delay(&self) -> Duration { - self.retry_at - .lock() - .unwrap() - .saturating_duration_since(Instant::now()) + fn cooldown(&self) -> Option<(u16, Duration)> { + let (until, status) = (*self.backoff.lock().unwrap())?; + let delay = until.saturating_duration_since(Instant::now()); + (!delay.is_zero()).then_some((status, delay)) } fn observe(&self, status: u16, headers: &HeaderMap) { if status == 429 || (status >= 500 && headers.contains_key("retry-after")) { let delay = retry_delay(headers).min(Duration::from_secs(u32::MAX as u64)); - let mut retry_at = self.retry_at.lock().unwrap(); - *retry_at = (*retry_at).max(Instant::now() + delay); + let until = Instant::now() + delay; + let mut backoff = self.backoff.lock().unwrap(); + if backoff.is_none_or(|(previous, _)| until > previous) { + *backoff = Some((until, status)); + } } } } @@ -110,17 +118,6 @@ impl HttpGatewayConfig { paths.insert((&route.path, &route.method)), "duplicate HTTP route" ); - ensure!( - route.forward_headers.iter().all(|name| matches!( - name.as_str(), - "content-type" - | "accept" - | "anthropic-version" - | "anthropic-beta" - | "user-agent" - )), - "unsupported forwarded header" - ); ensure!( route.headers.iter().all(|(name, _)| !matches!( name.as_str(), @@ -128,36 +125,68 @@ impl HttpGatewayConfig { )), "use a provider credential alias for authentication" ); - route.request(Bytes::new(), &HeaderMap::new()).validate()?; + route.request(Bytes::new(), &HeaderMap::new())?.validate()?; } Ok(()) } } impl HttpRoute { - fn request(&self, body: Bytes, incoming: &HeaderMap) -> HttpFetchRequest { + fn account(&self) -> String { + match &self.credential { + Some(alias) => format!("credential:{alias}"), + None => format!( + "origin:{}", + self.url + .parse::() + .expect("validated URL") + .origin() + .ascii_serialization() + ), + } + } + + fn request(&self, body: Bytes, incoming: &HeaderMap) -> anyhow::Result { let mut headers = self.headers.clone(); - for name in &self.forward_headers { - if !headers.iter().any(|(configured, _)| configured == name) { - for value in incoming.get_all(name) { - if let Ok(value) = value.to_str() { - headers.push((name.clone(), value.into())); - } - } + let connection = connection_headers( + incoming + .iter() + .map(|(name, value)| (name.as_str(), value.to_str().unwrap_or_default())), + ); + for (name, value) in incoming { + let name = name.as_str(); + if !hop_header(name) + && !connection.iter().any(|token| token == name) + && !matches!( + name, + "host" + | "content-length" + | "authorization" + | "x-api-key" + | "api-key" + | "x-goog-api-key" + | "cookie" + | "forwarded" + ) + && !name.starts_with("x-hellas-") + && !name.starts_with("x-forwarded-") + && !self + .headers + .iter() + .any(|(configured, _)| configured == name) + { + headers.push((name.into(), value.to_str()?.into())); } } - HttpFetchRequest { + Ok(HttpFetchRequest { url: self.url.clone(), method: self.method.clone(), headers, body_base64: STANDARD.encode(body), - tls: HttpTls { - roots: HttpTrustRoots::WebPki, - spki_sha256: vec![], - }, - credential: Some(self.credential.clone()), + tls: self.tls.clone(), + credential: self.credential.clone(), max_response_bytes: hellas_rpc::http_fetch::MAX_HTTP_RESPONSE_BYTES, - } + }) } } @@ -193,10 +222,10 @@ pub(super) async fn start(options: GatewayOptions) -> anyhow::Result>, request: Request) -> Respon }) else { return error(StatusCode::NOT_FOUND, "no configured HTTP route"); }; - if request.uri().query().is_some() { - return error( - StatusCode::BAD_REQUEST, - "query parameters are not supported on this route", - ); - } let mut observed = observation::Observation::new(&state.metrics, &route.path); - let account = state.accounts[&route.credential].clone(); - let delay = account.delay(); - if !delay.is_zero() { - observed.status(429); + let account = state.accounts[&route.account()].clone(); + if let Some((status, delay)) = account.cooldown() { + observed.status(status); observed.complete(); - return limited(StatusCode::TOO_MANY_REQUESTS, delay.as_secs() + 1); + return limited(StatusCode::from_u16(status).unwrap(), delay.as_secs() + 1); } let Ok(permit) = account.slots.clone().try_acquire_owned() else { observed.status(503); @@ -314,7 +336,20 @@ async fn handle(State(state): State>, request: Request) -> Respon ); } }; - let upstream = route.request(body, &parts.headers); + let mut upstream = match route.request(body, &parts.headers) { + Ok(upstream) => upstream, + Err(_) => { + observed.status(400); + observed.complete(); + return error(StatusCode::BAD_REQUEST, "unsupported HTTP header encoding"); + } + }; + if let Some(query) = parts.uri.query() { + upstream + .url + .push(if upstream.url.contains('?') { '&' } else { '?' }); + upstream.url.push_str(query); + } let payload = match serde_json::to_vec(&upstream) { Ok(payload) if upstream.validate().is_ok() => payload, _ => { @@ -334,13 +369,42 @@ async fn handle(State(state): State>, request: Request) -> Respon } }; observed.status(status); - let headers = response_headers(headers); + let representation_length = if parts.method == axum::http::Method::HEAD || status == 304 { + headers + .iter() + .find(|(name, _)| name == "content-length") + .and_then(|(_, value)| HeaderValue::from_str(value).ok()) + } else { + None + }; + let mut headers = response_headers(headers); + if let Some(length) = representation_length { + headers.insert("content-length", length); + } observed.content_type( headers .get("content-type") .and_then(|value| value.to_str().ok()), ); account.observe(status, &headers); + if parts.method == axum::http::Method::HEAD || status == 204 || status == 304 { + let end = events.next().instrument(observed.span.clone()).await; + if !matches!( + end, + Some(Ok(OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None + })) + ) { + observed.status(502); + return error(StatusCode::BAD_GATEWAY, "invalid bodyless HTTP completion"); + } + observed.complete(); + let mut response = Response::new(Body::empty()); + *response.status_mut() = StatusCode::from_u16(status).unwrap(); + *response.headers_mut() = std::mem::take(&mut headers); + return response; + } let stream = async_stream::try_stream! { let _permit = permit; let mut size = 0usize; @@ -420,13 +484,16 @@ async fn open( fn response_headers(headers: Vec<(String, String)>) -> HeaderMap { let mut result = HeaderMap::new(); + let connection = connection_headers( + headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_str())), + ); for (name, value) in headers { - if matches!( - name.as_str(), - "content-type" | "content-encoding" | "retry-after" | "request-id" | "x-request-id" - ) || name.starts_with("x-ratelimit-") - || name.starts_with("ratelimit-") - || name.starts_with("anthropic-ratelimit-") + if !hop_header(&name) + && !connection.contains(&name) + && !matches!(name.as_str(), "content-length" | "set-cookie") + && !name.starts_with("x-hellas-") { if let (Ok(name), Ok(value)) = ( name.parse::(), @@ -439,6 +506,29 @@ fn response_headers(headers: Vec<(String, String)>) -> HeaderMap { result } +fn hop_header(name: &str) -> bool { + matches!( + name, + "connection" + | "keep-alive" + | "proxy-authenticate" + | "proxy-authorization" + | "proxy-connection" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + ) +} + +fn connection_headers<'a>(headers: impl Iterator) -> Vec { + headers + .filter(|(name, _)| *name == "connection") + .flat_map(|(_, value)| value.split(',')) + .map(|name| name.trim().to_ascii_lowercase()) + .collect() +} + fn retry_delay(headers: &HeaderMap) -> Duration { let value = headers .get("retry-after") diff --git a/crates/gateway/src/http_fetch/observation.rs b/crates/gateway/src/http_fetch/observation.rs index 33fe50ce..b9b144a4 100644 --- a/crates/gateway/src/http_fetch/observation.rs +++ b/crates/gateway/src/http_fetch/observation.rs @@ -210,11 +210,11 @@ impl Usage { let Ok(value) = serde_json::from_slice::(bytes) else { return; }; - let Some(usage) = value.get("usage").or_else(|| { - value - .get("message") - .and_then(|message| message.get("usage")) - }) else { + let Some(usage) = value + .get("usage") + .or_else(|| value.pointer("/message/usage")) + .or_else(|| value.pointer("/response/usage")) + else { return; }; fn update(target: &mut Option, value: Option) { @@ -241,6 +241,7 @@ impl Usage { usage .get("cache_read_input_tokens") .or_else(|| usage.pointer("/prompt_tokens_details/cached_tokens")) + .or_else(|| usage.pointer("/input_tokens_details/cached_tokens")) .and_then(Value::as_u64), ); } @@ -283,4 +284,23 @@ mod tests { absent.finish(); assert_eq!(absent.input, None); } + + #[test] + fn responses_completed_usage_survives_fragmented_delivery() { + let wire = b"event: response.completed\ndata: {\"response\":{\"usage\":{\"input_tokens\":12,\"output_tokens\":3,\"input_tokens_details\":{\"cached_tokens\":8}}}}\n\n"; + for size in 1..=wire.len() { + let mut usage = Usage { + sse: true, + ..Default::default() + }; + for chunk in wire.chunks(size) { + usage.push(chunk); + } + usage.finish(); + assert_eq!( + (usage.input, usage.output, usage.cached), + (Some(12), Some(3), Some(8)) + ); + } + } } diff --git a/crates/gateway/src/http_fetch/tests.rs b/crates/gateway/src/http_fetch/tests.rs index 225bbe3d..e330df59 100644 --- a/crates/gateway/src/http_fetch/tests.rs +++ b/crates/gateway/src/http_fetch/tests.rs @@ -4,7 +4,7 @@ use super::*; fn account_backoff_is_shared_and_never_shortened_by_another_response() { let account = Arc::new(Account { slots: Arc::new(Semaphore::new(1)), - retry_at: Mutex::new(Instant::now()), + backoff: Mutex::new(None), }); let other_route = account.clone(); let header = |seconds: &'static str| { @@ -13,14 +13,21 @@ fn account_backoff_is_shared_and_never_shortened_by_another_response() { HeaderValue::from_static(seconds), )]) }; + assert!(account.cooldown().is_none()); account.observe(429, &header("60")); - assert!(other_route.delay() > Duration::from_secs(59)); + assert!(other_route.cooldown().unwrap().1 > Duration::from_secs(59)); other_route.observe(429, &header("1")); - assert!(account.delay() > Duration::from_secs(59)); + assert!(account.cooldown().unwrap().1 > Duration::from_secs(59)); other_route.observe(503, &header("120")); - assert!(account.delay() > Duration::from_secs(119)); + assert_eq!(account.cooldown().unwrap().0, 503); + assert!(account.cooldown().unwrap().1 > Duration::from_secs(119)); + // A short rate limit must not turn a retriable overload into a quota error. + other_route.observe(429, &header("1")); + assert_eq!(account.cooldown().unwrap().0, 503); account.observe(200, &header("600")); - assert!(account.delay() < Duration::from_secs(121)); + assert!(account.cooldown().unwrap().1 < Duration::from_secs(121)); + *account.backoff.lock().unwrap() = Some((Instant::now(), 503)); + assert!(account.cooldown().is_none()); let permit = account.slots.clone().try_acquire_owned().unwrap(); assert!(other_route.slots.clone().try_acquire_owned().is_err()); drop(permit); @@ -33,9 +40,9 @@ fn forward_only_protocol_headers_and_keep_retry_and_quota_metadata() { path: "/v1/messages".into(), method: "POST".into(), url: "https://api.example.com/v1/messages".into(), - credential: "account".into(), + credential: Some("account".into()), + tls: public_tls(), headers: vec![("anthropic-version".into(), "2023-06-01".into())], - forward_headers: vec!["content-type".into(), "anthropic-version".into()], }; let incoming = HeaderMap::from_iter([ ( @@ -55,7 +62,9 @@ fn forward_only_protocol_headers_and_keep_retry_and_quota_metadata() { HeaderValue::from_static("override"), ), ]); - let request = route.request(Bytes::from_static(b"{\"private\":1}"), &incoming); + let request = route + .request(Bytes::from_static(b"{\"private\":1}"), &incoming) + .unwrap(); assert_eq!(request.body().unwrap(), b"{\"private\":1}"); assert_eq!( request.headers, @@ -92,7 +101,7 @@ fn retry_after_supports_dates_and_has_a_nonzero_fallback() { } #[test] -fn credential_and_hop_headers_cannot_be_added_to_the_forward_list() { +fn credentials_and_hop_headers_cannot_be_configured_as_static_headers() { for name in ["authorization", "x-api-key", "cookie", "host", "connection"] { let config = HttpGatewayConfig { service: "http".into(), @@ -102,11 +111,53 @@ fn credential_and_hop_headers_cannot_be_added_to_the_forward_list() { path: "/v1/messages".into(), method: "POST".into(), url: "https://example.com/v1/messages".into(), - credential: "account".into(), - headers: vec![], - forward_headers: vec![name.into()], + credential: Some("account".into()), + tls: public_tls(), + headers: vec![(name.into(), "secret".into())], }], }; assert!(config.validate().is_err()); } } + +#[test] +fn extension_headers_and_duplicates_survive_but_connection_tokens_do_not() { + let route: HttpRoute = serde_json::from_value(serde_json::json!({ + "path":"/v1/chat/completions", "method":"POST", "url":"https://example.com/v1/chat/completions" + })).unwrap(); + let mut headers = HeaderMap::new(); + for (name, value) in [ + ("idempotency-key", "key"), + ("x-stainless-retry-count", "0"), + ("content-encoding", "gzip"), + ("connection", "X-Private-Hop"), + ("x-private-hop", "secret"), + ("x-beta", "one"), + ("x-beta", "two"), + ("x-hellas-zdr", "true"), + ("cookie", "secret"), + ] { + headers.append(name, value.parse().unwrap()); + } + let request = route.request(Bytes::new(), &headers).unwrap(); + assert_eq!(request.headers.len(), 5); + assert_eq!( + request + .headers + .iter() + .filter(|(n, _)| n == "x-beta") + .count(), + 2 + ); + assert!(!request.headers.iter().any(|(name, _)| { + ["connection", "x-private-hop", "cookie", "x-hellas-zdr"].contains(&name.as_str()) + })); + let response = response_headers(vec![ + ("connection".into(), "X-Private-Hop".into()), + ("x-private-hop".into(), "secret".into()), + ("location".into(), "/next".into()), + ("etag".into(), "v1".into()), + ]); + assert_eq!(response.len(), 2); + assert_eq!(response["location"], "/next"); +} diff --git a/crates/rpc/src/call.rs b/crates/rpc/src/call.rs index 751746b2..7ebb0de9 100644 --- a/crates/rpc/src/call.rs +++ b/crates/rpc/src/call.rs @@ -988,7 +988,22 @@ async fn write_streaming_response( mut responses: impl futures_util::Stream> + Unpin, call: &CallSpan, ) -> Result<(), TransportError> { - while let Some(response) = responses.next().await { + loop { + let next = match futures_util::future::select( + Box::pin(responses.next()), + Box::pin(send.stopped()), + ) + .await + { + futures_util::future::Either::Left((next, _)) => next, + futures_util::future::Either::Right(_) => { + call.finish(WireCode::Cancelled); + return Err(TransportError::Io("response consumer disconnected".into())); + } + }; + let Some(response) = next else { + break; + }; let response = match response { Ok(response) => response, Err(status) => return write_trailer(send, status.into(), call).await, @@ -1281,6 +1296,35 @@ mod streaming_call_tests { x: u32, } + #[tokio::test] + async fn stopped_consumer_cancels_an_idle_response_stream() { + struct StoppedSend; + impl SendHalf for StoppedSend { + type Error = std::io::Error; + async fn send_body(&mut self, _: Bytes) -> Result<(), Self::Error> { + panic!("no response available"); + } + async fn close_send(&mut self, _: Option) -> Result<(), Self::Error> { + panic!("cancelled transport"); + } + fn reset(&mut self, _: WireCode) {} + async fn stopped(&mut self) {} + } + let mut send = StoppedSend; + let call = CallSpan::new(tracing::Span::none()); + let result = tokio::time::timeout( + std::time::Duration::from_secs(1), + write_streaming_response( + &mut send, + futures_util::stream::pending::>(), + &call, + ), + ) + .await + .expect("idle cancellation must not wait for another response"); + assert!(result.is_err()); + } + #[derive(Clone, PartialEq, ::prost::Message)] struct BytesMsg { #[prost(bytes = "vec", tag = "1")] diff --git a/crates/wire/src/iroh/stream.rs b/crates/wire/src/iroh/stream.rs index 866f86c4..c8b955f1 100644 --- a/crates/wire/src/iroh/stream.rs +++ b/crates/wire/src/iroh/stream.rs @@ -208,6 +208,12 @@ impl Drop for IrohSendHalf { impl crate::transport::SendHalf for IrohSendHalf { type Error = std::io::Error; + async fn stopped(&mut self) { + if let Some(send) = self.send.as_mut() { + let _ = send.stopped().await; + } + } + async fn send_body(&mut self, payload: Bytes) -> Result<(), Self::Error> { self.write_framed(Frame::Body(payload)).await } diff --git a/crates/wire/src/transport.rs b/crates/wire/src/transport.rs index 6ffbef1e..5bb60394 100644 --- a/crates/wire/src/transport.rs +++ b/crates/wire/src/transport.rs @@ -163,6 +163,12 @@ pub trait SendHalf: Send { /// Cancel both directions of the underlying stream. Idempotent. fn reset(&mut self, code: WireCode); + + /// Wait until the peer stops receiving. Carriers without a cancellation + /// notification retain the default pending future and detect write errors. + fn stopped(&mut self) -> impl Future + Send { + std::future::pending() + } } /// Recv half. Yields body chunks; trailer is available after the diff --git a/docs/http-gateway.md b/docs/http-gateway.md index 1c1f3d05..c4e0bc5c 100644 --- a/docs/http-gateway.md +++ b/docs/http-gateway.md @@ -17,8 +17,7 @@ Example gateway configuration for Kimi Code: "path": "/v1/chat/completions", "method": "POST", "url": "https://api.kimi.com/coding/v1/chat/completions", - "credential": "kimi", - "forward_headers": ["content-type", "accept", "user-agent"] + "credential": "kimi" }] } ``` @@ -26,9 +25,19 @@ Example gateway configuration for Kimi Code: The service and method must match the provider's configured Fetch route. The credential alias belongs to that provider. Each alias shares a concurrency limit and cooldown across its HTTP routes. Paths and methods match exactly; -query parameters are rejected. The forward list only accepts protocol headers; -the caller's gateway bearer never becomes an upstream credential. Optional +query parameters retain their order, repeats and percent encoding. Ordinary +client headers, including idempotency keys and vendor extensions, pass through. +Connection-specific headers, credentials, cookies and gateway control headers +are removed; the caller's gateway bearer never becomes an upstream credential. Optional `headers` holds operator-supplied `[name, value]` pairs, with lowercase names. +Configured values override the corresponding client header. The former +`forward_headers` whitelist has been removed. + +Omit `credential` for an unauthenticated upstream. Such routes share admission +by origin. Optional `tls` uses the [Fetch TLS vocabulary](../crates/providers/HTTPS.md) +for exact trust anchors and pins, defaulting to WebPKI. Custom roots cannot be +combined with a provider credential; that restriction remains enforced by the +provider. Private destinations also require its explicit egress opt-in. ```sh hellas-cli --identity ./gateway.identity gateway \ @@ -54,7 +63,7 @@ CLI gateways archive authenticated requests and responses by default under `~/.hellas/gateway-archive`, or `--archive-dir DIRECTORY`. This applies to the existing inference routes as well as HTTP Fetch. Each exchange has owner-only `request.bin`, `response.bin` and `metadata.json` files. Metadata records status, -size, elapsed time, completion and trace context; it excludes authentication +size, content type/encoding, elapsed time, completion and trace context; it excludes authentication headers. `x-hellas-request-id` identifies the exchange. Failed or cancelled streams retain an incomplete archive. The gateway refuses requests when it cannot open their archive, and stops streaming on an archive write failure. @@ -64,7 +73,9 @@ There is no automatic archive pruning. `--zdr` enforces this for all requests. Ambiguous flags, `store: true`, and ZDR with inference caching enabled are rejected before archive writes. A non-ZDR request may be archived even when its upstream `store` is false. Unauthorized, -oversized and invalid ZDR requests are rejected without payload archives. +invalid ZDR requests and bodies exceeding the 2 MiB ingress limit are rejected +without payload archives. Non-ZDR requests that pass ingress but exceed the +smaller Fetch envelope limit are archived with their 413 response. These switches govern this gateway's application writes. They do not establish OS swap/dump protection, a remote provider's host policy, upstream retention, @@ -78,13 +89,19 @@ HTTP input is limited to 1,523,712 bytes, reserving envelope space inside the 2 MiB signed Fetch request. Responses are limited to 8 MiB and the Fetch stream to 32,768 events and 16 MiB of signed payload. The provider has a 20 minute total HTTP deadline and a 90 second response idle timeout. Exceeding a stream limit -closes it as incomplete. Individual upstream deliveries are forwarded promptly, +closes it as incomplete. A slow response consumer applies bounded backpressure, +with a 90 second drain timeout. QUIC cancellation stops an idle upstream without +waiting for its next byte. Individual upstream deliveries are forwarded promptly, split at 16 KiB; the provider does not wait for a full buffer. -No generation is automatically retried upstream. Upstream status, `Retry-After`, -request IDs and rate-limit headers reach the client. A 429, or a 5xx carrying -`Retry-After`, starts a shared account cooldown; delta seconds and HTTP dates -are accepted. Missing or invalid delays on a 429 use one second. Excess +No generation is automatically retried upstream. Upstream status and end-to-end +response headers reach the client, including Location, ETag, `Retry-After`, +request IDs and rate limits. Hop-by-hop headers and cookies are removed. Body +framing is regenerated; representation Content-Length is retained for HEAD/304. +A 429, or a 5xx carrying `Retry-After`, starts a shared account cooldown; +delta seconds and HTTP dates are accepted. Requests during cooldown receive +the status that established the longest remaining delay: a 503 remains a 503, +so clients can keep retrying overloads. Missing or invalid delays on a 429 use one second. Excess concurrency returns 503 with `Retry-After: 1`. With `otel`, traces connect HTTP ingress, Fetch RPCs, credential refresh and @@ -92,5 +109,14 @@ upstream HTTP. Span attributes exclude request/response bodies and credentials. `hellas.gateway.http.requests`, `.duration`, `.time_to_first_byte` and `.tokens` record status, completion, timings and standard OpenAI/Anthropic usage fields. Usage is unknown when upstream omits it; cached tokens are reported separately. +Bodies remain encoded on the wire and in archives; usage extraction currently +requires an uncompressed JSON/SSE response. Account quota windows still come from the upstream quota exporter. Request token totals alone cannot determine subscription quota or billing. + +This is an HTTP API bridge with explicit routes, authentication and resource +limits. It does not implement CONNECT, WebSocket upgrades, streaming uploads, +HTTP trailers or automatic rewriting of redirect URLs. Clients using Responses +WebSockets must select their HTTP/SSE transport. Archive unavailability currently +fails requests closed. These are contract differences from a general transparent +HTTP proxy, even when inference and tool-call payloads are preserved exactly. From dac1a9dbec70359c55a48a95d36b11d359e92cd9 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Thu, 24 Sep 2026 17:11:14 +0200 Subject: [PATCH 06/32] Keep serving when gateway archives fail and report failures --- Cargo.lock | 1 + crates/gateway/Cargo.toml | 1 + crates/gateway/src/archive.rs | 100 ++++++++--- crates/gateway/src/archive/tests.rs | 257 +++++++++++++++++++++++++++- crates/gateway/src/http_fetch.rs | 10 +- crates/gateway/src/lib.rs | 13 +- docs/http-gateway.md | 25 ++- 7 files changed, 361 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c115dabd..e8f8ea13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2869,6 +2869,7 @@ dependencies = [ "iroh", "libc", "opentelemetry", + "opentelemetry_sdk", "prometheus-client", "rand 0.10.2", "reqwest", diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index c7769936..eadc14f2 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -55,4 +55,5 @@ tracing-opentelemetry = { workspace = true, optional = true } [dev-dependencies] hellas-rpc = { workspace = true, features = ["host-control"] } hellas-wire = { workspace = true, features = ["unix"] } +opentelemetry_sdk = { workspace = true, features = ["testing"] } tempfile = "3" diff --git a/crates/gateway/src/archive.rs b/crates/gateway/src/archive.rs index 2d000271..89f6fe23 100644 --- a/crates/gateway/src/archive.rs +++ b/crates/gateway/src/archive.rs @@ -23,6 +23,49 @@ pub struct ArchiveOptions { pub(crate) struct Policy { pub options: ArchiveOptions, pub cache_enabled: bool, + #[cfg(feature = "otel")] + failures: opentelemetry::metrics::Counter, +} + +impl Policy { + pub(crate) fn new(options: ArchiveOptions, cache_enabled: bool) -> Self { + Self { + options, + cache_enabled, + #[cfg(feature = "otel")] + failures: opentelemetry::global::meter("hellas.gateway.archive") + .u64_counter("hellas.gateway.archive.failures") + .build(), + } + } + + pub(crate) fn prepare(&self) { + if !self.options.zdr + && let Err(error) = prepare(&self.options.directory) + { + self.failed("prepare", &error); + } + } + + fn failed(&self, stage: &'static str, error: &io::Error) { + // Filesystem error strings can contain paths. Report only bounded + // classifications, never request bodies, credentials or raw errors. + warn!( + target: "hellas_archive", + archive_stage = stage, + error_kind = ?error.kind(), + error_os_code = error.raw_os_error(), + "archive failed; continuing response delivery" + ); + #[cfg(feature = "otel")] + self.failures.add( + 1, + &[ + opentelemetry::KeyValue::new("archive.stage", stage), + opentelemetry::KeyValue::new("error.type", format!("{:?}", error.kind())), + ], + ); + } } pub(crate) fn zdr(headers: &HeaderMap, required: bool) -> Result { @@ -65,7 +108,7 @@ pub(crate) async fn record( .run(axum::extract::Request::from_parts(parts, Body::from(body))) .await; } - let mut archive = match Exchange::new( + let archive = match Exchange::new( &policy.options.directory, parts.uri.path(), parts.method.as_str(), @@ -74,28 +117,33 @@ pub(crate) async fn record( ) .await { - Ok(archive) => archive, - Err(_) => { - return ( - StatusCode::SERVICE_UNAVAILABLE, - "request archive unavailable", - ) - .into_response(); + Ok(archive) => Some(archive), + Err(error) => { + policy.failed("request", &error); + None } }; let response = next .run(axum::extract::Request::from_parts(parts, Body::from(body))) .await; - if archive + match archive { + Some(archive) => archive_response(policy, archive, response).await, + None => response, + } +} + +async fn archive_response( + policy: Policy, + mut archive: Exchange, + response: axum::response::Response, +) -> axum::response::Response { + use axum::body::Body; + if let Err(error) = archive .head(response.status().as_u16(), response.headers()) .await - .is_err() { - return ( - StatusCode::SERVICE_UNAVAILABLE, - "response archive unavailable", - ) - .into_response(); + policy.failed("response_head", &error); + return response; } let (mut parts, body) = response.into_parts(); if let Some(id) = archive.directory.file_name().and_then(|id| id.to_str()) { @@ -104,24 +152,30 @@ pub(crate) async fn record( .insert("x-hellas-request-id", id.parse().unwrap()); } if body.is_end_stream() { - if archive.finish().await.is_err() { - return ( - StatusCode::SERVICE_UNAVAILABLE, - "response archive unavailable", - ) - .into_response(); + if let Err(error) = archive.finish().await { + policy.failed("finish", &error); } return axum::response::Response::from_parts(parts, body); } let mut source = body.into_data_stream(); + let mut archive = Some(archive); let stream: futures::stream::BoxStream<'static, Result> = Box::pin(async_stream::try_stream! { while let Some(bytes) = source.next().await { let bytes = bytes.map_err(|_| io::Error::other("response stream failed"))?; - archive.chunk(&bytes).await?; + if let Some(exchange) = archive.as_mut() + && let Err(error) = exchange.chunk(&bytes).await + { + policy.failed("response_body", &error); + archive = None; + } yield bytes; } - archive.finish().await?; + if let Some(mut exchange) = archive + && let Err(error) = exchange.finish().await + { + policy.failed("finish", &error); + } }); axum::response::Response::from_parts(parts, Body::from_stream(stream)) } diff --git a/crates/gateway/src/archive/tests.rs b/crates/gateway/src/archive/tests.rs index e3976ca2..7c2f107a 100644 --- a/crates/gateway/src/archive/tests.rs +++ b/crates/gateway/src/archive/tests.rs @@ -7,6 +7,32 @@ use axum::{ }; use tower::ServiceExt; +fn policy(directory: &Path) -> Policy { + Policy::new( + ArchiveOptions { + directory: directory.into(), + zdr: false, + }, + false, + ) +} + +async fn exchange(directory: &Path) -> Exchange { + Exchange::new( + directory, + "/v1/test", + "POST", + &Bytes::from_static(b"synthetic request"), + &HeaderMap::new(), + ) + .await + .unwrap() +} + +fn metadata(directory: &Path) -> serde_json::Value { + serde_json::from_slice(&std::fs::read(directory.join("metadata.json")).unwrap()).unwrap() +} + fn router(directory: &Path, required: bool, cache_enabled: bool) -> Router { Router::new() .route( @@ -14,13 +40,13 @@ fn router(directory: &Path, required: bool, cache_enabled: bool) -> Router { post(|body: Bytes| async move { (StatusCode::TOO_MANY_REQUESTS, body) }), ) .layer(axum::middleware::from_fn_with_state( - Policy { - options: ArchiveOptions { + Policy::new( + ArchiveOptions { directory: directory.into(), zdr: required, }, cache_enabled, - }, + ), record, )) } @@ -148,3 +174,228 @@ async fn zdr_rejects_ambiguous_flags_retain_and_enabled_replay_before_disk_write ); assert!(!path.exists()); } + +#[tokio::test] +async fn unavailable_archive_preserves_errors_and_recovers_on_the_next_request() { + let root = tempfile::tempdir().unwrap(); + let archive = root.path().join("unavailable"); + std::fs::write(&archive, b"not a directory").unwrap(); + policy(&archive).prepare(); // Startup must also remain available. + let app = router(&archive, false, false); + for recovered in [false, true] { + if recovered { + std::fs::remove_file(&archive).unwrap(); + } + let response = app + .clone() + .oneshot( + Request::post("/v1/test") + .body(Body::from("unchanged error")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); + let id = response.headers().get("x-hellas-request-id").cloned(); + assert_eq!(id.is_some(), recovered); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "unchanged error" + ); + if let Some(id) = id { + assert_eq!( + metadata(&archive.join(id.to_str().unwrap()))["complete"], + true + ); + } + } +} + +#[tokio::test] +async fn response_head_archive_failure_preserves_status_headers_and_body() { + let root = tempfile::tempdir().unwrap(); + let archive = exchange(root.path()).await; + let moved = root.path().join("moved"); + std::fs::rename(&archive.directory, &moved).unwrap(); + std::fs::write(&archive.directory, b"unavailable").unwrap(); + let response = axum::response::Response::builder() + .status(429) + .header("retry-after", "3") + .body(Body::from("upstream error")) + .unwrap(); + let response = archive_response(policy(root.path()), archive, response).await; + assert_eq!(response.status(), 429); + assert_eq!(response.headers()["retry-after"], "3"); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "upstream error" + ); + assert_eq!(metadata(&moved)["complete"], false); +} + +#[cfg(target_os = "linux")] +#[tokio::test] +async fn disk_full_does_not_truncate_a_stream_or_replace_an_empty_response() { + let root = tempfile::tempdir().unwrap(); + for empty in [false, true] { + let mut archive = exchange(root.path()).await; + let directory = archive.directory.clone(); + // /dev/full injects ENOSPC on writes without filling the host filesystem. + archive.response = tokio::fs::OpenOptions::new() + .write(true) + .open("/dev/full") + .await + .unwrap(); + let body = if empty { + Body::empty() + } else { + Body::from_stream(futures::stream::iter( + ["first", "second", "third"] + .map(|s| Ok::<_, io::Error>(Bytes::from_static(s.as_bytes()))), + )) + }; + let response = axum::response::Response::builder() + .status(if empty { 204 } else { 200 }) + .body(body) + .unwrap(); + let response = archive_response(policy(root.path()), archive, response).await; + assert_eq!(response.status(), if empty { 204 } else { 200 }); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + if empty { "" } else { "firstsecondthird" } + ); + assert_eq!(metadata(&directory)["complete"], false); + } +} + +#[tokio::test] +async fn final_metadata_failure_keeps_all_delivered_bytes() { + let root = tempfile::tempdir().unwrap(); + let archive = exchange(root.path()).await; + let directory = archive.directory.clone(); + let moved = root.path().join("moved"); + let moved_in_stream = moved.clone(); + let body: futures::stream::BoxStream<'static, Result> = + Box::pin(async_stream::try_stream! { + yield Bytes::from_static(b"first"); + std::fs::rename(&directory, &moved_in_stream)?; + std::fs::write(&directory, b"unavailable")?; + yield Bytes::from_static(b"second"); + }); + let response = axum::response::Response::new(Body::from_stream(body)); + let response = archive_response(policy(root.path()), archive, response).await; + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "firstsecond" + ); + assert_eq!( + std::fs::read(moved.join("response.bin")).unwrap(), + b"firstsecond" + ); + assert_eq!(metadata(&moved)["complete"], false); +} + +#[tokio::test] +async fn genuine_upstream_stream_errors_still_fail() { + let root = tempfile::tempdir().unwrap(); + let archive = exchange(root.path()).await; + let directory = archive.directory.clone(); + let body = Body::from_stream(futures::stream::iter([ + Ok(Bytes::from_static(b"partial")), + Err(io::Error::other("synthetic upstream failure")), + ])); + let response = archive_response( + policy(root.path()), + archive, + axum::response::Response::new(body), + ) + .await; + let mut stream = response.into_body().into_data_stream(); + assert_eq!(stream.next().await.unwrap().unwrap(), "partial"); + assert!(stream.next().await.unwrap().is_err()); + assert_eq!(metadata(&directory)["complete"], false); +} + +#[cfg(feature = "otel")] +#[tokio::test] +async fn archive_failures_are_counted_without_changing_http_outcomes_or_counting_zdr() { + use opentelemetry::metrics::MeterProvider; + use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; + use opentelemetry_sdk::metrics::{InMemoryMetricExporter, SdkMeterProvider}; + + let exporter = InMemoryMetricExporter::default(); + let provider = SdkMeterProvider::builder() + .with_periodic_exporter(exporter.clone()) + .build(); + let root = tempfile::tempdir().unwrap(); + let unavailable = root.path().join("unavailable"); + std::fs::write(&unavailable, b"not a directory").unwrap(); + let mut policy = policy(&unavailable); + policy.failures = provider + .meter("archive-test") + .u64_counter("hellas.gateway.archive.failures") + .build(); + policy.prepare(); + let app = Router::new() + .route("/", post(|| async { "success" })) + .layer(axum::middleware::from_fn_with_state(policy, record)); + for zdr in [false, true] { + let response = app + .clone() + .oneshot( + Request::post("/") + .header("x-hellas-zdr", zdr.to_string()) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), 200); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "success" + ); + } + provider.force_flush().unwrap(); + let mut stages = Vec::new(); + for resource in exporter.get_finished_metrics().unwrap() { + for scope in resource.scope_metrics() { + for metric in scope.metrics() { + assert_eq!(metric.name(), "hellas.gateway.archive.failures"); + let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else { + panic!("failure counter must be a sum"); + }; + for point in sum.data_points() { + assert_eq!(point.value(), 1); + let attributes: Vec<_> = point.attributes().collect(); + assert_eq!(attributes.len(), 2); + stages.push( + attributes + .iter() + .find(|a| a.key.as_str() == "archive.stage") + .unwrap() + .value + .to_string(), + ); + assert!( + attributes + .iter() + .all(|a| !a.value.to_string().contains("unavailable")) + ); + } + } + } + } + stages.sort(); + assert_eq!(stages, ["prepare", "request"]); +} diff --git a/crates/gateway/src/http_fetch.rs b/crates/gateway/src/http_fetch.rs index 9024a77c..926b0e28 100644 --- a/crates/gateway/src/http_fetch.rs +++ b/crates/gateway/src/http_fetch.rs @@ -205,9 +205,8 @@ pub(super) async fn start(options: GatewayOptions) -> anyhow::Result anyhow::Result anyhow::Result ) .await?; let state = Arc::new(GatewayState::from_options(&options).await?); - if !options.archive.zdr { - archive::prepare(&options.archive.directory)?; - } + let archive_policy = archive::Policy::new( + options.archive.clone(), + options.output_cache.policy != cache::CachePolicy::Off, + ); + archive_policy.prepare(); // Every route below reaches an executor, so every route below is // behind this run's credential. The layer goes on last, which in axum @@ -265,10 +267,7 @@ async fn start_gateway(options: GatewayOptions) -> anyhow::Result .with_state(state.clone()) .layer(provenance_layer::ProvenanceLayer) .layer(axum::middleware::from_fn_with_state( - archive::Policy { - options: options.archive.clone(), - cache_enabled: options.output_cache.policy != cache::CachePolicy::Off, - }, + archive_policy, archive::record, )); #[cfg(feature = "otel")] diff --git a/docs/http-gateway.md b/docs/http-gateway.md index c4e0bc5c..fd3f8526 100644 --- a/docs/http-gateway.md +++ b/docs/http-gateway.md @@ -65,9 +65,21 @@ existing inference routes as well as HTTP Fetch. Each exchange has owner-only `request.bin`, `response.bin` and `metadata.json` files. Metadata records status, size, content type/encoding, elapsed time, completion and trace context; it excludes authentication headers. `x-hellas-request-id` identifies the exchange. Failed or cancelled -streams retain an incomplete archive. The gateway refuses requests when it -cannot open their archive, and stops streaming on an archive write failure. -There is no automatic archive pruning. +streams retain an incomplete archive. Archiving is best-effort: failures during +setup, request/response writes or finalization are reported without replacing +the upstream status or interrupting its response. After a write fails, archiving +stops for that exchange; the next ordinary request attempts a fresh archive. +`x-hellas-request-id` identifies an archive attempt, not a guarantee of a complete +durable record. There is no automatic archive pruning. + +Archive failures emit a `hellas_archive` warning with the operation, I/O error +kind and OS error code, without payloads, credentials, paths or raw error text. +With `otel`, `hellas.gateway.archive.failures` counts failures by `archive.stage` +(`prepare`, `request`, `response_head`, `response_body`, `finish`) and `error.type`. +The Prometheus name is `hellas_gateway_archive_failures_total`. Successful HTTP +responses remain successful in request telemetry even when their archive fails; +monitor the archive counter separately. ZDR requests do not attempt archival or +increment this counter. `x-hellas-zdr: true` disables application payload persistence for that request. `--zdr` enforces this for all requests. Ambiguous flags, `store: true`, and ZDR @@ -117,6 +129,7 @@ token totals alone cannot determine subscription quota or billing. This is an HTTP API bridge with explicit routes, authentication and resource limits. It does not implement CONNECT, WebSocket upgrades, streaming uploads, HTTP trailers or automatic rewriting of redirect URLs. Clients using Responses -WebSockets must select their HTTP/SSE transport. Archive unavailability currently -fails requests closed. These are contract differences from a general transparent -HTTP proxy, even when inference and tool-call payloads are preserved exactly. +WebSockets must select their HTTP/SSE transport. Archive I/O remains on the +response path, although its errors no longer fail requests. These are contract +differences from a general transparent HTTP proxy, even when inference and +tool-call payloads are preserved exactly. From 7643c6f4ee405059b4c6266b1277ff6debb1136a Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Thu, 24 Sep 2026 18:45:55 +0200 Subject: [PATCH 07/32] Fix native credentials, streaming usage, and pooled request traces --- Cargo.lock | 50 ++++++ crates/gateway/Cargo.toml | 1 + crates/gateway/src/http_fetch.rs | 5 +- crates/gateway/src/http_fetch/observation.rs | 175 ++++++++++++++++--- crates/providers/HTTPS.md | 4 + crates/providers/src/http/config.rs | 62 ++++++- crates/wire/Cargo.toml | 1 + crates/wire/src/iroh/pool.rs | 64 ++++++- docs/http-gateway.md | 13 +- 9 files changed, 345 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e8f8ea13..e32ec49c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aead" version = "0.5.2" @@ -1435,6 +1441,15 @@ dependencies = [ "spin 0.10.1", ] +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if 1.0.4", +] + [[package]] name = "criterion" version = "0.8.2" @@ -2215,6 +2230,17 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "fluent-uri" version = "0.4.1" @@ -2856,6 +2882,7 @@ dependencies = [ "async-stream", "axum", "base64 0.22.1", + "flate2", "futures", "hellas-adaptors", "hellas-client", @@ -3088,6 +3115,7 @@ dependencies = [ "tokio-stream", "tokio-tungstenite", "tracing", + "tracing-subscriber", "wasm-bindgen-futures", "web-time", "windows-sys 0.61.2", @@ -4328,6 +4356,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.1" @@ -6608,6 +6646,12 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "simd_cesu8" version = "1.1.1" @@ -8322,6 +8366,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.21" diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index eadc14f2..05564cce 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -28,6 +28,7 @@ async-stream = { workspace = true } axum = { workspace = true } base64.workspace = true httpdate = "1" +flate2 = "1" futures = { workspace = true } hellas-adaptors.workspace = true hellas-client = { workspace = true, features = ["iroh", "evaluate"] } diff --git a/crates/gateway/src/http_fetch.rs b/crates/gateway/src/http_fetch.rs index 926b0e28..26b53fc8 100644 --- a/crates/gateway/src/http_fetch.rs +++ b/crates/gateway/src/http_fetch.rs @@ -377,10 +377,13 @@ async fn handle(State(state): State>, request: Request) -> Respon if let Some(length) = representation_length { headers.insert("content-length", length); } - observed.content_type( + observed.content( headers .get("content-type") .and_then(|value| value.to_str().ok()), + headers + .get("content-encoding") + .and_then(|value| value.to_str().ok()), ); account.observe(status, &headers); if parts.method == axum::http::Method::HEAD || status == 204 || status == 304 { diff --git a/crates/gateway/src/http_fetch/observation.rs b/crates/gateway/src/http_fetch/observation.rs index b9b144a4..cdb6ece7 100644 --- a/crates/gateway/src/http_fetch/observation.rs +++ b/crates/gateway/src/http_fetch/observation.rs @@ -1,5 +1,5 @@ use serde_json::Value; -use std::time::Instant; +use std::{io::Write, time::Instant}; pub(super) struct Metrics { #[cfg(feature = "otel")] @@ -53,6 +53,7 @@ pub(super) struct Observation { bytes: u64, first_byte: Option, usage: Usage, + compressed_usage: Option>, #[cfg(feature = "otel")] route: String, #[cfg(feature = "otel")] @@ -71,6 +72,7 @@ impl Observation { gen_ai.usage.input_tokens = tracing::field::Empty, gen_ai.usage.output_tokens = tracing::field::Empty, gen_ai.usage.cache_read.input_tokens = tracing::field::Empty, + gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty, hellas.response.time_to_first_byte = tracing::field::Empty, hellas.response.complete = tracing::field::Empty, error.type = tracing::field::Empty, otel.status_code = tracing::field::Empty), @@ -80,6 +82,7 @@ impl Observation { bytes: 0, first_byte: None, usage: Usage::default(), + compressed_usage: None, #[cfg(feature = "otel")] route: route.into(), #[cfg(feature = "otel")] @@ -95,18 +98,41 @@ impl Observation { self.status = status; self.span.record("http.response.status_code", status); } - pub(super) fn content_type(&mut self, value: Option<&str>) { + pub(super) fn content(&mut self, value: Option<&str>, encoding: Option<&str>) { self.usage.sse = value.is_some_and(|value| value.split(';').next() == Some("text/event-stream")); + match encoding.map(str::trim) { + None | Some("") => {} + Some(value) if value.eq_ignore_ascii_case("identity") => {} + Some(value) if value.eq_ignore_ascii_case("gzip") => { + self.compressed_usage = Some(flate2::write::GzDecoder::new(std::mem::take( + &mut self.usage, + ))); + } + Some(_) => self.usage.overflow = true, + } } pub(super) fn chunk(&mut self, bytes: &[u8]) { if self.first_byte.is_none() { self.first_byte = Some(self.started.elapsed().as_secs_f64()); } self.bytes += bytes.len() as u64; - self.usage.push(bytes); + if let Some(decoder) = self.compressed_usage.as_mut() { + if decoder.write_all(bytes).is_err() { + self.compressed_usage = None; + self.usage.overflow = true; + } + } else { + self.usage.push(bytes); + } } pub(super) fn complete(&mut self) { + if let Some(decoder) = self.compressed_usage.take() { + self.usage = decoder.finish().unwrap_or_else(|_| Usage { + overflow: true, + ..Default::default() + }); + } self.usage.finish(); self.complete = true; } @@ -114,15 +140,24 @@ impl Observation { impl Drop for Observation { fn drop(&mut self) { + let usage = self + .compressed_usage + .as_ref() + .map(|decoder| decoder.get_ref()) + .unwrap_or(&self.usage); self.span.record("http.response.body.size", self.bytes); self.span.record("hellas.response.complete", self.complete); if let Some(ttfb) = self.first_byte { self.span.record("hellas.response.time_to_first_byte", ttfb); } for (field, value) in [ - ("gen_ai.usage.input_tokens", self.usage.input), - ("gen_ai.usage.output_tokens", self.usage.output), - ("gen_ai.usage.cache_read.input_tokens", self.usage.cached), + ("gen_ai.usage.input_tokens", usage.input), + ("gen_ai.usage.output_tokens", usage.output), + ("gen_ai.usage.cache_read.input_tokens", usage.cached), + ( + "gen_ai.usage.cache_creation.input_tokens", + usage.cache_write, + ), ] { if let Some(value) = value { self.span.record(field, value); @@ -157,9 +192,10 @@ impl Drop for Observation { self.metrics.first_byte.record(ttfb, &labels); } for (kind, value) in [ - ("input", self.usage.input), - ("output", self.usage.output), - ("cache_read", self.usage.cached), + ("input", usage.input), + ("output", usage.output), + ("cache_read", usage.cached), + ("cache_write", usage.cache_write), ] { if let Some(value) = value { let mut labels = labels.to_vec(); @@ -179,7 +215,26 @@ struct Usage { input: Option, output: Option, cached: Option, + cache_write: Option, overflow: bool, + decoded_bytes: usize, +} + +impl std::io::Write for Usage { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + // Observation must not allow compressed data to consume unbounded CPU + // or memory. Stopping this sink never changes the response sent onward. + self.decoded_bytes = self.decoded_bytes.saturating_add(bytes.len()); + if self.overflow || self.decoded_bytes > 32 * 1024 * 1024 { + return Err(std::io::Error::other("usage observation limit")); + } + self.push(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } } impl Usage { @@ -188,18 +243,24 @@ impl Usage { return; } self.pending.extend_from_slice(bytes); - if !self.sse { - return; - } - while let Some(end) = self.pending.iter().position(|byte| *byte == b'\n') { + // Some Responses endpoints omit Content-Type. Detect their SSE prelude + // after enough bytes have arrived, without changing the forwarded body. + self.sse |= self.pending.starts_with(b"event:") + || self.pending.starts_with(b"data:") + || self.pending.starts_with(b":"); + while self.sse + && let Some(end) = self.pending.iter().position(|byte| *byte == b'\n') + { let line: Vec<_> = self.pending.drain(..=end).collect(); if let Some(data) = line.strip_prefix(b"data:") { self.parse(data); } } if self.pending.len() > 512 * 1024 { - self.pending.clear(); - self.overflow = true; + *self = Self { + overflow: true, + ..Default::default() + }; } } fn finish(&mut self) { @@ -244,6 +305,14 @@ impl Usage { .or_else(|| usage.pointer("/input_tokens_details/cached_tokens")) .and_then(Value::as_u64), ); + update( + &mut self.cache_write, + usage + .get("cache_creation_input_tokens") + .or_else(|| usage.pointer("/input_tokens_details/cache_write_tokens")) + .or_else(|| usage.pointer("/prompt_tokens_details/cache_write_tokens")) + .and_then(Value::as_u64), + ); } } @@ -251,9 +320,73 @@ impl Usage { mod tests { use super::*; + fn gzip(bytes: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(bytes).unwrap(); + encoder.finish().unwrap() + } + + #[test] + fn compressed_usage_survives_fragmentation_without_changing_wire_accounting() { + let wire = gzip(b"event: message_start\ndata: {\"message\":{\"usage\":{\"input_tokens\":2,\"cache_creation_input_tokens\":400,\"cache_read_input_tokens\":1244}}}\n\ndata: {\"usage\":{\"output_tokens\":99}}\n\n"); + for size in 1..=wire.len() { + let mut observed = Observation::new(&Metrics::new(), "/v1/messages"); + observed.content(Some("text/event-stream; charset=utf-8"), Some("gzip")); + for chunk in wire.chunks(size) { + observed.chunk(chunk); + } + observed.complete(); + assert!(observed.complete); + assert_eq!(observed.bytes, wire.len() as u64); + assert_eq!( + ( + observed.usage.input, + observed.usage.output, + observed.usage.cached, + observed.usage.cache_write + ), + (Some(2), Some(99), Some(1244), Some(400)) + ); + } + } + + #[test] + fn bad_or_unsupported_compression_leaves_usage_unknown() { + let wire = gzip(b"{\"usage\":{\"input_tokens\":10,\"output_tokens\":5}}"); + let mut corrupt = wire.clone(); + let crc = corrupt.len() - 8; + corrupt[crc] ^= 1; + for (bytes, encoding) in [ + (&wire[..wire.len() - 4], "gzip"), + (corrupt.as_slice(), "gzip"), + (wire.as_slice(), "br"), + ] { + let mut observed = Observation::new(&Metrics::new(), "/v1/responses"); + observed.content(Some("application/json"), Some(encoding)); + observed.chunk(bytes); + observed.complete(); + assert!(observed.complete); + assert_eq!(observed.bytes, bytes.len() as u64); + assert_eq!((observed.usage.input, observed.usage.output), (None, None)); + } + } + + #[test] + fn compressed_observation_stops_at_its_budget() { + let data = b": ping\n\n".repeat(32 * 1024 * 1024 / 8 + 1); + let wire = gzip(&data); + let mut observed = Observation::new(&Metrics::new(), "/v1/messages"); + observed.content(Some("text/event-stream"), Some("gzip")); + observed.chunk(&wire); + observed.complete(); + assert!(observed.usage.overflow); + assert_eq!(observed.bytes, wire.len() as u64); + assert_eq!(observed.usage.input, None); + } + #[test] fn usage_survives_arbitrary_sse_boundaries_and_cumulative_updates() { - let wire = b"event: message_start\r\ndata: {\"message\":{\"usage\":{\"input_tokens\":11,\"output_tokens\":1,\"cache_read_input_tokens\":7}}}\r\n\r\ndata: {\"usage\":{\"output_tokens\":5}}\n\ndata: {\"usage\":{\"output_tokens\":5}}\n\ndata: [DONE]\n\n"; + let wire = b"event: message_start\r\ndata: {\"message\":{\"usage\":{\"input_tokens\":11,\"output_tokens\":1,\"cache_read_input_tokens\":7,\"cache_creation_input_tokens\":13}}}\r\n\r\ndata: {\"usage\":{\"output_tokens\":5}}\n\ndata: {\"usage\":{\"output_tokens\":5}}\n\ndata: [DONE]\n\n"; for size in 1..=wire.len() { let mut usage = Usage { sse: true, @@ -267,6 +400,7 @@ mod tests { (usage.input, usage.output, usage.cached), (Some(11), Some(5), Some(7)) ); + assert_eq!(usage.cache_write, Some(13)); } } @@ -287,12 +421,10 @@ mod tests { #[test] fn responses_completed_usage_survives_fragmented_delivery() { - let wire = b"event: response.completed\ndata: {\"response\":{\"usage\":{\"input_tokens\":12,\"output_tokens\":3,\"input_tokens_details\":{\"cached_tokens\":8}}}}\n\n"; + let wire = b"event: response.completed\ndata: {\"response\":{\"usage\":{\"input_tokens\":12,\"output_tokens\":3,\"input_tokens_details\":{\"cached_tokens\":8,\"cache_write_tokens\":2}}}}\n\n"; for size in 1..=wire.len() { - let mut usage = Usage { - sse: true, - ..Default::default() - }; + // Live subscription Responses can omit Content-Type entirely. + let mut usage = Usage::default(); for chunk in wire.chunks(size) { usage.push(chunk); } @@ -301,6 +433,7 @@ mod tests { (usage.input, usage.output, usage.cached), (Some(12), Some(3), Some(8)) ); + assert_eq!(usage.cache_write, Some(2)); } } } diff --git a/crates/providers/HTTPS.md b/crates/providers/HTTPS.md index e4555044..1370d719 100644 --- a/crates/providers/HTTPS.md +++ b/crates/providers/HTTPS.md @@ -69,6 +69,10 @@ memory. Alternatively, replace `secret_env` with `secret_file` and `secret_field to read a private JSON login file on each request. The loader rejects symlinks, non-regular files, files that are not owner-only, and files over 64 KiB; atomic token rotation is picked up without restarting the provider. +`secret_field` and `expires_field` accept a top-level key or a JSON Pointer +starting with `/`, for example `/tokens/access_token` or `/claudeAiOauth/accessToken`. +This lets the provider read native login files directly. The native login tool +still owns token refresh; do not independently rotate a CLI's refresh token. For short-lived tokens, a file credential can include: diff --git a/crates/providers/src/http/config.rs b/crates/providers/src/http/config.rs index 46ecc587..2ea2f8a6 100644 --- a/crates/providers/src/http/config.rs +++ b/crates/providers/src/http/config.rs @@ -51,8 +51,7 @@ pub struct CredentialRefresh { impl CredentialRefresh { fn due(&self, object: &serde_json::Value) -> Result { - let expiry = object - .get(&self.expires_field) + let expiry = credential_field(object, &self.expires_field) .and_then(serde_json::Value::as_f64) .filter(|value| value.is_finite()) .ok_or_else(|| super::fault("credential expiry unavailable"))?; @@ -141,8 +140,7 @@ impl HttpSecret { **retry_at = None; } } - let secret = object - .get(field) + let secret = credential_field(&object, field) .and_then(serde_json::Value::as_str) .filter(|secret| !secret.is_empty()) .ok_or_else(|| super::fault("credential field unavailable"))?; @@ -152,6 +150,17 @@ impl HttpSecret { } } +fn credential_field<'a>( + object: &'a serde_json::Value, + field: &str, +) -> Option<&'a serde_json::Value> { + if field.starts_with('/') { + object.pointer(field) + } else { + object.get(field) + } +} + async fn read_credential(path: PathBuf) -> Result { tokio::task::spawn_blocking(move || { let file = hellas_private::open_nofollow(&path) @@ -240,6 +249,51 @@ impl HttpProviderConfig { #[cfg(test)] mod tests { use super::*; + + #[tokio::test] + async fn nested_file_credentials_follow_rotation_and_reject_missing_fields() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("account.json"); + let source = HttpSecret::JsonFile { + path: path.clone(), + field: "/tokens/access_token".into(), + prefix: "Bearer ".into(), + refresh: None, + }; + for token in ["first", "rotated"] { + hellas_private::write_atomically( + &path, + ".tmp", + &serde_json::to_vec(&serde_json::json!({"tokens":{"access_token":token}})).unwrap(), + ) + .unwrap(); + assert_eq!(source.resolve().await.unwrap(), format!("Bearer {token}")); + } + hellas_private::write_atomically( + &path, + ".tmp", + b"{\"tokens\":{\"refresh_token\":\"private\"}}", + ) + .unwrap(); + assert!(source.resolve().await.is_err()); + let refresh = CredentialRefresh { + command: vec!["unused".into()], + expires_field: "/tokens/expires_at".into(), + lock: Default::default(), + }; + assert!( + refresh + .due(&serde_json::json!({"tokens":{"expires_at":0}})) + .unwrap() + ); + assert!( + !refresh + .due(&serde_json::json!({"tokens":{"expires_at":4102444800_u64}})) + .unwrap() + ); + assert!(refresh.due(&serde_json::json!({"tokens":{}})).is_err()); + } + #[cfg(unix)] #[tokio::test] async fn failed_refresh_is_not_repeated_by_waiting_requests() { diff --git a/crates/wire/Cargo.toml b/crates/wire/Cargo.toml index d2c91349..94040572 100644 --- a/crates/wire/Cargo.toml +++ b/crates/wire/Cargo.toml @@ -115,3 +115,4 @@ windows-sys = { version = "0.61", optional = true, features = [ [dev-dependencies] iroh = { workspace = true, features = ["tls-ring"] } tokio = { workspace = true, features = ["full"] } +tracing-subscriber = { workspace = true } diff --git a/crates/wire/src/iroh/pool.rs b/crates/wire/src/iroh/pool.rs index c22a5134..4de1db45 100644 --- a/crates/wire/src/iroh/pool.rs +++ b/crates/wire/src/iroh/pool.rs @@ -39,6 +39,7 @@ use ::iroh::{Endpoint, EndpointId}; // Xtensa (esp32-s3) lacks native 64-bit atomics; portable-atomic provides a // mutex-fallback so embedded targets still compile. use portable_atomic::{AtomicU64, Ordering}; +use tracing::Instrument; use web_time::Instant; use crate::iroh::transport::IrohTransport; @@ -235,10 +236,15 @@ impl Pool { // loser's connection is dropped immediately when we re-insert. // Pass the full EndpointAddr to iroh::Endpoint::connect so any // CLI-supplied direct addresses become dial hints. + // The QUIC driver retains the current span for the connection's entire + // lifetime. Give it a root span so it cannot keep the dialing request + // and its ancestors open while subsequent requests reuse the connection. + let span = tracing::info_span!(target: "hellas_request", parent: None, "quic.connection"); let connect = self .inner .endpoint - .connect(target, self.inner.alpn.as_slice()); + .connect(target, self.inner.alpn.as_slice()) + .instrument(span); let conn = n0_future::time::timeout(self.inner.options.connect_timeout, connect) .await .map_err(|_| PoolError::Timeout)? @@ -362,3 +368,59 @@ impl Drop for Pool { } } } + +#[cfg(test)] +mod tests { + use super::*; + use tracing::instrument::{Instrument, WithSubscriber}; + use tracing_subscriber::{Layer, layer::Context, prelude::*, registry::LookupSpan}; + + #[derive(Clone, Default)] + struct ClosedSpans(Arc>>); + + impl LookupSpan<'a>> Layer for ClosedSpans { + fn on_close(&self, id: tracing::span::Id, ctx: Context<'_, S>) { + self.0.lock().unwrap().push(ctx.span(&id).unwrap().name()); + } + } + + #[tokio::test] + async fn pooled_connection_does_not_hold_request_span_open() { + let closed = ClosedSpans::default(); + let dispatch = tracing::Dispatch::new( + tracing_subscriber::registry() + .with(tracing_subscriber::filter::LevelFilter::INFO) + .with(closed.clone()), + ); + async { + let alpn = b"/hellas-test/pool/1"; + let server = Endpoint::builder(::iroh::endpoint::presets::Minimal) + .alpns(vec![alpn.to_vec()]) + .bind() + .await + .unwrap(); + let client = Endpoint::builder(::iroh::endpoint::presets::Minimal) + .bind() + .await + .unwrap(); + let pool = Pool::new(client.clone(), alpn, PoolOptions::default()); + let request = tracing::info_span!("request"); + let (connected, accepted) = + tokio::join!(pool.connection(server.addr()).instrument(request), async { + server.accept().await.unwrap().await.unwrap() + },); + let connected = connected.unwrap(); + assert!(connected.close_reason().is_none()); + assert!(accepted.close_reason().is_none()); + assert!( + closed.0.lock().unwrap().contains(&"request"), + "request span must close while the pooled connection is still live" + ); + pool.shutdown(); + client.close().await; + server.close().await; + } + .with_subscriber(dispatch) + .await; + } +} diff --git a/docs/http-gateway.md b/docs/http-gateway.md index fd3f8526..d81212cf 100644 --- a/docs/http-gateway.md +++ b/docs/http-gateway.md @@ -118,11 +118,18 @@ concurrency returns 503 with `Retry-After: 1`. With `otel`, traces connect HTTP ingress, Fetch RPCs, credential refresh and upstream HTTP. Span attributes exclude request/response bodies and credentials. +Pooled QUIC connections have separate root spans so their lifetime cannot delay +exporting the first request's trace. `hellas.gateway.http.requests`, `.duration`, `.time_to_first_byte` and `.tokens` record status, completion, timings and standard OpenAI/Anthropic usage fields. -Usage is unknown when upstream omits it; cached tokens are reported separately. -Bodies remain encoded on the wire and in archives; usage extraction currently -requires an uncompressed JSON/SSE response. +SSE usage is recognized even when the upstream omits Content-Type. Cache reads +and writes are reported separately as `cache_read` and `cache_write` token kinds; +the input/output counts retain the upstream's meaning. Usage is unknown when +upstream omits it. +Bodies remain encoded on the wire and in archives. Usage observation accepts +plain JSON/SSE and gzip, with a 32 MiB decoded-byte budget and a 512 KiB pending +JSON/line budget. Invalid compression, unsupported encodings or exceeded budgets +leave usage unknown without affecting delivery. Account quota windows still come from the upstream quota exporter. Request token totals alone cannot determine subscription quota or billing. From 70a5a70205205c2f69cf3e66ef540bd7d6185542 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Thu, 24 Sep 2026 22:14:48 +0200 Subject: [PATCH 08/32] fix(fetch): reuse HTTPS connections and preserve trace and archive headers --- crates/gateway/src/archive.rs | 14 +- crates/gateway/src/archive/tests.rs | 8 + crates/providers/HTTPS.md | 6 +- crates/providers/src/http.rs | 23 +- crates/providers/src/http/clients.rs | 82 +++++++ crates/providers/src/http/tests.rs | 200 ++++++++++++++---- crates/providers/src/responses_fetch.rs | 8 +- .../src/responses_fetch/telemetry/noop.rs | 7 +- .../src/responses_fetch/telemetry/otel.rs | 100 ++++++++- crates/rpc/src/telemetry/http.rs | 68 +++++- docs/http-gateway.md | 15 +- 11 files changed, 453 insertions(+), 78 deletions(-) create mode 100644 crates/providers/src/http/clients.rs diff --git a/crates/gateway/src/archive.rs b/crates/gateway/src/archive.rs index 89f6fe23..546305b3 100644 --- a/crates/gateway/src/archive.rs +++ b/crates/gateway/src/archive.rs @@ -135,9 +135,14 @@ pub(crate) async fn record( async fn archive_response( policy: Policy, mut archive: Exchange, - response: axum::response::Response, + mut response: axum::response::Response, ) -> axum::response::Response { use axum::body::Body; + if let Some(id) = archive.directory.file_name().and_then(|id| id.to_str()) { + response + .headers_mut() + .insert("x-hellas-request-id", id.parse().unwrap()); + } if let Err(error) = archive .head(response.status().as_u16(), response.headers()) .await @@ -145,12 +150,7 @@ async fn archive_response( policy.failed("response_head", &error); return response; } - let (mut parts, body) = response.into_parts(); - if let Some(id) = archive.directory.file_name().and_then(|id| id.to_str()) { - parts - .headers - .insert("x-hellas-request-id", id.parse().unwrap()); - } + let (parts, body) = response.into_parts(); if body.is_end_stream() { if let Err(error) = archive.finish().await { policy.failed("finish", &error); diff --git a/crates/gateway/src/archive/tests.rs b/crates/gateway/src/archive/tests.rs index 7c2f107a..d1674256 100644 --- a/crates/gateway/src/archive/tests.rs +++ b/crates/gateway/src/archive/tests.rs @@ -217,6 +217,13 @@ async fn unavailable_archive_preserves_errors_and_recovers_on_the_next_request() async fn response_head_archive_failure_preserves_status_headers_and_body() { let root = tempfile::tempdir().unwrap(); let archive = exchange(root.path()).await; + let id = archive + .directory + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned(); let moved = root.path().join("moved"); std::fs::rename(&archive.directory, &moved).unwrap(); std::fs::write(&archive.directory, b"unavailable").unwrap(); @@ -227,6 +234,7 @@ async fn response_head_archive_failure_preserves_status_headers_and_body() { .unwrap(); let response = archive_response(policy(root.path()), archive, response).await; assert_eq!(response.status(), 429); + assert_eq!(response.headers()["x-hellas-request-id"], id); assert_eq!(response.headers()["retry-after"], "3"); assert_eq!( axum::body::to_bytes(response.into_body(), 1024) diff --git a/crates/providers/HTTPS.md b/crates/providers/HTTPS.md index 1370d719..e2a231af 100644 --- a/crates/providers/HTTPS.md +++ b/crates/providers/HTTPS.md @@ -98,7 +98,11 @@ with public WebPKI roots. Callers cannot override that account's header or change trust roots to impersonate its origin. Restrict paths to the inference endpoints the account is intended to expose. -DNS answers are checked and pinned to the request's connection. Private, +DNS answers are resolved and checked on every request. Connections are reused +only for the same origin, DNS address set, TLS roots/pins and credential alias. +Each provider retains at most 32 client pools, with at most four idle connections +per origin and a 90 second idle expiry. Credentials are still resolved per request; +the pools contain no default authorization headers. Private, loopback, link-local, multicast and other special-purpose addresses are denied by default. An operator can explicitly enable private addresses only with a nonempty exact host allowlist, for controlled private services. Environment diff --git a/crates/providers/src/http.rs b/crates/providers/src/http.rs index ccd7444d..b1c95605 100644 --- a/crates/providers/src/http.rs +++ b/crates/providers/src/http.rs @@ -23,6 +23,7 @@ use std::{ time::Duration, }; use tracing::Instrument; +mod clients; mod config; mod tls; pub use config::{CredentialRefresh, HttpCredentialConfig, HttpProviderConfig, HttpSecret}; @@ -63,6 +64,7 @@ impl std::fmt::Debug for HttpCredential { pub struct HttpFetchProvider { policy: HttpEgressPolicy, credentials: Arc>, + clients: Arc, } fn fault(message: &'static str) -> FetchProviderError { @@ -134,6 +136,7 @@ impl HttpFetchProvider { Ok(Self { policy, credentials: Arc::new(credentials), + clients: Arc::new(clients::Clients::default()), }) } @@ -226,21 +229,7 @@ impl HttpFetchProvider { { return Err(fault("HTTPS DNS addresses are outside the egress policy")); } - let tls = tls::config(&request.tls).map_err(fault)?; - let client = reqwest::Client::builder() - .no_proxy() - .no_gzip() - .no_brotli() - .no_deflate() - .no_zstd() - .redirect(reqwest::redirect::Policy::none()) - .retry(reqwest::retry::never()) - .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(20 * 60)) - .resolve_to_addrs(host, &addresses) - .tls_backend_preconfigured(tls) - .build() - .map_err(|_| fault("HTTPS client initialization failed"))?; + let client = self.clients.get(&request, &url, addresses)?; let method = request .method .parse() @@ -263,9 +252,7 @@ impl HttpFetchProvider { let mut trace = crate::responses_fetch::telemetry::Request::for_method(&url, &request.method); let response = trace - .propagate(outbound) - .body(request.body().map_err(|_| fault("invalid HTTP body"))?) - .send() + .send(outbound.body(request.body().map_err(|_| fault("invalid HTTP body"))?)) .instrument(trace.span.clone()) .await .map_err(|_| { diff --git a/crates/providers/src/http/clients.rs b/crates/providers/src/http/clients.rs new file mode 100644 index 00000000..890b1a75 --- /dev/null +++ b/crates/providers/src/http/clients.rs @@ -0,0 +1,82 @@ +//! Bounded connection reuse after per-request credential and DNS validation. +use super::{fault, tls}; +use hellas_executor::FetchProviderError; +use hellas_rpc::http_fetch::{HttpFetchRequest, HttpTls}; +use reqwest::{Client, Url}; +use std::{collections::VecDeque, net::SocketAddr, sync::Mutex, time::Duration}; + +const MAX_CLIENTS: usize = 32; + +#[derive(Debug, PartialEq, Eq)] +struct Key { + origin: String, + addresses: Vec, + tls: HttpTls, + credential: Option, +} + +#[derive(Debug, Default)] +pub(super) struct Clients(Mutex>); + +impl Clients { + /// `addresses` must have passed the provider's current egress policy. + /// No request bodies, authorization headers or credential values are cached. + pub(super) fn get( + &self, + request: &HttpFetchRequest, + url: &Url, + mut addresses: Vec, + ) -> Result { + // DNS answer order may rotate without changing the permitted set. + addresses.sort_unstable(); + addresses.dedup(); + let key = Key { + origin: url.origin().ascii_serialization(), + addresses, + tls: request.tls.clone(), + credential: request.credential.clone(), + }; + if let Some(client) = self.cached(&key) { + return Ok(client); + } + // Build outside the lock. A concurrent miss may build a second client, + // but the insertion check below shares one pool before either sends. + let client = Client::builder() + .no_proxy() + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() + .redirect(reqwest::redirect::Policy::none()) + .retry(reqwest::retry::never()) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(20 * 60)) + .pool_idle_timeout(Duration::from_secs(90)) + .pool_max_idle_per_host(4) + .resolve_to_addrs(url.host_str().expect("validated HTTPS URL"), &key.addresses) + .tls_backend_preconfigured(tls::config(&key.tls).map_err(fault)?) + .build() + .map_err(|_| fault("HTTPS client initialization failed"))?; + let mut entries = self.0.lock().unwrap(); + if let Some(index) = entries.iter().position(|(existing, _)| *existing == key) { + let entry = entries.remove(index).unwrap(); + let client = entry.1.clone(); + entries.push_back(entry); + return Ok(client); + } + if entries.len() == MAX_CLIENTS { + entries.pop_front(); + } + entries.push_back((key, client.clone())); + Ok(client) + } + + fn cached(&self, key: &Key) -> Option { + let mut entries = self.0.lock().unwrap(); + let index = entries.iter().position(|(existing, _)| existing == key)?; + let entry = entries.remove(index).unwrap(); + let client = entry.1.clone(); + entries.push_back(entry); + Some(client) + } +} diff --git a/crates/providers/src/http/tests.rs b/crates/providers/src/http/tests.rs index e50c9478..d95089e7 100644 --- a/crates/providers/src/http/tests.rs +++ b/crates/providers/src/http/tests.rs @@ -14,6 +14,7 @@ async fn server( ) -> ( HttpFetchRequest, Arc, + Arc, tokio::task::JoinHandle<()>, ) { server_with_pause(status, bytes, location, None).await @@ -27,6 +28,7 @@ async fn server_with_pause( ) -> ( HttpFetchRequest, Arc, + Arc, tokio::task::JoinHandle<()>, ) { let key = generate_simple_self_signed(vec!["localhost".into()]).unwrap(); @@ -59,8 +61,11 @@ async fn server_with_pause( ); let calls = Arc::new(AtomicUsize::new(0)); let seen = calls.clone(); + let connections = Arc::new(AtomicUsize::new(0)); + let accepted = connections.clone(); let task = tokio::spawn(async move { while let Ok((socket, _)) = listener.accept().await { + accepted.fetch_add(1, Ordering::SeqCst); let acceptor = acceptor.clone(); let seen = seen.clone(); let bytes = bytes.clone(); @@ -70,38 +75,42 @@ async fn server_with_pause( let Ok(mut socket) = acceptor.accept(socket).await else { return; }; - let mut request = Vec::new(); - let mut byte = [0u8; 1]; - while !request.ends_with(b"\r\n\r\n") && request.len() < 32768 { - if socket.read_exact(&mut byte).await.is_err() { + loop { + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + while !request.ends_with(b"\r\n\r\n") && request.len() < 32768 { + if socket.read_exact(&mut byte).await.is_err() { + return; + } + request.push(byte[0]); + } + seen.fetch_add(1, Ordering::SeqCst); + let location = location + .as_ref() + .map(|v| format!("Location: {v}\r\n")) + .unwrap_or_default(); + let retry_after = if status == 429 { + "Retry-After: 7\r\n" + } else { + "" + }; + let header = format!( + "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: keep-alive\r\n{location}{retry_after}\r\n", + bytes.len() + ); + let _ = socket.write_all(header.as_bytes()).await; + if let Some(pause) = pause.as_ref() { + let _ = socket.write_all(&bytes[..1]).await; + let _ = socket.flush().await; + pause.notified().await; + let _ = socket.write_all(&bytes[1..]).await; + let _ = socket.shutdown().await; + return; + } + if socket.write_all(&bytes).await.is_err() || socket.flush().await.is_err() { return; } - request.push(byte[0]); - } - seen.fetch_add(1, Ordering::SeqCst); - let location = location - .map(|v| format!("Location: {v}\r\n")) - .unwrap_or_default(); - let retry_after = if status == 429 { - "Retry-After: 7\r\n" - } else { - "" - }; - let header = format!( - "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: close\r\n{location}{retry_after}\r\n", - bytes.len() - ); - let _ = socket.write_all(header.as_bytes()).await; - if let Some(pause) = pause { - let _ = socket.write_all(&bytes[..1]).await; - let _ = socket.flush().await; - pause.notified().await; - let _ = socket.write_all(&bytes[1..]).await; - let _ = socket.shutdown().await; - return; } - let _ = socket.write_all(&bytes).await; - let _ = socket.shutdown().await; }); } }); @@ -121,6 +130,7 @@ async fn server_with_pause( max_response_bytes: 4096, }, calls, + connections, task, ) } @@ -128,7 +138,7 @@ async fn server_with_pause( #[tokio::test] async fn rate_limit_is_returned_with_its_body_and_delay_without_retrying() { let body = br#"{"error":{"type":"rate_limit"}}"#.to_vec(); - let (request, calls, task) = server(429, body.clone(), None).await; + let (request, calls, _, task) = server(429, body.clone(), None).await; let mut response = provider().run(prepared(&request)).await.unwrap(); let head = response.head.http.unwrap(); assert_eq!(head.status, 429); @@ -145,7 +155,7 @@ async fn rate_limit_is_returned_with_its_body_and_delay_without_retrying() { #[tokio::test] async fn coding_response_can_exceed_the_old_half_megabyte_ceiling() { let body = vec![b'x'; 1024 * 1024]; - let (mut request, calls, task) = server(200, body.clone(), None).await; + let (mut request, calls, _, task) = server(200, body.clone(), None).await; request.max_response_bytes = hellas_rpc::http_fetch::MAX_HTTP_RESPONSE_BYTES; let mut response = provider().run(prepared(&request)).await.unwrap(); let mut received = Vec::new(); @@ -162,7 +172,7 @@ async fn coding_response_can_exceed_the_old_half_megabyte_ceiling() { #[tokio::test] async fn small_stream_delivery_does_not_wait_for_record_capacity_or_eof() { let resume = Arc::new(tokio::sync::Notify::new()); - let (request, _, server) = + let (request, _, _, server) = server_with_pause(200, b"ab".to_vec(), None, Some(resume.clone())).await; let provider = provider(); let mut response = provider.run(prepared(&request)).await.unwrap(); @@ -204,7 +214,7 @@ fn prepared(request: &HttpFetchRequest) -> PreparedFetchRequest { #[tokio::test] async fn custom_roots_and_spki_deliver_exact_binary_bytes() { let bytes = vec![0, 255, 1, 13, 10, 128]; - let (request, calls, task) = server(200, bytes.clone(), None).await; + let (request, calls, _, task) = server(200, bytes.clone(), None).await; let mut response = provider().run(prepared(&request)).await.unwrap(); assert_eq!(response.head.http.as_ref().unwrap().status, 200); let mut body = Vec::new(); @@ -218,7 +228,7 @@ async fn custom_roots_and_spki_deliver_exact_binary_bytes() { #[tokio::test] async fn wrong_pin_wrong_roots_and_wrong_hostname_send_no_http_request() { - let (valid, calls, task) = server(200, b"ok".to_vec(), None).await; + let (valid, calls, _, task) = server(200, b"ok".to_vec(), None).await; let mut wrong_pin = valid.clone(); wrong_pin.tls.spki_sha256 = vec!["00".repeat(32)]; let mut wrong_root = valid.clone(); @@ -233,17 +243,131 @@ async fn wrong_pin_wrong_roots_and_wrong_hostname_send_no_http_request() { BTreeMap::new(), ) .unwrap(); + let response = configured.run(prepared(&valid)).await.unwrap(); + assert_eq!(collect(response).await, b"ok"); for request in [wrong_pin, wrong_root, wrong_name] { assert!(configured.run(prepared(&request)).await.is_err()); } - assert_eq!(calls.load(Ordering::SeqCst), 0); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "a warm connection must not bypass TLS validation" + ); + task.abort(); +} + +async fn collect(mut response: FetchProviderResponse) -> Vec { + let mut body = Vec::new(); + while let Some(chunk) = response.stream.next().await { + body.extend(chunk.unwrap()); + } + body +} + +#[tokio::test] +async fn repeated_requests_reuse_https_but_still_validate_each_request() { + let (request, calls, connections, task) = server(200, b"ok".to_vec(), None).await; + let provider = provider(); + for _ in 0..3 { + assert_eq!( + collect(provider.run(prepared(&request)).await.unwrap()).await, + b"ok" + ); + } + assert_eq!(calls.load(Ordering::SeqCst), 3); + assert_eq!(connections.load(Ordering::SeqCst), 1); + let mut forbidden = request.clone(); + forbidden.credential = Some("unknown-account".into()); + assert!(provider.run(prepared(&forbidden)).await.is_err()); + forbidden.credential = None; + forbidden.max_response_bytes = 1; + let mut response = provider.run(prepared(&forbidden)).await.unwrap(); + assert!(response.stream.next().await.unwrap().is_err()); + assert_eq!(connections.load(Ordering::SeqCst), 1); + task.abort(); +} + +#[tokio::test] +async fn client_pools_separate_dns_answers_aliases_and_tls_settings_and_evict() { + let (mut request, calls, connections, task) = server(200, b"ok".to_vec(), None).await; + let valid_tls = request.tls.clone(); + let url = request.parsed_url().unwrap(); + let address = SocketAddr::from(([127, 0, 0, 1], url.port().unwrap())); + let other = SocketAddr::from(([127, 0, 0, 2], url.port().unwrap())); + let clients = clients::Clients::default(); + for addresses in [vec![address, other], vec![other, address, address]] { + let client = clients.get(&request, &url, addresses).unwrap(); + assert_eq!( + client + .get(url.clone()) + .send() + .await + .unwrap() + .bytes() + .await + .unwrap(), + "ok" + ); + } + assert_eq!( + connections.load(Ordering::SeqCst), + 1, + "DNS order must not discard a usable pool" + ); + let denied = clients.get(&request, &url, vec![other]).unwrap(); + assert!( + denied.get(url.clone()).send().await.is_err(), + "a new DNS answer must not reuse the old address" + ); + assert_eq!(calls.load(Ordering::SeqCst), 2); + // Exercise the cache boundary directly: production credential authorization + // runs before this method and requires public roots, tested separately. + request.credential = Some("second-account".into()); + let client = clients.get(&request, &url, vec![address, other]).unwrap(); + assert_eq!( + client + .get(url.clone()) + .send() + .await + .unwrap() + .bytes() + .await + .unwrap(), + "ok" + ); + drop(client); + assert_eq!(connections.load(Ordering::SeqCst), 2); + request.tls.spki_sha256 = vec!["00".repeat(32)]; + let client = clients.get(&request, &url, vec![address, other]).unwrap(); + assert!(client.get(url.clone()).send().await.is_err()); + assert_eq!( + calls.load(Ordering::SeqCst), + 3, + "wrong pins must not reuse a verified socket" + ); + drop(client); + request.tls.spki_sha256.clear(); + // More distinct policies than the cache admits must evict its old pools. + for index in 0..40 { + request.credential = Some(format!("account-{index}")); + clients.get(&request, &url, vec![address]).unwrap(); + } + let before = connections.load(Ordering::SeqCst); + request.credential = None; + request.tls = valid_tls; + let client = clients.get(&request, &url, vec![address, other]).unwrap(); + assert_eq!( + client.get(url).send().await.unwrap().bytes().await.unwrap(), + "ok" + ); + assert_eq!(connections.load(Ordering::SeqCst), before + 1); task.abort(); } #[tokio::test] async fn redirects_are_returned_without_following_them() { - let (target, target_calls, target_task) = server(200, b"private".to_vec(), None).await; - let (origin, origin_calls, origin_task) = server(302, vec![], Some(target.url)).await; + let (target, target_calls, _, target_task) = server(200, b"private".to_vec(), None).await; + let (origin, origin_calls, _, origin_task) = server(302, vec![], Some(target.url)).await; let response = provider().run(prepared(&origin)).await.unwrap(); assert_eq!(response.head.http.unwrap().status, 302); assert_eq!(origin_calls.load(Ordering::SeqCst), 1); @@ -254,7 +378,7 @@ async fn redirects_are_returned_without_following_them() { #[tokio::test] async fn signed_response_size_and_default_private_address_denial_are_enforced() { - let (mut request, calls, task) = server(200, vec![42; 100], None).await; + let (mut request, calls, _, task) = server(200, vec![42; 100], None).await; let public = HttpFetchProvider::new(HttpEgressPolicy::default(), BTreeMap::new()).unwrap(); assert!(public.run(prepared(&request)).await.is_err()); assert_eq!(calls.load(Ordering::SeqCst), 0); diff --git a/crates/providers/src/responses_fetch.rs b/crates/providers/src/responses_fetch.rs index 5dc03432..98299fa7 100644 --- a/crates/providers/src/responses_fetch.rs +++ b/crates/providers/src/responses_fetch.rs @@ -39,14 +39,14 @@ pub async fn execute_responses_request( label: &str, ) -> Result { let mut telemetry = telemetry::Request::new(&endpoint); - let request = telemetry - .propagate(client.post(endpoint)) + let request = client + .post(endpoint) .header(CONTENT_TYPE, "application/json") .header(AUTHORIZATION, format!("Bearer {bearer_token}")) .header("Idempotency-Key", idempotency_key) .body(body); - let upstream = request - .send() + let upstream = telemetry + .send(request) .instrument(telemetry.span.clone()) .await .map_err(|source| { diff --git a/crates/providers/src/responses_fetch/telemetry/noop.rs b/crates/providers/src/responses_fetch/telemetry/noop.rs index 8948ce84..27f4dae8 100644 --- a/crates/providers/src/responses_fetch/telemetry/noop.rs +++ b/crates/providers/src/responses_fetch/telemetry/noop.rs @@ -12,8 +12,11 @@ impl Request { pub fn for_method(_: &reqwest::Url, _: &str) -> Self { Self { span: Span::none() } } - pub fn propagate(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder { - request + pub async fn send( + &self, + request: reqwest::RequestBuilder, + ) -> reqwest::Result { + request.send().await } pub fn status(&self, _: u16) {} pub fn fail(&mut self, _: &'static str) {} diff --git a/crates/providers/src/responses_fetch/telemetry/otel.rs b/crates/providers/src/responses_fetch/telemetry/otel.rs index 162ede5c..53d86194 100644 --- a/crates/providers/src/responses_fetch/telemetry/otel.rs +++ b/crates/providers/src/responses_fetch/telemetry/otel.rs @@ -25,15 +25,32 @@ impl Request { otel.status_code = tracing::field::Empty), } } - pub fn propagate(&self, mut request: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + pub async fn send( + &self, + builder: reqwest::RequestBuilder, + ) -> reqwest::Result { + let (client, request) = builder.build_split(); + let mut request = request?; let mut metadata = Metadata::new(); hellas_rpc::telemetry::inject(&self.span, &mut metadata); - for name in ["traceparent", "tracestate"] { - if let Some(value) = metadata.get(name).and_then(|v| v.as_text()) { - request = request.header(name, value); + // Without an active tracing context, preserve the caller's headers. + // Otherwise replace both fields, including stale tracestate when the + // new context has none. RequestBuilder::header would append duplicates. + if metadata.get("traceparent").is_some() { + for name in ["traceparent", "tracestate"] { + request.headers_mut().remove(name); + if let Some(value) = metadata + .get(name) + .and_then(|v| v.as_text()) + .filter(|value| !value.is_empty()) + { + if let Ok(value) = value.parse() { + request.headers_mut().insert(name, value); + } + } } } - request + client.execute(request).await } pub fn status(&self, code: u16) { self.span @@ -73,3 +90,76 @@ impl Drop for Request { } } } + +#[cfg(test)] +mod tests { + use super::*; + use opentelemetry::trace::TracerProvider; + use tracing_subscriber::prelude::*; + + #[tokio::test] + async fn propagation_replaces_duplicates_and_removes_stale_tracestate() { + opentelemetry::global::set_text_map_propagator( + opentelemetry_sdk::propagation::TraceContextPropagator::new(), + ); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build(); + let dispatch = tracing::Dispatch::new( + tracing_subscriber::registry() + .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("headers"))), + ); + let (tx, mut rx) = tokio::sync::mpsc::channel(2); + let app = axum::Router::new().route( + "/", + axum::routing::get(move |headers: axum::http::HeaderMap| { + let tx = tx.clone(); + async move { + tx.send(headers).await.unwrap(); + "ok" + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url: reqwest::Url = format!("http://{}/", listener.local_addr().unwrap()) + .parse() + .unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let client = reqwest::Client::new(); + for active in [true, false] { + let trace = if active { + tracing::dispatcher::with_default(&dispatch, || Request::new(&url)) + } else { + Request { + span: Span::none(), + complete: false, + } + }; + let mut context = Metadata::new(); + hellas_rpc::telemetry::inject(&trace.span, &mut context); + let builder = client + .get(url.clone()) + .header("traceparent", "old-one") + .header("traceparent", "old-two") + .header("tracestate", "vendor=stale") + .header("x-extension", "one") + .header("x-extension", "two"); + assert_eq!( + trace.send(builder).await.unwrap().text().await.unwrap(), + "ok" + ); + let headers = rx.recv().await.unwrap(); + assert_eq!(headers.get_all("x-extension").iter().count(), 2); + if active { + assert_eq!(headers.get_all("traceparent").iter().count(), 1); + assert_eq!( + headers["traceparent"], + context.get("traceparent").unwrap().as_text().unwrap() + ); + assert!(!headers.contains_key("tracestate")); + } else { + assert_eq!(headers.get_all("traceparent").iter().count(), 2); + assert_eq!(headers["tracestate"], "vendor=stale"); + } + } + server.abort(); + } +} diff --git a/crates/rpc/src/telemetry/http.rs b/crates/rpc/src/telemetry/http.rs index 8ba43228..3cfdf4cb 100644 --- a/crates/rpc/src/telemetry/http.rs +++ b/crates/rpc/src/telemetry/http.rs @@ -62,7 +62,10 @@ pub async fn trace_request(request: Request, next: Next) -> Response { super::inject(&trace.span, &mut context); if let Some(value) = context.get("traceparent").and_then(|value| value.as_text()) { if let Ok(header) = HeaderValue::from_str(value) { - response.headers_mut().insert("traceparent", header); + response + .headers_mut() + .entry("traceparent") + .or_insert(header); } if let Some(trace_id) = value.split('-').nth(1) && let Ok(header) = HeaderValue::from_str(trace_id) @@ -165,6 +168,69 @@ mod tests { use tracing::instrument::WithSubscriber; use tracing_subscriber::prelude::*; + #[tokio::test] + async fn upstream_trace_headers_survive_with_separate_gateway_correlation() { + opentelemetry::global::set_text_map_propagator( + opentelemetry_sdk::propagation::TraceContextPropagator::new(), + ); + let provider = SdkTracerProvider::builder().build(); + let dispatch = tracing::Dispatch::new( + tracing_subscriber::registry() + .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("headers"))), + ); + const UPSTREAM: &str = "00-33333333333333333333333333333333-4444444444444444-01"; + const CALLER: &str = "00-11111111111111111111111111111111-2222222222222222-01"; + let app = axum::Router::new() + .route("/plain", get(|| async { "ok" })) + .route( + "/upstream", + get(|| async { + Response::builder() + .header("traceparent", UPSTREAM) + .header("tracestate", "vendor=upstream") + .body(Body::from("ok")) + .unwrap() + }), + ) + .layer(axum::middleware::from_fn(trace_request)); + for path in ["/upstream", "/plain"] { + let response = app + .clone() + .oneshot( + Request::builder() + .uri(path) + .header("traceparent", CALLER) + .body(Body::empty()) + .unwrap(), + ) + .with_subscriber(dispatch.clone()) + .await + .unwrap(); + assert_eq!( + response.headers()["x-hellas-trace-id"], + "11111111111111111111111111111111" + ); + assert_eq!(response.headers().get_all("traceparent").iter().count(), 1); + if path == "/upstream" { + assert_eq!(response.headers()["traceparent"], UPSTREAM); + assert_eq!(response.headers()["tracestate"], "vendor=upstream"); + } else { + assert!( + response.headers()["traceparent"] + .to_str() + .unwrap() + .starts_with("00-11111111111111111111111111111111-") + ); + } + assert_eq!( + axum::body::to_bytes(response.into_body(), 16) + .await + .unwrap(), + "ok" + ); + } + } + #[tokio::test] async fn http_spans_cover_handler_and_body_completion_errors_and_cancellation() { let exporter = InMemorySpanExporter::default(); diff --git a/docs/http-gateway.md b/docs/http-gateway.md index d81212cf..a2c848a2 100644 --- a/docs/http-gateway.md +++ b/docs/http-gateway.md @@ -33,6 +33,13 @@ are removed; the caller's gateway bearer never becomes an upstream credential. O Configured values override the corresponding client header. The former `forward_headers` whitelist has been removed. +Several accounts at one upstream can use distinct aliases and gateway paths, +such as `/kimi-a/v1/chat/completions` and `/kimi-b/v1/chat/completions`, both +mapped to the upstream's chat endpoint. The client's base URL selects the route. +Each alias has independent admission and cooldown; routes sharing an alias +share those limits. This gateway targets one Hellas provider node. It does not +automatically select accounts, fail over between them or route using quota data. + Omit `credential` for an unauthenticated upstream. Such routes share admission by origin. Optional `tls` uses the [Fetch TLS vocabulary](../crates/providers/HTTPS.md) for exact trust anchors and pins, defaulting to WebPKI. Custom roots cannot be @@ -69,8 +76,9 @@ streams retain an incomplete archive. Archiving is best-effort: failures during setup, request/response writes or finalization are reported without replacing the upstream status or interrupting its response. After a write fails, archiving stops for that exchange; the next ordinary request attempts a fresh archive. -`x-hellas-request-id` identifies an archive attempt, not a guarantee of a complete -durable record. There is no automatic archive pruning. +Once an archive has been created, `x-hellas-request-id` remains present even if +saving its response metadata fails. It identifies an archive attempt, not a +guarantee of a complete durable record. There is no automatic archive pruning. Archive failures emit a `hellas_archive` warning with the operation, I/O error kind and OS error code, without payloads, credentials, paths or raw error text. @@ -118,6 +126,9 @@ concurrency returns 503 with `Retry-After: 1`. With `otel`, traces connect HTTP ingress, Fetch RPCs, credential refresh and upstream HTTP. Span attributes exclude request/response bodies and credentials. +Outbound trace context replaces the caller's propagation headers instead of +appending duplicates. Upstream response `traceparent` and `tracestate` are +preserved; `x-hellas-trace-id` identifies the gateway's trace independently. Pooled QUIC connections have separate root spans so their lifetime cannot delay exporting the first request's trace. `hellas.gateway.http.requests`, `.duration`, `.time_to_first_byte` and `.tokens` From b288a6ec93f4ea32b6fdd22d454979c024eabe61 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Thu, 24 Sep 2026 23:02:21 +0200 Subject: [PATCH 09/32] feat(gateway): route models across backends with session affinity --- Cargo.lock | 1 + crates/gateway/Cargo.toml | 1 + crates/gateway/src/archive.rs | 6 + crates/gateway/src/http_fetch.rs | 279 +++-------- crates/gateway/src/http_fetch/affinity.rs | 171 +++++++ crates/gateway/src/http_fetch/config.rs | 194 +++++++ crates/gateway/src/http_fetch/observation.rs | 43 +- crates/gateway/src/http_fetch/routing.rs | 474 ++++++++++++++++++ .../gateway/src/http_fetch/routing/tests.rs | 310 ++++++++++++ crates/gateway/src/http_fetch/tests.rs | 3 + crates/gateway/src/lib.rs | 19 +- docs/http-gateway.md | 116 ++++- 12 files changed, 1384 insertions(+), 233 deletions(-) create mode 100644 crates/gateway/src/http_fetch/affinity.rs create mode 100644 crates/gateway/src/http_fetch/config.rs create mode 100644 crates/gateway/src/http_fetch/routing.rs create mode 100644 crates/gateway/src/http_fetch/routing/tests.rs diff --git a/Cargo.lock b/Cargo.lock index e32ec49c..0723d279 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2908,6 +2908,7 @@ dependencies = [ "tower", "tracing", "tracing-opentelemetry", + "zstd", ] [[package]] diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index 05564cce..bf90c251 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -29,6 +29,7 @@ axum = { workspace = true } base64.workspace = true httpdate = "1" flate2 = "1" +zstd = "0.13" futures = { workspace = true } hellas-adaptors.workspace = true hellas-client = { workspace = true, features = ["iroh", "evaluate"] } diff --git a/crates/gateway/src/archive.rs b/crates/gateway/src/archive.rs index 546305b3..e183d5f8 100644 --- a/crates/gateway/src/archive.rs +++ b/crates/gateway/src/archive.rs @@ -143,6 +143,12 @@ async fn archive_response( .headers_mut() .insert("x-hellas-request-id", id.parse().unwrap()); } + if let Some(backend) = response + .extensions() + .get::() + { + archive.metadata["backend"] = json!(backend.0); + } if let Err(error) = archive .head(response.status().as_u16(), response.headers()) .await diff --git a/crates/gateway/src/http_fetch.rs b/crates/gateway/src/http_fetch.rs index 26b53fc8..1fc73266 100644 --- a/crates/gateway/src/http_fetch.rs +++ b/crates/gateway/src/http_fetch.rs @@ -1,5 +1,11 @@ //! HTTP bytes over authenticated Fetch, without translating vendor schemas. +mod affinity; +mod config; mod observation; +mod routing; + +pub use config::HttpGatewayConfig; +use config::HttpRoute; #[cfg(test)] mod tests; @@ -30,166 +36,23 @@ use tracing::Instrument; use super::{GatewayHandle, GatewayOptions, access, execution::CliRuntime}; -#[derive(Clone, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct HttpGatewayConfig { - pub service: String, - pub method: String, - pub routes: Vec, - #[serde(default = "default_concurrency")] - pub max_in_flight: usize, -} - -fn default_concurrency() -> usize { - 4 -} - -#[derive(Clone, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct HttpRoute { - pub path: String, - pub method: String, - pub url: String, - pub credential: Option, - #[serde(default = "public_tls")] - pub tls: HttpTls, - #[serde(default)] - pub headers: Vec<(String, String)>, -} - -fn public_tls() -> HttpTls { - HttpTls { - roots: HttpTrustRoots::WebPki, - spki_sha256: vec![], - } -} - -struct Account { - slots: Arc, - backoff: Mutex>, -} +#[derive(Clone)] +pub(crate) struct BackendName(pub String); -impl Account { - fn cooldown(&self) -> Option<(u16, Duration)> { - let (until, status) = (*self.backoff.lock().unwrap())?; - let delay = until.saturating_duration_since(Instant::now()); - (!delay.is_zero()).then_some((status, delay)) - } - - fn observe(&self, status: u16, headers: &HeaderMap) { - if status == 429 || (status >= 500 && headers.contains_key("retry-after")) { - let delay = retry_delay(headers).min(Duration::from_secs(u32::MAX as u64)); - let until = Instant::now() + delay; - let mut backoff = self.backoff.lock().unwrap(); - if backoff.is_none_or(|(previous, _)| until > previous) { - *backoff = Some((until, status)); - } - } - } +fn attributed(mut response: Response, name: &str) -> Response { + response.extensions_mut().insert(BackendName(name.into())); + response } struct HttpState { config: HttpGatewayConfig, runtime: CliRuntime, - route: ExecutionRoute, + routing: Arc, signer: Arc, assurance: Assurance, - accounts: BTreeMap>, metrics: observation::Metrics, } -impl HttpGatewayConfig { - fn validate(&self) -> anyhow::Result<()> { - ensure!( - !self.routes.is_empty(), - "HTTP gateway needs at least one route" - ); - ensure!( - self.max_in_flight > 0 && self.max_in_flight <= 1024, - "invalid HTTP concurrency limit" - ); - let mut paths = std::collections::BTreeSet::new(); - for route in &self.routes { - ensure!( - route.path.starts_with('/') && !route.path.contains(['?', '#', '{', '}']), - "HTTP routes must be exact paths" - ); - ensure!( - paths.insert((&route.path, &route.method)), - "duplicate HTTP route" - ); - ensure!( - route.headers.iter().all(|(name, _)| !matches!( - name.as_str(), - "authorization" | "x-api-key" | "cookie" - )), - "use a provider credential alias for authentication" - ); - route.request(Bytes::new(), &HeaderMap::new())?.validate()?; - } - Ok(()) - } -} - -impl HttpRoute { - fn account(&self) -> String { - match &self.credential { - Some(alias) => format!("credential:{alias}"), - None => format!( - "origin:{}", - self.url - .parse::() - .expect("validated URL") - .origin() - .ascii_serialization() - ), - } - } - - fn request(&self, body: Bytes, incoming: &HeaderMap) -> anyhow::Result { - let mut headers = self.headers.clone(); - let connection = connection_headers( - incoming - .iter() - .map(|(name, value)| (name.as_str(), value.to_str().unwrap_or_default())), - ); - for (name, value) in incoming { - let name = name.as_str(); - if !hop_header(name) - && !connection.iter().any(|token| token == name) - && !matches!( - name, - "host" - | "content-length" - | "authorization" - | "x-api-key" - | "api-key" - | "x-goog-api-key" - | "cookie" - | "forwarded" - ) - && !name.starts_with("x-hellas-") - && !name.starts_with("x-forwarded-") - && !self - .headers - .iter() - .any(|(configured, _)| configured == name) - { - headers.push((name.into(), value.to_str()?.into())); - } - } - Ok(HttpFetchRequest { - url: self.url.clone(), - method: self.method.clone(), - headers, - body_base64: STANDARD.encode(body), - tls: self.tls.clone(), - credential: self.credential.clone(), - max_response_bytes: hellas_rpc::http_fetch::MAX_HTTP_RESPONSE_BYTES, - }) - } -} - pub(super) async fn start(options: GatewayOptions) -> anyhow::Result { let config = options .http_fetch @@ -207,35 +70,23 @@ pub(super) async fn start(options: GatewayOptions) -> anyhow::Result anyhow::Result = state - .config - .routes + .routing + .backends .iter() - .map(|route| route.path.as_str()) + .flat_map(|backend| backend.routes.iter().map(|route| route.path.as_str())) .collect(); for path in paths { app = app.route(path, axum::routing::any(handle)); @@ -288,36 +139,8 @@ fn error(status: StatusCode, message: &'static str) -> Response { .into_response() } -fn limited(status: StatusCode, seconds: u64) -> Response { - let mut response = error( - status, - "provider temporarily unavailable; retry after the indicated delay", - ); - response.headers_mut().insert( - "retry-after", - HeaderValue::from_str(&seconds.max(1).to_string()).unwrap(), - ); - response -} - async fn handle(State(state): State>, request: Request) -> Response { - let Some(route) = state.config.routes.iter().find(|route| { - route.path == request.uri().path() && route.method == request.method().as_str() - }) else { - return error(StatusCode::NOT_FOUND, "no configured HTTP route"); - }; - let mut observed = observation::Observation::new(&state.metrics, &route.path); - let account = state.accounts[&route.account()].clone(); - if let Some((status, delay)) = account.cooldown() { - observed.status(status); - observed.complete(); - return limited(StatusCode::from_u16(status).unwrap(), delay.as_secs() + 1); - } - let Ok(permit) = account.slots.clone().try_acquire_owned() else { - observed.status(503); - observed.complete(); - return limited(StatusCode::SERVICE_UNAVAILABLE, 1); - }; + let mut observed = observation::Observation::new(&state.metrics, request.uri().path()); let (parts, body) = request.into_parts(); // Reserve space for URL, headers and JSON around the base64 body. let body_limit = (hellas_rpc::fetch::MAX_FETCH_REQUEST_BODY_BYTES - 64 * 1024) / 4 * 3; @@ -332,6 +155,45 @@ async fn handle(State(state): State>, request: Request) -> Respon ); } }; + let connection = parts + .extensions + .get::>() + .map(|c| c.0.0); + let selected = match state.routing.select( + parts.uri.path(), + parts.method.as_str(), + &parts.headers, + &body, + connection, + ) { + Ok(selected) => selected, + Err(failure) => { + observed.status(failure.status.as_u16()); + observed.complete(); + let mut response = error(failure.status, failure.message); + if let Some(seconds) = failure.retry { + response + .headers_mut() + .insert("retry-after", seconds.to_string().parse().unwrap()); + } + if let Some(backend) = failure.backend { + let name = &state.routing.backends[backend].name; + observed.backend(name, "session"); + return attributed(response, name); + } + return response; + } + }; + let backend = &state.routing.backends[selected.backend]; + let route = &backend.routes[selected.endpoint]; + let permit = selected.permit; + observed.backend(&backend.name, selected.affinity); + observed.model(selected.model.as_deref()); + observed.bind_response( + state + .routing + .response_binding(selected.backend, &route.path), + ); let mut upstream = match route.request(body, &parts.headers) { Ok(upstream) => upstream, Err(_) => { @@ -354,14 +216,18 @@ async fn handle(State(state): State>, request: Request) -> Respon return error(StatusCode::BAD_REQUEST, "invalid HTTP request"); } }; - let result = open(&state, &payload) + let result = open(&state, backend.remote.clone(), &payload) .instrument(observed.span.clone()) .await; let (status, headers, mut events) = match result { Ok(value) => value, Err(_) => { observed.status(502); - return error(StatusCode::BAD_GATEWAY, "authenticated Fetch failed"); + state.routing.transport_failed(selected.backend); + return attributed( + error(StatusCode::BAD_GATEWAY, "authenticated Fetch failed"), + &backend.name, + ); } }; observed.status(status); @@ -385,7 +251,7 @@ async fn handle(State(state): State>, request: Request) -> Respon .get("content-encoding") .and_then(|value| value.to_str().ok()), ); - account.observe(status, &headers); + state.routing.observe(selected.backend, status, &headers); if parts.method == axum::http::Method::HEAD || status == 204 || status == 304 { let end = events.next().instrument(observed.span.clone()).await; if !matches!( @@ -402,7 +268,7 @@ async fn handle(State(state): State>, request: Request) -> Respon let mut response = Response::new(Body::empty()); *response.status_mut() = StatusCode::from_u16(status).unwrap(); *response.headers_mut() = std::mem::take(&mut headers); - return response; + return attributed(response, &backend.name); } let stream = async_stream::try_stream! { let _permit = permit; @@ -434,11 +300,12 @@ async fn handle(State(state): State>, request: Request) -> Respon let mut response = Response::new(Body::from_stream(stream)); *response.status_mut() = StatusCode::from_u16(status).unwrap(); *response.headers_mut() = headers; - response + attributed(response, &backend.name) } async fn open( state: &HttpState, + remote: ExecutionRoute, payload: &[u8], ) -> anyhow::Result<( u16, @@ -463,7 +330,7 @@ async fn open( let mut stream = fetch_output_stream( state.runtime.clone(), request, - Some(state.route.clone()), + Some(remote), state.signer.clone(), None, ) diff --git a/crates/gateway/src/http_fetch/affinity.rs b/crates/gateway/src/http_fetch/affinity.rs new file mode 100644 index 00000000..8a85296e --- /dev/null +++ b/crates/gateway/src/http_fetch/affinity.rs @@ -0,0 +1,171 @@ +//! Read routing hints without changing the bytes forwarded upstream. +use super::*; +use serde_json::Value; +use std::io::Read; + +pub(super) struct Hints { + pub model: Option, + pub session: Option<(String, String)>, + pub previous: Option, + pub conversation: Option, +} + +impl Hints { + pub fn read(headers: &HeaderMap, body: &[u8]) -> Result { + let decoded; + let decode = |reader: &mut dyn Read| -> Result, &'static str> { + let mut output = Vec::new(); + reader + .take(8 * 1024 * 1024 + 1) + .read_to_end(&mut output) + .map_err(|_| "invalid compressed routing request")?; + if output.len() > 8 * 1024 * 1024 { + return Err("decoded routing request exceeds limit"); + } + Ok(output) + }; + let encoding = headers + .get("content-encoding") + .map(|v| v.to_str().map(|s| s.trim().to_ascii_lowercase())) + .transpose() + .map_err(|_| "invalid request encoding")?; + let bytes = match encoding.as_deref() { + None | Some("") | Some("identity") => body, + Some("gzip") => { + decoded = decode(&mut flate2::read::MultiGzDecoder::new(body))?; + &decoded + } + Some("zstd") => { + let mut decoder = zstd::stream::read::Decoder::new(body) + .map_err(|_| "invalid compressed routing request")?; + decoder + .window_log_max(23) + .map_err(|_| "invalid compressed routing request")?; + decoded = decode(&mut decoder)?; + &decoded + } + _ => return Err("unsupported request encoding for model routing"), + }; + let value = if bytes.is_empty() { + Value::Null + } else { + serde_json::from_slice::(bytes) + .map_err(|_| "model routing requires a JSON request")? + }; + fn field(value: Option<&Value>) -> Result, &'static str> { + match value { + None | Some(Value::Null) => Ok(None), + Some(Value::String(s)) if !s.is_empty() && s.len() <= 1024 => Ok(Some(s.clone())), + _ => Err("invalid model or session identifier"), + } + } + let mut session = None; + for name in [ + "x-hellas-session-id", + "x-claude-code-session-id", + "session-id", + "thread-id", + ] { + if let Some(value) = headers.get(name) { + if headers.get_all(name).iter().count() != 1 { + return Err("ambiguous session header"); + } + let value = value.to_str().map_err(|_| "invalid session header")?; + if value.is_empty() || value.len() > 1024 { + return Err("invalid session header"); + } + if session.is_none() { + session = Some((name.into(), value.into())); + } + } + } + if session.is_none() { + if let Some(user) = value.pointer("/metadata/user_id").and_then(Value::as_str) { + let id = serde_json::from_str::(user) + .ok() + .and_then(|v| { + v.get("session_id") + .and_then(Value::as_str) + .map(str::to_owned) + }) + .or_else(|| user.rsplit_once("_session_").map(|(_, id)| id.to_owned())); + if let Some(id) = id.filter(|s| !s.is_empty() && s.len() <= 1024) { + session = Some(("claude-session".into(), id)); + } + } + } + if session.is_none() { + session = field(value.get("prompt_cache_key"))?.map(|v| ("prompt-cache".into(), v)); + } + let conversation = value.get("conversation"); + Ok(Self { + model: field(value.get("model"))?, + session, + previous: field(value.get("previous_response_id"))?, + conversation: field( + conversation.and_then(|v| if v.is_object() { v.get("id") } else { Some(v) }), + )?, + }) + } +} + +pub(super) fn family(path: &str) -> &str { + path.strip_suffix("/compact") + .or_else(|| path.strip_suffix("/count_tokens")) + .unwrap_or(path) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn native_client_hints_and_compressed_json_are_read_without_rewriting() { + let body = br#"{"model":"k3","prompt_cache_key":"kimi-session"}"#; + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(body).unwrap(); + let wire = encoder.finish().unwrap(); + let headers = HeaderMap::from_iter([( + "content-encoding".parse().unwrap(), + HeaderValue::from_static("gzip"), + )]); + let hints = Hints::read(&headers, &wire).unwrap(); + assert_eq!(hints.model.as_deref(), Some("k3")); + assert_eq!(hints.session.unwrap().1, "kimi-session"); + let compressed = zstd::stream::encode_all(body.as_slice(), 1).unwrap(); + let headers = HeaderMap::from_iter([( + "content-encoding".parse().unwrap(), + HeaderValue::from_static("zstd"), + )]); + assert_eq!( + Hints::read(&headers, &compressed).unwrap().model.as_deref(), + Some("k3") + ); + let mut headers = HeaderMap::new(); + headers.insert("session-id", HeaderValue::from_static("codex-session")); + assert_eq!( + Hints::read(&headers, body).unwrap().session.unwrap().1, + "codex-session" + ); + let body=serde_json::to_vec(&serde_json::json!({"model":"claude", "metadata":{"user_id":"{\"device_id\":\"not-a-session\",\"session_id\":\"claude-session\"}"}})).unwrap(); + assert_eq!( + Hints::read(&HeaderMap::new(), &body) + .unwrap() + .session + .unwrap() + .1, + "claude-session" + ); + headers.append("session-id", HeaderValue::from_static("ambiguous")); + assert!(Hints::read(&headers, &body).is_err()); + assert!(Hints::read(&HeaderMap::new(), br#"{"model":4}"#).is_err()); + assert!( + Hints::read( + &HeaderMap::new(), + br#"{"model":"k3","previous_response_id":{}}"# + ) + .is_err() + ); + } +} diff --git a/crates/gateway/src/http_fetch/config.rs b/crates/gateway/src/http_fetch/config.rs new file mode 100644 index 00000000..055848b3 --- /dev/null +++ b/crates/gateway/src/http_fetch/config.rs @@ -0,0 +1,194 @@ +use super::*; + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpGatewayConfig { + pub service: String, + pub method: String, + #[serde(default)] + pub routes: Vec, + #[serde(default)] + pub backends: BTreeMap, + #[serde(default = "default_concurrency")] + pub max_in_flight: usize, +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpBackend { + pub models: Vec, + pub credential: Option, + pub routes: Vec, + pub max_in_flight: Option, + pub provider: Option, +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Provider { + pub node_id: iroh::EndpointId, + #[serde(default)] + pub node_addrs: Vec, + #[serde(deserialize_with = "genesis_from_hex")] + pub genesis: hellas_rpc::ContentId, +} + +fn genesis_from_hex<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result { + String::deserialize(deserializer)? + .parse() + .map_err(serde::de::Error::custom) +} + +fn default_concurrency() -> usize { + 4 +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HttpRoute { + pub path: String, + pub method: String, + pub url: String, + pub credential: Option, + #[serde(default = "public_tls")] + pub tls: HttpTls, + #[serde(default)] + pub headers: Vec<(String, String)>, +} + +pub(super) fn public_tls() -> HttpTls { + HttpTls { + roots: HttpTrustRoots::WebPki, + spki_sha256: vec![], + } +} + +impl HttpGatewayConfig { + pub(super) fn validate(&self) -> anyhow::Result<()> { + ensure!( + self.routes.is_empty() != self.backends.is_empty(), + "configure either HTTP backends or legacy routes" + ); + ensure!( + self.max_in_flight > 0 && self.max_in_flight <= 1024, + "invalid HTTP concurrency limit" + ); + if !self.routes.is_empty() { + return validate_routes(&self.routes); + } + ensure!(self.backends.len() <= 256, "too many HTTP backends"); + for (name, backend) in &self.backends { + ensure!(!name.is_empty() && name.len() <= 64, "invalid backend name"); + ensure!( + !backend.models.is_empty() + && backend.models.len() <= 256 + && backend + .models + .iter() + .all(|m| !m.is_empty() && m.len() <= 256), + "backend needs explicit model names" + ); + ensure!( + backend + .max_in_flight + .is_none_or(|n| (1..=1024).contains(&n)), + "invalid backend concurrency limit" + ); + ensure!( + backend.routes.iter().all(|r| r.credential.is_none()), + "set the credential on the backend, not its routes" + ); + validate_routes(&backend.routes)?; + } + Ok(()) + } +} + +fn validate_routes(routes: &[HttpRoute]) -> anyhow::Result<()> { + ensure!(!routes.is_empty(), "HTTP backend needs at least one route"); + let mut paths = std::collections::BTreeSet::new(); + for route in routes { + ensure!( + route.path.starts_with('/') && !route.path.contains(['?', '#', '{', '}']), + "HTTP routes must be exact paths" + ); + ensure!( + paths.insert((&route.path, &route.method)), + "duplicate HTTP route" + ); + ensure!( + route.headers.iter().all(|(name, _)| !matches!( + name.as_str(), + "authorization" | "x-api-key" | "cookie" + )), + "use a provider credential alias for authentication" + ); + route.request(Bytes::new(), &HeaderMap::new())?.validate()?; + } + Ok(()) +} + +impl HttpRoute { + pub(super) fn account(&self) -> String { + match &self.credential { + Some(alias) => format!("credential:{alias}"), + None => format!( + "origin:{}", + self.url + .parse::() + .expect("validated URL") + .origin() + .ascii_serialization() + ), + } + } + + pub(super) fn request( + &self, + body: Bytes, + incoming: &HeaderMap, + ) -> anyhow::Result { + let mut headers = self.headers.clone(); + let connection = connection_headers( + incoming + .iter() + .map(|(name, value)| (name.as_str(), value.to_str().unwrap_or_default())), + ); + for (name, value) in incoming { + let name = name.as_str(); + if !hop_header(name) + && !connection.iter().any(|token| token == name) + && !matches!( + name, + "host" + | "content-length" + | "authorization" + | "x-api-key" + | "api-key" + | "x-goog-api-key" + | "cookie" + | "forwarded" + ) + && !name.starts_with("x-hellas-") + && !name.starts_with("x-forwarded-") + && !self + .headers + .iter() + .any(|(configured, _)| configured == name) + { + headers.push((name.into(), value.to_str()?.into())); + } + } + Ok(HttpFetchRequest { + url: self.url.clone(), + method: self.method.clone(), + headers, + body_base64: STANDARD.encode(body), + tls: self.tls.clone(), + credential: self.credential.clone(), + max_response_bytes: hellas_rpc::http_fetch::MAX_HTTP_RESPONSE_BYTES, + }) + } +} diff --git a/crates/gateway/src/http_fetch/observation.rs b/crates/gateway/src/http_fetch/observation.rs index cdb6ece7..5c958718 100644 --- a/crates/gateway/src/http_fetch/observation.rs +++ b/crates/gateway/src/http_fetch/observation.rs @@ -55,6 +55,10 @@ pub(super) struct Observation { usage: Usage, compressed_usage: Option>, #[cfg(feature = "otel")] + backend: Option, + #[cfg(feature = "otel")] + model: Option, + #[cfg(feature = "otel")] route: String, #[cfg(feature = "otel")] metrics: Metrics, @@ -67,6 +71,9 @@ impl Observation { Self { span: hellas_rpc::request_span!(target: "hellas_request", "http.fetch", otel.kind = "client", http.route = route, + hellas.backend = tracing::field::Empty, + gen_ai.request.model = tracing::field::Empty, + hellas.routing.affinity = tracing::field::Empty, http.response.status_code = tracing::field::Empty, http.response.body.size = tracing::field::Empty, gen_ai.usage.input_tokens = tracing::field::Empty, @@ -84,6 +91,10 @@ impl Observation { usage: Usage::default(), compressed_usage: None, #[cfg(feature = "otel")] + backend: None, + #[cfg(feature = "otel")] + model: None, + #[cfg(feature = "otel")] route: route.into(), #[cfg(feature = "otel")] metrics: Metrics { @@ -94,6 +105,26 @@ impl Observation { }, } } + pub(super) fn backend(&mut self, name: &str, affinity: &'static str) { + self.span.record("hellas.backend", name); + self.span.record("hellas.routing.affinity", affinity); + #[cfg(feature = "otel")] + { + self.backend = Some(name.into()); + } + } + pub(super) fn bind_response(&mut self, binding: Option) { + self.usage.binding = binding; + } + pub(super) fn model(&mut self, model: Option<&str>) { + if let Some(model) = model { + self.span.record("gen_ai.request.model", model); + #[cfg(feature = "otel")] + { + self.model = Some(model.into()); + } + } + } pub(super) fn status(&mut self, status: u16) { self.status = status; self.span.record("http.response.status_code", status); @@ -179,11 +210,17 @@ impl Drop for Observation { #[cfg(feature = "otel")] { use opentelemetry::KeyValue; - let labels = [ + let mut labels = vec![ KeyValue::new("http.route", self.route.clone()), KeyValue::new("http.response.status_code", i64::from(self.status)), KeyValue::new("hellas.response.complete", self.complete), ]; + if let Some(backend) = &self.backend { + labels.push(KeyValue::new("hellas.backend", backend.clone())); + } + if let Some(model) = &self.model { + labels.push(KeyValue::new("gen_ai.request.model", model.clone())); + } self.metrics.requests.add(1, &labels); self.metrics .duration @@ -210,6 +247,7 @@ impl Drop for Observation { /// Observe standard usage fields without reserializing or delaying the wire body. #[derive(Default)] struct Usage { + binding: Option, sse: bool, pending: Vec, input: Option, @@ -271,6 +309,9 @@ impl Usage { let Ok(value) = serde_json::from_slice::(bytes) else { return; }; + if let Some(binding) = &self.binding { + binding.observe(&value); + } let Some(usage) = value .get("usage") .or_else(|| value.pointer("/message/usage")) diff --git a/crates/gateway/src/http_fetch/routing.rs b/crates/gateway/src/http_fetch/routing.rs new file mode 100644 index 00000000..30cb3d33 --- /dev/null +++ b/crates/gateway/src/http_fetch/routing.rs @@ -0,0 +1,474 @@ +use super::*; + +pub(super) struct Account { + pub(super) slots: Arc, + pub(super) backoff: Mutex>, +} + +impl Account { + pub(super) fn cooldown(&self) -> Option<(u16, Duration)> { + let (until, status) = (*self.backoff.lock().unwrap())?; + let delay = until.saturating_duration_since(Instant::now()); + (!delay.is_zero()).then_some((status, delay)) + } + + pub(super) fn observe(&self, status: u16, headers: &HeaderMap) { + if status == 429 || (status >= 500 && headers.contains_key("retry-after")) { + let delay = retry_delay(headers).min(Duration::from_secs(u32::MAX as u64)); + let until = Instant::now() + delay; + let mut backoff = self.backoff.lock().unwrap(); + if backoff.is_none_or(|(previous, _)| until > previous) { + *backoff = Some((until, status)); + } + } + } +} + +use super::affinity::{Hints, family}; +use hellas_rpc::ContentId; +use std::collections::HashMap; +use tokio::sync::OwnedSemaphorePermit; + +const MAX_BINDINGS: usize = 16_384; +const SESSION_IDLE: Duration = Duration::from_secs(24 * 60 * 60); + +pub(super) struct Backend { + pub name: String, + pub models: Vec, + pub routes: Vec, + pub remote: ExecutionRoute, + capacity: usize, + account: Arc, +} + +struct Binding { + backend: usize, + touched: Instant, +} + +#[derive(Default)] +struct State { + bindings: HashMap, + next: usize, +} + +pub(super) struct Routing { + pub backends: Vec, + pooled: bool, + salt: [u8; 32], + state: Mutex, +} + +pub(super) struct Selected { + pub backend: usize, + pub endpoint: usize, + pub permit: OwnedSemaphorePermit, + pub affinity: &'static str, + pub model: Option, +} + +#[derive(Debug)] +pub(super) struct Unavailable { + pub status: StatusCode, + pub message: &'static str, + pub retry: Option, + pub backend: Option, +} + +impl Unavailable { + fn conflict(message: &'static str) -> Self { + Self { + status: StatusCode::CONFLICT, + message, + retry: None, + backend: None, + } + } + fn busy(status: u16, delay: Duration) -> Self { + Self { + status: StatusCode::from_u16(status).unwrap(), + message: "selected backend is temporarily unavailable", + retry: Some(delay.as_secs() + 1), + backend: None, + } + } +} + +impl Routing { + pub fn new( + config: &HttpGatewayConfig, + remote: ExecutionRoute, + trust: hellas_client::ProviderTrustAnchor, + ) -> anyhow::Result { + let mut backends = Vec::new(); + let mut accounts: BTreeMap)> = BTreeMap::new(); + let mut add = |name: String, + models: Vec, + routes: Vec, + remote: ExecutionRoute, + limit: usize| + -> anyhow::Result<()> { + let account = routes[0].account(); + ensure!( + routes.iter().all(|r| r.account() == account), + "a backend must use one account or origin" + ); + // The enrollment pin identifies the provider even when one route + // uses discovery and another dials that provider directly. + let scope = match &remote { + ExecutionRoute::RemoteDirect(target) => { + target.provider_trust.expected_genesis.to_string() + } + ExecutionRoute::RemoteDiscovery { provider_trust, .. } => { + provider_trust.expected_genesis.to_string() + } + ExecutionRoute::Local => unreachable!(), + }; + let (configured_limit, shared) = accounts + .entry(format!("{scope}:{account}")) + .or_insert_with(|| { + ( + limit, + Arc::new(Account { + slots: Arc::new(Semaphore::new(limit)), + backoff: Mutex::new(None), + }), + ) + }); + ensure!( + *configured_limit == limit, + "shared accounts require the same concurrency limit" + ); + let shared = shared.clone(); + backends.push(Backend { + name, + models, + routes, + remote, + capacity: limit, + account: shared, + }); + Ok(()) + }; + if config.backends.is_empty() { + for (index, route) in config.routes.iter().enumerate() { + add( + format!("route-{index}"), + vec![], + vec![route.clone()], + remote.clone(), + config.max_in_flight, + )?; + } + } else { + for (name, backend) in &config.backends { + let remote = if let Some(provider) = &backend.provider { + let mut trust = trust.clone(); + trust.expected_genesis = provider.genesis; + ExecutionRoute::remote( + Some(provider.node_id), + provider.node_addrs.clone(), + 0, + trust, + ) + } else { + remote.clone() + }; + let routes = backend + .routes + .iter() + .cloned() + .map(|mut r| { + r.credential = backend.credential.clone(); + r + }) + .collect(); + add( + name.clone(), + backend.models.clone(), + routes, + remote, + backend.max_in_flight.unwrap_or(config.max_in_flight), + )?; + } + } + Ok(Self { + backends, + pooled: !config.backends.is_empty(), + salt: rand::random(), + state: Mutex::new(State::default()), + }) + } + + fn key(&self, domain: &str, parts: &[&str]) -> ContentId { + let mut fields: Vec<&[u8]> = vec![&self.salt]; + fields.extend(parts.iter().map(|s| s.as_bytes())); + ContentId::from_slice(hellas_rpc::hash_tuple(domain, &fields).as_bytes()).unwrap() + } + + pub fn select( + &self, + path: &str, + method: &str, + headers: &HeaderMap, + body: &[u8], + connection: Option, + ) -> Result { + let hints = if self.pooled { + Some(Hints::read(headers, body).map_err(|message| Unavailable { + status: StatusCode::BAD_REQUEST, + message, + retry: None, + backend: None, + })?) + } else { + None + }; + let model = hints.as_ref().and_then(|h| h.model.as_deref()); + let mut candidates = Vec::new(); + for (index, backend) in self.backends.iter().enumerate() { + if let Some(endpoint) = backend + .routes + .iter() + .position(|r| r.path == path && r.method == method) + { + if model.is_none_or(|model| { + backend.models.is_empty() || backend.models.iter().any(|m| m == model) + }) { + candidates.push((index, endpoint)); + } + } + } + if candidates.is_empty() { + return Err(Unavailable { + status: StatusCode::NOT_FOUND, + message: "no backend serves this route and model", + retry: None, + backend: None, + }); + } + if self.pooled + && model.is_none() + && !body.is_empty() + && !hints + .as_ref() + .is_some_and(|h| h.previous.is_some() || h.conversation.is_some()) + { + return Err(Unavailable { + status: StatusCode::BAD_REQUEST, + message: "model is required", + retry: None, + backend: None, + }); + } + let family = family(path); + let session = hints + .as_ref() + .and_then(|h| h.session.as_ref()) + .map(|(kind, id)| self.key("session", &[family, model.unwrap_or(""), kind, id])); + let connection_key = connection + .filter(|_| self.pooled && session.is_none()) + .map(|id| { + self.key( + "connection", + &[family, model.unwrap_or(""), &id.to_string()], + ) + }); + let affinity_key = session.or(connection_key); + let state_keys: Vec<_> = hints + .as_ref() + .into_iter() + .flat_map(|h| { + [ + h.previous + .as_ref() + .map(|id| self.key("response", &[family, id])), + h.conversation + .as_ref() + .map(|id| self.key("conversation", &[family, id])), + ] + }) + .flatten() + .collect(); + let mut state = self.state.lock().unwrap(); + let now = Instant::now(); + state + .bindings + .retain(|_, b| now.duration_since(b.touched) < SESSION_IDLE); + let mut pinned = None; + for key in &state_keys { + let binding = state.bindings.get_mut(key).ok_or_else(|| { + Unavailable::conflict( + "unknown or expired server-side continuation; resend full context", + ) + })?; + binding.touched = now; + if pinned.is_some_and(|p| p != binding.backend) { + return Err(Unavailable::conflict("conflicting continuation backends")); + } + pinned = Some(binding.backend); + } + if let Some(binding) = affinity_key.and_then(|k| state.bindings.get_mut(&k)) { + binding.touched = now; + if pinned.is_some_and(|p| p != binding.backend) { + return Err(Unavailable::conflict( + "session and continuation refer to different backends", + )); + } + pinned = Some(binding.backend); + } + if let Some(index) = pinned { + candidates.retain(|(backend, _)| *backend == index); + if candidates.is_empty() { + return Err(Unavailable::conflict( + "session backend cannot serve this model and route", + )); + } + } + if affinity_key.is_some_and(|key| !state.bindings.contains_key(&key)) + && state.bindings.len() >= MAX_BINDINGS + { + return Err(Unavailable::busy(503, Duration::ZERO)); + } + let count = self.backends.len(); + candidates.sort_by_key(|(index, _)| { + let backend = &self.backends[*index]; + // Least active first, rotating equal candidates. Shared account permits + // also cover aliases used by multiple model configurations. + ( + (backend.capacity - backend.account.slots.available_permits()) * 1024 + / backend.capacity, + (*index + count - state.next) % count, + ) + }); + let mut unavailable = None; + for (backend, endpoint) in candidates { + let account = &self.backends[backend].account; + if let Some((status, delay)) = account.cooldown() { + if unavailable.as_ref().is_none_or(|(_, old)| delay < *old) { + unavailable = Some((status, delay)); + } + continue; + } + let Ok(permit) = account.slots.clone().try_acquire_owned() else { + unavailable = Some((503, Duration::ZERO)); + continue; + }; + for key in state_keys.iter().copied().chain(affinity_key) { + state.bindings.insert( + key, + Binding { + backend, + touched: now, + }, + ); + } + state.next = (backend + 1) % count; + return Ok(Selected { + backend, + endpoint, + permit, + model: model.map(str::to_owned), + affinity: if !state_keys.is_empty() { + "continuation" + } else if pinned.is_some() && connection_key.is_some() { + "connection" + } else if pinned.is_some() { + "session" + } else { + "new" + }, + }); + } + let (status, delay) = unavailable.unwrap_or((503, Duration::ZERO)); + let mut failure = Unavailable::busy(status, delay); + failure.backend = pinned; + Err(failure) + } + + pub fn observe(&self, backend: usize, status: u16, headers: &HeaderMap) { + self.backends[backend].account.observe(status, headers); + } + + pub fn transport_failed(&self, backend: usize) { + if self.pooled { + self.observe( + backend, + 503, + &HeaderMap::from_iter([( + "retry-after".parse().unwrap(), + HeaderValue::from_static("1"), + )]), + ); + } + } + + pub fn response_binding( + self: &Arc, + backend: usize, + path: &str, + ) -> Option { + (self.pooled && family(path).ends_with("/responses")).then(|| ResponseBinding { + routing: self.clone(), + backend, + family: family(path).into(), + }) + } +} + +#[derive(Clone)] +pub(super) struct ResponseBinding { + routing: Arc, + backend: usize, + family: String, +} + +impl ResponseBinding { + pub fn observe(&self, value: &serde_json::Value) { + let response = value.get("response").unwrap_or(value); + for (kind, id) in [ + ( + "response", + response + .get("id") + .and_then(serde_json::Value::as_str) + .filter(|_| { + response.get("object").and_then(serde_json::Value::as_str) + == Some("response") + }), + ), + ( + "conversation", + response.get("conversation").and_then(|v| { + v.as_str() + .or_else(|| v.get("id").and_then(serde_json::Value::as_str)) + }), + ), + ] { + let Some(id) = id.filter(|id| !id.is_empty() && id.len() <= 1024) else { + continue; + }; + let key = self.routing.key(kind, &[&self.family, id]); + let mut state = self.routing.state.lock().unwrap(); + // Never silently overwrite another account's identifier or evict a + // live session. An unrecorded continuation will fail closed later. + if let Some(existing) = state.bindings.get_mut(&key) { + if existing.backend != self.backend { + existing.backend = usize::MAX; + } + existing.touched = Instant::now(); + } else if state.bindings.len() < MAX_BINDINGS { + state.bindings.insert( + key, + Binding { + backend: self.backend, + touched: Instant::now(), + }, + ); + } + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/gateway/src/http_fetch/routing/tests.rs b/crates/gateway/src/http_fetch/routing/tests.rs new file mode 100644 index 00000000..53e86c59 --- /dev/null +++ b/crates/gateway/src/http_fetch/routing/tests.rs @@ -0,0 +1,310 @@ +use super::*; +use serde_json::json; + +fn config() -> HttpGatewayConfig { + let backend = |credential: &str, models: &[&str]| { + json!({ + "credential":credential, "models":models, + "routes":[{"path":"/v1/chat/completions","method":"POST","url":"https://example.com/v1/chat/completions"}, + {"path":"/v1/responses","method":"POST","url":"https://example.com/v1/responses"}] + }) + }; + serde_json::from_value(json!({"service":"http","method":"request","max_in_flight":1, + "backends":{"a":backend("a", &["k3"]),"b":backend("b", &["k3"]),"c":backend("c", &["other"])}})).unwrap() +} + +fn routing(config: HttpGatewayConfig) -> Arc { + config.validate().unwrap(); + let trust = hellas_client::ProviderTrustAnchor { + expected_genesis: ContentId::hash(b"provider"), + required_assurance: Assurance::ProducerSigned, + apple_app_attest: None, + }; + Arc::new( + Routing::new( + &config, + ExecutionRoute::remote(None, vec![], 0, trust.clone()), + trust, + ) + .unwrap(), + ) +} + +fn select(routing: &Routing, model: &str, session: &str) -> Result { + routing.select( + "/v1/chat/completions", + "POST", + &HeaderMap::new(), + &serde_json::to_vec(&json!({"model":model,"prompt_cache_key":session})).unwrap(), + None, + ) +} + +#[test] +fn models_admission_and_cooldowns_do_not_move_existing_sessions() { + let routing = routing(config()); + let first = select(&routing, "k3", "first").unwrap(); + assert_eq!(first.backend, 0); + assert_eq!( + select(&routing, "k3", "first").err().unwrap().status, + StatusCode::SERVICE_UNAVAILABLE + ); + let second = select(&routing, "k3", "second").unwrap(); + assert_eq!(second.backend, 1); + assert_eq!( + select(&routing, "k3", "third").err().unwrap().status, + StatusCode::SERVICE_UNAVAILABLE + ); + assert_eq!(select(&routing, "other", "first").unwrap().backend, 2); + assert_eq!( + select(&routing, "unconfigured", "first") + .err() + .unwrap() + .status, + StatusCode::NOT_FOUND + ); + drop((first, second)); + let rate_limit = HeaderMap::from_iter([( + "retry-after".parse().unwrap(), + HeaderValue::from_static("60"), + )]); + routing.observe(0, 429, &rate_limit); + for binding in routing.state.lock().unwrap().bindings.values_mut() { + binding.touched = Instant::now() - SESSION_IDLE / 2; + } + let pinned = select(&routing, "k3", "first").err().unwrap(); + assert!( + routing + .state + .lock() + .unwrap() + .bindings + .values() + .any(|b| b.backend == 0 && b.touched.elapsed() < Duration::from_secs(1)), + "rate-limited retries still keep a session active" + ); + assert_eq!(pinned.status, StatusCode::TOO_MANY_REQUESTS); + assert!((59..=61).contains(&pinned.retry.unwrap())); + assert_eq!(select(&routing, "k3", "third").unwrap().backend, 1); + *routing.backends[0].account.backoff.lock().unwrap() = None; + assert_eq!(select(&routing, "k3", "first").unwrap().backend, 0); +} + +#[test] +fn connection_is_a_fallback_and_explicit_sessions_survive_reconnects() { + let routing = routing(config()); + let body = br#"{"model":"k3"}"#; + let request = |connection, headers: &HeaderMap| { + routing + .select( + "/v1/chat/completions", + "POST", + headers, + body, + Some(connection), + ) + .unwrap() + }; + let empty = HeaderMap::new(); + assert_eq!(request(1, &empty).backend, 0); + assert_eq!(request(1, &empty).backend, 0); + assert_eq!(request(2, &empty).backend, 1); + let headers = HeaderMap::from_iter([( + "session-id".parse().unwrap(), + HeaderValue::from_static("new-session"), + )]); + assert_eq!(request(2, &headers).backend, 0); + assert_eq!(request(3, &headers).backend, 0); +} + +#[test] +fn concurrent_first_requests_bind_atomically_and_release_capacity() { + let mut config = config(); + config.max_in_flight = 32; + let routing = routing(config); + let threads: Vec<_> = (0..16) + .map(|_| { + let routing = routing.clone(); + std::thread::spawn(move || select(&routing, "k3", "same-session").unwrap()) + }) + .collect(); + let requests: Vec<_> = threads.into_iter().map(|t| t.join().unwrap()).collect(); + assert!(requests.iter().all(|r| r.backend == requests[0].backend)); + let backend = requests[0].backend; + assert_eq!( + routing.backends[backend].account.slots.available_permits(), + 16 + ); + drop(requests); + assert_eq!( + routing.backends[backend].account.slots.available_permits(), + 32 + ); +} + +#[test] +fn server_state_uses_its_original_backend_and_unknown_or_conflicting_state_fails_closed() { + let routing = routing(config()); + let request = |value| { + routing.select( + "/v1/responses", + "POST", + &HeaderMap::new(), + &serde_json::to_vec(&value).unwrap(), + None, + ) + }; + let first = request(json!({"model":"k3","prompt_cache_key":"first"})).unwrap(); + let binding = routing + .response_binding(first.backend, "/v1/responses") + .unwrap(); + binding.observe( + &json!({"response":{"object":"response","id":"resp_a","conversation":{"id":"conv_a"}}}), + ); + drop(first); + assert_eq!( + request(json!({"previous_response_id":"resp_a"})) + .unwrap() + .backend, + 0 + ); + assert_eq!( + request(json!({"model":"k3","previous_response_id":"resp_a"})) + .unwrap() + .backend, + 0 + ); + assert_eq!( + request(json!({"model":"k3","conversation":"conv_a"})) + .unwrap() + .backend, + 0 + ); + assert_eq!( + request(json!({"model":"k3","previous_response_id":"unknown"})) + .err() + .unwrap() + .status, + StatusCode::CONFLICT + ); + assert_eq!( + request(json!({"model":"k3","prompt_cache_key":"second"})) + .unwrap() + .backend, + 1 + ); + assert_eq!( + request(json!({"model":"k3","prompt_cache_key":"second","previous_response_id":"resp_a"})) + .err() + .unwrap() + .status, + StatusCode::CONFLICT + ); + routing + .response_binding(1, "/v1/responses") + .unwrap() + .observe(&json!({"object":"response","id":"resp_a"})); + assert_eq!( + request(json!({"model":"k3","previous_response_id":"resp_a"})) + .err() + .unwrap() + .status, + StatusCode::CONFLICT + ); +} + +#[test] +fn full_affinity_table_does_not_evict_live_sessions_and_expired_server_state_is_rejected() { + let routing = routing(config()); + assert_eq!(select(&routing, "k3", "keep").unwrap().backend, 0); + { + let mut state = routing.state.lock().unwrap(); + for i in 1..MAX_BINDINGS { + state.bindings.insert( + ContentId::hash(&i.to_le_bytes()), + Binding { + backend: 1, + touched: Instant::now(), + }, + ); + } + } + assert_eq!(select(&routing, "k3", "keep").unwrap().backend, 0); + assert_eq!( + select(&routing, "k3", "new").err().unwrap().status, + StatusCode::SERVICE_UNAVAILABLE + ); + { + let mut state = routing.state.lock().unwrap(); + for binding in state.bindings.values_mut() { + binding.touched = Instant::now() - SESSION_IDLE - Duration::from_secs(1); + } + } + assert!(select(&routing, "k3", "new").is_ok()); + routing + .response_binding(0, "/v1/responses") + .unwrap() + .observe(&json!({"object":"response","id":"expired"})); + let key = routing.key("response", &["/v1/responses", "expired"]); + routing + .state + .lock() + .unwrap() + .bindings + .get_mut(&key) + .unwrap() + .touched = Instant::now() - SESSION_IDLE - Duration::from_secs(1); + assert_eq!( + routing + .select( + "/v1/responses", + "POST", + &HeaderMap::new(), + br#"{"model":"k3","previous_response_id":"expired"}"#, + None + ) + .err() + .unwrap() + .status, + StatusCode::CONFLICT + ); +} + +#[test] +fn repeated_aliases_share_limits_but_distinct_provider_nodes_keep_them_separate() { + let mut config = config(); + config.backends.get_mut("b").unwrap().credential = Some("a".into()); + let shared = routing(config.clone()); + let _request = select(&shared, "k3", "first").unwrap(); + assert_eq!( + select(&shared, "k3", "second").err().unwrap().status, + StatusCode::SERVICE_UNAVAILABLE + ); + let node = iroh::SecretKey::generate().public(); + config.backends.get_mut("b").unwrap().provider = Some(config::Provider { + node_id: node, + node_addrs: vec![], + genesis: ContentId::hash(b"other"), + }); + let mut same_provider = config.clone(); + same_provider + .backends + .get_mut("b") + .unwrap() + .provider + .as_mut() + .unwrap() + .genesis = ContentId::hash(b"provider"); + let same_provider = routing(same_provider); + let _first = select(&same_provider, "k3", "first").unwrap(); + assert_eq!( + select(&same_provider, "k3", "second").err().unwrap().status, + StatusCode::SERVICE_UNAVAILABLE + ); + let split = routing(config); + let _first = select(&split, "k3", "first").unwrap(); + assert_eq!(select(&split, "k3", "second").unwrap().backend, 1); + assert!( + matches!(&split.backends[1].remote, ExecutionRoute::RemoteDirect(target) if target.addr.id==node && target.provider_trust.expected_genesis==ContentId::hash(b"other")) + ); +} diff --git a/crates/gateway/src/http_fetch/tests.rs b/crates/gateway/src/http_fetch/tests.rs index e330df59..ed331681 100644 --- a/crates/gateway/src/http_fetch/tests.rs +++ b/crates/gateway/src/http_fetch/tests.rs @@ -1,3 +1,5 @@ +use super::config::public_tls; +use super::routing::Account; use super::*; #[test] @@ -107,6 +109,7 @@ fn credentials_and_hop_headers_cannot_be_configured_as_static_headers() { service: "http".into(), method: "request".into(), max_in_flight: 2, + backends: BTreeMap::new(), routes: vec![HttpRoute { path: "/v1/messages".into(), method: "POST".into(), diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index a84d2e4f..49d60ba1 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -89,6 +89,19 @@ pub trait PaidExecutionBackend: Send + Sync { static NEXT_ID: AtomicU64 = AtomicU64::new(1); +#[derive(Clone, Copy)] +struct ConnectionId(u64); + +impl + axum::extract::connect_info::Connected> + for ConnectionId +{ + fn connect_info(_: axum::serve::IncomingStream<'_, tokio::net::TcpListener>) -> Self { + static NEXT_CONNECTION: AtomicU64 = AtomicU64::new(1); + Self(NEXT_CONNECTION.fetch_add(1, Ordering::Relaxed)) + } +} + pub struct GatewayOptions { pub archive: ArchiveOptions, pub http_fetch: Option, @@ -370,7 +383,11 @@ async fn launch_gateway( let shutdown = Arc::new(tokio::sync::Notify::new()); let server_shutdown = shutdown.clone(); let server = std::future::IntoFuture::into_future( - axum::serve(listener, app).with_graceful_shutdown(async move { + axum::serve( + listener, + app.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { server_shutdown.notified().await; }), ); diff --git a/docs/http-gateway.md b/docs/http-gateway.md index a2c848a2..f97a1724 100644 --- a/docs/http-gateway.md +++ b/docs/http-gateway.md @@ -6,39 +6,65 @@ including SSE, tool calls and non-2xx errors. It requires a provider trust ancho and a provider route with a caller grant; it does not use the token-native paid pool. See [provider account and egress configuration](../crates/providers/HTTPS.md). -Example gateway configuration for Kimi Code: +The gateway's standard API paths can serve several accounts or provider nodes. +It reads the requested model, finds backends configured to serve that model and +route, and keeps an existing session on its assigned backend. Model names and +request/response bodies are forwarded unchanged. + +Example with two Kimi accounts behind one endpoint: ```json { "service": "https", "method": "request", "max_in_flight": 2, - "routes": [{ - "path": "/v1/chat/completions", - "method": "POST", - "url": "https://api.kimi.com/coding/v1/chat/completions", - "credential": "kimi" - }] + "backends": { + "kimi-a": { + "models": ["k3"], + "credential": "kimi-a", + "routes": [{ + "path": "/v1/chat/completions", + "method": "POST", + "url": "https://api.kimi.com/coding/v1/chat/completions" + }] + }, + "kimi-b": { + "models": ["k3"], + "credential": "kimi-b", + "routes": [{ + "path": "/v1/chat/completions", + "method": "POST", + "url": "https://api.kimi.com/coding/v1/chat/completions" + }] + } + } } ``` -The service and method must match the provider's configured Fetch route. The -credential alias belongs to that provider. Each alias shares a concurrency -limit and cooldown across its HTTP routes. Paths and methods match exactly; -query parameters retain their order, repeats and percent encoding. Ordinary -client headers, including idempotency keys and vendor extensions, pass through. -Connection-specific headers, credentials, cookies and gateway control headers -are removed; the caller's gateway bearer never becomes an upstream credential. Optional -`headers` holds operator-supplied `[name, value]` pairs, with lowercase names. -Configured values override the corresponding client header. The former -`forward_headers` whitelist has been removed. - -Several accounts at one upstream can use distinct aliases and gateway paths, -such as `/kimi-a/v1/chat/completions` and `/kimi-b/v1/chat/completions`, both -mapped to the upstream's chat endpoint. The client's base URL selects the route. -Each alias has independent admission and cooldown; routes sharing an alias -share those limits. This gateway targets one Hellas provider node. It does not -automatically select accounts, fail over between them or route using quota data. +Both accounts use the same client base URL, such as `http://127.0.0.1:8080/v1`. +Add backends with their exact supported models and API routes for other accounts. +The configured service and method must match the provider's Fetch route. An +optional backend `max_in_flight` overrides the global per-account limit. Routes +sharing a credential alias on the same provider share capacity and cooldown; +duplicating that alias does not multiply its allowance. + +Backends use the CLI's provider target by default. An optional backend `provider` +selects another node with `node_id`, `node_addrs` (an array of `IP:port` strings) +and `genesis` (the hex enrollment ContentId). The gateway's assurance and Apple +trust policy still apply; each node is verified against its configured enrollment +pin. Each provider must authorize this gateway's caller key and expose the +configured Fetch service/method. + +Paths and methods match exactly; query parameters retain their order, repeats +and percent encoding. Ordinary client headers, including idempotency keys and +vendor extensions, pass through. Connection-specific headers, credentials, +cookies and gateway control headers are removed; the caller's gateway bearer +never becomes an upstream credential. Optional route `headers` holds operator +`[name, value]` pairs, with lowercase names. Configured values override the +corresponding client header. Account credentials belong on the backend. + +The earlier top-level `routes` configuration remains supported for opaque HTTP +passthrough, without model selection. Use either `backends` or `routes`, not both. Omit `credential` for an unauthenticated upstream. Such routes share admission by origin. Optional `tls` uses the [Fetch TLS vocabulary](../crates/providers/HTTPS.md) @@ -64,13 +90,50 @@ the wrapper supplies `KIMI_MODEL_BASE_URL` and `KIMI_MODEL_API_KEY`. A separatel launched client can use the same variables and the private gateway bearer file. Choose a context size that fits the byte limits below. +## Model selection and session affinity + +New sessions use an eligible account outside cooldown with available capacity. +The gateway chooses the least occupied account, rotating ties. Existing sessions +stay pinned even when that account becomes busy or rate-limited: they receive a +retry response; new sessions can use another account. A request is sent once. +Neither a transport failure nor an upstream error causes automatic replay on a +different backend. A transport failure briefly excludes that backend from new +sessions. Quota polling remains monitoring; selection uses configured model +capabilities, current capacity and observed response cooldowns, not estimates +from token usage or external quota scrapes. + +Affinity recognizes `x-hellas-session-id`, Claude Code's +`x-claude-code-session-id`, Codex's `session-id`/`thread-id`, Claude's session ID +in `metadata.user_id`, and `prompt_cache_key` (used by Kimi Code), in that order. +An explicit session survives changes of client connection. Without one, requests +for the same model/API on the same accepted HTTP connection stay together. +Different explicit sessions may share that connection. Affinity is scoped by +model and API family; Responses compaction and Messages token counting share +their generation API's family. + +Responses API IDs observed in JSON or SSE bind `previous_response_id` and +`conversation` continuations to the original backend, including continuations +that omit the model. Unknown, expired or +conflicting state returns 409 before contacting an upstream. Session identifiers +are held as salted hashes in a bounded memory table (16,384 entries, 24 hours +idle). Unexpired entries are not evicted to admit new sessions; a full table +returns 503. Restarting the gateway clears affinity. Server-side continuations +whose mapping was lost require full context rather than being sent to a guessed +account. Full-history requests can establish new affinity after a restart. + +Model routing reads JSON, gzip or zstd requests with an 8 MiB decoded limit and +an 8 MiB zstd window limit. It forwards the original encoded bytes. An unsupported +model receives 404; malformed or unsupported routing input receives 400. These +checks precede upstream execution. Model-less, empty-body auxiliary requests use +an eligible backend; account-specific quota endpoints are not aggregated. + ## Archives and ZDR CLI gateways archive authenticated requests and responses by default under `~/.hellas/gateway-archive`, or `--archive-dir DIRECTORY`. This applies to the existing inference routes as well as HTTP Fetch. Each exchange has owner-only `request.bin`, `response.bin` and `metadata.json` files. Metadata records status, -size, content type/encoding, elapsed time, completion and trace context; it excludes authentication +size, content type/encoding, elapsed time, completion, selected backend and trace context; it excludes authentication headers. `x-hellas-request-id` identifies the exchange. Failed or cancelled streams retain an incomplete archive. Archiving is best-effort: failures during setup, request/response writes or finalization are reported without replacing @@ -126,6 +189,9 @@ concurrency returns 503 with `Retry-After: 1`. With `otel`, traces connect HTTP ingress, Fetch RPCs, credential refresh and upstream HTTP. Span attributes exclude request/response bodies and credentials. +`hellas.backend` and `gen_ai.request.model` identify configured backend/model +choices in request metrics and traces; `hellas.routing.affinity` records the +selection reason in traces. Raw session IDs are not exported. Outbound trace context replaces the caller's propagation headers instead of appending duplicates. Upstream response `traceparent` and `tracestate` are preserved; `x-hellas-trace-id` identifies the gateway's trace independently. From 462fa52e979bf20fb61a545bf00afad8eed1107f Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Thu, 24 Sep 2026 23:07:39 +0200 Subject: [PATCH 10/32] fix(gateway): reclaim affinity for closed client connections --- crates/gateway/src/http_fetch.rs | 2 +- crates/gateway/src/http_fetch/routing.rs | 33 +++++++++++------- .../gateway/src/http_fetch/routing/tests.rs | 34 +++++++++++++++++-- crates/gateway/src/lib.rs | 12 +++++-- docs/http-gateway.md | 3 +- 5 files changed, 65 insertions(+), 19 deletions(-) diff --git a/crates/gateway/src/http_fetch.rs b/crates/gateway/src/http_fetch.rs index 1fc73266..4c4369a9 100644 --- a/crates/gateway/src/http_fetch.rs +++ b/crates/gateway/src/http_fetch.rs @@ -158,7 +158,7 @@ async fn handle(State(state): State>, request: Request) -> Respon let connection = parts .extensions .get::>() - .map(|c| c.0.0); + .map(|c| &c.0); let selected = match state.routing.select( parts.uri.path(), parts.method.as_str(), diff --git a/crates/gateway/src/http_fetch/routing.rs b/crates/gateway/src/http_fetch/routing.rs index 30cb3d33..510830dd 100644 --- a/crates/gateway/src/http_fetch/routing.rs +++ b/crates/gateway/src/http_fetch/routing.rs @@ -44,6 +44,7 @@ pub(super) struct Backend { struct Binding { backend: usize, touched: Instant, + connection: Option>, } #[derive(Default)] @@ -212,7 +213,7 @@ impl Routing { method: &str, headers: &HeaderMap, body: &[u8], - connection: Option, + connection: Option<&crate::ConnectionId>, ) -> Result { let hints = if self.pooled { Some(Hints::read(headers, body).map_err(|message| Unavailable { @@ -266,14 +267,15 @@ impl Routing { .as_ref() .and_then(|h| h.session.as_ref()) .map(|(kind, id)| self.key("session", &[family, model.unwrap_or(""), kind, id])); - let connection_key = connection - .filter(|_| self.pooled && session.is_none()) - .map(|id| { - self.key( - "connection", - &[family, model.unwrap_or(""), &id.to_string()], - ) - }); + let connection_key = + connection + .filter(|_| self.pooled && session.is_none()) + .map(|connection| { + self.key( + "connection", + &[family, model.unwrap_or(""), &connection.id.to_string()], + ) + }); let affinity_key = session.or(connection_key); let state_keys: Vec<_> = hints .as_ref() @@ -292,9 +294,10 @@ impl Routing { .collect(); let mut state = self.state.lock().unwrap(); let now = Instant::now(); - state - .bindings - .retain(|_, b| now.duration_since(b.touched) < SESSION_IDLE); + state.bindings.retain(|_, b| { + now.duration_since(b.touched) < SESSION_IDLE + && b.connection.as_ref().is_none_or(|c| c.strong_count() > 0) + }); let mut pinned = None; for key in &state_keys { let binding = state.bindings.get_mut(key).ok_or_else(|| { @@ -360,6 +363,11 @@ impl Routing { Binding { backend, touched: now, + connection: if Some(key) == connection_key { + connection.map(|c| Arc::downgrade(&c.alive)) + } else { + None + }, }, ); } @@ -463,6 +471,7 @@ impl ResponseBinding { Binding { backend: self.backend, touched: Instant::now(), + connection: None, }, ); } diff --git a/crates/gateway/src/http_fetch/routing/tests.rs b/crates/gateway/src/http_fetch/routing/tests.rs index 53e86c59..04dc5fab 100644 --- a/crates/gateway/src/http_fetch/routing/tests.rs +++ b/crates/gateway/src/http_fetch/routing/tests.rs @@ -94,14 +94,20 @@ fn models_admission_and_cooldowns_do_not_move_existing_sessions() { fn connection_is_a_fallback_and_explicit_sessions_survive_reconnects() { let routing = routing(config()); let body = br#"{"model":"k3"}"#; - let request = |connection, headers: &HeaderMap| { + let connections: Vec<_> = (0..4) + .map(|id| crate::ConnectionId { + id, + alive: Arc::new(()), + }) + .collect(); + let request = |connection: usize, headers: &HeaderMap| { routing .select( "/v1/chat/completions", "POST", headers, body, - Some(connection), + Some(&connections[connection]), ) .unwrap() }; @@ -225,6 +231,7 @@ fn full_affinity_table_does_not_evict_live_sessions_and_expired_server_state_is_ Binding { backend: 1, touched: Instant::now(), + connection: None, }, ); } @@ -308,3 +315,26 @@ fn repeated_aliases_share_limits_but_distinct_provider_nodes_keep_them_separate( matches!(&split.backends[1].remote, ExecutionRoute::RemoteDirect(target) if target.addr.id==node && target.provider_trust.expected_genesis==ContentId::hash(b"other")) ); } + +#[test] +fn closed_client_connections_release_their_affinity_entries() { + let routing = routing(config()); + for id in 0..32 { + let connection = crate::ConnectionId { + id, + alive: Arc::new(()), + }; + routing + .select( + "/v1/chat/completions", + "POST", + &HeaderMap::new(), + br#"{"model":"k3"}"#, + Some(&connection), + ) + .unwrap(); + assert_eq!(routing.state.lock().unwrap().bindings.len(), 1); + } + select(&routing, "k3", "explicit-session").unwrap(); + assert_eq!(routing.state.lock().unwrap().bindings.len(), 1); +} diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index 49d60ba1..dc0e52a5 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -89,8 +89,11 @@ pub trait PaidExecutionBackend: Send + Sync { static NEXT_ID: AtomicU64 = AtomicU64::new(1); -#[derive(Clone, Copy)] -struct ConnectionId(u64); +#[derive(Clone)] +struct ConnectionId { + id: u64, + alive: Arc<()>, +} impl axum::extract::connect_info::Connected> @@ -98,7 +101,10 @@ impl { fn connect_info(_: axum::serve::IncomingStream<'_, tokio::net::TcpListener>) -> Self { static NEXT_CONNECTION: AtomicU64 = AtomicU64::new(1); - Self(NEXT_CONNECTION.fetch_add(1, Ordering::Relaxed)) + Self { + id: NEXT_CONNECTION.fetch_add(1, Ordering::Relaxed), + alive: Arc::new(()), + } } } diff --git a/docs/http-gateway.md b/docs/http-gateway.md index f97a1724..2abcf529 100644 --- a/docs/http-gateway.md +++ b/docs/http-gateway.md @@ -107,7 +107,8 @@ Affinity recognizes `x-hellas-session-id`, Claude Code's in `metadata.user_id`, and `prompt_cache_key` (used by Kimi Code), in that order. An explicit session survives changes of client connection. Without one, requests for the same model/API on the same accepted HTTP connection stay together. -Different explicit sessions may share that connection. Affinity is scoped by +Connection-only bindings are reclaimed once that connection closes. Different +explicit sessions may share that connection. Affinity is scoped by model and API family; Responses compaction and Messages token counting share their generation API's family. From 62052085d1881585fd4e5ce300da59ce93c6e95a Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Thu, 24 Sep 2026 23:38:22 +0200 Subject: [PATCH 11/32] refactor: simplify gateway routing and provider helpers Centralize account admission and session binding, normalize backend setup, and represent conflicting continuation IDs explicitly. Snapshot account load before sorting while keeping first-session assignment atomic. Remove duplicate stored configuration, connection-cache lookup logic, metric cloning and paid-provider RPC wrappers. Preserve opaque routes, strict session affinity, byte forwarding and best-effort archives. Validation: 334 Rust tests; 63 gateway tests without telemetry; 149 HTTP comparisons; 20 two-provider routing checks; native Kimi, Codex and Claude tool workflows against owned fixtures. Strict clippy, format, CLI check and release build passed. --- crates/gateway/src/http_fetch.rs | 50 +- crates/gateway/src/http_fetch/affinity.rs | 30 +- crates/gateway/src/http_fetch/config.rs | 10 +- crates/gateway/src/http_fetch/observation.rs | 8 +- crates/gateway/src/http_fetch/routing.rs | 537 +++++++++--------- .../gateway/src/http_fetch/routing/tests.rs | 74 ++- crates/gateway/src/http_fetch/tests.rs | 42 +- crates/providers/src/http/clients.rs | 10 +- crates/providers/src/http/config.rs | 24 +- crates/sdk/src/paid_provider.rs | 72 +-- docs/http-gateway.md | 4 +- 11 files changed, 421 insertions(+), 440 deletions(-) diff --git a/crates/gateway/src/http_fetch.rs b/crates/gateway/src/http_fetch.rs index 4c4369a9..00e3051c 100644 --- a/crates/gateway/src/http_fetch.rs +++ b/crates/gateway/src/http_fetch.rs @@ -5,7 +5,6 @@ mod observation; mod routing; pub use config::HttpGatewayConfig; -use config::HttpRoute; #[cfg(test)] mod tests; @@ -17,21 +16,13 @@ use axum::{ http::{HeaderMap, HeaderValue, StatusCode}, response::{IntoResponse, Response}, }; -use base64::{Engine as _, engine::general_purpose::STANDARD}; use futures::StreamExt; use hellas_client::{ExecutionRoute, cache::fetch_output_stream}; use hellas_rpc::{ Assurance, FetchEnvironment, ProducerSigningKey, Retention, - http_fetch::{HttpFetchRequest, HttpTls, HttpTrustRoots}, output::{AdaptorEvent, HttpResponseEvent, OutputEvent, StopReason}, }; -use serde::Deserialize; -use std::{ - collections::BTreeMap, - sync::{Arc, Mutex}, - time::{Duration, Instant, SystemTime}, -}; -use tokio::sync::Semaphore; +use std::sync::Arc; use tracing::Instrument; use super::{GatewayHandle, GatewayOptions, access, execution::CliRuntime}; @@ -45,7 +36,8 @@ fn attributed(mut response: Response, name: &str) -> Response { } struct HttpState { - config: HttpGatewayConfig, + service: String, + method: String, runtime: CliRuntime, routing: Arc, signer: Arc, @@ -82,7 +74,8 @@ pub(super) async fn start(options: GatewayOptions) -> anyhow::Result { let events = hellas_rpc::fetch::build_input_events_with_retention( - &state.config.service, - &state.config.method, + &state.service, + &state.method, payload, FetchEnvironment::Http.manifest_id(), state.assurance, @@ -360,13 +353,12 @@ fn response_headers(headers: Vec<(String, String)>) -> HeaderMap { && !connection.contains(&name) && !matches!(name.as_str(), "content-length" | "set-cookie") && !name.starts_with("x-hellas-") - { - if let (Ok(name), Ok(value)) = ( + && let (Ok(name), Ok(value)) = ( name.parse::(), value.parse::(), - ) { - result.append(name, value); - } + ) + { + result.append(name, value); } } result @@ -394,23 +386,3 @@ fn connection_headers<'a>(headers: impl Iterator) -> .map(|name| name.trim().to_ascii_lowercase()) .collect() } - -fn retry_delay(headers: &HeaderMap) -> Duration { - let value = headers - .get("retry-after") - .and_then(|value| value.to_str().ok()); - value - .and_then(|value| { - value - .parse::() - .ok() - .map(Duration::from_secs) - .or_else(|| { - httpdate::parse_http_date(value) - .ok() - .map(|date| date.duration_since(SystemTime::now()).unwrap_or_default()) - }) - }) - .unwrap_or(Duration::from_secs(1)) - .max(Duration::from_secs(1)) -} diff --git a/crates/gateway/src/http_fetch/affinity.rs b/crates/gateway/src/http_fetch/affinity.rs index 8a85296e..d97b64a7 100644 --- a/crates/gateway/src/http_fetch/affinity.rs +++ b/crates/gateway/src/http_fetch/affinity.rs @@ -1,8 +1,9 @@ //! Read routing hints without changing the bytes forwarded upstream. -use super::*; +use axum::http::HeaderMap; use serde_json::Value; use std::io::Read; +#[derive(Default)] pub(super) struct Hints { pub model: Option, pub session: Option<(String, String)>, @@ -79,19 +80,19 @@ impl Hints { } } } - if session.is_none() { - if let Some(user) = value.pointer("/metadata/user_id").and_then(Value::as_str) { - let id = serde_json::from_str::(user) - .ok() - .and_then(|v| { - v.get("session_id") - .and_then(Value::as_str) - .map(str::to_owned) - }) - .or_else(|| user.rsplit_once("_session_").map(|(_, id)| id.to_owned())); - if let Some(id) = id.filter(|s| !s.is_empty() && s.len() <= 1024) { - session = Some(("claude-session".into(), id)); - } + if session.is_none() + && let Some(user) = value.pointer("/metadata/user_id").and_then(Value::as_str) + { + let id = serde_json::from_str::(user) + .ok() + .and_then(|v| { + v.get("session_id") + .and_then(Value::as_str) + .map(str::to_owned) + }) + .or_else(|| user.rsplit_once("_session_").map(|(_, id)| id.to_owned())); + if let Some(id) = id.filter(|s| !s.is_empty() && s.len() <= 1024) { + session = Some(("claude-session".into(), id)); } } if session.is_none() { @@ -118,6 +119,7 @@ pub(super) fn family(path: &str) -> &str { #[cfg(test)] mod tests { use super::*; + use axum::http::HeaderValue; use std::io::Write; #[test] diff --git a/crates/gateway/src/http_fetch/config.rs b/crates/gateway/src/http_fetch/config.rs index 055848b3..700f50ed 100644 --- a/crates/gateway/src/http_fetch/config.rs +++ b/crates/gateway/src/http_fetch/config.rs @@ -1,4 +1,10 @@ -use super::*; +use super::{connection_headers, hop_header}; +use anyhow::ensure; +use axum::{body::Bytes, http::HeaderMap}; +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use hellas_rpc::http_fetch::{HttpFetchRequest, HttpTls, HttpTrustRoots}; +use serde::Deserialize; +use std::collections::BTreeMap; #[derive(Clone, Deserialize)] #[serde(deny_unknown_fields)] @@ -69,7 +75,7 @@ impl HttpGatewayConfig { pub(super) fn validate(&self) -> anyhow::Result<()> { ensure!( self.routes.is_empty() != self.backends.is_empty(), - "configure either HTTP backends or legacy routes" + "configure either HTTP backends or opaque routes" ); ensure!( self.max_in_flight > 0 && self.max_in_flight <= 1024, diff --git a/crates/gateway/src/http_fetch/observation.rs b/crates/gateway/src/http_fetch/observation.rs index 5c958718..333508c0 100644 --- a/crates/gateway/src/http_fetch/observation.rs +++ b/crates/gateway/src/http_fetch/observation.rs @@ -1,6 +1,7 @@ use serde_json::Value; use std::{io::Write, time::Instant}; +#[derive(Clone)] pub(super) struct Metrics { #[cfg(feature = "otel")] requests: opentelemetry::metrics::Counter, @@ -97,12 +98,7 @@ impl Observation { #[cfg(feature = "otel")] route: route.into(), #[cfg(feature = "otel")] - metrics: Metrics { - requests: metrics.requests.clone(), - duration: metrics.duration.clone(), - first_byte: metrics.first_byte.clone(), - tokens: metrics.tokens.clone(), - }, + metrics: metrics.clone(), } } pub(super) fn backend(&mut self, name: &str, affinity: &'static str) { diff --git a/crates/gateway/src/http_fetch/routing.rs b/crates/gateway/src/http_fetch/routing.rs index 510830dd..95e0b8cb 100644 --- a/crates/gateway/src/http_fetch/routing.rs +++ b/crates/gateway/src/http_fetch/routing.rs @@ -1,33 +1,67 @@ -use super::*; +use super::{ + affinity::{Hints, family}, + config::{HttpBackend, HttpGatewayConfig, HttpRoute}, +}; +use anyhow::ensure; +use axum::http::{HeaderMap, StatusCode}; +use hellas_client::{ExecutionRoute, ProviderTrustAnchor}; +use hellas_rpc::ContentId; +use std::{ + collections::HashMap, + sync::{Arc, Mutex, Weak}, + time::{Duration, Instant, SystemTime}, +}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; -pub(super) struct Account { - pub(super) slots: Arc, - pub(super) backoff: Mutex>, +struct Account { + capacity: usize, + slots: Arc, + backoff: Mutex>, } impl Account { - pub(super) fn cooldown(&self) -> Option<(u16, Duration)> { + fn new(capacity: usize) -> Self { + Self { + capacity, + slots: Arc::new(Semaphore::new(capacity)), + backoff: Mutex::new(None), + } + } + + fn load(&self) -> usize { + (self.capacity - self.slots.available_permits()) * 1024 / self.capacity + } + + fn acquire(&self) -> Result { + if let Some((status, delay)) = self.cooldown() { + return Err((status, delay)); + } + self.slots + .clone() + .try_acquire_owned() + .map_err(|_| (503, Duration::ZERO)) + } + + fn cooldown(&self) -> Option<(u16, Duration)> { let (until, status) = (*self.backoff.lock().unwrap())?; let delay = until.saturating_duration_since(Instant::now()); (!delay.is_zero()).then_some((status, delay)) } - pub(super) fn observe(&self, status: u16, headers: &HeaderMap) { + fn observe(&self, status: u16, headers: &HeaderMap) { if status == 429 || (status >= 500 && headers.contains_key("retry-after")) { - let delay = retry_delay(headers).min(Duration::from_secs(u32::MAX as u64)); - let until = Instant::now() + delay; - let mut backoff = self.backoff.lock().unwrap(); - if backoff.is_none_or(|(previous, _)| until > previous) { - *backoff = Some((until, status)); - } + self.back_off(status, retry_delay(headers)); } } -} -use super::affinity::{Hints, family}; -use hellas_rpc::ContentId; -use std::collections::HashMap; -use tokio::sync::OwnedSemaphorePermit; + fn back_off(&self, status: u16, delay: Duration) { + let until = Instant::now() + delay.min(Duration::from_secs(u32::MAX as u64)); + let mut backoff = self.backoff.lock().unwrap(); + if backoff.is_none_or(|(previous, _)| until > previous) { + *backoff = Some((until, status)); + } + } +} const MAX_BINDINGS: usize = 16_384; const SESSION_IDLE: Duration = Duration::from_secs(24 * 60 * 60); @@ -37,14 +71,14 @@ pub(super) struct Backend { pub models: Vec, pub routes: Vec, pub remote: ExecutionRoute, - capacity: usize, account: Arc, } struct Binding { - backend: usize, + // None means different backends reported the same server-side identifier. + backend: Option, touched: Instant, - connection: Option>, + connection: Option>, } #[derive(Default)] @@ -53,6 +87,76 @@ struct State { next: usize, } +impl State { + fn resolve( + &mut self, + session: Option, + continuations: &[ContentId], + ) -> Result, Unavailable> { + let now = Instant::now(); + self.bindings.retain(|_, binding| { + now.duration_since(binding.touched) < SESSION_IDLE + && binding + .connection + .as_ref() + .is_none_or(|c| c.strong_count() > 0) + }); + let mut pinned = None; + for (key, required) in continuations + .iter() + .map(|key| (*key, true)) + .chain(session.map(|key| (key, false))) + { + let Some(binding) = self.bindings.get_mut(&key) else { + if required { + return Err(Unavailable::conflict( + "unknown or expired server-side continuation; resend full context", + )); + } + continue; + }; + binding.touched = now; + let backend = binding + .backend + .ok_or_else(|| Unavailable::conflict("ambiguous server-side continuation"))?; + if pinned.is_some_and(|p| p != backend) { + return Err(Unavailable::conflict( + "conflicting session or continuation backends", + )); + } + pinned = Some(backend); + } + Ok(pinned) + } + + fn check_capacity(&self, session: Option) -> Result<(), Unavailable> { + if session.is_some_and(|key| !self.bindings.contains_key(&key)) + && self.bindings.len() >= MAX_BINDINGS + { + return Err(Unavailable::busy(503, Duration::ZERO)); + } + Ok(()) + } + + fn bind(&mut self, key: ContentId, backend: usize, connection: Option>) { + // Never overwrite a conflicting identifier or evict a live session. + // An unrecorded or ambiguous continuation will fail closed later. + if let Some(binding) = self.bindings.get_mut(&key) { + binding.backend = binding.backend.filter(|existing| *existing == backend); + binding.touched = Instant::now(); + } else if self.bindings.len() < MAX_BINDINGS { + self.bindings.insert( + key, + Binding { + backend: Some(backend), + touched: Instant::now(), + connection, + }, + ); + } + } +} + pub(super) struct Routing { pub backends: Vec, pooled: bool, @@ -77,20 +181,26 @@ pub(super) struct Unavailable { } impl Unavailable { - fn conflict(message: &'static str) -> Self { + fn new(status: StatusCode, message: &'static str) -> Self { Self { - status: StatusCode::CONFLICT, + status, message, retry: None, backend: None, } } + + fn conflict(message: &'static str) -> Self { + Self::new(StatusCode::CONFLICT, message) + } + fn busy(status: u16, delay: Duration) -> Self { Self { - status: StatusCode::from_u16(status).unwrap(), - message: "selected backend is temporarily unavailable", retry: Some(delay.as_secs() + 1), - backend: None, + ..Self::new( + StatusCode::from_u16(status).unwrap(), + "selected backend is temporarily unavailable", + ) } } } @@ -99,99 +209,66 @@ impl Routing { pub fn new( config: &HttpGatewayConfig, remote: ExecutionRoute, - trust: hellas_client::ProviderTrustAnchor, + trust: ProviderTrustAnchor, ) -> anyhow::Result { + let routes = config.routes.iter().enumerate().map(|(index, route)| { + ( + format!("route-{index}"), + HttpBackend { + models: vec![], + credential: route.credential.clone(), + routes: vec![route.clone()], + max_in_flight: None, + provider: None, + }, + ) + }); let mut backends = Vec::new(); - let mut accounts: BTreeMap)> = BTreeMap::new(); - let mut add = |name: String, - models: Vec, - routes: Vec, - remote: ExecutionRoute, - limit: usize| - -> anyhow::Result<()> { - let account = routes[0].account(); + let mut accounts = HashMap::new(); + for (name, mut backend) in routes.chain(config.backends.clone()) { + let remote = match backend.provider { + Some(provider) => ExecutionRoute::remote( + Some(provider.node_id), + provider.node_addrs, + 0, + ProviderTrustAnchor { + expected_genesis: provider.genesis, + ..trust.clone() + }, + ), + None => remote.clone(), + }; + for route in &mut backend.routes { + route.credential = backend.credential.clone(); + } + let account = backend.routes[0].account(); ensure!( - routes.iter().all(|r| r.account() == account), + backend.routes.iter().all(|r| r.account() == account), "a backend must use one account or origin" ); - // The enrollment pin identifies the provider even when one route - // uses discovery and another dials that provider directly. - let scope = match &remote { - ExecutionRoute::RemoteDirect(target) => { - target.provider_trust.expected_genesis.to_string() - } + // Enrollment pins share admission across direct and discovered routes. + let genesis = match &remote { + ExecutionRoute::RemoteDirect(target) => target.provider_trust.expected_genesis, ExecutionRoute::RemoteDiscovery { provider_trust, .. } => { - provider_trust.expected_genesis.to_string() + provider_trust.expected_genesis } ExecutionRoute::Local => unreachable!(), }; - let (configured_limit, shared) = accounts - .entry(format!("{scope}:{account}")) - .or_insert_with(|| { - ( - limit, - Arc::new(Account { - slots: Arc::new(Semaphore::new(limit)), - backoff: Mutex::new(None), - }), - ) - }); + let capacity = backend.max_in_flight.unwrap_or(config.max_in_flight); + let account = accounts + .entry((genesis, account)) + .or_insert_with(|| Arc::new(Account::new(capacity))); ensure!( - *configured_limit == limit, + account.capacity == capacity, "shared accounts require the same concurrency limit" ); - let shared = shared.clone(); backends.push(Backend { name, - models, - routes, + models: backend.models, + routes: backend.routes, remote, - capacity: limit, - account: shared, + account: account.clone(), }); - Ok(()) - }; - if config.backends.is_empty() { - for (index, route) in config.routes.iter().enumerate() { - add( - format!("route-{index}"), - vec![], - vec![route.clone()], - remote.clone(), - config.max_in_flight, - )?; - } - } else { - for (name, backend) in &config.backends { - let remote = if let Some(provider) = &backend.provider { - let mut trust = trust.clone(); - trust.expected_genesis = provider.genesis; - ExecutionRoute::remote( - Some(provider.node_id), - provider.node_addrs.clone(), - 0, - trust, - ) - } else { - remote.clone() - }; - let routes = backend - .routes - .iter() - .cloned() - .map(|mut r| { - r.credential = backend.credential.clone(); - r - }) - .collect(); - add( - name.clone(), - backend.models.clone(), - routes, - remote, - backend.max_in_flight.unwrap_or(config.max_in_flight), - )?; - } } Ok(Self { backends, @@ -216,110 +293,65 @@ impl Routing { connection: Option<&crate::ConnectionId>, ) -> Result { let hints = if self.pooled { - Some(Hints::read(headers, body).map_err(|message| Unavailable { - status: StatusCode::BAD_REQUEST, - message, - retry: None, - backend: None, - })?) + Hints::read(headers, body) + .map_err(|message| Unavailable::new(StatusCode::BAD_REQUEST, message))? } else { - None + Hints::default() }; - let model = hints.as_ref().and_then(|h| h.model.as_deref()); - let mut candidates = Vec::new(); - for (index, backend) in self.backends.iter().enumerate() { - if let Some(endpoint) = backend - .routes - .iter() - .position(|r| r.path == path && r.method == method) - { - if model.is_none_or(|model| { + let model = hints.model.as_deref(); + let mut candidates: Vec<_> = self + .backends + .iter() + .enumerate() + .filter(|(_, backend)| { + model.is_none_or(|model| { backend.models.is_empty() || backend.models.iter().any(|m| m == model) - }) { - candidates.push((index, endpoint)); - } - } - } + }) + }) + .filter_map(|(index, backend)| { + backend + .routes + .iter() + .position(|r| r.path == path && r.method == method) + .map(|endpoint| (index, endpoint)) + }) + .collect(); if candidates.is_empty() { - return Err(Unavailable { - status: StatusCode::NOT_FOUND, - message: "no backend serves this route and model", - retry: None, - backend: None, - }); - } - if self.pooled - && model.is_none() - && !body.is_empty() - && !hints - .as_ref() - .is_some_and(|h| h.previous.is_some() || h.conversation.is_some()) - { - return Err(Unavailable { - status: StatusCode::BAD_REQUEST, - message: "model is required", - retry: None, - backend: None, - }); + return Err(Unavailable::new( + StatusCode::NOT_FOUND, + "no backend serves this route and model", + )); } let family = family(path); + let continuations: Vec<_> = [ + ("response", &hints.previous), + ("conversation", &hints.conversation), + ] + .into_iter() + .filter_map(|(kind, id)| id.as_ref().map(|id| self.key(kind, &[family, id]))) + .collect(); + if self.pooled && model.is_none() && !body.is_empty() && continuations.is_empty() { + return Err(Unavailable::new( + StatusCode::BAD_REQUEST, + "model is required", + )); + } + let connection = connection.filter(|_| self.pooled && hints.session.is_none()); let session = hints + .session .as_ref() - .and_then(|h| h.session.as_ref()) - .map(|(kind, id)| self.key("session", &[family, model.unwrap_or(""), kind, id])); - let connection_key = - connection - .filter(|_| self.pooled && session.is_none()) - .map(|connection| { + .map(|(kind, id)| self.key("session", &[family, model.unwrap_or(""), kind, id])) + .or_else(|| { + connection.map(|c| { self.key( "connection", - &[family, model.unwrap_or(""), &connection.id.to_string()], + &[family, model.unwrap_or(""), &c.id.to_string()], ) - }); - let affinity_key = session.or(connection_key); - let state_keys: Vec<_> = hints - .as_ref() - .into_iter() - .flat_map(|h| { - [ - h.previous - .as_ref() - .map(|id| self.key("response", &[family, id])), - h.conversation - .as_ref() - .map(|id| self.key("conversation", &[family, id])), - ] - }) - .flatten() - .collect(); + }) + }); + // Keep resolution, admission and first binding atomic across requests. let mut state = self.state.lock().unwrap(); - let now = Instant::now(); - state.bindings.retain(|_, b| { - now.duration_since(b.touched) < SESSION_IDLE - && b.connection.as_ref().is_none_or(|c| c.strong_count() > 0) - }); - let mut pinned = None; - for key in &state_keys { - let binding = state.bindings.get_mut(key).ok_or_else(|| { - Unavailable::conflict( - "unknown or expired server-side continuation; resend full context", - ) - })?; - binding.touched = now; - if pinned.is_some_and(|p| p != binding.backend) { - return Err(Unavailable::conflict("conflicting continuation backends")); - } - pinned = Some(binding.backend); - } - if let Some(binding) = affinity_key.and_then(|k| state.bindings.get_mut(&k)) { - binding.touched = now; - if pinned.is_some_and(|p| p != binding.backend) { - return Err(Unavailable::conflict( - "session and continuation refer to different backends", - )); - } - pinned = Some(binding.backend); - } + let pinned = state.resolve(session, &continuations)?; if let Some(index) = pinned { candidates.retain(|(backend, _)| *backend == index); if candidates.is_empty() { @@ -328,67 +360,48 @@ impl Routing { )); } } - if affinity_key.is_some_and(|key| !state.bindings.contains_key(&key)) - && state.bindings.len() >= MAX_BINDINGS - { - return Err(Unavailable::busy(503, Duration::ZERO)); - } + state.check_capacity(session)?; let count = self.backends.len(); - candidates.sort_by_key(|(index, _)| { - let backend = &self.backends[*index]; - // Least active first, rotating equal candidates. Shared account permits - // also cover aliases used by multiple model configurations. + // Snapshot loads before sorting: other responses may release permits. + // Least occupied account first, rotating ties; aliases share its permits. + candidates.sort_by_cached_key(|(index, _)| { ( - (backend.capacity - backend.account.slots.available_permits()) * 1024 - / backend.capacity, + self.backends[*index].account.load(), (*index + count - state.next) % count, ) }); - let mut unavailable = None; + let mut unavailable: Option<(u16, Duration)> = None; for (backend, endpoint) in candidates { - let account = &self.backends[backend].account; - if let Some((status, delay)) = account.cooldown() { - if unavailable.as_ref().is_none_or(|(_, old)| delay < *old) { - unavailable = Some((status, delay)); + let permit = match self.backends[backend].account.acquire() { + Ok(permit) => permit, + Err(failure) => { + if unavailable.as_ref().is_none_or(|old| failure.1 < old.1) { + unavailable = Some(failure); + } + continue; } - continue; - } - let Ok(permit) = account.slots.clone().try_acquire_owned() else { - unavailable = Some((503, Duration::ZERO)); - continue; }; - for key in state_keys.iter().copied().chain(affinity_key) { - state.bindings.insert( - key, - Binding { - backend, - touched: now, - connection: if Some(key) == connection_key { - connection.map(|c| Arc::downgrade(&c.alive)) - } else { - None - }, - }, - ); + if let Some(key) = session { + state.bind(key, backend, connection.map(|c| Arc::downgrade(&c.alive))); } state.next = (backend + 1) % count; return Ok(Selected { backend, endpoint, permit, - model: model.map(str::to_owned), - affinity: if !state_keys.is_empty() { + model: hints.model, + affinity: if !continuations.is_empty() { "continuation" - } else if pinned.is_some() && connection_key.is_some() { + } else if pinned.is_none() { + "new" + } else if connection.is_some() { "connection" - } else if pinned.is_some() { - "session" } else { - "new" + "session" }, }); } - let (status, delay) = unavailable.unwrap_or((503, Duration::ZERO)); + let (status, delay) = unavailable.expect("nonempty backend candidates"); let mut failure = Unavailable::busy(status, delay); failure.backend = pinned; Err(failure) @@ -400,14 +413,9 @@ impl Routing { pub fn transport_failed(&self, backend: usize) { if self.pooled { - self.observe( - backend, - 503, - &HeaderMap::from_iter([( - "retry-after".parse().unwrap(), - HeaderValue::from_static("1"), - )]), - ); + self.backends[backend] + .account + .back_off(503, Duration::from_secs(1)); } } @@ -457,27 +465,34 @@ impl ResponseBinding { continue; }; let key = self.routing.key(kind, &[&self.family, id]); - let mut state = self.routing.state.lock().unwrap(); - // Never silently overwrite another account's identifier or evict a - // live session. An unrecorded continuation will fail closed later. - if let Some(existing) = state.bindings.get_mut(&key) { - if existing.backend != self.backend { - existing.backend = usize::MAX; - } - existing.touched = Instant::now(); - } else if state.bindings.len() < MAX_BINDINGS { - state.bindings.insert( - key, - Binding { - backend: self.backend, - touched: Instant::now(), - connection: None, - }, - ); - } + self.routing + .state + .lock() + .unwrap() + .bind(key, self.backend, None); } } } +pub(super) fn retry_delay(headers: &HeaderMap) -> Duration { + let value = headers + .get("retry-after") + .and_then(|value| value.to_str().ok()); + value + .and_then(|value| { + value + .parse::() + .ok() + .map(Duration::from_secs) + .or_else(|| { + httpdate::parse_http_date(value) + .ok() + .map(|date| date.duration_since(SystemTime::now()).unwrap_or_default()) + }) + }) + .unwrap_or(Duration::from_secs(1)) + .max(Duration::from_secs(1)) +} + #[cfg(test)] mod tests; diff --git a/crates/gateway/src/http_fetch/routing/tests.rs b/crates/gateway/src/http_fetch/routing/tests.rs index 04dc5fab..db590d4f 100644 --- a/crates/gateway/src/http_fetch/routing/tests.rs +++ b/crates/gateway/src/http_fetch/routing/tests.rs @@ -1,4 +1,7 @@ +use super::super::config; use super::*; +use axum::http::HeaderValue; +use hellas_rpc::Assurance; use serde_json::json; fn config() -> HttpGatewayConfig { @@ -80,7 +83,7 @@ fn models_admission_and_cooldowns_do_not_move_existing_sessions() { .unwrap() .bindings .values() - .any(|b| b.backend == 0 && b.touched.elapsed() < Duration::from_secs(1)), + .any(|b| b.backend == Some(0) && b.touched.elapsed() < Duration::from_secs(1)), "rate-limited retries still keep a session active" ); assert_eq!(pinned.status, StatusCode::TOO_MANY_REQUESTS); @@ -206,17 +209,20 @@ fn server_state_uses_its_original_backend_and_unknown_or_conflicting_state_fails .status, StatusCode::CONFLICT ); - routing - .response_binding(1, "/v1/responses") - .unwrap() - .observe(&json!({"object":"response","id":"resp_a"})); - assert_eq!( - request(json!({"model":"k3","previous_response_id":"resp_a"})) - .err() + // Neither account may repair a collision by reporting the ID again. + for backend in [1, 0, 1] { + routing + .response_binding(backend, "/v1/responses") .unwrap() - .status, - StatusCode::CONFLICT - ); + .observe(&json!({"object":"response","id":"resp_a"})); + assert_eq!( + request(json!({"model":"k3","previous_response_id":"resp_a"})) + .err() + .unwrap() + .status, + StatusCode::CONFLICT + ); + } } #[test] @@ -229,7 +235,7 @@ fn full_affinity_table_does_not_evict_live_sessions_and_expired_server_state_is_ state.bindings.insert( ContentId::hash(&i.to_le_bytes()), Binding { - backend: 1, + backend: Some(1), touched: Instant::now(), connection: None, }, @@ -338,3 +344,47 @@ fn closed_client_connections_release_their_affinity_entries() { select(&routing, "k3", "explicit-session").unwrap(); assert_eq!(routing.state.lock().unwrap().bindings.len(), 1); } + +#[test] +fn account_backoff_is_shared_and_never_shortened_by_another_response() { + let account = Arc::new(Account::new(1)); + let other_route = account.clone(); + let header = |seconds: &'static str| { + HeaderMap::from_iter([( + "retry-after".parse().unwrap(), + HeaderValue::from_static(seconds), + )]) + }; + assert!(account.cooldown().is_none()); + account.observe(429, &header("60")); + assert!(other_route.cooldown().unwrap().1 > Duration::from_secs(59)); + other_route.observe(429, &header("1")); + assert!(account.cooldown().unwrap().1 > Duration::from_secs(59)); + other_route.observe(503, &header("120")); + assert_eq!(account.cooldown().unwrap().0, 503); + assert!(account.cooldown().unwrap().1 > Duration::from_secs(119)); + // A short rate limit must not turn a retriable overload into a quota error. + other_route.observe(429, &header("1")); + assert_eq!(account.cooldown().unwrap().0, 503); + account.observe(200, &header("600")); + assert!(account.cooldown().unwrap().1 < Duration::from_secs(121)); + *account.backoff.lock().unwrap() = Some((Instant::now(), 503)); + assert!(account.cooldown().is_none()); + let permit = account.slots.clone().try_acquire_owned().unwrap(); + assert!(other_route.slots.clone().try_acquire_owned().is_err()); + drop(permit); + assert!(other_route.slots.clone().try_acquire_owned().is_ok()); +} + +#[test] +fn new_sessions_use_relative_account_load() { + let mut config = config(); + config.backends.get_mut("b").unwrap().max_in_flight = Some(2); + let routing = routing(config); + let a = select(&routing, "k3", "a").unwrap(); + let b = select(&routing, "k3", "b").unwrap(); + let c = select(&routing, "k3", "c").unwrap(); + assert_eq!((a.backend, b.backend, c.backend), (0, 1, 1)); + drop(a); + assert_eq!(select(&routing, "k3", "d").unwrap().backend, 0); +} diff --git a/crates/gateway/src/http_fetch/tests.rs b/crates/gateway/src/http_fetch/tests.rs index ed331681..cfb62d6b 100644 --- a/crates/gateway/src/http_fetch/tests.rs +++ b/crates/gateway/src/http_fetch/tests.rs @@ -1,40 +1,10 @@ -use super::config::public_tls; -use super::routing::Account; +use super::config::{HttpRoute, public_tls}; +use super::routing::retry_delay; use super::*; - -#[test] -fn account_backoff_is_shared_and_never_shortened_by_another_response() { - let account = Arc::new(Account { - slots: Arc::new(Semaphore::new(1)), - backoff: Mutex::new(None), - }); - let other_route = account.clone(); - let header = |seconds: &'static str| { - HeaderMap::from_iter([( - "retry-after".parse().unwrap(), - HeaderValue::from_static(seconds), - )]) - }; - assert!(account.cooldown().is_none()); - account.observe(429, &header("60")); - assert!(other_route.cooldown().unwrap().1 > Duration::from_secs(59)); - other_route.observe(429, &header("1")); - assert!(account.cooldown().unwrap().1 > Duration::from_secs(59)); - other_route.observe(503, &header("120")); - assert_eq!(account.cooldown().unwrap().0, 503); - assert!(account.cooldown().unwrap().1 > Duration::from_secs(119)); - // A short rate limit must not turn a retriable overload into a quota error. - other_route.observe(429, &header("1")); - assert_eq!(account.cooldown().unwrap().0, 503); - account.observe(200, &header("600")); - assert!(account.cooldown().unwrap().1 < Duration::from_secs(121)); - *account.backoff.lock().unwrap() = Some((Instant::now(), 503)); - assert!(account.cooldown().is_none()); - let permit = account.slots.clone().try_acquire_owned().unwrap(); - assert!(other_route.slots.clone().try_acquire_owned().is_err()); - drop(permit); - assert!(other_route.slots.clone().try_acquire_owned().is_ok()); -} +use std::{ + collections::BTreeMap, + time::{Duration, SystemTime}, +}; #[test] fn forward_only_protocol_headers_and_keep_retry_and_quota_metadata() { diff --git a/crates/providers/src/http/clients.rs b/crates/providers/src/http/clients.rs index 890b1a75..84c44f38 100644 --- a/crates/providers/src/http/clients.rs +++ b/crates/providers/src/http/clients.rs @@ -36,7 +36,7 @@ impl Clients { tls: request.tls.clone(), credential: request.credential.clone(), }; - if let Some(client) = self.cached(&key) { + if let Some(client) = Self::cached(&mut self.0.lock().unwrap(), &key) { return Ok(client); } // Build outside the lock. A concurrent miss may build a second client, @@ -58,10 +58,7 @@ impl Clients { .build() .map_err(|_| fault("HTTPS client initialization failed"))?; let mut entries = self.0.lock().unwrap(); - if let Some(index) = entries.iter().position(|(existing, _)| *existing == key) { - let entry = entries.remove(index).unwrap(); - let client = entry.1.clone(); - entries.push_back(entry); + if let Some(client) = Self::cached(&mut entries, &key) { return Ok(client); } if entries.len() == MAX_CLIENTS { @@ -71,8 +68,7 @@ impl Clients { Ok(client) } - fn cached(&self, key: &Key) -> Option { - let mut entries = self.0.lock().unwrap(); + fn cached(entries: &mut VecDeque<(Key, Client)>, key: &Key) -> Option { let index = entries.iter().position(|(existing, _)| existing == key)?; let entry = entries.remove(index).unwrap(); let client = entry.1.clone(); diff --git a/crates/providers/src/http/config.rs b/crates/providers/src/http/config.rs index 2ea2f8a6..25a5a076 100644 --- a/crates/providers/src/http/config.rs +++ b/crates/providers/src/http/config.rs @@ -125,20 +125,20 @@ impl HttpSecret { None => None, }; let mut object = read_credential(path.clone()).await?; - if let Some(refresh) = refresh { + if let Some(refresh) = refresh + && refresh.due(&object)? + { + let retry_at = guard.as_mut().expect("refresh holds its account lock"); + if retry_at.is_some_and(|deadline| deadline > std::time::Instant::now()) { + return Err(super::fault("credential refresh is cooling down")); + } + **retry_at = Some(std::time::Instant::now() + Duration::from_secs(30)); + refresh.run().await?; + object = read_credential(path.clone()).await?; if refresh.due(&object)? { - let retry_at = guard.as_mut().expect("refresh holds its account lock"); - if retry_at.is_some_and(|deadline| deadline > std::time::Instant::now()) { - return Err(super::fault("credential refresh is cooling down")); - } - **retry_at = Some(std::time::Instant::now() + Duration::from_secs(30)); - refresh.run().await?; - object = read_credential(path.clone()).await?; - if refresh.due(&object)? { - return Err(super::fault("credential refresh did not renew the token")); - } - **retry_at = None; + return Err(super::fault("credential refresh did not renew the token")); } + **retry_at = None; } let secret = credential_field(&object, field) .and_then(serde_json::Value::as_str) diff --git a/crates/sdk/src/paid_provider.rs b/crates/sdk/src/paid_provider.rs index 862d3720..2e3f782b 100644 --- a/crates/sdk/src/paid_provider.rs +++ b/crates/sdk/src/paid_provider.rs @@ -8,6 +8,7 @@ use hellas_chain::{ ConsensusInfo, ConsensusVerifier, FinalizedWorkView, WorkBlocks, WorkChannelQuery, }; use hellas_kernel::{EdgeId, NetworkId, Secp256k1Signer, Secp256k1Verifier}; +use hellas_rpc::call::WithTrailer; use hellas_rpc::pb::work::*; use hellas_rpc::peers::PeerId; use hellas_rpc::protocol::Digest; @@ -47,51 +48,36 @@ fn not_ready() -> WorkRefused { } impl WorkSetupHandler for UnmountedWork { - fn exchange_setup( + async fn exchange_setup( &self, _request: ExchangeSetupRequest, _context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - core::future::ready(Ok(ExchangeSetupResponse { + ) -> Result> + Send, WireStatus> { + Ok(ExchangeSetupResponse { outcome: Some(exchange_setup_response::Outcome::Refused(not_ready())), - })) + }) } } impl WorkHandler for UnmountedWork { - fn accept_work( + async fn accept_work( &self, _request: AcceptWorkRequest, _context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - core::future::ready(Ok(AcceptWorkResponse { + ) -> Result> + Send, WireStatus> { + Ok(AcceptWorkResponse { outcome: Some(accept_work_response::Outcome::Refused(not_ready())), - })) + }) } - fn deliver_result( + async fn deliver_result( &self, _request: DeliverResultRequest, _context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - core::future::ready(Ok(DeliverResultResponse { + ) -> Result> + Send, WireStatus> { + Ok(DeliverResultResponse { outcome: Some(deliver_result_response::Outcome::Refused(not_ready())), - })) + }) } async fn stream_result( @@ -105,19 +91,14 @@ impl WorkHandler for UnmountedWork { )) } - fn admit_certificate( + async fn admit_certificate( &self, _request: AdmitCertificateRequest, _context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - core::future::ready(Ok(AdmitCertificateResponse { + ) -> Result> + Send, WireStatus> { + Ok(AdmitCertificateResponse { outcome: Some(admit_certificate_response::Outcome::Refused(not_ready())), - })) + }) } } @@ -342,8 +323,7 @@ where &self, request: AcceptWorkRequest, _context: TransportContext, - ) -> Result> + Send, WireStatus> - { + ) -> Result> + Send, WireStatus> { if let Some(response) = self.service.precheck_acceptance(&request) { return Ok(response); } @@ -400,9 +380,8 @@ where &self, request: DeliverResultRequest, context: TransportContext, - ) -> Result> + Send, WireStatus> - { - let response: hellas_rpc::call::WithTrailer = + ) -> Result> + Send, WireStatus> { + let response: WithTrailer = if self.refresh_delivery(&request).await.is_ok() { self.service.deliver_result(request, context).await?.into() } else { @@ -453,17 +432,12 @@ where })) } - fn admit_certificate( + async fn admit_certificate( &self, request: AdmitCertificateRequest, context: TransportContext, - ) -> impl core::future::Future< - Output = Result< - impl Into> + Send, - WireStatus, - >, - > + Send { - self.service.admit_certificate(request, context) + ) -> Result> + Send, WireStatus> { + self.service.admit_certificate(request, context).await } } diff --git a/docs/http-gateway.md b/docs/http-gateway.md index 2abcf529..dfa3357e 100644 --- a/docs/http-gateway.md +++ b/docs/http-gateway.md @@ -63,8 +63,8 @@ never becomes an upstream credential. Optional route `headers` holds operator `[name, value]` pairs, with lowercase names. Configured values override the corresponding client header. Account credentials belong on the backend. -The earlier top-level `routes` configuration remains supported for opaque HTTP -passthrough, without model selection. Use either `backends` or `routes`, not both. +For opaque HTTP passthrough without model selection, configure top-level `routes` +instead of `backends`. Omit `credential` for an unauthenticated upstream. Such routes share admission by origin. Optional `tls` uses the [Fetch TLS vocabulary](../crates/providers/HTTPS.md) From f2a8c63098e2b8178739d923c940ceb87372ce88 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Fri, 25 Sep 2026 00:41:24 +0200 Subject: [PATCH 12/32] Route HTTP proxy through funded Fetch channels --- Cargo.lock | 1 + crates/cli/src/commands/paid_work.rs | 30 +- crates/cli/src/commands/paid_work/gateway.rs | 356 ++++++++++++----- crates/cli/src/commands/paid_work/tests.rs | 9 +- crates/cli/src/main.rs | 17 +- crates/cli/src/tests.rs | 19 + crates/executor/src/executor/actor/mod.rs | 8 +- .../executor/src/executor/actor/paid_fetch.rs | 58 ++- crates/executor/src/executor/mod.rs | 1 + crates/executor/src/work.rs | 15 + crates/gateway/src/http_fetch.rs | 106 +++-- crates/gateway/src/http_fetch/config.rs | 21 +- crates/gateway/src/http_fetch/routing.rs | 73 ++-- .../gateway/src/http_fetch/routing/tests.rs | 66 ++-- crates/gateway/src/http_fetch/tests.rs | 92 +++++ crates/gateway/src/lib.rs | 46 ++- crates/gateway/src/state.rs | 18 +- crates/gateway/src/state/tests.rs | 8 +- .../src/responses_fetch/telemetry/otel.rs | 5 +- crates/rpc/src/fetch.rs | 4 +- crates/rpc/src/protocol/work_fetch.rs | 4 + crates/sdk/Cargo.toml | 1 + crates/sdk/src/paid_client.rs | 370 +++++++++--------- crates/sdk/src/paid_client/error.rs | 90 +++++ crates/sdk/src/paid_client/tests.rs | 12 +- crates/wire/src/mux/state.rs | 8 +- crates/wire/src/mux/state/tests.rs | 37 ++ crates/work/src/work.rs | 65 ++- crates/work/src/work/stream.rs | 222 +++++++++-- crates/work/src/work_open.rs | 10 +- crates/work/src/work_store/channel/state.rs | 8 +- crates/work/src/work_store/channel/store.rs | 13 +- crates/work/tests/work_run/fetch.rs | 333 +++++++++++++++- crates/work/tests/work_store_channel.rs | 2 + docs/http-gateway.md | 69 +++- docs/paid-gateway.md | 19 +- proto/hellas/work/v1/work.proto | 12 +- 37 files changed, 1609 insertions(+), 619 deletions(-) create mode 100644 crates/sdk/src/paid_client/error.rs diff --git a/Cargo.lock b/Cargo.lock index 0723d279..a266acc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3066,6 +3066,7 @@ dependencies = [ "serde", "serde_json", "tempfile", + "thiserror 2.0.18", "tokio", "tracing", "url", diff --git a/crates/cli/src/commands/paid_work.rs b/crates/cli/src/commands/paid_work.rs index 053c3a2a..220ecd59 100644 --- a/crates/cli/src/commands/paid_work.rs +++ b/crates/cli/src/commands/paid_work.rs @@ -420,6 +420,22 @@ async fn inspect_chain(validators: &[String]) -> CliResult<()> { Ok(()) } +fn paid_provider_trust( + args: &RunArgs, + assurance: hellas_rpc::Assurance, +) -> CliResult> { + if args.provider_genesis.is_some() || assurance != hellas_rpc::Assurance::ProducerSigned { + Ok(Some(crate::identity::provider_trust( + args.provider_genesis, + assurance, + args.apple_app_id.clone(), + args.apple_cd_hashes.clone(), + )?)) + } else { + Ok(None) + } +} + async fn open_paid_channel( args: &RunArgs, endpoint: iroh::Endpoint, @@ -430,17 +446,7 @@ async fn open_paid_channel( !args.payment_coins.is_empty(), "at least one --payment-coin is required" ); - let provider_trust = - if args.provider_genesis.is_some() || assurance != hellas_rpc::Assurance::ProducerSigned { - Some(crate::identity::provider_trust( - args.provider_genesis, - assurance, - args.apple_app_id.clone(), - args.apple_cd_hashes.clone(), - )?) - } else { - None - }; + let provider_trust = paid_provider_trust(args, assurance)?; OpenPaidChannel::open( hellas_sdk::paid_client::PaidWorkOptions { config: load_work_config(&args.work_config)?, @@ -460,6 +466,7 @@ async fn open_paid_channel( settlement_key, ) .await + .map_err(Into::into) } async fn run_one( @@ -517,6 +524,7 @@ fn relative_deadlines(current: u64, args: &RunArgs) -> CliResult { args.terminal_blocks, args.payment_blocks, ) + .map_err(Into::into) } #[cfg(test)] use hellas_sdk::paid_client::check_genesis_payload; diff --git a/crates/cli/src/commands/paid_work/gateway.rs b/crates/cli/src/commands/paid_work/gateway.rs index cb30fbd6..ef8943b4 100644 --- a/crates/cli/src/commands/paid_work/gateway.rs +++ b/crates/cli/src/commands/paid_work/gateway.rs @@ -4,12 +4,15 @@ use super::*; use futures::future::BoxFuture; use futures::stream::BoxStream; use hellas_gateway::{ - ExecutionEvent, Outcome, PaidExecutionBackend, PaidExecutionRequest, StopReason, + ExecutionEvent, Outcome, PaidExecutionBackend, PaidExecutionRequest, PaidFetchRequest, + StopReason, }; +use hellas_rpc::output::OutputEvent as FetchEvent; use hellas_rpc::protocol::artifacts::{ BoundTermId, InputAddressed as _, OutputAddressed as _, SourceRef, TextArtifact, TextExecution, TextPolicy, TokenIds, }; +use hellas_rpc::protocol::work_profile::{PaidWorkPolicy, PreparedPaidWorkInput}; use iroh::Endpoint; use serde::Deserialize; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -68,7 +71,18 @@ const CHANNEL_FOLLOW_BUDGET: Duration = Duration::from_secs(1); const OUTPUT_BUFFER_BYTES: usize = 2 * MAX_RECORD_BYTES; const OUTPUT_EVENT_OVERHEAD: usize = 1024; const OUTPUT_BUFFER_EVENTS: usize = OUTPUT_BUFFER_BYTES / OUTPUT_EVENT_OVERHEAD; -type BufferedEvent = (CliResult, OwnedSemaphorePermit); +type BufferedEvent = (CliResult, OwnedSemaphorePermit); + +trait GatewayEvent: Send + 'static { + fn prefix(event: hellas_rpc::OutputEventEnvelope) -> CliResult + where + Self: Sized; + fn completed(output: PaidOutput) -> CliResult> + where + Self: Sized; + fn is_terminal(&self) -> bool; + fn bytes(&self) -> usize; +} #[derive(Deserialize)] #[serde(deny_unknown_fields)] @@ -81,11 +95,17 @@ struct ProviderFile { bond: String, payment_coins: Vec, omission_bond: u64, + /// Required enrollment pin for attested providers; optional for producer-signed work. + provider_genesis: Option, + apple_app_id: Option, + #[serde(default)] + apple_cd_hashes: Vec, } struct Provider { args: RunArgs, policy: ProviderChannelPolicy, + assurance: hellas_rpc::Assurance, /// A setup/channel journal has a single owner even with concurrent HTTP calls. serial: AsyncMutex>, pending: AtomicUsize, @@ -210,6 +230,7 @@ struct PaidGateway { next: AtomicUsize, endpoint: Endpoint, settlement_key: Secp256k1Signer, + producer_key: hellas_rpc::ProducerSigningKey, admission: Arc, tasks: Mutex>>, followers: Mutex>>, @@ -219,7 +240,14 @@ pub async fn load_gateway_backend( path: &Path, transport_key: SecretKey, settlement_key: Secp256k1Signer, + producer_key: hellas_rpc::ProducerSigningKey, + assurance: hellas_rpc::Assurance, ) -> CliResult> { + anyhow::ensure!( + producer_key.public_key() + == hellas_rpc::PublicKey::Secp256k1(settlement_key.party_key().to_bytes()), + "paid Fetch signer must match the settlement identity" + ); let bytes = crate::commands::read_bounded_regular_file(path, "paid gateway config", MAX_RECORD_BYTES)?; let file: PoolFile = serde_json::from_slice(&bytes) @@ -273,10 +301,18 @@ pub async fn load_gateway_backend( let config = load_work_config(&provider.work_config)?; providers.push(Arc::new(Provider { policy: config.provider_policy(), + assurance, args: RunArgs { - provider_genesis: None, - apple_app_id: None, - apple_cd_hashes: Vec::new(), + provider_genesis: provider + .provider_genesis + .map(|value| value.parse()) + .transpose()?, + apple_app_id: provider.apple_app_id, + apple_cd_hashes: provider + .apple_cd_hashes + .iter() + .map(|value| fixed_hex::<32>("apple_cd_hashes", value)) + .collect::>()?, work_config: provider.work_config, journal_root: provider.journal_root, provider: provider.provider, @@ -297,6 +333,8 @@ pub async fn load_gateway_backend( cache: Mutex::new(PrefixCache::default()), unavailable_until: Mutex::new(None), })); + let provider = providers.last().expect("provider just added"); + paid_provider_trust(&provider.args, provider.assurance)?; } let gateway = Arc::new(PaidGateway { admission: Arc::new(Semaphore::new(file.max_pending_requests)), @@ -306,13 +344,14 @@ pub async fn load_gateway_backend( // provider and request for the lifetime of this gateway. endpoint: bind_paid_endpoint(transport_key).await?, settlement_key, + producer_key, tasks: Mutex::new(Vec::new()), followers: Mutex::new(Vec::new()), }); // Restart recovery uses the retained input and certificate, never a new job. // Empty journal roots do not fund a channel until an HTTP request arrives. for provider in &gateway.providers { - let _recovery = gateway.submit( + let _recovery = gateway.submit::( vec![( provider.clone(), Route { @@ -364,14 +403,59 @@ pub async fn load_gateway_backend( } impl PaidGateway { - fn submit( + fn execute_tokens( + &self, + request: PaidExecutionRequest, + ) -> CliResult>> { + let permit = self + .admission + .clone() + .try_acquire_owned() + .map_err(|_| hellas_gateway::PaidGatewayBusy)?; + let input_ids = request.input_ids.clone(); + let cache_update = CacheUpdate::from_request(&request.environment, &input_ids); + let prepared = prepare_request(request, &self.settlement_key)?; + let eligible = self + .providers + .iter() + .filter(|provider| check_policy_input(&provider.policy, &prepared).is_ok()) + .collect::>(); + anyhow::ensure!( + !eligible.is_empty(), + "no provider policy matches this environment, token limit, and stop token list" + ); + let start = self.next.fetch_add(1, Ordering::Relaxed) % eligible.len(); + let mut candidates = (0..eligible.len()) + .map(|offset| { + let provider = eligible[(start + offset) % eligible.len()]; + let cache = provider.cache.lock().expect("provider cache poisoned"); + let route = Route { + available: provider.available(), + cache_affinity_tokens: cache.affinity(&input_ids), + pending: provider.pending.load(Ordering::Relaxed), + }; + (provider.clone(), route) + }) + .collect::>(); + // Stable sorting preserves the rotating order for equally ranked routes. + candidates.sort_by_key(|(_, route)| std::cmp::Reverse(route.score())); + self.submit( + candidates, + Some(prepared.into()), + Some(cache_update), + None, + Some(permit), + ) + } + + fn submit( &self, candidates: Vec<(Arc, Route)>, - prepared: Option, + prepared: Option, cache_update: Option, recovery_timeout: Option, permit: Option, - ) -> CliResult>> { + ) -> CliResult>> { // Admission can precede local request preparation. Serialize the final // gate check and registration with drain's close-and-snapshot boundary. let mut tasks = self.tasks.lock().expect("paid task list poisoned"); @@ -380,7 +464,7 @@ impl PaidGateway { } let endpoint = self.endpoint.clone(); let settlement_key = self.settlement_key.clone(); - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::>(OUTPUT_BUFFER_EVENTS); let (overflow, overflow_receiver) = watch::channel(false); let output_budget = Arc::new(Semaphore::new(OUTPUT_BUFFER_BYTES)); let (initial_provider, _initial_route) = candidates.first().expect("paid provider exists"); @@ -409,14 +493,12 @@ impl PaidGateway { let streamed = prepared.is_some(); let recovery = !streamed; let progress: hellas_work::work::PaidProgress = Arc::new(move |event| { - let delta = hellas_rpc::evaluate::decode_token_delta_payload(event.payload()) - .map_err(|error| hellas_work::work::BackendFault::new(error.to_string()))?; - let position = delta.end_position().map_err(|error| hellas_work::work::BackendFault::new(error.to_string()))?; - emit(&token_sender, &token_overflow, &token_budget, Ok(ExecutionEvent::Chunk { position, tokens: delta.token_bytes() })); + let event = E::prefix(event).map_err(|error| hellas_work::work::BackendFault::new(error.to_string()))?; + emit(&token_sender, &token_overflow, &token_budget, Ok(event)); Ok(()) }); let result = async { - let prepared_bytes = prepared.as_ref().map(PreparedPaidInputV1::encode).transpose()?; + let prepared_bytes = prepared.as_ref().map(PreparedPaidWorkInput::encode).transpose()?; let mut provider_errors = Vec::new(); for (provider, route) in candidates { let _occupied = occupied.take().or_else(|| { @@ -427,9 +509,9 @@ impl PaidGateway { task_span.record("hellas.route.pending", route.pending); // Wait for a brief follower pass to yield, but route around // channels still occupied by recovery or other requests. - let mut session = if recovery { + let mut session = if recovery || matches!(prepared, Some(PreparedPaidWorkInput::Fetch(_))) { before_proposal( - &sender, false, deadline, + &sender, streamed, deadline, |_| provider.serial.lock().instrument(hellas_rpc::request_span!(target: "hellas_request", "paid.queue")), ).await? } else { @@ -459,7 +541,7 @@ impl PaidGateway { if session.is_none() { match connect_before_deadline( &sender, streamed, deadline, PROVIDER_CONNECTION_TIMEOUT, - open_paid_channel(&provider.args, endpoint.clone(), settlement_key.clone(), hellas_rpc::Assurance::ProducerSigned), + open_paid_channel(&provider.args, endpoint.clone(), settlement_key.clone(), provider.assurance), ).await { Ok(opened) => { @@ -493,7 +575,7 @@ impl PaidGateway { .map_err(|_| anyhow::anyhow!( "retained paid work did not recover within {RECOVERY_ATTEMPT_TIMEOUT:?}" )) - .and_then(|result| result); + .and_then(|result| result.map_err(anyhow::Error::from)); if let Err(error) = recovery { // Nothing in this path has proposed the fresh request. // Keep the journal for a later recovery and route this @@ -522,11 +604,11 @@ impl PaidGateway { let result = before_proposal( &sender, streamed, deadline, |proposed| async move { - request_session.run_with_admission(input.map(Into::into), true, on_progress, Some(&proposed)).await + request_session.run_with_admission(input, true, on_progress, Some(&proposed)).await }, ).await - .and_then(|result| result) - .and_then(|output| output.map(output_events).transpose()) + .and_then(|result| result.map_err(anyhow::Error::from)) + .and_then(|output| output.map(E::completed).transpose()) .map(Option::unwrap_or_default); if let Err(error) = &result { if !recovery && error.is::() @@ -545,8 +627,8 @@ impl PaidGateway { // is not reflected in memory yet. Keep that failure with // this provider, just like a proposal with no response. let uncertain_append = matches!( - error.downcast_ref::(), - Some(hellas_work::work::ProposeError::Store(_)), + error.downcast_ref::(), + Some(hellas_sdk::paid_client::PaidClientError::Propose(hellas_work::work::ProposeError::Store(_))), ); if !already_proposed && !uncertain_append && session.state().proposal_nonce_high_water() == proposal_nonce @@ -584,7 +666,7 @@ impl PaidGateway { // unfinished operation for recovery. match result { Ok(events) => for event in events { - if !streamed || matches!(event, ExecutionEvent::Done(_)) { + if !streamed || event.is_terminal() { emit(&sender, &overflow, &output_budget, Ok(event)); } }, @@ -598,53 +680,69 @@ impl PaidGateway { } impl PaidExecutionBackend for PaidGateway { - fn timeout(&self) -> Duration { - Duration::from_secs(self.providers[0].args.timeout_secs) + fn fetch_providers(&self) -> Vec { + self.providers.iter().filter(|provider| matches!(provider.policy.execution_policy, + PaidWorkPolicy::Fetch { policy, .. } if policy.allowed_environment == hellas_rpc::FetchEnvironment::Http.manifest_id())) + .map(|provider| provider.args.provider).collect() } - fn execute( + fn fetch( &self, - request: PaidExecutionRequest, - ) -> CliResult>> { + request: PaidFetchRequest, + ) -> Result { + use hellas_gateway::PaidGatewayError; + let provider = self.providers.iter().find(|provider| provider.args.provider == request.provider + && matches!(provider.policy.execution_policy, PaidWorkPolicy::Fetch { policy, .. } + if policy.allowed_environment == hellas_rpc::FetchEnvironment::Http.manifest_id())) + .ok_or(PaidGatewayError::Provider(request.provider))?; let permit = self .admission .clone() .try_acquire_owned() .map_err(|_| hellas_gateway::PaidGatewayBusy)?; - let input_ids = request.input_ids.clone(); - let cache_update = CacheUpdate::from_request(&request.environment, &input_ids); - let prepared = prepare_request(request, &self.settlement_key)?; - let eligible = self - .providers - .iter() - .filter(|provider| check_policy_input(&provider.policy, &prepared).is_ok()) - .collect::>(); - anyhow::ensure!( - !eligible.is_empty(), - "no provider policy matches this environment, token limit, and stop token list" - ); - let start = self.next.fetch_add(1, Ordering::Relaxed) % eligible.len(); - let mut candidates = (0..eligible.len()) - .map(|offset| { - let provider = eligible[(start + offset) % eligible.len()]; - let cache = provider.cache.lock().expect("provider cache poisoned"); - let route = Route { - available: provider.available(), - cache_affinity_tokens: cache.affinity(&input_ids), - pending: provider.pending.load(Ordering::Relaxed), - }; - (provider.clone(), route) - }) - .collect::>(); - // Stable sorting preserves the rotating order for equally ranked routes. - candidates.sort_by_key(|(_, route)| std::cmp::Reverse(route.score())); - self.submit( - candidates, - Some(prepared), - Some(cache_update), - None, - Some(permit), - ) + let submit = || -> CliResult<_> { + let environment = hellas_rpc::FetchEnvironment::Http; + let input = hellas_rpc::fetch::build_input_events_with_retention( + &request.service, + &request.method, + &request.body, + environment.manifest_id(), + provider.assurance, + &self.producer_key, + hellas_rpc::Retention::Ephemeral, + )?; + let prepared = hellas_rpc::protocol::work_fetch::PreparedPaidFetchInputV1::new( + &input, + &environment.manifest(), + )?; + let route = Route { + available: provider.available(), + cache_affinity_tokens: 0, + pending: provider.pending.load(Ordering::Relaxed), + }; + // Account/session selection is already final. Never substitute another + // provider, including on failure before acceptance. + self.submit::( + vec![(provider.clone(), route)], + Some(prepared.into()), + None, + None, + Some(permit), + ) + }; + paid_stream(submit()) + } + + fn timeout(&self) -> Duration { + Duration::from_secs(self.providers[0].args.timeout_secs) + } + + fn execute( + &self, + request: PaidExecutionRequest, + ) -> Result, hellas_gateway::PaidGatewayError> + { + paid_stream(self.execute_tokens(request)) } fn drain(&self) -> BoxFuture<'_, ()> { @@ -687,33 +785,32 @@ impl PaidExecutionBackend for PaidGateway { } } -fn emit( - sender: &mpsc::Sender, +fn paid_stream( + stream: CliResult>>, +) -> Result, hellas_gateway::PaidGatewayError> { + use futures::StreamExt as _; + fn convert(error: anyhow::Error) -> hellas_gateway::PaidGatewayError { + match error.downcast::() { + Ok(busy) => busy.into(), + Err(error) => hellas_gateway::PaidGatewayError::Payment(error.into_boxed_dyn_error()), + } + } + Ok(Box::pin( + stream.map_err(convert)?.map(|event| event.map_err(convert)), + )) +} + +fn emit( + sender: &mpsc::Sender>, overflow: &watch::Sender, budget: &Arc, - event: CliResult, + event: CliResult, ) { if *overflow.borrow() || sender.is_closed() { return; } let bytes = match &event { - Ok(ExecutionEvent::Chunk { tokens, .. }) => tokens.len(), - // Payload bytes only. `OUTPUT_EVENT_OVERHEAD` is charged once below, - // for the one channel message this becomes -- it is the per-message - // constant that makes a full channel (`OUTPUT_BUFFER_EVENTS`) exactly - // exhaust `OUTPUT_BUFFER_BYTES`. Charging it per *contained* event - // instead conflated the transcript with the queue: a completion - // carrying ~8000 signed events, well inside the `MAX_RECORD_BYTES` - // transcript cap and ordinary for `max_new_tokens` in the thousands, - // asked for more permits than the semaphore can ever hold. The - // acquire then failed, `overflow` latched, and the client was told - // "paid output consumer is too slow" for work it had already paid - // for and was reading promptly. - Ok(ExecutionEvent::Done(Outcome::Completed { output_events, .. })) => output_events - .iter() - .map(|event| event.payload().len()) - .fold(0usize, usize::saturating_add), - Ok(ExecutionEvent::Done(_)) => MAX_RECORD_BYTES, + Ok(event) => event.bytes(), Err(error) => error.to_string().len(), } .saturating_add(OUTPUT_EVENT_OVERHEAD); @@ -732,10 +829,10 @@ fn emit( } } -fn response_stream( - mut receiver: mpsc::Receiver, +fn response_stream( + mut receiver: mpsc::Receiver>, mut overflow: watch::Receiver, -) -> BoxStream<'static, CliResult> { +) -> BoxStream<'static, CliResult> { Box::pin(async_stream::try_stream! { loop { let full = *overflow.borrow(); @@ -769,8 +866,8 @@ enum RequestStopped { // A route's short connection allowance is recoverable by trying another // provider; only the enclosing request deadline or disconnect is terminal. -async fn connect_before_deadline( - sender: &mpsc::Sender, +async fn connect_before_deadline( + sender: &mpsc::Sender>, cancel_on_disconnect: bool, deadline: tokio::time::Instant, connection_timeout: Duration, @@ -791,8 +888,8 @@ async fn connect_before_deadline( /// Cancel an unproposed HTTP operation without interrupting work whose /// proposal signature may already have reached a provider. -async fn before_proposal>( - sender: &mpsc::Sender, +async fn before_proposal, E>( + sender: &mpsc::Sender>, cancel_on_disconnect: bool, deadline: tokio::time::Instant, operation: impl FnOnce(Arc) -> F, @@ -903,6 +1000,62 @@ fn output_events(output: PaidOutput) -> CliResult> { Ok(result) } +impl GatewayEvent for ExecutionEvent { + fn prefix(event: hellas_rpc::OutputEventEnvelope) -> CliResult { + let delta = hellas_rpc::evaluate::decode_token_delta_payload(event.payload())?; + Ok(Self::Chunk { + position: delta.end_position()?, + tokens: delta.token_bytes(), + }) + } + fn completed(output: PaidOutput) -> CliResult> { + output_events(output) + } + fn is_terminal(&self) -> bool { + matches!(self, Self::Done(_)) + } + fn bytes(&self) -> usize { + match self { + Self::Chunk { tokens, .. } => tokens.len(), + Self::Done(Outcome::Completed { output_events, .. }) => output_events + .iter() + .map(|event| event.payload().len()) + .fold(0usize, usize::saturating_add), + Self::Done(_) => MAX_RECORD_BYTES, + } + } +} + +impl GatewayEvent for FetchEvent { + fn prefix(event: hellas_rpc::OutputEventEnvelope) -> CliResult { + Ok(hellas_rpc::fetch::decode_fetch_event_payload( + event.payload(), + )?) + } + fn completed(output: PaidOutput) -> CliResult> { + let events = hellas_rpc::protocol::work::decode_transcript( + &output.transcript, + hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES, + )?; + let terminal = events.last().context("paid Fetch omitted terminal")?; + anyhow::ensure!( + terminal.event().body().kind() == hellas_rpc::fetch::OUTPUT_TERMINAL_KIND, + "paid Fetch terminal kind" + ); + let terminal = + hellas_rpc::fetch::decode_fetch_terminal_payload(terminal.payload())?.to_output_event(); + tracing::info!(work_id = %hex::encode(output.work_id.as_bytes()), job_price = output.job_price, + credited_cumulative = output.credited_cumulative, result_bytes = output.transcript.len(), "paid Fetch result acknowledged"); + Ok(vec![terminal]) + } + fn is_terminal(&self) -> bool { + self.terminal().is_some() + } + fn bytes(&self) -> usize { + serde_json::to_vec(self).map_or(MAX_RECORD_BYTES, |bytes| bytes.len()) + } +} + #[cfg(test)] mod tests { use super::*; @@ -910,7 +1063,7 @@ mod tests { #[tokio::test] async fn disconnected_queued_request_never_starts_work() { - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); drop(receiver); let started = AtomicBool::new(false); let result = before_proposal( @@ -928,7 +1081,7 @@ mod tests { #[tokio::test] async fn disconnect_after_proposal_still_finishes_payment() { - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let (proposal_sent, proposal_seen) = tokio::sync::oneshot::channel(); let (payment_ready, payment_wait) = tokio::sync::oneshot::channel(); let work = before_proposal( @@ -954,7 +1107,7 @@ mod tests { #[tokio::test] async fn disconnect_during_dial_cancels_before_signature_release() { - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let signed = AtomicBool::new(false); let signed_ref = &signed; let (dial_started, dial_seen) = tokio::sync::oneshot::channel(); @@ -980,7 +1133,7 @@ mod tests { #[tokio::test] async fn connection_timeout_allows_fallback_within_the_request_deadline() { - let (sender, _receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, _receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let deadline = tokio::time::Instant::now() + Duration::from_secs(1); let failed = connect_before_deadline( &sender, @@ -1014,7 +1167,7 @@ mod tests { #[tokio::test] async fn previous_proposal_does_not_authorize_disconnected_next_operation() { - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let deadline = tokio::time::Instant::now() + Duration::from_secs(1); before_proposal(&sender, true, deadline, |proposed| async move { proposed.store(true, Ordering::Release); @@ -1043,6 +1196,7 @@ mod tests { .await .unwrap(), settlement_key: Secp256k1Signer::from_secret_scalar([7; 32]).unwrap(), + producer_key: hellas_rpc::ProducerSigningKey::from_secret_bytes([7; 32]).unwrap(), admission: Arc::new(Semaphore::new(1)), tasks: Mutex::new(Vec::new()), followers: Mutex::new(Vec::new()), @@ -1051,7 +1205,7 @@ mod tests { // before preparation/routing has reached task registration. let permit = gateway.admission.clone().try_acquire_owned().unwrap(); gateway.drain().await; - let result = gateway.submit(Vec::new(), None, None, None, Some(permit)); + let result = gateway.submit::(Vec::new(), None, None, None, Some(permit)); assert!( result .err() @@ -1087,7 +1241,7 @@ mod tests { #[tokio::test] async fn buffered_reconnect_burst_is_delivered_and_releases_byte_budget() { use futures::StreamExt; - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let (overflow, overflow_receiver) = watch::channel(false); let budget = Arc::new(Semaphore::new(OUTPUT_BUFFER_BYTES)); // Retained output can arrive in one burst before HTTP gets a poll. @@ -1117,7 +1271,7 @@ mod tests { #[tokio::test] async fn slow_reader_gets_an_error_without_blocking_payment() { use futures::StreamExt; - let (sender, receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let (overflow, overflow_receiver) = watch::channel(false); let budget = Arc::new(Semaphore::new(OUTPUT_BUFFER_BYTES)); // A stalled reader cannot retain more than the byte budget, and the @@ -1150,7 +1304,7 @@ mod tests { #[tokio::test] async fn queue_wait_uses_the_execution_deadline() { - let (sender, _receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, _receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let serial = AsyncMutex::new(()); let _busy = serial.lock().await; let result = before_proposal( @@ -1170,7 +1324,7 @@ mod tests { #[tokio::test] async fn expired_request_does_not_start_a_fallback_provider() { - let (sender, _receiver) = mpsc::channel(OUTPUT_BUFFER_EVENTS); + let (sender, _receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); let started = AtomicBool::new(false); let result = before_proposal(&sender, true, tokio::time::Instant::now(), |_| async { started.store(true, Ordering::Relaxed); diff --git a/crates/cli/src/commands/paid_work/tests.rs b/crates/cli/src/commands/paid_work/tests.rs index 32c60768..ff4bbc27 100644 --- a/crates/cli/src/commands/paid_work/tests.rs +++ b/crates/cli/src/commands/paid_work/tests.rs @@ -53,11 +53,10 @@ fn genesis_check_compares_the_configured_digest_with_block_ones_parent() { assert!(check_genesis_payload(&configured, &configured).is_ok()); let observed_parent = [0x32; 32]; - let error = check_genesis_payload(&configured, &observed_parent) - .unwrap_err() - .to_string(); - assert!(error.contains(&hex::encode(observed_parent)), "{error}"); - assert!(error.contains(&hex::encode(configured)), "{error}"); + let error = check_genesis_payload(&configured, &observed_parent).unwrap_err(); + assert!(matches!(error, + hellas_sdk::paid_client::PaidClientError::GenesisMismatch { expected, actual } + if expected == configured && actual == observed_parent)); } #[test] diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 56be9c4a..082f801a 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -469,8 +469,9 @@ enum Commands { mut_arg("tokenizer", |arg| arg.required(false).requires("environment")) )] Gateway { - /// Serve exact HTTP routes through the generic HTTPS Fetch environment. + /// Serve exact HTTP routes through paid HTTPS Fetch. #[arg(long, value_name = "FILE", conflicts_with_all = ["responses_backend", "environment"])] + #[cfg_attr(feature = "node", arg(requires = "paid_work_config"))] http_fetch_config: Option, /// Request/response archive directory (default: ~/.hellas/gateway-archive). #[arg(long, value_name = "DIRECTORY")] @@ -1284,19 +1285,21 @@ async fn async_main() { ); #[cfg(feature = "evaluate")] anyhow::ensure!(!verify_local, "--paid-work-config cannot use --verify-local"); + anyhow::ensure!(http_fetch_config.is_some() || assurance == hellas_rpc::Assurance::ProducerSigned, + "token-native paid work uses producer-signed assurance"); anyhow::ensure!( - assurance == hellas_rpc::Assurance::ProducerSigned, - "paid-work execution uses producer-signed assurance", - ); - anyhow::ensure!( - remote_trust.provider_genesis.is_none(), - "--paid-work-config uses the pool's bond and endpoint identities; omit the courtesy --provider pin", + remote_trust.provider_genesis.is_none() + && remote_trust.apple_app_attest_app_id.is_none() + && remote_trust.apple_app_attest_cdhashes.is_empty(), + "set provider enrollment and Apple trust pins in --paid-work-config", ); Some( commands::paid_work::load_gateway_backend( path, secret_key.clone(), identity::settlement_signer(&local_identity), + local_identity.producer_key.clone(), + assurance, ).await? ) } else { diff --git a/crates/cli/src/tests.rs b/crates/cli/src/tests.rs index 87db0177..36075d1e 100644 --- a/crates/cli/src/tests.rs +++ b/crates/cli/src/tests.rs @@ -445,6 +445,25 @@ fn gateway_requires_explicit_environment_and_tokenizer() { assert!(Cli::try_parse_from(["hellas", "gateway", "--tokenizer", TEST_TOKENIZER]).is_err()); } +#[cfg(all(feature = "gateway", feature = "node"))] +#[test] +fn http_gateway_requires_a_paid_pool() { + assert!( + Cli::try_parse_from(["hellas", "gateway", "--http-fetch-config", "/http.json"]).is_err() + ); + assert!( + Cli::try_parse_from([ + "hellas", + "gateway", + "--http-fetch-config", + "/http.json", + "--paid-work-config", + "/pool.json" + ]) + .is_ok() + ); +} + #[cfg(feature = "llm")] #[test] fn package_flags_are_not_accepted_as_compatibility_aliases() { diff --git a/crates/executor/src/executor/actor/mod.rs b/crates/executor/src/executor/actor/mod.rs index 62e11f8f..8c743449 100644 --- a/crates/executor/src/executor/actor/mod.rs +++ b/crates/executor/src/executor/actor/mod.rs @@ -490,8 +490,12 @@ impl Executor { async fn handle_owed_request(&mut self, request: ExecutorOwedRequest) { match request { - ExecutorOwedRequest::RunPaidFetch { input, reply } => { - self.start_paid_fetch(*input, reply); + ExecutorOwedRequest::RunPaidFetch { + input, + progress, + reply, + } => { + self.start_paid_fetch(*input, progress, reply); } ExecutorOwedRequest::RunPaidEvaluate { input, reply, span } => { #[cfg(feature = "evaluate")] diff --git a/crates/executor/src/executor/actor/paid_fetch.rs b/crates/executor/src/executor/actor/paid_fetch.rs index 618fbf45..4d0af6d1 100644 --- a/crates/executor/src/executor/actor/paid_fetch.rs +++ b/crates/executor/src/executor/actor/paid_fetch.rs @@ -17,6 +17,7 @@ impl Executor { pub(super) fn start_paid_fetch( &mut self, input: PreparedFetchInput, + progress: Option, reply: oneshot::Sender, ExecutorError>>, ) { let prepared = self.prepare_paid_fetch(input); @@ -41,27 +42,46 @@ impl Executor { &key, sender, ); - // The shared runner emits wire events as well as its final signed - // transcript. Drain the bounded channel without publishing or - // retaining a second copy of the response. - let drain = async { while receiver.recv().await.is_some() {} }; - let (result, ()) = tokio::join!(run, drain); - let result = result - .map_err(|_| { - ExecutorError::Execution("paid fetch upstream or projection failed".into()) - }) - .and_then(|run| { - hellas_rpc::protocol::work_fetch::check_fetch_output_limits( - &policy, - &run.output_events, - ) + let drain = async move { + while let Some(event) = receiver.recv().await { + let event = + event.map_err(|error| ExecutorError::Execution(error.to_string()))?; + if let Some(hellas_rpc::pb::execute::work_event::Kind::Chunk(chunk)) = + event.kind + && let Some(progress) = &progress + { + let event = chunk.output_event.ok_or_else(|| { + ExecutorError::Execution( + "paid Fetch chunk omitted its signature".into(), + ) + })?; + let event = hellas_rpc::stream::output_event_from_pb(event) + .map_err(|error| ExecutorError::Execution(error.to_string()))?; + progress(event) + .map_err(|error| ExecutorError::Execution(error.to_string()))?; + } + } + Ok::<_, ExecutorError>(()) + }; + let (result, drained) = tokio::join!(run, drain); + let result = drained.and_then(|()| { + result .map_err(|_| { - ExecutorError::Execution( - "paid fetch output exceeds its signed limits".into(), + ExecutorError::Execution("paid fetch upstream or projection failed".into()) + }) + .and_then(|run| { + hellas_rpc::protocol::work_fetch::check_fetch_output_limits( + &policy, + &run.output_events, ) - })?; - Ok(run.output_events) - }); + .map_err(|_| { + ExecutorError::Execution( + "paid fetch output exceeds its signed limits".into(), + ) + })?; + Ok(run.output_events) + }) + }); let _ = completion .send(ExecutorCompletion::PaidFetch { reply, result }) .await; diff --git a/crates/executor/src/executor/mod.rs b/crates/executor/src/executor/mod.rs index f6a99752..8f9a70b0 100644 --- a/crates/executor/src/executor/mod.rs +++ b/crates/executor/src/executor/mod.rs @@ -109,6 +109,7 @@ pub(crate) enum ExecutorRequest { pub(crate) enum ExecutorOwedRequest { RunPaidFetch { input: Box, + progress: Option, reply: oneshot::Sender, ExecutorError>>, }, /// Start one already-authorized paid job. diff --git a/crates/executor/src/work.rs b/crates/executor/src/work.rs index 73b6965a..896f8e7c 100644 --- a/crates/executor/src/work.rs +++ b/crates/executor/src/work.rs @@ -176,6 +176,21 @@ impl PaidWorkBackend for ExecutorHandle { ) -> Result, BackendFault> { self.send_owed(|reply| ExecutorOwedRequest::RunPaidFetch { input: Box::new(input), + progress: None, + reply, + }) + .await + .map_err(|error| BackendFault::new(error.to_string())) + } + + async fn fetch_stream( + &self, + input: hellas_work::work::PreparedFetchInput, + progress: PaidProgress, + ) -> Result, BackendFault> { + self.send_owed(|reply| ExecutorOwedRequest::RunPaidFetch { + input: Box::new(input), + progress: Some(progress), reply, }) .await diff --git a/crates/gateway/src/http_fetch.rs b/crates/gateway/src/http_fetch.rs index 00e3051c..23395276 100644 --- a/crates/gateway/src/http_fetch.rs +++ b/crates/gateway/src/http_fetch.rs @@ -8,7 +8,7 @@ pub use config::HttpGatewayConfig; #[cfg(test)] mod tests; -use anyhow::{Context, bail, ensure}; +use anyhow::{Context, ensure}; use axum::{ Router, body::{Body, Bytes}, @@ -17,15 +17,11 @@ use axum::{ response::{IntoResponse, Response}, }; use futures::StreamExt; -use hellas_client::{ExecutionRoute, cache::fetch_output_stream}; -use hellas_rpc::{ - Assurance, FetchEnvironment, ProducerSigningKey, Retention, - output::{AdaptorEvent, HttpResponseEvent, OutputEvent, StopReason}, -}; +use hellas_rpc::output::{AdaptorEvent, HttpResponseEvent, OutputEvent, StopReason}; use std::sync::Arc; use tracing::Instrument; -use super::{GatewayHandle, GatewayOptions, access, execution::CliRuntime}; +use super::{GatewayHandle, GatewayOptions, PaidExecutionBackend, PaidFetchRequest, access}; #[derive(Clone)] pub(crate) struct BackendName(pub String); @@ -38,13 +34,23 @@ fn attributed(mut response: Response, name: &str) -> Response { struct HttpState { service: String, method: String, - runtime: CliRuntime, + paid: Arc, routing: Arc, - signer: Arc, - assurance: Assurance, metrics: observation::Metrics, } +#[derive(Debug, thiserror::Error)] +enum HttpOpenError { + #[error("HTTP proxy requires a paid Fetch backend")] + MissingPaidBackend, + #[error("missing authenticated HTTP response head")] + MissingHead, + #[error(transparent)] + Paid(#[from] super::PaidGatewayError), + #[error(transparent)] + Headers(#[from] hellas_rpc::http_fetch::HttpRequestError), +} + pub(super) async fn start(options: GatewayOptions) -> anyhow::Result { let config = options .http_fetch @@ -52,34 +58,22 @@ pub(super) async fn start(options: GatewayOptions) -> anyhow::Result anyhow::Result>, request: Request) -> Respon return error(StatusCode::BAD_REQUEST, "invalid HTTP request"); } }; - let result = open(&state, backend.remote.clone(), &payload) + let result = open(&state, backend.provider, payload) .instrument(observed.span.clone()) .await; let (status, headers, mut events) = match result { Ok(value) => value, + Err(HttpOpenError::Paid(super::PaidGatewayError::Busy(_))) => { + observed.status(503); + return attributed( + error( + StatusCode::SERVICE_UNAVAILABLE, + "paid Fetch is busy; retry later", + ), + &backend.name, + ); + } Err(_) => { observed.status(502); state.routing.transport_failed(selected.backend); @@ -298,37 +302,15 @@ async fn handle(State(state): State>, request: Request) -> Respon async fn open( state: &HttpState, - remote: ExecutionRoute, - payload: &[u8], -) -> anyhow::Result<( - u16, - Vec<(String, String)>, - hellas_adaptors::OutputEventStream, -)> { - let events = hellas_rpc::fetch::build_input_events_with_retention( - &state.service, - &state.method, - payload, - FetchEnvironment::Http.manifest_id(), - state.assurance, - state.signer.as_ref(), - Retention::Ephemeral, - )?; - let request = hellas_rpc::pb::fetch::FetchRequest { - input: events - .iter() - .map(hellas_rpc::stream::input_event_to_pb) - .collect(), - }; - let mut stream = fetch_output_stream( - state.runtime.clone(), - request, - Some(remote), - state.signer.clone(), - None, - ) - .await? - .events; + provider: iroh::EndpointId, + payload: Vec, +) -> Result<(u16, Vec<(String, String)>, super::PaidFetchStream), HttpOpenError> { + let mut stream = state.paid.fetch(PaidFetchRequest { + provider, + service: state.service.clone(), + method: state.method.clone(), + body: payload, + })?; match stream.next().await.transpose()? { Some(OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Head { status, @@ -337,7 +319,7 @@ async fn open( hellas_rpc::http_fetch::check_headers(&headers, false)?; Ok((status, headers, stream)) } - _ => bail!("missing authenticated HTTP response head"), + _ => Err(HttpOpenError::MissingHead), } } diff --git a/crates/gateway/src/http_fetch/config.rs b/crates/gateway/src/http_fetch/config.rs index 700f50ed..5bfab44c 100644 --- a/crates/gateway/src/http_fetch/config.rs +++ b/crates/gateway/src/http_fetch/config.rs @@ -26,25 +26,8 @@ pub struct HttpBackend { pub credential: Option, pub routes: Vec, pub max_in_flight: Option, - pub provider: Option, -} - -#[derive(Clone, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct Provider { - pub node_id: iroh::EndpointId, - #[serde(default)] - pub node_addrs: Vec, - #[serde(deserialize_with = "genesis_from_hex")] - pub genesis: hellas_rpc::ContentId, -} - -fn genesis_from_hex<'de, D: serde::Deserializer<'de>>( - deserializer: D, -) -> Result { - String::deserialize(deserializer)? - .parse() - .map_err(serde::de::Error::custom) + /// Endpoint in the paid pool. May be omitted when there is exactly one. + pub provider: Option, } fn default_concurrency() -> usize { diff --git a/crates/gateway/src/http_fetch/routing.rs b/crates/gateway/src/http_fetch/routing.rs index 95e0b8cb..98da056f 100644 --- a/crates/gateway/src/http_fetch/routing.rs +++ b/crates/gateway/src/http_fetch/routing.rs @@ -2,9 +2,7 @@ use super::{ affinity::{Hints, family}, config::{HttpBackend, HttpGatewayConfig, HttpRoute}, }; -use anyhow::ensure; use axum::http::{HeaderMap, StatusCode}; -use hellas_client::{ExecutionRoute, ProviderTrustAnchor}; use hellas_rpc::ContentId; use std::{ collections::HashMap, @@ -66,11 +64,28 @@ impl Account { const MAX_BINDINGS: usize = 16_384; const SESSION_IDLE: Duration = Duration::from_secs(24 * 60 * 60); +#[derive(Debug, thiserror::Error)] +pub(super) enum RoutingError { + #[error("HTTP backend {backend} must name a provider from the paid pool")] + AmbiguousProvider { backend: String }, + #[error( + "HTTP backend {backend} names provider {provider} without a paid HTTP Fetch configuration" + )] + UnfundedProvider { + backend: String, + provider: iroh::EndpointId, + }, + #[error("HTTP backend {backend} must use one account or origin")] + MixedAccounts { backend: String }, + #[error("HTTP backend {backend} shares an account with a different concurrency limit")] + InconsistentCapacity { backend: String }, +} + pub(super) struct Backend { pub name: String, pub models: Vec, pub routes: Vec, - pub remote: ExecutionRoute, + pub provider: iroh::EndpointId, account: Arc, } @@ -208,9 +223,8 @@ impl Unavailable { impl Routing { pub fn new( config: &HttpGatewayConfig, - remote: ExecutionRoute, - trust: ProviderTrustAnchor, - ) -> anyhow::Result { + providers: &[iroh::EndpointId], + ) -> Result { let routes = config.routes.iter().enumerate().map(|(index, route)| { ( format!("route-{index}"), @@ -226,47 +240,36 @@ impl Routing { let mut backends = Vec::new(); let mut accounts = HashMap::new(); for (name, mut backend) in routes.chain(config.backends.clone()) { - let remote = match backend.provider { - Some(provider) => ExecutionRoute::remote( - Some(provider.node_id), - provider.node_addrs, - 0, - ProviderTrustAnchor { - expected_genesis: provider.genesis, - ..trust.clone() - }, - ), - None => remote.clone(), + let provider = match backend.provider { + Some(provider) => provider, + None if providers.len() == 1 => providers[0], + None => return Err(RoutingError::AmbiguousProvider { backend: name }), }; + if !providers.contains(&provider) { + return Err(RoutingError::UnfundedProvider { + backend: name, + provider, + }); + } for route in &mut backend.routes { route.credential = backend.credential.clone(); } let account = backend.routes[0].account(); - ensure!( - backend.routes.iter().all(|r| r.account() == account), - "a backend must use one account or origin" - ); - // Enrollment pins share admission across direct and discovered routes. - let genesis = match &remote { - ExecutionRoute::RemoteDirect(target) => target.provider_trust.expected_genesis, - ExecutionRoute::RemoteDiscovery { provider_trust, .. } => { - provider_trust.expected_genesis - } - ExecutionRoute::Local => unreachable!(), - }; + if backend.routes.iter().any(|r| r.account() != account) { + return Err(RoutingError::MixedAccounts { backend: name }); + } let capacity = backend.max_in_flight.unwrap_or(config.max_in_flight); let account = accounts - .entry((genesis, account)) + .entry((provider, account)) .or_insert_with(|| Arc::new(Account::new(capacity))); - ensure!( - account.capacity == capacity, - "shared accounts require the same concurrency limit" - ); + if account.capacity != capacity { + return Err(RoutingError::InconsistentCapacity { backend: name }); + } backends.push(Backend { name, models: backend.models, routes: backend.routes, - remote, + provider, account: account.clone(), }); } diff --git a/crates/gateway/src/http_fetch/routing/tests.rs b/crates/gateway/src/http_fetch/routing/tests.rs index db590d4f..be63395d 100644 --- a/crates/gateway/src/http_fetch/routing/tests.rs +++ b/crates/gateway/src/http_fetch/routing/tests.rs @@ -1,7 +1,5 @@ -use super::super::config; use super::*; use axum::http::HeaderValue; -use hellas_rpc::Assurance; use serde_json::json; fn config() -> HttpGatewayConfig { @@ -16,21 +14,37 @@ fn config() -> HttpGatewayConfig { "backends":{"a":backend("a", &["k3"]),"b":backend("b", &["k3"]),"c":backend("c", &["other"])}})).unwrap() } -fn routing(config: HttpGatewayConfig) -> Arc { +fn routing(mut config: HttpGatewayConfig) -> Arc { config.validate().unwrap(); - let trust = hellas_client::ProviderTrustAnchor { - expected_genesis: ContentId::hash(b"provider"), - required_assurance: Assurance::ProducerSigned, - apple_app_attest: None, - }; - Arc::new( - Routing::new( - &config, - ExecutionRoute::remote(None, vec![], 0, trust.clone()), - trust, - ) - .unwrap(), - ) + let default = iroh::SecretKey::from_bytes(&[1; 32]).public(); + let mut providers = vec![default]; + for backend in config.backends.values_mut() { + let provider = *backend.provider.get_or_insert(default); + if !providers.contains(&provider) { + providers.push(provider); + } + } + Arc::new(Routing::new(&config, &providers).unwrap()) +} + +#[test] +fn every_backend_must_resolve_to_a_configured_paid_provider() { + let first = iroh::SecretKey::from_bytes(&[1; 32]).public(); + let second = iroh::SecretKey::from_bytes(&[2; 32]).public(); + let mut config = config(); + assert!(matches!( + Routing::new(&config, &[]), + Err(RoutingError::AmbiguousProvider { .. }) + )); + assert!(matches!( + Routing::new(&config, &[first, second]), + Err(RoutingError::AmbiguousProvider { .. }) + )); + config.backends.get_mut("a").unwrap().provider = Some(second); + assert!(matches!( + Routing::new(&config, &[first]), + Err(RoutingError::UnfundedProvider { .. }) + )); } fn select(routing: &Routing, model: &str, session: &str) -> Result { @@ -294,20 +308,10 @@ fn repeated_aliases_share_limits_but_distinct_provider_nodes_keep_them_separate( StatusCode::SERVICE_UNAVAILABLE ); let node = iroh::SecretKey::generate().public(); - config.backends.get_mut("b").unwrap().provider = Some(config::Provider { - node_id: node, - node_addrs: vec![], - genesis: ContentId::hash(b"other"), - }); + config.backends.get_mut("b").unwrap().provider = Some(node); let mut same_provider = config.clone(); - same_provider - .backends - .get_mut("b") - .unwrap() - .provider - .as_mut() - .unwrap() - .genesis = ContentId::hash(b"provider"); + same_provider.backends.get_mut("b").unwrap().provider = + Some(iroh::SecretKey::from_bytes(&[1; 32]).public()); let same_provider = routing(same_provider); let _first = select(&same_provider, "k3", "first").unwrap(); assert_eq!( @@ -317,9 +321,7 @@ fn repeated_aliases_share_limits_but_distinct_provider_nodes_keep_them_separate( let split = routing(config); let _first = select(&split, "k3", "first").unwrap(); assert_eq!(select(&split, "k3", "second").unwrap().backend, 1); - assert!( - matches!(&split.backends[1].remote, ExecutionRoute::RemoteDirect(target) if target.addr.id==node && target.provider_trust.expected_genesis==ContentId::hash(b"other")) - ); + assert_eq!(split.backends[1].provider, node); } #[test] diff --git a/crates/gateway/src/http_fetch/tests.rs b/crates/gateway/src/http_fetch/tests.rs index cfb62d6b..66df894a 100644 --- a/crates/gateway/src/http_fetch/tests.rs +++ b/crates/gateway/src/http_fetch/tests.rs @@ -6,6 +6,98 @@ use std::{ time::{Duration, SystemTime}, }; +#[tokio::test] +async fn http_uses_the_paid_backend_and_waits_for_its_payment_completion() { + use std::sync::atomic::{AtomicUsize, Ordering}; + struct Paid { + requests: AtomicUsize, + ack: Arc, + provider: iroh::EndpointId, + busy: bool, + } + impl PaidExecutionBackend for Paid { + fn execute( + &self, + _: crate::PaidExecutionRequest, + ) -> Result, crate::PaidGatewayError> + { + unreachable!("HTTP must use Fetch") + } + fn fetch( + &self, + request: PaidFetchRequest, + ) -> Result { + self.requests.fetch_add(1, Ordering::Relaxed); + assert_eq!(request.provider, self.provider); + assert_eq!((&*request.service, &*request.method), ("http", "request")); + let http = hellas_rpc::http_fetch::HttpFetchRequest::decode(&request.body).unwrap(); + assert_eq!(http.url, "https://example.com/v1/responses?x=%2F&x=y"); + assert_eq!(http.body().unwrap(), b"opaque request"); + assert_eq!(http.credential.as_deref(), Some("account")); + if self.busy { + return Err(crate::PaidGatewayBusy.into()); + } + let ack = self.ack.clone(); + Ok(Box::pin(async_stream::try_stream! { + yield OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Head { + status: 200, headers: vec![("content-type".into(), "application/octet-stream".into())], + })); + yield OutputEvent::Adaptor(AdaptorEvent::Http(HttpResponseEvent::Body { base64: "AP8K".into() })); + ack.notified().await; + yield OutputEvent::Finished { stop_reason: StopReason::EndOfText, usage: None }; + })) + } + fn drain(&self) -> futures::future::BoxFuture<'_, ()> { + Box::pin(async {}) + } + } + for busy in [false, true] { + let provider = iroh::SecretKey::from_bytes(&[12; 32]).public(); + let paid = Arc::new(Paid { + requests: AtomicUsize::new(0), + ack: Arc::default(), + provider, + busy, + }); + let config: HttpGatewayConfig = serde_json::from_value(serde_json::json!({ + "service":"http", "method":"request", "routes":[{ + "path":"/v1/responses", "method":"POST", "url":"https://example.com/v1/responses", "credential":"account" + }] + })).unwrap(); + let state = Arc::new(HttpState { + service: config.service.clone(), + method: config.method.clone(), + routing: Arc::new(routing::Routing::new(&config, &[provider]).unwrap()), + paid: paid.clone(), + metrics: observation::Metrics::new(), + }); + let response = handle( + State(state), + Request::builder() + .method("POST") + .uri("/v1/responses?x=%2F&x=y") + .body(Body::from("opaque request")) + .unwrap(), + ) + .await; + assert_eq!(paid.requests.load(Ordering::Relaxed), 1); + if busy { + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + } else { + assert_eq!(response.status(), StatusCode::OK); + let mut body = response.into_body().into_data_stream(); + assert_eq!(body.next().await.unwrap().unwrap(), &b"\x00\xff\n"[..]); + assert!( + tokio::time::timeout(Duration::from_millis(20), body.next()) + .await + .is_err() + ); + paid.ack.notify_one(); + assert!(body.next().await.is_none()); + } + } +} + #[test] fn forward_only_protocol_headers_and_keep_retry_and_quota_metadata() { let route = HttpRoute { diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index dc0e52a5..d33dd64e 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -55,19 +55,46 @@ pub struct PaidExecutionRequest { pub stop_token_ids: Vec, } +/// A Fetch pinned to one funded provider by the HTTP account router. +pub struct PaidFetchRequest { + pub provider: EndpointId, + pub service: String, + pub method: String, + pub body: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum PaidGatewayError { + #[error("paid backend does not support HTTP Fetch")] + Unsupported, + #[error("provider {0} has no paid HTTP Fetch configuration")] + Provider(EndpointId), + #[error(transparent)] + Busy(#[from] PaidGatewayBusy), + #[error("paid work failed: {0}")] + Payment(#[source] Box), +} + +pub type PaidOutputStream = futures::stream::BoxStream<'static, Result>; +pub type PaidFetchStream = PaidOutputStream; + /// Paid admission capacity is exhausted or the backend is shutting down. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] +#[error("paid gateway is busy; retry later")] pub struct PaidGatewayBusy; -impl std::fmt::Display for PaidGatewayBusy { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("paid gateway is busy; retry later") +pub trait PaidExecutionBackend: Send + Sync { + /// Providers with a funded-pool configuration for the HTTP Fetch manifest. + fn fetch_providers(&self) -> Vec { + Vec::new() } -} -impl std::error::Error for PaidGatewayBusy {} + /// Authenticated prefixes, followed by completion only after payment ACK. + /// After proposal the backend owns collection and payment across disconnects. + fn fetch(&self, _request: PaidFetchRequest) -> Result { + Err(PaidGatewayError::Unsupported) + } -pub trait PaidExecutionBackend: Send + Sync { /// End-to-end budget, including queued time, advertised to HTTP consumers. fn timeout(&self) -> std::time::Duration { std::time::Duration::from_secs(300) @@ -81,7 +108,7 @@ pub trait PaidExecutionBackend: Send + Sync { fn execute( &self, request: PaidExecutionRequest, - ) -> anyhow::Result>>; + ) -> Result, PaidGatewayError>; /// Finish outstanding payment operations during graceful shutdown. fn drain(&self) -> futures::future::BoxFuture<'_, ()>; @@ -574,8 +601,7 @@ mod paid_shutdown_tests { fn execute( &self, _: PaidExecutionRequest, - ) -> anyhow::Result>> - { + ) -> Result, PaidGatewayError> { unreachable!("shutdown does not submit new work") } fn drain(&self) -> futures::future::BoxFuture<'_, ()> { diff --git a/crates/gateway/src/state.rs b/crates/gateway/src/state.rs index f859b0e5..557a0da4 100644 --- a/crates/gateway/src/state.rs +++ b/crates/gateway/src/state.rs @@ -395,22 +395,13 @@ impl GatewayState { max_new_tokens: max_tokens, stop_token_ids: self.stop_token_ids.clone(), }) - .map_err(|error| { - // anyhow's allocation-preserving conversion hides the - // concrete error type from std::error::Error downcasts. - let error: Box = - match error.downcast::() { - Ok(busy) => Box::new(busy), - Err(error) => error.into_boxed_dyn_error(), - }; - hellas_client::ClientError::External(error) - })?; + .map_err(|error| hellas_client::ClientError::External(Box::new(error)))?; Ok(( None, payment .map(|result| { result.map_err(|error| { - hellas_client::ClientError::External(error.into_boxed_dyn_error()) + hellas_client::ClientError::External(Box::new(error)) }) }) .boxed(), @@ -419,7 +410,10 @@ impl GatewayState { .await .map_err(|error| match error { hellas_client::ClientError::External(error) => HttpError { - status: if error.is::() { + status: if matches!( + error.downcast_ref::(), + Some(super::PaidGatewayError::Busy(_)) + ) { StatusCode::SERVICE_UNAVAILABLE } else { StatusCode::BAD_REQUEST diff --git a/crates/gateway/src/state/tests.rs b/crates/gateway/src/state/tests.rs index 8491c813..61afc596 100644 --- a/crates/gateway/src/state/tests.rs +++ b/crates/gateway/src/state/tests.rs @@ -355,8 +355,7 @@ async fn paid_generation_records_after_payment_and_replays_without_a_backend() { fn execute( &self, request: PaidExecutionRequest, - ) -> anyhow::Result>> - { + ) -> Result, crate::PaidGatewayError> { self.calls.fetch_add(1, Ordering::SeqCst); assert_eq!(request.input_ids, vec![0]); assert_eq!(request.max_new_tokens, 1); @@ -479,9 +478,8 @@ async fn paid_capacity_is_a_retryable_error_and_uses_the_pool_deadline() { fn execute( &self, _: crate::PaidExecutionRequest, - ) -> anyhow::Result< - futures::stream::BoxStream<'static, anyhow::Result>, - > { + ) -> Result, crate::PaidGatewayError> + { Err(crate::PaidGatewayBusy.into()) } fn drain(&self) -> futures::future::BoxFuture<'_, ()> { diff --git a/crates/providers/src/responses_fetch/telemetry/otel.rs b/crates/providers/src/responses_fetch/telemetry/otel.rs index 53d86194..177855c4 100644 --- a/crates/providers/src/responses_fetch/telemetry/otel.rs +++ b/crates/providers/src/responses_fetch/telemetry/otel.rs @@ -43,10 +43,9 @@ impl Request { .get(name) .and_then(|v| v.as_text()) .filter(|value| !value.is_empty()) + && let Ok(value) = value.parse() { - if let Ok(value) = value.parse() { - request.headers_mut().insert(name, value); - } + request.headers_mut().insert(name, value); } } } diff --git a/crates/rpc/src/fetch.rs b/crates/rpc/src/fetch.rs index 314066f2..24f55e41 100644 --- a/crates/rpc/src/fetch.rs +++ b/crates/rpc/src/fetch.rs @@ -11,8 +11,8 @@ use k256::elliptic_curve::rand_core::{OsRng, RngCore}; const INPUT_CANONICALIZATION: &[u8] = b"hellas.fetch.input.v3"; const OUTPUT_CANONICALIZATION: &[u8] = b"hellas.fetch.output.v2"; -const OUTPUT_EVENT_KIND: &str = "response.event"; -const OUTPUT_TERMINAL_KIND: &str = "response.terminal"; +pub const OUTPUT_EVENT_KIND: &str = "response.event"; +pub const OUTPUT_TERMINAL_KIND: &str = "response.terminal"; const INPUT_EVENT_KINDS: [&str; 8] = [ "assurance", "execution.environment", diff --git a/crates/rpc/src/protocol/work_fetch.rs b/crates/rpc/src/protocol/work_fetch.rs index 9b33bb45..ebe2a784 100644 --- a/crates/rpc/src/protocol/work_fetch.rs +++ b/crates/rpc/src/protocol/work_fetch.rs @@ -31,6 +31,10 @@ const PREPARED_FETCH_INPUT: &[u8] = b"hellas.work.prepared-fetch-input.v1"; /// The normalized fetch answer authenticated by the signed transcript. const FETCH_OUTPUT: &[u8] = b"hellas.work.fetch-output.v1"; +/// RAM bound for a complete Fetch transcript, including signed envelopes. +/// Wire frames remain independently bounded; Fetch journals omit this payload. +pub const MAX_FETCH_TRANSCRIPT_BYTES: usize = 32 * 1024 * 1024; + // ── The route policy ────────────────────────────────────────────────── /// Schema tag of the sealed-route body. diff --git a/crates/sdk/Cargo.toml b/crates/sdk/Cargo.toml index 7c789e53..486ff328 100644 --- a/crates/sdk/Cargo.toml +++ b/crates/sdk/Cargo.toml @@ -49,6 +49,7 @@ paid-work = [ ] [dependencies] +thiserror.workspace = true async-stream = { workspace = true, optional = true } hellas-chain = { workspace = true, default-features = false, optional = true } hellas-kernel = { workspace = true, default-features = false, optional = true } diff --git a/crates/sdk/src/paid_client.rs b/crates/sdk/src/paid_client.rs index e1ebb954..3cd97686 100644 --- a/crates/sdk/src/paid_client.rs +++ b/crates/sdk/src/paid_client.rs @@ -5,7 +5,9 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; -use anyhow::{Context as _, Result, bail}; +mod error; +pub use error::PaidClientError; +type Result = std::result::Result; use hellas_chain::client::VerifiedRemoteLightClient; use hellas_chain::{ ConsensusInfo, ConsensusVerifier, FinalizedWorkView as _, WorkBlocks, WorkChannelQuery, @@ -91,10 +93,9 @@ pub async fn run_paid_work( transport_key: SecretKey, settlement_key: Secp256k1Signer, ) -> Result { - anyhow::ensure!( - !args.timeout.is_zero(), - "paid-work timeout must be positive" - ); + if args.timeout.is_zero() { + return Err(PaidClientError::InvalidOptions("timeout must be positive")); + } tokio::time::timeout(args.timeout, async move { let PaidWorkRun { config, @@ -137,7 +138,9 @@ pub async fn run_paid_work( let mut result = session .run(Some(prepared_input), false, None) .await? - .context("paid execution returned no result")?; + .ok_or(PaidClientError::MissingState( + "paid execution returned no result", + ))?; if settle { result.settled_provider_payout = Some(session.settle().await?); } @@ -145,9 +148,7 @@ pub async fn run_paid_work( Ok(result) }) .await - .map_err(|_| { - anyhow::anyhow!("paid-work run timed out; client journals retain its payment state") - })? + .map_err(|_| PaidClientError::Timeout { stage: "paid job" })? } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct InputIdentities { @@ -158,20 +159,15 @@ pub struct InputIdentities { impl InputIdentities { pub fn from_prepared(prepared: &PreparedPaidInputV1) -> Result { - let parts = prepared - .parts() - .context("prepared input contains a non-canonical body")?; + let parts = prepared.parts()?; Self::from_parts(&parts) } fn from_parts(parts: &hellas_rpc::protocol::artifacts::PreparedPaidInputParts) -> Result { let allowed_environment = parts.manifest.content_id(); - anyhow::ensure!( - parts.evaluate_request.execution_environment == allowed_environment, - "prepared input request names environment {}, but its manifest derives {}", - parts.evaluate_request.execution_environment, - allowed_environment, - ); + if parts.evaluate_request.execution_environment != allowed_environment { + return Err(PaidClientError::InputMismatch("environment manifest")); + } Ok(Self { allowed_environment, generation_policy_digest: generation_policy_digest( @@ -202,14 +198,14 @@ impl PaidWorkSession { endpoint: Endpoint, settlement_key: Secp256k1Signer, ) -> Result { - anyhow::ensure!( - args.acceptance_blocks > 0 && args.terminal_blocks > 0 && args.payment_blocks > 0, - "all three deadline spans must be greater than zero", - ); - anyhow::ensure!( - !args.timeout.is_zero(), - "paid-work timeout must be positive" - ); + if !(args.acceptance_blocks > 0 && args.terminal_blocks > 0 && args.payment_blocks > 0) { + return Err(PaidClientError::InvalidOptions( + "deadline spans must be positive", + )); + } + if args.timeout.is_zero() { + return Err(PaidClientError::InvalidOptions("timeout must be positive")); + } let config = &args.config; let policy = config.provider_policy(); @@ -219,11 +215,11 @@ impl PaidWorkSession { let chain = connect_chain(config, &mut next_validator).await?; check_genesis(config, &chain).await?; - std::fs::create_dir_all(&args.journal_root).with_context(|| { - format!( - "failed to create client journal root {}", - args.journal_root.display(), - ) + std::fs::create_dir_all(&args.journal_root).map_err(|source| { + PaidClientError::JournalDirectory { + path: args.journal_root.clone(), + source, + } })?; let store = SetupStore::open( &args.journal_root, @@ -231,13 +227,7 @@ impl PaidWorkSession { bond, Role::Client, &Secp256k1Verifier::new(), - ) - .with_context(|| { - format!( - "failed to open client setup journal under {}", - args.journal_root.display(), - ) - })?; + )?; let mut setup = SetupEndpoint::new( store, settlement_key.clone(), @@ -257,17 +247,15 @@ impl PaidWorkSession { .state() .bundle() .cloned() - .context("provider returned no bond proposal")?; - anyhow::ensure!( - bundle.bond_edge() == bond, - "provider proposed a different bond edge" - ); - anyhow::ensure!( - bundle.bond_terms().parties.taker() == settlement_key.party_key(), - "provider bond names client settlement key {}, not this identity's {}", - hex::encode(bundle.bond_terms().parties.taker().to_bytes()), - hex::encode(settlement_key.party_key().to_bytes()), - ); + .ok_or(PaidClientError::MissingState( + "provider returned no bond proposal", + ))?; + if bundle.bond_edge() != bond { + return Err(PaidClientError::InputMismatch("bond edge")); + } + if bundle.bond_terms().parties.taker() != settlement_key.party_key() { + return Err(PaidClientError::InputMismatch("client settlement identity")); + } dialer.require_producer(hellas_rpc::PublicKey::Secp256k1( bundle.bond_terms().parties.maker().to_bytes(), ))?; @@ -281,10 +269,11 @@ impl PaidWorkSession { if setup.state().revision() == Some(2) { exchange_setup(&dialer, &mut setup).await?; } - anyhow::ensure!( - setup.state().revision() == Some(3), - "setup did not reach its countersigned revision", - ); + if setup.state().revision() != Some(3) { + return Err(PaidClientError::MissingState( + "countersigned setup revision", + )); + } let setup_service = SetupService::new(setup); let (mounted, descriptor) = @@ -320,16 +309,9 @@ impl PaidWorkSession { /// Opens the client close and waits for its finalized provider payout. pub async fn settle(&mut self) -> Result { - self.client - .prepare_close() - .context("failed to prepare the client payment close")?; + self.client.prepare_close()?; loop { - match self - .client - .advance_close(&self.chain, &self.chain) - .await - .context("failed to advance the client payment close")? - { + match self.client.advance_close(&self.chain, &self.chain).await? { CloseProgress::Settled { provider_payout } => return Ok(provider_payout), CloseProgress::Submitted { outcome, .. } => { tracing::info!(?outcome, "client payment close submitted") @@ -355,13 +337,13 @@ impl PaidWorkSession { } } } - bail!("no configured validator supplied finalized history") + Err(PaidClientError::NoValidators) } /// Runs a request and pays only after verifying its complete result. /// With `recover`, resume payable journaled work first and admit this as a new /// job; otherwise reuse a still-active job matching the input. `None` only - /// performs recovery. Incremental progress is currently supported for Evaluate. + /// performs recovery. Prefixes are authenticated before incremental delivery. pub async fn run( &mut self, prepared: Option, @@ -399,10 +381,6 @@ impl PaidWorkSession { dialer.trust.as_ref(), hellas_rpc::PublicKey::Secp256k1(descriptor.channel().client_key().to_bytes()), )?; - anyhow::ensure!( - progress.is_none() || matches!(prepared, PreparedPaidWorkInput::Evaluate(_)), - "incremental result delivery is only supported for Evaluate" - ); } let ready = caught_up_channel(descriptor, client, &*chain).await?; if recover && *needs_recovery { @@ -430,8 +408,17 @@ impl PaidWorkSession { payable }) .map(|job| { + // Fetch journals retain accounting only. A restart cannot + // reconstruct a lost request or authorize another execution. + if job.prepared_input().is_empty() { + return Err(PaidClientError::MissingPayload { + work_id: job.work_id(), + payment_deadline: job.authorization().payment_deadline, + }); + } PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) .map(|input| (job.work_id(), job.phase(), input)) + .map_err(PaidClientError::from) }) .collect::, _>>()?; for (work_id, phase, pending) in pending { @@ -457,8 +444,8 @@ impl PaidWorkSession { if let Err(error) = &result && ((phase == hellas_work::work_store::JobPhase::HalfSigned && matches!( - error.downcast_ref::(), - Some(hellas_work::work::ProposeError::Refused { + error, + PaidClientError::Propose(hellas_work::work::ProposeError::Refused { refusal, .. }) if !refusal.is_retryable() @@ -506,15 +493,14 @@ impl PaidWorkSession { } } -fn permanently_refused_delivery(error: &anyhow::Error) -> bool { +fn permanently_refused_delivery(error: &PaidClientError) -> bool { use hellas_client::work::CollectResultError; use hellas_work::work::DeliverError; - let delivery = error.downcast_ref::().or_else(|| { - match error.downcast_ref::() { - Some(CollectResultError::Deliver(delivery)) => Some(delivery), - _ => None, - } - }); + let delivery = match error { + PaidClientError::Deliver(error) + | PaidClientError::Collect(CollectResultError::Deliver(error)) => Some(error), + _ => None, + }; matches!(delivery, Some(DeliverError::Refused { refusal, .. }) if !refusal.is_retryable()) } @@ -536,7 +522,9 @@ async fn propose_when_ready( let mut delay = poll.max(Duration::from_secs(1)); loop { if Instant::now() >= deadline { - bail!("provider remained not ready for {timeout:?}"); + return Err(PaidClientError::Timeout { + stage: "provider admission", + }); } let transport = dialer.work().await?; // Once a proposal can leave this process, a lost acknowledgement must @@ -609,10 +597,9 @@ async fn execute_paid_job( }) .map(|job| (job.work_id(), job.phase(), *job.authorization())) .collect::>(); - anyhow::ensure!( - existing.len() <= 1, - "more than one active job matches this prepared input; inspect the retained channel journal", - ); + if existing.len() > 1 { + return Err(PaidClientError::AmbiguousRecovery); + } if !existing.is_empty() && let Some(proposed) = proposed { @@ -654,8 +641,12 @@ async fn execute_paid_job( .state() .job_by_id(work_id) .map(|job| job.transcript().to_vec()) - .context("collected job disappeared from its journal")? - } else if let Some(progress) = progress { + .ok_or(PaidClientError::MissingState( + "collected job disappeared from its journal", + ))? + } else if progress.is_some() + || matches!(proposal.prepared_input, PreparedPaidWorkInput::Fetch(_)) + { let mut emitted = false; let delivery = loop { let result = hellas_work::work::fetch_result_stream( @@ -665,9 +656,12 @@ async fn execute_paid_job( work_id, |event| { emitted = true; - progress(event.clone()).map_err(|error| { - hellas_rpc::protocol::work::PaidWorkError::Transcript(error.to_string()) - }) + if let Some(progress) = progress { + progress(event.clone()).map_err(|error| { + hellas_rpc::protocol::work::PaidWorkError::Transcript(error.to_string()) + })?; + } + Ok(()) }, ) .await; @@ -691,25 +685,24 @@ async fn execute_paid_job( let job = client .state() .job_by_id(work_id) - .context("accepted job disappeared")?; - anyhow::ensure!( - client.state().cursor().0 <= job.authorization().payment_deadline, - "payment deadline elapsed while waiting for result stream" - ); + .ok_or(PaidClientError::MissingState("accepted job disappeared"))?; + if client.state().cursor().0 > job.authorization().payment_deadline { + return Err(PaidClientError::PaymentExpired); + } tracing::debug!(%error, %work_id, "waiting for paid result stream readiness"); tokio::time::sleep(poll.max(Duration::from_secs(1))).await; }; client.catch_up(chain).await?; - anyhow::ensure!( - client.state().cursor().0 - <= client - .state() - .job_by_id(work_id) - .context("accepted job disappeared")? - .authorization() - .payment_deadline, - "payment deadline elapsed during delivery" - ); + if client.state().cursor().0 + > client + .state() + .job_by_id(work_id) + .ok_or(PaidClientError::MissingState("accepted job disappeared"))? + .authorization() + .payment_deadline + { + return Err(PaidClientError::PaymentExpired); + } delivery.transcript } else { collect_until_ready(dialer, client, ready, chain, work_id, poll).await? @@ -733,16 +726,19 @@ fn check_request( caller: hellas_rpc::PublicKey, ) -> Result<()> { let assurance = prepared.assurance()?; - anyhow::ensure!( - trust - .as_ref() - .is_none_or(|trust| trust.required_assurance == assurance), - "paid request assurance differs from provider trust" - ); - anyhow::ensure!( - assurance == hellas_rpc::Assurance::ProducerSigned || trust.is_some(), - "attested paid work requires a provider trust anchor before disclosure" - ); + if trust + .as_ref() + .is_some_and(|trust| trust.required_assurance != assurance) + { + return Err(PaidClientError::InputMismatch( + "assurance differs from provider trust", + )); + } + if assurance != hellas_rpc::Assurance::ProducerSigned && trust.is_none() { + return Err(PaidClientError::InvalidOptions( + "attested work requires a provider trust anchor", + )); + } match (prepared, &policy.execution_policy) { (PreparedPaidWorkInput::Evaluate(input), PaidWorkPolicy::Evaluate(_)) => { check_evaluate_input(policy, input)?; @@ -750,31 +746,29 @@ fn check_request( (PreparedPaidWorkInput::Fetch(input), PaidWorkPolicy::Fetch { policy, route }) => { let parts = input.parts()?; let request = hellas_rpc::fetch::verify_input_events(&parts.fetch_input_transcript)?; - anyhow::ensure!( - request.caller_key == caller, - "prepared fetch caller does not match the client identity" - ); - anyhow::ensure!( - request.execution_environment == policy.allowed_environment - && parts.manifest.content_id() == policy.allowed_environment, - "prepared fetch environment does not match work config" - ); - anyhow::ensure!( - request.retention == hellas_rpc::Retention::Ephemeral, - "paid fetch requires ephemeral retention" - ); + if request.caller_key != caller { + return Err(PaidClientError::InputMismatch("Fetch caller")); + } + if !(request.execution_environment == policy.allowed_environment + && parts.manifest.content_id() == policy.allowed_environment) + { + return Err(PaidClientError::InputMismatch("Fetch environment")); + } + if request.retention != hellas_rpc::Retention::Ephemeral { + return Err(PaidClientError::InputMismatch( + "Fetch retention must be ephemeral", + )); + } if let hellas_rpc::protocol::work_fetch::FetchRoutePolicy::SealedRoute { service, method, } = route + && (&request.service != service || &request.method != method) { - anyhow::ensure!( - &request.service == service && &request.method == method, - "prepared fetch route does not match work config" - ); + return Err(PaidClientError::InputMismatch("Fetch route")); } } - _ => bail!("prepared input and work config select different profiles"), + _ => return Err(PaidClientError::InputMismatch("work profile")), } Ok(()) } @@ -786,22 +780,17 @@ pub fn check_evaluate_input( let parts = prepared.parts()?; let input = InputIdentities::from_parts(&parts)?; let PaidWorkPolicy::Evaluate(expected) = &policy.execution_policy else { - bail!("work config does not select the Evaluate profile"); + return Err(PaidClientError::InputMismatch("expected Evaluate profile")); }; - anyhow::ensure!( - expected.allowed_environment == input.allowed_environment, - "work config allows environment {}, but prepared input uses {}", - expected.allowed_environment, - input.allowed_environment, - ); - anyhow::ensure!( - hellas_rpc::protocol::work::matches_generation_policy(expected, &parts.text_policy)?, - "work config generation_policy_digest does not match prepared input", - ); - anyhow::ensure!( - expected.identity_source_digest == input.identity_source_digest, - "work config identity_source_digest does not match prepared input", - ); + if expected.allowed_environment != input.allowed_environment { + return Err(PaidClientError::InputMismatch("Evaluate environment")); + } + if !(hellas_rpc::protocol::work::matches_generation_policy(expected, &parts.text_policy)?) { + return Err(PaidClientError::InputMismatch("generation policy")); + } + if expected.identity_source_digest != input.identity_source_digest { + return Err(PaidClientError::InputMismatch("identity source")); + } Ok(()) } @@ -832,21 +821,17 @@ async fn drive_setup( poll: Duration, ) -> Result<(hellas_work::work_store::ChannelStore, WorkChannelDescriptor)> { loop { - let SetupAdvance { progress, mounted } = setup - .advance_setup(chain, chain, chain) - .await - .context("failed to advance paid-work setup")?; + let SetupAdvance { progress, mounted } = setup.advance_setup(chain, chain, chain).await?; if let Some(store) = mounted { let channel = store.state().channel(); - let descriptor = policy - .admit(channel.payment_edge(), channel.payment_terms().clone()) - .context("the funded channel no longer satisfies the configured policy")?; + let descriptor = + policy.admit(channel.payment_edge(), channel.payment_terms().clone())?; return Ok((store, descriptor)); } match progress { - SetupProgress::Aborted(reason) => bail!("paid-work setup aborted: {reason:?}"), - SetupProgress::Faulted(reason) => bail!("paid-work setup faulted: {reason:?}"), - SetupProgress::TimeoutBond => bail!("provider bond timed out before setup completed"), + end @ (SetupProgress::Aborted(_) + | SetupProgress::Faulted(_) + | SetupProgress::TimeoutBond) => return Err(PaidClientError::SetupEnded(end)), _ => tokio::time::sleep(poll).await, } } @@ -861,13 +846,16 @@ async fn ready_channel( payment_edge: descriptor.channel().payment_edge(), funding: Default::default(), }; - let snapshot = chain - .work_channel_snapshot(query) - .await? - .context("no finalized channel snapshot is available")?; + let snapshot = + chain + .work_channel_snapshot(query) + .await? + .ok_or(PaidClientError::MissingState( + "no finalized channel snapshot is available", + ))?; descriptor .check_ready(&snapshot.observed_channel()) - .context("the finalized channel is not ready") + .map_err(PaidClientError::from) } /// Reads the ready snapshot once the client has processed it. @@ -923,15 +911,18 @@ pub fn deadlines( terminal_blocks: u64, payment_blocks: u64, ) -> Result { - let acceptance = current - .checked_add(acceptance_blocks) - .context("acceptance deadline overflow")?; + let acceptance = + current + .checked_add(acceptance_blocks) + .ok_or(PaidClientError::DeadlineOverflow { + stage: "acceptance", + })?; let terminal = acceptance .checked_add(terminal_blocks) - .context("terminal deadline overflow")?; + .ok_or(PaidClientError::DeadlineOverflow { stage: "terminal" })?; let payment = terminal .checked_add(payment_blocks) - .context("payment deadline overflow")?; + .ok_or(PaidClientError::DeadlineOverflow { stage: "payment" })?; Ok(JobDeadlines { acceptance, terminal, @@ -950,7 +941,7 @@ pub async fn bind_paid_endpoint(secret_key: SecretKey) -> Result { ]) .bind() .await - .context("failed to bind paid-work Iroh endpoint") + .map_err(PaidClientError::from) } async fn exchange_setup(dialer: &ProviderDialer, setup: &mut SetupEndpoint) -> Result<()> { @@ -989,11 +980,10 @@ impl ProviderDialer { let mut expected = self .producer .lock() - .map_err(|_| anyhow::anyhow!("provider key lock poisoned"))?; - anyhow::ensure!( - expected.as_ref().is_none_or(|old| *old == key), - "authenticated provider key differs from the payment channel" - ); + .map_err(|_| PaidClientError::ProviderIdentityPoisoned)?; + if expected.as_ref().is_some_and(|old| *old != key) { + return Err(PaidClientError::ProviderIdentityChanged); + } *expected = Some(key); Ok(()) } @@ -1013,7 +1003,10 @@ impl ProviderDialer { .endpoint .connect(self.provider.clone(), alpn) .await - .with_context(|| format!("failed to connect to provider {}", self.provider.id))?; + .map_err(|source| PaidClientError::Connect { + provider: self.provider.id, + source, + })?; let transport = IrohTransport::new(connection); if let Some(trust) = &self.trust { let producer = if alpn == hellas_rpc::services::work::Work::ALPN.as_bytes() { @@ -1041,8 +1034,7 @@ async fn connect_chain( validators: config.validators.clone(), threshold_identity: config.chain.threshold_identity.clone(), network_id: config.chain.network.as_str().to_owned(), - }) - .context("configured threshold identity is unusable")?; + })?; // A peer can accept connections while lacking a historical certificate. // Reconnects must make progress through the configured alternatives. let start = *next_validator; @@ -1057,10 +1049,10 @@ async fn connect_chain( *next_validator = (*next_validator + 1) % config.validators.len(); match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { Ok(client) => return Ok(WorkBlocks::new(client)), - Err(error) => failures.push(format!("{url}: {error}")), + Err(error) => failures.push((url.clone(), error)), } } - bail!("no configured validator answered: {}", failures.join("; ")) + Err(PaidClientError::ValidatorsUnavailable(failures)) } async fn check_genesis( @@ -1070,21 +1062,21 @@ async fn check_genesis( let first = chain .block_at(1) .await? - .context( - "configured validator has no finalized block 1; genesis cannot be authenticated until block 1 is finalized", - )?; + .ok_or(PaidClientError::MissingState( + "finalized block 1 for genesis authentication", + ))?; check_genesis_payload( config.chain.genesis_payload_digest.as_bytes(), &first.parent, ) } pub fn check_genesis_payload(expected: &[u8; 32], actual: &[u8; 32]) -> Result<()> { - anyhow::ensure!( - actual == expected, - "validator genesis payload {} does not match configured {}", - hex::encode(actual), - hex::encode(expected), - ); + if actual != expected { + return Err(PaidClientError::GenesisMismatch { + expected: *expected, + actual: *actual, + }); + } Ok(()) } @@ -1092,11 +1084,15 @@ async fn finalized_floor(chain: &WorkBlocks) -> Resul let height = chain .latest_height() .await? - .context("configured validator has finalized no blocks")?; + .ok_or(PaidClientError::MissingState( + "configured validator has finalized no blocks", + ))?; let block = chain .block_at(height) .await? - .context("configured validator did not return its finalized tip")?; + .ok_or(PaidClientError::MissingState( + "configured validator did not return its finalized tip", + ))?; Ok(SetupScan { height, payload: block.payload, diff --git a/crates/sdk/src/paid_client/error.rs b/crates/sdk/src/paid_client/error.rs new file mode 100644 index 00000000..a76688be --- /dev/null +++ b/crates/sdk/src/paid_client/error.rs @@ -0,0 +1,90 @@ +use std::path::PathBuf; + +/// Errors retain the admission/payment stage and the original typed cause. +#[derive(Debug, thiserror::Error)] +pub enum PaidClientError { + #[error("invalid paid-work options: {0}")] + InvalidOptions(&'static str), + #[error("paid input does not match {0}")] + InputMismatch(&'static str), + #[error("paid-work state is missing: {0}")] + MissingState(&'static str), + #[error( + "pending Fetch {work_id} lost its in-memory payload; channel remains reserved until payment deadline {payment_deadline}" + )] + MissingPayload { + work_id: hellas_rpc::Digest, + payment_deadline: u64, + }, + #[error("multiple active jobs match this prepared input")] + AmbiguousRecovery, + #[error("payment deadline elapsed during result delivery")] + PaymentExpired, + #[error("{stage} deadline overflow")] + DeadlineOverflow { stage: &'static str }, + #[error("{stage} timed out; journals retain payment state")] + Timeout { stage: &'static str }, + #[error("no validators are configured")] + NoValidators, + #[error("no configured validator answered ({0:?})")] + ValidatorsUnavailable(Vec<(String, hellas_chain::QueryError)>), + #[error("authenticated provider key differs from the payment channel")] + ProviderIdentityChanged, + #[error("provider identity lock poisoned")] + ProviderIdentityPoisoned, + #[error("validator genesis differs from the configured genesis")] + GenesisMismatch { + expected: [u8; 32], + actual: [u8; 32], + }, + #[error("paid-work setup ended: {0:?}")] + SetupEnded(hellas_work::work_open::SetupProgress), + #[error("cannot create journal directory {}: {source}", path.display())] + JournalDirectory { + path: PathBuf, + source: std::io::Error, + }, + #[error("cannot connect to provider {provider}: {source}")] + Connect { + provider: iroh::EndpointId, + source: iroh::endpoint::ConnectError, + }, + #[error(transparent)] + Bind(#[from] iroh::endpoint::BindError), + #[error(transparent)] + Client(#[from] hellas_client::ClientError), + #[error(transparent)] + Canonical(#[from] hellas_rpc::protocol::value::CanonicalDecodeError), + #[error(transparent)] + Work(#[from] hellas_rpc::protocol::work::PaidWorkError), + #[error(transparent)] + WorkSetup(#[from] hellas_rpc::protocol::work_setup::WorkSetupError), + #[error(transparent)] + Store(#[from] hellas_work::work_store::WorkStoreError), + #[error(transparent)] + Endpoint(#[from] hellas_work::work::EndpointError), + #[error(transparent)] + Propose(#[from] hellas_work::work::ProposeError), + #[error(transparent)] + Deliver(#[from] hellas_work::work::DeliverError), + #[error(transparent)] + Payment(#[from] hellas_work::work::PaymentError), + #[error(transparent)] + Collect(#[from] hellas_client::work::CollectResultError), + #[error(transparent)] + Close(#[from] hellas_work::work_close::CloseError), + #[error(transparent)] + CatchUp(#[from] hellas_work::work_close::CatchUpError), + #[error(transparent)] + Setup(#[from] hellas_work::work_handshake::SetupExchangeError), + #[error(transparent)] + SetupDrive(#[from] hellas_work::work_open::SetupDriveError), + #[error(transparent)] + BlockSource(#[from] hellas_work::work_close::BlockSourceError), + #[error(transparent)] + Consensus(#[from] hellas_chain::ConsensusVerificationError), + #[error(transparent)] + Query(#[from] hellas_chain::QueryError), + #[error(transparent)] + Fetch(#[from] hellas_rpc::fetch::FetchProtocolError), +} diff --git a/crates/sdk/src/paid_client/tests.rs b/crates/sdk/src/paid_client/tests.rs index 0f69397f..f28a2794 100644 --- a/crates/sdk/src/paid_client/tests.rs +++ b/crates/sdk/src/paid_client/tests.rs @@ -185,9 +185,9 @@ fn recovery_skips_only_permanent_delivery_refusals() { assert!(!permanently_refused_delivery( &DeliverError::Malformed("result").into() )); - assert!(!permanently_refused_delivery(&anyhow::anyhow!( - "connection lost" - ))); + assert!(!permanently_refused_delivery(&PaidClientError::Timeout { + stage: "connection" + })); } fn fetch_request( @@ -253,7 +253,7 @@ fn fetch_preflight_requires_matching_trust_caller_and_ephemeral_retention() { use hellas_rpc::Retention; let (policy, prepared, caller) = fetch_request(Assurance::AppleAppAttest, Retention::Ephemeral); let error = check_request(&policy, &prepared, None, caller).unwrap_err(); - assert!(error.to_string().contains("trust anchor before disclosure")); + assert!(matches!(error, PaidClientError::InvalidOptions(_))); let mut trust = hellas_client::ProviderTrustAnchor { expected_genesis: hellas_rpc::ContentId::from_bytes([8; 32]), required_assurance: Assurance::ProducerSigned, @@ -275,7 +275,7 @@ fn fetch_preflight_requires_matching_trust_caller_and_ephemeral_retention() { check_request(&policy, &prepared, Some(&trust), other) .unwrap_err() .to_string() - .contains("caller does not match") + .contains("Fetch caller") ); let (policy, prepared, caller) = fetch_request(Assurance::ProducerSigned, Retention::Ephemeral); check_request(&policy, &prepared, None, caller).unwrap(); @@ -285,6 +285,6 @@ fn fetch_preflight_requires_matching_trust_caller_and_ephemeral_retention() { check_request(&policy, &retained, None, caller) .unwrap_err() .to_string() - .contains("ephemeral retention") + .contains("ephemeral") ); } diff --git a/crates/wire/src/mux/state.rs b/crates/wire/src/mux/state.rs index 83f3c6e0..85cc86b9 100644 --- a/crates/wire/src/mux/state.rs +++ b/crates/wire/src/mux/state.rs @@ -514,8 +514,12 @@ impl Multiplexer { if slot.generation != keyed.key.generation { return Ok(events); // stale-generation drop } - if slot.peer_terminal { - return Ok(events); // peer already terminal; drop late frames + // End closes the peer's send half. It can still consume our response + // (returning credit) or cancel it until our own send half ends. + if slot.peer_terminal + && (slot.local_terminal || !matches!(keyed.frame, Frame::Credit(_) | Frame::Reset(_))) + { + return Ok(events); } match keyed.frame { diff --git a/crates/wire/src/mux/state/tests.rs b/crates/wire/src/mux/state/tests.rs index deb433a7..d97c96e1 100644 --- a/crates/wire/src/mux/state/tests.rs +++ b/crates/wire/src/mux/state/tests.rs @@ -168,6 +168,43 @@ fn credit_returns_only_after_application_consumption() { )); } +#[test] +fn request_end_still_allows_response_credit_and_cancellation() { + let cfg = MuxConfig { stream_window: 8 }; + let mut client: Multiplexer<8, _> = Multiplexer::new(Role::Client, DefaultClock, cfg); + let mut server: Multiplexer<8, _> = Multiplexer::new(Role::Server, DefaultClock, cfg); + let slot = client.open(7, Metadata::new()).unwrap(); + drain(&mut client, &mut server); + client.close_send(slot, None).unwrap(); + drain(&mut client, &mut server); + queue_body(&mut server, slot, Bytes::from_static(b"response")); + drain(&mut server, &mut client); + assert!(matches!( + server.try_send_body(slot, Bytes::from_static(b"next")), + Ok(SendBodyOutcome::Blocked(_)) + )); + client.consume(slot, 8).unwrap(); + assert!(matches!( + drain(&mut client, &mut server).as_slice(), + [Event::PeerCredit { .. }] + )); + queue_body(&mut server, slot, Bytes::from_static(b"next")); + drain(&mut server, &mut client); + client.reset(slot, WireCode::Cancelled); + assert!(matches!( + drain(&mut client, &mut server).as_slice(), + [Event::ResetStream { + code: WireCode::Cancelled, + .. + }] + )); + assert!( + server + .try_send_body(slot, Bytes::from_static(b"late")) + .is_err() + ); +} + #[test] fn stale_gen_discarded() { let (mut client, mut server) = pair::<32>(); diff --git a/crates/work/src/work.rs b/crates/work/src/work.rs index 01db3405..b0dd6d9b 100644 --- a/crates/work/src/work.rs +++ b/crates/work/src/work.rs @@ -949,16 +949,27 @@ impl ProviderEndpoint { // ledger charges this client for over an answer it was never // able to take. The client's own check is the other end of the // same bound, against a provider that does not apply this one. - let frame = u64::try_from( - WorkDelivered { - result: result.encode(), - provider_signature: signature.as_bytes().to_vec(), - transcript: spool.clone(), + let delivered = WorkDelivered { + result: result.encode(), + provider_signature: signature.as_bytes().to_vec(), + transcript: spool.clone(), + }; + let frame = if matches!(ready.execution_policy(), PaidWorkPolicy::Fetch { .. }) { + if spool.len() > hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES { + return Err(RunError::Record(PaidWorkError::OverEnvelope { + field: "Fetch transcript", + actual: spool.len() as u64, + limit: hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES as u64, + })); } - .encoded_len(), - ) - .unwrap_or(u64::MAX); - let frame_limit = u64::from(ready.execution_policy().max_encoded_result_frame()); + stream::fetch_frames(&delivered, transcript).try_fold(0, |size, frame| { + Ok::<_, RunError>(size.max(frame?.encoded_len() as u64)) + })? + } else { + delivered.encoded_len() as u64 + }; + let frame_limit = u64::from(ready.execution_policy().max_encoded_result_frame()) + .min(hellas_wire::frame::MAX_FRAME_BYTES as u64); if frame > frame_limit { return Err(RunError::Record(PaidWorkError::OverEnvelope { field: "encoded result frame", @@ -1741,6 +1752,24 @@ pub trait PaidWorkBackend: Sync { { async { Err(BackendFault::new("paid fetch backend is unavailable")) } } + + /// Runs once, exposing signed Fetch prefixes before terminal delivery. + fn fetch_stream( + &self, + input: PreparedFetchInput, + progress: PaidProgress, + ) -> impl core::future::Future, BackendFault>> + Send + { + async move { + let events = self.fetch(input).await?; + for event in &events { + if event.event().body().kind() == hellas_rpc::fetch::OUTPUT_EVENT_KIND { + progress(event.clone())?; + } + } + Ok(events) + } + } /// Runs one journaled Evaluate input to its terminal. fn evaluate( &self, @@ -1907,7 +1936,7 @@ where Arc::new(move |event| progress_service.publish_progress(work_id, event)); let invoked = match admission { RunAdmission::Invoke(input) => backend.evaluate_stream(*input, progress).await, - RunAdmission::InvokeFetch(input) => backend.fetch(*input).await, + RunAdmission::InvokeFetch(input) => backend.fetch_stream(*input, progress).await, RunAdmission::Running => return Ok(RunOutcome::Running), RunAdmission::Indeterminate => return Ok(RunOutcome::Indeterminate), RunAdmission::Ready { result, signature } => { @@ -2193,7 +2222,7 @@ pub struct WorkService { endpoint: Arc>, driving: Arc, changed: Arc, - progress: Arc>>>, + progress: Arc>>, } /// The authority to advance this channel's cursor, and the only thing @@ -3191,6 +3220,16 @@ impl ClientEndpoint { work_id: Digest, ready: &ReadyChannel, delivered: &WorkDelivered, + ) -> Result { + self.receive_inner(work_id, ready, delivered, false) + } + + fn receive_inner( + &mut self, + work_id: Digest, + ready: &ReadyChannel, + delivered: &WorkDelivered, + streamed: bool, ) -> Result { self.state() .job_by_id(work_id) @@ -3208,7 +3247,7 @@ impl ClientEndpoint { // it is the transport's and is not measured here. let limit = u64::from(ready.execution_policy().max_encoded_result_frame()); let actual = u64::try_from(delivered.encoded_len()).unwrap_or(u64::MAX); - if actual > limit { + if !streamed && actual > limit { return Err(DeliverError::OverFrame { actual, limit }); } @@ -3231,7 +3270,7 @@ impl ClientEndpoint { .authorization(); let transcript = hellas_rpc::protocol::work::decode_transcript( &delivered.transcript, - MAX_RECORD_BYTES, + hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES, )?; let input = PreparedPaidWorkInput::decode( self.state() diff --git a/crates/work/src/work/stream.rs b/crates/work/src/work/stream.rs index 58a93efa..618370f5 100644 --- a/crates/work/src/work/stream.rs +++ b/crates/work/src/work/stream.rs @@ -3,6 +3,7 @@ use super::*; use futures::{StreamExt, stream::BoxStream}; use hellas_rpc::pb::work::{WorkStreamEvent, work_stream_event}; use hellas_rpc::protocol::work::decode_transcript; +use hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES; use hellas_rpc::{ Operation, StreamId, output_genesis, scheme_id, verify_output_event_continuation, }; @@ -10,6 +11,45 @@ use hellas_rpc::{ pub type PaidProgress = Arc Result<(), BackendFault> + Send + Sync>; pub type PaidResultStream = BoxStream<'static, Result>; +#[derive(Debug, Default)] +pub(super) struct Progress { + events: Vec, + bytes: usize, +} + +/// Fetch's terminal authenticates the assembled transcript without repeating it +/// in one wire frame. Signatures, result digests and payment records are unchanged. +pub(super) fn fetch_frames<'a>( + delivered: &'a WorkDelivered, + events: &'a [OutputEventEnvelope], +) -> impl Iterator> + 'a { + events.iter().enumerate().map(|(index, event)| { + let transcript = encode_transcript(std::slice::from_ref(event))?; + let outcome = if index + 1 == events.len() { + work_stream_event::Outcome::Terminal(hellas_rpc::pb::work::WorkStreamTerminal { + result: delivered.result.clone(), + provider_signature: delivered.provider_signature.clone(), + terminal_transcript: transcript, + }) + } else { + work_stream_event::Outcome::Prefix(transcript) + }; + Ok(WorkStreamEvent { + outcome: Some(outcome), + }) + }) +} + +fn spool_charge(event: &OutputEventEnvelope, fetch: bool) -> usize { + // The canonical envelope represents payload bytes as CBOR integers, up to + // two encoded bytes each. Include signature/structure overhead before decode. + event + .payload() + .len() + .saturating_mul(if fetch { 2 } else { 1 }) + .saturating_add(1024) +} + impl ProviderEndpoint { fn reserve_stream( &mut self, @@ -52,28 +92,33 @@ impl WorkService { work_id: Digest, event: OutputEventEnvelope, ) -> Result<(), BackendFault> { - let limit = self + let policy = self .endpoint() .map_err(|error| BackendFault::new(error.to_string()))? .admitting() .map_err(|error| BackendFault::new(error.to_string()))? .execution_policy() - .max_spool_bytes(); - let mut progress = self.progress.lock().expect("paid progress poisoned"); - let events = progress.entry(work_id).or_default(); - // Signed token events are bounded by the same spool as final delivery. - let bytes = event.payload().len() + 1024; - let retained = events - .iter() - .map(|event| event.payload().len() + 1024) - .sum::(); - if retained.saturating_add(bytes) as u64 > limit { + .clone(); + let limit = match &policy { + PaidWorkPolicy::Fetch { .. } => policy + .max_spool_bytes() + .min(MAX_FETCH_TRANSCRIPT_BYTES as u64), + PaidWorkPolicy::Evaluate(_) => policy.max_spool_bytes(), + }; + let mut jobs = self.progress.lock().expect("paid progress poisoned"); + let progress = jobs.entry(work_id).or_default(); + // Signed prefixes are bounded by the same spool as final delivery. + let fetch = matches!(policy, PaidWorkPolicy::Fetch { .. }); + let bytes = spool_charge(&event, fetch); + let retained = progress.bytes.saturating_add(bytes); + if retained as u64 > limit { return Err(BackendFault::new( "live result exceeds its authorized spool", )); } - events.push(event); - drop(progress); + progress.bytes = retained; + progress.events.push(event); + drop(jobs); self.changed.notify_waiters(); Ok(()) } @@ -105,7 +150,20 @@ impl WorkService { let response = service.release(&request, &context); match response.outcome { Some(DeliverOutcome::Delivered(delivered)) => { - yield Ok(WorkStreamEvent { outcome: Some(work_stream_event::Outcome::Delivered(delivered)) }); + let events = match decode_transcript(&delivered.transcript, MAX_FETCH_TRANSCRIPT_BYTES) { + Ok(events) => events, + Err(error) => { + yield Err(WireStatus::new(hellas_wire::WireCode::Internal, error.to_string())); + return; + } + }; + if events.last().is_some_and(|event| event.event().body().kind() == hellas_rpc::fetch::OUTPUT_TERMINAL_KIND) { + for frame in fetch_frames(&delivered, &events).skip(position) { + yield frame.map_err(|error| WireStatus::new(hellas_wire::WireCode::Internal, error.to_string())); + } + } else { + yield Ok(WorkStreamEvent { outcome: Some(work_stream_event::Outcome::Delivered(delivered)) }); + } return; } Some(DeliverOutcome::Refused(refused)) if refused.code != WorkRefusal::NotReady.code() as i32 => { @@ -116,7 +174,7 @@ impl WorkService { } let pending = { let progress = service.progress.lock().expect("paid progress poisoned"); - progress.get(&work_id).map(|events| events[position.min(events.len())..].to_vec()).unwrap_or_default() + progress.get(&work_id).map(|progress| progress.events[position.min(progress.events.len())..].to_vec()).unwrap_or_default() }; for event in pending { match encode_transcript(&[event]) { @@ -148,7 +206,7 @@ impl WorkService { } } -/// Deliver verified token prefixes and finally journal the complete result. +/// Deliver verified prefixes and finally journal the complete result. /// The caller pays only after this returns; it must keep running if its UI drops. pub async fn fetch_result_stream( transport: T, @@ -171,27 +229,67 @@ where .state() .job_by_id(work_id) .ok_or(DeliverError::NoSuchJob)?; - let PreparedPaidWorkInput::Evaluate(prepared) = - PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) - .map_err(PaidWorkError::from)? - else { - return Err(DeliverError::Malformed( - "live token delivery requires Evaluate", - )); - }; - let parts = prepared.parts().map_err(PaidWorkError::from)?; - let input = hellas_rpc::evaluate::input_commitment(&parts.evaluate_request); + let prepared = PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) + .map_err(PaidWorkError::from)?; + let input = prepared.input_commitment()?; + let (operation, kind, max_tokens, max_events, max_bytes) = + match (&prepared, ready.execution_policy()) { + (PreparedPaidWorkInput::Evaluate(prepared), PaidWorkPolicy::Evaluate(_)) => ( + Operation::Evaluate, + hellas_rpc::evaluate::TOKEN_DELTA_EVENT_KIND, + u64::from( + prepared + .parts() + .map_err(PaidWorkError::from)? + .text_policy + .max_new_tokens(), + ), + u64::MAX, + u64::MAX, + ), + (PreparedPaidWorkInput::Fetch(_), PaidWorkPolicy::Fetch { policy, .. }) => ( + Operation::Fetch, + hellas_rpc::fetch::OUTPUT_EVENT_KIND, + 0, + u64::from(policy.max_output_events) + .min(hellas_rpc::fetch::MAX_FETCH_OUTPUT_EVENTS as u64), + u64::from(policy.max_output_bytes) + .min(hellas_rpc::fetch::MAX_FETCH_OUTPUT_PAYLOAD_BYTES as u64), + ), + _ => { + return Err(DeliverError::Malformed( + "stream profile differs from channel", + )); + } + }; let mut token_count = 0u64; + let mut payload_bytes = 0u64; let mut retained_bytes = 0usize; + let spool_limit = if operation == Operation::Fetch { + ready + .execution_policy() + .max_spool_bytes() + .min(MAX_FETCH_TRANSCRIPT_BYTES as u64) + } else { + ready.execution_policy().max_spool_bytes() + }; let key = hellas_rpc::PublicKey::Secp256k1(ready.channel().provider_key().to_bytes()); - let scheme = scheme_id(Operation::Evaluate, hellas_rpc::Assurance::ProducerSigned); + let scheme = scheme_id(operation, prepared.assurance()?); let mut previous = output_genesis(input, StreamId::from_input_commitment(input)); let mut streamed = Vec::new(); let mut stream = WorkClientImpl::new(transport) .stream_result(request) .await?; while let Some(event) = stream.next().await { - match event?.outcome { + let event = event?; + let frame_limit = ready.execution_policy().max_encoded_result_frame() as u64; + if operation == Operation::Fetch && event.encoded_len() as u64 > frame_limit { + return Err(DeliverError::OverFrame { + actual: event.encoded_len() as u64, + limit: frame_limit, + }); + } + match event.outcome { Some(work_stream_event::Outcome::Prefix(prefix)) => { let events = decode_transcript(&prefix, MAX_RECORD_BYTES)?; for event in events { @@ -204,17 +302,28 @@ where &event, ) .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; - if event.event().body().kind() != hellas_rpc::evaluate::TOKEN_DELTA_EVENT_KIND { + if event.event().body().kind() != kind { return Err(DeliverError::Malformed( - "live prefix must contain token deltas", + "live prefix has the wrong event kind", )); } - let delta = hellas_rpc::evaluate::decode_token_delta_payload(event.payload()) - .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; - token_count = token_count.saturating_add(delta.token_ids.len() as u64); - retained_bytes = retained_bytes.saturating_add(event.payload().len() + 1024); - if token_count > u64::from(parts.text_policy.max_new_tokens()) - || retained_bytes as u64 > ready.execution_policy().max_spool_bytes() + if operation == Operation::Evaluate { + let delta = + hellas_rpc::evaluate::decode_token_delta_payload(event.payload()) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + token_count = token_count.saturating_add(delta.token_ids.len() as u64); + } else { + hellas_rpc::fetch::decode_fetch_event_payload(event.payload()) + .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; + } + payload_bytes = payload_bytes.saturating_add(event.payload().len() as u64); + retained_bytes = retained_bytes + .saturating_add(spool_charge(&event, operation == Operation::Fetch)); + // Reserve one envelope for the required terminal. + if token_count > max_tokens + || streamed.len() as u64 + 1 >= max_events + || payload_bytes > max_bytes + || retained_bytes as u64 > spool_limit { return Err(DeliverError::Malformed( "live result exceeds authorized output", @@ -237,12 +346,51 @@ where stream.finish()?; let result = endpoint.receive(work_id, ready, &delivered)?; for event in &events[streamed.len()..] { - if event.event().body().kind() == hellas_rpc::evaluate::TOKEN_DELTA_EVENT_KIND { + if event.event().body().kind() == kind { progress(event)?; } } return Ok(result); } + Some(work_stream_event::Outcome::Terminal(terminal)) => { + if operation != Operation::Fetch { + return Err(DeliverError::Malformed("stream terminal requires Fetch")); + } + let mut tail = decode_transcript(&terminal.terminal_transcript, MAX_RECORD_BYTES)?; + if tail.len() != 1 + || tail[0].event().body().kind() != hellas_rpc::fetch::OUTPUT_TERMINAL_KIND + { + return Err(DeliverError::Malformed( + "stream terminal must contain one terminal envelope", + )); + } + if payload_bytes.saturating_add(tail[0].payload().len() as u64) > max_bytes + || retained_bytes.saturating_add(spool_charge(&tail[0], true)) as u64 + > spool_limit + { + return Err(DeliverError::Malformed( + "live result exceeds authorized output", + )); + } + // No delivery or payment is recorded before the stream's final + // status and the complete signed transcript have both verified. + if stream.next().await.transpose()?.is_some() { + return Err(DeliverError::Malformed("event after terminal result")); + } + stream.finish()?; + streamed.append(&mut tail); + let transcript = encode_transcript(&streamed)?; + return endpoint.receive_inner( + work_id, + ready, + &WorkDelivered { + result: terminal.result, + provider_signature: terminal.provider_signature, + transcript, + }, + true, + ); + } Some(work_stream_event::Outcome::Refused(refused)) => { return Err(DeliverError::Refused { refusal: WorkRefusal::from_code(refused.code) diff --git a/crates/work/src/work_open.rs b/crates/work/src/work_open.rs index ad14f739..062d0c70 100644 --- a/crates/work/src/work_open.rs +++ b/crates/work/src/work_open.rs @@ -755,15 +755,15 @@ fn mount( None => descriptor.expected_settlement().map_err(to_store)?, Some(payment) => descriptor.funded_settlement(payment).map_err(to_store)?, }; - let mut channel = if store.role() == crate::work_store::Role::Provider - && matches!( - descriptor.execution_policy(), - hellas_rpc::protocol::work_profile::PaidWorkPolicy::Fetch { .. } - ) { + let mut channel = if matches!( + descriptor.execution_policy(), + hellas_rpc::protocol::work_profile::PaidWorkPolicy::Fetch { .. } + ) { ChannelStore::open_metadata_only( store.root(), descriptor.channel().clone(), settlement, + store.role(), origin, verifier, )? diff --git a/crates/work/src/work_store/channel/state.rs b/crates/work/src/work_store/channel/state.rs index d24b552e..b7577791 100644 --- a/crates/work/src/work_store/channel/state.rs +++ b/crates/work/src/work_store/channel/state.rs @@ -20,9 +20,15 @@ impl ChannelState { Err(ChannelStateError::Malformed) }; } - let events = decode_transcript(transcript, MAX_RECORD_BYTES)?; let input = PreparedPaidWorkInput::decode(&job.prepared_input, MAX_RECORD_BYTES) .map_err(PaidWorkError::from)?; + let budget = match &input { + PreparedPaidWorkInput::Fetch(_) => { + hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES + } + PreparedPaidWorkInput::Evaluate(_) => MAX_RECORD_BYTES, + }; + let events = decode_transcript(transcript, budget)?; if input.terminal_result(&self.channel, &job.authorization, &events)? != *result { return Err(ChannelStateError::WrongChannel { field: "result against its transcript", diff --git a/crates/work/src/work_store/channel/store.rs b/crates/work/src/work_store/channel/store.rs index 26176e97..a978ea2f 100644 --- a/crates/work/src/work_store/channel/store.rs +++ b/crates/work/src/work_store/channel/store.rs @@ -50,7 +50,7 @@ impl ChannelStore { Self::open_inner(root, channel, settlement, role, origin, verifier, false) } - /// Opens a provider journal that stores accounting evidence only. + /// Opens a journal that stores accounting evidence only. /// /// Inputs and output transcripts are verified before commit and retained /// in memory. Append and checkpoint encodings omit both bodies. After a @@ -61,18 +61,11 @@ impl ChannelStore { root: &Path, channel: PaidChannel, settlement: WorkPaymentSettlement, + role: Role, origin: SetupOrigin, verifier: &V, ) -> Result { - Self::open_inner( - root, - channel, - settlement, - Role::Provider, - origin, - verifier, - true, - ) + Self::open_inner(root, channel, settlement, role, origin, verifier, true) } fn open_inner( diff --git a/crates/work/tests/work_run/fetch.rs b/crates/work/tests/work_run/fetch.rs index 36a29ea0..0d1b573f 100644 --- a/crates/work/tests/work_run/fetch.rs +++ b/crates/work/tests/work_run/fetch.rs @@ -18,6 +18,98 @@ use hellas_work::work::{ const PROMPT: &[u8] = br#"{"input":"private-customer-prompt-never-on-provider-disk"}"#; const RESPONSE: &str = "private-provider-response-never-on-provider-disk"; +#[path = "../support/transport.rs"] +mod transport; + +#[tokio::test] +async fn fetch_streams_before_terminal_then_pays_once_with_metadata_only_journals() { + use hellas_rpc::services::work::{WorkClientImpl, WorkServer}; + use hellas_wire::{Dispatcher, StreamTransport, mux::MuxTransport}; + use hellas_work::work::{PaidProgress, admit_payment, fetch_result_stream}; + + struct PausedFetch(Arc); + impl PaidWorkBackend for PausedFetch { + async fn fetch_stream( + &self, + input: PreparedFetchInput, + progress: PaidProgress, + ) -> Result, BackendFault> { + let events = FetchBackend::default().fetch(input).await?; + progress(events[0].clone())?; + self.0.notified().await; + Ok(events) + } + } + for assurance in [Assurance::ProducerSigned, Assurance::AppleAppAttest] { + let provider_dir = temp(); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let service = service(provider_dir.path()); + let proposal = JobProposal { + prepared_input: input_with_assurance(Retention::Ephemeral, assurance).into(), + deadlines: deadlines(), + }; + let request = client.propose(&proposal).unwrap(); + let id = client.accepted(&service.accept(&request)).unwrap(); + let release = Arc::new(tokio::sync::Notify::new()); + let worker = { + let service = service.clone(); + let backend = PausedFetch(release.clone()); + tokio::spawn( + async move { run_accepted_work(&service, &fetch_ready(), &backend, id).await }, + ) + }; + let (transport, server) = transport::transport_pair(); + let serving = { + let service = service.clone(); + tokio::spawn(async move { + let handler = WorkServer(service); + while let Ok(Some(inbound)) = server.accept().await { + let _ = Dispatcher::::dispatch(&handler, inbound).await; + } + }) + }; + let mut prefixes = 0; + tokio::time::timeout( + std::time::Duration::from_secs(5), + fetch_result_stream( + transport.clone(), + &mut client, + &fetch_ready(), + id, + |event| { + assert!(!worker.is_finished()); + assert_eq!( + service + .with_state(|s| s.job_by_id(id).unwrap().phase()) + .unwrap(), + JobPhase::Streaming + ); + assert!(hellas_rpc::fetch::decode_fetch_event_payload(event.payload()).is_ok()); + prefixes += 1; + release.notify_one(); + Ok(()) + }, + ), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(prefixes, 1); + assert!(matches!( + worker.await.unwrap().unwrap(), + RunOutcome::Completed { .. } + )); + let rpc = WorkClientImpl::new(transport); + assert_eq!(admit_payment(&rpc, &mut client, id).await.unwrap(), PRICE); + assert_eq!(admit_payment(&rpc, &mut client, id).await.unwrap(), PRICE); + assert_no_bodies(client_dir.path()); + assert_no_bodies(provider_dir.path()); + serving.abort(); + let _ = serving.await; + } +} + fn policy() -> PaidWorkPolicy { let route = FetchRoutePolicy::sealed_route("openai", "responses").unwrap(); PaidWorkPolicy::Fetch { @@ -43,6 +135,41 @@ fn fetch_ready() -> ReadyChannel { ready_of(descriptor_with(policy()), CURSOR) } +#[tokio::test] +async fn invalid_paid_authorizations_never_invoke_fetch() { + for invalid in ["signature", "price", "deadline"] { + let provider_dir = temp(); + let client_dir = temp(); + let service = service(provider_dir.path()); + let mut client = client_endpoint(client_dir.path()); + let mut request = client.propose(&proposal()).unwrap(); + let mut authorization = PaidJobAuthorizationV1::decode(&request.authorization).unwrap(); + match invalid { + "price" => authorization.price += 1, + "deadline" => authorization.acceptance_deadline = CURSOR - 1, + _ => {} + } + let id = work_id(fetch_ready().channel(), &authorization); + request.authorization = authorization.encode(); + request.client_signature = super::client().sign(signing_hash(id)).as_bytes().to_vec(); + if invalid == "signature" { + request.client_signature[0] ^= 1; + } + assert!(matches!( + service.accept(&request).outcome, + Some(hellas_rpc::pb::work::accept_work_response::Outcome::Refused(_)) + )); + let backend = FetchBackend::default(); + assert!(matches!( + run_accepted_work(&service, &fetch_ready(), &backend, id).await, + Err(RunError::NoSuchJob) + )); + assert_eq!(backend.0.load(Ordering::SeqCst), 0); + assert_no_bodies(provider_dir.path()); + assert_no_bodies(client_dir.path()); + } +} + fn input(retention: Retention) -> PreparedPaidFetchInputV1 { input_with_assurance(retention, Assurance::ProducerSigned) } @@ -69,10 +196,15 @@ fn proposal() -> JobProposal { } fn metadata_store(root: &std::path::Path) -> ChannelStore { + store_for(root, &fetch_ready(), Role::Provider) +} + +fn store_for(root: &std::path::Path, ready: &ReadyChannel, role: Role) -> ChannelStore { let mut store = ChannelStore::open_metadata_only( root, - fetch_ready().channel().clone(), + ready.channel().clone(), settlement(), + role, origin(), &Secp256k1Verifier::new(), ) @@ -88,17 +220,193 @@ fn service(root: &std::path::Path) -> WorkService { } fn client_endpoint(root: &std::path::Path) -> ClientEndpoint { - let mut store = ChannelStore::open( - root, - fetch_ready().channel().clone(), - settlement(), - Role::Client, - origin(), - &Secp256k1Verifier::new(), + let store = store_for(root, &fetch_ready(), Role::Client); + ClientEndpoint::new(fetch_ready(), store, client()).unwrap() +} + +#[tokio::test] +async fn paid_fetch_larger_than_a_wire_frame_streams_and_pays_without_disk_payloads() { + use hellas_rpc::services::work::{WorkClientImpl, WorkServer}; + use hellas_wire::{Dispatcher, StreamTransport, mux::MuxTransport}; + use hellas_work::work::{admit_payment, fetch_result_stream}; + struct LargeFetch; + impl PaidWorkBackend for LargeFetch { + async fn fetch( + &self, + input: PreparedFetchInput, + ) -> Result, BackendFault> { + let request = verify_input_events(&input.into_parts().fetch_input_transcript).unwrap(); + let key = provider_producer(); + let mut transcript = FetchOutputTranscriptBuilder::new( + request.input_commitment, + request.assurance, + &key, + ); + for index in 0..256 { + transcript + .push_event( + encode_fetch_event_payload(&OutputEvent::TextDelta { + index, + delta: RESPONSE.repeat(400), + channel: TextChannel::Output, + }) + .unwrap(), + ) + .unwrap(); + } + Ok(transcript + .finish( + encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .unwrap(), + ) + .unwrap()) + } + } + let mut profile = policy(); + let PaidWorkPolicy::Fetch { policy, .. } = &mut profile else { + unreachable!() + }; + policy.max_output_events = 512; + policy.max_output_bytes = 8 << 20; + policy.max_spool_bytes = 16 << 20; + policy.max_encoded_result_frame = 64 << 10; + let ready = ready_of(descriptor_with(profile), CURSOR); + let client_dir = temp(); + let provider_dir = temp(); + let mut client = ClientEndpoint::new( + ready.clone(), + store_for(client_dir.path(), &ready, Role::Client), + super::client(), ) .unwrap(); - advance(&mut store, CURSOR); - ClientEndpoint::new(fetch_ready(), store, client()).unwrap() + let service = WorkService::new( + ProviderEndpoint::new( + ready.clone(), + store_for(provider_dir.path(), &ready, Role::Provider), + provider(), + ) + .unwrap(), + ); + let request = client.propose(&proposal()).unwrap(); + let id = client.accepted(&service.accept(&request)).unwrap(); + run_accepted_work(&service, &ready, &LargeFetch, id) + .await + .unwrap(); + let (transport, server) = transport::transport_pair(); + let serving = tokio::spawn(async move { + let handler = WorkServer(service); + while let Ok(Some(inbound)) = server.accept().await { + let _ = Dispatcher::::dispatch(&handler, inbound).await; + } + }); + let mut received = 0; + let delivered = tokio::time::timeout( + std::time::Duration::from_secs(45), + fetch_result_stream(transport.clone(), &mut client, &ready, id, |_| { + received += 1; + Ok(()) + }), + ) + .await + .unwrap_or_else(|error| { + panic!( + "{error}: received {received} prefixes, server finished={}", + serving.is_finished() + ) + }) + .unwrap(); + assert_eq!(received, 256); + assert!(delivered.transcript.len() > hellas_wire::frame::MAX_FRAME_BYTES); + assert_eq!( + admit_payment(&WorkClientImpl::new(transport), &mut client, id) + .await + .unwrap(), + PRICE + ); + assert_no_bodies(client_dir.path()); + assert_no_bodies(provider_dir.path()); + serving.abort(); + let _ = serving.await; +} + +#[tokio::test] +async fn streamed_fetch_cannot_pay_without_a_complete_authenticated_terminal() { + use hellas_rpc::pb::work::{WorkStreamEvent, WorkStreamTerminal, work_stream_event::Outcome}; + use hellas_rpc::protocol::work::{decode_transcript, encode_transcript}; + use hellas_rpc::services::work::StreamResult; + use hellas_wire::{StreamTransport, WireCode, WireStatus, mux::MuxTransport}; + use hellas_work::work::fetch_result_stream; + for fault in ["missing-prefix", "signature", "duplicate", "status", "none"] { + let provider_dir = temp(); + let client_dir = temp(); + let mut client = client_endpoint(client_dir.path()); + let service = service(provider_dir.path()); + let request = client.propose(&proposal()).unwrap(); + let id = client.accepted(&service.accept(&request)).unwrap(); + run_accepted_work(&service, &fetch_ready(), &FetchBackend::default(), id) + .await + .unwrap(); + let exporter = [0x5e; 32]; + let delivered = service + .deliver(&client.request_delivery(id, &exporter).unwrap(), &exporter) + .unwrap(); + let events = decode_transcript(&delivered.transcript, 65536).unwrap(); + let mut terminal = WorkStreamTerminal { + result: delivered.result.encode(), + provider_signature: delivered.signature.as_bytes().to_vec(), + terminal_transcript: encode_transcript(&events[1..]).unwrap(), + }; + if fault == "signature" { + terminal.provider_signature[0] ^= 1; + } + let end = WorkStreamEvent { + outcome: Some(Outcome::Terminal(terminal)), + }; + let mut frames = Vec::new(); + if fault != "missing-prefix" { + frames.push(Ok(WorkStreamEvent { + outcome: Some(Outcome::Prefix(encode_transcript(&events[..1]).unwrap())), + })); + } + frames.push(Ok(end.clone())); + if fault == "duplicate" { + frames.push(Ok(end)); + } + if fault == "status" { + frames.push(Err(WireStatus::new(WireCode::Unavailable, "interrupted"))); + } + let (transport, server) = transport::transport_pair(); + let serving = tokio::spawn(async move { + let inbound = server.accept().await.unwrap().unwrap(); + hellas_rpc::call::dispatch_server_streaming::( + inbound, + move |_| async move { Ok(futures::stream::iter(frames)) }, + ) + .await + .unwrap(); + }); + let result = + fetch_result_stream(transport, &mut client, &fetch_ready(), id, |_| Ok(())).await; + serving.await.unwrap(); + if fault == "none" { + result.unwrap(); + assert!(client.pay(id).is_ok()); + } else { + assert!(result.is_err(), "{fault}"); + assert!( + client.pay(id).is_err(), + "{fault} must not create payable evidence" + ); + assert_eq!( + client.state().job_by_id(id).unwrap().phase(), + JobPhase::Accepted + ); + } + assert_no_bodies(client_dir.path()); + } } #[derive(Default)] @@ -188,7 +496,12 @@ async fn fetch_is_run_delivered_and_paid_without_provider_disk_bodies() { ) .unwrap(); assert_no_bodies(provider_dir.path()); + assert_no_bodies(client_dir.path()); let payment = client.pay(id).unwrap(); + drop(client); + let mut client = client_endpoint(client_dir.path()); + assert_eq!(client.pay(id).unwrap(), payment); + assert_no_bodies(client_dir.path()); // A delivery already happened: restart loses bodies but can still bank the signed payment. drop(service); diff --git a/crates/work/tests/work_store_channel.rs b/crates/work/tests/work_store_channel.rs index 45a1834c..8dbe9247 100644 --- a/crates/work/tests/work_store_channel.rs +++ b/crates/work/tests/work_store_channel.rs @@ -448,6 +448,7 @@ fn open_metadata(root: &std::path::Path) -> ChannelStore { root, channel(), settlement(), + Role::Provider, origin_of(&channel()), &Secp256k1Verifier::new(), ) @@ -580,6 +581,7 @@ fn journal_retention_mode_cannot_change_when_reopened() { root.path(), channel(), settlement(), + Role::Provider, origin_of(&channel()), &Secp256k1Verifier::new(), ) diff --git a/docs/http-gateway.md b/docs/http-gateway.md index dfa3357e..09e66e92 100644 --- a/docs/http-gateway.md +++ b/docs/http-gateway.md @@ -1,10 +1,10 @@ # HTTP APIs through Fetch -`gateway --http-fetch-config FILE` exposes exact HTTP routes over the generic -HTTPS Fetch environment. Request and response bodies keep the upstream format, -including SSE, tool calls and non-2xx errors. It requires a provider trust anchor -and a provider route with a caller grant; it does not use the token-native paid -pool. See [provider account and egress configuration](../crates/providers/HTTPS.md). +`gateway --http-fetch-config FILE --paid-work-config POOL` exposes exact HTTP +routes over paid HTTPS Fetch. Request and response bodies keep the upstream +format, including SSE, tool calls and non-2xx errors. Every request uses a funded +channel from the [paid pool](paid-gateway.md); missing payment configuration is a +startup error. See [provider account and egress configuration](../crates/providers/HTTPS.md). The gateway's standard API paths can serve several accounts or provider nodes. It reads the requested model, finds backends configured to serve that model and @@ -48,12 +48,27 @@ optional backend `max_in_flight` overrides the global per-account limit. Routes sharing a credential alias on the same provider share capacity and cooldown; duplicating that alias does not multiply its allowance. -Backends use the CLI's provider target by default. An optional backend `provider` -selects another node with `node_id`, `node_addrs` (an array of `IP:port` strings) -and `genesis` (the hex enrollment ContentId). The gateway's assurance and Apple -trust policy still apply; each node is verified against its configured enrollment -pin. Each provider must authorize this gateway's caller key and expose the -configured Fetch service/method. +A backend's optional `provider` is an endpoint ID from the paid pool. It may be +omitted only when the pool contains exactly one provider for the HTTP Fetch +manifest. Addresses, funding, journals and enrollment/Apple trust pins belong in +the pool. The provider must mount a paid Fetch channel for the gateway's transport +and settlement identities and expose the configured service/method. + +The gateway bearer authorizes access to the gateway's payment identity. The +router selects an account, then the paid client signs the exact Fetch request +and proposes it on that provider's channel. The provider checks the funded edge, +credit, deadlines, signature and Fetch policy before durably accepting work; +only that accepted job can invoke the upstream. The credential's origin, path +and method restrictions remain enforced by the provider. Courtesy caller grants +do not authorize this HTTP path. + +Signed response prefixes stream as they arrive. Successful HTTP completion +follows verification of the complete result and the provider's durable payment +acknowledgement. Each accepted valid terminal costs the channel's agreed fixed +price, including HTTP error statuses. Requests are not retried on another +provider after account selection. The pool serializes work within each provider +channel; concurrent requests assigned to that provider queue within its deadline, including requests for different +accounts on that provider. Separate providers have independent channels. Paths and methods match exactly; query parameters retain their order, repeats and percent encoding. Ordinary client headers, including idempotency keys and @@ -75,8 +90,7 @@ provider. Private destinations also require its explicit egress opt-in. ```sh hellas-cli --identity ./gateway.identity gateway \ --http-fetch-config ./http-gateway.json \ - --provider "$PROVIDER_ENROLLMENT" \ - --node-id "$PROVIDER_NODE_ID" --node-addr "$PROVIDER_ADDRESS" \ + --paid-work-config /srv/hellas/paid-pool.json \ --port 8080 --bearer-token-file ./gateway.bearer ``` @@ -130,6 +144,14 @@ an eligible backend; account-specific quota endpoints are not aggregated. ## Archives and ZDR +Fetch payment journals on both endpoints retain hashes, signatures and accounting +records, with payloads held in memory. After a restart they can re-send an existing +payment certificate, but cannot reconstruct or replay a lost request/response. +Unpaid work with a lost payload keeps its channel reserved until its payment +deadline; it is never re-executed from a new request. Existing payload-bearing +Fetch journals are refused instead of silently changing their retention policy. +The HTTP archive below is the gateway's separate payload persistence policy. + CLI gateways archive authenticated requests and responses by default under `~/.hellas/gateway-archive`, or `--archive-dir DIRECTORY`. This applies to the existing inference routes as well as HTTP Fetch. Each exchange has owner-only @@ -171,12 +193,21 @@ its host-managed storage policy. HTTP input is limited to 1,523,712 bytes, reserving envelope space inside the 2 MiB signed Fetch request. Responses are limited to 8 MiB and the Fetch stream -to 32,768 events and 16 MiB of signed payload. The provider has a 20 minute total -HTTP deadline and a 90 second response idle timeout. Exceeding a stream limit -closes it as incomplete. A slow response consumer applies bounded backpressure, -with a 90 second drain timeout. QUIC cancellation stops an idle upstream without -waiting for its next byte. Individual upstream deliveries are forwarded promptly, -split at 16 KiB; the provider does not wait for a full buffer. +to 32,768 events and 16 MiB of signed payload. The paid policy can impose tighter +limits; its transcript spool is capped at 32 MiB. Configure the spool and output +budgets for the encoded HTTP response (including base64 and signed envelopes). +Each wire frame remains bounded independently: Fetch completion authenticates +all preceding prefixes without packing the full response back into one frame. + +The provider has a 20 minute total HTTP deadline and a 90 second response idle +timeout. The pool's `timeout_secs` and on-chain block deadlines must also fit the +expected execution time. Exceeding a stream limit closes it as incomplete. +Individual upstream deliveries are forwarded promptly, split at 16 KiB. +Before proposal, a disconnect cancels the request. After proposal, collection and +payment continue independently of the HTTP reader. The bounded HTTP output queue +reports an error to a stalled reader without stopping settlement; graceful +shutdown drains accepted operations. Unfinished operations retain their accounting +evidence for recovery, subject to Fetch's in-memory payload lifetime. No generation is automatically retried upstream. Upstream status and end-to-end response headers reach the client, including Location, ETag, `Retry-After`, diff --git a/docs/paid-gateway.md b/docs/paid-gateway.md index 2c230f02..de451ae6 100644 --- a/docs/paid-gateway.md +++ b/docs/paid-gateway.md @@ -1,4 +1,4 @@ -# Paid inference gateway +# Paid gateway `hellas-cli gateway --paid-work-config /srv/hellas/pool.json` sends token-native inference to the configured providers and pays through their real funded work @@ -6,6 +6,11 @@ channels. Each provider has its own endpoint identity, bond, payment funding, and client journals. The gateway prefers idle matching providers and matching prompt prefixes, then serializes requests within each channel. +With `--http-fetch-config`, the same pool carries HTTP Fetch requests. Its work +config must select the HTTP Fetch manifest and a matching Fetch route policy; +see [HTTP routing](http-gateway.md). The HTTP router chooses the provider and +account, so this path never falls back to a different provider or Courtesy. + The pool file uses the provider's existing `--work-config` policy and chain configuration: @@ -30,6 +35,12 @@ configuration: } ``` +For `--assurance apple-app-attest`, each pool entry also needs +`provider_genesis` (hex enrollment ContentId), `apple_app_id`, and +`apple_cd_hashes` (an array of hex 32-byte hashes). Setup and work connections +verify this anchor before disclosing requests. Producer-signed entries may also +pin `provider_genesis`; their funded bond fixes the settlement identity in all cases. + Provision provider bonds with `hellas-cli provision`; coin values and policy terms must agree with the provider work config. Payment coins must be owned by the gateway settlement identity and cannot fund two channels. @@ -61,13 +72,15 @@ providers or validators. Keep gateway and provider identities and journals across service restarts, including when the operating system's store is ephemeral. Startup recovers -unfinished jobs and re-sends retained payments idempotently. Disconnecting an +retained Evaluate jobs and re-sends payment certificates idempotently. Fetch +journals omit bodies: a lost, unpaid Fetch payload cannot be recovered and keeps +the channel reserved until its payment deadline. It is never submitted anew. Disconnecting an HTTP client cancels work that has not yet been proposed. Once a signed proposal may have reached a provider, collection and payment continue despite disconnects. The pool admits at most `max_pending_requests` queued or running requests (default 64); additional requests receive HTTP 503 and may be retried. `timeout_secs` bounds queueing, recovery, provider fallback, execution and payment together, -rather than restarting for each provider. Interactive requests skip busy provider +rather than restarting for each provider. Token-native requests skip busy provider channels; each connection attempt gets at most 10 seconds before trying another route within that shared budget. The HTTP paid route uses this same configured budget; non-paid routes retain their existing 3600-second default. HTTP delivery has an diff --git a/proto/hellas/work/v1/work.proto b/proto/hellas/work/v1/work.proto index 73edfd97..c922dd8e 100644 --- a/proto/hellas/work/v1/work.proto +++ b/proto/hellas/work/v1/work.proto @@ -28,7 +28,8 @@ service Work { // This unary form waits for the durable terminal result. The job's // price is reserved as delivery credit before release; replay returns // the same bytes without a second debit. StreamResult offers signed - // token prefixes under the same authorization and delivery credit. + // prefixes under the same authorization and delivery credit. Use StreamResult + // for Fetch responses that do not fit in one wire frame. rpc DeliverResult(DeliverResultRequest) returns (DeliverResultResponse); // Authenticated live output. Delivery credit is durably reserved before @@ -257,5 +258,14 @@ message WorkStreamEvent { bytes prefix = 1; // canonical transcript containing signed output events WorkDelivered delivered = 2; WorkRefused refused = 3; + // Fetch completion after all nonterminal envelopes in prefix frames. + // The receiver verifies the assembled transcript against this result. + WorkStreamTerminal terminal = 4; } } + +message WorkStreamTerminal { + bytes result = 1; + bytes provider_signature = 2; + bytes terminal_transcript = 3; // exactly one signed terminal envelope +} From 5c3c9da5cbf3d418527234246cec15f6999a8271 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Fri, 25 Sep 2026 01:17:34 +0200 Subject: [PATCH 13/32] Validate paid funding early and batch ready Fetch prefixes --- crates/executor/src/executor/actor/mod.rs | 3 +- .../executor/src/executor/actor/paid_fetch.rs | 7 +- crates/executor/src/executor/mod.rs | 1 + crates/executor/src/work.rs | 2 + crates/sdk/src/paid_client.rs | 10 ++ crates/sdk/src/paid_client/tests.rs | 67 +++++++++ crates/sdk/src/paid_provider.rs | 6 + crates/work/src/work.rs | 7 +- crates/work/src/work/stream.rs | 136 +++++++++++++++--- docs/http-gateway.md | 5 +- docs/paid-gateway.md | 3 + 11 files changed, 226 insertions(+), 21 deletions(-) diff --git a/crates/executor/src/executor/actor/mod.rs b/crates/executor/src/executor/actor/mod.rs index 8c743449..9b0f8d53 100644 --- a/crates/executor/src/executor/actor/mod.rs +++ b/crates/executor/src/executor/actor/mod.rs @@ -491,11 +491,12 @@ impl Executor { async fn handle_owed_request(&mut self, request: ExecutorOwedRequest) { match request { ExecutorOwedRequest::RunPaidFetch { + span, input, progress, reply, } => { - self.start_paid_fetch(*input, progress, reply); + self.start_paid_fetch(*input, progress, reply, span); } ExecutorOwedRequest::RunPaidEvaluate { input, reply, span } => { #[cfg(feature = "evaluate")] diff --git a/crates/executor/src/executor/actor/paid_fetch.rs b/crates/executor/src/executor/actor/paid_fetch.rs index 4d0af6d1..57279aed 100644 --- a/crates/executor/src/executor/actor/paid_fetch.rs +++ b/crates/executor/src/executor/actor/paid_fetch.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use hellas_rpc::OutputEventEnvelope; use hellas_work::work::PreparedFetchInput; use tokio::sync::{mpsc, oneshot}; +use tracing::Instrument as _; use super::Executor; use crate::ExecutorError; @@ -19,6 +20,7 @@ impl Executor { input: PreparedFetchInput, progress: Option, reply: oneshot::Sender, ExecutorError>>, + span: tracing::Span, ) { let prepared = self.prepare_paid_fetch(input); let (entry, session, request, policy) = match prepared { @@ -31,7 +33,7 @@ impl Executor { self.active_fetches += 1; let completion = self.completion_tx.clone(); let key = Arc::clone(&self.provider.producer_key); - tokio::spawn(async move { + let task = async move { let (sender, mut receiver) = mpsc::channel(64); let run = super::execution::run_fetch_provider( entry.provider, @@ -85,7 +87,8 @@ impl Executor { let _ = completion .send(ExecutorCompletion::PaidFetch { reply, result }) .await; - }); + }; + tokio::spawn(task.instrument(span)); } fn prepare_paid_fetch( diff --git a/crates/executor/src/executor/mod.rs b/crates/executor/src/executor/mod.rs index 8f9a70b0..c415858f 100644 --- a/crates/executor/src/executor/mod.rs +++ b/crates/executor/src/executor/mod.rs @@ -108,6 +108,7 @@ pub(crate) enum ExecutorRequest { /// admission past queued best-effort execution. pub(crate) enum ExecutorOwedRequest { RunPaidFetch { + span: tracing::Span, input: Box, progress: Option, reply: oneshot::Sender, ExecutorError>>, diff --git a/crates/executor/src/work.rs b/crates/executor/src/work.rs index 896f8e7c..9158027e 100644 --- a/crates/executor/src/work.rs +++ b/crates/executor/src/work.rs @@ -175,6 +175,7 @@ impl PaidWorkBackend for ExecutorHandle { input: hellas_work::work::PreparedFetchInput, ) -> Result, BackendFault> { self.send_owed(|reply| ExecutorOwedRequest::RunPaidFetch { + span: hellas_rpc::request_span!(target: "hellas_request", "paid.executor.fetch"), input: Box::new(input), progress: None, reply, @@ -189,6 +190,7 @@ impl PaidWorkBackend for ExecutorHandle { progress: PaidProgress, ) -> Result, BackendFault> { self.send_owed(|reply| ExecutorOwedRequest::RunPaidFetch { + span: hellas_rpc::request_span!(target: "hellas_request", "paid.executor.fetch"), input: Box::new(input), progress: Some(progress), reply, diff --git a/crates/sdk/src/paid_client.rs b/crates/sdk/src/paid_client.rs index 3cd97686..df6a98cd 100644 --- a/crates/sdk/src/paid_client.rs +++ b/crates/sdk/src/paid_client.rs @@ -208,6 +208,16 @@ impl PaidWorkSession { } let config = &args.config; + // Reject impossible funding before signing either setup transaction. + let settlement = hellas_kernel::work_payment_settlement( + config.expected_payment_values, + args.omission_bond, + ) + .ok_or(hellas_rpc::protocol::work_setup::WorkSetupError::Unsettleable)?; + hellas_rpc::protocol::work_setup::check_collateral( + args.omission_bond, + settlement.capacity(), + )?; let policy = config.provider_policy(); let bond = args.bond; let payment_funding = args.payment_funding.clone(); diff --git a/crates/sdk/src/paid_client/tests.rs b/crates/sdk/src/paid_client/tests.rs index f28a2794..4ca11bcc 100644 --- a/crates/sdk/src/paid_client/tests.rs +++ b/crates/sdk/src/paid_client/tests.rs @@ -7,6 +7,73 @@ use hellas_rpc::{ }; use hellas_wire::{MethodMarker, StreamTransport, WireStatus}; +#[tokio::test] +async fn insufficient_collateral_is_rejected_before_network_or_journal_creation() { + let (policy, _, _) = fetch_request(Assurance::ProducerSigned, hellas_rpc::Retention::Ephemeral); + let root = tempfile::tempdir().unwrap(); + let journal_root = root.path().join("journal"); + let endpoint = Endpoint::builder(presets::Minimal) + .bind_addr("127.0.0.1:0".parse::().unwrap()) + .unwrap() + .bind() + .await + .unwrap(); + let args = PaidWorkOptions { + config: WorkConfig { + chain: crate::work_config::ChainCrossCheck { + network: policy.network, + genesis_payload_digest: [0; 32].into(), + threshold_identity: Vec::new(), + }, + // No chain is needed to reject these terms. + validators: Vec::new(), + journal_root: root.path().join("provider"), + routes: Default::default(), + policy_salt: policy.policy_salt, + channel_policy: policy.channel_policy, + execution_policy: policy.execution_policy, + poll: Duration::from_millis(200), + expected_payment_values: hellas_kernel::EdgeValues::new( + 1000, + 0, + hellas_kernel::Fees::ZERO, + ), + min_omit_response_blocks: policy.min_omit_response_blocks, + }, + journal_root: journal_root.clone(), + provider: endpoint.id(), + provider_addrs: Vec::new(), + provider_trust: None, + bond: EdgeId::from_bytes([0; 32]), + payment_funding: Funding::new( + hellas_kernel::List::empty(hellas_kernel::CoinId::from_bytes([0; 32])), + hellas_kernel::List::empty(hellas_kernel::CoinId::from_bytes([0; 32])), + ), + omission_bond: 51, + acceptance_blocks: 300, + terminal_blocks: 3000, + payment_blocks: 600, + timeout: Duration::from_secs(30), + }; + let result = PaidWorkSession::open( + args, + endpoint.clone(), + Secp256k1Signer::from_secret_scalar([1; 32]).unwrap(), + ) + .await; + assert!(matches!( + result, + Err(PaidClientError::WorkSetup( + hellas_rpc::protocol::work_setup::WorkSetupError::Undercollateralised { + bond: 51, + capacity: 949, + } + )) + )); + assert!(!journal_root.exists()); + endpoint.close().await; +} + fn enrollment(peer: EndpointId) -> (ProviderEnrollmentBundle, ProducerSigningKey) { let root = ProducerSigningKey::from_secret_bytes([1; 32]).unwrap(); let producer = ProducerSigningKey::from_secret_bytes([2; 32]).unwrap(); diff --git a/crates/sdk/src/paid_provider.rs b/crates/sdk/src/paid_provider.rs index 2e3f782b..1d2cc1c2 100644 --- a/crates/sdk/src/paid_provider.rs +++ b/crates/sdk/src/paid_provider.rs @@ -300,6 +300,12 @@ where .catch_up(source) .await .context("the mounted channel could not catch up to the fresh snapshot")?; + let cursor = service + .with_state(|state| state.cursor().0) + .context("the mounted channel cursor is unavailable")?; + if ready.check_caught_up(cursor).is_ok() { + break; + } } tokio::time::sleep(Duration::from_millis(250)).await; } diff --git a/crates/work/src/work.rs b/crates/work/src/work.rs index b0dd6d9b..9db3547f 100644 --- a/crates/work/src/work.rs +++ b/crates/work/src/work.rs @@ -962,7 +962,12 @@ impl ProviderEndpoint { limit: hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES as u64, })); } - stream::fetch_frames(&delivered, transcript).try_fold(0, |size, frame| { + stream::fetch_frames( + &delivered, + transcript, + ready.execution_policy().max_encoded_result_frame(), + ) + .try_fold(0, |size, frame| { Ok::<_, RunError>(size.max(frame?.encoded_len() as u64)) })? } else { diff --git a/crates/work/src/work/stream.rs b/crates/work/src/work/stream.rs index 618370f5..89012a96 100644 --- a/crates/work/src/work/stream.rs +++ b/crates/work/src/work/stream.rs @@ -22,21 +22,60 @@ pub(super) struct Progress { pub(super) fn fetch_frames<'a>( delivered: &'a WorkDelivered, events: &'a [OutputEventEnvelope], + frame_limit: u32, ) -> impl Iterator> + 'a { - events.iter().enumerate().map(|(index, event)| { - let transcript = encode_transcript(std::slice::from_ref(event))?; - let outcome = if index + 1 == events.len() { - work_stream_event::Outcome::Terminal(hellas_rpc::pb::work::WorkStreamTerminal { - result: delivered.result.clone(), - provider_signature: delivered.provider_signature.clone(), - terminal_transcript: transcript, + let prefixes = &events[..events.len().saturating_sub(1)]; + prefix_batches(prefixes, frame_limit, true) + .map(|batch| { + Ok(WorkStreamEvent { + outcome: Some(work_stream_event::Outcome::Prefix(encode_transcript( + batch, + )?)), }) - } else { - work_stream_event::Outcome::Prefix(transcript) - }; - Ok(WorkStreamEvent { - outcome: Some(outcome), }) + .chain(events.last().into_iter().map(|event| { + let transcript = encode_transcript(std::slice::from_ref(event))?; + let outcome = + work_stream_event::Outcome::Terminal(hellas_rpc::pb::work::WorkStreamTerminal { + result: delivered.result.clone(), + provider_signature: delivered.provider_signature.clone(), + terminal_transcript: transcript, + }); + Ok(WorkStreamEvent { + outcome: Some(outcome), + }) + })) +} + +// Batch only events already available. Each bounded frame still passes the +// provider's fresh chain check; each envelope is verified separately by the client. +fn prefix_batches( + mut events: &[OutputEventEnvelope], + frame_limit: u32, + fetch: bool, +) -> impl Iterator { + let budget = if fetch { + (frame_limit as usize).min(64 * 1024).saturating_sub(32) + } else { + 0 + }; + std::iter::from_fn(move || { + if events.is_empty() { + return None; + } + let mut bytes = 0usize; + let mut count = 0; + for event in events { + let next = bytes.saturating_add(spool_charge(event, fetch)); + if count > 0 && next > budget { + break; + } + bytes = next; + count += 1; + } + let (batch, rest) = events.split_at(count); + events = rest; + Some(batch) }) } @@ -147,6 +186,17 @@ impl WorkService { .and_then(|mut endpoint| endpoint.reserve_stream(&request, &exporter)); match reserved { Ok(()) => { + let limits = service.endpoint().and_then(|endpoint| { + let policy = endpoint.admitting()?.execution_policy(); + Ok((policy.max_encoded_result_frame(), matches!(policy, PaidWorkPolicy::Fetch { .. }))) + }); + let (frame_limit, fetch) = match limits { + Ok(limits) => limits, + Err(error) => { + yield Err(WireStatus::new(hellas_wire::WireCode::Unavailable, error.to_string())); + return; + } + }; let response = service.release(&request, &context); match response.outcome { Some(DeliverOutcome::Delivered(delivered)) => { @@ -158,7 +208,11 @@ impl WorkService { } }; if events.last().is_some_and(|event| event.event().body().kind() == hellas_rpc::fetch::OUTPUT_TERMINAL_KIND) { - for frame in fetch_frames(&delivered, &events).skip(position) { + let Some(tail) = events.get(position..) else { + yield Err(WireStatus::new(hellas_wire::WireCode::Internal, "terminal result is shorter than streamed output")); + return; + }; + for frame in fetch_frames(&delivered, tail, frame_limit) { yield frame.map_err(|error| WireStatus::new(hellas_wire::WireCode::Internal, error.to_string())); } } else { @@ -176,10 +230,10 @@ impl WorkService { let progress = service.progress.lock().expect("paid progress poisoned"); progress.get(&work_id).map(|progress| progress.events[position.min(progress.events.len())..].to_vec()).unwrap_or_default() }; - for event in pending { - match encode_transcript(&[event]) { + for batch in prefix_batches(&pending, frame_limit, fetch) { + match encode_transcript(batch) { Ok(prefix) => { - position += 1; + position += batch.len(); yield Ok(WorkStreamEvent { outcome: Some(work_stream_event::Outcome::Prefix(prefix)) }); } Err(error) => { @@ -406,3 +460,53 @@ where "result stream ended without terminal result", )) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn batched_fetch_preserves_every_envelope_and_bounds_each_frame() { + let key = hellas_rpc::ProducerSigningKey::from_secret_bytes([1; 32]).unwrap(); + let mut builder = hellas_rpc::fetch::FetchOutputTranscriptBuilder::new( + hellas_rpc::InputCommitment::from_digest(hellas_rpc::Digest::from_bytes([2; 32])), + hellas_rpc::Assurance::ProducerSigned, + &key, + ); + for _ in 0..100 { + builder.push_event(vec![7; 256]).unwrap(); + } + let events = builder.finish(vec![8; 16]).unwrap(); + let delivered = WorkDelivered { + result: vec![3; 128], + provider_signature: vec![4; 64], + transcript: encode_transcript(&events).unwrap(), + }; + for limit in [4096, 65536] { + // Resuming after an arbitrary prefix must skip envelopes, not batches. + for position in [0, 1, 47, 100] { + let frames = fetch_frames(&delivered, &events[position..], limit) + .collect::, _>>() + .unwrap(); + let mut decoded = Vec::new(); + for frame in &frames { + assert!(frame.encoded_len() <= limit as usize); + let bytes = match frame.outcome.as_ref().unwrap() { + work_stream_event::Outcome::Prefix(bytes) => bytes, + work_stream_event::Outcome::Terminal(end) => &end.terminal_transcript, + _ => panic!("unexpected frame"), + }; + decoded.extend(decode_transcript(bytes, MAX_RECORD_BYTES).unwrap()); + } + assert_eq!(decoded, events[position..]); + assert!(matches!( + frames.last().unwrap().outcome, + Some(work_stream_event::Outcome::Terminal(_)) + )); + if position == 0 { + assert!(frames.len() < events.len()); + } + } + } + } +} diff --git a/docs/http-gateway.md b/docs/http-gateway.md index 09e66e92..3e490716 100644 --- a/docs/http-gateway.md +++ b/docs/http-gateway.md @@ -65,7 +65,10 @@ do not authorize this HTTP path. Signed response prefixes stream as they arrive. Successful HTTP completion follows verification of the complete result and the provider's durable payment acknowledgement. Each accepted valid terminal costs the channel's agreed fixed -price, including HTTP error statuses. Requests are not retried on another +price, including HTTP error statuses. Already available prefixes share bounded +wire frames, with a fresh chain-state check before each frame and individual +signature checks for every event. Batching does not wait for more upstream data. +Requests are not retried on another provider after account selection. The pool serializes work within each provider channel; concurrent requests assigned to that provider queue within its deadline, including requests for different accounts on that provider. Separate providers have independent channels. diff --git a/docs/paid-gateway.md b/docs/paid-gateway.md index de451ae6..8124a89a 100644 --- a/docs/paid-gateway.md +++ b/docs/paid-gateway.md @@ -44,6 +44,9 @@ pin `provider_genesis`; their funded bond fixes the settlement identity in all c Provision provider bonds with `hellas-cli provision`; coin values and policy terms must agree with the provider work config. Payment coins must be owned by the gateway settlement identity and cannot fund two channels. +The omission bond must exceed the remaining payment capacity: with 1,000 units +and zero fees/reserve, a bond of 501 leaves 499 spendable units. The client checks +this before signing setup terms or creating a journal. The provider's route table must authorize the gateway transport and settlement identities. All participants must use the same compatible chain revision. From 08d4efef14bac9706ff1358d336031eb3f99acc7 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Fri, 25 Sep 2026 02:18:51 +0200 Subject: [PATCH 14/32] Observe paid channels independently of requests Remove synchronous validator refreshes from admission and streaming. Give each channel an independent observer and share journal state only across short synchronous operations, preserving payment ordering and restart recovery. Expire local readiness when finalized progress stalls, bound reconnects, and keep retained certificates retransmittable. Replace observation and work-config errors with typed causes and remove the competing gateway follower. Validated with 709 passing tests, strict Clippy, formatting, and two live Kimi/Codex/Claude tool-loop passes. Established request traces contain no validator RPCs. --- crates/chain/src/work_blocks.rs | 4 + crates/cli/src/commands/paid_work/gateway.rs | 69 +-- crates/cli/src/commands/serve/mod.rs | 1 + crates/cli/src/commands/serve/node.rs | 15 +- crates/cli/src/commands/serve/node/tests.rs | 180 +++++-- .../src/commands/serve/work_config/tests.rs | 26 +- crates/cli/src/main.rs | 4 +- crates/client/src/work/payment.rs | 4 +- crates/rpc/src/protocol/work_setup.rs | 14 +- crates/sdk/src/paid_client.rs | 403 ++++++++------ crates/sdk/src/paid_client/tests.rs | 1 + crates/sdk/src/paid_provider.rs | 491 +++++++----------- crates/sdk/src/provider.rs | 3 +- crates/sdk/src/work_config.rs | 207 +++++--- crates/sdk/src/work_config/error.rs | 78 +++ crates/sdk/src/work_provision/tests.rs | 2 +- crates/work/src/work.rs | 192 +++++-- crates/work/src/work/client_channel.rs | 212 ++++++++ crates/work/src/work/observation.rs | 92 ++++ crates/work/src/work/stream.rs | 70 ++- crates/work/tests/work_accept.rs | 141 +++++ crates/work/tests/work_payment.rs | 154 ++++++ docs/paid-gateway.md | 35 +- 23 files changed, 1681 insertions(+), 717 deletions(-) create mode 100644 crates/sdk/src/work_config/error.rs create mode 100644 crates/work/src/work/client_channel.rs create mode 100644 crates/work/src/work/observation.rs diff --git a/crates/chain/src/work_blocks.rs b/crates/chain/src/work_blocks.rs index 1a52b33d..193d94a0 100644 --- a/crates/chain/src/work_blocks.rs +++ b/crates/chain/src/work_blocks.rs @@ -63,6 +63,8 @@ pub struct WorkBlocks(C); /// What one production clock step did for a mounted paid channel. #[derive(Debug)] pub struct PaidWorkClockAdvance { + /// The verified snapshot already read for settlement, reusable by admission. + pub snapshot: Option, /// The journaled start-or-response drive that ran first. pub close: CloseProgress, /// Submission outcome for a due adjudicated payment close. @@ -158,6 +160,7 @@ where .map_err(PaidWorkClockError::Snapshot)? else { return Ok(PaidWorkClockAdvance { + snapshot: None, close, adjudication: None, bond_timeout: None, @@ -194,6 +197,7 @@ where }; Ok(PaidWorkClockAdvance { + snapshot: Some(snapshot), close, adjudication, bond_timeout, diff --git a/crates/cli/src/commands/paid_work/gateway.rs b/crates/cli/src/commands/paid_work/gateway.rs index ef8943b4..d8a18324 100644 --- a/crates/cli/src/commands/paid_work/gateway.rs +++ b/crates/cli/src/commands/paid_work/gateway.rs @@ -65,8 +65,8 @@ const PROVIDER_CONNECTION_TIMEOUT: Duration = Duration::from_secs(10); // A retained job is durable, but it must not monopolize the channel that // serves interactive requests after a restart or a provider interruption. const RECOVERY_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30); -// A background catch-up pass yields within the request's lock-wait budget. -const CHANNEL_FOLLOW_BUDGET: Duration = Duration::from_secs(1); +// Bound queueing for Evaluate routes that can try another funded provider. +const CHANNEL_QUEUE_BUDGET: Duration = Duration::from_secs(1); // Bound HTTP delivery independently of the authenticated transcript spool. const OUTPUT_BUFFER_BYTES: usize = 2 * MAX_RECORD_BYTES; const OUTPUT_EVENT_OVERHEAD: usize = 1024; @@ -233,7 +233,6 @@ struct PaidGateway { producer_key: hellas_rpc::ProducerSigningKey, admission: Arc, tasks: Mutex>>, - followers: Mutex>>, } pub async fn load_gateway_backend( @@ -346,7 +345,6 @@ pub async fn load_gateway_backend( settlement_key, producer_key, tasks: Mutex::new(Vec::new()), - followers: Mutex::new(Vec::new()), }); // Restart recovery uses the retained input and certificate, never a new job. // Empty journal roots do not fund a channel until an HTTP request arrives. @@ -365,39 +363,6 @@ pub async fn load_gateway_backend( Some(RECOVERY_ATTEMPT_TIMEOUT), None, )?; - let provider = provider.clone(); - gateway - .followers - .lock() - .expect("paid followers poisoned") - .push(tokio::spawn(async move { - let mut reported_failure = false; - loop { - // Leave a gap after each pass, including a slow one, and - // let startup recovery and waiting requests own the channel. - tokio::time::sleep(CHANNEL_FOLLOW_BUDGET).await; - let result = { - let Ok(mut session) = provider.serial.try_lock() else { - continue; - }; - let Some(session) = session.as_mut() else { - continue; - }; - // Catch-up journals complete blocks as it advances; - // cancellation resumes from that durable cursor. - tokio::time::timeout(CHANNEL_FOLLOW_BUDGET, session.follow_chain()).await - }; - match result { - Ok(Ok(())) => reported_failure = false, - Ok(Err(error)) if !reported_failure => { - reported_failure = true; - tracing::warn!(provider = %provider.args.provider, %error, - "paid channel chain follower will retry"); - } - Ok(Err(_)) | Err(_) => {} - } - } - })); } Ok(gateway) } @@ -507,8 +472,7 @@ impl PaidGateway { task_span.record("hellas.provider.id", tracing::field::display(provider.args.provider)); task_span.record("hellas.route.cache_affinity_tokens", route.cache_affinity_tokens); task_span.record("hellas.route.pending", route.pending); - // Wait for a brief follower pass to yield, but route around - // channels still occupied by recovery or other requests. + // Serialize jobs on one funded channel; its observer runs independently. let mut session = if recovery || matches!(prepared, Some(PreparedPaidWorkInput::Fetch(_))) { before_proposal( &sender, streamed, deadline, @@ -516,7 +480,7 @@ impl PaidGateway { ).await? } else { match tokio::time::timeout( - CHANNEL_FOLLOW_BUDGET, + CHANNEL_QUEUE_BUDGET, provider.serial.lock(), ).await { Ok(session) => session, @@ -594,10 +558,8 @@ impl PaidGateway { // ClientEndpoint journals the proposal nonce before releasing // its signature. Recovery and a failed dial need not propose // this request; a lost acceptance response does advance it. - let proposal_nonce = session.state().proposal_nonce_high_water(); - let already_proposed = prepared_bytes.as_ref().is_some_and(|input| { - session.state().jobs().any(|job| job.prepared_input() == input) - }); + let proposal_nonce = session.with_state(|state| state.proposal_nonce_high_water())?; + let already_proposed = session.with_state(|state| prepared_bytes.as_ref().is_some_and(|input| state.jobs().any(|job| job.prepared_input() == input)))?; let request_session = &mut *session; let input = prepared.clone(); let on_progress = streamed.then(|| progress.clone()); @@ -612,7 +574,7 @@ impl PaidGateway { .map(Option::unwrap_or_default); if let Err(error) = &result { if !recovery && error.is::() - && session.state().proposal_nonce_high_water() == proposal_nonce + && session.with_state(|state| state.proposal_nonce_high_water())? == proposal_nonce { return result; } @@ -631,7 +593,7 @@ impl PaidGateway { Some(hellas_sdk::paid_client::PaidClientError::Propose(hellas_work::work::ProposeError::Store(_))), ); if !already_proposed && !uncertain_append - && session.state().proposal_nonce_high_water() == proposal_nonce + && session.with_state(|state| state.proposal_nonce_high_water())? == proposal_nonce { provider.connection_failed(); tracing::debug!(provider = %provider.args.provider, error = %format!("{error:#}"), @@ -751,14 +713,6 @@ impl PaidExecutionBackend for PaidGateway { self.admission.close(); std::mem::take(&mut *tasks) }; - for follower in self - .followers - .lock() - .expect("paid followers poisoned") - .drain(..) - { - follower.abort(); - } Box::pin(async move { let deadline = tokio::time::Instant::now() + Duration::from_secs(60); let mut interrupted = 0; @@ -770,6 +724,7 @@ impl PaidExecutionBackend for PaidGateway { } Err(_) => { task.abort(); + let _ = task.await; interrupted += 1; } } @@ -780,6 +735,11 @@ impl PaidExecutionBackend for PaidGateway { "paid gateway shutdown deadline reached; retained work will recover on startup" ); } + for provider in &self.providers { + if let Some(mut session) = provider.serial.lock().await.take() { + session.shutdown().await; + } + } self.endpoint.close().await; }) } @@ -1199,7 +1159,6 @@ mod tests { producer_key: hellas_rpc::ProducerSigningKey::from_secret_bytes([7; 32]).unwrap(), admission: Arc::new(Semaphore::new(1)), tasks: Mutex::new(Vec::new()), - followers: Mutex::new(Vec::new()), }; // Deterministically pause execute at the point after admission but // before preparation/routing has reached task registration. diff --git a/crates/cli/src/commands/serve/mod.rs b/crates/cli/src/commands/serve/mod.rs index 7f65371a..f533392f 100644 --- a/crates/cli/src/commands/serve/mod.rs +++ b/crates/cli/src/commands/serve/mod.rs @@ -170,6 +170,7 @@ async fn run_with_store( routes: work.routes.clone(), validators: work.validators.clone(), poll: work.poll, + max_observation_age: work.max_observation_age, settlement_key: options.settlement_key.clone(), policy: work.provider_policy(), }); diff --git a/crates/cli/src/commands/serve/node.rs b/crates/cli/src/commands/serve/node.rs index bd7eb017..a2b38ab6 100644 --- a/crates/cli/src/commands/serve/node.rs +++ b/crates/cli/src/commands/serve/node.rs @@ -15,6 +15,7 @@ use std::sync::Mutex; use std::time::Duration; use anyhow::Context; +#[cfg(test)] use hellas_chain::FinalizedWorkView; #[cfg(feature = "evaluate")] use hellas_executor::ArtifactStoreConfig; @@ -56,6 +57,7 @@ use hellas_wire::{Dispatcher, ServiceMarker, StreamTransport}; use hellas_wire::{TransportContext, WireStatus}; #[cfg(test)] use hellas_work::work::PaidWorkBackend; +#[cfg(test)] use hellas_work::work_close::FinalizedBlocks; #[cfg(test)] use hellas_work::work_close::TxSink; @@ -75,7 +77,7 @@ use crate::commands::discovery::{DiscoveryAdvertiser, served_alpns, start_server use crate::identity::OpenIdentity; pub(super) use hellas_sdk::paid_provider::{ - MountedSetup, MountedWork, ProductionWorkSource, UnmountedWork, WorkRunner, WorkRunnerConfig, + MountedSetup, MountedWork, UnmountedWork, WorkRunner, WorkRunnerConfig, }; /// Keep peer-controlled transport state finite. A connection can multiplex @@ -282,7 +284,7 @@ pub(super) async fn spawn_node(config: NodeConfig) -> anyhow::Result executor: handle.clone(), open_identity: config.open_identity, }; - let work_mount: MountedWork = MountedWork::with_backend(handle); + let work_mount: MountedWork = MountedWork::with_backend(handle); let setup_mount = MountedSetup::default(); let work = config.work.map(|work| { let poll = work.poll; @@ -379,18 +381,15 @@ pub(super) async fn spawn_node(config: NodeConfig) -> anyhow::Result /// Per-connection serve: each inbound substream is dispatched to the /// service selected by the connection's negotiated ALPN. -async fn serve_connection( +async fn serve_connection( alpn: Vec, conn: Connection, remote_execution: RemoteExecutionServices, node_handler: NodeHandlerImpl, manager: PeerManager, setup: Option, - work: Option>, -) -> anyhow::Result<()> -where - S: FinalizedBlocks + FinalizedWorkView + Sync, -{ + work: Option, +) -> anyhow::Result<()> { let transport = Arc::new(IrohTransport::new(conn)); let context = transport.context(); diff --git a/crates/cli/src/commands/serve/node/tests.rs b/crates/cli/src/commands/serve/node/tests.rs index 2459bbde..528d94bf 100644 --- a/crates/cli/src/commands/serve/node/tests.rs +++ b/crates/cli/src/commands/serve/node/tests.rs @@ -363,7 +363,7 @@ async fn remote_cache_cli_uses_the_nodes_existing_dispatch_and_admin_grant() { let accepting = server.clone(); let task = tokio::spawn(async move { let connection = accepting.accept().await.unwrap().await.unwrap(); - serve_connection::( + serve_connection( connection.alpn().to_vec(), connection, execution, @@ -466,7 +466,7 @@ async fn production_fetch_alpn_dispatches_run_ticket_on_its_connection() { .expect("the server accepts the Fetch dial") .await .expect("the Fetch handshake completes"); - serve_connection::( + serve_connection( connection.alpn().to_vec(), connection, test_remote_execution(), @@ -541,7 +541,7 @@ async fn exchange_routed_setup( .expect("the routed setup connection starts") .await .expect("the routed setup handshake completes"); - serve_connection::( + serve_connection( connection.alpn().to_vec(), connection, test_remote_execution(), @@ -627,10 +627,11 @@ async fn two_vouched_peers_receive_their_distinct_configured_offers() { ]), validators: Vec::new(), poll: Duration::from_millis(1), + max_observation_age: Duration::from_secs(5), settlement_key: provider(), policy: provider_policy(), }, - MountedWork::::default(), + MountedWork::default(), setup_mount.clone(), ) .expect("both owned provider journals are discovered"); @@ -714,9 +715,9 @@ async fn two_vouched_peers_receive_their_distinct_configured_offers() { fn discover_two_route_runner( root: &Path, - work_mount: &MountedWork, + work_mount: &MountedWork, setup_mount: &MountedSetup, -) -> WorkRunner { +) -> WorkRunner { let first = OfferFixture::first(); let second = OfferFixture::second(); match WorkRunner::discover( @@ -738,6 +739,7 @@ fn discover_two_route_runner( ]), validators: Vec::new(), poll: Duration::from_millis(1), + max_observation_age: Duration::from_secs(5), settlement_key: provider(), policy: provider_policy(), }, @@ -750,7 +752,7 @@ fn discover_two_route_runner( } async fn accept_mounted_route( - mount: &MountedWork, + mount: &MountedWork, peer: PeerId, request: AcceptWorkRequest, ) -> AcceptWorkResponse { @@ -850,7 +852,7 @@ async fn completion_clears_and_mounts_only_the_completing_route() { /// B's independent snapshot and service remain ready. Retained answers and /// already-expired proposals do not wait for another admission decision. #[tokio::test] -async fn fresh_readiness_is_per_request_and_per_routed_channel() { +async fn requests_use_local_observation_and_one_contested_route_does_not_disable_another() { let dir = temp(); let first = OfferFixture::first(); let second = OfferFixture::second(); @@ -869,6 +871,7 @@ async fn fresh_readiness_is_per_request_and_per_routed_channel() { assert!(runner.tick(&source).await, "both completed routes mount"); let first_request = first.signed_accept_request(); + let reads_before_request = source.0.lock().unwrap().snapshot_reads; let accepted = accept_mounted_route(&work_mount, first_route_peer(), first_request.clone()).await; assert!( @@ -879,31 +882,24 @@ async fn fresh_readiness_is_per_request_and_per_routed_channel() { "A's first fresh snapshot permits its signature", ); + assert_eq!( + source.0.lock().unwrap().snapshot_reads, + reads_before_request, + "acceptance performs no validator reads" + ); + source.set_snapshot(first.ready_snapshot(ORIGIN, Some(pending_contest(false)))); let service = work_mount .service(&vouched_context(first_route_peer())) .unwrap(); let checkpoint = service.with_state(ChannelState::checkpoint).unwrap(); - // Hold a real fresh admission in its chain read through the public handler. + // Block the observer's read, not a request. Existing local evidence still + // permits replies, and those replies cannot initiate validator I/O. let entered = Arc::new(Semaphore::new(0)); let release = Arc::new(Semaphore::new(0)); source.0.lock().unwrap().next_read = Some((entered.clone(), release.clone())); - let mut waiting_authorization = first.authorization(); - waiting_authorization.proposal_nonce = 3; - let waiting_id = work_id(first.descriptor().channel(), &waiting_authorization); - let waiting_request = AcceptWorkRequest { - authorization: waiting_authorization.encode(), - client_signature: first - .client() - .sign(signing_hash(waiting_id)) - .as_bytes() - .to_vec(), - prepared_input: first_request.prepared_input.clone(), - }; - let waiting_mount = work_mount.clone(); - let waiting = tokio::spawn(async move { - accept_mounted_route(&waiting_mount, first_route_peer(), waiting_request).await - }); + let observer_source = source.clone(); + let observing = tokio::spawn(async move { runner.tick(&observer_source).await }); entered.acquire().await.unwrap().forget(); let repeated = tokio::time::timeout( Duration::from_secs(1), @@ -943,7 +939,7 @@ async fn fresh_readiness_is_per_request_and_per_routed_channel() { checkpoint ); release.add_permits(1); - waiting.await.unwrap(); + assert!(observing.await.unwrap()); let mut fresh_authorization = first.authorization(); fresh_authorization.proposal_nonce = 2; @@ -2132,6 +2128,7 @@ impl TxSink for TestChain { struct RoutedChain(Arc>); struct RoutedChainState { + snapshot_reads: usize, next_read: Option<(Arc, Arc)>, completed_setups: Vec, snapshots: Vec, @@ -2144,6 +2141,7 @@ impl RoutedChain { snapshots: impl IntoIterator, ) -> Self { Self(Arc::new(Mutex::new(RoutedChainState { + snapshot_reads: 0, next_read: None, completed_setups: completed_setups.into_iter().collect(), snapshots: snapshots.into_iter().collect(), @@ -2208,7 +2206,11 @@ impl FinalizedWorkView for RoutedChain { &self, query: WorkChannelQuery, ) -> Result, QueryError> { - let gate = self.0.lock().unwrap().next_read.take(); + let gate = { + let mut state = self.0.lock().unwrap(); + state.snapshot_reads += 1; + state.next_read.take() + }; if let Some((entered, release)) = gate { entered.add_permits(1); release.acquire().await.unwrap().forget(); @@ -2471,11 +2473,7 @@ impl TxSink for NodeChain { /// The runner a node starts with: the configured root, the stored /// identity, and the policy it was configured with. -fn runner( - root: &Path, - policy: ProviderChannelPolicy, - mount: &MountedWork, -) -> WorkRunner { +fn runner(root: &Path, policy: ProviderChannelPolicy, mount: &MountedWork) -> WorkRunner { match WorkRunner::discover( WorkRunnerConfig { network: network(), @@ -2484,6 +2482,7 @@ fn runner( routes: configured_routes(&[(default_route_peer(), bond_edge(), client().party_key())]), validators: Vec::new(), poll: Duration::from_millis(1), + max_observation_age: Duration::from_secs(5), settlement_key: provider(), policy, }, @@ -2947,7 +2946,7 @@ struct RunningPaidNode { runner_task: JoinHandle<()>, stop: Option>, setup_mount: MountedSetup, - work_mount: MountedWork, + work_mount: MountedWork, execution_calls: Arc, peer: PeerId, } @@ -2968,6 +2967,7 @@ impl RunningPaidNode { routes: configured_routes(&[(peer, bond_edge(), client().party_key())]), validators: Vec::new(), poll: Duration::from_millis(1), + max_observation_age: Duration::from_secs(5), settlement_key: provider(), policy, }, @@ -3312,17 +3312,22 @@ async fn run_advertised_paid_exchange( "the production clock submits and finalizes both setup Opens", ); - // Deliberately leave the raw driven service holding the readiness - // that was true when the channel mounted. The advertised Work - // wrapper must not trust that cached value: the contested arm below - // changes the coherent source before its first network request. - let mounted = node - .work_mount - .handler(&node.context()) - .expect("the production clock mounted a request handler"); - if let Err(error) = mounted.refresh_admission().await { - panic!("mount-time readiness primes the exact driven service: {error}"); - } + // The observer must publish readiness before the first request. Request + // handling itself has no validator connection and cannot refresh it. + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if node + .work_mount + .service(&node.context()) + .is_some_and(|service| service.readiness().is_ok()) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the channel observer publishes readiness"); if contested { let contest_height = source.open_contest(); @@ -3713,6 +3718,7 @@ fn the_clock_resumes_an_accepted_job_after_restart() { ); chain.set_snapshot(ready_channel_snapshot(ORIGIN, Some(pending_contest(false)))); + assert!(restarted.tick(&chain).await); let response: WithTrailer = handler .deliver_result(request, context) .await @@ -3766,12 +3772,21 @@ async fn open_result_stream_stops_when_a_contest_finalizes() { )); chain.set_snapshot(ready_channel_snapshot(ORIGIN, Some(pending_contest(false)))); + assert!(runner.tick(&chain).await); backend.finish(); let next = tokio::time::timeout(Duration::from_secs(5), stream.next()) .await .unwrap() .expect("the open stream terminates with a refusal"); - assert_eq!(next.unwrap_err().code, hellas_wire::WireCode::Unavailable); + assert!( + match next { + Err(status) => status.code == hellas_wire::WireCode::Unavailable, + Ok(event) => matches!(event.outcome, + Some(hellas_rpc::pb::work::work_stream_event::Outcome::Refused(refusal)) + if refusal.code == WorkRefusalCode::NotReady as i32 || refusal.code == WorkRefusalCode::Unavailable as i32), + }, + "no payload is released after the observer revokes readiness" + ); assert!(stream.next().await.is_none()); } @@ -3788,6 +3803,9 @@ async fn the_clock_does_not_starve_the_request_path() { let backend = BlockingPaidBackend::new(); let mount = MountedWork::with_backend(backend.clone()); let mut runner = runner(dir.path(), provider_policy(), &mount); + let initial = TestChain::new(); + initial.set_snapshot(ready_channel_snapshot(ORIGIN, None)); + assert!(runner.tick(&initial).await); let chain = TestChain::slow(); chain.set_snapshot(ready_channel_snapshot(ORIGIN, None)); @@ -3911,3 +3929,75 @@ async fn a_clean_shutdown_leaves_a_replayable_journal() { "the answer the clock fixed replays", ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_blocked_channel_observer_does_not_stop_other_channels_or_shutdown() { + let dir = temp(); + let first = OfferFixture::first(); + let second = OfferFixture::second(); + first.write_completed_setup(dir.path()); + second.write_completed_setup(dir.path()); + let source = RoutedChain::new( + [first.bond_edge(), second.bond_edge()], + [ + first.ready_snapshot(ORIGIN, None), + second.ready_snapshot(ORIGIN, None), + ], + ); + let entered = Arc::new(Semaphore::new(0)); + let release = Arc::new(Semaphore::new(0)); + source.0.lock().unwrap().next_read = Some((entered.clone(), release)); + let mount = MountedWork::default(); + let setups = MountedSetup::default(); + let runner = discover_two_route_runner(dir.path(), &mount, &setups); + let (stop, stopped) = oneshot::channel(); + let observing = tokio::spawn(async move { + runner + .run_over(stopped, move || { + let source = source.clone(); + async move { Some(source) } + }) + .await; + }); + tokio::time::timeout(Duration::from_secs(2), entered.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if mount + .service(&vouched_context(second_route_peer())) + .is_some_and(|service| service.readiness().is_ok()) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the second channel advances while the first source is blocked"); + let response = + accept_mounted_route(&mount, second_route_peer(), second.signed_accept_request()).await; + assert!(matches!( + response.outcome, + Some(accept_work_response::Outcome::Accepted(_)) + )); + let held = mount + .service(&vouched_context(second_route_peer())) + .unwrap(); + stop.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(2), observing) + .await + .expect("shutdown cancels the blocked read") + .unwrap(); + assert!( + held.readiness().is_err(), + "already-held handlers cannot admit after shutdown" + ); + assert!( + mount + .service(&vouched_context(second_route_peer())) + .is_none() + ); +} diff --git a/crates/cli/src/commands/serve/work_config/tests.rs b/crates/cli/src/commands/serve/work_config/tests.rs index b3f25ccd..498d07c0 100644 --- a/crates/cli/src/commands/serve/work_config/tests.rs +++ b/crates/cli/src/commands/serve/work_config/tests.rs @@ -128,7 +128,7 @@ fn write(dir: &tempfile::TempDir, value: &serde_json::Value) -> PathBuf { fn load(value: serde_json::Value) -> CliResult { let dir = tempfile::tempdir().unwrap(); - load_work_config(&write(&dir, &value)) + Ok(load_work_config(&write(&dir, &value))?) } /// Delete `field` from the object at `path`. @@ -830,3 +830,27 @@ fn a_restarted_node_rebuilds_its_endpoint_from_the_root_and_the_identity() { assert_eq!(state.revision(), Some(1)); } + +#[test] +fn observer_freshness_must_leave_time_for_a_poll() { + let poll = config()["poll_ms"].as_u64().unwrap(); + for age in [0, poll] { + let error = load(with( + config(), + "max_observation_age_ms", + serde_json::json!(age), + )) + .unwrap_err(); + assert!(format!("{error:#}").contains("max_observation_age_ms")); + } + let configured = load(with( + config(), + "max_observation_age_ms", + serde_json::json!(poll + 1), + )) + .unwrap(); + assert_eq!( + configured.max_observation_age, + std::time::Duration::from_millis(poll + 1) + ); +} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 082f801a..a408a4f9 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1070,7 +1070,7 @@ async fn async_main() { .map(commands::serve::load_work_config) .transpose() { - Err(error) => Err(error), + Err(error) => Err(error.into()), Ok(work_config) => { async { // The key every settlement this node signs is signed @@ -1149,7 +1149,7 @@ async fn async_main() { max_job_price, print_bond_only, } => match commands::serve::load_work_config(&work_config) { - Err(error) => Err(error), + Err(error) => Err(error.into()), Ok(work_config) => { commands::serve::run_provision(commands::serve::ProvisionOptions { work_config, diff --git a/crates/client/src/work/payment.rs b/crates/client/src/work/payment.rs index ff20fd8e..01e4d142 100644 --- a/crates/client/src/work/payment.rs +++ b/crates/client/src/work/payment.rs @@ -7,7 +7,7 @@ use hellas_rpc::protocol::Digest; use hellas_rpc::services::work::WorkClientImpl; use hellas_wire::StreamTransport; -use hellas_work::work::{ClientEndpoint, PaymentError, admit_payment}; +use hellas_work::work::{ClientChannel, PaymentError, admit_payment}; /// Signs and admits payment for one authenticated result. /// @@ -22,7 +22,7 @@ use hellas_work::work::{ClientEndpoint, PaymentError, admit_payment}; /// journal rejects the transition, or the acknowledgement names another amount. pub async fn pay_for_result( transport: T, - endpoint: &mut ClientEndpoint, + endpoint: &mut impl ClientChannel, work_id: Digest, ) -> Result where diff --git a/crates/rpc/src/protocol/work_setup.rs b/crates/rpc/src/protocol/work_setup.rs index 3b6b6447..d35ce8ef 100644 --- a/crates/rpc/src/protocol/work_setup.rs +++ b/crates/rpc/src/protocol/work_setup.rs @@ -859,14 +859,12 @@ pub struct ObservedChannel<'a> { /// method here, because the objects it would have to re-read are not /// carried. /// -/// So the obligation is the holder's, in the same way [`ObservedChannel`] -/// owes coherence: run [`WorkChannelDescriptor::check_ready`] again -/// against a fresh snapshot before each signature, and sign against the -/// `ReadyChannel` that read produced. [`Self::check_signable`] is the -/// per-signature *arithmetic* — the horizon and the deadline margins, -/// against the height the endpoint has actually reached. It is not a -/// substitute for the refresh, and it does not claim to be one: no -/// arithmetic over a stale read can see a contest that opened after it. +/// A live channel observer must apply finalized history and refresh this +/// decision independently of requests. Admission, delivery and new certificates +/// check local observer freshness and the journal's close cutoff under the same +/// state lock. An expired observer must suspend those operations; cached +/// arithmetic alone cannot establish that the channel remains open. +/// [`Self::check_signable`] checks deadline margins against the applied cursor. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ReadyChannel { channel: PaidChannel, diff --git a/crates/sdk/src/paid_client.rs b/crates/sdk/src/paid_client.rs index df6a98cd..0e123862 100644 --- a/crates/sdk/src/paid_client.rs +++ b/crates/sdk/src/paid_client.rs @@ -4,6 +4,7 @@ use std::net::SocketAddr; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; +use tracing::Instrument as _; mod error; pub use error::PaidClientError; @@ -13,7 +14,6 @@ use hellas_chain::{ ConsensusInfo, ConsensusVerifier, FinalizedWorkView as _, WorkBlocks, WorkChannelQuery, }; use hellas_client::work::payment::pay_for_result; -use hellas_client::work::{CollectResultOutcome, collect_result}; use hellas_kernel::{ EdgeId, Funding, MAX_START_VALIDITY_BLOCKS, Secp256k1Signer, Secp256k1Verifier, WorkPaymentTerms, @@ -26,8 +26,10 @@ use hellas_rpc::protocol::work_profile::{PaidWorkPolicy, PreparedPaidWorkInput}; use hellas_rpc::protocol::work_setup::{ProviderChannelPolicy, WorkChannelDescriptor}; use hellas_wire::ServiceMarker; use hellas_wire::iroh::IrohTransport; -use hellas_work::work::{ClientEndpoint, JobProposal, propose_work, resume_work_proposal}; -use hellas_work::work_close::CloseProgress; +use hellas_work::work::{ + ClientChannel as _, ClientEndpoint, ClientObserver, ClientService, JobProposal, fetch_result, + propose_work, resume_work_proposal, +}; use hellas_work::work_close::FinalizedBlocks as _; use hellas_work::work_handshake::{ PaymentAdmission, SetupEndpoint, SetupService, apply_setup_exchange, prepare_setup_exchange, @@ -144,6 +146,7 @@ pub async fn run_paid_work( if settle { result.settled_provider_payout = Some(session.settle().await?); } + session.shutdown().await; endpoint.close().await; Ok(result) }) @@ -186,12 +189,94 @@ pub struct PaidWorkSession { args: PaidWorkOptions, descriptor: WorkChannelDescriptor, dialer: ProviderDialer, - chain: WorkBlocks, - next_validator: usize, - client: ClientEndpoint, + client: ClientService, + observer: Option>, needs_recovery: bool, } +impl Drop for PaidWorkSession { + fn drop(&mut self) { + let _ = self.client.observer().suspend(); + if let Some(observer) = &self.observer { + observer.abort(); + } + } +} + +fn spawn_observer( + client: ClientObserver, + descriptor: WorkChannelDescriptor, + config: WorkConfig, + mut chain: WorkBlocks, + mut next_validator: usize, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + loop { + if client + .with_state(|state| state.close_settled().is_some()) + .unwrap_or(false) + { + break; + } + let started = hellas_work::work::ObservationTime::now(); + let span = hellas_rpc::request_span!(target: "hellas_request", parent: None, "paid.channel.observe", hellas.channel.role = "client"); + let result = tokio::time::timeout( + config.max_observation_age, + async { + let mut driver = client.drive()?; + driver.advance_close(&chain, &chain).await?; + let ready = ready_channel(&descriptor, &chain).await?; + if ready + .check_caught_up(client.with_state(|state| state.cursor().0)?) + .is_err() + { + driver.catch_up_to(&chain, ready.finalized_height()).await?; + } + client.observe_ready(ready, started, config.max_observation_age)?; + Ok::<_, PaidClientError>(()) + } + .instrument(span), + ) + .await + .unwrap_or(Err(PaidClientError::Timeout { + stage: "channel observer", + })); + if let Err(error) = result { + let _ = client.suspend(); + tracing::debug!(%error, "paid client observation unavailable"); + // Redial a bounded connection attempt; close duties remain armed. + if !matches!( + error, + PaidClientError::WorkSetup(_) | PaidClientError::Endpoint(_) + ) && let Ok(Ok(replacement)) = + tokio::time::timeout(config.max_observation_age, async { + let replacement = connect_chain(&config, &mut next_validator).await?; + check_genesis(&config, &replacement).await?; + Ok::<_, PaidClientError>(replacement) + }) + .await + { + chain = replacement; + } + } + tokio::time::sleep(config.poll).await; + } + }) +} + +fn check_payment_window(client: &ClientService, work_id: hellas_rpc::Digest) -> Result<()> { + client.with_state(|state| { + let job = state + .job_by_id(work_id) + .ok_or(PaidClientError::MissingState("accepted job disappeared"))?; + if state.cursor().0 > job.authorization().payment_deadline { + Err(PaidClientError::PaymentExpired) + } else { + Ok(()) + } + })? +} + impl PaidWorkSession { pub async fn open( args: PaidWorkOptions, @@ -288,18 +373,65 @@ impl PaidWorkSession { let setup_service = SetupService::new(setup); let (mounted, descriptor) = drive_setup(&setup_service, &policy, &chain, config.poll).await?; - let ready = ready_channel(&descriptor, &chain).await?; - let client = ClientEndpoint::new(ready.clone(), mounted, settlement_key)?; + let started = hellas_work::work::ObservationTime::now(); + let ready = match ready_channel(&descriptor, &chain).await { + Ok(ready) => Some(ready), + Err(PaidClientError::WorkSetup(_)) => None, + Err(error) => return Err(error), + }; + let initially_ready = ready.is_some(); + let mut endpoint = ClientEndpoint::recover(mounted, settlement_key)?; + endpoint.catch_up(&chain).await?; + let client = ClientService::new(endpoint); + if let Some(ready) = ready { + client + .observer() + .observe_ready(ready, started, config.max_observation_age)?; + } + let observer = spawn_observer( + client.observer(), + descriptor.clone(), + config.clone(), + chain, + next_validator, + ); - Ok(Self { + let session = Self { args, descriptor, dialer, - chain, - next_validator, client, + observer: Some(observer), needs_recovery: true, - }) + }; + if initially_ready { + // A long restart catch-up may outlive the first observation. Wait + // during channel opening; requests never perform this catch-up. + // The session already owns the task, so cancellation aborts it. + tokio::time::timeout(session.args.timeout, async { + while session.client.readiness().is_err() + && !session.client.with_state(|state| state.is_closing())? + { + tokio::time::sleep(session.args.config.poll).await; + } + Ok::<_, PaidClientError>(()) + }) + .await + .map_err(|_| PaidClientError::Timeout { + stage: "initial channel observation", + })??; + } + Ok(session) + } + + /// Stops observation and joins the task before releasing the journal. + /// Call after draining jobs; retained accounting is recovered on restart. + pub async fn shutdown(&mut self) { + let _ = self.client.observer().suspend(); + if let Some(observer) = self.observer.take() { + observer.abort(); + let _ = observer.await; + } } /// Identifies the funded channel and its admitted execution policy. @@ -307,9 +439,12 @@ impl PaidWorkSession { &self.descriptor } - /// Read-only journal state for routing, recovery, and admission decisions. - pub fn state(&self) -> &hellas_work::work_store::ChannelState { - self.client.state() + /// Reads local journal state under a short lock. + pub fn with_state( + &self, + read: impl FnOnce(&hellas_work::work_store::ChannelState) -> R, + ) -> Result { + Ok(self.client.with_state(read)?) } /// True after reopening a journal or an interrupted request. @@ -317,39 +452,19 @@ impl PaidWorkSession { self.needs_recovery } - /// Opens the client close and waits for its finalized provider payout. + /// Opens a close; the independent observer drives its finalized payout. pub async fn settle(&mut self) -> Result { - self.client.prepare_close()?; + if !self.client.with_state(|state| state.is_closing())? { + self.client.with_client(|client| client.prepare_close())??; + } loop { - match self.client.advance_close(&self.chain, &self.chain).await? { - CloseProgress::Settled { provider_payout } => return Ok(provider_payout), - CloseProgress::Submitted { outcome, .. } => { - tracing::info!(?outcome, "client payment close submitted") - } - CloseProgress::Opened { .. } | CloseProgress::Nothing => {} + if let Some(settlement) = self.client.with_state(|state| state.close_settled())? { + return Ok(settlement.provider_payout); } tokio::time::sleep(self.args.config.poll).await; } } - /// Refreshes finalized state, rotating through configured validators on failure. - pub async fn follow_chain(&mut self) -> Result<()> { - for attempt in 0..self.args.config.validators.len() { - match self.client.catch_up(&self.chain).await { - Ok(_) => return Ok(()), - Err(error) if attempt + 1 == self.args.config.validators.len() => { - return Err(error.into()); - } - Err(error) => { - tracing::debug!(%error, "paid channel will continue catch-up through another validator"); - self.chain = connect_chain(&self.args.config, &mut self.next_validator).await?; - check_genesis(&self.args.config, &self.chain).await?; - } - } - } - Err(PaidClientError::NoValidators) - } - /// Runs a request and pays only after verifying its complete result. /// With `recover`, resume payable journaled work first and admit this as a new /// job; otherwise reuse a still-active job matching the input. `None` only @@ -373,12 +488,10 @@ impl PaidWorkSession { progress: Option, proposed: Option<&AtomicBool>, ) -> Result> { - self.follow_chain().await?; let Self { args, descriptor, dialer, - chain, client, needs_recovery, .. @@ -392,45 +505,57 @@ impl PaidWorkSession { hellas_rpc::PublicKey::Secp256k1(descriptor.channel().client_key().to_bytes()), )?; } - let ready = caught_up_channel(descriptor, client, &*chain).await?; + if recover + && *needs_recovery + && let Some(work_id) = + client.with_state(|state| state.last_payment().map(|payment| payment.work_id))? + { + // A lost acknowledgement never requires fresh admission: these + // exact certificate bytes are already durable on the client. + pay_for_result(dialer.work().await?, client, work_id).await?; + } + if client.with_state(|state| state.is_closing())? { + return if prepared.is_none() { + Ok(None) + } else { + Err(hellas_work::work::EndpointError::NotAdmitting.into()) + }; + } + let ready = client.readiness()?; if recover && *needs_recovery { - if let Some(payment) = client.state().last_payment() { - // The provider may have committed payment while its acknowledgement - // was lost. Re-send the retained certificate before accepting work. - pay_for_result(dialer.work().await?, client, payment.work_id).await?; - } - let pending = client - .state() - .jobs() - .filter(|job| { - // The journal forbids signing payment after this height. - // Keep the evidence, but do not let an unpayable old job - // prevent this channel from serving a new request. Retained - // certificates are re-sent separately above. - let payable = job.authorization().payment_deadline >= client.state().cursor().0; - if !payable { - tracing::info!( - work_id = %hex::encode(job.work_id().as_bytes()), - payment_deadline = job.authorization().payment_deadline, - "retaining expired unpaid job without retrying execution", - ); - } - payable - }) - .map(|job| { - // Fetch journals retain accounting only. A restart cannot - // reconstruct a lost request or authorize another execution. - if job.prepared_input().is_empty() { - return Err(PaidClientError::MissingPayload { - work_id: job.work_id(), - payment_deadline: job.authorization().payment_deadline, - }); - } - PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) - .map(|input| (job.work_id(), job.phase(), input)) - .map_err(PaidClientError::from) - }) - .collect::, _>>()?; + let pending = client.with_state(|state| { + state + .jobs() + .filter(|job| { + // The journal forbids signing payment after this height. + // Keep the evidence, but do not let an unpayable old job + // prevent this channel from serving a new request. Retained + // certificates are re-sent separately above. + let payable = job.authorization().payment_deadline >= state.cursor().0; + if !payable { + tracing::info!( + work_id = %hex::encode(job.work_id().as_bytes()), + payment_deadline = job.authorization().payment_deadline, + "retaining expired unpaid job without retrying execution", + ); + } + payable + }) + .map(|job| { + // Fetch journals retain accounting only. A restart cannot + // reconstruct a lost request or authorize another execution. + if job.prepared_input().is_empty() { + return Err(PaidClientError::MissingPayload { + work_id: job.work_id(), + payment_deadline: job.authorization().payment_deadline, + }); + } + PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) + .map(|input| (job.work_id(), job.phase(), input)) + .map_err(PaidClientError::from) + }) + .collect::, _>>() + })??; for (work_id, phase, pending) in pending { check_request( &config.provider_policy(), @@ -444,7 +569,6 @@ impl PaidWorkSession { dialer, client, &ready, - &*chain, config.poll, None, JobLookup::Retained(work_id), @@ -475,7 +599,7 @@ impl PaidWorkSession { *needs_recovery = prepared.is_some(); let result = match prepared { Some(prepared) => { - let ready = caught_up_channel(descriptor, client, &*chain).await?; + let ready = client.readiness()?; Some( execute_paid_job( args, @@ -483,7 +607,6 @@ impl PaidWorkSession { dialer, client, &ready, - &*chain, config.poll, progress.as_ref(), if recover { @@ -521,7 +644,7 @@ fn permanently_refused_delivery(error: &PaidClientError) -> bool { /// flood. async fn propose_when_ready( dialer: &ProviderDialer, - client: &mut ClientEndpoint, + client: &mut ClientService, proposal: &JobProposal, retained: Option, poll: Duration, @@ -571,9 +694,8 @@ async fn execute_paid_job( args: &PaidWorkOptions, prepared: PreparedPaidWorkInput, dialer: &ProviderDialer, - client: &mut ClientEndpoint, + client: &mut ClientService, ready: &hellas_rpc::protocol::work_setup::ReadyChannel, - chain: &WorkBlocks, poll: Duration, progress: Option<&hellas_work::work::PaidProgress>, lookup: JobLookup, @@ -581,7 +703,7 @@ async fn execute_paid_job( ) -> Result { let readiness_timeout = args.timeout; let prepared_bytes = prepared.encode()?; - let current = client.state().cursor().0; + let current = client.with_state(|state| state.cursor().0)?; let deadlines = deadlines( current, args.acceptance_blocks, @@ -592,21 +714,22 @@ async fn execute_paid_job( prepared_input: prepared, deadlines, }; - let existing = client - .state() - .jobs() - .filter(|job| match &lookup { - JobLookup::Retained(work_id) => job.work_id() == *work_id, - JobLookup::New => false, - JobLookup::PreparedInput => { - job.prepared_input() == prepared_bytes.as_slice() - && job.authorization().payment_deadline >= current - && (job.phase() != hellas_work::work_store::JobPhase::HalfSigned - || job.authorization().acceptance_deadline >= current) - } - }) - .map(|job| (job.work_id(), job.phase(), *job.authorization())) - .collect::>(); + let existing = client.with_state(|state| { + state + .jobs() + .filter(|job| match &lookup { + JobLookup::Retained(work_id) => job.work_id() == *work_id, + JobLookup::New => false, + JobLookup::PreparedInput => { + job.prepared_input() == prepared_bytes.as_slice() + && job.authorization().payment_deadline >= current + && (job.phase() != hellas_work::work_store::JobPhase::HalfSigned + || job.authorization().acceptance_deadline >= current) + } + }) + .map(|job| (job.work_id(), job.phase(), *job.authorization())) + .collect::>() + })?; if existing.len() > 1 { return Err(PaidClientError::AmbiguousRecovery); } @@ -648,9 +771,11 @@ async fn execute_paid_job( }; let transcript = if already_collected { client - .state() - .job_by_id(work_id) - .map(|job| job.transcript().to_vec()) + .with_state(|state| { + state + .job_by_id(work_id) + .map(|job| job.transcript().to_vec()) + })? .ok_or(PaidClientError::MissingState( "collected job disappeared from its journal", ))? @@ -691,31 +816,14 @@ async fn execute_paid_job( if emitted || !retryable { return Err(error.into()); } - client.catch_up(chain).await?; - let job = client - .state() - .job_by_id(work_id) - .ok_or(PaidClientError::MissingState("accepted job disappeared"))?; - if client.state().cursor().0 > job.authorization().payment_deadline { - return Err(PaidClientError::PaymentExpired); - } + check_payment_window(client, work_id)?; tracing::debug!(%error, %work_id, "waiting for paid result stream readiness"); tokio::time::sleep(poll.max(Duration::from_secs(1))).await; }; - client.catch_up(chain).await?; - if client.state().cursor().0 - > client - .state() - .job_by_id(work_id) - .ok_or(PaidClientError::MissingState("accepted job disappeared"))? - .authorization() - .payment_deadline - { - return Err(PaidClientError::PaymentExpired); - } + check_payment_window(client, work_id)?; delivery.transcript } else { - collect_until_ready(dialer, client, ready, chain, work_id, poll).await? + collect_until_ready(dialer, client, ready, work_id, poll).await? }; let credited = pay_for_result(dialer.work().await?, client, work_id).await?; Ok(PaidWorkResult { @@ -868,49 +976,26 @@ async fn ready_channel( .map_err(PaidClientError::from) } -/// Reads the ready snapshot once the client has processed it. -/// -/// The snapshot and the blocks the cursor follows are answered by -/// validators independently, so the snapshot can name a height the -/// light client has not finalized yet, and a snapshot read after the -/// catch-up on a moving chain always lands a few blocks ahead of it. -/// The snapshot is sampled first and then the cursor is brought to it: -/// a fixed height is a target the catch-up reaches, where a fresh -/// snapshot every round was not. A state at or behind the cursor is -/// the direction `check_caught_up` accepts. -async fn caught_up_channel( - descriptor: &WorkChannelDescriptor, - client: &mut ClientEndpoint, - chain: &WorkBlocks, -) -> Result { - let ready = ready_channel(descriptor, chain).await?; - for _ in 0..16 { - let cursor = client.catch_up(chain).await?; - if ready.check_caught_up(cursor).is_ok() { - return Ok(ready); - } - tokio::time::sleep(Duration::from_millis(500)).await; - } - let cursor = client.catch_up(chain).await?; - ready.check_caught_up(cursor)?; - Ok(ready) -} - async fn collect_until_ready( dialer: &ProviderDialer, - client: &mut ClientEndpoint, + client: &mut ClientService, ready: &hellas_rpc::protocol::work_setup::ReadyChannel, - chain: &WorkBlocks, work_id: hellas_rpc::Digest, poll: Duration, ) -> Result> { loop { - match collect_result(dialer.work().await?, client, ready, chain, work_id).await? { - CollectResultOutcome::Collected(result) => return Ok(result.transcript), - CollectResultOutcome::NotReady { reason } => { + match fetch_result(dialer.work().await?, client, ready, work_id).await { + Ok(delivery) => { + check_payment_window(client, work_id)?; + return Ok(delivery.transcript); + } + Err(hellas_work::work::DeliverError::Refused { refusal, reason }) + if refusal.is_retryable() => + { tracing::debug!(%reason, "waiting for paid result"); tokio::time::sleep(poll.max(Duration::from_secs(1))).await; } + Err(error) => return Err(error.into()), } } } diff --git a/crates/sdk/src/paid_client/tests.rs b/crates/sdk/src/paid_client/tests.rs index 4ca11bcc..26fe890f 100644 --- a/crates/sdk/src/paid_client/tests.rs +++ b/crates/sdk/src/paid_client/tests.rs @@ -33,6 +33,7 @@ async fn insufficient_collateral_is_rejected_before_network_or_journal_creation( channel_policy: policy.channel_policy, execution_policy: policy.execution_policy, poll: Duration::from_millis(200), + max_observation_age: Duration::from_secs(5), expected_payment_values: hellas_kernel::EdgeValues::new( 1000, 0, diff --git a/crates/sdk/src/paid_provider.rs b/crates/sdk/src/paid_provider.rs index 1d2cc1c2..77445ee6 100644 --- a/crates/sdk/src/paid_provider.rs +++ b/crates/sdk/src/paid_provider.rs @@ -1,9 +1,8 @@ //! Shared provider routing and finalized-chain clock for paid work. use crate::work_config::WorkRoutes; -use anyhow::Context; use futures::future::BoxFuture; use hellas_chain::client::VerifiedRemoteLightClient; -use hellas_chain::work_blocks::{PaidWorkClockError, advance_paid_work_clock}; +use hellas_chain::work_blocks::advance_paid_work_clock; use hellas_chain::{ ConsensusInfo, ConsensusVerifier, FinalizedWorkView, WorkBlocks, WorkChannelQuery, }; @@ -24,7 +23,7 @@ use hellas_wire::{TransportContext, WireStatus}; use hellas_work::work::{ CloseEndpoint, PaidWorkBackend, RunError, RunOutcome, WorkService, run_accepted_work, }; -use hellas_work::work_close::{CatchUpError, FinalizedBlocks, TxSink}; +use hellas_work::work_close::{FinalizedBlocks, TxSink}; use hellas_work::work_handshake::{PaymentAdmission, SetupEndpoint, SetupService}; use hellas_work::work_open::{SetupAdvance, SetupDriveError, SetupProgress, SetupView}; use hellas_work::work_store::{ChannelStore, JobPhase, Role, SetupStore, discover_setups}; @@ -34,8 +33,38 @@ use std::{ sync::{Arc, Mutex}, time::Duration, }; -use tokio::sync::{Mutex as AsyncMutex, oneshot}; -use tracing::{debug, info, warn}; +use tokio::sync::oneshot; +use tracing::{Instrument as _, debug, info, warn}; +/// Provider observation and recovery failures preserve their typed causes. +#[derive(Debug, thiserror::Error)] +pub enum PaidProviderError { + #[error("channel has no admission descriptor")] + NoDescriptor, + #[error("no finalized channel snapshot is available")] + NoSnapshot, + #[error("finalized snapshot names another channel")] + WrongSnapshot, + #[error("accepted work has no execution backend")] + NoBackend, + #[error("observer age must exceed a positive polling interval")] + InvalidObservationPolicy, + #[error(transparent)] + Endpoint(#[from] hellas_work::work::EndpointError), + #[error(transparent)] + Setup(#[from] hellas_rpc::protocol::work_setup::WorkSetupError), + #[error(transparent)] + Query(#[from] hellas_chain::QueryError), + #[error(transparent)] + CatchUp(#[from] hellas_work::work_close::CatchUpError), + #[error(transparent)] + Consensus(#[from] hellas_chain::ConsensusVerificationError), + #[error("cannot discover work journals under {}: {source}", path.display())] + Discover { + path: PathBuf, + source: hellas_work::work_store::WorkStoreError, + }, +} + pub type ProductionWorkSource = WorkBlocks; #[derive(Clone, Copy, Debug)] pub struct UnmountedWork; @@ -116,6 +145,8 @@ pub struct WorkRunnerConfig { pub validators: Vec, /// How often the clock ticks. pub poll: Duration, + /// Bounds observer stalls and the age of local admission evidence. + pub max_observation_age: Duration, /// The key every settlement this node signs is signed with. pub settlement_key: Secp256k1Signer, /// What every setup endpoint this node builds countersigns over. @@ -125,12 +156,12 @@ pub struct WorkRunnerConfig { /// Channels indexed by authenticated peer. Handlers share the runner's state; /// mount locks are released before processing requests. #[derive(Clone)] -pub struct MountedWork { - mounted: Arc>>>>, +pub struct MountedWork { + mounted: Arc>>>, driver: Option, } -impl Default for MountedWork { +impl Default for MountedWork { fn default() -> Self { Self { mounted: Arc::new(Mutex::new(BTreeMap::new())), @@ -199,132 +230,14 @@ impl AcceptedWorkDriver { } } -/// One channel's handler. The runner replaces `source` when it redials a validator. -/// `accepting` serializes each fresh readiness read through signing to prevent -/// concurrent requests from consuming the same credit. +/// A peer's local paid channel. Validator connections belong to the observer. #[derive(Clone)] -pub struct MountedWorkService { - bond_edge: EdgeId, +pub struct MountedWorkService { service: WorkService, - descriptor: Option, - source: Arc>, - accepting: Arc>, driver: Option, } -impl MountedWorkService -where - S: FinalizedBlocks + FinalizedWorkView + Sync, -{ - /// Refreshes admission and checks the cursor using one coherent finalized read. - /// Any failure leaves admission disabled until a later successful refresh. - pub async fn refresh_admission(&self) -> anyhow::Result { - // Drop the source lock before awaiting the chain read. - let source = { - let held = self - .source - .lock() - .map_err(|_| anyhow::anyhow!("the finalized source lock is poisoned"))?; - held.clone() - }; - refresh_work_admission(&self.service, self.descriptor.as_ref(), &source).await - } - - async fn refresh_delivery(&self, request: &DeliverResultRequest) -> anyhow::Result<()> { - let Ok(bytes) = request.work_id.as_slice().try_into() else { - return Ok(()); - }; - let work_id = Digest::from_bytes(bytes); - let active = self - .service - .with_state(|state| state.job_by_id(work_id).is_some()); - if !matches!(active, Ok(true)) { - return Ok(()); - } - // A restarted mount has no readiness cached. A retained result must - // be collectable without first accepting another job. Terminal replies - // need no fresh admission and are authenticated by the core service. - let _accepting = self.accepting.lock().await; - let result = self.refresh_admission().await; - if let Err(error) = &result { - debug!(%error, "a delivery attempt found no fresh channel readiness"); - } - result.map(|_| ()) - } -} - -/// Shared readiness check for live requests and restart recovery. -/// Both paths use a fresh finalized snapshot. -async fn refresh_work_admission( - service: &WorkService, - descriptor: Option<&WorkChannelDescriptor>, - source: &S, -) -> anyhow::Result -where - S: FinalizedBlocks + FinalizedWorkView + Sync, -{ - let Some(descriptor) = descriptor else { - anyhow::bail!("this channel has no admission descriptor"); - }; - let query = WorkChannelQuery { - bond_edge: descriptor.bond_edge(), - payment_edge: descriptor.channel().payment_edge(), - funding: Default::default(), - }; - let Some(snapshot) = source - .work_channel_snapshot(query.clone()) - .await - .context("the fresh coherent channel read failed")? - else { - anyhow::bail!("no finalized channel snapshot is available"); - }; - if snapshot.query() != &query { - anyhow::bail!("the finalized source answered for another channel"); - } - let ready = descriptor - .check_ready(&snapshot.observed_channel()) - .context("the fresh channel snapshot is not ready")?; - // Keep one snapshot as the target. Recovery runs on the clock itself, - // so waiting for another tick here would prevent the cursor advancing. - // Take the service's existing driver when available; a concurrent clock - // drive keeps that authority until it finishes its own catch-up. - for _ in 0..16 { - let cursor = service - .with_state(|state| state.cursor().0) - .context("the mounted channel cursor is unavailable")?; - if ready.check_caught_up(cursor).is_ok() { - break; - } - if let Ok(mut driver) = service.drive() { - driver - .catch_up(source) - .await - .context("the mounted channel could not catch up to the fresh snapshot")?; - let cursor = service - .with_state(|state| state.cursor().0) - .context("the mounted channel cursor is unavailable")?; - if ready.check_caught_up(cursor).is_ok() { - break; - } - } - tokio::time::sleep(Duration::from_millis(250)).await; - } - let cursor = service - .with_state(|state| state.cursor().0) - .context("the mounted channel cursor is unavailable")?; - ready - .check_caught_up(cursor) - .context("the mounted channel has not caught up to the fresh snapshot")?; - service - .admit_new_work(ready.clone()) - .context("the driven work service refused its fresh readiness")?; - Ok(ready) -} - -impl WorkHandler for MountedWorkService -where - S: FinalizedBlocks + FinalizedWorkView + Sync, -{ +impl WorkHandler for MountedWorkService { async fn accept_work( &self, request: AcceptWorkRequest, @@ -333,20 +246,14 @@ where if let Some(response) = self.service.precheck_acceptance(&request) { return Ok(response); } - let _accepting = self.accepting.lock().await; - // A preceding request or the clock may have resolved this proposal - // while admission was serialized. Retained replies need no fresh read. - if let Some(response) = self.service.precheck_acceptance(&request) { - return Ok(response); - } - let ready = match self.refresh_admission().await { + let ready = match self.service.readiness() { Ok(ready) => ready, Err(error) => { - debug!(%error, "an acceptance attempt found no fresh channel readiness"); + debug!(%error, "channel observer is not ready for acceptance"); return Ok(AcceptWorkResponse { outcome: Some(accept_work_response::Outcome::Refused(WorkRefused { code: WorkRefusalCode::NotReady as i32, - reason: "fresh channel readiness is unavailable".to_string(), + reason: error.to_string(), })), }); } @@ -387,19 +294,7 @@ where request: DeliverResultRequest, context: TransportContext, ) -> Result> + Send, WireStatus> { - let response: WithTrailer = - if self.refresh_delivery(&request).await.is_ok() { - self.service.deliver_result(request, context).await?.into() - } else { - DeliverResultResponse { - outcome: Some(deliver_result_response::Outcome::Refused(WorkRefused { - code: WorkRefusalCode::NotReady as i32, - reason: "fresh channel readiness is unavailable".to_string(), - })), - } - .into() - }; - Ok(response) + self.service.deliver_result(request, context).await } async fn stream_result( @@ -407,35 +302,7 @@ where request: DeliverResultRequest, context: TransportContext, ) -> Result { - self.refresh_delivery(&request).await.map_err(|_| { - WireStatus::new( - hellas_wire::WireCode::Unavailable, - "fresh channel readiness is unavailable", - ) - })?; - let mounted = self.clone(); - let mut stream = self.service.stream_result(request.clone(), context).await?; - Ok(Box::pin(async_stream::try_stream! { - let mut refresh = tokio::time::interval(Duration::from_secs(1)); - loop { - let event = tokio::select! { - event = futures::StreamExt::next(&mut stream) => Some(event), - _ = refresh.tick() => None, - }; - // Even buffered prefixes need a fresh finalized decision before - // leaving the node; a contest can start after this stream opens. - mounted.refresh_delivery(&request).await.map_err(|_| WireStatus::new( - hellas_wire::WireCode::Unavailable, - "fresh channel readiness is unavailable", - ))?; - if let Some(event) = event { - match event { - Some(event) => yield event?, - None => break, - } - } - } - })) + self.service.stream_result(request, context).await } async fn admit_certificate( @@ -447,7 +314,7 @@ where } } -impl MountedWork { +impl MountedWork { pub fn with_backend(backend: B) -> Self where B: PaidWorkBackend + Send + Sync + 'static, @@ -459,24 +326,12 @@ impl MountedWork { } /// Mounts a channel for a peer. Multiple candidates disable routing for that peer. - pub fn mount( - &self, - peer: PeerId, - bond_edge: EdgeId, - service: &WorkService, - descriptor: Option, - accepting: Arc>, - source: &S, - ) -> bool { + pub fn mount(&self, peer: PeerId, service: &WorkService) -> bool { match self.mounted.lock() { Ok(mut held) => { let mounted = held.entry(peer).or_default(); mounted.push(MountedWorkService { - bond_edge, service: service.clone(), - descriptor, - source: Arc::new(Mutex::new(source.clone())), - accepting, driver: self.driver.clone(), }); mounted.len() == 1 @@ -486,7 +341,7 @@ impl MountedWork { } /// The one handler mounted for the transport-vouched peer. - pub fn handler(&self, context: &TransportContext) -> Option> { + pub fn handler(&self, context: &TransportContext) -> Option { let peer = context .vouched_peer() .map(|peer| PeerId::from_bytes(peer.0))?; @@ -499,29 +354,17 @@ impl MountedWork { } /// Returns the local service for journal inspection and recovery. - /// Remote requests must use `handler`, which refreshes finalized readiness. + /// Remote requests must use `handler`, which enforces local observer readiness. pub fn service(&self, context: &TransportContext) -> Option { self.handler(context).map(|mounted| mounted.service) } - /// Updates the shared source, including handlers held by live connections. - fn refresh_source(&self, peer: PeerId, bond_edge: EdgeId, source: &S) { - let source_slot = self.mounted.lock().ok().and_then(|held| { - held.get(&peer)? - .iter() - .find(|mounted| mounted.bond_edge == bond_edge) - .map(|mounted| Arc::clone(&mounted.source)) - }); - if let Some(source_slot) = source_slot - && let Ok(mut held) = source_slot.lock() - { - *held = source.clone(); - } - } - /// Unmounts all channels when the clock stops, releasing its journal handles. pub fn clear_all(&self) { if let Ok(mut held) = self.mounted.lock() { + for channel in held.values().flatten() { + let _ = channel.service.suspend(); + } held.clear(); } } @@ -609,43 +452,68 @@ enum Driven { } /// A driven channel recovers accepted jobs even without a peer route. -/// Live acceptance and recovery share the same admission lock. +/// Live acceptance and recovery use the same journaled running marker. struct DrivenChannel { service: WorkService, descriptor: Option, - accepting: Arc>, + max_observation_age: Duration, driver: Option, } impl DrivenChannel { - fn accepted_work_id(&self) -> anyhow::Result> { + async fn refresh( + &self, + source: &S, + started: hellas_work::work::ObservationTime, + snapshot: Option, + ) -> Result<(), PaidProviderError> + where + S: FinalizedBlocks + FinalizedWorkView + Sync, + { + let descriptor = self + .descriptor + .as_ref() + .ok_or(PaidProviderError::NoDescriptor)?; + let query = WorkChannelQuery { + bond_edge: descriptor.bond_edge(), + payment_edge: descriptor.channel().payment_edge(), + funding: Default::default(), + }; + let snapshot = snapshot.ok_or(PaidProviderError::NoSnapshot)?; + if snapshot.query() != &query { + return Err(PaidProviderError::WrongSnapshot); + } + let ready = descriptor.check_ready(&snapshot.observed_channel())?; + let cursor = self.service.with_state(|state| state.cursor().0)?; + if ready.check_caught_up(cursor).is_err() { + self.service + .drive()? + .catch_up_to(source, ready.finalized_height()) + .await?; + } + self.service + .observe_ready(ready, started, self.max_observation_age)?; + Ok(()) + } + + fn accepted_work_id(&self) -> Result, PaidProviderError> { self.service .with_state(|state| { let mut jobs = state.jobs(); let job = jobs.next()?; (jobs.next().is_none() && job.phase() == JobPhase::Accepted).then(|| job.work_id()) }) - .context("the driven channel state is unavailable") + .map_err(PaidProviderError::from) } /// Resumes accepted work after a fresh readiness check. The durable `JobRunning` /// record admits one invocation even when a live request races this clock tick. - async fn resume_accepted(&self, source: &S) -> anyhow::Result - where - S: FinalizedBlocks + FinalizedWorkView + Sync, - { - if self.accepted_work_id()?.is_none() { - return Ok(false); - } - let _accepting = self.accepting.lock().await; + fn resume_accepted(&self) -> Result { let Some(work_id) = self.accepted_work_id()? else { return Ok(false); }; - let driver = self - .driver - .as_ref() - .context("the accepted paid job has no execution backend")?; - let ready = refresh_work_admission(&self.service, self.descriptor.as_ref(), source).await?; + let driver = self.driver.as_ref().ok_or(PaidProviderError::NoBackend)?; + let ready = self.service.readiness()?; driver.spawn(self.service.clone(), ready, work_id); Ok(true) } @@ -653,6 +521,7 @@ impl DrivenChannel { /// One setup journal on a clock. struct SetupClock { + max_observation_age: Duration, /// The bond this journal stakes, so a log line names which one. bond_edge: EdgeId, /// The authenticated peer whose configured route names this bond. @@ -670,7 +539,7 @@ impl SetupClock { &mut self, source: &S, signer: &Secp256k1Signer, - work_mount: &MountedWork, + work_mount: &MountedWork, setup_mount: &MountedSetup, ) -> bool where @@ -695,7 +564,7 @@ impl SetupClock { if let Some(peer) = self.route_peer { setup_mount.clear(peer, self.bond_edge); } - self.take_mount(store, signer, &policy, source, work_mount); + self.take_mount(store, signer, &policy, work_mount); } else if matches!( progress, SetupProgress::Aborted(_) | SetupProgress::Faulted(_) @@ -713,37 +582,42 @@ impl SetupClock { } } if let Driven::Channel(channel) = &self.driven { - if let Some(peer) = self.route_peer { - work_mount.refresh_source(peer, self.bond_edge, source); - } - if let Err(error) = channel.resume_accepted(source).await { - warn!(bond, %error, "an accepted paid job did not resume"); - } - match advance_paid_work_clock(&channel.service, source).await { - Ok(progress) => debug!(bond, ?progress, "the channel advanced"), - // `resume_accepted` owns the channel while it starts the - // durable execution. The clock's concurrent close pass has - // no work to do until that owner returns the cursor. - Err(PaidWorkClockError::CloseDrive(CatchUpError::Busy)) => { - debug!(bond, "the channel is already being driven") + let started = hellas_work::work::ObservationTime::now(); + let snapshot = match advance_paid_work_clock(&channel.service, source).await { + Ok(progress) => { + debug!(bond, close = ?progress.close, "the channel advanced"); + progress.snapshot } Err(error) => { - answered &= !error.source_failed(); + let _ = channel.service.suspend(); warn!(bond, %error, "this channel's close did not advance"); + return !error.source_failed(); } + }; + if let Err(error) = channel.refresh(source, started, snapshot).await { + let _ = channel.service.suspend(); + debug!(bond, %error, "channel observation did not renew admission"); + answered &= !matches!( + error, + PaidProviderError::Query(_) + | PaidProviderError::CatchUp( + hellas_work::work_close::CatchUpError::Source(_) + ) + ); + } else if let Err(error) = channel.resume_accepted() { + warn!(bond, %error, "an accepted paid job did not resume"); } } answered } /// Transfers the mounted store returned by setup, preserving its exclusive lock. - fn take_mount( + fn take_mount( &mut self, store: ChannelStore, signer: &Secp256k1Signer, policy: &ProviderChannelPolicy, - source: &S, - mount: &MountedWork, + mount: &MountedWork, ) { let bond = hex::encode(self.bond_edge.to_bytes()); // Rebuild the descriptor from the retained policy and signed payment terms. @@ -760,17 +634,15 @@ impl SetupClock { match CloseEndpoint::new(store, signer.clone()) { Ok(close) => { let service = WorkService::close_only(close); - let accepting = Arc::new(AsyncMutex::new(())); - if self.route_peer.is_some_and(|peer| { - mount.mount( - peer, - self.bond_edge, - &service, - descriptor.clone(), - Arc::clone(&accepting), - source, - ) - }) { + if let Err(error) = service.require_observer() { + warn!(bond, %error, "channel observer could not be installed"); + self.driven = Driven::Done; + return; + } + if self + .route_peer + .is_some_and(|peer| mount.mount(peer, &service)) + { info!( bond, "this node now answers Work from the channel it mounted" @@ -784,7 +656,7 @@ impl SetupClock { self.driven = Driven::Channel(Box::new(DrivenChannel { service, descriptor, - accepting, + max_observation_age: self.max_observation_age, driver: mount.driver.clone(), })); } @@ -801,17 +673,17 @@ impl SetupClock { } /// The clock, over every paid-work journal this node owns. -pub struct WorkRunner { +pub struct WorkRunner { clocks: Vec, signer: Secp256k1Signer, - work_mount: MountedWork, + work_mount: MountedWork, setup_mount: MountedSetup, poll: Duration, validators: Vec, consensus_verifier: ConsensusVerifier, } -impl WorkRunner { +impl WorkRunner { pub fn journal_count(&self) -> usize { self.clocks.len() } @@ -823,30 +695,29 @@ impl WorkRunner { } } -impl WorkRunner -where - S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, -{ +impl WorkRunner { /// Discovers provider journals and mounts configured routes. Unrouted journals /// are still driven through close. Unreadable journals are logged; failure to /// enumerate the root is returned to the caller. pub fn discover( config: WorkRunnerConfig, - work_mount: MountedWork, + work_mount: MountedWork, setup_mount: MountedSetup, - ) -> anyhow::Result { + ) -> Result { + if config.poll.is_zero() || config.max_observation_age <= config.poll { + return Err(PaidProviderError::InvalidObservationPolicy); + } let consensus_verifier = ConsensusVerifier::new(&ConsensusInfo { validators: config.validators.clone(), threshold_identity: config.threshold_identity, network_id: config.network.as_str().to_owned(), - }) - .context("the configured threshold identity is not usable")?; + })?; let settlement_verifier = Secp256k1Verifier::new(); - let found = discover_setups(&config.journal_root, config.network).with_context(|| { - format!( - "failed to enumerate the work journals under {}", - config.journal_root.display(), - ) + let found = discover_setups(&config.journal_root, config.network).map_err(|source| { + PaidProviderError::Discover { + path: config.journal_root.clone(), + source, + } })?; for unnamed in &found.unidentified { warn!( @@ -907,6 +778,7 @@ where } let driven = Driven::Setup { service, policy }; clocks.push(SetupClock { + max_observation_age: config.max_observation_age, bond_edge: setup.bond_edge, route_peer, driven, @@ -925,7 +797,10 @@ where /// Takes one step of every journal, and says whether the chain /// answered all of them. - pub async fn tick(&mut self, source: &S) -> bool { + pub async fn tick(&mut self, source: &S) -> bool + where + S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, + { let mut answered = true; for clock in &mut self.clocks { answered &= clock @@ -936,8 +811,9 @@ where } /// Ticks each journal once per period and redials after a source failure. - pub async fn run_over(mut self, mut stop: oneshot::Receiver<()>, dial: D) + pub async fn run_over(mut self, mut stop: oneshot::Receiver<()>, dial: D) where + S: SetupView + FinalizedBlocks + FinalizedWorkView + TxSink + Sync, D: Fn() -> F, F: core::future::Future>, { @@ -947,18 +823,53 @@ where self.setup_mount.clear_all(); return; } - let mut chain = None; - loop { - tokio::select! { - _ = &mut stop => break, - () = tokio::time::sleep(self.poll) => {} - } - let Some(source) = chain.take() else { - chain = dial().await; - continue; - }; - if self.tick(&source).await { - chain = Some(source); + // Each channel owns its observer loop. A slow source or a large restart + // backlog on one channel cannot stop another channel's close response. + { + use futures::{StreamExt as _, stream::FuturesUnordered}; + let signer = &self.signer; + let work_mount = &self.work_mount; + let setup_mount = &self.setup_mount; + let dial = &dial; + let poll = self.poll; + let mut observers = self + .clocks + .iter_mut() + .map(|clock| async move { + let mut chain = None; + loop { + let budget = clock.max_observation_age; + if chain.is_none() { + chain = tokio::time::timeout(budget, dial()).await.ok().flatten(); + } + if let Some(source) = chain.as_ref() { + match tokio::time::timeout( + budget, + clock.tick(source, signer, work_mount, setup_mount).instrument(hellas_rpc::request_span!(target: "hellas_request", parent: None, "paid.channel.observe", hellas.channel.role = "provider")), + ) + .await + { + Ok(true) => {} + Ok(false) | Err(_) => { + if let Driven::Channel(channel) = &clock.driven { + let _ = channel.service.suspend(); + } + chain = None; + } + } + } + if matches!(clock.driven, Driven::Done) { + break; + } + tokio::time::sleep(poll).await; + } + }) + .collect::>(); + loop { + tokio::select! { + _ = &mut stop => break, + next = observers.next() => if next.is_none() { break; }, + } } } self.work_mount.clear_all(); @@ -967,7 +878,7 @@ where } } -impl WorkRunner { +impl WorkRunner { /// Ticks until told to stop, over the validators the configuration /// names. pub async fn run(self, stop: oneshot::Receiver<()>) { diff --git a/crates/sdk/src/provider.rs b/crates/sdk/src/provider.rs index e29c73f8..16b32312 100644 --- a/crates/sdk/src/provider.rs +++ b/crates/sdk/src/provider.rs @@ -215,7 +215,7 @@ where let executor = Executor::spawn_configured(executor_config).await?; #[cfg(feature = "paid-work")] - let work_mount = crate::paid_provider::MountedWork::::with_backend(executor.clone()); + let work_mount = crate::paid_provider::MountedWork::with_backend(executor.clone()); #[cfg(feature = "paid-work")] let setup_mount = crate::paid_provider::MountedSetup::default(); #[cfg(feature = "paid-work")] @@ -245,6 +245,7 @@ where routes: config.routes, validators: config.validators, poll: config.poll, + max_observation_age: config.max_observation_age, settlement_key, policy, }, diff --git a/crates/sdk/src/work_config.rs b/crates/sdk/src/work_config.rs index 0a78e04e..75e12330 100644 --- a/crates/sdk/src/work_config.rs +++ b/crates/sdk/src/work_config.rs @@ -9,7 +9,9 @@ use std::fs; use std::path::{Path, PathBuf}; use std::time::Duration; -use anyhow::{Context as _, Result, bail}; +mod error; +pub use error::WorkConfigError; +type Result = std::result::Result; use hellas_kernel::{ EdgeId, EdgeValues, Fees, Key, MIN_OMIT_RESPONSE_BLOCKS, NetworkId, Secp256k1Verifier, }; @@ -50,6 +52,8 @@ pub struct WorkConfig { pub execution_policy: PaidWorkPolicy, /// How often the watcher asks the chain for the next block. pub poll: Duration, + /// Maximum time without new verified finalized progress before admission stops. + pub max_observation_age: Duration, /// The payment edge's value, reserve, and close fees as this provider /// requires a client to fund them. pub expected_payment_values: EdgeValues, @@ -102,13 +106,10 @@ impl WorkRoutes { let client = Key::from_bytes(parse_fixed_hex("routes[].client", &file.client)?); let route = WorkRoute { peer, bond, client }; if by_peer.insert(peer, route).is_some() { - bail!("routes names peer {peer:#} twice; one authenticated peer has one route"); + return Err(WorkConfigError::DuplicatePeer(peer)); } if !bonds.insert(bond) { - bail!( - "routes names bond {} twice; one provider journal has one route", - hex::encode(bond.to_bytes()), - ); + return Err(WorkConfigError::DuplicateBond(bond)); } } Ok(Self { by_peer }) @@ -144,11 +145,19 @@ pub struct ChainCrossCheck { /// Loads configuration, checks chain identity and policy bounds, and normalizes /// validator URLs. Route-to-journal validation is deferred until serve startup. pub fn load_work_config(path: &Path) -> Result { - let bytes = fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; - let file: WorkConfigFile = serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse {}", path.display()))?; - file.into_config() - .with_context(|| format!("invalid work config {}", path.display())) + let bytes = fs::read(path).map_err(|source| WorkConfigError::Read { + path: path.to_path_buf(), + source, + })?; + let file: WorkConfigFile = + serde_json::from_slice(&bytes).map_err(|source| WorkConfigError::Parse { + path: path.to_path_buf(), + source, + })?; + file.into_config().map_err(|source| WorkConfigError::File { + path: path.to_path_buf(), + source: Box::new(source), + }) } /// Checks each route against a provider journal under the configured root, @@ -157,11 +166,11 @@ pub fn validate_work_routes(config: &WorkConfig) -> Result<()> { if config.routes.is_empty() { return Ok(()); } - let found = discover_setups(&config.journal_root, config.chain.network).with_context(|| { - format!( - "failed to enumerate configured work routes under journal.root {}", - config.journal_root.display(), - ) + let found = discover_setups(&config.journal_root, config.chain.network).map_err(|source| { + WorkConfigError::Journal { + root: config.journal_root.clone(), + source, + } })?; for route in config.routes.iter() { if !found @@ -169,13 +178,10 @@ pub fn validate_work_routes(config: &WorkConfig) -> Result<()> { .iter() .any(|setup| setup.role == Role::Provider && setup.bond_edge == route.bond) { - bail!( - "route for peer {:#} names bond {}, but its provider setup journal is not under \ - journal.root {}", - route.peer, - hex::encode(route.bond.to_bytes()), - config.journal_root.display(), - ); + return Err(WorkConfigError::MissingRoute { + bond: route.bond, + root: config.journal_root.clone(), + }); } let store = SetupStore::open( &config.journal_root, @@ -184,32 +190,20 @@ pub fn validate_work_routes(config: &WorkConfig) -> Result<()> { Role::Provider, &Secp256k1Verifier::new(), ) - .with_context(|| { - format!( - "route for peer {:#} could not open provider setup journal for bond {} under {}", - route.peer, - hex::encode(route.bond.to_bytes()), - config.journal_root.display(), - ) + .map_err(|source| WorkConfigError::Journal { + root: config.journal_root.clone(), + source, })?; let Some(bundle) = store.state().bundle() else { - bail!( - "route for peer {:#} names provider setup journal for bond {}, but it holds no \ - bond proposal", - route.peer, - hex::encode(route.bond.to_bytes()), - ); + return Err(WorkConfigError::MissingProposal(route.bond)); }; let journal_client = bundle.bond_terms().parties.taker(); if journal_client != route.client { - bail!( - "route for peer {:#} expects client settlement key {}, but provider setup journal \ - for bond {} names {} as its taker", - route.peer, - hex::encode(route.client.to_bytes()), - hex::encode(route.bond.to_bytes()), - hex::encode(journal_client.to_bytes()), - ); + return Err(WorkConfigError::WrongClient { + bond: route.bond, + expected: route.client, + actual: journal_client, + }); } } Ok(()) @@ -225,17 +219,23 @@ struct WorkConfigFile { policies: PoliciesFile, /// How often the watcher asks the chain for the next block. poll_ms: u64, + #[serde(default = "default_observation_age_ms")] + max_observation_age_ms: u64, expected_payment_values: PaymentValuesFile, min_omit_response_blocks: u64, } +fn default_observation_age_ms() -> u64 { + 5_000 +} + impl WorkConfigFile { fn into_config(self) -> Result { let Some(network) = NetworkId::new(self.chain.network_id.trim()) else { - bail!( - "chain.network_id {:?} is not a network id", - self.chain.network_id - ); + return Err(WorkConfigError::Invalid { + field: "chain.network_id", + reason: "not a network id", + }); }; let threshold_identity = parse_hex("chain.threshold_identity", &self.chain.threshold_identity)?; @@ -249,23 +249,31 @@ impl WorkConfigFile { validators: validators.clone(), threshold_identity: threshold_identity.clone(), network_id: self.chain.network_id.clone(), - }) - .map_err(|error| anyhow::anyhow!("chain.threshold_identity is not usable: {error}"))?; + })?; let journal_root = self.journal.into_root()?; let routes = WorkRoutes::from_files(self.routes)?; let policies = self.policies.into_policies()?; if self.poll_ms == 0 { - bail!("poll_ms must be greater than zero"); + return Err(WorkConfigError::Invalid { + field: "poll_ms", + reason: "must be greater than zero", + }); + } + if self.max_observation_age_ms <= self.poll_ms { + return Err(WorkConfigError::Invalid { + field: "max_observation_age_ms", + reason: "must exceed poll_ms", + }); } // The kernel refuses a shorter window at every payment open, so a // configuration under it would sign terms consensus then throws // away. if self.min_omit_response_blocks < MIN_OMIT_RESPONSE_BLOCKS { - bail!( - "min_omit_response_blocks {} is under the kernel's minimum {MIN_OMIT_RESPONSE_BLOCKS}", - self.min_omit_response_blocks, - ); + return Err(WorkConfigError::ResponseWindow { + actual: self.min_omit_response_blocks, + minimum: MIN_OMIT_RESPONSE_BLOCKS, + }); } Ok(WorkConfig { @@ -284,6 +292,7 @@ impl WorkConfigFile { channel_policy: policies.1, execution_policy: policies.2, poll: Duration::from_millis(self.poll_ms), + max_observation_age: Duration::from_millis(self.max_observation_age_ms), expected_payment_values: self.expected_payment_values.into_values(), min_omit_response_blocks: self.min_omit_response_blocks, }) @@ -305,24 +314,29 @@ fn parse_validators(entries: Vec) -> Result> { for entry in entries { let entry = entry.trim(); if entry.is_empty() { - bail!("validators entries must be non-empty"); + return Err(WorkConfigError::Invalid { + field: "validators", + reason: "entries must be non-empty", + }); } - let url = url::Url::parse(entry) - .with_context(|| format!("validators entry {entry:?} is not a URL"))?; + let url = url::Url::parse(entry).map_err(|source| WorkConfigError::ValidatorUrl { + entry: entry.to_owned(), + source, + })?; if url.host_str().is_none() { - bail!("validators entry {entry:?} names no host to dial"); + return Err(WorkConfigError::ValidatorHost(entry.to_owned())); } let normalised = url.as_str().to_string(); if validators.contains(&normalised) { - bail!("validators names {normalised} twice; a fan-out to five validators is not six"); + return Err(WorkConfigError::DuplicateValidator(normalised)); } validators.push(normalised); } if validators.len() != VALIDATOR_COUNT { - bail!( - "validators must name exactly {VALIDATOR_COUNT} validator URLs, found {}", - validators.len(), - ); + return Err(WorkConfigError::ValidatorCount { + expected: VALIDATOR_COUNT, + actual: validators.len(), + }); } Ok(validators) } @@ -344,7 +358,10 @@ struct JournalFile { impl JournalFile { fn into_root(self) -> Result { if self.root.as_os_str().is_empty() { - bail!("journal.root must be a path"); + return Err(WorkConfigError::Invalid { + field: "journal.root", + reason: "must be a path", + }); } Ok(self.root) } @@ -371,7 +388,12 @@ impl PoliciesFile { match (self.execution, self.fetch) { (Some(execution), None) => execution.into_policy()?.into(), (None, Some(fetch)) => fetch.into_policy()?, - _ => bail!("policies must select exactly one of execution or fetch"), + _ => { + return Err(WorkConfigError::Invalid { + field: "policies", + reason: "must select exactly one of execution or fetch", + }); + } }, )) } @@ -406,12 +428,12 @@ struct ExecutionPolicyFile { impl ExecutionPolicyFile { fn into_policy(self) -> Result { let allowed_environment: ContentId = - self.allowed_environment.parse().with_context(|| { - format!( - "policies.execution.allowed_environment {:?} is not a ContentId", - self.allowed_environment - ) - })?; + self.allowed_environment + .parse() + .map_err(|source| WorkConfigError::ContentId { + field: "policies.execution.allowed_environment", + source, + })?; let policy = PaidExecutionPolicyV1 { allowed_environment, generation_policy_digest: parse_digest( @@ -434,8 +456,7 @@ impl ExecutionPolicyFile { fixed_price: self.fixed_price, }; // Validate with the protocol rules before any channel is proposed. - check_execution_policy(&policy) - .map_err(|error| anyhow::anyhow!("policies.execution is not usable: {error}"))?; + check_execution_policy(&policy)?; Ok(policy) } } @@ -477,13 +498,20 @@ impl FetchPolicyFile { (None, None, Some(open)) => { PaidFetchRoutePolicy::open_fetch(open.require_spki_pin, open.allowed_hosts) } - _ => bail!("policies.fetch requires service+method or open_fetch"), + _ => { + return Err(WorkConfigError::Invalid { + field: "policies.fetch", + reason: "requires service+method or open_fetch", + }); + } }; let policy = PaidFetchPolicyV1 { - allowed_environment: self - .allowed_environment - .parse() - .context("policies.fetch.allowed_environment is not a ContentId")?, + allowed_environment: self.allowed_environment.parse().map_err(|source| { + WorkConfigError::ContentId { + field: "policies.fetch.allowed_environment", + source, + } + })?, route_commitment: fetch_route_commitment(&route.canonical_body_bytes())?, max_request_body_bytes: self.max_request_body_bytes, max_output_events: self.max_output_events, @@ -497,7 +525,7 @@ impl FetchPolicyFile { fixed_price: self.fixed_price, }; let profile = PaidWorkPolicy::Fetch { policy, route }; - profile.check().context("policies.fetch is not usable")?; + profile.check()?; Ok(profile) } } @@ -535,22 +563,29 @@ impl PaymentValuesFile { } } -fn parse_hex(field: &str, raw: &str) -> Result> { - let bytes = hex::decode(raw.trim()).with_context(|| format!("{field} is not hexadecimal"))?; +fn parse_hex(field: &'static str, raw: &str) -> Result> { + let bytes = hex::decode(raw.trim()).map_err(|source| WorkConfigError::Hex { field, source })?; if bytes.is_empty() { - bail!("{field} must not be empty"); + return Err(WorkConfigError::Invalid { + field, + reason: "must not be empty", + }); } Ok(bytes) } -fn parse_fixed_hex(field: &str, raw: &str) -> Result<[u8; N]> { +fn parse_fixed_hex(field: &'static str, raw: &str) -> Result<[u8; N]> { let bytes = parse_hex(field, raw)?; let Ok(bytes) = <[u8; N]>::try_from(bytes.as_slice()) else { - bail!("{field} must be {N} bytes, found {}", bytes.len()); + return Err(WorkConfigError::Length { + field, + expected: N, + actual: bytes.len(), + }); }; Ok(bytes) } -fn parse_digest(field: &str, raw: &str) -> Result { +fn parse_digest(field: &'static str, raw: &str) -> Result { Ok(Digest::from_bytes(parse_fixed_hex(field, raw)?)) } diff --git a/crates/sdk/src/work_config/error.rs b/crates/sdk/src/work_config/error.rs new file mode 100644 index 00000000..616f89c7 --- /dev/null +++ b/crates/sdk/src/work_config/error.rs @@ -0,0 +1,78 @@ +use super::{ContentId, EdgeId, Key, PeerId}; +use std::path::PathBuf; + +/// Configuration errors retain the field, route or file that failed. +#[derive(Debug, thiserror::Error)] +pub enum WorkConfigError { + #[error("failed to read {}: {source}", path.display())] + Read { + path: PathBuf, + source: std::io::Error, + }, + #[error("failed to parse {}: {source}", path.display())] + Parse { + path: PathBuf, + source: serde_json::Error, + }, + #[error("invalid work config {}: {source}", path.display())] + File { path: PathBuf, source: Box }, + #[error("{field}: {reason}")] + Invalid { + field: &'static str, + reason: &'static str, + }, + #[error("routes names peer {0:#} twice; one authenticated peer has one route")] + DuplicatePeer(PeerId), + #[error("routes names bond {} twice; one provider journal has one route", hex::encode(.0.to_bytes()))] + DuplicateBond(EdgeId), + #[error("provider setup journal for bond {} is not under journal.root {}", hex::encode(bond.to_bytes()), root.display())] + MissingRoute { bond: EdgeId, root: PathBuf }, + #[error("work journal under {}: {source}", root.display())] + Journal { + root: PathBuf, + source: hellas_work::work_store::WorkStoreError, + }, + #[error("provider setup journal for bond {} holds no bond proposal", hex::encode(.0.to_bytes()))] + MissingProposal(EdgeId), + #[error("route expects client settlement key {}, but provider setup journal for bond {} names {} as its taker", hex::encode(expected.to_bytes()), hex::encode(bond.to_bytes()), hex::encode(actual.to_bytes()))] + WrongClient { + bond: EdgeId, + expected: Key, + actual: Key, + }, + #[error("chain.threshold_identity is not usable: {0}")] + Consensus(#[from] hellas_chain::ConsensusVerificationError), + #[error("min_omit_response_blocks {actual} is under the kernel's minimum {minimum}")] + ResponseWindow { actual: u64, minimum: u64 }, + #[error("validators entry {entry:?} is not a URL: {source}")] + ValidatorUrl { + entry: String, + source: url::ParseError, + }, + #[error("validators entry {0:?} names no host to dial")] + ValidatorHost(String), + #[error("validators names {0} twice")] + DuplicateValidator(String), + #[error("validators must name exactly {expected} validator URLs, found {actual}")] + ValidatorCount { expected: usize, actual: usize }, + #[error("{field} is not a ContentId: {source}")] + ContentId { + field: &'static str, + source: ::Err, + }, + #[error(transparent)] + Fetch(#[from] hellas_rpc::fetch::FetchProtocolError), + #[error("paid execution policy is not usable: {0}")] + Policy(#[from] hellas_rpc::protocol::work::PaidWorkError), + #[error("{field} is not hexadecimal: {source}")] + Hex { + field: &'static str, + source: hex::FromHexError, + }, + #[error("{field} must be {expected} bytes, found {actual}")] + Length { + field: &'static str, + expected: usize, + actual: usize, + }, +} diff --git a/crates/sdk/src/work_provision/tests.rs b/crates/sdk/src/work_provision/tests.rs index 7d50a475..5556bade 100644 --- a/crates/sdk/src/work_provision/tests.rs +++ b/crates/sdk/src/work_provision/tests.rs @@ -137,7 +137,7 @@ fn routed_work_config(root: &Path, routes: Vec) -> Result ProvisionOptions { diff --git a/crates/work/src/work.rs b/crates/work/src/work.rs index 9db3547f..2013e963 100644 --- a/crates/work/src/work.rs +++ b/crates/work/src/work.rs @@ -117,6 +117,13 @@ //! What neither endpoint does here is wait. `advance_close` is one //! step, and the caller that owns a clock is the one that repeats it. +mod observation; +use observation::Observation; +pub use observation::ObservationTime; + +mod client_channel; +pub use client_channel::{ClientChannel, ClientDriver, ClientObserver, ClientService}; + mod stream; pub use stream::{PaidProgress, PaidResultStream, fetch_result_stream}; @@ -379,6 +386,9 @@ const fn channel_refusal(error: &ChannelStateError) -> WorkRefusal { /// new work. #[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] pub enum EndpointError { + /// The background observer has not confirmed recent finalized progress. + #[error("channel observation is stale; awaiting finalized progress")] + ObservationStale, /// The journal is another channel's. #[error("the store's channel is not the one this readiness decided")] WrongChannel, @@ -531,6 +541,7 @@ pub struct CloseEndpoint { pub struct ProviderEndpoint { close: CloseEndpoint, ready: Option, + observation: Option, } /// How far this process has got with the answer to a live contest. @@ -606,6 +617,7 @@ impl ProviderEndpoint { close_handoff: None, }, ready: Some(ready), + observation: None, }) } @@ -616,7 +628,11 @@ impl ProviderEndpoint { /// [`WorkService::close_only`] serves it behind, where one type has /// to carry both halves because one handler answers the wire. const fn close_only(close: CloseEndpoint) -> Self { - Self { close, ready: None } + Self { + close, + ready: None, + observation: None, + } } /// The readiness this endpoint admits new work under, or the refusal @@ -627,6 +643,9 @@ impl ProviderEndpoint { /// [`EndpointError::NotAdmitting`] when no readiness decision is /// held. Nothing about a close reaches this. fn admitting(&self) -> Result<&ReadyChannel, EndpointError> { + if let Some(observation) = &self.observation { + observation.check()?; + } self.ready.as_ref().ok_or(EndpointError::NotAdmitting) } @@ -921,7 +940,11 @@ impl ProviderEndpoint { let authorization = *job.authorization(); let input = PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) .map_err(|e| RunError::Transcript(e.into()))?; - let ready = self.admitting()?.clone(); + let ready = self + .ready + .as_ref() + .ok_or(EndpointError::NotAdmitting)? + .clone(); let channel = ready.channel(); let result = ready .execution_policy() @@ -1145,6 +1168,12 @@ impl ProviderEndpoint { let certificate_signature = signature(&request.certificate_signature) .ok_or(PaymentError::Malformed("certificate signature"))?; + if self.state().payment(work_id).is_none() + && let Some(observation) = &self.observation + { + observation.check()?; + } + let state = self.close.store.commit( ChannelRecord::JobTerminated { work_id, @@ -2094,7 +2123,9 @@ impl From for Refusal { let code = match error { DeliverError::NoSuchJob | DeliverError::Terminated { .. } => WorkRefusal::Declined, DeliverError::NoResult { .. } => WorkRefusal::NotReady, - DeliverError::Endpoint(EndpointError::CatchingUp) => WorkRefusal::NotReady, + DeliverError::Endpoint(EndpointError::CatchingUp | EndpointError::ObservationStale) => { + WorkRefusal::NotReady + } DeliverError::Endpoint(EndpointError::NotAdmitting) => WorkRefusal::Unavailable, DeliverError::Unbound => WorkRefusal::Invalid, DeliverError::Setup(setup) => return Refusal::from(setup), @@ -2114,6 +2145,9 @@ impl From for Refusal { /// only one of them can raise are documented where they are mapped. #[derive(Debug, thiserror::Error)] pub enum PaymentError { + /// The shared client endpoint is unavailable. + #[error(transparent)] + Endpoint(#[from] EndpointError), /// No open job on this channel carries this `work_id`. #[error("no open job on this channel carries this work id")] NoSuchJob, @@ -2173,6 +2207,7 @@ impl From for Refusal { let code = match error { PaymentError::NoSuchJob => WorkRefusal::Declined, PaymentError::NotPayable { .. } => WorkRefusal::NotReady, + PaymentError::Endpoint(error) => endpoint_refusal(error), PaymentError::Store(store) => return Refusal::from(store), _ => WorkRefusal::Invalid, }; @@ -2351,10 +2386,20 @@ impl ChannelDriver<'_> { &mut self, source: &S, ) -> Result { - let mut cursor = self.cursor()?; + let cursor = self.cursor()?; let Some(latest) = source.latest_height().await? else { return Ok(cursor); }; + self.catch_up_to(source, latest).await + } + + /// Applies history through a fixed snapshot height, respecting close duties. + pub async fn catch_up_to( + &mut self, + source: &S, + latest: u64, + ) -> Result { + let mut cursor = self.cursor()?; while cursor < latest { let next = cursor.saturating_add(1); let block = source @@ -2482,6 +2527,53 @@ impl WorkService { self.endpoint()?.admit_new_work(ready) } + /// Requires a live background observer before accepting or releasing work. + pub fn require_observer(&self) -> Result<(), EndpointError> { + self.endpoint()?.observation = Some(Observation::default()); + Ok(()) + } + + /// Publishes readiness after applying all blocks through its snapshot. + pub fn observe_ready( + &self, + ready: ReadyChannel, + started: ObservationTime, + max_age: std::time::Duration, + ) -> Result<(), EndpointError> { + let mut endpoint = self.endpoint()?; + if ready.check_caught_up(endpoint.state().cursor().0).is_err() { + return Err(EndpointError::CatchingUp); + } + let height = ready.finalized_height(); + endpoint.admit_new_work(ready)?; + endpoint + .observation + .get_or_insert_with(Observation::default) + .confirm(height, started, max_age); + drop(endpoint); + self.changed.notify_waiters(); + Ok(()) + } + + /// Reads current local admission; no chain I/O is performed. + pub fn readiness(&self) -> Result { + let endpoint = self.endpoint()?; + if endpoint.state().is_closing() { + return Err(EndpointError::NotAdmitting); + } + endpoint.admitting().cloned() + } + + /// Revokes observer freshness without discarding results or close duties. + pub fn suspend(&self) -> Result<(), EndpointError> { + self.endpoint()? + .observation + .get_or_insert_with(Observation::default) + .suspend(); + self.changed.notify_waiters(); + Ok(()) + } + /// Borrows the endpoint, privately. /// /// Private, and that is only half of the discipline. The other half @@ -2830,7 +2922,9 @@ impl WorkService { /// proposal is wrong. const fn endpoint_refusal(error: EndpointError) -> WorkRefusal { match error { - EndpointError::CatchingUp | EndpointError::NotAdmitting => WorkRefusal::NotReady, + EndpointError::CatchingUp + | EndpointError::NotAdmitting + | EndpointError::ObservationStale => WorkRefusal::NotReady, _ => WorkRefusal::Unavailable, } } @@ -2905,6 +2999,9 @@ pub struct JobProposal { /// Why a client could not complete one acceptance exchange. #[derive(Debug, thiserror::Error)] pub enum ProposeError { + /// The shared client endpoint is unavailable. + #[error(transparent)] + Endpoint(#[from] EndpointError), /// The provider refused. #[error("the provider refused as {refusal}: {reason}")] Refused { @@ -2949,7 +3046,8 @@ pub enum ProposeError { /// It owns one channel's journal and proposes on that channel alone. #[derive(Debug)] pub struct ClientEndpoint { - ready: ReadyChannel, + ready: Option, + observation: Option, store: ChannelStore, signer: Secp256k1Signer, } @@ -2968,7 +3066,19 @@ impl ClientEndpoint { ) -> Result { bind(&ready, &store, &signer, Role::Client)?; Ok(Self { - ready, + ready: Some(ready), + observation: None, + store, + signer, + }) + } + + /// Reopens accounting and close duties without requiring an open channel. + pub fn recover(store: ChannelStore, signer: Secp256k1Signer) -> Result { + bind_store(&store, &signer, Role::Client)?; + Ok(Self { + ready: None, + observation: None, store, signer, }) @@ -3001,8 +3111,12 @@ impl ClientEndpoint { /// outstanding, and the record, readiness, and journal errors the /// proposal itself raises. pub fn propose(&mut self, proposal: &JobProposal) -> Result { + if let Some(observation) = &self.observation { + observation.check()?; + } let (cursor_height, _) = self.state().cursor(); - let policy = self.ready.execution_policy(); + let ready = self.ready.as_ref().ok_or(EndpointError::NotAdmitting)?; + let policy = ready.execution_policy(); for job in self .state() @@ -3011,7 +3125,7 @@ impl ClientEndpoint { { let retained = *job.authorization(); let rebuilt = policy.propose( - self.ready.channel(), + self.state().channel(), &proposal.prepared_input, retained.proposal_nonce, proposal.deadlines, @@ -3031,18 +3145,18 @@ impl ClientEndpoint { // the sequence move backwards or reuse a number. let proposal_nonce = self.state().proposal_nonce_high_water().saturating_add(1); let authorization = policy.propose( - self.ready.channel(), + self.state().channel(), &proposal.prepared_input, proposal_nonce, proposal.deadlines, )?; - policy.check_authorization(self.ready.channel(), &authorization, cursor_height)?; + policy.check_authorization(self.state().channel(), &authorization, cursor_height)?; policy.check_input( - self.ready.channel(), + self.state().channel(), &authorization, &proposal.prepared_input, )?; - self.ready.check_signable( + ready.check_signable( cursor_height, authorization.terminal_deadline, authorization.payment_deadline, @@ -3051,7 +3165,7 @@ impl ClientEndpoint { .prepared_input .encode() .map_err(PaidWorkError::from)?; - let work_id = work_id(self.ready.channel(), &authorization); + let work_id = work_id(self.state().channel(), &authorization); let signature = self.signer.sign(signing_hash(work_id)); self.store.commit( @@ -3160,7 +3274,7 @@ impl ClientEndpoint { return Ok(retained.clone()); } let start = close_start( - self.ready.channel(), + self.state().channel(), Party::Maker, height, self.state().executable_certificate(), @@ -3241,7 +3355,13 @@ impl ClientEndpoint { .ok_or(DeliverError::NoSuchJob)?; bind(ready, &self.store, &self.signer, Role::Client)?; - if ready.execution_policy() != self.ready.execution_policy() { + if ready.execution_policy() + != self + .ready + .as_ref() + .ok_or(EndpointError::NotAdmitting)? + .execution_policy() + { return Err(DeliverError::Policy); } let (cursor_height, _) = self.state().cursor(); @@ -3338,7 +3458,7 @@ impl ClientEndpoint { .job_by_id(work_id) .ok_or(DeliverError::NoSuchJob)?; let signature = self.signer.sign(signing_hash(delivery_request_digest( - self.ready.channel(), + self.state().channel(), work_id, exporter, ))); @@ -3396,7 +3516,7 @@ impl ClientEndpoint { .job_by_id(work_id) .ok_or(DeliverError::NoSuchJob)?; let (result, _) = job.result().ok_or(DeliverError::NoSuchJob)?; - let result_digest = result_digest(self.ready.channel(), result); + let result_digest = result_digest(self.state().channel(), result); self.store.commit( ChannelRecord::JobTerminated { work_id, @@ -3443,6 +3563,9 @@ impl ClientEndpoint { if let Some(retained) = self.state().payment(work_id) { return Ok(admit_request(&retained)); } + if let Some(observation) = &self.observation { + observation.check()?; + } let job = self .state() .job_by_id(work_id) @@ -3453,19 +3576,19 @@ impl ClientEndpoint { let (authorization, result) = (*job.authorization(), *result); let (certificate, binding) = next_payment( - self.ready.channel(), + self.state().channel(), &authorization, &result, self.state().ledger().credited_cumulative(), self.state().settlement(), )?; let binding_signature = self.signer.sign(signing_hash(payment_binding_digest( - self.ready.channel(), + self.state().channel(), &binding, ))); let certificate_signature = self .signer - .sign(certificate.digest(self.ready.channel().network())); + .sign(certificate.digest(self.state().channel().network())); self.store.commit( ChannelRecord::JobTerminated { @@ -3550,7 +3673,7 @@ impl ClientEndpoint { /// raises. pub async fn admit_payment( client: &WorkClientImpl, - endpoint: &mut ClientEndpoint, + endpoint: &mut impl ClientChannel, work_id: Digest, ) -> Result where @@ -3558,9 +3681,9 @@ where T::Error: std::error::Error + Send + Sync + 'static, T::Stream: 'static, { - let request = endpoint.pay(work_id)?; + let request = endpoint.with_client(|endpoint| endpoint.pay(work_id))??; let response = client.admit_certificate(request).await?; - endpoint.acknowledged(work_id, &response) + endpoint.with_client(|endpoint| endpoint.acknowledged(work_id, &response))? } /// Asks for one accepted job's answer over a live transport and makes it @@ -3582,7 +3705,7 @@ where /// raise. pub async fn fetch_result( transport: T, - endpoint: &mut ClientEndpoint, + endpoint: &mut impl ClientChannel, ready: &ReadyChannel, work_id: Digest, ) -> Result @@ -3594,12 +3717,15 @@ where let Some(exporter) = transport.context().open_exporter else { return Err(DeliverError::Unbindable); }; - let request = endpoint.request_delivery(work_id, &exporter)?; + let request = + endpoint.with_client(|endpoint| endpoint.request_delivery(work_id, &exporter))??; let response = WorkClientImpl::new(transport) .deliver_result(request) .await?; match response.outcome { - Some(DeliverOutcome::Delivered(delivered)) => endpoint.receive(work_id, ready, &delivered), + Some(DeliverOutcome::Delivered(delivered)) => { + endpoint.with_client(|endpoint| endpoint.receive(work_id, ready, &delivered))? + } Some(DeliverOutcome::Refused(refused)) => Err(DeliverError::Refused { refusal: WorkRefusal::from_code(refused.code) .ok_or(DeliverError::Malformed("refusal code"))?, @@ -3623,7 +3749,7 @@ where /// raises. pub async fn propose_work( transport: T, - endpoint: &mut ClientEndpoint, + endpoint: &mut impl ClientChannel, proposal: &JobProposal, ) -> Result where @@ -3631,9 +3757,9 @@ where T::Error: std::error::Error + Send + Sync + 'static, T::Stream: 'static, { - let request = endpoint.propose(proposal)?; + let request = endpoint.with_client(|endpoint| endpoint.propose(proposal))??; let response = WorkClientImpl::new(transport).accept_work(request).await?; - endpoint.accepted(&response) + endpoint.with_client(|endpoint| endpoint.accepted(&response))? } /// Resend a retained proposal over a live transport and journal its answer. @@ -3643,7 +3769,7 @@ where /// or [`ClientEndpoint::accepted`]. pub async fn resume_work_proposal( transport: T, - endpoint: &mut ClientEndpoint, + endpoint: &mut impl ClientChannel, work_id: Digest, ) -> Result where @@ -3651,9 +3777,9 @@ where T::Error: std::error::Error + Send + Sync + 'static, T::Stream: 'static, { - let request = endpoint.resume_proposal(work_id)?; + let request = endpoint.with_client(|endpoint| endpoint.resume_proposal(work_id))??; let response = WorkClientImpl::new(transport).accept_work(request).await?; - endpoint.accepted(&response) + endpoint.with_client(|endpoint| endpoint.accepted(&response))? } // ── Wire shapes ─────────────────────────────────────────────────────── diff --git a/crates/work/src/work/client_channel.rs b/crates/work/src/work/client_channel.rs new file mode 100644 index 00000000..a26e8ead --- /dev/null +++ b/crates/work/src/work/client_channel.rs @@ -0,0 +1,212 @@ +//! Shared client state. Network waits never hold the journal lock. + +use super::*; + +/// Synchronous access to a client journal, used between transport awaits. +pub trait ClientChannel: Send { + /// Runs one journal operation without allowing the borrow to escape. + fn with_client( + &mut self, + step: impl FnOnce(&mut ClientEndpoint) -> R, + ) -> Result; +} + +impl ClientChannel for ClientEndpoint { + fn with_client( + &mut self, + step: impl FnOnce(&mut ClientEndpoint) -> R, + ) -> Result { + Ok(step(self)) + } +} + +/// The single request owner for a client channel. Requests retain exclusive +/// exchange ordering while the observer independently borrows the journal. +#[derive(Debug)] +pub struct ClientService { + observer: ClientObserver, +} + +impl ClientService { + /// Starts sharing an already opened journal with its observer. + #[must_use] + pub fn new(mut endpoint: ClientEndpoint) -> Self { + endpoint.observation = Some(Observation::default()); + Self { + observer: ClientObserver { + endpoint: Arc::new(Mutex::new(endpoint)), + driving: Arc::new(AtomicBool::new(false)), + }, + } + } + + /// Gives the chain task observation authority, never payment authority. + #[must_use] + pub fn observer(&self) -> ClientObserver { + self.observer.clone() + } + + /// Reads local accounting without holding a borrow across an await. + pub fn with_state(&self, read: impl FnOnce(&ChannelState) -> R) -> Result { + self.observer.with_state(read) + } + + /// Checks the observer's current admission decision locally. + pub fn readiness(&self) -> Result { + self.observer.readiness() + } +} + +/// Shares finalized-history and readiness updates with the request owner. +/// Cloning this handle cannot create another payer. Driving remains exclusive. +#[derive(Clone, Debug)] +pub struct ClientObserver { + endpoint: Arc>, + driving: Arc, +} + +impl ClientObserver { + fn endpoint(&self) -> Result, EndpointError> { + self.endpoint.lock().map_err(|_| EndpointError::Poisoned) + } + + /// Reads local state without holding it across an await. + pub fn with_state(&self, read: impl FnOnce(&ChannelState) -> R) -> Result { + Ok(read(self.endpoint()?.state())) + } + + /// Publishes a coherent snapshot only after its history has been applied. + pub fn observe_ready( + &self, + ready: ReadyChannel, + started: ObservationTime, + max_age: std::time::Duration, + ) -> Result<(), EndpointError> { + let mut endpoint = self.endpoint()?; + bind(&ready, &endpoint.store, &endpoint.signer, Role::Client)?; + if ready.check_caught_up(endpoint.state().cursor().0).is_err() { + return Err(EndpointError::CatchingUp); + } + let height = ready.finalized_height(); + endpoint.ready = Some(ready); + endpoint + .observation + .as_mut() + .expect("shared client observation") + .confirm(height, started, max_age); + Ok(()) + } + + /// Returns local readiness; never contacts a validator. + pub fn readiness(&self) -> Result { + let endpoint = self.endpoint()?; + endpoint + .observation + .as_ref() + .expect("shared client observation") + .check()?; + if endpoint.state().is_closing() { + return Err(EndpointError::NotAdmitting); + } + endpoint.ready.clone().ok_or(EndpointError::NotAdmitting) + } + + /// Stops new signatures while the observer reconnects or shuts down. + pub fn suspend(&self) -> Result<(), EndpointError> { + self.endpoint()? + .observation + .as_mut() + .expect("shared client observation") + .suspend(); + Ok(()) + } + + /// Takes exclusive authority to drive finalized history, not requests. + pub fn drive(&self) -> Result, EndpointError> { + self.driving + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| EndpointError::CatchingUp)?; + Ok(ClientDriver { service: self }) + } +} + +impl ClientChannel for ClientService { + fn with_client( + &mut self, + step: impl FnOnce(&mut ClientEndpoint) -> R, + ) -> Result { + Ok(step(&mut *self.observer.endpoint()?)) + } +} + +/// The client's one chain driver. Dropping it releases driving authority. +pub struct ClientDriver<'a> { + service: &'a ClientObserver, +} + +impl Drop for ClientDriver<'_> { + fn drop(&mut self) { + self.service.driving.store(false, Ordering::Release); + } +} + +impl CloseChannel for ClientDriver<'_> { + fn with_store( + &mut self, + step: impl FnOnce(&mut ChannelStore) -> R, + ) -> Result { + let mut endpoint = self.service.endpoint().map_err(|_| CatchUpError::Busy)?; + Ok(step(&mut endpoint.store)) + } +} + +impl ClientDriver<'_> { + /// Applies a verified block under the same lock used by payments. + pub fn observe_finalized(&mut self, block: &FinalizedWork) -> Result<(), CatchUpError> { + self.with_store(|store| observe(store, block, &Secp256k1Verifier::new()))??; + Ok(()) + } + + /// Fetches missing blocks without borrowing the journal across network I/O. + pub async fn catch_up( + &mut self, + source: &S, + ) -> Result { + let Some(latest) = source.latest_height().await? else { + return self.with_store(|store| store.state().cursor().0); + }; + self.catch_up_to(source, latest).await + } + + /// Applies a fixed snapshot's history without querying a moving chain tip. + pub async fn catch_up_to( + &mut self, + source: &S, + latest: u64, + ) -> Result { + let next = self + .with_store(|store| store.state().cursor().0)? + .saturating_add(1); + for height in next..=latest { + let block = source + .block_at(height) + .await? + .ok_or(CatchUpError::Missing { height })?; + self.observe_finalized(&block)?; + } + self.with_store(|store| store.state().cursor().0) + } + + /// Drives a retained close without blocking ordinary state access. + pub async fn advance_close( + &mut self, + source: &S, + sink: &T, + ) -> Result + where + S: FinalizedBlocks + ?Sized, + T: TxSink + ?Sized, + { + advance_close(source, sink, self, &Secp256k1Verifier::new()).await + } +} diff --git a/crates/work/src/work/observation.rs b/crates/work/src/work/observation.rs new file mode 100644 index 00000000..6dab3863 --- /dev/null +++ b/crates/work/src/work/observation.rs @@ -0,0 +1,92 @@ +//! Process-local freshness of a continuously observed channel. + +use super::EndpointError; +use std::time::{Duration, SystemTime}; +use tokio::time::Instant; + +/// Capture before reading the chain. Using both clocks counts host sleep even +/// where monotonic time excludes it, and fails closed on wall-clock rollback. +#[derive(Clone, Copy, Debug)] +pub struct ObservationTime { + monotonic: Instant, + wall: SystemTime, +} + +impl ObservationTime { + /// Marks the beginning of a chain observation. + #[must_use] + pub fn now() -> Self { + Self { + monotonic: Instant::now(), + wall: SystemTime::now(), + } + } + + fn elapsed(self) -> Option { + self.wall + .elapsed() + .ok() + .map(|wall| wall.max(self.monotonic.elapsed())) + } +} + +/// A successful read only renews freshness when finalized history advances. +/// Reading an old tip repeatedly cannot keep a disconnected channel admitting. +#[derive(Debug, Default)] +pub(super) struct Observation { + height: Option, + confirmed: Option<(ObservationTime, Duration)>, +} + +impl Observation { + pub(super) fn confirm(&mut self, height: u64, started: ObservationTime, max_age: Duration) { + if self.height.is_none_or(|previous| height > previous) { + self.height = Some(height); + self.confirmed = Some((started, max_age)); + } + } + + pub(super) fn check(&self) -> Result<(), EndpointError> { + if self + .confirmed + .is_some_and(|(started, max_age)| started.elapsed().is_some_and(|age| age < max_age)) + { + Ok(()) + } else { + Err(EndpointError::ObservationStale) + } + } + + pub(super) fn suspend(&mut self) { + self.confirmed = None; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn host_sleep_and_clock_rollback_cannot_extend_readiness() { + for wall in [ + SystemTime::now() - Duration::from_secs(60), + SystemTime::now() + Duration::from_secs(60), + ] { + let mut observation = Observation::default(); + observation.confirm( + 1, + ObservationTime { + monotonic: Instant::now(), + wall, + }, + Duration::from_secs(5), + ); + assert_eq!(observation.check(), Err(EndpointError::ObservationStale)); + observation.confirm(2, ObservationTime::now(), Duration::from_secs(5)); + assert!( + observation.check().is_ok(), + "new finalized progress restores readiness" + ); + } + } +} diff --git a/crates/work/src/work/stream.rs b/crates/work/src/work/stream.rs index 89012a96..95d44a35 100644 --- a/crates/work/src/work/stream.rs +++ b/crates/work/src/work/stream.rs @@ -48,7 +48,7 @@ pub(super) fn fetch_frames<'a>( } // Batch only events already available. Each bounded frame still passes the -// provider's fresh chain check; each envelope is verified separately by the client. +// provider's local readiness check; the client verifies each envelope separately. fn prefix_batches( mut events: &[OutputEventEnvelope], frame_limit: u32, @@ -126,6 +126,17 @@ impl ProviderEndpoint { } impl WorkService { + fn check_stream_release( + &self, + request: &DeliverResultRequest, + exporter: &[u8; 32], + ) -> Result<(), WireStatus> { + self.endpoint() + .map_err(DeliverError::from) + .and_then(|mut endpoint| endpoint.reserve_stream(request, exporter)) + .map_err(|error| WireStatus::new(hellas_wire::WireCode::Unavailable, error.to_string())) + } + pub(super) fn publish_progress( &self, work_id: Digest, @@ -134,8 +145,9 @@ impl WorkService { let policy = self .endpoint() .map_err(|error| BackendFault::new(error.to_string()))? - .admitting() - .map_err(|error| BackendFault::new(error.to_string()))? + .ready + .as_ref() + .ok_or_else(|| BackendFault::new("channel has no execution policy"))? .execution_policy() .clone(); let limit = match &policy { @@ -213,6 +225,10 @@ impl WorkService { return; }; for frame in fetch_frames(&delivered, tail, frame_limit) { + if let Err(error) = service.check_stream_release(&request, &exporter) { + yield Err(error); + return; + } yield frame.map_err(|error| WireStatus::new(hellas_wire::WireCode::Internal, error.to_string())); } } else { @@ -231,6 +247,10 @@ impl WorkService { progress.get(&work_id).map(|progress| progress.events[position.min(progress.events.len())..].to_vec()).unwrap_or_default() }; for batch in prefix_batches(&pending, frame_limit, fetch) { + if let Err(error) = service.check_stream_release(&request, &exporter) { + yield Err(error); + return; + } match encode_transcript(batch) { Ok(prefix) => { position += batch.len(); @@ -264,7 +284,7 @@ impl WorkService { /// The caller pays only after this returns; it must keep running if its UI drops. pub async fn fetch_result_stream( transport: T, - endpoint: &mut ClientEndpoint, + endpoint: &mut impl ClientChannel, ready: &ReadyChannel, work_id: Digest, mut progress: impl FnMut(&OutputEventEnvelope) -> Result<(), PaidWorkError>, @@ -278,13 +298,16 @@ where .context() .open_exporter .ok_or(DeliverError::Unbindable)?; - let request = endpoint.request_delivery(work_id, &exporter)?; - let job = endpoint - .state() - .job_by_id(work_id) - .ok_or(DeliverError::NoSuchJob)?; - let prepared = PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) - .map_err(PaidWorkError::from)?; + let (request, prepared) = endpoint.with_client(|endpoint| { + let request = endpoint.request_delivery(work_id, &exporter)?; + let job = endpoint + .state() + .job_by_id(work_id) + .ok_or(DeliverError::NoSuchJob)?; + let prepared = PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) + .map_err(PaidWorkError::from)?; + Ok::<_, DeliverError>((request, prepared)) + })??; let input = prepared.input_commitment()?; let (operation, kind, max_tokens, max_events, max_bytes) = match (&prepared, ready.execution_policy()) { @@ -398,7 +421,8 @@ where return Err(DeliverError::Malformed("event after terminal result")); } stream.finish()?; - let result = endpoint.receive(work_id, ready, &delivered)?; + let result = endpoint + .with_client(|endpoint| endpoint.receive(work_id, ready, &delivered))??; for event in &events[streamed.len()..] { if event.event().body().kind() == kind { progress(event)?; @@ -434,16 +458,18 @@ where stream.finish()?; streamed.append(&mut tail); let transcript = encode_transcript(&streamed)?; - return endpoint.receive_inner( - work_id, - ready, - &WorkDelivered { - result: terminal.result, - provider_signature: terminal.provider_signature, - transcript, - }, - true, - ); + return endpoint.with_client(|endpoint| { + endpoint.receive_inner( + work_id, + ready, + &WorkDelivered { + result: terminal.result, + provider_signature: terminal.provider_signature, + transcript, + }, + true, + ) + })?; } Some(work_stream_event::Outcome::Refused(refused)) => { return Err(DeliverError::Refused { diff --git a/crates/work/tests/work_accept.rs b/crates/work/tests/work_accept.rs index 64c2679a..d4a4ffd2 100644 --- a/crates/work/tests/work_accept.rs +++ b/crates/work/tests/work_accept.rs @@ -1987,3 +1987,144 @@ fn the_wire_carries_the_authorization_the_signature_covers() { "the signature on the wire verifies over the work id", ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn client_observer_does_not_hold_the_journal_during_chain_io() { + use hellas_work::work::{ClientChannel as _, ClientService}; + let root = temp(); + let mut service = ClientService::new(client_endpoint(root.path())); + service + .observer() + .observe_ready( + ready(), + hellas_work::work::ObservationTime::now(), + std::time::Duration::from_secs(30), + ) + .unwrap(); + let released = Arc::new(AtomicBool::new(false)); + let source = HeldChain::over(CURSOR + 1..=CURSOR + 1, &released); + let observer = service.observer(); + let held_source = source.clone(); + let driving = + tokio::spawn(async move { observer.drive().unwrap().catch_up(&*held_source).await }); + reaches(&source.fetches, 1).await; + assert!(matches!( + service.observer().drive(), + Err(EndpointError::CatchingUp) + )); + // The provider exchange can sign and journal while the observer waits. + let request = service + .with_client(|endpoint| endpoint.propose(&proposal(1))) + .unwrap() + .unwrap(); + assert!(!request.client_signature.is_empty()); + assert_eq!( + service + .with_state(|state| state.proposal_nonce_high_water()) + .unwrap(), + 1 + ); + released.store(true, Ordering::SeqCst); + assert_eq!(driving.await.unwrap().unwrap(), CURSOR + 1); + assert!(service.observer().drive().is_ok()); +} + +#[tokio::test] +async fn shared_client_can_follow_blocks_while_waiting_for_provider_acceptance() { + use hellas_work::work::ClientService; + let client_root = temp(); + let provider_root = temp(); + let mut client = ClientService::new(client_endpoint(client_root.path())); + client + .observer() + .observe_ready( + ready(), + hellas_work::work::ObservationTime::now(), + std::time::Duration::from_secs(30), + ) + .unwrap(); + let observer = client.observer(); + let (client_transport, server_transport) = transport_pair(); + let request = + tokio::spawn( + async move { propose_work(client_transport, &mut client, &proposal(1)).await }, + ); + // No server is running yet, so the request must wait for its response. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while observer + .with_state(|state| state.proposal_nonce_high_water()) + .unwrap() + == 0 + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + observer + .drive() + .unwrap() + .observe_finalized(&FinalizedWork { + height: CURSOR + 1, + parent: payload_at(CURSOR), + payload: payload_at(CURSOR + 1), + txs: Vec::new(), + }) + .unwrap(); + let serving = serve( + server_transport, + WorkService::new(provider_endpoint(provider_root.path())), + ); + request.await.unwrap().unwrap(); + assert_eq!( + observer.with_state(|state| state.cursor().0).unwrap(), + CURSOR + 1 + ); + serving.abort(); +} + +#[test] +fn managed_observers_start_closed_and_old_tips_cannot_renew_expired_readiness() { + use hellas_work::work::{ClientChannel as _, ClientService}; + let client_root = temp(); + let provider_root = temp(); + let mut client = ClientService::new(client_endpoint(client_root.path())); + let provider = WorkService::new(provider_endpoint(provider_root.path())); + provider.require_observer().unwrap(); + assert!(matches!( + client.readiness(), + Err(EndpointError::ObservationStale) + )); + assert_eq!( + refusal_code(&provider.accept(&signed_request(1, 1))), + WorkRefusalCode::NotReady + ); + for max_age in [ + std::time::Duration::ZERO, + std::time::Duration::from_secs(30), + ] { + let started = hellas_work::work::ObservationTime::now(); + client + .observer() + .observe_ready(ready(), started, max_age) + .unwrap(); + provider.observe_ready(ready(), started, max_age).unwrap(); + assert!(matches!( + client + .with_client(|endpoint| endpoint.propose(&proposal(1))) + .unwrap(), + Err(ProposeError::Endpoint(EndpointError::ObservationStale)) + )); + assert_eq!( + refusal_code(&provider.accept(&signed_request(1, 1))), + WorkRefusalCode::NotReady + ); + } + assert_eq!( + client + .with_state(|state| state.proposal_nonce_high_water()) + .unwrap(), + 0 + ); + assert_eq!(provider.with_state(|state| state.jobs().len()).unwrap(), 0); +} diff --git a/crates/work/tests/work_payment.rs b/crates/work/tests/work_payment.rs index 2fd49fe3..13007859 100644 --- a/crates/work/tests/work_payment.rs +++ b/crates/work/tests/work_payment.rs @@ -1160,3 +1160,157 @@ async fn a_defaulted_job_is_not_paid_for_as_well() { PRICE, ); } + +#[tokio::test] +async fn stale_observation_blocks_new_money_but_preserves_retransmission() { + use hellas_work::work::{ClientChannel as _, ClientService, EndpointError}; + let fixture = checked_job().await; + let mut client = ClientService::new(fixture.client); + let id = fixture.id; + assert!(matches!( + client.with_client(|endpoint| endpoint.pay(id)).unwrap(), + Err(PaymentError::Endpoint(EndpointError::ObservationStale)) + )); + assert_eq!( + client + .with_state(|state| state.ledger().credited_cumulative()) + .unwrap(), + 0 + ); + client + .observer() + .observe_ready( + fixture.ready.clone(), + hellas_work::work::ObservationTime::now(), + std::time::Duration::from_secs(30), + ) + .unwrap(); + let request = client + .with_client(|endpoint| endpoint.pay(id)) + .unwrap() + .unwrap(); + client.observer().suspend().unwrap(); + assert_eq!( + client + .with_client(|endpoint| endpoint.pay(id)) + .unwrap() + .unwrap(), + request + ); + + fixture.service.require_observer().unwrap(); + let refused = admit_response(&fixture.service, request.clone()).await; + assert_eq!(refusal_of(&refused), WorkRefusalCode::NotReady); + assert_nothing_credited(&fixture.service, "observer has not initialized").await; + fixture + .service + .observe_ready( + fixture.ready, + hellas_work::work::ObservationTime::now(), + std::time::Duration::from_secs(30), + ) + .unwrap(); + let accepted = admit_response(&fixture.service, request.clone()).await; + assert!(matches!(accepted.outcome, Some(AdmitOutcome::Paid(_)))); + fixture.service.suspend().unwrap(); + assert_eq!(admit_response(&fixture.service, request).await, accepted); + assert_eq!( + fixture + .service + .with_state(|state| state.ledger().credited_cumulative()) + .unwrap(), + PRICE + ); +} + +#[tokio::test] +async fn client_close_recovery_does_not_require_new_work_readiness() { + let mut fixture = checked_job().await; + fixture.client.pay(fixture.id).unwrap(); + let start = fixture.client.prepare_close().unwrap(); + drop(fixture.client); + let recovered_store = store_at( + fixture.client_root.path(), + &fixture.ready, + Role::Client, + CURSOR, + ); + let mut recovered = ClientEndpoint::recover(recovered_store, client()).unwrap(); + assert!(recovered.state().is_closing()); + assert_eq!(recovered.prepare_close().unwrap(), start); + assert_eq!(recovered.state().max_executable_certificate(), PRICE); +} + +#[tokio::test] +async fn observed_close_and_payment_have_one_journal_order() { + use hellas_work::work::{ClientChannel as _, ClientService, ObservationTime}; + use hellas_work::work_close::close_start; + for payment_first in [false, true] { + let fixture = checked_job().await; + let mut client = ClientService::new(fixture.client); + let observer = client.observer(); + observer + .observe_ready( + fixture.ready.clone(), + ObservationTime::now(), + std::time::Duration::from_secs(30), + ) + .unwrap(); + let start = close_start( + fixture.ready.channel(), + hellas_kernel::Party::Maker, + CURSOR, + None, + &self::client(), + ) + .unwrap(); + let block = FinalizedWork { + height: CURSOR + 1, + parent: payload_at(CURSOR), + payload: payload_at(CURSOR + 1), + txs: vec![hellas_kernel::Tx::move_action( + hellas_kernel::Move::StartPaymentClose(start), + )], + }; + if payment_first { + let request = client + .with_client(|endpoint| endpoint.pay(fixture.id)) + .unwrap() + .unwrap(); + let credited = admit_response(&fixture.service, request).await; + assert!(matches!(credited.outcome, Some(AdmitOutcome::Paid(_)))); + } + observer.drive().unwrap().observe_finalized(&block).unwrap(); + fixture + .service + .drive() + .unwrap() + .observe_finalized(&block) + .unwrap(); + assert!(observer.with_state(|state| state.is_closing()).unwrap()); + let retried = client + .with_client(|endpoint| endpoint.pay(fixture.id)) + .unwrap(); + if payment_first { + // Retrying the durable certificate is allowed after close observation. + let response = admit_response(&fixture.service, retried.unwrap()).await; + assert!(matches!(response.outcome, Some(AdmitOutcome::Paid(_)))); + } else { + assert!(matches!(retried, Err(PaymentError::NoSuchJob))); + } + let expected = if payment_first { PRICE } else { 0 }; + assert_eq!( + client + .with_state(|state| state.max_executable_certificate()) + .unwrap(), + expected + ); + assert_eq!( + fixture + .service + .with_state(|state| state.max_executable_certificate()) + .unwrap(), + expected + ); + } +} diff --git a/docs/paid-gateway.md b/docs/paid-gateway.md index 8124a89a..14bd9110 100644 --- a/docs/paid-gateway.md +++ b/docs/paid-gateway.md @@ -63,9 +63,36 @@ The provider reserves delivery credit before streaming signed token prefixes. The gateway verifies each prefix as it arrives, then authenticates and journals the complete result and obtains the provider's durable payment acknowledgement before reporting completion. One open subscription replaces repeated result -polls. Payment is an accumulating channel certificate. Channels stay open and -follow finalized blocks while idle; they settle through the existing close -protocol. The pool does not automatically refill exhausted channels. +polls. Payment is an accumulating channel certificate. An established, healthy +channel performs no validator RPCs on its request or streaming paths. Requests +use local credit checks, signatures and journal commits. Channels settle through +the existing close protocol; the pool does not automatically refill them. + +Each channel observes finalized history independently of its requests. The client +has one request owner and a separate observer handle, so a slow response cannot +block close monitoring or reorder payment exchanges. The provider also drives +channels independently: one stalled channel cannot block another's recovery. +Applying a close and crediting a certificate use the same short journal lock. +Jobs within a channel still finish in order, including collection and payment +after an HTTP client disconnects. A server that keeps an HTTP response open after +an application completion event therefore holds that channel until EOF. The +generic HTTP route preserves those trailing bytes. Separate funded channels +execute independently; `paid.queue` measures time waiting for a channel. + +The work configuration accepts `max_observation_age_ms` (default 5000), which +must exceed `poll_ms`. Admission, delivery and new certificates stop when the +observer fails or no new verified finalized height arrives within that age. +Repeated reads of an old tip do not renew it. The bound starts before the chain +read, not after it finishes, and readiness is published only after all intervening +blocks have been applied. Results may still be retained during an observation +outage; existing payment certificates remain safe to retransmit. + +This is an online channel: its observer must detect unilateral closes and submit +any response within `omit_response_blocks`. Configure observation age, network +recovery and submission latency to fit that window at the deployment's block +rate. The local age limit cannot guarantee recovery through a partition longer +than the on-chain response window. Setup and restart catch-up may wait on +validators; ordinary requests do not perform that work. The shared `--output-cache record` mode also covers paid inference. Repeating an identical recorded request reuses its output without another paid job. A @@ -126,7 +153,7 @@ Hellas RPC metadata carries the context over both Iroh and WebSocket mux connections, including validator queries and submissions. Transparent frame relays preserve this metadata. The explorer relay continues the context through its forwarding span, and proof-origin HTTP queries continue the caller trace. -Background indexer polling is independent of the paid request trace. +Background chain observation uses separate `paid.channel.observe` traces. Useful spans include `http.server`, `paid.gateway`, `paid.queue`, `paid.executor.stream`, GenAI `chat MODEL` and `text_completion` operations, From 048637c8628bb93722211ae0c344f56c57f78fdf Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Fri, 25 Sep 2026 02:59:55 +0200 Subject: [PATCH 15/32] Clarify local readiness checks in HTTP Fetch documentation --- docs/http-gateway.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/http-gateway.md b/docs/http-gateway.md index 3e490716..00a522f7 100644 --- a/docs/http-gateway.md +++ b/docs/http-gateway.md @@ -66,7 +66,7 @@ Signed response prefixes stream as they arrive. Successful HTTP completion follows verification of the complete result and the provider's durable payment acknowledgement. Each accepted valid terminal costs the channel's agreed fixed price, including HTTP error statuses. Already available prefixes share bounded -wire frames, with a fresh chain-state check before each frame and individual +wire frames, with a local observer-readiness check before each frame and individual signature checks for every event. Batching does not wait for more upstream data. Requests are not retried on another provider after account selection. The pool serializes work within each provider From f77be465d25bc7ec7175870be6293fd94efa8a0b Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Fri, 25 Sep 2026 03:52:00 +0200 Subject: [PATCH 16/32] Fix CI checks and preserve early RPC refusals Keep authoritative peer error trailers when an early rejection closes the request writer. Bound recovery and cover both unary and server-streaming calls. Refresh the Work and StreamResult wire-ID pins for the existing Fetch terminal schema, document coordinated upgrades, and apply dependency ordering and TOML formatting checks. --- crates/chain/src/server/tests.rs | 2 +- crates/cli/Cargo.toml | 2 +- crates/executor/Cargo.toml | 2 +- crates/gateway/Cargo.toml | 4 +- crates/providers/Cargo.toml | 27 ++++-- crates/rpc/Cargo.toml | 2 +- crates/rpc/src/call.rs | 161 +++++++++++++++++++++++-------- crates/rpc/src/pb/mod.rs | 7 +- crates/sdk/Cargo.toml | 18 ++-- docs/paid-gateway.md | 4 + 10 files changed, 164 insertions(+), 65 deletions(-) diff --git a/crates/chain/src/server/tests.rs b/crates/chain/src/server/tests.rs index 93df2714..70bea065 100644 --- a/crates/chain/src/server/tests.rs +++ b/crates/chain/src/server/tests.rs @@ -384,7 +384,7 @@ async fn bound_and_relay_transports_answer_from_one_node_state() { .await .expect("the second transport receives a prompt saturation result") .expect_err("all sixteen node response permits are held by the first transport"); - assert_eq!(overflow.code(), WireCode::ResourceExhausted); + assert_eq!(overflow.code(), WireCode::ResourceExhausted, "{overflow:?}"); blocker.release.add_permits(16); for call in held { diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index d049bf07..6aab2611 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -98,12 +98,12 @@ hellas-kernel = { workspace = true, default-features = false, optional = true } hellas-presentation = { workspace = true, optional = true } hellas-private.workspace = true hellas-providers.workspace = true -hellas-sdk = { workspace = true, default-features = false, optional = true } hellas-rpc = { workspace = true, default-features = false, features = [ "all-protocols", "work", "host-control", ] } +hellas-sdk = { workspace = true, default-features = false, optional = true } hellas-store = { workspace = true } hellas-wire = { workspace = true, features = [ "unix", diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml index 8c7a2b0b..3dbf533e 100644 --- a/crates/executor/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -49,6 +49,6 @@ uuid = { version = "1", features = ["v4"] } zeroize = "1" [dev-dependencies] -tokio = { workspace = true, features = ["test-util"] } opentelemetry_sdk = { workspace = true, features = ["testing"] } +tokio = { workspace = true, features = ["test-util"] } tracing-subscriber = { workspace = true } diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index bf90c251..3df26aec 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -27,9 +27,7 @@ anyhow = { workspace = true } async-stream = { workspace = true } axum = { workspace = true } base64.workspace = true -httpdate = "1" flate2 = "1" -zstd = "0.13" futures = { workspace = true } hellas-adaptors.workspace = true hellas-client = { workspace = true, features = ["iroh", "evaluate"] } @@ -39,6 +37,7 @@ hellas-private.workspace = true hellas-rpc = { workspace = true, features = ["courtesy", "fetch"] } hellas-store = { workspace = true, optional = true } hellas-wire = { workspace = true, features = ["iroh"] } +httpdate = "1" iroh.workspace = true libc = "0.2" opentelemetry = { workspace = true, optional = true } @@ -53,6 +52,7 @@ tokio.workspace = true tower = { version = "0.5", default-features = false, features = ["util"] } tracing.workspace = true tracing-opentelemetry = { workspace = true, optional = true } +zstd = "0.13" [dev-dependencies] hellas-rpc = { workspace = true, features = ["host-control"] } diff --git a/crates/providers/Cargo.toml b/crates/providers/Cargo.toml index d2da68d2..7194de1b 100644 --- a/crates/providers/Cargo.toml +++ b/crates/providers/Cargo.toml @@ -12,33 +12,40 @@ default = [] otel = ["hellas-rpc/otel", "dep:hellas-wire"] [dependencies] -base64.workspace = true -url = "2" -rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } -webpki-roots = "1" -sha2.workspace = true -x509-cert.workspace = true anyhow = { workspace = true } async-stream = { workspace = true } +base64.workspace = true futures.workspace = true hellas-adaptors.workspace = true hellas-executor = { workspace = true, default-features = false } -hellas-rpc = { workspace = true, features = ["fetch"] } hellas-private.workspace = true +hellas-rpc = { workspace = true, features = ["fetch"] } hellas-wire = { workspace = true, optional = true } reqwest.workspace = true +rustls = { version = "0.23", default-features = false, features = [ + "ring", + "std", + "tls12", +] } serde.workspace = true serde_json.workspace = true +sha2.workspace = true tokio.workspace = true tracing.workspace = true +url = "2" +webpki-roots = "1" +x509-cert.workspace = true [dev-dependencies] -tempfile.workspace = true -rcgen = "0.14" -tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } axum.workspace = true opentelemetry.workspace = true opentelemetry_sdk = { workspace = true, features = ["testing"] } +rcgen = "0.14" +tempfile.workspace = true tokio = { workspace = true, features = ["test-util"] } +tokio-rustls = { version = "0.26", default-features = false, features = [ + "ring", + "tls12", +] } tracing-opentelemetry.workspace = true tracing-subscriber.workspace = true diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 29f3f470..0902959b 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -55,7 +55,6 @@ work = ["dep:hellas-kernel", "evaluate", "fetch", "hellas-kernel/secp256k1"] axum = { workspace = true, optional = true } base64.workspace = true base64ct = { version = "1", optional = true, features = ["alloc"] } -url = "2" bytes = { workspace = true } ed25519-dalek = { workspace = true } futures-core = { workspace = true } @@ -79,6 +78,7 @@ thiserror = { workspace = true } # sample, and a node with no subscriber pays one atomic load per seam. tracing = { workspace = true } tracing-opentelemetry = { workspace = true, optional = true } +url = "2" web-time = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/crates/rpc/src/call.rs b/crates/rpc/src/call.rs index 7ebb0de9..21b8be6e 100644 --- a/crates/rpc/src/call.rs +++ b/crates/rpc/src/call.rs @@ -67,19 +67,8 @@ where .open(M::METHOD_ID, headers) .await .map_err(transport_to_status)?; - let (mut send, recv) = WireStream::split(stream); - let mut recv = Box::pin(recv); - - let mut buf = BytesMut::with_capacity(request.encoded_len()); - request - .encode(&mut buf) - .map_err(|e| WireStatus::internal(format!("prost encode: {e}")))?; - send.send_body(buf.freeze()) - .await - .map_err(|e| WireStatus::internal(format!("send: {e}")))?; - send.close_send(None) - .await - .map_err(|e| WireStatus::internal(format!("close_send: {e}")))?; + let (mut send, mut recv) = WireStream::split(stream); + send_request(&mut send, &mut recv, request).await?; // Unary protocol shape: exactly one body chunk, then EOF, then a // terminal trailer. Anything else is a server-side bug and must @@ -147,6 +136,42 @@ fn trailer_to_status(t: &Trailer) -> WireStatus { } } +/// A server may reject at Open, before accepting the request body. Preserve +/// its refusal when stopping the upload also makes a local write fail. +async fn send_request( + send: &mut S, + recv: &mut R, + request: Q, +) -> Result<(), WireStatus> { + let mut buf = BytesMut::with_capacity(request.encoded_len()); + request + .encode(&mut buf) + .map_err(|e| WireStatus::internal(format!("prost encode: {e}")))?; + let written = async { + send.send_body(buf.freeze()) + .await + .map_err(|e| WireStatus::internal(format!("send: {e}")))?; + send.close_send(None) + .await + .map_err(|e| WireStatus::internal(format!("close_send: {e}"))) + } + .await; + if let Err(error) = written { + // Only a terminal error can explain an early rejection. Do not drain + // arbitrary bodies or wait indefinitely for a peer that stopped reading. + if matches!( + n0_future::time::timeout(UNARY_RECEIVE_TIMEOUT, recv.next()).await, + Ok(None) + ) && let Some(trailer) = recv.trailer() + && trailer.status != WireCode::Ok + { + return Err(trailer_to_status(trailer)); + } + return Err(error); + } + Ok(()) +} + /// Server-streaming call: send one request, receive a stream of responses /// + a terminal trailer. /// @@ -181,18 +206,8 @@ where .open(M::METHOD_ID, headers) .await .map_err(transport_to_status)?; - let (mut send, recv) = WireStream::split(stream); - - let mut buf = BytesMut::with_capacity(request.encoded_len()); - request - .encode(&mut buf) - .map_err(|e| WireStatus::internal(format!("prost encode: {e}")))?; - send.send_body(buf.freeze()) - .await - .map_err(|e| WireStatus::internal(format!("send: {e}")))?; - send.close_send(None) - .await - .map_err(|e| WireStatus::internal(format!("close_send: {e}")))?; + let (mut send, mut recv) = WireStream::split(stream); + send_request(&mut send, &mut recv, request).await?; Ok(StreamingCall::new(recv, operation)) }, @@ -1492,6 +1507,21 @@ mod streaming_call_tests { struct MockTransport; + struct ReplyTransport(Mutex>); + + impl StreamTransport for ReplyTransport { + type Stream = MockWireStream; + type Error = std::io::Error; + + async fn open(&self, _: u32, _: Metadata) -> Result { + Ok(self.0.lock().unwrap().take().unwrap()) + } + + async fn accept(&self) -> Result>, Self::Error> { + Ok(None) + } + } + impl StreamTransport for MockTransport { type Stream = MockWireStream; type Error = std::io::Error; @@ -1538,6 +1568,74 @@ mod streaming_call_tests { ) } + #[tokio::test] + async fn early_refusal_survives_request_body_and_close_write_failures() { + for streaming in [false, true] { + for fail_close in [false, true] { + let (mut inbound, _) = route_inbound(None, false, None); + inbound.stream.send.state.fail_send = !fail_close; + inbound.stream.send.state.fail_close = fail_close; + let mut trailer = Trailer::from_status(WireCode::ResourceExhausted, "route full"); + trailer.metadata.insert_text("retry-after", "1"); + inbound.stream.recv.trailer = trailer; + let transport = ReplyTransport(Mutex::new(Some(inbound.stream))); + let request = BytesMsg::default(); + let result = if streaming { + server_streaming::<_, MockMethod>(&transport, request, Metadata::new()) + .await + .map(|_| ()) + } else { + unary::<_, MockMethod>(&transport, request, Metadata::new()) + .await + .map(|_| ()) + }; + let error = result.expect_err("early rejection remains a wire refusal"); + assert_eq!(error.code, WireCode::ResourceExhausted); + assert_eq!(error.message, "route full"); + assert_eq!( + error.metadata.get("retry-after").unwrap().as_text(), + Some("1") + ); + } + } + } + + #[tokio::test(start_paused = true)] + async fn failed_upload_does_not_accept_success_or_drain_a_stalled_response() { + for streaming in [false, true] { + for (body, pending) in [ + (None, false), + (Some(Bytes::from_static(b"unsolicited body")), false), + (None, true), + ] { + let (mut inbound, _) = route_inbound(body, pending, None); + inbound.stream.send.state.fail_send = true; + let transport = ReplyTransport(Mutex::new(Some(inbound.stream))); + let request = BytesMsg::default(); + let result = async { + if streaming { + server_streaming::<_, MockMethod>(&transport, request, Metadata::new()) + .await + .map(|_| ()) + } else { + unary::<_, MockMethod>(&transport, request, Metadata::new()) + .await + .map(|_| ()) + } + }; + let error = tokio::time::timeout( + UNARY_RECEIVE_TIMEOUT + std::time::Duration::from_secs(1), + result, + ) + .await + .expect("recovering an early refusal has a deadline") + .expect_err("a failed upload cannot become a successful call"); + assert_eq!(error.code, WireCode::Internal); + assert_eq!(error.message, "send: send failed"); + } + } + } + #[cfg(feature = "otel")] #[tokio::test] async fn telemetry_records_transport_protocol_and_cancellation_failures() { @@ -1546,19 +1644,6 @@ mod streaming_call_tests { use tracing::instrument::WithSubscriber; use tracing_subscriber::prelude::*; - struct ReplyTransport(Mutex>); - impl StreamTransport for ReplyTransport { - type Stream = MockWireStream; - type Error = std::io::Error; - async fn open(&self, _: u32, _: Metadata) -> Result { - Ok(self.0.lock().unwrap().take().unwrap()) - } - async fn accept( - &self, - ) -> Result>, Self::Error> { - Ok(None) - } - } let exporter = InMemorySpanExporter::default(); let provider = SdkTracerProvider::builder() .with_simple_exporter(exporter.clone()) diff --git a/crates/rpc/src/pb/mod.rs b/crates/rpc/src/pb/mod.rs index 6f692c77..083cb722 100644 --- a/crates/rpc/src/pb/mod.rs +++ b/crates/rpc/src/pb/mod.rs @@ -190,11 +190,14 @@ mod id_pins { use super::services::work::{ AcceptWork, AdmitCertificate, DeliverResult, Open, StreamResult, Work, }; - assert_eq!(::SERVICE_ID, 0x3b672fa8); + // Fetch's separate terminal frame extends WorkStreamEvent, deliberately + // changing StreamResult and its enclosing service. Other method IDs stay + // pinned: gateways and providers must upgrade together for this schema. + assert_eq!(::SERVICE_ID, 0x22b5_f1f8); assert_eq!(::METHOD_ID, 0x93cb0b39); assert_eq!(::METHOD_ID, 0xe6a7_13c2); assert_eq!(::METHOD_ID, 0xf15a_a80e); - assert_eq!(::METHOD_ID, 0xf9e8_388a); + assert_eq!(::METHOD_ID, 0x76bf_afcf); assert_eq!(::METHOD_ID, 0x0ffb_b4f9); } diff --git a/crates/sdk/Cargo.toml b/crates/sdk/Cargo.toml index 486ff328..de4048e8 100644 --- a/crates/sdk/Cargo.toml +++ b/crates/sdk/Cargo.toml @@ -49,29 +49,29 @@ paid-work = [ ] [dependencies] -thiserror.workspace = true +anyhow = { workspace = true, optional = true } async-stream = { workspace = true, optional = true } -hellas-chain = { workspace = true, default-features = false, optional = true } -hellas-kernel = { workspace = true, default-features = false, optional = true } -hellas-work = { workspace = true, optional = true } futures = { workspace = true, optional = true } -serde = { workspace = true, optional = true } -serde_json = { workspace = true, optional = true } -hex = { workspace = true, optional = true } -url = { version = "2", optional = true } -anyhow = { workspace = true, optional = true } futures-core = { workspace = true, optional = true } hellas-attestation.workspace = true +hellas-chain = { workspace = true, default-features = false, optional = true } hellas-client = { workspace = true, optional = true } hellas-executor = { workspace = true, default-features = false, optional = true } hellas-gateway = { workspace = true, optional = true } +hellas-kernel = { workspace = true, default-features = false, optional = true } hellas-providers = { workspace = true, optional = true } hellas-rpc = { workspace = true, features = ["execute", "fetch", "courtesy"] } hellas-wire = { workspace = true, features = ["mux"] } +hellas-work = { workspace = true, optional = true } +hex = { workspace = true, optional = true } iroh = { workspace = true, features = ["tls-ring"], optional = true } +serde = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } tempfile = { workspace = true } +thiserror.workspace = true tokio = { workspace = true, features = ["net", "sync"], optional = true } tracing = { workspace = true, optional = true } +url = { version = "2", optional = true } [dev-dependencies] tokio.workspace = true diff --git a/docs/paid-gateway.md b/docs/paid-gateway.md index 14bd9110..2fa38ebb 100644 --- a/docs/paid-gateway.md +++ b/docs/paid-gateway.md @@ -11,6 +11,10 @@ config must select the HTTP Fetch manifest and a matching Fetch route policy; see [HTTP routing](http-gateway.md). The HTTP router chooses the provider and account, so this path never falls back to a different provider or Courtesy. +Paid Fetch extends the Work/WorkSetup service descriptors and the StreamResult +response schema. Upgrade gateways and providers together: older peers reject +the changed wire IDs. The on-chain payment certificate format is unchanged. + The pool file uses the provider's existing `--work-config` policy and chain configuration: From d8e5b8017f3643bd57fd0bdc2580791e47759e4d Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Fri, 25 Sep 2026 04:01:39 +0200 Subject: [PATCH 17/32] Exercise the production paid input reader in CLI tests --- crates/cli/src/commands/paid_work.rs | 7 ------- crates/cli/src/commands/paid_work/tests.rs | 4 +++- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/crates/cli/src/commands/paid_work.rs b/crates/cli/src/commands/paid_work.rs index 220ecd59..ce45afad 100644 --- a/crates/cli/src/commands/paid_work.rs +++ b/crates/cli/src/commands/paid_work.rs @@ -535,13 +535,6 @@ fn read_prepared_work_input(path: &Path) -> CliResult { .map_err(|error| anyhow::anyhow!("invalid prepared paid input {}: {error}", path.display())) } -#[cfg(test)] -fn read_prepared_input(path: &Path) -> CliResult { - let bytes = super::read_bounded_regular_file(path, "prepared paid input", MAX_RECORD_BYTES)?; - PreparedPaidInputV1::decode(&bytes, MAX_RECORD_BYTES) - .map_err(|error| anyhow::anyhow!("invalid prepared paid input {}: {error}", path.display())) -} - fn write_private(path: &Path, bytes: &[u8]) -> CliResult<()> { if let Some(parent) = path .parent() diff --git a/crates/cli/src/commands/paid_work/tests.rs b/crates/cli/src/commands/paid_work/tests.rs index ff4bbc27..dc281a73 100644 --- a/crates/cli/src/commands/paid_work/tests.rs +++ b/crates/cli/src/commands/paid_work/tests.rs @@ -131,7 +131,9 @@ fn prepare_input_builds_a_bundle_from_an_environment_and_prompt() { ) .unwrap(); - let prepared = read_prepared_input(&output).unwrap(); + let PreparedPaidWorkInput::Evaluate(prepared) = read_prepared_work_input(&output).unwrap() else { + panic!("an Evaluate bundle"); + }; let parts = prepared.parts().unwrap(); assert_eq!( parts From e654802adbffa80c6f38e24b26d77098c2894f71 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Fri, 25 Sep 2026 04:09:45 +0200 Subject: [PATCH 18/32] Format the paid input regression test --- crates/cli/src/commands/paid_work/tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/cli/src/commands/paid_work/tests.rs b/crates/cli/src/commands/paid_work/tests.rs index dc281a73..42cae70b 100644 --- a/crates/cli/src/commands/paid_work/tests.rs +++ b/crates/cli/src/commands/paid_work/tests.rs @@ -131,7 +131,8 @@ fn prepare_input_builds_a_bundle_from_an_environment_and_prompt() { ) .unwrap(); - let PreparedPaidWorkInput::Evaluate(prepared) = read_prepared_work_input(&output).unwrap() else { + let PreparedPaidWorkInput::Evaluate(prepared) = read_prepared_work_input(&output).unwrap() + else { panic!("an Evaluate bundle"); }; let parts = prepared.parts().unwrap(); From 1ebd758ead953a6af5c8c40100ea31599ad96034 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Sat, 26 Sep 2026 20:54:50 +0200 Subject: [PATCH 19/32] Keep provider dispatch open for the life of a paid job The rewritten serve loop wrapped every RPC dispatch in a 30-second wall-clock timeout and dropped the failure logging the loop used to have. StreamResult drives the whole response stream inside dispatch and legitimately produces no frame for minutes while a job executes, so any paid job slower than 30 seconds had its stream cancelled mid-delivery and the connection torn down; the client refuses to retry once a prefix was emitted. Streaming methods already enforce their own application-level deadlines, so serve goes back to waiting without a total deadline and to logging dispatch and transport failures. --- crates/sdk/src/provider.rs | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/crates/sdk/src/provider.rs b/crates/sdk/src/provider.rs index 16b32312..a15afae7 100644 --- a/crates/sdk/src/provider.rs +++ b/crates/sdk/src/provider.rs @@ -16,7 +16,7 @@ use hellas_rpc::services::fetch::{Fetch, Open as FetchOpen}; use hellas_rpc::{ Assurance, OPEN_NONCE_LEN, ProviderEnrollmentBundle, PublicKey, RootProof, open_proof_binding, }; -use hellas_wire::iroh::IrohTransport; +use hellas_wire::iroh::{IrohTransport, IrohTransportError}; use hellas_wire::{ Dispatcher, ServiceMarker, StreamTransport, TransportContext, WireCode, WireStatus, }; @@ -398,28 +398,29 @@ where }) } +/// Serves one connection until the peer leaves or a dispatch fails. Dispatch +/// is deliberately unbounded here: streaming methods such as `StreamResult` +/// stay open for the lifetime of a job and enforce their own application-level +/// deadlines, so a wall-clock timeout at this layer would cancel paid work +/// mid-delivery. async fn serve(transport: Arc, server: S) where S: Dispatcher + Send + Sync + 'static, - S::Error: Send + Sync + 'static, + S::Error: std::fmt::Display + Send + Sync + 'static, { loop { - let incoming = - tokio::time::timeout(std::time::Duration::from_secs(120), transport.accept()).await; - match incoming { - Ok(Ok(Some(inbound))) => { - if !matches!( - tokio::time::timeout( - std::time::Duration::from_secs(30), - server.dispatch(inbound) - ) - .await, - Ok(Ok(())) - ) { + match transport.accept().await { + Ok(Some(inbound)) => { + if let Err(error) = Dispatcher::::dispatch(&server, inbound).await { + tracing::warn!(%error, "provider RPC failed"); break; } } - _ => break, + Ok(None) | Err(IrohTransportError::Connection(_)) => break, + Err(error) => { + tracing::warn!(%error, "provider transport failed"); + break; + } } } } From 08ab36f5ddb942e215de3ea77b0196481cac9394 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Sat, 26 Sep 2026 20:55:04 +0200 Subject: [PATCH 20/32] Validate paid-work configuration before executor side effects Two validation gaps let bad configuration past startup checks. The retained-transcript-capacity and route cross-checks ran only after Executor::spawn_configured had already created quota and transcript stores, so a config error aborted startup with the side effects committed; they now run before the spawn, and the Fetch-policy check the match already performs is no longer repeated. And an empty validator list reached connect_chain, which reduced a candidate index modulo zero inside the clock task: WorkRunner::discover now rejects it as configuration, the rotation counter is per runner rather than a process-global static, and ConsensusVerifier is built from the normalised network id the channel binds. --- crates/sdk/src/paid_provider.rs | 21 +++++++++++++++------ crates/sdk/src/provider.rs | 17 +++++------------ crates/sdk/src/work_config.rs | 4 +++- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/crates/sdk/src/paid_provider.rs b/crates/sdk/src/paid_provider.rs index 77445ee6..6ac2cb69 100644 --- a/crates/sdk/src/paid_provider.rs +++ b/crates/sdk/src/paid_provider.rs @@ -48,6 +48,8 @@ pub enum PaidProviderError { NoBackend, #[error("observer age must exceed a positive polling interval")] InvalidObservationPolicy, + #[error("the paid-work clock requires at least one validator")] + NoValidators, #[error(transparent)] Endpoint(#[from] hellas_work::work::EndpointError), #[error(transparent)] @@ -707,6 +709,9 @@ impl WorkRunner { if config.poll.is_zero() || config.max_observation_age <= config.poll { return Err(PaidProviderError::InvalidObservationPolicy); } + if config.validators.is_empty() { + return Err(PaidProviderError::NoValidators); + } let consensus_verifier = ConsensusVerifier::new(&ConsensusInfo { validators: config.validators.clone(), threshold_identity: config.threshold_identity, @@ -884,24 +889,28 @@ impl WorkRunner { pub async fn run(self, stop: oneshot::Receiver<()>) { let validators = self.validators.clone(); let verifier = self.consensus_verifier.clone(); + let next = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); self.run_over(stop, move || { let validators = validators.clone(); let verifier = verifier.clone(); - async move { connect_chain(&validators, verifier).await } + let next = next.clone(); + async move { connect_chain(&validators, verifier, &next).await } }) .await; } } -/// Rotates the first candidate on reconnect. Chain reads and submissions use -/// the selected verified connection. +/// Rotates the first candidate on reconnect, per runner. Chain reads and +/// submissions use the selected verified connection. async fn connect_chain( validators: &[String], verifier: ConsensusVerifier, + next: &std::sync::atomic::AtomicUsize, ) -> Option { - static NEXT_VALIDATOR: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); - let start = - NEXT_VALIDATOR.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % validators.len(); + if validators.is_empty() { + return None; + } + let start = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % validators.len(); for url in validators.iter().cycle().skip(start).take(validators.len()) { match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { Ok(client) => { diff --git a/crates/sdk/src/provider.rs b/crates/sdk/src/provider.rs index a15afae7..9dc5f338 100644 --- a/crates/sdk/src/provider.rs +++ b/crates/sdk/src/provider.rs @@ -193,6 +193,11 @@ where } _ => anyhow::bail!("Fetch provider requires a paid Fetch policy"), } + anyhow::ensure!( + options.retained_transcript_capacity == 0, + "paid Fetch provider requires zero retained transcript capacity" + ); + crate::work_config::validate_work_routes(config)?; } let producer_key = Arc::new(options.identity.caller_key().clone()); let access = FetchAccessPolicy::trusted_callers(options.allowed_callers).with_store( @@ -220,18 +225,6 @@ where let setup_mount = crate::paid_provider::MountedSetup::default(); #[cfg(feature = "paid-work")] let work = if let Some(config) = options.paid_work { - anyhow::ensure!( - matches!( - config.execution_policy, - hellas_rpc::protocol::work_profile::PaidWorkPolicy::Fetch { .. } - ), - "Fetch provider requires a paid Fetch policy" - ); - anyhow::ensure!( - options.retained_transcript_capacity == 0, - "paid Fetch provider requires zero retained transcript capacity" - ); - crate::work_config::validate_work_routes(&config)?; let policy = config.provider_policy(); let settlement_key = hellas_kernel::Secp256k1Signer::from_secret_scalar( options.identity.caller_secret_bytes(), diff --git a/crates/sdk/src/work_config.rs b/crates/sdk/src/work_config.rs index 75e12330..85768e35 100644 --- a/crates/sdk/src/work_config.rs +++ b/crates/sdk/src/work_config.rs @@ -53,6 +53,8 @@ pub struct WorkConfig { /// How often the watcher asks the chain for the next block. pub poll: Duration, /// Maximum time without new verified finalized progress before admission stops. + /// Renewal requires an advancing finalized height, so this must also fit + /// the deployment's block interval, not only the polling interval. pub max_observation_age: Duration, /// The payment edge's value, reserve, and close fees as this provider /// requires a client to fund them. @@ -248,7 +250,7 @@ impl WorkConfigFile { hellas_chain::ConsensusVerifier::new(&hellas_chain::light_client::ConsensusInfo { validators: validators.clone(), threshold_identity: threshold_identity.clone(), - network_id: self.chain.network_id.clone(), + network_id: network.as_str().to_owned(), })?; let journal_root = self.journal.into_root()?; From 89003707da5e829a0c762af6df13c79b6b00ea8d Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Sat, 26 Sep 2026 20:55:26 +0200 Subject: [PATCH 21/32] Settle delivered Fetch results during journal recovery Client Fetch journals are metadata-only, so after a restart every open Fetch job has an empty prepared input, and the recovery scan aborted with MissingPayload for all of them. For a job whose result was delivered, verified and committed before the crash that wedged two ways: the provider, which did the work, was never paid even though the retained result and authorization are everything signing payment needs, and needs_recovery stayed set so every later request failed again until the payment deadline filtered the job. Recovery now pays delivered jobs straight from their retained evidence, logs and skips jobs that can neither execute nor settle, and no longer aborts the whole pass on one unrecoverable job. Adjacent recovery fixes: a stream recovery with no progress callback marks nothing emitted, so retryable delivery failures actually retry instead of replaying prefixes into a user stream that does not exist; the unary collect loop checks the payment window between retries like the stream path does; declining recovery no longer disarms the flag; settle documents its missing deadline; the dead MissingPayload and NoValidators error variants go away. --- crates/sdk/src/paid_client.rs | 62 ++++++++++++++++++++++++----- crates/sdk/src/paid_client/error.rs | 9 ----- 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/crates/sdk/src/paid_client.rs b/crates/sdk/src/paid_client.rs index 0e123862..d0214bea 100644 --- a/crates/sdk/src/paid_client.rs +++ b/crates/sdk/src/paid_client.rs @@ -453,6 +453,9 @@ impl PaidWorkSession { } /// Opens a close; the independent observer drives its finalized payout. + /// There is no deadline here: with the observer stopped or validators + /// unreachable this waits as long as the process runs. Callers that need + /// a bound should wrap it in one. pub async fn settle(&mut self) -> Result { if !self.client.with_state(|state| state.is_closing())? { self.client.with_client(|client| client.prepare_close())??; @@ -523,6 +526,21 @@ impl PaidWorkSession { } let ready = client.readiness()?; if recover && *needs_recovery { + enum RecoveredJob { + /// The result was delivered, verified and committed before the + /// restart; only its payment is outstanding, and settling it + /// needs none of the payload a Fetch journal strips. + Delivered(hellas_rpc::protocol::Digest), + /// No payload and no result: the job can neither execute nor + /// settle. Its evidence is retained until the payment deadline + /// filters it out, without blocking every later request. + Unrecoverable(hellas_rpc::protocol::Digest), + Replay( + hellas_rpc::protocol::Digest, + hellas_work::work_store::JobPhase, + PreparedPaidWorkInput, + ), + } let pending = client.with_state(|state| { state .jobs() @@ -543,20 +561,38 @@ impl PaidWorkSession { }) .map(|job| { // Fetch journals retain accounting only. A restart cannot - // reconstruct a lost request or authorize another execution. + // reconstruct a lost request or authorize another + // execution, but an already delivered result still + // settles: signing payment needs no payload. if job.prepared_input().is_empty() { - return Err(PaidClientError::MissingPayload { - work_id: job.work_id(), - payment_deadline: job.authorization().payment_deadline, + return Ok(if job.result().is_some() { + RecoveredJob::Delivered(job.work_id()) + } else { + RecoveredJob::Unrecoverable(job.work_id()) }); } PreparedPaidWorkInput::decode(job.prepared_input(), MAX_RECORD_BYTES) - .map(|input| (job.work_id(), job.phase(), input)) + .map(|input| RecoveredJob::Replay(job.work_id(), job.phase(), input)) .map_err(PaidClientError::from) }) .collect::, _>>() })??; - for (work_id, phase, pending) in pending { + for recovered in pending { + let (work_id, phase, pending) = match recovered { + RecoveredJob::Delivered(work_id) => { + pay_for_result(dialer.work().await?, client, work_id).await?; + continue; + } + RecoveredJob::Unrecoverable(work_id) => { + tracing::warn!( + %work_id, + "retaining an unpaid job whose request payload is lost; \ + it can neither execute nor settle before its payment deadline" + ); + continue; + } + RecoveredJob::Replay(work_id, phase, input) => (work_id, phase, input), + }; check_request( &config.provider_policy(), &pending, @@ -596,7 +632,10 @@ impl PaidWorkSession { } } // Keep recovery armed across any error or cancellation after acceptance. - *needs_recovery = prepared.is_some(); + // A caller that declined recovery keeps the flag for a later run. + if recover { + *needs_recovery = prepared.is_some(); + } let result = match prepared { Some(prepared) => { let ready = client.readiness()?; @@ -621,7 +660,9 @@ impl PaidWorkSession { } None => None, }; - *needs_recovery = false; + if recover { + *needs_recovery = false; + } Ok(result) } } @@ -790,8 +831,10 @@ async fn execute_paid_job( ready, work_id, |event| { - emitted = true; + // Output is only exposed when a user stream observes it; + // recovery without one may safely retry after any event. if let Some(progress) = progress { + emitted = true; progress(event.clone()).map_err(|error| { hellas_rpc::protocol::work::PaidWorkError::Transcript(error.to_string()) })?; @@ -992,6 +1035,7 @@ async fn collect_until_ready( Err(hellas_work::work::DeliverError::Refused { refusal, reason }) if refusal.is_retryable() => { + check_payment_window(client, work_id)?; tracing::debug!(%reason, "waiting for paid result"); tokio::time::sleep(poll.max(Duration::from_secs(1))).await; } diff --git a/crates/sdk/src/paid_client/error.rs b/crates/sdk/src/paid_client/error.rs index a76688be..a96917bf 100644 --- a/crates/sdk/src/paid_client/error.rs +++ b/crates/sdk/src/paid_client/error.rs @@ -9,13 +9,6 @@ pub enum PaidClientError { InputMismatch(&'static str), #[error("paid-work state is missing: {0}")] MissingState(&'static str), - #[error( - "pending Fetch {work_id} lost its in-memory payload; channel remains reserved until payment deadline {payment_deadline}" - )] - MissingPayload { - work_id: hellas_rpc::Digest, - payment_deadline: u64, - }, #[error("multiple active jobs match this prepared input")] AmbiguousRecovery, #[error("payment deadline elapsed during result delivery")] @@ -24,8 +17,6 @@ pub enum PaidClientError { DeadlineOverflow { stage: &'static str }, #[error("{stage} timed out; journals retain payment state")] Timeout { stage: &'static str }, - #[error("no validators are configured")] - NoValidators, #[error("no configured validator answered ({0:?})")] ValidatorsUnavailable(Vec<(String, hellas_chain::QueryError)>), #[error("authenticated provider key differs from the payment channel")] From 4069dbfbd3189738141df69d6dd43c596b287f50 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Sat, 26 Sep 2026 20:55:42 +0200 Subject: [PATCH 22/32] Read ZDR store flags from decoded request bodies The ZDR store:true rejection parsed the raw wire bytes as JSON, so a gzip or zstd request body sailed past it and reached the upstream with retention enabled, exactly what an operator-enforced --zdr promises to forbid. The routing body decoder now lives in one shared helper, the archive middleware checks the decoded bytes and rejects bodies it cannot inspect under ZDR, and a regression test pins the compressed store:true refusal before anything is forwarded or archived. --- crates/gateway/src/archive.rs | 8 ++- crates/gateway/src/archive/tests.rs | 27 ++++++++ crates/gateway/src/http_fetch/affinity.rs | 76 ++++++++++++----------- docs/http-gateway.md | 4 +- 4 files changed, 78 insertions(+), 37 deletions(-) diff --git a/crates/gateway/src/archive.rs b/crates/gateway/src/archive.rs index e183d5f8..0db90a11 100644 --- a/crates/gateway/src/archive.rs +++ b/crates/gateway/src/archive.rs @@ -97,7 +97,13 @@ pub(crate) async fn record( Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(), }; if ephemeral { - if serde_json::from_slice::(&body) + // Decide retention on the decoded body: a compressed store=true must + // not slip past this check onto a retaining upstream. + let decoded = match super::http_fetch::affinity::decoded_body(&parts.headers, &body) { + Ok(decoded) => decoded, + Err(message) => return (StatusCode::BAD_REQUEST, message).into_response(), + }; + if serde_json::from_slice::(&decoded) .ok() .and_then(|value| value.get("store").cloned()) == Some(json!(true)) diff --git a/crates/gateway/src/archive/tests.rs b/crates/gateway/src/archive/tests.rs index d1674256..46cd8990 100644 --- a/crates/gateway/src/archive/tests.rs +++ b/crates/gateway/src/archive/tests.rs @@ -175,6 +175,33 @@ async fn zdr_rejects_ambiguous_flags_retain_and_enabled_replay_before_disk_write assert!(!path.exists()); } +#[tokio::test] +async fn zdr_rejects_compressed_store_true_before_forwarding() { + use std::io::Write as _; + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("must-not-exist"); + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(b"{\"store\":true}").unwrap(); + let body = encoder.finish().unwrap(); + let response = router(&path, true, false) + .oneshot( + Request::post("/v1/test") + .header("content-encoding", "gzip") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + "ZDR forbids store=true" + ); + assert!(!path.exists()); +} + #[tokio::test] async fn unavailable_archive_preserves_errors_and_recovers_on_the_next_request() { let root = tempfile::tempdir().unwrap(); diff --git a/crates/gateway/src/http_fetch/affinity.rs b/crates/gateway/src/http_fetch/affinity.rs index d97b64a7..7630424a 100644 --- a/crates/gateway/src/http_fetch/affinity.rs +++ b/crates/gateway/src/http_fetch/affinity.rs @@ -1,8 +1,47 @@ //! Read routing hints without changing the bytes forwarded upstream. use axum::http::HeaderMap; use serde_json::Value; +use std::borrow::Cow; use std::io::Read; +/// Decodes a gzip/zstd request body the same way routing does, bounding the +/// decoded size. Callers that make decisions on body contents must use this +/// rather than the wire bytes, or a compressed body bypasses their check. +pub(crate) fn decoded_body<'a>( + headers: &HeaderMap, + body: &'a [u8], +) -> Result, &'static str> { + let decode = |reader: &mut dyn Read| -> Result, &'static str> { + let mut output = Vec::new(); + reader + .take(8 * 1024 * 1024 + 1) + .read_to_end(&mut output) + .map_err(|_| "invalid compressed request body")?; + if output.len() > 8 * 1024 * 1024 { + return Err("decoded request body exceeds limit"); + } + Ok(output) + }; + let encoding = headers + .get("content-encoding") + .map(|v| v.to_str().map(|s| s.trim().to_ascii_lowercase())) + .transpose() + .map_err(|_| "invalid request encoding")?; + Ok(match encoding.as_deref() { + None | Some("") | Some("identity") => Cow::Borrowed(body), + Some("gzip") => Cow::Owned(decode(&mut flate2::read::MultiGzDecoder::new(body))?), + Some("zstd") => { + let mut decoder = zstd::stream::read::Decoder::new(body) + .map_err(|_| "invalid compressed request body")?; + decoder + .window_log_max(23) + .map_err(|_| "invalid compressed request body")?; + Cow::Owned(decode(&mut decoder)?) + } + _ => return Err("unsupported request encoding for model routing"), + }) +} + #[derive(Default)] pub(super) struct Hints { pub model: Option, @@ -13,44 +52,11 @@ pub(super) struct Hints { impl Hints { pub fn read(headers: &HeaderMap, body: &[u8]) -> Result { - let decoded; - let decode = |reader: &mut dyn Read| -> Result, &'static str> { - let mut output = Vec::new(); - reader - .take(8 * 1024 * 1024 + 1) - .read_to_end(&mut output) - .map_err(|_| "invalid compressed routing request")?; - if output.len() > 8 * 1024 * 1024 { - return Err("decoded routing request exceeds limit"); - } - Ok(output) - }; - let encoding = headers - .get("content-encoding") - .map(|v| v.to_str().map(|s| s.trim().to_ascii_lowercase())) - .transpose() - .map_err(|_| "invalid request encoding")?; - let bytes = match encoding.as_deref() { - None | Some("") | Some("identity") => body, - Some("gzip") => { - decoded = decode(&mut flate2::read::MultiGzDecoder::new(body))?; - &decoded - } - Some("zstd") => { - let mut decoder = zstd::stream::read::Decoder::new(body) - .map_err(|_| "invalid compressed routing request")?; - decoder - .window_log_max(23) - .map_err(|_| "invalid compressed routing request")?; - decoded = decode(&mut decoder)?; - &decoded - } - _ => return Err("unsupported request encoding for model routing"), - }; + let bytes = decoded_body(headers, body)?; let value = if bytes.is_empty() { Value::Null } else { - serde_json::from_slice::(bytes) + serde_json::from_slice::(&bytes) .map_err(|_| "model routing requires a JSON request")? }; fn field(value: Option<&Value>) -> Result, &'static str> { diff --git a/docs/http-gateway.md b/docs/http-gateway.md index 00a522f7..a506c374 100644 --- a/docs/http-gateway.md +++ b/docs/http-gateway.md @@ -180,7 +180,9 @@ increment this counter. `x-hellas-zdr: true` disables application payload persistence for that request. `--zdr` enforces this for all requests. Ambiguous flags, `store: true`, and ZDR -with inference caching enabled are rejected before archive writes. A non-ZDR +with inference caching enabled are rejected before archive writes. The +`store: true` check reads the decoded body, so a gzip or zstd request cannot +smuggle the flag past it. A non-ZDR request may be archived even when its upstream `store` is false. Unauthorized, invalid ZDR requests and bodies exceeding the 2 MiB ingress limit are rejected without payload archives. Non-ZDR requests that pass ingress but exceed the From 07e3c812ee0d96d69e7c0793662efe1360c929c7 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Sat, 26 Sep 2026 20:57:27 +0200 Subject: [PATCH 23/32] Tighten HTTP fetch gateway edge cases Four small correctness fixes. An empty body event no longer kills an otherwise valid paid response as a bogus size-limit error; zero-byte chunks are skipped and only real over-limit growth fails. A non-UTF-8 Connection request header is parsed lossily instead of dropping every token it named, so the headers it marks hop-by-hop stay stripped. Routing failures now attribute the pinned backend with the affinity that pinned it (continuation, connection or session) instead of a hardcoded session label. And --metrics-port warns in HTTP Fetch mode, where the Prometheus endpoint is not served, instead of going dark. --- crates/gateway/src/http_fetch.rs | 15 ++++++++++++--- crates/gateway/src/http_fetch/config.rs | 12 +++++++++--- crates/gateway/src/http_fetch/routing.rs | 11 +++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/crates/gateway/src/http_fetch.rs b/crates/gateway/src/http_fetch.rs index 23395276..fdef1801 100644 --- a/crates/gateway/src/http_fetch.rs +++ b/crates/gateway/src/http_fetch.rs @@ -1,5 +1,5 @@ //! HTTP bytes over authenticated Fetch, without translating vendor schemas. -mod affinity; +pub(crate) mod affinity; mod config; mod observation; mod routing; @@ -66,6 +66,12 @@ pub(super) async fn start(options: GatewayOptions) -> anyhow::Result>, request: Request) -> Respon } if let Some(backend) = failure.backend { let name = &state.routing.backends[backend].name; - observed.backend(name, "session"); + observed.backend(name, failure.affinity.unwrap_or("session")); return attributed(response, name); } return response; @@ -277,9 +283,12 @@ async fn handle(State(state): State>, request: Request) -> Respon let bytes = hellas_rpc::http_fetch::decode_base64(&base64) .map_err(|_| std::io::Error::other("invalid HTTP response encoding"))?; size = size.saturating_add(bytes.len()); - if bytes.is_empty() || size > upstream.max_response_bytes as usize { + if size > upstream.max_response_bytes as usize { Err(std::io::Error::other("HTTP response exceeds Fetch limit"))?; } + if bytes.is_empty() { + continue; + } observed.chunk(&bytes); yield Bytes::from(bytes); } diff --git a/crates/gateway/src/http_fetch/config.rs b/crates/gateway/src/http_fetch/config.rs index 5bfab44c..73b5dcf2 100644 --- a/crates/gateway/src/http_fetch/config.rs +++ b/crates/gateway/src/http_fetch/config.rs @@ -140,10 +140,16 @@ impl HttpRoute { incoming: &HeaderMap, ) -> anyhow::Result { let mut headers = self.headers.clone(); + // Connection values are ASCII by contract; a non-UTF-8 value still + // names its tokens, so parse lossily rather than silently keeping + // every header it named. + let values: Vec = incoming + .get_all("connection") + .iter() + .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned()) + .collect(); let connection = connection_headers( - incoming - .iter() - .map(|(name, value)| (name.as_str(), value.to_str().unwrap_or_default())), + std::iter::repeat("connection").zip(values.iter().map(String::as_str)), ); for (name, value) in incoming { let name = name.as_str(); diff --git a/crates/gateway/src/http_fetch/routing.rs b/crates/gateway/src/http_fetch/routing.rs index 98da056f..d6778192 100644 --- a/crates/gateway/src/http_fetch/routing.rs +++ b/crates/gateway/src/http_fetch/routing.rs @@ -193,6 +193,9 @@ pub(super) struct Unavailable { pub message: &'static str, pub retry: Option, pub backend: Option, + /// Affinity that produced `backend`, matching `Selected::affinity`'s + /// vocabulary, for failure attribution. None when nothing was pinned. + pub affinity: Option<&'static str>, } impl Unavailable { @@ -202,6 +205,7 @@ impl Unavailable { message, retry: None, backend: None, + affinity: None, } } @@ -407,6 +411,13 @@ impl Routing { let (status, delay) = unavailable.expect("nonempty backend candidates"); let mut failure = Unavailable::busy(status, delay); failure.backend = pinned; + failure.affinity = if !continuations.is_empty() { + Some("continuation") + } else if connection.is_some() { + Some("connection") + } else { + pinned.map(|_| "session") + }; Err(failure) } From 1a78445df47ed77029aae07bf63d93bf1db74c42 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Sat, 26 Sep 2026 20:57:50 +0200 Subject: [PATCH 24/32] Keep hop-by-hop headers out of the signed HTTP head The provider relayed upstream connection, transfer-encoding, content-length and keep-alive headers into the signed Adaptor.Http.Head event, so the evidence could carry a content-length contradicting the body its events actually hold. The signed head now carries end-to-end headers only; the body bytes are the single length truth. HTTPS.md gains the matching note, plus two operator clarifications from review: credential path scopes do not cover query strings, and successful courtesy http requests settle zero billable units, so courtesy spend quotas never accumulate on http routes (paid channels, which charge the fixed price, are unaffected). --- crates/providers/HTTPS.md | 16 ++++++++++++++-- crates/providers/src/http.rs | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/crates/providers/HTTPS.md b/crates/providers/HTTPS.md index e2a231af..35f23dc1 100644 --- a/crates/providers/HTTPS.md +++ b/crates/providers/HTTPS.md @@ -96,7 +96,9 @@ OAuth and refresh-token persistence; Hellas does not interpret vendor logins. Resolved secrets are injected only for an exact authorized origin, path and method, with public WebPKI roots. Callers cannot override that account's header or change trust roots to impersonate its origin. Restrict paths to the inference -endpoints the account is intended to expose. +endpoints the account is intended to expose. Path restrictions do not cover +query strings: if an API selects models or pricing through query parameters, +a caller can still steer cost within an allowed path. DNS answers are resolved and checked on every request. Connections are reused only for the same origin, DNS address set, TLS roots/pins and credential alias. @@ -126,6 +128,16 @@ route grants. On Gate's Run page, `http` is accepted as the execution environmen For CLI paid requests, `paid-work prepare-fetch --execution-environment http` prints the manifest ID to put in the work config. +Generic HTTP responses carry no token usage, so a successful courtesy `http` +request settles zero billable units: courtesy spend quotas (`max_units` per +window) never accumulate on `http` routes, only concurrency is bounded. Paid +channels are unaffected — they charge the payment policy's fixed price per job. +If courtesy callers reach provider-funded accounts, bound their exposure with +`fetch_max_in_flight`/queue limits, or require a paid channel. + Results contain one `Adaptor.Http.Head` event followed by base64 body events -and a signed terminal. `HttpFetchResponse::from_output` reconstructs the body +and a signed terminal. Hop-by-hop and framing headers (`connection`, +`transfer-encoding`, `content-length` and friends) describe the provider's own +connection, so they are not part of the signed head; the body bytes are the +only length truth. `HttpFetchResponse::from_output` reconstructs the body and checks ordering and size after transcript signature verification. diff --git a/crates/providers/src/http.rs b/crates/providers/src/http.rs index b1c95605..91950bab 100644 --- a/crates/providers/src/http.rs +++ b/crates/providers/src/http.rs @@ -262,9 +262,27 @@ impl HttpFetchProvider { trace.status(response.status().as_u16()); // A non-2xx status is still a completed HTTP exchange. Return it, with // its exact body, to the authenticated client; do not log it. + // The signed head carries end-to-end headers only: hop-by-hop and + // framing headers describe this connection, not the response, and a + // content-length could contradict the body the events actually carry. let headers = response .headers() .iter() + .filter(|(k, _)| { + !matches!( + k.as_str(), + "connection" + | "keep-alive" + | "proxy-authenticate" + | "proxy-authorization" + | "proxy-connection" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + | "content-length" + ) + }) .map(|(k, v)| { Ok(( k.to_string(), From c3e9bbefe4f76f73c83de3ea2ee75978482c0321 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Sat, 26 Sep 2026 20:58:05 +0200 Subject: [PATCH 25/32] Tighten paid HTTP vocabulary and admission coherence Reconstruction now refuses an interim 1xx as the single signed head, since only a final status completes a paid response, and the reserved request-header list covers the remaining RFC 9110 hop-by-hop names. Admission rejects an Open Fetch response cap the channel can never deliver: base64 body events carry at most three raw bytes per four payload bytes, so a max_response_bytes beyond three quarters of the policy's output budget would run the provider's upstream fetch and refuse the terminal at payment. The Open Fetch host allowlist is normalised to the lowercase punycode form admission compares against, so a case- or space-spelled entry can no longer brick a mounted policy. The close descriptor's version 2 gains a named constant, and the code generator's open-method default handler gains the comment explaining why a name match is the rule. --- crates/rpc/build.rs | 6 ++++++ crates/rpc/src/http_fetch.rs | 6 +++++- crates/rpc/src/protocol/work_fetch.rs | 22 +++++++++++++++++++++- crates/rpc/src/protocol/work_setup.rs | 9 ++++++--- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/crates/rpc/build.rs b/crates/rpc/build.rs index f06fd336..831da70a 100644 --- a/crates/rpc/build.rs +++ b/crates/rpc/build.rs @@ -776,6 +776,12 @@ fn handler_signature(m: &MethodPlan) -> TokenStream { Shape::Unary => quote! { impl Into> + Send }, _ => boxed_stream(response), }; + // Every service carries the confidential-Open handshake method, named + // `open` in each package. A dispatcher that never mounted an OpenHandler + // must still answer it: refusing with Unavailable keeps a missing mount + // a runtime refusal for the peer instead of a compile error here. The + // match is on the bare method name, so any future `open` rpc in any + // package gets this same default. if fn_name == "open" { let allow_open_context_unused = if m.connection_bound { quote! { let _ = context; } diff --git a/crates/rpc/src/http_fetch.rs b/crates/rpc/src/http_fetch.rs index fc25c12e..0ac65911 100644 --- a/crates/rpc/src/http_fetch.rs +++ b/crates/rpc/src/http_fetch.rs @@ -188,7 +188,9 @@ pub fn check_headers(headers: &[(String, String)], request: bool) -> Result<(), | "content-length" | "transfer-encoding" | "connection" + | "keep-alive" | "upgrade" + | "proxy-authenticate" | "proxy-authorization" | "proxy-connection" | "te" @@ -226,7 +228,9 @@ impl HttpFetchResponse { status, headers, })) if response.is_none() => { - if !(100..=599).contains(&status) { + // Only a final status can answer the job: an interim 1xx + // is a hop artifact, not a completed paid response. + if !(200..=599).contains(&status) { return Err(HttpRequestError("response status")); } check_headers(&headers, false)?; diff --git a/crates/rpc/src/protocol/work_fetch.rs b/crates/rpc/src/protocol/work_fetch.rs index ebe2a784..2d535234 100644 --- a/crates/rpc/src/protocol/work_fetch.rs +++ b/crates/rpc/src/protocol/work_fetch.rs @@ -99,7 +99,15 @@ impl FetchRoutePolicy { /// exact hosts and required pins; the HTTPS driver checks certificates and addresses. #[must_use] pub fn open_fetch(require_spki_pin: bool, allowed_hosts: impl Into>) -> Self { - let mut allowed_hosts = allowed_hosts.into(); + // Admission compares against Url::host_str, which is lowercase + // punycode; normalize here so a host spelled with case or padding + // can still match, and drop entries that never could. + let mut allowed_hosts: Vec = allowed_hosts + .into() + .into_iter() + .map(|host| host.trim().to_ascii_lowercase()) + .filter(|host| !host.is_empty()) + .collect(); allowed_hosts.sort_unstable(); allowed_hosts.dedup(); Self::OpenFetch { @@ -568,6 +576,18 @@ pub fn check_prepared_fetch_input( field: "open-fetch host or pin policy", }); } + // A response cap the channel can never deliver would run the + // provider's upstream fetch and refuse the terminal at payment. + // Base64 body events carry at most three raw bytes per four + // payload bytes, so that bound is decidable before admission. + let deliverable = u64::from(policy.max_output_bytes) / 4 * 3; + if u64::from(request.max_response_bytes) > deliverable { + return Err(PaidWorkError::OverEnvelope { + field: "max_response_bytes", + actual: u64::from(request.max_response_bytes), + limit: deliverable, + }); + } } let graph = [ diff --git a/crates/rpc/src/protocol/work_setup.rs b/crates/rpc/src/protocol/work_setup.rs index d35ce8ef..75b395bc 100644 --- a/crates/rpc/src/protocol/work_setup.rs +++ b/crates/rpc/src/protocol/work_setup.rs @@ -613,6 +613,9 @@ pub struct CloseDescriptor { /// First byte of the close descriptor stored in an armed setup record. const CLOSE_DESCRIPTOR_VERSION: u8 = 1; +/// Version 2 length-prefixes the execution policy so the Fetch profile can +/// extend it; v1 is the fixed-size Evaluate policy exactly. +const CLOSE_DESCRIPTOR_VERSION_V2: u8 = 2; impl CloseDescriptor { /// Returns the payment channel, including both complete terms bodies and @@ -676,7 +679,7 @@ impl CloseDescriptor { let version = if matches!(self.execution_policy, PaidWorkPolicy::Evaluate(_)) { CLOSE_DESCRIPTOR_VERSION } else { - 2 + CLOSE_DESCRIPTOR_VERSION_V2 }; let mut out = vec![version]; push_kernel(&mut out, &self.channel.network()); @@ -688,7 +691,7 @@ impl CloseDescriptor { out.extend_from_slice(&self.policy_salt); out.extend_from_slice(&self.channel.channel_policy().encode()); let policy_bytes = self.execution_policy.encode(); - if version == 2 { + if version == CLOSE_DESCRIPTOR_VERSION_V2 { out.extend_from_slice(&(policy_bytes.len() as u64).to_be_bytes()); } out.extend_from_slice(&policy_bytes); @@ -707,7 +710,7 @@ impl CloseDescriptor { pub fn decode(bytes: &[u8]) -> Result { let mut cursor = CloseCursor { bytes }; let version = cursor.byte()?; - if version != CLOSE_DESCRIPTOR_VERSION && version != 2 { + if version != CLOSE_DESCRIPTOR_VERSION && version != CLOSE_DESCRIPTOR_VERSION_V2 { return Err(WorkSetupError::DescriptorMalformed); } let network = cursor.network()?; From 78cdc881608f19d7733c749fce5b33bcadd94c00 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Sat, 26 Sep 2026 20:58:30 +0200 Subject: [PATCH 26/32] Skip the full-state clone for redundant stream releases The stream replay path commits a PlaintextReleased record per emitted frame, and every one after the first is redundant: nothing is written, but commit still cloned the whole channel state, transcript included, before apply could say so. At the documented 32 MiB spool that is gigabytes of memcpy per delivered response. A cheap exact redundancy check, mirroring apply_plaintext's own arms and falling through to the full apply whenever in doubt, now answers on the current state first. The same change documents why commit still validates bodies in full before they are stripped, why record_result reads the last fresh readiness rather than gating on observation, and why an input-less job answers Indeterminate. suspend no longer converts an observer-less endpoint to never-admitting, the README stops claiming client Fetch journals may retain payloads (mount is metadata-only on both endpoints), and the paid gateway doc notes the observation-age versus block-interval coupling. --- crates/work/README.md | 13 +++++---- crates/work/src/work.rs | 29 ++++++++++++++++----- crates/work/src/work_store/channel/state.rs | 22 ++++++++++++++++ crates/work/src/work_store/channel/store.rs | 12 ++++++++- docs/paid-gateway.md | 5 +++- 5 files changed, 67 insertions(+), 14 deletions(-) diff --git a/crates/work/README.md b/crates/work/README.md index 4efe558d..e697e246 100644 --- a/crates/work/README.md +++ b/crates/work/README.md @@ -23,9 +23,12 @@ through Courtesy's transcript store or replay cache. Upstream and projection faults are reduced to fixed error messages before they reach the paid-work driver's logs. -The client may store its own inputs and received responses. This policy covers -application-managed provider storage; it does not establish the upstream API's -retention policy or replace host memory/swap/crash-dump controls. +Fetch channels open metadata-only journals on both endpoints: a client +restart cannot reconstruct a lost request or replay a received response, +though a delivered result's retained evidence still settles payment. +This policy covers application-managed storage; it does not establish the +upstream API's retention policy or replace host memory/swap/crash-dump +controls. ## Restart behavior @@ -141,8 +144,8 @@ rejects a weaker output scheme. A session serializes its jobs. After cancellation, recover the journal before admitting another job. Evaluate supports authenticated incremental token delivery; -Fetch currently returns a complete bounded response. Client journals may retain -payloads; the Fetch provider's journal remains metadata-only. +Fetch currently returns a complete bounded response. Fetch journals are +metadata-only on both endpoints; Evaluate client journals may retain payloads. Apple App Attest requires a provisioned, signed macOS host. `ProducerSigned` verifies the key and transcript but does not attest the binary. These are the diff --git a/crates/work/src/work.rs b/crates/work/src/work.rs index 2013e963..1e0d009f 100644 --- a/crates/work/src/work.rs +++ b/crates/work/src/work.rs @@ -803,14 +803,17 @@ impl ProviderEndpoint { /// Decides whether the backend may be invoked for `work_id`, and /// makes that decision durable before it is returned. /// - /// [`RunAdmission::Invoke`] is returned only when the job was + /// [`RunAdmission::Invoke`] or [`RunAdmission::InvokeFetch`] is + /// returned only when the job was /// accepted and not yet running, and only after the running marker /// is on the disk — so a crash between the marker and the answer /// costs the invocation, never a second one. Every other phase past /// acceptance answers with what it already is: a job this process is /// running, a job whose result is already signed, or a job whose /// marker was found by a process that did not write it. A job that - /// was never co-signed is refused rather than answered. + /// was never co-signed is refused rather than answered. A job whose + /// input a metadata-only journal no longer holds is + /// [`RunAdmission::Indeterminate`]: it must not be executed again. /// /// `ready` is a *fresh* readiness decision, and the freshness is the /// caller's to owe in exactly the sense [`ReadyChannel`] already @@ -945,6 +948,11 @@ impl ProviderEndpoint { .as_ref() .ok_or(EndpointError::NotAdmitting)? .clone(); + // The last fresh readiness, deliberately not `admitting()`: a result + // signature commits nothing the payment path does not gate on its + // own freshness check, and refusing here would discard completed + // upstream work that the client may still pay for. Release and + // payment still require a fresh observation. let channel = ready.channel(); let result = ready .execution_policy() @@ -1753,6 +1761,12 @@ pub struct PreparedFetchInput { } impl PreparedFetchInput { + /// Builds an admitted input from its verified parts and channel policy. + #[must_use] + pub const fn new(parts: PreparedPaidFetchInputParts, policy: PaidFetchPolicyV1) -> Self { + Self { parts, policy } + } + /// Returns the fixed output bounds the provider must enforce while running. pub const fn policy(&self) -> &PaidFetchPolicyV1 { &self.policy @@ -2566,11 +2580,12 @@ impl WorkService { /// Revokes observer freshness without discarding results or close duties. pub fn suspend(&self) -> Result<(), EndpointError> { - self.endpoint()? - .observation - .get_or_insert_with(Observation::default) - .suspend(); - self.changed.notify_waiters(); + // An endpoint without an observer has no freshness to revoke; + // installing a defaulted one here would only close admission. + if let Some(observation) = self.endpoint()?.observation.as_mut() { + observation.suspend(); + self.changed.notify_waiters(); + } Ok(()) } diff --git a/crates/work/src/work_store/channel/state.rs b/crates/work/src/work_store/channel/state.rs index b7577791..240ebfe4 100644 --- a/crates/work/src/work_store/channel/state.rs +++ b/crates/work/src/work_store/channel/state.rs @@ -588,6 +588,28 @@ impl ChannelState { } } + /// Whether this record would apply as [`Applied::Redundant`], answered + /// without cloning the state. Only checks that stay cheap at the largest + /// legal state belong here; anything else falls through to the full + /// [`Self::apply`], which decides. + pub(super) fn is_redundant(&self, record: &ChannelRecord) -> bool { + match record { + // The stream replay path commits a release per emitted frame, + // and every one after the first is redundant. Detecting that + // here keeps a full-state clone — transcript included — off + // the per-frame hot path. The arms mirror apply_plaintext: + // any doubt falls through to the full apply and its refusal. + ChannelRecord::PlaintextReleased { work_id } => { + self.role == Role::Provider + && self + .jobs + .get(work_id) + .is_some_and(|job| job.phase.delivered()) + } + _ => false, + } + } + /// Applies one record, or says why it may not be applied. /// /// Every rule this endpoint has is here, and replay runs it too, so diff --git a/crates/work/src/work_store/channel/store.rs b/crates/work/src/work_store/channel/store.rs index a978ea2f..e871d525 100644 --- a/crates/work/src/work_store/channel/store.rs +++ b/crates/work/src/work_store/channel/store.rs @@ -173,8 +173,18 @@ impl ChannelStore { verifier: &V, ) -> Result<&ChannelState, WorkStoreError> { // Applied to a copy first: a record the rules refuse must leave - // neither the file nor the state touched. + // neither the file nor the state touched. A cheap exact check + // first: the stream replay path offers a redundant release per + // emitted frame, and the copy would clone the whole transcript. + if self.state.is_redundant(&record) { + return Ok(&self.state); + } let mut next = self.state.clone(); + // The `false` keeps commit-time validation at full strength even + // for a metadata-only journal: bodies are verified — digest and + // transcript-to-result reproduction — before they are stripped + // from the bytes written. Only replay and checkpoint validation + // downgrade for body-less records. if next.apply(&record, verifier, false)? == Applied::Changed { // The signature this record carries leaves after this // returns, so the state that authorises it has to be one a diff --git a/docs/paid-gateway.md b/docs/paid-gateway.md index 2fa38ebb..ceb16b20 100644 --- a/docs/paid-gateway.md +++ b/docs/paid-gateway.md @@ -89,7 +89,10 @@ observer fails or no new verified finalized height arrives within that age. Repeated reads of an old tip do not renew it. The bound starts before the chain read, not after it finishes, and readiness is published only after all intervening blocks have been applied. Results may still be retained during an observation -outage; existing payment certificates remain safe to retransmit. +outage; existing payment certificates remain safe to retransmit. Because renewal +requires an *advancing* finalized height, this age should also exceed the +deployment's block interval by a comfortable margin: a chain finalizing slower +than the age spends every gap stale, and admission flaps on each block. This is an online channel: its observer must detect unilateral closes and submit any response within `omit_response_blocks`. Configure observation age, network From bc33d4bc7c194576da19aa802adc1c53b6485a98 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Sat, 26 Sep 2026 20:58:50 +0200 Subject: [PATCH 27/32] Cover the executor paid fetch seam with tests The paid fetch admission path had no tests at all: none of prepare_paid_fetch's refusal branches, not the capacity accounting, not the progress wiring, sitting directly on the paid execution seam. Ten tests now drive the production path through the actor, from the tampered transcript and contract mismatches (retention, assurance, environment, route, capability) to the fail-fast capacity refusal and a happy path whose signed output verifies end to end. The seam also gets three fixes its review exposed: the PaidFetch completion runs the deferred quota retries like fetch completion does, so courtesy deferrals cannot linger until restart; the upstream/projection failure keeps its position and cause in the operator log while the peer still receives the sanitized message; and start_paid_fetch documents that a capacity refusal is terminal for the journaled job, not queued, so operators size the limit for bursts. PreparedFetchInput gains the public constructor the tests (and any out-of-crate backend) need. (PreparedFetchInput::new lives in hellas-work and is committed with the work changes.) --- .../executor/src/executor/actor/execution.rs | 6 +- crates/executor/src/executor/actor/mod.rs | 2 + .../executor/src/executor/actor/paid_fetch.rs | 545 +++++++++++++++++- 3 files changed, 548 insertions(+), 5 deletions(-) diff --git a/crates/executor/src/executor/actor/execution.rs b/crates/executor/src/executor/actor/execution.rs index 9ec85e4e..89ee39e7 100644 --- a/crates/executor/src/executor/actor/execution.rs +++ b/crates/executor/src/executor/actor/execution.rs @@ -62,7 +62,7 @@ use super::{ /// Backpressure buffer for the per-execution event channel. The worker keeps /// one slot reserved for the terminal frame. Backpressure waits in the spawned /// provider task, leaving the execution actor free to admit and finish work. -const PER_EXECUTION_CHANNEL_CAPACITY: usize = 64; +pub(super) const PER_EXECUTION_CHANNEL_CAPACITY: usize = 64; const FETCH_STREAM_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(90); const FETCH_STREAM_STALLED_ERROR: &str = "fetch stream consumer did not drain its bounded event channel"; @@ -595,7 +595,7 @@ impl Executor { result.map_err(|error| error.to_string()) } - fn retry_deferred_fetch_quota_cancellations(&mut self) { + pub(super) fn retry_deferred_fetch_quota_cancellations(&mut self) { let attempts = self .pending_fetch_quota_cancellations .len() @@ -668,7 +668,7 @@ impl Executor { } } - fn retry_deferred_fetch_quota_settlements(&mut self) { + pub(super) fn retry_deferred_fetch_quota_settlements(&mut self) { let attempts = self .pending_fetch_quota_settlements .len() diff --git a/crates/executor/src/executor/actor/mod.rs b/crates/executor/src/executor/actor/mod.rs index 9b0f8d53..69b65060 100644 --- a/crates/executor/src/executor/actor/mod.rs +++ b/crates/executor/src/executor/actor/mod.rs @@ -456,6 +456,8 @@ impl Executor { ExecutorCompletion::PaidFetch { reply, result } => { self.active_fetches = self.active_fetches.saturating_sub(1); self.dispatch_next_fetch(); + self.retry_deferred_fetch_quota_settlements(); + self.retry_deferred_fetch_quota_cancellations(); let _ = reply.send(result); false } diff --git a/crates/executor/src/executor/actor/paid_fetch.rs b/crates/executor/src/executor/actor/paid_fetch.rs index 57279aed..7017325b 100644 --- a/crates/executor/src/executor/actor/paid_fetch.rs +++ b/crates/executor/src/executor/actor/paid_fetch.rs @@ -15,6 +15,14 @@ use crate::fetch_policy::FetchRoute; use crate::fetch_provider::FetchCall; impl Executor { + /// Admits one journaled paid Fetch, or refuses it before any provider + /// work begins. + /// + /// Capacity refusal is deliberately fail-fast: the work layer journals it + /// as that job's terminal failure rather than queueing, unlike paid + /// Evaluate, which waits for a dispatch slot. A paid burst beyond + /// `fetch_max_in_flight` therefore ends the excess jobs, so operators + /// should size the limit for paid bursts, not for average load. pub(super) fn start_paid_fetch( &mut self, input: PreparedFetchInput, @@ -34,7 +42,8 @@ impl Executor { let completion = self.completion_tx.clone(); let key = Arc::clone(&self.provider.producer_key); let task = async move { - let (sender, mut receiver) = mpsc::channel(64); + let (sender, mut receiver) = + mpsc::channel(super::execution::PER_EXECUTION_CHANNEL_CAPACITY); let run = super::execution::run_fetch_provider( entry.provider, session.provider_request, @@ -68,7 +77,15 @@ impl Executor { let (result, drained) = tokio::join!(run, drain); let result = drained.and_then(|()| { result - .map_err(|_| { + .map_err(|failure| { + // The peer receives the sanitized message below; the + // operator needs the position and cause to tell + // upstream, transport and projection faults apart. + tracing::warn!( + position = failure.position, + error = %failure.error, + "paid fetch upstream or projection failed" + ); ExecutorError::Execution("paid fetch upstream or projection failed".into()) }) .and_then(|run| { @@ -147,3 +164,527 @@ impl Executor { Ok((entry, session, request, policy)) } } + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::time::Duration; + + use futures_util::stream; + use hellas_rpc::fetch::{ + FetchTerminalPayload, build_input_events_with_retention, decode_fetch_event_payload, + decode_fetch_terminal_payload, encode_fetch_event_payload, encode_fetch_terminal_payload, + verify_input_events, verify_output_events, + }; + use hellas_rpc::output::{OutputEvent, StopReason, TextChannel}; + use hellas_rpc::protocol::work_fetch::{PaidFetchPolicyV1, PreparedPaidFetchInputParts}; + use hellas_rpc::{ + Assurance, ContentId, Digest, FetchEnvironment, InputCommitment, InputEventEnvelope, + ProducerSigningKey, Retention, + }; + use hellas_work::work::PaidProgress; + use tokio::sync::Notify; + use tokio::time::timeout; + + use super::*; + use crate::ExecutorSpawnConfig; + use crate::executor::{ExecutorHandle, ExecutorOwedRequest}; + use crate::fetch_policy::FetchRoutePolicy; + use crate::fetch_projection::{ + FetchAdaptorError, FetchAdaptorFactory, FetchAdaptorSession, FetchProjector, + FetchRequestView, ProjectedFetch, + }; + use crate::fetch_provider::{ + FetchProvider, FetchProviderFuture, FetchProviderResponse, FetchProviderResponseHead, + FetchProviderStream, MockFetchProvider, PreparedFetchRequest, + }; + use crate::fetch_registry::{FetchRouteEntry, FetchRouteRegistry}; + + const SERVICE: &str = "openai"; + const METHOD: &str = "responses"; + const BODY: &[u8] = br#"{"input":"paid"}"#; + + fn environment() -> FetchEnvironment { + FetchEnvironment::OpenAiResponses + } + + fn signing_key() -> ProducerSigningKey { + ProducerSigningKey::from_secret_bytes([7; 32]).expect("valid test key") + } + + fn input_events(retention: Retention, assurance: Assurance) -> Vec { + build_input_events_with_retention( + SERVICE, + METHOD, + BODY, + environment().manifest_id(), + assurance, + &signing_key(), + retention, + ) + .expect("test input transcript builds") + } + + fn policy() -> PaidFetchPolicyV1 { + PaidFetchPolicyV1 { + allowed_environment: environment().manifest_id(), + route_commitment: Digest::from_bytes([3; 32]), + max_request_body_bytes: 4096, + max_output_events: 64, + max_output_bytes: 16384, + max_spool_bytes: 65536, + max_encoded_result_frame: 65536, + max_encoded_prepared_input: 65536, + dispatch_margin_blocks: 4, + delivery_margin_blocks: 2, + oracle_grace_blocks: 6, + fixed_price: 7, + } + } + + fn prepared_input(events: Vec) -> PreparedFetchInput { + PreparedFetchInput::new( + PreparedPaidFetchInputParts { + fetch_input_transcript: events, + manifest: environment().manifest(), + }, + policy(), + ) + } + + struct StubFetchAdaptorFactory { + environment: ContentId, + reject: bool, + } + + impl FetchAdaptorFactory for StubFetchAdaptorFactory { + fn execution_environment(&self) -> ContentId { + self.environment + } + + fn create(&self, request: &FetchCall) -> Result { + if self.reject { + return Err(FetchAdaptorError::failed( + "stub adaptor rejects the request", + )); + } + Ok(FetchAdaptorSession { + request_view: FetchRequestView::from_call(request), + provider_request: PreparedFetchRequest::new(request, request.body.clone()), + projector: Box::new(StubFetchProjector { projected: false }), + }) + } + } + + struct StubFetchProjector { + projected: bool, + } + + impl FetchProjector for StubFetchProjector { + fn project(&mut self, _bytes: &[u8]) -> Result, FetchAdaptorError> { + if std::mem::replace(&mut self.projected, true) { + return Ok(Vec::new()); + } + let event = encode_fetch_event_payload(&OutputEvent::TextDelta { + index: 0, + delta: "paid-output".to_string(), + channel: TextChannel::Output, + }) + .map_err(|error| FetchAdaptorError::failed(error.to_string()))?; + let terminal = encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .map_err(|error| FetchAdaptorError::failed(error.to_string()))?; + Ok(vec![ + ProjectedFetch::Event(event), + ProjectedFetch::Terminal(terminal), + ]) + } + + fn finish(&mut self) -> Result, FetchAdaptorError> { + Ok(Vec::new()) + } + } + + #[derive(Clone)] + struct BlockingFetchProvider { + environment: ContentId, + started: Arc, + released: Arc, + notify: Arc, + } + + impl BlockingFetchProvider { + fn new(environment: ContentId) -> Self { + Self { + environment, + started: Arc::new(AtomicUsize::new(0)), + released: Arc::new(AtomicBool::new(false)), + notify: Arc::new(Notify::new()), + } + } + + fn started(&self) -> usize { + self.started.load(Ordering::SeqCst) + } + + fn release(&self) { + self.released.store(true, Ordering::SeqCst); + self.notify.notify_waiters(); + } + } + + impl FetchProvider for BlockingFetchProvider { + fn execution_environment(&self) -> ContentId { + self.environment + } + + fn run(&self, _request: PreparedFetchRequest) -> FetchProviderFuture<'_> { + let started = Arc::clone(&self.started); + let released = Arc::clone(&self.released); + let notify = Arc::clone(&self.notify); + Box::pin(async move { + started.fetch_add(1, Ordering::SeqCst); + while !released.load(Ordering::SeqCst) { + notify.notified().await; + } + Ok(FetchProviderResponse { + head: FetchProviderResponseHead::default(), + stream: Box::pin(stream::iter([Ok(b"chunk".to_vec())])) as FetchProviderStream, + }) + }) + } + } + + fn routes( + provider: Arc, + adaptor_factory: Arc, + capabilities: FetchRoutePolicy, + ) -> FetchRouteRegistry { + let mut registry = FetchRouteRegistry::new(); + registry + .register( + FetchRoute::new(SERVICE, METHOD), + FetchRouteEntry::new(provider, adaptor_factory, capabilities) + .expect("test provider and adaptor identities match"), + ) + .expect("test route registers"); + registry + } + + fn stub_adaptor(environment: ContentId) -> Arc { + Arc::new(StubFetchAdaptorFactory { + environment, + reject: false, + }) + } + + async fn spawn_executor( + routes: FetchRouteRegistry, + fetch_max_in_flight: usize, + ) -> ExecutorHandle { + let mut config = ExecutorSpawnConfig::fetch_only( + Arc::new(signing_key()), + Arc::new(b"paid-fetch-tests".to_vec()), + Assurance::ProducerSigned, + routes, + ); + config.fetch_max_in_flight = fetch_max_in_flight; + Executor::spawn_configured(config) + .await + .expect("test executor spawns") + } + + async fn spawn_stub_executor(provider: MockFetchProvider) -> ExecutorHandle { + spawn_executor( + routes( + Arc::new(provider), + stub_adaptor(environment().manifest_id()), + FetchRoutePolicy::default(), + ), + 1, + ) + .await + } + + async fn run_paid_fetch( + handle: &ExecutorHandle, + input: PreparedFetchInput, + progress: Option, + ) -> Result, ExecutorError> { + let (reply, receive) = oneshot::channel(); + handle + .owed_tx + .send(ExecutorOwedRequest::RunPaidFetch { + span: tracing::Span::none(), + input: Box::new(input), + progress, + reply, + }) + .await + .expect("owed ingress remains open"); + timeout(Duration::from_secs(5), receive) + .await + .expect("paid fetch answers within the test budget") + .expect("actor answers the paid fetch") + } + + fn commitment_of(events: &[InputEventEnvelope]) -> InputCommitment { + verify_input_events(events) + .expect("test input verifies") + .input_commitment + } + + #[tokio::test] + async fn paid_fetch_rejects_a_tampered_input_transcript() { + let handle = spawn_stub_executor(MockFetchProvider::new(environment().manifest_id())).await; + let mut events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + let foreign = build_input_events_with_retention( + SERVICE, + METHOD, + BODY, + environment().manifest_id(), + Assurance::ProducerSigned, + &ProducerSigningKey::from_secret_bytes([8; 32]).expect("valid foreign key"), + Retention::Ephemeral, + ) + .expect("foreign transcript builds"); + // Both halves are well formed; the spliced terminal event fails + // signature verification under the transcript's signer. + *events.last_mut().expect("input transcript has events") = foreign + .last() + .expect("foreign transcript has events") + .clone(); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "invalid paid fetch input") + ); + } + + #[tokio::test] + async fn paid_fetch_rejects_retained_input() { + let handle = spawn_stub_executor(MockFetchProvider::new(environment().manifest_id())).await; + let events = input_events(Retention::Retain, Assurance::ProducerSigned); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "paid fetch contract mismatch") + ); + } + + #[tokio::test] + async fn paid_fetch_rejects_mismatched_assurance() { + let handle = spawn_stub_executor(MockFetchProvider::new(environment().manifest_id())).await; + let events = input_events(Retention::Ephemeral, Assurance::AppleAppAttest); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "paid fetch contract mismatch") + ); + } + + #[tokio::test] + async fn paid_fetch_rejects_a_manifest_other_than_the_signed_environment() { + let handle = spawn_stub_executor(MockFetchProvider::new(environment().manifest_id())).await; + // The transcript is valid but signs another environment than the + // bundled manifest commits to. + let events = build_input_events_with_retention( + SERVICE, + METHOD, + BODY, + ContentId::from_bytes([8; 32]), + Assurance::ProducerSigned, + &signing_key(), + Retention::Ephemeral, + ) + .expect("foreign environment transcript builds"); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "paid fetch contract mismatch") + ); + } + + #[tokio::test] + async fn paid_fetch_rejects_an_unregistered_route() { + let handle = spawn_executor(FetchRouteRegistry::new(), 1).await; + let events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "paid fetch route is unavailable") + ); + } + + #[tokio::test] + async fn paid_fetch_rejects_a_route_for_another_environment() { + let other = ContentId::from_bytes([8; 32]); + let handle = spawn_executor( + routes( + Arc::new(MockFetchProvider::new(other)), + stub_adaptor(other), + FetchRoutePolicy::default(), + ), + 1, + ) + .await; + let events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "paid fetch route manifest mismatch") + ); + } + + #[tokio::test] + async fn paid_fetch_rejects_an_adaptor_refusal_without_reaching_the_provider() { + let provider = MockFetchProvider::new(environment().manifest_id()); + let handle = spawn_executor( + routes( + Arc::new(provider.clone()), + Arc::new(StubFetchAdaptorFactory { + environment: environment().manifest_id(), + reject: true, + }), + FetchRoutePolicy::default(), + ), + 1, + ) + .await; + let events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::InvalidQuoteRequest(error)) if error == "paid fetch adaptor rejected request") + ); + assert_eq!(provider.calls(SERVICE, METHOD, BODY), 0); + } + + #[tokio::test] + async fn paid_fetch_rejects_a_request_beyond_route_capabilities() { + let capabilities = FetchRoutePolicy { + allowed_models: Some(BTreeSet::from(["gpt-5".to_string()])), + max_output_units: None, + }; + let handle = spawn_executor( + routes( + Arc::new(MockFetchProvider::new(environment().manifest_id())), + stub_adaptor(environment().manifest_id()), + capabilities, + ), + 1, + ) + .await; + let events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + + let result = run_paid_fetch(&handle, prepared_input(events), None).await; + + assert!( + matches!(result, Err(ExecutorError::PolicyDenied(error)) if error == "paid fetch exceeds route capabilities") + ); + } + + #[tokio::test] + async fn paid_fetch_at_capacity_fails_fast_instead_of_queueing() { + let provider = BlockingFetchProvider::new(environment().manifest_id()); + let handle = spawn_executor( + routes( + Arc::new(provider.clone()), + stub_adaptor(environment().manifest_id()), + FetchRoutePolicy::default(), + ), + 1, + ) + .await; + let first_events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + let first_commitment = commitment_of(&first_events); + let first = tokio::spawn({ + let handle = handle.clone(); + async move { run_paid_fetch(&handle, prepared_input(first_events), None).await } + }); + timeout(Duration::from_secs(5), async { + while provider.started() == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("first paid fetch reaches the provider"); + + // The single slot is occupied: the refusal must arrive while the + // first fetch is still blocked, not after it drains. + let second = run_paid_fetch( + &handle, + prepared_input(input_events( + Retention::Ephemeral, + Assurance::ProducerSigned, + )), + None, + ) + .await; + assert!( + matches!(second, Err(ExecutorError::ResourceExhausted(error)) if error == "fetch concurrency limit reached") + ); + + provider.release(); + let output = first + .await + .expect("first paid fetch task completes") + .expect("first paid fetch succeeds"); + assert_eq!(provider.started(), 1); + verify_output_events(first_commitment, Assurance::ProducerSigned, &output) + .expect("first paid fetch output verifies"); + } + + #[tokio::test] + async fn paid_fetch_streams_signed_progress_and_returns_signed_output() { + let provider = MockFetchProvider::new(environment().manifest_id()); + provider.insert(SERVICE, METHOD, BODY, [b"chunk".to_vec()]); + let handle = spawn_stub_executor(provider.clone()).await; + let events = input_events(Retention::Ephemeral, Assurance::ProducerSigned); + let input_commitment = commitment_of(&events); + let progressed = Arc::new(AtomicUsize::new(0)); + let progress: PaidProgress = { + let progressed = Arc::clone(&progressed); + Arc::new(move |event| { + assert_eq!( + event.event().body().kind(), + hellas_rpc::fetch::OUTPUT_EVENT_KIND + ); + progressed.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + }; + + let output = run_paid_fetch(&handle, prepared_input(events), Some(progress)) + .await + .expect("paid fetch succeeds"); + + assert_eq!(progressed.load(Ordering::SeqCst), 1); + assert_eq!(provider.calls(SERVICE, METHOD, BODY), 1); + let verified = verify_output_events(input_commitment, Assurance::ProducerSigned, &output) + .expect("paid fetch output verifies"); + assert_eq!(verified.producer_key, signing_key().public_key()); + let (payloads, terminal) = verified.output_event_payloads(); + let [payload] = payloads else { + panic!("one streamed event before the terminal") + }; + assert!(matches!( + decode_fetch_event_payload(payload), + Ok(OutputEvent::TextDelta { .. }) + )); + assert!(matches!( + decode_fetch_terminal_payload(terminal), + Ok(FetchTerminalPayload::Finished { .. }) + )); + } +} From 603dae113863184e4238deea6317780fbef64374 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Sat, 26 Sep 2026 20:59:04 +0200 Subject: [PATCH 28/32] Drop SDK alias shims in paid-work commands The CLI kept speaking names that no longer exist in the SDK (OpenPaidChannel, PaidOutput, check_policy_input) through import aliases, and carried two mid-file use statements where deleted code used to be. Call sites now use the SDK names directly. The --http-fetch-config read takes the bounded, path-labelled loader the pool file already used, and an empty provider candidate list is a startup error instead of a panic. --- crates/cli/src/commands/paid_work.rs | 16 ++++++---------- crates/cli/src/commands/paid_work/gateway.rs | 18 +++++++++++------- crates/cli/src/main.rs | 10 +++++++++- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/crates/cli/src/commands/paid_work.rs b/crates/cli/src/commands/paid_work.rs index ce45afad..052e9195 100644 --- a/crates/cli/src/commands/paid_work.rs +++ b/crates/cli/src/commands/paid_work.rs @@ -19,11 +19,11 @@ use hellas_rpc::protocol::work_fetch::PreparedPaidFetchInputV1; use hellas_rpc::protocol::work_profile::PreparedPaidWorkInput; #[cfg(feature = "gateway")] use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; +#[cfg(test)] +use hellas_sdk::paid_client::check_genesis_payload; +use hellas_sdk::paid_client::{InputIdentities, PaidWorkSession, bind_paid_endpoint}; #[cfg(feature = "gateway")] -use hellas_sdk::paid_client::{ - PaidWorkResult as PaidOutput, check_evaluate_input as check_policy_input, -}; -use hellas_sdk::paid_client::{PaidWorkSession as OpenPaidChannel, bind_paid_endpoint}; +use hellas_sdk::paid_client::{PaidWorkResult, check_evaluate_input}; use hellas_work::work_store::journal::MAX_RECORD_BYTES; use iroh::{EndpointId, SecretKey}; use std::net::SocketAddr; @@ -353,8 +353,6 @@ fn inspect_prepared( Ok(()) } -use hellas_sdk::paid_client::InputIdentities; - async fn inspect_chain(validators: &[String]) -> CliResult<()> { anyhow::ensure!( validators.len() == 6, @@ -441,13 +439,13 @@ async fn open_paid_channel( endpoint: iroh::Endpoint, settlement_key: Secp256k1Signer, assurance: hellas_rpc::Assurance, -) -> CliResult { +) -> CliResult { anyhow::ensure!( !args.payment_coins.is_empty(), "at least one --payment-coin is required" ); let provider_trust = paid_provider_trust(args, assurance)?; - OpenPaidChannel::open( + PaidWorkSession::open( hellas_sdk::paid_client::PaidWorkOptions { config: load_work_config(&args.work_config)?, journal_root: args.journal_root.clone(), @@ -526,8 +524,6 @@ fn relative_deadlines(current: u64, args: &RunArgs) -> CliResult { ) .map_err(Into::into) } -#[cfg(test)] -use hellas_sdk::paid_client::check_genesis_payload; fn read_prepared_work_input(path: &Path) -> CliResult { let bytes = super::read_bounded_regular_file(path, "prepared paid input", MAX_RECORD_BYTES)?; diff --git a/crates/cli/src/commands/paid_work/gateway.rs b/crates/cli/src/commands/paid_work/gateway.rs index d8a18324..c5a8d696 100644 --- a/crates/cli/src/commands/paid_work/gateway.rs +++ b/crates/cli/src/commands/paid_work/gateway.rs @@ -77,7 +77,7 @@ trait GatewayEvent: Send + 'static { fn prefix(event: hellas_rpc::OutputEventEnvelope) -> CliResult where Self: Sized; - fn completed(output: PaidOutput) -> CliResult> + fn completed(output: PaidWorkResult) -> CliResult> where Self: Sized; fn is_terminal(&self) -> bool; @@ -107,7 +107,7 @@ struct Provider { policy: ProviderChannelPolicy, assurance: hellas_rpc::Assurance, /// A setup/channel journal has a single owner even with concurrent HTTP calls. - serial: AsyncMutex>, + serial: AsyncMutex>, pending: AtomicUsize, cache: Mutex, unavailable_until: Mutex>, @@ -333,6 +333,8 @@ pub async fn load_gateway_backend( unavailable_until: Mutex::new(None), })); let provider = providers.last().expect("provider just added"); + // Validated now so a bad pin fails startup before the gateway binds; + // the anchor itself is recomputed when the channel opens. paid_provider_trust(&provider.args, provider.assurance)?; } let gateway = Arc::new(PaidGateway { @@ -383,7 +385,7 @@ impl PaidGateway { let eligible = self .providers .iter() - .filter(|provider| check_policy_input(&provider.policy, &prepared).is_ok()) + .filter(|provider| check_evaluate_input(&provider.policy, &prepared).is_ok()) .collect::>(); anyhow::ensure!( !eligible.is_empty(), @@ -432,7 +434,9 @@ impl PaidGateway { let (sender, receiver) = mpsc::channel::>(OUTPUT_BUFFER_EVENTS); let (overflow, overflow_receiver) = watch::channel(false); let output_budget = Arc::new(Semaphore::new(OUTPUT_BUFFER_BYTES)); - let (initial_provider, _initial_route) = candidates.first().expect("paid provider exists"); + let Some((initial_provider, _initial_route)) = candidates.first() else { + anyhow::bail!("paid gateway has no provider candidate"); + }; let span = hellas_rpc::request_span!( target: "hellas_request", "paid.gateway", hellas.provider.id = %initial_provider.args.provider, @@ -917,7 +921,7 @@ fn prepare_request( )) } -fn output_events(output: PaidOutput) -> CliResult> { +fn output_events(output: PaidWorkResult) -> CliResult> { let events = hellas_rpc::protocol::work::decode_transcript(&output.transcript, MAX_RECORD_BYTES)?; let verified = hellas_rpc::evaluate::verify_output_events_for_producer( @@ -968,7 +972,7 @@ impl GatewayEvent for ExecutionEvent { tokens: delta.token_bytes(), }) } - fn completed(output: PaidOutput) -> CliResult> { + fn completed(output: PaidWorkResult) -> CliResult> { output_events(output) } fn is_terminal(&self) -> bool { @@ -992,7 +996,7 @@ impl GatewayEvent for FetchEvent { event.payload(), )?) } - fn completed(output: PaidOutput) -> CliResult> { + fn completed(output: PaidWorkResult) -> CliResult> { let events = hellas_rpc::protocol::work::decode_transcript( &output.transcript, hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES, diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index a408a4f9..74742fef 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1328,7 +1328,15 @@ async fn async_main() { }, http_fetch: http_fetch_config .map(|path| -> anyhow::Result<_> { - Ok(serde_json::from_slice(&std::fs::read(path)?)?) + // 4 MiB, the same bound the pool-file loader + // takes from hellas-work, which this binary + // links only in some feature builds. + let bytes = commands::read_bounded_regular_file( + &path, + "HTTP gateway config", + 4 << 20, + )?; + Ok(serde_json::from_slice(&bytes)?) }) .transpose()?, output_cache: cache_options, From 366a2910127f04e4336deba2a378a2095205aa7c Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Sun, 27 Sep 2026 21:23:11 +0200 Subject: [PATCH 29/32] Harden paid Fetch provider boundaries and capacity recovery --- Cargo.lock | 1 + crates/cli/src/commands/serve/mod.rs | 1 + crates/cli/src/commands/serve/node/tests.rs | 12 +- .../executor/src/executor/actor/execution.rs | 1 + crates/executor/src/executor/actor/mod.rs | 32 ++- .../executor/src/executor/actor/paid_fetch.rs | 143 ++++++++-- crates/sdk/Cargo.toml | 3 +- crates/sdk/src/lib.rs | 3 + crates/sdk/src/paid_client/tests.rs | 34 +-- crates/sdk/src/paid_client/tests/recovery.rs | 222 +++++++++++++++ crates/sdk/src/paid_provider.rs | 107 ++++++- crates/sdk/src/provider.rs | 28 +- crates/sdk/src/provider/tests.rs | 186 +++++++++++++ crates/sdk/src/test_support.rs | 262 ++++++++++++++++++ docs/http-gateway.md | 8 + docs/paid-gateway.md | 6 +- nix/ci.nix | 6 + 17 files changed, 971 insertions(+), 84 deletions(-) create mode 100644 crates/sdk/src/paid_client/tests/recovery.rs create mode 100644 crates/sdk/src/provider/tests.rs create mode 100644 crates/sdk/src/test_support.rs diff --git a/Cargo.lock b/Cargo.lock index a266acc9..c980900a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3049,6 +3049,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-stream", + "bytes", "futures", "futures-core", "hellas-attestation", diff --git a/crates/cli/src/commands/serve/mod.rs b/crates/cli/src/commands/serve/mod.rs index f533392f..b42787f4 100644 --- a/crates/cli/src/commands/serve/mod.rs +++ b/crates/cli/src/commands/serve/mod.rs @@ -165,6 +165,7 @@ async fn run_with_store( ); work_runner = Some(node::WorkRunnerConfig { network: work.chain.network, + genesis_payload_digest: work.chain.genesis_payload_digest, threshold_identity: work.chain.threshold_identity.clone(), journal_root: work.journal_root.clone(), routes: work.routes.clone(), diff --git a/crates/cli/src/commands/serve/node/tests.rs b/crates/cli/src/commands/serve/node/tests.rs index 528d94bf..aa56be71 100644 --- a/crates/cli/src/commands/serve/node/tests.rs +++ b/crates/cli/src/commands/serve/node/tests.rs @@ -619,13 +619,14 @@ async fn two_vouched_peers_receive_their_distinct_configured_offers() { let runner = WorkRunner::discover( WorkRunnerConfig { network: network(), + genesis_payload_digest: [0; 32].into(), threshold_identity: threshold_identity(), journal_root: dir.path().to_path_buf(), routes: configured_routes(&[ (first_peer, first.bond_edge(), first.client().party_key()), (second_peer, second.bond_edge(), second.client().party_key()), ]), - validators: Vec::new(), + validators: vec!["ws://unused.invalid".to_owned()], poll: Duration::from_millis(1), max_observation_age: Duration::from_secs(5), settlement_key: provider(), @@ -723,6 +724,7 @@ fn discover_two_route_runner( match WorkRunner::discover( WorkRunnerConfig { network: network(), + genesis_payload_digest: [0; 32].into(), threshold_identity: threshold_identity(), journal_root: root.to_path_buf(), routes: configured_routes(&[ @@ -737,7 +739,7 @@ fn discover_two_route_runner( second.client().party_key(), ), ]), - validators: Vec::new(), + validators: vec!["ws://unused.invalid".to_owned()], poll: Duration::from_millis(1), max_observation_age: Duration::from_secs(5), settlement_key: provider(), @@ -2477,10 +2479,11 @@ fn runner(root: &Path, policy: ProviderChannelPolicy, mount: &MountedWork) -> Wo match WorkRunner::discover( WorkRunnerConfig { network: network(), + genesis_payload_digest: [0; 32].into(), threshold_identity: threshold_identity(), journal_root: root.to_path_buf(), routes: configured_routes(&[(default_route_peer(), bond_edge(), client().party_key())]), - validators: Vec::new(), + validators: vec!["ws://unused.invalid".to_owned()], poll: Duration::from_millis(1), max_observation_age: Duration::from_secs(5), settlement_key: provider(), @@ -2962,10 +2965,11 @@ impl RunningPaidNode { let runner = match WorkRunner::discover( WorkRunnerConfig { network: network(), + genesis_payload_digest: [0; 32].into(), threshold_identity: threshold_identity(), journal_root: root.to_path_buf(), routes: configured_routes(&[(peer, bond_edge(), client().party_key())]), - validators: Vec::new(), + validators: vec!["ws://unused.invalid".to_owned()], poll: Duration::from_millis(1), max_observation_age: Duration::from_secs(5), settlement_key: provider(), diff --git a/crates/executor/src/executor/actor/execution.rs b/crates/executor/src/executor/actor/execution.rs index 89ee39e7..57176a1a 100644 --- a/crates/executor/src/executor/actor/execution.rs +++ b/crates/executor/src/executor/actor/execution.rs @@ -809,6 +809,7 @@ impl Executor { } pub(super) fn dispatch_next_fetch(&mut self) { + self.dispatch_paid_fetches(); while self.active_fetches < self.fetch_max_in_flight { let Some(pending) = self.pending_fetches.pop_front() else { return; diff --git a/crates/executor/src/executor/actor/mod.rs b/crates/executor/src/executor/actor/mod.rs index 69b65060..8b2087ea 100644 --- a/crates/executor/src/executor/actor/mod.rs +++ b/crates/executor/src/executor/actor/mod.rs @@ -66,6 +66,7 @@ struct InboxArbiter { preference: TrustedPreference, request_open: bool, owed_open: bool, + owed_paused: bool, } impl Default for InboxArbiter { @@ -74,6 +75,7 @@ impl Default for InboxArbiter { preference: TrustedPreference::Owed, request_open: true, owed_open: true, + owed_paused: false, } } } @@ -123,6 +125,7 @@ pub struct Executor { pub(super) fetch_routes: FetchRouteRegistry, pub(super) fetch_cache: Option>, pub(super) pending_fetches: VecDeque, + pending_paid_fetches: VecDeque, pending_fetch_quota_cancellations: VecDeque, pending_fetch_quota_settlements: VecDeque, pub(super) fetch_max_in_flight: usize, @@ -402,6 +405,7 @@ impl Executor { fetch_access_policy: config.fetch_access_policy, fetch_routes: config.fetch_routes, pending_fetches: VecDeque::new(), + pending_paid_fetches: VecDeque::new(), fetch_cache: execution::cache::FetchCache::open(config.output_cache)?, pending_fetch_quota_cancellations: VecDeque::new(), pending_fetch_quota_settlements: VecDeque::new(), @@ -419,14 +423,18 @@ impl Executor { async fn run(mut self) { let mut arbiter = InboxArbiter::default(); - while let Some(message) = recv_next( - &mut self.request_rx, - &mut self.owed_rx, - &mut self.completion_rx, - &mut arbiter, - ) - .await - { + loop { + arbiter.owed_paused = self.pending_paid_fetches.len() >= EXECUTOR_OWED_MAILBOX_CAPACITY; + let Some(message) = recv_next( + &mut self.request_rx, + &mut self.owed_rx, + &mut self.completion_rx, + &mut arbiter, + ) + .await + else { + break; + }; match message { ExecutorInbox::Completion(completion) => { let evaluate_finished = self.handle_completion(completion).await; @@ -475,6 +483,9 @@ impl Executor { let ready = self.owed_rx.len(); let mut admitted = 0; for _ in 0..ready { + if self.pending_paid_fetches.len() >= EXECUTOR_OWED_MAILBOX_CAPACITY { + break; + } let Ok(request) = self.owed_rx.try_recv() else { break; }; @@ -616,7 +627,7 @@ async fn recv_next( TrustedPreference::Owed => { tokio::select! { biased; - owed = owed_rx.recv(), if arbiter.owed_open => { + owed = owed_rx.recv(), if arbiter.owed_open && !arbiter.owed_paused => { match owed { Some(request) => Some(ExecutorInbox::Owed(request)), None => { @@ -645,7 +656,7 @@ async fn recv_next( Some(completion) = completion_rx.recv() => { Some(ExecutorInbox::Completion(completion)) } - owed = owed_rx.recv(), if arbiter.owed_open => { + owed = owed_rx.recv(), if arbiter.owed_open && !arbiter.owed_paused => { match owed { Some(request) => Some(ExecutorInbox::Owed(request)), None => { @@ -785,6 +796,7 @@ mod mailbox_tests { fetch_access_policy: FetchAccessPolicy::trusted_callers([caller]), fetch_routes: FetchRouteRegistry::default(), pending_fetches: VecDeque::new(), + pending_paid_fetches: VecDeque::new(), pending_fetch_quota_cancellations: VecDeque::new(), pending_fetch_quota_settlements: VecDeque::new(), fetch_max_in_flight: 1, diff --git a/crates/executor/src/executor/actor/paid_fetch.rs b/crates/executor/src/executor/actor/paid_fetch.rs index 7017325b..cce5654b 100644 --- a/crates/executor/src/executor/actor/paid_fetch.rs +++ b/crates/executor/src/executor/actor/paid_fetch.rs @@ -14,15 +14,20 @@ use crate::executor::ExecutorCompletion; use crate::fetch_policy::FetchRoute; use crate::fetch_provider::FetchCall; +pub(super) struct PendingPaidFetch { + input: PreparedFetchInput, + progress: Option, + reply: oneshot::Sender, ExecutorError>>, + span: tracing::Span, +} + impl Executor { /// Admits one journaled paid Fetch, or refuses it before any provider /// work begins. /// - /// Capacity refusal is deliberately fail-fast: the work layer journals it - /// as that job's terminal failure rather than queueing, unlike paid - /// Evaluate, which waits for a dispatch slot. A paid burst beyond - /// `fetch_max_in_flight` therefore ends the excess jobs, so operators - /// should size the limit for paid bursts, not for average load. + /// Already accepted work waits in a bounded FIFO when dispatch is full. + /// Backpressure then reaches the owed mailbox; peer requests and completion + /// handling remain live while the upstream is busy. pub(super) fn start_paid_fetch( &mut self, input: PreparedFetchInput, @@ -30,6 +35,15 @@ impl Executor { reply: oneshot::Sender, ExecutorError>>, span: tracing::Span, ) { + if self.active_fetches >= self.fetch_max_in_flight { + self.pending_paid_fetches.push_back(PendingPaidFetch { + input, + progress, + reply, + span, + }); + return; + } let prepared = self.prepare_paid_fetch(input); let (entry, session, request, policy) = match prepared { Ok(prepared) => prepared, @@ -120,11 +134,6 @@ impl Executor { ), ExecutorError, > { - if self.active_fetches >= self.fetch_max_in_flight { - return Err(ExecutorError::ResourceExhausted( - "fetch concurrency limit reached".into(), - )); - } let policy = *input.policy(); let parts = input.into_parts(); let request = hellas_rpc::fetch::verify_input_events(&parts.fetch_input_transcript) @@ -163,6 +172,16 @@ impl Executor { })?; Ok((entry, session, request, policy)) } + + pub(super) fn dispatch_paid_fetches(&mut self) { + while self.active_fetches < self.fetch_max_in_flight { + let Some(pending) = self.pending_paid_fetches.pop_front() else { + break; + }; + // The durable invocation is owed even if its original waiter left. + self.start_paid_fetch(pending.input, pending.progress, pending.reply, pending.span); + } + } } #[cfg(test)] @@ -595,7 +614,7 @@ mod tests { } #[tokio::test] - async fn paid_fetch_at_capacity_fails_fast_instead_of_queueing() { + async fn paid_fetch_waits_for_capacity_without_blocking_the_actor() { let provider = BlockingFetchProvider::new(environment().manifest_id()); let handle = spawn_executor( routes( @@ -620,31 +639,103 @@ mod tests { .await .expect("first paid fetch reaches the provider"); - // The single slot is occupied: the refusal must arrive while the - // first fetch is still blocked, not after it drains. - let second = run_paid_fetch( - &handle, - prepared_input(input_events( - Retention::Ephemeral, - Assurance::ProducerSigned, - )), - None, - ) - .await; - assert!( - matches!(second, Err(ExecutorError::ResourceExhausted(error)) if error == "fetch concurrency limit reached") - ); + let second = tokio::spawn({ + let handle = handle.clone(); + async move { + run_paid_fetch( + &handle, + prepared_input(input_events( + Retention::Ephemeral, + Assurance::ProducerSigned, + )), + None, + ) + .await + } + }); + // Peer-facing requests remain serviceable while both paid jobs wait. + timeout(Duration::from_secs(1), handle.get_stats_handle()) + .await + .unwrap() + .unwrap(); + assert!(!second.is_finished()); + assert_eq!(provider.started(), 1); provider.release(); let output = first .await .expect("first paid fetch task completes") .expect("first paid fetch succeeds"); - assert_eq!(provider.started(), 1); + second.await.unwrap().expect("queued paid fetch succeeds"); + assert_eq!(provider.started(), 2); verify_output_events(first_commitment, Assurance::ProducerSigned, &output) .expect("first paid fetch output verifies"); } + #[tokio::test] + async fn full_paid_queue_backpressures_ingress_but_keeps_completions_live() { + timeout(Duration::from_secs(10), async { + let provider = BlockingFetchProvider::new(environment().manifest_id()); + let handle = spawn_executor( + routes( + Arc::new(provider.clone()), + stub_adaptor(environment().manifest_id()), + FetchRoutePolicy::default(), + ), + 1, + ) + .await; + let input = prepared_input(input_events( + Retention::Ephemeral, + Assurance::ProducerSigned, + )); + let request = |reply| crate::executor::ExecutorOwedRequest::RunPaidFetch { + span: tracing::Span::none(), + input: Box::new(input.clone()), + progress: None, + reply, + }; + let (reply, first) = oneshot::channel(); + handle.owed_tx.send(request(reply)).await.unwrap(); + while provider.started() == 0 { + tokio::task::yield_now().await; + } + let mut pending = Vec::new(); + // One bounded actor FIFO plus its bounded ingress mailbox. + for _ in 0..2 * super::super::EXECUTOR_OWED_MAILBOX_CAPACITY { + let (reply, result) = oneshot::channel(); + handle.owed_tx.send(request(reply)).await.unwrap(); + pending.push(result); + } + let (reply, last) = oneshot::channel(); + let blocked = handle.owed_tx.send(request(reply)); + tokio::pin!(blocked); + assert!( + timeout(Duration::from_millis(50), &mut blocked) + .await + .is_err() + ); + handle.get_stats_handle().await.unwrap(); + assert_eq!(handle.owed_tx.capacity(), 0); + assert_eq!(provider.started(), 1); + // An abandoned waiter cannot discard an already owed invocation. + drop(pending.pop().unwrap()); + provider.release(); + blocked.await.unwrap(); + first.await.unwrap().unwrap(); + for result in pending { + result.await.unwrap().unwrap(); + } + last.await.unwrap().unwrap(); + assert_eq!( + provider.started(), + 2 + 2 * super::super::EXECUTOR_OWED_MAILBOX_CAPACITY + ); + }) + .await + .expect("queue drains once the upstream is available"); + } + #[tokio::test] async fn paid_fetch_streams_signed_progress_and_returns_signed_output() { let provider = MockFetchProvider::new(environment().manifest_id()); diff --git a/crates/sdk/Cargo.toml b/crates/sdk/Cargo.toml index de4048e8..ea25a28e 100644 --- a/crates/sdk/Cargo.toml +++ b/crates/sdk/Cargo.toml @@ -74,4 +74,5 @@ tracing = { workspace = true, optional = true } url = { version = "2", optional = true } [dev-dependencies] -tokio.workspace = true +bytes.workspace = true +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/sdk/src/lib.rs b/crates/sdk/src/lib.rs index 92abd115..4cac0e85 100644 --- a/crates/sdk/src/lib.rs +++ b/crates/sdk/src/lib.rs @@ -58,3 +58,6 @@ pub mod paid_client; #[cfg(feature = "paid-work")] pub mod work_provision; + +#[cfg(all(test, feature = "paid-work"))] +mod test_support; diff --git a/crates/sdk/src/paid_client/tests.rs b/crates/sdk/src/paid_client/tests.rs index 26fe890f..fbb12230 100644 --- a/crates/sdk/src/paid_client/tests.rs +++ b/crates/sdk/src/paid_client/tests.rs @@ -1,10 +1,8 @@ use super::*; +use crate::test_support::enrollment; +mod recovery; use hellas_rpc::pb::execute::{OpenRequest, OpenResponse, open_response}; -use hellas_rpc::{ - Assurance, Digest, PlatformCredential, PlatformEnrollment, ProducerSigningKey, - ProviderEnrollmentBundle, ProviderGenesisStatement, PublicKey, RootKind, RootProof, - SignedProviderGenesis, -}; +use hellas_rpc::{Assurance, ProducerSigningKey, ProviderEnrollmentBundle, PublicKey}; use hellas_wire::{MethodMarker, StreamTransport, WireStatus}; #[tokio::test] @@ -75,32 +73,6 @@ async fn insufficient_collateral_is_rejected_before_network_or_journal_creation( endpoint.close().await; } -fn enrollment(peer: EndpointId) -> (ProviderEnrollmentBundle, ProducerSigningKey) { - let root = ProducerSigningKey::from_secret_bytes([1; 32]).unwrap(); - let producer = ProducerSigningKey::from_secret_bytes([2; 32]).unwrap(); - let statement = ProviderGenesisStatement { - root_kind: RootKind::Software, - root_public_key: root.public_key(), - producer_public_key: producer.public_key(), - transport_public_key: PublicKey::Ed25519(*peer.as_bytes()), - platform_credential: PlatformCredential::Absent, - installation_nonce: [3; 32], - }; - let proof = root - .sign_digest(Digest::hash(&statement.canonical_bytes())) - .unwrap(); - ( - ProviderEnrollmentBundle { - genesis: SignedProviderGenesis { - statement, - root_proof: RootProof::Software(proof), - }, - platform: PlatformEnrollment::Absent, - }, - producer, - ) -} - async fn serve_open( transport: &IrohTransport, bundle: ProviderEnrollmentBundle, diff --git a/crates/sdk/src/paid_client/tests/recovery.rs b/crates/sdk/src/paid_client/tests/recovery.rs new file mode 100644 index 00000000..68c544de --- /dev/null +++ b/crates/sdk/src/paid_client/tests/recovery.rs @@ -0,0 +1,222 @@ +use super::*; +use crate::test_support::{BODY, PRICE, PaidFixture, signer}; +use hellas_rpc::fetch::{ + FetchOutputTranscriptBuilder, encode_fetch_event_payload, encode_fetch_terminal_payload, + verify_input_events, +}; +use hellas_rpc::output::{OutputEvent, StopReason, TextChannel}; +use hellas_rpc::pb::work::WorkDelivered; +use hellas_rpc::protocol::work::PrivateRecord as _; +use hellas_rpc::services::work::{Work, WorkServer}; +use hellas_wire::Dispatcher; +use hellas_work::work::{ + BackendFault, ObservationTime, PaidWorkBackend, PreparedFetchInput, run_accepted_work, +}; +use std::sync::atomic::AtomicUsize; + +const RESPONSE: &str = "sdk-private-response"; + +#[derive(Default)] +struct Backend(AtomicUsize); + +impl PaidWorkBackend for Backend { + async fn fetch( + &self, + input: PreparedFetchInput, + ) -> Result, BackendFault> { + self.0.fetch_add(1, Ordering::SeqCst); + let request = verify_input_events(&input.into_parts().fetch_input_transcript).unwrap(); + let key = ProducerSigningKey::from_secret_bytes([2; 32]).unwrap(); + let mut output = + FetchOutputTranscriptBuilder::new(request.input_commitment, request.assurance, &key); + output + .push_event( + encode_fetch_event_payload(&OutputEvent::TextDelta { + index: 0, + delta: RESPONSE.into(), + channel: TextChannel::Output, + }) + .unwrap(), + ) + .unwrap(); + Ok(output + .finish( + encode_fetch_terminal_payload(&OutputEvent::Finished { + stop_reason: StopReason::EndOfText, + usage: None, + }) + .unwrap(), + ) + .unwrap()) + } +} + +fn assert_no_payloads(root: &std::path::Path) { + for entry in std::fs::read_dir(root).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + assert_no_payloads(&path); + } else { + let bytes = std::fs::read(&path).unwrap(); + for payload in [BODY, RESPONSE.as_bytes()] { + assert!( + !bytes.windows(payload.len()).any(|window| window == payload), + "payload in {}", + path.display() + ); + } + } + } +} + +#[tokio::test] +async fn restart_pays_delivered_fetch_once_and_skips_lost_payload_jobs() { + tokio::time::timeout(Duration::from_secs(15), async { + let fixture = PaidFixture::new(); + let provider = fixture.provider(); + let mut client = fixture.client(); + let request = client.propose(&fixture.proposal()).unwrap(); + let lost = client.accepted(&provider.accept(&request)).unwrap(); + let request = client.propose(&fixture.proposal()).unwrap(); + let delivered = client.accepted(&provider.accept(&request)).unwrap(); + assert_ne!(lost, delivered); + let backend = Backend::default(); + run_accepted_work(&provider, &fixture.ready, &backend, delivered) + .await + .unwrap(); + let request = client.request_delivery(delivered, &[8; 32]).unwrap(); + let result = provider.deliver(&request, &[8; 32]).unwrap(); + client + .receive( + delivered, + &fixture.ready, + &WorkDelivered { + result: result.result.encode(), + provider_signature: result.signature.as_bytes().to_vec(), + transcript: result.transcript, + }, + ) + .unwrap(); + // Crash after verified delivery, before creating any payment certificate. + assert!(client.state().last_payment().is_none()); + drop(client); + assert_no_payloads(fixture.root.path()); + + let server = Endpoint::builder(presets::Minimal) + .alpns(vec![Work::ALPN.as_bytes().to_vec()]) + .bind_addr("127.0.0.1:0".parse::().unwrap()) + .unwrap() + .bind() + .await + .unwrap(); + let (bundle, key) = enrollment(server.id()); + let trust = hellas_client::ProviderTrustAnchor { + expected_genesis: bundle.content_id(), + required_assurance: Assurance::ProducerSigned, + apple_app_attest: None, + }; + let serving = tokio::spawn({ + let server = server.clone(); + let provider = provider.clone(); + async move { + let mut tasks = tokio::task::JoinSet::new(); + while let Some(incoming) = server.accept().await { + let bundle = bundle.clone(); + let key = key.clone(); + let provider = provider.clone(); + tasks.spawn(async move { + let transport = IrohTransport::new(incoming.await.unwrap()); + serve_open::(&transport, bundle, key) + .await; + let handler = WorkServer(provider); + while let Ok(Some(inbound)) = transport.accept().await { + Dispatcher::::dispatch(&handler, inbound) + .await + .unwrap(); + } + }); + } + while let Some(result) = tasks.join_next().await { + result.unwrap(); + } + } + }); + for restart in 0..2 { + let store = fixture.store(Role::Client); + assert!(store.state().job_by_id(lost).is_some()); + assert_eq!(store.state().last_payment().is_some(), restart == 1); + for job in store.state().jobs() { + assert!(job.prepared_input().is_empty()); + assert!(job.transcript().is_empty()); + assert_eq!(job.result().is_some(), job.work_id() == delivered); + } + if restart == 0 { + assert!( + store + .state() + .job_by_id(delivered) + .unwrap() + .result() + .is_some() + ); + } + let client = ClientService::new(ClientEndpoint::recover(store, signer(1)).unwrap()); + client + .observer() + .observe_ready( + fixture.ready.clone(), + ObservationTime::now(), + Duration::from_secs(60), + ) + .unwrap(); + let endpoint = bind_paid_endpoint(SecretKey::from_bytes(&[5; 32])) + .await + .unwrap(); + let mut session = PaidWorkSession { + args: PaidWorkOptions { + config: fixture.config.clone(), + journal_root: fixture.root.path().join("client"), + provider: server.id(), + provider_addrs: server.bound_sockets(), + provider_trust: Some(trust.clone()), + bond: fixture.descriptor.bond_edge(), + payment_funding: Funding::new( + hellas_kernel::List::empty(hellas_kernel::CoinId::from_bytes([0; 32])), + hellas_kernel::List::empty(hellas_kernel::CoinId::from_bytes([0; 32])), + ), + omission_bond: 601, + acceptance_blocks: 50, + terminal_blocks: 100, + payment_blocks: 200, + timeout: Duration::from_secs(5), + }, + descriptor: fixture.descriptor.clone(), + dialer: ProviderDialer::new( + server.id(), + server.bound_sockets(), + endpoint.clone(), + Some(trust.clone()), + ), + client, + observer: None, + needs_recovery: true, + }; + assert!(session.run(None, true, None).await.unwrap().is_none()); + assert!(!session.needs_recovery); + assert_eq!( + provider + .with_state(|state| state.ledger().credited_cumulative()) + .unwrap(), + PRICE + ); + drop(session); + endpoint.close().await; + assert_no_payloads(fixture.root.path()); + } + assert_eq!(backend.0.load(Ordering::SeqCst), 1); + server.close().await; + serving.await.unwrap(); + }) + .await + .expect("recovery completes without a validator or payload replay"); +} diff --git a/crates/sdk/src/paid_provider.rs b/crates/sdk/src/paid_provider.rs index 6ac2cb69..f65644c3 100644 --- a/crates/sdk/src/paid_provider.rs +++ b/crates/sdk/src/paid_provider.rs @@ -50,6 +50,12 @@ pub enum PaidProviderError { InvalidObservationPolicy, #[error("the paid-work clock requires at least one validator")] NoValidators, + #[error("validator has no finalized block 1 for genesis authentication")] + MissingGenesis, + #[error("validator genesis does not match the configured payload digest")] + GenesisMismatch, + #[error(transparent)] + BlockSource(#[from] hellas_work::work_close::BlockSourceError), #[error(transparent)] Endpoint(#[from] hellas_work::work::EndpointError), #[error(transparent)] @@ -137,6 +143,8 @@ impl WorkHandler for UnmountedWork { pub struct WorkRunnerConfig { /// The network the journals are keyed and the signatures bound to. pub network: NetworkId, + /// The parent payload digest authenticated by finalized block 1. + pub genesis_payload_digest: Digest, /// The threshold identity finalized blocks must authenticate under. pub threshold_identity: Vec, /// The configured root the setup journals live under. @@ -683,6 +691,7 @@ pub struct WorkRunner { poll: Duration, validators: Vec, consensus_verifier: ConsensusVerifier, + genesis_payload_digest: Digest, } impl WorkRunner { @@ -797,6 +806,7 @@ impl WorkRunner { poll: config.poll, validators: config.validators, consensus_verifier, + genesis_payload_digest: config.genesis_payload_digest, }) } @@ -889,12 +899,13 @@ impl WorkRunner { pub async fn run(self, stop: oneshot::Receiver<()>) { let validators = self.validators.clone(); let verifier = self.consensus_verifier.clone(); + let genesis = self.genesis_payload_digest; let next = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); self.run_over(stop, move || { let validators = validators.clone(); let verifier = verifier.clone(); let next = next.clone(); - async move { connect_chain(&validators, verifier, &next).await } + async move { connect_chain(&validators, verifier, genesis, &next).await } }) .await; } @@ -905,6 +916,7 @@ impl WorkRunner { async fn connect_chain( validators: &[String], verifier: ConsensusVerifier, + genesis: Digest, next: &std::sync::atomic::AtomicUsize, ) -> Option { if validators.is_empty() { @@ -914,11 +926,102 @@ async fn connect_chain( for url in validators.iter().cycle().skip(start).take(validators.len()) { match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { Ok(client) => { + let source = WorkBlocks::new(client); + if let Err(error) = authenticate_genesis(&source, genesis).await { + warn!(validator = %url, %error, "validator failed genesis authentication"); + continue; + } info!(validator = %url, "the paid-work clock reads and submits here"); - return Some(WorkBlocks::new(client)); + return Some(source); } Err(error) => warn!(validator = %url, %error, "a configured validator did not answer"), } } None } + +async fn authenticate_genesis( + source: &impl FinalizedBlocks, + expected: Digest, +) -> Result<(), PaidProviderError> { + let first = source + .block_at(1) + .await? + .ok_or(PaidProviderError::MissingGenesis)?; + if &first.parent != expected.as_bytes() { + return Err(PaidProviderError::GenesisMismatch); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use hellas_work::work_close::{BlockSourceError, FinalizedWork}; + + struct Source(Result, BlockSourceError>); + + impl FinalizedBlocks for Source { + async fn latest_height(&self) -> Result, BlockSourceError> { + panic!("genesis authentication must request block 1 directly") + } + + async fn block_at(&self, height: u64) -> Result, BlockSourceError> { + assert_eq!(height, 1); + self.0.clone() + } + } + + #[tokio::test] + async fn validator_connection_requires_the_configured_genesis() { + let block = FinalizedWork { + height: 1, + parent: [7; 32], + payload: [8; 32], + txs: Vec::new(), + }; + authenticate_genesis(&Source(Ok(Some(block.clone()))), [7; 32].into()) + .await + .unwrap(); + assert!(matches!( + authenticate_genesis(&Source(Ok(Some(block))), [9; 32].into()).await, + Err(PaidProviderError::GenesisMismatch) + )); + assert!(matches!( + authenticate_genesis(&Source(Ok(None)), [7; 32].into()).await, + Err(PaidProviderError::MissingGenesis) + )); + assert!(matches!( + authenticate_genesis( + &Source(Err(BlockSourceError::new("offline"))), + [7; 32].into() + ) + .await, + Err(PaidProviderError::BlockSource(_)) + )); + } + + #[test] + fn missing_validators_are_rejected_before_discovering_journals() { + let fixture = crate::test_support::PaidFixture::new(); + let config = &fixture.config; + let runner = WorkRunner::discover( + WorkRunnerConfig { + network: config.chain.network, + genesis_payload_digest: config.chain.genesis_payload_digest, + threshold_identity: config.chain.threshold_identity.clone(), + journal_root: config.journal_root.clone(), + routes: config.routes.clone(), + validators: Vec::new(), + poll: config.poll, + max_observation_age: config.max_observation_age, + settlement_key: crate::test_support::signer(2), + policy: config.provider_policy(), + }, + MountedWork::default(), + MountedSetup::default(), + ); + assert!(matches!(runner, Err(PaidProviderError::NoValidators))); + assert!(!config.journal_root.exists()); + } +} diff --git a/crates/sdk/src/provider.rs b/crates/sdk/src/provider.rs index 9dc5f338..ba885d17 100644 --- a/crates/sdk/src/provider.rs +++ b/crates/sdk/src/provider.rs @@ -16,7 +16,7 @@ use hellas_rpc::services::fetch::{Fetch, Open as FetchOpen}; use hellas_rpc::{ Assurance, OPEN_NONCE_LEN, ProviderEnrollmentBundle, PublicKey, RootProof, open_proof_binding, }; -use hellas_wire::iroh::{IrohTransport, IrohTransportError}; +use hellas_wire::iroh::IrohTransport; use hellas_wire::{ Dispatcher, ServiceMarker, StreamTransport, TransportContext, WireCode, WireStatus, }; @@ -233,6 +233,7 @@ where let runner = crate::paid_provider::WorkRunner::discover( crate::paid_provider::WorkRunnerConfig { network: config.chain.network, + genesis_payload_digest: config.chain.genesis_payload_digest, threshold_identity: config.chain.threshold_identity, journal_root: config.journal_root, routes: config.routes, @@ -257,7 +258,13 @@ where root: options.root, enrollment: options.enrollment, }; - let alpns = vec![::ALPN.as_bytes().to_vec()]; + // A paid provider must not expose a second, unpaid route to the same + // upstream credentials, even when legacy Courtesy callers are configured. + let alpns = if has_paid_work { + Vec::new() + } else { + vec![::ALPN.as_bytes().to_vec()] + }; #[cfg(feature = "paid-work")] let alpns = if has_paid_work { let mut alpns = alpns; @@ -369,7 +376,7 @@ where } return; } - if alpn != Fetch::ALPN.as_bytes() { + if has_paid_work || alpn != Fetch::ALPN.as_bytes() { return; } let server = OpenDispatcher::<_, _, FetchOpen>::new( @@ -396,28 +403,33 @@ where /// stay open for the lifetime of a job and enforce their own application-level /// deadlines, so a wall-clock timeout at this layer would cancel paid work /// mid-delivery. -async fn serve(transport: Arc, server: S) +async fn serve(transport: Arc, server: S) where - S: Dispatcher + Send + Sync + 'static, + T: StreamTransport + Send + Sync + 'static, + T::Error: std::fmt::Display, + S: Dispatcher + Send + Sync + 'static, S::Error: std::fmt::Display + Send + Sync + 'static, { loop { match transport.accept().await { Ok(Some(inbound)) => { - if let Err(error) = Dispatcher::::dispatch(&server, inbound).await { + if let Err(error) = Dispatcher::::dispatch(&server, inbound).await { tracing::warn!(%error, "provider RPC failed"); break; } } - Ok(None) | Err(IrohTransportError::Connection(_)) => break, + Ok(None) => break, Err(error) => { - tracing::warn!(%error, "provider transport failed"); + tracing::debug!(%error, "provider transport ended"); break; } } } } +#[cfg(all(test, feature = "paid-work"))] +mod tests; + struct ProviderOpen { root: Arc, enrollment: ProviderEnrollmentBundle, diff --git a/crates/sdk/src/provider/tests.rs b/crates/sdk/src/provider/tests.rs new file mode 100644 index 00000000..512df1ae --- /dev/null +++ b/crates/sdk/src/provider/tests.rs @@ -0,0 +1,186 @@ +use super::*; +use bytes::Bytes; +use futures::StreamExt as _; +use hellas_rpc::pb::work::{DeliverResultRequest, WorkStreamEvent}; +use hellas_rpc::services::work::StreamResult; +use hellas_wire::mux::{MessagePipe, MuxConfig, MuxTransport, Role}; +use hellas_wire::{DefaultClock, Metadata}; +use tokio::sync::mpsc; +use tokio::time::{Duration, Instant}; + +struct UnusedRoot; + +impl RootProver for UnusedRoot { + async fn prove_statement( + &self, + _: &[u8], + ) -> Result { + panic!("ALPN negotiation must not invoke attestation") + } + + async fn prove_open_binding( + &self, + _: hellas_rpc::Digest, + ) -> Result { + panic!("ALPN negotiation must not invoke attestation") + } +} + +#[tokio::test] +async fn paid_provider_refuses_unpaid_fetch_even_with_legacy_allowed_callers() { + tokio::time::timeout(Duration::from_secs(15), async { + let fixture = crate::test_support::PaidFixture::new(); + let identity = ClientIdentity::from_secret_bytes([4; 32], [2; 32]).unwrap(); + let (enrollment, _) = crate::test_support::enrollment(identity.transport_key().public()); + let mut routes = FetchRouteRegistry::new(); + routes + .register( + FetchRoute::new("openai", "responses"), + FetchRouteEntry::new( + Arc::new( + hellas_providers::OpenAiResponsesFetchProvider::with_bearer("fixture") + .unwrap(), + ), + Arc::new(hellas_providers::ResponsesFetchAdaptorFactory::new( + hellas_rpc::FetchEnvironment::OpenAiResponses, + )), + FetchRoutePolicy::default(), + ) + .unwrap(), + ) + .unwrap(); + let provider = start_fetch_provider(FetchProviderOptions { + port: Some(0), + identity, + enrollment, + root: Arc::new(UnusedRoot), + state_directory: fixture.root.path().join("state"), + routes, + allowed_callers: vec![ + hellas_rpc::ProducerSigningKey::from_secret_bytes([1; 32]) + .unwrap() + .public_key(), + ], + fetch_max_in_flight: 1, + fetch_queue_capacity: 1, + retained_transcript_capacity: 0, + fetch_replay_max_in_flight: 1, + paid_work: Some(fixture.config), + }) + .await + .unwrap(); + let client = Endpoint::builder(presets::Minimal) + .bind_addr("127.0.0.1:0".parse::().unwrap()) + .unwrap() + .bind() + .await + .unwrap(); + let addr = iroh::EndpointAddr::new(provider.node_id()).with_addrs( + provider + .bound_sockets() + .into_iter() + .filter(|addr| addr.is_ipv4()) + .map(|mut addr| { + addr.set_ip(std::net::Ipv4Addr::LOCALHOST.into()); + iroh::TransportAddr::Ip(addr) + }), + ); + assert!( + client + .connect(addr.clone(), Fetch::ALPN.as_bytes()) + .await + .is_err() + ); + for alpn in [ + hellas_rpc::services::work::Work::ALPN, + hellas_rpc::services::work_setup::WorkSetup::ALPN, + ] { + let connection = client.connect(addr.clone(), alpn.as_bytes()).await.unwrap(); + connection.close(0u32.into(), b"tested"); + } + client.close().await; + provider.shutdown().await; + }) + .await + .expect("paid provider protocol negotiation completes"); +} + +struct Pipe(mpsc::UnboundedSender, mpsc::UnboundedReceiver); + +impl MessagePipe for Pipe { + type SendError = std::io::Error; + type RecvError = std::io::Error; + + async fn send_message(&mut self, bytes: Bytes) -> Result<(), Self::SendError> { + self.0 + .send(bytes) + .map_err(|_| std::io::ErrorKind::BrokenPipe.into()) + } + + async fn recv_message(&mut self) -> Result, Self::RecvError> { + Ok(self.1.recv().await) + } +} + +struct SlowStream; + +impl Dispatcher for SlowStream { + type Error = hellas_wire::TransportError; + + async fn dispatch( + &self, + inbound: hellas_wire::Inbound, + ) -> Result<(), Self::Error> { + hellas_rpc::call::dispatch_server_streaming::( + inbound, + |_: DeliverResultRequest| async { + Ok(Box::pin(async_stream::stream! { + yield Ok(WorkStreamEvent::default()); + tokio::time::sleep(Duration::from_secs(61)).await; + yield Ok(WorkStreamEvent::default()); + })) + }, + ) + .await + } +} + +#[tokio::test(start_paused = true)] +async fn paid_stream_survives_a_slow_upstream_and_an_idle_connection() { + let (to_server, server_inbox) = mpsc::unbounded_channel(); + let (to_client, client_inbox) = mpsc::unbounded_channel(); + let client = MuxTransport::spawn::<8, _, _>( + Role::Client, + DefaultClock, + MuxConfig::default(), + Pipe(to_server, client_inbox), + TransportContext::default(), + ); + let server = MuxTransport::spawn::<8, _, _>( + Role::Server, + DefaultClock, + MuxConfig::default(), + Pipe(to_client, server_inbox), + TransportContext::default(), + ); + let serving = tokio::spawn(serve(Arc::new(server), SlowStream)); + for _ in 0..2 { + let started = Instant::now(); + let mut response = hellas_rpc::call::server_streaming::<_, StreamResult>( + &client, + DeliverResultRequest::default(), + Metadata::new(), + ) + .await + .unwrap(); + assert!(response.next().await.unwrap().is_ok()); + assert!(response.next().await.unwrap().is_ok()); + assert!(response.next().await.is_none()); + response.finish().unwrap(); + assert!(started.elapsed() >= Duration::from_secs(61)); + tokio::time::sleep(Duration::from_secs(121)).await; + assert!(!serving.is_finished()); + } + serving.abort(); + let _ = serving.await; +} diff --git a/crates/sdk/src/test_support.rs b/crates/sdk/src/test_support.rs new file mode 100644 index 00000000..23db1cd6 --- /dev/null +++ b/crates/sdk/src/test_support.rs @@ -0,0 +1,262 @@ +//! A funded, metadata-only Fetch channel for SDK boundary tests. +use crate::work_config::{ChainCrossCheck, WorkConfig}; +use hellas_kernel::{ + BlockHeight, Decode as _, Edge, EdgeId, EdgeValues, Fees, Key, LeaseSlots, List, + MAX_EDGE_OUTPUTS, NetworkId, Parties, Payout, PendingSlot, RegistryChunk, RegistryNamespace, + RegistryRecordTag, Secp256k1Signer, Secp256k1Verifier, Terms, TermsHash, WorkPaymentTerms, + WorkStakeBondTerms, work_payment_settlement, +}; +use hellas_rpc::protocol::work::{JobDeadlines, PaidChannelPolicyV1, private_policy_commitment}; +use hellas_rpc::protocol::work_fetch::{ + FetchRoutePolicy, PaidFetchPolicyV1, PreparedPaidFetchInputV1, fetch_route_commitment, +}; +use hellas_rpc::protocol::work_profile::PaidWorkPolicy; +use hellas_rpc::protocol::work_setup::{ + ObservedChannel, ReadyChannel, WorkChannelConfig, WorkChannelDescriptor, payment_terms_hash, +}; +use hellas_rpc::{Assurance, FetchEnvironment, ProducerSigningKey, Retention}; +use hellas_work::work::{ClientEndpoint, JobProposal, ProviderEndpoint, WorkService}; +use hellas_work::work_store::{ChannelStore, Role, SetupOrigin}; + +use hellas_rpc::{ + Digest, PlatformCredential, PlatformEnrollment, ProviderEnrollmentBundle, + ProviderGenesisStatement, PublicKey, RootKind, RootProof, SignedProviderGenesis, +}; +use iroh::EndpointId; + +pub(crate) const PRICE: u64 = 10; +pub(crate) const BODY: &[u8] = br#"{"input":"sdk-private-request"}"#; +const HORIZON: u64 = 500; +const OMISSION_BOND: u64 = 601; + +pub(crate) struct PaidFixture { + pub root: tempfile::TempDir, + pub ready: ReadyChannel, + pub descriptor: WorkChannelDescriptor, + pub config: WorkConfig, +} + +pub(crate) fn signer(byte: u8) -> Secp256k1Signer { + Secp256k1Signer::from_secret_scalar([byte; 32]).unwrap() +} + +impl PaidFixture { + pub fn new() -> Self { + let root = tempfile::tempdir().unwrap(); + let network = NetworkId::new("sdk-paid-test").unwrap(); + let bond_id = EdgeId::from_bytes([11; 32]); + let payment_id = EdgeId::from_bytes([12; 32]); + let values = EdgeValues::new(1000, 200, Fees::ZERO); + let channel_policy = PaidChannelPolicyV1 { + compute_credit_limit: 40, + delivery_credit_limit: 40, + }; + let route = FetchRoutePolicy::sealed_route("openai", "responses").unwrap(); + let execution_policy = PaidWorkPolicy::Fetch { + policy: PaidFetchPolicyV1 { + allowed_environment: FetchEnvironment::OpenAiResponses.manifest_id(), + route_commitment: fetch_route_commitment(&route.canonical_body_bytes()).unwrap(), + max_request_body_bytes: 4096, + max_output_events: 64, + max_output_bytes: 16384, + max_spool_bytes: 65536, + max_encoded_result_frame: 65536, + max_encoded_prepared_input: 65536, + dispatch_margin_blocks: 4, + delivery_margin_blocks: 2, + oracle_grace_blocks: 6, + fixed_price: PRICE, + }, + route, + }; + let bond_terms = WorkStakeBondTerms { + parties: Parties::new(signer(2).party_key(), signer(1).party_key()), + timeout: BlockHeight::new(HORIZON), + timeout_outputs: List::take( + [Payout::new(signer(2).party_key(), 64); MAX_EDGE_OUTPUTS], + 1, + ), + max_job_price: 40, + }; + let terms = WorkPaymentTerms { + bond_edge: bond_id, + bond_terms: bond_terms.clone(), + private_policy_commitment: private_policy_commitment( + network, + &[9; 32], + &channel_policy, + ), + omit_response_blocks: hellas_kernel::MIN_OMIT_RESPONSE_BLOCKS, + start_validity_blocks: 8, + omission_bond: OMISSION_BOND, + }; + let descriptor = WorkChannelDescriptor::open(WorkChannelConfig { + network, + payment_edge: payment_id, + payment_terms: terms.clone(), + policy_salt: [9; 32], + channel_policy, + execution_policy: execution_policy.clone(), + expected_payment_values: values, + }) + .unwrap(); + let bond = edge( + 64, + 0, + signer(2).party_key(), + signer(1).party_key(), + Terms::work_stake_bond(bond_terms).hash(), + 0b10, + ); + let payment = edge( + 1000, + 200, + signer(1).party_key(), + signer(2).party_key(), + payment_terms_hash(terms.clone()), + 0b11000, + ); + let mut lease = vec![1, 31, 2]; + lease.extend_from_slice(&bond_id.to_bytes()); + lease.extend_from_slice(&payment_id.to_bytes()); + lease.extend_from_slice(payment_terms_hash(terms.clone()).as_bytes()); + lease.extend_from_slice(&terms.private_policy_commitment); + lease.extend_from_slice(&HORIZON.to_be_bytes()); + let lease = hellas_kernel::parse_bond_lease( + [0, 1].map(|index| { + RegistryChunk::split( + RegistryNamespace::BondLease, + RegistryRecordTag::BondLease, + &lease, + index, + ) + }), + bond_id, + ); + assert!(matches!(lease, LeaseSlots::Present(_))); + let ready = descriptor + .check_ready(&ObservedChannel { + height: 1, + bond: Some(&bond), + payment: Some(&payment), + lease, + pending: PendingSlot::Absent, + }) + .unwrap(); + let config = WorkConfig { + chain: ChainCrossCheck { network, genesis_payload_digest: [0; 32].into(), threshold_identity: hex::decode("97f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb").unwrap() }, + validators: vec!["ws://unused.invalid".to_owned()], journal_root: root.path().join("provider"), routes: Default::default(), + policy_salt: [9; 32], channel_policy, execution_policy, + poll: std::time::Duration::from_millis(10), max_observation_age: std::time::Duration::from_secs(60), + expected_payment_values: values, min_omit_response_blocks: hellas_kernel::MIN_OMIT_RESPONSE_BLOCKS, + }; + Self { + root, + ready, + descriptor, + config, + } + } + + pub fn store(&self, role: Role) -> ChannelStore { + let name = if role == Role::Client { + "client" + } else { + "provider" + }; + ChannelStore::open_metadata_only( + &self.root.path().join(name), + self.ready.channel().clone(), + work_payment_settlement(self.config.expected_payment_values, OMISSION_BOND).unwrap(), + role, + SetupOrigin { + payment_edge: self.ready.channel().payment_edge(), + height: 1, + payload: [1; 32], + parent: [0; 32], + }, + &Secp256k1Verifier::new(), + ) + .unwrap() + } + + pub fn client(&self) -> ClientEndpoint { + ClientEndpoint::new(self.ready.clone(), self.store(Role::Client), signer(1)).unwrap() + } + + pub fn provider(&self) -> WorkService { + WorkService::new( + ProviderEndpoint::new(self.ready.clone(), self.store(Role::Provider), signer(2)) + .unwrap(), + ) + } + + pub fn proposal(&self) -> JobProposal { + let events = hellas_rpc::fetch::build_input_events_with_retention( + "openai", + "responses", + BODY, + FetchEnvironment::OpenAiResponses.manifest_id(), + Assurance::ProducerSigned, + &ProducerSigningKey::from_secret_bytes([1; 32]).unwrap(), + Retention::Ephemeral, + ) + .unwrap(); + JobProposal { + prepared_input: PreparedPaidFetchInputV1::new( + &events, + &FetchEnvironment::OpenAiResponses.manifest(), + ) + .unwrap() + .into(), + deadlines: JobDeadlines { + acceptance: 50, + terminal: 100, + payment: 200, + }, + } + } +} + +fn edge(value: u64, reserve: u64, maker: Key, taker: Key, terms: TermsHash, allowed: u8) -> Edge { + // Canonical kernel edges, including their close permissions. + let mut bytes = vec![1, 5]; + bytes.extend_from_slice(&value.to_be_bytes()); + bytes.extend_from_slice(&reserve.to_be_bytes()); + bytes.extend_from_slice(&[1, 2]); + bytes.extend_from_slice(&[0; 32]); + bytes.extend_from_slice(&[1, 1]); + bytes.extend_from_slice(&HORIZON.to_be_bytes()); + bytes.extend_from_slice(&[1, 3]); + bytes.extend_from_slice(&maker.to_bytes()); + bytes.extend_from_slice(&taker.to_bytes()); + bytes.extend_from_slice(terms.as_bytes()); + bytes.push(allowed); + Edge::decode_exact(&bytes).unwrap() +} + +pub(crate) fn enrollment(peer: EndpointId) -> (ProviderEnrollmentBundle, ProducerSigningKey) { + let root = ProducerSigningKey::from_secret_bytes([1; 32]).unwrap(); + let producer = ProducerSigningKey::from_secret_bytes([2; 32]).unwrap(); + let statement = ProviderGenesisStatement { + root_kind: RootKind::Software, + root_public_key: root.public_key(), + producer_public_key: producer.public_key(), + transport_public_key: PublicKey::Ed25519(*peer.as_bytes()), + platform_credential: PlatformCredential::Absent, + installation_nonce: [3; 32], + }; + let proof = root + .sign_digest(Digest::hash(&statement.canonical_bytes())) + .unwrap(); + ( + ProviderEnrollmentBundle { + genesis: SignedProviderGenesis { + statement, + root_proof: RootProof::Software(proof), + }, + platform: PlatformEnrollment::Absent, + }, + producer, + ) +} diff --git a/docs/http-gateway.md b/docs/http-gateway.md index a506c374..3f8643a1 100644 --- a/docs/http-gateway.md +++ b/docs/http-gateway.md @@ -62,6 +62,14 @@ only that accepted job can invoke the upstream. The credential's origin, path and method restrictions remain enforced by the provider. Courtesy caller grants do not authorize this HTTP path. +The SDK Fetch provider advertises only Work and WorkSetup when configured for +paid work; legacy allowed callers cannot reach its credentials through unpaid +Fetch. Accepted paid Fetch jobs wait in a bounded provider queue when upstream +concurrency is full. Further submissions wait at the executor mailbox while +completion handling remains available. Provider validator connections check the +configured genesis digest before advancing payment state; this check runs on +connection, not per HTTP request. + Signed response prefixes stream as they arrive. Successful HTTP completion follows verification of the complete result and the provider's durable payment acknowledgement. Each accepted valid terminal costs the channel's agreed fixed diff --git a/docs/paid-gateway.md b/docs/paid-gateway.md index ceb16b20..5f0b6cf6 100644 --- a/docs/paid-gateway.md +++ b/docs/paid-gateway.md @@ -110,8 +110,10 @@ providers or validators. Keep gateway and provider identities and journals across service restarts, including when the operating system's store is ephemeral. Startup recovers retained Evaluate jobs and re-sends payment certificates idempotently. Fetch -journals omit bodies: a lost, unpaid Fetch payload cannot be recovered and keeps -the channel reserved until its payment deadline. It is never submitted anew. Disconnecting an +journals omit bodies. A verified delivery recorded before a crash is paid from +its retained evidence, even when the payload is gone. Jobs without either payload +or verified delivery retain their credit reservation until the payment deadline; +recovery skips them and never submits them anew. Disconnecting an HTTP client cancels work that has not yet been proposed. Once a signed proposal may have reached a provider, collection and payment continue despite disconnects. The pool admits at most `max_pending_requests` queued or running requests (default diff --git a/nix/ci.nix b/nix/ci.nix index a2256b1f..313a2316 100644 --- a/nix/ci.nix +++ b/nix/ci.nix @@ -129,6 +129,12 @@ let mkCargo "check-client-work" "cargo test -p hellas-client --features work && cargo clippy -p hellas-client --features work --all-targets -- -D warnings" (cargoEnv rustToolchain); + # Exercise the production provider loop and paid session recovery; their + # feature is absent from the SDK's default tests. + sdk-paid-work = + mkCargo "check-sdk-paid-work" + "cargo test -p hellas-sdk --features paid-work && cargo clippy -p hellas-sdk --features paid-work --all-targets -- -D warnings" + (cargoEnv rustToolchain); # The chain service's wire-id pins compile only under `chain`, which # `work` does not pull in. `check-validator` links hellas-rpc with # that feature but runs hellas-chain's tests, not hellas-rpc's, so From df129ca8af36be931827da20538aaf954454eca3 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Mon, 28 Sep 2026 03:39:44 +0200 Subject: [PATCH 30/32] Share the paid gateway pool across SDK hosts Separate paid client/provider features and type provider and provisioning errors. Give HTTP gateways dedicated paid options, reuse authenticated Work connections and verified HTTP output, and exercise credential injection over local TLS. Share bounded configuration reads and add the HTTP/storage and SDK feature CI gates. --- Cargo.lock | 4 +- crates/cli/Cargo.toml | 5 +- crates/cli/src/commands/mod.rs | 21 +- crates/cli/src/commands/paid_work.rs | 13 +- crates/cli/src/commands/serve/provision.rs | 4 +- crates/cli/src/main.rs | 57 +- crates/gateway/src/http_fetch.rs | 52 +- crates/gateway/src/http_fetch/tests.rs | 67 ++- crates/gateway/src/lib.rs | 16 +- crates/gateway/src/state/tests.rs | 1 - crates/private/src/lib.rs | 3 + crates/private/src/regular.rs | 184 +++++++ crates/private/src/regular/tests.rs | 101 ++++ crates/providers/Cargo.toml | 1 + crates/providers/HTTPS.md | 25 + crates/providers/src/http/clients.rs | 24 + crates/providers/src/http/tests.rs | 142 +++++ crates/providers/src/lib.rs | 2 +- crates/providers/src/openai.rs | 10 +- crates/rpc/src/protocol/work_fetch.rs | 31 +- crates/rpc/src/protocol/work_profile.rs | 13 +- crates/sdk/Cargo.toml | 18 +- crates/sdk/src/lib.rs | 19 +- crates/sdk/src/paid_client.rs | 20 + crates/sdk/src/paid_client/tests.rs | 16 +- .../gateway.rs => sdk/src/paid_gateway.rs} | 500 ++++++++---------- crates/sdk/src/paid_gateway/config.rs | 249 +++++++++ crates/sdk/src/paid_gateway/error.rs | 65 +++ crates/sdk/src/provider.rs | 122 +++-- crates/sdk/src/test_support.rs | 2 + crates/sdk/src/work_config.rs | 7 +- crates/sdk/src/work_provision.rs | 232 +++----- crates/sdk/src/work_provision/error.rs | 55 ++ crates/sdk/src/work_provision/tests.rs | 15 +- crates/store/src/lib.rs | 158 +----- crates/store/src/tests.rs | 88 +-- docs/paid-gateway.md | 28 + nix/ci.nix | 7 +- 38 files changed, 1518 insertions(+), 859 deletions(-) create mode 100644 crates/private/src/regular.rs create mode 100644 crates/private/src/regular/tests.rs rename crates/{cli/src/commands/paid_work/gateway.rs => sdk/src/paid_gateway.rs} (78%) create mode 100644 crates/sdk/src/paid_gateway/config.rs create mode 100644 crates/sdk/src/paid_gateway/error.rs create mode 100644 crates/sdk/src/work_provision/error.rs diff --git a/Cargo.lock b/Cargo.lock index c980900a..c82d4b77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2987,6 +2987,7 @@ dependencies = [ "serde_json", "sha2 0.11.0", "tempfile", + "thiserror 2.0.18", "tokio", "tokio-rustls", "tracing", @@ -3047,7 +3048,6 @@ dependencies = [ name = "hellas-sdk" version = "0.1.0" dependencies = [ - "anyhow", "async-stream", "bytes", "futures", @@ -3058,12 +3058,14 @@ dependencies = [ "hellas-executor", "hellas-gateway", "hellas-kernel", + "hellas-private", "hellas-providers", "hellas-rpc", "hellas-wire", "hellas-work", "hex", "iroh", + "rand 0.10.2", "serde", "serde_json", "tempfile", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 6aab2611..d5b3a88c 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -24,7 +24,8 @@ apple-app-attest = [] # anything. node = [ "dep:hellas-sdk", - "hellas-sdk/paid-work", + "hellas-sdk/paid-client", + "hellas-sdk/paid-provider", "chain", "dep:async-stream", "dep:axum", @@ -39,7 +40,7 @@ node = [ # This deliberately does not pull Catena into a client that only dials a node. llm = ["hellas-client/evaluate", "dep:hellas-presentation"] # HTTP gateway front-end (OpenAI/Anthropic/plain APIs). -gateway = ["llm", "dep:hellas-gateway"] +gateway = ["llm", "dep:hellas-gateway", "hellas-sdk?/paid-gateway"] evaluate = [ "node", "gateway", diff --git a/crates/cli/src/commands/mod.rs b/crates/cli/src/commands/mod.rs index 6309f832..b9715d52 100644 --- a/crates/cli/src/commands/mod.rs +++ b/crates/cli/src/commands/mod.rs @@ -1,7 +1,6 @@ pub type CliResult = anyhow::Result; use anyhow::Context as _; -use std::io::Read as _; use std::path::Path; use std::time::Duration; @@ -38,20 +37,12 @@ pub(crate) fn read_bounded_regular_file( label: &str, maximum: usize, ) -> CliResult> { - let file = hellas_store::open_regular_file(path) - .with_context(|| format!("failed to open {label} {}", path.display()))?; - let limit = u64::try_from(maximum).unwrap_or(u64::MAX).saturating_add(1); - let mut bytes = Vec::new(); - file.take(limit) - .read_to_end(&mut bytes) - .with_context(|| format!("failed to read {label} {}", path.display()))?; - anyhow::ensure!( - bytes.len() <= maximum, - "{label} {} is {} bytes, over the {maximum}-byte limit", - path.display(), - bytes.len() - ); - Ok(bytes) + hellas_private::read_bounded_regular_file(path, maximum).with_context(|| { + format!( + "failed to read {label} {} (limit {maximum} bytes)", + path.display() + ) + }) } pub(crate) fn http_client(request_timeout: Duration) -> reqwest::Client { diff --git a/crates/cli/src/commands/paid_work.rs b/crates/cli/src/commands/paid_work.rs index 052e9195..19d6d8e2 100644 --- a/crates/cli/src/commands/paid_work.rs +++ b/crates/cli/src/commands/paid_work.rs @@ -17,29 +17,18 @@ use hellas_rpc::protocol::artifacts::PreparedPaidInputV1; use hellas_rpc::protocol::work::{JobDeadlines, private_policy_commitment}; use hellas_rpc::protocol::work_fetch::PreparedPaidFetchInputV1; use hellas_rpc::protocol::work_profile::PreparedPaidWorkInput; -#[cfg(feature = "gateway")] -use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; #[cfg(test)] use hellas_sdk::paid_client::check_genesis_payload; use hellas_sdk::paid_client::{InputIdentities, PaidWorkSession, bind_paid_endpoint}; -#[cfg(feature = "gateway")] -use hellas_sdk::paid_client::{PaidWorkResult, check_evaluate_input}; use hellas_work::work_store::journal::MAX_RECORD_BYTES; use iroh::{EndpointId, SecretKey}; use std::net::SocketAddr; use std::path::{Path, PathBuf}; -#[cfg(feature = "gateway")] -use std::sync::atomic::AtomicBool; use std::time::Duration; use super::CliResult; use super::serve::work_config::load_work_config; -#[cfg(feature = "gateway")] -mod gateway; -#[cfg(feature = "gateway")] -pub use gateway::load_gateway_backend; - /// Paid-work commands intended for deployment bring-up and smoke tests. #[derive(Debug, Subcommand)] pub enum PaidWorkCommand { @@ -567,7 +556,7 @@ fn empty_coins() -> List { ) } -fn fixed_hex(flag: &str, value: &str) -> CliResult<[u8; N]> { +pub(crate) fn fixed_hex(flag: &str, value: &str) -> CliResult<[u8; N]> { let bytes = hex::decode(value).with_context(|| format!("{flag} is not hex"))?; bytes .try_into() diff --git a/crates/cli/src/commands/serve/provision.rs b/crates/cli/src/commands/serve/provision.rs index 62911fbf..31c9345a 100644 --- a/crates/cli/src/commands/serve/provision.rs +++ b/crates/cli/src/commands/serve/provision.rs @@ -3,8 +3,8 @@ use super::super::CliResult; pub use hellas_sdk::work_provision::ProvisionOptions; use hellas_sdk::work_provision::{preview_bond, provision_offer}; -pub async fn run_provision(options: ProvisionOptions) -> CliResult<()> { - if options.print_bond_only { +pub async fn run_provision(options: ProvisionOptions, print_bond_only: bool) -> CliResult<()> { + if print_bond_only { println!( "bond_edge: {}", hex::encode(preview_bond(&options)?.to_bytes()) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 74742fef..e88cdd1c 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1150,22 +1150,23 @@ async fn async_main() { print_bond_only, } => match commands::serve::load_work_config(&work_config) { Err(error) => Err(error.into()), - Ok(work_config) => { + Ok(work_config) => async { commands::serve::run_provision(commands::serve::ProvisionOptions { work_config, // The bond is staked by the party this node already // settles as, taken from the identity loaded above // and never made here. settlement_key: identity::settlement_signer(&local_identity), - client, - stake_coins: stake_coin, + client: hellas_kernel::Key::from_bytes(commands::paid_work::fixed_hex("--client", &client)?), + stake_coins: stake_coin.iter().map(|coin| { + commands::paid_work::fixed_hex("--stake-coin", coin).map(hellas_kernel::CoinId::from_bytes) + }).collect::>()?, bond_timeout, timeout_payout, max_job_price, - print_bond_only, - }) + }, print_bond_only) .await - } + }.await }, Commands::OutputCache(..) => unreachable!("cache commands handled before identity load"), #[cfg(feature = "gateway")] @@ -1294,13 +1295,10 @@ async fn async_main() { "set provider enrollment and Apple trust pins in --paid-work-config", ); Some( - commands::paid_work::load_gateway_backend( - path, - secret_key.clone(), - identity::settlement_signer(&local_identity), - local_identity.producer_key.clone(), - assurance, - ).await? + hellas_sdk::paid_gateway::PaidGateway::open( + hellas_sdk::paid_gateway::load_pool_options(path, assurance)?, + hellas_sdk::ClientIdentity::from_secret_bytes(secret_key.to_bytes(), local_identity.producer_key.to_secret_bytes())?, + ).await? as std::sync::Arc ) } else { None @@ -1321,24 +1319,23 @@ async fn async_main() { remote_trust.apple_app_attest_cdhashes, )? }; + let archive = hellas_gateway::ArchiveOptions { + directory: archive_dir.map(Ok).unwrap_or_else(identity::default_gateway_archive_path)?, + zdr, + }; + if let Some(path) = http_fetch_config { + anyhow::ensure!(output_cache == hellas_rpc::cache::CachePolicy::Off, + "HTTP routes archive exchanges; inference replay must be off"); + anyhow::ensure!(metrics_port.is_none(), "HTTP Fetch exports OpenTelemetry metrics; --metrics-port is unsupported"); + let bytes = commands::read_bounded_regular_file(&path, "HTTP gateway config", 4 << 20)?; + return hellas_gateway::run_http(hellas_gateway::HttpGatewayOptions { + config: serde_json::from_slice(&bytes)?, + paid: paid_work.ok_or_else(|| anyhow::anyhow!("HTTP proxy requires --paid-work-config"))?, + archive, bearer_token_file, allow_remote, host, port, wrap, wrap_args, + }).await; + } hellas_gateway::run(hellas_gateway::GatewayOptions { - archive: hellas_gateway::ArchiveOptions { - directory: archive_dir.map(Ok).unwrap_or_else(identity::default_gateway_archive_path)?, - zdr, - }, - http_fetch: http_fetch_config - .map(|path| -> anyhow::Result<_> { - // 4 MiB, the same bound the pool-file loader - // takes from hellas-work, which this binary - // links only in some feature builds. - let bytes = commands::read_bounded_regular_file( - &path, - "HTTP gateway config", - 4 << 20, - )?; - Ok(serde_json::from_slice(&bytes)?) - }) - .transpose()?, + archive, output_cache: cache_options, paid_work, bearer_token_file, diff --git a/crates/gateway/src/http_fetch.rs b/crates/gateway/src/http_fetch.rs index fdef1801..1089655a 100644 --- a/crates/gateway/src/http_fetch.rs +++ b/crates/gateway/src/http_fetch.rs @@ -8,7 +8,6 @@ pub use config::HttpGatewayConfig; #[cfg(test)] mod tests; -use anyhow::{Context, ensure}; use axum::{ Router, body::{Body, Bytes}, @@ -21,7 +20,7 @@ use hellas_rpc::output::{AdaptorEvent, HttpResponseEvent, OutputEvent, StopReaso use std::sync::Arc; use tracing::Instrument; -use super::{GatewayHandle, GatewayOptions, PaidExecutionBackend, PaidFetchRequest, access}; +use super::{GatewayHandle, PaidExecutionBackend, PaidFetchRequest, access}; #[derive(Clone)] pub(crate) struct BackendName(pub String); @@ -41,8 +40,6 @@ struct HttpState { #[derive(Debug, thiserror::Error)] enum HttpOpenError { - #[error("HTTP proxy requires a paid Fetch backend")] - MissingPaidBackend, #[error("missing authenticated HTTP response head")] MissingHead, #[error(transparent)] @@ -51,27 +48,34 @@ enum HttpOpenError { Headers(#[from] hellas_rpc::http_fetch::HttpRequestError), } -pub(super) async fn start(options: GatewayOptions) -> anyhow::Result { - let config = options - .http_fetch - .as_ref() - .context("missing HTTP configuration")? - .clone(); - config.validate()?; - let paid = options - .paid_work - .clone() - .ok_or(HttpOpenError::MissingPaidBackend)?; - ensure!( - options.output_cache.policy == hellas_rpc::cache::CachePolicy::Off, - "HTTP routes archive exchanges; inference replay must be off" - ); - if options.metrics_port.is_some() { - tracing::warn!( - "the Prometheus metrics endpoint is not served in HTTP Fetch mode; \ - --metrics-port has no effect (otel metrics remain on the usual exporter)" - ); +/// Configuration for transparent HTTP over paid Fetch. Every route uses the +/// supplied paid pool; tokenizers and inference replay do not apply here. +pub struct HttpGatewayOptions { + pub config: HttpGatewayConfig, + pub paid: Arc, + pub archive: super::ArchiveOptions, + pub host: String, + pub port: Option, + pub bearer_token_file: Option, + pub allow_remote: bool, + pub wrap: Option, + pub wrap_args: Vec, +} + +/// Start without signal handlers. Failed startup drains any recovering paid jobs. +pub async fn start_http(options: HttpGatewayOptions) -> anyhow::Result { + let paid = options.paid.clone(); + let result = start(options).await; + if result.is_err() { + paid.drain().await; } + result +} + +async fn start(options: HttpGatewayOptions) -> anyhow::Result { + let config = options.config; + config.validate()?; + let paid = options.paid; let archive_policy = super::archive::Policy::new(options.archive.clone(), false); archive_policy.prepare(); let routing = Arc::new(routing::Routing::new(&config, &paid.fetch_providers())?); diff --git a/crates/gateway/src/http_fetch/tests.rs b/crates/gateway/src/http_fetch/tests.rs index 66df894a..c789dfbf 100644 --- a/crates/gateway/src/http_fetch/tests.rs +++ b/crates/gateway/src/http_fetch/tests.rs @@ -8,14 +8,18 @@ use std::{ #[tokio::test] async fn http_uses_the_paid_backend_and_waits_for_its_payment_completion() { - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; struct Paid { requests: AtomicUsize, + drained: AtomicBool, ack: Arc, provider: iroh::EndpointId, busy: bool, } impl PaidExecutionBackend for Paid { + fn fetch_providers(&self) -> Vec { + vec![self.provider] + } fn execute( &self, _: crate::PaidExecutionRequest, @@ -48,13 +52,16 @@ async fn http_uses_the_paid_backend_and_waits_for_its_payment_completion() { })) } fn drain(&self) -> futures::future::BoxFuture<'_, ()> { - Box::pin(async {}) + Box::pin(async { + self.drained.store(true, Ordering::Relaxed); + }) } } for busy in [false, true] { let provider = iroh::SecretKey::from_bytes(&[12; 32]).public(); let paid = Arc::new(Paid { requests: AtomicUsize::new(0), + drained: AtomicBool::new(false), ack: Arc::default(), provider, busy, @@ -64,28 +71,49 @@ async fn http_uses_the_paid_backend_and_waits_for_its_payment_completion() { "path":"/v1/responses", "method":"POST", "url":"https://example.com/v1/responses", "credential":"account" }] })).unwrap(); - let state = Arc::new(HttpState { - service: config.service.clone(), - method: config.method.clone(), - routing: Arc::new(routing::Routing::new(&config, &[provider]).unwrap()), + let archive = tempfile::tempdir().unwrap(); + let handle = start_http(HttpGatewayOptions { + config, paid: paid.clone(), - metrics: observation::Metrics::new(), - }); - let response = handle( - State(state), - Request::builder() - .method("POST") - .uri("/v1/responses?x=%2F&x=y") - .body(Body::from("opaque request")) - .unwrap(), - ) - .await; + archive: crate::ArchiveOptions { + directory: archive.path().into(), + zdr: true, + }, + host: "127.0.0.1".into(), + port: Some(0), + bearer_token_file: None, + allow_remote: false, + wrap: None, + wrap_args: Vec::new(), + }) + .await + .unwrap(); + let client = reqwest::Client::new(); + let url = format!("http://{}/v1/responses?x=%2F&x=y", handle.address()); + assert_eq!( + client + .post(&url) + .body("opaque request") + .send() + .await + .unwrap() + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!(paid.requests.load(Ordering::Relaxed), 0); + let response = client + .post(&url) + .bearer_auth(handle.bearer()) + .body("opaque request") + .send() + .await + .unwrap(); assert_eq!(paid.requests.load(Ordering::Relaxed), 1); if busy { assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); } else { assert_eq!(response.status(), StatusCode::OK); - let mut body = response.into_body().into_data_stream(); + let mut body = response.bytes_stream(); assert_eq!(body.next().await.unwrap().unwrap(), &b"\x00\xff\n"[..]); assert!( tokio::time::timeout(Duration::from_millis(20), body.next()) @@ -95,6 +123,9 @@ async fn http_uses_the_paid_backend_and_waits_for_its_payment_completion() { paid.ack.notify_one(); assert!(body.next().await.is_none()); } + handle.shutdown().await.unwrap(); + assert!(paid.drained.load(Ordering::Relaxed)); + assert_eq!(std::fs::read_dir(archive.path()).unwrap().count(), 0); } } diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index d33dd64e..a9d7e39b 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -43,7 +43,7 @@ pub use execution::{ CausalLmExecutionEnvironment, CliRuntime, ExecutionEvent, ExecutionRequest, ExecutionRequestOptions, ExecutionStrategy, Outcome, PreparedExecution, StopReason, }; -pub use http_fetch::HttpGatewayConfig; +pub use http_fetch::{HttpGatewayConfig, HttpGatewayOptions, start_http}; const DEFAULT_HTTP_PORT: u16 = 8080; @@ -137,7 +137,6 @@ impl pub struct GatewayOptions { pub archive: ArchiveOptions, - pub http_fetch: Option, pub output_cache: cache::CacheOptions, pub paid_work: Option>, /// Load or create a stable bearer credential in a private file. @@ -281,9 +280,6 @@ pub async fn start(options: GatewayOptions) -> anyhow::Result { } async fn start_gateway(options: GatewayOptions) -> anyhow::Result { - if options.http_fetch.is_some() { - return http_fetch::start(options).await; - } let listener = bind_gateway( &options.host, options.port, @@ -479,7 +475,15 @@ async fn finish_paid_work( /// CLI lifecycle wrapper around [`start`]. pub async fn run(options: GatewayOptions) -> anyhow::Result<()> { - let mut handle = start(options).await?; + wait_for_shutdown(start(options).await?).await +} + +/// Run a paid HTTP gateway with process signal handling. +pub async fn run_http(options: HttpGatewayOptions) -> anyhow::Result<()> { + wait_for_shutdown(start_http(options).await?).await +} + +async fn wait_for_shutdown(mut handle: GatewayHandle) -> anyhow::Result<()> { tokio::select! { signal = shutdown_signal() => { signal?; diff --git a/crates/gateway/src/state/tests.rs b/crates/gateway/src/state/tests.rs index 61afc596..d96439ca 100644 --- a/crates/gateway/src/state/tests.rs +++ b/crates/gateway/src/state/tests.rs @@ -55,7 +55,6 @@ fn options(provider_trust: Option) -> GatewayOptions { directory: "unused-test-archive".into(), zdr: true, }, - http_fetch: None, output_cache: Default::default(), paid_work: None, bearer_token_file: None, diff --git a/crates/private/src/lib.rs b/crates/private/src/lib.rs index e0f1fb56..35ae217a 100644 --- a/crates/private/src/lib.rs +++ b/crates/private/src/lib.rs @@ -8,6 +8,9 @@ //! in a Windows DACL for the same reason root is on Unix: they can reach the //! object regardless of what it says. +mod regular; +pub use regular::{open_regular_file, read_bounded_regular_file}; + use std::fs::File; use std::io::{self, Write as _}; use std::path::Path; diff --git a/crates/private/src/regular.rs b/crates/private/src/regular.rs new file mode 100644 index 00000000..3b6e99df --- /dev/null +++ b/crates/private/src/regular.rs @@ -0,0 +1,184 @@ +use std::{fs::OpenOptions, io, path::Path}; + +/// Opens a local regular file without first opening a FIFO or device for I/O. +/// +/// Explicit content paths may be symlinks: the target descriptor, rather than +/// the symlink name, is the authority. Linux first acquires an `O_PATH` +/// descriptor, which does not open the underlying object, checks its type, and +/// then reopens that exact inode through `/proc/self/fd`. Consequently a path +/// replacement between the type check and the readable open cannot substitute +/// a FIFO or device. The returned descriptor is read-only, seekable, and +/// close-on-exec. +/// +/// Other platforms do not expose an equivalent through `std`: they preflight +/// the followed path, use nonblocking open where Unix provides it, and verify +/// the resulting descriptor. That rejects stable special files but cannot +/// close an adversarial replacement race as Linux does. +pub fn open_regular_file(path: &Path) -> io::Result { + open_regular_file_impl(path) +} + +fn not_a_regular_file(path: &Path) -> io::Error { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{} is not a regular file", path.display()), + ) +} + +#[cfg(target_os = "linux")] +fn open_regular_file_impl(path: &Path) -> io::Result { + let path_handle = open_path_handle(path)?; + reopen_regular_path_handle(path, &path_handle) +} + +/// Acquires an inode reference without invoking the target's file operations. +#[cfg(target_os = "linux")] +fn open_path_handle(path: &Path) -> io::Result { + use std::ffi::CString; + use std::os::fd::FromRawFd as _; + use std::os::unix::ffi::OsStrExt as _; + + let path = CString::new(path.as_os_str().as_bytes()) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + // OpenOptions masks custom flags with !O_ACCMODE. On musl that mask + // includes O_PATH, turning a descriptor-only open into a blocking read. + loop { + // SAFETY: path is NUL-terminated and live for the call. These flags + // do not create a file, so open needs no variadic mode argument. + let fd = unsafe { libc::open(path.as_ptr(), libc::O_PATH | libc::O_CLOEXEC) }; + if fd >= 0 { + // SAFETY: open returned a new descriptor, owned only here. + return Ok(unsafe { std::fs::File::from_raw_fd(fd) }); + } + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } + } +} + +/// Converts an `O_PATH` reference to a readable descriptor for the same inode. +#[cfg(target_os = "linux")] +fn reopen_regular_path_handle( + path: &Path, + path_handle: &std::fs::File, +) -> io::Result { + use std::os::fd::AsRawFd as _; + use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; + + let expected = path_handle.metadata()?; + if !expected.file_type().is_file() { + return Err(not_a_regular_file(path)); + } + + let descriptor_path = Path::new("/proc/self/fd").join(path_handle.as_raw_fd().to_string()); + let mut options = OpenOptions::new(); + options.read(true).custom_flags(libc::O_CLOEXEC); + let file = options.open(&descriptor_path).map_err(|source| { + // Once the held descriptor has been fstat-ed successfully, ENOENT can + // only mean procfs cannot provide the safe reopen. Do not report the + // caller's existing content as a cache miss. + let kind = if source.kind() == io::ErrorKind::NotFound { + io::ErrorKind::Unsupported + } else { + source.kind() + }; + io::Error::new( + kind, + format!( + "cannot safely reopen {} through {}: {source}", + path.display(), + descriptor_path.display() + ), + ) + })?; + let actual = file.metadata()?; + if !actual.file_type().is_file() + || (expected.dev(), expected.ino()) != (actual.dev(), actual.ino()) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{} did not reopen the inode held by its path descriptor", + path.display() + ), + )); + } + Ok(file) +} + +/// Best available fallback where `O_PATH` plus descriptor reopen is absent. +#[cfg(all(unix, not(target_os = "linux")))] +fn open_regular_file_impl(path: &Path) -> io::Result { + use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; + + let expected = std::fs::metadata(path)?; + if !expected.file_type().is_file() { + return Err(not_a_regular_file(path)); + } + + let mut options = OpenOptions::new(); + options + .read(true) + .custom_flags(libc::O_NONBLOCK | libc::O_CLOEXEC); + let file = options.open(path)?; + let actual = file.metadata()?; + if !actual.file_type().is_file() + || (expected.dev(), expected.ino()) != (actual.dev(), actual.ino()) + { + return Err(not_a_regular_file(path)); + } + Ok(file) +} + +/// Best available fallback where Unix descriptors are absent. Windows adds a +/// read-only share mode, so no writer can open the file while it is being +/// hashed -- the guarantee the Unix paths approximate with identity checks +/// (fastresume's Windows identity is deliberately weak; see there). +#[cfg(not(unix))] +fn open_regular_file_impl(path: &Path) -> io::Result { + let expected = std::fs::metadata(path)?; + if !expected.file_type().is_file() { + return Err(not_a_regular_file(path)); + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt as _; + // FILE_SHARE_READ: other readers may share the file; writers and + // deleters are refused until it is closed. + options.share_mode(0x0000_0001); + } + let file = options.open(path)?; + if !file.metadata()?.file_type().is_file() { + return Err(not_a_regular_file(path)); + } + Ok(file) +} + +/// Read a regular file with a one-byte sentinel so concurrent growth cannot +/// turn a bounded configuration read into an unbounded allocation. +pub fn read_bounded_regular_file(path: &Path, maximum: usize) -> io::Result> { + use io::Read as _; + let file = open_regular_file(path)?; + if file.metadata()?.len() > maximum as u64 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "file exceeds byte limit", + )); + } + let mut bytes = Vec::new(); + file.take((maximum as u64).saturating_add(1)) + .read_to_end(&mut bytes)?; + if bytes.len() > maximum { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "file exceeds byte limit", + )); + } + Ok(bytes) +} + +#[cfg(test)] +mod tests; diff --git a/crates/private/src/regular/tests.rs b/crates/private/src/regular/tests.rs new file mode 100644 index 00000000..9983c6dc --- /dev/null +++ b/crates/private/src/regular/tests.rs @@ -0,0 +1,101 @@ +use super::*; +#[cfg(target_os = "linux")] +use std::path::PathBuf; + +/// The Linux primitive is the contract, not merely an implementation +/// detail: a harmless path descriptor is acquired before type inspection, +/// and the readable descriptor remains bound to that inode even if the +/// name is replaced with a device. +#[cfg(target_os = "linux")] +#[test] +fn regular_open_is_safe_readable_seekable_and_close_on_exec() { + use std::io::{Read as _, Seek as _, SeekFrom}; + use std::os::fd::AsRawFd as _; + + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let target = dir.join("blob"); + let content = b"ordinary bytes"; + std::fs::write(&target, content).expect("write target"); + let link = dir.join("snapshot"); + std::os::unix::fs::symlink(&target, &link).expect("symlink"); + + let mut file = open_regular_file(&link).expect("open regular symlink"); + let mut read = Vec::new(); + file.read_to_end(&mut read).expect("read"); + assert_eq!(read, content); + file.seek(SeekFrom::Start(0)).expect("seek"); + read.clear(); + file.read_to_end(&mut read).expect("read again"); + assert_eq!(read, content); + // SAFETY: F_GETFD only observes the live descriptor owned by `file`. + let descriptor_flags = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFD) }; + assert!( + descriptor_flags >= 0, + "F_GETFD: {}", + io::Error::last_os_error() + ); + assert_ne!(descriptor_flags & libc::FD_CLOEXEC, 0); + + // Hold the ordinary inode without opening it for I/O, replace its + // public name with a device, then finish the reopen. The bytes must + // still come from the held inode. + let path_handle = open_path_handle(&link).expect("path handle"); + // O_PATH must survive libc-specific access-mode flags (notably musl). + // SAFETY: F_GETFL only observes the live descriptor owned by path_handle. + let path_flags = unsafe { libc::fcntl(path_handle.as_raw_fd(), libc::F_GETFL) }; + assert!(path_flags >= 0, "F_GETFL: {}", io::Error::last_os_error()); + assert_ne!(path_flags & libc::O_PATH, 0); + let replacement = dir.join("replacement"); + std::os::unix::fs::symlink("/dev/null", &replacement).expect("device symlink"); + std::fs::rename(&replacement, &link).expect("replace link"); + let mut held = reopen_regular_path_handle(&link, &path_handle).expect("reopen held inode"); + read.clear(); + held.read_to_end(&mut read).expect("read held inode"); + assert_eq!(read, content); + + let fifo = dir.join("fifo"); + let status = std::process::Command::new("mkfifo") + .arg(&fifo) + .status() + .expect("mkfifo"); + assert!(status.success(), "the fixture needs a fifo"); + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = sender.send(open_regular_file(&fifo)); + }); + let fifo_error = receiver + .recv_timeout(std::time::Duration::from_secs(2)) + .expect("opening the FIFO blocked before its type check") + .expect_err("a FIFO is not regular content"); + assert_eq!(fifo_error.kind(), io::ErrorKind::InvalidInput); + + if Path::new("/dev/null").exists() { + let device_error = open_regular_file(Path::new("/dev/null")) + .expect_err("a character device is not regular content"); + assert_eq!(device_error.kind(), io::ErrorKind::InvalidInput); + } + + use std::os::unix::ffi::{OsStrExt as _, OsStringExt as _}; + let mut nul_path = target.as_os_str().as_bytes().to_vec(); + nul_path.extend_from_slice(b"\0ignored-suffix"); + let nul_path = PathBuf::from(std::ffi::OsString::from_vec(nul_path)); + assert_eq!( + open_regular_file(&nul_path) + .expect_err("NUL must not truncate the path") + .kind(), + io::ErrorKind::InvalidInput, + ); +} + +#[test] +fn bounded_read_accepts_exact_limit_and_rejects_larger_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config"); + std::fs::write(&path, b"1234").unwrap(); + assert_eq!(read_bounded_regular_file(&path, 4).unwrap(), b"1234"); + assert_eq!( + read_bounded_regular_file(&path, 3).unwrap_err().kind(), + io::ErrorKind::InvalidData + ); +} diff --git a/crates/providers/Cargo.toml b/crates/providers/Cargo.toml index 7194de1b..ebac1b5d 100644 --- a/crates/providers/Cargo.toml +++ b/crates/providers/Cargo.toml @@ -12,6 +12,7 @@ default = [] otel = ["hellas-rpc/otel", "dep:hellas-wire"] [dependencies] +thiserror.workspace = true anyhow = { workspace = true } async-stream = { workspace = true } base64.workspace = true diff --git a/crates/providers/HTTPS.md b/crates/providers/HTTPS.md index 35f23dc1..2465e613 100644 --- a/crates/providers/HTTPS.md +++ b/crates/providers/HTTPS.md @@ -141,3 +141,28 @@ and a signed terminal. Hop-by-hop and framing headers (`connection`, connection, so they are not part of the signed head; the body bytes are the only length truth. `HttpFetchResponse::from_output` reconstructs the body and checks ordering and size after transcript signature verification. + +## Provider-owned credential demonstration + +Generic HTTPS preserves provider-side API-key injection. The buyer signs only +`credential: "account-1"`; the provider resolves that alias and inserts its +secret header immediately before sending HTTPS. The key is absent from the +buyer input transcript and the normal response transcript. Credentialed +requests require public trust roots, an exact permitted origin/path/method, +and no caller override of the credential header. Redirects are not followed. +Scope aliases to upstreams you trust: an upstream receives the header and could +include it in its response; a transparent proxy cannot promise to hide a secret +that the upstream itself returns. + +Run the offline TLS demonstration from the Hellas checkout: + +```sh +nix develop --command cargo test -p hellas-providers provider_injects_secret -- --nocapture +``` + +It runs the production HTTP provider and adaptor against a local TLS server, +checks that the server receives the key, verifies the signed input and output, +and confirms neither transcript contains the key. It also checks that warmed +connection pools cannot bypass credential scope or header-override checks. The +fixture installs its own CA in the test client pool; production credentialed +requests still refuse buyer-supplied roots. diff --git a/crates/providers/src/http/clients.rs b/crates/providers/src/http/clients.rs index 84c44f38..bec48cf3 100644 --- a/crates/providers/src/http/clients.rs +++ b/crates/providers/src/http/clients.rs @@ -68,6 +68,30 @@ impl Clients { Ok(client) } + // The fixture's CA acts as a provider-installed public root. Production + // callers still cannot combine their own trust anchors with credentials. + #[cfg(test)] + pub(super) fn trust_fixture( + &self, + request: &HttpFetchRequest, + url: &Url, + mut addresses: Vec, + ) { + let client = self.get(request, url, addresses.clone()).unwrap(); + addresses.sort_unstable(); + addresses.dedup(); + let key = Key { + origin: url.origin().ascii_serialization(), + addresses, + tls: HttpTls { + roots: hellas_rpc::http_fetch::HttpTrustRoots::WebPki, + spki_sha256: request.tls.spki_sha256.clone(), + }, + credential: request.credential.clone(), + }; + self.0.lock().unwrap().push_back((key, client)); + } + fn cached(entries: &mut VecDeque<(Key, Client)>, key: &Key) -> Option { let index = entries.iter().position(|(existing, _)| existing == key)?; let entry = entries.remove(index).unwrap(); diff --git a/crates/providers/src/http/tests.rs b/crates/providers/src/http/tests.rs index d95089e7..cc454630 100644 --- a/crates/providers/src/http/tests.rs +++ b/crates/providers/src/http/tests.rs @@ -30,6 +30,21 @@ async fn server_with_pause( Arc, Arc, tokio::task::JoinHandle<()>, +) { + server_fixture(status, bytes, location, pause, None).await +} + +async fn server_fixture( + status: u16, + bytes: Vec, + location: Option, + pause: Option>, + capture: Option>>, +) -> ( + HttpFetchRequest, + Arc, + Arc, + tokio::task::JoinHandle<()>, ) { let key = generate_simple_self_signed(vec!["localhost".into()]).unwrap(); let cert = key.cert.der().clone(); @@ -71,6 +86,7 @@ async fn server_with_pause( let bytes = bytes.clone(); let location = location.clone(); let pause = pause.clone(); + let capture = capture.clone(); tokio::spawn(async move { let Ok(mut socket) = acceptor.accept(socket).await else { return; @@ -84,6 +100,9 @@ async fn server_with_pause( } request.push(byte[0]); } + if let Some(capture) = &capture { + let _ = capture.send(request); + } seen.fetch_add(1, Ordering::SeqCst); let location = location .as_ref() @@ -471,3 +490,126 @@ fn special_addresses_never_qualify_as_public_egress() { assert!(public_address("8.8.8.8".parse().unwrap())); assert!(public_address("2606:4700:4700::1111".parse().unwrap())); } + +/// Offline demonstration: the TLS upstream sees the provider's credential; +/// the signed buyer request and authenticated response contain only the alias. +#[tokio::test] +async fn provider_injects_secret_without_disclosing_it_to_the_buyer() { + use hellas_rpc::fetch::{ + FetchOutputTranscriptBuilder, build_input_events_with_retention, verify_input_events, + verify_output_events, + }; + use hellas_rpc::{Assurance, ProducerSigningKey, Retention}; + const SECRET: &str = "Bearer provider-only-demo-key"; + let (capture, mut captured) = tokio::sync::mpsc::unbounded_channel(); + let (mut request, calls, _, server) = server_fixture( + 200, + b"upstream response".to_vec(), + None, + None, + Some(capture), + ) + .await; + let url = request.parsed_url().unwrap(); + request.credential = Some("provider-account".into()); + let provider = HttpFetchProvider::new( + HttpEgressPolicy { + allowed_hosts: vec!["localhost".into()], + allow_private_addresses: true, + }, + BTreeMap::from([( + "provider-account".into(), + HttpCredential { + allowed_origins: vec![url.origin().ascii_serialization()], + allowed_paths: vec!["/resource".into()], + allowed_methods: vec!["GET".into()], + header_name: "authorization".into(), + header_value: HttpSecret::Value(SECRET.into()), + }, + )]), + ) + .unwrap(); + let addresses = tokio::net::lookup_host(("localhost", url.port().unwrap())) + .await + .unwrap() + .take(65) + .collect(); + provider.clients.trust_fixture(&request, &url, addresses); + request.tls.roots = HttpTrustRoots::WebPki; + let buyer = ProducerSigningKey::from_secret_bytes([71; 32]).unwrap(); + let seller = ProducerSigningKey::from_secret_bytes([72; 32]).unwrap(); + let input_events = build_input_events_with_retention( + "http", + "request", + &serde_json::to_vec(&request).unwrap(), + FetchEnvironment::Http.manifest_id(), + Assurance::ProducerSigned, + &buyer, + Retention::Ephemeral, + ) + .unwrap(); + let input = verify_input_events(&input_events).unwrap(); + let call = FetchCall::new( + &input.service, + &input.method, + input.body.clone(), + input.input_commitment, + ); + let session = HttpFetchAdaptorFactory.create(&call).unwrap(); + let mut projector = session.projector; + let response = provider.run(session.provider_request).await.unwrap(); + let mut projected = projector.begin(response.head).unwrap(); + let mut stream = response.stream; + while let Some(chunk) = stream.next().await { + projected.extend(projector.project(&chunk.unwrap()).unwrap()); + } + projected.extend(projector.finish().unwrap()); + let terminal = match projected.pop().unwrap() { + ProjectedFetch::Terminal(bytes) => bytes, + _ => panic!("terminal"), + }; + let mut output = + FetchOutputTranscriptBuilder::new(input.input_commitment, input.assurance, &seller); + for event in projected { + let ProjectedFetch::Event(bytes) = event else { + panic!("event") + }; + output.push_event(bytes).unwrap(); + } + let output_events = output.finish(terminal).unwrap(); + let verified = + verify_output_events(input.input_commitment, input.assurance, &output_events).unwrap(); + let delivered = + hellas_rpc::http_fetch::HttpFetchResponse::from_output(&request, &verified).unwrap(); + assert_eq!(delivered.body, b"upstream response"); + let upstream = String::from_utf8(captured.recv().await.unwrap()).unwrap(); + assert!(upstream.contains(&format!("authorization: {SECRET}\r\n"))); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert!(input_events.iter().all(|event| { + !event + .payload() + .windows(SECRET.len()) + .any(|w| w == SECRET.as_bytes()) + })); + assert!(output_events.iter().all(|event| { + !event + .payload() + .windows(SECRET.len()) + .any(|w| w == SECRET.as_bytes()) + })); + // Reusing the TLS pool cannot bypass credential scope or override checks. + for (path, headers) in [ + ("/elsewhere", vec![]), + ( + "/resource", + vec![("authorization".into(), "buyer override".into())], + ), + ] { + let mut forbidden = request.clone(); + forbidden.url = format!("{}{path}", url.origin().ascii_serialization()); + forbidden.headers = headers; + assert!(provider.run(prepared(&forbidden)).await.is_err()); + } + assert_eq!(calls.load(Ordering::SeqCst), 1); + server.abort(); +} diff --git a/crates/providers/src/lib.rs b/crates/providers/src/lib.rs index 39a34dbf..843f8582 100644 --- a/crates/providers/src/lib.rs +++ b/crates/providers/src/lib.rs @@ -12,6 +12,6 @@ pub use http::{ HttpFetchAdaptorFactory, HttpFetchProvider, HttpProviderConfig, HttpSecret, }; -pub use openai::OpenAiResponsesFetchProvider; +pub use openai::{EmptyOpenAiKey, OpenAiResponsesFetchProvider}; pub use responses_fetch::{execute_responses_request, responses_http_client}; pub use responses_projector::ResponsesFetchAdaptorFactory; diff --git a/crates/providers/src/openai.rs b/crates/providers/src/openai.rs index 1a2c5b6c..30e40697 100644 --- a/crates/providers/src/openai.rs +++ b/crates/providers/src/openai.rs @@ -8,6 +8,10 @@ use reqwest::Url; use super::responses_fetch::{execute_responses_request, responses_http_client}; +#[derive(Debug, thiserror::Error)] +#[error("OpenAI API key is empty")] +pub struct EmptyOpenAiKey; + #[derive(Clone)] pub struct OpenAiResponsesFetchProvider { client: reqwest::Client, @@ -23,16 +27,16 @@ impl OpenAiResponsesFetchProvider { if bearer_token.is_empty() { bail!("environment variable {api_key_env} is empty"); } - Self::with_bearer(bearer_token) + Ok(Self::with_bearer(bearer_token)?) } /// Construct from a secret supplied by an embedding host without routing /// it through process-wide environment state. - pub fn with_bearer(bearer_token: impl Into) -> anyhow::Result { + pub fn with_bearer(bearer_token: impl Into) -> Result { let bearer_token = bearer_token.into(); let bearer_token = bearer_token.trim().to_owned(); if bearer_token.is_empty() { - bail!("OpenAI API key is empty"); + return Err(EmptyOpenAiKey); } let endpoint = Url::parse(OPENAI_RESPONSES_ENDPOINT) .expect("built-in OpenAI Responses endpoint is valid"); diff --git a/crates/rpc/src/protocol/work_fetch.rs b/crates/rpc/src/protocol/work_fetch.rs index 2d535234..b88c245e 100644 --- a/crates/rpc/src/protocol/work_fetch.rs +++ b/crates/rpc/src/protocol/work_fetch.rs @@ -701,6 +701,16 @@ pub fn terminal_fetch_result( transcript: &[OutputEventEnvelope], assurance: Assurance, ) -> Result { + Ok(verify_terminal_fetch_result(channel, authorization, transcript, assurance)?.0) +} + +/// Keep the authenticated output for profile-specific checks at the same boundary. +pub(super) fn verify_terminal_fetch_result( + channel: &PaidChannel, + authorization: &PaidJobAuthorizationV1, + transcript: &[OutputEventEnvelope], + assurance: Assurance, +) -> Result<(PaidJobResultV1, crate::fetch::FetchOutput), PaidWorkError> { let input = InputCommitment::from_digest(authorization.request_commitment.digest()); let output = crate::fetch::verify_output_events(input, assurance, transcript) .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; @@ -727,16 +737,19 @@ pub fn terminal_fetch_result( crate::fetch::decode_fetch_terminal_payload(terminal_payload) .map_err(|error| PaidWorkError::Transcript(error.to_string()))?; let work_id = work_id(channel, authorization); - Ok(PaidJobResultV1 { - work_id, - terminal_transcript_commitment: terminal_event.event_commitment(), - canonical_output_digest: fetch_canonical_output_digest( - channel.network(), + Ok(( + PaidJobResultV1 { work_id, - event_payloads, - terminal_payload, - ), - }) + terminal_transcript_commitment: terminal_event.event_commitment(), + canonical_output_digest: fetch_canonical_output_digest( + channel.network(), + work_id, + event_payloads, + terminal_payload, + ), + }, + output, + )) } // ── Bounds ──────────────────────────────────────────────────────────── diff --git a/crates/rpc/src/protocol/work_profile.rs b/crates/rpc/src/protocol/work_profile.rs index 071a50ec..a9eb24f8 100644 --- a/crates/rpc/src/protocol/work_profile.rs +++ b/crates/rpc/src/protocol/work_profile.rs @@ -311,22 +311,21 @@ impl PreparedPaidWorkInput { let parts = bundle.parts()?; let input = crate::fetch::verify_input_events(&parts.fetch_input_transcript) .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; - let result = work_fetch::terminal_fetch_result( + let (result, output) = work_fetch::verify_terminal_fetch_result( channel, authorization, transcript, input.assurance, )?; + if input.input_commitment.digest() != authorization.request_commitment.digest() { + return Err(PaidWorkError::Mismatch { + field: "Fetch input commitment", + }); + } if input.execution_environment == crate::FetchEnvironment::Http.manifest_id() { let request = crate::http_fetch::HttpFetchRequest::decode(input.body.as_bytes()) .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; - let output = crate::fetch::verify_output_events( - input.input_commitment, - input.assurance, - transcript, - ) - .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; crate::http_fetch::HttpFetchResponse::from_output(&request, &output) .map_err(|e| PaidWorkError::Transcript(e.to_string()))?; } diff --git a/crates/sdk/Cargo.toml b/crates/sdk/Cargo.toml index ea25a28e..bd954b32 100644 --- a/crates/sdk/Cargo.toml +++ b/crates/sdk/Cargo.toml @@ -20,7 +20,6 @@ apple-verifier = ["hellas-attestation/apple-app-attest"] gateway = ["dep:hellas-gateway"] provider = [ "client", - "dep:anyhow", "dep:hellas-executor", "dep:hellas-providers", "dep:iroh", @@ -30,16 +29,22 @@ provider = [ ] local-control = ["dep:tokio", "hellas-rpc/host-control", "hellas-wire/unix"] -paid-work = [ - "dep:async-stream", - "provider", +paid-client = ["work", "client", "dep:async-stream", "hellas-client/work"] +paid-provider = ["work", "provider"] +paid-gateway = ["paid-client", "gateway", "dep:rand"] +# Shared channel configuration and provisioning, independent of execution servers. +work = [ + "dep:iroh", + "hellas-wire/iroh", + "dep:tokio", + "dep:tracing", "dep:hellas-chain", "hellas-chain/client", "hellas-chain/work-watcher", "dep:hellas-kernel", "hellas-kernel/secp256k1", "dep:hellas-work", - "hellas-client/work", + "dep:hellas-private", "hellas-rpc/work", "dep:futures", "dep:serde", @@ -49,7 +54,6 @@ paid-work = [ ] [dependencies] -anyhow = { workspace = true, optional = true } async-stream = { workspace = true, optional = true } futures = { workspace = true, optional = true } futures-core = { workspace = true, optional = true } @@ -59,12 +63,14 @@ hellas-client = { workspace = true, optional = true } hellas-executor = { workspace = true, default-features = false, optional = true } hellas-gateway = { workspace = true, optional = true } hellas-kernel = { workspace = true, default-features = false, optional = true } +hellas-private = { workspace = true, optional = true } hellas-providers = { workspace = true, optional = true } hellas-rpc = { workspace = true, features = ["execute", "fetch", "courtesy"] } hellas-wire = { workspace = true, features = ["mux"] } hellas-work = { workspace = true, optional = true } hex = { workspace = true, optional = true } iroh = { workspace = true, features = ["tls-ring"], optional = true } +rand = { workspace = true, optional = true } serde = { workspace = true, optional = true } serde_json = { workspace = true, optional = true } tempfile = { workspace = true } diff --git a/crates/sdk/src/lib.rs b/crates/sdk/src/lib.rs index 4cac0e85..2bf8d168 100644 --- a/crates/sdk/src/lib.rs +++ b/crates/sdk/src/lib.rs @@ -25,7 +25,7 @@ pub use hellas_gateway as gateway; mod provider; #[cfg(feature = "provider")] pub use hellas_executor::{FetchRoute, FetchRouteEntry, FetchRoutePolicy, FetchRouteRegistry}; -#[cfg(feature = "paid-work")] +#[cfg(feature = "work")] pub use hellas_kernel as kernel; #[cfg(feature = "provider")] pub use hellas_providers::{ @@ -33,8 +33,8 @@ pub use hellas_providers::{ }; #[cfg(feature = "provider")] pub use provider::{ - FetchProviderOptions, OpenAiProviderOptions, ProviderHandle, start_fetch_provider, - start_openai_provider, + FetchProviderOptions, OpenAiProviderOptions, ProviderError, ProviderHandle, + start_fetch_provider, start_openai_provider, }; #[cfg(feature = "client")] pub use remote::{ClientIdentity, HellasClient, RemoteFetchRequest}; @@ -48,16 +48,19 @@ pub mod local { pub use hellas_wire::local::{LOCAL_MUX_SLOTS, LocalControlServer, connect, transport}; } -#[cfg(feature = "paid-work")] +#[cfg(feature = "paid-provider")] pub mod paid_provider; -#[cfg(feature = "paid-work")] +#[cfg(feature = "work")] pub mod work_config; -#[cfg(feature = "paid-work")] +#[cfg(feature = "paid-client")] pub mod paid_client; -#[cfg(feature = "paid-work")] +#[cfg(feature = "work")] pub mod work_provision; -#[cfg(all(test, feature = "paid-work"))] +#[cfg(all(test, any(feature = "paid-client", feature = "paid-provider")))] mod test_support; + +#[cfg(feature = "paid-gateway")] +pub mod paid_gateway; diff --git a/crates/sdk/src/paid_client.rs b/crates/sdk/src/paid_client.rs index d0214bea..1cf2db27 100644 --- a/crates/sdk/src/paid_client.rs +++ b/crates/sdk/src/paid_client.rs @@ -44,6 +44,7 @@ use iroh::{Endpoint, EndpointAddr, EndpointId, SecretKey, TransportAddr}; use crate::work_config::WorkConfig; /// Options shared by all jobs on a paid channel. +#[derive(Clone)] pub struct PaidWorkOptions { pub config: WorkConfig, pub journal_root: PathBuf, @@ -1095,6 +1096,10 @@ struct ProviderDialer { provider: EndpointAddr, trust: Option, producer: std::sync::Mutex>, + // Open is authenticated once per exact connection and ALPN. Serialize dials + // so concurrent callers cannot reuse a connection before its proof passes. + setup_connection: tokio::sync::Mutex>, + work_connection: tokio::sync::Mutex>, } impl ProviderDialer { @@ -1107,6 +1112,8 @@ impl ProviderDialer { Self { trust, producer: std::sync::Mutex::new(None), + setup_connection: tokio::sync::Mutex::new(None), + work_connection: tokio::sync::Mutex::new(None), endpoint, provider: EndpointAddr::from_parts( provider, @@ -1138,6 +1145,18 @@ impl ProviderDialer { } async fn connect(&self, alpn: &[u8]) -> Result { + let cache = if alpn == hellas_rpc::services::work::Work::ALPN.as_bytes() { + &self.work_connection + } else { + &self.setup_connection + }; + let mut cached = cache.lock().await; + if let Some(connection) = cached.as_ref().filter(|c| c.close_reason().is_none()) { + return Ok(IrohTransport::new(connection.clone())); + } + // A failed authentication never enters the cache. Reconnects repeat Open + // and still have to match the producer pinned by the payment channel. + *cached = None; let connection = self .endpoint .connect(self.provider.clone(), alpn) @@ -1161,6 +1180,7 @@ impl ProviderDialer { }; self.require_producer(producer)?; } + *cached = Some(transport.connection().clone()); Ok(transport) } } diff --git a/crates/sdk/src/paid_client/tests.rs b/crates/sdk/src/paid_client/tests.rs index fbb12230..38eabf0e 100644 --- a/crates/sdk/src/paid_client/tests.rs +++ b/crates/sdk/src/paid_client/tests.rs @@ -167,8 +167,19 @@ async fn both_paid_connections_open_before_disclosure_and_refuse_wrong_assurance .unwrap(), Some(trust), ); - dialer.setup().await.expect("setup authenticates"); - dialer.work().await.expect("work authenticates"); + let setup = dialer.setup().await.expect("setup authenticates"); + let work = dialer.work().await.expect("work authenticates"); + for _ in 0..3 { + assert_eq!( + setup.open_exporter().unwrap(), + dialer.setup().await.unwrap().open_exporter().unwrap() + ); + assert_eq!( + work.open_exporter().unwrap(), + dialer.work().await.unwrap().open_exporter().unwrap() + ); + } + work.connection().close(0u32.into(), b"test reconnect"); *dialer.producer.lock().unwrap() = Some( ProducerSigningKey::from_secret_bytes([6; 32]) .unwrap() @@ -185,6 +196,7 @@ async fn both_paid_connections_open_before_disclosure_and_refuse_wrong_assurance .contains("differs from the payment channel") ); *dialer.producer.lock().unwrap() = None; + setup.connection().close(0u32.into(), b"test reconnect"); dialer.trust.as_mut().unwrap().required_assurance = Assurance::AppleAppAttest; let error = dialer .setup() diff --git a/crates/cli/src/commands/paid_work/gateway.rs b/crates/sdk/src/paid_gateway.rs similarity index 78% rename from crates/cli/src/commands/paid_work/gateway.rs rename to crates/sdk/src/paid_gateway.rs index c5a8d696..ba032c1c 100644 --- a/crates/cli/src/commands/paid_work/gateway.rs +++ b/crates/sdk/src/paid_gateway.rs @@ -1,6 +1,22 @@ -//! HTTP gateway adapter over the same durable paid-work client as the CLI. +//! Reusable paid gateway pool over durable client-owned state channels. -use super::*; +use crate::paid_client::{ + PaidClientError, PaidWorkOptions, PaidWorkResult, PaidWorkSession, bind_paid_endpoint, + check_evaluate_input, +}; +use hellas_kernel::Secp256k1Signer; +use hellas_rpc::protocol::artifacts::PreparedPaidInputV1; +use hellas_rpc::protocol::work_setup::ProviderChannelPolicy; +use hellas_work::work_store::journal::MAX_RECORD_BYTES; +use iroh::EndpointId; +use std::sync::atomic::AtomicBool; +use std::time::Duration; + +mod config; +mod error; +pub use config::{PaidGatewayOptions, load_pool_options}; +pub use error::PoolError; +type Result = std::result::Result; use futures::future::BoxFuture; use futures::stream::BoxStream; use hellas_gateway::{ @@ -14,7 +30,6 @@ use hellas_rpc::protocol::artifacts::{ }; use hellas_rpc::protocol::work_profile::{PaidWorkPolicy, PreparedPaidWorkInput}; use iroh::Endpoint; -use serde::Deserialize; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Instant; @@ -22,39 +37,6 @@ use tokio::sync::{Mutex as AsyncMutex, OwnedSemaphorePermit, Semaphore, mpsc, wa use tokio::task::JoinHandle; use tracing::Instrument; -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct PoolFile { - providers: Vec, - #[serde(default = "acceptance_blocks")] - acceptance_blocks: u64, - #[serde(default = "terminal_blocks")] - terminal_blocks: u64, - #[serde(default = "payment_blocks")] - payment_blocks: u64, - #[serde(default = "timeout_secs")] - timeout_secs: u64, - #[serde(default = "max_pending_requests")] - max_pending_requests: usize, -} - -const fn max_pending_requests() -> usize { - 64 -} - -const fn acceptance_blocks() -> u64 { - 16 -} -const fn terminal_blocks() -> u64 { - 64 -} -const fn payment_blocks() -> u64 { - 32 -} -const fn timeout_secs() -> u64 { - 300 -} - // An OpenCode conversation has a substantial shared chat prefix. Smaller // checkpoints are not worth routing work around. const MIN_CACHE_AFFINITY_TOKENS: usize = 128; @@ -71,39 +53,21 @@ const CHANNEL_QUEUE_BUDGET: Duration = Duration::from_secs(1); const OUTPUT_BUFFER_BYTES: usize = 2 * MAX_RECORD_BYTES; const OUTPUT_EVENT_OVERHEAD: usize = 1024; const OUTPUT_BUFFER_EVENTS: usize = OUTPUT_BUFFER_BYTES / OUTPUT_EVENT_OVERHEAD; -type BufferedEvent = (CliResult, OwnedSemaphorePermit); +type BufferedEvent = (Result, OwnedSemaphorePermit); trait GatewayEvent: Send + 'static { - fn prefix(event: hellas_rpc::OutputEventEnvelope) -> CliResult + fn prefix(event: hellas_rpc::OutputEventEnvelope) -> Result where Self: Sized; - fn completed(output: PaidWorkResult) -> CliResult> + fn completed(output: PaidWorkResult) -> Result> where Self: Sized; fn is_terminal(&self) -> bool; fn bytes(&self) -> usize; } -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct ProviderFile { - work_config: PathBuf, - journal_root: PathBuf, - provider: EndpointId, - #[serde(default)] - provider_addrs: Vec, - bond: String, - payment_coins: Vec, - omission_bond: u64, - /// Required enrollment pin for attested providers; optional for producer-signed work. - provider_genesis: Option, - apple_app_id: Option, - #[serde(default)] - apple_cd_hashes: Vec, -} - struct Provider { - args: RunArgs, + args: PaidWorkOptions, policy: ProviderChannelPolicy, assurance: hellas_rpc::Assurance, /// A setup/channel journal has a single owner even with concurrent HTTP calls. @@ -225,7 +189,7 @@ impl Drop for ProviderUse { } } -struct PaidGateway { +pub struct PaidGateway { providers: Vec>, next: AtomicUsize, endpoint: Endpoint, @@ -235,145 +199,74 @@ struct PaidGateway { tasks: Mutex>>, } -pub async fn load_gateway_backend( - path: &Path, - transport_key: SecretKey, - settlement_key: Secp256k1Signer, - producer_key: hellas_rpc::ProducerSigningKey, - assurance: hellas_rpc::Assurance, -) -> CliResult> { - anyhow::ensure!( - producer_key.public_key() - == hellas_rpc::PublicKey::Secp256k1(settlement_key.party_key().to_bytes()), - "paid Fetch signer must match the settlement identity" - ); - let bytes = - crate::commands::read_bounded_regular_file(path, "paid gateway config", MAX_RECORD_BYTES)?; - let file: PoolFile = serde_json::from_slice(&bytes) - .with_context(|| format!("invalid paid gateway config {}", path.display()))?; - anyhow::ensure!( - !file.providers.is_empty(), - "paid gateway requires at least one provider" - ); - anyhow::ensure!( - file.timeout_secs > 0, - "paid gateway timeout_secs must be greater than zero" - ); - anyhow::ensure!( - file.acceptance_blocks > 0 && file.terminal_blocks > 0 && file.payment_blocks > 0, - "paid gateway deadline spans must be greater than zero" - ); - anyhow::ensure!( - file.max_pending_requests > 0 && file.max_pending_requests <= Semaphore::MAX_PERMITS, - "paid gateway max_pending_requests must be a positive supported semaphore capacity" - ); - let mut providers = Vec::new(); - let mut journals = std::collections::BTreeSet::new(); - let mut endpoints = std::collections::BTreeSet::new(); - let mut funding = std::collections::BTreeSet::new(); - for provider in file.providers { - anyhow::ensure!( - provider.work_config.is_absolute() && provider.journal_root.is_absolute(), - "paid gateway work_config and journal_root must be absolute runtime paths" - ); - anyhow::ensure!( - journals.insert(provider.journal_root.clone()), - "paid providers must have distinct journal roots" - ); - anyhow::ensure!( - endpoints.insert(provider.provider), - "paid gateway repeats provider {}", - provider.provider - ); - anyhow::ensure!( - !provider.payment_coins.is_empty(), - "paid provider needs payment_coins" - ); - coins(&provider.payment_coins)?; - edge_id("bond", &provider.bond)?; - for coin in &provider.payment_coins { - anyhow::ensure!( - funding.insert(fixed_hex::<32>("payment_coins", coin)?), - "a payment coin cannot fund two provider channels" - ); +impl PaidGateway { + /// Open a reusable pool. Funding, recovery, queueing and payment are shared + /// by CLI and embedding applications. Call `drain` during host shutdown. + pub async fn open( + options: PaidGatewayOptions, + identity: crate::ClientIdentity, + ) -> Result> { + options.validate()?; + let settlement_key = Secp256k1Signer::from_secret_scalar(identity.caller_secret_bytes()) + .map_err(|_| PoolError::Invalid("settlement identity is not secp256k1"))?; + let providers = options + .providers + .into_iter() + .map(|args| { + Arc::new(Provider { + policy: args.config.provider_policy(), + assurance: args + .provider_trust + .as_ref() + .map_or(hellas_rpc::Assurance::ProducerSigned, |trust| { + trust.required_assurance + }), + args, + serial: AsyncMutex::new(None), + pending: AtomicUsize::new(0), + cache: Mutex::new(PrefixCache::default()), + unavailable_until: Mutex::new(None), + }) + }) + .collect(); + let gateway = Arc::new(PaidGateway { + admission: Arc::new(Semaphore::new(options.max_pending_requests)), + providers, + next: AtomicUsize::new(0), + // One transport identity has one relay registration, shared by every + // provider and request for the lifetime of this gateway. + endpoint: bind_paid_endpoint(identity.transport_key()).await?, + settlement_key, + producer_key: identity.caller_key().clone(), + tasks: Mutex::new(Vec::new()), + }); + // Restart recovery uses the retained input and certificate, never a new job. + // Empty journal roots do not fund a channel until an HTTP request arrives. + for provider in &gateway.providers { + let _recovery = gateway.submit::( + vec![( + provider.clone(), + Route { + available: true, + cache_affinity_tokens: 0, + pending: 0, + }, + )], + None, + None, + Some(RECOVERY_ATTEMPT_TIMEOUT), + None, + )?; } - let config = load_work_config(&provider.work_config)?; - providers.push(Arc::new(Provider { - policy: config.provider_policy(), - assurance, - args: RunArgs { - provider_genesis: provider - .provider_genesis - .map(|value| value.parse()) - .transpose()?, - apple_app_id: provider.apple_app_id, - apple_cd_hashes: provider - .apple_cd_hashes - .iter() - .map(|value| fixed_hex::<32>("apple_cd_hashes", value)) - .collect::>()?, - work_config: provider.work_config, - journal_root: provider.journal_root, - provider: provider.provider, - provider_addrs: provider.provider_addrs, - bond: provider.bond, - payment_coins: provider.payment_coins, - omission_bond: provider.omission_bond, - prepared_input: PathBuf::new(), - output: None, - acceptance_blocks: file.acceptance_blocks, - terminal_blocks: file.terminal_blocks, - payment_blocks: file.payment_blocks, - timeout_secs: file.timeout_secs, - settle: false, - }, - serial: AsyncMutex::new(None), - pending: AtomicUsize::new(0), - cache: Mutex::new(PrefixCache::default()), - unavailable_until: Mutex::new(None), - })); - let provider = providers.last().expect("provider just added"); - // Validated now so a bad pin fails startup before the gateway binds; - // the anchor itself is recomputed when the channel opens. - paid_provider_trust(&provider.args, provider.assurance)?; + Ok(gateway) } - let gateway = Arc::new(PaidGateway { - admission: Arc::new(Semaphore::new(file.max_pending_requests)), - providers, - next: AtomicUsize::new(0), - // One transport identity has one relay registration, shared by every - // provider and request for the lifetime of this gateway. - endpoint: bind_paid_endpoint(transport_key).await?, - settlement_key, - producer_key, - tasks: Mutex::new(Vec::new()), - }); - // Restart recovery uses the retained input and certificate, never a new job. - // Empty journal roots do not fund a channel until an HTTP request arrives. - for provider in &gateway.providers { - let _recovery = gateway.submit::( - vec![( - provider.clone(), - Route { - available: true, - cache_affinity_tokens: 0, - pending: 0, - }, - )], - None, - None, - Some(RECOVERY_ATTEMPT_TIMEOUT), - None, - )?; - } - Ok(gateway) } impl PaidGateway { fn execute_tokens( &self, request: PaidExecutionRequest, - ) -> CliResult>> { + ) -> Result>> { let permit = self .admission .clone() @@ -387,10 +280,9 @@ impl PaidGateway { .iter() .filter(|provider| check_evaluate_input(&provider.policy, &prepared).is_ok()) .collect::>(); - anyhow::ensure!( - !eligible.is_empty(), - "no provider policy matches this environment, token limit, and stop token list" - ); + if eligible.is_empty() { + return Err(PoolError::NoMatchingPolicy); + } let start = self.next.fetch_add(1, Ordering::Relaxed) % eligible.len(); let mut candidates = (0..eligible.len()) .map(|offset| { @@ -422,7 +314,7 @@ impl PaidGateway { cache_update: Option, recovery_timeout: Option, permit: Option, - ) -> CliResult>> { + ) -> Result>> { // Admission can precede local request preparation. Serialize the final // gate check and registration with drain's close-and-snapshot boundary. let mut tasks = self.tasks.lock().expect("paid task list poisoned"); @@ -435,7 +327,7 @@ impl PaidGateway { let (overflow, overflow_receiver) = watch::channel(false); let output_budget = Arc::new(Semaphore::new(OUTPUT_BUFFER_BYTES)); let Some((initial_provider, _initial_route)) = candidates.first() else { - anyhow::bail!("paid gateway has no provider candidate"); + return Err(PoolError::NoMatchingPolicy); }; let span = hellas_rpc::request_span!( target: "hellas_request", "paid.gateway", @@ -449,8 +341,7 @@ impl PaidGateway { let mut occupied = prepared .as_ref() .map(|_| ProviderUse::new(initial_provider.clone())); - let timeout = recovery_timeout - .unwrap_or_else(|| Duration::from_secs(initial_provider.args.timeout_secs)); + let timeout = recovery_timeout.unwrap_or_else(|| initial_provider.args.timeout); // Queueing, recovery and fallback all consume the same request budget. let deadline = tokio::time::Instant::now() + timeout; let task_span = span.clone(); @@ -497,7 +388,7 @@ impl PaidGateway { } } }; - if prepared.is_none() && (!provider.args.journal_root.try_exists()? || std::fs::read_dir(&provider.args.journal_root)?.next().is_none()) { + if prepared.is_none() && !has_retained_setup(&provider.args)? { continue; } // Recheck after queueing, including restored channels whose @@ -509,14 +400,14 @@ impl PaidGateway { if session.is_none() { match connect_before_deadline( &sender, streamed, deadline, PROVIDER_CONNECTION_TIMEOUT, - open_paid_channel(&provider.args, endpoint.clone(), settlement_key.clone(), provider.assurance), + async { PaidWorkSession::open(provider.args.clone(), endpoint.clone(), settlement_key.clone()).await.map_err(PoolError::from) }, ).await { Ok(opened) => { *session = Some(opened); } Err(error) => { - if !recovery && error.is::() { return Err(error); } + if !recovery && matches!(error, PoolError::Stopped(_)) { return Err(error); } provider.connection_failed(); if recovery { tracing::debug!(provider = %provider.args.provider, error = %format!("{error:#}"), @@ -540,10 +431,8 @@ impl PaidGateway { session.run(None, true, None), ) .await - .map_err(|_| anyhow::anyhow!( - "retained paid work did not recover within {RECOVERY_ATTEMPT_TIMEOUT:?}" - )) - .and_then(|result| result.map_err(anyhow::Error::from)); + .map_err(|_| PoolError::RecoveryTimeout) + .and_then(|result| result.map_err(PoolError::from)); if let Err(error) = recovery { // Nothing in this path has proposed the fresh request. // Keep the journal for a later recovery and route this @@ -573,11 +462,11 @@ impl PaidGateway { request_session.run_with_admission(input, true, on_progress, Some(&proposed)).await }, ).await - .and_then(|result| result.map_err(anyhow::Error::from)) + .and_then(|result| result.map_err(PoolError::from)) .and_then(|output| output.map(E::completed).transpose()) .map(Option::unwrap_or_default); if let Err(error) = &result { - if !recovery && error.is::() + if !recovery && matches!(error, PoolError::Stopped(_)) && session.with_state(|state| state.proposal_nonce_high_water())? == proposal_nonce { return result; @@ -585,16 +474,14 @@ impl PaidGateway { provider.cache.lock().expect("provider cache poisoned").replace(None); if recovery { provider.connection_failed(); - return result.with_context(|| { - format!("retained paid work at provider {}", provider.args.provider) - }); + return result.map_err(|source| PoolError::Provider { provider: provider.args.provider, source: Box::new(source) }); } // A failed journal append can leave a durable proposal that // is not reflected in memory yet. Keep that failure with // this provider, just like a proposal with no response. let uncertain_append = matches!( - error.downcast_ref::(), - Some(hellas_sdk::paid_client::PaidClientError::Propose(hellas_work::work::ProposeError::Store(_))), + error, + PoolError::Client(PaidClientError::Propose(hellas_work::work::ProposeError::Store(_))), ); if !already_proposed && !uncertain_append && session.with_state(|state| state.proposal_nonce_high_water())? == proposal_nonce @@ -611,12 +498,9 @@ impl PaidGateway { cache_update.apply(&mut provider.cache.lock().expect("provider cache poisoned")); } } - return result.with_context(|| format!("paid provider {}", provider.args.provider)); + return result.map_err(|source| PoolError::Provider { provider: provider.args.provider, source: Box::new(source) }); } - Err(anyhow::anyhow!( - "no eligible paid provider could start this request: {}", - provider_errors.join("; ") - )) + Err(PoolError::ProvidersUnavailable(provider_errors)) }.await; if let Err(error) = &result { if recovery { @@ -645,6 +529,20 @@ impl PaidGateway { } } +fn has_retained_setup(options: &PaidWorkOptions) -> Result { + let found = hellas_work::work_store::discover_setups( + &options.journal_root, + options.config.chain.network, + )?; + // Counter files and other metadata alone must never trigger channel funding. + // Unidentified setup journals still go through ordinary recovery and fail + // closed; discovery is only a prefilter, not a signature check. + Ok(!found.unidentified.is_empty() + || found.setups.iter().any(|setup| { + setup.role == hellas_work::work_store::Role::Client && setup.bond_edge == options.bond + })) +} + impl PaidExecutionBackend for PaidGateway { fn fetch_providers(&self) -> Vec { self.providers.iter().filter(|provider| matches!(provider.policy.execution_policy, @@ -666,7 +564,7 @@ impl PaidExecutionBackend for PaidGateway { .clone() .try_acquire_owned() .map_err(|_| hellas_gateway::PaidGatewayBusy)?; - let submit = || -> CliResult<_> { + let submit = || -> Result<_> { let environment = hellas_rpc::FetchEnvironment::Http; let input = hellas_rpc::fetch::build_input_events_with_retention( &request.service, @@ -700,7 +598,11 @@ impl PaidExecutionBackend for PaidGateway { } fn timeout(&self) -> Duration { - Duration::from_secs(self.providers[0].args.timeout_secs) + self.providers + .iter() + .map(|provider| provider.args.timeout) + .max() + .unwrap_or_default() } fn execute( @@ -750,13 +652,13 @@ impl PaidExecutionBackend for PaidGateway { } fn paid_stream( - stream: CliResult>>, + stream: Result>>, ) -> Result, hellas_gateway::PaidGatewayError> { use futures::StreamExt as _; - fn convert(error: anyhow::Error) -> hellas_gateway::PaidGatewayError { - match error.downcast::() { - Ok(busy) => busy.into(), - Err(error) => hellas_gateway::PaidGatewayError::Payment(error.into_boxed_dyn_error()), + fn convert(error: PoolError) -> hellas_gateway::PaidGatewayError { + match error { + PoolError::Busy(busy) => busy.into(), + error => hellas_gateway::PaidGatewayError::Payment(Box::new(error)), } } Ok(Box::pin( @@ -768,7 +670,7 @@ fn emit( sender: &mpsc::Sender>, overflow: &watch::Sender, budget: &Arc, - event: CliResult, + event: Result, ) { if *overflow.borrow() || sender.is_closed() { return; @@ -796,13 +698,13 @@ fn emit( fn response_stream( mut receiver: mpsc::Receiver>, mut overflow: watch::Receiver, -) -> BoxStream<'static, CliResult> { +) -> BoxStream<'static, Result> { Box::pin(async_stream::try_stream! { loop { let full = *overflow.borrow(); if full { receiver.close(); - Err(anyhow::anyhow!("paid output consumer is too slow; accepted work continues settlement"))?; + Err(PoolError::SlowConsumer)?; } let event = tokio::select! { biased; @@ -821,7 +723,7 @@ fn response_stream( } #[derive(Debug, thiserror::Error)] -enum RequestStopped { +pub enum RequestStopped { #[error("paid request disconnected before proposal")] Disconnected, #[error("paid request deadline elapsed, including queue wait")] @@ -835,17 +737,13 @@ async fn connect_before_deadline( cancel_on_disconnect: bool, deadline: tokio::time::Instant, connection_timeout: Duration, - connection: impl std::future::Future>, -) -> CliResult { + connection: impl std::future::Future>, +) -> Result { let connection_deadline = deadline.min(tokio::time::Instant::now() + connection_timeout); before_proposal(sender, cancel_on_disconnect, deadline, |_| async { tokio::time::timeout_at(connection_deadline, connection) .await - .map_err(|_| { - anyhow::anyhow!( - "paid provider connection exceeded its {connection_timeout:?} limit" - ) - })? + .map_err(|_| PoolError::ConnectionTimeout(connection_timeout))? }) .await? } @@ -857,7 +755,7 @@ async fn before_proposal, E>( cancel_on_disconnect: bool, deadline: tokio::time::Instant, operation: impl FnOnce(Arc) -> F, -) -> CliResult { +) -> Result { if tokio::time::Instant::now() >= deadline { return Err(RequestStopped::Deadline.into()); } @@ -883,11 +781,12 @@ async fn before_proposal, E>( fn prepare_request( request: PaidExecutionRequest, signer: &Secp256k1Signer, -) -> CliResult { - anyhow::ensure!( - request.max_new_tokens > 0, - "max_new_tokens must be greater than zero" - ); +) -> Result { + if request.max_new_tokens == 0 { + return Err(PoolError::Invalid( + "max_new_tokens must be greater than zero", + )); + } let manifest = request.environment.manifest(); hellas_client::iroh::validate_causal_lm_quote_request( &hellas_rpc::pb::courtesy::QuoteTokensRequest { @@ -921,7 +820,7 @@ fn prepare_request( )) } -fn output_events(output: PaidWorkResult) -> CliResult> { +fn output_events(output: PaidWorkResult) -> Result> { let events = hellas_rpc::protocol::work::decode_transcript(&output.transcript, MAX_RECORD_BYTES)?; let verified = hellas_rpc::evaluate::verify_output_events_for_producer( @@ -940,11 +839,9 @@ fn output_events(output: PaidWorkResult) -> CliResult> { let terminal = verified.terminal; let stop_reason = if terminal.stop_reason == hellas_rpc::evaluate::EvaluateStopReason::STOP_TOKEN { - StopReason::StopToken( - terminal - .matched_stop_token_id - .context("signed stop-token result omitted its token ID")?, - ) + StopReason::StopToken(terminal.matched_stop_token_id.ok_or( + PoolError::MissingOutput("signed stop-token result omitted its token ID"), + )?) } else { StopReason::MaxNewTokens }; @@ -965,14 +862,14 @@ fn output_events(output: PaidWorkResult) -> CliResult> { } impl GatewayEvent for ExecutionEvent { - fn prefix(event: hellas_rpc::OutputEventEnvelope) -> CliResult { + fn prefix(event: hellas_rpc::OutputEventEnvelope) -> Result { let delta = hellas_rpc::evaluate::decode_token_delta_payload(event.payload())?; Ok(Self::Chunk { position: delta.end_position()?, tokens: delta.token_bytes(), }) } - fn completed(output: PaidWorkResult) -> CliResult> { + fn completed(output: PaidWorkResult) -> Result> { output_events(output) } fn is_terminal(&self) -> bool { @@ -991,21 +888,22 @@ impl GatewayEvent for ExecutionEvent { } impl GatewayEvent for FetchEvent { - fn prefix(event: hellas_rpc::OutputEventEnvelope) -> CliResult { + fn prefix(event: hellas_rpc::OutputEventEnvelope) -> Result { Ok(hellas_rpc::fetch::decode_fetch_event_payload( event.payload(), )?) } - fn completed(output: PaidWorkResult) -> CliResult> { + fn completed(output: PaidWorkResult) -> Result> { let events = hellas_rpc::protocol::work::decode_transcript( &output.transcript, hellas_rpc::protocol::work_fetch::MAX_FETCH_TRANSCRIPT_BYTES, )?; - let terminal = events.last().context("paid Fetch omitted terminal")?; - anyhow::ensure!( - terminal.event().body().kind() == hellas_rpc::fetch::OUTPUT_TERMINAL_KIND, - "paid Fetch terminal kind" - ); + let terminal = events + .last() + .ok_or(PoolError::MissingOutput("paid Fetch omitted terminal"))?; + if terminal.event().body().kind() != hellas_rpc::fetch::OUTPUT_TERMINAL_KIND { + return Err(PoolError::MissingOutput("paid Fetch terminal kind")); + } let terminal = hellas_rpc::fetch::decode_fetch_terminal_payload(terminal.payload())?.to_output_event(); tracing::info!(work_id = %hex::encode(output.work_id.as_bytes()), job_price = output.job_price, @@ -1025,6 +923,76 @@ mod tests { use super::*; use iroh::Endpoint; + fn pool_options(fixture: &crate::test_support::PaidFixture) -> PaidGatewayOptions { + use hellas_kernel::{CoinId, Funding, List}; + PaidGatewayOptions { + max_pending_requests: 4, + providers: vec![PaidWorkOptions { + config: fixture.config.clone(), + journal_root: fixture.root.path().join("client"), + provider: iroh::SecretKey::from_bytes(&[3; 32]).public(), + provider_addrs: Vec::new(), + provider_trust: None, + bond: fixture.descriptor.bond_edge(), + payment_funding: Funding::new( + List::take([CoinId::from_bytes([1; 32]); 4], 1), + List::empty(CoinId::from_bytes([0; 32])), + ), + omission_bond: 601, + acceptance_blocks: 16, + terminal_blocks: 64, + payment_blocks: 32, + timeout: Duration::from_secs(30), + }], + } + } + + #[test] + fn counter_files_do_not_trigger_funding_but_incomplete_setup_recovers() { + let fixture = crate::test_support::PaidFixture::new(); + let options = pool_options(&fixture); + let provider = &options.providers[0]; + assert!(!has_retained_setup(provider).unwrap()); + std::fs::create_dir_all(provider.journal_root.join("apple-counters")).unwrap(); + std::fs::write( + provider + .journal_root + .join("apple-counters/producer.counter"), + [1], + ) + .unwrap(); + assert!(!has_retained_setup(provider).unwrap()); + let store = hellas_work::work_store::SetupStore::open( + &provider.journal_root, + provider.config.chain.network, + provider.bond, + hellas_work::work_store::Role::Client, + &hellas_kernel::Secp256k1Verifier::new(), + ) + .unwrap(); + drop(store); + assert!(has_retained_setup(provider).unwrap()); + } + + #[tokio::test] + async fn shared_pool_rejects_reused_funding_before_opening_or_writing() { + let fixture = crate::test_support::PaidFixture::new(); + let mut options = pool_options(&fixture); + let mut second = options.providers[0].clone(); + second.provider = iroh::SecretKey::from_bytes(&[4; 32]).public(); + second.journal_root = fixture.root.path().join("other"); + options.providers.push(second); + let result = PaidGateway::open(options, crate::ClientIdentity::generate()).await; + assert!(matches!( + result, + Err(PoolError::Invalid( + "a payment coin cannot fund two provider channels or appear twice" + )) + )); + assert!(!fixture.root.path().join("client").exists()); + assert!(!fixture.root.path().join("other").exists()); + } + #[tokio::test] async fn disconnected_queued_request_never_starts_work() { let (sender, receiver) = mpsc::channel::(OUTPUT_BUFFER_EVENTS); @@ -1091,7 +1059,7 @@ mod tests { drop(receiver); }; let (result, ()) = tokio::join!(operation, disconnect); - assert!(result.unwrap_err().is::()); + assert!(matches!(result.unwrap_err(), PoolError::Stopped(_))); assert!(!signed.load(Ordering::Acquire)); } @@ -1104,11 +1072,11 @@ mod tests { true, deadline, Duration::from_millis(10), - std::future::pending::>(), + std::future::pending::>(), ) .await .unwrap_err(); - assert!(!failed.is::()); + assert!(!matches!(failed, PoolError::Stopped(_))); assert_eq!( connect_before_deadline(&sender, true, deadline, Duration::from_millis(10), async { Ok("second provider") @@ -1126,7 +1094,7 @@ mod tests { ) .await .unwrap_err(); - assert!(expired.is::()); + assert!(matches!(expired, PoolError::Stopped(_))); } #[tokio::test] @@ -1146,7 +1114,7 @@ mod tests { started.store(true, Ordering::Release); }) .await; - assert!(result.unwrap_err().is::()); + assert!(matches!(result.unwrap_err(), PoolError::Stopped(_))); assert!(!started.load(Ordering::Acquire)); } @@ -1169,12 +1137,10 @@ mod tests { let permit = gateway.admission.clone().try_acquire_owned().unwrap(); gateway.drain().await; let result = gateway.submit::(Vec::new(), None, None, None, Some(permit)); - assert!( - result - .err() - .expect("submission after drain") - .is::() - ); + assert!(matches!( + result.err().expect("submission after drain"), + PoolError::Busy(_) + )); assert!(gateway.tasks.lock().unwrap().is_empty()); assert_eq!(gateway.admission.available_permits(), 1); } diff --git a/crates/sdk/src/paid_gateway/config.rs b/crates/sdk/src/paid_gateway/config.rs new file mode 100644 index 00000000..3a55cdee --- /dev/null +++ b/crates/sdk/src/paid_gateway/config.rs @@ -0,0 +1,249 @@ +use super::{PaidWorkOptions, PoolError, Result}; +use crate::work_config::load_work_config; +use hellas_kernel::{CoinId, EdgeId, Funding, List, MAX_PARTY_INPUTS}; +use hellas_rpc::{Assurance, ContentId}; +use iroh::EndpointId; +use serde::Deserialize; +use std::{ + collections::BTreeSet, + net::SocketAddr, + path::{Path, PathBuf}, + time::Duration, +}; + +/// Typed funding and admission options for an application-owned paid pool. +pub struct PaidGatewayOptions { + pub providers: Vec, + pub max_pending_requests: usize, +} + +impl PaidGatewayOptions { + pub(super) fn validate(&self) -> Result<()> { + if self.providers.is_empty() { + return Err(PoolError::Invalid( + "paid gateway requires at least one provider", + )); + } + if self.max_pending_requests == 0 + || self.max_pending_requests > tokio::sync::Semaphore::MAX_PERMITS + { + return Err(PoolError::Invalid( + "max_pending_requests must be a positive supported semaphore capacity", + )); + } + let mut journals = BTreeSet::new(); + let mut endpoints = BTreeSet::new(); + let mut funding = BTreeSet::new(); + for provider in &self.providers { + if provider.timeout.is_zero() + || provider.acceptance_blocks == 0 + || provider.terminal_blocks == 0 + || provider.payment_blocks == 0 + { + return Err(PoolError::Invalid( + "timeouts and deadline spans must be positive", + )); + } + if !provider.journal_root.is_absolute() { + return Err(PoolError::Invalid("journal_root must be absolute")); + } + if !journals.insert(&provider.journal_root) { + return Err(PoolError::Invalid( + "paid providers must have distinct journal roots", + )); + } + if !endpoints.insert(provider.provider) { + return Err(PoolError::DuplicateProvider(provider.provider)); + } + let coins = provider + .payment_funding + .maker() + .iter() + .chain(provider.payment_funding.taker().iter()) + .collect::>(); + if coins.is_empty() { + return Err(PoolError::Invalid("paid provider needs payment coins")); + } + for coin in coins { + if !funding.insert(*coin) { + return Err(PoolError::Invalid( + "a payment coin cannot fund two provider channels or appear twice", + )); + } + } + } + Ok(()) + } +} + +/// Load the operator JSON format used by both CLI and native hosts. File paths +/// must be absolute; Apple counter state lives beside the channel journals. +pub fn load_pool_options(path: &Path, assurance: Assurance) -> Result { + let bytes = hellas_private::read_bounded_regular_file( + path, + hellas_work::work_store::journal::MAX_RECORD_BYTES, + ) + .map_err(|source| PoolError::Read { + path: path.into(), + source, + })?; + let file: PoolFile = serde_json::from_slice(&bytes).map_err(|source| PoolError::Parse { + path: path.into(), + source, + })?; + let providers = file + .providers + .into_iter() + .map(|p| { + if !p.work_config.is_absolute() { + return Err(PoolError::Invalid("work_config must be absolute")); + } + if p.payment_coins.is_empty() || p.payment_coins.len() > MAX_PARTY_INPUTS { + return Err(PoolError::Invalid("payment_coins must name 1 to 4 coins")); + } + let coins = p + .payment_coins + .iter() + .map(|value| fixed("payment_coins", value).map(CoinId::from_bytes)) + .collect::>>()?; + let slots = std::array::from_fn(|i| { + coins + .get(i) + .copied() + .unwrap_or(CoinId::from_bytes([0; CoinId::LENGTH])) + }); + let provider_trust = trust(&p, assurance)?; + Ok(PaidWorkOptions { + config: load_work_config(&p.work_config)?, + journal_root: p.journal_root, + provider: p.provider, + provider_addrs: p.provider_addrs, + provider_trust, + bond: EdgeId::from_bytes(fixed("bond", &p.bond)?), + payment_funding: Funding::new( + List::take(slots, coins.len()), + List::empty(CoinId::from_bytes([0; CoinId::LENGTH])), + ), + omission_bond: p.omission_bond, + acceptance_blocks: file.acceptance_blocks, + terminal_blocks: file.terminal_blocks, + payment_blocks: file.payment_blocks, + timeout: Duration::from_secs(file.timeout_secs), + }) + }) + .collect::>()?; + let options = PaidGatewayOptions { + providers, + max_pending_requests: file.max_pending_requests, + }; + options.validate()?; + Ok(options) +} + +fn trust( + p: &ProviderFile, + assurance: Assurance, +) -> Result> { + if p.provider_genesis.is_none() && assurance == Assurance::ProducerSigned { + if p.apple_app_id.is_some() || !p.apple_cd_hashes.is_empty() { + return Err(PoolError::Invalid("Apple trust requires provider_genesis")); + } + return Ok(None); + } + let expected_genesis = p.provider_genesis.ok_or(PoolError::Invalid( + "attested paid providers require provider_genesis", + ))?; + let apple_app_attest = if p.apple_app_id.is_none() + && p.apple_cd_hashes.is_empty() + && assurance == Assurance::ProducerSigned + { + None + } else { + #[cfg(feature = "apple-verifier")] + { + let app_id = p + .apple_app_id + .clone() + .filter(|v| !v.is_empty()) + .ok_or(PoolError::Invalid("Apple trust requires apple_app_id"))?; + if p.apple_cd_hashes.is_empty() { + return Err(PoolError::Invalid("Apple trust requires apple_cd_hashes")); + } + Some(hellas_client::AppleAppAttestTrust::new( + app_id, + p.apple_cd_hashes + .iter() + .map(|v| fixed("apple_cd_hashes", v)) + .collect::>()?, + std::sync::Arc::new(crate::FilesystemAssertionCounterStore::new( + p.journal_root.join("apple-counters"), + )), + )) + } + #[cfg(not(feature = "apple-verifier"))] + { + return Err(PoolError::Invalid( + "Apple trust requires the SDK apple-verifier feature", + )); + } + }; + Ok(Some(hellas_client::ProviderTrustAnchor { + expected_genesis, + required_assurance: assurance, + apple_app_attest, + })) +} + +fn fixed(field: &'static str, value: &str) -> Result<[u8; N]> { + let mut bytes = [0; N]; + hex::decode_to_slice(value, &mut bytes).map_err(|source| PoolError::Hex { field, source })?; + Ok(bytes) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PoolFile { + providers: Vec, + #[serde(default = "acceptance_blocks")] + acceptance_blocks: u64, + #[serde(default = "terminal_blocks")] + terminal_blocks: u64, + #[serde(default = "payment_blocks")] + payment_blocks: u64, + #[serde(default = "timeout_secs")] + timeout_secs: u64, + #[serde(default = "max_pending_requests")] + max_pending_requests: usize, +} +const fn acceptance_blocks() -> u64 { + 16 +} +const fn terminal_blocks() -> u64 { + 64 +} +const fn payment_blocks() -> u64 { + 32 +} +const fn timeout_secs() -> u64 { + 300 +} +const fn max_pending_requests() -> usize { + 64 +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ProviderFile { + work_config: PathBuf, + journal_root: PathBuf, + provider: EndpointId, + #[serde(default)] + provider_addrs: Vec, + bond: String, + payment_coins: Vec, + omission_bond: u64, + provider_genesis: Option, + apple_app_id: Option, + #[serde(default)] + apple_cd_hashes: Vec, +} diff --git a/crates/sdk/src/paid_gateway/error.rs b/crates/sdk/src/paid_gateway/error.rs new file mode 100644 index 00000000..a631bc08 --- /dev/null +++ b/crates/sdk/src/paid_gateway/error.rs @@ -0,0 +1,65 @@ +use std::{path::PathBuf, time::Duration}; + +#[derive(Debug, thiserror::Error)] +pub enum PoolError { + #[error("invalid paid pool: {0}")] + Invalid(&'static str), + #[error("paid gateway repeats provider {0}")] + DuplicateProvider(iroh::EndpointId), + #[error("no provider policy matches this environment, token limit, and stop token list")] + NoMatchingPolicy, + #[error("no eligible paid provider could start this request: {0:?}")] + ProvidersUnavailable(Vec), + #[error("retained paid work did not recover within its deadline")] + RecoveryTimeout, + #[error("paid provider connection exceeded its {0:?} limit")] + ConnectionTimeout(Duration), + #[error("paid output consumer is too slow; accepted work continues settlement")] + SlowConsumer, + #[error("{0}")] + MissingOutput(&'static str), + #[error("paid provider {provider}: {source}")] + Provider { + provider: iroh::EndpointId, + source: Box, + }, + #[error("cannot read {}: {source}", path.display())] + Read { + path: PathBuf, + source: std::io::Error, + }, + #[error("invalid paid gateway config {}: {source}", path.display())] + Parse { + path: PathBuf, + source: serde_json::Error, + }, + #[error("invalid {field}: {source}")] + Hex { + field: &'static str, + source: hex::FromHexError, + }, + #[error(transparent)] + Stopped(#[from] super::RequestStopped), + #[error(transparent)] + Busy(#[from] hellas_gateway::PaidGatewayBusy), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Config(#[from] crate::work_config::WorkConfigError), + #[error(transparent)] + Client(#[from] crate::paid_client::PaidClientError), + #[error(transparent)] + Canonical(#[from] hellas_rpc::protocol::value::CanonicalDecodeError), + #[error(transparent)] + Work(#[from] hellas_rpc::protocol::work::PaidWorkError), + #[error(transparent)] + Store(#[from] hellas_work::work_store::WorkStoreError), + #[error(transparent)] + Fetch(#[from] hellas_rpc::fetch::FetchProtocolError), + #[error(transparent)] + FetchPayload(#[from] hellas_rpc::fetch::FetchPayloadError), + #[error(transparent)] + Evaluate(#[from] hellas_rpc::evaluate::EvaluateProtocolError), + #[error(transparent)] + Remote(#[from] hellas_client::ClientError), +} diff --git a/crates/sdk/src/provider.rs b/crates/sdk/src/provider.rs index ba885d17..b9338df7 100644 --- a/crates/sdk/src/provider.rs +++ b/crates/sdk/src/provider.rs @@ -27,6 +27,41 @@ use crate::ClientIdentity; const MAX_ACTIVE_CONNECTIONS: usize = 64; const HANDSHAKE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); +/// Startup errors, before a provider accepts requests. +#[derive(Debug, thiserror::Error)] +pub enum ProviderError { + #[error("provider requires allowed callers or paid-work configuration")] + MissingAccessPolicy, + #[error("paid channel names an unavailable Fetch route or manifest")] + MissingPaidRoute, + #[error("Fetch provider requires a paid Fetch policy")] + WrongPaidPolicy, + #[error("paid Fetch provider requires zero retained transcript capacity")] + PaidRetention, + #[error("invalid provider settlement key")] + SettlementKey, + #[error(transparent)] + OpenAiKey(#[from] hellas_providers::EmptyOpenAiKey), + #[error(transparent)] + RouteBinding(#[from] hellas_executor::FetchRouteBindingError), + #[error(transparent)] + DuplicateRoute(#[from] hellas_executor::DuplicateFetchRoute), + #[error(transparent)] + Executor(#[from] hellas_executor::ExecutorError), + #[error(transparent)] + Bind(#[from] iroh::endpoint::BindError), + #[error(transparent)] + Address(#[from] iroh::endpoint::InvalidSocketAddr), + #[error(transparent)] + Io(#[from] std::io::Error), + #[cfg(feature = "paid-provider")] + #[error(transparent)] + Config(#[from] crate::work_config::WorkConfigError), + #[cfg(feature = "paid-provider")] + #[error(transparent)] + Paid(#[from] crate::paid_provider::PaidProviderError), +} + pub struct OpenAiProviderOptions { pub port: Option, pub identity: ClientIdentity, @@ -56,17 +91,17 @@ pub struct FetchProviderOptions { pub fetch_queue_capacity: usize, pub retained_transcript_capacity: usize, pub fetch_replay_max_in_flight: usize, - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] pub paid_work: Option, } -#[cfg(feature = "paid-work")] +#[cfg(feature = "paid-provider")] struct WorkWatcher { stop: Option>, task: tokio::task::JoinHandle<()>, } -#[cfg(feature = "paid-work")] +#[cfg(feature = "paid-provider")] impl Drop for WorkWatcher { fn drop(&mut self) { self.stop.take(); @@ -76,7 +111,7 @@ impl Drop for WorkWatcher { pub struct ProviderHandle { endpoint: Endpoint, accept_task: tokio::task::JoinHandle<()>, - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] work: Option, } @@ -92,7 +127,7 @@ impl ProviderHandle { pub async fn shutdown(mut self) { self.accept_task.abort(); let _ = (&mut self.accept_task).await; - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] if let Some(mut work) = self.work.take() { if let Some(stop) = work.stop.take() { let _ = stop.send(()); @@ -106,7 +141,7 @@ impl ProviderHandle { impl Drop for ProviderHandle { fn drop(&mut self) { self.accept_task.abort(); - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] if let Some(work) = &mut self.work { work.stop.take(); } @@ -115,7 +150,7 @@ impl Drop for ProviderHandle { pub async fn start_openai_provider( options: OpenAiProviderOptions, -) -> anyhow::Result +) -> Result where R: RootProver + Send + Sync + 'static, { @@ -142,7 +177,7 @@ where fetch_queue_capacity: options.fetch_queue_capacity, retained_transcript_capacity: options.retained_transcript_capacity, fetch_replay_max_in_flight: options.fetch_replay_max_in_flight, - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] paid_work: None, }) .await @@ -150,19 +185,18 @@ where pub async fn start_fetch_provider( options: FetchProviderOptions, -) -> anyhow::Result +) -> Result where R: RootProver + Send + Sync + 'static, { - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] let has_paid_work = options.paid_work.is_some(); - #[cfg(not(feature = "paid-work"))] + #[cfg(not(feature = "paid-provider"))] let has_paid_work = false; - anyhow::ensure!( - !options.allowed_callers.is_empty() || has_paid_work, - "provider requires allowed callers or paid-work configuration" - ); - #[cfg(feature = "paid-work")] + if options.allowed_callers.is_empty() && !has_paid_work { + return Err(ProviderError::MissingAccessPolicy); + } + #[cfg(feature = "paid-provider")] if let Some(config) = &options.paid_work { use hellas_rpc::protocol::{ work_fetch::FetchRoutePolicy as PaidRoute, work_profile::PaidWorkPolicy, @@ -172,31 +206,29 @@ where policy, route: PaidRoute::SealedRoute { service, method }, } => { - anyhow::ensure!( - options - .routes - .entry(&FetchRoute::new(service, method)) - .is_some_and( - |entry| entry.execution_environment() == policy.allowed_environment - ), - "paid channel names an unavailable Fetch route or manifest" - ); + if !options + .routes + .entry(&FetchRoute::new(service, method)) + .is_some_and(|entry| { + entry.execution_environment() == policy.allowed_environment + }) + { + return Err(ProviderError::MissingPaidRoute); + } } PaidWorkPolicy::Fetch { policy, route: PaidRoute::OpenFetch { .. }, } => { - anyhow::ensure!( - options.routes.has_environment(policy.allowed_environment), - "paid channel names an unavailable HTTPS manifest" - ); + if !options.routes.has_environment(policy.allowed_environment) { + return Err(ProviderError::MissingPaidRoute); + } } - _ => anyhow::bail!("Fetch provider requires a paid Fetch policy"), + _ => return Err(ProviderError::WrongPaidPolicy), + } + if options.retained_transcript_capacity != 0 { + return Err(ProviderError::PaidRetention); } - anyhow::ensure!( - options.retained_transcript_capacity == 0, - "paid Fetch provider requires zero retained transcript capacity" - ); crate::work_config::validate_work_routes(config)?; } let producer_key = Arc::new(options.identity.caller_key().clone()); @@ -219,17 +251,17 @@ where ); let executor = Executor::spawn_configured(executor_config).await?; - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] let work_mount = crate::paid_provider::MountedWork::with_backend(executor.clone()); - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] let setup_mount = crate::paid_provider::MountedSetup::default(); - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] let work = if let Some(config) = options.paid_work { let policy = config.provider_policy(); let settlement_key = hellas_kernel::Secp256k1Signer::from_secret_scalar( options.identity.caller_secret_bytes(), ) - .map_err(|_| anyhow::anyhow!("invalid provider settlement key"))?; + .map_err(|_| ProviderError::SettlementKey)?; let runner = crate::paid_provider::WorkRunner::discover( crate::paid_provider::WorkRunnerConfig { network: config.chain.network, @@ -265,7 +297,7 @@ where } else { vec![::ALPN.as_bytes().to_vec()] }; - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] let alpns = if has_paid_work { let mut alpns = alpns; alpns.extend([ @@ -282,7 +314,7 @@ where .secret_key(options.identity.transport_key()) .alpns(alpns); if let Some(port) = options.port { - builder = builder.bind_addr(format!("0.0.0.0:{port}").parse::()?)?; + builder = builder.bind_addr(std::net::SocketAddr::from(([0, 0, 0, 0], port)))?; } let endpoint = builder.bind().await?; let accept_endpoint = endpoint.clone(); @@ -314,7 +346,7 @@ where } }; let executor = executor.clone(); - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] let (work_mount, setup_mount) = (work_mount.clone(), setup_mount.clone()); let open = open.clone(); connections.spawn(async move { @@ -332,7 +364,7 @@ where }; let alpn = connection.alpn().to_vec(); let transport = Arc::new(IrohTransport::new(connection)); - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] if has_paid_work && alpn == hellas_rpc::services::work::Work::ALPN.as_bytes() { let context = transport.context(); if let Some(handler) = work_mount.handler(&context) { @@ -352,7 +384,7 @@ where } return; } - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] if has_paid_work && alpn == hellas_rpc::services::work_setup::WorkSetup::ALPN.as_bytes() { @@ -393,7 +425,7 @@ where Ok(ProviderHandle { endpoint, accept_task, - #[cfg(feature = "paid-work")] + #[cfg(feature = "paid-provider")] work, }) } @@ -427,7 +459,7 @@ where } } -#[cfg(all(test, feature = "paid-work"))] +#[cfg(all(test, feature = "paid-provider", feature = "paid-client"))] mod tests; struct ProviderOpen { diff --git a/crates/sdk/src/test_support.rs b/crates/sdk/src/test_support.rs index 23db1cd6..88245c00 100644 --- a/crates/sdk/src/test_support.rs +++ b/crates/sdk/src/test_support.rs @@ -1,4 +1,6 @@ //! A funded, metadata-only Fetch channel for SDK boundary tests. +// Client and provider feature suites use different parts of this fixture. +#![allow(dead_code)] use crate::work_config::{ChainCrossCheck, WorkConfig}; use hellas_kernel::{ BlockHeight, Decode as _, Edge, EdgeId, EdgeValues, Fees, Key, LeaseSlots, List, diff --git a/crates/sdk/src/work_config.rs b/crates/sdk/src/work_config.rs index 85768e35..d79880d0 100644 --- a/crates/sdk/src/work_config.rs +++ b/crates/sdk/src/work_config.rs @@ -5,7 +5,6 @@ //! `validate_work_routes` before serving to check the configuration against journals. use std::collections::{BTreeMap, BTreeSet}; -use std::fs; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -147,7 +146,11 @@ pub struct ChainCrossCheck { /// Loads configuration, checks chain identity and policy bounds, and normalizes /// validator URLs. Route-to-journal validation is deferred until serve startup. pub fn load_work_config(path: &Path) -> Result { - let bytes = fs::read(path).map_err(|source| WorkConfigError::Read { + let bytes = hellas_private::read_bounded_regular_file( + path, + hellas_work::work_store::journal::MAX_RECORD_BYTES, + ) + .map_err(|source| WorkConfigError::Read { path: path.to_path_buf(), source, })?; diff --git a/crates/sdk/src/work_provision.rs b/crates/sdk/src/work_provision.rs index a845f745..b0e8db7a 100644 --- a/crates/sdk/src/work_provision.rs +++ b/crates/sdk/src/work_provision.rs @@ -9,7 +9,9 @@ use std::collections::BTreeSet; use std::path::{Path, PathBuf}; -use anyhow::{Context as _, Result, bail}; +mod error; +pub use error::ProvisionError; +type Result = std::result::Result; use hellas_chain::client::VerifiedRemoteLightClient; use hellas_chain::domain::MAX_EDGE_LIFETIME_BLOCKS; use hellas_chain::{ConsensusInfo, ConsensusVerifier, WorkBlocks}; @@ -32,10 +34,10 @@ pub struct ProvisionOptions { /// The key this provider stakes and signs the bond with, read from /// the identity the operator already has and never made here. pub settlement_key: Secp256k1Signer, - /// The client this bond names as taker, hex-encoded. - pub client: String, - /// The coins this provider stakes, hex-encoded. - pub stake_coins: Vec, + /// The client this bond names as taker. + pub client: Key, + /// The coins this provider stakes. + pub stake_coins: Vec, /// Height the bond expires at, which is also the admission horizon of /// the channel it insures. pub bond_timeout: u64, @@ -43,8 +45,6 @@ pub struct ProvisionOptions { pub timeout_payout: u64, /// The largest job price this bond covers. pub max_job_price: u64, - /// Select bond preview in operator frontends. - pub print_bond_only: bool, } /// Compute the bond before the operator adds its bilateral route. @@ -88,10 +88,7 @@ impl BondCandidate { // whose staking party this key is not. let provider = options.settlement_key.party_key(); let bond_terms = WorkStakeBondTerms { - parties: Parties::new( - provider, - Key::from_bytes(fixed::<{ Key::LENGTH }>("--client", &options.client)?), - ), + parties: Parties::new(provider, options.client), timeout: BlockHeight::new(options.bond_timeout), timeout_outputs: List::take( [Payout::new(provider, options.timeout_payout); MAX_EDGE_OUTPUTS], @@ -145,15 +142,15 @@ impl Offer { /// journal replays it. fn journal(self, floor: SetupScan) -> Result { let timeout = self.candidate.bond_terms.timeout.get(); - anyhow::ensure!( - timeout > floor.height, - "bond timeout must be after finalized height {}", - floor.height - ); - anyhow::ensure!( - timeout - floor.height <= MAX_EDGE_LIFETIME_BLOCKS, - "bond timeout exceeds the chain maximum lifetime" - ); + if timeout <= floor.height { + return Err(ProvisionError::ExpiredBond { + timeout, + height: floor.height, + }); + } + if timeout - floor.height > MAX_EDGE_LIFETIME_BLOCKS { + return Err(ProvisionError::BondLifetime); + } let Self { candidate, admission, @@ -176,26 +173,20 @@ impl Offer { "this journal already holds its history floor, and a floor does not move", ); } else { - endpoint - .arm_scan(floor) - .context("failed to make this setup's immutable history floor durable")?; + endpoint.arm_scan(floor)?; } - endpoint - .propose_bond(network, bond_funding, bond_terms) - .context("failed to sign and journal the bond proposal")?; + endpoint.propose_bond(network, bond_funding, bond_terms)?; } // Reopen with the same replay and signature checks used at startup. let reopened = open_provider_journal(&journal_root, network, bond_edge)?; let state = reopened.state(); let (Some(1), Some(floor)) = (state.revision(), state.scan_armed()) else { - bail!( - "the journal under {} replays as revision {:?} over floor {:?}, not the armed \ - proposal that was just written", - journal_root.display(), - state.revision(), - state.scan_armed().map(|scan| scan.height), - ); + return Err(ProvisionError::JournalReplay { + root: journal_root, + revision: state.revision(), + floor: state.scan_armed().map(|scan| scan.height), + }); }; Ok(Provisioned { bond_edge, floor }) } @@ -209,12 +200,9 @@ fn open_provider_journal(root: &Path, network: NetworkId, bond_edge: EdgeId) -> Role::Provider, &Secp256k1Verifier::new(), ) - .with_context(|| { - format!( - "failed to open the provider setup journal for bond {} under {}", - hex::encode(bond_edge.to_bytes()), - root.display(), - ) + .map_err(|source| ProvisionError::Journal { + root: root.into(), + source, }) } @@ -224,23 +212,12 @@ fn route_for_candidate<'config>( bond_edge: EdgeId, bond_terms: &WorkStakeBondTerms, ) -> Result<&'config WorkRoute> { - let Some(route) = config.routes.iter().find(|route| route.bond == bond_edge) else { - bail!( - "bond {} has no bilateral route in this work configuration; an offer is signed only \ - after its peer, bond, and client are named together", - hex::encode(bond_edge.to_bytes()), - ); - }; - let client = bond_terms.parties.taker(); - if route.client != client { - bail!( - "route for peer {:#} expects client {}, but candidate bond {} names {} as its taker", - route.peer, - hex::encode(route.client.to_bytes()), - hex::encode(bond_edge.to_bytes()), - hex::encode(client.to_bytes()), - ); - } + let route = config + .routes + .iter() + .find(|route| route.bond == bond_edge) + .ok_or(ProvisionError::MissingRoute(bond_edge))?; + check_client(route, bond_edge, bond_terms.parties.taker())?; Ok(route) } @@ -253,25 +230,15 @@ fn refuse_offer_collisions( ) -> Result<()> { let root = &config.journal_root; let network = config.chain.network; - let found = discover_setups(root, network).with_context(|| { - format!( - "failed to enumerate the work journals under {}", - root.display(), - ) + let found = discover_setups(root, network).map_err(|source| ProvisionError::Journal { + root: root.clone(), + source, })?; - for unnamed in &found.unidentified { - warn!( - path = %unnamed.path.display(), - reason = %unnamed.reason, - "a setup journal under the work root could not be named", - ); - } - if let Some(unnamed) = found.unidentified.first() { - bail!( - "setup journal {} cannot be identified, so a new offer cannot be proved disjoint: {}", - unnamed.path.display(), - unnamed.reason, - ); + if let Some(unnamed) = found.unidentified.into_iter().next() { + return Err(ProvisionError::Unidentified { + path: unnamed.path, + source: unnamed.reason, + }); } let candidate_coins = funding_coins(candidate_funding); @@ -281,62 +248,48 @@ fn refuse_offer_collisions( .filter(|setup| setup.role == Role::Provider) { if held.bond_edge == candidate.bond { - bail!( - "candidate bond {} collides with a provider offer already under {}", - hex::encode(candidate.bond.to_bytes()), - root.display(), - ); + return Err(ProvisionError::BondCollision(candidate.bond)); } - let Some(route) = config + let route = config .routes .iter() .find(|route| route.bond == held.bond_edge) - else { - bail!( - "provider offer over bond {} under {} has no configured route, so the candidate \ - route cannot be proved disjoint", - hex::encode(held.bond_edge.to_bytes()), - root.display(), - ); - }; + .ok_or(ProvisionError::MissingRoute(held.bond_edge))?; let store = open_provider_journal(root, network, held.bond_edge)?; - let Some(bundle) = store.state().bundle() else { - bail!( - "provider offer over bond {} was discovered without a retained revision", - hex::encode(held.bond_edge.to_bytes()), - ); - }; - let held_client = bundle.bond_terms().parties.taker(); - if route.client != held_client { - bail!( - "route for peer {:#} expects client {}, but provider offer over bond {} names {} \ - as its taker", - route.peer, - hex::encode(route.client.to_bytes()), - hex::encode(held.bond_edge.to_bytes()), - hex::encode(held_client.to_bytes()), - ); - } + let bundle = store + .state() + .bundle() + .ok_or(ProvisionError::MissingProposal(held.bond_edge))?; + check_client(route, held.bond_edge, bundle.bond_terms().parties.taker())?; if route.peer == candidate.peer { - bail!( - "candidate route peer {:#} collides with the provider offer over bond {}", - candidate.peer, - hex::encode(held.bond_edge.to_bytes()), - ); + return Err(ProvisionError::PeerCollision { + peer: candidate.peer, + bond: held.bond_edge, + }); } - // Revision-one funding is already reserved, even before an executable Open exists. + // Revision-one funding is reserved even before an executable Open exists. let reserved = funding_coins(bundle.bond_funding()); if let Some(coin) = candidate_coins.intersection(&reserved).next() { - bail!( - "candidate stake coin {} is already reserved by provider offer over bond {}", - hex::encode(coin.to_bytes()), - hex::encode(held.bond_edge.to_bytes()), - ); + return Err(ProvisionError::ReservedCoin { + coin: *coin, + bond: held.bond_edge, + }); } } Ok(()) } +fn check_client(route: &WorkRoute, bond: EdgeId, actual: Key) -> Result<()> { + if route.client != actual { + return Err(ProvisionError::WrongClient { + bond, + expected: route.client, + actual, + }); + } + Ok(()) +} + /// Every input one bond funding consumes, irrespective of party position. fn funding_coins(funding: &Funding) -> BTreeSet { funding @@ -354,8 +307,7 @@ async fn finalized_floor(config: &WorkConfig) -> Result { validators: config.validators.clone(), threshold_identity: config.chain.threshold_identity.clone(), network_id: config.chain.network.as_str().to_owned(), - }) - .context("the configured threshold identity is not usable")?; + })?; for url in &config.validators { let client = match VerifiedRemoteLightClient::connect(url.clone(), verifier.clone()).await { Ok(client) => client, @@ -373,7 +325,7 @@ async fn finalized_floor(config: &WorkConfig) -> Result { Err(error) => warn!(validator = %url, %error, "a configured validator did not answer"), } } - bail!("no configured validator answered with a finalized block to floor this offer at") + Err(ProvisionError::NoFinalizedBlock) } /// Returns the finalized tip and its payload, or `None` before the first block. @@ -395,34 +347,22 @@ where } /// Reads the coins one provider stakes. -fn staked(ids: &[String]) -> Result> { - let mut slots = [CoinId::from_bytes([0; CoinId::LENGTH]); MAX_PARTY_INPUTS]; - for (slot, id) in slots.iter_mut().zip(ids) { - *slot = CoinId::from_bytes(fixed::<{ CoinId::LENGTH }>("--stake-coin", id)?); +fn staked(ids: &[CoinId]) -> Result> { + if ids.is_empty() || ids.len() > MAX_PARTY_INPUTS { + return Err(ProvisionError::StakeCount(ids.len())); } - // The zip above stops at the shorter side, so a list the array cannot - // hold is refused here rather than silently staking the first four of - // it. - List::new(slots, ids.len()).with_context(|| { - format!( - "--stake-coin names {} coins, and one party funds an open with at most \ - {MAX_PARTY_INPUTS}", - ids.len(), - ) - }) -} - -/// Reads exactly `N` bytes of hex, or says which flag was not that. -fn fixed(flag: &str, value: &str) -> Result<[u8; N]> { - let bytes = - hex::decode(value).with_context(|| format!("{flag} {value:?} is not hex-encoded bytes"))?; - let Ok(fixed) = <[u8; N]>::try_from(bytes.as_slice()) else { - bail!( - "{flag} {value:?} is {} bytes, and {N} are wanted", - bytes.len() - ); - }; - Ok(fixed) + let mut seen = BTreeSet::new(); + for coin in ids { + if !seen.insert(*coin) { + return Err(ProvisionError::DuplicateCoin(*coin)); + } + } + let slots = std::array::from_fn(|i| { + ids.get(i) + .copied() + .unwrap_or(CoinId::from_bytes([0; CoinId::LENGTH])) + }); + Ok(List::take(slots, ids.len())) } #[cfg(test)] diff --git a/crates/sdk/src/work_provision/error.rs b/crates/sdk/src/work_provision/error.rs new file mode 100644 index 00000000..6206c866 --- /dev/null +++ b/crates/sdk/src/work_provision/error.rs @@ -0,0 +1,55 @@ +use hellas_kernel::{CoinId, EdgeId, Key}; +use hellas_rpc::peers::PeerId; +use std::path::PathBuf; + +#[derive(Debug, thiserror::Error)] +pub enum ProvisionError { + #[error("bond timeout {timeout} must be after finalized height {height}")] + ExpiredBond { timeout: u64, height: u64 }, + #[error("bond timeout exceeds the chain maximum lifetime")] + BondLifetime, + #[error("stake requires between 1 and at most 4 coins, found {0}")] + StakeCount(usize), + #[error("duplicate stake coin {}", hex::encode(.0.to_bytes()))] + DuplicateCoin(CoinId), + #[error("bond {} has no bilateral route in this work configuration", hex::encode(.0.to_bytes()))] + MissingRoute(EdgeId), + #[error("provider offer over bond {} has no retained revision", hex::encode(.0.to_bytes()))] + MissingProposal(EdgeId), + #[error("candidate bond {} collides with an existing provider offer", hex::encode(.0.to_bytes()))] + BondCollision(EdgeId), + #[error("candidate peer {peer:#} collides with bond {}", hex::encode(bond.to_bytes()))] + PeerCollision { peer: PeerId, bond: EdgeId }, + #[error("candidate stake coin {} is reserved by bond {}", hex::encode(coin.to_bytes()), hex::encode(bond.to_bytes()))] + ReservedCoin { coin: CoinId, bond: EdgeId }, + #[error("route for bond {} expects client {}, but the bond names {}", hex::encode(bond.to_bytes()), hex::encode(expected.to_bytes()), hex::encode(actual.to_bytes()))] + WrongClient { + bond: EdgeId, + expected: Key, + actual: Key, + }, + #[error("journal under {} replays as revision {revision:?} over floor {floor:?}, not the armed proposal", root.display())] + JournalReplay { + root: PathBuf, + revision: Option, + floor: Option, + }, + #[error("work journal under {}: {source}", root.display())] + Journal { + root: PathBuf, + source: hellas_work::work_store::WorkStoreError, + }, + #[error("setup journal {} cannot be identified: {source}", path.display())] + Unidentified { + path: PathBuf, + source: hellas_work::work_store::SetupDiscoveryError, + }, + #[error("no configured validator answered with a finalized block to floor this offer at")] + NoFinalizedBlock, + #[error(transparent)] + Setup(#[from] hellas_work::work_handshake::SetupExchangeError), + #[error(transparent)] + Consensus(#[from] hellas_chain::ConsensusVerificationError), + #[error(transparent)] + BlockSource(#[from] hellas_work::work_close::BlockSourceError), +} diff --git a/crates/sdk/src/work_provision/tests.rs b/crates/sdk/src/work_provision/tests.rs index 5556bade..8c6fa8b3 100644 --- a/crates/sdk/src/work_provision/tests.rs +++ b/crates/sdk/src/work_provision/tests.rs @@ -91,7 +91,10 @@ fn route(peer: u8, bond: EdgeId, client: Key) -> serde_json::Value { /// Loads routes through the production parser, so their duplicate-peer /// and duplicate-bond invariants are facts these provisioning tests use, /// not a test-only constructor that can make impossible route tables. -fn routed_work_config(root: &Path, routes: Vec) -> Result { +fn routed_work_config( + root: &Path, + routes: Vec, +) -> std::result::Result> { let validators: Vec = (1..=6) .map(|index| format!("http://127.0.0.1:900{index}")) .collect(); @@ -135,8 +138,7 @@ fn routed_work_config(root: &Path, routes: Vec) -> Result io::Result { - open_regular_file_impl(path) -} - -fn not_a_regular_file(path: &Path) -> io::Error { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("{} is not a regular file", path.display()), - ) -} - -#[cfg(target_os = "linux")] -fn open_regular_file_impl(path: &Path) -> io::Result { - let path_handle = open_path_handle(path)?; - reopen_regular_path_handle(path, &path_handle) -} - -/// Acquires an inode reference without invoking the target's file operations. -#[cfg(target_os = "linux")] -fn open_path_handle(path: &Path) -> io::Result { - use std::ffi::CString; - use std::os::fd::FromRawFd as _; - use std::os::unix::ffi::OsStrExt as _; - - let path = CString::new(path.as_os_str().as_bytes()) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; - // OpenOptions masks custom flags with !O_ACCMODE. On musl that mask - // includes O_PATH, turning a descriptor-only open into a blocking read. - loop { - // SAFETY: path is NUL-terminated and live for the call. These flags - // do not create a file, so open needs no variadic mode argument. - let fd = unsafe { libc::open(path.as_ptr(), libc::O_PATH | libc::O_CLOEXEC) }; - if fd >= 0 { - // SAFETY: open returned a new descriptor, owned only here. - return Ok(unsafe { std::fs::File::from_raw_fd(fd) }); - } - let error = io::Error::last_os_error(); - if error.kind() != io::ErrorKind::Interrupted { - return Err(error); - } - } -} - -/// Converts an `O_PATH` reference to a readable descriptor for the same inode. -#[cfg(target_os = "linux")] -fn reopen_regular_path_handle( - path: &Path, - path_handle: &std::fs::File, -) -> io::Result { - use std::os::fd::AsRawFd as _; - use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; - - let expected = path_handle.metadata()?; - if !expected.file_type().is_file() { - return Err(not_a_regular_file(path)); - } - - let descriptor_path = Path::new("/proc/self/fd").join(path_handle.as_raw_fd().to_string()); - let mut options = OpenOptions::new(); - options.read(true).custom_flags(libc::O_CLOEXEC); - let file = options.open(&descriptor_path).map_err(|source| { - // Once the held descriptor has been fstat-ed successfully, ENOENT can - // only mean procfs cannot provide the safe reopen. Do not report the - // caller's existing content as a cache miss. - let kind = if source.kind() == io::ErrorKind::NotFound { - io::ErrorKind::Unsupported - } else { - source.kind() - }; - io::Error::new( - kind, - format!( - "cannot safely reopen {} through {}: {source}", - path.display(), - descriptor_path.display() - ), - ) - })?; - let actual = file.metadata()?; - if !actual.file_type().is_file() - || (expected.dev(), expected.ino()) != (actual.dev(), actual.ino()) - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "{} did not reopen the inode held by its path descriptor", - path.display() - ), - )); - } - Ok(file) -} - -/// Best available fallback where `O_PATH` plus descriptor reopen is absent. -#[cfg(all(unix, not(target_os = "linux")))] -fn open_regular_file_impl(path: &Path) -> io::Result { - use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; - - let expected = std::fs::metadata(path)?; - if !expected.file_type().is_file() { - return Err(not_a_regular_file(path)); - } - - let mut options = OpenOptions::new(); - options - .read(true) - .custom_flags(libc::O_NONBLOCK | libc::O_CLOEXEC); - let file = options.open(path)?; - let actual = file.metadata()?; - if !actual.file_type().is_file() - || (expected.dev(), expected.ino()) != (actual.dev(), actual.ino()) - { - return Err(not_a_regular_file(path)); - } - Ok(file) -} - -/// Best available fallback where Unix descriptors are absent. Windows adds a -/// read-only share mode, so no writer can open the file while it is being -/// hashed -- the guarantee the Unix paths approximate with identity checks -/// (fastresume's Windows identity is deliberately weak; see there). -#[cfg(not(unix))] -fn open_regular_file_impl(path: &Path) -> io::Result { - let expected = std::fs::metadata(path)?; - if !expected.file_type().is_file() { - return Err(not_a_regular_file(path)); - } - let mut options = OpenOptions::new(); - options.read(true); - #[cfg(windows)] - { - use std::os::windows::fs::OpenOptionsExt as _; - // FILE_SHARE_READ: other readers may share the file; writers and - // deleters are refused until it is closed. - options.share_mode(0x0000_0001); - } - let file = options.open(path)?; - if !file.metadata()?.file_type().is_file() { - return Err(not_a_regular_file(path)); - } - Ok(file) -} +pub use hellas_private::open_regular_file; /// What indexing one file learned about it. #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/crates/store/src/tests.rs b/crates/store/src/tests.rs index ea9517f6..a54d7298 100644 --- a/crates/store/src/tests.rs +++ b/crates/store/src/tests.rs @@ -1,4 +1,5 @@ use super::*; +use std::fs::OpenOptions; fn scratch(name: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!( @@ -16,93 +17,6 @@ fn opened(path: &Path) -> (std::fs::File, fastresume::FileIdentity) { (file, identity) } -/// The Linux primitive is the contract, not merely an implementation -/// detail: a harmless path descriptor is acquired before type inspection, -/// and the readable descriptor remains bound to that inode even if the -/// name is replaced with a device. -#[cfg(target_os = "linux")] -#[test] -fn regular_open_is_safe_readable_seekable_and_close_on_exec() { - use std::io::{Read as _, Seek as _, SeekFrom}; - use std::os::fd::AsRawFd as _; - - let dir = scratch("regular-open"); - let target = dir.join("blob"); - let content = b"ordinary bytes"; - std::fs::write(&target, content).expect("write target"); - let link = dir.join("snapshot"); - std::os::unix::fs::symlink(&target, &link).expect("symlink"); - - let mut file = open_regular_file(&link).expect("open regular symlink"); - let mut read = Vec::new(); - file.read_to_end(&mut read).expect("read"); - assert_eq!(read, content); - file.seek(SeekFrom::Start(0)).expect("seek"); - read.clear(); - file.read_to_end(&mut read).expect("read again"); - assert_eq!(read, content); - // SAFETY: F_GETFD only observes the live descriptor owned by `file`. - let descriptor_flags = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFD) }; - assert!( - descriptor_flags >= 0, - "F_GETFD: {}", - io::Error::last_os_error() - ); - assert_ne!(descriptor_flags & libc::FD_CLOEXEC, 0); - - // Hold the ordinary inode without opening it for I/O, replace its - // public name with a device, then finish the reopen. The bytes must - // still come from the held inode. - let path_handle = open_path_handle(&link).expect("path handle"); - // O_PATH must survive libc-specific access-mode flags (notably musl). - // SAFETY: F_GETFL only observes the live descriptor owned by path_handle. - let path_flags = unsafe { libc::fcntl(path_handle.as_raw_fd(), libc::F_GETFL) }; - assert!(path_flags >= 0, "F_GETFL: {}", io::Error::last_os_error()); - assert_ne!(path_flags & libc::O_PATH, 0); - let replacement = dir.join("replacement"); - std::os::unix::fs::symlink("/dev/null", &replacement).expect("device symlink"); - std::fs::rename(&replacement, &link).expect("replace link"); - let mut held = reopen_regular_path_handle(&link, &path_handle).expect("reopen held inode"); - read.clear(); - held.read_to_end(&mut read).expect("read held inode"); - assert_eq!(read, content); - - let fifo = dir.join("fifo"); - let status = std::process::Command::new("mkfifo") - .arg(&fifo) - .status() - .expect("mkfifo"); - assert!(status.success(), "the fixture needs a fifo"); - let (sender, receiver) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let _ = sender.send(open_regular_file(&fifo)); - }); - let fifo_error = receiver - .recv_timeout(std::time::Duration::from_secs(2)) - .expect("opening the FIFO blocked before its type check") - .expect_err("a FIFO is not regular content"); - assert_eq!(fifo_error.kind(), io::ErrorKind::InvalidInput); - - if Path::new("/dev/null").exists() { - let device_error = open_regular_file(Path::new("/dev/null")) - .expect_err("a character device is not regular content"); - assert_eq!(device_error.kind(), io::ErrorKind::InvalidInput); - } - - use std::os::unix::ffi::{OsStrExt as _, OsStringExt as _}; - let mut nul_path = target.as_os_str().as_bytes().to_vec(); - nul_path.extend_from_slice(b"\0ignored-suffix"); - let nul_path = PathBuf::from(std::ffi::OsString::from_vec(nul_path)); - assert_eq!( - open_regular_file(&nul_path) - .expect_err("NUL must not truncate the path") - .kind(), - io::ErrorKind::InvalidInput, - ); - - let _ = std::fs::remove_dir_all(&dir); -} - /// Hashing is bounded by the descriptor length captured before the read. /// Mutations are injected between real file reads, so growth is found by /// the one-byte sentinel and truncation by an early EOF. diff --git a/docs/paid-gateway.md b/docs/paid-gateway.md index 5f0b6cf6..d4efa113 100644 --- a/docs/paid-gateway.md +++ b/docs/paid-gateway.md @@ -215,3 +215,31 @@ prefill chunks when its memory budget permits. The gateway conservatively clears its affinity hint after short requests; the provider decides actual reuse. Worker spans report that observed reuse as `gen_ai.usage.cache_read.input_tokens` without recording token values. + +## Embedding + +The CLI and native hosts use `hellas_sdk::paid_gateway::PaidGateway`. Enable +`paid-gateway`, load the same pool file with `load_pool_options(path, assurance)`, +and pass the resulting typed `PaidGatewayOptions` and a `ClientIdentity` to +`PaidGateway::open`. Applications can also construct the options directly from +`PaidWorkOptions`; the pool validates duplicate providers, journals and funding +before opening any connections. It owns recovery, admission, per-channel +serialization and payment tasks. No CLI argument structures are part of its API. + +For transparent HTTP, pass this pool to `hellas_gateway::start_http` using +`HttpGatewayOptions`. Its `paid` field is required. The options contain HTTP +routes, archive policy, listener authentication and optional process wrapping; +they have no model, tokenizer, inference cache or Evaluate settings. Shutting +down the returned handle drains accepted paid work. Failed startup also drains +recovery tasks. A host using the pool without a gateway must call its +`PaidExecutionBackend::drain` method when shutting down. + +`paid-client` enables sessions without the executor or upstream-provider +crates. `paid-provider` enables the provider runner and provisioning. Both +include `work`, the shared typed configuration and provisioning API. Apple +verification additionally needs `apple-verifier`; the file loader stores +assertion counters under each provider's journal root in `apple-counters`. +Counter files alone do not trigger channel funding during startup recovery. +Work and WorkSetup each reuse an authenticated QUIC connection. Reconnection +repeats Open with that connection's exporter and the channel's producer pin; +no new consensus query is added to the per-request path. diff --git a/nix/ci.nix b/nix/ci.nix index 313a2316..64242f38 100644 --- a/nix/ci.nix +++ b/nix/ci.nix @@ -129,11 +129,16 @@ let mkCargo "check-client-work" "cargo test -p hellas-client --features work && cargo clippy -p hellas-client --features work --all-targets -- -D warnings" (cargoEnv rustToolchain); + # HTTP boundaries and the bounded file primitives shared by hosts. + http-fetch = + mkCargo "check-http-fetch" + "cargo test -p hellas-gateway -p hellas-providers -p hellas-private -p hellas-store" + (cargoEnv rustToolchain); # Exercise the production provider loop and paid session recovery; their # feature is absent from the SDK's default tests. sdk-paid-work = mkCargo "check-sdk-paid-work" - "cargo test -p hellas-sdk --features paid-work && cargo clippy -p hellas-sdk --features paid-work --all-targets -- -D warnings" + "cargo test -p hellas-sdk --features paid-client,paid-provider,paid-gateway && cargo clippy -p hellas-sdk --features paid-client,paid-provider,paid-gateway --all-targets -- -D warnings && cargo clippy -p hellas-sdk --no-default-features --features paid-client --all-targets -- -D warnings && cargo clippy -p hellas-sdk --no-default-features --features paid-provider --all-targets -- -D warnings && cargo clippy -p hellas-sdk --no-default-features --features paid-gateway --all-targets -- -D warnings" (cargoEnv rustToolchain); # The chain service's wire-id pins compile only under `chain`, which # `work` does not pull in. `check-validator` links hellas-rpc with From d9407a2d92ebf8e3e42145f07df757a062a6fb96 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Mon, 28 Sep 2026 03:50:43 +0200 Subject: [PATCH 31/32] Sort provider dependencies --- crates/providers/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/providers/Cargo.toml b/crates/providers/Cargo.toml index ebac1b5d..3146b9a7 100644 --- a/crates/providers/Cargo.toml +++ b/crates/providers/Cargo.toml @@ -12,7 +12,6 @@ default = [] otel = ["hellas-rpc/otel", "dep:hellas-wire"] [dependencies] -thiserror.workspace = true anyhow = { workspace = true } async-stream = { workspace = true } base64.workspace = true @@ -31,6 +30,7 @@ rustls = { version = "0.23", default-features = false, features = [ serde.workspace = true serde_json.workspace = true sha2.workspace = true +thiserror.workspace = true tokio.workspace = true tracing.workspace = true url = "2" From 420f5e7ef920adedf184d0d624221f85bbdd66ce Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Mon, 28 Sep 2026 19:21:48 +0200 Subject: [PATCH 32/32] Check typed file errors and run CLI tests in local CI --- crates/cli/src/commands/environment/tests.rs | 17 +++++++++++++---- crates/cli/src/commands/fetch.rs | 7 ++++--- crates/cli/src/commands/llm.rs | 5 ++++- nix/ci.nix | 4 ++++ 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/crates/cli/src/commands/environment/tests.rs b/crates/cli/src/commands/environment/tests.rs index 1fdffdc4..3ad46c56 100644 --- a/crates/cli/src/commands/environment/tests.rs +++ b/crates/cli/src/commands/environment/tests.rs @@ -86,7 +86,10 @@ fn build_rejects_oversized_settings_before_indexing_artifacts() { .expect_err("oversized settings must be refused before artifact indexing"); let message = error.to_string(); assert!(message.contains("environment settings"), "{message}"); - assert!(message.contains("byte limit"), "{message}"); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::InvalidData + ); assert!(!output.exists()); } @@ -103,7 +106,10 @@ fn inspect_rejects_oversized_environment_metadata() { let error = inspect(&environment).expect_err("oversized environment must be refused"); let message = error.to_string(); assert!(message.contains("environment"), "{message}"); - assert!(message.contains("byte limit"), "{message}"); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::InvalidData + ); } #[cfg(unix)] @@ -111,8 +117,11 @@ fn inspect_rejects_oversized_environment_metadata() { fn inspect_rejects_a_device_before_reading_from_it() { let error = inspect(Path::new("/dev/zero")).expect_err("environment metadata must be an ordinary file"); - assert!(error.to_string().contains("failed to open environment")); - assert!(format!("{error:#}").contains("not a regular file")); + assert!(error.to_string().contains("environment")); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::InvalidInput + ); } #[cfg(unix)] diff --git a/crates/cli/src/commands/fetch.rs b/crates/cli/src/commands/fetch.rs index dfa0dcb4..cebcdca3 100644 --- a/crates/cli/src/commands/fetch.rs +++ b/crates/cli/src/commands/fetch.rs @@ -147,9 +147,10 @@ mod tests { std::fs::write(&path, vec![b' '; MAX_FETCH_REQUEST_BODY_BYTES + 1]).unwrap(); let error = load_payload_file(&path).expect_err("oversized payload must be refused"); - assert!(error.to_string().contains(&format!( - "over the {MAX_FETCH_REQUEST_BODY_BYTES}-byte limit" - ))); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::InvalidData + ); } #[cfg(unix)] diff --git a/crates/cli/src/commands/llm.rs b/crates/cli/src/commands/llm.rs index b1a4cbe3..02e29235 100644 --- a/crates/cli/src/commands/llm.rs +++ b/crates/cli/src/commands/llm.rs @@ -411,7 +411,10 @@ mod tests { let error = load_environment(&path, None) .err() .expect("oversize is refused"); - assert!(error.to_string().contains("over the")); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::InvalidData + ); } #[cfg(unix)] diff --git a/nix/ci.nix b/nix/ci.nix index 64242f38..09ba37e3 100644 --- a/nix/ci.nix +++ b/nix/ci.nix @@ -96,6 +96,10 @@ let ++ lib.optionals isValidatorHost [ "validator" ] ) )) (cargoEnv rustToolchain); + # Run command tests locally as well as in the release package builds. + cli = mkCargo "check-cli" "cargo test -p hellas-cli --no-default-features --features node,gateway" ( + cargoEnv rustToolchain + ); # The kernel's whole suite, including `tests/itf.rs` — the Quint↔Rust # replay that the entire abstract-correspondence story rests on — and # the exact-error pins in `tests/channel/`. `--all-features` is load