diff --git a/Cargo.lock b/Cargo.lock index 21e6735..041c11c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -987,7 +987,7 @@ dependencies = [ [[package]] name = "printpapi-agent" -version = "3.3.0" +version = "3.4.0" dependencies = [ "chrono", "libc", @@ -1002,7 +1002,7 @@ dependencies = [ [[package]] name = "printpapi-server" -version = "3.3.0" +version = "3.4.0" dependencies = [ "base64", "bytes", diff --git a/Cargo.toml b/Cargo.toml index cac40c6..6404883 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = ["crates/server", "crates/agent"] exclude = ["crates/setup"] [workspace.package] -version = "3.3.0" +version = "3.4.0" edition = "2024" license-file = "LICENSE" publish = false diff --git a/crates/server/src/dispatch.rs b/crates/server/src/dispatch.rs index 20888d9..5224887 100644 --- a/crates/server/src/dispatch.rs +++ b/crates/server/src/dispatch.rs @@ -70,8 +70,10 @@ impl Drop for Slot<'_> { /// Outbound payload fetches in flight at once. Each one waits on a blocking thread, so the cap is /// what keeps a client pointing many jobs at a tarpit URL from taking every thread the server has -/// (main sizes the pool at twice this). ponytail: one global cap, not per org — a single tenant -/// can still use all of it, it just cannot stall /health, logins or agent claims any more. +/// (main sizes the pool at twice this). ponytail: this is the global cap; where it matters (open +/// signup) the per-org cap `MAX_FETCHES_PER_ORG` sits under it so one tenant cannot use all 256. +/// Without that cap a single tenant can still use all of it, it just cannot stall /health, logins +/// or agent claims any more. pub const OUTBOUND_IN_FLIGHT: usize = 256; /// Where outbound requests (payload URLs, job callbacks, org event URLs) may connect. @@ -341,7 +343,7 @@ pub fn is_http(url: &str) -> bool { } // A non-string truthy url gets the scheme error with its repr (Python crashed in urlparse). -fn checked_http_url(url: Option<&Value>) -> Result<&str, DispatchError> { +pub fn checked_http_url(url: Option<&Value>) -> Result<&str, DispatchError> { match url { Some(v) if truthy(v) => match v { Value::String(s) if is_http(s) => Ok(s), diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index f9f4d98..b4a55b9 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -4,7 +4,8 @@ //! (default 3460), PRINTAPI_SIGNUP, PUBLIC_URL, PRINTAPI_PLANS (JSON array or a path to a file //! holding one), PRINTAPI_BILLING_SECRET, PRINTAPI_TRUSTED_PROXIES (IPs or CIDRs whose //! `X-Forwarded-For` is believed), PRINTAPI_SIGNUP_QUOTA (monthly job quota of every self-signup -//! org), PRINTAPI_JOB_RETENTION_DAYS (days until finished jobs lose payload, title and +//! org), PRINTAPI_FETCHES_PER_ORG (payload downloads one org may run at once, 0 = no cap; unset: +//! 4 with open signup, else no cap), PRINTAPI_JOB_RETENTION_DAYS (days until finished jobs lose payload, title and //! callback, and until queued jobs without a deadline are failed; their row goes after //! max(N, 7) days if it is from before this UTC month; unset or 0 keeps everything), PRINTAPI_TERMS_VERSION (set: `POST /signup` //! must accept that version of the terms), PRINTAPI_TERMS_URL (required with it), PRINTAPI_DPA_URL, @@ -38,6 +39,17 @@ fn parse_quota(raw: &str) -> Result, String> { } } +/// Empty: unset (the default follows the signup mode). Otherwise a non-negative integer, 0 = no cap. +fn parse_fetch_cap(raw: &str) -> Result, String> { + let t = py_strip(raw); + if t.is_empty() { + return Ok(None); + } + t.parse::() + .map(Some) + .map_err(|_| format!("must be a non-negative integer, not {t:?}")) +} + /// Empty or 0: keep for ever. Otherwise a positive number of days. fn parse_retention(raw: &str) -> Result, String> { let t = py_strip(raw); @@ -174,6 +186,8 @@ fn config() -> Result<(Config, String, u16), String> { )); } }; + cfg.fetches_per_org = parse_fetch_cap(&env("PRINTAPI_FETCHES_PER_ORG").unwrap_or_default()) + .map_err(|e| format!("PRINTAPI_FETCHES_PER_ORG: {e}"))?; cfg.trusted_proxies = printpapi_server::proxy::TrustedProxies::parse( &env("PRINTAPI_TRUSTED_PROXIES").unwrap_or_default(), ) diff --git a/crates/server/src/server.rs b/crates/server/src/server.rs index 281611b..f215689 100644 --- a/crates/server/src/server.rs +++ b/crates/server/src/server.rs @@ -54,6 +54,9 @@ pub struct Config { pub job_retention_days: Option, /// PRINTAPI_TERMS_VERSION and its links: set, `POST /signup` needs `accept_terms` = version. pub terms: Option, + /// PRINTAPI_FETCHES_PER_ORG: payload downloads one org may have in flight; 0 = no cap. None: + /// `MAX_FETCHES_PER_ORG` with open signup, else no cap (see `Config::fetch_cap`). + pub fetches_per_org: Option, } /// The terms a signup must accept (`PRINTAPI_TERMS_*`). The URLs are `http(s)` (checked at startup). @@ -89,6 +92,7 @@ impl Config { signup_quota: None, job_retention_days: None, terms: None, + fetches_per_org: None, } } } @@ -182,6 +186,10 @@ pub struct AppState { pub render_slots: RenderSlots, /// When the retention purge last ran (the reaper ticks far more often than once an hour). last_retention: Mutex>, + /// Payload fetches running now, per org (see `MAX_FETCHES_PER_ORG`). + org_fetches: Mutex>, + /// Idempotency keys whose payload is being fetched now, per org (see `KeyGuard`). + keys_in_flight: Mutex>, /// A retention WAL truncate was blocked by a reader; the next pass retries it even if it /// changes no rows. wal_truncate_pending: std::sync::atomic::AtomicBool, @@ -215,6 +223,11 @@ impl RenderSlots { } } + /// Whether a slot is free right now, without taking it. + pub fn has_free(&self) -> bool { + self.in_flight.load(std::sync::atomic::Ordering::SeqCst) < self.max + } + /// A slot, or `None` when `max` renders are already running. pub fn try_enter(&self) -> Option> { use std::sync::atomic::Ordering::SeqCst; @@ -226,6 +239,105 @@ impl RenderSlots { } } +/// Payload downloads one org may have in flight at once where the cap applies (open signup, or +/// PRINTAPI_FETCHES_PER_ORG): the hosted default. ponytail: this stops one org from holding every +/// slot of the global `OUTBOUND_IN_FLIGHT` cap; an attacker with many orgs still shares that +/// global 256, and the cap does not queue, it answers 503. +pub const MAX_FETCHES_PER_ORG: usize = 4; + +impl Config { + /// Downloads one org may have in flight; 0 = no cap. Unset: the hosted default with open + /// signup, none on a self-hosted box (its one org would stall on four hung upstream hosts). + pub fn fetch_cap(&self) -> usize { + self.fetches_per_org.unwrap_or(if self.signup_open { + MAX_FETCHES_PER_ORG + } else { + 0 + }) + } +} + +/// Holds one of an org's download slots; dropping it (success, error or panic unwind) gives it back. +struct FetchGuard<'a> { + state: &'a AppState, + org_id: i64, +} + +impl Drop for FetchGuard<'_> { + fn drop(&mut self) { + let mut map = self + .state + .org_fetches + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(n) = map.get_mut(&self.org_id) { + *n -= 1; + if *n == 0 { + map.remove(&self.org_id); + } + } + } +} + +impl AppState { + /// A download slot for `org_id`, or `None` when it already has `Config::fetch_cap` running. + /// Without a cap nothing is counted. + fn try_fetch_slot(&self, org_id: i64) -> Option>> { + let cap = self.cfg.fetch_cap(); + if cap == 0 { + return Some(None); + } + let mut map = self + .org_fetches + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let n = map.entry(org_id).or_insert(0); + if *n >= cap { + return None; + } + *n += 1; + Some(Some(FetchGuard { + state: self, + org_id, + })) + } + + /// Marks `(org_id, key)` as being fetched, or `None` when another request already does. + fn try_key_in_flight(&self, org_id: i64, key: &str) -> Option> { + let mut set = self + .keys_in_flight + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !set.insert((org_id, key.to_string())) { + return None; + } + Some(KeyGuard { + state: self, + org_id, + key: key.to_string(), + }) + } +} + +/// Holds an idempotency key as "being fetched"; dropping it (success, error or panic unwind) +/// releases it. Two requests with the same new key must not both fetch the payload (a `*_uri_post` +/// would hit the downstream service twice); `enqueue_job` only dedupes at the end. +struct KeyGuard<'a> { + state: &'a AppState, + org_id: i64, + key: String, +} + +impl Drop for KeyGuard<'_> { + fn drop(&mut self) { + self.state + .keys_in_flight + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&(self.org_id, std::mem::take(&mut self.key))); + } +} + /// Concurrent long-poll DB calls (see `AppState::poll_slots`). pub const POLL_SLOTS: usize = 4; @@ -250,6 +362,8 @@ impl AppState { poll_slots: Arc::new(tokio::sync::Semaphore::new(POLL_SLOTS)), render_slots: RenderSlots::new(RENDER_SLOTS), last_retention: Mutex::new(None), + org_fetches: Mutex::new(std::collections::HashMap::new()), + keys_in_flight: Mutex::new(std::collections::HashSet::new()), wal_truncate_pending: std::sync::atomic::AtomicBool::new(false), } } @@ -602,13 +716,44 @@ impl Principal { } } +/// What `gate_printer` decided: the job already exists, or go on (holding the download slot and +/// the in-flight key when the payload is fetched). +enum Gate2<'a> { + Existing(store::Enqueued), + Go(Held<'a>), +} + +/// What a fetching request holds until its job is queued (or it fails). +#[derive(Default)] +struct Held<'a> { + slot: Option>, + _key: Option>, +} + +/// What `label_payload` made: an existing job (idempotent resubmit), or the job's mode, validated +/// options and rendered bytes. +enum Labeled<'a> { + Existing(store::Enqueued), + Ready(&'static str, Option, Vec, Held<'a>), +} + +/// The job types whose payload the server fetches itself (`label_uri` has its own path). +fn fetches_payload(type_v: &Value) -> bool { + matches!( + type_v.as_str(), + Some("raw_uri" | "pdf_uri" | "raw_uri_post" | "pdf_uri_post") + ) +} + /// Why a job submit failed — the callers map these onto their own status codes and shapes. enum SubmitErr { Fetch(String), Quota(String), Invalid(String), - /// Too many renders running; the client may retry. + /// Too many renders or downloads running; the client may retry. Busy(String), + /// The same new idempotency key is being fetched by another request. + InProgress(String), Store(StoreError), } @@ -990,13 +1135,36 @@ impl Ctx<'_> { }; let idempotency_key = dispatch::parse_idempotency_key(body)?; let expire_after = dispatch::parse_expire_after(body)?; + // the key stays marked until the job is queued, so a second request cannot fetch meanwhile + let mut _held = Held::default(); let data = match &label { - None => dispatch::decode_payload(body, &*self.s.fetcher)?, - Some(l) => { - let (m, o, d) = self.label_payload(body, l, org)?; - (mode, options) = (m, o); - d + None => { + // The gate runs only for types that fetch; the rest keep their order of checks. + if fetches_payload(type_v) { + // a bad url still 400s before the printer is looked at + dispatch::checked_http_url(body.get("url"))?; + let printer_v = body.get("printer_id").unwrap_or(&null); + let printer = self + .db(|c| store::get_printer(c, printer_v, org)) + .map_err(SubmitErr::Store)? + .ok_or_else(|| { + SubmitErr::Invalid(format!("unknown printer: {}", py_str(printer_v))) + })?; + match self.gate_printer(&printer, idempotency_key.as_deref(), true)? { + Gate2::Existing(e) => return Ok(e), + Gate2::Go(held) => _held = held, + } + } + dispatch::decode_payload(body, &*self.s.fetcher)? } + Some(l) => match self.label_payload(body, l, org, idempotency_key.as_deref())? { + Labeled::Existing(e) => return Ok(e), + Labeled::Ready(m, o, d, h) => { + (mode, options) = (m, o); + _held = h; + d + } + }, }; let job = store::NewJob { printer_id: body.get("printer_id").unwrap_or(&null), @@ -1015,6 +1183,56 @@ impl Ctx<'_> { self.enqueue(&job) } + /// The checks before any outbound fetch, cheapest first (the printer is already resolved): + /// an existing job under the idempotency key (returned as it is, nothing fetched), the quota, + /// and the org's download cap when `fetch`. `enqueue_job` repeats the printer, key and quota + /// checks inside its transaction; this only stops the expensive work early. + fn gate_printer( + &self, + printer: &store::PrinterRef, + key: Option<&str>, + fetch: bool, + ) -> Result, SubmitErr> { + // Marked before the lookup: a request that finished in between has queued its job by now, + // so the lookup below finds it; one that is still fetching holds the key. + let key_guard = match key.filter(|_| fetch) { + Some(k) => Some(self.s.try_key_in_flight(printer.org_id, k).ok_or_else(|| { + SubmitErr::InProgress( + "a request with this idempotency_key is still in progress, retry shortly" + .into(), + ) + })?), + None => None, + }; + // One closure, so a retry cannot see "no job yet" and then a quota the original used up. + let checked = self.db(|c| { + if let Some(key) = key + && let Some(e) = store::find_job_by_key(c, printer.org_id, key)? + { + return Ok(Some(e)); + } + store::check_quota(c, printer.org_id, crate::now()).map(|()| None) + }); + match checked { + Ok(Some(e)) => return Ok(Gate2::Existing(e)), + Ok(None) => {} + Err(StoreError::QuotaExceeded(m)) => return Err(SubmitErr::Quota(m)), + Err(e) => return Err(SubmitErr::Store(e)), + } + if !fetch { + return Ok(Gate2::Go(Held::default())); + } + match self.s.try_fetch_slot(printer.org_id) { + Some(slot) => Ok(Gate2::Go(Held { + slot, + _key: key_guard, + })), + None => Err(SubmitErr::Busy( + "too many downloads in flight for this org, try again later".into(), + )), + } + } + /// A `label_*` job: the printer, then the template, then the payload, each before the next so /// that an unknown printer or template never causes a fetch. The printer decides the output: /// PDF where it takes PDF, ZPL otherwise (gotcha #1: never a PDF to a label printer). Returns @@ -1024,7 +1242,8 @@ impl Ctx<'_> { body: &Obj, job: &dispatch::LabelJob, org: Option, - ) -> Result<(&'static str, Option, Vec), SubmitErr> { + key: Option<&str>, + ) -> Result, SubmitErr> { let null = Value::Null; let printer_v = body.get("printer_id").unwrap_or(&null); let printer = self @@ -1068,20 +1287,27 @@ impl Ctx<'_> { ("raw", label::Output::Zpl) }; let options = dispatch::parse_options(body, mode)?; - // Fetching and rendering cost a thread and a render slot, so an org past its quota stops - // before them, not at enqueue_job. A keyed job may be a resubmit, which spends no quota: - // that one still goes through and enqueue_job sorts it out. - if body.get("idempotency_key").is_none() { - match self.db(|c| store::check_quota(c, printer.org_id, crate::now())) { - Ok(()) => {} - Err(StoreError::QuotaExceeded(m)) => return Err(SubmitErr::Quota(m)), - Err(e) => return Err(SubmitErr::Store(e)), - } - } + // Fetching and rendering cost a thread and a render slot, so an existing key, a spent + // quota and a full download cap stop before them, not at enqueue_job. + // only `label_uri` fetches; `label_base64` decodes what it was sent + let fetches = body.get("type").and_then(Value::as_str) == Some("label_uri"); + let mut held = match self.gate_printer(&printer, key, fetches)? { + Gate2::Existing(e) => return Ok(Labeled::Existing(e)), + Gate2::Go(held) => held, + }; let rendered = match &job.text { Some(text) => label::render_text(&spec, text, out), None => { + // ponytail: a check, not a reservation: a slot can still be taken between here and + // the render, which then answers the same 503 after the fetch. + if fetches && !self.s.render_slots.has_free() { + return Err(SubmitErr::Busy( + "label rendering is busy, try again later".into(), + )); + } let img = dispatch::decode_payload(body, &*self.s.fetcher)?; + // the download slot is free again; the key stays held until the job is queued + held.slot = None; let Some(_slot) = self.s.render_slots.try_enter() else { return Err(SubmitErr::Busy( "label rendering is busy, try again later".into(), @@ -1091,7 +1317,7 @@ impl Ctx<'_> { } }; let data = rendered.map_err(|e| SubmitErr::Invalid(format!("label: {e}")))?; - Ok((mode, options, data)) + Ok(Labeled::Ready(mode, options, data, held)) } fn enqueue(&self, job: &store::NewJob<'_>) -> Result { @@ -1146,6 +1372,7 @@ impl Ctx<'_> { Err(SubmitErr::Quota(m)) => Ok(jerr(402, m)), Err(SubmitErr::Invalid(m)) => Ok(jerr(400, m)), Err(SubmitErr::Busy(m)) => Ok(jerr(503, m)), + Err(SubmitErr::InProgress(m)) => Ok(jerr(409, m)), Err(SubmitErr::Store(e)) => Err(e), } } @@ -1218,9 +1445,12 @@ impl Ctx<'_> { match self.enqueue(&job) { Ok(e) => Ok(Resp::json(200, &json!({ "job_id": e.job_id }))), Err(SubmitErr::Quota(m)) => Ok(jerr(402, m)), - Err(SubmitErr::Invalid(m) | SubmitErr::Fetch(m) | SubmitErr::Busy(m)) => { - Ok(jerr(400, m)) - } + Err( + SubmitErr::Invalid(m) + | SubmitErr::Fetch(m) + | SubmitErr::Busy(m) + | SubmitErr::InProgress(m), + ) => Ok(jerr(400, m)), Err(SubmitErr::Store(e)) => Err(e), } } @@ -1524,6 +1754,8 @@ impl Ctx<'_> { Err(SubmitErr::Quota(m)) => self.pn_error(402, "QuotaExceeded", m), Err(SubmitErr::Invalid(m)) => self.pn_error(400, "BadRequest", m), Err(SubmitErr::Busy(m)) => self.pn_error(503, "ServiceUnavailable", m), + // ponytail: unreachable today, this layer takes no idempotency key + Err(SubmitErr::InProgress(m)) => self.pn_error(409, "Conflict", m), Err(SubmitErr::Store(e)) => return Err(e), })) } diff --git a/crates/server/src/store.rs b/crates/server/src/store.rs index 68fe26a..f24fe3b 100644 --- a/crates/server/src/store.rs +++ b/crates/server/src/store.rs @@ -1331,6 +1331,24 @@ pub fn check_quota(conn: &Connection, org_id: i64, now: f64) -> Result<()> { Ok(()) } +/// The job an org already queued under `key`, as `enqueue_job` returns it for a resubmit — lets a +/// caller skip fetching a payload it would only throw away. +pub fn find_job_by_key(conn: &Connection, org_id: i64, key: &str) -> Result> { + Ok(conn + .query_row( + "SELECT id, agent_id FROM jobs WHERE org_id=? AND idempotency_key=?", + params![org_id, key], + |r| { + Ok(Enqueued { + job_id: r.get(0)?, + agent_id: r.get(1)?, + created: false, + }) + }, + ) + .optional()?) +} + pub fn enqueue_job(conn: &Connection, job: &NewJob<'_>) -> Result { let now = crate::now(); let unknown = || { diff --git a/crates/server/tests/binary.rs b/crates/server/tests/binary.rs index 953dcd7..71f8bd2 100644 --- a/crates/server/tests/binary.rs +++ b/crates/server/tests/binary.rs @@ -15,6 +15,7 @@ fn bin() -> Command { "PRINTAPI_SIGNUP", "PRINTAPI_TRUSTED_PROXIES", "PRINTAPI_SIGNUP_QUOTA", + "PRINTAPI_FETCHES_PER_ORG", "PRINTAPI_JOB_RETENTION_DAYS", "PRINTAPI_TERMS_VERSION", "PRINTAPI_TERMS_URL", @@ -94,6 +95,19 @@ fn bad_signup_quota_exits_1() { } } +#[test] +fn bad_fetches_per_org_exits_1() { + for bad in ["-1", "ten", "1.5"] { + let out = bin() + .env("PRINTAPI_TOKEN", "t") + .env("PRINTAPI_FETCHES_PER_ORG", bad) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(1), "{bad}"); + assert!(String::from_utf8_lossy(&out.stderr).contains("PRINTAPI_FETCHES_PER_ORG")); + } +} + #[test] fn bad_job_retention_exits_1() { for bad in ["-1", "ten", "1.5"] { diff --git a/crates/server/tests/http_fetch_gate.rs b/crates/server/tests/http_fetch_gate.rs new file mode 100644 index 0000000..34deb4e --- /dev/null +++ b/crates/server/tests/http_fetch_gate.rs @@ -0,0 +1,448 @@ +// printpapi — self-hosted PrintNode alternative. Elastic License 2.0 (see LICENSE). +//! The gate before an outbound fetch: for `*_uri` jobs the printer, the idempotency key, the quota +//! and a per-org cap on concurrent downloads are checked before the server fetches the payload. +mod common; + +use base64::{Engine, engine::general_purpose::STANDARD as B64}; +use common::*; +use printpapi_server::dispatch::Fetcher; +use printpapi_server::server::Config; +use printpapi_server::store::DEFAULT_ORG; +use serde_json::{Value, json}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Duration; + +fn png() -> Vec { + let img = image::RgbaImage::from_pixel(40, 20, image::Rgba([0, 0, 0, 255])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() +} + +/// An app with a client key "k1" in the default org, whose printer "P" has the id returned. +fn app_with_printer() -> (App, i64) { + let app = App::new(); + add_key(&app.db, "one", "k1", DEFAULT_ORG); + let pid = register(&app.db, "pc", "ak", &[decl("P", true)], DEFAULT_ORG).pid("P"); + *app.fetch.reply.lock().unwrap() = Some(Ok(png())); + (app, pid) +} + +fn template(app: &App, name: &str, content: &str) { + let (code, v) = app.post( + "/labels", + Some("k1"), + json!({ + "name": name, "content": content, + "label_mm": {"w": 50, "h": 30}, "dpi": 203, "rotate": 0, + "crop_percent": {"t": 0, "r": 0, "b": 0, "l": 0}, + "place_mm": {"x": 0, "y": 0, "w": 50, "h": 30}, + }), + ); + assert_eq!(code, 200, "{v}"); +} + +fn set_quota(app: &App, org: i64, quota: i64) { + let (code, v) = app.put( + &format!("/orgs/{org}"), + Some("t"), + json!({"job_quota": quota}), + ); + assert_eq!(code, 200, "{v}"); +} + +fn fetched(app: &App) -> usize { + app.fetch.gets.lock().unwrap().len() +} + +fn err_of(v: &Value) -> &str { + v["error"].as_str().unwrap() +} + +/// One submit body per fetching job type, each with a fresh idempotency key. +fn fetching_bodies(pid: i64, n: usize) -> Vec { + let key = |t: &str| format!("{t}-{n}"); + vec![ + json!({"printer_id": pid, "type": "raw_uri", "url": "https://x/a", "idempotency_key": key("r")}), + json!({"printer_id": pid, "type": "pdf_uri", "url": "https://x/a", "idempotency_key": key("p")}), + json!({"printer_id": pid, "type": "raw_uri_post", "url": "https://x/a", "json": {}, "idempotency_key": key("rp")}), + json!({"printer_id": pid, "type": "pdf_uri_post", "url": "https://x/a", "json": {}, "idempotency_key": key("pp")}), + json!({"printer_id": pid, "type": "label_uri", "url": "https://x/a", "template": "img", "idempotency_key": key("l")}), + ] +} + +#[test] +fn a_spent_quota_answers_402_without_fetching() { + let (app, pid) = app_with_printer(); + template(&app, "img", "image"); + set_quota(&app, DEFAULT_ORG, 0); + for round in 0..2 { + for mut body in fetching_bodies(pid, round) { + // round 0 sends a fresh key per request, round 1 none at all + if round == 1 { + body.as_object_mut().unwrap().remove("idempotency_key"); + } + let (code, v) = app.post("/jobs", Some("k1"), body.clone()); + assert_eq!(code, 402, "{body} -> {v}"); + } + } + assert_eq!(fetched(&app), 0, "{:?}", app.fetch.gets.lock().unwrap()); +} + +#[test] +fn an_unknown_or_foreign_printer_answers_400_without_fetching() { + let (app, _pid) = app_with_printer(); + let other = create_org(&app.db, "other"); + let foreign = register(&app.db, "pc2", "ak2", &[decl("Q", true)], other).pid("Q"); + for printer in [json!(4242), json!(foreign), json!("nope")] { + let (code, v) = app.post( + "/jobs", + Some("k1"), + json!({"printer_id": printer, "type": "raw_uri", "url": "https://x/a"}), + ); + assert_eq!(code, 400, "{v}"); + assert!(err_of(&v).starts_with("unknown printer: "), "{v}"); + } + assert_eq!(fetched(&app), 0); +} + +#[test] +fn a_keyed_resubmit_returns_the_job_without_fetching_again() { + let (app, pid) = app_with_printer(); + template(&app, "img", "image"); + for body in [ + json!({"printer_id": pid, "type": "raw_uri", "url": "https://x/a", "idempotency_key": "k-raw"}), + json!({"printer_id": pid, "type": "label_uri", "url": "https://x/a", "template": "img", "idempotency_key": "k-label"}), + ] { + let before = fetched(&app); + let (c1, v1) = app.post("/jobs", Some("k1"), body.clone()); + let (c2, v2) = app.post("/jobs", Some("k1"), body.clone()); + assert_eq!((c1, c2), (200, 200), "{v1} {v2}"); + assert_eq!(v1["job_id"], v2["job_id"], "{body}"); + assert_eq!(fetched(&app) - before, 1, "{body}"); + } +} + +#[test] +fn a_keyed_label_text_resubmit_is_answered_before_the_quota() { + // the keyed shortcut covers every label job: rendering costs a slot + let (app, pid) = app_with_printer(); + template(&app, "txt", "text"); + let body = json!({"printer_id": pid, "type": "label_text", "text": "hi", "template": "txt", "idempotency_key": "k-t"}); + let (c1, v1) = app.post("/jobs", Some("k1"), body.clone()); + assert_eq!(c1, 200, "{v1}"); + // a spent quota cannot stop the resubmit of a job that already exists + set_quota(&app, DEFAULT_ORG, 1); + let (c2, v2) = app.post("/jobs", Some("k1"), body); + assert_eq!(c2, 200, "{v2}"); + assert_eq!(v1["job_id"], v2["job_id"]); +} + +/// "slow" URLs block until `release` (or two seconds), "bad" ones fail, the rest answer at once. +struct GatedFetcher { + entered: AtomicUsize, + open: Mutex, + cv: Condvar, +} + +impl GatedFetcher { + fn new() -> Arc { + Arc::new(GatedFetcher { + entered: AtomicUsize::new(0), + open: Mutex::new(false), + cv: Condvar::new(), + }) + } + + fn release(&self) { + *self.open.lock().unwrap() = true; + self.cv.notify_all(); + } + + fn wait_entered(&self, n: usize) { + for _ in 0..400 { + if self.entered.load(Ordering::SeqCst) >= n { + return; + } + std::thread::sleep(Duration::from_millis(10)); + } + panic!( + "only {} fetches started", + self.entered.load(Ordering::SeqCst) + ); + } +} + +impl Fetcher for GatedFetcher { + fn get(&self, url: &str) -> Result, String> { + if url.contains("bad") { + return Err("HTTP Error 404: Not Found".into()); + } + if url.contains("slow") { + self.entered.fetch_add(1, Ordering::SeqCst); + let g = self.open.lock().unwrap(); + let _ = self + .cv + .wait_timeout_while(g, Duration::from_secs(2), |open| !*open) + .unwrap(); + } + Ok(b"FETCHED".to_vec()) + } + + fn post_json(&self, url: &str, _body: &Value) -> Result, String> { + self.get(url) + } +} + +fn uri_job(pid: i64, url: &str) -> Value { + json!({"printer_id": pid, "type": "raw_uri", "url": url}) +} + +#[test] +fn one_org_cannot_hold_more_than_four_downloads_in_flight() { + let mut app = App::with(|c| c.fetches_per_org = Some(4)); + let fetcher = GatedFetcher::new(); + app.state.fetcher = fetcher.clone(); + add_key(&app.db, "a", "ka", DEFAULT_ORG); + let pa = register(&app.db, "pa", "aka", &[decl("P", false)], DEFAULT_ORG).pid("P"); + let org_b = create_org(&app.db, "b"); + add_key(&app.db, "b", "kb", org_b); + let pb = register(&app.db, "pb", "akb", &[decl("P", false)], org_b).pid("P"); + std::thread::scope(|s| { + let held: Vec<_> = (0..4) + .map(|i| { + let app = &app; + s.spawn(move || { + app.post( + "/jobs", + Some("ka"), + uri_job(pa, &format!("https://x/slow{i}")), + ) + }) + }) + .collect(); + fetcher.wait_entered(4); + // the fifth from the same org is turned away, one from another org still goes through + let (code, v) = app.post("/jobs", Some("ka"), uri_job(pa, "https://x/fast")); + assert_eq!(code, 503, "{v}"); + assert!(err_of(&v).contains("too many downloads"), "{v}"); + let (code, v) = app.post("/jobs", Some("kb"), uri_job(pb, "https://x/fast")); + assert_eq!(code, 200, "{v}"); + // the PrintNode layer answers the same + let r = app + .call("POST", "/printjobs") + .basic("ka") + .json(&json!({"printerId": pa, "contentType": "raw_uri", "content": "https://x/fast", "source": "t"})) + .send(); + assert_eq!(r.status, 503); + fetcher.release(); + for h in held { + assert_eq!(h.join().unwrap().0, 200); + } + }); + // every slot is free again, also after fetches that fail + for _ in 0..6 { + let (code, v) = app.post("/jobs", Some("ka"), uri_job(pa, "https://x/bad")); + assert_eq!(code, 502, "{v}"); + } + let (code, v) = app.post("/jobs", Some("ka"), uri_job(pa, "https://x/fast")); + assert_eq!(code, 200, "{v}"); +} + +#[test] +fn the_printnode_layer_checks_before_it_fetches() { + let (app, pid) = app_with_printer(); + set_quota(&app, DEFAULT_ORG, 0); + let r = app + .call("POST", "/printjobs") + .basic("k1") + .json(&json!({"printerId": pid, "contentType": "raw_uri", "content": "https://x/a", "source": "t"})) + .send(); + assert_eq!(r.status, 402); + assert_eq!(r.json()["code"], "QuotaExceeded"); + let r = app + .call("POST", "/printjobs") + .basic("k1") + .json(&json!({"printerId": 4242, "contentType": "raw_uri", "content": "https://x/a", "source": "t"})) + .send(); + assert_eq!(r.status, 400); + assert_eq!(fetched(&app), 0); +} + +#[test] +fn base64_jobs_keep_their_order_of_checks() { + // no outbound IO: bad content still answers before an unknown printer does + let (app, _pid) = app_with_printer(); + let (code, v) = app.post( + "/jobs", + Some("k1"), + json!({"printer_id": 4242, "type": "raw_base64", "content": "!!"}), + ); + assert_eq!(code, 400); + assert!(err_of(&v).starts_with("bad base64"), "{v}"); + let (code, v) = app.post( + "/jobs", + Some("k1"), + json!({"printer_id": 4242, "type": "raw_base64", "content": B64.encode("x")}), + ); + assert_eq!(code, 400); + assert!(err_of(&v).starts_with("unknown printer: "), "{v}"); +} + +/// Holds `n` slow downloads of one org, then submits one more fast `raw_uri` job; its status. +fn status_with_held(tweak: impl FnOnce(&mut Config), n: usize) -> u16 { + let mut app = App::with(tweak); + let fetcher = GatedFetcher::new(); + app.state.fetcher = fetcher.clone(); + add_key(&app.db, "a", "ka", DEFAULT_ORG); + let pa = register(&app.db, "pa", "aka", &[decl("P", false)], DEFAULT_ORG).pid("P"); + std::thread::scope(|s| { + let held: Vec<_> = (0..n) + .map(|i| { + let app = &app; + s.spawn(move || { + app.post( + "/jobs", + Some("ka"), + uri_job(pa, &format!("https://x/slow{i}")), + ) + }) + }) + .collect(); + fetcher.wait_entered(n); + let (code, _) = app.post("/jobs", Some("ka"), uri_job(pa, "https://x/fast")); + fetcher.release(); + for h in held { + assert_eq!(h.join().unwrap().0, 200); + } + code + }) +} + +#[test] +fn without_open_signup_and_a_setting_there_is_no_per_org_cap() { + assert_eq!(status_with_held(|_| {}, 6), 200); +} + +#[test] +fn with_open_signup_and_no_setting_the_fifth_download_answers_503() { + assert_eq!(status_with_held(|c| c.signup_open = true, 4), 503); +} + +#[test] +fn an_explicit_setting_wins_either_way() { + assert_eq!( + status_with_held( + |c| { + c.signup_open = true; + c.fetches_per_org = Some(0); + }, + 6 + ), + 200 + ); + assert_eq!(status_with_held(|c| c.fetches_per_org = Some(2), 2), 503); +} + +#[test] +fn a_label_base64_job_needs_no_download_slot() { + let mut app = App::with(|c| c.fetches_per_org = Some(4)); + let fetcher = GatedFetcher::new(); + app.state.fetcher = fetcher.clone(); + add_key(&app.db, "a", "ka", DEFAULT_ORG); + let pa = register(&app.db, "pa", "aka", &[decl("P", true)], DEFAULT_ORG).pid("P"); + let (code, v) = app.post( + "/labels", + Some("ka"), + json!({ + "name": "img", "content": "image", + "label_mm": {"w": 50, "h": 30}, "dpi": 203, "rotate": 0, + "crop_percent": {"t": 0, "r": 0, "b": 0, "l": 0}, + "place_mm": {"x": 0, "y": 0, "w": 50, "h": 30}, + }), + ); + assert_eq!(code, 200, "{v}"); + std::thread::scope(|s| { + let held: Vec<_> = (0..4) + .map(|i| { + let app = &app; + s.spawn(move || { + app.post( + "/jobs", + Some("ka"), + uri_job(pa, &format!("https://x/slow{i}")), + ) + }) + }) + .collect(); + fetcher.wait_entered(4); + let (code, v) = app.post( + "/jobs", + Some("ka"), + json!({"printer_id": pa, "type": "label_base64", "template": "img", "content": B64.encode(png())}), + ); + assert_eq!(code, 200, "{v}"); + fetcher.release(); + for h in held { + h.join().unwrap(); + } + }); +} + +#[test] +fn a_second_request_with_the_same_new_key_is_refused_while_the_first_fetches() { + let mut app = App::new(); + let fetcher = GatedFetcher::new(); + app.state.fetcher = fetcher.clone(); + add_key(&app.db, "a", "ka", DEFAULT_ORG); + let pa = register(&app.db, "pa", "aka", &[decl("P", false)], DEFAULT_ORG).pid("P"); + let body = json!({"printer_id": pa, "type": "raw_uri_post", "url": "https://x/slow", "json": {}, "idempotency_key": "K"}); + std::thread::scope(|s| { + let first = { + let (app, body) = (&app, body.clone()); + s.spawn(move || app.post("/jobs", Some("ka"), body)) + }; + fetcher.wait_entered(1); + let (code, v) = app.post("/jobs", Some("ka"), body.clone()); + assert_eq!(code, 409, "{v}"); + assert!(err_of(&v).contains("still in progress"), "{v}"); + assert_eq!(fetcher.entered.load(Ordering::SeqCst), 1); + fetcher.release(); + let (c1, v1) = first.join().unwrap(); + assert_eq!(c1, 200, "{v1}"); + let (c2, v2) = app.post("/jobs", Some("ka"), body.clone()); + assert_eq!(c2, 200, "{v2}"); + assert_eq!(v1["job_id"], v2["job_id"]); + assert_eq!(fetcher.entered.load(Ordering::SeqCst), 1); + }); +} + +#[test] +fn a_failed_fetch_frees_its_key_again() { + let (app, pid) = app_with_printer(); + *app.fetch.reply.lock().unwrap() = Some(Err("HTTP Error 500".into())); + let body = + json!({"printer_id": pid, "type": "raw_uri", "url": "https://x/a", "idempotency_key": "K"}); + for _ in 0..2 { + let (code, v) = app.post("/jobs", Some("k1"), body.clone()); + assert_eq!(code, 502, "{v}"); + } +} + +#[test] +fn label_uri_with_every_render_slot_taken_answers_503_without_fetching() { + let (app, pid) = app_with_printer(); + template(&app, "img", "image"); + let _held: Vec<_> = (0..printpapi_server::server::RENDER_SLOTS) + .map(|_| app.state.render_slots.try_enter().unwrap()) + .collect(); + let (code, v) = app.post( + "/jobs", + Some("k1"), + json!({"printer_id": pid, "type": "label_uri", "url": "https://x/a", "template": "img"}), + ); + assert_eq!(code, 503, "{v}"); + assert!(err_of(&v).contains("rendering is busy"), "{v}"); + assert_eq!(fetched(&app), 0); +} diff --git a/crates/setup/Cargo.lock b/crates/setup/Cargo.lock index 7b2387e..9c42106 100644 --- a/crates/setup/Cargo.lock +++ b/crates/setup/Cargo.lock @@ -2412,7 +2412,7 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "printpapi-agent" -version = "3.3.0" +version = "3.4.0" dependencies = [ "chrono", "libc", @@ -2426,7 +2426,7 @@ dependencies = [ [[package]] name = "printpapi-setup" -version = "3.3.0" +version = "3.4.0" dependencies = [ "printpapi-agent", "serde", diff --git a/crates/setup/Cargo.toml b/crates/setup/Cargo.toml index 5ae2d32..626886c 100644 --- a/crates/setup/Cargo.toml +++ b/crates/setup/Cargo.toml @@ -5,7 +5,7 @@ name = "printpapi-setup" description = "printpapi agent setup: a window that installs the agent on this machine" # The same number as [workspace.package].version of the root Cargo.toml; a test keeps them equal. -version = "3.3.0" +version = "3.4.0" edition = "2024" license-file = "../../LICENSE" publish = false diff --git a/docs/api.md b/docs/api.md index e6460c4..cbe9411 100644 --- a/docs/api.md +++ b/docs/api.md @@ -149,6 +149,16 @@ agent 3.2.0; older agents ignore it. Measured with SumatraPDF 3.4.6, 3.5.2 and 3 URL fetches are `http(s)`-only and sent with a browser User-Agent (WAF/CDN-fronted services reject a default library one such as Python's). +For the types that fetch (`raw_uri`, `pdf_uri`, `raw_uri_post`, `pdf_uri_post`, `label_uri`) the +server checks the printer, an existing `idempotency_key` and the quota **before** it fetches the +URL, so a wrong printer (`400`) or a spent quota (`402`) never causes an outbound request, and a +resubmitted key returns the original job without fetching again. A request whose new +`idempotency_key` is still being fetched by another request answers `409` (`a request with this +idempotency_key is still in progress, retry shortly`); retry it and it returns the job. With open +signup each org may also have at most 4 downloads in flight at once, a fifth answers `503` (`too +many downloads in flight for this org, try again later`); `PRINTAPI_FETCHES_PER_ORG` changes the +number, `0` removes the cap (the default without open signup, see [server.md](server.md)). + The `label_*` types are the odd ones out: the server does not queue the bytes it received but a **rendered label**, a PDF for a `|pdf` printer and ZPL for any other. The job's mode follows the printer, so `options` are accepted on a PDF printer and refused on a ZPL one. The template is diff --git a/docs/labels.md b/docs/labels.md index 55a11f6..58060d4 100644 --- a/docs/labels.md +++ b/docs/labels.md @@ -112,8 +112,14 @@ agent downloads and prints it like any other payload, and nothing on the agent c ### Order of checks and errors The server checks the job fields, then looks up the printer, then the template, then checks that -the template's `content` matches the job type, and only then fetches the image. A bad printer or -template therefore never causes a download. +the template's `content` matches the job type. Before it fetches the image of a `label_uri` job it +also checks, cheapest first: an existing `idempotency_key` (the original job is returned, nothing is +fetched or rendered), the org's monthly quota (`402`), the org's download cap (`503`, only where +`PRINTAPI_FETCHES_PER_ORG` is in force, see [server.md](server.md)) and whether an image render slot +is free (`503`). A bad printer or template, a spent quota or a resubmitted key therefore never causes +a download. If another request with the same new `idempotency_key` is still fetching, yours answers +`409` and nothing is fetched; retry it shortly and it returns the job. `label_base64` and +`label_text` fetch nothing, so they take no download slot. | Situation | Answer | |---|---| @@ -123,6 +129,9 @@ template therefore never causes a download. | A text job on an image template, or the other way round | `400` `label template does not take text jobs` (or `image jobs`) | | The image is not a GIF, PNG or JPEG, is larger than 40 megapixels or 20 000 pixels on one edge, would need too much memory to scale into the template's box, or the text is empty or over 500 characters | `400` `label: ` | | The image URL could not be fetched | `502` | +| The monthly job quota is spent | `402`, before any download | +| The org already has its maximum of downloads in flight | `503` `too many downloads in flight for this org, try again later`; retry | +| Another request with the same new `idempotency_key` is still running | `409` `a request with this idempotency_key is still in progress, retry shortly`; retry | | Several image renders are already running (the server renders four at a time) | `503` `label rendering is busy, try again later`; retry | | `options` on a printer that gets ZPL | `400` `options are only supported on pdf jobs` | | A [CloudPRNT](cloudprnt.md) printer | `400` `CloudPRNT printers cannot print label templates — send raw Star commands` | diff --git a/docs/printnode-compat.md b/docs/printnode-compat.md index 88f0f31..4454389 100644 --- a/docs/printnode-compat.md +++ b/docs/printnode-compat.md @@ -80,6 +80,12 @@ Field mapping onto [`POST /jobs`](api.md#submitting-a-job): | `options.*` (anything else) | dropped | see below | | `source` | — | not stored; every job reports `"printpapi"` back | +A create can also fail with `402` `QuotaExceeded` (monthly job quota spent), `502` `DownstreamError` +(the `*_uri` content could not be fetched) and `503` `ServiceUnavailable` (the org has too many +downloads in flight, see `PRINTAPI_FETCHES_PER_ORG` in [server.md](server.md); retry later). This +layer takes no idempotency key, so it never answers the `409` that [`POST /jobs`](api.md#submitting-a-job) +gives for a key still in progress. + Two deliberate leniencies, because the point of this layer is that an unmodified client works: - **Unknown option keys are dropped, not rejected.** A client sends its whole option set with every diff --git a/docs/server.md b/docs/server.md index 64c8bad..90ed9f5 100644 --- a/docs/server.md +++ b/docs/server.md @@ -46,6 +46,7 @@ Environment variables: | `PRINTAPI_TERMS_VERSION` | *(unset: no consent asked)* | A non-empty version string, e.g. `2026-11-13`. Set: `POST /signup` must carry `accept_terms` equal to it, `GET /health` reports it with the links below, and the dashboard's sign-up form shows a checkbox for the terms and the data processing agreement. Each acceptance (org, user, version, time) is stored in the `terms_acceptances` table. See [Self-signup](api.md#self-signup). With `PRINTAPI_SIGNUP=open` and no version, startup prints a warning | | `PRINTAPI_TERMS_URL` | *(required with the version)* | `http(s)` link to the terms. Anything else, or a version without it, stops the server at startup | | `PRINTAPI_DPA_URL`, `PRINTAPI_PRIVACY_URL` | *(unset)* | Optional `http(s)` links to the data processing agreement and the privacy policy, shown next to the checkbox. Anything else stops the server at startup | +| `PRINTAPI_FETCHES_PER_ORG` | `4` with open signup, else `0` (no cap) | Payload downloads (`*_uri`, `*_uri_post`, `label_uri`) one org may have in flight at once; the next one answers `503`. A non-negative integer, `0` = no cap; anything else stops the server at startup. Keeps one hosted tenant from using every download slot; a single-org box has no reason for it, since four hung upstream hosts would then block every other fetching job | | `PRINTAPI_PRIVATE_URLS` | `deny` with open signup, else `allow` | `deny`: payload URLs (`*_uri`, `*_uri_post`), job `callback_url`s and org `event_url`s may only reach public addresses — no loopback, LAN, link-local/cloud-metadata or other private ranges, checked after DNS on every redirect hop. `allow`: anywhere (a self-hosted box fetching PDFs from an internal service). Set `deny` whenever people you do not trust get keys | | `PRINTAPI_TRUSTED_PROXIES` | *(unset: nobody)* | Comma-separated IPs or CIDRs (IPv4 and IPv6, e.g. `127.0.0.1,::1` or the one pinned proxy address `172.30.0.2` of the Compose example; list nothing wider) of the reverse proxies in front of printpapi. Only a request whose socket peer is listed has its `X-Forwarded-For` read: the rightmost entry that is not itself listed is the client, which the signup throttle (IPv6 per /64) and the `LOG_REQUESTS` line then use. Unset, the socket peer is the client, so behind a proxy every signup shares one throttle slot. An invalid entry stops the server at startup. See [Behind a reverse proxy](#behind-a-reverse-proxy) | | `PRINTAPI_OPEN_ENROLLMENT` | *(off)* | `1`: any Bearer value enrolls a new agent into the default org (the v2 rule). Off: a new agent needs an issued key or the root token |