From f25d080ca73f1e48501423f4413f4be9d932868d Mon Sep 17 00:00:00 2001 From: Dennis Wuitz Date: Wed, 21 Oct 2026 00:26:27 +0200 Subject: [PATCH 1/5] db: perf: stamp manifest blobs in one update --- .../gradient-db/src/build_request_blobs.rs | 37 +++++++------------ .../src/endpoints/build_requests/manifest.rs | 15 +++----- 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/backend/gradient-db/src/build_request_blobs.rs b/backend/gradient-db/src/build_request_blobs.rs index ddf8ee035..fc3426132 100644 --- a/backend/gradient-db/src/build_request_blobs.rs +++ b/backend/gradient-db/src/build_request_blobs.rs @@ -6,27 +6,12 @@ use chrono::NaiveDateTime; use gradient_types::*; -use sea_orm::ActiveValue::Set; +use sea_orm::sea_query::Expr; use sea_orm::{ - ActiveModelTrait, ColumnTrait, Condition, ConnectionTrait, DbErr, EntityTrait, IntoActiveModel, + ActiveModelTrait, ColumnTrait, ConnectionTrait, DbErr, EntityTrait, IntoActiveModel, QueryFilter, }; -pub async fn of_project_with_hashes( - db: &C, - project: ProjectId, - hashes: Vec>, -) -> Result, DbErr> { - EBuildRequestBlob::find() - .filter( - Condition::all() - .add(CBuildRequestBlob::Project.eq(project)) - .add(CBuildRequestBlob::Hash.is_in(hashes)), - ) - .all(db) - .await -} - pub async fn insert( db: &C, blob: MBuildRequestBlob, @@ -34,12 +19,18 @@ pub async fn insert( blob.into_active_model().insert(db).await } -pub async fn mark_used( +pub async fn mark_stored_as_used( db: &C, - blob: MBuildRequestBlob, + project: ProjectId, + hashes: Vec>, at: NaiveDateTime, -) -> Result { - let mut active: ABuildRequestBlob = blob.into(); - active.last_used_at = Set(at); - active.update(db).await +) -> Result>, DbErr> { + let stored = EBuildRequestBlob::update_many() + .col_expr(CBuildRequestBlob::LastUsedAt, Expr::value(at)) + .filter(CBuildRequestBlob::Project.eq(project)) + .filter(CBuildRequestBlob::Hash.is_in(hashes)) + .exec_with_returning(db) + .await?; + + Ok(stored.into_iter().map(|blob| blob.hash).collect()) } diff --git a/backend/gradient-web/src/endpoints/build_requests/manifest.rs b/backend/gradient-web/src/endpoints/build_requests/manifest.rs index 4f3e5cf74..4ca26fbc7 100644 --- a/backend/gradient-web/src/endpoints/build_requests/manifest.rs +++ b/backend/gradient-web/src/endpoints/build_requests/manifest.rs @@ -112,19 +112,16 @@ pub async fn post_manifest( .collect() }; - let existing = gradient_db::build_request_blobs::of_project_with_hashes( + let now_ts = now(); + let existing_set: HashSet> = gradient_db::build_request_blobs::mark_stored_as_used( &state.web_db, project.id, unique_hashes.clone(), + now_ts, ) - .await?; - - let existing_set: HashSet> = existing.iter().map(|b| b.hash.clone()).collect(); - - let now_ts = now(); - for blob in &existing { - gradient_db::build_request_blobs::mark_used(&state.web_db, blob.clone(), now_ts).await?; - } + .await? + .into_iter() + .collect(); let missing_hex: Vec = unique_hashes .iter() From dfba355751f82ad94b2d2d74763908aac2e9bd69 Mon Sep 17 00:00:00 2001 From: Dennis Wuitz Date: Wed, 21 Oct 2026 00:26:27 +0200 Subject: [PATCH 2/5] nix: perf: WAL defaults for the database --- docs/src/reference/configuration.md | 5 ++++- nix/modules/gradient.nix | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/src/reference/configuration.md b/docs/src/reference/configuration.md index dc58176d8..a7830fa23 100644 --- a/docs/src/reference/configuration.md +++ b/docs/src/reference/configuration.md @@ -41,6 +41,9 @@ With `postgres.enable`, host-independent defaults come from `mkDefault` entries. | `random_page_cost` | `1.1` | Matches SSD storage and keeps the planner on index-only scans of the graph tables | | `max_connections` | `200` | Covers the three server pools (56 connections by default), autovacuum and `psql` | | `max_locks_per_transaction` | `1024` | Graph writes lock every shared build they touch. The server logs warnings below `256` | +| `wal_compression` | `lz4` | Shrinks the full page images in the write-ahead log | +| `wal_buffers` | `64MB` | Keeps large graph writes from flushing the write-ahead log mid-transaction | +| `checkpoint_timeout` | `15min` | Fewer checkpoints mean fewer full page images after each one | RAM-dependent settings live in module options. @@ -51,7 +54,7 @@ RAM-dependent settings live in module options. | `postgres.workMem` | `"32MB"` for the default pool sizes | | `postgres.maintenanceWorkMem` | Up to `"1GB"` with RAM to spare | -An external database (`database.urlFile`) needs all seven values set by hand. +All ten values for an external database (`database.urlFile`) need manual entry. ## Prometheus and OpenTelemetry diff --git a/nix/modules/gradient.nix b/nix/modules/gradient.nix index d06e71a28..6d6e4624d 100644 --- a/nix/modules/gradient.nix +++ b/nix/modules/gradient.nix @@ -790,6 +790,9 @@ in { max_connections = lib.mkDefault 200; random_page_cost = lib.mkDefault 1.1; max_locks_per_transaction = lib.mkDefault 1024; + wal_compression = lib.mkDefault "lz4"; + wal_buffers = lib.mkDefault "64MB"; + checkpoint_timeout = lib.mkDefault "15min"; } // lib.optionalAttrs (cfg.postgres.sharedBuffers != null) { shared_buffers = cfg.postgres.sharedBuffers; } // lib.optionalAttrs (cfg.postgres.effectiveCacheSize != null) { From 92eaa4b6874c80107adafaa5060630e9605c212b Mon Sep 17 00:00:00 2001 From: Dennis Wuitz Date: Wed, 21 Oct 2026 00:27:33 +0200 Subject: [PATCH 3/5] nix: perf: asynchronous commits by default --- docs/src/reference/configuration.md | 3 ++- nix/modules/gradient.nix | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/src/reference/configuration.md b/docs/src/reference/configuration.md index a7830fa23..2716a2e11 100644 --- a/docs/src/reference/configuration.md +++ b/docs/src/reference/configuration.md @@ -44,6 +44,7 @@ With `postgres.enable`, host-independent defaults come from `mkDefault` entries. | `wal_compression` | `lz4` | Shrinks the full page images in the write-ahead log | | `wal_buffers` | `64MB` | Keeps large graph writes from flushing the write-ahead log mid-transaction | | `checkpoint_timeout` | `15min` | Fewer checkpoints mean fewer full page images after each one | +| `synchronous_commit` | `off` | A commit no longer waits for the disk. A database crash loses at most the last second of commits, with no corruption | RAM-dependent settings live in module options. @@ -54,7 +55,7 @@ RAM-dependent settings live in module options. | `postgres.workMem` | `"32MB"` for the default pool sizes | | `postgres.maintenanceWorkMem` | Up to `"1GB"` with RAM to spare | -All ten values for an external database (`database.urlFile`) need manual entry. +All eleven values are manual for an external database (`database.urlFile`). ## Prometheus and OpenTelemetry diff --git a/nix/modules/gradient.nix b/nix/modules/gradient.nix index 6d6e4624d..1e0e6a2d3 100644 --- a/nix/modules/gradient.nix +++ b/nix/modules/gradient.nix @@ -793,6 +793,7 @@ in { wal_compression = lib.mkDefault "lz4"; wal_buffers = lib.mkDefault "64MB"; checkpoint_timeout = lib.mkDefault "15min"; + synchronous_commit = lib.mkDefault "off"; } // lib.optionalAttrs (cfg.postgres.sharedBuffers != null) { shared_buffers = cfg.postgres.sharedBuffers; } // lib.optionalAttrs (cfg.postgres.effectiveCacheSize != null) { From c43f4b5f866596feb819836bbe1eb73374b1c2c8 Mon Sep 17 00:00:00 2001 From: Dennis Wuitz Date: Wed, 21 Oct 2026 00:30:13 +0200 Subject: [PATCH 4/5] nix: perf: zstd WAL compression --- docs/src/reference/configuration.md | 2 +- nix/modules/gradient.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/reference/configuration.md b/docs/src/reference/configuration.md index 2716a2e11..60d3ff761 100644 --- a/docs/src/reference/configuration.md +++ b/docs/src/reference/configuration.md @@ -41,7 +41,7 @@ With `postgres.enable`, host-independent defaults come from `mkDefault` entries. | `random_page_cost` | `1.1` | Matches SSD storage and keeps the planner on index-only scans of the graph tables | | `max_connections` | `200` | Covers the three server pools (56 connections by default), autovacuum and `psql` | | `max_locks_per_transaction` | `1024` | Graph writes lock every shared build they touch. The server logs warnings below `256` | -| `wal_compression` | `lz4` | Shrinks the full page images in the write-ahead log | +| `wal_compression` | `zstd` | Shrinks the full page images in the write-ahead log | | `wal_buffers` | `64MB` | Keeps large graph writes from flushing the write-ahead log mid-transaction | | `checkpoint_timeout` | `15min` | Fewer checkpoints mean fewer full page images after each one | | `synchronous_commit` | `off` | A commit no longer waits for the disk. A database crash loses at most the last second of commits, with no corruption | diff --git a/nix/modules/gradient.nix b/nix/modules/gradient.nix index 1e0e6a2d3..4ee142898 100644 --- a/nix/modules/gradient.nix +++ b/nix/modules/gradient.nix @@ -790,7 +790,7 @@ in { max_connections = lib.mkDefault 200; random_page_cost = lib.mkDefault 1.1; max_locks_per_transaction = lib.mkDefault 1024; - wal_compression = lib.mkDefault "lz4"; + wal_compression = lib.mkDefault "zstd"; wal_buffers = lib.mkDefault "64MB"; checkpoint_timeout = lib.mkDefault "15min"; synchronous_commit = lib.mkDefault "off"; From a096a3a89c2737b5d4db44fa9d5f9b9bf4ab3909 Mon Sep 17 00:00:00 2001 From: Dennis Wuitz Date: Wed, 21 Oct 2026 00:37:51 +0200 Subject: [PATCH 5/5] nix: fix: standalone VM with its own store image --- docs/src/get-started/standalone.md | 2 +- nix/standalone/vm.nix | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/src/get-started/standalone.md b/docs/src/get-started/standalone.md index 0c16af54a..f742b73cb 100644 --- a/docs/src/get-started/standalone.md +++ b/docs/src/get-started/standalone.md @@ -27,7 +27,7 @@ Standalone mode runs the whole Gradient stack on one machine. This setup suits p nix run github:wavelens/gradient/latest#standalone ``` - This runs a QEMU VM in the terminal with a root login. Projects, the cache and every build persist in the disk image `gradient.qcow2` in the current directory across a restart. Stop the VM with `Ctrl-a x`. + This runs a QEMU VM in the terminal with a root login. Projects, the cache and every build persist in the disk image `gradient.qcow2` in the current directory across a restart. Press `Ctrl-a x` to stop the VM. The run script can rebuild the image of the VM's Nix store before every boot. A startup can take longer than a plain VM. ??? note "Image Provenance" Release images come with a signed build provenance and an SBOM. The GitHub CLI can verify the provenance against the repository. diff --git a/nix/standalone/vm.nix b/nix/standalone/vm.nix index e20ae4302..b76ed00db 100644 --- a/nix/standalone/vm.nix +++ b/nix/standalone/vm.nix @@ -19,6 +19,8 @@ diskSize = 32768; graphics = false; writableStoreUseTmpfs = false; + useNixStoreImage = true; + writableStore = true; forwardPorts = [ { from = "host"; host.address = "127.0.0.1"; host.port = 8080; guest.port = 80; } ];