db: perf: fewer WAL flushes

backend/gradient-db/src/build_request_blobs.rs

@@ -6,40 +6,31 @@
 
 use chrono::NaiveDateTime;
 use gradient_types::*;
-use sea_orm::ActiveValue::Set;
+use sea_orm::sea_query::Expr;
 use sea_orm::{
-    ActiveModelTrait, ColumnTrait, Condition, ConnectionTrait, DbErr, EntityTrait, IntoActiveModel,
+    ActiveModelTrait, ColumnTrait, ConnectionTrait, DbErr, EntityTrait, IntoActiveModel,
     QueryFilter,
 };
 
-pub async fn of_project_with_hashes<C: ConnectionTrait>(
-    db: &C,
-    project: ProjectId,
-    hashes: Vec<Vec<u8>>,
-) -> Result<Vec<MBuildRequestBlob>, DbErr> {
-    EBuildRequestBlob::find()
-        .filter(
-            Condition::all()
-                .add(CBuildRequestBlob::Project.eq(project))
-                .add(CBuildRequestBlob::Hash.is_in(hashes)),
-        )
-        .all(db)
-        .await
-}
-
 pub async fn insert<C: ConnectionTrait>(
     db: &C,
     blob: MBuildRequestBlob,
 ) -> Result<MBuildRequestBlob, DbErr> {
     blob.into_active_model().insert(db).await
 }
 
-pub async fn mark_used<C: ConnectionTrait>(
+pub async fn mark_stored_as_used<C: ConnectionTrait>(
     db: &C,
-    blob: MBuildRequestBlob,
+    project: ProjectId,
+    hashes: Vec<Vec<u8>>,
     at: NaiveDateTime,
-) -> Result<MBuildRequestBlob, DbErr> {
-    let mut active: ABuildRequestBlob = blob.into();
-    active.last_used_at = Set(at);
-    active.update(db).await
+) -> Result<Vec<Vec<u8>>, DbErr> {
+    let stored = EBuildRequestBlob::update_many()
+        .col_expr(CBuildRequestBlob::LastUsedAt, Expr::value(at))
+        .filter(CBuildRequestBlob::Project.eq(project))
+        .filter(CBuildRequestBlob::Hash.is_in(hashes))
+        .exec_with_returning(db)
+        .await?;
+
+    Ok(stored.into_iter().map(|blob| blob.hash).collect())
 }

backend/gradient-web/src/endpoints/build_requests/manifest.rs

@@ -112,19 +112,16 @@ pub async fn post_manifest(
             .collect()
     };
 
-    let existing = gradient_db::build_request_blobs::of_project_with_hashes(
+    let now_ts = now();
+    let existing_set: HashSet<Vec<u8>> = gradient_db::build_request_blobs::mark_stored_as_used(
         &state.web_db,
         project.id,
         unique_hashes.clone(),
+        now_ts,
     )
-    .await?;
-
-    let existing_set: HashSet<Vec<u8>> = existing.iter().map(|b| b.hash.clone()).collect();
-
-    let now_ts = now();
-    for blob in &existing {
-        gradient_db::build_request_blobs::mark_used(&state.web_db, blob.clone(), now_ts).await?;
-    }
+    .await?
+    .into_iter()
+    .collect();
 
     let missing_hex: Vec<String> = unique_hashes
         .iter()

docs/src/get-started/standalone.md

@@ -27,7 +27,7 @@ Standalone mode runs the whole Gradient stack on one machine. This setup suits p
     nix run github:wavelens/gradient/latest#standalone
     ```
 
-    This runs a QEMU VM in the terminal with a root login. Projects, the cache and every build persist in the disk image `gradient.qcow2` in the current directory across a restart. Stop the VM with `Ctrl-a x`.
+    This runs a QEMU VM in the terminal with a root login. Projects, the cache and every build persist in the disk image `gradient.qcow2` in the current directory across a restart. Press `Ctrl-a x` to stop the VM. The run script can rebuild the image of the VM's Nix store before every boot. A startup can take longer than a plain VM.
 
 ??? note "Image Provenance"
     Release images come with a signed build provenance and an SBOM. The GitHub CLI can verify the provenance against the repository.

docs/src/reference/configuration.md

@@ -41,6 +41,10 @@ With `postgres.enable`, host-independent defaults come from `mkDefault` entries.
 | `random_page_cost` | `1.1` | Matches SSD storage and keeps the planner on index-only scans of the graph tables |
 | `max_connections` | `200` | Covers the three server pools (56 connections by default), autovacuum and `psql` |
 | `max_locks_per_transaction` | `1024` | Graph writes lock every shared build they touch. The server logs warnings below `256` |
+| `wal_compression` | `zstd` | Shrinks the full page images in the write-ahead log |
+| `wal_buffers` | `64MB` | Keeps large graph writes from flushing the write-ahead log mid-transaction |
+| `checkpoint_timeout` | `15min` | Fewer checkpoints mean fewer full page images after each one |
+| `synchronous_commit` | `off` | A commit no longer waits for the disk. A database crash loses at most the last second of commits, with no corruption |
 
 RAM-dependent settings live in module options.
 
@@ -51,7 +55,7 @@ RAM-dependent settings live in module options.
 | `postgres.workMem` | `"32MB"` for the default pool sizes |
 | `postgres.maintenanceWorkMem` | Up to `"1GB"` with RAM to spare |
 
-An external database (`database.urlFile`) needs all seven values set by hand.
+All eleven values are manual for an external database (`database.urlFile`).
 
 ## Prometheus and OpenTelemetry
 

nix/modules/gradient.nix

@@ -790,6 +790,10 @@ in {
           max_connections = lib.mkDefault 200;
           random_page_cost = lib.mkDefault 1.1;
           max_locks_per_transaction = lib.mkDefault 1024;
+          wal_compression = lib.mkDefault "zstd";
+          wal_buffers = lib.mkDefault "64MB";
+          checkpoint_timeout = lib.mkDefault "15min";
+          synchronous_commit = lib.mkDefault "off";
         } // lib.optionalAttrs (cfg.postgres.sharedBuffers != null) {
           shared_buffers = cfg.postgres.sharedBuffers;
         } // lib.optionalAttrs (cfg.postgres.effectiveCacheSize != null) {

nix/standalone/vm.nix

@@ -19,6 +19,8 @@
     diskSize = 32768;
     graphics = false;
     writableStoreUseTmpfs = false;
+    useNixStoreImage = true;
+    writableStore = true;
     forwardPorts = [
       { from = "host"; host.address = "127.0.0.1"; host.port = 8080; guest.port = 80; }
     ];