diff --git a/backend/gradient-db/src/build_request_blobs.rs b/backend/gradient-db/src/build_request_blobs.rs index ddf8ee035..fc3426132 100644 --- a/backend/gradient-db/src/build_request_blobs.rs +++ b/backend/gradient-db/src/build_request_blobs.rs @@ -6,27 +6,12 @@ use chrono::NaiveDateTime; use gradient_types::*; -use sea_orm::ActiveValue::Set; +use sea_orm::sea_query::Expr; use sea_orm::{ - ActiveModelTrait, ColumnTrait, Condition, ConnectionTrait, DbErr, EntityTrait, IntoActiveModel, + ActiveModelTrait, ColumnTrait, ConnectionTrait, DbErr, EntityTrait, IntoActiveModel, QueryFilter, }; -pub async fn of_project_with_hashes( - db: &C, - project: ProjectId, - hashes: Vec>, -) -> Result, DbErr> { - EBuildRequestBlob::find() - .filter( - Condition::all() - .add(CBuildRequestBlob::Project.eq(project)) - .add(CBuildRequestBlob::Hash.is_in(hashes)), - ) - .all(db) - .await -} - pub async fn insert( db: &C, blob: MBuildRequestBlob, @@ -34,12 +19,18 @@ pub async fn insert( blob.into_active_model().insert(db).await } -pub async fn mark_used( +pub async fn mark_stored_as_used( db: &C, - blob: MBuildRequestBlob, + project: ProjectId, + hashes: Vec>, at: NaiveDateTime, -) -> Result { - let mut active: ABuildRequestBlob = blob.into(); - active.last_used_at = Set(at); - active.update(db).await +) -> Result>, DbErr> { + let stored = EBuildRequestBlob::update_many() + .col_expr(CBuildRequestBlob::LastUsedAt, Expr::value(at)) + .filter(CBuildRequestBlob::Project.eq(project)) + .filter(CBuildRequestBlob::Hash.is_in(hashes)) + .exec_with_returning(db) + .await?; + + Ok(stored.into_iter().map(|blob| blob.hash).collect()) } diff --git a/backend/gradient-web/src/endpoints/build_requests/manifest.rs b/backend/gradient-web/src/endpoints/build_requests/manifest.rs index 4f3e5cf74..4ca26fbc7 100644 --- a/backend/gradient-web/src/endpoints/build_requests/manifest.rs +++ b/backend/gradient-web/src/endpoints/build_requests/manifest.rs @@ -112,19 +112,16 @@ pub async fn post_manifest( .collect() }; - let existing = gradient_db::build_request_blobs::of_project_with_hashes( + let now_ts = now(); + let existing_set: HashSet> = gradient_db::build_request_blobs::mark_stored_as_used( &state.web_db, project.id, unique_hashes.clone(), + now_ts, ) - .await?; - - let existing_set: HashSet> = existing.iter().map(|b| b.hash.clone()).collect(); - - let now_ts = now(); - for blob in &existing { - gradient_db::build_request_blobs::mark_used(&state.web_db, blob.clone(), now_ts).await?; - } + .await? + .into_iter() + .collect(); let missing_hex: Vec = unique_hashes .iter() diff --git a/docs/src/get-started/standalone.md b/docs/src/get-started/standalone.md index 0c16af54a..f742b73cb 100644 --- a/docs/src/get-started/standalone.md +++ b/docs/src/get-started/standalone.md @@ -27,7 +27,7 @@ Standalone mode runs the whole Gradient stack on one machine. This setup suits p nix run github:wavelens/gradient/latest#standalone ``` - This runs a QEMU VM in the terminal with a root login. Projects, the cache and every build persist in the disk image `gradient.qcow2` in the current directory across a restart. Stop the VM with `Ctrl-a x`. + This runs a QEMU VM in the terminal with a root login. Projects, the cache and every build persist in the disk image `gradient.qcow2` in the current directory across a restart. Press `Ctrl-a x` to stop the VM. The run script can rebuild the image of the VM's Nix store before every boot. A startup can take longer than a plain VM. ??? note "Image Provenance" Release images come with a signed build provenance and an SBOM. The GitHub CLI can verify the provenance against the repository. diff --git a/docs/src/reference/configuration.md b/docs/src/reference/configuration.md index dc58176d8..60d3ff761 100644 --- a/docs/src/reference/configuration.md +++ b/docs/src/reference/configuration.md @@ -41,6 +41,10 @@ With `postgres.enable`, host-independent defaults come from `mkDefault` entries. | `random_page_cost` | `1.1` | Matches SSD storage and keeps the planner on index-only scans of the graph tables | | `max_connections` | `200` | Covers the three server pools (56 connections by default), autovacuum and `psql` | | `max_locks_per_transaction` | `1024` | Graph writes lock every shared build they touch. The server logs warnings below `256` | +| `wal_compression` | `zstd` | Shrinks the full page images in the write-ahead log | +| `wal_buffers` | `64MB` | Keeps large graph writes from flushing the write-ahead log mid-transaction | +| `checkpoint_timeout` | `15min` | Fewer checkpoints mean fewer full page images after each one | +| `synchronous_commit` | `off` | A commit no longer waits for the disk. A database crash loses at most the last second of commits, with no corruption | RAM-dependent settings live in module options. @@ -51,7 +55,7 @@ RAM-dependent settings live in module options. | `postgres.workMem` | `"32MB"` for the default pool sizes | | `postgres.maintenanceWorkMem` | Up to `"1GB"` with RAM to spare | -An external database (`database.urlFile`) needs all seven values set by hand. +All eleven values are manual for an external database (`database.urlFile`). ## Prometheus and OpenTelemetry diff --git a/nix/modules/gradient.nix b/nix/modules/gradient.nix index d06e71a28..4ee142898 100644 --- a/nix/modules/gradient.nix +++ b/nix/modules/gradient.nix @@ -790,6 +790,10 @@ in { max_connections = lib.mkDefault 200; random_page_cost = lib.mkDefault 1.1; max_locks_per_transaction = lib.mkDefault 1024; + wal_compression = lib.mkDefault "zstd"; + wal_buffers = lib.mkDefault "64MB"; + checkpoint_timeout = lib.mkDefault "15min"; + synchronous_commit = lib.mkDefault "off"; } // lib.optionalAttrs (cfg.postgres.sharedBuffers != null) { shared_buffers = cfg.postgres.sharedBuffers; } // lib.optionalAttrs (cfg.postgres.effectiveCacheSize != null) { diff --git a/nix/standalone/vm.nix b/nix/standalone/vm.nix index e20ae4302..b76ed00db 100644 --- a/nix/standalone/vm.nix +++ b/nix/standalone/vm.nix @@ -19,6 +19,8 @@ diskSize = 32768; graphics = false; writableStoreUseTmpfs = false; + useNixStoreImage = true; + writableStore = true; forwardPorts = [ { from = "host"; host.address = "127.0.0.1"; host.port = 8080; guest.port = 80; } ];