From 11d13606e797d4599f71d93eb22bd45a469c8ef9 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 5 Aug 2026 14:42:50 -0700 Subject: [PATCH 01/27] on-demand backups --- client/www/components/dash/Backups.tsx | 323 ++++++++++++-- client/www/lib/types.ts | 14 + .../migrations/121_app_backup_jobs.down.sql | 1 + .../migrations/121_app_backup_jobs.up.sql | 34 ++ server/src/instant/backup.clj | 103 ++++- server/src/instant/core.clj | 6 + server/src/instant/dash/routes.clj | 90 ++++ server/src/instant/db/app_backup_jobs.clj | 399 ++++++++++++++++++ server/src/instant/flags.clj | 36 ++ server/src/instant/rate_limit.clj | 14 + server/src/instant/util/exception.clj | 28 +- 11 files changed, 988 insertions(+), 60 deletions(-) create mode 100644 server/resources/migrations/121_app_backup_jobs.down.sql create mode 100644 server/resources/migrations/121_app_backup_jobs.up.sql create mode 100644 server/src/instant/db/app_backup_jobs.clj diff --git a/client/www/components/dash/Backups.tsx b/client/www/components/dash/Backups.tsx index 99fcac6263..1e59a8f834 100644 --- a/client/www/components/dash/Backups.tsx +++ b/client/www/components/dash/Backups.tsx @@ -1,18 +1,34 @@ -import { useContext, useMemo } from 'react'; -import { ArrowDownTrayIcon } from '@heroicons/react/24/outline'; +import { + FormEvent, + ReactNode, + useContext, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; +import { ArrowDownTrayIcon, PlusIcon } from '@heroicons/react/24/outline'; +import { Transition } from '@headlessui/react'; import config from '@/lib/config'; import { useAuthedFetch } from '@/lib/auth'; +import { jsonFetch, jsonMutate } from '@/lib/fetch'; +import { errorToast, successToast } from '@/lib/toast'; import { TokenContext } from '@/lib/contexts'; -import { InstantApp, InstantAppBackup } from '@/lib/types'; +import { InstantApp, InstantAppBackup, InstantAppBackupJob } from '@/lib/types'; import { Button, Content, + Dialog, + Label, SectionHeading, + SubsectionHeading, + TextInput, Tooltip, TooltipContent, TooltipTrigger, + useDialog, } from '@/components/ui'; import { ErrorMessage, @@ -30,6 +46,93 @@ import { ItemTitle, } from '@/components/components/ui/item'; +function createBackup(token: string, appId: string, description?: string) { + return jsonMutate<{ job: InstantAppBackupJob }>( + `${config.apiURI}/dash/apps/${appId}/backups`, + { token, body: description ? { description } : {} }, + ); +} + +function fetchBackupJob(token: string, appId: string, jobId: string) { + return jsonFetch(`${config.apiURI}/dash/apps/${appId}/backup-jobs/${jobId}`, { + headers: { authorization: `Bearer ${token}` }, + }) as Promise<{ job: InstantAppBackupJob | null }>; +} + +// Shared layout for both in-progress jobs and finished backups. Rendering them +// through one component with the same structure (title, optional description, a +// two-row detail grid, and an action) keeps the row the exact same size, so it +// doesn't reflow when a job finishes and becomes a backup. +function BackupItem({ + title, + description, + rows, + action, +}: { + title: string; + description?: string | null; + rows: [string, ReactNode][]; + action: ReactNode; +}) { + return ( + + + {title} + {description ? {description} : null} +
+ {rows.map(([label, value]) => [ +
{label}
, +
{value}
, + ])} +
+ + {action} + + ); +} + +function ProgressBar({ pct }: { pct: number }) { + return ( +
+
+
+ ); +} + +function BackupJobRow({ job }: { job: InstantAppBackupJob }) { + const completed = job.work_completed ?? 0; + // Pad the estimate by 10% so the bar doesn't sit at 100% while the backup + // finishes uploading and finalizing. + const estimate = Math.round((job.work_estimate ?? 0) * 1.1); + const pct = + estimate > 0 + ? Math.min(100, Math.round((completed / estimate) * 100)) + : null; + const label = job.job_status === 'waiting' ? 'Starting…' : 'Backing up…'; + + return ( + ], + ]} + action={ + + {pct !== null ? `${pct}%` : ''} + + } + /> + ); +} + function BackupRow({ app, backup, @@ -69,30 +172,21 @@ function BackupRow({ ); return ( - - - {formatTimestamp(backup.backup_at)} - {backup.description ? ( - {backup.description} - ) : null} -
- {backup.expires_at ? ( - <> -
Expires
-
{formatTimestamp(backup.expires_at)}
- - ) : null} -
ID
-
- -
-
- - - {blockedByOther ? ( + , + ], + ]} + action={ + blockedByOther ? ( @@ -105,17 +199,78 @@ function BackupRow({ ) : ( downloadButton - )} - - + ) + } + /> ); } export function Backups({ app }: { app: InstantApp }) { + const token = useContext(TokenContext); + const createDialog = useDialog(); + const [description, setDescription] = useState(''); + const [creating, setCreating] = useState(false); + const [createError, setCreateError] = useState(null); + // Retains the message while the error box fades out (createError is null + // during the leave transition), so it doesn't blank before it's gone. + const lastCreateError = useRef(''); + + function openCreateDialog() { + setCreateError(null); + createDialog.onOpen(); + } + const backupsRes = useAuthedFetch<{ backups: InstantAppBackup[]; }>(`${config.apiURI}/dash/apps/${app.id}/backups`); + const jobsRes = useAuthedFetch<{ + jobs: InstantAppBackupJob[]; + }>(`${config.apiURI}/dash/apps/${app.id}/backup-jobs`); + + const jobs = jobsRes.data?.jobs ?? []; + const activeCount = jobs.length; + const hasActive = activeCount > 0; + const jobIds = jobs.map((j) => j.id); + const jobIdsKey = jobIds.join(','); + + // While a backup is running, poll both the jobs and the backups list so + // progress advances and the finished snapshot shows up. + useEffect(() => { + if (!hasActive) return; + const t = setInterval(() => { + jobsRes.mutate(); + backupsRes.mutate(); + }, 1000); + return () => clearInterval(t); + }, [hasActive, jobsRes.mutate, backupsRes.mutate]); + + // Detect when a job we were watching leaves the active list and alert on its + // outcome. `seenJobIds` starts empty, so we never alert for jobs that were + // already finished before this component mounted. + const seenJobIds = useRef>(new Set()); + useEffect(() => { + // Rebuild the id set from jobIdsKey (not jobIds) so the effect keys off the + // stable string and re-runs only when the set of active jobs changes. + const current = new Set(jobIdsKey ? jobIdsKey.split(',') : []); + const finished = [...seenJobIds.current].filter((id) => !current.has(id)); + seenJobIds.current = current; + if (finished.length === 0 || !token) return; + backupsRes.mutate(); + finished.forEach(async (id) => { + try { + const { job } = await fetchBackupJob(token, app.id, id); + if (job?.job_status === 'completed') { + successToast('Backup finished.'); + } else if (job?.job_status === 'errored') { + errorToast(job.error ?? 'Backup failed.', { autoClose: 8000 }); + } + } catch { + // Best-effort alert; ignore lookup failures. + } + }); + }, [jobIdsKey, token, app.id, backupsRes.mutate]); + const backups = useMemo( () => (backupsRes.data?.backups ?? []).filter( @@ -125,31 +280,127 @@ export function Backups({ app }: { app: InstantApp }) { [backupsRes.data?.backups], ); + async function onCreate(e: FormEvent) { + e.preventDefault(); + if (!token) return; + setCreating(true); + setCreateError(null); + try { + const desc = description.trim(); + await createBackup(token, app.id, desc || undefined); + successToast('Backup started.'); + setDescription(''); + createDialog.onClose(); + jobsRes.mutate(); + } catch (e: any) { + // Show the error (e.g. rate limit, too-large app) inline in the dialog so + // it stays put next to the form instead of flashing by as a toast. + const msg = e?.body?.message ?? 'Failed to start backup.'; + lastCreateError.current = msg; + setCreateError(msg); + } finally { + setCreating(false); + } + } + if (backupsRes.isLoading) return ; if (backupsRes.error) { return Failed to load backups.; } + const createButton = ( + + ); + return (
-
- Backups - - Point-in-time snapshots of your app's data. - +
+
+ Backups + + Point-in-time snapshots of your app's data. + +
+ {hasActive ? ( + + + + {createButton} + + + A backup is already in progress. + + ) : ( + createButton + )}
- {backups.length === 0 ? ( + {jobs.length === 0 && backups.length === 0 ? (
No backups yet.
) : ( + {jobs.map((job) => ( + + ))} {backups.map((b) => ( ))} )} + + +
+ Create backup + + Take a point-in-time snapshot of your app's data. Large apps can + take a while. + +
+ + +
+ + {createError ?? lastCreateError.current} + +
+ + +
+
+
); } diff --git a/client/www/lib/types.ts b/client/www/lib/types.ts index 2f960f85aa..385338534b 100644 --- a/client/www/lib/types.ts +++ b/client/www/lib/types.ts @@ -101,6 +101,20 @@ export type InstantAppBackup = { expires_at: string | null; }; +export type InstantAppBackupJob = { + id: string; + app_id: string; + app_backup_id: string | null; + job_status: 'waiting' | 'processing' | 'completed' | 'errored'; + description: string | null; + error: string | null; + work_estimate: number | null; + work_completed: number | null; + created_at: string; + updated_at: string; + done_at: string | null; +}; + export type InstantMember = { id: string; email: string; diff --git a/server/resources/migrations/121_app_backup_jobs.down.sql b/server/resources/migrations/121_app_backup_jobs.down.sql new file mode 100644 index 0000000000..59494775ec --- /dev/null +++ b/server/resources/migrations/121_app_backup_jobs.down.sql @@ -0,0 +1 @@ +drop table app_backup_jobs; diff --git a/server/resources/migrations/121_app_backup_jobs.up.sql b/server/resources/migrations/121_app_backup_jobs.up.sql new file mode 100644 index 0000000000..b31f20c885 --- /dev/null +++ b/server/resources/migrations/121_app_backup_jobs.up.sql @@ -0,0 +1,34 @@ +create table app_backup_jobs( + id uuid primary key, + app_id uuid not null references apps(id) on delete cascade, + app_backup_id uuid references app_backups(id) on delete set null, + worker_id text, + job_status text not null default 'waiting', + description text, + error text, + work_estimate bigint, + work_completed bigint, + created_at timestamptz not null default now(), + done_at timestamptz, + updated_at timestamptz not null default now() +); + +create index on app_backup_jobs(app_id); + +-- Enforces "at most one in-flight backup per app". A second enqueue while a +-- job is waiting/processing violates this and surfaces as a validation error. +create unique index app_backup_jobs_one_in_flight_per_app + on app_backup_jobs(app_id) + where job_status in ('waiting', 'processing'); + +-- Serves the worker claim query: oldest unclaimed waiting job. Ordered + partial +-- so FOR UPDATE SKIP LOCKED walks pending jobs in FIFO order with no sort, and +-- rows drop out the moment they're claimed (status -> processing). +create index app_backup_jobs_next_waiting + on app_backup_jobs(created_at) + where worker_id is null and job_status = 'waiting'; + +create trigger update_updated_at_trigger +before update on app_backup_jobs +for each row +execute function update_updated_at_column(); diff --git a/server/src/instant/backup.clj b/server/src/instant/backup.clj index d0ab57f92b..2e55564646 100644 --- a/server/src/instant/backup.clj +++ b/server/src/instant/backup.clj @@ -106,15 +106,16 @@ :db-size :?db-size :uncompressed-size :?uncompressed-size :description :?description - :expires-at :?expires-at}]})) + :expires-at :?expires-at}] + :returning :*})) (defn insert-app-backup! ([params] (insert-app-backup! (aurora/conn-pool :write) params)) ([conn {:keys [id app-id isn backup-at storage-prefix files-size db-size uncompressed-size description expires-at]}] - (sql/do-execute! ::insert-app-backup! - conn - (uhsql/formatp insert-app-backup-q - {:id id + (sql/execute-one! ::insert-app-backup! + conn + (uhsql/formatp insert-app-backup-q + {:id id :app-id app-id :isn isn :backup-at backup-at @@ -329,7 +330,9 @@ (defn complete-streams [query-conn {:keys [backup-id isn - backup-at]} + backup-at + description + expires-at]} {:keys [streams app-id]}] ;; First, write the streams out (->> (mapv (fn [^Stream stream] @@ -383,8 +386,9 @@ :files-size (get-storage-usage query-conn app-id) :db-size (get-app-usage query-conn app-id) :uncompressed-size uncompressed-size - :description "Automated Daily Snapshot" - :expires-at (.plus (Instant/now) (Duration/ofDays 7))}))) + :description (or description "Automated Daily Snapshot") + :expires-at (or expires-at + (.plus (Instant/now) (Duration/ofDays 7)))}))) ;; Then update the db with the app backup (defn update-entity @@ -739,12 +743,26 @@ (defn handle-app "Processes a single app. Delivers the result to `finished-promise` (either - a throwable or the {:triple-count } with the number of triples)." + a throwable or {:triple-count :app-backup } where the row is the + inserted `app_backups` record, or nil when the app had no triples and + `ensure-config?` was not set). + + `description` and `expires-at` override the defaults on the `app_backups` + row. When `ensure-config?` is true, an empty app still writes a config.json + and an `app_backups` row so the backup is restorable. + + `on-triple`, if given, is called once per triple as the app is copied. It + runs on the copy thread, so keep it cheap and non-blocking (e.g. bump a + counter) and do any I/O elsewhere." [{:keys [clone-pool process-id isn backup-at app-id + description + expires-at + ensure-config? + on-triple finished-promise]}] (try (let [triples-queue (LinkedBlockingQueue. 5000) @@ -755,7 +773,8 @@ (try (with-open [conn (next.jdbc/get-connection clone-pool)] (doseq [triple (app-triples-seq (.unwrap conn PgConnection) app-id)] - (.put triples-queue triple)) + (.put triples-queue triple) + (when on-triple (on-triple))) (.put triples-queue done-signal)) (catch Throwable t (deliver finished-promise t) @@ -777,23 +796,71 @@ (throw t)))) _ @copy-process _ @upload-process + ctx {:backup-id process-id + :isn isn + :backup-at backup-at + :description description + :expires-at expires-at} ;; This is a little awkward, since we'll only ever take 1 thing out of the queue, ;; but it allows us to use the same code as `process-with-copy` item (.take flush-streams-queue)] (if (= done-signal item) (when-not (realized? finished-promise) - (deliver finished-promise {:triple-count 0})) + (if ensure-config? + ;; The app had no triples, but we still want a restorable backup, so + ;; write a config.json with empty counts and record the row. + (with-open [conn (next.jdbc/get-connection clone-pool)] + (let [app-backup (complete-streams conn ctx {:streams {} :app-id app-id})] + (deliver finished-promise {:triple-count 0 + :app-backup app-backup}))) + (deliver finished-promise {:triple-count 0 + :app-backup nil}))) (with-open [conn (next.jdbc/get-connection clone-pool)] - (complete-streams conn - {:backup-id process-id - :isn isn - :backup-at backup-at} - item) - (deliver finished-promise - {:triple-count (:triple-count (.take upload-progress-queue))})))) + (let [app-backup (complete-streams conn ctx item)] + (deliver finished-promise + {:triple-count (:triple-count (.take upload-progress-queue)) + :app-backup app-backup}))))) (catch Throwable t (deliver finished-promise t)))) +(defn backup-app-on-primary! + "Runs an on-demand backup of a single app against the primary. + + Opens a repeatable-read, read-only snapshot connection and routes both the + triples COPY and the config.json reads through it (via `snapshot-datasource`) + so the whole backup sees a single consistent point in time. Because it reads + its own snapshot there's no clone and no replication slot. + + `on-triple`, if given, is called once per triple as the app is copied (see + `handle-app`); keep it cheap and non-blocking. + + Returns {:triple-count :app-backup } where the row is the + inserted `app_backups` record. Throws if the backup fails." + [{:keys [app-id description expires-at on-triple]}] + (let [backup-id (random-uuid)] + (with-open [conn (wal/get-pg-copy-ready-conn (config/get-aurora-config))] + (.setAutoCommit conn false) + ;; Fixes the snapshot for every read on this connection. + (sql/select conn ["set transaction isolation level repeatable read, read only"]) + (tracer/with-span! {:name "backup/backup-app-on-primary" + :attributes {:app-id app-id + :backup-id backup-id}} + (let [finished-promise (promise)] + (handle-app {:clone-pool (snapshot-datasource conn) + :process-id backup-id + :isn nil + :backup-at (Instant/now) + :app-id app-id + :description description + :expires-at expires-at + :ensure-config? true + :on-triple on-triple + :finished-promise finished-promise}) + (let [result @finished-promise] + (when (instance? Throwable result) + (throw result)) + result)))))) + (defn log-retry-queue! "Logs the apps that failed during a backup. Records a top-level exception span when the retry queue is non-empty so that it surfaces in our alerting." diff --git a/server/src/instant/core.clj b/server/src/instant/core.clj index 0959a3fdb3..c3b12c9d7a 100644 --- a/server/src/instant/core.clj +++ b/server/src/instant/core.clj @@ -14,6 +14,7 @@ [instant.config :as config] [instant.dash.ephemeral-app :as ephemeral-app] [instant.dash.routes :as dash-routes] + [instant.db.app-backup-jobs :as app-backup-jobs] [instant.db.indexing-jobs :as indexing-jobs] [instant.db.hint-testing :as hint-testing] [instant.db.model.wal-log :as wal-log-model] @@ -351,6 +352,9 @@ (future (tracer/with-span! {:name "stop-indexing-jobs"} (indexing-jobs/stop))) + (future + (tracer/with-span! {:name "stop-app-backup-jobs"} + (app-backup-jobs/stop))) (future (tracer/with-span! {:name "stop-join-room-logger"} (join-room-logger/stop))) @@ -467,6 +471,8 @@ (join-room-logger/start)) (with-log-init :indexing-jobs (indexing-jobs/start)) + (with-log-init :app-backup-jobs + (app-backup-jobs/start)) (with-log-init :storage-sweeper (storage-sweeper/start)) (with-log-init :hard-deletion-sweeper diff --git a/server/src/instant/dash/routes.clj b/server/src/instant/dash/routes.clj index 264cec881a..f4b4944a86 100644 --- a/server/src/instant/dash/routes.clj +++ b/server/src/instant/dash/routes.clj @@ -11,6 +11,7 @@ [instant.dash.admin :as dash-admin] [instant.dash.ephemeral-app :as ephemeral-app] [instant.dash.get-a-db :as get-a-db] + [instant.db.app-backup-jobs :as app-backup-jobs] [instant.db.indexing-jobs :as indexing-jobs] [instant.db.model.attr :as attr-model] [instant.db.transaction :as tx] @@ -59,6 +60,8 @@ [instant.model.webhook :as webhook-model] [instant.plans :as plans] [instant.postmark :as postmark] + [instant.rate-limit :as rate-limit] + [instant.reactive.ephemeral :as eph] [instant.runtime.magic-code-auth :as magic-code-auth :refer [check-send-rate-limit! check-verify-rate-limit! @@ -1941,6 +1944,90 @@ "X-Accel-Buffering" "no"} :body pipe-in})) +;; Hack to let us use the existing rate-limit machinery across apps +(def backup-ip-rate-limit-app-id #uuid "00000000-0000-0000-0000-000000000000") + +(defn ->backup-rate-limit-config + "Turns a {:capacity n :window-minutes m} flag value into a bucket4j config + that refills `capacity` tokens over the window." + [{:keys [capacity window-minutes]}] + {"limits" [{"capacity" capacity + "refill" {"period" (str window-minutes " minutes") + "amount" capacity + "type" "greedy"}}]}) + +(defn humanize-retry-in + "Human-readable \"try again\" duration until `retry-at`, e.g. \"45 seconds\" + or \"3 minutes\"." + [retry-at] + (let [secs (max 1 (ex/retry-after-seconds retry-at))] + (if (< secs 60) + (str secs " second" (when (not= 1 secs) "s")) + (let [mins (long (Math/ceil (/ secs 60.0)))] + (str mins " minute" (when (not= 1 mins) "s")))))) + +(defn consume-backup-token! + "Consumes one backup token from the named bucket. On exhaustion throws a + rate-limited error telling the user when to try again." + [rate-limiter params] + (when-let [retry-at (rate-limit/consume-user-rate-limit-retry-at rate-limiter params)] + (ex/throw-rate-limited-until! + (str "You've hit the backup rate limit. Please try again in " + (humanize-retry-in retry-at) ".") + retry-at))) + +(defn check-backup-rate-limits! + "Throttles on-demand backups per app and per client IP, both flag-tunable + (`on-demand-backup-app-rate-limit`, `on-demand-backup-ip-rate-limit`). Throws + a rate-limited error (with a retry time) when a bucket is exhausted. No-op + when the rate limiter isn't running (e.g. tests)." + [{:keys [app-id ip]}] + (when-let [rate-limiter (eph/get-rate-limit)] + (consume-backup-token! + rate-limiter + {:app-id app-id + :bucket-name "on-demand-backup-app" + :config (->backup-rate-limit-config (flags/on-demand-backup-app-rate-limit))}) + (when ip + (consume-backup-token! + rate-limiter + {:app-id backup-ip-rate-limit-app-id + :bucket-name "on-demand-backup-ip" + :bucket-key ip + :config (->backup-rate-limit-config (flags/on-demand-backup-ip-rate-limit))})))) + +(defn app-backup-job-post + "Kicks off an on-demand backup for the app. Returns the created job so the + client can poll its progress." + [req] + (let [{{app-id :id} :app} (req->app-accepting-superadmin-or-ref-token! :collaborator + :apps/write + req) + description (ex/get-optional-param! req [:body :description] string-util/coerce-non-blank-str) + _ (check-backup-rate-limits! {:app-id app-id + :ip (posthog/extract-client-ip req)}) + job (app-backup-jobs/enqueue! {:app-id app-id + :description description})] + (response/ok {:job (app-backup-jobs/job->client-format job)}))) + +(defn app-backup-job-get [req] + (let [{{app-id :id} :app} (req->app-accepting-superadmin-or-ref-token! :collaborator + :apps/read + req) + job-id (ex/get-param! req [:params :job_id] uuid-util/coerce) + job (app-backup-jobs/get-by-id-for-client app-id job-id)] + (response/ok {:job job}))) + +(defn app-backup-jobs-get + "Lists the app's in-progress backup jobs so the dashboard can show them even + without a job id (e.g. after a page reload or from another session)." + [req] + (let [{{app-id :id} :app} (req->app-accepting-superadmin-or-ref-token! :collaborator + :apps/read + req) + jobs (app-backup-jobs/get-active-for-client app-id)] + (response/ok {:jobs jobs}))) + (defn webhook-row->response [webhook] (select-keys webhook [:id :sink :namespaces :actions :status :disabled_reason :created_at :updated_at])) @@ -2588,6 +2675,9 @@ ;; Backups (GET "/dash/apps/:app_id/backups" [] app-backups-get) + (POST "/dash/apps/:app_id/backups" [] app-backup-job-post) + (GET "/dash/apps/:app_id/backup-jobs" [] app-backup-jobs-get) + (GET "/dash/apps/:app_id/backup-jobs/:job_id" [] app-backup-job-get) (GET "/dash/apps/:app_id/backups/:backup_id/files" [] app-backup-files-get) (GET "/dash/apps/:app_id/backups/:backup_id/file-url" [] app-backup-file-url-get) (GET "/dash/apps/:app_id/backups/:backup_id/storage-files" [] app-backup-storage-files-get) diff --git a/server/src/instant/db/app_backup_jobs.clj b/server/src/instant/db/app_backup_jobs.clj new file mode 100644 index 0000000000..98491d1dad --- /dev/null +++ b/server/src/instant/db/app_backup_jobs.clj @@ -0,0 +1,399 @@ +(ns instant.db.app-backup-jobs + "On-demand, per-app backups. A user asks for a backup, we insert a row into + `app_backup_jobs`, and a resizable pool of workers on each machine claims and + runs them (streaming the app to S3 via `instant.backup`). + + Concurrency per machine is capped by the `on-demand-backup-worker-count` + flag, applied live to the thread pool. A partial unique index on + `app_backup_jobs` allows at most one in-flight job per app." + (:require + [chime.core :as chime-core] + [honey.sql :as hsql] + [instant.backup :as backup] + [instant.config :as config] + [instant.dash.ephemeral-app :refer [ephemeral-creator]] + [instant.flags :as flags] + [instant.jdbc.aurora :as aurora] + [instant.jdbc.sql :as sql] + [instant.util.async :as ua] + [instant.util.exception :as ex] + [instant.util.tracer :as tracer]) + (:import + (clojure.lang ExceptionInfo) + (java.lang AutoCloseable) + (java.time Duration Instant) + (java.util.concurrent LinkedBlockingQueue ThreadPoolExecutor ThreadPoolExecutor$DiscardPolicy TimeUnit) + (java.util.concurrent.atomic AtomicLong))) + +;; `pool` is the live worker pool (a ThreadPoolExecutor). Each task grabs and +;; runs the oldest waiting job, then submits another task so the pool keeps +;; draining the table on its own. At most its core pool size run concurrently, +;; and that size tracks the worker-count flag. `flag-unsub` unregisters the flag +;; listener on stop. +(declare pool schedule flag-unsub run-next!) + +;; Absolute ceiling on per-machine concurrency, regardless of the flag. +(def max-worker-count 32) + +;; A backup that's been `processing` this long is assumed dead (its worker +;; crashed) and gets marked errored so the app isn't stuck behind the +;; one-in-flight limit forever. +(def stuck-threshold-minutes 120) + +;; How often a running job persists its progress (triples copied) to the db. +(def progress-report-interval-ms 1000) + +(defn worker-count + "Per-machine concurrency: the flag value, clamped to [1, max-worker-count]." + [] + (-> (or (flags/on-demand-backup-worker-count) 1) + (max 1) + (min max-worker-count))) + +(defn get-by-id + ([job-id] (get-by-id (aurora/conn-pool :read) job-id)) + ([conn job-id] + (sql/select-one ::get-by-id + conn + (hsql/format {:select :* + :from :app-backup-jobs + :where [:= :id job-id]})))) + +(defn job->client-format [job] + (select-keys job [:id + :app_id + :app_backup_id + :job_status + :description + :error + :work_estimate + :work_completed + :created_at + :updated_at + :done_at])) + +(defn get-by-id-for-client + ([app-id job-id] (get-by-id-for-client (aurora/conn-pool :read) app-id job-id)) + ([conn app-id job-id] + (some-> (sql/select-one ::get-by-id-for-client + conn + (hsql/format {:select :* + :from :app-backup-jobs + :where [:and + [:= :id job-id] + [:= :app-id app-id]]})) + job->client-format))) + +(defn get-active-for-client + "In-progress backup jobs (waiting/processing) for the app, so the dashboard + can show them even without a job id (e.g. after a page reload)." + ([app-id] (get-active-for-client (aurora/conn-pool :read) app-id)) + ([conn app-id] + (->> (sql/select ::get-active-for-client + conn + (hsql/format {:select :* + :from :app-backup-jobs + :where [:and + [:= :app-id app-id] + [:in :job-status ["waiting" "processing"]]] + :order-by [[:created-at :asc]]})) + (mapv job->client-format)))) + +(defn work-estimate + "Estimates the number of triples we'll process by summing the app's + attr_sketches totals. Used only to drive the progress display." + [conn app-id] + (-> (sql/select-one ::work-estimate + conn + (hsql/format {:select [[[:coalesce [:sum :total] [:inline 0]] :estimate]] + :from :attr-sketches + :where [:= :app-id app-id]})) + :estimate + (or 0))) + +(defn in-flight-job + "The app's currently waiting/processing backup job, if any." + [conn app-id] + (sql/select-one ::in-flight-job + conn + (hsql/format {:select :id + :from :app-backup-jobs + :where [:and + [:= :app-id app-id] + [:in :job-status ["waiting" "processing"]]] + :limit 1}))) + +(defn throw-in-flight! [app-id] + (ex/throw-validation-err! :app-backup-job + {:app-id app-id} + [{:message "A backup is already in progress for this app."}])) + +;; Discord invite the dashboard points users at (mirrors +;; `client/www/lib/config.ts`). We ask large apps to reach out here instead of +;; self-serving a backup. +(def discord-invite-url "https://discord.com/invite/VU53p7uQcE") + +(defn ephemeral-app? + "True if the app was created by the ephemeral-app creator (sandbox apps that + expire on their own and aren't worth backing up)." + [conn app-id] + (= (:id @ephemeral-creator) + (:creator_id (sql/select-one ::ephemeral-app? + conn + (hsql/format {:select :creator-id + :from :apps + :where [:= :id app-id]}))))) + +(defn assert-backup-allowed! + "Refuses on-demand backups for apps that shouldn't self-serve: ephemeral apps, + and apps whose estimated triple count is at or above + `on-demand-backup-max-triples` (too large to stream on-demand--we run those + manually, so we point them to Discord). `estimate` is the app's already- + computed work-estimate, threaded through so we don't recount." + [conn app-id estimate] + (when (ephemeral-app? conn app-id) + (ex/throw-validation-err! :app-backup-job + {:app-id app-id} + [{:message "Ephemeral apps can't be backed up."}])) + (let [max-triples (flags/on-demand-backup-max-triples)] + (when (and (pos-int? max-triples) + (>= estimate max-triples)) + (ex/throw-validation-err! + :app-backup-job + {:app-id app-id} + [{:message (str "This app is too large to back up from the dashboard. " + "Reach out in Discord and we'll run the backup for you: " + discord-invite-url)}])))) + +(defn create-job! + ([params] (create-job! (aurora/conn-pool :write) params)) + ([conn {:keys [app-id description]}] + (let [estimate (work-estimate conn app-id)] + (assert-backup-allowed! conn app-id estimate) + ;; Cheap check for the common case; the partial unique index below is the + ;; race-safe backstop. + (when (in-flight-job conn app-id) + (throw-in-flight! app-id)) + (try + (sql/execute-one! ::create-job! + conn + (hsql/format {:insert-into :app-backup-jobs + :values [{:id (random-uuid) + :app-id app-id + :description description + :work-estimate estimate + :job-status "waiting"}] + :returning :*})) + (catch ExceptionInfo e + (if (= ::ex/record-not-unique (::ex/type (ex-data e))) + (throw-in-flight! app-id) + (throw e))))))) + +(defn submit! + "Nudges the pool to grab and run the next waiting job. `.execute` uses the + pool's bounded queue; when it's full the task is silently discarded + (DiscardPolicy), which is fine because enough grab-tasks are already pending. + No-op if the pool isn't running (the schedule kick will pick things up)." + [] + (when (bound? #'pool) + (.execute ^ThreadPoolExecutor pool ^Runnable run-next!))) + +(defn enqueue! + "Creates a backup job for the app and nudges the pool to run it. Returns the + job row. Throws a validation error if the app already has a backup in + flight." + [{:keys [app-id description]}] + (let [job (create-job! {:app-id app-id + :description description})] + (submit!) + job)) + +(defn grab-oldest-job! + "Atomically claims the oldest waiting job for this process. Returns the + claimed row, or nil if there's nothing to do. Concurrent workers skip each + other's locked rows, so each claims a distinct job." + ([] (grab-oldest-job! (aurora/conn-pool :write))) + ([conn] + (sql/execute-one! ::grab-oldest-job! + conn + (hsql/format {:update :app-backup-jobs + :set {:worker-id @config/process-id + :job-status "processing"} + :where [:= :id {:select :id + :from :app-backup-jobs + :where [:and + [:= :worker-id nil] + [:= :job-status "waiting"]] + :order-by [[:created-at :asc]] + :limit 1 + :for [:update :skip-locked]}] + :returning :*})))) + +(defn mark-completed! [conn job-id {:keys [app-backup-id work-completed]}] + (sql/execute-one! ::mark-completed! + conn + (hsql/format {:update :app-backup-jobs + :set {:job-status "completed" + :app-backup-id app-backup-id + :work-completed work-completed + :done-at :%now} + :where [:= :id job-id]}))) + +(defn mark-error! [conn job-id ^Throwable t] + (sql/execute-one! ::mark-error! + conn + (hsql/format {:update :app-backup-jobs + :set {:job-status "errored" + :error (.getMessage t) + :done-at :%now} + :where [:= :id job-id]}))) + +(defn set-work-completed! [conn job-id n] + (sql/execute-one! ::set-work-completed! + conn + (hsql/format {:update :app-backup-jobs + :set {:work-completed n} + :where [:= :id job-id]}))) + +(defn run-job! + "Runs the backup for a claimed job and records the outcome. The backup runs on + a background vthread while this thread waits; between waits it persists the + copied-triple count so the dashboard can show live progress. The copy loop + only bumps `copied` (via the `on-triple` callback), so it never blocks on the + db." + [{:keys [id app_id description]}] + (tracer/with-span! {:name "app-backup-jobs/run-job" + :attributes {:job-id id + :app-id app_id}} + (let [copied (AtomicLong. 0) + expires-at (.plus (Instant/now) + (Duration/ofDays (flags/on-demand-backup-expiry-days))) + fut (ua/vfuture + (backup/backup-app-on-primary! {:app-id app_id + :description (or description "On-demand backup") + :expires-at expires-at + :on-triple (fn [] (.incrementAndGet copied))}))] + (try + ;; `last-written` avoids redundant db writes when no triples were copied + ;; since the previous tick (e.g. the copy has finished). + (loop [last-written 0] + (let [result (deref fut progress-report-interval-ms ::pending)] + (if (identical? ::pending result) + (let [n (.get copied)] + (if (= n last-written) + (recur last-written) + (do (set-work-completed! (aurora/conn-pool :write) id n) + (recur n)))) + (let [{:keys [triple-count app-backup]} result] + (mark-completed! (aurora/conn-pool :write) + id + {:app-backup-id (:id app-backup) + :work-completed triple-count}) + (tracer/add-data! {:attributes {:triple-count triple-count + :app-backup-id (:id app-backup)}}))))) + (catch Throwable t + (tracer/record-exception-span! t {:name "app-backup-jobs/run-job-error" + :escaping? false}) + (mark-error! (aurora/conn-pool :write) id t)))))) + +(defn run-next! + "Pool task: claim the oldest waiting job and run it. If it claimed one, submit + another task so the pool keeps draining the table until it's empty. The chain + ends when a task finds no waiting job." + [] + (tracer/with-span! {:name "app-backup-jobs/run-next"} + (try + (when-let [job (grab-oldest-job!)] + (run-job! job) + (submit!)) + (catch Throwable t + (tracer/record-exception-span! t {:name "app-backup-jobs/run-next-error" + :escaping? false}))))) + +(defn kick-workers! + "Submits enough tasks to get idle workers draining the table, e.g. after a + restart. Self-continuation keeps them going from there." + [] + (dotimes [_ (worker-count)] + (submit!))) + +(defn reclaim-stuck-jobs! + "Marks `processing` jobs with no progress for a while as errored, on the + assumption their worker died. Running jobs bump `updated_at` as they copy, so + a long-but-progressing backup won't be reclaimed. Releases the app from the + one-in-flight limit so it can retry; any partial S3 uploads expire on their + own via the `expire` tag." + ([] (reclaim-stuck-jobs! (aurora/conn-pool :write))) + ([conn] + (tracer/with-span! {:name "app-backup-jobs/reclaim-stuck-jobs!"} + (let [res (sql/do-execute! ::reclaim-stuck-jobs! + conn + (hsql/format {:update :app-backup-jobs + :set {:job-status "errored" + :error "Backup timed out" + :done-at :%now} + :where [:and + [:= :job-status "processing"] + [:< [:interval [:inline (format "%d minutes" + stuck-threshold-minutes)]] + [:- :%now :updated-at]]]})) + reclaimed (:next.jdbc/update-count (first res))] + (tracer/add-data! {:attributes {:reclaimed-count reclaimed}}) + reclaimed)))) + +(defn make-pool ^ThreadPoolExecutor [n] + ;; runs n tasks at a time. The queue is bounded at max-worker-count + ;; and DiscardPolicy drops `.execute` when it's full + ;; `allowCoreThreadTimeOut` lets idle threads die so shrinking + ;; the pool actually releases them. + (doto (ThreadPoolExecutor. (int n) + (int max-worker-count) + 60 TimeUnit/SECONDS + (LinkedBlockingQueue. (int max-worker-count)) + (ThreadPoolExecutor$DiscardPolicy.)) + (.allowCoreThreadTimeOut true))) + +(defn resize-pool! [n] + (when (bound? #'pool) + ;; setCorePoolSize starts threads for queued work when growing and reaps + ;; idle threads when shrinking. + (.setCorePoolSize ^ThreadPoolExecutor pool n))) + +(defn start [] + (tracer/record-info! {:name "app-backup-jobs/start"}) + (def pool (make-pool (worker-count))) + + ;; Apply worker-count flag changes to the live pool without a restart. + (def flag-unsub (flags/add-flag-listener + :on-demand-backup-worker-count + (fn [_path _old _new] + (resize-pool! (worker-count))))) + + ;; Pick up any jobs left waiting from before this process started. + (kick-workers!) + + (def schedule (chime-core/chime-at + (chime-core/periodic-seq (Instant/now) + (Duration/ofMinutes 10)) + (fn [_time] + (kick-workers!) + (reclaim-stuck-jobs!))))) + +(defn stop [] + (when (bound? #'schedule) + (.close ^AutoCloseable schedule)) + (when (and (bound? #'flag-unsub) flag-unsub) + (flag-unsub)) + (when (bound? #'pool) + (.shutdown ^ThreadPoolExecutor pool) + (.awaitTermination ^ThreadPoolExecutor pool 5 TimeUnit/MINUTES))) + +(defn restart [] + (stop) + (start)) + +(defn before-ns-unload [] + (stop)) + +(defn after-ns-reload [] + (start)) diff --git a/server/src/instant/flags.clj b/server/src/instant/flags.clj index 842f8ae2f3..f6e2d6ef1e 100644 --- a/server/src/instant/flags.clj +++ b/server/src/instant/flags.clj @@ -524,6 +524,42 @@ [] (flag :backup-skip-app-ids #{})) +(defn on-demand-backup-worker-count + "How many on-demand backups run concurrently per machine." + [] + (flag :on-demand-backup-worker-count 3)) + +(defn on-demand-backup-expiry-days + "How long an on-demand backup is retained before its S3 objects expire. + Hard-capped at 32 by the storage `expire` tag rule." + [] + (flag :on-demand-backup-expiry-days 30)) + +(defn on-demand-backup-max-triples + "Apps whose estimated triple count is at or above this can't run a self-serve + backup--they're too large to stream on-demand, so we route them to us in + Discord. A nil or non-positive value disables the ceiling." + [] + (flag :on-demand-backup-max-triples 25000000)) + +(defn on-demand-backup-app-rate-limit + "Per-app throttle for on-demand backups, as {:capacity n :window-minutes m} + (default 1 per 5 minutes). Set the flag to a JSON object like + {\"capacity\": 1, \"windowMinutes\": 5} to tune without a deploy." + [] + (let [v (flag :on-demand-backup-app-rate-limit)] + {:capacity (get v "capacity" 1) + :window-minutes (get v "windowMinutes" 5)})) + +(defn on-demand-backup-ip-rate-limit + "Per-IP throttle for on-demand backups, as {:capacity n :window-minutes m} + (default 2 per 5 minutes). Set the flag to a JSON object like + {\"capacity\": 2, \"windowMinutes\": 5} to tune without a deploy." + [] + (let [v (flag :on-demand-backup-ip-rate-limit)] + {:capacity (get v "capacity" 2) + :window-minutes (get v "windowMinutes" 5)})) + (defn use-cloudfront-signed-url? [app-id] (when-not (toggled? :disable-cloudfront-signed-urls-globally) (or (toggled? :enable-cloudfront-signed-urls-globally) diff --git a/server/src/instant/rate_limit.clj b/server/src/instant/rate_limit.clj index 2a358fb3fd..b4e5311dec 100644 --- a/server/src/instant/rate_limit.clj +++ b/server/src/instant/rate_limit.clj @@ -339,6 +339,20 @@ (.getNanosToWaitForRefill remaining)) (.getRemainingTokens remaining))))) +(defn consume-user-rate-limit-retry-at + "Like `consume-user-rate-limit`, but instead of throwing on exhaustion it + returns nil when the tokens were consumed, or the `Instant` at which the + caller may retry when the bucket is empty. Lets the caller craft its own + rate-limit message." + [{:keys [get-bucket-with-config]} + {:keys [app-id config tokens bucket-key bucket-name] + :or {tokens 1}}] + (let [key (user-key-hash app-id bucket-name config bucket-key) + ^Bucket bucket (get-bucket-with-config key (make-bucket-config-fn config)) + remaining (.tryConsumeAndReturnRemaining bucket tokens)] + (when-not (.isConsumed remaining) + (.plusNanos (Instant/now) (.getNanosToWaitForRefill remaining))))) + (defonce schedule (atom nil)) (def sweep-q (uhsql/formatp diff --git a/server/src/instant/util/exception.clj b/server/src/instant/util/exception.clj index ce20456026..8a0a37626f 100644 --- a/server/src/instant/util/exception.clj +++ b/server/src/instant/util/exception.clj @@ -431,18 +431,34 @@ (throw+ {::type ::rate-limited ::message "Too many verification codes requested for this email. Please try again later."})) +(defn retry-after-seconds + "Whole seconds until `retry-at`, rounded up and floored at 0. Suitable for a + Retry-After hint." + [retry-at] + (-> (Duration/between (Instant/now) retry-at) + (.toMillis) + (/ 1000) + (Math/ceil) + (long) + (max 0))) + (defn throw-permission-rate-limited! [retry-at remaining-tokens] (throw+ {::type ::rate-limited ::message "Your request exceeded the rate limit." ::hint {:retry-at retry-at - :retry-after (-> (Duration/between (Instant/now) retry-at) - (.toMillis) - (/ 1000) - (Math/ceil) - (long) - (max 0)) + :retry-after (retry-after-seconds retry-at) :remaining-tokens remaining-tokens}})) +(defn throw-rate-limited-until! + "Rate-limited error with a caller-supplied message and a retry-at instant so + the client knows when it can try again (both as a human-readable retry time + in the message and a machine-readable `retry-after` in the hint)." + [message retry-at] + (throw+ {::type ::rate-limited + ::message message + ::hint {:retry-at retry-at + :retry-after (retry-after-seconds retry-at)}})) + ;; ------- ;; Sockets From 323e3884c41164fe927f7fb78e834b2037fae2ed Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 5 Aug 2026 15:33:19 -0700 Subject: [PATCH 02/27] add delete and cancel --- client/www/components/dash/Backups.tsx | 261 +++++++++++++++--- client/www/lib/types.ts | 2 +- server/src/instant/backup.clj | 28 +- server/src/instant/dash/routes.clj | 30 ++ server/src/instant/db/app_backup_jobs.clj | 85 ++++-- .../test/instant/db/app_backup_jobs_test.clj | 116 ++++++++ 6 files changed, 461 insertions(+), 61 deletions(-) create mode 100644 server/test/instant/db/app_backup_jobs_test.clj diff --git a/client/www/components/dash/Backups.tsx b/client/www/components/dash/Backups.tsx index 1e59a8f834..1099ae139e 100644 --- a/client/www/components/dash/Backups.tsx +++ b/client/www/components/dash/Backups.tsx @@ -7,7 +7,12 @@ import { useRef, useState, } from 'react'; -import { ArrowDownTrayIcon, PlusIcon } from '@heroicons/react/24/outline'; +import { + ArrowDownTrayIcon, + EllipsisVerticalIcon, + PlusIcon, + TrashIcon, +} from '@heroicons/react/24/outline'; import { Transition } from '@headlessui/react'; import config from '@/lib/config'; @@ -30,6 +35,12 @@ import { TooltipTrigger, useDialog, } from '@/components/ui'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from '@/components/DropdownMenu'; import { ErrorMessage, Loading, @@ -59,6 +70,13 @@ function fetchBackupJob(token: string, appId: string, jobId: string) { }) as Promise<{ job: InstantAppBackupJob | null }>; } +function cancelBackupJob(token: string, appId: string, jobId: string) { + return jsonMutate<{ id: string }>( + `${config.apiURI}/dash/apps/${appId}/backup-jobs/${jobId}`, + { token, method: 'DELETE' }, + ); +} + // Shared layout for both in-progress jobs and finished backups. Rendering them // through one component with the same structure (title, optional description, a // two-row detail grid, and an action) keeps the row the exact same size, so it @@ -68,17 +86,20 @@ function BackupItem({ description, rows, action, + corner, }: { title: string; description?: string | null; rows: [string, ReactNode][]; action: ReactNode; + corner?: ReactNode; }) { return ( + {corner ?
{corner}
: null} {title} {description ? {description} : null} @@ -105,7 +126,18 @@ function ProgressBar({ pct }: { pct: number }) { ); } -function BackupJobRow({ job }: { job: InstantAppBackupJob }) { +function BackupJobRow({ + app, + job, + onCancelled, +}: { + app: InstantApp; + job: InstantAppBackupJob; + onCancelled: () => void; +}) { + const token = useContext(TokenContext); + const confirmDialog = useDialog(); + const [cancelling, setCancelling] = useState(false); const completed = job.work_completed ?? 0; // Pad the estimate by 10% so the bar doesn't sit at 100% while the backup // finishes uploading and finalizing. @@ -116,18 +148,74 @@ function BackupJobRow({ job }: { job: InstantAppBackupJob }) { : null; const label = job.job_status === 'waiting' ? 'Starting…' : 'Backing up…'; + async function onCancel() { + if (!token) return; + setCancelling(true); + try { + await cancelBackupJob(token, app.id, job.id); + successToast('Backup cancelled.'); + confirmDialog.onClose(); + // The server drops the job from the active list immediately, so refetch to + // clear the row (a running worker aborts at its next progress checkpoint). + onCancelled(); + } catch (e: any) { + errorToast(e?.body?.message ?? 'Failed to cancel backup.'); + setCancelling(false); + } + } + return ( ], + ['Status', cancelling ? 'Cancelling…' : label], + [ + 'Progress', +
+ + {pct !== null ? {pct}% : null} +
, + ], ]} action={ - - {pct !== null ? `${pct}%` : ''} - + <> + + +
+ Cancel backup + + Cancel this backup? Progress so far will be discarded and no + snapshot will be saved. + +
+ + +
+
+ + } /> ); @@ -136,12 +224,45 @@ function BackupJobRow({ job }: { job: InstantAppBackupJob }) { function BackupRow({ app, backup, + onDeleted, }: { app: InstantApp; backup: InstantAppBackup; + onDeleted: () => void; }) { const token = useContext(TokenContext); const { open, active } = useBackupDownloads(); + const deleteDialog = useDialog(); + const [deleting, setDeleting] = useState(false); + const [deleteError, setDeleteError] = useState(null); + // Deleting is admin-only (the server enforces this too); hide the affordance + // for everyone else. + const canDelete = + app.user_app_role === 'admin' || app.user_app_role === 'owner'; + + function openDeleteDialog() { + setDeleteError(null); + deleteDialog.onOpen(); + } + + async function deleteBackup() { + if (!token) return; + setDeleting(true); + setDeleteError(null); + try { + await jsonMutate( + `${config.apiURI}/dash/apps/${app.id}/backups/${backup.id}`, + { token, method: 'DELETE' }, + ); + deleteDialog.onClose(); + successToast('Backup deleted.'); + onDeleted(); + } catch (e: any) { + setDeleteError(e?.body?.message ?? 'Failed to delete backup.'); + } finally { + setDeleting(false); + } + } const isActive = active?.id === backup.id; // Only one download at a time: while another backup is actively downloading, @@ -171,37 +292,87 @@ function BackupRow({ ); + // Hidden until the row is hovered (or the menu is open/focused), so it stays + // out of the way until you go looking for it. + const actionsMenu = ( + + + + + + + Delete backup + + + + ); + return ( - , - ], - ]} - action={ - blockedByOther ? ( - - - - {downloadButton} - - - - You can only download one backup at a time. - - - ) : ( - downloadButton - ) - } - /> + <> + , + ], + ]} + action={ + blockedByOther ? ( + + + + {downloadButton} + + + + You can only download one backup at a time. + + + ) : ( + downloadButton + ) + } + corner={canDelete ? actionsMenu : null} + /> + +
+ Delete backup + + This removes the backup from your dashboard and can't be undone. + + {deleteError ? ( +
+ {deleteError} +
+ ) : null} +
+ + +
+
+ + ); } @@ -350,10 +521,20 @@ export function Backups({ app }: { app: InstantApp }) { ) : ( {jobs.map((job) => ( - + jobsRes.mutate()} + /> ))} {backups.map((b) => ( - + backupsRes.mutate()} + /> ))} )} diff --git a/client/www/lib/types.ts b/client/www/lib/types.ts index 385338534b..9a463eb3fc 100644 --- a/client/www/lib/types.ts +++ b/client/www/lib/types.ts @@ -105,7 +105,7 @@ export type InstantAppBackupJob = { id: string; app_id: string; app_backup_id: string | null; - job_status: 'waiting' | 'processing' | 'completed' | 'errored'; + job_status: 'waiting' | 'processing' | 'completed' | 'errored' | 'cancelled'; description: string | null; error: string | null; work_estimate: number | null; diff --git a/server/src/instant/backup.clj b/server/src/instant/backup.clj index 2e55564646..30a5c19837 100644 --- a/server/src/instant/backup.clj +++ b/server/src/instant/backup.clj @@ -134,7 +134,8 @@ :from :app-backups :where [:and [:= :app-id :?app-id] - [:> :expires-at :%now]] + [:> :expires-at :%now] + [:= nil :deletion-marked-at]] :order-by [[:backup-at :desc]]})) (defn get-app-backups-by-app-id @@ -150,7 +151,8 @@ :from :app-backups :where [:and [:= :id :?id] - [:= :app-id :?app-id]]})) + [:= :app-id :?app-id] + [:= nil :deletion-marked-at]]})) (defn get-app-backup-by-id ([params] (get-app-backup-by-id (aurora/conn-pool :read) params)) @@ -161,6 +163,28 @@ {:id id :app-id app-id})))) +(def mark-app-backup-deleted-q + (uhsql/preformat {:update :app-backups + :set {:deletion-marked-at :%now} + :where [:and + [:= :id :?id] + [:= :app-id :?app-id] + [:= nil :deletion-marked-at]] + :returning :id})) + +(defn mark-app-backup-deleted! + "Soft-deletes a backup by stamping `deletion_marked_at`. The row and its S3 + objects stay put--the objects expire on their own via their `expire` tag. + Returns the row when it flips a live backup, or nil if there was no matching + un-deleted backup." + ([params] (mark-app-backup-deleted! (aurora/conn-pool :write) params)) + ([conn {:keys [id app-id]}] + (sql/execute-one! ::mark-app-backup-deleted! + conn + (uhsql/formatp mark-app-backup-deleted-q + {:id id + :app-id app-id})))) + ;; Make sure the order of the columns matches the order of the record (defrecord-once Triple [app-id entity-id value created-at etype label many]) (def columns [{:name :app-id diff --git a/server/src/instant/dash/routes.clj b/server/src/instant/dash/routes.clj index f4b4944a86..e6d723c8fa 100644 --- a/server/src/instant/dash/routes.clj +++ b/server/src/instant/dash/routes.clj @@ -2028,6 +2028,34 @@ jobs (app-backup-jobs/get-active-for-client app-id)] (response/ok {:jobs jobs}))) +(defn app-backup-job-cancel + "Cancels an in-progress backup job. A waiting job never runs; a processing + job's worker notices at its next progress checkpoint and aborts the backup. + Whoever can start a backup can cancel one." + [req] + (let [{{app-id :id} :app} (req->app-accepting-superadmin-or-ref-token! :collaborator + :apps/write + req) + job-id (ex/get-param! req [:params :job_id] uuid-util/coerce)] + (ex/assert-record! (app-backup-jobs/cancel-job! app-id job-id) + :app-backup-job + {:id job-id}) + (response/ok {:id job-id}))) + +(defn app-backup-delete + "Soft-deletes a backup (admins only). The row is marked deleted, not removed; + its S3 objects expire on their own." + [req] + (let [{{app-id :id} :app} (req->app-accepting-superadmin-or-ref-token! :admin + :apps/write + req) + backup-id (ex/get-param! req [:params :backup_id] uuid-util/coerce)] + (ex/assert-record! (backup/mark-app-backup-deleted! {:id backup-id + :app-id app-id}) + :app-backup + {:id backup-id}) + (response/ok {:id backup-id}))) + (defn webhook-row->response [webhook] (select-keys webhook [:id :sink :namespaces :actions :status :disabled_reason :created_at :updated_at])) @@ -2676,8 +2704,10 @@ ;; Backups (GET "/dash/apps/:app_id/backups" [] app-backups-get) (POST "/dash/apps/:app_id/backups" [] app-backup-job-post) + (DELETE "/dash/apps/:app_id/backups/:backup_id" [] app-backup-delete) (GET "/dash/apps/:app_id/backup-jobs" [] app-backup-jobs-get) (GET "/dash/apps/:app_id/backup-jobs/:job_id" [] app-backup-job-get) + (DELETE "/dash/apps/:app_id/backup-jobs/:job_id" [] app-backup-job-cancel) (GET "/dash/apps/:app_id/backups/:backup_id/files" [] app-backup-files-get) (GET "/dash/apps/:app_id/backups/:backup_id/file-url" [] app-backup-file-url-get) (GET "/dash/apps/:app_id/backups/:backup_id/storage-files" [] app-backup-storage-files-get) diff --git a/server/src/instant/db/app_backup_jobs.clj b/server/src/instant/db/app_backup_jobs.clj index 98491d1dad..614b8352dc 100644 --- a/server/src/instant/db/app_backup_jobs.clj +++ b/server/src/instant/db/app_backup_jobs.clj @@ -229,6 +229,21 @@ :for [:update :skip-locked]}] :returning :*})))) +;; A worker only owns its job while `worker_id` still points at it and the job is +;; still `processing`. Every write it makes is scoped to that "still mine and +;; still running" predicate, so a cancel (which clears `worker_id` and flips the +;; status, see `cancel-job!`) or a stuck-reclaim (which flips the status) makes +;; the write match nothing--we don't resurrect a cancelled/errored job, and the +;; running worker learns it's been stopped. +(defn owned-and-processing + "Where-clause matching `job-id` only while it's still claimed by this process + and still `processing`." + [job-id] + [:and + [:= :id job-id] + [:= :worker-id @config/process-id] + [:= :job-status "processing"]]) + (defn mark-completed! [conn job-id {:keys [app-backup-id work-completed]}] (sql/execute-one! ::mark-completed! conn @@ -237,7 +252,7 @@ :app-backup-id app-backup-id :work-completed work-completed :done-at :%now} - :where [:= :id job-id]}))) + :where (owned-and-processing job-id)}))) (defn mark-error! [conn job-id ^Throwable t] (sql/execute-one! ::mark-error! @@ -246,21 +261,55 @@ :set {:job-status "errored" :error (.getMessage t) :done-at :%now} - :where [:= :id job-id]}))) - -(defn set-work-completed! [conn job-id n] - (sql/execute-one! ::set-work-completed! - conn - (hsql/format {:update :app-backup-jobs - :set {:work-completed n} - :where [:= :id job-id]}))) + :where (owned-and-processing job-id)}))) + +(defn report-progress! + "Persists the copied-triple count for a running job and doubles as a + cancellation checkpoint. The update only touches the row while it's still ours + and still `processing`, so it matches nothing once the job has been cancelled + (its `worker_id` cleared and status flipped) or reclaimed. Returns true while + the job is still ours to run, false once it's been taken from us." + [conn job-id n] + (let [res (sql/do-execute! ::report-progress! + conn + (hsql/format {:update :app-backup-jobs + :set {:work-completed n} + :where (owned-and-processing job-id)}))] + (pos? (:next.jdbc/update-count (first res))))) + +(defn cancel-job! + "Marks a waiting/processing backup job as cancelled so its worker stops. + Clearing `worker_id` (along with flipping the status) is what the worker's + `report-progress!` checkpoint detects: its next scoped write no longer matches + the row, so it cancels the in-flight backup and bails. A waiting job simply + never gets claimed. Scoped to the app so a caller can only cancel its own + app's job. Returns the updated row, or nil when there was no in-flight job + with that id (already done/cancelled)." + ([app-id job-id] (cancel-job! (aurora/conn-pool :write) app-id job-id)) + ([conn app-id job-id] + (sql/execute-one! ::cancel-job! + conn + (hsql/format {:update :app-backup-jobs + :set {:job-status "cancelled" + :worker-id nil + :done-at :%now} + :where [:and + [:= :id job-id] + [:= :app-id app-id] + [:in :job-status ["waiting" "processing"]]] + :returning :*})))) (defn run-job! "Runs the backup for a claimed job and records the outcome. The backup runs on a background vthread while this thread waits; between waits it persists the copied-triple count so the dashboard can show live progress. The copy loop only bumps `copied` (via the `on-triple` callback), so it never blocks on the - db." + db. + + Each progress tick doubles as a cancellation checkpoint: `report-progress!` + only touches the row while it's still ours and still `processing`, so once the + job has been cancelled (or reclaimed) the write matches nothing and we cancel + the in-flight backup and stop, leaving the cancelled state in place." [{:keys [id app_id description]}] (tracer/with-span! {:name "app-backup-jobs/run-job" :attributes {:job-id id @@ -274,16 +323,16 @@ :expires-at expires-at :on-triple (fn [] (.incrementAndGet copied))}))] (try - ;; `last-written` avoids redundant db writes when no triples were copied - ;; since the previous tick (e.g. the copy has finished). - (loop [last-written 0] + (loop [] (let [result (deref fut progress-report-interval-ms ::pending)] (if (identical? ::pending result) - (let [n (.get copied)] - (if (= n last-written) - (recur last-written) - (do (set-work-completed! (aurora/conn-pool :write) id n) - (recur n)))) + ;; Still running. Persist progress and, in the same write, confirm + ;; the job is still ours. If it isn't, it's been cancelled out from + ;; under us, so kill the backup and bail without touching the row. + (if (report-progress! (aurora/conn-pool :write) id (.get copied)) + (recur) + (do (future-cancel fut) + (tracer/add-data! {:attributes {:cancelled true}}))) (let [{:keys [triple-count app-backup]} result] (mark-completed! (aurora/conn-pool :write) id diff --git a/server/test/instant/db/app_backup_jobs_test.clj b/server/test/instant/db/app_backup_jobs_test.clj new file mode 100644 index 0000000000..ed6855b469 --- /dev/null +++ b/server/test/instant/db/app_backup_jobs_test.clj @@ -0,0 +1,116 @@ +(ns instant.db.app-backup-jobs-test + (:require + [clojure.test :refer [deftest is testing]] + [instant.backup :as backup] + [instant.data.constants :refer [test-user-id]] + [instant.db.app-backup-jobs :as app-backup-jobs] + [instant.model.app :as app-model] + [instant.util.test :refer [instant-ex-data wait-for]])) + +(def ^:private wait-timeout 10000) + +(defn- create-app! [] + (let [id (random-uuid)] + (app-model/create! {:title "backup-jobs-test-app" + :creator-id test-user-id + :id id + :admin-token (random-uuid)}))) + +(defn- with-apps + "Creates `n` throwaway apps, passes the vector to `f`, and deletes them after + (which cascades away their backup-job rows)." + [n f] + (let [apps (vec (repeatedly n create-app!))] + (try + (f apps) + (finally + (doseq [{:keys [id]} apps] + (app-model/delete-immediately-by-id! {:id id})))))) + +(defn- job-status [job] + (:job_status (app-backup-jobs/get-by-id (:id job)))) + +(deftest runs-at-most-n-backups-at-a-time + (let [n 2 + total 5 + ;; A fresh pool sized `n` so we get a clean concurrency cap that doesn't + ;; depend on the flag value the shared pool happened to start with. + pool (app-backup-jobs/make-pool n) + running (atom 0) + max-running (atom 0) + ;; The workers block here until we let them finish, so we can observe + ;; exactly how many run at once. + release (promise)] + (with-redefs [app-backup-jobs/pool pool + ;; Don't actually run the backup--just record concurrency and + ;; block until the test releases us. + backup/backup-app-on-primary! + (fn [_params] + (let [cur (swap! running inc)] + (swap! max-running max cur) + (try + @release + (finally + (swap! running dec))) + ;; `app-backup` is nil so `run-job!` writes a null + ;; `app_backup_id`; a fake uuid would trip the FK to + ;; `app_backups` and error the job instead of completing it. + {:triple-count 0 + :app-backup nil}))] + (with-apps total + (fn [apps] + (try + (let [jobs (mapv (fn [app] + (app-backup-jobs/enqueue! {:app-id (:id app) + :description "test backup"})) + apps)] + ;; Wait until the pool has spun up all the workers it's allowed to. + (wait-for #(= n @running) wait-timeout) + + (testing "only n workers run concurrently" + (is (= n @running)) + (is (= n @max-running))) + + (testing "the db reflects n processing and the rest still waiting" + (let [freq (frequencies (map job-status jobs))] + (is (= n (get freq "processing" 0))) + (is (= (- total n) (get freq "waiting" 0))))) + + ;; Let the blocked workers finish; the pool should drain the rest. + (deliver release true) + + (testing "every job completes once the workers are unblocked" + (wait-for (fn [] + (every? #(= "completed" (job-status %)) jobs)) + wait-timeout) + (is (every? #(= "completed" (job-status %)) jobs))) + + (testing "we never exceeded n concurrent backups" + (is (= n @max-running)))) + (finally + ;; Make sure nothing stays blocked if an assertion threw. + (deliver release true) + (.shutdown pool)))))))) + +(deftest rejects-a-second-in-flight-backup-for-the-same-app + (let [pool (app-backup-jobs/make-pool 1) + release (promise)] + (with-redefs [app-backup-jobs/pool pool + backup/backup-app-on-primary! + (fn [_params] + @release + {:triple-count 0 + :app-backup nil})] + (with-apps 1 + (fn [[app]] + (try + (app-backup-jobs/enqueue! {:app-id (:id app)}) + (let [err (instant-ex-data + (app-backup-jobs/enqueue! {:app-id (:id app)}))] + (is (= :instant.util.exception/validation-failed + (:instant.util.exception/type err))) + (is (re-find #"already in progress" + (:instant.util.exception/message err)))) + (finally + (deliver release true) + (.shutdown pool)))))))) From 35a27895dcdf248186caf40fe8d24a49a36630c9 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 5 Aug 2026 15:46:42 -0700 Subject: [PATCH 03/27] fail earlier --- server/src/instant/dash/routes.clj | 4 +++ server/src/instant/db/app_backup_jobs.clj | 30 ++++++++++++----------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/server/src/instant/dash/routes.clj b/server/src/instant/dash/routes.clj index e6d723c8fa..cdca955bd5 100644 --- a/server/src/instant/dash/routes.clj +++ b/server/src/instant/dash/routes.clj @@ -2004,6 +2004,10 @@ :apps/write req) description (ex/get-optional-param! req [:body :description] string-util/coerce-non-blank-str) + ;; Reject too-large (and ephemeral) apps before spending the caller's + ;; rate-limit budget, so they get the "too big" message rather than a + ;; rate-limit error. + _ (app-backup-jobs/assert-backup-allowed! app-id) _ (check-backup-rate-limits! {:app-id app-id :ip (posthog/extract-client-ip req)}) job (app-backup-jobs/enqueue! {:app-id app-id diff --git a/server/src/instant/db/app_backup_jobs.clj b/server/src/instant/db/app_backup_jobs.clj index 614b8352dc..ae78f6501e 100644 --- a/server/src/instant/db/app_backup_jobs.clj +++ b/server/src/instant/db/app_backup_jobs.clj @@ -150,20 +150,22 @@ `on-demand-backup-max-triples` (too large to stream on-demand--we run those manually, so we point them to Discord). `estimate` is the app's already- computed work-estimate, threaded through so we don't recount." - [conn app-id estimate] - (when (ephemeral-app? conn app-id) - (ex/throw-validation-err! :app-backup-job - {:app-id app-id} - [{:message "Ephemeral apps can't be backed up."}])) - (let [max-triples (flags/on-demand-backup-max-triples)] - (when (and (pos-int? max-triples) - (>= estimate max-triples)) - (ex/throw-validation-err! - :app-backup-job - {:app-id app-id} - [{:message (str "This app is too large to back up from the dashboard. " - "Reach out in Discord and we'll run the backup for you: " - discord-invite-url)}])))) + ([app-id] (assert-backup-allowed! (aurora/conn-pool :read) app-id)) + ([conn app-id] (assert-backup-allowed! conn app-id (work-estimate conn app-id))) + ([conn app-id estimate] + (when (ephemeral-app? conn app-id) + (ex/throw-validation-err! :app-backup-job + {:app-id app-id} + [{:message "Ephemeral apps can't be backed up."}])) + (let [max-triples (flags/on-demand-backup-max-triples)] + (when (and (pos-int? max-triples) + (>= estimate max-triples)) + (ex/throw-validation-err! + :app-backup-job + {:app-id app-id} + [{:message (str "This app is too large to back up from the dashboard. " + "Reach out in Discord and we'll run the backup for you: " + discord-invite-url)}]))))) (defn create-job! ([params] (create-job! (aurora/conn-pool :write) params)) From b4ae99fd45d0689d5b20b647b2f14b5c8bb59aa3 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 5 Aug 2026 15:51:19 -0700 Subject: [PATCH 04/27] missing migration --- server/resources/migrations/122_app_backups_deletion.down.sql | 1 + server/resources/migrations/122_app_backups_deletion.up.sql | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 server/resources/migrations/122_app_backups_deletion.down.sql create mode 100644 server/resources/migrations/122_app_backups_deletion.up.sql diff --git a/server/resources/migrations/122_app_backups_deletion.down.sql b/server/resources/migrations/122_app_backups_deletion.down.sql new file mode 100644 index 0000000000..1af87f5c03 --- /dev/null +++ b/server/resources/migrations/122_app_backups_deletion.down.sql @@ -0,0 +1 @@ +alter table app_backups drop column deletion_marked_at; diff --git a/server/resources/migrations/122_app_backups_deletion.up.sql b/server/resources/migrations/122_app_backups_deletion.up.sql new file mode 100644 index 0000000000..a432c023bc --- /dev/null +++ b/server/resources/migrations/122_app_backups_deletion.up.sql @@ -0,0 +1,4 @@ +-- Soft-delete for on-demand backups: the dashboard "delete" marks this instead +-- of removing the row. The backup's S3 objects are left to expire on their own +-- via their `expire` tag. +alter table app_backups add column deletion_marked_at timestamptz; From c088dfdee20aaf8a25507cc06bcdb799e4f45293 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 5 Aug 2026 15:57:53 -0700 Subject: [PATCH 05/27] don't expose raw error to client --- client/www/components/dash/Backups.tsx | 2 +- client/www/lib/types.ts | 1 - server/src/instant/db/app_backup_jobs.clj | 4 +++- server/src/instant/flags.clj | 5 +++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/client/www/components/dash/Backups.tsx b/client/www/components/dash/Backups.tsx index 1099ae139e..e17b7d2d9f 100644 --- a/client/www/components/dash/Backups.tsx +++ b/client/www/components/dash/Backups.tsx @@ -434,7 +434,7 @@ export function Backups({ app }: { app: InstantApp }) { if (job?.job_status === 'completed') { successToast('Backup finished.'); } else if (job?.job_status === 'errored') { - errorToast(job.error ?? 'Backup failed.', { autoClose: 8000 }); + errorToast('Backup failed.', { autoClose: 8000 }); } } catch { // Best-effort alert; ignore lookup failures. diff --git a/client/www/lib/types.ts b/client/www/lib/types.ts index 9a463eb3fc..37388f0cc4 100644 --- a/client/www/lib/types.ts +++ b/client/www/lib/types.ts @@ -107,7 +107,6 @@ export type InstantAppBackupJob = { app_backup_id: string | null; job_status: 'waiting' | 'processing' | 'completed' | 'errored' | 'cancelled'; description: string | null; - error: string | null; work_estimate: number | null; work_completed: number | null; created_at: string; diff --git a/server/src/instant/db/app_backup_jobs.clj b/server/src/instant/db/app_backup_jobs.clj index ae78f6501e..bddaac216f 100644 --- a/server/src/instant/db/app_backup_jobs.clj +++ b/server/src/instant/db/app_backup_jobs.clj @@ -60,12 +60,14 @@ :where [:= :id job-id]})))) (defn job->client-format [job] + ;; Deliberately omits `error`: the raw failure is kept in the db (and the + ;; run-job! span) for debugging, but the client only needs `job_status` to + ;; know it failed. (select-keys job [:id :app_id :app_backup_id :job_status :description - :error :work_estimate :work_completed :created_at diff --git a/server/src/instant/flags.clj b/server/src/instant/flags.clj index f6e2d6ef1e..12b1b0febd 100644 --- a/server/src/instant/flags.clj +++ b/server/src/instant/flags.clj @@ -531,9 +531,10 @@ (defn on-demand-backup-expiry-days "How long an on-demand backup is retained before its S3 objects expire. - Hard-capped at 32 by the storage `expire` tag rule." + Capped at 30 here so every consumer stays under the storage `expire` tag + rule's 32-day hard limit." [] - (flag :on-demand-backup-expiry-days 30)) + (min 30 (flag :on-demand-backup-expiry-days 30))) (defn on-demand-backup-max-triples "Apps whose estimated triple count is at or above this can't run a self-serve From bd39f6b210b4cbba431b5400372d1eba8accdf89 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 5 Aug 2026 16:26:23 -0700 Subject: [PATCH 06/27] wait for the future to unwind --- server/src/instant/db/app_backup_jobs.clj | 28 +++++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/server/src/instant/db/app_backup_jobs.clj b/server/src/instant/db/app_backup_jobs.clj index bddaac216f..bca59d8b5e 100644 --- a/server/src/instant/db/app_backup_jobs.clj +++ b/server/src/instant/db/app_backup_jobs.clj @@ -43,6 +43,10 @@ ;; How often a running job persists its progress (triples copied) to the db. (def progress-report-interval-ms 1000) +;; How long we let a cancelled backup unwind (release its db connection, abort +;; uploads) before abandoning the worker slot and letting it finish detached. +(def cancel-unwind-timeout-ms 5000) + (defn worker-count "Per-machine concurrency: the flag value, clamped to [1, max-worker-count]." [] @@ -321,11 +325,19 @@ (let [copied (AtomicLong. 0) expires-at (.plus (Instant/now) (Duration/ofDays (flags/on-demand-backup-expiry-days))) + ;; Delivered when the backup vthread actually finishes, including + ;; unwinding after an interrupt. `future-cancel` marks `fut` done + ;; immediately (so its deref would throw right away), so we wait on + ;; this instead to know the backup has really released its resources. + unwound (promise) fut (ua/vfuture - (backup/backup-app-on-primary! {:app-id app_id - :description (or description "On-demand backup") - :expires-at expires-at - :on-triple (fn [] (.incrementAndGet copied))}))] + (try + (backup/backup-app-on-primary! {:app-id app_id + :description (or description "On-demand backup") + :expires-at expires-at + :on-triple (fn [] (.incrementAndGet copied))}) + (finally + (deliver unwound true))))] (try (loop [] (let [result (deref fut progress-report-interval-ms ::pending)] @@ -335,8 +347,14 @@ ;; under us, so kill the backup and bail without touching the row. (if (report-progress! (aurora/conn-pool :write) id (.get copied)) (recur) + ;; Interrupt the backup and give it a bounded moment to unwind. + ;; If it's still running after that, abandon the worker slot and + ;; let it finish detached rather than block indefinitely. (do (future-cancel fut) - (tracer/add-data! {:attributes {:cancelled true}}))) + (let [unwound? (not= ::timeout + (deref unwound cancel-unwind-timeout-ms ::timeout))] + (tracer/add-data! {:attributes {:cancelled true + :unwound unwound?}})))) (let [{:keys [triple-count app-backup]} result] (mark-completed! (aurora/conn-pool :write) id From 6c1e01d9643421c55b2f2ccfbfe1e5cbb7fc83c4 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 5 Aug 2026 16:43:35 -0700 Subject: [PATCH 07/27] fix pool size and resizing pool --- server/src/instant/db/app_backup_jobs.clj | 25 +++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/server/src/instant/db/app_backup_jobs.clj b/server/src/instant/db/app_backup_jobs.clj index bca59d8b5e..1b265a6a0d 100644 --- a/server/src/instant/db/app_backup_jobs.clj +++ b/server/src/instant/db/app_backup_jobs.clj @@ -413,12 +413,13 @@ reclaimed)))) (defn make-pool ^ThreadPoolExecutor [n] - ;; runs n tasks at a time. The queue is bounded at max-worker-count - ;; and DiscardPolicy drops `.execute` when it's full - ;; `allowCoreThreadTimeOut` lets idle threads die so shrinking - ;; the pool actually releases them. + ;; core = max = n so exactly n tasks run at a time (n is the real cap). The + ;; queue is bounded at max-worker-count and DiscardPolicy drops `.execute` + ;; when it's full (self-continuation re-drives dropped tasks). + ;; `allowCoreThreadTimeOut` lets idle threads die so shrinking the pool + ;; actually releases them. (doto (ThreadPoolExecutor. (int n) - (int max-worker-count) + (int n) 60 TimeUnit/SECONDS (LinkedBlockingQueue. (int max-worker-count)) (ThreadPoolExecutor$DiscardPolicy.)) @@ -426,9 +427,17 @@ (defn resize-pool! [n] (when (bound? #'pool) - ;; setCorePoolSize starts threads for queued work when growing and reaps - ;; idle threads when shrinking. - (.setCorePoolSize ^ThreadPoolExecutor pool n))) + (let [n (int n) + ^ThreadPoolExecutor p pool] + ;; Move core and max together to n. They must satisfy core <= max at every + ;; step, so raise max first when growing and lower core first when + ;; shrinking. setCorePoolSize also starts threads for queued work and reaps + ;; idle ones. + (if (>= n (.getCorePoolSize p)) + (do (.setMaximumPoolSize p n) + (.setCorePoolSize p n)) + (do (.setCorePoolSize p n) + (.setMaximumPoolSize p n)))))) (defn start [] (tracer/record-info! {:name "app-backup-jobs/start"}) From f6f4458997ef5f88d04d008d3c1916a6d451a2f9 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Wed, 5 Aug 2026 16:57:08 -0700 Subject: [PATCH 08/27] reduce stuck threshold --- server/src/instant/db/app_backup_jobs.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/instant/db/app_backup_jobs.clj b/server/src/instant/db/app_backup_jobs.clj index 1b265a6a0d..40066e0838 100644 --- a/server/src/instant/db/app_backup_jobs.clj +++ b/server/src/instant/db/app_backup_jobs.clj @@ -38,7 +38,7 @@ ;; A backup that's been `processing` this long is assumed dead (its worker ;; crashed) and gets marked errored so the app isn't stuck behind the ;; one-in-flight limit forever. -(def stuck-threshold-minutes 120) +(def stuck-threshold-minutes 10) ;; How often a running job persists its progress (triples copied) to the db. (def progress-report-interval-ms 1000) From 8de30cbb0cdcda25c8428c13d91570897df5c562 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Thu, 6 Aug 2026 07:42:37 -0700 Subject: [PATCH 09/27] coderabbit feedback --- client/www/components/dash/Backups.tsx | 6 +-- server/src/instant/dash/routes.clj | 49 +++++++++++-------- server/src/instant/db/app_backup_jobs.clj | 2 + server/src/instant/flags.clj | 18 ++++--- server/src/instant/rate_limit.clj | 34 +++++++++---- .../test/instant/db/app_backup_jobs_test.clj | 5 +- 6 files changed, 73 insertions(+), 41 deletions(-) diff --git a/client/www/components/dash/Backups.tsx b/client/www/components/dash/Backups.tsx index e17b7d2d9f..d62d705566 100644 --- a/client/www/components/dash/Backups.tsx +++ b/client/www/components/dash/Backups.tsx @@ -405,16 +405,14 @@ export function Backups({ app }: { app: InstantApp }) { const jobIds = jobs.map((j) => j.id); const jobIdsKey = jobIds.join(','); - // While a backup is running, poll both the jobs and the backups list so - // progress advances and the finished snapshot shows up. + // While a backup is running, poll the jobs list so progress advances. useEffect(() => { if (!hasActive) return; const t = setInterval(() => { jobsRes.mutate(); - backupsRes.mutate(); }, 1000); return () => clearInterval(t); - }, [hasActive, jobsRes.mutate, backupsRes.mutate]); + }, [hasActive, jobsRes.mutate]); // Detect when a job we were watching leaves the active list and alert on its // outcome. `seenJobIds` starts empty, so we never alert for jobs that were diff --git a/server/src/instant/dash/routes.clj b/server/src/instant/dash/routes.clj index cdca955bd5..b6f9718ce9 100644 --- a/server/src/instant/dash/routes.clj +++ b/server/src/instant/dash/routes.clj @@ -1966,15 +1966,11 @@ (let [mins (long (Math/ceil (/ secs 60.0)))] (str mins " minute" (when (not= 1 mins) "s")))))) -(defn consume-backup-token! - "Consumes one backup token from the named bucket. On exhaustion throws a - rate-limited error telling the user when to try again." - [rate-limiter params] - (when-let [retry-at (rate-limit/consume-user-rate-limit-retry-at rate-limiter params)] - (ex/throw-rate-limited-until! - (str "You've hit the backup rate limit. Please try again in " - (humanize-retry-in retry-at) ".") - retry-at))) +(defn throw-backup-rate-limited! [retry-at] + (ex/throw-rate-limited-until! + (str "You've hit the backup rate limit. Please try again in " + (humanize-retry-in retry-at) ".") + retry-at)) (defn check-backup-rate-limits! "Throttles on-demand backups per app and per client IP, both flag-tunable @@ -1983,18 +1979,29 @@ when the rate limiter isn't running (e.g. tests)." [{:keys [app-id ip]}] (when-let [rate-limiter (eph/get-rate-limit)] - (consume-backup-token! - rate-limiter - {:app-id app-id - :bucket-name "on-demand-backup-app" - :config (->backup-rate-limit-config (flags/on-demand-backup-app-rate-limit))}) - (when ip - (consume-backup-token! - rate-limiter - {:app-id backup-ip-rate-limit-app-id - :bucket-name "on-demand-backup-ip" - :bucket-key ip - :config (->backup-rate-limit-config (flags/on-demand-backup-ip-rate-limit))})))) + (let [bucket-params (cond-> [{:app-id app-id + :bucket-name "on-demand-backup-app" + :config (->backup-rate-limit-config (flags/on-demand-backup-app-rate-limit))}] + ip (conj {:app-id backup-ip-rate-limit-app-id + :bucket-name "on-demand-backup-ip" + :bucket-key ip + :config (->backup-rate-limit-config (flags/on-demand-backup-ip-rate-limit))})) + ;; Peek every bucket before consuming from any, so a rejection on one + ;; bucket never wastes a token from another. Report the latest retry + ;; instant across the exhausted buckets, so a retry then isn't still + ;; blocked by another bucket. + retry-at (some->> bucket-params + (keep #(rate-limit/peek-user-rate-limit-retry-at rate-limiter %)) + seq + (reduce (fn [^Instant a ^Instant b] (if (.isAfter a b) a b))))] + (if retry-at + (throw-backup-rate-limited! retry-at) + ;; All buckets had capacity when peeked; consume one token from each. + ;; A concurrent request could still drain a bucket in the meantime, so + ;; honor a retry-at from the consume too. + (doseq [params bucket-params] + (when-let [retry-at (rate-limit/consume-user-rate-limit-retry-at rate-limiter params)] + (throw-backup-rate-limited! retry-at))))))) (defn app-backup-job-post "Kicks off an on-demand backup for the app. Returns the created job so the diff --git a/server/src/instant/db/app_backup_jobs.clj b/server/src/instant/db/app_backup_jobs.clj index 40066e0838..cc652042d4 100644 --- a/server/src/instant/db/app_backup_jobs.clj +++ b/server/src/instant/db/app_backup_jobs.clj @@ -363,6 +363,8 @@ (tracer/add-data! {:attributes {:triple-count triple-count :app-backup-id (:id app-backup)}}))))) (catch Throwable t + (future-cancel fut) + (deref unwound cancel-unwind-timeout-ms ::timeout) (tracer/record-exception-span! t {:name "app-backup-jobs/run-job-error" :escaping? false}) (mark-error! (aurora/conn-pool :write) id t)))))) diff --git a/server/src/instant/flags.clj b/server/src/instant/flags.clj index 12b1b0febd..6b24c38824 100644 --- a/server/src/instant/flags.clj +++ b/server/src/instant/flags.clj @@ -543,23 +543,29 @@ [] (flag :on-demand-backup-max-triples 25000000)) +(defn- rate-limit + "Builds a {:capacity n :window-minutes m} rate limit from a JSON flag value, + validating each field and falling back to the given defaults when it is + absent, non-integer, zero, or negative." + [v default-capacity default-window-minutes] + (let [capacity (get v "capacity") + window-minutes (get v "windowMinutes")] + {:capacity (if (pos-int? capacity) capacity default-capacity) + :window-minutes (if (pos-int? window-minutes) window-minutes default-window-minutes)})) + (defn on-demand-backup-app-rate-limit "Per-app throttle for on-demand backups, as {:capacity n :window-minutes m} (default 1 per 5 minutes). Set the flag to a JSON object like {\"capacity\": 1, \"windowMinutes\": 5} to tune without a deploy." [] - (let [v (flag :on-demand-backup-app-rate-limit)] - {:capacity (get v "capacity" 1) - :window-minutes (get v "windowMinutes" 5)})) + (rate-limit (flag :on-demand-backup-app-rate-limit) 1 5)) (defn on-demand-backup-ip-rate-limit "Per-IP throttle for on-demand backups, as {:capacity n :window-minutes m} (default 2 per 5 minutes). Set the flag to a JSON object like {\"capacity\": 2, \"windowMinutes\": 5} to tune without a deploy." [] - (let [v (flag :on-demand-backup-ip-rate-limit)] - {:capacity (get v "capacity" 2) - :window-minutes (get v "windowMinutes" 5)})) + (rate-limit (flag :on-demand-backup-ip-rate-limit) 2 5)) (defn use-cloudfront-signed-url? [app-id] (when-not (toggled? :disable-cloudfront-signed-urls-globally) diff --git a/server/src/instant/rate_limit.clj b/server/src/instant/rate_limit.clj index b4e5311dec..3a63c27d19 100644 --- a/server/src/instant/rate_limit.clj +++ b/server/src/instant/rate_limit.clj @@ -14,7 +14,7 @@ (com.hazelcast.config EvictionConfig EvictionPolicy MapStoreConfig MapStoreConfig$InitialLoadMode MaxSizePolicy) (com.hazelcast.core HazelcastInstance) (com.hazelcast.map MapStore) - (io.github.bucket4j Bandwidth Bucket BucketConfiguration) + (io.github.bucket4j Bandwidth Bucket BucketConfiguration ConsumptionProbe EstimationProbe) (io.github.bucket4j.grid.hazelcast Bucket4jHazelcast) (java.lang AutoCloseable) (java.time Duration Instant) @@ -325,14 +325,20 @@ ^Bucket bucket (get-bucket-with-config key (make-bucket-config-fn config))] (.tryConsume bucket tokens))) -(defn consume-user-rate-limit +(defn try-consume-user-rate-limit-and-return-remaining + "Consumes tokens from the user's bucket and returns the bucket-4j + `ConsumptionProbe`, exposing `isConsumed`, `getRemainingTokens`, and + `getNanosToWaitForRefill`." [{:keys [get-bucket-with-config]} - {:keys [app-id config tokens - bucket-key bucket-name] + {:keys [app-id config tokens bucket-key bucket-name] :or {tokens 1}}] (let [key (user-key-hash app-id bucket-name config bucket-key) - ^Bucket bucket (get-bucket-with-config key (make-bucket-config-fn config)) - remaining (.tryConsumeAndReturnRemaining bucket tokens)] + ^Bucket bucket (get-bucket-with-config key (make-bucket-config-fn config))] + (.tryConsumeAndReturnRemaining bucket tokens))) + +(defn consume-user-rate-limit + [store args] + (let [^ConsumptionProbe remaining (try-consume-user-rate-limit-and-return-remaining store args)] (if (.isConsumed remaining) true (ex/throw-permission-rate-limited! (.plusNanos (Instant/now) @@ -344,14 +350,24 @@ returns nil when the tokens were consumed, or the `Instant` at which the caller may retry when the bucket is empty. Lets the caller craft its own rate-limit message." + [store args] + (let [^ConsumptionProbe remaining (try-consume-user-rate-limit-and-return-remaining store args)] + (when-not (.isConsumed remaining) + (.plusNanos (Instant/now) (.getNanosToWaitForRefill remaining))))) + +(defn peek-user-rate-limit-retry-at + "Checks whether the user's bucket has enough tokens *without consuming any*. + Returns nil when the tokens are available, or the `Instant` at which the + caller may retry when the bucket is empty. Lets a caller probe several + buckets before deciding to consume from any of them." [{:keys [get-bucket-with-config]} {:keys [app-id config tokens bucket-key bucket-name] :or {tokens 1}}] (let [key (user-key-hash app-id bucket-name config bucket-key) ^Bucket bucket (get-bucket-with-config key (make-bucket-config-fn config)) - remaining (.tryConsumeAndReturnRemaining bucket tokens)] - (when-not (.isConsumed remaining) - (.plusNanos (Instant/now) (.getNanosToWaitForRefill remaining))))) + ^EstimationProbe estimate (.estimateAbilityToConsume bucket tokens)] + (when-not (.canBeConsumed estimate) + (.plusNanos (Instant/now) (.getNanosToWaitForRefill estimate))))) (defonce schedule (atom nil)) diff --git a/server/test/instant/db/app_backup_jobs_test.clj b/server/test/instant/db/app_backup_jobs_test.clj index ed6855b469..dc838c8047 100644 --- a/server/test/instant/db/app_backup_jobs_test.clj +++ b/server/test/instant/db/app_backup_jobs_test.clj @@ -32,7 +32,10 @@ (deftest runs-at-most-n-backups-at-a-time (let [n 2 - total 5 + ;; More jobs than the pool's queue can hold (`make-pool` bounds the queue + ;; at `max-worker-count`), so the extra `.execute` calls hit the + ;; DiscardPolicy path. Self-continuation still has to drain every job. + total (+ app-backup-jobs/max-worker-count 8) ;; A fresh pool sized `n` so we get a clean concurrency cap that doesn't ;; depend on the flag value the shared pool happened to start with. pool (app-backup-jobs/make-pool n) From c496390fac7a27222d86c3d3f45a53712d38ceea Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Thu, 6 Aug 2026 09:30:29 -0700 Subject: [PATCH 10/27] much better cancellation --- server/src/instant/db/app_backup_jobs.clj | 66 +++++++++++++++---- .../test/instant/db/app_backup_jobs_test.clj | 54 +++++++++++++++ 2 files changed, 107 insertions(+), 13 deletions(-) diff --git a/server/src/instant/db/app_backup_jobs.clj b/server/src/instant/db/app_backup_jobs.clj index cc652042d4..988c924a57 100644 --- a/server/src/instant/db/app_backup_jobs.clj +++ b/server/src/instant/db/app_backup_jobs.clj @@ -22,7 +22,8 @@ (clojure.lang ExceptionInfo) (java.lang AutoCloseable) (java.time Duration Instant) - (java.util.concurrent LinkedBlockingQueue ThreadPoolExecutor ThreadPoolExecutor$DiscardPolicy TimeUnit) + (com.google.common.collect MapMaker) + (java.util.concurrent ConcurrentMap LinkedBlockingQueue ThreadFactory ThreadPoolExecutor ThreadPoolExecutor$DiscardPolicy TimeUnit) (java.util.concurrent.atomic AtomicLong))) ;; `pool` is the live worker pool (a ThreadPoolExecutor). Each task grabs and @@ -414,18 +415,43 @@ (tracer/add-data! {:attributes {:reclaimed-count reclaimed}}) reclaimed)))) +;; Maps each pool to the set of backup vfutures currently running on it, so +;; `stop` can cancel them at shutdown. Weak keys (via MapMaker) so a pool +;; discarded by `restart` auto-evicts once its worker threads die and nothing +;; else holds it. +(defonce ^{:tag ConcurrentMap} pool->in-flight + (-> (MapMaker.) + (.weakKeys) + (.makeMap))) + (defn make-pool ^ThreadPoolExecutor [n] - ;; core = max = n so exactly n tasks run at a time (n is the real cap). The - ;; queue is bounded at max-worker-count and DiscardPolicy drops `.execute` + ;; The queue is bounded at max-worker-count and DiscardPolicy drops `.execute` ;; when it's full (self-continuation re-drives dropped tasks). - ;; `allowCoreThreadTimeOut` lets idle threads die so shrinking the pool - ;; actually releases them. - (doto (ThreadPoolExecutor. (int n) - (int n) - 60 TimeUnit/SECONDS - (LinkedBlockingQueue. (int max-worker-count)) - (ThreadPoolExecutor$DiscardPolicy.)) - (.allowCoreThreadTimeOut true))) + ;; + ;; Workers are virtual threads, and the factory binds `*child-vfutures*` to an + ;; `in-flight` map for each worker's whole lifetime. Every backup started with + ;; `ua/vfuture` inside a task therefore registers itself into that map, which + ;; we stash in `pool->in-flight` so `stop` can cancel the in-flight backups. + (let [in-flight (ua/new-child-vfutures) + vfactory (-> (Thread/ofVirtual) + (.name "app-backup-worker-" 0) + (.factory)) + factory (reify ThreadFactory + (newThread [_ r] + (.newThread vfactory + ^Runnable (fn [] + (binding [ua/*child-vfutures* in-flight] + (.run ^Runnable r)))))) + pool (doto (ThreadPoolExecutor. (int n) + (int n) + 60 TimeUnit/SECONDS + (LinkedBlockingQueue. (int max-worker-count)) + factory + (ThreadPoolExecutor$DiscardPolicy.)) + ;; lets idle threads die so shrinking the pool actually releases them. + (.allowCoreThreadTimeOut true))] + (.put pool->in-flight pool in-flight) + pool)) (defn resize-pool! [n] (when (bound? #'pool) @@ -441,6 +467,21 @@ (do (.setCorePoolSize p n) (.setMaximumPoolSize p n)))))) +(defn shutdown-pool + "Stops taking new work and gives running backups `grace-ms` to finish on their + own, then force-cancels any stragglers. Each cancelled worker waits for its + backup to unwind and release its resources before it exits. + Don't rely on shutdown to finish in `grace-ms`, it may take twice the time." + [^ThreadPoolExecutor p grace-ms] + (.shutdown p) + (.awaitTermination p grace-ms TimeUnit/MILLISECONDS) + ;; Force-cancel whatever outlasted the grace window. `cancel-children` cascades + ;; the interrupt to each backup vfuture and its copy sub-tasks. + (when-let [in-flight (.get pool->in-flight p)] + (ua/cancel-children in-flight true)) + (.awaitTermination p grace-ms TimeUnit/MILLISECONDS) + (.shutdownNow p)) + (defn start [] (tracer/record-info! {:name "app-backup-jobs/start"}) (def pool (make-pool (worker-count))) @@ -467,8 +508,7 @@ (when (and (bound? #'flag-unsub) flag-unsub) (flag-unsub)) (when (bound? #'pool) - (.shutdown ^ThreadPoolExecutor pool) - (.awaitTermination ^ThreadPoolExecutor pool 5 TimeUnit/MINUTES))) + (shutdown-pool pool (.toMillis TimeUnit/MINUTES 1)))) (defn restart [] (stop) diff --git a/server/test/instant/db/app_backup_jobs_test.clj b/server/test/instant/db/app_backup_jobs_test.clj index dc838c8047..64ceb9d6ed 100644 --- a/server/test/instant/db/app_backup_jobs_test.clj +++ b/server/test/instant/db/app_backup_jobs_test.clj @@ -5,6 +5,7 @@ [instant.data.constants :refer [test-user-id]] [instant.db.app-backup-jobs :as app-backup-jobs] [instant.model.app :as app-model] + [instant.util.async :as ua] [instant.util.test :refer [instant-ex-data wait-for]])) (def ^:private wait-timeout 10000) @@ -117,3 +118,56 @@ (finally (deliver release true) (.shutdown pool)))))))) + +(deftest killing-the-pool-cancels-in-flight-work + ;; Just the pool mechanics--no db, no real backup. We run a task shaped like + ;; `run-job!` (a vfuture waited on in a deref loop, with a catch standing in + ;; for `mark-error!`) and confirm that killing the pool cancels the registered + ;; vfuture and that the cancellation propagates out of the deref loop so the + ;; catch runs. + (let [pool (app-backup-jobs/make-pool 1) + started (promise) + unwound (promise) + errored (promise) + caught (atom nil) + ;; The "backup" blocks here until it's interrupted. + block (promise) + task (fn [] + (let [fut (ua/vfuture + (try + (deliver started true) + @block + (finally + (deliver unwound true))))] + (try + ;; Stand-in for run-job!'s progress loop: wait on the backup. + (loop [] + (when (identical? ::pending (deref fut 100 ::pending)) + (recur))) + (catch Throwable t + ;; Stand-in for mark-error!: the error propagated to us. + (future-cancel fut) + (deref unwound app-backup-jobs/cancel-unwind-timeout-ms ::timeout) + (reset! caught t) + (deliver errored true)))))] + (try + ;; Runs on a pool worker, so the vfuture registers with the pool's + ;; in-flight map via the worker's `*child-vfutures*` binding. + (.execute pool ^Runnable task) + (is (= true (deref started 5000 nil)) + "the task's backup vfuture started (and registered with the pool)") + (is (not (realized? errored)) + "nothing errored while the backup was still blocked") + + ;; Kill the pool with a tiny grace so we don't wait out the real window. + (app-backup-jobs/shutdown-pool pool 100) + + (is (= true (deref errored 5000 nil)) + "the deref loop threw, so the mark-error! stand-in ran") + (is (instance? java.util.concurrent.CancellationException @caught) + "the propagated error was the vfuture's cancellation") + (is (realized? unwound) + "the backup vfuture unwound its resources") + (finally + (deliver block true) + (.shutdownNow pool))))) From 3869ddd13fc6d2db58c15d63b3cdbe8d15477035 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Thu, 6 Aug 2026 14:55:35 -0700 Subject: [PATCH 11/27] restores from /intern --- client/www/app/intern/content.tsx | 7 + client/www/app/intern/restore/content.tsx | 401 ++++++++++++++++++ client/www/app/intern/restore/page.tsx | 10 + .../migrations/123_app_restore_jobs.down.sql | 1 + .../migrations/123_app_restore_jobs.up.sql | 27 ++ server/src/instant/core.clj | 3 +- server/src/instant/dash/routes.clj | 55 +++ server/src/instant/db/app_restore_jobs.clj | 213 ++++++++++ server/src/instant/restore.clj | 29 +- 9 files changed, 736 insertions(+), 10 deletions(-) create mode 100644 client/www/app/intern/restore/content.tsx create mode 100644 client/www/app/intern/restore/page.tsx create mode 100644 server/resources/migrations/123_app_restore_jobs.down.sql create mode 100644 server/resources/migrations/123_app_restore_jobs.up.sql create mode 100644 server/src/instant/db/app_restore_jobs.clj diff --git a/client/www/app/intern/content.tsx b/client/www/app/intern/content.tsx index be3d62d8e9..4b391d63f8 100644 --- a/client/www/app/intern/content.tsx +++ b/client/www/app/intern/content.tsx @@ -91,6 +91,13 @@ const tools: ToolCard[] = [ 'Preview all og:image cards across the site to make sure they look good before deploying.', category: 'Other', }, + { + title: 'Restore App', + href: '/intern/restore', + description: + 'Upload a backup zip to restore it into a new app. Runs in the background on the machine that receives the upload.', + category: 'Other', + }, ]; const categories = ['All', 'KPIs', 'Analytics', 'Comms', 'Other']; diff --git a/client/www/app/intern/restore/content.tsx b/client/www/app/intern/restore/content.tsx new file mode 100644 index 0000000000..bbf92a79e7 --- /dev/null +++ b/client/www/app/intern/restore/content.tsx @@ -0,0 +1,401 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { + LandingContainer, + LegacyNav, + Section, + H2, +} from '@/components/marketingUi'; +import { + Button, + Content, + Dialog, + FullscreenLoading, + SubsectionHeading, + TextInput, + useDialog, +} from '@/components/ui'; +import { Footer } from '@/components/new-landing/Footer'; +import { useAdmin, useAuthInfo, useTokenFetch } from '@/lib/auth'; +import { useIsHydrated } from '@/lib/hooks/useIsHydrated'; +import { errorToast, successToast } from '@/lib/toast'; +import config from '@/lib/config'; + +type RestoreJob = { + id: string; + app_id: string; + title: string | null; + job_status: 'waiting' | 'processing' | 'completed' | 'errored' | 'cancelled'; + progress: string | null; + error: string | null; + created_at: string; + done_at: string | null; + updated_at: string; +}; + +const TERMINAL = new Set(['completed', 'errored', 'cancelled']); + +function RestoreDialog({ + token, + email, + onStarted, +}: { + token: string | undefined; + email: string | undefined; + onStarted: () => void; +}) { + const dialog = useDialog(); + const [file, setFile] = useState(null); + const [appId, setAppId] = useState(''); + const [title, setTitle] = useState(''); + const [uploading, setUploading] = useState(false); + const [errorMsg, setErrorMsg] = useState(null); + + async function onSubmit(e: React.FormEvent) { + e.preventDefault(); + if (!file) { + setErrorMsg('Choose a zip file to restore.'); + return; + } + setUploading(true); + setErrorMsg(null); + try { + const params = new URLSearchParams(); + if (appId.trim()) params.set('app_id', appId.trim()); + if (title.trim()) params.set('title', title.trim()); + const qs = params.toString(); + + const res = await fetch( + `${config.apiURI}/dash/restores/zip${qs ? `?${qs}` : ''}`, + { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/zip', + }, + body: file, + }, + ); + const json = await res.json(); + if (!res.ok) { + throw new Error(json?.message ?? `Restore failed (${res.status})`); + } + successToast('Restore started'); + setFile(null); + setAppId(''); + setTitle(''); + dialog.onClose(); + onStarted(); + } catch (err: any) { + setErrorMsg(err?.message ?? 'Restore failed'); + } finally { + setUploading(false); + } + } + + return ( + <> + + +
+ Restore from backup + + Upload a backup zip to restore the app. The app will have the same + schema, rules, data, and files as the app that was backed up. + + + If you have OAuth client secrets for your social logins, you will + need to update them from the `Auth` tab after the restore finishes. + +
+ Only upload zip files that were downloaded from a valid Instant + backup. +
+ {email && ( + + The restored app will be owned by your account ( + {email}). + + )} + +
+
+ + Backup zip + + +
+ + + + + + {errorMsg && ( +
+ {errorMsg} +
+ )} + +
+ +
+
+
+ + + ); +} + +function statusText(job: RestoreJob) { + if (job.job_status === 'waiting') return 'Waiting…'; + if (job.job_status === 'processing') return job.progress ?? 'Restoring…'; + return job.job_status; +} + +function RecentRestores({ + jobs, + token, + onChanged, +}: { + jobs: RestoreJob[]; + token: string | undefined; + onChanged: () => void; +}) { + const [cancellingId, setCancellingId] = useState(null); + + async function cancel(id: string) { + setCancellingId(id); + try { + const res = await fetch(`${config.apiURI}/dash/restore-jobs/${id}`, { + method: 'DELETE', + headers: { authorization: `Bearer ${token}` }, + }); + if (!res.ok) { + const json = await res.json().catch(() => null); + throw new Error(json?.message ?? `Cancel failed (${res.status})`); + } + onChanged(); + } catch (err: any) { + errorToast(err?.message ?? 'Cancel failed'); + } finally { + setCancellingId(null); + } + } + + if (jobs.length === 0) { + return

No restores yet.

; + } + return ( +
+ {jobs.map((job) => ( +
+
+ {job.job_status === 'completed' ? ( + + {job.title || 'Restored app'} + + ) : ( + + {job.title || 'Restored app'} + + )} + + app id: {job.app_id} + + + {new Date(job.created_at).toLocaleString()} + +
+
+
+ + {statusText(job)} + + {job.job_status === 'errored' && job.error && ( + + {job.error} + + )} +
+ {!TERMINAL.has(job.job_status) && ( + + )} +
+
+ ))} +
+ ); +} + +function RestoreContent() { + const { token, user } = useAuthInfo(); + + const restoresRes = useTokenFetch<{ 'restore-jobs': RestoreJob[] }>( + `${config.apiURI}/dash/restore-jobs`, + token, + ); + const jobs = restoresRes.data?.['restore-jobs'] ?? []; + const anyActive = jobs.some((j) => !TERMINAL.has(j.job_status)); + + // Poll the list while any restore is in flight so progress advances. + useEffect(() => { + if (!anyActive) return; + const t = setInterval(() => restoresRes.mutate(), 1000); + return () => clearInterval(t); + }, [anyActive, restoresRes.mutate]); + + return ( +
+
+

Restore from backup

+

+ Restore an app from a backup zip you downloaded from Instant. +

+
+ +
+ restoresRes.mutate()} + /> +
+ +
+ + Recent restores + + restoresRes.mutate()} + /> +
+
+ ); +} + +export default function RestorePage() { + const isHydrated = useIsHydrated(); + const { isAdmin, isLoading, error } = useAdmin(); + + if (!isHydrated || isLoading) { + return ( + + +
+
+ +
+
+