From 3807f350c0f6b889485c7b63d26b0a08fd59525e Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Thu, 3 Sep 2026 23:45:29 +0800 Subject: [PATCH 1/4] fix(cli): snapshot the blob volume unless blobs live in external S3 `tale backup` and the pre-deploy snapshot never captured `object-store-data`: SNAPSHOT_VOLUMES predates the object store (blobs then lived in Convex `_storage` inside `convex-data`, which is snapshotted) and was not updated when #3107 moved blobs to MinIO. Every backup of a self-hosted deployment silently lacked its uploads. Add the blob volume to the snapshot set, decided the way the backend decides where blobs live: read `/object-storage/connection.json` from the config volume and compare the `default` tree's endpoint with the bundled store's address (the bootstrap's own bundled-vs-repointed test). A deployment default repointed at external S3 skips the local volume and prints a one-line notice naming endpoint and bucket; organizations with their own bucket are named the same way. Anything short of a readable default connection captures the volume (fail-safe). The blob archive is as large as the store, so it gets its own timeout bound, shared with restore. The bundled endpoint the compose generators hardcode moves into one constant both sites read. --- tools/cli/src/lib/backup/constants.ts | 39 +++- .../src/lib/backup/create-snapshot.test.ts | 206 +++++++++++++++++- tools/cli/src/lib/backup/create-snapshot.ts | 52 ++++- .../cli/src/lib/backup/inspect-blob-store.ts | 164 ++++++++++++++ .../src/lib/compose/generators/constants.ts | 11 + .../services/create-backend-services.ts | 9 +- 6 files changed, 468 insertions(+), 13 deletions(-) create mode 100644 tools/cli/src/lib/backup/inspect-blob-store.ts diff --git a/tools/cli/src/lib/backup/constants.ts b/tools/cli/src/lib/backup/constants.ts index b2684e6c4d..5c7085ce1d 100644 --- a/tools/cli/src/lib/backup/constants.ts +++ b/tools/cli/src/lib/backup/constants.ts @@ -1,3 +1,20 @@ +/** + * The org config store. The name predates the Convex retirement and is kept + * so no operator has to migrate a volume for a rename. Mounted at + * `TALE_CONFIG_DIR` (`/app/data`) in the backend tier, it holds every + * `//*.json` config file — including the object-storage + * connections the backup inspects to learn where the blobs live. + */ +export const CONFIG_VOLUME = 'convex-data'; + +/** + * The blob store's data: uploaded files, chat attachments, audio, generated + * media — non-rederivable, and as large as the store. Captured whenever the + * deployment default points at the bundled `object-store`; left out only + * when the default is an external S3 (see inspect-blob-store.ts). + */ +export const BLOB_VOLUME = 'object-store-data'; + /** * Volumes captured by a snapshot: every project volume that holds * non-rederivable state. `db-backup` is excluded (never back up backups — @@ -6,9 +23,8 @@ */ export const SNAPSHOT_VOLUMES = [ 'db-data', - // The org config store. The name predates the Convex retirement and is - // kept so no operator has to migrate a volume for a rename. - 'convex-data', + CONFIG_VOLUME, + BLOB_VOLUME, 'caddy-data', 'caddy-config', ] as const; @@ -28,6 +44,23 @@ export const BACKUP_VOLUME = 'backups'; */ export const BACKUP_HELPER_IMAGE = 'alpine:3.22'; +/** + * Bounds on a single volume's tar (snapshot) or extract (restore). The + * database, config and proxy volumes are small and settle in minutes; the + * blob store is as large as everything ever uploaded and gets a bound that + * covers a store in the hundreds of gigabytes. Both bounds guard against a + * hung docker, not against a slow but progressing archive — the snapshot + * side pauses the volume's containers for the duration either way. + */ +const ARCHIVE_TIMEOUT_SECONDS = 1800; +const BLOB_ARCHIVE_TIMEOUT_SECONDS = 4 * 3600; + +export function archiveTimeoutSeconds(volume: string): number { + return volume === BLOB_VOLUME + ? BLOB_ARCHIVE_TIMEOUT_SECONDS + : ARCHIVE_TIMEOUT_SECONDS; +} + /** * Snapshot ids are CLI-generated (`--`), but they * round-trip through manifests stored on the backups volume and back into diff --git a/tools/cli/src/lib/backup/create-snapshot.test.ts b/tools/cli/src/lib/backup/create-snapshot.test.ts index de3a806b2b..c3e1c4ab7d 100644 --- a/tools/cli/src/lib/backup/create-snapshot.test.ts +++ b/tools/cli/src/lib/backup/create-snapshot.test.ts @@ -6,6 +6,8 @@ const dockerMock = mock(); const execMock = mock(); const ensureVolumesMock = mock(); const volumeExistsMock = mock(); +const loggerWarnMock = mock(); +const loggerNoticeMock = mock(); mock.module('../docker/docker', () => ({ docker: dockerMock })); mock.module('../docker/exec', () => ({ exec: execMock })); @@ -16,13 +18,13 @@ mock.module('../docker/ensure-volumes', () => ({ mock.module('../../utils/logger', () => ({ info: mock(), error: mock(), - warn: mock(), + warn: loggerWarnMock, step: mock(), success: mock(), header: mock(), blank: mock(), debug: mock(), - notice: mock(), + notice: loggerNoticeMock, table: mock(), })); @@ -32,11 +34,71 @@ function ok(stdout = '') { return { success: true, stdout, stderr: '', exitCode: 0 }; } +function failed(stderr: string) { + return { success: false, stdout: '', stderr, exitCode: 1 }; +} + +/** One `\t` row of the config-store inspection output. */ +function connectionRow(slug: string, connection: Record) { + return `/data/${slug}/object-storage/connection.json\t${JSON.stringify(connection)}`; +} + +const BUNDLED_CONNECTION = { + region: 'us-east-1', + endpoint: 'http://object-store:9000', + forcePathStyle: true, + bucket: 'tale-blobs', +}; + +const EXTERNAL_CONNECTION = { + region: 'eu-central-1', + endpoint: 'https://s3.eu-central-1.amazonaws.com', + forcePathStyle: false, + bucket: 'acme-tale-blobs', +}; + +/** + * A full local stack: every snapshot volume exists, the config store answers + * the inspection with `rows`, and every tar succeeds with a per-volume sha. + */ +function seedLocalStack(rows: string[]) { + volumeExistsMock.mockResolvedValue(true); + ensureVolumesMock.mockResolvedValue(true); + dockerMock.mockImplementation((...args: string[]) => { + if (args[0] === 'ps') return Promise.resolve(ok('')); + return Promise.resolve(ok()); + }); + execMock.mockImplementation((_cmd: string, args: string[]) => { + const script = args[args.length - 1]; + if (script.includes('object-storage/connection.json')) { + return Promise.resolve(ok(rows.join('\n'))); + } + const tar = /tar czf \/backup\/[^/]+\/([a-z-]+)\.tar\.gz/.exec(script); + if (tar) { + return Promise.resolve(ok(`${SHA} ${tar[1]}.tar.gz\n4096`)); + } + return Promise.resolve(ok()); + }); +} + +function tarredVolumes(): string[] { + return execMock.mock.calls + .map((call) => + /tar czf \/backup\/[^/]+\/([a-z-]+)\.tar\.gz/.exec( + call[1][call[1].length - 1], + ), + ) + .filter((match): match is RegExpExecArray => match !== null) + .map((match) => match[1]); +} + afterEach(() => { dockerMock.mockReset(); execMock.mockReset(); ensureVolumesMock.mockReset(); volumeExistsMock.mockReset(); + loggerWarnMock.mockReset(); + loggerNoticeMock.mockReset(); }); describe('createSnapshot', () => { @@ -162,4 +224,144 @@ describe('createSnapshot', () => { }), ).rejects.toThrow('unparseable integrity output'); }); + + describe('the blob volume', () => { + test('is captured with the other data volumes when the deployment default is the bundled store', async () => { + seedLocalStack([connectionRow('default', BUNDLED_CONNECTION)]); + + const manifest = await createSnapshot({ + prefix: 'p_', + trigger: 'manual', + platformVersion: '0.5.7', + }); + + expect(Object.keys(manifest?.volumes ?? {}).sort()).toEqual([ + 'caddy-config', + 'caddy-data', + 'convex-data', + 'db-data', + 'object-store-data', + ]); + expect(manifest?.volumes['object-store-data']).toEqual({ + sha256: SHA, + sizeBytes: 4096, + }); + expect(tarredVolumes()).toContain('object-store-data'); + // The config store is read read-only, from the SAME volume the backend + // resolves connection files from. + const inspection = execMock.mock.calls.find((call) => + String(call[1][call[1].length - 1]).includes( + 'object-storage/connection.json', + ), + ); + expect(inspection?.[1]).toContain('p_convex-data:/data:ro'); + // Blobs are proportional to the store: the tar gets the wider bound. + const blobTar = execMock.mock.calls.find((call) => + String(call[1][call[1].length - 1]).includes( + 'object-store-data.tar.gz', + ), + ); + expect(blobTar?.[2]?.timeout).toBeGreaterThan(1800); + expect(loggerNoticeMock).not.toHaveBeenCalled(); + }); + + test('is left out, with a one-line notice, when the deployment default is external S3', async () => { + seedLocalStack([connectionRow('default', EXTERNAL_CONNECTION)]); + + const manifest = await createSnapshot({ + prefix: 'p_', + trigger: 'deploy', + platformVersion: '0.5.7', + }); + + expect(manifest?.volumes['object-store-data']).toBeUndefined(); + expect(tarredVolumes()).not.toContain('object-store-data'); + expect(Object.keys(manifest?.volumes ?? {})).toHaveLength(4); + const notices = loggerNoticeMock.mock.calls.map((call) => + String(call[0]), + ); + expect(notices).toHaveLength(1); + expect(notices[0]).toContain('https://s3.eu-central-1.amazonaws.com'); + expect(notices[0]).toContain('acme-tale-blobs'); + expect(notices[0]).toContain('not in this snapshot'); + }); + + test('is still captured when organizations bring their own bucket, and those orgs are named', async () => { + seedLocalStack([ + connectionRow('default', BUNDLED_CONNECTION), + connectionRow('acme', EXTERNAL_CONNECTION), + connectionRow('globex', { ...EXTERNAL_CONNECTION, bucket: 'globex' }), + ]); + + const manifest = await createSnapshot({ + prefix: 'p_', + trigger: 'manual', + platformVersion: '0.5.7', + }); + + expect(manifest?.volumes['object-store-data']).toBeDefined(); + const notices = loggerNoticeMock.mock.calls.map((call) => + String(call[0]), + ); + expect(notices).toHaveLength(1); + expect(notices[0]).toContain('2 organization(s)'); + expect(notices[0]).toContain('acme, globex'); + expect(notices[0]).toContain('not in this snapshot'); + }); + + test('is captured (fail-safe) when the config store cannot be inspected', async () => { + seedLocalStack([]); + execMock.mockImplementation((_cmd: string, args: string[]) => { + const script = args[args.length - 1]; + if (script.includes('object-storage/connection.json')) { + return Promise.resolve(failed('permission denied')); + } + const tar = /tar czf \/backup\/[^/]+\/([a-z-]+)\.tar\.gz/.exec(script); + if (tar) { + return Promise.resolve(ok(`${SHA} ${tar[1]}.tar.gz\n4096`)); + } + return Promise.resolve(ok()); + }); + + const manifest = await createSnapshot({ + prefix: 'p_', + trigger: 'manual', + platformVersion: null, + }); + + expect(manifest?.volumes['object-store-data']).toBeDefined(); + const warnings = loggerWarnMock.mock.calls.map((call) => String(call[0])); + expect(warnings.some((line) => line.includes('permission denied'))).toBe( + true, + ); + }); + + test('is captured (fail-safe) when the default connection file is unreadable', async () => { + seedLocalStack([ + '/data/default/object-storage/connection.json\t{not json', + ]); + + const manifest = await createSnapshot({ + prefix: 'p_', + trigger: 'manual', + platformVersion: null, + }); + + expect(manifest?.volumes['object-store-data']).toBeDefined(); + expect(loggerWarnMock).toHaveBeenCalled(); + }); + + test('is captured when no connection file exists yet (store never seeded)', async () => { + seedLocalStack([]); + + const manifest = await createSnapshot({ + prefix: 'p_', + trigger: 'manual', + platformVersion: null, + }); + + expect(manifest?.volumes['object-store-data']).toBeDefined(); + expect(loggerNoticeMock).not.toHaveBeenCalled(); + }); + }); }); diff --git a/tools/cli/src/lib/backup/create-snapshot.ts b/tools/cli/src/lib/backup/create-snapshot.ts index 5f4fef50a4..68387e44bb 100644 --- a/tools/cli/src/lib/backup/create-snapshot.ts +++ b/tools/cli/src/lib/backup/create-snapshot.ts @@ -5,10 +5,13 @@ import { docker } from '../docker/docker'; import { ensureVolumes, volumeExists } from '../docker/ensure-volumes'; import { exec } from '../docker/exec'; import { + archiveTimeoutSeconds, BACKUP_HELPER_IMAGE, BACKUP_VOLUME, + BLOB_VOLUME, SNAPSHOT_VOLUMES, } from './constants'; +import { type BlobStoreLayout, inspectBlobStore } from './inspect-blob-store'; export const SNAPSHOT_TRIGGERS = [ 'deploy', @@ -46,9 +49,6 @@ interface CreateSnapshotOptions { allowMissingVolumes?: boolean; } -/** Bounds a single volume tar; the slowest realistic volume is db-data. */ -const TAR_TIMEOUT_SECONDS = 1800; - function newSnapshotId(trigger: SnapshotTrigger, now = new Date()): string { const iso = now.toISOString(); // e.g. 2026-06-11T14:25:30.123Z const date = iso.slice(0, 10).replaceAll('-', ''); @@ -115,7 +115,7 @@ async function snapshotVolume( // are parseable from the last two stdout lines below. `mkdir -p /backup/${id} && tar czf /backup/${id}/${volume}.tar.gz -C /data . && cd /backup/${id} && sha256sum ${volume}.tar.gz | tee ${volume}.tar.gz.sha256 && wc -c < ${volume}.tar.gz`, ], - { timeout: TAR_TIMEOUT_SECONDS }, + { timeout: archiveTimeoutSeconds(volume) }, ); if (!tarResult.success) { throw new Error( @@ -156,12 +156,47 @@ async function snapshotVolume( } } +/** + * Say, once and plainly, which blobs a snapshot can NOT contain: those in an + * external S3 the deployment default points at, and those in buckets + * organizations bring themselves. Either way the operator's own backup of + * that bucket is the only copy — silence here would read as "everything is + * in the snapshot". + */ +function announceExternalBlobs(layout: BlobStoreLayout): void { + switch (layout.default.kind) { + case 'external': + logger.notice( + `Blobs live in external S3 (${layout.default.endpoint}, bucket "${layout.default.bucket}") — not in this snapshot: back that bucket up yourself; the local ${BLOB_VOLUME} volume is skipped.`, + ); + break; + case 'unknown': + logger.debug( + `Blob store layout unknown (${layout.default.reason}) — capturing ${BLOB_VOLUME} when present.`, + ); + break; + case 'bundled': + break; + } + if (layout.ownBucketOrgs.length > 0) { + logger.notice( + `Blobs of ${layout.ownBucketOrgs.length} organization(s) with their own bucket (${layout.ownBucketOrgs.join(', ')}) live in those buckets — not in this snapshot: back them up under your own contract.`, + ); + } +} + /** * Snapshot every existing data volume under `prefix` into the project's * backups volume. The manifest is written LAST — its presence marks the * snapshot complete. Listing and restore ignore manifest-less directories, * so a crash mid-tar can never surface as a restorable snapshot. * + * The blob volume is one of the data volumes — uploads are as + * non-rederivable as rows — unless the deployment default points at an + * external S3, in which case the local volume holds nothing the app reads + * and the operator is told the bucket is theirs to back up. Blobs of + * organizations that bring their own bucket are named for the same reason. + * * Throws on any failure (callers abort the surrounding deploy/migration); * returns null only in the `allowMissingVolumes` no-volumes case. */ @@ -170,8 +205,14 @@ export async function createSnapshot( ): Promise { const { prefix, trigger, platformVersion } = options; + const blobStore = await inspectBlobStore(prefix); + const candidates: readonly string[] = + blobStore.default.kind === 'external' + ? SNAPSHOT_VOLUMES.filter((volume) => volume !== BLOB_VOLUME) + : SNAPSHOT_VOLUMES; + const present: string[] = []; - for (const volume of SNAPSHOT_VOLUMES) { + for (const volume of candidates) { if (await volumeExists(`${prefix}${volume}`)) { present.push(volume); } @@ -193,6 +234,7 @@ export async function createSnapshot( const id = newSnapshotId(trigger); logger.step(`Creating volume snapshot ${id} (${present.join(', ')})...`); + announceExternalBlobs(blobStore); const volumes: Record = {}; for (const volume of present) { diff --git a/tools/cli/src/lib/backup/inspect-blob-store.ts b/tools/cli/src/lib/backup/inspect-blob-store.ts new file mode 100644 index 0000000000..eddbac9d11 --- /dev/null +++ b/tools/cli/src/lib/backup/inspect-blob-store.ts @@ -0,0 +1,164 @@ +import * as logger from '../../utils/logger'; +import { BUNDLED_OBJECT_STORE_ENDPOINT } from '../compose/generators/constants'; +import { volumeExists } from '../docker/ensure-volumes'; +import { exec } from '../docker/exec'; +import { BACKUP_HELPER_IMAGE, CONFIG_VOLUME } from './constants'; + +/** + * Where the deployment default blob store points — the `default` config + * tree's `object-storage/connection.json`, which the backend seeds against + * the bundled `object-store` at boot and an operator may repoint at their + * own S3. `unknown` covers a store that was never seeded and a config + * volume that cannot be read; callers treat it as "capture the volume". + */ +export type BlobStoreDefault = + | { kind: 'bundled' } + | { kind: 'external'; endpoint: string; bucket: string } + | { kind: 'unknown'; reason: string }; + +export interface BlobStoreLayout { + default: BlobStoreDefault; + /** + * Organizations that bring their own bucket (`/object-storage/ + * connection.json`, resolved by the backend BEFORE the default). Their + * blobs never touch the local volume, so no snapshot can contain them. + */ + ownBucketOrgs: string[]; +} + +const DEFAULT_ORG_SLUG = 'default'; + +/** + * The backend's file contract for the object-storage config domain + * (`backend/core/object_storage/file_utils.ts`), as seen from the config + * volume mounted at `/data`. + */ +const CONNECTION_PATH_RE = + /^\/data\/([^/]+)\/object-storage\/connection\.json$/; + +/** + * One `\t` row per connection file. The JSON is pretty-printed + * on disk, so raw newlines are dropped to keep one row per file — JSON + * strings escape theirs, so nothing is lost. `true` keeps the exit code 0 + * when no org has a connection file (the glob then matches nothing). + */ +const INSPECT_SCRIPT = [ + 'for f in /data/*/object-storage/connection.json; do', + '[ -f "$f" ] || continue;', + `printf '%s\\t' "$f"; tr -d '\\n' < "$f"; echo;`, + 'done; true', +].join(' '); + +interface ConnectionSummary { + /** Absent for AWS S3 proper (the schema leaves the endpoint optional). */ + endpoint?: string; + bucket: string; +} + +function parseConnection(raw: string): ConnectionSummary | null { + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + return null; + } + if (typeof value !== 'object' || value === null) return null; + const { endpoint, bucket } = value as Record; + if (typeof bucket !== 'string' || bucket.length === 0) return null; + if (endpoint !== undefined && typeof endpoint !== 'string') return null; + return endpoint === undefined ? { bucket } : { endpoint, bucket }; +} + +function normalizeEndpoint(endpoint: string): string { + return endpoint.trim().replace(/\/+$/, ''); +} + +/** + * Learn where this deployment's blobs live by reading the object-storage + * connection files from the config volume — the same files, with the same + * bundled-vs-repointed test (endpoint equality), the backend resolves them + * with. Never throws: anything short of a readable default connection is + * reported as `unknown`, and the backup then captures the blob volume + * rather than risk skipping data. + */ +export async function inspectBlobStore( + prefix: string, +): Promise { + const configVolume = `${prefix}${CONFIG_VOLUME}`; + if (!(await volumeExists(configVolume))) { + return { + default: { + kind: 'unknown', + reason: `config volume ${configVolume} does not exist`, + }, + ownBucketOrgs: [], + }; + } + + const result = await exec('docker', [ + 'run', + '--rm', + '-v', + `${configVolume}:/data:ro`, + BACKUP_HELPER_IMAGE, + 'sh', + '-c', + INSPECT_SCRIPT, + ]); + if (!result.success) { + const detail = result.stderr || result.stdout; + logger.warn( + `Could not inspect the object-storage config in ${configVolume} (${detail}) — capturing the blob volume to be safe.`, + ); + return { + default: { kind: 'unknown', reason: `inspection failed: ${detail}` }, + ownBucketOrgs: [], + }; + } + + let defaultStore: BlobStoreDefault = { + kind: 'unknown', + reason: 'no default/object-storage/connection.json (store not seeded yet)', + }; + const ownBucketOrgs: string[] = []; + for (const line of result.stdout.split('\n')) { + const row = line.trim(); + if (!row) continue; + const tab = row.indexOf('\t'); + if (tab === -1) continue; + const match = CONNECTION_PATH_RE.exec(row.slice(0, tab)); + if (!match) continue; + const slug = match[1] ?? ''; + + const connection = parseConnection(row.slice(tab + 1)); + if (!connection) { + logger.warn( + `Skipping unreadable object-storage connection file for "${slug}" — the backend fails closed on it too.`, + ); + if (slug === DEFAULT_ORG_SLUG) { + defaultStore = { + kind: 'unknown', + reason: 'default/object-storage/connection.json is unreadable', + }; + } + continue; + } + + if (slug !== DEFAULT_ORG_SLUG) { + ownBucketOrgs.push(slug); + continue; + } + defaultStore = + connection.endpoint !== undefined && + normalizeEndpoint(connection.endpoint) === BUNDLED_OBJECT_STORE_ENDPOINT + ? { kind: 'bundled' } + : { + kind: 'external', + endpoint: connection.endpoint ?? 'AWS S3', + bucket: connection.bucket, + }; + } + ownBucketOrgs.sort(); + + return { default: defaultStore, ownBucketOrgs }; +} diff --git a/tools/cli/src/lib/compose/generators/constants.ts b/tools/cli/src/lib/compose/generators/constants.ts index 1d771de64e..978c624645 100644 --- a/tools/cli/src/lib/compose/generators/constants.ts +++ b/tools/cli/src/lib/compose/generators/constants.ts @@ -35,6 +35,17 @@ export const REQUIRED_VOLUMES = [ 'llm-gateway-data', ] as const; +/** + * Where the backend tier reaches the BUNDLED blob store (`object-store`, the + * MinIO service on `object-store-data`). The backend seeds the deployment + * default `default/object-storage/connection.json` against this address at + * boot, so a default connection that still points here means the blobs live + * on the local volume — the test the backup uses to decide whether to + * snapshot it, and the same comparison the backend's own bootstrap makes to + * tell the bundled store from an operator-repointed one. + */ +export const BUNDLED_OBJECT_STORE_ENDPOINT = 'http://object-store:9000'; + // Enables containers to reach host services (e.g. Ollama on localhost:11434) // via `host.docker.internal`. `host-gateway` requires Docker 20.10+ (project // already requires 24.0+). Safe on Docker Desktop where host.docker.internal diff --git a/tools/cli/src/lib/compose/services/create-backend-services.ts b/tools/cli/src/lib/compose/services/create-backend-services.ts index af22bfb2f1..6b28e3fb9e 100644 --- a/tools/cli/src/lib/compose/services/create-backend-services.ts +++ b/tools/cli/src/lib/compose/services/create-backend-services.ts @@ -1,5 +1,8 @@ import { getProjectId } from '../../../utils/load-env'; -import { EXTRA_HOSTS } from '../generators/constants'; +import { + BUNDLED_OBJECT_STORE_ENDPOINT, + EXTRA_HOSTS, +} from '../generators/constants'; import type { ComposeService, ServiceConfig } from '../types'; import { DEFAULT_LOGGING, imageRef } from '../types'; @@ -75,7 +78,7 @@ export function createBackendApiService(config: ServiceConfig): ComposeService { // The bundled blob store the backend seeds the deployment default // against at boot. Internal address: presigned URLs are signed here // and forwarded by the proxy, so the store is never published. - OBJECT_STORE_ENDPOINT: 'http://object-store:9000', + OBJECT_STORE_ENDPOINT: BUNDLED_OBJECT_STORE_ENDPOINT, OBJECT_STORE_BUCKET: '${OBJECT_STORE_BUCKET:-tale-blobs}', OBJECT_STORE_ACCESS_KEY: '${OBJECT_STORE_ACCESS_KEY:-tale}', OBJECT_STORE_SECRET_KEY: @@ -122,7 +125,7 @@ export function createBackendWorkerService( // The bundled blob store the backend seeds the deployment default // against at boot. Internal address: presigned URLs are signed here // and forwarded by the proxy, so the store is never published. - OBJECT_STORE_ENDPOINT: 'http://object-store:9000', + OBJECT_STORE_ENDPOINT: BUNDLED_OBJECT_STORE_ENDPOINT, OBJECT_STORE_BUCKET: '${OBJECT_STORE_BUCKET:-tale-blobs}', OBJECT_STORE_ACCESS_KEY: '${OBJECT_STORE_ACCESS_KEY:-tale}', OBJECT_STORE_SECRET_KEY: From 49a67ff1283e83f59e2c7b30c760c40b27799ab0 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Thu, 3 Sep 2026 23:45:29 +0800 Subject: [PATCH 2/4] fix(cli): restore the blob archive and flag snapshots without one Restore already extracts every manifest volume the CLI snapshots, so the blob volume rides along once it is in SNAPSHOT_VOLUMES; the extract uses the same per-volume bound as the snapshot side. A snapshot taken before blobs were captured, or on a deployment whose blobs live in external S3, carries no blob archive: the listing marks it `without blobs`, the restore says so before asking for confirmation and leaves the blob volume untouched while restoring the rest. --- tools/cli/src/lib/actions/restore.test.ts | 98 ++++++++++++++++++++++- tools/cli/src/lib/actions/restore.ts | 21 +++-- 2 files changed, 113 insertions(+), 6 deletions(-) diff --git a/tools/cli/src/lib/actions/restore.test.ts b/tools/cli/src/lib/actions/restore.test.ts index 76a3db792a..f07b10232b 100644 --- a/tools/cli/src/lib/actions/restore.test.ts +++ b/tools/cli/src/lib/actions/restore.test.ts @@ -17,6 +17,7 @@ const ensureVolumesMock = mock(); const execMock = mock(); const confirmMock = mock(); const loggerInfoMock = mock(); +const loggerNoticeMock = mock(); const loggerTableMock = mock(); mock.module('../backup/list-snapshots', () => ({ @@ -52,7 +53,7 @@ mock.module('../../utils/logger', () => ({ header: mock(), blank: mock(), debug: mock(), - notice: mock(), + notice: loggerNoticeMock, table: loggerTableMock, })); @@ -65,6 +66,7 @@ const env: DeploymentEnv = { DEPLOY_DIR: '/tmp/tale-restore-test', }; +/** A snapshot from before blobs were captured: no `object-store-data` archive. */ const MANIFEST = { id: '20260611-120000-deploy', createdAt: '2026-06-11T12:00:00.000Z', @@ -77,6 +79,21 @@ const MANIFEST = { }, }; +const MANIFEST_WITH_BLOBS = { + ...MANIFEST, + id: '20260903-090000-manual', + createdAt: '2026-09-03T09:00:00.000Z', + trigger: 'manual', + volumes: { + ...MANIFEST.volumes, + 'object-store-data': { sha256: 'c'.repeat(64), sizeBytes: 4096 }, + }, +}; + +function restoreScripts(): string[] { + return execMock.mock.calls.map((call) => call[1][call[1].length - 1]); +} + afterEach(() => { listSnapshotsMock.mockReset(); resolveSnapshotPrefixMock.mockReset(); @@ -87,6 +104,7 @@ afterEach(() => { execMock.mockReset(); confirmMock.mockReset(); loggerInfoMock.mockReset(); + loggerNoticeMock.mockReset(); loggerTableMock.mockReset(); }); @@ -188,4 +206,82 @@ describe('restore', () => { expect(verifySnapshotMock).not.toHaveBeenCalled(); expect(execMock).not.toHaveBeenCalled(); }); + + describe('the blob archive', () => { + test('is restored into the blob volume when the snapshot carries one', async () => { + resolveSnapshotPrefixMock.mockResolvedValue('tale_'); + listSnapshotsMock.mockResolvedValue([MANIFEST_WITH_BLOBS, MANIFEST]); + isContainerRunningMock.mockResolvedValue(false); + verifySnapshotMock.mockResolvedValue(undefined); + ensureVolumesMock.mockResolvedValue(true); + execMock.mockResolvedValue({ + success: true, + stdout: '', + stderr: '', + exitCode: 0, + }); + + await restore({ + env, + snapshotId: MANIFEST_WITH_BLOBS.id, + assumeYes: true, + }); + + // The blob volume is re-created alongside the others on a fresh host. + expect(ensureVolumesMock).toHaveBeenCalledWith( + expect.arrayContaining(['object-store-data']), + 'tale_', + ); + // One wipe+extract per volume — the blob archive included. + expect(execMock).toHaveBeenCalledTimes(3); + const blobRestore = execMock.mock.calls.find((call) => + String(call[1][call[1].length - 1]).includes( + 'object-store-data.tar.gz', + ), + ); + expect(blobRestore?.[1]).toContain('tale_object-store-data:/data'); + // Blobs are proportional to the store: the extract gets the wider bound. + expect(blobRestore?.[2]?.timeout).toBeGreaterThan(1800); + expect(loggerNoticeMock).not.toHaveBeenCalled(); + }); + + test('is noted as absent when an older snapshot predates blob capture, and the rest restores', async () => { + resolveSnapshotPrefixMock.mockResolvedValue('tale_'); + listSnapshotsMock.mockResolvedValue([MANIFEST_WITH_BLOBS, MANIFEST]); + isContainerRunningMock.mockResolvedValue(false); + verifySnapshotMock.mockResolvedValue(undefined); + ensureVolumesMock.mockResolvedValue(true); + execMock.mockResolvedValue({ + success: true, + stdout: '', + stderr: '', + exitCode: 0, + }); + + await restore({ env, snapshotId: MANIFEST.id, assumeYes: true }); + + expect(execMock).toHaveBeenCalledTimes(2); + expect( + restoreScripts().some((script) => script.includes('object-store-data')), + ).toBe(false); + const notices = loggerNoticeMock.mock.calls.map((call) => + String(call[0]), + ); + expect(notices).toHaveLength(1); + expect(notices[0]).toContain('no object-store-data archive'); + expect(notices[0]).toContain('left untouched'); + }); + + test('is visible in the listing: snapshots without one are marked', async () => { + resolveSnapshotPrefixMock.mockResolvedValue('tale_'); + listSnapshotsMock.mockResolvedValue([MANIFEST_WITH_BLOBS, MANIFEST]); + + await restore({ env }); + + const rows = loggerTableMock.mock.calls[0][0] as [string, string][]; + const byId = new Map(rows); + expect(byId.get(MANIFEST_WITH_BLOBS.id)).not.toContain('without blobs'); + expect(byId.get(MANIFEST.id)).toContain('without blobs'); + }); + }); }); diff --git a/tools/cli/src/lib/actions/restore.ts b/tools/cli/src/lib/actions/restore.ts index 8f123eb3c2..ad0ee49d47 100644 --- a/tools/cli/src/lib/actions/restore.ts +++ b/tools/cli/src/lib/actions/restore.ts @@ -3,8 +3,10 @@ import { getProjectId, type DeploymentEnv } from '../../utils/load-env'; import * as logger from '../../utils/logger'; import { confirm } from '../../utils/prompt'; import { + archiveTimeoutSeconds, BACKUP_HELPER_IMAGE, BACKUP_VOLUME, + BLOB_VOLUME, SNAPSHOT_VOLUMES, isValidSnapshotId, } from '../backup/constants'; @@ -29,9 +31,6 @@ interface RestoreOptions { assumeYes?: boolean; } -/** Bounds a single volume extraction; mirrors the snapshot-side tar bound. */ -const RESTORE_TIMEOUT_SECONDS = 1800; - /** * Every container name this project can run: stateful (one instance each), * rotatable both uncolored (dev stack) and per blue/green color (prod stack). @@ -63,11 +62,15 @@ function totalSizeBytes(manifest: SnapshotManifest): number { ); } +function hasBlobArchive(manifest: SnapshotManifest): boolean { + return BLOB_VOLUME in manifest.volumes; +} + function printSnapshotList(snapshots: SnapshotManifest[]): void { logger.table( snapshots.map((snapshot) => [ snapshot.id, - `${snapshot.createdAt} · platform ${snapshot.platformVersion ?? 'unknown'} · ${formatBytes(totalSizeBytes(snapshot))} · ${snapshot.trigger}`, + `${snapshot.createdAt} · platform ${snapshot.platformVersion ?? 'unknown'} · ${formatBytes(totalSizeBytes(snapshot))} · ${snapshot.trigger}${hasBlobArchive(snapshot) ? '' : ' · without blobs'}`, ]), ); } @@ -143,6 +146,14 @@ export async function restore(options: RestoreOptions): Promise { if (volumes.length === 0) { throw new Error(`Snapshot ${snapshotId} contains no restorable volumes`); } + // Snapshots from before blobs were captured — and snapshots of a + // deployment whose blobs live in external S3 — carry no blob archive. + // Restore what the snapshot has; say what it does not touch. + if (!volumes.includes(BLOB_VOLUME)) { + logger.notice( + `Snapshot ${snapshotId} has no ${BLOB_VOLUME} archive (taken before blobs were captured, or with an external blob store) — the blob volume is left untouched.`, + ); + } if (!options.assumeYes) { logger.warn( @@ -182,7 +193,7 @@ export async function restore(options: RestoreOptions): Promise { '-c', `find /data -mindepth 1 -delete && tar xzf /backup/${snapshotId}/${volume}.tar.gz -C /data`, ], - { timeout: RESTORE_TIMEOUT_SECONDS }, + { timeout: archiveTimeoutSeconds(volume) }, ); if (!result.success) { throw new Error( From 264866d3471475be741f9f756a2f33338afaa709 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Thu, 3 Sep 2026 23:45:29 +0800 Subject: [PATCH 3/4] test(cli): cover listing and verifying snapshots with the blob archive Listing parses manifests with and without an `object-store-data` entry and orders them newest first; verification checks every `*.tar.gz.sha256` sidecar in the snapshot directory rather than a fixed list, so the blob archive is covered without a code change. --- .../cli/src/lib/backup/list-snapshots.test.ts | 119 ++++++++++++++++++ .../src/lib/backup/verify-snapshot.test.ts | 57 +++++++++ 2 files changed, 176 insertions(+) create mode 100644 tools/cli/src/lib/backup/list-snapshots.test.ts create mode 100644 tools/cli/src/lib/backup/verify-snapshot.test.ts diff --git a/tools/cli/src/lib/backup/list-snapshots.test.ts b/tools/cli/src/lib/backup/list-snapshots.test.ts new file mode 100644 index 0000000000..eb464f7967 --- /dev/null +++ b/tools/cli/src/lib/backup/list-snapshots.test.ts @@ -0,0 +1,119 @@ +import { afterEach, describe, expect, mock, test } from 'bun:test'; + +import { listSnapshots } from './list-snapshots'; + +const execMock = mock(); +const volumeExistsMock = mock(); +const loggerWarnMock = mock(); + +mock.module('../docker/exec', () => ({ exec: execMock })); +mock.module('../docker/ensure-volumes', () => ({ + ensureVolumes: mock(), + volumeExists: volumeExistsMock, +})); +mock.module('../../utils/logger', () => ({ + info: mock(), + error: mock(), + warn: loggerWarnMock, + step: mock(), + success: mock(), + header: mock(), + blank: mock(), + debug: mock(), + notice: mock(), + table: mock(), +})); + +function ok(stdout = '') { + return { success: true, stdout, stderr: '', exitCode: 0 }; +} + +const OLDER = { + id: '20260611-120000-deploy', + createdAt: '2026-06-11T12:00:00.000Z', + cliVersion: '1.0.0', + platformVersion: '0.9.6', + trigger: 'deploy', + volumes: { + 'db-data': { sha256: 'a'.repeat(64), sizeBytes: 1024 }, + 'convex-data': { sha256: 'b'.repeat(64), sizeBytes: 2048 }, + }, +}; + +const WITH_BLOBS = { + id: '20260903-090000-manual', + createdAt: '2026-09-03T09:00:00.000Z', + cliVersion: '1.0.0', + platformVersion: '0.5.7', + trigger: 'manual', + volumes: { + ...OLDER.volumes, + 'object-store-data': { sha256: 'c'.repeat(64), sizeBytes: 4096 }, + 'caddy-data': { sha256: 'd'.repeat(64), sizeBytes: 8 }, + 'caddy-config': { sha256: 'e'.repeat(64), sizeBytes: 8 }, + }, +}; + +afterEach(() => { + execMock.mockReset(); + volumeExistsMock.mockReset(); + loggerWarnMock.mockReset(); +}); + +describe('listSnapshots', () => { + test('returns [] before the backups volume exists', async () => { + volumeExistsMock.mockResolvedValue(false); + + expect(await listSnapshots('p_')).toEqual([]); + expect(execMock).not.toHaveBeenCalled(); + }); + + test('reads manifests with and without the blob archive, newest first', async () => { + volumeExistsMock.mockResolvedValue(true); + execMock.mockResolvedValue( + ok(`${JSON.stringify(OLDER)}\n${JSON.stringify(WITH_BLOBS)}\n`), + ); + + const snapshots = await listSnapshots('p_'); + + expect(snapshots.map((snapshot) => snapshot.id)).toEqual([ + WITH_BLOBS.id, + OLDER.id, + ]); + expect(snapshots[0]?.volumes['object-store-data']).toEqual({ + sha256: 'c'.repeat(64), + sizeBytes: 4096, + }); + expect(snapshots[1]?.volumes['object-store-data']).toBeUndefined(); + // Read-only mount of the backups volume. + expect(execMock.mock.calls[0][1]).toContain('p_backups:/backup:ro'); + }); + + test('skips a manifest whose volume entry is malformed and keeps the rest', async () => { + volumeExistsMock.mockResolvedValue(true); + const torn = { + ...WITH_BLOBS, + volumes: { 'object-store-data': { sha256: 'c'.repeat(64) } }, + }; + execMock.mockResolvedValue( + ok(`${JSON.stringify(torn)}\n${JSON.stringify(OLDER)}\nnot json\n`), + ); + + const snapshots = await listSnapshots('p_'); + + expect(snapshots.map((snapshot) => snapshot.id)).toEqual([OLDER.id]); + expect(loggerWarnMock).toHaveBeenCalledTimes(2); + }); + + test('throws when the backups volume cannot be read', async () => { + volumeExistsMock.mockResolvedValue(true); + execMock.mockResolvedValue({ + success: false, + stdout: '', + stderr: 'permission denied', + exitCode: 1, + }); + + await expect(listSnapshots('p_')).rejects.toThrow('Failed to list'); + }); +}); diff --git a/tools/cli/src/lib/backup/verify-snapshot.test.ts b/tools/cli/src/lib/backup/verify-snapshot.test.ts new file mode 100644 index 0000000000..5dd43dfb4d --- /dev/null +++ b/tools/cli/src/lib/backup/verify-snapshot.test.ts @@ -0,0 +1,57 @@ +import { afterEach, describe, expect, mock, test } from 'bun:test'; + +import { verifySnapshot } from './verify-snapshot'; + +const execMock = mock(); + +mock.module('../docker/exec', () => ({ exec: execMock })); + +afterEach(() => { + execMock.mockReset(); +}); + +describe('verifySnapshot', () => { + test('checks every archive sidecar in the snapshot directory, read-only', async () => { + execMock.mockResolvedValue({ + success: true, + // One line per sidecar — the blob archive is verified like the rest. + stdout: [ + 'db-data.tar.gz: OK', + 'convex-data.tar.gz: OK', + 'object-store-data.tar.gz: OK', + ].join('\n'), + stderr: '', + exitCode: 0, + }); + + await verifySnapshot('p_', '20260903-090000-manual'); + + expect(execMock).toHaveBeenCalledTimes(1); + const args = execMock.mock.calls[0][1] as string[]; + expect(args).toContain('p_backups:/backup:ro'); + const script = args[args.length - 1]; + expect(script).toContain('cd /backup/20260903-090000-manual'); + // The glob covers every volume archive, not a fixed list. + expect(script).toContain('sha256sum -c *.tar.gz.sha256'); + }); + + test('throws on the first mismatch', async () => { + execMock.mockResolvedValue({ + success: false, + stdout: 'object-store-data.tar.gz: FAILED', + stderr: '', + exitCode: 1, + }); + + await expect( + verifySnapshot('p_', '20260903-090000-manual'), + ).rejects.toThrow('failed integrity verification: object-store-data'); + }); + + test('rejects an id with shell metacharacters before running anything', async () => { + await expect(verifySnapshot('p_', '$(rm -rf /backup)')).rejects.toThrow( + 'Invalid snapshot id', + ); + expect(execMock).not.toHaveBeenCalled(); + }); +}); From d0b923009a2b5001819a75202d4f8b42fb337458 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Thu, 3 Sep 2026 23:45:29 +0800 Subject: [PATCH 4/4] docs(cli): say which blobs a backup captures and which are yours The backups page, the self-hosted overview and the Linux install hook described the blob store as deliberately outside the snapshot. State the shipped behaviour: `object-store-data` is captured with the other data volumes when the deployment default is the bundled store; a default repointed at external S3 and organizations with their own bucket are announced by the backup and are the operator's to back up; older snapshots restore without touching the blob volume. --- docs/de/self-hosted/install/linux-server.md | 2 +- .../operate/backups-and-restore.md | 19 ++++++++++++------- docs/de/self-hosted/overview.md | 2 +- docs/en/self-hosted/install/linux-server.md | 2 +- .../operate/backups-and-restore.md | 19 ++++++++++++------- docs/en/self-hosted/overview.md | 2 +- docs/fr/self-hosted/install/linux-server.md | 2 +- .../operate/backups-and-restore.md | 19 ++++++++++++------- docs/fr/self-hosted/overview.md | 2 +- 9 files changed, 42 insertions(+), 27 deletions(-) diff --git a/docs/de/self-hosted/install/linux-server.md b/docs/de/self-hosted/install/linux-server.md index 52c2cf9b93..952e1ed47e 100644 --- a/docs/de/self-hosted/install/linux-server.md +++ b/docs/de/self-hosted/install/linux-server.md @@ -92,7 +92,7 @@ Jeder Service sollte `running` oder `healthy` zeigen. Folg dem **Schritt 4 — D Bevor du User auf die URL zeigst, machen dir drei Haken später das Leben leichter: -- **Backups.** Richt dein bestehendes Snapshot-Tooling auf `db-data` und das Object-Store-Volume — siehe [Backups und Restore](/de/self-hosted/operate/backups-and-restore). +- **Backups.** `tale backup` snapshottet die Daten-Volumes — Blobs eingeschlossen — ins `backups`-Volume; richte dein Off-Host-Tooling auf dieses Volume plus Projekt-Workspace und `.env` — siehe [Backups und Restore](/de/self-hosted/operate/backups-and-restore). - **Logs.** Tale loggt auf stdout. Hat der Host journald, trägt `journalctl -u docker` alles; sonst pipe zu deinem Aggregator. - **Metriken.** Setze `METRICS_BEARER_TOKEN` in `.env` und scrap `/metrics` aus deinem Prometheus — siehe [Observability-Konfiguration](/de/self-hosted/configuration/observability-config). diff --git a/docs/de/self-hosted/operate/backups-and-restore.md b/docs/de/self-hosted/operate/backups-and-restore.md index 5aa5f19bec..8341ef4b96 100644 --- a/docs/de/self-hosted/operate/backups-and-restore.md +++ b/docs/de/self-hosted/operate/backups-and-restore.md @@ -9,17 +9,20 @@ Der Architektur-Kontext lebt in [Container-Architektur](/de/self-hosted/operate/ ## Was ein Snapshot enthält -| Volume | Enthält | -| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -| `db-data` | Postgres — der Anwendungsspeicher (Agents, Runs, das Audit-Log) und der Wissens-Korpus (Dokument-Chunks, Embeddings, gecrawlte Seiten) | -| `convex-data` | Org-Config, Anbieter-Secrets, hochgeladenes Branding | -| `caddy-data`, `caddy-config` | TLS-Zertifikate und Proxy-State | +| Volume | Enthält | +| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `db-data` | Postgres — der Anwendungsspeicher (Agents, Runs, das Audit-Log) und der Wissens-Korpus (Dokument-Chunks, Embeddings, gecrawlte Seiten) | +| `convex-data` | Org-Config, Anbieter-Secrets, hochgeladenes Branding | +| `object-store-data` | Der Blob-Store — hochgeladene Dateien, Chat-Anhänge, Audio, generierte Medien —, solange der Deployment-Default der mitgelieferte Objektspeicher ist | +| `caddy-data`, `caddy-config` | TLS-Zertifikate und Proxy-State | -Jeder Snapshot ist ein Verzeichnis mit einem Namen wie `20260611-142530-deploy` im `backups`-Volume des Projekts: ein `.tar.gz` pro Volume, je ein `.sha256`-Sidecar und ein zuletzt geschriebenes `manifest.json`. Ein Verzeichnis ohne Manifest ist ein unvollständiger Snapshot — er taucht nie in Listings auf und lässt sich nie wiederherstellen. Der Snapshot lässt `object-store-data` bewusst aus — den Blob-Store mit hochgeladenen Dateien und generierten Medien —, sodass diese Blobs ihre eigene Off-Host-Erfassung brauchen, neben den zwei Dingen, die ganz außerhalb der Volumes leben: dem Projekt-Workspace (das Verzeichnis mit `tale.json`) und `.env`. +Jeder Snapshot ist ein Verzeichnis mit einem Namen wie `20260611-142530-deploy` im `backups`-Volume des Projekts: ein `.tar.gz` pro Volume, je ein `.sha256`-Sidecar und ein zuletzt geschriebenes `manifest.json`. Ein Verzeichnis ohne Manifest ist ein unvollständiger Snapshot — er taucht nie in Listings auf und lässt sich nie wiederherstellen. Zwei Dinge leben ganz außerhalb der Volumes und brauchen ihren eigenen Platz in deinem Off-Host-Job: der Projekt-Workspace (das Verzeichnis mit `tale.json`) und `.env`. + +Blobs folgen dem Objektspeicher. Mit dem mitgelieferten `object-store` — dem Default — erfasst der Snapshot `object-store-data` wie jedes andere Volume, und sein Archiv ist so groß wie alles, was je hochgeladen wurde: Während des Tars pausiert der Store, Uploads und Downloads stehen also so lange still. In zwei Fällen liegen Blobs außerhalb des Snapshots, und beide sagt das Backup an, statt sie zu verschweigen. Zeigt der Deployment-Default auf ein externes S3 (`default/object-storage/connection.json` nennt nicht mehr den mitgelieferten Store), liegt im lokalen Volume nichts, was die App liest: Das Backup überspringt das Volume, und `tale backup` druckt eine einzeilige Notiz mit Endpoint und Bucket — dieses Bucket sicherst du mit deinem eigenen S3-Tooling. Eine Organisation, die unter **Einstellungen > Datenresidenz** ihren eigenen Bucket mitbringt, schreibt ebenfalls nie ins lokale Volume; die Notiz nennt die Organisation, und kein Snapshot kann diese Blobs enthalten. ## Wann Snapshots genommen werden -`tale deploy` snapshotet vor seinem ersten mutierenden Schritt, wann immer der Deploy Daten ändern kann: Die Zielversion weicht von der laufenden ab oder ein Host-Config-Push (`--override` / `--override-all`) ist angefordert. Während jedes Volume getart wird, sind die Container, die es nutzen, für ein paar Sekunden pausiert, damit das Archiv crash-konsistent ist — eine Live-Kopie eines laufenden Postgres-Verzeichnisses ist nicht wiederherstellbar. +`tale deploy` snapshotet vor seinem ersten mutierenden Schritt, wann immer der Deploy Daten ändern kann: Die Zielversion weicht von der laufenden ab oder ein Host-Config-Push (`--override` / `--override-all`) ist angefordert. Während jedes Volume getart wird, pausieren die Container, die es nutzen, für die Dauer des Tars — Sekunden bei Datenbank- und Config-Volumes, beim Blob-Volume so lange, wie der Store groß ist —, damit das Archiv crash-konsistent ist: Eine Live-Kopie eines laufenden Postgres-Verzeichnisses ist nicht wiederherstellbar. Ein gescheiterter Snapshot bricht den Deploy ab. `--skip-backup` übersteuert das auf `tale deploy` — dann sind deine eigenen externen Backups der einzige Recovery-Pfad, und genau deshalb loggt das Flag eine laute Warnung. @@ -62,6 +65,8 @@ tale deploy --stop Das Redeploy der passenden Version ist Teil des Restores, kein optionales Extra: Der Snapshot hat die Daten exakt so erfasst, wie diese Plattform-Version sie hinterlassen hat, und ein neueres Binary würde sofort wieder seine Migrationen darauf laufen lassen. Die Restore-Ausgabe druckt die exakte Version aus dem Manifest des Snapshots. +Ein Snapshot aus der Zeit, bevor Blobs erfasst wurden, oder von einem Deployment, dessen Blobs in einem externen S3 liegen, hat kein `object-store-data`-Archiv. `tale restore` listet solche Snapshots als `without blobs`, sagt es vor der Bestätigung noch einmal und lässt das Blob-Volume unangetastet, während es alles andere wiederherstellt — die Blobs bleiben genau so, wie sie auf dem Host sind. + ## Restore-Drill Lauf den Drill vierteljährlich auf einem Nicht-Produktions-Host. Der Drill ist nicht „existiert ein Snapshot" — er ist „kann ein frischer Host aus der Off-Host-Kopie des `backups`-Volumes, dem Projekt-Workspace und `.env` in unter einer Stunde wiederaufgebaut werden". Die Fehler-Modi, die der Drill fängt: ein Off-Host-Job, der den Workspace nie erfasst hat, und eine veraltete `.env`, die nicht mehr zu den Anforderungen des aktuellen Binarys passt. diff --git a/docs/de/self-hosted/overview.md b/docs/de/self-hosted/overview.md index 3542666439..0106322a52 100644 --- a/docs/de/self-hosted/overview.md +++ b/docs/de/self-hosted/overview.md @@ -37,7 +37,7 @@ Diese Volumes überleben ein `docker compose down`: - `caddy-data`, `caddy-config` — TLS-Zertifikate und Proxy-State. - `backups` — prüfsummengesicherte Volume-Snapshots, geschrieben von `tale backup` und automatisch vor migrierenden Deploys; [Backups und Restore](/de/self-hosted/operate/backups-and-restore) ist die Übung. -Alles andere ist ephemer. Container lassen sich ohne Datenverlust ersetzen, solange die Volumes überleben. Ein Vorbehalt, den du verinnerlichen solltest: `tale backup` snapshottet `db-data`, `convex-data` und die Caddy-Volumes, aber **nicht** `object-store-data` — die Blobs brauchen ihre eigene Off-Host-Sicherung, behandelt in [Backups und Restore](/de/self-hosted/operate/backups-and-restore). +Alles andere ist ephemer. Container lassen sich ohne Datenverlust ersetzen, solange die Volumes überleben. `tale backup` snapshottet die Daten-Volumes oben — `object-store-data` eingeschlossen, solange die Blobs im mitgelieferten Objektspeicher liegen. Blobs in einem externen S3-Bucket, ob umgebogener Deployment-Default oder eigener Bucket einer Organisation, sicherst du selbst, und das Backup sagt dir das; [Backups und Restore](/de/self-hosted/operate/backups-and-restore) hat die Liste und die Übung. ## Provider-Secrets und die SOPS-Schicht diff --git a/docs/en/self-hosted/install/linux-server.md b/docs/en/self-hosted/install/linux-server.md index d26b5713be..5ca266812c 100644 --- a/docs/en/self-hosted/install/linux-server.md +++ b/docs/en/self-hosted/install/linux-server.md @@ -92,7 +92,7 @@ Every service should be `running` or `healthy`. Walk through **Step 4 — Create Before pointing users at the URL, three hooks make life easier later: -- **Backups.** Point your existing snapshot tooling at `db-data` and the object store volume — see [Backups and restore](/self-hosted/operate/backups-and-restore). +- **Backups.** `tale backup` snapshots the data volumes — blobs included — into the `backups` volume; point your off-host tooling at that volume plus the project workspace and `.env` — see [Backups and restore](/self-hosted/operate/backups-and-restore). - **Logs.** Tale logs to stdout. If the host has journald, `journalctl -u docker` carries everything; otherwise pipe to your aggregator. - **Metrics.** Set `METRICS_BEARER_TOKEN` in `.env` and scrape `/metrics` from your Prometheus — see [Observability config](/self-hosted/configuration/observability-config). diff --git a/docs/en/self-hosted/operate/backups-and-restore.md b/docs/en/self-hosted/operate/backups-and-restore.md index b311eba385..25f7b630e8 100644 --- a/docs/en/self-hosted/operate/backups-and-restore.md +++ b/docs/en/self-hosted/operate/backups-and-restore.md @@ -9,17 +9,20 @@ The architecture context lives in [Container architecture](/self-hosted/operate/ ## What a snapshot contains -| Volume | Holds | -| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -| `db-data` | Postgres — the application store (agents, runs, the audit log) and the knowledge corpus (document chunks, embeddings, crawled pages) | -| `convex-data` | Org config, provider secrets, uploaded branding | -| `caddy-data`, `caddy-config` | TLS certificates and proxy state | +| Volume | Holds | +| ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| `db-data` | Postgres — the application store (agents, runs, the audit log) and the knowledge corpus (document chunks, embeddings, crawled pages) | +| `convex-data` | Org config, provider secrets, uploaded branding | +| `object-store-data` | The blob store — uploaded files, chat attachments, audio, generated media — whenever the deployment default is the bundled object store | +| `caddy-data`, `caddy-config` | TLS certificates and proxy state | -Each snapshot is a directory named like `20260611-142530-deploy` inside the project's `backups` volume: one `.tar.gz` per volume, a `.sha256` sidecar each, and a `manifest.json` written last. A directory without a manifest is an incomplete snapshot — it never shows up in listings and can never be restored. The snapshot deliberately skips `object-store-data` — the blob store holding uploaded files and generated media — so those blobs need their own off-host capture, alongside the two things that live outside the volumes entirely: the project workspace (the directory holding `tale.json`) and `.env`. +Each snapshot is a directory named like `20260611-142530-deploy` inside the project's `backups` volume: one `.tar.gz` per volume, a `.sha256` sidecar each, and a `manifest.json` written last. A directory without a manifest is an incomplete snapshot — it never shows up in listings and can never be restored. Two things live outside the volumes entirely and need their own place in your off-host job: the project workspace (the directory holding `tale.json`) and `.env`. + +Blobs follow the object store. With the bundled `object-store` — the default — `object-store-data` is captured like every other volume, and its archive is as large as everything ever uploaded: the store is paused while it is tarred, so uploads and downloads stall for that long. Two cases put blobs outside the snapshot, and both are announced rather than silent. A deployment default repointed at an external S3 (`default/object-storage/connection.json` no longer naming the bundled store) leaves the local volume with nothing the app reads, so the volume is skipped and `tale backup` prints a one-line notice with the endpoint and bucket — that bucket's backup runs under your own S3 tooling. An organization that brings its own bucket under **Settings > Data residency** never writes to the local volume either; the notice names the organization, and no snapshot can contain those blobs. ## When snapshots are taken -`tale deploy` snapshots before its first mutating step whenever the deploy can change data: the target version differs from the running one, or a host-config push (`--override` / `--override-all`) is requested. While each volume is tarred, the containers using it are paused for a few seconds so the archive is crash-consistent — a live copy of a running Postgres directory is not restorable. +`tale deploy` snapshots before its first mutating step whenever the deploy can change data: the target version differs from the running one, or a host-config push (`--override` / `--override-all`) is requested. While each volume is tarred, the containers using it are paused for the duration — seconds for the database and config volumes, as long as the store is large for the blob volume — so the archive is crash-consistent: a live copy of a running Postgres directory is not restorable. A failed snapshot aborts the deploy. `--skip-backup` overrides that on `tale deploy`, which leaves your own external backups as the only recovery path — the flag logs a loud warning for exactly that reason. @@ -62,6 +65,8 @@ tale deploy --stop The redeploy of the matching version is part of the restore, not an optional extra: the snapshot captured the data exactly as that platform version left it, and a newer binary would immediately re-run its migrations against it. The restore output prints the exact version recorded in the snapshot's manifest. +A snapshot taken before blobs were captured, or on a deployment whose blobs live in external S3, has no `object-store-data` archive. `tale restore` lists such snapshots as `without blobs`, says so again before asking for confirmation, and leaves the blob volume untouched while it restores everything else — the blobs stay exactly as they are on the host. + ## Restore drill Run the drill quarterly on a non-production host. The drill is not "does a snapshot exist" — it is "can a fresh host be rebuilt from the off-host copy of the `backups` volume, the project workspace, and `.env` in under an hour." The failure modes the drill catches: an off-host job that never captured the workspace, and a stale `.env` that no longer matches the current binary's requirements. diff --git a/docs/en/self-hosted/overview.md b/docs/en/self-hosted/overview.md index fb1f4c24b6..1f36ce4d0a 100644 --- a/docs/en/self-hosted/overview.md +++ b/docs/en/self-hosted/overview.md @@ -37,7 +37,7 @@ These volumes survive a `docker compose down`: - `caddy-data`, `caddy-config` — TLS certificates and proxy state. - `backups` — checksummed volume snapshots written by `tale backup` and automatically before migrating deploys; [Backups and restore](/self-hosted/operate/backups-and-restore) is the drill. -Everything else is ephemeral. Containers can be replaced without data loss as long as the volumes survive. One caveat worth internalising: `tale backup` snapshots `db-data`, `convex-data`, and the Caddy volumes, but **not** `object-store-data` — the blobs need their own off-host capture, covered in [Backups and restore](/self-hosted/operate/backups-and-restore). +Everything else is ephemeral. Containers can be replaced without data loss as long as the volumes survive. `tale backup` snapshots the data volumes above — `object-store-data` included, as long as the blobs live in the bundled object store. Blobs in an external S3 bucket, whether a repointed deployment default or an organization's own bucket, are yours to back up, and the backup says so; [Backups and restore](/self-hosted/operate/backups-and-restore) has the list and the drill. ## Provider secrets and the SOPS layer diff --git a/docs/fr/self-hosted/install/linux-server.md b/docs/fr/self-hosted/install/linux-server.md index 2d18b12043..f489e1045e 100644 --- a/docs/fr/self-hosted/install/linux-server.md +++ b/docs/fr/self-hosted/install/linux-server.md @@ -92,7 +92,7 @@ Chaque service devrait être `running` ou `healthy`. Parcours **Étape 4 — Cr Avant de pointer des utilisateurs sur l'URL, trois crochets te facilitent la vie plus tard : -- **Sauvegardes.** Pointe ton outillage de snapshot existant vers `db-data` et le volume du stockage objet — voir [Sauvegardes et restauration](/fr/self-hosted/operate/backups-and-restore). +- **Sauvegardes.** `tale backup` snapshotte les volumes de données — blobs compris — dans le volume `backups` ; pointe ton outillage hors-hôte vers ce volume plus le workspace du projet et `.env` — voir [Sauvegardes et restauration](/fr/self-hosted/operate/backups-and-restore). - **Logs.** Tale logue sur stdout. Si l'hôte a journald, `journalctl -u docker` transporte tout ; sinon, pipe vers ton agrégateur. - **Métriques.** Règle `METRICS_BEARER_TOKEN` dans `.env` et scrape `/metrics` depuis ton Prometheus — voir [Configuration de l'observabilité](/fr/self-hosted/configuration/observability-config). diff --git a/docs/fr/self-hosted/operate/backups-and-restore.md b/docs/fr/self-hosted/operate/backups-and-restore.md index bbcc0c0764..baa2e17f1d 100644 --- a/docs/fr/self-hosted/operate/backups-and-restore.md +++ b/docs/fr/self-hosted/operate/backups-and-restore.md @@ -9,17 +9,20 @@ Le contexte d'architecture vit dans [Architecture des conteneurs](/fr/self-hoste ## Ce qu'un snapshot contient -| Volume | Contient | -| ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | -| `db-data` | Postgres — le magasin applicatif (agents, runs, l'audit log) et le corpus de connaissances (fragments de documents, embeddings, pages crawlées) | -| `convex-data` | Config d'org, secrets de fournisseurs, branding téléversé | -| `caddy-data`, `caddy-config` | Certificats TLS et état du proxy | +| Volume | Contient | +| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `db-data` | Postgres — le magasin applicatif (agents, runs, l'audit log) et le corpus de connaissances (fragments de documents, embeddings, pages crawlées) | +| `convex-data` | Config d'org, secrets de fournisseurs, branding téléversé | +| `object-store-data` | Le store de blobs — fichiers téléversés, pièces jointes de chat, audio, médias générés — dès que le défaut du déploiement est le magasin d'objets fourni | +| `caddy-data`, `caddy-config` | Certificats TLS et état du proxy | -Chaque snapshot est un répertoire nommé comme `20260611-142530-deploy` dans le volume `backups` du projet : un `.tar.gz` par volume, un sidecar `.sha256` chacun et un `manifest.json` écrit en dernier. Un répertoire sans manifest est un snapshot incomplet — il n'apparaît jamais dans les listings et ne peut jamais être restauré. Le snapshot saute délibérément `object-store-data` — le store de blobs qui détient les fichiers téléversés et les médias générés — donc ces blobs demandent leur propre capture off-host, aux côtés des deux choses qui vivent entièrement hors des volumes : le workspace du projet (le répertoire qui contient `tale.json`) et `.env`. +Chaque snapshot est un répertoire nommé comme `20260611-142530-deploy` dans le volume `backups` du projet : un `.tar.gz` par volume, un sidecar `.sha256` chacun et un `manifest.json` écrit en dernier. Un répertoire sans manifest est un snapshot incomplet — il n'apparaît jamais dans les listings et ne peut jamais être restauré. Deux choses vivent entièrement hors des volumes et demandent leur propre place dans ton job hors-hôte : le workspace du projet (le répertoire qui contient `tale.json`) et `.env`. + +Les blobs suivent le magasin d'objets. Avec le service `object-store` fourni — le défaut — le snapshot capture `object-store-data` comme n'importe quel autre volume, et son archive pèse autant que tout ce qui a jamais été téléversé : le store est en pause pendant le tar, donc les téléversements et les téléchargements restent bloqués aussi longtemps. Deux cas placent des blobs hors du snapshot, et le backup les annonce tous les deux au lieu de les passer sous silence. Un défaut du déploiement repointé vers un S3 externe (`default/object-storage/connection.json` ne nomme plus le store fourni) ne laisse dans le volume local rien que l'app lise : le backup saute le volume et `tale backup` imprime une notice d'une ligne avec l'endpoint et le bucket — la sauvegarde de ce bucket relève de ton propre outillage S3. Une organisation qui apporte son propre bucket sous **Paramètres > Résidence des données** n'écrit jamais non plus dans le volume local ; la notice nomme l'organisation, et aucun snapshot ne peut contenir ces blobs. ## Quand les snapshots sont pris -`tale deploy` snapshotte avant sa première étape mutante dès que le déploiement peut changer des données : la version cible diffère de celle qui tourne, ou un push de config hôte (`--override` / `--override-all`) est demandé. Pendant que chaque volume est mis en tar, les conteneurs qui l'utilisent sont mis en pause quelques secondes pour que l'archive soit cohérente après crash — une copie à chaud d'un répertoire Postgres en marche n'est pas restaurable. +`tale deploy` snapshotte avant sa première étape mutante dès que le déploiement peut changer des données : la version cible diffère de celle qui tourne, ou un push de config hôte (`--override` / `--override-all`) est demandé. Pendant que chaque volume est mis en tar, les conteneurs qui l'utilisent sont mis en pause pour toute la durée — quelques secondes pour les volumes de base et de config, aussi longtemps que le store est gros pour le volume de blobs — pour que l'archive soit cohérente après crash : une copie à chaud d'un répertoire Postgres en marche n'est pas restaurable. Un snapshot échoué interrompt le déploiement. `--skip-backup` outrepasse cela sur `tale deploy` — tes propres backups externes deviennent alors le seul chemin de récupération, et c'est exactement pour ça que le flag logge un avertissement bien visible. @@ -62,6 +65,8 @@ tale deploy --stop Le redéploiement de la version correspondante fait partie de la restauration, ce n'est pas un extra optionnel : le snapshot a capturé les données exactement comme cette version de la plateforme les a laissées, et un binaire plus récent relancerait immédiatement ses migrations dessus. La sortie de la restauration imprime la version exacte enregistrée dans le manifest du snapshot. +Un snapshot pris avant que les blobs soient capturés, ou sur un déploiement dont les blobs vivent dans un S3 externe, n'a pas d'archive `object-store-data`. `tale restore` liste ces snapshots comme `without blobs`, le redit avant de demander confirmation et laisse le volume de blobs intact pendant qu'il restaure tout le reste — les blobs restent exactement tels qu'ils sont sur l'hôte. + ## Drill de restauration Fais tourner le drill trimestriellement sur un hôte non-production. Le drill n'est pas « un snapshot existe-t-il » — c'est « un hôte frais peut-il être reconstruit depuis la copie hors-hôte du volume `backups`, le workspace du projet et `.env` en moins d'une heure ». Les modes d'échec que le drill attrape : un job hors-hôte qui n'a jamais capturé le workspace, et un `.env` périmé qui ne correspond plus aux exigences du binaire courant. diff --git a/docs/fr/self-hosted/overview.md b/docs/fr/self-hosted/overview.md index a450225f90..b7867e898f 100644 --- a/docs/fr/self-hosted/overview.md +++ b/docs/fr/self-hosted/overview.md @@ -37,7 +37,7 @@ Ces volumes survivent à un `docker compose down` : - `caddy-data`, `caddy-config` — certificats TLS et état du proxy. - `backups` — snapshots de volumes vérifiés par somme de contrôle, écrits par `tale backup` et automatiquement avant les déploiements qui migrent ; [Sauvegardes et restauration](/fr/self-hosted/operate/backups-and-restore) est l'exercice. -Tout le reste est éphémère. Les conteneurs se remplacent sans perte de données tant que les volumes survivent. Une réserve à intérioriser : `tale backup` snapshotte `db-data`, `convex-data` et les volumes Caddy, mais **pas** `object-store-data` — les blobs ont besoin de leur propre capture off-host, traitée dans [Sauvegardes et restauration](/fr/self-hosted/operate/backups-and-restore). +Tout le reste est éphémère. Les conteneurs se remplacent sans perte de données tant que les volumes survivent. `tale backup` snapshotte les volumes de données ci-dessus — `object-store-data` compris, tant que les blobs vivent dans le magasin d'objets fourni. Les blobs d'un bucket S3 externe, qu'il s'agisse d'un défaut du déploiement repointé ou du propre bucket d'une organisation, sont à toi de sauvegarder, et le backup te le dit ; [Sauvegardes et restauration](/fr/self-hosted/operate/backups-and-restore) a la liste et l'exercice. ## Secrets de fournisseur et couche SOPS